diff --git a/easyflow-api/easyflow-api-admin/src/main/java/tech/easyflow/admin/controller/datacenter/DatacenterDatasetController.java b/easyflow-api/easyflow-api-admin/src/main/java/tech/easyflow/admin/controller/datacenter/DatacenterDatasetController.java index a6ec3e48..67478890 100644 --- a/easyflow-api/easyflow-api-admin/src/main/java/tech/easyflow/admin/controller/datacenter/DatacenterDatasetController.java +++ b/easyflow-api/easyflow-api-admin/src/main/java/tech/easyflow/admin/controller/datacenter/DatacenterDatasetController.java @@ -24,6 +24,11 @@ import java.util.List; @RequestMapping("/api/v1/datacenterDataset") public class DatacenterDatasetController { + /** 对外 Schema 接口的默认字段页码。 */ + private static final long DEFAULT_FIELD_PAGE_NUMBER = 1L; + /** 对外 Schema 接口的默认字段页大小。 */ + private static final long DEFAULT_FIELD_PAGE_SIZE = 200L; + @Resource private DatacenterDatasetQueryService queryService; @Resource @@ -32,13 +37,18 @@ public class DatacenterDatasetController { @PostMapping("/queryPage") @SaCheckPermission("/api/v1/datacenterSource/query") public Result> queryPage(@RequestBody DatacenterQueryRequest request) { - return Result.ok(queryService.queryPage(request)); + return Result.ok(queryService.queryPage( + request, SaTokenUtil.getLoginAccount())); } @GetMapping("/schema") @SaCheckPermission("/api/v1/datacenterSource/query") - public Result schema(DatasetRef datasetRef) { - return Result.ok(queryService.getSchema(datasetRef)); + public Result schema( + DatasetRef datasetRef, + @RequestParam(defaultValue = "1") Long fieldPageNumber, + @RequestParam(defaultValue = "200") Long fieldPageSize) { + return Result.ok(queryService.getSchema( + datasetRef, fieldPageNumber, fieldPageSize)); } @GetMapping("/managedTables") @@ -63,6 +73,13 @@ public class DatacenterDatasetController { request == null ? List.of() : request.getFields(), account ); - return Result.ok(queryService.getSchema(registryService.resolveDatasetRef(table.getId()))); + return Result.ok(queryService.getSchema( + registryService.resolveDatasetRef(table.getId()), + request == null || request.getFieldPageNumber() == null + ? DEFAULT_FIELD_PAGE_NUMBER + : request.getFieldPageNumber(), + request == null || request.getFieldPageSize() == null + ? DEFAULT_FIELD_PAGE_SIZE + : request.getFieldPageSize())); } } diff --git a/easyflow-api/easyflow-api-admin/src/main/java/tech/easyflow/admin/controller/datacenter/DatacenterQueryController.java b/easyflow-api/easyflow-api-admin/src/main/java/tech/easyflow/admin/controller/datacenter/DatacenterQueryController.java new file mode 100644 index 00000000..4ba972d4 --- /dev/null +++ b/easyflow-api/easyflow-api-admin/src/main/java/tech/easyflow/admin/controller/datacenter/DatacenterQueryController.java @@ -0,0 +1,84 @@ +package tech.easyflow.admin.controller.datacenter; + +import cn.dev33.satoken.annotation.SaCheckPermission; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RestController; +import tech.easyflow.common.domain.Result; +import tech.easyflow.common.entity.LoginAccount; +import tech.easyflow.common.satoken.util.SaTokenUtil; +import tech.easyflow.datacenter.execution.model.DatacenterSqlConsoleRequest; +import tech.easyflow.datacenter.execution.model.DatacenterSqlConsoleResult; +import tech.easyflow.datacenter.execution.model.DatacenterSqlCancelRequest; +import tech.easyflow.datacenter.federation.DatacenterFederationQueryService; +import tech.easyflow.datacenter.federation.DatacenterFederationQueryCancellationService; +import tech.easyflow.datacenter.meta.entity.DatacenterSource; +import tech.easyflow.datacenter.meta.service.DatacenterDatasetRegistryService; + +/** + * 数据中枢管理端只读 SQL 控制台。 + */ +@RestController +@RequestMapping("/api/v1/datacenterQuery") +public class DatacenterQueryController { + + private final DatacenterDatasetRegistryService registryService; + private final DatacenterFederationQueryService queryService; + private final DatacenterFederationQueryCancellationService cancellationService; + + /** + * 创建查询 Controller。 + * + * @param registryService 数据集注册服务 + * @param queryService Federation 查询服务 + * @param cancellationService 跨节点查询取消服务 + */ + public DatacenterQueryController( + DatacenterDatasetRegistryService registryService, + DatacenterFederationQueryService queryService, + DatacenterFederationQueryCancellationService cancellationService) { + this.registryService = registryService; + this.queryService = queryService; + this.cancellationService = cancellationService; + } + + /** + * 执行一条受 Calcite 与业务 Policy 校验的只读 SQL。 + * + * @param request 查询请求 + * @return 有界查询结果 + */ + @PostMapping("/execute") + @SaCheckPermission("/api/v1/datacenterSource/query") + public Result execute( + @RequestBody DatacenterSqlConsoleRequest request) { + LoginAccount account = SaTokenUtil.getLoginAccount(); + DatacenterSource source = registryService.getSourceRequired( + request == null ? null : request.sourceId()); + return Result.ok(queryService.execute( + source, + request == null ? null : request.sql(), + java.util.List.of(), + request == null ? null : request.maxRows(), + account, + "MANUAL", + account == null || account.getId() == null + ? null : account.getId().toString(), + request == null ? null : request.queryId())); + } + + /** + * 取消当前租户在任一节点执行的 SQL 查询。 + * + * @param request 取消请求 + * @return 是否在本地或集群中接受取消提示 + */ + @PostMapping("/cancel") + @SaCheckPermission("/api/v1/datacenterSource/query") + public Result cancel(@RequestBody DatacenterSqlCancelRequest request) { + LoginAccount account = SaTokenUtil.getLoginAccount(); + return Result.ok(cancellationService.cancel( + request == null ? null : request.queryId(), account)); + } +} diff --git a/easyflow-api/easyflow-api-admin/src/main/java/tech/easyflow/admin/controller/datacenter/DatacenterSourceController.java b/easyflow-api/easyflow-api-admin/src/main/java/tech/easyflow/admin/controller/datacenter/DatacenterSourceController.java index 47d013a2..eb3914e3 100644 --- a/easyflow-api/easyflow-api-admin/src/main/java/tech/easyflow/admin/controller/datacenter/DatacenterSourceController.java +++ b/easyflow-api/easyflow-api-admin/src/main/java/tech/easyflow/admin/controller/datacenter/DatacenterSourceController.java @@ -8,10 +8,16 @@ import tech.easyflow.common.entity.LoginAccount; import tech.easyflow.common.satoken.util.SaTokenUtil; import tech.easyflow.datacenter.entity.DatacenterTable; import tech.easyflow.datacenter.execution.model.DatacenterConnectionTestResult; -import tech.easyflow.datacenter.meta.entity.DatacenterSource; import tech.easyflow.datacenter.meta.model.DatacenterBatchRegisterRequest; import tech.easyflow.datacenter.meta.model.DatacenterCatalogMeta; +import tech.easyflow.datacenter.meta.model.DatacenterMetadataPage; import tech.easyflow.datacenter.meta.model.DatacenterRemoveSourceRequest; +import tech.easyflow.datacenter.meta.model.DatacenterSourceActivateRequest; +import tech.easyflow.datacenter.meta.model.DatacenterSourceCandidateMetadataRequest; +import tech.easyflow.datacenter.meta.model.DatacenterSourceCandidateCatalogRequest; +import tech.easyflow.datacenter.meta.model.DatacenterSourceDraftRequest; +import tech.easyflow.datacenter.meta.model.DatacenterSourceReconfigureRequest; +import tech.easyflow.datacenter.meta.model.DatacenterSourceView; import tech.easyflow.datacenter.meta.model.DatacenterTableDetailMeta; import tech.easyflow.datacenter.meta.service.DatacenterSourceService; @@ -19,6 +25,9 @@ import javax.annotation.Resource; import java.math.BigInteger; import java.util.List; +/** + * 数据源绑定、生命周期与元数据浏览接口。 + */ @RestController @RequestMapping("/api/v1/datacenterSource") public class DatacenterSourceController { @@ -26,23 +35,100 @@ public class DatacenterSourceController { @Resource private DatacenterSourceService sourceService; - @PostMapping("/testConnection") - @SaCheckPermission("/api/v1/datacenterSource/query") - public Result testConnection(@RequestBody DatacenterSource source) { + @PostMapping("/draft") + @SaCheckPermission("/api/v1/datacenterSource/save") + public Result saveDraft(@RequestBody DatacenterSourceDraftRequest request) { LoginAccount account = SaTokenUtil.getLoginAccount(); - return Result.ok(sourceService.testConnection(source, account)); + return Result.ok(sourceService.saveDraft(request, account)); } - @PostMapping("/save") - @SaCheckPermission("/api/v1/datacenterSource/save") - public Result save(@RequestBody DatacenterSource source) { + @PostMapping("/{sourceId}/probe") + @SaCheckPermission("/api/v1/datacenterSource/query") + public Result probe(@PathVariable BigInteger sourceId) { LoginAccount account = SaTokenUtil.getLoginAccount(); - return Result.ok(sourceService.saveSource(source, account)); + return Result.ok(sourceService.probe(sourceId, account)); + } + + @PostMapping("/activate") + @SaCheckPermission("/api/v1/datacenterSource/save") + public Result activate(@RequestBody DatacenterSourceActivateRequest request) { + LoginAccount account = SaTokenUtil.getLoginAccount(); + return Result.ok(sourceService.activate(request, account)); + } + + /** + * 探测活动数据源的未发布候选配置。 + * + * @param request 候选连接配置 + * @return 连接探测结果 + */ + @PostMapping("/candidate/probe") + @SaCheckPermission("/api/v1/datacenterSource/save") + public Result probeCandidate( + @RequestBody DatacenterSourceDraftRequest request) { + return Result.ok(sourceService.probeCandidate( + request, SaTokenUtil.getLoginAccount())); + } + + /** + * 浏览活动数据源候选配置可访问的命名空间。 + * + * @param request 候选连接配置 + * @return 命名空间列表 + */ + @PostMapping("/candidate/catalogs") + @SaCheckPermission("/api/v1/datacenterSource/save") + public Result> candidateCatalogs( + @RequestBody DatacenterSourceDraftRequest request) { + return Result.ok(sourceService.listCandidateCatalogs( + request, SaTokenUtil.getLoginAccount())); + } + + /** + * 分页浏览候选配置可访问的命名空间。 + * + * @param request 候选配置和分页条件 + * @return 有界命名空间列表 + */ + @PostMapping("/candidate/catalogs/page") + @SaCheckPermission("/api/v1/datacenterSource/save") + public Result> candidateCatalogsPage( + @RequestBody DatacenterSourceCandidateCatalogRequest request) { + return Result.ok(sourceService.listCandidateCatalogsPage( + request, SaTokenUtil.getLoginAccount())); + } + + /** + * 分页浏览活动数据源候选配置可访问的表。 + * + * @param request 候选配置与分页条件 + * @return 有界表列表 + */ + @PostMapping("/candidate/tables") + @SaCheckPermission("/api/v1/datacenterSource/save") + public Result> candidateTables( + @RequestBody DatacenterSourceCandidateMetadataRequest request) { + return Result.ok(sourceService.listCandidateTables( + request, SaTokenUtil.getLoginAccount())); + } + + /** + * 原地发布活动数据源的新连接配置和纳管范围。 + * + * @param request 重配置请求 + * @return 发布后的数据源视图 + */ + @PostMapping("/reconfigure") + @SaCheckPermission("/api/v1/datacenterSource/save") + public Result reconfigure( + @RequestBody DatacenterSourceReconfigureRequest request) { + return Result.ok(sourceService.reconfigure( + request, SaTokenUtil.getLoginAccount())); } @GetMapping("/page") @SaCheckPermission("/api/v1/datacenterSource/query") - public Result> page(Long pageNumber, Long pageSize) { + public Result> page(Long pageNumber, Long pageSize) { LoginAccount account = SaTokenUtil.getLoginAccount(); return Result.ok(sourceService.pageSources(pageNumber, pageSize, account)); } @@ -54,19 +140,53 @@ public class DatacenterSourceController { return Result.ok(sourceService.listCatalogs(sourceId, account)); } + /** + * 分页浏览当前数据源的命名空间。 + * + * @param sourceId 数据源 ID + * @param keyword 名称搜索词 + * @param pageNumber 页码 + * @param pageSize 每页大小 + * @return 有界命名空间列表 + */ + @GetMapping("/catalogs/page") + @SaCheckPermission("/api/v1/datacenterSource/query") + public Result> catalogsPage( + BigInteger sourceId, + String keyword, + Long pageNumber, + Long pageSize) { + return Result.ok(sourceService.listCatalogsPage( + sourceId, keyword, pageNumber, pageSize, + SaTokenUtil.getLoginAccount())); + } + @GetMapping("/tables") @SaCheckPermission("/api/v1/datacenterSource/query") - public Result> tables(BigInteger sourceId, String catalogName) { + public Result> tables( + BigInteger sourceId, + String catalogName, + String keyword, + Long pageNumber, + Long pageSize) { LoginAccount account = SaTokenUtil.getLoginAccount(); - return Result.ok(sourceService.listTables(sourceId, catalogName, account)); + return Result.ok(sourceService.listTables( + sourceId, catalogName, keyword, pageNumber, pageSize, account)); } @GetMapping("/tableDetail") @SaCheckPermission("/api/v1/datacenterSource/query") - public Result tableDetail(BigInteger sourceId, String catalogName, String tableName, - @RequestParam(defaultValue = "false") boolean register) { + public Result tableDetail( + BigInteger sourceId, + String catalogName, + String tableName, + @RequestParam(defaultValue = "false") boolean register, + Long fieldPageNumber, + Long fieldPageSize) { LoginAccount account = SaTokenUtil.getLoginAccount(); - return Result.ok(sourceService.getTableDetail(sourceId, catalogName, tableName, register, account)); + return Result.ok(sourceService.getTableDetail( + sourceId, catalogName, tableName, register, + fieldPageNumber, fieldPageSize, account)); } @PostMapping("/registerBatch") @@ -83,4 +203,44 @@ public class DatacenterSourceController { sourceService.removeSource(request == null ? null : request.getSourceId(), account); return Result.ok(); } + + /** + * 停用活动数据源。 + * + * @param sourceId 数据源 ID + * @return 停用后的数据源视图 + */ + @PostMapping("/{sourceId}/disable") + @SaCheckPermission("/api/v1/datacenterSource/save") + public Result disable(@PathVariable BigInteger sourceId) { + return Result.ok(sourceService.disable( + sourceId, SaTokenUtil.getLoginAccount())); + } + + /** + * 重新启用已停用数据源。 + * + * @param sourceId 数据源 ID + * @return 启用后的数据源视图 + */ + @PostMapping("/{sourceId}/enable") + @SaCheckPermission("/api/v1/datacenterSource/save") + public Result enable(@PathVariable BigInteger sourceId) { + return Result.ok(sourceService.enable( + sourceId, SaTokenUtil.getLoginAccount())); + } + + /** + * 刷新已纳管对象的 JDBC 元数据观测状态。 + * + * @param sourceId 数据源 ID + * @return 刷新后的数据源视图 + */ + @PostMapping("/{sourceId}/metadata/refresh") + @SaCheckPermission("/api/v1/datacenterSource/save") + public Result refreshMetadata( + @PathVariable BigInteger sourceId) { + return Result.ok(sourceService.refreshMetadata( + sourceId, SaTokenUtil.getLoginAccount())); + } } diff --git a/easyflow-api/easyflow-api-admin/src/main/java/tech/easyflow/admin/model/ai/WorkflowDesignerOptionsView.java b/easyflow-api/easyflow-api-admin/src/main/java/tech/easyflow/admin/model/ai/WorkflowDesignerOptionsView.java index 5b7674f9..8b175b4e 100644 --- a/easyflow-api/easyflow-api-admin/src/main/java/tech/easyflow/admin/model/ai/WorkflowDesignerOptionsView.java +++ b/easyflow-api/easyflow-api-admin/src/main/java/tech/easyflow/admin/model/ai/WorkflowDesignerOptionsView.java @@ -126,6 +126,7 @@ public record WorkflowDesignerOptionsView( * 已接入数据集安全选项。 * * @param id 数据集 ID + * @param tenantId 租户 ID * @param sourceId 数据源 ID * @param catalogId 目录 ID * @param tableName 数据表名称 @@ -133,6 +134,7 @@ public record WorkflowDesignerOptionsView( */ public record DatasetOption( @JsonSerialize(using = ToStringSerializer.class) BigInteger id, + @JsonSerialize(using = ToStringSerializer.class) BigInteger tenantId, @JsonSerialize(using = ToStringSerializer.class) BigInteger sourceId, @JsonSerialize(using = ToStringSerializer.class) BigInteger catalogId, String tableName, diff --git a/easyflow-api/easyflow-api-admin/src/main/java/tech/easyflow/admin/service/ai/WorkflowDesignerOptionService.java b/easyflow-api/easyflow-api-admin/src/main/java/tech/easyflow/admin/service/ai/WorkflowDesignerOptionService.java index 9f4e39ab..557203f4 100644 --- a/easyflow-api/easyflow-api-admin/src/main/java/tech/easyflow/admin/service/ai/WorkflowDesignerOptionService.java +++ b/easyflow-api/easyflow-api-admin/src/main/java/tech/easyflow/admin/service/ai/WorkflowDesignerOptionService.java @@ -653,6 +653,7 @@ public class WorkflowDesignerOptionService { private WorkflowDesignerOptionsView.DatasetOption toDatasetOption(DatacenterTable table) { return new WorkflowDesignerOptionsView.DatasetOption( table.getId(), + table.getTenantId(), table.getSourceId(), table.getCatalogId(), table.getTableName(), diff --git a/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/easyagentsflow/cancellation/WorkflowDatasetQueryCancellationListener.java b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/easyagentsflow/cancellation/WorkflowDatasetQueryCancellationListener.java new file mode 100644 index 00000000..0232d7d9 --- /dev/null +++ b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/easyagentsflow/cancellation/WorkflowDatasetQueryCancellationListener.java @@ -0,0 +1,42 @@ +package tech.easyflow.ai.easyagentsflow.cancellation; + +import com.easyagents.flow.core.chain.Chain; +import com.easyagents.flow.core.chain.ChainStatus; +import com.easyagents.flow.core.chain.Event; +import com.easyagents.flow.core.chain.event.ChainStatusChangeEvent; +import com.easyagents.flow.core.chain.listener.ChainEventListener; +import org.springframework.stereotype.Component; + +/** + * 将工作流取消终态桥接到活动数据集查询。 + */ +@Component +public class WorkflowDatasetQueryCancellationListener + implements ChainEventListener { + + private final WorkflowDatasetQueryCancellationRegistry registry; + + /** + * 创建工作流查询取消监听器。 + * + * @param registry 工作流查询取消登记表 + */ + public WorkflowDatasetQueryCancellationListener( + WorkflowDatasetQueryCancellationRegistry registry) { + this.registry = registry; + } + + /** + * 在工作流进入取消终态后取消该实例的活动查询。 + * + * @param event 工作流事件 + * @param chain 工作流实例 + */ + @Override + public void onEvent(Event event, Chain chain) { + if (event instanceof ChainStatusChangeEvent statusChangeEvent + && statusChangeEvent.getStatus() == ChainStatus.CANCELLED) { + registry.cancelExecution(chain.getStateInstanceId()); + } + } +} diff --git a/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/easyagentsflow/cancellation/WorkflowDatasetQueryCancellationRegistry.java b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/easyagentsflow/cancellation/WorkflowDatasetQueryCancellationRegistry.java new file mode 100644 index 00000000..d1308f20 --- /dev/null +++ b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/easyagentsflow/cancellation/WorkflowDatasetQueryCancellationRegistry.java @@ -0,0 +1,279 @@ +package tech.easyflow.ai.easyagentsflow.cancellation; + +import java.math.BigInteger; +import java.time.Duration; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.Objects; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicBoolean; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.ObjectProvider; +import org.springframework.data.redis.core.StringRedisTemplate; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Component; +import tech.easyflow.common.entity.LoginAccount; +import tech.easyflow.common.web.exceptions.BusinessException; +import tech.easyflow.datacenter.federation.DatacenterFederationQueryCancellationService; + +/** + * 维护工作流实例到活动数据集查询的跨节点取消映射。 + */ +@Component +public class WorkflowDatasetQueryCancellationRegistry { + + private static final String ACTIVE_KEY_PREFIX = + "easyflow:workflow:dataset-query:active:"; + private static final String CANCELLED_KEY_PREFIX = + "easyflow:workflow:dataset-query:cancelled:"; + private static final Duration STATE_TTL = Duration.ofHours(24); + private static final Logger log = LoggerFactory.getLogger( + WorkflowDatasetQueryCancellationRegistry.class); + + private final DatacenterFederationQueryCancellationService cancellationService; + private final ObjectProvider redisTemplateProvider; + private final ConcurrentHashMap> + localActive = new ConcurrentHashMap<>(); + private final ConcurrentHashMap localCancelledUntil = + new ConcurrentHashMap<>(); + + /** + * 创建工作流查询取消登记表。 + * + * @param cancellationService 数据中枢查询取消服务 + * @param redisTemplateProvider 可选 Redis 模板 + */ + public WorkflowDatasetQueryCancellationRegistry( + DatacenterFederationQueryCancellationService cancellationService, + ObjectProvider redisTemplateProvider) { + this.cancellationService = cancellationService; + this.redisTemplateProvider = redisTemplateProvider; + } + + /** + * 在数据库查询开始前登记工作流与 QueryId。 + * + * @param stateInstanceId 工作流实例 ID + * @param account 执行账号 + * @param queryId 查询 UUID + * @return 必须关闭的登记句柄 + * @throws BusinessException 参数缺失时抛出 + */ + public Registration register( + String stateInstanceId, + LoginAccount account, + String queryId) { + String instanceId = requireText(stateInstanceId, "工作流实例 ID 不能为空"); + String normalizedQueryId = requireText(queryId, "queryId 不能为空"); + BigInteger tenantId = requireTenantId(account); + localActive.computeIfAbsent( + instanceId, + ignored -> new ConcurrentHashMap<>()) + .put(normalizedQueryId, tenantId); + persistActive(instanceId, normalizedQueryId, tenantId); + Registration registration = new Registration( + instanceId, normalizedQueryId, tenantId); + if (isExecutionCancelled(instanceId)) { + cancelQuery(normalizedQueryId, tenantId); + } + return registration; + } + + /** + * 取消工作流当前登记的全部数据集查询。 + * + * @param stateInstanceId 工作流实例 ID + * @return 是否发现至少一个活动查询 + */ + public boolean cancelExecution(String stateInstanceId) { + String instanceId = requireText(stateInstanceId, "工作流实例 ID 不能为空"); + long expiresAt = System.currentTimeMillis() + STATE_TTL.toMillis(); + localCancelledUntil.put(instanceId, expiresAt); + persistCancellationMarker(instanceId); + + Map active = new LinkedHashMap<>(); + ConcurrentHashMap local = localActive.get(instanceId); + if (local != null) { + active.putAll(local); + } + loadPersistedActive(instanceId).forEach(active::putIfAbsent); + active.forEach((queryId, tenantId) -> { + try { + cancelQuery(queryId, tenantId); + } catch (RuntimeException exception) { + // 单个驱动取消失败不能阻断同一工作流的其他活动查询。 + log.error("Failed to cancel workflow dataset query {} for {}", + queryId, instanceId, exception); + } + }); + return !active.isEmpty(); + } + + /** + * 清理本机已过期的取消墓碑，避免不可达工作流持续占用内存。 + */ + @Scheduled(fixedDelayString = + "${easyflow.workflow.dataset-query-cancel-cleanup-ms:60000}") + public void cleanupExpiredLocalMarkers() { + long now = System.currentTimeMillis(); + localCancelledUntil.entrySet().removeIf( + entry -> entry.getValue() <= now); + } + + private void cancelQuery(String queryId, BigInteger tenantId) { + LoginAccount account = new LoginAccount(); + account.setTenantId(tenantId); + cancellationService.cancel(queryId, account); + } + + private boolean isExecutionCancelled(String stateInstanceId) { + Long localDeadline = localCancelledUntil.get(stateInstanceId); + if (localDeadline != null && localDeadline > System.currentTimeMillis()) { + return true; + } + StringRedisTemplate redisTemplate = redisTemplateProvider.getIfAvailable(); + if (redisTemplate == null) { + return false; + } + try { + return Boolean.TRUE.equals(redisTemplate.hasKey( + cancelledKey(stateInstanceId))); + } catch (RuntimeException exception) { + log.warn("Failed to read workflow dataset cancellation marker for {}", + stateInstanceId, exception); + return false; + } + } + + private void persistActive( + String stateInstanceId, + String queryId, + BigInteger tenantId) { + StringRedisTemplate redisTemplate = redisTemplateProvider.getIfAvailable(); + if (redisTemplate == null) { + return; + } + try { + String key = activeKey(stateInstanceId); + redisTemplate.opsForHash().put(key, queryId, tenantId.toString()); + redisTemplate.expire(key, STATE_TTL); + } catch (RuntimeException exception) { + log.warn("Failed to persist workflow dataset query mapping for {}", + stateInstanceId, exception); + } + } + + private Map loadPersistedActive(String stateInstanceId) { + StringRedisTemplate redisTemplate = redisTemplateProvider.getIfAvailable(); + if (redisTemplate == null) { + return Map.of(); + } + try { + Map entries = redisTemplate.opsForHash() + .entries(activeKey(stateInstanceId)); + Map active = new LinkedHashMap<>(); + entries.forEach((queryId, tenantId) -> { + try { + active.put(String.valueOf(queryId), + new BigInteger(String.valueOf(tenantId))); + } catch (RuntimeException exception) { + log.warn("Ignored invalid workflow dataset query mapping for {}", + stateInstanceId, exception); + } + }); + return active; + } catch (RuntimeException exception) { + log.warn("Failed to load workflow dataset query mappings for {}", + stateInstanceId, exception); + return Map.of(); + } + } + + private void persistCancellationMarker(String stateInstanceId) { + StringRedisTemplate redisTemplate = redisTemplateProvider.getIfAvailable(); + if (redisTemplate == null) { + return; + } + try { + redisTemplate.opsForValue().set( + cancelledKey(stateInstanceId), "1", STATE_TTL); + } catch (RuntimeException exception) { + log.warn("Failed to persist workflow dataset cancellation marker for {}", + stateInstanceId, exception); + } + } + + private void unregister( + String stateInstanceId, + String queryId, + BigInteger tenantId) { + localActive.computeIfPresent(stateInstanceId, (ignored, queries) -> { + queries.remove(queryId, tenantId); + return queries.isEmpty() ? null : queries; + }); + StringRedisTemplate redisTemplate = redisTemplateProvider.getIfAvailable(); + if (redisTemplate == null) { + return; + } + try { + redisTemplate.opsForHash().delete( + activeKey(stateInstanceId), queryId); + } catch (RuntimeException exception) { + log.warn("Failed to remove workflow dataset query mapping for {}", + stateInstanceId, exception); + } + } + + private String activeKey(String stateInstanceId) { + return ACTIVE_KEY_PREFIX + stateInstanceId; + } + + private String cancelledKey(String stateInstanceId) { + return CANCELLED_KEY_PREFIX + stateInstanceId; + } + + private String requireText(String value, String message) { + if (value == null || value.isBlank()) { + throw new BusinessException(message); + } + return value.trim(); + } + + private BigInteger requireTenantId(LoginAccount account) { + if (account == null || account.getTenantId() == null) { + throw new BusinessException("工作流数据集查询缺少执行租户"); + } + return account.getTenantId(); + } + + /** + * 单次工作流查询登记句柄。 + */ + public final class Registration implements AutoCloseable { + + private final String stateInstanceId; + private final String queryId; + private final BigInteger tenantId; + private final AtomicBoolean closed = new AtomicBoolean(); + + private Registration( + String stateInstanceId, + String queryId, + BigInteger tenantId) { + this.stateInstanceId = stateInstanceId; + this.queryId = queryId; + this.tenantId = tenantId; + } + + /** + * 幂等注销当前工作流查询。 + */ + @Override + public void close() { + if (closed.compareAndSet(false, true)) { + unregister(stateInstanceId, queryId, tenantId); + } + } + } +} diff --git a/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/easyagentsflow/config/ChainExecutorConfig.java b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/easyagentsflow/config/ChainExecutorConfig.java index 2d4d2f3e..f78506ff 100644 --- a/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/easyagentsflow/config/ChainExecutorConfig.java +++ b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/easyagentsflow/config/ChainExecutorConfig.java @@ -14,6 +14,7 @@ import org.springframework.context.annotation.Configuration; import tech.easyflow.ai.easyagentsflow.listener.ChainErrorListenerForSave; import tech.easyflow.ai.easyagentsflow.listener.ChainEventListenerForSave; import tech.easyflow.ai.easyagentsflow.listener.NodeErrorListenerForSave; +import tech.easyflow.ai.easyagentsflow.cancellation.WorkflowDatasetQueryCancellationListener; import tech.easyflow.ai.easyagentsflow.upload.WorkflowApiUploadCleanupListener; import javax.annotation.Resource; @@ -40,6 +41,9 @@ public class ChainExecutorConfig { @Resource private WorkflowApiUploadCleanupListener workflowApiUploadCleanupListener; @Resource + private WorkflowDatasetQueryCancellationListener + workflowDatasetQueryCancellationListener; + @Resource private WorkflowExecutionBudgetProperties workflowExecutionBudgetProperties; @Resource private WorkflowRuntimeProperties workflowRuntimeProperties; @@ -91,6 +95,9 @@ public class ChainExecutorConfig { chainExecutor.addEventListener( ChainStatusChangeEvent.class, workflowApiUploadCleanupListener); + chainExecutor.addEventListener( + ChainStatusChangeEvent.class, + workflowDatasetQueryCancellationListener); chainExecutor.addErrorListener(new ChainErrorListenerForSave()); chainExecutor.addNodeErrorListener(new NodeErrorListenerForSave()); } diff --git a/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/easyagentsflow/service/WorkflowDatacenterContentService.java b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/easyagentsflow/service/WorkflowDatacenterContentService.java index e35d242e..821a4eaa 100644 --- a/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/easyagentsflow/service/WorkflowDatacenterContentService.java +++ b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/easyagentsflow/service/WorkflowDatacenterContentService.java @@ -14,12 +14,13 @@ import tech.easyflow.datacenter.execution.model.DatasetRef; import tech.easyflow.datacenter.execution.service.DatacenterDatasetQueryService; import tech.easyflow.datacenter.meta.entity.DatacenterCatalog; import tech.easyflow.datacenter.meta.entity.DatacenterSource; +import tech.easyflow.datacenter.meta.enums.DatacenterMetadataStatus; +import tech.easyflow.datacenter.meta.enums.DatacenterSensitivityLevel; import tech.easyflow.datacenter.meta.service.DatacenterDatasetRegistryService; import javax.annotation.Resource; import java.math.BigInteger; import java.util.ArrayList; -import java.util.Comparator; import java.util.LinkedHashMap; import java.util.LinkedHashSet; import java.util.List; @@ -34,6 +35,7 @@ public class WorkflowDatacenterContentService { public static final String LLM_NODE_TYPE = "llmNode"; public static final String QUERY_DATA_CONTEXT = "queryDataContext"; public static final String SEARCH_SOURCE_MISSING_MESSAGE = "查询数据节点未选择连接服务"; + public static final String SEARCH_TABLE_MISSING_MESSAGE = "查询数据节点未选择已接入表"; public static final String SEARCH_SQL_MISSING_MESSAGE = "查询数据节点未设置 SQL"; public static final String SAVE_EXPIRED_MESSAGE = "写入数据节点配置已过期，请重新选择已接入表"; public static final String INVALID_QUERY_CONTEXT_MESSAGE = "查询上下文配置无效，请重新选择查询数据节点"; @@ -135,11 +137,16 @@ public class WorkflowDatacenterContentService { if (datasetRef == null || datasetRef.getSourceId() == null) { throw new BusinessException(SEARCH_SOURCE_MISSING_MESSAGE); } + if (datasetRef.getTableId() == null) { + throw new BusinessException(SEARCH_TABLE_MISSING_MESSAGE); + } String querySql = data == null ? null : trimToNull(data.getString("querySql")); if (!StringUtils.hasText(querySql)) { throw new BusinessException(SEARCH_SQL_MISSING_MESSAGE); } - return datasetRef; + DatasetRef boundRef = bindAuthoritativeTenant(datasetRef); + data.put("datasetRef", boundRef); + return boundRef; } public DatasetRef requireSaveDatasetRef(JSONObject data) { @@ -150,6 +157,31 @@ public class WorkflowDatacenterContentService { if (datasetRef == null || datasetRef.getTableId() == null) { throw new BusinessException(SAVE_EXPIRED_MESSAGE); } + DatasetRef boundRef = bindAuthoritativeTenant(datasetRef); + data.put("datasetRef", boundRef); + return boundRef; + } + + /** + * 依据当前租户可见的权威 Source/Table 覆盖调用方声明的租户字段。 + * + * @param datasetRef 工作流数据集引用 + * @return 已绑定权威租户的数据集引用 + */ + private DatasetRef bindAuthoritativeTenant(DatasetRef datasetRef) { + DatacenterTable table = datasetRef.getTableId() == null + ? null : registryService.getTableWithFields(datasetRef.getTableId()); + BigInteger sourceId = table == null ? datasetRef.getSourceId() : table.getSourceId(); + if (sourceId == null) { + throw new BusinessException(SEARCH_SOURCE_MISSING_MESSAGE); + } + DatacenterSource source = registryService.getSourceRequired(sourceId); + if (table != null && (!sourceId.equals(table.getSourceId()) + || !java.util.Objects.equals(source.getTenantId(), table.getTenantId()))) { + throw new BusinessException("数据集引用与当前租户不一致"); + } + datasetRef.setTenantId(source.getTenantId()); + datasetRef.setSourceId(sourceId); return datasetRef; } @@ -172,41 +204,68 @@ public class WorkflowDatacenterContentService { if (datasetRef == null || datasetRef.getSourceId() == null) { throw new BusinessException(SEARCH_SOURCE_MISSING_MESSAGE); } + if (datasetRef.getTableId() == null) { + throw new BusinessException(SEARCH_TABLE_MISSING_MESSAGE); + } DatacenterSource source = registryService.getSourceRequired(datasetRef.getSourceId()); - List managedTables = registryService.listManagedTables(datasetRef.getSourceId(), datasetRef.getCatalogId()); - managedTables.sort(Comparator.comparing(table -> table.getTableName() == null ? "" : table.getTableName())); + DatacenterTable fullTable = registryService.getTableWithFields( + datasetRef.getTableId()); + if (fullTable == null + || !datasetRef.getSourceId().equals(fullTable.getSourceId()) + || !Integer.valueOf(1).equals(fullTable.getQueryable()) + || !DatacenterMetadataStatus.ACTIVE.name().equals( + fullTable.getMetadataStatus())) { + throw new BusinessException("查询数据节点绑定的表不可用"); + } + DatacenterCatalog catalog = registryService.getCatalogById( + fullTable.getCatalogId()); + if (StringUtils.hasText(datasetRef.getCatalogName()) + && (catalog == null || !datasetRef.getCatalogName().equals( + catalog.getCatalogName()))) { + throw new BusinessException("查询数据节点绑定的命名空间已变化"); + } JSONObject sourceSummary = new JSONObject(); sourceSummary.put("sourceName", source.getSourceName()); sourceSummary.put("sourceType", source.getSourceType()); JSONArray tables = new JSONArray(); - for (DatacenterTable table : managedTables) { - DatacenterTable fullTable = registryService.getTableWithFields(table.getId()); - DatacenterCatalog catalog = registryService.getCatalogById(fullTable.getCatalogId()); - if (StringUtils.hasText(datasetRef.getCatalogName()) - && (catalog == null || !datasetRef.getCatalogName().equals(catalog.getCatalogName()))) { - continue; - } - JSONObject tableSummary = new JSONObject(); - tableSummary.put("catalogName", catalog == null ? null : catalog.getCatalogName()); - tableSummary.put("tableName", fullTable.getTableName()); - tableSummary.put("tableDesc", fullTable.getTableDesc()); - JSONArray fields = new JSONArray(); - if (fullTable.getFields() != null) { - for (DatacenterTableField field : fullTable.getFields()) { - JSONObject fieldSummary = new JSONObject(); - fieldSummary.put("fieldName", field.getFieldName()); - fieldSummary.put("fieldDesc", field.getFieldDesc()); - fieldSummary.put("fieldType", resolveFieldType(field)); - fields.add(fieldSummary); + JSONObject tableSummary = new JSONObject(); + tableSummary.put("catalogName", catalog == null ? null : catalog.getCatalogName()); + tableSummary.put("tableName", fullTable.getTableName()); + tableSummary.put("tableDesc", fullTable.getTableDesc()); + JSONArray fields = new JSONArray(); + if (fullTable.getFields() != null) { + for (DatacenterTableField field : fullTable.getFields()) { + if (!isQueryablePublicField(field)) { + continue; } + JSONObject fieldSummary = new JSONObject(); + fieldSummary.put("fieldName", field.getFieldName()); + fieldSummary.put("fieldDesc", field.getFieldDesc()); + fieldSummary.put("fieldType", resolveFieldType(field)); + fields.add(fieldSummary); } - tableSummary.put("fields", fields); - tables.add(tableSummary); } + tableSummary.put("fields", fields); + tables.add(tableSummary); sourceSummary.put("tables", tables); return sourceSummary; } + /** + * 判断字段是否允许暴露给 SQL 生成上下文。 + * + * @param field 字段元数据 + * @return 是否为当前可查询的公开字段 + */ + private boolean isQueryablePublicField(DatacenterTableField field) { + return field != null + && Integer.valueOf(1).equals(field.getQueryable()) + && DatacenterMetadataStatus.ACTIVE.name().equals( + field.getMetadataStatus()) + && DatacenterSensitivityLevel.PUBLIC.name().equals( + field.getSensitivityLevel()); + } + private void injectQueryDataContext(JSONObject data, Map nodeMap) { if (data == null) { return; @@ -217,7 +276,7 @@ public class WorkflowDatacenterContentService { removeQueryDataContextPlaceholder(data); return; } - Map sourceSummaries = new LinkedHashMap<>(); + Map sourceSummaries = new LinkedHashMap<>(); Set visitedNodeIds = new LinkedHashSet<>(); for (int i = 0; i < nodeIds.size(); i++) { String nodeId = trimToNull(nodeIds.getString(i)); @@ -229,7 +288,10 @@ public class WorkflowDatacenterContentService { throw new BusinessException(INVALID_QUERY_CONTEXT_MESSAGE); } DatasetRef datasetRef = requireSearchDatasetRef(targetNode.getJSONObject("data")); - sourceSummaries.putIfAbsent(datasetRef.getSourceId(), buildSourceSummary(datasetRef)); + String summaryKey = datasetRef.getSourceId() + ":" + datasetRef.getTableId(); + if (!sourceSummaries.containsKey(summaryKey)) { + sourceSummaries.put(summaryKey, buildSourceSummary(datasetRef)); + } } String contextValue = QUERY_CONTEXT_PROMPT + "\n" + JSON.toJSONString(new ArrayList<>(sourceSummaries.values())); upsertQueryDataContextParameter(data, contextValue); @@ -343,6 +405,7 @@ public class WorkflowDatacenterContentService { private DatasetRef copyDatasetRef(DatasetRef datasetRef) { DatasetRef copy = new DatasetRef(); + copy.setTenantId(datasetRef.getTenantId()); copy.setSourceId(datasetRef.getSourceId()); copy.setCatalogId(datasetRef.getCatalogId()); copy.setCatalogName(datasetRef.getCatalogName()); diff --git a/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/node/SaveDatasetNode.java b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/node/SaveDatasetNode.java index c9c38cc9..9acd8053 100644 --- a/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/node/SaveDatasetNode.java +++ b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/node/SaveDatasetNode.java @@ -6,7 +6,6 @@ import com.easyagents.flow.core.chain.Chain; import com.easyagents.flow.core.chain.runtime.RetryableTriggerException; import com.easyagents.flow.core.node.BaseNode; import com.easyagents.flow.core.util.IoBulkhead; -import com.mybatisflex.core.tenant.TenantManager; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import tech.easyflow.ai.utils.WorkFlowUtil; @@ -55,7 +54,6 @@ public class SaveDatasetNode extends BaseNode { rows.add(item instanceof JSONObject json ? json : JSONObject.from(item)); } try { - TenantManager.ignoreTenantCondition(); try (IoBulkhead.Permit ignored = IoBulkhead.dataset().acquire(resolveIoTarget())) { writeService.saveRowsIdempotently( datasetRef, @@ -77,8 +75,6 @@ public class SaveDatasetNode extends BaseNode { } catch (Exception ex) { log.error("工作流保存数据到统一数据集失败，datasetRef={}", datasetRef, ex); throw ex; - } finally { - TenantManager.restoreTenantCondition(); } } diff --git a/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/node/SearchDatasetNode.java b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/node/SearchDatasetNode.java index 742954a3..5628d8e2 100644 --- a/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/node/SearchDatasetNode.java +++ b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/node/SearchDatasetNode.java @@ -7,12 +7,13 @@ import com.easyagents.flow.core.chain.repository.LoopInputReference; import com.easyagents.flow.core.node.BaseNode; import com.easyagents.flow.core.util.IoBulkhead; import com.mybatisflex.core.row.Row; -import com.mybatisflex.core.tenant.TenantManager; import tech.easyflow.common.util.SpringContextUtil; +import tech.easyflow.common.entity.LoginAccount; import tech.easyflow.common.web.exceptions.BusinessException; import tech.easyflow.datacenter.execution.model.DatacenterSqlQueryRequest; import tech.easyflow.datacenter.execution.model.DatasetRef; import tech.easyflow.datacenter.execution.service.DatacenterDatasetQueryService; +import tech.easyflow.ai.easyagentsflow.cancellation.WorkflowDatasetQueryCancellationRegistry; import java.util.HashMap; import java.util.List; @@ -55,32 +56,38 @@ public class SearchDatasetNode extends BaseNode { Map params = chain.getExecutionState().resolveParameters(this); DatacenterDatasetQueryService queryService = SpringContextUtil.getBean(DatacenterDatasetQueryService.class); + WorkflowDatasetQueryCancellationRegistry cancellationRegistry = + SpringContextUtil.getBean( + WorkflowDatasetQueryCancellationRegistry.class); + LoginAccount account = tech.easyflow.ai.utils.WorkFlowUtil.getOperator(chain); DatacenterSqlQueryRequest request = buildRuntimeRequest(params); Map result = new HashMap<>(); - try { - TenantManager.ignoreTenantCondition(); - try (IoBulkhead.Permit ignored = IoBulkhead.dataset().acquire(resolveIoTarget())) { - String resultId = chain.getStateInstanceId() - + ":dataset:" - + UUID.randomUUID(); - int rowCount = - chain.storeProducedLoopInputOutsideLock( - resultId, - sink -> queryService.consumeBySql( - request, - QUERY_PAGE_SIZE, - sink::accept), - 0L, - chain.currentFencingClaimId(), - chain.currentClaimGeneration()); - result.put( - resolveOutputKey("data"), - new LoopInputReference( - resultId, rowCount)); - return result; - } - } finally { - TenantManager.restoreTenantCondition(); + String queryId = UUID.randomUUID().toString(); + try (WorkflowDatasetQueryCancellationRegistry.Registration registration = + cancellationRegistry.register( + chain.getStateInstanceId(), account, queryId); + IoBulkhead.Permit ignored = IoBulkhead.dataset() + .acquire(resolveIoTarget())) { + String resultId = chain.getStateInstanceId() + + ":dataset:" + + queryId; + int rowCount = + chain.storeProducedLoopInputOutsideLock( + resultId, + sink -> queryService.consumeBySql( + request, + QUERY_PAGE_SIZE, + account, + queryId, + sink::accept), + 0L, + chain.currentFencingClaimId(), + chain.currentClaimGeneration()); + result.put( + resolveOutputKey("data"), + new LoopInputReference( + resultId, rowCount)); + return result; } } @@ -97,6 +104,9 @@ public class SearchDatasetNode extends BaseNode { } private DatacenterSqlQueryRequest buildRuntimeRequest(Map params) { + if (datasetRef == null || datasetRef.getSourceId() == null) { + throw new BusinessException("数据集绑定缺少连接信息，请重新选择数据集"); + } DatacenterSqlQueryRequest request = new DatacenterSqlQueryRequest(); request.setDatasetRef(copyDatasetRef()); request.setSql(resolveQuerySql(params)); @@ -128,12 +138,13 @@ public class SearchDatasetNode extends BaseNode { private DatasetRef copyDatasetRef() { DatasetRef copy = new DatasetRef(); + copy.setTenantId(datasetRef == null ? null : datasetRef.getTenantId()); copy.setSourceId(datasetRef == null ? null : datasetRef.getSourceId()); copy.setCatalogId(datasetRef == null ? null : datasetRef.getCatalogId()); copy.setCatalogName(datasetRef == null ? null : datasetRef.getCatalogName()); - copy.setTableId(null); - copy.setTableName(null); - copy.setVersionId(null); + copy.setTableId(datasetRef == null ? null : datasetRef.getTableId()); + copy.setTableName(datasetRef == null ? null : datasetRef.getTableName()); + copy.setVersionId(datasetRef == null ? null : datasetRef.getVersionId()); return copy; } diff --git a/easyflow-modules/easyflow-module-ai/src/test/java/tech/easyflow/ai/easyagentsflow/cancellation/WorkflowDatasetQueryCancellationRegistryTest.java b/easyflow-modules/easyflow-module-ai/src/test/java/tech/easyflow/ai/easyagentsflow/cancellation/WorkflowDatasetQueryCancellationRegistryTest.java new file mode 100644 index 00000000..5f96b05a --- /dev/null +++ b/easyflow-modules/easyflow-module-ai/src/test/java/tech/easyflow/ai/easyagentsflow/cancellation/WorkflowDatasetQueryCancellationRegistryTest.java @@ -0,0 +1,131 @@ +package tech.easyflow.ai.easyagentsflow.cancellation; + +import com.easyagents.flow.core.chain.Chain; +import com.easyagents.flow.core.chain.ChainStatus; +import com.easyagents.flow.core.chain.event.ChainStatusChangeEvent; +import java.math.BigInteger; +import java.util.UUID; +import org.junit.Assert; +import org.junit.Test; +import org.mockito.ArgumentCaptor; +import org.mockito.Mockito; +import org.springframework.beans.factory.ObjectProvider; +import org.springframework.data.redis.core.StringRedisTemplate; +import tech.easyflow.common.entity.LoginAccount; +import tech.easyflow.datacenter.federation.DatacenterFederationQueryCancellationService; + +/** + * {@link WorkflowDatasetQueryCancellationRegistry} 取消竞态回归测试。 + */ +public class WorkflowDatasetQueryCancellationRegistryTest { + + /** + * 验证工作流取消会使用登记租户取消活动 QueryId。 + */ + @Test + public void shouldCancelRegisteredQuery() { + Fixture fixture = fixture(); + String queryId = UUID.randomUUID().toString(); + LoginAccount account = account(BigInteger.valueOf(7L)); + + try (WorkflowDatasetQueryCancellationRegistry.Registration ignored = + fixture.registry.register("instance-a", account, queryId)) { + Assert.assertTrue(fixture.registry.cancelExecution("instance-a")); + } + + ArgumentCaptor accountCaptor = + ArgumentCaptor.forClass(LoginAccount.class); + Mockito.verify(fixture.cancellationService).cancel( + Mockito.eq(queryId), accountCaptor.capture()); + Assert.assertEquals(BigInteger.valueOf(7L), + accountCaptor.getValue().getTenantId()); + } + + /** + * 验证单个查询取消失败不会阻断同一工作流的其他活动查询。 + */ + @Test + public void shouldIsolateCancellationFailuresBetweenQueries() { + Fixture fixture = fixture(); + String failedQueryId = UUID.randomUUID().toString(); + String succeedingQueryId = UUID.randomUUID().toString(); + LoginAccount account = account(BigInteger.valueOf(7L)); + Mockito.doThrow(new IllegalStateException("simulated cancellation failure")) + .when(fixture.cancellationService) + .cancel(Mockito.eq(failedQueryId), Mockito.any(LoginAccount.class)); + + try (WorkflowDatasetQueryCancellationRegistry.Registration ignoredA = + fixture.registry.register("instance-failure", account, failedQueryId); + WorkflowDatasetQueryCancellationRegistry.Registration ignoredB = + fixture.registry.register("instance-failure", account, succeedingQueryId)) { + Assert.assertTrue(fixture.registry.cancelExecution("instance-failure")); + } + + Mockito.verify(fixture.cancellationService).cancel( + Mockito.eq(failedQueryId), Mockito.any(LoginAccount.class)); + Mockito.verify(fixture.cancellationService).cancel( + Mockito.eq(succeedingQueryId), Mockito.any(LoginAccount.class)); + } + + /** + * 验证取消先于节点登记发生时，后续查询仍会立即收到取消。 + */ + @Test + public void shouldCancelQueryRegisteredAfterWorkflowCancellation() { + Fixture fixture = fixture(); + String queryId = UUID.randomUUID().toString(); + + Assert.assertFalse(fixture.registry.cancelExecution("instance-b")); + try (WorkflowDatasetQueryCancellationRegistry.Registration ignored = + fixture.registry.register( + "instance-b", account(BigInteger.ONE), queryId)) { + Mockito.verify(fixture.cancellationService).cancel( + Mockito.eq(queryId), Mockito.any(LoginAccount.class)); + } + } + + /** + * 验证监听器只桥接取消终态。 + */ + @Test + public void listenerShouldBridgeOnlyCancelledStatus() { + WorkflowDatasetQueryCancellationRegistry registry = Mockito.mock( + WorkflowDatasetQueryCancellationRegistry.class); + WorkflowDatasetQueryCancellationListener listener = + new WorkflowDatasetQueryCancellationListener(registry); + Chain chain = Mockito.mock(Chain.class); + Mockito.when(chain.getStateInstanceId()).thenReturn("instance-c"); + + listener.onEvent(new ChainStatusChangeEvent( + chain, ChainStatus.SUCCEEDED, ChainStatus.RUNNING), chain); + listener.onEvent(new ChainStatusChangeEvent( + chain, ChainStatus.CANCELLED, ChainStatus.RUNNING), chain); + + Mockito.verify(registry).cancelExecution("instance-c"); + } + + @SuppressWarnings("unchecked") + private Fixture fixture() { + DatacenterFederationQueryCancellationService cancellationService = + Mockito.mock( + DatacenterFederationQueryCancellationService.class); + ObjectProvider redisProvider = + Mockito.mock(ObjectProvider.class); + Mockito.when(redisProvider.getIfAvailable()).thenReturn(null); + return new Fixture( + new WorkflowDatasetQueryCancellationRegistry( + cancellationService, redisProvider), + cancellationService); + } + + private LoginAccount account(BigInteger tenantId) { + LoginAccount account = new LoginAccount(); + account.setTenantId(tenantId); + return account; + } + + private record Fixture( + WorkflowDatasetQueryCancellationRegistry registry, + DatacenterFederationQueryCancellationService cancellationService) { + } +} diff --git a/easyflow-modules/easyflow-module-ai/src/test/java/tech/easyflow/ai/easyagentsflow/service/WorkflowDatacenterContentServiceTest.java b/easyflow-modules/easyflow-module-ai/src/test/java/tech/easyflow/ai/easyagentsflow/service/WorkflowDatacenterContentServiceTest.java index fb9a1ffc..5ec76fb2 100644 --- a/easyflow-modules/easyflow-module-ai/src/test/java/tech/easyflow/ai/easyagentsflow/service/WorkflowDatacenterContentServiceTest.java +++ b/easyflow-modules/easyflow-module-ai/src/test/java/tech/easyflow/ai/easyagentsflow/service/WorkflowDatacenterContentServiceTest.java @@ -9,6 +9,8 @@ import org.mockito.Mockito; import tech.easyflow.datacenter.entity.DatacenterTable; import tech.easyflow.datacenter.entity.DatacenterTableField; import tech.easyflow.datacenter.meta.entity.DatacenterSource; +import tech.easyflow.datacenter.meta.enums.DatacenterMetadataStatus; +import tech.easyflow.datacenter.meta.enums.DatacenterSensitivityLevel; import tech.easyflow.datacenter.meta.service.DatacenterDatasetRegistryService; import java.lang.reflect.Field; @@ -41,13 +43,22 @@ public class WorkflowDatacenterContentServiceTest { DatacenterSource source = Mockito.mock(DatacenterSource.class); Mockito.when(source.getSourceName()).thenReturn("ama 实验基线模型预算"); Mockito.when(source.getSourceType()).thenReturn("EXCEL"); + Mockito.when(source.getTenantId()).thenReturn(BigInteger.ONE); DatacenterTableField modelId = mockField("col_id", "模型ID", "VARCHAR"); DatacenterTableField inputPrice = mockField("token", "输入价格", "DECIMAL"); + DatacenterTableField hidden = mockField("hidden_col", "受限字段", "VARCHAR"); + Mockito.when(hidden.getQueryable()).thenReturn(0); DatacenterTable table = Mockito.mock(DatacenterTable.class); Mockito.when(table.getId()).thenReturn(TABLE_ID); + Mockito.when(table.getSourceId()).thenReturn(SOURCE_ID); + Mockito.when(table.getTenantId()).thenReturn(BigInteger.ONE); Mockito.when(table.getTableName()).thenReturn("Sheet1"); - Mockito.when(table.getFields()).thenReturn(List.of(modelId, inputPrice)); + Mockito.when(table.getQueryable()).thenReturn(1); + Mockito.when(table.getMetadataStatus()).thenReturn( + DatacenterMetadataStatus.ACTIVE.name()); + Mockito.when(table.getFields()).thenReturn(List.of( + modelId, inputPrice, hidden)); Mockito.when(registryService.getSourceRequired(SOURCE_ID)).thenReturn(source); Mockito.when(registryService.listManagedTables(SOURCE_ID, null)) @@ -77,6 +88,7 @@ public class WorkflowDatacenterContentServiceTest { Assert.assertTrue(contextValue.contains("col_id")); Assert.assertTrue(contextValue.contains("模型ID")); Assert.assertTrue(contextValue.contains("token AS input_price")); + Assert.assertFalse(contextValue.contains("hidden_col")); } /** @@ -103,6 +115,7 @@ public class WorkflowDatacenterContentServiceTest { private JSONObject buildWorkflowRoot() { JSONObject datasetRef = new JSONObject(); datasetRef.put("sourceId", SOURCE_ID); + datasetRef.put("tableId", TABLE_ID); JSONObject queryData = new JSONObject(); queryData.put("datasetRef", datasetRef); @@ -160,6 +173,11 @@ public class WorkflowDatacenterContentServiceTest { Mockito.when(field.getFieldName()).thenReturn(fieldName); Mockito.when(field.getFieldDesc()).thenReturn(fieldDesc); Mockito.when(field.getJdbcType()).thenReturn(jdbcType); + Mockito.when(field.getQueryable()).thenReturn(1); + Mockito.when(field.getMetadataStatus()).thenReturn( + DatacenterMetadataStatus.ACTIVE.name()); + Mockito.when(field.getSensitivityLevel()).thenReturn( + DatacenterSensitivityLevel.PUBLIC.name()); return field; } diff --git a/easyflow-modules/easyflow-module-datacenter/pom.xml b/easyflow-modules/easyflow-module-datacenter/pom.xml index 5effd619..9a7b99a0 100644 --- a/easyflow-modules/easyflow-module-datacenter/pom.xml +++ b/easyflow-modules/easyflow-module-datacenter/pom.xml @@ -20,6 +20,10 @@ com.zaxxer HikariCP + + com.easyagents + easy-agents-federation-sql-adapter-jdbc + org.postgresql postgresql diff --git a/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/audit/DatacenterQueryAudit.java b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/audit/DatacenterQueryAudit.java new file mode 100644 index 00000000..43c5bc8c --- /dev/null +++ b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/audit/DatacenterQueryAudit.java @@ -0,0 +1,168 @@ +package tech.easyflow.datacenter.audit; + +import com.fasterxml.jackson.databind.annotation.JsonSerialize; +import com.fasterxml.jackson.databind.ser.std.ToStringSerializer; +import com.mybatisflex.annotation.Column; +import com.mybatisflex.annotation.Id; +import com.mybatisflex.annotation.KeyType; +import com.mybatisflex.annotation.Table; +import com.mybatisflex.core.handler.FastjsonTypeHandler; +import java.io.Serializable; +import java.math.BigInteger; +import java.util.Date; +import java.util.List; +import java.util.Map; + +/** + * 数据中枢统一只读查询审计记录。 + */ +@Table(value = "tb_datacenter_query_audit", comment = "数据中枢统一查询审计") +public class DatacenterQueryAudit implements Serializable { + + @Id(keyType = KeyType.Generator, value = "snowFlakeId", comment = "主键") + private BigInteger id; + @Column(comment = "查询标识") + private String queryId; + @Column(tenantId = true, comment = "租户ID") + private BigInteger tenantId; + @Column(comment = "部门ID") + private BigInteger deptId; + @Column(comment = "执行账号ID") + private BigInteger executorAccountId; + @Column(comment = "调用方类型") + private String callerType; + @Column(comment = "调用方标识") + private String callerId; + @Column(comment = "数据源ID") + private BigInteger sourceId; + @Column(comment = "数据源版本") + private Long sourceRevision; + @Column(comment = "纳管范围版本") + private Long scopeRevision; + @Column(typeHandler = FastjsonTypeHandler.class, comment = "引用表摘要") + private List referencedTablesJson; + @Column(typeHandler = FastjsonTypeHandler.class, comment = "引用字段摘要") + private List referencedFieldsJson; + @Column(comment = "参数化或脱敏SQL") + private String parameterizedSql; + @Column(typeHandler = FastjsonTypeHandler.class, comment = "脱敏参数摘要") + private Map maskedParametersJson; + @Column(comment = "执行状态") + private String status; + @Column(comment = "稳定错误码") + private String errorCode; + @Column(comment = "有界错误说明") + private String errorMessage; + @Column(comment = "开始时间") + private Date startedAt; + @Column(comment = "结束时间") + private Date finishedAt; + @Column(comment = "总耗时毫秒") + private Long durationMs; + @Column(comment = "数据库耗时毫秒") + private Long databaseDurationMs; + @Column(comment = "返回行数") + private Long returnedRows; + @Column(comment = "是否截断") + private Integer truncated; + @Column(comment = "创建时间") + private Date created; + + /** @return 主键 */ + @JsonSerialize(using = ToStringSerializer.class) + public BigInteger getId() { return id; } + /** @param id 主键 */ + public void setId(BigInteger id) { this.id = id; } + /** @return 查询标识 */ + public String getQueryId() { return queryId; } + /** @param queryId 查询标识 */ + public void setQueryId(String queryId) { this.queryId = queryId; } + /** @return 租户 ID */ + public BigInteger getTenantId() { return tenantId; } + /** @param tenantId 租户 ID */ + public void setTenantId(BigInteger tenantId) { this.tenantId = tenantId; } + /** @return 部门 ID */ + public BigInteger getDeptId() { return deptId; } + /** @param deptId 部门 ID */ + public void setDeptId(BigInteger deptId) { this.deptId = deptId; } + /** @return 执行账号 ID；系统任务为空 */ + public BigInteger getExecutorAccountId() { return executorAccountId; } + /** @param executorAccountId 执行账号 ID */ + public void setExecutorAccountId(BigInteger executorAccountId) { this.executorAccountId = executorAccountId; } + /** @return 调用方类型 */ + public String getCallerType() { return callerType; } + /** @param callerType 调用方类型 */ + public void setCallerType(String callerType) { this.callerType = callerType; } + /** @return 调用方标识 */ + public String getCallerId() { return callerId; } + /** @param callerId 调用方标识 */ + public void setCallerId(String callerId) { this.callerId = callerId; } + /** @return 数据源 ID */ + public BigInteger getSourceId() { return sourceId; } + /** @param sourceId 数据源 ID */ + public void setSourceId(BigInteger sourceId) { this.sourceId = sourceId; } + /** @return 数据源版本 */ + public Long getSourceRevision() { return sourceRevision; } + /** @param sourceRevision 数据源版本 */ + public void setSourceRevision(Long sourceRevision) { this.sourceRevision = sourceRevision; } + /** @return 纳管范围版本 */ + public Long getScopeRevision() { return scopeRevision; } + /** @param scopeRevision 纳管范围版本 */ + public void setScopeRevision(Long scopeRevision) { this.scopeRevision = scopeRevision; } + /** @return 引用表 */ + public List getReferencedTablesJson() { return referencedTablesJson; } + /** @param referencedTablesJson 引用表 */ + public void setReferencedTablesJson(List referencedTablesJson) { this.referencedTablesJson = referencedTablesJson; } + /** @return 引用字段 */ + public List getReferencedFieldsJson() { return referencedFieldsJson; } + /** @param referencedFieldsJson 引用字段 */ + public void setReferencedFieldsJson(List referencedFieldsJson) { this.referencedFieldsJson = referencedFieldsJson; } + /** @return 参数化 SQL */ + public String getParameterizedSql() { return parameterizedSql; } + /** @param parameterizedSql 参数化 SQL */ + public void setParameterizedSql(String parameterizedSql) { this.parameterizedSql = parameterizedSql; } + /** @return 脱敏参数 */ + public Map getMaskedParametersJson() { return maskedParametersJson; } + /** @param maskedParametersJson 脱敏参数 */ + public void setMaskedParametersJson(Map maskedParametersJson) { this.maskedParametersJson = maskedParametersJson; } + /** @return 状态 */ + public String getStatus() { return status; } + /** @param status 状态 */ + public void setStatus(String status) { this.status = status; } + /** @return 错误码 */ + public String getErrorCode() { return errorCode; } + /** @param errorCode 错误码 */ + public void setErrorCode(String errorCode) { this.errorCode = errorCode; } + /** @return 错误说明 */ + public String getErrorMessage() { return errorMessage; } + /** @param errorMessage 错误说明 */ + public void setErrorMessage(String errorMessage) { this.errorMessage = errorMessage; } + /** @return 开始时间 */ + public Date getStartedAt() { return startedAt; } + /** @param startedAt 开始时间 */ + public void setStartedAt(Date startedAt) { this.startedAt = startedAt; } + /** @return 结束时间 */ + public Date getFinishedAt() { return finishedAt; } + /** @param finishedAt 结束时间 */ + public void setFinishedAt(Date finishedAt) { this.finishedAt = finishedAt; } + /** @return 总耗时 */ + public Long getDurationMs() { return durationMs; } + /** @param durationMs 总耗时 */ + public void setDurationMs(Long durationMs) { this.durationMs = durationMs; } + /** @return 数据库耗时 */ + public Long getDatabaseDurationMs() { return databaseDurationMs; } + /** @param databaseDurationMs 数据库耗时 */ + public void setDatabaseDurationMs(Long databaseDurationMs) { this.databaseDurationMs = databaseDurationMs; } + /** @return 返回行数 */ + public Long getReturnedRows() { return returnedRows; } + /** @param returnedRows 返回行数 */ + public void setReturnedRows(Long returnedRows) { this.returnedRows = returnedRows; } + /** @return 是否截断 */ + public Integer getTruncated() { return truncated; } + /** @param truncated 是否截断 */ + public void setTruncated(Integer truncated) { this.truncated = truncated; } + /** @return 创建时间 */ + public Date getCreated() { return created; } + /** @param created 创建时间 */ + public void setCreated(Date created) { this.created = created; } +} diff --git a/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/audit/DatacenterQueryAuditService.java b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/audit/DatacenterQueryAuditService.java new file mode 100644 index 00000000..e6d0c236 --- /dev/null +++ b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/audit/DatacenterQueryAuditService.java @@ -0,0 +1,301 @@ +package tech.easyflow.datacenter.audit; + +import java.math.BigInteger; +import java.util.Date; +import java.util.List; +import java.util.Map; +import com.mybatisflex.core.query.QueryWrapper; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Service; +import tech.easyflow.common.entity.LoginAccount; +import tech.easyflow.datacenter.mapper.DatacenterQueryAuditMapper; +import tech.easyflow.datacenter.meta.entity.DatacenterSource; + +/** + * 在查询前落账并在资源释放后收口状态的审计服务。 + */ +@Service +public class DatacenterQueryAuditService { + + private final DatacenterQueryAuditMapper auditMapper; + + /** + * 创建审计服务。 + * + * @param auditMapper 审计 Mapper + */ + public DatacenterQueryAuditService(DatacenterQueryAuditMapper auditMapper) { + this.auditMapper = auditMapper; + } + + /** + * 在数据库查询开始前写入 RUNNING 记录。 + * + * @param queryId 查询标识 + * @param source 数据源快照 + * @param sql 参数化或无参数只读 SQL + * @param parameters 脱敏参数摘要 + * @param account 执行账号 + * @param callerType 调用方类型 + * @param callerId 调用方标识 + * @return 已持久化审计记录 + */ + public DatacenterQueryAudit start( + String queryId, + DatacenterSource source, + String sql, + Map parameters, + LoginAccount account, + String callerType, + String callerId) { + Date now = new Date(); + DatacenterQueryAudit audit = new DatacenterQueryAudit(); + audit.setQueryId(queryId); + audit.setTenantId(account == null || account.getTenantId() == null + ? value(source.getTenantId()) : account.getTenantId()); + audit.setDeptId(account == null || account.getDeptId() == null + ? value(source.getDeptId()) : account.getDeptId()); + audit.setExecutorAccountId(account == null ? null : account.getId()); + audit.setCallerType(callerType == null ? "SYSTEM" : callerType); + audit.setCallerId(callerId); + audit.setSourceId(source.getId()); + audit.setSourceRevision(value(source.getDefinitionRevision())); + audit.setScopeRevision(value(source.getScopeRevision())); + audit.setReferencedTablesJson(List.of()); + audit.setReferencedFieldsJson(List.of()); + audit.setParameterizedSql(redactSql(sql)); + audit.setMaskedParametersJson(parameters == null ? Map.of() : Map.copyOf(parameters)); + audit.setStatus("RUNNING"); + audit.setStartedAt(now); + audit.setReturnedRows(0L); + audit.setTruncated(0); + audit.setCreated(now); + auditMapper.insert(audit); + return audit; + } + + /** + * 标记查询成功。 + * + * @param audit 审计记录 + * @param referencedTables Calcite 解析出的引用表 + * @param referencedFields Calcite 解析出的引用字段 + * @param returnedRows 返回行数 + * @param truncated 是否截断 + * @param durationMs 总耗时 + * @param databaseDurationMs 数据库耗时 + */ + public void succeed( + DatacenterQueryAudit audit, + List referencedTables, + List referencedFields, + long returnedRows, + boolean truncated, + long durationMs, + long databaseDurationMs) { + finish(audit, "SUCCESS", null, null, referencedTables, referencedFields, + returnedRows, truncated, durationMs, databaseDurationMs); + } + + /** + * 标记查询失败。 + * + * @param audit 审计记录 + * @param errorCode 稳定错误码 + * @param message 有界错误说明 + * @param referencedTables Calcite 已解析出的引用表 + * @param referencedFields Calcite 已解析出的引用字段 + * @param durationMs 总耗时 + */ + public void fail( + DatacenterQueryAudit audit, + String errorCode, + String message, + List referencedTables, + List referencedFields, + long durationMs) { + finish(audit, "FAILED", errorCode, bounded(message), referencedTables, referencedFields, + 0L, false, durationMs, null); + } + + private void finish( + DatacenterQueryAudit audit, + String status, + String errorCode, + String errorMessage, + List referencedTables, + List referencedFields, + long returnedRows, + boolean truncated, + long durationMs, + Long databaseDurationMs) { + audit.setStatus(status); + audit.setErrorCode(errorCode); + audit.setErrorMessage(errorMessage); + audit.setReferencedTablesJson(List.copyOf( + referencedTables == null ? List.of() : referencedTables)); + audit.setReferencedFieldsJson(List.copyOf( + referencedFields == null ? List.of() : referencedFields)); + audit.setReturnedRows(returnedRows); + audit.setTruncated(truncated ? 1 : 0); + audit.setDurationMs(durationMs); + audit.setDatabaseDurationMs(databaseDurationMs); + audit.setFinishedAt(new Date()); + auditMapper.update(audit); + } + + /** + * 将超过十分钟仍为 RUNNING 的遗留审计收口为失败，覆盖进程异常退出场景。 + */ + @Scheduled(fixedDelayString = "${easyflow.datacenter.audit-reconcile-ms:60000}") + public void reconcileAbandonedQueries() { + Date now = new Date(); + DatacenterQueryAudit patch = new DatacenterQueryAudit(); + patch.setStatus("FAILED"); + patch.setErrorCode("QUERY_ABANDONED"); + patch.setErrorMessage("查询进程异常结束或超出审计存活时间"); + patch.setFinishedAt(now); + QueryWrapper guard = QueryWrapper.create() + .eq(DatacenterQueryAudit::getStatus, "RUNNING") + .lt(DatacenterQueryAudit::getStartedAt, new Date(now.getTime() - 600_000L)); + auditMapper.updateByQuery(patch, guard); + } + + private long value(Long value) { + return value == null ? 0L : value; + } + + private BigInteger value(BigInteger value) { + return value == null ? BigInteger.ZERO : value; + } + + private String redactSql(String sql) { + if (sql == null || sql.isBlank()) { + return sql; + } + StringBuilder result = new StringBuilder(Math.min(sql.length(), 100_000)); + boolean quoted = false; + for (int index = 0; index < sql.length(); index++) { + char current = sql.charAt(index); + if (quoted) { + if (current == '\'' && index + 1 < sql.length() && sql.charAt(index + 1) == '\'') { + index++; + continue; + } + if (current == '\'') { + quoted = false; + result.append('?'); + } + continue; + } + if (current == '\'') { + quoted = true; + continue; + } + if (current == '-' && index + 1 < sql.length() && sql.charAt(index + 1) == '-') { + result.append("--?"); + index += 2; + while (index < sql.length() && sql.charAt(index) != '\n' + && sql.charAt(index) != '\r') { + index++; + } + if (index < sql.length()) { + result.append(sql.charAt(index)); + } + continue; + } + if (current == '/' && index + 1 < sql.length() && sql.charAt(index + 1) == '*') { + result.append("/*?*/"); + index = skipBlockComment(sql, index + 2); + continue; + } + String dollarDelimiter = current == '$' ? dollarDelimiter(sql, index) : null; + if (dollarDelimiter != null) { + int end = sql.indexOf(dollarDelimiter, index + dollarDelimiter.length()); + result.append('?'); + index = end < 0 ? sql.length() : end + dollarDelimiter.length() - 1; + continue; + } + if (Character.isDigit(current) + && (index == 0 || !Character.isJavaIdentifierPart(sql.charAt(index - 1)))) { + result.append('?'); + while (index + 1 < sql.length()) { + char next = sql.charAt(index + 1); + if (!Character.isDigit(next) && next != '.' && next != 'e' && next != 'E' + && next != '+' && next != '-') { + break; + } + index++; + } + continue; + } + result.append(current); + if (result.length() >= 100_000) { + break; + } + } + if (quoted) { + result.append('?'); + } + return result.toString(); + } + + /** + * 跳过支持嵌套的块注释内容。 + * + * @param sql 原始 SQL + * @param contentStart 首个注释内容字符位置 + * @return 外层注释结束标记的末尾位置；未闭合时返回末尾 + */ + private int skipBlockComment(String sql, int contentStart) { + int depth = 1; + for (int index = contentStart; index < sql.length() - 1; index++) { + if (sql.charAt(index) == '/' && sql.charAt(index + 1) == '*') { + depth++; + index++; + } else if (sql.charAt(index) == '*' && sql.charAt(index + 1) == '/') { + depth--; + if (depth == 0) { + return index + 1; + } + index++; + } + } + return sql.length(); + } + + /** + * 识别 PostgreSQL dollar-quoted 字面量起始分隔符。 + * + * @param sql 原始 SQL + * @param start 美元符号位置 + * @return 完整分隔符；当前位置不是合法起始符时为空 + */ + private String dollarDelimiter(String sql, int start) { + int end = sql.indexOf('$', start + 1); + if (end < 0) { + return null; + } + String tag = sql.substring(start + 1, end); + if (tag.isEmpty()) { + return "$$"; + } + if (!Character.isLetter(tag.charAt(0)) && tag.charAt(0) != '_') { + return null; + } + for (int index = 1; index < tag.length(); index++) { + char character = tag.charAt(index); + if (!Character.isLetterOrDigit(character) && character != '_') { + return null; + } + } + return '$' + tag + '$'; + } + + private String bounded(String message) { + if (message == null) { + return null; + } + return message.length() <= 500 ? message : message.substring(0, 500); + } +} diff --git a/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/config/DatacenterFederationRedisConfig.java b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/config/DatacenterFederationRedisConfig.java new file mode 100644 index 00000000..3b8c011f --- /dev/null +++ b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/config/DatacenterFederationRedisConfig.java @@ -0,0 +1,55 @@ +package tech.easyflow.datacenter.config; + +import com.easyagents.federation.sql.source.SourceId; +import java.nio.charset.StandardCharsets; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.boot.autoconfigure.condition.ConditionalOnBean; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.data.redis.connection.RedisConnectionFactory; +import org.springframework.data.redis.listener.ChannelTopic; +import org.springframework.data.redis.listener.RedisMessageListenerContainer; +import tech.easyflow.datacenter.federation.DatacenterFederationChangeNotifier; +import tech.easyflow.datacenter.federation.DatacenterFederationQueryCancellationService; +import tech.easyflow.datacenter.federation.DatacenterFederationSourceStateProvider; + +/** + * 数据中枢无凭据 Definition 变更提示订阅配置。 + */ +@Configuration(proxyBeanMethods = false) +public class DatacenterFederationRedisConfig { + + private static final Logger log = LoggerFactory.getLogger(DatacenterFederationRedisConfig.class); + + /** + * 创建独立 Redis 监听容器，收到提示后始终回源数据库。 + * + * @param connectionFactory Redis 连接工厂 + * @param stateProvider 数据库权威状态 Provider + * @param queryCancellationService 查询取消服务 + * @return 监听容器 + */ + @Bean(name = "datacenterFederationRedisListenerContainer") + @ConditionalOnBean(RedisConnectionFactory.class) + public RedisMessageListenerContainer datacenterFederationRedisListenerContainer( + RedisConnectionFactory connectionFactory, + DatacenterFederationSourceStateProvider stateProvider, + DatacenterFederationQueryCancellationService queryCancellationService) { + RedisMessageListenerContainer container = new RedisMessageListenerContainer(); + container.setConnectionFactory(connectionFactory); + container.addMessageListener((message, pattern) -> { + String value = new String(message.getBody(), StandardCharsets.UTF_8); + try { + stateProvider.refresh(new SourceId(value)); + } catch (RuntimeException exception) { + log.warn("Failed to refresh datacenter Federation source {}", value, exception); + } + }, new ChannelTopic(DatacenterFederationChangeNotifier.CHANNEL)); + container.addMessageListener((message, pattern) -> + queryCancellationService.acceptCancellationHint( + new String(message.getBody(), StandardCharsets.UTF_8)), + new ChannelTopic(DatacenterFederationQueryCancellationService.CHANNEL)); + return container; + } +} diff --git a/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/connector/MetadataExplorer.java b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/connector/MetadataExplorer.java index 29f566d4..70b9f80b 100644 --- a/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/connector/MetadataExplorer.java +++ b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/connector/MetadataExplorer.java @@ -3,6 +3,8 @@ package tech.easyflow.datacenter.connector; import tech.easyflow.datacenter.entity.DatacenterTable; import tech.easyflow.datacenter.meta.entity.DatacenterSource; import tech.easyflow.datacenter.meta.model.DatacenterCatalogMeta; +import tech.easyflow.datacenter.meta.model.DatacenterManagedMetadataSnapshot; +import tech.easyflow.datacenter.meta.model.DatacenterMetadataPage; import tech.easyflow.datacenter.meta.model.DatacenterTableDetailMeta; import java.util.List; @@ -10,7 +12,97 @@ import java.util.List; public interface MetadataExplorer { List listCatalogs(DatacenterSource source); + /** + * 分页浏览 Catalog 或 Schema。 + * + * @param source 数据源 + * @param keyword 名称搜索词 + * @param pageNumber 页码 + * @param pageSize 每页大小 + * @return 有界命名空间列表 + */ + default DatacenterMetadataPage listCatalogsPage( + DatacenterSource source, + String keyword, + long pageNumber, + long pageSize) { + String normalizedKeyword = keyword == null + ? "" : keyword.trim().toLowerCase(java.util.Locale.ROOT); + List filtered = listCatalogs(source).stream() + .filter(catalog -> normalizedKeyword.isEmpty() + || String.valueOf(catalog.getCatalogName()) + .toLowerCase(java.util.Locale.ROOT) + .contains(normalizedKeyword)) + .toList(); + return DatacenterMetadataPage.slice(filtered, pageNumber, pageSize); + } + List listTables(DatacenterSource source, String catalogName); + /** + * 分页浏览表或视图。 + * + *

非 JDBC Connector 默认复用已有列表能力；外部 JDBC Connector 应覆盖此方法， + * 直接在元数据 ResultSet 上做有界读取。

+ * + * @param source 数据源 + * @param catalogName 物理命名空间 + * @param keyword 表名搜索词 + * @param pageNumber 页码 + * @param pageSize 每页大小 + * @return 有界表列表 + */ + default DatacenterMetadataPage listTablesPage( + DatacenterSource source, + String catalogName, + String keyword, + long pageNumber, + long pageSize) { + String normalizedKeyword = keyword == null ? "" : keyword.trim().toLowerCase(java.util.Locale.ROOT); + List filtered = listTables(source, catalogName).stream() + .filter(table -> normalizedKeyword.isEmpty() + || String.valueOf(table.getTableName()) + .toLowerCase(java.util.Locale.ROOT) + .contains(normalizedKeyword)) + .toList(); + return DatacenterMetadataPage.slice(filtered, pageNumber, pageSize); + } + DatacenterTableDetailMeta getTableDetail(DatacenterSource source, String catalogName, String tableName); + + /** + * 批量读取表详情。 + * + * @param source 数据源 + * @param catalogName 物理命名空间 + * @param tableNames 表名集合 + * @return 表详情集合 + */ + default List getTableDetails( + DatacenterSource source, + String catalogName, + List tableNames) { + return tableNames.stream() + .map(tableName -> getTableDetail(source, catalogName, tableName)) + .toList(); + } + + /** + * 读取已纳管表的刷新快照。 + * + *

默认实现保留 Connector 的既有逐表语义；JDBC Connector 应覆盖此方法以复用 + * 单个连接，并明确区分物理表缺失与连接故障。

+ * + * @param source 数据源 + * @param catalogName 物理命名空间 + * @param tableNames 已纳管表名集合 + * @return 存在与缺失对象的快照 + */ + default DatacenterManagedMetadataSnapshot inspectManagedTables( + DatacenterSource source, + String catalogName, + List tableNames) { + return new DatacenterManagedMetadataSnapshot( + getTableDetails(source, catalogName, tableNames), java.util.Set.of()); + } } diff --git a/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/connector/impl/MysqlConnector.java b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/connector/impl/MysqlConnector.java index 1f29b099..2138d84e 100644 --- a/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/connector/impl/MysqlConnector.java +++ b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/connector/impl/MysqlConnector.java @@ -28,13 +28,12 @@ public class MysqlConnector extends AbstractJdbcConnector { @Override protected T withConnection(DatacenterSource source, boolean cacheable, JdbcCallback callback) throws Exception { - HikariDataSource dataSource = cacheable ? datasourceManager.getOrCreateExternalDatasource(source) : datasourceManager.createExternalDatasource(source); + // MySQL 查询 Runtime 已由 Federation SQL 独占管理；此 Connector 只保留短生命周期元数据访问。 + HikariDataSource dataSource = datasourceManager.createExternalDatasource(source); try (Connection connection = dataSource.getConnection()) { return callback.apply(connection); } finally { - if (!cacheable || source.getId() == null) { - dataSource.close(); - } + dataSource.close(); } } } diff --git a/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/connector/impl/PostgresqlConnector.java b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/connector/impl/PostgresqlConnector.java index 8f09b12e..655d74c2 100644 --- a/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/connector/impl/PostgresqlConnector.java +++ b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/connector/impl/PostgresqlConnector.java @@ -28,13 +28,12 @@ public class PostgresqlConnector extends AbstractJdbcConnector { @Override protected T withConnection(DatacenterSource source, boolean cacheable, JdbcCallback callback) throws Exception { - HikariDataSource dataSource = cacheable ? datasourceManager.getOrCreateExternalDatasource(source) : datasourceManager.createExternalDatasource(source); + // PostgreSQL 查询 Runtime 已由 Federation SQL 独占管理；此 Connector 只保留短生命周期元数据访问。 + HikariDataSource dataSource = datasourceManager.createExternalDatasource(source); try (Connection connection = dataSource.getConnection()) { return callback.apply(connection); } finally { - if (!cacheable || source.getId() == null) { - dataSource.close(); - } + dataSource.close(); } } } diff --git a/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/connector/support/AbstractInternalTableConnector.java b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/connector/support/AbstractInternalTableConnector.java index 79316aed..d089b3ed 100644 --- a/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/connector/support/AbstractInternalTableConnector.java +++ b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/connector/support/AbstractInternalTableConnector.java @@ -3,6 +3,7 @@ package tech.easyflow.datacenter.connector.support; import cn.hutool.core.util.StrUtil; import com.alibaba.fastjson2.JSONObject; import com.mybatisflex.core.paginate.Page; +import com.mybatisflex.core.query.QueryColumn; import com.mybatisflex.core.query.QueryWrapper; import com.mybatisflex.core.row.Db; import com.mybatisflex.core.row.Row; @@ -15,6 +16,7 @@ import tech.easyflow.datacenter.connector.DatacenterConnector; import tech.easyflow.datacenter.entity.DatacenterTable; import tech.easyflow.datacenter.entity.DatacenterTableField; import tech.easyflow.datacenter.execution.model.DatacenterConnectionTestResult; +import tech.easyflow.datacenter.execution.model.DatacenterQueryFilter; import tech.easyflow.datacenter.execution.model.DatacenterQueryRequest; import tech.easyflow.datacenter.meta.entity.DatacenterSource; import tech.easyflow.datacenter.meta.enums.DatacenterCapability; @@ -85,35 +87,180 @@ public abstract class AbstractInternalTableConnector implements DatacenterConnec @Override public Page queryPage(DatacenterSource source, DatacenterTable table, DatacenterQueryRequest request) { + if (request == null || request.getPageSize() == null + || request.getPageSize() < 1L + || request.getPageSize() > 500L) { + throw new BusinessException("pageSize 必须在 1 到 500 之间"); + } String actualTable = resolveTableName(table); long count = Db.selectCountByQuery( - actualTable, createQueryWrapper(request.getWhere())); + actualTable, createQueryWrapper(table, request, false)); if (count == 0) { return new Page<>(new ArrayList<>(), request.getPageNumber(), request.getPageSize(), count); } // selectCountByQuery 会把无投影的 QueryWrapper 改为 COUNT(*)，分页查询必须使用独立实例。 + QueryWrapper pageQuery = createQueryWrapper(table, request, true); + pageQuery.select(resolveSelectedColumns(table, request).stream() + .map(this::quoteInternalIdentifier) + .toArray(String[]::new)); Page page = Db.paginate( actualTable, new Page<>(request.getPageNumber(), request.getPageSize(), count), - createQueryWrapper(request.getWhere())); + pageQuery); normalizeRows(page.getRecords()); return page; } + /** + * 将已在服务层校验的逻辑字段映射为内部物理列。 + * + * @param table 绑定表及字段 + * @param request 查询请求 + * @return 按请求顺序排列的物理列名 + * @throws BusinessException 字段元数据缺失时抛出 + */ + private List resolveSelectedColumns( + DatacenterTable table, + DatacenterQueryRequest request) { + Map fields = new LinkedHashMap<>(); + for (DatacenterTableField field : table.getFields()) { + fields.put(field.getFieldName(), field); + } + List columns = new ArrayList<>(); + for (String selected : request.getSelectedColumns()) { + DatacenterTableField field = fields.get(selected); + if (field == null) { + throw new BusinessException("字段不存在: " + selected); + } + String sourceColumn = StrUtil.blankToDefault( + field.getSourceColumnName(), field.getFieldName()); + columns.add(sourceColumn); + } + if (columns.isEmpty()) { + throw new BusinessException("当前数据集没有可查询字段"); + } + return columns; + } + + /** + * 引用可信元数据列名，避免保留字或特殊字符改变 SQL 结构。 + * + * @param identifier 物理列名 + * @return MySQL 反引号标识符 + */ + private String quoteInternalIdentifier(String identifier) { + return "`" + identifier.replace("`", "``") + "`"; + } + /** * 创建用于动态表查询的独立条件包装器。 * - * @param where 已校验的筛选表达式 + * @param where 调用方传入的旧原始筛选表达式；非空时拒绝 * @return 新建的查询条件包装器 */ static QueryWrapper createQueryWrapper(String where) { - QueryWrapper wrapper = QueryWrapper.create(); if (StrUtil.isNotBlank(where)) { - wrapper.where(where); + throw new BusinessException("不支持原始 WHERE 条件，请使用结构化筛选条件"); + } + QueryWrapper wrapper = QueryWrapper.create(); + return wrapper; + } + + private QueryWrapper createQueryWrapper( + DatacenterTable table, + DatacenterQueryRequest request, + boolean includeSorts) { + QueryWrapper wrapper = createQueryWrapper(request.getWhere()); + Map fields = table.getFields().stream() + .collect(java.util.stream.Collectors.toMap( + DatacenterTableField::getFieldName, + field -> field, + (first, ignored) -> first, + LinkedHashMap::new)); + List parameters = new ArrayList<>(); + StringBuilder condition = new StringBuilder(); + if (request.getFilters() != null) { + for (DatacenterQueryFilter filter : request.getFilters()) { + DatacenterTableField field = fields.get(filter.getColumn()); + if (field == null) { + throw new BusinessException("字段不存在: " + filter.getColumn()); + } + appendFilter(condition, parameters, field, filter); + } + } + if (!condition.isEmpty()) { + wrapper.where(condition.toString(), parameters.toArray()); + } + if (includeSorts && request.getSorts() != null && !request.getSorts().isEmpty()) { + request.getSorts().forEach(sort -> { + DatacenterTableField field = fields.get(sort.getColumn()); + if (field == null) { + throw new BusinessException("字段不存在: " + sort.getColumn()); + } + wrapper.orderBy( + new QueryColumn(sourceColumnName(field)), + !"DESC".equalsIgnoreCase(sort.getDirection())); + }); } return wrapper; } + private void appendFilter( + StringBuilder condition, + List parameters, + DatacenterTableField field, + DatacenterQueryFilter filter) { + if (!condition.isEmpty()) { + condition.append(" AND "); + } + String column = quoteInternalIdentifier(sourceColumnName(field)); + String operator = StrUtil.blankToDefault(filter.getOperator(), "EQ") + .toUpperCase(Locale.ROOT); + switch (operator) { + case "EQ" -> appendComparison(condition, parameters, column, "=", filter.getValue()); + case "LIKE" -> appendComparison( + condition, parameters, column, "LIKE", + "%" + String.valueOf(filter.getValue()) + "%"); + case "GT" -> appendComparison(condition, parameters, column, ">", filter.getValue()); + case "GTE" -> appendComparison(condition, parameters, column, ">=", filter.getValue()); + case "LT" -> appendComparison(condition, parameters, column, "<", filter.getValue()); + case "LTE" -> appendComparison(condition, parameters, column, "<=", filter.getValue()); + case "IS_NULL" -> condition.append(column).append(" IS NULL"); + case "IN" -> appendInFilter(condition, parameters, column, filter.getValues()); + default -> throw new BusinessException("不支持的过滤操作: " + operator); + } + } + + private void appendComparison( + StringBuilder condition, + List parameters, + String column, + String operator, + Object value) { + condition.append(column).append(' ').append(operator).append(" ?"); + parameters.add(value); + } + + private void appendInFilter( + StringBuilder condition, + List parameters, + String column, + List values) { + if (values == null || values.isEmpty()) { + condition.append("1 = 0"); + return; + } + condition.append(column).append(" IN (") + .append(Collections.nCopies(values.size(), "?").stream() + .collect(java.util.stream.Collectors.joining(", "))) + .append(')'); + parameters.addAll(values); + } + + private String sourceColumnName(DatacenterTableField field) { + return StrUtil.blankToDefault(field.getSourceColumnName(), field.getFieldName()); + } + @Override public List queryBySql(DatacenterSource source, String sql) { List rows = Db.selectListBySql(sql); diff --git a/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/connector/support/AbstractJdbcConnector.java b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/connector/support/AbstractJdbcConnector.java index 54435d3e..0d1e45b2 100644 --- a/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/connector/support/AbstractJdbcConnector.java +++ b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/connector/support/AbstractJdbcConnector.java @@ -16,12 +16,17 @@ import tech.easyflow.datacenter.execution.model.DatacenterConnectionTestResult; import tech.easyflow.datacenter.execution.model.DatacenterQueryFilter; import tech.easyflow.datacenter.execution.model.DatacenterQueryRequest; import tech.easyflow.datacenter.execution.model.DatacenterQuerySort; +import tech.easyflow.datacenter.federation.DatacenterMetadataIdentity; import tech.easyflow.datacenter.meta.entity.DatacenterSource; import tech.easyflow.datacenter.meta.enums.DatacenterCapability; import tech.easyflow.datacenter.meta.enums.DatacenterConnectionErrorCode; +import tech.easyflow.datacenter.meta.enums.DatacenterMetadataStatus; +import tech.easyflow.datacenter.meta.enums.DatacenterSensitivityLevel; import tech.easyflow.datacenter.meta.enums.DatacenterSourceType; import tech.easyflow.datacenter.meta.enums.DatacenterTableKind; import tech.easyflow.datacenter.meta.model.DatacenterCatalogMeta; +import tech.easyflow.datacenter.meta.model.DatacenterMetadataPage; +import tech.easyflow.datacenter.meta.model.DatacenterManagedMetadataSnapshot; import tech.easyflow.datacenter.meta.model.DatacenterTableDetailMeta; import java.math.BigDecimal; @@ -105,30 +110,39 @@ public abstract class AbstractJdbcConnector implements DatacenterConnector { return withConnection(source, true, connection -> { List result = new ArrayList<>(); DatabaseMetaData metaData = connection.getMetaData(); - try (ResultSet catalogs = metaData.getCatalogs()) { - while (catalogs.next()) { - String name = catalogs.getString("TABLE_CAT"); - if (StrUtil.isBlank(name)) { - continue; + if (usesCatalogNamespace()) { + try (ResultSet catalogs = metaData.getCatalogs()) { + while (catalogs.next()) { + String name = catalogs.getString("TABLE_CAT"); + if (StrUtil.isBlank(name)) { + continue; + } + DatacenterCatalogMeta meta = new DatacenterCatalogMeta(); + meta.setSourceId(source.getId()); + meta.setCatalogName(name); + meta.setCatalogType("CATALOG"); + meta.setLogicalSchemaName(name); + meta.setPhysicalCatalogName(name); + result.add(meta); } - DatacenterCatalogMeta meta = new DatacenterCatalogMeta(); - meta.setSourceId(source.getId()); - meta.setCatalogName(name); - meta.setCatalogType("CATALOG"); - result.add(meta); } } - try (ResultSet schemas = metaData.getSchemas()) { - while (schemas.next()) { - String name = schemas.getString("TABLE_SCHEM"); - if (StrUtil.isBlank(name) || containsCatalog(result, name)) { - continue; + if (!usesCatalogNamespace()) { + try (ResultSet schemas = metaData.getSchemas()) { + while (schemas.next()) { + String name = schemas.getString("TABLE_SCHEM"); + if (StrUtil.isBlank(name)) { + continue; + } + DatacenterCatalogMeta meta = new DatacenterCatalogMeta(); + meta.setSourceId(source.getId()); + meta.setCatalogName(name); + meta.setCatalogType("SCHEMA"); + meta.setLogicalSchemaName(name); + meta.setPhysicalCatalogName(source.getDatabaseName()); + meta.setPhysicalSchemaName(name); + result.add(meta); } - DatacenterCatalogMeta meta = new DatacenterCatalogMeta(); - meta.setSourceId(source.getId()); - meta.setCatalogName(name); - meta.setCatalogType("SCHEMA"); - result.add(meta); } } result = filterConfiguredCatalogs(source, result); @@ -139,6 +153,13 @@ public abstract class AbstractJdbcConnector implements DatacenterConnector { meta.setSourceId(source.getId()); meta.setCatalogName(fallback); meta.setCatalogType("DEFAULT"); + meta.setLogicalSchemaName(fallback); + if (usesCatalogNamespace()) { + meta.setPhysicalCatalogName(fallback); + } else { + meta.setPhysicalCatalogName(source.getDatabaseName()); + meta.setPhysicalSchemaName(fallback); + } result.add(meta); } } @@ -149,24 +170,98 @@ public abstract class AbstractJdbcConnector implements DatacenterConnector { } } + /** + * 直接在 JDBC Catalog/Schema 元数据游标上执行有界分页。 + * + * @param source 数据源 + * @param keyword 名称搜索词 + * @param pageNumber 页码 + * @param pageSize 每页大小 + * @return 有界命名空间列表 + */ + @Override + public DatacenterMetadataPage listCatalogsPage( + DatacenterSource source, + String keyword, + long pageNumber, + long pageSize) { + try { + return withConnection(source, true, connection -> { + DatabaseMetaData metadata = connection.getMetaData(); + String normalizedKeyword = keyword == null + ? "" : keyword.trim().toLowerCase(Locale.ROOT); + String configuredName = usesCatalogNamespace() + ? StrUtil.trimToNull(source.getDatabaseName()) + : StrUtil.trimToNull(source.getSchemaName()); + long offset = (pageNumber - 1L) * pageSize; + long matched = 0L; + boolean hasMore = false; + List records = new ArrayList<>((int) pageSize); + try (ResultSet names = usesCatalogNamespace() + ? metadata.getCatalogs() : metadata.getSchemas()) { + String column = usesCatalogNamespace() ? "TABLE_CAT" : "TABLE_SCHEM"; + while (names.next()) { + String name = names.getString(column); + if (StrUtil.isBlank(name) + || (configuredName != null && !configuredName.equals(name)) + || (!normalizedKeyword.isEmpty() + && !name.toLowerCase(Locale.ROOT).contains(normalizedKeyword))) { + continue; + } + if (matched++ < offset) { + continue; + } + if (records.size() >= pageSize) { + hasMore = true; + break; + } + records.add(catalogMeta(source, name)); + } + } + if (records.isEmpty() && offset == 0L && normalizedKeyword.isEmpty()) { + String fallback = resolveCatalogName(source, null); + if (StrUtil.isNotBlank(fallback)) { + records.add(catalogMeta(source, fallback)); + } + } + return new DatacenterMetadataPage<>( + records, pageNumber, pageSize, hasMore); + }); + } catch (Exception ex) { + throw DatacenterConnectorExceptionSupport.wrapAccessException("读取目录失败", ex); + } + } + + private DatacenterCatalogMeta catalogMeta( + DatacenterSource source, + String name) { + DatacenterCatalogMeta meta = new DatacenterCatalogMeta(); + meta.setSourceId(source.getId()); + meta.setCatalogName(name); + meta.setCatalogType(usesCatalogNamespace() ? "CATALOG" : "SCHEMA"); + meta.setLogicalSchemaName(name); + if (usesCatalogNamespace()) { + meta.setPhysicalCatalogName(name); + } else { + meta.setPhysicalCatalogName(source.getDatabaseName()); + meta.setPhysicalSchemaName(name); + } + return meta; + } + @Override public List listTables(DatacenterSource source, String catalogName) { try { return withConnection(source, true, connection -> { DatabaseMetaData metaData = connection.getMetaData(); List tables = new ArrayList<>(); - try (ResultSet resultSet = metaData.getTables(resolveCatalogArgument(source, catalogName), resolveSchemaArgument(source, catalogName), "%", new String[]{"TABLE", "VIEW"})) { + try (ResultSet resultSet = metaData.getTables( + resolveCatalogArgument(source, catalogName), + metadataPattern(metaData, resolveSchemaArgument(source, catalogName)), + "%", + new String[]{"TABLE", "VIEW"})) { while (resultSet.next()) { - DatacenterTable table = new DatacenterTable(); - table.setSourceId(source.getId()); - table.setTableName(resultSet.getString("TABLE_NAME")); - table.setTableDesc(resultSet.getString("REMARKS")); - table.setActualTable(resultSet.getString("TABLE_NAME")); - table.setMaterializedTable(resultSet.getString("TABLE_NAME")); - table.setAccessMode(capabilities.contains(DatacenterCapability.WRITE_MUTATION) ? "READ_WRITE" : "READ_ONLY"); - table.setTableKind(resolveTableKind(resultSet.getString("TABLE_TYPE")).name()); - table.setCapabilitiesJson(Map.of("capabilities", capabilities.stream().map(Enum::name).toList())); - tables.add(table); + tables.add(tableFromMetadata(source, resultSet)); } } return tables; @@ -176,10 +271,166 @@ public abstract class AbstractJdbcConnector implements DatacenterConnector { } } + /** + * 直接在 JDBC 元数据游标上完成表名筛选和有界分页。 + * + * @param source 数据源 + * @param catalogName 物理命名空间 + * @param keyword 表名搜索词 + * @param pageNumber 页码 + * @param pageSize 每页大小 + * @return 有界表列表 + */ + @Override + public DatacenterMetadataPage listTablesPage( + DatacenterSource source, + String catalogName, + String keyword, + long pageNumber, + long pageSize) { + try { + return withConnection(source, true, connection -> { + DatabaseMetaData metaData = connection.getMetaData(); + String normalizedKeyword = keyword == null + ? "" : keyword.trim().toLowerCase(Locale.ROOT); + long offset = (pageNumber - 1L) * pageSize; + long matched = 0L; + List tables = new ArrayList<>((int) pageSize); + boolean hasMore = false; + try (ResultSet resultSet = metaData.getTables( + resolveCatalogArgument(source, catalogName), + metadataPattern(metaData, resolveSchemaArgument(source, catalogName)), + "%", + new String[]{"TABLE", "VIEW"})) { + while (resultSet.next()) { + String tableName = resultSet.getString("TABLE_NAME"); + if (!normalizedKeyword.isEmpty() + && (tableName == null + || !tableName.toLowerCase(Locale.ROOT).contains(normalizedKeyword))) { + continue; + } + if (matched++ < offset) { + continue; + } + if (tables.size() >= pageSize) { + hasMore = true; + break; + } + tables.add(tableFromMetadata(source, resultSet)); + } + } + return new DatacenterMetadataPage<>( + tables, pageNumber, pageSize, hasMore); + }); + } catch (Exception ex) { + throw DatacenterConnectorExceptionSupport.wrapAccessException("读取表列表失败", ex); + } + } + + private DatacenterTable tableFromMetadata( + DatacenterSource source, + ResultSet resultSet) throws SQLException { + DatacenterTable table = new DatacenterTable(); + table.setSourceId(source.getId()); + table.setTableName(resultSet.getString("TABLE_NAME")); + table.setTableDesc(resultSet.getString("REMARKS")); + table.setActualTable(resultSet.getString("TABLE_NAME")); + table.setMaterializedTable(resultSet.getString("TABLE_NAME")); + table.setAccessMode(capabilities.contains(DatacenterCapability.WRITE_MUTATION) + ? "READ_WRITE" : "READ_ONLY"); + table.setTableKind(resolveTableKind(resultSet.getString("TABLE_TYPE")).name()); + table.setQueryable(1); + table.setMetadataStatus(DatacenterMetadataStatus.ACTIVE.name()); + table.setLastSeenAt(new java.util.Date()); + table.setTimeSemantics("NONE"); + table.setMetadataRevision(1L); + table.setCapabilitiesJson(Map.of( + "capabilities", capabilities.stream().map(Enum::name).toList())); + return table; + } + @Override public DatacenterTableDetailMeta getTableDetail(DatacenterSource source, String catalogName, String tableName) { + try { + return withConnection(source, true, + connection -> readTableDetail(connection, source, catalogName, tableName)); + } catch (Exception ex) { + throw DatacenterConnectorExceptionSupport.wrapAccessException("读取表详情失败", ex); + } + } + + /** + * 使用单个元数据连接批量读取表详情，避免每张表重复创建连接池。 + * + * @param source 数据源 + * @param catalogName 物理命名空间 + * @param tableNames 表名集合 + * @return 表详情集合 + */ + @Override + public List getTableDetails( + DatacenterSource source, + String catalogName, + List tableNames) { + if (tableNames == null || tableNames.isEmpty()) { + return List.of(); + } try { return withConnection(source, true, connection -> { + List details = new ArrayList<>(tableNames.size()); + for (String tableName : tableNames) { + details.add(readTableDetail(connection, source, catalogName, tableName)); + } + return List.copyOf(details); + }); + } catch (Exception ex) { + throw DatacenterConnectorExceptionSupport.wrapAccessException("批量读取表详情失败", ex); + } + } + + /** + * 使用单个连接刷新已纳管表，并把物理缺失作为快照状态返回。 + * + * @param source 数据源 + * @param catalogName 物理命名空间 + * @param tableNames 已纳管表名集合 + * @return 已发现详情与缺失表名 + */ + @Override + public DatacenterManagedMetadataSnapshot inspectManagedTables( + DatacenterSource source, + String catalogName, + List tableNames) { + if (tableNames == null || tableNames.isEmpty()) { + return new DatacenterManagedMetadataSnapshot(List.of(), Set.of()); + } + try { + return withConnection(source, true, connection -> { + List details = new ArrayList<>(tableNames.size()); + Set missing = new java.util.LinkedHashSet<>(); + for (String tableName : tableNames) { + try { + details.add(readTableDetail(connection, source, catalogName, tableName)); + } catch (BusinessException exception) { + if (exception.getMessage() == null + || !exception.getMessage().startsWith("所选数据表已变化")) { + throw exception; + } + missing.add(tableName); + } + } + return new DatacenterManagedMetadataSnapshot(details, missing); + }); + } catch (Exception ex) { + throw DatacenterConnectorExceptionSupport.wrapAccessException("刷新已纳管表失败", ex); + } + } + + private DatacenterTableDetailMeta readTableDetail( + Connection connection, + DatacenterSource source, + String catalogName, + String tableName) throws SQLException { DatabaseMetaData metaData = connection.getMetaData(); DatacenterTableDetailMeta detail = new DatacenterTableDetailMeta(); DatacenterTable table = new DatacenterTable(); @@ -189,24 +440,35 @@ public abstract class AbstractJdbcConnector implements DatacenterConnector { table.setMaterializedTable(tableName); table.setAccessMode(capabilities.contains(DatacenterCapability.WRITE_MUTATION) ? "READ_WRITE" : "READ_ONLY"); table.setTableKind(DatacenterTableKind.EXTERNAL_TABLE.name()); + table.setQueryable(1); + table.setMetadataStatus(DatacenterMetadataStatus.ACTIVE.name()); + table.setLastSeenAt(new java.util.Date()); + table.setTimeSemantics("NONE"); + table.setMetadataRevision(1L); table.setCapabilitiesJson(Map.of("capabilities", capabilities.stream().map(Enum::name).toList())); detail.setTable(table); + boolean tableFound = false; try (ResultSet tableSet = metaData.getTables( resolveCatalogArgument(source, catalogName), - resolveSchemaArgument(source, catalogName), - tableName, + metadataPattern(metaData, resolveSchemaArgument(source, catalogName)), + metadataPattern(metaData, tableName), new String[]{"TABLE", "VIEW"})) { while (tableSet.next()) { - String currentTableName = tableSet.getString("TABLE_NAME"); - if (!matchesTableName(currentTableName, tableName)) { + if (!matchesMetadataTable( + tableSet, source, catalogName, tableName)) { continue; } table.setTableDesc(tableSet.getString("REMARKS")); table.setTableKind(resolveTableKind(tableSet.getString("TABLE_TYPE")).name()); + tableFound = true; break; } } + if (!tableFound) { + throw new BusinessException( + "所选数据表已变化，请刷新后重试: " + tableName); + } Set primaryKeys = new HashSet<>(); try (ResultSet pkSet = metaData.getPrimaryKeys(resolveCatalogArgument(source, catalogName), resolveSchemaArgument(source, catalogName), tableName)) { @@ -216,13 +478,24 @@ public abstract class AbstractJdbcConnector implements DatacenterConnector { } List fields = new ArrayList<>(); - try (ResultSet columns = metaData.getColumns(resolveCatalogArgument(source, catalogName), resolveSchemaArgument(source, catalogName), tableName, "%")) { + try (ResultSet columns = metaData.getColumns( + resolveCatalogArgument(source, catalogName), + metadataPattern(metaData, resolveSchemaArgument(source, catalogName)), + metadataPattern(metaData, tableName), + "%")) { while (columns.next()) { + if (!matchesMetadataTable( + columns, source, catalogName, tableName)) { + continue; + } DatacenterTableField field = new DatacenterTableField(); field.setFieldName(columns.getString("COLUMN_NAME")); field.setSourceColumnName(columns.getString("COLUMN_NAME")); field.setFieldDesc(columns.getString("REMARKS")); field.setJdbcType(columns.getString("TYPE_NAME")); + field.setJdbcTypeCode(columns.getInt("DATA_TYPE")); + field.setNativeTypeName(columns.getString("TYPE_NAME")); + field.setOrdinalPosition(columns.getInt("ORDINAL_POSITION")); field.setPrecision(columns.getInt("COLUMN_SIZE")); field.setScale(columns.getInt("DECIMAL_DIGITS")); field.setRequired(columns.getInt("NULLABLE") == DatabaseMetaData.columnNoNulls ? 1 : 0); @@ -231,15 +504,21 @@ public abstract class AbstractJdbcConnector implements DatacenterConnector { field.setWritable(capabilities.contains(DatacenterCapability.WRITE_MUTATION) ? 1 : 0); field.setIndexed(primaryKeys.contains(field.getFieldName()) ? 1 : 0); field.setFieldType(mapFieldType(columns.getInt("DATA_TYPE"))); + field.setMetadataStatus(DatacenterMetadataStatus.ACTIVE.name()); + field.setLastSeenAt(new java.util.Date()); + field.setSensitivityLevel(DatacenterSensitivityLevel.PUBLIC.name()); + field.setMetadataFingerprint(DatacenterMetadataIdentity.fieldFingerprint(field)); fields.add(field); } } + if (fields.isEmpty()) { + throw new BusinessException( + "无法读取数据表字段，请检查权限: " + tableName); + } + table.setMetadataFingerprint(DatacenterMetadataIdentity.tableFingerprint( + table.getTableName(), table.getTableKind(), fields)); detail.setFields(fields); return detail; - }); - } catch (Exception ex) { - throw DatacenterConnectorExceptionSupport.wrapAccessException("读取表详情失败", ex); - } } @Override @@ -247,6 +526,11 @@ public abstract class AbstractJdbcConnector implements DatacenterConnector { if (!capabilities.contains(DatacenterCapability.READ_QUERY)) { throw new BusinessException("当前数据源暂不支持查询"); } + if (request == null || request.getPageSize() == null + || request.getPageSize() < 1L + || request.getPageSize() > 500L) { + throw new BusinessException("pageSize 必须在 1 到 500 之间"); + } try { return withConnection(source, true, connection -> doQueryPage(connection, source, table, request)); } catch (Exception ex) { @@ -325,7 +609,7 @@ public abstract class AbstractJdbcConnector implements DatacenterConnector { String qualifiedTable = sqlDialect.qualifyTable(resolveCatalogName(source, request.getDatasetRef() == null ? null : request.getDatasetRef().getCatalogName()), resolvePhysicalTableName(table)); StringBuilder whereClause = new StringBuilder(); if (StrUtil.isNotBlank(request.getWhere())) { - whereClause.append(" WHERE ").append(request.getWhere()); + throw new BusinessException("不支持原始 WHERE 条件，请使用结构化筛选条件"); } else if (!CollectionUtils.isEmpty(request.getFilters())) { whereClause.append(" WHERE 1=1 "); for (DatacenterQueryFilter filter : request.getFilters()) { @@ -622,13 +906,9 @@ public abstract class AbstractJdbcConnector implements DatacenterConnector { return items; } List matched = items.stream() - .filter(item -> configuredName.equalsIgnoreCase(item.getCatalogName())) + .filter(item -> configuredName.equals(item.getCatalogName())) .collect(Collectors.toList()); - return matched.isEmpty() ? items : matched; - } - - private boolean containsCatalog(List items, String catalogName) { - return items.stream().anyMatch(item -> catalogName.equalsIgnoreCase(item.getCatalogName())); + return matched; } private DatacenterTableKind resolveTableKind(String tableType) { @@ -639,8 +919,58 @@ public abstract class AbstractJdbcConnector implements DatacenterConnector { if (currentTableName == null || targetTableName == null) { return false; } - return currentTableName.equals(targetTableName) - || currentTableName.equalsIgnoreCase(targetTableName); + return currentTableName.equals(targetTableName); + } + + /** + * 将精确标识符转换为 JDBC 元数据 Pattern，转义驱动声明的通配符字符。 + * + * @param metaData JDBC 元数据 + * @param value 精确 Schema 或表名 + * @return 可安全用于元数据 Pattern 参数的文本 + * @throws SQLException 驱动无法返回转义字符时抛出 + */ + private String metadataPattern(DatabaseMetaData metaData, String value) + throws SQLException { + if (value == null) { + return null; + } + String escape = metaData.getSearchStringEscape(); + if (escape == null || escape.isEmpty()) { + return value; + } + return value.replace(escape, escape + escape) + .replace("%", escape + "%") + .replace("_", escape + "_"); + } + + /** + * 对元数据结果再次执行精确 Catalog、Schema 与表名校验，防御驱动忽略 Pattern 转义。 + * + * @param resultSet 元数据结果集 + * @param source 数据源 + * @param catalogName 用户选择的物理命名空间 + * @param tableName 目标表名 + * @return 当前结果行是否属于精确目标 + * @throws SQLException 读取元数据列失败时抛出 + */ + private boolean matchesMetadataTable( + ResultSet resultSet, + DatacenterSource source, + String catalogName, + String tableName) throws SQLException { + if (!matchesTableName(resultSet.getString("TABLE_NAME"), tableName)) { + return false; + } + String expectedCatalog = resolveCatalogArgument(source, catalogName); + String expectedSchema = resolveSchemaArgument(source, catalogName); + String actualCatalog = resultSet.getString("TABLE_CAT"); + String actualSchema = resultSet.getString("TABLE_SCHEM"); + // Schema 型数据库已由当前连接锁定 Database；部分 PostgreSQL 驱动不会回填 TABLE_CAT。 + return (!usesCatalogNamespace() + || expectedCatalog == null + || expectedCatalog.equals(actualCatalog)) + && (expectedSchema == null || expectedSchema.equals(actualSchema)); } private Integer mapFieldType(int jdbcType) { diff --git a/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/entity/base/DatacenterTableBase.java b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/entity/base/DatacenterTableBase.java index 9c2c6d83..ea2b30a3 100644 --- a/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/entity/base/DatacenterTableBase.java +++ b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/entity/base/DatacenterTableBase.java @@ -65,6 +65,38 @@ public class DatacenterTableBase extends DateEntity implements Serializable { @Column(comment = "物理表名") private String actualTable; + /** 物理对象稳定摘要。 */ + @Column(comment = "物理对象稳定摘要") + private String physicalIdentityKey; + + /** 表结构摘要。 */ + @Column(comment = "表结构摘要") + private String metadataFingerprint; + + /** 元数据状态。 */ + @Column(comment = "元数据状态") + private String metadataStatus; + + /** 最近发现时间。 */ + @Column(comment = "最近发现时间") + private Date lastSeenAt; + + /** 是否允许统一只读查询。 */ + @Column(comment = "是否允许统一只读查询") + private Integer queryable; + + /** 时间语义。 */ + @Column(comment = "时间语义") + private String timeSemantics; + + /** 默认时间字段 ID。 */ + @Column(comment = "默认时间字段ID") + private BigInteger defaultTimeFieldId; + + /** 表元数据版本。 */ + @Column(comment = "表元数据版本") + private Long metadataRevision; + /** * 表类型 */ @@ -198,6 +230,39 @@ public class DatacenterTableBase extends DateEntity implements Serializable { this.actualTable = actualTable; } + public String getPhysicalIdentityKey() { return physicalIdentityKey; } + + public void setPhysicalIdentityKey(String physicalIdentityKey) { this.physicalIdentityKey = physicalIdentityKey; } + + public String getMetadataFingerprint() { return metadataFingerprint; } + + public void setMetadataFingerprint(String metadataFingerprint) { this.metadataFingerprint = metadataFingerprint; } + + public String getMetadataStatus() { return metadataStatus; } + + public void setMetadataStatus(String metadataStatus) { this.metadataStatus = metadataStatus; } + + public Date getLastSeenAt() { return lastSeenAt; } + + public void setLastSeenAt(Date lastSeenAt) { this.lastSeenAt = lastSeenAt; } + + public Integer getQueryable() { return queryable; } + + public void setQueryable(Integer queryable) { this.queryable = queryable; } + + public String getTimeSemantics() { return timeSemantics; } + + public void setTimeSemantics(String timeSemantics) { this.timeSemantics = timeSemantics; } + + @JsonSerialize(using = ToStringSerializer.class) + public BigInteger getDefaultTimeFieldId() { return defaultTimeFieldId; } + + public void setDefaultTimeFieldId(BigInteger defaultTimeFieldId) { this.defaultTimeFieldId = defaultTimeFieldId; } + + public Long getMetadataRevision() { return metadataRevision; } + + public void setMetadataRevision(Long metadataRevision) { this.metadataRevision = metadataRevision; } + public String getTableKind() { return tableKind; } diff --git a/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/entity/base/DatacenterTableFieldBase.java b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/entity/base/DatacenterTableFieldBase.java index beed28c4..d2a9f67f 100644 --- a/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/entity/base/DatacenterTableFieldBase.java +++ b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/entity/base/DatacenterTableFieldBase.java @@ -41,6 +41,10 @@ public class DatacenterTableFieldBase extends DateEntity implements Serializable @Column(comment = "源字段名") private String sourceColumnName; + /** JDBC 字段顺序。 */ + @Column(comment = "JDBC字段顺序") + private Integer ordinalPosition; + /** * 字段描述 */ @@ -59,6 +63,14 @@ public class DatacenterTableFieldBase extends DateEntity implements Serializable @Column(comment = "JDBC类型") private String jdbcType; + /** java.sql.Types 数值。 */ + @Column(comment = "JDBC类型编码") + private Integer jdbcTypeCode; + + /** 数据库原生类型名。 */ + @Column(comment = "数据库原生类型名") + private String nativeTypeName; + /** * 精度 */ @@ -71,6 +83,26 @@ public class DatacenterTableFieldBase extends DateEntity implements Serializable @Column(comment = "小数位") private Integer scale; + /** 字段元数据摘要。 */ + @Column(comment = "字段元数据摘要") + private String metadataFingerprint; + + /** 元数据状态。 */ + @Column(comment = "元数据状态") + private String metadataStatus; + + /** 最近发现时间。 */ + @Column(comment = "最近发现时间") + private Date lastSeenAt; + + /** 敏感级别。 */ + @Column(comment = "敏感级别") + private String sensitivityLevel; + + /** 脱敏策略。 */ + @Column(comment = "脱敏策略") + private String maskingStrategy; + /** * 是否必填 */ @@ -165,6 +197,10 @@ public class DatacenterTableFieldBase extends DateEntity implements Serializable this.sourceColumnName = sourceColumnName; } + public Integer getOrdinalPosition() { return ordinalPosition; } + + public void setOrdinalPosition(Integer ordinalPosition) { this.ordinalPosition = ordinalPosition; } + public String getFieldDesc() { return fieldDesc; } @@ -189,6 +225,14 @@ public class DatacenterTableFieldBase extends DateEntity implements Serializable this.jdbcType = jdbcType; } + public Integer getJdbcTypeCode() { return jdbcTypeCode; } + + public void setJdbcTypeCode(Integer jdbcTypeCode) { this.jdbcTypeCode = jdbcTypeCode; } + + public String getNativeTypeName() { return nativeTypeName; } + + public void setNativeTypeName(String nativeTypeName) { this.nativeTypeName = nativeTypeName; } + public Integer getPrecision() { return precision; } @@ -205,6 +249,26 @@ public class DatacenterTableFieldBase extends DateEntity implements Serializable this.scale = scale; } + public String getMetadataFingerprint() { return metadataFingerprint; } + + public void setMetadataFingerprint(String metadataFingerprint) { this.metadataFingerprint = metadataFingerprint; } + + public String getMetadataStatus() { return metadataStatus; } + + public void setMetadataStatus(String metadataStatus) { this.metadataStatus = metadataStatus; } + + public Date getLastSeenAt() { return lastSeenAt; } + + public void setLastSeenAt(Date lastSeenAt) { this.lastSeenAt = lastSeenAt; } + + public String getSensitivityLevel() { return sensitivityLevel; } + + public void setSensitivityLevel(String sensitivityLevel) { this.sensitivityLevel = sensitivityLevel; } + + public String getMaskingStrategy() { return maskingStrategy; } + + public void setMaskingStrategy(String maskingStrategy) { this.maskingStrategy = maskingStrategy; } + public Integer getRequired() { return required; } diff --git a/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/excel/service/impl/DatacenterExcelImportServiceImpl.java b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/excel/service/impl/DatacenterExcelImportServiceImpl.java index 84751ce4..9946dcda 100644 --- a/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/excel/service/impl/DatacenterExcelImportServiceImpl.java +++ b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/excel/service/impl/DatacenterExcelImportServiceImpl.java @@ -346,7 +346,7 @@ public class DatacenterExcelImportServiceImpl implements DatacenterExcelImportSe queryRequest.setDatasetRef(registryService.resolveDatasetRef(table.getId())); queryRequest.setSelectedColumns(table.getFields().stream().map(DatacenterTableField::getFieldName).toList()); final int[] rowIndex = {1}; - totalRows += iterateRows(queryRequest, row -> { + totalRows += iterateRows(queryRequest, account, row -> { org.apache.poi.ss.usermodel.Row excelRow = sheet.createRow(rowIndex[0]++); for (int i = 0; i < table.getFields().size(); i++) { Cell cell = excelRow.createCell(i); @@ -434,7 +434,7 @@ public class DatacenterExcelImportServiceImpl implements DatacenterExcelImportSe String baseName = resolveSplitPrefix(request, sourceTable.getTableName()); List derivedIds = new ArrayList<>(); final Holder holder = new Holder(); - long totalRows = iterateRows(buildFullQuery(sourceTable), row -> { + long totalRows = iterateRows(buildFullQuery(sourceTable), account, row -> { if (holder.targetTable == null || holder.currentSize >= rowBatchSize) { holder.batchNo++; holder.targetTable = createDerivedTable(source, catalog, cloneFields(sourceTable.getFields()), @@ -466,7 +466,7 @@ public class DatacenterExcelImportServiceImpl implements DatacenterExcelImportSe String prefix = resolveSplitPrefix(request, sourceTable.getTableName()); Map targets = new LinkedHashMap<>(); List derivedIds = new ArrayList<>(); - long totalRows = iterateRows(buildFullQuery(sourceTable), row -> { + long totalRows = iterateRows(buildFullQuery(sourceTable), account, row -> { String fieldValue = stringify(row.get(splitField.getFieldName())); String bucket = fieldValue == null || fieldValue.isBlank() ? "empty" : fieldValue; DatacenterTable targetTable = targets.get(bucket); @@ -543,7 +543,7 @@ public class DatacenterExcelImportServiceImpl implements DatacenterExcelImportSe Map mergedRows = new LinkedHashMap<>(); for (DatacenterTable table : tables) { Map mapping = fieldMappings.get(table.getId()); - iterateRows(buildFullQuery(table), row -> { + iterateRows(buildFullQuery(table), account, row -> { String joinValue = stringify(row.get(request.getJoinKey())); if (joinValue == null || joinValue.isBlank()) { return; @@ -729,16 +729,22 @@ public class DatacenterExcelImportServiceImpl implements DatacenterExcelImportSe } private long copyRows(DatacenterQueryRequest queryRequest, RowMapper mapper, DatacenterTable targetTable, LoginAccount account) { - return iterateRows(queryRequest, row -> saveToTable(targetTable, mapper.map(row), account)); + return iterateRows( + queryRequest, + account, + row -> saveToTable(targetTable, mapper.map(row), account)); } - private long iterateRows(DatacenterQueryRequest queryRequest, RowConsumer consumer) { + private long iterateRows( + DatacenterQueryRequest queryRequest, + LoginAccount account, + RowConsumer consumer) { long total = 0L; long pageNumber = 1L; while (true) { queryRequest.setPageNumber(pageNumber); queryRequest.setPageSize(QUERY_BATCH_SIZE); - Page page = queryService.queryPage(queryRequest); + Page page = queryService.queryPage(queryRequest, account); if (page.getRecords() == null || page.getRecords().isEmpty()) { break; } diff --git a/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/execution/model/DatacenterSchemaResponse.java b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/execution/model/DatacenterSchemaResponse.java index b87765b3..58f7ac9c 100644 --- a/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/execution/model/DatacenterSchemaResponse.java +++ b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/execution/model/DatacenterSchemaResponse.java @@ -5,31 +5,46 @@ import tech.easyflow.datacenter.entity.DatacenterTableField; import tech.easyflow.datacenter.meta.entity.DatacenterCatalog; import tech.easyflow.datacenter.meta.entity.DatacenterDatasetVersion; import tech.easyflow.datacenter.meta.entity.DatacenterDerivedTable; -import tech.easyflow.datacenter.meta.entity.DatacenterSource; +import tech.easyflow.datacenter.meta.model.DatacenterSourceView; import java.util.ArrayList; import java.util.List; public class DatacenterSchemaResponse { private DatasetRef datasetRef; - private DatacenterSource source; + private DatacenterSourceView source; private DatacenterCatalog catalog; private DatacenterTable table; private List fields = new ArrayList<>(); + private long fieldPageNumber = 1L; + private long fieldPageSize; + private boolean hasMoreFields; private List versions = new ArrayList<>(); private List upstreamLineage = new ArrayList<>(); private List downstreamLineage = new ArrayList<>(); public DatasetRef getDatasetRef() { return datasetRef; } public void setDatasetRef(DatasetRef datasetRef) { this.datasetRef = datasetRef; } - public DatacenterSource getSource() { return source; } - public void setSource(DatacenterSource source) { this.source = source; } + public DatacenterSourceView getSource() { return source; } + public void setSource(DatacenterSourceView source) { this.source = source; } public DatacenterCatalog getCatalog() { return catalog; } public void setCatalog(DatacenterCatalog catalog) { this.catalog = catalog; } public DatacenterTable getTable() { return table; } public void setTable(DatacenterTable table) { this.table = table; } public List getFields() { return fields; } public void setFields(List fields) { this.fields = fields; } + /** @return 当前字段页码 */ + public long getFieldPageNumber() { return fieldPageNumber; } + /** @param fieldPageNumber 当前字段页码 */ + public void setFieldPageNumber(long fieldPageNumber) { this.fieldPageNumber = fieldPageNumber; } + /** @return 当前字段页大小 */ + public long getFieldPageSize() { return fieldPageSize; } + /** @param fieldPageSize 当前字段页大小 */ + public void setFieldPageSize(long fieldPageSize) { this.fieldPageSize = fieldPageSize; } + /** @return 是否还有下一页字段 */ + public boolean isHasMoreFields() { return hasMoreFields; } + /** @param hasMoreFields 是否还有下一页字段 */ + public void setHasMoreFields(boolean hasMoreFields) { this.hasMoreFields = hasMoreFields; } public List getVersions() { return versions; } public void setVersions(List versions) { this.versions = versions; } public List getUpstreamLineage() { return upstreamLineage; } diff --git a/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/execution/model/DatacenterSqlCancelRequest.java b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/execution/model/DatacenterSqlCancelRequest.java new file mode 100644 index 00000000..13c82442 --- /dev/null +++ b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/execution/model/DatacenterSqlCancelRequest.java @@ -0,0 +1,9 @@ +package tech.easyflow.datacenter.execution.model; + +/** + * 管理端取消只读 SQL 查询请求。 + * + * @param queryId 客户端在执行前生成的查询 UUID + */ +public record DatacenterSqlCancelRequest(String queryId) { +} diff --git a/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/execution/model/DatacenterSqlColumnView.java b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/execution/model/DatacenterSqlColumnView.java new file mode 100644 index 00000000..88faf51d --- /dev/null +++ b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/execution/model/DatacenterSqlColumnView.java @@ -0,0 +1,18 @@ +package tech.easyflow.datacenter.execution.model; + +/** + * SQL 控制台结果列。 + * + * @param key 前端稳定列键 + * @param label JDBC 列标签 + * @param jdbcType JDBC 类型编码 + * @param typeName 数据库类型名 + * @param nullable 是否允许空值 + */ +public record DatacenterSqlColumnView( + String key, + String label, + int jdbcType, + String typeName, + boolean nullable) { +} diff --git a/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/execution/model/DatacenterSqlConsoleRequest.java b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/execution/model/DatacenterSqlConsoleRequest.java new file mode 100644 index 00000000..87fd2ac7 --- /dev/null +++ b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/execution/model/DatacenterSqlConsoleRequest.java @@ -0,0 +1,18 @@ +package tech.easyflow.datacenter.execution.model; + +import java.math.BigInteger; + +/** + * 管理端只读 SQL 控制台请求。 + * + * @param sourceId 已激活数据源 ID + * @param queryId 客户端在执行前生成的查询 UUID + * @param sql 单条只读 SQL + * @param maxRows 最大返回行数 + */ +public record DatacenterSqlConsoleRequest( + BigInteger sourceId, + String queryId, + String sql, + Integer maxRows) { +} diff --git a/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/execution/model/DatacenterSqlConsoleResult.java b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/execution/model/DatacenterSqlConsoleResult.java new file mode 100644 index 00000000..1993489d --- /dev/null +++ b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/execution/model/DatacenterSqlConsoleResult.java @@ -0,0 +1,31 @@ +package tech.easyflow.datacenter.execution.model; + +import java.util.List; +import java.util.Map; + +/** + * 有界 SQL 控制台结果。 + * + * @param queryId 查询标识 + * @param columns 结果列 + * @param rows 结果行 + * @param returnedRows 返回行数 + * @param truncated 是否因上限截断 + * @param durationMs 总耗时毫秒 + */ +public record DatacenterSqlConsoleResult( + String queryId, + List columns, + List> rows, + long returnedRows, + boolean truncated, + long durationMs) { + + /** + * 防御性复制结果集合。 + */ + public DatacenterSqlConsoleResult { + columns = List.copyOf(columns == null ? List.of() : columns); + rows = List.copyOf(rows == null ? List.of() : rows); + } +} diff --git a/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/execution/model/DatasetRef.java b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/execution/model/DatasetRef.java index e334a7d1..f30a1959 100644 --- a/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/execution/model/DatasetRef.java +++ b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/execution/model/DatasetRef.java @@ -5,6 +5,7 @@ import java.math.BigInteger; public class DatasetRef implements java.io.Serializable { private static final long serialVersionUID = 1L; + private BigInteger tenantId; private BigInteger sourceId; private BigInteger catalogId; private String catalogName; @@ -12,6 +13,8 @@ public class DatasetRef implements java.io.Serializable { private String tableName; private BigInteger versionId; + public BigInteger getTenantId() { return tenantId; } + public void setTenantId(BigInteger tenantId) { this.tenantId = tenantId; } public BigInteger getSourceId() { return sourceId; } public void setSourceId(BigInteger sourceId) { this.sourceId = sourceId; } public BigInteger getCatalogId() { return catalogId; } diff --git a/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/execution/service/DatacenterDatasetQueryService.java b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/execution/service/DatacenterDatasetQueryService.java index 74766ccd..d9ce6de3 100644 --- a/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/execution/service/DatacenterDatasetQueryService.java +++ b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/execution/service/DatacenterDatasetQueryService.java @@ -2,6 +2,7 @@ package tech.easyflow.datacenter.execution.service; import com.mybatisflex.core.paginate.Page; import com.mybatisflex.core.row.Row; +import tech.easyflow.common.entity.LoginAccount; import tech.easyflow.datacenter.execution.model.DatacenterQueryRequest; import tech.easyflow.datacenter.execution.model.DatacenterSchemaResponse; import tech.easyflow.datacenter.execution.model.DatacenterSqlQueryRequest; @@ -20,6 +21,19 @@ public interface DatacenterDatasetQueryService { */ Page queryPage(DatacenterQueryRequest request); + /** + * 使用明确执行账号分页查询结构化数据集。 + * + * @param request 查询请求 + * @param account 执行账号 + * @return 分页结果 + */ + default Page queryPage( + DatacenterQueryRequest request, + LoginAccount account) { + return queryPage(request); + } + /** * 执行原生 SQL 并返回完整结果。 * @@ -35,9 +49,43 @@ public interface DatacenterDatasetQueryService { * @param fetchSize JDBC 建议拉取行数 * @param consumer 单行消费者 */ + default void consumeBySql( + DatacenterSqlQueryRequest request, + int fetchSize, + Consumer consumer) { + consumeBySql(request, fetchSize, null, consumer); + } + + /** + * 使用明确执行账号流式消费 SQL，后台工作流据此绑定权威租户。 + * + * @param request SQL 查询请求 + * @param fetchSize JDBC 建议拉取行数 + * @param account 执行账号 + * @param consumer 单行消费者 + */ + default void consumeBySql( + DatacenterSqlQueryRequest request, + int fetchSize, + LoginAccount account, + Consumer consumer) { + consumeBySql(request, fetchSize, account, null, consumer); + } + + /** + * 使用明确执行账号和调用方 QueryId 流式消费 SQL。 + * + * @param request SQL 查询请求 + * @param fetchSize JDBC 建议拉取行数 + * @param account 执行账号 + * @param requestedQueryId 调用方预生成的查询 UUID；为空时服务端生成 + * @param consumer 单行消费者 + */ void consumeBySql( DatacenterSqlQueryRequest request, int fetchSize, + LoginAccount account, + String requestedQueryId, Consumer consumer); /** @@ -48,6 +96,19 @@ public interface DatacenterDatasetQueryService { */ DatacenterSchemaResponse getSchema(DatasetRef datasetRef); + /** + * 获取有界字段页的数据集结构。 + * + * @param datasetRef 数据集引用 + * @param fieldPageNumber 字段页码 + * @param fieldPageSize 字段页大小 + * @return 数据集结构和当前字段页 + */ + DatacenterSchemaResponse getSchema( + DatasetRef datasetRef, + Long fieldPageNumber, + Long fieldPageSize); + /** * 仅解析数据集定位信息，不加载版本和血缘。 * diff --git a/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/execution/service/impl/DatacenterDatasetQueryServiceImpl.java b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/execution/service/impl/DatacenterDatasetQueryServiceImpl.java index fad8cadf..707ea735 100644 --- a/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/execution/service/impl/DatacenterDatasetQueryServiceImpl.java +++ b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/execution/service/impl/DatacenterDatasetQueryServiceImpl.java @@ -1,22 +1,43 @@ package tech.easyflow.datacenter.execution.service.impl; +import com.easyagents.federation.sql.execute.SqlParameter; +import com.easyagents.federation.sql.execute.QueryId; import com.mybatisflex.core.paginate.Page; import com.mybatisflex.core.query.QueryWrapper; import com.mybatisflex.core.row.Row; +import java.math.BigInteger; +import java.sql.Types; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.UUID; +import java.util.function.Consumer; +import java.util.function.Function; +import java.util.stream.Collectors; import org.springframework.stereotype.Service; import org.springframework.util.CollectionUtils; import org.springframework.util.StringUtils; import tech.easyflow.common.web.exceptions.BusinessException; +import tech.easyflow.common.entity.LoginAccount; import tech.easyflow.datacenter.connector.DatacenterConnector; import tech.easyflow.datacenter.connector.DatacenterConnectorRegistry; import tech.easyflow.datacenter.connector.support.AbstractInternalTableConnector; +import tech.easyflow.datacenter.audit.DatacenterQueryAudit; +import tech.easyflow.datacenter.audit.DatacenterQueryAuditService; import tech.easyflow.datacenter.entity.DatacenterTable; import tech.easyflow.datacenter.entity.DatacenterTableField; +import tech.easyflow.datacenter.execution.model.DatacenterQueryFilter; import tech.easyflow.datacenter.execution.model.DatacenterQueryRequest; +import tech.easyflow.datacenter.execution.model.DatacenterQuerySort; import tech.easyflow.datacenter.execution.model.DatacenterSchemaResponse; +import tech.easyflow.datacenter.execution.model.DatacenterSqlConsoleResult; import tech.easyflow.datacenter.execution.model.DatacenterSqlQueryRequest; import tech.easyflow.datacenter.execution.model.DatasetRef; import tech.easyflow.datacenter.execution.service.DatacenterDatasetQueryService; +import tech.easyflow.datacenter.federation.DatacenterFederationQueryService; import tech.easyflow.datacenter.mapper.DatacenterCatalogMapper; import tech.easyflow.datacenter.mapper.DatacenterDatasetVersionMapper; import tech.easyflow.datacenter.mapper.DatacenterDerivedTableMapper; @@ -25,19 +46,14 @@ import tech.easyflow.datacenter.meta.entity.DatacenterCatalog; import tech.easyflow.datacenter.meta.entity.DatacenterDatasetVersion; import tech.easyflow.datacenter.meta.entity.DatacenterDerivedTable; import tech.easyflow.datacenter.meta.entity.DatacenterSource; +import tech.easyflow.datacenter.meta.model.DatacenterSourceViews; +import tech.easyflow.datacenter.meta.enums.DatacenterMetadataStatus; +import tech.easyflow.datacenter.meta.enums.DatacenterSourceType; +import tech.easyflow.datacenter.meta.enums.DatacenterSensitivityLevel; import tech.easyflow.datacenter.meta.service.DatacenterDatasetRegistryService; import tech.easyflow.datacenter.utils.SqlSupportUtils; import javax.annotation.Resource; -import java.math.BigInteger; -import java.util.LinkedHashMap; -import java.util.LinkedHashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.function.Function; -import java.util.function.Consumer; -import java.util.stream.Collectors; @Service public class DatacenterDatasetQueryServiceImpl implements DatacenterDatasetQueryService { @@ -54,18 +70,34 @@ public class DatacenterDatasetQueryServiceImpl implements DatacenterDatasetQuery private DatacenterDatasetVersionMapper datasetVersionMapper; @Resource private DatacenterDerivedTableMapper derivedTableMapper; + @Resource + private DatacenterFederationQueryService federationQueryService; + @Resource + private DatacenterQueryAuditService queryAuditService; @Override public Page queryPage(DatacenterQueryRequest request) { + throw new BusinessException("结构化查询必须提供执行账号"); + } + + /** + * {@inheritDoc} + */ + @Override + public Page queryPage( + DatacenterQueryRequest request, + LoginAccount account) { if (request == null || request.getDatasetRef() == null) { throw new BusinessException("datasetRef 不能为空"); } normalizePage(request); DatacenterTable table = resolveTable(request.getDatasetRef()); DatacenterSource source = registryService.getSourceRequired(table.getSourceId()); + validateStructuredQueryTenant( + request.getDatasetRef(), table, source, account); DatacenterCatalog catalog = registryService.getCatalogById(table.getCatalogId()); DatacenterTable queryTable = resolveQueryTable(table, request.getDatasetRef()); - validateRequest(queryTable, request, source); + validateRequest(queryTable, request); request.getDatasetRef().setSourceId(table.getSourceId()); request.getDatasetRef().setCatalogId(table.getCatalogId()); request.getDatasetRef().setTableId(table.getId()); @@ -73,15 +105,199 @@ public class DatacenterDatasetQueryServiceImpl implements DatacenterDatasetQuery if (catalog != null) { request.getDatasetRef().setCatalogName(catalog.getCatalogName()); } + if (isFederated(source)) { + return queryFederatedPage( + source, catalog, queryTable, request, account); + } DatacenterConnector connector = connectorRegistry.getConnector(source.getSourceType()); - return connector.queryPage(source, queryTable, request); + QueryId queryId = QueryId.create(); + long startedNanos = System.nanoTime(); + DatacenterQueryAudit audit = queryAuditService.start( + queryId.value(), source, structuredQueryDescription(queryTable, request), + structuredParameterSummary(request), account, + "DATASET_PAGE", table.getId().toString()); + try { + Page result = connector.queryPage(source, queryTable, request); + long durationMs = elapsedMillis(startedNanos); + queryAuditService.succeed( + audit, + List.of(table.getTableName()), + structuredReferencedFields(request), + result.getRecords().size(), + false, + durationMs, + durationMs); + return result; + } catch (RuntimeException exception) { + queryAuditService.fail( + audit, + "DATASET_PAGE_QUERY_FAILED", + safeMessage(exception), + List.of(table.getTableName()), + structuredReferencedFields(request), + elapsedMillis(startedNanos)); + throw exception; + } + } + + /** + * 使用执行账号绑定结构化查询的权威租户。 + * + * @param datasetRef 数据集引用 + * @param table 已解析表 + * @param source 已解析数据源 + * @param account 执行账号 + * @throws BusinessException 账号或租户边界不合法时抛出 + */ + private void validateStructuredQueryTenant( + DatasetRef datasetRef, + DatacenterTable table, + DatacenterSource source, + LoginAccount account) { + if (account == null || account.getTenantId() == null) { + throw new BusinessException("结构化查询缺少执行账号租户"); + } + BigInteger tenantId = account.getTenantId(); + if ((datasetRef.getTenantId() != null + && !tenantId.equals(datasetRef.getTenantId())) + || !tenantId.equals(table.getTenantId()) + || !tenantId.equals(source.getTenantId())) { + throw new BusinessException("数据集不属于当前租户"); + } + datasetRef.setTenantId(tenantId); + } + + private String structuredQueryDescription( + DatacenterTable table, + DatacenterQueryRequest request) { + StringBuilder sql = new StringBuilder("SELECT ") + .append(String.join(", ", request.getSelectedColumns())) + .append(" FROM ").append(table.getTableName()); + if (!CollectionUtils.isEmpty(request.getFilters())) { + sql.append(" WHERE "); + for (int index = 0; index < request.getFilters().size(); index++) { + DatacenterQueryFilter filter = request.getFilters().get(index); + if (index > 0) { + sql.append(" AND "); + } + String operator = normalizeFilterOperator(filter.getOperator()); + sql.append(filter.getColumn()).append(' '); + switch (operator) { + case "IS_NULL" -> sql.append("IS NULL"); + case "IN" -> sql.append("IN (?)"); + default -> sql.append(operator).append(" ?"); + } + } + } + if (!CollectionUtils.isEmpty(request.getSorts())) { + sql.append(" ORDER BY "); + for (int index = 0; index < request.getSorts().size(); index++) { + DatacenterQuerySort sort = request.getSorts().get(index); + if (index > 0) { + sql.append(", "); + } + sql.append(sort.getColumn()).append(' ') + .append("DESC".equalsIgnoreCase(sort.getDirection()) + ? "DESC" : "ASC"); + } + } + return sql.toString(); + } + + /** + * 构建不包含实际值的结构化查询参数摘要。 + * + * @param request 查询请求 + * @return 脱敏参数摘要 + */ + private Map structuredParameterSummary( + DatacenterQueryRequest request) { + List> filters = new ArrayList<>(); + if (!CollectionUtils.isEmpty(request.getFilters())) { + for (DatacenterQueryFilter filter : request.getFilters()) { + String operator = normalizeFilterOperator(filter.getOperator()); + int valueCount = "IN".equals(operator) + ? (filter.getValues() == null ? 0 : filter.getValues().size()) + : "IS_NULL".equals(operator) ? 0 : 1; + filters.add(Map.of( + "column", filter.getColumn(), + "operator", operator, + "valueCount", valueCount)); + } + } + return Map.of( + "filters", List.copyOf(filters), + "pageNumber", request.getPageNumber(), + "pageSize", request.getPageSize()); + } + + /** + * 收集结构化查询实际引用的投影、筛选和排序字段。 + * + * @param request 查询请求 + * @return 去重后的字段列表 + */ + private List structuredReferencedFields( + DatacenterQueryRequest request) { + Set fields = new LinkedHashSet<>(request.getSelectedColumns()); + if (!CollectionUtils.isEmpty(request.getFilters())) { + request.getFilters().stream() + .map(DatacenterQueryFilter::getColumn) + .forEach(fields::add); + } + if (!CollectionUtils.isEmpty(request.getSorts())) { + request.getSorts().stream() + .map(DatacenterQuerySort::getColumn) + .forEach(fields::add); + } + return List.copyOf(fields); + } + + /** + * 将结构化筛选操作符归一化为审计中的稳定名称。 + * + * @param operator 原始操作符 + * @return 大写稳定操作符 + */ + private String normalizeFilterOperator(String operator) { + return trimToNull(operator) == null + ? "EQ" : operator.trim().toUpperCase(java.util.Locale.ROOT); } @Override public List queryBySql(DatacenterSqlQueryRequest request) { ResolvedSqlQuery query = resolveSqlQuery(request); - return query.connector.queryBySql( - query.source, query.sql); + if (isFederated(query.source)) { + DatacenterSqlConsoleResult result = federationQueryService.execute( + query.source, + query.sql, + List.of(), + 1_000, + null, + "DATASET", + request.getDatasetRef().getTableId() == null + ? null : request.getDatasetRef().getTableId().toString()); + if (result.truncated()) { + throw new BusinessException("数据集查询结果超过返回上限，请使用流式消费"); + } + return toRows(result); + } + QueryId queryId = QueryId.create(); + long startedNanos = System.nanoTime(); + DatacenterQueryAudit audit = queryAuditService.start( + queryId.value(), query.source, query.sql, Map.of(), null, + "DATASET", request.getDatasetRef().getTableId().toString()); + try { + List rows = query.connector.queryBySql(query.source, query.sql); + long durationMs = elapsedMillis(startedNanos); + queryAuditService.succeed(audit, query.referencedTables, List.of(), + rows.size(), false, durationMs, durationMs); + return rows; + } catch (RuntimeException exception) { + queryAuditService.fail(audit, "DATASET_QUERY_FAILED", safeMessage(exception), + query.referencedTables, List.of(), elapsedMillis(startedNanos)); + throw exception; + } } /** @@ -91,12 +307,38 @@ public class DatacenterDatasetQueryServiceImpl implements DatacenterDatasetQuery public void consumeBySql( DatacenterSqlQueryRequest request, int fetchSize, + LoginAccount account, + String requestedQueryId, Consumer consumer) { if (fetchSize <= 0 || consumer == null) { throw new IllegalArgumentException( "fetchSize and consumer must be valid"); } - ResolvedSqlQuery query = resolveSqlQuery(request); + ResolvedSqlQuery query = resolveSqlQuery(request, account); + if (isFederated(query.source)) { + int maxRows = Integer.getInteger( + "easyflow.datacenter.query.max-rows", 1_000_000); + long maxBytes = Long.getLong( + "easyflow.datacenter.query.max-bytes", 512L * 1024L * 1024L); + federationQueryService.consume( + query.source, + query.sql, + List.of(), + fetchSize, + maxRows, + maxBytes, + account, + "DATASET", + request.getDatasetRef().getTableId() == null + ? null : request.getDatasetRef().getTableId().toString(), + requestedQueryId, + sourceRow -> { + Row row = new Row(); + sourceRow.forEach(row::put); + consumer.accept(row); + }); + return; + } int maxRows = Integer.getInteger( "easyflow.datacenter.query.max-rows", 1_000_000); @@ -105,34 +347,48 @@ public class DatacenterDatasetQueryServiceImpl implements DatacenterDatasetQuery 512L * 1024L * 1024L); long[] accumulatedRows = {0L}; long[] accumulatedBytes = {0L}; - query.connector.consumeBySql( - query.source, - query.sql, - fetchSize, - row -> { - accumulatedRows[0]++; - if (maxRows > 0 - && accumulatedRows[0] > maxRows) { - throw new BusinessException( - "数据集查询结果超过行数上限: " - + maxRows); - } - for (Map.Entry entry - : row.entrySet()) { - accumulatedBytes[0] += - estimateQueryValueBytes( - entry.getKey(), - entry.getValue()); - if (maxBytes > 0L - && accumulatedBytes[0] > maxBytes) { + QueryId queryId = resolveQueryId(requestedQueryId); + long startedNanos = System.nanoTime(); + DatacenterQueryAudit audit = queryAuditService.start( + queryId.value(), query.source, query.sql, Map.of(), account, + "DATASET", request.getDatasetRef().getTableId().toString()); + try { + query.connector.consumeBySql( + query.source, + query.sql, + fetchSize, + row -> { + accumulatedRows[0]++; + if (maxRows > 0 + && accumulatedRows[0] > maxRows) { throw new BusinessException( - "数据集查询结果超过字节上限: " - + maxBytes); + "数据集查询结果超过行数上限: " + + maxRows); } + for (Map.Entry entry + : row.entrySet()) { + accumulatedBytes[0] += + estimateQueryValueBytes( + entry.getKey(), + entry.getValue()); + if (maxBytes > 0L + && accumulatedBytes[0] > maxBytes) { + throw new BusinessException( + "数据集查询结果超过字节上限: " + + maxBytes); + } + } + consumer.accept(row); } - consumer.accept(row); - } - ); + ); + long durationMs = elapsedMillis(startedNanos); + queryAuditService.succeed(audit, query.referencedTables, List.of(), + accumulatedRows[0], false, durationMs, durationMs); + } catch (RuntimeException exception) { + queryAuditService.fail(audit, "DATASET_QUERY_FAILED", safeMessage(exception), + query.referencedTables, List.of(), elapsedMillis(startedNanos)); + throw exception; + } } /** @@ -171,6 +427,19 @@ public class DatacenterDatasetQueryServiceImpl implements DatacenterDatasetQuery */ private ResolvedSqlQuery resolveSqlQuery( DatacenterSqlQueryRequest request) { + return resolveSqlQuery(request, null); + } + + /** + * 校验请求、执行账号租户并解析实际连接器与可执行 SQL。 + * + * @param request SQL 查询请求 + * @param account 后台执行账号；普通同步调用可为空 + * @return 已解析查询 + */ + private ResolvedSqlQuery resolveSqlQuery( + DatacenterSqlQueryRequest request, + LoginAccount account) { if (request == null || request.getDatasetRef() == null) { throw new BusinessException("datasetRef 不能为空"); } @@ -183,8 +452,28 @@ public class DatacenterDatasetQueryServiceImpl implements DatacenterDatasetQuery throw new BusinessException("缺少连接服务配置"); } DatacenterSource source = registryService.getSourceRequired(datasetRef.getSourceId()); + BigInteger authoritativeTenantId = account != null && account.getTenantId() != null + ? account.getTenantId() : datasetRef.getTenantId(); + if (authoritativeTenantId == null) { + throw new BusinessException("数据集绑定缺少租户信息"); + } + if ((datasetRef.getTenantId() != null + && !datasetRef.getTenantId().equals(authoritativeTenantId)) + || !authoritativeTenantId.equals(source.getTenantId())) { + throw new BusinessException("数据集不属于当前租户"); + } + datasetRef.setTenantId(authoritativeTenantId); BigInteger catalogId = resolveRequestedCatalogId(datasetRef); - List managedTables = registryService.listManagedTables(datasetRef.getSourceId(), catalogId); + if (datasetRef.getTableId() == null) { + throw new BusinessException("数据集 SQL 查询必须绑定具体表"); + } + DatacenterTable boundTable = registryService.getTableWithFields(datasetRef.getTableId()); + if (!datasetRef.getSourceId().equals(boundTable.getSourceId()) + || (catalogId != null && !catalogId.equals(boundTable.getCatalogId())) + || !authoritativeTenantId.equals(boundTable.getTenantId())) { + throw new BusinessException("数据集引用与绑定表不一致"); + } + List managedTables = List.of(boundTable); if (CollectionUtils.isEmpty(managedTables)) { throw new BusinessException("当前连接下没有已接入表"); } @@ -198,12 +487,15 @@ public class DatacenterDatasetQueryServiceImpl implements DatacenterDatasetQuery // 内部连接的 catalog 是逻辑命名空间，底层项目 MySQL 只执行物理表名。 SqlSupportUtils.ResolvedSql resolvedSql = connector instanceof AbstractInternalTableConnector + || DatacenterSourceType.PROJECT_MYSQL.name() + .equals(source.getSourceType()) ? SqlSupportUtils.resolveInternalMysql(sql, sqlTables) : SqlSupportUtils.resolve(sql, sqlTables); return new ResolvedSqlQuery( source, connector, - resolvedSql.getExecutableSql()); + isFederated(source) ? sql : resolvedSql.getExecutableSql(), + resolvedSql.getLogicalTables()); } /** @@ -212,30 +504,123 @@ public class DatacenterDatasetQueryServiceImpl implements DatacenterDatasetQuery * @param source 数据源 * @param connector 数据连接器 * @param sql 可执行 SQL + * @param referencedTables 已校验引用表 */ private record ResolvedSqlQuery( DatacenterSource source, DatacenterConnector connector, - String sql) { + String sql, + List referencedTables) { + } + + private long elapsedMillis(long startedNanos) { + return Math.max(0L, (System.nanoTime() - startedNanos) / 1_000_000L); + } + + /** + * 校验调用方查询 UUID，内部连接也使用同一标识完成审计关联。 + * + * @param requestedQueryId 调用方标识；为空时生成 + * @return 查询标识 + */ + private QueryId resolveQueryId(String requestedQueryId) { + if (!StringUtils.hasText(requestedQueryId)) { + return QueryId.create(); + } + try { + return new QueryId(UUID.fromString( + requestedQueryId.trim()).toString()); + } catch (IllegalArgumentException exception) { + throw new BusinessException("queryId 必须是 UUID"); + } + } + + private String safeMessage(RuntimeException exception) { + String message = exception.getMessage(); + if (message == null || message.isBlank()) { + return "数据集查询失败"; + } + return message.length() <= 500 ? message : message.substring(0, 500); } @Override public DatacenterSchemaResponse getSchema(DatasetRef datasetRef) { + return getSchema(datasetRef, null, null); + } + + /** + * {@inheritDoc} + */ + @Override + public DatacenterSchemaResponse getSchema( + DatasetRef datasetRef, + Long fieldPageNumber, + Long fieldPageSize) { DatacenterTable table = resolveTable(datasetRef); + List allFields = List.copyOf(table.getFields()); + long actualPage = fieldPageNumber == null ? 1L : Math.max(1L, fieldPageNumber); + long actualSize = fieldPageSize == null + ? Math.max(1L, allFields.size()) + : Math.min(200L, Math.max(1L, fieldPageSize)); + List pageFields = sliceFields( + allFields, actualPage, actualSize); + long offset = safePageOffset(actualPage, actualSize); + boolean hasMoreFields = offset < allFields.size() + && offset + pageFields.size() < allFields.size(); + // 字段只通过顶层分页字段返回，避免在 table 内重复序列化完整列表。 + table.setFields(List.of()); DatacenterSource source = registryService.getSourceRequired(table.getSourceId()); DatacenterCatalog catalog = registryService.getCatalogById(table.getCatalogId()); DatacenterSchemaResponse response = new DatacenterSchemaResponse(); response.setDatasetRef(registryService.resolveDatasetRef(table.getId())); - response.setSource(source); + response.setSource(DatacenterSourceViews.from(source)); response.setCatalog(catalog); response.setTable(table); - response.setFields(table.getFields()); + response.setFields(pageFields); + response.setFieldPageNumber(actualPage); + response.setFieldPageSize(actualSize); + response.setHasMoreFields(hasMoreFields); response.setVersions(listVersions(table.getId())); response.setUpstreamLineage(listUpstream(table.getId())); response.setDownstreamLineage(listDownstream(table.getId())); return response; } + /** + * 截取字段页并避免向响应暴露可变子列表。 + * + * @param fields 完整字段列表 + * @param pageNumber 页码 + * @param pageSize 页大小 + * @return 当前页字段 + */ + private List sliceFields( + List fields, + long pageNumber, + long pageSize) { + long offset = safePageOffset(pageNumber, pageSize); + if (offset >= fields.size()) { + return List.of(); + } + int fromIndex = Math.toIntExact(offset); + int toIndex = (int) Math.min((long) fields.size(), offset + pageSize); + return List.copyOf(fields.subList(fromIndex, toIndex)); + } + + /** + * 计算字段页偏移并防止长整型乘法溢出。 + * + * @param pageNumber 页码 + * @param pageSize 页大小 + * @return 安全偏移；溢出时返回最大长整型 + */ + private long safePageOffset(long pageNumber, long pageSize) { + if (pageNumber - 1L > Long.MAX_VALUE / pageSize) { + return Long.MAX_VALUE; + } + return (pageNumber - 1L) * pageSize; + } + /** * {@inheritDoc} */ @@ -245,8 +630,8 @@ public class DatacenterDatasetQueryServiceImpl implements DatacenterDatasetQuery DatacenterSchemaResponse response = new DatacenterSchemaResponse(); response.setDatasetRef(datasetRef); - response.setSource( - registryService.getSourceRequired(table.getSourceId())); + response.setSource(DatacenterSourceViews.from( + registryService.getSourceRequired(table.getSourceId()))); response.setCatalog( registryService.getCatalogById(table.getCatalogId())); response.setTable(table); @@ -319,13 +704,29 @@ public class DatacenterDatasetQueryServiceImpl implements DatacenterDatasetQuery DatacenterCatalog catalog = catalogId == null ? null : catalogsById.get(catalogId); + List knownColumns = table.getFields().stream() + .filter(field -> DatacenterMetadataStatus.ACTIVE.name() + .equals(field.getMetadataStatus())) + .map(this::sourceColumnName) + .toList(); + List queryableColumns = table.getFields().stream() + .filter(this::isQueryable) + .map(this::sourceColumnName) + .toList(); return new SqlSupportUtils.ManagedTable( catalog == null ? null : catalog.getCatalogName(), table.getTableName(), - resolvePhysicalTableName(table) + resolvePhysicalTableName(table), + knownColumns, + queryableColumns ); } + private String sourceColumnName(DatacenterTableField field) { + String sourceColumnName = trimToNull(field.getSourceColumnName()); + return sourceColumnName == null ? field.getFieldName() : sourceColumnName; + } + /** * 一次批量加载 SQL 白名单表关联的目录，避免逐表查询。 * @@ -367,6 +768,153 @@ public class DatacenterDatasetQueryServiceImpl implements DatacenterDatasetQuery return actualTable != null ? actualTable : table.getTableName(); } + /** + * 使用逻辑 Schema 构建参数化 ANSI SQL，再由 Calcite 转换为目标方言。 + */ + private Page queryFederatedPage( + DatacenterSource source, + DatacenterCatalog catalog, + DatacenterTable table, + DatacenterQueryRequest request, + LoginAccount account) { + if (catalog == null || catalog.getLogicalSchemaName() == null) { + throw new BusinessException("数据集缺少 Federation Schema 绑定"); + } + if (request.getPageSize() > 500L) { + throw new BusinessException("外部数据源单页最多返回 500 行"); + } + Map fields = table.getFields().stream() + .collect(Collectors.toMap( + DatacenterTableField::getFieldName, + Function.identity(), + (first, ignored) -> first, + LinkedHashMap::new)); + FederatedWhere where = buildFederatedWhere(request.getFilters(), fields); + String from = quote(catalog.getLogicalSchemaName()) + "." + quote(table.getTableName()); + String countSql = "SELECT COUNT(*) AS total FROM " + from + where.sql(); + DatacenterSqlConsoleResult countResult = federationQueryService.execute( + source, countSql, where.parameters(), 1, account, + "DATASET", table.getId().toString()); + long total = countResult.rows().isEmpty() + ? 0L : Long.parseLong(String.valueOf(countResult.rows().get(0).get("c1"))); + + if (total == 0L) { + return new Page<>(List.of(), request.getPageNumber(), request.getPageSize(), 0L); + } + String columns = request.getSelectedColumns().stream() + .map(name -> quote(requireField(fields, name).getSourceColumnName())) + .collect(Collectors.joining(", ")); + String orderBy = buildFederatedOrderBy(request.getSorts(), fields); + long offset = (request.getPageNumber() - 1L) * request.getPageSize(); + String pageSql = "SELECT " + columns + " FROM " + from + where.sql() + orderBy + + " OFFSET " + offset + " ROWS FETCH NEXT " + request.getPageSize() + " ROWS ONLY"; + DatacenterSqlConsoleResult pageResult = federationQueryService.execute( + source, pageSql, where.parameters(), request.getPageSize().intValue(), + account, "DATASET", table.getId().toString()); + return new Page<>(toRows(pageResult), request.getPageNumber(), request.getPageSize(), total); + } + + private FederatedWhere buildFederatedWhere( + List filters, + Map fields) { + if (CollectionUtils.isEmpty(filters)) { + return new FederatedWhere("", List.of()); + } + StringBuilder sql = new StringBuilder(" WHERE 1 = 1"); + List parameters = new ArrayList<>(); + for (DatacenterQueryFilter filter : filters) { + DatacenterTableField field = requireField(fields, filter.getColumn()); + String column = quote(field.getSourceColumnName()); + String operator = trimToNull(filter.getOperator()) == null + ? "EQ" : filter.getOperator().trim().toUpperCase(java.util.Locale.ROOT); + switch (operator) { + case "EQ", "GT", "GTE", "LT", "LTE", "LIKE" -> { + String sqlOperator = switch (operator) { + case "GT" -> ">"; + case "GTE" -> ">="; + case "LT" -> "<"; + case "LTE" -> "<="; + case "LIKE" -> "LIKE"; + default -> "="; + }; + sql.append(" AND ").append(column).append(' ').append(sqlOperator).append(" ?"); + Object value = "LIKE".equals(operator) + ? "%" + filter.getValue() + "%" : filter.getValue(); + parameters.add(parameter(field, value)); + } + case "IS_NULL" -> sql.append(" AND ").append(column).append(" IS NULL"); + case "IN" -> { + List values = filter.getValues() == null ? List.of() : filter.getValues(); + if (values.isEmpty()) { + sql.append(" AND 1 = 0"); + } else { + sql.append(" AND ").append(column).append(" IN (") + .append(values.stream().map(ignored -> "?").collect(Collectors.joining(","))) + .append(')'); + values.forEach(value -> parameters.add(parameter(field, value))); + } + } + default -> throw new BusinessException("不支持的过滤操作: " + operator); + } + } + return new FederatedWhere(sql.toString(), List.copyOf(parameters)); + } + + private String buildFederatedOrderBy( + List sorts, + Map fields) { + if (CollectionUtils.isEmpty(sorts)) { + return ""; + } + return " ORDER BY " + sorts.stream() + .map(sort -> quote(requireField(fields, sort.getColumn()).getSourceColumnName()) + + ("DESC".equalsIgnoreCase(sort.getDirection()) ? " DESC" : " ASC")) + .collect(Collectors.joining(", ")); + } + + private DatacenterTableField requireField( + Map fields, + String name) { + DatacenterTableField field = fields.get(name); + if (field == null || !isEnabled(field.getQueryable())) { + throw new BusinessException("字段不可查询: " + name); + } + return field; + } + + private SqlParameter parameter(DatacenterTableField field, Object value) { + int jdbcType = field.getJdbcTypeCode() == null ? Types.VARCHAR : field.getJdbcTypeCode(); + return value == null ? new SqlParameter(jdbcType, null) : new SqlParameter(jdbcType, value); + } + + private String quote(String identifier) { + if (!StringUtils.hasText(identifier)) { + throw new BusinessException("数据集物理名称为空"); + } + return '"' + identifier.replace("\"", "\"\"") + '"'; + } + + private List toRows(DatacenterSqlConsoleResult result) { + List rows = new ArrayList<>(result.rows().size()); + for (Map sourceRow : result.rows()) { + Row row = new Row(); + for (int index = 0; index < result.columns().size(); index++) { + var column = result.columns().get(index); + row.put(column.label(), sourceRow.get(column.key())); + } + rows.add(row); + } + return rows; + } + + private boolean isFederated(DatacenterSource source) { + return source != null && (DatacenterSourceType.MYSQL.name().equals(source.getSourceType()) + || DatacenterSourceType.POSTGRESQL.name().equals(source.getSourceType())); + } + + private record FederatedWhere(String sql, List parameters) { + } + private void normalizePage(DatacenterQueryRequest request) { if (request.getPageNumber() == null || request.getPageNumber() < 1L) { request.setPageNumber(1L); @@ -374,6 +922,9 @@ public class DatacenterDatasetQueryServiceImpl implements DatacenterDatasetQuery if (request.getPageSize() == null || request.getPageSize() < 1L) { throw new BusinessException("pageSize 必须大于 0"); } + if (request.getPageSize() > 500L) { + throw new BusinessException("单页最多返回 500 行"); + } } private DatacenterTable resolveQueryTable(DatacenterTable table, DatasetRef datasetRef) { @@ -400,7 +951,7 @@ public class DatacenterDatasetQueryServiceImpl implements DatacenterDatasetQuery return queryTable; } - private void validateRequest(DatacenterTable table, DatacenterQueryRequest request, DatacenterSource source) { + private void validateRequest(DatacenterTable table, DatacenterQueryRequest request) { Map fieldMap = new LinkedHashMap<>(); for (DatacenterTableField field : table.getFields()) { fieldMap.put(field.getFieldName(), field); @@ -408,22 +959,25 @@ public class DatacenterDatasetQueryServiceImpl implements DatacenterDatasetQuery if (!CollectionUtils.isEmpty(request.getSelectedColumns())) { for (String column : request.getSelectedColumns()) { DatacenterTableField field = fieldMap.get(column); - if (field == null || !isEnabled(field.getQueryable())) { + if (field == null || !isQueryable(field)) { throw new BusinessException("字段不可查询: " + column); } } } else { request.setSelectedColumns( table.getFields().stream() - .filter(field -> isEnabled(field.getQueryable())) + .filter(this::isQueryable) .map(DatacenterTableField::getFieldName) .toList() ); } + if (CollectionUtils.isEmpty(request.getSelectedColumns())) { + throw new BusinessException("当前数据集没有可查询字段"); + } if (!CollectionUtils.isEmpty(request.getFilters())) { request.getFilters().forEach(filter -> { DatacenterTableField field = fieldMap.get(filter.getColumn()); - if (field == null || !isEnabled(field.getQueryable())) { + if (field == null || !isQueryable(field)) { throw new BusinessException("字段不可过滤: " + filter.getColumn()); } }); @@ -431,18 +985,13 @@ public class DatacenterDatasetQueryServiceImpl implements DatacenterDatasetQuery if (!CollectionUtils.isEmpty(request.getSorts())) { request.getSorts().forEach(sort -> { DatacenterTableField field = fieldMap.get(sort.getColumn()); - if (field == null || !isEnabled(field.getSortable())) { + if (field == null || !isQueryable(field) || !isEnabled(field.getSortable())) { throw new BusinessException("字段不可排序: " + sort.getColumn()); } }); } if (request.getWhere() != null && !request.getWhere().isBlank()) { - boolean allowLegacyWhere = "PROJECT_MYSQL".equals(source.getSourceType()) - || "MYSQL".equals(source.getSourceType()) - || "POSTGRESQL".equals(source.getSourceType()); - if (!allowLegacyWhere) { - throw new BusinessException("当前数据源仅支持结构化 DSL 查询"); - } + throw new BusinessException("不支持原始 WHERE 条件，请使用结构化筛选条件"); } } @@ -450,6 +999,13 @@ public class DatacenterDatasetQueryServiceImpl implements DatacenterDatasetQuery return value == null || value == 1; } + private boolean isQueryable(DatacenterTableField field) { + return field != null + && isEnabled(field.getQueryable()) + && DatacenterMetadataStatus.ACTIVE.name().equals(field.getMetadataStatus()) + && DatacenterSensitivityLevel.PUBLIC.name().equals(field.getSensitivityLevel()); + } + private String trimToNull(String value) { if (!StringUtils.hasText(value)) { return null; diff --git a/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/execution/service/impl/DatacenterDatasetWriteServiceImpl.java b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/execution/service/impl/DatacenterDatasetWriteServiceImpl.java index 3da8ba7c..cedc475a 100644 --- a/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/execution/service/impl/DatacenterDatasetWriteServiceImpl.java +++ b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/execution/service/impl/DatacenterDatasetWriteServiceImpl.java @@ -38,7 +38,7 @@ public class DatacenterDatasetWriteServiceImpl implements DatacenterDatasetWrite */ @Override public void saveRow(DatasetRef datasetRef, JSONObject data, LoginAccount account) { - DatacenterTable table = resolveTable(datasetRef); + DatacenterTable table = resolveTable(datasetRef, account); DatacenterSource source = registryService.getSourceRequired(table.getSourceId()); DatacenterConnector connector = connectorRegistry.getConnector(source.getSourceType()); connector.saveRow(source, table, data, account); @@ -52,7 +52,7 @@ public class DatacenterDatasetWriteServiceImpl implements DatacenterDatasetWrite if (rows == null || rows.isEmpty()) { return; } - DatacenterTable table = resolveTable(datasetRef); + DatacenterTable table = resolveTable(datasetRef, account); DatacenterSource source = registryService.getSourceRequired(table.getSourceId()); DatacenterConnector connector = connectorRegistry.getConnector(source.getSourceType()); connector.saveRows(source, table, rows, account, Math.max(1, batchSize)); @@ -70,7 +70,7 @@ public class DatacenterDatasetWriteServiceImpl implements DatacenterDatasetWrite if (rows == null || rows.isEmpty()) { return true; } - DatacenterTable table = resolveTable(datasetRef); + DatacenterTable table = resolveTable(datasetRef, account); DatacenterSource source = registryService.getSourceRequired(table.getSourceId()); DatacenterConnector connector = connectorRegistry.getConnector(source.getSourceType()); String payloadHash = sha256Rows(rows); @@ -95,17 +95,27 @@ public class DatacenterDatasetWriteServiceImpl implements DatacenterDatasetWrite */ @Override public void deleteRow(DatasetRef datasetRef, BigInteger id, LoginAccount account) { - DatacenterTable table = resolveTable(datasetRef); + DatacenterTable table = resolveTable(datasetRef, account); DatacenterSource source = registryService.getSourceRequired(table.getSourceId()); DatacenterConnector connector = connectorRegistry.getConnector(source.getSourceType()); connector.deleteRow(source, table, id, account); } - private DatacenterTable resolveTable(DatasetRef datasetRef) { + private DatacenterTable resolveTable(DatasetRef datasetRef, LoginAccount account) { if (datasetRef == null || datasetRef.getTableId() == null) { throw new BusinessException("缺少 tableId"); } - return registryService.getTableWithFields(datasetRef.getTableId()); + if (account == null || account.getTenantId() == null) { + throw new BusinessException("数据集写入缺少执行租户"); + } + DatacenterTable table = registryService.getTableWithFields(datasetRef.getTableId()); + if (!account.getTenantId().equals(table.getTenantId()) + || (datasetRef.getTenantId() != null + && !account.getTenantId().equals(datasetRef.getTenantId()))) { + throw new BusinessException("数据集不属于当前租户"); + } + datasetRef.setTenantId(account.getTenantId()); + return table; } /** diff --git a/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/federation/DatacenterFederationAdmissionController.java b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/federation/DatacenterFederationAdmissionController.java new file mode 100644 index 00000000..a1357c5f --- /dev/null +++ b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/federation/DatacenterFederationAdmissionController.java @@ -0,0 +1,308 @@ +package tech.easyflow.datacenter.federation; + +import com.easyagents.federation.sql.api.FederationSqlErrorCode; +import com.easyagents.federation.sql.api.FederationSqlException; +import com.easyagents.federation.sql.execute.FederationQueryAdmissionController; +import com.easyagents.federation.sql.execute.FederationQueryPermit; +import com.easyagents.federation.sql.execute.LocalFederationQueryAdmissionController; +import com.easyagents.federation.sql.execute.QueryId; +import com.easyagents.federation.sql.source.SourceId; +import java.time.Duration; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.function.BooleanSupplier; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; + +/** + * 对查询执行租户、数据源与节点三级本地并发准入。 + */ +@Component +public class DatacenterFederationAdmissionController + implements FederationQueryAdmissionController { + + private final LocalFederationQueryAdmissionController globalAdmission; + private final int maximumConcurrentQueriesPerTenant; + private final int maximumConcurrentQueriesPerSource; + private final Map + tenantAdmissions = new ConcurrentHashMap<>(); + private final Map + sourceAdmissions = new ConcurrentHashMap<>(); + private final Object lifecycleLock = new Object(); + private final AtomicBoolean closed = new AtomicBoolean(); + + /** + * 创建三级准入控制器。 + * + * @param maximumConcurrentQueries 节点查询上限 + * @param maximumConcurrentQueriesPerTenant 单租户查询上限 + * @param maximumConcurrentQueriesPerSource 单数据源查询上限 + */ + public DatacenterFederationAdmissionController( + @Value("${easyflow.datacenter.query-admission.maximum-concurrent-queries:64}") + int maximumConcurrentQueries, + @Value("${easyflow.datacenter.query-admission.maximum-concurrent-queries-per-tenant:20}") + int maximumConcurrentQueriesPerTenant, + @Value("${easyflow.datacenter.query-admission.maximum-concurrent-queries-per-source:10}") + int maximumConcurrentQueriesPerSource) { + if (maximumConcurrentQueries <= 0 + || maximumConcurrentQueriesPerTenant <= 0 + || maximumConcurrentQueriesPerSource <= 0) { + throw new IllegalArgumentException( + "query admission limits must be positive"); + } + this.globalAdmission = new LocalFederationQueryAdmissionController( + maximumConcurrentQueries); + this.maximumConcurrentQueriesPerTenant = + maximumConcurrentQueriesPerTenant; + this.maximumConcurrentQueriesPerSource = + maximumConcurrentQueriesPerSource; + } + + /** + * 获取三级查询许可。 + * + * @param sourceId 数据源标识 + * @param queryId 查询标识 + * @param timeout 最大等待时间 + * @return 查询许可 + */ + @Override + public FederationQueryPermit acquire( + SourceId sourceId, + QueryId queryId, + Duration timeout) { + return acquire(sourceId, queryId, timeout, () -> false); + } + + /** + * 按数据源、租户、节点顺序获取许可，避免热点连接池等待占满全局配额。 + * + * @param sourceId 数据源标识 + * @param queryId 查询标识 + * @param timeout 最大等待时间 + * @param cancellationRequested 查询取消状态 + * @return 查询许可 + */ + @Override + public FederationQueryPermit acquire( + SourceId sourceId, + QueryId queryId, + Duration timeout, + BooleanSupplier cancellationRequested) { + long started = System.nanoTime(); + String tenantKey = tenantKey(sourceId); + AdmissionEntry sourceEntry; + AdmissionEntry tenantEntry; + synchronized (lifecycleLock) { + ensureOpen(); + sourceEntry = retain(sourceAdmissions, sourceId, + maximumConcurrentQueriesPerSource); + tenantEntry = retain(tenantAdmissions, tenantKey, + maximumConcurrentQueriesPerTenant); + } + FederationQueryPermit sourcePermit = null; + FederationQueryPermit tenantPermit = null; + FederationQueryPermit globalPermit = null; + try { + sourcePermit = sourceEntry.controller.acquire( + sourceId, queryId, remaining(timeout, started), + cancellationRequested); + tenantPermit = tenantEntry.controller.acquire( + sourceId, queryId, remaining(timeout, started), + cancellationRequested); + globalPermit = globalAdmission.acquire( + sourceId, queryId, remaining(timeout, started), + cancellationRequested); + FederationQueryPermit acquiredSource = sourcePermit; + FederationQueryPermit acquiredTenant = tenantPermit; + FederationQueryPermit acquiredGlobal = globalPermit; + AtomicBoolean released = new AtomicBoolean(); + return () -> { + if (released.compareAndSet(false, true)) { + try { + acquiredGlobal.close(); + } finally { + try { + acquiredTenant.close(); + } finally { + try { + acquiredSource.close(); + } finally { + releaseEntries( + sourceId, sourceEntry, + tenantKey, tenantEntry); + } + } + } + } + }; + } catch (RuntimeException exception) { + closeQuietly(globalPermit); + closeQuietly(tenantPermit); + closeQuietly(sourcePermit); + releaseEntries(sourceId, sourceEntry, tenantKey, tenantEntry); + throw exception; + } + } + + /** + * 关闭全部准入控制器并唤醒等待线程。 + */ + @Override + public void close() { + synchronized (lifecycleLock) { + if (!closed.compareAndSet(false, true)) { + return; + } + globalAdmission.close(); + tenantAdmissions.values().forEach( + entry -> entry.controller.close()); + sourceAdmissions.values().forEach( + entry -> entry.controller.close()); + tenantAdmissions.clear(); + sourceAdmissions.clear(); + } + } + + private Duration remaining(Duration timeout, long started) { + if (timeout == null || timeout.isZero() || timeout.isNegative()) { + return Duration.ZERO; + } + long remaining = timeout.toNanos() - (System.nanoTime() - started); + return remaining <= 0L ? Duration.ZERO : Duration.ofNanos(remaining); + } + + private String tenantKey(SourceId sourceId) { + String value = sourceId == null ? "" : sourceId.value(); + int marker = value.lastIndexOf("-source-"); + return marker <= 0 ? value : value.substring(0, marker); + } + + /** + * 获取或创建分层准入项并登记一个活动查询或等待者。 + * + * @param admissions 准入项映射 + * @param key 租户或数据源键 + * @param maximumConcurrentQueries 并发上限 + * @param 键类型 + * @return 已登记引用的准入项 + */ + private AdmissionEntry retain( + Map admissions, + K key, + int maximumConcurrentQueries) { + AdmissionEntry entry = admissions.computeIfAbsent( + key, + ignored -> new AdmissionEntry( + new LocalFederationQueryAdmissionController( + maximumConcurrentQueries))); + entry.references++; + return entry; + } + + /** + * 同步释放数据源与租户准入项引用。 + * + * @param sourceId 数据源标识 + * @param sourceEntry 数据源准入项 + * @param tenantKey 租户键 + * @param tenantEntry 租户准入项 + */ + private void releaseEntries( + SourceId sourceId, + AdmissionEntry sourceEntry, + String tenantKey, + AdmissionEntry tenantEntry) { + synchronized (lifecycleLock) { + release(sourceAdmissions, sourceId, sourceEntry); + release(tenantAdmissions, tenantKey, tenantEntry); + } + } + + /** + * 释放一个准入项引用，并在无查询或等待者时移除并关闭该项。 + * + * @param admissions 准入项映射 + * @param key 租户或数据源键 + * @param entry 待释放准入项 + * @param 键类型 + */ + private void release( + Map admissions, + K key, + AdmissionEntry entry) { + entry.references--; + if (entry.references < 0) { + throw new IllegalStateException( + "query admission reference count became negative"); + } + if (entry.references == 0 && admissions.remove(key, entry)) { + entry.controller.close(); + } + } + + /** + * 返回当前仍有查询或等待者的数据源准入项数量。 + * + * @return 活动数据源准入项数量 + */ + int trackedSourceAdmissionCount() { + return sourceAdmissions.size(); + } + + /** + * 返回当前仍有查询或等待者的租户准入项数量。 + * + * @return 活动租户准入项数量 + */ + int trackedTenantAdmissionCount() { + return tenantAdmissions.size(); + } + + /** + * 返回指定数据源当前活动查询与等待者的引用总数。 + * + * @param sourceId 数据源标识 + * @return 当前引用数 + */ + int trackedSourceReferenceCount(SourceId sourceId) { + synchronized (lifecycleLock) { + AdmissionEntry entry = sourceAdmissions.get(sourceId); + return entry == null ? 0 : entry.references; + } + } + + private void ensureOpen() { + if (closed.get()) { + throw new FederationSqlException( + FederationSqlErrorCode.ENGINE_CLOSED, + "datacenter query admission controller is closed"); + } + } + + private void closeQuietly(FederationQueryPermit permit) { + if (permit != null) { + permit.close(); + } + } + + /** + * 共享公平信号量及其活动查询、等待者引用计数。 + */ + private static final class AdmissionEntry { + + private final LocalFederationQueryAdmissionController controller; + private int references; + + /** + * 创建准入项。 + * + * @param controller 本地公平准入控制器 + */ + private AdmissionEntry(LocalFederationQueryAdmissionController controller) { + this.controller = controller; + } + } +} diff --git a/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/federation/DatacenterFederationChangeNotifier.java b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/federation/DatacenterFederationChangeNotifier.java new file mode 100644 index 00000000..ed0c5971 --- /dev/null +++ b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/federation/DatacenterFederationChangeNotifier.java @@ -0,0 +1,70 @@ +package tech.easyflow.datacenter.federation; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.ObjectProvider; +import org.springframework.data.redis.core.StringRedisTemplate; +import org.springframework.stereotype.Component; +import org.springframework.transaction.support.TransactionSynchronization; +import org.springframework.transaction.support.TransactionSynchronizationManager; +import tech.easyflow.datacenter.meta.entity.DatacenterSource; + +/** + * 发布不含凭据的 Federation Definition 变更提示。 + */ +@Component +public class DatacenterFederationChangeNotifier { + + /** Redis Pub/Sub 频道。 */ + public static final String CHANNEL = "easyflow:datacenter:federation-source-changed"; + + private static final Logger log = LoggerFactory.getLogger(DatacenterFederationChangeNotifier.class); + + private final ObjectProvider redisTemplateProvider; + private final DatacenterFederationDefinitionFactory definitionFactory; + + /** + * 创建通知器。 + * + * @param redisTemplateProvider 可选 Redis 模板 + * @param definitionFactory Definition 工厂 + */ + public DatacenterFederationChangeNotifier( + ObjectProvider redisTemplateProvider, + DatacenterFederationDefinitionFactory definitionFactory) { + this.redisTemplateProvider = redisTemplateProvider; + this.definitionFactory = definitionFactory; + } + + /** + * 在当前事务提交后发布 SourceId；无 Redis 时不影响数据库权威状态。 + * + * @param source 已提交或即将提交的数据源 + */ + public void publishAfterCommit(DatacenterSource source) { + String sourceId = definitionFactory.sourceId(source).value(); + if (TransactionSynchronizationManager.isSynchronizationActive()) { + TransactionSynchronizationManager.registerSynchronization(new TransactionSynchronization() { + @Override + public void afterCommit() { + publish(sourceId); + } + }); + return; + } + publish(sourceId); + } + + private void publish(String sourceId) { + StringRedisTemplate redisTemplate = redisTemplateProvider.getIfAvailable(); + if (redisTemplate == null) { + return; + } + try { + redisTemplate.convertAndSend(CHANNEL, sourceId); + } catch (RuntimeException exception) { + // Redis 仅缩短传播时间；数据库按需读取继续保证 minimumRevision 正确性。 + log.warn("Failed to publish datacenter source change hint for {}", sourceId, exception); + } + } +} diff --git a/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/federation/DatacenterFederationDataSourceFactory.java b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/federation/DatacenterFederationDataSourceFactory.java new file mode 100644 index 00000000..e093092d --- /dev/null +++ b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/federation/DatacenterFederationDataSourceFactory.java @@ -0,0 +1,64 @@ +package tech.easyflow.datacenter.federation; + +import com.zaxxer.hikari.HikariConfig; +import com.zaxxer.hikari.HikariDataSource; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; +import tech.easyflow.datacenter.meta.entity.DatacenterSource; +import tech.easyflow.datacenter.security.DatacenterCredentialCipher; + +/** + * 创建由 Federation Runtime 独占的只读 HikariCP 连接池。 + */ +@Component +public class DatacenterFederationDataSourceFactory { + + private final DatacenterCredentialCipher credentialCipher; + private final int maximumPoolSize; + + /** + * 创建数据源工厂。 + * + * @param credentialCipher 凭据解密器 + * @param maximumPoolSize 单数据源连接池上限，与单源查询准入共用配置 + */ + public DatacenterFederationDataSourceFactory( + DatacenterCredentialCipher credentialCipher, + @Value("${easyflow.datacenter.connection-pool.maximum-size:10}") + int maximumPoolSize) { + if (maximumPoolSize <= 0) { + throw new IllegalArgumentException( + "datacenter connection pool maximum size must be positive"); + } + this.credentialCipher = credentialCipher; + this.maximumPoolSize = maximumPoolSize; + } + + /** + * 根据持久化数据源快照创建只读池。 + * + * @param source 数据源快照 + * @return 未共享的 HikariCP 池 + */ + public HikariDataSource create(DatacenterSource source) { + HikariConfig config = new HikariConfig(); + config.setPoolName("dc-fed-" + source.getId() + "-r" + source.getDefinitionRevision()); + config.setJdbcUrl(source.getJdbcUrl()); + config.setUsername(source.getUsername()); + config.setPassword(credentialCipher.decrypt(source.getCredentialCipher())); + if (source.getDriverClassName() != null && !source.getDriverClassName().isBlank()) { + config.setDriverClassName(source.getDriverClassName()); + } + config.setReadOnly(true); + config.setAutoCommit(true); + config.setMaximumPoolSize(maximumPoolSize); + config.setMinimumIdle(Integer.getInteger("easyflow.datacenter.pool.minimum-idle", 0)); + config.setConnectionTimeout(Long.getLong("easyflow.datacenter.pool.connection-timeout-ms", 5_000L)); + config.setValidationTimeout(Long.getLong("easyflow.datacenter.pool.validation-timeout-ms", 3_000L)); + config.setIdleTimeout(Long.getLong("easyflow.datacenter.pool.idle-timeout-ms", 600_000L)); + config.setMaxLifetime(Long.getLong("easyflow.datacenter.pool.max-lifetime-ms", 1_800_000L)); + config.setInitializationFailTimeout( + Long.getLong("easyflow.datacenter.pool.initialization-timeout-ms", 5_000L)); + return new HikariDataSource(config); + } +} diff --git a/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/federation/DatacenterFederationDataSourceResolver.java b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/federation/DatacenterFederationDataSourceResolver.java new file mode 100644 index 00000000..a40e1d43 --- /dev/null +++ b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/federation/DatacenterFederationDataSourceResolver.java @@ -0,0 +1,140 @@ +package tech.easyflow.datacenter.federation; + +import com.easyagents.federation.sql.source.FederationDataSourceHandle; +import com.easyagents.federation.sql.source.FederationDataSourceHandles; +import com.easyagents.federation.sql.source.FederationDataSourceResolver; +import com.easyagents.federation.sql.source.FederationSourceDefinition; +import com.easyagents.federation.sql.source.RuntimeFingerprint; +import com.zaxxer.hikari.HikariDataSource; +import java.sql.Connection; +import java.sql.DatabaseMetaData; +import java.sql.SQLException; +import java.util.Objects; +import java.util.function.Supplier; +import org.springframework.stereotype.Component; +import tech.easyflow.datacenter.mapper.DatacenterSourceMapper; +import tech.easyflow.datacenter.meta.entity.DatacenterSource; +import tech.easyflow.datacenter.meta.enums.DatacenterSourceStatus; + +/** + * 从 EasyFlow 权威配置解析节点本地 Federation DataSource Handle。 + */ +@Component +public class DatacenterFederationDataSourceResolver implements FederationDataSourceResolver { + + private final DatacenterSourceMapper sourceMapper; + private final DatacenterFederationDefinitionFactory definitionFactory; + private final DatacenterFederationDataSourceFactory dataSourceFactory; + private final ThreadLocal candidateSource = new ThreadLocal<>(); + + /** + * 创建 Resolver。 + * + * @param sourceMapper 数据源 Mapper + * @param definitionFactory Definition 工厂 + * @param dataSourceFactory 连接池工厂 + */ + public DatacenterFederationDataSourceResolver( + DatacenterSourceMapper sourceMapper, + DatacenterFederationDefinitionFactory definitionFactory, + DatacenterFederationDataSourceFactory dataSourceFactory) { + this.sourceMapper = sourceMapper; + this.definitionFactory = definitionFactory; + this.dataSourceFactory = dataSourceFactory; + } + + /** + * 创建由 Runtime 独占并在 revision 淘汰后关闭的 Handle。 + * + * @param definition 无凭据 Definition + * @return 独占 Handle + */ + @Override + public FederationDataSourceHandle resolve(FederationSourceDefinition definition) { + CandidateResolution candidate = candidateSource.get(); + DatacenterSource source = candidate != null + && Objects.equals(candidate.source().getId(), definitionFactory.sourceRecordId(definition)) + ? candidate.source() : sourceMapper.selectOneById(definitionFactory.sourceRecordId(definition)); + if (source == null) { + throw new IllegalStateException("data source no longer exists"); + } + DatacenterSourceStatus status = DatacenterSourceStatus.fromCode(source.getStatus()); + boolean draftProbe = status == DatacenterSourceStatus.DRAFT; + if (!draftProbe + && status != DatacenterSourceStatus.READY + && status != DatacenterSourceStatus.DEGRADED) { + throw new IllegalStateException("data source cannot be resolved in its current state"); + } + if (!definition.sourceId().equals(definitionFactory.sourceId(source))) { + throw new IllegalStateException("data source tenant identity does not match definition"); + } + long revision = source.getDefinitionRevision() == null ? 0L : source.getDefinitionRevision(); + if (revision != definition.revision()) { + throw new IllegalStateException("data source revision changed while runtime was resolving"); + } + FederationSourceDefinition currentDefinition = candidate != null + ? candidate.definition() + : definitionFactory.create(source); + if (!currentDefinition.checksum().equals(definition.checksum())) { + throw new IllegalStateException("data source definition checksum mismatch"); + } + // 草稿只通过 probe 创建短生命周期连接池，尚未发布持久化 checksum； + // READY/DEGRADED Runtime 必须同时匹配已发布 checksum。 + if (!draftProbe && (source.getDefinitionChecksum() == null + || !source.getDefinitionChecksum().equals(definition.checksum()))) { + throw new IllegalStateException("published data source definition checksum mismatch"); + } + HikariDataSource pool = dataSourceFactory.create(source); + try (Connection connection = pool.getConnection()) { + DatabaseMetaData metadata = connection.getMetaData(); + RuntimeFingerprint fingerprint = new RuntimeFingerprint( + metadata.getDatabaseProductName(), + metadata.getDatabaseProductVersion(), + metadata.getDriverName(), + metadata.getDriverVersion(), + "easyflow-xl01-wp1"); + return FederationDataSourceHandles.owned(pool, fingerprint, pool::close); + } catch (SQLException | RuntimeException exception) { + try { + pool.close(); + } catch (RuntimeException closeException) { + exception.addSuppressed(closeException); + } + throw new IllegalStateException("failed to initialize data source runtime", exception); + } + } + + /** + * 在当前线程内使用未发布的候选配置完成一次 XL15 probe。 + * + *

候选配置不会进入共享状态、Redis 或持久化记录，并在操作结束后立即移除。

+ * + * @param source 已完成凭据加密和连接归一化的候选配置 + * @param action 需要解析候选配置的同步操作 + * @param 操作结果类型 + * @return 操作结果 + * @throws IllegalStateException 当前线程已经存在候选配置时抛出 + */ + public T withCandidate( + DatacenterSource source, + FederationSourceDefinition definition, + Supplier action) { + Objects.requireNonNull(source, "source"); + Objects.requireNonNull(definition, "definition"); + Objects.requireNonNull(action, "action"); + if (candidateSource.get() != null) { + throw new IllegalStateException("nested data source candidate resolution is not supported"); + } + candidateSource.set(new CandidateResolution(source, definition)); + try { + return action.get(); + } finally { + candidateSource.remove(); + } + } + + private record CandidateResolution( + DatacenterSource source, + FederationSourceDefinition definition) { + } +} diff --git a/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/federation/DatacenterFederationDefinitionFactory.java b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/federation/DatacenterFederationDefinitionFactory.java new file mode 100644 index 00000000..1f788938 --- /dev/null +++ b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/federation/DatacenterFederationDefinitionFactory.java @@ -0,0 +1,208 @@ +package tech.easyflow.datacenter.federation; + +import com.easyagents.federation.sql.adapter.jdbc.JdbcFederationSqlAdapterProvider; +import com.easyagents.federation.sql.adapter.jdbc.JdbcSchemaDefinition; +import com.easyagents.federation.sql.source.FederationSourceDefinition; +import com.easyagents.federation.sql.source.SourceId; +import com.mybatisflex.core.query.QueryWrapper; +import java.math.BigInteger; +import java.util.ArrayList; +import java.util.Collection; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; +import org.springframework.stereotype.Component; +import tech.easyflow.datacenter.mapper.DatacenterCatalogMapper; +import tech.easyflow.datacenter.meta.entity.DatacenterCatalog; +import tech.easyflow.datacenter.meta.entity.DatacenterSource; +import tech.easyflow.datacenter.meta.enums.DatacenterMetadataStatus; +import tech.easyflow.datacenter.meta.enums.DatacenterSourceType; + +/** + * 将 EasyFlow 持久化数据源快照转换为无凭据 Federation Definition。 + */ +@Component +public class DatacenterFederationDefinitionFactory { + + /** Definition 中用于解析业务数据源主键的选项。 */ + public static final String SOURCE_RECORD_ID_OPTION = "sourceRecordId"; + + private final DatacenterCatalogMapper catalogMapper; + + /** + * 创建 Definition 工厂。 + * + * @param catalogMapper 命名空间 Mapper + */ + public DatacenterFederationDefinitionFactory(DatacenterCatalogMapper catalogMapper) { + this.catalogMapper = catalogMapper; + } + + /** + * 创建用于探测或执行的 Definition。 + * + * @param source 数据源当前快照 + * @return 无凭据 Definition + * @throws IllegalArgumentException 数据源不是当前 JDBC 支持类型 + */ + public FederationSourceDefinition create(DatacenterSource source) { + return create(source, loadCatalogs(source)); + } + + /** + * 为尚未发布的连接候选创建只包含默认命名空间的 Definition。 + * + * @param source 候选数据源 + * @return 不读取旧持久化 Catalog 的候选 Definition + */ + public FederationSourceDefinition createCandidate(DatacenterSource source) { + return create(source, List.of()); + } + + /** + * 为即将发布的单命名空间配置创建与持久化结果一致的 Definition。 + * + * @param source 候选数据源 + * @param catalogName 即将纳管的物理 Catalog 或 Schema + * @return 不读取旧 Catalog 的候选 Definition + */ + public FederationSourceDefinition createCandidate( + DatacenterSource source, + String catalogName) { + if (catalogName == null || catalogName.isBlank()) { + throw new IllegalArgumentException("candidate catalog name is required"); + } + String normalized = catalogName.trim(); + DatacenterCatalog catalog = new DatacenterCatalog(); + catalog.setLogicalSchemaName(normalized); + if (DatacenterSourceType.MYSQL.name().equals(source.getSourceType())) { + catalog.setPhysicalCatalogName(normalized); + } else { + catalog.setPhysicalCatalogName(source.getDatabaseName()); + catalog.setPhysicalSchemaName(normalized); + } + return create(source, List.of(catalog)); + } + + /** + * 批量创建 Definition，一次加载全部活动 Catalog，供周期 reconcile 避免 N+1。 + * + * @param sources 数据源快照 + * @return 按数据源记录 ID 索引的 Definition + */ + public Map createBatch( + Collection sources) { + List sourceList = sources == null + ? List.of() : sources.stream().filter(java.util.Objects::nonNull).toList(); + if (sourceList.isEmpty()) { + return Map.of(); + } + List sourceIds = sourceList.stream() + .map(DatacenterSource::getId) + .filter(java.util.Objects::nonNull) + .toList(); + Map> catalogsBySource = sourceIds.isEmpty() + ? Map.of() + : catalogMapper.selectListByQuery(QueryWrapper.create() + .in(DatacenterCatalog::getSourceId, sourceIds) + .eq(DatacenterCatalog::getMetadataStatus, DatacenterMetadataStatus.ACTIVE.name()) + .orderBy("created asc")) + .stream() + .collect(Collectors.groupingBy(DatacenterCatalog::getSourceId)); + Map definitions = new LinkedHashMap<>(); + for (DatacenterSource source : sourceList) { + definitions.put(source.getId(), create( + source, catalogsBySource.getOrDefault(source.getId(), List.of()))); + } + return Map.copyOf(definitions); + } + + private FederationSourceDefinition create( + DatacenterSource source, + List catalogs) { + requireExternalJdbc(source); + long revision = Math.max(1L, source.getDefinitionRevision() == null + ? 0L : source.getDefinitionRevision()); + List schemas = schemas(source, catalogs); + Map options = new LinkedHashMap<>(); + if (source.getAdapterOptionsJson() != null) { + options.putAll(source.getAdapterOptionsJson()); + } + options.put(SOURCE_RECORD_ID_OPTION, source.getId().toString()); + return new FederationSourceDefinition( + sourceId(source), + revision, + source.getAdapterId() == null || source.getAdapterId().isBlank() + ? JdbcFederationSqlAdapterProvider.ADAPTER_ID + : source.getAdapterId(), + new ArrayList<>(schemas), + options); + } + + /** + * 派生租户隔离且不随显示名称变化的 SourceId。 + * + * @param source 数据源 + * @return Federation SourceId + */ + public SourceId sourceId(DatacenterSource source) { + BigInteger tenantId = source.getTenantId() == null ? BigInteger.ZERO : source.getTenantId(); + return new SourceId("tenant-" + tenantId + "-source-" + source.getId()); + } + + /** + * 从 Definition 读取 EasyFlow 数据源主键。 + * + * @param definition Definition + * @return 数据源主键 + */ + public BigInteger sourceRecordId(FederationSourceDefinition definition) { + String value = definition.adapterOptions().get(SOURCE_RECORD_ID_OPTION); + if (value == null || value.isBlank()) { + throw new IllegalArgumentException("sourceRecordId is missing from Federation Definition"); + } + return new BigInteger(value); + } + + private List loadCatalogs(DatacenterSource source) { + QueryWrapper query = QueryWrapper.create() + .eq(DatacenterCatalog::getSourceId, source.getId()) + .eq(DatacenterCatalog::getMetadataStatus, DatacenterMetadataStatus.ACTIVE.name()) + .orderBy("created asc"); + return catalogMapper.selectListByQuery(query); + } + + private List schemas( + DatacenterSource source, + List catalogs) { + if (catalogs.isEmpty()) { + return List.of(defaultSchema(source)); + } + return catalogs.stream() + .map(catalog -> new JdbcSchemaDefinition( + catalog.getLogicalSchemaName(), + catalog.getPhysicalCatalogName(), + catalog.getPhysicalSchemaName())) + .toList(); + } + + private JdbcSchemaDefinition defaultSchema(DatacenterSource source) { + if (DatacenterSourceType.MYSQL.name().equals(source.getSourceType())) { + return new JdbcSchemaDefinition("MAIN", source.getDatabaseName(), null); + } + String schema = source.getSchemaName() == null || source.getSchemaName().isBlank() + ? "public" : source.getSchemaName(); + return new JdbcSchemaDefinition("MAIN", source.getDatabaseName(), schema); + } + + private void requireExternalJdbc(DatacenterSource source) { + if (source == null || source.getId() == null) { + throw new IllegalArgumentException("persisted source is required"); + } + if (!DatacenterSourceType.MYSQL.name().equals(source.getSourceType()) + && !DatacenterSourceType.POSTGRESQL.name().equals(source.getSourceType())) { + throw new IllegalArgumentException("Federation JDBC currently supports MySQL and PostgreSQL sources"); + } + } +} diff --git a/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/federation/DatacenterFederationQueryCancellationService.java b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/federation/DatacenterFederationQueryCancellationService.java new file mode 100644 index 00000000..9bb9824e --- /dev/null +++ b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/federation/DatacenterFederationQueryCancellationService.java @@ -0,0 +1,292 @@ +package tech.easyflow.datacenter.federation; + +import com.easyagents.federation.sql.execute.QueryId; +import java.math.BigInteger; +import java.time.Duration; +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicBoolean; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.ObjectProvider; +import org.springframework.data.redis.core.StringRedisTemplate; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Component; +import tech.easyflow.common.entity.LoginAccount; +import tech.easyflow.common.web.exceptions.BusinessException; + +/** + * 管理查询从编译到游标关闭的取消状态，并通过 Redis 将取消提示广播到执行节点。 + */ +@Component +public class DatacenterFederationQueryCancellationService { + + /** 查询取消 Redis Pub/Sub 频道。 */ + public static final String CHANNEL = "easyflow:datacenter:federation-query-cancel"; + private static final String CANCEL_KEY_PREFIX = + "easyflow:datacenter:federation-query-cancelled:"; + private static final Duration CANCEL_TTL = Duration.ofMinutes(10); + + private static final Logger log = LoggerFactory.getLogger( + DatacenterFederationQueryCancellationService.class); + + private final DatacenterFederationRuntime runtime; + private final ObjectProvider redisTemplateProvider; + private final ConcurrentHashMap activeQueries = new ConcurrentHashMap<>(); + + /** + * 创建查询取消服务。 + * + * @param runtime 节点本地 Federation Runtime + * @param redisTemplateProvider 可选 Redis 模板 + */ + public DatacenterFederationQueryCancellationService( + DatacenterFederationRuntime runtime, + ObjectProvider redisTemplateProvider) { + this.runtime = runtime; + this.redisTemplateProvider = redisTemplateProvider; + } + + /** + * 校验或创建查询标识。 + * + * @param requestedQueryId 调用方预生成的 UUID；为空时由服务端生成 + * @return 已校验的查询标识 + * @throws BusinessException 标识格式不合法 + */ + public QueryId resolveQueryId(String requestedQueryId) { + if (requestedQueryId == null || requestedQueryId.isBlank()) { + return QueryId.create(); + } + try { + return new QueryId(UUID.fromString(requestedQueryId.trim()).toString()); + } catch (IllegalArgumentException exception) { + throw new BusinessException("queryId 必须是 UUID"); + } + } + + /** + * 在编译前登记查询，防止重复 QueryId 混淆取消目标。 + * + * @param queryId 查询标识 + * @param account 执行账号 + * @return 必须关闭的登记句柄 + * @throws BusinessException QueryId 正在使用 + */ + public QueryRegistration register(QueryId queryId, LoginAccount account) { + ActiveQuery activeQuery = new ActiveQuery(tenantId(account)); + if (activeQueries.putIfAbsent(queryId.value(), activeQuery) != null) { + throw new BusinessException(409, 14123, "queryId 正在使用", null); + } + if (isCancellationPersisted(activeQuery.tenantId, queryId)) { + try { + cancelLocal(activeQuery.tenantId, queryId); + } catch (RuntimeException exception) { + activeQueries.remove(queryId.value(), activeQuery); + throw exception; + } + } + return new QueryRegistration(queryId, activeQuery); + } + + /** + * 向所有节点广播取消提示，并立即取消当前节点已登记的 Statement。 + * + * @param requestedQueryId 查询标识 + * @param account 当前账号 + * @return 当前节点已找到查询或 Redis 已接收广播 + */ + public boolean cancel(String requestedQueryId, LoginAccount account) { + if (requestedQueryId == null || requestedQueryId.isBlank()) { + throw new BusinessException("queryId 不能为空"); + } + QueryId queryId = resolveQueryId(requestedQueryId); + BigInteger tenantId = tenantId(account); + boolean distributedCancellationPersisted = persistAndPublishCancellation( + tenantId, queryId); + boolean localCancelled; + try { + localCancelled = cancelLocal(tenantId, queryId); + } catch (RuntimeException exception) { + log.error("Failed to cancel local datacenter query for tenant {} query {}", + tenantId, queryId.value(), exception); + if (!distributedCancellationPersisted) { + throw exception; + } + localCancelled = false; + } + return localCancelled || distributedCancellationPersisted; + } + + /** + * 先持久化取消终态，再尽力发布低延迟提示。 + * + * @param tenantId 租户 ID + * @param queryId 查询标识 + * @return Redis 是否已可靠保存取消终态 + */ + private boolean persistAndPublishCancellation( + BigInteger tenantId, + QueryId queryId) { + StringRedisTemplate redisTemplate = redisTemplateProvider.getIfAvailable(); + if (redisTemplate == null) { + return false; + } + try { + redisTemplate.opsForValue().set( + cancellationKey(tenantId, queryId), "1", CANCEL_TTL); + } catch (RuntimeException exception) { + log.warn("Failed to persist datacenter query cancellation for tenant {} query {}", + tenantId, queryId.value(), exception); + return false; + } + try { + redisTemplate.convertAndSend( + CHANNEL, tenantId + "|" + queryId.value()); + } catch (RuntimeException exception) { + // Pub/Sub 仅缩短生效延迟；持久取消终态已由周期核对保证交付。 + log.warn("Failed to publish datacenter query cancellation hint for tenant {} query {}", + tenantId, queryId.value(), exception); + } + return true; + } + + /** + * 处理 Redis 取消提示；格式无效时安全忽略。 + * + * @param payload 租户与 QueryId 组合文本 + */ + public void acceptCancellationHint(String payload) { + if (payload == null) { + return; + } + int separator = payload.indexOf('|'); + if (separator <= 0 || separator == payload.length() - 1) { + return; + } + try { + BigInteger tenantId = new BigInteger(payload.substring(0, separator)); + QueryId queryId = resolveQueryId(payload.substring(separator + 1)); + cancelLocal(tenantId, queryId); + } catch (RuntimeException exception) { + log.warn("Ignored invalid datacenter query cancellation hint", exception); + } + } + + /** + * 在编译或取数边界检查查询是否已取消。 + * + * @param queryId 查询标识 + * @throws BusinessException 查询已取消 + */ + public void throwIfCancelled(QueryId queryId) { + ActiveQuery activeQuery = activeQueries.get(queryId.value()); + if (activeQuery != null && activeQuery.cancelled.get()) { + throw new BusinessException(409, 14118, "查询已取消", null); + } + } + + /** + * 返回查询当前是否已收到取消请求。 + * + * @param queryId 查询标识 + * @return 是否取消 + */ + public boolean isCancelled(QueryId queryId) { + ActiveQuery activeQuery = activeQueries.get(queryId.value()); + return activeQuery != null && activeQuery.cancelled.get(); + } + + /** + * 周期核对 Redis 持久取消标记，覆盖 Pub/Sub 提示丢失或节点短暂断连。 + */ + @Scheduled(fixedDelayString = + "${easyflow.datacenter.query-cancel-reconcile-ms:1000}") + public void reconcilePersistedCancellations() { + activeQueries.forEach((queryIdValue, activeQuery) -> { + if (activeQuery.localCancellationApplied.get()) { + return; + } + QueryId queryId = new QueryId(queryIdValue); + if (activeQuery.cancelled.get() + || isCancellationPersisted(activeQuery.tenantId, queryId)) { + try { + cancelLocal(activeQuery.tenantId, queryId); + } catch (RuntimeException exception) { + log.error("Failed to reconcile datacenter query cancellation for tenant {} query {}", + activeQuery.tenantId, queryId.value(), exception); + } + } + }); + } + + private boolean cancelLocal(BigInteger tenantId, QueryId queryId) { + ActiveQuery activeQuery = activeQueries.get(queryId.value()); + if (activeQuery == null || !activeQuery.tenantId.equals(tenantId)) { + return false; + } + activeQuery.cancelled.set(true); + runtime.engine().cancel(queryId); + activeQuery.localCancellationApplied.set(true); + return true; + } + + private boolean isCancellationPersisted(BigInteger tenantId, QueryId queryId) { + StringRedisTemplate redisTemplate = redisTemplateProvider.getIfAvailable(); + if (redisTemplate == null) { + return false; + } + try { + return Boolean.TRUE.equals(redisTemplate.hasKey( + cancellationKey(tenantId, queryId))); + } catch (RuntimeException exception) { + log.warn("Failed to read datacenter query cancellation for tenant {} query {}", + tenantId, queryId.value(), exception); + return false; + } + } + + private String cancellationKey(BigInteger tenantId, QueryId queryId) { + return CANCEL_KEY_PREFIX + tenantId + ":" + queryId.value(); + } + + private BigInteger tenantId(LoginAccount account) { + return account == null || account.getTenantId() == null + ? BigInteger.ZERO : account.getTenantId(); + } + + private record ActiveQuery( + BigInteger tenantId, + AtomicBoolean cancelled, + AtomicBoolean localCancellationApplied) { + + private ActiveQuery(BigInteger tenantId) { + this(tenantId, new AtomicBoolean(), new AtomicBoolean()); + } + } + + /** + * 查询活动登记句柄。 + */ + public final class QueryRegistration implements AutoCloseable { + + private final QueryId queryId; + private final ActiveQuery activeQuery; + private final AtomicBoolean closed = new AtomicBoolean(); + + private QueryRegistration(QueryId queryId, ActiveQuery activeQuery) { + this.queryId = queryId; + this.activeQuery = activeQuery; + } + + /** + * 仅移除当前登记实例，避免误删随后复用的同名查询。 + */ + @Override + public void close() { + if (closed.compareAndSet(false, true)) { + activeQueries.remove(queryId.value(), activeQuery); + } + } + } +} diff --git a/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/federation/DatacenterFederationQueryService.java b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/federation/DatacenterFederationQueryService.java new file mode 100644 index 00000000..9734b553 --- /dev/null +++ b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/federation/DatacenterFederationQueryService.java @@ -0,0 +1,731 @@ +package tech.easyflow.datacenter.federation; + +import com.easyagents.federation.sql.api.FederationSqlException; +import com.easyagents.federation.sql.api.SqlExecutionContext; +import com.easyagents.federation.sql.compile.FederationSqlPlan; +import com.easyagents.federation.sql.compile.SqlCompileRequest; +import com.easyagents.federation.sql.execute.FederationColumn; +import com.easyagents.federation.sql.execute.FederationResultCursor; +import com.easyagents.federation.sql.execute.QueryId; +import com.easyagents.federation.sql.execute.SqlExecutionOptions; +import com.easyagents.federation.sql.execute.SqlParameter; +import com.easyagents.federation.sql.federation.FederationQueryScopeDefinition; +import com.easyagents.federation.sql.source.SourceId; +import java.io.IOException; +import java.io.InputStream; +import java.io.Reader; +import java.math.BigInteger; +import java.sql.Blob; +import java.sql.Clob; +import java.sql.SQLXML; +import java.sql.SQLException; +import java.sql.Struct; +import java.sql.Types; +import java.time.Duration; +import java.util.ArrayList; +import java.util.Base64; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.function.Consumer; +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.RelVisitor; +import org.apache.calcite.rel.core.TableScan; +import org.apache.calcite.sql.SqlIdentifier; +import org.apache.calcite.sql.util.SqlBasicVisitor; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Service; +import tech.easyflow.common.entity.LoginAccount; +import tech.easyflow.common.web.exceptions.BusinessException; +import tech.easyflow.datacenter.audit.DatacenterQueryAudit; +import tech.easyflow.datacenter.audit.DatacenterQueryAuditService; +import tech.easyflow.datacenter.execution.model.DatacenterSqlColumnView; +import tech.easyflow.datacenter.execution.model.DatacenterSqlConsoleResult; +import tech.easyflow.datacenter.meta.entity.DatacenterSource; +import tech.easyflow.datacenter.meta.enums.DatacenterSourceStatus; + +/** + * 通过 Federation SQL 完成有界只读查询、资源关闭和统一审计。 + */ +@Service +public class DatacenterFederationQueryService { + + private static final Logger log = LoggerFactory.getLogger(DatacenterFederationQueryService.class); + private static final int DEFAULT_MAX_ROWS = 200; + private static final int MAX_ALLOWED_ROWS = 1_000; + private static final long MAX_RESULT_BYTES = 4L * 1024L * 1024L; + private static final int MAX_TEXT_CELL_CHARS = 65_536; + + private final DatacenterFederationRuntime runtime; + private final DatacenterQueryAuditService auditService; + private final DatacenterFederationQueryCancellationService cancellationService; + + /** + * 创建查询服务。 + * + * @param runtime Federation Runtime + * @param auditService 查询审计服务 + * @param cancellationService 跨节点查询取消服务 + */ + public DatacenterFederationQueryService( + DatacenterFederationRuntime runtime, + DatacenterQueryAuditService auditService, + DatacenterFederationQueryCancellationService cancellationService) { + this.runtime = runtime; + this.auditService = auditService; + this.cancellationService = cancellationService; + } + + /** + * 执行一条单数据源只读 SQL。 + * + * @param source 已激活数据源 + * @param sql 只读 SQL + * @param parameters 显式 JDBC 参数 + * @param requestedMaxRows 请求行数上限 + * @param account 执行账号 + * @param callerType 调用方类型 + * @param callerId 调用方标识 + * @return 有界查询结果 + */ + public DatacenterSqlConsoleResult execute( + DatacenterSource source, + String sql, + List parameters, + Integer requestedMaxRows, + LoginAccount account, + String callerType, + String callerId) { + return execute(source, sql, parameters, requestedMaxRows, account, + callerType, callerId, null); + } + + /** + * 使用调用方预生成的 QueryId 执行只读 SQL，以支持请求完成前取消。 + * + * @param source 已激活数据源 + * @param sql 只读 SQL + * @param parameters 显式 JDBC 参数 + * @param requestedMaxRows 请求行数上限 + * @param account 执行账号 + * @param callerType 调用方类型 + * @param callerId 调用方标识 + * @param requestedQueryId 调用方预生成的查询 UUID；为空时服务端生成 + * @return 有界查询结果 + */ + public DatacenterSqlConsoleResult execute( + DatacenterSource source, + String sql, + List parameters, + Integer requestedMaxRows, + LoginAccount account, + String callerType, + String callerId, + String requestedQueryId) { + requireExecutable(source, sql); + int maxRows = normalizeMaxRows(requestedMaxRows); + List safeParameters = List.copyOf(parameters == null ? List.of() : parameters); + QueryId queryId = cancellationService.resolveQueryId(requestedQueryId); + DatacenterFederationQueryCancellationService.QueryRegistration registration = + cancellationService.register(queryId, account); + long startedNanos = System.nanoTime(); + DatacenterQueryAudit audit; + try { + audit = auditService.start( + queryId.value(), source, sql, maskedParameterSummary(safeParameters), + account, callerType, callerId); + } catch (RuntimeException exception) { + registration.close(); + throw exception; + } + List referencedTables = List.of(); + List referencedFields = List.of(); + try { + cancellationService.throwIfCancelled(queryId); + SqlCompileRequest compileRequest = compileRequest(source, sql, safeParameters); + FederationSqlPlan plan = runtime.engine().compile(compileRequest); + cancellationService.throwIfCancelled(queryId); + referencedTables = referencedTables(plan.relRoot().rel); + referencedFields = referencedFields(plan.sqlNode()); + SqlExecutionOptions options = new SqlExecutionOptions( + 200, + maxRows + 1, + 30, + true); + long databaseStartedNanos = System.nanoTime(); + List columns; + List> rows = new ArrayList<>(maxRows); + boolean truncated = false; + long resultBytes = 0L; + try (FederationResultCursor cursor = runtime.engine().execute( + plan, + new SqlExecutionContext( + queryId, + safeParameters, + options, + Duration.ofSeconds(5)))) { + columns = columns(cursor.columns()); + while (cursor.next()) { + cancellationService.throwIfCancelled(queryId); + if (rows.size() >= maxRows) { + truncated = true; + break; + } + Map row = new LinkedHashMap<>(); + for (int index = 0; index < columns.size(); index++) { + Object value = readColumn(cursor, cursor.columns().get(index), index + 1); + resultBytes += estimateBytes(columns.get(index).label(), value); + if (resultBytes > MAX_RESULT_BYTES) { + truncated = true; + break; + } + row.put(columns.get(index).key(), value); + } + if (truncated && resultBytes > MAX_RESULT_BYTES) { + break; + } + rows.add(Collections.unmodifiableMap(new LinkedHashMap<>(row))); + } + } + long databaseDurationMs = elapsedMillis(databaseStartedNanos); + long durationMs = elapsedMillis(startedNanos); + auditService.succeed(audit, referencedTables, referencedFields, rows.size(), truncated, + durationMs, databaseDurationMs); + return new DatacenterSqlConsoleResult( + queryId.value(), columns, rows, rows.size(), truncated, durationMs); + } catch (RuntimeException exception) { + long durationMs = elapsedMillis(startedNanos); + String errorCode = cancellationService.isCancelled(queryId) + ? "QUERY_CANCELLED" + : exception instanceof FederationSqlException federationException + ? federationException.errorCode().name() + : "DATACENTER_QUERY_FAILED"; + try { + auditService.fail(audit, errorCode, safeMessage(exception), + referencedTables, referencedFields, durationMs); + } catch (RuntimeException auditException) { + exception.addSuppressed(auditException); + log.error("Failed to finalize datacenter query audit {}", queryId.value(), auditException); + } + if (exception instanceof BusinessException businessException) { + throw businessException; + } + logUnexpectedFailure(queryId, source, exception); + throw toBusinessException(exception); + } finally { + registration.close(); + } + } + + /** + * 使用 Federation 游标流式消费单数据源只读 SQL，并在触及资源上限时显式失败。 + * + * @param source 已激活数据源 + * @param sql 只读 SQL + * @param parameters 显式 JDBC 参数 + * @param fetchSize JDBC 拉取批次 + * @param maxRows 最大消费行数 + * @param maxBytes 最大估算字节数 + * @param account 执行账号 + * @param callerType 调用方类型 + * @param callerId 调用方标识 + * @param consumer 单行消费者 + * @return 已消费行数 + */ + public long consume( + DatacenterSource source, + String sql, + List parameters, + int fetchSize, + long maxRows, + long maxBytes, + LoginAccount account, + String callerType, + String callerId, + Consumer> consumer) { + return consume(source, sql, parameters, fetchSize, maxRows, maxBytes, + account, callerType, callerId, null, consumer); + } + + /** + * 使用调用方预生成的 QueryId 流式消费只读 SQL，以支持流消费期间取消。 + * + * @param source 已激活数据源 + * @param sql 只读 SQL + * @param parameters 显式 JDBC 参数 + * @param fetchSize JDBC 拉取批次 + * @param maxRows 最大消费行数 + * @param maxBytes 最大估算字节数 + * @param account 执行账号 + * @param callerType 调用方类型 + * @param callerId 调用方标识 + * @param requestedQueryId 调用方预生成的查询 UUID；为空时服务端生成 + * @param consumer 单行消费者 + * @return 已消费行数 + */ + public long consume( + DatacenterSource source, + String sql, + List parameters, + int fetchSize, + long maxRows, + long maxBytes, + LoginAccount account, + String callerType, + String callerId, + String requestedQueryId, + Consumer> consumer) { + requireExecutable(source, sql); + if (fetchSize <= 0 || maxRows <= 0 || maxRows >= Integer.MAX_VALUE + || maxBytes <= 0 || consumer == null) { + throw new IllegalArgumentException("stream query limits and consumer must be valid"); + } + List safeParameters = List.copyOf(parameters == null ? List.of() : parameters); + QueryId queryId = cancellationService.resolveQueryId(requestedQueryId); + DatacenterFederationQueryCancellationService.QueryRegistration registration = + cancellationService.register(queryId, account); + long startedNanos = System.nanoTime(); + DatacenterQueryAudit audit; + try { + audit = auditService.start( + queryId.value(), source, sql, + maskedParameterSummary(safeParameters), + account, callerType, callerId); + } catch (RuntimeException exception) { + registration.close(); + throw exception; + } + List referencedTables = List.of(); + List referencedFields = List.of(); + long consumedRows = 0L; + try { + cancellationService.throwIfCancelled(queryId); + SqlCompileRequest compileRequest = compileRequest(source, sql, safeParameters); + FederationSqlPlan plan = runtime.engine().compile(compileRequest); + cancellationService.throwIfCancelled(queryId); + referencedTables = referencedTables(plan.relRoot().rel); + referencedFields = referencedFields(plan.sqlNode()); + SqlExecutionOptions options = new SqlExecutionOptions( + fetchSize, Math.toIntExact(maxRows + 1L), 30, true); + long databaseStartedNanos = System.nanoTime(); + long resultBytes = 0L; + try (FederationResultCursor cursor = runtime.engine().execute( + plan, + new SqlExecutionContext(queryId, safeParameters, options, Duration.ofSeconds(5)))) { + List columns = columns(cursor.columns()); + while (cursor.next()) { + cancellationService.throwIfCancelled(queryId); + if (consumedRows >= maxRows) { + throw new BusinessException("数据集查询结果超过行数上限: " + maxRows); + } + Map row = new LinkedHashMap<>(); + for (int index = 0; index < columns.size(); index++) { + Object value = readColumn(cursor, cursor.columns().get(index), index + 1); + resultBytes += estimateBytes(columns.get(index).label(), value); + if (resultBytes > maxBytes) { + throw new BusinessException("数据集查询结果超过字节上限: " + maxBytes); + } + row.put(columns.get(index).label(), value); + } + consumer.accept(Collections.unmodifiableMap(row)); + consumedRows++; + } + } + long durationMs = elapsedMillis(startedNanos); + auditService.succeed( + audit, referencedTables, referencedFields, consumedRows, false, + durationMs, elapsedMillis(databaseStartedNanos)); + return consumedRows; + } catch (RuntimeException exception) { + long durationMs = elapsedMillis(startedNanos); + String errorCode = cancellationService.isCancelled(queryId) + ? "QUERY_CANCELLED" + : exception instanceof FederationSqlException federationException + ? federationException.errorCode().name() + : "DATACENTER_QUERY_FAILED"; + try { + auditService.fail(audit, errorCode, safeMessage(exception), + referencedTables, referencedFields, durationMs); + } catch (RuntimeException auditException) { + exception.addSuppressed(auditException); + log.error("Failed to finalize datacenter stream query audit {}", queryId.value(), auditException); + } + if (exception instanceof BusinessException businessException) { + throw businessException; + } + logUnexpectedFailure(queryId, source, exception); + throw toBusinessException(exception); + } finally { + registration.close(); + } + } + + private SqlCompileRequest compileRequest( + DatacenterSource source, + String sql, + List parameters) { + SourceId sourceId = runtime.definitions().sourceId(source); + return new SqlCompileRequest( + sql, + FederationQueryScopeDefinition.single( + "datacenter-source:" + source.getId(), + source.getScopeRevision(), + sourceId.value(), + sourceId, + source.getDefinitionRevision()), + parameters.stream().map(SqlParameter::jdbcType).toList(), + "scope-" + source.getScopeRevision()); + } + + private void requireExecutable(DatacenterSource source, String sql) { + if (source == null || DatacenterSourceStatus.fromCode(source.getStatus()) + != DatacenterSourceStatus.READY) { + throw new BusinessException("数据连接尚未就绪"); + } + if (sql == null || sql.isBlank()) { + throw new BusinessException("SQL 不能为空"); + } + if (sql.length() > 100_000) { + throw new BusinessException("SQL 长度超过限制"); + } + } + + private int normalizeMaxRows(Integer requested) { + if (requested == null) { + return DEFAULT_MAX_ROWS; + } + if (requested <= 0) { + throw new BusinessException("maxRows 必须大于 0"); + } + return Math.min(requested, MAX_ALLOWED_ROWS); + } + + private List columns(List columns) { + List result = new ArrayList<>(columns.size()); + for (FederationColumn column : columns) { + result.add(new DatacenterSqlColumnView( + "c" + column.index(), + column.label(), + column.jdbcType(), + column.typeName(), + column.nullable())); + } + return List.copyOf(result); + } + + private List referencedTables(RelNode root) { + Set names = new LinkedHashSet<>(); + new RelVisitor() { + @Override + public void visit(RelNode node, int ordinal, RelNode parent) { + if (node instanceof TableScan scan) { + names.add(String.join(".", scan.getTable().getQualifiedName())); + } + super.visit(node, ordinal, parent); + } + }.go(root); + return List.copyOf(names); + } + + private List referencedFields(org.apache.calcite.sql.SqlNode sqlNode) { + Set names = new LinkedHashSet<>(); + sqlNode.accept(new SqlBasicVisitor() { + @Override + public Void visit(SqlIdentifier identifier) { + if (!identifier.isStar() && !identifier.names.isEmpty()) { + names.add(String.join(".", identifier.names)); + } + return null; + } + }); + return List.copyOf(names); + } + + private Map maskedParameterSummary(List parameters) { + Map summary = new LinkedHashMap<>(); + for (int index = 0; index < parameters.size(); index++) { + SqlParameter parameter = parameters.get(index); + summary.put("p" + (index + 1), Map.of( + "jdbcType", parameter.jdbcType(), + "present", parameter.value() != null)); + } + return Map.copyOf(summary); + } + + private Object normalizeValue(Object value) { + try { + if (value instanceof byte[] bytes) { + if (bytes.length > MAX_TEXT_CELL_CHARS) { + return "[binary " + bytes.length + " bytes]"; + } + return Base64.getEncoder().encodeToString(bytes); + } + if (value instanceof Blob blob) { + return "[binary " + blob.length() + " bytes]"; + } + if (value instanceof Clob clob) { + long length = Math.min(clob.length(), 4_096L); + return clob.getSubString(1L, (int) length); + } + if (value instanceof SQLXML sqlxml) { + String text = sqlxml.getString(); + return boundedText(text); + } + if (value instanceof java.sql.Array array) { + return "[array " + array.getBaseTypeName() + "]"; + } + if (value instanceof Struct struct) { + return "[struct " + struct.getSQLTypeName() + "]"; + } + if (value instanceof CharSequence text) { + return boundedText(text.toString()); + } + if (value instanceof BigInteger || value instanceof Long) { + return value.toString(); + } + Package valuePackage = value.getClass().getPackage(); + if (valuePackage != null && valuePackage.getName().startsWith("org.postgresql")) { + return boundedText(String.valueOf(value)); + } + return value instanceof Number || value instanceof Boolean + || value instanceof java.time.temporal.TemporalAccessor + || value instanceof java.util.Date + ? value : boundedText(String.valueOf(value)); + } catch (SQLException exception) { + throw new BusinessException("查询结果包含无法读取的大对象"); + } + } + + /** + * 按 JDBC 类型选择有界流式读取或普通对象读取。 + * + * @param cursor 当前结果游标 + * @param column 列元数据 + * @param columnIndex 从 1 开始的列序号 + * @return 可安全序列化的有界值 + */ + private Object readColumn( + FederationResultCursor cursor, + FederationColumn column, + int columnIndex) { + try { + if (isBinaryType(column.jdbcType())) { + try (InputStream stream = cursor.getBinaryStream(columnIndex)) { + return readBoundedBinary(stream); + } + } + if (isCharacterType(column.jdbcType(), column.typeName())) { + try (Reader reader = cursor.getCharacterStream(columnIndex)) { + return readBoundedText(reader); + } + } + } catch (UnsupportedOperationException ignored) { + // 第三方 Adapter 可暂不实现流式列读取；保留兼容的对象读取路径。 + } catch (IOException exception) { + throw new BusinessException("查询结果流读取失败"); + } + return normalizeValue(cursor.getObject(columnIndex)); + } + + /** + * 有界读取二进制流，小值编码为 Base64，大值返回摘要。 + * + * @param stream 二进制流；可为空 + * @return Base64 文本、大小摘要或 null + * @throws IOException 流读取失败 + */ + private Object readBoundedBinary(InputStream stream) throws IOException { + if (stream == null) { + return null; + } + byte[] bytes = stream.readNBytes(MAX_TEXT_CELL_CHARS + 1); + if (bytes.length > MAX_TEXT_CELL_CHARS) { + return "[binary exceeds " + MAX_TEXT_CELL_CHARS + " bytes]"; + } + return Base64.getEncoder().encodeToString(bytes); + } + + /** + * 有界读取字符流，超过单元格限制时附加截断标识。 + * + * @param reader 字符流；可为空 + * @return 有界文本或 null + * @throws IOException 流读取失败 + */ + private String readBoundedText(Reader reader) throws IOException { + if (reader == null) { + return null; + } + StringBuilder text = new StringBuilder(Math.min(MAX_TEXT_CELL_CHARS, 4_096)); + char[] buffer = new char[4_096]; + while (text.length() <= MAX_TEXT_CELL_CHARS) { + int remaining = MAX_TEXT_CELL_CHARS + 1 - text.length(); + int read = reader.read(buffer, 0, Math.min(buffer.length, remaining)); + if (read < 0) { + break; + } + text.append(buffer, 0, read); + } + if (text.length() > MAX_TEXT_CELL_CHARS) { + return text.substring(0, MAX_TEXT_CELL_CHARS) + "…[truncated]"; + } + return text.toString(); + } + + /** + * 判断 JDBC 类型是否应使用二进制流读取。 + * + * @param jdbcType JDBC 类型 + * @return 是否为二进制大字段类型 + */ + private boolean isBinaryType(int jdbcType) { + return jdbcType == Types.BINARY + || jdbcType == Types.VARBINARY + || jdbcType == Types.LONGVARBINARY + || jdbcType == Types.BLOB; + } + + /** + * 判断 JDBC 类型或数据库类型名是否应使用字符流读取。 + * + * @param jdbcType JDBC 类型 + * @param typeName 数据库类型名 + * @return 是否为字符大字段类型 + */ + private boolean isCharacterType(int jdbcType, String typeName) { + if (jdbcType == Types.CHAR + || jdbcType == Types.VARCHAR + || jdbcType == Types.LONGVARCHAR + || jdbcType == Types.NCHAR + || jdbcType == Types.NVARCHAR + || jdbcType == Types.LONGNVARCHAR + || jdbcType == Types.CLOB + || jdbcType == Types.NCLOB + || jdbcType == Types.SQLXML) { + return true; + } + String normalized = typeName == null ? "" : typeName.toLowerCase(java.util.Locale.ROOT); + return normalized.equals("json") || normalized.equals("jsonb") || normalized.equals("xml"); + } + + private String boundedText(String value) { + if (value == null || value.length() <= MAX_TEXT_CELL_CHARS) { + return value; + } + return value.substring(0, MAX_TEXT_CELL_CHARS) + + "…[truncated " + value.length() + " chars]"; + } + + private long estimateBytes(String label, Object value) { + long bytes = label == null ? 0L : (long) label.length() * Character.BYTES; + if (value == null) { + return bytes + 8L; + } + if (value instanceof CharSequence text) { + return bytes + (long) text.length() * Character.BYTES; + } + return bytes + 64L; + } + + private long elapsedMillis(long startedNanos) { + return Math.max(0L, (System.nanoTime() - startedNanos) / 1_000_000L); + } + + private String safeMessage(Throwable exception) { + String message = exception.getMessage(); + if (message == null || message.isBlank()) { + return "数据查询失败"; + } + return message.length() <= 500 ? message : message.substring(0, 500); + } + + private BusinessException toBusinessException(RuntimeException exception) { + if (exception instanceof FederationSqlException federationException) { + int status = switch (federationException.errorCode()) { + case QUERY_ADMISSION_TIMEOUT, QUERY_TIMEOUT, + SQL_COMPILE_TIMEOUT -> 408; + case NODE_MEMORY_ADMISSION_TIMEOUT -> 429; + case SOURCE_NOT_FOUND, SOURCE_REMOVED -> 404; + case SOURCE_REVISION_NOT_READY, SOURCE_DEFINITION_CONFLICT -> 409; + case PLAN_STALE -> 409; + case FEDERATION_RESOURCE_LIMIT_EXCEEDED -> 422; + case EXECUTION_FAILED, RESOURCE_CLOSE_FAILED, + SOURCE_INITIALIZATION_FAILED, ENGINE_CLOSED, + EXPLAIN_FAILED, SQL_COMPLETION_FAILED, + CONNECTION_ACQUISITION_TIMEOUT, + CONNECTION_ACQUISITION_FAILED -> 503; + default -> 400; + }; + int code = switch (federationException.errorCode()) { + case INVALID_ARGUMENT -> 14101; + case SQL_PARSE_FAILED -> 14102; + case SQL_VALIDATION_FAILED -> 14103; + case SQL_NOT_READ_ONLY -> 14104; + case SQL_COMPILE_FAILED -> 14105; + case SQL_NOT_FULLY_PUSHDOWN -> 14106; + case CROSS_SOURCE_DISABLED -> 14107; + case CROSS_SOURCE_EXECUTION_UNSUPPORTED -> 14108; + case SOURCE_NOT_FOUND -> 14109; + case SOURCE_REMOVED -> 14110; + case SOURCE_REVISION_NOT_READY -> 14111; + case SOURCE_DEFINITION_CONFLICT -> 14112; + case SOURCE_INITIALIZATION_FAILED -> 14113; + case ADAPTER_NOT_FOUND -> 14114; + case ADAPTER_UNSUPPORTED -> 14115; + case PARAMETER_COUNT_MISMATCH -> 14116; + case QUERY_ADMISSION_TIMEOUT -> 14117; + case QUERY_CANCELLED -> 14118; + case QUERY_TIMEOUT -> 14119; + case EXECUTION_FAILED -> 14120; + case RESOURCE_CLOSE_FAILED -> 14121; + case ENGINE_CLOSED -> 14122; + case INVALID_QUERY_SCOPE -> 14124; + case FEDERATION_OPERATOR_UNSUPPORTED -> 14125; + case FEDERATION_RESOURCE_LIMIT_EXCEEDED -> 14126; + case PLAN_STALE -> 14127; + case EXPLAIN_FAILED -> 14128; + case SQL_COMPLETION_FAILED -> 14129; + case SQL_COMPILE_TIMEOUT -> 14130; + case NODE_MEMORY_ADMISSION_TIMEOUT -> 14131; + case CONNECTION_ACQUISITION_TIMEOUT -> 14132; + case CONNECTION_ACQUISITION_FAILED -> 14133; + }; + String message = switch (federationException.errorCode()) { + case EXECUTION_FAILED, RESOURCE_CLOSE_FAILED, + SOURCE_INITIALIZATION_FAILED, ENGINE_CLOSED -> "数据库查询暂时不可用"; + case QUERY_TIMEOUT, QUERY_ADMISSION_TIMEOUT -> "数据库查询超时"; + case QUERY_CANCELLED -> "查询已取消"; + case FEDERATION_RESOURCE_LIMIT_EXCEEDED -> "联邦查询超过资源限制"; + case EXPLAIN_FAILED -> "数据库查询计划分析失败"; + case SQL_COMPLETION_FAILED -> "SQL 补全暂时不可用"; + case SQL_COMPILE_TIMEOUT -> "SQL 编译超时，请简化语句后重试"; + case NODE_MEMORY_ADMISSION_TIMEOUT -> "节点查询资源繁忙，请稍后重试"; + case CONNECTION_ACQUISITION_TIMEOUT -> "数据库连接池繁忙，请稍后重试"; + case CONNECTION_ACQUISITION_FAILED -> "无法获取数据库连接，请检查连接状态"; + default -> safeMessage(exception); + }; + return new BusinessException(status, code, message, exception); + } + return new BusinessException(500, 14999, "数据查询失败", exception); + } + + private void logUnexpectedFailure( + QueryId queryId, + DatacenterSource source, + RuntimeException exception) { + if (!(exception instanceof FederationSqlException federationException) + || switch (federationException.errorCode()) { + case EXECUTION_FAILED, RESOURCE_CLOSE_FAILED, + SOURCE_INITIALIZATION_FAILED, ENGINE_CLOSED, + EXPLAIN_FAILED, CONNECTION_ACQUISITION_FAILED -> true; + default -> false; + }) { + log.error("Datacenter query failed, queryId={}, sourceId={}", + queryId.value(), source == null ? null : source.getId(), exception); + } + } +} diff --git a/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/federation/DatacenterFederationRuntime.java b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/federation/DatacenterFederationRuntime.java new file mode 100644 index 00000000..92d00344 --- /dev/null +++ b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/federation/DatacenterFederationRuntime.java @@ -0,0 +1,175 @@ +package tech.easyflow.datacenter.federation; + +import com.easyagents.federation.sql.adapter.jdbc.JdbcFederationSqlAdapterProvider; +import com.easyagents.federation.sql.api.FederationSqlEngine; +import com.easyagents.federation.sql.api.FederationSqlEngines; +import com.easyagents.federation.sql.source.FederationSourceDefinition; +import com.easyagents.federation.sql.source.PreparedSourceRuntime; +import com.easyagents.federation.sql.source.SourceApplyOptions; +import com.easyagents.federation.sql.source.SourceApplyResult; +import com.easyagents.federation.sql.source.SourceApplyStatus; +import com.easyagents.federation.sql.source.SourceProbeResult; +import com.easyagents.federation.sql.source.SourceRemoveResult; +import com.easyagents.federation.sql.source.SourceTombstone; +import jakarta.annotation.PreDestroy; +import org.springframework.stereotype.Component; +import tech.easyflow.datacenter.meta.entity.DatacenterSource; + +/** + * EasyFlow 数据中枢持有的单例 Federation SQL Engine。 + */ +@Component +public class DatacenterFederationRuntime { + + private final DatacenterFederationDefinitionFactory definitionFactory; + private final DatacenterFederationDataSourceResolver resolver; + private final FederationSqlEngine engine; + + /** + * 组装 DataSource Resolver、数据库状态 Provider 与业务 Policy。 + * + * @param definitionFactory Definition 工厂 + * @param resolver HikariCP Resolver + * @param stateProvider 数据库权威状态 Provider + * @param policy 纳管范围 Policy + * @param admissionController 租户与数据源分层准入控制器 + */ + public DatacenterFederationRuntime( + DatacenterFederationDefinitionFactory definitionFactory, + DatacenterFederationDataSourceResolver resolver, + DatacenterFederationSourceStateProvider stateProvider, + DatacenterFederationSqlPolicy policy, + DatacenterFederationAdmissionController admissionController) { + this.definitionFactory = definitionFactory; + this.resolver = resolver; + this.engine = FederationSqlEngines.builder() + .dataSourceResolver(resolver) + .stateProvider(stateProvider) + .adapter(new JdbcFederationSqlAdapterProvider()) + .policy(policy) + .admissionController(admissionController) + .maximumPlanCacheEntries(512) + .maximumConcurrentCompilations(Math.max(1, Math.min(4, + Runtime.getRuntime().availableProcessors()))) + .build(); + } + + /** + * 探测草稿数据源，不保留临时连接池。 + * + * @param source 数据源草稿 + * @return 探测结果 + */ + public SourceProbeResult probe(DatacenterSource source) { + return engine.sources().probe(definitionFactory.create(source)); + } + + /** + * 使用未发布候选配置执行短生命周期探测。 + * + * @param source 候选数据源 + * @return 探测结果 + */ + public SourceProbeResult probeCandidate(DatacenterSource source) { + FederationSourceDefinition definition = definitionFactory.createCandidate(source); + return resolver.withCandidate(source, definition, + () -> engine.sources().probe(definition)); + } + + /** + * 预构建单命名空间候选 Runtime，且不修改共享 Source Slot。 + * + * @param source 候选数据源 + * @param catalogName 即将发布的 Catalog 或 Schema + * @return 由调用方暂时持有的候选 Runtime + */ + public PreparedSourceRuntime prepareCandidate( + DatacenterSource source, + String catalogName) { + FederationSourceDefinition definition = + definitionFactory.createCandidate(source, catalogName); + return resolver.withCandidate(source, definition, + () -> engine.sources().prepare(definition)); + } + + /** + * 使用当前持久化命名空间预构建候选 Runtime，且不修改共享 Source Slot。 + * + * @param source 候选数据源 + * @return 由调用方暂时持有的候选 Runtime + */ + public PreparedSourceRuntime prepareCandidate(DatacenterSource source) { + FederationSourceDefinition definition = definitionFactory.create(source); + return resolver.withCandidate(source, definition, + () -> engine.sources().prepare(definition)); + } + + /** + * 原子接管已经通过业务 revision CAS 的候选 Runtime。 + * + * @param prepared 预构建 Runtime + * @return Definition 应用结果 + * @throws IllegalStateException Slot 已被更高版本或冲突版本占用时抛出 + */ + public SourceApplyResult commitPrepared(PreparedSourceRuntime prepared) { + SourceApplyResult result = engine.sources().commit(prepared); + if (result.status() == SourceApplyStatus.CONFLICT + || result.status() == SourceApplyStatus.IGNORED_STALE) { + throw new IllegalStateException( + "prepared runtime was superseded before commit: " + result.status()); + } + return result; + } + + /** + * 应用已持久化 Definition。 + * + * @param source 数据源当前快照 + * @param prewarm 是否立即预热 + * @return 应用结果 + */ + public SourceApplyResult apply(DatacenterSource source, boolean prewarm) { + FederationSourceDefinition definition = definitionFactory.create(source); + return engine.sources().apply( + definition, + prewarm ? SourceApplyOptions.prewarmNow() : SourceApplyOptions.lazy()); + } + + /** + * 应用禁用或删除墓碑。 + * + * @param source 数据源当前快照 + * @return 删除结果 + */ + public SourceRemoveResult remove(DatacenterSource source) { + return engine.sources().remove(SourceTombstone.of( + definitionFactory.sourceId(source), + source.getDefinitionRevision() == null ? 0L : source.getDefinitionRevision())); + } + + /** + * 返回统一查询 Engine。 + * + * @return Engine + */ + public FederationSqlEngine engine() { + return engine; + } + + /** + * 返回 Definition 工厂。 + * + * @return Definition 工厂 + */ + public DatacenterFederationDefinitionFactory definitions() { + return definitionFactory; + } + + /** + * 应用停止时关闭查询、计划缓存和所有 Runtime 句柄。 + */ + @PreDestroy + public void close() { + engine.close(); + } +} diff --git a/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/federation/DatacenterFederationSourceStateProvider.java b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/federation/DatacenterFederationSourceStateProvider.java new file mode 100644 index 00000000..19c303d9 --- /dev/null +++ b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/federation/DatacenterFederationSourceStateProvider.java @@ -0,0 +1,177 @@ +package tech.easyflow.datacenter.federation; + +import com.easyagents.federation.sql.source.ActiveSourceState; +import com.easyagents.federation.sql.source.FederationSourceState; +import com.easyagents.federation.sql.source.FederationSourceStateProvider; +import com.easyagents.federation.sql.source.SourceId; +import com.easyagents.federation.sql.source.SourceStateSubscription; +import com.easyagents.federation.sql.source.SourceTombstone; +import com.mybatisflex.core.query.QueryWrapper; +import java.math.BigInteger; +import java.util.Collection; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.function.Consumer; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Component; +import tech.easyflow.datacenter.mapper.DatacenterSourceMapper; +import tech.easyflow.datacenter.meta.entity.DatacenterSource; +import tech.easyflow.datacenter.meta.enums.DatacenterSourceStatus; +import tech.easyflow.datacenter.meta.enums.DatacenterSourceType; + +/** + * 数据库权威、Redis 仅提示的 Federation Source State Provider。 + */ +@Component +public class DatacenterFederationSourceStateProvider implements FederationSourceStateProvider { + + private static final Logger log = LoggerFactory.getLogger(DatacenterFederationSourceStateProvider.class); + + private final DatacenterSourceMapper sourceMapper; + private final DatacenterFederationDefinitionFactory definitionFactory; + private final CopyOnWriteArrayList> subscribers = + new CopyOnWriteArrayList<>(); + + /** + * 创建状态 Provider。 + * + * @param sourceMapper 数据源 Mapper + * @param definitionFactory Definition 工厂 + */ + public DatacenterFederationSourceStateProvider( + DatacenterSourceMapper sourceMapper, + DatacenterFederationDefinitionFactory definitionFactory) { + this.sourceMapper = sourceMapper; + this.definitionFactory = definitionFactory; + } + + /** + * 从数据库读取指定数据源的最新 Definition 或墓碑。 + * + * @param sourceId Federation SourceId + * @return 当前共享状态 + */ + @Override + public Optional find(SourceId sourceId) { + BigInteger recordId = parseRecordId(sourceId); + if (recordId == null) { + return Optional.empty(); + } + return toState(sourceMapper.selectOneById(recordId)); + } + + /** + * 加载当前数据库上下文中可见的外部数据源快照。 + * + * @return Definition 与墓碑集合 + */ + @Override + public Collection loadSnapshot() { + QueryWrapper query = QueryWrapper.create() + .in(DatacenterSource::getSourceType, List.of( + DatacenterSourceType.MYSQL.name(), + DatacenterSourceType.POSTGRESQL.name())) + .in(DatacenterSource::getStatus, List.of( + DatacenterSourceStatus.DRAFT.code(), + DatacenterSourceStatus.READY.code(), + DatacenterSourceStatus.DEGRADED.code(), + DatacenterSourceStatus.DISABLED.code(), + DatacenterSourceStatus.DELETED.code())); + List sources = sourceMapper.selectListByQuery(query); + Map definitions = + definitionFactory.createBatch(sources.stream() + .filter(this::isActive) + .toList()); + return sources.stream() + .map(source -> toState(source, definitions.get(source.getId()))) + .flatMap(Optional::stream) + .toList(); + } + + /** + * 注册节点本地变更消费者。 + * + * @param consumer 状态消费者 + * @return 可关闭订阅 + */ + @Override + public SourceStateSubscription subscribe(Consumer consumer) { + subscribers.add(consumer); + return () -> subscribers.remove(consumer); + } + + /** + * 响应无凭据 Redis 提示，从数据库重读并发布最新状态。 + * + * @param sourceId Federation SourceId + */ + public void refresh(SourceId sourceId) { + find(sourceId).ifPresent(state -> subscribers.forEach(consumer -> consumer.accept(state))); + } + + /** + * 周期性回源数据库，弥补 Redis Pub/Sub 在节点离线或网络抖动时可能丢失的提示。 + */ + @Scheduled(fixedDelayString = "${easyflow.datacenter.federation-reconcile-ms:30000}") + public void reconcile() { + try { + loadSnapshot().forEach(state -> subscribers.forEach(consumer -> consumer.accept(state))); + } catch (RuntimeException exception) { + log.warn("Failed to reconcile datacenter Federation source states", exception); + } + } + + private Optional toState(DatacenterSource source) { + return toState(source, null); + } + + private Optional toState( + DatacenterSource source, + com.easyagents.federation.sql.source.FederationSourceDefinition preloadedDefinition) { + if (source == null || source.getId() == null) { + return Optional.empty(); + } + DatacenterSourceStatus status = DatacenterSourceStatus.fromCode(source.getStatus()); + long revision = source.getDefinitionRevision() == null ? 0L : source.getDefinitionRevision(); + if (status == DatacenterSourceStatus.READY || status == DatacenterSourceStatus.DEGRADED) { + if (revision <= 0L) { + return Optional.empty(); + } + return Optional.of(ActiveSourceState.of(preloadedDefinition == null + ? definitionFactory.create(source) : preloadedDefinition)); + } + if ((status == DatacenterSourceStatus.DRAFT && revision > 0L) + || status == DatacenterSourceStatus.DISABLED + || status == DatacenterSourceStatus.DELETED) { + return Optional.of(SourceTombstone.of(definitionFactory.sourceId(source), revision)); + } + return Optional.empty(); + } + + private boolean isActive(DatacenterSource source) { + DatacenterSourceStatus status = source == null + ? DatacenterSourceStatus.DRAFT + : DatacenterSourceStatus.fromCode(source.getStatus()); + return status == DatacenterSourceStatus.READY || status == DatacenterSourceStatus.DEGRADED; + } + + private BigInteger parseRecordId(SourceId sourceId) { + if (sourceId == null) { + return null; + } + String value = sourceId.value(); + int marker = value.lastIndexOf("-source-"); + if (marker < 0) { + return null; + } + try { + return new BigInteger(value.substring(marker + "-source-".length())); + } catch (NumberFormatException ignored) { + return null; + } + } +} diff --git a/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/federation/DatacenterFederationSqlPolicy.java b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/federation/DatacenterFederationSqlPolicy.java new file mode 100644 index 00000000..1fce04b1 --- /dev/null +++ b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/federation/DatacenterFederationSqlPolicy.java @@ -0,0 +1,859 @@ +package tech.easyflow.datacenter.federation; + +import com.easyagents.federation.sql.api.FederationSqlErrorCode; +import com.easyagents.federation.sql.api.FederationSqlException; +import com.easyagents.federation.sql.compile.FederationSqlPolicy; +import com.easyagents.federation.sql.compile.SqlPolicyContext; +import com.mybatisflex.core.query.QueryWrapper; +import java.math.BigInteger; +import java.util.ArrayDeque; +import java.util.Collections; +import java.util.Deque; +import java.util.HashMap; +import java.util.HashSet; +import java.util.IdentityHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentLinkedQueue; +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.RelVisitor; +import org.apache.calcite.rel.core.TableScan; +import org.apache.calcite.sql.SqlIdentifier; +import org.apache.calcite.sql.SqlCall; +import org.apache.calcite.sql.SqlKind; +import org.apache.calcite.sql.SqlJoin; +import org.apache.calcite.sql.SqlNode; +import org.apache.calcite.sql.SqlNodeList; +import org.apache.calcite.sql.SqlOrderBy; +import org.apache.calcite.sql.SqlSelect; +import org.apache.calcite.sql.SqlWith; +import org.apache.calcite.sql.SqlWithItem; +import org.apache.calcite.sql.util.SqlBasicVisitor; +import org.springframework.stereotype.Component; +import tech.easyflow.datacenter.entity.DatacenterTable; +import tech.easyflow.datacenter.entity.DatacenterTableField; +import tech.easyflow.datacenter.mapper.DatacenterCatalogMapper; +import tech.easyflow.datacenter.mapper.DatacenterSourceMapper; +import tech.easyflow.datacenter.mapper.DatacenterTableFieldMapper; +import tech.easyflow.datacenter.mapper.DatacenterTableMapper; +import tech.easyflow.datacenter.meta.entity.DatacenterCatalog; +import tech.easyflow.datacenter.meta.entity.DatacenterSource; +import tech.easyflow.datacenter.meta.enums.DatacenterMetadataStatus; +import tech.easyflow.datacenter.meta.enums.DatacenterSensitivityLevel; +import tech.easyflow.datacenter.meta.enums.DatacenterSourceStatus; + +/** + * 将 Calcite 解析出的表与字段引用限制在当前数据源的可查询纳管范围内。 + */ +@Component +public class DatacenterFederationSqlPolicy implements FederationSqlPolicy { + + private static final int MAX_POLICY_CACHE_ENTRIES = 512; + + private final DatacenterTableMapper tableMapper; + private final DatacenterTableFieldMapper tableFieldMapper; + private final DatacenterCatalogMapper catalogMapper; + private final DatacenterSourceMapper sourceMapper; + private final Map policyCache = new ConcurrentHashMap<>(); + private final ConcurrentLinkedQueue policyCacheOrder = new ConcurrentLinkedQueue<>(); + + /** + * 创建业务 SQL Policy。 + * + * @param tableMapper 表 Mapper + * @param tableFieldMapper 字段 Mapper + * @param catalogMapper 命名空间 Mapper + * @param sourceMapper 数据源 Mapper + */ + public DatacenterFederationSqlPolicy( + DatacenterTableMapper tableMapper, + DatacenterTableFieldMapper tableFieldMapper, + DatacenterCatalogMapper catalogMapper, + DatacenterSourceMapper sourceMapper) { + this.tableMapper = tableMapper; + this.tableFieldMapper = tableFieldMapper; + this.catalogMapper = catalogMapper; + this.sourceMapper = sourceMapper; + } + + /** + * 返回 Policy 代码版本。 + * + * @return 稳定版本 + */ + @Override + public String version() { + return "xl01-wp1-v6"; + } + + /** + * 拒绝未纳管表、不可查询字段和任何尚未支持脱敏的敏感字段。 + * + * @param context Calcite 策略上下文 + */ + @Override + public void validate(SqlPolicyContext context) { + if (context.request().queryScope().bindings().size() != 1 + || context.referencedSources().size() != 1 + || !context.referencedSources().contains( + context.request().sourceId())) { + throw validationFailure("当前数据中枢尚未开放跨数据源授权范围"); + } + BigInteger sourceId = parseRecordId(context.request().sourceId().value()); + requireCurrentScopeRevision( + sourceId, context.request().policyVersion()); + PolicyCacheKey cacheKey = new PolicyCacheKey(sourceId, context.request().policyVersion()); + PolicySnapshot snapshot = cachedSnapshot(cacheKey); + Set referencedNames = referencedTables(context.relRoot().rel); + Set referencedTableIds = new LinkedHashSet<>(); + for (String referencedName : referencedNames) { + Set tableIds = snapshot.tableIdsByName().get(referencedName); + if (tableIds == null || tableIds.isEmpty()) { + throw validationFailure("SQL 引用了未纳管表: " + referencedName); + } + referencedTableIds.addAll(tableIds); + } + validatePhysicalColumns(context.relRoot().rel, snapshot); + validateFields(context, snapshot, referencedTableIds); + } + + /** + * 直接核对数据库权威范围版本，避免跨节点通知延迟复用旧策略。 + * + * @param sourceId 数据源记录 ID + * @param policyVersion 编译请求携带的范围版本 + */ + private void requireCurrentScopeRevision( + BigInteger sourceId, + String policyVersion) { + long requestedRevision = parseScopeRevision(policyVersion); + DatacenterSource current = sourceMapper.selectOneById(sourceId); + if (current == null + || DatacenterSourceStatus.fromCode(current.getStatus()) + != DatacenterSourceStatus.READY + || current.getScopeRevision() == null + || current.getScopeRevision() != requestedRevision) { + throw validationFailure("数据范围已变化，请重新执行查询"); + } + } + + /** + * 解析范围策略版本。 + * + * @param policyVersion 策略版本 + * @return 范围 revision + */ + private long parseScopeRevision(String policyVersion) { + if (policyVersion == null || !policyVersion.startsWith("scope-")) { + throw validationFailure("数据范围版本无效"); + } + try { + return Long.parseLong(policyVersion.substring("scope-".length())); + } catch (NumberFormatException exception) { + throw validationFailure("数据范围版本无效"); + } + } + + private PolicySnapshot cachedSnapshot(PolicyCacheKey cacheKey) { + PolicySnapshot existing = policyCache.get(cacheKey); + if (existing != null) { + return existing; + } + PolicySnapshot loaded = loadSnapshot(cacheKey.sourceId()); + PolicySnapshot raced = policyCache.putIfAbsent(cacheKey, loaded); + if (raced != null) { + return raced; + } + policyCacheOrder.add(cacheKey); + trimCache(); + return loaded; + } + + private void trimCache() { + while (policyCache.size() > MAX_POLICY_CACHE_ENTRIES) { + PolicyCacheKey eldest = policyCacheOrder.poll(); + if (eldest == null) { + return; + } + policyCache.remove(eldest); + } + } + + private PolicySnapshot loadSnapshot(BigInteger sourceId) { + QueryWrapper tableQuery = QueryWrapper.create() + .eq(DatacenterTable::getSourceId, sourceId) + .eq(DatacenterTable::getQueryable, 1) + .eq(DatacenterTable::getMetadataStatus, DatacenterMetadataStatus.ACTIVE.name()); + List tables = tableMapper.selectListByQuery(tableQuery); + if (tables.isEmpty()) { + return PolicySnapshot.empty(); + } + Set catalogIds = new HashSet<>(); + Set tableIds = new HashSet<>(); + for (DatacenterTable table : tables) { + tableIds.add(table.getId()); + if (table.getCatalogId() != null) { + catalogIds.add(table.getCatalogId()); + } + } + Map logicalSchemas = new HashMap<>(); + if (!catalogIds.isEmpty()) { + QueryWrapper catalogQuery = QueryWrapper.create().in(DatacenterCatalog::getId, catalogIds); + for (DatacenterCatalog catalog : catalogMapper.selectListByQuery(catalogQuery)) { + logicalSchemas.put(catalog.getId(), catalog.getLogicalSchemaName()); + } + } + Map> idsByName = new HashMap<>(); + Set tableReferenceNames = new HashSet<>(); + for (DatacenterTable table : tables) { + addTableName(idsByName, tableReferenceNames, table.getTableName(), table.getId()); + addTableName(idsByName, tableReferenceNames, table.getActualTable(), table.getId()); + String schema = logicalSchemas.get(table.getCatalogId()); + if (schema != null && !schema.isBlank()) { + addTableName(idsByName, tableReferenceNames, schema + "." + table.getTableName(), table.getId()); + addTableName(idsByName, tableReferenceNames, schema + "." + table.getActualTable(), table.getId()); + } + } + + Map columnsByTable = new HashMap<>(); + QueryWrapper fieldQuery = QueryWrapper.create().in(DatacenterTableField::getTableId, tableIds); + for (DatacenterTableField field : tableFieldMapper.selectListByQuery(fieldQuery)) { + ColumnPolicy columns = columnsByTable.computeIfAbsent( + field.getTableId(), ignored -> ColumnPolicy.mutable()); + String columnName = sourceColumnName(field); + if (columnName != null && !columnName.isBlank()) { + columns.knownExact().add(columnName); + columns.knownFolded().add(columnName.toUpperCase(Locale.ROOT)); + if (fieldAllowed(field)) { + columns.allowedExact().add(columnName); + columns.allowedFolded().add(columnName.toUpperCase(Locale.ROOT)); + } + } + } + Map> immutableNames = new HashMap<>(); + idsByName.forEach((name, ids) -> immutableNames.put(name, Set.copyOf(ids))); + return new PolicySnapshot( + Map.copyOf(immutableNames), + immutableColumnPolicies(columnsByTable), + Set.copyOf(tableReferenceNames)); + } + + private Map immutableColumnPolicies( + Map mutablePolicies) { + Map immutable = new HashMap<>(); + mutablePolicies.forEach((tableId, policy) -> immutable.put(tableId, new ColumnPolicy( + Set.copyOf(policy.knownExact()), + Set.copyOf(policy.knownFolded()), + Set.copyOf(policy.allowedExact()), + Set.copyOf(policy.allowedFolded())))); + return Map.copyOf(immutable); + } + + private void validatePhysicalColumns(RelNode root, PolicySnapshot snapshot) { + new RelVisitor() { + @Override + public void visit(RelNode node, int ordinal, RelNode parent) { + if (node instanceof TableScan scan) { + String referencedName = referencedTableName(scan); + Set tableIds = snapshot.tableIdsByName().get(referencedName); + if (tableIds == null || tableIds.isEmpty()) { + throw validationFailure("SQL 引用了未纳管表: " + referencedName); + } + for (BigInteger tableId : tableIds) { + ColumnPolicy policy = snapshot.columnsByTable().get(tableId); + if (policy == null || policy.knownExact().isEmpty()) { + throw validationFailure("数据表尚未纳管任何字段: " + referencedName); + } + for (String fieldName : scan.getRowType().getFieldNames()) { + if (!policy.knownExact().contains(fieldName) + && !policy.knownFolded().contains(fieldName.toUpperCase(Locale.ROOT))) { + throw validationFailure("物理表包含尚未纳管的字段: " + fieldName); + } + } + } + } + super.visit(node, ordinal, parent); + } + }.go(root); + } + + private void validateFields( + SqlPolicyContext context, + PolicySnapshot snapshot, + Set referencedTableIds) { + Deque scopes = new ArrayDeque<>(); + Map commonTables = collectCommonTables( + context.validatedSql(), snapshot, referencedTableIds); + context.validatedSql().accept(new SqlBasicVisitor() { + @Override + public Void visit(SqlCall call) { + AliasContext scope = scopeFor( + call, snapshot, referencedTableIds, commonTables); + if (scope == null) { + return super.visit(call); + } + scopes.push(scope); + try { + return super.visit(call); + } finally { + scopes.pop(); + } + } + + @Override + public Void visit(SqlIdentifier identifier) { + if (scopes.isEmpty()) { + return null; + } + AliasContext aliasContext = scopes.peek(); + if (aliasContext.syntaxAliases().contains(identifier) + || aliasContext.tableIdentifiers().contains(identifier) + || aliasContext.outputAliasReferences().contains(identifier)) { + return null; + } + if (identifier.isStar()) { + boolean allAllowed = referencedTableIds.stream() + .map(snapshot.columnsByTable()::get) + .allMatch(policy -> policy != null + && policy.knownExact().equals(policy.allowedExact())); + if (!allAllowed) { + throw validationFailure("查询范围包含不可查询或敏感字段，请显式选择允许字段"); + } + return null; + } + List names = identifier.names; + if (names.isEmpty()) { + return null; + } + int last = names.size() - 1; + String columnName = names.get(last); + boolean quoted = identifier.isComponentQuoted(last); + if (derivedColumnReference(identifier, aliasContext)) { + return null; + } + Set targetTableIds = resolveColumnTables( + identifier, snapshot, referencedTableIds, aliasContext); + boolean allowed = !targetTableIds.isEmpty() && targetTableIds.stream() + .allMatch(tableId -> columnAllowed( + snapshot.columnsByTable().get(tableId), columnName, quoted)); + if (!allowed) { + throw validationFailure("SQL 引用了不可查询或敏感字段: " + columnName); + } + return null; + } + }); + } + + private AliasContext scopeFor( + SqlCall call, + PolicySnapshot snapshot, + Set referencedTableIds, + Map commonTables) { + if (call instanceof SqlOrderBy orderBy + && orderBy.query instanceof SqlSelect select) { + return collectAliases( + select, orderBy.orderList, snapshot, + referencedTableIds, commonTables); + } + if (call instanceof SqlSelect select) { + return collectAliases( + select, null, snapshot, referencedTableIds, commonTables); + } + return null; + } + + private boolean derivedColumnReference( + SqlIdentifier identifier, + AliasContext aliasContext) { + List names = identifier.names; + int columnIndex = names.size() - 1; + String columnKey = identifierKey( + names.get(columnIndex), + identifier.isComponentQuoted(columnIndex)); + if (names.size() == 1) { + return aliasContext.unqualifiedDerivedColumns().contains(columnKey); + } + int qualifierIndex = names.size() - 2; + String qualifierKey = identifierKey( + names.get(qualifierIndex), + identifier.isComponentQuoted(qualifierIndex)); + Set derivedColumns = aliasContext + .derivedColumnsByAlias().get(qualifierKey); + return derivedColumns != null && derivedColumns.contains(columnKey); + } + + private Set resolveColumnTables( + SqlIdentifier identifier, + PolicySnapshot snapshot, + Set referencedTableIds, + AliasContext aliasContext) { + List identifierNames = identifier.names; + if (identifierNames.size() >= 2) { + int qualifierIndex = identifierNames.size() - 2; + String qualifier = identifierNames.get(qualifierIndex); + Set aliased = aliasContext.tableIdsByAlias().get( + identifierKey(qualifier, identifier.isComponentQuoted(qualifierIndex))); + if (aliased != null && !aliased.isEmpty()) { + return aliased; + } + Set qualified = snapshot.tableIdsByName().get(qualifier); + if (qualified != null && !qualified.isEmpty()) { + Set referencedQualified = new LinkedHashSet<>(qualified); + referencedQualified.retainAll(referencedTableIds); + if (!referencedQualified.isEmpty()) { + return referencedQualified; + } + } + } + if (referencedTableIds.size() == 1) { + return referencedTableIds; + } + // 无法将别名还原到单表时采用交集语义，避免多表同名列扩大权限。 + return referencedTableIds; + } + + private AliasContext collectAliases( + SqlSelect select, + SqlNodeList externalOrderList, + PolicySnapshot snapshot, + Set referencedTableIds, + Map commonTables) { + Map> tableIdsByAlias = new HashMap<>(); + Map> derivedColumnsByAlias = new HashMap<>(); + Set outputAliases = new HashSet<>(); + Set syntaxAliases = Collections.newSetFromMap( + new IdentityHashMap<>()); + Set tableIdentifiers = Collections.newSetFromMap( + new IdentityHashMap<>()); + Set outputAliasReferences = Collections.newSetFromMap( + new IdentityHashMap<>()); + collectOutputAliases( + select, outputAliases, syntaxAliases, outputAliasReferences); + FromSummary fromSummary = collectFrom( + select.getFrom(), snapshot, referencedTableIds, + commonTables, tableIdsByAlias, derivedColumnsByAlias, + syntaxAliases, tableIdentifiers); + markOutputAliasReferences( + externalOrderList, outputAliases, outputAliasReferences); + return new AliasContext( + Map.copyOf(tableIdsByAlias), + Map.copyOf(derivedColumnsByAlias), + fromSummary.singleDerived() + ? fromSummary.outputColumns() : Set.of(), + Set.copyOf(outputAliases), + syntaxAliases, + tableIdentifiers, + outputAliasReferences); + } + + private void collectOutputAliases( + SqlSelect select, + Set outputAliases, + Set syntaxAliases, + Set outputAliasReferences) { + if (select.getSelectList() == null) { + return; + } + for (SqlNode item : select.getSelectList()) { + if (!(item instanceof SqlCall call) + || call.getKind() != SqlKind.AS + || call.operandCount() < 2 + || !(call.operand(1) instanceof SqlIdentifier alias) + || alias.names.isEmpty()) { + continue; + } + int aliasIndex = alias.names.size() - 1; + outputAliases.add(identifierKey( + alias.names.get(aliasIndex), + alias.isComponentQuoted(aliasIndex))); + syntaxAliases.add(alias); + } + markOutputAliasReferences( + select.getOrderList(), outputAliases, outputAliasReferences); + markOutputAliasReferences( + select.getGroup(), outputAliases, outputAliasReferences); + } + + private void markOutputAliasReferences( + SqlNode node, + Set outputAliases, + Set outputAliasReferences) { + if (node == null) { + return; + } + if (node instanceof SqlNodeList nodes) { + for (SqlNode item : nodes) { + markOutputAliasReferences(item, outputAliases, outputAliasReferences); + } + return; + } + if (node instanceof SqlIdentifier identifier && identifier.names.size() == 1) { + if (outputAliases.contains(identifierKey( + identifier.names.get(0), identifier.isComponentQuoted(0)))) { + outputAliasReferences.add(identifier); + } + return; + } + if (node instanceof SqlCall call + && (call.getKind() == SqlKind.DESCENDING + || call.getKind() == SqlKind.NULLS_FIRST + || call.getKind() == SqlKind.NULLS_LAST) + && call.operandCount() > 0) { + markOutputAliasReferences( + call.operand(0), outputAliases, outputAliasReferences); + } + } + + private FromSummary collectFrom( + SqlNode from, + PolicySnapshot snapshot, + Set referencedTableIds, + Map commonTables, + Map> tableIdsByAlias, + Map> derivedColumnsByAlias, + Set syntaxAliases, + Set tableIdentifiers) { + if (from == null) { + return FromSummary.empty(); + } + if (from instanceof SqlIdentifier identifier) { + tableIdentifiers.add(identifier); + String relationKey = identifierKey( + identifier.names.get(identifier.names.size() - 1), + identifier.isComponentQuoted(identifier.names.size() - 1)); + DerivedRelation commonTable = commonTables.get(relationKey); + if (commonTable != null) { + tableIdsByAlias.put(relationKey, commonTable.tableIds()); + derivedColumnsByAlias.put( + relationKey, commonTable.outputColumns()); + return new FromSummary( + commonTable.tableIds(), commonTable.outputColumns(), true); + } + return new FromSummary( + resolveReferencedTableIds( + identifier, snapshot, referencedTableIds), + Set.of(), + false); + } + if (!(from instanceof SqlCall call)) { + return FromSummary.empty(); + } + if (call instanceof SqlSelect nestedSelect) { + FromSummary nested = collectFrom( + nestedSelect.getFrom(), snapshot, referencedTableIds, + commonTables, tableIdsByAlias, derivedColumnsByAlias, + syntaxAliases, tableIdentifiers); + return new FromSummary( + nested.tableIds(), outputColumnKeys(nestedSelect), true); + } + if (call instanceof SqlJoin join) { + FromSummary left = collectFrom( + join.getLeft(), snapshot, referencedTableIds, + commonTables, tableIdsByAlias, derivedColumnsByAlias, + syntaxAliases, tableIdentifiers); + FromSummary right = collectFrom( + join.getRight(), snapshot, referencedTableIds, + commonTables, tableIdsByAlias, derivedColumnsByAlias, + syntaxAliases, tableIdentifiers); + Set tableIds = new LinkedHashSet<>(left.tableIds()); + tableIds.addAll(right.tableIds()); + return new FromSummary(Set.copyOf(tableIds), Set.of(), false); + } + if (call.getKind() == SqlKind.AS && call.operandCount() >= 2) { + SqlNode relationNode = call.operand(0); + FromSummary relation = collectFrom( + relationNode, snapshot, referencedTableIds, + commonTables, tableIdsByAlias, derivedColumnsByAlias, + syntaxAliases, tableIdentifiers); + SqlNode aliasNode = call.operand(1); + if (aliasNode instanceof SqlIdentifier alias && !alias.names.isEmpty()) { + int aliasIndex = alias.names.size() - 1; + String aliasKey = identifierKey( + alias.names.get(aliasIndex), + alias.isComponentQuoted(aliasIndex)); + syntaxAliases.add(alias); + if (!relation.tableIds().isEmpty()) { + tableIdsByAlias.merge(aliasKey, relation.tableIds(), (left, right) -> { + Set merged = new LinkedHashSet<>(left); + merged.addAll(right); + return Set.copyOf(merged); + }); + } + if (relation.singleDerived() + && !relation.outputColumns().isEmpty()) { + derivedColumnsByAlias.put( + aliasKey, relation.outputColumns()); + } + } + return relation; + } + if (call instanceof SqlOrderBy orderBy) { + return collectFrom( + orderBy.query, snapshot, referencedTableIds, + commonTables, tableIdsByAlias, derivedColumnsByAlias, + syntaxAliases, tableIdentifiers); + } + if (call instanceof SqlWith sqlWith) { + return collectFrom( + sqlWith.body, snapshot, referencedTableIds, + commonTables, tableIdsByAlias, derivedColumnsByAlias, + syntaxAliases, tableIdentifiers); + } + return FromSummary.empty(); + } + + private Map collectCommonTables( + SqlNode root, + PolicySnapshot snapshot, + Set referencedTableIds) { + Map commonTables = new HashMap<>(); + collectCommonTables( + root, snapshot, referencedTableIds, commonTables); + return Map.copyOf(commonTables); + } + + private void collectCommonTables( + SqlNode node, + PolicySnapshot snapshot, + Set referencedTableIds, + Map commonTables) { + if (node instanceof SqlOrderBy orderBy) { + collectCommonTables( + orderBy.query, snapshot, referencedTableIds, commonTables); + return; + } + if (!(node instanceof SqlWith sqlWith)) { + return; + } + for (SqlNode withNode : sqlWith.withList) { + if (!(withNode instanceof SqlWithItem withItem) + || withItem.name == null + || withItem.name.names.isEmpty()) { + continue; + } + collectCommonTables( + withItem.query, snapshot, referencedTableIds, commonTables); + SqlSelect select = selectBody(withItem.query); + if (select == null) { + continue; + } + FromSummary source = collectFrom( + select.getFrom(), snapshot, referencedTableIds, + commonTables, new HashMap<>(), new HashMap<>(), + Collections.newSetFromMap(new IdentityHashMap<>()), + Collections.newSetFromMap(new IdentityHashMap<>())); + Set outputColumns = withItem.columnList == null + || withItem.columnList.isEmpty() + ? outputColumnKeys(select) + : identifierKeys(withItem.columnList); + int nameIndex = withItem.name.names.size() - 1; + commonTables.put( + identifierKey( + withItem.name.names.get(nameIndex), + withItem.name.isComponentQuoted(nameIndex)), + new DerivedRelation( + source.tableIds(), outputColumns)); + } + collectCommonTables( + sqlWith.body, snapshot, referencedTableIds, commonTables); + } + + private SqlSelect selectBody(SqlNode node) { + if (node instanceof SqlSelect select) { + return select; + } + if (node instanceof SqlOrderBy orderBy) { + return selectBody(orderBy.query); + } + if (node instanceof SqlWith sqlWith) { + return selectBody(sqlWith.body); + } + return null; + } + + private Set outputColumnKeys(SqlNode query) { + SqlSelect select = selectBody(query); + if (select == null || select.getSelectList() == null) { + return Set.of(); + } + Set outputColumns = new LinkedHashSet<>(); + for (SqlNode item : select.getSelectList()) { + if (item instanceof SqlCall call + && call.getKind() == SqlKind.AS + && call.operandCount() >= 2 + && call.operand(1) instanceof SqlIdentifier alias + && !alias.names.isEmpty()) { + int index = alias.names.size() - 1; + outputColumns.add(identifierKey( + alias.names.get(index), + alias.isComponentQuoted(index))); + continue; + } + if (item instanceof SqlIdentifier identifier + && !identifier.isStar() + && !identifier.names.isEmpty()) { + int index = identifier.names.size() - 1; + outputColumns.add(identifierKey( + identifier.names.get(index), + identifier.isComponentQuoted(index))); + } + } + return Set.copyOf(outputColumns); + } + + private Set identifierKeys(SqlNodeList identifiers) { + Set keys = new LinkedHashSet<>(); + for (SqlNode node : identifiers) { + if (node instanceof SqlIdentifier identifier + && !identifier.names.isEmpty()) { + int index = identifier.names.size() - 1; + keys.add(identifierKey( + identifier.names.get(index), + identifier.isComponentQuoted(index))); + } + } + return Set.copyOf(keys); + } + + private Set resolveReferencedTableIds( + SqlIdentifier identifier, + PolicySnapshot snapshot, + Set referencedTableIds) { + Set candidates = snapshot.tableIdsByName().get( + String.join(".", identifier.names)); + if (candidates == null || candidates.isEmpty()) { + return Set.of(); + } + Set referenced = new LinkedHashSet<>(candidates); + referenced.retainAll(referencedTableIds); + return Set.copyOf(referenced); + } + + private String identifierKey(String name, boolean quoted) { + return quoted ? "Q:" + name : "U:" + name.toUpperCase(Locale.ROOT); + } + + private boolean columnAllowed(ColumnPolicy policy, String columnName, boolean quoted) { + if (policy == null) { + return false; + } + return quoted + ? policy.allowedExact().contains(columnName) + : policy.allowedFolded().contains(columnName.toUpperCase(Locale.ROOT)); + } + + private boolean fieldAllowed(DatacenterTableField field) { + return Integer.valueOf(1).equals(field.getQueryable()) + && DatacenterMetadataStatus.ACTIVE.name().equals(field.getMetadataStatus()) + && DatacenterSensitivityLevel.PUBLIC.name().equals(field.getSensitivityLevel()); + } + + private void addTableName( + Map> idsByName, + Set tableReferenceNames, + String name, + BigInteger tableId) { + if (name == null || name.isBlank()) { + return; + } + idsByName.computeIfAbsent(name, ignored -> new LinkedHashSet<>()).add(tableId); + tableReferenceNames.add(name); + } + + private Set referencedTables(RelNode root) { + Set names = new LinkedHashSet<>(); + new RelVisitor() { + @Override + public void visit(RelNode node, int ordinal, RelNode parent) { + if (node instanceof TableScan scan) { + names.add(referencedTableName(scan)); + } + super.visit(node, ordinal, parent); + } + }.go(root); + return names; + } + + private String referencedTableName(TableScan scan) { + List qualified = scan.getTable().getQualifiedName(); + if (qualified.isEmpty()) { + return ""; + } + String table = qualified.get(qualified.size() - 1); + return qualified.size() >= 2 + ? qualified.get(qualified.size() - 2) + "." + table + : table; + } + + private BigInteger parseRecordId(String sourceId) { + int marker = sourceId.lastIndexOf("-source-"); + if (marker < 0) { + throw new FederationSqlException( + FederationSqlErrorCode.INVALID_ARGUMENT, + "invalid data source id"); + } + return new BigInteger(sourceId.substring(marker + "-source-".length())); + } + + private String sourceColumnName(DatacenterTableField field) { + return field.getSourceColumnName() == null || field.getSourceColumnName().isBlank() + ? field.getFieldName() : field.getSourceColumnName(); + } + + private FederationSqlException validationFailure(String message) { + return new FederationSqlException(FederationSqlErrorCode.SQL_VALIDATION_FAILED, message); + } + + private record PolicyCacheKey(BigInteger sourceId, String policyVersion) { + } + + private record AliasContext( + Map> tableIdsByAlias, + Map> derivedColumnsByAlias, + Set unqualifiedDerivedColumns, + Set outputAliases, + Set syntaxAliases, + Set tableIdentifiers, + Set outputAliasReferences) { + } + + private record DerivedRelation( + Set tableIds, + Set outputColumns) { + } + + private record FromSummary( + Set tableIds, + Set outputColumns, + boolean singleDerived) { + + private static FromSummary empty() { + return new FromSummary(Set.of(), Set.of(), false); + } + } + + private record ColumnPolicy( + Set knownExact, + Set knownFolded, + Set allowedExact, + Set allowedFolded) { + + private static ColumnPolicy mutable() { + return new ColumnPolicy(new HashSet<>(), new HashSet<>(), new HashSet<>(), new HashSet<>()); + } + } + + private record PolicySnapshot( + Map> tableIdsByName, + Map columnsByTable, + Set tableReferenceNames) { + + private static PolicySnapshot empty() { + return new PolicySnapshot(Map.of(), Map.of(), Set.of()); + } + } +} diff --git a/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/federation/DatacenterMetadataIdentity.java b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/federation/DatacenterMetadataIdentity.java new file mode 100644 index 00000000..f566fd92 --- /dev/null +++ b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/federation/DatacenterMetadataIdentity.java @@ -0,0 +1,174 @@ +package tech.easyflow.datacenter.federation; + +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.HexFormat; +import java.util.List; +import tech.easyflow.datacenter.entity.DatacenterTableField; + +/** + * 使用 presence 与长度前缀生成稳定元数据摘要。 + */ +public final class DatacenterMetadataIdentity { + + private DatacenterMetadataIdentity() { + } + + /** + * 计算物理命名空间摘要。 + * + * @param catalog 物理 Catalog + * @param schema 物理 Schema + * @return SHA-256 摘要 + */ + public static String namespaceKey(String catalog, String schema) { + return digest(List.of(nullable(catalog), nullable(schema))); + } + + /** + * 计算物理表身份摘要。 + * + * @param sourceId 数据源 ID + * @param namespaceKey 命名空间摘要 + * @param physicalTable 物理表名 + * @param tableKind 对象类型 + * @return SHA-256 摘要 + */ + public static String tableIdentity( + String sourceId, + String namespaceKey, + String physicalTable, + String tableKind) { + return digest(List.of( + "table-v1", + nullable(sourceId), + nullable(namespaceKey), + nullable(physicalTable), + nullable(tableKind))); + } + + /** + * 计算表结构观测摘要。 + * + * @param tableName 表名 + * @param tableKind 对象类型 + * @param fields 按 JDBC 顺序排列的字段 + * @return SHA-256 摘要 + */ + public static String tableFingerprint( + String tableName, + String tableKind, + List fields) { + MessageDigest digest = newDigest(); + update(digest, "table-metadata-v1"); + update(digest, nullable(tableName)); + update(digest, nullable(tableKind)); + List safeFields = fields == null ? List.of() : fields; + digest.update(ByteBuffer.allocate(Integer.BYTES).putInt(safeFields.size()).array()); + for (DatacenterTableField field : safeFields) { + update(digest, nullable(field.getSourceColumnName())); + update(digest, nullable(field.getNativeTypeName())); + update(digest, nullable(field.getJdbcTypeCode())); + update(digest, nullable(field.getPrecision())); + update(digest, nullable(field.getScale())); + update(digest, nullable(field.getRequired())); + } + return HexFormat.of().formatHex(digest.digest()); + } + + /** + * 计算单字段元数据摘要。 + * + * @param field 字段元数据 + * @return SHA-256 摘要 + */ + public static String fieldFingerprint(DatacenterTableField field) { + return digest(List.of( + "field-v1", + nullable(field.getSourceColumnName()), + nullable(field.getNativeTypeName()), + nullable(field.getJdbcTypeCode()), + nullable(field.getPrecision()), + nullable(field.getScale()), + nullable(field.getRequired()))); + } + + /** + * 复现 V58 的存量表身份回填算法，仅用于首次刷新时识别可安全升级的旧摘要。 + * + * @param sourceId 数据源 ID + * @param catalogId 目录 ID + * @param physicalTable 物理表名 + * @param tableKind 表类型 + * @return V58 SHA-256 摘要 + */ + public static String legacyV58TableIdentity( + Object sourceId, + Object catalogId, + String physicalTable, + String tableKind) { + String source = sourceId == null ? "0" : String.valueOf(sourceId); + String catalog = catalogId == null ? "0" : String.valueOf(catalogId); + String table = physicalTable == null ? "" : physicalTable; + String kind = tableKind == null ? "" : tableKind; + return rawDigest("S" + codePointLength(source) + ":" + source + + "C" + codePointLength(catalog) + ":" + catalog + + "T" + codePointLength(table) + ":" + table + + "K" + codePointLength(kind) + ":" + kind); + } + + /** + * 复现 V58 的存量表结构摘要回填算法。 + * + * @param tableName 表名 + * @param tableKind 表类型 + * @return V58 SHA-256 摘要 + */ + public static String legacyV58TableFingerprint(String tableName, String tableKind) { + String table = tableName == null ? "" : tableName; + String kind = tableKind == null ? "" : tableKind; + return rawDigest(codePointLength(table) + ":" + table + + codePointLength(kind) + ":" + kind); + } + + private static int codePointLength(String value) { + return value.codePointCount(0, value.length()); + } + + private static String rawDigest(String value) { + MessageDigest digest = newDigest(); + return HexFormat.of().formatHex(digest.digest(value.getBytes(StandardCharsets.UTF_8))); + } + + private static String digest(List values) { + MessageDigest digest = newDigest(); + for (String value : values) { + update(digest, value); + } + return HexFormat.of().formatHex(digest.digest()); + } + + private static void update(MessageDigest digest, String value) { + byte[] bytes = value.getBytes(StandardCharsets.UTF_8); + digest.update(ByteBuffer.allocate(Integer.BYTES).putInt(bytes.length).array()); + digest.update(bytes); + } + + private static String nullable(Object value) { + if (value == null) { + return "N"; + } + String text = String.valueOf(value); + return "V" + text.length() + ":" + text; + } + + private static MessageDigest newDigest() { + try { + return MessageDigest.getInstance("SHA-256"); + } catch (NoSuchAlgorithmException exception) { + throw new IllegalStateException("SHA-256 is not available", exception); + } + } +} diff --git a/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/integration/AssistantDatacenterBridge.java b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/integration/AssistantDatacenterBridge.java index a006f57d..f7d565a2 100644 --- a/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/integration/AssistantDatacenterBridge.java +++ b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/integration/AssistantDatacenterBridge.java @@ -1,7 +1,18 @@ package tech.easyflow.datacenter.integration; +import tech.easyflow.common.entity.LoginAccount; import tech.easyflow.datacenter.execution.model.DatacenterQueryRequest; public interface AssistantDatacenterBridge { - AssistantDatacenterResult queryPage(DatacenterQueryRequest request); + + /** + * 使用明确执行账号查询数据中枢。 + * + * @param request 查询请求 + * @param account 执行账号 + * @return 查询结果 + */ + AssistantDatacenterResult queryPage( + DatacenterQueryRequest request, + LoginAccount account); } diff --git a/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/integration/AssistantDatacenterResult.java b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/integration/AssistantDatacenterResult.java index e3fe9d32..d2bd29ef 100644 --- a/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/integration/AssistantDatacenterResult.java +++ b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/integration/AssistantDatacenterResult.java @@ -4,7 +4,7 @@ import com.mybatisflex.core.row.Row; import tech.easyflow.datacenter.entity.DatacenterTable; import tech.easyflow.datacenter.meta.entity.DatacenterCatalog; import tech.easyflow.datacenter.meta.entity.DatacenterDatasetVersion; -import tech.easyflow.datacenter.meta.entity.DatacenterSource; +import tech.easyflow.datacenter.meta.model.DatacenterSourceView; import java.util.ArrayList; import java.util.LinkedHashMap; @@ -13,7 +13,7 @@ import java.util.Map; public class AssistantDatacenterResult { private List rows = new ArrayList<>(); - private DatacenterSource source; + private DatacenterSourceView source; private DatacenterCatalog catalog; private DatacenterTable table; private DatacenterDatasetVersion version; @@ -21,8 +21,8 @@ public class AssistantDatacenterResult { public List getRows() { return rows; } public void setRows(List rows) { this.rows = rows; } - public DatacenterSource getSource() { return source; } - public void setSource(DatacenterSource source) { this.source = source; } + public DatacenterSourceView getSource() { return source; } + public void setSource(DatacenterSourceView source) { this.source = source; } public DatacenterCatalog getCatalog() { return catalog; } public void setCatalog(DatacenterCatalog catalog) { this.catalog = catalog; } public DatacenterTable getTable() { return table; } diff --git a/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/integration/DefaultAssistantDatacenterBridge.java b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/integration/DefaultAssistantDatacenterBridge.java index b78e1492..2524b44a 100644 --- a/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/integration/DefaultAssistantDatacenterBridge.java +++ b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/integration/DefaultAssistantDatacenterBridge.java @@ -1,6 +1,7 @@ package tech.easyflow.datacenter.integration; import org.springframework.stereotype.Component; +import tech.easyflow.common.entity.LoginAccount; import tech.easyflow.datacenter.execution.model.DatacenterQueryRequest; import tech.easyflow.datacenter.execution.model.DatacenterSchemaResponse; import tech.easyflow.datacenter.execution.service.DatacenterDatasetQueryService; @@ -17,8 +18,10 @@ public class DefaultAssistantDatacenterBridge implements AssistantDatacenterBrid private DatacenterDatasetQueryService queryService; @Override - public AssistantDatacenterResult queryPage(DatacenterQueryRequest request) { - var page = queryService.queryPage(request); + public AssistantDatacenterResult queryPage( + DatacenterQueryRequest request, + LoginAccount account) { + var page = queryService.queryPage(request, account); DatacenterSchemaResponse schema = queryService.getSchema(request.getDatasetRef()); AssistantDatacenterResult result = new AssistantDatacenterResult(); result.setRows(page.getRecords()); diff --git a/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/mapper/DatacenterQueryAuditMapper.java b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/mapper/DatacenterQueryAuditMapper.java new file mode 100644 index 00000000..54989c13 --- /dev/null +++ b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/mapper/DatacenterQueryAuditMapper.java @@ -0,0 +1,10 @@ +package tech.easyflow.datacenter.mapper; + +import com.mybatisflex.core.BaseMapper; +import tech.easyflow.datacenter.audit.DatacenterQueryAudit; + +/** + * 数据中枢查询审计 Mapper。 + */ +public interface DatacenterQueryAuditMapper extends BaseMapper { +} diff --git a/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/meta/entity/DatacenterCatalog.java b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/meta/entity/DatacenterCatalog.java index 45b838f4..f9ea5572 100644 --- a/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/meta/entity/DatacenterCatalog.java +++ b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/meta/entity/DatacenterCatalog.java @@ -31,6 +31,18 @@ public class DatacenterCatalog extends DateEntity implements Serializable { private String catalogDesc; @Column(comment = "目录类型") private String catalogType; + @Column(comment = "Calcite 逻辑 Schema") + private String logicalSchemaName; + @Column(comment = "JDBC 物理 Catalog") + private String physicalCatalogName; + @Column(comment = "JDBC 物理 Schema") + private String physicalSchemaName; + @Column(comment = "物理命名空间稳定摘要") + private String namespaceKey; + @Column(comment = "元数据状态") + private String metadataStatus; + @Column(comment = "最近发现时间") + private Date lastSeenAt; @Column(comment = "状态") private Integer status; @Column(typeHandler = FastjsonTypeHandler.class, comment = "扩展项") @@ -59,6 +71,18 @@ public class DatacenterCatalog extends DateEntity implements Serializable { public void setCatalogDesc(String catalogDesc) { this.catalogDesc = catalogDesc; } public String getCatalogType() { return catalogType; } public void setCatalogType(String catalogType) { this.catalogType = catalogType; } + public String getLogicalSchemaName() { return logicalSchemaName; } + public void setLogicalSchemaName(String logicalSchemaName) { this.logicalSchemaName = logicalSchemaName; } + public String getPhysicalCatalogName() { return physicalCatalogName; } + public void setPhysicalCatalogName(String physicalCatalogName) { this.physicalCatalogName = physicalCatalogName; } + public String getPhysicalSchemaName() { return physicalSchemaName; } + public void setPhysicalSchemaName(String physicalSchemaName) { this.physicalSchemaName = physicalSchemaName; } + public String getNamespaceKey() { return namespaceKey; } + public void setNamespaceKey(String namespaceKey) { this.namespaceKey = namespaceKey; } + public String getMetadataStatus() { return metadataStatus; } + public void setMetadataStatus(String metadataStatus) { this.metadataStatus = metadataStatus; } + public Date getLastSeenAt() { return lastSeenAt; } + public void setLastSeenAt(Date lastSeenAt) { this.lastSeenAt = lastSeenAt; } public Integer getStatus() { return status; } public void setStatus(Integer status) { this.status = status; } public Map getOptions() { return options; } diff --git a/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/meta/entity/DatacenterSource.java b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/meta/entity/DatacenterSource.java index 01261e5b..bb4c4ffd 100644 --- a/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/meta/entity/DatacenterSource.java +++ b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/meta/entity/DatacenterSource.java @@ -1,5 +1,6 @@ package tech.easyflow.datacenter.meta.entity; +import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.databind.annotation.JsonSerialize; import com.fasterxml.jackson.databind.ser.std.ToStringSerializer; import com.mybatisflex.annotation.Column; @@ -29,6 +30,30 @@ public class DatacenterSource extends DateEntity implements Serializable { private String sourceCode; @Column(comment = "数据源类型") private String sourceType; + @Column(comment = "Federation Adapter 标识") + private String adapterId; + @Column(typeHandler = FastjsonTypeHandler.class, comment = "不含凭据的 Adapter 选项") + private Map adapterOptionsJson; + @Column(comment = "数据源 Definition 版本") + private Long definitionRevision; + @Column(comment = "Definition SHA-256 校验和") + private String definitionChecksum; + @Column(comment = "纳管范围版本") + private Long scopeRevision; + @Column(comment = "Adapter 兼容状态") + private String compatibilityStatus; + @Column(comment = "数据库产品名称") + private String databaseProductName; + @Column(comment = "数据库产品版本") + private String databaseProductVersion; + @Column(comment = "JDBC Driver 名称") + private String driverName; + @Column(comment = "JDBC Driver 版本") + private String driverVersion; + @Column(comment = "元数据刷新状态") + private String metadataRefreshStatus; + @Column(comment = "元数据最近刷新时间") + private Date metadataRefreshedAt; @Column(comment = "访问模式") private String accessMode; @Column(comment = "是否内置") @@ -85,6 +110,30 @@ public class DatacenterSource extends DateEntity implements Serializable { public void setSourceCode(String sourceCode) { this.sourceCode = sourceCode; } public String getSourceType() { return sourceType; } public void setSourceType(String sourceType) { this.sourceType = sourceType; } + public String getAdapterId() { return adapterId; } + public void setAdapterId(String adapterId) { this.adapterId = adapterId; } + public Map getAdapterOptionsJson() { return adapterOptionsJson; } + public void setAdapterOptionsJson(Map adapterOptionsJson) { this.adapterOptionsJson = adapterOptionsJson; } + public Long getDefinitionRevision() { return definitionRevision; } + public void setDefinitionRevision(Long definitionRevision) { this.definitionRevision = definitionRevision; } + public String getDefinitionChecksum() { return definitionChecksum; } + public void setDefinitionChecksum(String definitionChecksum) { this.definitionChecksum = definitionChecksum; } + public Long getScopeRevision() { return scopeRevision; } + public void setScopeRevision(Long scopeRevision) { this.scopeRevision = scopeRevision; } + public String getCompatibilityStatus() { return compatibilityStatus; } + public void setCompatibilityStatus(String compatibilityStatus) { this.compatibilityStatus = compatibilityStatus; } + public String getDatabaseProductName() { return databaseProductName; } + public void setDatabaseProductName(String databaseProductName) { this.databaseProductName = databaseProductName; } + public String getDatabaseProductVersion() { return databaseProductVersion; } + public void setDatabaseProductVersion(String databaseProductVersion) { this.databaseProductVersion = databaseProductVersion; } + public String getDriverName() { return driverName; } + public void setDriverName(String driverName) { this.driverName = driverName; } + public String getDriverVersion() { return driverVersion; } + public void setDriverVersion(String driverVersion) { this.driverVersion = driverVersion; } + public String getMetadataRefreshStatus() { return metadataRefreshStatus; } + public void setMetadataRefreshStatus(String metadataRefreshStatus) { this.metadataRefreshStatus = metadataRefreshStatus; } + public Date getMetadataRefreshedAt() { return metadataRefreshedAt; } + public void setMetadataRefreshedAt(Date metadataRefreshedAt) { this.metadataRefreshedAt = metadataRefreshedAt; } public String getAccessMode() { return accessMode; } public void setAccessMode(String accessMode) { this.accessMode = accessMode; } public Integer getBuiltinFlag() { return builtinFlag; } @@ -103,8 +152,10 @@ public class DatacenterSource extends DateEntity implements Serializable { public void setSchemaName(String schemaName) { this.schemaName = schemaName; } public String getUsername() { return username; } public void setUsername(String username) { this.username = username; } + @JsonIgnore public String getCredentialCipher() { return credentialCipher; } public void setCredentialCipher(String credentialCipher) { this.credentialCipher = credentialCipher; } + @JsonIgnore public Map getConfigJson() { return configJson; } public void setConfigJson(Map configJson) { this.configJson = configJson; } public Map getCapabilitiesJson() { return capabilitiesJson; } diff --git a/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/meta/enums/DatacenterMetadataStatus.java b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/meta/enums/DatacenterMetadataStatus.java new file mode 100644 index 00000000..402c51a4 --- /dev/null +++ b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/meta/enums/DatacenterMetadataStatus.java @@ -0,0 +1,15 @@ +package tech.easyflow.datacenter.meta.enums; + +/** + * 数据库对象元数据状态。 + */ +public enum DatacenterMetadataStatus { + /** 当前仍能从目标数据库发现。 */ + ACTIVE, + /** 本次刷新未发现。 */ + MISSING, + /** 结构已变化，等待用户确认。 */ + CHANGED, + /** 已由用户停止纳管。 */ + RETIRED +} diff --git a/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/meta/enums/DatacenterSensitivityLevel.java b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/meta/enums/DatacenterSensitivityLevel.java new file mode 100644 index 00000000..19323695 --- /dev/null +++ b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/meta/enums/DatacenterSensitivityLevel.java @@ -0,0 +1,15 @@ +package tech.easyflow.datacenter.meta.enums; + +/** + * 数据字段敏感级别。 + */ +public enum DatacenterSensitivityLevel { + /** 可直接用于受控查询。 */ + PUBLIC, + /** 仅限组织内部。 */ + INTERNAL, + /** 需要脱敏后使用。 */ + SENSITIVE, + /** 默认禁止进入查询结果。 */ + RESTRICTED +} diff --git a/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/meta/enums/DatacenterSourceStatus.java b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/meta/enums/DatacenterSourceStatus.java new file mode 100644 index 00000000..7f7280c9 --- /dev/null +++ b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/meta/enums/DatacenterSourceStatus.java @@ -0,0 +1,56 @@ +package tech.easyflow.datacenter.meta.enums; + +import java.util.Arrays; + +/** + * 数据源持久化生命周期。 + */ +public enum DatacenterSourceStatus { + + /** 配置已保存，尚未激活。 */ + DRAFT(0), + /** Definition 已发布且节点可查询。 */ + READY(1), + /** 配置仍保留，但当前 Runtime 不可用。 */ + DEGRADED(2), + /** 管理员主动禁用。 */ + DISABLED(3), + /** 删除墓碑，禁止新查询。 */ + DELETED(4); + + private final int code; + + /** + * 创建状态。 + * + * @param code 持久化编码 + */ + DatacenterSourceStatus(int code) { + this.code = code; + } + + /** + * 返回持久化编码。 + * + * @return 状态编码 + */ + public int code() { + return code; + } + + /** + * 按持久化编码解析状态。 + * + * @param code 状态编码 + * @return 对应状态；未知编码按草稿处理 + */ + public static DatacenterSourceStatus fromCode(Integer code) { + if (code == null) { + return DRAFT; + } + return Arrays.stream(values()) + .filter(status -> status.code == code) + .findFirst() + .orElse(DRAFT); + } +} diff --git a/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/meta/model/DatacenterCatalogMeta.java b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/meta/model/DatacenterCatalogMeta.java index e1fe9a3a..77fb6578 100644 --- a/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/meta/model/DatacenterCatalogMeta.java +++ b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/meta/model/DatacenterCatalogMeta.java @@ -14,6 +14,9 @@ public class DatacenterCatalogMeta { private String catalogName; private String catalogType; private String catalogDesc; + private String logicalSchemaName; + private String physicalCatalogName; + private String physicalSchemaName; @JsonSerialize(using = ToStringSerializer.class) public BigInteger getId() { return id; } @@ -27,4 +30,10 @@ public class DatacenterCatalogMeta { public void setCatalogType(String catalogType) { this.catalogType = catalogType; } public String getCatalogDesc() { return catalogDesc; } public void setCatalogDesc(String catalogDesc) { this.catalogDesc = catalogDesc; } + public String getLogicalSchemaName() { return logicalSchemaName; } + public void setLogicalSchemaName(String logicalSchemaName) { this.logicalSchemaName = logicalSchemaName; } + public String getPhysicalCatalogName() { return physicalCatalogName; } + public void setPhysicalCatalogName(String physicalCatalogName) { this.physicalCatalogName = physicalCatalogName; } + public String getPhysicalSchemaName() { return physicalSchemaName; } + public void setPhysicalSchemaName(String physicalSchemaName) { this.physicalSchemaName = physicalSchemaName; } } diff --git a/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/meta/model/DatacenterFieldDescriptionUpdate.java b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/meta/model/DatacenterFieldDescriptionUpdate.java index b4fb2676..7f2e74c9 100644 --- a/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/meta/model/DatacenterFieldDescriptionUpdate.java +++ b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/meta/model/DatacenterFieldDescriptionUpdate.java @@ -2,24 +2,85 @@ package tech.easyflow.datacenter.meta.model; import java.math.BigInteger; +/** + * 字段说明与只读查询治理更新。 + */ public class DatacenterFieldDescriptionUpdate { private BigInteger fieldId; private String fieldDesc; + private Integer queryable; + private String sensitivityLevel; + /** + * 获取字段 ID。 + * + * @return 字段 ID + */ public BigInteger getFieldId() { return fieldId; } + /** + * 设置字段 ID。 + * + * @param fieldId 字段 ID + */ public void setFieldId(BigInteger fieldId) { this.fieldId = fieldId; } + /** + * 获取字段说明。 + * + * @return 字段说明 + */ public String getFieldDesc() { return fieldDesc; } + /** + * 设置字段说明。 + * + * @param fieldDesc 字段说明 + */ public void setFieldDesc(String fieldDesc) { this.fieldDesc = fieldDesc; } + + /** + * 获取是否允许只读查询。 + * + * @return 1 为允许，0 为禁止，空表示保持原值 + */ + public Integer getQueryable() { + return queryable; + } + + /** + * 设置是否允许只读查询。 + * + * @param queryable 1 为允许，0 为禁止，空表示保持原值 + */ + public void setQueryable(Integer queryable) { + this.queryable = queryable; + } + + /** + * 获取敏感级别。 + * + * @return 敏感级别，空表示保持原值 + */ + public String getSensitivityLevel() { + return sensitivityLevel; + } + + /** + * 设置敏感级别。 + * + * @param sensitivityLevel PUBLIC、INTERNAL、SENSITIVE 或 RESTRICTED + */ + public void setSensitivityLevel(String sensitivityLevel) { + this.sensitivityLevel = sensitivityLevel; + } } diff --git a/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/meta/model/DatacenterManagedMetadataSnapshot.java b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/meta/model/DatacenterManagedMetadataSnapshot.java new file mode 100644 index 00000000..43c792f2 --- /dev/null +++ b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/meta/model/DatacenterManagedMetadataSnapshot.java @@ -0,0 +1,24 @@ +package tech.easyflow.datacenter.meta.model; + +import java.util.List; +import java.util.Set; + +/** + * 一次连接内读取的已纳管表元数据快照。 + * + * @param details 当前仍存在的表详情 + * @param missingTableNames 当前连接中未发现的表名 + */ +public record DatacenterManagedMetadataSnapshot( + List details, + Set missingTableNames) { + + /** + * 对返回集合做防御性复制。 + */ + public DatacenterManagedMetadataSnapshot { + details = List.copyOf(details == null ? List.of() : details); + missingTableNames = Set.copyOf( + missingTableNames == null ? Set.of() : missingTableNames); + } +} diff --git a/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/meta/model/DatacenterMetadataPage.java b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/meta/model/DatacenterMetadataPage.java new file mode 100644 index 00000000..d48d30fe --- /dev/null +++ b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/meta/model/DatacenterMetadataPage.java @@ -0,0 +1,55 @@ +package tech.easyflow.datacenter.meta.model; + +import java.util.List; + +/** + * 有界元数据分页结果。 + * + * @param records 当前页记录 + * @param pageNumber 页码，从 1 开始 + * @param pageSize 每页记录上限 + * @param hasMore 是否还有下一页 + * @param 元数据类型 + */ +public record DatacenterMetadataPage( + List records, + long pageNumber, + long pageSize, + boolean hasMore) { + + /** + * 防御性复制当前页记录。 + */ + public DatacenterMetadataPage { + records = List.copyOf(records == null ? List.of() : records); + } + + /** + * 从已加载记录中创建有界分页结果。 + * + * @param records 全部候选记录 + * @param pageNumber 页码 + * @param pageSize 每页大小 + * @param 元数据类型 + * @return 当前页结果 + */ + public static DatacenterMetadataPage slice( + List records, + long pageNumber, + long pageSize) { + List safeRecords = records == null ? List.of() : records; + long safePage = Math.max(1L, pageNumber); + long safeSize = Math.max(1L, pageSize); + if (safePage - 1L > Long.MAX_VALUE / safeSize) { + return new DatacenterMetadataPage<>(List.of(), safePage, safeSize, false); + } + long offset = (safePage - 1L) * safeSize; + if (offset >= safeRecords.size()) { + return new DatacenterMetadataPage<>(List.of(), safePage, safeSize, false); + } + int from = Math.toIntExact(offset); + int to = (int) Math.min((long) safeRecords.size(), offset + safeSize); + return new DatacenterMetadataPage<>( + safeRecords.subList(from, to), safePage, safeSize, to < safeRecords.size()); + } +} diff --git a/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/meta/model/DatacenterSaveDescriptionsRequest.java b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/meta/model/DatacenterSaveDescriptionsRequest.java index b0968a76..07cfe4c4 100644 --- a/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/meta/model/DatacenterSaveDescriptionsRequest.java +++ b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/meta/model/DatacenterSaveDescriptionsRequest.java @@ -9,6 +9,8 @@ public class DatacenterSaveDescriptionsRequest { private BigInteger tableId; private String tableDesc; private List fields = new ArrayList<>(); + private Long fieldPageNumber; + private Long fieldPageSize; public BigInteger getTableId() { return tableId; @@ -33,4 +35,24 @@ public class DatacenterSaveDescriptionsRequest { public void setFields(List fields) { this.fields = fields; } + + /** @return 保存后返回的字段页码 */ + public Long getFieldPageNumber() { + return fieldPageNumber; + } + + /** @param fieldPageNumber 保存后返回的字段页码 */ + public void setFieldPageNumber(Long fieldPageNumber) { + this.fieldPageNumber = fieldPageNumber; + } + + /** @return 保存后返回的字段页大小 */ + public Long getFieldPageSize() { + return fieldPageSize; + } + + /** @param fieldPageSize 保存后返回的字段页大小 */ + public void setFieldPageSize(Long fieldPageSize) { + this.fieldPageSize = fieldPageSize; + } } diff --git a/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/meta/model/DatacenterSourceActivateRequest.java b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/meta/model/DatacenterSourceActivateRequest.java new file mode 100644 index 00000000..d1b20b19 --- /dev/null +++ b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/meta/model/DatacenterSourceActivateRequest.java @@ -0,0 +1,26 @@ +package tech.easyflow.datacenter.meta.model; + +import java.math.BigInteger; +import java.util.List; + +/** + * 激活数据源并纳管选定对象的请求。 + * + * @param sourceId 数据源 ID + * @param catalogName 物理 Catalog 或 Schema 名称 + * @param tableNames 需要纳管的表或视图 + * @param prewarm 是否立即预热节点本地连接池 + */ +public record DatacenterSourceActivateRequest( + BigInteger sourceId, + String catalogName, + List tableNames, + boolean prewarm) { + + /** + * 防御性复制表名。 + */ + public DatacenterSourceActivateRequest { + tableNames = List.copyOf(tableNames == null ? List.of() : tableNames); + } +} diff --git a/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/meta/model/DatacenterSourceCandidateCatalogRequest.java b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/meta/model/DatacenterSourceCandidateCatalogRequest.java new file mode 100644 index 00000000..e15a8ed9 --- /dev/null +++ b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/meta/model/DatacenterSourceCandidateCatalogRequest.java @@ -0,0 +1,16 @@ +package tech.easyflow.datacenter.meta.model; + +/** + * 未发布候选连接的命名空间分页请求。 + * + * @param definition 候选连接配置 + * @param keyword 命名空间搜索词 + * @param pageNumber 页码 + * @param pageSize 每页大小 + */ +public record DatacenterSourceCandidateCatalogRequest( + DatacenterSourceDraftRequest definition, + String keyword, + Long pageNumber, + Long pageSize) { +} diff --git a/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/meta/model/DatacenterSourceCandidateMetadataRequest.java b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/meta/model/DatacenterSourceCandidateMetadataRequest.java new file mode 100644 index 00000000..f46cfd13 --- /dev/null +++ b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/meta/model/DatacenterSourceCandidateMetadataRequest.java @@ -0,0 +1,18 @@ +package tech.easyflow.datacenter.meta.model; + +/** + * 活动数据源候选配置的元数据浏览请求。 + * + * @param definition 候选连接配置 + * @param catalogName 目标 Catalog 或 Schema + * @param keyword 表名搜索词 + * @param pageNumber 页码 + * @param pageSize 每页大小 + */ +public record DatacenterSourceCandidateMetadataRequest( + DatacenterSourceDraftRequest definition, + String catalogName, + String keyword, + Long pageNumber, + Long pageSize) { +} diff --git a/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/meta/model/DatacenterSourceDraftRequest.java b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/meta/model/DatacenterSourceDraftRequest.java new file mode 100644 index 00000000..c8f0e300 --- /dev/null +++ b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/meta/model/DatacenterSourceDraftRequest.java @@ -0,0 +1,47 @@ +package tech.easyflow.datacenter.meta.model; + +import java.math.BigInteger; +import java.util.Map; + +/** + * 创建或更新数据源草稿的受控请求。 + * + * @param id 编辑时的数据源 ID + * @param sourceName 连接名称 + * @param sourceCode 租户内业务编码 + * @param sourceType 数据源类型 + * @param host 主机地址 + * @param port 端口 + * @param databaseName 数据库名 + * @param schemaName 默认 Schema + * @param username 用户名 + * @param password 本次提交的明文密码；只在请求内存中短暂存在 + * @param driverClassName JDBC Driver 类名 + * @param jdbcUrl JDBC URL + * @param adapterOptions 不含凭据的 Adapter 选项 + * @param configJson 不含凭据的连接扩展项 + */ +public record DatacenterSourceDraftRequest( + BigInteger id, + String sourceName, + String sourceCode, + String sourceType, + String host, + Integer port, + String databaseName, + String schemaName, + String username, + String password, + String driverClassName, + String jdbcUrl, + Map adapterOptions, + Map configJson) { + + /** + * 防御性复制可变配置。 + */ + public DatacenterSourceDraftRequest { + adapterOptions = Map.copyOf(adapterOptions == null ? Map.of() : adapterOptions); + configJson = Map.copyOf(configJson == null ? Map.of() : configJson); + } +} diff --git a/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/meta/model/DatacenterSourceReconfigureRequest.java b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/meta/model/DatacenterSourceReconfigureRequest.java new file mode 100644 index 00000000..190d08f9 --- /dev/null +++ b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/meta/model/DatacenterSourceReconfigureRequest.java @@ -0,0 +1,29 @@ +package tech.easyflow.datacenter.meta.model; + +import java.util.List; + +/** + * 活动数据源原地重配置请求。 + * + * @param definition 候选连接配置 + * @param catalogName 显式选择的 Catalog 或 Schema + * @param tableNames 显式选择的表或视图 + * @param prewarm 是否在发布后预热 + * @param expectedDefinitionRevision 调用方读取到的 Definition 版本 + * @param expectedScopeRevision 调用方读取到的范围版本 + */ +public record DatacenterSourceReconfigureRequest( + DatacenterSourceDraftRequest definition, + String catalogName, + List tableNames, + boolean prewarm, + Long expectedDefinitionRevision, + Long expectedScopeRevision) { + + /** + * 防御性复制表名集合。 + */ + public DatacenterSourceReconfigureRequest { + tableNames = List.copyOf(tableNames == null ? List.of() : tableNames); + } +} diff --git a/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/meta/model/DatacenterSourceView.java b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/meta/model/DatacenterSourceView.java new file mode 100644 index 00000000..593f5f9b --- /dev/null +++ b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/meta/model/DatacenterSourceView.java @@ -0,0 +1,75 @@ +package tech.easyflow.datacenter.meta.model; + +import com.fasterxml.jackson.databind.annotation.JsonSerialize; +import com.fasterxml.jackson.databind.ser.std.ToStringSerializer; +import java.math.BigInteger; +import java.util.Date; +import java.util.Map; + +/** + * 不含凭据、密文和内部 Runtime 对象的数据源视图。 + * + * @param id 数据源 ID + * @param sourceName 连接名称 + * @param sourceCode 业务编码 + * @param sourceType 数据源类型 + * @param status 生命周期名称 + * @param builtinFlag 是否内置 + * @param host 主机 + * @param port 端口 + * @param databaseName 数据库名 + * @param schemaName 默认 Schema + * @param username 用户名 + * @param driverClassName Driver 类名 + * @param jdbcUrl JDBC URL + * @param passwordConfigured 是否已保存密码 + * @param definitionRevision Definition 版本 + * @param scopeRevision 纳管范围版本 + * @param compatibilityStatus 兼容状态 + * @param databaseProductName 数据库产品 + * @param databaseProductVersion 数据库版本 + * @param driverName 实际 Driver 名称 + * @param driverVersion 实际 Driver 版本 + * @param lastTestStatus 最近测试状态 + * @param lastTestMessage 最近测试说明 + * @param lastTestedAt 最近测试时间 + * @param metadataRefreshStatus 元数据刷新状态 + * @param metadataRefreshedAt 元数据刷新时间 + * @param adapterOptions 不含凭据的 Adapter 选项 + */ +public record DatacenterSourceView( + @JsonSerialize(using = ToStringSerializer.class) BigInteger id, + String sourceName, + String sourceCode, + String sourceType, + String status, + Integer builtinFlag, + String host, + Integer port, + String databaseName, + String schemaName, + String username, + String driverClassName, + String jdbcUrl, + boolean passwordConfigured, + long definitionRevision, + long scopeRevision, + String compatibilityStatus, + String databaseProductName, + String databaseProductVersion, + String driverName, + String driverVersion, + String lastTestStatus, + String lastTestMessage, + Date lastTestedAt, + String metadataRefreshStatus, + Date metadataRefreshedAt, + Map adapterOptions) { + + /** + * 防御性复制 Adapter 选项。 + */ + public DatacenterSourceView { + adapterOptions = Map.copyOf(adapterOptions == null ? Map.of() : adapterOptions); + } +} diff --git a/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/meta/model/DatacenterSourceViews.java b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/meta/model/DatacenterSourceViews.java new file mode 100644 index 00000000..a967700c --- /dev/null +++ b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/meta/model/DatacenterSourceViews.java @@ -0,0 +1,59 @@ +package tech.easyflow.datacenter.meta.model; + +import tech.easyflow.datacenter.meta.entity.DatacenterSource; +import tech.easyflow.datacenter.meta.enums.DatacenterSourceStatus; + +/** + * 将持久化数据源实体转换为安全 API 视图。 + */ +public final class DatacenterSourceViews { + + private DatacenterSourceViews() { + } + + /** + * 创建不含凭据与内部对象的数据源视图。 + * + * @param source 持久化数据源实体 + * @return 安全数据源视图 + * @throws IllegalArgumentException 数据源为空时抛出 + */ + public static DatacenterSourceView from(DatacenterSource source) { + if (source == null) { + throw new IllegalArgumentException("source must not be null"); + } + return new DatacenterSourceView( + source.getId(), + source.getSourceName(), + source.getSourceCode(), + source.getSourceType(), + DatacenterSourceStatus.fromCode(source.getStatus()).name(), + source.getBuiltinFlag(), + source.getHost(), + source.getPort(), + source.getDatabaseName(), + source.getSchemaName(), + source.getUsername(), + source.getDriverClassName(), + source.getJdbcUrl(), + source.getCredentialCipher() != null + && !source.getCredentialCipher().isBlank(), + value(source.getDefinitionRevision()), + Math.max(1L, value(source.getScopeRevision())), + source.getCompatibilityStatus(), + source.getDatabaseProductName(), + source.getDatabaseProductVersion(), + source.getDriverName(), + source.getDriverVersion(), + source.getLastTestStatus(), + source.getLastTestMessage(), + source.getLastTestedAt(), + source.getMetadataRefreshStatus(), + source.getMetadataRefreshedAt(), + source.getAdapterOptionsJson()); + } + + private static long value(Long value) { + return value == null ? 0L : value; + } +} diff --git a/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/meta/model/DatacenterTableDetailMeta.java b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/meta/model/DatacenterTableDetailMeta.java index 572c59bd..c3cdeb10 100644 --- a/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/meta/model/DatacenterTableDetailMeta.java +++ b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/meta/model/DatacenterTableDetailMeta.java @@ -6,12 +6,36 @@ import tech.easyflow.datacenter.entity.DatacenterTableField; import java.util.ArrayList; import java.util.List; +/** + * 数据表及其有界字段详情。 + */ public class DatacenterTableDetailMeta { private DatacenterTable table; private List fields = new ArrayList<>(); + private long fieldPageNumber = 1L; + private long fieldPageSize; + private boolean hasMoreFields; + /** @return 数据表元数据 */ public DatacenterTable getTable() { return table; } + /** @param table 数据表元数据 */ public void setTable(DatacenterTable table) { this.table = table; } + /** @return 当前页字段 */ public List getFields() { return fields; } - public void setFields(List fields) { this.fields = fields; } + /** @param fields 当前页字段 */ + public void setFields(List fields) { + this.fields = fields == null ? new ArrayList<>() : new ArrayList<>(fields); + } + /** @return 字段页码 */ + public long getFieldPageNumber() { return fieldPageNumber; } + /** @param fieldPageNumber 字段页码 */ + public void setFieldPageNumber(long fieldPageNumber) { this.fieldPageNumber = fieldPageNumber; } + /** @return 字段页大小 */ + public long getFieldPageSize() { return fieldPageSize; } + /** @param fieldPageSize 字段页大小 */ + public void setFieldPageSize(long fieldPageSize) { this.fieldPageSize = fieldPageSize; } + /** @return 是否还有字段 */ + public boolean isHasMoreFields() { return hasMoreFields; } + /** @param hasMoreFields 是否还有字段 */ + public void setHasMoreFields(boolean hasMoreFields) { this.hasMoreFields = hasMoreFields; } } diff --git a/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/meta/service/DatacenterDatasetRegistryService.java b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/meta/service/DatacenterDatasetRegistryService.java index 1d972e5a..73385cc1 100644 --- a/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/meta/service/DatacenterDatasetRegistryService.java +++ b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/meta/service/DatacenterDatasetRegistryService.java @@ -20,6 +20,21 @@ public interface DatacenterDatasetRegistryService { DatacenterTable registerTable(DatacenterSource source, DatacenterCatalog catalog, DatacenterTableDetailMeta detail, LoginAccount account); + /** + * 在管理员显式重配置连接时重新绑定物理表身份和字段结构。 + * + * @param source 新数据源 Definition + * @param catalog 新物理命名空间 + * @param detail 新物理表元数据 + * @param account 操作账号 + * @return 重绑定后的纳管表 + */ + DatacenterTable rebindTable( + DatacenterSource source, + DatacenterCatalog catalog, + DatacenterTableDetailMeta detail, + LoginAccount account); + DatacenterTable getTableWithFields(BigInteger tableId); List getFields(BigInteger tableId); @@ -34,5 +49,35 @@ public interface DatacenterDatasetRegistryService { int removeTables(List tableIds); + /** + * 刷新一个已纳管表的观测元数据，不自动插入新表或新字段。 + * + * @param source 当前数据源 + * @param catalog 已纳管命名空间 + * @param tableId 已纳管表 ID + * @param detail 当前 JDBC 元数据 + * @param account 当前账号 + * @return 是否收缩或改变了可查询范围 + */ + boolean refreshManagedTable( + DatacenterSource source, + DatacenterCatalog catalog, + BigInteger tableId, + DatacenterTableDetailMeta detail, + LoginAccount account); + + /** + * 将本次无法发现的已纳管表标记为缺失。 + * + * @param source 当前数据源 + * @param tableId 已纳管表 ID + * @param account 当前账号 + * @return 是否收缩或改变了可查询范围 + */ + boolean markManagedTableMissing( + DatacenterSource source, + BigInteger tableId, + LoginAccount account); + DatacenterTable saveDescriptions(BigInteger tableId, String tableDesc, List fields, LoginAccount account); } diff --git a/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/meta/service/DatacenterSourceService.java b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/meta/service/DatacenterSourceService.java index c27bac56..4da9fcb4 100644 --- a/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/meta/service/DatacenterSourceService.java +++ b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/meta/service/DatacenterSourceService.java @@ -9,24 +9,183 @@ import tech.easyflow.datacenter.meta.entity.DatacenterSource; import tech.easyflow.datacenter.meta.model.DatacenterBatchRegisterRequest; import tech.easyflow.datacenter.meta.model.DatacenterCatalogMeta; import tech.easyflow.datacenter.meta.model.DatacenterTableDetailMeta; +import tech.easyflow.datacenter.meta.model.DatacenterSourceActivateRequest; +import tech.easyflow.datacenter.meta.model.DatacenterMetadataPage; +import tech.easyflow.datacenter.meta.model.DatacenterSourceCandidateMetadataRequest; +import tech.easyflow.datacenter.meta.model.DatacenterSourceCandidateCatalogRequest; +import tech.easyflow.datacenter.meta.model.DatacenterSourceDraftRequest; +import tech.easyflow.datacenter.meta.model.DatacenterSourceReconfigureRequest; +import tech.easyflow.datacenter.meta.model.DatacenterSourceView; import java.math.BigInteger; import java.util.List; public interface DatacenterSourceService extends IService { + /** + * 保存内部 Excel/项目数据源实体，外部 Controller 不应调用。 + * + * @param source 内部数据源实体 + * @param account 当前账号 + * @return 保存后的实体 + */ DatacenterSource saveSource(DatacenterSource source, LoginAccount account); - Page pageSources(Long pageNumber, Long pageSize, LoginAccount account); + /** + * 保存外部 JDBC 数据源草稿。 + * + * @param request 受控草稿请求 + * @param account 当前账号 + * @return 安全数据源视图 + */ + DatacenterSourceView saveDraft(DatacenterSourceDraftRequest request, LoginAccount account); - DatacenterConnectionTestResult testConnection(DatacenterSource source, LoginAccount account); + /** + * 分页返回不含凭据的数据源。 + * + * @param pageNumber 页码 + * @param pageSize 页大小 + * @param account 当前账号 + * @return 安全数据源分页 + */ + Page pageSources(Long pageNumber, Long pageSize, LoginAccount account); + + /** + * 探测已保存草稿并记录数据库/Driver 指纹。 + * + * @param sourceId 数据源 ID + * @param account 当前账号 + * @return 连接探测结果 + */ + DatacenterConnectionTestResult probe(BigInteger sourceId, LoginAccount account); + + /** + * 激活数据源、纳管选择对象并发布 Definition。 + * + * @param request 激活请求 + * @param account 当前账号 + * @return READY 数据源视图 + */ + DatacenterSourceView activate(DatacenterSourceActivateRequest request, LoginAccount account); + + /** + * 使用活动数据源的候选配置执行无状态连接探测。 + * + * @param request 候选连接配置 + * @param account 当前账号 + * @return 连接探测结果 + */ + DatacenterConnectionTestResult probeCandidate( + DatacenterSourceDraftRequest request, + LoginAccount account); + + /** + * 浏览活动数据源候选配置可访问的命名空间。 + * + * @param request 候选连接配置 + * @param account 当前账号 + * @return 命名空间列表 + */ + List listCandidateCatalogs( + DatacenterSourceDraftRequest request, + LoginAccount account); + + /** + * 分页浏览活动数据源候选配置可访问的命名空间。 + * + * @param request 候选配置与分页条件 + * @param account 当前账号 + * @return 有界命名空间列表 + */ + DatacenterMetadataPage listCandidateCatalogsPage( + DatacenterSourceCandidateCatalogRequest request, + LoginAccount account); + + /** + * 分页浏览活动数据源候选配置可访问的表。 + * + * @param request 候选配置与分页条件 + * @param account 当前账号 + * @return 有界表列表 + */ + DatacenterMetadataPage listCandidateTables( + DatacenterSourceCandidateMetadataRequest request, + LoginAccount account); + + /** + * 原地发布活动数据源的新连接配置和显式纳管范围。 + * + * @param request 重配置请求 + * @param account 当前账号 + * @return 发布后的安全视图 + */ + DatacenterSourceView reconfigure( + DatacenterSourceReconfigureRequest request, + LoginAccount account); List listCatalogs(BigInteger sourceId, LoginAccount account); - List listTables(BigInteger sourceId, String catalogName, LoginAccount account); + /** + * 分页浏览当前数据源的命名空间。 + * + * @param sourceId 数据源 ID + * @param keyword 名称搜索词 + * @param pageNumber 页码 + * @param pageSize 每页大小 + * @param account 当前账号 + * @return 有界命名空间列表 + */ + DatacenterMetadataPage listCatalogsPage( + BigInteger sourceId, + String keyword, + Long pageNumber, + Long pageSize, + LoginAccount account); - DatacenterTableDetailMeta getTableDetail(BigInteger sourceId, String catalogName, String tableName, boolean register, LoginAccount account); + DatacenterMetadataPage listTables( + BigInteger sourceId, + String catalogName, + String keyword, + Long pageNumber, + Long pageSize, + LoginAccount account); + + DatacenterTableDetailMeta getTableDetail( + BigInteger sourceId, + String catalogName, + String tableName, + boolean register, + Long fieldPageNumber, + Long fieldPageSize, + LoginAccount account); List batchRegisterTables(DatacenterBatchRegisterRequest request, LoginAccount account); void removeSource(BigInteger sourceId, LoginAccount account); + + /** + * 停用活动数据源并发布墓碑。 + * + * @param sourceId 数据源 ID + * @param account 当前账号 + * @return 停用后的安全视图 + */ + DatacenterSourceView disable(BigInteger sourceId, LoginAccount account); + + /** + * 重新探测、预热并启用已停用数据源。 + * + * @param sourceId 数据源 ID + * @param account 当前账号 + * @return 启用后的安全视图 + */ + DatacenterSourceView enable(BigInteger sourceId, LoginAccount account); + + /** + * 刷新已纳管对象的 JDBC 元数据观测状态。 + * + * @param sourceId 数据源 ID + * @param account 当前账号 + * @return 刷新后的安全视图 + */ + DatacenterSourceView refreshMetadata(BigInteger sourceId, LoginAccount account); } diff --git a/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/meta/service/impl/DatacenterDatasetRegistryServiceImpl.java b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/meta/service/impl/DatacenterDatasetRegistryServiceImpl.java index c5cfd9f2..7e4517bb 100644 --- a/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/meta/service/impl/DatacenterDatasetRegistryServiceImpl.java +++ b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/meta/service/impl/DatacenterDatasetRegistryServiceImpl.java @@ -9,6 +9,8 @@ import tech.easyflow.datacenter.adapter.DbHandleManager; import tech.easyflow.datacenter.entity.DatacenterTable; import tech.easyflow.datacenter.entity.DatacenterTableField; import tech.easyflow.datacenter.execution.model.DatasetRef; +import tech.easyflow.datacenter.federation.DatacenterMetadataIdentity; +import tech.easyflow.datacenter.federation.DatacenterFederationChangeNotifier; import tech.easyflow.datacenter.mapper.DatacenterCatalogMapper; import tech.easyflow.datacenter.mapper.DatacenterDatasetVersionMapper; import tech.easyflow.datacenter.mapper.DatacenterDerivedTableMapper; @@ -20,6 +22,9 @@ import tech.easyflow.datacenter.meta.entity.DatacenterCatalog; import tech.easyflow.datacenter.meta.entity.DatacenterSource; import tech.easyflow.datacenter.meta.model.DatacenterFieldDescriptionUpdate; import tech.easyflow.datacenter.meta.enums.DatacenterSourceType; +import tech.easyflow.datacenter.meta.enums.DatacenterMetadataStatus; +import tech.easyflow.datacenter.meta.enums.DatacenterSensitivityLevel; +import tech.easyflow.datacenter.meta.enums.DatacenterSourceStatus; import tech.easyflow.datacenter.meta.service.DatacenterDatasetRegistryService; import tech.easyflow.datacenter.meta.service.DatacenterMetaConstants; import tech.easyflow.datacenter.meta.model.DatacenterTableDetailMeta; @@ -30,7 +35,11 @@ import java.math.BigInteger; import java.util.Date; import java.util.LinkedHashMap; import java.util.List; +import java.util.Locale; import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.stream.Collectors; @Service public class DatacenterDatasetRegistryServiceImpl implements DatacenterDatasetRegistryService { @@ -51,6 +60,8 @@ public class DatacenterDatasetRegistryServiceImpl implements DatacenterDatasetRe private DatacenterImportJobMapper importJobMapper; @Resource private DbHandleManager dbHandleManager; + @Resource + private DatacenterFederationChangeNotifier federationChangeNotifier; @Override public DatacenterSource ensureBuiltinSource(DatacenterSourceType sourceType, LoginAccount account) { @@ -67,7 +78,11 @@ public class DatacenterDatasetRegistryServiceImpl implements DatacenterDatasetRe source.setDeptId(deptId); source.setSourceType(sourceType.name()); source.setBuiltinFlag(1); - source.setStatus(0); + source.setStatus(DatacenterSourceStatus.READY.code()); + source.setDefinitionRevision(1L); + source.setScopeRevision(1L); + source.setMetadataRefreshStatus("SUCCESS"); + source.setMetadataRefreshedAt(new Date()); source.setCreated(new Date()); source.setModified(new Date()); source.setCreatedBy(account == null ? BigInteger.ZERO : account.getId()); @@ -92,6 +107,12 @@ public class DatacenterDatasetRegistryServiceImpl implements DatacenterDatasetRe wrapper.eq(DatacenterCatalog::getCatalogName, catalogName); DatacenterCatalog catalog = catalogMapper.selectOneByQuery(wrapper); if (catalog != null) { + boolean changed = applyCatalogIdentity(catalog, source, catalogName); + if (changed) { + catalog.setModified(new Date()); + catalog.setModifiedBy(account == null ? BigInteger.ZERO : account.getId()); + catalogMapper.update(catalog); + } return catalog; } catalog = new DatacenterCatalog(); @@ -105,6 +126,7 @@ public class DatacenterDatasetRegistryServiceImpl implements DatacenterDatasetRe catalog.setCatalogName(catalogName); catalog.setCatalogDesc(catalogName); catalog.setCatalogType("DATABASE"); + applyCatalogIdentity(catalog, source, catalogName); catalog.setStatus(0); catalog.setCreated(new Date()); catalog.setModified(new Date()); @@ -116,13 +138,43 @@ public class DatacenterDatasetRegistryServiceImpl implements DatacenterDatasetRe @Override public DatacenterTable registerTable(DatacenterSource source, DatacenterCatalog catalog, DatacenterTableDetailMeta detail, LoginAccount account) { + return registerTable(source, catalog, detail, account, false); + } + + /** + * 显式重配置时允许在同一稳定表记录上确认新的物理身份，并重建字段治理基线。 + */ + @Override + public DatacenterTable rebindTable( + DatacenterSource source, + DatacenterCatalog catalog, + DatacenterTableDetailMeta detail, + LoginAccount account) { + return registerTable(source, catalog, detail, account, true); + } + + private DatacenterTable registerTable( + DatacenterSource source, + DatacenterCatalog catalog, + DatacenterTableDetailMeta detail, + LoginAccount account, + boolean allowExplicitRebind) { DatacenterTable table = detail.getTable(); applyTableDefaults(table, source, detail); + table.setPhysicalIdentityKey(DatacenterMetadataIdentity.tableIdentity( + source.getId().toString(), + catalog.getNamespaceKey(), + table.getActualTable(), + table.getTableKind())); + table.setMetadataFingerprint(DatacenterMetadataIdentity.tableFingerprint( + table.getTableName(), table.getTableKind(), detail.getFields())); QueryWrapper wrapper = QueryWrapper.create(); wrapper.eq(DatacenterTable::getSourceId, source.getId()); wrapper.eq(DatacenterTable::getCatalogId, catalog.getId()); wrapper.eq(DatacenterTable::getTableName, table.getTableName()); DatacenterTable existing = tableMapper.selectOneByQuery(wrapper); + List persistedFields = existing == null + ? List.of() : getFields(existing.getId()); Date now = new Date(); if (existing == null) { table.setSourceId(source.getId()); @@ -136,6 +188,39 @@ public class DatacenterDatasetRegistryServiceImpl implements DatacenterDatasetRe table.setModifiedBy(account == null ? BigInteger.ZERO : account.getId()); tableMapper.insert(table); } else { + boolean identityChanged = hasText(existing.getPhysicalIdentityKey()) + && !existing.getPhysicalIdentityKey().equals(table.getPhysicalIdentityKey()) + && !existing.getPhysicalIdentityKey().equals( + DatacenterMetadataIdentity.legacyV58TableIdentity( + existing.getSourceId(), existing.getCatalogId(), + existing.getActualTable(), existing.getTableKind())); + if (identityChanged && !allowExplicitRebind) { + throw new BusinessException("物理表身份已变化，请移除原绑定后重新接入: " + table.getTableName()); + } + boolean structureChanged = false; + if (hasText(existing.getMetadataFingerprint()) + && !existing.getMetadataFingerprint().equals(table.getMetadataFingerprint())) { + boolean legacyFingerprint = existing.getMetadataFingerprint().equals( + DatacenterMetadataIdentity.legacyV58TableFingerprint( + existing.getTableName(), existing.getTableKind())); + if (!legacyFingerprint + || !legacyStructureMatches( + persistedFields, detail.getFields())) { + structureChanged = true; + } + } + if (structureChanged && !allowExplicitRebind) { + throw new BusinessException( + "表结构已变化，请移除原绑定后重新接入: " + table.getTableName()); + } + boolean resetGovernance = allowExplicitRebind + && (identityChanged || structureChanged); + if (resetGovernance) { + tableFieldMapper.deleteByQuery(QueryWrapper.create() + .eq(DatacenterTableField::getTableId, existing.getId())); + persistedFields = List.of(); + existing.setQueryable(table.getQueryable()); + } if (!hasText(existing.getTableDesc())) { existing.setTableDesc(normalizeDescription(table.getTableDesc())); } @@ -145,30 +230,61 @@ public class DatacenterDatasetRegistryServiceImpl implements DatacenterDatasetRe existing.setAccessMode(table.getAccessMode()); existing.setVersioningEnabled(table.getVersioningEnabled()); existing.setCapabilitiesJson(table.getCapabilitiesJson()); + existing.setPhysicalIdentityKey(table.getPhysicalIdentityKey()); + existing.setMetadataFingerprint(table.getMetadataFingerprint()); + existing.setMetadataStatus(DatacenterMetadataStatus.ACTIVE.name()); + existing.setLastSeenAt(now); + // 查询开关属于治理配置，元数据刷新不得覆盖人工决策。 + existing.setTimeSemantics(table.getTimeSemantics()); + existing.setMetadataRevision((existing.getMetadataRevision() == null + ? 0L : existing.getMetadataRevision()) + 1L); existing.setModified(now); existing.setModifiedBy(account == null ? BigInteger.ZERO : account.getId()); tableMapper.update(existing); table = existing; } - Map existingFieldMap = getFields(table.getId()).stream() - .collect(LinkedHashMap::new, (map, field) -> map.put(field.getFieldName(), field), Map::putAll); - QueryWrapper deleteWrapper = QueryWrapper.create(); - deleteWrapper.eq(DatacenterTableField::getTableId, table.getId()); - tableFieldMapper.deleteByQuery(deleteWrapper); + Map existingFieldMap = persistedFields.stream() + .collect(LinkedHashMap::new, + (map, field) -> map.put(sourceColumnName(field), field), + Map::putAll); for (DatacenterTableField field : detail.getFields()) { - DatacenterTableField existingField = existingFieldMap.get(field.getFieldName()); + applyFieldDefaults(field); + DatacenterTableField existingField = existingFieldMap.remove(sourceColumnName(field)); if (existingField != null && hasText(existingField.getFieldDesc())) { field.setFieldDesc(existingField.getFieldDesc()); } else { field.setFieldDesc(normalizeDescription(field.getFieldDesc())); } - field.setId(null); field.setTableId(table.getId()); - field.setCreated(now); field.setModified(now); - field.setCreatedBy(account == null ? BigInteger.ZERO : account.getId()); field.setModifiedBy(account == null ? BigInteger.ZERO : account.getId()); - tableFieldMapper.insert(field); + if (existingField == null) { + field.setId(null); + field.setCreated(now); + field.setCreatedBy(account == null ? BigInteger.ZERO : account.getId()); + tableFieldMapper.insert(field); + } else { + field.setId(existingField.getId()); + field.setCreated(existingField.getCreated()); + field.setCreatedBy(existingField.getCreatedBy()); + if (existingField.getSensitivityLevel() != null) { + field.setSensitivityLevel(existingField.getSensitivityLevel()); + field.setMaskingStrategy(existingField.getMaskingStrategy()); + } + // 元数据刷新只更新物理事实，保留字段级治理设置。 + field.setQueryable(existingField.getQueryable()); + field.setSortable(existingField.getSortable()); + tableFieldMapper.update(field); + } + } + for (DatacenterTableField missing : existingFieldMap.values()) { + missing.setMetadataStatus(DatacenterMetadataStatus.MISSING.name()); + missing.setQueryable(0); + missing.setSortable(0); + missing.setWritable(0); + missing.setModified(now); + missing.setModifiedBy(account == null ? BigInteger.ZERO : account.getId()); + tableFieldMapper.update(missing); } table.setFields(getFields(table.getId())); return table; @@ -196,19 +312,71 @@ public class DatacenterDatasetRegistryServiceImpl implements DatacenterDatasetRe fieldUpdateMap.put(field.getFieldId(), field); } } + boolean policyChanged = false; for (DatacenterTableField field : table.getFields()) { DatacenterFieldDescriptionUpdate update = fieldUpdateMap.get(field.getId()); if (update == null) { continue; } field.setFieldDesc(normalizeDescription(update.getFieldDesc())); + Integer queryable = normalizedQueryable(update.getQueryable(), field.getQueryable()); + String sensitivityLevel = normalizedSensitivityLevel( + update.getSensitivityLevel(), field.getSensitivityLevel()); + if (!DatacenterSensitivityLevel.PUBLIC.name().equals(sensitivityLevel) + && Integer.valueOf(1).equals(queryable)) { + throw new BusinessException("敏感字段暂不支持直接查询，请关闭可查询后保存"); + } + policyChanged |= !Objects.equals(field.getQueryable(), queryable) + || !Objects.equals(field.getSensitivityLevel(), sensitivityLevel); + field.setQueryable(queryable); + field.setSensitivityLevel(sensitivityLevel); field.setModified(now); field.setModifiedBy(account == null ? BigInteger.ZERO : account.getId()); tableFieldMapper.update(field); } + if (policyChanged) { + incrementExternalScopeRevisions(List.of(table)); + } return getTableWithFields(tableId); } + /** + * 校验并归一化字段查询开关。 + * + * @param requested 请求值 + * @param existing 当前值 + * @return 归一化后的 0 或 1 + */ + private Integer normalizedQueryable(Integer requested, Integer existing) { + if (requested == null) { + return existing == null ? 0 : existing; + } + if (requested != 0 && requested != 1) { + throw new BusinessException("queryable 只能为 0 或 1"); + } + return requested; + } + + /** + * 校验并归一化字段敏感级别。 + * + * @param requested 请求值 + * @param existing 当前值 + * @return 有效枚举名称 + */ + private String normalizedSensitivityLevel(String requested, String existing) { + if (requested == null || requested.isBlank()) { + return existing == null || existing.isBlank() + ? DatacenterSensitivityLevel.PUBLIC.name() : existing; + } + try { + return DatacenterSensitivityLevel.valueOf( + requested.trim().toUpperCase(Locale.ROOT)).name(); + } catch (IllegalArgumentException exception) { + throw new BusinessException("不支持的字段敏感级别: " + requested); + } + } + @Override public DatacenterTable getTableWithFields(BigInteger tableId) { DatacenterTable table = tableMapper.selectOneById(tableId); @@ -223,7 +391,7 @@ public class DatacenterDatasetRegistryServiceImpl implements DatacenterDatasetRe public List getFields(BigInteger tableId) { QueryWrapper wrapper = QueryWrapper.create(); wrapper.eq(DatacenterTableField::getTableId, tableId); - wrapper.orderBy("id"); + wrapper.orderBy("ordinal_position asc, id asc"); return tableFieldMapper.selectListByQuery(wrapper); } @@ -231,6 +399,7 @@ public class DatacenterDatasetRegistryServiceImpl implements DatacenterDatasetRe public DatasetRef resolveDatasetRef(BigInteger tableId) { DatacenterTable table = getTableWithFields(tableId); DatasetRef ref = new DatasetRef(); + ref.setTenantId(table.getTenantId()); ref.setTableId(tableId); ref.setSourceId(table.getSourceId()); ref.setCatalogId(table.getCatalogId()); @@ -266,6 +435,7 @@ public class DatacenterDatasetRegistryServiceImpl implements DatacenterDatasetRe if (catalogId != null) { wrapper.eq(DatacenterTable::getCatalogId, catalogId); } + wrapper.eq(DatacenterTable::getMetadataStatus, DatacenterMetadataStatus.ACTIVE.name()); wrapper.orderBy("created desc"); return tableMapper.selectListByQuery(wrapper); } @@ -280,31 +450,231 @@ public class DatacenterDatasetRegistryServiceImpl implements DatacenterDatasetRe QueryWrapper physicalTableWrapper = QueryWrapper.create(); physicalTableWrapper.in(DatacenterTable::getId, ids); List tables = tableMapper.selectListByQuery(physicalTableWrapper); + List authorizedIds = tables.stream() + .map(DatacenterTable::getId) + .distinct() + .toList(); + if (authorizedIds.size() != ids.size()) { + throw new BusinessException("部分数据表不存在或无权删除"); + } tables.forEach(this::deletePhysicalTableIfNecessary); QueryWrapper fieldWrapper = QueryWrapper.create(); - fieldWrapper.in(DatacenterTableField::getTableId, ids); + fieldWrapper.in(DatacenterTableField::getTableId, authorizedIds); tableFieldMapper.deleteByQuery(fieldWrapper); QueryWrapper versionWrapper = QueryWrapper.create(); - versionWrapper.in(tech.easyflow.datacenter.meta.entity.DatacenterDatasetVersion::getTableId, ids); + versionWrapper.in(tech.easyflow.datacenter.meta.entity.DatacenterDatasetVersion::getTableId, authorizedIds); datasetVersionMapper.deleteByQuery(versionWrapper); QueryWrapper importJobWrapper = QueryWrapper.create(); - importJobWrapper.in(tech.easyflow.datacenter.meta.entity.DatacenterImportJob::getTableId, ids); + importJobWrapper.in(tech.easyflow.datacenter.meta.entity.DatacenterImportJob::getTableId, authorizedIds); importJobMapper.deleteByQuery(importJobWrapper); QueryWrapper upstreamWrapper = QueryWrapper.create(); - upstreamWrapper.in(tech.easyflow.datacenter.meta.entity.DatacenterDerivedTable::getSourceTableId, ids); + upstreamWrapper.in(tech.easyflow.datacenter.meta.entity.DatacenterDerivedTable::getSourceTableId, authorizedIds); derivedTableMapper.deleteByQuery(upstreamWrapper); QueryWrapper downstreamWrapper = QueryWrapper.create(); - downstreamWrapper.in(tech.easyflow.datacenter.meta.entity.DatacenterDerivedTable::getDerivedTableId, ids); + downstreamWrapper.in(tech.easyflow.datacenter.meta.entity.DatacenterDerivedTable::getDerivedTableId, authorizedIds); derivedTableMapper.deleteByQuery(downstreamWrapper); QueryWrapper tableWrapper = QueryWrapper.create(); - tableWrapper.in(DatacenterTable::getId, ids); - return tableMapper.deleteByQuery(tableWrapper); + tableWrapper.in(DatacenterTable::getId, authorizedIds); + int removed = tableMapper.deleteByQuery(tableWrapper); + incrementExternalScopeRevisions(tables); + return removed; + } + + /** + * 刷新已纳管表的物理观测值；身份、结构或历史状态存在歧义时立即收缩权限。 + */ + @Override + @Transactional(rollbackFor = Exception.class) + public boolean refreshManagedTable( + DatacenterSource source, + DatacenterCatalog catalog, + BigInteger tableId, + DatacenterTableDetailMeta detail, + LoginAccount account) { + DatacenterTable existing = tableMapper.selectOneById(tableId); + if (existing == null + || !Objects.equals(existing.getSourceId(), source.getId()) + || !Objects.equals(existing.getCatalogId(), catalog.getId())) { + throw new BusinessException("已纳管数据表不存在或不属于当前连接"); + } + DatacenterTable discovered = detail == null ? null : detail.getTable(); + if (discovered == null) { + throw new BusinessException("未返回数据表元数据: " + existing.getTableName()); + } + applyTableDefaults(discovered, source, detail); + String discoveredIdentity = DatacenterMetadataIdentity.tableIdentity( + source.getId().toString(), + catalog.getNamespaceKey(), + discovered.getActualTable(), + discovered.getTableKind()); + String discoveredFingerprint = DatacenterMetadataIdentity.tableFingerprint( + discovered.getTableName(), discovered.getTableKind(), detail.getFields()); + List persistedFields = getFields(existing.getId()); + boolean identityMatches = Objects.equals( + existing.getPhysicalIdentityKey(), discoveredIdentity) + || Objects.equals( + existing.getPhysicalIdentityKey(), + DatacenterMetadataIdentity.legacyV58TableIdentity( + existing.getSourceId(), existing.getCatalogId(), + existing.getActualTable(), existing.getTableKind())); + boolean legacyCompatible = Objects.equals( + existing.getMetadataFingerprint(), + DatacenterMetadataIdentity.legacyV58TableFingerprint( + existing.getTableName(), existing.getTableKind())) + && legacyStructureMatches(persistedFields, detail.getFields()); + boolean fingerprintMatches = Objects.equals( + existing.getMetadataFingerprint(), discoveredFingerprint) + || legacyCompatible; + boolean ambiguous = !DatacenterMetadataStatus.ACTIVE.name().equals(existing.getMetadataStatus()) + || !identityMatches + || !fingerprintMatches; + if (ambiguous) { + return markTableUnavailable( + existing, DatacenterMetadataStatus.CHANGED, true, account); + } + + Date now = new Date(); + existing.setActualTable(discovered.getActualTable()); + existing.setTableKind(discovered.getTableKind()); + existing.setPhysicalIdentityKey(discoveredIdentity); + existing.setMetadataFingerprint(discoveredFingerprint); + existing.setLastSeenAt(now); + existing.setMetadataRevision((existing.getMetadataRevision() == null + ? 0L : existing.getMetadataRevision()) + 1L); + existing.setModified(now); + existing.setModifiedBy(account == null ? BigInteger.ZERO : account.getId()); + tableMapper.update(existing); + + Map discoveredFields = detail.getFields().stream() + .collect(Collectors.toMap( + this::sourceColumnName, + field -> field, + (first, ignored) -> first, + LinkedHashMap::new)); + for (DatacenterTableField persisted : persistedFields) { + DatacenterTableField observed = discoveredFields.get(sourceColumnName(persisted)); + if (observed == null + || (!legacyCompatible && !Objects.equals( + persisted.getMetadataFingerprint(), + DatacenterMetadataIdentity.fieldFingerprint(observed)))) { + // 表 fingerprint 相同的前提下不应进入该分支；保留 fail-closed 防线。 + return markTableUnavailable( + existing, DatacenterMetadataStatus.CHANGED, true, account); + } + persisted.setOrdinalPosition(observed.getOrdinalPosition()); + persisted.setJdbcType(observed.getJdbcType()); + persisted.setJdbcTypeCode(observed.getJdbcTypeCode()); + persisted.setNativeTypeName(observed.getNativeTypeName()); + persisted.setPrecision(observed.getPrecision()); + persisted.setScale(observed.getScale()); + persisted.setRequired(observed.getRequired()); + persisted.setIndexed(observed.getIndexed()); + persisted.setMetadataFingerprint( + DatacenterMetadataIdentity.fieldFingerprint(observed)); + persisted.setMetadataStatus(DatacenterMetadataStatus.ACTIVE.name()); + persisted.setLastSeenAt(now); + persisted.setModified(now); + persisted.setModifiedBy(account == null ? BigInteger.ZERO : account.getId()); + tableFieldMapper.update(persisted); + } + return false; + } + + /** + * 标记未发现的已纳管表，并关闭其全部查询入口。 + */ + @Override + @Transactional(rollbackFor = Exception.class) + public boolean markManagedTableMissing( + DatacenterSource source, + BigInteger tableId, + LoginAccount account) { + DatacenterTable existing = tableMapper.selectOneById(tableId); + if (existing == null || !Objects.equals(existing.getSourceId(), source.getId())) { + throw new BusinessException("已纳管数据表不存在或不属于当前连接"); + } + return markTableUnavailable( + existing, DatacenterMetadataStatus.MISSING, false, account); + } + + private boolean markTableUnavailable( + DatacenterTable table, + DatacenterMetadataStatus status, + boolean seen, + LoginAccount account) { + boolean changed = !status.name().equals(table.getMetadataStatus()) + || !Integer.valueOf(0).equals(table.getQueryable()); + Date now = new Date(); + table.setMetadataStatus(status.name()); + table.setQueryable(0); + if (seen) { + table.setLastSeenAt(now); + } + table.setModified(now); + table.setModifiedBy(account == null ? BigInteger.ZERO : account.getId()); + tableMapper.update(table); + for (DatacenterTableField field : getFields(table.getId())) { + changed |= !status.name().equals(field.getMetadataStatus()) + || !Integer.valueOf(0).equals(field.getQueryable()); + field.setMetadataStatus(status.name()); + field.setQueryable(0); + field.setSortable(0); + field.setWritable(0); + if (seen) { + field.setLastSeenAt(now); + } + field.setModified(now); + field.setModifiedBy(account == null ? BigInteger.ZERO : account.getId()); + tableFieldMapper.update(field); + } + return changed; + } + + /** + * 删除外部纳管表后以 CAS 推进策略版本，并在提交后通知其他节点。 + * + * @param removedTables 本事务删除的纳管表 + */ + private void incrementExternalScopeRevisions(List removedTables) { + Set sourceIds = removedTables.stream() + .map(DatacenterTable::getSourceId) + .filter(java.util.Objects::nonNull) + .collect(Collectors.toSet()); + for (BigInteger sourceId : sourceIds) { + DatacenterSource source = sourceMapper.selectOneById(sourceId); + if (source == null || (!DatacenterSourceType.MYSQL.name().equals(source.getSourceType()) + && !DatacenterSourceType.POSTGRESQL.name().equals(source.getSourceType()))) { + continue; + } + long previous = source.getScopeRevision() == null ? 0L : source.getScopeRevision(); + long definitionRevision = source.getDefinitionRevision() == null + ? 0L : source.getDefinitionRevision(); + DatacenterSource patch = new DatacenterSource(); + patch.setId(source.getId()); + patch.setScopeRevision(Math.max(1L, previous) + 1L); + patch.setModified(new Date()); + QueryWrapper guard = QueryWrapper.create() + .eq(DatacenterSource::getId, source.getId()) + .eq(DatacenterSource::getTenantId, source.getTenantId()) + .eq(DatacenterSource::getDefinitionRevision, definitionRevision) + .eq(DatacenterSource::getScopeRevision, previous) + .ne(DatacenterSource::getStatus, + DatacenterSourceStatus.DELETED.code()); + if (sourceMapper.updateByQuery(patch, guard) != 1) { + throw new BusinessException("数据范围已被其他请求更新，请刷新后重试"); + } + DatacenterSource updated = sourceMapper.selectOneById(source.getId()); + if (updated == null) { + throw new BusinessException("数据连接已被删除"); + } + federationChangeNotifier.publishAfterCommit(updated); + } } private void deletePhysicalTableIfNecessary(DatacenterTable table) { @@ -360,12 +730,130 @@ public class DatacenterDatasetRegistryServiceImpl implements DatacenterDatasetRe if ((table.getActualTable() == null || table.getActualTable().isBlank()) && table.getTableName() != null) { table.setActualTable(table.getTableName()); } + if (table.getQueryable() == null) { + table.setQueryable(1); + } + if (table.getMetadataStatus() == null) { + table.setMetadataStatus(DatacenterMetadataStatus.ACTIVE.name()); + } + if (table.getLastSeenAt() == null) { + table.setLastSeenAt(new Date()); + } + if (table.getTimeSemantics() == null) { + table.setTimeSemantics("NONE"); + } + if (table.getMetadataRevision() == null) { + table.setMetadataRevision(1L); + } table.setTableDesc(normalizeDescription(table.getTableDesc())); if (detail.getFields() == null) { detail.setFields(List.of()); } } + private boolean applyCatalogIdentity( + DatacenterCatalog catalog, + DatacenterSource source, + String catalogName) { + boolean mysql = DatacenterSourceType.MYSQL.name().equals(source.getSourceType()) + || DatacenterSourceType.PROJECT_MYSQL.name().equals(source.getSourceType()); + String physicalCatalog = mysql ? catalogName : source.getDatabaseName(); + String physicalSchema = mysql ? null : catalogName; + String logicalSchema = catalogName; + String namespaceKey = DatacenterMetadataIdentity.namespaceKey(physicalCatalog, physicalSchema); + boolean changed = !java.util.Objects.equals(catalog.getLogicalSchemaName(), logicalSchema) + || !java.util.Objects.equals(catalog.getPhysicalCatalogName(), physicalCatalog) + || !java.util.Objects.equals(catalog.getPhysicalSchemaName(), physicalSchema) + || !java.util.Objects.equals(catalog.getNamespaceKey(), namespaceKey) + || !DatacenterMetadataStatus.ACTIVE.name().equals(catalog.getMetadataStatus()); + catalog.setLogicalSchemaName(logicalSchema); + catalog.setPhysicalCatalogName(physicalCatalog); + catalog.setPhysicalSchemaName(physicalSchema); + catalog.setNamespaceKey(namespaceKey); + catalog.setMetadataStatus(DatacenterMetadataStatus.ACTIVE.name()); + catalog.setLastSeenAt(new Date()); + return changed; + } + + private void applyFieldDefaults(DatacenterTableField field) { + if (!hasText(field.getSourceColumnName())) { + field.setSourceColumnName(field.getFieldName()); + } + if (field.getNativeTypeName() == null) { + field.setNativeTypeName(field.getJdbcType()); + } + field.setMetadataStatus(DatacenterMetadataStatus.ACTIVE.name()); + field.setLastSeenAt(new Date()); + if (field.getSensitivityLevel() == null) { + field.setSensitivityLevel(DatacenterSensitivityLevel.PUBLIC.name()); + } + field.setMetadataFingerprint(DatacenterMetadataIdentity.fieldFingerprint(field)); + } + + /** + * 使用 V58 已持久化的旧字段信息校验首次元数据升级。 + * + *

V58 新增的 JDBC 编码和顺序列没有回填，因此这些空值只表示未知； + * 字段名称、旧 JDBC 类型、精度、小数位和必填属性仍必须保持一致。

+ * + * @param persistedFields V58 存量字段 + * @param discoveredFields 当前 JDBC 发现字段 + * @return 已知旧结构是否与当前结构一致 + */ + private boolean legacyStructureMatches( + List persistedFields, + List discoveredFields) { + List safePersisted = persistedFields == null + ? List.of() : persistedFields; + List safeDiscovered = discoveredFields == null + ? List.of() : discoveredFields; + if (safePersisted.size() != safeDiscovered.size()) { + return false; + } + Map discoveredByName = safeDiscovered.stream() + .collect(Collectors.toMap( + this::sourceColumnName, + field -> field, + (first, ignored) -> first, + LinkedHashMap::new)); + if (discoveredByName.size() != safeDiscovered.size()) { + return false; + } + for (DatacenterTableField persisted : safePersisted) { + DatacenterTableField discovered = discoveredByName.get( + sourceColumnName(persisted)); + if (discovered == null + || !sameWhenKnown(persisted.getFieldType(), discovered.getFieldType()) + || !sameWhenKnown(persisted.getJdbcType(), discovered.getJdbcType()) + || !sameWhenKnown(persisted.getNativeTypeName(), discovered.getNativeTypeName()) + || !sameWhenKnown(persisted.getJdbcTypeCode(), discovered.getJdbcTypeCode()) + || !sameWhenKnown(persisted.getPrecision(), discovered.getPrecision()) + || !sameWhenKnown(persisted.getScale(), discovered.getScale()) + || !sameWhenKnown(persisted.getRequired(), discovered.getRequired())) { + return false; + } + } + return true; + } + + /** + * 比较旧元数据中的已知值，空值按 V58 未回填处理。 + * + * @param persisted 旧值 + * @param discovered 当前值 + * @return 旧值为空或两者相等时返回 true + */ + private boolean sameWhenKnown(Object persisted, Object discovered) { + return persisted == null + || java.util.Objects.equals(persisted, discovered); + } + + private String sourceColumnName(DatacenterTableField field) { + return hasText(field.getSourceColumnName()) + ? field.getSourceColumnName() + : field.getFieldName(); + } + private String normalizeDescription(String description) { if (description == null) { return ""; diff --git a/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/meta/service/impl/DatacenterSourceCandidateService.java b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/meta/service/impl/DatacenterSourceCandidateService.java new file mode 100644 index 00000000..c248c646 --- /dev/null +++ b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/meta/service/impl/DatacenterSourceCandidateService.java @@ -0,0 +1,361 @@ +package tech.easyflow.datacenter.meta.service.impl; + +import com.easyagents.federation.sql.api.FederationSqlException; +import com.easyagents.federation.sql.source.RuntimeFingerprint; +import com.easyagents.federation.sql.source.SourceProbeResult; +import java.math.BigInteger; +import java.util.Date; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Service; +import tech.easyflow.common.entity.LoginAccount; +import tech.easyflow.common.web.exceptions.BusinessException; +import tech.easyflow.datacenter.execution.model.DatacenterConnectionTestResult; +import tech.easyflow.datacenter.federation.DatacenterFederationRuntime; +import tech.easyflow.datacenter.meta.entity.DatacenterSource; +import tech.easyflow.datacenter.meta.enums.DatacenterSourceStatus; +import tech.easyflow.datacenter.meta.enums.DatacenterSourceType; +import tech.easyflow.datacenter.meta.model.DatacenterSourceDraftRequest; +import tech.easyflow.datacenter.meta.support.DatacenterSourceConnectionDefaults; +import tech.easyflow.datacenter.security.DatacenterCredentialCipher; + +/** + * 构造并探测未发布的数据源候选配置，确保凭据和连接默认值只在一个边界内处理。 + */ +@Service +public class DatacenterSourceCandidateService { + + private static final Logger LOGGER = + LoggerFactory.getLogger(DatacenterSourceCandidateService.class); + + private final DatacenterCredentialCipher credentialCipher; + private final DatacenterSourceConnectionDefaults connectionDefaults; + private final DatacenterFederationRuntime federationRuntime; + + /** + * 创建候选配置服务。 + * + * @param credentialCipher 凭据加密器 + * @param connectionDefaults JDBC 默认配置 + * @param federationRuntime Federation Runtime + */ + public DatacenterSourceCandidateService( + DatacenterCredentialCipher credentialCipher, + DatacenterSourceConnectionDefaults connectionDefaults, + DatacenterFederationRuntime federationRuntime) { + this.credentialCipher = credentialCipher; + this.connectionDefaults = connectionDefaults; + this.federationRuntime = federationRuntime; + } + + /** + * 构造仅可探测、不可查询的持久化草稿。 + * + * @param request 草稿请求 + * @param existing 当前草稿；新建时为空 + * @param account 当前账号 + * @return 已归一化草稿 + */ + public DatacenterSource buildDraft( + DatacenterSourceDraftRequest request, + DatacenterSource existing, + LoginAccount account) { + validateDefinition(request); + requireEditableDraft(existing, request.sourceType()); + DatacenterSource source = existing == null + ? new DatacenterSource() : copy(existing); + source.setSourceName(request.sourceName().trim()); + source.setSourceCode(trimToNull(request.sourceCode())); + source.setSourceType(request.sourceType()); + source.setAccessMode("READ_ONLY"); + source.setBuiltinFlag(0); + source.setAdapterId("jdbc"); + source.setAdapterOptionsJson(Map.of()); + source.setHost(request.host().trim()); + source.setPort(request.port()); + source.setDatabaseName(request.databaseName().trim()); + source.setSchemaName(trimToNull(request.schemaName())); + source.setUsername(request.username().trim()); + source.setDriverClassName( + connectionDefaults.defaultDriverClassName(request.sourceType())); + source.setJdbcUrl(null); + source.setConfigJson(Map.of()); + applyCredential(source, existing, trimToNull(request.password())); + connectionDefaults.normalize(source); + requireCredential(source); + source.setDefinitionRevision(existing == null + ? 1L : value(existing.getDefinitionRevision()) + 1L); + source.setScopeRevision(Math.max(1L, value(source.getScopeRevision()))); + source.setDefinitionChecksum(null); + source.setCompatibilityStatus(null); + source.setMetadataRefreshStatus("PENDING"); + source.setStatus(DatacenterSourceStatus.DRAFT.code()); + source.setCapabilitiesJson(Map.of( + "capabilities", List.of( + "TEST_CONNECTION", "BROWSE_METADATA", "READ_QUERY"))); + if (source.getTenantId() == null && account != null) { + source.setTenantId(account.getTenantId()); + source.setDeptId(account.getDeptId()); + } + return source; + } + + /** + * 构造活动连接的下一 revision 候选配置。 + * + * @param current 当前已发布配置 + * @param request 新连接定义 + * @return 未发布候选配置 + */ + public DatacenterSource buildReconfigurationCandidate( + DatacenterSource current, + DatacenterSourceDraftRequest request) { + validateDefinition(request); + DatacenterSourceStatus status = DatacenterSourceStatus.fromCode(current.getStatus()); + if (status != DatacenterSourceStatus.READY + && status != DatacenterSourceStatus.DEGRADED) { + throw new BusinessException("只有已激活连接可以使用候选配置"); + } + if (!java.util.Objects.equals(current.getSourceType(), request.sourceType())) { + throw new BusinessException("数据连接类型创建后不可修改"); + } + DatacenterSource candidate = copy(current); + candidate.setSourceName(request.sourceName().trim()); + String requestedCode = trimToNull(request.sourceCode()); + candidate.setSourceCode(requestedCode == null + ? current.getSourceCode() : requestedCode); + candidate.setAccessMode("READ_ONLY"); + candidate.setBuiltinFlag(0); + candidate.setAdapterId("jdbc"); + candidate.setAdapterOptionsJson(Map.of()); + candidate.setHost(request.host().trim()); + candidate.setPort(request.port()); + candidate.setDatabaseName(request.databaseName().trim()); + candidate.setSchemaName(trimToNull(request.schemaName())); + candidate.setUsername(request.username().trim()); + candidate.setDriverClassName( + connectionDefaults.defaultDriverClassName(request.sourceType())); + candidate.setJdbcUrl(null); + candidate.setConfigJson(Map.of()); + applyCredential(candidate, current, trimToNull(request.password())); + connectionDefaults.normalize(candidate); + requireCredential(candidate); + candidate.setDefinitionRevision(value(current.getDefinitionRevision()) + 1L); + candidate.setScopeRevision(current.getScopeRevision()); + candidate.setDefinitionChecksum(null); + candidate.setStatus(DatacenterSourceStatus.DRAFT.code()); + candidate.setLastTestStatus(null); + candidate.setLastTestMessage(null); + candidate.setLastTestedAt(null); + return candidate; + } + + /** + * 使用短生命周期候选上下文探测连接，并将数据库指纹写回候选实体。 + * + * @param candidate 候选配置 + * @return 安全探测结果 + */ + public DatacenterConnectionTestResult probe(DatacenterSource candidate) { + Date testedAt = new Date(); + DatacenterConnectionTestResult result = new DatacenterConnectionTestResult(); + try { + DatacenterSourceStatus status = DatacenterSourceStatus.fromCode( + candidate.getStatus()); + SourceProbeResult probe = status == DatacenterSourceStatus.READY + || status == DatacenterSourceStatus.DEGRADED + ? federationRuntime.probe(candidate) + : federationRuntime.probeCandidate(candidate); + RuntimeFingerprint fingerprint = probe.fingerprint(); + result.setSuccess(probe.supported()); + result.setErrorCode(probe.supported() ? null : "ADAPTER_UNSUPPORTED"); + result.setMessage(bounded(probe.diagnostic())); + result.setCapabilities(List.of( + "TEST_CONNECTION", "BROWSE_METADATA", "READ_QUERY")); + result.setDetails(safeProbeDetails(candidate, fingerprint)); + candidate.setLastTestStatus(probe.supported() ? "SUCCESS" : "FAILED"); + candidate.setLastTestMessage(result.getMessage()); + candidate.setCompatibilityStatus( + probe.supported() ? "CODE_SUPPORTED_UNVERIFIED" : "UNSUPPORTED"); + applyFingerprint(candidate, fingerprint); + } catch (FederationSqlException exception) { + LOGGER.warn("候选数据源探测失败，sourceId={}, code={}", + candidate.getId(), exception.errorCode(), exception); + result.setSuccess(false); + result.setErrorCode(exception.errorCode().name()); + result.setMessage(bounded(exception.getMessage())); + candidate.setLastTestStatus("FAILED"); + candidate.setLastTestMessage(result.getMessage()); + } catch (RuntimeException exception) { + LOGGER.error("候选数据源探测发生未预期错误，sourceId={}", + candidate.getId(), exception); + result.setSuccess(false); + result.setErrorCode("CONNECTION_FAILED"); + result.setMessage("无法连接到数据库，请检查地址、账号和网络"); + candidate.setLastTestStatus("FAILED"); + candidate.setLastTestMessage(result.getMessage()); + } + candidate.setLastTestedAt(testedAt); + return result; + } + + /** + * 加密新密码，空密码则保留已有密文。 + * + * @param target 目标实体 + * @param existing 当前实体 + * @param password 可选明文密码 + */ + public void applyCredential( + DatacenterSource target, + DatacenterSource existing, + String password) { + if (password != null) { + target.setCredentialCipher(credentialCipher.encrypt(password)); + return; + } + if ((target.getCredentialCipher() == null + || target.getCredentialCipher().isBlank()) && existing != null) { + target.setCredentialCipher(existing.getCredentialCipher()); + } + } + + /** + * 从旧兼容配置中读取待加密密码。 + * + * @param configJson 兼容配置 + * @return 归一化密码；不存在时为空 + */ + public String extractPassword(Map configJson) { + if (configJson == null || configJson.get("password") == null) { + return null; + } + return trimToNull(String.valueOf(configJson.get("password"))); + } + + /** + * 复制数据源快照，避免候选探测修改权威实体。 + * + * @param source 原实体 + * @return 独立副本 + */ + public DatacenterSource copy(DatacenterSource source) { + DatacenterSource copy = new DatacenterSource(); + copy.setId(source.getId()); + copy.setTenantId(source.getTenantId()); + copy.setDeptId(source.getDeptId()); + copy.setCreated(source.getCreated()); + copy.setCreatedBy(source.getCreatedBy()); + copy.setSourceName(source.getSourceName()); + copy.setSourceCode(source.getSourceCode()); + copy.setSourceType(source.getSourceType()); + copy.setAdapterId(source.getAdapterId()); + copy.setAdapterOptionsJson(source.getAdapterOptionsJson()); + copy.setDefinitionRevision(source.getDefinitionRevision()); + copy.setDefinitionChecksum(source.getDefinitionChecksum()); + copy.setScopeRevision(source.getScopeRevision()); + copy.setAccessMode(source.getAccessMode()); + copy.setBuiltinFlag(source.getBuiltinFlag()); + copy.setCredentialCipher(source.getCredentialCipher()); + copy.setDriverClassName(source.getDriverClassName()); + copy.setJdbcUrl(source.getJdbcUrl()); + copy.setHost(source.getHost()); + copy.setPort(source.getPort()); + copy.setDatabaseName(source.getDatabaseName()); + copy.setSchemaName(source.getSchemaName()); + copy.setUsername(source.getUsername()); + copy.setConfigJson(source.getConfigJson()); + copy.setCapabilitiesJson(source.getCapabilitiesJson()); + copy.setOptions(source.getOptions()); + copy.setStatus(source.getStatus()); + copy.setCompatibilityStatus(source.getCompatibilityStatus()); + copy.setDatabaseProductName(source.getDatabaseProductName()); + copy.setDatabaseProductVersion(source.getDatabaseProductVersion()); + copy.setDriverName(source.getDriverName()); + copy.setDriverVersion(source.getDriverVersion()); + copy.setMetadataRefreshStatus(source.getMetadataRefreshStatus()); + copy.setMetadataRefreshedAt(source.getMetadataRefreshedAt()); + copy.setLastTestStatus(source.getLastTestStatus()); + copy.setLastTestMessage(source.getLastTestMessage()); + copy.setLastTestedAt(source.getLastTestedAt()); + return copy; + } + + private void validateDefinition(DatacenterSourceDraftRequest request) { + if (request == null || request.sourceName() == null + || request.sourceName().isBlank()) { + throw new BusinessException("请输入连接名称"); + } + if (!DatacenterSourceType.MYSQL.name().equals(request.sourceType()) + && !DatacenterSourceType.POSTGRESQL.name().equals(request.sourceType())) { + throw new BusinessException("当前批次仅支持 MySQL 和 PostgreSQL"); + } + if (request.host() == null || request.host().isBlank() + || request.databaseName() == null || request.databaseName().isBlank() + || request.username() == null || request.username().isBlank()) { + throw new BusinessException("请完整填写主机、数据库和用户名"); + } + } + + private void requireEditableDraft( + DatacenterSource existing, + String requestedSourceType) { + if (existing == null) { + return; + } + if (DatacenterSourceStatus.fromCode(existing.getStatus()) + != DatacenterSourceStatus.DRAFT) { + throw new BusinessException("已激活连接不可直接修改，请使用重配置"); + } + if (!java.util.Objects.equals(existing.getSourceType(), requestedSourceType)) { + throw new BusinessException("数据连接类型创建后不可修改"); + } + } + + private void requireCredential(DatacenterSource source) { + if (source.getCredentialCipher() == null + || source.getCredentialCipher().isBlank()) { + throw new BusinessException("请输入数据库密码"); + } + } + + private Map safeProbeDetails( + DatacenterSource source, + RuntimeFingerprint fingerprint) { + Map details = new LinkedHashMap<>(); + details.put("databaseProductName", fingerprint.databaseProduct()); + details.put("databaseProductVersion", fingerprint.databaseVersion()); + details.put("driverName", fingerprint.driverName()); + details.put("driverVersion", fingerprint.driverVersion()); + details.put("effectiveDriverClassName", source.getDriverClassName()); + details.put("effectivePort", source.getPort()); + return java.util.Collections.unmodifiableMap(details); + } + + private void applyFingerprint( + DatacenterSource source, + RuntimeFingerprint fingerprint) { + source.setDatabaseProductName(fingerprint.databaseProduct()); + source.setDatabaseProductVersion(fingerprint.databaseVersion()); + source.setDriverName(fingerprint.driverName()); + source.setDriverVersion(fingerprint.driverVersion()); + } + + private String trimToNull(String value) { + return value == null || value.isBlank() ? null : value.trim(); + } + + private long value(Long sourceValue) { + return sourceValue == null ? 0L : sourceValue; + } + + private String bounded(String value) { + if (value == null) { + return null; + } + return value.length() <= 500 ? value : value.substring(0, 500); + } +} diff --git a/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/meta/service/impl/DatacenterSourceMetadataRefreshService.java b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/meta/service/impl/DatacenterSourceMetadataRefreshService.java new file mode 100644 index 00000000..1f96d36f --- /dev/null +++ b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/meta/service/impl/DatacenterSourceMetadataRefreshService.java @@ -0,0 +1,259 @@ +package tech.easyflow.datacenter.meta.service.impl; + +import com.mybatisflex.core.query.QueryWrapper; +import java.math.BigInteger; +import java.util.ArrayList; +import java.util.Date; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Service; +import org.springframework.transaction.PlatformTransactionManager; +import org.springframework.transaction.support.TransactionTemplate; +import tech.easyflow.common.entity.LoginAccount; +import tech.easyflow.common.web.exceptions.BusinessException; +import tech.easyflow.datacenter.connector.DatacenterConnector; +import tech.easyflow.datacenter.connector.DatacenterConnectorRegistry; +import tech.easyflow.datacenter.entity.DatacenterTable; +import tech.easyflow.datacenter.federation.DatacenterFederationChangeNotifier; +import tech.easyflow.datacenter.mapper.DatacenterCatalogMapper; +import tech.easyflow.datacenter.mapper.DatacenterSourceMapper; +import tech.easyflow.datacenter.mapper.DatacenterTableMapper; +import tech.easyflow.datacenter.meta.entity.DatacenterCatalog; +import tech.easyflow.datacenter.meta.entity.DatacenterSource; +import tech.easyflow.datacenter.meta.enums.DatacenterMetadataStatus; +import tech.easyflow.datacenter.meta.enums.DatacenterSourceStatus; +import tech.easyflow.datacenter.meta.model.DatacenterManagedMetadataSnapshot; +import tech.easyflow.datacenter.meta.model.DatacenterTableDetailMeta; +import tech.easyflow.datacenter.meta.service.DatacenterDatasetRegistryService; + +/** + * 已纳管元数据刷新协调器，将远程 JDBC 读取与平台数据库短事务明确分离。 + */ +@Service +public class DatacenterSourceMetadataRefreshService { + + private static final Logger LOGGER = + LoggerFactory.getLogger(DatacenterSourceMetadataRefreshService.class); + + private final DatacenterConnectorRegistry connectorRegistry; + private final DatacenterDatasetRegistryService registryService; + private final DatacenterCatalogMapper catalogMapper; + private final DatacenterTableMapper tableMapper; + private final DatacenterSourceMapper sourceMapper; + private final DatacenterFederationChangeNotifier changeNotifier; + private final TransactionTemplate transactionTemplate; + + /** + * 创建元数据刷新协调器。 + * + * @param connectorRegistry Connector 注册表 + * @param registryService 数据集注册服务 + * @param catalogMapper 命名空间 Mapper + * @param tableMapper 表 Mapper + * @param sourceMapper 数据源 Mapper + * @param changeNotifier 多节点变更通知器 + * @param transactionManager 事务管理器 + */ + public DatacenterSourceMetadataRefreshService( + DatacenterConnectorRegistry connectorRegistry, + DatacenterDatasetRegistryService registryService, + DatacenterCatalogMapper catalogMapper, + DatacenterTableMapper tableMapper, + DatacenterSourceMapper sourceMapper, + DatacenterFederationChangeNotifier changeNotifier, + PlatformTransactionManager transactionManager) { + this.connectorRegistry = connectorRegistry; + this.registryService = registryService; + this.catalogMapper = catalogMapper; + this.tableMapper = tableMapper; + this.sourceMapper = sourceMapper; + this.changeNotifier = changeNotifier; + this.transactionTemplate = new TransactionTemplate(transactionManager); + } + + /** + * 刷新已纳管对象；外部新表和新字段不会自动扩张查询范围。 + * + * @param source 已完成租户校验的数据源 revision 快照 + * @param account 当前账号 + * @return 刷新后的权威数据源 + */ + public DatacenterSource refresh( + DatacenterSource source, + LoginAccount account) { + DatacenterSourceStatus sourceStatus = + DatacenterSourceStatus.fromCode(source.getStatus()); + if (sourceStatus != DatacenterSourceStatus.READY + && sourceStatus != DatacenterSourceStatus.DEGRADED) { + throw new BusinessException("当前连接状态不支持刷新元数据"); + } + DatacenterSource refreshed; + try { + // 远程元数据读取不持有平台数据库事务连接。 + List snapshot = inspect(source); + refreshed = transactionTemplate.execute(status -> + applySnapshot(source, snapshot, account)); + } catch (RuntimeException exception) { + markFailed(source, account); + throw exception; + } + if (refreshed == null) { + throw new BusinessException("元数据刷新事务未完成"); + } + changeNotifier.publishAfterCommit(refreshed); + return refreshed; + } + + private List inspect(DatacenterSource source) { + DatacenterConnector connector = + connectorRegistry.getConnector(source.getSourceType()); + List catalogs = catalogMapper.selectListByQuery( + QueryWrapper.create() + .eq(DatacenterCatalog::getSourceId, source.getId()) + .ne(DatacenterCatalog::getMetadataStatus, + DatacenterMetadataStatus.RETIRED.name())); + List refreshes = new ArrayList<>(catalogs.size()); + for (DatacenterCatalog catalog : catalogs) { + List tables = tableMapper.selectListByQuery( + QueryWrapper.create() + .eq(DatacenterTable::getSourceId, source.getId()) + .eq(DatacenterTable::getCatalogId, catalog.getId()) + .ne(DatacenterTable::getMetadataStatus, + DatacenterMetadataStatus.RETIRED.name())); + List physicalNames = tables.stream() + .map(this::physicalTableName) + .toList(); + DatacenterManagedMetadataSnapshot snapshot = + connector.inspectManagedTables( + source, catalog.getCatalogName(), physicalNames); + Map detailsByPhysicalName = + snapshot.details().stream() + .filter(detail -> detail.getTable() != null) + .collect(Collectors.toMap( + detail -> physicalTableName(detail.getTable()), + detail -> detail, + (first, ignored) -> first, + LinkedHashMap::new)); + List tableRefreshes = + new ArrayList<>(tables.size()); + for (DatacenterTable table : tables) { + String physicalName = physicalTableName(table); + DatacenterTableDetailMeta detail = + detailsByPhysicalName.get(physicalName); + if (detail != null) { + tableRefreshes.add(new ManagedTableRefresh( + table.getId(), detail, false)); + } else if (snapshot.missingTableNames().contains(physicalName)) { + tableRefreshes.add(new ManagedTableRefresh( + table.getId(), null, true)); + } else { + throw new BusinessException( + "元数据刷新结果不完整: " + physicalName); + } + } + refreshes.add(new ManagedCatalogRefresh(catalog, tableRefreshes)); + } + return List.copyOf(refreshes); + } + + private DatacenterSource applySnapshot( + DatacenterSource source, + List refreshes, + LoginAccount account) { + boolean scopeChanged = false; + for (ManagedCatalogRefresh catalogRefresh : refreshes) { + for (ManagedTableRefresh tableRefresh : catalogRefresh.tables()) { + if (tableRefresh.missing()) { + scopeChanged |= registryService.markManagedTableMissing( + source, tableRefresh.tableId(), account); + } else { + scopeChanged |= registryService.refreshManagedTable( + source, + catalogRefresh.catalog(), + tableRefresh.tableId(), + tableRefresh.detail(), + account); + } + } + } + DatacenterSource patch = new DatacenterSource(); + patch.setId(source.getId()); + patch.setScopeRevision(scopeChanged + ? Math.max(1L, value(source.getScopeRevision())) + 1L + : source.getScopeRevision()); + patch.setMetadataRefreshStatus(scopeChanged ? "CHANGED" : "SUCCESS"); + patch.setMetadataRefreshedAt(new Date()); + patch.setModified(new Date()); + patch.setModifiedBy(accountId(account)); + QueryWrapper guard = QueryWrapper.create() + .eq(DatacenterSource::getId, source.getId()) + .eq(DatacenterSource::getTenantId, source.getTenantId()) + .eq(DatacenterSource::getDefinitionRevision, + source.getDefinitionRevision()) + .eq(DatacenterSource::getScopeRevision, source.getScopeRevision()) + .eq(DatacenterSource::getStatus, source.getStatus()); + if (sourceMapper.updateByQuery(patch, guard) != 1) { + throw new BusinessException("连接或数据范围已变化，请刷新后重试"); + } + return sourceMapper.selectOneByQuery(QueryWrapper.create() + .eq(DatacenterSource::getId, source.getId()) + .eq(DatacenterSource::getTenantId, source.getTenantId())); + } + + private void markFailed(DatacenterSource source, LoginAccount account) { + DatacenterSource patch = new DatacenterSource(); + patch.setId(source.getId()); + patch.setMetadataRefreshStatus("FAILED"); + patch.setModified(new Date()); + patch.setModifiedBy(accountId(account)); + try { + sourceMapper.updateByQuery(patch, QueryWrapper.create() + .eq(DatacenterSource::getId, source.getId()) + .eq(DatacenterSource::getTenantId, source.getTenantId()) + .eq(DatacenterSource::getDefinitionRevision, + source.getDefinitionRevision()) + .eq(DatacenterSource::getScopeRevision, + source.getScopeRevision())); + } catch (RuntimeException updateException) { + LOGGER.warn("记录元数据刷新失败状态时发生错误，sourceId={}", + source.getId(), updateException); + } + } + + private String physicalTableName(DatacenterTable table) { + String actualTable = trimToNull(table.getActualTable()); + return actualTable == null ? table.getTableName() : actualTable; + } + + private String trimToNull(String value) { + return value == null || value.isBlank() ? null : value.trim(); + } + + private long value(Long sourceValue) { + return sourceValue == null ? 0L : sourceValue; + } + + private BigInteger accountId(LoginAccount account) { + return account == null || account.getId() == null + ? BigInteger.ZERO : account.getId(); + } + + private record ManagedCatalogRefresh( + DatacenterCatalog catalog, + List tables) { + + private ManagedCatalogRefresh { + tables = List.copyOf(tables); + } + } + + private record ManagedTableRefresh( + BigInteger tableId, + DatacenterTableDetailMeta detail, + boolean missing) { + } +} diff --git a/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/meta/service/impl/DatacenterSourceMetadataService.java b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/meta/service/impl/DatacenterSourceMetadataService.java new file mode 100644 index 00000000..e4554b9e --- /dev/null +++ b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/meta/service/impl/DatacenterSourceMetadataService.java @@ -0,0 +1,317 @@ +package tech.easyflow.datacenter.meta.service.impl; + +import com.mybatisflex.core.query.QueryWrapper; +import java.math.BigInteger; +import java.util.List; +import org.springframework.stereotype.Service; +import tech.easyflow.common.web.exceptions.BusinessException; +import tech.easyflow.datacenter.connector.DatacenterConnectorRegistry; +import tech.easyflow.datacenter.entity.DatacenterTable; +import tech.easyflow.datacenter.mapper.DatacenterCatalogMapper; +import tech.easyflow.datacenter.meta.entity.DatacenterCatalog; +import tech.easyflow.datacenter.meta.entity.DatacenterSource; +import tech.easyflow.datacenter.meta.enums.DatacenterMetadataStatus; +import tech.easyflow.datacenter.meta.enums.DatacenterSourceStatus; +import tech.easyflow.datacenter.meta.enums.DatacenterSourceType; +import tech.easyflow.datacenter.meta.model.DatacenterCatalogMeta; +import tech.easyflow.datacenter.meta.model.DatacenterMetadataPage; +import tech.easyflow.datacenter.meta.model.DatacenterTableDetailMeta; +import tech.easyflow.datacenter.meta.service.DatacenterDatasetRegistryService; + +/** + * 数据源元数据浏览服务，统一处理物理元数据和已纳管元数据的分页边界。 + */ +@Service +public class DatacenterSourceMetadataService { + + private static final long DEFAULT_METADATA_PAGE_SIZE = 100L; + private static final long MAX_METADATA_PAGE_SIZE = 200L; + + private final DatacenterConnectorRegistry connectorRegistry; + private final DatacenterDatasetRegistryService registryService; + private final DatacenterCatalogMapper catalogMapper; + + /** + * 创建元数据浏览服务。 + * + * @param connectorRegistry 数据源 Connector 注册表 + * @param registryService 已纳管数据集注册服务 + * @param catalogMapper 命名空间 Mapper + */ + public DatacenterSourceMetadataService( + DatacenterConnectorRegistry connectorRegistry, + DatacenterDatasetRegistryService registryService, + DatacenterCatalogMapper catalogMapper) { + this.connectorRegistry = connectorRegistry; + this.registryService = registryService; + this.catalogMapper = catalogMapper; + } + + /** + * 返回已发布命名空间，草稿则浏览物理命名空间。 + * + * @param source 已完成租户校验的数据源 + * @return 命名空间列表 + */ + public List listCatalogs(DatacenterSource source) { + DatacenterSourceStatus status = DatacenterSourceStatus.fromCode(source.getStatus()); + if (usesManagedMetadata(source) + || status == DatacenterSourceStatus.READY + || status == DatacenterSourceStatus.DEGRADED + || status == DatacenterSourceStatus.DISABLED) { + return managedCatalogs(source.getId()).stream().map(this::toCatalogMeta).toList(); + } + requireSuccessfulProbe(source); + return connectorRegistry.getConnector(source.getSourceType()).listCatalogs(source); + } + + /** + * 分页返回已发布或物理命名空间。 + * + * @param source 已完成租户校验的数据源 + * @param keyword 名称关键字 + * @param pageNumber 页码 + * @param pageSize 页大小 + * @return 命名空间分页 + */ + public DatacenterMetadataPage listCatalogsPage( + DatacenterSource source, + String keyword, + Long pageNumber, + Long pageSize) { + long actualPage = pageNumber(pageNumber); + long actualSize = pageSize(pageSize); + DatacenterSourceStatus status = DatacenterSourceStatus.fromCode(source.getStatus()); + if (usesManagedMetadata(source) + || status == DatacenterSourceStatus.READY + || status == DatacenterSourceStatus.DEGRADED + || status == DatacenterSourceStatus.DISABLED) { + List managed = managedCatalogs(source.getId()).stream() + .map(this::toCatalogMeta) + .filter(catalog -> matchesKeyword(catalog.getCatalogName(), keyword)) + .toList(); + return DatacenterMetadataPage.slice(managed, actualPage, actualSize); + } + requireSuccessfulProbe(source); + return connectorRegistry.getConnector(source.getSourceType()).listCatalogsPage( + source, keyword, actualPage, actualSize); + } + + /** + * 分页返回命名空间中的表与视图。 + * + * @param source 已完成租户校验的数据源 + * @param catalogName 命名空间名称 + * @param keyword 表名关键字 + * @param pageNumber 页码 + * @param pageSize 页大小 + * @return 表分页 + */ + public DatacenterMetadataPage listTables( + DatacenterSource source, + String catalogName, + String keyword, + Long pageNumber, + Long pageSize) { + long actualPage = pageNumber(pageNumber); + long actualSize = pageSize(pageSize); + DatacenterSourceStatus status = DatacenterSourceStatus.fromCode(source.getStatus()); + if (usesManagedMetadata(source) || status == DatacenterSourceStatus.DISABLED) { + List managed = registryService + .listManagedTables( + source.getId(), resolveCatalogId(source.getId(), catalogName)) + .stream() + .filter(table -> matchesKeyword(table.getTableName(), keyword)) + .toList(); + return DatacenterMetadataPage.slice(managed, actualPage, actualSize); + } + requireSuccessfulProbe(source); + return connectorRegistry.getConnector(source.getSourceType()).listTablesPage( + source, catalogName, keyword, actualPage, actualSize); + } + + /** + * 读取单表字段元数据。 + * + * @param source 已完成租户校验的数据源 + * @param catalogName 命名空间名称 + * @param tableName 表名 + * @param register 是否请求兼容的即时纳管 + * @param fieldPageNumber 字段页码 + * @param fieldPageSize 字段页大小 + * @return 表详情 + */ + public DatacenterTableDetailMeta getTableDetail( + DatacenterSource source, + String catalogName, + String tableName, + boolean register, + Long fieldPageNumber, + Long fieldPageSize) { + if (usesManagedMetadata(source)) { + BigInteger catalogId = resolveCatalogId(source.getId(), catalogName); + DatacenterTable target = registryService + .listManagedTables(source.getId(), catalogId).stream() + .filter(item -> tableName.equals(item.getTableName())) + .findFirst() + .orElseThrow(() -> new BusinessException("数据集不存在: " + tableName)); + DatacenterTableDetailMeta detail = new DatacenterTableDetailMeta(); + detail.setTable(registryService.getTableWithFields(target.getId())); + detail.setFields(detail.getTable().getFields()); + return pageFields(detail, fieldPageNumber, fieldPageSize); + } + requireSuccessfulProbe(source); + DatacenterTableDetailMeta detail = connectorRegistry + .getConnector(source.getSourceType()) + .getTableDetail(source, catalogName, tableName); + if (register) { + throw new BusinessException("请使用批量接入以原子更新数据范围"); + } + return pageFields(detail, fieldPageNumber, fieldPageSize); + } + + /** + * 浏览未发布候选连接的命名空间。 + * + * @param candidate 候选数据源 + * @return 物理命名空间列表 + */ + public List listCandidateCatalogs( + DatacenterSource candidate) { + return connectorRegistry.getConnector(candidate.getSourceType()) + .listCatalogs(candidate); + } + + /** + * 分页浏览未发布候选连接的命名空间。 + * + * @param candidate 候选数据源 + * @param keyword 名称关键字 + * @param pageNumber 页码 + * @param pageSize 页大小 + * @return 命名空间分页 + */ + public DatacenterMetadataPage listCandidateCatalogsPage( + DatacenterSource candidate, + String keyword, + Long pageNumber, + Long pageSize) { + return connectorRegistry.getConnector(candidate.getSourceType()).listCatalogsPage( + candidate, keyword, pageNumber(pageNumber), pageSize(pageSize)); + } + + /** + * 分页浏览未发布候选连接的表。 + * + * @param candidate 候选数据源 + * @param catalogName 命名空间名称 + * @param keyword 表名关键字 + * @param pageNumber 页码 + * @param pageSize 页大小 + * @return 表分页 + */ + public DatacenterMetadataPage listCandidateTables( + DatacenterSource candidate, + String catalogName, + String keyword, + Long pageNumber, + Long pageSize) { + return connectorRegistry.getConnector(candidate.getSourceType()).listTablesPage( + candidate, + trimToNull(catalogName), + keyword, + pageNumber(pageNumber), + pageSize(pageSize)); + } + + private List managedCatalogs(BigInteger sourceId) { + return catalogMapper.selectListByQuery(QueryWrapper.create() + .eq(DatacenterCatalog::getSourceId, sourceId) + .ne(DatacenterCatalog::getMetadataStatus, + DatacenterMetadataStatus.RETIRED.name())); + } + + private DatacenterTableDetailMeta pageFields( + DatacenterTableDetailMeta detail, + Long fieldPageNumber, + Long fieldPageSize) { + List fields = + detail.getFields() == null ? List.of() : detail.getFields(); + DatacenterMetadataPage page = + DatacenterMetadataPage.slice( + fields, pageNumber(fieldPageNumber), pageSize(fieldPageSize)); + detail.setFields(page.records()); + detail.setFieldPageNumber(page.pageNumber()); + detail.setFieldPageSize(page.pageSize()); + detail.setHasMoreFields(page.hasMore()); + if (detail.getTable() != null) { + detail.getTable().setFields(page.records()); + } + return detail; + } + + private BigInteger resolveCatalogId(BigInteger sourceId, String catalogName) { + if (catalogName == null || catalogName.isBlank()) { + return null; + } + DatacenterCatalog catalog = catalogMapper.selectOneByQuery(QueryWrapper.create() + .eq(DatacenterCatalog::getSourceId, sourceId) + .eq(DatacenterCatalog::getCatalogName, catalogName) + .ne(DatacenterCatalog::getMetadataStatus, + DatacenterMetadataStatus.RETIRED.name())); + if (catalog == null) { + throw new BusinessException("命名空间不存在: " + catalogName); + } + return catalog.getId(); + } + + private DatacenterCatalogMeta toCatalogMeta(DatacenterCatalog catalog) { + DatacenterCatalogMeta meta = new DatacenterCatalogMeta(); + meta.setId(catalog.getId()); + meta.setSourceId(catalog.getSourceId()); + meta.setCatalogName(catalog.getCatalogName()); + meta.setCatalogDesc(catalog.getCatalogDesc()); + meta.setCatalogType(catalog.getCatalogType()); + meta.setLogicalSchemaName(catalog.getLogicalSchemaName()); + meta.setPhysicalCatalogName(catalog.getPhysicalCatalogName()); + meta.setPhysicalSchemaName(catalog.getPhysicalSchemaName()); + return meta; + } + + private void requireSuccessfulProbe(DatacenterSource source) { + if (!"SUCCESS".equalsIgnoreCase(source.getLastTestStatus())) { + throw new BusinessException("请先测试连接"); + } + } + + private boolean usesManagedMetadata(DatacenterSource source) { + return DatacenterSourceType.EXCEL.name().equals(source.getSourceType()) + || DatacenterSourceType.EXCEL_MATERIALIZED.name().equals( + source.getSourceType()); + } + + private boolean matchesKeyword(String value, String keyword) { + String normalized = trimToNull(keyword); + return normalized == null || (value != null + && value.toLowerCase(java.util.Locale.ROOT) + .contains(normalized.toLowerCase(java.util.Locale.ROOT))); + } + + private long pageNumber(Long requested) { + return requested == null || requested < 1L + ? 1L : Math.min(requested, 1_000_000L); + } + + private long pageSize(Long requested) { + return requested == null || requested < 1L + ? DEFAULT_METADATA_PAGE_SIZE + : Math.min(requested, MAX_METADATA_PAGE_SIZE); + } + + private String trimToNull(String value) { + if (value == null || value.isBlank()) { + return null; + } + return value.trim(); + } +} diff --git a/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/meta/service/impl/DatacenterSourceServiceImpl.java b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/meta/service/impl/DatacenterSourceServiceImpl.java index ffe74c96..417a7ed6 100644 --- a/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/meta/service/impl/DatacenterSourceServiceImpl.java +++ b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/meta/service/impl/DatacenterSourceServiceImpl.java @@ -1,220 +1,1063 @@ package tech.easyflow.datacenter.meta.service.impl; +import com.easyagents.federation.sql.source.PreparedSourceRuntime; import com.mybatisflex.core.paginate.Page; import com.mybatisflex.core.query.QueryWrapper; import com.mybatisflex.spring.service.impl.ServiceImpl; +import java.math.BigInteger; +import java.util.Date; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.springframework.stereotype.Service; +import org.springframework.transaction.PlatformTransactionManager; import org.springframework.transaction.annotation.Transactional; +import org.springframework.transaction.support.TransactionSynchronization; +import org.springframework.transaction.support.TransactionSynchronizationManager; +import org.springframework.transaction.support.TransactionTemplate; import tech.easyflow.common.entity.LoginAccount; import tech.easyflow.common.web.exceptions.BusinessException; import tech.easyflow.datacenter.connector.DatacenterConnector; import tech.easyflow.datacenter.connector.DatacenterConnectorRegistry; import tech.easyflow.datacenter.entity.DatacenterTable; +import tech.easyflow.datacenter.execution.model.DatacenterConnectionTestResult; +import tech.easyflow.datacenter.federation.DatacenterFederationChangeNotifier; +import tech.easyflow.datacenter.federation.DatacenterMetadataIdentity; +import tech.easyflow.datacenter.federation.DatacenterFederationRuntime; import tech.easyflow.datacenter.mapper.DatacenterCatalogMapper; import tech.easyflow.datacenter.mapper.DatacenterSourceMapper; +import tech.easyflow.datacenter.mapper.DatacenterTableMapper; import tech.easyflow.datacenter.meta.entity.DatacenterCatalog; import tech.easyflow.datacenter.meta.entity.DatacenterSource; +import tech.easyflow.datacenter.meta.enums.DatacenterMetadataStatus; +import tech.easyflow.datacenter.meta.enums.DatacenterSourceStatus; import tech.easyflow.datacenter.meta.enums.DatacenterSourceType; import tech.easyflow.datacenter.meta.model.DatacenterBatchRegisterRequest; import tech.easyflow.datacenter.meta.model.DatacenterCatalogMeta; +import tech.easyflow.datacenter.meta.model.DatacenterMetadataPage; +import tech.easyflow.datacenter.meta.model.DatacenterSourceActivateRequest; +import tech.easyflow.datacenter.meta.model.DatacenterSourceCandidateMetadataRequest; +import tech.easyflow.datacenter.meta.model.DatacenterSourceCandidateCatalogRequest; +import tech.easyflow.datacenter.meta.model.DatacenterSourceDraftRequest; +import tech.easyflow.datacenter.meta.model.DatacenterSourceReconfigureRequest; +import tech.easyflow.datacenter.meta.model.DatacenterSourceView; +import tech.easyflow.datacenter.meta.model.DatacenterSourceViews; import tech.easyflow.datacenter.meta.model.DatacenterTableDetailMeta; import tech.easyflow.datacenter.meta.service.DatacenterDatasetRegistryService; import tech.easyflow.datacenter.meta.service.DatacenterSourceService; import tech.easyflow.datacenter.meta.support.DatacenterSourceConnectionDefaults; -import tech.easyflow.datacenter.execution.model.DatacenterConnectionTestResult; -import tech.easyflow.datacenter.security.DatacenterCredentialCipher; - -import javax.annotation.Resource; -import java.math.BigInteger; -import java.util.Date; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; +/** + * 数据源草稿、探测、元数据选择、激活和墓碑的统一应用服务。 + */ @Service -public class DatacenterSourceServiceImpl extends ServiceImpl implements DatacenterSourceService { +public class DatacenterSourceServiceImpl + extends ServiceImpl + implements DatacenterSourceService { - @Resource - private DatacenterConnectorRegistry connectorRegistry; - @Resource - private DatacenterDatasetRegistryService registryService; - @Resource - private DatacenterCredentialCipher credentialCipher; - @Resource - private DatacenterCatalogMapper catalogMapper; - @Resource - private DatacenterSourceConnectionDefaults connectionDefaults; + private static final Logger LOGGER = LoggerFactory.getLogger(DatacenterSourceServiceImpl.class); + private static final int MAX_TABLES_PER_ACTIVATION = 200; + private final DatacenterConnectorRegistry connectorRegistry; + private final DatacenterDatasetRegistryService registryService; + private final DatacenterCatalogMapper catalogMapper; + private final DatacenterTableMapper tableMapper; + private final DatacenterSourceConnectionDefaults connectionDefaults; + private final DatacenterSourceCandidateService candidateService; + private final DatacenterSourceMetadataService metadataService; + private final DatacenterSourceMetadataRefreshService metadataRefreshService; + private final DatacenterFederationRuntime federationRuntime; + private final DatacenterFederationChangeNotifier changeNotifier; + private final TransactionTemplate transactionTemplate; + + /** + * 创建数据源应用服务。 + * + * @param connectorRegistry 兼容元数据 Connector 注册表 + * @param registryService 数据集注册服务 + * @param catalogMapper 命名空间 Mapper + * @param tableMapper 表 Mapper + * @param connectionDefaults JDBC 默认配置 + * @param candidateService 候选配置服务 + * @param metadataService 元数据浏览服务 + * @param metadataRefreshService 已纳管元数据刷新服务 + * @param federationRuntime Federation Runtime + * @param changeNotifier 多节点变更提示发布器 + * @param transactionManager 事务管理器 + */ + public DatacenterSourceServiceImpl( + DatacenterConnectorRegistry connectorRegistry, + DatacenterDatasetRegistryService registryService, + DatacenterCatalogMapper catalogMapper, + DatacenterTableMapper tableMapper, + DatacenterSourceConnectionDefaults connectionDefaults, + DatacenterSourceCandidateService candidateService, + DatacenterSourceMetadataService metadataService, + DatacenterSourceMetadataRefreshService metadataRefreshService, + DatacenterFederationRuntime federationRuntime, + DatacenterFederationChangeNotifier changeNotifier, + PlatformTransactionManager transactionManager) { + this.connectorRegistry = connectorRegistry; + this.registryService = registryService; + this.catalogMapper = catalogMapper; + this.tableMapper = tableMapper; + this.connectionDefaults = connectionDefaults; + this.candidateService = candidateService; + this.metadataService = metadataService; + this.metadataRefreshService = metadataRefreshService; + this.federationRuntime = federationRuntime; + this.changeNotifier = changeNotifier; + this.transactionTemplate = new TransactionTemplate(transactionManager); + } + + /** + * 保存 Excel 或项目内置数据源；外部 JDBC 接入使用 {@link #saveDraft}。 + * + * @param source 内部数据源实体 + * @param account 当前账号 + * @return 保存后的实体 + */ @Override + @Transactional(rollbackFor = Exception.class) public DatacenterSource saveSource(DatacenterSource source, LoginAccount account) { if (source == null || source.getSourceType() == null || source.getSourceType().isBlank()) { throw new BusinessException("数据源类型不能为空"); } - DatacenterSource existing = source.getId() == null ? null : getById(source.getId()); + DatacenterSource existing = source.getId() == null + ? null : requireSource(source.getId(), account); DatacenterSource normalized = mergeWithExisting(existing, source); DatacenterConnector connector = connectorRegistry.getConnector(normalized.getSourceType()); - applyCredentialCipher(normalized, existing); + candidateService.applyCredential( + normalized, existing, + candidateService.extractPassword(normalized.getConfigJson())); normalized.setConfigJson(connectionDefaults.sanitizeConfig(normalized.getConfigJson())); connectionDefaults.normalize(normalized); - normalized.setCapabilitiesJson(Map.of("capabilities", connector.getCapabilities().stream().map(Enum::name).toList())); - Date now = new Date(); - if (normalized.getId() == null) { - normalized.setCreated(now); - normalized.setCreatedBy(account == null ? BigInteger.ZERO : account.getId()); - normalized.setTenantId(account == null ? BigInteger.ZERO : account.getTenantId()); - normalized.setDeptId(account == null ? BigInteger.ZERO : account.getDeptId()); - normalized.setStatus(normalized.getStatus() == null ? 0 : normalized.getStatus()); - normalized.setModified(now); - normalized.setModifiedBy(account == null ? BigInteger.ZERO : account.getId()); - save(normalized); - } else { - if (existing == null) { - throw new BusinessException("连接不存在"); - } - normalized.setModified(now); - normalized.setModifiedBy(account == null ? BigInteger.ZERO : account.getId()); - updateById(normalized); + normalized.setCapabilitiesJson(Map.of( + "capabilities", + connector.getCapabilities().stream().map(Enum::name).toList())); + normalized.setDefinitionRevision(value(normalized.getDefinitionRevision())); + normalized.setScopeRevision(Math.max(1L, value(normalized.getScopeRevision()))); + if (normalized.getStatus() == null) { + normalized.setStatus(DatacenterSourceType.PROJECT_MYSQL.name().equals(normalized.getSourceType()) + ? DatacenterSourceStatus.READY.code() + : DatacenterSourceStatus.DRAFT.code()); } + saveOrUpdate(normalized, existing, account); + ensureGeneratedSourceCode(normalized); return getById(normalized.getId()); } + /** + * 保存不直接进入查询状态的外部 JDBC 草稿。 + * + * @param request 草稿请求 + * @param account 当前账号 + * @return 安全视图 + */ @Override - public Page pageSources(Long pageNumber, Long pageSize, LoginAccount account) { - registryService.ensureBuiltinSource(DatacenterSourceType.PROJECT_MYSQL, account); - return page(new Page<>(pageNumber == null ? 1L : pageNumber, pageSize == null ? 10L : pageSize), QueryWrapper.create()); + @Transactional(rollbackFor = Exception.class) + public DatacenterSourceView saveDraft(DatacenterSourceDraftRequest request, LoginAccount account) { + if (request == null) { + throw new BusinessException("连接配置不能为空"); + } + DatacenterSource existing = request.id() == null + ? null : requireSource(request.id(), account); + DatacenterSource source = candidateService.buildDraft( + request, existing, account); + saveOrUpdate(source, existing, account); + ensureGeneratedSourceCode(source); + DatacenterSource saved = requireSource(source.getId(), account); + retireRuntimeAfterCommit(saved); + changeNotifier.publishAfterCommit(saved); + return toView(saved); } + /** + * 分页返回未删除且不含凭据的数据源视图。 + */ @Override - public DatacenterConnectionTestResult testConnection(DatacenterSource source, LoginAccount account) { - DatacenterSource existing = source != null && source.getId() != null ? getById(source.getId()) : null; - if (source != null && source.getId() != null && existing == null) { - throw new BusinessException("连接不存在"); + public Page pageSources(Long pageNumber, Long pageSize, LoginAccount account) { + registryService.ensureBuiltinSource(DatacenterSourceType.PROJECT_MYSQL, account); + long actualPage = pageNumber == null || pageNumber < 1L ? 1L : pageNumber; + long actualSize = pageSize == null || pageSize < 1L ? 20L : Math.min(pageSize, 100L); + QueryWrapper query = QueryWrapper.create() + .eq(DatacenterSource::getTenantId, tenantId(account)) + .ne(DatacenterSource::getStatus, DatacenterSourceStatus.DELETED.code()) + .orderBy("created desc"); + Page sourcePage = page(new Page<>(actualPage, actualSize), query); + return new Page<>(sourcePage.getRecords().stream().map(this::toView).toList(), + sourcePage.getPageNumber(), sourcePage.getPageSize(), sourcePage.getTotalRow()); + } + + /** + * 使用 Federation Adapter 探测草稿并保存实际数据库/Driver 指纹。 + */ + @Override + public DatacenterConnectionTestResult probe(BigInteger sourceId, LoginAccount account) { + DatacenterSource source = requireSource(sourceId, account); + if (!isFederated(source.getSourceType())) { + return connectorRegistry.getConnector(source.getSourceType()).testConnection(source); } - DatacenterSource actual = mergeWithExisting(existing, source); - if (DatacenterSourceType.PROJECT_MYSQL.name().equals(actual.getSourceType())) { - actual = registryService.ensureBuiltinSource(DatacenterSourceType.PROJECT_MYSQL, account); - } - applyCredentialCipher(actual, existing); - actual.setConfigJson(connectionDefaults.sanitizeConfig(actual.getConfigJson())); - connectionDefaults.normalize(actual); - DatacenterConnector connector = connectorRegistry.getConnector(actual.getSourceType()); - DatacenterConnectionTestResult result = connector.testConnection(actual); - Map details = new LinkedHashMap<>(); - if (result.getDetails() != null) { - details.putAll(result.getDetails()); - } - details.put("effectiveDriverClassName", actual.getDriverClassName()); - details.put("effectiveJdbcUrl", actual.getJdbcUrl()); - details.put("effectivePort", actual.getPort()); - result.setDetails(details); - if (actual.getId() != null) { - DatacenterSource persisted = new DatacenterSource(); - persisted.setId(actual.getId()); - persisted.setLastTestStatus(result.isSuccess() ? "SUCCESS" : "FAILED"); - persisted.setLastTestMessage(result.getMessage()); - persisted.setLastTestedAt(new Date()); - updateById(persisted); + DatacenterConnectionTestResult result = candidateService.probe(source); + Date testedAt = source.getLastTestedAt(); + DatacenterSource update = new DatacenterSource(); + update.setId(source.getId()); + update.setLastTestStatus(source.getLastTestStatus()); + update.setLastTestMessage(source.getLastTestMessage()); + update.setLastTestedAt(source.getLastTestedAt()); + update.setCompatibilityStatus(source.getCompatibilityStatus()); + update.setDatabaseProductName(source.getDatabaseProductName()); + update.setDatabaseProductVersion(source.getDatabaseProductVersion()); + update.setDriverName(source.getDriverName()); + update.setDriverVersion(source.getDriverVersion()); + update.setModified(testedAt); + update.setModifiedBy(accountId(account)); + QueryWrapper probeGuard = QueryWrapper.create() + .eq(DatacenterSource::getId, source.getId()) + .eq(DatacenterSource::getTenantId, tenantId(account)) + .eq(DatacenterSource::getDefinitionRevision, source.getDefinitionRevision()); + if (getMapper().updateByQuery(update, probeGuard) != 1) { + throw new BusinessException("连接配置已变化，请重新测试"); } return result; } + /** + * 返回当前账号可见的物理 Catalog/Schema。 + */ @Override public List listCatalogs(BigInteger sourceId, LoginAccount account) { - DatacenterSource source = registryService.getSourceRequired(sourceId); - if (isManagedOnly(source.getSourceType())) { - QueryWrapper wrapper = QueryWrapper.create(); - wrapper.eq(DatacenterCatalog::getSourceId, sourceId); - return catalogMapper.selectListByQuery(wrapper).stream().map(this::toCatalogMeta).collect(Collectors.toList()); - } - return connectorRegistry.getConnector(source.getSourceType()).listCatalogs(source); + DatacenterSource source = requireSource(sourceId, account); + return metadataService.listCatalogs(source); } + /** + * 分页浏览当前账号可见的 Catalog 或 Schema。 + */ @Override - public List listTables(BigInteger sourceId, String catalogName, LoginAccount account) { - DatacenterSource source = registryService.getSourceRequired(sourceId); - if (isManagedOnly(source.getSourceType())) { - BigInteger catalogId = resolveCatalogId(sourceId, catalogName); - return registryService.listManagedTables(sourceId, catalogId); - } - return connectorRegistry.getConnector(source.getSourceType()).listTables(source, catalogName); + public DatacenterMetadataPage listCatalogsPage( + BigInteger sourceId, + String keyword, + Long pageNumber, + Long pageSize, + LoginAccount account) { + DatacenterSource source = requireSource(sourceId, account); + return metadataService.listCatalogsPage( + source, keyword, pageNumber, pageSize); } + /** + * 返回目标命名空间中可访问的表与视图。 + */ @Override - public DatacenterTableDetailMeta getTableDetail(BigInteger sourceId, String catalogName, String tableName, boolean register, LoginAccount account) { - DatacenterSource source = registryService.getSourceRequired(sourceId); - if (isManagedOnly(source.getSourceType())) { - BigInteger catalogId = resolveCatalogId(sourceId, catalogName); - List tables = registryService.listManagedTables(sourceId, catalogId); - DatacenterTable target = tables.stream().filter(item -> tableName.equals(item.getTableName())).findFirst() - .orElseThrow(() -> new BusinessException("数据集不存在: " + tableName)); - DatacenterTableDetailMeta detail = new DatacenterTableDetailMeta(); - detail.setTable(registryService.getTableWithFields(target.getId())); - detail.setFields(detail.getTable().getFields()); - return detail; + public DatacenterMetadataPage listTables( + BigInteger sourceId, + String catalogName, + String keyword, + Long pageNumber, + Long pageSize, + LoginAccount account) { + DatacenterSource source = requireSource(sourceId, account); + return metadataService.listTables( + source, catalogName, keyword, pageNumber, pageSize); + } + + /** + * 读取表字段；兼容旧页面的单表纳管入口。 + */ + @Override + public DatacenterTableDetailMeta getTableDetail( + BigInteger sourceId, + String catalogName, + String tableName, + boolean register, + Long fieldPageNumber, + Long fieldPageSize, + LoginAccount account) { + DatacenterSource source = requireSource(sourceId, account); + return metadataService.getTableDetail( + source, catalogName, tableName, register, + fieldPageNumber, fieldPageSize); + } + + /** + * 激活单个物理命名空间及其选定表，并发布新 revision。 + */ + @Override + public DatacenterSourceView activate(DatacenterSourceActivateRequest request, LoginAccount account) { + if (request == null || request.sourceId() == null) { + throw new BusinessException("数据连接不能为空"); } + String catalogName = trimToNull(request.catalogName()); + Set selectedTables = (request.tableNames() == null + ? List.of() : request.tableNames()).stream() + .map(this::trimToNull) + .filter(java.util.Objects::nonNull) + .collect(Collectors.toCollection(LinkedHashSet::new)); + if (catalogName == null || selectedTables.isEmpty()) { + throw new BusinessException("请选择数据范围"); + } + if (selectedTables.size() > MAX_TABLES_PER_ACTIVATION) { + throw new BusinessException("单次最多接入 " + MAX_TABLES_PER_ACTIVATION + " 张表"); + } + DatacenterSource source = requireSource(request.sourceId(), account); + if (!isFederated(source.getSourceType())) { + throw new BusinessException("当前连接不使用 Federation SQL 激活流程"); + } + if (DatacenterSourceStatus.fromCode(source.getStatus()) + != DatacenterSourceStatus.DRAFT) { + throw new BusinessException("只有草稿连接可以激活，请使用重新配置功能"); + } + DatacenterConnectionTestResult probe = probe(source.getId(), account); + if (!probe.isSuccess()) { + throw new BusinessException(probe.getMessage()); + } + source = requireSource(source.getId(), account); DatacenterConnector connector = connectorRegistry.getConnector(source.getSourceType()); - DatacenterTableDetailMeta detail = connector.getTableDetail(source, catalogName, tableName); - if (register) { - DatacenterCatalog catalog = registryService.ensureCatalog(source, catalogName, account); - DatacenterTable table = registryService.registerTable(source, catalog, detail, account); - detail.setTable(table); - detail.setFields(table.getFields()); + // 批量详情读取会逐项执行精确身份校验，避免为少量选择全量扫描大型 Schema。 + List tableDetails = connector.getTableDetails( + source, catalogName, List.copyOf(selectedTables)); + DatacenterSource activationSource = candidateService.copy(source); + activationSource.setDefinitionRevision(value(source.getDefinitionRevision()) + 1L); + DatacenterSource sourceSnapshot = source; + try (PreparedSourceRuntime prepared = federationRuntime.prepareCandidate( + activationSource, catalogName)) { + DatacenterSource published = transactionTemplate.execute(status -> + persistActivation( + sourceSnapshot, catalogName, tableDetails, + prepared.definition().checksum(), account)); + if (published == null) { + throw new BusinessException("连接激活事务未完成"); + } + commitPreparedRuntime( + published, prepared, account, "连接激活后的 Runtime 切换失败"); + changeNotifier.publishAfterCommit(published); + return toView(published); } - return detail; } + /** + * 使用未发布候选配置探测活动数据源。 + */ + @Override + public DatacenterConnectionTestResult probeCandidate( + DatacenterSourceDraftRequest request, + LoginAccount account) { + DatacenterSource candidate = buildActiveCandidate(request, account); + return candidateService.probe(candidate); + } + + /** + * 浏览未发布候选配置的可访问命名空间。 + */ + @Override + public List listCandidateCatalogs( + DatacenterSourceDraftRequest request, + LoginAccount account) { + DatacenterSource candidate = buildActiveCandidate(request, account); + return metadataService.listCandidateCatalogs(candidate); + } + + /** + * 分页浏览未发布候选配置的命名空间。 + */ + @Override + public DatacenterMetadataPage listCandidateCatalogsPage( + DatacenterSourceCandidateCatalogRequest request, + LoginAccount account) { + if (request == null || request.definition() == null) { + throw new BusinessException("候选连接配置不能为空"); + } + DatacenterSource candidate = buildActiveCandidate(request.definition(), account); + return metadataService.listCandidateCatalogsPage( + candidate, request.keyword(), request.pageNumber(), request.pageSize()); + } + + /** + * 分页浏览未发布候选配置的可访问表。 + */ + @Override + public DatacenterMetadataPage listCandidateTables( + DatacenterSourceCandidateMetadataRequest request, + LoginAccount account) { + if (request == null || request.definition() == null) { + throw new BusinessException("候选连接配置不能为空"); + } + DatacenterSource candidate = buildActiveCandidate(request.definition(), account); + return metadataService.listCandidateTables( + candidate, request.catalogName(), request.keyword(), + request.pageNumber(), request.pageSize()); + } + + /** + * 候选连接和范围验证成功后，以 revision CAS 原地发布新配置。 + */ + @Override + public DatacenterSourceView reconfigure( + DatacenterSourceReconfigureRequest request, + LoginAccount account) { + if (request == null || request.definition() == null) { + throw new BusinessException("候选连接配置不能为空"); + } + DatacenterSource current = requireSource(request.definition().id(), account); + requireActiveReconfiguration(current, request); + DatacenterSource candidate = candidateService.buildReconfigurationCandidate( + current, request.definition()); + DatacenterConnectionTestResult probe = candidateService.probe(candidate); + if (!probe.isSuccess()) { + throw new BusinessException(probe.getMessage()); + } + String catalogName = requireCatalogName(request.catalogName()); + Set selectedTables = selectedTableNames(request.tableNames()); + DatacenterConnector connector = connectorRegistry.getConnector(candidate.getSourceType()); + List tableDetails = connector.getTableDetails( + candidate, catalogName, List.copyOf(selectedTables)); + try (PreparedSourceRuntime prepared = federationRuntime.prepareCandidate( + candidate, catalogName)) { + DatacenterSource published = transactionTemplate.execute(status -> + persistReconfiguration( + current, candidate, catalogName, tableDetails, + prepared.definition().checksum(), account)); + if (published == null) { + throw new BusinessException("连接重配置事务未完成"); + } + commitPreparedRuntime( + published, prepared, account, "连接重配置后的 Runtime 切换失败"); + changeNotifier.publishAfterCommit(published); + return toView(published); + } + } + + /** + * 兼容旧页面的批量纳管入口。 + */ @Override public List batchRegisterTables(DatacenterBatchRegisterRequest request, LoginAccount account) { if (request == null || request.getSourceId() == null) { throw new BusinessException("数据连接不能为空"); } - if (request.getCatalogName() == null || request.getCatalogName().isBlank()) { - throw new BusinessException("库不能为空"); - } - List tableNames = request.getTableNames() == null - ? List.of() - : request.getTableNames().stream().filter(name -> name != null && !name.isBlank()).distinct().toList(); - if (tableNames.isEmpty()) { - throw new BusinessException("至少选择一张表"); - } - DatacenterSource source = registryService.getSourceRequired(request.getSourceId()); + DatacenterSource source = requireSource(request.getSourceId(), account); if (isManagedOnly(source.getSourceType())) { throw new BusinessException("当前数据连接不支持批量接入"); } + DatacenterSourceStatus status = DatacenterSourceStatus.fromCode(source.getStatus()); + if (status != DatacenterSourceStatus.READY + && status != DatacenterSourceStatus.DEGRADED) { + throw new BusinessException("请先激活数据连接"); + } + String catalogName = requireCatalogName(request.getCatalogName()); + BigInteger catalogId = resolveCatalogId(source.getId(), catalogName); + List existingTables = registryService.listManagedTables( + source.getId(), catalogId); + Set existingNames = existingTables.stream() + .map(DatacenterTable::getTableName) + .collect(Collectors.toCollection(LinkedHashSet::new)); + Set requestedNames = selectedTableNames(request.getTableNames()); + requestedNames.removeAll(existingNames); + if (requestedNames.isEmpty()) { + return existingTables; + } DatacenterConnector connector = connectorRegistry.getConnector(source.getSourceType()); - DatacenterCatalog catalog = registryService.ensureCatalog(source, request.getCatalogName(), account); - return tableNames.stream().map(tableName -> { - DatacenterTableDetailMeta detail = connector.getTableDetail(source, request.getCatalogName(), tableName); - return registryService.registerTable(source, catalog, detail, account); - }).collect(Collectors.toList()); + List details = connector.getTableDetails( + source, catalogName, List.copyOf(requestedNames)); + DatacenterSource updated = transactionTemplate.execute(transactionStatus -> + persistScopeAddition(source, catalogId, details, account)); + if (updated == null) { + throw new BusinessException("数据范围更新事务未完成"); + } + changeNotifier.publishAfterCommit(updated); + return registryService.listManagedTables(source.getId(), catalogId); } + /** + * 外部数据源写入墓碑；Excel 数据源保留旧的物理清理契约。 + */ @Override - @Transactional(rollbackFor = Exception.class) public void removeSource(BigInteger sourceId, LoginAccount account) { + DatacenterSource source = requireSource(sourceId, account); + if (Integer.valueOf(1).equals(source.getBuiltinFlag())) { + throw new BusinessException("内置数据连接不支持删除"); + } + if (!isFederated(source.getSourceType())) { + transactionTemplate.executeWithoutResult(status -> removeManagedSource(source)); + return; + } + DatacenterSource tombstone = transactionTemplate.execute(status -> persistTombstone(source, account)); + if (tombstone == null) { + throw new BusinessException("连接删除事务未完成"); + } + try { + federationRuntime.remove(tombstone); + } catch (RuntimeException exception) { + // 数据库墓碑已经提交；本地清理失败交由 Runtime lease 排空和周期 reconcile 收口。 + LOGGER.warn("Failed to retire local runtime after source tombstone, sourceId={}", + tombstone.getId(), exception); + } finally { + changeNotifier.publishAfterCommit(tombstone); + } + } + + /** + * 以 CAS 停用外部数据源，并向本地及其他节点发布墓碑。 + */ + @Override + public DatacenterSourceView disable(BigInteger sourceId, LoginAccount account) { + DatacenterSource source = requireSource(sourceId, account); + requireFederated(source); + DatacenterSourceStatus currentStatus = DatacenterSourceStatus.fromCode(source.getStatus()); + if (currentStatus == DatacenterSourceStatus.DISABLED) { + return toView(source); + } + if (currentStatus != DatacenterSourceStatus.READY + && currentStatus != DatacenterSourceStatus.DEGRADED) { + throw new BusinessException("当前连接状态不支持停用"); + } + DatacenterSource disabled = transactionTemplate.execute(status -> + persistStatusTransition( + source, DatacenterSourceStatus.DISABLED, true, account)); + if (disabled == null) { + throw new BusinessException("连接停用事务未完成"); + } + try { + federationRuntime.remove(disabled); + } catch (RuntimeException exception) { + // 权威状态已经停用；查询入口会拒绝新请求，节点本地清理由 reconcile 重试。 + LOGGER.warn("停用后回收本地 Runtime 失败，sourceId={}, revision={}", + disabled.getId(), disabled.getDefinitionRevision(), exception); + } finally { + changeNotifier.publishAfterCommit(disabled); + } + return toView(disabled); + } + + /** + * 探测候选配置并以 CAS 发布 READY revision，再从权威记录构建 Runtime。 + */ + @Override + public DatacenterSourceView enable(BigInteger sourceId, LoginAccount account) { + DatacenterSource disabled = requireSource(sourceId, account); + requireFederated(disabled); + if (DatacenterSourceStatus.fromCode(disabled.getStatus()) != DatacenterSourceStatus.DISABLED) { + throw new BusinessException("只有已停用连接可以重新启用"); + } + DatacenterSource candidate = candidateService.copy(disabled); + candidate.setDefinitionRevision(value(disabled.getDefinitionRevision()) + 1L); + candidate.setStatus(DatacenterSourceStatus.DRAFT.code()); + DatacenterConnectionTestResult probe = candidateService.probe(candidate); + if (!probe.isSuccess()) { + throw new BusinessException(probe.getMessage()); + } + try (PreparedSourceRuntime prepared = federationRuntime.prepareCandidate(candidate)) { + // 候选保持 DRAFT 完成预热；READY 必须与 prepared checksum 同步发布。 + candidate.setStatus(DatacenterSourceStatus.READY.code()); + candidate.setMetadataRefreshStatus("SUCCESS"); + candidate.setMetadataRefreshedAt(new Date()); + candidate.setDefinitionChecksum(prepared.definition().checksum()); + DatacenterSource enabled = transactionTemplate.execute(status -> + persistEnable(disabled, candidate, account)); + if (enabled == null) { + throw new BusinessException("连接启用事务未完成"); + } + commitPreparedRuntime( + enabled, prepared, account, "连接启用后的 Runtime 切换失败"); + changeNotifier.publishAfterCommit(enabled); + return toView(enabled); + } + } + + /** + * 只刷新已纳管对象；新对象不会进入 Registry。 + */ + @Override + public DatacenterSourceView refreshMetadata(BigInteger sourceId, LoginAccount account) { + DatacenterSource source = requireSource(sourceId, account); + requireFederated(source); + return toView(metadataRefreshService.refresh(source, account)); + } + + private DatacenterSource persistActivation( + DatacenterSource source, + String catalogName, + List tableDetails, + String expectedChecksum, + LoginAccount account) { + retireExistingScope(source, account); + DatacenterCatalog catalog = registryService.ensureCatalog(source, catalogName, account); + for (DatacenterTableDetailMeta detail : tableDetails) { + registryService.registerTable(source, catalog, detail, account); + } + long previousRevision = value(source.getDefinitionRevision()); + long previousScopeRevision = value(source.getScopeRevision()); + source.setDefinitionRevision(previousRevision + 1L); + source.setScopeRevision(Math.max(1L, previousScopeRevision) + 1L); + source.setStatus(DatacenterSourceStatus.READY.code()); + source.setCompatibilityStatus("CODE_SUPPORTED_UNVERIFIED"); + source.setMetadataRefreshStatus("SUCCESS"); + source.setMetadataRefreshedAt(new Date()); + String persistedChecksum = federationRuntime.definitions().create(source).checksum(); + requirePreparedChecksum(expectedChecksum, persistedChecksum); + source.setDefinitionChecksum(persistedChecksum); + source.setModified(new Date()); + source.setModifiedBy(accountId(account)); + QueryWrapper revisionGuard = QueryWrapper.create() + .eq(DatacenterSource::getId, source.getId()) + .eq(DatacenterSource::getTenantId, source.getTenantId()) + .eq(DatacenterSource::getDefinitionRevision, previousRevision) + .eq(DatacenterSource::getScopeRevision, previousScopeRevision); + if (getMapper().updateByQuery(source, revisionGuard) != 1) { + throw new BusinessException("连接配置已被其他节点更新，请刷新后重试"); + } + return getMapper().selectOneByQuery(QueryWrapper.create() + .eq(DatacenterSource::getId, source.getId()) + .eq(DatacenterSource::getTenantId, source.getTenantId())); + } + + private DatacenterSource persistReconfiguration( + DatacenterSource current, + DatacenterSource candidate, + String catalogName, + List tableDetails, + String expectedChecksum, + LoginAccount account) { + boolean scopeChanged = reconfigurationChangesScope( + current, candidate, catalogName, tableDetails); + retireExistingScope(current, account); + DatacenterCatalog catalog = registryService.ensureCatalog(candidate, catalogName, account); + for (DatacenterTableDetailMeta detail : tableDetails) { + registryService.rebindTable(candidate, catalog, detail, account); + } + candidate.setStatus(DatacenterSourceStatus.READY.code()); + candidate.setScopeRevision(scopeChanged + ? Math.max(1L, value(current.getScopeRevision())) + 1L + : current.getScopeRevision()); + candidate.setCompatibilityStatus("CODE_SUPPORTED_UNVERIFIED"); + candidate.setMetadataRefreshStatus("SUCCESS"); + candidate.setMetadataRefreshedAt(new Date()); + String persistedChecksum = federationRuntime.definitions().create(candidate).checksum(); + requirePreparedChecksum(expectedChecksum, persistedChecksum); + candidate.setDefinitionChecksum(persistedChecksum); + candidate.setCreated(current.getCreated()); + candidate.setCreatedBy(current.getCreatedBy()); + candidate.setTenantId(current.getTenantId()); + candidate.setDeptId(current.getDeptId()); + candidate.setModified(new Date()); + candidate.setModifiedBy(accountId(account)); + QueryWrapper guard = QueryWrapper.create() + .eq(DatacenterSource::getId, current.getId()) + .eq(DatacenterSource::getTenantId, current.getTenantId()) + .eq(DatacenterSource::getDefinitionRevision, current.getDefinitionRevision()) + .eq(DatacenterSource::getScopeRevision, current.getScopeRevision()) + .in(DatacenterSource::getStatus, List.of( + DatacenterSourceStatus.READY.code(), + DatacenterSourceStatus.DEGRADED.code())); + if (getMapper().updateByQuery(candidate, guard) != 1) { + throw new BusinessException("连接配置已被其他节点更新，请刷新后重试"); + } + return getMapper().selectOneByQuery(QueryWrapper.create() + .eq(DatacenterSource::getId, current.getId()) + .eq(DatacenterSource::getTenantId, current.getTenantId())); + } + + private DatacenterSource persistScopeAddition( + DatacenterSource source, + BigInteger catalogId, + List details, + LoginAccount account) { + DatacenterCatalog catalog = catalogMapper.selectOneByQuery(QueryWrapper.create() + .eq(DatacenterCatalog::getId, catalogId) + .eq(DatacenterCatalog::getSourceId, source.getId()) + .eq(DatacenterCatalog::getMetadataStatus, DatacenterMetadataStatus.ACTIVE.name())); + if (catalog == null) { + throw new BusinessException("命名空间已变化，请刷新后重试"); + } + for (DatacenterTableDetailMeta detail : details) { + registryService.registerTable(source, catalog, detail, account); + } + DatacenterSource patch = new DatacenterSource(); + patch.setId(source.getId()); + patch.setScopeRevision(Math.max(1L, value(source.getScopeRevision())) + 1L); + patch.setModified(new Date()); + patch.setModifiedBy(accountId(account)); + QueryWrapper guard = QueryWrapper.create() + .eq(DatacenterSource::getId, source.getId()) + .eq(DatacenterSource::getTenantId, source.getTenantId()) + .eq(DatacenterSource::getDefinitionRevision, source.getDefinitionRevision()) + .eq(DatacenterSource::getScopeRevision, source.getScopeRevision()) + .in(DatacenterSource::getStatus, List.of( + DatacenterSourceStatus.READY.code(), + DatacenterSourceStatus.DEGRADED.code())); + if (getMapper().updateByQuery(patch, guard) != 1) { + throw new BusinessException("数据范围已被其他请求更新，请刷新后重试"); + } + return getMapper().selectOneByQuery(QueryWrapper.create() + .eq(DatacenterSource::getId, source.getId()) + .eq(DatacenterSource::getTenantId, source.getTenantId())); + } + + private DatacenterSource persistStatusTransition( + DatacenterSource source, + DatacenterSourceStatus targetStatus, + boolean clearChecksum, + LoginAccount account) { + DatacenterSource patch = new DatacenterSource(); + patch.setId(source.getId()); + patch.setDefinitionRevision(value(source.getDefinitionRevision()) + 1L); + patch.setStatus(targetStatus.code()); + if (clearChecksum) { + patch.setDefinitionChecksum(null); + } + patch.setModified(new Date()); + patch.setModifiedBy(accountId(account)); + QueryWrapper guard = QueryWrapper.create() + .eq(DatacenterSource::getId, source.getId()) + .eq(DatacenterSource::getTenantId, source.getTenantId()) + .eq(DatacenterSource::getDefinitionRevision, source.getDefinitionRevision()) + .eq(DatacenterSource::getScopeRevision, source.getScopeRevision()) + .eq(DatacenterSource::getStatus, source.getStatus()); + if (getMapper().updateByQuery(patch, guard) != 1) { + throw new BusinessException("连接状态已变化，请刷新后重试"); + } + return getMapper().selectOneByQuery(QueryWrapper.create() + .eq(DatacenterSource::getId, source.getId()) + .eq(DatacenterSource::getTenantId, source.getTenantId())); + } + + private DatacenterSource persistEnable( + DatacenterSource disabled, + DatacenterSource candidate, + LoginAccount account) { + candidate.setCreated(disabled.getCreated()); + candidate.setCreatedBy(disabled.getCreatedBy()); + candidate.setTenantId(disabled.getTenantId()); + candidate.setDeptId(disabled.getDeptId()); + candidate.setModified(new Date()); + candidate.setModifiedBy(accountId(account)); + QueryWrapper guard = QueryWrapper.create() + .eq(DatacenterSource::getId, disabled.getId()) + .eq(DatacenterSource::getTenantId, disabled.getTenantId()) + .eq(DatacenterSource::getDefinitionRevision, disabled.getDefinitionRevision()) + .eq(DatacenterSource::getScopeRevision, disabled.getScopeRevision()) + .eq(DatacenterSource::getStatus, DatacenterSourceStatus.DISABLED.code()); + if (getMapper().updateByQuery(candidate, guard) != 1) { + throw new BusinessException("连接状态已变化，请刷新后重试"); + } + return getMapper().selectOneByQuery(QueryWrapper.create() + .eq(DatacenterSource::getId, disabled.getId()) + .eq(DatacenterSource::getTenantId, disabled.getTenantId())); + } + + private DatacenterSource persistTombstone(DatacenterSource source, LoginAccount account) { + long previousRevision = value(source.getDefinitionRevision()); + long previousScopeRevision = value(source.getScopeRevision()); + source.setDefinitionRevision(previousRevision + 1L); + source.setScopeRevision(Math.max(1L, previousScopeRevision) + 1L); + source.setStatus(DatacenterSourceStatus.DELETED.code()); + source.setModified(new Date()); + source.setModifiedBy(accountId(account)); + QueryWrapper guard = QueryWrapper.create() + .eq(DatacenterSource::getId, source.getId()) + .eq(DatacenterSource::getTenantId, source.getTenantId()) + .eq(DatacenterSource::getDefinitionRevision, previousRevision) + .eq(DatacenterSource::getScopeRevision, previousScopeRevision); + if (getMapper().updateByQuery(source, guard) != 1) { + throw new BusinessException("连接配置已变化，请刷新后重试"); + } + return getMapper().selectOneByQuery(QueryWrapper.create() + .eq(DatacenterSource::getId, source.getId()) + .eq(DatacenterSource::getTenantId, source.getTenantId())); + } + + private void saveOrUpdate(DatacenterSource source, DatacenterSource existing, LoginAccount account) { + Date now = new Date(); + if (existing == null) { + source.setCreated(now); + source.setCreatedBy(accountId(account)); + source.setTenantId(tenantId(account)); + source.setDeptId(deptId(account)); + source.setModified(now); + source.setModifiedBy(accountId(account)); + save(source); + return; + } + source.setCreated(existing.getCreated()); + source.setCreatedBy(existing.getCreatedBy()); + source.setTenantId(existing.getTenantId()); + source.setDeptId(existing.getDeptId()); + source.setModified(now); + source.setModifiedBy(accountId(account)); + QueryWrapper guard = QueryWrapper.create() + .eq(DatacenterSource::getId, existing.getId()) + .eq(DatacenterSource::getTenantId, existing.getTenantId()) + .eq(DatacenterSource::getDefinitionRevision, existing.getDefinitionRevision()); + if (getMapper().updateByQuery(source, guard) != 1) { + throw new BusinessException("连接配置已被其他请求更新，请刷新后重试"); + } + } + + private void ensureGeneratedSourceCode(DatacenterSource source) { + if (source.getSourceCode() != null && !source.getSourceCode().isBlank()) { + return; + } + source.setSourceCode("DC_" + source.getId()); + updateById(source); + } + + private DatacenterSource buildActiveCandidate( + DatacenterSourceDraftRequest request, + LoginAccount account) { + if (request == null || request.id() == null) { + throw new BusinessException("活动连接 ID 不能为空"); + } + DatacenterSource current = requireSource(request.id(), account); + return candidateService.buildReconfigurationCandidate(current, request); + } + + private void requireActiveReconfiguration( + DatacenterSource current, + DatacenterSourceReconfigureRequest request) { + DatacenterSourceStatus status = DatacenterSourceStatus.fromCode(current.getStatus()); + if (status != DatacenterSourceStatus.READY + && status != DatacenterSourceStatus.DEGRADED) { + throw new BusinessException("只有已激活连接可以原地重配置"); + } + if (!java.util.Objects.equals( + current.getDefinitionRevision(), request.expectedDefinitionRevision()) + || !java.util.Objects.equals( + current.getScopeRevision(), request.expectedScopeRevision())) { + throw new BusinessException("连接配置已变化，请刷新后重试"); + } + } + + private String requireCatalogName(String catalogName) { + String normalized = trimToNull(catalogName); + if (normalized == null) { + throw new BusinessException("请选择数据范围"); + } + return normalized; + } + + private Set selectedTableNames(List tableNames) { + Set selected = (tableNames == null ? List.of() : tableNames).stream() + .map(this::trimToNull) + .filter(java.util.Objects::nonNull) + .collect(Collectors.toCollection(LinkedHashSet::new)); + if (selected.isEmpty()) { + throw new BusinessException("请选择数据范围"); + } + if (selected.size() > MAX_TABLES_PER_ACTIVATION) { + throw new BusinessException("单次最多接入 " + MAX_TABLES_PER_ACTIVATION + " 张表"); + } + return selected; + } + + private String physicalTableName(DatacenterTable table) { + String actualTable = trimToNull(table.getActualTable()); + return actualTable == null ? table.getTableName() : actualTable; + } + + private void requireFederated(DatacenterSource source) { + if (!isFederated(source.getSourceType())) { + throw new BusinessException("当前连接不支持 Federation 生命周期操作"); + } + } + + private void retireExistingScope(DatacenterSource source, LoginAccount account) { + Date now = new Date(); + for (DatacenterCatalog catalog : catalogMapper.selectListByQuery( + QueryWrapper.create().eq(DatacenterCatalog::getSourceId, source.getId()))) { + catalog.setMetadataStatus(DatacenterMetadataStatus.RETIRED.name()); + catalog.setModified(now); + catalog.setModifiedBy(accountId(account)); + catalogMapper.update(catalog); + } + for (DatacenterTable table : tableMapper.selectListByQuery( + QueryWrapper.create().eq(DatacenterTable::getSourceId, source.getId()))) { + table.setMetadataStatus(DatacenterMetadataStatus.RETIRED.name()); + table.setModified(now); + table.setModifiedBy(accountId(account)); + tableMapper.update(table); + } + } + + private void markDegraded(DatacenterSource source, LoginAccount account) { + DatacenterSource degraded = new DatacenterSource(); + degraded.setId(source.getId()); + degraded.setStatus(DatacenterSourceStatus.DEGRADED.code()); + degraded.setLastTestStatus("FAILED"); + degraded.setLastTestMessage("连接池预热失败"); + degraded.setModified(new Date()); + degraded.setModifiedBy(accountId(account)); + getMapper().updateByQuery(degraded, + QueryWrapper.create() + .eq(DatacenterSource::getId, source.getId()) + .eq(DatacenterSource::getTenantId, source.getTenantId()) + .eq(DatacenterSource::getDefinitionRevision, source.getDefinitionRevision())); + } + + /** + * 发布已通过业务 CAS 的预构建 Runtime；极端冲突时将当前 revision 标记为降级。 + * + * @param published 已发布的权威数据源记录 + * @param prepared 预构建 Runtime + * @param account 操作账号 + * @param failureMessage 对外失败信息 + */ + private void commitPreparedRuntime( + DatacenterSource published, + PreparedSourceRuntime prepared, + LoginAccount account, + String failureMessage) { + try { + federationRuntime.commitPrepared(prepared); + } catch (RuntimeException exception) { + LOGGER.error("预构建 Runtime 原子切换失败，sourceId={}, revision={}", + published.getId(), published.getDefinitionRevision(), exception); + markDegraded(published, account); + changeNotifier.publishAfterCommit(requireSource(published.getId(), account)); + throw new BusinessException(failureMessage); + } + } + + /** + * 确保事务内持久化 Definition 与事务外预构建 Runtime 完全一致。 + * + * @param preparedChecksum 预构建 Definition 摘要 + * @param persistedChecksum 即将持久化的 Definition 摘要 + */ + private void requirePreparedChecksum( + String preparedChecksum, + String persistedChecksum) { + if (!java.util.Objects.equals(preparedChecksum, persistedChecksum)) { + throw new BusinessException("连接元数据在发布前发生变化，请重试"); + } + } + + /** + * 判断重配置是否改变 Policy 可见的物理命名空间、表集合或表结构。 + * + * @param current 当前已发布数据源 + * @param candidate 新连接候选 + * @param catalogName 新命名空间 + * @param details 新表元数据 + * @return 是否需要推进 scopeRevision + */ + private boolean reconfigurationChangesScope( + DatacenterSource current, + DatacenterSource candidate, + String catalogName, + List details) { + List activeCatalogs = catalogMapper.selectListByQuery( + QueryWrapper.create() + .eq(DatacenterCatalog::getSourceId, current.getId()) + .eq(DatacenterCatalog::getMetadataStatus, + DatacenterMetadataStatus.ACTIVE.name())); + if (activeCatalogs.size() != 1) { + return true; + } + DatacenterCatalog currentCatalog = activeCatalogs.get(0); + boolean mysql = DatacenterSourceType.MYSQL.name().equals(candidate.getSourceType()); + String physicalCatalog = mysql ? catalogName : candidate.getDatabaseName(); + String physicalSchema = mysql ? null : catalogName; + if (!java.util.Objects.equals(currentCatalog.getCatalogName(), catalogName) + || !java.util.Objects.equals( + currentCatalog.getPhysicalCatalogName(), physicalCatalog) + || !java.util.Objects.equals( + currentCatalog.getPhysicalSchemaName(), physicalSchema)) { + return true; + } + List activeTables = tableMapper.selectListByQuery( + QueryWrapper.create() + .eq(DatacenterTable::getSourceId, current.getId()) + .eq(DatacenterTable::getCatalogId, currentCatalog.getId()) + .eq(DatacenterTable::getMetadataStatus, + DatacenterMetadataStatus.ACTIVE.name())); + if (activeTables.size() != details.size()) { + return true; + } + Map existingByName = activeTables.stream() + .collect(Collectors.toMap( + DatacenterTable::getTableName, + table -> table, + (first, ignored) -> first, + LinkedHashMap::new)); + String namespaceKey = DatacenterMetadataIdentity.namespaceKey( + physicalCatalog, physicalSchema); + for (DatacenterTableDetailMeta detail : details) { + DatacenterTable discovered = detail.getTable(); + if (discovered == null) { + return true; + } + DatacenterTable existing = existingByName.get(discovered.getTableName()); + if (existing == null) { + return true; + } + String identity = DatacenterMetadataIdentity.tableIdentity( + candidate.getId().toString(), + namespaceKey, + physicalTableName(discovered), + discovered.getTableKind()); + String fingerprint = DatacenterMetadataIdentity.tableFingerprint( + discovered.getTableName(), discovered.getTableKind(), detail.getFields()); + if (!java.util.Objects.equals(existing.getPhysicalIdentityKey(), identity) + || !java.util.Objects.equals(existing.getMetadataFingerprint(), fingerprint)) { + return true; + } + } + return false; + } + + private void removeManagedSource(DatacenterSource source) { + List tableIds = registryService.listManagedTables(source.getId(), null).stream() + .map(DatacenterTable::getId).filter(java.util.Objects::nonNull).toList(); + registryService.removeTables(tableIds); + catalogMapper.deleteByQuery(QueryWrapper.create() + .eq(DatacenterCatalog::getSourceId, source.getId())); + removeById(source.getId()); + } + + private DatacenterSource requireSource(BigInteger sourceId, LoginAccount account) { if (sourceId == null) { throw new BusinessException("数据连接不能为空"); } - DatacenterSource source = getById(sourceId); - if (source == null) { + DatacenterSource source = getMapper().selectOneByQuery(QueryWrapper.create() + .eq(DatacenterSource::getId, sourceId) + .eq(DatacenterSource::getTenantId, tenantId(account))); + if (source == null || DatacenterSourceStatus.fromCode(source.getStatus()) + == DatacenterSourceStatus.DELETED) { throw new BusinessException("连接不存在"); } - if (Boolean.TRUE.equals(source.getBuiltinFlag())) { - throw new BusinessException("内置数据连接不支持删除"); + return source; + } + + private void retireRuntimeAfterCommit(DatacenterSource source) { + Runnable retirement = () -> { + try { + federationRuntime.remove(source); + } catch (RuntimeException exception) { + LOGGER.warn("Failed to retire stale datacenter runtime for source {} revision {}", + source.getId(), source.getDefinitionRevision(), exception); + } + }; + if (TransactionSynchronizationManager.isSynchronizationActive()) { + TransactionSynchronizationManager.registerSynchronization(new TransactionSynchronization() { + @Override + public void afterCommit() { + retirement.run(); + } + }); + return; } + retirement.run(); + } - List tableIds = registryService.listManagedTables(sourceId, null).stream() - .map(DatacenterTable::getId) - .filter(id -> id != null) - .toList(); - registryService.removeTables(tableIds); - - QueryWrapper catalogWrapper = QueryWrapper.create(); - catalogWrapper.eq(DatacenterCatalog::getSourceId, sourceId); - catalogMapper.deleteByQuery(catalogWrapper); - - removeById(sourceId); + private boolean isFederated(String sourceType) { + return DatacenterSourceType.MYSQL.name().equals(sourceType) + || DatacenterSourceType.POSTGRESQL.name().equals(sourceType); } private boolean isManagedOnly(String sourceType) { @@ -222,72 +1065,29 @@ public class DatacenterSourceServiceImpl extends ServiceImpl configJson) { - if (configJson == null) { - return null; - } - Object password = configJson.get("password"); - if (password == null) { - return null; - } - String value = String.valueOf(password); - return value.isBlank() ? null : value; + private DatacenterSourceView toView(DatacenterSource source) { + return DatacenterSourceViews.from(source); } private DatacenterSource mergeWithExisting(DatacenterSource existing, DatacenterSource incoming) { - if (incoming == null) { - return existing; - } if (existing == null) { return incoming; } - DatacenterSource merged = new DatacenterSource(); - merged.setId(existing.getId()); - merged.setCreated(existing.getCreated()); - merged.setCreatedBy(existing.getCreatedBy()); - merged.setTenantId(existing.getTenantId()); - merged.setDeptId(existing.getDeptId()); - merged.setBuiltinFlag(existing.getBuiltinFlag()); - merged.setStatus(existing.getStatus()); - merged.setLastTestStatus(existing.getLastTestStatus()); - merged.setLastTestMessage(existing.getLastTestMessage()); - merged.setLastTestedAt(existing.getLastTestedAt()); - merged.setOptions(existing.getOptions()); + DatacenterSource merged = candidateService.copy(existing); merged.setSourceName(valueOrExisting(incoming.getSourceName(), existing.getSourceName())); merged.setSourceCode(valueOrExisting(incoming.getSourceCode(), existing.getSourceCode())); merged.setSourceType(valueOrExisting(incoming.getSourceType(), existing.getSourceType())); @@ -299,13 +1099,32 @@ public class DatacenterSourceServiceImpl extends ServiceImpl entry : configJson.entrySet()) { + String key = entry.getKey(); + if (key == null || isSensitiveKey(key)) { + continue; + } + sanitized.put(key, entry.getValue()); + } return sanitized; } + /** + * 判断扩展字段名是否可能承载凭据。 + * + * @param key 配置键 + * @return 是否属于敏感键 + */ + public boolean isSensitiveKey(String key) { + String normalized = StrUtil.trimToEmpty(key).toLowerCase(java.util.Locale.ROOT); + return normalized.contains("password") + || normalized.contains("passwd") + || normalized.contains("secret") + || normalized.contains("token") + || normalized.contains("credential") + || normalized.endsWith("key"); + } + public Integer defaultPort(String sourceType) { DatacenterSourceType resolved = requireSourceType(sourceType); return switch (resolved) { @@ -69,11 +91,12 @@ public class DatacenterSourceConnectionDefaults { public String defaultDriverClassName(String sourceType) { DatacenterSourceType resolved = requireSourceType(sourceType); return switch (resolved) { - case MYSQL -> "com.mysql.cj.jdbc.Driver"; - case POSTGRESQL, GAUSSDB_NATIVE -> "org.postgresql.Driver"; - case ORACLE -> "oracle.jdbc.OracleDriver"; - case GBASE_8A -> "com.gbase.jdbc.Driver"; - case GBASE_8S -> "com.gbasedbt.jdbc.Driver"; + case MYSQL -> KnownJdbcDriver.MYSQL.driverClassName(); + case POSTGRESQL -> KnownJdbcDriver.POSTGRESQL.driverClassName(); + case GAUSSDB_NATIVE -> KnownJdbcDriver.GAUSSDB.driverClassName(); + case ORACLE -> KnownJdbcDriver.ORACLE.driverClassName(); + case GBASE_8A -> KnownJdbcDriver.GBASE_8A.driverClassName(); + case GBASE_8S -> KnownJdbcDriver.GBASE_8S.driverClassName(); default -> ""; }; } @@ -91,10 +114,12 @@ public class DatacenterSourceConnectionDefaults { DatacenterSourceType resolved = requireSourceType(source.getSourceType()); return switch (resolved) { case MYSQL -> String.format( - "jdbc:mysql://%s:%d/%s?useUnicode=true&characterEncoding=utf-8&serverTimezone=Asia/Shanghai&useSSL=false", + "jdbc:mysql://%s:%d/%s?useUnicode=true&characterEncoding=utf-8&serverTimezone=Asia/Shanghai&useSSL=false&useCursorFetch=true&useServerPrepStmts=true&connectTimeout=5000&socketTimeout=35000", host, port, databaseName ); - case POSTGRESQL -> String.format("jdbc:postgresql://%s:%d/%s", host, port, databaseName); + case POSTGRESQL -> String.format( + "jdbc:postgresql://%s:%d/%s?connectTimeout=5&socketTimeout=35&cancelSignalTimeout=5", + host, port, databaseName); case ORACLE -> String.format("jdbc:oracle:thin:@//%s:%d/%s", host, port, databaseName); case GAUSSDB_NATIVE -> String.format("jdbc:postgresql://%s:%d/%s", host, port, databaseName); case GBASE_8A -> String.format("jdbc:gbase://%s:%d/%s", host, port, databaseName); diff --git a/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/security/DatacenterCredentialCipher.java b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/security/DatacenterCredentialCipher.java index 53db8f44..ce950bd2 100644 --- a/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/security/DatacenterCredentialCipher.java +++ b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/security/DatacenterCredentialCipher.java @@ -1,28 +1,109 @@ package tech.easyflow.datacenter.security; -import cn.hutool.crypto.SecureUtil; -import cn.hutool.crypto.symmetric.AES; +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.security.GeneralSecurityException; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.security.SecureRandom; +import java.util.Base64; +import javax.crypto.Cipher; +import javax.crypto.spec.GCMParameterSpec; +import javax.crypto.spec.SecretKeySpec; +import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Component; -import java.nio.charset.StandardCharsets; - +/** + * 使用部署侧主密钥对数据源密码执行带认证加密。 + */ @Component public class DatacenterCredentialCipher { - private static final String DEFAULT_KEY = "easyflow-datacenter-phase1-key"; - private final AES aes = SecureUtil.aes(SecureUtil.sha256(DEFAULT_KEY).substring(0, 16).getBytes(StandardCharsets.UTF_8)); + private static final String VERSION = "v1:"; + private static final String KEY_DERIVATION_DOMAIN = "easyflow-datacenter-credential-v1\u0000"; + private static final int IV_BYTES = 12; + private static final int GCM_TAG_BITS = 128; + private final SecretKeySpec key; + private final SecureRandom secureRandom = new SecureRandom(); + + /** + * 创建凭据加密器。 + * + * @param masterKey 部署环境提供的独立主密钥，至少 32 个字符 + * @throws IllegalStateException 主密钥缺失或长度不足时抛出 + */ + public DatacenterCredentialCipher( + @Value("${easyflow.datacenter.credential-key:}") String masterKey) { + if (masterKey == null || masterKey.length() < 32) { + throw new IllegalStateException( + "easyflow.datacenter.credential-key must contain at least 32 characters"); + } + // 使用固定域标签派生独立子密钥，避免与部署主密钥的其他用途共享同一 AES key。 + this.key = new SecretKeySpec(sha256(KEY_DERIVATION_DOMAIN + masterKey), "AES"); + } + + /** + * 加密明文密码。 + * + * @param plainText 明文密码 + * @return 带版本和随机 IV 的密文；空输入返回 null + */ public String encrypt(String plainText) { if (plainText == null || plainText.isBlank()) { return null; } - return aes.encryptHex(plainText); + byte[] iv = new byte[IV_BYTES]; + secureRandom.nextBytes(iv); + try { + Cipher cipher = Cipher.getInstance("AES/GCM/NoPadding"); + cipher.init(Cipher.ENCRYPT_MODE, key, new GCMParameterSpec(GCM_TAG_BITS, iv)); + byte[] encrypted = cipher.doFinal(plainText.getBytes(StandardCharsets.UTF_8)); + return VERSION + Base64.getEncoder().encodeToString( + ByteBuffer.allocate(iv.length + encrypted.length) + .put(iv) + .put(encrypted) + .array()); + } catch (GeneralSecurityException exception) { + throw new IllegalStateException("failed to encrypt datacenter credential", exception); + } } + /** + * 解密持久化密码。 + * + * @param cipherText 版本化密文 + * @return 明文密码；空输入返回 null + * @throws IllegalStateException 密文格式或认证校验失败时抛出 + */ public String decrypt(String cipherText) { if (cipherText == null || cipherText.isBlank()) { return null; } - return aes.decryptStr(cipherText); + if (!cipherText.startsWith(VERSION)) { + throw new IllegalStateException("unsupported datacenter credential format"); + } + try { + byte[] payload = Base64.getDecoder().decode(cipherText.substring(VERSION.length())); + if (payload.length <= IV_BYTES) { + throw new IllegalStateException("invalid datacenter credential payload"); + } + byte[] iv = java.util.Arrays.copyOfRange(payload, 0, IV_BYTES); + byte[] encrypted = java.util.Arrays.copyOfRange(payload, IV_BYTES, payload.length); + Cipher cipher = Cipher.getInstance("AES/GCM/NoPadding"); + cipher.init(Cipher.DECRYPT_MODE, key, new GCMParameterSpec(GCM_TAG_BITS, iv)); + return new String(cipher.doFinal(encrypted), StandardCharsets.UTF_8); + } catch (GeneralSecurityException | IllegalArgumentException exception) { + throw new IllegalStateException("failed to decrypt datacenter credential", exception); + } + } + + private byte[] sha256(String value) { + try { + return MessageDigest.getInstance("SHA-256") + .digest(value.getBytes(StandardCharsets.UTF_8)); + } catch (NoSuchAlgorithmException exception) { + throw new IllegalStateException("SHA-256 is not available", exception); + } } } diff --git a/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/utils/SqlSupportUtils.java b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/utils/SqlSupportUtils.java index e6965f27..c75e9ce3 100644 --- a/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/utils/SqlSupportUtils.java +++ b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/utils/SqlSupportUtils.java @@ -1,17 +1,42 @@ package tech.easyflow.datacenter.utils; import net.sf.jsqlparser.JSQLParserException; +import net.sf.jsqlparser.expression.AnalyticExpression; +import net.sf.jsqlparser.expression.Expression; +import net.sf.jsqlparser.expression.ExpressionVisitorAdapter; +import net.sf.jsqlparser.expression.Function; +import net.sf.jsqlparser.expression.KeepExpression; +import net.sf.jsqlparser.expression.MySQLGroupConcat; +import net.sf.jsqlparser.expression.WindowDefinition; +import net.sf.jsqlparser.expression.WindowElement; +import net.sf.jsqlparser.expression.WindowOffset; +import net.sf.jsqlparser.expression.XMLSerializeExpr; +import net.sf.jsqlparser.expression.operators.relational.FullTextSearch; +import net.sf.jsqlparser.expression.operators.relational.IsBooleanExpression; +import net.sf.jsqlparser.expression.operators.relational.IsNullExpression; import net.sf.jsqlparser.parser.CCJSqlParserUtil; +import net.sf.jsqlparser.schema.Column; import net.sf.jsqlparser.schema.Table; import net.sf.jsqlparser.statement.Statement; import net.sf.jsqlparser.statement.Statements; +import net.sf.jsqlparser.statement.select.AllColumns; +import net.sf.jsqlparser.statement.select.AllTableColumns; +import net.sf.jsqlparser.statement.select.FromItem; +import net.sf.jsqlparser.statement.select.ParenthesedFromItem; +import net.sf.jsqlparser.statement.select.ParenthesedSelect; +import net.sf.jsqlparser.statement.select.PlainSelect; import net.sf.jsqlparser.statement.select.Select; +import net.sf.jsqlparser.statement.select.SelectItem; import net.sf.jsqlparser.statement.select.WithItem; import net.sf.jsqlparser.util.TablesNamesFinder; import tech.easyflow.common.web.exceptions.BusinessException; +import java.util.ArrayDeque; import java.util.ArrayList; import java.util.Collection; +import java.util.Collections; +import java.util.Deque; +import java.util.IdentityHashMap; import java.util.LinkedHashMap; import java.util.LinkedHashSet; import java.util.List; @@ -21,11 +46,15 @@ import java.util.Set; public final class SqlSupportUtils { + private static final Set BLOCKED_READ_FUNCTIONS = Set.of( + "BENCHMARK", "GET_LOCK", "LOAD_FILE", "MASTER_POS_WAIT", + "RELEASE_LOCK", "SLEEP", "WAIT_FOR_EXECUTED_GTID_SET"); + private SqlSupportUtils() { } public static ResolvedSql resolve(String sql, Collection managedTables) { - return resolve(sql, managedTables, true, false); + return resolve(sql, managedTables, true, false, false); } /** @@ -38,7 +67,7 @@ public final class SqlSupportUtils { public static ResolvedSql resolveInternalMysql( String sql, Collection managedTables) { - return resolve(sql, managedTables, false, true); + return resolve(sql, managedTables, false, true, true); } /** @@ -54,7 +83,8 @@ public final class SqlSupportUtils { String sql, Collection managedTables, boolean retainCatalog, - boolean mysqlIdentifierQuotes) { + boolean mysqlIdentifierQuotes, + boolean enforceColumnPolicy) { String normalizedSql = normalizeSql(sql); Statement statement = parseSingleStatement(normalizedSql); if (!(statement instanceof Select select)) { @@ -66,13 +96,15 @@ public final class SqlSupportUtils { if (managedTable == null || !hasText(managedTable.getTableName())) { continue; } - String tableKey = normalizeIdentifier(managedTable.getTableName()); - byTableName.computeIfAbsent(tableKey, key -> new ArrayList<>()).add(managedTable); + for (String tableKey : managedIdentifierKeys(managedTable.getTableName())) { + byTableName.computeIfAbsent(tableKey, key -> new ArrayList<>()).add(managedTable); + } if (hasText(managedTable.getCatalogName())) { - byCatalogAndTable.put( - catalogTableKey(managedTable.getCatalogName(), managedTable.getTableName()), - managedTable - ); + for (String catalogKey : managedIdentifierKeys(managedTable.getCatalogName())) { + for (String tableKey : managedIdentifierKeys(managedTable.getTableName())) { + byCatalogAndTable.put(catalogKey + "." + tableKey, managedTable); + } + } } } @@ -82,11 +114,18 @@ public final class SqlSupportUtils { throw new BusinessException("SQL 必须引用至少一张已接入表"); } Set logicalTables = new LinkedHashSet<>(); + Map managedTablesByNode = new IdentityHashMap<>(); for (Table table : referencedTables) { ManagedTable managedTable = resolveManagedTable(table, byTableName, byCatalogAndTable); + managedTablesByNode.put(table, managedTable); rewriteTable(table, managedTable, retainCatalog); logicalTables.add(renderLogicalTable(managedTable)); } + if (enforceColumnPolicy) { + validateReadPolicy( + collector, + managedTablesByNode); + } String executableSql = select.toString(); if (mysqlIdentifierQuotes) { executableSql = normalizeMysqlIdentifierQuotes(executableSql); @@ -94,6 +133,60 @@ public final class SqlSupportUtils { return new ResolvedSql(executableSql, new ArrayList<>(logicalTables)); } + private static void validateReadPolicy( + SqlTableCollector collector, + Map managedTablesByNode) { + if (collector.unsafeSelect || !Collections.disjoint( + collector.functions, BLOCKED_READ_FUNCTIONS)) { + throw new BusinessException("SQL 包含不允许的只读操作"); + } + if (collector.unresolvedProjectionStar + || collector.projectionStarPhysicalTables.stream() + .map(managedTablesByNode::get) + .anyMatch(table -> table == null || !table.allColumnsQueryable())) { + throw new BusinessException("查询范围包含不可查询或敏感字段，请显式选择允许字段"); + } + for (Column column : collector.columns) { + String columnName = trimToNull(column.getColumnName()); + if (!hasText(columnName)) { + continue; + } + if (collector.derivedColumnReferences.contains(column)) { + continue; + } + if (collector.outputAliasReferences.contains(column) + && collector.outputAliasPhysicalTables + .getOrDefault(column, Set.of()) + .stream() + .map(managedTablesByNode::get) + .noneMatch(table -> table != null + && table.columnKnown(columnName))) { + continue; + } + boolean qualified = column.getTable() != null + && hasText(column.getTable().getName()); + Table physicalTable = collector.physicalTablesByColumn.get(column); + ManagedTable qualifiedTable = physicalTable == null + ? null : managedTablesByNode.get(physicalTable); + boolean allowed = qualified + ? collector.scopedQualifiedColumns.contains(column) + && qualifiedTable != null + && qualifiedTable.columnQueryable(columnName) + : collector.unqualifiedPhysicalTables + .getOrDefault(column, Set.of()) + .stream() + .map(managedTablesByNode::get) + .allMatch(table -> table != null + && table.columnQueryable(columnName)) + && !collector.unqualifiedPhysicalTables + .getOrDefault(column, Set.of()) + .isEmpty(); + if (!allowed) { + throw new BusinessException("SQL 引用了不可查询或敏感字段: " + columnName); + } + } + } + private static Statement parseSingleStatement(String sql) { try { Statements statements = CCJSqlParserUtil.parseStatements(sql); @@ -120,13 +213,14 @@ public final class SqlSupportUtils { } String catalogName = trimToNull(table.getSchemaName()); if (hasText(catalogName)) { - ManagedTable managedTable = byCatalogAndTable.get(catalogTableKey(catalogName, tableName)); + ManagedTable managedTable = byCatalogAndTable.get( + identifierLookupKey(catalogName) + "." + identifierLookupKey(tableName)); if (managedTable == null) { throw new BusinessException("SQL 引用了未接入表: " + catalogName + "." + tableName); } return managedTable; } - List matches = byTableName.get(normalizeIdentifier(tableName)); + List matches = byTableName.get(identifierLookupKey(tableName)); if (matches == null || matches.isEmpty()) { throw new BusinessException("SQL 引用了未接入表: " + tableName); } @@ -220,21 +314,33 @@ public final class SqlSupportUtils { return normalized; } - private static String catalogTableKey(String catalogName, String tableName) { - return normalizeIdentifier(catalogName) + "." + normalizeIdentifier(tableName); - } - - private static String normalizeIdentifier(String value) { + private static String identifierLookupKey(String value) { String normalized = trimToNull(value); if (!hasText(normalized)) { return ""; } - if ((normalized.startsWith("`") && normalized.endsWith("`")) + boolean quoted = (normalized.startsWith("`") && normalized.endsWith("`")) || (normalized.startsWith("\"") && normalized.endsWith("\"")) - || (normalized.startsWith("[") && normalized.endsWith("]"))) { + || (normalized.startsWith("[") && normalized.endsWith("]")); + if (quoted) { normalized = normalized.substring(1, normalized.length() - 1); } - return normalized.trim().toLowerCase(Locale.ROOT); + normalized = normalized.trim(); + return quoted ? "Q:" + normalized : "U:" + normalized.toLowerCase(Locale.ROOT); + } + + private static Set managedIdentifierKeys(String value) { + String normalized = trimToNull(value); + if (!hasText(normalized)) { + return Set.of(); + } + Set keys = new LinkedHashSet<>(); + keys.add("Q:" + normalized); + String folded = normalized.toLowerCase(Locale.ROOT); + if (normalized.equals(folded)) { + keys.add("U:" + folded); + } + return keys; } private static boolean hasText(String value) { @@ -253,11 +359,24 @@ public final class SqlSupportUtils { private final String catalogName; private final String tableName; private final String physicalTableName; + private final Set knownColumnKeys; + private final Set queryableColumnKeys; public ManagedTable(String catalogName, String tableName, String physicalTableName) { + this(catalogName, tableName, physicalTableName, Set.of(), Set.of()); + } + + public ManagedTable( + String catalogName, + String tableName, + String physicalTableName, + Collection knownColumns, + Collection queryableColumns) { this.catalogName = trimToNull(catalogName); this.tableName = trimToNull(tableName); this.physicalTableName = hasText(physicalTableName) ? physicalTableName.trim() : this.tableName; + this.knownColumnKeys = columnKeys(knownColumns); + this.queryableColumnKeys = columnKeys(queryableColumns); } public String getCatalogName() { @@ -271,6 +390,38 @@ public final class SqlSupportUtils { public String getPhysicalTableName() { return physicalTableName; } + + private boolean allColumnsQueryable() { + return !knownColumnKeys.isEmpty() + && knownColumnKeys.equals(queryableColumnKeys); + } + + private boolean columnQueryable(String columnName) { + return queryableColumnKeys.contains(identifierLookupKey(columnName)); + } + + /** + * 判断字段是否属于当前纳管表。 + * + * @param columnName 字段名称 + * @return 已纳管时返回 {@code true} + */ + private boolean columnKnown(String columnName) { + return knownColumnKeys.contains(identifierLookupKey(columnName)); + } + + private static Set columnKeys(Collection columns) { + if (columns == null || columns.isEmpty()) { + return Set.of(); + } + Set keys = new LinkedHashSet<>(); + for (String column : columns) { + if (hasText(column)) { + keys.addAll(managedIdentifierKeys(column)); + } + } + return Set.copyOf(keys); + } } public static class ResolvedSql { @@ -293,29 +444,761 @@ public final class SqlSupportUtils { private static class SqlTableCollector extends TablesNamesFinder { private final List tables = new ArrayList<>(); - private final Set withNames = new LinkedHashSet<>(); + private final Set
seenTables = Collections.newSetFromMap(new IdentityHashMap<>()); + private final List columns = new ArrayList<>(); + private final Set functions = new LinkedHashSet<>(); + private final Set outputAliasReferences = Collections.newSetFromMap( + new IdentityHashMap<>()); + private final Map> outputAliasPhysicalTables = + new IdentityHashMap<>(); + private final Set derivedColumnReferences = Collections.newSetFromMap( + new IdentityHashMap<>()); + private final Set scopedQualifiedColumns = Collections.newSetFromMap( + new IdentityHashMap<>()); + private final Map physicalTablesByColumn = new IdentityHashMap<>(); + private final Map> unqualifiedPhysicalTables = + new IdentityHashMap<>(); + private final Set
projectionStarPhysicalTables = Collections.newSetFromMap( + new IdentityHashMap<>()); + private final Deque queryScopes = new ArrayDeque<>(); + private boolean unresolvedProjectionStar; + private boolean unsafeSelect; + /** + * 收集 SQL 中的物理表、字段、函数和只读风险标记。 + * + * @param statement SQL 语句 + * @return 去重后的物理表节点 + */ public List
collect(Statement statement) { tables.clear(); - withNames.clear(); + seenTables.clear(); + columns.clear(); + functions.clear(); + outputAliasReferences.clear(); + outputAliasPhysicalTables.clear(); + derivedColumnReferences.clear(); + scopedQualifiedColumns.clear(); + physicalTablesByColumn.clear(); + unqualifiedPhysicalTables.clear(); + projectionStarPhysicalTables.clear(); + queryScopes.clear(); + unresolvedProjectionStar = false; + unsafeSelect = false; statement.accept(this); return new ArrayList<>(tables); } + /** + * 将顶层 Select 直接分派到具体查询节点，避免父实现提前在作用域外遍历 CTE。 + * + * @param select SELECT 语句 + */ + @Override + public void visit(Select select) { + select.accept((net.sf.jsqlparser.statement.select.SelectVisitor) this); + } + + @Override + public void visit(PlainSelect plainSelect) { + QueryScope parentScope = queryScopes.peek(); + Map> visibleCtes = parentScope == null + ? new LinkedHashMap<>() + : new LinkedHashMap<>(parentScope.visibleCtes()); + registerCteOutputs(plainSelect, visibleCtes); + + Set outputAliases = outputColumns(plainSelect); + QueryScope currentScope = createQueryScope(plainSelect, visibleCtes); + registerProjectionStars(plainSelect, currentScope); + markOutputAliasReferences( + plainSelect, + outputAliases, + currentScope.physicalTables()); + unsafeSelect |= (plainSelect.getIntoTables() != null + && !plainSelect.getIntoTables().isEmpty()) + || plainSelect.getIntoTempTable() != null + || plainSelect.getForUpdateTable() != null + || plainSelect.isNoWait() + || plainSelect.isSkipLocked() + || plainSelect.getWait() != null; + queryScopes.push(currentScope); + try { + super.visit(plainSelect); + visitAdditionalClauses(plainSelect); + } finally { + queryScopes.pop(); + } + } + @Override public void visit(WithItem withItem) { - if (withItem.getAlias() != null && hasText(withItem.getAlias().getName())) { - withNames.add(normalizeIdentifier(withItem.getAlias().getName())); - } withItem.getSelect().accept((net.sf.jsqlparser.statement.select.SelectVisitor) this); } @Override public void visit(Table table) { - String tableName = normalizeIdentifier(table.getName()); - if (!withNames.contains(tableName)) { + QueryScope currentScope = queryScopes.peek(); + String tableName = identifierLookupKey(table.getName()); + boolean cteReference = currentScope != null + && isUnqualifiedTable(table) + && currentScope.visibleCtes().containsKey(tableName); + if (!cteReference && seenTables.add(table)) { tables.add(table); } } + + @Override + public void visit(Column column) { + classifyColumn(column); + columns.add(column); + super.visit(column); + } + + @Override + public void visit(AllColumns allColumns) { + super.visit(allColumns); + } + + @Override + public void visit(AllTableColumns allTableColumns) { + super.visit(allTableColumns); + } + + @Override + public void visit(Function function) { + if (hasText(function.getName())) { + functions.add(function.getName().toUpperCase(Locale.ROOT)); + } + super.visit(function); + } + + /** + * 遍历父实现遗漏的 IS NULL 左侧表达式。 + * + * @param isNullExpression IS NULL 表达式 + */ + @Override + public void visit(IsNullExpression isNullExpression) { + if (isNullExpression.getLeftExpression() != null) { + isNullExpression.getLeftExpression().accept(this); + } + } + + /** + * 遍历父实现遗漏的 IS TRUE/IS FALSE 左侧表达式。 + * + * @param isBooleanExpression 布尔判断表达式 + */ + @Override + public void visit(IsBooleanExpression isBooleanExpression) { + if (isBooleanExpression.getLeftExpression() != null) { + isBooleanExpression.getLeftExpression().accept(this); + } + } + + /** + * 遍历 MySQL 全文检索涉及的字段和检索值。 + * + * @param fullTextSearch 全文检索表达式 + */ + @Override + public void visit(FullTextSearch fullTextSearch) { + if (fullTextSearch.getMatchColumns() != null) { + fullTextSearch.getMatchColumns().accept(this); + } + if (fullTextSearch.getAgainstValue() != null) { + fullTextSearch.getAgainstValue().accept(this); + } + } + + /** + * 遍历 GROUP_CONCAT 的参数和内部排序字段。 + * + * @param groupConcat GROUP_CONCAT 表达式 + */ + @Override + public void visit(MySQLGroupConcat groupConcat) { + if (groupConcat.getExpressionList() != null) { + groupConcat.getExpressionList().accept(this); + } + visitOrderByElements(groupConcat.getOrderByElements()); + } + + /** + * 遍历 KEEP 表达式中的排序字段。 + * + * @param keepExpression KEEP 表达式 + */ + @Override + public void visit(KeepExpression keepExpression) { + visitOrderByElements(keepExpression.getOrderByElements()); + } + + /** + * 遍历 XMLSERIALIZE 的值和排序字段。 + * + * @param expression XMLSERIALIZE 表达式 + */ + @Override + public void visit(XMLSerializeExpr expression) { + if (expression.getExpression() != null) { + expression.getExpression().accept(this); + } + visitOrderByElements(expression.getOrderByElements()); + } + + /** + * 完整且空值安全地遍历窗口函数可引用字段的全部位置。 + * + * @param analytic 窗口函数表达式 + */ + @Override + public void visit(AnalyticExpression analytic) { + visitExpression(analytic.getExpression()); + visitExpression(analytic.getOffset()); + visitExpression(analytic.getDefaultValue()); + if (analytic.getKeep() != null) { + analytic.getKeep().accept(this); + } + visitOrderByElements(analytic.getFuncOrderBy()); + visitWindowDefinition(analytic.getWindowDefinition()); + visitExpression(analytic.getFilterExpression()); + } + + /** + * 遍历窗口定义中的分区、排序和 frame 偏移表达式。 + * + * @param windowDefinition 窗口定义 + */ + private void visitWindowDefinition(WindowDefinition windowDefinition) { + if (windowDefinition == null) { + return; + } + if (windowDefinition.getPartitionExpressionList() != null) { + windowDefinition.getPartitionExpressionList().accept(this); + } + visitOrderByElements(windowDefinition.getOrderByElements()); + visitWindowElement(windowDefinition.getWindowElement()); + } + + /** + * 遍历窗口 frame 的单边或区间偏移表达式。 + * + * @param windowElement 窗口 frame + */ + private void visitWindowElement(WindowElement windowElement) { + if (windowElement == null) { + return; + } + visitWindowOffset(windowElement.getOffset()); + if (windowElement.getRange() != null) { + visitWindowOffset(windowElement.getRange().getStart()); + visitWindowOffset(windowElement.getRange().getEnd()); + } + } + + /** + * 遍历窗口偏移中的动态表达式。 + * + * @param windowOffset 窗口偏移 + */ + private void visitWindowOffset(WindowOffset windowOffset) { + if (windowOffset != null) { + visitExpression(windowOffset.getExpression()); + } + } + + /** + * 标记 ORDER BY/GROUP BY 中真正引用当前投影别名的字段节点。 + * + * @param plainSelect 查询块 + * @param outputAliases 当前投影输出名 + */ + private void markOutputAliasReferences( + PlainSelect plainSelect, + Set outputAliases, + Set
physicalTables) { + if (plainSelect.getOrderByElements() != null) { + plainSelect.getOrderByElements().forEach(orderBy -> + markOutputAliasReferences( + orderBy.getExpression(), outputAliases, physicalTables)); + } + if (plainSelect.getGroupBy() != null + && plainSelect.getGroupBy().getGroupByExpressionList() != null) { + plainSelect.getGroupBy().getGroupByExpressionList().forEach( + expression -> { + if (expression instanceof Expression typedExpression) { + markOutputAliasReferences( + typedExpression, outputAliases, physicalTables); + } + }); + } + if (plainSelect.getGroupBy() != null + && plainSelect.getGroupBy().getGroupingSets() != null) { + plainSelect.getGroupBy().getGroupingSets().forEach( + expressions -> expressions.forEach(expression -> { + if (expression instanceof Expression typedExpression) { + markOutputAliasReferences( + typedExpression, outputAliases, physicalTables); + } + })); + } + markOutputAliasReferences( + plainSelect.getHaving(), outputAliases, physicalTables); + markOutputAliasReferences( + plainSelect.getQualify(), outputAliases, physicalTables); + } + + /** + * 标记当前子句表达式中的投影别名引用，且不进入嵌套查询块。 + * + * @param expression 待遍历表达式 + * @param outputAliases 当前投影输出名 + */ + private void markOutputAliasReferences( + Expression expression, + Set outputAliases, + Set
physicalTables) { + if (expression != null && !outputAliases.isEmpty()) { + expression.accept(new OutputAliasReferenceMarker( + outputAliases, + physicalTables)); + } + } + + /** + * 遍历 TablesNamesFinder 未覆盖、仍可引用字段的查询子句。 + * + * @param plainSelect 查询块 + */ + private void visitAdditionalClauses(PlainSelect plainSelect) { + if (plainSelect.getGroupBy() != null) { + if (plainSelect.getGroupBy().getGroupByExpressionList() != null) { + plainSelect.getGroupBy().getGroupByExpressionList().accept(this); + } + if (plainSelect.getGroupBy().getGroupingSets() != null) { + plainSelect.getGroupBy().getGroupingSets().forEach( + expressions -> expressions.accept(this)); + } + } + visitOrderByElements(plainSelect.getOrderByElements()); + if (plainSelect.getQualify() != null) { + plainSelect.getQualify().accept(this); + } + if (plainSelect.getWindowDefinitions() != null) { + plainSelect.getWindowDefinitions().forEach(this::visitWindowDefinition); + } + } + + /** + * 遍历可空表达式。 + * + * @param expression 表达式 + */ + private void visitExpression(Expression expression) { + if (expression != null) { + expression.accept(this); + } + } + + /** + * 遍历排序元素中的字段表达式。 + * + * @param orderByElements 排序元素 + */ + private void visitOrderByElements( + List orderByElements) { + if (orderByElements == null) { + return; + } + orderByElements.forEach(orderBy -> { + if (orderBy.getExpression() != null) { + orderBy.getExpression().accept(this); + } + }); + } + + /** + * 注册当前查询块可见的 CTE 输出列。 + * + * @param select 当前查询块 + * @param visibleCtes 可见 CTE 映射 + */ + private void registerCteOutputs( + Select select, + Map> visibleCtes) { + if (select.getWithItemsList() == null) { + return; + } + for (WithItem withItem : select.getWithItemsList()) { + if (withItem.getAlias() == null + || !hasText(withItem.getAlias().getName())) { + continue; + } + Set outputs = explicitCteColumns(withItem); + if (outputs.isEmpty()) { + outputs = outputColumns(withItem.getSelect()); + } + if (!outputs.isEmpty()) { + visibleCtes.put( + identifierLookupKey(withItem.getAlias().getName()), + outputs); + } + } + } + + /** + * 创建当前查询块的派生关系作用域。 + * + * @param plainSelect 查询块 + * @param visibleCtes 当前可见 CTE + * @return 查询作用域 + */ + private QueryScope createQueryScope( + PlainSelect plainSelect, + Map> visibleCtes) { + List relations = new ArrayList<>(); + collectScopeRelations(plainSelect.getFromItem(), relations); + if (plainSelect.getJoins() != null) { + plainSelect.getJoins().stream() + .map(join -> join.getRightItem()) + .forEach(relation -> collectScopeRelations(relation, relations)); + } + + Map> derivedByQualifier = new LinkedHashMap<>(); + Map physicalByQualifier = new LinkedHashMap<>(); + Set relationQualifiers = new LinkedHashSet<>(); + DerivedRelation soleDerivedRelation = null; + for (FromItem relation : relations) { + String qualifier = relationQualifier(relation); + String qualifierKey = hasText(qualifier) + ? identifierLookupKey(qualifier) + : null; + if (hasText(qualifierKey)) { + relationQualifiers.add(qualifierKey); + } + DerivedRelation derivedRelation = resolveDerivedRelation( + relation, + visibleCtes); + if (derivedRelation != null) { + derivedByQualifier.put( + derivedRelation.qualifier(), + derivedRelation.outputColumns()); + soleDerivedRelation = derivedRelation; + } else if (relation instanceof Table table + && hasText(qualifierKey)) { + physicalByQualifier.put(qualifierKey, table); + } + } + Set soleDerivedOutputs = relations.size() == 1 + && soleDerivedRelation != null + ? soleDerivedRelation.outputColumns() + : Set.of(); + return new QueryScope( + Map.copyOf(visibleCtes), + Map.copyOf(derivedByQualifier), + Map.copyOf(physicalByQualifier), + Set.copyOf(physicalByQualifier.values()), + Set.copyOf(relationQualifiers), + soleDerivedOutputs); + } + + /** + * 递归展开括号包裹的 JOIN，使星号与未限定字段仍绑定到底层物理表。 + * + * @param relation 待展开的 FROM 关系 + * @param relations 当前查询块的扁平关系集合 + */ + private void collectScopeRelations( + FromItem relation, + List relations) { + if (relation == null) { + return; + } + if (relation instanceof ParenthesedFromItem parenthesedFromItem) { + collectScopeRelations(parenthesedFromItem.getFromItem(), relations); + if (parenthesedFromItem.getJoins() != null) { + parenthesedFromItem.getJoins().stream() + .map(join -> join.getRightItem()) + .forEach(item -> collectScopeRelations(item, relations)); + } + return; + } + relations.add(relation); + } + + /** + * 按查询块从内向外解析字段限定符，并固定到首个声明该限定符的关系。 + * + * @param column 字段节点 + */ + private void classifyColumn(Column column) { + QueryScope currentScope = queryScopes.peek(); + if (currentScope == null) { + return; + } + String qualifier = column.getTable() == null + ? null : trimToNull(column.getTable().getName()); + String columnKey = identifierLookupKey(column.getColumnName()); + if (!hasText(qualifier)) { + if (currentScope.soleDerivedOutputs().contains(columnKey)) { + derivedColumnReferences.add(column); + } else if (!currentScope.physicalTables().isEmpty()) { + unqualifiedPhysicalTables.put( + column, + currentScope.physicalTables()); + } + return; + } + + String qualifierKey = identifierLookupKey(qualifier); + for (QueryScope scope : queryScopes) { + if (!scope.relationQualifiers().contains(qualifierKey)) { + continue; + } + scopedQualifiedColumns.add(column); + Set derivedOutputs = scope.derivedByQualifier().get(qualifierKey); + if (derivedOutputs != null && derivedOutputs.contains(columnKey)) { + derivedColumnReferences.add(column); + } else { + Table physicalTable = scope.physicalByQualifier().get(qualifierKey); + if (physicalTable != null) { + physicalTablesByColumn.put(column, physicalTable); + } + } + return; + } + } + + /** + * 返回 FROM 关系在当前查询块声明的限定符。 + * + * @param fromItem FROM 关系 + * @return 别名或物理表名；无法确定时返回 {@code null} + */ + private String relationQualifier(FromItem fromItem) { + if (fromItem.getAlias() != null + && hasText(fromItem.getAlias().getName())) { + return fromItem.getAlias().getName(); + } + return fromItem instanceof Table table + ? table.getName() + : null; + } + + /** + * 将 FROM 项解析为 CTE 或派生表关系。 + * + * @param fromItem FROM 项 + * @param visibleCtes 当前可见 CTE + * @return 可确定输出列的派生关系；无法确定时返回 {@code null} + */ + private DerivedRelation resolveDerivedRelation( + FromItem fromItem, + Map> visibleCtes) { + if (fromItem instanceof ParenthesedSelect parenthesedSelect + && parenthesedSelect.getAlias() != null + && hasText(parenthesedSelect.getAlias().getName())) { + Set outputs = outputColumns(parenthesedSelect.getSelect()); + if (!outputs.isEmpty()) { + return new DerivedRelation( + identifierLookupKey(parenthesedSelect.getAlias().getName()), + outputs); + } + } + if (fromItem instanceof Table table) { + Set outputs = isUnqualifiedTable(table) + ? visibleCtes.get(identifierLookupKey(table.getName())) + : null; + if (outputs != null && !outputs.isEmpty()) { + String qualifier = table.getAlias() != null + && hasText(table.getAlias().getName()) + ? table.getAlias().getName() + : table.getName(); + return new DerivedRelation(identifierLookupKey(qualifier), outputs); + } + } + return null; + } + + /** + * 判断表引用是否没有数据库或 Schema 限定符，只有此类引用才可能指向 CTE。 + * + * @param table 表节点 + * @return 无限定符时返回 {@code true} + */ + private boolean isUnqualifiedTable(Table table) { + return table.getNameParts() != null + && table.getNameParts().size() == 1; + } + + /** + * 将投影星号绑定到当前查询块直接展开的物理关系。 + * + * @param plainSelect 查询块 + * @param currentScope 当前查询作用域 + */ + private void registerProjectionStars( + PlainSelect plainSelect, + QueryScope currentScope) { + if (plainSelect.getSelectItems() == null) { + return; + } + for (SelectItem selectItem : plainSelect.getSelectItems()) { + Expression expression = selectItem.getExpression(); + if (expression instanceof AllTableColumns allTableColumns) { + String qualifier = allTableColumns.getTable() == null + ? null : trimToNull(allTableColumns.getTable().getName()); + if (!hasText(qualifier)) { + unresolvedProjectionStar = true; + continue; + } + String qualifierKey = identifierLookupKey(qualifier); + Table physicalTable = currentScope.physicalByQualifier() + .get(qualifierKey); + if (physicalTable != null) { + projectionStarPhysicalTables.add(physicalTable); + } else if (!currentScope.derivedByQualifier() + .containsKey(qualifierKey)) { + unresolvedProjectionStar = true; + } + } else if (expression instanceof AllColumns) { + projectionStarPhysicalTables.addAll( + currentScope.physicalTables()); + } + } + } + + /** + * 读取 CTE 显式列清单。 + * + * @param withItem CTE 定义 + * @return 列标识键集合 + */ + private Set explicitCteColumns(WithItem withItem) { + if (withItem.getWithItemList() == null + || withItem.getWithItemList().isEmpty()) { + return Set.of(); + } + return outputColumns(withItem.getWithItemList()); + } + + /** + * 读取 SELECT 的确定输出列。 + * + * @param select SELECT 节点 + * @return 列标识键集合；无法静态确定时返回空集合 + */ + private Set outputColumns(Select select) { + if (select instanceof ParenthesedSelect parenthesedSelect) { + return outputColumns(parenthesedSelect.getSelect()); + } + if (!(select instanceof PlainSelect plainSelect) + || plainSelect.getSelectItems() == null) { + return Set.of(); + } + return outputColumns(plainSelect.getSelectItems()); + } + + /** + * 从选择项中提取可确定的输出列。 + * + * @param selectItems 选择项 + * @return 列标识键集合 + */ + private Set outputColumns(List> selectItems) { + Set outputs = new LinkedHashSet<>(); + for (SelectItem selectItem : selectItems) { + String outputName = null; + if (selectItem.getAlias() != null + && hasText(selectItem.getAlias().getName())) { + outputName = selectItem.getAlias().getName(); + } else { + Expression expression = selectItem.getExpression(); + if (expression instanceof Column column) { + outputName = column.getColumnName(); + } + } + if (hasText(outputName)) { + outputs.addAll(managedIdentifierKeys(outputName)); + } + } + return outputs.isEmpty() ? Set.of() : Set.copyOf(outputs); + } + + /** + * 只在当前子句表达式内收集未限定的投影别名引用。 + */ + private final class OutputAliasReferenceMarker extends ExpressionVisitorAdapter { + private final Set outputAliases; + private final Set
physicalTables; + + /** + * 创建投影别名标记器。 + * + * @param outputAliases 当前查询块的投影输出名 + * @param physicalTables 当前查询块直接引用的物理表 + */ + private OutputAliasReferenceMarker( + Set outputAliases, + Set
physicalTables) { + this.outputAliases = outputAliases; + this.physicalTables = physicalTables; + } + + /** + * 标记未限定且命中当前投影输出名的字段节点。 + * + * @param column 字段节点 + */ + @Override + public void visit(Column column) { + if ((column.getTable() == null + || !hasText(column.getTable().getName())) + && outputAliases.contains(identifierLookupKey( + column.getColumnName()))) { + outputAliasReferences.add(column); + outputAliasPhysicalTables.put(column, physicalTables); + } + } + + /** + * 子查询拥有独立投影作用域，不继承外层别名标记上下文。 + * + * @param select 子查询 + */ + @Override + public void visit(Select select) { + // 当前标记器只处理所属查询块的子句表达式。 + } + } + } + + /** + * 查询块内可见的派生关系和输出列。 + * + * @param visibleCtes 可见 CTE 输出 + * @param derivedByQualifier 派生关系限定符与输出 + * @param physicalByQualifier 物理关系限定符与表节点 + * @param physicalTables 当前查询块直接引用的物理表 + * @param relationQualifiers 当前查询块声明的全部关系限定符 + * @param soleDerivedOutputs 唯一 FROM 派生关系的输出 + */ + private record QueryScope( + Map> visibleCtes, + Map> derivedByQualifier, + Map physicalByQualifier, + Set
physicalTables, + Set relationQualifiers, + Set soleDerivedOutputs) { + } + + /** + * 可静态确定输出列的派生关系。 + * + * @param qualifier 关系限定符 + * @param outputColumns 输出列 + */ + private record DerivedRelation(String qualifier, Set outputColumns) { } } diff --git a/easyflow-modules/easyflow-module-datacenter/src/test/java/tech/easyflow/datacenter/audit/DatacenterQueryAuditServiceTest.java b/easyflow-modules/easyflow-module-datacenter/src/test/java/tech/easyflow/datacenter/audit/DatacenterQueryAuditServiceTest.java new file mode 100644 index 00000000..a3f6ab03 --- /dev/null +++ b/easyflow-modules/easyflow-module-datacenter/src/test/java/tech/easyflow/datacenter/audit/DatacenterQueryAuditServiceTest.java @@ -0,0 +1,79 @@ +package tech.easyflow.datacenter.audit; + +import java.math.BigInteger; +import java.util.Map; +import org.junit.Assert; +import org.junit.Test; +import org.mockito.ArgumentCaptor; +import org.mockito.Mockito; +import tech.easyflow.common.entity.LoginAccount; +import tech.easyflow.datacenter.mapper.DatacenterQueryAuditMapper; +import tech.easyflow.datacenter.meta.entity.DatacenterSource; + +/** + * {@link DatacenterQueryAuditService} 审计脱敏回归测试。 + */ +public class DatacenterQueryAuditServiceTest { + + /** + * 验证 PostgreSQL dollar quote、字符串、数字和注释内容均不会进入审计 SQL。 + */ + @Test + public void startShouldRedactLiteralsAndComments() { + DatacenterQueryAuditMapper mapper = Mockito.mock( + DatacenterQueryAuditMapper.class); + DatacenterQueryAuditService service = new DatacenterQueryAuditService(mapper); + DatacenterSource source = new DatacenterSource(); + source.setId(BigInteger.ONE); + source.setTenantId(BigInteger.ONE); + + service.start( + "query-1", + source, + "SELECT $$secret$$, $tag$hidden$tag$, 'password', 42 " + + "/* api-key /* nested */ value */ -- bearer-token\nFROM orders", + Map.of(), + null, + "CONSOLE", + null); + + ArgumentCaptor captor = ArgumentCaptor.forClass( + DatacenterQueryAudit.class); + Mockito.verify(mapper).insert(captor.capture()); + String sql = captor.getValue().getParameterizedSql(); + Assert.assertFalse(sql.contains("secret")); + Assert.assertFalse(sql.contains("hidden")); + Assert.assertFalse(sql.contains("password")); + Assert.assertFalse(sql.contains("api-key")); + Assert.assertFalse(sql.contains("bearer-token")); + Assert.assertFalse(sql.contains("42")); + Assert.assertTrue(sql.contains("FROM orders")); + } + + /** + * 验证执行账号与业务调用对象分别落入独立审计字段。 + */ + @Test + public void startShouldPersistExecutorSeparatelyFromCaller() { + DatacenterQueryAuditMapper mapper = Mockito.mock( + DatacenterQueryAuditMapper.class); + DatacenterQueryAuditService service = new DatacenterQueryAuditService(mapper); + DatacenterSource source = new DatacenterSource(); + source.setId(BigInteger.ONE); + source.setTenantId(BigInteger.ONE); + LoginAccount account = new LoginAccount(); + account.setId(BigInteger.valueOf(9001L)); + account.setTenantId(BigInteger.ONE); + + service.start( + "query-2", source, "SELECT id FROM orders", Map.of(), + account, "DATASET", "table-2001"); + + ArgumentCaptor captor = ArgumentCaptor.forClass( + DatacenterQueryAudit.class); + Mockito.verify(mapper).insert(captor.capture()); + Assert.assertEquals(BigInteger.valueOf(9001L), + captor.getValue().getExecutorAccountId()); + Assert.assertEquals("table-2001", captor.getValue().getCallerId()); + } +} diff --git a/easyflow-modules/easyflow-module-datacenter/src/test/java/tech/easyflow/datacenter/connector/support/AbstractInternalTableConnectorTest.java b/easyflow-modules/easyflow-module-datacenter/src/test/java/tech/easyflow/datacenter/connector/support/AbstractInternalTableConnectorTest.java index ed95ed26..193c005e 100644 --- a/easyflow-modules/easyflow-module-datacenter/src/test/java/tech/easyflow/datacenter/connector/support/AbstractInternalTableConnectorTest.java +++ b/easyflow-modules/easyflow-module-datacenter/src/test/java/tech/easyflow/datacenter/connector/support/AbstractInternalTableConnectorTest.java @@ -8,8 +8,12 @@ import org.junit.Assert; import org.junit.Test; import org.mockito.MockedStatic; import org.mockito.Mockito; +import tech.easyflow.common.web.exceptions.BusinessException; import tech.easyflow.datacenter.entity.DatacenterTable; +import tech.easyflow.datacenter.entity.DatacenterTableField; import tech.easyflow.datacenter.execution.model.DatacenterQueryRequest; +import tech.easyflow.datacenter.execution.model.DatacenterQueryFilter; +import tech.easyflow.datacenter.execution.model.DatacenterQuerySort; import tech.easyflow.datacenter.meta.enums.DatacenterSourceType; import java.math.BigInteger; @@ -47,9 +51,23 @@ public class AbstractInternalTableConnectorTest { public void shouldUseIndependentQueryWrapperForPaginationAfterCount() { DatacenterTable table = new DatacenterTable(); table.setActualTable("preview_table"); + DatacenterTableField field = new DatacenterTableField(); + field.setFieldName("model_id"); + field.setSourceColumnName("MODEL_ID"); + table.setFields(List.of(field)); DatacenterQueryRequest request = new DatacenterQueryRequest(); request.setPageNumber(1L); request.setPageSize(10L); + request.setSelectedColumns(List.of("model_id")); + DatacenterQueryFilter filter = new DatacenterQueryFilter(); + filter.setColumn("model_id"); + filter.setOperator("EQ"); + filter.setValue("deepseek-v4-pro"); + request.setFilters(List.of(filter)); + DatacenterQuerySort sort = new DatacenterQuerySort(); + sort.setColumn("model_id"); + sort.setDirection("DESC"); + request.setSorts(List.of(sort)); Row record = new Row(); record.put("MODEL_ID", "deepseek-v4-pro"); @@ -79,10 +97,22 @@ public class AbstractInternalTableConnectorTest { null, table, request); Assert.assertNotSame(countWrapper.get(), pageWrapper.get()); + Assert.assertTrue(countWrapper.get().toSQL().contains("`MODEL_ID` =")); + Assert.assertFalse(countWrapper.get().toSQL().contains("DESC")); + Assert.assertTrue(pageWrapper.get().toSQL().contains("`MODEL_ID` DESC")); Assert.assertEquals("deepseek-v4-pro", page.getRecords().get(0).get("model_id")); } } + /** + * 验证结构化分页入口拒绝调用方拼接原始 WHERE 片段。 + */ + @Test(expected = BusinessException.class) + public void shouldRejectRawWhereClause() { + AbstractInternalTableConnector.createQueryWrapper( + "1 = 1 UNION SELECT secret FROM another_table"); + } + /** * 供内部动态表查询测试使用的最小连接器实现。 */ diff --git a/easyflow-modules/easyflow-module-datacenter/src/test/java/tech/easyflow/datacenter/connector/support/AbstractJdbcConnectorMetadataTest.java b/easyflow-modules/easyflow-module-datacenter/src/test/java/tech/easyflow/datacenter/connector/support/AbstractJdbcConnectorMetadataTest.java new file mode 100644 index 00000000..cbecdf84 --- /dev/null +++ b/easyflow-modules/easyflow-module-datacenter/src/test/java/tech/easyflow/datacenter/connector/support/AbstractJdbcConnectorMetadataTest.java @@ -0,0 +1,270 @@ +package tech.easyflow.datacenter.connector.support; + +import java.math.BigInteger; +import java.sql.Connection; +import java.sql.DatabaseMetaData; +import java.sql.ResultSet; +import java.sql.Types; +import java.util.List; +import java.util.Set; +import org.junit.Assert; +import org.junit.Test; +import org.mockito.ArgumentMatchers; +import org.mockito.Mockito; +import tech.easyflow.datacenter.connector.SqlDialect; +import tech.easyflow.datacenter.meta.entity.DatacenterSource; +import tech.easyflow.datacenter.meta.enums.DatacenterSourceType; +import tech.easyflow.datacenter.meta.model.DatacenterManagedMetadataSnapshot; +import tech.easyflow.datacenter.meta.model.DatacenterTableDetailMeta; + +/** + * {@link AbstractJdbcConnector} JDBC 元数据精确匹配回归测试。 + */ +public class AbstractJdbcConnectorMetadataTest { + + /** + * 验证包含下划线的表名会转义元数据 Pattern，并过滤驱动返回的近似匹配项。 + * + * @throws Exception 模拟 JDBC 调用失败 + */ + @Test + public void getTableDetailShouldEscapePatternsAndFilterMetadataRows() + throws Exception { + Connection connection = Mockito.mock(Connection.class); + DatabaseMetaData metadata = Mockito.mock(DatabaseMetaData.class); + ResultSet tables = Mockito.mock(ResultSet.class); + ResultSet primaryKeys = Mockito.mock(ResultSet.class); + ResultSet columns = Mockito.mock(ResultSet.class); + Mockito.when(connection.getMetaData()).thenReturn(metadata); + Mockito.when(metadata.getSearchStringEscape()).thenReturn("\\"); + Mockito.when(metadata.getTables( + Mockito.eq("analytics"), Mockito.isNull(), + Mockito.eq("order\\_item"), ArgumentMatchers.any(String[].class))) + .thenReturn(tables); + Mockito.when(metadata.getPrimaryKeys( + "analytics", null, "order_item")) + .thenReturn(primaryKeys); + Mockito.when(metadata.getColumns( + "analytics", null, "order\\_item", "%")) + .thenReturn(columns); + + Mockito.when(tables.next()).thenReturn(true, true, false); + Mockito.when(tables.getString("TABLE_NAME")) + .thenReturn("orderXitem", "order_item"); + Mockito.when(tables.getString("TABLE_CAT")).thenReturn("analytics"); + Mockito.when(tables.getString("TABLE_SCHEM")).thenReturn(null); + Mockito.when(tables.getString("REMARKS")).thenReturn("订单"); + Mockito.when(tables.getString("TABLE_TYPE")).thenReturn("TABLE"); + Mockito.when(primaryKeys.next()).thenReturn(false); + + Mockito.when(columns.next()).thenReturn(true, true, false); + Mockito.when(columns.getString("TABLE_NAME")) + .thenReturn("orderXitem", "order_item"); + Mockito.when(columns.getString("TABLE_CAT")).thenReturn("analytics"); + Mockito.when(columns.getString("TABLE_SCHEM")).thenReturn(null); + Mockito.when(columns.getString("COLUMN_NAME")).thenReturn("id"); + Mockito.when(columns.getString("REMARKS")).thenReturn("主键"); + Mockito.when(columns.getString("TYPE_NAME")).thenReturn("BIGINT"); + Mockito.when(columns.getInt("DATA_TYPE")).thenReturn(Types.BIGINT); + Mockito.when(columns.getInt("ORDINAL_POSITION")).thenReturn(1); + Mockito.when(columns.getInt("COLUMN_SIZE")).thenReturn(19); + Mockito.when(columns.getInt("DECIMAL_DIGITS")).thenReturn(0); + Mockito.when(columns.getInt("NULLABLE")) + .thenReturn(DatabaseMetaData.columnNoNulls); + + DatacenterSource source = new DatacenterSource(); + source.setId(BigInteger.ONE); + source.setDatabaseName("analytics"); + DatacenterTableDetailMeta detail = new TestJdbcConnector(connection) + .getTableDetail(source, "analytics", "order_item"); + + Assert.assertEquals("订单", detail.getTable().getTableDesc()); + Assert.assertEquals(1, detail.getFields().size()); + Assert.assertEquals("id", detail.getFields().get(0).getFieldName()); + Mockito.verify(metadata).getTables( + Mockito.eq("analytics"), Mockito.isNull(), + Mockito.eq("order\\_item"), ArgumentMatchers.any(String[].class)); + Mockito.verify(metadata).getColumns( + "analytics", null, "order\\_item", "%"); + } + + /** + * 刷新多个已纳管表时复用一个连接，并显式返回缺失对象。 + * + * @throws Exception 模拟 JDBC 调用失败 + */ + @Test + public void inspectManagedTablesShouldReuseConnectionAndReportMissing() + throws Exception { + Connection connection = Mockito.mock(Connection.class); + DatabaseMetaData metadata = Mockito.mock(DatabaseMetaData.class); + ResultSet orders = Mockito.mock(ResultSet.class); + ResultSet missing = Mockito.mock(ResultSet.class); + ResultSet primaryKeys = Mockito.mock(ResultSet.class); + ResultSet columns = Mockito.mock(ResultSet.class); + Mockito.when(connection.getMetaData()).thenReturn(metadata); + Mockito.when(metadata.getSearchStringEscape()).thenReturn("\\"); + Mockito.when(metadata.getTables( + Mockito.eq("analytics"), Mockito.isNull(), Mockito.eq("orders"), + ArgumentMatchers.any(String[].class))) + .thenReturn(orders); + Mockito.when(metadata.getTables( + Mockito.eq("analytics"), Mockito.isNull(), Mockito.eq("deleted\\_table"), + ArgumentMatchers.any(String[].class))) + .thenReturn(missing); + Mockito.when(metadata.getPrimaryKeys("analytics", null, "orders")) + .thenReturn(primaryKeys); + Mockito.when(metadata.getColumns("analytics", null, "orders", "%")) + .thenReturn(columns); + Mockito.when(orders.next()).thenReturn(true, false); + Mockito.when(orders.getString("TABLE_NAME")).thenReturn("orders"); + Mockito.when(orders.getString("TABLE_CAT")).thenReturn("analytics"); + Mockito.when(orders.getString("TABLE_SCHEM")).thenReturn(null); + Mockito.when(orders.getString("TABLE_TYPE")).thenReturn("TABLE"); + Mockito.when(missing.next()).thenReturn(false); + Mockito.when(primaryKeys.next()).thenReturn(false); + Mockito.when(columns.next()).thenReturn(true, false); + Mockito.when(columns.getString("TABLE_NAME")).thenReturn("orders"); + Mockito.when(columns.getString("TABLE_CAT")).thenReturn("analytics"); + Mockito.when(columns.getString("TABLE_SCHEM")).thenReturn(null); + Mockito.when(columns.getString("COLUMN_NAME")).thenReturn("id"); + Mockito.when(columns.getString("TYPE_NAME")).thenReturn("BIGINT"); + Mockito.when(columns.getInt("DATA_TYPE")).thenReturn(Types.BIGINT); + Mockito.when(columns.getInt("ORDINAL_POSITION")).thenReturn(1); + Mockito.when(columns.getInt("COLUMN_SIZE")).thenReturn(19); + Mockito.when(columns.getInt("DECIMAL_DIGITS")).thenReturn(0); + Mockito.when(columns.getInt("NULLABLE")) + .thenReturn(DatabaseMetaData.columnNoNulls); + + DatacenterSource source = new DatacenterSource(); + source.setId(BigInteger.ONE); + source.setDatabaseName("analytics"); + TestJdbcConnector connector = new TestJdbcConnector(connection); + DatacenterManagedMetadataSnapshot snapshot = connector.inspectManagedTables( + source, "analytics", List.of("orders", "deleted_table")); + + Assert.assertEquals(1, connector.connectionCalls()); + Assert.assertEquals(1, snapshot.details().size()); + Assert.assertEquals("orders", snapshot.details().get(0).getTable().getTableName()); + Assert.assertEquals(Set.of("deleted_table"), snapshot.missingTableNames()); + } + + /** + * 验证 Schema 型数据库不依赖驱动可选的 TABLE_CAT 回填。 + * + * @throws Exception 模拟 JDBC 调用失败 + */ + @Test + public void getTableDetailShouldAcceptMissingCatalogForPostgresql() + throws Exception { + Connection connection = Mockito.mock(Connection.class); + DatabaseMetaData metadata = Mockito.mock(DatabaseMetaData.class); + ResultSet tables = Mockito.mock(ResultSet.class); + ResultSet primaryKeys = Mockito.mock(ResultSet.class); + ResultSet columns = Mockito.mock(ResultSet.class); + Mockito.when(connection.getMetaData()).thenReturn(metadata); + Mockito.when(metadata.getSearchStringEscape()).thenReturn("\\"); + Mockito.when(metadata.getTables( + Mockito.eq("harmony_adapter"), Mockito.eq("public"), + Mockito.eq("flyway\\_schema\\_history"), + ArgumentMatchers.any(String[].class))) + .thenReturn(tables); + Mockito.when(metadata.getPrimaryKeys( + "harmony_adapter", "public", "flyway_schema_history")) + .thenReturn(primaryKeys); + Mockito.when(metadata.getColumns( + "harmony_adapter", "public", + "flyway\\_schema\\_history", "%")) + .thenReturn(columns); + Mockito.when(tables.next()).thenReturn(true, false); + Mockito.when(tables.getString("TABLE_NAME")) + .thenReturn("flyway_schema_history"); + Mockito.when(tables.getString("TABLE_CAT")).thenReturn(null); + Mockito.when(tables.getString("TABLE_SCHEM")).thenReturn("public"); + Mockito.when(tables.getString("TABLE_TYPE")).thenReturn("TABLE"); + Mockito.when(primaryKeys.next()).thenReturn(false); + Mockito.when(columns.next()).thenReturn(true, false); + Mockito.when(columns.getString("TABLE_NAME")) + .thenReturn("flyway_schema_history"); + Mockito.when(columns.getString("TABLE_CAT")).thenReturn(null); + Mockito.when(columns.getString("TABLE_SCHEM")).thenReturn("public"); + Mockito.when(columns.getString("COLUMN_NAME")).thenReturn("installed_rank"); + Mockito.when(columns.getString("TYPE_NAME")).thenReturn("INTEGER"); + Mockito.when(columns.getInt("DATA_TYPE")).thenReturn(Types.INTEGER); + Mockito.when(columns.getInt("ORDINAL_POSITION")).thenReturn(1); + Mockito.when(columns.getInt("COLUMN_SIZE")).thenReturn(10); + Mockito.when(columns.getInt("DECIMAL_DIGITS")).thenReturn(0); + Mockito.when(columns.getInt("NULLABLE")) + .thenReturn(DatabaseMetaData.columnNoNulls); + + DatacenterSource source = new DatacenterSource(); + source.setId(BigInteger.ONE); + source.setDatabaseName("harmony_adapter"); + source.setSchemaName("public"); + DatacenterTableDetailMeta detail = new TestJdbcConnector( + connection, DatacenterSourceType.POSTGRESQL) + .getTableDetail(source, "public", "flyway_schema_history"); + + Assert.assertEquals("flyway_schema_history", detail.getTable().getTableName()); + Assert.assertEquals("installed_rank", detail.getFields().get(0).getFieldName()); + } + + /** + * 使用固定模拟连接的测试连接器。 + */ + private static final class TestJdbcConnector extends AbstractJdbcConnector { + + private final Connection connection; + private int connectionCalls; + + /** + * 创建测试连接器。 + * + * @param connection 模拟 JDBC 连接 + */ + private TestJdbcConnector(Connection connection) { + this(connection, DatacenterSourceType.MYSQL); + } + + /** + * 创建指定数据库类型的测试连接器。 + * + * @param connection 模拟 JDBC 连接 + * @param sourceType 数据源类型 + */ + private TestJdbcConnector( + Connection connection, + DatacenterSourceType sourceType) { + super(sourceType, Mockito.mock(SqlDialect.class), Set.of()); + this.connection = connection; + } + + /** + * 将元数据操作交给固定测试连接。 + * + * @param source 数据源定义 + * @param cacheable 是否允许连接缓存 + * @param callback JDBC 回调 + * @param 返回值类型 + * @return 回调结果 + * @throws Exception 回调失败 + */ + @Override + protected T withConnection( + DatacenterSource source, + boolean cacheable, + JdbcCallback callback) throws Exception { + connectionCalls++; + return callback.apply(connection); + } + + /** + * 返回元数据操作请求连接的次数。 + * + * @return 调用次数 + */ + private int connectionCalls() { + return connectionCalls; + } + } +} diff --git a/easyflow-modules/easyflow-module-datacenter/src/test/java/tech/easyflow/datacenter/execution/service/impl/DatacenterDatasetQueryServiceImplTest.java b/easyflow-modules/easyflow-module-datacenter/src/test/java/tech/easyflow/datacenter/execution/service/impl/DatacenterDatasetQueryServiceImplTest.java index 86db4fb0..29290c07 100644 --- a/easyflow-modules/easyflow-module-datacenter/src/test/java/tech/easyflow/datacenter/execution/service/impl/DatacenterDatasetQueryServiceImplTest.java +++ b/easyflow-modules/easyflow-module-datacenter/src/test/java/tech/easyflow/datacenter/execution/service/impl/DatacenterDatasetQueryServiceImplTest.java @@ -1,33 +1,105 @@ package tech.easyflow.datacenter.execution.service.impl; +import com.mybatisflex.core.paginate.Page; import com.mybatisflex.core.row.Row; import org.junit.Assert; import org.junit.Test; import org.mockito.ArgumentCaptor; import org.mockito.ArgumentMatchers; import org.mockito.Mockito; +import tech.easyflow.datacenter.audit.DatacenterQueryAuditService; import tech.easyflow.datacenter.connector.DatacenterConnector; import tech.easyflow.datacenter.connector.DatacenterConnectorRegistry; import tech.easyflow.datacenter.connector.support.AbstractInternalTableConnector; import tech.easyflow.datacenter.entity.DatacenterTable; +import tech.easyflow.datacenter.entity.DatacenterTableField; import tech.easyflow.datacenter.execution.model.DatacenterSqlQueryRequest; +import tech.easyflow.datacenter.execution.model.DatacenterSqlColumnView; +import tech.easyflow.datacenter.execution.model.DatacenterSqlConsoleResult; +import tech.easyflow.datacenter.execution.model.DatacenterQueryFilter; +import tech.easyflow.datacenter.execution.model.DatacenterQueryRequest; +import tech.easyflow.datacenter.execution.model.DatacenterQuerySort; +import tech.easyflow.datacenter.execution.model.DatacenterSchemaResponse; import tech.easyflow.datacenter.execution.model.DatasetRef; +import tech.easyflow.datacenter.federation.DatacenterFederationQueryService; import tech.easyflow.datacenter.mapper.DatacenterCatalogMapper; +import tech.easyflow.datacenter.mapper.DatacenterDatasetVersionMapper; +import tech.easyflow.datacenter.mapper.DatacenterDerivedTableMapper; import tech.easyflow.datacenter.meta.entity.DatacenterCatalog; import tech.easyflow.datacenter.meta.entity.DatacenterSource; +import tech.easyflow.datacenter.meta.enums.DatacenterMetadataStatus; +import tech.easyflow.datacenter.meta.enums.DatacenterSensitivityLevel; import tech.easyflow.datacenter.meta.service.DatacenterDatasetRegistryService; import java.lang.reflect.Field; import java.math.BigInteger; import java.util.ArrayList; import java.util.List; +import java.util.Map; import java.util.function.Consumer; +import tech.easyflow.common.entity.LoginAccount; +import tech.easyflow.common.web.exceptions.BusinessException; /** * {@link DatacenterDatasetQueryServiceImpl} 分页 SQL 读取回归测试。 */ public class DatacenterDatasetQueryServiceImplTest { + /** + * 验证结构接口只返回请求的字段页，且不在表对象内重复序列化完整字段。 + * + * @throws Exception 测试依赖注入失败时抛出 + */ + @Test + public void getSchemaShouldReturnBoundedFieldPage() throws Exception { + BigInteger sourceId = BigInteger.valueOf(6101L); + BigInteger catalogId = BigInteger.valueOf(6102L); + BigInteger tableId = BigInteger.valueOf(6103L); + DatacenterSource source = new DatacenterSource(); + source.setId(sourceId); + DatacenterCatalog catalog = new DatacenterCatalog(); + catalog.setId(catalogId); + DatacenterTable table = new DatacenterTable(); + table.setId(tableId); + table.setSourceId(sourceId); + table.setCatalogId(catalogId); + table.setFields(List.of( + queryableField("first"), + queryableField("second"), + queryableField("third"))); + DatasetRef datasetRef = new DatasetRef(); + datasetRef.setTableId(tableId); + DatacenterDatasetRegistryService registry = Mockito.mock( + DatacenterDatasetRegistryService.class); + Mockito.when(registry.getTableWithFields(tableId)).thenReturn(table); + Mockito.when(registry.resolveDatasetRef(tableId)).thenReturn(datasetRef); + Mockito.when(registry.getSourceRequired(sourceId)).thenReturn(source); + Mockito.when(registry.getCatalogById(catalogId)).thenReturn(catalog); + DatacenterDatasetVersionMapper versionMapper = Mockito.mock( + DatacenterDatasetVersionMapper.class); + DatacenterDerivedTableMapper derivedMapper = Mockito.mock( + DatacenterDerivedTableMapper.class); + Mockito.when(versionMapper.selectListByQuery(ArgumentMatchers.any())) + .thenReturn(List.of()); + Mockito.when(derivedMapper.selectListByQuery(ArgumentMatchers.any())) + .thenReturn(List.of()); + DatacenterDatasetQueryServiceImpl service = + new DatacenterDatasetQueryServiceImpl(); + setField(service, "registryService", registry); + setField(service, "datasetVersionMapper", versionMapper); + setField(service, "derivedTableMapper", derivedMapper); + + DatacenterSchemaResponse response = service.getSchema( + datasetRef, 2L, 2L); + + Assert.assertEquals(2L, response.getFieldPageNumber()); + Assert.assertEquals(2L, response.getFieldPageSize()); + Assert.assertFalse(response.isHasMoreFields()); + Assert.assertEquals(List.of("third"), response.getFields().stream() + .map(DatacenterTableField::getFieldName).toList()); + Assert.assertTrue(response.getTable().getFields().isEmpty()); + } + /** * 验证惰性迭代器逐页读取并保持原始行顺序。 * @@ -39,12 +111,15 @@ public class DatacenterDatasetQueryServiceImplTest { BigInteger sourceId = BigInteger.valueOf(1001L); DatacenterSource source = new DatacenterSource(); source.setId(sourceId); - source.setSourceType("MYSQL"); + source.setTenantId(BigInteger.ONE); + source.setSourceType("PROJECT_MYSQL"); DatacenterTable table = new DatacenterTable(); table.setId(BigInteger.valueOf(2001L)); + table.setTenantId(BigInteger.ONE); table.setSourceId(sourceId); table.setTableName("orders"); table.setActualTable("orders_actual"); + table.setFields(List.of(queryableField("id"))); DatacenterDatasetRegistryService registry = Mockito.mock( @@ -54,6 +129,8 @@ public class DatacenterDatasetQueryServiceImplTest { Mockito.when(registry.listManagedTables( sourceId, null)) .thenReturn(List.of(table)); + Mockito.when(registry.getTableWithFields(table.getId())) + .thenReturn(table); DatacenterConnector connector = Mockito.mock(DatacenterConnector.class); Mockito.doAnswer(invocation -> { @@ -74,17 +151,21 @@ public class DatacenterDatasetQueryServiceImplTest { DatacenterConnectorRegistry connectors = Mockito.mock( DatacenterConnectorRegistry.class); - Mockito.when(connectors.getConnector("MYSQL")) + Mockito.when(connectors.getConnector("PROJECT_MYSQL")) .thenReturn(connector); DatacenterDatasetQueryServiceImpl service = new DatacenterDatasetQueryServiceImpl(); setField(service, "registryService", registry); setField(service, "connectorRegistry", connectors); + setField(service, "queryAuditService", + Mockito.mock(DatacenterQueryAuditService.class)); DatacenterSqlQueryRequest request = new DatacenterSqlQueryRequest(); DatasetRef datasetRef = new DatasetRef(); + datasetRef.setTenantId(BigInteger.ONE); datasetRef.setSourceId(sourceId); + datasetRef.setTableId(table.getId()); request.setDatasetRef(datasetRef); request.setSql("SELECT * FROM orders ORDER BY id"); @@ -105,6 +186,77 @@ public class DatacenterDatasetQueryServiceImplTest { ArgumentMatchers.any()); } + /** + * 验证外部 MySQL SQL 查询统一进入 Federation SQL 并保持列标签。 + * + * @throws Exception 测试依赖注入失败时抛出 + */ + @Test + public void queryBySqlShouldUseFederationForExternalMysql() + throws Exception { + BigInteger sourceId = BigInteger.valueOf(4101L); + BigInteger tableId = BigInteger.valueOf(4102L); + DatacenterSource source = new DatacenterSource(); + source.setId(sourceId); + source.setTenantId(BigInteger.ONE); + source.setSourceType("MYSQL"); + DatacenterTable table = new DatacenterTable(); + table.setId(tableId); + table.setTenantId(BigInteger.ONE); + table.setSourceId(sourceId); + table.setTableName("orders"); + table.setActualTable("orders"); + + DatacenterDatasetRegistryService registry = Mockito.mock( + DatacenterDatasetRegistryService.class); + Mockito.when(registry.getSourceRequired(sourceId)).thenReturn(source); + Mockito.when(registry.listManagedTables(sourceId, null)) + .thenReturn(List.of(table)); + Mockito.when(registry.getTableWithFields(tableId)).thenReturn(table); + DatacenterConnector connector = Mockito.mock(DatacenterConnector.class); + DatacenterConnectorRegistry connectors = Mockito.mock( + DatacenterConnectorRegistry.class); + Mockito.when(connectors.getConnector("MYSQL")).thenReturn(connector); + DatacenterFederationQueryService federation = Mockito.mock( + DatacenterFederationQueryService.class); + Mockito.when(federation.execute( + Mockito.eq(source), + Mockito.anyString(), + Mockito.eq(List.of()), + Mockito.eq(1_000), + Mockito.isNull(), + Mockito.eq("DATASET"), + Mockito.eq(tableId.toString()))) + .thenReturn(new DatacenterSqlConsoleResult( + "query-1", + List.of(new DatacenterSqlColumnView( + "c1", "order_id", java.sql.Types.BIGINT, + "BIGINT", false)), + List.of(java.util.Map.of("c1", "12")), + 1L, + false, + 3L)); + + DatacenterDatasetQueryServiceImpl service = + new DatacenterDatasetQueryServiceImpl(); + setField(service, "registryService", registry); + setField(service, "connectorRegistry", connectors); + setField(service, "federationQueryService", federation); + DatacenterSqlQueryRequest request = new DatacenterSqlQueryRequest(); + DatasetRef datasetRef = new DatasetRef(); + datasetRef.setTenantId(BigInteger.ONE); + datasetRef.setSourceId(sourceId); + datasetRef.setTableId(tableId); + request.setDatasetRef(datasetRef); + request.setSql("SELECT order_id FROM orders"); + + List rows = service.queryBySql(request); + + Assert.assertEquals("12", rows.get(0).get("order_id")); + Mockito.verify(connector, Mockito.never()).queryBySql( + Mockito.any(), Mockito.anyString()); + } + /** * 验证内部 Excel 查询移除逻辑目录并转换 MySQL 标识符引号。 * @@ -117,13 +269,19 @@ public class DatacenterDatasetQueryServiceImplTest { BigInteger catalogId = BigInteger.valueOf(3002L); DatacenterSource source = new DatacenterSource(); source.setId(sourceId); + source.setTenantId(BigInteger.ONE); source.setSourceType("EXCEL"); DatacenterTable table = new DatacenterTable(); table.setId(BigInteger.valueOf(3003L)); + table.setTenantId(BigInteger.ONE); table.setSourceId(sourceId); table.setCatalogId(catalogId); table.setTableName("Sheet1"); table.setMaterializedTable("tb_excel_budget"); + table.setFields(List.of( + queryableField("token"), + queryableField("token_1"), + queryableField("col_id"))); DatacenterCatalog catalog = new DatacenterCatalog(); catalog.setId(catalogId); @@ -134,6 +292,8 @@ public class DatacenterDatasetQueryServiceImplTest { .thenReturn(source); Mockito.when(registry.listManagedTables(sourceId, null)) .thenReturn(List.of(table)); + Mockito.when(registry.getTableWithFields(table.getId())) + .thenReturn(table); DatacenterCatalogMapper catalogMapper = Mockito.mock(DatacenterCatalogMapper.class); Mockito.when(catalogMapper.selectListByQuery( @@ -151,10 +311,14 @@ public class DatacenterDatasetQueryServiceImplTest { setField(service, "registryService", registry); setField(service, "connectorRegistry", connectors); setField(service, "catalogMapper", catalogMapper); + setField(service, "queryAuditService", + Mockito.mock(DatacenterQueryAuditService.class)); DatacenterSqlQueryRequest request = new DatacenterSqlQueryRequest(); DatasetRef datasetRef = new DatasetRef(); + datasetRef.setTenantId(BigInteger.ONE); datasetRef.setSourceId(sourceId); + datasetRef.setTableId(table.getId()); request.setDatasetRef(datasetRef); request.setSql(""" SELECT "token", "token_1" @@ -175,6 +339,187 @@ public class DatacenterDatasetQueryServiceImplTest { sqlCaptor.getValue()); } + /** + * 验证项目 MySQL 原生 SQL 也会拒绝不可查询字段和同名输出别名绕过。 + * + * @throws Exception 测试依赖注入失败时抛出 + */ + @Test(expected = BusinessException.class) + public void queryBySqlShouldEnforceProjectMysqlColumnPolicy() + throws Exception { + BigInteger sourceId = BigInteger.valueOf(5101L); + BigInteger tableId = BigInteger.valueOf(5102L); + DatacenterSource source = new DatacenterSource(); + source.setId(sourceId); + source.setTenantId(BigInteger.ONE); + source.setSourceType("PROJECT_MYSQL"); + DatacenterTable table = new DatacenterTable(); + table.setId(tableId); + table.setTenantId(BigInteger.ONE); + table.setSourceId(sourceId); + table.setTableName("orders"); + table.setActualTable("orders"); + DatacenterTableField hidden = queryableField("secret"); + hidden.setQueryable(0); + hidden.setSensitivityLevel(DatacenterSensitivityLevel.RESTRICTED.name()); + table.setFields(List.of(queryableField("id"), hidden)); + + DatacenterDatasetRegistryService registry = Mockito.mock( + DatacenterDatasetRegistryService.class); + Mockito.when(registry.getSourceRequired(sourceId)).thenReturn(source); + Mockito.when(registry.getTableWithFields(tableId)).thenReturn(table); + DatacenterConnector connector = Mockito.mock(DatacenterConnector.class); + DatacenterConnectorRegistry connectors = Mockito.mock( + DatacenterConnectorRegistry.class); + Mockito.when(connectors.getConnector("PROJECT_MYSQL")) + .thenReturn(connector); + + DatacenterDatasetQueryServiceImpl service = + new DatacenterDatasetQueryServiceImpl(); + setField(service, "registryService", registry); + setField(service, "connectorRegistry", connectors); + DatacenterSqlQueryRequest request = new DatacenterSqlQueryRequest(); + DatasetRef datasetRef = new DatasetRef(); + datasetRef.setTenantId(BigInteger.ONE); + datasetRef.setSourceId(sourceId); + datasetRef.setTableId(tableId); + request.setDatasetRef(datasetRef); + request.setSql("SELECT secret AS secret FROM orders"); + + service.queryBySql(request); + } + + /** + * 验证结构化分页审计绑定执行账号且只记录脱敏筛选摘要。 + * + * @throws Exception 测试依赖注入失败时抛出 + */ + @Test + @SuppressWarnings({"rawtypes", "unchecked"}) + public void queryPageShouldAuditAccountAndMaskedFilters() + throws Exception { + BigInteger sourceId = BigInteger.valueOf(5201L); + BigInteger tableId = BigInteger.valueOf(5202L); + DatacenterSource source = new DatacenterSource(); + source.setId(sourceId); + source.setTenantId(BigInteger.ONE); + source.setSourceType("PROJECT_MYSQL"); + DatacenterTable table = new DatacenterTable(); + table.setId(tableId); + table.setTenantId(BigInteger.ONE); + table.setSourceId(sourceId); + table.setTableName("orders"); + DatacenterTableField idField = queryableField("id"); + idField.setSortable(1); + table.setFields(List.of(idField)); + + DatacenterDatasetRegistryService registry = Mockito.mock( + DatacenterDatasetRegistryService.class); + Mockito.when(registry.getTableWithFields(tableId)).thenReturn(table); + Mockito.when(registry.getSourceRequired(sourceId)).thenReturn(source); + DatacenterConnector connector = Mockito.mock(DatacenterConnector.class); + Mockito.when(connector.queryPage( + Mockito.eq(source), Mockito.eq(table), Mockito.any())) + .thenReturn(new Page<>(List.of(row(1)), 1L, 20L, 1L)); + DatacenterConnectorRegistry connectors = Mockito.mock( + DatacenterConnectorRegistry.class); + Mockito.when(connectors.getConnector("PROJECT_MYSQL")) + .thenReturn(connector); + DatacenterQueryAuditService auditService = Mockito.mock( + DatacenterQueryAuditService.class); + + DatacenterDatasetQueryServiceImpl service = + new DatacenterDatasetQueryServiceImpl(); + setField(service, "registryService", registry); + setField(service, "connectorRegistry", connectors); + setField(service, "queryAuditService", auditService); + DatacenterQueryRequest request = new DatacenterQueryRequest(); + DatasetRef datasetRef = new DatasetRef(); + datasetRef.setTenantId(BigInteger.ONE); + datasetRef.setTableId(tableId); + request.setDatasetRef(datasetRef); + request.setPageNumber(1L); + request.setPageSize(20L); + request.setSelectedColumns(List.of("id")); + DatacenterQueryFilter filter = new DatacenterQueryFilter(); + filter.setColumn("id"); + filter.setOperator("EQ"); + filter.setValue("sensitive-value"); + request.setFilters(List.of(filter)); + DatacenterQuerySort sort = new DatacenterQuerySort(); + sort.setColumn("id"); + sort.setDirection("DESC"); + request.setSorts(List.of(sort)); + LoginAccount account = new LoginAccount(); + account.setId(BigInteger.valueOf(9001L)); + account.setTenantId(BigInteger.ONE); + + service.queryPage(request, account); + + ArgumentCaptor parameters = ArgumentCaptor.forClass(Map.class); + Mockito.verify(auditService).start( + Mockito.anyString(), Mockito.eq(source), + Mockito.contains("WHERE id EQ ?"), parameters.capture(), + Mockito.same(account), Mockito.eq("DATASET_PAGE"), + Mockito.eq(tableId.toString())); + Assert.assertFalse(parameters.getValue().toString() + .contains("sensitive-value")); + Assert.assertTrue(parameters.getValue().toString() + .contains("valueCount=1")); + } + + /** + * 验证后台结构化查询不能凭其他租户的表 ID 越权读取。 + */ + @Test(expected = BusinessException.class) + public void queryPageShouldRejectCrossTenantTable() throws Exception { + BigInteger sourceId = BigInteger.valueOf(5301L); + BigInteger tableId = BigInteger.valueOf(5302L); + DatacenterSource source = new DatacenterSource(); + source.setId(sourceId); + source.setTenantId(BigInteger.TWO); + source.setSourceType("PROJECT_MYSQL"); + DatacenterTable table = new DatacenterTable(); + table.setId(tableId); + table.setTenantId(BigInteger.TWO); + table.setSourceId(sourceId); + table.setTableName("orders"); + table.setFields(List.of(queryableField("id"))); + + DatacenterDatasetRegistryService registry = Mockito.mock( + DatacenterDatasetRegistryService.class); + Mockito.when(registry.getTableWithFields(tableId)).thenReturn(table); + Mockito.when(registry.getSourceRequired(sourceId)).thenReturn(source); + DatacenterDatasetQueryServiceImpl service = + new DatacenterDatasetQueryServiceImpl(); + setField(service, "registryService", registry); + DatacenterQueryRequest request = new DatacenterQueryRequest(); + DatasetRef datasetRef = new DatasetRef(); + datasetRef.setTenantId(BigInteger.ONE); + datasetRef.setTableId(tableId); + request.setDatasetRef(datasetRef); + LoginAccount account = new LoginAccount(); + account.setId(BigInteger.ONE); + account.setTenantId(BigInteger.ONE); + + service.queryPage(request, account); + } + + /** + * 验证内部与外部结构化查询统一限制单页行数。 + */ + @Test(expected = BusinessException.class) + public void queryPageShouldRejectOversizedPage() { + DatacenterDatasetQueryServiceImpl service = + new DatacenterDatasetQueryServiceImpl(); + DatacenterQueryRequest request = new DatacenterQueryRequest(); + request.setDatasetRef(new DatasetRef()); + request.setPageNumber(1L); + request.setPageSize(501L); + + service.queryPage(request); + } + /** * 创建测试数据行。 * @@ -187,6 +532,22 @@ public class DatacenterDatasetQueryServiceImplTest { return row; } + /** + * 创建可查询的公开字段。 + * + * @param name 字段名称 + * @return 字段定义 + */ + private DatacenterTableField queryableField(String name) { + DatacenterTableField field = new DatacenterTableField(); + field.setFieldName(name); + field.setSourceColumnName(name); + field.setQueryable(1); + field.setMetadataStatus(DatacenterMetadataStatus.ACTIVE.name()); + field.setSensitivityLevel(DatacenterSensitivityLevel.PUBLIC.name()); + return field; + } + /** * 注入被测服务依赖。 * diff --git a/easyflow-modules/easyflow-module-datacenter/src/test/java/tech/easyflow/datacenter/federation/DatacenterFederationAdmissionControllerTest.java b/easyflow-modules/easyflow-module-datacenter/src/test/java/tech/easyflow/datacenter/federation/DatacenterFederationAdmissionControllerTest.java new file mode 100644 index 00000000..7a6beb0c --- /dev/null +++ b/easyflow-modules/easyflow-module-datacenter/src/test/java/tech/easyflow/datacenter/federation/DatacenterFederationAdmissionControllerTest.java @@ -0,0 +1,238 @@ +package tech.easyflow.datacenter.federation; + +import com.easyagents.federation.sql.api.FederationSqlErrorCode; +import com.easyagents.federation.sql.api.FederationSqlException; +import com.easyagents.federation.sql.execute.FederationQueryPermit; +import com.easyagents.federation.sql.execute.QueryId; +import com.easyagents.federation.sql.source.SourceId; +import java.time.Duration; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import org.junit.Test; + +/** + * {@link DatacenterFederationAdmissionController} 分层隔离测试。 + */ +public class DatacenterFederationAdmissionControllerTest { + + /** + * 验证已结束查询不会永久保留历史租户和数据源准入项。 + */ + @Test + public void completedQueriesShouldReleaseAdmissionEntries() { + DatacenterFederationAdmissionController controller = + new DatacenterFederationAdmissionController(8, 4, 2); + try { + for (int index = 0; index < 100; index++) { + SourceId sourceId = new SourceId( + "tenant-" + index + "-source-" + index); + try (FederationQueryPermit ignored = controller.acquire( + sourceId, + new QueryId("query-" + index), + Duration.ZERO)) { + org.junit.Assert.assertEquals( + 1, controller.trackedSourceAdmissionCount()); + org.junit.Assert.assertEquals( + 1, controller.trackedTenantAdmissionCount()); + } + org.junit.Assert.assertEquals( + 0, controller.trackedSourceAdmissionCount()); + org.junit.Assert.assertEquals( + 0, controller.trackedTenantAdmissionCount()); + } + } finally { + controller.close(); + } + } + + /** + * 验证热点数据源耗尽自身配额后不会占用其他数据源的全局许可。 + */ + @Test + public void hotSourceShouldNotStarveAnotherSource() { + DatacenterFederationAdmissionController controller = + new DatacenterFederationAdmissionController(2, 2, 1); + SourceId firstSource = new SourceId("tenant-1-source-1"); + SourceId secondSource = new SourceId("tenant-1-source-2"); + try (FederationQueryPermit ignored = controller.acquire( + firstSource, new QueryId("query-1"), Duration.ZERO)) { + try { + controller.acquire( + firstSource, + new QueryId("query-2"), + Duration.ZERO); + org.junit.Assert.fail("source limit should reject query"); + } catch (FederationSqlException expected) { + // 单源配额按预期先于租户和全局配额生效。 + } + try (FederationQueryPermit second = controller.acquire( + secondSource, + new QueryId("query-3"), + Duration.ZERO)) { + org.junit.Assert.assertNotNull(second); + } + } finally { + controller.close(); + } + } + + /** + * 验证活动查询释放后等待者取得许可，最终清理全部准入项。 + * + * @throws Exception 并发执行失败 + */ + @Test + public void waitingQueryShouldAcquireAndReleaseWithoutLeakingEntries() + throws Exception { + DatacenterFederationAdmissionController controller = + new DatacenterFederationAdmissionController(2, 2, 1); + SourceId sourceId = new SourceId("tenant-1-source-1"); + ExecutorService executor = Executors.newSingleThreadExecutor(); + FederationQueryPermit active = controller.acquire( + sourceId, new QueryId("active"), Duration.ZERO); + try { + Future waiting = executor.submit(() -> + controller.acquire( + sourceId, + new QueryId("waiting"), + Duration.ofSeconds(2))); + awaitSourceReferences(controller, sourceId, 2); + active.close(); + try (FederationQueryPermit acquired = waiting.get( + 2, TimeUnit.SECONDS)) { + org.junit.Assert.assertNotNull(acquired); + } + org.junit.Assert.assertEquals( + 0, controller.trackedSourceAdmissionCount()); + org.junit.Assert.assertEquals( + 0, controller.trackedTenantAdmissionCount()); + } finally { + active.close(); + executor.shutdownNow(); + controller.close(); + } + } + + /** + * 验证等待者取消与 Controller 提前关闭均会正确归还引用。 + * + * @throws Exception 并发执行失败 + */ + @Test + public void cancelledAndClosedWaitersShouldReleaseReferences() + throws Exception { + DatacenterFederationAdmissionController controller = + new DatacenterFederationAdmissionController(2, 2, 1); + SourceId sourceId = new SourceId("tenant-1-source-1"); + ExecutorService executor = Executors.newSingleThreadExecutor(); + FederationQueryPermit active = controller.acquire( + sourceId, new QueryId("active"), Duration.ZERO); + AtomicBoolean cancellationRequested = new AtomicBoolean(); + try { + Future cancelled = executor.submit(() -> + controller.acquire( + sourceId, + new QueryId("cancelled"), + Duration.ofSeconds(2), + cancellationRequested::get)); + awaitSourceReferences(controller, sourceId, 2); + cancellationRequested.set(true); + assertFailureCode( + cancelled, + FederationSqlErrorCode.QUERY_CANCELLED); + org.junit.Assert.assertEquals( + 1, controller.trackedSourceReferenceCount(sourceId)); + + Future closing = executor.submit(() -> + controller.acquire( + sourceId, + new QueryId("closing"), + Duration.ofSeconds(2))); + awaitSourceReferences(controller, sourceId, 2); + controller.close(); + assertFailureCode( + closing, + FederationSqlErrorCode.ENGINE_CLOSED); + active.close(); + active.close(); + org.junit.Assert.assertEquals( + 0, controller.trackedSourceAdmissionCount()); + org.junit.Assert.assertEquals( + 0, controller.trackedTenantAdmissionCount()); + } finally { + active.close(); + executor.shutdownNow(); + controller.close(); + } + } + + /** + * 验证关闭完成后的新查询仍返回统一的引擎关闭错误码。 + */ + @Test + public void acquireAfterCloseShouldReportEngineClosed() { + DatacenterFederationAdmissionController controller = + new DatacenterFederationAdmissionController(2, 2, 1); + controller.close(); + try { + controller.acquire( + new SourceId("tenant-1-source-1"), + new QueryId("closed"), + Duration.ZERO); + org.junit.Assert.fail("closed admission should reject query"); + } catch (FederationSqlException exception) { + org.junit.Assert.assertEquals( + FederationSqlErrorCode.ENGINE_CLOSED, + exception.errorCode()); + } + } + + /** + * 等待指定数据源达到预期引用数。 + * + * @param controller 准入控制器 + * @param sourceId 数据源标识 + * @param expected 预期引用数 + * @throws InterruptedException 等待被中断 + */ + private static void awaitSourceReferences( + DatacenterFederationAdmissionController controller, + SourceId sourceId, + int expected) throws InterruptedException { + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(2); + while (System.nanoTime() < deadline) { + if (controller.trackedSourceReferenceCount(sourceId) == expected) { + return; + } + Thread.sleep(5L); + } + org.junit.Assert.fail( + "source admission references did not reach " + expected); + } + + /** + * 断言异步准入以指定错误码失败。 + * + * @param future 异步准入结果 + * @param errorCode 预期错误码 + * @throws Exception 获取异步结果失败 + */ + private static void assertFailureCode( + Future future, + FederationSqlErrorCode errorCode) + throws Exception { + try { + future.get(2, TimeUnit.SECONDS); + org.junit.Assert.fail("query admission should fail"); + } catch (ExecutionException exception) { + Throwable cause = exception.getCause(); + org.junit.Assert.assertTrue(cause instanceof FederationSqlException); + org.junit.Assert.assertEquals( + errorCode, ((FederationSqlException) cause).errorCode()); + } + } +} diff --git a/easyflow-modules/easyflow-module-datacenter/src/test/java/tech/easyflow/datacenter/federation/DatacenterFederationDataSourceResolverTest.java b/easyflow-modules/easyflow-module-datacenter/src/test/java/tech/easyflow/datacenter/federation/DatacenterFederationDataSourceResolverTest.java new file mode 100644 index 00000000..1c315eb4 --- /dev/null +++ b/easyflow-modules/easyflow-module-datacenter/src/test/java/tech/easyflow/datacenter/federation/DatacenterFederationDataSourceResolverTest.java @@ -0,0 +1,191 @@ +package tech.easyflow.datacenter.federation; + +import com.easyagents.federation.sql.source.FederationDataSourceHandle; +import com.easyagents.federation.sql.source.FederationSourceDefinition; +import com.zaxxer.hikari.HikariDataSource; +import java.math.BigInteger; +import java.sql.Connection; +import java.sql.DatabaseMetaData; +import java.util.List; +import java.util.Map; +import org.junit.Assert; +import org.junit.Test; +import org.mockito.Mockito; +import tech.easyflow.datacenter.mapper.DatacenterCatalogMapper; +import tech.easyflow.datacenter.mapper.DatacenterSourceMapper; +import tech.easyflow.datacenter.meta.entity.DatacenterSource; +import tech.easyflow.datacenter.meta.enums.DatacenterSourceStatus; +import tech.easyflow.datacenter.meta.enums.DatacenterSourceType; + +/** + * {@link DatacenterFederationDataSourceResolver} 状态与 checksum 契约测试。 + */ +public class DatacenterFederationDataSourceResolverTest { + + /** + * 验证未发布 checksum 的草稿可以创建仅供 probe 使用的短生命周期句柄。 + * + * @throws Exception JDBC mock 初始化失败 + */ + @Test + public void shouldResolveDraftForProbeWithoutPublishedChecksum() throws Exception { + DatacenterSource source = source(DatacenterSourceStatus.DRAFT); + DatacenterCatalogMapper catalogMapper = Mockito.mock(DatacenterCatalogMapper.class); + Mockito.when(catalogMapper.selectListByQuery(Mockito.any())).thenReturn(List.of()); + DatacenterFederationDefinitionFactory definitionFactory = + new DatacenterFederationDefinitionFactory(catalogMapper); + FederationSourceDefinition definition = definitionFactory.create(source); + + DatacenterSourceMapper sourceMapper = Mockito.mock(DatacenterSourceMapper.class); + Mockito.when(sourceMapper.selectOneById(source.getId())).thenReturn(source); + DatacenterFederationDataSourceFactory dataSourceFactory = + Mockito.mock(DatacenterFederationDataSourceFactory.class); + HikariDataSource pool = Mockito.mock(HikariDataSource.class); + Connection connection = Mockito.mock(Connection.class); + DatabaseMetaData metadata = Mockito.mock(DatabaseMetaData.class); + Mockito.when(dataSourceFactory.create(source)).thenReturn(pool); + Mockito.when(pool.getConnection()).thenReturn(connection); + Mockito.when(connection.getMetaData()).thenReturn(metadata); + Mockito.when(metadata.getDatabaseProductName()).thenReturn("MySQL"); + Mockito.when(metadata.getDatabaseProductVersion()).thenReturn("8.0"); + Mockito.when(metadata.getDriverName()).thenReturn("MySQL Connector/J"); + Mockito.when(metadata.getDriverVersion()).thenReturn("9.0"); + + DatacenterFederationDataSourceResolver resolver = new DatacenterFederationDataSourceResolver( + sourceMapper, definitionFactory, dataSourceFactory); + try (FederationDataSourceHandle handle = resolver.resolve(definition)) { + Assert.assertSame(pool, handle.dataSource()); + } + Mockito.verify(pool).close(); + } + + /** + * 验证活动 Runtime 必须匹配已持久化的发布 checksum。 + */ + @Test + public void shouldRejectActiveSourceWithoutPublishedChecksum() { + DatacenterSource source = source(DatacenterSourceStatus.READY); + DatacenterCatalogMapper catalogMapper = Mockito.mock(DatacenterCatalogMapper.class); + Mockito.when(catalogMapper.selectListByQuery(Mockito.any())).thenReturn(List.of()); + DatacenterFederationDefinitionFactory definitionFactory = + new DatacenterFederationDefinitionFactory(catalogMapper); + FederationSourceDefinition definition = definitionFactory.create(source); + DatacenterSourceMapper sourceMapper = Mockito.mock(DatacenterSourceMapper.class); + Mockito.when(sourceMapper.selectOneById(source.getId())).thenReturn(source); + + DatacenterFederationDataSourceResolver resolver = new DatacenterFederationDataSourceResolver( + sourceMapper, definitionFactory, Mockito.mock(DatacenterFederationDataSourceFactory.class)); + try { + resolver.resolve(definition); + Assert.fail("active source without persisted checksum must be rejected"); + } catch (IllegalStateException expected) { + Assert.assertTrue(expected.getMessage().contains("published")); + } + } + + /** + * 验证候选配置只在同步操作内生效，结束后恢复数据库权威解析。 + * + * @throws Exception JDBC mock 初始化失败 + */ + @Test + public void candidateResolutionShouldBeThreadBoundAndAlwaysCleared() throws Exception { + DatacenterSource persisted = source(DatacenterSourceStatus.READY); + DatacenterSource candidate = source(DatacenterSourceStatus.DRAFT); + candidate.setDefinitionRevision(2L); + candidate.setDatabaseName("candidate_db"); + DatacenterCatalogMapper catalogMapper = Mockito.mock(DatacenterCatalogMapper.class); + Mockito.when(catalogMapper.selectListByQuery(Mockito.any())).thenReturn(List.of()); + DatacenterFederationDefinitionFactory definitionFactory = + new DatacenterFederationDefinitionFactory(catalogMapper); + FederationSourceDefinition definition = definitionFactory.createCandidate(candidate); + DatacenterSourceMapper sourceMapper = Mockito.mock(DatacenterSourceMapper.class); + Mockito.when(sourceMapper.selectOneById(candidate.getId())).thenReturn(persisted); + DatacenterFederationDataSourceFactory dataSourceFactory = + Mockito.mock(DatacenterFederationDataSourceFactory.class); + HikariDataSource pool = Mockito.mock(HikariDataSource.class); + Connection connection = Mockito.mock(Connection.class); + DatabaseMetaData metadata = Mockito.mock(DatabaseMetaData.class); + Mockito.when(dataSourceFactory.create(candidate)).thenReturn(pool); + Mockito.when(pool.getConnection()).thenReturn(connection); + Mockito.when(connection.getMetaData()).thenReturn(metadata); + Mockito.when(metadata.getDatabaseProductName()).thenReturn("MySQL"); + Mockito.when(metadata.getDatabaseProductVersion()).thenReturn("8.0"); + Mockito.when(metadata.getDriverName()).thenReturn("MySQL Connector/J"); + Mockito.when(metadata.getDriverVersion()).thenReturn("9.0"); + DatacenterFederationDataSourceResolver resolver = + new DatacenterFederationDataSourceResolver( + sourceMapper, definitionFactory, dataSourceFactory); + + FederationDataSourceHandle handle = resolver.withCandidate( + candidate, definition, () -> resolver.resolve(definition)); + handle.close(); + Mockito.verify(sourceMapper, Mockito.never()).selectOneById(candidate.getId()); + + try { + resolver.resolve(definition); + Assert.fail("candidate context must be cleared after the action"); + } catch (IllegalStateException expected) { + Assert.assertTrue(expected.getMessage().contains("revision")); + } + Mockito.verify(sourceMapper).selectOneById(candidate.getId()); + } + + /** + * 候选操作抛错时也必须清除线程上下文，避免后续请求复用错误凭据。 + */ + @Test + public void candidateResolutionShouldClearContextAfterFailure() { + DatacenterSource persisted = source(DatacenterSourceStatus.READY); + DatacenterSource candidate = source(DatacenterSourceStatus.DRAFT); + candidate.setDefinitionRevision(2L); + DatacenterCatalogMapper catalogMapper = Mockito.mock(DatacenterCatalogMapper.class); + Mockito.when(catalogMapper.selectListByQuery(Mockito.any())).thenReturn(List.of()); + DatacenterFederationDefinitionFactory definitionFactory = + new DatacenterFederationDefinitionFactory(catalogMapper); + FederationSourceDefinition definition = definitionFactory.createCandidate(candidate); + DatacenterSourceMapper sourceMapper = Mockito.mock(DatacenterSourceMapper.class); + Mockito.when(sourceMapper.selectOneById(candidate.getId())).thenReturn(persisted); + DatacenterFederationDataSourceResolver resolver = + new DatacenterFederationDataSourceResolver( + sourceMapper, + definitionFactory, + Mockito.mock(DatacenterFederationDataSourceFactory.class)); + + try { + resolver.withCandidate(candidate, definition, () -> { + throw new IllegalStateException("simulated candidate failure"); + }); + Assert.fail("candidate action must fail"); + } catch (IllegalStateException expected) { + Assert.assertEquals("simulated candidate failure", expected.getMessage()); + } + + try { + resolver.resolve(definition); + Assert.fail("resolver must return to persisted source resolution"); + } catch (IllegalStateException expected) { + Assert.assertTrue(expected.getMessage().contains("revision")); + } + Mockito.verify(sourceMapper).selectOneById(candidate.getId()); + } + + /** + * 创建满足 Definition 最小字段要求的测试数据源。 + * + * @param status 数据源状态 + * @return 测试数据源 + */ + private DatacenterSource source(DatacenterSourceStatus status) { + DatacenterSource source = new DatacenterSource(); + source.setId(BigInteger.valueOf(101L)); + source.setTenantId(BigInteger.valueOf(7L)); + source.setSourceType(DatacenterSourceType.MYSQL.name()); + source.setDatabaseName("test_db"); + source.setAdapterId("jdbc"); + source.setAdapterOptionsJson(Map.of()); + source.setDefinitionRevision(1L); + source.setStatus(status.code()); + return source; + } +} diff --git a/easyflow-modules/easyflow-module-datacenter/src/test/java/tech/easyflow/datacenter/federation/DatacenterFederationQueryCancellationServiceTest.java b/easyflow-modules/easyflow-module-datacenter/src/test/java/tech/easyflow/datacenter/federation/DatacenterFederationQueryCancellationServiceTest.java new file mode 100644 index 00000000..464cb7b5 --- /dev/null +++ b/easyflow-modules/easyflow-module-datacenter/src/test/java/tech/easyflow/datacenter/federation/DatacenterFederationQueryCancellationServiceTest.java @@ -0,0 +1,184 @@ +package tech.easyflow.datacenter.federation; + +import com.easyagents.federation.sql.api.FederationSqlEngine; +import com.easyagents.federation.sql.execute.QueryId; +import java.math.BigInteger; +import java.time.Duration; +import java.util.UUID; +import org.junit.Assert; +import org.junit.Test; +import org.mockito.Mockito; +import org.springframework.beans.factory.ObjectProvider; +import org.springframework.data.redis.core.StringRedisTemplate; +import org.springframework.data.redis.core.ValueOperations; +import tech.easyflow.common.entity.LoginAccount; +import tech.easyflow.common.web.exceptions.BusinessException; + +/** + * {@link DatacenterFederationQueryCancellationService} 租户隔离与登记回归测试。 + */ +public class DatacenterFederationQueryCancellationServiceTest { + + /** + * 验证其他租户无法取消查询，所属租户可以取消并在关闭登记后释放状态。 + */ + @Test + public void cancelShouldBeTenantScopedAndReleaseRegistration() { + FederationSqlEngine engine = Mockito.mock(FederationSqlEngine.class); + DatacenterFederationQueryCancellationService service = service(engine); + QueryId queryId = new QueryId(UUID.randomUUID().toString()); + + try (DatacenterFederationQueryCancellationService.QueryRegistration ignored = + service.register(queryId, account(1L))) { + Assert.assertFalse(service.cancel(queryId.value(), account(2L))); + Mockito.verify(engine, Mockito.never()).cancel(queryId); + + Assert.assertTrue(service.cancel(queryId.value(), account(1L))); + Assert.assertTrue(service.isCancelled(queryId)); + Mockito.verify(engine).cancel(queryId); + } + + Assert.assertFalse(service.isCancelled(queryId)); + } + + /** + * 验证活动 QueryId 不能被并发查询复用。 + */ + @Test(expected = BusinessException.class) + public void registerShouldRejectDuplicateActiveQueryId() { + DatacenterFederationQueryCancellationService service = service( + Mockito.mock(FederationSqlEngine.class)); + QueryId queryId = new QueryId(UUID.randomUUID().toString()); + try (DatacenterFederationQueryCancellationService.QueryRegistration ignored = + service.register(queryId, account(1L))) { + service.register(queryId, account(1L)); + } + } + + /** + * 验证先到达 Redis 的取消请求会在查询登记时立即生效。 + */ + @Test + public void registerShouldObservePersistedCancellation() { + FederationSqlEngine engine = Mockito.mock(FederationSqlEngine.class); + StringRedisTemplate redis = Mockito.mock(StringRedisTemplate.class); + Mockito.when(redis.hasKey(Mockito.anyString())).thenReturn(true); + DatacenterFederationQueryCancellationService service = service(engine, redis); + QueryId queryId = new QueryId(UUID.randomUUID().toString()); + + try (DatacenterFederationQueryCancellationService.QueryRegistration ignored = + service.register(queryId, account(1L))) { + Assert.assertTrue(service.isCancelled(queryId)); + Mockito.verify(engine).cancel(queryId); + } + } + + /** + * 验证活动查询在丢失 Pub/Sub 提示后仍会通过 Redis 终态核对被取消。 + */ + @Test + public void reconcileShouldObserveCancellationCreatedAfterRegister() { + FederationSqlEngine engine = Mockito.mock(FederationSqlEngine.class); + StringRedisTemplate redis = Mockito.mock(StringRedisTemplate.class); + Mockito.when(redis.hasKey(Mockito.anyString())) + .thenReturn(false, true); + DatacenterFederationQueryCancellationService service = service(engine, redis); + QueryId queryId = new QueryId(UUID.randomUUID().toString()); + + try (DatacenterFederationQueryCancellationService.QueryRegistration ignored = + service.register(queryId, account(1L))) { + Assert.assertFalse(service.isCancelled(queryId)); + + service.reconcilePersistedCancellations(); + + Assert.assertTrue(service.isCancelled(queryId)); + Mockito.verify(engine).cancel(queryId); + } + } + + /** + * 验证取消请求先持久化终态，再发布跨节点通知。 + */ + @Test + @SuppressWarnings("unchecked") + public void cancelShouldPersistCancellationBeforePublishing() { + FederationSqlEngine engine = Mockito.mock(FederationSqlEngine.class); + StringRedisTemplate redis = Mockito.mock(StringRedisTemplate.class); + ValueOperations values = Mockito.mock(ValueOperations.class); + Mockito.when(redis.opsForValue()).thenReturn(values); + Mockito.when(redis.convertAndSend( + Mockito.eq(DatacenterFederationQueryCancellationService.CHANNEL), + Mockito.anyString())).thenReturn(1L); + DatacenterFederationQueryCancellationService service = service(engine, redis); + QueryId queryId = new QueryId(UUID.randomUUID().toString()); + + Assert.assertTrue(service.cancel(queryId.value(), account(1L))); + + org.mockito.InOrder order = Mockito.inOrder(values, redis); + order.verify(values).set( + Mockito.contains(queryId.value()), Mockito.eq("1"), + Mockito.eq(Duration.ofMinutes(10))); + order.verify(redis).convertAndSend( + Mockito.eq(DatacenterFederationQueryCancellationService.CHANNEL), + Mockito.contains(queryId.value())); + } + + /** + * 验证本地驱动取消失败时仍保留分布式终态，并由周期核对重试本地取消。 + */ + @Test + @SuppressWarnings("unchecked") + public void cancelShouldPersistAndRetryAfterLocalFailure() { + FederationSqlEngine engine = Mockito.mock(FederationSqlEngine.class); + StringRedisTemplate redis = Mockito.mock(StringRedisTemplate.class); + ValueOperations values = Mockito.mock(ValueOperations.class); + Mockito.when(redis.opsForValue()).thenReturn(values); + Mockito.when(redis.convertAndSend( + Mockito.eq(DatacenterFederationQueryCancellationService.CHANNEL), + Mockito.anyString())).thenThrow( + new IllegalStateException("simulated publish failure")); + DatacenterFederationQueryCancellationService service = service(engine, redis); + QueryId queryId = new QueryId(UUID.randomUUID().toString()); + Mockito.doThrow(new IllegalStateException("simulated driver failure")) + .doReturn(true) + .when(engine).cancel(queryId); + + try (DatacenterFederationQueryCancellationService.QueryRegistration ignored = + service.register(queryId, account(1L))) { + Assert.assertTrue(service.cancel(queryId.value(), account(1L))); + Assert.assertTrue(service.isCancelled(queryId)); + service.reconcilePersistedCancellations(); + } + + Mockito.verify(values).set( + Mockito.contains(queryId.value()), Mockito.eq("1"), + Mockito.eq(Duration.ofMinutes(10))); + Mockito.verify(engine, Mockito.times(2)).cancel(queryId); + } + + @SuppressWarnings("unchecked") + private DatacenterFederationQueryCancellationService service( + FederationSqlEngine engine) { + return service(engine, null); + } + + @SuppressWarnings("unchecked") + private DatacenterFederationQueryCancellationService service( + FederationSqlEngine engine, + StringRedisTemplate redis) { + DatacenterFederationRuntime runtime = Mockito.mock( + DatacenterFederationRuntime.class); + Mockito.when(runtime.engine()).thenReturn(engine); + ObjectProvider redisProvider = Mockito.mock( + ObjectProvider.class); + Mockito.when(redisProvider.getIfAvailable()).thenReturn(redis); + return new DatacenterFederationQueryCancellationService( + runtime, redisProvider); + } + + private LoginAccount account(long tenantId) { + LoginAccount account = new LoginAccount(); + account.setTenantId(BigInteger.valueOf(tenantId)); + return account; + } +} diff --git a/easyflow-modules/easyflow-module-datacenter/src/test/java/tech/easyflow/datacenter/federation/DatacenterFederationQueryServiceTest.java b/easyflow-modules/easyflow-module-datacenter/src/test/java/tech/easyflow/datacenter/federation/DatacenterFederationQueryServiceTest.java new file mode 100644 index 00000000..ba1e2587 --- /dev/null +++ b/easyflow-modules/easyflow-module-datacenter/src/test/java/tech/easyflow/datacenter/federation/DatacenterFederationQueryServiceTest.java @@ -0,0 +1,109 @@ +package tech.easyflow.datacenter.federation; + +import com.easyagents.federation.sql.api.FederationSqlEngine; +import com.easyagents.federation.sql.api.SqlExecutionContext; +import com.easyagents.federation.sql.compile.FederationSqlPlan; +import com.easyagents.federation.sql.compile.SqlCompileRequest; +import com.easyagents.federation.sql.execute.FederationResultCursor; +import com.easyagents.federation.sql.source.SourceId; +import java.math.BigInteger; +import java.util.List; +import java.util.UUID; +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.RelRoot; +import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.sql.SqlKind; +import org.apache.calcite.sql.parser.SqlParser; +import org.junit.Assert; +import org.junit.Test; +import org.mockito.ArgumentCaptor; +import org.mockito.Mockito; +import org.springframework.beans.factory.ObjectProvider; +import org.springframework.data.redis.core.StringRedisTemplate; +import tech.easyflow.datacenter.audit.DatacenterQueryAuditService; +import tech.easyflow.datacenter.meta.entity.DatacenterSource; +import tech.easyflow.datacenter.meta.enums.DatacenterSourceStatus; + +/** + * {@link DatacenterFederationQueryService} 流式查询取消生命周期回归测试。 + */ +public class DatacenterFederationQueryServiceTest { + + /** + * 验证调用方 QueryId 会进入执行上下文，完成后释放登记并允许安全复用。 + * + * @throws Exception SQL 解析失败 + */ + @Test + @SuppressWarnings("unchecked") + public void consumeShouldUseRequestedQueryIdAndReleaseRegistration() + throws Exception { + DatacenterFederationRuntime runtime = Mockito.mock( + DatacenterFederationRuntime.class); + FederationSqlEngine engine = Mockito.mock(FederationSqlEngine.class); + DatacenterFederationDefinitionFactory definitions = Mockito.mock( + DatacenterFederationDefinitionFactory.class); + Mockito.when(runtime.engine()).thenReturn(engine); + Mockito.when(runtime.definitions()).thenReturn(definitions); + SourceId sourceId = new SourceId("tenant-1-source-1001"); + + DatacenterSource source = new DatacenterSource(); + source.setId(BigInteger.valueOf(1001L)); + source.setStatus(DatacenterSourceStatus.READY.code()); + source.setDefinitionRevision(1L); + source.setScopeRevision(1L); + Mockito.when(definitions.sourceId(source)).thenReturn(sourceId); + + RelDataType rowType = Mockito.mock(RelDataType.class); + RelNode root = Mockito.mock(RelNode.class); + Mockito.when(root.getRowType()).thenReturn(rowType); + RelRoot relRoot = RelRoot.of(root, rowType, SqlKind.SELECT); + FederationSqlPlan plan = Mockito.mock(FederationSqlPlan.class); + Mockito.when(plan.relRoot()).thenReturn(relRoot); + Mockito.when(plan.sqlNode()).thenReturn( + SqlParser.create("SELECT 1").parseQuery()); + Mockito.when(engine.compile(Mockito.any())).thenReturn(plan); + FederationResultCursor cursor = Mockito.mock( + FederationResultCursor.class); + Mockito.when(cursor.columns()).thenReturn(List.of()); + Mockito.when(cursor.next()).thenReturn(false); + Mockito.when(engine.execute(Mockito.eq(plan), Mockito.any())) + .thenReturn(cursor); + + ObjectProvider redisProvider = Mockito.mock( + ObjectProvider.class); + DatacenterFederationQueryCancellationService cancellationService = + new DatacenterFederationQueryCancellationService( + runtime, redisProvider); + DatacenterFederationQueryService service = + new DatacenterFederationQueryService( + runtime, + Mockito.mock(DatacenterQueryAuditService.class), + cancellationService); + String requestedQueryId = UUID.randomUUID().toString(); + + Assert.assertEquals(0L, service.consume( + source, "SELECT 1", List.of(), 10, 100, 1_024, + null, "TEST", "stream-1", requestedQueryId, + ignored -> { })); + Assert.assertEquals(0L, service.consume( + source, "SELECT 1", List.of(), 10, 100, 1_024, + null, "TEST", "stream-2", requestedQueryId, + ignored -> { })); + + ArgumentCaptor contexts = + ArgumentCaptor.forClass(SqlExecutionContext.class); + Mockito.verify(engine, Mockito.times(2)).execute( + Mockito.eq(plan), contexts.capture()); + Assert.assertTrue(contexts.getAllValues().stream().allMatch( + context -> requestedQueryId.equals(context.queryId().value()))); + ArgumentCaptor compileRequests = + ArgumentCaptor.forClass(SqlCompileRequest.class); + Mockito.verify(engine, Mockito.times(2)).compile(compileRequests.capture()); + Assert.assertTrue(compileRequests.getAllValues().stream().allMatch(request -> + request.queryScope().revision() == source.getScopeRevision() + && request.minimumRevision() == source.getDefinitionRevision() + && sourceId.equals(request.sourceId()))); + Mockito.verify(cursor, Mockito.times(2)).close(); + } +} diff --git a/easyflow-modules/easyflow-module-datacenter/src/test/java/tech/easyflow/datacenter/federation/DatacenterFederationSqlPolicyTest.java b/easyflow-modules/easyflow-module-datacenter/src/test/java/tech/easyflow/datacenter/federation/DatacenterFederationSqlPolicyTest.java new file mode 100644 index 00000000..3dc04e88 --- /dev/null +++ b/easyflow-modules/easyflow-module-datacenter/src/test/java/tech/easyflow/datacenter/federation/DatacenterFederationSqlPolicyTest.java @@ -0,0 +1,286 @@ +package tech.easyflow.datacenter.federation; + +import com.easyagents.federation.sql.api.FederationSqlException; +import com.easyagents.federation.sql.compile.SqlCompileRequest; +import com.easyagents.federation.sql.compile.SqlPolicyContext; +import com.easyagents.federation.sql.federation.FederationExecutionPolicy; +import com.easyagents.federation.sql.federation.FederationQueryScopeDefinition; +import com.easyagents.federation.sql.federation.FederationSourceBindingDefinition; +import com.easyagents.federation.sql.source.SourceId; +import java.math.BigInteger; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Set; +import org.apache.calcite.plan.RelOptTable; +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.RelRoot; +import org.apache.calcite.rel.RelVisitor; +import org.apache.calcite.rel.core.TableScan; +import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.sql.SqlKind; +import org.apache.calcite.sql.SqlNode; +import org.apache.calcite.sql.parser.SqlParser; +import org.junit.Test; +import org.mockito.ArgumentMatchers; +import org.mockito.Mockito; +import tech.easyflow.datacenter.entity.DatacenterTable; +import tech.easyflow.datacenter.entity.DatacenterTableField; +import tech.easyflow.datacenter.mapper.DatacenterCatalogMapper; +import tech.easyflow.datacenter.mapper.DatacenterSourceMapper; +import tech.easyflow.datacenter.mapper.DatacenterTableFieldMapper; +import tech.easyflow.datacenter.mapper.DatacenterTableMapper; +import tech.easyflow.datacenter.meta.entity.DatacenterSource; +import tech.easyflow.datacenter.meta.enums.DatacenterMetadataStatus; +import tech.easyflow.datacenter.meta.enums.DatacenterSensitivityLevel; +import tech.easyflow.datacenter.meta.enums.DatacenterSourceStatus; + +/** + * {@link DatacenterFederationSqlPolicy} 字段白名单和别名回归测试。 + */ +public class DatacenterFederationSqlPolicyTest { + + /** + * 验证表别名、输出别名和 ORDER BY 别名不会被误判为物理字段。 + * + * @throws Exception SQL 解析失败 + */ + @Test + public void validateShouldAllowTableAndOutputAliases() throws Exception { + DatacenterFederationSqlPolicy policy = policy(); + + policy.validate(context( + "SELECT o.id AS order_id FROM orders AS o ORDER BY order_id")); + } + + /** + * 验证不可查询字段即使通过表别名限定也会被拒绝。 + * + * @throws Exception SQL 解析失败 + */ + @Test(expected = FederationSqlException.class) + public void validateShouldRejectHiddenQualifiedColumn() throws Exception { + policy().validate(context("SELECT o.secret FROM orders AS o")); + } + + /** + * 验证与表同名的字段不会因名称碰撞绕过字段白名单。 + * + * @throws Exception SQL 解析失败 + */ + @Test(expected = FederationSqlException.class) + public void validateShouldRejectHiddenColumnNamedLikeTable() throws Exception { + policy().validate(context("SELECT orders AS display_name FROM orders")); + } + + /** + * 验证输出别名与隐藏物理字段同名时不会绕过字段策略。 + * + * @throws Exception SQL 解析失败 + */ + @Test(expected = FederationSqlException.class) + public void validateShouldRejectHiddenColumnWithSameOutputAlias() + throws Exception { + policy().validate(context("SELECT secret AS secret FROM orders")); + } + + /** + * 验证内层子查询的表别名不会污染外层同名物理表限定符。 + * + * @throws Exception SQL 解析失败 + */ + @Test(expected = FederationSqlException.class) + public void validateShouldKeepNestedAliasScopesIndependent() + throws Exception { + policy().validate(context( + "SELECT orders.secret FROM orders " + + "WHERE EXISTS (SELECT 1 FROM public_orders AS orders)", + "orders", "public_orders")); + } + + /** + * 验证派生表的输出别名由内层字段策略校验后可在外层引用。 + * + * @throws Exception SQL 解析失败 + */ + @Test + public void validateShouldAllowDerivedTableOutputAlias() + throws Exception { + policy().validate(context( + "SELECT d.order_id FROM " + + "(SELECT id AS order_id FROM orders) AS d")); + } + + /** + * 验证 CTE 输出别名由内层字段策略校验后可在外层引用。 + * + * @throws Exception SQL 解析失败 + */ + @Test + public void validateShouldAllowCommonTableExpressionOutputAlias() + throws Exception { + policy().validate(context( + "WITH x AS (SELECT id AS order_id FROM orders) " + + "SELECT order_id FROM x")); + } + + /** + * 验证派生输出仍不能隐藏内层敏感字段引用。 + * + * @throws Exception SQL 解析失败 + */ + @Test(expected = FederationSqlException.class) + public void validateShouldRejectHiddenColumnInsideDerivedTable() + throws Exception { + policy().validate(context( + "SELECT d.secret_alias FROM " + + "(SELECT secret AS secret_alias FROM orders) AS d")); + } + + /** + * 验证数据库权威范围版本变化后，旧节点缓存不能继续放行查询。 + * + * @throws Exception SQL 解析失败 + */ + @Test(expected = FederationSqlException.class) + public void validateShouldRejectStaleScopeRevision() + throws Exception { + policy(2L).validate(context("SELECT id FROM orders")); + } + + /** + * 验证 XL01 尚未建立逐源授权模型时不会误开放底座联邦能力。 + * + * @throws Exception SQL 解析失败 + */ + @Test(expected = FederationSqlException.class) + public void validateShouldRejectMultiSourceScopeUntilProductAuthorizationExists() + throws Exception { + SqlPolicyContext single = context("SELECT id FROM orders"); + SourceId second = new SourceId("tenant-1-source-1002"); + FederationQueryScopeDefinition virtualScope = + FederationQueryScopeDefinition.virtual( + "not-yet-authorized", + 1, + Map.of( + "A", FederationSourceBindingDefinition.of( + single.request().sourceId(), 1), + "B", FederationSourceBindingDefinition.of(second, 1)), + "A", + FederationExecutionPolicy.basic()); + SqlCompileRequest request = new SqlCompileRequest( + single.request().sql(), virtualScope, List.of(), "scope-1"); + policy().validate(new SqlPolicyContext( + request, + single.validatedSql(), + single.relRoot(), + Set.of(second))); + } + + private DatacenterFederationSqlPolicy policy() { + return policy(1L); + } + + private DatacenterFederationSqlPolicy policy(long currentScopeRevision) { + BigInteger tableId = BigInteger.valueOf(2001L); + BigInteger publicTableId = BigInteger.valueOf(2002L); + DatacenterTable table = table(tableId, "orders"); + DatacenterTable publicTable = table(publicTableId, "public_orders"); + + DatacenterTableMapper tableMapper = Mockito.mock(DatacenterTableMapper.class); + DatacenterTableFieldMapper fieldMapper = Mockito.mock( + DatacenterTableFieldMapper.class); + DatacenterCatalogMapper catalogMapper = Mockito.mock(DatacenterCatalogMapper.class); + DatacenterSourceMapper sourceMapper = Mockito.mock( + DatacenterSourceMapper.class); + DatacenterSource source = new DatacenterSource(); + source.setId(BigInteger.valueOf(1001L)); + source.setStatus(DatacenterSourceStatus.READY.code()); + source.setScopeRevision(currentScopeRevision); + Mockito.when(sourceMapper.selectOneById(BigInteger.valueOf(1001L))) + .thenReturn(source); + Mockito.when(tableMapper.selectListByQuery(ArgumentMatchers.any())) + .thenReturn(List.of(table, publicTable)); + Mockito.when(fieldMapper.selectListByQuery(ArgumentMatchers.any())) + .thenReturn(List.of( + field(tableId, "id", true), + field(tableId, "secret", false), + field(tableId, "orders", false), + field(publicTableId, "id", true), + field(publicTableId, "secret", true))); + return new DatacenterFederationSqlPolicy( + tableMapper, fieldMapper, catalogMapper, sourceMapper); + } + + private SqlPolicyContext context(String sql, String... tableNames) + throws Exception { + SqlNode parsed = SqlParser.create(sql).parseQuery(); + String[] effectiveTableNames = tableNames.length == 0 + ? new String[]{"orders"} : tableNames; + List scans = new ArrayList<>(); + for (String tableName : effectiveTableNames) { + scans.add(scan(tableName)); + } + RelDataType rowType = Mockito.mock(RelDataType.class); + Mockito.when(rowType.getFieldNames()) + .thenReturn(List.of("id", "secret", "orders")); + RelNode root = scans.size() == 1 + ? scans.get(0) : Mockito.mock(RelNode.class); + if (scans.size() > 1) { + Mockito.when(root.getRowType()).thenReturn(rowType); + Mockito.doAnswer(invocation -> { + RelVisitor visitor = invocation.getArgument(0); + for (int index = 0; index < scans.size(); index++) { + visitor.visit(scans.get(index), index, root); + } + return null; + }).when(root).childrenAccept(ArgumentMatchers.any(RelVisitor.class)); + } + RelRoot relRoot = RelRoot.of(root, rowType, SqlKind.SELECT); + SourceId sourceId = new SourceId("tenant-1-source-1001"); + SqlCompileRequest request = new SqlCompileRequest( + sql, sourceId, 1L, List.of(), "scope-1"); + return new SqlPolicyContext(request, parsed, relRoot, Set.of(sourceId)); + } + + private TableScan scan(String tableName) { + RelDataType rowType = Mockito.mock(RelDataType.class); + Mockito.when(rowType.getFieldNames()).thenReturn( + "orders".equals(tableName) + ? List.of("id", "secret", "orders") + : List.of("id", "secret")); + TableScan scan = Mockito.mock(TableScan.class); + RelOptTable relOptTable = Mockito.mock(RelOptTable.class); + Mockito.when(relOptTable.getQualifiedName()) + .thenReturn(List.of(tableName)); + Mockito.when(scan.getTable()).thenReturn(relOptTable); + Mockito.when(scan.getRowType()).thenReturn(rowType); + return scan; + } + + private DatacenterTable table(BigInteger tableId, String tableName) { + DatacenterTable table = new DatacenterTable(); + table.setId(tableId); + table.setTableName(tableName); + table.setActualTable(tableName); + table.setQueryable(1); + table.setMetadataStatus(DatacenterMetadataStatus.ACTIVE.name()); + return table; + } + + private DatacenterTableField field( + BigInteger tableId, + String name, + boolean queryable) { + DatacenterTableField field = new DatacenterTableField(); + field.setTableId(tableId); + field.setFieldName(name); + field.setSourceColumnName(name); + field.setQueryable(queryable ? 1 : 0); + field.setMetadataStatus(DatacenterMetadataStatus.ACTIVE.name()); + field.setSensitivityLevel(queryable + ? DatacenterSensitivityLevel.PUBLIC.name() + : DatacenterSensitivityLevel.RESTRICTED.name()); + return field; + } +} diff --git a/easyflow-modules/easyflow-module-datacenter/src/test/java/tech/easyflow/datacenter/federation/DatacenterMetadataIdentityTest.java b/easyflow-modules/easyflow-module-datacenter/src/test/java/tech/easyflow/datacenter/federation/DatacenterMetadataIdentityTest.java new file mode 100644 index 00000000..71c79998 --- /dev/null +++ b/easyflow-modules/easyflow-module-datacenter/src/test/java/tech/easyflow/datacenter/federation/DatacenterMetadataIdentityTest.java @@ -0,0 +1,53 @@ +package tech.easyflow.datacenter.federation; + +import org.junit.Assert; +import org.junit.Test; + +/** + * {@link DatacenterMetadataIdentity} 稳定编码回归测试。 + */ +public class DatacenterMetadataIdentityTest { + + /** + * 验证 null、用户字面量和字段边界不会产生相同摘要。 + */ + @Test + public void namespaceKeyShouldPreservePresenceAndBoundaries() { + String nullCatalog = DatacenterMetadataIdentity.namespaceKey(null, "public"); + String literalCatalog = DatacenterMetadataIdentity.namespaceKey("N", "public"); + String firstBoundary = DatacenterMetadataIdentity.namespaceKey("ab", "c"); + String secondBoundary = DatacenterMetadataIdentity.namespaceKey("a", "bc"); + + Assert.assertNotEquals(nullCatalog, literalCatalog); + Assert.assertNotEquals(firstBoundary, secondBoundary); + Assert.assertEquals(64, nullCatalog.length()); + } + + /** + * 验证相同物理身份在重复计算时保持稳定。 + */ + @Test + public void tableIdentityShouldBeDeterministic() { + String first = DatacenterMetadataIdentity.tableIdentity( + "source-1", "namespace-1", "orders", "TABLE"); + String second = DatacenterMetadataIdentity.tableIdentity( + "source-1", "namespace-1", "orders", "TABLE"); + + Assert.assertEquals(first, second); + } + + /** + * 验证旧表摘要与 V58 迁移中的 MySQL SHA2 输入完全一致。 + */ + @Test + public void legacyV58DigestsShouldMatchMigrationAlgorithm() { + Assert.assertEquals( + "1b0b9f334b8ce200ce3c5480930d9bb537af2b630e17f7f1bc47e9dbe2a72274", + DatacenterMetadataIdentity.legacyV58TableIdentity( + 12L, 34L, "orders", "TABLE")); + Assert.assertEquals( + "983026e4b253d124c629e19de6b97c30a0e3146ba8c45cdee2540df80a466bb3", + DatacenterMetadataIdentity.legacyV58TableFingerprint( + "orders", "TABLE")); + } +} diff --git a/easyflow-modules/easyflow-module-datacenter/src/test/java/tech/easyflow/datacenter/meta/entity/DatacenterIdJsonSerializationTest.java b/easyflow-modules/easyflow-module-datacenter/src/test/java/tech/easyflow/datacenter/meta/entity/DatacenterIdJsonSerializationTest.java index 6e8de6cf..f79e6018 100644 --- a/easyflow-modules/easyflow-module-datacenter/src/test/java/tech/easyflow/datacenter/meta/entity/DatacenterIdJsonSerializationTest.java +++ b/easyflow-modules/easyflow-module-datacenter/src/test/java/tech/easyflow/datacenter/meta/entity/DatacenterIdJsonSerializationTest.java @@ -8,6 +8,7 @@ import tech.easyflow.datacenter.entity.DatacenterTable; import tech.easyflow.datacenter.meta.model.DatacenterCatalogMeta; import java.math.BigInteger; +import java.util.Map; /** * 数据中枢雪花 ID 的 HTTP JSON 序列化测试。 @@ -47,6 +48,24 @@ public class DatacenterIdJsonSerializationTest { assertTextualId(table, "catalogId"); } + /** + * 验证实体被旧 Schema API 间接序列化时也不会暴露凭据密文和连接扩展配置。 + * + * @throws Exception JSON 序列化失败时抛出 + */ + @Test + public void shouldHideSourceCredentialMaterial() throws Exception { + DatacenterSource source = new DatacenterSource(); + source.setCredentialCipher("v1:encrypted-secret"); + source.setConfigJson(Map.of("clientSecret", "plain-secret")); + + JsonNode node = objectMapper.readTree(objectMapper.writeValueAsString(source)); + + Assert.assertFalse(node.has("credentialCipher")); + Assert.assertFalse(node.has("configJson")); + Assert.assertFalse(node.toString().contains("plain-secret")); + } + /** * 验证给定属性被输出为精确的文本 ID。 * diff --git a/easyflow-modules/easyflow-module-datacenter/src/test/java/tech/easyflow/datacenter/meta/service/impl/DatacenterDatasetRegistryServiceImplTest.java b/easyflow-modules/easyflow-module-datacenter/src/test/java/tech/easyflow/datacenter/meta/service/impl/DatacenterDatasetRegistryServiceImplTest.java new file mode 100644 index 00000000..8225cc27 --- /dev/null +++ b/easyflow-modules/easyflow-module-datacenter/src/test/java/tech/easyflow/datacenter/meta/service/impl/DatacenterDatasetRegistryServiceImplTest.java @@ -0,0 +1,361 @@ +package tech.easyflow.datacenter.meta.service.impl; + +import java.lang.reflect.Field; +import java.lang.reflect.Method; +import java.math.BigInteger; +import java.sql.Types; +import java.util.List; +import org.junit.Assert; +import org.junit.Test; +import org.mockito.ArgumentMatchers; +import org.mockito.ArgumentCaptor; +import org.mockito.Mockito; +import tech.easyflow.common.entity.LoginAccount; +import tech.easyflow.common.web.exceptions.BusinessException; +import tech.easyflow.datacenter.entity.DatacenterTable; +import tech.easyflow.datacenter.entity.DatacenterTableField; +import tech.easyflow.datacenter.federation.DatacenterFederationChangeNotifier; +import tech.easyflow.datacenter.federation.DatacenterMetadataIdentity; +import tech.easyflow.datacenter.meta.entity.DatacenterCatalog; +import tech.easyflow.datacenter.mapper.DatacenterSourceMapper; +import tech.easyflow.datacenter.mapper.DatacenterTableFieldMapper; +import tech.easyflow.datacenter.mapper.DatacenterTableMapper; +import tech.easyflow.datacenter.meta.entity.DatacenterSource; +import tech.easyflow.datacenter.meta.model.DatacenterFieldDescriptionUpdate; +import tech.easyflow.datacenter.meta.enums.DatacenterSourceStatus; +import tech.easyflow.datacenter.meta.model.DatacenterTableDetailMeta; + +/** + * {@link DatacenterDatasetRegistryServiceImpl} 查询范围版本回归测试。 + */ +public class DatacenterDatasetRegistryServiceImplTest { + + /** + * 验证 V58 未回填的新 JDBC 字段不会让未变化表误报，同时仍检测旧结构变化。 + * + * @throws Exception 反射调用失败 + */ + @Test + public void legacyStructureShouldIgnoreUnknownNewMetadataOnly() + throws Exception { + DatacenterDatasetRegistryServiceImpl service = + new DatacenterDatasetRegistryServiceImpl(); + DatacenterTableField persisted = legacyField("id", 19); + DatacenterTableField discovered = legacyField("id", 19); + discovered.setJdbcTypeCode(Types.BIGINT); + discovered.setOrdinalPosition(1); + + Assert.assertTrue(invokeLegacyStructureMatches( + service, List.of(persisted), List.of(discovered))); + + DatacenterTableField changed = legacyField("id", 20); + changed.setJdbcTypeCode(Types.BIGINT); + changed.setOrdinalPosition(1); + Assert.assertFalse(invokeLegacyStructureMatches( + service, List.of(persisted), List.of(changed))); + } + + /** + * 验证字段治理变更会以 CAS 推进外部数据源范围版本并发布通知。 + * + * @throws Exception 测试依赖注入失败 + */ + @Test + public void saveDescriptionsShouldAdvanceScopeRevisionForPolicyChange() + throws Exception { + BigInteger sourceId = BigInteger.valueOf(1001L); + BigInteger tableId = BigInteger.valueOf(2001L); + DatacenterSource source = new DatacenterSource(); + source.setId(sourceId); + source.setTenantId(BigInteger.ONE); + source.setSourceType("MYSQL"); + source.setStatus(DatacenterSourceStatus.READY.code()); + source.setDefinitionRevision(1L); + source.setScopeRevision(1L); + DatacenterSource refreshedSource = new DatacenterSource(); + refreshedSource.setId(sourceId); + refreshedSource.setTenantId(BigInteger.ONE); + refreshedSource.setSourceType("MYSQL"); + refreshedSource.setStatus(DatacenterSourceStatus.READY.code()); + refreshedSource.setDefinitionRevision(1L); + refreshedSource.setScopeRevision(2L); + + DatacenterTable table = new DatacenterTable(); + table.setId(tableId); + table.setSourceId(sourceId); + table.setTenantId(BigInteger.ONE); + DatacenterTableField field = new DatacenterTableField(); + field.setId(BigInteger.valueOf(3001L)); + field.setTableId(tableId); + field.setQueryable(1); + field.setSensitivityLevel("PUBLIC"); + + DatacenterTableMapper tableMapper = Mockito.mock(DatacenterTableMapper.class); + DatacenterTableFieldMapper fieldMapper = Mockito.mock( + DatacenterTableFieldMapper.class); + DatacenterSourceMapper sourceMapper = Mockito.mock(DatacenterSourceMapper.class); + DatacenterFederationChangeNotifier notifier = Mockito.mock( + DatacenterFederationChangeNotifier.class); + Mockito.when(tableMapper.selectOneById(tableId)).thenReturn(table); + Mockito.when(fieldMapper.selectListByQuery(ArgumentMatchers.any())) + .thenReturn(List.of(field)); + Mockito.when(sourceMapper.selectOneById(sourceId)) + .thenReturn(source, refreshedSource); + Mockito.when(sourceMapper.updateByQuery( + ArgumentMatchers.any(), ArgumentMatchers.any())) + .thenReturn(1); + + DatacenterDatasetRegistryServiceImpl service = + new DatacenterDatasetRegistryServiceImpl(); + setField(service, "tableMapper", tableMapper); + setField(service, "tableFieldMapper", fieldMapper); + setField(service, "sourceMapper", sourceMapper); + setField(service, "federationChangeNotifier", notifier); + + DatacenterFieldDescriptionUpdate update = + new DatacenterFieldDescriptionUpdate(); + update.setFieldId(field.getId()); + update.setFieldDesc("内部编号"); + update.setQueryable(0); + update.setSensitivityLevel("INTERNAL"); + LoginAccount account = new LoginAccount(); + account.setId(BigInteger.TEN); + account.setTenantId(BigInteger.ONE); + + service.saveDescriptions(tableId, "订单", List.of(update), account); + + Assert.assertEquals(Integer.valueOf(0), field.getQueryable()); + Assert.assertEquals("INTERNAL", field.getSensitivityLevel()); + ArgumentCaptor patchCaptor = + ArgumentCaptor.forClass(DatacenterSource.class); + Mockito.verify(sourceMapper).updateByQuery( + patchCaptor.capture(), ArgumentMatchers.any()); + DatacenterSource patch = patchCaptor.getValue(); + Assert.assertEquals(sourceId, patch.getId()); + Assert.assertEquals(Long.valueOf(2L), patch.getScopeRevision()); + Assert.assertNull(patch.getStatus()); + Assert.assertNull(patch.getDefinitionRevision()); + Mockito.verify(notifier).publishAfterCommit(refreshedSource); + } + + /** + * 验证刷新发现未纳管新字段时只标记结构变化并收缩权限，不自动插入字段。 + * + * @throws Exception 测试依赖注入失败 + */ + @Test + public void refreshShouldFailClosedWithoutRegisteringNewField() + throws Exception { + BigInteger sourceId = BigInteger.valueOf(1101L); + BigInteger catalogId = BigInteger.valueOf(2101L); + BigInteger tableId = BigInteger.valueOf(3101L); + DatacenterSource source = new DatacenterSource(); + source.setId(sourceId); + source.setTenantId(BigInteger.ONE); + source.setSourceType("MYSQL"); + DatacenterCatalog catalog = new DatacenterCatalog(); + catalog.setId(catalogId); + catalog.setSourceId(sourceId); + catalog.setNamespaceKey(DatacenterMetadataIdentity.namespaceKey("easyflow", null)); + + DatacenterTableField idField = modernField("id", Types.BIGINT); + idField.setId(BigInteger.valueOf(4101L)); + idField.setTableId(tableId); + idField.setQueryable(1); + DatacenterTable table = new DatacenterTable(); + table.setId(tableId); + table.setSourceId(sourceId); + table.setCatalogId(catalogId); + table.setTableName("orders"); + table.setActualTable("orders"); + table.setTableKind("EXTERNAL_TABLE"); + table.setMetadataStatus("ACTIVE"); + table.setQueryable(1); + table.setPhysicalIdentityKey(DatacenterMetadataIdentity.tableIdentity( + sourceId.toString(), catalog.getNamespaceKey(), "orders", "EXTERNAL_TABLE")); + table.setMetadataFingerprint(DatacenterMetadataIdentity.tableFingerprint( + "orders", "EXTERNAL_TABLE", List.of(idField))); + + DatacenterTableField newField = modernField("secret", Types.VARCHAR); + DatacenterTable discovered = new DatacenterTable(); + discovered.setTableName("orders"); + discovered.setActualTable("orders"); + discovered.setTableKind("EXTERNAL_TABLE"); + DatacenterTableDetailMeta detail = new DatacenterTableDetailMeta(); + detail.setTable(discovered); + detail.setFields(List.of(modernField("id", Types.BIGINT), newField)); + + DatacenterTableMapper tableMapper = Mockito.mock(DatacenterTableMapper.class); + DatacenterTableFieldMapper fieldMapper = Mockito.mock( + DatacenterTableFieldMapper.class); + Mockito.when(tableMapper.selectOneById(tableId)).thenReturn(table); + Mockito.when(fieldMapper.selectListByQuery(ArgumentMatchers.any())) + .thenReturn(List.of(idField)); + DatacenterDatasetRegistryServiceImpl service = + new DatacenterDatasetRegistryServiceImpl(); + setField(service, "tableMapper", tableMapper); + setField(service, "tableFieldMapper", fieldMapper); + + boolean changed = service.refreshManagedTable( + source, catalog, tableId, detail, account()); + + Assert.assertTrue(changed); + Assert.assertEquals("CHANGED", table.getMetadataStatus()); + Assert.assertEquals(Integer.valueOf(0), table.getQueryable()); + Assert.assertEquals("CHANGED", idField.getMetadataStatus()); + Assert.assertEquals(Integer.valueOf(0), idField.getQueryable()); + Mockito.verify(fieldMapper, Mockito.never()) + .insert(ArgumentMatchers.any(DatacenterTableField.class)); + } + + /** + * 显式重配置允许将稳定表 ID 绑定到新物理命名空间，并重建字段治理基线。 + * 常规刷新对相同变化仍保持 fail-closed。 + * + * @throws Exception 测试依赖注入失败 + */ + @Test + public void explicitRebindShouldPreserveTableIdAndReplaceFields() + throws Exception { + BigInteger sourceId = BigInteger.valueOf(1201L); + BigInteger catalogId = BigInteger.valueOf(2201L); + BigInteger tableId = BigInteger.valueOf(3201L); + DatacenterSource source = new DatacenterSource(); + source.setId(sourceId); + source.setTenantId(BigInteger.ONE); + source.setSourceType("MYSQL"); + DatacenterCatalog catalog = new DatacenterCatalog(); + catalog.setId(catalogId); + catalog.setSourceId(sourceId); + catalog.setNamespaceKey(DatacenterMetadataIdentity.namespaceKey("new_db", null)); + + DatacenterTableField persistedField = modernField("legacy_id", Types.BIGINT); + persistedField.setId(BigInteger.valueOf(4201L)); + persistedField.setTableId(tableId); + DatacenterTable existing = new DatacenterTable(); + existing.setId(tableId); + existing.setSourceId(sourceId); + existing.setCatalogId(catalogId); + existing.setTableName("orders"); + existing.setActualTable("orders"); + existing.setTableKind("EXTERNAL_TABLE"); + existing.setQueryable(0); + existing.setPhysicalIdentityKey(DatacenterMetadataIdentity.tableIdentity( + sourceId.toString(), + DatacenterMetadataIdentity.namespaceKey("old_db", null), + "orders", + "EXTERNAL_TABLE")); + existing.setMetadataFingerprint(DatacenterMetadataIdentity.tableFingerprint( + "orders", "EXTERNAL_TABLE", List.of(persistedField))); + + DatacenterTableField discoveredField = modernField("id", Types.BIGINT); + DatacenterTable discovered = new DatacenterTable(); + discovered.setTableName("orders"); + discovered.setActualTable("orders"); + discovered.setTableKind("EXTERNAL_TABLE"); + discovered.setQueryable(1); + DatacenterTableDetailMeta detail = new DatacenterTableDetailMeta(); + detail.setTable(discovered); + detail.setFields(List.of(discoveredField)); + + DatacenterTableMapper tableMapper = Mockito.mock(DatacenterTableMapper.class); + DatacenterTableFieldMapper fieldMapper = Mockito.mock( + DatacenterTableFieldMapper.class); + Mockito.when(tableMapper.selectOneByQuery(ArgumentMatchers.any())) + .thenReturn(existing); + Mockito.when(fieldMapper.selectListByQuery(ArgumentMatchers.any())) + .thenReturn( + List.of(persistedField), + List.of(persistedField), + List.of(discoveredField)); + DatacenterDatasetRegistryServiceImpl service = + new DatacenterDatasetRegistryServiceImpl(); + setField(service, "tableMapper", tableMapper); + setField(service, "tableFieldMapper", fieldMapper); + + try { + service.registerTable(source, catalog, detail, account()); + Assert.fail("ordinary refresh must reject changed physical identity"); + } catch (BusinessException expected) { + Assert.assertTrue(expected.getMessage().contains("物理表身份已变化")); + } + + DatacenterTable rebound = service.rebindTable( + source, catalog, detail, account()); + + Assert.assertEquals(tableId, rebound.getId()); + Assert.assertEquals(Integer.valueOf(1), rebound.getQueryable()); + Assert.assertEquals(DatacenterMetadataIdentity.tableIdentity( + sourceId.toString(), catalog.getNamespaceKey(), + "orders", "EXTERNAL_TABLE"), rebound.getPhysicalIdentityKey()); + Mockito.verify(fieldMapper).deleteByQuery(ArgumentMatchers.any()); + Mockito.verify(fieldMapper).insert(discoveredField); + Mockito.verify(tableMapper).update(existing); + } + + /** + * 注入被测服务依赖。 + * + * @param target 被测对象 + * @param name 字段名 + * @param value 依赖值 + * @throws ReflectiveOperationException 反射失败 + */ + private void setField(Object target, String name, Object value) + throws ReflectiveOperationException { + Field field = target.getClass().getDeclaredField(name); + field.setAccessible(true); + field.set(target, value); + } + + private DatacenterTableField legacyField(String name, int precision) { + DatacenterTableField field = new DatacenterTableField(); + field.setFieldName(name); + field.setSourceColumnName(name); + field.setFieldType(0); + field.setJdbcType("BIGINT"); + field.setNativeTypeName("BIGINT"); + field.setPrecision(precision); + field.setScale(0); + field.setRequired(1); + return field; + } + + private DatacenterTableField modernField(String name, int jdbcType) { + DatacenterTableField field = new DatacenterTableField(); + field.setFieldName(name); + field.setSourceColumnName(name); + field.setJdbcType(jdbcType == Types.BIGINT ? "BIGINT" : "VARCHAR"); + field.setNativeTypeName(field.getJdbcType()); + field.setJdbcTypeCode(jdbcType); + field.setPrecision(jdbcType == Types.BIGINT ? 19 : 255); + field.setScale(0); + field.setRequired(0); + field.setQueryable(1); + field.setSortable(1); + field.setWritable(0); + field.setMetadataStatus("ACTIVE"); + field.setMetadataFingerprint(DatacenterMetadataIdentity.fieldFingerprint(field)); + return field; + } + + private LoginAccount account() { + LoginAccount account = new LoginAccount(); + account.setId(BigInteger.TEN); + account.setTenantId(BigInteger.ONE); + return account; + } + + @SuppressWarnings("unchecked") + private boolean invokeLegacyStructureMatches( + DatacenterDatasetRegistryServiceImpl service, + List persisted, + List discovered) + throws ReflectiveOperationException { + Method method = DatacenterDatasetRegistryServiceImpl.class + .getDeclaredMethod( + "legacyStructureMatches", List.class, List.class); + method.setAccessible(true); + return (boolean) method.invoke(service, persisted, discovered); + } +} diff --git a/easyflow-modules/easyflow-module-datacenter/src/test/java/tech/easyflow/datacenter/meta/service/impl/DatacenterSourceMetadataRefreshServiceTest.java b/easyflow-modules/easyflow-module-datacenter/src/test/java/tech/easyflow/datacenter/meta/service/impl/DatacenterSourceMetadataRefreshServiceTest.java new file mode 100644 index 00000000..4dfd7285 --- /dev/null +++ b/easyflow-modules/easyflow-module-datacenter/src/test/java/tech/easyflow/datacenter/meta/service/impl/DatacenterSourceMetadataRefreshServiceTest.java @@ -0,0 +1,125 @@ +package tech.easyflow.datacenter.meta.service.impl; + +import java.math.BigInteger; +import java.util.List; +import java.util.Set; +import org.junit.Assert; +import org.junit.Test; +import org.mockito.ArgumentMatchers; +import org.mockito.InOrder; +import org.mockito.Mockito; +import org.springframework.transaction.PlatformTransactionManager; +import org.springframework.transaction.TransactionDefinition; +import org.springframework.transaction.TransactionStatus; +import tech.easyflow.common.entity.LoginAccount; +import tech.easyflow.datacenter.connector.DatacenterConnector; +import tech.easyflow.datacenter.connector.DatacenterConnectorRegistry; +import tech.easyflow.datacenter.entity.DatacenterTable; +import tech.easyflow.datacenter.federation.DatacenterFederationChangeNotifier; +import tech.easyflow.datacenter.mapper.DatacenterCatalogMapper; +import tech.easyflow.datacenter.mapper.DatacenterSourceMapper; +import tech.easyflow.datacenter.mapper.DatacenterTableMapper; +import tech.easyflow.datacenter.meta.entity.DatacenterCatalog; +import tech.easyflow.datacenter.meta.entity.DatacenterSource; +import tech.easyflow.datacenter.meta.enums.DatacenterSourceStatus; +import tech.easyflow.datacenter.meta.model.DatacenterManagedMetadataSnapshot; +import tech.easyflow.datacenter.meta.model.DatacenterTableDetailMeta; +import tech.easyflow.datacenter.meta.service.DatacenterDatasetRegistryService; + +/** + * {@link DatacenterSourceMetadataRefreshService} 事务与远程 I/O 边界测试。 + */ +public class DatacenterSourceMetadataRefreshServiceTest { + + /** + * 远程 JDBC 元数据必须在开启平台数据库事务前完成。 + */ + @Test + public void refreshShouldInspectRemoteMetadataBeforeOpeningTransaction() { + BigInteger sourceId = BigInteger.valueOf(1301L); + DatacenterSource source = new DatacenterSource(); + source.setId(sourceId); + source.setTenantId(BigInteger.ONE); + source.setSourceType("MYSQL"); + source.setStatus(DatacenterSourceStatus.READY.code()); + source.setDefinitionRevision(3L); + source.setScopeRevision(5L); + DatacenterCatalog catalog = new DatacenterCatalog(); + catalog.setId(BigInteger.valueOf(2301L)); + catalog.setSourceId(sourceId); + catalog.setCatalogName("orders_db"); + DatacenterTable table = new DatacenterTable(); + table.setId(BigInteger.valueOf(3301L)); + table.setSourceId(sourceId); + table.setCatalogId(catalog.getId()); + table.setTableName("orders"); + table.setActualTable("orders"); + DatacenterTable discovered = new DatacenterTable(); + discovered.setTableName("orders"); + discovered.setActualTable("orders"); + DatacenterTableDetailMeta detail = new DatacenterTableDetailMeta(); + detail.setTable(discovered); + LoginAccount account = account(); + + DatacenterConnector connector = Mockito.mock(DatacenterConnector.class); + DatacenterConnectorRegistry connectorRegistry = + Mockito.mock(DatacenterConnectorRegistry.class); + DatacenterDatasetRegistryService registryService = + Mockito.mock(DatacenterDatasetRegistryService.class); + DatacenterCatalogMapper catalogMapper = + Mockito.mock(DatacenterCatalogMapper.class); + DatacenterTableMapper tableMapper = Mockito.mock(DatacenterTableMapper.class); + DatacenterSourceMapper sourceMapper = + Mockito.mock(DatacenterSourceMapper.class); + DatacenterFederationChangeNotifier notifier = + Mockito.mock(DatacenterFederationChangeNotifier.class); + PlatformTransactionManager transactionManager = + Mockito.mock(PlatformTransactionManager.class); + TransactionStatus transactionStatus = Mockito.mock(TransactionStatus.class); + Mockito.when(connectorRegistry.getConnector("MYSQL")).thenReturn(connector); + Mockito.when(catalogMapper.selectListByQuery(ArgumentMatchers.any())) + .thenReturn(List.of(catalog)); + Mockito.when(tableMapper.selectListByQuery(ArgumentMatchers.any())) + .thenReturn(List.of(table)); + Mockito.when(connector.inspectManagedTables( + source, "orders_db", List.of("orders"))) + .thenReturn(new DatacenterManagedMetadataSnapshot( + List.of(detail), Set.of())); + Mockito.when(registryService.refreshManagedTable( + source, catalog, table.getId(), detail, account)) + .thenReturn(false); + Mockito.when(transactionManager.getTransaction( + ArgumentMatchers.any(TransactionDefinition.class))) + .thenReturn(transactionStatus); + Mockito.when(sourceMapper.updateByQuery( + ArgumentMatchers.any(), ArgumentMatchers.any())) + .thenReturn(1); + DatacenterSource refreshed = new DatacenterSource(); + refreshed.setId(sourceId); + Mockito.when(sourceMapper.selectOneByQuery(ArgumentMatchers.any())) + .thenReturn(refreshed); + DatacenterSourceMetadataRefreshService service = + new DatacenterSourceMetadataRefreshService( + connectorRegistry, registryService, catalogMapper, + tableMapper, sourceMapper, notifier, transactionManager); + + DatacenterSource result = service.refresh(source, account); + + Assert.assertSame(refreshed, result); + InOrder order = Mockito.inOrder(connector, transactionManager, sourceMapper); + order.verify(connector).inspectManagedTables( + source, "orders_db", List.of("orders")); + order.verify(transactionManager).getTransaction( + ArgumentMatchers.any(TransactionDefinition.class)); + order.verify(sourceMapper).updateByQuery( + ArgumentMatchers.any(), ArgumentMatchers.any()); + Mockito.verify(notifier).publishAfterCommit(refreshed); + } + + private LoginAccount account() { + LoginAccount account = new LoginAccount(); + account.setId(BigInteger.TEN); + account.setTenantId(BigInteger.ONE); + return account; + } +} diff --git a/easyflow-modules/easyflow-module-datacenter/src/test/java/tech/easyflow/datacenter/meta/service/impl/DatacenterSourceServiceImplTest.java b/easyflow-modules/easyflow-module-datacenter/src/test/java/tech/easyflow/datacenter/meta/service/impl/DatacenterSourceServiceImplTest.java new file mode 100644 index 00000000..f5b64cdb --- /dev/null +++ b/easyflow-modules/easyflow-module-datacenter/src/test/java/tech/easyflow/datacenter/meta/service/impl/DatacenterSourceServiceImplTest.java @@ -0,0 +1,410 @@ +package tech.easyflow.datacenter.meta.service.impl; + +import com.mybatisflex.core.query.QueryWrapper; +import com.mybatisflex.spring.service.impl.ServiceImpl; +import java.lang.reflect.Field; +import java.math.BigInteger; +import java.util.List; +import org.junit.Assert; +import org.junit.Test; +import org.mockito.ArgumentCaptor; +import org.mockito.ArgumentMatchers; +import org.mockito.InOrder; +import org.mockito.Mockito; +import org.springframework.transaction.PlatformTransactionManager; +import org.springframework.transaction.TransactionDefinition; +import org.springframework.transaction.TransactionStatus; +import tech.easyflow.common.entity.LoginAccount; +import tech.easyflow.datacenter.connector.DatacenterConnector; +import tech.easyflow.datacenter.connector.DatacenterConnectorRegistry; +import tech.easyflow.datacenter.entity.DatacenterTable; +import tech.easyflow.datacenter.federation.DatacenterFederationChangeNotifier; +import tech.easyflow.datacenter.federation.DatacenterFederationDefinitionFactory; +import tech.easyflow.datacenter.federation.DatacenterFederationRuntime; +import tech.easyflow.datacenter.mapper.DatacenterCatalogMapper; +import tech.easyflow.datacenter.mapper.DatacenterSourceMapper; +import tech.easyflow.datacenter.mapper.DatacenterTableMapper; +import tech.easyflow.datacenter.meta.entity.DatacenterCatalog; +import tech.easyflow.datacenter.meta.entity.DatacenterSource; +import tech.easyflow.datacenter.meta.enums.DatacenterMetadataStatus; +import tech.easyflow.datacenter.meta.enums.DatacenterSourceStatus; +import tech.easyflow.datacenter.meta.model.DatacenterBatchRegisterRequest; +import tech.easyflow.datacenter.meta.model.DatacenterSourceActivateRequest; +import tech.easyflow.datacenter.meta.model.DatacenterTableDetailMeta; +import tech.easyflow.datacenter.meta.model.DatacenterSourceDraftRequest; +import tech.easyflow.datacenter.meta.model.DatacenterSourceReconfigureRequest; +import tech.easyflow.datacenter.meta.service.DatacenterDatasetRegistryService; +import tech.easyflow.datacenter.meta.support.DatacenterSourceConnectionDefaults; +import tech.easyflow.datacenter.security.DatacenterCredentialCipher; +import com.easyagents.federation.sql.source.ExternalSchemaDefinition; +import com.easyagents.federation.sql.source.FederationSourceDefinition; +import com.easyagents.federation.sql.source.PreparedSourceRuntime; +import com.easyagents.federation.sql.source.RuntimeFingerprint; +import com.easyagents.federation.sql.source.SourceId; +import com.easyagents.federation.sql.source.SourceProbeResult; + +/** + * {@link DatacenterSourceServiceImpl} 生命周期和 revision 回归测试。 + */ +public class DatacenterSourceServiceImplTest { + + /** + * 已发布连接不得复用初次激活入口绕过重配置 revision CAS。 + * + * @throws Exception 测试依赖注入失败 + */ + @Test + public void activateShouldRejectPublishedSource() throws Exception { + Fixture fixture = fixture(); + DatacenterSource ready = source(DatacenterSourceStatus.READY, 3L, 7L); + Mockito.when(fixture.sourceMapper.selectOneByQuery(ArgumentMatchers.any())) + .thenReturn(ready); + + try { + fixture.service.activate(new DatacenterSourceActivateRequest( + ready.getId(), "public", List.of("orders"), true), account()); + Assert.fail("published source must use reconfiguration"); + } catch (tech.easyflow.common.web.exceptions.BusinessException expected) { + Assert.assertTrue(expected.getMessage().contains("重新配置")); + } + + Mockito.verifyNoInteractions(fixture.connectorRegistry); + Mockito.verify(fixture.runtime, Mockito.never()) + .probe(ArgumentMatchers.any()); + Mockito.verify(fixture.runtime, Mockito.never()) + .probeCandidate(ArgumentMatchers.any()); + } + + /** + * 已发布连接按持久化 Definition 探测，避免候选 MAIN Schema 造成 checksum 冲突。 + * + * @throws Exception 测试依赖注入失败 + */ + @Test + public void publishedSourceProbeShouldUsePublishedDefinition() throws Exception { + Fixture fixture = fixture(); + DatacenterSource ready = source(DatacenterSourceStatus.READY, 3L, 7L); + ready.setSchemaName("tenant_schema"); + ready.setDefinitionChecksum("published-checksum"); + Mockito.when(fixture.sourceMapper.selectOneByQuery(ArgumentMatchers.any())) + .thenReturn(ready); + Mockito.when(fixture.sourceMapper.updateByQuery( + ArgumentMatchers.any(), ArgumentMatchers.any())).thenReturn(1); + Mockito.when(fixture.runtime.probe(ready)).thenReturn(new SourceProbeResult( + new SourceId("tenant-1-source-100"), true, + new RuntimeFingerprint( + "PostgreSQL", "17", "PostgreSQL JDBC", "42", "test"), + "ok")); + + Assert.assertTrue(fixture.service.probe(ready.getId(), account()).isSuccess()); + + Mockito.verify(fixture.runtime).probe(ready); + Mockito.verify(fixture.runtime, Mockito.never()) + .probeCandidate(ArgumentMatchers.any()); + } + + /** + * 停用只推进 Definition 版本并发布墓碑，不改变业务范围版本。 + * + * @throws Exception 测试依赖注入失败 + */ + @Test + public void disableShouldAdvanceDefinitionRevisionOnly() throws Exception { + Fixture fixture = fixture(); + DatacenterSource ready = source(DatacenterSourceStatus.READY, 3L, 7L); + DatacenterSource disabled = source(DatacenterSourceStatus.DISABLED, 4L, 7L); + Mockito.when(fixture.sourceMapper.selectOneByQuery(ArgumentMatchers.any())) + .thenReturn(ready, disabled); + Mockito.when(fixture.sourceMapper.updateByQuery( + ArgumentMatchers.any(), ArgumentMatchers.any())).thenReturn(1); + + fixture.service.disable(ready.getId(), account()); + + ArgumentCaptor patch = + ArgumentCaptor.forClass(DatacenterSource.class); + Mockito.verify(fixture.sourceMapper).updateByQuery( + patch.capture(), ArgumentMatchers.any(QueryWrapper.class)); + Assert.assertEquals(Long.valueOf(4L), patch.getValue().getDefinitionRevision()); + Assert.assertNull(patch.getValue().getScopeRevision()); + Assert.assertEquals(Integer.valueOf(DatacenterSourceStatus.DISABLED.code()), + patch.getValue().getStatus()); + Mockito.verify(fixture.runtime).remove(disabled); + Mockito.verify(fixture.notifier).publishAfterCommit(disabled); + } + + /** + * 已激活连接新增纳管表时只推进 scopeRevision，避免无意义重建连接池。 + * + * @throws Exception 测试依赖注入失败 + */ + @Test + public void batchRegisterShouldAdvanceScopeRevisionOnly() throws Exception { + Fixture fixture = fixture(); + DatacenterSource ready = source(DatacenterSourceStatus.READY, 5L, 9L); + ready.setLastTestStatus("SUCCESS"); + DatacenterSource updated = source(DatacenterSourceStatus.READY, 5L, 10L); + DatacenterCatalog catalog = new DatacenterCatalog(); + catalog.setId(BigInteger.valueOf(200L)); + catalog.setSourceId(ready.getId()); + catalog.setCatalogName("public"); + catalog.setMetadataStatus(DatacenterMetadataStatus.ACTIVE.name()); + DatacenterTableDetailMeta detail = new DatacenterTableDetailMeta(); + DatacenterTable discovered = new DatacenterTable(); + discovered.setTableName("orders"); + detail.setTable(discovered); + DatacenterConnector connector = Mockito.mock(DatacenterConnector.class); + + Mockito.when(fixture.sourceMapper.selectOneByQuery(ArgumentMatchers.any())) + .thenReturn(ready, updated); + Mockito.when(fixture.sourceMapper.updateByQuery( + ArgumentMatchers.any(), ArgumentMatchers.any())).thenReturn(1); + Mockito.when(fixture.registry.listManagedTables( + ready.getId(), catalog.getId())).thenReturn(List.of()); + Mockito.when(fixture.catalogMapper.selectOneByQuery(ArgumentMatchers.any())) + .thenReturn(catalog); + Mockito.when(fixture.connectorRegistry.getConnector("MYSQL")) + .thenReturn(connector); + Mockito.when(connector.getTableDetails( + ready, "public", List.of("orders"))).thenReturn(List.of(detail)); + + DatacenterBatchRegisterRequest request = new DatacenterBatchRegisterRequest(); + request.setSourceId(ready.getId()); + request.setCatalogName("public"); + request.setTableNames(List.of("orders")); + fixture.service.batchRegisterTables(request, account()); + + ArgumentCaptor patch = + ArgumentCaptor.forClass(DatacenterSource.class); + Mockito.verify(fixture.sourceMapper).updateByQuery( + patch.capture(), ArgumentMatchers.any(QueryWrapper.class)); + Assert.assertNull(patch.getValue().getDefinitionRevision()); + Assert.assertEquals(Long.valueOf(10L), patch.getValue().getScopeRevision()); + Mockito.verify(fixture.runtime, Mockito.never()).apply( + ArgumentMatchers.any(), ArgumentMatchers.anyBoolean()); + Mockito.verify(fixture.notifier).publishAfterCommit(updated); + } + + /** + * 候选配置探测失败时不写数据库，也不替换当前 Runtime。 + * + * @throws Exception 测试依赖注入失败 + */ + @Test + public void failedReconfigurationProbeShouldKeepReadySourceUntouched() + throws Exception { + Fixture fixture = fixture(); + DatacenterSource ready = source(DatacenterSourceStatus.READY, 8L, 12L); + ready.setSourceName("orders-db"); + ready.setSourceCode("ORDERS_DB"); + ready.setHost("127.0.0.1"); + ready.setPort(3306); + ready.setDatabaseName("orders"); + ready.setUsername("reader"); + ready.setCredentialCipher("encrypted"); + Mockito.when(fixture.sourceMapper.selectOneByQuery(ArgumentMatchers.any())) + .thenReturn(ready); + Mockito.when(fixture.runtime.probeCandidate(ArgumentMatchers.any())) + .thenThrow(new IllegalStateException("simulated connection failure")); + Mockito.when(fixture.defaults.defaultDriverClassName("MYSQL")) + .thenReturn("com.mysql.cj.jdbc.Driver"); + DatacenterSourceDraftRequest definition = new DatacenterSourceDraftRequest( + ready.getId(), "orders-db", "ORDERS_DB", "MYSQL", + "127.0.0.2", 3306, "orders", null, "reader", null, + null, null, java.util.Map.of(), java.util.Map.of()); + DatacenterSourceReconfigureRequest request = + new DatacenterSourceReconfigureRequest( + definition, "orders", List.of("orders"), true, 8L, 12L); + + try { + fixture.service.reconfigure(request, account()); + Assert.fail("failed candidate probe must reject reconfiguration"); + } catch (tech.easyflow.common.web.exceptions.BusinessException expected) { + Assert.assertTrue(expected.getMessage().contains("无法连接")); + } + + Mockito.verify(fixture.sourceMapper, Mockito.never()).updateByQuery( + ArgumentMatchers.any(), ArgumentMatchers.any()); + Mockito.verify(fixture.runtime, Mockito.never()).apply( + ArgumentMatchers.any(), ArgumentMatchers.anyBoolean()); + } + + /** + * 重新启用时先预构建 Runtime，再发布权威 READY 并原子接管。 + * + * @throws Exception 测试依赖注入失败 + */ + @Test + public void enableShouldPrepareBeforePublishingAndCommitAfterward() throws Exception { + Fixture fixture = fixture(); + DatacenterSource disabled = source(DatacenterSourceStatus.DISABLED, 4L, 7L); + disabled.setSourceName("orders-db"); + disabled.setSourceCode("ORDERS_DB"); + disabled.setDatabaseName("orders"); + disabled.setSchemaName("public"); + disabled.setCredentialCipher("encrypted"); + disabled.setAdapterId("jdbc"); + DatacenterSource enabled = source(DatacenterSourceStatus.READY, 5L, 7L); + Mockito.when(fixture.sourceMapper.selectOneByQuery(ArgumentMatchers.any())) + .thenReturn(disabled, enabled); + Mockito.when(fixture.sourceMapper.updateByQuery( + ArgumentMatchers.any(), ArgumentMatchers.any())).thenReturn(1); + RuntimeFingerprint fingerprint = new RuntimeFingerprint( + "MySQL", "8.0", "MySQL Connector/J", "9.0", "test"); + Mockito.when(fixture.runtime.probeCandidate(ArgumentMatchers.any())) + .thenReturn(new SourceProbeResult( + new SourceId("tenant-1-source-100"), true, fingerprint, "ok")); + FederationSourceDefinition definition = new FederationSourceDefinition( + new SourceId("tenant-1-source-100"), + 5L, + "jdbc", + List.of(new ExternalSchemaDefinition("public", "snapshot", 1L)), + java.util.Map.of()); + PreparedSourceRuntime prepared = Mockito.mock(PreparedSourceRuntime.class); + Mockito.when(prepared.definition()).thenReturn(definition); + Mockito.doAnswer(invocation -> { + DatacenterSource candidate = invocation.getArgument(0); + Assert.assertEquals(Integer.valueOf(DatacenterSourceStatus.DRAFT.code()), + candidate.getStatus()); + Assert.assertNull(candidate.getDefinitionChecksum()); + return prepared; + }).when(fixture.runtime).prepareCandidate(ArgumentMatchers.any()); + + fixture.service.enable(disabled.getId(), account()); + + InOrder order = Mockito.inOrder(fixture.runtime, fixture.sourceMapper); + order.verify(fixture.runtime).probeCandidate(ArgumentMatchers.any()); + order.verify(fixture.runtime).prepareCandidate(ArgumentMatchers.any()); + order.verify(fixture.sourceMapper).updateByQuery( + ArgumentMatchers.any(), ArgumentMatchers.any(QueryWrapper.class)); + order.verify(fixture.runtime).commitPrepared(prepared); + } + + /** + * 重新启用的 revision CAS 失败时丢弃预构建 Runtime，且不得发布候选连接池。 + * + * @throws Exception 测试依赖注入失败 + */ + @Test + public void enableCasFailureShouldDiscardPreparedRuntime() throws Exception { + Fixture fixture = fixture(); + DatacenterSource disabled = source(DatacenterSourceStatus.DISABLED, 4L, 7L); + disabled.setSourceName("orders-db"); + disabled.setSourceCode("ORDERS_DB"); + disabled.setDatabaseName("orders"); + disabled.setSchemaName("public"); + disabled.setCredentialCipher("encrypted"); + disabled.setAdapterId("jdbc"); + Mockito.when(fixture.sourceMapper.selectOneByQuery(ArgumentMatchers.any())) + .thenReturn(disabled); + Mockito.when(fixture.sourceMapper.updateByQuery( + ArgumentMatchers.any(), ArgumentMatchers.any())).thenReturn(0); + RuntimeFingerprint fingerprint = new RuntimeFingerprint( + "MySQL", "8.0", "MySQL Connector/J", "9.0", "test"); + Mockito.when(fixture.runtime.probeCandidate(ArgumentMatchers.any())) + .thenReturn(new SourceProbeResult( + new SourceId("tenant-1-source-100"), true, fingerprint, "ok")); + FederationSourceDefinition definition = new FederationSourceDefinition( + new SourceId("tenant-1-source-100"), + 5L, + "jdbc", + List.of(new ExternalSchemaDefinition("public", "snapshot", 1L)), + java.util.Map.of()); + PreparedSourceRuntime prepared = Mockito.mock(PreparedSourceRuntime.class); + Mockito.when(prepared.definition()).thenReturn(definition); + Mockito.doAnswer(invocation -> { + DatacenterSource candidate = invocation.getArgument(0); + Assert.assertEquals(Integer.valueOf(DatacenterSourceStatus.DRAFT.code()), + candidate.getStatus()); + Assert.assertNull(candidate.getDefinitionChecksum()); + return prepared; + }).when(fixture.runtime).prepareCandidate(ArgumentMatchers.any()); + + try { + fixture.service.enable(disabled.getId(), account()); + Assert.fail("stale revision must reject enable"); + } catch (tech.easyflow.common.web.exceptions.BusinessException expected) { + Assert.assertNotNull(expected.getMessage()); + } + + Mockito.verify(prepared).close(); + Mockito.verify(fixture.runtime, Mockito.never()).commitPrepared(prepared); + Mockito.verify(fixture.notifier, Mockito.never()) + .publishAfterCommit(ArgumentMatchers.any()); + } + + private Fixture fixture() throws Exception { + DatacenterConnectorRegistry connectorRegistry = + Mockito.mock(DatacenterConnectorRegistry.class); + DatacenterDatasetRegistryService registry = + Mockito.mock(DatacenterDatasetRegistryService.class); + DatacenterCredentialCipher cipher = Mockito.mock(DatacenterCredentialCipher.class); + DatacenterCatalogMapper catalogMapper = Mockito.mock(DatacenterCatalogMapper.class); + DatacenterTableMapper tableMapper = Mockito.mock(DatacenterTableMapper.class); + DatacenterSourceConnectionDefaults defaults = + Mockito.mock(DatacenterSourceConnectionDefaults.class); + DatacenterFederationRuntime runtime = Mockito.mock(DatacenterFederationRuntime.class); + Mockito.when(catalogMapper.selectListByQuery(ArgumentMatchers.any())) + .thenReturn(List.of()); + Mockito.when(runtime.definitions()) + .thenReturn(new DatacenterFederationDefinitionFactory(catalogMapper)); + DatacenterFederationChangeNotifier notifier = + Mockito.mock(DatacenterFederationChangeNotifier.class); + DatacenterSourceMapper sourceMapper = Mockito.mock(DatacenterSourceMapper.class); + DatacenterSourceMetadataService metadataService = + new DatacenterSourceMetadataService( + connectorRegistry, registry, catalogMapper); + DatacenterSourceCandidateService candidateService = + new DatacenterSourceCandidateService(cipher, defaults, runtime); + PlatformTransactionManager transactionManager = + Mockito.mock(PlatformTransactionManager.class); + Mockito.when(transactionManager.getTransaction( + ArgumentMatchers.any(TransactionDefinition.class))) + .thenReturn(Mockito.mock(TransactionStatus.class)); + DatacenterSourceMetadataRefreshService metadataRefreshService = + new DatacenterSourceMetadataRefreshService( + connectorRegistry, registry, catalogMapper, tableMapper, + sourceMapper, notifier, transactionManager); + DatacenterSourceServiceImpl service = new DatacenterSourceServiceImpl( + connectorRegistry, registry, catalogMapper, tableMapper, + defaults, candidateService, metadataService, + metadataRefreshService, runtime, notifier, transactionManager); + Field mapper = ServiceImpl.class.getDeclaredField("mapper"); + mapper.setAccessible(true); + mapper.set(service, sourceMapper); + return new Fixture( + service, connectorRegistry, registry, catalogMapper, + sourceMapper, runtime, notifier, defaults); + } + + private DatacenterSource source( + DatacenterSourceStatus status, + long definitionRevision, + long scopeRevision) { + DatacenterSource source = new DatacenterSource(); + source.setId(BigInteger.valueOf(100L)); + source.setTenantId(BigInteger.ONE); + source.setSourceType("MYSQL"); + source.setStatus(status.code()); + source.setDefinitionRevision(definitionRevision); + source.setScopeRevision(scopeRevision); + return source; + } + + private LoginAccount account() { + LoginAccount account = new LoginAccount(); + account.setId(BigInteger.TEN); + account.setTenantId(BigInteger.ONE); + return account; + } + + private record Fixture( + DatacenterSourceServiceImpl service, + DatacenterConnectorRegistry connectorRegistry, + DatacenterDatasetRegistryService registry, + DatacenterCatalogMapper catalogMapper, + DatacenterSourceMapper sourceMapper, + DatacenterFederationRuntime runtime, + DatacenterFederationChangeNotifier notifier, + DatacenterSourceConnectionDefaults defaults) { + } +} diff --git a/easyflow-modules/easyflow-module-datacenter/src/test/java/tech/easyflow/datacenter/meta/support/DatacenterSourceConnectionDefaultsTest.java b/easyflow-modules/easyflow-module-datacenter/src/test/java/tech/easyflow/datacenter/meta/support/DatacenterSourceConnectionDefaultsTest.java new file mode 100644 index 00000000..da908018 --- /dev/null +++ b/easyflow-modules/easyflow-module-datacenter/src/test/java/tech/easyflow/datacenter/meta/support/DatacenterSourceConnectionDefaultsTest.java @@ -0,0 +1,53 @@ +package tech.easyflow.datacenter.meta.support; + +import java.util.Map; +import org.junit.Assert; +import org.junit.Test; +import tech.easyflow.datacenter.meta.entity.DatacenterSource; + +/** + * {@link DatacenterSourceConnectionDefaults} 安全连接默认值回归测试。 + */ +public class DatacenterSourceConnectionDefaultsTest { + + private final DatacenterSourceConnectionDefaults defaults = + new DatacenterSourceConnectionDefaults(); + + /** + * 验证可能承载凭据的配置项不会进入持久化配置或 Definition。 + */ + @Test + public void shouldRemoveSensitiveOptions() { + Map sanitized = defaults.sanitizeConfig(Map.of( + "socketTimeout", 30, + "password", "plain", + "apiToken", "token", + "clientSecret", "secret", + "privateKey", "key")); + + Assert.assertEquals(Map.of("socketTimeout", 30), sanitized); + } + + /** + * 验证首批 MySQL 与 PostgreSQL URL 默认具有有界连接和查询等待。 + */ + @Test + public void shouldGenerateBoundedJdbcUrls() { + DatacenterSource mysql = source("MYSQL", 3306, "easyflow"); + DatacenterSource postgres = source("POSTGRESQL", 5432, "harmony_adapter"); + + Assert.assertTrue(defaults.buildJdbcUrl(mysql).contains("connectTimeout=5000")); + Assert.assertTrue(defaults.buildJdbcUrl(mysql).contains("socketTimeout=35000")); + Assert.assertTrue(defaults.buildJdbcUrl(postgres).contains("connectTimeout=5")); + Assert.assertTrue(defaults.buildJdbcUrl(postgres).contains("socketTimeout=35")); + } + + private DatacenterSource source(String type, int port, String databaseName) { + DatacenterSource source = new DatacenterSource(); + source.setSourceType(type); + source.setHost("127.0.0.1"); + source.setPort(port); + source.setDatabaseName(databaseName); + return source; + } +} diff --git a/easyflow-modules/easyflow-module-datacenter/src/test/java/tech/easyflow/datacenter/security/DatacenterCredentialCipherTest.java b/easyflow-modules/easyflow-module-datacenter/src/test/java/tech/easyflow/datacenter/security/DatacenterCredentialCipherTest.java new file mode 100644 index 00000000..bdf26c48 --- /dev/null +++ b/easyflow-modules/easyflow-module-datacenter/src/test/java/tech/easyflow/datacenter/security/DatacenterCredentialCipherTest.java @@ -0,0 +1,45 @@ +package tech.easyflow.datacenter.security; + +import org.junit.Assert; +import org.junit.Test; + +/** + * {@link DatacenterCredentialCipher} 认证加密回归测试。 + */ +public class DatacenterCredentialCipherTest { + + /** + * 验证随机 IV、正确解密和篡改检测。 + */ + @Test + public void shouldEncryptWithRandomIvAndRejectTampering() { + DatacenterCredentialCipher cipher = new DatacenterCredentialCipher( + "test-only-master-key-with-at-least-32-characters"); + + String first = cipher.encrypt("secret-value"); + String second = cipher.encrypt("secret-value"); + + Assert.assertNotEquals(first, second); + Assert.assertEquals("secret-value", cipher.decrypt(first)); + String tampered = first.substring(0, first.length() - 2) + "AA"; + try { + cipher.decrypt(tampered); + Assert.fail("tampered credential must be rejected"); + } catch (IllegalStateException expected) { + Assert.assertTrue(expected.getMessage().contains("decrypt")); + } + } + + /** + * 验证部署未提供足够强度的主密钥时立即停止启动。 + */ + @Test + public void shouldRejectMissingMasterKey() { + try { + new DatacenterCredentialCipher("too-short"); + Assert.fail("short master key must be rejected"); + } catch (IllegalStateException expected) { + Assert.assertTrue(expected.getMessage().contains("32")); + } + } +} diff --git a/easyflow-modules/easyflow-module-datacenter/src/test/java/tech/easyflow/datacenter/utils/SqlSupportUtilsTest.java b/easyflow-modules/easyflow-module-datacenter/src/test/java/tech/easyflow/datacenter/utils/SqlSupportUtilsTest.java index 2673cb13..5ff12773 100644 --- a/easyflow-modules/easyflow-module-datacenter/src/test/java/tech/easyflow/datacenter/utils/SqlSupportUtilsTest.java +++ b/easyflow-modules/easyflow-module-datacenter/src/test/java/tech/easyflow/datacenter/utils/SqlSupportUtilsTest.java @@ -2,6 +2,7 @@ package tech.easyflow.datacenter.utils; import org.junit.Assert; import org.junit.Test; +import tech.easyflow.common.web.exceptions.BusinessException; import java.util.List; @@ -24,7 +25,9 @@ public class SqlSupportUtilsTest { List.of(new SqlSupportUtils.ManagedTable( "ama 实验基线模型预算", "Sheet1", - "tb_excel_budget"))); + "tb_excel_budget", + List.of("token", "token_1", "col_id"), + List.of("token", "token_1", "col_id")))); Assert.assertEquals( "SELECT `token`, `token_1` FROM tb_excel_budget " @@ -42,10 +45,564 @@ public class SqlSupportUtilsTest { List.of(new SqlSupportUtils.ManagedTable( null, "Sheet1", - "tb_excel_budget"))); + "tb_excel_budget", + List.of("token", "col_id"), + List.of("token", "col_id")))); Assert.assertEquals( "SELECT `token` FROM tb_excel_budget WHERE `col_id` = 'deep\"seek'", resolved.getExecutableSql()); } + + /** + * 验证 PostgreSQL 引号标识符不会与未引号的小写对象混为同一张表。 + */ + @Test(expected = BusinessException.class) + public void resolveShouldNotFoldQuotedManagedTableToUnquotedReference() { + SqlSupportUtils.resolve( + "SELECT * FROM foo", + List.of(new SqlSupportUtils.ManagedTable( + null, + "Foo", + "Foo"))); + } + + /** + * 验证大小写敏感表可通过精确引号标识符访问。 + */ + @Test + public void resolveShouldMatchQuotedManagedTableExactly() { + SqlSupportUtils.ResolvedSql resolved = SqlSupportUtils.resolve( + "SELECT * FROM \"Foo\"", + List.of(new SqlSupportUtils.ManagedTable( + null, + "Foo", + "Foo"))); + + Assert.assertEquals(List.of("Foo"), resolved.getLogicalTables()); + } + + /** + * 验证内部 SQL 无法读取未开放字段。 + */ + @Test(expected = BusinessException.class) + public void resolveInternalMysqlShouldRejectHiddenColumn() { + SqlSupportUtils.resolveInternalMysql( + "SELECT secret FROM Sheet1", + List.of(internalTable( + List.of("public_col", "secret"), + List.of("public_col")))); + } + + /** + * 验证 IS NULL 条件中的隐藏字段仍受字段策略约束。 + */ + @Test(expected = BusinessException.class) + public void resolveInternalMysqlShouldRejectHiddenColumnInIsNull() { + SqlSupportUtils.resolveInternalMysql( + "SELECT public_col FROM \"Sheet1\" WHERE secret IS NULL", + List.of(internalTable( + List.of("public_col", "secret"), + List.of("public_col")))); + } + + /** + * 验证排序条件中的隐藏字段仍受字段策略约束。 + */ + @Test(expected = BusinessException.class) + public void resolveInternalMysqlShouldRejectHiddenColumnInOrderBy() { + SqlSupportUtils.resolveInternalMysql( + "SELECT public_col FROM \"Sheet1\" ORDER BY secret", + List.of(internalTable( + List.of("public_col", "secret"), + List.of("public_col")))); + } + + /** + * 验证分组条件中的隐藏字段仍受字段策略约束。 + */ + @Test(expected = BusinessException.class) + public void resolveInternalMysqlShouldRejectHiddenColumnInGroupBy() { + SqlSupportUtils.resolveInternalMysql( + "SELECT COUNT(*) FROM \"Sheet1\" GROUP BY secret", + List.of(internalTable( + List.of("public_col", "secret"), + List.of("public_col")))); + } + + /** + * 验证窗口排序中的隐藏字段仍受字段策略约束。 + */ + @Test(expected = BusinessException.class) + public void resolveInternalMysqlShouldRejectHiddenColumnInWindowOrderBy() { + SqlSupportUtils.resolveInternalMysql( + "SELECT public_col, ROW_NUMBER() OVER (ORDER BY secret) AS rn " + + "FROM \"Sheet1\"", + List.of(internalTable( + List.of("public_col", "secret"), + List.of("public_col")))); + } + + /** + * 验证命名窗口排序中的隐藏字段仍受字段策略约束。 + */ + @Test(expected = BusinessException.class) + public void resolveInternalMysqlShouldRejectHiddenColumnInNamedWindow() { + SqlSupportUtils.resolveInternalMysql( + "SELECT public_col, ROW_NUMBER() OVER w AS rn FROM \"Sheet1\" " + + "WINDOW w AS (ORDER BY secret)", + List.of(internalTable( + List.of("public_col", "secret"), + List.of("public_col")))); + } + + /** + * 验证合法窗口 frame 可以安全解析且不会触发空值异常。 + */ + @Test + public void resolveInternalMysqlShouldAllowSafeWindowFrame() { + SqlSupportUtils.ResolvedSql resolved = SqlSupportUtils.resolveInternalMysql( + "SELECT SUM(public_col) OVER (ORDER BY public_col " + + "ROWS BETWEEN 1 PRECEDING AND CURRENT ROW) AS total " + + "FROM \"Sheet1\"", + List.of(internalTable( + List.of("public_col", "secret"), + List.of("public_col")))); + + Assert.assertEquals( + "SELECT SUM(public_col) OVER (ORDER BY public_col ROWS BETWEEN 1 PRECEDING " + + "AND CURRENT ROW) AS total FROM tb_excel_budget", + resolved.getExecutableSql()); + } + + /** + * 验证 HAVING 中包装的合法投影别名不会被误判为物理字段。 + */ + @Test + public void resolveInternalMysqlShouldAllowOutputAliasInHavingExpression() { + SqlSupportUtils.ResolvedSql resolved = SqlSupportUtils.resolveInternalMysql( + "SELECT public_col AS visible FROM \"Sheet1\" " + + "HAVING visible IS NOT NULL", + List.of(internalTable( + List.of("public_col", "secret"), + List.of("public_col")))); + + Assert.assertEquals( + "SELECT public_col AS visible FROM tb_excel_budget " + + "HAVING visible IS NOT NULL", + resolved.getExecutableSql()); + } + + /** + * 验证 ORDER BY 包装表达式中的合法投影别名可以使用。 + */ + @Test + public void resolveInternalMysqlShouldAllowOutputAliasInWrappedOrderBy() { + SqlSupportUtils.ResolvedSql resolved = SqlSupportUtils.resolveInternalMysql( + "SELECT public_col AS visible FROM \"Sheet1\" " + + "ORDER BY COALESCE(visible, '')", + List.of(internalTable( + List.of("public_col", "secret"), + List.of("public_col")))); + + Assert.assertEquals( + "SELECT public_col AS visible FROM tb_excel_budget " + + "ORDER BY COALESCE(visible, '')", + resolved.getExecutableSql()); + } + + /** + * 验证包装排序表达式不能放行隐藏物理字段。 + */ + @Test(expected = BusinessException.class) + public void resolveInternalMysqlShouldRejectHiddenColumnInWrappedOrderBy() { + SqlSupportUtils.resolveInternalMysql( + "SELECT public_col FROM \"Sheet1\" ORDER BY COALESCE(secret, '')", + List.of(internalTable( + List.of("public_col", "secret"), + List.of("public_col")))); + } + + /** + * 验证 IS TRUE 条件中的隐藏字段仍受字段策略约束。 + */ + @Test(expected = BusinessException.class) + public void resolveInternalMysqlShouldRejectHiddenColumnInIsBoolean() { + SqlSupportUtils.resolveInternalMysql( + "SELECT public_col FROM \"Sheet1\" WHERE secret IS TRUE", + List.of(internalTable( + List.of("public_col", "secret"), + List.of("public_col")))); + } + + /** + * 验证 QUALIFY 条件中的隐藏字段仍受字段策略约束。 + */ + @Test(expected = BusinessException.class) + public void resolveInternalMysqlShouldRejectHiddenColumnInQualify() { + SqlSupportUtils.resolveInternalMysql( + "SELECT public_col FROM \"Sheet1\" QUALIFY secret IS NOT NULL", + List.of(internalTable( + List.of("public_col", "secret"), + List.of("public_col")))); + } + + /** + * 验证隐藏字段不能通过同名输出别名绕过字段白名单。 + */ + @Test(expected = BusinessException.class) + public void resolveInternalMysqlShouldRejectHiddenColumnWithSameAlias() { + SqlSupportUtils.resolveInternalMysql( + "SELECT secret AS secret FROM \"Sheet1\"", + List.of(internalTable( + List.of("public_col", "secret"), + List.of("public_col")))); + } + + /** + * 验证存在未开放字段时禁止星号扩展。 + */ + @Test(expected = BusinessException.class) + public void resolveInternalMysqlShouldRejectStarWithHiddenColumns() { + SqlSupportUtils.resolveInternalMysql( + "SELECT * FROM Sheet1", + List.of(internalTable( + List.of("public_col", "secret"), + List.of("public_col")))); + } + + /** + * 验证括号包裹的 JOIN 仍会将星号绑定到底层物理表。 + */ + @Test(expected = BusinessException.class) + public void resolveInternalMysqlShouldRejectStarOverParenthesizedJoinWithHiddenColumn() { + SqlSupportUtils.resolveInternalMysql( + "SELECT * FROM (Sheet1 CROSS JOIN Sheet2)", + List.of( + internalTable( + List.of("public_col", "secret"), + List.of("public_col")), + new SqlSupportUtils.ManagedTable( + null, + "Sheet2", + "tb_excel_public", + List.of("other_col"), + List.of("other_col")))); + } + + /** + * 验证聚合函数中的星号不代表读取全部物理字段。 + */ + @Test + public void resolveInternalMysqlShouldAllowCountStarWithHiddenColumns() { + SqlSupportUtils.ResolvedSql resolved = SqlSupportUtils.resolveInternalMysql( + "SELECT COUNT(*) FROM \"Sheet1\"", + List.of(internalTable( + List.of("public_col", "secret"), + List.of("public_col")))); + + Assert.assertEquals( + "SELECT COUNT(*) FROM tb_excel_budget", + resolved.getExecutableSql()); + } + + /** + * 验证表别名和输出别名不会被误判为物理字段。 + */ + @Test + public void resolveInternalMysqlShouldAllowSafeAliases() { + SqlSupportUtils.ResolvedSql resolved = SqlSupportUtils.resolveInternalMysql( + "SELECT s.public_col AS display_col FROM \"Sheet1\" s ORDER BY display_col", + List.of(internalTable( + List.of("public_col"), + List.of("public_col")))); + + Assert.assertEquals( + "SELECT s.public_col AS display_col FROM tb_excel_budget s " + + "ORDER BY display_col", + resolved.getExecutableSql()); + } + + /** + * 验证内层子查询的同名字段不会污染外层合法输出别名。 + */ + @Test + public void resolveInternalMysqlShouldScopeOutputAliasToCurrentQueryBlock() { + SqlSupportUtils.ResolvedSql resolved = SqlSupportUtils.resolveInternalMysql( + "SELECT public_col AS secret FROM \"Sheet1\" " + + "WHERE EXISTS (SELECT 1 FROM \"Sheet2\") ORDER BY secret", + List.of( + internalTable( + List.of("public_col"), + List.of("public_col")), + new SqlSupportUtils.ManagedTable( + null, + "Sheet2", + "tb_excel_private", + List.of("secret"), + List.of()))); + + Assert.assertEquals( + "SELECT public_col AS secret FROM tb_excel_budget WHERE EXISTS " + + "(SELECT 1 FROM tb_excel_private) ORDER BY secret", + resolved.getExecutableSql()); + } + + /** + * 验证内层子查询的物理表不会参与外层未限定字段授权判断。 + */ + @Test + public void resolveInternalMysqlShouldScopeUnqualifiedColumnToCurrentQueryBlock() { + SqlSupportUtils.ResolvedSql resolved = SqlSupportUtils.resolveInternalMysql( + "SELECT public_col FROM \"Sheet1\" " + + "WHERE EXISTS (SELECT 1 FROM \"Sheet2\")", + List.of( + internalTable( + List.of("public_col"), + List.of("public_col")), + new SqlSupportUtils.ManagedTable( + null, + "Sheet2", + "tb_excel_private", + List.of("secret"), + List.of()))); + + Assert.assertEquals( + "SELECT public_col FROM tb_excel_budget WHERE EXISTS " + + "(SELECT 1 FROM tb_excel_private)", + resolved.getExecutableSql()); + } + + /** + * 验证派生表输出别名可以在外层查询中使用。 + */ + @Test + public void resolveInternalMysqlShouldAllowDerivedTableOutputAlias() { + SqlSupportUtils.ResolvedSql resolved = SqlSupportUtils.resolveInternalMysql( + "SELECT d.order_id FROM (SELECT public_col AS order_id FROM \"Sheet1\") d", + List.of(internalTable( + List.of("public_col", "secret"), + List.of("public_col")))); + + Assert.assertEquals( + "SELECT d.order_id FROM (SELECT public_col AS order_id " + + "FROM tb_excel_budget) d", + resolved.getExecutableSql()); + } + + /** + * 验证 CTE 输出别名可以在主查询中使用。 + */ + @Test + public void resolveInternalMysqlShouldAllowCteOutputAlias() { + SqlSupportUtils.ResolvedSql resolved = SqlSupportUtils.resolveInternalMysql( + "WITH x AS (SELECT public_col AS order_id FROM \"Sheet1\") SELECT order_id FROM x", + List.of(internalTable( + List.of("public_col", "secret"), + List.of("public_col")))); + + Assert.assertEquals( + "WITH x AS (SELECT public_col AS order_id FROM tb_excel_budget) " + + "SELECT order_id FROM x", + resolved.getExecutableSql()); + } + + /** + * 验证外层星号只展开 CTE 已校验的输出字段。 + */ + @Test + public void resolveInternalMysqlShouldAllowStarOverCteOutput() { + SqlSupportUtils.ResolvedSql resolved = SqlSupportUtils.resolveInternalMysql( + "WITH x AS (SELECT public_col FROM \"Sheet1\") SELECT * FROM x", + List.of(internalTable( + List.of("public_col", "secret"), + List.of("public_col")))); + + Assert.assertEquals( + "WITH x AS (SELECT public_col FROM tb_excel_budget) SELECT * FROM x", + resolved.getExecutableSql()); + } + + /** + * 验证带 Schema 的物理表不能伪装成同名 CTE 绕过表白名单。 + */ + @Test(expected = BusinessException.class) + public void resolveInternalMysqlShouldRejectSchemaQualifiedTableShadowingCte() { + SqlSupportUtils.resolveInternalMysql( + "WITH x AS (SELECT public_col AS secret FROM \"Sheet1\") " + + "SELECT x.secret FROM private_db.x", + List.of(internalTable( + List.of("public_col", "secret"), + List.of("public_col")))); + } + + /** + * 验证后置 CTE 可以引用同一 WITH 列表中的前置 CTE。 + */ + @Test + public void resolveInternalMysqlShouldAllowChainedCtes() { + SqlSupportUtils.ResolvedSql resolved = SqlSupportUtils.resolveInternalMysql( + "WITH x AS (SELECT public_col AS a FROM \"Sheet1\"), " + + "y AS (SELECT a AS b FROM x) SELECT b FROM y", + List.of(internalTable( + List.of("public_col", "secret"), + List.of("public_col")))); + + Assert.assertEquals( + "WITH x AS (SELECT public_col AS a FROM tb_excel_budget), " + + "y AS (SELECT a AS b FROM x) SELECT b FROM y", + resolved.getExecutableSql()); + } + + /** + * 验证 CTE 显式列清单可以作为主查询输出字段使用。 + */ + @Test + public void resolveInternalMysqlShouldAllowExplicitCteColumnList() { + SqlSupportUtils.ResolvedSql resolved = SqlSupportUtils.resolveInternalMysql( + "WITH x(order_id) AS (SELECT public_col FROM \"Sheet1\") SELECT order_id FROM x", + List.of(internalTable( + List.of("public_col", "secret"), + List.of("public_col")))); + + Assert.assertEquals( + "WITH x (order_id) AS (SELECT public_col FROM tb_excel_budget) " + + "SELECT order_id FROM x", + resolved.getExecutableSql()); + } + + /** + * 验证派生表别名不能洗白内部不可查询字段。 + */ + @Test(expected = BusinessException.class) + public void resolveInternalMysqlShouldRejectHiddenColumnInsideDerivedTable() { + SqlSupportUtils.resolveInternalMysql( + "SELECT d.order_id FROM (SELECT secret AS order_id FROM \"Sheet1\") d", + List.of(internalTable( + List.of("public_col", "secret"), + List.of("public_col")))); + } + + /** + * 验证 CTE 输出别名不能洗白内部不可查询字段。 + */ + @Test(expected = BusinessException.class) + public void resolveInternalMysqlShouldRejectHiddenColumnInsideCte() { + SqlSupportUtils.resolveInternalMysql( + "WITH x AS (SELECT secret AS order_id FROM \"Sheet1\") SELECT order_id FROM x", + List.of(internalTable( + List.of("public_col", "secret"), + List.of("public_col")))); + } + + /** + * 验证外层派生表别名不会污染内层同名物理表别名。 + */ + @Test(expected = BusinessException.class) + public void resolveInternalMysqlShouldNotLeakDerivedAliasAcrossNestedScope() { + SqlSupportUtils.resolveInternalMysql( + "SELECT d.secret FROM (SELECT public_col AS secret FROM \"Sheet1\") d " + + "WHERE EXISTS (SELECT 1 FROM \"Sheet1\" d WHERE d.secret = 1)", + List.of(internalTable( + List.of("public_col", "secret"), + List.of("public_col")))); + } + + /** + * 验证相关子查询可以引用外层派生关系的已校验输出字段。 + */ + @Test + public void resolveInternalMysqlShouldAllowCorrelatedOuterDerivedColumn() { + SqlSupportUtils.ResolvedSql resolved = SqlSupportUtils.resolveInternalMysql( + "SELECT d.order_id FROM (SELECT public_col AS order_id FROM \"Sheet1\") d " + + "WHERE EXISTS (SELECT 1 WHERE d.order_id IS NOT NULL)", + List.of(internalTable( + List.of("public_col", "secret"), + List.of("public_col")))); + + Assert.assertEquals( + "SELECT d.order_id FROM (SELECT public_col AS order_id " + + "FROM tb_excel_budget) d WHERE EXISTS " + + "(SELECT 1 WHERE d.order_id IS NOT NULL)", + resolved.getExecutableSql()); + } + + /** + * 验证内层物理表别名会遮蔽外层派生关系，不能借外层输出绕过字段策略。 + */ + @Test(expected = BusinessException.class) + public void resolveInternalMysqlShouldRejectCorrelatedDerivedAliasShadowedByInnerPhysicalAlias() { + SqlSupportUtils.resolveInternalMysql( + "SELECT d.order_id FROM (SELECT public_col AS order_id FROM \"Sheet1\") d " + + "WHERE EXISTS (SELECT 1 FROM \"Sheet1\" d " + + "WHERE d.order_id IS NOT NULL)", + List.of(internalTable( + List.of("public_col", "secret"), + List.of("public_col")))); + } + + /** + * 验证不同查询块的同名物理别名不会覆盖彼此的字段授权范围。 + */ + @Test(expected = BusinessException.class) + public void resolveInternalMysqlShouldIsolatePhysicalAliasesAcrossScopes() { + SqlSupportUtils.resolveInternalMysql( + "SELECT x.secret FROM \"Sheet1\" x WHERE EXISTS " + + "(SELECT 1 FROM \"Sheet2\" x WHERE x.secret IS NOT NULL)", + List.of( + new SqlSupportUtils.ManagedTable( + null, + "Sheet1", + "tb_excel_budget", + List.of("public_col", "secret"), + List.of("public_col")), + new SqlSupportUtils.ManagedTable( + null, + "Sheet2", + "tb_excel_public", + List.of("secret"), + List.of("secret")))); + } + + /** + * 验证外层输出别名不能污染内层作用域并放行隐藏字段。 + */ + @Test(expected = BusinessException.class) + public void resolveInternalMysqlShouldRejectNestedHiddenColumnShadowedByOuterAlias() { + SqlSupportUtils.resolveInternalMysql( + "SELECT public_col AS secret FROM Sheet1 " + + "WHERE public_col = (SELECT public_col FROM Sheet1 " + + "ORDER BY secret LIMIT 1)", + List.of(internalTable( + List.of("public_col", "secret"), + List.of("public_col")))); + } + + /** + * 验证内部 SQL 拒绝具有等待或资源消耗风险的函数。 + */ + @Test(expected = BusinessException.class) + public void resolveInternalMysqlShouldRejectDangerousFunction() { + SqlSupportUtils.resolveInternalMysql( + "SELECT SLEEP(10) FROM Sheet1", + List.of(internalTable(List.of("public_col"), List.of("public_col")))); + } + + /** + * 创建内部动态表测试定义。 + * + * @param knownColumns 已纳管字段 + * @param queryableColumns 可查询字段 + * @return 测试表定义 + */ + private SqlSupportUtils.ManagedTable internalTable( + List knownColumns, + List queryableColumns) { + return new SqlSupportUtils.ManagedTable( + null, + "Sheet1", + "tb_excel_budget", + knownColumns, + queryableColumns); + } } diff --git a/easyflow-starter/easyflow-starter-all/src/main/resources/db/migration/mysql/V58__mysql_datacenter_federation_refactor.sql b/easyflow-starter/easyflow-starter-all/src/main/resources/db/migration/mysql/V58__mysql_datacenter_federation_refactor.sql new file mode 100644 index 00000000..97a819e4 --- /dev/null +++ b/easyflow-starter/easyflow-starter-all/src/main/resources/db/migration/mysql/V58__mysql_datacenter_federation_refactor.sql @@ -0,0 +1,152 @@ +-- XL01 WP1: 数据中枢 Federation SQL 数据契约。 +-- 历史数据中枢表继续保留；本迁移只做可回填的增量调整。 + +ALTER TABLE `tb_datacenter_source` + ADD COLUMN `adapter_id` varchar(64) NULL COMMENT 'Federation Adapter 标识' AFTER `source_type`, + ADD COLUMN `adapter_options_json` text NULL COMMENT '不含凭据的 Adapter 选项' AFTER `adapter_id`, + ADD COLUMN `definition_revision` bigint UNSIGNED NOT NULL DEFAULT 0 COMMENT '数据源 Definition 版本' AFTER `adapter_options_json`, + ADD COLUMN `definition_checksum` char(64) NULL COMMENT 'Definition SHA-256 校验和' AFTER `definition_revision`, + ADD COLUMN `scope_revision` bigint UNSIGNED NOT NULL DEFAULT 1 COMMENT '纳管范围版本' AFTER `definition_checksum`, + ADD COLUMN `compatibility_status` varchar(32) NULL COMMENT 'Adapter 兼容状态' AFTER `scope_revision`, + ADD COLUMN `database_product_name` varchar(128) NULL COMMENT '数据库产品名称' AFTER `compatibility_status`, + ADD COLUMN `database_product_version` varchar(128) NULL COMMENT '数据库产品版本' AFTER `database_product_name`, + ADD COLUMN `driver_name` varchar(128) NULL COMMENT 'JDBC Driver 名称' AFTER `database_product_version`, + ADD COLUMN `driver_version` varchar(128) NULL COMMENT 'JDBC Driver 版本' AFTER `driver_name`, + ADD COLUMN `metadata_refresh_status` varchar(32) NULL COMMENT '元数据刷新状态' AFTER `driver_version`, + ADD COLUMN `metadata_refreshed_at` datetime NULL COMMENT '元数据最近刷新时间' AFTER `metadata_refresh_status`; + +UPDATE `tb_datacenter_source` +SET `adapter_id` = CASE + WHEN `source_type` IN ('MYSQL', 'POSTGRESQL') THEN 'jdbc' + ELSE `adapter_id` + END, + `definition_revision` = CASE + WHEN `source_type` = 'PROJECT_MYSQL' THEN 1 + ELSE `definition_revision` + END, + `status` = CASE + WHEN `source_type` = 'PROJECT_MYSQL' THEN 1 + ELSE `status` + END, + `metadata_refresh_status` = COALESCE(`metadata_refresh_status`, 'PENDING'); + +ALTER TABLE `tb_datacenter_source` + ADD UNIQUE KEY `uk_datacenter_source_tenant_code` (`tenant_id`, `source_code`), + ADD KEY `idx_datacenter_source_runtime` (`status`, `definition_revision`); + +ALTER TABLE `tb_datacenter_catalog` + ADD COLUMN `logical_schema_name` varchar(128) NULL COMMENT 'Calcite 逻辑 Schema' AFTER `catalog_type`, + ADD COLUMN `physical_catalog_name` varchar(128) NULL COMMENT 'JDBC 物理 Catalog' AFTER `logical_schema_name`, + ADD COLUMN `physical_schema_name` varchar(128) NULL COMMENT 'JDBC 物理 Schema' AFTER `physical_catalog_name`, + ADD COLUMN `namespace_key` char(64) NULL COMMENT '物理命名空间稳定摘要' AFTER `physical_schema_name`, + ADD COLUMN `metadata_status` varchar(32) NOT NULL DEFAULT 'ACTIVE' COMMENT '元数据状态' AFTER `namespace_key`, + ADD COLUMN `last_seen_at` datetime NULL COMMENT '最近发现时间' AFTER `metadata_status`; + +UPDATE `tb_datacenter_catalog` +SET `logical_schema_name` = `catalog_name`, + `physical_catalog_name` = `catalog_name`, + `namespace_key` = SHA2(CONCAT( + 'C', CHAR_LENGTH(`catalog_name`), ':', `catalog_name`, + 'S', '-1:' + ), 256), + `last_seen_at` = COALESCE(`modified`, `created`) +WHERE `logical_schema_name` IS NULL OR `namespace_key` IS NULL; + +ALTER TABLE `tb_datacenter_catalog` + MODIFY COLUMN `logical_schema_name` varchar(128) NOT NULL COMMENT 'Calcite 逻辑 Schema', + MODIFY COLUMN `namespace_key` char(64) NOT NULL COMMENT '物理命名空间稳定摘要', + ADD UNIQUE KEY `uk_datacenter_catalog_logical_schema` (`source_id`, `logical_schema_name`), + ADD UNIQUE KEY `uk_datacenter_catalog_namespace` (`source_id`, `namespace_key`); + +ALTER TABLE `tb_datacenter_table` + MODIFY COLUMN `table_name` varchar(128) NOT NULL COMMENT '数据表名', + ADD COLUMN `physical_identity_key` char(64) NULL COMMENT '物理对象稳定摘要' AFTER `actual_table`, + ADD COLUMN `metadata_fingerprint` char(64) NULL COMMENT '表结构摘要' AFTER `physical_identity_key`, + ADD COLUMN `metadata_status` varchar(32) NOT NULL DEFAULT 'ACTIVE' COMMENT '元数据状态' AFTER `metadata_fingerprint`, + ADD COLUMN `last_seen_at` datetime NULL COMMENT '最近发现时间' AFTER `metadata_status`, + ADD COLUMN `queryable` tinyint NOT NULL DEFAULT 1 COMMENT '是否允许统一只读查询' AFTER `last_seen_at`, + ADD COLUMN `time_semantics` varchar(32) NOT NULL DEFAULT 'NONE' COMMENT '时间语义' AFTER `queryable`, + ADD COLUMN `default_time_field_id` bigint UNSIGNED NULL COMMENT '默认时间字段 ID' AFTER `time_semantics`, + ADD COLUMN `metadata_revision` bigint UNSIGNED NOT NULL DEFAULT 1 COMMENT '表元数据版本' AFTER `default_time_field_id`; + +UPDATE `tb_datacenter_table` +SET `actual_table` = COALESCE(NULLIF(`actual_table`, ''), `table_name`), + `physical_identity_key` = SHA2(CONCAT( + 'S', CHAR_LENGTH(CAST(COALESCE(`source_id`, 0) AS CHAR)), ':', COALESCE(`source_id`, 0), + 'C', CHAR_LENGTH(CAST(COALESCE(`catalog_id`, 0) AS CHAR)), ':', COALESCE(`catalog_id`, 0), + 'T', CHAR_LENGTH(COALESCE(NULLIF(`actual_table`, ''), `table_name`)), ':', COALESCE(NULLIF(`actual_table`, ''), `table_name`), + 'K', CHAR_LENGTH(`table_kind`), ':', `table_kind` + ), 256), + `metadata_fingerprint` = SHA2(CONCAT( + CHAR_LENGTH(`table_name`), ':', `table_name`, + CHAR_LENGTH(`table_kind`), ':', `table_kind` + ), 256), + `last_seen_at` = COALESCE(`modified`, `created`) +WHERE `physical_identity_key` IS NULL OR `metadata_fingerprint` IS NULL; + +ALTER TABLE `tb_datacenter_table` + MODIFY COLUMN `physical_identity_key` char(64) NOT NULL COMMENT '物理对象稳定摘要', + ADD UNIQUE KEY `uk_datacenter_table_physical_identity` (`physical_identity_key`), + ADD KEY `idx_datacenter_table_metadata` (`source_id`, `catalog_id`, `metadata_status`); + +ALTER TABLE `tb_datacenter_table_field` + MODIFY COLUMN `field_name` varchar(128) NOT NULL COMMENT '字段名称', + ADD COLUMN `ordinal_position` int NULL COMMENT 'JDBC 字段顺序' AFTER `source_column_name`, + ADD COLUMN `jdbc_type_code` int NULL COMMENT 'java.sql.Types 数值' AFTER `jdbc_type`, + ADD COLUMN `native_type_name` varchar(128) NULL COMMENT '数据库原生类型名' AFTER `jdbc_type_code`, + ADD COLUMN `metadata_fingerprint` char(64) NULL COMMENT '字段元数据摘要' AFTER `scale`, + ADD COLUMN `metadata_status` varchar(32) NOT NULL DEFAULT 'ACTIVE' COMMENT '元数据状态' AFTER `metadata_fingerprint`, + ADD COLUMN `last_seen_at` datetime NULL COMMENT '最近发现时间' AFTER `metadata_status`, + ADD COLUMN `sensitivity_level` varchar(32) NOT NULL DEFAULT 'PUBLIC' COMMENT '敏感级别' AFTER `last_seen_at`, + ADD COLUMN `masking_strategy` varchar(32) NULL COMMENT '脱敏策略' AFTER `sensitivity_level`; + +UPDATE `tb_datacenter_table_field` +SET `source_column_name` = COALESCE(NULLIF(`source_column_name`, ''), `field_name`), + `native_type_name` = COALESCE(`native_type_name`, `jdbc_type`), + `metadata_fingerprint` = SHA2(CONCAT( + CHAR_LENGTH(`field_name`), ':', `field_name`, + CHAR_LENGTH(COALESCE(`jdbc_type`, '')), ':', COALESCE(`jdbc_type`, ''), + 'P', COALESCE(`precision`, -1), ':S', COALESCE(`scale`, -1), ':R', `required` + ), 256), + `last_seen_at` = COALESCE(`modified`, `created`), + `writable` = CASE WHEN `writable` IS NULL THEN 0 ELSE `writable` END +WHERE `metadata_fingerprint` IS NULL OR `source_column_name` IS NULL; + +ALTER TABLE `tb_datacenter_table_field` + MODIFY COLUMN `source_column_name` varchar(128) NOT NULL COMMENT '源字段名', + MODIFY COLUMN `metadata_fingerprint` char(64) NOT NULL COMMENT '字段元数据摘要', + ADD UNIQUE KEY `uk_datacenter_field_source_column` (`table_id`, `source_column_name`), + ADD KEY `idx_datacenter_field_metadata` (`table_id`, `metadata_status`, `ordinal_position`); + +CREATE TABLE `tb_datacenter_query_audit` +( + `id` bigint UNSIGNED NOT NULL COMMENT '主键', + `query_id` varchar(64) NOT NULL COMMENT '查询标识', + `tenant_id` bigint UNSIGNED NOT NULL DEFAULT 0 COMMENT '租户ID', + `dept_id` bigint UNSIGNED NOT NULL DEFAULT 0 COMMENT '部门ID', + `caller_type` varchar(32) NOT NULL COMMENT '调用方类型', + `caller_id` varchar(128) NULL COMMENT '调用方标识', + `source_id` bigint UNSIGNED NOT NULL COMMENT '数据源ID', + `source_revision` bigint UNSIGNED NOT NULL COMMENT '数据源版本', + `scope_revision` bigint UNSIGNED NOT NULL COMMENT '纳管范围版本', + `referenced_tables_json` text NULL COMMENT '引用表摘要', + `referenced_fields_json` text NULL COMMENT '引用字段摘要', + `parameterized_sql` longtext NOT NULL COMMENT '参数化或脱敏 SQL', + `masked_parameters_json` text NULL COMMENT '脱敏参数摘要', + `status` varchar(32) NOT NULL COMMENT '执行状态', + `error_code` varchar(64) NULL COMMENT '稳定错误码', + `error_message` varchar(500) NULL COMMENT '有界错误说明', + `started_at` datetime NOT NULL COMMENT '开始时间', + `finished_at` datetime NULL COMMENT '结束时间', + `duration_ms` bigint NULL COMMENT '总耗时毫秒', + `database_duration_ms` bigint NULL COMMENT '数据库耗时毫秒', + `returned_rows` bigint NOT NULL DEFAULT 0 COMMENT '返回行数', + `truncated` tinyint NOT NULL DEFAULT 0 COMMENT '是否截断', + `created` datetime NOT NULL COMMENT '创建时间', + PRIMARY KEY (`id`), + UNIQUE KEY `uk_datacenter_query_audit_query_id` (`query_id`), + KEY `idx_datacenter_query_audit_tenant_created` (`tenant_id`, `created`), + KEY `idx_datacenter_query_audit_source_created` (`source_id`, `created`), + KEY `idx_datacenter_query_audit_caller_created` (`caller_type`, `caller_id`, `created`), + KEY `idx_datacenter_query_audit_status_created` (`status`, `created`) +) ENGINE = InnoDB DEFAULT CHARSET = utf8mb4 COLLATE = utf8mb4_0900_ai_ci COMMENT = '数据中枢统一查询审计'; diff --git a/easyflow-starter/easyflow-starter-all/src/main/resources/db/migration/mysql/V59__mysql_datacenter_case_sensitive_identity.sql b/easyflow-starter/easyflow-starter-all/src/main/resources/db/migration/mysql/V59__mysql_datacenter_case_sensitive_identity.sql new file mode 100644 index 00000000..0f6d5b77 --- /dev/null +++ b/easyflow-starter/easyflow-starter-all/src/main/resources/db/migration/mysql/V59__mysql_datacenter_case_sensitive_identity.sql @@ -0,0 +1,19 @@ +ALTER TABLE `tb_datacenter_catalog` + MODIFY COLUMN `catalog_name` varchar(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL COMMENT '目录名称', + MODIFY COLUMN `logical_schema_name` varchar(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL COMMENT 'Calcite逻辑Schema', + MODIFY COLUMN `physical_catalog_name` varchar(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NULL COMMENT '物理Catalog', + MODIFY COLUMN `physical_schema_name` varchar(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NULL COMMENT '物理Schema'; + +ALTER TABLE `tb_datacenter_table` + MODIFY COLUMN `table_name` varchar(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL COMMENT '数据表名', + MODIFY COLUMN `actual_table` varchar(200) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NULL COMMENT '物理表名'; + +ALTER TABLE `tb_datacenter_table_field` + MODIFY COLUMN `field_name` varchar(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL COMMENT '字段名称', + MODIFY COLUMN `source_column_name` varchar(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL COMMENT '源字段名'; + +UPDATE `tb_datacenter_table_field` field_meta +JOIN `tb_datacenter_table` table_meta ON table_meta.`id` = field_meta.`table_id` +JOIN `tb_datacenter_source` source_meta ON source_meta.`id` = table_meta.`source_id` +SET field_meta.`writable` = 0 +WHERE source_meta.`source_type` IN ('MYSQL', 'POSTGRESQL'); diff --git a/easyflow-starter/easyflow-starter-all/src/main/resources/db/migration/mysql/V60__mysql_datacenter_query_audit_executor.sql b/easyflow-starter/easyflow-starter-all/src/main/resources/db/migration/mysql/V60__mysql_datacenter_query_audit_executor.sql new file mode 100644 index 00000000..2ca692b8 --- /dev/null +++ b/easyflow-starter/easyflow-starter-all/src/main/resources/db/migration/mysql/V60__mysql_datacenter_query_audit_executor.sql @@ -0,0 +1,4 @@ +ALTER TABLE `tb_datacenter_query_audit` + ADD COLUMN `executor_account_id` bigint UNSIGNED NULL COMMENT '执行账号ID' AFTER `dept_id`, + ADD KEY `idx_datacenter_query_audit_executor_created` + (`tenant_id`, `executor_account_id`, `created`); diff --git a/easyflow-ui-admin/app/src/views/ai/workflow/customNode/datasetNodeRenderer.ts b/easyflow-ui-admin/app/src/views/ai/workflow/customNode/datasetNodeRenderer.ts index 1c667290..c235d691 100644 --- a/easyflow-ui-admin/app/src/views/ai/workflow/customNode/datasetNodeRenderer.ts +++ b/easyflow-ui-admin/app/src/views/ai/workflow/customNode/datasetNodeRenderer.ts @@ -90,6 +90,7 @@ function createSourceOnlyDatasetRef( source: ManagedDatasetSourceOption, ): DatasetRefPayload { return { + tenantId: source.tables[0]?.datasetRef.tenantId ?? null, sourceId: source.sourceId, catalogId: null, catalogName: '', diff --git a/easyflow-ui-admin/app/src/views/ai/workflow/customNode/datasetOptions.ts b/easyflow-ui-admin/app/src/views/ai/workflow/customNode/datasetOptions.ts index a2ba1c6e..2888345c 100644 --- a/easyflow-ui-admin/app/src/views/ai/workflow/customNode/datasetOptions.ts +++ b/easyflow-ui-admin/app/src/views/ai/workflow/customNode/datasetOptions.ts @@ -1,6 +1,7 @@ import { api } from '#/api/request'; export interface DatasetRefPayload { + tenantId: null | number | string; sourceId: null | number | string; catalogId?: null | number | string; catalogName?: string; @@ -110,6 +111,7 @@ export async function loadManagedDatasetOptions(): Promise< catalogName: catalog.catalogName, tableName: table.tableName, datasetRef: { + tenantId: table.tenantId, sourceId: source.id, catalogId: catalog.id, catalogName: catalog.catalogName, diff --git a/easyflow-ui-admin/app/src/views/datacenter/DatacenterWorkspace.vue b/easyflow-ui-admin/app/src/views/datacenter/DatacenterWorkspace.vue index 23603ddc..0e76f576 100644 --- a/easyflow-ui-admin/app/src/views/datacenter/DatacenterWorkspace.vue +++ b/easyflow-ui-admin/app/src/views/datacenter/DatacenterWorkspace.vue @@ -1,9 +1,10 @@ diff --git a/easyflow-ui-admin/app/src/views/datacenter/components/SqlConsoleView.vue b/easyflow-ui-admin/app/src/views/datacenter/components/SqlConsoleView.vue new file mode 100644 index 00000000..97f70f8c --- /dev/null +++ b/easyflow-ui-admin/app/src/views/datacenter/components/SqlConsoleView.vue @@ -0,0 +1,287 @@ + + + + + diff --git a/easyflow-ui-admin/app/src/views/datacenter/components/TableDetailView.vue b/easyflow-ui-admin/app/src/views/datacenter/components/TableDetailView.vue index d752a9b0..fea6cbd4 100644 --- a/easyflow-ui-admin/app/src/views/datacenter/components/TableDetailView.vue +++ b/easyflow-ui-admin/app/src/views/datacenter/components/TableDetailView.vue @@ -7,6 +7,9 @@ import { ElEmpty, ElIcon, ElInput, + ElOption, + ElSelect, + ElSwitch, ElTable, ElTableColumn, ElTabPane, @@ -20,11 +23,20 @@ import { } from '../composables/datacenter-constants'; const props = defineProps<{ + fieldHasMore?: boolean; + fieldLoading?: boolean; + fieldPageNumber?: number; jobs: any[]; + loadError?: string; loading: boolean; previewRows: any[]; saveDescriptions: (payload: { - fields?: Array<{ fieldDesc: string; fieldId: number | string }>; + fields?: Array<{ + fieldDesc: string; + fieldId: number | string; + queryable?: number; + sensitivityLevel?: string; + }>; tableDesc?: string; tableId: number | string; }) => Promise; @@ -35,21 +47,46 @@ const props = defineProps<{ const emit = defineEmits<{ back: []; + fieldPageChange: [pageNumber: number]; + retry: []; }>(); const activeTab = ref('data'); const editingFieldId = ref(null); const editingFieldDesc = ref(''); +const editingFieldQueryable = ref(false); +const editingSensitivityLevel = ref('PUBLIC'); const savingFieldId = ref(null); +const sensitivityLabels: Record = { + INTERNAL: '内部', + PUBLIC: '公开', + RESTRICTED: '受限', + SENSITIVE: '敏感', +}; + function startFieldEdit(field: any) { editingFieldId.value = field.id; editingFieldDesc.value = field.fieldDesc || ''; + editingFieldQueryable.value = Number(field.queryable) === 1; + editingSensitivityLevel.value = field.sensitivityLevel || 'PUBLIC'; } function cancelFieldEdit() { editingFieldId.value = null; editingFieldDesc.value = ''; + editingFieldQueryable.value = false; + editingSensitivityLevel.value = 'PUBLIC'; +} + +function handleSensitivityChange(value: string) { + if (value !== 'PUBLIC') { + editingFieldQueryable.value = false; + } +} + +function formatSensitivityLevel(value?: string) { + return sensitivityLabels[value || 'PUBLIC'] || '公开'; } async function saveFieldDescription(field: any) { @@ -64,6 +101,8 @@ async function saveFieldDescription(field: any) { { fieldDesc: editingFieldDesc.value, fieldId: field.id, + queryable: editingFieldQueryable.value ? 1 : 0, + sensitivityLevel: editingSensitivityLevel.value, }, ], }); @@ -75,7 +114,7 @@ async function saveFieldDescription(field: any) { + + + + + + @@ -246,6 +327,30 @@ async function saveFieldDescription(field: any) { + +
+ + 上一页 + + 第 {{ fieldPageNumber || 1 }} 页 + + 下一页 + +
@@ -358,6 +463,17 @@ async function saveFieldDescription(field: any) { overflow: auto; } +.field-page-controls { + display: flex; + flex: 0 0 44px; + gap: 8px; + align-items: center; + justify-content: flex-end; + padding-right: 8px; + font-size: 13px; + color: hsl(var(--text-muted)); +} + .tag-flow { display: flex; flex-wrap: wrap; diff --git a/easyflow-ui-admin/app/src/views/datacenter/components/TableListView.vue b/easyflow-ui-admin/app/src/views/datacenter/components/TableListView.vue index 3a7e3d95..c18ef735 100644 --- a/easyflow-ui-admin/app/src/views/datacenter/components/TableListView.vue +++ b/easyflow-ui-admin/app/src/views/datacenter/components/TableListView.vue @@ -1,5 +1,5 @@