feat: 完善数据中枢联邦查询闭环
- 重构数据源生命周期、元数据纳管与运行时切换 - 增加只读 SQL、查询审计、跨节点取消与工作流联动 - 完善管理端连接配置、元数据浏览与 SQL 工作台
This commit is contained in:
@@ -24,6 +24,11 @@ import java.util.List;
|
||||
@RequestMapping("/api/v1/datacenterDataset")
|
||||
public class DatacenterDatasetController {
|
||||
|
||||
/** 对外 Schema 接口的默认字段页码。 */
|
||||
private static final long DEFAULT_FIELD_PAGE_NUMBER = 1L;
|
||||
/** 对外 Schema 接口的默认字段页大小。 */
|
||||
private static final long DEFAULT_FIELD_PAGE_SIZE = 200L;
|
||||
|
||||
@Resource
|
||||
private DatacenterDatasetQueryService queryService;
|
||||
@Resource
|
||||
@@ -32,13 +37,18 @@ public class DatacenterDatasetController {
|
||||
@PostMapping("/queryPage")
|
||||
@SaCheckPermission("/api/v1/datacenterSource/query")
|
||||
public Result<Page<Row>> queryPage(@RequestBody DatacenterQueryRequest request) {
|
||||
return Result.ok(queryService.queryPage(request));
|
||||
return Result.ok(queryService.queryPage(
|
||||
request, SaTokenUtil.getLoginAccount()));
|
||||
}
|
||||
|
||||
@GetMapping("/schema")
|
||||
@SaCheckPermission("/api/v1/datacenterSource/query")
|
||||
public Result<DatacenterSchemaResponse> schema(DatasetRef datasetRef) {
|
||||
return Result.ok(queryService.getSchema(datasetRef));
|
||||
public Result<DatacenterSchemaResponse> schema(
|
||||
DatasetRef datasetRef,
|
||||
@RequestParam(defaultValue = "1") Long fieldPageNumber,
|
||||
@RequestParam(defaultValue = "200") Long fieldPageSize) {
|
||||
return Result.ok(queryService.getSchema(
|
||||
datasetRef, fieldPageNumber, fieldPageSize));
|
||||
}
|
||||
|
||||
@GetMapping("/managedTables")
|
||||
@@ -63,6 +73,13 @@ public class DatacenterDatasetController {
|
||||
request == null ? List.of() : request.getFields(),
|
||||
account
|
||||
);
|
||||
return Result.ok(queryService.getSchema(registryService.resolveDatasetRef(table.getId())));
|
||||
return Result.ok(queryService.getSchema(
|
||||
registryService.resolveDatasetRef(table.getId()),
|
||||
request == null || request.getFieldPageNumber() == null
|
||||
? DEFAULT_FIELD_PAGE_NUMBER
|
||||
: request.getFieldPageNumber(),
|
||||
request == null || request.getFieldPageSize() == null
|
||||
? DEFAULT_FIELD_PAGE_SIZE
|
||||
: request.getFieldPageSize()));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,84 @@
|
||||
package tech.easyflow.admin.controller.datacenter;
|
||||
|
||||
import cn.dev33.satoken.annotation.SaCheckPermission;
|
||||
import org.springframework.web.bind.annotation.PostMapping;
|
||||
import org.springframework.web.bind.annotation.RequestBody;
|
||||
import org.springframework.web.bind.annotation.RequestMapping;
|
||||
import org.springframework.web.bind.annotation.RestController;
|
||||
import tech.easyflow.common.domain.Result;
|
||||
import tech.easyflow.common.entity.LoginAccount;
|
||||
import tech.easyflow.common.satoken.util.SaTokenUtil;
|
||||
import tech.easyflow.datacenter.execution.model.DatacenterSqlConsoleRequest;
|
||||
import tech.easyflow.datacenter.execution.model.DatacenterSqlConsoleResult;
|
||||
import tech.easyflow.datacenter.execution.model.DatacenterSqlCancelRequest;
|
||||
import tech.easyflow.datacenter.federation.DatacenterFederationQueryService;
|
||||
import tech.easyflow.datacenter.federation.DatacenterFederationQueryCancellationService;
|
||||
import tech.easyflow.datacenter.meta.entity.DatacenterSource;
|
||||
import tech.easyflow.datacenter.meta.service.DatacenterDatasetRegistryService;
|
||||
|
||||
/**
|
||||
* 数据中枢管理端只读 SQL 控制台。
|
||||
*/
|
||||
@RestController
|
||||
@RequestMapping("/api/v1/datacenterQuery")
|
||||
public class DatacenterQueryController {
|
||||
|
||||
private final DatacenterDatasetRegistryService registryService;
|
||||
private final DatacenterFederationQueryService queryService;
|
||||
private final DatacenterFederationQueryCancellationService cancellationService;
|
||||
|
||||
/**
|
||||
* 创建查询 Controller。
|
||||
*
|
||||
* @param registryService 数据集注册服务
|
||||
* @param queryService Federation 查询服务
|
||||
* @param cancellationService 跨节点查询取消服务
|
||||
*/
|
||||
public DatacenterQueryController(
|
||||
DatacenterDatasetRegistryService registryService,
|
||||
DatacenterFederationQueryService queryService,
|
||||
DatacenterFederationQueryCancellationService cancellationService) {
|
||||
this.registryService = registryService;
|
||||
this.queryService = queryService;
|
||||
this.cancellationService = cancellationService;
|
||||
}
|
||||
|
||||
/**
|
||||
* 执行一条受 Calcite 与业务 Policy 校验的只读 SQL。
|
||||
*
|
||||
* @param request 查询请求
|
||||
* @return 有界查询结果
|
||||
*/
|
||||
@PostMapping("/execute")
|
||||
@SaCheckPermission("/api/v1/datacenterSource/query")
|
||||
public Result<DatacenterSqlConsoleResult> execute(
|
||||
@RequestBody DatacenterSqlConsoleRequest request) {
|
||||
LoginAccount account = SaTokenUtil.getLoginAccount();
|
||||
DatacenterSource source = registryService.getSourceRequired(
|
||||
request == null ? null : request.sourceId());
|
||||
return Result.ok(queryService.execute(
|
||||
source,
|
||||
request == null ? null : request.sql(),
|
||||
java.util.List.of(),
|
||||
request == null ? null : request.maxRows(),
|
||||
account,
|
||||
"MANUAL",
|
||||
account == null || account.getId() == null
|
||||
? null : account.getId().toString(),
|
||||
request == null ? null : request.queryId()));
|
||||
}
|
||||
|
||||
/**
|
||||
* 取消当前租户在任一节点执行的 SQL 查询。
|
||||
*
|
||||
* @param request 取消请求
|
||||
* @return 是否在本地或集群中接受取消提示
|
||||
*/
|
||||
@PostMapping("/cancel")
|
||||
@SaCheckPermission("/api/v1/datacenterSource/query")
|
||||
public Result<Boolean> cancel(@RequestBody DatacenterSqlCancelRequest request) {
|
||||
LoginAccount account = SaTokenUtil.getLoginAccount();
|
||||
return Result.ok(cancellationService.cancel(
|
||||
request == null ? null : request.queryId(), account));
|
||||
}
|
||||
}
|
||||
@@ -8,10 +8,16 @@ import tech.easyflow.common.entity.LoginAccount;
|
||||
import tech.easyflow.common.satoken.util.SaTokenUtil;
|
||||
import tech.easyflow.datacenter.entity.DatacenterTable;
|
||||
import tech.easyflow.datacenter.execution.model.DatacenterConnectionTestResult;
|
||||
import tech.easyflow.datacenter.meta.entity.DatacenterSource;
|
||||
import tech.easyflow.datacenter.meta.model.DatacenterBatchRegisterRequest;
|
||||
import tech.easyflow.datacenter.meta.model.DatacenterCatalogMeta;
|
||||
import tech.easyflow.datacenter.meta.model.DatacenterMetadataPage;
|
||||
import tech.easyflow.datacenter.meta.model.DatacenterRemoveSourceRequest;
|
||||
import tech.easyflow.datacenter.meta.model.DatacenterSourceActivateRequest;
|
||||
import tech.easyflow.datacenter.meta.model.DatacenterSourceCandidateMetadataRequest;
|
||||
import tech.easyflow.datacenter.meta.model.DatacenterSourceCandidateCatalogRequest;
|
||||
import tech.easyflow.datacenter.meta.model.DatacenterSourceDraftRequest;
|
||||
import tech.easyflow.datacenter.meta.model.DatacenterSourceReconfigureRequest;
|
||||
import tech.easyflow.datacenter.meta.model.DatacenterSourceView;
|
||||
import tech.easyflow.datacenter.meta.model.DatacenterTableDetailMeta;
|
||||
import tech.easyflow.datacenter.meta.service.DatacenterSourceService;
|
||||
|
||||
@@ -19,6 +25,9 @@ import javax.annotation.Resource;
|
||||
import java.math.BigInteger;
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* 数据源绑定、生命周期与元数据浏览接口。
|
||||
*/
|
||||
@RestController
|
||||
@RequestMapping("/api/v1/datacenterSource")
|
||||
public class DatacenterSourceController {
|
||||
@@ -26,23 +35,100 @@ public class DatacenterSourceController {
|
||||
@Resource
|
||||
private DatacenterSourceService sourceService;
|
||||
|
||||
@PostMapping("/testConnection")
|
||||
@SaCheckPermission("/api/v1/datacenterSource/query")
|
||||
public Result<DatacenterConnectionTestResult> testConnection(@RequestBody DatacenterSource source) {
|
||||
@PostMapping("/draft")
|
||||
@SaCheckPermission("/api/v1/datacenterSource/save")
|
||||
public Result<DatacenterSourceView> saveDraft(@RequestBody DatacenterSourceDraftRequest request) {
|
||||
LoginAccount account = SaTokenUtil.getLoginAccount();
|
||||
return Result.ok(sourceService.testConnection(source, account));
|
||||
return Result.ok(sourceService.saveDraft(request, account));
|
||||
}
|
||||
|
||||
@PostMapping("/save")
|
||||
@SaCheckPermission("/api/v1/datacenterSource/save")
|
||||
public Result<DatacenterSource> save(@RequestBody DatacenterSource source) {
|
||||
@PostMapping("/{sourceId}/probe")
|
||||
@SaCheckPermission("/api/v1/datacenterSource/query")
|
||||
public Result<DatacenterConnectionTestResult> probe(@PathVariable BigInteger sourceId) {
|
||||
LoginAccount account = SaTokenUtil.getLoginAccount();
|
||||
return Result.ok(sourceService.saveSource(source, account));
|
||||
return Result.ok(sourceService.probe(sourceId, account));
|
||||
}
|
||||
|
||||
@PostMapping("/activate")
|
||||
@SaCheckPermission("/api/v1/datacenterSource/save")
|
||||
public Result<DatacenterSourceView> activate(@RequestBody DatacenterSourceActivateRequest request) {
|
||||
LoginAccount account = SaTokenUtil.getLoginAccount();
|
||||
return Result.ok(sourceService.activate(request, account));
|
||||
}
|
||||
|
||||
/**
|
||||
* 探测活动数据源的未发布候选配置。
|
||||
*
|
||||
* @param request 候选连接配置
|
||||
* @return 连接探测结果
|
||||
*/
|
||||
@PostMapping("/candidate/probe")
|
||||
@SaCheckPermission("/api/v1/datacenterSource/save")
|
||||
public Result<DatacenterConnectionTestResult> probeCandidate(
|
||||
@RequestBody DatacenterSourceDraftRequest request) {
|
||||
return Result.ok(sourceService.probeCandidate(
|
||||
request, SaTokenUtil.getLoginAccount()));
|
||||
}
|
||||
|
||||
/**
|
||||
* 浏览活动数据源候选配置可访问的命名空间。
|
||||
*
|
||||
* @param request 候选连接配置
|
||||
* @return 命名空间列表
|
||||
*/
|
||||
@PostMapping("/candidate/catalogs")
|
||||
@SaCheckPermission("/api/v1/datacenterSource/save")
|
||||
public Result<List<DatacenterCatalogMeta>> candidateCatalogs(
|
||||
@RequestBody DatacenterSourceDraftRequest request) {
|
||||
return Result.ok(sourceService.listCandidateCatalogs(
|
||||
request, SaTokenUtil.getLoginAccount()));
|
||||
}
|
||||
|
||||
/**
|
||||
* 分页浏览候选配置可访问的命名空间。
|
||||
*
|
||||
* @param request 候选配置和分页条件
|
||||
* @return 有界命名空间列表
|
||||
*/
|
||||
@PostMapping("/candidate/catalogs/page")
|
||||
@SaCheckPermission("/api/v1/datacenterSource/save")
|
||||
public Result<DatacenterMetadataPage<DatacenterCatalogMeta>> candidateCatalogsPage(
|
||||
@RequestBody DatacenterSourceCandidateCatalogRequest request) {
|
||||
return Result.ok(sourceService.listCandidateCatalogsPage(
|
||||
request, SaTokenUtil.getLoginAccount()));
|
||||
}
|
||||
|
||||
/**
|
||||
* 分页浏览活动数据源候选配置可访问的表。
|
||||
*
|
||||
* @param request 候选配置与分页条件
|
||||
* @return 有界表列表
|
||||
*/
|
||||
@PostMapping("/candidate/tables")
|
||||
@SaCheckPermission("/api/v1/datacenterSource/save")
|
||||
public Result<DatacenterMetadataPage<DatacenterTable>> candidateTables(
|
||||
@RequestBody DatacenterSourceCandidateMetadataRequest request) {
|
||||
return Result.ok(sourceService.listCandidateTables(
|
||||
request, SaTokenUtil.getLoginAccount()));
|
||||
}
|
||||
|
||||
/**
|
||||
* 原地发布活动数据源的新连接配置和纳管范围。
|
||||
*
|
||||
* @param request 重配置请求
|
||||
* @return 发布后的数据源视图
|
||||
*/
|
||||
@PostMapping("/reconfigure")
|
||||
@SaCheckPermission("/api/v1/datacenterSource/save")
|
||||
public Result<DatacenterSourceView> reconfigure(
|
||||
@RequestBody DatacenterSourceReconfigureRequest request) {
|
||||
return Result.ok(sourceService.reconfigure(
|
||||
request, SaTokenUtil.getLoginAccount()));
|
||||
}
|
||||
|
||||
@GetMapping("/page")
|
||||
@SaCheckPermission("/api/v1/datacenterSource/query")
|
||||
public Result<Page<DatacenterSource>> page(Long pageNumber, Long pageSize) {
|
||||
public Result<Page<DatacenterSourceView>> page(Long pageNumber, Long pageSize) {
|
||||
LoginAccount account = SaTokenUtil.getLoginAccount();
|
||||
return Result.ok(sourceService.pageSources(pageNumber, pageSize, account));
|
||||
}
|
||||
@@ -54,19 +140,53 @@ public class DatacenterSourceController {
|
||||
return Result.ok(sourceService.listCatalogs(sourceId, account));
|
||||
}
|
||||
|
||||
/**
|
||||
* 分页浏览当前数据源的命名空间。
|
||||
*
|
||||
* @param sourceId 数据源 ID
|
||||
* @param keyword 名称搜索词
|
||||
* @param pageNumber 页码
|
||||
* @param pageSize 每页大小
|
||||
* @return 有界命名空间列表
|
||||
*/
|
||||
@GetMapping("/catalogs/page")
|
||||
@SaCheckPermission("/api/v1/datacenterSource/query")
|
||||
public Result<DatacenterMetadataPage<DatacenterCatalogMeta>> catalogsPage(
|
||||
BigInteger sourceId,
|
||||
String keyword,
|
||||
Long pageNumber,
|
||||
Long pageSize) {
|
||||
return Result.ok(sourceService.listCatalogsPage(
|
||||
sourceId, keyword, pageNumber, pageSize,
|
||||
SaTokenUtil.getLoginAccount()));
|
||||
}
|
||||
|
||||
@GetMapping("/tables")
|
||||
@SaCheckPermission("/api/v1/datacenterSource/query")
|
||||
public Result<List<DatacenterTable>> tables(BigInteger sourceId, String catalogName) {
|
||||
public Result<DatacenterMetadataPage<DatacenterTable>> tables(
|
||||
BigInteger sourceId,
|
||||
String catalogName,
|
||||
String keyword,
|
||||
Long pageNumber,
|
||||
Long pageSize) {
|
||||
LoginAccount account = SaTokenUtil.getLoginAccount();
|
||||
return Result.ok(sourceService.listTables(sourceId, catalogName, account));
|
||||
return Result.ok(sourceService.listTables(
|
||||
sourceId, catalogName, keyword, pageNumber, pageSize, account));
|
||||
}
|
||||
|
||||
@GetMapping("/tableDetail")
|
||||
@SaCheckPermission("/api/v1/datacenterSource/query")
|
||||
public Result<DatacenterTableDetailMeta> tableDetail(BigInteger sourceId, String catalogName, String tableName,
|
||||
@RequestParam(defaultValue = "false") boolean register) {
|
||||
public Result<DatacenterTableDetailMeta> tableDetail(
|
||||
BigInteger sourceId,
|
||||
String catalogName,
|
||||
String tableName,
|
||||
@RequestParam(defaultValue = "false") boolean register,
|
||||
Long fieldPageNumber,
|
||||
Long fieldPageSize) {
|
||||
LoginAccount account = SaTokenUtil.getLoginAccount();
|
||||
return Result.ok(sourceService.getTableDetail(sourceId, catalogName, tableName, register, account));
|
||||
return Result.ok(sourceService.getTableDetail(
|
||||
sourceId, catalogName, tableName, register,
|
||||
fieldPageNumber, fieldPageSize, account));
|
||||
}
|
||||
|
||||
@PostMapping("/registerBatch")
|
||||
@@ -83,4 +203,44 @@ public class DatacenterSourceController {
|
||||
sourceService.removeSource(request == null ? null : request.getSourceId(), account);
|
||||
return Result.ok();
|
||||
}
|
||||
|
||||
/**
|
||||
* 停用活动数据源。
|
||||
*
|
||||
* @param sourceId 数据源 ID
|
||||
* @return 停用后的数据源视图
|
||||
*/
|
||||
@PostMapping("/{sourceId}/disable")
|
||||
@SaCheckPermission("/api/v1/datacenterSource/save")
|
||||
public Result<DatacenterSourceView> disable(@PathVariable BigInteger sourceId) {
|
||||
return Result.ok(sourceService.disable(
|
||||
sourceId, SaTokenUtil.getLoginAccount()));
|
||||
}
|
||||
|
||||
/**
|
||||
* 重新启用已停用数据源。
|
||||
*
|
||||
* @param sourceId 数据源 ID
|
||||
* @return 启用后的数据源视图
|
||||
*/
|
||||
@PostMapping("/{sourceId}/enable")
|
||||
@SaCheckPermission("/api/v1/datacenterSource/save")
|
||||
public Result<DatacenterSourceView> enable(@PathVariable BigInteger sourceId) {
|
||||
return Result.ok(sourceService.enable(
|
||||
sourceId, SaTokenUtil.getLoginAccount()));
|
||||
}
|
||||
|
||||
/**
|
||||
* 刷新已纳管对象的 JDBC 元数据观测状态。
|
||||
*
|
||||
* @param sourceId 数据源 ID
|
||||
* @return 刷新后的数据源视图
|
||||
*/
|
||||
@PostMapping("/{sourceId}/metadata/refresh")
|
||||
@SaCheckPermission("/api/v1/datacenterSource/save")
|
||||
public Result<DatacenterSourceView> refreshMetadata(
|
||||
@PathVariable BigInteger sourceId) {
|
||||
return Result.ok(sourceService.refreshMetadata(
|
||||
sourceId, SaTokenUtil.getLoginAccount()));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -126,6 +126,7 @@ public record WorkflowDesignerOptionsView(
|
||||
* 已接入数据集安全选项。
|
||||
*
|
||||
* @param id 数据集 ID
|
||||
* @param tenantId 租户 ID
|
||||
* @param sourceId 数据源 ID
|
||||
* @param catalogId 目录 ID
|
||||
* @param tableName 数据表名称
|
||||
@@ -133,6 +134,7 @@ public record WorkflowDesignerOptionsView(
|
||||
*/
|
||||
public record DatasetOption(
|
||||
@JsonSerialize(using = ToStringSerializer.class) BigInteger id,
|
||||
@JsonSerialize(using = ToStringSerializer.class) BigInteger tenantId,
|
||||
@JsonSerialize(using = ToStringSerializer.class) BigInteger sourceId,
|
||||
@JsonSerialize(using = ToStringSerializer.class) BigInteger catalogId,
|
||||
String tableName,
|
||||
|
||||
@@ -653,6 +653,7 @@ public class WorkflowDesignerOptionService {
|
||||
private WorkflowDesignerOptionsView.DatasetOption toDatasetOption(DatacenterTable table) {
|
||||
return new WorkflowDesignerOptionsView.DatasetOption(
|
||||
table.getId(),
|
||||
table.getTenantId(),
|
||||
table.getSourceId(),
|
||||
table.getCatalogId(),
|
||||
table.getTableName(),
|
||||
|
||||
@@ -0,0 +1,42 @@
|
||||
package tech.easyflow.ai.easyagentsflow.cancellation;
|
||||
|
||||
import com.easyagents.flow.core.chain.Chain;
|
||||
import com.easyagents.flow.core.chain.ChainStatus;
|
||||
import com.easyagents.flow.core.chain.Event;
|
||||
import com.easyagents.flow.core.chain.event.ChainStatusChangeEvent;
|
||||
import com.easyagents.flow.core.chain.listener.ChainEventListener;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
/**
|
||||
* 将工作流取消终态桥接到活动数据集查询。
|
||||
*/
|
||||
@Component
|
||||
public class WorkflowDatasetQueryCancellationListener
|
||||
implements ChainEventListener {
|
||||
|
||||
private final WorkflowDatasetQueryCancellationRegistry registry;
|
||||
|
||||
/**
|
||||
* 创建工作流查询取消监听器。
|
||||
*
|
||||
* @param registry 工作流查询取消登记表
|
||||
*/
|
||||
public WorkflowDatasetQueryCancellationListener(
|
||||
WorkflowDatasetQueryCancellationRegistry registry) {
|
||||
this.registry = registry;
|
||||
}
|
||||
|
||||
/**
|
||||
* 在工作流进入取消终态后取消该实例的活动查询。
|
||||
*
|
||||
* @param event 工作流事件
|
||||
* @param chain 工作流实例
|
||||
*/
|
||||
@Override
|
||||
public void onEvent(Event event, Chain chain) {
|
||||
if (event instanceof ChainStatusChangeEvent statusChangeEvent
|
||||
&& statusChangeEvent.getStatus() == ChainStatus.CANCELLED) {
|
||||
registry.cancelExecution(chain.getStateInstanceId());
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,279 @@
|
||||
package tech.easyflow.ai.easyagentsflow.cancellation;
|
||||
|
||||
import java.math.BigInteger;
|
||||
import java.time.Duration;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.Map;
|
||||
import java.util.Objects;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
import java.util.concurrent.atomic.AtomicBoolean;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.beans.factory.ObjectProvider;
|
||||
import org.springframework.data.redis.core.StringRedisTemplate;
|
||||
import org.springframework.scheduling.annotation.Scheduled;
|
||||
import org.springframework.stereotype.Component;
|
||||
import tech.easyflow.common.entity.LoginAccount;
|
||||
import tech.easyflow.common.web.exceptions.BusinessException;
|
||||
import tech.easyflow.datacenter.federation.DatacenterFederationQueryCancellationService;
|
||||
|
||||
/**
|
||||
* 维护工作流实例到活动数据集查询的跨节点取消映射。
|
||||
*/
|
||||
@Component
|
||||
public class WorkflowDatasetQueryCancellationRegistry {
|
||||
|
||||
private static final String ACTIVE_KEY_PREFIX =
|
||||
"easyflow:workflow:dataset-query:active:";
|
||||
private static final String CANCELLED_KEY_PREFIX =
|
||||
"easyflow:workflow:dataset-query:cancelled:";
|
||||
private static final Duration STATE_TTL = Duration.ofHours(24);
|
||||
private static final Logger log = LoggerFactory.getLogger(
|
||||
WorkflowDatasetQueryCancellationRegistry.class);
|
||||
|
||||
private final DatacenterFederationQueryCancellationService cancellationService;
|
||||
private final ObjectProvider<StringRedisTemplate> redisTemplateProvider;
|
||||
private final ConcurrentHashMap<String, ConcurrentHashMap<String, BigInteger>>
|
||||
localActive = new ConcurrentHashMap<>();
|
||||
private final ConcurrentHashMap<String, Long> localCancelledUntil =
|
||||
new ConcurrentHashMap<>();
|
||||
|
||||
/**
|
||||
* 创建工作流查询取消登记表。
|
||||
*
|
||||
* @param cancellationService 数据中枢查询取消服务
|
||||
* @param redisTemplateProvider 可选 Redis 模板
|
||||
*/
|
||||
public WorkflowDatasetQueryCancellationRegistry(
|
||||
DatacenterFederationQueryCancellationService cancellationService,
|
||||
ObjectProvider<StringRedisTemplate> redisTemplateProvider) {
|
||||
this.cancellationService = cancellationService;
|
||||
this.redisTemplateProvider = redisTemplateProvider;
|
||||
}
|
||||
|
||||
/**
|
||||
* 在数据库查询开始前登记工作流与 QueryId。
|
||||
*
|
||||
* @param stateInstanceId 工作流实例 ID
|
||||
* @param account 执行账号
|
||||
* @param queryId 查询 UUID
|
||||
* @return 必须关闭的登记句柄
|
||||
* @throws BusinessException 参数缺失时抛出
|
||||
*/
|
||||
public Registration register(
|
||||
String stateInstanceId,
|
||||
LoginAccount account,
|
||||
String queryId) {
|
||||
String instanceId = requireText(stateInstanceId, "工作流实例 ID 不能为空");
|
||||
String normalizedQueryId = requireText(queryId, "queryId 不能为空");
|
||||
BigInteger tenantId = requireTenantId(account);
|
||||
localActive.computeIfAbsent(
|
||||
instanceId,
|
||||
ignored -> new ConcurrentHashMap<>())
|
||||
.put(normalizedQueryId, tenantId);
|
||||
persistActive(instanceId, normalizedQueryId, tenantId);
|
||||
Registration registration = new Registration(
|
||||
instanceId, normalizedQueryId, tenantId);
|
||||
if (isExecutionCancelled(instanceId)) {
|
||||
cancelQuery(normalizedQueryId, tenantId);
|
||||
}
|
||||
return registration;
|
||||
}
|
||||
|
||||
/**
|
||||
* 取消工作流当前登记的全部数据集查询。
|
||||
*
|
||||
* @param stateInstanceId 工作流实例 ID
|
||||
* @return 是否发现至少一个活动查询
|
||||
*/
|
||||
public boolean cancelExecution(String stateInstanceId) {
|
||||
String instanceId = requireText(stateInstanceId, "工作流实例 ID 不能为空");
|
||||
long expiresAt = System.currentTimeMillis() + STATE_TTL.toMillis();
|
||||
localCancelledUntil.put(instanceId, expiresAt);
|
||||
persistCancellationMarker(instanceId);
|
||||
|
||||
Map<String, BigInteger> active = new LinkedHashMap<>();
|
||||
ConcurrentHashMap<String, BigInteger> local = localActive.get(instanceId);
|
||||
if (local != null) {
|
||||
active.putAll(local);
|
||||
}
|
||||
loadPersistedActive(instanceId).forEach(active::putIfAbsent);
|
||||
active.forEach((queryId, tenantId) -> {
|
||||
try {
|
||||
cancelQuery(queryId, tenantId);
|
||||
} catch (RuntimeException exception) {
|
||||
// 单个驱动取消失败不能阻断同一工作流的其他活动查询。
|
||||
log.error("Failed to cancel workflow dataset query {} for {}",
|
||||
queryId, instanceId, exception);
|
||||
}
|
||||
});
|
||||
return !active.isEmpty();
|
||||
}
|
||||
|
||||
/**
|
||||
* 清理本机已过期的取消墓碑,避免不可达工作流持续占用内存。
|
||||
*/
|
||||
@Scheduled(fixedDelayString =
|
||||
"${easyflow.workflow.dataset-query-cancel-cleanup-ms:60000}")
|
||||
public void cleanupExpiredLocalMarkers() {
|
||||
long now = System.currentTimeMillis();
|
||||
localCancelledUntil.entrySet().removeIf(
|
||||
entry -> entry.getValue() <= now);
|
||||
}
|
||||
|
||||
private void cancelQuery(String queryId, BigInteger tenantId) {
|
||||
LoginAccount account = new LoginAccount();
|
||||
account.setTenantId(tenantId);
|
||||
cancellationService.cancel(queryId, account);
|
||||
}
|
||||
|
||||
private boolean isExecutionCancelled(String stateInstanceId) {
|
||||
Long localDeadline = localCancelledUntil.get(stateInstanceId);
|
||||
if (localDeadline != null && localDeadline > System.currentTimeMillis()) {
|
||||
return true;
|
||||
}
|
||||
StringRedisTemplate redisTemplate = redisTemplateProvider.getIfAvailable();
|
||||
if (redisTemplate == null) {
|
||||
return false;
|
||||
}
|
||||
try {
|
||||
return Boolean.TRUE.equals(redisTemplate.hasKey(
|
||||
cancelledKey(stateInstanceId)));
|
||||
} catch (RuntimeException exception) {
|
||||
log.warn("Failed to read workflow dataset cancellation marker for {}",
|
||||
stateInstanceId, exception);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
private void persistActive(
|
||||
String stateInstanceId,
|
||||
String queryId,
|
||||
BigInteger tenantId) {
|
||||
StringRedisTemplate redisTemplate = redisTemplateProvider.getIfAvailable();
|
||||
if (redisTemplate == null) {
|
||||
return;
|
||||
}
|
||||
try {
|
||||
String key = activeKey(stateInstanceId);
|
||||
redisTemplate.opsForHash().put(key, queryId, tenantId.toString());
|
||||
redisTemplate.expire(key, STATE_TTL);
|
||||
} catch (RuntimeException exception) {
|
||||
log.warn("Failed to persist workflow dataset query mapping for {}",
|
||||
stateInstanceId, exception);
|
||||
}
|
||||
}
|
||||
|
||||
private Map<String, BigInteger> loadPersistedActive(String stateInstanceId) {
|
||||
StringRedisTemplate redisTemplate = redisTemplateProvider.getIfAvailable();
|
||||
if (redisTemplate == null) {
|
||||
return Map.of();
|
||||
}
|
||||
try {
|
||||
Map<Object, Object> entries = redisTemplate.opsForHash()
|
||||
.entries(activeKey(stateInstanceId));
|
||||
Map<String, BigInteger> active = new LinkedHashMap<>();
|
||||
entries.forEach((queryId, tenantId) -> {
|
||||
try {
|
||||
active.put(String.valueOf(queryId),
|
||||
new BigInteger(String.valueOf(tenantId)));
|
||||
} catch (RuntimeException exception) {
|
||||
log.warn("Ignored invalid workflow dataset query mapping for {}",
|
||||
stateInstanceId, exception);
|
||||
}
|
||||
});
|
||||
return active;
|
||||
} catch (RuntimeException exception) {
|
||||
log.warn("Failed to load workflow dataset query mappings for {}",
|
||||
stateInstanceId, exception);
|
||||
return Map.of();
|
||||
}
|
||||
}
|
||||
|
||||
private void persistCancellationMarker(String stateInstanceId) {
|
||||
StringRedisTemplate redisTemplate = redisTemplateProvider.getIfAvailable();
|
||||
if (redisTemplate == null) {
|
||||
return;
|
||||
}
|
||||
try {
|
||||
redisTemplate.opsForValue().set(
|
||||
cancelledKey(stateInstanceId), "1", STATE_TTL);
|
||||
} catch (RuntimeException exception) {
|
||||
log.warn("Failed to persist workflow dataset cancellation marker for {}",
|
||||
stateInstanceId, exception);
|
||||
}
|
||||
}
|
||||
|
||||
private void unregister(
|
||||
String stateInstanceId,
|
||||
String queryId,
|
||||
BigInteger tenantId) {
|
||||
localActive.computeIfPresent(stateInstanceId, (ignored, queries) -> {
|
||||
queries.remove(queryId, tenantId);
|
||||
return queries.isEmpty() ? null : queries;
|
||||
});
|
||||
StringRedisTemplate redisTemplate = redisTemplateProvider.getIfAvailable();
|
||||
if (redisTemplate == null) {
|
||||
return;
|
||||
}
|
||||
try {
|
||||
redisTemplate.opsForHash().delete(
|
||||
activeKey(stateInstanceId), queryId);
|
||||
} catch (RuntimeException exception) {
|
||||
log.warn("Failed to remove workflow dataset query mapping for {}",
|
||||
stateInstanceId, exception);
|
||||
}
|
||||
}
|
||||
|
||||
private String activeKey(String stateInstanceId) {
|
||||
return ACTIVE_KEY_PREFIX + stateInstanceId;
|
||||
}
|
||||
|
||||
private String cancelledKey(String stateInstanceId) {
|
||||
return CANCELLED_KEY_PREFIX + stateInstanceId;
|
||||
}
|
||||
|
||||
private String requireText(String value, String message) {
|
||||
if (value == null || value.isBlank()) {
|
||||
throw new BusinessException(message);
|
||||
}
|
||||
return value.trim();
|
||||
}
|
||||
|
||||
private BigInteger requireTenantId(LoginAccount account) {
|
||||
if (account == null || account.getTenantId() == null) {
|
||||
throw new BusinessException("工作流数据集查询缺少执行租户");
|
||||
}
|
||||
return account.getTenantId();
|
||||
}
|
||||
|
||||
/**
|
||||
* 单次工作流查询登记句柄。
|
||||
*/
|
||||
public final class Registration implements AutoCloseable {
|
||||
|
||||
private final String stateInstanceId;
|
||||
private final String queryId;
|
||||
private final BigInteger tenantId;
|
||||
private final AtomicBoolean closed = new AtomicBoolean();
|
||||
|
||||
private Registration(
|
||||
String stateInstanceId,
|
||||
String queryId,
|
||||
BigInteger tenantId) {
|
||||
this.stateInstanceId = stateInstanceId;
|
||||
this.queryId = queryId;
|
||||
this.tenantId = tenantId;
|
||||
}
|
||||
|
||||
/**
|
||||
* 幂等注销当前工作流查询。
|
||||
*/
|
||||
@Override
|
||||
public void close() {
|
||||
if (closed.compareAndSet(false, true)) {
|
||||
unregister(stateInstanceId, queryId, tenantId);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -14,6 +14,7 @@ import org.springframework.context.annotation.Configuration;
|
||||
import tech.easyflow.ai.easyagentsflow.listener.ChainErrorListenerForSave;
|
||||
import tech.easyflow.ai.easyagentsflow.listener.ChainEventListenerForSave;
|
||||
import tech.easyflow.ai.easyagentsflow.listener.NodeErrorListenerForSave;
|
||||
import tech.easyflow.ai.easyagentsflow.cancellation.WorkflowDatasetQueryCancellationListener;
|
||||
import tech.easyflow.ai.easyagentsflow.upload.WorkflowApiUploadCleanupListener;
|
||||
|
||||
import javax.annotation.Resource;
|
||||
@@ -40,6 +41,9 @@ public class ChainExecutorConfig {
|
||||
@Resource
|
||||
private WorkflowApiUploadCleanupListener workflowApiUploadCleanupListener;
|
||||
@Resource
|
||||
private WorkflowDatasetQueryCancellationListener
|
||||
workflowDatasetQueryCancellationListener;
|
||||
@Resource
|
||||
private WorkflowExecutionBudgetProperties workflowExecutionBudgetProperties;
|
||||
@Resource
|
||||
private WorkflowRuntimeProperties workflowRuntimeProperties;
|
||||
@@ -91,6 +95,9 @@ public class ChainExecutorConfig {
|
||||
chainExecutor.addEventListener(
|
||||
ChainStatusChangeEvent.class,
|
||||
workflowApiUploadCleanupListener);
|
||||
chainExecutor.addEventListener(
|
||||
ChainStatusChangeEvent.class,
|
||||
workflowDatasetQueryCancellationListener);
|
||||
chainExecutor.addErrorListener(new ChainErrorListenerForSave());
|
||||
chainExecutor.addNodeErrorListener(new NodeErrorListenerForSave());
|
||||
}
|
||||
|
||||
@@ -14,12 +14,13 @@ import tech.easyflow.datacenter.execution.model.DatasetRef;
|
||||
import tech.easyflow.datacenter.execution.service.DatacenterDatasetQueryService;
|
||||
import tech.easyflow.datacenter.meta.entity.DatacenterCatalog;
|
||||
import tech.easyflow.datacenter.meta.entity.DatacenterSource;
|
||||
import tech.easyflow.datacenter.meta.enums.DatacenterMetadataStatus;
|
||||
import tech.easyflow.datacenter.meta.enums.DatacenterSensitivityLevel;
|
||||
import tech.easyflow.datacenter.meta.service.DatacenterDatasetRegistryService;
|
||||
|
||||
import javax.annotation.Resource;
|
||||
import java.math.BigInteger;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Comparator;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.LinkedHashSet;
|
||||
import java.util.List;
|
||||
@@ -34,6 +35,7 @@ public class WorkflowDatacenterContentService {
|
||||
public static final String LLM_NODE_TYPE = "llmNode";
|
||||
public static final String QUERY_DATA_CONTEXT = "queryDataContext";
|
||||
public static final String SEARCH_SOURCE_MISSING_MESSAGE = "查询数据节点未选择连接服务";
|
||||
public static final String SEARCH_TABLE_MISSING_MESSAGE = "查询数据节点未选择已接入表";
|
||||
public static final String SEARCH_SQL_MISSING_MESSAGE = "查询数据节点未设置 SQL";
|
||||
public static final String SAVE_EXPIRED_MESSAGE = "写入数据节点配置已过期,请重新选择已接入表";
|
||||
public static final String INVALID_QUERY_CONTEXT_MESSAGE = "查询上下文配置无效,请重新选择查询数据节点";
|
||||
@@ -135,11 +137,16 @@ public class WorkflowDatacenterContentService {
|
||||
if (datasetRef == null || datasetRef.getSourceId() == null) {
|
||||
throw new BusinessException(SEARCH_SOURCE_MISSING_MESSAGE);
|
||||
}
|
||||
if (datasetRef.getTableId() == null) {
|
||||
throw new BusinessException(SEARCH_TABLE_MISSING_MESSAGE);
|
||||
}
|
||||
String querySql = data == null ? null : trimToNull(data.getString("querySql"));
|
||||
if (!StringUtils.hasText(querySql)) {
|
||||
throw new BusinessException(SEARCH_SQL_MISSING_MESSAGE);
|
||||
}
|
||||
return datasetRef;
|
||||
DatasetRef boundRef = bindAuthoritativeTenant(datasetRef);
|
||||
data.put("datasetRef", boundRef);
|
||||
return boundRef;
|
||||
}
|
||||
|
||||
public DatasetRef requireSaveDatasetRef(JSONObject data) {
|
||||
@@ -150,6 +157,31 @@ public class WorkflowDatacenterContentService {
|
||||
if (datasetRef == null || datasetRef.getTableId() == null) {
|
||||
throw new BusinessException(SAVE_EXPIRED_MESSAGE);
|
||||
}
|
||||
DatasetRef boundRef = bindAuthoritativeTenant(datasetRef);
|
||||
data.put("datasetRef", boundRef);
|
||||
return boundRef;
|
||||
}
|
||||
|
||||
/**
|
||||
* 依据当前租户可见的权威 Source/Table 覆盖调用方声明的租户字段。
|
||||
*
|
||||
* @param datasetRef 工作流数据集引用
|
||||
* @return 已绑定权威租户的数据集引用
|
||||
*/
|
||||
private DatasetRef bindAuthoritativeTenant(DatasetRef datasetRef) {
|
||||
DatacenterTable table = datasetRef.getTableId() == null
|
||||
? null : registryService.getTableWithFields(datasetRef.getTableId());
|
||||
BigInteger sourceId = table == null ? datasetRef.getSourceId() : table.getSourceId();
|
||||
if (sourceId == null) {
|
||||
throw new BusinessException(SEARCH_SOURCE_MISSING_MESSAGE);
|
||||
}
|
||||
DatacenterSource source = registryService.getSourceRequired(sourceId);
|
||||
if (table != null && (!sourceId.equals(table.getSourceId())
|
||||
|| !java.util.Objects.equals(source.getTenantId(), table.getTenantId()))) {
|
||||
throw new BusinessException("数据集引用与当前租户不一致");
|
||||
}
|
||||
datasetRef.setTenantId(source.getTenantId());
|
||||
datasetRef.setSourceId(sourceId);
|
||||
return datasetRef;
|
||||
}
|
||||
|
||||
@@ -172,41 +204,68 @@ public class WorkflowDatacenterContentService {
|
||||
if (datasetRef == null || datasetRef.getSourceId() == null) {
|
||||
throw new BusinessException(SEARCH_SOURCE_MISSING_MESSAGE);
|
||||
}
|
||||
if (datasetRef.getTableId() == null) {
|
||||
throw new BusinessException(SEARCH_TABLE_MISSING_MESSAGE);
|
||||
}
|
||||
DatacenterSource source = registryService.getSourceRequired(datasetRef.getSourceId());
|
||||
List<DatacenterTable> managedTables = registryService.listManagedTables(datasetRef.getSourceId(), datasetRef.getCatalogId());
|
||||
managedTables.sort(Comparator.comparing(table -> table.getTableName() == null ? "" : table.getTableName()));
|
||||
DatacenterTable fullTable = registryService.getTableWithFields(
|
||||
datasetRef.getTableId());
|
||||
if (fullTable == null
|
||||
|| !datasetRef.getSourceId().equals(fullTable.getSourceId())
|
||||
|| !Integer.valueOf(1).equals(fullTable.getQueryable())
|
||||
|| !DatacenterMetadataStatus.ACTIVE.name().equals(
|
||||
fullTable.getMetadataStatus())) {
|
||||
throw new BusinessException("查询数据节点绑定的表不可用");
|
||||
}
|
||||
DatacenterCatalog catalog = registryService.getCatalogById(
|
||||
fullTable.getCatalogId());
|
||||
if (StringUtils.hasText(datasetRef.getCatalogName())
|
||||
&& (catalog == null || !datasetRef.getCatalogName().equals(
|
||||
catalog.getCatalogName()))) {
|
||||
throw new BusinessException("查询数据节点绑定的命名空间已变化");
|
||||
}
|
||||
JSONObject sourceSummary = new JSONObject();
|
||||
sourceSummary.put("sourceName", source.getSourceName());
|
||||
sourceSummary.put("sourceType", source.getSourceType());
|
||||
JSONArray tables = new JSONArray();
|
||||
for (DatacenterTable table : managedTables) {
|
||||
DatacenterTable fullTable = registryService.getTableWithFields(table.getId());
|
||||
DatacenterCatalog catalog = registryService.getCatalogById(fullTable.getCatalogId());
|
||||
if (StringUtils.hasText(datasetRef.getCatalogName())
|
||||
&& (catalog == null || !datasetRef.getCatalogName().equals(catalog.getCatalogName()))) {
|
||||
continue;
|
||||
}
|
||||
JSONObject tableSummary = new JSONObject();
|
||||
tableSummary.put("catalogName", catalog == null ? null : catalog.getCatalogName());
|
||||
tableSummary.put("tableName", fullTable.getTableName());
|
||||
tableSummary.put("tableDesc", fullTable.getTableDesc());
|
||||
JSONArray fields = new JSONArray();
|
||||
if (fullTable.getFields() != null) {
|
||||
for (DatacenterTableField field : fullTable.getFields()) {
|
||||
JSONObject fieldSummary = new JSONObject();
|
||||
fieldSummary.put("fieldName", field.getFieldName());
|
||||
fieldSummary.put("fieldDesc", field.getFieldDesc());
|
||||
fieldSummary.put("fieldType", resolveFieldType(field));
|
||||
fields.add(fieldSummary);
|
||||
JSONObject tableSummary = new JSONObject();
|
||||
tableSummary.put("catalogName", catalog == null ? null : catalog.getCatalogName());
|
||||
tableSummary.put("tableName", fullTable.getTableName());
|
||||
tableSummary.put("tableDesc", fullTable.getTableDesc());
|
||||
JSONArray fields = new JSONArray();
|
||||
if (fullTable.getFields() != null) {
|
||||
for (DatacenterTableField field : fullTable.getFields()) {
|
||||
if (!isQueryablePublicField(field)) {
|
||||
continue;
|
||||
}
|
||||
JSONObject fieldSummary = new JSONObject();
|
||||
fieldSummary.put("fieldName", field.getFieldName());
|
||||
fieldSummary.put("fieldDesc", field.getFieldDesc());
|
||||
fieldSummary.put("fieldType", resolveFieldType(field));
|
||||
fields.add(fieldSummary);
|
||||
}
|
||||
tableSummary.put("fields", fields);
|
||||
tables.add(tableSummary);
|
||||
}
|
||||
tableSummary.put("fields", fields);
|
||||
tables.add(tableSummary);
|
||||
sourceSummary.put("tables", tables);
|
||||
return sourceSummary;
|
||||
}
|
||||
|
||||
/**
|
||||
* 判断字段是否允许暴露给 SQL 生成上下文。
|
||||
*
|
||||
* @param field 字段元数据
|
||||
* @return 是否为当前可查询的公开字段
|
||||
*/
|
||||
private boolean isQueryablePublicField(DatacenterTableField field) {
|
||||
return field != null
|
||||
&& Integer.valueOf(1).equals(field.getQueryable())
|
||||
&& DatacenterMetadataStatus.ACTIVE.name().equals(
|
||||
field.getMetadataStatus())
|
||||
&& DatacenterSensitivityLevel.PUBLIC.name().equals(
|
||||
field.getSensitivityLevel());
|
||||
}
|
||||
|
||||
private void injectQueryDataContext(JSONObject data, Map<String, JSONObject> nodeMap) {
|
||||
if (data == null) {
|
||||
return;
|
||||
@@ -217,7 +276,7 @@ public class WorkflowDatacenterContentService {
|
||||
removeQueryDataContextPlaceholder(data);
|
||||
return;
|
||||
}
|
||||
Map<BigInteger, JSONObject> sourceSummaries = new LinkedHashMap<>();
|
||||
Map<String, JSONObject> sourceSummaries = new LinkedHashMap<>();
|
||||
Set<String> visitedNodeIds = new LinkedHashSet<>();
|
||||
for (int i = 0; i < nodeIds.size(); i++) {
|
||||
String nodeId = trimToNull(nodeIds.getString(i));
|
||||
@@ -229,7 +288,10 @@ public class WorkflowDatacenterContentService {
|
||||
throw new BusinessException(INVALID_QUERY_CONTEXT_MESSAGE);
|
||||
}
|
||||
DatasetRef datasetRef = requireSearchDatasetRef(targetNode.getJSONObject("data"));
|
||||
sourceSummaries.putIfAbsent(datasetRef.getSourceId(), buildSourceSummary(datasetRef));
|
||||
String summaryKey = datasetRef.getSourceId() + ":" + datasetRef.getTableId();
|
||||
if (!sourceSummaries.containsKey(summaryKey)) {
|
||||
sourceSummaries.put(summaryKey, buildSourceSummary(datasetRef));
|
||||
}
|
||||
}
|
||||
String contextValue = QUERY_CONTEXT_PROMPT + "\n" + JSON.toJSONString(new ArrayList<>(sourceSummaries.values()));
|
||||
upsertQueryDataContextParameter(data, contextValue);
|
||||
@@ -343,6 +405,7 @@ public class WorkflowDatacenterContentService {
|
||||
|
||||
private DatasetRef copyDatasetRef(DatasetRef datasetRef) {
|
||||
DatasetRef copy = new DatasetRef();
|
||||
copy.setTenantId(datasetRef.getTenantId());
|
||||
copy.setSourceId(datasetRef.getSourceId());
|
||||
copy.setCatalogId(datasetRef.getCatalogId());
|
||||
copy.setCatalogName(datasetRef.getCatalogName());
|
||||
|
||||
@@ -6,7 +6,6 @@ import com.easyagents.flow.core.chain.Chain;
|
||||
import com.easyagents.flow.core.chain.runtime.RetryableTriggerException;
|
||||
import com.easyagents.flow.core.node.BaseNode;
|
||||
import com.easyagents.flow.core.util.IoBulkhead;
|
||||
import com.mybatisflex.core.tenant.TenantManager;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import tech.easyflow.ai.utils.WorkFlowUtil;
|
||||
@@ -55,7 +54,6 @@ public class SaveDatasetNode extends BaseNode {
|
||||
rows.add(item instanceof JSONObject json ? json : JSONObject.from(item));
|
||||
}
|
||||
try {
|
||||
TenantManager.ignoreTenantCondition();
|
||||
try (IoBulkhead.Permit ignored = IoBulkhead.dataset().acquire(resolveIoTarget())) {
|
||||
writeService.saveRowsIdempotently(
|
||||
datasetRef,
|
||||
@@ -77,8 +75,6 @@ public class SaveDatasetNode extends BaseNode {
|
||||
} catch (Exception ex) {
|
||||
log.error("工作流保存数据到统一数据集失败,datasetRef={}", datasetRef, ex);
|
||||
throw ex;
|
||||
} finally {
|
||||
TenantManager.restoreTenantCondition();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -7,12 +7,13 @@ import com.easyagents.flow.core.chain.repository.LoopInputReference;
|
||||
import com.easyagents.flow.core.node.BaseNode;
|
||||
import com.easyagents.flow.core.util.IoBulkhead;
|
||||
import com.mybatisflex.core.row.Row;
|
||||
import com.mybatisflex.core.tenant.TenantManager;
|
||||
import tech.easyflow.common.util.SpringContextUtil;
|
||||
import tech.easyflow.common.entity.LoginAccount;
|
||||
import tech.easyflow.common.web.exceptions.BusinessException;
|
||||
import tech.easyflow.datacenter.execution.model.DatacenterSqlQueryRequest;
|
||||
import tech.easyflow.datacenter.execution.model.DatasetRef;
|
||||
import tech.easyflow.datacenter.execution.service.DatacenterDatasetQueryService;
|
||||
import tech.easyflow.ai.easyagentsflow.cancellation.WorkflowDatasetQueryCancellationRegistry;
|
||||
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
@@ -55,32 +56,38 @@ public class SearchDatasetNode extends BaseNode {
|
||||
Map<String, Object> params =
|
||||
chain.getExecutionState().resolveParameters(this);
|
||||
DatacenterDatasetQueryService queryService = SpringContextUtil.getBean(DatacenterDatasetQueryService.class);
|
||||
WorkflowDatasetQueryCancellationRegistry cancellationRegistry =
|
||||
SpringContextUtil.getBean(
|
||||
WorkflowDatasetQueryCancellationRegistry.class);
|
||||
LoginAccount account = tech.easyflow.ai.utils.WorkFlowUtil.getOperator(chain);
|
||||
DatacenterSqlQueryRequest request = buildRuntimeRequest(params);
|
||||
Map<String, Object> result = new HashMap<>();
|
||||
try {
|
||||
TenantManager.ignoreTenantCondition();
|
||||
try (IoBulkhead.Permit ignored = IoBulkhead.dataset().acquire(resolveIoTarget())) {
|
||||
String resultId = chain.getStateInstanceId()
|
||||
+ ":dataset:"
|
||||
+ UUID.randomUUID();
|
||||
int rowCount =
|
||||
chain.storeProducedLoopInputOutsideLock(
|
||||
resultId,
|
||||
sink -> queryService.consumeBySql(
|
||||
request,
|
||||
QUERY_PAGE_SIZE,
|
||||
sink::accept),
|
||||
0L,
|
||||
chain.currentFencingClaimId(),
|
||||
chain.currentClaimGeneration());
|
||||
result.put(
|
||||
resolveOutputKey("data"),
|
||||
new LoopInputReference(
|
||||
resultId, rowCount));
|
||||
return result;
|
||||
}
|
||||
} finally {
|
||||
TenantManager.restoreTenantCondition();
|
||||
String queryId = UUID.randomUUID().toString();
|
||||
try (WorkflowDatasetQueryCancellationRegistry.Registration registration =
|
||||
cancellationRegistry.register(
|
||||
chain.getStateInstanceId(), account, queryId);
|
||||
IoBulkhead.Permit ignored = IoBulkhead.dataset()
|
||||
.acquire(resolveIoTarget())) {
|
||||
String resultId = chain.getStateInstanceId()
|
||||
+ ":dataset:"
|
||||
+ queryId;
|
||||
int rowCount =
|
||||
chain.storeProducedLoopInputOutsideLock(
|
||||
resultId,
|
||||
sink -> queryService.consumeBySql(
|
||||
request,
|
||||
QUERY_PAGE_SIZE,
|
||||
account,
|
||||
queryId,
|
||||
sink::accept),
|
||||
0L,
|
||||
chain.currentFencingClaimId(),
|
||||
chain.currentClaimGeneration());
|
||||
result.put(
|
||||
resolveOutputKey("data"),
|
||||
new LoopInputReference(
|
||||
resultId, rowCount));
|
||||
return result;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -97,6 +104,9 @@ public class SearchDatasetNode extends BaseNode {
|
||||
}
|
||||
|
||||
private DatacenterSqlQueryRequest buildRuntimeRequest(Map<String, Object> params) {
|
||||
if (datasetRef == null || datasetRef.getSourceId() == null) {
|
||||
throw new BusinessException("数据集绑定缺少连接信息,请重新选择数据集");
|
||||
}
|
||||
DatacenterSqlQueryRequest request = new DatacenterSqlQueryRequest();
|
||||
request.setDatasetRef(copyDatasetRef());
|
||||
request.setSql(resolveQuerySql(params));
|
||||
@@ -128,12 +138,13 @@ public class SearchDatasetNode extends BaseNode {
|
||||
|
||||
private DatasetRef copyDatasetRef() {
|
||||
DatasetRef copy = new DatasetRef();
|
||||
copy.setTenantId(datasetRef == null ? null : datasetRef.getTenantId());
|
||||
copy.setSourceId(datasetRef == null ? null : datasetRef.getSourceId());
|
||||
copy.setCatalogId(datasetRef == null ? null : datasetRef.getCatalogId());
|
||||
copy.setCatalogName(datasetRef == null ? null : datasetRef.getCatalogName());
|
||||
copy.setTableId(null);
|
||||
copy.setTableName(null);
|
||||
copy.setVersionId(null);
|
||||
copy.setTableId(datasetRef == null ? null : datasetRef.getTableId());
|
||||
copy.setTableName(datasetRef == null ? null : datasetRef.getTableName());
|
||||
copy.setVersionId(datasetRef == null ? null : datasetRef.getVersionId());
|
||||
return copy;
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,131 @@
|
||||
package tech.easyflow.ai.easyagentsflow.cancellation;
|
||||
|
||||
import com.easyagents.flow.core.chain.Chain;
|
||||
import com.easyagents.flow.core.chain.ChainStatus;
|
||||
import com.easyagents.flow.core.chain.event.ChainStatusChangeEvent;
|
||||
import java.math.BigInteger;
|
||||
import java.util.UUID;
|
||||
import org.junit.Assert;
|
||||
import org.junit.Test;
|
||||
import org.mockito.ArgumentCaptor;
|
||||
import org.mockito.Mockito;
|
||||
import org.springframework.beans.factory.ObjectProvider;
|
||||
import org.springframework.data.redis.core.StringRedisTemplate;
|
||||
import tech.easyflow.common.entity.LoginAccount;
|
||||
import tech.easyflow.datacenter.federation.DatacenterFederationQueryCancellationService;
|
||||
|
||||
/**
|
||||
* {@link WorkflowDatasetQueryCancellationRegistry} 取消竞态回归测试。
|
||||
*/
|
||||
public class WorkflowDatasetQueryCancellationRegistryTest {
|
||||
|
||||
/**
|
||||
* 验证工作流取消会使用登记租户取消活动 QueryId。
|
||||
*/
|
||||
@Test
|
||||
public void shouldCancelRegisteredQuery() {
|
||||
Fixture fixture = fixture();
|
||||
String queryId = UUID.randomUUID().toString();
|
||||
LoginAccount account = account(BigInteger.valueOf(7L));
|
||||
|
||||
try (WorkflowDatasetQueryCancellationRegistry.Registration ignored =
|
||||
fixture.registry.register("instance-a", account, queryId)) {
|
||||
Assert.assertTrue(fixture.registry.cancelExecution("instance-a"));
|
||||
}
|
||||
|
||||
ArgumentCaptor<LoginAccount> accountCaptor =
|
||||
ArgumentCaptor.forClass(LoginAccount.class);
|
||||
Mockito.verify(fixture.cancellationService).cancel(
|
||||
Mockito.eq(queryId), accountCaptor.capture());
|
||||
Assert.assertEquals(BigInteger.valueOf(7L),
|
||||
accountCaptor.getValue().getTenantId());
|
||||
}
|
||||
|
||||
/**
|
||||
* 验证单个查询取消失败不会阻断同一工作流的其他活动查询。
|
||||
*/
|
||||
@Test
|
||||
public void shouldIsolateCancellationFailuresBetweenQueries() {
|
||||
Fixture fixture = fixture();
|
||||
String failedQueryId = UUID.randomUUID().toString();
|
||||
String succeedingQueryId = UUID.randomUUID().toString();
|
||||
LoginAccount account = account(BigInteger.valueOf(7L));
|
||||
Mockito.doThrow(new IllegalStateException("simulated cancellation failure"))
|
||||
.when(fixture.cancellationService)
|
||||
.cancel(Mockito.eq(failedQueryId), Mockito.any(LoginAccount.class));
|
||||
|
||||
try (WorkflowDatasetQueryCancellationRegistry.Registration ignoredA =
|
||||
fixture.registry.register("instance-failure", account, failedQueryId);
|
||||
WorkflowDatasetQueryCancellationRegistry.Registration ignoredB =
|
||||
fixture.registry.register("instance-failure", account, succeedingQueryId)) {
|
||||
Assert.assertTrue(fixture.registry.cancelExecution("instance-failure"));
|
||||
}
|
||||
|
||||
Mockito.verify(fixture.cancellationService).cancel(
|
||||
Mockito.eq(failedQueryId), Mockito.any(LoginAccount.class));
|
||||
Mockito.verify(fixture.cancellationService).cancel(
|
||||
Mockito.eq(succeedingQueryId), Mockito.any(LoginAccount.class));
|
||||
}
|
||||
|
||||
/**
|
||||
* 验证取消先于节点登记发生时,后续查询仍会立即收到取消。
|
||||
*/
|
||||
@Test
|
||||
public void shouldCancelQueryRegisteredAfterWorkflowCancellation() {
|
||||
Fixture fixture = fixture();
|
||||
String queryId = UUID.randomUUID().toString();
|
||||
|
||||
Assert.assertFalse(fixture.registry.cancelExecution("instance-b"));
|
||||
try (WorkflowDatasetQueryCancellationRegistry.Registration ignored =
|
||||
fixture.registry.register(
|
||||
"instance-b", account(BigInteger.ONE), queryId)) {
|
||||
Mockito.verify(fixture.cancellationService).cancel(
|
||||
Mockito.eq(queryId), Mockito.any(LoginAccount.class));
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 验证监听器只桥接取消终态。
|
||||
*/
|
||||
@Test
|
||||
public void listenerShouldBridgeOnlyCancelledStatus() {
|
||||
WorkflowDatasetQueryCancellationRegistry registry = Mockito.mock(
|
||||
WorkflowDatasetQueryCancellationRegistry.class);
|
||||
WorkflowDatasetQueryCancellationListener listener =
|
||||
new WorkflowDatasetQueryCancellationListener(registry);
|
||||
Chain chain = Mockito.mock(Chain.class);
|
||||
Mockito.when(chain.getStateInstanceId()).thenReturn("instance-c");
|
||||
|
||||
listener.onEvent(new ChainStatusChangeEvent(
|
||||
chain, ChainStatus.SUCCEEDED, ChainStatus.RUNNING), chain);
|
||||
listener.onEvent(new ChainStatusChangeEvent(
|
||||
chain, ChainStatus.CANCELLED, ChainStatus.RUNNING), chain);
|
||||
|
||||
Mockito.verify(registry).cancelExecution("instance-c");
|
||||
}
|
||||
|
||||
@SuppressWarnings("unchecked")
|
||||
private Fixture fixture() {
|
||||
DatacenterFederationQueryCancellationService cancellationService =
|
||||
Mockito.mock(
|
||||
DatacenterFederationQueryCancellationService.class);
|
||||
ObjectProvider<StringRedisTemplate> redisProvider =
|
||||
Mockito.mock(ObjectProvider.class);
|
||||
Mockito.when(redisProvider.getIfAvailable()).thenReturn(null);
|
||||
return new Fixture(
|
||||
new WorkflowDatasetQueryCancellationRegistry(
|
||||
cancellationService, redisProvider),
|
||||
cancellationService);
|
||||
}
|
||||
|
||||
private LoginAccount account(BigInteger tenantId) {
|
||||
LoginAccount account = new LoginAccount();
|
||||
account.setTenantId(tenantId);
|
||||
return account;
|
||||
}
|
||||
|
||||
private record Fixture(
|
||||
WorkflowDatasetQueryCancellationRegistry registry,
|
||||
DatacenterFederationQueryCancellationService cancellationService) {
|
||||
}
|
||||
}
|
||||
@@ -9,6 +9,8 @@ import org.mockito.Mockito;
|
||||
import tech.easyflow.datacenter.entity.DatacenterTable;
|
||||
import tech.easyflow.datacenter.entity.DatacenterTableField;
|
||||
import tech.easyflow.datacenter.meta.entity.DatacenterSource;
|
||||
import tech.easyflow.datacenter.meta.enums.DatacenterMetadataStatus;
|
||||
import tech.easyflow.datacenter.meta.enums.DatacenterSensitivityLevel;
|
||||
import tech.easyflow.datacenter.meta.service.DatacenterDatasetRegistryService;
|
||||
|
||||
import java.lang.reflect.Field;
|
||||
@@ -41,13 +43,22 @@ public class WorkflowDatacenterContentServiceTest {
|
||||
DatacenterSource source = Mockito.mock(DatacenterSource.class);
|
||||
Mockito.when(source.getSourceName()).thenReturn("ama 实验基线模型预算");
|
||||
Mockito.when(source.getSourceType()).thenReturn("EXCEL");
|
||||
Mockito.when(source.getTenantId()).thenReturn(BigInteger.ONE);
|
||||
|
||||
DatacenterTableField modelId = mockField("col_id", "模型ID", "VARCHAR");
|
||||
DatacenterTableField inputPrice = mockField("token", "输入价格", "DECIMAL");
|
||||
DatacenterTableField hidden = mockField("hidden_col", "受限字段", "VARCHAR");
|
||||
Mockito.when(hidden.getQueryable()).thenReturn(0);
|
||||
DatacenterTable table = Mockito.mock(DatacenterTable.class);
|
||||
Mockito.when(table.getId()).thenReturn(TABLE_ID);
|
||||
Mockito.when(table.getSourceId()).thenReturn(SOURCE_ID);
|
||||
Mockito.when(table.getTenantId()).thenReturn(BigInteger.ONE);
|
||||
Mockito.when(table.getTableName()).thenReturn("Sheet1");
|
||||
Mockito.when(table.getFields()).thenReturn(List.of(modelId, inputPrice));
|
||||
Mockito.when(table.getQueryable()).thenReturn(1);
|
||||
Mockito.when(table.getMetadataStatus()).thenReturn(
|
||||
DatacenterMetadataStatus.ACTIVE.name());
|
||||
Mockito.when(table.getFields()).thenReturn(List.of(
|
||||
modelId, inputPrice, hidden));
|
||||
|
||||
Mockito.when(registryService.getSourceRequired(SOURCE_ID)).thenReturn(source);
|
||||
Mockito.when(registryService.listManagedTables(SOURCE_ID, null))
|
||||
@@ -77,6 +88,7 @@ public class WorkflowDatacenterContentServiceTest {
|
||||
Assert.assertTrue(contextValue.contains("col_id"));
|
||||
Assert.assertTrue(contextValue.contains("模型ID"));
|
||||
Assert.assertTrue(contextValue.contains("token AS input_price"));
|
||||
Assert.assertFalse(contextValue.contains("hidden_col"));
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -103,6 +115,7 @@ public class WorkflowDatacenterContentServiceTest {
|
||||
private JSONObject buildWorkflowRoot() {
|
||||
JSONObject datasetRef = new JSONObject();
|
||||
datasetRef.put("sourceId", SOURCE_ID);
|
||||
datasetRef.put("tableId", TABLE_ID);
|
||||
|
||||
JSONObject queryData = new JSONObject();
|
||||
queryData.put("datasetRef", datasetRef);
|
||||
@@ -160,6 +173,11 @@ public class WorkflowDatacenterContentServiceTest {
|
||||
Mockito.when(field.getFieldName()).thenReturn(fieldName);
|
||||
Mockito.when(field.getFieldDesc()).thenReturn(fieldDesc);
|
||||
Mockito.when(field.getJdbcType()).thenReturn(jdbcType);
|
||||
Mockito.when(field.getQueryable()).thenReturn(1);
|
||||
Mockito.when(field.getMetadataStatus()).thenReturn(
|
||||
DatacenterMetadataStatus.ACTIVE.name());
|
||||
Mockito.when(field.getSensitivityLevel()).thenReturn(
|
||||
DatacenterSensitivityLevel.PUBLIC.name());
|
||||
return field;
|
||||
}
|
||||
|
||||
|
||||
@@ -20,6 +20,10 @@
|
||||
<groupId>com.zaxxer</groupId>
|
||||
<artifactId>HikariCP</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>com.easyagents</groupId>
|
||||
<artifactId>easy-agents-federation-sql-adapter-jdbc</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.postgresql</groupId>
|
||||
<artifactId>postgresql</artifactId>
|
||||
|
||||
@@ -0,0 +1,168 @@
|
||||
package tech.easyflow.datacenter.audit;
|
||||
|
||||
import com.fasterxml.jackson.databind.annotation.JsonSerialize;
|
||||
import com.fasterxml.jackson.databind.ser.std.ToStringSerializer;
|
||||
import com.mybatisflex.annotation.Column;
|
||||
import com.mybatisflex.annotation.Id;
|
||||
import com.mybatisflex.annotation.KeyType;
|
||||
import com.mybatisflex.annotation.Table;
|
||||
import com.mybatisflex.core.handler.FastjsonTypeHandler;
|
||||
import java.io.Serializable;
|
||||
import java.math.BigInteger;
|
||||
import java.util.Date;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* 数据中枢统一只读查询审计记录。
|
||||
*/
|
||||
@Table(value = "tb_datacenter_query_audit", comment = "数据中枢统一查询审计")
|
||||
public class DatacenterQueryAudit implements Serializable {
|
||||
|
||||
@Id(keyType = KeyType.Generator, value = "snowFlakeId", comment = "主键")
|
||||
private BigInteger id;
|
||||
@Column(comment = "查询标识")
|
||||
private String queryId;
|
||||
@Column(tenantId = true, comment = "租户ID")
|
||||
private BigInteger tenantId;
|
||||
@Column(comment = "部门ID")
|
||||
private BigInteger deptId;
|
||||
@Column(comment = "执行账号ID")
|
||||
private BigInteger executorAccountId;
|
||||
@Column(comment = "调用方类型")
|
||||
private String callerType;
|
||||
@Column(comment = "调用方标识")
|
||||
private String callerId;
|
||||
@Column(comment = "数据源ID")
|
||||
private BigInteger sourceId;
|
||||
@Column(comment = "数据源版本")
|
||||
private Long sourceRevision;
|
||||
@Column(comment = "纳管范围版本")
|
||||
private Long scopeRevision;
|
||||
@Column(typeHandler = FastjsonTypeHandler.class, comment = "引用表摘要")
|
||||
private List<String> referencedTablesJson;
|
||||
@Column(typeHandler = FastjsonTypeHandler.class, comment = "引用字段摘要")
|
||||
private List<String> referencedFieldsJson;
|
||||
@Column(comment = "参数化或脱敏SQL")
|
||||
private String parameterizedSql;
|
||||
@Column(typeHandler = FastjsonTypeHandler.class, comment = "脱敏参数摘要")
|
||||
private Map<String, Object> maskedParametersJson;
|
||||
@Column(comment = "执行状态")
|
||||
private String status;
|
||||
@Column(comment = "稳定错误码")
|
||||
private String errorCode;
|
||||
@Column(comment = "有界错误说明")
|
||||
private String errorMessage;
|
||||
@Column(comment = "开始时间")
|
||||
private Date startedAt;
|
||||
@Column(comment = "结束时间")
|
||||
private Date finishedAt;
|
||||
@Column(comment = "总耗时毫秒")
|
||||
private Long durationMs;
|
||||
@Column(comment = "数据库耗时毫秒")
|
||||
private Long databaseDurationMs;
|
||||
@Column(comment = "返回行数")
|
||||
private Long returnedRows;
|
||||
@Column(comment = "是否截断")
|
||||
private Integer truncated;
|
||||
@Column(comment = "创建时间")
|
||||
private Date created;
|
||||
|
||||
/** @return 主键 */
|
||||
@JsonSerialize(using = ToStringSerializer.class)
|
||||
public BigInteger getId() { return id; }
|
||||
/** @param id 主键 */
|
||||
public void setId(BigInteger id) { this.id = id; }
|
||||
/** @return 查询标识 */
|
||||
public String getQueryId() { return queryId; }
|
||||
/** @param queryId 查询标识 */
|
||||
public void setQueryId(String queryId) { this.queryId = queryId; }
|
||||
/** @return 租户 ID */
|
||||
public BigInteger getTenantId() { return tenantId; }
|
||||
/** @param tenantId 租户 ID */
|
||||
public void setTenantId(BigInteger tenantId) { this.tenantId = tenantId; }
|
||||
/** @return 部门 ID */
|
||||
public BigInteger getDeptId() { return deptId; }
|
||||
/** @param deptId 部门 ID */
|
||||
public void setDeptId(BigInteger deptId) { this.deptId = deptId; }
|
||||
/** @return 执行账号 ID;系统任务为空 */
|
||||
public BigInteger getExecutorAccountId() { return executorAccountId; }
|
||||
/** @param executorAccountId 执行账号 ID */
|
||||
public void setExecutorAccountId(BigInteger executorAccountId) { this.executorAccountId = executorAccountId; }
|
||||
/** @return 调用方类型 */
|
||||
public String getCallerType() { return callerType; }
|
||||
/** @param callerType 调用方类型 */
|
||||
public void setCallerType(String callerType) { this.callerType = callerType; }
|
||||
/** @return 调用方标识 */
|
||||
public String getCallerId() { return callerId; }
|
||||
/** @param callerId 调用方标识 */
|
||||
public void setCallerId(String callerId) { this.callerId = callerId; }
|
||||
/** @return 数据源 ID */
|
||||
public BigInteger getSourceId() { return sourceId; }
|
||||
/** @param sourceId 数据源 ID */
|
||||
public void setSourceId(BigInteger sourceId) { this.sourceId = sourceId; }
|
||||
/** @return 数据源版本 */
|
||||
public Long getSourceRevision() { return sourceRevision; }
|
||||
/** @param sourceRevision 数据源版本 */
|
||||
public void setSourceRevision(Long sourceRevision) { this.sourceRevision = sourceRevision; }
|
||||
/** @return 纳管范围版本 */
|
||||
public Long getScopeRevision() { return scopeRevision; }
|
||||
/** @param scopeRevision 纳管范围版本 */
|
||||
public void setScopeRevision(Long scopeRevision) { this.scopeRevision = scopeRevision; }
|
||||
/** @return 引用表 */
|
||||
public List<String> getReferencedTablesJson() { return referencedTablesJson; }
|
||||
/** @param referencedTablesJson 引用表 */
|
||||
public void setReferencedTablesJson(List<String> referencedTablesJson) { this.referencedTablesJson = referencedTablesJson; }
|
||||
/** @return 引用字段 */
|
||||
public List<String> getReferencedFieldsJson() { return referencedFieldsJson; }
|
||||
/** @param referencedFieldsJson 引用字段 */
|
||||
public void setReferencedFieldsJson(List<String> referencedFieldsJson) { this.referencedFieldsJson = referencedFieldsJson; }
|
||||
/** @return 参数化 SQL */
|
||||
public String getParameterizedSql() { return parameterizedSql; }
|
||||
/** @param parameterizedSql 参数化 SQL */
|
||||
public void setParameterizedSql(String parameterizedSql) { this.parameterizedSql = parameterizedSql; }
|
||||
/** @return 脱敏参数 */
|
||||
public Map<String, Object> getMaskedParametersJson() { return maskedParametersJson; }
|
||||
/** @param maskedParametersJson 脱敏参数 */
|
||||
public void setMaskedParametersJson(Map<String, Object> maskedParametersJson) { this.maskedParametersJson = maskedParametersJson; }
|
||||
/** @return 状态 */
|
||||
public String getStatus() { return status; }
|
||||
/** @param status 状态 */
|
||||
public void setStatus(String status) { this.status = status; }
|
||||
/** @return 错误码 */
|
||||
public String getErrorCode() { return errorCode; }
|
||||
/** @param errorCode 错误码 */
|
||||
public void setErrorCode(String errorCode) { this.errorCode = errorCode; }
|
||||
/** @return 错误说明 */
|
||||
public String getErrorMessage() { return errorMessage; }
|
||||
/** @param errorMessage 错误说明 */
|
||||
public void setErrorMessage(String errorMessage) { this.errorMessage = errorMessage; }
|
||||
/** @return 开始时间 */
|
||||
public Date getStartedAt() { return startedAt; }
|
||||
/** @param startedAt 开始时间 */
|
||||
public void setStartedAt(Date startedAt) { this.startedAt = startedAt; }
|
||||
/** @return 结束时间 */
|
||||
public Date getFinishedAt() { return finishedAt; }
|
||||
/** @param finishedAt 结束时间 */
|
||||
public void setFinishedAt(Date finishedAt) { this.finishedAt = finishedAt; }
|
||||
/** @return 总耗时 */
|
||||
public Long getDurationMs() { return durationMs; }
|
||||
/** @param durationMs 总耗时 */
|
||||
public void setDurationMs(Long durationMs) { this.durationMs = durationMs; }
|
||||
/** @return 数据库耗时 */
|
||||
public Long getDatabaseDurationMs() { return databaseDurationMs; }
|
||||
/** @param databaseDurationMs 数据库耗时 */
|
||||
public void setDatabaseDurationMs(Long databaseDurationMs) { this.databaseDurationMs = databaseDurationMs; }
|
||||
/** @return 返回行数 */
|
||||
public Long getReturnedRows() { return returnedRows; }
|
||||
/** @param returnedRows 返回行数 */
|
||||
public void setReturnedRows(Long returnedRows) { this.returnedRows = returnedRows; }
|
||||
/** @return 是否截断 */
|
||||
public Integer getTruncated() { return truncated; }
|
||||
/** @param truncated 是否截断 */
|
||||
public void setTruncated(Integer truncated) { this.truncated = truncated; }
|
||||
/** @return 创建时间 */
|
||||
public Date getCreated() { return created; }
|
||||
/** @param created 创建时间 */
|
||||
public void setCreated(Date created) { this.created = created; }
|
||||
}
|
||||
@@ -0,0 +1,301 @@
|
||||
package tech.easyflow.datacenter.audit;
|
||||
|
||||
import java.math.BigInteger;
|
||||
import java.util.Date;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import com.mybatisflex.core.query.QueryWrapper;
|
||||
import org.springframework.scheduling.annotation.Scheduled;
|
||||
import org.springframework.stereotype.Service;
|
||||
import tech.easyflow.common.entity.LoginAccount;
|
||||
import tech.easyflow.datacenter.mapper.DatacenterQueryAuditMapper;
|
||||
import tech.easyflow.datacenter.meta.entity.DatacenterSource;
|
||||
|
||||
/**
|
||||
* 在查询前落账并在资源释放后收口状态的审计服务。
|
||||
*/
|
||||
@Service
|
||||
public class DatacenterQueryAuditService {
|
||||
|
||||
private final DatacenterQueryAuditMapper auditMapper;
|
||||
|
||||
/**
|
||||
* 创建审计服务。
|
||||
*
|
||||
* @param auditMapper 审计 Mapper
|
||||
*/
|
||||
public DatacenterQueryAuditService(DatacenterQueryAuditMapper auditMapper) {
|
||||
this.auditMapper = auditMapper;
|
||||
}
|
||||
|
||||
/**
|
||||
* 在数据库查询开始前写入 RUNNING 记录。
|
||||
*
|
||||
* @param queryId 查询标识
|
||||
* @param source 数据源快照
|
||||
* @param sql 参数化或无参数只读 SQL
|
||||
* @param parameters 脱敏参数摘要
|
||||
* @param account 执行账号
|
||||
* @param callerType 调用方类型
|
||||
* @param callerId 调用方标识
|
||||
* @return 已持久化审计记录
|
||||
*/
|
||||
public DatacenterQueryAudit start(
|
||||
String queryId,
|
||||
DatacenterSource source,
|
||||
String sql,
|
||||
Map<String, Object> parameters,
|
||||
LoginAccount account,
|
||||
String callerType,
|
||||
String callerId) {
|
||||
Date now = new Date();
|
||||
DatacenterQueryAudit audit = new DatacenterQueryAudit();
|
||||
audit.setQueryId(queryId);
|
||||
audit.setTenantId(account == null || account.getTenantId() == null
|
||||
? value(source.getTenantId()) : account.getTenantId());
|
||||
audit.setDeptId(account == null || account.getDeptId() == null
|
||||
? value(source.getDeptId()) : account.getDeptId());
|
||||
audit.setExecutorAccountId(account == null ? null : account.getId());
|
||||
audit.setCallerType(callerType == null ? "SYSTEM" : callerType);
|
||||
audit.setCallerId(callerId);
|
||||
audit.setSourceId(source.getId());
|
||||
audit.setSourceRevision(value(source.getDefinitionRevision()));
|
||||
audit.setScopeRevision(value(source.getScopeRevision()));
|
||||
audit.setReferencedTablesJson(List.of());
|
||||
audit.setReferencedFieldsJson(List.of());
|
||||
audit.setParameterizedSql(redactSql(sql));
|
||||
audit.setMaskedParametersJson(parameters == null ? Map.of() : Map.copyOf(parameters));
|
||||
audit.setStatus("RUNNING");
|
||||
audit.setStartedAt(now);
|
||||
audit.setReturnedRows(0L);
|
||||
audit.setTruncated(0);
|
||||
audit.setCreated(now);
|
||||
auditMapper.insert(audit);
|
||||
return audit;
|
||||
}
|
||||
|
||||
/**
|
||||
* 标记查询成功。
|
||||
*
|
||||
* @param audit 审计记录
|
||||
* @param referencedTables Calcite 解析出的引用表
|
||||
* @param referencedFields Calcite 解析出的引用字段
|
||||
* @param returnedRows 返回行数
|
||||
* @param truncated 是否截断
|
||||
* @param durationMs 总耗时
|
||||
* @param databaseDurationMs 数据库耗时
|
||||
*/
|
||||
public void succeed(
|
||||
DatacenterQueryAudit audit,
|
||||
List<String> referencedTables,
|
||||
List<String> referencedFields,
|
||||
long returnedRows,
|
||||
boolean truncated,
|
||||
long durationMs,
|
||||
long databaseDurationMs) {
|
||||
finish(audit, "SUCCESS", null, null, referencedTables, referencedFields,
|
||||
returnedRows, truncated, durationMs, databaseDurationMs);
|
||||
}
|
||||
|
||||
/**
|
||||
* 标记查询失败。
|
||||
*
|
||||
* @param audit 审计记录
|
||||
* @param errorCode 稳定错误码
|
||||
* @param message 有界错误说明
|
||||
* @param referencedTables Calcite 已解析出的引用表
|
||||
* @param referencedFields Calcite 已解析出的引用字段
|
||||
* @param durationMs 总耗时
|
||||
*/
|
||||
public void fail(
|
||||
DatacenterQueryAudit audit,
|
||||
String errorCode,
|
||||
String message,
|
||||
List<String> referencedTables,
|
||||
List<String> referencedFields,
|
||||
long durationMs) {
|
||||
finish(audit, "FAILED", errorCode, bounded(message), referencedTables, referencedFields,
|
||||
0L, false, durationMs, null);
|
||||
}
|
||||
|
||||
private void finish(
|
||||
DatacenterQueryAudit audit,
|
||||
String status,
|
||||
String errorCode,
|
||||
String errorMessage,
|
||||
List<String> referencedTables,
|
||||
List<String> referencedFields,
|
||||
long returnedRows,
|
||||
boolean truncated,
|
||||
long durationMs,
|
||||
Long databaseDurationMs) {
|
||||
audit.setStatus(status);
|
||||
audit.setErrorCode(errorCode);
|
||||
audit.setErrorMessage(errorMessage);
|
||||
audit.setReferencedTablesJson(List.copyOf(
|
||||
referencedTables == null ? List.of() : referencedTables));
|
||||
audit.setReferencedFieldsJson(List.copyOf(
|
||||
referencedFields == null ? List.of() : referencedFields));
|
||||
audit.setReturnedRows(returnedRows);
|
||||
audit.setTruncated(truncated ? 1 : 0);
|
||||
audit.setDurationMs(durationMs);
|
||||
audit.setDatabaseDurationMs(databaseDurationMs);
|
||||
audit.setFinishedAt(new Date());
|
||||
auditMapper.update(audit);
|
||||
}
|
||||
|
||||
/**
|
||||
* 将超过十分钟仍为 RUNNING 的遗留审计收口为失败,覆盖进程异常退出场景。
|
||||
*/
|
||||
@Scheduled(fixedDelayString = "${easyflow.datacenter.audit-reconcile-ms:60000}")
|
||||
public void reconcileAbandonedQueries() {
|
||||
Date now = new Date();
|
||||
DatacenterQueryAudit patch = new DatacenterQueryAudit();
|
||||
patch.setStatus("FAILED");
|
||||
patch.setErrorCode("QUERY_ABANDONED");
|
||||
patch.setErrorMessage("查询进程异常结束或超出审计存活时间");
|
||||
patch.setFinishedAt(now);
|
||||
QueryWrapper guard = QueryWrapper.create()
|
||||
.eq(DatacenterQueryAudit::getStatus, "RUNNING")
|
||||
.lt(DatacenterQueryAudit::getStartedAt, new Date(now.getTime() - 600_000L));
|
||||
auditMapper.updateByQuery(patch, guard);
|
||||
}
|
||||
|
||||
private long value(Long value) {
|
||||
return value == null ? 0L : value;
|
||||
}
|
||||
|
||||
private BigInteger value(BigInteger value) {
|
||||
return value == null ? BigInteger.ZERO : value;
|
||||
}
|
||||
|
||||
private String redactSql(String sql) {
|
||||
if (sql == null || sql.isBlank()) {
|
||||
return sql;
|
||||
}
|
||||
StringBuilder result = new StringBuilder(Math.min(sql.length(), 100_000));
|
||||
boolean quoted = false;
|
||||
for (int index = 0; index < sql.length(); index++) {
|
||||
char current = sql.charAt(index);
|
||||
if (quoted) {
|
||||
if (current == '\'' && index + 1 < sql.length() && sql.charAt(index + 1) == '\'') {
|
||||
index++;
|
||||
continue;
|
||||
}
|
||||
if (current == '\'') {
|
||||
quoted = false;
|
||||
result.append('?');
|
||||
}
|
||||
continue;
|
||||
}
|
||||
if (current == '\'') {
|
||||
quoted = true;
|
||||
continue;
|
||||
}
|
||||
if (current == '-' && index + 1 < sql.length() && sql.charAt(index + 1) == '-') {
|
||||
result.append("--?");
|
||||
index += 2;
|
||||
while (index < sql.length() && sql.charAt(index) != '\n'
|
||||
&& sql.charAt(index) != '\r') {
|
||||
index++;
|
||||
}
|
||||
if (index < sql.length()) {
|
||||
result.append(sql.charAt(index));
|
||||
}
|
||||
continue;
|
||||
}
|
||||
if (current == '/' && index + 1 < sql.length() && sql.charAt(index + 1) == '*') {
|
||||
result.append("/*?*/");
|
||||
index = skipBlockComment(sql, index + 2);
|
||||
continue;
|
||||
}
|
||||
String dollarDelimiter = current == '$' ? dollarDelimiter(sql, index) : null;
|
||||
if (dollarDelimiter != null) {
|
||||
int end = sql.indexOf(dollarDelimiter, index + dollarDelimiter.length());
|
||||
result.append('?');
|
||||
index = end < 0 ? sql.length() : end + dollarDelimiter.length() - 1;
|
||||
continue;
|
||||
}
|
||||
if (Character.isDigit(current)
|
||||
&& (index == 0 || !Character.isJavaIdentifierPart(sql.charAt(index - 1)))) {
|
||||
result.append('?');
|
||||
while (index + 1 < sql.length()) {
|
||||
char next = sql.charAt(index + 1);
|
||||
if (!Character.isDigit(next) && next != '.' && next != 'e' && next != 'E'
|
||||
&& next != '+' && next != '-') {
|
||||
break;
|
||||
}
|
||||
index++;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
result.append(current);
|
||||
if (result.length() >= 100_000) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (quoted) {
|
||||
result.append('?');
|
||||
}
|
||||
return result.toString();
|
||||
}
|
||||
|
||||
/**
|
||||
* 跳过支持嵌套的块注释内容。
|
||||
*
|
||||
* @param sql 原始 SQL
|
||||
* @param contentStart 首个注释内容字符位置
|
||||
* @return 外层注释结束标记的末尾位置;未闭合时返回末尾
|
||||
*/
|
||||
private int skipBlockComment(String sql, int contentStart) {
|
||||
int depth = 1;
|
||||
for (int index = contentStart; index < sql.length() - 1; index++) {
|
||||
if (sql.charAt(index) == '/' && sql.charAt(index + 1) == '*') {
|
||||
depth++;
|
||||
index++;
|
||||
} else if (sql.charAt(index) == '*' && sql.charAt(index + 1) == '/') {
|
||||
depth--;
|
||||
if (depth == 0) {
|
||||
return index + 1;
|
||||
}
|
||||
index++;
|
||||
}
|
||||
}
|
||||
return sql.length();
|
||||
}
|
||||
|
||||
/**
|
||||
* 识别 PostgreSQL dollar-quoted 字面量起始分隔符。
|
||||
*
|
||||
* @param sql 原始 SQL
|
||||
* @param start 美元符号位置
|
||||
* @return 完整分隔符;当前位置不是合法起始符时为空
|
||||
*/
|
||||
private String dollarDelimiter(String sql, int start) {
|
||||
int end = sql.indexOf('$', start + 1);
|
||||
if (end < 0) {
|
||||
return null;
|
||||
}
|
||||
String tag = sql.substring(start + 1, end);
|
||||
if (tag.isEmpty()) {
|
||||
return "$$";
|
||||
}
|
||||
if (!Character.isLetter(tag.charAt(0)) && tag.charAt(0) != '_') {
|
||||
return null;
|
||||
}
|
||||
for (int index = 1; index < tag.length(); index++) {
|
||||
char character = tag.charAt(index);
|
||||
if (!Character.isLetterOrDigit(character) && character != '_') {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
return '$' + tag + '$';
|
||||
}
|
||||
|
||||
private String bounded(String message) {
|
||||
if (message == null) {
|
||||
return null;
|
||||
}
|
||||
return message.length() <= 500 ? message : message.substring(0, 500);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,55 @@
|
||||
package tech.easyflow.datacenter.config;
|
||||
|
||||
import com.easyagents.federation.sql.source.SourceId;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.boot.autoconfigure.condition.ConditionalOnBean;
|
||||
import org.springframework.context.annotation.Bean;
|
||||
import org.springframework.context.annotation.Configuration;
|
||||
import org.springframework.data.redis.connection.RedisConnectionFactory;
|
||||
import org.springframework.data.redis.listener.ChannelTopic;
|
||||
import org.springframework.data.redis.listener.RedisMessageListenerContainer;
|
||||
import tech.easyflow.datacenter.federation.DatacenterFederationChangeNotifier;
|
||||
import tech.easyflow.datacenter.federation.DatacenterFederationQueryCancellationService;
|
||||
import tech.easyflow.datacenter.federation.DatacenterFederationSourceStateProvider;
|
||||
|
||||
/**
|
||||
* 数据中枢无凭据 Definition 变更提示订阅配置。
|
||||
*/
|
||||
@Configuration(proxyBeanMethods = false)
|
||||
public class DatacenterFederationRedisConfig {
|
||||
|
||||
private static final Logger log = LoggerFactory.getLogger(DatacenterFederationRedisConfig.class);
|
||||
|
||||
/**
|
||||
* 创建独立 Redis 监听容器,收到提示后始终回源数据库。
|
||||
*
|
||||
* @param connectionFactory Redis 连接工厂
|
||||
* @param stateProvider 数据库权威状态 Provider
|
||||
* @param queryCancellationService 查询取消服务
|
||||
* @return 监听容器
|
||||
*/
|
||||
@Bean(name = "datacenterFederationRedisListenerContainer")
|
||||
@ConditionalOnBean(RedisConnectionFactory.class)
|
||||
public RedisMessageListenerContainer datacenterFederationRedisListenerContainer(
|
||||
RedisConnectionFactory connectionFactory,
|
||||
DatacenterFederationSourceStateProvider stateProvider,
|
||||
DatacenterFederationQueryCancellationService queryCancellationService) {
|
||||
RedisMessageListenerContainer container = new RedisMessageListenerContainer();
|
||||
container.setConnectionFactory(connectionFactory);
|
||||
container.addMessageListener((message, pattern) -> {
|
||||
String value = new String(message.getBody(), StandardCharsets.UTF_8);
|
||||
try {
|
||||
stateProvider.refresh(new SourceId(value));
|
||||
} catch (RuntimeException exception) {
|
||||
log.warn("Failed to refresh datacenter Federation source {}", value, exception);
|
||||
}
|
||||
}, new ChannelTopic(DatacenterFederationChangeNotifier.CHANNEL));
|
||||
container.addMessageListener((message, pattern) ->
|
||||
queryCancellationService.acceptCancellationHint(
|
||||
new String(message.getBody(), StandardCharsets.UTF_8)),
|
||||
new ChannelTopic(DatacenterFederationQueryCancellationService.CHANNEL));
|
||||
return container;
|
||||
}
|
||||
}
|
||||
@@ -3,6 +3,8 @@ package tech.easyflow.datacenter.connector;
|
||||
import tech.easyflow.datacenter.entity.DatacenterTable;
|
||||
import tech.easyflow.datacenter.meta.entity.DatacenterSource;
|
||||
import tech.easyflow.datacenter.meta.model.DatacenterCatalogMeta;
|
||||
import tech.easyflow.datacenter.meta.model.DatacenterManagedMetadataSnapshot;
|
||||
import tech.easyflow.datacenter.meta.model.DatacenterMetadataPage;
|
||||
import tech.easyflow.datacenter.meta.model.DatacenterTableDetailMeta;
|
||||
|
||||
import java.util.List;
|
||||
@@ -10,7 +12,97 @@ import java.util.List;
|
||||
public interface MetadataExplorer {
|
||||
List<DatacenterCatalogMeta> listCatalogs(DatacenterSource source);
|
||||
|
||||
/**
|
||||
* 分页浏览 Catalog 或 Schema。
|
||||
*
|
||||
* @param source 数据源
|
||||
* @param keyword 名称搜索词
|
||||
* @param pageNumber 页码
|
||||
* @param pageSize 每页大小
|
||||
* @return 有界命名空间列表
|
||||
*/
|
||||
default DatacenterMetadataPage<DatacenterCatalogMeta> listCatalogsPage(
|
||||
DatacenterSource source,
|
||||
String keyword,
|
||||
long pageNumber,
|
||||
long pageSize) {
|
||||
String normalizedKeyword = keyword == null
|
||||
? "" : keyword.trim().toLowerCase(java.util.Locale.ROOT);
|
||||
List<DatacenterCatalogMeta> filtered = listCatalogs(source).stream()
|
||||
.filter(catalog -> normalizedKeyword.isEmpty()
|
||||
|| String.valueOf(catalog.getCatalogName())
|
||||
.toLowerCase(java.util.Locale.ROOT)
|
||||
.contains(normalizedKeyword))
|
||||
.toList();
|
||||
return DatacenterMetadataPage.slice(filtered, pageNumber, pageSize);
|
||||
}
|
||||
|
||||
List<DatacenterTable> listTables(DatacenterSource source, String catalogName);
|
||||
|
||||
/**
|
||||
* 分页浏览表或视图。
|
||||
*
|
||||
* <p>非 JDBC Connector 默认复用已有列表能力;外部 JDBC Connector 应覆盖此方法,
|
||||
* 直接在元数据 ResultSet 上做有界读取。</p>
|
||||
*
|
||||
* @param source 数据源
|
||||
* @param catalogName 物理命名空间
|
||||
* @param keyword 表名搜索词
|
||||
* @param pageNumber 页码
|
||||
* @param pageSize 每页大小
|
||||
* @return 有界表列表
|
||||
*/
|
||||
default DatacenterMetadataPage<DatacenterTable> listTablesPage(
|
||||
DatacenterSource source,
|
||||
String catalogName,
|
||||
String keyword,
|
||||
long pageNumber,
|
||||
long pageSize) {
|
||||
String normalizedKeyword = keyword == null ? "" : keyword.trim().toLowerCase(java.util.Locale.ROOT);
|
||||
List<DatacenterTable> filtered = listTables(source, catalogName).stream()
|
||||
.filter(table -> normalizedKeyword.isEmpty()
|
||||
|| String.valueOf(table.getTableName())
|
||||
.toLowerCase(java.util.Locale.ROOT)
|
||||
.contains(normalizedKeyword))
|
||||
.toList();
|
||||
return DatacenterMetadataPage.slice(filtered, pageNumber, pageSize);
|
||||
}
|
||||
|
||||
DatacenterTableDetailMeta getTableDetail(DatacenterSource source, String catalogName, String tableName);
|
||||
|
||||
/**
|
||||
* 批量读取表详情。
|
||||
*
|
||||
* @param source 数据源
|
||||
* @param catalogName 物理命名空间
|
||||
* @param tableNames 表名集合
|
||||
* @return 表详情集合
|
||||
*/
|
||||
default List<DatacenterTableDetailMeta> getTableDetails(
|
||||
DatacenterSource source,
|
||||
String catalogName,
|
||||
List<String> tableNames) {
|
||||
return tableNames.stream()
|
||||
.map(tableName -> getTableDetail(source, catalogName, tableName))
|
||||
.toList();
|
||||
}
|
||||
|
||||
/**
|
||||
* 读取已纳管表的刷新快照。
|
||||
*
|
||||
* <p>默认实现保留 Connector 的既有逐表语义;JDBC Connector 应覆盖此方法以复用
|
||||
* 单个连接,并明确区分物理表缺失与连接故障。</p>
|
||||
*
|
||||
* @param source 数据源
|
||||
* @param catalogName 物理命名空间
|
||||
* @param tableNames 已纳管表名集合
|
||||
* @return 存在与缺失对象的快照
|
||||
*/
|
||||
default DatacenterManagedMetadataSnapshot inspectManagedTables(
|
||||
DatacenterSource source,
|
||||
String catalogName,
|
||||
List<String> tableNames) {
|
||||
return new DatacenterManagedMetadataSnapshot(
|
||||
getTableDetails(source, catalogName, tableNames), java.util.Set.of());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -28,13 +28,12 @@ public class MysqlConnector extends AbstractJdbcConnector {
|
||||
|
||||
@Override
|
||||
protected <T> T withConnection(DatacenterSource source, boolean cacheable, JdbcCallback<T> callback) throws Exception {
|
||||
HikariDataSource dataSource = cacheable ? datasourceManager.getOrCreateExternalDatasource(source) : datasourceManager.createExternalDatasource(source);
|
||||
// MySQL 查询 Runtime 已由 Federation SQL 独占管理;此 Connector 只保留短生命周期元数据访问。
|
||||
HikariDataSource dataSource = datasourceManager.createExternalDatasource(source);
|
||||
try (Connection connection = dataSource.getConnection()) {
|
||||
return callback.apply(connection);
|
||||
} finally {
|
||||
if (!cacheable || source.getId() == null) {
|
||||
dataSource.close();
|
||||
}
|
||||
dataSource.close();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -28,13 +28,12 @@ public class PostgresqlConnector extends AbstractJdbcConnector {
|
||||
|
||||
@Override
|
||||
protected <T> T withConnection(DatacenterSource source, boolean cacheable, JdbcCallback<T> callback) throws Exception {
|
||||
HikariDataSource dataSource = cacheable ? datasourceManager.getOrCreateExternalDatasource(source) : datasourceManager.createExternalDatasource(source);
|
||||
// PostgreSQL 查询 Runtime 已由 Federation SQL 独占管理;此 Connector 只保留短生命周期元数据访问。
|
||||
HikariDataSource dataSource = datasourceManager.createExternalDatasource(source);
|
||||
try (Connection connection = dataSource.getConnection()) {
|
||||
return callback.apply(connection);
|
||||
} finally {
|
||||
if (!cacheable || source.getId() == null) {
|
||||
dataSource.close();
|
||||
}
|
||||
dataSource.close();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3,6 +3,7 @@ package tech.easyflow.datacenter.connector.support;
|
||||
import cn.hutool.core.util.StrUtil;
|
||||
import com.alibaba.fastjson2.JSONObject;
|
||||
import com.mybatisflex.core.paginate.Page;
|
||||
import com.mybatisflex.core.query.QueryColumn;
|
||||
import com.mybatisflex.core.query.QueryWrapper;
|
||||
import com.mybatisflex.core.row.Db;
|
||||
import com.mybatisflex.core.row.Row;
|
||||
@@ -15,6 +16,7 @@ import tech.easyflow.datacenter.connector.DatacenterConnector;
|
||||
import tech.easyflow.datacenter.entity.DatacenterTable;
|
||||
import tech.easyflow.datacenter.entity.DatacenterTableField;
|
||||
import tech.easyflow.datacenter.execution.model.DatacenterConnectionTestResult;
|
||||
import tech.easyflow.datacenter.execution.model.DatacenterQueryFilter;
|
||||
import tech.easyflow.datacenter.execution.model.DatacenterQueryRequest;
|
||||
import tech.easyflow.datacenter.meta.entity.DatacenterSource;
|
||||
import tech.easyflow.datacenter.meta.enums.DatacenterCapability;
|
||||
@@ -85,35 +87,180 @@ public abstract class AbstractInternalTableConnector implements DatacenterConnec
|
||||
|
||||
@Override
|
||||
public Page<Row> queryPage(DatacenterSource source, DatacenterTable table, DatacenterQueryRequest request) {
|
||||
if (request == null || request.getPageSize() == null
|
||||
|| request.getPageSize() < 1L
|
||||
|| request.getPageSize() > 500L) {
|
||||
throw new BusinessException("pageSize 必须在 1 到 500 之间");
|
||||
}
|
||||
String actualTable = resolveTableName(table);
|
||||
long count = Db.selectCountByQuery(
|
||||
actualTable, createQueryWrapper(request.getWhere()));
|
||||
actualTable, createQueryWrapper(table, request, false));
|
||||
if (count == 0) {
|
||||
return new Page<>(new ArrayList<>(), request.getPageNumber(), request.getPageSize(), count);
|
||||
}
|
||||
// selectCountByQuery 会把无投影的 QueryWrapper 改为 COUNT(*),分页查询必须使用独立实例。
|
||||
QueryWrapper pageQuery = createQueryWrapper(table, request, true);
|
||||
pageQuery.select(resolveSelectedColumns(table, request).stream()
|
||||
.map(this::quoteInternalIdentifier)
|
||||
.toArray(String[]::new));
|
||||
Page<Row> page = Db.paginate(
|
||||
actualTable,
|
||||
new Page<>(request.getPageNumber(), request.getPageSize(), count),
|
||||
createQueryWrapper(request.getWhere()));
|
||||
pageQuery);
|
||||
normalizeRows(page.getRecords());
|
||||
return page;
|
||||
}
|
||||
|
||||
/**
|
||||
* 将已在服务层校验的逻辑字段映射为内部物理列。
|
||||
*
|
||||
* @param table 绑定表及字段
|
||||
* @param request 查询请求
|
||||
* @return 按请求顺序排列的物理列名
|
||||
* @throws BusinessException 字段元数据缺失时抛出
|
||||
*/
|
||||
private List<String> resolveSelectedColumns(
|
||||
DatacenterTable table,
|
||||
DatacenterQueryRequest request) {
|
||||
Map<String, DatacenterTableField> fields = new LinkedHashMap<>();
|
||||
for (DatacenterTableField field : table.getFields()) {
|
||||
fields.put(field.getFieldName(), field);
|
||||
}
|
||||
List<String> columns = new ArrayList<>();
|
||||
for (String selected : request.getSelectedColumns()) {
|
||||
DatacenterTableField field = fields.get(selected);
|
||||
if (field == null) {
|
||||
throw new BusinessException("字段不存在: " + selected);
|
||||
}
|
||||
String sourceColumn = StrUtil.blankToDefault(
|
||||
field.getSourceColumnName(), field.getFieldName());
|
||||
columns.add(sourceColumn);
|
||||
}
|
||||
if (columns.isEmpty()) {
|
||||
throw new BusinessException("当前数据集没有可查询字段");
|
||||
}
|
||||
return columns;
|
||||
}
|
||||
|
||||
/**
|
||||
* 引用可信元数据列名,避免保留字或特殊字符改变 SQL 结构。
|
||||
*
|
||||
* @param identifier 物理列名
|
||||
* @return MySQL 反引号标识符
|
||||
*/
|
||||
private String quoteInternalIdentifier(String identifier) {
|
||||
return "`" + identifier.replace("`", "``") + "`";
|
||||
}
|
||||
|
||||
/**
|
||||
* 创建用于动态表查询的独立条件包装器。
|
||||
*
|
||||
* @param where 已校验的筛选表达式
|
||||
* @param where 调用方传入的旧原始筛选表达式;非空时拒绝
|
||||
* @return 新建的查询条件包装器
|
||||
*/
|
||||
static QueryWrapper createQueryWrapper(String where) {
|
||||
QueryWrapper wrapper = QueryWrapper.create();
|
||||
if (StrUtil.isNotBlank(where)) {
|
||||
wrapper.where(where);
|
||||
throw new BusinessException("不支持原始 WHERE 条件,请使用结构化筛选条件");
|
||||
}
|
||||
QueryWrapper wrapper = QueryWrapper.create();
|
||||
return wrapper;
|
||||
}
|
||||
|
||||
private QueryWrapper createQueryWrapper(
|
||||
DatacenterTable table,
|
||||
DatacenterQueryRequest request,
|
||||
boolean includeSorts) {
|
||||
QueryWrapper wrapper = createQueryWrapper(request.getWhere());
|
||||
Map<String, DatacenterTableField> fields = table.getFields().stream()
|
||||
.collect(java.util.stream.Collectors.toMap(
|
||||
DatacenterTableField::getFieldName,
|
||||
field -> field,
|
||||
(first, ignored) -> first,
|
||||
LinkedHashMap::new));
|
||||
List<Object> parameters = new ArrayList<>();
|
||||
StringBuilder condition = new StringBuilder();
|
||||
if (request.getFilters() != null) {
|
||||
for (DatacenterQueryFilter filter : request.getFilters()) {
|
||||
DatacenterTableField field = fields.get(filter.getColumn());
|
||||
if (field == null) {
|
||||
throw new BusinessException("字段不存在: " + filter.getColumn());
|
||||
}
|
||||
appendFilter(condition, parameters, field, filter);
|
||||
}
|
||||
}
|
||||
if (!condition.isEmpty()) {
|
||||
wrapper.where(condition.toString(), parameters.toArray());
|
||||
}
|
||||
if (includeSorts && request.getSorts() != null && !request.getSorts().isEmpty()) {
|
||||
request.getSorts().forEach(sort -> {
|
||||
DatacenterTableField field = fields.get(sort.getColumn());
|
||||
if (field == null) {
|
||||
throw new BusinessException("字段不存在: " + sort.getColumn());
|
||||
}
|
||||
wrapper.orderBy(
|
||||
new QueryColumn(sourceColumnName(field)),
|
||||
!"DESC".equalsIgnoreCase(sort.getDirection()));
|
||||
});
|
||||
}
|
||||
return wrapper;
|
||||
}
|
||||
|
||||
private void appendFilter(
|
||||
StringBuilder condition,
|
||||
List<Object> parameters,
|
||||
DatacenterTableField field,
|
||||
DatacenterQueryFilter filter) {
|
||||
if (!condition.isEmpty()) {
|
||||
condition.append(" AND ");
|
||||
}
|
||||
String column = quoteInternalIdentifier(sourceColumnName(field));
|
||||
String operator = StrUtil.blankToDefault(filter.getOperator(), "EQ")
|
||||
.toUpperCase(Locale.ROOT);
|
||||
switch (operator) {
|
||||
case "EQ" -> appendComparison(condition, parameters, column, "=", filter.getValue());
|
||||
case "LIKE" -> appendComparison(
|
||||
condition, parameters, column, "LIKE",
|
||||
"%" + String.valueOf(filter.getValue()) + "%");
|
||||
case "GT" -> appendComparison(condition, parameters, column, ">", filter.getValue());
|
||||
case "GTE" -> appendComparison(condition, parameters, column, ">=", filter.getValue());
|
||||
case "LT" -> appendComparison(condition, parameters, column, "<", filter.getValue());
|
||||
case "LTE" -> appendComparison(condition, parameters, column, "<=", filter.getValue());
|
||||
case "IS_NULL" -> condition.append(column).append(" IS NULL");
|
||||
case "IN" -> appendInFilter(condition, parameters, column, filter.getValues());
|
||||
default -> throw new BusinessException("不支持的过滤操作: " + operator);
|
||||
}
|
||||
}
|
||||
|
||||
private void appendComparison(
|
||||
StringBuilder condition,
|
||||
List<Object> parameters,
|
||||
String column,
|
||||
String operator,
|
||||
Object value) {
|
||||
condition.append(column).append(' ').append(operator).append(" ?");
|
||||
parameters.add(value);
|
||||
}
|
||||
|
||||
private void appendInFilter(
|
||||
StringBuilder condition,
|
||||
List<Object> parameters,
|
||||
String column,
|
||||
List<Object> values) {
|
||||
if (values == null || values.isEmpty()) {
|
||||
condition.append("1 = 0");
|
||||
return;
|
||||
}
|
||||
condition.append(column).append(" IN (")
|
||||
.append(Collections.nCopies(values.size(), "?").stream()
|
||||
.collect(java.util.stream.Collectors.joining(", ")))
|
||||
.append(')');
|
||||
parameters.addAll(values);
|
||||
}
|
||||
|
||||
private String sourceColumnName(DatacenterTableField field) {
|
||||
return StrUtil.blankToDefault(field.getSourceColumnName(), field.getFieldName());
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<Row> queryBySql(DatacenterSource source, String sql) {
|
||||
List<Row> rows = Db.selectListBySql(sql);
|
||||
|
||||
@@ -16,12 +16,17 @@ import tech.easyflow.datacenter.execution.model.DatacenterConnectionTestResult;
|
||||
import tech.easyflow.datacenter.execution.model.DatacenterQueryFilter;
|
||||
import tech.easyflow.datacenter.execution.model.DatacenterQueryRequest;
|
||||
import tech.easyflow.datacenter.execution.model.DatacenterQuerySort;
|
||||
import tech.easyflow.datacenter.federation.DatacenterMetadataIdentity;
|
||||
import tech.easyflow.datacenter.meta.entity.DatacenterSource;
|
||||
import tech.easyflow.datacenter.meta.enums.DatacenterCapability;
|
||||
import tech.easyflow.datacenter.meta.enums.DatacenterConnectionErrorCode;
|
||||
import tech.easyflow.datacenter.meta.enums.DatacenterMetadataStatus;
|
||||
import tech.easyflow.datacenter.meta.enums.DatacenterSensitivityLevel;
|
||||
import tech.easyflow.datacenter.meta.enums.DatacenterSourceType;
|
||||
import tech.easyflow.datacenter.meta.enums.DatacenterTableKind;
|
||||
import tech.easyflow.datacenter.meta.model.DatacenterCatalogMeta;
|
||||
import tech.easyflow.datacenter.meta.model.DatacenterMetadataPage;
|
||||
import tech.easyflow.datacenter.meta.model.DatacenterManagedMetadataSnapshot;
|
||||
import tech.easyflow.datacenter.meta.model.DatacenterTableDetailMeta;
|
||||
|
||||
import java.math.BigDecimal;
|
||||
@@ -105,30 +110,39 @@ public abstract class AbstractJdbcConnector implements DatacenterConnector {
|
||||
return withConnection(source, true, connection -> {
|
||||
List<DatacenterCatalogMeta> result = new ArrayList<>();
|
||||
DatabaseMetaData metaData = connection.getMetaData();
|
||||
try (ResultSet catalogs = metaData.getCatalogs()) {
|
||||
while (catalogs.next()) {
|
||||
String name = catalogs.getString("TABLE_CAT");
|
||||
if (StrUtil.isBlank(name)) {
|
||||
continue;
|
||||
if (usesCatalogNamespace()) {
|
||||
try (ResultSet catalogs = metaData.getCatalogs()) {
|
||||
while (catalogs.next()) {
|
||||
String name = catalogs.getString("TABLE_CAT");
|
||||
if (StrUtil.isBlank(name)) {
|
||||
continue;
|
||||
}
|
||||
DatacenterCatalogMeta meta = new DatacenterCatalogMeta();
|
||||
meta.setSourceId(source.getId());
|
||||
meta.setCatalogName(name);
|
||||
meta.setCatalogType("CATALOG");
|
||||
meta.setLogicalSchemaName(name);
|
||||
meta.setPhysicalCatalogName(name);
|
||||
result.add(meta);
|
||||
}
|
||||
DatacenterCatalogMeta meta = new DatacenterCatalogMeta();
|
||||
meta.setSourceId(source.getId());
|
||||
meta.setCatalogName(name);
|
||||
meta.setCatalogType("CATALOG");
|
||||
result.add(meta);
|
||||
}
|
||||
}
|
||||
try (ResultSet schemas = metaData.getSchemas()) {
|
||||
while (schemas.next()) {
|
||||
String name = schemas.getString("TABLE_SCHEM");
|
||||
if (StrUtil.isBlank(name) || containsCatalog(result, name)) {
|
||||
continue;
|
||||
if (!usesCatalogNamespace()) {
|
||||
try (ResultSet schemas = metaData.getSchemas()) {
|
||||
while (schemas.next()) {
|
||||
String name = schemas.getString("TABLE_SCHEM");
|
||||
if (StrUtil.isBlank(name)) {
|
||||
continue;
|
||||
}
|
||||
DatacenterCatalogMeta meta = new DatacenterCatalogMeta();
|
||||
meta.setSourceId(source.getId());
|
||||
meta.setCatalogName(name);
|
||||
meta.setCatalogType("SCHEMA");
|
||||
meta.setLogicalSchemaName(name);
|
||||
meta.setPhysicalCatalogName(source.getDatabaseName());
|
||||
meta.setPhysicalSchemaName(name);
|
||||
result.add(meta);
|
||||
}
|
||||
DatacenterCatalogMeta meta = new DatacenterCatalogMeta();
|
||||
meta.setSourceId(source.getId());
|
||||
meta.setCatalogName(name);
|
||||
meta.setCatalogType("SCHEMA");
|
||||
result.add(meta);
|
||||
}
|
||||
}
|
||||
result = filterConfiguredCatalogs(source, result);
|
||||
@@ -139,6 +153,13 @@ public abstract class AbstractJdbcConnector implements DatacenterConnector {
|
||||
meta.setSourceId(source.getId());
|
||||
meta.setCatalogName(fallback);
|
||||
meta.setCatalogType("DEFAULT");
|
||||
meta.setLogicalSchemaName(fallback);
|
||||
if (usesCatalogNamespace()) {
|
||||
meta.setPhysicalCatalogName(fallback);
|
||||
} else {
|
||||
meta.setPhysicalCatalogName(source.getDatabaseName());
|
||||
meta.setPhysicalSchemaName(fallback);
|
||||
}
|
||||
result.add(meta);
|
||||
}
|
||||
}
|
||||
@@ -149,24 +170,98 @@ public abstract class AbstractJdbcConnector implements DatacenterConnector {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 直接在 JDBC Catalog/Schema 元数据游标上执行有界分页。
|
||||
*
|
||||
* @param source 数据源
|
||||
* @param keyword 名称搜索词
|
||||
* @param pageNumber 页码
|
||||
* @param pageSize 每页大小
|
||||
* @return 有界命名空间列表
|
||||
*/
|
||||
@Override
|
||||
public DatacenterMetadataPage<DatacenterCatalogMeta> listCatalogsPage(
|
||||
DatacenterSource source,
|
||||
String keyword,
|
||||
long pageNumber,
|
||||
long pageSize) {
|
||||
try {
|
||||
return withConnection(source, true, connection -> {
|
||||
DatabaseMetaData metadata = connection.getMetaData();
|
||||
String normalizedKeyword = keyword == null
|
||||
? "" : keyword.trim().toLowerCase(Locale.ROOT);
|
||||
String configuredName = usesCatalogNamespace()
|
||||
? StrUtil.trimToNull(source.getDatabaseName())
|
||||
: StrUtil.trimToNull(source.getSchemaName());
|
||||
long offset = (pageNumber - 1L) * pageSize;
|
||||
long matched = 0L;
|
||||
boolean hasMore = false;
|
||||
List<DatacenterCatalogMeta> records = new ArrayList<>((int) pageSize);
|
||||
try (ResultSet names = usesCatalogNamespace()
|
||||
? metadata.getCatalogs() : metadata.getSchemas()) {
|
||||
String column = usesCatalogNamespace() ? "TABLE_CAT" : "TABLE_SCHEM";
|
||||
while (names.next()) {
|
||||
String name = names.getString(column);
|
||||
if (StrUtil.isBlank(name)
|
||||
|| (configuredName != null && !configuredName.equals(name))
|
||||
|| (!normalizedKeyword.isEmpty()
|
||||
&& !name.toLowerCase(Locale.ROOT).contains(normalizedKeyword))) {
|
||||
continue;
|
||||
}
|
||||
if (matched++ < offset) {
|
||||
continue;
|
||||
}
|
||||
if (records.size() >= pageSize) {
|
||||
hasMore = true;
|
||||
break;
|
||||
}
|
||||
records.add(catalogMeta(source, name));
|
||||
}
|
||||
}
|
||||
if (records.isEmpty() && offset == 0L && normalizedKeyword.isEmpty()) {
|
||||
String fallback = resolveCatalogName(source, null);
|
||||
if (StrUtil.isNotBlank(fallback)) {
|
||||
records.add(catalogMeta(source, fallback));
|
||||
}
|
||||
}
|
||||
return new DatacenterMetadataPage<>(
|
||||
records, pageNumber, pageSize, hasMore);
|
||||
});
|
||||
} catch (Exception ex) {
|
||||
throw DatacenterConnectorExceptionSupport.wrapAccessException("读取目录失败", ex);
|
||||
}
|
||||
}
|
||||
|
||||
private DatacenterCatalogMeta catalogMeta(
|
||||
DatacenterSource source,
|
||||
String name) {
|
||||
DatacenterCatalogMeta meta = new DatacenterCatalogMeta();
|
||||
meta.setSourceId(source.getId());
|
||||
meta.setCatalogName(name);
|
||||
meta.setCatalogType(usesCatalogNamespace() ? "CATALOG" : "SCHEMA");
|
||||
meta.setLogicalSchemaName(name);
|
||||
if (usesCatalogNamespace()) {
|
||||
meta.setPhysicalCatalogName(name);
|
||||
} else {
|
||||
meta.setPhysicalCatalogName(source.getDatabaseName());
|
||||
meta.setPhysicalSchemaName(name);
|
||||
}
|
||||
return meta;
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<DatacenterTable> listTables(DatacenterSource source, String catalogName) {
|
||||
try {
|
||||
return withConnection(source, true, connection -> {
|
||||
DatabaseMetaData metaData = connection.getMetaData();
|
||||
List<DatacenterTable> tables = new ArrayList<>();
|
||||
try (ResultSet resultSet = metaData.getTables(resolveCatalogArgument(source, catalogName), resolveSchemaArgument(source, catalogName), "%", new String[]{"TABLE", "VIEW"})) {
|
||||
try (ResultSet resultSet = metaData.getTables(
|
||||
resolveCatalogArgument(source, catalogName),
|
||||
metadataPattern(metaData, resolveSchemaArgument(source, catalogName)),
|
||||
"%",
|
||||
new String[]{"TABLE", "VIEW"})) {
|
||||
while (resultSet.next()) {
|
||||
DatacenterTable table = new DatacenterTable();
|
||||
table.setSourceId(source.getId());
|
||||
table.setTableName(resultSet.getString("TABLE_NAME"));
|
||||
table.setTableDesc(resultSet.getString("REMARKS"));
|
||||
table.setActualTable(resultSet.getString("TABLE_NAME"));
|
||||
table.setMaterializedTable(resultSet.getString("TABLE_NAME"));
|
||||
table.setAccessMode(capabilities.contains(DatacenterCapability.WRITE_MUTATION) ? "READ_WRITE" : "READ_ONLY");
|
||||
table.setTableKind(resolveTableKind(resultSet.getString("TABLE_TYPE")).name());
|
||||
table.setCapabilitiesJson(Map.of("capabilities", capabilities.stream().map(Enum::name).toList()));
|
||||
tables.add(table);
|
||||
tables.add(tableFromMetadata(source, resultSet));
|
||||
}
|
||||
}
|
||||
return tables;
|
||||
@@ -176,10 +271,166 @@ public abstract class AbstractJdbcConnector implements DatacenterConnector {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 直接在 JDBC 元数据游标上完成表名筛选和有界分页。
|
||||
*
|
||||
* @param source 数据源
|
||||
* @param catalogName 物理命名空间
|
||||
* @param keyword 表名搜索词
|
||||
* @param pageNumber 页码
|
||||
* @param pageSize 每页大小
|
||||
* @return 有界表列表
|
||||
*/
|
||||
@Override
|
||||
public DatacenterMetadataPage<DatacenterTable> listTablesPage(
|
||||
DatacenterSource source,
|
||||
String catalogName,
|
||||
String keyword,
|
||||
long pageNumber,
|
||||
long pageSize) {
|
||||
try {
|
||||
return withConnection(source, true, connection -> {
|
||||
DatabaseMetaData metaData = connection.getMetaData();
|
||||
String normalizedKeyword = keyword == null
|
||||
? "" : keyword.trim().toLowerCase(Locale.ROOT);
|
||||
long offset = (pageNumber - 1L) * pageSize;
|
||||
long matched = 0L;
|
||||
List<DatacenterTable> tables = new ArrayList<>((int) pageSize);
|
||||
boolean hasMore = false;
|
||||
try (ResultSet resultSet = metaData.getTables(
|
||||
resolveCatalogArgument(source, catalogName),
|
||||
metadataPattern(metaData, resolveSchemaArgument(source, catalogName)),
|
||||
"%",
|
||||
new String[]{"TABLE", "VIEW"})) {
|
||||
while (resultSet.next()) {
|
||||
String tableName = resultSet.getString("TABLE_NAME");
|
||||
if (!normalizedKeyword.isEmpty()
|
||||
&& (tableName == null
|
||||
|| !tableName.toLowerCase(Locale.ROOT).contains(normalizedKeyword))) {
|
||||
continue;
|
||||
}
|
||||
if (matched++ < offset) {
|
||||
continue;
|
||||
}
|
||||
if (tables.size() >= pageSize) {
|
||||
hasMore = true;
|
||||
break;
|
||||
}
|
||||
tables.add(tableFromMetadata(source, resultSet));
|
||||
}
|
||||
}
|
||||
return new DatacenterMetadataPage<>(
|
||||
tables, pageNumber, pageSize, hasMore);
|
||||
});
|
||||
} catch (Exception ex) {
|
||||
throw DatacenterConnectorExceptionSupport.wrapAccessException("读取表列表失败", ex);
|
||||
}
|
||||
}
|
||||
|
||||
private DatacenterTable tableFromMetadata(
|
||||
DatacenterSource source,
|
||||
ResultSet resultSet) throws SQLException {
|
||||
DatacenterTable table = new DatacenterTable();
|
||||
table.setSourceId(source.getId());
|
||||
table.setTableName(resultSet.getString("TABLE_NAME"));
|
||||
table.setTableDesc(resultSet.getString("REMARKS"));
|
||||
table.setActualTable(resultSet.getString("TABLE_NAME"));
|
||||
table.setMaterializedTable(resultSet.getString("TABLE_NAME"));
|
||||
table.setAccessMode(capabilities.contains(DatacenterCapability.WRITE_MUTATION)
|
||||
? "READ_WRITE" : "READ_ONLY");
|
||||
table.setTableKind(resolveTableKind(resultSet.getString("TABLE_TYPE")).name());
|
||||
table.setQueryable(1);
|
||||
table.setMetadataStatus(DatacenterMetadataStatus.ACTIVE.name());
|
||||
table.setLastSeenAt(new java.util.Date());
|
||||
table.setTimeSemantics("NONE");
|
||||
table.setMetadataRevision(1L);
|
||||
table.setCapabilitiesJson(Map.of(
|
||||
"capabilities", capabilities.stream().map(Enum::name).toList()));
|
||||
return table;
|
||||
}
|
||||
|
||||
@Override
|
||||
public DatacenterTableDetailMeta getTableDetail(DatacenterSource source, String catalogName, String tableName) {
|
||||
try {
|
||||
return withConnection(source, true,
|
||||
connection -> readTableDetail(connection, source, catalogName, tableName));
|
||||
} catch (Exception ex) {
|
||||
throw DatacenterConnectorExceptionSupport.wrapAccessException("读取表详情失败", ex);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 使用单个元数据连接批量读取表详情,避免每张表重复创建连接池。
|
||||
*
|
||||
* @param source 数据源
|
||||
* @param catalogName 物理命名空间
|
||||
* @param tableNames 表名集合
|
||||
* @return 表详情集合
|
||||
*/
|
||||
@Override
|
||||
public List<DatacenterTableDetailMeta> getTableDetails(
|
||||
DatacenterSource source,
|
||||
String catalogName,
|
||||
List<String> tableNames) {
|
||||
if (tableNames == null || tableNames.isEmpty()) {
|
||||
return List.of();
|
||||
}
|
||||
try {
|
||||
return withConnection(source, true, connection -> {
|
||||
List<DatacenterTableDetailMeta> details = new ArrayList<>(tableNames.size());
|
||||
for (String tableName : tableNames) {
|
||||
details.add(readTableDetail(connection, source, catalogName, tableName));
|
||||
}
|
||||
return List.copyOf(details);
|
||||
});
|
||||
} catch (Exception ex) {
|
||||
throw DatacenterConnectorExceptionSupport.wrapAccessException("批量读取表详情失败", ex);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 使用单个连接刷新已纳管表,并把物理缺失作为快照状态返回。
|
||||
*
|
||||
* @param source 数据源
|
||||
* @param catalogName 物理命名空间
|
||||
* @param tableNames 已纳管表名集合
|
||||
* @return 已发现详情与缺失表名
|
||||
*/
|
||||
@Override
|
||||
public DatacenterManagedMetadataSnapshot inspectManagedTables(
|
||||
DatacenterSource source,
|
||||
String catalogName,
|
||||
List<String> tableNames) {
|
||||
if (tableNames == null || tableNames.isEmpty()) {
|
||||
return new DatacenterManagedMetadataSnapshot(List.of(), Set.of());
|
||||
}
|
||||
try {
|
||||
return withConnection(source, true, connection -> {
|
||||
List<DatacenterTableDetailMeta> details = new ArrayList<>(tableNames.size());
|
||||
Set<String> missing = new java.util.LinkedHashSet<>();
|
||||
for (String tableName : tableNames) {
|
||||
try {
|
||||
details.add(readTableDetail(connection, source, catalogName, tableName));
|
||||
} catch (BusinessException exception) {
|
||||
if (exception.getMessage() == null
|
||||
|| !exception.getMessage().startsWith("所选数据表已变化")) {
|
||||
throw exception;
|
||||
}
|
||||
missing.add(tableName);
|
||||
}
|
||||
}
|
||||
return new DatacenterManagedMetadataSnapshot(details, missing);
|
||||
});
|
||||
} catch (Exception ex) {
|
||||
throw DatacenterConnectorExceptionSupport.wrapAccessException("刷新已纳管表失败", ex);
|
||||
}
|
||||
}
|
||||
|
||||
private DatacenterTableDetailMeta readTableDetail(
|
||||
Connection connection,
|
||||
DatacenterSource source,
|
||||
String catalogName,
|
||||
String tableName) throws SQLException {
|
||||
DatabaseMetaData metaData = connection.getMetaData();
|
||||
DatacenterTableDetailMeta detail = new DatacenterTableDetailMeta();
|
||||
DatacenterTable table = new DatacenterTable();
|
||||
@@ -189,24 +440,35 @@ public abstract class AbstractJdbcConnector implements DatacenterConnector {
|
||||
table.setMaterializedTable(tableName);
|
||||
table.setAccessMode(capabilities.contains(DatacenterCapability.WRITE_MUTATION) ? "READ_WRITE" : "READ_ONLY");
|
||||
table.setTableKind(DatacenterTableKind.EXTERNAL_TABLE.name());
|
||||
table.setQueryable(1);
|
||||
table.setMetadataStatus(DatacenterMetadataStatus.ACTIVE.name());
|
||||
table.setLastSeenAt(new java.util.Date());
|
||||
table.setTimeSemantics("NONE");
|
||||
table.setMetadataRevision(1L);
|
||||
table.setCapabilitiesJson(Map.of("capabilities", capabilities.stream().map(Enum::name).toList()));
|
||||
detail.setTable(table);
|
||||
|
||||
boolean tableFound = false;
|
||||
try (ResultSet tableSet = metaData.getTables(
|
||||
resolveCatalogArgument(source, catalogName),
|
||||
resolveSchemaArgument(source, catalogName),
|
||||
tableName,
|
||||
metadataPattern(metaData, resolveSchemaArgument(source, catalogName)),
|
||||
metadataPattern(metaData, tableName),
|
||||
new String[]{"TABLE", "VIEW"})) {
|
||||
while (tableSet.next()) {
|
||||
String currentTableName = tableSet.getString("TABLE_NAME");
|
||||
if (!matchesTableName(currentTableName, tableName)) {
|
||||
if (!matchesMetadataTable(
|
||||
tableSet, source, catalogName, tableName)) {
|
||||
continue;
|
||||
}
|
||||
table.setTableDesc(tableSet.getString("REMARKS"));
|
||||
table.setTableKind(resolveTableKind(tableSet.getString("TABLE_TYPE")).name());
|
||||
tableFound = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (!tableFound) {
|
||||
throw new BusinessException(
|
||||
"所选数据表已变化,请刷新后重试: " + tableName);
|
||||
}
|
||||
|
||||
Set<String> primaryKeys = new HashSet<>();
|
||||
try (ResultSet pkSet = metaData.getPrimaryKeys(resolveCatalogArgument(source, catalogName), resolveSchemaArgument(source, catalogName), tableName)) {
|
||||
@@ -216,13 +478,24 @@ public abstract class AbstractJdbcConnector implements DatacenterConnector {
|
||||
}
|
||||
|
||||
List<DatacenterTableField> fields = new ArrayList<>();
|
||||
try (ResultSet columns = metaData.getColumns(resolveCatalogArgument(source, catalogName), resolveSchemaArgument(source, catalogName), tableName, "%")) {
|
||||
try (ResultSet columns = metaData.getColumns(
|
||||
resolveCatalogArgument(source, catalogName),
|
||||
metadataPattern(metaData, resolveSchemaArgument(source, catalogName)),
|
||||
metadataPattern(metaData, tableName),
|
||||
"%")) {
|
||||
while (columns.next()) {
|
||||
if (!matchesMetadataTable(
|
||||
columns, source, catalogName, tableName)) {
|
||||
continue;
|
||||
}
|
||||
DatacenterTableField field = new DatacenterTableField();
|
||||
field.setFieldName(columns.getString("COLUMN_NAME"));
|
||||
field.setSourceColumnName(columns.getString("COLUMN_NAME"));
|
||||
field.setFieldDesc(columns.getString("REMARKS"));
|
||||
field.setJdbcType(columns.getString("TYPE_NAME"));
|
||||
field.setJdbcTypeCode(columns.getInt("DATA_TYPE"));
|
||||
field.setNativeTypeName(columns.getString("TYPE_NAME"));
|
||||
field.setOrdinalPosition(columns.getInt("ORDINAL_POSITION"));
|
||||
field.setPrecision(columns.getInt("COLUMN_SIZE"));
|
||||
field.setScale(columns.getInt("DECIMAL_DIGITS"));
|
||||
field.setRequired(columns.getInt("NULLABLE") == DatabaseMetaData.columnNoNulls ? 1 : 0);
|
||||
@@ -231,15 +504,21 @@ public abstract class AbstractJdbcConnector implements DatacenterConnector {
|
||||
field.setWritable(capabilities.contains(DatacenterCapability.WRITE_MUTATION) ? 1 : 0);
|
||||
field.setIndexed(primaryKeys.contains(field.getFieldName()) ? 1 : 0);
|
||||
field.setFieldType(mapFieldType(columns.getInt("DATA_TYPE")));
|
||||
field.setMetadataStatus(DatacenterMetadataStatus.ACTIVE.name());
|
||||
field.setLastSeenAt(new java.util.Date());
|
||||
field.setSensitivityLevel(DatacenterSensitivityLevel.PUBLIC.name());
|
||||
field.setMetadataFingerprint(DatacenterMetadataIdentity.fieldFingerprint(field));
|
||||
fields.add(field);
|
||||
}
|
||||
}
|
||||
if (fields.isEmpty()) {
|
||||
throw new BusinessException(
|
||||
"无法读取数据表字段,请检查权限: " + tableName);
|
||||
}
|
||||
table.setMetadataFingerprint(DatacenterMetadataIdentity.tableFingerprint(
|
||||
table.getTableName(), table.getTableKind(), fields));
|
||||
detail.setFields(fields);
|
||||
return detail;
|
||||
});
|
||||
} catch (Exception ex) {
|
||||
throw DatacenterConnectorExceptionSupport.wrapAccessException("读取表详情失败", ex);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
@@ -247,6 +526,11 @@ public abstract class AbstractJdbcConnector implements DatacenterConnector {
|
||||
if (!capabilities.contains(DatacenterCapability.READ_QUERY)) {
|
||||
throw new BusinessException("当前数据源暂不支持查询");
|
||||
}
|
||||
if (request == null || request.getPageSize() == null
|
||||
|| request.getPageSize() < 1L
|
||||
|| request.getPageSize() > 500L) {
|
||||
throw new BusinessException("pageSize 必须在 1 到 500 之间");
|
||||
}
|
||||
try {
|
||||
return withConnection(source, true, connection -> doQueryPage(connection, source, table, request));
|
||||
} catch (Exception ex) {
|
||||
@@ -325,7 +609,7 @@ public abstract class AbstractJdbcConnector implements DatacenterConnector {
|
||||
String qualifiedTable = sqlDialect.qualifyTable(resolveCatalogName(source, request.getDatasetRef() == null ? null : request.getDatasetRef().getCatalogName()), resolvePhysicalTableName(table));
|
||||
StringBuilder whereClause = new StringBuilder();
|
||||
if (StrUtil.isNotBlank(request.getWhere())) {
|
||||
whereClause.append(" WHERE ").append(request.getWhere());
|
||||
throw new BusinessException("不支持原始 WHERE 条件,请使用结构化筛选条件");
|
||||
} else if (!CollectionUtils.isEmpty(request.getFilters())) {
|
||||
whereClause.append(" WHERE 1=1 ");
|
||||
for (DatacenterQueryFilter filter : request.getFilters()) {
|
||||
@@ -622,13 +906,9 @@ public abstract class AbstractJdbcConnector implements DatacenterConnector {
|
||||
return items;
|
||||
}
|
||||
List<DatacenterCatalogMeta> matched = items.stream()
|
||||
.filter(item -> configuredName.equalsIgnoreCase(item.getCatalogName()))
|
||||
.filter(item -> configuredName.equals(item.getCatalogName()))
|
||||
.collect(Collectors.toList());
|
||||
return matched.isEmpty() ? items : matched;
|
||||
}
|
||||
|
||||
private boolean containsCatalog(List<DatacenterCatalogMeta> items, String catalogName) {
|
||||
return items.stream().anyMatch(item -> catalogName.equalsIgnoreCase(item.getCatalogName()));
|
||||
return matched;
|
||||
}
|
||||
|
||||
private DatacenterTableKind resolveTableKind(String tableType) {
|
||||
@@ -639,8 +919,58 @@ public abstract class AbstractJdbcConnector implements DatacenterConnector {
|
||||
if (currentTableName == null || targetTableName == null) {
|
||||
return false;
|
||||
}
|
||||
return currentTableName.equals(targetTableName)
|
||||
|| currentTableName.equalsIgnoreCase(targetTableName);
|
||||
return currentTableName.equals(targetTableName);
|
||||
}
|
||||
|
||||
/**
|
||||
* 将精确标识符转换为 JDBC 元数据 Pattern,转义驱动声明的通配符字符。
|
||||
*
|
||||
* @param metaData JDBC 元数据
|
||||
* @param value 精确 Schema 或表名
|
||||
* @return 可安全用于元数据 Pattern 参数的文本
|
||||
* @throws SQLException 驱动无法返回转义字符时抛出
|
||||
*/
|
||||
private String metadataPattern(DatabaseMetaData metaData, String value)
|
||||
throws SQLException {
|
||||
if (value == null) {
|
||||
return null;
|
||||
}
|
||||
String escape = metaData.getSearchStringEscape();
|
||||
if (escape == null || escape.isEmpty()) {
|
||||
return value;
|
||||
}
|
||||
return value.replace(escape, escape + escape)
|
||||
.replace("%", escape + "%")
|
||||
.replace("_", escape + "_");
|
||||
}
|
||||
|
||||
/**
|
||||
* 对元数据结果再次执行精确 Catalog、Schema 与表名校验,防御驱动忽略 Pattern 转义。
|
||||
*
|
||||
* @param resultSet 元数据结果集
|
||||
* @param source 数据源
|
||||
* @param catalogName 用户选择的物理命名空间
|
||||
* @param tableName 目标表名
|
||||
* @return 当前结果行是否属于精确目标
|
||||
* @throws SQLException 读取元数据列失败时抛出
|
||||
*/
|
||||
private boolean matchesMetadataTable(
|
||||
ResultSet resultSet,
|
||||
DatacenterSource source,
|
||||
String catalogName,
|
||||
String tableName) throws SQLException {
|
||||
if (!matchesTableName(resultSet.getString("TABLE_NAME"), tableName)) {
|
||||
return false;
|
||||
}
|
||||
String expectedCatalog = resolveCatalogArgument(source, catalogName);
|
||||
String expectedSchema = resolveSchemaArgument(source, catalogName);
|
||||
String actualCatalog = resultSet.getString("TABLE_CAT");
|
||||
String actualSchema = resultSet.getString("TABLE_SCHEM");
|
||||
// Schema 型数据库已由当前连接锁定 Database;部分 PostgreSQL 驱动不会回填 TABLE_CAT。
|
||||
return (!usesCatalogNamespace()
|
||||
|| expectedCatalog == null
|
||||
|| expectedCatalog.equals(actualCatalog))
|
||||
&& (expectedSchema == null || expectedSchema.equals(actualSchema));
|
||||
}
|
||||
|
||||
private Integer mapFieldType(int jdbcType) {
|
||||
|
||||
@@ -65,6 +65,38 @@ public class DatacenterTableBase extends DateEntity implements Serializable {
|
||||
@Column(comment = "物理表名")
|
||||
private String actualTable;
|
||||
|
||||
/** 物理对象稳定摘要。 */
|
||||
@Column(comment = "物理对象稳定摘要")
|
||||
private String physicalIdentityKey;
|
||||
|
||||
/** 表结构摘要。 */
|
||||
@Column(comment = "表结构摘要")
|
||||
private String metadataFingerprint;
|
||||
|
||||
/** 元数据状态。 */
|
||||
@Column(comment = "元数据状态")
|
||||
private String metadataStatus;
|
||||
|
||||
/** 最近发现时间。 */
|
||||
@Column(comment = "最近发现时间")
|
||||
private Date lastSeenAt;
|
||||
|
||||
/** 是否允许统一只读查询。 */
|
||||
@Column(comment = "是否允许统一只读查询")
|
||||
private Integer queryable;
|
||||
|
||||
/** 时间语义。 */
|
||||
@Column(comment = "时间语义")
|
||||
private String timeSemantics;
|
||||
|
||||
/** 默认时间字段 ID。 */
|
||||
@Column(comment = "默认时间字段ID")
|
||||
private BigInteger defaultTimeFieldId;
|
||||
|
||||
/** 表元数据版本。 */
|
||||
@Column(comment = "表元数据版本")
|
||||
private Long metadataRevision;
|
||||
|
||||
/**
|
||||
* 表类型
|
||||
*/
|
||||
@@ -198,6 +230,39 @@ public class DatacenterTableBase extends DateEntity implements Serializable {
|
||||
this.actualTable = actualTable;
|
||||
}
|
||||
|
||||
public String getPhysicalIdentityKey() { return physicalIdentityKey; }
|
||||
|
||||
public void setPhysicalIdentityKey(String physicalIdentityKey) { this.physicalIdentityKey = physicalIdentityKey; }
|
||||
|
||||
public String getMetadataFingerprint() { return metadataFingerprint; }
|
||||
|
||||
public void setMetadataFingerprint(String metadataFingerprint) { this.metadataFingerprint = metadataFingerprint; }
|
||||
|
||||
public String getMetadataStatus() { return metadataStatus; }
|
||||
|
||||
public void setMetadataStatus(String metadataStatus) { this.metadataStatus = metadataStatus; }
|
||||
|
||||
public Date getLastSeenAt() { return lastSeenAt; }
|
||||
|
||||
public void setLastSeenAt(Date lastSeenAt) { this.lastSeenAt = lastSeenAt; }
|
||||
|
||||
public Integer getQueryable() { return queryable; }
|
||||
|
||||
public void setQueryable(Integer queryable) { this.queryable = queryable; }
|
||||
|
||||
public String getTimeSemantics() { return timeSemantics; }
|
||||
|
||||
public void setTimeSemantics(String timeSemantics) { this.timeSemantics = timeSemantics; }
|
||||
|
||||
@JsonSerialize(using = ToStringSerializer.class)
|
||||
public BigInteger getDefaultTimeFieldId() { return defaultTimeFieldId; }
|
||||
|
||||
public void setDefaultTimeFieldId(BigInteger defaultTimeFieldId) { this.defaultTimeFieldId = defaultTimeFieldId; }
|
||||
|
||||
public Long getMetadataRevision() { return metadataRevision; }
|
||||
|
||||
public void setMetadataRevision(Long metadataRevision) { this.metadataRevision = metadataRevision; }
|
||||
|
||||
public String getTableKind() {
|
||||
return tableKind;
|
||||
}
|
||||
|
||||
@@ -41,6 +41,10 @@ public class DatacenterTableFieldBase extends DateEntity implements Serializable
|
||||
@Column(comment = "源字段名")
|
||||
private String sourceColumnName;
|
||||
|
||||
/** JDBC 字段顺序。 */
|
||||
@Column(comment = "JDBC字段顺序")
|
||||
private Integer ordinalPosition;
|
||||
|
||||
/**
|
||||
* 字段描述
|
||||
*/
|
||||
@@ -59,6 +63,14 @@ public class DatacenterTableFieldBase extends DateEntity implements Serializable
|
||||
@Column(comment = "JDBC类型")
|
||||
private String jdbcType;
|
||||
|
||||
/** java.sql.Types 数值。 */
|
||||
@Column(comment = "JDBC类型编码")
|
||||
private Integer jdbcTypeCode;
|
||||
|
||||
/** 数据库原生类型名。 */
|
||||
@Column(comment = "数据库原生类型名")
|
||||
private String nativeTypeName;
|
||||
|
||||
/**
|
||||
* 精度
|
||||
*/
|
||||
@@ -71,6 +83,26 @@ public class DatacenterTableFieldBase extends DateEntity implements Serializable
|
||||
@Column(comment = "小数位")
|
||||
private Integer scale;
|
||||
|
||||
/** 字段元数据摘要。 */
|
||||
@Column(comment = "字段元数据摘要")
|
||||
private String metadataFingerprint;
|
||||
|
||||
/** 元数据状态。 */
|
||||
@Column(comment = "元数据状态")
|
||||
private String metadataStatus;
|
||||
|
||||
/** 最近发现时间。 */
|
||||
@Column(comment = "最近发现时间")
|
||||
private Date lastSeenAt;
|
||||
|
||||
/** 敏感级别。 */
|
||||
@Column(comment = "敏感级别")
|
||||
private String sensitivityLevel;
|
||||
|
||||
/** 脱敏策略。 */
|
||||
@Column(comment = "脱敏策略")
|
||||
private String maskingStrategy;
|
||||
|
||||
/**
|
||||
* 是否必填
|
||||
*/
|
||||
@@ -165,6 +197,10 @@ public class DatacenterTableFieldBase extends DateEntity implements Serializable
|
||||
this.sourceColumnName = sourceColumnName;
|
||||
}
|
||||
|
||||
public Integer getOrdinalPosition() { return ordinalPosition; }
|
||||
|
||||
public void setOrdinalPosition(Integer ordinalPosition) { this.ordinalPosition = ordinalPosition; }
|
||||
|
||||
public String getFieldDesc() {
|
||||
return fieldDesc;
|
||||
}
|
||||
@@ -189,6 +225,14 @@ public class DatacenterTableFieldBase extends DateEntity implements Serializable
|
||||
this.jdbcType = jdbcType;
|
||||
}
|
||||
|
||||
public Integer getJdbcTypeCode() { return jdbcTypeCode; }
|
||||
|
||||
public void setJdbcTypeCode(Integer jdbcTypeCode) { this.jdbcTypeCode = jdbcTypeCode; }
|
||||
|
||||
public String getNativeTypeName() { return nativeTypeName; }
|
||||
|
||||
public void setNativeTypeName(String nativeTypeName) { this.nativeTypeName = nativeTypeName; }
|
||||
|
||||
public Integer getPrecision() {
|
||||
return precision;
|
||||
}
|
||||
@@ -205,6 +249,26 @@ public class DatacenterTableFieldBase extends DateEntity implements Serializable
|
||||
this.scale = scale;
|
||||
}
|
||||
|
||||
public String getMetadataFingerprint() { return metadataFingerprint; }
|
||||
|
||||
public void setMetadataFingerprint(String metadataFingerprint) { this.metadataFingerprint = metadataFingerprint; }
|
||||
|
||||
public String getMetadataStatus() { return metadataStatus; }
|
||||
|
||||
public void setMetadataStatus(String metadataStatus) { this.metadataStatus = metadataStatus; }
|
||||
|
||||
public Date getLastSeenAt() { return lastSeenAt; }
|
||||
|
||||
public void setLastSeenAt(Date lastSeenAt) { this.lastSeenAt = lastSeenAt; }
|
||||
|
||||
public String getSensitivityLevel() { return sensitivityLevel; }
|
||||
|
||||
public void setSensitivityLevel(String sensitivityLevel) { this.sensitivityLevel = sensitivityLevel; }
|
||||
|
||||
public String getMaskingStrategy() { return maskingStrategy; }
|
||||
|
||||
public void setMaskingStrategy(String maskingStrategy) { this.maskingStrategy = maskingStrategy; }
|
||||
|
||||
public Integer getRequired() {
|
||||
return required;
|
||||
}
|
||||
|
||||
@@ -346,7 +346,7 @@ public class DatacenterExcelImportServiceImpl implements DatacenterExcelImportSe
|
||||
queryRequest.setDatasetRef(registryService.resolveDatasetRef(table.getId()));
|
||||
queryRequest.setSelectedColumns(table.getFields().stream().map(DatacenterTableField::getFieldName).toList());
|
||||
final int[] rowIndex = {1};
|
||||
totalRows += iterateRows(queryRequest, row -> {
|
||||
totalRows += iterateRows(queryRequest, account, row -> {
|
||||
org.apache.poi.ss.usermodel.Row excelRow = sheet.createRow(rowIndex[0]++);
|
||||
for (int i = 0; i < table.getFields().size(); i++) {
|
||||
Cell cell = excelRow.createCell(i);
|
||||
@@ -434,7 +434,7 @@ public class DatacenterExcelImportServiceImpl implements DatacenterExcelImportSe
|
||||
String baseName = resolveSplitPrefix(request, sourceTable.getTableName());
|
||||
List<BigInteger> derivedIds = new ArrayList<>();
|
||||
final Holder holder = new Holder();
|
||||
long totalRows = iterateRows(buildFullQuery(sourceTable), row -> {
|
||||
long totalRows = iterateRows(buildFullQuery(sourceTable), account, row -> {
|
||||
if (holder.targetTable == null || holder.currentSize >= rowBatchSize) {
|
||||
holder.batchNo++;
|
||||
holder.targetTable = createDerivedTable(source, catalog, cloneFields(sourceTable.getFields()),
|
||||
@@ -466,7 +466,7 @@ public class DatacenterExcelImportServiceImpl implements DatacenterExcelImportSe
|
||||
String prefix = resolveSplitPrefix(request, sourceTable.getTableName());
|
||||
Map<String, DatacenterTable> targets = new LinkedHashMap<>();
|
||||
List<BigInteger> derivedIds = new ArrayList<>();
|
||||
long totalRows = iterateRows(buildFullQuery(sourceTable), row -> {
|
||||
long totalRows = iterateRows(buildFullQuery(sourceTable), account, row -> {
|
||||
String fieldValue = stringify(row.get(splitField.getFieldName()));
|
||||
String bucket = fieldValue == null || fieldValue.isBlank() ? "empty" : fieldValue;
|
||||
DatacenterTable targetTable = targets.get(bucket);
|
||||
@@ -543,7 +543,7 @@ public class DatacenterExcelImportServiceImpl implements DatacenterExcelImportSe
|
||||
Map<String, JSONObject> mergedRows = new LinkedHashMap<>();
|
||||
for (DatacenterTable table : tables) {
|
||||
Map<String, String> mapping = fieldMappings.get(table.getId());
|
||||
iterateRows(buildFullQuery(table), row -> {
|
||||
iterateRows(buildFullQuery(table), account, row -> {
|
||||
String joinValue = stringify(row.get(request.getJoinKey()));
|
||||
if (joinValue == null || joinValue.isBlank()) {
|
||||
return;
|
||||
@@ -729,16 +729,22 @@ public class DatacenterExcelImportServiceImpl implements DatacenterExcelImportSe
|
||||
}
|
||||
|
||||
private long copyRows(DatacenterQueryRequest queryRequest, RowMapper mapper, DatacenterTable targetTable, LoginAccount account) {
|
||||
return iterateRows(queryRequest, row -> saveToTable(targetTable, mapper.map(row), account));
|
||||
return iterateRows(
|
||||
queryRequest,
|
||||
account,
|
||||
row -> saveToTable(targetTable, mapper.map(row), account));
|
||||
}
|
||||
|
||||
private long iterateRows(DatacenterQueryRequest queryRequest, RowConsumer consumer) {
|
||||
private long iterateRows(
|
||||
DatacenterQueryRequest queryRequest,
|
||||
LoginAccount account,
|
||||
RowConsumer consumer) {
|
||||
long total = 0L;
|
||||
long pageNumber = 1L;
|
||||
while (true) {
|
||||
queryRequest.setPageNumber(pageNumber);
|
||||
queryRequest.setPageSize(QUERY_BATCH_SIZE);
|
||||
Page<Row> page = queryService.queryPage(queryRequest);
|
||||
Page<Row> page = queryService.queryPage(queryRequest, account);
|
||||
if (page.getRecords() == null || page.getRecords().isEmpty()) {
|
||||
break;
|
||||
}
|
||||
|
||||
@@ -5,31 +5,46 @@ import tech.easyflow.datacenter.entity.DatacenterTableField;
|
||||
import tech.easyflow.datacenter.meta.entity.DatacenterCatalog;
|
||||
import tech.easyflow.datacenter.meta.entity.DatacenterDatasetVersion;
|
||||
import tech.easyflow.datacenter.meta.entity.DatacenterDerivedTable;
|
||||
import tech.easyflow.datacenter.meta.entity.DatacenterSource;
|
||||
import tech.easyflow.datacenter.meta.model.DatacenterSourceView;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
public class DatacenterSchemaResponse {
|
||||
private DatasetRef datasetRef;
|
||||
private DatacenterSource source;
|
||||
private DatacenterSourceView source;
|
||||
private DatacenterCatalog catalog;
|
||||
private DatacenterTable table;
|
||||
private List<DatacenterTableField> fields = new ArrayList<>();
|
||||
private long fieldPageNumber = 1L;
|
||||
private long fieldPageSize;
|
||||
private boolean hasMoreFields;
|
||||
private List<DatacenterDatasetVersion> versions = new ArrayList<>();
|
||||
private List<DatacenterDerivedTable> upstreamLineage = new ArrayList<>();
|
||||
private List<DatacenterDerivedTable> downstreamLineage = new ArrayList<>();
|
||||
|
||||
public DatasetRef getDatasetRef() { return datasetRef; }
|
||||
public void setDatasetRef(DatasetRef datasetRef) { this.datasetRef = datasetRef; }
|
||||
public DatacenterSource getSource() { return source; }
|
||||
public void setSource(DatacenterSource source) { this.source = source; }
|
||||
public DatacenterSourceView getSource() { return source; }
|
||||
public void setSource(DatacenterSourceView source) { this.source = source; }
|
||||
public DatacenterCatalog getCatalog() { return catalog; }
|
||||
public void setCatalog(DatacenterCatalog catalog) { this.catalog = catalog; }
|
||||
public DatacenterTable getTable() { return table; }
|
||||
public void setTable(DatacenterTable table) { this.table = table; }
|
||||
public List<DatacenterTableField> getFields() { return fields; }
|
||||
public void setFields(List<DatacenterTableField> fields) { this.fields = fields; }
|
||||
/** @return 当前字段页码 */
|
||||
public long getFieldPageNumber() { return fieldPageNumber; }
|
||||
/** @param fieldPageNumber 当前字段页码 */
|
||||
public void setFieldPageNumber(long fieldPageNumber) { this.fieldPageNumber = fieldPageNumber; }
|
||||
/** @return 当前字段页大小 */
|
||||
public long getFieldPageSize() { return fieldPageSize; }
|
||||
/** @param fieldPageSize 当前字段页大小 */
|
||||
public void setFieldPageSize(long fieldPageSize) { this.fieldPageSize = fieldPageSize; }
|
||||
/** @return 是否还有下一页字段 */
|
||||
public boolean isHasMoreFields() { return hasMoreFields; }
|
||||
/** @param hasMoreFields 是否还有下一页字段 */
|
||||
public void setHasMoreFields(boolean hasMoreFields) { this.hasMoreFields = hasMoreFields; }
|
||||
public List<DatacenterDatasetVersion> getVersions() { return versions; }
|
||||
public void setVersions(List<DatacenterDatasetVersion> versions) { this.versions = versions; }
|
||||
public List<DatacenterDerivedTable> getUpstreamLineage() { return upstreamLineage; }
|
||||
|
||||
@@ -0,0 +1,9 @@
|
||||
package tech.easyflow.datacenter.execution.model;
|
||||
|
||||
/**
|
||||
* 管理端取消只读 SQL 查询请求。
|
||||
*
|
||||
* @param queryId 客户端在执行前生成的查询 UUID
|
||||
*/
|
||||
public record DatacenterSqlCancelRequest(String queryId) {
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
package tech.easyflow.datacenter.execution.model;
|
||||
|
||||
/**
|
||||
* SQL 控制台结果列。
|
||||
*
|
||||
* @param key 前端稳定列键
|
||||
* @param label JDBC 列标签
|
||||
* @param jdbcType JDBC 类型编码
|
||||
* @param typeName 数据库类型名
|
||||
* @param nullable 是否允许空值
|
||||
*/
|
||||
public record DatacenterSqlColumnView(
|
||||
String key,
|
||||
String label,
|
||||
int jdbcType,
|
||||
String typeName,
|
||||
boolean nullable) {
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
package tech.easyflow.datacenter.execution.model;
|
||||
|
||||
import java.math.BigInteger;
|
||||
|
||||
/**
|
||||
* 管理端只读 SQL 控制台请求。
|
||||
*
|
||||
* @param sourceId 已激活数据源 ID
|
||||
* @param queryId 客户端在执行前生成的查询 UUID
|
||||
* @param sql 单条只读 SQL
|
||||
* @param maxRows 最大返回行数
|
||||
*/
|
||||
public record DatacenterSqlConsoleRequest(
|
||||
BigInteger sourceId,
|
||||
String queryId,
|
||||
String sql,
|
||||
Integer maxRows) {
|
||||
}
|
||||
@@ -0,0 +1,31 @@
|
||||
package tech.easyflow.datacenter.execution.model;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* 有界 SQL 控制台结果。
|
||||
*
|
||||
* @param queryId 查询标识
|
||||
* @param columns 结果列
|
||||
* @param rows 结果行
|
||||
* @param returnedRows 返回行数
|
||||
* @param truncated 是否因上限截断
|
||||
* @param durationMs 总耗时毫秒
|
||||
*/
|
||||
public record DatacenterSqlConsoleResult(
|
||||
String queryId,
|
||||
List<DatacenterSqlColumnView> columns,
|
||||
List<Map<String, Object>> rows,
|
||||
long returnedRows,
|
||||
boolean truncated,
|
||||
long durationMs) {
|
||||
|
||||
/**
|
||||
* 防御性复制结果集合。
|
||||
*/
|
||||
public DatacenterSqlConsoleResult {
|
||||
columns = List.copyOf(columns == null ? List.of() : columns);
|
||||
rows = List.copyOf(rows == null ? List.of() : rows);
|
||||
}
|
||||
}
|
||||
@@ -5,6 +5,7 @@ import java.math.BigInteger;
|
||||
public class DatasetRef implements java.io.Serializable {
|
||||
private static final long serialVersionUID = 1L;
|
||||
|
||||
private BigInteger tenantId;
|
||||
private BigInteger sourceId;
|
||||
private BigInteger catalogId;
|
||||
private String catalogName;
|
||||
@@ -12,6 +13,8 @@ public class DatasetRef implements java.io.Serializable {
|
||||
private String tableName;
|
||||
private BigInteger versionId;
|
||||
|
||||
public BigInteger getTenantId() { return tenantId; }
|
||||
public void setTenantId(BigInteger tenantId) { this.tenantId = tenantId; }
|
||||
public BigInteger getSourceId() { return sourceId; }
|
||||
public void setSourceId(BigInteger sourceId) { this.sourceId = sourceId; }
|
||||
public BigInteger getCatalogId() { return catalogId; }
|
||||
|
||||
@@ -2,6 +2,7 @@ package tech.easyflow.datacenter.execution.service;
|
||||
|
||||
import com.mybatisflex.core.paginate.Page;
|
||||
import com.mybatisflex.core.row.Row;
|
||||
import tech.easyflow.common.entity.LoginAccount;
|
||||
import tech.easyflow.datacenter.execution.model.DatacenterQueryRequest;
|
||||
import tech.easyflow.datacenter.execution.model.DatacenterSchemaResponse;
|
||||
import tech.easyflow.datacenter.execution.model.DatacenterSqlQueryRequest;
|
||||
@@ -20,6 +21,19 @@ public interface DatacenterDatasetQueryService {
|
||||
*/
|
||||
Page<Row> queryPage(DatacenterQueryRequest request);
|
||||
|
||||
/**
|
||||
* 使用明确执行账号分页查询结构化数据集。
|
||||
*
|
||||
* @param request 查询请求
|
||||
* @param account 执行账号
|
||||
* @return 分页结果
|
||||
*/
|
||||
default Page<Row> queryPage(
|
||||
DatacenterQueryRequest request,
|
||||
LoginAccount account) {
|
||||
return queryPage(request);
|
||||
}
|
||||
|
||||
/**
|
||||
* 执行原生 SQL 并返回完整结果。
|
||||
*
|
||||
@@ -35,9 +49,43 @@ public interface DatacenterDatasetQueryService {
|
||||
* @param fetchSize JDBC 建议拉取行数
|
||||
* @param consumer 单行消费者
|
||||
*/
|
||||
default void consumeBySql(
|
||||
DatacenterSqlQueryRequest request,
|
||||
int fetchSize,
|
||||
Consumer<Row> consumer) {
|
||||
consumeBySql(request, fetchSize, null, consumer);
|
||||
}
|
||||
|
||||
/**
|
||||
* 使用明确执行账号流式消费 SQL,后台工作流据此绑定权威租户。
|
||||
*
|
||||
* @param request SQL 查询请求
|
||||
* @param fetchSize JDBC 建议拉取行数
|
||||
* @param account 执行账号
|
||||
* @param consumer 单行消费者
|
||||
*/
|
||||
default void consumeBySql(
|
||||
DatacenterSqlQueryRequest request,
|
||||
int fetchSize,
|
||||
LoginAccount account,
|
||||
Consumer<Row> consumer) {
|
||||
consumeBySql(request, fetchSize, account, null, consumer);
|
||||
}
|
||||
|
||||
/**
|
||||
* 使用明确执行账号和调用方 QueryId 流式消费 SQL。
|
||||
*
|
||||
* @param request SQL 查询请求
|
||||
* @param fetchSize JDBC 建议拉取行数
|
||||
* @param account 执行账号
|
||||
* @param requestedQueryId 调用方预生成的查询 UUID;为空时服务端生成
|
||||
* @param consumer 单行消费者
|
||||
*/
|
||||
void consumeBySql(
|
||||
DatacenterSqlQueryRequest request,
|
||||
int fetchSize,
|
||||
LoginAccount account,
|
||||
String requestedQueryId,
|
||||
Consumer<Row> consumer);
|
||||
|
||||
/**
|
||||
@@ -48,6 +96,19 @@ public interface DatacenterDatasetQueryService {
|
||||
*/
|
||||
DatacenterSchemaResponse getSchema(DatasetRef datasetRef);
|
||||
|
||||
/**
|
||||
* 获取有界字段页的数据集结构。
|
||||
*
|
||||
* @param datasetRef 数据集引用
|
||||
* @param fieldPageNumber 字段页码
|
||||
* @param fieldPageSize 字段页大小
|
||||
* @return 数据集结构和当前字段页
|
||||
*/
|
||||
DatacenterSchemaResponse getSchema(
|
||||
DatasetRef datasetRef,
|
||||
Long fieldPageNumber,
|
||||
Long fieldPageSize);
|
||||
|
||||
/**
|
||||
* 仅解析数据集定位信息,不加载版本和血缘。
|
||||
*
|
||||
|
||||
@@ -1,22 +1,43 @@
|
||||
package tech.easyflow.datacenter.execution.service.impl;
|
||||
|
||||
import com.easyagents.federation.sql.execute.SqlParameter;
|
||||
import com.easyagents.federation.sql.execute.QueryId;
|
||||
import com.mybatisflex.core.paginate.Page;
|
||||
import com.mybatisflex.core.query.QueryWrapper;
|
||||
import com.mybatisflex.core.row.Row;
|
||||
import java.math.BigInteger;
|
||||
import java.sql.Types;
|
||||
import java.util.ArrayList;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.LinkedHashSet;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.UUID;
|
||||
import java.util.function.Consumer;
|
||||
import java.util.function.Function;
|
||||
import java.util.stream.Collectors;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.springframework.util.CollectionUtils;
|
||||
import org.springframework.util.StringUtils;
|
||||
import tech.easyflow.common.web.exceptions.BusinessException;
|
||||
import tech.easyflow.common.entity.LoginAccount;
|
||||
import tech.easyflow.datacenter.connector.DatacenterConnector;
|
||||
import tech.easyflow.datacenter.connector.DatacenterConnectorRegistry;
|
||||
import tech.easyflow.datacenter.connector.support.AbstractInternalTableConnector;
|
||||
import tech.easyflow.datacenter.audit.DatacenterQueryAudit;
|
||||
import tech.easyflow.datacenter.audit.DatacenterQueryAuditService;
|
||||
import tech.easyflow.datacenter.entity.DatacenterTable;
|
||||
import tech.easyflow.datacenter.entity.DatacenterTableField;
|
||||
import tech.easyflow.datacenter.execution.model.DatacenterQueryFilter;
|
||||
import tech.easyflow.datacenter.execution.model.DatacenterQueryRequest;
|
||||
import tech.easyflow.datacenter.execution.model.DatacenterQuerySort;
|
||||
import tech.easyflow.datacenter.execution.model.DatacenterSchemaResponse;
|
||||
import tech.easyflow.datacenter.execution.model.DatacenterSqlConsoleResult;
|
||||
import tech.easyflow.datacenter.execution.model.DatacenterSqlQueryRequest;
|
||||
import tech.easyflow.datacenter.execution.model.DatasetRef;
|
||||
import tech.easyflow.datacenter.execution.service.DatacenterDatasetQueryService;
|
||||
import tech.easyflow.datacenter.federation.DatacenterFederationQueryService;
|
||||
import tech.easyflow.datacenter.mapper.DatacenterCatalogMapper;
|
||||
import tech.easyflow.datacenter.mapper.DatacenterDatasetVersionMapper;
|
||||
import tech.easyflow.datacenter.mapper.DatacenterDerivedTableMapper;
|
||||
@@ -25,19 +46,14 @@ import tech.easyflow.datacenter.meta.entity.DatacenterCatalog;
|
||||
import tech.easyflow.datacenter.meta.entity.DatacenterDatasetVersion;
|
||||
import tech.easyflow.datacenter.meta.entity.DatacenterDerivedTable;
|
||||
import tech.easyflow.datacenter.meta.entity.DatacenterSource;
|
||||
import tech.easyflow.datacenter.meta.model.DatacenterSourceViews;
|
||||
import tech.easyflow.datacenter.meta.enums.DatacenterMetadataStatus;
|
||||
import tech.easyflow.datacenter.meta.enums.DatacenterSourceType;
|
||||
import tech.easyflow.datacenter.meta.enums.DatacenterSensitivityLevel;
|
||||
import tech.easyflow.datacenter.meta.service.DatacenterDatasetRegistryService;
|
||||
import tech.easyflow.datacenter.utils.SqlSupportUtils;
|
||||
|
||||
import javax.annotation.Resource;
|
||||
import java.math.BigInteger;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.LinkedHashSet;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.function.Function;
|
||||
import java.util.function.Consumer;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
@Service
|
||||
public class DatacenterDatasetQueryServiceImpl implements DatacenterDatasetQueryService {
|
||||
@@ -54,18 +70,34 @@ public class DatacenterDatasetQueryServiceImpl implements DatacenterDatasetQuery
|
||||
private DatacenterDatasetVersionMapper datasetVersionMapper;
|
||||
@Resource
|
||||
private DatacenterDerivedTableMapper derivedTableMapper;
|
||||
@Resource
|
||||
private DatacenterFederationQueryService federationQueryService;
|
||||
@Resource
|
||||
private DatacenterQueryAuditService queryAuditService;
|
||||
|
||||
@Override
|
||||
public Page<Row> queryPage(DatacenterQueryRequest request) {
|
||||
throw new BusinessException("结构化查询必须提供执行账号");
|
||||
}
|
||||
|
||||
/**
|
||||
* {@inheritDoc}
|
||||
*/
|
||||
@Override
|
||||
public Page<Row> queryPage(
|
||||
DatacenterQueryRequest request,
|
||||
LoginAccount account) {
|
||||
if (request == null || request.getDatasetRef() == null) {
|
||||
throw new BusinessException("datasetRef 不能为空");
|
||||
}
|
||||
normalizePage(request);
|
||||
DatacenterTable table = resolveTable(request.getDatasetRef());
|
||||
DatacenterSource source = registryService.getSourceRequired(table.getSourceId());
|
||||
validateStructuredQueryTenant(
|
||||
request.getDatasetRef(), table, source, account);
|
||||
DatacenterCatalog catalog = registryService.getCatalogById(table.getCatalogId());
|
||||
DatacenterTable queryTable = resolveQueryTable(table, request.getDatasetRef());
|
||||
validateRequest(queryTable, request, source);
|
||||
validateRequest(queryTable, request);
|
||||
request.getDatasetRef().setSourceId(table.getSourceId());
|
||||
request.getDatasetRef().setCatalogId(table.getCatalogId());
|
||||
request.getDatasetRef().setTableId(table.getId());
|
||||
@@ -73,15 +105,199 @@ public class DatacenterDatasetQueryServiceImpl implements DatacenterDatasetQuery
|
||||
if (catalog != null) {
|
||||
request.getDatasetRef().setCatalogName(catalog.getCatalogName());
|
||||
}
|
||||
if (isFederated(source)) {
|
||||
return queryFederatedPage(
|
||||
source, catalog, queryTable, request, account);
|
||||
}
|
||||
DatacenterConnector connector = connectorRegistry.getConnector(source.getSourceType());
|
||||
return connector.queryPage(source, queryTable, request);
|
||||
QueryId queryId = QueryId.create();
|
||||
long startedNanos = System.nanoTime();
|
||||
DatacenterQueryAudit audit = queryAuditService.start(
|
||||
queryId.value(), source, structuredQueryDescription(queryTable, request),
|
||||
structuredParameterSummary(request), account,
|
||||
"DATASET_PAGE", table.getId().toString());
|
||||
try {
|
||||
Page<Row> result = connector.queryPage(source, queryTable, request);
|
||||
long durationMs = elapsedMillis(startedNanos);
|
||||
queryAuditService.succeed(
|
||||
audit,
|
||||
List.of(table.getTableName()),
|
||||
structuredReferencedFields(request),
|
||||
result.getRecords().size(),
|
||||
false,
|
||||
durationMs,
|
||||
durationMs);
|
||||
return result;
|
||||
} catch (RuntimeException exception) {
|
||||
queryAuditService.fail(
|
||||
audit,
|
||||
"DATASET_PAGE_QUERY_FAILED",
|
||||
safeMessage(exception),
|
||||
List.of(table.getTableName()),
|
||||
structuredReferencedFields(request),
|
||||
elapsedMillis(startedNanos));
|
||||
throw exception;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 使用执行账号绑定结构化查询的权威租户。
|
||||
*
|
||||
* @param datasetRef 数据集引用
|
||||
* @param table 已解析表
|
||||
* @param source 已解析数据源
|
||||
* @param account 执行账号
|
||||
* @throws BusinessException 账号或租户边界不合法时抛出
|
||||
*/
|
||||
private void validateStructuredQueryTenant(
|
||||
DatasetRef datasetRef,
|
||||
DatacenterTable table,
|
||||
DatacenterSource source,
|
||||
LoginAccount account) {
|
||||
if (account == null || account.getTenantId() == null) {
|
||||
throw new BusinessException("结构化查询缺少执行账号租户");
|
||||
}
|
||||
BigInteger tenantId = account.getTenantId();
|
||||
if ((datasetRef.getTenantId() != null
|
||||
&& !tenantId.equals(datasetRef.getTenantId()))
|
||||
|| !tenantId.equals(table.getTenantId())
|
||||
|| !tenantId.equals(source.getTenantId())) {
|
||||
throw new BusinessException("数据集不属于当前租户");
|
||||
}
|
||||
datasetRef.setTenantId(tenantId);
|
||||
}
|
||||
|
||||
private String structuredQueryDescription(
|
||||
DatacenterTable table,
|
||||
DatacenterQueryRequest request) {
|
||||
StringBuilder sql = new StringBuilder("SELECT ")
|
||||
.append(String.join(", ", request.getSelectedColumns()))
|
||||
.append(" FROM ").append(table.getTableName());
|
||||
if (!CollectionUtils.isEmpty(request.getFilters())) {
|
||||
sql.append(" WHERE ");
|
||||
for (int index = 0; index < request.getFilters().size(); index++) {
|
||||
DatacenterQueryFilter filter = request.getFilters().get(index);
|
||||
if (index > 0) {
|
||||
sql.append(" AND ");
|
||||
}
|
||||
String operator = normalizeFilterOperator(filter.getOperator());
|
||||
sql.append(filter.getColumn()).append(' ');
|
||||
switch (operator) {
|
||||
case "IS_NULL" -> sql.append("IS NULL");
|
||||
case "IN" -> sql.append("IN (?)");
|
||||
default -> sql.append(operator).append(" ?");
|
||||
}
|
||||
}
|
||||
}
|
||||
if (!CollectionUtils.isEmpty(request.getSorts())) {
|
||||
sql.append(" ORDER BY ");
|
||||
for (int index = 0; index < request.getSorts().size(); index++) {
|
||||
DatacenterQuerySort sort = request.getSorts().get(index);
|
||||
if (index > 0) {
|
||||
sql.append(", ");
|
||||
}
|
||||
sql.append(sort.getColumn()).append(' ')
|
||||
.append("DESC".equalsIgnoreCase(sort.getDirection())
|
||||
? "DESC" : "ASC");
|
||||
}
|
||||
}
|
||||
return sql.toString();
|
||||
}
|
||||
|
||||
/**
|
||||
* 构建不包含实际值的结构化查询参数摘要。
|
||||
*
|
||||
* @param request 查询请求
|
||||
* @return 脱敏参数摘要
|
||||
*/
|
||||
private Map<String, Object> structuredParameterSummary(
|
||||
DatacenterQueryRequest request) {
|
||||
List<Map<String, Object>> filters = new ArrayList<>();
|
||||
if (!CollectionUtils.isEmpty(request.getFilters())) {
|
||||
for (DatacenterQueryFilter filter : request.getFilters()) {
|
||||
String operator = normalizeFilterOperator(filter.getOperator());
|
||||
int valueCount = "IN".equals(operator)
|
||||
? (filter.getValues() == null ? 0 : filter.getValues().size())
|
||||
: "IS_NULL".equals(operator) ? 0 : 1;
|
||||
filters.add(Map.of(
|
||||
"column", filter.getColumn(),
|
||||
"operator", operator,
|
||||
"valueCount", valueCount));
|
||||
}
|
||||
}
|
||||
return Map.of(
|
||||
"filters", List.copyOf(filters),
|
||||
"pageNumber", request.getPageNumber(),
|
||||
"pageSize", request.getPageSize());
|
||||
}
|
||||
|
||||
/**
|
||||
* 收集结构化查询实际引用的投影、筛选和排序字段。
|
||||
*
|
||||
* @param request 查询请求
|
||||
* @return 去重后的字段列表
|
||||
*/
|
||||
private List<String> structuredReferencedFields(
|
||||
DatacenterQueryRequest request) {
|
||||
Set<String> fields = new LinkedHashSet<>(request.getSelectedColumns());
|
||||
if (!CollectionUtils.isEmpty(request.getFilters())) {
|
||||
request.getFilters().stream()
|
||||
.map(DatacenterQueryFilter::getColumn)
|
||||
.forEach(fields::add);
|
||||
}
|
||||
if (!CollectionUtils.isEmpty(request.getSorts())) {
|
||||
request.getSorts().stream()
|
||||
.map(DatacenterQuerySort::getColumn)
|
||||
.forEach(fields::add);
|
||||
}
|
||||
return List.copyOf(fields);
|
||||
}
|
||||
|
||||
/**
|
||||
* 将结构化筛选操作符归一化为审计中的稳定名称。
|
||||
*
|
||||
* @param operator 原始操作符
|
||||
* @return 大写稳定操作符
|
||||
*/
|
||||
private String normalizeFilterOperator(String operator) {
|
||||
return trimToNull(operator) == null
|
||||
? "EQ" : operator.trim().toUpperCase(java.util.Locale.ROOT);
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<Row> queryBySql(DatacenterSqlQueryRequest request) {
|
||||
ResolvedSqlQuery query = resolveSqlQuery(request);
|
||||
return query.connector.queryBySql(
|
||||
query.source, query.sql);
|
||||
if (isFederated(query.source)) {
|
||||
DatacenterSqlConsoleResult result = federationQueryService.execute(
|
||||
query.source,
|
||||
query.sql,
|
||||
List.of(),
|
||||
1_000,
|
||||
null,
|
||||
"DATASET",
|
||||
request.getDatasetRef().getTableId() == null
|
||||
? null : request.getDatasetRef().getTableId().toString());
|
||||
if (result.truncated()) {
|
||||
throw new BusinessException("数据集查询结果超过返回上限,请使用流式消费");
|
||||
}
|
||||
return toRows(result);
|
||||
}
|
||||
QueryId queryId = QueryId.create();
|
||||
long startedNanos = System.nanoTime();
|
||||
DatacenterQueryAudit audit = queryAuditService.start(
|
||||
queryId.value(), query.source, query.sql, Map.of(), null,
|
||||
"DATASET", request.getDatasetRef().getTableId().toString());
|
||||
try {
|
||||
List<Row> rows = query.connector.queryBySql(query.source, query.sql);
|
||||
long durationMs = elapsedMillis(startedNanos);
|
||||
queryAuditService.succeed(audit, query.referencedTables, List.of(),
|
||||
rows.size(), false, durationMs, durationMs);
|
||||
return rows;
|
||||
} catch (RuntimeException exception) {
|
||||
queryAuditService.fail(audit, "DATASET_QUERY_FAILED", safeMessage(exception),
|
||||
query.referencedTables, List.of(), elapsedMillis(startedNanos));
|
||||
throw exception;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -91,12 +307,38 @@ public class DatacenterDatasetQueryServiceImpl implements DatacenterDatasetQuery
|
||||
public void consumeBySql(
|
||||
DatacenterSqlQueryRequest request,
|
||||
int fetchSize,
|
||||
LoginAccount account,
|
||||
String requestedQueryId,
|
||||
Consumer<Row> consumer) {
|
||||
if (fetchSize <= 0 || consumer == null) {
|
||||
throw new IllegalArgumentException(
|
||||
"fetchSize and consumer must be valid");
|
||||
}
|
||||
ResolvedSqlQuery query = resolveSqlQuery(request);
|
||||
ResolvedSqlQuery query = resolveSqlQuery(request, account);
|
||||
if (isFederated(query.source)) {
|
||||
int maxRows = Integer.getInteger(
|
||||
"easyflow.datacenter.query.max-rows", 1_000_000);
|
||||
long maxBytes = Long.getLong(
|
||||
"easyflow.datacenter.query.max-bytes", 512L * 1024L * 1024L);
|
||||
federationQueryService.consume(
|
||||
query.source,
|
||||
query.sql,
|
||||
List.of(),
|
||||
fetchSize,
|
||||
maxRows,
|
||||
maxBytes,
|
||||
account,
|
||||
"DATASET",
|
||||
request.getDatasetRef().getTableId() == null
|
||||
? null : request.getDatasetRef().getTableId().toString(),
|
||||
requestedQueryId,
|
||||
sourceRow -> {
|
||||
Row row = new Row();
|
||||
sourceRow.forEach(row::put);
|
||||
consumer.accept(row);
|
||||
});
|
||||
return;
|
||||
}
|
||||
int maxRows = Integer.getInteger(
|
||||
"easyflow.datacenter.query.max-rows",
|
||||
1_000_000);
|
||||
@@ -105,34 +347,48 @@ public class DatacenterDatasetQueryServiceImpl implements DatacenterDatasetQuery
|
||||
512L * 1024L * 1024L);
|
||||
long[] accumulatedRows = {0L};
|
||||
long[] accumulatedBytes = {0L};
|
||||
query.connector.consumeBySql(
|
||||
query.source,
|
||||
query.sql,
|
||||
fetchSize,
|
||||
row -> {
|
||||
accumulatedRows[0]++;
|
||||
if (maxRows > 0
|
||||
&& accumulatedRows[0] > maxRows) {
|
||||
throw new BusinessException(
|
||||
"数据集查询结果超过行数上限: "
|
||||
+ maxRows);
|
||||
}
|
||||
for (Map.Entry<String, Object> entry
|
||||
: row.entrySet()) {
|
||||
accumulatedBytes[0] +=
|
||||
estimateQueryValueBytes(
|
||||
entry.getKey(),
|
||||
entry.getValue());
|
||||
if (maxBytes > 0L
|
||||
&& accumulatedBytes[0] > maxBytes) {
|
||||
QueryId queryId = resolveQueryId(requestedQueryId);
|
||||
long startedNanos = System.nanoTime();
|
||||
DatacenterQueryAudit audit = queryAuditService.start(
|
||||
queryId.value(), query.source, query.sql, Map.of(), account,
|
||||
"DATASET", request.getDatasetRef().getTableId().toString());
|
||||
try {
|
||||
query.connector.consumeBySql(
|
||||
query.source,
|
||||
query.sql,
|
||||
fetchSize,
|
||||
row -> {
|
||||
accumulatedRows[0]++;
|
||||
if (maxRows > 0
|
||||
&& accumulatedRows[0] > maxRows) {
|
||||
throw new BusinessException(
|
||||
"数据集查询结果超过字节上限: "
|
||||
+ maxBytes);
|
||||
"数据集查询结果超过行数上限: "
|
||||
+ maxRows);
|
||||
}
|
||||
for (Map.Entry<String, Object> entry
|
||||
: row.entrySet()) {
|
||||
accumulatedBytes[0] +=
|
||||
estimateQueryValueBytes(
|
||||
entry.getKey(),
|
||||
entry.getValue());
|
||||
if (maxBytes > 0L
|
||||
&& accumulatedBytes[0] > maxBytes) {
|
||||
throw new BusinessException(
|
||||
"数据集查询结果超过字节上限: "
|
||||
+ maxBytes);
|
||||
}
|
||||
}
|
||||
consumer.accept(row);
|
||||
}
|
||||
consumer.accept(row);
|
||||
}
|
||||
);
|
||||
);
|
||||
long durationMs = elapsedMillis(startedNanos);
|
||||
queryAuditService.succeed(audit, query.referencedTables, List.of(),
|
||||
accumulatedRows[0], false, durationMs, durationMs);
|
||||
} catch (RuntimeException exception) {
|
||||
queryAuditService.fail(audit, "DATASET_QUERY_FAILED", safeMessage(exception),
|
||||
query.referencedTables, List.of(), elapsedMillis(startedNanos));
|
||||
throw exception;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -171,6 +427,19 @@ public class DatacenterDatasetQueryServiceImpl implements DatacenterDatasetQuery
|
||||
*/
|
||||
private ResolvedSqlQuery resolveSqlQuery(
|
||||
DatacenterSqlQueryRequest request) {
|
||||
return resolveSqlQuery(request, null);
|
||||
}
|
||||
|
||||
/**
|
||||
* 校验请求、执行账号租户并解析实际连接器与可执行 SQL。
|
||||
*
|
||||
* @param request SQL 查询请求
|
||||
* @param account 后台执行账号;普通同步调用可为空
|
||||
* @return 已解析查询
|
||||
*/
|
||||
private ResolvedSqlQuery resolveSqlQuery(
|
||||
DatacenterSqlQueryRequest request,
|
||||
LoginAccount account) {
|
||||
if (request == null || request.getDatasetRef() == null) {
|
||||
throw new BusinessException("datasetRef 不能为空");
|
||||
}
|
||||
@@ -183,8 +452,28 @@ public class DatacenterDatasetQueryServiceImpl implements DatacenterDatasetQuery
|
||||
throw new BusinessException("缺少连接服务配置");
|
||||
}
|
||||
DatacenterSource source = registryService.getSourceRequired(datasetRef.getSourceId());
|
||||
BigInteger authoritativeTenantId = account != null && account.getTenantId() != null
|
||||
? account.getTenantId() : datasetRef.getTenantId();
|
||||
if (authoritativeTenantId == null) {
|
||||
throw new BusinessException("数据集绑定缺少租户信息");
|
||||
}
|
||||
if ((datasetRef.getTenantId() != null
|
||||
&& !datasetRef.getTenantId().equals(authoritativeTenantId))
|
||||
|| !authoritativeTenantId.equals(source.getTenantId())) {
|
||||
throw new BusinessException("数据集不属于当前租户");
|
||||
}
|
||||
datasetRef.setTenantId(authoritativeTenantId);
|
||||
BigInteger catalogId = resolveRequestedCatalogId(datasetRef);
|
||||
List<DatacenterTable> managedTables = registryService.listManagedTables(datasetRef.getSourceId(), catalogId);
|
||||
if (datasetRef.getTableId() == null) {
|
||||
throw new BusinessException("数据集 SQL 查询必须绑定具体表");
|
||||
}
|
||||
DatacenterTable boundTable = registryService.getTableWithFields(datasetRef.getTableId());
|
||||
if (!datasetRef.getSourceId().equals(boundTable.getSourceId())
|
||||
|| (catalogId != null && !catalogId.equals(boundTable.getCatalogId()))
|
||||
|| !authoritativeTenantId.equals(boundTable.getTenantId())) {
|
||||
throw new BusinessException("数据集引用与绑定表不一致");
|
||||
}
|
||||
List<DatacenterTable> managedTables = List.of(boundTable);
|
||||
if (CollectionUtils.isEmpty(managedTables)) {
|
||||
throw new BusinessException("当前连接下没有已接入表");
|
||||
}
|
||||
@@ -198,12 +487,15 @@ public class DatacenterDatasetQueryServiceImpl implements DatacenterDatasetQuery
|
||||
// 内部连接的 catalog 是逻辑命名空间,底层项目 MySQL 只执行物理表名。
|
||||
SqlSupportUtils.ResolvedSql resolvedSql =
|
||||
connector instanceof AbstractInternalTableConnector
|
||||
|| DatacenterSourceType.PROJECT_MYSQL.name()
|
||||
.equals(source.getSourceType())
|
||||
? SqlSupportUtils.resolveInternalMysql(sql, sqlTables)
|
||||
: SqlSupportUtils.resolve(sql, sqlTables);
|
||||
return new ResolvedSqlQuery(
|
||||
source,
|
||||
connector,
|
||||
resolvedSql.getExecutableSql());
|
||||
isFederated(source) ? sql : resolvedSql.getExecutableSql(),
|
||||
resolvedSql.getLogicalTables());
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -212,30 +504,123 @@ public class DatacenterDatasetQueryServiceImpl implements DatacenterDatasetQuery
|
||||
* @param source 数据源
|
||||
* @param connector 数据连接器
|
||||
* @param sql 可执行 SQL
|
||||
* @param referencedTables 已校验引用表
|
||||
*/
|
||||
private record ResolvedSqlQuery(
|
||||
DatacenterSource source,
|
||||
DatacenterConnector connector,
|
||||
String sql) {
|
||||
String sql,
|
||||
List<String> referencedTables) {
|
||||
}
|
||||
|
||||
private long elapsedMillis(long startedNanos) {
|
||||
return Math.max(0L, (System.nanoTime() - startedNanos) / 1_000_000L);
|
||||
}
|
||||
|
||||
/**
|
||||
* 校验调用方查询 UUID,内部连接也使用同一标识完成审计关联。
|
||||
*
|
||||
* @param requestedQueryId 调用方标识;为空时生成
|
||||
* @return 查询标识
|
||||
*/
|
||||
private QueryId resolveQueryId(String requestedQueryId) {
|
||||
if (!StringUtils.hasText(requestedQueryId)) {
|
||||
return QueryId.create();
|
||||
}
|
||||
try {
|
||||
return new QueryId(UUID.fromString(
|
||||
requestedQueryId.trim()).toString());
|
||||
} catch (IllegalArgumentException exception) {
|
||||
throw new BusinessException("queryId 必须是 UUID");
|
||||
}
|
||||
}
|
||||
|
||||
private String safeMessage(RuntimeException exception) {
|
||||
String message = exception.getMessage();
|
||||
if (message == null || message.isBlank()) {
|
||||
return "数据集查询失败";
|
||||
}
|
||||
return message.length() <= 500 ? message : message.substring(0, 500);
|
||||
}
|
||||
|
||||
@Override
|
||||
public DatacenterSchemaResponse getSchema(DatasetRef datasetRef) {
|
||||
return getSchema(datasetRef, null, null);
|
||||
}
|
||||
|
||||
/**
|
||||
* {@inheritDoc}
|
||||
*/
|
||||
@Override
|
||||
public DatacenterSchemaResponse getSchema(
|
||||
DatasetRef datasetRef,
|
||||
Long fieldPageNumber,
|
||||
Long fieldPageSize) {
|
||||
DatacenterTable table = resolveTable(datasetRef);
|
||||
List<DatacenterTableField> allFields = List.copyOf(table.getFields());
|
||||
long actualPage = fieldPageNumber == null ? 1L : Math.max(1L, fieldPageNumber);
|
||||
long actualSize = fieldPageSize == null
|
||||
? Math.max(1L, allFields.size())
|
||||
: Math.min(200L, Math.max(1L, fieldPageSize));
|
||||
List<DatacenterTableField> pageFields = sliceFields(
|
||||
allFields, actualPage, actualSize);
|
||||
long offset = safePageOffset(actualPage, actualSize);
|
||||
boolean hasMoreFields = offset < allFields.size()
|
||||
&& offset + pageFields.size() < allFields.size();
|
||||
// 字段只通过顶层分页字段返回,避免在 table 内重复序列化完整列表。
|
||||
table.setFields(List.of());
|
||||
DatacenterSource source = registryService.getSourceRequired(table.getSourceId());
|
||||
DatacenterCatalog catalog = registryService.getCatalogById(table.getCatalogId());
|
||||
DatacenterSchemaResponse response = new DatacenterSchemaResponse();
|
||||
response.setDatasetRef(registryService.resolveDatasetRef(table.getId()));
|
||||
response.setSource(source);
|
||||
response.setSource(DatacenterSourceViews.from(source));
|
||||
response.setCatalog(catalog);
|
||||
response.setTable(table);
|
||||
response.setFields(table.getFields());
|
||||
response.setFields(pageFields);
|
||||
response.setFieldPageNumber(actualPage);
|
||||
response.setFieldPageSize(actualSize);
|
||||
response.setHasMoreFields(hasMoreFields);
|
||||
response.setVersions(listVersions(table.getId()));
|
||||
response.setUpstreamLineage(listUpstream(table.getId()));
|
||||
response.setDownstreamLineage(listDownstream(table.getId()));
|
||||
return response;
|
||||
}
|
||||
|
||||
/**
|
||||
* 截取字段页并避免向响应暴露可变子列表。
|
||||
*
|
||||
* @param fields 完整字段列表
|
||||
* @param pageNumber 页码
|
||||
* @param pageSize 页大小
|
||||
* @return 当前页字段
|
||||
*/
|
||||
private List<DatacenterTableField> sliceFields(
|
||||
List<DatacenterTableField> fields,
|
||||
long pageNumber,
|
||||
long pageSize) {
|
||||
long offset = safePageOffset(pageNumber, pageSize);
|
||||
if (offset >= fields.size()) {
|
||||
return List.of();
|
||||
}
|
||||
int fromIndex = Math.toIntExact(offset);
|
||||
int toIndex = (int) Math.min((long) fields.size(), offset + pageSize);
|
||||
return List.copyOf(fields.subList(fromIndex, toIndex));
|
||||
}
|
||||
|
||||
/**
|
||||
* 计算字段页偏移并防止长整型乘法溢出。
|
||||
*
|
||||
* @param pageNumber 页码
|
||||
* @param pageSize 页大小
|
||||
* @return 安全偏移;溢出时返回最大长整型
|
||||
*/
|
||||
private long safePageOffset(long pageNumber, long pageSize) {
|
||||
if (pageNumber - 1L > Long.MAX_VALUE / pageSize) {
|
||||
return Long.MAX_VALUE;
|
||||
}
|
||||
return (pageNumber - 1L) * pageSize;
|
||||
}
|
||||
|
||||
/**
|
||||
* {@inheritDoc}
|
||||
*/
|
||||
@@ -245,8 +630,8 @@ public class DatacenterDatasetQueryServiceImpl implements DatacenterDatasetQuery
|
||||
DatacenterSchemaResponse response =
|
||||
new DatacenterSchemaResponse();
|
||||
response.setDatasetRef(datasetRef);
|
||||
response.setSource(
|
||||
registryService.getSourceRequired(table.getSourceId()));
|
||||
response.setSource(DatacenterSourceViews.from(
|
||||
registryService.getSourceRequired(table.getSourceId())));
|
||||
response.setCatalog(
|
||||
registryService.getCatalogById(table.getCatalogId()));
|
||||
response.setTable(table);
|
||||
@@ -319,13 +704,29 @@ public class DatacenterDatasetQueryServiceImpl implements DatacenterDatasetQuery
|
||||
DatacenterCatalog catalog = catalogId == null
|
||||
? null
|
||||
: catalogsById.get(catalogId);
|
||||
List<String> knownColumns = table.getFields().stream()
|
||||
.filter(field -> DatacenterMetadataStatus.ACTIVE.name()
|
||||
.equals(field.getMetadataStatus()))
|
||||
.map(this::sourceColumnName)
|
||||
.toList();
|
||||
List<String> queryableColumns = table.getFields().stream()
|
||||
.filter(this::isQueryable)
|
||||
.map(this::sourceColumnName)
|
||||
.toList();
|
||||
return new SqlSupportUtils.ManagedTable(
|
||||
catalog == null ? null : catalog.getCatalogName(),
|
||||
table.getTableName(),
|
||||
resolvePhysicalTableName(table)
|
||||
resolvePhysicalTableName(table),
|
||||
knownColumns,
|
||||
queryableColumns
|
||||
);
|
||||
}
|
||||
|
||||
private String sourceColumnName(DatacenterTableField field) {
|
||||
String sourceColumnName = trimToNull(field.getSourceColumnName());
|
||||
return sourceColumnName == null ? field.getFieldName() : sourceColumnName;
|
||||
}
|
||||
|
||||
/**
|
||||
* 一次批量加载 SQL 白名单表关联的目录,避免逐表查询。
|
||||
*
|
||||
@@ -367,6 +768,153 @@ public class DatacenterDatasetQueryServiceImpl implements DatacenterDatasetQuery
|
||||
return actualTable != null ? actualTable : table.getTableName();
|
||||
}
|
||||
|
||||
/**
|
||||
* 使用逻辑 Schema 构建参数化 ANSI SQL,再由 Calcite 转换为目标方言。
|
||||
*/
|
||||
private Page<Row> queryFederatedPage(
|
||||
DatacenterSource source,
|
||||
DatacenterCatalog catalog,
|
||||
DatacenterTable table,
|
||||
DatacenterQueryRequest request,
|
||||
LoginAccount account) {
|
||||
if (catalog == null || catalog.getLogicalSchemaName() == null) {
|
||||
throw new BusinessException("数据集缺少 Federation Schema 绑定");
|
||||
}
|
||||
if (request.getPageSize() > 500L) {
|
||||
throw new BusinessException("外部数据源单页最多返回 500 行");
|
||||
}
|
||||
Map<String, DatacenterTableField> fields = table.getFields().stream()
|
||||
.collect(Collectors.toMap(
|
||||
DatacenterTableField::getFieldName,
|
||||
Function.identity(),
|
||||
(first, ignored) -> first,
|
||||
LinkedHashMap::new));
|
||||
FederatedWhere where = buildFederatedWhere(request.getFilters(), fields);
|
||||
String from = quote(catalog.getLogicalSchemaName()) + "." + quote(table.getTableName());
|
||||
String countSql = "SELECT COUNT(*) AS total FROM " + from + where.sql();
|
||||
DatacenterSqlConsoleResult countResult = federationQueryService.execute(
|
||||
source, countSql, where.parameters(), 1, account,
|
||||
"DATASET", table.getId().toString());
|
||||
long total = countResult.rows().isEmpty()
|
||||
? 0L : Long.parseLong(String.valueOf(countResult.rows().get(0).get("c1")));
|
||||
|
||||
if (total == 0L) {
|
||||
return new Page<>(List.of(), request.getPageNumber(), request.getPageSize(), 0L);
|
||||
}
|
||||
String columns = request.getSelectedColumns().stream()
|
||||
.map(name -> quote(requireField(fields, name).getSourceColumnName()))
|
||||
.collect(Collectors.joining(", "));
|
||||
String orderBy = buildFederatedOrderBy(request.getSorts(), fields);
|
||||
long offset = (request.getPageNumber() - 1L) * request.getPageSize();
|
||||
String pageSql = "SELECT " + columns + " FROM " + from + where.sql() + orderBy
|
||||
+ " OFFSET " + offset + " ROWS FETCH NEXT " + request.getPageSize() + " ROWS ONLY";
|
||||
DatacenterSqlConsoleResult pageResult = federationQueryService.execute(
|
||||
source, pageSql, where.parameters(), request.getPageSize().intValue(),
|
||||
account, "DATASET", table.getId().toString());
|
||||
return new Page<>(toRows(pageResult), request.getPageNumber(), request.getPageSize(), total);
|
||||
}
|
||||
|
||||
private FederatedWhere buildFederatedWhere(
|
||||
List<DatacenterQueryFilter> filters,
|
||||
Map<String, DatacenterTableField> fields) {
|
||||
if (CollectionUtils.isEmpty(filters)) {
|
||||
return new FederatedWhere("", List.of());
|
||||
}
|
||||
StringBuilder sql = new StringBuilder(" WHERE 1 = 1");
|
||||
List<SqlParameter> parameters = new ArrayList<>();
|
||||
for (DatacenterQueryFilter filter : filters) {
|
||||
DatacenterTableField field = requireField(fields, filter.getColumn());
|
||||
String column = quote(field.getSourceColumnName());
|
||||
String operator = trimToNull(filter.getOperator()) == null
|
||||
? "EQ" : filter.getOperator().trim().toUpperCase(java.util.Locale.ROOT);
|
||||
switch (operator) {
|
||||
case "EQ", "GT", "GTE", "LT", "LTE", "LIKE" -> {
|
||||
String sqlOperator = switch (operator) {
|
||||
case "GT" -> ">";
|
||||
case "GTE" -> ">=";
|
||||
case "LT" -> "<";
|
||||
case "LTE" -> "<=";
|
||||
case "LIKE" -> "LIKE";
|
||||
default -> "=";
|
||||
};
|
||||
sql.append(" AND ").append(column).append(' ').append(sqlOperator).append(" ?");
|
||||
Object value = "LIKE".equals(operator)
|
||||
? "%" + filter.getValue() + "%" : filter.getValue();
|
||||
parameters.add(parameter(field, value));
|
||||
}
|
||||
case "IS_NULL" -> sql.append(" AND ").append(column).append(" IS NULL");
|
||||
case "IN" -> {
|
||||
List<Object> values = filter.getValues() == null ? List.of() : filter.getValues();
|
||||
if (values.isEmpty()) {
|
||||
sql.append(" AND 1 = 0");
|
||||
} else {
|
||||
sql.append(" AND ").append(column).append(" IN (")
|
||||
.append(values.stream().map(ignored -> "?").collect(Collectors.joining(",")))
|
||||
.append(')');
|
||||
values.forEach(value -> parameters.add(parameter(field, value)));
|
||||
}
|
||||
}
|
||||
default -> throw new BusinessException("不支持的过滤操作: " + operator);
|
||||
}
|
||||
}
|
||||
return new FederatedWhere(sql.toString(), List.copyOf(parameters));
|
||||
}
|
||||
|
||||
private String buildFederatedOrderBy(
|
||||
List<DatacenterQuerySort> sorts,
|
||||
Map<String, DatacenterTableField> fields) {
|
||||
if (CollectionUtils.isEmpty(sorts)) {
|
||||
return "";
|
||||
}
|
||||
return " ORDER BY " + sorts.stream()
|
||||
.map(sort -> quote(requireField(fields, sort.getColumn()).getSourceColumnName())
|
||||
+ ("DESC".equalsIgnoreCase(sort.getDirection()) ? " DESC" : " ASC"))
|
||||
.collect(Collectors.joining(", "));
|
||||
}
|
||||
|
||||
private DatacenterTableField requireField(
|
||||
Map<String, DatacenterTableField> fields,
|
||||
String name) {
|
||||
DatacenterTableField field = fields.get(name);
|
||||
if (field == null || !isEnabled(field.getQueryable())) {
|
||||
throw new BusinessException("字段不可查询: " + name);
|
||||
}
|
||||
return field;
|
||||
}
|
||||
|
||||
private SqlParameter parameter(DatacenterTableField field, Object value) {
|
||||
int jdbcType = field.getJdbcTypeCode() == null ? Types.VARCHAR : field.getJdbcTypeCode();
|
||||
return value == null ? new SqlParameter(jdbcType, null) : new SqlParameter(jdbcType, value);
|
||||
}
|
||||
|
||||
private String quote(String identifier) {
|
||||
if (!StringUtils.hasText(identifier)) {
|
||||
throw new BusinessException("数据集物理名称为空");
|
||||
}
|
||||
return '"' + identifier.replace("\"", "\"\"") + '"';
|
||||
}
|
||||
|
||||
private List<Row> toRows(DatacenterSqlConsoleResult result) {
|
||||
List<Row> rows = new ArrayList<>(result.rows().size());
|
||||
for (Map<String, Object> sourceRow : result.rows()) {
|
||||
Row row = new Row();
|
||||
for (int index = 0; index < result.columns().size(); index++) {
|
||||
var column = result.columns().get(index);
|
||||
row.put(column.label(), sourceRow.get(column.key()));
|
||||
}
|
||||
rows.add(row);
|
||||
}
|
||||
return rows;
|
||||
}
|
||||
|
||||
private boolean isFederated(DatacenterSource source) {
|
||||
return source != null && (DatacenterSourceType.MYSQL.name().equals(source.getSourceType())
|
||||
|| DatacenterSourceType.POSTGRESQL.name().equals(source.getSourceType()));
|
||||
}
|
||||
|
||||
private record FederatedWhere(String sql, List<SqlParameter> parameters) {
|
||||
}
|
||||
|
||||
private void normalizePage(DatacenterQueryRequest request) {
|
||||
if (request.getPageNumber() == null || request.getPageNumber() < 1L) {
|
||||
request.setPageNumber(1L);
|
||||
@@ -374,6 +922,9 @@ public class DatacenterDatasetQueryServiceImpl implements DatacenterDatasetQuery
|
||||
if (request.getPageSize() == null || request.getPageSize() < 1L) {
|
||||
throw new BusinessException("pageSize 必须大于 0");
|
||||
}
|
||||
if (request.getPageSize() > 500L) {
|
||||
throw new BusinessException("单页最多返回 500 行");
|
||||
}
|
||||
}
|
||||
|
||||
private DatacenterTable resolveQueryTable(DatacenterTable table, DatasetRef datasetRef) {
|
||||
@@ -400,7 +951,7 @@ public class DatacenterDatasetQueryServiceImpl implements DatacenterDatasetQuery
|
||||
return queryTable;
|
||||
}
|
||||
|
||||
private void validateRequest(DatacenterTable table, DatacenterQueryRequest request, DatacenterSource source) {
|
||||
private void validateRequest(DatacenterTable table, DatacenterQueryRequest request) {
|
||||
Map<String, DatacenterTableField> fieldMap = new LinkedHashMap<>();
|
||||
for (DatacenterTableField field : table.getFields()) {
|
||||
fieldMap.put(field.getFieldName(), field);
|
||||
@@ -408,22 +959,25 @@ public class DatacenterDatasetQueryServiceImpl implements DatacenterDatasetQuery
|
||||
if (!CollectionUtils.isEmpty(request.getSelectedColumns())) {
|
||||
for (String column : request.getSelectedColumns()) {
|
||||
DatacenterTableField field = fieldMap.get(column);
|
||||
if (field == null || !isEnabled(field.getQueryable())) {
|
||||
if (field == null || !isQueryable(field)) {
|
||||
throw new BusinessException("字段不可查询: " + column);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
request.setSelectedColumns(
|
||||
table.getFields().stream()
|
||||
.filter(field -> isEnabled(field.getQueryable()))
|
||||
.filter(this::isQueryable)
|
||||
.map(DatacenterTableField::getFieldName)
|
||||
.toList()
|
||||
);
|
||||
}
|
||||
if (CollectionUtils.isEmpty(request.getSelectedColumns())) {
|
||||
throw new BusinessException("当前数据集没有可查询字段");
|
||||
}
|
||||
if (!CollectionUtils.isEmpty(request.getFilters())) {
|
||||
request.getFilters().forEach(filter -> {
|
||||
DatacenterTableField field = fieldMap.get(filter.getColumn());
|
||||
if (field == null || !isEnabled(field.getQueryable())) {
|
||||
if (field == null || !isQueryable(field)) {
|
||||
throw new BusinessException("字段不可过滤: " + filter.getColumn());
|
||||
}
|
||||
});
|
||||
@@ -431,18 +985,13 @@ public class DatacenterDatasetQueryServiceImpl implements DatacenterDatasetQuery
|
||||
if (!CollectionUtils.isEmpty(request.getSorts())) {
|
||||
request.getSorts().forEach(sort -> {
|
||||
DatacenterTableField field = fieldMap.get(sort.getColumn());
|
||||
if (field == null || !isEnabled(field.getSortable())) {
|
||||
if (field == null || !isQueryable(field) || !isEnabled(field.getSortable())) {
|
||||
throw new BusinessException("字段不可排序: " + sort.getColumn());
|
||||
}
|
||||
});
|
||||
}
|
||||
if (request.getWhere() != null && !request.getWhere().isBlank()) {
|
||||
boolean allowLegacyWhere = "PROJECT_MYSQL".equals(source.getSourceType())
|
||||
|| "MYSQL".equals(source.getSourceType())
|
||||
|| "POSTGRESQL".equals(source.getSourceType());
|
||||
if (!allowLegacyWhere) {
|
||||
throw new BusinessException("当前数据源仅支持结构化 DSL 查询");
|
||||
}
|
||||
throw new BusinessException("不支持原始 WHERE 条件,请使用结构化筛选条件");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -450,6 +999,13 @@ public class DatacenterDatasetQueryServiceImpl implements DatacenterDatasetQuery
|
||||
return value == null || value == 1;
|
||||
}
|
||||
|
||||
private boolean isQueryable(DatacenterTableField field) {
|
||||
return field != null
|
||||
&& isEnabled(field.getQueryable())
|
||||
&& DatacenterMetadataStatus.ACTIVE.name().equals(field.getMetadataStatus())
|
||||
&& DatacenterSensitivityLevel.PUBLIC.name().equals(field.getSensitivityLevel());
|
||||
}
|
||||
|
||||
private String trimToNull(String value) {
|
||||
if (!StringUtils.hasText(value)) {
|
||||
return null;
|
||||
|
||||
@@ -38,7 +38,7 @@ public class DatacenterDatasetWriteServiceImpl implements DatacenterDatasetWrite
|
||||
*/
|
||||
@Override
|
||||
public void saveRow(DatasetRef datasetRef, JSONObject data, LoginAccount account) {
|
||||
DatacenterTable table = resolveTable(datasetRef);
|
||||
DatacenterTable table = resolveTable(datasetRef, account);
|
||||
DatacenterSource source = registryService.getSourceRequired(table.getSourceId());
|
||||
DatacenterConnector connector = connectorRegistry.getConnector(source.getSourceType());
|
||||
connector.saveRow(source, table, data, account);
|
||||
@@ -52,7 +52,7 @@ public class DatacenterDatasetWriteServiceImpl implements DatacenterDatasetWrite
|
||||
if (rows == null || rows.isEmpty()) {
|
||||
return;
|
||||
}
|
||||
DatacenterTable table = resolveTable(datasetRef);
|
||||
DatacenterTable table = resolveTable(datasetRef, account);
|
||||
DatacenterSource source = registryService.getSourceRequired(table.getSourceId());
|
||||
DatacenterConnector connector = connectorRegistry.getConnector(source.getSourceType());
|
||||
connector.saveRows(source, table, rows, account, Math.max(1, batchSize));
|
||||
@@ -70,7 +70,7 @@ public class DatacenterDatasetWriteServiceImpl implements DatacenterDatasetWrite
|
||||
if (rows == null || rows.isEmpty()) {
|
||||
return true;
|
||||
}
|
||||
DatacenterTable table = resolveTable(datasetRef);
|
||||
DatacenterTable table = resolveTable(datasetRef, account);
|
||||
DatacenterSource source = registryService.getSourceRequired(table.getSourceId());
|
||||
DatacenterConnector connector = connectorRegistry.getConnector(source.getSourceType());
|
||||
String payloadHash = sha256Rows(rows);
|
||||
@@ -95,17 +95,27 @@ public class DatacenterDatasetWriteServiceImpl implements DatacenterDatasetWrite
|
||||
*/
|
||||
@Override
|
||||
public void deleteRow(DatasetRef datasetRef, BigInteger id, LoginAccount account) {
|
||||
DatacenterTable table = resolveTable(datasetRef);
|
||||
DatacenterTable table = resolveTable(datasetRef, account);
|
||||
DatacenterSource source = registryService.getSourceRequired(table.getSourceId());
|
||||
DatacenterConnector connector = connectorRegistry.getConnector(source.getSourceType());
|
||||
connector.deleteRow(source, table, id, account);
|
||||
}
|
||||
|
||||
private DatacenterTable resolveTable(DatasetRef datasetRef) {
|
||||
private DatacenterTable resolveTable(DatasetRef datasetRef, LoginAccount account) {
|
||||
if (datasetRef == null || datasetRef.getTableId() == null) {
|
||||
throw new BusinessException("缺少 tableId");
|
||||
}
|
||||
return registryService.getTableWithFields(datasetRef.getTableId());
|
||||
if (account == null || account.getTenantId() == null) {
|
||||
throw new BusinessException("数据集写入缺少执行租户");
|
||||
}
|
||||
DatacenterTable table = registryService.getTableWithFields(datasetRef.getTableId());
|
||||
if (!account.getTenantId().equals(table.getTenantId())
|
||||
|| (datasetRef.getTenantId() != null
|
||||
&& !account.getTenantId().equals(datasetRef.getTenantId()))) {
|
||||
throw new BusinessException("数据集不属于当前租户");
|
||||
}
|
||||
datasetRef.setTenantId(account.getTenantId());
|
||||
return table;
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
@@ -0,0 +1,308 @@
|
||||
package tech.easyflow.datacenter.federation;
|
||||
|
||||
import com.easyagents.federation.sql.api.FederationSqlErrorCode;
|
||||
import com.easyagents.federation.sql.api.FederationSqlException;
|
||||
import com.easyagents.federation.sql.execute.FederationQueryAdmissionController;
|
||||
import com.easyagents.federation.sql.execute.FederationQueryPermit;
|
||||
import com.easyagents.federation.sql.execute.LocalFederationQueryAdmissionController;
|
||||
import com.easyagents.federation.sql.execute.QueryId;
|
||||
import com.easyagents.federation.sql.source.SourceId;
|
||||
import java.time.Duration;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
import java.util.concurrent.atomic.AtomicBoolean;
|
||||
import java.util.function.BooleanSupplier;
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
/**
|
||||
* 对查询执行租户、数据源与节点三级本地并发准入。
|
||||
*/
|
||||
@Component
|
||||
public class DatacenterFederationAdmissionController
|
||||
implements FederationQueryAdmissionController {
|
||||
|
||||
private final LocalFederationQueryAdmissionController globalAdmission;
|
||||
private final int maximumConcurrentQueriesPerTenant;
|
||||
private final int maximumConcurrentQueriesPerSource;
|
||||
private final Map<String, AdmissionEntry>
|
||||
tenantAdmissions = new ConcurrentHashMap<>();
|
||||
private final Map<SourceId, AdmissionEntry>
|
||||
sourceAdmissions = new ConcurrentHashMap<>();
|
||||
private final Object lifecycleLock = new Object();
|
||||
private final AtomicBoolean closed = new AtomicBoolean();
|
||||
|
||||
/**
|
||||
* 创建三级准入控制器。
|
||||
*
|
||||
* @param maximumConcurrentQueries 节点查询上限
|
||||
* @param maximumConcurrentQueriesPerTenant 单租户查询上限
|
||||
* @param maximumConcurrentQueriesPerSource 单数据源查询上限
|
||||
*/
|
||||
public DatacenterFederationAdmissionController(
|
||||
@Value("${easyflow.datacenter.query-admission.maximum-concurrent-queries:64}")
|
||||
int maximumConcurrentQueries,
|
||||
@Value("${easyflow.datacenter.query-admission.maximum-concurrent-queries-per-tenant:20}")
|
||||
int maximumConcurrentQueriesPerTenant,
|
||||
@Value("${easyflow.datacenter.query-admission.maximum-concurrent-queries-per-source:10}")
|
||||
int maximumConcurrentQueriesPerSource) {
|
||||
if (maximumConcurrentQueries <= 0
|
||||
|| maximumConcurrentQueriesPerTenant <= 0
|
||||
|| maximumConcurrentQueriesPerSource <= 0) {
|
||||
throw new IllegalArgumentException(
|
||||
"query admission limits must be positive");
|
||||
}
|
||||
this.globalAdmission = new LocalFederationQueryAdmissionController(
|
||||
maximumConcurrentQueries);
|
||||
this.maximumConcurrentQueriesPerTenant =
|
||||
maximumConcurrentQueriesPerTenant;
|
||||
this.maximumConcurrentQueriesPerSource =
|
||||
maximumConcurrentQueriesPerSource;
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取三级查询许可。
|
||||
*
|
||||
* @param sourceId 数据源标识
|
||||
* @param queryId 查询标识
|
||||
* @param timeout 最大等待时间
|
||||
* @return 查询许可
|
||||
*/
|
||||
@Override
|
||||
public FederationQueryPermit acquire(
|
||||
SourceId sourceId,
|
||||
QueryId queryId,
|
||||
Duration timeout) {
|
||||
return acquire(sourceId, queryId, timeout, () -> false);
|
||||
}
|
||||
|
||||
/**
|
||||
* 按数据源、租户、节点顺序获取许可,避免热点连接池等待占满全局配额。
|
||||
*
|
||||
* @param sourceId 数据源标识
|
||||
* @param queryId 查询标识
|
||||
* @param timeout 最大等待时间
|
||||
* @param cancellationRequested 查询取消状态
|
||||
* @return 查询许可
|
||||
*/
|
||||
@Override
|
||||
public FederationQueryPermit acquire(
|
||||
SourceId sourceId,
|
||||
QueryId queryId,
|
||||
Duration timeout,
|
||||
BooleanSupplier cancellationRequested) {
|
||||
long started = System.nanoTime();
|
||||
String tenantKey = tenantKey(sourceId);
|
||||
AdmissionEntry sourceEntry;
|
||||
AdmissionEntry tenantEntry;
|
||||
synchronized (lifecycleLock) {
|
||||
ensureOpen();
|
||||
sourceEntry = retain(sourceAdmissions, sourceId,
|
||||
maximumConcurrentQueriesPerSource);
|
||||
tenantEntry = retain(tenantAdmissions, tenantKey,
|
||||
maximumConcurrentQueriesPerTenant);
|
||||
}
|
||||
FederationQueryPermit sourcePermit = null;
|
||||
FederationQueryPermit tenantPermit = null;
|
||||
FederationQueryPermit globalPermit = null;
|
||||
try {
|
||||
sourcePermit = sourceEntry.controller.acquire(
|
||||
sourceId, queryId, remaining(timeout, started),
|
||||
cancellationRequested);
|
||||
tenantPermit = tenantEntry.controller.acquire(
|
||||
sourceId, queryId, remaining(timeout, started),
|
||||
cancellationRequested);
|
||||
globalPermit = globalAdmission.acquire(
|
||||
sourceId, queryId, remaining(timeout, started),
|
||||
cancellationRequested);
|
||||
FederationQueryPermit acquiredSource = sourcePermit;
|
||||
FederationQueryPermit acquiredTenant = tenantPermit;
|
||||
FederationQueryPermit acquiredGlobal = globalPermit;
|
||||
AtomicBoolean released = new AtomicBoolean();
|
||||
return () -> {
|
||||
if (released.compareAndSet(false, true)) {
|
||||
try {
|
||||
acquiredGlobal.close();
|
||||
} finally {
|
||||
try {
|
||||
acquiredTenant.close();
|
||||
} finally {
|
||||
try {
|
||||
acquiredSource.close();
|
||||
} finally {
|
||||
releaseEntries(
|
||||
sourceId, sourceEntry,
|
||||
tenantKey, tenantEntry);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
} catch (RuntimeException exception) {
|
||||
closeQuietly(globalPermit);
|
||||
closeQuietly(tenantPermit);
|
||||
closeQuietly(sourcePermit);
|
||||
releaseEntries(sourceId, sourceEntry, tenantKey, tenantEntry);
|
||||
throw exception;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 关闭全部准入控制器并唤醒等待线程。
|
||||
*/
|
||||
@Override
|
||||
public void close() {
|
||||
synchronized (lifecycleLock) {
|
||||
if (!closed.compareAndSet(false, true)) {
|
||||
return;
|
||||
}
|
||||
globalAdmission.close();
|
||||
tenantAdmissions.values().forEach(
|
||||
entry -> entry.controller.close());
|
||||
sourceAdmissions.values().forEach(
|
||||
entry -> entry.controller.close());
|
||||
tenantAdmissions.clear();
|
||||
sourceAdmissions.clear();
|
||||
}
|
||||
}
|
||||
|
||||
private Duration remaining(Duration timeout, long started) {
|
||||
if (timeout == null || timeout.isZero() || timeout.isNegative()) {
|
||||
return Duration.ZERO;
|
||||
}
|
||||
long remaining = timeout.toNanos() - (System.nanoTime() - started);
|
||||
return remaining <= 0L ? Duration.ZERO : Duration.ofNanos(remaining);
|
||||
}
|
||||
|
||||
private String tenantKey(SourceId sourceId) {
|
||||
String value = sourceId == null ? "" : sourceId.value();
|
||||
int marker = value.lastIndexOf("-source-");
|
||||
return marker <= 0 ? value : value.substring(0, marker);
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取或创建分层准入项并登记一个活动查询或等待者。
|
||||
*
|
||||
* @param admissions 准入项映射
|
||||
* @param key 租户或数据源键
|
||||
* @param maximumConcurrentQueries 并发上限
|
||||
* @param <K> 键类型
|
||||
* @return 已登记引用的准入项
|
||||
*/
|
||||
private <K> AdmissionEntry retain(
|
||||
Map<K, AdmissionEntry> admissions,
|
||||
K key,
|
||||
int maximumConcurrentQueries) {
|
||||
AdmissionEntry entry = admissions.computeIfAbsent(
|
||||
key,
|
||||
ignored -> new AdmissionEntry(
|
||||
new LocalFederationQueryAdmissionController(
|
||||
maximumConcurrentQueries)));
|
||||
entry.references++;
|
||||
return entry;
|
||||
}
|
||||
|
||||
/**
|
||||
* 同步释放数据源与租户准入项引用。
|
||||
*
|
||||
* @param sourceId 数据源标识
|
||||
* @param sourceEntry 数据源准入项
|
||||
* @param tenantKey 租户键
|
||||
* @param tenantEntry 租户准入项
|
||||
*/
|
||||
private void releaseEntries(
|
||||
SourceId sourceId,
|
||||
AdmissionEntry sourceEntry,
|
||||
String tenantKey,
|
||||
AdmissionEntry tenantEntry) {
|
||||
synchronized (lifecycleLock) {
|
||||
release(sourceAdmissions, sourceId, sourceEntry);
|
||||
release(tenantAdmissions, tenantKey, tenantEntry);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 释放一个准入项引用,并在无查询或等待者时移除并关闭该项。
|
||||
*
|
||||
* @param admissions 准入项映射
|
||||
* @param key 租户或数据源键
|
||||
* @param entry 待释放准入项
|
||||
* @param <K> 键类型
|
||||
*/
|
||||
private <K> void release(
|
||||
Map<K, AdmissionEntry> admissions,
|
||||
K key,
|
||||
AdmissionEntry entry) {
|
||||
entry.references--;
|
||||
if (entry.references < 0) {
|
||||
throw new IllegalStateException(
|
||||
"query admission reference count became negative");
|
||||
}
|
||||
if (entry.references == 0 && admissions.remove(key, entry)) {
|
||||
entry.controller.close();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 返回当前仍有查询或等待者的数据源准入项数量。
|
||||
*
|
||||
* @return 活动数据源准入项数量
|
||||
*/
|
||||
int trackedSourceAdmissionCount() {
|
||||
return sourceAdmissions.size();
|
||||
}
|
||||
|
||||
/**
|
||||
* 返回当前仍有查询或等待者的租户准入项数量。
|
||||
*
|
||||
* @return 活动租户准入项数量
|
||||
*/
|
||||
int trackedTenantAdmissionCount() {
|
||||
return tenantAdmissions.size();
|
||||
}
|
||||
|
||||
/**
|
||||
* 返回指定数据源当前活动查询与等待者的引用总数。
|
||||
*
|
||||
* @param sourceId 数据源标识
|
||||
* @return 当前引用数
|
||||
*/
|
||||
int trackedSourceReferenceCount(SourceId sourceId) {
|
||||
synchronized (lifecycleLock) {
|
||||
AdmissionEntry entry = sourceAdmissions.get(sourceId);
|
||||
return entry == null ? 0 : entry.references;
|
||||
}
|
||||
}
|
||||
|
||||
private void ensureOpen() {
|
||||
if (closed.get()) {
|
||||
throw new FederationSqlException(
|
||||
FederationSqlErrorCode.ENGINE_CLOSED,
|
||||
"datacenter query admission controller is closed");
|
||||
}
|
||||
}
|
||||
|
||||
private void closeQuietly(FederationQueryPermit permit) {
|
||||
if (permit != null) {
|
||||
permit.close();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 共享公平信号量及其活动查询、等待者引用计数。
|
||||
*/
|
||||
private static final class AdmissionEntry {
|
||||
|
||||
private final LocalFederationQueryAdmissionController controller;
|
||||
private int references;
|
||||
|
||||
/**
|
||||
* 创建准入项。
|
||||
*
|
||||
* @param controller 本地公平准入控制器
|
||||
*/
|
||||
private AdmissionEntry(LocalFederationQueryAdmissionController controller) {
|
||||
this.controller = controller;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,70 @@
|
||||
package tech.easyflow.datacenter.federation;
|
||||
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.beans.factory.ObjectProvider;
|
||||
import org.springframework.data.redis.core.StringRedisTemplate;
|
||||
import org.springframework.stereotype.Component;
|
||||
import org.springframework.transaction.support.TransactionSynchronization;
|
||||
import org.springframework.transaction.support.TransactionSynchronizationManager;
|
||||
import tech.easyflow.datacenter.meta.entity.DatacenterSource;
|
||||
|
||||
/**
|
||||
* 发布不含凭据的 Federation Definition 变更提示。
|
||||
*/
|
||||
@Component
|
||||
public class DatacenterFederationChangeNotifier {
|
||||
|
||||
/** Redis Pub/Sub 频道。 */
|
||||
public static final String CHANNEL = "easyflow:datacenter:federation-source-changed";
|
||||
|
||||
private static final Logger log = LoggerFactory.getLogger(DatacenterFederationChangeNotifier.class);
|
||||
|
||||
private final ObjectProvider<StringRedisTemplate> redisTemplateProvider;
|
||||
private final DatacenterFederationDefinitionFactory definitionFactory;
|
||||
|
||||
/**
|
||||
* 创建通知器。
|
||||
*
|
||||
* @param redisTemplateProvider 可选 Redis 模板
|
||||
* @param definitionFactory Definition 工厂
|
||||
*/
|
||||
public DatacenterFederationChangeNotifier(
|
||||
ObjectProvider<StringRedisTemplate> redisTemplateProvider,
|
||||
DatacenterFederationDefinitionFactory definitionFactory) {
|
||||
this.redisTemplateProvider = redisTemplateProvider;
|
||||
this.definitionFactory = definitionFactory;
|
||||
}
|
||||
|
||||
/**
|
||||
* 在当前事务提交后发布 SourceId;无 Redis 时不影响数据库权威状态。
|
||||
*
|
||||
* @param source 已提交或即将提交的数据源
|
||||
*/
|
||||
public void publishAfterCommit(DatacenterSource source) {
|
||||
String sourceId = definitionFactory.sourceId(source).value();
|
||||
if (TransactionSynchronizationManager.isSynchronizationActive()) {
|
||||
TransactionSynchronizationManager.registerSynchronization(new TransactionSynchronization() {
|
||||
@Override
|
||||
public void afterCommit() {
|
||||
publish(sourceId);
|
||||
}
|
||||
});
|
||||
return;
|
||||
}
|
||||
publish(sourceId);
|
||||
}
|
||||
|
||||
private void publish(String sourceId) {
|
||||
StringRedisTemplate redisTemplate = redisTemplateProvider.getIfAvailable();
|
||||
if (redisTemplate == null) {
|
||||
return;
|
||||
}
|
||||
try {
|
||||
redisTemplate.convertAndSend(CHANNEL, sourceId);
|
||||
} catch (RuntimeException exception) {
|
||||
// Redis 仅缩短传播时间;数据库按需读取继续保证 minimumRevision 正确性。
|
||||
log.warn("Failed to publish datacenter source change hint for {}", sourceId, exception);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,64 @@
|
||||
package tech.easyflow.datacenter.federation;
|
||||
|
||||
import com.zaxxer.hikari.HikariConfig;
|
||||
import com.zaxxer.hikari.HikariDataSource;
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
import org.springframework.stereotype.Component;
|
||||
import tech.easyflow.datacenter.meta.entity.DatacenterSource;
|
||||
import tech.easyflow.datacenter.security.DatacenterCredentialCipher;
|
||||
|
||||
/**
|
||||
* 创建由 Federation Runtime 独占的只读 HikariCP 连接池。
|
||||
*/
|
||||
@Component
|
||||
public class DatacenterFederationDataSourceFactory {
|
||||
|
||||
private final DatacenterCredentialCipher credentialCipher;
|
||||
private final int maximumPoolSize;
|
||||
|
||||
/**
|
||||
* 创建数据源工厂。
|
||||
*
|
||||
* @param credentialCipher 凭据解密器
|
||||
* @param maximumPoolSize 单数据源连接池上限,与单源查询准入共用配置
|
||||
*/
|
||||
public DatacenterFederationDataSourceFactory(
|
||||
DatacenterCredentialCipher credentialCipher,
|
||||
@Value("${easyflow.datacenter.connection-pool.maximum-size:10}")
|
||||
int maximumPoolSize) {
|
||||
if (maximumPoolSize <= 0) {
|
||||
throw new IllegalArgumentException(
|
||||
"datacenter connection pool maximum size must be positive");
|
||||
}
|
||||
this.credentialCipher = credentialCipher;
|
||||
this.maximumPoolSize = maximumPoolSize;
|
||||
}
|
||||
|
||||
/**
|
||||
* 根据持久化数据源快照创建只读池。
|
||||
*
|
||||
* @param source 数据源快照
|
||||
* @return 未共享的 HikariCP 池
|
||||
*/
|
||||
public HikariDataSource create(DatacenterSource source) {
|
||||
HikariConfig config = new HikariConfig();
|
||||
config.setPoolName("dc-fed-" + source.getId() + "-r" + source.getDefinitionRevision());
|
||||
config.setJdbcUrl(source.getJdbcUrl());
|
||||
config.setUsername(source.getUsername());
|
||||
config.setPassword(credentialCipher.decrypt(source.getCredentialCipher()));
|
||||
if (source.getDriverClassName() != null && !source.getDriverClassName().isBlank()) {
|
||||
config.setDriverClassName(source.getDriverClassName());
|
||||
}
|
||||
config.setReadOnly(true);
|
||||
config.setAutoCommit(true);
|
||||
config.setMaximumPoolSize(maximumPoolSize);
|
||||
config.setMinimumIdle(Integer.getInteger("easyflow.datacenter.pool.minimum-idle", 0));
|
||||
config.setConnectionTimeout(Long.getLong("easyflow.datacenter.pool.connection-timeout-ms", 5_000L));
|
||||
config.setValidationTimeout(Long.getLong("easyflow.datacenter.pool.validation-timeout-ms", 3_000L));
|
||||
config.setIdleTimeout(Long.getLong("easyflow.datacenter.pool.idle-timeout-ms", 600_000L));
|
||||
config.setMaxLifetime(Long.getLong("easyflow.datacenter.pool.max-lifetime-ms", 1_800_000L));
|
||||
config.setInitializationFailTimeout(
|
||||
Long.getLong("easyflow.datacenter.pool.initialization-timeout-ms", 5_000L));
|
||||
return new HikariDataSource(config);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,140 @@
|
||||
package tech.easyflow.datacenter.federation;
|
||||
|
||||
import com.easyagents.federation.sql.source.FederationDataSourceHandle;
|
||||
import com.easyagents.federation.sql.source.FederationDataSourceHandles;
|
||||
import com.easyagents.federation.sql.source.FederationDataSourceResolver;
|
||||
import com.easyagents.federation.sql.source.FederationSourceDefinition;
|
||||
import com.easyagents.federation.sql.source.RuntimeFingerprint;
|
||||
import com.zaxxer.hikari.HikariDataSource;
|
||||
import java.sql.Connection;
|
||||
import java.sql.DatabaseMetaData;
|
||||
import java.sql.SQLException;
|
||||
import java.util.Objects;
|
||||
import java.util.function.Supplier;
|
||||
import org.springframework.stereotype.Component;
|
||||
import tech.easyflow.datacenter.mapper.DatacenterSourceMapper;
|
||||
import tech.easyflow.datacenter.meta.entity.DatacenterSource;
|
||||
import tech.easyflow.datacenter.meta.enums.DatacenterSourceStatus;
|
||||
|
||||
/**
|
||||
* 从 EasyFlow 权威配置解析节点本地 Federation DataSource Handle。
|
||||
*/
|
||||
@Component
|
||||
public class DatacenterFederationDataSourceResolver implements FederationDataSourceResolver {
|
||||
|
||||
private final DatacenterSourceMapper sourceMapper;
|
||||
private final DatacenterFederationDefinitionFactory definitionFactory;
|
||||
private final DatacenterFederationDataSourceFactory dataSourceFactory;
|
||||
private final ThreadLocal<CandidateResolution> candidateSource = new ThreadLocal<>();
|
||||
|
||||
/**
|
||||
* 创建 Resolver。
|
||||
*
|
||||
* @param sourceMapper 数据源 Mapper
|
||||
* @param definitionFactory Definition 工厂
|
||||
* @param dataSourceFactory 连接池工厂
|
||||
*/
|
||||
public DatacenterFederationDataSourceResolver(
|
||||
DatacenterSourceMapper sourceMapper,
|
||||
DatacenterFederationDefinitionFactory definitionFactory,
|
||||
DatacenterFederationDataSourceFactory dataSourceFactory) {
|
||||
this.sourceMapper = sourceMapper;
|
||||
this.definitionFactory = definitionFactory;
|
||||
this.dataSourceFactory = dataSourceFactory;
|
||||
}
|
||||
|
||||
/**
|
||||
* 创建由 Runtime 独占并在 revision 淘汰后关闭的 Handle。
|
||||
*
|
||||
* @param definition 无凭据 Definition
|
||||
* @return 独占 Handle
|
||||
*/
|
||||
@Override
|
||||
public FederationDataSourceHandle resolve(FederationSourceDefinition definition) {
|
||||
CandidateResolution candidate = candidateSource.get();
|
||||
DatacenterSource source = candidate != null
|
||||
&& Objects.equals(candidate.source().getId(), definitionFactory.sourceRecordId(definition))
|
||||
? candidate.source() : sourceMapper.selectOneById(definitionFactory.sourceRecordId(definition));
|
||||
if (source == null) {
|
||||
throw new IllegalStateException("data source no longer exists");
|
||||
}
|
||||
DatacenterSourceStatus status = DatacenterSourceStatus.fromCode(source.getStatus());
|
||||
boolean draftProbe = status == DatacenterSourceStatus.DRAFT;
|
||||
if (!draftProbe
|
||||
&& status != DatacenterSourceStatus.READY
|
||||
&& status != DatacenterSourceStatus.DEGRADED) {
|
||||
throw new IllegalStateException("data source cannot be resolved in its current state");
|
||||
}
|
||||
if (!definition.sourceId().equals(definitionFactory.sourceId(source))) {
|
||||
throw new IllegalStateException("data source tenant identity does not match definition");
|
||||
}
|
||||
long revision = source.getDefinitionRevision() == null ? 0L : source.getDefinitionRevision();
|
||||
if (revision != definition.revision()) {
|
||||
throw new IllegalStateException("data source revision changed while runtime was resolving");
|
||||
}
|
||||
FederationSourceDefinition currentDefinition = candidate != null
|
||||
? candidate.definition()
|
||||
: definitionFactory.create(source);
|
||||
if (!currentDefinition.checksum().equals(definition.checksum())) {
|
||||
throw new IllegalStateException("data source definition checksum mismatch");
|
||||
}
|
||||
// 草稿只通过 probe 创建短生命周期连接池,尚未发布持久化 checksum;
|
||||
// READY/DEGRADED Runtime 必须同时匹配已发布 checksum。
|
||||
if (!draftProbe && (source.getDefinitionChecksum() == null
|
||||
|| !source.getDefinitionChecksum().equals(definition.checksum()))) {
|
||||
throw new IllegalStateException("published data source definition checksum mismatch");
|
||||
}
|
||||
HikariDataSource pool = dataSourceFactory.create(source);
|
||||
try (Connection connection = pool.getConnection()) {
|
||||
DatabaseMetaData metadata = connection.getMetaData();
|
||||
RuntimeFingerprint fingerprint = new RuntimeFingerprint(
|
||||
metadata.getDatabaseProductName(),
|
||||
metadata.getDatabaseProductVersion(),
|
||||
metadata.getDriverName(),
|
||||
metadata.getDriverVersion(),
|
||||
"easyflow-xl01-wp1");
|
||||
return FederationDataSourceHandles.owned(pool, fingerprint, pool::close);
|
||||
} catch (SQLException | RuntimeException exception) {
|
||||
try {
|
||||
pool.close();
|
||||
} catch (RuntimeException closeException) {
|
||||
exception.addSuppressed(closeException);
|
||||
}
|
||||
throw new IllegalStateException("failed to initialize data source runtime", exception);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 在当前线程内使用未发布的候选配置完成一次 XL15 probe。
|
||||
*
|
||||
* <p>候选配置不会进入共享状态、Redis 或持久化记录,并在操作结束后立即移除。</p>
|
||||
*
|
||||
* @param source 已完成凭据加密和连接归一化的候选配置
|
||||
* @param action 需要解析候选配置的同步操作
|
||||
* @param <T> 操作结果类型
|
||||
* @return 操作结果
|
||||
* @throws IllegalStateException 当前线程已经存在候选配置时抛出
|
||||
*/
|
||||
public <T> T withCandidate(
|
||||
DatacenterSource source,
|
||||
FederationSourceDefinition definition,
|
||||
Supplier<T> action) {
|
||||
Objects.requireNonNull(source, "source");
|
||||
Objects.requireNonNull(definition, "definition");
|
||||
Objects.requireNonNull(action, "action");
|
||||
if (candidateSource.get() != null) {
|
||||
throw new IllegalStateException("nested data source candidate resolution is not supported");
|
||||
}
|
||||
candidateSource.set(new CandidateResolution(source, definition));
|
||||
try {
|
||||
return action.get();
|
||||
} finally {
|
||||
candidateSource.remove();
|
||||
}
|
||||
}
|
||||
|
||||
private record CandidateResolution(
|
||||
DatacenterSource source,
|
||||
FederationSourceDefinition definition) {
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,208 @@
|
||||
package tech.easyflow.datacenter.federation;
|
||||
|
||||
import com.easyagents.federation.sql.adapter.jdbc.JdbcFederationSqlAdapterProvider;
|
||||
import com.easyagents.federation.sql.adapter.jdbc.JdbcSchemaDefinition;
|
||||
import com.easyagents.federation.sql.source.FederationSourceDefinition;
|
||||
import com.easyagents.federation.sql.source.SourceId;
|
||||
import com.mybatisflex.core.query.QueryWrapper;
|
||||
import java.math.BigInteger;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collection;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.stream.Collectors;
|
||||
import org.springframework.stereotype.Component;
|
||||
import tech.easyflow.datacenter.mapper.DatacenterCatalogMapper;
|
||||
import tech.easyflow.datacenter.meta.entity.DatacenterCatalog;
|
||||
import tech.easyflow.datacenter.meta.entity.DatacenterSource;
|
||||
import tech.easyflow.datacenter.meta.enums.DatacenterMetadataStatus;
|
||||
import tech.easyflow.datacenter.meta.enums.DatacenterSourceType;
|
||||
|
||||
/**
|
||||
* 将 EasyFlow 持久化数据源快照转换为无凭据 Federation Definition。
|
||||
*/
|
||||
@Component
|
||||
public class DatacenterFederationDefinitionFactory {
|
||||
|
||||
/** Definition 中用于解析业务数据源主键的选项。 */
|
||||
public static final String SOURCE_RECORD_ID_OPTION = "sourceRecordId";
|
||||
|
||||
private final DatacenterCatalogMapper catalogMapper;
|
||||
|
||||
/**
|
||||
* 创建 Definition 工厂。
|
||||
*
|
||||
* @param catalogMapper 命名空间 Mapper
|
||||
*/
|
||||
public DatacenterFederationDefinitionFactory(DatacenterCatalogMapper catalogMapper) {
|
||||
this.catalogMapper = catalogMapper;
|
||||
}
|
||||
|
||||
/**
|
||||
* 创建用于探测或执行的 Definition。
|
||||
*
|
||||
* @param source 数据源当前快照
|
||||
* @return 无凭据 Definition
|
||||
* @throws IllegalArgumentException 数据源不是当前 JDBC 支持类型
|
||||
*/
|
||||
public FederationSourceDefinition create(DatacenterSource source) {
|
||||
return create(source, loadCatalogs(source));
|
||||
}
|
||||
|
||||
/**
|
||||
* 为尚未发布的连接候选创建只包含默认命名空间的 Definition。
|
||||
*
|
||||
* @param source 候选数据源
|
||||
* @return 不读取旧持久化 Catalog 的候选 Definition
|
||||
*/
|
||||
public FederationSourceDefinition createCandidate(DatacenterSource source) {
|
||||
return create(source, List.of());
|
||||
}
|
||||
|
||||
/**
|
||||
* 为即将发布的单命名空间配置创建与持久化结果一致的 Definition。
|
||||
*
|
||||
* @param source 候选数据源
|
||||
* @param catalogName 即将纳管的物理 Catalog 或 Schema
|
||||
* @return 不读取旧 Catalog 的候选 Definition
|
||||
*/
|
||||
public FederationSourceDefinition createCandidate(
|
||||
DatacenterSource source,
|
||||
String catalogName) {
|
||||
if (catalogName == null || catalogName.isBlank()) {
|
||||
throw new IllegalArgumentException("candidate catalog name is required");
|
||||
}
|
||||
String normalized = catalogName.trim();
|
||||
DatacenterCatalog catalog = new DatacenterCatalog();
|
||||
catalog.setLogicalSchemaName(normalized);
|
||||
if (DatacenterSourceType.MYSQL.name().equals(source.getSourceType())) {
|
||||
catalog.setPhysicalCatalogName(normalized);
|
||||
} else {
|
||||
catalog.setPhysicalCatalogName(source.getDatabaseName());
|
||||
catalog.setPhysicalSchemaName(normalized);
|
||||
}
|
||||
return create(source, List.of(catalog));
|
||||
}
|
||||
|
||||
/**
|
||||
* 批量创建 Definition,一次加载全部活动 Catalog,供周期 reconcile 避免 N+1。
|
||||
*
|
||||
* @param sources 数据源快照
|
||||
* @return 按数据源记录 ID 索引的 Definition
|
||||
*/
|
||||
public Map<BigInteger, FederationSourceDefinition> createBatch(
|
||||
Collection<DatacenterSource> sources) {
|
||||
List<DatacenterSource> sourceList = sources == null
|
||||
? List.of() : sources.stream().filter(java.util.Objects::nonNull).toList();
|
||||
if (sourceList.isEmpty()) {
|
||||
return Map.of();
|
||||
}
|
||||
List<BigInteger> sourceIds = sourceList.stream()
|
||||
.map(DatacenterSource::getId)
|
||||
.filter(java.util.Objects::nonNull)
|
||||
.toList();
|
||||
Map<BigInteger, List<DatacenterCatalog>> catalogsBySource = sourceIds.isEmpty()
|
||||
? Map.of()
|
||||
: catalogMapper.selectListByQuery(QueryWrapper.create()
|
||||
.in(DatacenterCatalog::getSourceId, sourceIds)
|
||||
.eq(DatacenterCatalog::getMetadataStatus, DatacenterMetadataStatus.ACTIVE.name())
|
||||
.orderBy("created asc"))
|
||||
.stream()
|
||||
.collect(Collectors.groupingBy(DatacenterCatalog::getSourceId));
|
||||
Map<BigInteger, FederationSourceDefinition> definitions = new LinkedHashMap<>();
|
||||
for (DatacenterSource source : sourceList) {
|
||||
definitions.put(source.getId(), create(
|
||||
source, catalogsBySource.getOrDefault(source.getId(), List.of())));
|
||||
}
|
||||
return Map.copyOf(definitions);
|
||||
}
|
||||
|
||||
private FederationSourceDefinition create(
|
||||
DatacenterSource source,
|
||||
List<DatacenterCatalog> catalogs) {
|
||||
requireExternalJdbc(source);
|
||||
long revision = Math.max(1L, source.getDefinitionRevision() == null
|
||||
? 0L : source.getDefinitionRevision());
|
||||
List<JdbcSchemaDefinition> schemas = schemas(source, catalogs);
|
||||
Map<String, String> options = new LinkedHashMap<>();
|
||||
if (source.getAdapterOptionsJson() != null) {
|
||||
options.putAll(source.getAdapterOptionsJson());
|
||||
}
|
||||
options.put(SOURCE_RECORD_ID_OPTION, source.getId().toString());
|
||||
return new FederationSourceDefinition(
|
||||
sourceId(source),
|
||||
revision,
|
||||
source.getAdapterId() == null || source.getAdapterId().isBlank()
|
||||
? JdbcFederationSqlAdapterProvider.ADAPTER_ID
|
||||
: source.getAdapterId(),
|
||||
new ArrayList<>(schemas),
|
||||
options);
|
||||
}
|
||||
|
||||
/**
|
||||
* 派生租户隔离且不随显示名称变化的 SourceId。
|
||||
*
|
||||
* @param source 数据源
|
||||
* @return Federation SourceId
|
||||
*/
|
||||
public SourceId sourceId(DatacenterSource source) {
|
||||
BigInteger tenantId = source.getTenantId() == null ? BigInteger.ZERO : source.getTenantId();
|
||||
return new SourceId("tenant-" + tenantId + "-source-" + source.getId());
|
||||
}
|
||||
|
||||
/**
|
||||
* 从 Definition 读取 EasyFlow 数据源主键。
|
||||
*
|
||||
* @param definition Definition
|
||||
* @return 数据源主键
|
||||
*/
|
||||
public BigInteger sourceRecordId(FederationSourceDefinition definition) {
|
||||
String value = definition.adapterOptions().get(SOURCE_RECORD_ID_OPTION);
|
||||
if (value == null || value.isBlank()) {
|
||||
throw new IllegalArgumentException("sourceRecordId is missing from Federation Definition");
|
||||
}
|
||||
return new BigInteger(value);
|
||||
}
|
||||
|
||||
private List<DatacenterCatalog> loadCatalogs(DatacenterSource source) {
|
||||
QueryWrapper query = QueryWrapper.create()
|
||||
.eq(DatacenterCatalog::getSourceId, source.getId())
|
||||
.eq(DatacenterCatalog::getMetadataStatus, DatacenterMetadataStatus.ACTIVE.name())
|
||||
.orderBy("created asc");
|
||||
return catalogMapper.selectListByQuery(query);
|
||||
}
|
||||
|
||||
private List<JdbcSchemaDefinition> schemas(
|
||||
DatacenterSource source,
|
||||
List<DatacenterCatalog> catalogs) {
|
||||
if (catalogs.isEmpty()) {
|
||||
return List.of(defaultSchema(source));
|
||||
}
|
||||
return catalogs.stream()
|
||||
.map(catalog -> new JdbcSchemaDefinition(
|
||||
catalog.getLogicalSchemaName(),
|
||||
catalog.getPhysicalCatalogName(),
|
||||
catalog.getPhysicalSchemaName()))
|
||||
.toList();
|
||||
}
|
||||
|
||||
private JdbcSchemaDefinition defaultSchema(DatacenterSource source) {
|
||||
if (DatacenterSourceType.MYSQL.name().equals(source.getSourceType())) {
|
||||
return new JdbcSchemaDefinition("MAIN", source.getDatabaseName(), null);
|
||||
}
|
||||
String schema = source.getSchemaName() == null || source.getSchemaName().isBlank()
|
||||
? "public" : source.getSchemaName();
|
||||
return new JdbcSchemaDefinition("MAIN", source.getDatabaseName(), schema);
|
||||
}
|
||||
|
||||
private void requireExternalJdbc(DatacenterSource source) {
|
||||
if (source == null || source.getId() == null) {
|
||||
throw new IllegalArgumentException("persisted source is required");
|
||||
}
|
||||
if (!DatacenterSourceType.MYSQL.name().equals(source.getSourceType())
|
||||
&& !DatacenterSourceType.POSTGRESQL.name().equals(source.getSourceType())) {
|
||||
throw new IllegalArgumentException("Federation JDBC currently supports MySQL and PostgreSQL sources");
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,292 @@
|
||||
package tech.easyflow.datacenter.federation;
|
||||
|
||||
import com.easyagents.federation.sql.execute.QueryId;
|
||||
import java.math.BigInteger;
|
||||
import java.time.Duration;
|
||||
import java.util.UUID;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
import java.util.concurrent.atomic.AtomicBoolean;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.beans.factory.ObjectProvider;
|
||||
import org.springframework.data.redis.core.StringRedisTemplate;
|
||||
import org.springframework.scheduling.annotation.Scheduled;
|
||||
import org.springframework.stereotype.Component;
|
||||
import tech.easyflow.common.entity.LoginAccount;
|
||||
import tech.easyflow.common.web.exceptions.BusinessException;
|
||||
|
||||
/**
|
||||
* 管理查询从编译到游标关闭的取消状态,并通过 Redis 将取消提示广播到执行节点。
|
||||
*/
|
||||
@Component
|
||||
public class DatacenterFederationQueryCancellationService {
|
||||
|
||||
/** 查询取消 Redis Pub/Sub 频道。 */
|
||||
public static final String CHANNEL = "easyflow:datacenter:federation-query-cancel";
|
||||
private static final String CANCEL_KEY_PREFIX =
|
||||
"easyflow:datacenter:federation-query-cancelled:";
|
||||
private static final Duration CANCEL_TTL = Duration.ofMinutes(10);
|
||||
|
||||
private static final Logger log = LoggerFactory.getLogger(
|
||||
DatacenterFederationQueryCancellationService.class);
|
||||
|
||||
private final DatacenterFederationRuntime runtime;
|
||||
private final ObjectProvider<StringRedisTemplate> redisTemplateProvider;
|
||||
private final ConcurrentHashMap<String, ActiveQuery> activeQueries = new ConcurrentHashMap<>();
|
||||
|
||||
/**
|
||||
* 创建查询取消服务。
|
||||
*
|
||||
* @param runtime 节点本地 Federation Runtime
|
||||
* @param redisTemplateProvider 可选 Redis 模板
|
||||
*/
|
||||
public DatacenterFederationQueryCancellationService(
|
||||
DatacenterFederationRuntime runtime,
|
||||
ObjectProvider<StringRedisTemplate> redisTemplateProvider) {
|
||||
this.runtime = runtime;
|
||||
this.redisTemplateProvider = redisTemplateProvider;
|
||||
}
|
||||
|
||||
/**
|
||||
* 校验或创建查询标识。
|
||||
*
|
||||
* @param requestedQueryId 调用方预生成的 UUID;为空时由服务端生成
|
||||
* @return 已校验的查询标识
|
||||
* @throws BusinessException 标识格式不合法
|
||||
*/
|
||||
public QueryId resolveQueryId(String requestedQueryId) {
|
||||
if (requestedQueryId == null || requestedQueryId.isBlank()) {
|
||||
return QueryId.create();
|
||||
}
|
||||
try {
|
||||
return new QueryId(UUID.fromString(requestedQueryId.trim()).toString());
|
||||
} catch (IllegalArgumentException exception) {
|
||||
throw new BusinessException("queryId 必须是 UUID");
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 在编译前登记查询,防止重复 QueryId 混淆取消目标。
|
||||
*
|
||||
* @param queryId 查询标识
|
||||
* @param account 执行账号
|
||||
* @return 必须关闭的登记句柄
|
||||
* @throws BusinessException QueryId 正在使用
|
||||
*/
|
||||
public QueryRegistration register(QueryId queryId, LoginAccount account) {
|
||||
ActiveQuery activeQuery = new ActiveQuery(tenantId(account));
|
||||
if (activeQueries.putIfAbsent(queryId.value(), activeQuery) != null) {
|
||||
throw new BusinessException(409, 14123, "queryId 正在使用", null);
|
||||
}
|
||||
if (isCancellationPersisted(activeQuery.tenantId, queryId)) {
|
||||
try {
|
||||
cancelLocal(activeQuery.tenantId, queryId);
|
||||
} catch (RuntimeException exception) {
|
||||
activeQueries.remove(queryId.value(), activeQuery);
|
||||
throw exception;
|
||||
}
|
||||
}
|
||||
return new QueryRegistration(queryId, activeQuery);
|
||||
}
|
||||
|
||||
/**
|
||||
* 向所有节点广播取消提示,并立即取消当前节点已登记的 Statement。
|
||||
*
|
||||
* @param requestedQueryId 查询标识
|
||||
* @param account 当前账号
|
||||
* @return 当前节点已找到查询或 Redis 已接收广播
|
||||
*/
|
||||
public boolean cancel(String requestedQueryId, LoginAccount account) {
|
||||
if (requestedQueryId == null || requestedQueryId.isBlank()) {
|
||||
throw new BusinessException("queryId 不能为空");
|
||||
}
|
||||
QueryId queryId = resolveQueryId(requestedQueryId);
|
||||
BigInteger tenantId = tenantId(account);
|
||||
boolean distributedCancellationPersisted = persistAndPublishCancellation(
|
||||
tenantId, queryId);
|
||||
boolean localCancelled;
|
||||
try {
|
||||
localCancelled = cancelLocal(tenantId, queryId);
|
||||
} catch (RuntimeException exception) {
|
||||
log.error("Failed to cancel local datacenter query for tenant {} query {}",
|
||||
tenantId, queryId.value(), exception);
|
||||
if (!distributedCancellationPersisted) {
|
||||
throw exception;
|
||||
}
|
||||
localCancelled = false;
|
||||
}
|
||||
return localCancelled || distributedCancellationPersisted;
|
||||
}
|
||||
|
||||
/**
|
||||
* 先持久化取消终态,再尽力发布低延迟提示。
|
||||
*
|
||||
* @param tenantId 租户 ID
|
||||
* @param queryId 查询标识
|
||||
* @return Redis 是否已可靠保存取消终态
|
||||
*/
|
||||
private boolean persistAndPublishCancellation(
|
||||
BigInteger tenantId,
|
||||
QueryId queryId) {
|
||||
StringRedisTemplate redisTemplate = redisTemplateProvider.getIfAvailable();
|
||||
if (redisTemplate == null) {
|
||||
return false;
|
||||
}
|
||||
try {
|
||||
redisTemplate.opsForValue().set(
|
||||
cancellationKey(tenantId, queryId), "1", CANCEL_TTL);
|
||||
} catch (RuntimeException exception) {
|
||||
log.warn("Failed to persist datacenter query cancellation for tenant {} query {}",
|
||||
tenantId, queryId.value(), exception);
|
||||
return false;
|
||||
}
|
||||
try {
|
||||
redisTemplate.convertAndSend(
|
||||
CHANNEL, tenantId + "|" + queryId.value());
|
||||
} catch (RuntimeException exception) {
|
||||
// Pub/Sub 仅缩短生效延迟;持久取消终态已由周期核对保证交付。
|
||||
log.warn("Failed to publish datacenter query cancellation hint for tenant {} query {}",
|
||||
tenantId, queryId.value(), exception);
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* 处理 Redis 取消提示;格式无效时安全忽略。
|
||||
*
|
||||
* @param payload 租户与 QueryId 组合文本
|
||||
*/
|
||||
public void acceptCancellationHint(String payload) {
|
||||
if (payload == null) {
|
||||
return;
|
||||
}
|
||||
int separator = payload.indexOf('|');
|
||||
if (separator <= 0 || separator == payload.length() - 1) {
|
||||
return;
|
||||
}
|
||||
try {
|
||||
BigInteger tenantId = new BigInteger(payload.substring(0, separator));
|
||||
QueryId queryId = resolveQueryId(payload.substring(separator + 1));
|
||||
cancelLocal(tenantId, queryId);
|
||||
} catch (RuntimeException exception) {
|
||||
log.warn("Ignored invalid datacenter query cancellation hint", exception);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 在编译或取数边界检查查询是否已取消。
|
||||
*
|
||||
* @param queryId 查询标识
|
||||
* @throws BusinessException 查询已取消
|
||||
*/
|
||||
public void throwIfCancelled(QueryId queryId) {
|
||||
ActiveQuery activeQuery = activeQueries.get(queryId.value());
|
||||
if (activeQuery != null && activeQuery.cancelled.get()) {
|
||||
throw new BusinessException(409, 14118, "查询已取消", null);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 返回查询当前是否已收到取消请求。
|
||||
*
|
||||
* @param queryId 查询标识
|
||||
* @return 是否取消
|
||||
*/
|
||||
public boolean isCancelled(QueryId queryId) {
|
||||
ActiveQuery activeQuery = activeQueries.get(queryId.value());
|
||||
return activeQuery != null && activeQuery.cancelled.get();
|
||||
}
|
||||
|
||||
/**
|
||||
* 周期核对 Redis 持久取消标记,覆盖 Pub/Sub 提示丢失或节点短暂断连。
|
||||
*/
|
||||
@Scheduled(fixedDelayString =
|
||||
"${easyflow.datacenter.query-cancel-reconcile-ms:1000}")
|
||||
public void reconcilePersistedCancellations() {
|
||||
activeQueries.forEach((queryIdValue, activeQuery) -> {
|
||||
if (activeQuery.localCancellationApplied.get()) {
|
||||
return;
|
||||
}
|
||||
QueryId queryId = new QueryId(queryIdValue);
|
||||
if (activeQuery.cancelled.get()
|
||||
|| isCancellationPersisted(activeQuery.tenantId, queryId)) {
|
||||
try {
|
||||
cancelLocal(activeQuery.tenantId, queryId);
|
||||
} catch (RuntimeException exception) {
|
||||
log.error("Failed to reconcile datacenter query cancellation for tenant {} query {}",
|
||||
activeQuery.tenantId, queryId.value(), exception);
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
private boolean cancelLocal(BigInteger tenantId, QueryId queryId) {
|
||||
ActiveQuery activeQuery = activeQueries.get(queryId.value());
|
||||
if (activeQuery == null || !activeQuery.tenantId.equals(tenantId)) {
|
||||
return false;
|
||||
}
|
||||
activeQuery.cancelled.set(true);
|
||||
runtime.engine().cancel(queryId);
|
||||
activeQuery.localCancellationApplied.set(true);
|
||||
return true;
|
||||
}
|
||||
|
||||
private boolean isCancellationPersisted(BigInteger tenantId, QueryId queryId) {
|
||||
StringRedisTemplate redisTemplate = redisTemplateProvider.getIfAvailable();
|
||||
if (redisTemplate == null) {
|
||||
return false;
|
||||
}
|
||||
try {
|
||||
return Boolean.TRUE.equals(redisTemplate.hasKey(
|
||||
cancellationKey(tenantId, queryId)));
|
||||
} catch (RuntimeException exception) {
|
||||
log.warn("Failed to read datacenter query cancellation for tenant {} query {}",
|
||||
tenantId, queryId.value(), exception);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
private String cancellationKey(BigInteger tenantId, QueryId queryId) {
|
||||
return CANCEL_KEY_PREFIX + tenantId + ":" + queryId.value();
|
||||
}
|
||||
|
||||
private BigInteger tenantId(LoginAccount account) {
|
||||
return account == null || account.getTenantId() == null
|
||||
? BigInteger.ZERO : account.getTenantId();
|
||||
}
|
||||
|
||||
private record ActiveQuery(
|
||||
BigInteger tenantId,
|
||||
AtomicBoolean cancelled,
|
||||
AtomicBoolean localCancellationApplied) {
|
||||
|
||||
private ActiveQuery(BigInteger tenantId) {
|
||||
this(tenantId, new AtomicBoolean(), new AtomicBoolean());
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 查询活动登记句柄。
|
||||
*/
|
||||
public final class QueryRegistration implements AutoCloseable {
|
||||
|
||||
private final QueryId queryId;
|
||||
private final ActiveQuery activeQuery;
|
||||
private final AtomicBoolean closed = new AtomicBoolean();
|
||||
|
||||
private QueryRegistration(QueryId queryId, ActiveQuery activeQuery) {
|
||||
this.queryId = queryId;
|
||||
this.activeQuery = activeQuery;
|
||||
}
|
||||
|
||||
/**
|
||||
* 仅移除当前登记实例,避免误删随后复用的同名查询。
|
||||
*/
|
||||
@Override
|
||||
public void close() {
|
||||
if (closed.compareAndSet(false, true)) {
|
||||
activeQueries.remove(queryId.value(), activeQuery);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,731 @@
|
||||
package tech.easyflow.datacenter.federation;
|
||||
|
||||
import com.easyagents.federation.sql.api.FederationSqlException;
|
||||
import com.easyagents.federation.sql.api.SqlExecutionContext;
|
||||
import com.easyagents.federation.sql.compile.FederationSqlPlan;
|
||||
import com.easyagents.federation.sql.compile.SqlCompileRequest;
|
||||
import com.easyagents.federation.sql.execute.FederationColumn;
|
||||
import com.easyagents.federation.sql.execute.FederationResultCursor;
|
||||
import com.easyagents.federation.sql.execute.QueryId;
|
||||
import com.easyagents.federation.sql.execute.SqlExecutionOptions;
|
||||
import com.easyagents.federation.sql.execute.SqlParameter;
|
||||
import com.easyagents.federation.sql.federation.FederationQueryScopeDefinition;
|
||||
import com.easyagents.federation.sql.source.SourceId;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.io.Reader;
|
||||
import java.math.BigInteger;
|
||||
import java.sql.Blob;
|
||||
import java.sql.Clob;
|
||||
import java.sql.SQLXML;
|
||||
import java.sql.SQLException;
|
||||
import java.sql.Struct;
|
||||
import java.sql.Types;
|
||||
import java.time.Duration;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Base64;
|
||||
import java.util.Collections;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.LinkedHashSet;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.function.Consumer;
|
||||
import org.apache.calcite.rel.RelNode;
|
||||
import org.apache.calcite.rel.RelVisitor;
|
||||
import org.apache.calcite.rel.core.TableScan;
|
||||
import org.apache.calcite.sql.SqlIdentifier;
|
||||
import org.apache.calcite.sql.util.SqlBasicVisitor;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.stereotype.Service;
|
||||
import tech.easyflow.common.entity.LoginAccount;
|
||||
import tech.easyflow.common.web.exceptions.BusinessException;
|
||||
import tech.easyflow.datacenter.audit.DatacenterQueryAudit;
|
||||
import tech.easyflow.datacenter.audit.DatacenterQueryAuditService;
|
||||
import tech.easyflow.datacenter.execution.model.DatacenterSqlColumnView;
|
||||
import tech.easyflow.datacenter.execution.model.DatacenterSqlConsoleResult;
|
||||
import tech.easyflow.datacenter.meta.entity.DatacenterSource;
|
||||
import tech.easyflow.datacenter.meta.enums.DatacenterSourceStatus;
|
||||
|
||||
/**
|
||||
* 通过 Federation SQL 完成有界只读查询、资源关闭和统一审计。
|
||||
*/
|
||||
@Service
|
||||
public class DatacenterFederationQueryService {
|
||||
|
||||
private static final Logger log = LoggerFactory.getLogger(DatacenterFederationQueryService.class);
|
||||
private static final int DEFAULT_MAX_ROWS = 200;
|
||||
private static final int MAX_ALLOWED_ROWS = 1_000;
|
||||
private static final long MAX_RESULT_BYTES = 4L * 1024L * 1024L;
|
||||
private static final int MAX_TEXT_CELL_CHARS = 65_536;
|
||||
|
||||
private final DatacenterFederationRuntime runtime;
|
||||
private final DatacenterQueryAuditService auditService;
|
||||
private final DatacenterFederationQueryCancellationService cancellationService;
|
||||
|
||||
/**
|
||||
* 创建查询服务。
|
||||
*
|
||||
* @param runtime Federation Runtime
|
||||
* @param auditService 查询审计服务
|
||||
* @param cancellationService 跨节点查询取消服务
|
||||
*/
|
||||
public DatacenterFederationQueryService(
|
||||
DatacenterFederationRuntime runtime,
|
||||
DatacenterQueryAuditService auditService,
|
||||
DatacenterFederationQueryCancellationService cancellationService) {
|
||||
this.runtime = runtime;
|
||||
this.auditService = auditService;
|
||||
this.cancellationService = cancellationService;
|
||||
}
|
||||
|
||||
/**
|
||||
* 执行一条单数据源只读 SQL。
|
||||
*
|
||||
* @param source 已激活数据源
|
||||
* @param sql 只读 SQL
|
||||
* @param parameters 显式 JDBC 参数
|
||||
* @param requestedMaxRows 请求行数上限
|
||||
* @param account 执行账号
|
||||
* @param callerType 调用方类型
|
||||
* @param callerId 调用方标识
|
||||
* @return 有界查询结果
|
||||
*/
|
||||
public DatacenterSqlConsoleResult execute(
|
||||
DatacenterSource source,
|
||||
String sql,
|
||||
List<SqlParameter> parameters,
|
||||
Integer requestedMaxRows,
|
||||
LoginAccount account,
|
||||
String callerType,
|
||||
String callerId) {
|
||||
return execute(source, sql, parameters, requestedMaxRows, account,
|
||||
callerType, callerId, null);
|
||||
}
|
||||
|
||||
/**
|
||||
* 使用调用方预生成的 QueryId 执行只读 SQL,以支持请求完成前取消。
|
||||
*
|
||||
* @param source 已激活数据源
|
||||
* @param sql 只读 SQL
|
||||
* @param parameters 显式 JDBC 参数
|
||||
* @param requestedMaxRows 请求行数上限
|
||||
* @param account 执行账号
|
||||
* @param callerType 调用方类型
|
||||
* @param callerId 调用方标识
|
||||
* @param requestedQueryId 调用方预生成的查询 UUID;为空时服务端生成
|
||||
* @return 有界查询结果
|
||||
*/
|
||||
public DatacenterSqlConsoleResult execute(
|
||||
DatacenterSource source,
|
||||
String sql,
|
||||
List<SqlParameter> parameters,
|
||||
Integer requestedMaxRows,
|
||||
LoginAccount account,
|
||||
String callerType,
|
||||
String callerId,
|
||||
String requestedQueryId) {
|
||||
requireExecutable(source, sql);
|
||||
int maxRows = normalizeMaxRows(requestedMaxRows);
|
||||
List<SqlParameter> safeParameters = List.copyOf(parameters == null ? List.of() : parameters);
|
||||
QueryId queryId = cancellationService.resolveQueryId(requestedQueryId);
|
||||
DatacenterFederationQueryCancellationService.QueryRegistration registration =
|
||||
cancellationService.register(queryId, account);
|
||||
long startedNanos = System.nanoTime();
|
||||
DatacenterQueryAudit audit;
|
||||
try {
|
||||
audit = auditService.start(
|
||||
queryId.value(), source, sql, maskedParameterSummary(safeParameters),
|
||||
account, callerType, callerId);
|
||||
} catch (RuntimeException exception) {
|
||||
registration.close();
|
||||
throw exception;
|
||||
}
|
||||
List<String> referencedTables = List.of();
|
||||
List<String> referencedFields = List.of();
|
||||
try {
|
||||
cancellationService.throwIfCancelled(queryId);
|
||||
SqlCompileRequest compileRequest = compileRequest(source, sql, safeParameters);
|
||||
FederationSqlPlan plan = runtime.engine().compile(compileRequest);
|
||||
cancellationService.throwIfCancelled(queryId);
|
||||
referencedTables = referencedTables(plan.relRoot().rel);
|
||||
referencedFields = referencedFields(plan.sqlNode());
|
||||
SqlExecutionOptions options = new SqlExecutionOptions(
|
||||
200,
|
||||
maxRows + 1,
|
||||
30,
|
||||
true);
|
||||
long databaseStartedNanos = System.nanoTime();
|
||||
List<DatacenterSqlColumnView> columns;
|
||||
List<Map<String, Object>> rows = new ArrayList<>(maxRows);
|
||||
boolean truncated = false;
|
||||
long resultBytes = 0L;
|
||||
try (FederationResultCursor cursor = runtime.engine().execute(
|
||||
plan,
|
||||
new SqlExecutionContext(
|
||||
queryId,
|
||||
safeParameters,
|
||||
options,
|
||||
Duration.ofSeconds(5)))) {
|
||||
columns = columns(cursor.columns());
|
||||
while (cursor.next()) {
|
||||
cancellationService.throwIfCancelled(queryId);
|
||||
if (rows.size() >= maxRows) {
|
||||
truncated = true;
|
||||
break;
|
||||
}
|
||||
Map<String, Object> row = new LinkedHashMap<>();
|
||||
for (int index = 0; index < columns.size(); index++) {
|
||||
Object value = readColumn(cursor, cursor.columns().get(index), index + 1);
|
||||
resultBytes += estimateBytes(columns.get(index).label(), value);
|
||||
if (resultBytes > MAX_RESULT_BYTES) {
|
||||
truncated = true;
|
||||
break;
|
||||
}
|
||||
row.put(columns.get(index).key(), value);
|
||||
}
|
||||
if (truncated && resultBytes > MAX_RESULT_BYTES) {
|
||||
break;
|
||||
}
|
||||
rows.add(Collections.unmodifiableMap(new LinkedHashMap<>(row)));
|
||||
}
|
||||
}
|
||||
long databaseDurationMs = elapsedMillis(databaseStartedNanos);
|
||||
long durationMs = elapsedMillis(startedNanos);
|
||||
auditService.succeed(audit, referencedTables, referencedFields, rows.size(), truncated,
|
||||
durationMs, databaseDurationMs);
|
||||
return new DatacenterSqlConsoleResult(
|
||||
queryId.value(), columns, rows, rows.size(), truncated, durationMs);
|
||||
} catch (RuntimeException exception) {
|
||||
long durationMs = elapsedMillis(startedNanos);
|
||||
String errorCode = cancellationService.isCancelled(queryId)
|
||||
? "QUERY_CANCELLED"
|
||||
: exception instanceof FederationSqlException federationException
|
||||
? federationException.errorCode().name()
|
||||
: "DATACENTER_QUERY_FAILED";
|
||||
try {
|
||||
auditService.fail(audit, errorCode, safeMessage(exception),
|
||||
referencedTables, referencedFields, durationMs);
|
||||
} catch (RuntimeException auditException) {
|
||||
exception.addSuppressed(auditException);
|
||||
log.error("Failed to finalize datacenter query audit {}", queryId.value(), auditException);
|
||||
}
|
||||
if (exception instanceof BusinessException businessException) {
|
||||
throw businessException;
|
||||
}
|
||||
logUnexpectedFailure(queryId, source, exception);
|
||||
throw toBusinessException(exception);
|
||||
} finally {
|
||||
registration.close();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 使用 Federation 游标流式消费单数据源只读 SQL,并在触及资源上限时显式失败。
|
||||
*
|
||||
* @param source 已激活数据源
|
||||
* @param sql 只读 SQL
|
||||
* @param parameters 显式 JDBC 参数
|
||||
* @param fetchSize JDBC 拉取批次
|
||||
* @param maxRows 最大消费行数
|
||||
* @param maxBytes 最大估算字节数
|
||||
* @param account 执行账号
|
||||
* @param callerType 调用方类型
|
||||
* @param callerId 调用方标识
|
||||
* @param consumer 单行消费者
|
||||
* @return 已消费行数
|
||||
*/
|
||||
public long consume(
|
||||
DatacenterSource source,
|
||||
String sql,
|
||||
List<SqlParameter> parameters,
|
||||
int fetchSize,
|
||||
long maxRows,
|
||||
long maxBytes,
|
||||
LoginAccount account,
|
||||
String callerType,
|
||||
String callerId,
|
||||
Consumer<Map<String, Object>> consumer) {
|
||||
return consume(source, sql, parameters, fetchSize, maxRows, maxBytes,
|
||||
account, callerType, callerId, null, consumer);
|
||||
}
|
||||
|
||||
/**
|
||||
* 使用调用方预生成的 QueryId 流式消费只读 SQL,以支持流消费期间取消。
|
||||
*
|
||||
* @param source 已激活数据源
|
||||
* @param sql 只读 SQL
|
||||
* @param parameters 显式 JDBC 参数
|
||||
* @param fetchSize JDBC 拉取批次
|
||||
* @param maxRows 最大消费行数
|
||||
* @param maxBytes 最大估算字节数
|
||||
* @param account 执行账号
|
||||
* @param callerType 调用方类型
|
||||
* @param callerId 调用方标识
|
||||
* @param requestedQueryId 调用方预生成的查询 UUID;为空时服务端生成
|
||||
* @param consumer 单行消费者
|
||||
* @return 已消费行数
|
||||
*/
|
||||
public long consume(
|
||||
DatacenterSource source,
|
||||
String sql,
|
||||
List<SqlParameter> parameters,
|
||||
int fetchSize,
|
||||
long maxRows,
|
||||
long maxBytes,
|
||||
LoginAccount account,
|
||||
String callerType,
|
||||
String callerId,
|
||||
String requestedQueryId,
|
||||
Consumer<Map<String, Object>> consumer) {
|
||||
requireExecutable(source, sql);
|
||||
if (fetchSize <= 0 || maxRows <= 0 || maxRows >= Integer.MAX_VALUE
|
||||
|| maxBytes <= 0 || consumer == null) {
|
||||
throw new IllegalArgumentException("stream query limits and consumer must be valid");
|
||||
}
|
||||
List<SqlParameter> safeParameters = List.copyOf(parameters == null ? List.of() : parameters);
|
||||
QueryId queryId = cancellationService.resolveQueryId(requestedQueryId);
|
||||
DatacenterFederationQueryCancellationService.QueryRegistration registration =
|
||||
cancellationService.register(queryId, account);
|
||||
long startedNanos = System.nanoTime();
|
||||
DatacenterQueryAudit audit;
|
||||
try {
|
||||
audit = auditService.start(
|
||||
queryId.value(), source, sql,
|
||||
maskedParameterSummary(safeParameters),
|
||||
account, callerType, callerId);
|
||||
} catch (RuntimeException exception) {
|
||||
registration.close();
|
||||
throw exception;
|
||||
}
|
||||
List<String> referencedTables = List.of();
|
||||
List<String> referencedFields = List.of();
|
||||
long consumedRows = 0L;
|
||||
try {
|
||||
cancellationService.throwIfCancelled(queryId);
|
||||
SqlCompileRequest compileRequest = compileRequest(source, sql, safeParameters);
|
||||
FederationSqlPlan plan = runtime.engine().compile(compileRequest);
|
||||
cancellationService.throwIfCancelled(queryId);
|
||||
referencedTables = referencedTables(plan.relRoot().rel);
|
||||
referencedFields = referencedFields(plan.sqlNode());
|
||||
SqlExecutionOptions options = new SqlExecutionOptions(
|
||||
fetchSize, Math.toIntExact(maxRows + 1L), 30, true);
|
||||
long databaseStartedNanos = System.nanoTime();
|
||||
long resultBytes = 0L;
|
||||
try (FederationResultCursor cursor = runtime.engine().execute(
|
||||
plan,
|
||||
new SqlExecutionContext(queryId, safeParameters, options, Duration.ofSeconds(5)))) {
|
||||
List<DatacenterSqlColumnView> columns = columns(cursor.columns());
|
||||
while (cursor.next()) {
|
||||
cancellationService.throwIfCancelled(queryId);
|
||||
if (consumedRows >= maxRows) {
|
||||
throw new BusinessException("数据集查询结果超过行数上限: " + maxRows);
|
||||
}
|
||||
Map<String, Object> row = new LinkedHashMap<>();
|
||||
for (int index = 0; index < columns.size(); index++) {
|
||||
Object value = readColumn(cursor, cursor.columns().get(index), index + 1);
|
||||
resultBytes += estimateBytes(columns.get(index).label(), value);
|
||||
if (resultBytes > maxBytes) {
|
||||
throw new BusinessException("数据集查询结果超过字节上限: " + maxBytes);
|
||||
}
|
||||
row.put(columns.get(index).label(), value);
|
||||
}
|
||||
consumer.accept(Collections.unmodifiableMap(row));
|
||||
consumedRows++;
|
||||
}
|
||||
}
|
||||
long durationMs = elapsedMillis(startedNanos);
|
||||
auditService.succeed(
|
||||
audit, referencedTables, referencedFields, consumedRows, false,
|
||||
durationMs, elapsedMillis(databaseStartedNanos));
|
||||
return consumedRows;
|
||||
} catch (RuntimeException exception) {
|
||||
long durationMs = elapsedMillis(startedNanos);
|
||||
String errorCode = cancellationService.isCancelled(queryId)
|
||||
? "QUERY_CANCELLED"
|
||||
: exception instanceof FederationSqlException federationException
|
||||
? federationException.errorCode().name()
|
||||
: "DATACENTER_QUERY_FAILED";
|
||||
try {
|
||||
auditService.fail(audit, errorCode, safeMessage(exception),
|
||||
referencedTables, referencedFields, durationMs);
|
||||
} catch (RuntimeException auditException) {
|
||||
exception.addSuppressed(auditException);
|
||||
log.error("Failed to finalize datacenter stream query audit {}", queryId.value(), auditException);
|
||||
}
|
||||
if (exception instanceof BusinessException businessException) {
|
||||
throw businessException;
|
||||
}
|
||||
logUnexpectedFailure(queryId, source, exception);
|
||||
throw toBusinessException(exception);
|
||||
} finally {
|
||||
registration.close();
|
||||
}
|
||||
}
|
||||
|
||||
private SqlCompileRequest compileRequest(
|
||||
DatacenterSource source,
|
||||
String sql,
|
||||
List<SqlParameter> parameters) {
|
||||
SourceId sourceId = runtime.definitions().sourceId(source);
|
||||
return new SqlCompileRequest(
|
||||
sql,
|
||||
FederationQueryScopeDefinition.single(
|
||||
"datacenter-source:" + source.getId(),
|
||||
source.getScopeRevision(),
|
||||
sourceId.value(),
|
||||
sourceId,
|
||||
source.getDefinitionRevision()),
|
||||
parameters.stream().map(SqlParameter::jdbcType).toList(),
|
||||
"scope-" + source.getScopeRevision());
|
||||
}
|
||||
|
||||
private void requireExecutable(DatacenterSource source, String sql) {
|
||||
if (source == null || DatacenterSourceStatus.fromCode(source.getStatus())
|
||||
!= DatacenterSourceStatus.READY) {
|
||||
throw new BusinessException("数据连接尚未就绪");
|
||||
}
|
||||
if (sql == null || sql.isBlank()) {
|
||||
throw new BusinessException("SQL 不能为空");
|
||||
}
|
||||
if (sql.length() > 100_000) {
|
||||
throw new BusinessException("SQL 长度超过限制");
|
||||
}
|
||||
}
|
||||
|
||||
private int normalizeMaxRows(Integer requested) {
|
||||
if (requested == null) {
|
||||
return DEFAULT_MAX_ROWS;
|
||||
}
|
||||
if (requested <= 0) {
|
||||
throw new BusinessException("maxRows 必须大于 0");
|
||||
}
|
||||
return Math.min(requested, MAX_ALLOWED_ROWS);
|
||||
}
|
||||
|
||||
private List<DatacenterSqlColumnView> columns(List<FederationColumn> columns) {
|
||||
List<DatacenterSqlColumnView> result = new ArrayList<>(columns.size());
|
||||
for (FederationColumn column : columns) {
|
||||
result.add(new DatacenterSqlColumnView(
|
||||
"c" + column.index(),
|
||||
column.label(),
|
||||
column.jdbcType(),
|
||||
column.typeName(),
|
||||
column.nullable()));
|
||||
}
|
||||
return List.copyOf(result);
|
||||
}
|
||||
|
||||
private List<String> referencedTables(RelNode root) {
|
||||
Set<String> names = new LinkedHashSet<>();
|
||||
new RelVisitor() {
|
||||
@Override
|
||||
public void visit(RelNode node, int ordinal, RelNode parent) {
|
||||
if (node instanceof TableScan scan) {
|
||||
names.add(String.join(".", scan.getTable().getQualifiedName()));
|
||||
}
|
||||
super.visit(node, ordinal, parent);
|
||||
}
|
||||
}.go(root);
|
||||
return List.copyOf(names);
|
||||
}
|
||||
|
||||
private List<String> referencedFields(org.apache.calcite.sql.SqlNode sqlNode) {
|
||||
Set<String> names = new LinkedHashSet<>();
|
||||
sqlNode.accept(new SqlBasicVisitor<Void>() {
|
||||
@Override
|
||||
public Void visit(SqlIdentifier identifier) {
|
||||
if (!identifier.isStar() && !identifier.names.isEmpty()) {
|
||||
names.add(String.join(".", identifier.names));
|
||||
}
|
||||
return null;
|
||||
}
|
||||
});
|
||||
return List.copyOf(names);
|
||||
}
|
||||
|
||||
private Map<String, Object> maskedParameterSummary(List<SqlParameter> parameters) {
|
||||
Map<String, Object> summary = new LinkedHashMap<>();
|
||||
for (int index = 0; index < parameters.size(); index++) {
|
||||
SqlParameter parameter = parameters.get(index);
|
||||
summary.put("p" + (index + 1), Map.of(
|
||||
"jdbcType", parameter.jdbcType(),
|
||||
"present", parameter.value() != null));
|
||||
}
|
||||
return Map.copyOf(summary);
|
||||
}
|
||||
|
||||
private Object normalizeValue(Object value) {
|
||||
try {
|
||||
if (value instanceof byte[] bytes) {
|
||||
if (bytes.length > MAX_TEXT_CELL_CHARS) {
|
||||
return "[binary " + bytes.length + " bytes]";
|
||||
}
|
||||
return Base64.getEncoder().encodeToString(bytes);
|
||||
}
|
||||
if (value instanceof Blob blob) {
|
||||
return "[binary " + blob.length() + " bytes]";
|
||||
}
|
||||
if (value instanceof Clob clob) {
|
||||
long length = Math.min(clob.length(), 4_096L);
|
||||
return clob.getSubString(1L, (int) length);
|
||||
}
|
||||
if (value instanceof SQLXML sqlxml) {
|
||||
String text = sqlxml.getString();
|
||||
return boundedText(text);
|
||||
}
|
||||
if (value instanceof java.sql.Array array) {
|
||||
return "[array " + array.getBaseTypeName() + "]";
|
||||
}
|
||||
if (value instanceof Struct struct) {
|
||||
return "[struct " + struct.getSQLTypeName() + "]";
|
||||
}
|
||||
if (value instanceof CharSequence text) {
|
||||
return boundedText(text.toString());
|
||||
}
|
||||
if (value instanceof BigInteger || value instanceof Long) {
|
||||
return value.toString();
|
||||
}
|
||||
Package valuePackage = value.getClass().getPackage();
|
||||
if (valuePackage != null && valuePackage.getName().startsWith("org.postgresql")) {
|
||||
return boundedText(String.valueOf(value));
|
||||
}
|
||||
return value instanceof Number || value instanceof Boolean
|
||||
|| value instanceof java.time.temporal.TemporalAccessor
|
||||
|| value instanceof java.util.Date
|
||||
? value : boundedText(String.valueOf(value));
|
||||
} catch (SQLException exception) {
|
||||
throw new BusinessException("查询结果包含无法读取的大对象");
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 按 JDBC 类型选择有界流式读取或普通对象读取。
|
||||
*
|
||||
* @param cursor 当前结果游标
|
||||
* @param column 列元数据
|
||||
* @param columnIndex 从 1 开始的列序号
|
||||
* @return 可安全序列化的有界值
|
||||
*/
|
||||
private Object readColumn(
|
||||
FederationResultCursor cursor,
|
||||
FederationColumn column,
|
||||
int columnIndex) {
|
||||
try {
|
||||
if (isBinaryType(column.jdbcType())) {
|
||||
try (InputStream stream = cursor.getBinaryStream(columnIndex)) {
|
||||
return readBoundedBinary(stream);
|
||||
}
|
||||
}
|
||||
if (isCharacterType(column.jdbcType(), column.typeName())) {
|
||||
try (Reader reader = cursor.getCharacterStream(columnIndex)) {
|
||||
return readBoundedText(reader);
|
||||
}
|
||||
}
|
||||
} catch (UnsupportedOperationException ignored) {
|
||||
// 第三方 Adapter 可暂不实现流式列读取;保留兼容的对象读取路径。
|
||||
} catch (IOException exception) {
|
||||
throw new BusinessException("查询结果流读取失败");
|
||||
}
|
||||
return normalizeValue(cursor.getObject(columnIndex));
|
||||
}
|
||||
|
||||
/**
|
||||
* 有界读取二进制流,小值编码为 Base64,大值返回摘要。
|
||||
*
|
||||
* @param stream 二进制流;可为空
|
||||
* @return Base64 文本、大小摘要或 null
|
||||
* @throws IOException 流读取失败
|
||||
*/
|
||||
private Object readBoundedBinary(InputStream stream) throws IOException {
|
||||
if (stream == null) {
|
||||
return null;
|
||||
}
|
||||
byte[] bytes = stream.readNBytes(MAX_TEXT_CELL_CHARS + 1);
|
||||
if (bytes.length > MAX_TEXT_CELL_CHARS) {
|
||||
return "[binary exceeds " + MAX_TEXT_CELL_CHARS + " bytes]";
|
||||
}
|
||||
return Base64.getEncoder().encodeToString(bytes);
|
||||
}
|
||||
|
||||
/**
|
||||
* 有界读取字符流,超过单元格限制时附加截断标识。
|
||||
*
|
||||
* @param reader 字符流;可为空
|
||||
* @return 有界文本或 null
|
||||
* @throws IOException 流读取失败
|
||||
*/
|
||||
private String readBoundedText(Reader reader) throws IOException {
|
||||
if (reader == null) {
|
||||
return null;
|
||||
}
|
||||
StringBuilder text = new StringBuilder(Math.min(MAX_TEXT_CELL_CHARS, 4_096));
|
||||
char[] buffer = new char[4_096];
|
||||
while (text.length() <= MAX_TEXT_CELL_CHARS) {
|
||||
int remaining = MAX_TEXT_CELL_CHARS + 1 - text.length();
|
||||
int read = reader.read(buffer, 0, Math.min(buffer.length, remaining));
|
||||
if (read < 0) {
|
||||
break;
|
||||
}
|
||||
text.append(buffer, 0, read);
|
||||
}
|
||||
if (text.length() > MAX_TEXT_CELL_CHARS) {
|
||||
return text.substring(0, MAX_TEXT_CELL_CHARS) + "…[truncated]";
|
||||
}
|
||||
return text.toString();
|
||||
}
|
||||
|
||||
/**
|
||||
* 判断 JDBC 类型是否应使用二进制流读取。
|
||||
*
|
||||
* @param jdbcType JDBC 类型
|
||||
* @return 是否为二进制大字段类型
|
||||
*/
|
||||
private boolean isBinaryType(int jdbcType) {
|
||||
return jdbcType == Types.BINARY
|
||||
|| jdbcType == Types.VARBINARY
|
||||
|| jdbcType == Types.LONGVARBINARY
|
||||
|| jdbcType == Types.BLOB;
|
||||
}
|
||||
|
||||
/**
|
||||
* 判断 JDBC 类型或数据库类型名是否应使用字符流读取。
|
||||
*
|
||||
* @param jdbcType JDBC 类型
|
||||
* @param typeName 数据库类型名
|
||||
* @return 是否为字符大字段类型
|
||||
*/
|
||||
private boolean isCharacterType(int jdbcType, String typeName) {
|
||||
if (jdbcType == Types.CHAR
|
||||
|| jdbcType == Types.VARCHAR
|
||||
|| jdbcType == Types.LONGVARCHAR
|
||||
|| jdbcType == Types.NCHAR
|
||||
|| jdbcType == Types.NVARCHAR
|
||||
|| jdbcType == Types.LONGNVARCHAR
|
||||
|| jdbcType == Types.CLOB
|
||||
|| jdbcType == Types.NCLOB
|
||||
|| jdbcType == Types.SQLXML) {
|
||||
return true;
|
||||
}
|
||||
String normalized = typeName == null ? "" : typeName.toLowerCase(java.util.Locale.ROOT);
|
||||
return normalized.equals("json") || normalized.equals("jsonb") || normalized.equals("xml");
|
||||
}
|
||||
|
||||
private String boundedText(String value) {
|
||||
if (value == null || value.length() <= MAX_TEXT_CELL_CHARS) {
|
||||
return value;
|
||||
}
|
||||
return value.substring(0, MAX_TEXT_CELL_CHARS)
|
||||
+ "…[truncated " + value.length() + " chars]";
|
||||
}
|
||||
|
||||
private long estimateBytes(String label, Object value) {
|
||||
long bytes = label == null ? 0L : (long) label.length() * Character.BYTES;
|
||||
if (value == null) {
|
||||
return bytes + 8L;
|
||||
}
|
||||
if (value instanceof CharSequence text) {
|
||||
return bytes + (long) text.length() * Character.BYTES;
|
||||
}
|
||||
return bytes + 64L;
|
||||
}
|
||||
|
||||
private long elapsedMillis(long startedNanos) {
|
||||
return Math.max(0L, (System.nanoTime() - startedNanos) / 1_000_000L);
|
||||
}
|
||||
|
||||
private String safeMessage(Throwable exception) {
|
||||
String message = exception.getMessage();
|
||||
if (message == null || message.isBlank()) {
|
||||
return "数据查询失败";
|
||||
}
|
||||
return message.length() <= 500 ? message : message.substring(0, 500);
|
||||
}
|
||||
|
||||
private BusinessException toBusinessException(RuntimeException exception) {
|
||||
if (exception instanceof FederationSqlException federationException) {
|
||||
int status = switch (federationException.errorCode()) {
|
||||
case QUERY_ADMISSION_TIMEOUT, QUERY_TIMEOUT,
|
||||
SQL_COMPILE_TIMEOUT -> 408;
|
||||
case NODE_MEMORY_ADMISSION_TIMEOUT -> 429;
|
||||
case SOURCE_NOT_FOUND, SOURCE_REMOVED -> 404;
|
||||
case SOURCE_REVISION_NOT_READY, SOURCE_DEFINITION_CONFLICT -> 409;
|
||||
case PLAN_STALE -> 409;
|
||||
case FEDERATION_RESOURCE_LIMIT_EXCEEDED -> 422;
|
||||
case EXECUTION_FAILED, RESOURCE_CLOSE_FAILED,
|
||||
SOURCE_INITIALIZATION_FAILED, ENGINE_CLOSED,
|
||||
EXPLAIN_FAILED, SQL_COMPLETION_FAILED,
|
||||
CONNECTION_ACQUISITION_TIMEOUT,
|
||||
CONNECTION_ACQUISITION_FAILED -> 503;
|
||||
default -> 400;
|
||||
};
|
||||
int code = switch (federationException.errorCode()) {
|
||||
case INVALID_ARGUMENT -> 14101;
|
||||
case SQL_PARSE_FAILED -> 14102;
|
||||
case SQL_VALIDATION_FAILED -> 14103;
|
||||
case SQL_NOT_READ_ONLY -> 14104;
|
||||
case SQL_COMPILE_FAILED -> 14105;
|
||||
case SQL_NOT_FULLY_PUSHDOWN -> 14106;
|
||||
case CROSS_SOURCE_DISABLED -> 14107;
|
||||
case CROSS_SOURCE_EXECUTION_UNSUPPORTED -> 14108;
|
||||
case SOURCE_NOT_FOUND -> 14109;
|
||||
case SOURCE_REMOVED -> 14110;
|
||||
case SOURCE_REVISION_NOT_READY -> 14111;
|
||||
case SOURCE_DEFINITION_CONFLICT -> 14112;
|
||||
case SOURCE_INITIALIZATION_FAILED -> 14113;
|
||||
case ADAPTER_NOT_FOUND -> 14114;
|
||||
case ADAPTER_UNSUPPORTED -> 14115;
|
||||
case PARAMETER_COUNT_MISMATCH -> 14116;
|
||||
case QUERY_ADMISSION_TIMEOUT -> 14117;
|
||||
case QUERY_CANCELLED -> 14118;
|
||||
case QUERY_TIMEOUT -> 14119;
|
||||
case EXECUTION_FAILED -> 14120;
|
||||
case RESOURCE_CLOSE_FAILED -> 14121;
|
||||
case ENGINE_CLOSED -> 14122;
|
||||
case INVALID_QUERY_SCOPE -> 14124;
|
||||
case FEDERATION_OPERATOR_UNSUPPORTED -> 14125;
|
||||
case FEDERATION_RESOURCE_LIMIT_EXCEEDED -> 14126;
|
||||
case PLAN_STALE -> 14127;
|
||||
case EXPLAIN_FAILED -> 14128;
|
||||
case SQL_COMPLETION_FAILED -> 14129;
|
||||
case SQL_COMPILE_TIMEOUT -> 14130;
|
||||
case NODE_MEMORY_ADMISSION_TIMEOUT -> 14131;
|
||||
case CONNECTION_ACQUISITION_TIMEOUT -> 14132;
|
||||
case CONNECTION_ACQUISITION_FAILED -> 14133;
|
||||
};
|
||||
String message = switch (federationException.errorCode()) {
|
||||
case EXECUTION_FAILED, RESOURCE_CLOSE_FAILED,
|
||||
SOURCE_INITIALIZATION_FAILED, ENGINE_CLOSED -> "数据库查询暂时不可用";
|
||||
case QUERY_TIMEOUT, QUERY_ADMISSION_TIMEOUT -> "数据库查询超时";
|
||||
case QUERY_CANCELLED -> "查询已取消";
|
||||
case FEDERATION_RESOURCE_LIMIT_EXCEEDED -> "联邦查询超过资源限制";
|
||||
case EXPLAIN_FAILED -> "数据库查询计划分析失败";
|
||||
case SQL_COMPLETION_FAILED -> "SQL 补全暂时不可用";
|
||||
case SQL_COMPILE_TIMEOUT -> "SQL 编译超时,请简化语句后重试";
|
||||
case NODE_MEMORY_ADMISSION_TIMEOUT -> "节点查询资源繁忙,请稍后重试";
|
||||
case CONNECTION_ACQUISITION_TIMEOUT -> "数据库连接池繁忙,请稍后重试";
|
||||
case CONNECTION_ACQUISITION_FAILED -> "无法获取数据库连接,请检查连接状态";
|
||||
default -> safeMessage(exception);
|
||||
};
|
||||
return new BusinessException(status, code, message, exception);
|
||||
}
|
||||
return new BusinessException(500, 14999, "数据查询失败", exception);
|
||||
}
|
||||
|
||||
private void logUnexpectedFailure(
|
||||
QueryId queryId,
|
||||
DatacenterSource source,
|
||||
RuntimeException exception) {
|
||||
if (!(exception instanceof FederationSqlException federationException)
|
||||
|| switch (federationException.errorCode()) {
|
||||
case EXECUTION_FAILED, RESOURCE_CLOSE_FAILED,
|
||||
SOURCE_INITIALIZATION_FAILED, ENGINE_CLOSED,
|
||||
EXPLAIN_FAILED, CONNECTION_ACQUISITION_FAILED -> true;
|
||||
default -> false;
|
||||
}) {
|
||||
log.error("Datacenter query failed, queryId={}, sourceId={}",
|
||||
queryId.value(), source == null ? null : source.getId(), exception);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,175 @@
|
||||
package tech.easyflow.datacenter.federation;
|
||||
|
||||
import com.easyagents.federation.sql.adapter.jdbc.JdbcFederationSqlAdapterProvider;
|
||||
import com.easyagents.federation.sql.api.FederationSqlEngine;
|
||||
import com.easyagents.federation.sql.api.FederationSqlEngines;
|
||||
import com.easyagents.federation.sql.source.FederationSourceDefinition;
|
||||
import com.easyagents.federation.sql.source.PreparedSourceRuntime;
|
||||
import com.easyagents.federation.sql.source.SourceApplyOptions;
|
||||
import com.easyagents.federation.sql.source.SourceApplyResult;
|
||||
import com.easyagents.federation.sql.source.SourceApplyStatus;
|
||||
import com.easyagents.federation.sql.source.SourceProbeResult;
|
||||
import com.easyagents.federation.sql.source.SourceRemoveResult;
|
||||
import com.easyagents.federation.sql.source.SourceTombstone;
|
||||
import jakarta.annotation.PreDestroy;
|
||||
import org.springframework.stereotype.Component;
|
||||
import tech.easyflow.datacenter.meta.entity.DatacenterSource;
|
||||
|
||||
/**
|
||||
* EasyFlow 数据中枢持有的单例 Federation SQL Engine。
|
||||
*/
|
||||
@Component
|
||||
public class DatacenterFederationRuntime {
|
||||
|
||||
private final DatacenterFederationDefinitionFactory definitionFactory;
|
||||
private final DatacenterFederationDataSourceResolver resolver;
|
||||
private final FederationSqlEngine engine;
|
||||
|
||||
/**
|
||||
* 组装 DataSource Resolver、数据库状态 Provider 与业务 Policy。
|
||||
*
|
||||
* @param definitionFactory Definition 工厂
|
||||
* @param resolver HikariCP Resolver
|
||||
* @param stateProvider 数据库权威状态 Provider
|
||||
* @param policy 纳管范围 Policy
|
||||
* @param admissionController 租户与数据源分层准入控制器
|
||||
*/
|
||||
public DatacenterFederationRuntime(
|
||||
DatacenterFederationDefinitionFactory definitionFactory,
|
||||
DatacenterFederationDataSourceResolver resolver,
|
||||
DatacenterFederationSourceStateProvider stateProvider,
|
||||
DatacenterFederationSqlPolicy policy,
|
||||
DatacenterFederationAdmissionController admissionController) {
|
||||
this.definitionFactory = definitionFactory;
|
||||
this.resolver = resolver;
|
||||
this.engine = FederationSqlEngines.builder()
|
||||
.dataSourceResolver(resolver)
|
||||
.stateProvider(stateProvider)
|
||||
.adapter(new JdbcFederationSqlAdapterProvider())
|
||||
.policy(policy)
|
||||
.admissionController(admissionController)
|
||||
.maximumPlanCacheEntries(512)
|
||||
.maximumConcurrentCompilations(Math.max(1, Math.min(4,
|
||||
Runtime.getRuntime().availableProcessors())))
|
||||
.build();
|
||||
}
|
||||
|
||||
/**
|
||||
* 探测草稿数据源,不保留临时连接池。
|
||||
*
|
||||
* @param source 数据源草稿
|
||||
* @return 探测结果
|
||||
*/
|
||||
public SourceProbeResult probe(DatacenterSource source) {
|
||||
return engine.sources().probe(definitionFactory.create(source));
|
||||
}
|
||||
|
||||
/**
|
||||
* 使用未发布候选配置执行短生命周期探测。
|
||||
*
|
||||
* @param source 候选数据源
|
||||
* @return 探测结果
|
||||
*/
|
||||
public SourceProbeResult probeCandidate(DatacenterSource source) {
|
||||
FederationSourceDefinition definition = definitionFactory.createCandidate(source);
|
||||
return resolver.withCandidate(source, definition,
|
||||
() -> engine.sources().probe(definition));
|
||||
}
|
||||
|
||||
/**
|
||||
* 预构建单命名空间候选 Runtime,且不修改共享 Source Slot。
|
||||
*
|
||||
* @param source 候选数据源
|
||||
* @param catalogName 即将发布的 Catalog 或 Schema
|
||||
* @return 由调用方暂时持有的候选 Runtime
|
||||
*/
|
||||
public PreparedSourceRuntime prepareCandidate(
|
||||
DatacenterSource source,
|
||||
String catalogName) {
|
||||
FederationSourceDefinition definition =
|
||||
definitionFactory.createCandidate(source, catalogName);
|
||||
return resolver.withCandidate(source, definition,
|
||||
() -> engine.sources().prepare(definition));
|
||||
}
|
||||
|
||||
/**
|
||||
* 使用当前持久化命名空间预构建候选 Runtime,且不修改共享 Source Slot。
|
||||
*
|
||||
* @param source 候选数据源
|
||||
* @return 由调用方暂时持有的候选 Runtime
|
||||
*/
|
||||
public PreparedSourceRuntime prepareCandidate(DatacenterSource source) {
|
||||
FederationSourceDefinition definition = definitionFactory.create(source);
|
||||
return resolver.withCandidate(source, definition,
|
||||
() -> engine.sources().prepare(definition));
|
||||
}
|
||||
|
||||
/**
|
||||
* 原子接管已经通过业务 revision CAS 的候选 Runtime。
|
||||
*
|
||||
* @param prepared 预构建 Runtime
|
||||
* @return Definition 应用结果
|
||||
* @throws IllegalStateException Slot 已被更高版本或冲突版本占用时抛出
|
||||
*/
|
||||
public SourceApplyResult commitPrepared(PreparedSourceRuntime prepared) {
|
||||
SourceApplyResult result = engine.sources().commit(prepared);
|
||||
if (result.status() == SourceApplyStatus.CONFLICT
|
||||
|| result.status() == SourceApplyStatus.IGNORED_STALE) {
|
||||
throw new IllegalStateException(
|
||||
"prepared runtime was superseded before commit: " + result.status());
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* 应用已持久化 Definition。
|
||||
*
|
||||
* @param source 数据源当前快照
|
||||
* @param prewarm 是否立即预热
|
||||
* @return 应用结果
|
||||
*/
|
||||
public SourceApplyResult apply(DatacenterSource source, boolean prewarm) {
|
||||
FederationSourceDefinition definition = definitionFactory.create(source);
|
||||
return engine.sources().apply(
|
||||
definition,
|
||||
prewarm ? SourceApplyOptions.prewarmNow() : SourceApplyOptions.lazy());
|
||||
}
|
||||
|
||||
/**
|
||||
* 应用禁用或删除墓碑。
|
||||
*
|
||||
* @param source 数据源当前快照
|
||||
* @return 删除结果
|
||||
*/
|
||||
public SourceRemoveResult remove(DatacenterSource source) {
|
||||
return engine.sources().remove(SourceTombstone.of(
|
||||
definitionFactory.sourceId(source),
|
||||
source.getDefinitionRevision() == null ? 0L : source.getDefinitionRevision()));
|
||||
}
|
||||
|
||||
/**
|
||||
* 返回统一查询 Engine。
|
||||
*
|
||||
* @return Engine
|
||||
*/
|
||||
public FederationSqlEngine engine() {
|
||||
return engine;
|
||||
}
|
||||
|
||||
/**
|
||||
* 返回 Definition 工厂。
|
||||
*
|
||||
* @return Definition 工厂
|
||||
*/
|
||||
public DatacenterFederationDefinitionFactory definitions() {
|
||||
return definitionFactory;
|
||||
}
|
||||
|
||||
/**
|
||||
* 应用停止时关闭查询、计划缓存和所有 Runtime 句柄。
|
||||
*/
|
||||
@PreDestroy
|
||||
public void close() {
|
||||
engine.close();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,177 @@
|
||||
package tech.easyflow.datacenter.federation;
|
||||
|
||||
import com.easyagents.federation.sql.source.ActiveSourceState;
|
||||
import com.easyagents.federation.sql.source.FederationSourceState;
|
||||
import com.easyagents.federation.sql.source.FederationSourceStateProvider;
|
||||
import com.easyagents.federation.sql.source.SourceId;
|
||||
import com.easyagents.federation.sql.source.SourceStateSubscription;
|
||||
import com.easyagents.federation.sql.source.SourceTombstone;
|
||||
import com.mybatisflex.core.query.QueryWrapper;
|
||||
import java.math.BigInteger;
|
||||
import java.util.Collection;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Optional;
|
||||
import java.util.concurrent.CopyOnWriteArrayList;
|
||||
import java.util.function.Consumer;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.scheduling.annotation.Scheduled;
|
||||
import org.springframework.stereotype.Component;
|
||||
import tech.easyflow.datacenter.mapper.DatacenterSourceMapper;
|
||||
import tech.easyflow.datacenter.meta.entity.DatacenterSource;
|
||||
import tech.easyflow.datacenter.meta.enums.DatacenterSourceStatus;
|
||||
import tech.easyflow.datacenter.meta.enums.DatacenterSourceType;
|
||||
|
||||
/**
|
||||
* 数据库权威、Redis 仅提示的 Federation Source State Provider。
|
||||
*/
|
||||
@Component
|
||||
public class DatacenterFederationSourceStateProvider implements FederationSourceStateProvider {
|
||||
|
||||
private static final Logger log = LoggerFactory.getLogger(DatacenterFederationSourceStateProvider.class);
|
||||
|
||||
private final DatacenterSourceMapper sourceMapper;
|
||||
private final DatacenterFederationDefinitionFactory definitionFactory;
|
||||
private final CopyOnWriteArrayList<Consumer<FederationSourceState>> subscribers =
|
||||
new CopyOnWriteArrayList<>();
|
||||
|
||||
/**
|
||||
* 创建状态 Provider。
|
||||
*
|
||||
* @param sourceMapper 数据源 Mapper
|
||||
* @param definitionFactory Definition 工厂
|
||||
*/
|
||||
public DatacenterFederationSourceStateProvider(
|
||||
DatacenterSourceMapper sourceMapper,
|
||||
DatacenterFederationDefinitionFactory definitionFactory) {
|
||||
this.sourceMapper = sourceMapper;
|
||||
this.definitionFactory = definitionFactory;
|
||||
}
|
||||
|
||||
/**
|
||||
* 从数据库读取指定数据源的最新 Definition 或墓碑。
|
||||
*
|
||||
* @param sourceId Federation SourceId
|
||||
* @return 当前共享状态
|
||||
*/
|
||||
@Override
|
||||
public Optional<FederationSourceState> find(SourceId sourceId) {
|
||||
BigInteger recordId = parseRecordId(sourceId);
|
||||
if (recordId == null) {
|
||||
return Optional.empty();
|
||||
}
|
||||
return toState(sourceMapper.selectOneById(recordId));
|
||||
}
|
||||
|
||||
/**
|
||||
* 加载当前数据库上下文中可见的外部数据源快照。
|
||||
*
|
||||
* @return Definition 与墓碑集合
|
||||
*/
|
||||
@Override
|
||||
public Collection<FederationSourceState> loadSnapshot() {
|
||||
QueryWrapper query = QueryWrapper.create()
|
||||
.in(DatacenterSource::getSourceType, List.of(
|
||||
DatacenterSourceType.MYSQL.name(),
|
||||
DatacenterSourceType.POSTGRESQL.name()))
|
||||
.in(DatacenterSource::getStatus, List.of(
|
||||
DatacenterSourceStatus.DRAFT.code(),
|
||||
DatacenterSourceStatus.READY.code(),
|
||||
DatacenterSourceStatus.DEGRADED.code(),
|
||||
DatacenterSourceStatus.DISABLED.code(),
|
||||
DatacenterSourceStatus.DELETED.code()));
|
||||
List<DatacenterSource> sources = sourceMapper.selectListByQuery(query);
|
||||
Map<BigInteger, com.easyagents.federation.sql.source.FederationSourceDefinition> definitions =
|
||||
definitionFactory.createBatch(sources.stream()
|
||||
.filter(this::isActive)
|
||||
.toList());
|
||||
return sources.stream()
|
||||
.map(source -> toState(source, definitions.get(source.getId())))
|
||||
.flatMap(Optional::stream)
|
||||
.toList();
|
||||
}
|
||||
|
||||
/**
|
||||
* 注册节点本地变更消费者。
|
||||
*
|
||||
* @param consumer 状态消费者
|
||||
* @return 可关闭订阅
|
||||
*/
|
||||
@Override
|
||||
public SourceStateSubscription subscribe(Consumer<FederationSourceState> consumer) {
|
||||
subscribers.add(consumer);
|
||||
return () -> subscribers.remove(consumer);
|
||||
}
|
||||
|
||||
/**
|
||||
* 响应无凭据 Redis 提示,从数据库重读并发布最新状态。
|
||||
*
|
||||
* @param sourceId Federation SourceId
|
||||
*/
|
||||
public void refresh(SourceId sourceId) {
|
||||
find(sourceId).ifPresent(state -> subscribers.forEach(consumer -> consumer.accept(state)));
|
||||
}
|
||||
|
||||
/**
|
||||
* 周期性回源数据库,弥补 Redis Pub/Sub 在节点离线或网络抖动时可能丢失的提示。
|
||||
*/
|
||||
@Scheduled(fixedDelayString = "${easyflow.datacenter.federation-reconcile-ms:30000}")
|
||||
public void reconcile() {
|
||||
try {
|
||||
loadSnapshot().forEach(state -> subscribers.forEach(consumer -> consumer.accept(state)));
|
||||
} catch (RuntimeException exception) {
|
||||
log.warn("Failed to reconcile datacenter Federation source states", exception);
|
||||
}
|
||||
}
|
||||
|
||||
private Optional<FederationSourceState> toState(DatacenterSource source) {
|
||||
return toState(source, null);
|
||||
}
|
||||
|
||||
private Optional<FederationSourceState> toState(
|
||||
DatacenterSource source,
|
||||
com.easyagents.federation.sql.source.FederationSourceDefinition preloadedDefinition) {
|
||||
if (source == null || source.getId() == null) {
|
||||
return Optional.empty();
|
||||
}
|
||||
DatacenterSourceStatus status = DatacenterSourceStatus.fromCode(source.getStatus());
|
||||
long revision = source.getDefinitionRevision() == null ? 0L : source.getDefinitionRevision();
|
||||
if (status == DatacenterSourceStatus.READY || status == DatacenterSourceStatus.DEGRADED) {
|
||||
if (revision <= 0L) {
|
||||
return Optional.empty();
|
||||
}
|
||||
return Optional.of(ActiveSourceState.of(preloadedDefinition == null
|
||||
? definitionFactory.create(source) : preloadedDefinition));
|
||||
}
|
||||
if ((status == DatacenterSourceStatus.DRAFT && revision > 0L)
|
||||
|| status == DatacenterSourceStatus.DISABLED
|
||||
|| status == DatacenterSourceStatus.DELETED) {
|
||||
return Optional.of(SourceTombstone.of(definitionFactory.sourceId(source), revision));
|
||||
}
|
||||
return Optional.empty();
|
||||
}
|
||||
|
||||
private boolean isActive(DatacenterSource source) {
|
||||
DatacenterSourceStatus status = source == null
|
||||
? DatacenterSourceStatus.DRAFT
|
||||
: DatacenterSourceStatus.fromCode(source.getStatus());
|
||||
return status == DatacenterSourceStatus.READY || status == DatacenterSourceStatus.DEGRADED;
|
||||
}
|
||||
|
||||
private BigInteger parseRecordId(SourceId sourceId) {
|
||||
if (sourceId == null) {
|
||||
return null;
|
||||
}
|
||||
String value = sourceId.value();
|
||||
int marker = value.lastIndexOf("-source-");
|
||||
if (marker < 0) {
|
||||
return null;
|
||||
}
|
||||
try {
|
||||
return new BigInteger(value.substring(marker + "-source-".length()));
|
||||
} catch (NumberFormatException ignored) {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,859 @@
|
||||
package tech.easyflow.datacenter.federation;
|
||||
|
||||
import com.easyagents.federation.sql.api.FederationSqlErrorCode;
|
||||
import com.easyagents.federation.sql.api.FederationSqlException;
|
||||
import com.easyagents.federation.sql.compile.FederationSqlPolicy;
|
||||
import com.easyagents.federation.sql.compile.SqlPolicyContext;
|
||||
import com.mybatisflex.core.query.QueryWrapper;
|
||||
import java.math.BigInteger;
|
||||
import java.util.ArrayDeque;
|
||||
import java.util.Collections;
|
||||
import java.util.Deque;
|
||||
import java.util.HashMap;
|
||||
import java.util.HashSet;
|
||||
import java.util.IdentityHashMap;
|
||||
import java.util.LinkedHashSet;
|
||||
import java.util.List;
|
||||
import java.util.Locale;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
import java.util.concurrent.ConcurrentLinkedQueue;
|
||||
import org.apache.calcite.rel.RelNode;
|
||||
import org.apache.calcite.rel.RelVisitor;
|
||||
import org.apache.calcite.rel.core.TableScan;
|
||||
import org.apache.calcite.sql.SqlIdentifier;
|
||||
import org.apache.calcite.sql.SqlCall;
|
||||
import org.apache.calcite.sql.SqlKind;
|
||||
import org.apache.calcite.sql.SqlJoin;
|
||||
import org.apache.calcite.sql.SqlNode;
|
||||
import org.apache.calcite.sql.SqlNodeList;
|
||||
import org.apache.calcite.sql.SqlOrderBy;
|
||||
import org.apache.calcite.sql.SqlSelect;
|
||||
import org.apache.calcite.sql.SqlWith;
|
||||
import org.apache.calcite.sql.SqlWithItem;
|
||||
import org.apache.calcite.sql.util.SqlBasicVisitor;
|
||||
import org.springframework.stereotype.Component;
|
||||
import tech.easyflow.datacenter.entity.DatacenterTable;
|
||||
import tech.easyflow.datacenter.entity.DatacenterTableField;
|
||||
import tech.easyflow.datacenter.mapper.DatacenterCatalogMapper;
|
||||
import tech.easyflow.datacenter.mapper.DatacenterSourceMapper;
|
||||
import tech.easyflow.datacenter.mapper.DatacenterTableFieldMapper;
|
||||
import tech.easyflow.datacenter.mapper.DatacenterTableMapper;
|
||||
import tech.easyflow.datacenter.meta.entity.DatacenterCatalog;
|
||||
import tech.easyflow.datacenter.meta.entity.DatacenterSource;
|
||||
import tech.easyflow.datacenter.meta.enums.DatacenterMetadataStatus;
|
||||
import tech.easyflow.datacenter.meta.enums.DatacenterSensitivityLevel;
|
||||
import tech.easyflow.datacenter.meta.enums.DatacenterSourceStatus;
|
||||
|
||||
/**
|
||||
* 将 Calcite 解析出的表与字段引用限制在当前数据源的可查询纳管范围内。
|
||||
*/
|
||||
@Component
|
||||
public class DatacenterFederationSqlPolicy implements FederationSqlPolicy {
|
||||
|
||||
private static final int MAX_POLICY_CACHE_ENTRIES = 512;
|
||||
|
||||
private final DatacenterTableMapper tableMapper;
|
||||
private final DatacenterTableFieldMapper tableFieldMapper;
|
||||
private final DatacenterCatalogMapper catalogMapper;
|
||||
private final DatacenterSourceMapper sourceMapper;
|
||||
private final Map<PolicyCacheKey, PolicySnapshot> policyCache = new ConcurrentHashMap<>();
|
||||
private final ConcurrentLinkedQueue<PolicyCacheKey> policyCacheOrder = new ConcurrentLinkedQueue<>();
|
||||
|
||||
/**
|
||||
* 创建业务 SQL Policy。
|
||||
*
|
||||
* @param tableMapper 表 Mapper
|
||||
* @param tableFieldMapper 字段 Mapper
|
||||
* @param catalogMapper 命名空间 Mapper
|
||||
* @param sourceMapper 数据源 Mapper
|
||||
*/
|
||||
public DatacenterFederationSqlPolicy(
|
||||
DatacenterTableMapper tableMapper,
|
||||
DatacenterTableFieldMapper tableFieldMapper,
|
||||
DatacenterCatalogMapper catalogMapper,
|
||||
DatacenterSourceMapper sourceMapper) {
|
||||
this.tableMapper = tableMapper;
|
||||
this.tableFieldMapper = tableFieldMapper;
|
||||
this.catalogMapper = catalogMapper;
|
||||
this.sourceMapper = sourceMapper;
|
||||
}
|
||||
|
||||
/**
|
||||
* 返回 Policy 代码版本。
|
||||
*
|
||||
* @return 稳定版本
|
||||
*/
|
||||
@Override
|
||||
public String version() {
|
||||
return "xl01-wp1-v6";
|
||||
}
|
||||
|
||||
/**
|
||||
* 拒绝未纳管表、不可查询字段和任何尚未支持脱敏的敏感字段。
|
||||
*
|
||||
* @param context Calcite 策略上下文
|
||||
*/
|
||||
@Override
|
||||
public void validate(SqlPolicyContext context) {
|
||||
if (context.request().queryScope().bindings().size() != 1
|
||||
|| context.referencedSources().size() != 1
|
||||
|| !context.referencedSources().contains(
|
||||
context.request().sourceId())) {
|
||||
throw validationFailure("当前数据中枢尚未开放跨数据源授权范围");
|
||||
}
|
||||
BigInteger sourceId = parseRecordId(context.request().sourceId().value());
|
||||
requireCurrentScopeRevision(
|
||||
sourceId, context.request().policyVersion());
|
||||
PolicyCacheKey cacheKey = new PolicyCacheKey(sourceId, context.request().policyVersion());
|
||||
PolicySnapshot snapshot = cachedSnapshot(cacheKey);
|
||||
Set<String> referencedNames = referencedTables(context.relRoot().rel);
|
||||
Set<BigInteger> referencedTableIds = new LinkedHashSet<>();
|
||||
for (String referencedName : referencedNames) {
|
||||
Set<BigInteger> tableIds = snapshot.tableIdsByName().get(referencedName);
|
||||
if (tableIds == null || tableIds.isEmpty()) {
|
||||
throw validationFailure("SQL 引用了未纳管表: " + referencedName);
|
||||
}
|
||||
referencedTableIds.addAll(tableIds);
|
||||
}
|
||||
validatePhysicalColumns(context.relRoot().rel, snapshot);
|
||||
validateFields(context, snapshot, referencedTableIds);
|
||||
}
|
||||
|
||||
/**
|
||||
* 直接核对数据库权威范围版本,避免跨节点通知延迟复用旧策略。
|
||||
*
|
||||
* @param sourceId 数据源记录 ID
|
||||
* @param policyVersion 编译请求携带的范围版本
|
||||
*/
|
||||
private void requireCurrentScopeRevision(
|
||||
BigInteger sourceId,
|
||||
String policyVersion) {
|
||||
long requestedRevision = parseScopeRevision(policyVersion);
|
||||
DatacenterSource current = sourceMapper.selectOneById(sourceId);
|
||||
if (current == null
|
||||
|| DatacenterSourceStatus.fromCode(current.getStatus())
|
||||
!= DatacenterSourceStatus.READY
|
||||
|| current.getScopeRevision() == null
|
||||
|| current.getScopeRevision() != requestedRevision) {
|
||||
throw validationFailure("数据范围已变化,请重新执行查询");
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 解析范围策略版本。
|
||||
*
|
||||
* @param policyVersion 策略版本
|
||||
* @return 范围 revision
|
||||
*/
|
||||
private long parseScopeRevision(String policyVersion) {
|
||||
if (policyVersion == null || !policyVersion.startsWith("scope-")) {
|
||||
throw validationFailure("数据范围版本无效");
|
||||
}
|
||||
try {
|
||||
return Long.parseLong(policyVersion.substring("scope-".length()));
|
||||
} catch (NumberFormatException exception) {
|
||||
throw validationFailure("数据范围版本无效");
|
||||
}
|
||||
}
|
||||
|
||||
private PolicySnapshot cachedSnapshot(PolicyCacheKey cacheKey) {
|
||||
PolicySnapshot existing = policyCache.get(cacheKey);
|
||||
if (existing != null) {
|
||||
return existing;
|
||||
}
|
||||
PolicySnapshot loaded = loadSnapshot(cacheKey.sourceId());
|
||||
PolicySnapshot raced = policyCache.putIfAbsent(cacheKey, loaded);
|
||||
if (raced != null) {
|
||||
return raced;
|
||||
}
|
||||
policyCacheOrder.add(cacheKey);
|
||||
trimCache();
|
||||
return loaded;
|
||||
}
|
||||
|
||||
private void trimCache() {
|
||||
while (policyCache.size() > MAX_POLICY_CACHE_ENTRIES) {
|
||||
PolicyCacheKey eldest = policyCacheOrder.poll();
|
||||
if (eldest == null) {
|
||||
return;
|
||||
}
|
||||
policyCache.remove(eldest);
|
||||
}
|
||||
}
|
||||
|
||||
private PolicySnapshot loadSnapshot(BigInteger sourceId) {
|
||||
QueryWrapper tableQuery = QueryWrapper.create()
|
||||
.eq(DatacenterTable::getSourceId, sourceId)
|
||||
.eq(DatacenterTable::getQueryable, 1)
|
||||
.eq(DatacenterTable::getMetadataStatus, DatacenterMetadataStatus.ACTIVE.name());
|
||||
List<DatacenterTable> tables = tableMapper.selectListByQuery(tableQuery);
|
||||
if (tables.isEmpty()) {
|
||||
return PolicySnapshot.empty();
|
||||
}
|
||||
Set<BigInteger> catalogIds = new HashSet<>();
|
||||
Set<BigInteger> tableIds = new HashSet<>();
|
||||
for (DatacenterTable table : tables) {
|
||||
tableIds.add(table.getId());
|
||||
if (table.getCatalogId() != null) {
|
||||
catalogIds.add(table.getCatalogId());
|
||||
}
|
||||
}
|
||||
Map<BigInteger, String> logicalSchemas = new HashMap<>();
|
||||
if (!catalogIds.isEmpty()) {
|
||||
QueryWrapper catalogQuery = QueryWrapper.create().in(DatacenterCatalog::getId, catalogIds);
|
||||
for (DatacenterCatalog catalog : catalogMapper.selectListByQuery(catalogQuery)) {
|
||||
logicalSchemas.put(catalog.getId(), catalog.getLogicalSchemaName());
|
||||
}
|
||||
}
|
||||
Map<String, Set<BigInteger>> idsByName = new HashMap<>();
|
||||
Set<String> tableReferenceNames = new HashSet<>();
|
||||
for (DatacenterTable table : tables) {
|
||||
addTableName(idsByName, tableReferenceNames, table.getTableName(), table.getId());
|
||||
addTableName(idsByName, tableReferenceNames, table.getActualTable(), table.getId());
|
||||
String schema = logicalSchemas.get(table.getCatalogId());
|
||||
if (schema != null && !schema.isBlank()) {
|
||||
addTableName(idsByName, tableReferenceNames, schema + "." + table.getTableName(), table.getId());
|
||||
addTableName(idsByName, tableReferenceNames, schema + "." + table.getActualTable(), table.getId());
|
||||
}
|
||||
}
|
||||
|
||||
Map<BigInteger, ColumnPolicy> columnsByTable = new HashMap<>();
|
||||
QueryWrapper fieldQuery = QueryWrapper.create().in(DatacenterTableField::getTableId, tableIds);
|
||||
for (DatacenterTableField field : tableFieldMapper.selectListByQuery(fieldQuery)) {
|
||||
ColumnPolicy columns = columnsByTable.computeIfAbsent(
|
||||
field.getTableId(), ignored -> ColumnPolicy.mutable());
|
||||
String columnName = sourceColumnName(field);
|
||||
if (columnName != null && !columnName.isBlank()) {
|
||||
columns.knownExact().add(columnName);
|
||||
columns.knownFolded().add(columnName.toUpperCase(Locale.ROOT));
|
||||
if (fieldAllowed(field)) {
|
||||
columns.allowedExact().add(columnName);
|
||||
columns.allowedFolded().add(columnName.toUpperCase(Locale.ROOT));
|
||||
}
|
||||
}
|
||||
}
|
||||
Map<String, Set<BigInteger>> immutableNames = new HashMap<>();
|
||||
idsByName.forEach((name, ids) -> immutableNames.put(name, Set.copyOf(ids)));
|
||||
return new PolicySnapshot(
|
||||
Map.copyOf(immutableNames),
|
||||
immutableColumnPolicies(columnsByTable),
|
||||
Set.copyOf(tableReferenceNames));
|
||||
}
|
||||
|
||||
private Map<BigInteger, ColumnPolicy> immutableColumnPolicies(
|
||||
Map<BigInteger, ColumnPolicy> mutablePolicies) {
|
||||
Map<BigInteger, ColumnPolicy> immutable = new HashMap<>();
|
||||
mutablePolicies.forEach((tableId, policy) -> immutable.put(tableId, new ColumnPolicy(
|
||||
Set.copyOf(policy.knownExact()),
|
||||
Set.copyOf(policy.knownFolded()),
|
||||
Set.copyOf(policy.allowedExact()),
|
||||
Set.copyOf(policy.allowedFolded()))));
|
||||
return Map.copyOf(immutable);
|
||||
}
|
||||
|
||||
private void validatePhysicalColumns(RelNode root, PolicySnapshot snapshot) {
|
||||
new RelVisitor() {
|
||||
@Override
|
||||
public void visit(RelNode node, int ordinal, RelNode parent) {
|
||||
if (node instanceof TableScan scan) {
|
||||
String referencedName = referencedTableName(scan);
|
||||
Set<BigInteger> tableIds = snapshot.tableIdsByName().get(referencedName);
|
||||
if (tableIds == null || tableIds.isEmpty()) {
|
||||
throw validationFailure("SQL 引用了未纳管表: " + referencedName);
|
||||
}
|
||||
for (BigInteger tableId : tableIds) {
|
||||
ColumnPolicy policy = snapshot.columnsByTable().get(tableId);
|
||||
if (policy == null || policy.knownExact().isEmpty()) {
|
||||
throw validationFailure("数据表尚未纳管任何字段: " + referencedName);
|
||||
}
|
||||
for (String fieldName : scan.getRowType().getFieldNames()) {
|
||||
if (!policy.knownExact().contains(fieldName)
|
||||
&& !policy.knownFolded().contains(fieldName.toUpperCase(Locale.ROOT))) {
|
||||
throw validationFailure("物理表包含尚未纳管的字段: " + fieldName);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
super.visit(node, ordinal, parent);
|
||||
}
|
||||
}.go(root);
|
||||
}
|
||||
|
||||
private void validateFields(
|
||||
SqlPolicyContext context,
|
||||
PolicySnapshot snapshot,
|
||||
Set<BigInteger> referencedTableIds) {
|
||||
Deque<AliasContext> scopes = new ArrayDeque<>();
|
||||
Map<String, DerivedRelation> commonTables = collectCommonTables(
|
||||
context.validatedSql(), snapshot, referencedTableIds);
|
||||
context.validatedSql().accept(new SqlBasicVisitor<Void>() {
|
||||
@Override
|
||||
public Void visit(SqlCall call) {
|
||||
AliasContext scope = scopeFor(
|
||||
call, snapshot, referencedTableIds, commonTables);
|
||||
if (scope == null) {
|
||||
return super.visit(call);
|
||||
}
|
||||
scopes.push(scope);
|
||||
try {
|
||||
return super.visit(call);
|
||||
} finally {
|
||||
scopes.pop();
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public Void visit(SqlIdentifier identifier) {
|
||||
if (scopes.isEmpty()) {
|
||||
return null;
|
||||
}
|
||||
AliasContext aliasContext = scopes.peek();
|
||||
if (aliasContext.syntaxAliases().contains(identifier)
|
||||
|| aliasContext.tableIdentifiers().contains(identifier)
|
||||
|| aliasContext.outputAliasReferences().contains(identifier)) {
|
||||
return null;
|
||||
}
|
||||
if (identifier.isStar()) {
|
||||
boolean allAllowed = referencedTableIds.stream()
|
||||
.map(snapshot.columnsByTable()::get)
|
||||
.allMatch(policy -> policy != null
|
||||
&& policy.knownExact().equals(policy.allowedExact()));
|
||||
if (!allAllowed) {
|
||||
throw validationFailure("查询范围包含不可查询或敏感字段,请显式选择允许字段");
|
||||
}
|
||||
return null;
|
||||
}
|
||||
List<String> names = identifier.names;
|
||||
if (names.isEmpty()) {
|
||||
return null;
|
||||
}
|
||||
int last = names.size() - 1;
|
||||
String columnName = names.get(last);
|
||||
boolean quoted = identifier.isComponentQuoted(last);
|
||||
if (derivedColumnReference(identifier, aliasContext)) {
|
||||
return null;
|
||||
}
|
||||
Set<BigInteger> targetTableIds = resolveColumnTables(
|
||||
identifier, snapshot, referencedTableIds, aliasContext);
|
||||
boolean allowed = !targetTableIds.isEmpty() && targetTableIds.stream()
|
||||
.allMatch(tableId -> columnAllowed(
|
||||
snapshot.columnsByTable().get(tableId), columnName, quoted));
|
||||
if (!allowed) {
|
||||
throw validationFailure("SQL 引用了不可查询或敏感字段: " + columnName);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
private AliasContext scopeFor(
|
||||
SqlCall call,
|
||||
PolicySnapshot snapshot,
|
||||
Set<BigInteger> referencedTableIds,
|
||||
Map<String, DerivedRelation> commonTables) {
|
||||
if (call instanceof SqlOrderBy orderBy
|
||||
&& orderBy.query instanceof SqlSelect select) {
|
||||
return collectAliases(
|
||||
select, orderBy.orderList, snapshot,
|
||||
referencedTableIds, commonTables);
|
||||
}
|
||||
if (call instanceof SqlSelect select) {
|
||||
return collectAliases(
|
||||
select, null, snapshot, referencedTableIds, commonTables);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
private boolean derivedColumnReference(
|
||||
SqlIdentifier identifier,
|
||||
AliasContext aliasContext) {
|
||||
List<String> names = identifier.names;
|
||||
int columnIndex = names.size() - 1;
|
||||
String columnKey = identifierKey(
|
||||
names.get(columnIndex),
|
||||
identifier.isComponentQuoted(columnIndex));
|
||||
if (names.size() == 1) {
|
||||
return aliasContext.unqualifiedDerivedColumns().contains(columnKey);
|
||||
}
|
||||
int qualifierIndex = names.size() - 2;
|
||||
String qualifierKey = identifierKey(
|
||||
names.get(qualifierIndex),
|
||||
identifier.isComponentQuoted(qualifierIndex));
|
||||
Set<String> derivedColumns = aliasContext
|
||||
.derivedColumnsByAlias().get(qualifierKey);
|
||||
return derivedColumns != null && derivedColumns.contains(columnKey);
|
||||
}
|
||||
|
||||
private Set<BigInteger> resolveColumnTables(
|
||||
SqlIdentifier identifier,
|
||||
PolicySnapshot snapshot,
|
||||
Set<BigInteger> referencedTableIds,
|
||||
AliasContext aliasContext) {
|
||||
List<String> identifierNames = identifier.names;
|
||||
if (identifierNames.size() >= 2) {
|
||||
int qualifierIndex = identifierNames.size() - 2;
|
||||
String qualifier = identifierNames.get(qualifierIndex);
|
||||
Set<BigInteger> aliased = aliasContext.tableIdsByAlias().get(
|
||||
identifierKey(qualifier, identifier.isComponentQuoted(qualifierIndex)));
|
||||
if (aliased != null && !aliased.isEmpty()) {
|
||||
return aliased;
|
||||
}
|
||||
Set<BigInteger> qualified = snapshot.tableIdsByName().get(qualifier);
|
||||
if (qualified != null && !qualified.isEmpty()) {
|
||||
Set<BigInteger> referencedQualified = new LinkedHashSet<>(qualified);
|
||||
referencedQualified.retainAll(referencedTableIds);
|
||||
if (!referencedQualified.isEmpty()) {
|
||||
return referencedQualified;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (referencedTableIds.size() == 1) {
|
||||
return referencedTableIds;
|
||||
}
|
||||
// 无法将别名还原到单表时采用交集语义,避免多表同名列扩大权限。
|
||||
return referencedTableIds;
|
||||
}
|
||||
|
||||
private AliasContext collectAliases(
|
||||
SqlSelect select,
|
||||
SqlNodeList externalOrderList,
|
||||
PolicySnapshot snapshot,
|
||||
Set<BigInteger> referencedTableIds,
|
||||
Map<String, DerivedRelation> commonTables) {
|
||||
Map<String, Set<BigInteger>> tableIdsByAlias = new HashMap<>();
|
||||
Map<String, Set<String>> derivedColumnsByAlias = new HashMap<>();
|
||||
Set<String> outputAliases = new HashSet<>();
|
||||
Set<SqlIdentifier> syntaxAliases = Collections.newSetFromMap(
|
||||
new IdentityHashMap<>());
|
||||
Set<SqlIdentifier> tableIdentifiers = Collections.newSetFromMap(
|
||||
new IdentityHashMap<>());
|
||||
Set<SqlIdentifier> outputAliasReferences = Collections.newSetFromMap(
|
||||
new IdentityHashMap<>());
|
||||
collectOutputAliases(
|
||||
select, outputAliases, syntaxAliases, outputAliasReferences);
|
||||
FromSummary fromSummary = collectFrom(
|
||||
select.getFrom(), snapshot, referencedTableIds,
|
||||
commonTables, tableIdsByAlias, derivedColumnsByAlias,
|
||||
syntaxAliases, tableIdentifiers);
|
||||
markOutputAliasReferences(
|
||||
externalOrderList, outputAliases, outputAliasReferences);
|
||||
return new AliasContext(
|
||||
Map.copyOf(tableIdsByAlias),
|
||||
Map.copyOf(derivedColumnsByAlias),
|
||||
fromSummary.singleDerived()
|
||||
? fromSummary.outputColumns() : Set.of(),
|
||||
Set.copyOf(outputAliases),
|
||||
syntaxAliases,
|
||||
tableIdentifiers,
|
||||
outputAliasReferences);
|
||||
}
|
||||
|
||||
private void collectOutputAliases(
|
||||
SqlSelect select,
|
||||
Set<String> outputAliases,
|
||||
Set<SqlIdentifier> syntaxAliases,
|
||||
Set<SqlIdentifier> outputAliasReferences) {
|
||||
if (select.getSelectList() == null) {
|
||||
return;
|
||||
}
|
||||
for (SqlNode item : select.getSelectList()) {
|
||||
if (!(item instanceof SqlCall call)
|
||||
|| call.getKind() != SqlKind.AS
|
||||
|| call.operandCount() < 2
|
||||
|| !(call.operand(1) instanceof SqlIdentifier alias)
|
||||
|| alias.names.isEmpty()) {
|
||||
continue;
|
||||
}
|
||||
int aliasIndex = alias.names.size() - 1;
|
||||
outputAliases.add(identifierKey(
|
||||
alias.names.get(aliasIndex),
|
||||
alias.isComponentQuoted(aliasIndex)));
|
||||
syntaxAliases.add(alias);
|
||||
}
|
||||
markOutputAliasReferences(
|
||||
select.getOrderList(), outputAliases, outputAliasReferences);
|
||||
markOutputAliasReferences(
|
||||
select.getGroup(), outputAliases, outputAliasReferences);
|
||||
}
|
||||
|
||||
private void markOutputAliasReferences(
|
||||
SqlNode node,
|
||||
Set<String> outputAliases,
|
||||
Set<SqlIdentifier> outputAliasReferences) {
|
||||
if (node == null) {
|
||||
return;
|
||||
}
|
||||
if (node instanceof SqlNodeList nodes) {
|
||||
for (SqlNode item : nodes) {
|
||||
markOutputAliasReferences(item, outputAliases, outputAliasReferences);
|
||||
}
|
||||
return;
|
||||
}
|
||||
if (node instanceof SqlIdentifier identifier && identifier.names.size() == 1) {
|
||||
if (outputAliases.contains(identifierKey(
|
||||
identifier.names.get(0), identifier.isComponentQuoted(0)))) {
|
||||
outputAliasReferences.add(identifier);
|
||||
}
|
||||
return;
|
||||
}
|
||||
if (node instanceof SqlCall call
|
||||
&& (call.getKind() == SqlKind.DESCENDING
|
||||
|| call.getKind() == SqlKind.NULLS_FIRST
|
||||
|| call.getKind() == SqlKind.NULLS_LAST)
|
||||
&& call.operandCount() > 0) {
|
||||
markOutputAliasReferences(
|
||||
call.operand(0), outputAliases, outputAliasReferences);
|
||||
}
|
||||
}
|
||||
|
||||
private FromSummary collectFrom(
|
||||
SqlNode from,
|
||||
PolicySnapshot snapshot,
|
||||
Set<BigInteger> referencedTableIds,
|
||||
Map<String, DerivedRelation> commonTables,
|
||||
Map<String, Set<BigInteger>> tableIdsByAlias,
|
||||
Map<String, Set<String>> derivedColumnsByAlias,
|
||||
Set<SqlIdentifier> syntaxAliases,
|
||||
Set<SqlIdentifier> tableIdentifiers) {
|
||||
if (from == null) {
|
||||
return FromSummary.empty();
|
||||
}
|
||||
if (from instanceof SqlIdentifier identifier) {
|
||||
tableIdentifiers.add(identifier);
|
||||
String relationKey = identifierKey(
|
||||
identifier.names.get(identifier.names.size() - 1),
|
||||
identifier.isComponentQuoted(identifier.names.size() - 1));
|
||||
DerivedRelation commonTable = commonTables.get(relationKey);
|
||||
if (commonTable != null) {
|
||||
tableIdsByAlias.put(relationKey, commonTable.tableIds());
|
||||
derivedColumnsByAlias.put(
|
||||
relationKey, commonTable.outputColumns());
|
||||
return new FromSummary(
|
||||
commonTable.tableIds(), commonTable.outputColumns(), true);
|
||||
}
|
||||
return new FromSummary(
|
||||
resolveReferencedTableIds(
|
||||
identifier, snapshot, referencedTableIds),
|
||||
Set.of(),
|
||||
false);
|
||||
}
|
||||
if (!(from instanceof SqlCall call)) {
|
||||
return FromSummary.empty();
|
||||
}
|
||||
if (call instanceof SqlSelect nestedSelect) {
|
||||
FromSummary nested = collectFrom(
|
||||
nestedSelect.getFrom(), snapshot, referencedTableIds,
|
||||
commonTables, tableIdsByAlias, derivedColumnsByAlias,
|
||||
syntaxAliases, tableIdentifiers);
|
||||
return new FromSummary(
|
||||
nested.tableIds(), outputColumnKeys(nestedSelect), true);
|
||||
}
|
||||
if (call instanceof SqlJoin join) {
|
||||
FromSummary left = collectFrom(
|
||||
join.getLeft(), snapshot, referencedTableIds,
|
||||
commonTables, tableIdsByAlias, derivedColumnsByAlias,
|
||||
syntaxAliases, tableIdentifiers);
|
||||
FromSummary right = collectFrom(
|
||||
join.getRight(), snapshot, referencedTableIds,
|
||||
commonTables, tableIdsByAlias, derivedColumnsByAlias,
|
||||
syntaxAliases, tableIdentifiers);
|
||||
Set<BigInteger> tableIds = new LinkedHashSet<>(left.tableIds());
|
||||
tableIds.addAll(right.tableIds());
|
||||
return new FromSummary(Set.copyOf(tableIds), Set.of(), false);
|
||||
}
|
||||
if (call.getKind() == SqlKind.AS && call.operandCount() >= 2) {
|
||||
SqlNode relationNode = call.operand(0);
|
||||
FromSummary relation = collectFrom(
|
||||
relationNode, snapshot, referencedTableIds,
|
||||
commonTables, tableIdsByAlias, derivedColumnsByAlias,
|
||||
syntaxAliases, tableIdentifiers);
|
||||
SqlNode aliasNode = call.operand(1);
|
||||
if (aliasNode instanceof SqlIdentifier alias && !alias.names.isEmpty()) {
|
||||
int aliasIndex = alias.names.size() - 1;
|
||||
String aliasKey = identifierKey(
|
||||
alias.names.get(aliasIndex),
|
||||
alias.isComponentQuoted(aliasIndex));
|
||||
syntaxAliases.add(alias);
|
||||
if (!relation.tableIds().isEmpty()) {
|
||||
tableIdsByAlias.merge(aliasKey, relation.tableIds(), (left, right) -> {
|
||||
Set<BigInteger> merged = new LinkedHashSet<>(left);
|
||||
merged.addAll(right);
|
||||
return Set.copyOf(merged);
|
||||
});
|
||||
}
|
||||
if (relation.singleDerived()
|
||||
&& !relation.outputColumns().isEmpty()) {
|
||||
derivedColumnsByAlias.put(
|
||||
aliasKey, relation.outputColumns());
|
||||
}
|
||||
}
|
||||
return relation;
|
||||
}
|
||||
if (call instanceof SqlOrderBy orderBy) {
|
||||
return collectFrom(
|
||||
orderBy.query, snapshot, referencedTableIds,
|
||||
commonTables, tableIdsByAlias, derivedColumnsByAlias,
|
||||
syntaxAliases, tableIdentifiers);
|
||||
}
|
||||
if (call instanceof SqlWith sqlWith) {
|
||||
return collectFrom(
|
||||
sqlWith.body, snapshot, referencedTableIds,
|
||||
commonTables, tableIdsByAlias, derivedColumnsByAlias,
|
||||
syntaxAliases, tableIdentifiers);
|
||||
}
|
||||
return FromSummary.empty();
|
||||
}
|
||||
|
||||
private Map<String, DerivedRelation> collectCommonTables(
|
||||
SqlNode root,
|
||||
PolicySnapshot snapshot,
|
||||
Set<BigInteger> referencedTableIds) {
|
||||
Map<String, DerivedRelation> commonTables = new HashMap<>();
|
||||
collectCommonTables(
|
||||
root, snapshot, referencedTableIds, commonTables);
|
||||
return Map.copyOf(commonTables);
|
||||
}
|
||||
|
||||
private void collectCommonTables(
|
||||
SqlNode node,
|
||||
PolicySnapshot snapshot,
|
||||
Set<BigInteger> referencedTableIds,
|
||||
Map<String, DerivedRelation> commonTables) {
|
||||
if (node instanceof SqlOrderBy orderBy) {
|
||||
collectCommonTables(
|
||||
orderBy.query, snapshot, referencedTableIds, commonTables);
|
||||
return;
|
||||
}
|
||||
if (!(node instanceof SqlWith sqlWith)) {
|
||||
return;
|
||||
}
|
||||
for (SqlNode withNode : sqlWith.withList) {
|
||||
if (!(withNode instanceof SqlWithItem withItem)
|
||||
|| withItem.name == null
|
||||
|| withItem.name.names.isEmpty()) {
|
||||
continue;
|
||||
}
|
||||
collectCommonTables(
|
||||
withItem.query, snapshot, referencedTableIds, commonTables);
|
||||
SqlSelect select = selectBody(withItem.query);
|
||||
if (select == null) {
|
||||
continue;
|
||||
}
|
||||
FromSummary source = collectFrom(
|
||||
select.getFrom(), snapshot, referencedTableIds,
|
||||
commonTables, new HashMap<>(), new HashMap<>(),
|
||||
Collections.newSetFromMap(new IdentityHashMap<>()),
|
||||
Collections.newSetFromMap(new IdentityHashMap<>()));
|
||||
Set<String> outputColumns = withItem.columnList == null
|
||||
|| withItem.columnList.isEmpty()
|
||||
? outputColumnKeys(select)
|
||||
: identifierKeys(withItem.columnList);
|
||||
int nameIndex = withItem.name.names.size() - 1;
|
||||
commonTables.put(
|
||||
identifierKey(
|
||||
withItem.name.names.get(nameIndex),
|
||||
withItem.name.isComponentQuoted(nameIndex)),
|
||||
new DerivedRelation(
|
||||
source.tableIds(), outputColumns));
|
||||
}
|
||||
collectCommonTables(
|
||||
sqlWith.body, snapshot, referencedTableIds, commonTables);
|
||||
}
|
||||
|
||||
private SqlSelect selectBody(SqlNode node) {
|
||||
if (node instanceof SqlSelect select) {
|
||||
return select;
|
||||
}
|
||||
if (node instanceof SqlOrderBy orderBy) {
|
||||
return selectBody(orderBy.query);
|
||||
}
|
||||
if (node instanceof SqlWith sqlWith) {
|
||||
return selectBody(sqlWith.body);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
private Set<String> outputColumnKeys(SqlNode query) {
|
||||
SqlSelect select = selectBody(query);
|
||||
if (select == null || select.getSelectList() == null) {
|
||||
return Set.of();
|
||||
}
|
||||
Set<String> outputColumns = new LinkedHashSet<>();
|
||||
for (SqlNode item : select.getSelectList()) {
|
||||
if (item instanceof SqlCall call
|
||||
&& call.getKind() == SqlKind.AS
|
||||
&& call.operandCount() >= 2
|
||||
&& call.operand(1) instanceof SqlIdentifier alias
|
||||
&& !alias.names.isEmpty()) {
|
||||
int index = alias.names.size() - 1;
|
||||
outputColumns.add(identifierKey(
|
||||
alias.names.get(index),
|
||||
alias.isComponentQuoted(index)));
|
||||
continue;
|
||||
}
|
||||
if (item instanceof SqlIdentifier identifier
|
||||
&& !identifier.isStar()
|
||||
&& !identifier.names.isEmpty()) {
|
||||
int index = identifier.names.size() - 1;
|
||||
outputColumns.add(identifierKey(
|
||||
identifier.names.get(index),
|
||||
identifier.isComponentQuoted(index)));
|
||||
}
|
||||
}
|
||||
return Set.copyOf(outputColumns);
|
||||
}
|
||||
|
||||
private Set<String> identifierKeys(SqlNodeList identifiers) {
|
||||
Set<String> keys = new LinkedHashSet<>();
|
||||
for (SqlNode node : identifiers) {
|
||||
if (node instanceof SqlIdentifier identifier
|
||||
&& !identifier.names.isEmpty()) {
|
||||
int index = identifier.names.size() - 1;
|
||||
keys.add(identifierKey(
|
||||
identifier.names.get(index),
|
||||
identifier.isComponentQuoted(index)));
|
||||
}
|
||||
}
|
||||
return Set.copyOf(keys);
|
||||
}
|
||||
|
||||
private Set<BigInteger> resolveReferencedTableIds(
|
||||
SqlIdentifier identifier,
|
||||
PolicySnapshot snapshot,
|
||||
Set<BigInteger> referencedTableIds) {
|
||||
Set<BigInteger> candidates = snapshot.tableIdsByName().get(
|
||||
String.join(".", identifier.names));
|
||||
if (candidates == null || candidates.isEmpty()) {
|
||||
return Set.of();
|
||||
}
|
||||
Set<BigInteger> referenced = new LinkedHashSet<>(candidates);
|
||||
referenced.retainAll(referencedTableIds);
|
||||
return Set.copyOf(referenced);
|
||||
}
|
||||
|
||||
private String identifierKey(String name, boolean quoted) {
|
||||
return quoted ? "Q:" + name : "U:" + name.toUpperCase(Locale.ROOT);
|
||||
}
|
||||
|
||||
private boolean columnAllowed(ColumnPolicy policy, String columnName, boolean quoted) {
|
||||
if (policy == null) {
|
||||
return false;
|
||||
}
|
||||
return quoted
|
||||
? policy.allowedExact().contains(columnName)
|
||||
: policy.allowedFolded().contains(columnName.toUpperCase(Locale.ROOT));
|
||||
}
|
||||
|
||||
private boolean fieldAllowed(DatacenterTableField field) {
|
||||
return Integer.valueOf(1).equals(field.getQueryable())
|
||||
&& DatacenterMetadataStatus.ACTIVE.name().equals(field.getMetadataStatus())
|
||||
&& DatacenterSensitivityLevel.PUBLIC.name().equals(field.getSensitivityLevel());
|
||||
}
|
||||
|
||||
private void addTableName(
|
||||
Map<String, Set<BigInteger>> idsByName,
|
||||
Set<String> tableReferenceNames,
|
||||
String name,
|
||||
BigInteger tableId) {
|
||||
if (name == null || name.isBlank()) {
|
||||
return;
|
||||
}
|
||||
idsByName.computeIfAbsent(name, ignored -> new LinkedHashSet<>()).add(tableId);
|
||||
tableReferenceNames.add(name);
|
||||
}
|
||||
|
||||
private Set<String> referencedTables(RelNode root) {
|
||||
Set<String> names = new LinkedHashSet<>();
|
||||
new RelVisitor() {
|
||||
@Override
|
||||
public void visit(RelNode node, int ordinal, RelNode parent) {
|
||||
if (node instanceof TableScan scan) {
|
||||
names.add(referencedTableName(scan));
|
||||
}
|
||||
super.visit(node, ordinal, parent);
|
||||
}
|
||||
}.go(root);
|
||||
return names;
|
||||
}
|
||||
|
||||
private String referencedTableName(TableScan scan) {
|
||||
List<String> qualified = scan.getTable().getQualifiedName();
|
||||
if (qualified.isEmpty()) {
|
||||
return "";
|
||||
}
|
||||
String table = qualified.get(qualified.size() - 1);
|
||||
return qualified.size() >= 2
|
||||
? qualified.get(qualified.size() - 2) + "." + table
|
||||
: table;
|
||||
}
|
||||
|
||||
private BigInteger parseRecordId(String sourceId) {
|
||||
int marker = sourceId.lastIndexOf("-source-");
|
||||
if (marker < 0) {
|
||||
throw new FederationSqlException(
|
||||
FederationSqlErrorCode.INVALID_ARGUMENT,
|
||||
"invalid data source id");
|
||||
}
|
||||
return new BigInteger(sourceId.substring(marker + "-source-".length()));
|
||||
}
|
||||
|
||||
private String sourceColumnName(DatacenterTableField field) {
|
||||
return field.getSourceColumnName() == null || field.getSourceColumnName().isBlank()
|
||||
? field.getFieldName() : field.getSourceColumnName();
|
||||
}
|
||||
|
||||
private FederationSqlException validationFailure(String message) {
|
||||
return new FederationSqlException(FederationSqlErrorCode.SQL_VALIDATION_FAILED, message);
|
||||
}
|
||||
|
||||
private record PolicyCacheKey(BigInteger sourceId, String policyVersion) {
|
||||
}
|
||||
|
||||
private record AliasContext(
|
||||
Map<String, Set<BigInteger>> tableIdsByAlias,
|
||||
Map<String, Set<String>> derivedColumnsByAlias,
|
||||
Set<String> unqualifiedDerivedColumns,
|
||||
Set<String> outputAliases,
|
||||
Set<SqlIdentifier> syntaxAliases,
|
||||
Set<SqlIdentifier> tableIdentifiers,
|
||||
Set<SqlIdentifier> outputAliasReferences) {
|
||||
}
|
||||
|
||||
private record DerivedRelation(
|
||||
Set<BigInteger> tableIds,
|
||||
Set<String> outputColumns) {
|
||||
}
|
||||
|
||||
private record FromSummary(
|
||||
Set<BigInteger> tableIds,
|
||||
Set<String> outputColumns,
|
||||
boolean singleDerived) {
|
||||
|
||||
private static FromSummary empty() {
|
||||
return new FromSummary(Set.of(), Set.of(), false);
|
||||
}
|
||||
}
|
||||
|
||||
private record ColumnPolicy(
|
||||
Set<String> knownExact,
|
||||
Set<String> knownFolded,
|
||||
Set<String> allowedExact,
|
||||
Set<String> allowedFolded) {
|
||||
|
||||
private static ColumnPolicy mutable() {
|
||||
return new ColumnPolicy(new HashSet<>(), new HashSet<>(), new HashSet<>(), new HashSet<>());
|
||||
}
|
||||
}
|
||||
|
||||
private record PolicySnapshot(
|
||||
Map<String, Set<BigInteger>> tableIdsByName,
|
||||
Map<BigInteger, ColumnPolicy> columnsByTable,
|
||||
Set<String> tableReferenceNames) {
|
||||
|
||||
private static PolicySnapshot empty() {
|
||||
return new PolicySnapshot(Map.of(), Map.of(), Set.of());
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,174 @@
|
||||
package tech.easyflow.datacenter.federation;
|
||||
|
||||
import java.nio.ByteBuffer;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.security.MessageDigest;
|
||||
import java.security.NoSuchAlgorithmException;
|
||||
import java.util.HexFormat;
|
||||
import java.util.List;
|
||||
import tech.easyflow.datacenter.entity.DatacenterTableField;
|
||||
|
||||
/**
|
||||
* 使用 presence 与长度前缀生成稳定元数据摘要。
|
||||
*/
|
||||
public final class DatacenterMetadataIdentity {
|
||||
|
||||
private DatacenterMetadataIdentity() {
|
||||
}
|
||||
|
||||
/**
|
||||
* 计算物理命名空间摘要。
|
||||
*
|
||||
* @param catalog 物理 Catalog
|
||||
* @param schema 物理 Schema
|
||||
* @return SHA-256 摘要
|
||||
*/
|
||||
public static String namespaceKey(String catalog, String schema) {
|
||||
return digest(List.of(nullable(catalog), nullable(schema)));
|
||||
}
|
||||
|
||||
/**
|
||||
* 计算物理表身份摘要。
|
||||
*
|
||||
* @param sourceId 数据源 ID
|
||||
* @param namespaceKey 命名空间摘要
|
||||
* @param physicalTable 物理表名
|
||||
* @param tableKind 对象类型
|
||||
* @return SHA-256 摘要
|
||||
*/
|
||||
public static String tableIdentity(
|
||||
String sourceId,
|
||||
String namespaceKey,
|
||||
String physicalTable,
|
||||
String tableKind) {
|
||||
return digest(List.of(
|
||||
"table-v1",
|
||||
nullable(sourceId),
|
||||
nullable(namespaceKey),
|
||||
nullable(physicalTable),
|
||||
nullable(tableKind)));
|
||||
}
|
||||
|
||||
/**
|
||||
* 计算表结构观测摘要。
|
||||
*
|
||||
* @param tableName 表名
|
||||
* @param tableKind 对象类型
|
||||
* @param fields 按 JDBC 顺序排列的字段
|
||||
* @return SHA-256 摘要
|
||||
*/
|
||||
public static String tableFingerprint(
|
||||
String tableName,
|
||||
String tableKind,
|
||||
List<DatacenterTableField> fields) {
|
||||
MessageDigest digest = newDigest();
|
||||
update(digest, "table-metadata-v1");
|
||||
update(digest, nullable(tableName));
|
||||
update(digest, nullable(tableKind));
|
||||
List<DatacenterTableField> safeFields = fields == null ? List.of() : fields;
|
||||
digest.update(ByteBuffer.allocate(Integer.BYTES).putInt(safeFields.size()).array());
|
||||
for (DatacenterTableField field : safeFields) {
|
||||
update(digest, nullable(field.getSourceColumnName()));
|
||||
update(digest, nullable(field.getNativeTypeName()));
|
||||
update(digest, nullable(field.getJdbcTypeCode()));
|
||||
update(digest, nullable(field.getPrecision()));
|
||||
update(digest, nullable(field.getScale()));
|
||||
update(digest, nullable(field.getRequired()));
|
||||
}
|
||||
return HexFormat.of().formatHex(digest.digest());
|
||||
}
|
||||
|
||||
/**
|
||||
* 计算单字段元数据摘要。
|
||||
*
|
||||
* @param field 字段元数据
|
||||
* @return SHA-256 摘要
|
||||
*/
|
||||
public static String fieldFingerprint(DatacenterTableField field) {
|
||||
return digest(List.of(
|
||||
"field-v1",
|
||||
nullable(field.getSourceColumnName()),
|
||||
nullable(field.getNativeTypeName()),
|
||||
nullable(field.getJdbcTypeCode()),
|
||||
nullable(field.getPrecision()),
|
||||
nullable(field.getScale()),
|
||||
nullable(field.getRequired())));
|
||||
}
|
||||
|
||||
/**
|
||||
* 复现 V58 的存量表身份回填算法,仅用于首次刷新时识别可安全升级的旧摘要。
|
||||
*
|
||||
* @param sourceId 数据源 ID
|
||||
* @param catalogId 目录 ID
|
||||
* @param physicalTable 物理表名
|
||||
* @param tableKind 表类型
|
||||
* @return V58 SHA-256 摘要
|
||||
*/
|
||||
public static String legacyV58TableIdentity(
|
||||
Object sourceId,
|
||||
Object catalogId,
|
||||
String physicalTable,
|
||||
String tableKind) {
|
||||
String source = sourceId == null ? "0" : String.valueOf(sourceId);
|
||||
String catalog = catalogId == null ? "0" : String.valueOf(catalogId);
|
||||
String table = physicalTable == null ? "" : physicalTable;
|
||||
String kind = tableKind == null ? "" : tableKind;
|
||||
return rawDigest("S" + codePointLength(source) + ":" + source
|
||||
+ "C" + codePointLength(catalog) + ":" + catalog
|
||||
+ "T" + codePointLength(table) + ":" + table
|
||||
+ "K" + codePointLength(kind) + ":" + kind);
|
||||
}
|
||||
|
||||
/**
|
||||
* 复现 V58 的存量表结构摘要回填算法。
|
||||
*
|
||||
* @param tableName 表名
|
||||
* @param tableKind 表类型
|
||||
* @return V58 SHA-256 摘要
|
||||
*/
|
||||
public static String legacyV58TableFingerprint(String tableName, String tableKind) {
|
||||
String table = tableName == null ? "" : tableName;
|
||||
String kind = tableKind == null ? "" : tableKind;
|
||||
return rawDigest(codePointLength(table) + ":" + table
|
||||
+ codePointLength(kind) + ":" + kind);
|
||||
}
|
||||
|
||||
private static int codePointLength(String value) {
|
||||
return value.codePointCount(0, value.length());
|
||||
}
|
||||
|
||||
private static String rawDigest(String value) {
|
||||
MessageDigest digest = newDigest();
|
||||
return HexFormat.of().formatHex(digest.digest(value.getBytes(StandardCharsets.UTF_8)));
|
||||
}
|
||||
|
||||
private static String digest(List<String> values) {
|
||||
MessageDigest digest = newDigest();
|
||||
for (String value : values) {
|
||||
update(digest, value);
|
||||
}
|
||||
return HexFormat.of().formatHex(digest.digest());
|
||||
}
|
||||
|
||||
private static void update(MessageDigest digest, String value) {
|
||||
byte[] bytes = value.getBytes(StandardCharsets.UTF_8);
|
||||
digest.update(ByteBuffer.allocate(Integer.BYTES).putInt(bytes.length).array());
|
||||
digest.update(bytes);
|
||||
}
|
||||
|
||||
private static String nullable(Object value) {
|
||||
if (value == null) {
|
||||
return "N";
|
||||
}
|
||||
String text = String.valueOf(value);
|
||||
return "V" + text.length() + ":" + text;
|
||||
}
|
||||
|
||||
private static MessageDigest newDigest() {
|
||||
try {
|
||||
return MessageDigest.getInstance("SHA-256");
|
||||
} catch (NoSuchAlgorithmException exception) {
|
||||
throw new IllegalStateException("SHA-256 is not available", exception);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,7 +1,18 @@
|
||||
package tech.easyflow.datacenter.integration;
|
||||
|
||||
import tech.easyflow.common.entity.LoginAccount;
|
||||
import tech.easyflow.datacenter.execution.model.DatacenterQueryRequest;
|
||||
|
||||
public interface AssistantDatacenterBridge {
|
||||
AssistantDatacenterResult queryPage(DatacenterQueryRequest request);
|
||||
|
||||
/**
|
||||
* 使用明确执行账号查询数据中枢。
|
||||
*
|
||||
* @param request 查询请求
|
||||
* @param account 执行账号
|
||||
* @return 查询结果
|
||||
*/
|
||||
AssistantDatacenterResult queryPage(
|
||||
DatacenterQueryRequest request,
|
||||
LoginAccount account);
|
||||
}
|
||||
|
||||
@@ -4,7 +4,7 @@ import com.mybatisflex.core.row.Row;
|
||||
import tech.easyflow.datacenter.entity.DatacenterTable;
|
||||
import tech.easyflow.datacenter.meta.entity.DatacenterCatalog;
|
||||
import tech.easyflow.datacenter.meta.entity.DatacenterDatasetVersion;
|
||||
import tech.easyflow.datacenter.meta.entity.DatacenterSource;
|
||||
import tech.easyflow.datacenter.meta.model.DatacenterSourceView;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.LinkedHashMap;
|
||||
@@ -13,7 +13,7 @@ import java.util.Map;
|
||||
|
||||
public class AssistantDatacenterResult {
|
||||
private List<Row> rows = new ArrayList<>();
|
||||
private DatacenterSource source;
|
||||
private DatacenterSourceView source;
|
||||
private DatacenterCatalog catalog;
|
||||
private DatacenterTable table;
|
||||
private DatacenterDatasetVersion version;
|
||||
@@ -21,8 +21,8 @@ public class AssistantDatacenterResult {
|
||||
|
||||
public List<Row> getRows() { return rows; }
|
||||
public void setRows(List<Row> rows) { this.rows = rows; }
|
||||
public DatacenterSource getSource() { return source; }
|
||||
public void setSource(DatacenterSource source) { this.source = source; }
|
||||
public DatacenterSourceView getSource() { return source; }
|
||||
public void setSource(DatacenterSourceView source) { this.source = source; }
|
||||
public DatacenterCatalog getCatalog() { return catalog; }
|
||||
public void setCatalog(DatacenterCatalog catalog) { this.catalog = catalog; }
|
||||
public DatacenterTable getTable() { return table; }
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
package tech.easyflow.datacenter.integration;
|
||||
|
||||
import org.springframework.stereotype.Component;
|
||||
import tech.easyflow.common.entity.LoginAccount;
|
||||
import tech.easyflow.datacenter.execution.model.DatacenterQueryRequest;
|
||||
import tech.easyflow.datacenter.execution.model.DatacenterSchemaResponse;
|
||||
import tech.easyflow.datacenter.execution.service.DatacenterDatasetQueryService;
|
||||
@@ -17,8 +18,10 @@ public class DefaultAssistantDatacenterBridge implements AssistantDatacenterBrid
|
||||
private DatacenterDatasetQueryService queryService;
|
||||
|
||||
@Override
|
||||
public AssistantDatacenterResult queryPage(DatacenterQueryRequest request) {
|
||||
var page = queryService.queryPage(request);
|
||||
public AssistantDatacenterResult queryPage(
|
||||
DatacenterQueryRequest request,
|
||||
LoginAccount account) {
|
||||
var page = queryService.queryPage(request, account);
|
||||
DatacenterSchemaResponse schema = queryService.getSchema(request.getDatasetRef());
|
||||
AssistantDatacenterResult result = new AssistantDatacenterResult();
|
||||
result.setRows(page.getRecords());
|
||||
|
||||
@@ -0,0 +1,10 @@
|
||||
package tech.easyflow.datacenter.mapper;
|
||||
|
||||
import com.mybatisflex.core.BaseMapper;
|
||||
import tech.easyflow.datacenter.audit.DatacenterQueryAudit;
|
||||
|
||||
/**
|
||||
* 数据中枢查询审计 Mapper。
|
||||
*/
|
||||
public interface DatacenterQueryAuditMapper extends BaseMapper<DatacenterQueryAudit> {
|
||||
}
|
||||
@@ -31,6 +31,18 @@ public class DatacenterCatalog extends DateEntity implements Serializable {
|
||||
private String catalogDesc;
|
||||
@Column(comment = "目录类型")
|
||||
private String catalogType;
|
||||
@Column(comment = "Calcite 逻辑 Schema")
|
||||
private String logicalSchemaName;
|
||||
@Column(comment = "JDBC 物理 Catalog")
|
||||
private String physicalCatalogName;
|
||||
@Column(comment = "JDBC 物理 Schema")
|
||||
private String physicalSchemaName;
|
||||
@Column(comment = "物理命名空间稳定摘要")
|
||||
private String namespaceKey;
|
||||
@Column(comment = "元数据状态")
|
||||
private String metadataStatus;
|
||||
@Column(comment = "最近发现时间")
|
||||
private Date lastSeenAt;
|
||||
@Column(comment = "状态")
|
||||
private Integer status;
|
||||
@Column(typeHandler = FastjsonTypeHandler.class, comment = "扩展项")
|
||||
@@ -59,6 +71,18 @@ public class DatacenterCatalog extends DateEntity implements Serializable {
|
||||
public void setCatalogDesc(String catalogDesc) { this.catalogDesc = catalogDesc; }
|
||||
public String getCatalogType() { return catalogType; }
|
||||
public void setCatalogType(String catalogType) { this.catalogType = catalogType; }
|
||||
public String getLogicalSchemaName() { return logicalSchemaName; }
|
||||
public void setLogicalSchemaName(String logicalSchemaName) { this.logicalSchemaName = logicalSchemaName; }
|
||||
public String getPhysicalCatalogName() { return physicalCatalogName; }
|
||||
public void setPhysicalCatalogName(String physicalCatalogName) { this.physicalCatalogName = physicalCatalogName; }
|
||||
public String getPhysicalSchemaName() { return physicalSchemaName; }
|
||||
public void setPhysicalSchemaName(String physicalSchemaName) { this.physicalSchemaName = physicalSchemaName; }
|
||||
public String getNamespaceKey() { return namespaceKey; }
|
||||
public void setNamespaceKey(String namespaceKey) { this.namespaceKey = namespaceKey; }
|
||||
public String getMetadataStatus() { return metadataStatus; }
|
||||
public void setMetadataStatus(String metadataStatus) { this.metadataStatus = metadataStatus; }
|
||||
public Date getLastSeenAt() { return lastSeenAt; }
|
||||
public void setLastSeenAt(Date lastSeenAt) { this.lastSeenAt = lastSeenAt; }
|
||||
public Integer getStatus() { return status; }
|
||||
public void setStatus(Integer status) { this.status = status; }
|
||||
public Map<String, Object> getOptions() { return options; }
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
package tech.easyflow.datacenter.meta.entity;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonIgnore;
|
||||
import com.fasterxml.jackson.databind.annotation.JsonSerialize;
|
||||
import com.fasterxml.jackson.databind.ser.std.ToStringSerializer;
|
||||
import com.mybatisflex.annotation.Column;
|
||||
@@ -29,6 +30,30 @@ public class DatacenterSource extends DateEntity implements Serializable {
|
||||
private String sourceCode;
|
||||
@Column(comment = "数据源类型")
|
||||
private String sourceType;
|
||||
@Column(comment = "Federation Adapter 标识")
|
||||
private String adapterId;
|
||||
@Column(typeHandler = FastjsonTypeHandler.class, comment = "不含凭据的 Adapter 选项")
|
||||
private Map<String, String> adapterOptionsJson;
|
||||
@Column(comment = "数据源 Definition 版本")
|
||||
private Long definitionRevision;
|
||||
@Column(comment = "Definition SHA-256 校验和")
|
||||
private String definitionChecksum;
|
||||
@Column(comment = "纳管范围版本")
|
||||
private Long scopeRevision;
|
||||
@Column(comment = "Adapter 兼容状态")
|
||||
private String compatibilityStatus;
|
||||
@Column(comment = "数据库产品名称")
|
||||
private String databaseProductName;
|
||||
@Column(comment = "数据库产品版本")
|
||||
private String databaseProductVersion;
|
||||
@Column(comment = "JDBC Driver 名称")
|
||||
private String driverName;
|
||||
@Column(comment = "JDBC Driver 版本")
|
||||
private String driverVersion;
|
||||
@Column(comment = "元数据刷新状态")
|
||||
private String metadataRefreshStatus;
|
||||
@Column(comment = "元数据最近刷新时间")
|
||||
private Date metadataRefreshedAt;
|
||||
@Column(comment = "访问模式")
|
||||
private String accessMode;
|
||||
@Column(comment = "是否内置")
|
||||
@@ -85,6 +110,30 @@ public class DatacenterSource extends DateEntity implements Serializable {
|
||||
public void setSourceCode(String sourceCode) { this.sourceCode = sourceCode; }
|
||||
public String getSourceType() { return sourceType; }
|
||||
public void setSourceType(String sourceType) { this.sourceType = sourceType; }
|
||||
public String getAdapterId() { return adapterId; }
|
||||
public void setAdapterId(String adapterId) { this.adapterId = adapterId; }
|
||||
public Map<String, String> getAdapterOptionsJson() { return adapterOptionsJson; }
|
||||
public void setAdapterOptionsJson(Map<String, String> adapterOptionsJson) { this.adapterOptionsJson = adapterOptionsJson; }
|
||||
public Long getDefinitionRevision() { return definitionRevision; }
|
||||
public void setDefinitionRevision(Long definitionRevision) { this.definitionRevision = definitionRevision; }
|
||||
public String getDefinitionChecksum() { return definitionChecksum; }
|
||||
public void setDefinitionChecksum(String definitionChecksum) { this.definitionChecksum = definitionChecksum; }
|
||||
public Long getScopeRevision() { return scopeRevision; }
|
||||
public void setScopeRevision(Long scopeRevision) { this.scopeRevision = scopeRevision; }
|
||||
public String getCompatibilityStatus() { return compatibilityStatus; }
|
||||
public void setCompatibilityStatus(String compatibilityStatus) { this.compatibilityStatus = compatibilityStatus; }
|
||||
public String getDatabaseProductName() { return databaseProductName; }
|
||||
public void setDatabaseProductName(String databaseProductName) { this.databaseProductName = databaseProductName; }
|
||||
public String getDatabaseProductVersion() { return databaseProductVersion; }
|
||||
public void setDatabaseProductVersion(String databaseProductVersion) { this.databaseProductVersion = databaseProductVersion; }
|
||||
public String getDriverName() { return driverName; }
|
||||
public void setDriverName(String driverName) { this.driverName = driverName; }
|
||||
public String getDriverVersion() { return driverVersion; }
|
||||
public void setDriverVersion(String driverVersion) { this.driverVersion = driverVersion; }
|
||||
public String getMetadataRefreshStatus() { return metadataRefreshStatus; }
|
||||
public void setMetadataRefreshStatus(String metadataRefreshStatus) { this.metadataRefreshStatus = metadataRefreshStatus; }
|
||||
public Date getMetadataRefreshedAt() { return metadataRefreshedAt; }
|
||||
public void setMetadataRefreshedAt(Date metadataRefreshedAt) { this.metadataRefreshedAt = metadataRefreshedAt; }
|
||||
public String getAccessMode() { return accessMode; }
|
||||
public void setAccessMode(String accessMode) { this.accessMode = accessMode; }
|
||||
public Integer getBuiltinFlag() { return builtinFlag; }
|
||||
@@ -103,8 +152,10 @@ public class DatacenterSource extends DateEntity implements Serializable {
|
||||
public void setSchemaName(String schemaName) { this.schemaName = schemaName; }
|
||||
public String getUsername() { return username; }
|
||||
public void setUsername(String username) { this.username = username; }
|
||||
@JsonIgnore
|
||||
public String getCredentialCipher() { return credentialCipher; }
|
||||
public void setCredentialCipher(String credentialCipher) { this.credentialCipher = credentialCipher; }
|
||||
@JsonIgnore
|
||||
public Map<String, Object> getConfigJson() { return configJson; }
|
||||
public void setConfigJson(Map<String, Object> configJson) { this.configJson = configJson; }
|
||||
public Map<String, Object> getCapabilitiesJson() { return capabilitiesJson; }
|
||||
|
||||
@@ -0,0 +1,15 @@
|
||||
package tech.easyflow.datacenter.meta.enums;
|
||||
|
||||
/**
|
||||
* 数据库对象元数据状态。
|
||||
*/
|
||||
public enum DatacenterMetadataStatus {
|
||||
/** 当前仍能从目标数据库发现。 */
|
||||
ACTIVE,
|
||||
/** 本次刷新未发现。 */
|
||||
MISSING,
|
||||
/** 结构已变化,等待用户确认。 */
|
||||
CHANGED,
|
||||
/** 已由用户停止纳管。 */
|
||||
RETIRED
|
||||
}
|
||||
@@ -0,0 +1,15 @@
|
||||
package tech.easyflow.datacenter.meta.enums;
|
||||
|
||||
/**
|
||||
* 数据字段敏感级别。
|
||||
*/
|
||||
public enum DatacenterSensitivityLevel {
|
||||
/** 可直接用于受控查询。 */
|
||||
PUBLIC,
|
||||
/** 仅限组织内部。 */
|
||||
INTERNAL,
|
||||
/** 需要脱敏后使用。 */
|
||||
SENSITIVE,
|
||||
/** 默认禁止进入查询结果。 */
|
||||
RESTRICTED
|
||||
}
|
||||
@@ -0,0 +1,56 @@
|
||||
package tech.easyflow.datacenter.meta.enums;
|
||||
|
||||
import java.util.Arrays;
|
||||
|
||||
/**
|
||||
* 数据源持久化生命周期。
|
||||
*/
|
||||
public enum DatacenterSourceStatus {
|
||||
|
||||
/** 配置已保存,尚未激活。 */
|
||||
DRAFT(0),
|
||||
/** Definition 已发布且节点可查询。 */
|
||||
READY(1),
|
||||
/** 配置仍保留,但当前 Runtime 不可用。 */
|
||||
DEGRADED(2),
|
||||
/** 管理员主动禁用。 */
|
||||
DISABLED(3),
|
||||
/** 删除墓碑,禁止新查询。 */
|
||||
DELETED(4);
|
||||
|
||||
private final int code;
|
||||
|
||||
/**
|
||||
* 创建状态。
|
||||
*
|
||||
* @param code 持久化编码
|
||||
*/
|
||||
DatacenterSourceStatus(int code) {
|
||||
this.code = code;
|
||||
}
|
||||
|
||||
/**
|
||||
* 返回持久化编码。
|
||||
*
|
||||
* @return 状态编码
|
||||
*/
|
||||
public int code() {
|
||||
return code;
|
||||
}
|
||||
|
||||
/**
|
||||
* 按持久化编码解析状态。
|
||||
*
|
||||
* @param code 状态编码
|
||||
* @return 对应状态;未知编码按草稿处理
|
||||
*/
|
||||
public static DatacenterSourceStatus fromCode(Integer code) {
|
||||
if (code == null) {
|
||||
return DRAFT;
|
||||
}
|
||||
return Arrays.stream(values())
|
||||
.filter(status -> status.code == code)
|
||||
.findFirst()
|
||||
.orElse(DRAFT);
|
||||
}
|
||||
}
|
||||
@@ -14,6 +14,9 @@ public class DatacenterCatalogMeta {
|
||||
private String catalogName;
|
||||
private String catalogType;
|
||||
private String catalogDesc;
|
||||
private String logicalSchemaName;
|
||||
private String physicalCatalogName;
|
||||
private String physicalSchemaName;
|
||||
|
||||
@JsonSerialize(using = ToStringSerializer.class)
|
||||
public BigInteger getId() { return id; }
|
||||
@@ -27,4 +30,10 @@ public class DatacenterCatalogMeta {
|
||||
public void setCatalogType(String catalogType) { this.catalogType = catalogType; }
|
||||
public String getCatalogDesc() { return catalogDesc; }
|
||||
public void setCatalogDesc(String catalogDesc) { this.catalogDesc = catalogDesc; }
|
||||
public String getLogicalSchemaName() { return logicalSchemaName; }
|
||||
public void setLogicalSchemaName(String logicalSchemaName) { this.logicalSchemaName = logicalSchemaName; }
|
||||
public String getPhysicalCatalogName() { return physicalCatalogName; }
|
||||
public void setPhysicalCatalogName(String physicalCatalogName) { this.physicalCatalogName = physicalCatalogName; }
|
||||
public String getPhysicalSchemaName() { return physicalSchemaName; }
|
||||
public void setPhysicalSchemaName(String physicalSchemaName) { this.physicalSchemaName = physicalSchemaName; }
|
||||
}
|
||||
|
||||
@@ -2,24 +2,85 @@ package tech.easyflow.datacenter.meta.model;
|
||||
|
||||
import java.math.BigInteger;
|
||||
|
||||
/**
|
||||
* 字段说明与只读查询治理更新。
|
||||
*/
|
||||
public class DatacenterFieldDescriptionUpdate {
|
||||
|
||||
private BigInteger fieldId;
|
||||
private String fieldDesc;
|
||||
private Integer queryable;
|
||||
private String sensitivityLevel;
|
||||
|
||||
/**
|
||||
* 获取字段 ID。
|
||||
*
|
||||
* @return 字段 ID
|
||||
*/
|
||||
public BigInteger getFieldId() {
|
||||
return fieldId;
|
||||
}
|
||||
|
||||
/**
|
||||
* 设置字段 ID。
|
||||
*
|
||||
* @param fieldId 字段 ID
|
||||
*/
|
||||
public void setFieldId(BigInteger fieldId) {
|
||||
this.fieldId = fieldId;
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取字段说明。
|
||||
*
|
||||
* @return 字段说明
|
||||
*/
|
||||
public String getFieldDesc() {
|
||||
return fieldDesc;
|
||||
}
|
||||
|
||||
/**
|
||||
* 设置字段说明。
|
||||
*
|
||||
* @param fieldDesc 字段说明
|
||||
*/
|
||||
public void setFieldDesc(String fieldDesc) {
|
||||
this.fieldDesc = fieldDesc;
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取是否允许只读查询。
|
||||
*
|
||||
* @return 1 为允许,0 为禁止,空表示保持原值
|
||||
*/
|
||||
public Integer getQueryable() {
|
||||
return queryable;
|
||||
}
|
||||
|
||||
/**
|
||||
* 设置是否允许只读查询。
|
||||
*
|
||||
* @param queryable 1 为允许,0 为禁止,空表示保持原值
|
||||
*/
|
||||
public void setQueryable(Integer queryable) {
|
||||
this.queryable = queryable;
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取敏感级别。
|
||||
*
|
||||
* @return 敏感级别,空表示保持原值
|
||||
*/
|
||||
public String getSensitivityLevel() {
|
||||
return sensitivityLevel;
|
||||
}
|
||||
|
||||
/**
|
||||
* 设置敏感级别。
|
||||
*
|
||||
* @param sensitivityLevel PUBLIC、INTERNAL、SENSITIVE 或 RESTRICTED
|
||||
*/
|
||||
public void setSensitivityLevel(String sensitivityLevel) {
|
||||
this.sensitivityLevel = sensitivityLevel;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,24 @@
|
||||
package tech.easyflow.datacenter.meta.model;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
|
||||
/**
|
||||
* 一次连接内读取的已纳管表元数据快照。
|
||||
*
|
||||
* @param details 当前仍存在的表详情
|
||||
* @param missingTableNames 当前连接中未发现的表名
|
||||
*/
|
||||
public record DatacenterManagedMetadataSnapshot(
|
||||
List<DatacenterTableDetailMeta> details,
|
||||
Set<String> missingTableNames) {
|
||||
|
||||
/**
|
||||
* 对返回集合做防御性复制。
|
||||
*/
|
||||
public DatacenterManagedMetadataSnapshot {
|
||||
details = List.copyOf(details == null ? List.of() : details);
|
||||
missingTableNames = Set.copyOf(
|
||||
missingTableNames == null ? Set.of() : missingTableNames);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,55 @@
|
||||
package tech.easyflow.datacenter.meta.model;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* 有界元数据分页结果。
|
||||
*
|
||||
* @param records 当前页记录
|
||||
* @param pageNumber 页码,从 1 开始
|
||||
* @param pageSize 每页记录上限
|
||||
* @param hasMore 是否还有下一页
|
||||
* @param <T> 元数据类型
|
||||
*/
|
||||
public record DatacenterMetadataPage<T>(
|
||||
List<T> records,
|
||||
long pageNumber,
|
||||
long pageSize,
|
||||
boolean hasMore) {
|
||||
|
||||
/**
|
||||
* 防御性复制当前页记录。
|
||||
*/
|
||||
public DatacenterMetadataPage {
|
||||
records = List.copyOf(records == null ? List.of() : records);
|
||||
}
|
||||
|
||||
/**
|
||||
* 从已加载记录中创建有界分页结果。
|
||||
*
|
||||
* @param records 全部候选记录
|
||||
* @param pageNumber 页码
|
||||
* @param pageSize 每页大小
|
||||
* @param <T> 元数据类型
|
||||
* @return 当前页结果
|
||||
*/
|
||||
public static <T> DatacenterMetadataPage<T> slice(
|
||||
List<T> records,
|
||||
long pageNumber,
|
||||
long pageSize) {
|
||||
List<T> safeRecords = records == null ? List.of() : records;
|
||||
long safePage = Math.max(1L, pageNumber);
|
||||
long safeSize = Math.max(1L, pageSize);
|
||||
if (safePage - 1L > Long.MAX_VALUE / safeSize) {
|
||||
return new DatacenterMetadataPage<>(List.of(), safePage, safeSize, false);
|
||||
}
|
||||
long offset = (safePage - 1L) * safeSize;
|
||||
if (offset >= safeRecords.size()) {
|
||||
return new DatacenterMetadataPage<>(List.of(), safePage, safeSize, false);
|
||||
}
|
||||
int from = Math.toIntExact(offset);
|
||||
int to = (int) Math.min((long) safeRecords.size(), offset + safeSize);
|
||||
return new DatacenterMetadataPage<>(
|
||||
safeRecords.subList(from, to), safePage, safeSize, to < safeRecords.size());
|
||||
}
|
||||
}
|
||||
@@ -9,6 +9,8 @@ public class DatacenterSaveDescriptionsRequest {
|
||||
private BigInteger tableId;
|
||||
private String tableDesc;
|
||||
private List<DatacenterFieldDescriptionUpdate> fields = new ArrayList<>();
|
||||
private Long fieldPageNumber;
|
||||
private Long fieldPageSize;
|
||||
|
||||
public BigInteger getTableId() {
|
||||
return tableId;
|
||||
@@ -33,4 +35,24 @@ public class DatacenterSaveDescriptionsRequest {
|
||||
public void setFields(List<DatacenterFieldDescriptionUpdate> fields) {
|
||||
this.fields = fields;
|
||||
}
|
||||
|
||||
/** @return 保存后返回的字段页码 */
|
||||
public Long getFieldPageNumber() {
|
||||
return fieldPageNumber;
|
||||
}
|
||||
|
||||
/** @param fieldPageNumber 保存后返回的字段页码 */
|
||||
public void setFieldPageNumber(Long fieldPageNumber) {
|
||||
this.fieldPageNumber = fieldPageNumber;
|
||||
}
|
||||
|
||||
/** @return 保存后返回的字段页大小 */
|
||||
public Long getFieldPageSize() {
|
||||
return fieldPageSize;
|
||||
}
|
||||
|
||||
/** @param fieldPageSize 保存后返回的字段页大小 */
|
||||
public void setFieldPageSize(Long fieldPageSize) {
|
||||
this.fieldPageSize = fieldPageSize;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,26 @@
|
||||
package tech.easyflow.datacenter.meta.model;
|
||||
|
||||
import java.math.BigInteger;
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* 激活数据源并纳管选定对象的请求。
|
||||
*
|
||||
* @param sourceId 数据源 ID
|
||||
* @param catalogName 物理 Catalog 或 Schema 名称
|
||||
* @param tableNames 需要纳管的表或视图
|
||||
* @param prewarm 是否立即预热节点本地连接池
|
||||
*/
|
||||
public record DatacenterSourceActivateRequest(
|
||||
BigInteger sourceId,
|
||||
String catalogName,
|
||||
List<String> tableNames,
|
||||
boolean prewarm) {
|
||||
|
||||
/**
|
||||
* 防御性复制表名。
|
||||
*/
|
||||
public DatacenterSourceActivateRequest {
|
||||
tableNames = List.copyOf(tableNames == null ? List.of() : tableNames);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,16 @@
|
||||
package tech.easyflow.datacenter.meta.model;
|
||||
|
||||
/**
|
||||
* 未发布候选连接的命名空间分页请求。
|
||||
*
|
||||
* @param definition 候选连接配置
|
||||
* @param keyword 命名空间搜索词
|
||||
* @param pageNumber 页码
|
||||
* @param pageSize 每页大小
|
||||
*/
|
||||
public record DatacenterSourceCandidateCatalogRequest(
|
||||
DatacenterSourceDraftRequest definition,
|
||||
String keyword,
|
||||
Long pageNumber,
|
||||
Long pageSize) {
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
package tech.easyflow.datacenter.meta.model;
|
||||
|
||||
/**
|
||||
* 活动数据源候选配置的元数据浏览请求。
|
||||
*
|
||||
* @param definition 候选连接配置
|
||||
* @param catalogName 目标 Catalog 或 Schema
|
||||
* @param keyword 表名搜索词
|
||||
* @param pageNumber 页码
|
||||
* @param pageSize 每页大小
|
||||
*/
|
||||
public record DatacenterSourceCandidateMetadataRequest(
|
||||
DatacenterSourceDraftRequest definition,
|
||||
String catalogName,
|
||||
String keyword,
|
||||
Long pageNumber,
|
||||
Long pageSize) {
|
||||
}
|
||||
@@ -0,0 +1,47 @@
|
||||
package tech.easyflow.datacenter.meta.model;
|
||||
|
||||
import java.math.BigInteger;
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* 创建或更新数据源草稿的受控请求。
|
||||
*
|
||||
* @param id 编辑时的数据源 ID
|
||||
* @param sourceName 连接名称
|
||||
* @param sourceCode 租户内业务编码
|
||||
* @param sourceType 数据源类型
|
||||
* @param host 主机地址
|
||||
* @param port 端口
|
||||
* @param databaseName 数据库名
|
||||
* @param schemaName 默认 Schema
|
||||
* @param username 用户名
|
||||
* @param password 本次提交的明文密码;只在请求内存中短暂存在
|
||||
* @param driverClassName JDBC Driver 类名
|
||||
* @param jdbcUrl JDBC URL
|
||||
* @param adapterOptions 不含凭据的 Adapter 选项
|
||||
* @param configJson 不含凭据的连接扩展项
|
||||
*/
|
||||
public record DatacenterSourceDraftRequest(
|
||||
BigInteger id,
|
||||
String sourceName,
|
||||
String sourceCode,
|
||||
String sourceType,
|
||||
String host,
|
||||
Integer port,
|
||||
String databaseName,
|
||||
String schemaName,
|
||||
String username,
|
||||
String password,
|
||||
String driverClassName,
|
||||
String jdbcUrl,
|
||||
Map<String, String> adapterOptions,
|
||||
Map<String, Object> configJson) {
|
||||
|
||||
/**
|
||||
* 防御性复制可变配置。
|
||||
*/
|
||||
public DatacenterSourceDraftRequest {
|
||||
adapterOptions = Map.copyOf(adapterOptions == null ? Map.of() : adapterOptions);
|
||||
configJson = Map.copyOf(configJson == null ? Map.of() : configJson);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,29 @@
|
||||
package tech.easyflow.datacenter.meta.model;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* 活动数据源原地重配置请求。
|
||||
*
|
||||
* @param definition 候选连接配置
|
||||
* @param catalogName 显式选择的 Catalog 或 Schema
|
||||
* @param tableNames 显式选择的表或视图
|
||||
* @param prewarm 是否在发布后预热
|
||||
* @param expectedDefinitionRevision 调用方读取到的 Definition 版本
|
||||
* @param expectedScopeRevision 调用方读取到的范围版本
|
||||
*/
|
||||
public record DatacenterSourceReconfigureRequest(
|
||||
DatacenterSourceDraftRequest definition,
|
||||
String catalogName,
|
||||
List<String> tableNames,
|
||||
boolean prewarm,
|
||||
Long expectedDefinitionRevision,
|
||||
Long expectedScopeRevision) {
|
||||
|
||||
/**
|
||||
* 防御性复制表名集合。
|
||||
*/
|
||||
public DatacenterSourceReconfigureRequest {
|
||||
tableNames = List.copyOf(tableNames == null ? List.of() : tableNames);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,75 @@
|
||||
package tech.easyflow.datacenter.meta.model;
|
||||
|
||||
import com.fasterxml.jackson.databind.annotation.JsonSerialize;
|
||||
import com.fasterxml.jackson.databind.ser.std.ToStringSerializer;
|
||||
import java.math.BigInteger;
|
||||
import java.util.Date;
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* 不含凭据、密文和内部 Runtime 对象的数据源视图。
|
||||
*
|
||||
* @param id 数据源 ID
|
||||
* @param sourceName 连接名称
|
||||
* @param sourceCode 业务编码
|
||||
* @param sourceType 数据源类型
|
||||
* @param status 生命周期名称
|
||||
* @param builtinFlag 是否内置
|
||||
* @param host 主机
|
||||
* @param port 端口
|
||||
* @param databaseName 数据库名
|
||||
* @param schemaName 默认 Schema
|
||||
* @param username 用户名
|
||||
* @param driverClassName Driver 类名
|
||||
* @param jdbcUrl JDBC URL
|
||||
* @param passwordConfigured 是否已保存密码
|
||||
* @param definitionRevision Definition 版本
|
||||
* @param scopeRevision 纳管范围版本
|
||||
* @param compatibilityStatus 兼容状态
|
||||
* @param databaseProductName 数据库产品
|
||||
* @param databaseProductVersion 数据库版本
|
||||
* @param driverName 实际 Driver 名称
|
||||
* @param driverVersion 实际 Driver 版本
|
||||
* @param lastTestStatus 最近测试状态
|
||||
* @param lastTestMessage 最近测试说明
|
||||
* @param lastTestedAt 最近测试时间
|
||||
* @param metadataRefreshStatus 元数据刷新状态
|
||||
* @param metadataRefreshedAt 元数据刷新时间
|
||||
* @param adapterOptions 不含凭据的 Adapter 选项
|
||||
*/
|
||||
public record DatacenterSourceView(
|
||||
@JsonSerialize(using = ToStringSerializer.class) BigInteger id,
|
||||
String sourceName,
|
||||
String sourceCode,
|
||||
String sourceType,
|
||||
String status,
|
||||
Integer builtinFlag,
|
||||
String host,
|
||||
Integer port,
|
||||
String databaseName,
|
||||
String schemaName,
|
||||
String username,
|
||||
String driverClassName,
|
||||
String jdbcUrl,
|
||||
boolean passwordConfigured,
|
||||
long definitionRevision,
|
||||
long scopeRevision,
|
||||
String compatibilityStatus,
|
||||
String databaseProductName,
|
||||
String databaseProductVersion,
|
||||
String driverName,
|
||||
String driverVersion,
|
||||
String lastTestStatus,
|
||||
String lastTestMessage,
|
||||
Date lastTestedAt,
|
||||
String metadataRefreshStatus,
|
||||
Date metadataRefreshedAt,
|
||||
Map<String, String> adapterOptions) {
|
||||
|
||||
/**
|
||||
* 防御性复制 Adapter 选项。
|
||||
*/
|
||||
public DatacenterSourceView {
|
||||
adapterOptions = Map.copyOf(adapterOptions == null ? Map.of() : adapterOptions);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,59 @@
|
||||
package tech.easyflow.datacenter.meta.model;
|
||||
|
||||
import tech.easyflow.datacenter.meta.entity.DatacenterSource;
|
||||
import tech.easyflow.datacenter.meta.enums.DatacenterSourceStatus;
|
||||
|
||||
/**
|
||||
* 将持久化数据源实体转换为安全 API 视图。
|
||||
*/
|
||||
public final class DatacenterSourceViews {
|
||||
|
||||
private DatacenterSourceViews() {
|
||||
}
|
||||
|
||||
/**
|
||||
* 创建不含凭据与内部对象的数据源视图。
|
||||
*
|
||||
* @param source 持久化数据源实体
|
||||
* @return 安全数据源视图
|
||||
* @throws IllegalArgumentException 数据源为空时抛出
|
||||
*/
|
||||
public static DatacenterSourceView from(DatacenterSource source) {
|
||||
if (source == null) {
|
||||
throw new IllegalArgumentException("source must not be null");
|
||||
}
|
||||
return new DatacenterSourceView(
|
||||
source.getId(),
|
||||
source.getSourceName(),
|
||||
source.getSourceCode(),
|
||||
source.getSourceType(),
|
||||
DatacenterSourceStatus.fromCode(source.getStatus()).name(),
|
||||
source.getBuiltinFlag(),
|
||||
source.getHost(),
|
||||
source.getPort(),
|
||||
source.getDatabaseName(),
|
||||
source.getSchemaName(),
|
||||
source.getUsername(),
|
||||
source.getDriverClassName(),
|
||||
source.getJdbcUrl(),
|
||||
source.getCredentialCipher() != null
|
||||
&& !source.getCredentialCipher().isBlank(),
|
||||
value(source.getDefinitionRevision()),
|
||||
Math.max(1L, value(source.getScopeRevision())),
|
||||
source.getCompatibilityStatus(),
|
||||
source.getDatabaseProductName(),
|
||||
source.getDatabaseProductVersion(),
|
||||
source.getDriverName(),
|
||||
source.getDriverVersion(),
|
||||
source.getLastTestStatus(),
|
||||
source.getLastTestMessage(),
|
||||
source.getLastTestedAt(),
|
||||
source.getMetadataRefreshStatus(),
|
||||
source.getMetadataRefreshedAt(),
|
||||
source.getAdapterOptionsJson());
|
||||
}
|
||||
|
||||
private static long value(Long value) {
|
||||
return value == null ? 0L : value;
|
||||
}
|
||||
}
|
||||
@@ -6,12 +6,36 @@ import tech.easyflow.datacenter.entity.DatacenterTableField;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* 数据表及其有界字段详情。
|
||||
*/
|
||||
public class DatacenterTableDetailMeta {
|
||||
private DatacenterTable table;
|
||||
private List<DatacenterTableField> fields = new ArrayList<>();
|
||||
private long fieldPageNumber = 1L;
|
||||
private long fieldPageSize;
|
||||
private boolean hasMoreFields;
|
||||
|
||||
/** @return 数据表元数据 */
|
||||
public DatacenterTable getTable() { return table; }
|
||||
/** @param table 数据表元数据 */
|
||||
public void setTable(DatacenterTable table) { this.table = table; }
|
||||
/** @return 当前页字段 */
|
||||
public List<DatacenterTableField> getFields() { return fields; }
|
||||
public void setFields(List<DatacenterTableField> fields) { this.fields = fields; }
|
||||
/** @param fields 当前页字段 */
|
||||
public void setFields(List<DatacenterTableField> fields) {
|
||||
this.fields = fields == null ? new ArrayList<>() : new ArrayList<>(fields);
|
||||
}
|
||||
/** @return 字段页码 */
|
||||
public long getFieldPageNumber() { return fieldPageNumber; }
|
||||
/** @param fieldPageNumber 字段页码 */
|
||||
public void setFieldPageNumber(long fieldPageNumber) { this.fieldPageNumber = fieldPageNumber; }
|
||||
/** @return 字段页大小 */
|
||||
public long getFieldPageSize() { return fieldPageSize; }
|
||||
/** @param fieldPageSize 字段页大小 */
|
||||
public void setFieldPageSize(long fieldPageSize) { this.fieldPageSize = fieldPageSize; }
|
||||
/** @return 是否还有字段 */
|
||||
public boolean isHasMoreFields() { return hasMoreFields; }
|
||||
/** @param hasMoreFields 是否还有字段 */
|
||||
public void setHasMoreFields(boolean hasMoreFields) { this.hasMoreFields = hasMoreFields; }
|
||||
}
|
||||
|
||||
@@ -20,6 +20,21 @@ public interface DatacenterDatasetRegistryService {
|
||||
|
||||
DatacenterTable registerTable(DatacenterSource source, DatacenterCatalog catalog, DatacenterTableDetailMeta detail, LoginAccount account);
|
||||
|
||||
/**
|
||||
* 在管理员显式重配置连接时重新绑定物理表身份和字段结构。
|
||||
*
|
||||
* @param source 新数据源 Definition
|
||||
* @param catalog 新物理命名空间
|
||||
* @param detail 新物理表元数据
|
||||
* @param account 操作账号
|
||||
* @return 重绑定后的纳管表
|
||||
*/
|
||||
DatacenterTable rebindTable(
|
||||
DatacenterSource source,
|
||||
DatacenterCatalog catalog,
|
||||
DatacenterTableDetailMeta detail,
|
||||
LoginAccount account);
|
||||
|
||||
DatacenterTable getTableWithFields(BigInteger tableId);
|
||||
|
||||
List<DatacenterTableField> getFields(BigInteger tableId);
|
||||
@@ -34,5 +49,35 @@ public interface DatacenterDatasetRegistryService {
|
||||
|
||||
int removeTables(List<BigInteger> tableIds);
|
||||
|
||||
/**
|
||||
* 刷新一个已纳管表的观测元数据,不自动插入新表或新字段。
|
||||
*
|
||||
* @param source 当前数据源
|
||||
* @param catalog 已纳管命名空间
|
||||
* @param tableId 已纳管表 ID
|
||||
* @param detail 当前 JDBC 元数据
|
||||
* @param account 当前账号
|
||||
* @return 是否收缩或改变了可查询范围
|
||||
*/
|
||||
boolean refreshManagedTable(
|
||||
DatacenterSource source,
|
||||
DatacenterCatalog catalog,
|
||||
BigInteger tableId,
|
||||
DatacenterTableDetailMeta detail,
|
||||
LoginAccount account);
|
||||
|
||||
/**
|
||||
* 将本次无法发现的已纳管表标记为缺失。
|
||||
*
|
||||
* @param source 当前数据源
|
||||
* @param tableId 已纳管表 ID
|
||||
* @param account 当前账号
|
||||
* @return 是否收缩或改变了可查询范围
|
||||
*/
|
||||
boolean markManagedTableMissing(
|
||||
DatacenterSource source,
|
||||
BigInteger tableId,
|
||||
LoginAccount account);
|
||||
|
||||
DatacenterTable saveDescriptions(BigInteger tableId, String tableDesc, List<DatacenterFieldDescriptionUpdate> fields, LoginAccount account);
|
||||
}
|
||||
|
||||
@@ -9,24 +9,183 @@ import tech.easyflow.datacenter.meta.entity.DatacenterSource;
|
||||
import tech.easyflow.datacenter.meta.model.DatacenterBatchRegisterRequest;
|
||||
import tech.easyflow.datacenter.meta.model.DatacenterCatalogMeta;
|
||||
import tech.easyflow.datacenter.meta.model.DatacenterTableDetailMeta;
|
||||
import tech.easyflow.datacenter.meta.model.DatacenterSourceActivateRequest;
|
||||
import tech.easyflow.datacenter.meta.model.DatacenterMetadataPage;
|
||||
import tech.easyflow.datacenter.meta.model.DatacenterSourceCandidateMetadataRequest;
|
||||
import tech.easyflow.datacenter.meta.model.DatacenterSourceCandidateCatalogRequest;
|
||||
import tech.easyflow.datacenter.meta.model.DatacenterSourceDraftRequest;
|
||||
import tech.easyflow.datacenter.meta.model.DatacenterSourceReconfigureRequest;
|
||||
import tech.easyflow.datacenter.meta.model.DatacenterSourceView;
|
||||
|
||||
import java.math.BigInteger;
|
||||
import java.util.List;
|
||||
|
||||
public interface DatacenterSourceService extends IService<DatacenterSource> {
|
||||
/**
|
||||
* 保存内部 Excel/项目数据源实体,外部 Controller 不应调用。
|
||||
*
|
||||
* @param source 内部数据源实体
|
||||
* @param account 当前账号
|
||||
* @return 保存后的实体
|
||||
*/
|
||||
DatacenterSource saveSource(DatacenterSource source, LoginAccount account);
|
||||
|
||||
Page<DatacenterSource> pageSources(Long pageNumber, Long pageSize, LoginAccount account);
|
||||
/**
|
||||
* 保存外部 JDBC 数据源草稿。
|
||||
*
|
||||
* @param request 受控草稿请求
|
||||
* @param account 当前账号
|
||||
* @return 安全数据源视图
|
||||
*/
|
||||
DatacenterSourceView saveDraft(DatacenterSourceDraftRequest request, LoginAccount account);
|
||||
|
||||
DatacenterConnectionTestResult testConnection(DatacenterSource source, LoginAccount account);
|
||||
/**
|
||||
* 分页返回不含凭据的数据源。
|
||||
*
|
||||
* @param pageNumber 页码
|
||||
* @param pageSize 页大小
|
||||
* @param account 当前账号
|
||||
* @return 安全数据源分页
|
||||
*/
|
||||
Page<DatacenterSourceView> pageSources(Long pageNumber, Long pageSize, LoginAccount account);
|
||||
|
||||
/**
|
||||
* 探测已保存草稿并记录数据库/Driver 指纹。
|
||||
*
|
||||
* @param sourceId 数据源 ID
|
||||
* @param account 当前账号
|
||||
* @return 连接探测结果
|
||||
*/
|
||||
DatacenterConnectionTestResult probe(BigInteger sourceId, LoginAccount account);
|
||||
|
||||
/**
|
||||
* 激活数据源、纳管选择对象并发布 Definition。
|
||||
*
|
||||
* @param request 激活请求
|
||||
* @param account 当前账号
|
||||
* @return READY 数据源视图
|
||||
*/
|
||||
DatacenterSourceView activate(DatacenterSourceActivateRequest request, LoginAccount account);
|
||||
|
||||
/**
|
||||
* 使用活动数据源的候选配置执行无状态连接探测。
|
||||
*
|
||||
* @param request 候选连接配置
|
||||
* @param account 当前账号
|
||||
* @return 连接探测结果
|
||||
*/
|
||||
DatacenterConnectionTestResult probeCandidate(
|
||||
DatacenterSourceDraftRequest request,
|
||||
LoginAccount account);
|
||||
|
||||
/**
|
||||
* 浏览活动数据源候选配置可访问的命名空间。
|
||||
*
|
||||
* @param request 候选连接配置
|
||||
* @param account 当前账号
|
||||
* @return 命名空间列表
|
||||
*/
|
||||
List<DatacenterCatalogMeta> listCandidateCatalogs(
|
||||
DatacenterSourceDraftRequest request,
|
||||
LoginAccount account);
|
||||
|
||||
/**
|
||||
* 分页浏览活动数据源候选配置可访问的命名空间。
|
||||
*
|
||||
* @param request 候选配置与分页条件
|
||||
* @param account 当前账号
|
||||
* @return 有界命名空间列表
|
||||
*/
|
||||
DatacenterMetadataPage<DatacenterCatalogMeta> listCandidateCatalogsPage(
|
||||
DatacenterSourceCandidateCatalogRequest request,
|
||||
LoginAccount account);
|
||||
|
||||
/**
|
||||
* 分页浏览活动数据源候选配置可访问的表。
|
||||
*
|
||||
* @param request 候选配置与分页条件
|
||||
* @param account 当前账号
|
||||
* @return 有界表列表
|
||||
*/
|
||||
DatacenterMetadataPage<DatacenterTable> listCandidateTables(
|
||||
DatacenterSourceCandidateMetadataRequest request,
|
||||
LoginAccount account);
|
||||
|
||||
/**
|
||||
* 原地发布活动数据源的新连接配置和显式纳管范围。
|
||||
*
|
||||
* @param request 重配置请求
|
||||
* @param account 当前账号
|
||||
* @return 发布后的安全视图
|
||||
*/
|
||||
DatacenterSourceView reconfigure(
|
||||
DatacenterSourceReconfigureRequest request,
|
||||
LoginAccount account);
|
||||
|
||||
List<DatacenterCatalogMeta> listCatalogs(BigInteger sourceId, LoginAccount account);
|
||||
|
||||
List<DatacenterTable> listTables(BigInteger sourceId, String catalogName, LoginAccount account);
|
||||
/**
|
||||
* 分页浏览当前数据源的命名空间。
|
||||
*
|
||||
* @param sourceId 数据源 ID
|
||||
* @param keyword 名称搜索词
|
||||
* @param pageNumber 页码
|
||||
* @param pageSize 每页大小
|
||||
* @param account 当前账号
|
||||
* @return 有界命名空间列表
|
||||
*/
|
||||
DatacenterMetadataPage<DatacenterCatalogMeta> listCatalogsPage(
|
||||
BigInteger sourceId,
|
||||
String keyword,
|
||||
Long pageNumber,
|
||||
Long pageSize,
|
||||
LoginAccount account);
|
||||
|
||||
DatacenterTableDetailMeta getTableDetail(BigInteger sourceId, String catalogName, String tableName, boolean register, LoginAccount account);
|
||||
DatacenterMetadataPage<DatacenterTable> listTables(
|
||||
BigInteger sourceId,
|
||||
String catalogName,
|
||||
String keyword,
|
||||
Long pageNumber,
|
||||
Long pageSize,
|
||||
LoginAccount account);
|
||||
|
||||
DatacenterTableDetailMeta getTableDetail(
|
||||
BigInteger sourceId,
|
||||
String catalogName,
|
||||
String tableName,
|
||||
boolean register,
|
||||
Long fieldPageNumber,
|
||||
Long fieldPageSize,
|
||||
LoginAccount account);
|
||||
|
||||
List<DatacenterTable> batchRegisterTables(DatacenterBatchRegisterRequest request, LoginAccount account);
|
||||
|
||||
void removeSource(BigInteger sourceId, LoginAccount account);
|
||||
|
||||
/**
|
||||
* 停用活动数据源并发布墓碑。
|
||||
*
|
||||
* @param sourceId 数据源 ID
|
||||
* @param account 当前账号
|
||||
* @return 停用后的安全视图
|
||||
*/
|
||||
DatacenterSourceView disable(BigInteger sourceId, LoginAccount account);
|
||||
|
||||
/**
|
||||
* 重新探测、预热并启用已停用数据源。
|
||||
*
|
||||
* @param sourceId 数据源 ID
|
||||
* @param account 当前账号
|
||||
* @return 启用后的安全视图
|
||||
*/
|
||||
DatacenterSourceView enable(BigInteger sourceId, LoginAccount account);
|
||||
|
||||
/**
|
||||
* 刷新已纳管对象的 JDBC 元数据观测状态。
|
||||
*
|
||||
* @param sourceId 数据源 ID
|
||||
* @param account 当前账号
|
||||
* @return 刷新后的安全视图
|
||||
*/
|
||||
DatacenterSourceView refreshMetadata(BigInteger sourceId, LoginAccount account);
|
||||
}
|
||||
|
||||
@@ -9,6 +9,8 @@ import tech.easyflow.datacenter.adapter.DbHandleManager;
|
||||
import tech.easyflow.datacenter.entity.DatacenterTable;
|
||||
import tech.easyflow.datacenter.entity.DatacenterTableField;
|
||||
import tech.easyflow.datacenter.execution.model.DatasetRef;
|
||||
import tech.easyflow.datacenter.federation.DatacenterMetadataIdentity;
|
||||
import tech.easyflow.datacenter.federation.DatacenterFederationChangeNotifier;
|
||||
import tech.easyflow.datacenter.mapper.DatacenterCatalogMapper;
|
||||
import tech.easyflow.datacenter.mapper.DatacenterDatasetVersionMapper;
|
||||
import tech.easyflow.datacenter.mapper.DatacenterDerivedTableMapper;
|
||||
@@ -20,6 +22,9 @@ import tech.easyflow.datacenter.meta.entity.DatacenterCatalog;
|
||||
import tech.easyflow.datacenter.meta.entity.DatacenterSource;
|
||||
import tech.easyflow.datacenter.meta.model.DatacenterFieldDescriptionUpdate;
|
||||
import tech.easyflow.datacenter.meta.enums.DatacenterSourceType;
|
||||
import tech.easyflow.datacenter.meta.enums.DatacenterMetadataStatus;
|
||||
import tech.easyflow.datacenter.meta.enums.DatacenterSensitivityLevel;
|
||||
import tech.easyflow.datacenter.meta.enums.DatacenterSourceStatus;
|
||||
import tech.easyflow.datacenter.meta.service.DatacenterDatasetRegistryService;
|
||||
import tech.easyflow.datacenter.meta.service.DatacenterMetaConstants;
|
||||
import tech.easyflow.datacenter.meta.model.DatacenterTableDetailMeta;
|
||||
@@ -30,7 +35,11 @@ import java.math.BigInteger;
|
||||
import java.util.Date;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Locale;
|
||||
import java.util.Map;
|
||||
import java.util.Objects;
|
||||
import java.util.Set;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
@Service
|
||||
public class DatacenterDatasetRegistryServiceImpl implements DatacenterDatasetRegistryService {
|
||||
@@ -51,6 +60,8 @@ public class DatacenterDatasetRegistryServiceImpl implements DatacenterDatasetRe
|
||||
private DatacenterImportJobMapper importJobMapper;
|
||||
@Resource
|
||||
private DbHandleManager dbHandleManager;
|
||||
@Resource
|
||||
private DatacenterFederationChangeNotifier federationChangeNotifier;
|
||||
|
||||
@Override
|
||||
public DatacenterSource ensureBuiltinSource(DatacenterSourceType sourceType, LoginAccount account) {
|
||||
@@ -67,7 +78,11 @@ public class DatacenterDatasetRegistryServiceImpl implements DatacenterDatasetRe
|
||||
source.setDeptId(deptId);
|
||||
source.setSourceType(sourceType.name());
|
||||
source.setBuiltinFlag(1);
|
||||
source.setStatus(0);
|
||||
source.setStatus(DatacenterSourceStatus.READY.code());
|
||||
source.setDefinitionRevision(1L);
|
||||
source.setScopeRevision(1L);
|
||||
source.setMetadataRefreshStatus("SUCCESS");
|
||||
source.setMetadataRefreshedAt(new Date());
|
||||
source.setCreated(new Date());
|
||||
source.setModified(new Date());
|
||||
source.setCreatedBy(account == null ? BigInteger.ZERO : account.getId());
|
||||
@@ -92,6 +107,12 @@ public class DatacenterDatasetRegistryServiceImpl implements DatacenterDatasetRe
|
||||
wrapper.eq(DatacenterCatalog::getCatalogName, catalogName);
|
||||
DatacenterCatalog catalog = catalogMapper.selectOneByQuery(wrapper);
|
||||
if (catalog != null) {
|
||||
boolean changed = applyCatalogIdentity(catalog, source, catalogName);
|
||||
if (changed) {
|
||||
catalog.setModified(new Date());
|
||||
catalog.setModifiedBy(account == null ? BigInteger.ZERO : account.getId());
|
||||
catalogMapper.update(catalog);
|
||||
}
|
||||
return catalog;
|
||||
}
|
||||
catalog = new DatacenterCatalog();
|
||||
@@ -105,6 +126,7 @@ public class DatacenterDatasetRegistryServiceImpl implements DatacenterDatasetRe
|
||||
catalog.setCatalogName(catalogName);
|
||||
catalog.setCatalogDesc(catalogName);
|
||||
catalog.setCatalogType("DATABASE");
|
||||
applyCatalogIdentity(catalog, source, catalogName);
|
||||
catalog.setStatus(0);
|
||||
catalog.setCreated(new Date());
|
||||
catalog.setModified(new Date());
|
||||
@@ -116,13 +138,43 @@ public class DatacenterDatasetRegistryServiceImpl implements DatacenterDatasetRe
|
||||
|
||||
@Override
|
||||
public DatacenterTable registerTable(DatacenterSource source, DatacenterCatalog catalog, DatacenterTableDetailMeta detail, LoginAccount account) {
|
||||
return registerTable(source, catalog, detail, account, false);
|
||||
}
|
||||
|
||||
/**
|
||||
* 显式重配置时允许在同一稳定表记录上确认新的物理身份,并重建字段治理基线。
|
||||
*/
|
||||
@Override
|
||||
public DatacenterTable rebindTable(
|
||||
DatacenterSource source,
|
||||
DatacenterCatalog catalog,
|
||||
DatacenterTableDetailMeta detail,
|
||||
LoginAccount account) {
|
||||
return registerTable(source, catalog, detail, account, true);
|
||||
}
|
||||
|
||||
private DatacenterTable registerTable(
|
||||
DatacenterSource source,
|
||||
DatacenterCatalog catalog,
|
||||
DatacenterTableDetailMeta detail,
|
||||
LoginAccount account,
|
||||
boolean allowExplicitRebind) {
|
||||
DatacenterTable table = detail.getTable();
|
||||
applyTableDefaults(table, source, detail);
|
||||
table.setPhysicalIdentityKey(DatacenterMetadataIdentity.tableIdentity(
|
||||
source.getId().toString(),
|
||||
catalog.getNamespaceKey(),
|
||||
table.getActualTable(),
|
||||
table.getTableKind()));
|
||||
table.setMetadataFingerprint(DatacenterMetadataIdentity.tableFingerprint(
|
||||
table.getTableName(), table.getTableKind(), detail.getFields()));
|
||||
QueryWrapper wrapper = QueryWrapper.create();
|
||||
wrapper.eq(DatacenterTable::getSourceId, source.getId());
|
||||
wrapper.eq(DatacenterTable::getCatalogId, catalog.getId());
|
||||
wrapper.eq(DatacenterTable::getTableName, table.getTableName());
|
||||
DatacenterTable existing = tableMapper.selectOneByQuery(wrapper);
|
||||
List<DatacenterTableField> persistedFields = existing == null
|
||||
? List.of() : getFields(existing.getId());
|
||||
Date now = new Date();
|
||||
if (existing == null) {
|
||||
table.setSourceId(source.getId());
|
||||
@@ -136,6 +188,39 @@ public class DatacenterDatasetRegistryServiceImpl implements DatacenterDatasetRe
|
||||
table.setModifiedBy(account == null ? BigInteger.ZERO : account.getId());
|
||||
tableMapper.insert(table);
|
||||
} else {
|
||||
boolean identityChanged = hasText(existing.getPhysicalIdentityKey())
|
||||
&& !existing.getPhysicalIdentityKey().equals(table.getPhysicalIdentityKey())
|
||||
&& !existing.getPhysicalIdentityKey().equals(
|
||||
DatacenterMetadataIdentity.legacyV58TableIdentity(
|
||||
existing.getSourceId(), existing.getCatalogId(),
|
||||
existing.getActualTable(), existing.getTableKind()));
|
||||
if (identityChanged && !allowExplicitRebind) {
|
||||
throw new BusinessException("物理表身份已变化,请移除原绑定后重新接入: " + table.getTableName());
|
||||
}
|
||||
boolean structureChanged = false;
|
||||
if (hasText(existing.getMetadataFingerprint())
|
||||
&& !existing.getMetadataFingerprint().equals(table.getMetadataFingerprint())) {
|
||||
boolean legacyFingerprint = existing.getMetadataFingerprint().equals(
|
||||
DatacenterMetadataIdentity.legacyV58TableFingerprint(
|
||||
existing.getTableName(), existing.getTableKind()));
|
||||
if (!legacyFingerprint
|
||||
|| !legacyStructureMatches(
|
||||
persistedFields, detail.getFields())) {
|
||||
structureChanged = true;
|
||||
}
|
||||
}
|
||||
if (structureChanged && !allowExplicitRebind) {
|
||||
throw new BusinessException(
|
||||
"表结构已变化,请移除原绑定后重新接入: " + table.getTableName());
|
||||
}
|
||||
boolean resetGovernance = allowExplicitRebind
|
||||
&& (identityChanged || structureChanged);
|
||||
if (resetGovernance) {
|
||||
tableFieldMapper.deleteByQuery(QueryWrapper.create()
|
||||
.eq(DatacenterTableField::getTableId, existing.getId()));
|
||||
persistedFields = List.of();
|
||||
existing.setQueryable(table.getQueryable());
|
||||
}
|
||||
if (!hasText(existing.getTableDesc())) {
|
||||
existing.setTableDesc(normalizeDescription(table.getTableDesc()));
|
||||
}
|
||||
@@ -145,30 +230,61 @@ public class DatacenterDatasetRegistryServiceImpl implements DatacenterDatasetRe
|
||||
existing.setAccessMode(table.getAccessMode());
|
||||
existing.setVersioningEnabled(table.getVersioningEnabled());
|
||||
existing.setCapabilitiesJson(table.getCapabilitiesJson());
|
||||
existing.setPhysicalIdentityKey(table.getPhysicalIdentityKey());
|
||||
existing.setMetadataFingerprint(table.getMetadataFingerprint());
|
||||
existing.setMetadataStatus(DatacenterMetadataStatus.ACTIVE.name());
|
||||
existing.setLastSeenAt(now);
|
||||
// 查询开关属于治理配置,元数据刷新不得覆盖人工决策。
|
||||
existing.setTimeSemantics(table.getTimeSemantics());
|
||||
existing.setMetadataRevision((existing.getMetadataRevision() == null
|
||||
? 0L : existing.getMetadataRevision()) + 1L);
|
||||
existing.setModified(now);
|
||||
existing.setModifiedBy(account == null ? BigInteger.ZERO : account.getId());
|
||||
tableMapper.update(existing);
|
||||
table = existing;
|
||||
}
|
||||
Map<String, DatacenterTableField> existingFieldMap = getFields(table.getId()).stream()
|
||||
.collect(LinkedHashMap::new, (map, field) -> map.put(field.getFieldName(), field), Map::putAll);
|
||||
QueryWrapper deleteWrapper = QueryWrapper.create();
|
||||
deleteWrapper.eq(DatacenterTableField::getTableId, table.getId());
|
||||
tableFieldMapper.deleteByQuery(deleteWrapper);
|
||||
Map<String, DatacenterTableField> existingFieldMap = persistedFields.stream()
|
||||
.collect(LinkedHashMap::new,
|
||||
(map, field) -> map.put(sourceColumnName(field), field),
|
||||
Map::putAll);
|
||||
for (DatacenterTableField field : detail.getFields()) {
|
||||
DatacenterTableField existingField = existingFieldMap.get(field.getFieldName());
|
||||
applyFieldDefaults(field);
|
||||
DatacenterTableField existingField = existingFieldMap.remove(sourceColumnName(field));
|
||||
if (existingField != null && hasText(existingField.getFieldDesc())) {
|
||||
field.setFieldDesc(existingField.getFieldDesc());
|
||||
} else {
|
||||
field.setFieldDesc(normalizeDescription(field.getFieldDesc()));
|
||||
}
|
||||
field.setId(null);
|
||||
field.setTableId(table.getId());
|
||||
field.setCreated(now);
|
||||
field.setModified(now);
|
||||
field.setCreatedBy(account == null ? BigInteger.ZERO : account.getId());
|
||||
field.setModifiedBy(account == null ? BigInteger.ZERO : account.getId());
|
||||
tableFieldMapper.insert(field);
|
||||
if (existingField == null) {
|
||||
field.setId(null);
|
||||
field.setCreated(now);
|
||||
field.setCreatedBy(account == null ? BigInteger.ZERO : account.getId());
|
||||
tableFieldMapper.insert(field);
|
||||
} else {
|
||||
field.setId(existingField.getId());
|
||||
field.setCreated(existingField.getCreated());
|
||||
field.setCreatedBy(existingField.getCreatedBy());
|
||||
if (existingField.getSensitivityLevel() != null) {
|
||||
field.setSensitivityLevel(existingField.getSensitivityLevel());
|
||||
field.setMaskingStrategy(existingField.getMaskingStrategy());
|
||||
}
|
||||
// 元数据刷新只更新物理事实,保留字段级治理设置。
|
||||
field.setQueryable(existingField.getQueryable());
|
||||
field.setSortable(existingField.getSortable());
|
||||
tableFieldMapper.update(field);
|
||||
}
|
||||
}
|
||||
for (DatacenterTableField missing : existingFieldMap.values()) {
|
||||
missing.setMetadataStatus(DatacenterMetadataStatus.MISSING.name());
|
||||
missing.setQueryable(0);
|
||||
missing.setSortable(0);
|
||||
missing.setWritable(0);
|
||||
missing.setModified(now);
|
||||
missing.setModifiedBy(account == null ? BigInteger.ZERO : account.getId());
|
||||
tableFieldMapper.update(missing);
|
||||
}
|
||||
table.setFields(getFields(table.getId()));
|
||||
return table;
|
||||
@@ -196,19 +312,71 @@ public class DatacenterDatasetRegistryServiceImpl implements DatacenterDatasetRe
|
||||
fieldUpdateMap.put(field.getFieldId(), field);
|
||||
}
|
||||
}
|
||||
boolean policyChanged = false;
|
||||
for (DatacenterTableField field : table.getFields()) {
|
||||
DatacenterFieldDescriptionUpdate update = fieldUpdateMap.get(field.getId());
|
||||
if (update == null) {
|
||||
continue;
|
||||
}
|
||||
field.setFieldDesc(normalizeDescription(update.getFieldDesc()));
|
||||
Integer queryable = normalizedQueryable(update.getQueryable(), field.getQueryable());
|
||||
String sensitivityLevel = normalizedSensitivityLevel(
|
||||
update.getSensitivityLevel(), field.getSensitivityLevel());
|
||||
if (!DatacenterSensitivityLevel.PUBLIC.name().equals(sensitivityLevel)
|
||||
&& Integer.valueOf(1).equals(queryable)) {
|
||||
throw new BusinessException("敏感字段暂不支持直接查询,请关闭可查询后保存");
|
||||
}
|
||||
policyChanged |= !Objects.equals(field.getQueryable(), queryable)
|
||||
|| !Objects.equals(field.getSensitivityLevel(), sensitivityLevel);
|
||||
field.setQueryable(queryable);
|
||||
field.setSensitivityLevel(sensitivityLevel);
|
||||
field.setModified(now);
|
||||
field.setModifiedBy(account == null ? BigInteger.ZERO : account.getId());
|
||||
tableFieldMapper.update(field);
|
||||
}
|
||||
if (policyChanged) {
|
||||
incrementExternalScopeRevisions(List.of(table));
|
||||
}
|
||||
return getTableWithFields(tableId);
|
||||
}
|
||||
|
||||
/**
|
||||
* 校验并归一化字段查询开关。
|
||||
*
|
||||
* @param requested 请求值
|
||||
* @param existing 当前值
|
||||
* @return 归一化后的 0 或 1
|
||||
*/
|
||||
private Integer normalizedQueryable(Integer requested, Integer existing) {
|
||||
if (requested == null) {
|
||||
return existing == null ? 0 : existing;
|
||||
}
|
||||
if (requested != 0 && requested != 1) {
|
||||
throw new BusinessException("queryable 只能为 0 或 1");
|
||||
}
|
||||
return requested;
|
||||
}
|
||||
|
||||
/**
|
||||
* 校验并归一化字段敏感级别。
|
||||
*
|
||||
* @param requested 请求值
|
||||
* @param existing 当前值
|
||||
* @return 有效枚举名称
|
||||
*/
|
||||
private String normalizedSensitivityLevel(String requested, String existing) {
|
||||
if (requested == null || requested.isBlank()) {
|
||||
return existing == null || existing.isBlank()
|
||||
? DatacenterSensitivityLevel.PUBLIC.name() : existing;
|
||||
}
|
||||
try {
|
||||
return DatacenterSensitivityLevel.valueOf(
|
||||
requested.trim().toUpperCase(Locale.ROOT)).name();
|
||||
} catch (IllegalArgumentException exception) {
|
||||
throw new BusinessException("不支持的字段敏感级别: " + requested);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public DatacenterTable getTableWithFields(BigInteger tableId) {
|
||||
DatacenterTable table = tableMapper.selectOneById(tableId);
|
||||
@@ -223,7 +391,7 @@ public class DatacenterDatasetRegistryServiceImpl implements DatacenterDatasetRe
|
||||
public List<DatacenterTableField> getFields(BigInteger tableId) {
|
||||
QueryWrapper wrapper = QueryWrapper.create();
|
||||
wrapper.eq(DatacenterTableField::getTableId, tableId);
|
||||
wrapper.orderBy("id");
|
||||
wrapper.orderBy("ordinal_position asc, id asc");
|
||||
return tableFieldMapper.selectListByQuery(wrapper);
|
||||
}
|
||||
|
||||
@@ -231,6 +399,7 @@ public class DatacenterDatasetRegistryServiceImpl implements DatacenterDatasetRe
|
||||
public DatasetRef resolveDatasetRef(BigInteger tableId) {
|
||||
DatacenterTable table = getTableWithFields(tableId);
|
||||
DatasetRef ref = new DatasetRef();
|
||||
ref.setTenantId(table.getTenantId());
|
||||
ref.setTableId(tableId);
|
||||
ref.setSourceId(table.getSourceId());
|
||||
ref.setCatalogId(table.getCatalogId());
|
||||
@@ -266,6 +435,7 @@ public class DatacenterDatasetRegistryServiceImpl implements DatacenterDatasetRe
|
||||
if (catalogId != null) {
|
||||
wrapper.eq(DatacenterTable::getCatalogId, catalogId);
|
||||
}
|
||||
wrapper.eq(DatacenterTable::getMetadataStatus, DatacenterMetadataStatus.ACTIVE.name());
|
||||
wrapper.orderBy("created desc");
|
||||
return tableMapper.selectListByQuery(wrapper);
|
||||
}
|
||||
@@ -280,31 +450,231 @@ public class DatacenterDatasetRegistryServiceImpl implements DatacenterDatasetRe
|
||||
QueryWrapper physicalTableWrapper = QueryWrapper.create();
|
||||
physicalTableWrapper.in(DatacenterTable::getId, ids);
|
||||
List<DatacenterTable> tables = tableMapper.selectListByQuery(physicalTableWrapper);
|
||||
List<BigInteger> authorizedIds = tables.stream()
|
||||
.map(DatacenterTable::getId)
|
||||
.distinct()
|
||||
.toList();
|
||||
if (authorizedIds.size() != ids.size()) {
|
||||
throw new BusinessException("部分数据表不存在或无权删除");
|
||||
}
|
||||
tables.forEach(this::deletePhysicalTableIfNecessary);
|
||||
|
||||
QueryWrapper fieldWrapper = QueryWrapper.create();
|
||||
fieldWrapper.in(DatacenterTableField::getTableId, ids);
|
||||
fieldWrapper.in(DatacenterTableField::getTableId, authorizedIds);
|
||||
tableFieldMapper.deleteByQuery(fieldWrapper);
|
||||
|
||||
QueryWrapper versionWrapper = QueryWrapper.create();
|
||||
versionWrapper.in(tech.easyflow.datacenter.meta.entity.DatacenterDatasetVersion::getTableId, ids);
|
||||
versionWrapper.in(tech.easyflow.datacenter.meta.entity.DatacenterDatasetVersion::getTableId, authorizedIds);
|
||||
datasetVersionMapper.deleteByQuery(versionWrapper);
|
||||
|
||||
QueryWrapper importJobWrapper = QueryWrapper.create();
|
||||
importJobWrapper.in(tech.easyflow.datacenter.meta.entity.DatacenterImportJob::getTableId, ids);
|
||||
importJobWrapper.in(tech.easyflow.datacenter.meta.entity.DatacenterImportJob::getTableId, authorizedIds);
|
||||
importJobMapper.deleteByQuery(importJobWrapper);
|
||||
|
||||
QueryWrapper upstreamWrapper = QueryWrapper.create();
|
||||
upstreamWrapper.in(tech.easyflow.datacenter.meta.entity.DatacenterDerivedTable::getSourceTableId, ids);
|
||||
upstreamWrapper.in(tech.easyflow.datacenter.meta.entity.DatacenterDerivedTable::getSourceTableId, authorizedIds);
|
||||
derivedTableMapper.deleteByQuery(upstreamWrapper);
|
||||
|
||||
QueryWrapper downstreamWrapper = QueryWrapper.create();
|
||||
downstreamWrapper.in(tech.easyflow.datacenter.meta.entity.DatacenterDerivedTable::getDerivedTableId, ids);
|
||||
downstreamWrapper.in(tech.easyflow.datacenter.meta.entity.DatacenterDerivedTable::getDerivedTableId, authorizedIds);
|
||||
derivedTableMapper.deleteByQuery(downstreamWrapper);
|
||||
|
||||
QueryWrapper tableWrapper = QueryWrapper.create();
|
||||
tableWrapper.in(DatacenterTable::getId, ids);
|
||||
return tableMapper.deleteByQuery(tableWrapper);
|
||||
tableWrapper.in(DatacenterTable::getId, authorizedIds);
|
||||
int removed = tableMapper.deleteByQuery(tableWrapper);
|
||||
incrementExternalScopeRevisions(tables);
|
||||
return removed;
|
||||
}
|
||||
|
||||
/**
|
||||
* 刷新已纳管表的物理观测值;身份、结构或历史状态存在歧义时立即收缩权限。
|
||||
*/
|
||||
@Override
|
||||
@Transactional(rollbackFor = Exception.class)
|
||||
public boolean refreshManagedTable(
|
||||
DatacenterSource source,
|
||||
DatacenterCatalog catalog,
|
||||
BigInteger tableId,
|
||||
DatacenterTableDetailMeta detail,
|
||||
LoginAccount account) {
|
||||
DatacenterTable existing = tableMapper.selectOneById(tableId);
|
||||
if (existing == null
|
||||
|| !Objects.equals(existing.getSourceId(), source.getId())
|
||||
|| !Objects.equals(existing.getCatalogId(), catalog.getId())) {
|
||||
throw new BusinessException("已纳管数据表不存在或不属于当前连接");
|
||||
}
|
||||
DatacenterTable discovered = detail == null ? null : detail.getTable();
|
||||
if (discovered == null) {
|
||||
throw new BusinessException("未返回数据表元数据: " + existing.getTableName());
|
||||
}
|
||||
applyTableDefaults(discovered, source, detail);
|
||||
String discoveredIdentity = DatacenterMetadataIdentity.tableIdentity(
|
||||
source.getId().toString(),
|
||||
catalog.getNamespaceKey(),
|
||||
discovered.getActualTable(),
|
||||
discovered.getTableKind());
|
||||
String discoveredFingerprint = DatacenterMetadataIdentity.tableFingerprint(
|
||||
discovered.getTableName(), discovered.getTableKind(), detail.getFields());
|
||||
List<DatacenterTableField> persistedFields = getFields(existing.getId());
|
||||
boolean identityMatches = Objects.equals(
|
||||
existing.getPhysicalIdentityKey(), discoveredIdentity)
|
||||
|| Objects.equals(
|
||||
existing.getPhysicalIdentityKey(),
|
||||
DatacenterMetadataIdentity.legacyV58TableIdentity(
|
||||
existing.getSourceId(), existing.getCatalogId(),
|
||||
existing.getActualTable(), existing.getTableKind()));
|
||||
boolean legacyCompatible = Objects.equals(
|
||||
existing.getMetadataFingerprint(),
|
||||
DatacenterMetadataIdentity.legacyV58TableFingerprint(
|
||||
existing.getTableName(), existing.getTableKind()))
|
||||
&& legacyStructureMatches(persistedFields, detail.getFields());
|
||||
boolean fingerprintMatches = Objects.equals(
|
||||
existing.getMetadataFingerprint(), discoveredFingerprint)
|
||||
|| legacyCompatible;
|
||||
boolean ambiguous = !DatacenterMetadataStatus.ACTIVE.name().equals(existing.getMetadataStatus())
|
||||
|| !identityMatches
|
||||
|| !fingerprintMatches;
|
||||
if (ambiguous) {
|
||||
return markTableUnavailable(
|
||||
existing, DatacenterMetadataStatus.CHANGED, true, account);
|
||||
}
|
||||
|
||||
Date now = new Date();
|
||||
existing.setActualTable(discovered.getActualTable());
|
||||
existing.setTableKind(discovered.getTableKind());
|
||||
existing.setPhysicalIdentityKey(discoveredIdentity);
|
||||
existing.setMetadataFingerprint(discoveredFingerprint);
|
||||
existing.setLastSeenAt(now);
|
||||
existing.setMetadataRevision((existing.getMetadataRevision() == null
|
||||
? 0L : existing.getMetadataRevision()) + 1L);
|
||||
existing.setModified(now);
|
||||
existing.setModifiedBy(account == null ? BigInteger.ZERO : account.getId());
|
||||
tableMapper.update(existing);
|
||||
|
||||
Map<String, DatacenterTableField> discoveredFields = detail.getFields().stream()
|
||||
.collect(Collectors.toMap(
|
||||
this::sourceColumnName,
|
||||
field -> field,
|
||||
(first, ignored) -> first,
|
||||
LinkedHashMap::new));
|
||||
for (DatacenterTableField persisted : persistedFields) {
|
||||
DatacenterTableField observed = discoveredFields.get(sourceColumnName(persisted));
|
||||
if (observed == null
|
||||
|| (!legacyCompatible && !Objects.equals(
|
||||
persisted.getMetadataFingerprint(),
|
||||
DatacenterMetadataIdentity.fieldFingerprint(observed)))) {
|
||||
// 表 fingerprint 相同的前提下不应进入该分支;保留 fail-closed 防线。
|
||||
return markTableUnavailable(
|
||||
existing, DatacenterMetadataStatus.CHANGED, true, account);
|
||||
}
|
||||
persisted.setOrdinalPosition(observed.getOrdinalPosition());
|
||||
persisted.setJdbcType(observed.getJdbcType());
|
||||
persisted.setJdbcTypeCode(observed.getJdbcTypeCode());
|
||||
persisted.setNativeTypeName(observed.getNativeTypeName());
|
||||
persisted.setPrecision(observed.getPrecision());
|
||||
persisted.setScale(observed.getScale());
|
||||
persisted.setRequired(observed.getRequired());
|
||||
persisted.setIndexed(observed.getIndexed());
|
||||
persisted.setMetadataFingerprint(
|
||||
DatacenterMetadataIdentity.fieldFingerprint(observed));
|
||||
persisted.setMetadataStatus(DatacenterMetadataStatus.ACTIVE.name());
|
||||
persisted.setLastSeenAt(now);
|
||||
persisted.setModified(now);
|
||||
persisted.setModifiedBy(account == null ? BigInteger.ZERO : account.getId());
|
||||
tableFieldMapper.update(persisted);
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* 标记未发现的已纳管表,并关闭其全部查询入口。
|
||||
*/
|
||||
@Override
|
||||
@Transactional(rollbackFor = Exception.class)
|
||||
public boolean markManagedTableMissing(
|
||||
DatacenterSource source,
|
||||
BigInteger tableId,
|
||||
LoginAccount account) {
|
||||
DatacenterTable existing = tableMapper.selectOneById(tableId);
|
||||
if (existing == null || !Objects.equals(existing.getSourceId(), source.getId())) {
|
||||
throw new BusinessException("已纳管数据表不存在或不属于当前连接");
|
||||
}
|
||||
return markTableUnavailable(
|
||||
existing, DatacenterMetadataStatus.MISSING, false, account);
|
||||
}
|
||||
|
||||
private boolean markTableUnavailable(
|
||||
DatacenterTable table,
|
||||
DatacenterMetadataStatus status,
|
||||
boolean seen,
|
||||
LoginAccount account) {
|
||||
boolean changed = !status.name().equals(table.getMetadataStatus())
|
||||
|| !Integer.valueOf(0).equals(table.getQueryable());
|
||||
Date now = new Date();
|
||||
table.setMetadataStatus(status.name());
|
||||
table.setQueryable(0);
|
||||
if (seen) {
|
||||
table.setLastSeenAt(now);
|
||||
}
|
||||
table.setModified(now);
|
||||
table.setModifiedBy(account == null ? BigInteger.ZERO : account.getId());
|
||||
tableMapper.update(table);
|
||||
for (DatacenterTableField field : getFields(table.getId())) {
|
||||
changed |= !status.name().equals(field.getMetadataStatus())
|
||||
|| !Integer.valueOf(0).equals(field.getQueryable());
|
||||
field.setMetadataStatus(status.name());
|
||||
field.setQueryable(0);
|
||||
field.setSortable(0);
|
||||
field.setWritable(0);
|
||||
if (seen) {
|
||||
field.setLastSeenAt(now);
|
||||
}
|
||||
field.setModified(now);
|
||||
field.setModifiedBy(account == null ? BigInteger.ZERO : account.getId());
|
||||
tableFieldMapper.update(field);
|
||||
}
|
||||
return changed;
|
||||
}
|
||||
|
||||
/**
|
||||
* 删除外部纳管表后以 CAS 推进策略版本,并在提交后通知其他节点。
|
||||
*
|
||||
* @param removedTables 本事务删除的纳管表
|
||||
*/
|
||||
private void incrementExternalScopeRevisions(List<DatacenterTable> removedTables) {
|
||||
Set<BigInteger> sourceIds = removedTables.stream()
|
||||
.map(DatacenterTable::getSourceId)
|
||||
.filter(java.util.Objects::nonNull)
|
||||
.collect(Collectors.toSet());
|
||||
for (BigInteger sourceId : sourceIds) {
|
||||
DatacenterSource source = sourceMapper.selectOneById(sourceId);
|
||||
if (source == null || (!DatacenterSourceType.MYSQL.name().equals(source.getSourceType())
|
||||
&& !DatacenterSourceType.POSTGRESQL.name().equals(source.getSourceType()))) {
|
||||
continue;
|
||||
}
|
||||
long previous = source.getScopeRevision() == null ? 0L : source.getScopeRevision();
|
||||
long definitionRevision = source.getDefinitionRevision() == null
|
||||
? 0L : source.getDefinitionRevision();
|
||||
DatacenterSource patch = new DatacenterSource();
|
||||
patch.setId(source.getId());
|
||||
patch.setScopeRevision(Math.max(1L, previous) + 1L);
|
||||
patch.setModified(new Date());
|
||||
QueryWrapper guard = QueryWrapper.create()
|
||||
.eq(DatacenterSource::getId, source.getId())
|
||||
.eq(DatacenterSource::getTenantId, source.getTenantId())
|
||||
.eq(DatacenterSource::getDefinitionRevision, definitionRevision)
|
||||
.eq(DatacenterSource::getScopeRevision, previous)
|
||||
.ne(DatacenterSource::getStatus,
|
||||
DatacenterSourceStatus.DELETED.code());
|
||||
if (sourceMapper.updateByQuery(patch, guard) != 1) {
|
||||
throw new BusinessException("数据范围已被其他请求更新,请刷新后重试");
|
||||
}
|
||||
DatacenterSource updated = sourceMapper.selectOneById(source.getId());
|
||||
if (updated == null) {
|
||||
throw new BusinessException("数据连接已被删除");
|
||||
}
|
||||
federationChangeNotifier.publishAfterCommit(updated);
|
||||
}
|
||||
}
|
||||
|
||||
private void deletePhysicalTableIfNecessary(DatacenterTable table) {
|
||||
@@ -360,12 +730,130 @@ public class DatacenterDatasetRegistryServiceImpl implements DatacenterDatasetRe
|
||||
if ((table.getActualTable() == null || table.getActualTable().isBlank()) && table.getTableName() != null) {
|
||||
table.setActualTable(table.getTableName());
|
||||
}
|
||||
if (table.getQueryable() == null) {
|
||||
table.setQueryable(1);
|
||||
}
|
||||
if (table.getMetadataStatus() == null) {
|
||||
table.setMetadataStatus(DatacenterMetadataStatus.ACTIVE.name());
|
||||
}
|
||||
if (table.getLastSeenAt() == null) {
|
||||
table.setLastSeenAt(new Date());
|
||||
}
|
||||
if (table.getTimeSemantics() == null) {
|
||||
table.setTimeSemantics("NONE");
|
||||
}
|
||||
if (table.getMetadataRevision() == null) {
|
||||
table.setMetadataRevision(1L);
|
||||
}
|
||||
table.setTableDesc(normalizeDescription(table.getTableDesc()));
|
||||
if (detail.getFields() == null) {
|
||||
detail.setFields(List.of());
|
||||
}
|
||||
}
|
||||
|
||||
private boolean applyCatalogIdentity(
|
||||
DatacenterCatalog catalog,
|
||||
DatacenterSource source,
|
||||
String catalogName) {
|
||||
boolean mysql = DatacenterSourceType.MYSQL.name().equals(source.getSourceType())
|
||||
|| DatacenterSourceType.PROJECT_MYSQL.name().equals(source.getSourceType());
|
||||
String physicalCatalog = mysql ? catalogName : source.getDatabaseName();
|
||||
String physicalSchema = mysql ? null : catalogName;
|
||||
String logicalSchema = catalogName;
|
||||
String namespaceKey = DatacenterMetadataIdentity.namespaceKey(physicalCatalog, physicalSchema);
|
||||
boolean changed = !java.util.Objects.equals(catalog.getLogicalSchemaName(), logicalSchema)
|
||||
|| !java.util.Objects.equals(catalog.getPhysicalCatalogName(), physicalCatalog)
|
||||
|| !java.util.Objects.equals(catalog.getPhysicalSchemaName(), physicalSchema)
|
||||
|| !java.util.Objects.equals(catalog.getNamespaceKey(), namespaceKey)
|
||||
|| !DatacenterMetadataStatus.ACTIVE.name().equals(catalog.getMetadataStatus());
|
||||
catalog.setLogicalSchemaName(logicalSchema);
|
||||
catalog.setPhysicalCatalogName(physicalCatalog);
|
||||
catalog.setPhysicalSchemaName(physicalSchema);
|
||||
catalog.setNamespaceKey(namespaceKey);
|
||||
catalog.setMetadataStatus(DatacenterMetadataStatus.ACTIVE.name());
|
||||
catalog.setLastSeenAt(new Date());
|
||||
return changed;
|
||||
}
|
||||
|
||||
private void applyFieldDefaults(DatacenterTableField field) {
|
||||
if (!hasText(field.getSourceColumnName())) {
|
||||
field.setSourceColumnName(field.getFieldName());
|
||||
}
|
||||
if (field.getNativeTypeName() == null) {
|
||||
field.setNativeTypeName(field.getJdbcType());
|
||||
}
|
||||
field.setMetadataStatus(DatacenterMetadataStatus.ACTIVE.name());
|
||||
field.setLastSeenAt(new Date());
|
||||
if (field.getSensitivityLevel() == null) {
|
||||
field.setSensitivityLevel(DatacenterSensitivityLevel.PUBLIC.name());
|
||||
}
|
||||
field.setMetadataFingerprint(DatacenterMetadataIdentity.fieldFingerprint(field));
|
||||
}
|
||||
|
||||
/**
|
||||
* 使用 V58 已持久化的旧字段信息校验首次元数据升级。
|
||||
*
|
||||
* <p>V58 新增的 JDBC 编码和顺序列没有回填,因此这些空值只表示未知;
|
||||
* 字段名称、旧 JDBC 类型、精度、小数位和必填属性仍必须保持一致。</p>
|
||||
*
|
||||
* @param persistedFields V58 存量字段
|
||||
* @param discoveredFields 当前 JDBC 发现字段
|
||||
* @return 已知旧结构是否与当前结构一致
|
||||
*/
|
||||
private boolean legacyStructureMatches(
|
||||
List<DatacenterTableField> persistedFields,
|
||||
List<DatacenterTableField> discoveredFields) {
|
||||
List<DatacenterTableField> safePersisted = persistedFields == null
|
||||
? List.of() : persistedFields;
|
||||
List<DatacenterTableField> safeDiscovered = discoveredFields == null
|
||||
? List.of() : discoveredFields;
|
||||
if (safePersisted.size() != safeDiscovered.size()) {
|
||||
return false;
|
||||
}
|
||||
Map<String, DatacenterTableField> discoveredByName = safeDiscovered.stream()
|
||||
.collect(Collectors.toMap(
|
||||
this::sourceColumnName,
|
||||
field -> field,
|
||||
(first, ignored) -> first,
|
||||
LinkedHashMap::new));
|
||||
if (discoveredByName.size() != safeDiscovered.size()) {
|
||||
return false;
|
||||
}
|
||||
for (DatacenterTableField persisted : safePersisted) {
|
||||
DatacenterTableField discovered = discoveredByName.get(
|
||||
sourceColumnName(persisted));
|
||||
if (discovered == null
|
||||
|| !sameWhenKnown(persisted.getFieldType(), discovered.getFieldType())
|
||||
|| !sameWhenKnown(persisted.getJdbcType(), discovered.getJdbcType())
|
||||
|| !sameWhenKnown(persisted.getNativeTypeName(), discovered.getNativeTypeName())
|
||||
|| !sameWhenKnown(persisted.getJdbcTypeCode(), discovered.getJdbcTypeCode())
|
||||
|| !sameWhenKnown(persisted.getPrecision(), discovered.getPrecision())
|
||||
|| !sameWhenKnown(persisted.getScale(), discovered.getScale())
|
||||
|| !sameWhenKnown(persisted.getRequired(), discovered.getRequired())) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* 比较旧元数据中的已知值,空值按 V58 未回填处理。
|
||||
*
|
||||
* @param persisted 旧值
|
||||
* @param discovered 当前值
|
||||
* @return 旧值为空或两者相等时返回 true
|
||||
*/
|
||||
private boolean sameWhenKnown(Object persisted, Object discovered) {
|
||||
return persisted == null
|
||||
|| java.util.Objects.equals(persisted, discovered);
|
||||
}
|
||||
|
||||
private String sourceColumnName(DatacenterTableField field) {
|
||||
return hasText(field.getSourceColumnName())
|
||||
? field.getSourceColumnName()
|
||||
: field.getFieldName();
|
||||
}
|
||||
|
||||
private String normalizeDescription(String description) {
|
||||
if (description == null) {
|
||||
return "";
|
||||
|
||||
@@ -0,0 +1,361 @@
|
||||
package tech.easyflow.datacenter.meta.service.impl;
|
||||
|
||||
import com.easyagents.federation.sql.api.FederationSqlException;
|
||||
import com.easyagents.federation.sql.source.RuntimeFingerprint;
|
||||
import com.easyagents.federation.sql.source.SourceProbeResult;
|
||||
import java.math.BigInteger;
|
||||
import java.util.Date;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.stereotype.Service;
|
||||
import tech.easyflow.common.entity.LoginAccount;
|
||||
import tech.easyflow.common.web.exceptions.BusinessException;
|
||||
import tech.easyflow.datacenter.execution.model.DatacenterConnectionTestResult;
|
||||
import tech.easyflow.datacenter.federation.DatacenterFederationRuntime;
|
||||
import tech.easyflow.datacenter.meta.entity.DatacenterSource;
|
||||
import tech.easyflow.datacenter.meta.enums.DatacenterSourceStatus;
|
||||
import tech.easyflow.datacenter.meta.enums.DatacenterSourceType;
|
||||
import tech.easyflow.datacenter.meta.model.DatacenterSourceDraftRequest;
|
||||
import tech.easyflow.datacenter.meta.support.DatacenterSourceConnectionDefaults;
|
||||
import tech.easyflow.datacenter.security.DatacenterCredentialCipher;
|
||||
|
||||
/**
|
||||
* 构造并探测未发布的数据源候选配置,确保凭据和连接默认值只在一个边界内处理。
|
||||
*/
|
||||
@Service
|
||||
public class DatacenterSourceCandidateService {
|
||||
|
||||
private static final Logger LOGGER =
|
||||
LoggerFactory.getLogger(DatacenterSourceCandidateService.class);
|
||||
|
||||
private final DatacenterCredentialCipher credentialCipher;
|
||||
private final DatacenterSourceConnectionDefaults connectionDefaults;
|
||||
private final DatacenterFederationRuntime federationRuntime;
|
||||
|
||||
/**
|
||||
* 创建候选配置服务。
|
||||
*
|
||||
* @param credentialCipher 凭据加密器
|
||||
* @param connectionDefaults JDBC 默认配置
|
||||
* @param federationRuntime Federation Runtime
|
||||
*/
|
||||
public DatacenterSourceCandidateService(
|
||||
DatacenterCredentialCipher credentialCipher,
|
||||
DatacenterSourceConnectionDefaults connectionDefaults,
|
||||
DatacenterFederationRuntime federationRuntime) {
|
||||
this.credentialCipher = credentialCipher;
|
||||
this.connectionDefaults = connectionDefaults;
|
||||
this.federationRuntime = federationRuntime;
|
||||
}
|
||||
|
||||
/**
|
||||
* 构造仅可探测、不可查询的持久化草稿。
|
||||
*
|
||||
* @param request 草稿请求
|
||||
* @param existing 当前草稿;新建时为空
|
||||
* @param account 当前账号
|
||||
* @return 已归一化草稿
|
||||
*/
|
||||
public DatacenterSource buildDraft(
|
||||
DatacenterSourceDraftRequest request,
|
||||
DatacenterSource existing,
|
||||
LoginAccount account) {
|
||||
validateDefinition(request);
|
||||
requireEditableDraft(existing, request.sourceType());
|
||||
DatacenterSource source = existing == null
|
||||
? new DatacenterSource() : copy(existing);
|
||||
source.setSourceName(request.sourceName().trim());
|
||||
source.setSourceCode(trimToNull(request.sourceCode()));
|
||||
source.setSourceType(request.sourceType());
|
||||
source.setAccessMode("READ_ONLY");
|
||||
source.setBuiltinFlag(0);
|
||||
source.setAdapterId("jdbc");
|
||||
source.setAdapterOptionsJson(Map.of());
|
||||
source.setHost(request.host().trim());
|
||||
source.setPort(request.port());
|
||||
source.setDatabaseName(request.databaseName().trim());
|
||||
source.setSchemaName(trimToNull(request.schemaName()));
|
||||
source.setUsername(request.username().trim());
|
||||
source.setDriverClassName(
|
||||
connectionDefaults.defaultDriverClassName(request.sourceType()));
|
||||
source.setJdbcUrl(null);
|
||||
source.setConfigJson(Map.of());
|
||||
applyCredential(source, existing, trimToNull(request.password()));
|
||||
connectionDefaults.normalize(source);
|
||||
requireCredential(source);
|
||||
source.setDefinitionRevision(existing == null
|
||||
? 1L : value(existing.getDefinitionRevision()) + 1L);
|
||||
source.setScopeRevision(Math.max(1L, value(source.getScopeRevision())));
|
||||
source.setDefinitionChecksum(null);
|
||||
source.setCompatibilityStatus(null);
|
||||
source.setMetadataRefreshStatus("PENDING");
|
||||
source.setStatus(DatacenterSourceStatus.DRAFT.code());
|
||||
source.setCapabilitiesJson(Map.of(
|
||||
"capabilities", List.of(
|
||||
"TEST_CONNECTION", "BROWSE_METADATA", "READ_QUERY")));
|
||||
if (source.getTenantId() == null && account != null) {
|
||||
source.setTenantId(account.getTenantId());
|
||||
source.setDeptId(account.getDeptId());
|
||||
}
|
||||
return source;
|
||||
}
|
||||
|
||||
/**
|
||||
* 构造活动连接的下一 revision 候选配置。
|
||||
*
|
||||
* @param current 当前已发布配置
|
||||
* @param request 新连接定义
|
||||
* @return 未发布候选配置
|
||||
*/
|
||||
public DatacenterSource buildReconfigurationCandidate(
|
||||
DatacenterSource current,
|
||||
DatacenterSourceDraftRequest request) {
|
||||
validateDefinition(request);
|
||||
DatacenterSourceStatus status = DatacenterSourceStatus.fromCode(current.getStatus());
|
||||
if (status != DatacenterSourceStatus.READY
|
||||
&& status != DatacenterSourceStatus.DEGRADED) {
|
||||
throw new BusinessException("只有已激活连接可以使用候选配置");
|
||||
}
|
||||
if (!java.util.Objects.equals(current.getSourceType(), request.sourceType())) {
|
||||
throw new BusinessException("数据连接类型创建后不可修改");
|
||||
}
|
||||
DatacenterSource candidate = copy(current);
|
||||
candidate.setSourceName(request.sourceName().trim());
|
||||
String requestedCode = trimToNull(request.sourceCode());
|
||||
candidate.setSourceCode(requestedCode == null
|
||||
? current.getSourceCode() : requestedCode);
|
||||
candidate.setAccessMode("READ_ONLY");
|
||||
candidate.setBuiltinFlag(0);
|
||||
candidate.setAdapterId("jdbc");
|
||||
candidate.setAdapterOptionsJson(Map.of());
|
||||
candidate.setHost(request.host().trim());
|
||||
candidate.setPort(request.port());
|
||||
candidate.setDatabaseName(request.databaseName().trim());
|
||||
candidate.setSchemaName(trimToNull(request.schemaName()));
|
||||
candidate.setUsername(request.username().trim());
|
||||
candidate.setDriverClassName(
|
||||
connectionDefaults.defaultDriverClassName(request.sourceType()));
|
||||
candidate.setJdbcUrl(null);
|
||||
candidate.setConfigJson(Map.of());
|
||||
applyCredential(candidate, current, trimToNull(request.password()));
|
||||
connectionDefaults.normalize(candidate);
|
||||
requireCredential(candidate);
|
||||
candidate.setDefinitionRevision(value(current.getDefinitionRevision()) + 1L);
|
||||
candidate.setScopeRevision(current.getScopeRevision());
|
||||
candidate.setDefinitionChecksum(null);
|
||||
candidate.setStatus(DatacenterSourceStatus.DRAFT.code());
|
||||
candidate.setLastTestStatus(null);
|
||||
candidate.setLastTestMessage(null);
|
||||
candidate.setLastTestedAt(null);
|
||||
return candidate;
|
||||
}
|
||||
|
||||
/**
|
||||
* 使用短生命周期候选上下文探测连接,并将数据库指纹写回候选实体。
|
||||
*
|
||||
* @param candidate 候选配置
|
||||
* @return 安全探测结果
|
||||
*/
|
||||
public DatacenterConnectionTestResult probe(DatacenterSource candidate) {
|
||||
Date testedAt = new Date();
|
||||
DatacenterConnectionTestResult result = new DatacenterConnectionTestResult();
|
||||
try {
|
||||
DatacenterSourceStatus status = DatacenterSourceStatus.fromCode(
|
||||
candidate.getStatus());
|
||||
SourceProbeResult probe = status == DatacenterSourceStatus.READY
|
||||
|| status == DatacenterSourceStatus.DEGRADED
|
||||
? federationRuntime.probe(candidate)
|
||||
: federationRuntime.probeCandidate(candidate);
|
||||
RuntimeFingerprint fingerprint = probe.fingerprint();
|
||||
result.setSuccess(probe.supported());
|
||||
result.setErrorCode(probe.supported() ? null : "ADAPTER_UNSUPPORTED");
|
||||
result.setMessage(bounded(probe.diagnostic()));
|
||||
result.setCapabilities(List.of(
|
||||
"TEST_CONNECTION", "BROWSE_METADATA", "READ_QUERY"));
|
||||
result.setDetails(safeProbeDetails(candidate, fingerprint));
|
||||
candidate.setLastTestStatus(probe.supported() ? "SUCCESS" : "FAILED");
|
||||
candidate.setLastTestMessage(result.getMessage());
|
||||
candidate.setCompatibilityStatus(
|
||||
probe.supported() ? "CODE_SUPPORTED_UNVERIFIED" : "UNSUPPORTED");
|
||||
applyFingerprint(candidate, fingerprint);
|
||||
} catch (FederationSqlException exception) {
|
||||
LOGGER.warn("候选数据源探测失败,sourceId={}, code={}",
|
||||
candidate.getId(), exception.errorCode(), exception);
|
||||
result.setSuccess(false);
|
||||
result.setErrorCode(exception.errorCode().name());
|
||||
result.setMessage(bounded(exception.getMessage()));
|
||||
candidate.setLastTestStatus("FAILED");
|
||||
candidate.setLastTestMessage(result.getMessage());
|
||||
} catch (RuntimeException exception) {
|
||||
LOGGER.error("候选数据源探测发生未预期错误,sourceId={}",
|
||||
candidate.getId(), exception);
|
||||
result.setSuccess(false);
|
||||
result.setErrorCode("CONNECTION_FAILED");
|
||||
result.setMessage("无法连接到数据库,请检查地址、账号和网络");
|
||||
candidate.setLastTestStatus("FAILED");
|
||||
candidate.setLastTestMessage(result.getMessage());
|
||||
}
|
||||
candidate.setLastTestedAt(testedAt);
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* 加密新密码,空密码则保留已有密文。
|
||||
*
|
||||
* @param target 目标实体
|
||||
* @param existing 当前实体
|
||||
* @param password 可选明文密码
|
||||
*/
|
||||
public void applyCredential(
|
||||
DatacenterSource target,
|
||||
DatacenterSource existing,
|
||||
String password) {
|
||||
if (password != null) {
|
||||
target.setCredentialCipher(credentialCipher.encrypt(password));
|
||||
return;
|
||||
}
|
||||
if ((target.getCredentialCipher() == null
|
||||
|| target.getCredentialCipher().isBlank()) && existing != null) {
|
||||
target.setCredentialCipher(existing.getCredentialCipher());
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 从旧兼容配置中读取待加密密码。
|
||||
*
|
||||
* @param configJson 兼容配置
|
||||
* @return 归一化密码;不存在时为空
|
||||
*/
|
||||
public String extractPassword(Map<String, Object> configJson) {
|
||||
if (configJson == null || configJson.get("password") == null) {
|
||||
return null;
|
||||
}
|
||||
return trimToNull(String.valueOf(configJson.get("password")));
|
||||
}
|
||||
|
||||
/**
|
||||
* 复制数据源快照,避免候选探测修改权威实体。
|
||||
*
|
||||
* @param source 原实体
|
||||
* @return 独立副本
|
||||
*/
|
||||
public DatacenterSource copy(DatacenterSource source) {
|
||||
DatacenterSource copy = new DatacenterSource();
|
||||
copy.setId(source.getId());
|
||||
copy.setTenantId(source.getTenantId());
|
||||
copy.setDeptId(source.getDeptId());
|
||||
copy.setCreated(source.getCreated());
|
||||
copy.setCreatedBy(source.getCreatedBy());
|
||||
copy.setSourceName(source.getSourceName());
|
||||
copy.setSourceCode(source.getSourceCode());
|
||||
copy.setSourceType(source.getSourceType());
|
||||
copy.setAdapterId(source.getAdapterId());
|
||||
copy.setAdapterOptionsJson(source.getAdapterOptionsJson());
|
||||
copy.setDefinitionRevision(source.getDefinitionRevision());
|
||||
copy.setDefinitionChecksum(source.getDefinitionChecksum());
|
||||
copy.setScopeRevision(source.getScopeRevision());
|
||||
copy.setAccessMode(source.getAccessMode());
|
||||
copy.setBuiltinFlag(source.getBuiltinFlag());
|
||||
copy.setCredentialCipher(source.getCredentialCipher());
|
||||
copy.setDriverClassName(source.getDriverClassName());
|
||||
copy.setJdbcUrl(source.getJdbcUrl());
|
||||
copy.setHost(source.getHost());
|
||||
copy.setPort(source.getPort());
|
||||
copy.setDatabaseName(source.getDatabaseName());
|
||||
copy.setSchemaName(source.getSchemaName());
|
||||
copy.setUsername(source.getUsername());
|
||||
copy.setConfigJson(source.getConfigJson());
|
||||
copy.setCapabilitiesJson(source.getCapabilitiesJson());
|
||||
copy.setOptions(source.getOptions());
|
||||
copy.setStatus(source.getStatus());
|
||||
copy.setCompatibilityStatus(source.getCompatibilityStatus());
|
||||
copy.setDatabaseProductName(source.getDatabaseProductName());
|
||||
copy.setDatabaseProductVersion(source.getDatabaseProductVersion());
|
||||
copy.setDriverName(source.getDriverName());
|
||||
copy.setDriverVersion(source.getDriverVersion());
|
||||
copy.setMetadataRefreshStatus(source.getMetadataRefreshStatus());
|
||||
copy.setMetadataRefreshedAt(source.getMetadataRefreshedAt());
|
||||
copy.setLastTestStatus(source.getLastTestStatus());
|
||||
copy.setLastTestMessage(source.getLastTestMessage());
|
||||
copy.setLastTestedAt(source.getLastTestedAt());
|
||||
return copy;
|
||||
}
|
||||
|
||||
private void validateDefinition(DatacenterSourceDraftRequest request) {
|
||||
if (request == null || request.sourceName() == null
|
||||
|| request.sourceName().isBlank()) {
|
||||
throw new BusinessException("请输入连接名称");
|
||||
}
|
||||
if (!DatacenterSourceType.MYSQL.name().equals(request.sourceType())
|
||||
&& !DatacenterSourceType.POSTGRESQL.name().equals(request.sourceType())) {
|
||||
throw new BusinessException("当前批次仅支持 MySQL 和 PostgreSQL");
|
||||
}
|
||||
if (request.host() == null || request.host().isBlank()
|
||||
|| request.databaseName() == null || request.databaseName().isBlank()
|
||||
|| request.username() == null || request.username().isBlank()) {
|
||||
throw new BusinessException("请完整填写主机、数据库和用户名");
|
||||
}
|
||||
}
|
||||
|
||||
private void requireEditableDraft(
|
||||
DatacenterSource existing,
|
||||
String requestedSourceType) {
|
||||
if (existing == null) {
|
||||
return;
|
||||
}
|
||||
if (DatacenterSourceStatus.fromCode(existing.getStatus())
|
||||
!= DatacenterSourceStatus.DRAFT) {
|
||||
throw new BusinessException("已激活连接不可直接修改,请使用重配置");
|
||||
}
|
||||
if (!java.util.Objects.equals(existing.getSourceType(), requestedSourceType)) {
|
||||
throw new BusinessException("数据连接类型创建后不可修改");
|
||||
}
|
||||
}
|
||||
|
||||
private void requireCredential(DatacenterSource source) {
|
||||
if (source.getCredentialCipher() == null
|
||||
|| source.getCredentialCipher().isBlank()) {
|
||||
throw new BusinessException("请输入数据库密码");
|
||||
}
|
||||
}
|
||||
|
||||
private Map<String, Object> safeProbeDetails(
|
||||
DatacenterSource source,
|
||||
RuntimeFingerprint fingerprint) {
|
||||
Map<String, Object> details = new LinkedHashMap<>();
|
||||
details.put("databaseProductName", fingerprint.databaseProduct());
|
||||
details.put("databaseProductVersion", fingerprint.databaseVersion());
|
||||
details.put("driverName", fingerprint.driverName());
|
||||
details.put("driverVersion", fingerprint.driverVersion());
|
||||
details.put("effectiveDriverClassName", source.getDriverClassName());
|
||||
details.put("effectivePort", source.getPort());
|
||||
return java.util.Collections.unmodifiableMap(details);
|
||||
}
|
||||
|
||||
private void applyFingerprint(
|
||||
DatacenterSource source,
|
||||
RuntimeFingerprint fingerprint) {
|
||||
source.setDatabaseProductName(fingerprint.databaseProduct());
|
||||
source.setDatabaseProductVersion(fingerprint.databaseVersion());
|
||||
source.setDriverName(fingerprint.driverName());
|
||||
source.setDriverVersion(fingerprint.driverVersion());
|
||||
}
|
||||
|
||||
private String trimToNull(String value) {
|
||||
return value == null || value.isBlank() ? null : value.trim();
|
||||
}
|
||||
|
||||
private long value(Long sourceValue) {
|
||||
return sourceValue == null ? 0L : sourceValue;
|
||||
}
|
||||
|
||||
private String bounded(String value) {
|
||||
if (value == null) {
|
||||
return null;
|
||||
}
|
||||
return value.length() <= 500 ? value : value.substring(0, 500);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,259 @@
|
||||
package tech.easyflow.datacenter.meta.service.impl;
|
||||
|
||||
import com.mybatisflex.core.query.QueryWrapper;
|
||||
import java.math.BigInteger;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Date;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.stream.Collectors;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.springframework.transaction.PlatformTransactionManager;
|
||||
import org.springframework.transaction.support.TransactionTemplate;
|
||||
import tech.easyflow.common.entity.LoginAccount;
|
||||
import tech.easyflow.common.web.exceptions.BusinessException;
|
||||
import tech.easyflow.datacenter.connector.DatacenterConnector;
|
||||
import tech.easyflow.datacenter.connector.DatacenterConnectorRegistry;
|
||||
import tech.easyflow.datacenter.entity.DatacenterTable;
|
||||
import tech.easyflow.datacenter.federation.DatacenterFederationChangeNotifier;
|
||||
import tech.easyflow.datacenter.mapper.DatacenterCatalogMapper;
|
||||
import tech.easyflow.datacenter.mapper.DatacenterSourceMapper;
|
||||
import tech.easyflow.datacenter.mapper.DatacenterTableMapper;
|
||||
import tech.easyflow.datacenter.meta.entity.DatacenterCatalog;
|
||||
import tech.easyflow.datacenter.meta.entity.DatacenterSource;
|
||||
import tech.easyflow.datacenter.meta.enums.DatacenterMetadataStatus;
|
||||
import tech.easyflow.datacenter.meta.enums.DatacenterSourceStatus;
|
||||
import tech.easyflow.datacenter.meta.model.DatacenterManagedMetadataSnapshot;
|
||||
import tech.easyflow.datacenter.meta.model.DatacenterTableDetailMeta;
|
||||
import tech.easyflow.datacenter.meta.service.DatacenterDatasetRegistryService;
|
||||
|
||||
/**
|
||||
* 已纳管元数据刷新协调器,将远程 JDBC 读取与平台数据库短事务明确分离。
|
||||
*/
|
||||
@Service
|
||||
public class DatacenterSourceMetadataRefreshService {
|
||||
|
||||
private static final Logger LOGGER =
|
||||
LoggerFactory.getLogger(DatacenterSourceMetadataRefreshService.class);
|
||||
|
||||
private final DatacenterConnectorRegistry connectorRegistry;
|
||||
private final DatacenterDatasetRegistryService registryService;
|
||||
private final DatacenterCatalogMapper catalogMapper;
|
||||
private final DatacenterTableMapper tableMapper;
|
||||
private final DatacenterSourceMapper sourceMapper;
|
||||
private final DatacenterFederationChangeNotifier changeNotifier;
|
||||
private final TransactionTemplate transactionTemplate;
|
||||
|
||||
/**
|
||||
* 创建元数据刷新协调器。
|
||||
*
|
||||
* @param connectorRegistry Connector 注册表
|
||||
* @param registryService 数据集注册服务
|
||||
* @param catalogMapper 命名空间 Mapper
|
||||
* @param tableMapper 表 Mapper
|
||||
* @param sourceMapper 数据源 Mapper
|
||||
* @param changeNotifier 多节点变更通知器
|
||||
* @param transactionManager 事务管理器
|
||||
*/
|
||||
public DatacenterSourceMetadataRefreshService(
|
||||
DatacenterConnectorRegistry connectorRegistry,
|
||||
DatacenterDatasetRegistryService registryService,
|
||||
DatacenterCatalogMapper catalogMapper,
|
||||
DatacenterTableMapper tableMapper,
|
||||
DatacenterSourceMapper sourceMapper,
|
||||
DatacenterFederationChangeNotifier changeNotifier,
|
||||
PlatformTransactionManager transactionManager) {
|
||||
this.connectorRegistry = connectorRegistry;
|
||||
this.registryService = registryService;
|
||||
this.catalogMapper = catalogMapper;
|
||||
this.tableMapper = tableMapper;
|
||||
this.sourceMapper = sourceMapper;
|
||||
this.changeNotifier = changeNotifier;
|
||||
this.transactionTemplate = new TransactionTemplate(transactionManager);
|
||||
}
|
||||
|
||||
/**
|
||||
* 刷新已纳管对象;外部新表和新字段不会自动扩张查询范围。
|
||||
*
|
||||
* @param source 已完成租户校验的数据源 revision 快照
|
||||
* @param account 当前账号
|
||||
* @return 刷新后的权威数据源
|
||||
*/
|
||||
public DatacenterSource refresh(
|
||||
DatacenterSource source,
|
||||
LoginAccount account) {
|
||||
DatacenterSourceStatus sourceStatus =
|
||||
DatacenterSourceStatus.fromCode(source.getStatus());
|
||||
if (sourceStatus != DatacenterSourceStatus.READY
|
||||
&& sourceStatus != DatacenterSourceStatus.DEGRADED) {
|
||||
throw new BusinessException("当前连接状态不支持刷新元数据");
|
||||
}
|
||||
DatacenterSource refreshed;
|
||||
try {
|
||||
// 远程元数据读取不持有平台数据库事务连接。
|
||||
List<ManagedCatalogRefresh> snapshot = inspect(source);
|
||||
refreshed = transactionTemplate.execute(status ->
|
||||
applySnapshot(source, snapshot, account));
|
||||
} catch (RuntimeException exception) {
|
||||
markFailed(source, account);
|
||||
throw exception;
|
||||
}
|
||||
if (refreshed == null) {
|
||||
throw new BusinessException("元数据刷新事务未完成");
|
||||
}
|
||||
changeNotifier.publishAfterCommit(refreshed);
|
||||
return refreshed;
|
||||
}
|
||||
|
||||
private List<ManagedCatalogRefresh> inspect(DatacenterSource source) {
|
||||
DatacenterConnector connector =
|
||||
connectorRegistry.getConnector(source.getSourceType());
|
||||
List<DatacenterCatalog> catalogs = catalogMapper.selectListByQuery(
|
||||
QueryWrapper.create()
|
||||
.eq(DatacenterCatalog::getSourceId, source.getId())
|
||||
.ne(DatacenterCatalog::getMetadataStatus,
|
||||
DatacenterMetadataStatus.RETIRED.name()));
|
||||
List<ManagedCatalogRefresh> refreshes = new ArrayList<>(catalogs.size());
|
||||
for (DatacenterCatalog catalog : catalogs) {
|
||||
List<DatacenterTable> tables = tableMapper.selectListByQuery(
|
||||
QueryWrapper.create()
|
||||
.eq(DatacenterTable::getSourceId, source.getId())
|
||||
.eq(DatacenterTable::getCatalogId, catalog.getId())
|
||||
.ne(DatacenterTable::getMetadataStatus,
|
||||
DatacenterMetadataStatus.RETIRED.name()));
|
||||
List<String> physicalNames = tables.stream()
|
||||
.map(this::physicalTableName)
|
||||
.toList();
|
||||
DatacenterManagedMetadataSnapshot snapshot =
|
||||
connector.inspectManagedTables(
|
||||
source, catalog.getCatalogName(), physicalNames);
|
||||
Map<String, DatacenterTableDetailMeta> detailsByPhysicalName =
|
||||
snapshot.details().stream()
|
||||
.filter(detail -> detail.getTable() != null)
|
||||
.collect(Collectors.toMap(
|
||||
detail -> physicalTableName(detail.getTable()),
|
||||
detail -> detail,
|
||||
(first, ignored) -> first,
|
||||
LinkedHashMap::new));
|
||||
List<ManagedTableRefresh> tableRefreshes =
|
||||
new ArrayList<>(tables.size());
|
||||
for (DatacenterTable table : tables) {
|
||||
String physicalName = physicalTableName(table);
|
||||
DatacenterTableDetailMeta detail =
|
||||
detailsByPhysicalName.get(physicalName);
|
||||
if (detail != null) {
|
||||
tableRefreshes.add(new ManagedTableRefresh(
|
||||
table.getId(), detail, false));
|
||||
} else if (snapshot.missingTableNames().contains(physicalName)) {
|
||||
tableRefreshes.add(new ManagedTableRefresh(
|
||||
table.getId(), null, true));
|
||||
} else {
|
||||
throw new BusinessException(
|
||||
"元数据刷新结果不完整: " + physicalName);
|
||||
}
|
||||
}
|
||||
refreshes.add(new ManagedCatalogRefresh(catalog, tableRefreshes));
|
||||
}
|
||||
return List.copyOf(refreshes);
|
||||
}
|
||||
|
||||
private DatacenterSource applySnapshot(
|
||||
DatacenterSource source,
|
||||
List<ManagedCatalogRefresh> refreshes,
|
||||
LoginAccount account) {
|
||||
boolean scopeChanged = false;
|
||||
for (ManagedCatalogRefresh catalogRefresh : refreshes) {
|
||||
for (ManagedTableRefresh tableRefresh : catalogRefresh.tables()) {
|
||||
if (tableRefresh.missing()) {
|
||||
scopeChanged |= registryService.markManagedTableMissing(
|
||||
source, tableRefresh.tableId(), account);
|
||||
} else {
|
||||
scopeChanged |= registryService.refreshManagedTable(
|
||||
source,
|
||||
catalogRefresh.catalog(),
|
||||
tableRefresh.tableId(),
|
||||
tableRefresh.detail(),
|
||||
account);
|
||||
}
|
||||
}
|
||||
}
|
||||
DatacenterSource patch = new DatacenterSource();
|
||||
patch.setId(source.getId());
|
||||
patch.setScopeRevision(scopeChanged
|
||||
? Math.max(1L, value(source.getScopeRevision())) + 1L
|
||||
: source.getScopeRevision());
|
||||
patch.setMetadataRefreshStatus(scopeChanged ? "CHANGED" : "SUCCESS");
|
||||
patch.setMetadataRefreshedAt(new Date());
|
||||
patch.setModified(new Date());
|
||||
patch.setModifiedBy(accountId(account));
|
||||
QueryWrapper guard = QueryWrapper.create()
|
||||
.eq(DatacenterSource::getId, source.getId())
|
||||
.eq(DatacenterSource::getTenantId, source.getTenantId())
|
||||
.eq(DatacenterSource::getDefinitionRevision,
|
||||
source.getDefinitionRevision())
|
||||
.eq(DatacenterSource::getScopeRevision, source.getScopeRevision())
|
||||
.eq(DatacenterSource::getStatus, source.getStatus());
|
||||
if (sourceMapper.updateByQuery(patch, guard) != 1) {
|
||||
throw new BusinessException("连接或数据范围已变化,请刷新后重试");
|
||||
}
|
||||
return sourceMapper.selectOneByQuery(QueryWrapper.create()
|
||||
.eq(DatacenterSource::getId, source.getId())
|
||||
.eq(DatacenterSource::getTenantId, source.getTenantId()));
|
||||
}
|
||||
|
||||
private void markFailed(DatacenterSource source, LoginAccount account) {
|
||||
DatacenterSource patch = new DatacenterSource();
|
||||
patch.setId(source.getId());
|
||||
patch.setMetadataRefreshStatus("FAILED");
|
||||
patch.setModified(new Date());
|
||||
patch.setModifiedBy(accountId(account));
|
||||
try {
|
||||
sourceMapper.updateByQuery(patch, QueryWrapper.create()
|
||||
.eq(DatacenterSource::getId, source.getId())
|
||||
.eq(DatacenterSource::getTenantId, source.getTenantId())
|
||||
.eq(DatacenterSource::getDefinitionRevision,
|
||||
source.getDefinitionRevision())
|
||||
.eq(DatacenterSource::getScopeRevision,
|
||||
source.getScopeRevision()));
|
||||
} catch (RuntimeException updateException) {
|
||||
LOGGER.warn("记录元数据刷新失败状态时发生错误,sourceId={}",
|
||||
source.getId(), updateException);
|
||||
}
|
||||
}
|
||||
|
||||
private String physicalTableName(DatacenterTable table) {
|
||||
String actualTable = trimToNull(table.getActualTable());
|
||||
return actualTable == null ? table.getTableName() : actualTable;
|
||||
}
|
||||
|
||||
private String trimToNull(String value) {
|
||||
return value == null || value.isBlank() ? null : value.trim();
|
||||
}
|
||||
|
||||
private long value(Long sourceValue) {
|
||||
return sourceValue == null ? 0L : sourceValue;
|
||||
}
|
||||
|
||||
private BigInteger accountId(LoginAccount account) {
|
||||
return account == null || account.getId() == null
|
||||
? BigInteger.ZERO : account.getId();
|
||||
}
|
||||
|
||||
private record ManagedCatalogRefresh(
|
||||
DatacenterCatalog catalog,
|
||||
List<ManagedTableRefresh> tables) {
|
||||
|
||||
private ManagedCatalogRefresh {
|
||||
tables = List.copyOf(tables);
|
||||
}
|
||||
}
|
||||
|
||||
private record ManagedTableRefresh(
|
||||
BigInteger tableId,
|
||||
DatacenterTableDetailMeta detail,
|
||||
boolean missing) {
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,317 @@
|
||||
package tech.easyflow.datacenter.meta.service.impl;
|
||||
|
||||
import com.mybatisflex.core.query.QueryWrapper;
|
||||
import java.math.BigInteger;
|
||||
import java.util.List;
|
||||
import org.springframework.stereotype.Service;
|
||||
import tech.easyflow.common.web.exceptions.BusinessException;
|
||||
import tech.easyflow.datacenter.connector.DatacenterConnectorRegistry;
|
||||
import tech.easyflow.datacenter.entity.DatacenterTable;
|
||||
import tech.easyflow.datacenter.mapper.DatacenterCatalogMapper;
|
||||
import tech.easyflow.datacenter.meta.entity.DatacenterCatalog;
|
||||
import tech.easyflow.datacenter.meta.entity.DatacenterSource;
|
||||
import tech.easyflow.datacenter.meta.enums.DatacenterMetadataStatus;
|
||||
import tech.easyflow.datacenter.meta.enums.DatacenterSourceStatus;
|
||||
import tech.easyflow.datacenter.meta.enums.DatacenterSourceType;
|
||||
import tech.easyflow.datacenter.meta.model.DatacenterCatalogMeta;
|
||||
import tech.easyflow.datacenter.meta.model.DatacenterMetadataPage;
|
||||
import tech.easyflow.datacenter.meta.model.DatacenterTableDetailMeta;
|
||||
import tech.easyflow.datacenter.meta.service.DatacenterDatasetRegistryService;
|
||||
|
||||
/**
|
||||
* 数据源元数据浏览服务,统一处理物理元数据和已纳管元数据的分页边界。
|
||||
*/
|
||||
@Service
|
||||
public class DatacenterSourceMetadataService {
|
||||
|
||||
private static final long DEFAULT_METADATA_PAGE_SIZE = 100L;
|
||||
private static final long MAX_METADATA_PAGE_SIZE = 200L;
|
||||
|
||||
private final DatacenterConnectorRegistry connectorRegistry;
|
||||
private final DatacenterDatasetRegistryService registryService;
|
||||
private final DatacenterCatalogMapper catalogMapper;
|
||||
|
||||
/**
|
||||
* 创建元数据浏览服务。
|
||||
*
|
||||
* @param connectorRegistry 数据源 Connector 注册表
|
||||
* @param registryService 已纳管数据集注册服务
|
||||
* @param catalogMapper 命名空间 Mapper
|
||||
*/
|
||||
public DatacenterSourceMetadataService(
|
||||
DatacenterConnectorRegistry connectorRegistry,
|
||||
DatacenterDatasetRegistryService registryService,
|
||||
DatacenterCatalogMapper catalogMapper) {
|
||||
this.connectorRegistry = connectorRegistry;
|
||||
this.registryService = registryService;
|
||||
this.catalogMapper = catalogMapper;
|
||||
}
|
||||
|
||||
/**
|
||||
* 返回已发布命名空间,草稿则浏览物理命名空间。
|
||||
*
|
||||
* @param source 已完成租户校验的数据源
|
||||
* @return 命名空间列表
|
||||
*/
|
||||
public List<DatacenterCatalogMeta> listCatalogs(DatacenterSource source) {
|
||||
DatacenterSourceStatus status = DatacenterSourceStatus.fromCode(source.getStatus());
|
||||
if (usesManagedMetadata(source)
|
||||
|| status == DatacenterSourceStatus.READY
|
||||
|| status == DatacenterSourceStatus.DEGRADED
|
||||
|| status == DatacenterSourceStatus.DISABLED) {
|
||||
return managedCatalogs(source.getId()).stream().map(this::toCatalogMeta).toList();
|
||||
}
|
||||
requireSuccessfulProbe(source);
|
||||
return connectorRegistry.getConnector(source.getSourceType()).listCatalogs(source);
|
||||
}
|
||||
|
||||
/**
|
||||
* 分页返回已发布或物理命名空间。
|
||||
*
|
||||
* @param source 已完成租户校验的数据源
|
||||
* @param keyword 名称关键字
|
||||
* @param pageNumber 页码
|
||||
* @param pageSize 页大小
|
||||
* @return 命名空间分页
|
||||
*/
|
||||
public DatacenterMetadataPage<DatacenterCatalogMeta> listCatalogsPage(
|
||||
DatacenterSource source,
|
||||
String keyword,
|
||||
Long pageNumber,
|
||||
Long pageSize) {
|
||||
long actualPage = pageNumber(pageNumber);
|
||||
long actualSize = pageSize(pageSize);
|
||||
DatacenterSourceStatus status = DatacenterSourceStatus.fromCode(source.getStatus());
|
||||
if (usesManagedMetadata(source)
|
||||
|| status == DatacenterSourceStatus.READY
|
||||
|| status == DatacenterSourceStatus.DEGRADED
|
||||
|| status == DatacenterSourceStatus.DISABLED) {
|
||||
List<DatacenterCatalogMeta> managed = managedCatalogs(source.getId()).stream()
|
||||
.map(this::toCatalogMeta)
|
||||
.filter(catalog -> matchesKeyword(catalog.getCatalogName(), keyword))
|
||||
.toList();
|
||||
return DatacenterMetadataPage.slice(managed, actualPage, actualSize);
|
||||
}
|
||||
requireSuccessfulProbe(source);
|
||||
return connectorRegistry.getConnector(source.getSourceType()).listCatalogsPage(
|
||||
source, keyword, actualPage, actualSize);
|
||||
}
|
||||
|
||||
/**
|
||||
* 分页返回命名空间中的表与视图。
|
||||
*
|
||||
* @param source 已完成租户校验的数据源
|
||||
* @param catalogName 命名空间名称
|
||||
* @param keyword 表名关键字
|
||||
* @param pageNumber 页码
|
||||
* @param pageSize 页大小
|
||||
* @return 表分页
|
||||
*/
|
||||
public DatacenterMetadataPage<DatacenterTable> listTables(
|
||||
DatacenterSource source,
|
||||
String catalogName,
|
||||
String keyword,
|
||||
Long pageNumber,
|
||||
Long pageSize) {
|
||||
long actualPage = pageNumber(pageNumber);
|
||||
long actualSize = pageSize(pageSize);
|
||||
DatacenterSourceStatus status = DatacenterSourceStatus.fromCode(source.getStatus());
|
||||
if (usesManagedMetadata(source) || status == DatacenterSourceStatus.DISABLED) {
|
||||
List<DatacenterTable> managed = registryService
|
||||
.listManagedTables(
|
||||
source.getId(), resolveCatalogId(source.getId(), catalogName))
|
||||
.stream()
|
||||
.filter(table -> matchesKeyword(table.getTableName(), keyword))
|
||||
.toList();
|
||||
return DatacenterMetadataPage.slice(managed, actualPage, actualSize);
|
||||
}
|
||||
requireSuccessfulProbe(source);
|
||||
return connectorRegistry.getConnector(source.getSourceType()).listTablesPage(
|
||||
source, catalogName, keyword, actualPage, actualSize);
|
||||
}
|
||||
|
||||
/**
|
||||
* 读取单表字段元数据。
|
||||
*
|
||||
* @param source 已完成租户校验的数据源
|
||||
* @param catalogName 命名空间名称
|
||||
* @param tableName 表名
|
||||
* @param register 是否请求兼容的即时纳管
|
||||
* @param fieldPageNumber 字段页码
|
||||
* @param fieldPageSize 字段页大小
|
||||
* @return 表详情
|
||||
*/
|
||||
public DatacenterTableDetailMeta getTableDetail(
|
||||
DatacenterSource source,
|
||||
String catalogName,
|
||||
String tableName,
|
||||
boolean register,
|
||||
Long fieldPageNumber,
|
||||
Long fieldPageSize) {
|
||||
if (usesManagedMetadata(source)) {
|
||||
BigInteger catalogId = resolveCatalogId(source.getId(), catalogName);
|
||||
DatacenterTable target = registryService
|
||||
.listManagedTables(source.getId(), catalogId).stream()
|
||||
.filter(item -> tableName.equals(item.getTableName()))
|
||||
.findFirst()
|
||||
.orElseThrow(() -> new BusinessException("数据集不存在: " + tableName));
|
||||
DatacenterTableDetailMeta detail = new DatacenterTableDetailMeta();
|
||||
detail.setTable(registryService.getTableWithFields(target.getId()));
|
||||
detail.setFields(detail.getTable().getFields());
|
||||
return pageFields(detail, fieldPageNumber, fieldPageSize);
|
||||
}
|
||||
requireSuccessfulProbe(source);
|
||||
DatacenterTableDetailMeta detail = connectorRegistry
|
||||
.getConnector(source.getSourceType())
|
||||
.getTableDetail(source, catalogName, tableName);
|
||||
if (register) {
|
||||
throw new BusinessException("请使用批量接入以原子更新数据范围");
|
||||
}
|
||||
return pageFields(detail, fieldPageNumber, fieldPageSize);
|
||||
}
|
||||
|
||||
/**
|
||||
* 浏览未发布候选连接的命名空间。
|
||||
*
|
||||
* @param candidate 候选数据源
|
||||
* @return 物理命名空间列表
|
||||
*/
|
||||
public List<DatacenterCatalogMeta> listCandidateCatalogs(
|
||||
DatacenterSource candidate) {
|
||||
return connectorRegistry.getConnector(candidate.getSourceType())
|
||||
.listCatalogs(candidate);
|
||||
}
|
||||
|
||||
/**
|
||||
* 分页浏览未发布候选连接的命名空间。
|
||||
*
|
||||
* @param candidate 候选数据源
|
||||
* @param keyword 名称关键字
|
||||
* @param pageNumber 页码
|
||||
* @param pageSize 页大小
|
||||
* @return 命名空间分页
|
||||
*/
|
||||
public DatacenterMetadataPage<DatacenterCatalogMeta> listCandidateCatalogsPage(
|
||||
DatacenterSource candidate,
|
||||
String keyword,
|
||||
Long pageNumber,
|
||||
Long pageSize) {
|
||||
return connectorRegistry.getConnector(candidate.getSourceType()).listCatalogsPage(
|
||||
candidate, keyword, pageNumber(pageNumber), pageSize(pageSize));
|
||||
}
|
||||
|
||||
/**
|
||||
* 分页浏览未发布候选连接的表。
|
||||
*
|
||||
* @param candidate 候选数据源
|
||||
* @param catalogName 命名空间名称
|
||||
* @param keyword 表名关键字
|
||||
* @param pageNumber 页码
|
||||
* @param pageSize 页大小
|
||||
* @return 表分页
|
||||
*/
|
||||
public DatacenterMetadataPage<DatacenterTable> listCandidateTables(
|
||||
DatacenterSource candidate,
|
||||
String catalogName,
|
||||
String keyword,
|
||||
Long pageNumber,
|
||||
Long pageSize) {
|
||||
return connectorRegistry.getConnector(candidate.getSourceType()).listTablesPage(
|
||||
candidate,
|
||||
trimToNull(catalogName),
|
||||
keyword,
|
||||
pageNumber(pageNumber),
|
||||
pageSize(pageSize));
|
||||
}
|
||||
|
||||
private List<DatacenterCatalog> managedCatalogs(BigInteger sourceId) {
|
||||
return catalogMapper.selectListByQuery(QueryWrapper.create()
|
||||
.eq(DatacenterCatalog::getSourceId, sourceId)
|
||||
.ne(DatacenterCatalog::getMetadataStatus,
|
||||
DatacenterMetadataStatus.RETIRED.name()));
|
||||
}
|
||||
|
||||
private DatacenterTableDetailMeta pageFields(
|
||||
DatacenterTableDetailMeta detail,
|
||||
Long fieldPageNumber,
|
||||
Long fieldPageSize) {
|
||||
List<tech.easyflow.datacenter.entity.DatacenterTableField> fields =
|
||||
detail.getFields() == null ? List.of() : detail.getFields();
|
||||
DatacenterMetadataPage<tech.easyflow.datacenter.entity.DatacenterTableField> page =
|
||||
DatacenterMetadataPage.slice(
|
||||
fields, pageNumber(fieldPageNumber), pageSize(fieldPageSize));
|
||||
detail.setFields(page.records());
|
||||
detail.setFieldPageNumber(page.pageNumber());
|
||||
detail.setFieldPageSize(page.pageSize());
|
||||
detail.setHasMoreFields(page.hasMore());
|
||||
if (detail.getTable() != null) {
|
||||
detail.getTable().setFields(page.records());
|
||||
}
|
||||
return detail;
|
||||
}
|
||||
|
||||
private BigInteger resolveCatalogId(BigInteger sourceId, String catalogName) {
|
||||
if (catalogName == null || catalogName.isBlank()) {
|
||||
return null;
|
||||
}
|
||||
DatacenterCatalog catalog = catalogMapper.selectOneByQuery(QueryWrapper.create()
|
||||
.eq(DatacenterCatalog::getSourceId, sourceId)
|
||||
.eq(DatacenterCatalog::getCatalogName, catalogName)
|
||||
.ne(DatacenterCatalog::getMetadataStatus,
|
||||
DatacenterMetadataStatus.RETIRED.name()));
|
||||
if (catalog == null) {
|
||||
throw new BusinessException("命名空间不存在: " + catalogName);
|
||||
}
|
||||
return catalog.getId();
|
||||
}
|
||||
|
||||
private DatacenterCatalogMeta toCatalogMeta(DatacenterCatalog catalog) {
|
||||
DatacenterCatalogMeta meta = new DatacenterCatalogMeta();
|
||||
meta.setId(catalog.getId());
|
||||
meta.setSourceId(catalog.getSourceId());
|
||||
meta.setCatalogName(catalog.getCatalogName());
|
||||
meta.setCatalogDesc(catalog.getCatalogDesc());
|
||||
meta.setCatalogType(catalog.getCatalogType());
|
||||
meta.setLogicalSchemaName(catalog.getLogicalSchemaName());
|
||||
meta.setPhysicalCatalogName(catalog.getPhysicalCatalogName());
|
||||
meta.setPhysicalSchemaName(catalog.getPhysicalSchemaName());
|
||||
return meta;
|
||||
}
|
||||
|
||||
private void requireSuccessfulProbe(DatacenterSource source) {
|
||||
if (!"SUCCESS".equalsIgnoreCase(source.getLastTestStatus())) {
|
||||
throw new BusinessException("请先测试连接");
|
||||
}
|
||||
}
|
||||
|
||||
private boolean usesManagedMetadata(DatacenterSource source) {
|
||||
return DatacenterSourceType.EXCEL.name().equals(source.getSourceType())
|
||||
|| DatacenterSourceType.EXCEL_MATERIALIZED.name().equals(
|
||||
source.getSourceType());
|
||||
}
|
||||
|
||||
private boolean matchesKeyword(String value, String keyword) {
|
||||
String normalized = trimToNull(keyword);
|
||||
return normalized == null || (value != null
|
||||
&& value.toLowerCase(java.util.Locale.ROOT)
|
||||
.contains(normalized.toLowerCase(java.util.Locale.ROOT)));
|
||||
}
|
||||
|
||||
private long pageNumber(Long requested) {
|
||||
return requested == null || requested < 1L
|
||||
? 1L : Math.min(requested, 1_000_000L);
|
||||
}
|
||||
|
||||
private long pageSize(Long requested) {
|
||||
return requested == null || requested < 1L
|
||||
? DEFAULT_METADATA_PAGE_SIZE
|
||||
: Math.min(requested, MAX_METADATA_PAGE_SIZE);
|
||||
}
|
||||
|
||||
private String trimToNull(String value) {
|
||||
if (value == null || value.isBlank()) {
|
||||
return null;
|
||||
}
|
||||
return value.trim();
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,6 +1,7 @@
|
||||
package tech.easyflow.datacenter.meta.support;
|
||||
|
||||
import cn.hutool.core.util.StrUtil;
|
||||
import com.easyagents.federation.sql.source.KnownJdbcDriver;
|
||||
import org.springframework.stereotype.Component;
|
||||
import tech.easyflow.datacenter.meta.entity.DatacenterSource;
|
||||
import tech.easyflow.datacenter.meta.enums.DatacenterSourceType;
|
||||
@@ -49,11 +50,32 @@ public class DatacenterSourceConnectionDefaults {
|
||||
if (configJson == null || configJson.isEmpty()) {
|
||||
return sanitized;
|
||||
}
|
||||
sanitized.putAll(configJson);
|
||||
sanitized.remove("password");
|
||||
for (Map.Entry<String, Object> entry : configJson.entrySet()) {
|
||||
String key = entry.getKey();
|
||||
if (key == null || isSensitiveKey(key)) {
|
||||
continue;
|
||||
}
|
||||
sanitized.put(key, entry.getValue());
|
||||
}
|
||||
return sanitized;
|
||||
}
|
||||
|
||||
/**
|
||||
* 判断扩展字段名是否可能承载凭据。
|
||||
*
|
||||
* @param key 配置键
|
||||
* @return 是否属于敏感键
|
||||
*/
|
||||
public boolean isSensitiveKey(String key) {
|
||||
String normalized = StrUtil.trimToEmpty(key).toLowerCase(java.util.Locale.ROOT);
|
||||
return normalized.contains("password")
|
||||
|| normalized.contains("passwd")
|
||||
|| normalized.contains("secret")
|
||||
|| normalized.contains("token")
|
||||
|| normalized.contains("credential")
|
||||
|| normalized.endsWith("key");
|
||||
}
|
||||
|
||||
public Integer defaultPort(String sourceType) {
|
||||
DatacenterSourceType resolved = requireSourceType(sourceType);
|
||||
return switch (resolved) {
|
||||
@@ -69,11 +91,12 @@ public class DatacenterSourceConnectionDefaults {
|
||||
public String defaultDriverClassName(String sourceType) {
|
||||
DatacenterSourceType resolved = requireSourceType(sourceType);
|
||||
return switch (resolved) {
|
||||
case MYSQL -> "com.mysql.cj.jdbc.Driver";
|
||||
case POSTGRESQL, GAUSSDB_NATIVE -> "org.postgresql.Driver";
|
||||
case ORACLE -> "oracle.jdbc.OracleDriver";
|
||||
case GBASE_8A -> "com.gbase.jdbc.Driver";
|
||||
case GBASE_8S -> "com.gbasedbt.jdbc.Driver";
|
||||
case MYSQL -> KnownJdbcDriver.MYSQL.driverClassName();
|
||||
case POSTGRESQL -> KnownJdbcDriver.POSTGRESQL.driverClassName();
|
||||
case GAUSSDB_NATIVE -> KnownJdbcDriver.GAUSSDB.driverClassName();
|
||||
case ORACLE -> KnownJdbcDriver.ORACLE.driverClassName();
|
||||
case GBASE_8A -> KnownJdbcDriver.GBASE_8A.driverClassName();
|
||||
case GBASE_8S -> KnownJdbcDriver.GBASE_8S.driverClassName();
|
||||
default -> "";
|
||||
};
|
||||
}
|
||||
@@ -91,10 +114,12 @@ public class DatacenterSourceConnectionDefaults {
|
||||
DatacenterSourceType resolved = requireSourceType(source.getSourceType());
|
||||
return switch (resolved) {
|
||||
case MYSQL -> String.format(
|
||||
"jdbc:mysql://%s:%d/%s?useUnicode=true&characterEncoding=utf-8&serverTimezone=Asia/Shanghai&useSSL=false",
|
||||
"jdbc:mysql://%s:%d/%s?useUnicode=true&characterEncoding=utf-8&serverTimezone=Asia/Shanghai&useSSL=false&useCursorFetch=true&useServerPrepStmts=true&connectTimeout=5000&socketTimeout=35000",
|
||||
host, port, databaseName
|
||||
);
|
||||
case POSTGRESQL -> String.format("jdbc:postgresql://%s:%d/%s", host, port, databaseName);
|
||||
case POSTGRESQL -> String.format(
|
||||
"jdbc:postgresql://%s:%d/%s?connectTimeout=5&socketTimeout=35&cancelSignalTimeout=5",
|
||||
host, port, databaseName);
|
||||
case ORACLE -> String.format("jdbc:oracle:thin:@//%s:%d/%s", host, port, databaseName);
|
||||
case GAUSSDB_NATIVE -> String.format("jdbc:postgresql://%s:%d/%s", host, port, databaseName);
|
||||
case GBASE_8A -> String.format("jdbc:gbase://%s:%d/%s", host, port, databaseName);
|
||||
|
||||
@@ -1,28 +1,109 @@
|
||||
package tech.easyflow.datacenter.security;
|
||||
|
||||
import cn.hutool.crypto.SecureUtil;
|
||||
import cn.hutool.crypto.symmetric.AES;
|
||||
import java.nio.ByteBuffer;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.security.GeneralSecurityException;
|
||||
import java.security.MessageDigest;
|
||||
import java.security.NoSuchAlgorithmException;
|
||||
import java.security.SecureRandom;
|
||||
import java.util.Base64;
|
||||
import javax.crypto.Cipher;
|
||||
import javax.crypto.spec.GCMParameterSpec;
|
||||
import javax.crypto.spec.SecretKeySpec;
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
import java.nio.charset.StandardCharsets;
|
||||
|
||||
/**
|
||||
* 使用部署侧主密钥对数据源密码执行带认证加密。
|
||||
*/
|
||||
@Component
|
||||
public class DatacenterCredentialCipher {
|
||||
|
||||
private static final String DEFAULT_KEY = "easyflow-datacenter-phase1-key";
|
||||
private final AES aes = SecureUtil.aes(SecureUtil.sha256(DEFAULT_KEY).substring(0, 16).getBytes(StandardCharsets.UTF_8));
|
||||
private static final String VERSION = "v1:";
|
||||
private static final String KEY_DERIVATION_DOMAIN = "easyflow-datacenter-credential-v1\u0000";
|
||||
private static final int IV_BYTES = 12;
|
||||
private static final int GCM_TAG_BITS = 128;
|
||||
|
||||
private final SecretKeySpec key;
|
||||
private final SecureRandom secureRandom = new SecureRandom();
|
||||
|
||||
/**
|
||||
* 创建凭据加密器。
|
||||
*
|
||||
* @param masterKey 部署环境提供的独立主密钥,至少 32 个字符
|
||||
* @throws IllegalStateException 主密钥缺失或长度不足时抛出
|
||||
*/
|
||||
public DatacenterCredentialCipher(
|
||||
@Value("${easyflow.datacenter.credential-key:}") String masterKey) {
|
||||
if (masterKey == null || masterKey.length() < 32) {
|
||||
throw new IllegalStateException(
|
||||
"easyflow.datacenter.credential-key must contain at least 32 characters");
|
||||
}
|
||||
// 使用固定域标签派生独立子密钥,避免与部署主密钥的其他用途共享同一 AES key。
|
||||
this.key = new SecretKeySpec(sha256(KEY_DERIVATION_DOMAIN + masterKey), "AES");
|
||||
}
|
||||
|
||||
/**
|
||||
* 加密明文密码。
|
||||
*
|
||||
* @param plainText 明文密码
|
||||
* @return 带版本和随机 IV 的密文;空输入返回 null
|
||||
*/
|
||||
public String encrypt(String plainText) {
|
||||
if (plainText == null || plainText.isBlank()) {
|
||||
return null;
|
||||
}
|
||||
return aes.encryptHex(plainText);
|
||||
byte[] iv = new byte[IV_BYTES];
|
||||
secureRandom.nextBytes(iv);
|
||||
try {
|
||||
Cipher cipher = Cipher.getInstance("AES/GCM/NoPadding");
|
||||
cipher.init(Cipher.ENCRYPT_MODE, key, new GCMParameterSpec(GCM_TAG_BITS, iv));
|
||||
byte[] encrypted = cipher.doFinal(plainText.getBytes(StandardCharsets.UTF_8));
|
||||
return VERSION + Base64.getEncoder().encodeToString(
|
||||
ByteBuffer.allocate(iv.length + encrypted.length)
|
||||
.put(iv)
|
||||
.put(encrypted)
|
||||
.array());
|
||||
} catch (GeneralSecurityException exception) {
|
||||
throw new IllegalStateException("failed to encrypt datacenter credential", exception);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 解密持久化密码。
|
||||
*
|
||||
* @param cipherText 版本化密文
|
||||
* @return 明文密码;空输入返回 null
|
||||
* @throws IllegalStateException 密文格式或认证校验失败时抛出
|
||||
*/
|
||||
public String decrypt(String cipherText) {
|
||||
if (cipherText == null || cipherText.isBlank()) {
|
||||
return null;
|
||||
}
|
||||
return aes.decryptStr(cipherText);
|
||||
if (!cipherText.startsWith(VERSION)) {
|
||||
throw new IllegalStateException("unsupported datacenter credential format");
|
||||
}
|
||||
try {
|
||||
byte[] payload = Base64.getDecoder().decode(cipherText.substring(VERSION.length()));
|
||||
if (payload.length <= IV_BYTES) {
|
||||
throw new IllegalStateException("invalid datacenter credential payload");
|
||||
}
|
||||
byte[] iv = java.util.Arrays.copyOfRange(payload, 0, IV_BYTES);
|
||||
byte[] encrypted = java.util.Arrays.copyOfRange(payload, IV_BYTES, payload.length);
|
||||
Cipher cipher = Cipher.getInstance("AES/GCM/NoPadding");
|
||||
cipher.init(Cipher.DECRYPT_MODE, key, new GCMParameterSpec(GCM_TAG_BITS, iv));
|
||||
return new String(cipher.doFinal(encrypted), StandardCharsets.UTF_8);
|
||||
} catch (GeneralSecurityException | IllegalArgumentException exception) {
|
||||
throw new IllegalStateException("failed to decrypt datacenter credential", exception);
|
||||
}
|
||||
}
|
||||
|
||||
private byte[] sha256(String value) {
|
||||
try {
|
||||
return MessageDigest.getInstance("SHA-256")
|
||||
.digest(value.getBytes(StandardCharsets.UTF_8));
|
||||
} catch (NoSuchAlgorithmException exception) {
|
||||
throw new IllegalStateException("SHA-256 is not available", exception);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,79 @@
|
||||
package tech.easyflow.datacenter.audit;
|
||||
|
||||
import java.math.BigInteger;
|
||||
import java.util.Map;
|
||||
import org.junit.Assert;
|
||||
import org.junit.Test;
|
||||
import org.mockito.ArgumentCaptor;
|
||||
import org.mockito.Mockito;
|
||||
import tech.easyflow.common.entity.LoginAccount;
|
||||
import tech.easyflow.datacenter.mapper.DatacenterQueryAuditMapper;
|
||||
import tech.easyflow.datacenter.meta.entity.DatacenterSource;
|
||||
|
||||
/**
|
||||
* {@link DatacenterQueryAuditService} 审计脱敏回归测试。
|
||||
*/
|
||||
public class DatacenterQueryAuditServiceTest {
|
||||
|
||||
/**
|
||||
* 验证 PostgreSQL dollar quote、字符串、数字和注释内容均不会进入审计 SQL。
|
||||
*/
|
||||
@Test
|
||||
public void startShouldRedactLiteralsAndComments() {
|
||||
DatacenterQueryAuditMapper mapper = Mockito.mock(
|
||||
DatacenterQueryAuditMapper.class);
|
||||
DatacenterQueryAuditService service = new DatacenterQueryAuditService(mapper);
|
||||
DatacenterSource source = new DatacenterSource();
|
||||
source.setId(BigInteger.ONE);
|
||||
source.setTenantId(BigInteger.ONE);
|
||||
|
||||
service.start(
|
||||
"query-1",
|
||||
source,
|
||||
"SELECT $$secret$$, $tag$hidden$tag$, 'password', 42 "
|
||||
+ "/* api-key /* nested */ value */ -- bearer-token\nFROM orders",
|
||||
Map.of(),
|
||||
null,
|
||||
"CONSOLE",
|
||||
null);
|
||||
|
||||
ArgumentCaptor<DatacenterQueryAudit> captor = ArgumentCaptor.forClass(
|
||||
DatacenterQueryAudit.class);
|
||||
Mockito.verify(mapper).insert(captor.capture());
|
||||
String sql = captor.getValue().getParameterizedSql();
|
||||
Assert.assertFalse(sql.contains("secret"));
|
||||
Assert.assertFalse(sql.contains("hidden"));
|
||||
Assert.assertFalse(sql.contains("password"));
|
||||
Assert.assertFalse(sql.contains("api-key"));
|
||||
Assert.assertFalse(sql.contains("bearer-token"));
|
||||
Assert.assertFalse(sql.contains("42"));
|
||||
Assert.assertTrue(sql.contains("FROM orders"));
|
||||
}
|
||||
|
||||
/**
|
||||
* 验证执行账号与业务调用对象分别落入独立审计字段。
|
||||
*/
|
||||
@Test
|
||||
public void startShouldPersistExecutorSeparatelyFromCaller() {
|
||||
DatacenterQueryAuditMapper mapper = Mockito.mock(
|
||||
DatacenterQueryAuditMapper.class);
|
||||
DatacenterQueryAuditService service = new DatacenterQueryAuditService(mapper);
|
||||
DatacenterSource source = new DatacenterSource();
|
||||
source.setId(BigInteger.ONE);
|
||||
source.setTenantId(BigInteger.ONE);
|
||||
LoginAccount account = new LoginAccount();
|
||||
account.setId(BigInteger.valueOf(9001L));
|
||||
account.setTenantId(BigInteger.ONE);
|
||||
|
||||
service.start(
|
||||
"query-2", source, "SELECT id FROM orders", Map.of(),
|
||||
account, "DATASET", "table-2001");
|
||||
|
||||
ArgumentCaptor<DatacenterQueryAudit> captor = ArgumentCaptor.forClass(
|
||||
DatacenterQueryAudit.class);
|
||||
Mockito.verify(mapper).insert(captor.capture());
|
||||
Assert.assertEquals(BigInteger.valueOf(9001L),
|
||||
captor.getValue().getExecutorAccountId());
|
||||
Assert.assertEquals("table-2001", captor.getValue().getCallerId());
|
||||
}
|
||||
}
|
||||
@@ -8,8 +8,12 @@ import org.junit.Assert;
|
||||
import org.junit.Test;
|
||||
import org.mockito.MockedStatic;
|
||||
import org.mockito.Mockito;
|
||||
import tech.easyflow.common.web.exceptions.BusinessException;
|
||||
import tech.easyflow.datacenter.entity.DatacenterTable;
|
||||
import tech.easyflow.datacenter.entity.DatacenterTableField;
|
||||
import tech.easyflow.datacenter.execution.model.DatacenterQueryRequest;
|
||||
import tech.easyflow.datacenter.execution.model.DatacenterQueryFilter;
|
||||
import tech.easyflow.datacenter.execution.model.DatacenterQuerySort;
|
||||
import tech.easyflow.datacenter.meta.enums.DatacenterSourceType;
|
||||
|
||||
import java.math.BigInteger;
|
||||
@@ -47,9 +51,23 @@ public class AbstractInternalTableConnectorTest {
|
||||
public void shouldUseIndependentQueryWrapperForPaginationAfterCount() {
|
||||
DatacenterTable table = new DatacenterTable();
|
||||
table.setActualTable("preview_table");
|
||||
DatacenterTableField field = new DatacenterTableField();
|
||||
field.setFieldName("model_id");
|
||||
field.setSourceColumnName("MODEL_ID");
|
||||
table.setFields(List.of(field));
|
||||
DatacenterQueryRequest request = new DatacenterQueryRequest();
|
||||
request.setPageNumber(1L);
|
||||
request.setPageSize(10L);
|
||||
request.setSelectedColumns(List.of("model_id"));
|
||||
DatacenterQueryFilter filter = new DatacenterQueryFilter();
|
||||
filter.setColumn("model_id");
|
||||
filter.setOperator("EQ");
|
||||
filter.setValue("deepseek-v4-pro");
|
||||
request.setFilters(List.of(filter));
|
||||
DatacenterQuerySort sort = new DatacenterQuerySort();
|
||||
sort.setColumn("model_id");
|
||||
sort.setDirection("DESC");
|
||||
request.setSorts(List.of(sort));
|
||||
|
||||
Row record = new Row();
|
||||
record.put("MODEL_ID", "deepseek-v4-pro");
|
||||
@@ -79,10 +97,22 @@ public class AbstractInternalTableConnectorTest {
|
||||
null, table, request);
|
||||
|
||||
Assert.assertNotSame(countWrapper.get(), pageWrapper.get());
|
||||
Assert.assertTrue(countWrapper.get().toSQL().contains("`MODEL_ID` ="));
|
||||
Assert.assertFalse(countWrapper.get().toSQL().contains("DESC"));
|
||||
Assert.assertTrue(pageWrapper.get().toSQL().contains("`MODEL_ID` DESC"));
|
||||
Assert.assertEquals("deepseek-v4-pro", page.getRecords().get(0).get("model_id"));
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 验证结构化分页入口拒绝调用方拼接原始 WHERE 片段。
|
||||
*/
|
||||
@Test(expected = BusinessException.class)
|
||||
public void shouldRejectRawWhereClause() {
|
||||
AbstractInternalTableConnector.createQueryWrapper(
|
||||
"1 = 1 UNION SELECT secret FROM another_table");
|
||||
}
|
||||
|
||||
/**
|
||||
* 供内部动态表查询测试使用的最小连接器实现。
|
||||
*/
|
||||
|
||||
@@ -0,0 +1,270 @@
|
||||
package tech.easyflow.datacenter.connector.support;
|
||||
|
||||
import java.math.BigInteger;
|
||||
import java.sql.Connection;
|
||||
import java.sql.DatabaseMetaData;
|
||||
import java.sql.ResultSet;
|
||||
import java.sql.Types;
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
import org.junit.Assert;
|
||||
import org.junit.Test;
|
||||
import org.mockito.ArgumentMatchers;
|
||||
import org.mockito.Mockito;
|
||||
import tech.easyflow.datacenter.connector.SqlDialect;
|
||||
import tech.easyflow.datacenter.meta.entity.DatacenterSource;
|
||||
import tech.easyflow.datacenter.meta.enums.DatacenterSourceType;
|
||||
import tech.easyflow.datacenter.meta.model.DatacenterManagedMetadataSnapshot;
|
||||
import tech.easyflow.datacenter.meta.model.DatacenterTableDetailMeta;
|
||||
|
||||
/**
|
||||
* {@link AbstractJdbcConnector} JDBC 元数据精确匹配回归测试。
|
||||
*/
|
||||
public class AbstractJdbcConnectorMetadataTest {
|
||||
|
||||
/**
|
||||
* 验证包含下划线的表名会转义元数据 Pattern,并过滤驱动返回的近似匹配项。
|
||||
*
|
||||
* @throws Exception 模拟 JDBC 调用失败
|
||||
*/
|
||||
@Test
|
||||
public void getTableDetailShouldEscapePatternsAndFilterMetadataRows()
|
||||
throws Exception {
|
||||
Connection connection = Mockito.mock(Connection.class);
|
||||
DatabaseMetaData metadata = Mockito.mock(DatabaseMetaData.class);
|
||||
ResultSet tables = Mockito.mock(ResultSet.class);
|
||||
ResultSet primaryKeys = Mockito.mock(ResultSet.class);
|
||||
ResultSet columns = Mockito.mock(ResultSet.class);
|
||||
Mockito.when(connection.getMetaData()).thenReturn(metadata);
|
||||
Mockito.when(metadata.getSearchStringEscape()).thenReturn("\\");
|
||||
Mockito.when(metadata.getTables(
|
||||
Mockito.eq("analytics"), Mockito.isNull(),
|
||||
Mockito.eq("order\\_item"), ArgumentMatchers.any(String[].class)))
|
||||
.thenReturn(tables);
|
||||
Mockito.when(metadata.getPrimaryKeys(
|
||||
"analytics", null, "order_item"))
|
||||
.thenReturn(primaryKeys);
|
||||
Mockito.when(metadata.getColumns(
|
||||
"analytics", null, "order\\_item", "%"))
|
||||
.thenReturn(columns);
|
||||
|
||||
Mockito.when(tables.next()).thenReturn(true, true, false);
|
||||
Mockito.when(tables.getString("TABLE_NAME"))
|
||||
.thenReturn("orderXitem", "order_item");
|
||||
Mockito.when(tables.getString("TABLE_CAT")).thenReturn("analytics");
|
||||
Mockito.when(tables.getString("TABLE_SCHEM")).thenReturn(null);
|
||||
Mockito.when(tables.getString("REMARKS")).thenReturn("订单");
|
||||
Mockito.when(tables.getString("TABLE_TYPE")).thenReturn("TABLE");
|
||||
Mockito.when(primaryKeys.next()).thenReturn(false);
|
||||
|
||||
Mockito.when(columns.next()).thenReturn(true, true, false);
|
||||
Mockito.when(columns.getString("TABLE_NAME"))
|
||||
.thenReturn("orderXitem", "order_item");
|
||||
Mockito.when(columns.getString("TABLE_CAT")).thenReturn("analytics");
|
||||
Mockito.when(columns.getString("TABLE_SCHEM")).thenReturn(null);
|
||||
Mockito.when(columns.getString("COLUMN_NAME")).thenReturn("id");
|
||||
Mockito.when(columns.getString("REMARKS")).thenReturn("主键");
|
||||
Mockito.when(columns.getString("TYPE_NAME")).thenReturn("BIGINT");
|
||||
Mockito.when(columns.getInt("DATA_TYPE")).thenReturn(Types.BIGINT);
|
||||
Mockito.when(columns.getInt("ORDINAL_POSITION")).thenReturn(1);
|
||||
Mockito.when(columns.getInt("COLUMN_SIZE")).thenReturn(19);
|
||||
Mockito.when(columns.getInt("DECIMAL_DIGITS")).thenReturn(0);
|
||||
Mockito.when(columns.getInt("NULLABLE"))
|
||||
.thenReturn(DatabaseMetaData.columnNoNulls);
|
||||
|
||||
DatacenterSource source = new DatacenterSource();
|
||||
source.setId(BigInteger.ONE);
|
||||
source.setDatabaseName("analytics");
|
||||
DatacenterTableDetailMeta detail = new TestJdbcConnector(connection)
|
||||
.getTableDetail(source, "analytics", "order_item");
|
||||
|
||||
Assert.assertEquals("订单", detail.getTable().getTableDesc());
|
||||
Assert.assertEquals(1, detail.getFields().size());
|
||||
Assert.assertEquals("id", detail.getFields().get(0).getFieldName());
|
||||
Mockito.verify(metadata).getTables(
|
||||
Mockito.eq("analytics"), Mockito.isNull(),
|
||||
Mockito.eq("order\\_item"), ArgumentMatchers.any(String[].class));
|
||||
Mockito.verify(metadata).getColumns(
|
||||
"analytics", null, "order\\_item", "%");
|
||||
}
|
||||
|
||||
/**
|
||||
* 刷新多个已纳管表时复用一个连接,并显式返回缺失对象。
|
||||
*
|
||||
* @throws Exception 模拟 JDBC 调用失败
|
||||
*/
|
||||
@Test
|
||||
public void inspectManagedTablesShouldReuseConnectionAndReportMissing()
|
||||
throws Exception {
|
||||
Connection connection = Mockito.mock(Connection.class);
|
||||
DatabaseMetaData metadata = Mockito.mock(DatabaseMetaData.class);
|
||||
ResultSet orders = Mockito.mock(ResultSet.class);
|
||||
ResultSet missing = Mockito.mock(ResultSet.class);
|
||||
ResultSet primaryKeys = Mockito.mock(ResultSet.class);
|
||||
ResultSet columns = Mockito.mock(ResultSet.class);
|
||||
Mockito.when(connection.getMetaData()).thenReturn(metadata);
|
||||
Mockito.when(metadata.getSearchStringEscape()).thenReturn("\\");
|
||||
Mockito.when(metadata.getTables(
|
||||
Mockito.eq("analytics"), Mockito.isNull(), Mockito.eq("orders"),
|
||||
ArgumentMatchers.any(String[].class)))
|
||||
.thenReturn(orders);
|
||||
Mockito.when(metadata.getTables(
|
||||
Mockito.eq("analytics"), Mockito.isNull(), Mockito.eq("deleted\\_table"),
|
||||
ArgumentMatchers.any(String[].class)))
|
||||
.thenReturn(missing);
|
||||
Mockito.when(metadata.getPrimaryKeys("analytics", null, "orders"))
|
||||
.thenReturn(primaryKeys);
|
||||
Mockito.when(metadata.getColumns("analytics", null, "orders", "%"))
|
||||
.thenReturn(columns);
|
||||
Mockito.when(orders.next()).thenReturn(true, false);
|
||||
Mockito.when(orders.getString("TABLE_NAME")).thenReturn("orders");
|
||||
Mockito.when(orders.getString("TABLE_CAT")).thenReturn("analytics");
|
||||
Mockito.when(orders.getString("TABLE_SCHEM")).thenReturn(null);
|
||||
Mockito.when(orders.getString("TABLE_TYPE")).thenReturn("TABLE");
|
||||
Mockito.when(missing.next()).thenReturn(false);
|
||||
Mockito.when(primaryKeys.next()).thenReturn(false);
|
||||
Mockito.when(columns.next()).thenReturn(true, false);
|
||||
Mockito.when(columns.getString("TABLE_NAME")).thenReturn("orders");
|
||||
Mockito.when(columns.getString("TABLE_CAT")).thenReturn("analytics");
|
||||
Mockito.when(columns.getString("TABLE_SCHEM")).thenReturn(null);
|
||||
Mockito.when(columns.getString("COLUMN_NAME")).thenReturn("id");
|
||||
Mockito.when(columns.getString("TYPE_NAME")).thenReturn("BIGINT");
|
||||
Mockito.when(columns.getInt("DATA_TYPE")).thenReturn(Types.BIGINT);
|
||||
Mockito.when(columns.getInt("ORDINAL_POSITION")).thenReturn(1);
|
||||
Mockito.when(columns.getInt("COLUMN_SIZE")).thenReturn(19);
|
||||
Mockito.when(columns.getInt("DECIMAL_DIGITS")).thenReturn(0);
|
||||
Mockito.when(columns.getInt("NULLABLE"))
|
||||
.thenReturn(DatabaseMetaData.columnNoNulls);
|
||||
|
||||
DatacenterSource source = new DatacenterSource();
|
||||
source.setId(BigInteger.ONE);
|
||||
source.setDatabaseName("analytics");
|
||||
TestJdbcConnector connector = new TestJdbcConnector(connection);
|
||||
DatacenterManagedMetadataSnapshot snapshot = connector.inspectManagedTables(
|
||||
source, "analytics", List.of("orders", "deleted_table"));
|
||||
|
||||
Assert.assertEquals(1, connector.connectionCalls());
|
||||
Assert.assertEquals(1, snapshot.details().size());
|
||||
Assert.assertEquals("orders", snapshot.details().get(0).getTable().getTableName());
|
||||
Assert.assertEquals(Set.of("deleted_table"), snapshot.missingTableNames());
|
||||
}
|
||||
|
||||
/**
|
||||
* 验证 Schema 型数据库不依赖驱动可选的 TABLE_CAT 回填。
|
||||
*
|
||||
* @throws Exception 模拟 JDBC 调用失败
|
||||
*/
|
||||
@Test
|
||||
public void getTableDetailShouldAcceptMissingCatalogForPostgresql()
|
||||
throws Exception {
|
||||
Connection connection = Mockito.mock(Connection.class);
|
||||
DatabaseMetaData metadata = Mockito.mock(DatabaseMetaData.class);
|
||||
ResultSet tables = Mockito.mock(ResultSet.class);
|
||||
ResultSet primaryKeys = Mockito.mock(ResultSet.class);
|
||||
ResultSet columns = Mockito.mock(ResultSet.class);
|
||||
Mockito.when(connection.getMetaData()).thenReturn(metadata);
|
||||
Mockito.when(metadata.getSearchStringEscape()).thenReturn("\\");
|
||||
Mockito.when(metadata.getTables(
|
||||
Mockito.eq("harmony_adapter"), Mockito.eq("public"),
|
||||
Mockito.eq("flyway\\_schema\\_history"),
|
||||
ArgumentMatchers.any(String[].class)))
|
||||
.thenReturn(tables);
|
||||
Mockito.when(metadata.getPrimaryKeys(
|
||||
"harmony_adapter", "public", "flyway_schema_history"))
|
||||
.thenReturn(primaryKeys);
|
||||
Mockito.when(metadata.getColumns(
|
||||
"harmony_adapter", "public",
|
||||
"flyway\\_schema\\_history", "%"))
|
||||
.thenReturn(columns);
|
||||
Mockito.when(tables.next()).thenReturn(true, false);
|
||||
Mockito.when(tables.getString("TABLE_NAME"))
|
||||
.thenReturn("flyway_schema_history");
|
||||
Mockito.when(tables.getString("TABLE_CAT")).thenReturn(null);
|
||||
Mockito.when(tables.getString("TABLE_SCHEM")).thenReturn("public");
|
||||
Mockito.when(tables.getString("TABLE_TYPE")).thenReturn("TABLE");
|
||||
Mockito.when(primaryKeys.next()).thenReturn(false);
|
||||
Mockito.when(columns.next()).thenReturn(true, false);
|
||||
Mockito.when(columns.getString("TABLE_NAME"))
|
||||
.thenReturn("flyway_schema_history");
|
||||
Mockito.when(columns.getString("TABLE_CAT")).thenReturn(null);
|
||||
Mockito.when(columns.getString("TABLE_SCHEM")).thenReturn("public");
|
||||
Mockito.when(columns.getString("COLUMN_NAME")).thenReturn("installed_rank");
|
||||
Mockito.when(columns.getString("TYPE_NAME")).thenReturn("INTEGER");
|
||||
Mockito.when(columns.getInt("DATA_TYPE")).thenReturn(Types.INTEGER);
|
||||
Mockito.when(columns.getInt("ORDINAL_POSITION")).thenReturn(1);
|
||||
Mockito.when(columns.getInt("COLUMN_SIZE")).thenReturn(10);
|
||||
Mockito.when(columns.getInt("DECIMAL_DIGITS")).thenReturn(0);
|
||||
Mockito.when(columns.getInt("NULLABLE"))
|
||||
.thenReturn(DatabaseMetaData.columnNoNulls);
|
||||
|
||||
DatacenterSource source = new DatacenterSource();
|
||||
source.setId(BigInteger.ONE);
|
||||
source.setDatabaseName("harmony_adapter");
|
||||
source.setSchemaName("public");
|
||||
DatacenterTableDetailMeta detail = new TestJdbcConnector(
|
||||
connection, DatacenterSourceType.POSTGRESQL)
|
||||
.getTableDetail(source, "public", "flyway_schema_history");
|
||||
|
||||
Assert.assertEquals("flyway_schema_history", detail.getTable().getTableName());
|
||||
Assert.assertEquals("installed_rank", detail.getFields().get(0).getFieldName());
|
||||
}
|
||||
|
||||
/**
|
||||
* 使用固定模拟连接的测试连接器。
|
||||
*/
|
||||
private static final class TestJdbcConnector extends AbstractJdbcConnector {
|
||||
|
||||
private final Connection connection;
|
||||
private int connectionCalls;
|
||||
|
||||
/**
|
||||
* 创建测试连接器。
|
||||
*
|
||||
* @param connection 模拟 JDBC 连接
|
||||
*/
|
||||
private TestJdbcConnector(Connection connection) {
|
||||
this(connection, DatacenterSourceType.MYSQL);
|
||||
}
|
||||
|
||||
/**
|
||||
* 创建指定数据库类型的测试连接器。
|
||||
*
|
||||
* @param connection 模拟 JDBC 连接
|
||||
* @param sourceType 数据源类型
|
||||
*/
|
||||
private TestJdbcConnector(
|
||||
Connection connection,
|
||||
DatacenterSourceType sourceType) {
|
||||
super(sourceType, Mockito.mock(SqlDialect.class), Set.of());
|
||||
this.connection = connection;
|
||||
}
|
||||
|
||||
/**
|
||||
* 将元数据操作交给固定测试连接。
|
||||
*
|
||||
* @param source 数据源定义
|
||||
* @param cacheable 是否允许连接缓存
|
||||
* @param callback JDBC 回调
|
||||
* @param <T> 返回值类型
|
||||
* @return 回调结果
|
||||
* @throws Exception 回调失败
|
||||
*/
|
||||
@Override
|
||||
protected <T> T withConnection(
|
||||
DatacenterSource source,
|
||||
boolean cacheable,
|
||||
JdbcCallback<T> callback) throws Exception {
|
||||
connectionCalls++;
|
||||
return callback.apply(connection);
|
||||
}
|
||||
|
||||
/**
|
||||
* 返回元数据操作请求连接的次数。
|
||||
*
|
||||
* @return 调用次数
|
||||
*/
|
||||
private int connectionCalls() {
|
||||
return connectionCalls;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,33 +1,105 @@
|
||||
package tech.easyflow.datacenter.execution.service.impl;
|
||||
|
||||
import com.mybatisflex.core.paginate.Page;
|
||||
import com.mybatisflex.core.row.Row;
|
||||
import org.junit.Assert;
|
||||
import org.junit.Test;
|
||||
import org.mockito.ArgumentCaptor;
|
||||
import org.mockito.ArgumentMatchers;
|
||||
import org.mockito.Mockito;
|
||||
import tech.easyflow.datacenter.audit.DatacenterQueryAuditService;
|
||||
import tech.easyflow.datacenter.connector.DatacenterConnector;
|
||||
import tech.easyflow.datacenter.connector.DatacenterConnectorRegistry;
|
||||
import tech.easyflow.datacenter.connector.support.AbstractInternalTableConnector;
|
||||
import tech.easyflow.datacenter.entity.DatacenterTable;
|
||||
import tech.easyflow.datacenter.entity.DatacenterTableField;
|
||||
import tech.easyflow.datacenter.execution.model.DatacenterSqlQueryRequest;
|
||||
import tech.easyflow.datacenter.execution.model.DatacenterSqlColumnView;
|
||||
import tech.easyflow.datacenter.execution.model.DatacenterSqlConsoleResult;
|
||||
import tech.easyflow.datacenter.execution.model.DatacenterQueryFilter;
|
||||
import tech.easyflow.datacenter.execution.model.DatacenterQueryRequest;
|
||||
import tech.easyflow.datacenter.execution.model.DatacenterQuerySort;
|
||||
import tech.easyflow.datacenter.execution.model.DatacenterSchemaResponse;
|
||||
import tech.easyflow.datacenter.execution.model.DatasetRef;
|
||||
import tech.easyflow.datacenter.federation.DatacenterFederationQueryService;
|
||||
import tech.easyflow.datacenter.mapper.DatacenterCatalogMapper;
|
||||
import tech.easyflow.datacenter.mapper.DatacenterDatasetVersionMapper;
|
||||
import tech.easyflow.datacenter.mapper.DatacenterDerivedTableMapper;
|
||||
import tech.easyflow.datacenter.meta.entity.DatacenterCatalog;
|
||||
import tech.easyflow.datacenter.meta.entity.DatacenterSource;
|
||||
import tech.easyflow.datacenter.meta.enums.DatacenterMetadataStatus;
|
||||
import tech.easyflow.datacenter.meta.enums.DatacenterSensitivityLevel;
|
||||
import tech.easyflow.datacenter.meta.service.DatacenterDatasetRegistryService;
|
||||
|
||||
import java.lang.reflect.Field;
|
||||
import java.math.BigInteger;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.function.Consumer;
|
||||
import tech.easyflow.common.entity.LoginAccount;
|
||||
import tech.easyflow.common.web.exceptions.BusinessException;
|
||||
|
||||
/**
|
||||
* {@link DatacenterDatasetQueryServiceImpl} 分页 SQL 读取回归测试。
|
||||
*/
|
||||
public class DatacenterDatasetQueryServiceImplTest {
|
||||
|
||||
/**
|
||||
* 验证结构接口只返回请求的字段页,且不在表对象内重复序列化完整字段。
|
||||
*
|
||||
* @throws Exception 测试依赖注入失败时抛出
|
||||
*/
|
||||
@Test
|
||||
public void getSchemaShouldReturnBoundedFieldPage() throws Exception {
|
||||
BigInteger sourceId = BigInteger.valueOf(6101L);
|
||||
BigInteger catalogId = BigInteger.valueOf(6102L);
|
||||
BigInteger tableId = BigInteger.valueOf(6103L);
|
||||
DatacenterSource source = new DatacenterSource();
|
||||
source.setId(sourceId);
|
||||
DatacenterCatalog catalog = new DatacenterCatalog();
|
||||
catalog.setId(catalogId);
|
||||
DatacenterTable table = new DatacenterTable();
|
||||
table.setId(tableId);
|
||||
table.setSourceId(sourceId);
|
||||
table.setCatalogId(catalogId);
|
||||
table.setFields(List.of(
|
||||
queryableField("first"),
|
||||
queryableField("second"),
|
||||
queryableField("third")));
|
||||
DatasetRef datasetRef = new DatasetRef();
|
||||
datasetRef.setTableId(tableId);
|
||||
DatacenterDatasetRegistryService registry = Mockito.mock(
|
||||
DatacenterDatasetRegistryService.class);
|
||||
Mockito.when(registry.getTableWithFields(tableId)).thenReturn(table);
|
||||
Mockito.when(registry.resolveDatasetRef(tableId)).thenReturn(datasetRef);
|
||||
Mockito.when(registry.getSourceRequired(sourceId)).thenReturn(source);
|
||||
Mockito.when(registry.getCatalogById(catalogId)).thenReturn(catalog);
|
||||
DatacenterDatasetVersionMapper versionMapper = Mockito.mock(
|
||||
DatacenterDatasetVersionMapper.class);
|
||||
DatacenterDerivedTableMapper derivedMapper = Mockito.mock(
|
||||
DatacenterDerivedTableMapper.class);
|
||||
Mockito.when(versionMapper.selectListByQuery(ArgumentMatchers.any()))
|
||||
.thenReturn(List.of());
|
||||
Mockito.when(derivedMapper.selectListByQuery(ArgumentMatchers.any()))
|
||||
.thenReturn(List.of());
|
||||
DatacenterDatasetQueryServiceImpl service =
|
||||
new DatacenterDatasetQueryServiceImpl();
|
||||
setField(service, "registryService", registry);
|
||||
setField(service, "datasetVersionMapper", versionMapper);
|
||||
setField(service, "derivedTableMapper", derivedMapper);
|
||||
|
||||
DatacenterSchemaResponse response = service.getSchema(
|
||||
datasetRef, 2L, 2L);
|
||||
|
||||
Assert.assertEquals(2L, response.getFieldPageNumber());
|
||||
Assert.assertEquals(2L, response.getFieldPageSize());
|
||||
Assert.assertFalse(response.isHasMoreFields());
|
||||
Assert.assertEquals(List.of("third"), response.getFields().stream()
|
||||
.map(DatacenterTableField::getFieldName).toList());
|
||||
Assert.assertTrue(response.getTable().getFields().isEmpty());
|
||||
}
|
||||
|
||||
/**
|
||||
* 验证惰性迭代器逐页读取并保持原始行顺序。
|
||||
*
|
||||
@@ -39,12 +111,15 @@ public class DatacenterDatasetQueryServiceImplTest {
|
||||
BigInteger sourceId = BigInteger.valueOf(1001L);
|
||||
DatacenterSource source = new DatacenterSource();
|
||||
source.setId(sourceId);
|
||||
source.setSourceType("MYSQL");
|
||||
source.setTenantId(BigInteger.ONE);
|
||||
source.setSourceType("PROJECT_MYSQL");
|
||||
DatacenterTable table = new DatacenterTable();
|
||||
table.setId(BigInteger.valueOf(2001L));
|
||||
table.setTenantId(BigInteger.ONE);
|
||||
table.setSourceId(sourceId);
|
||||
table.setTableName("orders");
|
||||
table.setActualTable("orders_actual");
|
||||
table.setFields(List.of(queryableField("id")));
|
||||
|
||||
DatacenterDatasetRegistryService registry =
|
||||
Mockito.mock(
|
||||
@@ -54,6 +129,8 @@ public class DatacenterDatasetQueryServiceImplTest {
|
||||
Mockito.when(registry.listManagedTables(
|
||||
sourceId, null))
|
||||
.thenReturn(List.of(table));
|
||||
Mockito.when(registry.getTableWithFields(table.getId()))
|
||||
.thenReturn(table);
|
||||
DatacenterConnector connector =
|
||||
Mockito.mock(DatacenterConnector.class);
|
||||
Mockito.doAnswer(invocation -> {
|
||||
@@ -74,17 +151,21 @@ public class DatacenterDatasetQueryServiceImplTest {
|
||||
DatacenterConnectorRegistry connectors =
|
||||
Mockito.mock(
|
||||
DatacenterConnectorRegistry.class);
|
||||
Mockito.when(connectors.getConnector("MYSQL"))
|
||||
Mockito.when(connectors.getConnector("PROJECT_MYSQL"))
|
||||
.thenReturn(connector);
|
||||
|
||||
DatacenterDatasetQueryServiceImpl service =
|
||||
new DatacenterDatasetQueryServiceImpl();
|
||||
setField(service, "registryService", registry);
|
||||
setField(service, "connectorRegistry", connectors);
|
||||
setField(service, "queryAuditService",
|
||||
Mockito.mock(DatacenterQueryAuditService.class));
|
||||
DatacenterSqlQueryRequest request =
|
||||
new DatacenterSqlQueryRequest();
|
||||
DatasetRef datasetRef = new DatasetRef();
|
||||
datasetRef.setTenantId(BigInteger.ONE);
|
||||
datasetRef.setSourceId(sourceId);
|
||||
datasetRef.setTableId(table.getId());
|
||||
request.setDatasetRef(datasetRef);
|
||||
request.setSql("SELECT * FROM orders ORDER BY id");
|
||||
|
||||
@@ -105,6 +186,77 @@ public class DatacenterDatasetQueryServiceImplTest {
|
||||
ArgumentMatchers.any());
|
||||
}
|
||||
|
||||
/**
|
||||
* 验证外部 MySQL SQL 查询统一进入 Federation SQL 并保持列标签。
|
||||
*
|
||||
* @throws Exception 测试依赖注入失败时抛出
|
||||
*/
|
||||
@Test
|
||||
public void queryBySqlShouldUseFederationForExternalMysql()
|
||||
throws Exception {
|
||||
BigInteger sourceId = BigInteger.valueOf(4101L);
|
||||
BigInteger tableId = BigInteger.valueOf(4102L);
|
||||
DatacenterSource source = new DatacenterSource();
|
||||
source.setId(sourceId);
|
||||
source.setTenantId(BigInteger.ONE);
|
||||
source.setSourceType("MYSQL");
|
||||
DatacenterTable table = new DatacenterTable();
|
||||
table.setId(tableId);
|
||||
table.setTenantId(BigInteger.ONE);
|
||||
table.setSourceId(sourceId);
|
||||
table.setTableName("orders");
|
||||
table.setActualTable("orders");
|
||||
|
||||
DatacenterDatasetRegistryService registry = Mockito.mock(
|
||||
DatacenterDatasetRegistryService.class);
|
||||
Mockito.when(registry.getSourceRequired(sourceId)).thenReturn(source);
|
||||
Mockito.when(registry.listManagedTables(sourceId, null))
|
||||
.thenReturn(List.of(table));
|
||||
Mockito.when(registry.getTableWithFields(tableId)).thenReturn(table);
|
||||
DatacenterConnector connector = Mockito.mock(DatacenterConnector.class);
|
||||
DatacenterConnectorRegistry connectors = Mockito.mock(
|
||||
DatacenterConnectorRegistry.class);
|
||||
Mockito.when(connectors.getConnector("MYSQL")).thenReturn(connector);
|
||||
DatacenterFederationQueryService federation = Mockito.mock(
|
||||
DatacenterFederationQueryService.class);
|
||||
Mockito.when(federation.execute(
|
||||
Mockito.eq(source),
|
||||
Mockito.anyString(),
|
||||
Mockito.eq(List.of()),
|
||||
Mockito.eq(1_000),
|
||||
Mockito.isNull(),
|
||||
Mockito.eq("DATASET"),
|
||||
Mockito.eq(tableId.toString())))
|
||||
.thenReturn(new DatacenterSqlConsoleResult(
|
||||
"query-1",
|
||||
List.of(new DatacenterSqlColumnView(
|
||||
"c1", "order_id", java.sql.Types.BIGINT,
|
||||
"BIGINT", false)),
|
||||
List.of(java.util.Map.of("c1", "12")),
|
||||
1L,
|
||||
false,
|
||||
3L));
|
||||
|
||||
DatacenterDatasetQueryServiceImpl service =
|
||||
new DatacenterDatasetQueryServiceImpl();
|
||||
setField(service, "registryService", registry);
|
||||
setField(service, "connectorRegistry", connectors);
|
||||
setField(service, "federationQueryService", federation);
|
||||
DatacenterSqlQueryRequest request = new DatacenterSqlQueryRequest();
|
||||
DatasetRef datasetRef = new DatasetRef();
|
||||
datasetRef.setTenantId(BigInteger.ONE);
|
||||
datasetRef.setSourceId(sourceId);
|
||||
datasetRef.setTableId(tableId);
|
||||
request.setDatasetRef(datasetRef);
|
||||
request.setSql("SELECT order_id FROM orders");
|
||||
|
||||
List<Row> rows = service.queryBySql(request);
|
||||
|
||||
Assert.assertEquals("12", rows.get(0).get("order_id"));
|
||||
Mockito.verify(connector, Mockito.never()).queryBySql(
|
||||
Mockito.any(), Mockito.anyString());
|
||||
}
|
||||
|
||||
/**
|
||||
* 验证内部 Excel 查询移除逻辑目录并转换 MySQL 标识符引号。
|
||||
*
|
||||
@@ -117,13 +269,19 @@ public class DatacenterDatasetQueryServiceImplTest {
|
||||
BigInteger catalogId = BigInteger.valueOf(3002L);
|
||||
DatacenterSource source = new DatacenterSource();
|
||||
source.setId(sourceId);
|
||||
source.setTenantId(BigInteger.ONE);
|
||||
source.setSourceType("EXCEL");
|
||||
DatacenterTable table = new DatacenterTable();
|
||||
table.setId(BigInteger.valueOf(3003L));
|
||||
table.setTenantId(BigInteger.ONE);
|
||||
table.setSourceId(sourceId);
|
||||
table.setCatalogId(catalogId);
|
||||
table.setTableName("Sheet1");
|
||||
table.setMaterializedTable("tb_excel_budget");
|
||||
table.setFields(List.of(
|
||||
queryableField("token"),
|
||||
queryableField("token_1"),
|
||||
queryableField("col_id")));
|
||||
|
||||
DatacenterCatalog catalog = new DatacenterCatalog();
|
||||
catalog.setId(catalogId);
|
||||
@@ -134,6 +292,8 @@ public class DatacenterDatasetQueryServiceImplTest {
|
||||
.thenReturn(source);
|
||||
Mockito.when(registry.listManagedTables(sourceId, null))
|
||||
.thenReturn(List.of(table));
|
||||
Mockito.when(registry.getTableWithFields(table.getId()))
|
||||
.thenReturn(table);
|
||||
DatacenterCatalogMapper catalogMapper =
|
||||
Mockito.mock(DatacenterCatalogMapper.class);
|
||||
Mockito.when(catalogMapper.selectListByQuery(
|
||||
@@ -151,10 +311,14 @@ public class DatacenterDatasetQueryServiceImplTest {
|
||||
setField(service, "registryService", registry);
|
||||
setField(service, "connectorRegistry", connectors);
|
||||
setField(service, "catalogMapper", catalogMapper);
|
||||
setField(service, "queryAuditService",
|
||||
Mockito.mock(DatacenterQueryAuditService.class));
|
||||
DatacenterSqlQueryRequest request =
|
||||
new DatacenterSqlQueryRequest();
|
||||
DatasetRef datasetRef = new DatasetRef();
|
||||
datasetRef.setTenantId(BigInteger.ONE);
|
||||
datasetRef.setSourceId(sourceId);
|
||||
datasetRef.setTableId(table.getId());
|
||||
request.setDatasetRef(datasetRef);
|
||||
request.setSql("""
|
||||
SELECT "token", "token_1"
|
||||
@@ -175,6 +339,187 @@ public class DatacenterDatasetQueryServiceImplTest {
|
||||
sqlCaptor.getValue());
|
||||
}
|
||||
|
||||
/**
|
||||
* 验证项目 MySQL 原生 SQL 也会拒绝不可查询字段和同名输出别名绕过。
|
||||
*
|
||||
* @throws Exception 测试依赖注入失败时抛出
|
||||
*/
|
||||
@Test(expected = BusinessException.class)
|
||||
public void queryBySqlShouldEnforceProjectMysqlColumnPolicy()
|
||||
throws Exception {
|
||||
BigInteger sourceId = BigInteger.valueOf(5101L);
|
||||
BigInteger tableId = BigInteger.valueOf(5102L);
|
||||
DatacenterSource source = new DatacenterSource();
|
||||
source.setId(sourceId);
|
||||
source.setTenantId(BigInteger.ONE);
|
||||
source.setSourceType("PROJECT_MYSQL");
|
||||
DatacenterTable table = new DatacenterTable();
|
||||
table.setId(tableId);
|
||||
table.setTenantId(BigInteger.ONE);
|
||||
table.setSourceId(sourceId);
|
||||
table.setTableName("orders");
|
||||
table.setActualTable("orders");
|
||||
DatacenterTableField hidden = queryableField("secret");
|
||||
hidden.setQueryable(0);
|
||||
hidden.setSensitivityLevel(DatacenterSensitivityLevel.RESTRICTED.name());
|
||||
table.setFields(List.of(queryableField("id"), hidden));
|
||||
|
||||
DatacenterDatasetRegistryService registry = Mockito.mock(
|
||||
DatacenterDatasetRegistryService.class);
|
||||
Mockito.when(registry.getSourceRequired(sourceId)).thenReturn(source);
|
||||
Mockito.when(registry.getTableWithFields(tableId)).thenReturn(table);
|
||||
DatacenterConnector connector = Mockito.mock(DatacenterConnector.class);
|
||||
DatacenterConnectorRegistry connectors = Mockito.mock(
|
||||
DatacenterConnectorRegistry.class);
|
||||
Mockito.when(connectors.getConnector("PROJECT_MYSQL"))
|
||||
.thenReturn(connector);
|
||||
|
||||
DatacenterDatasetQueryServiceImpl service =
|
||||
new DatacenterDatasetQueryServiceImpl();
|
||||
setField(service, "registryService", registry);
|
||||
setField(service, "connectorRegistry", connectors);
|
||||
DatacenterSqlQueryRequest request = new DatacenterSqlQueryRequest();
|
||||
DatasetRef datasetRef = new DatasetRef();
|
||||
datasetRef.setTenantId(BigInteger.ONE);
|
||||
datasetRef.setSourceId(sourceId);
|
||||
datasetRef.setTableId(tableId);
|
||||
request.setDatasetRef(datasetRef);
|
||||
request.setSql("SELECT secret AS secret FROM orders");
|
||||
|
||||
service.queryBySql(request);
|
||||
}
|
||||
|
||||
/**
|
||||
* 验证结构化分页审计绑定执行账号且只记录脱敏筛选摘要。
|
||||
*
|
||||
* @throws Exception 测试依赖注入失败时抛出
|
||||
*/
|
||||
@Test
|
||||
@SuppressWarnings({"rawtypes", "unchecked"})
|
||||
public void queryPageShouldAuditAccountAndMaskedFilters()
|
||||
throws Exception {
|
||||
BigInteger sourceId = BigInteger.valueOf(5201L);
|
||||
BigInteger tableId = BigInteger.valueOf(5202L);
|
||||
DatacenterSource source = new DatacenterSource();
|
||||
source.setId(sourceId);
|
||||
source.setTenantId(BigInteger.ONE);
|
||||
source.setSourceType("PROJECT_MYSQL");
|
||||
DatacenterTable table = new DatacenterTable();
|
||||
table.setId(tableId);
|
||||
table.setTenantId(BigInteger.ONE);
|
||||
table.setSourceId(sourceId);
|
||||
table.setTableName("orders");
|
||||
DatacenterTableField idField = queryableField("id");
|
||||
idField.setSortable(1);
|
||||
table.setFields(List.of(idField));
|
||||
|
||||
DatacenterDatasetRegistryService registry = Mockito.mock(
|
||||
DatacenterDatasetRegistryService.class);
|
||||
Mockito.when(registry.getTableWithFields(tableId)).thenReturn(table);
|
||||
Mockito.when(registry.getSourceRequired(sourceId)).thenReturn(source);
|
||||
DatacenterConnector connector = Mockito.mock(DatacenterConnector.class);
|
||||
Mockito.when(connector.queryPage(
|
||||
Mockito.eq(source), Mockito.eq(table), Mockito.any()))
|
||||
.thenReturn(new Page<>(List.of(row(1)), 1L, 20L, 1L));
|
||||
DatacenterConnectorRegistry connectors = Mockito.mock(
|
||||
DatacenterConnectorRegistry.class);
|
||||
Mockito.when(connectors.getConnector("PROJECT_MYSQL"))
|
||||
.thenReturn(connector);
|
||||
DatacenterQueryAuditService auditService = Mockito.mock(
|
||||
DatacenterQueryAuditService.class);
|
||||
|
||||
DatacenterDatasetQueryServiceImpl service =
|
||||
new DatacenterDatasetQueryServiceImpl();
|
||||
setField(service, "registryService", registry);
|
||||
setField(service, "connectorRegistry", connectors);
|
||||
setField(service, "queryAuditService", auditService);
|
||||
DatacenterQueryRequest request = new DatacenterQueryRequest();
|
||||
DatasetRef datasetRef = new DatasetRef();
|
||||
datasetRef.setTenantId(BigInteger.ONE);
|
||||
datasetRef.setTableId(tableId);
|
||||
request.setDatasetRef(datasetRef);
|
||||
request.setPageNumber(1L);
|
||||
request.setPageSize(20L);
|
||||
request.setSelectedColumns(List.of("id"));
|
||||
DatacenterQueryFilter filter = new DatacenterQueryFilter();
|
||||
filter.setColumn("id");
|
||||
filter.setOperator("EQ");
|
||||
filter.setValue("sensitive-value");
|
||||
request.setFilters(List.of(filter));
|
||||
DatacenterQuerySort sort = new DatacenterQuerySort();
|
||||
sort.setColumn("id");
|
||||
sort.setDirection("DESC");
|
||||
request.setSorts(List.of(sort));
|
||||
LoginAccount account = new LoginAccount();
|
||||
account.setId(BigInteger.valueOf(9001L));
|
||||
account.setTenantId(BigInteger.ONE);
|
||||
|
||||
service.queryPage(request, account);
|
||||
|
||||
ArgumentCaptor<Map> parameters = ArgumentCaptor.forClass(Map.class);
|
||||
Mockito.verify(auditService).start(
|
||||
Mockito.anyString(), Mockito.eq(source),
|
||||
Mockito.contains("WHERE id EQ ?"), parameters.capture(),
|
||||
Mockito.same(account), Mockito.eq("DATASET_PAGE"),
|
||||
Mockito.eq(tableId.toString()));
|
||||
Assert.assertFalse(parameters.getValue().toString()
|
||||
.contains("sensitive-value"));
|
||||
Assert.assertTrue(parameters.getValue().toString()
|
||||
.contains("valueCount=1"));
|
||||
}
|
||||
|
||||
/**
|
||||
* 验证后台结构化查询不能凭其他租户的表 ID 越权读取。
|
||||
*/
|
||||
@Test(expected = BusinessException.class)
|
||||
public void queryPageShouldRejectCrossTenantTable() throws Exception {
|
||||
BigInteger sourceId = BigInteger.valueOf(5301L);
|
||||
BigInteger tableId = BigInteger.valueOf(5302L);
|
||||
DatacenterSource source = new DatacenterSource();
|
||||
source.setId(sourceId);
|
||||
source.setTenantId(BigInteger.TWO);
|
||||
source.setSourceType("PROJECT_MYSQL");
|
||||
DatacenterTable table = new DatacenterTable();
|
||||
table.setId(tableId);
|
||||
table.setTenantId(BigInteger.TWO);
|
||||
table.setSourceId(sourceId);
|
||||
table.setTableName("orders");
|
||||
table.setFields(List.of(queryableField("id")));
|
||||
|
||||
DatacenterDatasetRegistryService registry = Mockito.mock(
|
||||
DatacenterDatasetRegistryService.class);
|
||||
Mockito.when(registry.getTableWithFields(tableId)).thenReturn(table);
|
||||
Mockito.when(registry.getSourceRequired(sourceId)).thenReturn(source);
|
||||
DatacenterDatasetQueryServiceImpl service =
|
||||
new DatacenterDatasetQueryServiceImpl();
|
||||
setField(service, "registryService", registry);
|
||||
DatacenterQueryRequest request = new DatacenterQueryRequest();
|
||||
DatasetRef datasetRef = new DatasetRef();
|
||||
datasetRef.setTenantId(BigInteger.ONE);
|
||||
datasetRef.setTableId(tableId);
|
||||
request.setDatasetRef(datasetRef);
|
||||
LoginAccount account = new LoginAccount();
|
||||
account.setId(BigInteger.ONE);
|
||||
account.setTenantId(BigInteger.ONE);
|
||||
|
||||
service.queryPage(request, account);
|
||||
}
|
||||
|
||||
/**
|
||||
* 验证内部与外部结构化查询统一限制单页行数。
|
||||
*/
|
||||
@Test(expected = BusinessException.class)
|
||||
public void queryPageShouldRejectOversizedPage() {
|
||||
DatacenterDatasetQueryServiceImpl service =
|
||||
new DatacenterDatasetQueryServiceImpl();
|
||||
DatacenterQueryRequest request = new DatacenterQueryRequest();
|
||||
request.setDatasetRef(new DatasetRef());
|
||||
request.setPageNumber(1L);
|
||||
request.setPageSize(501L);
|
||||
|
||||
service.queryPage(request);
|
||||
}
|
||||
|
||||
/**
|
||||
* 创建测试数据行。
|
||||
*
|
||||
@@ -187,6 +532,22 @@ public class DatacenterDatasetQueryServiceImplTest {
|
||||
return row;
|
||||
}
|
||||
|
||||
/**
|
||||
* 创建可查询的公开字段。
|
||||
*
|
||||
* @param name 字段名称
|
||||
* @return 字段定义
|
||||
*/
|
||||
private DatacenterTableField queryableField(String name) {
|
||||
DatacenterTableField field = new DatacenterTableField();
|
||||
field.setFieldName(name);
|
||||
field.setSourceColumnName(name);
|
||||
field.setQueryable(1);
|
||||
field.setMetadataStatus(DatacenterMetadataStatus.ACTIVE.name());
|
||||
field.setSensitivityLevel(DatacenterSensitivityLevel.PUBLIC.name());
|
||||
return field;
|
||||
}
|
||||
|
||||
/**
|
||||
* 注入被测服务依赖。
|
||||
*
|
||||
|
||||
@@ -0,0 +1,238 @@
|
||||
package tech.easyflow.datacenter.federation;
|
||||
|
||||
import com.easyagents.federation.sql.api.FederationSqlErrorCode;
|
||||
import com.easyagents.federation.sql.api.FederationSqlException;
|
||||
import com.easyagents.federation.sql.execute.FederationQueryPermit;
|
||||
import com.easyagents.federation.sql.execute.QueryId;
|
||||
import com.easyagents.federation.sql.source.SourceId;
|
||||
import java.time.Duration;
|
||||
import java.util.concurrent.ExecutionException;
|
||||
import java.util.concurrent.ExecutorService;
|
||||
import java.util.concurrent.Executors;
|
||||
import java.util.concurrent.Future;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.concurrent.atomic.AtomicBoolean;
|
||||
import org.junit.Test;
|
||||
|
||||
/**
|
||||
* {@link DatacenterFederationAdmissionController} 分层隔离测试。
|
||||
*/
|
||||
public class DatacenterFederationAdmissionControllerTest {
|
||||
|
||||
/**
|
||||
* 验证已结束查询不会永久保留历史租户和数据源准入项。
|
||||
*/
|
||||
@Test
|
||||
public void completedQueriesShouldReleaseAdmissionEntries() {
|
||||
DatacenterFederationAdmissionController controller =
|
||||
new DatacenterFederationAdmissionController(8, 4, 2);
|
||||
try {
|
||||
for (int index = 0; index < 100; index++) {
|
||||
SourceId sourceId = new SourceId(
|
||||
"tenant-" + index + "-source-" + index);
|
||||
try (FederationQueryPermit ignored = controller.acquire(
|
||||
sourceId,
|
||||
new QueryId("query-" + index),
|
||||
Duration.ZERO)) {
|
||||
org.junit.Assert.assertEquals(
|
||||
1, controller.trackedSourceAdmissionCount());
|
||||
org.junit.Assert.assertEquals(
|
||||
1, controller.trackedTenantAdmissionCount());
|
||||
}
|
||||
org.junit.Assert.assertEquals(
|
||||
0, controller.trackedSourceAdmissionCount());
|
||||
org.junit.Assert.assertEquals(
|
||||
0, controller.trackedTenantAdmissionCount());
|
||||
}
|
||||
} finally {
|
||||
controller.close();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 验证热点数据源耗尽自身配额后不会占用其他数据源的全局许可。
|
||||
*/
|
||||
@Test
|
||||
public void hotSourceShouldNotStarveAnotherSource() {
|
||||
DatacenterFederationAdmissionController controller =
|
||||
new DatacenterFederationAdmissionController(2, 2, 1);
|
||||
SourceId firstSource = new SourceId("tenant-1-source-1");
|
||||
SourceId secondSource = new SourceId("tenant-1-source-2");
|
||||
try (FederationQueryPermit ignored = controller.acquire(
|
||||
firstSource, new QueryId("query-1"), Duration.ZERO)) {
|
||||
try {
|
||||
controller.acquire(
|
||||
firstSource,
|
||||
new QueryId("query-2"),
|
||||
Duration.ZERO);
|
||||
org.junit.Assert.fail("source limit should reject query");
|
||||
} catch (FederationSqlException expected) {
|
||||
// 单源配额按预期先于租户和全局配额生效。
|
||||
}
|
||||
try (FederationQueryPermit second = controller.acquire(
|
||||
secondSource,
|
||||
new QueryId("query-3"),
|
||||
Duration.ZERO)) {
|
||||
org.junit.Assert.assertNotNull(second);
|
||||
}
|
||||
} finally {
|
||||
controller.close();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 验证活动查询释放后等待者取得许可,最终清理全部准入项。
|
||||
*
|
||||
* @throws Exception 并发执行失败
|
||||
*/
|
||||
@Test
|
||||
public void waitingQueryShouldAcquireAndReleaseWithoutLeakingEntries()
|
||||
throws Exception {
|
||||
DatacenterFederationAdmissionController controller =
|
||||
new DatacenterFederationAdmissionController(2, 2, 1);
|
||||
SourceId sourceId = new SourceId("tenant-1-source-1");
|
||||
ExecutorService executor = Executors.newSingleThreadExecutor();
|
||||
FederationQueryPermit active = controller.acquire(
|
||||
sourceId, new QueryId("active"), Duration.ZERO);
|
||||
try {
|
||||
Future<FederationQueryPermit> waiting = executor.submit(() ->
|
||||
controller.acquire(
|
||||
sourceId,
|
||||
new QueryId("waiting"),
|
||||
Duration.ofSeconds(2)));
|
||||
awaitSourceReferences(controller, sourceId, 2);
|
||||
active.close();
|
||||
try (FederationQueryPermit acquired = waiting.get(
|
||||
2, TimeUnit.SECONDS)) {
|
||||
org.junit.Assert.assertNotNull(acquired);
|
||||
}
|
||||
org.junit.Assert.assertEquals(
|
||||
0, controller.trackedSourceAdmissionCount());
|
||||
org.junit.Assert.assertEquals(
|
||||
0, controller.trackedTenantAdmissionCount());
|
||||
} finally {
|
||||
active.close();
|
||||
executor.shutdownNow();
|
||||
controller.close();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 验证等待者取消与 Controller 提前关闭均会正确归还引用。
|
||||
*
|
||||
* @throws Exception 并发执行失败
|
||||
*/
|
||||
@Test
|
||||
public void cancelledAndClosedWaitersShouldReleaseReferences()
|
||||
throws Exception {
|
||||
DatacenterFederationAdmissionController controller =
|
||||
new DatacenterFederationAdmissionController(2, 2, 1);
|
||||
SourceId sourceId = new SourceId("tenant-1-source-1");
|
||||
ExecutorService executor = Executors.newSingleThreadExecutor();
|
||||
FederationQueryPermit active = controller.acquire(
|
||||
sourceId, new QueryId("active"), Duration.ZERO);
|
||||
AtomicBoolean cancellationRequested = new AtomicBoolean();
|
||||
try {
|
||||
Future<FederationQueryPermit> cancelled = executor.submit(() ->
|
||||
controller.acquire(
|
||||
sourceId,
|
||||
new QueryId("cancelled"),
|
||||
Duration.ofSeconds(2),
|
||||
cancellationRequested::get));
|
||||
awaitSourceReferences(controller, sourceId, 2);
|
||||
cancellationRequested.set(true);
|
||||
assertFailureCode(
|
||||
cancelled,
|
||||
FederationSqlErrorCode.QUERY_CANCELLED);
|
||||
org.junit.Assert.assertEquals(
|
||||
1, controller.trackedSourceReferenceCount(sourceId));
|
||||
|
||||
Future<FederationQueryPermit> closing = executor.submit(() ->
|
||||
controller.acquire(
|
||||
sourceId,
|
||||
new QueryId("closing"),
|
||||
Duration.ofSeconds(2)));
|
||||
awaitSourceReferences(controller, sourceId, 2);
|
||||
controller.close();
|
||||
assertFailureCode(
|
||||
closing,
|
||||
FederationSqlErrorCode.ENGINE_CLOSED);
|
||||
active.close();
|
||||
active.close();
|
||||
org.junit.Assert.assertEquals(
|
||||
0, controller.trackedSourceAdmissionCount());
|
||||
org.junit.Assert.assertEquals(
|
||||
0, controller.trackedTenantAdmissionCount());
|
||||
} finally {
|
||||
active.close();
|
||||
executor.shutdownNow();
|
||||
controller.close();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 验证关闭完成后的新查询仍返回统一的引擎关闭错误码。
|
||||
*/
|
||||
@Test
|
||||
public void acquireAfterCloseShouldReportEngineClosed() {
|
||||
DatacenterFederationAdmissionController controller =
|
||||
new DatacenterFederationAdmissionController(2, 2, 1);
|
||||
controller.close();
|
||||
try {
|
||||
controller.acquire(
|
||||
new SourceId("tenant-1-source-1"),
|
||||
new QueryId("closed"),
|
||||
Duration.ZERO);
|
||||
org.junit.Assert.fail("closed admission should reject query");
|
||||
} catch (FederationSqlException exception) {
|
||||
org.junit.Assert.assertEquals(
|
||||
FederationSqlErrorCode.ENGINE_CLOSED,
|
||||
exception.errorCode());
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 等待指定数据源达到预期引用数。
|
||||
*
|
||||
* @param controller 准入控制器
|
||||
* @param sourceId 数据源标识
|
||||
* @param expected 预期引用数
|
||||
* @throws InterruptedException 等待被中断
|
||||
*/
|
||||
private static void awaitSourceReferences(
|
||||
DatacenterFederationAdmissionController controller,
|
||||
SourceId sourceId,
|
||||
int expected) throws InterruptedException {
|
||||
long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(2);
|
||||
while (System.nanoTime() < deadline) {
|
||||
if (controller.trackedSourceReferenceCount(sourceId) == expected) {
|
||||
return;
|
||||
}
|
||||
Thread.sleep(5L);
|
||||
}
|
||||
org.junit.Assert.fail(
|
||||
"source admission references did not reach " + expected);
|
||||
}
|
||||
|
||||
/**
|
||||
* 断言异步准入以指定错误码失败。
|
||||
*
|
||||
* @param future 异步准入结果
|
||||
* @param errorCode 预期错误码
|
||||
* @throws Exception 获取异步结果失败
|
||||
*/
|
||||
private static void assertFailureCode(
|
||||
Future<FederationQueryPermit> future,
|
||||
FederationSqlErrorCode errorCode)
|
||||
throws Exception {
|
||||
try {
|
||||
future.get(2, TimeUnit.SECONDS);
|
||||
org.junit.Assert.fail("query admission should fail");
|
||||
} catch (ExecutionException exception) {
|
||||
Throwable cause = exception.getCause();
|
||||
org.junit.Assert.assertTrue(cause instanceof FederationSqlException);
|
||||
org.junit.Assert.assertEquals(
|
||||
errorCode, ((FederationSqlException) cause).errorCode());
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,191 @@
|
||||
package tech.easyflow.datacenter.federation;
|
||||
|
||||
import com.easyagents.federation.sql.source.FederationDataSourceHandle;
|
||||
import com.easyagents.federation.sql.source.FederationSourceDefinition;
|
||||
import com.zaxxer.hikari.HikariDataSource;
|
||||
import java.math.BigInteger;
|
||||
import java.sql.Connection;
|
||||
import java.sql.DatabaseMetaData;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import org.junit.Assert;
|
||||
import org.junit.Test;
|
||||
import org.mockito.Mockito;
|
||||
import tech.easyflow.datacenter.mapper.DatacenterCatalogMapper;
|
||||
import tech.easyflow.datacenter.mapper.DatacenterSourceMapper;
|
||||
import tech.easyflow.datacenter.meta.entity.DatacenterSource;
|
||||
import tech.easyflow.datacenter.meta.enums.DatacenterSourceStatus;
|
||||
import tech.easyflow.datacenter.meta.enums.DatacenterSourceType;
|
||||
|
||||
/**
|
||||
* {@link DatacenterFederationDataSourceResolver} 状态与 checksum 契约测试。
|
||||
*/
|
||||
public class DatacenterFederationDataSourceResolverTest {
|
||||
|
||||
/**
|
||||
* 验证未发布 checksum 的草稿可以创建仅供 probe 使用的短生命周期句柄。
|
||||
*
|
||||
* @throws Exception JDBC mock 初始化失败
|
||||
*/
|
||||
@Test
|
||||
public void shouldResolveDraftForProbeWithoutPublishedChecksum() throws Exception {
|
||||
DatacenterSource source = source(DatacenterSourceStatus.DRAFT);
|
||||
DatacenterCatalogMapper catalogMapper = Mockito.mock(DatacenterCatalogMapper.class);
|
||||
Mockito.when(catalogMapper.selectListByQuery(Mockito.any())).thenReturn(List.of());
|
||||
DatacenterFederationDefinitionFactory definitionFactory =
|
||||
new DatacenterFederationDefinitionFactory(catalogMapper);
|
||||
FederationSourceDefinition definition = definitionFactory.create(source);
|
||||
|
||||
DatacenterSourceMapper sourceMapper = Mockito.mock(DatacenterSourceMapper.class);
|
||||
Mockito.when(sourceMapper.selectOneById(source.getId())).thenReturn(source);
|
||||
DatacenterFederationDataSourceFactory dataSourceFactory =
|
||||
Mockito.mock(DatacenterFederationDataSourceFactory.class);
|
||||
HikariDataSource pool = Mockito.mock(HikariDataSource.class);
|
||||
Connection connection = Mockito.mock(Connection.class);
|
||||
DatabaseMetaData metadata = Mockito.mock(DatabaseMetaData.class);
|
||||
Mockito.when(dataSourceFactory.create(source)).thenReturn(pool);
|
||||
Mockito.when(pool.getConnection()).thenReturn(connection);
|
||||
Mockito.when(connection.getMetaData()).thenReturn(metadata);
|
||||
Mockito.when(metadata.getDatabaseProductName()).thenReturn("MySQL");
|
||||
Mockito.when(metadata.getDatabaseProductVersion()).thenReturn("8.0");
|
||||
Mockito.when(metadata.getDriverName()).thenReturn("MySQL Connector/J");
|
||||
Mockito.when(metadata.getDriverVersion()).thenReturn("9.0");
|
||||
|
||||
DatacenterFederationDataSourceResolver resolver = new DatacenterFederationDataSourceResolver(
|
||||
sourceMapper, definitionFactory, dataSourceFactory);
|
||||
try (FederationDataSourceHandle handle = resolver.resolve(definition)) {
|
||||
Assert.assertSame(pool, handle.dataSource());
|
||||
}
|
||||
Mockito.verify(pool).close();
|
||||
}
|
||||
|
||||
/**
|
||||
* 验证活动 Runtime 必须匹配已持久化的发布 checksum。
|
||||
*/
|
||||
@Test
|
||||
public void shouldRejectActiveSourceWithoutPublishedChecksum() {
|
||||
DatacenterSource source = source(DatacenterSourceStatus.READY);
|
||||
DatacenterCatalogMapper catalogMapper = Mockito.mock(DatacenterCatalogMapper.class);
|
||||
Mockito.when(catalogMapper.selectListByQuery(Mockito.any())).thenReturn(List.of());
|
||||
DatacenterFederationDefinitionFactory definitionFactory =
|
||||
new DatacenterFederationDefinitionFactory(catalogMapper);
|
||||
FederationSourceDefinition definition = definitionFactory.create(source);
|
||||
DatacenterSourceMapper sourceMapper = Mockito.mock(DatacenterSourceMapper.class);
|
||||
Mockito.when(sourceMapper.selectOneById(source.getId())).thenReturn(source);
|
||||
|
||||
DatacenterFederationDataSourceResolver resolver = new DatacenterFederationDataSourceResolver(
|
||||
sourceMapper, definitionFactory, Mockito.mock(DatacenterFederationDataSourceFactory.class));
|
||||
try {
|
||||
resolver.resolve(definition);
|
||||
Assert.fail("active source without persisted checksum must be rejected");
|
||||
} catch (IllegalStateException expected) {
|
||||
Assert.assertTrue(expected.getMessage().contains("published"));
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 验证候选配置只在同步操作内生效,结束后恢复数据库权威解析。
|
||||
*
|
||||
* @throws Exception JDBC mock 初始化失败
|
||||
*/
|
||||
@Test
|
||||
public void candidateResolutionShouldBeThreadBoundAndAlwaysCleared() throws Exception {
|
||||
DatacenterSource persisted = source(DatacenterSourceStatus.READY);
|
||||
DatacenterSource candidate = source(DatacenterSourceStatus.DRAFT);
|
||||
candidate.setDefinitionRevision(2L);
|
||||
candidate.setDatabaseName("candidate_db");
|
||||
DatacenterCatalogMapper catalogMapper = Mockito.mock(DatacenterCatalogMapper.class);
|
||||
Mockito.when(catalogMapper.selectListByQuery(Mockito.any())).thenReturn(List.of());
|
||||
DatacenterFederationDefinitionFactory definitionFactory =
|
||||
new DatacenterFederationDefinitionFactory(catalogMapper);
|
||||
FederationSourceDefinition definition = definitionFactory.createCandidate(candidate);
|
||||
DatacenterSourceMapper sourceMapper = Mockito.mock(DatacenterSourceMapper.class);
|
||||
Mockito.when(sourceMapper.selectOneById(candidate.getId())).thenReturn(persisted);
|
||||
DatacenterFederationDataSourceFactory dataSourceFactory =
|
||||
Mockito.mock(DatacenterFederationDataSourceFactory.class);
|
||||
HikariDataSource pool = Mockito.mock(HikariDataSource.class);
|
||||
Connection connection = Mockito.mock(Connection.class);
|
||||
DatabaseMetaData metadata = Mockito.mock(DatabaseMetaData.class);
|
||||
Mockito.when(dataSourceFactory.create(candidate)).thenReturn(pool);
|
||||
Mockito.when(pool.getConnection()).thenReturn(connection);
|
||||
Mockito.when(connection.getMetaData()).thenReturn(metadata);
|
||||
Mockito.when(metadata.getDatabaseProductName()).thenReturn("MySQL");
|
||||
Mockito.when(metadata.getDatabaseProductVersion()).thenReturn("8.0");
|
||||
Mockito.when(metadata.getDriverName()).thenReturn("MySQL Connector/J");
|
||||
Mockito.when(metadata.getDriverVersion()).thenReturn("9.0");
|
||||
DatacenterFederationDataSourceResolver resolver =
|
||||
new DatacenterFederationDataSourceResolver(
|
||||
sourceMapper, definitionFactory, dataSourceFactory);
|
||||
|
||||
FederationDataSourceHandle handle = resolver.withCandidate(
|
||||
candidate, definition, () -> resolver.resolve(definition));
|
||||
handle.close();
|
||||
Mockito.verify(sourceMapper, Mockito.never()).selectOneById(candidate.getId());
|
||||
|
||||
try {
|
||||
resolver.resolve(definition);
|
||||
Assert.fail("candidate context must be cleared after the action");
|
||||
} catch (IllegalStateException expected) {
|
||||
Assert.assertTrue(expected.getMessage().contains("revision"));
|
||||
}
|
||||
Mockito.verify(sourceMapper).selectOneById(candidate.getId());
|
||||
}
|
||||
|
||||
/**
|
||||
* 候选操作抛错时也必须清除线程上下文,避免后续请求复用错误凭据。
|
||||
*/
|
||||
@Test
|
||||
public void candidateResolutionShouldClearContextAfterFailure() {
|
||||
DatacenterSource persisted = source(DatacenterSourceStatus.READY);
|
||||
DatacenterSource candidate = source(DatacenterSourceStatus.DRAFT);
|
||||
candidate.setDefinitionRevision(2L);
|
||||
DatacenterCatalogMapper catalogMapper = Mockito.mock(DatacenterCatalogMapper.class);
|
||||
Mockito.when(catalogMapper.selectListByQuery(Mockito.any())).thenReturn(List.of());
|
||||
DatacenterFederationDefinitionFactory definitionFactory =
|
||||
new DatacenterFederationDefinitionFactory(catalogMapper);
|
||||
FederationSourceDefinition definition = definitionFactory.createCandidate(candidate);
|
||||
DatacenterSourceMapper sourceMapper = Mockito.mock(DatacenterSourceMapper.class);
|
||||
Mockito.when(sourceMapper.selectOneById(candidate.getId())).thenReturn(persisted);
|
||||
DatacenterFederationDataSourceResolver resolver =
|
||||
new DatacenterFederationDataSourceResolver(
|
||||
sourceMapper,
|
||||
definitionFactory,
|
||||
Mockito.mock(DatacenterFederationDataSourceFactory.class));
|
||||
|
||||
try {
|
||||
resolver.withCandidate(candidate, definition, () -> {
|
||||
throw new IllegalStateException("simulated candidate failure");
|
||||
});
|
||||
Assert.fail("candidate action must fail");
|
||||
} catch (IllegalStateException expected) {
|
||||
Assert.assertEquals("simulated candidate failure", expected.getMessage());
|
||||
}
|
||||
|
||||
try {
|
||||
resolver.resolve(definition);
|
||||
Assert.fail("resolver must return to persisted source resolution");
|
||||
} catch (IllegalStateException expected) {
|
||||
Assert.assertTrue(expected.getMessage().contains("revision"));
|
||||
}
|
||||
Mockito.verify(sourceMapper).selectOneById(candidate.getId());
|
||||
}
|
||||
|
||||
/**
|
||||
* 创建满足 Definition 最小字段要求的测试数据源。
|
||||
*
|
||||
* @param status 数据源状态
|
||||
* @return 测试数据源
|
||||
*/
|
||||
private DatacenterSource source(DatacenterSourceStatus status) {
|
||||
DatacenterSource source = new DatacenterSource();
|
||||
source.setId(BigInteger.valueOf(101L));
|
||||
source.setTenantId(BigInteger.valueOf(7L));
|
||||
source.setSourceType(DatacenterSourceType.MYSQL.name());
|
||||
source.setDatabaseName("test_db");
|
||||
source.setAdapterId("jdbc");
|
||||
source.setAdapterOptionsJson(Map.of());
|
||||
source.setDefinitionRevision(1L);
|
||||
source.setStatus(status.code());
|
||||
return source;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,184 @@
|
||||
package tech.easyflow.datacenter.federation;
|
||||
|
||||
import com.easyagents.federation.sql.api.FederationSqlEngine;
|
||||
import com.easyagents.federation.sql.execute.QueryId;
|
||||
import java.math.BigInteger;
|
||||
import java.time.Duration;
|
||||
import java.util.UUID;
|
||||
import org.junit.Assert;
|
||||
import org.junit.Test;
|
||||
import org.mockito.Mockito;
|
||||
import org.springframework.beans.factory.ObjectProvider;
|
||||
import org.springframework.data.redis.core.StringRedisTemplate;
|
||||
import org.springframework.data.redis.core.ValueOperations;
|
||||
import tech.easyflow.common.entity.LoginAccount;
|
||||
import tech.easyflow.common.web.exceptions.BusinessException;
|
||||
|
||||
/**
|
||||
* {@link DatacenterFederationQueryCancellationService} 租户隔离与登记回归测试。
|
||||
*/
|
||||
public class DatacenterFederationQueryCancellationServiceTest {
|
||||
|
||||
/**
|
||||
* 验证其他租户无法取消查询,所属租户可以取消并在关闭登记后释放状态。
|
||||
*/
|
||||
@Test
|
||||
public void cancelShouldBeTenantScopedAndReleaseRegistration() {
|
||||
FederationSqlEngine engine = Mockito.mock(FederationSqlEngine.class);
|
||||
DatacenterFederationQueryCancellationService service = service(engine);
|
||||
QueryId queryId = new QueryId(UUID.randomUUID().toString());
|
||||
|
||||
try (DatacenterFederationQueryCancellationService.QueryRegistration ignored =
|
||||
service.register(queryId, account(1L))) {
|
||||
Assert.assertFalse(service.cancel(queryId.value(), account(2L)));
|
||||
Mockito.verify(engine, Mockito.never()).cancel(queryId);
|
||||
|
||||
Assert.assertTrue(service.cancel(queryId.value(), account(1L)));
|
||||
Assert.assertTrue(service.isCancelled(queryId));
|
||||
Mockito.verify(engine).cancel(queryId);
|
||||
}
|
||||
|
||||
Assert.assertFalse(service.isCancelled(queryId));
|
||||
}
|
||||
|
||||
/**
|
||||
* 验证活动 QueryId 不能被并发查询复用。
|
||||
*/
|
||||
@Test(expected = BusinessException.class)
|
||||
public void registerShouldRejectDuplicateActiveQueryId() {
|
||||
DatacenterFederationQueryCancellationService service = service(
|
||||
Mockito.mock(FederationSqlEngine.class));
|
||||
QueryId queryId = new QueryId(UUID.randomUUID().toString());
|
||||
try (DatacenterFederationQueryCancellationService.QueryRegistration ignored =
|
||||
service.register(queryId, account(1L))) {
|
||||
service.register(queryId, account(1L));
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 验证先到达 Redis 的取消请求会在查询登记时立即生效。
|
||||
*/
|
||||
@Test
|
||||
public void registerShouldObservePersistedCancellation() {
|
||||
FederationSqlEngine engine = Mockito.mock(FederationSqlEngine.class);
|
||||
StringRedisTemplate redis = Mockito.mock(StringRedisTemplate.class);
|
||||
Mockito.when(redis.hasKey(Mockito.anyString())).thenReturn(true);
|
||||
DatacenterFederationQueryCancellationService service = service(engine, redis);
|
||||
QueryId queryId = new QueryId(UUID.randomUUID().toString());
|
||||
|
||||
try (DatacenterFederationQueryCancellationService.QueryRegistration ignored =
|
||||
service.register(queryId, account(1L))) {
|
||||
Assert.assertTrue(service.isCancelled(queryId));
|
||||
Mockito.verify(engine).cancel(queryId);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 验证活动查询在丢失 Pub/Sub 提示后仍会通过 Redis 终态核对被取消。
|
||||
*/
|
||||
@Test
|
||||
public void reconcileShouldObserveCancellationCreatedAfterRegister() {
|
||||
FederationSqlEngine engine = Mockito.mock(FederationSqlEngine.class);
|
||||
StringRedisTemplate redis = Mockito.mock(StringRedisTemplate.class);
|
||||
Mockito.when(redis.hasKey(Mockito.anyString()))
|
||||
.thenReturn(false, true);
|
||||
DatacenterFederationQueryCancellationService service = service(engine, redis);
|
||||
QueryId queryId = new QueryId(UUID.randomUUID().toString());
|
||||
|
||||
try (DatacenterFederationQueryCancellationService.QueryRegistration ignored =
|
||||
service.register(queryId, account(1L))) {
|
||||
Assert.assertFalse(service.isCancelled(queryId));
|
||||
|
||||
service.reconcilePersistedCancellations();
|
||||
|
||||
Assert.assertTrue(service.isCancelled(queryId));
|
||||
Mockito.verify(engine).cancel(queryId);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 验证取消请求先持久化终态,再发布跨节点通知。
|
||||
*/
|
||||
@Test
|
||||
@SuppressWarnings("unchecked")
|
||||
public void cancelShouldPersistCancellationBeforePublishing() {
|
||||
FederationSqlEngine engine = Mockito.mock(FederationSqlEngine.class);
|
||||
StringRedisTemplate redis = Mockito.mock(StringRedisTemplate.class);
|
||||
ValueOperations<String, String> values = Mockito.mock(ValueOperations.class);
|
||||
Mockito.when(redis.opsForValue()).thenReturn(values);
|
||||
Mockito.when(redis.convertAndSend(
|
||||
Mockito.eq(DatacenterFederationQueryCancellationService.CHANNEL),
|
||||
Mockito.anyString())).thenReturn(1L);
|
||||
DatacenterFederationQueryCancellationService service = service(engine, redis);
|
||||
QueryId queryId = new QueryId(UUID.randomUUID().toString());
|
||||
|
||||
Assert.assertTrue(service.cancel(queryId.value(), account(1L)));
|
||||
|
||||
org.mockito.InOrder order = Mockito.inOrder(values, redis);
|
||||
order.verify(values).set(
|
||||
Mockito.contains(queryId.value()), Mockito.eq("1"),
|
||||
Mockito.eq(Duration.ofMinutes(10)));
|
||||
order.verify(redis).convertAndSend(
|
||||
Mockito.eq(DatacenterFederationQueryCancellationService.CHANNEL),
|
||||
Mockito.contains(queryId.value()));
|
||||
}
|
||||
|
||||
/**
|
||||
* 验证本地驱动取消失败时仍保留分布式终态,并由周期核对重试本地取消。
|
||||
*/
|
||||
@Test
|
||||
@SuppressWarnings("unchecked")
|
||||
public void cancelShouldPersistAndRetryAfterLocalFailure() {
|
||||
FederationSqlEngine engine = Mockito.mock(FederationSqlEngine.class);
|
||||
StringRedisTemplate redis = Mockito.mock(StringRedisTemplate.class);
|
||||
ValueOperations<String, String> values = Mockito.mock(ValueOperations.class);
|
||||
Mockito.when(redis.opsForValue()).thenReturn(values);
|
||||
Mockito.when(redis.convertAndSend(
|
||||
Mockito.eq(DatacenterFederationQueryCancellationService.CHANNEL),
|
||||
Mockito.anyString())).thenThrow(
|
||||
new IllegalStateException("simulated publish failure"));
|
||||
DatacenterFederationQueryCancellationService service = service(engine, redis);
|
||||
QueryId queryId = new QueryId(UUID.randomUUID().toString());
|
||||
Mockito.doThrow(new IllegalStateException("simulated driver failure"))
|
||||
.doReturn(true)
|
||||
.when(engine).cancel(queryId);
|
||||
|
||||
try (DatacenterFederationQueryCancellationService.QueryRegistration ignored =
|
||||
service.register(queryId, account(1L))) {
|
||||
Assert.assertTrue(service.cancel(queryId.value(), account(1L)));
|
||||
Assert.assertTrue(service.isCancelled(queryId));
|
||||
service.reconcilePersistedCancellations();
|
||||
}
|
||||
|
||||
Mockito.verify(values).set(
|
||||
Mockito.contains(queryId.value()), Mockito.eq("1"),
|
||||
Mockito.eq(Duration.ofMinutes(10)));
|
||||
Mockito.verify(engine, Mockito.times(2)).cancel(queryId);
|
||||
}
|
||||
|
||||
@SuppressWarnings("unchecked")
|
||||
private DatacenterFederationQueryCancellationService service(
|
||||
FederationSqlEngine engine) {
|
||||
return service(engine, null);
|
||||
}
|
||||
|
||||
@SuppressWarnings("unchecked")
|
||||
private DatacenterFederationQueryCancellationService service(
|
||||
FederationSqlEngine engine,
|
||||
StringRedisTemplate redis) {
|
||||
DatacenterFederationRuntime runtime = Mockito.mock(
|
||||
DatacenterFederationRuntime.class);
|
||||
Mockito.when(runtime.engine()).thenReturn(engine);
|
||||
ObjectProvider<StringRedisTemplate> redisProvider = Mockito.mock(
|
||||
ObjectProvider.class);
|
||||
Mockito.when(redisProvider.getIfAvailable()).thenReturn(redis);
|
||||
return new DatacenterFederationQueryCancellationService(
|
||||
runtime, redisProvider);
|
||||
}
|
||||
|
||||
private LoginAccount account(long tenantId) {
|
||||
LoginAccount account = new LoginAccount();
|
||||
account.setTenantId(BigInteger.valueOf(tenantId));
|
||||
return account;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,109 @@
|
||||
package tech.easyflow.datacenter.federation;
|
||||
|
||||
import com.easyagents.federation.sql.api.FederationSqlEngine;
|
||||
import com.easyagents.federation.sql.api.SqlExecutionContext;
|
||||
import com.easyagents.federation.sql.compile.FederationSqlPlan;
|
||||
import com.easyagents.federation.sql.compile.SqlCompileRequest;
|
||||
import com.easyagents.federation.sql.execute.FederationResultCursor;
|
||||
import com.easyagents.federation.sql.source.SourceId;
|
||||
import java.math.BigInteger;
|
||||
import java.util.List;
|
||||
import java.util.UUID;
|
||||
import org.apache.calcite.rel.RelNode;
|
||||
import org.apache.calcite.rel.RelRoot;
|
||||
import org.apache.calcite.rel.type.RelDataType;
|
||||
import org.apache.calcite.sql.SqlKind;
|
||||
import org.apache.calcite.sql.parser.SqlParser;
|
||||
import org.junit.Assert;
|
||||
import org.junit.Test;
|
||||
import org.mockito.ArgumentCaptor;
|
||||
import org.mockito.Mockito;
|
||||
import org.springframework.beans.factory.ObjectProvider;
|
||||
import org.springframework.data.redis.core.StringRedisTemplate;
|
||||
import tech.easyflow.datacenter.audit.DatacenterQueryAuditService;
|
||||
import tech.easyflow.datacenter.meta.entity.DatacenterSource;
|
||||
import tech.easyflow.datacenter.meta.enums.DatacenterSourceStatus;
|
||||
|
||||
/**
|
||||
* {@link DatacenterFederationQueryService} 流式查询取消生命周期回归测试。
|
||||
*/
|
||||
public class DatacenterFederationQueryServiceTest {
|
||||
|
||||
/**
|
||||
* 验证调用方 QueryId 会进入执行上下文,完成后释放登记并允许安全复用。
|
||||
*
|
||||
* @throws Exception SQL 解析失败
|
||||
*/
|
||||
@Test
|
||||
@SuppressWarnings("unchecked")
|
||||
public void consumeShouldUseRequestedQueryIdAndReleaseRegistration()
|
||||
throws Exception {
|
||||
DatacenterFederationRuntime runtime = Mockito.mock(
|
||||
DatacenterFederationRuntime.class);
|
||||
FederationSqlEngine engine = Mockito.mock(FederationSqlEngine.class);
|
||||
DatacenterFederationDefinitionFactory definitions = Mockito.mock(
|
||||
DatacenterFederationDefinitionFactory.class);
|
||||
Mockito.when(runtime.engine()).thenReturn(engine);
|
||||
Mockito.when(runtime.definitions()).thenReturn(definitions);
|
||||
SourceId sourceId = new SourceId("tenant-1-source-1001");
|
||||
|
||||
DatacenterSource source = new DatacenterSource();
|
||||
source.setId(BigInteger.valueOf(1001L));
|
||||
source.setStatus(DatacenterSourceStatus.READY.code());
|
||||
source.setDefinitionRevision(1L);
|
||||
source.setScopeRevision(1L);
|
||||
Mockito.when(definitions.sourceId(source)).thenReturn(sourceId);
|
||||
|
||||
RelDataType rowType = Mockito.mock(RelDataType.class);
|
||||
RelNode root = Mockito.mock(RelNode.class);
|
||||
Mockito.when(root.getRowType()).thenReturn(rowType);
|
||||
RelRoot relRoot = RelRoot.of(root, rowType, SqlKind.SELECT);
|
||||
FederationSqlPlan plan = Mockito.mock(FederationSqlPlan.class);
|
||||
Mockito.when(plan.relRoot()).thenReturn(relRoot);
|
||||
Mockito.when(plan.sqlNode()).thenReturn(
|
||||
SqlParser.create("SELECT 1").parseQuery());
|
||||
Mockito.when(engine.compile(Mockito.any())).thenReturn(plan);
|
||||
FederationResultCursor cursor = Mockito.mock(
|
||||
FederationResultCursor.class);
|
||||
Mockito.when(cursor.columns()).thenReturn(List.of());
|
||||
Mockito.when(cursor.next()).thenReturn(false);
|
||||
Mockito.when(engine.execute(Mockito.eq(plan), Mockito.any()))
|
||||
.thenReturn(cursor);
|
||||
|
||||
ObjectProvider<StringRedisTemplate> redisProvider = Mockito.mock(
|
||||
ObjectProvider.class);
|
||||
DatacenterFederationQueryCancellationService cancellationService =
|
||||
new DatacenterFederationQueryCancellationService(
|
||||
runtime, redisProvider);
|
||||
DatacenterFederationQueryService service =
|
||||
new DatacenterFederationQueryService(
|
||||
runtime,
|
||||
Mockito.mock(DatacenterQueryAuditService.class),
|
||||
cancellationService);
|
||||
String requestedQueryId = UUID.randomUUID().toString();
|
||||
|
||||
Assert.assertEquals(0L, service.consume(
|
||||
source, "SELECT 1", List.of(), 10, 100, 1_024,
|
||||
null, "TEST", "stream-1", requestedQueryId,
|
||||
ignored -> { }));
|
||||
Assert.assertEquals(0L, service.consume(
|
||||
source, "SELECT 1", List.of(), 10, 100, 1_024,
|
||||
null, "TEST", "stream-2", requestedQueryId,
|
||||
ignored -> { }));
|
||||
|
||||
ArgumentCaptor<SqlExecutionContext> contexts =
|
||||
ArgumentCaptor.forClass(SqlExecutionContext.class);
|
||||
Mockito.verify(engine, Mockito.times(2)).execute(
|
||||
Mockito.eq(plan), contexts.capture());
|
||||
Assert.assertTrue(contexts.getAllValues().stream().allMatch(
|
||||
context -> requestedQueryId.equals(context.queryId().value())));
|
||||
ArgumentCaptor<SqlCompileRequest> compileRequests =
|
||||
ArgumentCaptor.forClass(SqlCompileRequest.class);
|
||||
Mockito.verify(engine, Mockito.times(2)).compile(compileRequests.capture());
|
||||
Assert.assertTrue(compileRequests.getAllValues().stream().allMatch(request ->
|
||||
request.queryScope().revision() == source.getScopeRevision()
|
||||
&& request.minimumRevision() == source.getDefinitionRevision()
|
||||
&& sourceId.equals(request.sourceId())));
|
||||
Mockito.verify(cursor, Mockito.times(2)).close();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,286 @@
|
||||
package tech.easyflow.datacenter.federation;
|
||||
|
||||
import com.easyagents.federation.sql.api.FederationSqlException;
|
||||
import com.easyagents.federation.sql.compile.SqlCompileRequest;
|
||||
import com.easyagents.federation.sql.compile.SqlPolicyContext;
|
||||
import com.easyagents.federation.sql.federation.FederationExecutionPolicy;
|
||||
import com.easyagents.federation.sql.federation.FederationQueryScopeDefinition;
|
||||
import com.easyagents.federation.sql.federation.FederationSourceBindingDefinition;
|
||||
import com.easyagents.federation.sql.source.SourceId;
|
||||
import java.math.BigInteger;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import org.apache.calcite.plan.RelOptTable;
|
||||
import org.apache.calcite.rel.RelNode;
|
||||
import org.apache.calcite.rel.RelRoot;
|
||||
import org.apache.calcite.rel.RelVisitor;
|
||||
import org.apache.calcite.rel.core.TableScan;
|
||||
import org.apache.calcite.rel.type.RelDataType;
|
||||
import org.apache.calcite.sql.SqlKind;
|
||||
import org.apache.calcite.sql.SqlNode;
|
||||
import org.apache.calcite.sql.parser.SqlParser;
|
||||
import org.junit.Test;
|
||||
import org.mockito.ArgumentMatchers;
|
||||
import org.mockito.Mockito;
|
||||
import tech.easyflow.datacenter.entity.DatacenterTable;
|
||||
import tech.easyflow.datacenter.entity.DatacenterTableField;
|
||||
import tech.easyflow.datacenter.mapper.DatacenterCatalogMapper;
|
||||
import tech.easyflow.datacenter.mapper.DatacenterSourceMapper;
|
||||
import tech.easyflow.datacenter.mapper.DatacenterTableFieldMapper;
|
||||
import tech.easyflow.datacenter.mapper.DatacenterTableMapper;
|
||||
import tech.easyflow.datacenter.meta.entity.DatacenterSource;
|
||||
import tech.easyflow.datacenter.meta.enums.DatacenterMetadataStatus;
|
||||
import tech.easyflow.datacenter.meta.enums.DatacenterSensitivityLevel;
|
||||
import tech.easyflow.datacenter.meta.enums.DatacenterSourceStatus;
|
||||
|
||||
/**
|
||||
* {@link DatacenterFederationSqlPolicy} 字段白名单和别名回归测试。
|
||||
*/
|
||||
public class DatacenterFederationSqlPolicyTest {
|
||||
|
||||
/**
|
||||
* 验证表别名、输出别名和 ORDER BY 别名不会被误判为物理字段。
|
||||
*
|
||||
* @throws Exception SQL 解析失败
|
||||
*/
|
||||
@Test
|
||||
public void validateShouldAllowTableAndOutputAliases() throws Exception {
|
||||
DatacenterFederationSqlPolicy policy = policy();
|
||||
|
||||
policy.validate(context(
|
||||
"SELECT o.id AS order_id FROM orders AS o ORDER BY order_id"));
|
||||
}
|
||||
|
||||
/**
|
||||
* 验证不可查询字段即使通过表别名限定也会被拒绝。
|
||||
*
|
||||
* @throws Exception SQL 解析失败
|
||||
*/
|
||||
@Test(expected = FederationSqlException.class)
|
||||
public void validateShouldRejectHiddenQualifiedColumn() throws Exception {
|
||||
policy().validate(context("SELECT o.secret FROM orders AS o"));
|
||||
}
|
||||
|
||||
/**
|
||||
* 验证与表同名的字段不会因名称碰撞绕过字段白名单。
|
||||
*
|
||||
* @throws Exception SQL 解析失败
|
||||
*/
|
||||
@Test(expected = FederationSqlException.class)
|
||||
public void validateShouldRejectHiddenColumnNamedLikeTable() throws Exception {
|
||||
policy().validate(context("SELECT orders AS display_name FROM orders"));
|
||||
}
|
||||
|
||||
/**
|
||||
* 验证输出别名与隐藏物理字段同名时不会绕过字段策略。
|
||||
*
|
||||
* @throws Exception SQL 解析失败
|
||||
*/
|
||||
@Test(expected = FederationSqlException.class)
|
||||
public void validateShouldRejectHiddenColumnWithSameOutputAlias()
|
||||
throws Exception {
|
||||
policy().validate(context("SELECT secret AS secret FROM orders"));
|
||||
}
|
||||
|
||||
/**
|
||||
* 验证内层子查询的表别名不会污染外层同名物理表限定符。
|
||||
*
|
||||
* @throws Exception SQL 解析失败
|
||||
*/
|
||||
@Test(expected = FederationSqlException.class)
|
||||
public void validateShouldKeepNestedAliasScopesIndependent()
|
||||
throws Exception {
|
||||
policy().validate(context(
|
||||
"SELECT orders.secret FROM orders "
|
||||
+ "WHERE EXISTS (SELECT 1 FROM public_orders AS orders)",
|
||||
"orders", "public_orders"));
|
||||
}
|
||||
|
||||
/**
|
||||
* 验证派生表的输出别名由内层字段策略校验后可在外层引用。
|
||||
*
|
||||
* @throws Exception SQL 解析失败
|
||||
*/
|
||||
@Test
|
||||
public void validateShouldAllowDerivedTableOutputAlias()
|
||||
throws Exception {
|
||||
policy().validate(context(
|
||||
"SELECT d.order_id FROM "
|
||||
+ "(SELECT id AS order_id FROM orders) AS d"));
|
||||
}
|
||||
|
||||
/**
|
||||
* 验证 CTE 输出别名由内层字段策略校验后可在外层引用。
|
||||
*
|
||||
* @throws Exception SQL 解析失败
|
||||
*/
|
||||
@Test
|
||||
public void validateShouldAllowCommonTableExpressionOutputAlias()
|
||||
throws Exception {
|
||||
policy().validate(context(
|
||||
"WITH x AS (SELECT id AS order_id FROM orders) "
|
||||
+ "SELECT order_id FROM x"));
|
||||
}
|
||||
|
||||
/**
|
||||
* 验证派生输出仍不能隐藏内层敏感字段引用。
|
||||
*
|
||||
* @throws Exception SQL 解析失败
|
||||
*/
|
||||
@Test(expected = FederationSqlException.class)
|
||||
public void validateShouldRejectHiddenColumnInsideDerivedTable()
|
||||
throws Exception {
|
||||
policy().validate(context(
|
||||
"SELECT d.secret_alias FROM "
|
||||
+ "(SELECT secret AS secret_alias FROM orders) AS d"));
|
||||
}
|
||||
|
||||
/**
|
||||
* 验证数据库权威范围版本变化后,旧节点缓存不能继续放行查询。
|
||||
*
|
||||
* @throws Exception SQL 解析失败
|
||||
*/
|
||||
@Test(expected = FederationSqlException.class)
|
||||
public void validateShouldRejectStaleScopeRevision()
|
||||
throws Exception {
|
||||
policy(2L).validate(context("SELECT id FROM orders"));
|
||||
}
|
||||
|
||||
/**
|
||||
* 验证 XL01 尚未建立逐源授权模型时不会误开放底座联邦能力。
|
||||
*
|
||||
* @throws Exception SQL 解析失败
|
||||
*/
|
||||
@Test(expected = FederationSqlException.class)
|
||||
public void validateShouldRejectMultiSourceScopeUntilProductAuthorizationExists()
|
||||
throws Exception {
|
||||
SqlPolicyContext single = context("SELECT id FROM orders");
|
||||
SourceId second = new SourceId("tenant-1-source-1002");
|
||||
FederationQueryScopeDefinition virtualScope =
|
||||
FederationQueryScopeDefinition.virtual(
|
||||
"not-yet-authorized",
|
||||
1,
|
||||
Map.of(
|
||||
"A", FederationSourceBindingDefinition.of(
|
||||
single.request().sourceId(), 1),
|
||||
"B", FederationSourceBindingDefinition.of(second, 1)),
|
||||
"A",
|
||||
FederationExecutionPolicy.basic());
|
||||
SqlCompileRequest request = new SqlCompileRequest(
|
||||
single.request().sql(), virtualScope, List.of(), "scope-1");
|
||||
policy().validate(new SqlPolicyContext(
|
||||
request,
|
||||
single.validatedSql(),
|
||||
single.relRoot(),
|
||||
Set.of(second)));
|
||||
}
|
||||
|
||||
private DatacenterFederationSqlPolicy policy() {
|
||||
return policy(1L);
|
||||
}
|
||||
|
||||
private DatacenterFederationSqlPolicy policy(long currentScopeRevision) {
|
||||
BigInteger tableId = BigInteger.valueOf(2001L);
|
||||
BigInteger publicTableId = BigInteger.valueOf(2002L);
|
||||
DatacenterTable table = table(tableId, "orders");
|
||||
DatacenterTable publicTable = table(publicTableId, "public_orders");
|
||||
|
||||
DatacenterTableMapper tableMapper = Mockito.mock(DatacenterTableMapper.class);
|
||||
DatacenterTableFieldMapper fieldMapper = Mockito.mock(
|
||||
DatacenterTableFieldMapper.class);
|
||||
DatacenterCatalogMapper catalogMapper = Mockito.mock(DatacenterCatalogMapper.class);
|
||||
DatacenterSourceMapper sourceMapper = Mockito.mock(
|
||||
DatacenterSourceMapper.class);
|
||||
DatacenterSource source = new DatacenterSource();
|
||||
source.setId(BigInteger.valueOf(1001L));
|
||||
source.setStatus(DatacenterSourceStatus.READY.code());
|
||||
source.setScopeRevision(currentScopeRevision);
|
||||
Mockito.when(sourceMapper.selectOneById(BigInteger.valueOf(1001L)))
|
||||
.thenReturn(source);
|
||||
Mockito.when(tableMapper.selectListByQuery(ArgumentMatchers.any()))
|
||||
.thenReturn(List.of(table, publicTable));
|
||||
Mockito.when(fieldMapper.selectListByQuery(ArgumentMatchers.any()))
|
||||
.thenReturn(List.of(
|
||||
field(tableId, "id", true),
|
||||
field(tableId, "secret", false),
|
||||
field(tableId, "orders", false),
|
||||
field(publicTableId, "id", true),
|
||||
field(publicTableId, "secret", true)));
|
||||
return new DatacenterFederationSqlPolicy(
|
||||
tableMapper, fieldMapper, catalogMapper, sourceMapper);
|
||||
}
|
||||
|
||||
private SqlPolicyContext context(String sql, String... tableNames)
|
||||
throws Exception {
|
||||
SqlNode parsed = SqlParser.create(sql).parseQuery();
|
||||
String[] effectiveTableNames = tableNames.length == 0
|
||||
? new String[]{"orders"} : tableNames;
|
||||
List<TableScan> scans = new ArrayList<>();
|
||||
for (String tableName : effectiveTableNames) {
|
||||
scans.add(scan(tableName));
|
||||
}
|
||||
RelDataType rowType = Mockito.mock(RelDataType.class);
|
||||
Mockito.when(rowType.getFieldNames())
|
||||
.thenReturn(List.of("id", "secret", "orders"));
|
||||
RelNode root = scans.size() == 1
|
||||
? scans.get(0) : Mockito.mock(RelNode.class);
|
||||
if (scans.size() > 1) {
|
||||
Mockito.when(root.getRowType()).thenReturn(rowType);
|
||||
Mockito.doAnswer(invocation -> {
|
||||
RelVisitor visitor = invocation.getArgument(0);
|
||||
for (int index = 0; index < scans.size(); index++) {
|
||||
visitor.visit(scans.get(index), index, root);
|
||||
}
|
||||
return null;
|
||||
}).when(root).childrenAccept(ArgumentMatchers.any(RelVisitor.class));
|
||||
}
|
||||
RelRoot relRoot = RelRoot.of(root, rowType, SqlKind.SELECT);
|
||||
SourceId sourceId = new SourceId("tenant-1-source-1001");
|
||||
SqlCompileRequest request = new SqlCompileRequest(
|
||||
sql, sourceId, 1L, List.of(), "scope-1");
|
||||
return new SqlPolicyContext(request, parsed, relRoot, Set.of(sourceId));
|
||||
}
|
||||
|
||||
private TableScan scan(String tableName) {
|
||||
RelDataType rowType = Mockito.mock(RelDataType.class);
|
||||
Mockito.when(rowType.getFieldNames()).thenReturn(
|
||||
"orders".equals(tableName)
|
||||
? List.of("id", "secret", "orders")
|
||||
: List.of("id", "secret"));
|
||||
TableScan scan = Mockito.mock(TableScan.class);
|
||||
RelOptTable relOptTable = Mockito.mock(RelOptTable.class);
|
||||
Mockito.when(relOptTable.getQualifiedName())
|
||||
.thenReturn(List.of(tableName));
|
||||
Mockito.when(scan.getTable()).thenReturn(relOptTable);
|
||||
Mockito.when(scan.getRowType()).thenReturn(rowType);
|
||||
return scan;
|
||||
}
|
||||
|
||||
private DatacenterTable table(BigInteger tableId, String tableName) {
|
||||
DatacenterTable table = new DatacenterTable();
|
||||
table.setId(tableId);
|
||||
table.setTableName(tableName);
|
||||
table.setActualTable(tableName);
|
||||
table.setQueryable(1);
|
||||
table.setMetadataStatus(DatacenterMetadataStatus.ACTIVE.name());
|
||||
return table;
|
||||
}
|
||||
|
||||
private DatacenterTableField field(
|
||||
BigInteger tableId,
|
||||
String name,
|
||||
boolean queryable) {
|
||||
DatacenterTableField field = new DatacenterTableField();
|
||||
field.setTableId(tableId);
|
||||
field.setFieldName(name);
|
||||
field.setSourceColumnName(name);
|
||||
field.setQueryable(queryable ? 1 : 0);
|
||||
field.setMetadataStatus(DatacenterMetadataStatus.ACTIVE.name());
|
||||
field.setSensitivityLevel(queryable
|
||||
? DatacenterSensitivityLevel.PUBLIC.name()
|
||||
: DatacenterSensitivityLevel.RESTRICTED.name());
|
||||
return field;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,53 @@
|
||||
package tech.easyflow.datacenter.federation;
|
||||
|
||||
import org.junit.Assert;
|
||||
import org.junit.Test;
|
||||
|
||||
/**
|
||||
* {@link DatacenterMetadataIdentity} 稳定编码回归测试。
|
||||
*/
|
||||
public class DatacenterMetadataIdentityTest {
|
||||
|
||||
/**
|
||||
* 验证 null、用户字面量和字段边界不会产生相同摘要。
|
||||
*/
|
||||
@Test
|
||||
public void namespaceKeyShouldPreservePresenceAndBoundaries() {
|
||||
String nullCatalog = DatacenterMetadataIdentity.namespaceKey(null, "public");
|
||||
String literalCatalog = DatacenterMetadataIdentity.namespaceKey("N", "public");
|
||||
String firstBoundary = DatacenterMetadataIdentity.namespaceKey("ab", "c");
|
||||
String secondBoundary = DatacenterMetadataIdentity.namespaceKey("a", "bc");
|
||||
|
||||
Assert.assertNotEquals(nullCatalog, literalCatalog);
|
||||
Assert.assertNotEquals(firstBoundary, secondBoundary);
|
||||
Assert.assertEquals(64, nullCatalog.length());
|
||||
}
|
||||
|
||||
/**
|
||||
* 验证相同物理身份在重复计算时保持稳定。
|
||||
*/
|
||||
@Test
|
||||
public void tableIdentityShouldBeDeterministic() {
|
||||
String first = DatacenterMetadataIdentity.tableIdentity(
|
||||
"source-1", "namespace-1", "orders", "TABLE");
|
||||
String second = DatacenterMetadataIdentity.tableIdentity(
|
||||
"source-1", "namespace-1", "orders", "TABLE");
|
||||
|
||||
Assert.assertEquals(first, second);
|
||||
}
|
||||
|
||||
/**
|
||||
* 验证旧表摘要与 V58 迁移中的 MySQL SHA2 输入完全一致。
|
||||
*/
|
||||
@Test
|
||||
public void legacyV58DigestsShouldMatchMigrationAlgorithm() {
|
||||
Assert.assertEquals(
|
||||
"1b0b9f334b8ce200ce3c5480930d9bb537af2b630e17f7f1bc47e9dbe2a72274",
|
||||
DatacenterMetadataIdentity.legacyV58TableIdentity(
|
||||
12L, 34L, "orders", "TABLE"));
|
||||
Assert.assertEquals(
|
||||
"983026e4b253d124c629e19de6b97c30a0e3146ba8c45cdee2540df80a466bb3",
|
||||
DatacenterMetadataIdentity.legacyV58TableFingerprint(
|
||||
"orders", "TABLE"));
|
||||
}
|
||||
}
|
||||
@@ -8,6 +8,7 @@ import tech.easyflow.datacenter.entity.DatacenterTable;
|
||||
import tech.easyflow.datacenter.meta.model.DatacenterCatalogMeta;
|
||||
|
||||
import java.math.BigInteger;
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* 数据中枢雪花 ID 的 HTTP JSON 序列化测试。
|
||||
@@ -47,6 +48,24 @@ public class DatacenterIdJsonSerializationTest {
|
||||
assertTextualId(table, "catalogId");
|
||||
}
|
||||
|
||||
/**
|
||||
* 验证实体被旧 Schema API 间接序列化时也不会暴露凭据密文和连接扩展配置。
|
||||
*
|
||||
* @throws Exception JSON 序列化失败时抛出
|
||||
*/
|
||||
@Test
|
||||
public void shouldHideSourceCredentialMaterial() throws Exception {
|
||||
DatacenterSource source = new DatacenterSource();
|
||||
source.setCredentialCipher("v1:encrypted-secret");
|
||||
source.setConfigJson(Map.of("clientSecret", "plain-secret"));
|
||||
|
||||
JsonNode node = objectMapper.readTree(objectMapper.writeValueAsString(source));
|
||||
|
||||
Assert.assertFalse(node.has("credentialCipher"));
|
||||
Assert.assertFalse(node.has("configJson"));
|
||||
Assert.assertFalse(node.toString().contains("plain-secret"));
|
||||
}
|
||||
|
||||
/**
|
||||
* 验证给定属性被输出为精确的文本 ID。
|
||||
*
|
||||
|
||||
@@ -0,0 +1,361 @@
|
||||
package tech.easyflow.datacenter.meta.service.impl;
|
||||
|
||||
import java.lang.reflect.Field;
|
||||
import java.lang.reflect.Method;
|
||||
import java.math.BigInteger;
|
||||
import java.sql.Types;
|
||||
import java.util.List;
|
||||
import org.junit.Assert;
|
||||
import org.junit.Test;
|
||||
import org.mockito.ArgumentMatchers;
|
||||
import org.mockito.ArgumentCaptor;
|
||||
import org.mockito.Mockito;
|
||||
import tech.easyflow.common.entity.LoginAccount;
|
||||
import tech.easyflow.common.web.exceptions.BusinessException;
|
||||
import tech.easyflow.datacenter.entity.DatacenterTable;
|
||||
import tech.easyflow.datacenter.entity.DatacenterTableField;
|
||||
import tech.easyflow.datacenter.federation.DatacenterFederationChangeNotifier;
|
||||
import tech.easyflow.datacenter.federation.DatacenterMetadataIdentity;
|
||||
import tech.easyflow.datacenter.meta.entity.DatacenterCatalog;
|
||||
import tech.easyflow.datacenter.mapper.DatacenterSourceMapper;
|
||||
import tech.easyflow.datacenter.mapper.DatacenterTableFieldMapper;
|
||||
import tech.easyflow.datacenter.mapper.DatacenterTableMapper;
|
||||
import tech.easyflow.datacenter.meta.entity.DatacenterSource;
|
||||
import tech.easyflow.datacenter.meta.model.DatacenterFieldDescriptionUpdate;
|
||||
import tech.easyflow.datacenter.meta.enums.DatacenterSourceStatus;
|
||||
import tech.easyflow.datacenter.meta.model.DatacenterTableDetailMeta;
|
||||
|
||||
/**
|
||||
* {@link DatacenterDatasetRegistryServiceImpl} 查询范围版本回归测试。
|
||||
*/
|
||||
public class DatacenterDatasetRegistryServiceImplTest {
|
||||
|
||||
/**
|
||||
* 验证 V58 未回填的新 JDBC 字段不会让未变化表误报,同时仍检测旧结构变化。
|
||||
*
|
||||
* @throws Exception 反射调用失败
|
||||
*/
|
||||
@Test
|
||||
public void legacyStructureShouldIgnoreUnknownNewMetadataOnly()
|
||||
throws Exception {
|
||||
DatacenterDatasetRegistryServiceImpl service =
|
||||
new DatacenterDatasetRegistryServiceImpl();
|
||||
DatacenterTableField persisted = legacyField("id", 19);
|
||||
DatacenterTableField discovered = legacyField("id", 19);
|
||||
discovered.setJdbcTypeCode(Types.BIGINT);
|
||||
discovered.setOrdinalPosition(1);
|
||||
|
||||
Assert.assertTrue(invokeLegacyStructureMatches(
|
||||
service, List.of(persisted), List.of(discovered)));
|
||||
|
||||
DatacenterTableField changed = legacyField("id", 20);
|
||||
changed.setJdbcTypeCode(Types.BIGINT);
|
||||
changed.setOrdinalPosition(1);
|
||||
Assert.assertFalse(invokeLegacyStructureMatches(
|
||||
service, List.of(persisted), List.of(changed)));
|
||||
}
|
||||
|
||||
/**
|
||||
* 验证字段治理变更会以 CAS 推进外部数据源范围版本并发布通知。
|
||||
*
|
||||
* @throws Exception 测试依赖注入失败
|
||||
*/
|
||||
@Test
|
||||
public void saveDescriptionsShouldAdvanceScopeRevisionForPolicyChange()
|
||||
throws Exception {
|
||||
BigInteger sourceId = BigInteger.valueOf(1001L);
|
||||
BigInteger tableId = BigInteger.valueOf(2001L);
|
||||
DatacenterSource source = new DatacenterSource();
|
||||
source.setId(sourceId);
|
||||
source.setTenantId(BigInteger.ONE);
|
||||
source.setSourceType("MYSQL");
|
||||
source.setStatus(DatacenterSourceStatus.READY.code());
|
||||
source.setDefinitionRevision(1L);
|
||||
source.setScopeRevision(1L);
|
||||
DatacenterSource refreshedSource = new DatacenterSource();
|
||||
refreshedSource.setId(sourceId);
|
||||
refreshedSource.setTenantId(BigInteger.ONE);
|
||||
refreshedSource.setSourceType("MYSQL");
|
||||
refreshedSource.setStatus(DatacenterSourceStatus.READY.code());
|
||||
refreshedSource.setDefinitionRevision(1L);
|
||||
refreshedSource.setScopeRevision(2L);
|
||||
|
||||
DatacenterTable table = new DatacenterTable();
|
||||
table.setId(tableId);
|
||||
table.setSourceId(sourceId);
|
||||
table.setTenantId(BigInteger.ONE);
|
||||
DatacenterTableField field = new DatacenterTableField();
|
||||
field.setId(BigInteger.valueOf(3001L));
|
||||
field.setTableId(tableId);
|
||||
field.setQueryable(1);
|
||||
field.setSensitivityLevel("PUBLIC");
|
||||
|
||||
DatacenterTableMapper tableMapper = Mockito.mock(DatacenterTableMapper.class);
|
||||
DatacenterTableFieldMapper fieldMapper = Mockito.mock(
|
||||
DatacenterTableFieldMapper.class);
|
||||
DatacenterSourceMapper sourceMapper = Mockito.mock(DatacenterSourceMapper.class);
|
||||
DatacenterFederationChangeNotifier notifier = Mockito.mock(
|
||||
DatacenterFederationChangeNotifier.class);
|
||||
Mockito.when(tableMapper.selectOneById(tableId)).thenReturn(table);
|
||||
Mockito.when(fieldMapper.selectListByQuery(ArgumentMatchers.any()))
|
||||
.thenReturn(List.of(field));
|
||||
Mockito.when(sourceMapper.selectOneById(sourceId))
|
||||
.thenReturn(source, refreshedSource);
|
||||
Mockito.when(sourceMapper.updateByQuery(
|
||||
ArgumentMatchers.any(), ArgumentMatchers.any()))
|
||||
.thenReturn(1);
|
||||
|
||||
DatacenterDatasetRegistryServiceImpl service =
|
||||
new DatacenterDatasetRegistryServiceImpl();
|
||||
setField(service, "tableMapper", tableMapper);
|
||||
setField(service, "tableFieldMapper", fieldMapper);
|
||||
setField(service, "sourceMapper", sourceMapper);
|
||||
setField(service, "federationChangeNotifier", notifier);
|
||||
|
||||
DatacenterFieldDescriptionUpdate update =
|
||||
new DatacenterFieldDescriptionUpdate();
|
||||
update.setFieldId(field.getId());
|
||||
update.setFieldDesc("内部编号");
|
||||
update.setQueryable(0);
|
||||
update.setSensitivityLevel("INTERNAL");
|
||||
LoginAccount account = new LoginAccount();
|
||||
account.setId(BigInteger.TEN);
|
||||
account.setTenantId(BigInteger.ONE);
|
||||
|
||||
service.saveDescriptions(tableId, "订单", List.of(update), account);
|
||||
|
||||
Assert.assertEquals(Integer.valueOf(0), field.getQueryable());
|
||||
Assert.assertEquals("INTERNAL", field.getSensitivityLevel());
|
||||
ArgumentCaptor<DatacenterSource> patchCaptor =
|
||||
ArgumentCaptor.forClass(DatacenterSource.class);
|
||||
Mockito.verify(sourceMapper).updateByQuery(
|
||||
patchCaptor.capture(), ArgumentMatchers.any());
|
||||
DatacenterSource patch = patchCaptor.getValue();
|
||||
Assert.assertEquals(sourceId, patch.getId());
|
||||
Assert.assertEquals(Long.valueOf(2L), patch.getScopeRevision());
|
||||
Assert.assertNull(patch.getStatus());
|
||||
Assert.assertNull(patch.getDefinitionRevision());
|
||||
Mockito.verify(notifier).publishAfterCommit(refreshedSource);
|
||||
}
|
||||
|
||||
/**
|
||||
* 验证刷新发现未纳管新字段时只标记结构变化并收缩权限,不自动插入字段。
|
||||
*
|
||||
* @throws Exception 测试依赖注入失败
|
||||
*/
|
||||
@Test
|
||||
public void refreshShouldFailClosedWithoutRegisteringNewField()
|
||||
throws Exception {
|
||||
BigInteger sourceId = BigInteger.valueOf(1101L);
|
||||
BigInteger catalogId = BigInteger.valueOf(2101L);
|
||||
BigInteger tableId = BigInteger.valueOf(3101L);
|
||||
DatacenterSource source = new DatacenterSource();
|
||||
source.setId(sourceId);
|
||||
source.setTenantId(BigInteger.ONE);
|
||||
source.setSourceType("MYSQL");
|
||||
DatacenterCatalog catalog = new DatacenterCatalog();
|
||||
catalog.setId(catalogId);
|
||||
catalog.setSourceId(sourceId);
|
||||
catalog.setNamespaceKey(DatacenterMetadataIdentity.namespaceKey("easyflow", null));
|
||||
|
||||
DatacenterTableField idField = modernField("id", Types.BIGINT);
|
||||
idField.setId(BigInteger.valueOf(4101L));
|
||||
idField.setTableId(tableId);
|
||||
idField.setQueryable(1);
|
||||
DatacenterTable table = new DatacenterTable();
|
||||
table.setId(tableId);
|
||||
table.setSourceId(sourceId);
|
||||
table.setCatalogId(catalogId);
|
||||
table.setTableName("orders");
|
||||
table.setActualTable("orders");
|
||||
table.setTableKind("EXTERNAL_TABLE");
|
||||
table.setMetadataStatus("ACTIVE");
|
||||
table.setQueryable(1);
|
||||
table.setPhysicalIdentityKey(DatacenterMetadataIdentity.tableIdentity(
|
||||
sourceId.toString(), catalog.getNamespaceKey(), "orders", "EXTERNAL_TABLE"));
|
||||
table.setMetadataFingerprint(DatacenterMetadataIdentity.tableFingerprint(
|
||||
"orders", "EXTERNAL_TABLE", List.of(idField)));
|
||||
|
||||
DatacenterTableField newField = modernField("secret", Types.VARCHAR);
|
||||
DatacenterTable discovered = new DatacenterTable();
|
||||
discovered.setTableName("orders");
|
||||
discovered.setActualTable("orders");
|
||||
discovered.setTableKind("EXTERNAL_TABLE");
|
||||
DatacenterTableDetailMeta detail = new DatacenterTableDetailMeta();
|
||||
detail.setTable(discovered);
|
||||
detail.setFields(List.of(modernField("id", Types.BIGINT), newField));
|
||||
|
||||
DatacenterTableMapper tableMapper = Mockito.mock(DatacenterTableMapper.class);
|
||||
DatacenterTableFieldMapper fieldMapper = Mockito.mock(
|
||||
DatacenterTableFieldMapper.class);
|
||||
Mockito.when(tableMapper.selectOneById(tableId)).thenReturn(table);
|
||||
Mockito.when(fieldMapper.selectListByQuery(ArgumentMatchers.any()))
|
||||
.thenReturn(List.of(idField));
|
||||
DatacenterDatasetRegistryServiceImpl service =
|
||||
new DatacenterDatasetRegistryServiceImpl();
|
||||
setField(service, "tableMapper", tableMapper);
|
||||
setField(service, "tableFieldMapper", fieldMapper);
|
||||
|
||||
boolean changed = service.refreshManagedTable(
|
||||
source, catalog, tableId, detail, account());
|
||||
|
||||
Assert.assertTrue(changed);
|
||||
Assert.assertEquals("CHANGED", table.getMetadataStatus());
|
||||
Assert.assertEquals(Integer.valueOf(0), table.getQueryable());
|
||||
Assert.assertEquals("CHANGED", idField.getMetadataStatus());
|
||||
Assert.assertEquals(Integer.valueOf(0), idField.getQueryable());
|
||||
Mockito.verify(fieldMapper, Mockito.never())
|
||||
.insert(ArgumentMatchers.any(DatacenterTableField.class));
|
||||
}
|
||||
|
||||
/**
|
||||
* 显式重配置允许将稳定表 ID 绑定到新物理命名空间,并重建字段治理基线。
|
||||
* 常规刷新对相同变化仍保持 fail-closed。
|
||||
*
|
||||
* @throws Exception 测试依赖注入失败
|
||||
*/
|
||||
@Test
|
||||
public void explicitRebindShouldPreserveTableIdAndReplaceFields()
|
||||
throws Exception {
|
||||
BigInteger sourceId = BigInteger.valueOf(1201L);
|
||||
BigInteger catalogId = BigInteger.valueOf(2201L);
|
||||
BigInteger tableId = BigInteger.valueOf(3201L);
|
||||
DatacenterSource source = new DatacenterSource();
|
||||
source.setId(sourceId);
|
||||
source.setTenantId(BigInteger.ONE);
|
||||
source.setSourceType("MYSQL");
|
||||
DatacenterCatalog catalog = new DatacenterCatalog();
|
||||
catalog.setId(catalogId);
|
||||
catalog.setSourceId(sourceId);
|
||||
catalog.setNamespaceKey(DatacenterMetadataIdentity.namespaceKey("new_db", null));
|
||||
|
||||
DatacenterTableField persistedField = modernField("legacy_id", Types.BIGINT);
|
||||
persistedField.setId(BigInteger.valueOf(4201L));
|
||||
persistedField.setTableId(tableId);
|
||||
DatacenterTable existing = new DatacenterTable();
|
||||
existing.setId(tableId);
|
||||
existing.setSourceId(sourceId);
|
||||
existing.setCatalogId(catalogId);
|
||||
existing.setTableName("orders");
|
||||
existing.setActualTable("orders");
|
||||
existing.setTableKind("EXTERNAL_TABLE");
|
||||
existing.setQueryable(0);
|
||||
existing.setPhysicalIdentityKey(DatacenterMetadataIdentity.tableIdentity(
|
||||
sourceId.toString(),
|
||||
DatacenterMetadataIdentity.namespaceKey("old_db", null),
|
||||
"orders",
|
||||
"EXTERNAL_TABLE"));
|
||||
existing.setMetadataFingerprint(DatacenterMetadataIdentity.tableFingerprint(
|
||||
"orders", "EXTERNAL_TABLE", List.of(persistedField)));
|
||||
|
||||
DatacenterTableField discoveredField = modernField("id", Types.BIGINT);
|
||||
DatacenterTable discovered = new DatacenterTable();
|
||||
discovered.setTableName("orders");
|
||||
discovered.setActualTable("orders");
|
||||
discovered.setTableKind("EXTERNAL_TABLE");
|
||||
discovered.setQueryable(1);
|
||||
DatacenterTableDetailMeta detail = new DatacenterTableDetailMeta();
|
||||
detail.setTable(discovered);
|
||||
detail.setFields(List.of(discoveredField));
|
||||
|
||||
DatacenterTableMapper tableMapper = Mockito.mock(DatacenterTableMapper.class);
|
||||
DatacenterTableFieldMapper fieldMapper = Mockito.mock(
|
||||
DatacenterTableFieldMapper.class);
|
||||
Mockito.when(tableMapper.selectOneByQuery(ArgumentMatchers.any()))
|
||||
.thenReturn(existing);
|
||||
Mockito.when(fieldMapper.selectListByQuery(ArgumentMatchers.any()))
|
||||
.thenReturn(
|
||||
List.of(persistedField),
|
||||
List.of(persistedField),
|
||||
List.of(discoveredField));
|
||||
DatacenterDatasetRegistryServiceImpl service =
|
||||
new DatacenterDatasetRegistryServiceImpl();
|
||||
setField(service, "tableMapper", tableMapper);
|
||||
setField(service, "tableFieldMapper", fieldMapper);
|
||||
|
||||
try {
|
||||
service.registerTable(source, catalog, detail, account());
|
||||
Assert.fail("ordinary refresh must reject changed physical identity");
|
||||
} catch (BusinessException expected) {
|
||||
Assert.assertTrue(expected.getMessage().contains("物理表身份已变化"));
|
||||
}
|
||||
|
||||
DatacenterTable rebound = service.rebindTable(
|
||||
source, catalog, detail, account());
|
||||
|
||||
Assert.assertEquals(tableId, rebound.getId());
|
||||
Assert.assertEquals(Integer.valueOf(1), rebound.getQueryable());
|
||||
Assert.assertEquals(DatacenterMetadataIdentity.tableIdentity(
|
||||
sourceId.toString(), catalog.getNamespaceKey(),
|
||||
"orders", "EXTERNAL_TABLE"), rebound.getPhysicalIdentityKey());
|
||||
Mockito.verify(fieldMapper).deleteByQuery(ArgumentMatchers.any());
|
||||
Mockito.verify(fieldMapper).insert(discoveredField);
|
||||
Mockito.verify(tableMapper).update(existing);
|
||||
}
|
||||
|
||||
/**
|
||||
* 注入被测服务依赖。
|
||||
*
|
||||
* @param target 被测对象
|
||||
* @param name 字段名
|
||||
* @param value 依赖值
|
||||
* @throws ReflectiveOperationException 反射失败
|
||||
*/
|
||||
private void setField(Object target, String name, Object value)
|
||||
throws ReflectiveOperationException {
|
||||
Field field = target.getClass().getDeclaredField(name);
|
||||
field.setAccessible(true);
|
||||
field.set(target, value);
|
||||
}
|
||||
|
||||
private DatacenterTableField legacyField(String name, int precision) {
|
||||
DatacenterTableField field = new DatacenterTableField();
|
||||
field.setFieldName(name);
|
||||
field.setSourceColumnName(name);
|
||||
field.setFieldType(0);
|
||||
field.setJdbcType("BIGINT");
|
||||
field.setNativeTypeName("BIGINT");
|
||||
field.setPrecision(precision);
|
||||
field.setScale(0);
|
||||
field.setRequired(1);
|
||||
return field;
|
||||
}
|
||||
|
||||
private DatacenterTableField modernField(String name, int jdbcType) {
|
||||
DatacenterTableField field = new DatacenterTableField();
|
||||
field.setFieldName(name);
|
||||
field.setSourceColumnName(name);
|
||||
field.setJdbcType(jdbcType == Types.BIGINT ? "BIGINT" : "VARCHAR");
|
||||
field.setNativeTypeName(field.getJdbcType());
|
||||
field.setJdbcTypeCode(jdbcType);
|
||||
field.setPrecision(jdbcType == Types.BIGINT ? 19 : 255);
|
||||
field.setScale(0);
|
||||
field.setRequired(0);
|
||||
field.setQueryable(1);
|
||||
field.setSortable(1);
|
||||
field.setWritable(0);
|
||||
field.setMetadataStatus("ACTIVE");
|
||||
field.setMetadataFingerprint(DatacenterMetadataIdentity.fieldFingerprint(field));
|
||||
return field;
|
||||
}
|
||||
|
||||
private LoginAccount account() {
|
||||
LoginAccount account = new LoginAccount();
|
||||
account.setId(BigInteger.TEN);
|
||||
account.setTenantId(BigInteger.ONE);
|
||||
return account;
|
||||
}
|
||||
|
||||
@SuppressWarnings("unchecked")
|
||||
private boolean invokeLegacyStructureMatches(
|
||||
DatacenterDatasetRegistryServiceImpl service,
|
||||
List<DatacenterTableField> persisted,
|
||||
List<DatacenterTableField> discovered)
|
||||
throws ReflectiveOperationException {
|
||||
Method method = DatacenterDatasetRegistryServiceImpl.class
|
||||
.getDeclaredMethod(
|
||||
"legacyStructureMatches", List.class, List.class);
|
||||
method.setAccessible(true);
|
||||
return (boolean) method.invoke(service, persisted, discovered);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,125 @@
|
||||
package tech.easyflow.datacenter.meta.service.impl;
|
||||
|
||||
import java.math.BigInteger;
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
import org.junit.Assert;
|
||||
import org.junit.Test;
|
||||
import org.mockito.ArgumentMatchers;
|
||||
import org.mockito.InOrder;
|
||||
import org.mockito.Mockito;
|
||||
import org.springframework.transaction.PlatformTransactionManager;
|
||||
import org.springframework.transaction.TransactionDefinition;
|
||||
import org.springframework.transaction.TransactionStatus;
|
||||
import tech.easyflow.common.entity.LoginAccount;
|
||||
import tech.easyflow.datacenter.connector.DatacenterConnector;
|
||||
import tech.easyflow.datacenter.connector.DatacenterConnectorRegistry;
|
||||
import tech.easyflow.datacenter.entity.DatacenterTable;
|
||||
import tech.easyflow.datacenter.federation.DatacenterFederationChangeNotifier;
|
||||
import tech.easyflow.datacenter.mapper.DatacenterCatalogMapper;
|
||||
import tech.easyflow.datacenter.mapper.DatacenterSourceMapper;
|
||||
import tech.easyflow.datacenter.mapper.DatacenterTableMapper;
|
||||
import tech.easyflow.datacenter.meta.entity.DatacenterCatalog;
|
||||
import tech.easyflow.datacenter.meta.entity.DatacenterSource;
|
||||
import tech.easyflow.datacenter.meta.enums.DatacenterSourceStatus;
|
||||
import tech.easyflow.datacenter.meta.model.DatacenterManagedMetadataSnapshot;
|
||||
import tech.easyflow.datacenter.meta.model.DatacenterTableDetailMeta;
|
||||
import tech.easyflow.datacenter.meta.service.DatacenterDatasetRegistryService;
|
||||
|
||||
/**
|
||||
* {@link DatacenterSourceMetadataRefreshService} 事务与远程 I/O 边界测试。
|
||||
*/
|
||||
public class DatacenterSourceMetadataRefreshServiceTest {
|
||||
|
||||
/**
|
||||
* 远程 JDBC 元数据必须在开启平台数据库事务前完成。
|
||||
*/
|
||||
@Test
|
||||
public void refreshShouldInspectRemoteMetadataBeforeOpeningTransaction() {
|
||||
BigInteger sourceId = BigInteger.valueOf(1301L);
|
||||
DatacenterSource source = new DatacenterSource();
|
||||
source.setId(sourceId);
|
||||
source.setTenantId(BigInteger.ONE);
|
||||
source.setSourceType("MYSQL");
|
||||
source.setStatus(DatacenterSourceStatus.READY.code());
|
||||
source.setDefinitionRevision(3L);
|
||||
source.setScopeRevision(5L);
|
||||
DatacenterCatalog catalog = new DatacenterCatalog();
|
||||
catalog.setId(BigInteger.valueOf(2301L));
|
||||
catalog.setSourceId(sourceId);
|
||||
catalog.setCatalogName("orders_db");
|
||||
DatacenterTable table = new DatacenterTable();
|
||||
table.setId(BigInteger.valueOf(3301L));
|
||||
table.setSourceId(sourceId);
|
||||
table.setCatalogId(catalog.getId());
|
||||
table.setTableName("orders");
|
||||
table.setActualTable("orders");
|
||||
DatacenterTable discovered = new DatacenterTable();
|
||||
discovered.setTableName("orders");
|
||||
discovered.setActualTable("orders");
|
||||
DatacenterTableDetailMeta detail = new DatacenterTableDetailMeta();
|
||||
detail.setTable(discovered);
|
||||
LoginAccount account = account();
|
||||
|
||||
DatacenterConnector connector = Mockito.mock(DatacenterConnector.class);
|
||||
DatacenterConnectorRegistry connectorRegistry =
|
||||
Mockito.mock(DatacenterConnectorRegistry.class);
|
||||
DatacenterDatasetRegistryService registryService =
|
||||
Mockito.mock(DatacenterDatasetRegistryService.class);
|
||||
DatacenterCatalogMapper catalogMapper =
|
||||
Mockito.mock(DatacenterCatalogMapper.class);
|
||||
DatacenterTableMapper tableMapper = Mockito.mock(DatacenterTableMapper.class);
|
||||
DatacenterSourceMapper sourceMapper =
|
||||
Mockito.mock(DatacenterSourceMapper.class);
|
||||
DatacenterFederationChangeNotifier notifier =
|
||||
Mockito.mock(DatacenterFederationChangeNotifier.class);
|
||||
PlatformTransactionManager transactionManager =
|
||||
Mockito.mock(PlatformTransactionManager.class);
|
||||
TransactionStatus transactionStatus = Mockito.mock(TransactionStatus.class);
|
||||
Mockito.when(connectorRegistry.getConnector("MYSQL")).thenReturn(connector);
|
||||
Mockito.when(catalogMapper.selectListByQuery(ArgumentMatchers.any()))
|
||||
.thenReturn(List.of(catalog));
|
||||
Mockito.when(tableMapper.selectListByQuery(ArgumentMatchers.any()))
|
||||
.thenReturn(List.of(table));
|
||||
Mockito.when(connector.inspectManagedTables(
|
||||
source, "orders_db", List.of("orders")))
|
||||
.thenReturn(new DatacenterManagedMetadataSnapshot(
|
||||
List.of(detail), Set.of()));
|
||||
Mockito.when(registryService.refreshManagedTable(
|
||||
source, catalog, table.getId(), detail, account))
|
||||
.thenReturn(false);
|
||||
Mockito.when(transactionManager.getTransaction(
|
||||
ArgumentMatchers.any(TransactionDefinition.class)))
|
||||
.thenReturn(transactionStatus);
|
||||
Mockito.when(sourceMapper.updateByQuery(
|
||||
ArgumentMatchers.any(), ArgumentMatchers.any()))
|
||||
.thenReturn(1);
|
||||
DatacenterSource refreshed = new DatacenterSource();
|
||||
refreshed.setId(sourceId);
|
||||
Mockito.when(sourceMapper.selectOneByQuery(ArgumentMatchers.any()))
|
||||
.thenReturn(refreshed);
|
||||
DatacenterSourceMetadataRefreshService service =
|
||||
new DatacenterSourceMetadataRefreshService(
|
||||
connectorRegistry, registryService, catalogMapper,
|
||||
tableMapper, sourceMapper, notifier, transactionManager);
|
||||
|
||||
DatacenterSource result = service.refresh(source, account);
|
||||
|
||||
Assert.assertSame(refreshed, result);
|
||||
InOrder order = Mockito.inOrder(connector, transactionManager, sourceMapper);
|
||||
order.verify(connector).inspectManagedTables(
|
||||
source, "orders_db", List.of("orders"));
|
||||
order.verify(transactionManager).getTransaction(
|
||||
ArgumentMatchers.any(TransactionDefinition.class));
|
||||
order.verify(sourceMapper).updateByQuery(
|
||||
ArgumentMatchers.any(), ArgumentMatchers.any());
|
||||
Mockito.verify(notifier).publishAfterCommit(refreshed);
|
||||
}
|
||||
|
||||
private LoginAccount account() {
|
||||
LoginAccount account = new LoginAccount();
|
||||
account.setId(BigInteger.TEN);
|
||||
account.setTenantId(BigInteger.ONE);
|
||||
return account;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,410 @@
|
||||
package tech.easyflow.datacenter.meta.service.impl;
|
||||
|
||||
import com.mybatisflex.core.query.QueryWrapper;
|
||||
import com.mybatisflex.spring.service.impl.ServiceImpl;
|
||||
import java.lang.reflect.Field;
|
||||
import java.math.BigInteger;
|
||||
import java.util.List;
|
||||
import org.junit.Assert;
|
||||
import org.junit.Test;
|
||||
import org.mockito.ArgumentCaptor;
|
||||
import org.mockito.ArgumentMatchers;
|
||||
import org.mockito.InOrder;
|
||||
import org.mockito.Mockito;
|
||||
import org.springframework.transaction.PlatformTransactionManager;
|
||||
import org.springframework.transaction.TransactionDefinition;
|
||||
import org.springframework.transaction.TransactionStatus;
|
||||
import tech.easyflow.common.entity.LoginAccount;
|
||||
import tech.easyflow.datacenter.connector.DatacenterConnector;
|
||||
import tech.easyflow.datacenter.connector.DatacenterConnectorRegistry;
|
||||
import tech.easyflow.datacenter.entity.DatacenterTable;
|
||||
import tech.easyflow.datacenter.federation.DatacenterFederationChangeNotifier;
|
||||
import tech.easyflow.datacenter.federation.DatacenterFederationDefinitionFactory;
|
||||
import tech.easyflow.datacenter.federation.DatacenterFederationRuntime;
|
||||
import tech.easyflow.datacenter.mapper.DatacenterCatalogMapper;
|
||||
import tech.easyflow.datacenter.mapper.DatacenterSourceMapper;
|
||||
import tech.easyflow.datacenter.mapper.DatacenterTableMapper;
|
||||
import tech.easyflow.datacenter.meta.entity.DatacenterCatalog;
|
||||
import tech.easyflow.datacenter.meta.entity.DatacenterSource;
|
||||
import tech.easyflow.datacenter.meta.enums.DatacenterMetadataStatus;
|
||||
import tech.easyflow.datacenter.meta.enums.DatacenterSourceStatus;
|
||||
import tech.easyflow.datacenter.meta.model.DatacenterBatchRegisterRequest;
|
||||
import tech.easyflow.datacenter.meta.model.DatacenterSourceActivateRequest;
|
||||
import tech.easyflow.datacenter.meta.model.DatacenterTableDetailMeta;
|
||||
import tech.easyflow.datacenter.meta.model.DatacenterSourceDraftRequest;
|
||||
import tech.easyflow.datacenter.meta.model.DatacenterSourceReconfigureRequest;
|
||||
import tech.easyflow.datacenter.meta.service.DatacenterDatasetRegistryService;
|
||||
import tech.easyflow.datacenter.meta.support.DatacenterSourceConnectionDefaults;
|
||||
import tech.easyflow.datacenter.security.DatacenterCredentialCipher;
|
||||
import com.easyagents.federation.sql.source.ExternalSchemaDefinition;
|
||||
import com.easyagents.federation.sql.source.FederationSourceDefinition;
|
||||
import com.easyagents.federation.sql.source.PreparedSourceRuntime;
|
||||
import com.easyagents.federation.sql.source.RuntimeFingerprint;
|
||||
import com.easyagents.federation.sql.source.SourceId;
|
||||
import com.easyagents.federation.sql.source.SourceProbeResult;
|
||||
|
||||
/**
|
||||
* {@link DatacenterSourceServiceImpl} 生命周期和 revision 回归测试。
|
||||
*/
|
||||
public class DatacenterSourceServiceImplTest {
|
||||
|
||||
/**
|
||||
* 已发布连接不得复用初次激活入口绕过重配置 revision CAS。
|
||||
*
|
||||
* @throws Exception 测试依赖注入失败
|
||||
*/
|
||||
@Test
|
||||
public void activateShouldRejectPublishedSource() throws Exception {
|
||||
Fixture fixture = fixture();
|
||||
DatacenterSource ready = source(DatacenterSourceStatus.READY, 3L, 7L);
|
||||
Mockito.when(fixture.sourceMapper.selectOneByQuery(ArgumentMatchers.any()))
|
||||
.thenReturn(ready);
|
||||
|
||||
try {
|
||||
fixture.service.activate(new DatacenterSourceActivateRequest(
|
||||
ready.getId(), "public", List.of("orders"), true), account());
|
||||
Assert.fail("published source must use reconfiguration");
|
||||
} catch (tech.easyflow.common.web.exceptions.BusinessException expected) {
|
||||
Assert.assertTrue(expected.getMessage().contains("重新配置"));
|
||||
}
|
||||
|
||||
Mockito.verifyNoInteractions(fixture.connectorRegistry);
|
||||
Mockito.verify(fixture.runtime, Mockito.never())
|
||||
.probe(ArgumentMatchers.any());
|
||||
Mockito.verify(fixture.runtime, Mockito.never())
|
||||
.probeCandidate(ArgumentMatchers.any());
|
||||
}
|
||||
|
||||
/**
|
||||
* 已发布连接按持久化 Definition 探测,避免候选 MAIN Schema 造成 checksum 冲突。
|
||||
*
|
||||
* @throws Exception 测试依赖注入失败
|
||||
*/
|
||||
@Test
|
||||
public void publishedSourceProbeShouldUsePublishedDefinition() throws Exception {
|
||||
Fixture fixture = fixture();
|
||||
DatacenterSource ready = source(DatacenterSourceStatus.READY, 3L, 7L);
|
||||
ready.setSchemaName("tenant_schema");
|
||||
ready.setDefinitionChecksum("published-checksum");
|
||||
Mockito.when(fixture.sourceMapper.selectOneByQuery(ArgumentMatchers.any()))
|
||||
.thenReturn(ready);
|
||||
Mockito.when(fixture.sourceMapper.updateByQuery(
|
||||
ArgumentMatchers.any(), ArgumentMatchers.any())).thenReturn(1);
|
||||
Mockito.when(fixture.runtime.probe(ready)).thenReturn(new SourceProbeResult(
|
||||
new SourceId("tenant-1-source-100"), true,
|
||||
new RuntimeFingerprint(
|
||||
"PostgreSQL", "17", "PostgreSQL JDBC", "42", "test"),
|
||||
"ok"));
|
||||
|
||||
Assert.assertTrue(fixture.service.probe(ready.getId(), account()).isSuccess());
|
||||
|
||||
Mockito.verify(fixture.runtime).probe(ready);
|
||||
Mockito.verify(fixture.runtime, Mockito.never())
|
||||
.probeCandidate(ArgumentMatchers.any());
|
||||
}
|
||||
|
||||
/**
|
||||
* 停用只推进 Definition 版本并发布墓碑,不改变业务范围版本。
|
||||
*
|
||||
* @throws Exception 测试依赖注入失败
|
||||
*/
|
||||
@Test
|
||||
public void disableShouldAdvanceDefinitionRevisionOnly() throws Exception {
|
||||
Fixture fixture = fixture();
|
||||
DatacenterSource ready = source(DatacenterSourceStatus.READY, 3L, 7L);
|
||||
DatacenterSource disabled = source(DatacenterSourceStatus.DISABLED, 4L, 7L);
|
||||
Mockito.when(fixture.sourceMapper.selectOneByQuery(ArgumentMatchers.any()))
|
||||
.thenReturn(ready, disabled);
|
||||
Mockito.when(fixture.sourceMapper.updateByQuery(
|
||||
ArgumentMatchers.any(), ArgumentMatchers.any())).thenReturn(1);
|
||||
|
||||
fixture.service.disable(ready.getId(), account());
|
||||
|
||||
ArgumentCaptor<DatacenterSource> patch =
|
||||
ArgumentCaptor.forClass(DatacenterSource.class);
|
||||
Mockito.verify(fixture.sourceMapper).updateByQuery(
|
||||
patch.capture(), ArgumentMatchers.any(QueryWrapper.class));
|
||||
Assert.assertEquals(Long.valueOf(4L), patch.getValue().getDefinitionRevision());
|
||||
Assert.assertNull(patch.getValue().getScopeRevision());
|
||||
Assert.assertEquals(Integer.valueOf(DatacenterSourceStatus.DISABLED.code()),
|
||||
patch.getValue().getStatus());
|
||||
Mockito.verify(fixture.runtime).remove(disabled);
|
||||
Mockito.verify(fixture.notifier).publishAfterCommit(disabled);
|
||||
}
|
||||
|
||||
/**
|
||||
* 已激活连接新增纳管表时只推进 scopeRevision,避免无意义重建连接池。
|
||||
*
|
||||
* @throws Exception 测试依赖注入失败
|
||||
*/
|
||||
@Test
|
||||
public void batchRegisterShouldAdvanceScopeRevisionOnly() throws Exception {
|
||||
Fixture fixture = fixture();
|
||||
DatacenterSource ready = source(DatacenterSourceStatus.READY, 5L, 9L);
|
||||
ready.setLastTestStatus("SUCCESS");
|
||||
DatacenterSource updated = source(DatacenterSourceStatus.READY, 5L, 10L);
|
||||
DatacenterCatalog catalog = new DatacenterCatalog();
|
||||
catalog.setId(BigInteger.valueOf(200L));
|
||||
catalog.setSourceId(ready.getId());
|
||||
catalog.setCatalogName("public");
|
||||
catalog.setMetadataStatus(DatacenterMetadataStatus.ACTIVE.name());
|
||||
DatacenterTableDetailMeta detail = new DatacenterTableDetailMeta();
|
||||
DatacenterTable discovered = new DatacenterTable();
|
||||
discovered.setTableName("orders");
|
||||
detail.setTable(discovered);
|
||||
DatacenterConnector connector = Mockito.mock(DatacenterConnector.class);
|
||||
|
||||
Mockito.when(fixture.sourceMapper.selectOneByQuery(ArgumentMatchers.any()))
|
||||
.thenReturn(ready, updated);
|
||||
Mockito.when(fixture.sourceMapper.updateByQuery(
|
||||
ArgumentMatchers.any(), ArgumentMatchers.any())).thenReturn(1);
|
||||
Mockito.when(fixture.registry.listManagedTables(
|
||||
ready.getId(), catalog.getId())).thenReturn(List.of());
|
||||
Mockito.when(fixture.catalogMapper.selectOneByQuery(ArgumentMatchers.any()))
|
||||
.thenReturn(catalog);
|
||||
Mockito.when(fixture.connectorRegistry.getConnector("MYSQL"))
|
||||
.thenReturn(connector);
|
||||
Mockito.when(connector.getTableDetails(
|
||||
ready, "public", List.of("orders"))).thenReturn(List.of(detail));
|
||||
|
||||
DatacenterBatchRegisterRequest request = new DatacenterBatchRegisterRequest();
|
||||
request.setSourceId(ready.getId());
|
||||
request.setCatalogName("public");
|
||||
request.setTableNames(List.of("orders"));
|
||||
fixture.service.batchRegisterTables(request, account());
|
||||
|
||||
ArgumentCaptor<DatacenterSource> patch =
|
||||
ArgumentCaptor.forClass(DatacenterSource.class);
|
||||
Mockito.verify(fixture.sourceMapper).updateByQuery(
|
||||
patch.capture(), ArgumentMatchers.any(QueryWrapper.class));
|
||||
Assert.assertNull(patch.getValue().getDefinitionRevision());
|
||||
Assert.assertEquals(Long.valueOf(10L), patch.getValue().getScopeRevision());
|
||||
Mockito.verify(fixture.runtime, Mockito.never()).apply(
|
||||
ArgumentMatchers.any(), ArgumentMatchers.anyBoolean());
|
||||
Mockito.verify(fixture.notifier).publishAfterCommit(updated);
|
||||
}
|
||||
|
||||
/**
|
||||
* 候选配置探测失败时不写数据库,也不替换当前 Runtime。
|
||||
*
|
||||
* @throws Exception 测试依赖注入失败
|
||||
*/
|
||||
@Test
|
||||
public void failedReconfigurationProbeShouldKeepReadySourceUntouched()
|
||||
throws Exception {
|
||||
Fixture fixture = fixture();
|
||||
DatacenterSource ready = source(DatacenterSourceStatus.READY, 8L, 12L);
|
||||
ready.setSourceName("orders-db");
|
||||
ready.setSourceCode("ORDERS_DB");
|
||||
ready.setHost("127.0.0.1");
|
||||
ready.setPort(3306);
|
||||
ready.setDatabaseName("orders");
|
||||
ready.setUsername("reader");
|
||||
ready.setCredentialCipher("encrypted");
|
||||
Mockito.when(fixture.sourceMapper.selectOneByQuery(ArgumentMatchers.any()))
|
||||
.thenReturn(ready);
|
||||
Mockito.when(fixture.runtime.probeCandidate(ArgumentMatchers.any()))
|
||||
.thenThrow(new IllegalStateException("simulated connection failure"));
|
||||
Mockito.when(fixture.defaults.defaultDriverClassName("MYSQL"))
|
||||
.thenReturn("com.mysql.cj.jdbc.Driver");
|
||||
DatacenterSourceDraftRequest definition = new DatacenterSourceDraftRequest(
|
||||
ready.getId(), "orders-db", "ORDERS_DB", "MYSQL",
|
||||
"127.0.0.2", 3306, "orders", null, "reader", null,
|
||||
null, null, java.util.Map.of(), java.util.Map.of());
|
||||
DatacenterSourceReconfigureRequest request =
|
||||
new DatacenterSourceReconfigureRequest(
|
||||
definition, "orders", List.of("orders"), true, 8L, 12L);
|
||||
|
||||
try {
|
||||
fixture.service.reconfigure(request, account());
|
||||
Assert.fail("failed candidate probe must reject reconfiguration");
|
||||
} catch (tech.easyflow.common.web.exceptions.BusinessException expected) {
|
||||
Assert.assertTrue(expected.getMessage().contains("无法连接"));
|
||||
}
|
||||
|
||||
Mockito.verify(fixture.sourceMapper, Mockito.never()).updateByQuery(
|
||||
ArgumentMatchers.any(), ArgumentMatchers.any());
|
||||
Mockito.verify(fixture.runtime, Mockito.never()).apply(
|
||||
ArgumentMatchers.any(), ArgumentMatchers.anyBoolean());
|
||||
}
|
||||
|
||||
/**
|
||||
* 重新启用时先预构建 Runtime,再发布权威 READY 并原子接管。
|
||||
*
|
||||
* @throws Exception 测试依赖注入失败
|
||||
*/
|
||||
@Test
|
||||
public void enableShouldPrepareBeforePublishingAndCommitAfterward() throws Exception {
|
||||
Fixture fixture = fixture();
|
||||
DatacenterSource disabled = source(DatacenterSourceStatus.DISABLED, 4L, 7L);
|
||||
disabled.setSourceName("orders-db");
|
||||
disabled.setSourceCode("ORDERS_DB");
|
||||
disabled.setDatabaseName("orders");
|
||||
disabled.setSchemaName("public");
|
||||
disabled.setCredentialCipher("encrypted");
|
||||
disabled.setAdapterId("jdbc");
|
||||
DatacenterSource enabled = source(DatacenterSourceStatus.READY, 5L, 7L);
|
||||
Mockito.when(fixture.sourceMapper.selectOneByQuery(ArgumentMatchers.any()))
|
||||
.thenReturn(disabled, enabled);
|
||||
Mockito.when(fixture.sourceMapper.updateByQuery(
|
||||
ArgumentMatchers.any(), ArgumentMatchers.any())).thenReturn(1);
|
||||
RuntimeFingerprint fingerprint = new RuntimeFingerprint(
|
||||
"MySQL", "8.0", "MySQL Connector/J", "9.0", "test");
|
||||
Mockito.when(fixture.runtime.probeCandidate(ArgumentMatchers.any()))
|
||||
.thenReturn(new SourceProbeResult(
|
||||
new SourceId("tenant-1-source-100"), true, fingerprint, "ok"));
|
||||
FederationSourceDefinition definition = new FederationSourceDefinition(
|
||||
new SourceId("tenant-1-source-100"),
|
||||
5L,
|
||||
"jdbc",
|
||||
List.of(new ExternalSchemaDefinition("public", "snapshot", 1L)),
|
||||
java.util.Map.of());
|
||||
PreparedSourceRuntime prepared = Mockito.mock(PreparedSourceRuntime.class);
|
||||
Mockito.when(prepared.definition()).thenReturn(definition);
|
||||
Mockito.doAnswer(invocation -> {
|
||||
DatacenterSource candidate = invocation.getArgument(0);
|
||||
Assert.assertEquals(Integer.valueOf(DatacenterSourceStatus.DRAFT.code()),
|
||||
candidate.getStatus());
|
||||
Assert.assertNull(candidate.getDefinitionChecksum());
|
||||
return prepared;
|
||||
}).when(fixture.runtime).prepareCandidate(ArgumentMatchers.any());
|
||||
|
||||
fixture.service.enable(disabled.getId(), account());
|
||||
|
||||
InOrder order = Mockito.inOrder(fixture.runtime, fixture.sourceMapper);
|
||||
order.verify(fixture.runtime).probeCandidate(ArgumentMatchers.any());
|
||||
order.verify(fixture.runtime).prepareCandidate(ArgumentMatchers.any());
|
||||
order.verify(fixture.sourceMapper).updateByQuery(
|
||||
ArgumentMatchers.any(), ArgumentMatchers.any(QueryWrapper.class));
|
||||
order.verify(fixture.runtime).commitPrepared(prepared);
|
||||
}
|
||||
|
||||
/**
|
||||
* 重新启用的 revision CAS 失败时丢弃预构建 Runtime,且不得发布候选连接池。
|
||||
*
|
||||
* @throws Exception 测试依赖注入失败
|
||||
*/
|
||||
@Test
|
||||
public void enableCasFailureShouldDiscardPreparedRuntime() throws Exception {
|
||||
Fixture fixture = fixture();
|
||||
DatacenterSource disabled = source(DatacenterSourceStatus.DISABLED, 4L, 7L);
|
||||
disabled.setSourceName("orders-db");
|
||||
disabled.setSourceCode("ORDERS_DB");
|
||||
disabled.setDatabaseName("orders");
|
||||
disabled.setSchemaName("public");
|
||||
disabled.setCredentialCipher("encrypted");
|
||||
disabled.setAdapterId("jdbc");
|
||||
Mockito.when(fixture.sourceMapper.selectOneByQuery(ArgumentMatchers.any()))
|
||||
.thenReturn(disabled);
|
||||
Mockito.when(fixture.sourceMapper.updateByQuery(
|
||||
ArgumentMatchers.any(), ArgumentMatchers.any())).thenReturn(0);
|
||||
RuntimeFingerprint fingerprint = new RuntimeFingerprint(
|
||||
"MySQL", "8.0", "MySQL Connector/J", "9.0", "test");
|
||||
Mockito.when(fixture.runtime.probeCandidate(ArgumentMatchers.any()))
|
||||
.thenReturn(new SourceProbeResult(
|
||||
new SourceId("tenant-1-source-100"), true, fingerprint, "ok"));
|
||||
FederationSourceDefinition definition = new FederationSourceDefinition(
|
||||
new SourceId("tenant-1-source-100"),
|
||||
5L,
|
||||
"jdbc",
|
||||
List.of(new ExternalSchemaDefinition("public", "snapshot", 1L)),
|
||||
java.util.Map.of());
|
||||
PreparedSourceRuntime prepared = Mockito.mock(PreparedSourceRuntime.class);
|
||||
Mockito.when(prepared.definition()).thenReturn(definition);
|
||||
Mockito.doAnswer(invocation -> {
|
||||
DatacenterSource candidate = invocation.getArgument(0);
|
||||
Assert.assertEquals(Integer.valueOf(DatacenterSourceStatus.DRAFT.code()),
|
||||
candidate.getStatus());
|
||||
Assert.assertNull(candidate.getDefinitionChecksum());
|
||||
return prepared;
|
||||
}).when(fixture.runtime).prepareCandidate(ArgumentMatchers.any());
|
||||
|
||||
try {
|
||||
fixture.service.enable(disabled.getId(), account());
|
||||
Assert.fail("stale revision must reject enable");
|
||||
} catch (tech.easyflow.common.web.exceptions.BusinessException expected) {
|
||||
Assert.assertNotNull(expected.getMessage());
|
||||
}
|
||||
|
||||
Mockito.verify(prepared).close();
|
||||
Mockito.verify(fixture.runtime, Mockito.never()).commitPrepared(prepared);
|
||||
Mockito.verify(fixture.notifier, Mockito.never())
|
||||
.publishAfterCommit(ArgumentMatchers.any());
|
||||
}
|
||||
|
||||
private Fixture fixture() throws Exception {
|
||||
DatacenterConnectorRegistry connectorRegistry =
|
||||
Mockito.mock(DatacenterConnectorRegistry.class);
|
||||
DatacenterDatasetRegistryService registry =
|
||||
Mockito.mock(DatacenterDatasetRegistryService.class);
|
||||
DatacenterCredentialCipher cipher = Mockito.mock(DatacenterCredentialCipher.class);
|
||||
DatacenterCatalogMapper catalogMapper = Mockito.mock(DatacenterCatalogMapper.class);
|
||||
DatacenterTableMapper tableMapper = Mockito.mock(DatacenterTableMapper.class);
|
||||
DatacenterSourceConnectionDefaults defaults =
|
||||
Mockito.mock(DatacenterSourceConnectionDefaults.class);
|
||||
DatacenterFederationRuntime runtime = Mockito.mock(DatacenterFederationRuntime.class);
|
||||
Mockito.when(catalogMapper.selectListByQuery(ArgumentMatchers.any()))
|
||||
.thenReturn(List.of());
|
||||
Mockito.when(runtime.definitions())
|
||||
.thenReturn(new DatacenterFederationDefinitionFactory(catalogMapper));
|
||||
DatacenterFederationChangeNotifier notifier =
|
||||
Mockito.mock(DatacenterFederationChangeNotifier.class);
|
||||
DatacenterSourceMapper sourceMapper = Mockito.mock(DatacenterSourceMapper.class);
|
||||
DatacenterSourceMetadataService metadataService =
|
||||
new DatacenterSourceMetadataService(
|
||||
connectorRegistry, registry, catalogMapper);
|
||||
DatacenterSourceCandidateService candidateService =
|
||||
new DatacenterSourceCandidateService(cipher, defaults, runtime);
|
||||
PlatformTransactionManager transactionManager =
|
||||
Mockito.mock(PlatformTransactionManager.class);
|
||||
Mockito.when(transactionManager.getTransaction(
|
||||
ArgumentMatchers.any(TransactionDefinition.class)))
|
||||
.thenReturn(Mockito.mock(TransactionStatus.class));
|
||||
DatacenterSourceMetadataRefreshService metadataRefreshService =
|
||||
new DatacenterSourceMetadataRefreshService(
|
||||
connectorRegistry, registry, catalogMapper, tableMapper,
|
||||
sourceMapper, notifier, transactionManager);
|
||||
DatacenterSourceServiceImpl service = new DatacenterSourceServiceImpl(
|
||||
connectorRegistry, registry, catalogMapper, tableMapper,
|
||||
defaults, candidateService, metadataService,
|
||||
metadataRefreshService, runtime, notifier, transactionManager);
|
||||
Field mapper = ServiceImpl.class.getDeclaredField("mapper");
|
||||
mapper.setAccessible(true);
|
||||
mapper.set(service, sourceMapper);
|
||||
return new Fixture(
|
||||
service, connectorRegistry, registry, catalogMapper,
|
||||
sourceMapper, runtime, notifier, defaults);
|
||||
}
|
||||
|
||||
private DatacenterSource source(
|
||||
DatacenterSourceStatus status,
|
||||
long definitionRevision,
|
||||
long scopeRevision) {
|
||||
DatacenterSource source = new DatacenterSource();
|
||||
source.setId(BigInteger.valueOf(100L));
|
||||
source.setTenantId(BigInteger.ONE);
|
||||
source.setSourceType("MYSQL");
|
||||
source.setStatus(status.code());
|
||||
source.setDefinitionRevision(definitionRevision);
|
||||
source.setScopeRevision(scopeRevision);
|
||||
return source;
|
||||
}
|
||||
|
||||
private LoginAccount account() {
|
||||
LoginAccount account = new LoginAccount();
|
||||
account.setId(BigInteger.TEN);
|
||||
account.setTenantId(BigInteger.ONE);
|
||||
return account;
|
||||
}
|
||||
|
||||
private record Fixture(
|
||||
DatacenterSourceServiceImpl service,
|
||||
DatacenterConnectorRegistry connectorRegistry,
|
||||
DatacenterDatasetRegistryService registry,
|
||||
DatacenterCatalogMapper catalogMapper,
|
||||
DatacenterSourceMapper sourceMapper,
|
||||
DatacenterFederationRuntime runtime,
|
||||
DatacenterFederationChangeNotifier notifier,
|
||||
DatacenterSourceConnectionDefaults defaults) {
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,53 @@
|
||||
package tech.easyflow.datacenter.meta.support;
|
||||
|
||||
import java.util.Map;
|
||||
import org.junit.Assert;
|
||||
import org.junit.Test;
|
||||
import tech.easyflow.datacenter.meta.entity.DatacenterSource;
|
||||
|
||||
/**
|
||||
* {@link DatacenterSourceConnectionDefaults} 安全连接默认值回归测试。
|
||||
*/
|
||||
public class DatacenterSourceConnectionDefaultsTest {
|
||||
|
||||
private final DatacenterSourceConnectionDefaults defaults =
|
||||
new DatacenterSourceConnectionDefaults();
|
||||
|
||||
/**
|
||||
* 验证可能承载凭据的配置项不会进入持久化配置或 Definition。
|
||||
*/
|
||||
@Test
|
||||
public void shouldRemoveSensitiveOptions() {
|
||||
Map<String, Object> sanitized = defaults.sanitizeConfig(Map.of(
|
||||
"socketTimeout", 30,
|
||||
"password", "plain",
|
||||
"apiToken", "token",
|
||||
"clientSecret", "secret",
|
||||
"privateKey", "key"));
|
||||
|
||||
Assert.assertEquals(Map.of("socketTimeout", 30), sanitized);
|
||||
}
|
||||
|
||||
/**
|
||||
* 验证首批 MySQL 与 PostgreSQL URL 默认具有有界连接和查询等待。
|
||||
*/
|
||||
@Test
|
||||
public void shouldGenerateBoundedJdbcUrls() {
|
||||
DatacenterSource mysql = source("MYSQL", 3306, "easyflow");
|
||||
DatacenterSource postgres = source("POSTGRESQL", 5432, "harmony_adapter");
|
||||
|
||||
Assert.assertTrue(defaults.buildJdbcUrl(mysql).contains("connectTimeout=5000"));
|
||||
Assert.assertTrue(defaults.buildJdbcUrl(mysql).contains("socketTimeout=35000"));
|
||||
Assert.assertTrue(defaults.buildJdbcUrl(postgres).contains("connectTimeout=5"));
|
||||
Assert.assertTrue(defaults.buildJdbcUrl(postgres).contains("socketTimeout=35"));
|
||||
}
|
||||
|
||||
private DatacenterSource source(String type, int port, String databaseName) {
|
||||
DatacenterSource source = new DatacenterSource();
|
||||
source.setSourceType(type);
|
||||
source.setHost("127.0.0.1");
|
||||
source.setPort(port);
|
||||
source.setDatabaseName(databaseName);
|
||||
return source;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,45 @@
|
||||
package tech.easyflow.datacenter.security;
|
||||
|
||||
import org.junit.Assert;
|
||||
import org.junit.Test;
|
||||
|
||||
/**
|
||||
* {@link DatacenterCredentialCipher} 认证加密回归测试。
|
||||
*/
|
||||
public class DatacenterCredentialCipherTest {
|
||||
|
||||
/**
|
||||
* 验证随机 IV、正确解密和篡改检测。
|
||||
*/
|
||||
@Test
|
||||
public void shouldEncryptWithRandomIvAndRejectTampering() {
|
||||
DatacenterCredentialCipher cipher = new DatacenterCredentialCipher(
|
||||
"test-only-master-key-with-at-least-32-characters");
|
||||
|
||||
String first = cipher.encrypt("secret-value");
|
||||
String second = cipher.encrypt("secret-value");
|
||||
|
||||
Assert.assertNotEquals(first, second);
|
||||
Assert.assertEquals("secret-value", cipher.decrypt(first));
|
||||
String tampered = first.substring(0, first.length() - 2) + "AA";
|
||||
try {
|
||||
cipher.decrypt(tampered);
|
||||
Assert.fail("tampered credential must be rejected");
|
||||
} catch (IllegalStateException expected) {
|
||||
Assert.assertTrue(expected.getMessage().contains("decrypt"));
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 验证部署未提供足够强度的主密钥时立即停止启动。
|
||||
*/
|
||||
@Test
|
||||
public void shouldRejectMissingMasterKey() {
|
||||
try {
|
||||
new DatacenterCredentialCipher("too-short");
|
||||
Assert.fail("short master key must be rejected");
|
||||
} catch (IllegalStateException expected) {
|
||||
Assert.assertTrue(expected.getMessage().contains("32"));
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -2,6 +2,7 @@ package tech.easyflow.datacenter.utils;
|
||||
|
||||
import org.junit.Assert;
|
||||
import org.junit.Test;
|
||||
import tech.easyflow.common.web.exceptions.BusinessException;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
@@ -24,7 +25,9 @@ public class SqlSupportUtilsTest {
|
||||
List.of(new SqlSupportUtils.ManagedTable(
|
||||
"ama 实验基线模型预算",
|
||||
"Sheet1",
|
||||
"tb_excel_budget")));
|
||||
"tb_excel_budget",
|
||||
List.of("token", "token_1", "col_id"),
|
||||
List.of("token", "token_1", "col_id"))));
|
||||
|
||||
Assert.assertEquals(
|
||||
"SELECT `token`, `token_1` FROM tb_excel_budget "
|
||||
@@ -42,10 +45,564 @@ public class SqlSupportUtilsTest {
|
||||
List.of(new SqlSupportUtils.ManagedTable(
|
||||
null,
|
||||
"Sheet1",
|
||||
"tb_excel_budget")));
|
||||
"tb_excel_budget",
|
||||
List.of("token", "col_id"),
|
||||
List.of("token", "col_id"))));
|
||||
|
||||
Assert.assertEquals(
|
||||
"SELECT `token` FROM tb_excel_budget WHERE `col_id` = 'deep\"seek'",
|
||||
resolved.getExecutableSql());
|
||||
}
|
||||
|
||||
/**
|
||||
* 验证 PostgreSQL 引号标识符不会与未引号的小写对象混为同一张表。
|
||||
*/
|
||||
@Test(expected = BusinessException.class)
|
||||
public void resolveShouldNotFoldQuotedManagedTableToUnquotedReference() {
|
||||
SqlSupportUtils.resolve(
|
||||
"SELECT * FROM foo",
|
||||
List.of(new SqlSupportUtils.ManagedTable(
|
||||
null,
|
||||
"Foo",
|
||||
"Foo")));
|
||||
}
|
||||
|
||||
/**
|
||||
* 验证大小写敏感表可通过精确引号标识符访问。
|
||||
*/
|
||||
@Test
|
||||
public void resolveShouldMatchQuotedManagedTableExactly() {
|
||||
SqlSupportUtils.ResolvedSql resolved = SqlSupportUtils.resolve(
|
||||
"SELECT * FROM \"Foo\"",
|
||||
List.of(new SqlSupportUtils.ManagedTable(
|
||||
null,
|
||||
"Foo",
|
||||
"Foo")));
|
||||
|
||||
Assert.assertEquals(List.of("Foo"), resolved.getLogicalTables());
|
||||
}
|
||||
|
||||
/**
|
||||
* 验证内部 SQL 无法读取未开放字段。
|
||||
*/
|
||||
@Test(expected = BusinessException.class)
|
||||
public void resolveInternalMysqlShouldRejectHiddenColumn() {
|
||||
SqlSupportUtils.resolveInternalMysql(
|
||||
"SELECT secret FROM Sheet1",
|
||||
List.of(internalTable(
|
||||
List.of("public_col", "secret"),
|
||||
List.of("public_col"))));
|
||||
}
|
||||
|
||||
/**
|
||||
* 验证 IS NULL 条件中的隐藏字段仍受字段策略约束。
|
||||
*/
|
||||
@Test(expected = BusinessException.class)
|
||||
public void resolveInternalMysqlShouldRejectHiddenColumnInIsNull() {
|
||||
SqlSupportUtils.resolveInternalMysql(
|
||||
"SELECT public_col FROM \"Sheet1\" WHERE secret IS NULL",
|
||||
List.of(internalTable(
|
||||
List.of("public_col", "secret"),
|
||||
List.of("public_col"))));
|
||||
}
|
||||
|
||||
/**
|
||||
* 验证排序条件中的隐藏字段仍受字段策略约束。
|
||||
*/
|
||||
@Test(expected = BusinessException.class)
|
||||
public void resolveInternalMysqlShouldRejectHiddenColumnInOrderBy() {
|
||||
SqlSupportUtils.resolveInternalMysql(
|
||||
"SELECT public_col FROM \"Sheet1\" ORDER BY secret",
|
||||
List.of(internalTable(
|
||||
List.of("public_col", "secret"),
|
||||
List.of("public_col"))));
|
||||
}
|
||||
|
||||
/**
|
||||
* 验证分组条件中的隐藏字段仍受字段策略约束。
|
||||
*/
|
||||
@Test(expected = BusinessException.class)
|
||||
public void resolveInternalMysqlShouldRejectHiddenColumnInGroupBy() {
|
||||
SqlSupportUtils.resolveInternalMysql(
|
||||
"SELECT COUNT(*) FROM \"Sheet1\" GROUP BY secret",
|
||||
List.of(internalTable(
|
||||
List.of("public_col", "secret"),
|
||||
List.of("public_col"))));
|
||||
}
|
||||
|
||||
/**
|
||||
* 验证窗口排序中的隐藏字段仍受字段策略约束。
|
||||
*/
|
||||
@Test(expected = BusinessException.class)
|
||||
public void resolveInternalMysqlShouldRejectHiddenColumnInWindowOrderBy() {
|
||||
SqlSupportUtils.resolveInternalMysql(
|
||||
"SELECT public_col, ROW_NUMBER() OVER (ORDER BY secret) AS rn "
|
||||
+ "FROM \"Sheet1\"",
|
||||
List.of(internalTable(
|
||||
List.of("public_col", "secret"),
|
||||
List.of("public_col"))));
|
||||
}
|
||||
|
||||
/**
|
||||
* 验证命名窗口排序中的隐藏字段仍受字段策略约束。
|
||||
*/
|
||||
@Test(expected = BusinessException.class)
|
||||
public void resolveInternalMysqlShouldRejectHiddenColumnInNamedWindow() {
|
||||
SqlSupportUtils.resolveInternalMysql(
|
||||
"SELECT public_col, ROW_NUMBER() OVER w AS rn FROM \"Sheet1\" "
|
||||
+ "WINDOW w AS (ORDER BY secret)",
|
||||
List.of(internalTable(
|
||||
List.of("public_col", "secret"),
|
||||
List.of("public_col"))));
|
||||
}
|
||||
|
||||
/**
|
||||
* 验证合法窗口 frame 可以安全解析且不会触发空值异常。
|
||||
*/
|
||||
@Test
|
||||
public void resolveInternalMysqlShouldAllowSafeWindowFrame() {
|
||||
SqlSupportUtils.ResolvedSql resolved = SqlSupportUtils.resolveInternalMysql(
|
||||
"SELECT SUM(public_col) OVER (ORDER BY public_col "
|
||||
+ "ROWS BETWEEN 1 PRECEDING AND CURRENT ROW) AS total "
|
||||
+ "FROM \"Sheet1\"",
|
||||
List.of(internalTable(
|
||||
List.of("public_col", "secret"),
|
||||
List.of("public_col"))));
|
||||
|
||||
Assert.assertEquals(
|
||||
"SELECT SUM(public_col) OVER (ORDER BY public_col ROWS BETWEEN 1 PRECEDING "
|
||||
+ "AND CURRENT ROW) AS total FROM tb_excel_budget",
|
||||
resolved.getExecutableSql());
|
||||
}
|
||||
|
||||
/**
|
||||
* 验证 HAVING 中包装的合法投影别名不会被误判为物理字段。
|
||||
*/
|
||||
@Test
|
||||
public void resolveInternalMysqlShouldAllowOutputAliasInHavingExpression() {
|
||||
SqlSupportUtils.ResolvedSql resolved = SqlSupportUtils.resolveInternalMysql(
|
||||
"SELECT public_col AS visible FROM \"Sheet1\" "
|
||||
+ "HAVING visible IS NOT NULL",
|
||||
List.of(internalTable(
|
||||
List.of("public_col", "secret"),
|
||||
List.of("public_col"))));
|
||||
|
||||
Assert.assertEquals(
|
||||
"SELECT public_col AS visible FROM tb_excel_budget "
|
||||
+ "HAVING visible IS NOT NULL",
|
||||
resolved.getExecutableSql());
|
||||
}
|
||||
|
||||
/**
|
||||
* 验证 ORDER BY 包装表达式中的合法投影别名可以使用。
|
||||
*/
|
||||
@Test
|
||||
public void resolveInternalMysqlShouldAllowOutputAliasInWrappedOrderBy() {
|
||||
SqlSupportUtils.ResolvedSql resolved = SqlSupportUtils.resolveInternalMysql(
|
||||
"SELECT public_col AS visible FROM \"Sheet1\" "
|
||||
+ "ORDER BY COALESCE(visible, '')",
|
||||
List.of(internalTable(
|
||||
List.of("public_col", "secret"),
|
||||
List.of("public_col"))));
|
||||
|
||||
Assert.assertEquals(
|
||||
"SELECT public_col AS visible FROM tb_excel_budget "
|
||||
+ "ORDER BY COALESCE(visible, '')",
|
||||
resolved.getExecutableSql());
|
||||
}
|
||||
|
||||
/**
|
||||
* 验证包装排序表达式不能放行隐藏物理字段。
|
||||
*/
|
||||
@Test(expected = BusinessException.class)
|
||||
public void resolveInternalMysqlShouldRejectHiddenColumnInWrappedOrderBy() {
|
||||
SqlSupportUtils.resolveInternalMysql(
|
||||
"SELECT public_col FROM \"Sheet1\" ORDER BY COALESCE(secret, '')",
|
||||
List.of(internalTable(
|
||||
List.of("public_col", "secret"),
|
||||
List.of("public_col"))));
|
||||
}
|
||||
|
||||
/**
|
||||
* 验证 IS TRUE 条件中的隐藏字段仍受字段策略约束。
|
||||
*/
|
||||
@Test(expected = BusinessException.class)
|
||||
public void resolveInternalMysqlShouldRejectHiddenColumnInIsBoolean() {
|
||||
SqlSupportUtils.resolveInternalMysql(
|
||||
"SELECT public_col FROM \"Sheet1\" WHERE secret IS TRUE",
|
||||
List.of(internalTable(
|
||||
List.of("public_col", "secret"),
|
||||
List.of("public_col"))));
|
||||
}
|
||||
|
||||
/**
|
||||
* 验证 QUALIFY 条件中的隐藏字段仍受字段策略约束。
|
||||
*/
|
||||
@Test(expected = BusinessException.class)
|
||||
public void resolveInternalMysqlShouldRejectHiddenColumnInQualify() {
|
||||
SqlSupportUtils.resolveInternalMysql(
|
||||
"SELECT public_col FROM \"Sheet1\" QUALIFY secret IS NOT NULL",
|
||||
List.of(internalTable(
|
||||
List.of("public_col", "secret"),
|
||||
List.of("public_col"))));
|
||||
}
|
||||
|
||||
/**
|
||||
* 验证隐藏字段不能通过同名输出别名绕过字段白名单。
|
||||
*/
|
||||
@Test(expected = BusinessException.class)
|
||||
public void resolveInternalMysqlShouldRejectHiddenColumnWithSameAlias() {
|
||||
SqlSupportUtils.resolveInternalMysql(
|
||||
"SELECT secret AS secret FROM \"Sheet1\"",
|
||||
List.of(internalTable(
|
||||
List.of("public_col", "secret"),
|
||||
List.of("public_col"))));
|
||||
}
|
||||
|
||||
/**
|
||||
* 验证存在未开放字段时禁止星号扩展。
|
||||
*/
|
||||
@Test(expected = BusinessException.class)
|
||||
public void resolveInternalMysqlShouldRejectStarWithHiddenColumns() {
|
||||
SqlSupportUtils.resolveInternalMysql(
|
||||
"SELECT * FROM Sheet1",
|
||||
List.of(internalTable(
|
||||
List.of("public_col", "secret"),
|
||||
List.of("public_col"))));
|
||||
}
|
||||
|
||||
/**
|
||||
* 验证括号包裹的 JOIN 仍会将星号绑定到底层物理表。
|
||||
*/
|
||||
@Test(expected = BusinessException.class)
|
||||
public void resolveInternalMysqlShouldRejectStarOverParenthesizedJoinWithHiddenColumn() {
|
||||
SqlSupportUtils.resolveInternalMysql(
|
||||
"SELECT * FROM (Sheet1 CROSS JOIN Sheet2)",
|
||||
List.of(
|
||||
internalTable(
|
||||
List.of("public_col", "secret"),
|
||||
List.of("public_col")),
|
||||
new SqlSupportUtils.ManagedTable(
|
||||
null,
|
||||
"Sheet2",
|
||||
"tb_excel_public",
|
||||
List.of("other_col"),
|
||||
List.of("other_col"))));
|
||||
}
|
||||
|
||||
/**
|
||||
* 验证聚合函数中的星号不代表读取全部物理字段。
|
||||
*/
|
||||
@Test
|
||||
public void resolveInternalMysqlShouldAllowCountStarWithHiddenColumns() {
|
||||
SqlSupportUtils.ResolvedSql resolved = SqlSupportUtils.resolveInternalMysql(
|
||||
"SELECT COUNT(*) FROM \"Sheet1\"",
|
||||
List.of(internalTable(
|
||||
List.of("public_col", "secret"),
|
||||
List.of("public_col"))));
|
||||
|
||||
Assert.assertEquals(
|
||||
"SELECT COUNT(*) FROM tb_excel_budget",
|
||||
resolved.getExecutableSql());
|
||||
}
|
||||
|
||||
/**
|
||||
* 验证表别名和输出别名不会被误判为物理字段。
|
||||
*/
|
||||
@Test
|
||||
public void resolveInternalMysqlShouldAllowSafeAliases() {
|
||||
SqlSupportUtils.ResolvedSql resolved = SqlSupportUtils.resolveInternalMysql(
|
||||
"SELECT s.public_col AS display_col FROM \"Sheet1\" s ORDER BY display_col",
|
||||
List.of(internalTable(
|
||||
List.of("public_col"),
|
||||
List.of("public_col"))));
|
||||
|
||||
Assert.assertEquals(
|
||||
"SELECT s.public_col AS display_col FROM tb_excel_budget s "
|
||||
+ "ORDER BY display_col",
|
||||
resolved.getExecutableSql());
|
||||
}
|
||||
|
||||
/**
|
||||
* 验证内层子查询的同名字段不会污染外层合法输出别名。
|
||||
*/
|
||||
@Test
|
||||
public void resolveInternalMysqlShouldScopeOutputAliasToCurrentQueryBlock() {
|
||||
SqlSupportUtils.ResolvedSql resolved = SqlSupportUtils.resolveInternalMysql(
|
||||
"SELECT public_col AS secret FROM \"Sheet1\" "
|
||||
+ "WHERE EXISTS (SELECT 1 FROM \"Sheet2\") ORDER BY secret",
|
||||
List.of(
|
||||
internalTable(
|
||||
List.of("public_col"),
|
||||
List.of("public_col")),
|
||||
new SqlSupportUtils.ManagedTable(
|
||||
null,
|
||||
"Sheet2",
|
||||
"tb_excel_private",
|
||||
List.of("secret"),
|
||||
List.of())));
|
||||
|
||||
Assert.assertEquals(
|
||||
"SELECT public_col AS secret FROM tb_excel_budget WHERE EXISTS "
|
||||
+ "(SELECT 1 FROM tb_excel_private) ORDER BY secret",
|
||||
resolved.getExecutableSql());
|
||||
}
|
||||
|
||||
/**
|
||||
* 验证内层子查询的物理表不会参与外层未限定字段授权判断。
|
||||
*/
|
||||
@Test
|
||||
public void resolveInternalMysqlShouldScopeUnqualifiedColumnToCurrentQueryBlock() {
|
||||
SqlSupportUtils.ResolvedSql resolved = SqlSupportUtils.resolveInternalMysql(
|
||||
"SELECT public_col FROM \"Sheet1\" "
|
||||
+ "WHERE EXISTS (SELECT 1 FROM \"Sheet2\")",
|
||||
List.of(
|
||||
internalTable(
|
||||
List.of("public_col"),
|
||||
List.of("public_col")),
|
||||
new SqlSupportUtils.ManagedTable(
|
||||
null,
|
||||
"Sheet2",
|
||||
"tb_excel_private",
|
||||
List.of("secret"),
|
||||
List.of())));
|
||||
|
||||
Assert.assertEquals(
|
||||
"SELECT public_col FROM tb_excel_budget WHERE EXISTS "
|
||||
+ "(SELECT 1 FROM tb_excel_private)",
|
||||
resolved.getExecutableSql());
|
||||
}
|
||||
|
||||
/**
|
||||
* 验证派生表输出别名可以在外层查询中使用。
|
||||
*/
|
||||
@Test
|
||||
public void resolveInternalMysqlShouldAllowDerivedTableOutputAlias() {
|
||||
SqlSupportUtils.ResolvedSql resolved = SqlSupportUtils.resolveInternalMysql(
|
||||
"SELECT d.order_id FROM (SELECT public_col AS order_id FROM \"Sheet1\") d",
|
||||
List.of(internalTable(
|
||||
List.of("public_col", "secret"),
|
||||
List.of("public_col"))));
|
||||
|
||||
Assert.assertEquals(
|
||||
"SELECT d.order_id FROM (SELECT public_col AS order_id "
|
||||
+ "FROM tb_excel_budget) d",
|
||||
resolved.getExecutableSql());
|
||||
}
|
||||
|
||||
/**
|
||||
* 验证 CTE 输出别名可以在主查询中使用。
|
||||
*/
|
||||
@Test
|
||||
public void resolveInternalMysqlShouldAllowCteOutputAlias() {
|
||||
SqlSupportUtils.ResolvedSql resolved = SqlSupportUtils.resolveInternalMysql(
|
||||
"WITH x AS (SELECT public_col AS order_id FROM \"Sheet1\") SELECT order_id FROM x",
|
||||
List.of(internalTable(
|
||||
List.of("public_col", "secret"),
|
||||
List.of("public_col"))));
|
||||
|
||||
Assert.assertEquals(
|
||||
"WITH x AS (SELECT public_col AS order_id FROM tb_excel_budget) "
|
||||
+ "SELECT order_id FROM x",
|
||||
resolved.getExecutableSql());
|
||||
}
|
||||
|
||||
/**
|
||||
* 验证外层星号只展开 CTE 已校验的输出字段。
|
||||
*/
|
||||
@Test
|
||||
public void resolveInternalMysqlShouldAllowStarOverCteOutput() {
|
||||
SqlSupportUtils.ResolvedSql resolved = SqlSupportUtils.resolveInternalMysql(
|
||||
"WITH x AS (SELECT public_col FROM \"Sheet1\") SELECT * FROM x",
|
||||
List.of(internalTable(
|
||||
List.of("public_col", "secret"),
|
||||
List.of("public_col"))));
|
||||
|
||||
Assert.assertEquals(
|
||||
"WITH x AS (SELECT public_col FROM tb_excel_budget) SELECT * FROM x",
|
||||
resolved.getExecutableSql());
|
||||
}
|
||||
|
||||
/**
|
||||
* 验证带 Schema 的物理表不能伪装成同名 CTE 绕过表白名单。
|
||||
*/
|
||||
@Test(expected = BusinessException.class)
|
||||
public void resolveInternalMysqlShouldRejectSchemaQualifiedTableShadowingCte() {
|
||||
SqlSupportUtils.resolveInternalMysql(
|
||||
"WITH x AS (SELECT public_col AS secret FROM \"Sheet1\") "
|
||||
+ "SELECT x.secret FROM private_db.x",
|
||||
List.of(internalTable(
|
||||
List.of("public_col", "secret"),
|
||||
List.of("public_col"))));
|
||||
}
|
||||
|
||||
/**
|
||||
* 验证后置 CTE 可以引用同一 WITH 列表中的前置 CTE。
|
||||
*/
|
||||
@Test
|
||||
public void resolveInternalMysqlShouldAllowChainedCtes() {
|
||||
SqlSupportUtils.ResolvedSql resolved = SqlSupportUtils.resolveInternalMysql(
|
||||
"WITH x AS (SELECT public_col AS a FROM \"Sheet1\"), "
|
||||
+ "y AS (SELECT a AS b FROM x) SELECT b FROM y",
|
||||
List.of(internalTable(
|
||||
List.of("public_col", "secret"),
|
||||
List.of("public_col"))));
|
||||
|
||||
Assert.assertEquals(
|
||||
"WITH x AS (SELECT public_col AS a FROM tb_excel_budget), "
|
||||
+ "y AS (SELECT a AS b FROM x) SELECT b FROM y",
|
||||
resolved.getExecutableSql());
|
||||
}
|
||||
|
||||
/**
|
||||
* 验证 CTE 显式列清单可以作为主查询输出字段使用。
|
||||
*/
|
||||
@Test
|
||||
public void resolveInternalMysqlShouldAllowExplicitCteColumnList() {
|
||||
SqlSupportUtils.ResolvedSql resolved = SqlSupportUtils.resolveInternalMysql(
|
||||
"WITH x(order_id) AS (SELECT public_col FROM \"Sheet1\") SELECT order_id FROM x",
|
||||
List.of(internalTable(
|
||||
List.of("public_col", "secret"),
|
||||
List.of("public_col"))));
|
||||
|
||||
Assert.assertEquals(
|
||||
"WITH x (order_id) AS (SELECT public_col FROM tb_excel_budget) "
|
||||
+ "SELECT order_id FROM x",
|
||||
resolved.getExecutableSql());
|
||||
}
|
||||
|
||||
/**
|
||||
* 验证派生表别名不能洗白内部不可查询字段。
|
||||
*/
|
||||
@Test(expected = BusinessException.class)
|
||||
public void resolveInternalMysqlShouldRejectHiddenColumnInsideDerivedTable() {
|
||||
SqlSupportUtils.resolveInternalMysql(
|
||||
"SELECT d.order_id FROM (SELECT secret AS order_id FROM \"Sheet1\") d",
|
||||
List.of(internalTable(
|
||||
List.of("public_col", "secret"),
|
||||
List.of("public_col"))));
|
||||
}
|
||||
|
||||
/**
|
||||
* 验证 CTE 输出别名不能洗白内部不可查询字段。
|
||||
*/
|
||||
@Test(expected = BusinessException.class)
|
||||
public void resolveInternalMysqlShouldRejectHiddenColumnInsideCte() {
|
||||
SqlSupportUtils.resolveInternalMysql(
|
||||
"WITH x AS (SELECT secret AS order_id FROM \"Sheet1\") SELECT order_id FROM x",
|
||||
List.of(internalTable(
|
||||
List.of("public_col", "secret"),
|
||||
List.of("public_col"))));
|
||||
}
|
||||
|
||||
/**
|
||||
* 验证外层派生表别名不会污染内层同名物理表别名。
|
||||
*/
|
||||
@Test(expected = BusinessException.class)
|
||||
public void resolveInternalMysqlShouldNotLeakDerivedAliasAcrossNestedScope() {
|
||||
SqlSupportUtils.resolveInternalMysql(
|
||||
"SELECT d.secret FROM (SELECT public_col AS secret FROM \"Sheet1\") d "
|
||||
+ "WHERE EXISTS (SELECT 1 FROM \"Sheet1\" d WHERE d.secret = 1)",
|
||||
List.of(internalTable(
|
||||
List.of("public_col", "secret"),
|
||||
List.of("public_col"))));
|
||||
}
|
||||
|
||||
/**
|
||||
* 验证相关子查询可以引用外层派生关系的已校验输出字段。
|
||||
*/
|
||||
@Test
|
||||
public void resolveInternalMysqlShouldAllowCorrelatedOuterDerivedColumn() {
|
||||
SqlSupportUtils.ResolvedSql resolved = SqlSupportUtils.resolveInternalMysql(
|
||||
"SELECT d.order_id FROM (SELECT public_col AS order_id FROM \"Sheet1\") d "
|
||||
+ "WHERE EXISTS (SELECT 1 WHERE d.order_id IS NOT NULL)",
|
||||
List.of(internalTable(
|
||||
List.of("public_col", "secret"),
|
||||
List.of("public_col"))));
|
||||
|
||||
Assert.assertEquals(
|
||||
"SELECT d.order_id FROM (SELECT public_col AS order_id "
|
||||
+ "FROM tb_excel_budget) d WHERE EXISTS "
|
||||
+ "(SELECT 1 WHERE d.order_id IS NOT NULL)",
|
||||
resolved.getExecutableSql());
|
||||
}
|
||||
|
||||
/**
|
||||
* 验证内层物理表别名会遮蔽外层派生关系,不能借外层输出绕过字段策略。
|
||||
*/
|
||||
@Test(expected = BusinessException.class)
|
||||
public void resolveInternalMysqlShouldRejectCorrelatedDerivedAliasShadowedByInnerPhysicalAlias() {
|
||||
SqlSupportUtils.resolveInternalMysql(
|
||||
"SELECT d.order_id FROM (SELECT public_col AS order_id FROM \"Sheet1\") d "
|
||||
+ "WHERE EXISTS (SELECT 1 FROM \"Sheet1\" d "
|
||||
+ "WHERE d.order_id IS NOT NULL)",
|
||||
List.of(internalTable(
|
||||
List.of("public_col", "secret"),
|
||||
List.of("public_col"))));
|
||||
}
|
||||
|
||||
/**
|
||||
* 验证不同查询块的同名物理别名不会覆盖彼此的字段授权范围。
|
||||
*/
|
||||
@Test(expected = BusinessException.class)
|
||||
public void resolveInternalMysqlShouldIsolatePhysicalAliasesAcrossScopes() {
|
||||
SqlSupportUtils.resolveInternalMysql(
|
||||
"SELECT x.secret FROM \"Sheet1\" x WHERE EXISTS "
|
||||
+ "(SELECT 1 FROM \"Sheet2\" x WHERE x.secret IS NOT NULL)",
|
||||
List.of(
|
||||
new SqlSupportUtils.ManagedTable(
|
||||
null,
|
||||
"Sheet1",
|
||||
"tb_excel_budget",
|
||||
List.of("public_col", "secret"),
|
||||
List.of("public_col")),
|
||||
new SqlSupportUtils.ManagedTable(
|
||||
null,
|
||||
"Sheet2",
|
||||
"tb_excel_public",
|
||||
List.of("secret"),
|
||||
List.of("secret"))));
|
||||
}
|
||||
|
||||
/**
|
||||
* 验证外层输出别名不能污染内层作用域并放行隐藏字段。
|
||||
*/
|
||||
@Test(expected = BusinessException.class)
|
||||
public void resolveInternalMysqlShouldRejectNestedHiddenColumnShadowedByOuterAlias() {
|
||||
SqlSupportUtils.resolveInternalMysql(
|
||||
"SELECT public_col AS secret FROM Sheet1 "
|
||||
+ "WHERE public_col = (SELECT public_col FROM Sheet1 "
|
||||
+ "ORDER BY secret LIMIT 1)",
|
||||
List.of(internalTable(
|
||||
List.of("public_col", "secret"),
|
||||
List.of("public_col"))));
|
||||
}
|
||||
|
||||
/**
|
||||
* 验证内部 SQL 拒绝具有等待或资源消耗风险的函数。
|
||||
*/
|
||||
@Test(expected = BusinessException.class)
|
||||
public void resolveInternalMysqlShouldRejectDangerousFunction() {
|
||||
SqlSupportUtils.resolveInternalMysql(
|
||||
"SELECT SLEEP(10) FROM Sheet1",
|
||||
List.of(internalTable(List.of("public_col"), List.of("public_col"))));
|
||||
}
|
||||
|
||||
/**
|
||||
* 创建内部动态表测试定义。
|
||||
*
|
||||
* @param knownColumns 已纳管字段
|
||||
* @param queryableColumns 可查询字段
|
||||
* @return 测试表定义
|
||||
*/
|
||||
private SqlSupportUtils.ManagedTable internalTable(
|
||||
List<String> knownColumns,
|
||||
List<String> queryableColumns) {
|
||||
return new SqlSupportUtils.ManagedTable(
|
||||
null,
|
||||
"Sheet1",
|
||||
"tb_excel_budget",
|
||||
knownColumns,
|
||||
queryableColumns);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,152 @@
|
||||
-- XL01 WP1: 数据中枢 Federation SQL 数据契约。
|
||||
-- 历史数据中枢表继续保留;本迁移只做可回填的增量调整。
|
||||
|
||||
ALTER TABLE `tb_datacenter_source`
|
||||
ADD COLUMN `adapter_id` varchar(64) NULL COMMENT 'Federation Adapter 标识' AFTER `source_type`,
|
||||
ADD COLUMN `adapter_options_json` text NULL COMMENT '不含凭据的 Adapter 选项' AFTER `adapter_id`,
|
||||
ADD COLUMN `definition_revision` bigint UNSIGNED NOT NULL DEFAULT 0 COMMENT '数据源 Definition 版本' AFTER `adapter_options_json`,
|
||||
ADD COLUMN `definition_checksum` char(64) NULL COMMENT 'Definition SHA-256 校验和' AFTER `definition_revision`,
|
||||
ADD COLUMN `scope_revision` bigint UNSIGNED NOT NULL DEFAULT 1 COMMENT '纳管范围版本' AFTER `definition_checksum`,
|
||||
ADD COLUMN `compatibility_status` varchar(32) NULL COMMENT 'Adapter 兼容状态' AFTER `scope_revision`,
|
||||
ADD COLUMN `database_product_name` varchar(128) NULL COMMENT '数据库产品名称' AFTER `compatibility_status`,
|
||||
ADD COLUMN `database_product_version` varchar(128) NULL COMMENT '数据库产品版本' AFTER `database_product_name`,
|
||||
ADD COLUMN `driver_name` varchar(128) NULL COMMENT 'JDBC Driver 名称' AFTER `database_product_version`,
|
||||
ADD COLUMN `driver_version` varchar(128) NULL COMMENT 'JDBC Driver 版本' AFTER `driver_name`,
|
||||
ADD COLUMN `metadata_refresh_status` varchar(32) NULL COMMENT '元数据刷新状态' AFTER `driver_version`,
|
||||
ADD COLUMN `metadata_refreshed_at` datetime NULL COMMENT '元数据最近刷新时间' AFTER `metadata_refresh_status`;
|
||||
|
||||
UPDATE `tb_datacenter_source`
|
||||
SET `adapter_id` = CASE
|
||||
WHEN `source_type` IN ('MYSQL', 'POSTGRESQL') THEN 'jdbc'
|
||||
ELSE `adapter_id`
|
||||
END,
|
||||
`definition_revision` = CASE
|
||||
WHEN `source_type` = 'PROJECT_MYSQL' THEN 1
|
||||
ELSE `definition_revision`
|
||||
END,
|
||||
`status` = CASE
|
||||
WHEN `source_type` = 'PROJECT_MYSQL' THEN 1
|
||||
ELSE `status`
|
||||
END,
|
||||
`metadata_refresh_status` = COALESCE(`metadata_refresh_status`, 'PENDING');
|
||||
|
||||
ALTER TABLE `tb_datacenter_source`
|
||||
ADD UNIQUE KEY `uk_datacenter_source_tenant_code` (`tenant_id`, `source_code`),
|
||||
ADD KEY `idx_datacenter_source_runtime` (`status`, `definition_revision`);
|
||||
|
||||
ALTER TABLE `tb_datacenter_catalog`
|
||||
ADD COLUMN `logical_schema_name` varchar(128) NULL COMMENT 'Calcite 逻辑 Schema' AFTER `catalog_type`,
|
||||
ADD COLUMN `physical_catalog_name` varchar(128) NULL COMMENT 'JDBC 物理 Catalog' AFTER `logical_schema_name`,
|
||||
ADD COLUMN `physical_schema_name` varchar(128) NULL COMMENT 'JDBC 物理 Schema' AFTER `physical_catalog_name`,
|
||||
ADD COLUMN `namespace_key` char(64) NULL COMMENT '物理命名空间稳定摘要' AFTER `physical_schema_name`,
|
||||
ADD COLUMN `metadata_status` varchar(32) NOT NULL DEFAULT 'ACTIVE' COMMENT '元数据状态' AFTER `namespace_key`,
|
||||
ADD COLUMN `last_seen_at` datetime NULL COMMENT '最近发现时间' AFTER `metadata_status`;
|
||||
|
||||
UPDATE `tb_datacenter_catalog`
|
||||
SET `logical_schema_name` = `catalog_name`,
|
||||
`physical_catalog_name` = `catalog_name`,
|
||||
`namespace_key` = SHA2(CONCAT(
|
||||
'C', CHAR_LENGTH(`catalog_name`), ':', `catalog_name`,
|
||||
'S', '-1:'
|
||||
), 256),
|
||||
`last_seen_at` = COALESCE(`modified`, `created`)
|
||||
WHERE `logical_schema_name` IS NULL OR `namespace_key` IS NULL;
|
||||
|
||||
ALTER TABLE `tb_datacenter_catalog`
|
||||
MODIFY COLUMN `logical_schema_name` varchar(128) NOT NULL COMMENT 'Calcite 逻辑 Schema',
|
||||
MODIFY COLUMN `namespace_key` char(64) NOT NULL COMMENT '物理命名空间稳定摘要',
|
||||
ADD UNIQUE KEY `uk_datacenter_catalog_logical_schema` (`source_id`, `logical_schema_name`),
|
||||
ADD UNIQUE KEY `uk_datacenter_catalog_namespace` (`source_id`, `namespace_key`);
|
||||
|
||||
ALTER TABLE `tb_datacenter_table`
|
||||
MODIFY COLUMN `table_name` varchar(128) NOT NULL COMMENT '数据表名',
|
||||
ADD COLUMN `physical_identity_key` char(64) NULL COMMENT '物理对象稳定摘要' AFTER `actual_table`,
|
||||
ADD COLUMN `metadata_fingerprint` char(64) NULL COMMENT '表结构摘要' AFTER `physical_identity_key`,
|
||||
ADD COLUMN `metadata_status` varchar(32) NOT NULL DEFAULT 'ACTIVE' COMMENT '元数据状态' AFTER `metadata_fingerprint`,
|
||||
ADD COLUMN `last_seen_at` datetime NULL COMMENT '最近发现时间' AFTER `metadata_status`,
|
||||
ADD COLUMN `queryable` tinyint NOT NULL DEFAULT 1 COMMENT '是否允许统一只读查询' AFTER `last_seen_at`,
|
||||
ADD COLUMN `time_semantics` varchar(32) NOT NULL DEFAULT 'NONE' COMMENT '时间语义' AFTER `queryable`,
|
||||
ADD COLUMN `default_time_field_id` bigint UNSIGNED NULL COMMENT '默认时间字段 ID' AFTER `time_semantics`,
|
||||
ADD COLUMN `metadata_revision` bigint UNSIGNED NOT NULL DEFAULT 1 COMMENT '表元数据版本' AFTER `default_time_field_id`;
|
||||
|
||||
UPDATE `tb_datacenter_table`
|
||||
SET `actual_table` = COALESCE(NULLIF(`actual_table`, ''), `table_name`),
|
||||
`physical_identity_key` = SHA2(CONCAT(
|
||||
'S', CHAR_LENGTH(CAST(COALESCE(`source_id`, 0) AS CHAR)), ':', COALESCE(`source_id`, 0),
|
||||
'C', CHAR_LENGTH(CAST(COALESCE(`catalog_id`, 0) AS CHAR)), ':', COALESCE(`catalog_id`, 0),
|
||||
'T', CHAR_LENGTH(COALESCE(NULLIF(`actual_table`, ''), `table_name`)), ':', COALESCE(NULLIF(`actual_table`, ''), `table_name`),
|
||||
'K', CHAR_LENGTH(`table_kind`), ':', `table_kind`
|
||||
), 256),
|
||||
`metadata_fingerprint` = SHA2(CONCAT(
|
||||
CHAR_LENGTH(`table_name`), ':', `table_name`,
|
||||
CHAR_LENGTH(`table_kind`), ':', `table_kind`
|
||||
), 256),
|
||||
`last_seen_at` = COALESCE(`modified`, `created`)
|
||||
WHERE `physical_identity_key` IS NULL OR `metadata_fingerprint` IS NULL;
|
||||
|
||||
ALTER TABLE `tb_datacenter_table`
|
||||
MODIFY COLUMN `physical_identity_key` char(64) NOT NULL COMMENT '物理对象稳定摘要',
|
||||
ADD UNIQUE KEY `uk_datacenter_table_physical_identity` (`physical_identity_key`),
|
||||
ADD KEY `idx_datacenter_table_metadata` (`source_id`, `catalog_id`, `metadata_status`);
|
||||
|
||||
ALTER TABLE `tb_datacenter_table_field`
|
||||
MODIFY COLUMN `field_name` varchar(128) NOT NULL COMMENT '字段名称',
|
||||
ADD COLUMN `ordinal_position` int NULL COMMENT 'JDBC 字段顺序' AFTER `source_column_name`,
|
||||
ADD COLUMN `jdbc_type_code` int NULL COMMENT 'java.sql.Types 数值' AFTER `jdbc_type`,
|
||||
ADD COLUMN `native_type_name` varchar(128) NULL COMMENT '数据库原生类型名' AFTER `jdbc_type_code`,
|
||||
ADD COLUMN `metadata_fingerprint` char(64) NULL COMMENT '字段元数据摘要' AFTER `scale`,
|
||||
ADD COLUMN `metadata_status` varchar(32) NOT NULL DEFAULT 'ACTIVE' COMMENT '元数据状态' AFTER `metadata_fingerprint`,
|
||||
ADD COLUMN `last_seen_at` datetime NULL COMMENT '最近发现时间' AFTER `metadata_status`,
|
||||
ADD COLUMN `sensitivity_level` varchar(32) NOT NULL DEFAULT 'PUBLIC' COMMENT '敏感级别' AFTER `last_seen_at`,
|
||||
ADD COLUMN `masking_strategy` varchar(32) NULL COMMENT '脱敏策略' AFTER `sensitivity_level`;
|
||||
|
||||
UPDATE `tb_datacenter_table_field`
|
||||
SET `source_column_name` = COALESCE(NULLIF(`source_column_name`, ''), `field_name`),
|
||||
`native_type_name` = COALESCE(`native_type_name`, `jdbc_type`),
|
||||
`metadata_fingerprint` = SHA2(CONCAT(
|
||||
CHAR_LENGTH(`field_name`), ':', `field_name`,
|
||||
CHAR_LENGTH(COALESCE(`jdbc_type`, '')), ':', COALESCE(`jdbc_type`, ''),
|
||||
'P', COALESCE(`precision`, -1), ':S', COALESCE(`scale`, -1), ':R', `required`
|
||||
), 256),
|
||||
`last_seen_at` = COALESCE(`modified`, `created`),
|
||||
`writable` = CASE WHEN `writable` IS NULL THEN 0 ELSE `writable` END
|
||||
WHERE `metadata_fingerprint` IS NULL OR `source_column_name` IS NULL;
|
||||
|
||||
ALTER TABLE `tb_datacenter_table_field`
|
||||
MODIFY COLUMN `source_column_name` varchar(128) NOT NULL COMMENT '源字段名',
|
||||
MODIFY COLUMN `metadata_fingerprint` char(64) NOT NULL COMMENT '字段元数据摘要',
|
||||
ADD UNIQUE KEY `uk_datacenter_field_source_column` (`table_id`, `source_column_name`),
|
||||
ADD KEY `idx_datacenter_field_metadata` (`table_id`, `metadata_status`, `ordinal_position`);
|
||||
|
||||
CREATE TABLE `tb_datacenter_query_audit`
|
||||
(
|
||||
`id` bigint UNSIGNED NOT NULL COMMENT '主键',
|
||||
`query_id` varchar(64) NOT NULL COMMENT '查询标识',
|
||||
`tenant_id` bigint UNSIGNED NOT NULL DEFAULT 0 COMMENT '租户ID',
|
||||
`dept_id` bigint UNSIGNED NOT NULL DEFAULT 0 COMMENT '部门ID',
|
||||
`caller_type` varchar(32) NOT NULL COMMENT '调用方类型',
|
||||
`caller_id` varchar(128) NULL COMMENT '调用方标识',
|
||||
`source_id` bigint UNSIGNED NOT NULL COMMENT '数据源ID',
|
||||
`source_revision` bigint UNSIGNED NOT NULL COMMENT '数据源版本',
|
||||
`scope_revision` bigint UNSIGNED NOT NULL COMMENT '纳管范围版本',
|
||||
`referenced_tables_json` text NULL COMMENT '引用表摘要',
|
||||
`referenced_fields_json` text NULL COMMENT '引用字段摘要',
|
||||
`parameterized_sql` longtext NOT NULL COMMENT '参数化或脱敏 SQL',
|
||||
`masked_parameters_json` text NULL COMMENT '脱敏参数摘要',
|
||||
`status` varchar(32) NOT NULL COMMENT '执行状态',
|
||||
`error_code` varchar(64) NULL COMMENT '稳定错误码',
|
||||
`error_message` varchar(500) NULL COMMENT '有界错误说明',
|
||||
`started_at` datetime NOT NULL COMMENT '开始时间',
|
||||
`finished_at` datetime NULL COMMENT '结束时间',
|
||||
`duration_ms` bigint NULL COMMENT '总耗时毫秒',
|
||||
`database_duration_ms` bigint NULL COMMENT '数据库耗时毫秒',
|
||||
`returned_rows` bigint NOT NULL DEFAULT 0 COMMENT '返回行数',
|
||||
`truncated` tinyint NOT NULL DEFAULT 0 COMMENT '是否截断',
|
||||
`created` datetime NOT NULL COMMENT '创建时间',
|
||||
PRIMARY KEY (`id`),
|
||||
UNIQUE KEY `uk_datacenter_query_audit_query_id` (`query_id`),
|
||||
KEY `idx_datacenter_query_audit_tenant_created` (`tenant_id`, `created`),
|
||||
KEY `idx_datacenter_query_audit_source_created` (`source_id`, `created`),
|
||||
KEY `idx_datacenter_query_audit_caller_created` (`caller_type`, `caller_id`, `created`),
|
||||
KEY `idx_datacenter_query_audit_status_created` (`status`, `created`)
|
||||
) ENGINE = InnoDB DEFAULT CHARSET = utf8mb4 COLLATE = utf8mb4_0900_ai_ci COMMENT = '数据中枢统一查询审计';
|
||||
@@ -0,0 +1,19 @@
|
||||
ALTER TABLE `tb_datacenter_catalog`
|
||||
MODIFY COLUMN `catalog_name` varchar(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL COMMENT '目录名称',
|
||||
MODIFY COLUMN `logical_schema_name` varchar(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL COMMENT 'Calcite逻辑Schema',
|
||||
MODIFY COLUMN `physical_catalog_name` varchar(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NULL COMMENT '物理Catalog',
|
||||
MODIFY COLUMN `physical_schema_name` varchar(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NULL COMMENT '物理Schema';
|
||||
|
||||
ALTER TABLE `tb_datacenter_table`
|
||||
MODIFY COLUMN `table_name` varchar(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL COMMENT '数据表名',
|
||||
MODIFY COLUMN `actual_table` varchar(200) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NULL COMMENT '物理表名';
|
||||
|
||||
ALTER TABLE `tb_datacenter_table_field`
|
||||
MODIFY COLUMN `field_name` varchar(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL COMMENT '字段名称',
|
||||
MODIFY COLUMN `source_column_name` varchar(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL COMMENT '源字段名';
|
||||
|
||||
UPDATE `tb_datacenter_table_field` field_meta
|
||||
JOIN `tb_datacenter_table` table_meta ON table_meta.`id` = field_meta.`table_id`
|
||||
JOIN `tb_datacenter_source` source_meta ON source_meta.`id` = table_meta.`source_id`
|
||||
SET field_meta.`writable` = 0
|
||||
WHERE source_meta.`source_type` IN ('MYSQL', 'POSTGRESQL');
|
||||
@@ -0,0 +1,4 @@
|
||||
ALTER TABLE `tb_datacenter_query_audit`
|
||||
ADD COLUMN `executor_account_id` bigint UNSIGNED NULL COMMENT '执行账号ID' AFTER `dept_id`,
|
||||
ADD KEY `idx_datacenter_query_audit_executor_created`
|
||||
(`tenant_id`, `executor_account_id`, `created`);
|
||||
@@ -90,6 +90,7 @@ function createSourceOnlyDatasetRef(
|
||||
source: ManagedDatasetSourceOption,
|
||||
): DatasetRefPayload {
|
||||
return {
|
||||
tenantId: source.tables[0]?.datasetRef.tenantId ?? null,
|
||||
sourceId: source.sourceId,
|
||||
catalogId: null,
|
||||
catalogName: '',
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import { api } from '#/api/request';
|
||||
|
||||
export interface DatasetRefPayload {
|
||||
tenantId: null | number | string;
|
||||
sourceId: null | number | string;
|
||||
catalogId?: null | number | string;
|
||||
catalogName?: string;
|
||||
@@ -110,6 +111,7 @@ export async function loadManagedDatasetOptions(): Promise<
|
||||
catalogName: catalog.catalogName,
|
||||
tableName: table.tableName,
|
||||
datasetRef: {
|
||||
tenantId: table.tenantId,
|
||||
sourceId: source.id,
|
||||
catalogId: catalog.id,
|
||||
catalogName: catalog.catalogName,
|
||||
|
||||
@@ -1,9 +1,10 @@
|
||||
<script setup lang="ts">
|
||||
import type { TreeNode } from './composables/use-connection-tree';
|
||||
|
||||
import { nextTick, onBeforeUnmount, onMounted, ref } from 'vue';
|
||||
import { computed, nextTick, onBeforeUnmount, onMounted, ref } from 'vue';
|
||||
|
||||
import {
|
||||
EasyFlowButton,
|
||||
ResizableHandle,
|
||||
ResizablePanel,
|
||||
ResizablePanelGroup,
|
||||
@@ -13,6 +14,7 @@ import { ElEmpty, ElMessage, ElMessageBox } from 'element-plus';
|
||||
|
||||
import ConnectionTree from './components/ConnectionTree.vue';
|
||||
import SourceFormDrawer from './components/SourceFormDrawer.vue';
|
||||
import SqlConsoleView from './components/SqlConsoleView.vue';
|
||||
import TableDetailView from './components/TableDetailView.vue';
|
||||
import TableListView from './components/TableListView.vue';
|
||||
import { useConnectionTree } from './composables/use-connection-tree';
|
||||
@@ -23,19 +25,36 @@ const sourceFormRef = ref<InstanceType<typeof SourceFormDrawer>>();
|
||||
const sourceFormVisible = ref(false);
|
||||
const workspaceRef = ref<HTMLElement>();
|
||||
const workspaceHeight = ref('100%');
|
||||
const isCompactLayout = ref(false);
|
||||
const pendingSourceActionIds = ref<Set<string>>(new Set());
|
||||
const pendingSourceIds = computed(() => pendingSourceActionIds.value);
|
||||
|
||||
const {
|
||||
sources,
|
||||
selectedSourceId,
|
||||
selectedSource,
|
||||
hasMoreSources,
|
||||
loading,
|
||||
loadingMoreSources,
|
||||
loadMoreSourcesFailed,
|
||||
saving,
|
||||
sourceLoadError,
|
||||
testing,
|
||||
activateSource,
|
||||
changeSourceState,
|
||||
importExcelSource,
|
||||
loadCandidateCatalogs,
|
||||
loadCandidateTables,
|
||||
loadBindableCatalogs,
|
||||
loadBindableTables,
|
||||
loadManagedScopeTables,
|
||||
loadSources,
|
||||
loadMoreSources,
|
||||
probeCandidate,
|
||||
probeSource,
|
||||
reconfigureSource,
|
||||
removeSource,
|
||||
saveSource,
|
||||
testConnection,
|
||||
saveDraft,
|
||||
markSourceRuntimeUnavailable,
|
||||
clearSourceRuntimeUnavailable,
|
||||
} = useDatacenterSources();
|
||||
@@ -43,15 +62,29 @@ const {
|
||||
const {
|
||||
sourceTables,
|
||||
managedTables,
|
||||
catalogs,
|
||||
schema,
|
||||
previewRows,
|
||||
jobs,
|
||||
sourceTableHasMore,
|
||||
sourceTableLoadingMore,
|
||||
sourceUnavailable,
|
||||
tableRuntimeError,
|
||||
selectedCatalogId,
|
||||
selectedTableId,
|
||||
selectedTable,
|
||||
previewLoading,
|
||||
fieldPageNumber,
|
||||
fieldHasMore,
|
||||
fieldLoading,
|
||||
tableMutationKind,
|
||||
contextError,
|
||||
contextLoading,
|
||||
loadTableRuntime,
|
||||
loadFieldPage,
|
||||
loadMoreSourceTables,
|
||||
retrySourceContext,
|
||||
searchSourceTables,
|
||||
syncSourceContext,
|
||||
batchRegisterTables,
|
||||
batchRemoveTables,
|
||||
@@ -66,6 +99,9 @@ async function reloadAll(options?: {
|
||||
resetTable?: boolean;
|
||||
}) {
|
||||
await loadSources();
|
||||
if (selectedSource.value?.status !== 'READY') {
|
||||
sourceViewMode.value = 'tables';
|
||||
}
|
||||
if (selectedSourceId.value === null) {
|
||||
selectedCatalogId.value = null;
|
||||
selectedTableId.value = null;
|
||||
@@ -87,9 +123,31 @@ async function reloadAll(options?: {
|
||||
viewMode.value = 'list';
|
||||
}
|
||||
|
||||
async function refreshSelectedSourceContext() {
|
||||
sourceViewMode.value = 'tables';
|
||||
selectedCatalogId.value = null;
|
||||
selectedTableId.value = null;
|
||||
if (selectedSourceId.value === null) {
|
||||
selectedNodeKey.value = '';
|
||||
viewMode.value = 'empty';
|
||||
await syncSourceContext();
|
||||
return;
|
||||
}
|
||||
selectedNodeKey.value = `source-${selectedSourceId.value}`;
|
||||
viewMode.value = 'list';
|
||||
await syncSourceContext();
|
||||
}
|
||||
|
||||
async function refreshLifecycleTarget(sourceId: number | string) {
|
||||
if (String(selectedSourceId.value) !== String(sourceId)) return;
|
||||
await refreshSelectedSourceContext();
|
||||
}
|
||||
|
||||
// 视图状态:empty | list | detail
|
||||
const selectedNodeKey = ref('');
|
||||
const viewMode = ref<'detail' | 'empty' | 'list'>('empty');
|
||||
const sourceViewMode = ref<'sql' | 'tables'>('tables');
|
||||
let loadAllInFlight = 0;
|
||||
|
||||
const { treeData, parseNodeKey } = useConnectionTree(sources);
|
||||
|
||||
@@ -99,13 +157,14 @@ async function handleNodeSelect(node: TreeNode) {
|
||||
switch (node.type) {
|
||||
case 'source': {
|
||||
const { id } = parseNodeKey(node.id);
|
||||
sourceViewMode.value = 'tables';
|
||||
viewMode.value = 'list';
|
||||
if (id !== selectedSourceId.value) {
|
||||
selectedSourceId.value = id;
|
||||
selectedCatalogId.value = null;
|
||||
selectedTableId.value = null;
|
||||
await syncSourceContext();
|
||||
}
|
||||
viewMode.value = 'list';
|
||||
|
||||
break;
|
||||
}
|
||||
@@ -147,11 +206,11 @@ async function handleSaveTableDescription(
|
||||
});
|
||||
}
|
||||
|
||||
function handleSelectTable(row: any) {
|
||||
async function handleSelectTable(row: any) {
|
||||
selectedTableId.value = row.id;
|
||||
selectedNodeKey.value = `table-${row.id}`;
|
||||
viewMode.value = 'detail';
|
||||
loadTableRuntime();
|
||||
await loadTableRuntime();
|
||||
}
|
||||
|
||||
function handleBackToList() {
|
||||
@@ -168,64 +227,133 @@ function openCreate() {
|
||||
|
||||
function openEdit(node: TreeNode) {
|
||||
if (node.type !== 'source') return;
|
||||
if (node.meta?.sourceType === 'EXCEL') {
|
||||
ElMessage.info('Excel 连接通过重新导入文件更新');
|
||||
return;
|
||||
}
|
||||
if (!['DEGRADED', 'DRAFT', 'READY'].includes(node.meta?.status)) {
|
||||
ElMessage.info('当前连接状态不支持编辑');
|
||||
return;
|
||||
}
|
||||
sourceFormRef.value?.open(node.meta);
|
||||
sourceFormVisible.value = true;
|
||||
}
|
||||
|
||||
async function handleDisableSource(node: TreeNode) {
|
||||
if (node.type !== 'source' || !node.meta?.id) return;
|
||||
if (pendingSourceActionIds.value.has(String(node.meta.id))) return;
|
||||
try {
|
||||
await ElMessageBox.confirm(
|
||||
`停用“${node.label}”后将拒绝新查询。`,
|
||||
'停用连接',
|
||||
{
|
||||
type: 'warning',
|
||||
confirmButtonText: '停用',
|
||||
cancelButtonText: '取消',
|
||||
},
|
||||
);
|
||||
} catch (error) {
|
||||
if (error === 'cancel' || error === 'close') return;
|
||||
throw error;
|
||||
}
|
||||
await runSourceAction(node.meta.id, async () => {
|
||||
await changeSourceState(node.meta.id, 'disable');
|
||||
await refreshLifecycleTarget(node.meta.id);
|
||||
ElMessage.success('连接已停用');
|
||||
});
|
||||
}
|
||||
|
||||
async function handleEnableSource(node: TreeNode) {
|
||||
if (node.type !== 'source' || !node.meta?.id) return;
|
||||
await runSourceAction(node.meta.id, async () => {
|
||||
await changeSourceState(node.meta.id, 'enable');
|
||||
await refreshLifecycleTarget(node.meta.id);
|
||||
ElMessage.success('连接已启用');
|
||||
});
|
||||
}
|
||||
|
||||
async function handleRefreshMetadata(node: TreeNode) {
|
||||
if (node.type !== 'source' || !node.meta?.id) return;
|
||||
await runSourceAction(node.meta.id, async () => {
|
||||
await changeSourceState(node.meta.id, 'metadata/refresh');
|
||||
await refreshLifecycleTarget(node.meta.id);
|
||||
ElMessage.success('元数据已刷新');
|
||||
});
|
||||
}
|
||||
|
||||
async function runSourceAction(
|
||||
sourceId: number | string,
|
||||
action: () => Promise<void>,
|
||||
) {
|
||||
const key = String(sourceId);
|
||||
if (pendingSourceActionIds.value.has(key)) return;
|
||||
pendingSourceActionIds.value = new Set(pendingSourceActionIds.value).add(key);
|
||||
try {
|
||||
await action();
|
||||
} finally {
|
||||
const next = new Set(pendingSourceActionIds.value);
|
||||
next.delete(key);
|
||||
pendingSourceActionIds.value = next;
|
||||
}
|
||||
}
|
||||
|
||||
async function handleRemoveSource(node: TreeNode) {
|
||||
if (node.type !== 'source' || !node.meta?.id) return;
|
||||
await ElMessageBox.confirm(`确认删除“${node.label}”吗?`, '删除连接', {
|
||||
type: 'warning',
|
||||
confirmButtonText: '删除',
|
||||
cancelButtonText: '取消',
|
||||
confirmButtonClass: 'el-button--danger',
|
||||
});
|
||||
try {
|
||||
await ElMessageBox.confirm(`确认删除“${node.label}”吗?`, '删除连接', {
|
||||
type: 'warning',
|
||||
confirmButtonText: '删除',
|
||||
cancelButtonText: '取消',
|
||||
confirmButtonClass: 'el-button--danger',
|
||||
});
|
||||
} catch (error) {
|
||||
if (error === 'cancel' || error === 'close') return;
|
||||
throw error;
|
||||
}
|
||||
const removingCurrent = selectedSourceId.value === node.meta.id;
|
||||
await removeSource(node.meta.id);
|
||||
if (removingCurrent) {
|
||||
selectedTableId.value = null;
|
||||
await refreshSelectedSourceContext();
|
||||
}
|
||||
await reloadAll({ focus: 'source', resetTable: true });
|
||||
}
|
||||
|
||||
async function handleSaveSource(
|
||||
formData: Record<string, any>,
|
||||
excelFile?: File,
|
||||
) {
|
||||
if (formData.sourceType === 'EXCEL' && !formData.id) {
|
||||
if (!excelFile) return;
|
||||
await importExcelSource(excelFile, formData.sourceName);
|
||||
ElMessage.success('Excel 已导入');
|
||||
} else {
|
||||
await saveSource(formData);
|
||||
ElMessage.success(formData.id ? '连接已更新' : '连接已创建');
|
||||
}
|
||||
sourceFormRef.value?.close();
|
||||
sourceFormVisible.value = false;
|
||||
async function handleSourceCompleted(source?: any) {
|
||||
ElMessage.success(source ? '数据连接已激活' : 'Excel 已导入');
|
||||
await nextTick();
|
||||
await reloadAll({ focus: 'source', resetTable: true });
|
||||
}
|
||||
|
||||
async function handleTestConnection(formData: Record<string, any>) {
|
||||
const result = await testConnection(formData);
|
||||
sourceFormRef.value?.setTestResult(result);
|
||||
if (formData.id) await loadSources();
|
||||
await refreshSelectedSourceContext();
|
||||
}
|
||||
|
||||
async function loadAll() {
|
||||
loadAllInFlight += 1;
|
||||
loading.value = true;
|
||||
try {
|
||||
await reloadAll({ focus: 'source', resetTable: true });
|
||||
} catch (error) {
|
||||
console.error('数据连接加载失败', error);
|
||||
} finally {
|
||||
loading.value = false;
|
||||
loadAllInFlight = Math.max(0, loadAllInFlight - 1);
|
||||
loading.value = loadAllInFlight > 0;
|
||||
}
|
||||
}
|
||||
|
||||
async function handleLoadMoreSources() {
|
||||
try {
|
||||
await loadMoreSources();
|
||||
} catch (error) {
|
||||
console.error('更多数据连接加载失败', error);
|
||||
}
|
||||
}
|
||||
|
||||
function updateWorkspaceHeight() {
|
||||
const element = workspaceRef.value;
|
||||
if (!element) return;
|
||||
isCompactLayout.value = window.innerWidth < 900;
|
||||
const top = element.getBoundingClientRect().top;
|
||||
workspaceHeight.value = `${Math.max(window.innerHeight - top, 480)}px`;
|
||||
workspaceHeight.value = `${Math.max(
|
||||
window.innerHeight - top,
|
||||
isCompactLayout.value ? 640 : 480,
|
||||
)}px`;
|
||||
}
|
||||
|
||||
onMounted(async () => {
|
||||
@@ -248,13 +376,29 @@ onBeforeUnmount(() => {
|
||||
v-loading="loading"
|
||||
>
|
||||
<!-- 双栏主体 -->
|
||||
<ResizablePanelGroup direction="horizontal" class="workspace-body">
|
||||
<ResizablePanel :default-size="20" :min-size="16" :max-size="30">
|
||||
<ResizablePanelGroup
|
||||
:direction="isCompactLayout ? 'vertical' : 'horizontal'"
|
||||
class="workspace-body"
|
||||
>
|
||||
<ResizablePanel
|
||||
:default-size="isCompactLayout ? 34 : 20"
|
||||
:min-size="isCompactLayout ? 24 : 16"
|
||||
:max-size="isCompactLayout ? 54 : 30"
|
||||
>
|
||||
<ConnectionTree
|
||||
:has-more="hasMoreSources"
|
||||
:load-error="sourceLoadError"
|
||||
:load-more-failed="loadMoreSourcesFailed"
|
||||
:loading-more="loadingMoreSources"
|
||||
:pending-source-ids="pendingSourceIds"
|
||||
:tree-data="treeData"
|
||||
:selected-key="selectedNodeKey"
|
||||
@create="openCreate"
|
||||
@edit="openEdit"
|
||||
@disable="handleDisableSource"
|
||||
@enable="handleEnableSource"
|
||||
@metadata-refresh="handleRefreshMetadata"
|
||||
@load-more="handleLoadMoreSources"
|
||||
@refresh="loadAll"
|
||||
@remove="handleRemoveSource"
|
||||
@select="handleNodeSelect"
|
||||
@@ -263,28 +407,92 @@ onBeforeUnmount(() => {
|
||||
|
||||
<ResizableHandle />
|
||||
|
||||
<ResizablePanel :default-size="80" :min-size="44">
|
||||
<TableListView
|
||||
v-if="viewMode === 'list'"
|
||||
:source-tables="sourceTables"
|
||||
:managed-tables="managedTables"
|
||||
:save-table-description="handleSaveTableDescription"
|
||||
:source-label="selectedSource?.sourceName || ''"
|
||||
:source-unavailable="sourceUnavailable"
|
||||
@register-tables="handleBatchRegister"
|
||||
@remove-tables="handleBatchRemove"
|
||||
@select-table="handleSelectTable"
|
||||
/>
|
||||
<ResizablePanel
|
||||
:default-size="isCompactLayout ? 66 : 80"
|
||||
:min-size="isCompactLayout ? 40 : 44"
|
||||
>
|
||||
<div v-if="viewMode === 'list'" class="source-workspace">
|
||||
<div v-if="selectedSource" class="source-mode-bar">
|
||||
<div
|
||||
class="source-mode-switch"
|
||||
role="tablist"
|
||||
aria-label="数据源视图"
|
||||
>
|
||||
<EasyFlowButton
|
||||
size="sm"
|
||||
:variant="sourceViewMode === 'tables' ? 'default' : 'ghost'"
|
||||
role="tab"
|
||||
:aria-selected="sourceViewMode === 'tables'"
|
||||
@click="sourceViewMode = 'tables'"
|
||||
>
|
||||
数据表
|
||||
</EasyFlowButton>
|
||||
<EasyFlowButton
|
||||
v-if="
|
||||
['MYSQL', 'POSTGRESQL'].includes(selectedSource.sourceType) &&
|
||||
selectedSource.status === 'READY'
|
||||
"
|
||||
size="sm"
|
||||
:variant="sourceViewMode === 'sql' ? 'default' : 'ghost'"
|
||||
role="tab"
|
||||
:aria-selected="sourceViewMode === 'sql'"
|
||||
@click="sourceViewMode = 'sql'"
|
||||
>
|
||||
SQL 查询
|
||||
</EasyFlowButton>
|
||||
</div>
|
||||
<span
|
||||
class="source-runtime-status"
|
||||
:class="`is-${String(selectedSource.status || '').toLowerCase()}`"
|
||||
>
|
||||
{{ selectedSource.status || 'READY' }}
|
||||
</span>
|
||||
</div>
|
||||
<TableListView
|
||||
v-if="sourceViewMode === 'tables'"
|
||||
class="source-view-content"
|
||||
:source-tables="sourceTables"
|
||||
:has-more="sourceTableHasMore"
|
||||
:managed-tables="managedTables"
|
||||
:load-error="contextError"
|
||||
:loading="contextLoading"
|
||||
:loading-more="sourceTableLoadingMore"
|
||||
:mutation-kind="tableMutationKind"
|
||||
:save-table-description="handleSaveTableDescription"
|
||||
:source-id="String(selectedSourceId)"
|
||||
:source-label="selectedSource?.sourceName || ''"
|
||||
:source-unavailable="sourceUnavailable"
|
||||
@register-tables="handleBatchRegister"
|
||||
@load-more="loadMoreSourceTables"
|
||||
@remove-tables="handleBatchRemove"
|
||||
@retry="retrySourceContext"
|
||||
@search="searchSourceTables"
|
||||
@select-table="handleSelectTable"
|
||||
/>
|
||||
<SqlConsoleView
|
||||
v-else
|
||||
class="source-view-content"
|
||||
:source="selectedSource"
|
||||
:catalogs="catalogs"
|
||||
:tables="managedTables"
|
||||
/>
|
||||
</div>
|
||||
<TableDetailView
|
||||
v-else-if="viewMode === 'detail' && selectedTable"
|
||||
:table="selectedTable"
|
||||
:schema="schema"
|
||||
:preview-rows="previewRows"
|
||||
:jobs="jobs"
|
||||
:field-has-more="fieldHasMore"
|
||||
:field-loading="fieldLoading"
|
||||
:field-page-number="fieldPageNumber"
|
||||
:load-error="tableRuntimeError"
|
||||
:loading="previewLoading"
|
||||
:save-descriptions="saveDescriptions"
|
||||
:source-unavailable="sourceUnavailable"
|
||||
@back="handleBackToList"
|
||||
@field-page-change="loadFieldPage"
|
||||
@retry="loadTableRuntime"
|
||||
/>
|
||||
<div v-else class="empty-state">
|
||||
<ElEmpty description="从左侧选择连接或表开始浏览" />
|
||||
@@ -298,8 +506,18 @@ onBeforeUnmount(() => {
|
||||
v-model:visible="sourceFormVisible"
|
||||
:saving="saving"
|
||||
:testing="testing"
|
||||
@save="handleSaveSource"
|
||||
@test="handleTestConnection"
|
||||
:save-draft="saveDraft"
|
||||
:probe-source="probeSource"
|
||||
:load-catalogs="loadBindableCatalogs"
|
||||
:load-tables="loadBindableTables"
|
||||
:load-candidate-catalogs="loadCandidateCatalogs"
|
||||
:load-candidate-tables="loadCandidateTables"
|
||||
:load-managed-tables="loadManagedScopeTables"
|
||||
:probe-candidate="probeCandidate"
|
||||
:activate-source="activateSource"
|
||||
:reconfigure-source="reconfigureSource"
|
||||
:import-excel-source="importExcelSource"
|
||||
@completed="handleSourceCompleted"
|
||||
/>
|
||||
</div>
|
||||
</template>
|
||||
@@ -345,6 +563,85 @@ onBeforeUnmount(() => {
|
||||
transform: translateX(-50%);
|
||||
}
|
||||
|
||||
.workspace-body :deep([data-panel-group-direction='vertical']) {
|
||||
width: 100%;
|
||||
}
|
||||
|
||||
.workspace-body
|
||||
:deep(
|
||||
[data-panel-group-direction='vertical'] [data-panel-resize-handle-enabled]
|
||||
) {
|
||||
width: 100%;
|
||||
height: 10px;
|
||||
}
|
||||
|
||||
.workspace-body
|
||||
:deep(
|
||||
[data-panel-group-direction='vertical']
|
||||
[data-panel-resize-handle-enabled]::before
|
||||
) {
|
||||
top: 50%;
|
||||
right: 20px;
|
||||
bottom: auto;
|
||||
left: 20px;
|
||||
width: auto;
|
||||
height: 1px;
|
||||
transform: translateY(-50%);
|
||||
}
|
||||
|
||||
.source-workspace {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
height: 100%;
|
||||
min-height: 0;
|
||||
}
|
||||
|
||||
.source-mode-bar {
|
||||
display: flex;
|
||||
flex: 0 0 auto;
|
||||
align-items: center;
|
||||
justify-content: space-between;
|
||||
min-height: 44px;
|
||||
padding: 0 16px 8px 20px;
|
||||
}
|
||||
|
||||
.source-mode-switch {
|
||||
display: inline-flex;
|
||||
gap: 4px;
|
||||
padding: 3px;
|
||||
background: hsl(var(--surface-subtle));
|
||||
border-radius: var(--radius-control);
|
||||
}
|
||||
|
||||
.source-mode-switch :deep(button) {
|
||||
min-height: 30px;
|
||||
box-shadow: none;
|
||||
}
|
||||
|
||||
.source-runtime-status {
|
||||
padding: 4px 8px;
|
||||
font-size: 11px;
|
||||
font-weight: 600;
|
||||
color: hsl(var(--muted-foreground));
|
||||
background: hsl(var(--muted));
|
||||
border-radius: 999px;
|
||||
}
|
||||
|
||||
.source-runtime-status.is-ready {
|
||||
color: hsl(var(--success));
|
||||
background: hsl(var(--success) / 9%);
|
||||
}
|
||||
|
||||
.source-runtime-status.is-degraded {
|
||||
color: hsl(var(--warning));
|
||||
background: hsl(var(--warning) / 9%);
|
||||
}
|
||||
|
||||
.source-view-content {
|
||||
flex: 1;
|
||||
min-height: 0;
|
||||
}
|
||||
|
||||
.empty-state {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user