feat: 完善数据中枢联邦查询闭环

- 重构数据源生命周期、元数据纳管与运行时切换

- 增加只读 SQL、查询审计、跨节点取消与工作流联动

- 完善管理端连接配置、元数据浏览与 SQL 工作台
This commit is contained in:
2026-08-26 18:14:34 +08:00
parent 3e79e99925
commit e38821e48a
113 changed files with 17032 additions and 1151 deletions

View File

@@ -20,6 +20,10 @@
<groupId>com.zaxxer</groupId>
<artifactId>HikariCP</artifactId>
</dependency>
<dependency>
<groupId>com.easyagents</groupId>
<artifactId>easy-agents-federation-sql-adapter-jdbc</artifactId>
</dependency>
<dependency>
<groupId>org.postgresql</groupId>
<artifactId>postgresql</artifactId>

View File

@@ -0,0 +1,168 @@
package tech.easyflow.datacenter.audit;
import com.fasterxml.jackson.databind.annotation.JsonSerialize;
import com.fasterxml.jackson.databind.ser.std.ToStringSerializer;
import com.mybatisflex.annotation.Column;
import com.mybatisflex.annotation.Id;
import com.mybatisflex.annotation.KeyType;
import com.mybatisflex.annotation.Table;
import com.mybatisflex.core.handler.FastjsonTypeHandler;
import java.io.Serializable;
import java.math.BigInteger;
import java.util.Date;
import java.util.List;
import java.util.Map;
/**
* 数据中枢统一只读查询审计记录。
*/
@Table(value = "tb_datacenter_query_audit", comment = "数据中枢统一查询审计")
public class DatacenterQueryAudit implements Serializable {
@Id(keyType = KeyType.Generator, value = "snowFlakeId", comment = "主键")
private BigInteger id;
@Column(comment = "查询标识")
private String queryId;
@Column(tenantId = true, comment = "租户ID")
private BigInteger tenantId;
@Column(comment = "部门ID")
private BigInteger deptId;
@Column(comment = "执行账号ID")
private BigInteger executorAccountId;
@Column(comment = "调用方类型")
private String callerType;
@Column(comment = "调用方标识")
private String callerId;
@Column(comment = "数据源ID")
private BigInteger sourceId;
@Column(comment = "数据源版本")
private Long sourceRevision;
@Column(comment = "纳管范围版本")
private Long scopeRevision;
@Column(typeHandler = FastjsonTypeHandler.class, comment = "引用表摘要")
private List<String> referencedTablesJson;
@Column(typeHandler = FastjsonTypeHandler.class, comment = "引用字段摘要")
private List<String> referencedFieldsJson;
@Column(comment = "参数化或脱敏SQL")
private String parameterizedSql;
@Column(typeHandler = FastjsonTypeHandler.class, comment = "脱敏参数摘要")
private Map<String, Object> maskedParametersJson;
@Column(comment = "执行状态")
private String status;
@Column(comment = "稳定错误码")
private String errorCode;
@Column(comment = "有界错误说明")
private String errorMessage;
@Column(comment = "开始时间")
private Date startedAt;
@Column(comment = "结束时间")
private Date finishedAt;
@Column(comment = "总耗时毫秒")
private Long durationMs;
@Column(comment = "数据库耗时毫秒")
private Long databaseDurationMs;
@Column(comment = "返回行数")
private Long returnedRows;
@Column(comment = "是否截断")
private Integer truncated;
@Column(comment = "创建时间")
private Date created;
/** @return 主键 */
@JsonSerialize(using = ToStringSerializer.class)
public BigInteger getId() { return id; }
/** @param id 主键 */
public void setId(BigInteger id) { this.id = id; }
/** @return 查询标识 */
public String getQueryId() { return queryId; }
/** @param queryId 查询标识 */
public void setQueryId(String queryId) { this.queryId = queryId; }
/** @return 租户 ID */
public BigInteger getTenantId() { return tenantId; }
/** @param tenantId 租户 ID */
public void setTenantId(BigInteger tenantId) { this.tenantId = tenantId; }
/** @return 部门 ID */
public BigInteger getDeptId() { return deptId; }
/** @param deptId 部门 ID */
public void setDeptId(BigInteger deptId) { this.deptId = deptId; }
/** @return 执行账号 ID系统任务为空 */
public BigInteger getExecutorAccountId() { return executorAccountId; }
/** @param executorAccountId 执行账号 ID */
public void setExecutorAccountId(BigInteger executorAccountId) { this.executorAccountId = executorAccountId; }
/** @return 调用方类型 */
public String getCallerType() { return callerType; }
/** @param callerType 调用方类型 */
public void setCallerType(String callerType) { this.callerType = callerType; }
/** @return 调用方标识 */
public String getCallerId() { return callerId; }
/** @param callerId 调用方标识 */
public void setCallerId(String callerId) { this.callerId = callerId; }
/** @return 数据源 ID */
public BigInteger getSourceId() { return sourceId; }
/** @param sourceId 数据源 ID */
public void setSourceId(BigInteger sourceId) { this.sourceId = sourceId; }
/** @return 数据源版本 */
public Long getSourceRevision() { return sourceRevision; }
/** @param sourceRevision 数据源版本 */
public void setSourceRevision(Long sourceRevision) { this.sourceRevision = sourceRevision; }
/** @return 纳管范围版本 */
public Long getScopeRevision() { return scopeRevision; }
/** @param scopeRevision 纳管范围版本 */
public void setScopeRevision(Long scopeRevision) { this.scopeRevision = scopeRevision; }
/** @return 引用表 */
public List<String> getReferencedTablesJson() { return referencedTablesJson; }
/** @param referencedTablesJson 引用表 */
public void setReferencedTablesJson(List<String> referencedTablesJson) { this.referencedTablesJson = referencedTablesJson; }
/** @return 引用字段 */
public List<String> getReferencedFieldsJson() { return referencedFieldsJson; }
/** @param referencedFieldsJson 引用字段 */
public void setReferencedFieldsJson(List<String> referencedFieldsJson) { this.referencedFieldsJson = referencedFieldsJson; }
/** @return 参数化 SQL */
public String getParameterizedSql() { return parameterizedSql; }
/** @param parameterizedSql 参数化 SQL */
public void setParameterizedSql(String parameterizedSql) { this.parameterizedSql = parameterizedSql; }
/** @return 脱敏参数 */
public Map<String, Object> getMaskedParametersJson() { return maskedParametersJson; }
/** @param maskedParametersJson 脱敏参数 */
public void setMaskedParametersJson(Map<String, Object> maskedParametersJson) { this.maskedParametersJson = maskedParametersJson; }
/** @return 状态 */
public String getStatus() { return status; }
/** @param status 状态 */
public void setStatus(String status) { this.status = status; }
/** @return 错误码 */
public String getErrorCode() { return errorCode; }
/** @param errorCode 错误码 */
public void setErrorCode(String errorCode) { this.errorCode = errorCode; }
/** @return 错误说明 */
public String getErrorMessage() { return errorMessage; }
/** @param errorMessage 错误说明 */
public void setErrorMessage(String errorMessage) { this.errorMessage = errorMessage; }
/** @return 开始时间 */
public Date getStartedAt() { return startedAt; }
/** @param startedAt 开始时间 */
public void setStartedAt(Date startedAt) { this.startedAt = startedAt; }
/** @return 结束时间 */
public Date getFinishedAt() { return finishedAt; }
/** @param finishedAt 结束时间 */
public void setFinishedAt(Date finishedAt) { this.finishedAt = finishedAt; }
/** @return 总耗时 */
public Long getDurationMs() { return durationMs; }
/** @param durationMs 总耗时 */
public void setDurationMs(Long durationMs) { this.durationMs = durationMs; }
/** @return 数据库耗时 */
public Long getDatabaseDurationMs() { return databaseDurationMs; }
/** @param databaseDurationMs 数据库耗时 */
public void setDatabaseDurationMs(Long databaseDurationMs) { this.databaseDurationMs = databaseDurationMs; }
/** @return 返回行数 */
public Long getReturnedRows() { return returnedRows; }
/** @param returnedRows 返回行数 */
public void setReturnedRows(Long returnedRows) { this.returnedRows = returnedRows; }
/** @return 是否截断 */
public Integer getTruncated() { return truncated; }
/** @param truncated 是否截断 */
public void setTruncated(Integer truncated) { this.truncated = truncated; }
/** @return 创建时间 */
public Date getCreated() { return created; }
/** @param created 创建时间 */
public void setCreated(Date created) { this.created = created; }
}

View File

@@ -0,0 +1,301 @@
package tech.easyflow.datacenter.audit;
import java.math.BigInteger;
import java.util.Date;
import java.util.List;
import java.util.Map;
import com.mybatisflex.core.query.QueryWrapper;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Service;
import tech.easyflow.common.entity.LoginAccount;
import tech.easyflow.datacenter.mapper.DatacenterQueryAuditMapper;
import tech.easyflow.datacenter.meta.entity.DatacenterSource;
/**
* 在查询前落账并在资源释放后收口状态的审计服务。
*/
@Service
public class DatacenterQueryAuditService {
private final DatacenterQueryAuditMapper auditMapper;
/**
* 创建审计服务。
*
* @param auditMapper 审计 Mapper
*/
public DatacenterQueryAuditService(DatacenterQueryAuditMapper auditMapper) {
this.auditMapper = auditMapper;
}
/**
* 在数据库查询开始前写入 RUNNING 记录。
*
* @param queryId 查询标识
* @param source 数据源快照
* @param sql 参数化或无参数只读 SQL
* @param parameters 脱敏参数摘要
* @param account 执行账号
* @param callerType 调用方类型
* @param callerId 调用方标识
* @return 已持久化审计记录
*/
public DatacenterQueryAudit start(
String queryId,
DatacenterSource source,
String sql,
Map<String, Object> parameters,
LoginAccount account,
String callerType,
String callerId) {
Date now = new Date();
DatacenterQueryAudit audit = new DatacenterQueryAudit();
audit.setQueryId(queryId);
audit.setTenantId(account == null || account.getTenantId() == null
? value(source.getTenantId()) : account.getTenantId());
audit.setDeptId(account == null || account.getDeptId() == null
? value(source.getDeptId()) : account.getDeptId());
audit.setExecutorAccountId(account == null ? null : account.getId());
audit.setCallerType(callerType == null ? "SYSTEM" : callerType);
audit.setCallerId(callerId);
audit.setSourceId(source.getId());
audit.setSourceRevision(value(source.getDefinitionRevision()));
audit.setScopeRevision(value(source.getScopeRevision()));
audit.setReferencedTablesJson(List.of());
audit.setReferencedFieldsJson(List.of());
audit.setParameterizedSql(redactSql(sql));
audit.setMaskedParametersJson(parameters == null ? Map.of() : Map.copyOf(parameters));
audit.setStatus("RUNNING");
audit.setStartedAt(now);
audit.setReturnedRows(0L);
audit.setTruncated(0);
audit.setCreated(now);
auditMapper.insert(audit);
return audit;
}
/**
* 标记查询成功。
*
* @param audit 审计记录
* @param referencedTables Calcite 解析出的引用表
* @param referencedFields Calcite 解析出的引用字段
* @param returnedRows 返回行数
* @param truncated 是否截断
* @param durationMs 总耗时
* @param databaseDurationMs 数据库耗时
*/
public void succeed(
DatacenterQueryAudit audit,
List<String> referencedTables,
List<String> referencedFields,
long returnedRows,
boolean truncated,
long durationMs,
long databaseDurationMs) {
finish(audit, "SUCCESS", null, null, referencedTables, referencedFields,
returnedRows, truncated, durationMs, databaseDurationMs);
}
/**
* 标记查询失败。
*
* @param audit 审计记录
* @param errorCode 稳定错误码
* @param message 有界错误说明
* @param referencedTables Calcite 已解析出的引用表
* @param referencedFields Calcite 已解析出的引用字段
* @param durationMs 总耗时
*/
public void fail(
DatacenterQueryAudit audit,
String errorCode,
String message,
List<String> referencedTables,
List<String> referencedFields,
long durationMs) {
finish(audit, "FAILED", errorCode, bounded(message), referencedTables, referencedFields,
0L, false, durationMs, null);
}
private void finish(
DatacenterQueryAudit audit,
String status,
String errorCode,
String errorMessage,
List<String> referencedTables,
List<String> referencedFields,
long returnedRows,
boolean truncated,
long durationMs,
Long databaseDurationMs) {
audit.setStatus(status);
audit.setErrorCode(errorCode);
audit.setErrorMessage(errorMessage);
audit.setReferencedTablesJson(List.copyOf(
referencedTables == null ? List.of() : referencedTables));
audit.setReferencedFieldsJson(List.copyOf(
referencedFields == null ? List.of() : referencedFields));
audit.setReturnedRows(returnedRows);
audit.setTruncated(truncated ? 1 : 0);
audit.setDurationMs(durationMs);
audit.setDatabaseDurationMs(databaseDurationMs);
audit.setFinishedAt(new Date());
auditMapper.update(audit);
}
/**
* 将超过十分钟仍为 RUNNING 的遗留审计收口为失败,覆盖进程异常退出场景。
*/
@Scheduled(fixedDelayString = "${easyflow.datacenter.audit-reconcile-ms:60000}")
public void reconcileAbandonedQueries() {
Date now = new Date();
DatacenterQueryAudit patch = new DatacenterQueryAudit();
patch.setStatus("FAILED");
patch.setErrorCode("QUERY_ABANDONED");
patch.setErrorMessage("查询进程异常结束或超出审计存活时间");
patch.setFinishedAt(now);
QueryWrapper guard = QueryWrapper.create()
.eq(DatacenterQueryAudit::getStatus, "RUNNING")
.lt(DatacenterQueryAudit::getStartedAt, new Date(now.getTime() - 600_000L));
auditMapper.updateByQuery(patch, guard);
}
private long value(Long value) {
return value == null ? 0L : value;
}
private BigInteger value(BigInteger value) {
return value == null ? BigInteger.ZERO : value;
}
private String redactSql(String sql) {
if (sql == null || sql.isBlank()) {
return sql;
}
StringBuilder result = new StringBuilder(Math.min(sql.length(), 100_000));
boolean quoted = false;
for (int index = 0; index < sql.length(); index++) {
char current = sql.charAt(index);
if (quoted) {
if (current == '\'' && index + 1 < sql.length() && sql.charAt(index + 1) == '\'') {
index++;
continue;
}
if (current == '\'') {
quoted = false;
result.append('?');
}
continue;
}
if (current == '\'') {
quoted = true;
continue;
}
if (current == '-' && index + 1 < sql.length() && sql.charAt(index + 1) == '-') {
result.append("--?");
index += 2;
while (index < sql.length() && sql.charAt(index) != '\n'
&& sql.charAt(index) != '\r') {
index++;
}
if (index < sql.length()) {
result.append(sql.charAt(index));
}
continue;
}
if (current == '/' && index + 1 < sql.length() && sql.charAt(index + 1) == '*') {
result.append("/*?*/");
index = skipBlockComment(sql, index + 2);
continue;
}
String dollarDelimiter = current == '$' ? dollarDelimiter(sql, index) : null;
if (dollarDelimiter != null) {
int end = sql.indexOf(dollarDelimiter, index + dollarDelimiter.length());
result.append('?');
index = end < 0 ? sql.length() : end + dollarDelimiter.length() - 1;
continue;
}
if (Character.isDigit(current)
&& (index == 0 || !Character.isJavaIdentifierPart(sql.charAt(index - 1)))) {
result.append('?');
while (index + 1 < sql.length()) {
char next = sql.charAt(index + 1);
if (!Character.isDigit(next) && next != '.' && next != 'e' && next != 'E'
&& next != '+' && next != '-') {
break;
}
index++;
}
continue;
}
result.append(current);
if (result.length() >= 100_000) {
break;
}
}
if (quoted) {
result.append('?');
}
return result.toString();
}
/**
* 跳过支持嵌套的块注释内容。
*
* @param sql 原始 SQL
* @param contentStart 首个注释内容字符位置
* @return 外层注释结束标记的末尾位置;未闭合时返回末尾
*/
private int skipBlockComment(String sql, int contentStart) {
int depth = 1;
for (int index = contentStart; index < sql.length() - 1; index++) {
if (sql.charAt(index) == '/' && sql.charAt(index + 1) == '*') {
depth++;
index++;
} else if (sql.charAt(index) == '*' && sql.charAt(index + 1) == '/') {
depth--;
if (depth == 0) {
return index + 1;
}
index++;
}
}
return sql.length();
}
/**
* 识别 PostgreSQL dollar-quoted 字面量起始分隔符。
*
* @param sql 原始 SQL
* @param start 美元符号位置
* @return 完整分隔符;当前位置不是合法起始符时为空
*/
private String dollarDelimiter(String sql, int start) {
int end = sql.indexOf('$', start + 1);
if (end < 0) {
return null;
}
String tag = sql.substring(start + 1, end);
if (tag.isEmpty()) {
return "$$";
}
if (!Character.isLetter(tag.charAt(0)) && tag.charAt(0) != '_') {
return null;
}
for (int index = 1; index < tag.length(); index++) {
char character = tag.charAt(index);
if (!Character.isLetterOrDigit(character) && character != '_') {
return null;
}
}
return '$' + tag + '$';
}
private String bounded(String message) {
if (message == null) {
return null;
}
return message.length() <= 500 ? message : message.substring(0, 500);
}
}

View File

@@ -0,0 +1,55 @@
package tech.easyflow.datacenter.config;
import com.easyagents.federation.sql.source.SourceId;
import java.nio.charset.StandardCharsets;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.boot.autoconfigure.condition.ConditionalOnBean;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.data.redis.connection.RedisConnectionFactory;
import org.springframework.data.redis.listener.ChannelTopic;
import org.springframework.data.redis.listener.RedisMessageListenerContainer;
import tech.easyflow.datacenter.federation.DatacenterFederationChangeNotifier;
import tech.easyflow.datacenter.federation.DatacenterFederationQueryCancellationService;
import tech.easyflow.datacenter.federation.DatacenterFederationSourceStateProvider;
/**
* 数据中枢无凭据 Definition 变更提示订阅配置。
*/
@Configuration(proxyBeanMethods = false)
public class DatacenterFederationRedisConfig {
private static final Logger log = LoggerFactory.getLogger(DatacenterFederationRedisConfig.class);
/**
* 创建独立 Redis 监听容器,收到提示后始终回源数据库。
*
* @param connectionFactory Redis 连接工厂
* @param stateProvider 数据库权威状态 Provider
* @param queryCancellationService 查询取消服务
* @return 监听容器
*/
@Bean(name = "datacenterFederationRedisListenerContainer")
@ConditionalOnBean(RedisConnectionFactory.class)
public RedisMessageListenerContainer datacenterFederationRedisListenerContainer(
RedisConnectionFactory connectionFactory,
DatacenterFederationSourceStateProvider stateProvider,
DatacenterFederationQueryCancellationService queryCancellationService) {
RedisMessageListenerContainer container = new RedisMessageListenerContainer();
container.setConnectionFactory(connectionFactory);
container.addMessageListener((message, pattern) -> {
String value = new String(message.getBody(), StandardCharsets.UTF_8);
try {
stateProvider.refresh(new SourceId(value));
} catch (RuntimeException exception) {
log.warn("Failed to refresh datacenter Federation source {}", value, exception);
}
}, new ChannelTopic(DatacenterFederationChangeNotifier.CHANNEL));
container.addMessageListener((message, pattern) ->
queryCancellationService.acceptCancellationHint(
new String(message.getBody(), StandardCharsets.UTF_8)),
new ChannelTopic(DatacenterFederationQueryCancellationService.CHANNEL));
return container;
}
}

View File

@@ -3,6 +3,8 @@ package tech.easyflow.datacenter.connector;
import tech.easyflow.datacenter.entity.DatacenterTable;
import tech.easyflow.datacenter.meta.entity.DatacenterSource;
import tech.easyflow.datacenter.meta.model.DatacenterCatalogMeta;
import tech.easyflow.datacenter.meta.model.DatacenterManagedMetadataSnapshot;
import tech.easyflow.datacenter.meta.model.DatacenterMetadataPage;
import tech.easyflow.datacenter.meta.model.DatacenterTableDetailMeta;
import java.util.List;
@@ -10,7 +12,97 @@ import java.util.List;
public interface MetadataExplorer {
List<DatacenterCatalogMeta> listCatalogs(DatacenterSource source);
/**
* 分页浏览 Catalog 或 Schema。
*
* @param source 数据源
* @param keyword 名称搜索词
* @param pageNumber 页码
* @param pageSize 每页大小
* @return 有界命名空间列表
*/
default DatacenterMetadataPage<DatacenterCatalogMeta> listCatalogsPage(
DatacenterSource source,
String keyword,
long pageNumber,
long pageSize) {
String normalizedKeyword = keyword == null
? "" : keyword.trim().toLowerCase(java.util.Locale.ROOT);
List<DatacenterCatalogMeta> filtered = listCatalogs(source).stream()
.filter(catalog -> normalizedKeyword.isEmpty()
|| String.valueOf(catalog.getCatalogName())
.toLowerCase(java.util.Locale.ROOT)
.contains(normalizedKeyword))
.toList();
return DatacenterMetadataPage.slice(filtered, pageNumber, pageSize);
}
List<DatacenterTable> listTables(DatacenterSource source, String catalogName);
/**
* 分页浏览表或视图。
*
* <p>非 JDBC Connector 默认复用已有列表能力;外部 JDBC Connector 应覆盖此方法,
* 直接在元数据 ResultSet 上做有界读取。</p>
*
* @param source 数据源
* @param catalogName 物理命名空间
* @param keyword 表名搜索词
* @param pageNumber 页码
* @param pageSize 每页大小
* @return 有界表列表
*/
default DatacenterMetadataPage<DatacenterTable> listTablesPage(
DatacenterSource source,
String catalogName,
String keyword,
long pageNumber,
long pageSize) {
String normalizedKeyword = keyword == null ? "" : keyword.trim().toLowerCase(java.util.Locale.ROOT);
List<DatacenterTable> filtered = listTables(source, catalogName).stream()
.filter(table -> normalizedKeyword.isEmpty()
|| String.valueOf(table.getTableName())
.toLowerCase(java.util.Locale.ROOT)
.contains(normalizedKeyword))
.toList();
return DatacenterMetadataPage.slice(filtered, pageNumber, pageSize);
}
DatacenterTableDetailMeta getTableDetail(DatacenterSource source, String catalogName, String tableName);
/**
* 批量读取表详情。
*
* @param source 数据源
* @param catalogName 物理命名空间
* @param tableNames 表名集合
* @return 表详情集合
*/
default List<DatacenterTableDetailMeta> getTableDetails(
DatacenterSource source,
String catalogName,
List<String> tableNames) {
return tableNames.stream()
.map(tableName -> getTableDetail(source, catalogName, tableName))
.toList();
}
/**
* 读取已纳管表的刷新快照。
*
* <p>默认实现保留 Connector 的既有逐表语义JDBC Connector 应覆盖此方法以复用
* 单个连接,并明确区分物理表缺失与连接故障。</p>
*
* @param source 数据源
* @param catalogName 物理命名空间
* @param tableNames 已纳管表名集合
* @return 存在与缺失对象的快照
*/
default DatacenterManagedMetadataSnapshot inspectManagedTables(
DatacenterSource source,
String catalogName,
List<String> tableNames) {
return new DatacenterManagedMetadataSnapshot(
getTableDetails(source, catalogName, tableNames), java.util.Set.of());
}
}

View File

@@ -28,13 +28,12 @@ public class MysqlConnector extends AbstractJdbcConnector {
@Override
protected <T> T withConnection(DatacenterSource source, boolean cacheable, JdbcCallback<T> callback) throws Exception {
HikariDataSource dataSource = cacheable ? datasourceManager.getOrCreateExternalDatasource(source) : datasourceManager.createExternalDatasource(source);
// MySQL 查询 Runtime 已由 Federation SQL 独占管理;此 Connector 只保留短生命周期元数据访问。
HikariDataSource dataSource = datasourceManager.createExternalDatasource(source);
try (Connection connection = dataSource.getConnection()) {
return callback.apply(connection);
} finally {
if (!cacheable || source.getId() == null) {
dataSource.close();
}
dataSource.close();
}
}
}

View File

@@ -28,13 +28,12 @@ public class PostgresqlConnector extends AbstractJdbcConnector {
@Override
protected <T> T withConnection(DatacenterSource source, boolean cacheable, JdbcCallback<T> callback) throws Exception {
HikariDataSource dataSource = cacheable ? datasourceManager.getOrCreateExternalDatasource(source) : datasourceManager.createExternalDatasource(source);
// PostgreSQL 查询 Runtime 已由 Federation SQL 独占管理;此 Connector 只保留短生命周期元数据访问。
HikariDataSource dataSource = datasourceManager.createExternalDatasource(source);
try (Connection connection = dataSource.getConnection()) {
return callback.apply(connection);
} finally {
if (!cacheable || source.getId() == null) {
dataSource.close();
}
dataSource.close();
}
}
}

View File

@@ -3,6 +3,7 @@ package tech.easyflow.datacenter.connector.support;
import cn.hutool.core.util.StrUtil;
import com.alibaba.fastjson2.JSONObject;
import com.mybatisflex.core.paginate.Page;
import com.mybatisflex.core.query.QueryColumn;
import com.mybatisflex.core.query.QueryWrapper;
import com.mybatisflex.core.row.Db;
import com.mybatisflex.core.row.Row;
@@ -15,6 +16,7 @@ import tech.easyflow.datacenter.connector.DatacenterConnector;
import tech.easyflow.datacenter.entity.DatacenterTable;
import tech.easyflow.datacenter.entity.DatacenterTableField;
import tech.easyflow.datacenter.execution.model.DatacenterConnectionTestResult;
import tech.easyflow.datacenter.execution.model.DatacenterQueryFilter;
import tech.easyflow.datacenter.execution.model.DatacenterQueryRequest;
import tech.easyflow.datacenter.meta.entity.DatacenterSource;
import tech.easyflow.datacenter.meta.enums.DatacenterCapability;
@@ -85,35 +87,180 @@ public abstract class AbstractInternalTableConnector implements DatacenterConnec
@Override
public Page<Row> queryPage(DatacenterSource source, DatacenterTable table, DatacenterQueryRequest request) {
if (request == null || request.getPageSize() == null
|| request.getPageSize() < 1L
|| request.getPageSize() > 500L) {
throw new BusinessException("pageSize 必须在 1 到 500 之间");
}
String actualTable = resolveTableName(table);
long count = Db.selectCountByQuery(
actualTable, createQueryWrapper(request.getWhere()));
actualTable, createQueryWrapper(table, request, false));
if (count == 0) {
return new Page<>(new ArrayList<>(), request.getPageNumber(), request.getPageSize(), count);
}
// selectCountByQuery 会把无投影的 QueryWrapper 改为 COUNT(*),分页查询必须使用独立实例。
QueryWrapper pageQuery = createQueryWrapper(table, request, true);
pageQuery.select(resolveSelectedColumns(table, request).stream()
.map(this::quoteInternalIdentifier)
.toArray(String[]::new));
Page<Row> page = Db.paginate(
actualTable,
new Page<>(request.getPageNumber(), request.getPageSize(), count),
createQueryWrapper(request.getWhere()));
pageQuery);
normalizeRows(page.getRecords());
return page;
}
/**
* 将已在服务层校验的逻辑字段映射为内部物理列。
*
* @param table 绑定表及字段
* @param request 查询请求
* @return 按请求顺序排列的物理列名
* @throws BusinessException 字段元数据缺失时抛出
*/
private List<String> resolveSelectedColumns(
DatacenterTable table,
DatacenterQueryRequest request) {
Map<String, DatacenterTableField> fields = new LinkedHashMap<>();
for (DatacenterTableField field : table.getFields()) {
fields.put(field.getFieldName(), field);
}
List<String> columns = new ArrayList<>();
for (String selected : request.getSelectedColumns()) {
DatacenterTableField field = fields.get(selected);
if (field == null) {
throw new BusinessException("字段不存在: " + selected);
}
String sourceColumn = StrUtil.blankToDefault(
field.getSourceColumnName(), field.getFieldName());
columns.add(sourceColumn);
}
if (columns.isEmpty()) {
throw new BusinessException("当前数据集没有可查询字段");
}
return columns;
}
/**
* 引用可信元数据列名,避免保留字或特殊字符改变 SQL 结构。
*
* @param identifier 物理列名
* @return MySQL 反引号标识符
*/
private String quoteInternalIdentifier(String identifier) {
return "`" + identifier.replace("`", "``") + "`";
}
/**
* 创建用于动态表查询的独立条件包装器。
*
* @param where 已校验的筛选表达式
* @param where 调用方传入的旧原始筛选表达式;非空时拒绝
* @return 新建的查询条件包装器
*/
static QueryWrapper createQueryWrapper(String where) {
QueryWrapper wrapper = QueryWrapper.create();
if (StrUtil.isNotBlank(where)) {
wrapper.where(where);
throw new BusinessException("不支持原始 WHERE 条件,请使用结构化筛选条件");
}
QueryWrapper wrapper = QueryWrapper.create();
return wrapper;
}
private QueryWrapper createQueryWrapper(
DatacenterTable table,
DatacenterQueryRequest request,
boolean includeSorts) {
QueryWrapper wrapper = createQueryWrapper(request.getWhere());
Map<String, DatacenterTableField> fields = table.getFields().stream()
.collect(java.util.stream.Collectors.toMap(
DatacenterTableField::getFieldName,
field -> field,
(first, ignored) -> first,
LinkedHashMap::new));
List<Object> parameters = new ArrayList<>();
StringBuilder condition = new StringBuilder();
if (request.getFilters() != null) {
for (DatacenterQueryFilter filter : request.getFilters()) {
DatacenterTableField field = fields.get(filter.getColumn());
if (field == null) {
throw new BusinessException("字段不存在: " + filter.getColumn());
}
appendFilter(condition, parameters, field, filter);
}
}
if (!condition.isEmpty()) {
wrapper.where(condition.toString(), parameters.toArray());
}
if (includeSorts && request.getSorts() != null && !request.getSorts().isEmpty()) {
request.getSorts().forEach(sort -> {
DatacenterTableField field = fields.get(sort.getColumn());
if (field == null) {
throw new BusinessException("字段不存在: " + sort.getColumn());
}
wrapper.orderBy(
new QueryColumn(sourceColumnName(field)),
!"DESC".equalsIgnoreCase(sort.getDirection()));
});
}
return wrapper;
}
private void appendFilter(
StringBuilder condition,
List<Object> parameters,
DatacenterTableField field,
DatacenterQueryFilter filter) {
if (!condition.isEmpty()) {
condition.append(" AND ");
}
String column = quoteInternalIdentifier(sourceColumnName(field));
String operator = StrUtil.blankToDefault(filter.getOperator(), "EQ")
.toUpperCase(Locale.ROOT);
switch (operator) {
case "EQ" -> appendComparison(condition, parameters, column, "=", filter.getValue());
case "LIKE" -> appendComparison(
condition, parameters, column, "LIKE",
"%" + String.valueOf(filter.getValue()) + "%");
case "GT" -> appendComparison(condition, parameters, column, ">", filter.getValue());
case "GTE" -> appendComparison(condition, parameters, column, ">=", filter.getValue());
case "LT" -> appendComparison(condition, parameters, column, "<", filter.getValue());
case "LTE" -> appendComparison(condition, parameters, column, "<=", filter.getValue());
case "IS_NULL" -> condition.append(column).append(" IS NULL");
case "IN" -> appendInFilter(condition, parameters, column, filter.getValues());
default -> throw new BusinessException("不支持的过滤操作: " + operator);
}
}
private void appendComparison(
StringBuilder condition,
List<Object> parameters,
String column,
String operator,
Object value) {
condition.append(column).append(' ').append(operator).append(" ?");
parameters.add(value);
}
private void appendInFilter(
StringBuilder condition,
List<Object> parameters,
String column,
List<Object> values) {
if (values == null || values.isEmpty()) {
condition.append("1 = 0");
return;
}
condition.append(column).append(" IN (")
.append(Collections.nCopies(values.size(), "?").stream()
.collect(java.util.stream.Collectors.joining(", ")))
.append(')');
parameters.addAll(values);
}
private String sourceColumnName(DatacenterTableField field) {
return StrUtil.blankToDefault(field.getSourceColumnName(), field.getFieldName());
}
@Override
public List<Row> queryBySql(DatacenterSource source, String sql) {
List<Row> rows = Db.selectListBySql(sql);

View File

@@ -16,12 +16,17 @@ import tech.easyflow.datacenter.execution.model.DatacenterConnectionTestResult;
import tech.easyflow.datacenter.execution.model.DatacenterQueryFilter;
import tech.easyflow.datacenter.execution.model.DatacenterQueryRequest;
import tech.easyflow.datacenter.execution.model.DatacenterQuerySort;
import tech.easyflow.datacenter.federation.DatacenterMetadataIdentity;
import tech.easyflow.datacenter.meta.entity.DatacenterSource;
import tech.easyflow.datacenter.meta.enums.DatacenterCapability;
import tech.easyflow.datacenter.meta.enums.DatacenterConnectionErrorCode;
import tech.easyflow.datacenter.meta.enums.DatacenterMetadataStatus;
import tech.easyflow.datacenter.meta.enums.DatacenterSensitivityLevel;
import tech.easyflow.datacenter.meta.enums.DatacenterSourceType;
import tech.easyflow.datacenter.meta.enums.DatacenterTableKind;
import tech.easyflow.datacenter.meta.model.DatacenterCatalogMeta;
import tech.easyflow.datacenter.meta.model.DatacenterMetadataPage;
import tech.easyflow.datacenter.meta.model.DatacenterManagedMetadataSnapshot;
import tech.easyflow.datacenter.meta.model.DatacenterTableDetailMeta;
import java.math.BigDecimal;
@@ -105,30 +110,39 @@ public abstract class AbstractJdbcConnector implements DatacenterConnector {
return withConnection(source, true, connection -> {
List<DatacenterCatalogMeta> result = new ArrayList<>();
DatabaseMetaData metaData = connection.getMetaData();
try (ResultSet catalogs = metaData.getCatalogs()) {
while (catalogs.next()) {
String name = catalogs.getString("TABLE_CAT");
if (StrUtil.isBlank(name)) {
continue;
if (usesCatalogNamespace()) {
try (ResultSet catalogs = metaData.getCatalogs()) {
while (catalogs.next()) {
String name = catalogs.getString("TABLE_CAT");
if (StrUtil.isBlank(name)) {
continue;
}
DatacenterCatalogMeta meta = new DatacenterCatalogMeta();
meta.setSourceId(source.getId());
meta.setCatalogName(name);
meta.setCatalogType("CATALOG");
meta.setLogicalSchemaName(name);
meta.setPhysicalCatalogName(name);
result.add(meta);
}
DatacenterCatalogMeta meta = new DatacenterCatalogMeta();
meta.setSourceId(source.getId());
meta.setCatalogName(name);
meta.setCatalogType("CATALOG");
result.add(meta);
}
}
try (ResultSet schemas = metaData.getSchemas()) {
while (schemas.next()) {
String name = schemas.getString("TABLE_SCHEM");
if (StrUtil.isBlank(name) || containsCatalog(result, name)) {
continue;
if (!usesCatalogNamespace()) {
try (ResultSet schemas = metaData.getSchemas()) {
while (schemas.next()) {
String name = schemas.getString("TABLE_SCHEM");
if (StrUtil.isBlank(name)) {
continue;
}
DatacenterCatalogMeta meta = new DatacenterCatalogMeta();
meta.setSourceId(source.getId());
meta.setCatalogName(name);
meta.setCatalogType("SCHEMA");
meta.setLogicalSchemaName(name);
meta.setPhysicalCatalogName(source.getDatabaseName());
meta.setPhysicalSchemaName(name);
result.add(meta);
}
DatacenterCatalogMeta meta = new DatacenterCatalogMeta();
meta.setSourceId(source.getId());
meta.setCatalogName(name);
meta.setCatalogType("SCHEMA");
result.add(meta);
}
}
result = filterConfiguredCatalogs(source, result);
@@ -139,6 +153,13 @@ public abstract class AbstractJdbcConnector implements DatacenterConnector {
meta.setSourceId(source.getId());
meta.setCatalogName(fallback);
meta.setCatalogType("DEFAULT");
meta.setLogicalSchemaName(fallback);
if (usesCatalogNamespace()) {
meta.setPhysicalCatalogName(fallback);
} else {
meta.setPhysicalCatalogName(source.getDatabaseName());
meta.setPhysicalSchemaName(fallback);
}
result.add(meta);
}
}
@@ -149,24 +170,98 @@ public abstract class AbstractJdbcConnector implements DatacenterConnector {
}
}
/**
* 直接在 JDBC Catalog/Schema 元数据游标上执行有界分页。
*
* @param source 数据源
* @param keyword 名称搜索词
* @param pageNumber 页码
* @param pageSize 每页大小
* @return 有界命名空间列表
*/
@Override
public DatacenterMetadataPage<DatacenterCatalogMeta> listCatalogsPage(
DatacenterSource source,
String keyword,
long pageNumber,
long pageSize) {
try {
return withConnection(source, true, connection -> {
DatabaseMetaData metadata = connection.getMetaData();
String normalizedKeyword = keyword == null
? "" : keyword.trim().toLowerCase(Locale.ROOT);
String configuredName = usesCatalogNamespace()
? StrUtil.trimToNull(source.getDatabaseName())
: StrUtil.trimToNull(source.getSchemaName());
long offset = (pageNumber - 1L) * pageSize;
long matched = 0L;
boolean hasMore = false;
List<DatacenterCatalogMeta> records = new ArrayList<>((int) pageSize);
try (ResultSet names = usesCatalogNamespace()
? metadata.getCatalogs() : metadata.getSchemas()) {
String column = usesCatalogNamespace() ? "TABLE_CAT" : "TABLE_SCHEM";
while (names.next()) {
String name = names.getString(column);
if (StrUtil.isBlank(name)
|| (configuredName != null && !configuredName.equals(name))
|| (!normalizedKeyword.isEmpty()
&& !name.toLowerCase(Locale.ROOT).contains(normalizedKeyword))) {
continue;
}
if (matched++ < offset) {
continue;
}
if (records.size() >= pageSize) {
hasMore = true;
break;
}
records.add(catalogMeta(source, name));
}
}
if (records.isEmpty() && offset == 0L && normalizedKeyword.isEmpty()) {
String fallback = resolveCatalogName(source, null);
if (StrUtil.isNotBlank(fallback)) {
records.add(catalogMeta(source, fallback));
}
}
return new DatacenterMetadataPage<>(
records, pageNumber, pageSize, hasMore);
});
} catch (Exception ex) {
throw DatacenterConnectorExceptionSupport.wrapAccessException("读取目录失败", ex);
}
}
private DatacenterCatalogMeta catalogMeta(
DatacenterSource source,
String name) {
DatacenterCatalogMeta meta = new DatacenterCatalogMeta();
meta.setSourceId(source.getId());
meta.setCatalogName(name);
meta.setCatalogType(usesCatalogNamespace() ? "CATALOG" : "SCHEMA");
meta.setLogicalSchemaName(name);
if (usesCatalogNamespace()) {
meta.setPhysicalCatalogName(name);
} else {
meta.setPhysicalCatalogName(source.getDatabaseName());
meta.setPhysicalSchemaName(name);
}
return meta;
}
@Override
public List<DatacenterTable> listTables(DatacenterSource source, String catalogName) {
try {
return withConnection(source, true, connection -> {
DatabaseMetaData metaData = connection.getMetaData();
List<DatacenterTable> tables = new ArrayList<>();
try (ResultSet resultSet = metaData.getTables(resolveCatalogArgument(source, catalogName), resolveSchemaArgument(source, catalogName), "%", new String[]{"TABLE", "VIEW"})) {
try (ResultSet resultSet = metaData.getTables(
resolveCatalogArgument(source, catalogName),
metadataPattern(metaData, resolveSchemaArgument(source, catalogName)),
"%",
new String[]{"TABLE", "VIEW"})) {
while (resultSet.next()) {
DatacenterTable table = new DatacenterTable();
table.setSourceId(source.getId());
table.setTableName(resultSet.getString("TABLE_NAME"));
table.setTableDesc(resultSet.getString("REMARKS"));
table.setActualTable(resultSet.getString("TABLE_NAME"));
table.setMaterializedTable(resultSet.getString("TABLE_NAME"));
table.setAccessMode(capabilities.contains(DatacenterCapability.WRITE_MUTATION) ? "READ_WRITE" : "READ_ONLY");
table.setTableKind(resolveTableKind(resultSet.getString("TABLE_TYPE")).name());
table.setCapabilitiesJson(Map.of("capabilities", capabilities.stream().map(Enum::name).toList()));
tables.add(table);
tables.add(tableFromMetadata(source, resultSet));
}
}
return tables;
@@ -176,10 +271,166 @@ public abstract class AbstractJdbcConnector implements DatacenterConnector {
}
}
/**
* 直接在 JDBC 元数据游标上完成表名筛选和有界分页。
*
* @param source 数据源
* @param catalogName 物理命名空间
* @param keyword 表名搜索词
* @param pageNumber 页码
* @param pageSize 每页大小
* @return 有界表列表
*/
@Override
public DatacenterMetadataPage<DatacenterTable> listTablesPage(
DatacenterSource source,
String catalogName,
String keyword,
long pageNumber,
long pageSize) {
try {
return withConnection(source, true, connection -> {
DatabaseMetaData metaData = connection.getMetaData();
String normalizedKeyword = keyword == null
? "" : keyword.trim().toLowerCase(Locale.ROOT);
long offset = (pageNumber - 1L) * pageSize;
long matched = 0L;
List<DatacenterTable> tables = new ArrayList<>((int) pageSize);
boolean hasMore = false;
try (ResultSet resultSet = metaData.getTables(
resolveCatalogArgument(source, catalogName),
metadataPattern(metaData, resolveSchemaArgument(source, catalogName)),
"%",
new String[]{"TABLE", "VIEW"})) {
while (resultSet.next()) {
String tableName = resultSet.getString("TABLE_NAME");
if (!normalizedKeyword.isEmpty()
&& (tableName == null
|| !tableName.toLowerCase(Locale.ROOT).contains(normalizedKeyword))) {
continue;
}
if (matched++ < offset) {
continue;
}
if (tables.size() >= pageSize) {
hasMore = true;
break;
}
tables.add(tableFromMetadata(source, resultSet));
}
}
return new DatacenterMetadataPage<>(
tables, pageNumber, pageSize, hasMore);
});
} catch (Exception ex) {
throw DatacenterConnectorExceptionSupport.wrapAccessException("读取表列表失败", ex);
}
}
private DatacenterTable tableFromMetadata(
DatacenterSource source,
ResultSet resultSet) throws SQLException {
DatacenterTable table = new DatacenterTable();
table.setSourceId(source.getId());
table.setTableName(resultSet.getString("TABLE_NAME"));
table.setTableDesc(resultSet.getString("REMARKS"));
table.setActualTable(resultSet.getString("TABLE_NAME"));
table.setMaterializedTable(resultSet.getString("TABLE_NAME"));
table.setAccessMode(capabilities.contains(DatacenterCapability.WRITE_MUTATION)
? "READ_WRITE" : "READ_ONLY");
table.setTableKind(resolveTableKind(resultSet.getString("TABLE_TYPE")).name());
table.setQueryable(1);
table.setMetadataStatus(DatacenterMetadataStatus.ACTIVE.name());
table.setLastSeenAt(new java.util.Date());
table.setTimeSemantics("NONE");
table.setMetadataRevision(1L);
table.setCapabilitiesJson(Map.of(
"capabilities", capabilities.stream().map(Enum::name).toList()));
return table;
}
@Override
public DatacenterTableDetailMeta getTableDetail(DatacenterSource source, String catalogName, String tableName) {
try {
return withConnection(source, true,
connection -> readTableDetail(connection, source, catalogName, tableName));
} catch (Exception ex) {
throw DatacenterConnectorExceptionSupport.wrapAccessException("读取表详情失败", ex);
}
}
/**
* 使用单个元数据连接批量读取表详情,避免每张表重复创建连接池。
*
* @param source 数据源
* @param catalogName 物理命名空间
* @param tableNames 表名集合
* @return 表详情集合
*/
@Override
public List<DatacenterTableDetailMeta> getTableDetails(
DatacenterSource source,
String catalogName,
List<String> tableNames) {
if (tableNames == null || tableNames.isEmpty()) {
return List.of();
}
try {
return withConnection(source, true, connection -> {
List<DatacenterTableDetailMeta> details = new ArrayList<>(tableNames.size());
for (String tableName : tableNames) {
details.add(readTableDetail(connection, source, catalogName, tableName));
}
return List.copyOf(details);
});
} catch (Exception ex) {
throw DatacenterConnectorExceptionSupport.wrapAccessException("批量读取表详情失败", ex);
}
}
/**
* 使用单个连接刷新已纳管表,并把物理缺失作为快照状态返回。
*
* @param source 数据源
* @param catalogName 物理命名空间
* @param tableNames 已纳管表名集合
* @return 已发现详情与缺失表名
*/
@Override
public DatacenterManagedMetadataSnapshot inspectManagedTables(
DatacenterSource source,
String catalogName,
List<String> tableNames) {
if (tableNames == null || tableNames.isEmpty()) {
return new DatacenterManagedMetadataSnapshot(List.of(), Set.of());
}
try {
return withConnection(source, true, connection -> {
List<DatacenterTableDetailMeta> details = new ArrayList<>(tableNames.size());
Set<String> missing = new java.util.LinkedHashSet<>();
for (String tableName : tableNames) {
try {
details.add(readTableDetail(connection, source, catalogName, tableName));
} catch (BusinessException exception) {
if (exception.getMessage() == null
|| !exception.getMessage().startsWith("所选数据表已变化")) {
throw exception;
}
missing.add(tableName);
}
}
return new DatacenterManagedMetadataSnapshot(details, missing);
});
} catch (Exception ex) {
throw DatacenterConnectorExceptionSupport.wrapAccessException("刷新已纳管表失败", ex);
}
}
private DatacenterTableDetailMeta readTableDetail(
Connection connection,
DatacenterSource source,
String catalogName,
String tableName) throws SQLException {
DatabaseMetaData metaData = connection.getMetaData();
DatacenterTableDetailMeta detail = new DatacenterTableDetailMeta();
DatacenterTable table = new DatacenterTable();
@@ -189,24 +440,35 @@ public abstract class AbstractJdbcConnector implements DatacenterConnector {
table.setMaterializedTable(tableName);
table.setAccessMode(capabilities.contains(DatacenterCapability.WRITE_MUTATION) ? "READ_WRITE" : "READ_ONLY");
table.setTableKind(DatacenterTableKind.EXTERNAL_TABLE.name());
table.setQueryable(1);
table.setMetadataStatus(DatacenterMetadataStatus.ACTIVE.name());
table.setLastSeenAt(new java.util.Date());
table.setTimeSemantics("NONE");
table.setMetadataRevision(1L);
table.setCapabilitiesJson(Map.of("capabilities", capabilities.stream().map(Enum::name).toList()));
detail.setTable(table);
boolean tableFound = false;
try (ResultSet tableSet = metaData.getTables(
resolveCatalogArgument(source, catalogName),
resolveSchemaArgument(source, catalogName),
tableName,
metadataPattern(metaData, resolveSchemaArgument(source, catalogName)),
metadataPattern(metaData, tableName),
new String[]{"TABLE", "VIEW"})) {
while (tableSet.next()) {
String currentTableName = tableSet.getString("TABLE_NAME");
if (!matchesTableName(currentTableName, tableName)) {
if (!matchesMetadataTable(
tableSet, source, catalogName, tableName)) {
continue;
}
table.setTableDesc(tableSet.getString("REMARKS"));
table.setTableKind(resolveTableKind(tableSet.getString("TABLE_TYPE")).name());
tableFound = true;
break;
}
}
if (!tableFound) {
throw new BusinessException(
"所选数据表已变化,请刷新后重试: " + tableName);
}
Set<String> primaryKeys = new HashSet<>();
try (ResultSet pkSet = metaData.getPrimaryKeys(resolveCatalogArgument(source, catalogName), resolveSchemaArgument(source, catalogName), tableName)) {
@@ -216,13 +478,24 @@ public abstract class AbstractJdbcConnector implements DatacenterConnector {
}
List<DatacenterTableField> fields = new ArrayList<>();
try (ResultSet columns = metaData.getColumns(resolveCatalogArgument(source, catalogName), resolveSchemaArgument(source, catalogName), tableName, "%")) {
try (ResultSet columns = metaData.getColumns(
resolveCatalogArgument(source, catalogName),
metadataPattern(metaData, resolveSchemaArgument(source, catalogName)),
metadataPattern(metaData, tableName),
"%")) {
while (columns.next()) {
if (!matchesMetadataTable(
columns, source, catalogName, tableName)) {
continue;
}
DatacenterTableField field = new DatacenterTableField();
field.setFieldName(columns.getString("COLUMN_NAME"));
field.setSourceColumnName(columns.getString("COLUMN_NAME"));
field.setFieldDesc(columns.getString("REMARKS"));
field.setJdbcType(columns.getString("TYPE_NAME"));
field.setJdbcTypeCode(columns.getInt("DATA_TYPE"));
field.setNativeTypeName(columns.getString("TYPE_NAME"));
field.setOrdinalPosition(columns.getInt("ORDINAL_POSITION"));
field.setPrecision(columns.getInt("COLUMN_SIZE"));
field.setScale(columns.getInt("DECIMAL_DIGITS"));
field.setRequired(columns.getInt("NULLABLE") == DatabaseMetaData.columnNoNulls ? 1 : 0);
@@ -231,15 +504,21 @@ public abstract class AbstractJdbcConnector implements DatacenterConnector {
field.setWritable(capabilities.contains(DatacenterCapability.WRITE_MUTATION) ? 1 : 0);
field.setIndexed(primaryKeys.contains(field.getFieldName()) ? 1 : 0);
field.setFieldType(mapFieldType(columns.getInt("DATA_TYPE")));
field.setMetadataStatus(DatacenterMetadataStatus.ACTIVE.name());
field.setLastSeenAt(new java.util.Date());
field.setSensitivityLevel(DatacenterSensitivityLevel.PUBLIC.name());
field.setMetadataFingerprint(DatacenterMetadataIdentity.fieldFingerprint(field));
fields.add(field);
}
}
if (fields.isEmpty()) {
throw new BusinessException(
"无法读取数据表字段,请检查权限: " + tableName);
}
table.setMetadataFingerprint(DatacenterMetadataIdentity.tableFingerprint(
table.getTableName(), table.getTableKind(), fields));
detail.setFields(fields);
return detail;
});
} catch (Exception ex) {
throw DatacenterConnectorExceptionSupport.wrapAccessException("读取表详情失败", ex);
}
}
@Override
@@ -247,6 +526,11 @@ public abstract class AbstractJdbcConnector implements DatacenterConnector {
if (!capabilities.contains(DatacenterCapability.READ_QUERY)) {
throw new BusinessException("当前数据源暂不支持查询");
}
if (request == null || request.getPageSize() == null
|| request.getPageSize() < 1L
|| request.getPageSize() > 500L) {
throw new BusinessException("pageSize 必须在 1 到 500 之间");
}
try {
return withConnection(source, true, connection -> doQueryPage(connection, source, table, request));
} catch (Exception ex) {
@@ -325,7 +609,7 @@ public abstract class AbstractJdbcConnector implements DatacenterConnector {
String qualifiedTable = sqlDialect.qualifyTable(resolveCatalogName(source, request.getDatasetRef() == null ? null : request.getDatasetRef().getCatalogName()), resolvePhysicalTableName(table));
StringBuilder whereClause = new StringBuilder();
if (StrUtil.isNotBlank(request.getWhere())) {
whereClause.append(" WHERE ").append(request.getWhere());
throw new BusinessException("不支持原始 WHERE 条件,请使用结构化筛选条件");
} else if (!CollectionUtils.isEmpty(request.getFilters())) {
whereClause.append(" WHERE 1=1 ");
for (DatacenterQueryFilter filter : request.getFilters()) {
@@ -622,13 +906,9 @@ public abstract class AbstractJdbcConnector implements DatacenterConnector {
return items;
}
List<DatacenterCatalogMeta> matched = items.stream()
.filter(item -> configuredName.equalsIgnoreCase(item.getCatalogName()))
.filter(item -> configuredName.equals(item.getCatalogName()))
.collect(Collectors.toList());
return matched.isEmpty() ? items : matched;
}
private boolean containsCatalog(List<DatacenterCatalogMeta> items, String catalogName) {
return items.stream().anyMatch(item -> catalogName.equalsIgnoreCase(item.getCatalogName()));
return matched;
}
private DatacenterTableKind resolveTableKind(String tableType) {
@@ -639,8 +919,58 @@ public abstract class AbstractJdbcConnector implements DatacenterConnector {
if (currentTableName == null || targetTableName == null) {
return false;
}
return currentTableName.equals(targetTableName)
|| currentTableName.equalsIgnoreCase(targetTableName);
return currentTableName.equals(targetTableName);
}
/**
* 将精确标识符转换为 JDBC 元数据 Pattern转义驱动声明的通配符字符。
*
* @param metaData JDBC 元数据
* @param value 精确 Schema 或表名
* @return 可安全用于元数据 Pattern 参数的文本
* @throws SQLException 驱动无法返回转义字符时抛出
*/
private String metadataPattern(DatabaseMetaData metaData, String value)
throws SQLException {
if (value == null) {
return null;
}
String escape = metaData.getSearchStringEscape();
if (escape == null || escape.isEmpty()) {
return value;
}
return value.replace(escape, escape + escape)
.replace("%", escape + "%")
.replace("_", escape + "_");
}
/**
* 对元数据结果再次执行精确 Catalog、Schema 与表名校验,防御驱动忽略 Pattern 转义。
*
* @param resultSet 元数据结果集
* @param source 数据源
* @param catalogName 用户选择的物理命名空间
* @param tableName 目标表名
* @return 当前结果行是否属于精确目标
* @throws SQLException 读取元数据列失败时抛出
*/
private boolean matchesMetadataTable(
ResultSet resultSet,
DatacenterSource source,
String catalogName,
String tableName) throws SQLException {
if (!matchesTableName(resultSet.getString("TABLE_NAME"), tableName)) {
return false;
}
String expectedCatalog = resolveCatalogArgument(source, catalogName);
String expectedSchema = resolveSchemaArgument(source, catalogName);
String actualCatalog = resultSet.getString("TABLE_CAT");
String actualSchema = resultSet.getString("TABLE_SCHEM");
// Schema 型数据库已由当前连接锁定 Database部分 PostgreSQL 驱动不会回填 TABLE_CAT。
return (!usesCatalogNamespace()
|| expectedCatalog == null
|| expectedCatalog.equals(actualCatalog))
&& (expectedSchema == null || expectedSchema.equals(actualSchema));
}
private Integer mapFieldType(int jdbcType) {

View File

@@ -65,6 +65,38 @@ public class DatacenterTableBase extends DateEntity implements Serializable {
@Column(comment = "物理表名")
private String actualTable;
/** 物理对象稳定摘要。 */
@Column(comment = "物理对象稳定摘要")
private String physicalIdentityKey;
/** 表结构摘要。 */
@Column(comment = "表结构摘要")
private String metadataFingerprint;
/** 元数据状态。 */
@Column(comment = "元数据状态")
private String metadataStatus;
/** 最近发现时间。 */
@Column(comment = "最近发现时间")
private Date lastSeenAt;
/** 是否允许统一只读查询。 */
@Column(comment = "是否允许统一只读查询")
private Integer queryable;
/** 时间语义。 */
@Column(comment = "时间语义")
private String timeSemantics;
/** 默认时间字段 ID。 */
@Column(comment = "默认时间字段ID")
private BigInteger defaultTimeFieldId;
/** 表元数据版本。 */
@Column(comment = "表元数据版本")
private Long metadataRevision;
/**
* 表类型
*/
@@ -198,6 +230,39 @@ public class DatacenterTableBase extends DateEntity implements Serializable {
this.actualTable = actualTable;
}
public String getPhysicalIdentityKey() { return physicalIdentityKey; }
public void setPhysicalIdentityKey(String physicalIdentityKey) { this.physicalIdentityKey = physicalIdentityKey; }
public String getMetadataFingerprint() { return metadataFingerprint; }
public void setMetadataFingerprint(String metadataFingerprint) { this.metadataFingerprint = metadataFingerprint; }
public String getMetadataStatus() { return metadataStatus; }
public void setMetadataStatus(String metadataStatus) { this.metadataStatus = metadataStatus; }
public Date getLastSeenAt() { return lastSeenAt; }
public void setLastSeenAt(Date lastSeenAt) { this.lastSeenAt = lastSeenAt; }
public Integer getQueryable() { return queryable; }
public void setQueryable(Integer queryable) { this.queryable = queryable; }
public String getTimeSemantics() { return timeSemantics; }
public void setTimeSemantics(String timeSemantics) { this.timeSemantics = timeSemantics; }
@JsonSerialize(using = ToStringSerializer.class)
public BigInteger getDefaultTimeFieldId() { return defaultTimeFieldId; }
public void setDefaultTimeFieldId(BigInteger defaultTimeFieldId) { this.defaultTimeFieldId = defaultTimeFieldId; }
public Long getMetadataRevision() { return metadataRevision; }
public void setMetadataRevision(Long metadataRevision) { this.metadataRevision = metadataRevision; }
public String getTableKind() {
return tableKind;
}

View File

@@ -41,6 +41,10 @@ public class DatacenterTableFieldBase extends DateEntity implements Serializable
@Column(comment = "源字段名")
private String sourceColumnName;
/** JDBC 字段顺序。 */
@Column(comment = "JDBC字段顺序")
private Integer ordinalPosition;
/**
* 字段描述
*/
@@ -59,6 +63,14 @@ public class DatacenterTableFieldBase extends DateEntity implements Serializable
@Column(comment = "JDBC类型")
private String jdbcType;
/** java.sql.Types 数值。 */
@Column(comment = "JDBC类型编码")
private Integer jdbcTypeCode;
/** 数据库原生类型名。 */
@Column(comment = "数据库原生类型名")
private String nativeTypeName;
/**
* 精度
*/
@@ -71,6 +83,26 @@ public class DatacenterTableFieldBase extends DateEntity implements Serializable
@Column(comment = "小数位")
private Integer scale;
/** 字段元数据摘要。 */
@Column(comment = "字段元数据摘要")
private String metadataFingerprint;
/** 元数据状态。 */
@Column(comment = "元数据状态")
private String metadataStatus;
/** 最近发现时间。 */
@Column(comment = "最近发现时间")
private Date lastSeenAt;
/** 敏感级别。 */
@Column(comment = "敏感级别")
private String sensitivityLevel;
/** 脱敏策略。 */
@Column(comment = "脱敏策略")
private String maskingStrategy;
/**
* 是否必填
*/
@@ -165,6 +197,10 @@ public class DatacenterTableFieldBase extends DateEntity implements Serializable
this.sourceColumnName = sourceColumnName;
}
public Integer getOrdinalPosition() { return ordinalPosition; }
public void setOrdinalPosition(Integer ordinalPosition) { this.ordinalPosition = ordinalPosition; }
public String getFieldDesc() {
return fieldDesc;
}
@@ -189,6 +225,14 @@ public class DatacenterTableFieldBase extends DateEntity implements Serializable
this.jdbcType = jdbcType;
}
public Integer getJdbcTypeCode() { return jdbcTypeCode; }
public void setJdbcTypeCode(Integer jdbcTypeCode) { this.jdbcTypeCode = jdbcTypeCode; }
public String getNativeTypeName() { return nativeTypeName; }
public void setNativeTypeName(String nativeTypeName) { this.nativeTypeName = nativeTypeName; }
public Integer getPrecision() {
return precision;
}
@@ -205,6 +249,26 @@ public class DatacenterTableFieldBase extends DateEntity implements Serializable
this.scale = scale;
}
public String getMetadataFingerprint() { return metadataFingerprint; }
public void setMetadataFingerprint(String metadataFingerprint) { this.metadataFingerprint = metadataFingerprint; }
public String getMetadataStatus() { return metadataStatus; }
public void setMetadataStatus(String metadataStatus) { this.metadataStatus = metadataStatus; }
public Date getLastSeenAt() { return lastSeenAt; }
public void setLastSeenAt(Date lastSeenAt) { this.lastSeenAt = lastSeenAt; }
public String getSensitivityLevel() { return sensitivityLevel; }
public void setSensitivityLevel(String sensitivityLevel) { this.sensitivityLevel = sensitivityLevel; }
public String getMaskingStrategy() { return maskingStrategy; }
public void setMaskingStrategy(String maskingStrategy) { this.maskingStrategy = maskingStrategy; }
public Integer getRequired() {
return required;
}

View File

@@ -346,7 +346,7 @@ public class DatacenterExcelImportServiceImpl implements DatacenterExcelImportSe
queryRequest.setDatasetRef(registryService.resolveDatasetRef(table.getId()));
queryRequest.setSelectedColumns(table.getFields().stream().map(DatacenterTableField::getFieldName).toList());
final int[] rowIndex = {1};
totalRows += iterateRows(queryRequest, row -> {
totalRows += iterateRows(queryRequest, account, row -> {
org.apache.poi.ss.usermodel.Row excelRow = sheet.createRow(rowIndex[0]++);
for (int i = 0; i < table.getFields().size(); i++) {
Cell cell = excelRow.createCell(i);
@@ -434,7 +434,7 @@ public class DatacenterExcelImportServiceImpl implements DatacenterExcelImportSe
String baseName = resolveSplitPrefix(request, sourceTable.getTableName());
List<BigInteger> derivedIds = new ArrayList<>();
final Holder holder = new Holder();
long totalRows = iterateRows(buildFullQuery(sourceTable), row -> {
long totalRows = iterateRows(buildFullQuery(sourceTable), account, row -> {
if (holder.targetTable == null || holder.currentSize >= rowBatchSize) {
holder.batchNo++;
holder.targetTable = createDerivedTable(source, catalog, cloneFields(sourceTable.getFields()),
@@ -466,7 +466,7 @@ public class DatacenterExcelImportServiceImpl implements DatacenterExcelImportSe
String prefix = resolveSplitPrefix(request, sourceTable.getTableName());
Map<String, DatacenterTable> targets = new LinkedHashMap<>();
List<BigInteger> derivedIds = new ArrayList<>();
long totalRows = iterateRows(buildFullQuery(sourceTable), row -> {
long totalRows = iterateRows(buildFullQuery(sourceTable), account, row -> {
String fieldValue = stringify(row.get(splitField.getFieldName()));
String bucket = fieldValue == null || fieldValue.isBlank() ? "empty" : fieldValue;
DatacenterTable targetTable = targets.get(bucket);
@@ -543,7 +543,7 @@ public class DatacenterExcelImportServiceImpl implements DatacenterExcelImportSe
Map<String, JSONObject> mergedRows = new LinkedHashMap<>();
for (DatacenterTable table : tables) {
Map<String, String> mapping = fieldMappings.get(table.getId());
iterateRows(buildFullQuery(table), row -> {
iterateRows(buildFullQuery(table), account, row -> {
String joinValue = stringify(row.get(request.getJoinKey()));
if (joinValue == null || joinValue.isBlank()) {
return;
@@ -729,16 +729,22 @@ public class DatacenterExcelImportServiceImpl implements DatacenterExcelImportSe
}
private long copyRows(DatacenterQueryRequest queryRequest, RowMapper mapper, DatacenterTable targetTable, LoginAccount account) {
return iterateRows(queryRequest, row -> saveToTable(targetTable, mapper.map(row), account));
return iterateRows(
queryRequest,
account,
row -> saveToTable(targetTable, mapper.map(row), account));
}
private long iterateRows(DatacenterQueryRequest queryRequest, RowConsumer consumer) {
private long iterateRows(
DatacenterQueryRequest queryRequest,
LoginAccount account,
RowConsumer consumer) {
long total = 0L;
long pageNumber = 1L;
while (true) {
queryRequest.setPageNumber(pageNumber);
queryRequest.setPageSize(QUERY_BATCH_SIZE);
Page<Row> page = queryService.queryPage(queryRequest);
Page<Row> page = queryService.queryPage(queryRequest, account);
if (page.getRecords() == null || page.getRecords().isEmpty()) {
break;
}

View File

@@ -5,31 +5,46 @@ import tech.easyflow.datacenter.entity.DatacenterTableField;
import tech.easyflow.datacenter.meta.entity.DatacenterCatalog;
import tech.easyflow.datacenter.meta.entity.DatacenterDatasetVersion;
import tech.easyflow.datacenter.meta.entity.DatacenterDerivedTable;
import tech.easyflow.datacenter.meta.entity.DatacenterSource;
import tech.easyflow.datacenter.meta.model.DatacenterSourceView;
import java.util.ArrayList;
import java.util.List;
public class DatacenterSchemaResponse {
private DatasetRef datasetRef;
private DatacenterSource source;
private DatacenterSourceView source;
private DatacenterCatalog catalog;
private DatacenterTable table;
private List<DatacenterTableField> fields = new ArrayList<>();
private long fieldPageNumber = 1L;
private long fieldPageSize;
private boolean hasMoreFields;
private List<DatacenterDatasetVersion> versions = new ArrayList<>();
private List<DatacenterDerivedTable> upstreamLineage = new ArrayList<>();
private List<DatacenterDerivedTable> downstreamLineage = new ArrayList<>();
public DatasetRef getDatasetRef() { return datasetRef; }
public void setDatasetRef(DatasetRef datasetRef) { this.datasetRef = datasetRef; }
public DatacenterSource getSource() { return source; }
public void setSource(DatacenterSource source) { this.source = source; }
public DatacenterSourceView getSource() { return source; }
public void setSource(DatacenterSourceView source) { this.source = source; }
public DatacenterCatalog getCatalog() { return catalog; }
public void setCatalog(DatacenterCatalog catalog) { this.catalog = catalog; }
public DatacenterTable getTable() { return table; }
public void setTable(DatacenterTable table) { this.table = table; }
public List<DatacenterTableField> getFields() { return fields; }
public void setFields(List<DatacenterTableField> fields) { this.fields = fields; }
/** @return 当前字段页码 */
public long getFieldPageNumber() { return fieldPageNumber; }
/** @param fieldPageNumber 当前字段页码 */
public void setFieldPageNumber(long fieldPageNumber) { this.fieldPageNumber = fieldPageNumber; }
/** @return 当前字段页大小 */
public long getFieldPageSize() { return fieldPageSize; }
/** @param fieldPageSize 当前字段页大小 */
public void setFieldPageSize(long fieldPageSize) { this.fieldPageSize = fieldPageSize; }
/** @return 是否还有下一页字段 */
public boolean isHasMoreFields() { return hasMoreFields; }
/** @param hasMoreFields 是否还有下一页字段 */
public void setHasMoreFields(boolean hasMoreFields) { this.hasMoreFields = hasMoreFields; }
public List<DatacenterDatasetVersion> getVersions() { return versions; }
public void setVersions(List<DatacenterDatasetVersion> versions) { this.versions = versions; }
public List<DatacenterDerivedTable> getUpstreamLineage() { return upstreamLineage; }

View File

@@ -0,0 +1,9 @@
package tech.easyflow.datacenter.execution.model;
/**
* 管理端取消只读 SQL 查询请求。
*
* @param queryId 客户端在执行前生成的查询 UUID
*/
public record DatacenterSqlCancelRequest(String queryId) {
}

View File

@@ -0,0 +1,18 @@
package tech.easyflow.datacenter.execution.model;
/**
* SQL 控制台结果列。
*
* @param key 前端稳定列键
* @param label JDBC 列标签
* @param jdbcType JDBC 类型编码
* @param typeName 数据库类型名
* @param nullable 是否允许空值
*/
public record DatacenterSqlColumnView(
String key,
String label,
int jdbcType,
String typeName,
boolean nullable) {
}

View File

@@ -0,0 +1,18 @@
package tech.easyflow.datacenter.execution.model;
import java.math.BigInteger;
/**
* 管理端只读 SQL 控制台请求。
*
* @param sourceId 已激活数据源 ID
* @param queryId 客户端在执行前生成的查询 UUID
* @param sql 单条只读 SQL
* @param maxRows 最大返回行数
*/
public record DatacenterSqlConsoleRequest(
BigInteger sourceId,
String queryId,
String sql,
Integer maxRows) {
}

View File

@@ -0,0 +1,31 @@
package tech.easyflow.datacenter.execution.model;
import java.util.List;
import java.util.Map;
/**
* 有界 SQL 控制台结果。
*
* @param queryId 查询标识
* @param columns 结果列
* @param rows 结果行
* @param returnedRows 返回行数
* @param truncated 是否因上限截断
* @param durationMs 总耗时毫秒
*/
public record DatacenterSqlConsoleResult(
String queryId,
List<DatacenterSqlColumnView> columns,
List<Map<String, Object>> rows,
long returnedRows,
boolean truncated,
long durationMs) {
/**
* 防御性复制结果集合。
*/
public DatacenterSqlConsoleResult {
columns = List.copyOf(columns == null ? List.of() : columns);
rows = List.copyOf(rows == null ? List.of() : rows);
}
}

View File

@@ -5,6 +5,7 @@ import java.math.BigInteger;
public class DatasetRef implements java.io.Serializable {
private static final long serialVersionUID = 1L;
private BigInteger tenantId;
private BigInteger sourceId;
private BigInteger catalogId;
private String catalogName;
@@ -12,6 +13,8 @@ public class DatasetRef implements java.io.Serializable {
private String tableName;
private BigInteger versionId;
public BigInteger getTenantId() { return tenantId; }
public void setTenantId(BigInteger tenantId) { this.tenantId = tenantId; }
public BigInteger getSourceId() { return sourceId; }
public void setSourceId(BigInteger sourceId) { this.sourceId = sourceId; }
public BigInteger getCatalogId() { return catalogId; }

View File

@@ -2,6 +2,7 @@ package tech.easyflow.datacenter.execution.service;
import com.mybatisflex.core.paginate.Page;
import com.mybatisflex.core.row.Row;
import tech.easyflow.common.entity.LoginAccount;
import tech.easyflow.datacenter.execution.model.DatacenterQueryRequest;
import tech.easyflow.datacenter.execution.model.DatacenterSchemaResponse;
import tech.easyflow.datacenter.execution.model.DatacenterSqlQueryRequest;
@@ -20,6 +21,19 @@ public interface DatacenterDatasetQueryService {
*/
Page<Row> queryPage(DatacenterQueryRequest request);
/**
* 使用明确执行账号分页查询结构化数据集。
*
* @param request 查询请求
* @param account 执行账号
* @return 分页结果
*/
default Page<Row> queryPage(
DatacenterQueryRequest request,
LoginAccount account) {
return queryPage(request);
}
/**
* 执行原生 SQL 并返回完整结果。
*
@@ -35,9 +49,43 @@ public interface DatacenterDatasetQueryService {
* @param fetchSize JDBC 建议拉取行数
* @param consumer 单行消费者
*/
default void consumeBySql(
DatacenterSqlQueryRequest request,
int fetchSize,
Consumer<Row> consumer) {
consumeBySql(request, fetchSize, null, consumer);
}
/**
* 使用明确执行账号流式消费 SQL后台工作流据此绑定权威租户。
*
* @param request SQL 查询请求
* @param fetchSize JDBC 建议拉取行数
* @param account 执行账号
* @param consumer 单行消费者
*/
default void consumeBySql(
DatacenterSqlQueryRequest request,
int fetchSize,
LoginAccount account,
Consumer<Row> consumer) {
consumeBySql(request, fetchSize, account, null, consumer);
}
/**
* 使用明确执行账号和调用方 QueryId 流式消费 SQL。
*
* @param request SQL 查询请求
* @param fetchSize JDBC 建议拉取行数
* @param account 执行账号
* @param requestedQueryId 调用方预生成的查询 UUID为空时服务端生成
* @param consumer 单行消费者
*/
void consumeBySql(
DatacenterSqlQueryRequest request,
int fetchSize,
LoginAccount account,
String requestedQueryId,
Consumer<Row> consumer);
/**
@@ -48,6 +96,19 @@ public interface DatacenterDatasetQueryService {
*/
DatacenterSchemaResponse getSchema(DatasetRef datasetRef);
/**
* 获取有界字段页的数据集结构。
*
* @param datasetRef 数据集引用
* @param fieldPageNumber 字段页码
* @param fieldPageSize 字段页大小
* @return 数据集结构和当前字段页
*/
DatacenterSchemaResponse getSchema(
DatasetRef datasetRef,
Long fieldPageNumber,
Long fieldPageSize);
/**
* 仅解析数据集定位信息,不加载版本和血缘。
*

View File

@@ -1,22 +1,43 @@
package tech.easyflow.datacenter.execution.service.impl;
import com.easyagents.federation.sql.execute.SqlParameter;
import com.easyagents.federation.sql.execute.QueryId;
import com.mybatisflex.core.paginate.Page;
import com.mybatisflex.core.query.QueryWrapper;
import com.mybatisflex.core.row.Row;
import java.math.BigInteger;
import java.sql.Types;
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.UUID;
import java.util.function.Consumer;
import java.util.function.Function;
import java.util.stream.Collectors;
import org.springframework.stereotype.Service;
import org.springframework.util.CollectionUtils;
import org.springframework.util.StringUtils;
import tech.easyflow.common.web.exceptions.BusinessException;
import tech.easyflow.common.entity.LoginAccount;
import tech.easyflow.datacenter.connector.DatacenterConnector;
import tech.easyflow.datacenter.connector.DatacenterConnectorRegistry;
import tech.easyflow.datacenter.connector.support.AbstractInternalTableConnector;
import tech.easyflow.datacenter.audit.DatacenterQueryAudit;
import tech.easyflow.datacenter.audit.DatacenterQueryAuditService;
import tech.easyflow.datacenter.entity.DatacenterTable;
import tech.easyflow.datacenter.entity.DatacenterTableField;
import tech.easyflow.datacenter.execution.model.DatacenterQueryFilter;
import tech.easyflow.datacenter.execution.model.DatacenterQueryRequest;
import tech.easyflow.datacenter.execution.model.DatacenterQuerySort;
import tech.easyflow.datacenter.execution.model.DatacenterSchemaResponse;
import tech.easyflow.datacenter.execution.model.DatacenterSqlConsoleResult;
import tech.easyflow.datacenter.execution.model.DatacenterSqlQueryRequest;
import tech.easyflow.datacenter.execution.model.DatasetRef;
import tech.easyflow.datacenter.execution.service.DatacenterDatasetQueryService;
import tech.easyflow.datacenter.federation.DatacenterFederationQueryService;
import tech.easyflow.datacenter.mapper.DatacenterCatalogMapper;
import tech.easyflow.datacenter.mapper.DatacenterDatasetVersionMapper;
import tech.easyflow.datacenter.mapper.DatacenterDerivedTableMapper;
@@ -25,19 +46,14 @@ import tech.easyflow.datacenter.meta.entity.DatacenterCatalog;
import tech.easyflow.datacenter.meta.entity.DatacenterDatasetVersion;
import tech.easyflow.datacenter.meta.entity.DatacenterDerivedTable;
import tech.easyflow.datacenter.meta.entity.DatacenterSource;
import tech.easyflow.datacenter.meta.model.DatacenterSourceViews;
import tech.easyflow.datacenter.meta.enums.DatacenterMetadataStatus;
import tech.easyflow.datacenter.meta.enums.DatacenterSourceType;
import tech.easyflow.datacenter.meta.enums.DatacenterSensitivityLevel;
import tech.easyflow.datacenter.meta.service.DatacenterDatasetRegistryService;
import tech.easyflow.datacenter.utils.SqlSupportUtils;
import javax.annotation.Resource;
import java.math.BigInteger;
import java.util.LinkedHashMap;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.function.Function;
import java.util.function.Consumer;
import java.util.stream.Collectors;
@Service
public class DatacenterDatasetQueryServiceImpl implements DatacenterDatasetQueryService {
@@ -54,18 +70,34 @@ public class DatacenterDatasetQueryServiceImpl implements DatacenterDatasetQuery
private DatacenterDatasetVersionMapper datasetVersionMapper;
@Resource
private DatacenterDerivedTableMapper derivedTableMapper;
@Resource
private DatacenterFederationQueryService federationQueryService;
@Resource
private DatacenterQueryAuditService queryAuditService;
@Override
public Page<Row> queryPage(DatacenterQueryRequest request) {
throw new BusinessException("结构化查询必须提供执行账号");
}
/**
* {@inheritDoc}
*/
@Override
public Page<Row> queryPage(
DatacenterQueryRequest request,
LoginAccount account) {
if (request == null || request.getDatasetRef() == null) {
throw new BusinessException("datasetRef 不能为空");
}
normalizePage(request);
DatacenterTable table = resolveTable(request.getDatasetRef());
DatacenterSource source = registryService.getSourceRequired(table.getSourceId());
validateStructuredQueryTenant(
request.getDatasetRef(), table, source, account);
DatacenterCatalog catalog = registryService.getCatalogById(table.getCatalogId());
DatacenterTable queryTable = resolveQueryTable(table, request.getDatasetRef());
validateRequest(queryTable, request, source);
validateRequest(queryTable, request);
request.getDatasetRef().setSourceId(table.getSourceId());
request.getDatasetRef().setCatalogId(table.getCatalogId());
request.getDatasetRef().setTableId(table.getId());
@@ -73,15 +105,199 @@ public class DatacenterDatasetQueryServiceImpl implements DatacenterDatasetQuery
if (catalog != null) {
request.getDatasetRef().setCatalogName(catalog.getCatalogName());
}
if (isFederated(source)) {
return queryFederatedPage(
source, catalog, queryTable, request, account);
}
DatacenterConnector connector = connectorRegistry.getConnector(source.getSourceType());
return connector.queryPage(source, queryTable, request);
QueryId queryId = QueryId.create();
long startedNanos = System.nanoTime();
DatacenterQueryAudit audit = queryAuditService.start(
queryId.value(), source, structuredQueryDescription(queryTable, request),
structuredParameterSummary(request), account,
"DATASET_PAGE", table.getId().toString());
try {
Page<Row> result = connector.queryPage(source, queryTable, request);
long durationMs = elapsedMillis(startedNanos);
queryAuditService.succeed(
audit,
List.of(table.getTableName()),
structuredReferencedFields(request),
result.getRecords().size(),
false,
durationMs,
durationMs);
return result;
} catch (RuntimeException exception) {
queryAuditService.fail(
audit,
"DATASET_PAGE_QUERY_FAILED",
safeMessage(exception),
List.of(table.getTableName()),
structuredReferencedFields(request),
elapsedMillis(startedNanos));
throw exception;
}
}
/**
* 使用执行账号绑定结构化查询的权威租户。
*
* @param datasetRef 数据集引用
* @param table 已解析表
* @param source 已解析数据源
* @param account 执行账号
* @throws BusinessException 账号或租户边界不合法时抛出
*/
private void validateStructuredQueryTenant(
DatasetRef datasetRef,
DatacenterTable table,
DatacenterSource source,
LoginAccount account) {
if (account == null || account.getTenantId() == null) {
throw new BusinessException("结构化查询缺少执行账号租户");
}
BigInteger tenantId = account.getTenantId();
if ((datasetRef.getTenantId() != null
&& !tenantId.equals(datasetRef.getTenantId()))
|| !tenantId.equals(table.getTenantId())
|| !tenantId.equals(source.getTenantId())) {
throw new BusinessException("数据集不属于当前租户");
}
datasetRef.setTenantId(tenantId);
}
private String structuredQueryDescription(
DatacenterTable table,
DatacenterQueryRequest request) {
StringBuilder sql = new StringBuilder("SELECT ")
.append(String.join(", ", request.getSelectedColumns()))
.append(" FROM ").append(table.getTableName());
if (!CollectionUtils.isEmpty(request.getFilters())) {
sql.append(" WHERE ");
for (int index = 0; index < request.getFilters().size(); index++) {
DatacenterQueryFilter filter = request.getFilters().get(index);
if (index > 0) {
sql.append(" AND ");
}
String operator = normalizeFilterOperator(filter.getOperator());
sql.append(filter.getColumn()).append(' ');
switch (operator) {
case "IS_NULL" -> sql.append("IS NULL");
case "IN" -> sql.append("IN (?)");
default -> sql.append(operator).append(" ?");
}
}
}
if (!CollectionUtils.isEmpty(request.getSorts())) {
sql.append(" ORDER BY ");
for (int index = 0; index < request.getSorts().size(); index++) {
DatacenterQuerySort sort = request.getSorts().get(index);
if (index > 0) {
sql.append(", ");
}
sql.append(sort.getColumn()).append(' ')
.append("DESC".equalsIgnoreCase(sort.getDirection())
? "DESC" : "ASC");
}
}
return sql.toString();
}
/**
* 构建不包含实际值的结构化查询参数摘要。
*
* @param request 查询请求
* @return 脱敏参数摘要
*/
private Map<String, Object> structuredParameterSummary(
DatacenterQueryRequest request) {
List<Map<String, Object>> filters = new ArrayList<>();
if (!CollectionUtils.isEmpty(request.getFilters())) {
for (DatacenterQueryFilter filter : request.getFilters()) {
String operator = normalizeFilterOperator(filter.getOperator());
int valueCount = "IN".equals(operator)
? (filter.getValues() == null ? 0 : filter.getValues().size())
: "IS_NULL".equals(operator) ? 0 : 1;
filters.add(Map.of(
"column", filter.getColumn(),
"operator", operator,
"valueCount", valueCount));
}
}
return Map.of(
"filters", List.copyOf(filters),
"pageNumber", request.getPageNumber(),
"pageSize", request.getPageSize());
}
/**
* 收集结构化查询实际引用的投影、筛选和排序字段。
*
* @param request 查询请求
* @return 去重后的字段列表
*/
private List<String> structuredReferencedFields(
DatacenterQueryRequest request) {
Set<String> fields = new LinkedHashSet<>(request.getSelectedColumns());
if (!CollectionUtils.isEmpty(request.getFilters())) {
request.getFilters().stream()
.map(DatacenterQueryFilter::getColumn)
.forEach(fields::add);
}
if (!CollectionUtils.isEmpty(request.getSorts())) {
request.getSorts().stream()
.map(DatacenterQuerySort::getColumn)
.forEach(fields::add);
}
return List.copyOf(fields);
}
/**
* 将结构化筛选操作符归一化为审计中的稳定名称。
*
* @param operator 原始操作符
* @return 大写稳定操作符
*/
private String normalizeFilterOperator(String operator) {
return trimToNull(operator) == null
? "EQ" : operator.trim().toUpperCase(java.util.Locale.ROOT);
}
@Override
public List<Row> queryBySql(DatacenterSqlQueryRequest request) {
ResolvedSqlQuery query = resolveSqlQuery(request);
return query.connector.queryBySql(
query.source, query.sql);
if (isFederated(query.source)) {
DatacenterSqlConsoleResult result = federationQueryService.execute(
query.source,
query.sql,
List.of(),
1_000,
null,
"DATASET",
request.getDatasetRef().getTableId() == null
? null : request.getDatasetRef().getTableId().toString());
if (result.truncated()) {
throw new BusinessException("数据集查询结果超过返回上限,请使用流式消费");
}
return toRows(result);
}
QueryId queryId = QueryId.create();
long startedNanos = System.nanoTime();
DatacenterQueryAudit audit = queryAuditService.start(
queryId.value(), query.source, query.sql, Map.of(), null,
"DATASET", request.getDatasetRef().getTableId().toString());
try {
List<Row> rows = query.connector.queryBySql(query.source, query.sql);
long durationMs = elapsedMillis(startedNanos);
queryAuditService.succeed(audit, query.referencedTables, List.of(),
rows.size(), false, durationMs, durationMs);
return rows;
} catch (RuntimeException exception) {
queryAuditService.fail(audit, "DATASET_QUERY_FAILED", safeMessage(exception),
query.referencedTables, List.of(), elapsedMillis(startedNanos));
throw exception;
}
}
/**
@@ -91,12 +307,38 @@ public class DatacenterDatasetQueryServiceImpl implements DatacenterDatasetQuery
public void consumeBySql(
DatacenterSqlQueryRequest request,
int fetchSize,
LoginAccount account,
String requestedQueryId,
Consumer<Row> consumer) {
if (fetchSize <= 0 || consumer == null) {
throw new IllegalArgumentException(
"fetchSize and consumer must be valid");
}
ResolvedSqlQuery query = resolveSqlQuery(request);
ResolvedSqlQuery query = resolveSqlQuery(request, account);
if (isFederated(query.source)) {
int maxRows = Integer.getInteger(
"easyflow.datacenter.query.max-rows", 1_000_000);
long maxBytes = Long.getLong(
"easyflow.datacenter.query.max-bytes", 512L * 1024L * 1024L);
federationQueryService.consume(
query.source,
query.sql,
List.of(),
fetchSize,
maxRows,
maxBytes,
account,
"DATASET",
request.getDatasetRef().getTableId() == null
? null : request.getDatasetRef().getTableId().toString(),
requestedQueryId,
sourceRow -> {
Row row = new Row();
sourceRow.forEach(row::put);
consumer.accept(row);
});
return;
}
int maxRows = Integer.getInteger(
"easyflow.datacenter.query.max-rows",
1_000_000);
@@ -105,34 +347,48 @@ public class DatacenterDatasetQueryServiceImpl implements DatacenterDatasetQuery
512L * 1024L * 1024L);
long[] accumulatedRows = {0L};
long[] accumulatedBytes = {0L};
query.connector.consumeBySql(
query.source,
query.sql,
fetchSize,
row -> {
accumulatedRows[0]++;
if (maxRows > 0
&& accumulatedRows[0] > maxRows) {
throw new BusinessException(
"数据集查询结果超过行数上限: "
+ maxRows);
}
for (Map.Entry<String, Object> entry
: row.entrySet()) {
accumulatedBytes[0] +=
estimateQueryValueBytes(
entry.getKey(),
entry.getValue());
if (maxBytes > 0L
&& accumulatedBytes[0] > maxBytes) {
QueryId queryId = resolveQueryId(requestedQueryId);
long startedNanos = System.nanoTime();
DatacenterQueryAudit audit = queryAuditService.start(
queryId.value(), query.source, query.sql, Map.of(), account,
"DATASET", request.getDatasetRef().getTableId().toString());
try {
query.connector.consumeBySql(
query.source,
query.sql,
fetchSize,
row -> {
accumulatedRows[0]++;
if (maxRows > 0
&& accumulatedRows[0] > maxRows) {
throw new BusinessException(
"数据集查询结果超过字节上限: "
+ maxBytes);
"数据集查询结果超过行数上限: "
+ maxRows);
}
for (Map.Entry<String, Object> entry
: row.entrySet()) {
accumulatedBytes[0] +=
estimateQueryValueBytes(
entry.getKey(),
entry.getValue());
if (maxBytes > 0L
&& accumulatedBytes[0] > maxBytes) {
throw new BusinessException(
"数据集查询结果超过字节上限: "
+ maxBytes);
}
}
consumer.accept(row);
}
consumer.accept(row);
}
);
);
long durationMs = elapsedMillis(startedNanos);
queryAuditService.succeed(audit, query.referencedTables, List.of(),
accumulatedRows[0], false, durationMs, durationMs);
} catch (RuntimeException exception) {
queryAuditService.fail(audit, "DATASET_QUERY_FAILED", safeMessage(exception),
query.referencedTables, List.of(), elapsedMillis(startedNanos));
throw exception;
}
}
/**
@@ -171,6 +427,19 @@ public class DatacenterDatasetQueryServiceImpl implements DatacenterDatasetQuery
*/
private ResolvedSqlQuery resolveSqlQuery(
DatacenterSqlQueryRequest request) {
return resolveSqlQuery(request, null);
}
/**
* 校验请求、执行账号租户并解析实际连接器与可执行 SQL。
*
* @param request SQL 查询请求
* @param account 后台执行账号;普通同步调用可为空
* @return 已解析查询
*/
private ResolvedSqlQuery resolveSqlQuery(
DatacenterSqlQueryRequest request,
LoginAccount account) {
if (request == null || request.getDatasetRef() == null) {
throw new BusinessException("datasetRef 不能为空");
}
@@ -183,8 +452,28 @@ public class DatacenterDatasetQueryServiceImpl implements DatacenterDatasetQuery
throw new BusinessException("缺少连接服务配置");
}
DatacenterSource source = registryService.getSourceRequired(datasetRef.getSourceId());
BigInteger authoritativeTenantId = account != null && account.getTenantId() != null
? account.getTenantId() : datasetRef.getTenantId();
if (authoritativeTenantId == null) {
throw new BusinessException("数据集绑定缺少租户信息");
}
if ((datasetRef.getTenantId() != null
&& !datasetRef.getTenantId().equals(authoritativeTenantId))
|| !authoritativeTenantId.equals(source.getTenantId())) {
throw new BusinessException("数据集不属于当前租户");
}
datasetRef.setTenantId(authoritativeTenantId);
BigInteger catalogId = resolveRequestedCatalogId(datasetRef);
List<DatacenterTable> managedTables = registryService.listManagedTables(datasetRef.getSourceId(), catalogId);
if (datasetRef.getTableId() == null) {
throw new BusinessException("数据集 SQL 查询必须绑定具体表");
}
DatacenterTable boundTable = registryService.getTableWithFields(datasetRef.getTableId());
if (!datasetRef.getSourceId().equals(boundTable.getSourceId())
|| (catalogId != null && !catalogId.equals(boundTable.getCatalogId()))
|| !authoritativeTenantId.equals(boundTable.getTenantId())) {
throw new BusinessException("数据集引用与绑定表不一致");
}
List<DatacenterTable> managedTables = List.of(boundTable);
if (CollectionUtils.isEmpty(managedTables)) {
throw new BusinessException("当前连接下没有已接入表");
}
@@ -198,12 +487,15 @@ public class DatacenterDatasetQueryServiceImpl implements DatacenterDatasetQuery
// 内部连接的 catalog 是逻辑命名空间,底层项目 MySQL 只执行物理表名。
SqlSupportUtils.ResolvedSql resolvedSql =
connector instanceof AbstractInternalTableConnector
|| DatacenterSourceType.PROJECT_MYSQL.name()
.equals(source.getSourceType())
? SqlSupportUtils.resolveInternalMysql(sql, sqlTables)
: SqlSupportUtils.resolve(sql, sqlTables);
return new ResolvedSqlQuery(
source,
connector,
resolvedSql.getExecutableSql());
isFederated(source) ? sql : resolvedSql.getExecutableSql(),
resolvedSql.getLogicalTables());
}
/**
@@ -212,30 +504,123 @@ public class DatacenterDatasetQueryServiceImpl implements DatacenterDatasetQuery
* @param source 数据源
* @param connector 数据连接器
* @param sql 可执行 SQL
* @param referencedTables 已校验引用表
*/
private record ResolvedSqlQuery(
DatacenterSource source,
DatacenterConnector connector,
String sql) {
String sql,
List<String> referencedTables) {
}
private long elapsedMillis(long startedNanos) {
return Math.max(0L, (System.nanoTime() - startedNanos) / 1_000_000L);
}
/**
* 校验调用方查询 UUID内部连接也使用同一标识完成审计关联。
*
* @param requestedQueryId 调用方标识;为空时生成
* @return 查询标识
*/
private QueryId resolveQueryId(String requestedQueryId) {
if (!StringUtils.hasText(requestedQueryId)) {
return QueryId.create();
}
try {
return new QueryId(UUID.fromString(
requestedQueryId.trim()).toString());
} catch (IllegalArgumentException exception) {
throw new BusinessException("queryId 必须是 UUID");
}
}
private String safeMessage(RuntimeException exception) {
String message = exception.getMessage();
if (message == null || message.isBlank()) {
return "数据集查询失败";
}
return message.length() <= 500 ? message : message.substring(0, 500);
}
@Override
public DatacenterSchemaResponse getSchema(DatasetRef datasetRef) {
return getSchema(datasetRef, null, null);
}
/**
* {@inheritDoc}
*/
@Override
public DatacenterSchemaResponse getSchema(
DatasetRef datasetRef,
Long fieldPageNumber,
Long fieldPageSize) {
DatacenterTable table = resolveTable(datasetRef);
List<DatacenterTableField> allFields = List.copyOf(table.getFields());
long actualPage = fieldPageNumber == null ? 1L : Math.max(1L, fieldPageNumber);
long actualSize = fieldPageSize == null
? Math.max(1L, allFields.size())
: Math.min(200L, Math.max(1L, fieldPageSize));
List<DatacenterTableField> pageFields = sliceFields(
allFields, actualPage, actualSize);
long offset = safePageOffset(actualPage, actualSize);
boolean hasMoreFields = offset < allFields.size()
&& offset + pageFields.size() < allFields.size();
// 字段只通过顶层分页字段返回,避免在 table 内重复序列化完整列表。
table.setFields(List.of());
DatacenterSource source = registryService.getSourceRequired(table.getSourceId());
DatacenterCatalog catalog = registryService.getCatalogById(table.getCatalogId());
DatacenterSchemaResponse response = new DatacenterSchemaResponse();
response.setDatasetRef(registryService.resolveDatasetRef(table.getId()));
response.setSource(source);
response.setSource(DatacenterSourceViews.from(source));
response.setCatalog(catalog);
response.setTable(table);
response.setFields(table.getFields());
response.setFields(pageFields);
response.setFieldPageNumber(actualPage);
response.setFieldPageSize(actualSize);
response.setHasMoreFields(hasMoreFields);
response.setVersions(listVersions(table.getId()));
response.setUpstreamLineage(listUpstream(table.getId()));
response.setDownstreamLineage(listDownstream(table.getId()));
return response;
}
/**
* 截取字段页并避免向响应暴露可变子列表。
*
* @param fields 完整字段列表
* @param pageNumber 页码
* @param pageSize 页大小
* @return 当前页字段
*/
private List<DatacenterTableField> sliceFields(
List<DatacenterTableField> fields,
long pageNumber,
long pageSize) {
long offset = safePageOffset(pageNumber, pageSize);
if (offset >= fields.size()) {
return List.of();
}
int fromIndex = Math.toIntExact(offset);
int toIndex = (int) Math.min((long) fields.size(), offset + pageSize);
return List.copyOf(fields.subList(fromIndex, toIndex));
}
/**
* 计算字段页偏移并防止长整型乘法溢出。
*
* @param pageNumber 页码
* @param pageSize 页大小
* @return 安全偏移;溢出时返回最大长整型
*/
private long safePageOffset(long pageNumber, long pageSize) {
if (pageNumber - 1L > Long.MAX_VALUE / pageSize) {
return Long.MAX_VALUE;
}
return (pageNumber - 1L) * pageSize;
}
/**
* {@inheritDoc}
*/
@@ -245,8 +630,8 @@ public class DatacenterDatasetQueryServiceImpl implements DatacenterDatasetQuery
DatacenterSchemaResponse response =
new DatacenterSchemaResponse();
response.setDatasetRef(datasetRef);
response.setSource(
registryService.getSourceRequired(table.getSourceId()));
response.setSource(DatacenterSourceViews.from(
registryService.getSourceRequired(table.getSourceId())));
response.setCatalog(
registryService.getCatalogById(table.getCatalogId()));
response.setTable(table);
@@ -319,13 +704,29 @@ public class DatacenterDatasetQueryServiceImpl implements DatacenterDatasetQuery
DatacenterCatalog catalog = catalogId == null
? null
: catalogsById.get(catalogId);
List<String> knownColumns = table.getFields().stream()
.filter(field -> DatacenterMetadataStatus.ACTIVE.name()
.equals(field.getMetadataStatus()))
.map(this::sourceColumnName)
.toList();
List<String> queryableColumns = table.getFields().stream()
.filter(this::isQueryable)
.map(this::sourceColumnName)
.toList();
return new SqlSupportUtils.ManagedTable(
catalog == null ? null : catalog.getCatalogName(),
table.getTableName(),
resolvePhysicalTableName(table)
resolvePhysicalTableName(table),
knownColumns,
queryableColumns
);
}
private String sourceColumnName(DatacenterTableField field) {
String sourceColumnName = trimToNull(field.getSourceColumnName());
return sourceColumnName == null ? field.getFieldName() : sourceColumnName;
}
/**
* 一次批量加载 SQL 白名单表关联的目录,避免逐表查询。
*
@@ -367,6 +768,153 @@ public class DatacenterDatasetQueryServiceImpl implements DatacenterDatasetQuery
return actualTable != null ? actualTable : table.getTableName();
}
/**
* 使用逻辑 Schema 构建参数化 ANSI SQL再由 Calcite 转换为目标方言。
*/
private Page<Row> queryFederatedPage(
DatacenterSource source,
DatacenterCatalog catalog,
DatacenterTable table,
DatacenterQueryRequest request,
LoginAccount account) {
if (catalog == null || catalog.getLogicalSchemaName() == null) {
throw new BusinessException("数据集缺少 Federation Schema 绑定");
}
if (request.getPageSize() > 500L) {
throw new BusinessException("外部数据源单页最多返回 500 行");
}
Map<String, DatacenterTableField> fields = table.getFields().stream()
.collect(Collectors.toMap(
DatacenterTableField::getFieldName,
Function.identity(),
(first, ignored) -> first,
LinkedHashMap::new));
FederatedWhere where = buildFederatedWhere(request.getFilters(), fields);
String from = quote(catalog.getLogicalSchemaName()) + "." + quote(table.getTableName());
String countSql = "SELECT COUNT(*) AS total FROM " + from + where.sql();
DatacenterSqlConsoleResult countResult = federationQueryService.execute(
source, countSql, where.parameters(), 1, account,
"DATASET", table.getId().toString());
long total = countResult.rows().isEmpty()
? 0L : Long.parseLong(String.valueOf(countResult.rows().get(0).get("c1")));
if (total == 0L) {
return new Page<>(List.of(), request.getPageNumber(), request.getPageSize(), 0L);
}
String columns = request.getSelectedColumns().stream()
.map(name -> quote(requireField(fields, name).getSourceColumnName()))
.collect(Collectors.joining(", "));
String orderBy = buildFederatedOrderBy(request.getSorts(), fields);
long offset = (request.getPageNumber() - 1L) * request.getPageSize();
String pageSql = "SELECT " + columns + " FROM " + from + where.sql() + orderBy
+ " OFFSET " + offset + " ROWS FETCH NEXT " + request.getPageSize() + " ROWS ONLY";
DatacenterSqlConsoleResult pageResult = federationQueryService.execute(
source, pageSql, where.parameters(), request.getPageSize().intValue(),
account, "DATASET", table.getId().toString());
return new Page<>(toRows(pageResult), request.getPageNumber(), request.getPageSize(), total);
}
private FederatedWhere buildFederatedWhere(
List<DatacenterQueryFilter> filters,
Map<String, DatacenterTableField> fields) {
if (CollectionUtils.isEmpty(filters)) {
return new FederatedWhere("", List.of());
}
StringBuilder sql = new StringBuilder(" WHERE 1 = 1");
List<SqlParameter> parameters = new ArrayList<>();
for (DatacenterQueryFilter filter : filters) {
DatacenterTableField field = requireField(fields, filter.getColumn());
String column = quote(field.getSourceColumnName());
String operator = trimToNull(filter.getOperator()) == null
? "EQ" : filter.getOperator().trim().toUpperCase(java.util.Locale.ROOT);
switch (operator) {
case "EQ", "GT", "GTE", "LT", "LTE", "LIKE" -> {
String sqlOperator = switch (operator) {
case "GT" -> ">";
case "GTE" -> ">=";
case "LT" -> "<";
case "LTE" -> "<=";
case "LIKE" -> "LIKE";
default -> "=";
};
sql.append(" AND ").append(column).append(' ').append(sqlOperator).append(" ?");
Object value = "LIKE".equals(operator)
? "%" + filter.getValue() + "%" : filter.getValue();
parameters.add(parameter(field, value));
}
case "IS_NULL" -> sql.append(" AND ").append(column).append(" IS NULL");
case "IN" -> {
List<Object> values = filter.getValues() == null ? List.of() : filter.getValues();
if (values.isEmpty()) {
sql.append(" AND 1 = 0");
} else {
sql.append(" AND ").append(column).append(" IN (")
.append(values.stream().map(ignored -> "?").collect(Collectors.joining(",")))
.append(')');
values.forEach(value -> parameters.add(parameter(field, value)));
}
}
default -> throw new BusinessException("不支持的过滤操作: " + operator);
}
}
return new FederatedWhere(sql.toString(), List.copyOf(parameters));
}
private String buildFederatedOrderBy(
List<DatacenterQuerySort> sorts,
Map<String, DatacenterTableField> fields) {
if (CollectionUtils.isEmpty(sorts)) {
return "";
}
return " ORDER BY " + sorts.stream()
.map(sort -> quote(requireField(fields, sort.getColumn()).getSourceColumnName())
+ ("DESC".equalsIgnoreCase(sort.getDirection()) ? " DESC" : " ASC"))
.collect(Collectors.joining(", "));
}
private DatacenterTableField requireField(
Map<String, DatacenterTableField> fields,
String name) {
DatacenterTableField field = fields.get(name);
if (field == null || !isEnabled(field.getQueryable())) {
throw new BusinessException("字段不可查询: " + name);
}
return field;
}
private SqlParameter parameter(DatacenterTableField field, Object value) {
int jdbcType = field.getJdbcTypeCode() == null ? Types.VARCHAR : field.getJdbcTypeCode();
return value == null ? new SqlParameter(jdbcType, null) : new SqlParameter(jdbcType, value);
}
private String quote(String identifier) {
if (!StringUtils.hasText(identifier)) {
throw new BusinessException("数据集物理名称为空");
}
return '"' + identifier.replace("\"", "\"\"") + '"';
}
private List<Row> toRows(DatacenterSqlConsoleResult result) {
List<Row> rows = new ArrayList<>(result.rows().size());
for (Map<String, Object> sourceRow : result.rows()) {
Row row = new Row();
for (int index = 0; index < result.columns().size(); index++) {
var column = result.columns().get(index);
row.put(column.label(), sourceRow.get(column.key()));
}
rows.add(row);
}
return rows;
}
private boolean isFederated(DatacenterSource source) {
return source != null && (DatacenterSourceType.MYSQL.name().equals(source.getSourceType())
|| DatacenterSourceType.POSTGRESQL.name().equals(source.getSourceType()));
}
private record FederatedWhere(String sql, List<SqlParameter> parameters) {
}
private void normalizePage(DatacenterQueryRequest request) {
if (request.getPageNumber() == null || request.getPageNumber() < 1L) {
request.setPageNumber(1L);
@@ -374,6 +922,9 @@ public class DatacenterDatasetQueryServiceImpl implements DatacenterDatasetQuery
if (request.getPageSize() == null || request.getPageSize() < 1L) {
throw new BusinessException("pageSize 必须大于 0");
}
if (request.getPageSize() > 500L) {
throw new BusinessException("单页最多返回 500 行");
}
}
private DatacenterTable resolveQueryTable(DatacenterTable table, DatasetRef datasetRef) {
@@ -400,7 +951,7 @@ public class DatacenterDatasetQueryServiceImpl implements DatacenterDatasetQuery
return queryTable;
}
private void validateRequest(DatacenterTable table, DatacenterQueryRequest request, DatacenterSource source) {
private void validateRequest(DatacenterTable table, DatacenterQueryRequest request) {
Map<String, DatacenterTableField> fieldMap = new LinkedHashMap<>();
for (DatacenterTableField field : table.getFields()) {
fieldMap.put(field.getFieldName(), field);
@@ -408,22 +959,25 @@ public class DatacenterDatasetQueryServiceImpl implements DatacenterDatasetQuery
if (!CollectionUtils.isEmpty(request.getSelectedColumns())) {
for (String column : request.getSelectedColumns()) {
DatacenterTableField field = fieldMap.get(column);
if (field == null || !isEnabled(field.getQueryable())) {
if (field == null || !isQueryable(field)) {
throw new BusinessException("字段不可查询: " + column);
}
}
} else {
request.setSelectedColumns(
table.getFields().stream()
.filter(field -> isEnabled(field.getQueryable()))
.filter(this::isQueryable)
.map(DatacenterTableField::getFieldName)
.toList()
);
}
if (CollectionUtils.isEmpty(request.getSelectedColumns())) {
throw new BusinessException("当前数据集没有可查询字段");
}
if (!CollectionUtils.isEmpty(request.getFilters())) {
request.getFilters().forEach(filter -> {
DatacenterTableField field = fieldMap.get(filter.getColumn());
if (field == null || !isEnabled(field.getQueryable())) {
if (field == null || !isQueryable(field)) {
throw new BusinessException("字段不可过滤: " + filter.getColumn());
}
});
@@ -431,18 +985,13 @@ public class DatacenterDatasetQueryServiceImpl implements DatacenterDatasetQuery
if (!CollectionUtils.isEmpty(request.getSorts())) {
request.getSorts().forEach(sort -> {
DatacenterTableField field = fieldMap.get(sort.getColumn());
if (field == null || !isEnabled(field.getSortable())) {
if (field == null || !isQueryable(field) || !isEnabled(field.getSortable())) {
throw new BusinessException("字段不可排序: " + sort.getColumn());
}
});
}
if (request.getWhere() != null && !request.getWhere().isBlank()) {
boolean allowLegacyWhere = "PROJECT_MYSQL".equals(source.getSourceType())
|| "MYSQL".equals(source.getSourceType())
|| "POSTGRESQL".equals(source.getSourceType());
if (!allowLegacyWhere) {
throw new BusinessException("当前数据源仅支持结构化 DSL 查询");
}
throw new BusinessException("不支持原始 WHERE 条件,请使用结构化筛选条件");
}
}
@@ -450,6 +999,13 @@ public class DatacenterDatasetQueryServiceImpl implements DatacenterDatasetQuery
return value == null || value == 1;
}
private boolean isQueryable(DatacenterTableField field) {
return field != null
&& isEnabled(field.getQueryable())
&& DatacenterMetadataStatus.ACTIVE.name().equals(field.getMetadataStatus())
&& DatacenterSensitivityLevel.PUBLIC.name().equals(field.getSensitivityLevel());
}
private String trimToNull(String value) {
if (!StringUtils.hasText(value)) {
return null;

View File

@@ -38,7 +38,7 @@ public class DatacenterDatasetWriteServiceImpl implements DatacenterDatasetWrite
*/
@Override
public void saveRow(DatasetRef datasetRef, JSONObject data, LoginAccount account) {
DatacenterTable table = resolveTable(datasetRef);
DatacenterTable table = resolveTable(datasetRef, account);
DatacenterSource source = registryService.getSourceRequired(table.getSourceId());
DatacenterConnector connector = connectorRegistry.getConnector(source.getSourceType());
connector.saveRow(source, table, data, account);
@@ -52,7 +52,7 @@ public class DatacenterDatasetWriteServiceImpl implements DatacenterDatasetWrite
if (rows == null || rows.isEmpty()) {
return;
}
DatacenterTable table = resolveTable(datasetRef);
DatacenterTable table = resolveTable(datasetRef, account);
DatacenterSource source = registryService.getSourceRequired(table.getSourceId());
DatacenterConnector connector = connectorRegistry.getConnector(source.getSourceType());
connector.saveRows(source, table, rows, account, Math.max(1, batchSize));
@@ -70,7 +70,7 @@ public class DatacenterDatasetWriteServiceImpl implements DatacenterDatasetWrite
if (rows == null || rows.isEmpty()) {
return true;
}
DatacenterTable table = resolveTable(datasetRef);
DatacenterTable table = resolveTable(datasetRef, account);
DatacenterSource source = registryService.getSourceRequired(table.getSourceId());
DatacenterConnector connector = connectorRegistry.getConnector(source.getSourceType());
String payloadHash = sha256Rows(rows);
@@ -95,17 +95,27 @@ public class DatacenterDatasetWriteServiceImpl implements DatacenterDatasetWrite
*/
@Override
public void deleteRow(DatasetRef datasetRef, BigInteger id, LoginAccount account) {
DatacenterTable table = resolveTable(datasetRef);
DatacenterTable table = resolveTable(datasetRef, account);
DatacenterSource source = registryService.getSourceRequired(table.getSourceId());
DatacenterConnector connector = connectorRegistry.getConnector(source.getSourceType());
connector.deleteRow(source, table, id, account);
}
private DatacenterTable resolveTable(DatasetRef datasetRef) {
private DatacenterTable resolveTable(DatasetRef datasetRef, LoginAccount account) {
if (datasetRef == null || datasetRef.getTableId() == null) {
throw new BusinessException("缺少 tableId");
}
return registryService.getTableWithFields(datasetRef.getTableId());
if (account == null || account.getTenantId() == null) {
throw new BusinessException("数据集写入缺少执行租户");
}
DatacenterTable table = registryService.getTableWithFields(datasetRef.getTableId());
if (!account.getTenantId().equals(table.getTenantId())
|| (datasetRef.getTenantId() != null
&& !account.getTenantId().equals(datasetRef.getTenantId()))) {
throw new BusinessException("数据集不属于当前租户");
}
datasetRef.setTenantId(account.getTenantId());
return table;
}
/**

View File

@@ -0,0 +1,308 @@
package tech.easyflow.datacenter.federation;
import com.easyagents.federation.sql.api.FederationSqlErrorCode;
import com.easyagents.federation.sql.api.FederationSqlException;
import com.easyagents.federation.sql.execute.FederationQueryAdmissionController;
import com.easyagents.federation.sql.execute.FederationQueryPermit;
import com.easyagents.federation.sql.execute.LocalFederationQueryAdmissionController;
import com.easyagents.federation.sql.execute.QueryId;
import com.easyagents.federation.sql.source.SourceId;
import java.time.Duration;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.function.BooleanSupplier;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
/**
* 对查询执行租户、数据源与节点三级本地并发准入。
*/
@Component
public class DatacenterFederationAdmissionController
implements FederationQueryAdmissionController {
private final LocalFederationQueryAdmissionController globalAdmission;
private final int maximumConcurrentQueriesPerTenant;
private final int maximumConcurrentQueriesPerSource;
private final Map<String, AdmissionEntry>
tenantAdmissions = new ConcurrentHashMap<>();
private final Map<SourceId, AdmissionEntry>
sourceAdmissions = new ConcurrentHashMap<>();
private final Object lifecycleLock = new Object();
private final AtomicBoolean closed = new AtomicBoolean();
/**
* 创建三级准入控制器。
*
* @param maximumConcurrentQueries 节点查询上限
* @param maximumConcurrentQueriesPerTenant 单租户查询上限
* @param maximumConcurrentQueriesPerSource 单数据源查询上限
*/
public DatacenterFederationAdmissionController(
@Value("${easyflow.datacenter.query-admission.maximum-concurrent-queries:64}")
int maximumConcurrentQueries,
@Value("${easyflow.datacenter.query-admission.maximum-concurrent-queries-per-tenant:20}")
int maximumConcurrentQueriesPerTenant,
@Value("${easyflow.datacenter.query-admission.maximum-concurrent-queries-per-source:10}")
int maximumConcurrentQueriesPerSource) {
if (maximumConcurrentQueries <= 0
|| maximumConcurrentQueriesPerTenant <= 0
|| maximumConcurrentQueriesPerSource <= 0) {
throw new IllegalArgumentException(
"query admission limits must be positive");
}
this.globalAdmission = new LocalFederationQueryAdmissionController(
maximumConcurrentQueries);
this.maximumConcurrentQueriesPerTenant =
maximumConcurrentQueriesPerTenant;
this.maximumConcurrentQueriesPerSource =
maximumConcurrentQueriesPerSource;
}
/**
* 获取三级查询许可。
*
* @param sourceId 数据源标识
* @param queryId 查询标识
* @param timeout 最大等待时间
* @return 查询许可
*/
@Override
public FederationQueryPermit acquire(
SourceId sourceId,
QueryId queryId,
Duration timeout) {
return acquire(sourceId, queryId, timeout, () -> false);
}
/**
* 按数据源、租户、节点顺序获取许可,避免热点连接池等待占满全局配额。
*
* @param sourceId 数据源标识
* @param queryId 查询标识
* @param timeout 最大等待时间
* @param cancellationRequested 查询取消状态
* @return 查询许可
*/
@Override
public FederationQueryPermit acquire(
SourceId sourceId,
QueryId queryId,
Duration timeout,
BooleanSupplier cancellationRequested) {
long started = System.nanoTime();
String tenantKey = tenantKey(sourceId);
AdmissionEntry sourceEntry;
AdmissionEntry tenantEntry;
synchronized (lifecycleLock) {
ensureOpen();
sourceEntry = retain(sourceAdmissions, sourceId,
maximumConcurrentQueriesPerSource);
tenantEntry = retain(tenantAdmissions, tenantKey,
maximumConcurrentQueriesPerTenant);
}
FederationQueryPermit sourcePermit = null;
FederationQueryPermit tenantPermit = null;
FederationQueryPermit globalPermit = null;
try {
sourcePermit = sourceEntry.controller.acquire(
sourceId, queryId, remaining(timeout, started),
cancellationRequested);
tenantPermit = tenantEntry.controller.acquire(
sourceId, queryId, remaining(timeout, started),
cancellationRequested);
globalPermit = globalAdmission.acquire(
sourceId, queryId, remaining(timeout, started),
cancellationRequested);
FederationQueryPermit acquiredSource = sourcePermit;
FederationQueryPermit acquiredTenant = tenantPermit;
FederationQueryPermit acquiredGlobal = globalPermit;
AtomicBoolean released = new AtomicBoolean();
return () -> {
if (released.compareAndSet(false, true)) {
try {
acquiredGlobal.close();
} finally {
try {
acquiredTenant.close();
} finally {
try {
acquiredSource.close();
} finally {
releaseEntries(
sourceId, sourceEntry,
tenantKey, tenantEntry);
}
}
}
}
};
} catch (RuntimeException exception) {
closeQuietly(globalPermit);
closeQuietly(tenantPermit);
closeQuietly(sourcePermit);
releaseEntries(sourceId, sourceEntry, tenantKey, tenantEntry);
throw exception;
}
}
/**
* 关闭全部准入控制器并唤醒等待线程。
*/
@Override
public void close() {
synchronized (lifecycleLock) {
if (!closed.compareAndSet(false, true)) {
return;
}
globalAdmission.close();
tenantAdmissions.values().forEach(
entry -> entry.controller.close());
sourceAdmissions.values().forEach(
entry -> entry.controller.close());
tenantAdmissions.clear();
sourceAdmissions.clear();
}
}
private Duration remaining(Duration timeout, long started) {
if (timeout == null || timeout.isZero() || timeout.isNegative()) {
return Duration.ZERO;
}
long remaining = timeout.toNanos() - (System.nanoTime() - started);
return remaining <= 0L ? Duration.ZERO : Duration.ofNanos(remaining);
}
private String tenantKey(SourceId sourceId) {
String value = sourceId == null ? "" : sourceId.value();
int marker = value.lastIndexOf("-source-");
return marker <= 0 ? value : value.substring(0, marker);
}
/**
* 获取或创建分层准入项并登记一个活动查询或等待者。
*
* @param admissions 准入项映射
* @param key 租户或数据源键
* @param maximumConcurrentQueries 并发上限
* @param <K> 键类型
* @return 已登记引用的准入项
*/
private <K> AdmissionEntry retain(
Map<K, AdmissionEntry> admissions,
K key,
int maximumConcurrentQueries) {
AdmissionEntry entry = admissions.computeIfAbsent(
key,
ignored -> new AdmissionEntry(
new LocalFederationQueryAdmissionController(
maximumConcurrentQueries)));
entry.references++;
return entry;
}
/**
* 同步释放数据源与租户准入项引用。
*
* @param sourceId 数据源标识
* @param sourceEntry 数据源准入项
* @param tenantKey 租户键
* @param tenantEntry 租户准入项
*/
private void releaseEntries(
SourceId sourceId,
AdmissionEntry sourceEntry,
String tenantKey,
AdmissionEntry tenantEntry) {
synchronized (lifecycleLock) {
release(sourceAdmissions, sourceId, sourceEntry);
release(tenantAdmissions, tenantKey, tenantEntry);
}
}
/**
* 释放一个准入项引用,并在无查询或等待者时移除并关闭该项。
*
* @param admissions 准入项映射
* @param key 租户或数据源键
* @param entry 待释放准入项
* @param <K> 键类型
*/
private <K> void release(
Map<K, AdmissionEntry> admissions,
K key,
AdmissionEntry entry) {
entry.references--;
if (entry.references < 0) {
throw new IllegalStateException(
"query admission reference count became negative");
}
if (entry.references == 0 && admissions.remove(key, entry)) {
entry.controller.close();
}
}
/**
* 返回当前仍有查询或等待者的数据源准入项数量。
*
* @return 活动数据源准入项数量
*/
int trackedSourceAdmissionCount() {
return sourceAdmissions.size();
}
/**
* 返回当前仍有查询或等待者的租户准入项数量。
*
* @return 活动租户准入项数量
*/
int trackedTenantAdmissionCount() {
return tenantAdmissions.size();
}
/**
* 返回指定数据源当前活动查询与等待者的引用总数。
*
* @param sourceId 数据源标识
* @return 当前引用数
*/
int trackedSourceReferenceCount(SourceId sourceId) {
synchronized (lifecycleLock) {
AdmissionEntry entry = sourceAdmissions.get(sourceId);
return entry == null ? 0 : entry.references;
}
}
private void ensureOpen() {
if (closed.get()) {
throw new FederationSqlException(
FederationSqlErrorCode.ENGINE_CLOSED,
"datacenter query admission controller is closed");
}
}
private void closeQuietly(FederationQueryPermit permit) {
if (permit != null) {
permit.close();
}
}
/**
* 共享公平信号量及其活动查询、等待者引用计数。
*/
private static final class AdmissionEntry {
private final LocalFederationQueryAdmissionController controller;
private int references;
/**
* 创建准入项。
*
* @param controller 本地公平准入控制器
*/
private AdmissionEntry(LocalFederationQueryAdmissionController controller) {
this.controller = controller;
}
}
}

View File

@@ -0,0 +1,70 @@
package tech.easyflow.datacenter.federation;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.ObjectProvider;
import org.springframework.data.redis.core.StringRedisTemplate;
import org.springframework.stereotype.Component;
import org.springframework.transaction.support.TransactionSynchronization;
import org.springframework.transaction.support.TransactionSynchronizationManager;
import tech.easyflow.datacenter.meta.entity.DatacenterSource;
/**
* 发布不含凭据的 Federation Definition 变更提示。
*/
@Component
public class DatacenterFederationChangeNotifier {
/** Redis Pub/Sub 频道。 */
public static final String CHANNEL = "easyflow:datacenter:federation-source-changed";
private static final Logger log = LoggerFactory.getLogger(DatacenterFederationChangeNotifier.class);
private final ObjectProvider<StringRedisTemplate> redisTemplateProvider;
private final DatacenterFederationDefinitionFactory definitionFactory;
/**
* 创建通知器。
*
* @param redisTemplateProvider 可选 Redis 模板
* @param definitionFactory Definition 工厂
*/
public DatacenterFederationChangeNotifier(
ObjectProvider<StringRedisTemplate> redisTemplateProvider,
DatacenterFederationDefinitionFactory definitionFactory) {
this.redisTemplateProvider = redisTemplateProvider;
this.definitionFactory = definitionFactory;
}
/**
* 在当前事务提交后发布 SourceId无 Redis 时不影响数据库权威状态。
*
* @param source 已提交或即将提交的数据源
*/
public void publishAfterCommit(DatacenterSource source) {
String sourceId = definitionFactory.sourceId(source).value();
if (TransactionSynchronizationManager.isSynchronizationActive()) {
TransactionSynchronizationManager.registerSynchronization(new TransactionSynchronization() {
@Override
public void afterCommit() {
publish(sourceId);
}
});
return;
}
publish(sourceId);
}
private void publish(String sourceId) {
StringRedisTemplate redisTemplate = redisTemplateProvider.getIfAvailable();
if (redisTemplate == null) {
return;
}
try {
redisTemplate.convertAndSend(CHANNEL, sourceId);
} catch (RuntimeException exception) {
// Redis 仅缩短传播时间;数据库按需读取继续保证 minimumRevision 正确性。
log.warn("Failed to publish datacenter source change hint for {}", sourceId, exception);
}
}
}

View File

@@ -0,0 +1,64 @@
package tech.easyflow.datacenter.federation;
import com.zaxxer.hikari.HikariConfig;
import com.zaxxer.hikari.HikariDataSource;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
import tech.easyflow.datacenter.meta.entity.DatacenterSource;
import tech.easyflow.datacenter.security.DatacenterCredentialCipher;
/**
* 创建由 Federation Runtime 独占的只读 HikariCP 连接池。
*/
@Component
public class DatacenterFederationDataSourceFactory {
private final DatacenterCredentialCipher credentialCipher;
private final int maximumPoolSize;
/**
* 创建数据源工厂。
*
* @param credentialCipher 凭据解密器
* @param maximumPoolSize 单数据源连接池上限,与单源查询准入共用配置
*/
public DatacenterFederationDataSourceFactory(
DatacenterCredentialCipher credentialCipher,
@Value("${easyflow.datacenter.connection-pool.maximum-size:10}")
int maximumPoolSize) {
if (maximumPoolSize <= 0) {
throw new IllegalArgumentException(
"datacenter connection pool maximum size must be positive");
}
this.credentialCipher = credentialCipher;
this.maximumPoolSize = maximumPoolSize;
}
/**
* 根据持久化数据源快照创建只读池。
*
* @param source 数据源快照
* @return 未共享的 HikariCP 池
*/
public HikariDataSource create(DatacenterSource source) {
HikariConfig config = new HikariConfig();
config.setPoolName("dc-fed-" + source.getId() + "-r" + source.getDefinitionRevision());
config.setJdbcUrl(source.getJdbcUrl());
config.setUsername(source.getUsername());
config.setPassword(credentialCipher.decrypt(source.getCredentialCipher()));
if (source.getDriverClassName() != null && !source.getDriverClassName().isBlank()) {
config.setDriverClassName(source.getDriverClassName());
}
config.setReadOnly(true);
config.setAutoCommit(true);
config.setMaximumPoolSize(maximumPoolSize);
config.setMinimumIdle(Integer.getInteger("easyflow.datacenter.pool.minimum-idle", 0));
config.setConnectionTimeout(Long.getLong("easyflow.datacenter.pool.connection-timeout-ms", 5_000L));
config.setValidationTimeout(Long.getLong("easyflow.datacenter.pool.validation-timeout-ms", 3_000L));
config.setIdleTimeout(Long.getLong("easyflow.datacenter.pool.idle-timeout-ms", 600_000L));
config.setMaxLifetime(Long.getLong("easyflow.datacenter.pool.max-lifetime-ms", 1_800_000L));
config.setInitializationFailTimeout(
Long.getLong("easyflow.datacenter.pool.initialization-timeout-ms", 5_000L));
return new HikariDataSource(config);
}
}

View File

@@ -0,0 +1,140 @@
package tech.easyflow.datacenter.federation;
import com.easyagents.federation.sql.source.FederationDataSourceHandle;
import com.easyagents.federation.sql.source.FederationDataSourceHandles;
import com.easyagents.federation.sql.source.FederationDataSourceResolver;
import com.easyagents.federation.sql.source.FederationSourceDefinition;
import com.easyagents.federation.sql.source.RuntimeFingerprint;
import com.zaxxer.hikari.HikariDataSource;
import java.sql.Connection;
import java.sql.DatabaseMetaData;
import java.sql.SQLException;
import java.util.Objects;
import java.util.function.Supplier;
import org.springframework.stereotype.Component;
import tech.easyflow.datacenter.mapper.DatacenterSourceMapper;
import tech.easyflow.datacenter.meta.entity.DatacenterSource;
import tech.easyflow.datacenter.meta.enums.DatacenterSourceStatus;
/**
* 从 EasyFlow 权威配置解析节点本地 Federation DataSource Handle。
*/
@Component
public class DatacenterFederationDataSourceResolver implements FederationDataSourceResolver {
private final DatacenterSourceMapper sourceMapper;
private final DatacenterFederationDefinitionFactory definitionFactory;
private final DatacenterFederationDataSourceFactory dataSourceFactory;
private final ThreadLocal<CandidateResolution> candidateSource = new ThreadLocal<>();
/**
* 创建 Resolver。
*
* @param sourceMapper 数据源 Mapper
* @param definitionFactory Definition 工厂
* @param dataSourceFactory 连接池工厂
*/
public DatacenterFederationDataSourceResolver(
DatacenterSourceMapper sourceMapper,
DatacenterFederationDefinitionFactory definitionFactory,
DatacenterFederationDataSourceFactory dataSourceFactory) {
this.sourceMapper = sourceMapper;
this.definitionFactory = definitionFactory;
this.dataSourceFactory = dataSourceFactory;
}
/**
* 创建由 Runtime 独占并在 revision 淘汰后关闭的 Handle。
*
* @param definition 无凭据 Definition
* @return 独占 Handle
*/
@Override
public FederationDataSourceHandle resolve(FederationSourceDefinition definition) {
CandidateResolution candidate = candidateSource.get();
DatacenterSource source = candidate != null
&& Objects.equals(candidate.source().getId(), definitionFactory.sourceRecordId(definition))
? candidate.source() : sourceMapper.selectOneById(definitionFactory.sourceRecordId(definition));
if (source == null) {
throw new IllegalStateException("data source no longer exists");
}
DatacenterSourceStatus status = DatacenterSourceStatus.fromCode(source.getStatus());
boolean draftProbe = status == DatacenterSourceStatus.DRAFT;
if (!draftProbe
&& status != DatacenterSourceStatus.READY
&& status != DatacenterSourceStatus.DEGRADED) {
throw new IllegalStateException("data source cannot be resolved in its current state");
}
if (!definition.sourceId().equals(definitionFactory.sourceId(source))) {
throw new IllegalStateException("data source tenant identity does not match definition");
}
long revision = source.getDefinitionRevision() == null ? 0L : source.getDefinitionRevision();
if (revision != definition.revision()) {
throw new IllegalStateException("data source revision changed while runtime was resolving");
}
FederationSourceDefinition currentDefinition = candidate != null
? candidate.definition()
: definitionFactory.create(source);
if (!currentDefinition.checksum().equals(definition.checksum())) {
throw new IllegalStateException("data source definition checksum mismatch");
}
// 草稿只通过 probe 创建短生命周期连接池,尚未发布持久化 checksum
// READY/DEGRADED Runtime 必须同时匹配已发布 checksum。
if (!draftProbe && (source.getDefinitionChecksum() == null
|| !source.getDefinitionChecksum().equals(definition.checksum()))) {
throw new IllegalStateException("published data source definition checksum mismatch");
}
HikariDataSource pool = dataSourceFactory.create(source);
try (Connection connection = pool.getConnection()) {
DatabaseMetaData metadata = connection.getMetaData();
RuntimeFingerprint fingerprint = new RuntimeFingerprint(
metadata.getDatabaseProductName(),
metadata.getDatabaseProductVersion(),
metadata.getDriverName(),
metadata.getDriverVersion(),
"easyflow-xl01-wp1");
return FederationDataSourceHandles.owned(pool, fingerprint, pool::close);
} catch (SQLException | RuntimeException exception) {
try {
pool.close();
} catch (RuntimeException closeException) {
exception.addSuppressed(closeException);
}
throw new IllegalStateException("failed to initialize data source runtime", exception);
}
}
/**
* 在当前线程内使用未发布的候选配置完成一次 XL15 probe。
*
* <p>候选配置不会进入共享状态、Redis 或持久化记录,并在操作结束后立即移除。</p>
*
* @param source 已完成凭据加密和连接归一化的候选配置
* @param action 需要解析候选配置的同步操作
* @param <T> 操作结果类型
* @return 操作结果
* @throws IllegalStateException 当前线程已经存在候选配置时抛出
*/
public <T> T withCandidate(
DatacenterSource source,
FederationSourceDefinition definition,
Supplier<T> action) {
Objects.requireNonNull(source, "source");
Objects.requireNonNull(definition, "definition");
Objects.requireNonNull(action, "action");
if (candidateSource.get() != null) {
throw new IllegalStateException("nested data source candidate resolution is not supported");
}
candidateSource.set(new CandidateResolution(source, definition));
try {
return action.get();
} finally {
candidateSource.remove();
}
}
private record CandidateResolution(
DatacenterSource source,
FederationSourceDefinition definition) {
}
}

View File

@@ -0,0 +1,208 @@
package tech.easyflow.datacenter.federation;
import com.easyagents.federation.sql.adapter.jdbc.JdbcFederationSqlAdapterProvider;
import com.easyagents.federation.sql.adapter.jdbc.JdbcSchemaDefinition;
import com.easyagents.federation.sql.source.FederationSourceDefinition;
import com.easyagents.federation.sql.source.SourceId;
import com.mybatisflex.core.query.QueryWrapper;
import java.math.BigInteger;
import java.util.ArrayList;
import java.util.Collection;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
import org.springframework.stereotype.Component;
import tech.easyflow.datacenter.mapper.DatacenterCatalogMapper;
import tech.easyflow.datacenter.meta.entity.DatacenterCatalog;
import tech.easyflow.datacenter.meta.entity.DatacenterSource;
import tech.easyflow.datacenter.meta.enums.DatacenterMetadataStatus;
import tech.easyflow.datacenter.meta.enums.DatacenterSourceType;
/**
* 将 EasyFlow 持久化数据源快照转换为无凭据 Federation Definition。
*/
@Component
public class DatacenterFederationDefinitionFactory {
/** Definition 中用于解析业务数据源主键的选项。 */
public static final String SOURCE_RECORD_ID_OPTION = "sourceRecordId";
private final DatacenterCatalogMapper catalogMapper;
/**
* 创建 Definition 工厂。
*
* @param catalogMapper 命名空间 Mapper
*/
public DatacenterFederationDefinitionFactory(DatacenterCatalogMapper catalogMapper) {
this.catalogMapper = catalogMapper;
}
/**
* 创建用于探测或执行的 Definition。
*
* @param source 数据源当前快照
* @return 无凭据 Definition
* @throws IllegalArgumentException 数据源不是当前 JDBC 支持类型
*/
public FederationSourceDefinition create(DatacenterSource source) {
return create(source, loadCatalogs(source));
}
/**
* 为尚未发布的连接候选创建只包含默认命名空间的 Definition。
*
* @param source 候选数据源
* @return 不读取旧持久化 Catalog 的候选 Definition
*/
public FederationSourceDefinition createCandidate(DatacenterSource source) {
return create(source, List.of());
}
/**
* 为即将发布的单命名空间配置创建与持久化结果一致的 Definition。
*
* @param source 候选数据源
* @param catalogName 即将纳管的物理 Catalog 或 Schema
* @return 不读取旧 Catalog 的候选 Definition
*/
public FederationSourceDefinition createCandidate(
DatacenterSource source,
String catalogName) {
if (catalogName == null || catalogName.isBlank()) {
throw new IllegalArgumentException("candidate catalog name is required");
}
String normalized = catalogName.trim();
DatacenterCatalog catalog = new DatacenterCatalog();
catalog.setLogicalSchemaName(normalized);
if (DatacenterSourceType.MYSQL.name().equals(source.getSourceType())) {
catalog.setPhysicalCatalogName(normalized);
} else {
catalog.setPhysicalCatalogName(source.getDatabaseName());
catalog.setPhysicalSchemaName(normalized);
}
return create(source, List.of(catalog));
}
/**
* 批量创建 Definition一次加载全部活动 Catalog供周期 reconcile 避免 N+1。
*
* @param sources 数据源快照
* @return 按数据源记录 ID 索引的 Definition
*/
public Map<BigInteger, FederationSourceDefinition> createBatch(
Collection<DatacenterSource> sources) {
List<DatacenterSource> sourceList = sources == null
? List.of() : sources.stream().filter(java.util.Objects::nonNull).toList();
if (sourceList.isEmpty()) {
return Map.of();
}
List<BigInteger> sourceIds = sourceList.stream()
.map(DatacenterSource::getId)
.filter(java.util.Objects::nonNull)
.toList();
Map<BigInteger, List<DatacenterCatalog>> catalogsBySource = sourceIds.isEmpty()
? Map.of()
: catalogMapper.selectListByQuery(QueryWrapper.create()
.in(DatacenterCatalog::getSourceId, sourceIds)
.eq(DatacenterCatalog::getMetadataStatus, DatacenterMetadataStatus.ACTIVE.name())
.orderBy("created asc"))
.stream()
.collect(Collectors.groupingBy(DatacenterCatalog::getSourceId));
Map<BigInteger, FederationSourceDefinition> definitions = new LinkedHashMap<>();
for (DatacenterSource source : sourceList) {
definitions.put(source.getId(), create(
source, catalogsBySource.getOrDefault(source.getId(), List.of())));
}
return Map.copyOf(definitions);
}
private FederationSourceDefinition create(
DatacenterSource source,
List<DatacenterCatalog> catalogs) {
requireExternalJdbc(source);
long revision = Math.max(1L, source.getDefinitionRevision() == null
? 0L : source.getDefinitionRevision());
List<JdbcSchemaDefinition> schemas = schemas(source, catalogs);
Map<String, String> options = new LinkedHashMap<>();
if (source.getAdapterOptionsJson() != null) {
options.putAll(source.getAdapterOptionsJson());
}
options.put(SOURCE_RECORD_ID_OPTION, source.getId().toString());
return new FederationSourceDefinition(
sourceId(source),
revision,
source.getAdapterId() == null || source.getAdapterId().isBlank()
? JdbcFederationSqlAdapterProvider.ADAPTER_ID
: source.getAdapterId(),
new ArrayList<>(schemas),
options);
}
/**
* 派生租户隔离且不随显示名称变化的 SourceId。
*
* @param source 数据源
* @return Federation SourceId
*/
public SourceId sourceId(DatacenterSource source) {
BigInteger tenantId = source.getTenantId() == null ? BigInteger.ZERO : source.getTenantId();
return new SourceId("tenant-" + tenantId + "-source-" + source.getId());
}
/**
* 从 Definition 读取 EasyFlow 数据源主键。
*
* @param definition Definition
* @return 数据源主键
*/
public BigInteger sourceRecordId(FederationSourceDefinition definition) {
String value = definition.adapterOptions().get(SOURCE_RECORD_ID_OPTION);
if (value == null || value.isBlank()) {
throw new IllegalArgumentException("sourceRecordId is missing from Federation Definition");
}
return new BigInteger(value);
}
private List<DatacenterCatalog> loadCatalogs(DatacenterSource source) {
QueryWrapper query = QueryWrapper.create()
.eq(DatacenterCatalog::getSourceId, source.getId())
.eq(DatacenterCatalog::getMetadataStatus, DatacenterMetadataStatus.ACTIVE.name())
.orderBy("created asc");
return catalogMapper.selectListByQuery(query);
}
private List<JdbcSchemaDefinition> schemas(
DatacenterSource source,
List<DatacenterCatalog> catalogs) {
if (catalogs.isEmpty()) {
return List.of(defaultSchema(source));
}
return catalogs.stream()
.map(catalog -> new JdbcSchemaDefinition(
catalog.getLogicalSchemaName(),
catalog.getPhysicalCatalogName(),
catalog.getPhysicalSchemaName()))
.toList();
}
private JdbcSchemaDefinition defaultSchema(DatacenterSource source) {
if (DatacenterSourceType.MYSQL.name().equals(source.getSourceType())) {
return new JdbcSchemaDefinition("MAIN", source.getDatabaseName(), null);
}
String schema = source.getSchemaName() == null || source.getSchemaName().isBlank()
? "public" : source.getSchemaName();
return new JdbcSchemaDefinition("MAIN", source.getDatabaseName(), schema);
}
private void requireExternalJdbc(DatacenterSource source) {
if (source == null || source.getId() == null) {
throw new IllegalArgumentException("persisted source is required");
}
if (!DatacenterSourceType.MYSQL.name().equals(source.getSourceType())
&& !DatacenterSourceType.POSTGRESQL.name().equals(source.getSourceType())) {
throw new IllegalArgumentException("Federation JDBC currently supports MySQL and PostgreSQL sources");
}
}
}

View File

@@ -0,0 +1,292 @@
package tech.easyflow.datacenter.federation;
import com.easyagents.federation.sql.execute.QueryId;
import java.math.BigInteger;
import java.time.Duration;
import java.util.UUID;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.atomic.AtomicBoolean;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.ObjectProvider;
import org.springframework.data.redis.core.StringRedisTemplate;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Component;
import tech.easyflow.common.entity.LoginAccount;
import tech.easyflow.common.web.exceptions.BusinessException;
/**
* 管理查询从编译到游标关闭的取消状态,并通过 Redis 将取消提示广播到执行节点。
*/
@Component
public class DatacenterFederationQueryCancellationService {
/** 查询取消 Redis Pub/Sub 频道。 */
public static final String CHANNEL = "easyflow:datacenter:federation-query-cancel";
private static final String CANCEL_KEY_PREFIX =
"easyflow:datacenter:federation-query-cancelled:";
private static final Duration CANCEL_TTL = Duration.ofMinutes(10);
private static final Logger log = LoggerFactory.getLogger(
DatacenterFederationQueryCancellationService.class);
private final DatacenterFederationRuntime runtime;
private final ObjectProvider<StringRedisTemplate> redisTemplateProvider;
private final ConcurrentHashMap<String, ActiveQuery> activeQueries = new ConcurrentHashMap<>();
/**
* 创建查询取消服务。
*
* @param runtime 节点本地 Federation Runtime
* @param redisTemplateProvider 可选 Redis 模板
*/
public DatacenterFederationQueryCancellationService(
DatacenterFederationRuntime runtime,
ObjectProvider<StringRedisTemplate> redisTemplateProvider) {
this.runtime = runtime;
this.redisTemplateProvider = redisTemplateProvider;
}
/**
* 校验或创建查询标识。
*
* @param requestedQueryId 调用方预生成的 UUID为空时由服务端生成
* @return 已校验的查询标识
* @throws BusinessException 标识格式不合法
*/
public QueryId resolveQueryId(String requestedQueryId) {
if (requestedQueryId == null || requestedQueryId.isBlank()) {
return QueryId.create();
}
try {
return new QueryId(UUID.fromString(requestedQueryId.trim()).toString());
} catch (IllegalArgumentException exception) {
throw new BusinessException("queryId 必须是 UUID");
}
}
/**
* 在编译前登记查询,防止重复 QueryId 混淆取消目标。
*
* @param queryId 查询标识
* @param account 执行账号
* @return 必须关闭的登记句柄
* @throws BusinessException QueryId 正在使用
*/
public QueryRegistration register(QueryId queryId, LoginAccount account) {
ActiveQuery activeQuery = new ActiveQuery(tenantId(account));
if (activeQueries.putIfAbsent(queryId.value(), activeQuery) != null) {
throw new BusinessException(409, 14123, "queryId 正在使用", null);
}
if (isCancellationPersisted(activeQuery.tenantId, queryId)) {
try {
cancelLocal(activeQuery.tenantId, queryId);
} catch (RuntimeException exception) {
activeQueries.remove(queryId.value(), activeQuery);
throw exception;
}
}
return new QueryRegistration(queryId, activeQuery);
}
/**
* 向所有节点广播取消提示,并立即取消当前节点已登记的 Statement。
*
* @param requestedQueryId 查询标识
* @param account 当前账号
* @return 当前节点已找到查询或 Redis 已接收广播
*/
public boolean cancel(String requestedQueryId, LoginAccount account) {
if (requestedQueryId == null || requestedQueryId.isBlank()) {
throw new BusinessException("queryId 不能为空");
}
QueryId queryId = resolveQueryId(requestedQueryId);
BigInteger tenantId = tenantId(account);
boolean distributedCancellationPersisted = persistAndPublishCancellation(
tenantId, queryId);
boolean localCancelled;
try {
localCancelled = cancelLocal(tenantId, queryId);
} catch (RuntimeException exception) {
log.error("Failed to cancel local datacenter query for tenant {} query {}",
tenantId, queryId.value(), exception);
if (!distributedCancellationPersisted) {
throw exception;
}
localCancelled = false;
}
return localCancelled || distributedCancellationPersisted;
}
/**
* 先持久化取消终态,再尽力发布低延迟提示。
*
* @param tenantId 租户 ID
* @param queryId 查询标识
* @return Redis 是否已可靠保存取消终态
*/
private boolean persistAndPublishCancellation(
BigInteger tenantId,
QueryId queryId) {
StringRedisTemplate redisTemplate = redisTemplateProvider.getIfAvailable();
if (redisTemplate == null) {
return false;
}
try {
redisTemplate.opsForValue().set(
cancellationKey(tenantId, queryId), "1", CANCEL_TTL);
} catch (RuntimeException exception) {
log.warn("Failed to persist datacenter query cancellation for tenant {} query {}",
tenantId, queryId.value(), exception);
return false;
}
try {
redisTemplate.convertAndSend(
CHANNEL, tenantId + "|" + queryId.value());
} catch (RuntimeException exception) {
// Pub/Sub 仅缩短生效延迟;持久取消终态已由周期核对保证交付。
log.warn("Failed to publish datacenter query cancellation hint for tenant {} query {}",
tenantId, queryId.value(), exception);
}
return true;
}
/**
* 处理 Redis 取消提示;格式无效时安全忽略。
*
* @param payload 租户与 QueryId 组合文本
*/
public void acceptCancellationHint(String payload) {
if (payload == null) {
return;
}
int separator = payload.indexOf('|');
if (separator <= 0 || separator == payload.length() - 1) {
return;
}
try {
BigInteger tenantId = new BigInteger(payload.substring(0, separator));
QueryId queryId = resolveQueryId(payload.substring(separator + 1));
cancelLocal(tenantId, queryId);
} catch (RuntimeException exception) {
log.warn("Ignored invalid datacenter query cancellation hint", exception);
}
}
/**
* 在编译或取数边界检查查询是否已取消。
*
* @param queryId 查询标识
* @throws BusinessException 查询已取消
*/
public void throwIfCancelled(QueryId queryId) {
ActiveQuery activeQuery = activeQueries.get(queryId.value());
if (activeQuery != null && activeQuery.cancelled.get()) {
throw new BusinessException(409, 14118, "查询已取消", null);
}
}
/**
* 返回查询当前是否已收到取消请求。
*
* @param queryId 查询标识
* @return 是否取消
*/
public boolean isCancelled(QueryId queryId) {
ActiveQuery activeQuery = activeQueries.get(queryId.value());
return activeQuery != null && activeQuery.cancelled.get();
}
/**
* 周期核对 Redis 持久取消标记,覆盖 Pub/Sub 提示丢失或节点短暂断连。
*/
@Scheduled(fixedDelayString =
"${easyflow.datacenter.query-cancel-reconcile-ms:1000}")
public void reconcilePersistedCancellations() {
activeQueries.forEach((queryIdValue, activeQuery) -> {
if (activeQuery.localCancellationApplied.get()) {
return;
}
QueryId queryId = new QueryId(queryIdValue);
if (activeQuery.cancelled.get()
|| isCancellationPersisted(activeQuery.tenantId, queryId)) {
try {
cancelLocal(activeQuery.tenantId, queryId);
} catch (RuntimeException exception) {
log.error("Failed to reconcile datacenter query cancellation for tenant {} query {}",
activeQuery.tenantId, queryId.value(), exception);
}
}
});
}
private boolean cancelLocal(BigInteger tenantId, QueryId queryId) {
ActiveQuery activeQuery = activeQueries.get(queryId.value());
if (activeQuery == null || !activeQuery.tenantId.equals(tenantId)) {
return false;
}
activeQuery.cancelled.set(true);
runtime.engine().cancel(queryId);
activeQuery.localCancellationApplied.set(true);
return true;
}
private boolean isCancellationPersisted(BigInteger tenantId, QueryId queryId) {
StringRedisTemplate redisTemplate = redisTemplateProvider.getIfAvailable();
if (redisTemplate == null) {
return false;
}
try {
return Boolean.TRUE.equals(redisTemplate.hasKey(
cancellationKey(tenantId, queryId)));
} catch (RuntimeException exception) {
log.warn("Failed to read datacenter query cancellation for tenant {} query {}",
tenantId, queryId.value(), exception);
return false;
}
}
private String cancellationKey(BigInteger tenantId, QueryId queryId) {
return CANCEL_KEY_PREFIX + tenantId + ":" + queryId.value();
}
private BigInteger tenantId(LoginAccount account) {
return account == null || account.getTenantId() == null
? BigInteger.ZERO : account.getTenantId();
}
private record ActiveQuery(
BigInteger tenantId,
AtomicBoolean cancelled,
AtomicBoolean localCancellationApplied) {
private ActiveQuery(BigInteger tenantId) {
this(tenantId, new AtomicBoolean(), new AtomicBoolean());
}
}
/**
* 查询活动登记句柄。
*/
public final class QueryRegistration implements AutoCloseable {
private final QueryId queryId;
private final ActiveQuery activeQuery;
private final AtomicBoolean closed = new AtomicBoolean();
private QueryRegistration(QueryId queryId, ActiveQuery activeQuery) {
this.queryId = queryId;
this.activeQuery = activeQuery;
}
/**
* 仅移除当前登记实例,避免误删随后复用的同名查询。
*/
@Override
public void close() {
if (closed.compareAndSet(false, true)) {
activeQueries.remove(queryId.value(), activeQuery);
}
}
}
}

View File

@@ -0,0 +1,731 @@
package tech.easyflow.datacenter.federation;
import com.easyagents.federation.sql.api.FederationSqlException;
import com.easyagents.federation.sql.api.SqlExecutionContext;
import com.easyagents.federation.sql.compile.FederationSqlPlan;
import com.easyagents.federation.sql.compile.SqlCompileRequest;
import com.easyagents.federation.sql.execute.FederationColumn;
import com.easyagents.federation.sql.execute.FederationResultCursor;
import com.easyagents.federation.sql.execute.QueryId;
import com.easyagents.federation.sql.execute.SqlExecutionOptions;
import com.easyagents.federation.sql.execute.SqlParameter;
import com.easyagents.federation.sql.federation.FederationQueryScopeDefinition;
import com.easyagents.federation.sql.source.SourceId;
import java.io.IOException;
import java.io.InputStream;
import java.io.Reader;
import java.math.BigInteger;
import java.sql.Blob;
import java.sql.Clob;
import java.sql.SQLXML;
import java.sql.SQLException;
import java.sql.Struct;
import java.sql.Types;
import java.time.Duration;
import java.util.ArrayList;
import java.util.Base64;
import java.util.Collections;
import java.util.LinkedHashMap;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.function.Consumer;
import org.apache.calcite.rel.RelNode;
import org.apache.calcite.rel.RelVisitor;
import org.apache.calcite.rel.core.TableScan;
import org.apache.calcite.sql.SqlIdentifier;
import org.apache.calcite.sql.util.SqlBasicVisitor;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.stereotype.Service;
import tech.easyflow.common.entity.LoginAccount;
import tech.easyflow.common.web.exceptions.BusinessException;
import tech.easyflow.datacenter.audit.DatacenterQueryAudit;
import tech.easyflow.datacenter.audit.DatacenterQueryAuditService;
import tech.easyflow.datacenter.execution.model.DatacenterSqlColumnView;
import tech.easyflow.datacenter.execution.model.DatacenterSqlConsoleResult;
import tech.easyflow.datacenter.meta.entity.DatacenterSource;
import tech.easyflow.datacenter.meta.enums.DatacenterSourceStatus;
/**
* 通过 Federation SQL 完成有界只读查询、资源关闭和统一审计。
*/
@Service
public class DatacenterFederationQueryService {
private static final Logger log = LoggerFactory.getLogger(DatacenterFederationQueryService.class);
private static final int DEFAULT_MAX_ROWS = 200;
private static final int MAX_ALLOWED_ROWS = 1_000;
private static final long MAX_RESULT_BYTES = 4L * 1024L * 1024L;
private static final int MAX_TEXT_CELL_CHARS = 65_536;
private final DatacenterFederationRuntime runtime;
private final DatacenterQueryAuditService auditService;
private final DatacenterFederationQueryCancellationService cancellationService;
/**
* 创建查询服务。
*
* @param runtime Federation Runtime
* @param auditService 查询审计服务
* @param cancellationService 跨节点查询取消服务
*/
public DatacenterFederationQueryService(
DatacenterFederationRuntime runtime,
DatacenterQueryAuditService auditService,
DatacenterFederationQueryCancellationService cancellationService) {
this.runtime = runtime;
this.auditService = auditService;
this.cancellationService = cancellationService;
}
/**
* 执行一条单数据源只读 SQL。
*
* @param source 已激活数据源
* @param sql 只读 SQL
* @param parameters 显式 JDBC 参数
* @param requestedMaxRows 请求行数上限
* @param account 执行账号
* @param callerType 调用方类型
* @param callerId 调用方标识
* @return 有界查询结果
*/
public DatacenterSqlConsoleResult execute(
DatacenterSource source,
String sql,
List<SqlParameter> parameters,
Integer requestedMaxRows,
LoginAccount account,
String callerType,
String callerId) {
return execute(source, sql, parameters, requestedMaxRows, account,
callerType, callerId, null);
}
/**
* 使用调用方预生成的 QueryId 执行只读 SQL以支持请求完成前取消。
*
* @param source 已激活数据源
* @param sql 只读 SQL
* @param parameters 显式 JDBC 参数
* @param requestedMaxRows 请求行数上限
* @param account 执行账号
* @param callerType 调用方类型
* @param callerId 调用方标识
* @param requestedQueryId 调用方预生成的查询 UUID为空时服务端生成
* @return 有界查询结果
*/
public DatacenterSqlConsoleResult execute(
DatacenterSource source,
String sql,
List<SqlParameter> parameters,
Integer requestedMaxRows,
LoginAccount account,
String callerType,
String callerId,
String requestedQueryId) {
requireExecutable(source, sql);
int maxRows = normalizeMaxRows(requestedMaxRows);
List<SqlParameter> safeParameters = List.copyOf(parameters == null ? List.of() : parameters);
QueryId queryId = cancellationService.resolveQueryId(requestedQueryId);
DatacenterFederationQueryCancellationService.QueryRegistration registration =
cancellationService.register(queryId, account);
long startedNanos = System.nanoTime();
DatacenterQueryAudit audit;
try {
audit = auditService.start(
queryId.value(), source, sql, maskedParameterSummary(safeParameters),
account, callerType, callerId);
} catch (RuntimeException exception) {
registration.close();
throw exception;
}
List<String> referencedTables = List.of();
List<String> referencedFields = List.of();
try {
cancellationService.throwIfCancelled(queryId);
SqlCompileRequest compileRequest = compileRequest(source, sql, safeParameters);
FederationSqlPlan plan = runtime.engine().compile(compileRequest);
cancellationService.throwIfCancelled(queryId);
referencedTables = referencedTables(plan.relRoot().rel);
referencedFields = referencedFields(plan.sqlNode());
SqlExecutionOptions options = new SqlExecutionOptions(
200,
maxRows + 1,
30,
true);
long databaseStartedNanos = System.nanoTime();
List<DatacenterSqlColumnView> columns;
List<Map<String, Object>> rows = new ArrayList<>(maxRows);
boolean truncated = false;
long resultBytes = 0L;
try (FederationResultCursor cursor = runtime.engine().execute(
plan,
new SqlExecutionContext(
queryId,
safeParameters,
options,
Duration.ofSeconds(5)))) {
columns = columns(cursor.columns());
while (cursor.next()) {
cancellationService.throwIfCancelled(queryId);
if (rows.size() >= maxRows) {
truncated = true;
break;
}
Map<String, Object> row = new LinkedHashMap<>();
for (int index = 0; index < columns.size(); index++) {
Object value = readColumn(cursor, cursor.columns().get(index), index + 1);
resultBytes += estimateBytes(columns.get(index).label(), value);
if (resultBytes > MAX_RESULT_BYTES) {
truncated = true;
break;
}
row.put(columns.get(index).key(), value);
}
if (truncated && resultBytes > MAX_RESULT_BYTES) {
break;
}
rows.add(Collections.unmodifiableMap(new LinkedHashMap<>(row)));
}
}
long databaseDurationMs = elapsedMillis(databaseStartedNanos);
long durationMs = elapsedMillis(startedNanos);
auditService.succeed(audit, referencedTables, referencedFields, rows.size(), truncated,
durationMs, databaseDurationMs);
return new DatacenterSqlConsoleResult(
queryId.value(), columns, rows, rows.size(), truncated, durationMs);
} catch (RuntimeException exception) {
long durationMs = elapsedMillis(startedNanos);
String errorCode = cancellationService.isCancelled(queryId)
? "QUERY_CANCELLED"
: exception instanceof FederationSqlException federationException
? federationException.errorCode().name()
: "DATACENTER_QUERY_FAILED";
try {
auditService.fail(audit, errorCode, safeMessage(exception),
referencedTables, referencedFields, durationMs);
} catch (RuntimeException auditException) {
exception.addSuppressed(auditException);
log.error("Failed to finalize datacenter query audit {}", queryId.value(), auditException);
}
if (exception instanceof BusinessException businessException) {
throw businessException;
}
logUnexpectedFailure(queryId, source, exception);
throw toBusinessException(exception);
} finally {
registration.close();
}
}
/**
* 使用 Federation 游标流式消费单数据源只读 SQL并在触及资源上限时显式失败。
*
* @param source 已激活数据源
* @param sql 只读 SQL
* @param parameters 显式 JDBC 参数
* @param fetchSize JDBC 拉取批次
* @param maxRows 最大消费行数
* @param maxBytes 最大估算字节数
* @param account 执行账号
* @param callerType 调用方类型
* @param callerId 调用方标识
* @param consumer 单行消费者
* @return 已消费行数
*/
public long consume(
DatacenterSource source,
String sql,
List<SqlParameter> parameters,
int fetchSize,
long maxRows,
long maxBytes,
LoginAccount account,
String callerType,
String callerId,
Consumer<Map<String, Object>> consumer) {
return consume(source, sql, parameters, fetchSize, maxRows, maxBytes,
account, callerType, callerId, null, consumer);
}
/**
* 使用调用方预生成的 QueryId 流式消费只读 SQL以支持流消费期间取消。
*
* @param source 已激活数据源
* @param sql 只读 SQL
* @param parameters 显式 JDBC 参数
* @param fetchSize JDBC 拉取批次
* @param maxRows 最大消费行数
* @param maxBytes 最大估算字节数
* @param account 执行账号
* @param callerType 调用方类型
* @param callerId 调用方标识
* @param requestedQueryId 调用方预生成的查询 UUID为空时服务端生成
* @param consumer 单行消费者
* @return 已消费行数
*/
public long consume(
DatacenterSource source,
String sql,
List<SqlParameter> parameters,
int fetchSize,
long maxRows,
long maxBytes,
LoginAccount account,
String callerType,
String callerId,
String requestedQueryId,
Consumer<Map<String, Object>> consumer) {
requireExecutable(source, sql);
if (fetchSize <= 0 || maxRows <= 0 || maxRows >= Integer.MAX_VALUE
|| maxBytes <= 0 || consumer == null) {
throw new IllegalArgumentException("stream query limits and consumer must be valid");
}
List<SqlParameter> safeParameters = List.copyOf(parameters == null ? List.of() : parameters);
QueryId queryId = cancellationService.resolveQueryId(requestedQueryId);
DatacenterFederationQueryCancellationService.QueryRegistration registration =
cancellationService.register(queryId, account);
long startedNanos = System.nanoTime();
DatacenterQueryAudit audit;
try {
audit = auditService.start(
queryId.value(), source, sql,
maskedParameterSummary(safeParameters),
account, callerType, callerId);
} catch (RuntimeException exception) {
registration.close();
throw exception;
}
List<String> referencedTables = List.of();
List<String> referencedFields = List.of();
long consumedRows = 0L;
try {
cancellationService.throwIfCancelled(queryId);
SqlCompileRequest compileRequest = compileRequest(source, sql, safeParameters);
FederationSqlPlan plan = runtime.engine().compile(compileRequest);
cancellationService.throwIfCancelled(queryId);
referencedTables = referencedTables(plan.relRoot().rel);
referencedFields = referencedFields(plan.sqlNode());
SqlExecutionOptions options = new SqlExecutionOptions(
fetchSize, Math.toIntExact(maxRows + 1L), 30, true);
long databaseStartedNanos = System.nanoTime();
long resultBytes = 0L;
try (FederationResultCursor cursor = runtime.engine().execute(
plan,
new SqlExecutionContext(queryId, safeParameters, options, Duration.ofSeconds(5)))) {
List<DatacenterSqlColumnView> columns = columns(cursor.columns());
while (cursor.next()) {
cancellationService.throwIfCancelled(queryId);
if (consumedRows >= maxRows) {
throw new BusinessException("数据集查询结果超过行数上限: " + maxRows);
}
Map<String, Object> row = new LinkedHashMap<>();
for (int index = 0; index < columns.size(); index++) {
Object value = readColumn(cursor, cursor.columns().get(index), index + 1);
resultBytes += estimateBytes(columns.get(index).label(), value);
if (resultBytes > maxBytes) {
throw new BusinessException("数据集查询结果超过字节上限: " + maxBytes);
}
row.put(columns.get(index).label(), value);
}
consumer.accept(Collections.unmodifiableMap(row));
consumedRows++;
}
}
long durationMs = elapsedMillis(startedNanos);
auditService.succeed(
audit, referencedTables, referencedFields, consumedRows, false,
durationMs, elapsedMillis(databaseStartedNanos));
return consumedRows;
} catch (RuntimeException exception) {
long durationMs = elapsedMillis(startedNanos);
String errorCode = cancellationService.isCancelled(queryId)
? "QUERY_CANCELLED"
: exception instanceof FederationSqlException federationException
? federationException.errorCode().name()
: "DATACENTER_QUERY_FAILED";
try {
auditService.fail(audit, errorCode, safeMessage(exception),
referencedTables, referencedFields, durationMs);
} catch (RuntimeException auditException) {
exception.addSuppressed(auditException);
log.error("Failed to finalize datacenter stream query audit {}", queryId.value(), auditException);
}
if (exception instanceof BusinessException businessException) {
throw businessException;
}
logUnexpectedFailure(queryId, source, exception);
throw toBusinessException(exception);
} finally {
registration.close();
}
}
private SqlCompileRequest compileRequest(
DatacenterSource source,
String sql,
List<SqlParameter> parameters) {
SourceId sourceId = runtime.definitions().sourceId(source);
return new SqlCompileRequest(
sql,
FederationQueryScopeDefinition.single(
"datacenter-source:" + source.getId(),
source.getScopeRevision(),
sourceId.value(),
sourceId,
source.getDefinitionRevision()),
parameters.stream().map(SqlParameter::jdbcType).toList(),
"scope-" + source.getScopeRevision());
}
private void requireExecutable(DatacenterSource source, String sql) {
if (source == null || DatacenterSourceStatus.fromCode(source.getStatus())
!= DatacenterSourceStatus.READY) {
throw new BusinessException("数据连接尚未就绪");
}
if (sql == null || sql.isBlank()) {
throw new BusinessException("SQL 不能为空");
}
if (sql.length() > 100_000) {
throw new BusinessException("SQL 长度超过限制");
}
}
private int normalizeMaxRows(Integer requested) {
if (requested == null) {
return DEFAULT_MAX_ROWS;
}
if (requested <= 0) {
throw new BusinessException("maxRows 必须大于 0");
}
return Math.min(requested, MAX_ALLOWED_ROWS);
}
private List<DatacenterSqlColumnView> columns(List<FederationColumn> columns) {
List<DatacenterSqlColumnView> result = new ArrayList<>(columns.size());
for (FederationColumn column : columns) {
result.add(new DatacenterSqlColumnView(
"c" + column.index(),
column.label(),
column.jdbcType(),
column.typeName(),
column.nullable()));
}
return List.copyOf(result);
}
private List<String> referencedTables(RelNode root) {
Set<String> names = new LinkedHashSet<>();
new RelVisitor() {
@Override
public void visit(RelNode node, int ordinal, RelNode parent) {
if (node instanceof TableScan scan) {
names.add(String.join(".", scan.getTable().getQualifiedName()));
}
super.visit(node, ordinal, parent);
}
}.go(root);
return List.copyOf(names);
}
private List<String> referencedFields(org.apache.calcite.sql.SqlNode sqlNode) {
Set<String> names = new LinkedHashSet<>();
sqlNode.accept(new SqlBasicVisitor<Void>() {
@Override
public Void visit(SqlIdentifier identifier) {
if (!identifier.isStar() && !identifier.names.isEmpty()) {
names.add(String.join(".", identifier.names));
}
return null;
}
});
return List.copyOf(names);
}
private Map<String, Object> maskedParameterSummary(List<SqlParameter> parameters) {
Map<String, Object> summary = new LinkedHashMap<>();
for (int index = 0; index < parameters.size(); index++) {
SqlParameter parameter = parameters.get(index);
summary.put("p" + (index + 1), Map.of(
"jdbcType", parameter.jdbcType(),
"present", parameter.value() != null));
}
return Map.copyOf(summary);
}
private Object normalizeValue(Object value) {
try {
if (value instanceof byte[] bytes) {
if (bytes.length > MAX_TEXT_CELL_CHARS) {
return "[binary " + bytes.length + " bytes]";
}
return Base64.getEncoder().encodeToString(bytes);
}
if (value instanceof Blob blob) {
return "[binary " + blob.length() + " bytes]";
}
if (value instanceof Clob clob) {
long length = Math.min(clob.length(), 4_096L);
return clob.getSubString(1L, (int) length);
}
if (value instanceof SQLXML sqlxml) {
String text = sqlxml.getString();
return boundedText(text);
}
if (value instanceof java.sql.Array array) {
return "[array " + array.getBaseTypeName() + "]";
}
if (value instanceof Struct struct) {
return "[struct " + struct.getSQLTypeName() + "]";
}
if (value instanceof CharSequence text) {
return boundedText(text.toString());
}
if (value instanceof BigInteger || value instanceof Long) {
return value.toString();
}
Package valuePackage = value.getClass().getPackage();
if (valuePackage != null && valuePackage.getName().startsWith("org.postgresql")) {
return boundedText(String.valueOf(value));
}
return value instanceof Number || value instanceof Boolean
|| value instanceof java.time.temporal.TemporalAccessor
|| value instanceof java.util.Date
? value : boundedText(String.valueOf(value));
} catch (SQLException exception) {
throw new BusinessException("查询结果包含无法读取的大对象");
}
}
/**
* 按 JDBC 类型选择有界流式读取或普通对象读取。
*
* @param cursor 当前结果游标
* @param column 列元数据
* @param columnIndex 从 1 开始的列序号
* @return 可安全序列化的有界值
*/
private Object readColumn(
FederationResultCursor cursor,
FederationColumn column,
int columnIndex) {
try {
if (isBinaryType(column.jdbcType())) {
try (InputStream stream = cursor.getBinaryStream(columnIndex)) {
return readBoundedBinary(stream);
}
}
if (isCharacterType(column.jdbcType(), column.typeName())) {
try (Reader reader = cursor.getCharacterStream(columnIndex)) {
return readBoundedText(reader);
}
}
} catch (UnsupportedOperationException ignored) {
// 第三方 Adapter 可暂不实现流式列读取;保留兼容的对象读取路径。
} catch (IOException exception) {
throw new BusinessException("查询结果流读取失败");
}
return normalizeValue(cursor.getObject(columnIndex));
}
/**
* 有界读取二进制流,小值编码为 Base64大值返回摘要。
*
* @param stream 二进制流;可为空
* @return Base64 文本、大小摘要或 null
* @throws IOException 流读取失败
*/
private Object readBoundedBinary(InputStream stream) throws IOException {
if (stream == null) {
return null;
}
byte[] bytes = stream.readNBytes(MAX_TEXT_CELL_CHARS + 1);
if (bytes.length > MAX_TEXT_CELL_CHARS) {
return "[binary exceeds " + MAX_TEXT_CELL_CHARS + " bytes]";
}
return Base64.getEncoder().encodeToString(bytes);
}
/**
* 有界读取字符流,超过单元格限制时附加截断标识。
*
* @param reader 字符流;可为空
* @return 有界文本或 null
* @throws IOException 流读取失败
*/
private String readBoundedText(Reader reader) throws IOException {
if (reader == null) {
return null;
}
StringBuilder text = new StringBuilder(Math.min(MAX_TEXT_CELL_CHARS, 4_096));
char[] buffer = new char[4_096];
while (text.length() <= MAX_TEXT_CELL_CHARS) {
int remaining = MAX_TEXT_CELL_CHARS + 1 - text.length();
int read = reader.read(buffer, 0, Math.min(buffer.length, remaining));
if (read < 0) {
break;
}
text.append(buffer, 0, read);
}
if (text.length() > MAX_TEXT_CELL_CHARS) {
return text.substring(0, MAX_TEXT_CELL_CHARS) + "…[truncated]";
}
return text.toString();
}
/**
* 判断 JDBC 类型是否应使用二进制流读取。
*
* @param jdbcType JDBC 类型
* @return 是否为二进制大字段类型
*/
private boolean isBinaryType(int jdbcType) {
return jdbcType == Types.BINARY
|| jdbcType == Types.VARBINARY
|| jdbcType == Types.LONGVARBINARY
|| jdbcType == Types.BLOB;
}
/**
* 判断 JDBC 类型或数据库类型名是否应使用字符流读取。
*
* @param jdbcType JDBC 类型
* @param typeName 数据库类型名
* @return 是否为字符大字段类型
*/
private boolean isCharacterType(int jdbcType, String typeName) {
if (jdbcType == Types.CHAR
|| jdbcType == Types.VARCHAR
|| jdbcType == Types.LONGVARCHAR
|| jdbcType == Types.NCHAR
|| jdbcType == Types.NVARCHAR
|| jdbcType == Types.LONGNVARCHAR
|| jdbcType == Types.CLOB
|| jdbcType == Types.NCLOB
|| jdbcType == Types.SQLXML) {
return true;
}
String normalized = typeName == null ? "" : typeName.toLowerCase(java.util.Locale.ROOT);
return normalized.equals("json") || normalized.equals("jsonb") || normalized.equals("xml");
}
private String boundedText(String value) {
if (value == null || value.length() <= MAX_TEXT_CELL_CHARS) {
return value;
}
return value.substring(0, MAX_TEXT_CELL_CHARS)
+ "…[truncated " + value.length() + " chars]";
}
private long estimateBytes(String label, Object value) {
long bytes = label == null ? 0L : (long) label.length() * Character.BYTES;
if (value == null) {
return bytes + 8L;
}
if (value instanceof CharSequence text) {
return bytes + (long) text.length() * Character.BYTES;
}
return bytes + 64L;
}
private long elapsedMillis(long startedNanos) {
return Math.max(0L, (System.nanoTime() - startedNanos) / 1_000_000L);
}
private String safeMessage(Throwable exception) {
String message = exception.getMessage();
if (message == null || message.isBlank()) {
return "数据查询失败";
}
return message.length() <= 500 ? message : message.substring(0, 500);
}
private BusinessException toBusinessException(RuntimeException exception) {
if (exception instanceof FederationSqlException federationException) {
int status = switch (federationException.errorCode()) {
case QUERY_ADMISSION_TIMEOUT, QUERY_TIMEOUT,
SQL_COMPILE_TIMEOUT -> 408;
case NODE_MEMORY_ADMISSION_TIMEOUT -> 429;
case SOURCE_NOT_FOUND, SOURCE_REMOVED -> 404;
case SOURCE_REVISION_NOT_READY, SOURCE_DEFINITION_CONFLICT -> 409;
case PLAN_STALE -> 409;
case FEDERATION_RESOURCE_LIMIT_EXCEEDED -> 422;
case EXECUTION_FAILED, RESOURCE_CLOSE_FAILED,
SOURCE_INITIALIZATION_FAILED, ENGINE_CLOSED,
EXPLAIN_FAILED, SQL_COMPLETION_FAILED,
CONNECTION_ACQUISITION_TIMEOUT,
CONNECTION_ACQUISITION_FAILED -> 503;
default -> 400;
};
int code = switch (federationException.errorCode()) {
case INVALID_ARGUMENT -> 14101;
case SQL_PARSE_FAILED -> 14102;
case SQL_VALIDATION_FAILED -> 14103;
case SQL_NOT_READ_ONLY -> 14104;
case SQL_COMPILE_FAILED -> 14105;
case SQL_NOT_FULLY_PUSHDOWN -> 14106;
case CROSS_SOURCE_DISABLED -> 14107;
case CROSS_SOURCE_EXECUTION_UNSUPPORTED -> 14108;
case SOURCE_NOT_FOUND -> 14109;
case SOURCE_REMOVED -> 14110;
case SOURCE_REVISION_NOT_READY -> 14111;
case SOURCE_DEFINITION_CONFLICT -> 14112;
case SOURCE_INITIALIZATION_FAILED -> 14113;
case ADAPTER_NOT_FOUND -> 14114;
case ADAPTER_UNSUPPORTED -> 14115;
case PARAMETER_COUNT_MISMATCH -> 14116;
case QUERY_ADMISSION_TIMEOUT -> 14117;
case QUERY_CANCELLED -> 14118;
case QUERY_TIMEOUT -> 14119;
case EXECUTION_FAILED -> 14120;
case RESOURCE_CLOSE_FAILED -> 14121;
case ENGINE_CLOSED -> 14122;
case INVALID_QUERY_SCOPE -> 14124;
case FEDERATION_OPERATOR_UNSUPPORTED -> 14125;
case FEDERATION_RESOURCE_LIMIT_EXCEEDED -> 14126;
case PLAN_STALE -> 14127;
case EXPLAIN_FAILED -> 14128;
case SQL_COMPLETION_FAILED -> 14129;
case SQL_COMPILE_TIMEOUT -> 14130;
case NODE_MEMORY_ADMISSION_TIMEOUT -> 14131;
case CONNECTION_ACQUISITION_TIMEOUT -> 14132;
case CONNECTION_ACQUISITION_FAILED -> 14133;
};
String message = switch (federationException.errorCode()) {
case EXECUTION_FAILED, RESOURCE_CLOSE_FAILED,
SOURCE_INITIALIZATION_FAILED, ENGINE_CLOSED -> "数据库查询暂时不可用";
case QUERY_TIMEOUT, QUERY_ADMISSION_TIMEOUT -> "数据库查询超时";
case QUERY_CANCELLED -> "查询已取消";
case FEDERATION_RESOURCE_LIMIT_EXCEEDED -> "联邦查询超过资源限制";
case EXPLAIN_FAILED -> "数据库查询计划分析失败";
case SQL_COMPLETION_FAILED -> "SQL 补全暂时不可用";
case SQL_COMPILE_TIMEOUT -> "SQL 编译超时,请简化语句后重试";
case NODE_MEMORY_ADMISSION_TIMEOUT -> "节点查询资源繁忙,请稍后重试";
case CONNECTION_ACQUISITION_TIMEOUT -> "数据库连接池繁忙,请稍后重试";
case CONNECTION_ACQUISITION_FAILED -> "无法获取数据库连接,请检查连接状态";
default -> safeMessage(exception);
};
return new BusinessException(status, code, message, exception);
}
return new BusinessException(500, 14999, "数据查询失败", exception);
}
private void logUnexpectedFailure(
QueryId queryId,
DatacenterSource source,
RuntimeException exception) {
if (!(exception instanceof FederationSqlException federationException)
|| switch (federationException.errorCode()) {
case EXECUTION_FAILED, RESOURCE_CLOSE_FAILED,
SOURCE_INITIALIZATION_FAILED, ENGINE_CLOSED,
EXPLAIN_FAILED, CONNECTION_ACQUISITION_FAILED -> true;
default -> false;
}) {
log.error("Datacenter query failed, queryId={}, sourceId={}",
queryId.value(), source == null ? null : source.getId(), exception);
}
}
}

View File

@@ -0,0 +1,175 @@
package tech.easyflow.datacenter.federation;
import com.easyagents.federation.sql.adapter.jdbc.JdbcFederationSqlAdapterProvider;
import com.easyagents.federation.sql.api.FederationSqlEngine;
import com.easyagents.federation.sql.api.FederationSqlEngines;
import com.easyagents.federation.sql.source.FederationSourceDefinition;
import com.easyagents.federation.sql.source.PreparedSourceRuntime;
import com.easyagents.federation.sql.source.SourceApplyOptions;
import com.easyagents.federation.sql.source.SourceApplyResult;
import com.easyagents.federation.sql.source.SourceApplyStatus;
import com.easyagents.federation.sql.source.SourceProbeResult;
import com.easyagents.federation.sql.source.SourceRemoveResult;
import com.easyagents.federation.sql.source.SourceTombstone;
import jakarta.annotation.PreDestroy;
import org.springframework.stereotype.Component;
import tech.easyflow.datacenter.meta.entity.DatacenterSource;
/**
* EasyFlow 数据中枢持有的单例 Federation SQL Engine。
*/
@Component
public class DatacenterFederationRuntime {
private final DatacenterFederationDefinitionFactory definitionFactory;
private final DatacenterFederationDataSourceResolver resolver;
private final FederationSqlEngine engine;
/**
* 组装 DataSource Resolver、数据库状态 Provider 与业务 Policy。
*
* @param definitionFactory Definition 工厂
* @param resolver HikariCP Resolver
* @param stateProvider 数据库权威状态 Provider
* @param policy 纳管范围 Policy
* @param admissionController 租户与数据源分层准入控制器
*/
public DatacenterFederationRuntime(
DatacenterFederationDefinitionFactory definitionFactory,
DatacenterFederationDataSourceResolver resolver,
DatacenterFederationSourceStateProvider stateProvider,
DatacenterFederationSqlPolicy policy,
DatacenterFederationAdmissionController admissionController) {
this.definitionFactory = definitionFactory;
this.resolver = resolver;
this.engine = FederationSqlEngines.builder()
.dataSourceResolver(resolver)
.stateProvider(stateProvider)
.adapter(new JdbcFederationSqlAdapterProvider())
.policy(policy)
.admissionController(admissionController)
.maximumPlanCacheEntries(512)
.maximumConcurrentCompilations(Math.max(1, Math.min(4,
Runtime.getRuntime().availableProcessors())))
.build();
}
/**
* 探测草稿数据源,不保留临时连接池。
*
* @param source 数据源草稿
* @return 探测结果
*/
public SourceProbeResult probe(DatacenterSource source) {
return engine.sources().probe(definitionFactory.create(source));
}
/**
* 使用未发布候选配置执行短生命周期探测。
*
* @param source 候选数据源
* @return 探测结果
*/
public SourceProbeResult probeCandidate(DatacenterSource source) {
FederationSourceDefinition definition = definitionFactory.createCandidate(source);
return resolver.withCandidate(source, definition,
() -> engine.sources().probe(definition));
}
/**
* 预构建单命名空间候选 Runtime且不修改共享 Source Slot。
*
* @param source 候选数据源
* @param catalogName 即将发布的 Catalog 或 Schema
* @return 由调用方暂时持有的候选 Runtime
*/
public PreparedSourceRuntime prepareCandidate(
DatacenterSource source,
String catalogName) {
FederationSourceDefinition definition =
definitionFactory.createCandidate(source, catalogName);
return resolver.withCandidate(source, definition,
() -> engine.sources().prepare(definition));
}
/**
* 使用当前持久化命名空间预构建候选 Runtime且不修改共享 Source Slot。
*
* @param source 候选数据源
* @return 由调用方暂时持有的候选 Runtime
*/
public PreparedSourceRuntime prepareCandidate(DatacenterSource source) {
FederationSourceDefinition definition = definitionFactory.create(source);
return resolver.withCandidate(source, definition,
() -> engine.sources().prepare(definition));
}
/**
* 原子接管已经通过业务 revision CAS 的候选 Runtime。
*
* @param prepared 预构建 Runtime
* @return Definition 应用结果
* @throws IllegalStateException Slot 已被更高版本或冲突版本占用时抛出
*/
public SourceApplyResult commitPrepared(PreparedSourceRuntime prepared) {
SourceApplyResult result = engine.sources().commit(prepared);
if (result.status() == SourceApplyStatus.CONFLICT
|| result.status() == SourceApplyStatus.IGNORED_STALE) {
throw new IllegalStateException(
"prepared runtime was superseded before commit: " + result.status());
}
return result;
}
/**
* 应用已持久化 Definition。
*
* @param source 数据源当前快照
* @param prewarm 是否立即预热
* @return 应用结果
*/
public SourceApplyResult apply(DatacenterSource source, boolean prewarm) {
FederationSourceDefinition definition = definitionFactory.create(source);
return engine.sources().apply(
definition,
prewarm ? SourceApplyOptions.prewarmNow() : SourceApplyOptions.lazy());
}
/**
* 应用禁用或删除墓碑。
*
* @param source 数据源当前快照
* @return 删除结果
*/
public SourceRemoveResult remove(DatacenterSource source) {
return engine.sources().remove(SourceTombstone.of(
definitionFactory.sourceId(source),
source.getDefinitionRevision() == null ? 0L : source.getDefinitionRevision()));
}
/**
* 返回统一查询 Engine。
*
* @return Engine
*/
public FederationSqlEngine engine() {
return engine;
}
/**
* 返回 Definition 工厂。
*
* @return Definition 工厂
*/
public DatacenterFederationDefinitionFactory definitions() {
return definitionFactory;
}
/**
* 应用停止时关闭查询、计划缓存和所有 Runtime 句柄。
*/
@PreDestroy
public void close() {
engine.close();
}
}

View File

@@ -0,0 +1,177 @@
package tech.easyflow.datacenter.federation;
import com.easyagents.federation.sql.source.ActiveSourceState;
import com.easyagents.federation.sql.source.FederationSourceState;
import com.easyagents.federation.sql.source.FederationSourceStateProvider;
import com.easyagents.federation.sql.source.SourceId;
import com.easyagents.federation.sql.source.SourceStateSubscription;
import com.easyagents.federation.sql.source.SourceTombstone;
import com.mybatisflex.core.query.QueryWrapper;
import java.math.BigInteger;
import java.util.Collection;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.concurrent.CopyOnWriteArrayList;
import java.util.function.Consumer;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Component;
import tech.easyflow.datacenter.mapper.DatacenterSourceMapper;
import tech.easyflow.datacenter.meta.entity.DatacenterSource;
import tech.easyflow.datacenter.meta.enums.DatacenterSourceStatus;
import tech.easyflow.datacenter.meta.enums.DatacenterSourceType;
/**
* 数据库权威、Redis 仅提示的 Federation Source State Provider。
*/
@Component
public class DatacenterFederationSourceStateProvider implements FederationSourceStateProvider {
private static final Logger log = LoggerFactory.getLogger(DatacenterFederationSourceStateProvider.class);
private final DatacenterSourceMapper sourceMapper;
private final DatacenterFederationDefinitionFactory definitionFactory;
private final CopyOnWriteArrayList<Consumer<FederationSourceState>> subscribers =
new CopyOnWriteArrayList<>();
/**
* 创建状态 Provider。
*
* @param sourceMapper 数据源 Mapper
* @param definitionFactory Definition 工厂
*/
public DatacenterFederationSourceStateProvider(
DatacenterSourceMapper sourceMapper,
DatacenterFederationDefinitionFactory definitionFactory) {
this.sourceMapper = sourceMapper;
this.definitionFactory = definitionFactory;
}
/**
* 从数据库读取指定数据源的最新 Definition 或墓碑。
*
* @param sourceId Federation SourceId
* @return 当前共享状态
*/
@Override
public Optional<FederationSourceState> find(SourceId sourceId) {
BigInteger recordId = parseRecordId(sourceId);
if (recordId == null) {
return Optional.empty();
}
return toState(sourceMapper.selectOneById(recordId));
}
/**
* 加载当前数据库上下文中可见的外部数据源快照。
*
* @return Definition 与墓碑集合
*/
@Override
public Collection<FederationSourceState> loadSnapshot() {
QueryWrapper query = QueryWrapper.create()
.in(DatacenterSource::getSourceType, List.of(
DatacenterSourceType.MYSQL.name(),
DatacenterSourceType.POSTGRESQL.name()))
.in(DatacenterSource::getStatus, List.of(
DatacenterSourceStatus.DRAFT.code(),
DatacenterSourceStatus.READY.code(),
DatacenterSourceStatus.DEGRADED.code(),
DatacenterSourceStatus.DISABLED.code(),
DatacenterSourceStatus.DELETED.code()));
List<DatacenterSource> sources = sourceMapper.selectListByQuery(query);
Map<BigInteger, com.easyagents.federation.sql.source.FederationSourceDefinition> definitions =
definitionFactory.createBatch(sources.stream()
.filter(this::isActive)
.toList());
return sources.stream()
.map(source -> toState(source, definitions.get(source.getId())))
.flatMap(Optional::stream)
.toList();
}
/**
* 注册节点本地变更消费者。
*
* @param consumer 状态消费者
* @return 可关闭订阅
*/
@Override
public SourceStateSubscription subscribe(Consumer<FederationSourceState> consumer) {
subscribers.add(consumer);
return () -> subscribers.remove(consumer);
}
/**
* 响应无凭据 Redis 提示,从数据库重读并发布最新状态。
*
* @param sourceId Federation SourceId
*/
public void refresh(SourceId sourceId) {
find(sourceId).ifPresent(state -> subscribers.forEach(consumer -> consumer.accept(state)));
}
/**
* 周期性回源数据库,弥补 Redis Pub/Sub 在节点离线或网络抖动时可能丢失的提示。
*/
@Scheduled(fixedDelayString = "${easyflow.datacenter.federation-reconcile-ms:30000}")
public void reconcile() {
try {
loadSnapshot().forEach(state -> subscribers.forEach(consumer -> consumer.accept(state)));
} catch (RuntimeException exception) {
log.warn("Failed to reconcile datacenter Federation source states", exception);
}
}
private Optional<FederationSourceState> toState(DatacenterSource source) {
return toState(source, null);
}
private Optional<FederationSourceState> toState(
DatacenterSource source,
com.easyagents.federation.sql.source.FederationSourceDefinition preloadedDefinition) {
if (source == null || source.getId() == null) {
return Optional.empty();
}
DatacenterSourceStatus status = DatacenterSourceStatus.fromCode(source.getStatus());
long revision = source.getDefinitionRevision() == null ? 0L : source.getDefinitionRevision();
if (status == DatacenterSourceStatus.READY || status == DatacenterSourceStatus.DEGRADED) {
if (revision <= 0L) {
return Optional.empty();
}
return Optional.of(ActiveSourceState.of(preloadedDefinition == null
? definitionFactory.create(source) : preloadedDefinition));
}
if ((status == DatacenterSourceStatus.DRAFT && revision > 0L)
|| status == DatacenterSourceStatus.DISABLED
|| status == DatacenterSourceStatus.DELETED) {
return Optional.of(SourceTombstone.of(definitionFactory.sourceId(source), revision));
}
return Optional.empty();
}
private boolean isActive(DatacenterSource source) {
DatacenterSourceStatus status = source == null
? DatacenterSourceStatus.DRAFT
: DatacenterSourceStatus.fromCode(source.getStatus());
return status == DatacenterSourceStatus.READY || status == DatacenterSourceStatus.DEGRADED;
}
private BigInteger parseRecordId(SourceId sourceId) {
if (sourceId == null) {
return null;
}
String value = sourceId.value();
int marker = value.lastIndexOf("-source-");
if (marker < 0) {
return null;
}
try {
return new BigInteger(value.substring(marker + "-source-".length()));
} catch (NumberFormatException ignored) {
return null;
}
}
}

View File

@@ -0,0 +1,859 @@
package tech.easyflow.datacenter.federation;
import com.easyagents.federation.sql.api.FederationSqlErrorCode;
import com.easyagents.federation.sql.api.FederationSqlException;
import com.easyagents.federation.sql.compile.FederationSqlPolicy;
import com.easyagents.federation.sql.compile.SqlPolicyContext;
import com.mybatisflex.core.query.QueryWrapper;
import java.math.BigInteger;
import java.util.ArrayDeque;
import java.util.Collections;
import java.util.Deque;
import java.util.HashMap;
import java.util.HashSet;
import java.util.IdentityHashMap;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentLinkedQueue;
import org.apache.calcite.rel.RelNode;
import org.apache.calcite.rel.RelVisitor;
import org.apache.calcite.rel.core.TableScan;
import org.apache.calcite.sql.SqlIdentifier;
import org.apache.calcite.sql.SqlCall;
import org.apache.calcite.sql.SqlKind;
import org.apache.calcite.sql.SqlJoin;
import org.apache.calcite.sql.SqlNode;
import org.apache.calcite.sql.SqlNodeList;
import org.apache.calcite.sql.SqlOrderBy;
import org.apache.calcite.sql.SqlSelect;
import org.apache.calcite.sql.SqlWith;
import org.apache.calcite.sql.SqlWithItem;
import org.apache.calcite.sql.util.SqlBasicVisitor;
import org.springframework.stereotype.Component;
import tech.easyflow.datacenter.entity.DatacenterTable;
import tech.easyflow.datacenter.entity.DatacenterTableField;
import tech.easyflow.datacenter.mapper.DatacenterCatalogMapper;
import tech.easyflow.datacenter.mapper.DatacenterSourceMapper;
import tech.easyflow.datacenter.mapper.DatacenterTableFieldMapper;
import tech.easyflow.datacenter.mapper.DatacenterTableMapper;
import tech.easyflow.datacenter.meta.entity.DatacenterCatalog;
import tech.easyflow.datacenter.meta.entity.DatacenterSource;
import tech.easyflow.datacenter.meta.enums.DatacenterMetadataStatus;
import tech.easyflow.datacenter.meta.enums.DatacenterSensitivityLevel;
import tech.easyflow.datacenter.meta.enums.DatacenterSourceStatus;
/**
* 将 Calcite 解析出的表与字段引用限制在当前数据源的可查询纳管范围内。
*/
@Component
public class DatacenterFederationSqlPolicy implements FederationSqlPolicy {
private static final int MAX_POLICY_CACHE_ENTRIES = 512;
private final DatacenterTableMapper tableMapper;
private final DatacenterTableFieldMapper tableFieldMapper;
private final DatacenterCatalogMapper catalogMapper;
private final DatacenterSourceMapper sourceMapper;
private final Map<PolicyCacheKey, PolicySnapshot> policyCache = new ConcurrentHashMap<>();
private final ConcurrentLinkedQueue<PolicyCacheKey> policyCacheOrder = new ConcurrentLinkedQueue<>();
/**
* 创建业务 SQL Policy。
*
* @param tableMapper 表 Mapper
* @param tableFieldMapper 字段 Mapper
* @param catalogMapper 命名空间 Mapper
* @param sourceMapper 数据源 Mapper
*/
public DatacenterFederationSqlPolicy(
DatacenterTableMapper tableMapper,
DatacenterTableFieldMapper tableFieldMapper,
DatacenterCatalogMapper catalogMapper,
DatacenterSourceMapper sourceMapper) {
this.tableMapper = tableMapper;
this.tableFieldMapper = tableFieldMapper;
this.catalogMapper = catalogMapper;
this.sourceMapper = sourceMapper;
}
/**
* 返回 Policy 代码版本。
*
* @return 稳定版本
*/
@Override
public String version() {
return "xl01-wp1-v6";
}
/**
* 拒绝未纳管表、不可查询字段和任何尚未支持脱敏的敏感字段。
*
* @param context Calcite 策略上下文
*/
@Override
public void validate(SqlPolicyContext context) {
if (context.request().queryScope().bindings().size() != 1
|| context.referencedSources().size() != 1
|| !context.referencedSources().contains(
context.request().sourceId())) {
throw validationFailure("当前数据中枢尚未开放跨数据源授权范围");
}
BigInteger sourceId = parseRecordId(context.request().sourceId().value());
requireCurrentScopeRevision(
sourceId, context.request().policyVersion());
PolicyCacheKey cacheKey = new PolicyCacheKey(sourceId, context.request().policyVersion());
PolicySnapshot snapshot = cachedSnapshot(cacheKey);
Set<String> referencedNames = referencedTables(context.relRoot().rel);
Set<BigInteger> referencedTableIds = new LinkedHashSet<>();
for (String referencedName : referencedNames) {
Set<BigInteger> tableIds = snapshot.tableIdsByName().get(referencedName);
if (tableIds == null || tableIds.isEmpty()) {
throw validationFailure("SQL 引用了未纳管表: " + referencedName);
}
referencedTableIds.addAll(tableIds);
}
validatePhysicalColumns(context.relRoot().rel, snapshot);
validateFields(context, snapshot, referencedTableIds);
}
/**
* 直接核对数据库权威范围版本,避免跨节点通知延迟复用旧策略。
*
* @param sourceId 数据源记录 ID
* @param policyVersion 编译请求携带的范围版本
*/
private void requireCurrentScopeRevision(
BigInteger sourceId,
String policyVersion) {
long requestedRevision = parseScopeRevision(policyVersion);
DatacenterSource current = sourceMapper.selectOneById(sourceId);
if (current == null
|| DatacenterSourceStatus.fromCode(current.getStatus())
!= DatacenterSourceStatus.READY
|| current.getScopeRevision() == null
|| current.getScopeRevision() != requestedRevision) {
throw validationFailure("数据范围已变化,请重新执行查询");
}
}
/**
* 解析范围策略版本。
*
* @param policyVersion 策略版本
* @return 范围 revision
*/
private long parseScopeRevision(String policyVersion) {
if (policyVersion == null || !policyVersion.startsWith("scope-")) {
throw validationFailure("数据范围版本无效");
}
try {
return Long.parseLong(policyVersion.substring("scope-".length()));
} catch (NumberFormatException exception) {
throw validationFailure("数据范围版本无效");
}
}
private PolicySnapshot cachedSnapshot(PolicyCacheKey cacheKey) {
PolicySnapshot existing = policyCache.get(cacheKey);
if (existing != null) {
return existing;
}
PolicySnapshot loaded = loadSnapshot(cacheKey.sourceId());
PolicySnapshot raced = policyCache.putIfAbsent(cacheKey, loaded);
if (raced != null) {
return raced;
}
policyCacheOrder.add(cacheKey);
trimCache();
return loaded;
}
private void trimCache() {
while (policyCache.size() > MAX_POLICY_CACHE_ENTRIES) {
PolicyCacheKey eldest = policyCacheOrder.poll();
if (eldest == null) {
return;
}
policyCache.remove(eldest);
}
}
private PolicySnapshot loadSnapshot(BigInteger sourceId) {
QueryWrapper tableQuery = QueryWrapper.create()
.eq(DatacenterTable::getSourceId, sourceId)
.eq(DatacenterTable::getQueryable, 1)
.eq(DatacenterTable::getMetadataStatus, DatacenterMetadataStatus.ACTIVE.name());
List<DatacenterTable> tables = tableMapper.selectListByQuery(tableQuery);
if (tables.isEmpty()) {
return PolicySnapshot.empty();
}
Set<BigInteger> catalogIds = new HashSet<>();
Set<BigInteger> tableIds = new HashSet<>();
for (DatacenterTable table : tables) {
tableIds.add(table.getId());
if (table.getCatalogId() != null) {
catalogIds.add(table.getCatalogId());
}
}
Map<BigInteger, String> logicalSchemas = new HashMap<>();
if (!catalogIds.isEmpty()) {
QueryWrapper catalogQuery = QueryWrapper.create().in(DatacenterCatalog::getId, catalogIds);
for (DatacenterCatalog catalog : catalogMapper.selectListByQuery(catalogQuery)) {
logicalSchemas.put(catalog.getId(), catalog.getLogicalSchemaName());
}
}
Map<String, Set<BigInteger>> idsByName = new HashMap<>();
Set<String> tableReferenceNames = new HashSet<>();
for (DatacenterTable table : tables) {
addTableName(idsByName, tableReferenceNames, table.getTableName(), table.getId());
addTableName(idsByName, tableReferenceNames, table.getActualTable(), table.getId());
String schema = logicalSchemas.get(table.getCatalogId());
if (schema != null && !schema.isBlank()) {
addTableName(idsByName, tableReferenceNames, schema + "." + table.getTableName(), table.getId());
addTableName(idsByName, tableReferenceNames, schema + "." + table.getActualTable(), table.getId());
}
}
Map<BigInteger, ColumnPolicy> columnsByTable = new HashMap<>();
QueryWrapper fieldQuery = QueryWrapper.create().in(DatacenterTableField::getTableId, tableIds);
for (DatacenterTableField field : tableFieldMapper.selectListByQuery(fieldQuery)) {
ColumnPolicy columns = columnsByTable.computeIfAbsent(
field.getTableId(), ignored -> ColumnPolicy.mutable());
String columnName = sourceColumnName(field);
if (columnName != null && !columnName.isBlank()) {
columns.knownExact().add(columnName);
columns.knownFolded().add(columnName.toUpperCase(Locale.ROOT));
if (fieldAllowed(field)) {
columns.allowedExact().add(columnName);
columns.allowedFolded().add(columnName.toUpperCase(Locale.ROOT));
}
}
}
Map<String, Set<BigInteger>> immutableNames = new HashMap<>();
idsByName.forEach((name, ids) -> immutableNames.put(name, Set.copyOf(ids)));
return new PolicySnapshot(
Map.copyOf(immutableNames),
immutableColumnPolicies(columnsByTable),
Set.copyOf(tableReferenceNames));
}
private Map<BigInteger, ColumnPolicy> immutableColumnPolicies(
Map<BigInteger, ColumnPolicy> mutablePolicies) {
Map<BigInteger, ColumnPolicy> immutable = new HashMap<>();
mutablePolicies.forEach((tableId, policy) -> immutable.put(tableId, new ColumnPolicy(
Set.copyOf(policy.knownExact()),
Set.copyOf(policy.knownFolded()),
Set.copyOf(policy.allowedExact()),
Set.copyOf(policy.allowedFolded()))));
return Map.copyOf(immutable);
}
private void validatePhysicalColumns(RelNode root, PolicySnapshot snapshot) {
new RelVisitor() {
@Override
public void visit(RelNode node, int ordinal, RelNode parent) {
if (node instanceof TableScan scan) {
String referencedName = referencedTableName(scan);
Set<BigInteger> tableIds = snapshot.tableIdsByName().get(referencedName);
if (tableIds == null || tableIds.isEmpty()) {
throw validationFailure("SQL 引用了未纳管表: " + referencedName);
}
for (BigInteger tableId : tableIds) {
ColumnPolicy policy = snapshot.columnsByTable().get(tableId);
if (policy == null || policy.knownExact().isEmpty()) {
throw validationFailure("数据表尚未纳管任何字段: " + referencedName);
}
for (String fieldName : scan.getRowType().getFieldNames()) {
if (!policy.knownExact().contains(fieldName)
&& !policy.knownFolded().contains(fieldName.toUpperCase(Locale.ROOT))) {
throw validationFailure("物理表包含尚未纳管的字段: " + fieldName);
}
}
}
}
super.visit(node, ordinal, parent);
}
}.go(root);
}
private void validateFields(
SqlPolicyContext context,
PolicySnapshot snapshot,
Set<BigInteger> referencedTableIds) {
Deque<AliasContext> scopes = new ArrayDeque<>();
Map<String, DerivedRelation> commonTables = collectCommonTables(
context.validatedSql(), snapshot, referencedTableIds);
context.validatedSql().accept(new SqlBasicVisitor<Void>() {
@Override
public Void visit(SqlCall call) {
AliasContext scope = scopeFor(
call, snapshot, referencedTableIds, commonTables);
if (scope == null) {
return super.visit(call);
}
scopes.push(scope);
try {
return super.visit(call);
} finally {
scopes.pop();
}
}
@Override
public Void visit(SqlIdentifier identifier) {
if (scopes.isEmpty()) {
return null;
}
AliasContext aliasContext = scopes.peek();
if (aliasContext.syntaxAliases().contains(identifier)
|| aliasContext.tableIdentifiers().contains(identifier)
|| aliasContext.outputAliasReferences().contains(identifier)) {
return null;
}
if (identifier.isStar()) {
boolean allAllowed = referencedTableIds.stream()
.map(snapshot.columnsByTable()::get)
.allMatch(policy -> policy != null
&& policy.knownExact().equals(policy.allowedExact()));
if (!allAllowed) {
throw validationFailure("查询范围包含不可查询或敏感字段,请显式选择允许字段");
}
return null;
}
List<String> names = identifier.names;
if (names.isEmpty()) {
return null;
}
int last = names.size() - 1;
String columnName = names.get(last);
boolean quoted = identifier.isComponentQuoted(last);
if (derivedColumnReference(identifier, aliasContext)) {
return null;
}
Set<BigInteger> targetTableIds = resolveColumnTables(
identifier, snapshot, referencedTableIds, aliasContext);
boolean allowed = !targetTableIds.isEmpty() && targetTableIds.stream()
.allMatch(tableId -> columnAllowed(
snapshot.columnsByTable().get(tableId), columnName, quoted));
if (!allowed) {
throw validationFailure("SQL 引用了不可查询或敏感字段: " + columnName);
}
return null;
}
});
}
private AliasContext scopeFor(
SqlCall call,
PolicySnapshot snapshot,
Set<BigInteger> referencedTableIds,
Map<String, DerivedRelation> commonTables) {
if (call instanceof SqlOrderBy orderBy
&& orderBy.query instanceof SqlSelect select) {
return collectAliases(
select, orderBy.orderList, snapshot,
referencedTableIds, commonTables);
}
if (call instanceof SqlSelect select) {
return collectAliases(
select, null, snapshot, referencedTableIds, commonTables);
}
return null;
}
private boolean derivedColumnReference(
SqlIdentifier identifier,
AliasContext aliasContext) {
List<String> names = identifier.names;
int columnIndex = names.size() - 1;
String columnKey = identifierKey(
names.get(columnIndex),
identifier.isComponentQuoted(columnIndex));
if (names.size() == 1) {
return aliasContext.unqualifiedDerivedColumns().contains(columnKey);
}
int qualifierIndex = names.size() - 2;
String qualifierKey = identifierKey(
names.get(qualifierIndex),
identifier.isComponentQuoted(qualifierIndex));
Set<String> derivedColumns = aliasContext
.derivedColumnsByAlias().get(qualifierKey);
return derivedColumns != null && derivedColumns.contains(columnKey);
}
private Set<BigInteger> resolveColumnTables(
SqlIdentifier identifier,
PolicySnapshot snapshot,
Set<BigInteger> referencedTableIds,
AliasContext aliasContext) {
List<String> identifierNames = identifier.names;
if (identifierNames.size() >= 2) {
int qualifierIndex = identifierNames.size() - 2;
String qualifier = identifierNames.get(qualifierIndex);
Set<BigInteger> aliased = aliasContext.tableIdsByAlias().get(
identifierKey(qualifier, identifier.isComponentQuoted(qualifierIndex)));
if (aliased != null && !aliased.isEmpty()) {
return aliased;
}
Set<BigInteger> qualified = snapshot.tableIdsByName().get(qualifier);
if (qualified != null && !qualified.isEmpty()) {
Set<BigInteger> referencedQualified = new LinkedHashSet<>(qualified);
referencedQualified.retainAll(referencedTableIds);
if (!referencedQualified.isEmpty()) {
return referencedQualified;
}
}
}
if (referencedTableIds.size() == 1) {
return referencedTableIds;
}
// 无法将别名还原到单表时采用交集语义,避免多表同名列扩大权限。
return referencedTableIds;
}
private AliasContext collectAliases(
SqlSelect select,
SqlNodeList externalOrderList,
PolicySnapshot snapshot,
Set<BigInteger> referencedTableIds,
Map<String, DerivedRelation> commonTables) {
Map<String, Set<BigInteger>> tableIdsByAlias = new HashMap<>();
Map<String, Set<String>> derivedColumnsByAlias = new HashMap<>();
Set<String> outputAliases = new HashSet<>();
Set<SqlIdentifier> syntaxAliases = Collections.newSetFromMap(
new IdentityHashMap<>());
Set<SqlIdentifier> tableIdentifiers = Collections.newSetFromMap(
new IdentityHashMap<>());
Set<SqlIdentifier> outputAliasReferences = Collections.newSetFromMap(
new IdentityHashMap<>());
collectOutputAliases(
select, outputAliases, syntaxAliases, outputAliasReferences);
FromSummary fromSummary = collectFrom(
select.getFrom(), snapshot, referencedTableIds,
commonTables, tableIdsByAlias, derivedColumnsByAlias,
syntaxAliases, tableIdentifiers);
markOutputAliasReferences(
externalOrderList, outputAliases, outputAliasReferences);
return new AliasContext(
Map.copyOf(tableIdsByAlias),
Map.copyOf(derivedColumnsByAlias),
fromSummary.singleDerived()
? fromSummary.outputColumns() : Set.of(),
Set.copyOf(outputAliases),
syntaxAliases,
tableIdentifiers,
outputAliasReferences);
}
private void collectOutputAliases(
SqlSelect select,
Set<String> outputAliases,
Set<SqlIdentifier> syntaxAliases,
Set<SqlIdentifier> outputAliasReferences) {
if (select.getSelectList() == null) {
return;
}
for (SqlNode item : select.getSelectList()) {
if (!(item instanceof SqlCall call)
|| call.getKind() != SqlKind.AS
|| call.operandCount() < 2
|| !(call.operand(1) instanceof SqlIdentifier alias)
|| alias.names.isEmpty()) {
continue;
}
int aliasIndex = alias.names.size() - 1;
outputAliases.add(identifierKey(
alias.names.get(aliasIndex),
alias.isComponentQuoted(aliasIndex)));
syntaxAliases.add(alias);
}
markOutputAliasReferences(
select.getOrderList(), outputAliases, outputAliasReferences);
markOutputAliasReferences(
select.getGroup(), outputAliases, outputAliasReferences);
}
private void markOutputAliasReferences(
SqlNode node,
Set<String> outputAliases,
Set<SqlIdentifier> outputAliasReferences) {
if (node == null) {
return;
}
if (node instanceof SqlNodeList nodes) {
for (SqlNode item : nodes) {
markOutputAliasReferences(item, outputAliases, outputAliasReferences);
}
return;
}
if (node instanceof SqlIdentifier identifier && identifier.names.size() == 1) {
if (outputAliases.contains(identifierKey(
identifier.names.get(0), identifier.isComponentQuoted(0)))) {
outputAliasReferences.add(identifier);
}
return;
}
if (node instanceof SqlCall call
&& (call.getKind() == SqlKind.DESCENDING
|| call.getKind() == SqlKind.NULLS_FIRST
|| call.getKind() == SqlKind.NULLS_LAST)
&& call.operandCount() > 0) {
markOutputAliasReferences(
call.operand(0), outputAliases, outputAliasReferences);
}
}
private FromSummary collectFrom(
SqlNode from,
PolicySnapshot snapshot,
Set<BigInteger> referencedTableIds,
Map<String, DerivedRelation> commonTables,
Map<String, Set<BigInteger>> tableIdsByAlias,
Map<String, Set<String>> derivedColumnsByAlias,
Set<SqlIdentifier> syntaxAliases,
Set<SqlIdentifier> tableIdentifiers) {
if (from == null) {
return FromSummary.empty();
}
if (from instanceof SqlIdentifier identifier) {
tableIdentifiers.add(identifier);
String relationKey = identifierKey(
identifier.names.get(identifier.names.size() - 1),
identifier.isComponentQuoted(identifier.names.size() - 1));
DerivedRelation commonTable = commonTables.get(relationKey);
if (commonTable != null) {
tableIdsByAlias.put(relationKey, commonTable.tableIds());
derivedColumnsByAlias.put(
relationKey, commonTable.outputColumns());
return new FromSummary(
commonTable.tableIds(), commonTable.outputColumns(), true);
}
return new FromSummary(
resolveReferencedTableIds(
identifier, snapshot, referencedTableIds),
Set.of(),
false);
}
if (!(from instanceof SqlCall call)) {
return FromSummary.empty();
}
if (call instanceof SqlSelect nestedSelect) {
FromSummary nested = collectFrom(
nestedSelect.getFrom(), snapshot, referencedTableIds,
commonTables, tableIdsByAlias, derivedColumnsByAlias,
syntaxAliases, tableIdentifiers);
return new FromSummary(
nested.tableIds(), outputColumnKeys(nestedSelect), true);
}
if (call instanceof SqlJoin join) {
FromSummary left = collectFrom(
join.getLeft(), snapshot, referencedTableIds,
commonTables, tableIdsByAlias, derivedColumnsByAlias,
syntaxAliases, tableIdentifiers);
FromSummary right = collectFrom(
join.getRight(), snapshot, referencedTableIds,
commonTables, tableIdsByAlias, derivedColumnsByAlias,
syntaxAliases, tableIdentifiers);
Set<BigInteger> tableIds = new LinkedHashSet<>(left.tableIds());
tableIds.addAll(right.tableIds());
return new FromSummary(Set.copyOf(tableIds), Set.of(), false);
}
if (call.getKind() == SqlKind.AS && call.operandCount() >= 2) {
SqlNode relationNode = call.operand(0);
FromSummary relation = collectFrom(
relationNode, snapshot, referencedTableIds,
commonTables, tableIdsByAlias, derivedColumnsByAlias,
syntaxAliases, tableIdentifiers);
SqlNode aliasNode = call.operand(1);
if (aliasNode instanceof SqlIdentifier alias && !alias.names.isEmpty()) {
int aliasIndex = alias.names.size() - 1;
String aliasKey = identifierKey(
alias.names.get(aliasIndex),
alias.isComponentQuoted(aliasIndex));
syntaxAliases.add(alias);
if (!relation.tableIds().isEmpty()) {
tableIdsByAlias.merge(aliasKey, relation.tableIds(), (left, right) -> {
Set<BigInteger> merged = new LinkedHashSet<>(left);
merged.addAll(right);
return Set.copyOf(merged);
});
}
if (relation.singleDerived()
&& !relation.outputColumns().isEmpty()) {
derivedColumnsByAlias.put(
aliasKey, relation.outputColumns());
}
}
return relation;
}
if (call instanceof SqlOrderBy orderBy) {
return collectFrom(
orderBy.query, snapshot, referencedTableIds,
commonTables, tableIdsByAlias, derivedColumnsByAlias,
syntaxAliases, tableIdentifiers);
}
if (call instanceof SqlWith sqlWith) {
return collectFrom(
sqlWith.body, snapshot, referencedTableIds,
commonTables, tableIdsByAlias, derivedColumnsByAlias,
syntaxAliases, tableIdentifiers);
}
return FromSummary.empty();
}
private Map<String, DerivedRelation> collectCommonTables(
SqlNode root,
PolicySnapshot snapshot,
Set<BigInteger> referencedTableIds) {
Map<String, DerivedRelation> commonTables = new HashMap<>();
collectCommonTables(
root, snapshot, referencedTableIds, commonTables);
return Map.copyOf(commonTables);
}
private void collectCommonTables(
SqlNode node,
PolicySnapshot snapshot,
Set<BigInteger> referencedTableIds,
Map<String, DerivedRelation> commonTables) {
if (node instanceof SqlOrderBy orderBy) {
collectCommonTables(
orderBy.query, snapshot, referencedTableIds, commonTables);
return;
}
if (!(node instanceof SqlWith sqlWith)) {
return;
}
for (SqlNode withNode : sqlWith.withList) {
if (!(withNode instanceof SqlWithItem withItem)
|| withItem.name == null
|| withItem.name.names.isEmpty()) {
continue;
}
collectCommonTables(
withItem.query, snapshot, referencedTableIds, commonTables);
SqlSelect select = selectBody(withItem.query);
if (select == null) {
continue;
}
FromSummary source = collectFrom(
select.getFrom(), snapshot, referencedTableIds,
commonTables, new HashMap<>(), new HashMap<>(),
Collections.newSetFromMap(new IdentityHashMap<>()),
Collections.newSetFromMap(new IdentityHashMap<>()));
Set<String> outputColumns = withItem.columnList == null
|| withItem.columnList.isEmpty()
? outputColumnKeys(select)
: identifierKeys(withItem.columnList);
int nameIndex = withItem.name.names.size() - 1;
commonTables.put(
identifierKey(
withItem.name.names.get(nameIndex),
withItem.name.isComponentQuoted(nameIndex)),
new DerivedRelation(
source.tableIds(), outputColumns));
}
collectCommonTables(
sqlWith.body, snapshot, referencedTableIds, commonTables);
}
private SqlSelect selectBody(SqlNode node) {
if (node instanceof SqlSelect select) {
return select;
}
if (node instanceof SqlOrderBy orderBy) {
return selectBody(orderBy.query);
}
if (node instanceof SqlWith sqlWith) {
return selectBody(sqlWith.body);
}
return null;
}
private Set<String> outputColumnKeys(SqlNode query) {
SqlSelect select = selectBody(query);
if (select == null || select.getSelectList() == null) {
return Set.of();
}
Set<String> outputColumns = new LinkedHashSet<>();
for (SqlNode item : select.getSelectList()) {
if (item instanceof SqlCall call
&& call.getKind() == SqlKind.AS
&& call.operandCount() >= 2
&& call.operand(1) instanceof SqlIdentifier alias
&& !alias.names.isEmpty()) {
int index = alias.names.size() - 1;
outputColumns.add(identifierKey(
alias.names.get(index),
alias.isComponentQuoted(index)));
continue;
}
if (item instanceof SqlIdentifier identifier
&& !identifier.isStar()
&& !identifier.names.isEmpty()) {
int index = identifier.names.size() - 1;
outputColumns.add(identifierKey(
identifier.names.get(index),
identifier.isComponentQuoted(index)));
}
}
return Set.copyOf(outputColumns);
}
private Set<String> identifierKeys(SqlNodeList identifiers) {
Set<String> keys = new LinkedHashSet<>();
for (SqlNode node : identifiers) {
if (node instanceof SqlIdentifier identifier
&& !identifier.names.isEmpty()) {
int index = identifier.names.size() - 1;
keys.add(identifierKey(
identifier.names.get(index),
identifier.isComponentQuoted(index)));
}
}
return Set.copyOf(keys);
}
private Set<BigInteger> resolveReferencedTableIds(
SqlIdentifier identifier,
PolicySnapshot snapshot,
Set<BigInteger> referencedTableIds) {
Set<BigInteger> candidates = snapshot.tableIdsByName().get(
String.join(".", identifier.names));
if (candidates == null || candidates.isEmpty()) {
return Set.of();
}
Set<BigInteger> referenced = new LinkedHashSet<>(candidates);
referenced.retainAll(referencedTableIds);
return Set.copyOf(referenced);
}
private String identifierKey(String name, boolean quoted) {
return quoted ? "Q:" + name : "U:" + name.toUpperCase(Locale.ROOT);
}
private boolean columnAllowed(ColumnPolicy policy, String columnName, boolean quoted) {
if (policy == null) {
return false;
}
return quoted
? policy.allowedExact().contains(columnName)
: policy.allowedFolded().contains(columnName.toUpperCase(Locale.ROOT));
}
private boolean fieldAllowed(DatacenterTableField field) {
return Integer.valueOf(1).equals(field.getQueryable())
&& DatacenterMetadataStatus.ACTIVE.name().equals(field.getMetadataStatus())
&& DatacenterSensitivityLevel.PUBLIC.name().equals(field.getSensitivityLevel());
}
private void addTableName(
Map<String, Set<BigInteger>> idsByName,
Set<String> tableReferenceNames,
String name,
BigInteger tableId) {
if (name == null || name.isBlank()) {
return;
}
idsByName.computeIfAbsent(name, ignored -> new LinkedHashSet<>()).add(tableId);
tableReferenceNames.add(name);
}
private Set<String> referencedTables(RelNode root) {
Set<String> names = new LinkedHashSet<>();
new RelVisitor() {
@Override
public void visit(RelNode node, int ordinal, RelNode parent) {
if (node instanceof TableScan scan) {
names.add(referencedTableName(scan));
}
super.visit(node, ordinal, parent);
}
}.go(root);
return names;
}
private String referencedTableName(TableScan scan) {
List<String> qualified = scan.getTable().getQualifiedName();
if (qualified.isEmpty()) {
return "";
}
String table = qualified.get(qualified.size() - 1);
return qualified.size() >= 2
? qualified.get(qualified.size() - 2) + "." + table
: table;
}
private BigInteger parseRecordId(String sourceId) {
int marker = sourceId.lastIndexOf("-source-");
if (marker < 0) {
throw new FederationSqlException(
FederationSqlErrorCode.INVALID_ARGUMENT,
"invalid data source id");
}
return new BigInteger(sourceId.substring(marker + "-source-".length()));
}
private String sourceColumnName(DatacenterTableField field) {
return field.getSourceColumnName() == null || field.getSourceColumnName().isBlank()
? field.getFieldName() : field.getSourceColumnName();
}
private FederationSqlException validationFailure(String message) {
return new FederationSqlException(FederationSqlErrorCode.SQL_VALIDATION_FAILED, message);
}
private record PolicyCacheKey(BigInteger sourceId, String policyVersion) {
}
private record AliasContext(
Map<String, Set<BigInteger>> tableIdsByAlias,
Map<String, Set<String>> derivedColumnsByAlias,
Set<String> unqualifiedDerivedColumns,
Set<String> outputAliases,
Set<SqlIdentifier> syntaxAliases,
Set<SqlIdentifier> tableIdentifiers,
Set<SqlIdentifier> outputAliasReferences) {
}
private record DerivedRelation(
Set<BigInteger> tableIds,
Set<String> outputColumns) {
}
private record FromSummary(
Set<BigInteger> tableIds,
Set<String> outputColumns,
boolean singleDerived) {
private static FromSummary empty() {
return new FromSummary(Set.of(), Set.of(), false);
}
}
private record ColumnPolicy(
Set<String> knownExact,
Set<String> knownFolded,
Set<String> allowedExact,
Set<String> allowedFolded) {
private static ColumnPolicy mutable() {
return new ColumnPolicy(new HashSet<>(), new HashSet<>(), new HashSet<>(), new HashSet<>());
}
}
private record PolicySnapshot(
Map<String, Set<BigInteger>> tableIdsByName,
Map<BigInteger, ColumnPolicy> columnsByTable,
Set<String> tableReferenceNames) {
private static PolicySnapshot empty() {
return new PolicySnapshot(Map.of(), Map.of(), Set.of());
}
}
}

View File

@@ -0,0 +1,174 @@
package tech.easyflow.datacenter.federation;
import java.nio.ByteBuffer;
import java.nio.charset.StandardCharsets;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.util.HexFormat;
import java.util.List;
import tech.easyflow.datacenter.entity.DatacenterTableField;
/**
* 使用 presence 与长度前缀生成稳定元数据摘要。
*/
public final class DatacenterMetadataIdentity {
private DatacenterMetadataIdentity() {
}
/**
* 计算物理命名空间摘要。
*
* @param catalog 物理 Catalog
* @param schema 物理 Schema
* @return SHA-256 摘要
*/
public static String namespaceKey(String catalog, String schema) {
return digest(List.of(nullable(catalog), nullable(schema)));
}
/**
* 计算物理表身份摘要。
*
* @param sourceId 数据源 ID
* @param namespaceKey 命名空间摘要
* @param physicalTable 物理表名
* @param tableKind 对象类型
* @return SHA-256 摘要
*/
public static String tableIdentity(
String sourceId,
String namespaceKey,
String physicalTable,
String tableKind) {
return digest(List.of(
"table-v1",
nullable(sourceId),
nullable(namespaceKey),
nullable(physicalTable),
nullable(tableKind)));
}
/**
* 计算表结构观测摘要。
*
* @param tableName 表名
* @param tableKind 对象类型
* @param fields 按 JDBC 顺序排列的字段
* @return SHA-256 摘要
*/
public static String tableFingerprint(
String tableName,
String tableKind,
List<DatacenterTableField> fields) {
MessageDigest digest = newDigest();
update(digest, "table-metadata-v1");
update(digest, nullable(tableName));
update(digest, nullable(tableKind));
List<DatacenterTableField> safeFields = fields == null ? List.of() : fields;
digest.update(ByteBuffer.allocate(Integer.BYTES).putInt(safeFields.size()).array());
for (DatacenterTableField field : safeFields) {
update(digest, nullable(field.getSourceColumnName()));
update(digest, nullable(field.getNativeTypeName()));
update(digest, nullable(field.getJdbcTypeCode()));
update(digest, nullable(field.getPrecision()));
update(digest, nullable(field.getScale()));
update(digest, nullable(field.getRequired()));
}
return HexFormat.of().formatHex(digest.digest());
}
/**
* 计算单字段元数据摘要。
*
* @param field 字段元数据
* @return SHA-256 摘要
*/
public static String fieldFingerprint(DatacenterTableField field) {
return digest(List.of(
"field-v1",
nullable(field.getSourceColumnName()),
nullable(field.getNativeTypeName()),
nullable(field.getJdbcTypeCode()),
nullable(field.getPrecision()),
nullable(field.getScale()),
nullable(field.getRequired())));
}
/**
* 复现 V58 的存量表身份回填算法,仅用于首次刷新时识别可安全升级的旧摘要。
*
* @param sourceId 数据源 ID
* @param catalogId 目录 ID
* @param physicalTable 物理表名
* @param tableKind 表类型
* @return V58 SHA-256 摘要
*/
public static String legacyV58TableIdentity(
Object sourceId,
Object catalogId,
String physicalTable,
String tableKind) {
String source = sourceId == null ? "0" : String.valueOf(sourceId);
String catalog = catalogId == null ? "0" : String.valueOf(catalogId);
String table = physicalTable == null ? "" : physicalTable;
String kind = tableKind == null ? "" : tableKind;
return rawDigest("S" + codePointLength(source) + ":" + source
+ "C" + codePointLength(catalog) + ":" + catalog
+ "T" + codePointLength(table) + ":" + table
+ "K" + codePointLength(kind) + ":" + kind);
}
/**
* 复现 V58 的存量表结构摘要回填算法。
*
* @param tableName 表名
* @param tableKind 表类型
* @return V58 SHA-256 摘要
*/
public static String legacyV58TableFingerprint(String tableName, String tableKind) {
String table = tableName == null ? "" : tableName;
String kind = tableKind == null ? "" : tableKind;
return rawDigest(codePointLength(table) + ":" + table
+ codePointLength(kind) + ":" + kind);
}
private static int codePointLength(String value) {
return value.codePointCount(0, value.length());
}
private static String rawDigest(String value) {
MessageDigest digest = newDigest();
return HexFormat.of().formatHex(digest.digest(value.getBytes(StandardCharsets.UTF_8)));
}
private static String digest(List<String> values) {
MessageDigest digest = newDigest();
for (String value : values) {
update(digest, value);
}
return HexFormat.of().formatHex(digest.digest());
}
private static void update(MessageDigest digest, String value) {
byte[] bytes = value.getBytes(StandardCharsets.UTF_8);
digest.update(ByteBuffer.allocate(Integer.BYTES).putInt(bytes.length).array());
digest.update(bytes);
}
private static String nullable(Object value) {
if (value == null) {
return "N";
}
String text = String.valueOf(value);
return "V" + text.length() + ":" + text;
}
private static MessageDigest newDigest() {
try {
return MessageDigest.getInstance("SHA-256");
} catch (NoSuchAlgorithmException exception) {
throw new IllegalStateException("SHA-256 is not available", exception);
}
}
}

View File

@@ -1,7 +1,18 @@
package tech.easyflow.datacenter.integration;
import tech.easyflow.common.entity.LoginAccount;
import tech.easyflow.datacenter.execution.model.DatacenterQueryRequest;
public interface AssistantDatacenterBridge {
AssistantDatacenterResult queryPage(DatacenterQueryRequest request);
/**
* 使用明确执行账号查询数据中枢。
*
* @param request 查询请求
* @param account 执行账号
* @return 查询结果
*/
AssistantDatacenterResult queryPage(
DatacenterQueryRequest request,
LoginAccount account);
}

View File

@@ -4,7 +4,7 @@ import com.mybatisflex.core.row.Row;
import tech.easyflow.datacenter.entity.DatacenterTable;
import tech.easyflow.datacenter.meta.entity.DatacenterCatalog;
import tech.easyflow.datacenter.meta.entity.DatacenterDatasetVersion;
import tech.easyflow.datacenter.meta.entity.DatacenterSource;
import tech.easyflow.datacenter.meta.model.DatacenterSourceView;
import java.util.ArrayList;
import java.util.LinkedHashMap;
@@ -13,7 +13,7 @@ import java.util.Map;
public class AssistantDatacenterResult {
private List<Row> rows = new ArrayList<>();
private DatacenterSource source;
private DatacenterSourceView source;
private DatacenterCatalog catalog;
private DatacenterTable table;
private DatacenterDatasetVersion version;
@@ -21,8 +21,8 @@ public class AssistantDatacenterResult {
public List<Row> getRows() { return rows; }
public void setRows(List<Row> rows) { this.rows = rows; }
public DatacenterSource getSource() { return source; }
public void setSource(DatacenterSource source) { this.source = source; }
public DatacenterSourceView getSource() { return source; }
public void setSource(DatacenterSourceView source) { this.source = source; }
public DatacenterCatalog getCatalog() { return catalog; }
public void setCatalog(DatacenterCatalog catalog) { this.catalog = catalog; }
public DatacenterTable getTable() { return table; }

View File

@@ -1,6 +1,7 @@
package tech.easyflow.datacenter.integration;
import org.springframework.stereotype.Component;
import tech.easyflow.common.entity.LoginAccount;
import tech.easyflow.datacenter.execution.model.DatacenterQueryRequest;
import tech.easyflow.datacenter.execution.model.DatacenterSchemaResponse;
import tech.easyflow.datacenter.execution.service.DatacenterDatasetQueryService;
@@ -17,8 +18,10 @@ public class DefaultAssistantDatacenterBridge implements AssistantDatacenterBrid
private DatacenterDatasetQueryService queryService;
@Override
public AssistantDatacenterResult queryPage(DatacenterQueryRequest request) {
var page = queryService.queryPage(request);
public AssistantDatacenterResult queryPage(
DatacenterQueryRequest request,
LoginAccount account) {
var page = queryService.queryPage(request, account);
DatacenterSchemaResponse schema = queryService.getSchema(request.getDatasetRef());
AssistantDatacenterResult result = new AssistantDatacenterResult();
result.setRows(page.getRecords());

View File

@@ -0,0 +1,10 @@
package tech.easyflow.datacenter.mapper;
import com.mybatisflex.core.BaseMapper;
import tech.easyflow.datacenter.audit.DatacenterQueryAudit;
/**
* 数据中枢查询审计 Mapper。
*/
public interface DatacenterQueryAuditMapper extends BaseMapper<DatacenterQueryAudit> {
}

View File

@@ -31,6 +31,18 @@ public class DatacenterCatalog extends DateEntity implements Serializable {
private String catalogDesc;
@Column(comment = "目录类型")
private String catalogType;
@Column(comment = "Calcite 逻辑 Schema")
private String logicalSchemaName;
@Column(comment = "JDBC 物理 Catalog")
private String physicalCatalogName;
@Column(comment = "JDBC 物理 Schema")
private String physicalSchemaName;
@Column(comment = "物理命名空间稳定摘要")
private String namespaceKey;
@Column(comment = "元数据状态")
private String metadataStatus;
@Column(comment = "最近发现时间")
private Date lastSeenAt;
@Column(comment = "状态")
private Integer status;
@Column(typeHandler = FastjsonTypeHandler.class, comment = "扩展项")
@@ -59,6 +71,18 @@ public class DatacenterCatalog extends DateEntity implements Serializable {
public void setCatalogDesc(String catalogDesc) { this.catalogDesc = catalogDesc; }
public String getCatalogType() { return catalogType; }
public void setCatalogType(String catalogType) { this.catalogType = catalogType; }
public String getLogicalSchemaName() { return logicalSchemaName; }
public void setLogicalSchemaName(String logicalSchemaName) { this.logicalSchemaName = logicalSchemaName; }
public String getPhysicalCatalogName() { return physicalCatalogName; }
public void setPhysicalCatalogName(String physicalCatalogName) { this.physicalCatalogName = physicalCatalogName; }
public String getPhysicalSchemaName() { return physicalSchemaName; }
public void setPhysicalSchemaName(String physicalSchemaName) { this.physicalSchemaName = physicalSchemaName; }
public String getNamespaceKey() { return namespaceKey; }
public void setNamespaceKey(String namespaceKey) { this.namespaceKey = namespaceKey; }
public String getMetadataStatus() { return metadataStatus; }
public void setMetadataStatus(String metadataStatus) { this.metadataStatus = metadataStatus; }
public Date getLastSeenAt() { return lastSeenAt; }
public void setLastSeenAt(Date lastSeenAt) { this.lastSeenAt = lastSeenAt; }
public Integer getStatus() { return status; }
public void setStatus(Integer status) { this.status = status; }
public Map<String, Object> getOptions() { return options; }

View File

@@ -1,5 +1,6 @@
package tech.easyflow.datacenter.meta.entity;
import com.fasterxml.jackson.annotation.JsonIgnore;
import com.fasterxml.jackson.databind.annotation.JsonSerialize;
import com.fasterxml.jackson.databind.ser.std.ToStringSerializer;
import com.mybatisflex.annotation.Column;
@@ -29,6 +30,30 @@ public class DatacenterSource extends DateEntity implements Serializable {
private String sourceCode;
@Column(comment = "数据源类型")
private String sourceType;
@Column(comment = "Federation Adapter 标识")
private String adapterId;
@Column(typeHandler = FastjsonTypeHandler.class, comment = "不含凭据的 Adapter 选项")
private Map<String, String> adapterOptionsJson;
@Column(comment = "数据源 Definition 版本")
private Long definitionRevision;
@Column(comment = "Definition SHA-256 校验和")
private String definitionChecksum;
@Column(comment = "纳管范围版本")
private Long scopeRevision;
@Column(comment = "Adapter 兼容状态")
private String compatibilityStatus;
@Column(comment = "数据库产品名称")
private String databaseProductName;
@Column(comment = "数据库产品版本")
private String databaseProductVersion;
@Column(comment = "JDBC Driver 名称")
private String driverName;
@Column(comment = "JDBC Driver 版本")
private String driverVersion;
@Column(comment = "元数据刷新状态")
private String metadataRefreshStatus;
@Column(comment = "元数据最近刷新时间")
private Date metadataRefreshedAt;
@Column(comment = "访问模式")
private String accessMode;
@Column(comment = "是否内置")
@@ -85,6 +110,30 @@ public class DatacenterSource extends DateEntity implements Serializable {
public void setSourceCode(String sourceCode) { this.sourceCode = sourceCode; }
public String getSourceType() { return sourceType; }
public void setSourceType(String sourceType) { this.sourceType = sourceType; }
public String getAdapterId() { return adapterId; }
public void setAdapterId(String adapterId) { this.adapterId = adapterId; }
public Map<String, String> getAdapterOptionsJson() { return adapterOptionsJson; }
public void setAdapterOptionsJson(Map<String, String> adapterOptionsJson) { this.adapterOptionsJson = adapterOptionsJson; }
public Long getDefinitionRevision() { return definitionRevision; }
public void setDefinitionRevision(Long definitionRevision) { this.definitionRevision = definitionRevision; }
public String getDefinitionChecksum() { return definitionChecksum; }
public void setDefinitionChecksum(String definitionChecksum) { this.definitionChecksum = definitionChecksum; }
public Long getScopeRevision() { return scopeRevision; }
public void setScopeRevision(Long scopeRevision) { this.scopeRevision = scopeRevision; }
public String getCompatibilityStatus() { return compatibilityStatus; }
public void setCompatibilityStatus(String compatibilityStatus) { this.compatibilityStatus = compatibilityStatus; }
public String getDatabaseProductName() { return databaseProductName; }
public void setDatabaseProductName(String databaseProductName) { this.databaseProductName = databaseProductName; }
public String getDatabaseProductVersion() { return databaseProductVersion; }
public void setDatabaseProductVersion(String databaseProductVersion) { this.databaseProductVersion = databaseProductVersion; }
public String getDriverName() { return driverName; }
public void setDriverName(String driverName) { this.driverName = driverName; }
public String getDriverVersion() { return driverVersion; }
public void setDriverVersion(String driverVersion) { this.driverVersion = driverVersion; }
public String getMetadataRefreshStatus() { return metadataRefreshStatus; }
public void setMetadataRefreshStatus(String metadataRefreshStatus) { this.metadataRefreshStatus = metadataRefreshStatus; }
public Date getMetadataRefreshedAt() { return metadataRefreshedAt; }
public void setMetadataRefreshedAt(Date metadataRefreshedAt) { this.metadataRefreshedAt = metadataRefreshedAt; }
public String getAccessMode() { return accessMode; }
public void setAccessMode(String accessMode) { this.accessMode = accessMode; }
public Integer getBuiltinFlag() { return builtinFlag; }
@@ -103,8 +152,10 @@ public class DatacenterSource extends DateEntity implements Serializable {
public void setSchemaName(String schemaName) { this.schemaName = schemaName; }
public String getUsername() { return username; }
public void setUsername(String username) { this.username = username; }
@JsonIgnore
public String getCredentialCipher() { return credentialCipher; }
public void setCredentialCipher(String credentialCipher) { this.credentialCipher = credentialCipher; }
@JsonIgnore
public Map<String, Object> getConfigJson() { return configJson; }
public void setConfigJson(Map<String, Object> configJson) { this.configJson = configJson; }
public Map<String, Object> getCapabilitiesJson() { return capabilitiesJson; }

View File

@@ -0,0 +1,15 @@
package tech.easyflow.datacenter.meta.enums;
/**
* 数据库对象元数据状态。
*/
public enum DatacenterMetadataStatus {
/** 当前仍能从目标数据库发现。 */
ACTIVE,
/** 本次刷新未发现。 */
MISSING,
/** 结构已变化,等待用户确认。 */
CHANGED,
/** 已由用户停止纳管。 */
RETIRED
}

View File

@@ -0,0 +1,15 @@
package tech.easyflow.datacenter.meta.enums;
/**
* 数据字段敏感级别。
*/
public enum DatacenterSensitivityLevel {
/** 可直接用于受控查询。 */
PUBLIC,
/** 仅限组织内部。 */
INTERNAL,
/** 需要脱敏后使用。 */
SENSITIVE,
/** 默认禁止进入查询结果。 */
RESTRICTED
}

View File

@@ -0,0 +1,56 @@
package tech.easyflow.datacenter.meta.enums;
import java.util.Arrays;
/**
* 数据源持久化生命周期。
*/
public enum DatacenterSourceStatus {
/** 配置已保存,尚未激活。 */
DRAFT(0),
/** Definition 已发布且节点可查询。 */
READY(1),
/** 配置仍保留,但当前 Runtime 不可用。 */
DEGRADED(2),
/** 管理员主动禁用。 */
DISABLED(3),
/** 删除墓碑,禁止新查询。 */
DELETED(4);
private final int code;
/**
* 创建状态。
*
* @param code 持久化编码
*/
DatacenterSourceStatus(int code) {
this.code = code;
}
/**
* 返回持久化编码。
*
* @return 状态编码
*/
public int code() {
return code;
}
/**
* 按持久化编码解析状态。
*
* @param code 状态编码
* @return 对应状态;未知编码按草稿处理
*/
public static DatacenterSourceStatus fromCode(Integer code) {
if (code == null) {
return DRAFT;
}
return Arrays.stream(values())
.filter(status -> status.code == code)
.findFirst()
.orElse(DRAFT);
}
}

View File

@@ -14,6 +14,9 @@ public class DatacenterCatalogMeta {
private String catalogName;
private String catalogType;
private String catalogDesc;
private String logicalSchemaName;
private String physicalCatalogName;
private String physicalSchemaName;
@JsonSerialize(using = ToStringSerializer.class)
public BigInteger getId() { return id; }
@@ -27,4 +30,10 @@ public class DatacenterCatalogMeta {
public void setCatalogType(String catalogType) { this.catalogType = catalogType; }
public String getCatalogDesc() { return catalogDesc; }
public void setCatalogDesc(String catalogDesc) { this.catalogDesc = catalogDesc; }
public String getLogicalSchemaName() { return logicalSchemaName; }
public void setLogicalSchemaName(String logicalSchemaName) { this.logicalSchemaName = logicalSchemaName; }
public String getPhysicalCatalogName() { return physicalCatalogName; }
public void setPhysicalCatalogName(String physicalCatalogName) { this.physicalCatalogName = physicalCatalogName; }
public String getPhysicalSchemaName() { return physicalSchemaName; }
public void setPhysicalSchemaName(String physicalSchemaName) { this.physicalSchemaName = physicalSchemaName; }
}

View File

@@ -2,24 +2,85 @@ package tech.easyflow.datacenter.meta.model;
import java.math.BigInteger;
/**
* 字段说明与只读查询治理更新。
*/
public class DatacenterFieldDescriptionUpdate {
private BigInteger fieldId;
private String fieldDesc;
private Integer queryable;
private String sensitivityLevel;
/**
* 获取字段 ID。
*
* @return 字段 ID
*/
public BigInteger getFieldId() {
return fieldId;
}
/**
* 设置字段 ID。
*
* @param fieldId 字段 ID
*/
public void setFieldId(BigInteger fieldId) {
this.fieldId = fieldId;
}
/**
* 获取字段说明。
*
* @return 字段说明
*/
public String getFieldDesc() {
return fieldDesc;
}
/**
* 设置字段说明。
*
* @param fieldDesc 字段说明
*/
public void setFieldDesc(String fieldDesc) {
this.fieldDesc = fieldDesc;
}
/**
* 获取是否允许只读查询。
*
* @return 1 为允许0 为禁止,空表示保持原值
*/
public Integer getQueryable() {
return queryable;
}
/**
* 设置是否允许只读查询。
*
* @param queryable 1 为允许0 为禁止,空表示保持原值
*/
public void setQueryable(Integer queryable) {
this.queryable = queryable;
}
/**
* 获取敏感级别。
*
* @return 敏感级别,空表示保持原值
*/
public String getSensitivityLevel() {
return sensitivityLevel;
}
/**
* 设置敏感级别。
*
* @param sensitivityLevel PUBLIC、INTERNAL、SENSITIVE 或 RESTRICTED
*/
public void setSensitivityLevel(String sensitivityLevel) {
this.sensitivityLevel = sensitivityLevel;
}
}

View File

@@ -0,0 +1,24 @@
package tech.easyflow.datacenter.meta.model;
import java.util.List;
import java.util.Set;
/**
* 一次连接内读取的已纳管表元数据快照。
*
* @param details 当前仍存在的表详情
* @param missingTableNames 当前连接中未发现的表名
*/
public record DatacenterManagedMetadataSnapshot(
List<DatacenterTableDetailMeta> details,
Set<String> missingTableNames) {
/**
* 对返回集合做防御性复制。
*/
public DatacenterManagedMetadataSnapshot {
details = List.copyOf(details == null ? List.of() : details);
missingTableNames = Set.copyOf(
missingTableNames == null ? Set.of() : missingTableNames);
}
}

View File

@@ -0,0 +1,55 @@
package tech.easyflow.datacenter.meta.model;
import java.util.List;
/**
* 有界元数据分页结果。
*
* @param records 当前页记录
* @param pageNumber 页码,从 1 开始
* @param pageSize 每页记录上限
* @param hasMore 是否还有下一页
* @param <T> 元数据类型
*/
public record DatacenterMetadataPage<T>(
List<T> records,
long pageNumber,
long pageSize,
boolean hasMore) {
/**
* 防御性复制当前页记录。
*/
public DatacenterMetadataPage {
records = List.copyOf(records == null ? List.of() : records);
}
/**
* 从已加载记录中创建有界分页结果。
*
* @param records 全部候选记录
* @param pageNumber 页码
* @param pageSize 每页大小
* @param <T> 元数据类型
* @return 当前页结果
*/
public static <T> DatacenterMetadataPage<T> slice(
List<T> records,
long pageNumber,
long pageSize) {
List<T> safeRecords = records == null ? List.of() : records;
long safePage = Math.max(1L, pageNumber);
long safeSize = Math.max(1L, pageSize);
if (safePage - 1L > Long.MAX_VALUE / safeSize) {
return new DatacenterMetadataPage<>(List.of(), safePage, safeSize, false);
}
long offset = (safePage - 1L) * safeSize;
if (offset >= safeRecords.size()) {
return new DatacenterMetadataPage<>(List.of(), safePage, safeSize, false);
}
int from = Math.toIntExact(offset);
int to = (int) Math.min((long) safeRecords.size(), offset + safeSize);
return new DatacenterMetadataPage<>(
safeRecords.subList(from, to), safePage, safeSize, to < safeRecords.size());
}
}

View File

@@ -9,6 +9,8 @@ public class DatacenterSaveDescriptionsRequest {
private BigInteger tableId;
private String tableDesc;
private List<DatacenterFieldDescriptionUpdate> fields = new ArrayList<>();
private Long fieldPageNumber;
private Long fieldPageSize;
public BigInteger getTableId() {
return tableId;
@@ -33,4 +35,24 @@ public class DatacenterSaveDescriptionsRequest {
public void setFields(List<DatacenterFieldDescriptionUpdate> fields) {
this.fields = fields;
}
/** @return 保存后返回的字段页码 */
public Long getFieldPageNumber() {
return fieldPageNumber;
}
/** @param fieldPageNumber 保存后返回的字段页码 */
public void setFieldPageNumber(Long fieldPageNumber) {
this.fieldPageNumber = fieldPageNumber;
}
/** @return 保存后返回的字段页大小 */
public Long getFieldPageSize() {
return fieldPageSize;
}
/** @param fieldPageSize 保存后返回的字段页大小 */
public void setFieldPageSize(Long fieldPageSize) {
this.fieldPageSize = fieldPageSize;
}
}

View File

@@ -0,0 +1,26 @@
package tech.easyflow.datacenter.meta.model;
import java.math.BigInteger;
import java.util.List;
/**
* 激活数据源并纳管选定对象的请求。
*
* @param sourceId 数据源 ID
* @param catalogName 物理 Catalog 或 Schema 名称
* @param tableNames 需要纳管的表或视图
* @param prewarm 是否立即预热节点本地连接池
*/
public record DatacenterSourceActivateRequest(
BigInteger sourceId,
String catalogName,
List<String> tableNames,
boolean prewarm) {
/**
* 防御性复制表名。
*/
public DatacenterSourceActivateRequest {
tableNames = List.copyOf(tableNames == null ? List.of() : tableNames);
}
}

View File

@@ -0,0 +1,16 @@
package tech.easyflow.datacenter.meta.model;
/**
* 未发布候选连接的命名空间分页请求。
*
* @param definition 候选连接配置
* @param keyword 命名空间搜索词
* @param pageNumber 页码
* @param pageSize 每页大小
*/
public record DatacenterSourceCandidateCatalogRequest(
DatacenterSourceDraftRequest definition,
String keyword,
Long pageNumber,
Long pageSize) {
}

View File

@@ -0,0 +1,18 @@
package tech.easyflow.datacenter.meta.model;
/**
* 活动数据源候选配置的元数据浏览请求。
*
* @param definition 候选连接配置
* @param catalogName 目标 Catalog 或 Schema
* @param keyword 表名搜索词
* @param pageNumber 页码
* @param pageSize 每页大小
*/
public record DatacenterSourceCandidateMetadataRequest(
DatacenterSourceDraftRequest definition,
String catalogName,
String keyword,
Long pageNumber,
Long pageSize) {
}

View File

@@ -0,0 +1,47 @@
package tech.easyflow.datacenter.meta.model;
import java.math.BigInteger;
import java.util.Map;
/**
* 创建或更新数据源草稿的受控请求。
*
* @param id 编辑时的数据源 ID
* @param sourceName 连接名称
* @param sourceCode 租户内业务编码
* @param sourceType 数据源类型
* @param host 主机地址
* @param port 端口
* @param databaseName 数据库名
* @param schemaName 默认 Schema
* @param username 用户名
* @param password 本次提交的明文密码;只在请求内存中短暂存在
* @param driverClassName JDBC Driver 类名
* @param jdbcUrl JDBC URL
* @param adapterOptions 不含凭据的 Adapter 选项
* @param configJson 不含凭据的连接扩展项
*/
public record DatacenterSourceDraftRequest(
BigInteger id,
String sourceName,
String sourceCode,
String sourceType,
String host,
Integer port,
String databaseName,
String schemaName,
String username,
String password,
String driverClassName,
String jdbcUrl,
Map<String, String> adapterOptions,
Map<String, Object> configJson) {
/**
* 防御性复制可变配置。
*/
public DatacenterSourceDraftRequest {
adapterOptions = Map.copyOf(adapterOptions == null ? Map.of() : adapterOptions);
configJson = Map.copyOf(configJson == null ? Map.of() : configJson);
}
}

View File

@@ -0,0 +1,29 @@
package tech.easyflow.datacenter.meta.model;
import java.util.List;
/**
* 活动数据源原地重配置请求。
*
* @param definition 候选连接配置
* @param catalogName 显式选择的 Catalog 或 Schema
* @param tableNames 显式选择的表或视图
* @param prewarm 是否在发布后预热
* @param expectedDefinitionRevision 调用方读取到的 Definition 版本
* @param expectedScopeRevision 调用方读取到的范围版本
*/
public record DatacenterSourceReconfigureRequest(
DatacenterSourceDraftRequest definition,
String catalogName,
List<String> tableNames,
boolean prewarm,
Long expectedDefinitionRevision,
Long expectedScopeRevision) {
/**
* 防御性复制表名集合。
*/
public DatacenterSourceReconfigureRequest {
tableNames = List.copyOf(tableNames == null ? List.of() : tableNames);
}
}

View File

@@ -0,0 +1,75 @@
package tech.easyflow.datacenter.meta.model;
import com.fasterxml.jackson.databind.annotation.JsonSerialize;
import com.fasterxml.jackson.databind.ser.std.ToStringSerializer;
import java.math.BigInteger;
import java.util.Date;
import java.util.Map;
/**
* 不含凭据、密文和内部 Runtime 对象的数据源视图。
*
* @param id 数据源 ID
* @param sourceName 连接名称
* @param sourceCode 业务编码
* @param sourceType 数据源类型
* @param status 生命周期名称
* @param builtinFlag 是否内置
* @param host 主机
* @param port 端口
* @param databaseName 数据库名
* @param schemaName 默认 Schema
* @param username 用户名
* @param driverClassName Driver 类名
* @param jdbcUrl JDBC URL
* @param passwordConfigured 是否已保存密码
* @param definitionRevision Definition 版本
* @param scopeRevision 纳管范围版本
* @param compatibilityStatus 兼容状态
* @param databaseProductName 数据库产品
* @param databaseProductVersion 数据库版本
* @param driverName 实际 Driver 名称
* @param driverVersion 实际 Driver 版本
* @param lastTestStatus 最近测试状态
* @param lastTestMessage 最近测试说明
* @param lastTestedAt 最近测试时间
* @param metadataRefreshStatus 元数据刷新状态
* @param metadataRefreshedAt 元数据刷新时间
* @param adapterOptions 不含凭据的 Adapter 选项
*/
public record DatacenterSourceView(
@JsonSerialize(using = ToStringSerializer.class) BigInteger id,
String sourceName,
String sourceCode,
String sourceType,
String status,
Integer builtinFlag,
String host,
Integer port,
String databaseName,
String schemaName,
String username,
String driverClassName,
String jdbcUrl,
boolean passwordConfigured,
long definitionRevision,
long scopeRevision,
String compatibilityStatus,
String databaseProductName,
String databaseProductVersion,
String driverName,
String driverVersion,
String lastTestStatus,
String lastTestMessage,
Date lastTestedAt,
String metadataRefreshStatus,
Date metadataRefreshedAt,
Map<String, String> adapterOptions) {
/**
* 防御性复制 Adapter 选项。
*/
public DatacenterSourceView {
adapterOptions = Map.copyOf(adapterOptions == null ? Map.of() : adapterOptions);
}
}

View File

@@ -0,0 +1,59 @@
package tech.easyflow.datacenter.meta.model;
import tech.easyflow.datacenter.meta.entity.DatacenterSource;
import tech.easyflow.datacenter.meta.enums.DatacenterSourceStatus;
/**
* 将持久化数据源实体转换为安全 API 视图。
*/
public final class DatacenterSourceViews {
private DatacenterSourceViews() {
}
/**
* 创建不含凭据与内部对象的数据源视图。
*
* @param source 持久化数据源实体
* @return 安全数据源视图
* @throws IllegalArgumentException 数据源为空时抛出
*/
public static DatacenterSourceView from(DatacenterSource source) {
if (source == null) {
throw new IllegalArgumentException("source must not be null");
}
return new DatacenterSourceView(
source.getId(),
source.getSourceName(),
source.getSourceCode(),
source.getSourceType(),
DatacenterSourceStatus.fromCode(source.getStatus()).name(),
source.getBuiltinFlag(),
source.getHost(),
source.getPort(),
source.getDatabaseName(),
source.getSchemaName(),
source.getUsername(),
source.getDriverClassName(),
source.getJdbcUrl(),
source.getCredentialCipher() != null
&& !source.getCredentialCipher().isBlank(),
value(source.getDefinitionRevision()),
Math.max(1L, value(source.getScopeRevision())),
source.getCompatibilityStatus(),
source.getDatabaseProductName(),
source.getDatabaseProductVersion(),
source.getDriverName(),
source.getDriverVersion(),
source.getLastTestStatus(),
source.getLastTestMessage(),
source.getLastTestedAt(),
source.getMetadataRefreshStatus(),
source.getMetadataRefreshedAt(),
source.getAdapterOptionsJson());
}
private static long value(Long value) {
return value == null ? 0L : value;
}
}

View File

@@ -6,12 +6,36 @@ import tech.easyflow.datacenter.entity.DatacenterTableField;
import java.util.ArrayList;
import java.util.List;
/**
* 数据表及其有界字段详情。
*/
public class DatacenterTableDetailMeta {
private DatacenterTable table;
private List<DatacenterTableField> fields = new ArrayList<>();
private long fieldPageNumber = 1L;
private long fieldPageSize;
private boolean hasMoreFields;
/** @return 数据表元数据 */
public DatacenterTable getTable() { return table; }
/** @param table 数据表元数据 */
public void setTable(DatacenterTable table) { this.table = table; }
/** @return 当前页字段 */
public List<DatacenterTableField> getFields() { return fields; }
public void setFields(List<DatacenterTableField> fields) { this.fields = fields; }
/** @param fields 当前页字段 */
public void setFields(List<DatacenterTableField> fields) {
this.fields = fields == null ? new ArrayList<>() : new ArrayList<>(fields);
}
/** @return 字段页码 */
public long getFieldPageNumber() { return fieldPageNumber; }
/** @param fieldPageNumber 字段页码 */
public void setFieldPageNumber(long fieldPageNumber) { this.fieldPageNumber = fieldPageNumber; }
/** @return 字段页大小 */
public long getFieldPageSize() { return fieldPageSize; }
/** @param fieldPageSize 字段页大小 */
public void setFieldPageSize(long fieldPageSize) { this.fieldPageSize = fieldPageSize; }
/** @return 是否还有字段 */
public boolean isHasMoreFields() { return hasMoreFields; }
/** @param hasMoreFields 是否还有字段 */
public void setHasMoreFields(boolean hasMoreFields) { this.hasMoreFields = hasMoreFields; }
}

View File

@@ -20,6 +20,21 @@ public interface DatacenterDatasetRegistryService {
DatacenterTable registerTable(DatacenterSource source, DatacenterCatalog catalog, DatacenterTableDetailMeta detail, LoginAccount account);
/**
* 在管理员显式重配置连接时重新绑定物理表身份和字段结构。
*
* @param source 新数据源 Definition
* @param catalog 新物理命名空间
* @param detail 新物理表元数据
* @param account 操作账号
* @return 重绑定后的纳管表
*/
DatacenterTable rebindTable(
DatacenterSource source,
DatacenterCatalog catalog,
DatacenterTableDetailMeta detail,
LoginAccount account);
DatacenterTable getTableWithFields(BigInteger tableId);
List<DatacenterTableField> getFields(BigInteger tableId);
@@ -34,5 +49,35 @@ public interface DatacenterDatasetRegistryService {
int removeTables(List<BigInteger> tableIds);
/**
* 刷新一个已纳管表的观测元数据,不自动插入新表或新字段。
*
* @param source 当前数据源
* @param catalog 已纳管命名空间
* @param tableId 已纳管表 ID
* @param detail 当前 JDBC 元数据
* @param account 当前账号
* @return 是否收缩或改变了可查询范围
*/
boolean refreshManagedTable(
DatacenterSource source,
DatacenterCatalog catalog,
BigInteger tableId,
DatacenterTableDetailMeta detail,
LoginAccount account);
/**
* 将本次无法发现的已纳管表标记为缺失。
*
* @param source 当前数据源
* @param tableId 已纳管表 ID
* @param account 当前账号
* @return 是否收缩或改变了可查询范围
*/
boolean markManagedTableMissing(
DatacenterSource source,
BigInteger tableId,
LoginAccount account);
DatacenterTable saveDescriptions(BigInteger tableId, String tableDesc, List<DatacenterFieldDescriptionUpdate> fields, LoginAccount account);
}

View File

@@ -9,24 +9,183 @@ import tech.easyflow.datacenter.meta.entity.DatacenterSource;
import tech.easyflow.datacenter.meta.model.DatacenterBatchRegisterRequest;
import tech.easyflow.datacenter.meta.model.DatacenterCatalogMeta;
import tech.easyflow.datacenter.meta.model.DatacenterTableDetailMeta;
import tech.easyflow.datacenter.meta.model.DatacenterSourceActivateRequest;
import tech.easyflow.datacenter.meta.model.DatacenterMetadataPage;
import tech.easyflow.datacenter.meta.model.DatacenterSourceCandidateMetadataRequest;
import tech.easyflow.datacenter.meta.model.DatacenterSourceCandidateCatalogRequest;
import tech.easyflow.datacenter.meta.model.DatacenterSourceDraftRequest;
import tech.easyflow.datacenter.meta.model.DatacenterSourceReconfigureRequest;
import tech.easyflow.datacenter.meta.model.DatacenterSourceView;
import java.math.BigInteger;
import java.util.List;
public interface DatacenterSourceService extends IService<DatacenterSource> {
/**
* 保存内部 Excel/项目数据源实体,外部 Controller 不应调用。
*
* @param source 内部数据源实体
* @param account 当前账号
* @return 保存后的实体
*/
DatacenterSource saveSource(DatacenterSource source, LoginAccount account);
Page<DatacenterSource> pageSources(Long pageNumber, Long pageSize, LoginAccount account);
/**
* 保存外部 JDBC 数据源草稿。
*
* @param request 受控草稿请求
* @param account 当前账号
* @return 安全数据源视图
*/
DatacenterSourceView saveDraft(DatacenterSourceDraftRequest request, LoginAccount account);
DatacenterConnectionTestResult testConnection(DatacenterSource source, LoginAccount account);
/**
* 分页返回不含凭据的数据源。
*
* @param pageNumber 页码
* @param pageSize 页大小
* @param account 当前账号
* @return 安全数据源分页
*/
Page<DatacenterSourceView> pageSources(Long pageNumber, Long pageSize, LoginAccount account);
/**
* 探测已保存草稿并记录数据库/Driver 指纹。
*
* @param sourceId 数据源 ID
* @param account 当前账号
* @return 连接探测结果
*/
DatacenterConnectionTestResult probe(BigInteger sourceId, LoginAccount account);
/**
* 激活数据源、纳管选择对象并发布 Definition。
*
* @param request 激活请求
* @param account 当前账号
* @return READY 数据源视图
*/
DatacenterSourceView activate(DatacenterSourceActivateRequest request, LoginAccount account);
/**
* 使用活动数据源的候选配置执行无状态连接探测。
*
* @param request 候选连接配置
* @param account 当前账号
* @return 连接探测结果
*/
DatacenterConnectionTestResult probeCandidate(
DatacenterSourceDraftRequest request,
LoginAccount account);
/**
* 浏览活动数据源候选配置可访问的命名空间。
*
* @param request 候选连接配置
* @param account 当前账号
* @return 命名空间列表
*/
List<DatacenterCatalogMeta> listCandidateCatalogs(
DatacenterSourceDraftRequest request,
LoginAccount account);
/**
* 分页浏览活动数据源候选配置可访问的命名空间。
*
* @param request 候选配置与分页条件
* @param account 当前账号
* @return 有界命名空间列表
*/
DatacenterMetadataPage<DatacenterCatalogMeta> listCandidateCatalogsPage(
DatacenterSourceCandidateCatalogRequest request,
LoginAccount account);
/**
* 分页浏览活动数据源候选配置可访问的表。
*
* @param request 候选配置与分页条件
* @param account 当前账号
* @return 有界表列表
*/
DatacenterMetadataPage<DatacenterTable> listCandidateTables(
DatacenterSourceCandidateMetadataRequest request,
LoginAccount account);
/**
* 原地发布活动数据源的新连接配置和显式纳管范围。
*
* @param request 重配置请求
* @param account 当前账号
* @return 发布后的安全视图
*/
DatacenterSourceView reconfigure(
DatacenterSourceReconfigureRequest request,
LoginAccount account);
List<DatacenterCatalogMeta> listCatalogs(BigInteger sourceId, LoginAccount account);
List<DatacenterTable> listTables(BigInteger sourceId, String catalogName, LoginAccount account);
/**
* 分页浏览当前数据源的命名空间。
*
* @param sourceId 数据源 ID
* @param keyword 名称搜索词
* @param pageNumber 页码
* @param pageSize 每页大小
* @param account 当前账号
* @return 有界命名空间列表
*/
DatacenterMetadataPage<DatacenterCatalogMeta> listCatalogsPage(
BigInteger sourceId,
String keyword,
Long pageNumber,
Long pageSize,
LoginAccount account);
DatacenterTableDetailMeta getTableDetail(BigInteger sourceId, String catalogName, String tableName, boolean register, LoginAccount account);
DatacenterMetadataPage<DatacenterTable> listTables(
BigInteger sourceId,
String catalogName,
String keyword,
Long pageNumber,
Long pageSize,
LoginAccount account);
DatacenterTableDetailMeta getTableDetail(
BigInteger sourceId,
String catalogName,
String tableName,
boolean register,
Long fieldPageNumber,
Long fieldPageSize,
LoginAccount account);
List<DatacenterTable> batchRegisterTables(DatacenterBatchRegisterRequest request, LoginAccount account);
void removeSource(BigInteger sourceId, LoginAccount account);
/**
* 停用活动数据源并发布墓碑。
*
* @param sourceId 数据源 ID
* @param account 当前账号
* @return 停用后的安全视图
*/
DatacenterSourceView disable(BigInteger sourceId, LoginAccount account);
/**
* 重新探测、预热并启用已停用数据源。
*
* @param sourceId 数据源 ID
* @param account 当前账号
* @return 启用后的安全视图
*/
DatacenterSourceView enable(BigInteger sourceId, LoginAccount account);
/**
* 刷新已纳管对象的 JDBC 元数据观测状态。
*
* @param sourceId 数据源 ID
* @param account 当前账号
* @return 刷新后的安全视图
*/
DatacenterSourceView refreshMetadata(BigInteger sourceId, LoginAccount account);
}

View File

@@ -9,6 +9,8 @@ import tech.easyflow.datacenter.adapter.DbHandleManager;
import tech.easyflow.datacenter.entity.DatacenterTable;
import tech.easyflow.datacenter.entity.DatacenterTableField;
import tech.easyflow.datacenter.execution.model.DatasetRef;
import tech.easyflow.datacenter.federation.DatacenterMetadataIdentity;
import tech.easyflow.datacenter.federation.DatacenterFederationChangeNotifier;
import tech.easyflow.datacenter.mapper.DatacenterCatalogMapper;
import tech.easyflow.datacenter.mapper.DatacenterDatasetVersionMapper;
import tech.easyflow.datacenter.mapper.DatacenterDerivedTableMapper;
@@ -20,6 +22,9 @@ import tech.easyflow.datacenter.meta.entity.DatacenterCatalog;
import tech.easyflow.datacenter.meta.entity.DatacenterSource;
import tech.easyflow.datacenter.meta.model.DatacenterFieldDescriptionUpdate;
import tech.easyflow.datacenter.meta.enums.DatacenterSourceType;
import tech.easyflow.datacenter.meta.enums.DatacenterMetadataStatus;
import tech.easyflow.datacenter.meta.enums.DatacenterSensitivityLevel;
import tech.easyflow.datacenter.meta.enums.DatacenterSourceStatus;
import tech.easyflow.datacenter.meta.service.DatacenterDatasetRegistryService;
import tech.easyflow.datacenter.meta.service.DatacenterMetaConstants;
import tech.easyflow.datacenter.meta.model.DatacenterTableDetailMeta;
@@ -30,7 +35,11 @@ import java.math.BigInteger;
import java.util.Date;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.stream.Collectors;
@Service
public class DatacenterDatasetRegistryServiceImpl implements DatacenterDatasetRegistryService {
@@ -51,6 +60,8 @@ public class DatacenterDatasetRegistryServiceImpl implements DatacenterDatasetRe
private DatacenterImportJobMapper importJobMapper;
@Resource
private DbHandleManager dbHandleManager;
@Resource
private DatacenterFederationChangeNotifier federationChangeNotifier;
@Override
public DatacenterSource ensureBuiltinSource(DatacenterSourceType sourceType, LoginAccount account) {
@@ -67,7 +78,11 @@ public class DatacenterDatasetRegistryServiceImpl implements DatacenterDatasetRe
source.setDeptId(deptId);
source.setSourceType(sourceType.name());
source.setBuiltinFlag(1);
source.setStatus(0);
source.setStatus(DatacenterSourceStatus.READY.code());
source.setDefinitionRevision(1L);
source.setScopeRevision(1L);
source.setMetadataRefreshStatus("SUCCESS");
source.setMetadataRefreshedAt(new Date());
source.setCreated(new Date());
source.setModified(new Date());
source.setCreatedBy(account == null ? BigInteger.ZERO : account.getId());
@@ -92,6 +107,12 @@ public class DatacenterDatasetRegistryServiceImpl implements DatacenterDatasetRe
wrapper.eq(DatacenterCatalog::getCatalogName, catalogName);
DatacenterCatalog catalog = catalogMapper.selectOneByQuery(wrapper);
if (catalog != null) {
boolean changed = applyCatalogIdentity(catalog, source, catalogName);
if (changed) {
catalog.setModified(new Date());
catalog.setModifiedBy(account == null ? BigInteger.ZERO : account.getId());
catalogMapper.update(catalog);
}
return catalog;
}
catalog = new DatacenterCatalog();
@@ -105,6 +126,7 @@ public class DatacenterDatasetRegistryServiceImpl implements DatacenterDatasetRe
catalog.setCatalogName(catalogName);
catalog.setCatalogDesc(catalogName);
catalog.setCatalogType("DATABASE");
applyCatalogIdentity(catalog, source, catalogName);
catalog.setStatus(0);
catalog.setCreated(new Date());
catalog.setModified(new Date());
@@ -116,13 +138,43 @@ public class DatacenterDatasetRegistryServiceImpl implements DatacenterDatasetRe
@Override
public DatacenterTable registerTable(DatacenterSource source, DatacenterCatalog catalog, DatacenterTableDetailMeta detail, LoginAccount account) {
return registerTable(source, catalog, detail, account, false);
}
/**
* 显式重配置时允许在同一稳定表记录上确认新的物理身份,并重建字段治理基线。
*/
@Override
public DatacenterTable rebindTable(
DatacenterSource source,
DatacenterCatalog catalog,
DatacenterTableDetailMeta detail,
LoginAccount account) {
return registerTable(source, catalog, detail, account, true);
}
private DatacenterTable registerTable(
DatacenterSource source,
DatacenterCatalog catalog,
DatacenterTableDetailMeta detail,
LoginAccount account,
boolean allowExplicitRebind) {
DatacenterTable table = detail.getTable();
applyTableDefaults(table, source, detail);
table.setPhysicalIdentityKey(DatacenterMetadataIdentity.tableIdentity(
source.getId().toString(),
catalog.getNamespaceKey(),
table.getActualTable(),
table.getTableKind()));
table.setMetadataFingerprint(DatacenterMetadataIdentity.tableFingerprint(
table.getTableName(), table.getTableKind(), detail.getFields()));
QueryWrapper wrapper = QueryWrapper.create();
wrapper.eq(DatacenterTable::getSourceId, source.getId());
wrapper.eq(DatacenterTable::getCatalogId, catalog.getId());
wrapper.eq(DatacenterTable::getTableName, table.getTableName());
DatacenterTable existing = tableMapper.selectOneByQuery(wrapper);
List<DatacenterTableField> persistedFields = existing == null
? List.of() : getFields(existing.getId());
Date now = new Date();
if (existing == null) {
table.setSourceId(source.getId());
@@ -136,6 +188,39 @@ public class DatacenterDatasetRegistryServiceImpl implements DatacenterDatasetRe
table.setModifiedBy(account == null ? BigInteger.ZERO : account.getId());
tableMapper.insert(table);
} else {
boolean identityChanged = hasText(existing.getPhysicalIdentityKey())
&& !existing.getPhysicalIdentityKey().equals(table.getPhysicalIdentityKey())
&& !existing.getPhysicalIdentityKey().equals(
DatacenterMetadataIdentity.legacyV58TableIdentity(
existing.getSourceId(), existing.getCatalogId(),
existing.getActualTable(), existing.getTableKind()));
if (identityChanged && !allowExplicitRebind) {
throw new BusinessException("物理表身份已变化,请移除原绑定后重新接入: " + table.getTableName());
}
boolean structureChanged = false;
if (hasText(existing.getMetadataFingerprint())
&& !existing.getMetadataFingerprint().equals(table.getMetadataFingerprint())) {
boolean legacyFingerprint = existing.getMetadataFingerprint().equals(
DatacenterMetadataIdentity.legacyV58TableFingerprint(
existing.getTableName(), existing.getTableKind()));
if (!legacyFingerprint
|| !legacyStructureMatches(
persistedFields, detail.getFields())) {
structureChanged = true;
}
}
if (structureChanged && !allowExplicitRebind) {
throw new BusinessException(
"表结构已变化,请移除原绑定后重新接入: " + table.getTableName());
}
boolean resetGovernance = allowExplicitRebind
&& (identityChanged || structureChanged);
if (resetGovernance) {
tableFieldMapper.deleteByQuery(QueryWrapper.create()
.eq(DatacenterTableField::getTableId, existing.getId()));
persistedFields = List.of();
existing.setQueryable(table.getQueryable());
}
if (!hasText(existing.getTableDesc())) {
existing.setTableDesc(normalizeDescription(table.getTableDesc()));
}
@@ -145,30 +230,61 @@ public class DatacenterDatasetRegistryServiceImpl implements DatacenterDatasetRe
existing.setAccessMode(table.getAccessMode());
existing.setVersioningEnabled(table.getVersioningEnabled());
existing.setCapabilitiesJson(table.getCapabilitiesJson());
existing.setPhysicalIdentityKey(table.getPhysicalIdentityKey());
existing.setMetadataFingerprint(table.getMetadataFingerprint());
existing.setMetadataStatus(DatacenterMetadataStatus.ACTIVE.name());
existing.setLastSeenAt(now);
// 查询开关属于治理配置,元数据刷新不得覆盖人工决策。
existing.setTimeSemantics(table.getTimeSemantics());
existing.setMetadataRevision((existing.getMetadataRevision() == null
? 0L : existing.getMetadataRevision()) + 1L);
existing.setModified(now);
existing.setModifiedBy(account == null ? BigInteger.ZERO : account.getId());
tableMapper.update(existing);
table = existing;
}
Map<String, DatacenterTableField> existingFieldMap = getFields(table.getId()).stream()
.collect(LinkedHashMap::new, (map, field) -> map.put(field.getFieldName(), field), Map::putAll);
QueryWrapper deleteWrapper = QueryWrapper.create();
deleteWrapper.eq(DatacenterTableField::getTableId, table.getId());
tableFieldMapper.deleteByQuery(deleteWrapper);
Map<String, DatacenterTableField> existingFieldMap = persistedFields.stream()
.collect(LinkedHashMap::new,
(map, field) -> map.put(sourceColumnName(field), field),
Map::putAll);
for (DatacenterTableField field : detail.getFields()) {
DatacenterTableField existingField = existingFieldMap.get(field.getFieldName());
applyFieldDefaults(field);
DatacenterTableField existingField = existingFieldMap.remove(sourceColumnName(field));
if (existingField != null && hasText(existingField.getFieldDesc())) {
field.setFieldDesc(existingField.getFieldDesc());
} else {
field.setFieldDesc(normalizeDescription(field.getFieldDesc()));
}
field.setId(null);
field.setTableId(table.getId());
field.setCreated(now);
field.setModified(now);
field.setCreatedBy(account == null ? BigInteger.ZERO : account.getId());
field.setModifiedBy(account == null ? BigInteger.ZERO : account.getId());
tableFieldMapper.insert(field);
if (existingField == null) {
field.setId(null);
field.setCreated(now);
field.setCreatedBy(account == null ? BigInteger.ZERO : account.getId());
tableFieldMapper.insert(field);
} else {
field.setId(existingField.getId());
field.setCreated(existingField.getCreated());
field.setCreatedBy(existingField.getCreatedBy());
if (existingField.getSensitivityLevel() != null) {
field.setSensitivityLevel(existingField.getSensitivityLevel());
field.setMaskingStrategy(existingField.getMaskingStrategy());
}
// 元数据刷新只更新物理事实,保留字段级治理设置。
field.setQueryable(existingField.getQueryable());
field.setSortable(existingField.getSortable());
tableFieldMapper.update(field);
}
}
for (DatacenterTableField missing : existingFieldMap.values()) {
missing.setMetadataStatus(DatacenterMetadataStatus.MISSING.name());
missing.setQueryable(0);
missing.setSortable(0);
missing.setWritable(0);
missing.setModified(now);
missing.setModifiedBy(account == null ? BigInteger.ZERO : account.getId());
tableFieldMapper.update(missing);
}
table.setFields(getFields(table.getId()));
return table;
@@ -196,19 +312,71 @@ public class DatacenterDatasetRegistryServiceImpl implements DatacenterDatasetRe
fieldUpdateMap.put(field.getFieldId(), field);
}
}
boolean policyChanged = false;
for (DatacenterTableField field : table.getFields()) {
DatacenterFieldDescriptionUpdate update = fieldUpdateMap.get(field.getId());
if (update == null) {
continue;
}
field.setFieldDesc(normalizeDescription(update.getFieldDesc()));
Integer queryable = normalizedQueryable(update.getQueryable(), field.getQueryable());
String sensitivityLevel = normalizedSensitivityLevel(
update.getSensitivityLevel(), field.getSensitivityLevel());
if (!DatacenterSensitivityLevel.PUBLIC.name().equals(sensitivityLevel)
&& Integer.valueOf(1).equals(queryable)) {
throw new BusinessException("敏感字段暂不支持直接查询,请关闭可查询后保存");
}
policyChanged |= !Objects.equals(field.getQueryable(), queryable)
|| !Objects.equals(field.getSensitivityLevel(), sensitivityLevel);
field.setQueryable(queryable);
field.setSensitivityLevel(sensitivityLevel);
field.setModified(now);
field.setModifiedBy(account == null ? BigInteger.ZERO : account.getId());
tableFieldMapper.update(field);
}
if (policyChanged) {
incrementExternalScopeRevisions(List.of(table));
}
return getTableWithFields(tableId);
}
/**
* 校验并归一化字段查询开关。
*
* @param requested 请求值
* @param existing 当前值
* @return 归一化后的 0 或 1
*/
private Integer normalizedQueryable(Integer requested, Integer existing) {
if (requested == null) {
return existing == null ? 0 : existing;
}
if (requested != 0 && requested != 1) {
throw new BusinessException("queryable 只能为 0 或 1");
}
return requested;
}
/**
* 校验并归一化字段敏感级别。
*
* @param requested 请求值
* @param existing 当前值
* @return 有效枚举名称
*/
private String normalizedSensitivityLevel(String requested, String existing) {
if (requested == null || requested.isBlank()) {
return existing == null || existing.isBlank()
? DatacenterSensitivityLevel.PUBLIC.name() : existing;
}
try {
return DatacenterSensitivityLevel.valueOf(
requested.trim().toUpperCase(Locale.ROOT)).name();
} catch (IllegalArgumentException exception) {
throw new BusinessException("不支持的字段敏感级别: " + requested);
}
}
@Override
public DatacenterTable getTableWithFields(BigInteger tableId) {
DatacenterTable table = tableMapper.selectOneById(tableId);
@@ -223,7 +391,7 @@ public class DatacenterDatasetRegistryServiceImpl implements DatacenterDatasetRe
public List<DatacenterTableField> getFields(BigInteger tableId) {
QueryWrapper wrapper = QueryWrapper.create();
wrapper.eq(DatacenterTableField::getTableId, tableId);
wrapper.orderBy("id");
wrapper.orderBy("ordinal_position asc, id asc");
return tableFieldMapper.selectListByQuery(wrapper);
}
@@ -231,6 +399,7 @@ public class DatacenterDatasetRegistryServiceImpl implements DatacenterDatasetRe
public DatasetRef resolveDatasetRef(BigInteger tableId) {
DatacenterTable table = getTableWithFields(tableId);
DatasetRef ref = new DatasetRef();
ref.setTenantId(table.getTenantId());
ref.setTableId(tableId);
ref.setSourceId(table.getSourceId());
ref.setCatalogId(table.getCatalogId());
@@ -266,6 +435,7 @@ public class DatacenterDatasetRegistryServiceImpl implements DatacenterDatasetRe
if (catalogId != null) {
wrapper.eq(DatacenterTable::getCatalogId, catalogId);
}
wrapper.eq(DatacenterTable::getMetadataStatus, DatacenterMetadataStatus.ACTIVE.name());
wrapper.orderBy("created desc");
return tableMapper.selectListByQuery(wrapper);
}
@@ -280,31 +450,231 @@ public class DatacenterDatasetRegistryServiceImpl implements DatacenterDatasetRe
QueryWrapper physicalTableWrapper = QueryWrapper.create();
physicalTableWrapper.in(DatacenterTable::getId, ids);
List<DatacenterTable> tables = tableMapper.selectListByQuery(physicalTableWrapper);
List<BigInteger> authorizedIds = tables.stream()
.map(DatacenterTable::getId)
.distinct()
.toList();
if (authorizedIds.size() != ids.size()) {
throw new BusinessException("部分数据表不存在或无权删除");
}
tables.forEach(this::deletePhysicalTableIfNecessary);
QueryWrapper fieldWrapper = QueryWrapper.create();
fieldWrapper.in(DatacenterTableField::getTableId, ids);
fieldWrapper.in(DatacenterTableField::getTableId, authorizedIds);
tableFieldMapper.deleteByQuery(fieldWrapper);
QueryWrapper versionWrapper = QueryWrapper.create();
versionWrapper.in(tech.easyflow.datacenter.meta.entity.DatacenterDatasetVersion::getTableId, ids);
versionWrapper.in(tech.easyflow.datacenter.meta.entity.DatacenterDatasetVersion::getTableId, authorizedIds);
datasetVersionMapper.deleteByQuery(versionWrapper);
QueryWrapper importJobWrapper = QueryWrapper.create();
importJobWrapper.in(tech.easyflow.datacenter.meta.entity.DatacenterImportJob::getTableId, ids);
importJobWrapper.in(tech.easyflow.datacenter.meta.entity.DatacenterImportJob::getTableId, authorizedIds);
importJobMapper.deleteByQuery(importJobWrapper);
QueryWrapper upstreamWrapper = QueryWrapper.create();
upstreamWrapper.in(tech.easyflow.datacenter.meta.entity.DatacenterDerivedTable::getSourceTableId, ids);
upstreamWrapper.in(tech.easyflow.datacenter.meta.entity.DatacenterDerivedTable::getSourceTableId, authorizedIds);
derivedTableMapper.deleteByQuery(upstreamWrapper);
QueryWrapper downstreamWrapper = QueryWrapper.create();
downstreamWrapper.in(tech.easyflow.datacenter.meta.entity.DatacenterDerivedTable::getDerivedTableId, ids);
downstreamWrapper.in(tech.easyflow.datacenter.meta.entity.DatacenterDerivedTable::getDerivedTableId, authorizedIds);
derivedTableMapper.deleteByQuery(downstreamWrapper);
QueryWrapper tableWrapper = QueryWrapper.create();
tableWrapper.in(DatacenterTable::getId, ids);
return tableMapper.deleteByQuery(tableWrapper);
tableWrapper.in(DatacenterTable::getId, authorizedIds);
int removed = tableMapper.deleteByQuery(tableWrapper);
incrementExternalScopeRevisions(tables);
return removed;
}
/**
* 刷新已纳管表的物理观测值;身份、结构或历史状态存在歧义时立即收缩权限。
*/
@Override
@Transactional(rollbackFor = Exception.class)
public boolean refreshManagedTable(
DatacenterSource source,
DatacenterCatalog catalog,
BigInteger tableId,
DatacenterTableDetailMeta detail,
LoginAccount account) {
DatacenterTable existing = tableMapper.selectOneById(tableId);
if (existing == null
|| !Objects.equals(existing.getSourceId(), source.getId())
|| !Objects.equals(existing.getCatalogId(), catalog.getId())) {
throw new BusinessException("已纳管数据表不存在或不属于当前连接");
}
DatacenterTable discovered = detail == null ? null : detail.getTable();
if (discovered == null) {
throw new BusinessException("未返回数据表元数据: " + existing.getTableName());
}
applyTableDefaults(discovered, source, detail);
String discoveredIdentity = DatacenterMetadataIdentity.tableIdentity(
source.getId().toString(),
catalog.getNamespaceKey(),
discovered.getActualTable(),
discovered.getTableKind());
String discoveredFingerprint = DatacenterMetadataIdentity.tableFingerprint(
discovered.getTableName(), discovered.getTableKind(), detail.getFields());
List<DatacenterTableField> persistedFields = getFields(existing.getId());
boolean identityMatches = Objects.equals(
existing.getPhysicalIdentityKey(), discoveredIdentity)
|| Objects.equals(
existing.getPhysicalIdentityKey(),
DatacenterMetadataIdentity.legacyV58TableIdentity(
existing.getSourceId(), existing.getCatalogId(),
existing.getActualTable(), existing.getTableKind()));
boolean legacyCompatible = Objects.equals(
existing.getMetadataFingerprint(),
DatacenterMetadataIdentity.legacyV58TableFingerprint(
existing.getTableName(), existing.getTableKind()))
&& legacyStructureMatches(persistedFields, detail.getFields());
boolean fingerprintMatches = Objects.equals(
existing.getMetadataFingerprint(), discoveredFingerprint)
|| legacyCompatible;
boolean ambiguous = !DatacenterMetadataStatus.ACTIVE.name().equals(existing.getMetadataStatus())
|| !identityMatches
|| !fingerprintMatches;
if (ambiguous) {
return markTableUnavailable(
existing, DatacenterMetadataStatus.CHANGED, true, account);
}
Date now = new Date();
existing.setActualTable(discovered.getActualTable());
existing.setTableKind(discovered.getTableKind());
existing.setPhysicalIdentityKey(discoveredIdentity);
existing.setMetadataFingerprint(discoveredFingerprint);
existing.setLastSeenAt(now);
existing.setMetadataRevision((existing.getMetadataRevision() == null
? 0L : existing.getMetadataRevision()) + 1L);
existing.setModified(now);
existing.setModifiedBy(account == null ? BigInteger.ZERO : account.getId());
tableMapper.update(existing);
Map<String, DatacenterTableField> discoveredFields = detail.getFields().stream()
.collect(Collectors.toMap(
this::sourceColumnName,
field -> field,
(first, ignored) -> first,
LinkedHashMap::new));
for (DatacenterTableField persisted : persistedFields) {
DatacenterTableField observed = discoveredFields.get(sourceColumnName(persisted));
if (observed == null
|| (!legacyCompatible && !Objects.equals(
persisted.getMetadataFingerprint(),
DatacenterMetadataIdentity.fieldFingerprint(observed)))) {
// 表 fingerprint 相同的前提下不应进入该分支;保留 fail-closed 防线。
return markTableUnavailable(
existing, DatacenterMetadataStatus.CHANGED, true, account);
}
persisted.setOrdinalPosition(observed.getOrdinalPosition());
persisted.setJdbcType(observed.getJdbcType());
persisted.setJdbcTypeCode(observed.getJdbcTypeCode());
persisted.setNativeTypeName(observed.getNativeTypeName());
persisted.setPrecision(observed.getPrecision());
persisted.setScale(observed.getScale());
persisted.setRequired(observed.getRequired());
persisted.setIndexed(observed.getIndexed());
persisted.setMetadataFingerprint(
DatacenterMetadataIdentity.fieldFingerprint(observed));
persisted.setMetadataStatus(DatacenterMetadataStatus.ACTIVE.name());
persisted.setLastSeenAt(now);
persisted.setModified(now);
persisted.setModifiedBy(account == null ? BigInteger.ZERO : account.getId());
tableFieldMapper.update(persisted);
}
return false;
}
/**
* 标记未发现的已纳管表,并关闭其全部查询入口。
*/
@Override
@Transactional(rollbackFor = Exception.class)
public boolean markManagedTableMissing(
DatacenterSource source,
BigInteger tableId,
LoginAccount account) {
DatacenterTable existing = tableMapper.selectOneById(tableId);
if (existing == null || !Objects.equals(existing.getSourceId(), source.getId())) {
throw new BusinessException("已纳管数据表不存在或不属于当前连接");
}
return markTableUnavailable(
existing, DatacenterMetadataStatus.MISSING, false, account);
}
private boolean markTableUnavailable(
DatacenterTable table,
DatacenterMetadataStatus status,
boolean seen,
LoginAccount account) {
boolean changed = !status.name().equals(table.getMetadataStatus())
|| !Integer.valueOf(0).equals(table.getQueryable());
Date now = new Date();
table.setMetadataStatus(status.name());
table.setQueryable(0);
if (seen) {
table.setLastSeenAt(now);
}
table.setModified(now);
table.setModifiedBy(account == null ? BigInteger.ZERO : account.getId());
tableMapper.update(table);
for (DatacenterTableField field : getFields(table.getId())) {
changed |= !status.name().equals(field.getMetadataStatus())
|| !Integer.valueOf(0).equals(field.getQueryable());
field.setMetadataStatus(status.name());
field.setQueryable(0);
field.setSortable(0);
field.setWritable(0);
if (seen) {
field.setLastSeenAt(now);
}
field.setModified(now);
field.setModifiedBy(account == null ? BigInteger.ZERO : account.getId());
tableFieldMapper.update(field);
}
return changed;
}
/**
* 删除外部纳管表后以 CAS 推进策略版本,并在提交后通知其他节点。
*
* @param removedTables 本事务删除的纳管表
*/
private void incrementExternalScopeRevisions(List<DatacenterTable> removedTables) {
Set<BigInteger> sourceIds = removedTables.stream()
.map(DatacenterTable::getSourceId)
.filter(java.util.Objects::nonNull)
.collect(Collectors.toSet());
for (BigInteger sourceId : sourceIds) {
DatacenterSource source = sourceMapper.selectOneById(sourceId);
if (source == null || (!DatacenterSourceType.MYSQL.name().equals(source.getSourceType())
&& !DatacenterSourceType.POSTGRESQL.name().equals(source.getSourceType()))) {
continue;
}
long previous = source.getScopeRevision() == null ? 0L : source.getScopeRevision();
long definitionRevision = source.getDefinitionRevision() == null
? 0L : source.getDefinitionRevision();
DatacenterSource patch = new DatacenterSource();
patch.setId(source.getId());
patch.setScopeRevision(Math.max(1L, previous) + 1L);
patch.setModified(new Date());
QueryWrapper guard = QueryWrapper.create()
.eq(DatacenterSource::getId, source.getId())
.eq(DatacenterSource::getTenantId, source.getTenantId())
.eq(DatacenterSource::getDefinitionRevision, definitionRevision)
.eq(DatacenterSource::getScopeRevision, previous)
.ne(DatacenterSource::getStatus,
DatacenterSourceStatus.DELETED.code());
if (sourceMapper.updateByQuery(patch, guard) != 1) {
throw new BusinessException("数据范围已被其他请求更新,请刷新后重试");
}
DatacenterSource updated = sourceMapper.selectOneById(source.getId());
if (updated == null) {
throw new BusinessException("数据连接已被删除");
}
federationChangeNotifier.publishAfterCommit(updated);
}
}
private void deletePhysicalTableIfNecessary(DatacenterTable table) {
@@ -360,12 +730,130 @@ public class DatacenterDatasetRegistryServiceImpl implements DatacenterDatasetRe
if ((table.getActualTable() == null || table.getActualTable().isBlank()) && table.getTableName() != null) {
table.setActualTable(table.getTableName());
}
if (table.getQueryable() == null) {
table.setQueryable(1);
}
if (table.getMetadataStatus() == null) {
table.setMetadataStatus(DatacenterMetadataStatus.ACTIVE.name());
}
if (table.getLastSeenAt() == null) {
table.setLastSeenAt(new Date());
}
if (table.getTimeSemantics() == null) {
table.setTimeSemantics("NONE");
}
if (table.getMetadataRevision() == null) {
table.setMetadataRevision(1L);
}
table.setTableDesc(normalizeDescription(table.getTableDesc()));
if (detail.getFields() == null) {
detail.setFields(List.of());
}
}
private boolean applyCatalogIdentity(
DatacenterCatalog catalog,
DatacenterSource source,
String catalogName) {
boolean mysql = DatacenterSourceType.MYSQL.name().equals(source.getSourceType())
|| DatacenterSourceType.PROJECT_MYSQL.name().equals(source.getSourceType());
String physicalCatalog = mysql ? catalogName : source.getDatabaseName();
String physicalSchema = mysql ? null : catalogName;
String logicalSchema = catalogName;
String namespaceKey = DatacenterMetadataIdentity.namespaceKey(physicalCatalog, physicalSchema);
boolean changed = !java.util.Objects.equals(catalog.getLogicalSchemaName(), logicalSchema)
|| !java.util.Objects.equals(catalog.getPhysicalCatalogName(), physicalCatalog)
|| !java.util.Objects.equals(catalog.getPhysicalSchemaName(), physicalSchema)
|| !java.util.Objects.equals(catalog.getNamespaceKey(), namespaceKey)
|| !DatacenterMetadataStatus.ACTIVE.name().equals(catalog.getMetadataStatus());
catalog.setLogicalSchemaName(logicalSchema);
catalog.setPhysicalCatalogName(physicalCatalog);
catalog.setPhysicalSchemaName(physicalSchema);
catalog.setNamespaceKey(namespaceKey);
catalog.setMetadataStatus(DatacenterMetadataStatus.ACTIVE.name());
catalog.setLastSeenAt(new Date());
return changed;
}
private void applyFieldDefaults(DatacenterTableField field) {
if (!hasText(field.getSourceColumnName())) {
field.setSourceColumnName(field.getFieldName());
}
if (field.getNativeTypeName() == null) {
field.setNativeTypeName(field.getJdbcType());
}
field.setMetadataStatus(DatacenterMetadataStatus.ACTIVE.name());
field.setLastSeenAt(new Date());
if (field.getSensitivityLevel() == null) {
field.setSensitivityLevel(DatacenterSensitivityLevel.PUBLIC.name());
}
field.setMetadataFingerprint(DatacenterMetadataIdentity.fieldFingerprint(field));
}
/**
* 使用 V58 已持久化的旧字段信息校验首次元数据升级。
*
* <p>V58 新增的 JDBC 编码和顺序列没有回填,因此这些空值只表示未知;
* 字段名称、旧 JDBC 类型、精度、小数位和必填属性仍必须保持一致。</p>
*
* @param persistedFields V58 存量字段
* @param discoveredFields 当前 JDBC 发现字段
* @return 已知旧结构是否与当前结构一致
*/
private boolean legacyStructureMatches(
List<DatacenterTableField> persistedFields,
List<DatacenterTableField> discoveredFields) {
List<DatacenterTableField> safePersisted = persistedFields == null
? List.of() : persistedFields;
List<DatacenterTableField> safeDiscovered = discoveredFields == null
? List.of() : discoveredFields;
if (safePersisted.size() != safeDiscovered.size()) {
return false;
}
Map<String, DatacenterTableField> discoveredByName = safeDiscovered.stream()
.collect(Collectors.toMap(
this::sourceColumnName,
field -> field,
(first, ignored) -> first,
LinkedHashMap::new));
if (discoveredByName.size() != safeDiscovered.size()) {
return false;
}
for (DatacenterTableField persisted : safePersisted) {
DatacenterTableField discovered = discoveredByName.get(
sourceColumnName(persisted));
if (discovered == null
|| !sameWhenKnown(persisted.getFieldType(), discovered.getFieldType())
|| !sameWhenKnown(persisted.getJdbcType(), discovered.getJdbcType())
|| !sameWhenKnown(persisted.getNativeTypeName(), discovered.getNativeTypeName())
|| !sameWhenKnown(persisted.getJdbcTypeCode(), discovered.getJdbcTypeCode())
|| !sameWhenKnown(persisted.getPrecision(), discovered.getPrecision())
|| !sameWhenKnown(persisted.getScale(), discovered.getScale())
|| !sameWhenKnown(persisted.getRequired(), discovered.getRequired())) {
return false;
}
}
return true;
}
/**
* 比较旧元数据中的已知值,空值按 V58 未回填处理。
*
* @param persisted 旧值
* @param discovered 当前值
* @return 旧值为空或两者相等时返回 true
*/
private boolean sameWhenKnown(Object persisted, Object discovered) {
return persisted == null
|| java.util.Objects.equals(persisted, discovered);
}
private String sourceColumnName(DatacenterTableField field) {
return hasText(field.getSourceColumnName())
? field.getSourceColumnName()
: field.getFieldName();
}
private String normalizeDescription(String description) {
if (description == null) {
return "";

View File

@@ -0,0 +1,361 @@
package tech.easyflow.datacenter.meta.service.impl;
import com.easyagents.federation.sql.api.FederationSqlException;
import com.easyagents.federation.sql.source.RuntimeFingerprint;
import com.easyagents.federation.sql.source.SourceProbeResult;
import java.math.BigInteger;
import java.util.Date;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.stereotype.Service;
import tech.easyflow.common.entity.LoginAccount;
import tech.easyflow.common.web.exceptions.BusinessException;
import tech.easyflow.datacenter.execution.model.DatacenterConnectionTestResult;
import tech.easyflow.datacenter.federation.DatacenterFederationRuntime;
import tech.easyflow.datacenter.meta.entity.DatacenterSource;
import tech.easyflow.datacenter.meta.enums.DatacenterSourceStatus;
import tech.easyflow.datacenter.meta.enums.DatacenterSourceType;
import tech.easyflow.datacenter.meta.model.DatacenterSourceDraftRequest;
import tech.easyflow.datacenter.meta.support.DatacenterSourceConnectionDefaults;
import tech.easyflow.datacenter.security.DatacenterCredentialCipher;
/**
* 构造并探测未发布的数据源候选配置,确保凭据和连接默认值只在一个边界内处理。
*/
@Service
public class DatacenterSourceCandidateService {
private static final Logger LOGGER =
LoggerFactory.getLogger(DatacenterSourceCandidateService.class);
private final DatacenterCredentialCipher credentialCipher;
private final DatacenterSourceConnectionDefaults connectionDefaults;
private final DatacenterFederationRuntime federationRuntime;
/**
* 创建候选配置服务。
*
* @param credentialCipher 凭据加密器
* @param connectionDefaults JDBC 默认配置
* @param federationRuntime Federation Runtime
*/
public DatacenterSourceCandidateService(
DatacenterCredentialCipher credentialCipher,
DatacenterSourceConnectionDefaults connectionDefaults,
DatacenterFederationRuntime federationRuntime) {
this.credentialCipher = credentialCipher;
this.connectionDefaults = connectionDefaults;
this.federationRuntime = federationRuntime;
}
/**
* 构造仅可探测、不可查询的持久化草稿。
*
* @param request 草稿请求
* @param existing 当前草稿;新建时为空
* @param account 当前账号
* @return 已归一化草稿
*/
public DatacenterSource buildDraft(
DatacenterSourceDraftRequest request,
DatacenterSource existing,
LoginAccount account) {
validateDefinition(request);
requireEditableDraft(existing, request.sourceType());
DatacenterSource source = existing == null
? new DatacenterSource() : copy(existing);
source.setSourceName(request.sourceName().trim());
source.setSourceCode(trimToNull(request.sourceCode()));
source.setSourceType(request.sourceType());
source.setAccessMode("READ_ONLY");
source.setBuiltinFlag(0);
source.setAdapterId("jdbc");
source.setAdapterOptionsJson(Map.of());
source.setHost(request.host().trim());
source.setPort(request.port());
source.setDatabaseName(request.databaseName().trim());
source.setSchemaName(trimToNull(request.schemaName()));
source.setUsername(request.username().trim());
source.setDriverClassName(
connectionDefaults.defaultDriverClassName(request.sourceType()));
source.setJdbcUrl(null);
source.setConfigJson(Map.of());
applyCredential(source, existing, trimToNull(request.password()));
connectionDefaults.normalize(source);
requireCredential(source);
source.setDefinitionRevision(existing == null
? 1L : value(existing.getDefinitionRevision()) + 1L);
source.setScopeRevision(Math.max(1L, value(source.getScopeRevision())));
source.setDefinitionChecksum(null);
source.setCompatibilityStatus(null);
source.setMetadataRefreshStatus("PENDING");
source.setStatus(DatacenterSourceStatus.DRAFT.code());
source.setCapabilitiesJson(Map.of(
"capabilities", List.of(
"TEST_CONNECTION", "BROWSE_METADATA", "READ_QUERY")));
if (source.getTenantId() == null && account != null) {
source.setTenantId(account.getTenantId());
source.setDeptId(account.getDeptId());
}
return source;
}
/**
* 构造活动连接的下一 revision 候选配置。
*
* @param current 当前已发布配置
* @param request 新连接定义
* @return 未发布候选配置
*/
public DatacenterSource buildReconfigurationCandidate(
DatacenterSource current,
DatacenterSourceDraftRequest request) {
validateDefinition(request);
DatacenterSourceStatus status = DatacenterSourceStatus.fromCode(current.getStatus());
if (status != DatacenterSourceStatus.READY
&& status != DatacenterSourceStatus.DEGRADED) {
throw new BusinessException("只有已激活连接可以使用候选配置");
}
if (!java.util.Objects.equals(current.getSourceType(), request.sourceType())) {
throw new BusinessException("数据连接类型创建后不可修改");
}
DatacenterSource candidate = copy(current);
candidate.setSourceName(request.sourceName().trim());
String requestedCode = trimToNull(request.sourceCode());
candidate.setSourceCode(requestedCode == null
? current.getSourceCode() : requestedCode);
candidate.setAccessMode("READ_ONLY");
candidate.setBuiltinFlag(0);
candidate.setAdapterId("jdbc");
candidate.setAdapterOptionsJson(Map.of());
candidate.setHost(request.host().trim());
candidate.setPort(request.port());
candidate.setDatabaseName(request.databaseName().trim());
candidate.setSchemaName(trimToNull(request.schemaName()));
candidate.setUsername(request.username().trim());
candidate.setDriverClassName(
connectionDefaults.defaultDriverClassName(request.sourceType()));
candidate.setJdbcUrl(null);
candidate.setConfigJson(Map.of());
applyCredential(candidate, current, trimToNull(request.password()));
connectionDefaults.normalize(candidate);
requireCredential(candidate);
candidate.setDefinitionRevision(value(current.getDefinitionRevision()) + 1L);
candidate.setScopeRevision(current.getScopeRevision());
candidate.setDefinitionChecksum(null);
candidate.setStatus(DatacenterSourceStatus.DRAFT.code());
candidate.setLastTestStatus(null);
candidate.setLastTestMessage(null);
candidate.setLastTestedAt(null);
return candidate;
}
/**
* 使用短生命周期候选上下文探测连接,并将数据库指纹写回候选实体。
*
* @param candidate 候选配置
* @return 安全探测结果
*/
public DatacenterConnectionTestResult probe(DatacenterSource candidate) {
Date testedAt = new Date();
DatacenterConnectionTestResult result = new DatacenterConnectionTestResult();
try {
DatacenterSourceStatus status = DatacenterSourceStatus.fromCode(
candidate.getStatus());
SourceProbeResult probe = status == DatacenterSourceStatus.READY
|| status == DatacenterSourceStatus.DEGRADED
? federationRuntime.probe(candidate)
: federationRuntime.probeCandidate(candidate);
RuntimeFingerprint fingerprint = probe.fingerprint();
result.setSuccess(probe.supported());
result.setErrorCode(probe.supported() ? null : "ADAPTER_UNSUPPORTED");
result.setMessage(bounded(probe.diagnostic()));
result.setCapabilities(List.of(
"TEST_CONNECTION", "BROWSE_METADATA", "READ_QUERY"));
result.setDetails(safeProbeDetails(candidate, fingerprint));
candidate.setLastTestStatus(probe.supported() ? "SUCCESS" : "FAILED");
candidate.setLastTestMessage(result.getMessage());
candidate.setCompatibilityStatus(
probe.supported() ? "CODE_SUPPORTED_UNVERIFIED" : "UNSUPPORTED");
applyFingerprint(candidate, fingerprint);
} catch (FederationSqlException exception) {
LOGGER.warn("候选数据源探测失败sourceId={}, code={}",
candidate.getId(), exception.errorCode(), exception);
result.setSuccess(false);
result.setErrorCode(exception.errorCode().name());
result.setMessage(bounded(exception.getMessage()));
candidate.setLastTestStatus("FAILED");
candidate.setLastTestMessage(result.getMessage());
} catch (RuntimeException exception) {
LOGGER.error("候选数据源探测发生未预期错误sourceId={}",
candidate.getId(), exception);
result.setSuccess(false);
result.setErrorCode("CONNECTION_FAILED");
result.setMessage("无法连接到数据库,请检查地址、账号和网络");
candidate.setLastTestStatus("FAILED");
candidate.setLastTestMessage(result.getMessage());
}
candidate.setLastTestedAt(testedAt);
return result;
}
/**
* 加密新密码,空密码则保留已有密文。
*
* @param target 目标实体
* @param existing 当前实体
* @param password 可选明文密码
*/
public void applyCredential(
DatacenterSource target,
DatacenterSource existing,
String password) {
if (password != null) {
target.setCredentialCipher(credentialCipher.encrypt(password));
return;
}
if ((target.getCredentialCipher() == null
|| target.getCredentialCipher().isBlank()) && existing != null) {
target.setCredentialCipher(existing.getCredentialCipher());
}
}
/**
* 从旧兼容配置中读取待加密密码。
*
* @param configJson 兼容配置
* @return 归一化密码;不存在时为空
*/
public String extractPassword(Map<String, Object> configJson) {
if (configJson == null || configJson.get("password") == null) {
return null;
}
return trimToNull(String.valueOf(configJson.get("password")));
}
/**
* 复制数据源快照,避免候选探测修改权威实体。
*
* @param source 原实体
* @return 独立副本
*/
public DatacenterSource copy(DatacenterSource source) {
DatacenterSource copy = new DatacenterSource();
copy.setId(source.getId());
copy.setTenantId(source.getTenantId());
copy.setDeptId(source.getDeptId());
copy.setCreated(source.getCreated());
copy.setCreatedBy(source.getCreatedBy());
copy.setSourceName(source.getSourceName());
copy.setSourceCode(source.getSourceCode());
copy.setSourceType(source.getSourceType());
copy.setAdapterId(source.getAdapterId());
copy.setAdapterOptionsJson(source.getAdapterOptionsJson());
copy.setDefinitionRevision(source.getDefinitionRevision());
copy.setDefinitionChecksum(source.getDefinitionChecksum());
copy.setScopeRevision(source.getScopeRevision());
copy.setAccessMode(source.getAccessMode());
copy.setBuiltinFlag(source.getBuiltinFlag());
copy.setCredentialCipher(source.getCredentialCipher());
copy.setDriverClassName(source.getDriverClassName());
copy.setJdbcUrl(source.getJdbcUrl());
copy.setHost(source.getHost());
copy.setPort(source.getPort());
copy.setDatabaseName(source.getDatabaseName());
copy.setSchemaName(source.getSchemaName());
copy.setUsername(source.getUsername());
copy.setConfigJson(source.getConfigJson());
copy.setCapabilitiesJson(source.getCapabilitiesJson());
copy.setOptions(source.getOptions());
copy.setStatus(source.getStatus());
copy.setCompatibilityStatus(source.getCompatibilityStatus());
copy.setDatabaseProductName(source.getDatabaseProductName());
copy.setDatabaseProductVersion(source.getDatabaseProductVersion());
copy.setDriverName(source.getDriverName());
copy.setDriverVersion(source.getDriverVersion());
copy.setMetadataRefreshStatus(source.getMetadataRefreshStatus());
copy.setMetadataRefreshedAt(source.getMetadataRefreshedAt());
copy.setLastTestStatus(source.getLastTestStatus());
copy.setLastTestMessage(source.getLastTestMessage());
copy.setLastTestedAt(source.getLastTestedAt());
return copy;
}
private void validateDefinition(DatacenterSourceDraftRequest request) {
if (request == null || request.sourceName() == null
|| request.sourceName().isBlank()) {
throw new BusinessException("请输入连接名称");
}
if (!DatacenterSourceType.MYSQL.name().equals(request.sourceType())
&& !DatacenterSourceType.POSTGRESQL.name().equals(request.sourceType())) {
throw new BusinessException("当前批次仅支持 MySQL 和 PostgreSQL");
}
if (request.host() == null || request.host().isBlank()
|| request.databaseName() == null || request.databaseName().isBlank()
|| request.username() == null || request.username().isBlank()) {
throw new BusinessException("请完整填写主机、数据库和用户名");
}
}
private void requireEditableDraft(
DatacenterSource existing,
String requestedSourceType) {
if (existing == null) {
return;
}
if (DatacenterSourceStatus.fromCode(existing.getStatus())
!= DatacenterSourceStatus.DRAFT) {
throw new BusinessException("已激活连接不可直接修改,请使用重配置");
}
if (!java.util.Objects.equals(existing.getSourceType(), requestedSourceType)) {
throw new BusinessException("数据连接类型创建后不可修改");
}
}
private void requireCredential(DatacenterSource source) {
if (source.getCredentialCipher() == null
|| source.getCredentialCipher().isBlank()) {
throw new BusinessException("请输入数据库密码");
}
}
private Map<String, Object> safeProbeDetails(
DatacenterSource source,
RuntimeFingerprint fingerprint) {
Map<String, Object> details = new LinkedHashMap<>();
details.put("databaseProductName", fingerprint.databaseProduct());
details.put("databaseProductVersion", fingerprint.databaseVersion());
details.put("driverName", fingerprint.driverName());
details.put("driverVersion", fingerprint.driverVersion());
details.put("effectiveDriverClassName", source.getDriverClassName());
details.put("effectivePort", source.getPort());
return java.util.Collections.unmodifiableMap(details);
}
private void applyFingerprint(
DatacenterSource source,
RuntimeFingerprint fingerprint) {
source.setDatabaseProductName(fingerprint.databaseProduct());
source.setDatabaseProductVersion(fingerprint.databaseVersion());
source.setDriverName(fingerprint.driverName());
source.setDriverVersion(fingerprint.driverVersion());
}
private String trimToNull(String value) {
return value == null || value.isBlank() ? null : value.trim();
}
private long value(Long sourceValue) {
return sourceValue == null ? 0L : sourceValue;
}
private String bounded(String value) {
if (value == null) {
return null;
}
return value.length() <= 500 ? value : value.substring(0, 500);
}
}

View File

@@ -0,0 +1,259 @@
package tech.easyflow.datacenter.meta.service.impl;
import com.mybatisflex.core.query.QueryWrapper;
import java.math.BigInteger;
import java.util.ArrayList;
import java.util.Date;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.stereotype.Service;
import org.springframework.transaction.PlatformTransactionManager;
import org.springframework.transaction.support.TransactionTemplate;
import tech.easyflow.common.entity.LoginAccount;
import tech.easyflow.common.web.exceptions.BusinessException;
import tech.easyflow.datacenter.connector.DatacenterConnector;
import tech.easyflow.datacenter.connector.DatacenterConnectorRegistry;
import tech.easyflow.datacenter.entity.DatacenterTable;
import tech.easyflow.datacenter.federation.DatacenterFederationChangeNotifier;
import tech.easyflow.datacenter.mapper.DatacenterCatalogMapper;
import tech.easyflow.datacenter.mapper.DatacenterSourceMapper;
import tech.easyflow.datacenter.mapper.DatacenterTableMapper;
import tech.easyflow.datacenter.meta.entity.DatacenterCatalog;
import tech.easyflow.datacenter.meta.entity.DatacenterSource;
import tech.easyflow.datacenter.meta.enums.DatacenterMetadataStatus;
import tech.easyflow.datacenter.meta.enums.DatacenterSourceStatus;
import tech.easyflow.datacenter.meta.model.DatacenterManagedMetadataSnapshot;
import tech.easyflow.datacenter.meta.model.DatacenterTableDetailMeta;
import tech.easyflow.datacenter.meta.service.DatacenterDatasetRegistryService;
/**
* 已纳管元数据刷新协调器,将远程 JDBC 读取与平台数据库短事务明确分离。
*/
@Service
public class DatacenterSourceMetadataRefreshService {
private static final Logger LOGGER =
LoggerFactory.getLogger(DatacenterSourceMetadataRefreshService.class);
private final DatacenterConnectorRegistry connectorRegistry;
private final DatacenterDatasetRegistryService registryService;
private final DatacenterCatalogMapper catalogMapper;
private final DatacenterTableMapper tableMapper;
private final DatacenterSourceMapper sourceMapper;
private final DatacenterFederationChangeNotifier changeNotifier;
private final TransactionTemplate transactionTemplate;
/**
* 创建元数据刷新协调器。
*
* @param connectorRegistry Connector 注册表
* @param registryService 数据集注册服务
* @param catalogMapper 命名空间 Mapper
* @param tableMapper 表 Mapper
* @param sourceMapper 数据源 Mapper
* @param changeNotifier 多节点变更通知器
* @param transactionManager 事务管理器
*/
public DatacenterSourceMetadataRefreshService(
DatacenterConnectorRegistry connectorRegistry,
DatacenterDatasetRegistryService registryService,
DatacenterCatalogMapper catalogMapper,
DatacenterTableMapper tableMapper,
DatacenterSourceMapper sourceMapper,
DatacenterFederationChangeNotifier changeNotifier,
PlatformTransactionManager transactionManager) {
this.connectorRegistry = connectorRegistry;
this.registryService = registryService;
this.catalogMapper = catalogMapper;
this.tableMapper = tableMapper;
this.sourceMapper = sourceMapper;
this.changeNotifier = changeNotifier;
this.transactionTemplate = new TransactionTemplate(transactionManager);
}
/**
* 刷新已纳管对象;外部新表和新字段不会自动扩张查询范围。
*
* @param source 已完成租户校验的数据源 revision 快照
* @param account 当前账号
* @return 刷新后的权威数据源
*/
public DatacenterSource refresh(
DatacenterSource source,
LoginAccount account) {
DatacenterSourceStatus sourceStatus =
DatacenterSourceStatus.fromCode(source.getStatus());
if (sourceStatus != DatacenterSourceStatus.READY
&& sourceStatus != DatacenterSourceStatus.DEGRADED) {
throw new BusinessException("当前连接状态不支持刷新元数据");
}
DatacenterSource refreshed;
try {
// 远程元数据读取不持有平台数据库事务连接。
List<ManagedCatalogRefresh> snapshot = inspect(source);
refreshed = transactionTemplate.execute(status ->
applySnapshot(source, snapshot, account));
} catch (RuntimeException exception) {
markFailed(source, account);
throw exception;
}
if (refreshed == null) {
throw new BusinessException("元数据刷新事务未完成");
}
changeNotifier.publishAfterCommit(refreshed);
return refreshed;
}
private List<ManagedCatalogRefresh> inspect(DatacenterSource source) {
DatacenterConnector connector =
connectorRegistry.getConnector(source.getSourceType());
List<DatacenterCatalog> catalogs = catalogMapper.selectListByQuery(
QueryWrapper.create()
.eq(DatacenterCatalog::getSourceId, source.getId())
.ne(DatacenterCatalog::getMetadataStatus,
DatacenterMetadataStatus.RETIRED.name()));
List<ManagedCatalogRefresh> refreshes = new ArrayList<>(catalogs.size());
for (DatacenterCatalog catalog : catalogs) {
List<DatacenterTable> tables = tableMapper.selectListByQuery(
QueryWrapper.create()
.eq(DatacenterTable::getSourceId, source.getId())
.eq(DatacenterTable::getCatalogId, catalog.getId())
.ne(DatacenterTable::getMetadataStatus,
DatacenterMetadataStatus.RETIRED.name()));
List<String> physicalNames = tables.stream()
.map(this::physicalTableName)
.toList();
DatacenterManagedMetadataSnapshot snapshot =
connector.inspectManagedTables(
source, catalog.getCatalogName(), physicalNames);
Map<String, DatacenterTableDetailMeta> detailsByPhysicalName =
snapshot.details().stream()
.filter(detail -> detail.getTable() != null)
.collect(Collectors.toMap(
detail -> physicalTableName(detail.getTable()),
detail -> detail,
(first, ignored) -> first,
LinkedHashMap::new));
List<ManagedTableRefresh> tableRefreshes =
new ArrayList<>(tables.size());
for (DatacenterTable table : tables) {
String physicalName = physicalTableName(table);
DatacenterTableDetailMeta detail =
detailsByPhysicalName.get(physicalName);
if (detail != null) {
tableRefreshes.add(new ManagedTableRefresh(
table.getId(), detail, false));
} else if (snapshot.missingTableNames().contains(physicalName)) {
tableRefreshes.add(new ManagedTableRefresh(
table.getId(), null, true));
} else {
throw new BusinessException(
"元数据刷新结果不完整: " + physicalName);
}
}
refreshes.add(new ManagedCatalogRefresh(catalog, tableRefreshes));
}
return List.copyOf(refreshes);
}
private DatacenterSource applySnapshot(
DatacenterSource source,
List<ManagedCatalogRefresh> refreshes,
LoginAccount account) {
boolean scopeChanged = false;
for (ManagedCatalogRefresh catalogRefresh : refreshes) {
for (ManagedTableRefresh tableRefresh : catalogRefresh.tables()) {
if (tableRefresh.missing()) {
scopeChanged |= registryService.markManagedTableMissing(
source, tableRefresh.tableId(), account);
} else {
scopeChanged |= registryService.refreshManagedTable(
source,
catalogRefresh.catalog(),
tableRefresh.tableId(),
tableRefresh.detail(),
account);
}
}
}
DatacenterSource patch = new DatacenterSource();
patch.setId(source.getId());
patch.setScopeRevision(scopeChanged
? Math.max(1L, value(source.getScopeRevision())) + 1L
: source.getScopeRevision());
patch.setMetadataRefreshStatus(scopeChanged ? "CHANGED" : "SUCCESS");
patch.setMetadataRefreshedAt(new Date());
patch.setModified(new Date());
patch.setModifiedBy(accountId(account));
QueryWrapper guard = QueryWrapper.create()
.eq(DatacenterSource::getId, source.getId())
.eq(DatacenterSource::getTenantId, source.getTenantId())
.eq(DatacenterSource::getDefinitionRevision,
source.getDefinitionRevision())
.eq(DatacenterSource::getScopeRevision, source.getScopeRevision())
.eq(DatacenterSource::getStatus, source.getStatus());
if (sourceMapper.updateByQuery(patch, guard) != 1) {
throw new BusinessException("连接或数据范围已变化,请刷新后重试");
}
return sourceMapper.selectOneByQuery(QueryWrapper.create()
.eq(DatacenterSource::getId, source.getId())
.eq(DatacenterSource::getTenantId, source.getTenantId()));
}
private void markFailed(DatacenterSource source, LoginAccount account) {
DatacenterSource patch = new DatacenterSource();
patch.setId(source.getId());
patch.setMetadataRefreshStatus("FAILED");
patch.setModified(new Date());
patch.setModifiedBy(accountId(account));
try {
sourceMapper.updateByQuery(patch, QueryWrapper.create()
.eq(DatacenterSource::getId, source.getId())
.eq(DatacenterSource::getTenantId, source.getTenantId())
.eq(DatacenterSource::getDefinitionRevision,
source.getDefinitionRevision())
.eq(DatacenterSource::getScopeRevision,
source.getScopeRevision()));
} catch (RuntimeException updateException) {
LOGGER.warn("记录元数据刷新失败状态时发生错误sourceId={}",
source.getId(), updateException);
}
}
private String physicalTableName(DatacenterTable table) {
String actualTable = trimToNull(table.getActualTable());
return actualTable == null ? table.getTableName() : actualTable;
}
private String trimToNull(String value) {
return value == null || value.isBlank() ? null : value.trim();
}
private long value(Long sourceValue) {
return sourceValue == null ? 0L : sourceValue;
}
private BigInteger accountId(LoginAccount account) {
return account == null || account.getId() == null
? BigInteger.ZERO : account.getId();
}
private record ManagedCatalogRefresh(
DatacenterCatalog catalog,
List<ManagedTableRefresh> tables) {
private ManagedCatalogRefresh {
tables = List.copyOf(tables);
}
}
private record ManagedTableRefresh(
BigInteger tableId,
DatacenterTableDetailMeta detail,
boolean missing) {
}
}

View File

@@ -0,0 +1,317 @@
package tech.easyflow.datacenter.meta.service.impl;
import com.mybatisflex.core.query.QueryWrapper;
import java.math.BigInteger;
import java.util.List;
import org.springframework.stereotype.Service;
import tech.easyflow.common.web.exceptions.BusinessException;
import tech.easyflow.datacenter.connector.DatacenterConnectorRegistry;
import tech.easyflow.datacenter.entity.DatacenterTable;
import tech.easyflow.datacenter.mapper.DatacenterCatalogMapper;
import tech.easyflow.datacenter.meta.entity.DatacenterCatalog;
import tech.easyflow.datacenter.meta.entity.DatacenterSource;
import tech.easyflow.datacenter.meta.enums.DatacenterMetadataStatus;
import tech.easyflow.datacenter.meta.enums.DatacenterSourceStatus;
import tech.easyflow.datacenter.meta.enums.DatacenterSourceType;
import tech.easyflow.datacenter.meta.model.DatacenterCatalogMeta;
import tech.easyflow.datacenter.meta.model.DatacenterMetadataPage;
import tech.easyflow.datacenter.meta.model.DatacenterTableDetailMeta;
import tech.easyflow.datacenter.meta.service.DatacenterDatasetRegistryService;
/**
* 数据源元数据浏览服务,统一处理物理元数据和已纳管元数据的分页边界。
*/
@Service
public class DatacenterSourceMetadataService {
private static final long DEFAULT_METADATA_PAGE_SIZE = 100L;
private static final long MAX_METADATA_PAGE_SIZE = 200L;
private final DatacenterConnectorRegistry connectorRegistry;
private final DatacenterDatasetRegistryService registryService;
private final DatacenterCatalogMapper catalogMapper;
/**
* 创建元数据浏览服务。
*
* @param connectorRegistry 数据源 Connector 注册表
* @param registryService 已纳管数据集注册服务
* @param catalogMapper 命名空间 Mapper
*/
public DatacenterSourceMetadataService(
DatacenterConnectorRegistry connectorRegistry,
DatacenterDatasetRegistryService registryService,
DatacenterCatalogMapper catalogMapper) {
this.connectorRegistry = connectorRegistry;
this.registryService = registryService;
this.catalogMapper = catalogMapper;
}
/**
* 返回已发布命名空间,草稿则浏览物理命名空间。
*
* @param source 已完成租户校验的数据源
* @return 命名空间列表
*/
public List<DatacenterCatalogMeta> listCatalogs(DatacenterSource source) {
DatacenterSourceStatus status = DatacenterSourceStatus.fromCode(source.getStatus());
if (usesManagedMetadata(source)
|| status == DatacenterSourceStatus.READY
|| status == DatacenterSourceStatus.DEGRADED
|| status == DatacenterSourceStatus.DISABLED) {
return managedCatalogs(source.getId()).stream().map(this::toCatalogMeta).toList();
}
requireSuccessfulProbe(source);
return connectorRegistry.getConnector(source.getSourceType()).listCatalogs(source);
}
/**
* 分页返回已发布或物理命名空间。
*
* @param source 已完成租户校验的数据源
* @param keyword 名称关键字
* @param pageNumber 页码
* @param pageSize 页大小
* @return 命名空间分页
*/
public DatacenterMetadataPage<DatacenterCatalogMeta> listCatalogsPage(
DatacenterSource source,
String keyword,
Long pageNumber,
Long pageSize) {
long actualPage = pageNumber(pageNumber);
long actualSize = pageSize(pageSize);
DatacenterSourceStatus status = DatacenterSourceStatus.fromCode(source.getStatus());
if (usesManagedMetadata(source)
|| status == DatacenterSourceStatus.READY
|| status == DatacenterSourceStatus.DEGRADED
|| status == DatacenterSourceStatus.DISABLED) {
List<DatacenterCatalogMeta> managed = managedCatalogs(source.getId()).stream()
.map(this::toCatalogMeta)
.filter(catalog -> matchesKeyword(catalog.getCatalogName(), keyword))
.toList();
return DatacenterMetadataPage.slice(managed, actualPage, actualSize);
}
requireSuccessfulProbe(source);
return connectorRegistry.getConnector(source.getSourceType()).listCatalogsPage(
source, keyword, actualPage, actualSize);
}
/**
* 分页返回命名空间中的表与视图。
*
* @param source 已完成租户校验的数据源
* @param catalogName 命名空间名称
* @param keyword 表名关键字
* @param pageNumber 页码
* @param pageSize 页大小
* @return 表分页
*/
public DatacenterMetadataPage<DatacenterTable> listTables(
DatacenterSource source,
String catalogName,
String keyword,
Long pageNumber,
Long pageSize) {
long actualPage = pageNumber(pageNumber);
long actualSize = pageSize(pageSize);
DatacenterSourceStatus status = DatacenterSourceStatus.fromCode(source.getStatus());
if (usesManagedMetadata(source) || status == DatacenterSourceStatus.DISABLED) {
List<DatacenterTable> managed = registryService
.listManagedTables(
source.getId(), resolveCatalogId(source.getId(), catalogName))
.stream()
.filter(table -> matchesKeyword(table.getTableName(), keyword))
.toList();
return DatacenterMetadataPage.slice(managed, actualPage, actualSize);
}
requireSuccessfulProbe(source);
return connectorRegistry.getConnector(source.getSourceType()).listTablesPage(
source, catalogName, keyword, actualPage, actualSize);
}
/**
* 读取单表字段元数据。
*
* @param source 已完成租户校验的数据源
* @param catalogName 命名空间名称
* @param tableName 表名
* @param register 是否请求兼容的即时纳管
* @param fieldPageNumber 字段页码
* @param fieldPageSize 字段页大小
* @return 表详情
*/
public DatacenterTableDetailMeta getTableDetail(
DatacenterSource source,
String catalogName,
String tableName,
boolean register,
Long fieldPageNumber,
Long fieldPageSize) {
if (usesManagedMetadata(source)) {
BigInteger catalogId = resolveCatalogId(source.getId(), catalogName);
DatacenterTable target = registryService
.listManagedTables(source.getId(), catalogId).stream()
.filter(item -> tableName.equals(item.getTableName()))
.findFirst()
.orElseThrow(() -> new BusinessException("数据集不存在: " + tableName));
DatacenterTableDetailMeta detail = new DatacenterTableDetailMeta();
detail.setTable(registryService.getTableWithFields(target.getId()));
detail.setFields(detail.getTable().getFields());
return pageFields(detail, fieldPageNumber, fieldPageSize);
}
requireSuccessfulProbe(source);
DatacenterTableDetailMeta detail = connectorRegistry
.getConnector(source.getSourceType())
.getTableDetail(source, catalogName, tableName);
if (register) {
throw new BusinessException("请使用批量接入以原子更新数据范围");
}
return pageFields(detail, fieldPageNumber, fieldPageSize);
}
/**
* 浏览未发布候选连接的命名空间。
*
* @param candidate 候选数据源
* @return 物理命名空间列表
*/
public List<DatacenterCatalogMeta> listCandidateCatalogs(
DatacenterSource candidate) {
return connectorRegistry.getConnector(candidate.getSourceType())
.listCatalogs(candidate);
}
/**
* 分页浏览未发布候选连接的命名空间。
*
* @param candidate 候选数据源
* @param keyword 名称关键字
* @param pageNumber 页码
* @param pageSize 页大小
* @return 命名空间分页
*/
public DatacenterMetadataPage<DatacenterCatalogMeta> listCandidateCatalogsPage(
DatacenterSource candidate,
String keyword,
Long pageNumber,
Long pageSize) {
return connectorRegistry.getConnector(candidate.getSourceType()).listCatalogsPage(
candidate, keyword, pageNumber(pageNumber), pageSize(pageSize));
}
/**
* 分页浏览未发布候选连接的表。
*
* @param candidate 候选数据源
* @param catalogName 命名空间名称
* @param keyword 表名关键字
* @param pageNumber 页码
* @param pageSize 页大小
* @return 表分页
*/
public DatacenterMetadataPage<DatacenterTable> listCandidateTables(
DatacenterSource candidate,
String catalogName,
String keyword,
Long pageNumber,
Long pageSize) {
return connectorRegistry.getConnector(candidate.getSourceType()).listTablesPage(
candidate,
trimToNull(catalogName),
keyword,
pageNumber(pageNumber),
pageSize(pageSize));
}
private List<DatacenterCatalog> managedCatalogs(BigInteger sourceId) {
return catalogMapper.selectListByQuery(QueryWrapper.create()
.eq(DatacenterCatalog::getSourceId, sourceId)
.ne(DatacenterCatalog::getMetadataStatus,
DatacenterMetadataStatus.RETIRED.name()));
}
private DatacenterTableDetailMeta pageFields(
DatacenterTableDetailMeta detail,
Long fieldPageNumber,
Long fieldPageSize) {
List<tech.easyflow.datacenter.entity.DatacenterTableField> fields =
detail.getFields() == null ? List.of() : detail.getFields();
DatacenterMetadataPage<tech.easyflow.datacenter.entity.DatacenterTableField> page =
DatacenterMetadataPage.slice(
fields, pageNumber(fieldPageNumber), pageSize(fieldPageSize));
detail.setFields(page.records());
detail.setFieldPageNumber(page.pageNumber());
detail.setFieldPageSize(page.pageSize());
detail.setHasMoreFields(page.hasMore());
if (detail.getTable() != null) {
detail.getTable().setFields(page.records());
}
return detail;
}
private BigInteger resolveCatalogId(BigInteger sourceId, String catalogName) {
if (catalogName == null || catalogName.isBlank()) {
return null;
}
DatacenterCatalog catalog = catalogMapper.selectOneByQuery(QueryWrapper.create()
.eq(DatacenterCatalog::getSourceId, sourceId)
.eq(DatacenterCatalog::getCatalogName, catalogName)
.ne(DatacenterCatalog::getMetadataStatus,
DatacenterMetadataStatus.RETIRED.name()));
if (catalog == null) {
throw new BusinessException("命名空间不存在: " + catalogName);
}
return catalog.getId();
}
private DatacenterCatalogMeta toCatalogMeta(DatacenterCatalog catalog) {
DatacenterCatalogMeta meta = new DatacenterCatalogMeta();
meta.setId(catalog.getId());
meta.setSourceId(catalog.getSourceId());
meta.setCatalogName(catalog.getCatalogName());
meta.setCatalogDesc(catalog.getCatalogDesc());
meta.setCatalogType(catalog.getCatalogType());
meta.setLogicalSchemaName(catalog.getLogicalSchemaName());
meta.setPhysicalCatalogName(catalog.getPhysicalCatalogName());
meta.setPhysicalSchemaName(catalog.getPhysicalSchemaName());
return meta;
}
private void requireSuccessfulProbe(DatacenterSource source) {
if (!"SUCCESS".equalsIgnoreCase(source.getLastTestStatus())) {
throw new BusinessException("请先测试连接");
}
}
private boolean usesManagedMetadata(DatacenterSource source) {
return DatacenterSourceType.EXCEL.name().equals(source.getSourceType())
|| DatacenterSourceType.EXCEL_MATERIALIZED.name().equals(
source.getSourceType());
}
private boolean matchesKeyword(String value, String keyword) {
String normalized = trimToNull(keyword);
return normalized == null || (value != null
&& value.toLowerCase(java.util.Locale.ROOT)
.contains(normalized.toLowerCase(java.util.Locale.ROOT)));
}
private long pageNumber(Long requested) {
return requested == null || requested < 1L
? 1L : Math.min(requested, 1_000_000L);
}
private long pageSize(Long requested) {
return requested == null || requested < 1L
? DEFAULT_METADATA_PAGE_SIZE
: Math.min(requested, MAX_METADATA_PAGE_SIZE);
}
private String trimToNull(String value) {
if (value == null || value.isBlank()) {
return null;
}
return value.trim();
}
}

View File

@@ -1,6 +1,7 @@
package tech.easyflow.datacenter.meta.support;
import cn.hutool.core.util.StrUtil;
import com.easyagents.federation.sql.source.KnownJdbcDriver;
import org.springframework.stereotype.Component;
import tech.easyflow.datacenter.meta.entity.DatacenterSource;
import tech.easyflow.datacenter.meta.enums.DatacenterSourceType;
@@ -49,11 +50,32 @@ public class DatacenterSourceConnectionDefaults {
if (configJson == null || configJson.isEmpty()) {
return sanitized;
}
sanitized.putAll(configJson);
sanitized.remove("password");
for (Map.Entry<String, Object> entry : configJson.entrySet()) {
String key = entry.getKey();
if (key == null || isSensitiveKey(key)) {
continue;
}
sanitized.put(key, entry.getValue());
}
return sanitized;
}
/**
* 判断扩展字段名是否可能承载凭据。
*
* @param key 配置键
* @return 是否属于敏感键
*/
public boolean isSensitiveKey(String key) {
String normalized = StrUtil.trimToEmpty(key).toLowerCase(java.util.Locale.ROOT);
return normalized.contains("password")
|| normalized.contains("passwd")
|| normalized.contains("secret")
|| normalized.contains("token")
|| normalized.contains("credential")
|| normalized.endsWith("key");
}
public Integer defaultPort(String sourceType) {
DatacenterSourceType resolved = requireSourceType(sourceType);
return switch (resolved) {
@@ -69,11 +91,12 @@ public class DatacenterSourceConnectionDefaults {
public String defaultDriverClassName(String sourceType) {
DatacenterSourceType resolved = requireSourceType(sourceType);
return switch (resolved) {
case MYSQL -> "com.mysql.cj.jdbc.Driver";
case POSTGRESQL, GAUSSDB_NATIVE -> "org.postgresql.Driver";
case ORACLE -> "oracle.jdbc.OracleDriver";
case GBASE_8A -> "com.gbase.jdbc.Driver";
case GBASE_8S -> "com.gbasedbt.jdbc.Driver";
case MYSQL -> KnownJdbcDriver.MYSQL.driverClassName();
case POSTGRESQL -> KnownJdbcDriver.POSTGRESQL.driverClassName();
case GAUSSDB_NATIVE -> KnownJdbcDriver.GAUSSDB.driverClassName();
case ORACLE -> KnownJdbcDriver.ORACLE.driverClassName();
case GBASE_8A -> KnownJdbcDriver.GBASE_8A.driverClassName();
case GBASE_8S -> KnownJdbcDriver.GBASE_8S.driverClassName();
default -> "";
};
}
@@ -91,10 +114,12 @@ public class DatacenterSourceConnectionDefaults {
DatacenterSourceType resolved = requireSourceType(source.getSourceType());
return switch (resolved) {
case MYSQL -> String.format(
"jdbc:mysql://%s:%d/%s?useUnicode=true&characterEncoding=utf-8&serverTimezone=Asia/Shanghai&useSSL=false",
"jdbc:mysql://%s:%d/%s?useUnicode=true&characterEncoding=utf-8&serverTimezone=Asia/Shanghai&useSSL=false&useCursorFetch=true&useServerPrepStmts=true&connectTimeout=5000&socketTimeout=35000",
host, port, databaseName
);
case POSTGRESQL -> String.format("jdbc:postgresql://%s:%d/%s", host, port, databaseName);
case POSTGRESQL -> String.format(
"jdbc:postgresql://%s:%d/%s?connectTimeout=5&socketTimeout=35&cancelSignalTimeout=5",
host, port, databaseName);
case ORACLE -> String.format("jdbc:oracle:thin:@//%s:%d/%s", host, port, databaseName);
case GAUSSDB_NATIVE -> String.format("jdbc:postgresql://%s:%d/%s", host, port, databaseName);
case GBASE_8A -> String.format("jdbc:gbase://%s:%d/%s", host, port, databaseName);

View File

@@ -1,28 +1,109 @@
package tech.easyflow.datacenter.security;
import cn.hutool.crypto.SecureUtil;
import cn.hutool.crypto.symmetric.AES;
import java.nio.ByteBuffer;
import java.nio.charset.StandardCharsets;
import java.security.GeneralSecurityException;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.security.SecureRandom;
import java.util.Base64;
import javax.crypto.Cipher;
import javax.crypto.spec.GCMParameterSpec;
import javax.crypto.spec.SecretKeySpec;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
import java.nio.charset.StandardCharsets;
/**
* 使用部署侧主密钥对数据源密码执行带认证加密。
*/
@Component
public class DatacenterCredentialCipher {
private static final String DEFAULT_KEY = "easyflow-datacenter-phase1-key";
private final AES aes = SecureUtil.aes(SecureUtil.sha256(DEFAULT_KEY).substring(0, 16).getBytes(StandardCharsets.UTF_8));
private static final String VERSION = "v1:";
private static final String KEY_DERIVATION_DOMAIN = "easyflow-datacenter-credential-v1\u0000";
private static final int IV_BYTES = 12;
private static final int GCM_TAG_BITS = 128;
private final SecretKeySpec key;
private final SecureRandom secureRandom = new SecureRandom();
/**
* 创建凭据加密器。
*
* @param masterKey 部署环境提供的独立主密钥,至少 32 个字符
* @throws IllegalStateException 主密钥缺失或长度不足时抛出
*/
public DatacenterCredentialCipher(
@Value("${easyflow.datacenter.credential-key:}") String masterKey) {
if (masterKey == null || masterKey.length() < 32) {
throw new IllegalStateException(
"easyflow.datacenter.credential-key must contain at least 32 characters");
}
// 使用固定域标签派生独立子密钥,避免与部署主密钥的其他用途共享同一 AES key。
this.key = new SecretKeySpec(sha256(KEY_DERIVATION_DOMAIN + masterKey), "AES");
}
/**
* 加密明文密码。
*
* @param plainText 明文密码
* @return 带版本和随机 IV 的密文;空输入返回 null
*/
public String encrypt(String plainText) {
if (plainText == null || plainText.isBlank()) {
return null;
}
return aes.encryptHex(plainText);
byte[] iv = new byte[IV_BYTES];
secureRandom.nextBytes(iv);
try {
Cipher cipher = Cipher.getInstance("AES/GCM/NoPadding");
cipher.init(Cipher.ENCRYPT_MODE, key, new GCMParameterSpec(GCM_TAG_BITS, iv));
byte[] encrypted = cipher.doFinal(plainText.getBytes(StandardCharsets.UTF_8));
return VERSION + Base64.getEncoder().encodeToString(
ByteBuffer.allocate(iv.length + encrypted.length)
.put(iv)
.put(encrypted)
.array());
} catch (GeneralSecurityException exception) {
throw new IllegalStateException("failed to encrypt datacenter credential", exception);
}
}
/**
* 解密持久化密码。
*
* @param cipherText 版本化密文
* @return 明文密码;空输入返回 null
* @throws IllegalStateException 密文格式或认证校验失败时抛出
*/
public String decrypt(String cipherText) {
if (cipherText == null || cipherText.isBlank()) {
return null;
}
return aes.decryptStr(cipherText);
if (!cipherText.startsWith(VERSION)) {
throw new IllegalStateException("unsupported datacenter credential format");
}
try {
byte[] payload = Base64.getDecoder().decode(cipherText.substring(VERSION.length()));
if (payload.length <= IV_BYTES) {
throw new IllegalStateException("invalid datacenter credential payload");
}
byte[] iv = java.util.Arrays.copyOfRange(payload, 0, IV_BYTES);
byte[] encrypted = java.util.Arrays.copyOfRange(payload, IV_BYTES, payload.length);
Cipher cipher = Cipher.getInstance("AES/GCM/NoPadding");
cipher.init(Cipher.DECRYPT_MODE, key, new GCMParameterSpec(GCM_TAG_BITS, iv));
return new String(cipher.doFinal(encrypted), StandardCharsets.UTF_8);
} catch (GeneralSecurityException | IllegalArgumentException exception) {
throw new IllegalStateException("failed to decrypt datacenter credential", exception);
}
}
private byte[] sha256(String value) {
try {
return MessageDigest.getInstance("SHA-256")
.digest(value.getBytes(StandardCharsets.UTF_8));
} catch (NoSuchAlgorithmException exception) {
throw new IllegalStateException("SHA-256 is not available", exception);
}
}
}

View File

@@ -0,0 +1,79 @@
package tech.easyflow.datacenter.audit;
import java.math.BigInteger;
import java.util.Map;
import org.junit.Assert;
import org.junit.Test;
import org.mockito.ArgumentCaptor;
import org.mockito.Mockito;
import tech.easyflow.common.entity.LoginAccount;
import tech.easyflow.datacenter.mapper.DatacenterQueryAuditMapper;
import tech.easyflow.datacenter.meta.entity.DatacenterSource;
/**
* {@link DatacenterQueryAuditService} 审计脱敏回归测试。
*/
public class DatacenterQueryAuditServiceTest {
/**
* 验证 PostgreSQL dollar quote、字符串、数字和注释内容均不会进入审计 SQL。
*/
@Test
public void startShouldRedactLiteralsAndComments() {
DatacenterQueryAuditMapper mapper = Mockito.mock(
DatacenterQueryAuditMapper.class);
DatacenterQueryAuditService service = new DatacenterQueryAuditService(mapper);
DatacenterSource source = new DatacenterSource();
source.setId(BigInteger.ONE);
source.setTenantId(BigInteger.ONE);
service.start(
"query-1",
source,
"SELECT $$secret$$, $tag$hidden$tag$, 'password', 42 "
+ "/* api-key /* nested */ value */ -- bearer-token\nFROM orders",
Map.of(),
null,
"CONSOLE",
null);
ArgumentCaptor<DatacenterQueryAudit> captor = ArgumentCaptor.forClass(
DatacenterQueryAudit.class);
Mockito.verify(mapper).insert(captor.capture());
String sql = captor.getValue().getParameterizedSql();
Assert.assertFalse(sql.contains("secret"));
Assert.assertFalse(sql.contains("hidden"));
Assert.assertFalse(sql.contains("password"));
Assert.assertFalse(sql.contains("api-key"));
Assert.assertFalse(sql.contains("bearer-token"));
Assert.assertFalse(sql.contains("42"));
Assert.assertTrue(sql.contains("FROM orders"));
}
/**
* 验证执行账号与业务调用对象分别落入独立审计字段。
*/
@Test
public void startShouldPersistExecutorSeparatelyFromCaller() {
DatacenterQueryAuditMapper mapper = Mockito.mock(
DatacenterQueryAuditMapper.class);
DatacenterQueryAuditService service = new DatacenterQueryAuditService(mapper);
DatacenterSource source = new DatacenterSource();
source.setId(BigInteger.ONE);
source.setTenantId(BigInteger.ONE);
LoginAccount account = new LoginAccount();
account.setId(BigInteger.valueOf(9001L));
account.setTenantId(BigInteger.ONE);
service.start(
"query-2", source, "SELECT id FROM orders", Map.of(),
account, "DATASET", "table-2001");
ArgumentCaptor<DatacenterQueryAudit> captor = ArgumentCaptor.forClass(
DatacenterQueryAudit.class);
Mockito.verify(mapper).insert(captor.capture());
Assert.assertEquals(BigInteger.valueOf(9001L),
captor.getValue().getExecutorAccountId());
Assert.assertEquals("table-2001", captor.getValue().getCallerId());
}
}

View File

@@ -8,8 +8,12 @@ import org.junit.Assert;
import org.junit.Test;
import org.mockito.MockedStatic;
import org.mockito.Mockito;
import tech.easyflow.common.web.exceptions.BusinessException;
import tech.easyflow.datacenter.entity.DatacenterTable;
import tech.easyflow.datacenter.entity.DatacenterTableField;
import tech.easyflow.datacenter.execution.model.DatacenterQueryRequest;
import tech.easyflow.datacenter.execution.model.DatacenterQueryFilter;
import tech.easyflow.datacenter.execution.model.DatacenterQuerySort;
import tech.easyflow.datacenter.meta.enums.DatacenterSourceType;
import java.math.BigInteger;
@@ -47,9 +51,23 @@ public class AbstractInternalTableConnectorTest {
public void shouldUseIndependentQueryWrapperForPaginationAfterCount() {
DatacenterTable table = new DatacenterTable();
table.setActualTable("preview_table");
DatacenterTableField field = new DatacenterTableField();
field.setFieldName("model_id");
field.setSourceColumnName("MODEL_ID");
table.setFields(List.of(field));
DatacenterQueryRequest request = new DatacenterQueryRequest();
request.setPageNumber(1L);
request.setPageSize(10L);
request.setSelectedColumns(List.of("model_id"));
DatacenterQueryFilter filter = new DatacenterQueryFilter();
filter.setColumn("model_id");
filter.setOperator("EQ");
filter.setValue("deepseek-v4-pro");
request.setFilters(List.of(filter));
DatacenterQuerySort sort = new DatacenterQuerySort();
sort.setColumn("model_id");
sort.setDirection("DESC");
request.setSorts(List.of(sort));
Row record = new Row();
record.put("MODEL_ID", "deepseek-v4-pro");
@@ -79,10 +97,22 @@ public class AbstractInternalTableConnectorTest {
null, table, request);
Assert.assertNotSame(countWrapper.get(), pageWrapper.get());
Assert.assertTrue(countWrapper.get().toSQL().contains("`MODEL_ID` ="));
Assert.assertFalse(countWrapper.get().toSQL().contains("DESC"));
Assert.assertTrue(pageWrapper.get().toSQL().contains("`MODEL_ID` DESC"));
Assert.assertEquals("deepseek-v4-pro", page.getRecords().get(0).get("model_id"));
}
}
/**
* 验证结构化分页入口拒绝调用方拼接原始 WHERE 片段。
*/
@Test(expected = BusinessException.class)
public void shouldRejectRawWhereClause() {
AbstractInternalTableConnector.createQueryWrapper(
"1 = 1 UNION SELECT secret FROM another_table");
}
/**
* 供内部动态表查询测试使用的最小连接器实现。
*/

View File

@@ -0,0 +1,270 @@
package tech.easyflow.datacenter.connector.support;
import java.math.BigInteger;
import java.sql.Connection;
import java.sql.DatabaseMetaData;
import java.sql.ResultSet;
import java.sql.Types;
import java.util.List;
import java.util.Set;
import org.junit.Assert;
import org.junit.Test;
import org.mockito.ArgumentMatchers;
import org.mockito.Mockito;
import tech.easyflow.datacenter.connector.SqlDialect;
import tech.easyflow.datacenter.meta.entity.DatacenterSource;
import tech.easyflow.datacenter.meta.enums.DatacenterSourceType;
import tech.easyflow.datacenter.meta.model.DatacenterManagedMetadataSnapshot;
import tech.easyflow.datacenter.meta.model.DatacenterTableDetailMeta;
/**
* {@link AbstractJdbcConnector} JDBC 元数据精确匹配回归测试。
*/
public class AbstractJdbcConnectorMetadataTest {
/**
* 验证包含下划线的表名会转义元数据 Pattern并过滤驱动返回的近似匹配项。
*
* @throws Exception 模拟 JDBC 调用失败
*/
@Test
public void getTableDetailShouldEscapePatternsAndFilterMetadataRows()
throws Exception {
Connection connection = Mockito.mock(Connection.class);
DatabaseMetaData metadata = Mockito.mock(DatabaseMetaData.class);
ResultSet tables = Mockito.mock(ResultSet.class);
ResultSet primaryKeys = Mockito.mock(ResultSet.class);
ResultSet columns = Mockito.mock(ResultSet.class);
Mockito.when(connection.getMetaData()).thenReturn(metadata);
Mockito.when(metadata.getSearchStringEscape()).thenReturn("\\");
Mockito.when(metadata.getTables(
Mockito.eq("analytics"), Mockito.isNull(),
Mockito.eq("order\\_item"), ArgumentMatchers.any(String[].class)))
.thenReturn(tables);
Mockito.when(metadata.getPrimaryKeys(
"analytics", null, "order_item"))
.thenReturn(primaryKeys);
Mockito.when(metadata.getColumns(
"analytics", null, "order\\_item", "%"))
.thenReturn(columns);
Mockito.when(tables.next()).thenReturn(true, true, false);
Mockito.when(tables.getString("TABLE_NAME"))
.thenReturn("orderXitem", "order_item");
Mockito.when(tables.getString("TABLE_CAT")).thenReturn("analytics");
Mockito.when(tables.getString("TABLE_SCHEM")).thenReturn(null);
Mockito.when(tables.getString("REMARKS")).thenReturn("订单");
Mockito.when(tables.getString("TABLE_TYPE")).thenReturn("TABLE");
Mockito.when(primaryKeys.next()).thenReturn(false);
Mockito.when(columns.next()).thenReturn(true, true, false);
Mockito.when(columns.getString("TABLE_NAME"))
.thenReturn("orderXitem", "order_item");
Mockito.when(columns.getString("TABLE_CAT")).thenReturn("analytics");
Mockito.when(columns.getString("TABLE_SCHEM")).thenReturn(null);
Mockito.when(columns.getString("COLUMN_NAME")).thenReturn("id");
Mockito.when(columns.getString("REMARKS")).thenReturn("主键");
Mockito.when(columns.getString("TYPE_NAME")).thenReturn("BIGINT");
Mockito.when(columns.getInt("DATA_TYPE")).thenReturn(Types.BIGINT);
Mockito.when(columns.getInt("ORDINAL_POSITION")).thenReturn(1);
Mockito.when(columns.getInt("COLUMN_SIZE")).thenReturn(19);
Mockito.when(columns.getInt("DECIMAL_DIGITS")).thenReturn(0);
Mockito.when(columns.getInt("NULLABLE"))
.thenReturn(DatabaseMetaData.columnNoNulls);
DatacenterSource source = new DatacenterSource();
source.setId(BigInteger.ONE);
source.setDatabaseName("analytics");
DatacenterTableDetailMeta detail = new TestJdbcConnector(connection)
.getTableDetail(source, "analytics", "order_item");
Assert.assertEquals("订单", detail.getTable().getTableDesc());
Assert.assertEquals(1, detail.getFields().size());
Assert.assertEquals("id", detail.getFields().get(0).getFieldName());
Mockito.verify(metadata).getTables(
Mockito.eq("analytics"), Mockito.isNull(),
Mockito.eq("order\\_item"), ArgumentMatchers.any(String[].class));
Mockito.verify(metadata).getColumns(
"analytics", null, "order\\_item", "%");
}
/**
* 刷新多个已纳管表时复用一个连接,并显式返回缺失对象。
*
* @throws Exception 模拟 JDBC 调用失败
*/
@Test
public void inspectManagedTablesShouldReuseConnectionAndReportMissing()
throws Exception {
Connection connection = Mockito.mock(Connection.class);
DatabaseMetaData metadata = Mockito.mock(DatabaseMetaData.class);
ResultSet orders = Mockito.mock(ResultSet.class);
ResultSet missing = Mockito.mock(ResultSet.class);
ResultSet primaryKeys = Mockito.mock(ResultSet.class);
ResultSet columns = Mockito.mock(ResultSet.class);
Mockito.when(connection.getMetaData()).thenReturn(metadata);
Mockito.when(metadata.getSearchStringEscape()).thenReturn("\\");
Mockito.when(metadata.getTables(
Mockito.eq("analytics"), Mockito.isNull(), Mockito.eq("orders"),
ArgumentMatchers.any(String[].class)))
.thenReturn(orders);
Mockito.when(metadata.getTables(
Mockito.eq("analytics"), Mockito.isNull(), Mockito.eq("deleted\\_table"),
ArgumentMatchers.any(String[].class)))
.thenReturn(missing);
Mockito.when(metadata.getPrimaryKeys("analytics", null, "orders"))
.thenReturn(primaryKeys);
Mockito.when(metadata.getColumns("analytics", null, "orders", "%"))
.thenReturn(columns);
Mockito.when(orders.next()).thenReturn(true, false);
Mockito.when(orders.getString("TABLE_NAME")).thenReturn("orders");
Mockito.when(orders.getString("TABLE_CAT")).thenReturn("analytics");
Mockito.when(orders.getString("TABLE_SCHEM")).thenReturn(null);
Mockito.when(orders.getString("TABLE_TYPE")).thenReturn("TABLE");
Mockito.when(missing.next()).thenReturn(false);
Mockito.when(primaryKeys.next()).thenReturn(false);
Mockito.when(columns.next()).thenReturn(true, false);
Mockito.when(columns.getString("TABLE_NAME")).thenReturn("orders");
Mockito.when(columns.getString("TABLE_CAT")).thenReturn("analytics");
Mockito.when(columns.getString("TABLE_SCHEM")).thenReturn(null);
Mockito.when(columns.getString("COLUMN_NAME")).thenReturn("id");
Mockito.when(columns.getString("TYPE_NAME")).thenReturn("BIGINT");
Mockito.when(columns.getInt("DATA_TYPE")).thenReturn(Types.BIGINT);
Mockito.when(columns.getInt("ORDINAL_POSITION")).thenReturn(1);
Mockito.when(columns.getInt("COLUMN_SIZE")).thenReturn(19);
Mockito.when(columns.getInt("DECIMAL_DIGITS")).thenReturn(0);
Mockito.when(columns.getInt("NULLABLE"))
.thenReturn(DatabaseMetaData.columnNoNulls);
DatacenterSource source = new DatacenterSource();
source.setId(BigInteger.ONE);
source.setDatabaseName("analytics");
TestJdbcConnector connector = new TestJdbcConnector(connection);
DatacenterManagedMetadataSnapshot snapshot = connector.inspectManagedTables(
source, "analytics", List.of("orders", "deleted_table"));
Assert.assertEquals(1, connector.connectionCalls());
Assert.assertEquals(1, snapshot.details().size());
Assert.assertEquals("orders", snapshot.details().get(0).getTable().getTableName());
Assert.assertEquals(Set.of("deleted_table"), snapshot.missingTableNames());
}
/**
* 验证 Schema 型数据库不依赖驱动可选的 TABLE_CAT 回填。
*
* @throws Exception 模拟 JDBC 调用失败
*/
@Test
public void getTableDetailShouldAcceptMissingCatalogForPostgresql()
throws Exception {
Connection connection = Mockito.mock(Connection.class);
DatabaseMetaData metadata = Mockito.mock(DatabaseMetaData.class);
ResultSet tables = Mockito.mock(ResultSet.class);
ResultSet primaryKeys = Mockito.mock(ResultSet.class);
ResultSet columns = Mockito.mock(ResultSet.class);
Mockito.when(connection.getMetaData()).thenReturn(metadata);
Mockito.when(metadata.getSearchStringEscape()).thenReturn("\\");
Mockito.when(metadata.getTables(
Mockito.eq("harmony_adapter"), Mockito.eq("public"),
Mockito.eq("flyway\\_schema\\_history"),
ArgumentMatchers.any(String[].class)))
.thenReturn(tables);
Mockito.when(metadata.getPrimaryKeys(
"harmony_adapter", "public", "flyway_schema_history"))
.thenReturn(primaryKeys);
Mockito.when(metadata.getColumns(
"harmony_adapter", "public",
"flyway\\_schema\\_history", "%"))
.thenReturn(columns);
Mockito.when(tables.next()).thenReturn(true, false);
Mockito.when(tables.getString("TABLE_NAME"))
.thenReturn("flyway_schema_history");
Mockito.when(tables.getString("TABLE_CAT")).thenReturn(null);
Mockito.when(tables.getString("TABLE_SCHEM")).thenReturn("public");
Mockito.when(tables.getString("TABLE_TYPE")).thenReturn("TABLE");
Mockito.when(primaryKeys.next()).thenReturn(false);
Mockito.when(columns.next()).thenReturn(true, false);
Mockito.when(columns.getString("TABLE_NAME"))
.thenReturn("flyway_schema_history");
Mockito.when(columns.getString("TABLE_CAT")).thenReturn(null);
Mockito.when(columns.getString("TABLE_SCHEM")).thenReturn("public");
Mockito.when(columns.getString("COLUMN_NAME")).thenReturn("installed_rank");
Mockito.when(columns.getString("TYPE_NAME")).thenReturn("INTEGER");
Mockito.when(columns.getInt("DATA_TYPE")).thenReturn(Types.INTEGER);
Mockito.when(columns.getInt("ORDINAL_POSITION")).thenReturn(1);
Mockito.when(columns.getInt("COLUMN_SIZE")).thenReturn(10);
Mockito.when(columns.getInt("DECIMAL_DIGITS")).thenReturn(0);
Mockito.when(columns.getInt("NULLABLE"))
.thenReturn(DatabaseMetaData.columnNoNulls);
DatacenterSource source = new DatacenterSource();
source.setId(BigInteger.ONE);
source.setDatabaseName("harmony_adapter");
source.setSchemaName("public");
DatacenterTableDetailMeta detail = new TestJdbcConnector(
connection, DatacenterSourceType.POSTGRESQL)
.getTableDetail(source, "public", "flyway_schema_history");
Assert.assertEquals("flyway_schema_history", detail.getTable().getTableName());
Assert.assertEquals("installed_rank", detail.getFields().get(0).getFieldName());
}
/**
* 使用固定模拟连接的测试连接器。
*/
private static final class TestJdbcConnector extends AbstractJdbcConnector {
private final Connection connection;
private int connectionCalls;
/**
* 创建测试连接器。
*
* @param connection 模拟 JDBC 连接
*/
private TestJdbcConnector(Connection connection) {
this(connection, DatacenterSourceType.MYSQL);
}
/**
* 创建指定数据库类型的测试连接器。
*
* @param connection 模拟 JDBC 连接
* @param sourceType 数据源类型
*/
private TestJdbcConnector(
Connection connection,
DatacenterSourceType sourceType) {
super(sourceType, Mockito.mock(SqlDialect.class), Set.of());
this.connection = connection;
}
/**
* 将元数据操作交给固定测试连接。
*
* @param source 数据源定义
* @param cacheable 是否允许连接缓存
* @param callback JDBC 回调
* @param <T> 返回值类型
* @return 回调结果
* @throws Exception 回调失败
*/
@Override
protected <T> T withConnection(
DatacenterSource source,
boolean cacheable,
JdbcCallback<T> callback) throws Exception {
connectionCalls++;
return callback.apply(connection);
}
/**
* 返回元数据操作请求连接的次数。
*
* @return 调用次数
*/
private int connectionCalls() {
return connectionCalls;
}
}
}

View File

@@ -1,33 +1,105 @@
package tech.easyflow.datacenter.execution.service.impl;
import com.mybatisflex.core.paginate.Page;
import com.mybatisflex.core.row.Row;
import org.junit.Assert;
import org.junit.Test;
import org.mockito.ArgumentCaptor;
import org.mockito.ArgumentMatchers;
import org.mockito.Mockito;
import tech.easyflow.datacenter.audit.DatacenterQueryAuditService;
import tech.easyflow.datacenter.connector.DatacenterConnector;
import tech.easyflow.datacenter.connector.DatacenterConnectorRegistry;
import tech.easyflow.datacenter.connector.support.AbstractInternalTableConnector;
import tech.easyflow.datacenter.entity.DatacenterTable;
import tech.easyflow.datacenter.entity.DatacenterTableField;
import tech.easyflow.datacenter.execution.model.DatacenterSqlQueryRequest;
import tech.easyflow.datacenter.execution.model.DatacenterSqlColumnView;
import tech.easyflow.datacenter.execution.model.DatacenterSqlConsoleResult;
import tech.easyflow.datacenter.execution.model.DatacenterQueryFilter;
import tech.easyflow.datacenter.execution.model.DatacenterQueryRequest;
import tech.easyflow.datacenter.execution.model.DatacenterQuerySort;
import tech.easyflow.datacenter.execution.model.DatacenterSchemaResponse;
import tech.easyflow.datacenter.execution.model.DatasetRef;
import tech.easyflow.datacenter.federation.DatacenterFederationQueryService;
import tech.easyflow.datacenter.mapper.DatacenterCatalogMapper;
import tech.easyflow.datacenter.mapper.DatacenterDatasetVersionMapper;
import tech.easyflow.datacenter.mapper.DatacenterDerivedTableMapper;
import tech.easyflow.datacenter.meta.entity.DatacenterCatalog;
import tech.easyflow.datacenter.meta.entity.DatacenterSource;
import tech.easyflow.datacenter.meta.enums.DatacenterMetadataStatus;
import tech.easyflow.datacenter.meta.enums.DatacenterSensitivityLevel;
import tech.easyflow.datacenter.meta.service.DatacenterDatasetRegistryService;
import java.lang.reflect.Field;
import java.math.BigInteger;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.function.Consumer;
import tech.easyflow.common.entity.LoginAccount;
import tech.easyflow.common.web.exceptions.BusinessException;
/**
* {@link DatacenterDatasetQueryServiceImpl} 分页 SQL 读取回归测试。
*/
public class DatacenterDatasetQueryServiceImplTest {
/**
* 验证结构接口只返回请求的字段页,且不在表对象内重复序列化完整字段。
*
* @throws Exception 测试依赖注入失败时抛出
*/
@Test
public void getSchemaShouldReturnBoundedFieldPage() throws Exception {
BigInteger sourceId = BigInteger.valueOf(6101L);
BigInteger catalogId = BigInteger.valueOf(6102L);
BigInteger tableId = BigInteger.valueOf(6103L);
DatacenterSource source = new DatacenterSource();
source.setId(sourceId);
DatacenterCatalog catalog = new DatacenterCatalog();
catalog.setId(catalogId);
DatacenterTable table = new DatacenterTable();
table.setId(tableId);
table.setSourceId(sourceId);
table.setCatalogId(catalogId);
table.setFields(List.of(
queryableField("first"),
queryableField("second"),
queryableField("third")));
DatasetRef datasetRef = new DatasetRef();
datasetRef.setTableId(tableId);
DatacenterDatasetRegistryService registry = Mockito.mock(
DatacenterDatasetRegistryService.class);
Mockito.when(registry.getTableWithFields(tableId)).thenReturn(table);
Mockito.when(registry.resolveDatasetRef(tableId)).thenReturn(datasetRef);
Mockito.when(registry.getSourceRequired(sourceId)).thenReturn(source);
Mockito.when(registry.getCatalogById(catalogId)).thenReturn(catalog);
DatacenterDatasetVersionMapper versionMapper = Mockito.mock(
DatacenterDatasetVersionMapper.class);
DatacenterDerivedTableMapper derivedMapper = Mockito.mock(
DatacenterDerivedTableMapper.class);
Mockito.when(versionMapper.selectListByQuery(ArgumentMatchers.any()))
.thenReturn(List.of());
Mockito.when(derivedMapper.selectListByQuery(ArgumentMatchers.any()))
.thenReturn(List.of());
DatacenterDatasetQueryServiceImpl service =
new DatacenterDatasetQueryServiceImpl();
setField(service, "registryService", registry);
setField(service, "datasetVersionMapper", versionMapper);
setField(service, "derivedTableMapper", derivedMapper);
DatacenterSchemaResponse response = service.getSchema(
datasetRef, 2L, 2L);
Assert.assertEquals(2L, response.getFieldPageNumber());
Assert.assertEquals(2L, response.getFieldPageSize());
Assert.assertFalse(response.isHasMoreFields());
Assert.assertEquals(List.of("third"), response.getFields().stream()
.map(DatacenterTableField::getFieldName).toList());
Assert.assertTrue(response.getTable().getFields().isEmpty());
}
/**
* 验证惰性迭代器逐页读取并保持原始行顺序。
*
@@ -39,12 +111,15 @@ public class DatacenterDatasetQueryServiceImplTest {
BigInteger sourceId = BigInteger.valueOf(1001L);
DatacenterSource source = new DatacenterSource();
source.setId(sourceId);
source.setSourceType("MYSQL");
source.setTenantId(BigInteger.ONE);
source.setSourceType("PROJECT_MYSQL");
DatacenterTable table = new DatacenterTable();
table.setId(BigInteger.valueOf(2001L));
table.setTenantId(BigInteger.ONE);
table.setSourceId(sourceId);
table.setTableName("orders");
table.setActualTable("orders_actual");
table.setFields(List.of(queryableField("id")));
DatacenterDatasetRegistryService registry =
Mockito.mock(
@@ -54,6 +129,8 @@ public class DatacenterDatasetQueryServiceImplTest {
Mockito.when(registry.listManagedTables(
sourceId, null))
.thenReturn(List.of(table));
Mockito.when(registry.getTableWithFields(table.getId()))
.thenReturn(table);
DatacenterConnector connector =
Mockito.mock(DatacenterConnector.class);
Mockito.doAnswer(invocation -> {
@@ -74,17 +151,21 @@ public class DatacenterDatasetQueryServiceImplTest {
DatacenterConnectorRegistry connectors =
Mockito.mock(
DatacenterConnectorRegistry.class);
Mockito.when(connectors.getConnector("MYSQL"))
Mockito.when(connectors.getConnector("PROJECT_MYSQL"))
.thenReturn(connector);
DatacenterDatasetQueryServiceImpl service =
new DatacenterDatasetQueryServiceImpl();
setField(service, "registryService", registry);
setField(service, "connectorRegistry", connectors);
setField(service, "queryAuditService",
Mockito.mock(DatacenterQueryAuditService.class));
DatacenterSqlQueryRequest request =
new DatacenterSqlQueryRequest();
DatasetRef datasetRef = new DatasetRef();
datasetRef.setTenantId(BigInteger.ONE);
datasetRef.setSourceId(sourceId);
datasetRef.setTableId(table.getId());
request.setDatasetRef(datasetRef);
request.setSql("SELECT * FROM orders ORDER BY id");
@@ -105,6 +186,77 @@ public class DatacenterDatasetQueryServiceImplTest {
ArgumentMatchers.any());
}
/**
* 验证外部 MySQL SQL 查询统一进入 Federation SQL 并保持列标签。
*
* @throws Exception 测试依赖注入失败时抛出
*/
@Test
public void queryBySqlShouldUseFederationForExternalMysql()
throws Exception {
BigInteger sourceId = BigInteger.valueOf(4101L);
BigInteger tableId = BigInteger.valueOf(4102L);
DatacenterSource source = new DatacenterSource();
source.setId(sourceId);
source.setTenantId(BigInteger.ONE);
source.setSourceType("MYSQL");
DatacenterTable table = new DatacenterTable();
table.setId(tableId);
table.setTenantId(BigInteger.ONE);
table.setSourceId(sourceId);
table.setTableName("orders");
table.setActualTable("orders");
DatacenterDatasetRegistryService registry = Mockito.mock(
DatacenterDatasetRegistryService.class);
Mockito.when(registry.getSourceRequired(sourceId)).thenReturn(source);
Mockito.when(registry.listManagedTables(sourceId, null))
.thenReturn(List.of(table));
Mockito.when(registry.getTableWithFields(tableId)).thenReturn(table);
DatacenterConnector connector = Mockito.mock(DatacenterConnector.class);
DatacenterConnectorRegistry connectors = Mockito.mock(
DatacenterConnectorRegistry.class);
Mockito.when(connectors.getConnector("MYSQL")).thenReturn(connector);
DatacenterFederationQueryService federation = Mockito.mock(
DatacenterFederationQueryService.class);
Mockito.when(federation.execute(
Mockito.eq(source),
Mockito.anyString(),
Mockito.eq(List.of()),
Mockito.eq(1_000),
Mockito.isNull(),
Mockito.eq("DATASET"),
Mockito.eq(tableId.toString())))
.thenReturn(new DatacenterSqlConsoleResult(
"query-1",
List.of(new DatacenterSqlColumnView(
"c1", "order_id", java.sql.Types.BIGINT,
"BIGINT", false)),
List.of(java.util.Map.of("c1", "12")),
1L,
false,
3L));
DatacenterDatasetQueryServiceImpl service =
new DatacenterDatasetQueryServiceImpl();
setField(service, "registryService", registry);
setField(service, "connectorRegistry", connectors);
setField(service, "federationQueryService", federation);
DatacenterSqlQueryRequest request = new DatacenterSqlQueryRequest();
DatasetRef datasetRef = new DatasetRef();
datasetRef.setTenantId(BigInteger.ONE);
datasetRef.setSourceId(sourceId);
datasetRef.setTableId(tableId);
request.setDatasetRef(datasetRef);
request.setSql("SELECT order_id FROM orders");
List<Row> rows = service.queryBySql(request);
Assert.assertEquals("12", rows.get(0).get("order_id"));
Mockito.verify(connector, Mockito.never()).queryBySql(
Mockito.any(), Mockito.anyString());
}
/**
* 验证内部 Excel 查询移除逻辑目录并转换 MySQL 标识符引号。
*
@@ -117,13 +269,19 @@ public class DatacenterDatasetQueryServiceImplTest {
BigInteger catalogId = BigInteger.valueOf(3002L);
DatacenterSource source = new DatacenterSource();
source.setId(sourceId);
source.setTenantId(BigInteger.ONE);
source.setSourceType("EXCEL");
DatacenterTable table = new DatacenterTable();
table.setId(BigInteger.valueOf(3003L));
table.setTenantId(BigInteger.ONE);
table.setSourceId(sourceId);
table.setCatalogId(catalogId);
table.setTableName("Sheet1");
table.setMaterializedTable("tb_excel_budget");
table.setFields(List.of(
queryableField("token"),
queryableField("token_1"),
queryableField("col_id")));
DatacenterCatalog catalog = new DatacenterCatalog();
catalog.setId(catalogId);
@@ -134,6 +292,8 @@ public class DatacenterDatasetQueryServiceImplTest {
.thenReturn(source);
Mockito.when(registry.listManagedTables(sourceId, null))
.thenReturn(List.of(table));
Mockito.when(registry.getTableWithFields(table.getId()))
.thenReturn(table);
DatacenterCatalogMapper catalogMapper =
Mockito.mock(DatacenterCatalogMapper.class);
Mockito.when(catalogMapper.selectListByQuery(
@@ -151,10 +311,14 @@ public class DatacenterDatasetQueryServiceImplTest {
setField(service, "registryService", registry);
setField(service, "connectorRegistry", connectors);
setField(service, "catalogMapper", catalogMapper);
setField(service, "queryAuditService",
Mockito.mock(DatacenterQueryAuditService.class));
DatacenterSqlQueryRequest request =
new DatacenterSqlQueryRequest();
DatasetRef datasetRef = new DatasetRef();
datasetRef.setTenantId(BigInteger.ONE);
datasetRef.setSourceId(sourceId);
datasetRef.setTableId(table.getId());
request.setDatasetRef(datasetRef);
request.setSql("""
SELECT "token", "token_1"
@@ -175,6 +339,187 @@ public class DatacenterDatasetQueryServiceImplTest {
sqlCaptor.getValue());
}
/**
* 验证项目 MySQL 原生 SQL 也会拒绝不可查询字段和同名输出别名绕过。
*
* @throws Exception 测试依赖注入失败时抛出
*/
@Test(expected = BusinessException.class)
public void queryBySqlShouldEnforceProjectMysqlColumnPolicy()
throws Exception {
BigInteger sourceId = BigInteger.valueOf(5101L);
BigInteger tableId = BigInteger.valueOf(5102L);
DatacenterSource source = new DatacenterSource();
source.setId(sourceId);
source.setTenantId(BigInteger.ONE);
source.setSourceType("PROJECT_MYSQL");
DatacenterTable table = new DatacenterTable();
table.setId(tableId);
table.setTenantId(BigInteger.ONE);
table.setSourceId(sourceId);
table.setTableName("orders");
table.setActualTable("orders");
DatacenterTableField hidden = queryableField("secret");
hidden.setQueryable(0);
hidden.setSensitivityLevel(DatacenterSensitivityLevel.RESTRICTED.name());
table.setFields(List.of(queryableField("id"), hidden));
DatacenterDatasetRegistryService registry = Mockito.mock(
DatacenterDatasetRegistryService.class);
Mockito.when(registry.getSourceRequired(sourceId)).thenReturn(source);
Mockito.when(registry.getTableWithFields(tableId)).thenReturn(table);
DatacenterConnector connector = Mockito.mock(DatacenterConnector.class);
DatacenterConnectorRegistry connectors = Mockito.mock(
DatacenterConnectorRegistry.class);
Mockito.when(connectors.getConnector("PROJECT_MYSQL"))
.thenReturn(connector);
DatacenterDatasetQueryServiceImpl service =
new DatacenterDatasetQueryServiceImpl();
setField(service, "registryService", registry);
setField(service, "connectorRegistry", connectors);
DatacenterSqlQueryRequest request = new DatacenterSqlQueryRequest();
DatasetRef datasetRef = new DatasetRef();
datasetRef.setTenantId(BigInteger.ONE);
datasetRef.setSourceId(sourceId);
datasetRef.setTableId(tableId);
request.setDatasetRef(datasetRef);
request.setSql("SELECT secret AS secret FROM orders");
service.queryBySql(request);
}
/**
* 验证结构化分页审计绑定执行账号且只记录脱敏筛选摘要。
*
* @throws Exception 测试依赖注入失败时抛出
*/
@Test
@SuppressWarnings({"rawtypes", "unchecked"})
public void queryPageShouldAuditAccountAndMaskedFilters()
throws Exception {
BigInteger sourceId = BigInteger.valueOf(5201L);
BigInteger tableId = BigInteger.valueOf(5202L);
DatacenterSource source = new DatacenterSource();
source.setId(sourceId);
source.setTenantId(BigInteger.ONE);
source.setSourceType("PROJECT_MYSQL");
DatacenterTable table = new DatacenterTable();
table.setId(tableId);
table.setTenantId(BigInteger.ONE);
table.setSourceId(sourceId);
table.setTableName("orders");
DatacenterTableField idField = queryableField("id");
idField.setSortable(1);
table.setFields(List.of(idField));
DatacenterDatasetRegistryService registry = Mockito.mock(
DatacenterDatasetRegistryService.class);
Mockito.when(registry.getTableWithFields(tableId)).thenReturn(table);
Mockito.when(registry.getSourceRequired(sourceId)).thenReturn(source);
DatacenterConnector connector = Mockito.mock(DatacenterConnector.class);
Mockito.when(connector.queryPage(
Mockito.eq(source), Mockito.eq(table), Mockito.any()))
.thenReturn(new Page<>(List.of(row(1)), 1L, 20L, 1L));
DatacenterConnectorRegistry connectors = Mockito.mock(
DatacenterConnectorRegistry.class);
Mockito.when(connectors.getConnector("PROJECT_MYSQL"))
.thenReturn(connector);
DatacenterQueryAuditService auditService = Mockito.mock(
DatacenterQueryAuditService.class);
DatacenterDatasetQueryServiceImpl service =
new DatacenterDatasetQueryServiceImpl();
setField(service, "registryService", registry);
setField(service, "connectorRegistry", connectors);
setField(service, "queryAuditService", auditService);
DatacenterQueryRequest request = new DatacenterQueryRequest();
DatasetRef datasetRef = new DatasetRef();
datasetRef.setTenantId(BigInteger.ONE);
datasetRef.setTableId(tableId);
request.setDatasetRef(datasetRef);
request.setPageNumber(1L);
request.setPageSize(20L);
request.setSelectedColumns(List.of("id"));
DatacenterQueryFilter filter = new DatacenterQueryFilter();
filter.setColumn("id");
filter.setOperator("EQ");
filter.setValue("sensitive-value");
request.setFilters(List.of(filter));
DatacenterQuerySort sort = new DatacenterQuerySort();
sort.setColumn("id");
sort.setDirection("DESC");
request.setSorts(List.of(sort));
LoginAccount account = new LoginAccount();
account.setId(BigInteger.valueOf(9001L));
account.setTenantId(BigInteger.ONE);
service.queryPage(request, account);
ArgumentCaptor<Map> parameters = ArgumentCaptor.forClass(Map.class);
Mockito.verify(auditService).start(
Mockito.anyString(), Mockito.eq(source),
Mockito.contains("WHERE id EQ ?"), parameters.capture(),
Mockito.same(account), Mockito.eq("DATASET_PAGE"),
Mockito.eq(tableId.toString()));
Assert.assertFalse(parameters.getValue().toString()
.contains("sensitive-value"));
Assert.assertTrue(parameters.getValue().toString()
.contains("valueCount=1"));
}
/**
* 验证后台结构化查询不能凭其他租户的表 ID 越权读取。
*/
@Test(expected = BusinessException.class)
public void queryPageShouldRejectCrossTenantTable() throws Exception {
BigInteger sourceId = BigInteger.valueOf(5301L);
BigInteger tableId = BigInteger.valueOf(5302L);
DatacenterSource source = new DatacenterSource();
source.setId(sourceId);
source.setTenantId(BigInteger.TWO);
source.setSourceType("PROJECT_MYSQL");
DatacenterTable table = new DatacenterTable();
table.setId(tableId);
table.setTenantId(BigInteger.TWO);
table.setSourceId(sourceId);
table.setTableName("orders");
table.setFields(List.of(queryableField("id")));
DatacenterDatasetRegistryService registry = Mockito.mock(
DatacenterDatasetRegistryService.class);
Mockito.when(registry.getTableWithFields(tableId)).thenReturn(table);
Mockito.when(registry.getSourceRequired(sourceId)).thenReturn(source);
DatacenterDatasetQueryServiceImpl service =
new DatacenterDatasetQueryServiceImpl();
setField(service, "registryService", registry);
DatacenterQueryRequest request = new DatacenterQueryRequest();
DatasetRef datasetRef = new DatasetRef();
datasetRef.setTenantId(BigInteger.ONE);
datasetRef.setTableId(tableId);
request.setDatasetRef(datasetRef);
LoginAccount account = new LoginAccount();
account.setId(BigInteger.ONE);
account.setTenantId(BigInteger.ONE);
service.queryPage(request, account);
}
/**
* 验证内部与外部结构化查询统一限制单页行数。
*/
@Test(expected = BusinessException.class)
public void queryPageShouldRejectOversizedPage() {
DatacenterDatasetQueryServiceImpl service =
new DatacenterDatasetQueryServiceImpl();
DatacenterQueryRequest request = new DatacenterQueryRequest();
request.setDatasetRef(new DatasetRef());
request.setPageNumber(1L);
request.setPageSize(501L);
service.queryPage(request);
}
/**
* 创建测试数据行。
*
@@ -187,6 +532,22 @@ public class DatacenterDatasetQueryServiceImplTest {
return row;
}
/**
* 创建可查询的公开字段。
*
* @param name 字段名称
* @return 字段定义
*/
private DatacenterTableField queryableField(String name) {
DatacenterTableField field = new DatacenterTableField();
field.setFieldName(name);
field.setSourceColumnName(name);
field.setQueryable(1);
field.setMetadataStatus(DatacenterMetadataStatus.ACTIVE.name());
field.setSensitivityLevel(DatacenterSensitivityLevel.PUBLIC.name());
return field;
}
/**
* 注入被测服务依赖。
*

View File

@@ -0,0 +1,238 @@
package tech.easyflow.datacenter.federation;
import com.easyagents.federation.sql.api.FederationSqlErrorCode;
import com.easyagents.federation.sql.api.FederationSqlException;
import com.easyagents.federation.sql.execute.FederationQueryPermit;
import com.easyagents.federation.sql.execute.QueryId;
import com.easyagents.federation.sql.source.SourceId;
import java.time.Duration;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import org.junit.Test;
/**
* {@link DatacenterFederationAdmissionController} 分层隔离测试。
*/
public class DatacenterFederationAdmissionControllerTest {
/**
* 验证已结束查询不会永久保留历史租户和数据源准入项。
*/
@Test
public void completedQueriesShouldReleaseAdmissionEntries() {
DatacenterFederationAdmissionController controller =
new DatacenterFederationAdmissionController(8, 4, 2);
try {
for (int index = 0; index < 100; index++) {
SourceId sourceId = new SourceId(
"tenant-" + index + "-source-" + index);
try (FederationQueryPermit ignored = controller.acquire(
sourceId,
new QueryId("query-" + index),
Duration.ZERO)) {
org.junit.Assert.assertEquals(
1, controller.trackedSourceAdmissionCount());
org.junit.Assert.assertEquals(
1, controller.trackedTenantAdmissionCount());
}
org.junit.Assert.assertEquals(
0, controller.trackedSourceAdmissionCount());
org.junit.Assert.assertEquals(
0, controller.trackedTenantAdmissionCount());
}
} finally {
controller.close();
}
}
/**
* 验证热点数据源耗尽自身配额后不会占用其他数据源的全局许可。
*/
@Test
public void hotSourceShouldNotStarveAnotherSource() {
DatacenterFederationAdmissionController controller =
new DatacenterFederationAdmissionController(2, 2, 1);
SourceId firstSource = new SourceId("tenant-1-source-1");
SourceId secondSource = new SourceId("tenant-1-source-2");
try (FederationQueryPermit ignored = controller.acquire(
firstSource, new QueryId("query-1"), Duration.ZERO)) {
try {
controller.acquire(
firstSource,
new QueryId("query-2"),
Duration.ZERO);
org.junit.Assert.fail("source limit should reject query");
} catch (FederationSqlException expected) {
// 单源配额按预期先于租户和全局配额生效。
}
try (FederationQueryPermit second = controller.acquire(
secondSource,
new QueryId("query-3"),
Duration.ZERO)) {
org.junit.Assert.assertNotNull(second);
}
} finally {
controller.close();
}
}
/**
* 验证活动查询释放后等待者取得许可,最终清理全部准入项。
*
* @throws Exception 并发执行失败
*/
@Test
public void waitingQueryShouldAcquireAndReleaseWithoutLeakingEntries()
throws Exception {
DatacenterFederationAdmissionController controller =
new DatacenterFederationAdmissionController(2, 2, 1);
SourceId sourceId = new SourceId("tenant-1-source-1");
ExecutorService executor = Executors.newSingleThreadExecutor();
FederationQueryPermit active = controller.acquire(
sourceId, new QueryId("active"), Duration.ZERO);
try {
Future<FederationQueryPermit> waiting = executor.submit(() ->
controller.acquire(
sourceId,
new QueryId("waiting"),
Duration.ofSeconds(2)));
awaitSourceReferences(controller, sourceId, 2);
active.close();
try (FederationQueryPermit acquired = waiting.get(
2, TimeUnit.SECONDS)) {
org.junit.Assert.assertNotNull(acquired);
}
org.junit.Assert.assertEquals(
0, controller.trackedSourceAdmissionCount());
org.junit.Assert.assertEquals(
0, controller.trackedTenantAdmissionCount());
} finally {
active.close();
executor.shutdownNow();
controller.close();
}
}
/**
* 验证等待者取消与 Controller 提前关闭均会正确归还引用。
*
* @throws Exception 并发执行失败
*/
@Test
public void cancelledAndClosedWaitersShouldReleaseReferences()
throws Exception {
DatacenterFederationAdmissionController controller =
new DatacenterFederationAdmissionController(2, 2, 1);
SourceId sourceId = new SourceId("tenant-1-source-1");
ExecutorService executor = Executors.newSingleThreadExecutor();
FederationQueryPermit active = controller.acquire(
sourceId, new QueryId("active"), Duration.ZERO);
AtomicBoolean cancellationRequested = new AtomicBoolean();
try {
Future<FederationQueryPermit> cancelled = executor.submit(() ->
controller.acquire(
sourceId,
new QueryId("cancelled"),
Duration.ofSeconds(2),
cancellationRequested::get));
awaitSourceReferences(controller, sourceId, 2);
cancellationRequested.set(true);
assertFailureCode(
cancelled,
FederationSqlErrorCode.QUERY_CANCELLED);
org.junit.Assert.assertEquals(
1, controller.trackedSourceReferenceCount(sourceId));
Future<FederationQueryPermit> closing = executor.submit(() ->
controller.acquire(
sourceId,
new QueryId("closing"),
Duration.ofSeconds(2)));
awaitSourceReferences(controller, sourceId, 2);
controller.close();
assertFailureCode(
closing,
FederationSqlErrorCode.ENGINE_CLOSED);
active.close();
active.close();
org.junit.Assert.assertEquals(
0, controller.trackedSourceAdmissionCount());
org.junit.Assert.assertEquals(
0, controller.trackedTenantAdmissionCount());
} finally {
active.close();
executor.shutdownNow();
controller.close();
}
}
/**
* 验证关闭完成后的新查询仍返回统一的引擎关闭错误码。
*/
@Test
public void acquireAfterCloseShouldReportEngineClosed() {
DatacenterFederationAdmissionController controller =
new DatacenterFederationAdmissionController(2, 2, 1);
controller.close();
try {
controller.acquire(
new SourceId("tenant-1-source-1"),
new QueryId("closed"),
Duration.ZERO);
org.junit.Assert.fail("closed admission should reject query");
} catch (FederationSqlException exception) {
org.junit.Assert.assertEquals(
FederationSqlErrorCode.ENGINE_CLOSED,
exception.errorCode());
}
}
/**
* 等待指定数据源达到预期引用数。
*
* @param controller 准入控制器
* @param sourceId 数据源标识
* @param expected 预期引用数
* @throws InterruptedException 等待被中断
*/
private static void awaitSourceReferences(
DatacenterFederationAdmissionController controller,
SourceId sourceId,
int expected) throws InterruptedException {
long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(2);
while (System.nanoTime() < deadline) {
if (controller.trackedSourceReferenceCount(sourceId) == expected) {
return;
}
Thread.sleep(5L);
}
org.junit.Assert.fail(
"source admission references did not reach " + expected);
}
/**
* 断言异步准入以指定错误码失败。
*
* @param future 异步准入结果
* @param errorCode 预期错误码
* @throws Exception 获取异步结果失败
*/
private static void assertFailureCode(
Future<FederationQueryPermit> future,
FederationSqlErrorCode errorCode)
throws Exception {
try {
future.get(2, TimeUnit.SECONDS);
org.junit.Assert.fail("query admission should fail");
} catch (ExecutionException exception) {
Throwable cause = exception.getCause();
org.junit.Assert.assertTrue(cause instanceof FederationSqlException);
org.junit.Assert.assertEquals(
errorCode, ((FederationSqlException) cause).errorCode());
}
}
}

View File

@@ -0,0 +1,191 @@
package tech.easyflow.datacenter.federation;
import com.easyagents.federation.sql.source.FederationDataSourceHandle;
import com.easyagents.federation.sql.source.FederationSourceDefinition;
import com.zaxxer.hikari.HikariDataSource;
import java.math.BigInteger;
import java.sql.Connection;
import java.sql.DatabaseMetaData;
import java.util.List;
import java.util.Map;
import org.junit.Assert;
import org.junit.Test;
import org.mockito.Mockito;
import tech.easyflow.datacenter.mapper.DatacenterCatalogMapper;
import tech.easyflow.datacenter.mapper.DatacenterSourceMapper;
import tech.easyflow.datacenter.meta.entity.DatacenterSource;
import tech.easyflow.datacenter.meta.enums.DatacenterSourceStatus;
import tech.easyflow.datacenter.meta.enums.DatacenterSourceType;
/**
* {@link DatacenterFederationDataSourceResolver} 状态与 checksum 契约测试。
*/
public class DatacenterFederationDataSourceResolverTest {
/**
* 验证未发布 checksum 的草稿可以创建仅供 probe 使用的短生命周期句柄。
*
* @throws Exception JDBC mock 初始化失败
*/
@Test
public void shouldResolveDraftForProbeWithoutPublishedChecksum() throws Exception {
DatacenterSource source = source(DatacenterSourceStatus.DRAFT);
DatacenterCatalogMapper catalogMapper = Mockito.mock(DatacenterCatalogMapper.class);
Mockito.when(catalogMapper.selectListByQuery(Mockito.any())).thenReturn(List.of());
DatacenterFederationDefinitionFactory definitionFactory =
new DatacenterFederationDefinitionFactory(catalogMapper);
FederationSourceDefinition definition = definitionFactory.create(source);
DatacenterSourceMapper sourceMapper = Mockito.mock(DatacenterSourceMapper.class);
Mockito.when(sourceMapper.selectOneById(source.getId())).thenReturn(source);
DatacenterFederationDataSourceFactory dataSourceFactory =
Mockito.mock(DatacenterFederationDataSourceFactory.class);
HikariDataSource pool = Mockito.mock(HikariDataSource.class);
Connection connection = Mockito.mock(Connection.class);
DatabaseMetaData metadata = Mockito.mock(DatabaseMetaData.class);
Mockito.when(dataSourceFactory.create(source)).thenReturn(pool);
Mockito.when(pool.getConnection()).thenReturn(connection);
Mockito.when(connection.getMetaData()).thenReturn(metadata);
Mockito.when(metadata.getDatabaseProductName()).thenReturn("MySQL");
Mockito.when(metadata.getDatabaseProductVersion()).thenReturn("8.0");
Mockito.when(metadata.getDriverName()).thenReturn("MySQL Connector/J");
Mockito.when(metadata.getDriverVersion()).thenReturn("9.0");
DatacenterFederationDataSourceResolver resolver = new DatacenterFederationDataSourceResolver(
sourceMapper, definitionFactory, dataSourceFactory);
try (FederationDataSourceHandle handle = resolver.resolve(definition)) {
Assert.assertSame(pool, handle.dataSource());
}
Mockito.verify(pool).close();
}
/**
* 验证活动 Runtime 必须匹配已持久化的发布 checksum。
*/
@Test
public void shouldRejectActiveSourceWithoutPublishedChecksum() {
DatacenterSource source = source(DatacenterSourceStatus.READY);
DatacenterCatalogMapper catalogMapper = Mockito.mock(DatacenterCatalogMapper.class);
Mockito.when(catalogMapper.selectListByQuery(Mockito.any())).thenReturn(List.of());
DatacenterFederationDefinitionFactory definitionFactory =
new DatacenterFederationDefinitionFactory(catalogMapper);
FederationSourceDefinition definition = definitionFactory.create(source);
DatacenterSourceMapper sourceMapper = Mockito.mock(DatacenterSourceMapper.class);
Mockito.when(sourceMapper.selectOneById(source.getId())).thenReturn(source);
DatacenterFederationDataSourceResolver resolver = new DatacenterFederationDataSourceResolver(
sourceMapper, definitionFactory, Mockito.mock(DatacenterFederationDataSourceFactory.class));
try {
resolver.resolve(definition);
Assert.fail("active source without persisted checksum must be rejected");
} catch (IllegalStateException expected) {
Assert.assertTrue(expected.getMessage().contains("published"));
}
}
/**
* 验证候选配置只在同步操作内生效,结束后恢复数据库权威解析。
*
* @throws Exception JDBC mock 初始化失败
*/
@Test
public void candidateResolutionShouldBeThreadBoundAndAlwaysCleared() throws Exception {
DatacenterSource persisted = source(DatacenterSourceStatus.READY);
DatacenterSource candidate = source(DatacenterSourceStatus.DRAFT);
candidate.setDefinitionRevision(2L);
candidate.setDatabaseName("candidate_db");
DatacenterCatalogMapper catalogMapper = Mockito.mock(DatacenterCatalogMapper.class);
Mockito.when(catalogMapper.selectListByQuery(Mockito.any())).thenReturn(List.of());
DatacenterFederationDefinitionFactory definitionFactory =
new DatacenterFederationDefinitionFactory(catalogMapper);
FederationSourceDefinition definition = definitionFactory.createCandidate(candidate);
DatacenterSourceMapper sourceMapper = Mockito.mock(DatacenterSourceMapper.class);
Mockito.when(sourceMapper.selectOneById(candidate.getId())).thenReturn(persisted);
DatacenterFederationDataSourceFactory dataSourceFactory =
Mockito.mock(DatacenterFederationDataSourceFactory.class);
HikariDataSource pool = Mockito.mock(HikariDataSource.class);
Connection connection = Mockito.mock(Connection.class);
DatabaseMetaData metadata = Mockito.mock(DatabaseMetaData.class);
Mockito.when(dataSourceFactory.create(candidate)).thenReturn(pool);
Mockito.when(pool.getConnection()).thenReturn(connection);
Mockito.when(connection.getMetaData()).thenReturn(metadata);
Mockito.when(metadata.getDatabaseProductName()).thenReturn("MySQL");
Mockito.when(metadata.getDatabaseProductVersion()).thenReturn("8.0");
Mockito.when(metadata.getDriverName()).thenReturn("MySQL Connector/J");
Mockito.when(metadata.getDriverVersion()).thenReturn("9.0");
DatacenterFederationDataSourceResolver resolver =
new DatacenterFederationDataSourceResolver(
sourceMapper, definitionFactory, dataSourceFactory);
FederationDataSourceHandle handle = resolver.withCandidate(
candidate, definition, () -> resolver.resolve(definition));
handle.close();
Mockito.verify(sourceMapper, Mockito.never()).selectOneById(candidate.getId());
try {
resolver.resolve(definition);
Assert.fail("candidate context must be cleared after the action");
} catch (IllegalStateException expected) {
Assert.assertTrue(expected.getMessage().contains("revision"));
}
Mockito.verify(sourceMapper).selectOneById(candidate.getId());
}
/**
* 候选操作抛错时也必须清除线程上下文,避免后续请求复用错误凭据。
*/
@Test
public void candidateResolutionShouldClearContextAfterFailure() {
DatacenterSource persisted = source(DatacenterSourceStatus.READY);
DatacenterSource candidate = source(DatacenterSourceStatus.DRAFT);
candidate.setDefinitionRevision(2L);
DatacenterCatalogMapper catalogMapper = Mockito.mock(DatacenterCatalogMapper.class);
Mockito.when(catalogMapper.selectListByQuery(Mockito.any())).thenReturn(List.of());
DatacenterFederationDefinitionFactory definitionFactory =
new DatacenterFederationDefinitionFactory(catalogMapper);
FederationSourceDefinition definition = definitionFactory.createCandidate(candidate);
DatacenterSourceMapper sourceMapper = Mockito.mock(DatacenterSourceMapper.class);
Mockito.when(sourceMapper.selectOneById(candidate.getId())).thenReturn(persisted);
DatacenterFederationDataSourceResolver resolver =
new DatacenterFederationDataSourceResolver(
sourceMapper,
definitionFactory,
Mockito.mock(DatacenterFederationDataSourceFactory.class));
try {
resolver.withCandidate(candidate, definition, () -> {
throw new IllegalStateException("simulated candidate failure");
});
Assert.fail("candidate action must fail");
} catch (IllegalStateException expected) {
Assert.assertEquals("simulated candidate failure", expected.getMessage());
}
try {
resolver.resolve(definition);
Assert.fail("resolver must return to persisted source resolution");
} catch (IllegalStateException expected) {
Assert.assertTrue(expected.getMessage().contains("revision"));
}
Mockito.verify(sourceMapper).selectOneById(candidate.getId());
}
/**
* 创建满足 Definition 最小字段要求的测试数据源。
*
* @param status 数据源状态
* @return 测试数据源
*/
private DatacenterSource source(DatacenterSourceStatus status) {
DatacenterSource source = new DatacenterSource();
source.setId(BigInteger.valueOf(101L));
source.setTenantId(BigInteger.valueOf(7L));
source.setSourceType(DatacenterSourceType.MYSQL.name());
source.setDatabaseName("test_db");
source.setAdapterId("jdbc");
source.setAdapterOptionsJson(Map.of());
source.setDefinitionRevision(1L);
source.setStatus(status.code());
return source;
}
}

View File

@@ -0,0 +1,184 @@
package tech.easyflow.datacenter.federation;
import com.easyagents.federation.sql.api.FederationSqlEngine;
import com.easyagents.federation.sql.execute.QueryId;
import java.math.BigInteger;
import java.time.Duration;
import java.util.UUID;
import org.junit.Assert;
import org.junit.Test;
import org.mockito.Mockito;
import org.springframework.beans.factory.ObjectProvider;
import org.springframework.data.redis.core.StringRedisTemplate;
import org.springframework.data.redis.core.ValueOperations;
import tech.easyflow.common.entity.LoginAccount;
import tech.easyflow.common.web.exceptions.BusinessException;
/**
* {@link DatacenterFederationQueryCancellationService} 租户隔离与登记回归测试。
*/
public class DatacenterFederationQueryCancellationServiceTest {
/**
* 验证其他租户无法取消查询,所属租户可以取消并在关闭登记后释放状态。
*/
@Test
public void cancelShouldBeTenantScopedAndReleaseRegistration() {
FederationSqlEngine engine = Mockito.mock(FederationSqlEngine.class);
DatacenterFederationQueryCancellationService service = service(engine);
QueryId queryId = new QueryId(UUID.randomUUID().toString());
try (DatacenterFederationQueryCancellationService.QueryRegistration ignored =
service.register(queryId, account(1L))) {
Assert.assertFalse(service.cancel(queryId.value(), account(2L)));
Mockito.verify(engine, Mockito.never()).cancel(queryId);
Assert.assertTrue(service.cancel(queryId.value(), account(1L)));
Assert.assertTrue(service.isCancelled(queryId));
Mockito.verify(engine).cancel(queryId);
}
Assert.assertFalse(service.isCancelled(queryId));
}
/**
* 验证活动 QueryId 不能被并发查询复用。
*/
@Test(expected = BusinessException.class)
public void registerShouldRejectDuplicateActiveQueryId() {
DatacenterFederationQueryCancellationService service = service(
Mockito.mock(FederationSqlEngine.class));
QueryId queryId = new QueryId(UUID.randomUUID().toString());
try (DatacenterFederationQueryCancellationService.QueryRegistration ignored =
service.register(queryId, account(1L))) {
service.register(queryId, account(1L));
}
}
/**
* 验证先到达 Redis 的取消请求会在查询登记时立即生效。
*/
@Test
public void registerShouldObservePersistedCancellation() {
FederationSqlEngine engine = Mockito.mock(FederationSqlEngine.class);
StringRedisTemplate redis = Mockito.mock(StringRedisTemplate.class);
Mockito.when(redis.hasKey(Mockito.anyString())).thenReturn(true);
DatacenterFederationQueryCancellationService service = service(engine, redis);
QueryId queryId = new QueryId(UUID.randomUUID().toString());
try (DatacenterFederationQueryCancellationService.QueryRegistration ignored =
service.register(queryId, account(1L))) {
Assert.assertTrue(service.isCancelled(queryId));
Mockito.verify(engine).cancel(queryId);
}
}
/**
* 验证活动查询在丢失 Pub/Sub 提示后仍会通过 Redis 终态核对被取消。
*/
@Test
public void reconcileShouldObserveCancellationCreatedAfterRegister() {
FederationSqlEngine engine = Mockito.mock(FederationSqlEngine.class);
StringRedisTemplate redis = Mockito.mock(StringRedisTemplate.class);
Mockito.when(redis.hasKey(Mockito.anyString()))
.thenReturn(false, true);
DatacenterFederationQueryCancellationService service = service(engine, redis);
QueryId queryId = new QueryId(UUID.randomUUID().toString());
try (DatacenterFederationQueryCancellationService.QueryRegistration ignored =
service.register(queryId, account(1L))) {
Assert.assertFalse(service.isCancelled(queryId));
service.reconcilePersistedCancellations();
Assert.assertTrue(service.isCancelled(queryId));
Mockito.verify(engine).cancel(queryId);
}
}
/**
* 验证取消请求先持久化终态,再发布跨节点通知。
*/
@Test
@SuppressWarnings("unchecked")
public void cancelShouldPersistCancellationBeforePublishing() {
FederationSqlEngine engine = Mockito.mock(FederationSqlEngine.class);
StringRedisTemplate redis = Mockito.mock(StringRedisTemplate.class);
ValueOperations<String, String> values = Mockito.mock(ValueOperations.class);
Mockito.when(redis.opsForValue()).thenReturn(values);
Mockito.when(redis.convertAndSend(
Mockito.eq(DatacenterFederationQueryCancellationService.CHANNEL),
Mockito.anyString())).thenReturn(1L);
DatacenterFederationQueryCancellationService service = service(engine, redis);
QueryId queryId = new QueryId(UUID.randomUUID().toString());
Assert.assertTrue(service.cancel(queryId.value(), account(1L)));
org.mockito.InOrder order = Mockito.inOrder(values, redis);
order.verify(values).set(
Mockito.contains(queryId.value()), Mockito.eq("1"),
Mockito.eq(Duration.ofMinutes(10)));
order.verify(redis).convertAndSend(
Mockito.eq(DatacenterFederationQueryCancellationService.CHANNEL),
Mockito.contains(queryId.value()));
}
/**
* 验证本地驱动取消失败时仍保留分布式终态,并由周期核对重试本地取消。
*/
@Test
@SuppressWarnings("unchecked")
public void cancelShouldPersistAndRetryAfterLocalFailure() {
FederationSqlEngine engine = Mockito.mock(FederationSqlEngine.class);
StringRedisTemplate redis = Mockito.mock(StringRedisTemplate.class);
ValueOperations<String, String> values = Mockito.mock(ValueOperations.class);
Mockito.when(redis.opsForValue()).thenReturn(values);
Mockito.when(redis.convertAndSend(
Mockito.eq(DatacenterFederationQueryCancellationService.CHANNEL),
Mockito.anyString())).thenThrow(
new IllegalStateException("simulated publish failure"));
DatacenterFederationQueryCancellationService service = service(engine, redis);
QueryId queryId = new QueryId(UUID.randomUUID().toString());
Mockito.doThrow(new IllegalStateException("simulated driver failure"))
.doReturn(true)
.when(engine).cancel(queryId);
try (DatacenterFederationQueryCancellationService.QueryRegistration ignored =
service.register(queryId, account(1L))) {
Assert.assertTrue(service.cancel(queryId.value(), account(1L)));
Assert.assertTrue(service.isCancelled(queryId));
service.reconcilePersistedCancellations();
}
Mockito.verify(values).set(
Mockito.contains(queryId.value()), Mockito.eq("1"),
Mockito.eq(Duration.ofMinutes(10)));
Mockito.verify(engine, Mockito.times(2)).cancel(queryId);
}
@SuppressWarnings("unchecked")
private DatacenterFederationQueryCancellationService service(
FederationSqlEngine engine) {
return service(engine, null);
}
@SuppressWarnings("unchecked")
private DatacenterFederationQueryCancellationService service(
FederationSqlEngine engine,
StringRedisTemplate redis) {
DatacenterFederationRuntime runtime = Mockito.mock(
DatacenterFederationRuntime.class);
Mockito.when(runtime.engine()).thenReturn(engine);
ObjectProvider<StringRedisTemplate> redisProvider = Mockito.mock(
ObjectProvider.class);
Mockito.when(redisProvider.getIfAvailable()).thenReturn(redis);
return new DatacenterFederationQueryCancellationService(
runtime, redisProvider);
}
private LoginAccount account(long tenantId) {
LoginAccount account = new LoginAccount();
account.setTenantId(BigInteger.valueOf(tenantId));
return account;
}
}

View File

@@ -0,0 +1,109 @@
package tech.easyflow.datacenter.federation;
import com.easyagents.federation.sql.api.FederationSqlEngine;
import com.easyagents.federation.sql.api.SqlExecutionContext;
import com.easyagents.federation.sql.compile.FederationSqlPlan;
import com.easyagents.federation.sql.compile.SqlCompileRequest;
import com.easyagents.federation.sql.execute.FederationResultCursor;
import com.easyagents.federation.sql.source.SourceId;
import java.math.BigInteger;
import java.util.List;
import java.util.UUID;
import org.apache.calcite.rel.RelNode;
import org.apache.calcite.rel.RelRoot;
import org.apache.calcite.rel.type.RelDataType;
import org.apache.calcite.sql.SqlKind;
import org.apache.calcite.sql.parser.SqlParser;
import org.junit.Assert;
import org.junit.Test;
import org.mockito.ArgumentCaptor;
import org.mockito.Mockito;
import org.springframework.beans.factory.ObjectProvider;
import org.springframework.data.redis.core.StringRedisTemplate;
import tech.easyflow.datacenter.audit.DatacenterQueryAuditService;
import tech.easyflow.datacenter.meta.entity.DatacenterSource;
import tech.easyflow.datacenter.meta.enums.DatacenterSourceStatus;
/**
* {@link DatacenterFederationQueryService} 流式查询取消生命周期回归测试。
*/
public class DatacenterFederationQueryServiceTest {
/**
* 验证调用方 QueryId 会进入执行上下文,完成后释放登记并允许安全复用。
*
* @throws Exception SQL 解析失败
*/
@Test
@SuppressWarnings("unchecked")
public void consumeShouldUseRequestedQueryIdAndReleaseRegistration()
throws Exception {
DatacenterFederationRuntime runtime = Mockito.mock(
DatacenterFederationRuntime.class);
FederationSqlEngine engine = Mockito.mock(FederationSqlEngine.class);
DatacenterFederationDefinitionFactory definitions = Mockito.mock(
DatacenterFederationDefinitionFactory.class);
Mockito.when(runtime.engine()).thenReturn(engine);
Mockito.when(runtime.definitions()).thenReturn(definitions);
SourceId sourceId = new SourceId("tenant-1-source-1001");
DatacenterSource source = new DatacenterSource();
source.setId(BigInteger.valueOf(1001L));
source.setStatus(DatacenterSourceStatus.READY.code());
source.setDefinitionRevision(1L);
source.setScopeRevision(1L);
Mockito.when(definitions.sourceId(source)).thenReturn(sourceId);
RelDataType rowType = Mockito.mock(RelDataType.class);
RelNode root = Mockito.mock(RelNode.class);
Mockito.when(root.getRowType()).thenReturn(rowType);
RelRoot relRoot = RelRoot.of(root, rowType, SqlKind.SELECT);
FederationSqlPlan plan = Mockito.mock(FederationSqlPlan.class);
Mockito.when(plan.relRoot()).thenReturn(relRoot);
Mockito.when(plan.sqlNode()).thenReturn(
SqlParser.create("SELECT 1").parseQuery());
Mockito.when(engine.compile(Mockito.any())).thenReturn(plan);
FederationResultCursor cursor = Mockito.mock(
FederationResultCursor.class);
Mockito.when(cursor.columns()).thenReturn(List.of());
Mockito.when(cursor.next()).thenReturn(false);
Mockito.when(engine.execute(Mockito.eq(plan), Mockito.any()))
.thenReturn(cursor);
ObjectProvider<StringRedisTemplate> redisProvider = Mockito.mock(
ObjectProvider.class);
DatacenterFederationQueryCancellationService cancellationService =
new DatacenterFederationQueryCancellationService(
runtime, redisProvider);
DatacenterFederationQueryService service =
new DatacenterFederationQueryService(
runtime,
Mockito.mock(DatacenterQueryAuditService.class),
cancellationService);
String requestedQueryId = UUID.randomUUID().toString();
Assert.assertEquals(0L, service.consume(
source, "SELECT 1", List.of(), 10, 100, 1_024,
null, "TEST", "stream-1", requestedQueryId,
ignored -> { }));
Assert.assertEquals(0L, service.consume(
source, "SELECT 1", List.of(), 10, 100, 1_024,
null, "TEST", "stream-2", requestedQueryId,
ignored -> { }));
ArgumentCaptor<SqlExecutionContext> contexts =
ArgumentCaptor.forClass(SqlExecutionContext.class);
Mockito.verify(engine, Mockito.times(2)).execute(
Mockito.eq(plan), contexts.capture());
Assert.assertTrue(contexts.getAllValues().stream().allMatch(
context -> requestedQueryId.equals(context.queryId().value())));
ArgumentCaptor<SqlCompileRequest> compileRequests =
ArgumentCaptor.forClass(SqlCompileRequest.class);
Mockito.verify(engine, Mockito.times(2)).compile(compileRequests.capture());
Assert.assertTrue(compileRequests.getAllValues().stream().allMatch(request ->
request.queryScope().revision() == source.getScopeRevision()
&& request.minimumRevision() == source.getDefinitionRevision()
&& sourceId.equals(request.sourceId())));
Mockito.verify(cursor, Mockito.times(2)).close();
}
}

View File

@@ -0,0 +1,286 @@
package tech.easyflow.datacenter.federation;
import com.easyagents.federation.sql.api.FederationSqlException;
import com.easyagents.federation.sql.compile.SqlCompileRequest;
import com.easyagents.federation.sql.compile.SqlPolicyContext;
import com.easyagents.federation.sql.federation.FederationExecutionPolicy;
import com.easyagents.federation.sql.federation.FederationQueryScopeDefinition;
import com.easyagents.federation.sql.federation.FederationSourceBindingDefinition;
import com.easyagents.federation.sql.source.SourceId;
import java.math.BigInteger;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Set;
import org.apache.calcite.plan.RelOptTable;
import org.apache.calcite.rel.RelNode;
import org.apache.calcite.rel.RelRoot;
import org.apache.calcite.rel.RelVisitor;
import org.apache.calcite.rel.core.TableScan;
import org.apache.calcite.rel.type.RelDataType;
import org.apache.calcite.sql.SqlKind;
import org.apache.calcite.sql.SqlNode;
import org.apache.calcite.sql.parser.SqlParser;
import org.junit.Test;
import org.mockito.ArgumentMatchers;
import org.mockito.Mockito;
import tech.easyflow.datacenter.entity.DatacenterTable;
import tech.easyflow.datacenter.entity.DatacenterTableField;
import tech.easyflow.datacenter.mapper.DatacenterCatalogMapper;
import tech.easyflow.datacenter.mapper.DatacenterSourceMapper;
import tech.easyflow.datacenter.mapper.DatacenterTableFieldMapper;
import tech.easyflow.datacenter.mapper.DatacenterTableMapper;
import tech.easyflow.datacenter.meta.entity.DatacenterSource;
import tech.easyflow.datacenter.meta.enums.DatacenterMetadataStatus;
import tech.easyflow.datacenter.meta.enums.DatacenterSensitivityLevel;
import tech.easyflow.datacenter.meta.enums.DatacenterSourceStatus;
/**
* {@link DatacenterFederationSqlPolicy} 字段白名单和别名回归测试。
*/
public class DatacenterFederationSqlPolicyTest {
/**
* 验证表别名、输出别名和 ORDER BY 别名不会被误判为物理字段。
*
* @throws Exception SQL 解析失败
*/
@Test
public void validateShouldAllowTableAndOutputAliases() throws Exception {
DatacenterFederationSqlPolicy policy = policy();
policy.validate(context(
"SELECT o.id AS order_id FROM orders AS o ORDER BY order_id"));
}
/**
* 验证不可查询字段即使通过表别名限定也会被拒绝。
*
* @throws Exception SQL 解析失败
*/
@Test(expected = FederationSqlException.class)
public void validateShouldRejectHiddenQualifiedColumn() throws Exception {
policy().validate(context("SELECT o.secret FROM orders AS o"));
}
/**
* 验证与表同名的字段不会因名称碰撞绕过字段白名单。
*
* @throws Exception SQL 解析失败
*/
@Test(expected = FederationSqlException.class)
public void validateShouldRejectHiddenColumnNamedLikeTable() throws Exception {
policy().validate(context("SELECT orders AS display_name FROM orders"));
}
/**
* 验证输出别名与隐藏物理字段同名时不会绕过字段策略。
*
* @throws Exception SQL 解析失败
*/
@Test(expected = FederationSqlException.class)
public void validateShouldRejectHiddenColumnWithSameOutputAlias()
throws Exception {
policy().validate(context("SELECT secret AS secret FROM orders"));
}
/**
* 验证内层子查询的表别名不会污染外层同名物理表限定符。
*
* @throws Exception SQL 解析失败
*/
@Test(expected = FederationSqlException.class)
public void validateShouldKeepNestedAliasScopesIndependent()
throws Exception {
policy().validate(context(
"SELECT orders.secret FROM orders "
+ "WHERE EXISTS (SELECT 1 FROM public_orders AS orders)",
"orders", "public_orders"));
}
/**
* 验证派生表的输出别名由内层字段策略校验后可在外层引用。
*
* @throws Exception SQL 解析失败
*/
@Test
public void validateShouldAllowDerivedTableOutputAlias()
throws Exception {
policy().validate(context(
"SELECT d.order_id FROM "
+ "(SELECT id AS order_id FROM orders) AS d"));
}
/**
* 验证 CTE 输出别名由内层字段策略校验后可在外层引用。
*
* @throws Exception SQL 解析失败
*/
@Test
public void validateShouldAllowCommonTableExpressionOutputAlias()
throws Exception {
policy().validate(context(
"WITH x AS (SELECT id AS order_id FROM orders) "
+ "SELECT order_id FROM x"));
}
/**
* 验证派生输出仍不能隐藏内层敏感字段引用。
*
* @throws Exception SQL 解析失败
*/
@Test(expected = FederationSqlException.class)
public void validateShouldRejectHiddenColumnInsideDerivedTable()
throws Exception {
policy().validate(context(
"SELECT d.secret_alias FROM "
+ "(SELECT secret AS secret_alias FROM orders) AS d"));
}
/**
* 验证数据库权威范围版本变化后,旧节点缓存不能继续放行查询。
*
* @throws Exception SQL 解析失败
*/
@Test(expected = FederationSqlException.class)
public void validateShouldRejectStaleScopeRevision()
throws Exception {
policy(2L).validate(context("SELECT id FROM orders"));
}
/**
* 验证 XL01 尚未建立逐源授权模型时不会误开放底座联邦能力。
*
* @throws Exception SQL 解析失败
*/
@Test(expected = FederationSqlException.class)
public void validateShouldRejectMultiSourceScopeUntilProductAuthorizationExists()
throws Exception {
SqlPolicyContext single = context("SELECT id FROM orders");
SourceId second = new SourceId("tenant-1-source-1002");
FederationQueryScopeDefinition virtualScope =
FederationQueryScopeDefinition.virtual(
"not-yet-authorized",
1,
Map.of(
"A", FederationSourceBindingDefinition.of(
single.request().sourceId(), 1),
"B", FederationSourceBindingDefinition.of(second, 1)),
"A",
FederationExecutionPolicy.basic());
SqlCompileRequest request = new SqlCompileRequest(
single.request().sql(), virtualScope, List.of(), "scope-1");
policy().validate(new SqlPolicyContext(
request,
single.validatedSql(),
single.relRoot(),
Set.of(second)));
}
private DatacenterFederationSqlPolicy policy() {
return policy(1L);
}
private DatacenterFederationSqlPolicy policy(long currentScopeRevision) {
BigInteger tableId = BigInteger.valueOf(2001L);
BigInteger publicTableId = BigInteger.valueOf(2002L);
DatacenterTable table = table(tableId, "orders");
DatacenterTable publicTable = table(publicTableId, "public_orders");
DatacenterTableMapper tableMapper = Mockito.mock(DatacenterTableMapper.class);
DatacenterTableFieldMapper fieldMapper = Mockito.mock(
DatacenterTableFieldMapper.class);
DatacenterCatalogMapper catalogMapper = Mockito.mock(DatacenterCatalogMapper.class);
DatacenterSourceMapper sourceMapper = Mockito.mock(
DatacenterSourceMapper.class);
DatacenterSource source = new DatacenterSource();
source.setId(BigInteger.valueOf(1001L));
source.setStatus(DatacenterSourceStatus.READY.code());
source.setScopeRevision(currentScopeRevision);
Mockito.when(sourceMapper.selectOneById(BigInteger.valueOf(1001L)))
.thenReturn(source);
Mockito.when(tableMapper.selectListByQuery(ArgumentMatchers.any()))
.thenReturn(List.of(table, publicTable));
Mockito.when(fieldMapper.selectListByQuery(ArgumentMatchers.any()))
.thenReturn(List.of(
field(tableId, "id", true),
field(tableId, "secret", false),
field(tableId, "orders", false),
field(publicTableId, "id", true),
field(publicTableId, "secret", true)));
return new DatacenterFederationSqlPolicy(
tableMapper, fieldMapper, catalogMapper, sourceMapper);
}
private SqlPolicyContext context(String sql, String... tableNames)
throws Exception {
SqlNode parsed = SqlParser.create(sql).parseQuery();
String[] effectiveTableNames = tableNames.length == 0
? new String[]{"orders"} : tableNames;
List<TableScan> scans = new ArrayList<>();
for (String tableName : effectiveTableNames) {
scans.add(scan(tableName));
}
RelDataType rowType = Mockito.mock(RelDataType.class);
Mockito.when(rowType.getFieldNames())
.thenReturn(List.of("id", "secret", "orders"));
RelNode root = scans.size() == 1
? scans.get(0) : Mockito.mock(RelNode.class);
if (scans.size() > 1) {
Mockito.when(root.getRowType()).thenReturn(rowType);
Mockito.doAnswer(invocation -> {
RelVisitor visitor = invocation.getArgument(0);
for (int index = 0; index < scans.size(); index++) {
visitor.visit(scans.get(index), index, root);
}
return null;
}).when(root).childrenAccept(ArgumentMatchers.any(RelVisitor.class));
}
RelRoot relRoot = RelRoot.of(root, rowType, SqlKind.SELECT);
SourceId sourceId = new SourceId("tenant-1-source-1001");
SqlCompileRequest request = new SqlCompileRequest(
sql, sourceId, 1L, List.of(), "scope-1");
return new SqlPolicyContext(request, parsed, relRoot, Set.of(sourceId));
}
private TableScan scan(String tableName) {
RelDataType rowType = Mockito.mock(RelDataType.class);
Mockito.when(rowType.getFieldNames()).thenReturn(
"orders".equals(tableName)
? List.of("id", "secret", "orders")
: List.of("id", "secret"));
TableScan scan = Mockito.mock(TableScan.class);
RelOptTable relOptTable = Mockito.mock(RelOptTable.class);
Mockito.when(relOptTable.getQualifiedName())
.thenReturn(List.of(tableName));
Mockito.when(scan.getTable()).thenReturn(relOptTable);
Mockito.when(scan.getRowType()).thenReturn(rowType);
return scan;
}
private DatacenterTable table(BigInteger tableId, String tableName) {
DatacenterTable table = new DatacenterTable();
table.setId(tableId);
table.setTableName(tableName);
table.setActualTable(tableName);
table.setQueryable(1);
table.setMetadataStatus(DatacenterMetadataStatus.ACTIVE.name());
return table;
}
private DatacenterTableField field(
BigInteger tableId,
String name,
boolean queryable) {
DatacenterTableField field = new DatacenterTableField();
field.setTableId(tableId);
field.setFieldName(name);
field.setSourceColumnName(name);
field.setQueryable(queryable ? 1 : 0);
field.setMetadataStatus(DatacenterMetadataStatus.ACTIVE.name());
field.setSensitivityLevel(queryable
? DatacenterSensitivityLevel.PUBLIC.name()
: DatacenterSensitivityLevel.RESTRICTED.name());
return field;
}
}

View File

@@ -0,0 +1,53 @@
package tech.easyflow.datacenter.federation;
import org.junit.Assert;
import org.junit.Test;
/**
* {@link DatacenterMetadataIdentity} 稳定编码回归测试。
*/
public class DatacenterMetadataIdentityTest {
/**
* 验证 null、用户字面量和字段边界不会产生相同摘要。
*/
@Test
public void namespaceKeyShouldPreservePresenceAndBoundaries() {
String nullCatalog = DatacenterMetadataIdentity.namespaceKey(null, "public");
String literalCatalog = DatacenterMetadataIdentity.namespaceKey("N", "public");
String firstBoundary = DatacenterMetadataIdentity.namespaceKey("ab", "c");
String secondBoundary = DatacenterMetadataIdentity.namespaceKey("a", "bc");
Assert.assertNotEquals(nullCatalog, literalCatalog);
Assert.assertNotEquals(firstBoundary, secondBoundary);
Assert.assertEquals(64, nullCatalog.length());
}
/**
* 验证相同物理身份在重复计算时保持稳定。
*/
@Test
public void tableIdentityShouldBeDeterministic() {
String first = DatacenterMetadataIdentity.tableIdentity(
"source-1", "namespace-1", "orders", "TABLE");
String second = DatacenterMetadataIdentity.tableIdentity(
"source-1", "namespace-1", "orders", "TABLE");
Assert.assertEquals(first, second);
}
/**
* 验证旧表摘要与 V58 迁移中的 MySQL SHA2 输入完全一致。
*/
@Test
public void legacyV58DigestsShouldMatchMigrationAlgorithm() {
Assert.assertEquals(
"1b0b9f334b8ce200ce3c5480930d9bb537af2b630e17f7f1bc47e9dbe2a72274",
DatacenterMetadataIdentity.legacyV58TableIdentity(
12L, 34L, "orders", "TABLE"));
Assert.assertEquals(
"983026e4b253d124c629e19de6b97c30a0e3146ba8c45cdee2540df80a466bb3",
DatacenterMetadataIdentity.legacyV58TableFingerprint(
"orders", "TABLE"));
}
}

View File

@@ -8,6 +8,7 @@ import tech.easyflow.datacenter.entity.DatacenterTable;
import tech.easyflow.datacenter.meta.model.DatacenterCatalogMeta;
import java.math.BigInteger;
import java.util.Map;
/**
* 数据中枢雪花 ID 的 HTTP JSON 序列化测试。
@@ -47,6 +48,24 @@ public class DatacenterIdJsonSerializationTest {
assertTextualId(table, "catalogId");
}
/**
* 验证实体被旧 Schema API 间接序列化时也不会暴露凭据密文和连接扩展配置。
*
* @throws Exception JSON 序列化失败时抛出
*/
@Test
public void shouldHideSourceCredentialMaterial() throws Exception {
DatacenterSource source = new DatacenterSource();
source.setCredentialCipher("v1:encrypted-secret");
source.setConfigJson(Map.of("clientSecret", "plain-secret"));
JsonNode node = objectMapper.readTree(objectMapper.writeValueAsString(source));
Assert.assertFalse(node.has("credentialCipher"));
Assert.assertFalse(node.has("configJson"));
Assert.assertFalse(node.toString().contains("plain-secret"));
}
/**
* 验证给定属性被输出为精确的文本 ID。
*

View File

@@ -0,0 +1,361 @@
package tech.easyflow.datacenter.meta.service.impl;
import java.lang.reflect.Field;
import java.lang.reflect.Method;
import java.math.BigInteger;
import java.sql.Types;
import java.util.List;
import org.junit.Assert;
import org.junit.Test;
import org.mockito.ArgumentMatchers;
import org.mockito.ArgumentCaptor;
import org.mockito.Mockito;
import tech.easyflow.common.entity.LoginAccount;
import tech.easyflow.common.web.exceptions.BusinessException;
import tech.easyflow.datacenter.entity.DatacenterTable;
import tech.easyflow.datacenter.entity.DatacenterTableField;
import tech.easyflow.datacenter.federation.DatacenterFederationChangeNotifier;
import tech.easyflow.datacenter.federation.DatacenterMetadataIdentity;
import tech.easyflow.datacenter.meta.entity.DatacenterCatalog;
import tech.easyflow.datacenter.mapper.DatacenterSourceMapper;
import tech.easyflow.datacenter.mapper.DatacenterTableFieldMapper;
import tech.easyflow.datacenter.mapper.DatacenterTableMapper;
import tech.easyflow.datacenter.meta.entity.DatacenterSource;
import tech.easyflow.datacenter.meta.model.DatacenterFieldDescriptionUpdate;
import tech.easyflow.datacenter.meta.enums.DatacenterSourceStatus;
import tech.easyflow.datacenter.meta.model.DatacenterTableDetailMeta;
/**
* {@link DatacenterDatasetRegistryServiceImpl} 查询范围版本回归测试。
*/
public class DatacenterDatasetRegistryServiceImplTest {
/**
* 验证 V58 未回填的新 JDBC 字段不会让未变化表误报,同时仍检测旧结构变化。
*
* @throws Exception 反射调用失败
*/
@Test
public void legacyStructureShouldIgnoreUnknownNewMetadataOnly()
throws Exception {
DatacenterDatasetRegistryServiceImpl service =
new DatacenterDatasetRegistryServiceImpl();
DatacenterTableField persisted = legacyField("id", 19);
DatacenterTableField discovered = legacyField("id", 19);
discovered.setJdbcTypeCode(Types.BIGINT);
discovered.setOrdinalPosition(1);
Assert.assertTrue(invokeLegacyStructureMatches(
service, List.of(persisted), List.of(discovered)));
DatacenterTableField changed = legacyField("id", 20);
changed.setJdbcTypeCode(Types.BIGINT);
changed.setOrdinalPosition(1);
Assert.assertFalse(invokeLegacyStructureMatches(
service, List.of(persisted), List.of(changed)));
}
/**
* 验证字段治理变更会以 CAS 推进外部数据源范围版本并发布通知。
*
* @throws Exception 测试依赖注入失败
*/
@Test
public void saveDescriptionsShouldAdvanceScopeRevisionForPolicyChange()
throws Exception {
BigInteger sourceId = BigInteger.valueOf(1001L);
BigInteger tableId = BigInteger.valueOf(2001L);
DatacenterSource source = new DatacenterSource();
source.setId(sourceId);
source.setTenantId(BigInteger.ONE);
source.setSourceType("MYSQL");
source.setStatus(DatacenterSourceStatus.READY.code());
source.setDefinitionRevision(1L);
source.setScopeRevision(1L);
DatacenterSource refreshedSource = new DatacenterSource();
refreshedSource.setId(sourceId);
refreshedSource.setTenantId(BigInteger.ONE);
refreshedSource.setSourceType("MYSQL");
refreshedSource.setStatus(DatacenterSourceStatus.READY.code());
refreshedSource.setDefinitionRevision(1L);
refreshedSource.setScopeRevision(2L);
DatacenterTable table = new DatacenterTable();
table.setId(tableId);
table.setSourceId(sourceId);
table.setTenantId(BigInteger.ONE);
DatacenterTableField field = new DatacenterTableField();
field.setId(BigInteger.valueOf(3001L));
field.setTableId(tableId);
field.setQueryable(1);
field.setSensitivityLevel("PUBLIC");
DatacenterTableMapper tableMapper = Mockito.mock(DatacenterTableMapper.class);
DatacenterTableFieldMapper fieldMapper = Mockito.mock(
DatacenterTableFieldMapper.class);
DatacenterSourceMapper sourceMapper = Mockito.mock(DatacenterSourceMapper.class);
DatacenterFederationChangeNotifier notifier = Mockito.mock(
DatacenterFederationChangeNotifier.class);
Mockito.when(tableMapper.selectOneById(tableId)).thenReturn(table);
Mockito.when(fieldMapper.selectListByQuery(ArgumentMatchers.any()))
.thenReturn(List.of(field));
Mockito.when(sourceMapper.selectOneById(sourceId))
.thenReturn(source, refreshedSource);
Mockito.when(sourceMapper.updateByQuery(
ArgumentMatchers.any(), ArgumentMatchers.any()))
.thenReturn(1);
DatacenterDatasetRegistryServiceImpl service =
new DatacenterDatasetRegistryServiceImpl();
setField(service, "tableMapper", tableMapper);
setField(service, "tableFieldMapper", fieldMapper);
setField(service, "sourceMapper", sourceMapper);
setField(service, "federationChangeNotifier", notifier);
DatacenterFieldDescriptionUpdate update =
new DatacenterFieldDescriptionUpdate();
update.setFieldId(field.getId());
update.setFieldDesc("内部编号");
update.setQueryable(0);
update.setSensitivityLevel("INTERNAL");
LoginAccount account = new LoginAccount();
account.setId(BigInteger.TEN);
account.setTenantId(BigInteger.ONE);
service.saveDescriptions(tableId, "订单", List.of(update), account);
Assert.assertEquals(Integer.valueOf(0), field.getQueryable());
Assert.assertEquals("INTERNAL", field.getSensitivityLevel());
ArgumentCaptor<DatacenterSource> patchCaptor =
ArgumentCaptor.forClass(DatacenterSource.class);
Mockito.verify(sourceMapper).updateByQuery(
patchCaptor.capture(), ArgumentMatchers.any());
DatacenterSource patch = patchCaptor.getValue();
Assert.assertEquals(sourceId, patch.getId());
Assert.assertEquals(Long.valueOf(2L), patch.getScopeRevision());
Assert.assertNull(patch.getStatus());
Assert.assertNull(patch.getDefinitionRevision());
Mockito.verify(notifier).publishAfterCommit(refreshedSource);
}
/**
* 验证刷新发现未纳管新字段时只标记结构变化并收缩权限,不自动插入字段。
*
* @throws Exception 测试依赖注入失败
*/
@Test
public void refreshShouldFailClosedWithoutRegisteringNewField()
throws Exception {
BigInteger sourceId = BigInteger.valueOf(1101L);
BigInteger catalogId = BigInteger.valueOf(2101L);
BigInteger tableId = BigInteger.valueOf(3101L);
DatacenterSource source = new DatacenterSource();
source.setId(sourceId);
source.setTenantId(BigInteger.ONE);
source.setSourceType("MYSQL");
DatacenterCatalog catalog = new DatacenterCatalog();
catalog.setId(catalogId);
catalog.setSourceId(sourceId);
catalog.setNamespaceKey(DatacenterMetadataIdentity.namespaceKey("easyflow", null));
DatacenterTableField idField = modernField("id", Types.BIGINT);
idField.setId(BigInteger.valueOf(4101L));
idField.setTableId(tableId);
idField.setQueryable(1);
DatacenterTable table = new DatacenterTable();
table.setId(tableId);
table.setSourceId(sourceId);
table.setCatalogId(catalogId);
table.setTableName("orders");
table.setActualTable("orders");
table.setTableKind("EXTERNAL_TABLE");
table.setMetadataStatus("ACTIVE");
table.setQueryable(1);
table.setPhysicalIdentityKey(DatacenterMetadataIdentity.tableIdentity(
sourceId.toString(), catalog.getNamespaceKey(), "orders", "EXTERNAL_TABLE"));
table.setMetadataFingerprint(DatacenterMetadataIdentity.tableFingerprint(
"orders", "EXTERNAL_TABLE", List.of(idField)));
DatacenterTableField newField = modernField("secret", Types.VARCHAR);
DatacenterTable discovered = new DatacenterTable();
discovered.setTableName("orders");
discovered.setActualTable("orders");
discovered.setTableKind("EXTERNAL_TABLE");
DatacenterTableDetailMeta detail = new DatacenterTableDetailMeta();
detail.setTable(discovered);
detail.setFields(List.of(modernField("id", Types.BIGINT), newField));
DatacenterTableMapper tableMapper = Mockito.mock(DatacenterTableMapper.class);
DatacenterTableFieldMapper fieldMapper = Mockito.mock(
DatacenterTableFieldMapper.class);
Mockito.when(tableMapper.selectOneById(tableId)).thenReturn(table);
Mockito.when(fieldMapper.selectListByQuery(ArgumentMatchers.any()))
.thenReturn(List.of(idField));
DatacenterDatasetRegistryServiceImpl service =
new DatacenterDatasetRegistryServiceImpl();
setField(service, "tableMapper", tableMapper);
setField(service, "tableFieldMapper", fieldMapper);
boolean changed = service.refreshManagedTable(
source, catalog, tableId, detail, account());
Assert.assertTrue(changed);
Assert.assertEquals("CHANGED", table.getMetadataStatus());
Assert.assertEquals(Integer.valueOf(0), table.getQueryable());
Assert.assertEquals("CHANGED", idField.getMetadataStatus());
Assert.assertEquals(Integer.valueOf(0), idField.getQueryable());
Mockito.verify(fieldMapper, Mockito.never())
.insert(ArgumentMatchers.any(DatacenterTableField.class));
}
/**
* 显式重配置允许将稳定表 ID 绑定到新物理命名空间,并重建字段治理基线。
* 常规刷新对相同变化仍保持 fail-closed。
*
* @throws Exception 测试依赖注入失败
*/
@Test
public void explicitRebindShouldPreserveTableIdAndReplaceFields()
throws Exception {
BigInteger sourceId = BigInteger.valueOf(1201L);
BigInteger catalogId = BigInteger.valueOf(2201L);
BigInteger tableId = BigInteger.valueOf(3201L);
DatacenterSource source = new DatacenterSource();
source.setId(sourceId);
source.setTenantId(BigInteger.ONE);
source.setSourceType("MYSQL");
DatacenterCatalog catalog = new DatacenterCatalog();
catalog.setId(catalogId);
catalog.setSourceId(sourceId);
catalog.setNamespaceKey(DatacenterMetadataIdentity.namespaceKey("new_db", null));
DatacenterTableField persistedField = modernField("legacy_id", Types.BIGINT);
persistedField.setId(BigInteger.valueOf(4201L));
persistedField.setTableId(tableId);
DatacenterTable existing = new DatacenterTable();
existing.setId(tableId);
existing.setSourceId(sourceId);
existing.setCatalogId(catalogId);
existing.setTableName("orders");
existing.setActualTable("orders");
existing.setTableKind("EXTERNAL_TABLE");
existing.setQueryable(0);
existing.setPhysicalIdentityKey(DatacenterMetadataIdentity.tableIdentity(
sourceId.toString(),
DatacenterMetadataIdentity.namespaceKey("old_db", null),
"orders",
"EXTERNAL_TABLE"));
existing.setMetadataFingerprint(DatacenterMetadataIdentity.tableFingerprint(
"orders", "EXTERNAL_TABLE", List.of(persistedField)));
DatacenterTableField discoveredField = modernField("id", Types.BIGINT);
DatacenterTable discovered = new DatacenterTable();
discovered.setTableName("orders");
discovered.setActualTable("orders");
discovered.setTableKind("EXTERNAL_TABLE");
discovered.setQueryable(1);
DatacenterTableDetailMeta detail = new DatacenterTableDetailMeta();
detail.setTable(discovered);
detail.setFields(List.of(discoveredField));
DatacenterTableMapper tableMapper = Mockito.mock(DatacenterTableMapper.class);
DatacenterTableFieldMapper fieldMapper = Mockito.mock(
DatacenterTableFieldMapper.class);
Mockito.when(tableMapper.selectOneByQuery(ArgumentMatchers.any()))
.thenReturn(existing);
Mockito.when(fieldMapper.selectListByQuery(ArgumentMatchers.any()))
.thenReturn(
List.of(persistedField),
List.of(persistedField),
List.of(discoveredField));
DatacenterDatasetRegistryServiceImpl service =
new DatacenterDatasetRegistryServiceImpl();
setField(service, "tableMapper", tableMapper);
setField(service, "tableFieldMapper", fieldMapper);
try {
service.registerTable(source, catalog, detail, account());
Assert.fail("ordinary refresh must reject changed physical identity");
} catch (BusinessException expected) {
Assert.assertTrue(expected.getMessage().contains("物理表身份已变化"));
}
DatacenterTable rebound = service.rebindTable(
source, catalog, detail, account());
Assert.assertEquals(tableId, rebound.getId());
Assert.assertEquals(Integer.valueOf(1), rebound.getQueryable());
Assert.assertEquals(DatacenterMetadataIdentity.tableIdentity(
sourceId.toString(), catalog.getNamespaceKey(),
"orders", "EXTERNAL_TABLE"), rebound.getPhysicalIdentityKey());
Mockito.verify(fieldMapper).deleteByQuery(ArgumentMatchers.any());
Mockito.verify(fieldMapper).insert(discoveredField);
Mockito.verify(tableMapper).update(existing);
}
/**
* 注入被测服务依赖。
*
* @param target 被测对象
* @param name 字段名
* @param value 依赖值
* @throws ReflectiveOperationException 反射失败
*/
private void setField(Object target, String name, Object value)
throws ReflectiveOperationException {
Field field = target.getClass().getDeclaredField(name);
field.setAccessible(true);
field.set(target, value);
}
private DatacenterTableField legacyField(String name, int precision) {
DatacenterTableField field = new DatacenterTableField();
field.setFieldName(name);
field.setSourceColumnName(name);
field.setFieldType(0);
field.setJdbcType("BIGINT");
field.setNativeTypeName("BIGINT");
field.setPrecision(precision);
field.setScale(0);
field.setRequired(1);
return field;
}
private DatacenterTableField modernField(String name, int jdbcType) {
DatacenterTableField field = new DatacenterTableField();
field.setFieldName(name);
field.setSourceColumnName(name);
field.setJdbcType(jdbcType == Types.BIGINT ? "BIGINT" : "VARCHAR");
field.setNativeTypeName(field.getJdbcType());
field.setJdbcTypeCode(jdbcType);
field.setPrecision(jdbcType == Types.BIGINT ? 19 : 255);
field.setScale(0);
field.setRequired(0);
field.setQueryable(1);
field.setSortable(1);
field.setWritable(0);
field.setMetadataStatus("ACTIVE");
field.setMetadataFingerprint(DatacenterMetadataIdentity.fieldFingerprint(field));
return field;
}
private LoginAccount account() {
LoginAccount account = new LoginAccount();
account.setId(BigInteger.TEN);
account.setTenantId(BigInteger.ONE);
return account;
}
@SuppressWarnings("unchecked")
private boolean invokeLegacyStructureMatches(
DatacenterDatasetRegistryServiceImpl service,
List<DatacenterTableField> persisted,
List<DatacenterTableField> discovered)
throws ReflectiveOperationException {
Method method = DatacenterDatasetRegistryServiceImpl.class
.getDeclaredMethod(
"legacyStructureMatches", List.class, List.class);
method.setAccessible(true);
return (boolean) method.invoke(service, persisted, discovered);
}
}

View File

@@ -0,0 +1,125 @@
package tech.easyflow.datacenter.meta.service.impl;
import java.math.BigInteger;
import java.util.List;
import java.util.Set;
import org.junit.Assert;
import org.junit.Test;
import org.mockito.ArgumentMatchers;
import org.mockito.InOrder;
import org.mockito.Mockito;
import org.springframework.transaction.PlatformTransactionManager;
import org.springframework.transaction.TransactionDefinition;
import org.springframework.transaction.TransactionStatus;
import tech.easyflow.common.entity.LoginAccount;
import tech.easyflow.datacenter.connector.DatacenterConnector;
import tech.easyflow.datacenter.connector.DatacenterConnectorRegistry;
import tech.easyflow.datacenter.entity.DatacenterTable;
import tech.easyflow.datacenter.federation.DatacenterFederationChangeNotifier;
import tech.easyflow.datacenter.mapper.DatacenterCatalogMapper;
import tech.easyflow.datacenter.mapper.DatacenterSourceMapper;
import tech.easyflow.datacenter.mapper.DatacenterTableMapper;
import tech.easyflow.datacenter.meta.entity.DatacenterCatalog;
import tech.easyflow.datacenter.meta.entity.DatacenterSource;
import tech.easyflow.datacenter.meta.enums.DatacenterSourceStatus;
import tech.easyflow.datacenter.meta.model.DatacenterManagedMetadataSnapshot;
import tech.easyflow.datacenter.meta.model.DatacenterTableDetailMeta;
import tech.easyflow.datacenter.meta.service.DatacenterDatasetRegistryService;
/**
* {@link DatacenterSourceMetadataRefreshService} 事务与远程 I/O 边界测试。
*/
public class DatacenterSourceMetadataRefreshServiceTest {
/**
* 远程 JDBC 元数据必须在开启平台数据库事务前完成。
*/
@Test
public void refreshShouldInspectRemoteMetadataBeforeOpeningTransaction() {
BigInteger sourceId = BigInteger.valueOf(1301L);
DatacenterSource source = new DatacenterSource();
source.setId(sourceId);
source.setTenantId(BigInteger.ONE);
source.setSourceType("MYSQL");
source.setStatus(DatacenterSourceStatus.READY.code());
source.setDefinitionRevision(3L);
source.setScopeRevision(5L);
DatacenterCatalog catalog = new DatacenterCatalog();
catalog.setId(BigInteger.valueOf(2301L));
catalog.setSourceId(sourceId);
catalog.setCatalogName("orders_db");
DatacenterTable table = new DatacenterTable();
table.setId(BigInteger.valueOf(3301L));
table.setSourceId(sourceId);
table.setCatalogId(catalog.getId());
table.setTableName("orders");
table.setActualTable("orders");
DatacenterTable discovered = new DatacenterTable();
discovered.setTableName("orders");
discovered.setActualTable("orders");
DatacenterTableDetailMeta detail = new DatacenterTableDetailMeta();
detail.setTable(discovered);
LoginAccount account = account();
DatacenterConnector connector = Mockito.mock(DatacenterConnector.class);
DatacenterConnectorRegistry connectorRegistry =
Mockito.mock(DatacenterConnectorRegistry.class);
DatacenterDatasetRegistryService registryService =
Mockito.mock(DatacenterDatasetRegistryService.class);
DatacenterCatalogMapper catalogMapper =
Mockito.mock(DatacenterCatalogMapper.class);
DatacenterTableMapper tableMapper = Mockito.mock(DatacenterTableMapper.class);
DatacenterSourceMapper sourceMapper =
Mockito.mock(DatacenterSourceMapper.class);
DatacenterFederationChangeNotifier notifier =
Mockito.mock(DatacenterFederationChangeNotifier.class);
PlatformTransactionManager transactionManager =
Mockito.mock(PlatformTransactionManager.class);
TransactionStatus transactionStatus = Mockito.mock(TransactionStatus.class);
Mockito.when(connectorRegistry.getConnector("MYSQL")).thenReturn(connector);
Mockito.when(catalogMapper.selectListByQuery(ArgumentMatchers.any()))
.thenReturn(List.of(catalog));
Mockito.when(tableMapper.selectListByQuery(ArgumentMatchers.any()))
.thenReturn(List.of(table));
Mockito.when(connector.inspectManagedTables(
source, "orders_db", List.of("orders")))
.thenReturn(new DatacenterManagedMetadataSnapshot(
List.of(detail), Set.of()));
Mockito.when(registryService.refreshManagedTable(
source, catalog, table.getId(), detail, account))
.thenReturn(false);
Mockito.when(transactionManager.getTransaction(
ArgumentMatchers.any(TransactionDefinition.class)))
.thenReturn(transactionStatus);
Mockito.when(sourceMapper.updateByQuery(
ArgumentMatchers.any(), ArgumentMatchers.any()))
.thenReturn(1);
DatacenterSource refreshed = new DatacenterSource();
refreshed.setId(sourceId);
Mockito.when(sourceMapper.selectOneByQuery(ArgumentMatchers.any()))
.thenReturn(refreshed);
DatacenterSourceMetadataRefreshService service =
new DatacenterSourceMetadataRefreshService(
connectorRegistry, registryService, catalogMapper,
tableMapper, sourceMapper, notifier, transactionManager);
DatacenterSource result = service.refresh(source, account);
Assert.assertSame(refreshed, result);
InOrder order = Mockito.inOrder(connector, transactionManager, sourceMapper);
order.verify(connector).inspectManagedTables(
source, "orders_db", List.of("orders"));
order.verify(transactionManager).getTransaction(
ArgumentMatchers.any(TransactionDefinition.class));
order.verify(sourceMapper).updateByQuery(
ArgumentMatchers.any(), ArgumentMatchers.any());
Mockito.verify(notifier).publishAfterCommit(refreshed);
}
private LoginAccount account() {
LoginAccount account = new LoginAccount();
account.setId(BigInteger.TEN);
account.setTenantId(BigInteger.ONE);
return account;
}
}

View File

@@ -0,0 +1,410 @@
package tech.easyflow.datacenter.meta.service.impl;
import com.mybatisflex.core.query.QueryWrapper;
import com.mybatisflex.spring.service.impl.ServiceImpl;
import java.lang.reflect.Field;
import java.math.BigInteger;
import java.util.List;
import org.junit.Assert;
import org.junit.Test;
import org.mockito.ArgumentCaptor;
import org.mockito.ArgumentMatchers;
import org.mockito.InOrder;
import org.mockito.Mockito;
import org.springframework.transaction.PlatformTransactionManager;
import org.springframework.transaction.TransactionDefinition;
import org.springframework.transaction.TransactionStatus;
import tech.easyflow.common.entity.LoginAccount;
import tech.easyflow.datacenter.connector.DatacenterConnector;
import tech.easyflow.datacenter.connector.DatacenterConnectorRegistry;
import tech.easyflow.datacenter.entity.DatacenterTable;
import tech.easyflow.datacenter.federation.DatacenterFederationChangeNotifier;
import tech.easyflow.datacenter.federation.DatacenterFederationDefinitionFactory;
import tech.easyflow.datacenter.federation.DatacenterFederationRuntime;
import tech.easyflow.datacenter.mapper.DatacenterCatalogMapper;
import tech.easyflow.datacenter.mapper.DatacenterSourceMapper;
import tech.easyflow.datacenter.mapper.DatacenterTableMapper;
import tech.easyflow.datacenter.meta.entity.DatacenterCatalog;
import tech.easyflow.datacenter.meta.entity.DatacenterSource;
import tech.easyflow.datacenter.meta.enums.DatacenterMetadataStatus;
import tech.easyflow.datacenter.meta.enums.DatacenterSourceStatus;
import tech.easyflow.datacenter.meta.model.DatacenterBatchRegisterRequest;
import tech.easyflow.datacenter.meta.model.DatacenterSourceActivateRequest;
import tech.easyflow.datacenter.meta.model.DatacenterTableDetailMeta;
import tech.easyflow.datacenter.meta.model.DatacenterSourceDraftRequest;
import tech.easyflow.datacenter.meta.model.DatacenterSourceReconfigureRequest;
import tech.easyflow.datacenter.meta.service.DatacenterDatasetRegistryService;
import tech.easyflow.datacenter.meta.support.DatacenterSourceConnectionDefaults;
import tech.easyflow.datacenter.security.DatacenterCredentialCipher;
import com.easyagents.federation.sql.source.ExternalSchemaDefinition;
import com.easyagents.federation.sql.source.FederationSourceDefinition;
import com.easyagents.federation.sql.source.PreparedSourceRuntime;
import com.easyagents.federation.sql.source.RuntimeFingerprint;
import com.easyagents.federation.sql.source.SourceId;
import com.easyagents.federation.sql.source.SourceProbeResult;
/**
* {@link DatacenterSourceServiceImpl} 生命周期和 revision 回归测试。
*/
public class DatacenterSourceServiceImplTest {
/**
* 已发布连接不得复用初次激活入口绕过重配置 revision CAS。
*
* @throws Exception 测试依赖注入失败
*/
@Test
public void activateShouldRejectPublishedSource() throws Exception {
Fixture fixture = fixture();
DatacenterSource ready = source(DatacenterSourceStatus.READY, 3L, 7L);
Mockito.when(fixture.sourceMapper.selectOneByQuery(ArgumentMatchers.any()))
.thenReturn(ready);
try {
fixture.service.activate(new DatacenterSourceActivateRequest(
ready.getId(), "public", List.of("orders"), true), account());
Assert.fail("published source must use reconfiguration");
} catch (tech.easyflow.common.web.exceptions.BusinessException expected) {
Assert.assertTrue(expected.getMessage().contains("重新配置"));
}
Mockito.verifyNoInteractions(fixture.connectorRegistry);
Mockito.verify(fixture.runtime, Mockito.never())
.probe(ArgumentMatchers.any());
Mockito.verify(fixture.runtime, Mockito.never())
.probeCandidate(ArgumentMatchers.any());
}
/**
* 已发布连接按持久化 Definition 探测,避免候选 MAIN Schema 造成 checksum 冲突。
*
* @throws Exception 测试依赖注入失败
*/
@Test
public void publishedSourceProbeShouldUsePublishedDefinition() throws Exception {
Fixture fixture = fixture();
DatacenterSource ready = source(DatacenterSourceStatus.READY, 3L, 7L);
ready.setSchemaName("tenant_schema");
ready.setDefinitionChecksum("published-checksum");
Mockito.when(fixture.sourceMapper.selectOneByQuery(ArgumentMatchers.any()))
.thenReturn(ready);
Mockito.when(fixture.sourceMapper.updateByQuery(
ArgumentMatchers.any(), ArgumentMatchers.any())).thenReturn(1);
Mockito.when(fixture.runtime.probe(ready)).thenReturn(new SourceProbeResult(
new SourceId("tenant-1-source-100"), true,
new RuntimeFingerprint(
"PostgreSQL", "17", "PostgreSQL JDBC", "42", "test"),
"ok"));
Assert.assertTrue(fixture.service.probe(ready.getId(), account()).isSuccess());
Mockito.verify(fixture.runtime).probe(ready);
Mockito.verify(fixture.runtime, Mockito.never())
.probeCandidate(ArgumentMatchers.any());
}
/**
* 停用只推进 Definition 版本并发布墓碑,不改变业务范围版本。
*
* @throws Exception 测试依赖注入失败
*/
@Test
public void disableShouldAdvanceDefinitionRevisionOnly() throws Exception {
Fixture fixture = fixture();
DatacenterSource ready = source(DatacenterSourceStatus.READY, 3L, 7L);
DatacenterSource disabled = source(DatacenterSourceStatus.DISABLED, 4L, 7L);
Mockito.when(fixture.sourceMapper.selectOneByQuery(ArgumentMatchers.any()))
.thenReturn(ready, disabled);
Mockito.when(fixture.sourceMapper.updateByQuery(
ArgumentMatchers.any(), ArgumentMatchers.any())).thenReturn(1);
fixture.service.disable(ready.getId(), account());
ArgumentCaptor<DatacenterSource> patch =
ArgumentCaptor.forClass(DatacenterSource.class);
Mockito.verify(fixture.sourceMapper).updateByQuery(
patch.capture(), ArgumentMatchers.any(QueryWrapper.class));
Assert.assertEquals(Long.valueOf(4L), patch.getValue().getDefinitionRevision());
Assert.assertNull(patch.getValue().getScopeRevision());
Assert.assertEquals(Integer.valueOf(DatacenterSourceStatus.DISABLED.code()),
patch.getValue().getStatus());
Mockito.verify(fixture.runtime).remove(disabled);
Mockito.verify(fixture.notifier).publishAfterCommit(disabled);
}
/**
* 已激活连接新增纳管表时只推进 scopeRevision避免无意义重建连接池。
*
* @throws Exception 测试依赖注入失败
*/
@Test
public void batchRegisterShouldAdvanceScopeRevisionOnly() throws Exception {
Fixture fixture = fixture();
DatacenterSource ready = source(DatacenterSourceStatus.READY, 5L, 9L);
ready.setLastTestStatus("SUCCESS");
DatacenterSource updated = source(DatacenterSourceStatus.READY, 5L, 10L);
DatacenterCatalog catalog = new DatacenterCatalog();
catalog.setId(BigInteger.valueOf(200L));
catalog.setSourceId(ready.getId());
catalog.setCatalogName("public");
catalog.setMetadataStatus(DatacenterMetadataStatus.ACTIVE.name());
DatacenterTableDetailMeta detail = new DatacenterTableDetailMeta();
DatacenterTable discovered = new DatacenterTable();
discovered.setTableName("orders");
detail.setTable(discovered);
DatacenterConnector connector = Mockito.mock(DatacenterConnector.class);
Mockito.when(fixture.sourceMapper.selectOneByQuery(ArgumentMatchers.any()))
.thenReturn(ready, updated);
Mockito.when(fixture.sourceMapper.updateByQuery(
ArgumentMatchers.any(), ArgumentMatchers.any())).thenReturn(1);
Mockito.when(fixture.registry.listManagedTables(
ready.getId(), catalog.getId())).thenReturn(List.of());
Mockito.when(fixture.catalogMapper.selectOneByQuery(ArgumentMatchers.any()))
.thenReturn(catalog);
Mockito.when(fixture.connectorRegistry.getConnector("MYSQL"))
.thenReturn(connector);
Mockito.when(connector.getTableDetails(
ready, "public", List.of("orders"))).thenReturn(List.of(detail));
DatacenterBatchRegisterRequest request = new DatacenterBatchRegisterRequest();
request.setSourceId(ready.getId());
request.setCatalogName("public");
request.setTableNames(List.of("orders"));
fixture.service.batchRegisterTables(request, account());
ArgumentCaptor<DatacenterSource> patch =
ArgumentCaptor.forClass(DatacenterSource.class);
Mockito.verify(fixture.sourceMapper).updateByQuery(
patch.capture(), ArgumentMatchers.any(QueryWrapper.class));
Assert.assertNull(patch.getValue().getDefinitionRevision());
Assert.assertEquals(Long.valueOf(10L), patch.getValue().getScopeRevision());
Mockito.verify(fixture.runtime, Mockito.never()).apply(
ArgumentMatchers.any(), ArgumentMatchers.anyBoolean());
Mockito.verify(fixture.notifier).publishAfterCommit(updated);
}
/**
* 候选配置探测失败时不写数据库,也不替换当前 Runtime。
*
* @throws Exception 测试依赖注入失败
*/
@Test
public void failedReconfigurationProbeShouldKeepReadySourceUntouched()
throws Exception {
Fixture fixture = fixture();
DatacenterSource ready = source(DatacenterSourceStatus.READY, 8L, 12L);
ready.setSourceName("orders-db");
ready.setSourceCode("ORDERS_DB");
ready.setHost("127.0.0.1");
ready.setPort(3306);
ready.setDatabaseName("orders");
ready.setUsername("reader");
ready.setCredentialCipher("encrypted");
Mockito.when(fixture.sourceMapper.selectOneByQuery(ArgumentMatchers.any()))
.thenReturn(ready);
Mockito.when(fixture.runtime.probeCandidate(ArgumentMatchers.any()))
.thenThrow(new IllegalStateException("simulated connection failure"));
Mockito.when(fixture.defaults.defaultDriverClassName("MYSQL"))
.thenReturn("com.mysql.cj.jdbc.Driver");
DatacenterSourceDraftRequest definition = new DatacenterSourceDraftRequest(
ready.getId(), "orders-db", "ORDERS_DB", "MYSQL",
"127.0.0.2", 3306, "orders", null, "reader", null,
null, null, java.util.Map.of(), java.util.Map.of());
DatacenterSourceReconfigureRequest request =
new DatacenterSourceReconfigureRequest(
definition, "orders", List.of("orders"), true, 8L, 12L);
try {
fixture.service.reconfigure(request, account());
Assert.fail("failed candidate probe must reject reconfiguration");
} catch (tech.easyflow.common.web.exceptions.BusinessException expected) {
Assert.assertTrue(expected.getMessage().contains("无法连接"));
}
Mockito.verify(fixture.sourceMapper, Mockito.never()).updateByQuery(
ArgumentMatchers.any(), ArgumentMatchers.any());
Mockito.verify(fixture.runtime, Mockito.never()).apply(
ArgumentMatchers.any(), ArgumentMatchers.anyBoolean());
}
/**
* 重新启用时先预构建 Runtime再发布权威 READY 并原子接管。
*
* @throws Exception 测试依赖注入失败
*/
@Test
public void enableShouldPrepareBeforePublishingAndCommitAfterward() throws Exception {
Fixture fixture = fixture();
DatacenterSource disabled = source(DatacenterSourceStatus.DISABLED, 4L, 7L);
disabled.setSourceName("orders-db");
disabled.setSourceCode("ORDERS_DB");
disabled.setDatabaseName("orders");
disabled.setSchemaName("public");
disabled.setCredentialCipher("encrypted");
disabled.setAdapterId("jdbc");
DatacenterSource enabled = source(DatacenterSourceStatus.READY, 5L, 7L);
Mockito.when(fixture.sourceMapper.selectOneByQuery(ArgumentMatchers.any()))
.thenReturn(disabled, enabled);
Mockito.when(fixture.sourceMapper.updateByQuery(
ArgumentMatchers.any(), ArgumentMatchers.any())).thenReturn(1);
RuntimeFingerprint fingerprint = new RuntimeFingerprint(
"MySQL", "8.0", "MySQL Connector/J", "9.0", "test");
Mockito.when(fixture.runtime.probeCandidate(ArgumentMatchers.any()))
.thenReturn(new SourceProbeResult(
new SourceId("tenant-1-source-100"), true, fingerprint, "ok"));
FederationSourceDefinition definition = new FederationSourceDefinition(
new SourceId("tenant-1-source-100"),
5L,
"jdbc",
List.of(new ExternalSchemaDefinition("public", "snapshot", 1L)),
java.util.Map.of());
PreparedSourceRuntime prepared = Mockito.mock(PreparedSourceRuntime.class);
Mockito.when(prepared.definition()).thenReturn(definition);
Mockito.doAnswer(invocation -> {
DatacenterSource candidate = invocation.getArgument(0);
Assert.assertEquals(Integer.valueOf(DatacenterSourceStatus.DRAFT.code()),
candidate.getStatus());
Assert.assertNull(candidate.getDefinitionChecksum());
return prepared;
}).when(fixture.runtime).prepareCandidate(ArgumentMatchers.any());
fixture.service.enable(disabled.getId(), account());
InOrder order = Mockito.inOrder(fixture.runtime, fixture.sourceMapper);
order.verify(fixture.runtime).probeCandidate(ArgumentMatchers.any());
order.verify(fixture.runtime).prepareCandidate(ArgumentMatchers.any());
order.verify(fixture.sourceMapper).updateByQuery(
ArgumentMatchers.any(), ArgumentMatchers.any(QueryWrapper.class));
order.verify(fixture.runtime).commitPrepared(prepared);
}
/**
* 重新启用的 revision CAS 失败时丢弃预构建 Runtime且不得发布候选连接池。
*
* @throws Exception 测试依赖注入失败
*/
@Test
public void enableCasFailureShouldDiscardPreparedRuntime() throws Exception {
Fixture fixture = fixture();
DatacenterSource disabled = source(DatacenterSourceStatus.DISABLED, 4L, 7L);
disabled.setSourceName("orders-db");
disabled.setSourceCode("ORDERS_DB");
disabled.setDatabaseName("orders");
disabled.setSchemaName("public");
disabled.setCredentialCipher("encrypted");
disabled.setAdapterId("jdbc");
Mockito.when(fixture.sourceMapper.selectOneByQuery(ArgumentMatchers.any()))
.thenReturn(disabled);
Mockito.when(fixture.sourceMapper.updateByQuery(
ArgumentMatchers.any(), ArgumentMatchers.any())).thenReturn(0);
RuntimeFingerprint fingerprint = new RuntimeFingerprint(
"MySQL", "8.0", "MySQL Connector/J", "9.0", "test");
Mockito.when(fixture.runtime.probeCandidate(ArgumentMatchers.any()))
.thenReturn(new SourceProbeResult(
new SourceId("tenant-1-source-100"), true, fingerprint, "ok"));
FederationSourceDefinition definition = new FederationSourceDefinition(
new SourceId("tenant-1-source-100"),
5L,
"jdbc",
List.of(new ExternalSchemaDefinition("public", "snapshot", 1L)),
java.util.Map.of());
PreparedSourceRuntime prepared = Mockito.mock(PreparedSourceRuntime.class);
Mockito.when(prepared.definition()).thenReturn(definition);
Mockito.doAnswer(invocation -> {
DatacenterSource candidate = invocation.getArgument(0);
Assert.assertEquals(Integer.valueOf(DatacenterSourceStatus.DRAFT.code()),
candidate.getStatus());
Assert.assertNull(candidate.getDefinitionChecksum());
return prepared;
}).when(fixture.runtime).prepareCandidate(ArgumentMatchers.any());
try {
fixture.service.enable(disabled.getId(), account());
Assert.fail("stale revision must reject enable");
} catch (tech.easyflow.common.web.exceptions.BusinessException expected) {
Assert.assertNotNull(expected.getMessage());
}
Mockito.verify(prepared).close();
Mockito.verify(fixture.runtime, Mockito.never()).commitPrepared(prepared);
Mockito.verify(fixture.notifier, Mockito.never())
.publishAfterCommit(ArgumentMatchers.any());
}
private Fixture fixture() throws Exception {
DatacenterConnectorRegistry connectorRegistry =
Mockito.mock(DatacenterConnectorRegistry.class);
DatacenterDatasetRegistryService registry =
Mockito.mock(DatacenterDatasetRegistryService.class);
DatacenterCredentialCipher cipher = Mockito.mock(DatacenterCredentialCipher.class);
DatacenterCatalogMapper catalogMapper = Mockito.mock(DatacenterCatalogMapper.class);
DatacenterTableMapper tableMapper = Mockito.mock(DatacenterTableMapper.class);
DatacenterSourceConnectionDefaults defaults =
Mockito.mock(DatacenterSourceConnectionDefaults.class);
DatacenterFederationRuntime runtime = Mockito.mock(DatacenterFederationRuntime.class);
Mockito.when(catalogMapper.selectListByQuery(ArgumentMatchers.any()))
.thenReturn(List.of());
Mockito.when(runtime.definitions())
.thenReturn(new DatacenterFederationDefinitionFactory(catalogMapper));
DatacenterFederationChangeNotifier notifier =
Mockito.mock(DatacenterFederationChangeNotifier.class);
DatacenterSourceMapper sourceMapper = Mockito.mock(DatacenterSourceMapper.class);
DatacenterSourceMetadataService metadataService =
new DatacenterSourceMetadataService(
connectorRegistry, registry, catalogMapper);
DatacenterSourceCandidateService candidateService =
new DatacenterSourceCandidateService(cipher, defaults, runtime);
PlatformTransactionManager transactionManager =
Mockito.mock(PlatformTransactionManager.class);
Mockito.when(transactionManager.getTransaction(
ArgumentMatchers.any(TransactionDefinition.class)))
.thenReturn(Mockito.mock(TransactionStatus.class));
DatacenterSourceMetadataRefreshService metadataRefreshService =
new DatacenterSourceMetadataRefreshService(
connectorRegistry, registry, catalogMapper, tableMapper,
sourceMapper, notifier, transactionManager);
DatacenterSourceServiceImpl service = new DatacenterSourceServiceImpl(
connectorRegistry, registry, catalogMapper, tableMapper,
defaults, candidateService, metadataService,
metadataRefreshService, runtime, notifier, transactionManager);
Field mapper = ServiceImpl.class.getDeclaredField("mapper");
mapper.setAccessible(true);
mapper.set(service, sourceMapper);
return new Fixture(
service, connectorRegistry, registry, catalogMapper,
sourceMapper, runtime, notifier, defaults);
}
private DatacenterSource source(
DatacenterSourceStatus status,
long definitionRevision,
long scopeRevision) {
DatacenterSource source = new DatacenterSource();
source.setId(BigInteger.valueOf(100L));
source.setTenantId(BigInteger.ONE);
source.setSourceType("MYSQL");
source.setStatus(status.code());
source.setDefinitionRevision(definitionRevision);
source.setScopeRevision(scopeRevision);
return source;
}
private LoginAccount account() {
LoginAccount account = new LoginAccount();
account.setId(BigInteger.TEN);
account.setTenantId(BigInteger.ONE);
return account;
}
private record Fixture(
DatacenterSourceServiceImpl service,
DatacenterConnectorRegistry connectorRegistry,
DatacenterDatasetRegistryService registry,
DatacenterCatalogMapper catalogMapper,
DatacenterSourceMapper sourceMapper,
DatacenterFederationRuntime runtime,
DatacenterFederationChangeNotifier notifier,
DatacenterSourceConnectionDefaults defaults) {
}
}

View File

@@ -0,0 +1,53 @@
package tech.easyflow.datacenter.meta.support;
import java.util.Map;
import org.junit.Assert;
import org.junit.Test;
import tech.easyflow.datacenter.meta.entity.DatacenterSource;
/**
* {@link DatacenterSourceConnectionDefaults} 安全连接默认值回归测试。
*/
public class DatacenterSourceConnectionDefaultsTest {
private final DatacenterSourceConnectionDefaults defaults =
new DatacenterSourceConnectionDefaults();
/**
* 验证可能承载凭据的配置项不会进入持久化配置或 Definition。
*/
@Test
public void shouldRemoveSensitiveOptions() {
Map<String, Object> sanitized = defaults.sanitizeConfig(Map.of(
"socketTimeout", 30,
"password", "plain",
"apiToken", "token",
"clientSecret", "secret",
"privateKey", "key"));
Assert.assertEquals(Map.of("socketTimeout", 30), sanitized);
}
/**
* 验证首批 MySQL 与 PostgreSQL URL 默认具有有界连接和查询等待。
*/
@Test
public void shouldGenerateBoundedJdbcUrls() {
DatacenterSource mysql = source("MYSQL", 3306, "easyflow");
DatacenterSource postgres = source("POSTGRESQL", 5432, "harmony_adapter");
Assert.assertTrue(defaults.buildJdbcUrl(mysql).contains("connectTimeout=5000"));
Assert.assertTrue(defaults.buildJdbcUrl(mysql).contains("socketTimeout=35000"));
Assert.assertTrue(defaults.buildJdbcUrl(postgres).contains("connectTimeout=5"));
Assert.assertTrue(defaults.buildJdbcUrl(postgres).contains("socketTimeout=35"));
}
private DatacenterSource source(String type, int port, String databaseName) {
DatacenterSource source = new DatacenterSource();
source.setSourceType(type);
source.setHost("127.0.0.1");
source.setPort(port);
source.setDatabaseName(databaseName);
return source;
}
}

View File

@@ -0,0 +1,45 @@
package tech.easyflow.datacenter.security;
import org.junit.Assert;
import org.junit.Test;
/**
* {@link DatacenterCredentialCipher} 认证加密回归测试。
*/
public class DatacenterCredentialCipherTest {
/**
* 验证随机 IV、正确解密和篡改检测。
*/
@Test
public void shouldEncryptWithRandomIvAndRejectTampering() {
DatacenterCredentialCipher cipher = new DatacenterCredentialCipher(
"test-only-master-key-with-at-least-32-characters");
String first = cipher.encrypt("secret-value");
String second = cipher.encrypt("secret-value");
Assert.assertNotEquals(first, second);
Assert.assertEquals("secret-value", cipher.decrypt(first));
String tampered = first.substring(0, first.length() - 2) + "AA";
try {
cipher.decrypt(tampered);
Assert.fail("tampered credential must be rejected");
} catch (IllegalStateException expected) {
Assert.assertTrue(expected.getMessage().contains("decrypt"));
}
}
/**
* 验证部署未提供足够强度的主密钥时立即停止启动。
*/
@Test
public void shouldRejectMissingMasterKey() {
try {
new DatacenterCredentialCipher("too-short");
Assert.fail("short master key must be rejected");
} catch (IllegalStateException expected) {
Assert.assertTrue(expected.getMessage().contains("32"));
}
}
}

View File

@@ -2,6 +2,7 @@ package tech.easyflow.datacenter.utils;
import org.junit.Assert;
import org.junit.Test;
import tech.easyflow.common.web.exceptions.BusinessException;
import java.util.List;
@@ -24,7 +25,9 @@ public class SqlSupportUtilsTest {
List.of(new SqlSupportUtils.ManagedTable(
"ama 实验基线模型预算",
"Sheet1",
"tb_excel_budget")));
"tb_excel_budget",
List.of("token", "token_1", "col_id"),
List.of("token", "token_1", "col_id"))));
Assert.assertEquals(
"SELECT `token`, `token_1` FROM tb_excel_budget "
@@ -42,10 +45,564 @@ public class SqlSupportUtilsTest {
List.of(new SqlSupportUtils.ManagedTable(
null,
"Sheet1",
"tb_excel_budget")));
"tb_excel_budget",
List.of("token", "col_id"),
List.of("token", "col_id"))));
Assert.assertEquals(
"SELECT `token` FROM tb_excel_budget WHERE `col_id` = 'deep\"seek'",
resolved.getExecutableSql());
}
/**
* 验证 PostgreSQL 引号标识符不会与未引号的小写对象混为同一张表。
*/
@Test(expected = BusinessException.class)
public void resolveShouldNotFoldQuotedManagedTableToUnquotedReference() {
SqlSupportUtils.resolve(
"SELECT * FROM foo",
List.of(new SqlSupportUtils.ManagedTable(
null,
"Foo",
"Foo")));
}
/**
* 验证大小写敏感表可通过精确引号标识符访问。
*/
@Test
public void resolveShouldMatchQuotedManagedTableExactly() {
SqlSupportUtils.ResolvedSql resolved = SqlSupportUtils.resolve(
"SELECT * FROM \"Foo\"",
List.of(new SqlSupportUtils.ManagedTable(
null,
"Foo",
"Foo")));
Assert.assertEquals(List.of("Foo"), resolved.getLogicalTables());
}
/**
* 验证内部 SQL 无法读取未开放字段。
*/
@Test(expected = BusinessException.class)
public void resolveInternalMysqlShouldRejectHiddenColumn() {
SqlSupportUtils.resolveInternalMysql(
"SELECT secret FROM Sheet1",
List.of(internalTable(
List.of("public_col", "secret"),
List.of("public_col"))));
}
/**
* 验证 IS NULL 条件中的隐藏字段仍受字段策略约束。
*/
@Test(expected = BusinessException.class)
public void resolveInternalMysqlShouldRejectHiddenColumnInIsNull() {
SqlSupportUtils.resolveInternalMysql(
"SELECT public_col FROM \"Sheet1\" WHERE secret IS NULL",
List.of(internalTable(
List.of("public_col", "secret"),
List.of("public_col"))));
}
/**
* 验证排序条件中的隐藏字段仍受字段策略约束。
*/
@Test(expected = BusinessException.class)
public void resolveInternalMysqlShouldRejectHiddenColumnInOrderBy() {
SqlSupportUtils.resolveInternalMysql(
"SELECT public_col FROM \"Sheet1\" ORDER BY secret",
List.of(internalTable(
List.of("public_col", "secret"),
List.of("public_col"))));
}
/**
* 验证分组条件中的隐藏字段仍受字段策略约束。
*/
@Test(expected = BusinessException.class)
public void resolveInternalMysqlShouldRejectHiddenColumnInGroupBy() {
SqlSupportUtils.resolveInternalMysql(
"SELECT COUNT(*) FROM \"Sheet1\" GROUP BY secret",
List.of(internalTable(
List.of("public_col", "secret"),
List.of("public_col"))));
}
/**
* 验证窗口排序中的隐藏字段仍受字段策略约束。
*/
@Test(expected = BusinessException.class)
public void resolveInternalMysqlShouldRejectHiddenColumnInWindowOrderBy() {
SqlSupportUtils.resolveInternalMysql(
"SELECT public_col, ROW_NUMBER() OVER (ORDER BY secret) AS rn "
+ "FROM \"Sheet1\"",
List.of(internalTable(
List.of("public_col", "secret"),
List.of("public_col"))));
}
/**
* 验证命名窗口排序中的隐藏字段仍受字段策略约束。
*/
@Test(expected = BusinessException.class)
public void resolveInternalMysqlShouldRejectHiddenColumnInNamedWindow() {
SqlSupportUtils.resolveInternalMysql(
"SELECT public_col, ROW_NUMBER() OVER w AS rn FROM \"Sheet1\" "
+ "WINDOW w AS (ORDER BY secret)",
List.of(internalTable(
List.of("public_col", "secret"),
List.of("public_col"))));
}
/**
* 验证合法窗口 frame 可以安全解析且不会触发空值异常。
*/
@Test
public void resolveInternalMysqlShouldAllowSafeWindowFrame() {
SqlSupportUtils.ResolvedSql resolved = SqlSupportUtils.resolveInternalMysql(
"SELECT SUM(public_col) OVER (ORDER BY public_col "
+ "ROWS BETWEEN 1 PRECEDING AND CURRENT ROW) AS total "
+ "FROM \"Sheet1\"",
List.of(internalTable(
List.of("public_col", "secret"),
List.of("public_col"))));
Assert.assertEquals(
"SELECT SUM(public_col) OVER (ORDER BY public_col ROWS BETWEEN 1 PRECEDING "
+ "AND CURRENT ROW) AS total FROM tb_excel_budget",
resolved.getExecutableSql());
}
/**
* 验证 HAVING 中包装的合法投影别名不会被误判为物理字段。
*/
@Test
public void resolveInternalMysqlShouldAllowOutputAliasInHavingExpression() {
SqlSupportUtils.ResolvedSql resolved = SqlSupportUtils.resolveInternalMysql(
"SELECT public_col AS visible FROM \"Sheet1\" "
+ "HAVING visible IS NOT NULL",
List.of(internalTable(
List.of("public_col", "secret"),
List.of("public_col"))));
Assert.assertEquals(
"SELECT public_col AS visible FROM tb_excel_budget "
+ "HAVING visible IS NOT NULL",
resolved.getExecutableSql());
}
/**
* 验证 ORDER BY 包装表达式中的合法投影别名可以使用。
*/
@Test
public void resolveInternalMysqlShouldAllowOutputAliasInWrappedOrderBy() {
SqlSupportUtils.ResolvedSql resolved = SqlSupportUtils.resolveInternalMysql(
"SELECT public_col AS visible FROM \"Sheet1\" "
+ "ORDER BY COALESCE(visible, '')",
List.of(internalTable(
List.of("public_col", "secret"),
List.of("public_col"))));
Assert.assertEquals(
"SELECT public_col AS visible FROM tb_excel_budget "
+ "ORDER BY COALESCE(visible, '')",
resolved.getExecutableSql());
}
/**
* 验证包装排序表达式不能放行隐藏物理字段。
*/
@Test(expected = BusinessException.class)
public void resolveInternalMysqlShouldRejectHiddenColumnInWrappedOrderBy() {
SqlSupportUtils.resolveInternalMysql(
"SELECT public_col FROM \"Sheet1\" ORDER BY COALESCE(secret, '')",
List.of(internalTable(
List.of("public_col", "secret"),
List.of("public_col"))));
}
/**
* 验证 IS TRUE 条件中的隐藏字段仍受字段策略约束。
*/
@Test(expected = BusinessException.class)
public void resolveInternalMysqlShouldRejectHiddenColumnInIsBoolean() {
SqlSupportUtils.resolveInternalMysql(
"SELECT public_col FROM \"Sheet1\" WHERE secret IS TRUE",
List.of(internalTable(
List.of("public_col", "secret"),
List.of("public_col"))));
}
/**
* 验证 QUALIFY 条件中的隐藏字段仍受字段策略约束。
*/
@Test(expected = BusinessException.class)
public void resolveInternalMysqlShouldRejectHiddenColumnInQualify() {
SqlSupportUtils.resolveInternalMysql(
"SELECT public_col FROM \"Sheet1\" QUALIFY secret IS NOT NULL",
List.of(internalTable(
List.of("public_col", "secret"),
List.of("public_col"))));
}
/**
* 验证隐藏字段不能通过同名输出别名绕过字段白名单。
*/
@Test(expected = BusinessException.class)
public void resolveInternalMysqlShouldRejectHiddenColumnWithSameAlias() {
SqlSupportUtils.resolveInternalMysql(
"SELECT secret AS secret FROM \"Sheet1\"",
List.of(internalTable(
List.of("public_col", "secret"),
List.of("public_col"))));
}
/**
* 验证存在未开放字段时禁止星号扩展。
*/
@Test(expected = BusinessException.class)
public void resolveInternalMysqlShouldRejectStarWithHiddenColumns() {
SqlSupportUtils.resolveInternalMysql(
"SELECT * FROM Sheet1",
List.of(internalTable(
List.of("public_col", "secret"),
List.of("public_col"))));
}
/**
* 验证括号包裹的 JOIN 仍会将星号绑定到底层物理表。
*/
@Test(expected = BusinessException.class)
public void resolveInternalMysqlShouldRejectStarOverParenthesizedJoinWithHiddenColumn() {
SqlSupportUtils.resolveInternalMysql(
"SELECT * FROM (Sheet1 CROSS JOIN Sheet2)",
List.of(
internalTable(
List.of("public_col", "secret"),
List.of("public_col")),
new SqlSupportUtils.ManagedTable(
null,
"Sheet2",
"tb_excel_public",
List.of("other_col"),
List.of("other_col"))));
}
/**
* 验证聚合函数中的星号不代表读取全部物理字段。
*/
@Test
public void resolveInternalMysqlShouldAllowCountStarWithHiddenColumns() {
SqlSupportUtils.ResolvedSql resolved = SqlSupportUtils.resolveInternalMysql(
"SELECT COUNT(*) FROM \"Sheet1\"",
List.of(internalTable(
List.of("public_col", "secret"),
List.of("public_col"))));
Assert.assertEquals(
"SELECT COUNT(*) FROM tb_excel_budget",
resolved.getExecutableSql());
}
/**
* 验证表别名和输出别名不会被误判为物理字段。
*/
@Test
public void resolveInternalMysqlShouldAllowSafeAliases() {
SqlSupportUtils.ResolvedSql resolved = SqlSupportUtils.resolveInternalMysql(
"SELECT s.public_col AS display_col FROM \"Sheet1\" s ORDER BY display_col",
List.of(internalTable(
List.of("public_col"),
List.of("public_col"))));
Assert.assertEquals(
"SELECT s.public_col AS display_col FROM tb_excel_budget s "
+ "ORDER BY display_col",
resolved.getExecutableSql());
}
/**
* 验证内层子查询的同名字段不会污染外层合法输出别名。
*/
@Test
public void resolveInternalMysqlShouldScopeOutputAliasToCurrentQueryBlock() {
SqlSupportUtils.ResolvedSql resolved = SqlSupportUtils.resolveInternalMysql(
"SELECT public_col AS secret FROM \"Sheet1\" "
+ "WHERE EXISTS (SELECT 1 FROM \"Sheet2\") ORDER BY secret",
List.of(
internalTable(
List.of("public_col"),
List.of("public_col")),
new SqlSupportUtils.ManagedTable(
null,
"Sheet2",
"tb_excel_private",
List.of("secret"),
List.of())));
Assert.assertEquals(
"SELECT public_col AS secret FROM tb_excel_budget WHERE EXISTS "
+ "(SELECT 1 FROM tb_excel_private) ORDER BY secret",
resolved.getExecutableSql());
}
/**
* 验证内层子查询的物理表不会参与外层未限定字段授权判断。
*/
@Test
public void resolveInternalMysqlShouldScopeUnqualifiedColumnToCurrentQueryBlock() {
SqlSupportUtils.ResolvedSql resolved = SqlSupportUtils.resolveInternalMysql(
"SELECT public_col FROM \"Sheet1\" "
+ "WHERE EXISTS (SELECT 1 FROM \"Sheet2\")",
List.of(
internalTable(
List.of("public_col"),
List.of("public_col")),
new SqlSupportUtils.ManagedTable(
null,
"Sheet2",
"tb_excel_private",
List.of("secret"),
List.of())));
Assert.assertEquals(
"SELECT public_col FROM tb_excel_budget WHERE EXISTS "
+ "(SELECT 1 FROM tb_excel_private)",
resolved.getExecutableSql());
}
/**
* 验证派生表输出别名可以在外层查询中使用。
*/
@Test
public void resolveInternalMysqlShouldAllowDerivedTableOutputAlias() {
SqlSupportUtils.ResolvedSql resolved = SqlSupportUtils.resolveInternalMysql(
"SELECT d.order_id FROM (SELECT public_col AS order_id FROM \"Sheet1\") d",
List.of(internalTable(
List.of("public_col", "secret"),
List.of("public_col"))));
Assert.assertEquals(
"SELECT d.order_id FROM (SELECT public_col AS order_id "
+ "FROM tb_excel_budget) d",
resolved.getExecutableSql());
}
/**
* 验证 CTE 输出别名可以在主查询中使用。
*/
@Test
public void resolveInternalMysqlShouldAllowCteOutputAlias() {
SqlSupportUtils.ResolvedSql resolved = SqlSupportUtils.resolveInternalMysql(
"WITH x AS (SELECT public_col AS order_id FROM \"Sheet1\") SELECT order_id FROM x",
List.of(internalTable(
List.of("public_col", "secret"),
List.of("public_col"))));
Assert.assertEquals(
"WITH x AS (SELECT public_col AS order_id FROM tb_excel_budget) "
+ "SELECT order_id FROM x",
resolved.getExecutableSql());
}
/**
* 验证外层星号只展开 CTE 已校验的输出字段。
*/
@Test
public void resolveInternalMysqlShouldAllowStarOverCteOutput() {
SqlSupportUtils.ResolvedSql resolved = SqlSupportUtils.resolveInternalMysql(
"WITH x AS (SELECT public_col FROM \"Sheet1\") SELECT * FROM x",
List.of(internalTable(
List.of("public_col", "secret"),
List.of("public_col"))));
Assert.assertEquals(
"WITH x AS (SELECT public_col FROM tb_excel_budget) SELECT * FROM x",
resolved.getExecutableSql());
}
/**
* 验证带 Schema 的物理表不能伪装成同名 CTE 绕过表白名单。
*/
@Test(expected = BusinessException.class)
public void resolveInternalMysqlShouldRejectSchemaQualifiedTableShadowingCte() {
SqlSupportUtils.resolveInternalMysql(
"WITH x AS (SELECT public_col AS secret FROM \"Sheet1\") "
+ "SELECT x.secret FROM private_db.x",
List.of(internalTable(
List.of("public_col", "secret"),
List.of("public_col"))));
}
/**
* 验证后置 CTE 可以引用同一 WITH 列表中的前置 CTE。
*/
@Test
public void resolveInternalMysqlShouldAllowChainedCtes() {
SqlSupportUtils.ResolvedSql resolved = SqlSupportUtils.resolveInternalMysql(
"WITH x AS (SELECT public_col AS a FROM \"Sheet1\"), "
+ "y AS (SELECT a AS b FROM x) SELECT b FROM y",
List.of(internalTable(
List.of("public_col", "secret"),
List.of("public_col"))));
Assert.assertEquals(
"WITH x AS (SELECT public_col AS a FROM tb_excel_budget), "
+ "y AS (SELECT a AS b FROM x) SELECT b FROM y",
resolved.getExecutableSql());
}
/**
* 验证 CTE 显式列清单可以作为主查询输出字段使用。
*/
@Test
public void resolveInternalMysqlShouldAllowExplicitCteColumnList() {
SqlSupportUtils.ResolvedSql resolved = SqlSupportUtils.resolveInternalMysql(
"WITH x(order_id) AS (SELECT public_col FROM \"Sheet1\") SELECT order_id FROM x",
List.of(internalTable(
List.of("public_col", "secret"),
List.of("public_col"))));
Assert.assertEquals(
"WITH x (order_id) AS (SELECT public_col FROM tb_excel_budget) "
+ "SELECT order_id FROM x",
resolved.getExecutableSql());
}
/**
* 验证派生表别名不能洗白内部不可查询字段。
*/
@Test(expected = BusinessException.class)
public void resolveInternalMysqlShouldRejectHiddenColumnInsideDerivedTable() {
SqlSupportUtils.resolveInternalMysql(
"SELECT d.order_id FROM (SELECT secret AS order_id FROM \"Sheet1\") d",
List.of(internalTable(
List.of("public_col", "secret"),
List.of("public_col"))));
}
/**
* 验证 CTE 输出别名不能洗白内部不可查询字段。
*/
@Test(expected = BusinessException.class)
public void resolveInternalMysqlShouldRejectHiddenColumnInsideCte() {
SqlSupportUtils.resolveInternalMysql(
"WITH x AS (SELECT secret AS order_id FROM \"Sheet1\") SELECT order_id FROM x",
List.of(internalTable(
List.of("public_col", "secret"),
List.of("public_col"))));
}
/**
* 验证外层派生表别名不会污染内层同名物理表别名。
*/
@Test(expected = BusinessException.class)
public void resolveInternalMysqlShouldNotLeakDerivedAliasAcrossNestedScope() {
SqlSupportUtils.resolveInternalMysql(
"SELECT d.secret FROM (SELECT public_col AS secret FROM \"Sheet1\") d "
+ "WHERE EXISTS (SELECT 1 FROM \"Sheet1\" d WHERE d.secret = 1)",
List.of(internalTable(
List.of("public_col", "secret"),
List.of("public_col"))));
}
/**
* 验证相关子查询可以引用外层派生关系的已校验输出字段。
*/
@Test
public void resolveInternalMysqlShouldAllowCorrelatedOuterDerivedColumn() {
SqlSupportUtils.ResolvedSql resolved = SqlSupportUtils.resolveInternalMysql(
"SELECT d.order_id FROM (SELECT public_col AS order_id FROM \"Sheet1\") d "
+ "WHERE EXISTS (SELECT 1 WHERE d.order_id IS NOT NULL)",
List.of(internalTable(
List.of("public_col", "secret"),
List.of("public_col"))));
Assert.assertEquals(
"SELECT d.order_id FROM (SELECT public_col AS order_id "
+ "FROM tb_excel_budget) d WHERE EXISTS "
+ "(SELECT 1 WHERE d.order_id IS NOT NULL)",
resolved.getExecutableSql());
}
/**
* 验证内层物理表别名会遮蔽外层派生关系,不能借外层输出绕过字段策略。
*/
@Test(expected = BusinessException.class)
public void resolveInternalMysqlShouldRejectCorrelatedDerivedAliasShadowedByInnerPhysicalAlias() {
SqlSupportUtils.resolveInternalMysql(
"SELECT d.order_id FROM (SELECT public_col AS order_id FROM \"Sheet1\") d "
+ "WHERE EXISTS (SELECT 1 FROM \"Sheet1\" d "
+ "WHERE d.order_id IS NOT NULL)",
List.of(internalTable(
List.of("public_col", "secret"),
List.of("public_col"))));
}
/**
* 验证不同查询块的同名物理别名不会覆盖彼此的字段授权范围。
*/
@Test(expected = BusinessException.class)
public void resolveInternalMysqlShouldIsolatePhysicalAliasesAcrossScopes() {
SqlSupportUtils.resolveInternalMysql(
"SELECT x.secret FROM \"Sheet1\" x WHERE EXISTS "
+ "(SELECT 1 FROM \"Sheet2\" x WHERE x.secret IS NOT NULL)",
List.of(
new SqlSupportUtils.ManagedTable(
null,
"Sheet1",
"tb_excel_budget",
List.of("public_col", "secret"),
List.of("public_col")),
new SqlSupportUtils.ManagedTable(
null,
"Sheet2",
"tb_excel_public",
List.of("secret"),
List.of("secret"))));
}
/**
* 验证外层输出别名不能污染内层作用域并放行隐藏字段。
*/
@Test(expected = BusinessException.class)
public void resolveInternalMysqlShouldRejectNestedHiddenColumnShadowedByOuterAlias() {
SqlSupportUtils.resolveInternalMysql(
"SELECT public_col AS secret FROM Sheet1 "
+ "WHERE public_col = (SELECT public_col FROM Sheet1 "
+ "ORDER BY secret LIMIT 1)",
List.of(internalTable(
List.of("public_col", "secret"),
List.of("public_col"))));
}
/**
* 验证内部 SQL 拒绝具有等待或资源消耗风险的函数。
*/
@Test(expected = BusinessException.class)
public void resolveInternalMysqlShouldRejectDangerousFunction() {
SqlSupportUtils.resolveInternalMysql(
"SELECT SLEEP(10) FROM Sheet1",
List.of(internalTable(List.of("public_col"), List.of("public_col"))));
}
/**
* 创建内部动态表测试定义。
*
* @param knownColumns 已纳管字段
* @param queryableColumns 可查询字段
* @return 测试表定义
*/
private SqlSupportUtils.ManagedTable internalTable(
List<String> knownColumns,
List<String> queryableColumns) {
return new SqlSupportUtils.ManagedTable(
null,
"Sheet1",
"tb_excel_budget",
knownColumns,
queryableColumns);
}
}