feat: 重构数据中枢工作台与接入管理

- 新增统一的数据源、目录、纳管表与 Excel 处理后端能力

- 重建管理端数据中枢工作台并替换旧表管理页面

- 补充数据中枢迁移脚本、连接器底座与说明字段支持
This commit is contained in:
2026-04-02 18:55:31 +08:00
parent b6213d0933
commit 798effbd5b
117 changed files with 9739 additions and 1824 deletions

View File

@@ -16,6 +16,19 @@
<groupId>com.mybatis-flex</groupId>
<artifactId>mybatis-flex-spring-boot3-starter</artifactId>
</dependency>
<dependency>
<groupId>com.zaxxer</groupId>
<artifactId>HikariCP</artifactId>
</dependency>
<dependency>
<groupId>org.postgresql</groupId>
<artifactId>postgresql</artifactId>
<version>42.7.3</version>
</dependency>
<dependency>
<groupId>cn.hutool</groupId>
<artifactId>hutool-crypto</artifactId>
</dependency>
<dependency>
<groupId>tech.easyflow</groupId>
<artifactId>easyflow-common-base</artifactId>

View File

@@ -0,0 +1,12 @@
package tech.easyflow.datacenter.connector;
import tech.easyflow.datacenter.meta.enums.DatacenterCapability;
import tech.easyflow.datacenter.meta.enums.DatacenterSourceType;
import java.util.Set;
public interface DatacenterConnector extends SourceHealthChecker, MetadataExplorer, QueryExecutor, WriteExecutor {
DatacenterSourceType getSourceType();
Set<DatacenterCapability> getCapabilities();
}

View File

@@ -0,0 +1,29 @@
package tech.easyflow.datacenter.connector;
import org.springframework.stereotype.Component;
import tech.easyflow.common.web.exceptions.BusinessException;
import tech.easyflow.datacenter.meta.enums.DatacenterSourceType;
import java.util.List;
import java.util.Map;
import java.util.function.Function;
import java.util.stream.Collectors;
@Component
public class DatacenterConnectorRegistry {
private final Map<DatacenterSourceType, DatacenterConnector> connectorMap;
public DatacenterConnectorRegistry(List<DatacenterConnector> connectors) {
this.connectorMap = connectors.stream().collect(Collectors.toMap(DatacenterConnector::getSourceType, Function.identity()));
}
public DatacenterConnector getConnector(String sourceType) {
DatacenterSourceType type = DatacenterSourceType.valueOf(sourceType);
DatacenterConnector connector = connectorMap.get(type);
if (connector == null) {
throw new BusinessException("未找到连接器: " + sourceType);
}
return connector;
}
}

View File

@@ -0,0 +1,16 @@
package tech.easyflow.datacenter.connector;
import tech.easyflow.datacenter.entity.DatacenterTable;
import tech.easyflow.datacenter.meta.entity.DatacenterSource;
import tech.easyflow.datacenter.meta.model.DatacenterCatalogMeta;
import tech.easyflow.datacenter.meta.model.DatacenterTableDetailMeta;
import java.util.List;
public interface MetadataExplorer {
List<DatacenterCatalogMeta> listCatalogs(DatacenterSource source);
List<DatacenterTable> listTables(DatacenterSource source, String catalogName);
DatacenterTableDetailMeta getTableDetail(DatacenterSource source, String catalogName, String tableName);
}

View File

@@ -0,0 +1,15 @@
package tech.easyflow.datacenter.connector;
import com.mybatisflex.core.paginate.Page;
import com.mybatisflex.core.row.Row;
import tech.easyflow.datacenter.entity.DatacenterTable;
import tech.easyflow.datacenter.execution.model.DatacenterQueryRequest;
import tech.easyflow.datacenter.meta.entity.DatacenterSource;
import java.util.List;
public interface QueryExecutor {
Page<Row> queryPage(DatacenterSource source, DatacenterTable table, DatacenterQueryRequest request);
List<Row> queryBySql(DatacenterSource source, String sql);
}

View File

@@ -0,0 +1,8 @@
package tech.easyflow.datacenter.connector;
import tech.easyflow.datacenter.execution.model.DatacenterConnectionTestResult;
import tech.easyflow.datacenter.meta.entity.DatacenterSource;
public interface SourceHealthChecker {
DatacenterConnectionTestResult testConnection(DatacenterSource source);
}

View File

@@ -0,0 +1,9 @@
package tech.easyflow.datacenter.connector;
public interface SqlDialect {
String quoteIdentifier(String identifier);
String qualifyTable(String namespace, String tableName);
String buildPageSql(String baseSql, long pageNumber, long pageSize);
}

View File

@@ -0,0 +1,14 @@
package tech.easyflow.datacenter.connector;
import com.alibaba.fastjson2.JSONObject;
import tech.easyflow.common.entity.LoginAccount;
import tech.easyflow.datacenter.entity.DatacenterTable;
import tech.easyflow.datacenter.meta.entity.DatacenterSource;
import java.math.BigInteger;
public interface WriteExecutor {
void saveRow(DatacenterSource source, DatacenterTable table, JSONObject data, LoginAccount account);
void deleteRow(DatacenterSource source, DatacenterTable table, BigInteger id, LoginAccount account);
}

View File

@@ -0,0 +1,24 @@
package tech.easyflow.datacenter.connector.dialect;
import cn.hutool.core.util.StrUtil;
import tech.easyflow.datacenter.connector.SqlDialect;
public class GaussdbSqlDialect implements SqlDialect {
@Override
public String quoteIdentifier(String identifier) {
return '"' + identifier + '"';
}
@Override
public String qualifyTable(String namespace, String tableName) {
if (StrUtil.isBlank(namespace)) {
return quoteIdentifier(tableName);
}
return quoteIdentifier(namespace) + "." + quoteIdentifier(tableName);
}
@Override
public String buildPageSql(String baseSql, long pageNumber, long pageSize) {
return baseSql + " OFFSET ? LIMIT ?";
}
}

View File

@@ -0,0 +1,24 @@
package tech.easyflow.datacenter.connector.dialect;
import cn.hutool.core.util.StrUtil;
import tech.easyflow.datacenter.connector.SqlDialect;
public class MysqlSqlDialect implements SqlDialect {
@Override
public String quoteIdentifier(String identifier) {
return "`" + identifier + "`";
}
@Override
public String qualifyTable(String namespace, String tableName) {
if (StrUtil.isBlank(namespace)) {
return quoteIdentifier(tableName);
}
return quoteIdentifier(namespace) + "." + quoteIdentifier(tableName);
}
@Override
public String buildPageSql(String baseSql, long pageNumber, long pageSize) {
return baseSql + " LIMIT ?, ?";
}
}

View File

@@ -0,0 +1,24 @@
package tech.easyflow.datacenter.connector.dialect;
import cn.hutool.core.util.StrUtil;
import tech.easyflow.datacenter.connector.SqlDialect;
public class OracleSqlDialect implements SqlDialect {
@Override
public String quoteIdentifier(String identifier) {
return '"' + identifier + '"';
}
@Override
public String qualifyTable(String namespace, String tableName) {
if (StrUtil.isBlank(namespace)) {
return quoteIdentifier(tableName);
}
return quoteIdentifier(namespace) + "." + quoteIdentifier(tableName);
}
@Override
public String buildPageSql(String baseSql, long pageNumber, long pageSize) {
return baseSql + " OFFSET ? ROWS FETCH NEXT ? ROWS ONLY";
}
}

View File

@@ -0,0 +1,24 @@
package tech.easyflow.datacenter.connector.dialect;
import cn.hutool.core.util.StrUtil;
import tech.easyflow.datacenter.connector.SqlDialect;
public class PostgresqlSqlDialect implements SqlDialect {
@Override
public String quoteIdentifier(String identifier) {
return '"' + identifier + '"';
}
@Override
public String qualifyTable(String namespace, String tableName) {
if (StrUtil.isBlank(namespace)) {
return quoteIdentifier(tableName);
}
return quoteIdentifier(namespace) + "." + quoteIdentifier(tableName);
}
@Override
public String buildPageSql(String baseSql, long pageNumber, long pageSize) {
return baseSql + " OFFSET ? LIMIT ?";
}
}

View File

@@ -0,0 +1,22 @@
package tech.easyflow.datacenter.connector.impl;
import org.springframework.stereotype.Component;
import tech.easyflow.datacenter.connector.support.AbstractInternalTableConnector;
import tech.easyflow.datacenter.meta.enums.DatacenterCapability;
import tech.easyflow.datacenter.meta.enums.DatacenterSourceType;
import java.util.EnumSet;
@Component
public class ExcelConnector extends AbstractInternalTableConnector {
public ExcelConnector() {
super(DatacenterSourceType.EXCEL, EnumSet.of(
DatacenterCapability.TEST_CONNECTION,
DatacenterCapability.BROWSE_METADATA,
DatacenterCapability.READ_QUERY,
DatacenterCapability.WRITE_MUTATION,
DatacenterCapability.MATERIALIZE,
DatacenterCapability.EXPORT
));
}
}

View File

@@ -0,0 +1,22 @@
package tech.easyflow.datacenter.connector.impl;
import org.springframework.stereotype.Component;
import tech.easyflow.datacenter.connector.support.AbstractInternalTableConnector;
import tech.easyflow.datacenter.meta.enums.DatacenterCapability;
import tech.easyflow.datacenter.meta.enums.DatacenterSourceType;
import java.util.EnumSet;
@Component
public class ExcelMaterializedConnector extends AbstractInternalTableConnector {
public ExcelMaterializedConnector() {
super(DatacenterSourceType.EXCEL_MATERIALIZED, EnumSet.of(
DatacenterCapability.TEST_CONNECTION,
DatacenterCapability.BROWSE_METADATA,
DatacenterCapability.READ_QUERY,
DatacenterCapability.WRITE_MUTATION,
DatacenterCapability.MATERIALIZE,
DatacenterCapability.EXPORT
));
}
}

View File

@@ -0,0 +1,40 @@
package tech.easyflow.datacenter.connector.impl;
import com.zaxxer.hikari.HikariDataSource;
import org.springframework.stereotype.Component;
import tech.easyflow.datacenter.connector.dialect.GaussdbSqlDialect;
import tech.easyflow.datacenter.connector.support.AbstractJdbcConnector;
import tech.easyflow.datacenter.connector.support.DatacenterDatasourceManager;
import tech.easyflow.datacenter.meta.entity.DatacenterSource;
import tech.easyflow.datacenter.meta.enums.DatacenterCapability;
import tech.easyflow.datacenter.meta.enums.DatacenterSourceType;
import java.sql.Connection;
import java.util.EnumSet;
@Component
public class GaussdbNativeConnector extends AbstractJdbcConnector {
private final DatacenterDatasourceManager datasourceManager;
public GaussdbNativeConnector(DatacenterDatasourceManager datasourceManager) {
super(DatacenterSourceType.GAUSSDB_NATIVE, new GaussdbSqlDialect(), EnumSet.of(
DatacenterCapability.TEST_CONNECTION,
DatacenterCapability.BROWSE_METADATA,
DatacenterCapability.READ_QUERY
));
this.datasourceManager = datasourceManager;
}
@Override
protected <T> T withConnection(DatacenterSource source, boolean cacheable, JdbcCallback<T> callback) throws Exception {
HikariDataSource dataSource = cacheable ? datasourceManager.getOrCreateExternalDatasource(source) : datasourceManager.createExternalDatasource(source);
try (Connection connection = dataSource.getConnection()) {
return callback.apply(connection);
} finally {
if (!cacheable || source.getId() == null) {
dataSource.close();
}
}
}
}

View File

@@ -0,0 +1,40 @@
package tech.easyflow.datacenter.connector.impl;
import com.zaxxer.hikari.HikariDataSource;
import org.springframework.stereotype.Component;
import tech.easyflow.datacenter.connector.dialect.MysqlSqlDialect;
import tech.easyflow.datacenter.connector.support.AbstractJdbcConnector;
import tech.easyflow.datacenter.connector.support.DatacenterDatasourceManager;
import tech.easyflow.datacenter.meta.entity.DatacenterSource;
import tech.easyflow.datacenter.meta.enums.DatacenterCapability;
import tech.easyflow.datacenter.meta.enums.DatacenterSourceType;
import java.sql.Connection;
import java.util.EnumSet;
@Component
public class Gbase8aConnector extends AbstractJdbcConnector {
private final DatacenterDatasourceManager datasourceManager;
public Gbase8aConnector(DatacenterDatasourceManager datasourceManager) {
super(DatacenterSourceType.GBASE_8A, new MysqlSqlDialect(), EnumSet.of(
DatacenterCapability.TEST_CONNECTION,
DatacenterCapability.BROWSE_METADATA,
DatacenterCapability.READ_QUERY
));
this.datasourceManager = datasourceManager;
}
@Override
protected <T> T withConnection(DatacenterSource source, boolean cacheable, JdbcCallback<T> callback) throws Exception {
HikariDataSource dataSource = cacheable ? datasourceManager.getOrCreateExternalDatasource(source) : datasourceManager.createExternalDatasource(source);
try (Connection connection = dataSource.getConnection()) {
return callback.apply(connection);
} finally {
if (!cacheable || source.getId() == null) {
dataSource.close();
}
}
}
}

View File

@@ -0,0 +1,40 @@
package tech.easyflow.datacenter.connector.impl;
import com.zaxxer.hikari.HikariDataSource;
import org.springframework.stereotype.Component;
import tech.easyflow.datacenter.connector.dialect.MysqlSqlDialect;
import tech.easyflow.datacenter.connector.support.AbstractJdbcConnector;
import tech.easyflow.datacenter.connector.support.DatacenterDatasourceManager;
import tech.easyflow.datacenter.meta.entity.DatacenterSource;
import tech.easyflow.datacenter.meta.enums.DatacenterCapability;
import tech.easyflow.datacenter.meta.enums.DatacenterSourceType;
import java.sql.Connection;
import java.util.EnumSet;
@Component
public class Gbase8sConnector extends AbstractJdbcConnector {
private final DatacenterDatasourceManager datasourceManager;
public Gbase8sConnector(DatacenterDatasourceManager datasourceManager) {
super(DatacenterSourceType.GBASE_8S, new MysqlSqlDialect(), EnumSet.of(
DatacenterCapability.TEST_CONNECTION,
DatacenterCapability.BROWSE_METADATA,
DatacenterCapability.READ_QUERY
));
this.datasourceManager = datasourceManager;
}
@Override
protected <T> T withConnection(DatacenterSource source, boolean cacheable, JdbcCallback<T> callback) throws Exception {
HikariDataSource dataSource = cacheable ? datasourceManager.getOrCreateExternalDatasource(source) : datasourceManager.createExternalDatasource(source);
try (Connection connection = dataSource.getConnection()) {
return callback.apply(connection);
} finally {
if (!cacheable || source.getId() == null) {
dataSource.close();
}
}
}
}

View File

@@ -0,0 +1,40 @@
package tech.easyflow.datacenter.connector.impl;
import com.zaxxer.hikari.HikariDataSource;
import org.springframework.stereotype.Component;
import tech.easyflow.datacenter.connector.dialect.MysqlSqlDialect;
import tech.easyflow.datacenter.connector.support.AbstractJdbcConnector;
import tech.easyflow.datacenter.connector.support.DatacenterDatasourceManager;
import tech.easyflow.datacenter.meta.entity.DatacenterSource;
import tech.easyflow.datacenter.meta.enums.DatacenterCapability;
import tech.easyflow.datacenter.meta.enums.DatacenterSourceType;
import java.sql.Connection;
import java.util.EnumSet;
@Component
public class MysqlConnector extends AbstractJdbcConnector {
private final DatacenterDatasourceManager datasourceManager;
public MysqlConnector(DatacenterDatasourceManager datasourceManager) {
super(DatacenterSourceType.MYSQL, new MysqlSqlDialect(), EnumSet.of(
DatacenterCapability.TEST_CONNECTION,
DatacenterCapability.BROWSE_METADATA,
DatacenterCapability.READ_QUERY
));
this.datasourceManager = datasourceManager;
}
@Override
protected <T> T withConnection(DatacenterSource source, boolean cacheable, JdbcCallback<T> callback) throws Exception {
HikariDataSource dataSource = cacheable ? datasourceManager.getOrCreateExternalDatasource(source) : datasourceManager.createExternalDatasource(source);
try (Connection connection = dataSource.getConnection()) {
return callback.apply(connection);
} finally {
if (!cacheable || source.getId() == null) {
dataSource.close();
}
}
}
}

View File

@@ -0,0 +1,40 @@
package tech.easyflow.datacenter.connector.impl;
import com.zaxxer.hikari.HikariDataSource;
import org.springframework.stereotype.Component;
import tech.easyflow.datacenter.connector.dialect.OracleSqlDialect;
import tech.easyflow.datacenter.connector.support.AbstractJdbcConnector;
import tech.easyflow.datacenter.connector.support.DatacenterDatasourceManager;
import tech.easyflow.datacenter.meta.entity.DatacenterSource;
import tech.easyflow.datacenter.meta.enums.DatacenterCapability;
import tech.easyflow.datacenter.meta.enums.DatacenterSourceType;
import java.sql.Connection;
import java.util.EnumSet;
@Component
public class OracleConnector extends AbstractJdbcConnector {
private final DatacenterDatasourceManager datasourceManager;
public OracleConnector(DatacenterDatasourceManager datasourceManager) {
super(DatacenterSourceType.ORACLE, new OracleSqlDialect(), EnumSet.of(
DatacenterCapability.TEST_CONNECTION,
DatacenterCapability.BROWSE_METADATA,
DatacenterCapability.READ_QUERY
));
this.datasourceManager = datasourceManager;
}
@Override
protected <T> T withConnection(DatacenterSource source, boolean cacheable, JdbcCallback<T> callback) throws Exception {
HikariDataSource dataSource = cacheable ? datasourceManager.getOrCreateExternalDatasource(source) : datasourceManager.createExternalDatasource(source);
try (Connection connection = dataSource.getConnection()) {
return callback.apply(connection);
} finally {
if (!cacheable || source.getId() == null) {
dataSource.close();
}
}
}
}

View File

@@ -0,0 +1,40 @@
package tech.easyflow.datacenter.connector.impl;
import com.zaxxer.hikari.HikariDataSource;
import org.springframework.stereotype.Component;
import tech.easyflow.datacenter.connector.dialect.PostgresqlSqlDialect;
import tech.easyflow.datacenter.connector.support.AbstractJdbcConnector;
import tech.easyflow.datacenter.connector.support.DatacenterDatasourceManager;
import tech.easyflow.datacenter.meta.entity.DatacenterSource;
import tech.easyflow.datacenter.meta.enums.DatacenterCapability;
import tech.easyflow.datacenter.meta.enums.DatacenterSourceType;
import java.sql.Connection;
import java.util.EnumSet;
@Component
public class PostgresqlConnector extends AbstractJdbcConnector {
private final DatacenterDatasourceManager datasourceManager;
public PostgresqlConnector(DatacenterDatasourceManager datasourceManager) {
super(DatacenterSourceType.POSTGRESQL, new PostgresqlSqlDialect(), EnumSet.of(
DatacenterCapability.TEST_CONNECTION,
DatacenterCapability.BROWSE_METADATA,
DatacenterCapability.READ_QUERY
));
this.datasourceManager = datasourceManager;
}
@Override
protected <T> T withConnection(DatacenterSource source, boolean cacheable, JdbcCallback<T> callback) throws Exception {
HikariDataSource dataSource = cacheable ? datasourceManager.getOrCreateExternalDatasource(source) : datasourceManager.createExternalDatasource(source);
try (Connection connection = dataSource.getConnection()) {
return callback.apply(connection);
} finally {
if (!cacheable || source.getId() == null) {
dataSource.close();
}
}
}
}

View File

@@ -0,0 +1,135 @@
package tech.easyflow.datacenter.connector.impl;
import cn.hutool.core.util.StrUtil;
import com.alibaba.fastjson2.JSONObject;
import com.mybatisflex.core.paginate.Page;
import com.mybatisflex.core.row.Row;
import org.springframework.stereotype.Component;
import tech.easyflow.common.entity.LoginAccount;
import tech.easyflow.common.web.exceptions.BusinessException;
import tech.easyflow.datacenter.connector.dialect.MysqlSqlDialect;
import tech.easyflow.datacenter.connector.support.AbstractJdbcConnector;
import tech.easyflow.datacenter.entity.DatacenterTable;
import tech.easyflow.datacenter.entity.DatacenterTableField;
import tech.easyflow.datacenter.execution.model.DatacenterQueryRequest;
import tech.easyflow.datacenter.meta.entity.DatacenterSource;
import tech.easyflow.datacenter.meta.enums.DatacenterCapability;
import tech.easyflow.datacenter.meta.enums.DatacenterSourceType;
import javax.sql.DataSource;
import java.math.BigInteger;
import java.sql.Connection;
import java.sql.PreparedStatement;
import java.util.ArrayList;
import java.util.EnumSet;
import java.util.List;
import java.util.stream.Collectors;
@Component
public class ProjectMysqlConnector extends AbstractJdbcConnector {
private final DataSource dataSource;
private final MysqlSqlDialect dialect = new MysqlSqlDialect();
public ProjectMysqlConnector(DataSource dataSource) {
super(DatacenterSourceType.PROJECT_MYSQL, new MysqlSqlDialect(), EnumSet.of(
DatacenterCapability.TEST_CONNECTION,
DatacenterCapability.BROWSE_METADATA,
DatacenterCapability.READ_QUERY,
DatacenterCapability.WRITE_MUTATION
));
this.dataSource = dataSource;
}
@Override
protected <T> T withConnection(DatacenterSource source, boolean cacheable, JdbcCallback<T> callback) throws Exception {
try (Connection connection = dataSource.getConnection()) {
return callback.apply(connection);
}
}
@Override
protected boolean requiresJdbcUrl() {
return false;
}
@Override
public Page<Row> queryPage(DatacenterSource source, DatacenterTable table, DatacenterQueryRequest request) {
return super.queryPage(source, table, request);
}
@Override
public void saveRow(DatacenterSource source, DatacenterTable table, JSONObject data, LoginAccount account) {
List<DatacenterTableField> writableFields = table.getFields().stream()
.filter(field -> field.getWritable() == null || field.getWritable() == 1)
.collect(Collectors.toList());
Object id = data.get("id");
try (Connection connection = dataSource.getConnection()) {
if (id == null) {
List<String> columns = new ArrayList<>();
List<Object> values = new ArrayList<>();
for (DatacenterTableField field : writableFields) {
Object value = data.get(field.getFieldName());
if (value != null) {
columns.add(field.getFieldName());
values.add(value);
}
}
if (columns.isEmpty()) {
throw new BusinessException("没有可写字段");
}
String sql = "INSERT INTO " + dialect.qualifyTable(source.getDatabaseName(), resolvePhysicalTableName(table))
+ " (" + columns.stream().map(dialect::quoteIdentifier).collect(Collectors.joining(",")) + ") VALUES ("
+ columns.stream().map(item -> "?").collect(Collectors.joining(",")) + ")";
try (PreparedStatement statement = connection.prepareStatement(sql)) {
for (int i = 0; i < values.size(); i++) {
statement.setObject(i + 1, values.get(i));
}
statement.executeUpdate();
}
return;
}
List<String> setClauses = new ArrayList<>();
List<Object> values = new ArrayList<>();
for (DatacenterTableField field : writableFields) {
if (!data.containsKey(field.getFieldName())) {
continue;
}
setClauses.add(dialect.quoteIdentifier(field.getFieldName()) + " = ?");
values.add(data.get(field.getFieldName()));
}
if (setClauses.isEmpty()) {
return;
}
String sql = "UPDATE " + dialect.qualifyTable(source.getDatabaseName(), resolvePhysicalTableName(table))
+ " SET " + String.join(",", setClauses)
+ " WHERE " + dialect.quoteIdentifier("id") + " = ?";
values.add(id);
try (PreparedStatement statement = connection.prepareStatement(sql)) {
for (int i = 0; i < values.size(); i++) {
statement.setObject(i + 1, values.get(i));
}
statement.executeUpdate();
}
} catch (Exception ex) {
throw new BusinessException("项目 MySQL 写入失败: " + ex.getMessage());
}
}
@Override
public void deleteRow(DatacenterSource source, DatacenterTable table, BigInteger id, LoginAccount account) {
String sql = "DELETE FROM " + dialect.qualifyTable(source.getDatabaseName(), resolvePhysicalTableName(table))
+ " WHERE " + dialect.quoteIdentifier("id") + " = ?";
try (Connection connection = dataSource.getConnection(); PreparedStatement statement = connection.prepareStatement(sql)) {
statement.setObject(1, id);
statement.executeUpdate();
} catch (Exception ex) {
throw new BusinessException("项目 MySQL 删除失败: " + ex.getMessage());
}
}
@Override
protected String resolveCatalogName(DatacenterSource source, String requestedCatalogName) {
return StrUtil.blankToDefault(requestedCatalogName, source.getDatabaseName());
}
}

View File

@@ -0,0 +1,152 @@
package tech.easyflow.datacenter.connector.support;
import cn.hutool.core.util.StrUtil;
import com.alibaba.fastjson2.JSONObject;
import com.mybatisflex.core.paginate.Page;
import com.mybatisflex.core.query.QueryWrapper;
import com.mybatisflex.core.row.Db;
import com.mybatisflex.core.row.Row;
import com.mybatisflex.core.row.RowKey;
import org.springframework.util.CollectionUtils;
import tech.easyflow.common.entity.LoginAccount;
import tech.easyflow.common.web.exceptions.BusinessException;
import tech.easyflow.datacenter.connector.DatacenterConnector;
import tech.easyflow.datacenter.entity.DatacenterTable;
import tech.easyflow.datacenter.entity.DatacenterTableField;
import tech.easyflow.datacenter.execution.model.DatacenterConnectionTestResult;
import tech.easyflow.datacenter.execution.model.DatacenterQueryRequest;
import tech.easyflow.datacenter.meta.entity.DatacenterSource;
import tech.easyflow.datacenter.meta.enums.DatacenterCapability;
import tech.easyflow.datacenter.meta.enums.DatacenterSourceType;
import tech.easyflow.datacenter.meta.model.DatacenterCatalogMeta;
import tech.easyflow.datacenter.meta.model.DatacenterTableDetailMeta;
import java.math.BigDecimal;
import java.math.BigInteger;
import java.util.*;
public abstract class AbstractInternalTableConnector implements DatacenterConnector {
private final DatacenterSourceType sourceType;
private final Set<DatacenterCapability> capabilities;
protected AbstractInternalTableConnector(DatacenterSourceType sourceType, Set<DatacenterCapability> capabilities) {
this.sourceType = sourceType;
this.capabilities = capabilities;
}
@Override
public DatacenterSourceType getSourceType() {
return sourceType;
}
@Override
public Set<DatacenterCapability> getCapabilities() {
return capabilities;
}
@Override
public DatacenterConnectionTestResult testConnection(DatacenterSource source) {
DatacenterConnectionTestResult result = new DatacenterConnectionTestResult();
result.setSuccess(true);
result.setMessage("连接成功");
result.setCapabilities(capabilities.stream().map(Enum::name).toList());
result.setDetails(Map.of("sourceType", sourceType.name()));
return result;
}
@Override
public List<DatacenterCatalogMeta> listCatalogs(DatacenterSource source) {
return Collections.emptyList();
}
@Override
public List<DatacenterTable> listTables(DatacenterSource source, String catalogName) {
return Collections.emptyList();
}
@Override
public DatacenterTableDetailMeta getTableDetail(DatacenterSource source, String catalogName, String tableName) {
throw new BusinessException("内部数据源请从元数据注册表读取表结构");
}
@Override
public Page<Row> queryPage(DatacenterSource source, DatacenterTable table, DatacenterQueryRequest request) {
String actualTable = resolveTableName(table);
QueryWrapper wrapper = QueryWrapper.create();
if (StrUtil.isNotBlank(request.getWhere())) {
wrapper.where(request.getWhere());
}
long count = Db.selectCountByQuery(actualTable, wrapper);
if (count == 0) {
return new Page<>(new ArrayList<>(), request.getPageNumber(), request.getPageSize(), count);
}
Page<Row> page = Db.paginate(actualTable, new Page<>(request.getPageNumber(), request.getPageSize(), count), wrapper);
normalizeRows(page.getRecords());
return page;
}
@Override
public List<Row> queryBySql(DatacenterSource source, String sql) {
List<Row> rows = Db.selectListBySql(sql);
normalizeRows(rows);
return rows;
}
@Override
public void saveRow(DatacenterSource source, DatacenterTable table, JSONObject data, LoginAccount account) {
List<DatacenterTableField> fields = table.getFields();
if (CollectionUtils.isEmpty(fields)) {
throw new BusinessException("数据集字段为空,无法写入");
}
String actualTable = resolveTableName(table);
Object id = data.get("id");
if (id == null) {
Row row = Row.ofKey(RowKey.SNOW_FLAKE_ID);
row.put("dept_id", account.getDeptId());
row.put("tenant_id", account.getTenantId());
row.put("created", new Date());
row.put("created_by", account.getId());
row.put("modified", new Date());
row.put("modified_by", account.getId());
row.put("remark", data.get("remark"));
for (DatacenterTableField field : fields) {
row.put(field.getFieldName(), data.get(field.getFieldName()));
}
Db.insert(actualTable, row);
return;
}
Row row = Row.ofKey("id", id);
row.put("modified", new Date());
row.put("modified_by", account.getId());
for (DatacenterTableField field : fields) {
row.put(field.getFieldName(), data.get(field.getFieldName()));
}
Db.updateById(actualTable, row);
}
@Override
public void deleteRow(DatacenterSource source, DatacenterTable table, BigInteger id, LoginAccount account) {
Db.deleteById(resolveTableName(table), Row.ofKey("id", id));
}
protected String resolveTableName(DatacenterTable table) {
return StrUtil.blankToDefault(table.getMaterializedTable(), table.getActualTable());
}
private void normalizeRows(List<Row> records) {
for (Row record : records) {
Map<String, Object> converted = new LinkedHashMap<>();
for (Map.Entry<String, Object> entry : record.entrySet()) {
Object value = entry.getValue();
if (value instanceof BigInteger || value instanceof BigDecimal || value instanceof Long) {
converted.put(entry.getKey(), value.toString());
} else {
converted.put(entry.getKey(), value);
}
}
record.clear();
record.putAll(converted);
}
}
}

View File

@@ -0,0 +1,523 @@
package tech.easyflow.datacenter.connector.support;
import cn.hutool.core.util.StrUtil;
import com.alibaba.fastjson2.JSONObject;
import com.mybatisflex.core.paginate.Page;
import com.mybatisflex.core.row.Row;
import org.springframework.util.CollectionUtils;
import tech.easyflow.common.constant.enums.EnumFieldType;
import tech.easyflow.common.entity.LoginAccount;
import tech.easyflow.common.web.exceptions.BusinessException;
import tech.easyflow.datacenter.connector.DatacenterConnector;
import tech.easyflow.datacenter.connector.SqlDialect;
import tech.easyflow.datacenter.entity.DatacenterTable;
import tech.easyflow.datacenter.entity.DatacenterTableField;
import tech.easyflow.datacenter.execution.model.DatacenterConnectionTestResult;
import tech.easyflow.datacenter.execution.model.DatacenterQueryFilter;
import tech.easyflow.datacenter.execution.model.DatacenterQueryRequest;
import tech.easyflow.datacenter.execution.model.DatacenterQuerySort;
import tech.easyflow.datacenter.meta.entity.DatacenterSource;
import tech.easyflow.datacenter.meta.enums.DatacenterCapability;
import tech.easyflow.datacenter.meta.enums.DatacenterConnectionErrorCode;
import tech.easyflow.datacenter.meta.enums.DatacenterSourceType;
import tech.easyflow.datacenter.meta.enums.DatacenterTableKind;
import tech.easyflow.datacenter.meta.model.DatacenterCatalogMeta;
import tech.easyflow.datacenter.meta.model.DatacenterTableDetailMeta;
import java.math.BigDecimal;
import java.math.BigInteger;
import java.sql.*;
import java.util.*;
import java.util.stream.Collectors;
public abstract class AbstractJdbcConnector implements DatacenterConnector {
private final DatacenterSourceType sourceType;
private final SqlDialect sqlDialect;
private final Set<DatacenterCapability> capabilities;
protected AbstractJdbcConnector(DatacenterSourceType sourceType, SqlDialect sqlDialect, Set<DatacenterCapability> capabilities) {
this.sourceType = sourceType;
this.sqlDialect = sqlDialect;
this.capabilities = capabilities;
}
@Override
public DatacenterSourceType getSourceType() {
return sourceType;
}
@Override
public Set<DatacenterCapability> getCapabilities() {
return capabilities;
}
protected abstract <T> T withConnection(DatacenterSource source, boolean cacheable, JdbcCallback<T> callback) throws Exception;
@Override
public DatacenterConnectionTestResult testConnection(DatacenterSource source) {
DatacenterConnectionTestResult result = new DatacenterConnectionTestResult();
result.setCapabilities(capabilities.stream().map(Enum::name).toList());
if (StrUtil.isBlank(source.getJdbcUrl()) && requiresJdbcUrl()) {
result.setSuccess(false);
result.setErrorCode(DatacenterConnectionErrorCode.INVALID_ARGUMENT.name());
result.setMessage("缺少 JDBC URL");
return result;
}
try {
if (StrUtil.isNotBlank(source.getDriverClassName())) {
Class.forName(source.getDriverClassName());
}
Map<String, Object> details = withConnection(source, false, connection -> {
DatabaseMetaData metaData = connection.getMetaData();
Map<String, Object> map = new LinkedHashMap<>();
map.put("databaseProductName", metaData.getDatabaseProductName());
map.put("databaseProductVersion", metaData.getDatabaseProductVersion());
map.put("url", metaData.getURL());
return map;
});
result.setSuccess(true);
result.setMessage("连接成功");
result.setDetails(details);
return result;
} catch (ClassNotFoundException ex) {
result.setSuccess(false);
result.setErrorCode(DatacenterConnectionErrorCode.DRIVER_NOT_FOUND.name());
result.setMessage(ex.getMessage());
return result;
} catch (SQLException ex) {
result.setSuccess(false);
result.setErrorCode(mapSqlError(ex));
result.setMessage(ex.getMessage());
return result;
} catch (Exception ex) {
result.setSuccess(false);
result.setErrorCode(DatacenterConnectionErrorCode.UNKNOWN_ERROR.name());
result.setMessage(ex.getMessage());
return result;
}
}
@Override
public List<DatacenterCatalogMeta> listCatalogs(DatacenterSource source) {
try {
return withConnection(source, true, connection -> {
List<DatacenterCatalogMeta> result = new ArrayList<>();
DatabaseMetaData metaData = connection.getMetaData();
try (ResultSet catalogs = metaData.getCatalogs()) {
while (catalogs.next()) {
String name = catalogs.getString("TABLE_CAT");
if (StrUtil.isBlank(name)) {
continue;
}
DatacenterCatalogMeta meta = new DatacenterCatalogMeta();
meta.setSourceId(source.getId());
meta.setCatalogName(name);
meta.setCatalogType("CATALOG");
result.add(meta);
}
}
try (ResultSet schemas = metaData.getSchemas()) {
while (schemas.next()) {
String name = schemas.getString("TABLE_SCHEM");
if (StrUtil.isBlank(name) || containsCatalog(result, name)) {
continue;
}
DatacenterCatalogMeta meta = new DatacenterCatalogMeta();
meta.setSourceId(source.getId());
meta.setCatalogName(name);
meta.setCatalogType("SCHEMA");
result.add(meta);
}
}
result = filterConfiguredCatalogs(source, result);
if (result.isEmpty()) {
String fallback = resolveCatalogName(source, null);
if (StrUtil.isNotBlank(fallback)) {
DatacenterCatalogMeta meta = new DatacenterCatalogMeta();
meta.setSourceId(source.getId());
meta.setCatalogName(fallback);
meta.setCatalogType("DEFAULT");
result.add(meta);
}
}
return result;
});
} catch (Exception ex) {
throw DatacenterConnectorExceptionSupport.wrapAccessException("读取目录失败", ex);
}
}
@Override
public List<DatacenterTable> listTables(DatacenterSource source, String catalogName) {
try {
return withConnection(source, true, connection -> {
DatabaseMetaData metaData = connection.getMetaData();
List<DatacenterTable> tables = new ArrayList<>();
try (ResultSet resultSet = metaData.getTables(resolveCatalogArgument(source, catalogName), resolveSchemaArgument(source, catalogName), "%", new String[]{"TABLE", "VIEW"})) {
while (resultSet.next()) {
DatacenterTable table = new DatacenterTable();
table.setSourceId(source.getId());
table.setTableName(resultSet.getString("TABLE_NAME"));
table.setTableDesc(resultSet.getString("REMARKS"));
table.setActualTable(resultSet.getString("TABLE_NAME"));
table.setMaterializedTable(resultSet.getString("TABLE_NAME"));
table.setAccessMode(capabilities.contains(DatacenterCapability.WRITE_MUTATION) ? "READ_WRITE" : "READ_ONLY");
table.setTableKind(resolveTableKind(resultSet.getString("TABLE_TYPE")).name());
table.setCapabilitiesJson(Map.of("capabilities", capabilities.stream().map(Enum::name).toList()));
tables.add(table);
}
}
return tables;
});
} catch (Exception ex) {
throw DatacenterConnectorExceptionSupport.wrapAccessException("读取表列表失败", ex);
}
}
@Override
public DatacenterTableDetailMeta getTableDetail(DatacenterSource source, String catalogName, String tableName) {
try {
return withConnection(source, true, connection -> {
DatabaseMetaData metaData = connection.getMetaData();
DatacenterTableDetailMeta detail = new DatacenterTableDetailMeta();
DatacenterTable table = new DatacenterTable();
table.setSourceId(source.getId());
table.setTableName(tableName);
table.setActualTable(tableName);
table.setMaterializedTable(tableName);
table.setAccessMode(capabilities.contains(DatacenterCapability.WRITE_MUTATION) ? "READ_WRITE" : "READ_ONLY");
table.setTableKind(DatacenterTableKind.EXTERNAL_TABLE.name());
table.setCapabilitiesJson(Map.of("capabilities", capabilities.stream().map(Enum::name).toList()));
detail.setTable(table);
try (ResultSet tableSet = metaData.getTables(
resolveCatalogArgument(source, catalogName),
resolveSchemaArgument(source, catalogName),
tableName,
new String[]{"TABLE", "VIEW"})) {
while (tableSet.next()) {
String currentTableName = tableSet.getString("TABLE_NAME");
if (!matchesTableName(currentTableName, tableName)) {
continue;
}
table.setTableDesc(tableSet.getString("REMARKS"));
table.setTableKind(resolveTableKind(tableSet.getString("TABLE_TYPE")).name());
break;
}
}
Set<String> primaryKeys = new HashSet<>();
try (ResultSet pkSet = metaData.getPrimaryKeys(resolveCatalogArgument(source, catalogName), resolveSchemaArgument(source, catalogName), tableName)) {
while (pkSet.next()) {
primaryKeys.add(pkSet.getString("COLUMN_NAME"));
}
}
List<DatacenterTableField> fields = new ArrayList<>();
try (ResultSet columns = metaData.getColumns(resolveCatalogArgument(source, catalogName), resolveSchemaArgument(source, catalogName), tableName, "%")) {
while (columns.next()) {
DatacenterTableField field = new DatacenterTableField();
field.setFieldName(columns.getString("COLUMN_NAME"));
field.setSourceColumnName(columns.getString("COLUMN_NAME"));
field.setFieldDesc(columns.getString("REMARKS"));
field.setJdbcType(columns.getString("TYPE_NAME"));
field.setPrecision(columns.getInt("COLUMN_SIZE"));
field.setScale(columns.getInt("DECIMAL_DIGITS"));
field.setRequired(columns.getInt("NULLABLE") == DatabaseMetaData.columnNoNulls ? 1 : 0);
field.setQueryable(1);
field.setSortable(1);
field.setWritable(capabilities.contains(DatacenterCapability.WRITE_MUTATION) ? 1 : 0);
field.setIndexed(primaryKeys.contains(field.getFieldName()) ? 1 : 0);
field.setFieldType(mapFieldType(columns.getInt("DATA_TYPE")));
fields.add(field);
}
}
detail.setFields(fields);
return detail;
});
} catch (Exception ex) {
throw DatacenterConnectorExceptionSupport.wrapAccessException("读取表详情失败", ex);
}
}
@Override
public Page<Row> queryPage(DatacenterSource source, DatacenterTable table, DatacenterQueryRequest request) {
if (!capabilities.contains(DatacenterCapability.READ_QUERY)) {
throw new BusinessException("当前数据源暂不支持查询");
}
try {
return withConnection(source, true, connection -> doQueryPage(connection, source, table, request));
} catch (Exception ex) {
throw DatacenterConnectorExceptionSupport.wrapAccessException("查询失败", ex);
}
}
@Override
public List<Row> queryBySql(DatacenterSource source, String sql) {
if (!capabilities.contains(DatacenterCapability.READ_QUERY)) {
throw new BusinessException("当前数据源暂不支持查询");
}
try {
return withConnection(source, true, connection -> doQueryBySql(connection, sql));
} catch (Exception ex) {
throw DatacenterConnectorExceptionSupport.wrapAccessException("SQL 查询失败", ex);
}
}
@Override
public void saveRow(DatacenterSource source, DatacenterTable table, JSONObject data, LoginAccount account) {
throw new BusinessException("当前数据源不支持写入");
}
@Override
public void deleteRow(DatacenterSource source, DatacenterTable table, BigInteger id, LoginAccount account) {
throw new BusinessException("当前数据源不支持删除");
}
protected boolean requiresJdbcUrl() {
return true;
}
protected Page<Row> doQueryPage(Connection connection, DatacenterSource source, DatacenterTable table, DatacenterQueryRequest request) throws SQLException {
List<Object> params = new ArrayList<>();
String selectColumns = CollectionUtils.isEmpty(request.getSelectedColumns())
? "*"
: request.getSelectedColumns().stream().map(sqlDialect::quoteIdentifier).collect(Collectors.joining(", "));
String qualifiedTable = sqlDialect.qualifyTable(resolveCatalogName(source, request.getDatasetRef() == null ? null : request.getDatasetRef().getCatalogName()), resolvePhysicalTableName(table));
StringBuilder whereClause = new StringBuilder();
if (StrUtil.isNotBlank(request.getWhere())) {
whereClause.append(" WHERE ").append(request.getWhere());
} else if (!CollectionUtils.isEmpty(request.getFilters())) {
whereClause.append(" WHERE 1=1 ");
for (DatacenterQueryFilter filter : request.getFilters()) {
appendFilter(whereClause, params, filter);
}
}
String orderClause = buildOrderClause(request.getSorts());
String baseSql = "SELECT " + selectColumns + " FROM " + qualifiedTable + whereClause + orderClause;
String countSql = "SELECT COUNT(1) FROM " + qualifiedTable + whereClause;
long total = queryCount(connection, countSql, params);
if (total == 0L) {
return new Page<>(new ArrayList<>(), request.getPageNumber(), request.getPageSize(), total);
}
String pageSql = sqlDialect.buildPageSql(baseSql, request.getPageNumber(), request.getPageSize());
List<Object> pageParams = new ArrayList<>(params);
if (pageSql.contains("FETCH NEXT") || pageSql.toLowerCase(Locale.ROOT).contains("limit")) {
pageParams.add((request.getPageNumber() - 1) * request.getPageSize());
pageParams.add(request.getPageSize());
}
List<Row> records = new ArrayList<>();
try (PreparedStatement statement = connection.prepareStatement(pageSql)) {
bindParameters(statement, pageParams);
try (ResultSet resultSet = statement.executeQuery()) {
ResultSetMetaData metaData = resultSet.getMetaData();
while (resultSet.next()) {
Row row = new Row();
for (int i = 1; i <= metaData.getColumnCount(); i++) {
String columnLabel = metaData.getColumnLabel(i);
row.put(columnLabel, normalizeValue(resultSet.getObject(i)));
}
records.add(row);
}
}
}
return new Page<>(records, request.getPageNumber(), request.getPageSize(), total);
}
protected String resolvePhysicalTableName(DatacenterTable table) {
return StrUtil.blankToDefault(table.getActualTable(), table.getTableName());
}
protected List<Row> doQueryBySql(Connection connection, String sql) throws SQLException {
try (PreparedStatement statement = connection.prepareStatement(sql);
ResultSet resultSet = statement.executeQuery()) {
return readRows(resultSet);
}
}
protected String resolveCatalogName(DatacenterSource source, String requestedCatalogName) {
if (StrUtil.isNotBlank(requestedCatalogName)) {
return requestedCatalogName;
}
if (usesCatalogNamespace()) {
return source.getDatabaseName();
}
return source.getSchemaName();
}
protected List<Row> readRows(ResultSet resultSet) throws SQLException {
List<Row> records = new ArrayList<>();
ResultSetMetaData metaData = resultSet.getMetaData();
while (resultSet.next()) {
Row row = new Row();
for (int i = 1; i <= metaData.getColumnCount(); i++) {
String columnLabel = metaData.getColumnLabel(i);
row.put(columnLabel, normalizeValue(resultSet.getObject(i)));
}
records.add(row);
}
return records;
}
protected String resolveCatalogArgument(DatacenterSource source, String catalogName) {
return usesCatalogNamespace() ? resolveCatalogName(source, catalogName) : source.getDatabaseName();
}
protected String resolveSchemaArgument(DatacenterSource source, String catalogName) {
return usesCatalogNamespace() ? source.getSchemaName() : resolveCatalogName(source, catalogName);
}
protected boolean usesCatalogNamespace() {
return sourceType == DatacenterSourceType.MYSQL
|| sourceType == DatacenterSourceType.PROJECT_MYSQL
|| sourceType == DatacenterSourceType.GBASE_8A
|| sourceType == DatacenterSourceType.GBASE_8S;
}
private List<DatacenterCatalogMeta> filterConfiguredCatalogs(DatacenterSource source, List<DatacenterCatalogMeta> items) {
if (items == null || items.isEmpty() || source == null) {
return items;
}
String configuredName = usesCatalogNamespace()
? StrUtil.trimToNull(source.getDatabaseName())
: StrUtil.trimToNull(source.getSchemaName());
if (StrUtil.isBlank(configuredName)) {
return items;
}
List<DatacenterCatalogMeta> matched = items.stream()
.filter(item -> configuredName.equalsIgnoreCase(item.getCatalogName()))
.collect(Collectors.toList());
return matched.isEmpty() ? items : matched;
}
private boolean containsCatalog(List<DatacenterCatalogMeta> items, String catalogName) {
return items.stream().anyMatch(item -> catalogName.equalsIgnoreCase(item.getCatalogName()));
}
private DatacenterTableKind resolveTableKind(String tableType) {
return "VIEW".equalsIgnoreCase(tableType) ? DatacenterTableKind.EXTERNAL_VIEW : DatacenterTableKind.EXTERNAL_TABLE;
}
private boolean matchesTableName(String currentTableName, String targetTableName) {
if (currentTableName == null || targetTableName == null) {
return false;
}
return currentTableName.equals(targetTableName)
|| currentTableName.equalsIgnoreCase(targetTableName);
}
private Integer mapFieldType(int jdbcType) {
return switch (jdbcType) {
case Types.INTEGER, Types.TINYINT, Types.SMALLINT, Types.BIGINT -> EnumFieldType.INTEGER.getCode();
case Types.FLOAT, Types.DOUBLE, Types.REAL, Types.NUMERIC, Types.DECIMAL -> EnumFieldType.NUMBER.getCode();
case Types.TIMESTAMP, Types.DATE, Types.TIME -> EnumFieldType.TIME.getCode();
case Types.BOOLEAN, Types.BIT -> EnumFieldType.BOOLEAN.getCode();
default -> EnumFieldType.STRING.getCode();
};
}
private void appendFilter(StringBuilder sql, List<Object> params, DatacenterQueryFilter filter) {
String operator = StrUtil.blankToDefault(filter.getOperator(), "EQ").toUpperCase(Locale.ROOT);
String column = sqlDialect.quoteIdentifier(filter.getColumn());
switch (operator) {
case "EQ" -> {
sql.append(" AND ").append(column).append(" = ?");
params.add(filter.getValue());
}
case "LIKE" -> {
sql.append(" AND ").append(column).append(" LIKE ?");
params.add("%" + filter.getValue() + "%");
}
case "GT" -> {
sql.append(" AND ").append(column).append(" > ?");
params.add(filter.getValue());
}
case "GTE" -> {
sql.append(" AND ").append(column).append(" >= ?");
params.add(filter.getValue());
}
case "LT" -> {
sql.append(" AND ").append(column).append(" < ?");
params.add(filter.getValue());
}
case "LTE" -> {
sql.append(" AND ").append(column).append(" <= ?");
params.add(filter.getValue());
}
case "IN" -> {
List<Object> values = filter.getValues() == null ? List.of() : filter.getValues();
if (values.isEmpty()) {
sql.append(" AND 1 = 0");
} else {
sql.append(" AND ").append(column).append(" IN (");
sql.append(values.stream().map(item -> "?").collect(Collectors.joining(",")));
sql.append(")");
params.addAll(values);
}
}
case "IS_NULL" -> sql.append(" AND ").append(column).append(" IS NULL");
default -> throw new BusinessException("不支持的过滤操作: " + operator);
}
}
private String buildOrderClause(List<DatacenterQuerySort> sorts) {
if (CollectionUtils.isEmpty(sorts)) {
return "";
}
return " ORDER BY " + sorts.stream()
.map(sort -> sqlDialect.quoteIdentifier(sort.getColumn()) + " " + ("DESC".equalsIgnoreCase(sort.getDirection()) ? "DESC" : "ASC"))
.collect(Collectors.joining(", "));
}
private long queryCount(Connection connection, String sql, List<Object> params) throws SQLException {
try (PreparedStatement statement = connection.prepareStatement(sql)) {
bindParameters(statement, params);
try (ResultSet resultSet = statement.executeQuery()) {
if (resultSet.next()) {
return resultSet.getLong(1);
}
return 0L;
}
}
}
private void bindParameters(PreparedStatement statement, List<Object> params) throws SQLException {
for (int i = 0; i < params.size(); i++) {
statement.setObject(i + 1, params.get(i));
}
}
private Object normalizeValue(Object value) {
if (value instanceof BigDecimal || value instanceof BigInteger || value instanceof Long) {
return value.toString();
}
return value;
}
private String mapSqlError(SQLException ex) {
String state = ex.getSQLState();
String message = ex.getMessage() == null ? "" : ex.getMessage().toLowerCase(Locale.ROOT);
if (message.contains("access denied") || message.contains("password") || message.contains("authentication")) {
return DatacenterConnectionErrorCode.AUTH_FAILED.name();
}
if (message.contains("unknown database") || message.contains("database does not exist")) {
return DatacenterConnectionErrorCode.DATABASE_NOT_FOUND.name();
}
if (message.contains("schema") && message.contains("does not exist")) {
return DatacenterConnectionErrorCode.SCHEMA_NOT_FOUND.name();
}
if (message.contains("permission denied") || message.contains("insufficient privilege")) {
return DatacenterConnectionErrorCode.PERMISSION_DENIED.name();
}
if (state != null && state.startsWith("08")) {
return DatacenterConnectionErrorCode.NETWORK_UNREACHABLE.name();
}
return DatacenterConnectionErrorCode.UNKNOWN_ERROR.name();
}
@FunctionalInterface
protected interface JdbcCallback<T> {
T apply(Connection connection) throws Exception;
}
}

View File

@@ -0,0 +1,80 @@
package tech.easyflow.datacenter.connector.support;
import tech.easyflow.common.web.exceptions.BusinessException;
import java.net.ConnectException;
import java.net.NoRouteToHostException;
import java.net.SocketException;
import java.net.SocketTimeoutException;
import java.net.UnknownHostException;
import java.sql.SQLException;
import java.util.Locale;
public final class DatacenterConnectorExceptionSupport {
public static final String SOURCE_UNAVAILABLE_MESSAGE = "当前连接不可用,请检查连接配置后重试";
private DatacenterConnectorExceptionSupport() {
}
public static BusinessException wrapAccessException(String fallbackMessage, Exception ex) {
if (ex instanceof BusinessException businessException && !isConnectionUnavailable(ex)) {
return businessException;
}
if (isConnectionUnavailable(ex)) {
return new BusinessException(SOURCE_UNAVAILABLE_MESSAGE);
}
return new BusinessException(fallbackMessage);
}
public static boolean isConnectionUnavailable(Throwable throwable) {
Throwable current = throwable;
while (current != null) {
if (current instanceof ClassNotFoundException
|| current instanceof ConnectException
|| current instanceof NoRouteToHostException
|| current instanceof SocketTimeoutException
|| current instanceof UnknownHostException) {
return true;
}
if (current instanceof SocketException socketException) {
String socketMessage = lowerCase(socketException.getMessage());
if (socketMessage.contains("broken pipe")
|| socketMessage.contains("connection reset")
|| socketMessage.contains("network is unreachable")) {
return true;
}
}
if (current instanceof SQLException sqlException) {
String sqlState = sqlException.getSQLState();
String message = lowerCase(sqlException.getMessage());
if ((sqlState != null && sqlState.startsWith("08"))
|| message.contains("communications link failure")
|| message.contains("connection refused")
|| message.contains("connection attempt failed")
|| message.contains("connect timed out")
|| message.contains("i/o error")
|| message.contains("io error")
|| message.contains("the network adapter could not establish the connection")
|| message.contains("unknown host")
|| message.contains("access denied")
|| message.contains("authentication failed")
|| message.contains("login failed")
|| message.contains("password authentication failed")
|| message.contains("unknown database")
|| message.contains("database does not exist")
|| (message.contains("schema") && message.contains("does not exist"))
|| message.contains("insufficient privilege")
|| message.contains("permission denied")) {
return true;
}
}
current = current.getCause();
}
return false;
}
private static String lowerCase(String message) {
return message == null ? "" : message.toLowerCase(Locale.ROOT);
}
}

View File

@@ -0,0 +1,44 @@
package tech.easyflow.datacenter.connector.support;
import com.zaxxer.hikari.HikariConfig;
import com.zaxxer.hikari.HikariDataSource;
import org.springframework.stereotype.Component;
import tech.easyflow.datacenter.meta.entity.DatacenterSource;
import tech.easyflow.datacenter.security.DatacenterCredentialCipher;
import javax.annotation.Resource;
import java.math.BigInteger;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
@Component
public class DatacenterDatasourceManager {
private final Map<BigInteger, HikariDataSource> externalDatasourceCache = new ConcurrentHashMap<>();
@Resource
private DatacenterCredentialCipher credentialCipher;
public HikariDataSource createExternalDatasource(DatacenterSource source) {
HikariConfig config = new HikariConfig();
config.setPoolName("dc-ext-" + (source.getId() == null ? "temp" : source.getId()));
config.setJdbcUrl(source.getJdbcUrl());
config.setUsername(source.getUsername());
config.setPassword(credentialCipher.decrypt(source.getCredentialCipher()));
config.setMaximumPoolSize(3);
config.setMinimumIdle(0);
config.setConnectionTimeout(5000);
config.setValidationTimeout(3000);
if (source.getDriverClassName() != null && !source.getDriverClassName().isBlank()) {
config.setDriverClassName(source.getDriverClassName());
}
return new HikariDataSource(config);
}
public HikariDataSource getOrCreateExternalDatasource(DatacenterSource source) {
if (source.getId() == null) {
return createExternalDatasource(source);
}
return externalDatasourceCache.computeIfAbsent(source.getId(), key -> createExternalDatasource(source));
}
}

View File

@@ -33,6 +33,18 @@ public class DatacenterTableBase extends DateEntity implements Serializable {
@Column(tenantId = true, comment = "租户ID")
private BigInteger tenantId;
/**
* 数据源ID
*/
@Column(comment = "数据源ID")
private BigInteger sourceId;
/**
* 目录ID
*/
@Column(comment = "目录ID")
private BigInteger catalogId;
/**
* 数据表名
*/
@@ -51,6 +63,30 @@ public class DatacenterTableBase extends DateEntity implements Serializable {
@Column(comment = "物理表名")
private String actualTable;
/**
* 表类型
*/
@Column(comment = "表类型")
private String tableKind;
/**
* 访问模式
*/
@Column(comment = "访问模式")
private String accessMode;
/**
* 物化表名
*/
@Column(comment = "物化表名")
private String materializedTable;
/**
* 是否开启版本
*/
@Column(comment = "是否开启版本")
private Integer versioningEnabled;
/**
* 数据状态
*/
@@ -87,6 +123,12 @@ public class DatacenterTableBase extends DateEntity implements Serializable {
@Column(typeHandler = FastjsonTypeHandler.class, comment = "扩展项")
private Map<String, Object> options;
/**
* 能力声明
*/
@Column(typeHandler = FastjsonTypeHandler.class, comment = "能力声明")
private Map<String, Object> capabilitiesJson;
public BigInteger getId() {
return id;
}
@@ -111,6 +153,22 @@ public class DatacenterTableBase extends DateEntity implements Serializable {
this.tenantId = tenantId;
}
public BigInteger getSourceId() {
return sourceId;
}
public void setSourceId(BigInteger sourceId) {
this.sourceId = sourceId;
}
public BigInteger getCatalogId() {
return catalogId;
}
public void setCatalogId(BigInteger catalogId) {
this.catalogId = catalogId;
}
public String getTableName() {
return tableName;
}
@@ -135,6 +193,38 @@ public class DatacenterTableBase extends DateEntity implements Serializable {
this.actualTable = actualTable;
}
public String getTableKind() {
return tableKind;
}
public void setTableKind(String tableKind) {
this.tableKind = tableKind;
}
public String getAccessMode() {
return accessMode;
}
public void setAccessMode(String accessMode) {
this.accessMode = accessMode;
}
public String getMaterializedTable() {
return materializedTable;
}
public void setMaterializedTable(String materializedTable) {
this.materializedTable = materializedTable;
}
public Integer getVersioningEnabled() {
return versioningEnabled;
}
public void setVersioningEnabled(Integer versioningEnabled) {
this.versioningEnabled = versioningEnabled;
}
public Integer getStatus() {
return status;
}
@@ -183,4 +273,12 @@ public class DatacenterTableBase extends DateEntity implements Serializable {
this.options = options;
}
public Map<String, Object> getCapabilitiesJson() {
return capabilitiesJson;
}
public void setCapabilitiesJson(Map<String, Object> capabilitiesJson) {
this.capabilitiesJson = capabilitiesJson;
}
}

View File

@@ -33,6 +33,12 @@ public class DatacenterTableFieldBase extends DateEntity implements Serializable
@Column(comment = "字段名称")
private String fieldName;
/**
* 源字段名
*/
@Column(comment = "源字段名")
private String sourceColumnName;
/**
* 字段描述
*/
@@ -45,12 +51,54 @@ public class DatacenterTableFieldBase extends DateEntity implements Serializable
@Column(comment = "字段类型")
private Integer fieldType;
/**
* JDBC 类型
*/
@Column(comment = "JDBC类型")
private String jdbcType;
/**
* 精度
*/
@Column(comment = "精度")
private Integer precision;
/**
* 小数位
*/
@Column(comment = "小数位")
private Integer scale;
/**
* 是否必填
*/
@Column(comment = "是否必填")
private Integer required;
/**
* 可查询
*/
@Column(comment = "可查询")
private Integer queryable;
/**
* 可排序
*/
@Column(comment = "可排序")
private Integer sortable;
/**
* 可写入
*/
@Column(comment = "可写入")
private Integer writable;
/**
* 是否索引
*/
@Column(comment = "是否索引")
private Integer indexed;
/**
* 扩展项
*/
@@ -105,6 +153,14 @@ public class DatacenterTableFieldBase extends DateEntity implements Serializable
this.fieldName = fieldName;
}
public String getSourceColumnName() {
return sourceColumnName;
}
public void setSourceColumnName(String sourceColumnName) {
this.sourceColumnName = sourceColumnName;
}
public String getFieldDesc() {
return fieldDesc;
}
@@ -121,6 +177,30 @@ public class DatacenterTableFieldBase extends DateEntity implements Serializable
this.fieldType = fieldType;
}
public String getJdbcType() {
return jdbcType;
}
public void setJdbcType(String jdbcType) {
this.jdbcType = jdbcType;
}
public Integer getPrecision() {
return precision;
}
public void setPrecision(Integer precision) {
this.precision = precision;
}
public Integer getScale() {
return scale;
}
public void setScale(Integer scale) {
this.scale = scale;
}
public Integer getRequired() {
return required;
}
@@ -129,6 +209,38 @@ public class DatacenterTableFieldBase extends DateEntity implements Serializable
this.required = required;
}
public Integer getQueryable() {
return queryable;
}
public void setQueryable(Integer queryable) {
this.queryable = queryable;
}
public Integer getSortable() {
return sortable;
}
public void setSortable(Integer sortable) {
this.sortable = sortable;
}
public Integer getWritable() {
return writable;
}
public void setWritable(Integer writable) {
this.writable = writable;
}
public Integer getIndexed() {
return indexed;
}
public void setIndexed(Integer indexed) {
this.indexed = indexed;
}
public Map<String, Object> getOptions() {
return options;
}

View File

@@ -1,107 +0,0 @@
package tech.easyflow.datacenter.excel;
import cn.idev.excel.context.AnalysisContext;
import cn.idev.excel.metadata.data.ReadCellData;
import cn.idev.excel.read.listener.ReadListener;
import com.alibaba.fastjson2.JSONObject;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import tech.easyflow.common.entity.LoginAccount;
import tech.easyflow.common.util.SpringContextUtil;
import tech.easyflow.datacenter.entity.DatacenterTableField;
import tech.easyflow.datacenter.service.DatacenterTableService;
import java.math.BigInteger;
import java.util.*;
public class ReadDataListener implements ReadListener<LinkedHashMap<Integer, Object>> {
private static final Logger log = LoggerFactory.getLogger(ReadDataListener.class);
private BigInteger tableId;
private List<DatacenterTableField> fields;
private LoginAccount loginAccount;
private final Map<String, Integer> headFieldIndex = new HashMap<>();
private int successCount = 0;
private int errorCount = 0;
private int totalCount = 0;
private final List<JSONObject> errorRows = new ArrayList<>();
public ReadDataListener() {
}
public ReadDataListener(BigInteger tableId, List<DatacenterTableField> fields, LoginAccount loginAccount) {
this.tableId = tableId;
this.fields = fields;
this.loginAccount = loginAccount;
}
@Override
public void invoke(LinkedHashMap<Integer, Object> o, AnalysisContext analysisContext) {
DatacenterTableService service = SpringContextUtil.getBean(DatacenterTableService.class);
JSONObject obj = new JSONObject();
for (DatacenterTableField field : fields) {
String fieldName = field.getFieldName();
Integer i = headFieldIndex.get(fieldName);
if (i != null) {
obj.put(fieldName, o.get(i));
}
}
try {
service.saveValue(tableId, obj, loginAccount);
successCount++;
} catch (Exception e) {
errorCount++;
log.error("导入数据到数据中枢失败,具体值:{}", obj, e);
errorRows.add(obj);
}
totalCount++;
}
@Override
public void invokeHead(Map<Integer, ReadCellData<?>> headMap, AnalysisContext context) {
Set<Map.Entry<Integer, ReadCellData<?>>> entries = headMap.entrySet();
for (Map.Entry<Integer, ReadCellData<?>> entry : entries) {
Integer key = entry.getKey();
String field = entry.getValue().getStringValue();
headFieldIndex.put(field, key);
}
if (headFieldIndex.size() != fields.size()) {
throw new RuntimeException("表头字段数量与表结构对应不上!");
}
}
@Override
public void doAfterAllAnalysed(AnalysisContext analysisContext) {
}
public List<DatacenterTableField> getFields() {
return fields;
}
public void setFields(List<DatacenterTableField> fields) {
this.fields = fields;
}
public int getSuccessCount() {
return successCount;
}
public int getErrorCount() {
return errorCount;
}
public int getTotalCount() {
return totalCount;
}
public List<JSONObject> getErrorRows() {
return errorRows;
}
}

View File

@@ -1,68 +0,0 @@
package tech.easyflow.datacenter.excel;
import com.alibaba.fastjson2.JSONObject;
import java.util.List;
public class ReadResVo {
/**
* 成功数
*/
private int successCount = 0;
/**
* 失败数
*/
private int errorCount = 0;
/**
* 总数
*/
private int totalCount = 0;
/**
* 错误行
*/
private List<JSONObject> errorRows;
public ReadResVo() {
}
public ReadResVo(int successCount, int errorCount, int totalCount, List<JSONObject> errorRows) {
this.successCount = successCount;
this.errorCount = errorCount;
this.totalCount = totalCount;
this.errorRows = errorRows;
}
public int getSuccessCount() {
return successCount;
}
public void setSuccessCount(int successCount) {
this.successCount = successCount;
}
public int getErrorCount() {
return errorCount;
}
public void setErrorCount(int errorCount) {
this.errorCount = errorCount;
}
public int getTotalCount() {
return totalCount;
}
public void setTotalCount(int totalCount) {
this.totalCount = totalCount;
}
public List<JSONObject> getErrorRows() {
return errorRows;
}
public void setErrorRows(List<JSONObject> errorRows) {
this.errorRows = errorRows;
}
}

View File

@@ -0,0 +1,28 @@
package tech.easyflow.datacenter.excel.model;
import tech.easyflow.datacenter.execution.model.DatacenterQueryFilter;
import tech.easyflow.datacenter.execution.model.DatasetRef;
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
public class DatacenterExcelDeriveRequest {
private DatasetRef datasetRef;
private String targetTableName;
private List<String> selectedColumns = new ArrayList<>();
private Map<String, String> renameMappings = new LinkedHashMap<>();
private List<DatacenterQueryFilter> filters = new ArrayList<>();
public DatasetRef getDatasetRef() { return datasetRef; }
public void setDatasetRef(DatasetRef datasetRef) { this.datasetRef = datasetRef; }
public String getTargetTableName() { return targetTableName; }
public void setTargetTableName(String targetTableName) { this.targetTableName = targetTableName; }
public List<String> getSelectedColumns() { return selectedColumns; }
public void setSelectedColumns(List<String> selectedColumns) { this.selectedColumns = selectedColumns; }
public Map<String, String> getRenameMappings() { return renameMappings; }
public void setRenameMappings(Map<String, String> renameMappings) { this.renameMappings = renameMappings; }
public List<DatacenterQueryFilter> getFilters() { return filters; }
public void setFilters(List<DatacenterQueryFilter> filters) { this.filters = filters; }
}

View File

@@ -0,0 +1,23 @@
package tech.easyflow.datacenter.excel.model;
import tech.easyflow.datacenter.execution.model.DatasetRef;
import java.math.BigInteger;
import java.util.ArrayList;
import java.util.List;
public class DatacenterExcelExportRequest {
private BigInteger sourceId;
private BigInteger catalogId;
private List<DatasetRef> datasetRefs = new ArrayList<>();
private String fileName;
public BigInteger getSourceId() { return sourceId; }
public void setSourceId(BigInteger sourceId) { this.sourceId = sourceId; }
public BigInteger getCatalogId() { return catalogId; }
public void setCatalogId(BigInteger catalogId) { this.catalogId = catalogId; }
public List<DatasetRef> getDatasetRefs() { return datasetRefs; }
public void setDatasetRefs(List<DatasetRef> datasetRefs) { this.datasetRefs = datasetRefs; }
public String getFileName() { return fileName; }
public void setFileName(String fileName) { this.fileName = fileName; }
}

View File

@@ -0,0 +1,22 @@
package tech.easyflow.datacenter.excel.model;
import tech.easyflow.datacenter.execution.model.DatasetRef;
import java.util.ArrayList;
import java.util.List;
public class DatacenterExcelMergeRequest {
private List<DatasetRef> datasetRefs = new ArrayList<>();
private String mergeMode;
private String targetTableName;
private String joinKey;
public List<DatasetRef> getDatasetRefs() { return datasetRefs; }
public void setDatasetRefs(List<DatasetRef> datasetRefs) { this.datasetRefs = datasetRefs; }
public String getMergeMode() { return mergeMode; }
public void setMergeMode(String mergeMode) { this.mergeMode = mergeMode; }
public String getTargetTableName() { return targetTableName; }
public void setTargetTableName(String targetTableName) { this.targetTableName = targetTableName; }
public String getJoinKey() { return joinKey; }
public void setJoinKey(String joinKey) { this.joinKey = joinKey; }
}

View File

@@ -0,0 +1,30 @@
package tech.easyflow.datacenter.excel.model;
import tech.easyflow.datacenter.execution.model.DatasetRef;
import java.math.BigInteger;
public class DatacenterExcelSplitRequest {
private BigInteger sourceId;
private BigInteger catalogId;
private DatasetRef datasetRef;
private String splitMode;
private Integer rowBatchSize;
private String fieldName;
private String targetNamePrefix;
public BigInteger getSourceId() { return sourceId; }
public void setSourceId(BigInteger sourceId) { this.sourceId = sourceId; }
public BigInteger getCatalogId() { return catalogId; }
public void setCatalogId(BigInteger catalogId) { this.catalogId = catalogId; }
public DatasetRef getDatasetRef() { return datasetRef; }
public void setDatasetRef(DatasetRef datasetRef) { this.datasetRef = datasetRef; }
public String getSplitMode() { return splitMode; }
public void setSplitMode(String splitMode) { this.splitMode = splitMode; }
public Integer getRowBatchSize() { return rowBatchSize; }
public void setRowBatchSize(Integer rowBatchSize) { this.rowBatchSize = rowBatchSize; }
public String getFieldName() { return fieldName; }
public void setFieldName(String fieldName) { this.fieldName = fieldName; }
public String getTargetNamePrefix() { return targetNamePrefix; }
public void setTargetNamePrefix(String targetNamePrefix) { this.targetNamePrefix = targetNamePrefix; }
}

View File

@@ -0,0 +1,28 @@
package tech.easyflow.datacenter.excel.service;
import org.springframework.web.multipart.MultipartFile;
import tech.easyflow.common.entity.LoginAccount;
import tech.easyflow.datacenter.excel.model.DatacenterExcelDeriveRequest;
import tech.easyflow.datacenter.excel.model.DatacenterExcelExportRequest;
import tech.easyflow.datacenter.excel.model.DatacenterExcelMergeRequest;
import tech.easyflow.datacenter.excel.model.DatacenterExcelSplitRequest;
import tech.easyflow.datacenter.meta.entity.DatacenterImportJob;
import java.math.BigInteger;
import java.util.List;
public interface DatacenterExcelImportService {
DatacenterImportJob importWorkbook(MultipartFile file, LoginAccount account) throws Exception;
DatacenterImportJob splitWorkbook(DatacenterExcelSplitRequest request, LoginAccount account);
DatacenterImportJob mergeWorkbook(DatacenterExcelMergeRequest request, LoginAccount account);
DatacenterImportJob deriveWorkbook(DatacenterExcelDeriveRequest request, LoginAccount account);
DatacenterImportJob exportWorkbook(DatacenterExcelExportRequest request, LoginAccount account) throws Exception;
DatacenterImportJob getImportJobDetail(BigInteger jobId);
List<DatacenterImportJob> listJobs(BigInteger sourceId, BigInteger tableId);
}

View File

@@ -0,0 +1,950 @@
package tech.easyflow.datacenter.excel.service.impl;
import com.alibaba.fastjson2.JSONObject;
import com.mybatisflex.core.keygen.impl.SnowFlakeIDKeyGenerator;
import com.mybatisflex.core.paginate.Page;
import com.mybatisflex.core.row.Row;
import org.apache.poi.ss.usermodel.Cell;
import org.apache.poi.ss.usermodel.DataFormatter;
import org.apache.poi.ss.usermodel.Sheet;
import org.apache.poi.ss.usermodel.Workbook;
import org.apache.poi.ss.usermodel.WorkbookFactory;
import org.apache.poi.xssf.streaming.SXSSFWorkbook;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import org.springframework.util.CollectionUtils;
import org.springframework.web.multipart.MultipartFile;
import tech.easyflow.common.constant.enums.EnumFieldType;
import tech.easyflow.common.entity.LoginAccount;
import tech.easyflow.common.web.exceptions.BusinessException;
import tech.easyflow.datacenter.adapter.DbHandleManager;
import tech.easyflow.datacenter.entity.DatacenterTable;
import tech.easyflow.datacenter.entity.DatacenterTableField;
import tech.easyflow.datacenter.excel.model.DatacenterExcelDeriveRequest;
import tech.easyflow.datacenter.excel.model.DatacenterExcelExportRequest;
import tech.easyflow.datacenter.excel.model.DatacenterExcelMergeRequest;
import tech.easyflow.datacenter.excel.model.DatacenterExcelSplitRequest;
import tech.easyflow.datacenter.excel.service.DatacenterExcelImportService;
import tech.easyflow.datacenter.execution.model.DatacenterQueryFilter;
import tech.easyflow.datacenter.execution.model.DatacenterQueryRequest;
import tech.easyflow.datacenter.execution.model.DatasetRef;
import tech.easyflow.datacenter.execution.service.DatacenterDatasetQueryService;
import tech.easyflow.datacenter.mapper.DatacenterDatasetVersionMapper;
import tech.easyflow.datacenter.mapper.DatacenterDerivedTableMapper;
import tech.easyflow.datacenter.mapper.DatacenterImportJobMapper;
import tech.easyflow.datacenter.meta.entity.DatacenterCatalog;
import tech.easyflow.datacenter.meta.entity.DatacenterDatasetVersion;
import tech.easyflow.datacenter.meta.entity.DatacenterDerivedTable;
import tech.easyflow.datacenter.meta.entity.DatacenterImportJob;
import tech.easyflow.datacenter.meta.entity.DatacenterSource;
import tech.easyflow.datacenter.meta.enums.DatacenterImportStatus;
import tech.easyflow.datacenter.meta.enums.DatacenterSourceType;
import tech.easyflow.datacenter.meta.enums.DatacenterTableKind;
import tech.easyflow.datacenter.meta.model.DatacenterTableDetailMeta;
import tech.easyflow.datacenter.meta.service.DatacenterDatasetRegistryService;
import tech.easyflow.datacenter.meta.service.DatacenterSourceService;
import javax.annotation.Resource;
import java.io.File;
import java.io.FileOutputStream;
import java.io.InputStream;
import java.math.BigInteger;
import java.nio.file.Files;
import java.nio.file.Path;
import java.time.LocalDateTime;
import java.time.format.DateTimeFormatter;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedHashMap;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.UUID;
@Service
public class DatacenterExcelImportServiceImpl implements DatacenterExcelImportService {
private static final long QUERY_BATCH_SIZE = 500L;
private static final DateTimeFormatter EXPORT_TIME_FORMAT = DateTimeFormatter.ofPattern("yyyyMMddHHmmss");
@Resource
private DatacenterSourceService sourceService;
@Resource
private DatacenterDatasetRegistryService registryService;
@Resource
private DatacenterImportJobMapper importJobMapper;
@Resource
private DatacenterDatasetVersionMapper datasetVersionMapper;
@Resource
private DatacenterDerivedTableMapper derivedTableMapper;
@Resource
private DatacenterDatasetQueryService queryService;
@Resource
private DbHandleManager dbHandleManager;
@Override
@Transactional(rollbackFor = Exception.class)
public DatacenterImportJob importWorkbook(MultipartFile file, LoginAccount account) throws Exception {
if (file == null || file.isEmpty()) {
throw new BusinessException("Excel 文件不能为空");
}
String workbookName = extractWorkbookName(file.getOriginalFilename());
DatacenterSource source = new DatacenterSource();
source.setSourceName(workbookName);
source.setSourceCode("EXCEL_" + UUID.randomUUID());
source.setSourceType(DatacenterSourceType.EXCEL.name());
source.setAccessMode("READ_WRITE");
source.setBuiltinFlag(0);
source.setConfigJson(Map.of("originFileName", file.getOriginalFilename()));
source = sourceService.saveSource(source, account);
DatacenterCatalog catalog = registryService.ensureCatalog(source, workbookName, account);
DatacenterImportJob job = createJob("EXCEL_IMPORT", source.getId(), catalog.getId(), null,
file.getOriginalFilename(), Map.of("operation", "import"), account);
long totalRows = 0L;
long successRows = 0L;
List<BigInteger> createdTableIds = new ArrayList<>();
try (InputStream inputStream = file.getInputStream(); Workbook workbook = WorkbookFactory.create(inputStream)) {
DataFormatter formatter = new DataFormatter();
for (int sheetIndex = 0; sheetIndex < workbook.getNumberOfSheets(); sheetIndex++) {
Sheet sheet = workbook.getSheetAt(sheetIndex);
org.apache.poi.ss.usermodel.Row headerRow = sheet.getRow(sheet.getFirstRowNum());
if (headerRow == null) {
continue;
}
List<DatacenterTableField> fields = buildFields(headerRow, formatter);
if (fields.isEmpty()) {
continue;
}
DatacenterTable table = new DatacenterTable();
table.setTableName(uniqueTableName(source.getId(), catalog.getId(), sheet.getSheetName()));
table.setTableDesc(sheet.getSheetName());
table.setActualTable(buildMaterializedTableName(source.getId(), sheetIndex));
table.setMaterializedTable(table.getActualTable());
table.setTableKind(DatacenterTableKind.EXCEL_MATERIALIZED.name());
table.setAccessMode("READ_WRITE");
table.setVersioningEnabled(1);
table.setCapabilitiesJson(defaultExcelCapabilities());
table.setFields(fields);
DatacenterTableDetailMeta detail = new DatacenterTableDetailMeta();
detail.setTable(table);
detail.setFields(fields);
dbHandleManager.getDbHandler().createTable(table);
DatacenterTable savedTable = registryService.registerTable(source, catalog, detail, account);
savedTable.setFields(registryService.getFields(savedTable.getId()));
createdTableIds.add(savedTable.getId());
for (int rowIndex = sheet.getFirstRowNum() + 1; rowIndex <= sheet.getLastRowNum(); rowIndex++) {
org.apache.poi.ss.usermodel.Row row = sheet.getRow(rowIndex);
if (row == null) {
continue;
}
JSONObject payload = new JSONObject();
boolean hasValue = false;
for (int cellIndex = 0; cellIndex < savedTable.getFields().size(); cellIndex++) {
String value = formatter.formatCellValue(row.getCell(cellIndex));
if (value != null && !value.isBlank()) {
hasValue = true;
}
payload.put(savedTable.getFields().get(cellIndex).getFieldName(), value);
}
if (!hasValue) {
continue;
}
dbHandleManager.getDbHandler().saveValue(savedTable, payload, account);
totalRows++;
successRows++;
}
createVersion(savedTable, "initial-import", Map.of(
"sheetName", sheet.getSheetName(),
"sourceId", source.getId(),
"originFileName", file.getOriginalFilename()
), account);
}
job.setTableId(createdTableIds.isEmpty() ? null : createdTableIds.get(0));
finishJobSuccess(job, totalRows, successRows, Map.of("tableIds", createdTableIds));
return job;
} catch (Exception ex) {
finishJobFailure(job, ex);
throw ex;
}
}
@Override
@Transactional(rollbackFor = Exception.class)
public DatacenterImportJob splitWorkbook(DatacenterExcelSplitRequest request, LoginAccount account) {
DatasetRef datasetRef = request == null ? null : request.getDatasetRef();
DatacenterImportJob job = createJob("EXCEL_SPLIT",
request == null ? null : request.getSourceId(),
request == null ? null : request.getCatalogId(),
datasetRef == null ? null : datasetRef.getTableId(),
null,
buildPayload("request", request),
account);
try {
String splitMode = normalizeMode(request == null ? null : request.getSplitMode(), "BY_ROW_COUNT");
return switch (splitMode) {
case "BY_SHEET" -> doSplitBySheet(request, account, job);
case "BY_FIELD_VALUE" -> doSplitByFieldValue(request, account, job);
default -> doSplitByRowCount(request, account, job);
};
} catch (Exception ex) {
finishJobFailure(job, ex);
throw ex;
}
}
@Override
@Transactional(rollbackFor = Exception.class)
public DatacenterImportJob mergeWorkbook(DatacenterExcelMergeRequest request, LoginAccount account) {
if (request == null || CollectionUtils.isEmpty(request.getDatasetRefs())) {
throw new BusinessException("合并数据集不能为空");
}
DatacenterImportJob job = createJob("EXCEL_MERGE", null, null, null, null, buildPayload("request", request), account);
try {
String mergeMode = normalizeMode(request.getMergeMode(), "VERTICAL");
return switch (mergeMode) {
case "HORIZONTAL" -> doHorizontalMerge(request, account, job);
default -> doVerticalMerge(request, account, job);
};
} catch (Exception ex) {
finishJobFailure(job, ex);
throw ex;
}
}
@Override
@Transactional(rollbackFor = Exception.class)
public DatacenterImportJob deriveWorkbook(DatacenterExcelDeriveRequest request, LoginAccount account) {
if (request == null || request.getDatasetRef() == null) {
throw new BusinessException("派生数据集不能为空");
}
DatacenterImportJob job = createJob("EXCEL_DERIVE",
request.getDatasetRef().getSourceId(),
request.getDatasetRef().getCatalogId(),
request.getDatasetRef().getTableId(),
null,
buildPayload("request", request),
account);
try {
DatacenterTable sourceTable = resolveTable(request.getDatasetRef());
DatacenterSource source = registryService.getSourceRequired(sourceTable.getSourceId());
DatacenterCatalog catalog = requireCatalog(sourceTable.getCatalogId());
List<String> selectedColumns = resolveSelectedColumns(sourceTable, request.getSelectedColumns());
List<DatacenterTableField> targetFields = buildDerivedFields(sourceTable, request);
DatacenterTable targetTable = createDerivedTable(source, catalog, targetFields,
request.getTargetTableName(), "DERIVE", Map.of("sourceTableId", sourceTable.getId()), account);
DatacenterQueryRequest queryRequest = new DatacenterQueryRequest();
queryRequest.setDatasetRef(registryService.resolveDatasetRef(sourceTable.getId()));
queryRequest.setSelectedColumns(selectedColumns);
queryRequest.setFilters(request.getFilters());
long successRows = copyRows(queryRequest, rows -> mapDerivedRow(rows, selectedColumns, request), targetTable, account);
createLineage(sourceTable.getId(), targetTable.getId(), "DERIVE", Map.of("request", request), account);
finishJobSuccess(job, successRows, successRows, Map.of("derivedTableId", targetTable.getId()));
job.setTableId(targetTable.getId());
return job;
} catch (Exception ex) {
finishJobFailure(job, ex);
throw ex;
}
}
@Override
@Transactional(rollbackFor = Exception.class)
public DatacenterImportJob exportWorkbook(DatacenterExcelExportRequest request, LoginAccount account) throws Exception {
List<DatacenterTable> tables = resolveExportTables(request);
if (tables.isEmpty()) {
throw new BusinessException("没有可导出的 Excel 数据集");
}
String fileName = buildExportFileName(request == null ? null : request.getFileName());
DatacenterImportJob job = createJob("EXCEL_EXPORT",
request == null ? null : request.getSourceId(),
request == null ? null : request.getCatalogId(),
null,
fileName,
buildPayload("request", request),
account);
Path exportDir = ensureExportDir();
Path exportFile = exportDir.resolve(fileName);
long totalRows = 0L;
try (SXSSFWorkbook workbook = new SXSSFWorkbook(200); FileOutputStream outputStream = new FileOutputStream(exportFile.toFile())) {
workbook.setCompressTempFiles(true);
Set<String> usedSheetNames = new HashSet<>();
for (DatacenterTable table : tables) {
String sheetName = uniqueSheetName(table.getTableName(), usedSheetNames);
org.apache.poi.ss.usermodel.Sheet sheet = workbook.createSheet(sheetName);
writeHeaderRow(sheet, table.getFields());
DatacenterQueryRequest queryRequest = new DatacenterQueryRequest();
queryRequest.setDatasetRef(registryService.resolveDatasetRef(table.getId()));
queryRequest.setSelectedColumns(table.getFields().stream().map(DatacenterTableField::getFieldName).toList());
final int[] rowIndex = {1};
totalRows += iterateRows(queryRequest, row -> {
org.apache.poi.ss.usermodel.Row excelRow = sheet.createRow(rowIndex[0]++);
for (int i = 0; i < table.getFields().size(); i++) {
Cell cell = excelRow.createCell(i);
Object value = row.get(table.getFields().get(i).getFieldName());
cell.setCellValue(value == null ? "" : String.valueOf(value));
}
});
}
workbook.write(outputStream);
workbook.dispose();
job.setStoragePath(exportFile.toAbsolutePath().toString());
finishJobSuccess(job, totalRows, totalRows, Map.of("storagePath", job.getStoragePath(), "fileName", fileName));
return job;
} catch (Exception ex) {
finishJobFailure(job, ex);
throw ex;
}
}
@Override
public DatacenterImportJob getImportJobDetail(BigInteger jobId) {
DatacenterImportJob job = importJobMapper.selectOneById(jobId);
if (job == null) {
throw new BusinessException("导入任务不存在: " + jobId);
}
return job;
}
@Override
public List<DatacenterImportJob> listJobs(BigInteger sourceId, BigInteger tableId) {
var wrapper = com.mybatisflex.core.query.QueryWrapper.create();
if (sourceId != null) {
wrapper.eq(DatacenterImportJob::getSourceId, sourceId);
}
if (tableId != null) {
wrapper.eq(DatacenterImportJob::getTableId, tableId);
}
wrapper.orderBy("created desc");
wrapper.limit(20L);
return importJobMapper.selectListByQuery(wrapper);
}
private DatacenterImportJob doSplitBySheet(DatacenterExcelSplitRequest request, LoginAccount account, DatacenterImportJob job) {
BigInteger sourceId = request.getSourceId();
BigInteger catalogId = request.getCatalogId();
if (sourceId == null && request.getDatasetRef() != null) {
sourceId = request.getDatasetRef().getSourceId();
catalogId = request.getDatasetRef().getCatalogId();
}
if (sourceId == null) {
throw new BusinessException("按 sheet 拆分需要 sourceId");
}
DatacenterSource source = registryService.getSourceRequired(sourceId);
DatacenterCatalog catalog = requireCatalog(catalogId);
List<DatacenterTable> sourceTables = registryService.listManagedTables(sourceId, catalogId);
if (sourceTables.isEmpty()) {
throw new BusinessException("当前 workbook 下没有可拆分的 sheet 表");
}
List<BigInteger> derivedIds = new ArrayList<>();
long successRows = 0L;
for (DatacenterTable sourceTable : sourceTables) {
DatacenterTable fullTable = registryService.getTableWithFields(sourceTable.getId());
DatacenterTable targetTable = createDerivedTable(source, catalog, cloneFields(fullTable.getFields()),
resolveSplitPrefix(request, fullTable.getTableName()) + "_copy", "SPLIT_BY_SHEET",
Map.of("sourceTableId", fullTable.getId()), account);
DatacenterQueryRequest queryRequest = new DatacenterQueryRequest();
queryRequest.setDatasetRef(registryService.resolveDatasetRef(fullTable.getId()));
queryRequest.setSelectedColumns(fullTable.getFields().stream().map(DatacenterTableField::getFieldName).toList());
successRows += copyRows(queryRequest, this::mapRow, targetTable, account);
createLineage(fullTable.getId(), targetTable.getId(), "SPLIT_BY_SHEET", Map.of("sourceTableId", fullTable.getId()), account);
derivedIds.add(targetTable.getId());
}
finishJobSuccess(job, successRows, successRows, Map.of("derivedTableIds", derivedIds));
return job;
}
private DatacenterImportJob doSplitByRowCount(DatacenterExcelSplitRequest request, LoginAccount account, DatacenterImportJob job) {
if (request == null || request.getDatasetRef() == null) {
throw new BusinessException("按行数拆分需要数据集");
}
int rowBatchSize = request.getRowBatchSize() == null || request.getRowBatchSize() < 1 ? 1000 : request.getRowBatchSize();
DatacenterTable sourceTable = resolveTable(request.getDatasetRef());
DatacenterSource source = registryService.getSourceRequired(sourceTable.getSourceId());
DatacenterCatalog catalog = requireCatalog(sourceTable.getCatalogId());
String baseName = resolveSplitPrefix(request, sourceTable.getTableName());
List<BigInteger> derivedIds = new ArrayList<>();
final Holder holder = new Holder();
long totalRows = iterateRows(buildFullQuery(sourceTable), row -> {
if (holder.targetTable == null || holder.currentSize >= rowBatchSize) {
holder.batchNo++;
holder.targetTable = createDerivedTable(source, catalog, cloneFields(sourceTable.getFields()),
baseName + "_part_" + holder.batchNo, "SPLIT_BY_ROW_COUNT",
Map.of("sourceTableId", sourceTable.getId(), "batchNo", holder.batchNo, "rowBatchSize", rowBatchSize), account);
createLineage(sourceTable.getId(), holder.targetTable.getId(), "SPLIT_BY_ROW_COUNT",
Map.of("sourceTableId", sourceTable.getId(), "batchNo", holder.batchNo), account);
derivedIds.add(holder.targetTable.getId());
holder.currentSize = 0;
}
saveToTable(holder.targetTable, mapRow(row), account);
holder.currentSize++;
});
finishJobSuccess(job, totalRows, totalRows, Map.of("derivedTableIds", derivedIds));
return job;
}
private DatacenterImportJob doSplitByFieldValue(DatacenterExcelSplitRequest request, LoginAccount account, DatacenterImportJob job) {
if (request == null || request.getDatasetRef() == null || request.getFieldName() == null || request.getFieldName().isBlank()) {
throw new BusinessException("按字段值拆分需要数据集和字段名");
}
DatacenterTable sourceTable = resolveTable(request.getDatasetRef());
DatacenterSource source = registryService.getSourceRequired(sourceTable.getSourceId());
DatacenterCatalog catalog = requireCatalog(sourceTable.getCatalogId());
DatacenterTableField splitField = sourceTable.getFields().stream()
.filter(field -> request.getFieldName().equals(field.getFieldName()))
.findFirst()
.orElseThrow(() -> new BusinessException("拆分字段不存在: " + request.getFieldName()));
String prefix = resolveSplitPrefix(request, sourceTable.getTableName());
Map<String, DatacenterTable> targets = new LinkedHashMap<>();
List<BigInteger> derivedIds = new ArrayList<>();
long totalRows = iterateRows(buildFullQuery(sourceTable), row -> {
String fieldValue = stringify(row.get(splitField.getFieldName()));
String bucket = fieldValue == null || fieldValue.isBlank() ? "empty" : fieldValue;
DatacenterTable targetTable = targets.get(bucket);
if (targetTable == null) {
targetTable = createDerivedTable(source, catalog, cloneFields(sourceTable.getFields()),
prefix + "_" + normalizeIdentifier(bucket), "SPLIT_BY_FIELD_VALUE",
Map.of("sourceTableId", sourceTable.getId(), "fieldName", splitField.getFieldName(), "fieldValue", bucket), account);
createLineage(sourceTable.getId(), targetTable.getId(), "SPLIT_BY_FIELD_VALUE",
Map.of("sourceTableId", sourceTable.getId(), "fieldName", splitField.getFieldName(), "fieldValue", bucket), account);
targets.put(bucket, targetTable);
derivedIds.add(targetTable.getId());
}
saveToTable(targetTable, mapRow(row), account);
});
finishJobSuccess(job, totalRows, totalRows, Map.of("derivedTableIds", derivedIds));
return job;
}
private DatacenterImportJob doVerticalMerge(DatacenterExcelMergeRequest request, LoginAccount account, DatacenterImportJob job) {
List<DatacenterTable> tables = request.getDatasetRefs().stream().map(this::resolveTable).toList();
DatacenterTable firstTable = tables.get(0);
DatacenterSource source = registryService.getSourceRequired(firstTable.getSourceId());
DatacenterCatalog catalog = requireCatalog(firstTable.getCatalogId());
assertSameCatalog(tables);
assertSameFields(tables);
DatacenterTable targetTable = createDerivedTable(source, catalog, cloneFields(firstTable.getFields()),
request.getTargetTableName(), "MERGE_VERTICAL", Map.of("sourceTableIds", tables.stream().map(DatacenterTable::getId).toList()), account);
long successRows = 0L;
for (DatacenterTable table : tables) {
successRows += copyRows(buildFullQuery(table), this::mapRow, targetTable, account);
createLineage(table.getId(), targetTable.getId(), "MERGE_VERTICAL", Map.of("sourceTableId", table.getId()), account);
}
job.setTableId(targetTable.getId());
finishJobSuccess(job, successRows, successRows, Map.of("derivedTableId", targetTable.getId()));
return job;
}
private DatacenterImportJob doHorizontalMerge(DatacenterExcelMergeRequest request, LoginAccount account, DatacenterImportJob job) {
if (request.getJoinKey() == null || request.getJoinKey().isBlank()) {
throw new BusinessException("横向合并必须指定 joinKey");
}
List<DatacenterTable> tables = request.getDatasetRefs().stream().map(this::resolveTable).toList();
DatacenterTable firstTable = tables.get(0);
DatacenterSource source = registryService.getSourceRequired(firstTable.getSourceId());
DatacenterCatalog catalog = requireCatalog(firstTable.getCatalogId());
assertSameCatalog(tables);
List<DatacenterTableField> mergedFields = new ArrayList<>();
Set<String> usedFieldNames = new LinkedHashSet<>();
Map<BigInteger, Map<String, String>> fieldMappings = new LinkedHashMap<>();
for (DatacenterTable table : tables) {
Map<String, String> mapping = new LinkedHashMap<>();
for (DatacenterTableField field : table.getFields()) {
String targetFieldName;
if (field.getFieldName().equals(request.getJoinKey())) {
targetFieldName = field.getFieldName();
} else {
targetFieldName = field.getFieldName();
if (usedFieldNames.contains(targetFieldName)) {
targetFieldName = normalizeIdentifier(table.getTableName()) + "_" + targetFieldName;
}
}
if (!usedFieldNames.contains(targetFieldName)) {
usedFieldNames.add(targetFieldName);
mergedFields.add(cloneField(field, targetFieldName, field.getFieldDesc()));
}
mapping.put(field.getFieldName(), targetFieldName);
}
fieldMappings.put(table.getId(), mapping);
}
DatacenterTable targetTable = createDerivedTable(source, catalog, mergedFields,
request.getTargetTableName(), "MERGE_HORIZONTAL", Map.of("sourceTableIds", tables.stream().map(DatacenterTable::getId).toList(), "joinKey", request.getJoinKey()), account);
Map<String, JSONObject> mergedRows = new LinkedHashMap<>();
for (DatacenterTable table : tables) {
Map<String, String> mapping = fieldMappings.get(table.getId());
iterateRows(buildFullQuery(table), row -> {
String joinValue = stringify(row.get(request.getJoinKey()));
if (joinValue == null || joinValue.isBlank()) {
return;
}
JSONObject target = mergedRows.computeIfAbsent(joinValue, key -> new JSONObject());
mapping.forEach((sourceField, targetField) -> target.put(targetField, row.get(sourceField)));
});
createLineage(table.getId(), targetTable.getId(), "MERGE_HORIZONTAL", Map.of("sourceTableId", table.getId(), "joinKey", request.getJoinKey()), account);
}
for (JSONObject row : mergedRows.values()) {
saveToTable(targetTable, row, account);
}
job.setTableId(targetTable.getId());
finishJobSuccess(job, (long) mergedRows.size(), (long) mergedRows.size(), Map.of("derivedTableId", targetTable.getId()));
return job;
}
private DatacenterImportJob createJob(String jobType, BigInteger sourceId, BigInteger catalogId, BigInteger tableId,
String fileName, Map<String, Object> payload, LoginAccount account) {
DatacenterImportJob job = new DatacenterImportJob();
job.setSourceId(sourceId);
job.setCatalogId(catalogId);
job.setTableId(tableId);
job.setTenantId(account == null || account.getTenantId() == null ? BigInteger.ZERO : account.getTenantId());
job.setDeptId(account == null || account.getDeptId() == null ? BigInteger.ZERO : account.getDeptId());
job.setJobType(jobType);
job.setFileName(fileName);
job.setStatus(DatacenterImportStatus.RUNNING.name());
job.setPayloadJson(payload == null ? new LinkedHashMap<>() : new LinkedHashMap<>(payload));
job.setStartedAt(new Date());
job.setCreated(new Date());
job.setModified(new Date());
job.setCreatedBy(account == null ? BigInteger.ZERO : account.getId());
job.setModifiedBy(account == null ? BigInteger.ZERO : account.getId());
importJobMapper.insert(job);
return job;
}
private Map<String, Object> buildPayload(String key, Object value) {
Map<String, Object> payload = new LinkedHashMap<>();
payload.put(key, value);
return payload;
}
private void finishJobSuccess(DatacenterImportJob job, long totalRows, long successRows, Map<String, Object> payload) {
job.setStatus(DatacenterImportStatus.SUCCESS.name());
job.setTotalRows(totalRows);
job.setSuccessRows(successRows);
job.setErrorRows(Math.max(0L, totalRows - successRows));
if (payload != null) {
job.setPayloadJson(new LinkedHashMap<>(payload));
}
job.setFinishedAt(new Date());
job.setModified(new Date());
importJobMapper.update(job);
}
private void finishJobFailure(DatacenterImportJob job, Exception ex) {
job.setStatus(DatacenterImportStatus.FAILED.name());
job.setErrorSummary(ex.getMessage());
job.setFinishedAt(new Date());
job.setModified(new Date());
importJobMapper.update(job);
}
private DatacenterTable resolveTable(DatasetRef datasetRef) {
if (datasetRef == null || datasetRef.getTableId() == null) {
throw new BusinessException("缺少数据集 tableId");
}
return registryService.getTableWithFields(datasetRef.getTableId());
}
private DatacenterCatalog requireCatalog(BigInteger catalogId) {
DatacenterCatalog catalog = registryService.getCatalogById(catalogId);
if (catalog == null) {
throw new BusinessException("目录不存在: " + catalogId);
}
return catalog;
}
private DatacenterTable createDerivedTable(DatacenterSource source, DatacenterCatalog catalog, List<DatacenterTableField> fields,
String tableName, String deriveType, Map<String, Object> config, LoginAccount account) {
DatacenterTable table = new DatacenterTable();
String resolvedName = uniqueTableName(source.getId(), catalog.getId(), normalizeLogicalName(tableName, deriveType));
table.setTableName(resolvedName);
table.setTableDesc(resolvedName);
table.setActualTable(buildMaterializedTableName(source.getId(), Math.abs(Objects.hash(resolvedName, deriveType))));
table.setMaterializedTable(table.getActualTable());
table.setTableKind(DatacenterTableKind.DERIVED_TABLE.name());
table.setAccessMode("READ_WRITE");
table.setVersioningEnabled(1);
table.setCapabilitiesJson(defaultExcelCapabilities());
table.setFields(fields);
DatacenterTableDetailMeta detail = new DatacenterTableDetailMeta();
detail.setTable(table);
detail.setFields(fields);
dbHandleManager.getDbHandler().createTable(table);
DatacenterTable savedTable = registryService.registerTable(source, catalog, detail, account);
savedTable.setFields(registryService.getFields(savedTable.getId()));
createVersion(savedTable, deriveType.toLowerCase(Locale.ROOT), config, account);
return savedTable;
}
private DatacenterDatasetVersion createVersion(DatacenterTable table, String versionLabel, Map<String, Object> snapshot, LoginAccount account) {
QueryWrapperWrapper wrapper = new QueryWrapperWrapper(table.getId());
DatacenterDatasetVersion version = new DatacenterDatasetVersion();
version.setTableId(table.getId());
version.setTenantId(table.getTenantId());
version.setDeptId(table.getDeptId());
version.setVersionNo(wrapper.nextVersionNo(datasetVersionMapper));
version.setVersionLabel(versionLabel);
version.setMaterializedTable(table.getMaterializedTable());
version.setSnapshotJson(snapshot == null ? new LinkedHashMap<>() : new LinkedHashMap<>(snapshot));
version.setStatus(0);
version.setCreated(new Date());
version.setModified(new Date());
version.setCreatedBy(account == null ? BigInteger.ZERO : account.getId());
version.setModifiedBy(account == null ? BigInteger.ZERO : account.getId());
datasetVersionMapper.insert(version);
return version;
}
private void createLineage(BigInteger sourceTableId, BigInteger derivedTableId, String deriveType, Map<String, Object> config, LoginAccount account) {
DatacenterDerivedTable relation = new DatacenterDerivedTable();
relation.setSourceTableId(sourceTableId);
relation.setDerivedTableId(derivedTableId);
relation.setDeriveType(deriveType);
relation.setDeriveConfigJson(config == null ? new LinkedHashMap<>() : new LinkedHashMap<>(config));
relation.setStatus(0);
relation.setTenantId(account == null || account.getTenantId() == null ? BigInteger.ZERO : account.getTenantId());
relation.setDeptId(account == null || account.getDeptId() == null ? BigInteger.ZERO : account.getDeptId());
relation.setCreated(new Date());
relation.setModified(new Date());
relation.setCreatedBy(account == null ? BigInteger.ZERO : account.getId());
relation.setModifiedBy(account == null ? BigInteger.ZERO : account.getId());
derivedTableMapper.insert(relation);
}
private long copyRows(DatacenterQueryRequest queryRequest, RowMapper mapper, DatacenterTable targetTable, LoginAccount account) {
return iterateRows(queryRequest, row -> saveToTable(targetTable, mapper.map(row), account));
}
private long iterateRows(DatacenterQueryRequest queryRequest, RowConsumer consumer) {
long total = 0L;
long pageNumber = 1L;
while (true) {
queryRequest.setPageNumber(pageNumber);
queryRequest.setPageSize(QUERY_BATCH_SIZE);
Page<Row> page = queryService.queryPage(queryRequest);
if (page.getRecords() == null || page.getRecords().isEmpty()) {
break;
}
for (Row row : page.getRecords()) {
consumer.accept(row);
total++;
}
if (page.getRecords().size() < QUERY_BATCH_SIZE) {
break;
}
pageNumber++;
}
return total;
}
private void saveToTable(DatacenterTable targetTable, JSONObject data, LoginAccount account) {
dbHandleManager.getDbHandler().saveValue(targetTable, data, account);
}
private DatacenterQueryRequest buildFullQuery(DatacenterTable table) {
DatacenterQueryRequest queryRequest = new DatacenterQueryRequest();
queryRequest.setDatasetRef(registryService.resolveDatasetRef(table.getId()));
queryRequest.setSelectedColumns(table.getFields().stream().map(DatacenterTableField::getFieldName).toList());
return queryRequest;
}
private JSONObject mapRow(Row row) {
JSONObject payload = new JSONObject();
row.forEach(payload::put);
payload.remove("id");
payload.remove("dept_id");
payload.remove("tenant_id");
payload.remove("created");
payload.remove("created_by");
payload.remove("modified");
payload.remove("modified_by");
payload.remove("remark");
return payload;
}
private JSONObject mapDerivedRow(Row row, List<String> selectedColumns, DatacenterExcelDeriveRequest request) {
JSONObject payload = new JSONObject();
for (String column : selectedColumns) {
String targetName = request.getRenameMappings().getOrDefault(column, column);
payload.put(targetName, row.get(column));
}
return payload;
}
private List<String> resolveSelectedColumns(DatacenterTable sourceTable, List<String> selectedColumns) {
if (CollectionUtils.isEmpty(selectedColumns)) {
return sourceTable.getFields().stream().map(DatacenterTableField::getFieldName).toList();
}
return selectedColumns;
}
private List<DatacenterTableField> buildDerivedFields(DatacenterTable sourceTable, DatacenterExcelDeriveRequest request) {
List<String> selectedColumns = resolveSelectedColumns(sourceTable, request.getSelectedColumns());
Map<String, DatacenterTableField> fieldMap = new LinkedHashMap<>();
for (DatacenterTableField field : sourceTable.getFields()) {
fieldMap.put(field.getFieldName(), field);
}
List<DatacenterTableField> fields = new ArrayList<>();
for (String column : selectedColumns) {
DatacenterTableField sourceField = fieldMap.get(column);
if (sourceField == null) {
throw new BusinessException("派生字段不存在: " + column);
}
String targetName = request.getRenameMappings().getOrDefault(column, column);
fields.add(cloneField(sourceField, targetName, targetName));
}
return fields;
}
private List<DatacenterTableField> cloneFields(List<DatacenterTableField> sourceFields) {
List<DatacenterTableField> fields = new ArrayList<>();
for (DatacenterTableField field : sourceFields) {
fields.add(cloneField(field, field.getFieldName(), field.getFieldDesc()));
}
return fields;
}
private DatacenterTableField cloneField(DatacenterTableField source, String fieldName, String fieldDesc) {
DatacenterTableField field = new DatacenterTableField();
field.setFieldName(fieldName);
field.setSourceColumnName(source.getSourceColumnName());
field.setFieldDesc(fieldDesc);
field.setFieldType(source.getFieldType());
field.setJdbcType(source.getJdbcType());
field.setPrecision(source.getPrecision());
field.setScale(source.getScale());
field.setRequired(source.getRequired());
field.setQueryable(source.getQueryable());
field.setSortable(source.getSortable());
field.setWritable(source.getWritable());
field.setIndexed(source.getIndexed());
field.setOptions(source.getOptions());
return field;
}
private Map<String, Object> defaultExcelCapabilities() {
return Map.of("capabilities", List.of("READ_QUERY", "WRITE_MUTATION", "MATERIALIZE", "EXPORT"));
}
private void assertSameCatalog(List<DatacenterTable> tables) {
Set<BigInteger> catalogIds = new HashSet<>();
Set<BigInteger> sourceIds = new HashSet<>();
for (DatacenterTable table : tables) {
catalogIds.add(table.getCatalogId());
sourceIds.add(table.getSourceId());
}
if (catalogIds.size() > 1 || sourceIds.size() > 1) {
throw new BusinessException("Excel 操作暂只支持同一 workbook/catalog 下的数据集");
}
}
private void assertSameFields(List<DatacenterTable> tables) {
List<String> first = tables.get(0).getFields().stream().map(DatacenterTableField::getFieldName).toList();
for (int i = 1; i < tables.size(); i++) {
List<String> current = tables.get(i).getFields().stream().map(DatacenterTableField::getFieldName).toList();
if (!first.equals(current)) {
throw new BusinessException("纵向合并仅支持同结构表");
}
}
}
private List<DatacenterTable> resolveExportTables(DatacenterExcelExportRequest request) {
List<DatacenterTable> tables = new ArrayList<>();
if (request != null && !CollectionUtils.isEmpty(request.getDatasetRefs())) {
for (DatasetRef datasetRef : request.getDatasetRefs()) {
tables.add(resolveTable(datasetRef));
}
return tables;
}
if (request == null || request.getSourceId() == null) {
throw new BusinessException("导出需要 sourceId 或 datasetRefs");
}
tables.addAll(registryService.listManagedTables(request.getSourceId(), request.getCatalogId()));
return tables.stream().map(table -> registryService.getTableWithFields(table.getId())).toList();
}
private void writeHeaderRow(org.apache.poi.ss.usermodel.Sheet sheet, List<DatacenterTableField> fields) {
org.apache.poi.ss.usermodel.Row headerRow = sheet.createRow(0);
for (int i = 0; i < fields.size(); i++) {
Cell cell = headerRow.createCell(i);
cell.setCellValue(fields.get(i).getFieldDesc());
}
}
private Path ensureExportDir() throws Exception {
Path dir = Path.of(System.getProperty("java.io.tmpdir"), "easyflow-datacenter", "exports");
Files.createDirectories(dir);
return dir;
}
private String buildExportFileName(String rawFileName) {
String baseName = rawFileName == null || rawFileName.isBlank() ? "excel_export" : extractWorkbookName(rawFileName);
return normalizeIdentifier(baseName) + "_" + EXPORT_TIME_FORMAT.format(LocalDateTime.now()) + ".xlsx";
}
private String uniqueSheetName(String rawName, Set<String> usedSheetNames) {
String base = rawName == null || rawName.isBlank() ? "Sheet" : rawName;
base = base.length() > 31 ? base.substring(0, 31) : base;
String result = base;
int suffix = 1;
while (usedSheetNames.contains(result)) {
String suffixText = "_" + suffix++;
int limit = Math.max(1, 31 - suffixText.length());
result = base.substring(0, Math.min(base.length(), limit)) + suffixText;
}
usedSheetNames.add(result);
return result;
}
private List<DatacenterTableField> buildFields(org.apache.poi.ss.usermodel.Row headerRow, DataFormatter formatter) {
List<DatacenterTableField> fields = new ArrayList<>();
Set<String> usedNames = new HashSet<>();
short lastCellNum = headerRow.getLastCellNum();
for (int cellIndex = 0; cellIndex < lastCellNum; cellIndex++) {
String header = formatter.formatCellValue(headerRow.getCell(cellIndex));
String fieldName = normalizeIdentifier(header, cellIndex, usedNames);
DatacenterTableField field = new DatacenterTableField();
field.setFieldName(fieldName);
field.setSourceColumnName(header);
field.setFieldDesc(header == null || header.isBlank() ? fieldName : header);
field.setFieldType(EnumFieldType.STRING.getCode());
field.setJdbcType("VARCHAR");
field.setPrecision(255);
field.setScale(0);
field.setRequired(0);
field.setQueryable(1);
field.setSortable(1);
field.setWritable(1);
field.setIndexed(0);
fields.add(field);
}
return fields;
}
private String normalizeMode(String value, String defaultValue) {
return value == null || value.isBlank() ? defaultValue : value.trim().toUpperCase(Locale.ROOT);
}
private String resolveSplitPrefix(DatacenterExcelSplitRequest request, String fallback) {
if (request != null && request.getTargetNamePrefix() != null && !request.getTargetNamePrefix().isBlank()) {
return request.getTargetNamePrefix();
}
return fallback + "_split";
}
private String normalizeLogicalName(String tableName, String deriveType) {
if (tableName != null && !tableName.isBlank()) {
return tableName;
}
return deriveType.toLowerCase(Locale.ROOT) + "_" + System.currentTimeMillis();
}
private String uniqueTableName(BigInteger sourceId, BigInteger catalogId, String rawName) {
String baseName = rawName == null || rawName.isBlank() ? "dataset" : rawName;
baseName = baseName.trim();
String result = baseName;
int suffix = 1;
while (tableNameExists(sourceId, catalogId, result)) {
result = baseName + "_" + suffix++;
}
return result;
}
private boolean tableNameExists(BigInteger sourceId, BigInteger catalogId, String tableName) {
List<DatacenterTable> tables = registryService.listManagedTables(sourceId, catalogId);
return tables.stream().anyMatch(table -> tableName.equals(table.getTableName()));
}
private String extractWorkbookName(String originalFileName) {
if (originalFileName == null || originalFileName.isBlank()) {
return "excel_workbook";
}
int index = originalFileName.lastIndexOf('.');
return index > 0 ? originalFileName.substring(0, index) : originalFileName;
}
private String buildMaterializedTableName(BigInteger sourceId, int sheetIndex) {
long snowId = new SnowFlakeIDKeyGenerator().nextId();
return "tb_excel_" + sourceId + "_" + sheetIndex + "_" + snowId;
}
private String normalizeIdentifier(String raw) {
if (raw == null || raw.isBlank()) {
return "value";
}
String normalized = raw.trim().toLowerCase(Locale.ROOT).replaceAll("[^a-z0-9_\\u4e00-\\u9fa5]+", "_");
normalized = normalized.replaceAll("_+", "_");
if (normalized.isBlank()) {
return "value";
}
return normalized;
}
private String normalizeIdentifier(String raw, int index, Set<String> usedNames) {
String value = normalizeIdentifier(raw);
if (value.isBlank() || "value".equals(value)) {
value = "col_" + (index + 1);
}
if (Character.isDigit(value.charAt(0))) {
value = "col_" + value;
}
String result = value;
int suffix = 1;
while (usedNames.contains(result)) {
result = value + "_" + suffix++;
}
usedNames.add(result);
return result;
}
private String stringify(Object value) {
return value == null ? null : String.valueOf(value);
}
private static final class Holder {
private DatacenterTable targetTable;
private int batchNo;
private int currentSize;
}
private interface RowConsumer {
void accept(Row row);
}
private interface RowMapper {
JSONObject map(Row row);
}
private static final class QueryWrapperWrapper {
private final BigInteger tableId;
private QueryWrapperWrapper(BigInteger tableId) {
this.tableId = tableId;
}
private int nextVersionNo(DatacenterDatasetVersionMapper mapper) {
return mapper.selectListByQuery(com.mybatisflex.core.query.QueryWrapper.create()
.eq(DatacenterDatasetVersion::getTableId, tableId)
.orderBy("version_no desc"))
.stream()
.findFirst()
.map(version -> version.getVersionNo() + 1)
.orElse(1);
}
}
}

View File

@@ -0,0 +1,24 @@
package tech.easyflow.datacenter.execution.model;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
public class DatacenterConnectionTestResult {
private boolean success;
private String errorCode;
private String message;
private List<String> capabilities = new ArrayList<>();
private Map<String, Object> details;
public boolean isSuccess() { return success; }
public void setSuccess(boolean success) { this.success = success; }
public String getErrorCode() { return errorCode; }
public void setErrorCode(String errorCode) { this.errorCode = errorCode; }
public String getMessage() { return message; }
public void setMessage(String message) { this.message = message; }
public List<String> getCapabilities() { return capabilities; }
public void setCapabilities(List<String> capabilities) { this.capabilities = capabilities; }
public Map<String, Object> getDetails() { return details; }
public void setDetails(Map<String, Object> details) { this.details = details; }
}

View File

@@ -0,0 +1,19 @@
package tech.easyflow.datacenter.execution.model;
import java.util.List;
public class DatacenterQueryFilter {
private String column;
private String operator;
private Object value;
private List<Object> values;
public String getColumn() { return column; }
public void setColumn(String column) { this.column = column; }
public String getOperator() { return operator; }
public void setOperator(String operator) { this.operator = operator; }
public Object getValue() { return value; }
public void setValue(Object value) { this.value = value; }
public List<Object> getValues() { return values; }
public void setValues(List<Object> values) { this.values = values; }
}

View File

@@ -0,0 +1,29 @@
package tech.easyflow.datacenter.execution.model;
import java.util.ArrayList;
import java.util.List;
public class DatacenterQueryRequest {
private DatasetRef datasetRef;
private Long pageNumber = 1L;
private Long pageSize = 10L;
private List<DatacenterQueryFilter> filters = new ArrayList<>();
private List<DatacenterQuerySort> sorts = new ArrayList<>();
private List<String> selectedColumns = new ArrayList<>();
private String where;
public DatasetRef getDatasetRef() { return datasetRef; }
public void setDatasetRef(DatasetRef datasetRef) { this.datasetRef = datasetRef; }
public Long getPageNumber() { return pageNumber; }
public void setPageNumber(Long pageNumber) { this.pageNumber = pageNumber; }
public Long getPageSize() { return pageSize; }
public void setPageSize(Long pageSize) { this.pageSize = pageSize; }
public List<DatacenterQueryFilter> getFilters() { return filters; }
public void setFilters(List<DatacenterQueryFilter> filters) { this.filters = filters; }
public List<DatacenterQuerySort> getSorts() { return sorts; }
public void setSorts(List<DatacenterQuerySort> sorts) { this.sorts = sorts; }
public List<String> getSelectedColumns() { return selectedColumns; }
public void setSelectedColumns(List<String> selectedColumns) { this.selectedColumns = selectedColumns; }
public String getWhere() { return where; }
public void setWhere(String where) { this.where = where; }
}

View File

@@ -0,0 +1,11 @@
package tech.easyflow.datacenter.execution.model;
public class DatacenterQuerySort {
private String column;
private String direction;
public String getColumn() { return column; }
public void setColumn(String column) { this.column = column; }
public String getDirection() { return direction; }
public void setDirection(String direction) { this.direction = direction; }
}

View File

@@ -0,0 +1,39 @@
package tech.easyflow.datacenter.execution.model;
import tech.easyflow.datacenter.entity.DatacenterTable;
import tech.easyflow.datacenter.entity.DatacenterTableField;
import tech.easyflow.datacenter.meta.entity.DatacenterCatalog;
import tech.easyflow.datacenter.meta.entity.DatacenterDatasetVersion;
import tech.easyflow.datacenter.meta.entity.DatacenterDerivedTable;
import tech.easyflow.datacenter.meta.entity.DatacenterSource;
import java.util.ArrayList;
import java.util.List;
public class DatacenterSchemaResponse {
private DatasetRef datasetRef;
private DatacenterSource source;
private DatacenterCatalog catalog;
private DatacenterTable table;
private List<DatacenterTableField> fields = new ArrayList<>();
private List<DatacenterDatasetVersion> versions = new ArrayList<>();
private List<DatacenterDerivedTable> upstreamLineage = new ArrayList<>();
private List<DatacenterDerivedTable> downstreamLineage = new ArrayList<>();
public DatasetRef getDatasetRef() { return datasetRef; }
public void setDatasetRef(DatasetRef datasetRef) { this.datasetRef = datasetRef; }
public DatacenterSource getSource() { return source; }
public void setSource(DatacenterSource source) { this.source = source; }
public DatacenterCatalog getCatalog() { return catalog; }
public void setCatalog(DatacenterCatalog catalog) { this.catalog = catalog; }
public DatacenterTable getTable() { return table; }
public void setTable(DatacenterTable table) { this.table = table; }
public List<DatacenterTableField> getFields() { return fields; }
public void setFields(List<DatacenterTableField> fields) { this.fields = fields; }
public List<DatacenterDatasetVersion> getVersions() { return versions; }
public void setVersions(List<DatacenterDatasetVersion> versions) { this.versions = versions; }
public List<DatacenterDerivedTable> getUpstreamLineage() { return upstreamLineage; }
public void setUpstreamLineage(List<DatacenterDerivedTable> upstreamLineage) { this.upstreamLineage = upstreamLineage; }
public List<DatacenterDerivedTable> getDownstreamLineage() { return downstreamLineage; }
public void setDownstreamLineage(List<DatacenterDerivedTable> downstreamLineage) { this.downstreamLineage = downstreamLineage; }
}

View File

@@ -0,0 +1,23 @@
package tech.easyflow.datacenter.execution.model;
public class DatacenterSqlQueryRequest {
private DatasetRef datasetRef;
private String sql;
public DatasetRef getDatasetRef() {
return datasetRef;
}
public void setDatasetRef(DatasetRef datasetRef) {
this.datasetRef = datasetRef;
}
public String getSql() {
return sql;
}
public void setSql(String sql) {
this.sql = sql;
}
}

View File

@@ -0,0 +1,25 @@
package tech.easyflow.datacenter.execution.model;
import java.math.BigInteger;
public class DatasetRef {
private BigInteger sourceId;
private BigInteger catalogId;
private String catalogName;
private BigInteger tableId;
private String tableName;
private BigInteger versionId;
public BigInteger getSourceId() { return sourceId; }
public void setSourceId(BigInteger sourceId) { this.sourceId = sourceId; }
public BigInteger getCatalogId() { return catalogId; }
public void setCatalogId(BigInteger catalogId) { this.catalogId = catalogId; }
public String getCatalogName() { return catalogName; }
public void setCatalogName(String catalogName) { this.catalogName = catalogName; }
public BigInteger getTableId() { return tableId; }
public void setTableId(BigInteger tableId) { this.tableId = tableId; }
public String getTableName() { return tableName; }
public void setTableName(String tableName) { this.tableName = tableName; }
public BigInteger getVersionId() { return versionId; }
public void setVersionId(BigInteger versionId) { this.versionId = versionId; }
}

View File

@@ -0,0 +1,18 @@
package tech.easyflow.datacenter.execution.service;
import com.mybatisflex.core.paginate.Page;
import com.mybatisflex.core.row.Row;
import tech.easyflow.datacenter.execution.model.DatacenterQueryRequest;
import tech.easyflow.datacenter.execution.model.DatacenterSchemaResponse;
import tech.easyflow.datacenter.execution.model.DatacenterSqlQueryRequest;
import tech.easyflow.datacenter.execution.model.DatasetRef;
import java.util.List;
public interface DatacenterDatasetQueryService {
Page<Row> queryPage(DatacenterQueryRequest request);
List<Row> queryBySql(DatacenterSqlQueryRequest request);
DatacenterSchemaResponse getSchema(DatasetRef datasetRef);
}

View File

@@ -0,0 +1,13 @@
package tech.easyflow.datacenter.execution.service;
import com.alibaba.fastjson2.JSONObject;
import tech.easyflow.common.entity.LoginAccount;
import tech.easyflow.datacenter.execution.model.DatasetRef;
import java.math.BigInteger;
public interface DatacenterDatasetWriteService {
void saveRow(DatasetRef datasetRef, JSONObject data, LoginAccount account);
void deleteRow(DatasetRef datasetRef, BigInteger id, LoginAccount account);
}

View File

@@ -0,0 +1,311 @@
package tech.easyflow.datacenter.execution.service.impl;
import com.mybatisflex.core.paginate.Page;
import com.mybatisflex.core.query.QueryWrapper;
import com.mybatisflex.core.row.Row;
import org.springframework.stereotype.Service;
import org.springframework.util.CollectionUtils;
import org.springframework.util.StringUtils;
import tech.easyflow.common.web.exceptions.BusinessException;
import tech.easyflow.datacenter.connector.DatacenterConnector;
import tech.easyflow.datacenter.connector.DatacenterConnectorRegistry;
import tech.easyflow.datacenter.entity.DatacenterTable;
import tech.easyflow.datacenter.entity.DatacenterTableField;
import tech.easyflow.datacenter.execution.model.DatacenterQueryRequest;
import tech.easyflow.datacenter.execution.model.DatacenterSchemaResponse;
import tech.easyflow.datacenter.execution.model.DatacenterSqlQueryRequest;
import tech.easyflow.datacenter.execution.model.DatasetRef;
import tech.easyflow.datacenter.execution.service.DatacenterDatasetQueryService;
import tech.easyflow.datacenter.mapper.DatacenterCatalogMapper;
import tech.easyflow.datacenter.mapper.DatacenterDatasetVersionMapper;
import tech.easyflow.datacenter.mapper.DatacenterDerivedTableMapper;
import tech.easyflow.datacenter.mapper.DatacenterTableMapper;
import tech.easyflow.datacenter.meta.entity.DatacenterCatalog;
import tech.easyflow.datacenter.meta.entity.DatacenterDatasetVersion;
import tech.easyflow.datacenter.meta.entity.DatacenterDerivedTable;
import tech.easyflow.datacenter.meta.entity.DatacenterSource;
import tech.easyflow.datacenter.meta.service.DatacenterDatasetRegistryService;
import tech.easyflow.datacenter.utils.SqlSupportUtils;
import javax.annotation.Resource;
import java.math.BigInteger;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
@Service
public class DatacenterDatasetQueryServiceImpl implements DatacenterDatasetQueryService {
@Resource
private DatacenterDatasetRegistryService registryService;
@Resource
private DatacenterConnectorRegistry connectorRegistry;
@Resource
private DatacenterTableMapper tableMapper;
@Resource
private DatacenterCatalogMapper catalogMapper;
@Resource
private DatacenterDatasetVersionMapper datasetVersionMapper;
@Resource
private DatacenterDerivedTableMapper derivedTableMapper;
@Override
public Page<Row> queryPage(DatacenterQueryRequest request) {
if (request == null || request.getDatasetRef() == null) {
throw new BusinessException("datasetRef 不能为空");
}
normalizePage(request);
DatacenterTable table = resolveTable(request.getDatasetRef());
DatacenterSource source = registryService.getSourceRequired(table.getSourceId());
DatacenterCatalog catalog = registryService.getCatalogById(table.getCatalogId());
DatacenterTable queryTable = resolveQueryTable(table, request.getDatasetRef());
validateRequest(queryTable, request, source);
request.getDatasetRef().setSourceId(table.getSourceId());
request.getDatasetRef().setCatalogId(table.getCatalogId());
request.getDatasetRef().setTableId(table.getId());
request.getDatasetRef().setTableName(table.getTableName());
if (catalog != null) {
request.getDatasetRef().setCatalogName(catalog.getCatalogName());
}
DatacenterConnector connector = connectorRegistry.getConnector(source.getSourceType());
return connector.queryPage(source, queryTable, request);
}
@Override
public List<Row> queryBySql(DatacenterSqlQueryRequest request) {
if (request == null || request.getDatasetRef() == null) {
throw new BusinessException("datasetRef 不能为空");
}
String sql = trimToNull(request.getSql());
if (!StringUtils.hasText(sql)) {
throw new BusinessException("SQL 不能为空");
}
DatasetRef datasetRef = request.getDatasetRef();
if (datasetRef.getSourceId() == null) {
throw new BusinessException("缺少连接服务配置");
}
DatacenterSource source = registryService.getSourceRequired(datasetRef.getSourceId());
BigInteger catalogId = resolveRequestedCatalogId(datasetRef);
List<DatacenterTable> managedTables = registryService.listManagedTables(datasetRef.getSourceId(), catalogId);
if (CollectionUtils.isEmpty(managedTables)) {
throw new BusinessException("当前连接下没有已接入表");
}
SqlSupportUtils.ResolvedSql resolvedSql = SqlSupportUtils.resolve(
sql,
managedTables.stream().map(this::toManagedSqlTable).toList()
);
DatacenterConnector connector = connectorRegistry.getConnector(source.getSourceType());
return connector.queryBySql(source, resolvedSql.getExecutableSql());
}
@Override
public DatacenterSchemaResponse getSchema(DatasetRef datasetRef) {
DatacenterTable table = resolveTable(datasetRef);
DatacenterSource source = registryService.getSourceRequired(table.getSourceId());
DatacenterCatalog catalog = registryService.getCatalogById(table.getCatalogId());
DatacenterSchemaResponse response = new DatacenterSchemaResponse();
response.setDatasetRef(registryService.resolveDatasetRef(table.getId()));
response.setSource(source);
response.setCatalog(catalog);
response.setTable(table);
response.setFields(table.getFields());
response.setVersions(listVersions(table.getId()));
response.setUpstreamLineage(listUpstream(table.getId()));
response.setDownstreamLineage(listDownstream(table.getId()));
return response;
}
private DatacenterTable resolveTable(DatasetRef datasetRef) {
if (datasetRef.getTableId() != null) {
return registryService.getTableWithFields(datasetRef.getTableId());
}
if (datasetRef.getSourceId() == null || !StringUtils.hasText(datasetRef.getTableName())) {
throw new BusinessException("缺少数据集定位信息");
}
QueryWrapper wrapper = QueryWrapper.create();
wrapper.eq(DatacenterTable::getSourceId, datasetRef.getSourceId());
wrapper.eq(DatacenterTable::getTableName, datasetRef.getTableName().trim());
boolean hasCatalogCondition = false;
if (datasetRef.getCatalogId() != null) {
wrapper.eq(DatacenterTable::getCatalogId, datasetRef.getCatalogId());
hasCatalogCondition = true;
} else if (StringUtils.hasText(datasetRef.getCatalogName())) {
DatacenterCatalog catalog = resolveCatalog(datasetRef.getSourceId(), datasetRef.getCatalogName().trim());
wrapper.eq(DatacenterTable::getCatalogId, catalog.getId());
hasCatalogCondition = true;
}
List<DatacenterTable> tables = tableMapper.selectListByQuery(wrapper);
if (CollectionUtils.isEmpty(tables)) {
throw new BusinessException("数据集不存在: " + datasetRef.getTableName());
}
if (!hasCatalogCondition && tables.size() > 1) {
throw new BusinessException("数据集存在重名表,请指定库名: " + datasetRef.getTableName());
}
return registryService.getTableWithFields(tables.get(0).getId());
}
private DatacenterCatalog resolveCatalog(java.math.BigInteger sourceId, String catalogName) {
QueryWrapper wrapper = QueryWrapper.create();
wrapper.eq(DatacenterCatalog::getSourceId, sourceId);
wrapper.eq(DatacenterCatalog::getCatalogName, catalogName);
List<DatacenterCatalog> catalogs = catalogMapper.selectListByQuery(wrapper);
if (CollectionUtils.isEmpty(catalogs)) {
throw new BusinessException("库不存在: " + catalogName);
}
if (catalogs.size() > 1) {
throw new BusinessException("库存在重复配置,请检查: " + catalogName);
}
return catalogs.get(0);
}
private BigInteger resolveRequestedCatalogId(DatasetRef datasetRef) {
if (datasetRef == null) {
return null;
}
if (datasetRef.getCatalogId() != null) {
return datasetRef.getCatalogId();
}
if (StringUtils.hasText(datasetRef.getCatalogName())) {
DatacenterCatalog catalog = resolveCatalog(datasetRef.getSourceId(), datasetRef.getCatalogName().trim());
return catalog.getId();
}
return null;
}
private SqlSupportUtils.ManagedTable toManagedSqlTable(DatacenterTable table) {
DatacenterCatalog catalog = registryService.getCatalogById(table.getCatalogId());
return new SqlSupportUtils.ManagedTable(
catalog == null ? null : catalog.getCatalogName(),
table.getTableName(),
resolvePhysicalTableName(table)
);
}
private String resolvePhysicalTableName(DatacenterTable table) {
if (table == null) {
return null;
}
if ("EXTERNAL_TABLE".equals(table.getTableKind()) || "EXTERNAL_VIEW".equals(table.getTableKind())) {
return trimToNull(table.getActualTable()) == null ? table.getTableName() : table.getActualTable().trim();
}
String materializedTable = trimToNull(table.getMaterializedTable());
if (materializedTable != null) {
return materializedTable;
}
String actualTable = trimToNull(table.getActualTable());
return actualTable != null ? actualTable : table.getTableName();
}
private void normalizePage(DatacenterQueryRequest request) {
if (request.getPageNumber() == null || request.getPageNumber() < 1L) {
request.setPageNumber(1L);
}
if (request.getPageSize() == null || request.getPageSize() < 1L) {
throw new BusinessException("pageSize 必须大于 0");
}
}
private DatacenterTable resolveQueryTable(DatacenterTable table, DatasetRef datasetRef) {
if (datasetRef == null || datasetRef.getVersionId() == null) {
return table;
}
DatacenterDatasetVersion version = datasetVersionMapper.selectOneById(datasetRef.getVersionId());
if (version == null || !table.getId().equals(version.getTableId())) {
throw new BusinessException("数据集版本不存在: " + datasetRef.getVersionId());
}
DatacenterTable queryTable = new DatacenterTable();
queryTable.setId(table.getId());
queryTable.setSourceId(table.getSourceId());
queryTable.setCatalogId(table.getCatalogId());
queryTable.setTableName(table.getTableName());
queryTable.setTableDesc(table.getTableDesc());
queryTable.setActualTable(table.getActualTable());
queryTable.setMaterializedTable(version.getMaterializedTable());
queryTable.setAccessMode(table.getAccessMode());
queryTable.setTableKind(table.getTableKind());
queryTable.setVersioningEnabled(table.getVersioningEnabled());
queryTable.setCapabilitiesJson(table.getCapabilitiesJson());
queryTable.setFields(table.getFields());
return queryTable;
}
private void validateRequest(DatacenterTable table, DatacenterQueryRequest request, DatacenterSource source) {
Map<String, DatacenterTableField> fieldMap = new LinkedHashMap<>();
for (DatacenterTableField field : table.getFields()) {
fieldMap.put(field.getFieldName(), field);
}
if (!CollectionUtils.isEmpty(request.getSelectedColumns())) {
for (String column : request.getSelectedColumns()) {
DatacenterTableField field = fieldMap.get(column);
if (field == null || !isEnabled(field.getQueryable())) {
throw new BusinessException("字段不可查询: " + column);
}
}
} else {
request.setSelectedColumns(
table.getFields().stream()
.filter(field -> isEnabled(field.getQueryable()))
.map(DatacenterTableField::getFieldName)
.toList()
);
}
if (!CollectionUtils.isEmpty(request.getFilters())) {
request.getFilters().forEach(filter -> {
DatacenterTableField field = fieldMap.get(filter.getColumn());
if (field == null || !isEnabled(field.getQueryable())) {
throw new BusinessException("字段不可过滤: " + filter.getColumn());
}
});
}
if (!CollectionUtils.isEmpty(request.getSorts())) {
request.getSorts().forEach(sort -> {
DatacenterTableField field = fieldMap.get(sort.getColumn());
if (field == null || !isEnabled(field.getSortable())) {
throw new BusinessException("字段不可排序: " + sort.getColumn());
}
});
}
if (request.getWhere() != null && !request.getWhere().isBlank()) {
boolean allowLegacyWhere = "PROJECT_MYSQL".equals(source.getSourceType())
|| "MYSQL".equals(source.getSourceType())
|| "POSTGRESQL".equals(source.getSourceType());
if (!allowLegacyWhere) {
throw new BusinessException("当前数据源仅支持结构化 DSL 查询");
}
}
}
private boolean isEnabled(Integer value) {
return value == null || value == 1;
}
private String trimToNull(String value) {
if (!StringUtils.hasText(value)) {
return null;
}
return value.trim();
}
private List<DatacenterDatasetVersion> listVersions(java.math.BigInteger tableId) {
QueryWrapper wrapper = QueryWrapper.create();
wrapper.eq(DatacenterDatasetVersion::getTableId, tableId);
wrapper.orderBy("version_no desc");
return datasetVersionMapper.selectListByQuery(wrapper);
}
private List<DatacenterDerivedTable> listUpstream(java.math.BigInteger tableId) {
QueryWrapper wrapper = QueryWrapper.create();
wrapper.eq(DatacenterDerivedTable::getDerivedTableId, tableId);
wrapper.orderBy("created desc");
return derivedTableMapper.selectListByQuery(wrapper);
}
private List<DatacenterDerivedTable> listDownstream(java.math.BigInteger tableId) {
QueryWrapper wrapper = QueryWrapper.create();
wrapper.eq(DatacenterDerivedTable::getSourceTableId, tableId);
wrapper.orderBy("created desc");
return derivedTableMapper.selectListByQuery(wrapper);
}
}

View File

@@ -0,0 +1,48 @@
package tech.easyflow.datacenter.execution.service.impl;
import com.alibaba.fastjson2.JSONObject;
import org.springframework.stereotype.Service;
import tech.easyflow.common.entity.LoginAccount;
import tech.easyflow.common.web.exceptions.BusinessException;
import tech.easyflow.datacenter.connector.DatacenterConnector;
import tech.easyflow.datacenter.connector.DatacenterConnectorRegistry;
import tech.easyflow.datacenter.entity.DatacenterTable;
import tech.easyflow.datacenter.execution.model.DatasetRef;
import tech.easyflow.datacenter.meta.entity.DatacenterSource;
import tech.easyflow.datacenter.meta.service.DatacenterDatasetRegistryService;
import tech.easyflow.datacenter.execution.service.DatacenterDatasetWriteService;
import javax.annotation.Resource;
import java.math.BigInteger;
@Service
public class DatacenterDatasetWriteServiceImpl implements DatacenterDatasetWriteService {
@Resource
private DatacenterDatasetRegistryService registryService;
@Resource
private DatacenterConnectorRegistry connectorRegistry;
@Override
public void saveRow(DatasetRef datasetRef, JSONObject data, LoginAccount account) {
DatacenterTable table = resolveTable(datasetRef);
DatacenterSource source = registryService.getSourceRequired(table.getSourceId());
DatacenterConnector connector = connectorRegistry.getConnector(source.getSourceType());
connector.saveRow(source, table, data, account);
}
@Override
public void deleteRow(DatasetRef datasetRef, BigInteger id, LoginAccount account) {
DatacenterTable table = resolveTable(datasetRef);
DatacenterSource source = registryService.getSourceRequired(table.getSourceId());
DatacenterConnector connector = connectorRegistry.getConnector(source.getSourceType());
connector.deleteRow(source, table, id, account);
}
private DatacenterTable resolveTable(DatasetRef datasetRef) {
if (datasetRef == null || datasetRef.getTableId() == null) {
throw new BusinessException("缺少 tableId");
}
return registryService.getTableWithFields(datasetRef.getTableId());
}
}

View File

@@ -0,0 +1,7 @@
package tech.easyflow.datacenter.integration;
import tech.easyflow.datacenter.execution.model.DatacenterQueryRequest;
public interface AssistantDatacenterBridge {
AssistantDatacenterResult queryPage(DatacenterQueryRequest request);
}

View File

@@ -0,0 +1,34 @@
package tech.easyflow.datacenter.integration;
import com.mybatisflex.core.row.Row;
import tech.easyflow.datacenter.entity.DatacenterTable;
import tech.easyflow.datacenter.meta.entity.DatacenterCatalog;
import tech.easyflow.datacenter.meta.entity.DatacenterDatasetVersion;
import tech.easyflow.datacenter.meta.entity.DatacenterSource;
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
public class AssistantDatacenterResult {
private List<Row> rows = new ArrayList<>();
private DatacenterSource source;
private DatacenterCatalog catalog;
private DatacenterTable table;
private DatacenterDatasetVersion version;
private Map<String, Object> querySummary = new LinkedHashMap<>();
public List<Row> getRows() { return rows; }
public void setRows(List<Row> rows) { this.rows = rows; }
public DatacenterSource getSource() { return source; }
public void setSource(DatacenterSource source) { this.source = source; }
public DatacenterCatalog getCatalog() { return catalog; }
public void setCatalog(DatacenterCatalog catalog) { this.catalog = catalog; }
public DatacenterTable getTable() { return table; }
public void setTable(DatacenterTable table) { this.table = table; }
public DatacenterDatasetVersion getVersion() { return version; }
public void setVersion(DatacenterDatasetVersion version) { this.version = version; }
public Map<String, Object> getQuerySummary() { return querySummary; }
public void setQuerySummary(Map<String, Object> querySummary) { this.querySummary = querySummary; }
}

View File

@@ -0,0 +1,49 @@
package tech.easyflow.datacenter.integration;
import org.springframework.stereotype.Component;
import tech.easyflow.datacenter.execution.model.DatacenterQueryRequest;
import tech.easyflow.datacenter.execution.model.DatacenterSchemaResponse;
import tech.easyflow.datacenter.execution.service.DatacenterDatasetQueryService;
import tech.easyflow.datacenter.meta.entity.DatacenterDatasetVersion;
import javax.annotation.Resource;
import java.util.LinkedHashMap;
import java.util.List;
@Component
public class DefaultAssistantDatacenterBridge implements AssistantDatacenterBridge {
@Resource
private DatacenterDatasetQueryService queryService;
@Override
public AssistantDatacenterResult queryPage(DatacenterQueryRequest request) {
var page = queryService.queryPage(request);
DatacenterSchemaResponse schema = queryService.getSchema(request.getDatasetRef());
AssistantDatacenterResult result = new AssistantDatacenterResult();
result.setRows(page.getRecords());
result.setSource(schema.getSource());
result.setCatalog(schema.getCatalog());
result.setTable(schema.getTable());
result.setVersion(resolveVersion(schema.getVersions(), request == null || request.getDatasetRef() == null ? null : request.getDatasetRef().getVersionId()));
result.setQuerySummary(new LinkedHashMap<>() {{
put("pageNumber", page.getPageNumber());
put("pageSize", page.getPageSize());
put("totalRows", page.getTotalRow());
put("selectedColumns", request == null ? List.of() : request.getSelectedColumns());
put("filterCount", request == null || request.getFilters() == null ? 0 : request.getFilters().size());
put("sortCount", request == null || request.getSorts() == null ? 0 : request.getSorts().size());
}});
return result;
}
private DatacenterDatasetVersion resolveVersion(List<DatacenterDatasetVersion> versions, java.math.BigInteger versionId) {
if (versions == null || versions.isEmpty()) {
return null;
}
if (versionId == null) {
return versions.get(0);
}
return versions.stream().filter(version -> versionId.equals(version.getId())).findFirst().orElse(null);
}
}

View File

@@ -0,0 +1,7 @@
package tech.easyflow.datacenter.mapper;
import com.mybatisflex.core.BaseMapper;
import tech.easyflow.datacenter.meta.entity.DatacenterCatalog;
public interface DatacenterCatalogMapper extends BaseMapper<DatacenterCatalog> {
}

View File

@@ -0,0 +1,7 @@
package tech.easyflow.datacenter.mapper;
import com.mybatisflex.core.BaseMapper;
import tech.easyflow.datacenter.meta.entity.DatacenterDatasetVersion;
public interface DatacenterDatasetVersionMapper extends BaseMapper<DatacenterDatasetVersion> {
}

View File

@@ -0,0 +1,7 @@
package tech.easyflow.datacenter.mapper;
import com.mybatisflex.core.BaseMapper;
import tech.easyflow.datacenter.meta.entity.DatacenterDerivedTable;
public interface DatacenterDerivedTableMapper extends BaseMapper<DatacenterDerivedTable> {
}

View File

@@ -0,0 +1,7 @@
package tech.easyflow.datacenter.mapper;
import com.mybatisflex.core.BaseMapper;
import tech.easyflow.datacenter.meta.entity.DatacenterImportJob;
public interface DatacenterImportJobMapper extends BaseMapper<DatacenterImportJob> {
}

View File

@@ -0,0 +1,7 @@
package tech.easyflow.datacenter.mapper;
import com.mybatisflex.core.BaseMapper;
import tech.easyflow.datacenter.meta.entity.DatacenterSource;
public interface DatacenterSourceMapper extends BaseMapper<DatacenterSource> {
}

View File

@@ -0,0 +1,75 @@
package tech.easyflow.datacenter.meta.entity;
import com.mybatisflex.annotation.Column;
import com.mybatisflex.annotation.Id;
import com.mybatisflex.annotation.KeyType;
import com.mybatisflex.annotation.Table;
import com.mybatisflex.core.handler.FastjsonTypeHandler;
import tech.easyflow.common.entity.DateEntity;
import java.io.Serializable;
import java.math.BigInteger;
import java.util.Date;
import java.util.Map;
@Table(value = "tb_datacenter_catalog", comment = "数据中心逻辑库/命名空间")
public class DatacenterCatalog extends DateEntity implements Serializable {
@Id(keyType = KeyType.Generator, value = "snowFlakeId", comment = "主键")
private BigInteger id;
@Column(comment = "部门ID")
private BigInteger deptId;
@Column(tenantId = true, comment = "租户ID")
private BigInteger tenantId;
@Column(comment = "数据源ID")
private BigInteger sourceId;
@Column(comment = "目录名")
private String catalogName;
@Column(comment = "目录描述")
private String catalogDesc;
@Column(comment = "目录类型")
private String catalogType;
@Column(comment = "状态")
private Integer status;
@Column(typeHandler = FastjsonTypeHandler.class, comment = "扩展项")
private Map<String, Object> options;
@Column(comment = "创建时间")
private Date created;
@Column(comment = "创建人")
private BigInteger createdBy;
@Column(comment = "修改时间")
private Date modified;
@Column(comment = "修改人")
private BigInteger modifiedBy;
public BigInteger getId() { return id; }
public void setId(BigInteger id) { this.id = id; }
public BigInteger getDeptId() { return deptId; }
public void setDeptId(BigInteger deptId) { this.deptId = deptId; }
public BigInteger getTenantId() { return tenantId; }
public void setTenantId(BigInteger tenantId) { this.tenantId = tenantId; }
public BigInteger getSourceId() { return sourceId; }
public void setSourceId(BigInteger sourceId) { this.sourceId = sourceId; }
public String getCatalogName() { return catalogName; }
public void setCatalogName(String catalogName) { this.catalogName = catalogName; }
public String getCatalogDesc() { return catalogDesc; }
public void setCatalogDesc(String catalogDesc) { this.catalogDesc = catalogDesc; }
public String getCatalogType() { return catalogType; }
public void setCatalogType(String catalogType) { this.catalogType = catalogType; }
public Integer getStatus() { return status; }
public void setStatus(Integer status) { this.status = status; }
public Map<String, Object> getOptions() { return options; }
public void setOptions(Map<String, Object> options) { this.options = options; }
@Override
public Date getCreated() { return created; }
@Override
public void setCreated(Date created) { this.created = created; }
public BigInteger getCreatedBy() { return createdBy; }
public void setCreatedBy(BigInteger createdBy) { this.createdBy = createdBy; }
@Override
public Date getModified() { return modified; }
@Override
public void setModified(Date modified) { this.modified = modified; }
public BigInteger getModifiedBy() { return modifiedBy; }
public void setModifiedBy(BigInteger modifiedBy) { this.modifiedBy = modifiedBy; }
}

View File

@@ -0,0 +1,70 @@
package tech.easyflow.datacenter.meta.entity;
import com.mybatisflex.annotation.Column;
import com.mybatisflex.annotation.Id;
import com.mybatisflex.annotation.KeyType;
import com.mybatisflex.annotation.Table;
import com.mybatisflex.core.handler.FastjsonTypeHandler;
import tech.easyflow.common.entity.DateEntity;
import java.io.Serializable;
import java.math.BigInteger;
import java.util.Date;
import java.util.Map;
@Table(value = "tb_datacenter_dataset_version", comment = "数据集版本")
public class DatacenterDatasetVersion extends DateEntity implements Serializable {
@Id(keyType = KeyType.Generator, value = "snowFlakeId")
private BigInteger id;
@Column(comment = "部门ID")
private BigInteger deptId;
@Column(tenantId = true, comment = "租户ID")
private BigInteger tenantId;
@Column(comment = "表ID")
private BigInteger tableId;
@Column(comment = "版本号")
private Integer versionNo;
@Column(comment = "版本标签")
private String versionLabel;
@Column(comment = "物化表名")
private String materializedTable;
@Column(typeHandler = FastjsonTypeHandler.class, comment = "版本快照")
private Map<String, Object> snapshotJson;
@Column(comment = "状态")
private Integer status;
@Column(comment = "创建时间")
private Date created;
@Column(comment = "创建人")
private BigInteger createdBy;
@Column(comment = "修改时间")
private Date modified;
@Column(comment = "修改人")
private BigInteger modifiedBy;
public BigInteger getId() { return id; }
public void setId(BigInteger id) { this.id = id; }
public BigInteger getDeptId() { return deptId; }
public void setDeptId(BigInteger deptId) { this.deptId = deptId; }
public BigInteger getTenantId() { return tenantId; }
public void setTenantId(BigInteger tenantId) { this.tenantId = tenantId; }
public BigInteger getTableId() { return tableId; }
public void setTableId(BigInteger tableId) { this.tableId = tableId; }
public Integer getVersionNo() { return versionNo; }
public void setVersionNo(Integer versionNo) { this.versionNo = versionNo; }
public String getVersionLabel() { return versionLabel; }
public void setVersionLabel(String versionLabel) { this.versionLabel = versionLabel; }
public String getMaterializedTable() { return materializedTable; }
public void setMaterializedTable(String materializedTable) { this.materializedTable = materializedTable; }
public Map<String, Object> getSnapshotJson() { return snapshotJson; }
public void setSnapshotJson(Map<String, Object> snapshotJson) { this.snapshotJson = snapshotJson; }
public Integer getStatus() { return status; }
public void setStatus(Integer status) { this.status = status; }
@Override public Date getCreated() { return created; }
@Override public void setCreated(Date created) { this.created = created; }
public BigInteger getCreatedBy() { return createdBy; }
public void setCreatedBy(BigInteger createdBy) { this.createdBy = createdBy; }
@Override public Date getModified() { return modified; }
@Override public void setModified(Date modified) { this.modified = modified; }
public BigInteger getModifiedBy() { return modifiedBy; }
public void setModifiedBy(BigInteger modifiedBy) { this.modifiedBy = modifiedBy; }
}

View File

@@ -0,0 +1,66 @@
package tech.easyflow.datacenter.meta.entity;
import com.mybatisflex.annotation.Column;
import com.mybatisflex.annotation.Id;
import com.mybatisflex.annotation.KeyType;
import com.mybatisflex.annotation.Table;
import com.mybatisflex.core.handler.FastjsonTypeHandler;
import tech.easyflow.common.entity.DateEntity;
import java.io.Serializable;
import java.math.BigInteger;
import java.util.Date;
import java.util.Map;
@Table(value = "tb_datacenter_derived_table", comment = "数据中心派生表关系")
public class DatacenterDerivedTable extends DateEntity implements Serializable {
@Id(keyType = KeyType.Generator, value = "snowFlakeId")
private BigInteger id;
@Column(comment = "部门ID")
private BigInteger deptId;
@Column(tenantId = true, comment = "租户ID")
private BigInteger tenantId;
@Column(comment = "源表ID")
private BigInteger sourceTableId;
@Column(comment = "派生表ID")
private BigInteger derivedTableId;
@Column(comment = "派生类型")
private String deriveType;
@Column(typeHandler = FastjsonTypeHandler.class, comment = "派生配置")
private Map<String, Object> deriveConfigJson;
@Column(comment = "状态")
private Integer status;
@Column(comment = "创建时间")
private Date created;
@Column(comment = "创建人")
private BigInteger createdBy;
@Column(comment = "修改时间")
private Date modified;
@Column(comment = "修改人")
private BigInteger modifiedBy;
public BigInteger getId() { return id; }
public void setId(BigInteger id) { this.id = id; }
public BigInteger getDeptId() { return deptId; }
public void setDeptId(BigInteger deptId) { this.deptId = deptId; }
public BigInteger getTenantId() { return tenantId; }
public void setTenantId(BigInteger tenantId) { this.tenantId = tenantId; }
public BigInteger getSourceTableId() { return sourceTableId; }
public void setSourceTableId(BigInteger sourceTableId) { this.sourceTableId = sourceTableId; }
public BigInteger getDerivedTableId() { return derivedTableId; }
public void setDerivedTableId(BigInteger derivedTableId) { this.derivedTableId = derivedTableId; }
public String getDeriveType() { return deriveType; }
public void setDeriveType(String deriveType) { this.deriveType = deriveType; }
public Map<String, Object> getDeriveConfigJson() { return deriveConfigJson; }
public void setDeriveConfigJson(Map<String, Object> deriveConfigJson) { this.deriveConfigJson = deriveConfigJson; }
public Integer getStatus() { return status; }
public void setStatus(Integer status) { this.status = status; }
@Override public Date getCreated() { return created; }
@Override public void setCreated(Date created) { this.created = created; }
public BigInteger getCreatedBy() { return createdBy; }
public void setCreatedBy(BigInteger createdBy) { this.createdBy = createdBy; }
@Override public Date getModified() { return modified; }
@Override public void setModified(Date modified) { this.modified = modified; }
public BigInteger getModifiedBy() { return modifiedBy; }
public void setModifiedBy(BigInteger modifiedBy) { this.modifiedBy = modifiedBy; }
}

View File

@@ -0,0 +1,102 @@
package tech.easyflow.datacenter.meta.entity;
import com.mybatisflex.annotation.Column;
import com.mybatisflex.annotation.Id;
import com.mybatisflex.annotation.KeyType;
import com.mybatisflex.annotation.Table;
import com.mybatisflex.core.handler.FastjsonTypeHandler;
import tech.easyflow.common.entity.DateEntity;
import java.io.Serializable;
import java.math.BigInteger;
import java.util.Date;
import java.util.Map;
@Table(value = "tb_datacenter_import_job", comment = "数据中心导入任务")
public class DatacenterImportJob extends DateEntity implements Serializable {
@Id(keyType = KeyType.Generator, value = "snowFlakeId")
private BigInteger id;
@Column(comment = "部门ID")
private BigInteger deptId;
@Column(tenantId = true, comment = "租户ID")
private BigInteger tenantId;
@Column(comment = "数据源ID")
private BigInteger sourceId;
@Column(comment = "目录ID")
private BigInteger catalogId;
@Column(comment = "表ID")
private BigInteger tableId;
@Column(comment = "任务类型")
private String jobType;
@Column(comment = "文件名")
private String fileName;
@Column(comment = "文件存储路径")
private String storagePath;
@Column(comment = "任务状态")
private String status;
@Column(comment = "总行数")
private Long totalRows;
@Column(comment = "成功行数")
private Long successRows;
@Column(comment = "失败行数")
private Long errorRows;
@Column(comment = "错误摘要")
private String errorSummary;
@Column(typeHandler = FastjsonTypeHandler.class, comment = "任务载荷")
private Map<String, Object> payloadJson;
@Column(comment = "开始时间")
private Date startedAt;
@Column(comment = "结束时间")
private Date finishedAt;
@Column(comment = "创建时间")
private Date created;
@Column(comment = "创建人")
private BigInteger createdBy;
@Column(comment = "修改时间")
private Date modified;
@Column(comment = "修改人")
private BigInteger modifiedBy;
public BigInteger getId() { return id; }
public void setId(BigInteger id) { this.id = id; }
public BigInteger getDeptId() { return deptId; }
public void setDeptId(BigInteger deptId) { this.deptId = deptId; }
public BigInteger getTenantId() { return tenantId; }
public void setTenantId(BigInteger tenantId) { this.tenantId = tenantId; }
public BigInteger getSourceId() { return sourceId; }
public void setSourceId(BigInteger sourceId) { this.sourceId = sourceId; }
public BigInteger getCatalogId() { return catalogId; }
public void setCatalogId(BigInteger catalogId) { this.catalogId = catalogId; }
public BigInteger getTableId() { return tableId; }
public void setTableId(BigInteger tableId) { this.tableId = tableId; }
public String getJobType() { return jobType; }
public void setJobType(String jobType) { this.jobType = jobType; }
public String getFileName() { return fileName; }
public void setFileName(String fileName) { this.fileName = fileName; }
public String getStoragePath() { return storagePath; }
public void setStoragePath(String storagePath) { this.storagePath = storagePath; }
public String getStatus() { return status; }
public void setStatus(String status) { this.status = status; }
public Long getTotalRows() { return totalRows; }
public void setTotalRows(Long totalRows) { this.totalRows = totalRows; }
public Long getSuccessRows() { return successRows; }
public void setSuccessRows(Long successRows) { this.successRows = successRows; }
public Long getErrorRows() { return errorRows; }
public void setErrorRows(Long errorRows) { this.errorRows = errorRows; }
public String getErrorSummary() { return errorSummary; }
public void setErrorSummary(String errorSummary) { this.errorSummary = errorSummary; }
public Map<String, Object> getPayloadJson() { return payloadJson; }
public void setPayloadJson(Map<String, Object> payloadJson) { this.payloadJson = payloadJson; }
public Date getStartedAt() { return startedAt; }
public void setStartedAt(Date startedAt) { this.startedAt = startedAt; }
public Date getFinishedAt() { return finishedAt; }
public void setFinishedAt(Date finishedAt) { this.finishedAt = finishedAt; }
@Override public Date getCreated() { return created; }
@Override public void setCreated(Date created) { this.created = created; }
public BigInteger getCreatedBy() { return createdBy; }
public void setCreatedBy(BigInteger createdBy) { this.createdBy = createdBy; }
@Override public Date getModified() { return modified; }
@Override public void setModified(Date modified) { this.modified = modified; }
public BigInteger getModifiedBy() { return modifiedBy; }
public void setModifiedBy(BigInteger modifiedBy) { this.modifiedBy = modifiedBy; }
}

View File

@@ -0,0 +1,131 @@
package tech.easyflow.datacenter.meta.entity;
import com.mybatisflex.annotation.Column;
import com.mybatisflex.annotation.Id;
import com.mybatisflex.annotation.KeyType;
import com.mybatisflex.annotation.Table;
import com.mybatisflex.core.handler.FastjsonTypeHandler;
import tech.easyflow.common.entity.DateEntity;
import java.io.Serializable;
import java.math.BigInteger;
import java.util.Date;
import java.util.Map;
@Table(value = "tb_datacenter_source", comment = "数据中心数据源")
public class DatacenterSource extends DateEntity implements Serializable {
@Id(keyType = KeyType.Generator, value = "snowFlakeId", comment = "主键")
private BigInteger id;
@Column(comment = "部门ID")
private BigInteger deptId;
@Column(tenantId = true, comment = "租户ID")
private BigInteger tenantId;
@Column(comment = "数据源名称")
private String sourceName;
@Column(comment = "数据源编码")
private String sourceCode;
@Column(comment = "数据源类型")
private String sourceType;
@Column(comment = "访问模式")
private String accessMode;
@Column(comment = "是否内置")
private Integer builtinFlag;
@Column(comment = "驱动类名")
private String driverClassName;
@Column(comment = "JDBC URL")
private String jdbcUrl;
@Column(comment = "主机")
private String host;
@Column(comment = "端口")
private Integer port;
@Column(comment = "数据库名")
private String databaseName;
@Column(comment = "Schema名")
private String schemaName;
@Column(comment = "用户名")
private String username;
@Column(comment = "凭据密文")
private String credentialCipher;
@Column(typeHandler = FastjsonTypeHandler.class, comment = "连接配置")
private Map<String, Object> configJson;
@Column(typeHandler = FastjsonTypeHandler.class, comment = "能力声明")
private Map<String, Object> capabilitiesJson;
@Column(comment = "最近测试状态")
private String lastTestStatus;
@Column(comment = "最近测试信息")
private String lastTestMessage;
@Column(comment = "最近测试时间")
private Date lastTestedAt;
@Column(comment = "状态")
private Integer status;
@Column(typeHandler = FastjsonTypeHandler.class, comment = "扩展项")
private Map<String, Object> options;
@Column(comment = "创建时间")
private Date created;
@Column(comment = "创建人")
private BigInteger createdBy;
@Column(comment = "修改时间")
private Date modified;
@Column(comment = "修改人")
private BigInteger modifiedBy;
public BigInteger getId() { return id; }
public void setId(BigInteger id) { this.id = id; }
public BigInteger getDeptId() { return deptId; }
public void setDeptId(BigInteger deptId) { this.deptId = deptId; }
public BigInteger getTenantId() { return tenantId; }
public void setTenantId(BigInteger tenantId) { this.tenantId = tenantId; }
public String getSourceName() { return sourceName; }
public void setSourceName(String sourceName) { this.sourceName = sourceName; }
public String getSourceCode() { return sourceCode; }
public void setSourceCode(String sourceCode) { this.sourceCode = sourceCode; }
public String getSourceType() { return sourceType; }
public void setSourceType(String sourceType) { this.sourceType = sourceType; }
public String getAccessMode() { return accessMode; }
public void setAccessMode(String accessMode) { this.accessMode = accessMode; }
public Integer getBuiltinFlag() { return builtinFlag; }
public void setBuiltinFlag(Integer builtinFlag) { this.builtinFlag = builtinFlag; }
public String getDriverClassName() { return driverClassName; }
public void setDriverClassName(String driverClassName) { this.driverClassName = driverClassName; }
public String getJdbcUrl() { return jdbcUrl; }
public void setJdbcUrl(String jdbcUrl) { this.jdbcUrl = jdbcUrl; }
public String getHost() { return host; }
public void setHost(String host) { this.host = host; }
public Integer getPort() { return port; }
public void setPort(Integer port) { this.port = port; }
public String getDatabaseName() { return databaseName; }
public void setDatabaseName(String databaseName) { this.databaseName = databaseName; }
public String getSchemaName() { return schemaName; }
public void setSchemaName(String schemaName) { this.schemaName = schemaName; }
public String getUsername() { return username; }
public void setUsername(String username) { this.username = username; }
public String getCredentialCipher() { return credentialCipher; }
public void setCredentialCipher(String credentialCipher) { this.credentialCipher = credentialCipher; }
public Map<String, Object> getConfigJson() { return configJson; }
public void setConfigJson(Map<String, Object> configJson) { this.configJson = configJson; }
public Map<String, Object> getCapabilitiesJson() { return capabilitiesJson; }
public void setCapabilitiesJson(Map<String, Object> capabilitiesJson) { this.capabilitiesJson = capabilitiesJson; }
public String getLastTestStatus() { return lastTestStatus; }
public void setLastTestStatus(String lastTestStatus) { this.lastTestStatus = lastTestStatus; }
public String getLastTestMessage() { return lastTestMessage; }
public void setLastTestMessage(String lastTestMessage) { this.lastTestMessage = lastTestMessage; }
public Date getLastTestedAt() { return lastTestedAt; }
public void setLastTestedAt(Date lastTestedAt) { this.lastTestedAt = lastTestedAt; }
public Integer getStatus() { return status; }
public void setStatus(Integer status) { this.status = status; }
public Map<String, Object> getOptions() { return options; }
public void setOptions(Map<String, Object> options) { this.options = options; }
@Override
public Date getCreated() { return created; }
@Override
public void setCreated(Date created) { this.created = created; }
public BigInteger getCreatedBy() { return createdBy; }
public void setCreatedBy(BigInteger createdBy) { this.createdBy = createdBy; }
@Override
public Date getModified() { return modified; }
@Override
public void setModified(Date modified) { this.modified = modified; }
public BigInteger getModifiedBy() { return modifiedBy; }
public void setModifiedBy(BigInteger modifiedBy) { this.modifiedBy = modifiedBy; }
}

View File

@@ -0,0 +1,6 @@
package tech.easyflow.datacenter.meta.enums;
public enum DatacenterAccessMode {
READ_ONLY,
READ_WRITE
}

View File

@@ -0,0 +1,10 @@
package tech.easyflow.datacenter.meta.enums;
public enum DatacenterCapability {
TEST_CONNECTION,
BROWSE_METADATA,
READ_QUERY,
WRITE_MUTATION,
EXPORT,
MATERIALIZE
}

View File

@@ -0,0 +1,12 @@
package tech.easyflow.datacenter.meta.enums;
public enum DatacenterConnectionErrorCode {
INVALID_ARGUMENT,
DRIVER_NOT_FOUND,
NETWORK_UNREACHABLE,
AUTH_FAILED,
DATABASE_NOT_FOUND,
SCHEMA_NOT_FOUND,
PERMISSION_DENIED,
UNKNOWN_ERROR
}

View File

@@ -0,0 +1,9 @@
package tech.easyflow.datacenter.meta.enums;
public enum DatacenterImportStatus {
PENDING,
RUNNING,
SUCCESS,
FAILED,
NOT_IMPLEMENTED
}

View File

@@ -0,0 +1,13 @@
package tech.easyflow.datacenter.meta.enums;
public enum DatacenterSourceType {
PROJECT_MYSQL,
EXCEL,
EXCEL_MATERIALIZED,
MYSQL,
POSTGRESQL,
ORACLE,
GAUSSDB_NATIVE,
GBASE_8A,
GBASE_8S
}

View File

@@ -0,0 +1,10 @@
package tech.easyflow.datacenter.meta.enums;
public enum DatacenterTableKind {
PROJECT_MANAGED,
EXTERNAL_TABLE,
EXTERNAL_VIEW,
EXCEL_SHEET,
EXCEL_MATERIALIZED,
DERIVED_TABLE
}

View File

@@ -0,0 +1,36 @@
package tech.easyflow.datacenter.meta.model;
import java.math.BigInteger;
import java.util.ArrayList;
import java.util.List;
public class DatacenterBatchRegisterRequest {
private BigInteger sourceId;
private String catalogName;
private List<String> tableNames = new ArrayList<>();
public BigInteger getSourceId() {
return sourceId;
}
public void setSourceId(BigInteger sourceId) {
this.sourceId = sourceId;
}
public String getCatalogName() {
return catalogName;
}
public void setCatalogName(String catalogName) {
this.catalogName = catalogName;
}
public List<String> getTableNames() {
return tableNames;
}
public void setTableNames(List<String> tableNames) {
this.tableNames = tableNames;
}
}

View File

@@ -0,0 +1,18 @@
package tech.easyflow.datacenter.meta.model;
import java.math.BigInteger;
import java.util.ArrayList;
import java.util.List;
public class DatacenterBatchRemoveRequest {
private List<BigInteger> tableIds = new ArrayList<>();
public List<BigInteger> getTableIds() {
return tableIds;
}
public void setTableIds(List<BigInteger> tableIds) {
this.tableIds = tableIds;
}
}

View File

@@ -0,0 +1,22 @@
package tech.easyflow.datacenter.meta.model;
import java.math.BigInteger;
public class DatacenterCatalogMeta {
private BigInteger id;
private BigInteger sourceId;
private String catalogName;
private String catalogType;
private String catalogDesc;
public BigInteger getId() { return id; }
public void setId(BigInteger id) { this.id = id; }
public BigInteger getSourceId() { return sourceId; }
public void setSourceId(BigInteger sourceId) { this.sourceId = sourceId; }
public String getCatalogName() { return catalogName; }
public void setCatalogName(String catalogName) { this.catalogName = catalogName; }
public String getCatalogType() { return catalogType; }
public void setCatalogType(String catalogType) { this.catalogType = catalogType; }
public String getCatalogDesc() { return catalogDesc; }
public void setCatalogDesc(String catalogDesc) { this.catalogDesc = catalogDesc; }
}

View File

@@ -0,0 +1,25 @@
package tech.easyflow.datacenter.meta.model;
import java.math.BigInteger;
public class DatacenterFieldDescriptionUpdate {
private BigInteger fieldId;
private String fieldDesc;
public BigInteger getFieldId() {
return fieldId;
}
public void setFieldId(BigInteger fieldId) {
this.fieldId = fieldId;
}
public String getFieldDesc() {
return fieldDesc;
}
public void setFieldDesc(String fieldDesc) {
this.fieldDesc = fieldDesc;
}
}

View File

@@ -0,0 +1,16 @@
package tech.easyflow.datacenter.meta.model;
import java.math.BigInteger;
public class DatacenterRemoveSourceRequest {
private BigInteger sourceId;
public BigInteger getSourceId() {
return sourceId;
}
public void setSourceId(BigInteger sourceId) {
this.sourceId = sourceId;
}
}

View File

@@ -0,0 +1,36 @@
package tech.easyflow.datacenter.meta.model;
import java.math.BigInteger;
import java.util.ArrayList;
import java.util.List;
public class DatacenterSaveDescriptionsRequest {
private BigInteger tableId;
private String tableDesc;
private List<DatacenterFieldDescriptionUpdate> fields = new ArrayList<>();
public BigInteger getTableId() {
return tableId;
}
public void setTableId(BigInteger tableId) {
this.tableId = tableId;
}
public String getTableDesc() {
return tableDesc;
}
public void setTableDesc(String tableDesc) {
this.tableDesc = tableDesc;
}
public List<DatacenterFieldDescriptionUpdate> getFields() {
return fields;
}
public void setFields(List<DatacenterFieldDescriptionUpdate> fields) {
this.fields = fields;
}
}

View File

@@ -0,0 +1,17 @@
package tech.easyflow.datacenter.meta.model;
import tech.easyflow.datacenter.entity.DatacenterTable;
import tech.easyflow.datacenter.entity.DatacenterTableField;
import java.util.ArrayList;
import java.util.List;
public class DatacenterTableDetailMeta {
private DatacenterTable table;
private List<DatacenterTableField> fields = new ArrayList<>();
public DatacenterTable getTable() { return table; }
public void setTable(DatacenterTable table) { this.table = table; }
public List<DatacenterTableField> getFields() { return fields; }
public void setFields(List<DatacenterTableField> fields) { this.fields = fields; }
}

View File

@@ -0,0 +1,38 @@
package tech.easyflow.datacenter.meta.service;
import tech.easyflow.common.entity.LoginAccount;
import tech.easyflow.datacenter.entity.DatacenterTable;
import tech.easyflow.datacenter.entity.DatacenterTableField;
import tech.easyflow.datacenter.execution.model.DatasetRef;
import tech.easyflow.datacenter.meta.entity.DatacenterCatalog;
import tech.easyflow.datacenter.meta.entity.DatacenterSource;
import tech.easyflow.datacenter.meta.model.DatacenterFieldDescriptionUpdate;
import tech.easyflow.datacenter.meta.enums.DatacenterSourceType;
import tech.easyflow.datacenter.meta.model.DatacenterTableDetailMeta;
import java.math.BigInteger;
import java.util.List;
public interface DatacenterDatasetRegistryService {
DatacenterSource ensureBuiltinSource(DatacenterSourceType sourceType, LoginAccount account);
DatacenterCatalog ensureCatalog(DatacenterSource source, String catalogName, LoginAccount account);
DatacenterTable registerTable(DatacenterSource source, DatacenterCatalog catalog, DatacenterTableDetailMeta detail, LoginAccount account);
DatacenterTable getTableWithFields(BigInteger tableId);
List<DatacenterTableField> getFields(BigInteger tableId);
DatasetRef resolveDatasetRef(BigInteger tableId);
DatacenterSource getSourceRequired(BigInteger sourceId);
DatacenterCatalog getCatalogById(BigInteger catalogId);
List<DatacenterTable> listManagedTables(BigInteger sourceId, BigInteger catalogId);
int removeTables(List<BigInteger> tableIds);
DatacenterTable saveDescriptions(BigInteger tableId, String tableDesc, List<DatacenterFieldDescriptionUpdate> fields, LoginAccount account);
}

View File

@@ -0,0 +1,12 @@
package tech.easyflow.datacenter.meta.service;
import java.math.BigInteger;
public final class DatacenterMetaConstants {
private DatacenterMetaConstants() {
}
public static final BigInteger PROJECT_SOURCE_BASE = new BigInteger("9000000000002000000");
public static final BigInteger PROJECT_CATALOG_BASE = new BigInteger("9000000000003000000");
}

View File

@@ -0,0 +1,32 @@
package tech.easyflow.datacenter.meta.service;
import com.mybatisflex.core.paginate.Page;
import com.mybatisflex.core.service.IService;
import tech.easyflow.common.entity.LoginAccount;
import tech.easyflow.datacenter.entity.DatacenterTable;
import tech.easyflow.datacenter.execution.model.DatacenterConnectionTestResult;
import tech.easyflow.datacenter.meta.entity.DatacenterSource;
import tech.easyflow.datacenter.meta.model.DatacenterBatchRegisterRequest;
import tech.easyflow.datacenter.meta.model.DatacenterCatalogMeta;
import tech.easyflow.datacenter.meta.model.DatacenterTableDetailMeta;
import java.math.BigInteger;
import java.util.List;
public interface DatacenterSourceService extends IService<DatacenterSource> {
DatacenterSource saveSource(DatacenterSource source, LoginAccount account);
Page<DatacenterSource> pageSources(Long pageNumber, Long pageSize, LoginAccount account);
DatacenterConnectionTestResult testConnection(DatacenterSource source, LoginAccount account);
List<DatacenterCatalogMeta> listCatalogs(BigInteger sourceId, LoginAccount account);
List<DatacenterTable> listTables(BigInteger sourceId, String catalogName, LoginAccount account);
DatacenterTableDetailMeta getTableDetail(BigInteger sourceId, String catalogName, String tableName, boolean register, LoginAccount account);
List<DatacenterTable> batchRegisterTables(DatacenterBatchRegisterRequest request, LoginAccount account);
void removeSource(BigInteger sourceId, LoginAccount account);
}

View File

@@ -0,0 +1,405 @@
package tech.easyflow.datacenter.meta.service.impl;
import com.mybatisflex.core.query.QueryWrapper;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import tech.easyflow.common.entity.LoginAccount;
import tech.easyflow.common.web.exceptions.BusinessException;
import tech.easyflow.datacenter.adapter.DbHandleManager;
import tech.easyflow.datacenter.entity.DatacenterTable;
import tech.easyflow.datacenter.entity.DatacenterTableField;
import tech.easyflow.datacenter.execution.model.DatasetRef;
import tech.easyflow.datacenter.mapper.DatacenterCatalogMapper;
import tech.easyflow.datacenter.mapper.DatacenterDatasetVersionMapper;
import tech.easyflow.datacenter.mapper.DatacenterDerivedTableMapper;
import tech.easyflow.datacenter.mapper.DatacenterImportJobMapper;
import tech.easyflow.datacenter.mapper.DatacenterSourceMapper;
import tech.easyflow.datacenter.mapper.DatacenterTableFieldMapper;
import tech.easyflow.datacenter.mapper.DatacenterTableMapper;
import tech.easyflow.datacenter.meta.entity.DatacenterCatalog;
import tech.easyflow.datacenter.meta.entity.DatacenterSource;
import tech.easyflow.datacenter.meta.model.DatacenterFieldDescriptionUpdate;
import tech.easyflow.datacenter.meta.enums.DatacenterSourceType;
import tech.easyflow.datacenter.meta.service.DatacenterDatasetRegistryService;
import tech.easyflow.datacenter.meta.service.DatacenterMetaConstants;
import tech.easyflow.datacenter.meta.model.DatacenterTableDetailMeta;
import tech.easyflow.datacenter.meta.enums.DatacenterTableKind;
import javax.annotation.Resource;
import java.math.BigInteger;
import java.util.Date;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
@Service
public class DatacenterDatasetRegistryServiceImpl implements DatacenterDatasetRegistryService {
@Resource
private DatacenterSourceMapper sourceMapper;
@Resource
private DatacenterCatalogMapper catalogMapper;
@Resource
private DatacenterTableMapper tableMapper;
@Resource
private DatacenterTableFieldMapper tableFieldMapper;
@Resource
private DatacenterDatasetVersionMapper datasetVersionMapper;
@Resource
private DatacenterDerivedTableMapper derivedTableMapper;
@Resource
private DatacenterImportJobMapper importJobMapper;
@Resource
private DbHandleManager dbHandleManager;
@Override
public DatacenterSource ensureBuiltinSource(DatacenterSourceType sourceType, LoginAccount account) {
BigInteger tenantId = account == null || account.getTenantId() == null ? BigInteger.ZERO : account.getTenantId();
BigInteger deptId = account == null || account.getDeptId() == null ? BigInteger.ZERO : account.getDeptId();
BigInteger id = builtinSourceId(sourceType, tenantId);
DatacenterSource source = sourceMapper.selectOneById(id);
if (source != null) {
return source;
}
source = new DatacenterSource();
source.setId(id);
source.setTenantId(tenantId);
source.setDeptId(deptId);
source.setSourceType(sourceType.name());
source.setBuiltinFlag(1);
source.setStatus(0);
source.setCreated(new Date());
source.setModified(new Date());
source.setCreatedBy(account == null ? BigInteger.ZERO : account.getId());
source.setModifiedBy(account == null ? BigInteger.ZERO : account.getId());
source.setCapabilitiesJson(Map.of("capabilities", builtinCapabilities(sourceType)));
if (sourceType == DatacenterSourceType.PROJECT_MYSQL) {
source.setSourceName("项目 MySQL");
source.setSourceCode("PROJECT_MYSQL_" + tenantId);
source.setAccessMode("READ_WRITE");
} else {
throw new BusinessException("不支持的内置数据源类型: " + sourceType);
}
sourceMapper.insert(source);
ensureCatalog(source, defaultCatalogName(sourceType), account);
return source;
}
@Override
public DatacenterCatalog ensureCatalog(DatacenterSource source, String catalogName, LoginAccount account) {
QueryWrapper wrapper = QueryWrapper.create();
wrapper.eq(DatacenterCatalog::getSourceId, source.getId());
wrapper.eq(DatacenterCatalog::getCatalogName, catalogName);
DatacenterCatalog catalog = catalogMapper.selectOneByQuery(wrapper);
if (catalog != null) {
return catalog;
}
catalog = new DatacenterCatalog();
BigInteger catalogId = builtinCatalogId(source, catalogName);
if (catalogId != null) {
catalog.setId(catalogId);
}
catalog.setSourceId(source.getId());
catalog.setTenantId(source.getTenantId());
catalog.setDeptId(source.getDeptId());
catalog.setCatalogName(catalogName);
catalog.setCatalogDesc(catalogName);
catalog.setCatalogType("DATABASE");
catalog.setStatus(0);
catalog.setCreated(new Date());
catalog.setModified(new Date());
catalog.setCreatedBy(account == null ? BigInteger.ZERO : account.getId());
catalog.setModifiedBy(account == null ? BigInteger.ZERO : account.getId());
catalogMapper.insert(catalog);
return catalog;
}
@Override
public DatacenterTable registerTable(DatacenterSource source, DatacenterCatalog catalog, DatacenterTableDetailMeta detail, LoginAccount account) {
DatacenterTable table = detail.getTable();
applyTableDefaults(table, source, detail);
QueryWrapper wrapper = QueryWrapper.create();
wrapper.eq(DatacenterTable::getSourceId, source.getId());
wrapper.eq(DatacenterTable::getCatalogId, catalog.getId());
wrapper.eq(DatacenterTable::getTableName, table.getTableName());
DatacenterTable existing = tableMapper.selectOneByQuery(wrapper);
Date now = new Date();
if (existing == null) {
table.setSourceId(source.getId());
table.setCatalogId(catalog.getId());
table.setTenantId(source.getTenantId());
table.setDeptId(source.getDeptId());
table.setStatus(0);
table.setCreated(now);
table.setModified(now);
table.setCreatedBy(account == null ? BigInteger.ZERO : account.getId());
table.setModifiedBy(account == null ? BigInteger.ZERO : account.getId());
tableMapper.insert(table);
} else {
if (!hasText(existing.getTableDesc())) {
existing.setTableDesc(normalizeDescription(table.getTableDesc()));
}
existing.setActualTable(table.getActualTable());
existing.setMaterializedTable(table.getMaterializedTable());
existing.setTableKind(table.getTableKind());
existing.setAccessMode(table.getAccessMode());
existing.setVersioningEnabled(table.getVersioningEnabled());
existing.setCapabilitiesJson(table.getCapabilitiesJson());
existing.setModified(now);
existing.setModifiedBy(account == null ? BigInteger.ZERO : account.getId());
tableMapper.update(existing);
table = existing;
}
Map<String, DatacenterTableField> existingFieldMap = getFields(table.getId()).stream()
.collect(LinkedHashMap::new, (map, field) -> map.put(field.getFieldName(), field), Map::putAll);
QueryWrapper deleteWrapper = QueryWrapper.create();
deleteWrapper.eq(DatacenterTableField::getTableId, table.getId());
tableFieldMapper.deleteByQuery(deleteWrapper);
for (DatacenterTableField field : detail.getFields()) {
DatacenterTableField existingField = existingFieldMap.get(field.getFieldName());
if (existingField != null && hasText(existingField.getFieldDesc())) {
field.setFieldDesc(existingField.getFieldDesc());
} else {
field.setFieldDesc(normalizeDescription(field.getFieldDesc()));
}
field.setId(null);
field.setTableId(table.getId());
field.setCreated(now);
field.setModified(now);
field.setCreatedBy(account == null ? BigInteger.ZERO : account.getId());
field.setModifiedBy(account == null ? BigInteger.ZERO : account.getId());
tableFieldMapper.insert(field);
}
table.setFields(getFields(table.getId()));
return table;
}
@Override
@Transactional(rollbackFor = Exception.class)
public DatacenterTable saveDescriptions(BigInteger tableId, String tableDesc, List<DatacenterFieldDescriptionUpdate> fields, LoginAccount account) {
if (tableId == null) {
throw new BusinessException("缺少表 ID");
}
DatacenterTable table = getTableWithFields(tableId);
Date now = new Date();
table.setTableDesc(normalizeDescription(tableDesc));
table.setModified(now);
table.setModifiedBy(account == null ? BigInteger.ZERO : account.getId());
tableMapper.update(table);
Map<BigInteger, DatacenterFieldDescriptionUpdate> fieldUpdateMap = new LinkedHashMap<>();
if (fields != null) {
for (DatacenterFieldDescriptionUpdate field : fields) {
if (field == null || field.getFieldId() == null) {
continue;
}
fieldUpdateMap.put(field.getFieldId(), field);
}
}
for (DatacenterTableField field : table.getFields()) {
DatacenterFieldDescriptionUpdate update = fieldUpdateMap.get(field.getId());
if (update == null) {
continue;
}
field.setFieldDesc(normalizeDescription(update.getFieldDesc()));
field.setModified(now);
field.setModifiedBy(account == null ? BigInteger.ZERO : account.getId());
tableFieldMapper.update(field);
}
return getTableWithFields(tableId);
}
@Override
public DatacenterTable getTableWithFields(BigInteger tableId) {
DatacenterTable table = tableMapper.selectOneById(tableId);
if (table == null) {
throw new BusinessException("数据集不存在: " + tableId);
}
table.setFields(getFields(tableId));
return table;
}
@Override
public List<DatacenterTableField> getFields(BigInteger tableId) {
QueryWrapper wrapper = QueryWrapper.create();
wrapper.eq(DatacenterTableField::getTableId, tableId);
wrapper.orderBy("id");
return tableFieldMapper.selectListByQuery(wrapper);
}
@Override
public DatasetRef resolveDatasetRef(BigInteger tableId) {
DatacenterTable table = getTableWithFields(tableId);
DatasetRef ref = new DatasetRef();
ref.setTableId(tableId);
ref.setSourceId(table.getSourceId());
ref.setCatalogId(table.getCatalogId());
ref.setTableName(table.getTableName());
DatacenterCatalog catalog = getCatalogById(table.getCatalogId());
if (catalog != null) {
ref.setCatalogName(catalog.getCatalogName());
}
return ref;
}
@Override
public DatacenterSource getSourceRequired(BigInteger sourceId) {
DatacenterSource source = sourceMapper.selectOneById(sourceId);
if (source == null) {
throw new BusinessException("连接不存在");
}
return source;
}
@Override
public DatacenterCatalog getCatalogById(BigInteger catalogId) {
if (catalogId == null) {
return null;
}
return catalogMapper.selectOneById(catalogId);
}
@Override
public List<DatacenterTable> listManagedTables(BigInteger sourceId, BigInteger catalogId) {
QueryWrapper wrapper = QueryWrapper.create();
wrapper.eq(DatacenterTable::getSourceId, sourceId);
if (catalogId != null) {
wrapper.eq(DatacenterTable::getCatalogId, catalogId);
}
wrapper.orderBy("created desc");
return tableMapper.selectListByQuery(wrapper);
}
@Override
@Transactional(rollbackFor = Exception.class)
public int removeTables(List<BigInteger> tableIds) {
List<BigInteger> ids = tableIds == null ? List.of() : tableIds.stream().filter(id -> id != null).distinct().toList();
if (ids.isEmpty()) {
return 0;
}
QueryWrapper physicalTableWrapper = QueryWrapper.create();
physicalTableWrapper.in(DatacenterTable::getId, ids);
List<DatacenterTable> tables = tableMapper.selectListByQuery(physicalTableWrapper);
tables.forEach(this::deletePhysicalTableIfNecessary);
QueryWrapper fieldWrapper = QueryWrapper.create();
fieldWrapper.in(DatacenterTableField::getTableId, ids);
tableFieldMapper.deleteByQuery(fieldWrapper);
QueryWrapper versionWrapper = QueryWrapper.create();
versionWrapper.in(tech.easyflow.datacenter.meta.entity.DatacenterDatasetVersion::getTableId, ids);
datasetVersionMapper.deleteByQuery(versionWrapper);
QueryWrapper importJobWrapper = QueryWrapper.create();
importJobWrapper.in(tech.easyflow.datacenter.meta.entity.DatacenterImportJob::getTableId, ids);
importJobMapper.deleteByQuery(importJobWrapper);
QueryWrapper upstreamWrapper = QueryWrapper.create();
upstreamWrapper.in(tech.easyflow.datacenter.meta.entity.DatacenterDerivedTable::getSourceTableId, ids);
derivedTableMapper.deleteByQuery(upstreamWrapper);
QueryWrapper downstreamWrapper = QueryWrapper.create();
downstreamWrapper.in(tech.easyflow.datacenter.meta.entity.DatacenterDerivedTable::getDerivedTableId, ids);
derivedTableMapper.deleteByQuery(downstreamWrapper);
QueryWrapper tableWrapper = QueryWrapper.create();
tableWrapper.in(DatacenterTable::getId, ids);
return tableMapper.deleteByQuery(tableWrapper);
}
private void deletePhysicalTableIfNecessary(DatacenterTable table) {
if (table == null || !shouldDropPhysicalTable(table)) {
return;
}
String physicalTableName = table.getActualTable();
if (physicalTableName == null || physicalTableName.isBlank()) {
physicalTableName = table.getMaterializedTable();
}
if (physicalTableName == null || physicalTableName.isBlank()) {
return;
}
DatacenterTable physicalTable = new DatacenterTable();
physicalTable.setActualTable(physicalTableName);
dbHandleManager.getDbHandler().deleteTable(physicalTable);
}
private boolean shouldDropPhysicalTable(DatacenterTable table) {
String tableKind = table.getTableKind();
if (tableKind == null || tableKind.isBlank()) {
return false;
}
try {
DatacenterTableKind kind = DatacenterTableKind.valueOf(tableKind);
return kind == DatacenterTableKind.PROJECT_MANAGED
|| kind == DatacenterTableKind.EXCEL_SHEET
|| kind == DatacenterTableKind.EXCEL_MATERIALIZED
|| kind == DatacenterTableKind.DERIVED_TABLE;
} catch (IllegalArgumentException ignored) {
return false;
}
}
private void applyTableDefaults(DatacenterTable table, DatacenterSource source, DatacenterTableDetailMeta detail) {
if (table.getTableKind() == null || table.getTableKind().isBlank()) {
table.setTableKind(DatacenterTableKind.EXTERNAL_TABLE.name());
}
if (table.getAccessMode() == null || table.getAccessMode().isBlank()) {
table.setAccessMode("READ_ONLY");
}
if (table.getVersioningEnabled() == null) {
table.setVersioningEnabled(0);
}
if (table.getCapabilitiesJson() == null || table.getCapabilitiesJson().isEmpty()) {
table.setCapabilitiesJson(Map.of(
"capabilities",
source.getCapabilitiesJson() == null
? List.of()
: source.getCapabilitiesJson().getOrDefault("capabilities", List.of())
));
}
if ((table.getActualTable() == null || table.getActualTable().isBlank()) && table.getTableName() != null) {
table.setActualTable(table.getTableName());
}
table.setTableDesc(normalizeDescription(table.getTableDesc()));
if (detail.getFields() == null) {
detail.setFields(List.of());
}
}
private String normalizeDescription(String description) {
if (description == null) {
return "";
}
return description.trim();
}
private boolean hasText(String value) {
return value != null && !value.trim().isEmpty();
}
private BigInteger builtinSourceId(DatacenterSourceType sourceType, BigInteger tenantId) {
if (sourceType == DatacenterSourceType.PROJECT_MYSQL) {
return DatacenterMetaConstants.PROJECT_SOURCE_BASE.add(tenantId);
}
throw new BusinessException("不支持的内置源 ID 计算");
}
private BigInteger builtinCatalogId(DatacenterSource source, String catalogName) {
if (source.getId().compareTo(DatacenterMetaConstants.PROJECT_SOURCE_BASE) >= 0
&& defaultCatalogName(DatacenterSourceType.PROJECT_MYSQL).equals(catalogName)) {
return DatacenterMetaConstants.PROJECT_CATALOG_BASE.add(source.getTenantId());
}
return null;
}
private String defaultCatalogName(DatacenterSourceType sourceType) {
if (sourceType == DatacenterSourceType.PROJECT_MYSQL) {
return "project_mysql";
}
return sourceType.name().toLowerCase();
}
private List<String> builtinCapabilities(DatacenterSourceType sourceType) {
return List.of("TEST_CONNECTION", "BROWSE_METADATA", "READ_QUERY", "WRITE_MUTATION");
}
}

View File

@@ -0,0 +1,311 @@
package tech.easyflow.datacenter.meta.service.impl;
import com.mybatisflex.core.paginate.Page;
import com.mybatisflex.core.query.QueryWrapper;
import com.mybatisflex.spring.service.impl.ServiceImpl;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import tech.easyflow.common.entity.LoginAccount;
import tech.easyflow.common.web.exceptions.BusinessException;
import tech.easyflow.datacenter.connector.DatacenterConnector;
import tech.easyflow.datacenter.connector.DatacenterConnectorRegistry;
import tech.easyflow.datacenter.entity.DatacenterTable;
import tech.easyflow.datacenter.mapper.DatacenterCatalogMapper;
import tech.easyflow.datacenter.mapper.DatacenterSourceMapper;
import tech.easyflow.datacenter.meta.entity.DatacenterCatalog;
import tech.easyflow.datacenter.meta.entity.DatacenterSource;
import tech.easyflow.datacenter.meta.enums.DatacenterSourceType;
import tech.easyflow.datacenter.meta.model.DatacenterBatchRegisterRequest;
import tech.easyflow.datacenter.meta.model.DatacenterCatalogMeta;
import tech.easyflow.datacenter.meta.model.DatacenterTableDetailMeta;
import tech.easyflow.datacenter.meta.service.DatacenterDatasetRegistryService;
import tech.easyflow.datacenter.meta.service.DatacenterSourceService;
import tech.easyflow.datacenter.meta.support.DatacenterSourceConnectionDefaults;
import tech.easyflow.datacenter.execution.model.DatacenterConnectionTestResult;
import tech.easyflow.datacenter.security.DatacenterCredentialCipher;
import javax.annotation.Resource;
import java.math.BigInteger;
import java.util.Date;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
@Service
public class DatacenterSourceServiceImpl extends ServiceImpl<DatacenterSourceMapper, DatacenterSource> implements DatacenterSourceService {
@Resource
private DatacenterConnectorRegistry connectorRegistry;
@Resource
private DatacenterDatasetRegistryService registryService;
@Resource
private DatacenterCredentialCipher credentialCipher;
@Resource
private DatacenterCatalogMapper catalogMapper;
@Resource
private DatacenterSourceConnectionDefaults connectionDefaults;
@Override
public DatacenterSource saveSource(DatacenterSource source, LoginAccount account) {
if (source == null || source.getSourceType() == null || source.getSourceType().isBlank()) {
throw new BusinessException("数据源类型不能为空");
}
DatacenterSource existing = source.getId() == null ? null : getById(source.getId());
DatacenterSource normalized = mergeWithExisting(existing, source);
DatacenterConnector connector = connectorRegistry.getConnector(normalized.getSourceType());
applyCredentialCipher(normalized, existing);
normalized.setConfigJson(connectionDefaults.sanitizeConfig(normalized.getConfigJson()));
connectionDefaults.normalize(normalized);
normalized.setCapabilitiesJson(Map.of("capabilities", connector.getCapabilities().stream().map(Enum::name).toList()));
Date now = new Date();
if (normalized.getId() == null) {
normalized.setCreated(now);
normalized.setCreatedBy(account == null ? BigInteger.ZERO : account.getId());
normalized.setTenantId(account == null ? BigInteger.ZERO : account.getTenantId());
normalized.setDeptId(account == null ? BigInteger.ZERO : account.getDeptId());
normalized.setStatus(normalized.getStatus() == null ? 0 : normalized.getStatus());
normalized.setModified(now);
normalized.setModifiedBy(account == null ? BigInteger.ZERO : account.getId());
save(normalized);
} else {
if (existing == null) {
throw new BusinessException("连接不存在");
}
normalized.setModified(now);
normalized.setModifiedBy(account == null ? BigInteger.ZERO : account.getId());
updateById(normalized);
}
return getById(normalized.getId());
}
@Override
public Page<DatacenterSource> pageSources(Long pageNumber, Long pageSize, LoginAccount account) {
registryService.ensureBuiltinSource(DatacenterSourceType.PROJECT_MYSQL, account);
return page(new Page<>(pageNumber == null ? 1L : pageNumber, pageSize == null ? 10L : pageSize), QueryWrapper.create());
}
@Override
public DatacenterConnectionTestResult testConnection(DatacenterSource source, LoginAccount account) {
DatacenterSource existing = source != null && source.getId() != null ? getById(source.getId()) : null;
if (source != null && source.getId() != null && existing == null) {
throw new BusinessException("连接不存在");
}
DatacenterSource actual = mergeWithExisting(existing, source);
if (DatacenterSourceType.PROJECT_MYSQL.name().equals(actual.getSourceType())) {
actual = registryService.ensureBuiltinSource(DatacenterSourceType.PROJECT_MYSQL, account);
}
applyCredentialCipher(actual, existing);
actual.setConfigJson(connectionDefaults.sanitizeConfig(actual.getConfigJson()));
connectionDefaults.normalize(actual);
DatacenterConnector connector = connectorRegistry.getConnector(actual.getSourceType());
DatacenterConnectionTestResult result = connector.testConnection(actual);
Map<String, Object> details = new LinkedHashMap<>();
if (result.getDetails() != null) {
details.putAll(result.getDetails());
}
details.put("effectiveDriverClassName", actual.getDriverClassName());
details.put("effectiveJdbcUrl", actual.getJdbcUrl());
details.put("effectivePort", actual.getPort());
result.setDetails(details);
if (actual.getId() != null) {
DatacenterSource persisted = new DatacenterSource();
persisted.setId(actual.getId());
persisted.setLastTestStatus(result.isSuccess() ? "SUCCESS" : "FAILED");
persisted.setLastTestMessage(result.getMessage());
persisted.setLastTestedAt(new Date());
updateById(persisted);
}
return result;
}
@Override
public List<DatacenterCatalogMeta> listCatalogs(BigInteger sourceId, LoginAccount account) {
DatacenterSource source = registryService.getSourceRequired(sourceId);
if (isManagedOnly(source.getSourceType())) {
QueryWrapper wrapper = QueryWrapper.create();
wrapper.eq(DatacenterCatalog::getSourceId, sourceId);
return catalogMapper.selectListByQuery(wrapper).stream().map(this::toCatalogMeta).collect(Collectors.toList());
}
return connectorRegistry.getConnector(source.getSourceType()).listCatalogs(source);
}
@Override
public List<DatacenterTable> listTables(BigInteger sourceId, String catalogName, LoginAccount account) {
DatacenterSource source = registryService.getSourceRequired(sourceId);
if (isManagedOnly(source.getSourceType())) {
BigInteger catalogId = resolveCatalogId(sourceId, catalogName);
return registryService.listManagedTables(sourceId, catalogId);
}
return connectorRegistry.getConnector(source.getSourceType()).listTables(source, catalogName);
}
@Override
public DatacenterTableDetailMeta getTableDetail(BigInteger sourceId, String catalogName, String tableName, boolean register, LoginAccount account) {
DatacenterSource source = registryService.getSourceRequired(sourceId);
if (isManagedOnly(source.getSourceType())) {
BigInteger catalogId = resolveCatalogId(sourceId, catalogName);
List<DatacenterTable> tables = registryService.listManagedTables(sourceId, catalogId);
DatacenterTable target = tables.stream().filter(item -> tableName.equals(item.getTableName())).findFirst()
.orElseThrow(() -> new BusinessException("数据集不存在: " + tableName));
DatacenterTableDetailMeta detail = new DatacenterTableDetailMeta();
detail.setTable(registryService.getTableWithFields(target.getId()));
detail.setFields(detail.getTable().getFields());
return detail;
}
DatacenterConnector connector = connectorRegistry.getConnector(source.getSourceType());
DatacenterTableDetailMeta detail = connector.getTableDetail(source, catalogName, tableName);
if (register) {
DatacenterCatalog catalog = registryService.ensureCatalog(source, catalogName, account);
DatacenterTable table = registryService.registerTable(source, catalog, detail, account);
detail.setTable(table);
detail.setFields(table.getFields());
}
return detail;
}
@Override
public List<DatacenterTable> batchRegisterTables(DatacenterBatchRegisterRequest request, LoginAccount account) {
if (request == null || request.getSourceId() == null) {
throw new BusinessException("数据连接不能为空");
}
if (request.getCatalogName() == null || request.getCatalogName().isBlank()) {
throw new BusinessException("库不能为空");
}
List<String> tableNames = request.getTableNames() == null
? List.of()
: request.getTableNames().stream().filter(name -> name != null && !name.isBlank()).distinct().toList();
if (tableNames.isEmpty()) {
throw new BusinessException("至少选择一张表");
}
DatacenterSource source = registryService.getSourceRequired(request.getSourceId());
if (isManagedOnly(source.getSourceType())) {
throw new BusinessException("当前数据连接不支持批量接入");
}
DatacenterConnector connector = connectorRegistry.getConnector(source.getSourceType());
DatacenterCatalog catalog = registryService.ensureCatalog(source, request.getCatalogName(), account);
return tableNames.stream().map(tableName -> {
DatacenterTableDetailMeta detail = connector.getTableDetail(source, request.getCatalogName(), tableName);
return registryService.registerTable(source, catalog, detail, account);
}).collect(Collectors.toList());
}
@Override
@Transactional(rollbackFor = Exception.class)
public void removeSource(BigInteger sourceId, LoginAccount account) {
if (sourceId == null) {
throw new BusinessException("数据连接不能为空");
}
DatacenterSource source = getById(sourceId);
if (source == null) {
throw new BusinessException("连接不存在");
}
if (Boolean.TRUE.equals(source.getBuiltinFlag())) {
throw new BusinessException("内置数据连接不支持删除");
}
List<BigInteger> tableIds = registryService.listManagedTables(sourceId, null).stream()
.map(DatacenterTable::getId)
.filter(id -> id != null)
.toList();
registryService.removeTables(tableIds);
QueryWrapper catalogWrapper = QueryWrapper.create();
catalogWrapper.eq(DatacenterCatalog::getSourceId, sourceId);
catalogMapper.deleteByQuery(catalogWrapper);
removeById(sourceId);
}
private boolean isManagedOnly(String sourceType) {
return DatacenterSourceType.EXCEL.name().equals(sourceType)
|| DatacenterSourceType.EXCEL_MATERIALIZED.name().equals(sourceType);
}
private DatacenterCatalogMeta toCatalogMeta(DatacenterCatalog catalog) {
DatacenterCatalogMeta meta = new DatacenterCatalogMeta();
meta.setId(catalog.getId());
meta.setSourceId(catalog.getSourceId());
meta.setCatalogName(catalog.getCatalogName());
meta.setCatalogDesc(catalog.getCatalogDesc());
meta.setCatalogType(catalog.getCatalogType());
return meta;
}
private BigInteger resolveCatalogId(BigInteger sourceId, String catalogName) {
if (catalogName == null || catalogName.isBlank()) {
return null;
}
QueryWrapper wrapper = QueryWrapper.create();
wrapper.eq(DatacenterCatalog::getSourceId, sourceId);
wrapper.eq(DatacenterCatalog::getCatalogName, catalogName);
DatacenterCatalog catalog = catalogMapper.selectOneByQuery(wrapper);
if (catalog == null) {
throw new BusinessException("目录不存在: " + catalogName);
}
return catalog.getId();
}
private void applyCredentialCipher(DatacenterSource target, DatacenterSource existing) {
String password = extractPassword(target.getConfigJson());
if (password != null) {
target.setCredentialCipher(credentialCipher.encrypt(password));
return;
}
if ((target.getCredentialCipher() == null || target.getCredentialCipher().isBlank()) && existing != null) {
target.setCredentialCipher(existing.getCredentialCipher());
}
}
private String extractPassword(Map<String, Object> configJson) {
if (configJson == null) {
return null;
}
Object password = configJson.get("password");
if (password == null) {
return null;
}
String value = String.valueOf(password);
return value.isBlank() ? null : value;
}
private DatacenterSource mergeWithExisting(DatacenterSource existing, DatacenterSource incoming) {
if (incoming == null) {
return existing;
}
if (existing == null) {
return incoming;
}
DatacenterSource merged = new DatacenterSource();
merged.setId(existing.getId());
merged.setCreated(existing.getCreated());
merged.setCreatedBy(existing.getCreatedBy());
merged.setTenantId(existing.getTenantId());
merged.setDeptId(existing.getDeptId());
merged.setBuiltinFlag(existing.getBuiltinFlag());
merged.setStatus(existing.getStatus());
merged.setLastTestStatus(existing.getLastTestStatus());
merged.setLastTestMessage(existing.getLastTestMessage());
merged.setLastTestedAt(existing.getLastTestedAt());
merged.setOptions(existing.getOptions());
merged.setSourceName(valueOrExisting(incoming.getSourceName(), existing.getSourceName()));
merged.setSourceCode(valueOrExisting(incoming.getSourceCode(), existing.getSourceCode()));
merged.setSourceType(valueOrExisting(incoming.getSourceType(), existing.getSourceType()));
merged.setAccessMode(valueOrExisting(incoming.getAccessMode(), existing.getAccessMode()));
merged.setDriverClassName(valueOrExisting(incoming.getDriverClassName(), existing.getDriverClassName()));
merged.setJdbcUrl(valueOrExisting(incoming.getJdbcUrl(), existing.getJdbcUrl()));
merged.setHost(valueOrExisting(incoming.getHost(), existing.getHost()));
merged.setPort(incoming.getPort() != null ? incoming.getPort() : existing.getPort());
merged.setDatabaseName(valueOrExisting(incoming.getDatabaseName(), existing.getDatabaseName()));
merged.setSchemaName(valueOrExisting(incoming.getSchemaName(), existing.getSchemaName()));
merged.setUsername(valueOrExisting(incoming.getUsername(), existing.getUsername()));
merged.setCredentialCipher(valueOrExisting(incoming.getCredentialCipher(), existing.getCredentialCipher()));
merged.setConfigJson(incoming.getConfigJson() != null ? incoming.getConfigJson() : existing.getConfigJson());
merged.setCapabilitiesJson(existing.getCapabilitiesJson());
return merged;
}
private String valueOrExisting(String incoming, String existing) {
return incoming != null ? incoming : existing;
}
}

View File

@@ -0,0 +1,153 @@
package tech.easyflow.datacenter.meta.support;
import cn.hutool.core.util.StrUtil;
import org.springframework.stereotype.Component;
import tech.easyflow.datacenter.meta.entity.DatacenterSource;
import tech.easyflow.datacenter.meta.enums.DatacenterSourceType;
import java.util.LinkedHashMap;
import java.util.Map;
@Component
public class DatacenterSourceConnectionDefaults {
public boolean supportsExternalConnection(String sourceType) {
if (StrUtil.isBlank(sourceType)) {
return false;
}
DatacenterSourceType resolved = resolveSourceType(sourceType);
if (resolved == null) {
return false;
}
return switch (resolved) {
case MYSQL, POSTGRESQL, ORACLE, GAUSSDB_NATIVE, GBASE_8A, GBASE_8S -> true;
default -> false;
};
}
public DatacenterSource normalize(DatacenterSource source) {
if (source == null || !supportsExternalConnection(source.getSourceType())) {
return source;
}
if (source.getPort() == null || source.getPort() <= 0) {
source.setPort(defaultPort(source.getSourceType()));
}
if (StrUtil.isBlank(source.getDriverClassName())) {
source.setDriverClassName(defaultDriverClassName(source.getSourceType()));
}
if (StrUtil.isBlank(source.getJdbcUrl())) {
String jdbcUrl = buildJdbcUrl(source);
if (StrUtil.isNotBlank(jdbcUrl)) {
source.setJdbcUrl(jdbcUrl);
}
}
return source;
}
public Map<String, Object> sanitizeConfig(Map<String, Object> configJson) {
Map<String, Object> sanitized = new LinkedHashMap<>();
if (configJson == null || configJson.isEmpty()) {
return sanitized;
}
sanitized.putAll(configJson);
sanitized.remove("password");
return sanitized;
}
public Integer defaultPort(String sourceType) {
DatacenterSourceType resolved = requireSourceType(sourceType);
return switch (resolved) {
case MYSQL -> 3306;
case POSTGRESQL, GAUSSDB_NATIVE -> 5432;
case ORACLE -> 1521;
case GBASE_8A -> 5258;
case GBASE_8S -> 9088;
default -> null;
};
}
public String defaultDriverClassName(String sourceType) {
DatacenterSourceType resolved = requireSourceType(sourceType);
return switch (resolved) {
case MYSQL -> "com.mysql.cj.jdbc.Driver";
case POSTGRESQL, GAUSSDB_NATIVE -> "org.postgresql.Driver";
case ORACLE -> "oracle.jdbc.OracleDriver";
case GBASE_8A -> "com.gbase.jdbc.Driver";
case GBASE_8S -> "com.gbasedbt.jdbc.Driver";
default -> "";
};
}
public String buildJdbcUrl(DatacenterSource source) {
if (source == null || !supportsExternalConnection(source.getSourceType())) {
return null;
}
String host = StrUtil.trimToEmpty(source.getHost());
Integer port = source.getPort() == null || source.getPort() <= 0 ? defaultPort(source.getSourceType()) : source.getPort();
String databaseName = resolveDatabaseName(source);
if (StrUtil.isBlank(host) || port == null || StrUtil.isBlank(databaseName)) {
return null;
}
DatacenterSourceType resolved = requireSourceType(source.getSourceType());
return switch (resolved) {
case MYSQL -> String.format(
"jdbc:mysql://%s:%d/%s?useUnicode=true&characterEncoding=utf-8&serverTimezone=Asia/Shanghai&useSSL=false",
host, port, databaseName
);
case POSTGRESQL -> String.format("jdbc:postgresql://%s:%d/%s", host, port, databaseName);
case ORACLE -> String.format("jdbc:oracle:thin:@//%s:%d/%s", host, port, databaseName);
case GAUSSDB_NATIVE -> String.format("jdbc:postgresql://%s:%d/%s", host, port, databaseName);
case GBASE_8A -> String.format("jdbc:gbase://%s:%d/%s", host, port, databaseName);
case GBASE_8S -> {
String informixServer = resolveInformixServer(source);
if (StrUtil.isBlank(informixServer)) {
yield null;
}
yield String.format("jdbc:gbasedbt-sqli://%s:%d/%s:INFORMIXSERVER=%s", host, port, databaseName, informixServer);
}
default -> null;
};
}
private DatacenterSourceType requireSourceType(String sourceType) {
DatacenterSourceType resolved = resolveSourceType(sourceType);
if (resolved == null) {
throw new IllegalArgumentException("Unsupported datacenter source type: " + sourceType);
}
return resolved;
}
private DatacenterSourceType resolveSourceType(String sourceType) {
try {
return DatacenterSourceType.valueOf(sourceType);
} catch (IllegalArgumentException ex) {
return null;
}
}
private String resolveDatabaseName(DatacenterSource source) {
if (StrUtil.isNotBlank(source.getDatabaseName())) {
return source.getDatabaseName().trim();
}
if (source.getConfigJson() == null) {
return null;
}
Object serviceName = source.getConfigJson().get("serviceName");
if (serviceName != null && StrUtil.isNotBlank(String.valueOf(serviceName))) {
return String.valueOf(serviceName).trim();
}
return null;
}
private String resolveInformixServer(DatacenterSource source) {
if (source.getConfigJson() == null) {
return null;
}
Object informixServer = source.getConfigJson().get("informixServer");
if (informixServer == null) {
return null;
}
String value = String.valueOf(informixServer).trim();
return value.isEmpty() ? null : value;
}
}

View File

@@ -0,0 +1,28 @@
package tech.easyflow.datacenter.security;
import cn.hutool.crypto.SecureUtil;
import cn.hutool.crypto.symmetric.AES;
import org.springframework.stereotype.Component;
import java.nio.charset.StandardCharsets;
@Component
public class DatacenterCredentialCipher {
private static final String DEFAULT_KEY = "easyflow-datacenter-phase1-key";
private final AES aes = SecureUtil.aes(SecureUtil.sha256(DEFAULT_KEY).substring(0, 16).getBytes(StandardCharsets.UTF_8));
public String encrypt(String plainText) {
if (plainText == null || plainText.isBlank()) {
return null;
}
return aes.encryptHex(plainText);
}
public String decrypt(String cipherText) {
if (cipherText == null || cipherText.isBlank()) {
return null;
}
return aes.decryptStr(cipherText);
}
}

View File

@@ -1,14 +0,0 @@
package tech.easyflow.datacenter.service;
import com.mybatisflex.core.service.IService;
import tech.easyflow.datacenter.entity.DatacenterTableField;
/**
* 服务层。
*
* @author ArkLight
* @since 2025-07-10
*/
public interface DatacenterTableFieldService extends IService<DatacenterTableField> {
}

View File

@@ -1,35 +0,0 @@
package tech.easyflow.datacenter.service;
import com.alibaba.fastjson2.JSONObject;
import com.mybatisflex.core.paginate.Page;
import com.mybatisflex.core.row.Row;
import com.mybatisflex.core.service.IService;
import tech.easyflow.common.entity.DatacenterQuery;
import tech.easyflow.common.entity.LoginAccount;
import tech.easyflow.datacenter.entity.DatacenterTable;
import tech.easyflow.datacenter.entity.DatacenterTableField;
import tech.easyflow.datacenter.entity.vo.HeaderVo;
import java.math.BigInteger;
import java.util.List;
public interface DatacenterTableService extends IService<DatacenterTable> {
void saveTable(DatacenterTable entity, LoginAccount loginUser);
void removeTable(BigInteger tableId);
Long getCount(DatacenterQuery where);
List<Row> getListData(DatacenterQuery where);
Page<Row> getPageData(DatacenterQuery where);
List<HeaderVo> getHeaders(BigInteger tableId);
void saveValue(BigInteger tableId, JSONObject object, LoginAccount account);
void removeValue(BigInteger tableId, BigInteger id, LoginAccount account);
List<DatacenterTableField> getFields(BigInteger tableId);
}

View File

@@ -1,18 +0,0 @@
package tech.easyflow.datacenter.service.impl;
import com.mybatisflex.spring.service.impl.ServiceImpl;
import org.springframework.stereotype.Service;
import tech.easyflow.datacenter.entity.DatacenterTableField;
import tech.easyflow.datacenter.mapper.DatacenterTableFieldMapper;
import tech.easyflow.datacenter.service.DatacenterTableFieldService;
/**
* 服务层实现。
*
* @author ArkLight
* @since 2025-07-10
*/
@Service
public class DatacenterTableFieldServiceImpl extends ServiceImpl<DatacenterTableFieldMapper, DatacenterTableField> implements DatacenterTableFieldService {
}

View File

@@ -1,293 +0,0 @@
package tech.easyflow.datacenter.service.impl;
import cn.hutool.core.collection.CollectionUtil;
import cn.hutool.core.util.StrUtil;
import com.alibaba.fastjson2.JSONObject;
import com.mybatisflex.core.keygen.impl.SnowFlakeIDKeyGenerator;
import com.mybatisflex.core.paginate.Page;
import com.mybatisflex.core.query.QueryWrapper;
import com.mybatisflex.core.row.Db;
import com.mybatisflex.core.row.Row;
import com.mybatisflex.spring.service.impl.ServiceImpl;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import tech.easyflow.common.cache.RedisLockExecutor;
import tech.easyflow.common.entity.DatacenterQuery;
import tech.easyflow.common.entity.LoginAccount;
import tech.easyflow.common.web.exceptions.BusinessException;
import tech.easyflow.datacenter.adapter.DbHandleManager;
import tech.easyflow.datacenter.adapter.DbHandleService;
import tech.easyflow.datacenter.entity.DatacenterTable;
import tech.easyflow.datacenter.entity.DatacenterTableField;
import tech.easyflow.datacenter.entity.vo.HeaderVo;
import tech.easyflow.datacenter.mapper.DatacenterTableFieldMapper;
import tech.easyflow.datacenter.mapper.DatacenterTableMapper;
import tech.easyflow.datacenter.service.DatacenterTableService;
import javax.annotation.Resource;
import java.math.BigDecimal;
import java.math.BigInteger;
import java.time.Duration;
import java.util.*;
import java.util.stream.Collectors;
@Service
public class DatacenterTableServiceImpl extends ServiceImpl<DatacenterTableMapper, DatacenterTable> implements DatacenterTableService {
private static final String DATACENTER_TABLE_LOCK_KEY_PREFIX = "easyflow:lock:datacenter:table:";
private static final String DATACENTER_TABLE_CREATE_LOCK_KEY_PREFIX = "easyflow:lock:datacenter:table:create:";
private static final Duration LOCK_WAIT_TIMEOUT = Duration.ofSeconds(2);
private static final Duration LOCK_LEASE_TIMEOUT = Duration.ofSeconds(15);
@Resource
private DbHandleManager dbHandleManager;
@Resource
private DatacenterTableFieldMapper fieldsMapper;
@Resource
private RedisLockExecutor redisLockExecutor;
@Override
@Transactional(rollbackFor = Exception.class)
public void saveTable(DatacenterTable entity, LoginAccount loginUser) {
String lockKey = buildTableLockKey(entity, loginUser);
redisLockExecutor.executeWithLock(lockKey, LOCK_WAIT_TIMEOUT, LOCK_LEASE_TIMEOUT, () -> {
doSaveTable(entity, loginUser);
});
}
private void doSaveTable(DatacenterTable entity, LoginAccount loginUser) {
DbHandleService dbHandler = dbHandleManager.getDbHandler();
List<DatacenterTableField> fields = entity.getFields();
BigInteger tableId = entity.getId();
if (tableId == null) {
long snowId = new SnowFlakeIDKeyGenerator().nextId();
entity.setId(new BigInteger(String.valueOf(snowId)));
String actualTable = getActualTableName(entity);
entity.setActualTable(actualTable);
// 先 DDL 操作DDL会默认提交事务不然报错了事务不会回滚。
dbHandler.createTable(entity);
// 保存主表和字段表
save(entity);
for (DatacenterTableField field : fields) {
// 插入
field.setCreated(new Date());
field.setCreatedBy(loginUser.getId());
field.setModified(new Date());
field.setModifiedBy(loginUser.getId());
field.setTableId(entity.getId());
fieldsMapper.insert(field);
}
} else {
// actualTable 前端不可见,所以要设置
DatacenterTable tableRecord = getById(tableId);
entity.setActualTable(tableRecord.getActualTable());
dbHandler.updateTable(entity, tableRecord);
updateById(entity);
// 查询所有字段
QueryWrapper w = QueryWrapper.create();
w.eq(DatacenterTableField::getTableId, entity.getId());
List<DatacenterTableField> fieldRecords = fieldsMapper.selectListByQuery(w);
Map<BigInteger, DatacenterTableField> fieldsMap = fieldRecords.stream()
.collect(Collectors.toMap(DatacenterTableField::getId, field -> field));
for (DatacenterTableField field : fields) {
BigInteger id = field.getId();
if (id == null) {
// 新增字段到物理表
dbHandler.addField(entity, field);
// 插入
field.setCreated(new Date());
field.setCreatedBy(loginUser.getId());
field.setModified(new Date());
field.setModifiedBy(loginUser.getId());
field.setTableId(entity.getId());
fieldsMapper.insert(field);
} else {
// 删除的字段
if (field.isHandleDelete()) {
// 删除物理表中的字段
dbHandler.deleteField(entity, field);
// 删除字段
fieldsMapper.deleteById(id);
} else {
// 修改物理表中的字段
DatacenterTableField fieldRecord = fieldsMap.get(id);
dbHandler.updateField(entity, fieldRecord, field);
// 更新字段,字段类型不允许修改
field.setFieldType(field.getFieldType());
field.setModified(new Date());
field.setModifiedBy(loginUser.getId());
fieldsMapper.update(field);
}
}
}
}
}
@Override
@Transactional(rollbackFor = Exception.class)
public void removeTable(BigInteger tableId) {
redisLockExecutor.executeWithLock(
DATACENTER_TABLE_LOCK_KEY_PREFIX + tableId,
LOCK_WAIT_TIMEOUT,
LOCK_LEASE_TIMEOUT,
() -> {
DatacenterTable record = getById(tableId);
dbHandleManager.getDbHandler().deleteTable(record);
removeById(tableId);
QueryWrapper wrapper = QueryWrapper.create();
wrapper.eq(DatacenterTableField::getTableId, tableId);
fieldsMapper.deleteByQuery(wrapper);
}
);
}
@Override
public Long getCount(DatacenterQuery where) {
String actualTable = getActualTable(where.getTableId());
QueryWrapper wrapper = QueryWrapper.create();
buildCondition(wrapper, where);
return Db.selectCountByQuery(actualTable, wrapper);
}
@Override
public List<Row> getListData(DatacenterQuery where) {
String actualTable = getActualTable(where.getTableId());
QueryWrapper wrapper = QueryWrapper.create();
buildCondition(wrapper, where);
List<Row> rows = Db.selectListByQuery(actualTable, wrapper);
handleBigNumber(rows);
return rows;
}
@Override
public Page<Row> getPageData(DatacenterQuery where) {
Long pageNumber = where.getPageNumber();
Long pageSize = where.getPageSize();
Long count = getCount(where);
if (count == 0) {
return new Page<>(new ArrayList<>(), pageNumber, pageSize, count);
}
String actualTable = getActualTable(where.getTableId());
QueryWrapper wrapper = QueryWrapper.create();
buildCondition(wrapper, where);
Page<Row> page = new Page<>(pageNumber, pageSize, count);
Page<Row> paginate = Db.paginate(actualTable, page, wrapper);
handleBigNumber(paginate.getRecords());
return paginate;
}
private void handleBigNumber(List<Row> records) {
for (Row record : records) {
Map<String, Object> newMap = new LinkedHashMap<>();
for (Map.Entry<String, Object> entry : record.entrySet()) {
Object value = entry.getValue();
if ((value instanceof BigInteger ||
value instanceof BigDecimal ||
value instanceof Long)) {
newMap.put(entry.getKey(), value.toString());
} else {
newMap.put(entry.getKey(), value);
}
}
record.clear();
record.putAll(newMap);
}
}
@Override
public List<HeaderVo> getHeaders(BigInteger tableId) {
QueryWrapper wrapper = QueryWrapper.create();
wrapper.eq(DatacenterTableField::getTableId, tableId);
wrapper.orderBy("id");
List<DatacenterTableField> fields = fieldsMapper.selectListByQuery(wrapper);
List<HeaderVo> headers = new ArrayList<>();
for (DatacenterTableField field : fields) {
HeaderVo header = new HeaderVo();
header.setKey(field.getFieldName());
header.setDataIndex(field.getFieldName());
header.setTitle(field.getFieldDesc());
header.setFieldType(field.getFieldType());
header.setRequired(field.getRequired());
header.setFieldId(field.getId());
header.setTableId(field.getTableId());
headers.add(header);
}
return headers;
}
@Override
public void saveValue(BigInteger tableId, JSONObject object, LoginAccount account) {
DatacenterTable table = getById(tableId);
QueryWrapper wrapper = QueryWrapper.create();
wrapper.eq(DatacenterTableField::getTableId, tableId);
List<DatacenterTableField> fields = fieldsMapper.selectListByQuery(wrapper);
if (CollectionUtil.isEmpty(fields)) {
throw new BusinessException("请先添加字段");
}
table.setFields(fields);
Object valueId = object.get("id");
if (valueId == null) {
dbHandleManager.getDbHandler().saveValue(table, object, account);
} else {
dbHandleManager.getDbHandler().updateValue(table, object, account);
}
}
@Override
public void removeValue(BigInteger tableId, BigInteger id, LoginAccount account) {
DatacenterTable record = getById(tableId);
dbHandleManager.getDbHandler().removeValue(record, id, account);
}
@Override
public List<DatacenterTableField> getFields(BigInteger tableId) {
QueryWrapper wrapper = QueryWrapper.create();
wrapper.eq(DatacenterTableField::getTableId, tableId);
return fieldsMapper.selectListByQuery(wrapper);
}
private String getActualTable(BigInteger tableId) {
DatacenterTable record = getById(tableId);
return record.getActualTable();
}
private String getActualTableName(DatacenterTable table) {
String tableName = table.getTableName();
BigInteger id = table.getId();
return "tb_dynamic_" + tableName + "_" + id;
}
private String buildTableLockKey(DatacenterTable table, LoginAccount loginUser) {
if (table.getId() != null) {
return DATACENTER_TABLE_LOCK_KEY_PREFIX + table.getId();
}
String tenant = table.getTenantId() != null
? table.getTenantId().toString()
: (loginUser != null && loginUser.getTenantId() != null ? loginUser.getTenantId().toString() : "0");
String tableName = table.getTableName() == null ? "unknown" : table.getTableName();
return DATACENTER_TABLE_CREATE_LOCK_KEY_PREFIX + tenant + ":" + tableName;
}
/**
* 构建查询条件
*/
private void buildCondition(QueryWrapper wrapper, DatacenterQuery where) {
// 构建查询条件
String condition = where.getWhere();
if (StrUtil.isNotEmpty(condition)) {
wrapper.where(condition);
}
}
}

View File

@@ -0,0 +1,223 @@
package tech.easyflow.datacenter.utils;
import net.sf.jsqlparser.JSQLParserException;
import net.sf.jsqlparser.parser.CCJSqlParserUtil;
import net.sf.jsqlparser.schema.Table;
import net.sf.jsqlparser.statement.Statement;
import net.sf.jsqlparser.statement.Statements;
import net.sf.jsqlparser.statement.select.Select;
import net.sf.jsqlparser.statement.select.WithItem;
import net.sf.jsqlparser.util.TablesNamesFinder;
import tech.easyflow.common.web.exceptions.BusinessException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.LinkedHashMap;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Set;
public final class SqlSupportUtils {
private SqlSupportUtils() {
}
public static ResolvedSql resolve(String sql, Collection<ManagedTable> managedTables) {
String normalizedSql = normalizeSql(sql);
Statement statement = parseSingleStatement(normalizedSql);
if (!(statement instanceof Select select)) {
throw new BusinessException("查询数据节点只支持只读 SELECT SQL");
}
Map<String, List<ManagedTable>> byTableName = new LinkedHashMap<>();
Map<String, ManagedTable> byCatalogAndTable = new LinkedHashMap<>();
for (ManagedTable managedTable : managedTables) {
if (managedTable == null || !hasText(managedTable.getTableName())) {
continue;
}
String tableKey = normalizeIdentifier(managedTable.getTableName());
byTableName.computeIfAbsent(tableKey, key -> new ArrayList<>()).add(managedTable);
if (hasText(managedTable.getCatalogName())) {
byCatalogAndTable.put(
catalogTableKey(managedTable.getCatalogName(), managedTable.getTableName()),
managedTable
);
}
}
SqlTableCollector collector = new SqlTableCollector();
List<Table> referencedTables = collector.collect(select);
if (referencedTables.isEmpty()) {
throw new BusinessException("SQL 必须引用至少一张已接入表");
}
Set<String> logicalTables = new LinkedHashSet<>();
for (Table table : referencedTables) {
ManagedTable managedTable = resolveManagedTable(table, byTableName, byCatalogAndTable);
rewriteTable(table, managedTable);
logicalTables.add(renderLogicalTable(managedTable));
}
return new ResolvedSql(select.toString(), new ArrayList<>(logicalTables));
}
private static Statement parseSingleStatement(String sql) {
try {
Statements statements = CCJSqlParserUtil.parseStatements(sql);
if (statements == null || statements.getStatements() == null || statements.getStatements().isEmpty()) {
throw new BusinessException("SQL 不能为空");
}
if (statements.getStatements().size() != 1) {
throw new BusinessException("查询数据节点仅支持单条 SQL");
}
return statements.getStatements().get(0);
} catch (BusinessException e) {
throw e;
} catch (JSQLParserException e) {
throw new BusinessException("SQL 解析失败,请检查语法");
}
}
private static ManagedTable resolveManagedTable(Table table,
Map<String, List<ManagedTable>> byTableName,
Map<String, ManagedTable> byCatalogAndTable) {
String tableName = trimToNull(table.getName());
if (!hasText(tableName)) {
throw new BusinessException("SQL 包含无法识别的表名");
}
String catalogName = trimToNull(table.getSchemaName());
if (hasText(catalogName)) {
ManagedTable managedTable = byCatalogAndTable.get(catalogTableKey(catalogName, tableName));
if (managedTable == null) {
throw new BusinessException("SQL 引用了未接入表: " + catalogName + "." + tableName);
}
return managedTable;
}
List<ManagedTable> matches = byTableName.get(normalizeIdentifier(tableName));
if (matches == null || matches.isEmpty()) {
throw new BusinessException("SQL 引用了未接入表: " + tableName);
}
if (matches.size() > 1) {
throw new BusinessException("SQL 引用了重名表,请使用 catalog.table: " + tableName);
}
return matches.get(0);
}
private static void rewriteTable(Table table, ManagedTable managedTable) {
table.setName(managedTable.getPhysicalTableName());
table.setSchemaName(trimToNull(managedTable.getCatalogName()));
}
private static String renderLogicalTable(ManagedTable managedTable) {
if (!hasText(managedTable.getCatalogName())) {
return managedTable.getTableName();
}
return managedTable.getCatalogName() + "." + managedTable.getTableName();
}
private static String normalizeSql(String sql) {
String normalized = trimToNull(sql);
if (!hasText(normalized)) {
throw new BusinessException("SQL 不能为空");
}
return normalized;
}
private static String catalogTableKey(String catalogName, String tableName) {
return normalizeIdentifier(catalogName) + "." + normalizeIdentifier(tableName);
}
private static String normalizeIdentifier(String value) {
String normalized = trimToNull(value);
if (!hasText(normalized)) {
return "";
}
if ((normalized.startsWith("`") && normalized.endsWith("`"))
|| (normalized.startsWith("\"") && normalized.endsWith("\""))
|| (normalized.startsWith("[") && normalized.endsWith("]"))) {
normalized = normalized.substring(1, normalized.length() - 1);
}
return normalized.trim().toLowerCase(Locale.ROOT);
}
private static boolean hasText(String value) {
return value != null && !value.isBlank();
}
private static String trimToNull(String value) {
if (value == null) {
return null;
}
String trimmed = value.trim();
return trimmed.isEmpty() ? null : trimmed;
}
public static class ManagedTable {
private final String catalogName;
private final String tableName;
private final String physicalTableName;
public ManagedTable(String catalogName, String tableName, String physicalTableName) {
this.catalogName = trimToNull(catalogName);
this.tableName = trimToNull(tableName);
this.physicalTableName = hasText(physicalTableName) ? physicalTableName.trim() : this.tableName;
}
public String getCatalogName() {
return catalogName;
}
public String getTableName() {
return tableName;
}
public String getPhysicalTableName() {
return physicalTableName;
}
}
public static class ResolvedSql {
private final String executableSql;
private final List<String> logicalTables;
public ResolvedSql(String executableSql, List<String> logicalTables) {
this.executableSql = executableSql;
this.logicalTables = logicalTables;
}
public String getExecutableSql() {
return executableSql;
}
public List<String> getLogicalTables() {
return logicalTables;
}
}
private static class SqlTableCollector extends TablesNamesFinder {
private final List<Table> tables = new ArrayList<>();
private final Set<String> withNames = new LinkedHashSet<>();
public List<Table> collect(Statement statement) {
tables.clear();
withNames.clear();
statement.accept(this);
return new ArrayList<>(tables);
}
@Override
public void visit(WithItem withItem) {
if (withItem.getAlias() != null && hasText(withItem.getAlias().getName())) {
withNames.add(normalizeIdentifier(withItem.getAlias().getName()));
}
withItem.getSelect().accept((net.sf.jsqlparser.statement.select.SelectVisitor) this);
}
@Override
public void visit(Table table) {
String tableName = normalizeIdentifier(table.getName());
if (!withNames.contains(tableName)) {
tables.add(table);
}
}
}
}