feat: 重构数据中枢工作台与接入管理
- 新增统一的数据源、目录、纳管表与 Excel 处理后端能力 - 重建管理端数据中枢工作台并替换旧表管理页面 - 补充数据中枢迁移脚本、连接器底座与说明字段支持
This commit is contained in:
@@ -16,6 +16,19 @@
|
||||
<groupId>com.mybatis-flex</groupId>
|
||||
<artifactId>mybatis-flex-spring-boot3-starter</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>com.zaxxer</groupId>
|
||||
<artifactId>HikariCP</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.postgresql</groupId>
|
||||
<artifactId>postgresql</artifactId>
|
||||
<version>42.7.3</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>cn.hutool</groupId>
|
||||
<artifactId>hutool-crypto</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>tech.easyflow</groupId>
|
||||
<artifactId>easyflow-common-base</artifactId>
|
||||
|
||||
@@ -0,0 +1,12 @@
|
||||
package tech.easyflow.datacenter.connector;
|
||||
|
||||
import tech.easyflow.datacenter.meta.enums.DatacenterCapability;
|
||||
import tech.easyflow.datacenter.meta.enums.DatacenterSourceType;
|
||||
|
||||
import java.util.Set;
|
||||
|
||||
public interface DatacenterConnector extends SourceHealthChecker, MetadataExplorer, QueryExecutor, WriteExecutor {
|
||||
DatacenterSourceType getSourceType();
|
||||
|
||||
Set<DatacenterCapability> getCapabilities();
|
||||
}
|
||||
@@ -0,0 +1,29 @@
|
||||
package tech.easyflow.datacenter.connector;
|
||||
|
||||
import org.springframework.stereotype.Component;
|
||||
import tech.easyflow.common.web.exceptions.BusinessException;
|
||||
import tech.easyflow.datacenter.meta.enums.DatacenterSourceType;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.function.Function;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
@Component
|
||||
public class DatacenterConnectorRegistry {
|
||||
|
||||
private final Map<DatacenterSourceType, DatacenterConnector> connectorMap;
|
||||
|
||||
public DatacenterConnectorRegistry(List<DatacenterConnector> connectors) {
|
||||
this.connectorMap = connectors.stream().collect(Collectors.toMap(DatacenterConnector::getSourceType, Function.identity()));
|
||||
}
|
||||
|
||||
public DatacenterConnector getConnector(String sourceType) {
|
||||
DatacenterSourceType type = DatacenterSourceType.valueOf(sourceType);
|
||||
DatacenterConnector connector = connectorMap.get(type);
|
||||
if (connector == null) {
|
||||
throw new BusinessException("未找到连接器: " + sourceType);
|
||||
}
|
||||
return connector;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,16 @@
|
||||
package tech.easyflow.datacenter.connector;
|
||||
|
||||
import tech.easyflow.datacenter.entity.DatacenterTable;
|
||||
import tech.easyflow.datacenter.meta.entity.DatacenterSource;
|
||||
import tech.easyflow.datacenter.meta.model.DatacenterCatalogMeta;
|
||||
import tech.easyflow.datacenter.meta.model.DatacenterTableDetailMeta;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
public interface MetadataExplorer {
|
||||
List<DatacenterCatalogMeta> listCatalogs(DatacenterSource source);
|
||||
|
||||
List<DatacenterTable> listTables(DatacenterSource source, String catalogName);
|
||||
|
||||
DatacenterTableDetailMeta getTableDetail(DatacenterSource source, String catalogName, String tableName);
|
||||
}
|
||||
@@ -0,0 +1,15 @@
|
||||
package tech.easyflow.datacenter.connector;
|
||||
|
||||
import com.mybatisflex.core.paginate.Page;
|
||||
import com.mybatisflex.core.row.Row;
|
||||
import tech.easyflow.datacenter.entity.DatacenterTable;
|
||||
import tech.easyflow.datacenter.execution.model.DatacenterQueryRequest;
|
||||
import tech.easyflow.datacenter.meta.entity.DatacenterSource;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
public interface QueryExecutor {
|
||||
Page<Row> queryPage(DatacenterSource source, DatacenterTable table, DatacenterQueryRequest request);
|
||||
|
||||
List<Row> queryBySql(DatacenterSource source, String sql);
|
||||
}
|
||||
@@ -0,0 +1,8 @@
|
||||
package tech.easyflow.datacenter.connector;
|
||||
|
||||
import tech.easyflow.datacenter.execution.model.DatacenterConnectionTestResult;
|
||||
import tech.easyflow.datacenter.meta.entity.DatacenterSource;
|
||||
|
||||
public interface SourceHealthChecker {
|
||||
DatacenterConnectionTestResult testConnection(DatacenterSource source);
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
package tech.easyflow.datacenter.connector;
|
||||
|
||||
public interface SqlDialect {
|
||||
String quoteIdentifier(String identifier);
|
||||
|
||||
String qualifyTable(String namespace, String tableName);
|
||||
|
||||
String buildPageSql(String baseSql, long pageNumber, long pageSize);
|
||||
}
|
||||
@@ -0,0 +1,14 @@
|
||||
package tech.easyflow.datacenter.connector;
|
||||
|
||||
import com.alibaba.fastjson2.JSONObject;
|
||||
import tech.easyflow.common.entity.LoginAccount;
|
||||
import tech.easyflow.datacenter.entity.DatacenterTable;
|
||||
import tech.easyflow.datacenter.meta.entity.DatacenterSource;
|
||||
|
||||
import java.math.BigInteger;
|
||||
|
||||
public interface WriteExecutor {
|
||||
void saveRow(DatacenterSource source, DatacenterTable table, JSONObject data, LoginAccount account);
|
||||
|
||||
void deleteRow(DatacenterSource source, DatacenterTable table, BigInteger id, LoginAccount account);
|
||||
}
|
||||
@@ -0,0 +1,24 @@
|
||||
package tech.easyflow.datacenter.connector.dialect;
|
||||
|
||||
import cn.hutool.core.util.StrUtil;
|
||||
import tech.easyflow.datacenter.connector.SqlDialect;
|
||||
|
||||
public class GaussdbSqlDialect implements SqlDialect {
|
||||
@Override
|
||||
public String quoteIdentifier(String identifier) {
|
||||
return '"' + identifier + '"';
|
||||
}
|
||||
|
||||
@Override
|
||||
public String qualifyTable(String namespace, String tableName) {
|
||||
if (StrUtil.isBlank(namespace)) {
|
||||
return quoteIdentifier(tableName);
|
||||
}
|
||||
return quoteIdentifier(namespace) + "." + quoteIdentifier(tableName);
|
||||
}
|
||||
|
||||
@Override
|
||||
public String buildPageSql(String baseSql, long pageNumber, long pageSize) {
|
||||
return baseSql + " OFFSET ? LIMIT ?";
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,24 @@
|
||||
package tech.easyflow.datacenter.connector.dialect;
|
||||
|
||||
import cn.hutool.core.util.StrUtil;
|
||||
import tech.easyflow.datacenter.connector.SqlDialect;
|
||||
|
||||
public class MysqlSqlDialect implements SqlDialect {
|
||||
@Override
|
||||
public String quoteIdentifier(String identifier) {
|
||||
return "`" + identifier + "`";
|
||||
}
|
||||
|
||||
@Override
|
||||
public String qualifyTable(String namespace, String tableName) {
|
||||
if (StrUtil.isBlank(namespace)) {
|
||||
return quoteIdentifier(tableName);
|
||||
}
|
||||
return quoteIdentifier(namespace) + "." + quoteIdentifier(tableName);
|
||||
}
|
||||
|
||||
@Override
|
||||
public String buildPageSql(String baseSql, long pageNumber, long pageSize) {
|
||||
return baseSql + " LIMIT ?, ?";
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,24 @@
|
||||
package tech.easyflow.datacenter.connector.dialect;
|
||||
|
||||
import cn.hutool.core.util.StrUtil;
|
||||
import tech.easyflow.datacenter.connector.SqlDialect;
|
||||
|
||||
public class OracleSqlDialect implements SqlDialect {
|
||||
@Override
|
||||
public String quoteIdentifier(String identifier) {
|
||||
return '"' + identifier + '"';
|
||||
}
|
||||
|
||||
@Override
|
||||
public String qualifyTable(String namespace, String tableName) {
|
||||
if (StrUtil.isBlank(namespace)) {
|
||||
return quoteIdentifier(tableName);
|
||||
}
|
||||
return quoteIdentifier(namespace) + "." + quoteIdentifier(tableName);
|
||||
}
|
||||
|
||||
@Override
|
||||
public String buildPageSql(String baseSql, long pageNumber, long pageSize) {
|
||||
return baseSql + " OFFSET ? ROWS FETCH NEXT ? ROWS ONLY";
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,24 @@
|
||||
package tech.easyflow.datacenter.connector.dialect;
|
||||
|
||||
import cn.hutool.core.util.StrUtil;
|
||||
import tech.easyflow.datacenter.connector.SqlDialect;
|
||||
|
||||
public class PostgresqlSqlDialect implements SqlDialect {
|
||||
@Override
|
||||
public String quoteIdentifier(String identifier) {
|
||||
return '"' + identifier + '"';
|
||||
}
|
||||
|
||||
@Override
|
||||
public String qualifyTable(String namespace, String tableName) {
|
||||
if (StrUtil.isBlank(namespace)) {
|
||||
return quoteIdentifier(tableName);
|
||||
}
|
||||
return quoteIdentifier(namespace) + "." + quoteIdentifier(tableName);
|
||||
}
|
||||
|
||||
@Override
|
||||
public String buildPageSql(String baseSql, long pageNumber, long pageSize) {
|
||||
return baseSql + " OFFSET ? LIMIT ?";
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,22 @@
|
||||
package tech.easyflow.datacenter.connector.impl;
|
||||
|
||||
import org.springframework.stereotype.Component;
|
||||
import tech.easyflow.datacenter.connector.support.AbstractInternalTableConnector;
|
||||
import tech.easyflow.datacenter.meta.enums.DatacenterCapability;
|
||||
import tech.easyflow.datacenter.meta.enums.DatacenterSourceType;
|
||||
|
||||
import java.util.EnumSet;
|
||||
|
||||
@Component
|
||||
public class ExcelConnector extends AbstractInternalTableConnector {
|
||||
public ExcelConnector() {
|
||||
super(DatacenterSourceType.EXCEL, EnumSet.of(
|
||||
DatacenterCapability.TEST_CONNECTION,
|
||||
DatacenterCapability.BROWSE_METADATA,
|
||||
DatacenterCapability.READ_QUERY,
|
||||
DatacenterCapability.WRITE_MUTATION,
|
||||
DatacenterCapability.MATERIALIZE,
|
||||
DatacenterCapability.EXPORT
|
||||
));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,22 @@
|
||||
package tech.easyflow.datacenter.connector.impl;
|
||||
|
||||
import org.springframework.stereotype.Component;
|
||||
import tech.easyflow.datacenter.connector.support.AbstractInternalTableConnector;
|
||||
import tech.easyflow.datacenter.meta.enums.DatacenterCapability;
|
||||
import tech.easyflow.datacenter.meta.enums.DatacenterSourceType;
|
||||
|
||||
import java.util.EnumSet;
|
||||
|
||||
@Component
|
||||
public class ExcelMaterializedConnector extends AbstractInternalTableConnector {
|
||||
public ExcelMaterializedConnector() {
|
||||
super(DatacenterSourceType.EXCEL_MATERIALIZED, EnumSet.of(
|
||||
DatacenterCapability.TEST_CONNECTION,
|
||||
DatacenterCapability.BROWSE_METADATA,
|
||||
DatacenterCapability.READ_QUERY,
|
||||
DatacenterCapability.WRITE_MUTATION,
|
||||
DatacenterCapability.MATERIALIZE,
|
||||
DatacenterCapability.EXPORT
|
||||
));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,40 @@
|
||||
package tech.easyflow.datacenter.connector.impl;
|
||||
|
||||
import com.zaxxer.hikari.HikariDataSource;
|
||||
import org.springframework.stereotype.Component;
|
||||
import tech.easyflow.datacenter.connector.dialect.GaussdbSqlDialect;
|
||||
import tech.easyflow.datacenter.connector.support.AbstractJdbcConnector;
|
||||
import tech.easyflow.datacenter.connector.support.DatacenterDatasourceManager;
|
||||
import tech.easyflow.datacenter.meta.entity.DatacenterSource;
|
||||
import tech.easyflow.datacenter.meta.enums.DatacenterCapability;
|
||||
import tech.easyflow.datacenter.meta.enums.DatacenterSourceType;
|
||||
|
||||
import java.sql.Connection;
|
||||
import java.util.EnumSet;
|
||||
|
||||
@Component
|
||||
public class GaussdbNativeConnector extends AbstractJdbcConnector {
|
||||
|
||||
private final DatacenterDatasourceManager datasourceManager;
|
||||
|
||||
public GaussdbNativeConnector(DatacenterDatasourceManager datasourceManager) {
|
||||
super(DatacenterSourceType.GAUSSDB_NATIVE, new GaussdbSqlDialect(), EnumSet.of(
|
||||
DatacenterCapability.TEST_CONNECTION,
|
||||
DatacenterCapability.BROWSE_METADATA,
|
||||
DatacenterCapability.READ_QUERY
|
||||
));
|
||||
this.datasourceManager = datasourceManager;
|
||||
}
|
||||
|
||||
@Override
|
||||
protected <T> T withConnection(DatacenterSource source, boolean cacheable, JdbcCallback<T> callback) throws Exception {
|
||||
HikariDataSource dataSource = cacheable ? datasourceManager.getOrCreateExternalDatasource(source) : datasourceManager.createExternalDatasource(source);
|
||||
try (Connection connection = dataSource.getConnection()) {
|
||||
return callback.apply(connection);
|
||||
} finally {
|
||||
if (!cacheable || source.getId() == null) {
|
||||
dataSource.close();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,40 @@
|
||||
package tech.easyflow.datacenter.connector.impl;
|
||||
|
||||
import com.zaxxer.hikari.HikariDataSource;
|
||||
import org.springframework.stereotype.Component;
|
||||
import tech.easyflow.datacenter.connector.dialect.MysqlSqlDialect;
|
||||
import tech.easyflow.datacenter.connector.support.AbstractJdbcConnector;
|
||||
import tech.easyflow.datacenter.connector.support.DatacenterDatasourceManager;
|
||||
import tech.easyflow.datacenter.meta.entity.DatacenterSource;
|
||||
import tech.easyflow.datacenter.meta.enums.DatacenterCapability;
|
||||
import tech.easyflow.datacenter.meta.enums.DatacenterSourceType;
|
||||
|
||||
import java.sql.Connection;
|
||||
import java.util.EnumSet;
|
||||
|
||||
@Component
|
||||
public class Gbase8aConnector extends AbstractJdbcConnector {
|
||||
|
||||
private final DatacenterDatasourceManager datasourceManager;
|
||||
|
||||
public Gbase8aConnector(DatacenterDatasourceManager datasourceManager) {
|
||||
super(DatacenterSourceType.GBASE_8A, new MysqlSqlDialect(), EnumSet.of(
|
||||
DatacenterCapability.TEST_CONNECTION,
|
||||
DatacenterCapability.BROWSE_METADATA,
|
||||
DatacenterCapability.READ_QUERY
|
||||
));
|
||||
this.datasourceManager = datasourceManager;
|
||||
}
|
||||
|
||||
@Override
|
||||
protected <T> T withConnection(DatacenterSource source, boolean cacheable, JdbcCallback<T> callback) throws Exception {
|
||||
HikariDataSource dataSource = cacheable ? datasourceManager.getOrCreateExternalDatasource(source) : datasourceManager.createExternalDatasource(source);
|
||||
try (Connection connection = dataSource.getConnection()) {
|
||||
return callback.apply(connection);
|
||||
} finally {
|
||||
if (!cacheable || source.getId() == null) {
|
||||
dataSource.close();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,40 @@
|
||||
package tech.easyflow.datacenter.connector.impl;
|
||||
|
||||
import com.zaxxer.hikari.HikariDataSource;
|
||||
import org.springframework.stereotype.Component;
|
||||
import tech.easyflow.datacenter.connector.dialect.MysqlSqlDialect;
|
||||
import tech.easyflow.datacenter.connector.support.AbstractJdbcConnector;
|
||||
import tech.easyflow.datacenter.connector.support.DatacenterDatasourceManager;
|
||||
import tech.easyflow.datacenter.meta.entity.DatacenterSource;
|
||||
import tech.easyflow.datacenter.meta.enums.DatacenterCapability;
|
||||
import tech.easyflow.datacenter.meta.enums.DatacenterSourceType;
|
||||
|
||||
import java.sql.Connection;
|
||||
import java.util.EnumSet;
|
||||
|
||||
@Component
|
||||
public class Gbase8sConnector extends AbstractJdbcConnector {
|
||||
|
||||
private final DatacenterDatasourceManager datasourceManager;
|
||||
|
||||
public Gbase8sConnector(DatacenterDatasourceManager datasourceManager) {
|
||||
super(DatacenterSourceType.GBASE_8S, new MysqlSqlDialect(), EnumSet.of(
|
||||
DatacenterCapability.TEST_CONNECTION,
|
||||
DatacenterCapability.BROWSE_METADATA,
|
||||
DatacenterCapability.READ_QUERY
|
||||
));
|
||||
this.datasourceManager = datasourceManager;
|
||||
}
|
||||
|
||||
@Override
|
||||
protected <T> T withConnection(DatacenterSource source, boolean cacheable, JdbcCallback<T> callback) throws Exception {
|
||||
HikariDataSource dataSource = cacheable ? datasourceManager.getOrCreateExternalDatasource(source) : datasourceManager.createExternalDatasource(source);
|
||||
try (Connection connection = dataSource.getConnection()) {
|
||||
return callback.apply(connection);
|
||||
} finally {
|
||||
if (!cacheable || source.getId() == null) {
|
||||
dataSource.close();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,40 @@
|
||||
package tech.easyflow.datacenter.connector.impl;
|
||||
|
||||
import com.zaxxer.hikari.HikariDataSource;
|
||||
import org.springframework.stereotype.Component;
|
||||
import tech.easyflow.datacenter.connector.dialect.MysqlSqlDialect;
|
||||
import tech.easyflow.datacenter.connector.support.AbstractJdbcConnector;
|
||||
import tech.easyflow.datacenter.connector.support.DatacenterDatasourceManager;
|
||||
import tech.easyflow.datacenter.meta.entity.DatacenterSource;
|
||||
import tech.easyflow.datacenter.meta.enums.DatacenterCapability;
|
||||
import tech.easyflow.datacenter.meta.enums.DatacenterSourceType;
|
||||
|
||||
import java.sql.Connection;
|
||||
import java.util.EnumSet;
|
||||
|
||||
@Component
|
||||
public class MysqlConnector extends AbstractJdbcConnector {
|
||||
|
||||
private final DatacenterDatasourceManager datasourceManager;
|
||||
|
||||
public MysqlConnector(DatacenterDatasourceManager datasourceManager) {
|
||||
super(DatacenterSourceType.MYSQL, new MysqlSqlDialect(), EnumSet.of(
|
||||
DatacenterCapability.TEST_CONNECTION,
|
||||
DatacenterCapability.BROWSE_METADATA,
|
||||
DatacenterCapability.READ_QUERY
|
||||
));
|
||||
this.datasourceManager = datasourceManager;
|
||||
}
|
||||
|
||||
@Override
|
||||
protected <T> T withConnection(DatacenterSource source, boolean cacheable, JdbcCallback<T> callback) throws Exception {
|
||||
HikariDataSource dataSource = cacheable ? datasourceManager.getOrCreateExternalDatasource(source) : datasourceManager.createExternalDatasource(source);
|
||||
try (Connection connection = dataSource.getConnection()) {
|
||||
return callback.apply(connection);
|
||||
} finally {
|
||||
if (!cacheable || source.getId() == null) {
|
||||
dataSource.close();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,40 @@
|
||||
package tech.easyflow.datacenter.connector.impl;
|
||||
|
||||
import com.zaxxer.hikari.HikariDataSource;
|
||||
import org.springframework.stereotype.Component;
|
||||
import tech.easyflow.datacenter.connector.dialect.OracleSqlDialect;
|
||||
import tech.easyflow.datacenter.connector.support.AbstractJdbcConnector;
|
||||
import tech.easyflow.datacenter.connector.support.DatacenterDatasourceManager;
|
||||
import tech.easyflow.datacenter.meta.entity.DatacenterSource;
|
||||
import tech.easyflow.datacenter.meta.enums.DatacenterCapability;
|
||||
import tech.easyflow.datacenter.meta.enums.DatacenterSourceType;
|
||||
|
||||
import java.sql.Connection;
|
||||
import java.util.EnumSet;
|
||||
|
||||
@Component
|
||||
public class OracleConnector extends AbstractJdbcConnector {
|
||||
|
||||
private final DatacenterDatasourceManager datasourceManager;
|
||||
|
||||
public OracleConnector(DatacenterDatasourceManager datasourceManager) {
|
||||
super(DatacenterSourceType.ORACLE, new OracleSqlDialect(), EnumSet.of(
|
||||
DatacenterCapability.TEST_CONNECTION,
|
||||
DatacenterCapability.BROWSE_METADATA,
|
||||
DatacenterCapability.READ_QUERY
|
||||
));
|
||||
this.datasourceManager = datasourceManager;
|
||||
}
|
||||
|
||||
@Override
|
||||
protected <T> T withConnection(DatacenterSource source, boolean cacheable, JdbcCallback<T> callback) throws Exception {
|
||||
HikariDataSource dataSource = cacheable ? datasourceManager.getOrCreateExternalDatasource(source) : datasourceManager.createExternalDatasource(source);
|
||||
try (Connection connection = dataSource.getConnection()) {
|
||||
return callback.apply(connection);
|
||||
} finally {
|
||||
if (!cacheable || source.getId() == null) {
|
||||
dataSource.close();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,40 @@
|
||||
package tech.easyflow.datacenter.connector.impl;
|
||||
|
||||
import com.zaxxer.hikari.HikariDataSource;
|
||||
import org.springframework.stereotype.Component;
|
||||
import tech.easyflow.datacenter.connector.dialect.PostgresqlSqlDialect;
|
||||
import tech.easyflow.datacenter.connector.support.AbstractJdbcConnector;
|
||||
import tech.easyflow.datacenter.connector.support.DatacenterDatasourceManager;
|
||||
import tech.easyflow.datacenter.meta.entity.DatacenterSource;
|
||||
import tech.easyflow.datacenter.meta.enums.DatacenterCapability;
|
||||
import tech.easyflow.datacenter.meta.enums.DatacenterSourceType;
|
||||
|
||||
import java.sql.Connection;
|
||||
import java.util.EnumSet;
|
||||
|
||||
@Component
|
||||
public class PostgresqlConnector extends AbstractJdbcConnector {
|
||||
|
||||
private final DatacenterDatasourceManager datasourceManager;
|
||||
|
||||
public PostgresqlConnector(DatacenterDatasourceManager datasourceManager) {
|
||||
super(DatacenterSourceType.POSTGRESQL, new PostgresqlSqlDialect(), EnumSet.of(
|
||||
DatacenterCapability.TEST_CONNECTION,
|
||||
DatacenterCapability.BROWSE_METADATA,
|
||||
DatacenterCapability.READ_QUERY
|
||||
));
|
||||
this.datasourceManager = datasourceManager;
|
||||
}
|
||||
|
||||
@Override
|
||||
protected <T> T withConnection(DatacenterSource source, boolean cacheable, JdbcCallback<T> callback) throws Exception {
|
||||
HikariDataSource dataSource = cacheable ? datasourceManager.getOrCreateExternalDatasource(source) : datasourceManager.createExternalDatasource(source);
|
||||
try (Connection connection = dataSource.getConnection()) {
|
||||
return callback.apply(connection);
|
||||
} finally {
|
||||
if (!cacheable || source.getId() == null) {
|
||||
dataSource.close();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,135 @@
|
||||
package tech.easyflow.datacenter.connector.impl;
|
||||
|
||||
import cn.hutool.core.util.StrUtil;
|
||||
import com.alibaba.fastjson2.JSONObject;
|
||||
import com.mybatisflex.core.paginate.Page;
|
||||
import com.mybatisflex.core.row.Row;
|
||||
import org.springframework.stereotype.Component;
|
||||
import tech.easyflow.common.entity.LoginAccount;
|
||||
import tech.easyflow.common.web.exceptions.BusinessException;
|
||||
import tech.easyflow.datacenter.connector.dialect.MysqlSqlDialect;
|
||||
import tech.easyflow.datacenter.connector.support.AbstractJdbcConnector;
|
||||
import tech.easyflow.datacenter.entity.DatacenterTable;
|
||||
import tech.easyflow.datacenter.entity.DatacenterTableField;
|
||||
import tech.easyflow.datacenter.execution.model.DatacenterQueryRequest;
|
||||
import tech.easyflow.datacenter.meta.entity.DatacenterSource;
|
||||
import tech.easyflow.datacenter.meta.enums.DatacenterCapability;
|
||||
import tech.easyflow.datacenter.meta.enums.DatacenterSourceType;
|
||||
|
||||
import javax.sql.DataSource;
|
||||
import java.math.BigInteger;
|
||||
import java.sql.Connection;
|
||||
import java.sql.PreparedStatement;
|
||||
import java.util.ArrayList;
|
||||
import java.util.EnumSet;
|
||||
import java.util.List;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
@Component
|
||||
public class ProjectMysqlConnector extends AbstractJdbcConnector {
|
||||
|
||||
private final DataSource dataSource;
|
||||
private final MysqlSqlDialect dialect = new MysqlSqlDialect();
|
||||
|
||||
public ProjectMysqlConnector(DataSource dataSource) {
|
||||
super(DatacenterSourceType.PROJECT_MYSQL, new MysqlSqlDialect(), EnumSet.of(
|
||||
DatacenterCapability.TEST_CONNECTION,
|
||||
DatacenterCapability.BROWSE_METADATA,
|
||||
DatacenterCapability.READ_QUERY,
|
||||
DatacenterCapability.WRITE_MUTATION
|
||||
));
|
||||
this.dataSource = dataSource;
|
||||
}
|
||||
|
||||
@Override
|
||||
protected <T> T withConnection(DatacenterSource source, boolean cacheable, JdbcCallback<T> callback) throws Exception {
|
||||
try (Connection connection = dataSource.getConnection()) {
|
||||
return callback.apply(connection);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
protected boolean requiresJdbcUrl() {
|
||||
return false;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Page<Row> queryPage(DatacenterSource source, DatacenterTable table, DatacenterQueryRequest request) {
|
||||
return super.queryPage(source, table, request);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void saveRow(DatacenterSource source, DatacenterTable table, JSONObject data, LoginAccount account) {
|
||||
List<DatacenterTableField> writableFields = table.getFields().stream()
|
||||
.filter(field -> field.getWritable() == null || field.getWritable() == 1)
|
||||
.collect(Collectors.toList());
|
||||
Object id = data.get("id");
|
||||
try (Connection connection = dataSource.getConnection()) {
|
||||
if (id == null) {
|
||||
List<String> columns = new ArrayList<>();
|
||||
List<Object> values = new ArrayList<>();
|
||||
for (DatacenterTableField field : writableFields) {
|
||||
Object value = data.get(field.getFieldName());
|
||||
if (value != null) {
|
||||
columns.add(field.getFieldName());
|
||||
values.add(value);
|
||||
}
|
||||
}
|
||||
if (columns.isEmpty()) {
|
||||
throw new BusinessException("没有可写字段");
|
||||
}
|
||||
String sql = "INSERT INTO " + dialect.qualifyTable(source.getDatabaseName(), resolvePhysicalTableName(table))
|
||||
+ " (" + columns.stream().map(dialect::quoteIdentifier).collect(Collectors.joining(",")) + ") VALUES ("
|
||||
+ columns.stream().map(item -> "?").collect(Collectors.joining(",")) + ")";
|
||||
try (PreparedStatement statement = connection.prepareStatement(sql)) {
|
||||
for (int i = 0; i < values.size(); i++) {
|
||||
statement.setObject(i + 1, values.get(i));
|
||||
}
|
||||
statement.executeUpdate();
|
||||
}
|
||||
return;
|
||||
}
|
||||
List<String> setClauses = new ArrayList<>();
|
||||
List<Object> values = new ArrayList<>();
|
||||
for (DatacenterTableField field : writableFields) {
|
||||
if (!data.containsKey(field.getFieldName())) {
|
||||
continue;
|
||||
}
|
||||
setClauses.add(dialect.quoteIdentifier(field.getFieldName()) + " = ?");
|
||||
values.add(data.get(field.getFieldName()));
|
||||
}
|
||||
if (setClauses.isEmpty()) {
|
||||
return;
|
||||
}
|
||||
String sql = "UPDATE " + dialect.qualifyTable(source.getDatabaseName(), resolvePhysicalTableName(table))
|
||||
+ " SET " + String.join(",", setClauses)
|
||||
+ " WHERE " + dialect.quoteIdentifier("id") + " = ?";
|
||||
values.add(id);
|
||||
try (PreparedStatement statement = connection.prepareStatement(sql)) {
|
||||
for (int i = 0; i < values.size(); i++) {
|
||||
statement.setObject(i + 1, values.get(i));
|
||||
}
|
||||
statement.executeUpdate();
|
||||
}
|
||||
} catch (Exception ex) {
|
||||
throw new BusinessException("项目 MySQL 写入失败: " + ex.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void deleteRow(DatacenterSource source, DatacenterTable table, BigInteger id, LoginAccount account) {
|
||||
String sql = "DELETE FROM " + dialect.qualifyTable(source.getDatabaseName(), resolvePhysicalTableName(table))
|
||||
+ " WHERE " + dialect.quoteIdentifier("id") + " = ?";
|
||||
try (Connection connection = dataSource.getConnection(); PreparedStatement statement = connection.prepareStatement(sql)) {
|
||||
statement.setObject(1, id);
|
||||
statement.executeUpdate();
|
||||
} catch (Exception ex) {
|
||||
throw new BusinessException("项目 MySQL 删除失败: " + ex.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
protected String resolveCatalogName(DatacenterSource source, String requestedCatalogName) {
|
||||
return StrUtil.blankToDefault(requestedCatalogName, source.getDatabaseName());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,152 @@
|
||||
package tech.easyflow.datacenter.connector.support;
|
||||
|
||||
import cn.hutool.core.util.StrUtil;
|
||||
import com.alibaba.fastjson2.JSONObject;
|
||||
import com.mybatisflex.core.paginate.Page;
|
||||
import com.mybatisflex.core.query.QueryWrapper;
|
||||
import com.mybatisflex.core.row.Db;
|
||||
import com.mybatisflex.core.row.Row;
|
||||
import com.mybatisflex.core.row.RowKey;
|
||||
import org.springframework.util.CollectionUtils;
|
||||
import tech.easyflow.common.entity.LoginAccount;
|
||||
import tech.easyflow.common.web.exceptions.BusinessException;
|
||||
import tech.easyflow.datacenter.connector.DatacenterConnector;
|
||||
import tech.easyflow.datacenter.entity.DatacenterTable;
|
||||
import tech.easyflow.datacenter.entity.DatacenterTableField;
|
||||
import tech.easyflow.datacenter.execution.model.DatacenterConnectionTestResult;
|
||||
import tech.easyflow.datacenter.execution.model.DatacenterQueryRequest;
|
||||
import tech.easyflow.datacenter.meta.entity.DatacenterSource;
|
||||
import tech.easyflow.datacenter.meta.enums.DatacenterCapability;
|
||||
import tech.easyflow.datacenter.meta.enums.DatacenterSourceType;
|
||||
import tech.easyflow.datacenter.meta.model.DatacenterCatalogMeta;
|
||||
import tech.easyflow.datacenter.meta.model.DatacenterTableDetailMeta;
|
||||
|
||||
import java.math.BigDecimal;
|
||||
import java.math.BigInteger;
|
||||
import java.util.*;
|
||||
|
||||
public abstract class AbstractInternalTableConnector implements DatacenterConnector {
|
||||
|
||||
private final DatacenterSourceType sourceType;
|
||||
private final Set<DatacenterCapability> capabilities;
|
||||
|
||||
protected AbstractInternalTableConnector(DatacenterSourceType sourceType, Set<DatacenterCapability> capabilities) {
|
||||
this.sourceType = sourceType;
|
||||
this.capabilities = capabilities;
|
||||
}
|
||||
|
||||
@Override
|
||||
public DatacenterSourceType getSourceType() {
|
||||
return sourceType;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Set<DatacenterCapability> getCapabilities() {
|
||||
return capabilities;
|
||||
}
|
||||
|
||||
@Override
|
||||
public DatacenterConnectionTestResult testConnection(DatacenterSource source) {
|
||||
DatacenterConnectionTestResult result = new DatacenterConnectionTestResult();
|
||||
result.setSuccess(true);
|
||||
result.setMessage("连接成功");
|
||||
result.setCapabilities(capabilities.stream().map(Enum::name).toList());
|
||||
result.setDetails(Map.of("sourceType", sourceType.name()));
|
||||
return result;
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<DatacenterCatalogMeta> listCatalogs(DatacenterSource source) {
|
||||
return Collections.emptyList();
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<DatacenterTable> listTables(DatacenterSource source, String catalogName) {
|
||||
return Collections.emptyList();
|
||||
}
|
||||
|
||||
@Override
|
||||
public DatacenterTableDetailMeta getTableDetail(DatacenterSource source, String catalogName, String tableName) {
|
||||
throw new BusinessException("内部数据源请从元数据注册表读取表结构");
|
||||
}
|
||||
|
||||
@Override
|
||||
public Page<Row> queryPage(DatacenterSource source, DatacenterTable table, DatacenterQueryRequest request) {
|
||||
String actualTable = resolveTableName(table);
|
||||
QueryWrapper wrapper = QueryWrapper.create();
|
||||
if (StrUtil.isNotBlank(request.getWhere())) {
|
||||
wrapper.where(request.getWhere());
|
||||
}
|
||||
long count = Db.selectCountByQuery(actualTable, wrapper);
|
||||
if (count == 0) {
|
||||
return new Page<>(new ArrayList<>(), request.getPageNumber(), request.getPageSize(), count);
|
||||
}
|
||||
Page<Row> page = Db.paginate(actualTable, new Page<>(request.getPageNumber(), request.getPageSize(), count), wrapper);
|
||||
normalizeRows(page.getRecords());
|
||||
return page;
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<Row> queryBySql(DatacenterSource source, String sql) {
|
||||
List<Row> rows = Db.selectListBySql(sql);
|
||||
normalizeRows(rows);
|
||||
return rows;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void saveRow(DatacenterSource source, DatacenterTable table, JSONObject data, LoginAccount account) {
|
||||
List<DatacenterTableField> fields = table.getFields();
|
||||
if (CollectionUtils.isEmpty(fields)) {
|
||||
throw new BusinessException("数据集字段为空,无法写入");
|
||||
}
|
||||
String actualTable = resolveTableName(table);
|
||||
Object id = data.get("id");
|
||||
if (id == null) {
|
||||
Row row = Row.ofKey(RowKey.SNOW_FLAKE_ID);
|
||||
row.put("dept_id", account.getDeptId());
|
||||
row.put("tenant_id", account.getTenantId());
|
||||
row.put("created", new Date());
|
||||
row.put("created_by", account.getId());
|
||||
row.put("modified", new Date());
|
||||
row.put("modified_by", account.getId());
|
||||
row.put("remark", data.get("remark"));
|
||||
for (DatacenterTableField field : fields) {
|
||||
row.put(field.getFieldName(), data.get(field.getFieldName()));
|
||||
}
|
||||
Db.insert(actualTable, row);
|
||||
return;
|
||||
}
|
||||
Row row = Row.ofKey("id", id);
|
||||
row.put("modified", new Date());
|
||||
row.put("modified_by", account.getId());
|
||||
for (DatacenterTableField field : fields) {
|
||||
row.put(field.getFieldName(), data.get(field.getFieldName()));
|
||||
}
|
||||
Db.updateById(actualTable, row);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void deleteRow(DatacenterSource source, DatacenterTable table, BigInteger id, LoginAccount account) {
|
||||
Db.deleteById(resolveTableName(table), Row.ofKey("id", id));
|
||||
}
|
||||
|
||||
protected String resolveTableName(DatacenterTable table) {
|
||||
return StrUtil.blankToDefault(table.getMaterializedTable(), table.getActualTable());
|
||||
}
|
||||
|
||||
private void normalizeRows(List<Row> records) {
|
||||
for (Row record : records) {
|
||||
Map<String, Object> converted = new LinkedHashMap<>();
|
||||
for (Map.Entry<String, Object> entry : record.entrySet()) {
|
||||
Object value = entry.getValue();
|
||||
if (value instanceof BigInteger || value instanceof BigDecimal || value instanceof Long) {
|
||||
converted.put(entry.getKey(), value.toString());
|
||||
} else {
|
||||
converted.put(entry.getKey(), value);
|
||||
}
|
||||
}
|
||||
record.clear();
|
||||
record.putAll(converted);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,523 @@
|
||||
package tech.easyflow.datacenter.connector.support;
|
||||
|
||||
import cn.hutool.core.util.StrUtil;
|
||||
import com.alibaba.fastjson2.JSONObject;
|
||||
import com.mybatisflex.core.paginate.Page;
|
||||
import com.mybatisflex.core.row.Row;
|
||||
import org.springframework.util.CollectionUtils;
|
||||
import tech.easyflow.common.constant.enums.EnumFieldType;
|
||||
import tech.easyflow.common.entity.LoginAccount;
|
||||
import tech.easyflow.common.web.exceptions.BusinessException;
|
||||
import tech.easyflow.datacenter.connector.DatacenterConnector;
|
||||
import tech.easyflow.datacenter.connector.SqlDialect;
|
||||
import tech.easyflow.datacenter.entity.DatacenterTable;
|
||||
import tech.easyflow.datacenter.entity.DatacenterTableField;
|
||||
import tech.easyflow.datacenter.execution.model.DatacenterConnectionTestResult;
|
||||
import tech.easyflow.datacenter.execution.model.DatacenterQueryFilter;
|
||||
import tech.easyflow.datacenter.execution.model.DatacenterQueryRequest;
|
||||
import tech.easyflow.datacenter.execution.model.DatacenterQuerySort;
|
||||
import tech.easyflow.datacenter.meta.entity.DatacenterSource;
|
||||
import tech.easyflow.datacenter.meta.enums.DatacenterCapability;
|
||||
import tech.easyflow.datacenter.meta.enums.DatacenterConnectionErrorCode;
|
||||
import tech.easyflow.datacenter.meta.enums.DatacenterSourceType;
|
||||
import tech.easyflow.datacenter.meta.enums.DatacenterTableKind;
|
||||
import tech.easyflow.datacenter.meta.model.DatacenterCatalogMeta;
|
||||
import tech.easyflow.datacenter.meta.model.DatacenterTableDetailMeta;
|
||||
|
||||
import java.math.BigDecimal;
|
||||
import java.math.BigInteger;
|
||||
import java.sql.*;
|
||||
import java.util.*;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
public abstract class AbstractJdbcConnector implements DatacenterConnector {
|
||||
|
||||
private final DatacenterSourceType sourceType;
|
||||
private final SqlDialect sqlDialect;
|
||||
private final Set<DatacenterCapability> capabilities;
|
||||
|
||||
protected AbstractJdbcConnector(DatacenterSourceType sourceType, SqlDialect sqlDialect, Set<DatacenterCapability> capabilities) {
|
||||
this.sourceType = sourceType;
|
||||
this.sqlDialect = sqlDialect;
|
||||
this.capabilities = capabilities;
|
||||
}
|
||||
|
||||
@Override
|
||||
public DatacenterSourceType getSourceType() {
|
||||
return sourceType;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Set<DatacenterCapability> getCapabilities() {
|
||||
return capabilities;
|
||||
}
|
||||
|
||||
protected abstract <T> T withConnection(DatacenterSource source, boolean cacheable, JdbcCallback<T> callback) throws Exception;
|
||||
|
||||
@Override
|
||||
public DatacenterConnectionTestResult testConnection(DatacenterSource source) {
|
||||
DatacenterConnectionTestResult result = new DatacenterConnectionTestResult();
|
||||
result.setCapabilities(capabilities.stream().map(Enum::name).toList());
|
||||
if (StrUtil.isBlank(source.getJdbcUrl()) && requiresJdbcUrl()) {
|
||||
result.setSuccess(false);
|
||||
result.setErrorCode(DatacenterConnectionErrorCode.INVALID_ARGUMENT.name());
|
||||
result.setMessage("缺少 JDBC URL");
|
||||
return result;
|
||||
}
|
||||
try {
|
||||
if (StrUtil.isNotBlank(source.getDriverClassName())) {
|
||||
Class.forName(source.getDriverClassName());
|
||||
}
|
||||
Map<String, Object> details = withConnection(source, false, connection -> {
|
||||
DatabaseMetaData metaData = connection.getMetaData();
|
||||
Map<String, Object> map = new LinkedHashMap<>();
|
||||
map.put("databaseProductName", metaData.getDatabaseProductName());
|
||||
map.put("databaseProductVersion", metaData.getDatabaseProductVersion());
|
||||
map.put("url", metaData.getURL());
|
||||
return map;
|
||||
});
|
||||
result.setSuccess(true);
|
||||
result.setMessage("连接成功");
|
||||
result.setDetails(details);
|
||||
return result;
|
||||
} catch (ClassNotFoundException ex) {
|
||||
result.setSuccess(false);
|
||||
result.setErrorCode(DatacenterConnectionErrorCode.DRIVER_NOT_FOUND.name());
|
||||
result.setMessage(ex.getMessage());
|
||||
return result;
|
||||
} catch (SQLException ex) {
|
||||
result.setSuccess(false);
|
||||
result.setErrorCode(mapSqlError(ex));
|
||||
result.setMessage(ex.getMessage());
|
||||
return result;
|
||||
} catch (Exception ex) {
|
||||
result.setSuccess(false);
|
||||
result.setErrorCode(DatacenterConnectionErrorCode.UNKNOWN_ERROR.name());
|
||||
result.setMessage(ex.getMessage());
|
||||
return result;
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<DatacenterCatalogMeta> listCatalogs(DatacenterSource source) {
|
||||
try {
|
||||
return withConnection(source, true, connection -> {
|
||||
List<DatacenterCatalogMeta> result = new ArrayList<>();
|
||||
DatabaseMetaData metaData = connection.getMetaData();
|
||||
try (ResultSet catalogs = metaData.getCatalogs()) {
|
||||
while (catalogs.next()) {
|
||||
String name = catalogs.getString("TABLE_CAT");
|
||||
if (StrUtil.isBlank(name)) {
|
||||
continue;
|
||||
}
|
||||
DatacenterCatalogMeta meta = new DatacenterCatalogMeta();
|
||||
meta.setSourceId(source.getId());
|
||||
meta.setCatalogName(name);
|
||||
meta.setCatalogType("CATALOG");
|
||||
result.add(meta);
|
||||
}
|
||||
}
|
||||
try (ResultSet schemas = metaData.getSchemas()) {
|
||||
while (schemas.next()) {
|
||||
String name = schemas.getString("TABLE_SCHEM");
|
||||
if (StrUtil.isBlank(name) || containsCatalog(result, name)) {
|
||||
continue;
|
||||
}
|
||||
DatacenterCatalogMeta meta = new DatacenterCatalogMeta();
|
||||
meta.setSourceId(source.getId());
|
||||
meta.setCatalogName(name);
|
||||
meta.setCatalogType("SCHEMA");
|
||||
result.add(meta);
|
||||
}
|
||||
}
|
||||
result = filterConfiguredCatalogs(source, result);
|
||||
if (result.isEmpty()) {
|
||||
String fallback = resolveCatalogName(source, null);
|
||||
if (StrUtil.isNotBlank(fallback)) {
|
||||
DatacenterCatalogMeta meta = new DatacenterCatalogMeta();
|
||||
meta.setSourceId(source.getId());
|
||||
meta.setCatalogName(fallback);
|
||||
meta.setCatalogType("DEFAULT");
|
||||
result.add(meta);
|
||||
}
|
||||
}
|
||||
return result;
|
||||
});
|
||||
} catch (Exception ex) {
|
||||
throw DatacenterConnectorExceptionSupport.wrapAccessException("读取目录失败", ex);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<DatacenterTable> listTables(DatacenterSource source, String catalogName) {
|
||||
try {
|
||||
return withConnection(source, true, connection -> {
|
||||
DatabaseMetaData metaData = connection.getMetaData();
|
||||
List<DatacenterTable> tables = new ArrayList<>();
|
||||
try (ResultSet resultSet = metaData.getTables(resolveCatalogArgument(source, catalogName), resolveSchemaArgument(source, catalogName), "%", new String[]{"TABLE", "VIEW"})) {
|
||||
while (resultSet.next()) {
|
||||
DatacenterTable table = new DatacenterTable();
|
||||
table.setSourceId(source.getId());
|
||||
table.setTableName(resultSet.getString("TABLE_NAME"));
|
||||
table.setTableDesc(resultSet.getString("REMARKS"));
|
||||
table.setActualTable(resultSet.getString("TABLE_NAME"));
|
||||
table.setMaterializedTable(resultSet.getString("TABLE_NAME"));
|
||||
table.setAccessMode(capabilities.contains(DatacenterCapability.WRITE_MUTATION) ? "READ_WRITE" : "READ_ONLY");
|
||||
table.setTableKind(resolveTableKind(resultSet.getString("TABLE_TYPE")).name());
|
||||
table.setCapabilitiesJson(Map.of("capabilities", capabilities.stream().map(Enum::name).toList()));
|
||||
tables.add(table);
|
||||
}
|
||||
}
|
||||
return tables;
|
||||
});
|
||||
} catch (Exception ex) {
|
||||
throw DatacenterConnectorExceptionSupport.wrapAccessException("读取表列表失败", ex);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public DatacenterTableDetailMeta getTableDetail(DatacenterSource source, String catalogName, String tableName) {
|
||||
try {
|
||||
return withConnection(source, true, connection -> {
|
||||
DatabaseMetaData metaData = connection.getMetaData();
|
||||
DatacenterTableDetailMeta detail = new DatacenterTableDetailMeta();
|
||||
DatacenterTable table = new DatacenterTable();
|
||||
table.setSourceId(source.getId());
|
||||
table.setTableName(tableName);
|
||||
table.setActualTable(tableName);
|
||||
table.setMaterializedTable(tableName);
|
||||
table.setAccessMode(capabilities.contains(DatacenterCapability.WRITE_MUTATION) ? "READ_WRITE" : "READ_ONLY");
|
||||
table.setTableKind(DatacenterTableKind.EXTERNAL_TABLE.name());
|
||||
table.setCapabilitiesJson(Map.of("capabilities", capabilities.stream().map(Enum::name).toList()));
|
||||
detail.setTable(table);
|
||||
|
||||
try (ResultSet tableSet = metaData.getTables(
|
||||
resolveCatalogArgument(source, catalogName),
|
||||
resolveSchemaArgument(source, catalogName),
|
||||
tableName,
|
||||
new String[]{"TABLE", "VIEW"})) {
|
||||
while (tableSet.next()) {
|
||||
String currentTableName = tableSet.getString("TABLE_NAME");
|
||||
if (!matchesTableName(currentTableName, tableName)) {
|
||||
continue;
|
||||
}
|
||||
table.setTableDesc(tableSet.getString("REMARKS"));
|
||||
table.setTableKind(resolveTableKind(tableSet.getString("TABLE_TYPE")).name());
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
Set<String> primaryKeys = new HashSet<>();
|
||||
try (ResultSet pkSet = metaData.getPrimaryKeys(resolveCatalogArgument(source, catalogName), resolveSchemaArgument(source, catalogName), tableName)) {
|
||||
while (pkSet.next()) {
|
||||
primaryKeys.add(pkSet.getString("COLUMN_NAME"));
|
||||
}
|
||||
}
|
||||
|
||||
List<DatacenterTableField> fields = new ArrayList<>();
|
||||
try (ResultSet columns = metaData.getColumns(resolveCatalogArgument(source, catalogName), resolveSchemaArgument(source, catalogName), tableName, "%")) {
|
||||
while (columns.next()) {
|
||||
DatacenterTableField field = new DatacenterTableField();
|
||||
field.setFieldName(columns.getString("COLUMN_NAME"));
|
||||
field.setSourceColumnName(columns.getString("COLUMN_NAME"));
|
||||
field.setFieldDesc(columns.getString("REMARKS"));
|
||||
field.setJdbcType(columns.getString("TYPE_NAME"));
|
||||
field.setPrecision(columns.getInt("COLUMN_SIZE"));
|
||||
field.setScale(columns.getInt("DECIMAL_DIGITS"));
|
||||
field.setRequired(columns.getInt("NULLABLE") == DatabaseMetaData.columnNoNulls ? 1 : 0);
|
||||
field.setQueryable(1);
|
||||
field.setSortable(1);
|
||||
field.setWritable(capabilities.contains(DatacenterCapability.WRITE_MUTATION) ? 1 : 0);
|
||||
field.setIndexed(primaryKeys.contains(field.getFieldName()) ? 1 : 0);
|
||||
field.setFieldType(mapFieldType(columns.getInt("DATA_TYPE")));
|
||||
fields.add(field);
|
||||
}
|
||||
}
|
||||
detail.setFields(fields);
|
||||
return detail;
|
||||
});
|
||||
} catch (Exception ex) {
|
||||
throw DatacenterConnectorExceptionSupport.wrapAccessException("读取表详情失败", ex);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public Page<Row> queryPage(DatacenterSource source, DatacenterTable table, DatacenterQueryRequest request) {
|
||||
if (!capabilities.contains(DatacenterCapability.READ_QUERY)) {
|
||||
throw new BusinessException("当前数据源暂不支持查询");
|
||||
}
|
||||
try {
|
||||
return withConnection(source, true, connection -> doQueryPage(connection, source, table, request));
|
||||
} catch (Exception ex) {
|
||||
throw DatacenterConnectorExceptionSupport.wrapAccessException("查询失败", ex);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<Row> queryBySql(DatacenterSource source, String sql) {
|
||||
if (!capabilities.contains(DatacenterCapability.READ_QUERY)) {
|
||||
throw new BusinessException("当前数据源暂不支持查询");
|
||||
}
|
||||
try {
|
||||
return withConnection(source, true, connection -> doQueryBySql(connection, sql));
|
||||
} catch (Exception ex) {
|
||||
throw DatacenterConnectorExceptionSupport.wrapAccessException("SQL 查询失败", ex);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void saveRow(DatacenterSource source, DatacenterTable table, JSONObject data, LoginAccount account) {
|
||||
throw new BusinessException("当前数据源不支持写入");
|
||||
}
|
||||
|
||||
@Override
|
||||
public void deleteRow(DatacenterSource source, DatacenterTable table, BigInteger id, LoginAccount account) {
|
||||
throw new BusinessException("当前数据源不支持删除");
|
||||
}
|
||||
|
||||
protected boolean requiresJdbcUrl() {
|
||||
return true;
|
||||
}
|
||||
|
||||
protected Page<Row> doQueryPage(Connection connection, DatacenterSource source, DatacenterTable table, DatacenterQueryRequest request) throws SQLException {
|
||||
List<Object> params = new ArrayList<>();
|
||||
String selectColumns = CollectionUtils.isEmpty(request.getSelectedColumns())
|
||||
? "*"
|
||||
: request.getSelectedColumns().stream().map(sqlDialect::quoteIdentifier).collect(Collectors.joining(", "));
|
||||
String qualifiedTable = sqlDialect.qualifyTable(resolveCatalogName(source, request.getDatasetRef() == null ? null : request.getDatasetRef().getCatalogName()), resolvePhysicalTableName(table));
|
||||
StringBuilder whereClause = new StringBuilder();
|
||||
if (StrUtil.isNotBlank(request.getWhere())) {
|
||||
whereClause.append(" WHERE ").append(request.getWhere());
|
||||
} else if (!CollectionUtils.isEmpty(request.getFilters())) {
|
||||
whereClause.append(" WHERE 1=1 ");
|
||||
for (DatacenterQueryFilter filter : request.getFilters()) {
|
||||
appendFilter(whereClause, params, filter);
|
||||
}
|
||||
}
|
||||
String orderClause = buildOrderClause(request.getSorts());
|
||||
String baseSql = "SELECT " + selectColumns + " FROM " + qualifiedTable + whereClause + orderClause;
|
||||
String countSql = "SELECT COUNT(1) FROM " + qualifiedTable + whereClause;
|
||||
long total = queryCount(connection, countSql, params);
|
||||
if (total == 0L) {
|
||||
return new Page<>(new ArrayList<>(), request.getPageNumber(), request.getPageSize(), total);
|
||||
}
|
||||
String pageSql = sqlDialect.buildPageSql(baseSql, request.getPageNumber(), request.getPageSize());
|
||||
List<Object> pageParams = new ArrayList<>(params);
|
||||
if (pageSql.contains("FETCH NEXT") || pageSql.toLowerCase(Locale.ROOT).contains("limit")) {
|
||||
pageParams.add((request.getPageNumber() - 1) * request.getPageSize());
|
||||
pageParams.add(request.getPageSize());
|
||||
}
|
||||
List<Row> records = new ArrayList<>();
|
||||
try (PreparedStatement statement = connection.prepareStatement(pageSql)) {
|
||||
bindParameters(statement, pageParams);
|
||||
try (ResultSet resultSet = statement.executeQuery()) {
|
||||
ResultSetMetaData metaData = resultSet.getMetaData();
|
||||
while (resultSet.next()) {
|
||||
Row row = new Row();
|
||||
for (int i = 1; i <= metaData.getColumnCount(); i++) {
|
||||
String columnLabel = metaData.getColumnLabel(i);
|
||||
row.put(columnLabel, normalizeValue(resultSet.getObject(i)));
|
||||
}
|
||||
records.add(row);
|
||||
}
|
||||
}
|
||||
}
|
||||
return new Page<>(records, request.getPageNumber(), request.getPageSize(), total);
|
||||
}
|
||||
|
||||
protected String resolvePhysicalTableName(DatacenterTable table) {
|
||||
return StrUtil.blankToDefault(table.getActualTable(), table.getTableName());
|
||||
}
|
||||
|
||||
protected List<Row> doQueryBySql(Connection connection, String sql) throws SQLException {
|
||||
try (PreparedStatement statement = connection.prepareStatement(sql);
|
||||
ResultSet resultSet = statement.executeQuery()) {
|
||||
return readRows(resultSet);
|
||||
}
|
||||
}
|
||||
|
||||
protected String resolveCatalogName(DatacenterSource source, String requestedCatalogName) {
|
||||
if (StrUtil.isNotBlank(requestedCatalogName)) {
|
||||
return requestedCatalogName;
|
||||
}
|
||||
if (usesCatalogNamespace()) {
|
||||
return source.getDatabaseName();
|
||||
}
|
||||
return source.getSchemaName();
|
||||
}
|
||||
|
||||
protected List<Row> readRows(ResultSet resultSet) throws SQLException {
|
||||
List<Row> records = new ArrayList<>();
|
||||
ResultSetMetaData metaData = resultSet.getMetaData();
|
||||
while (resultSet.next()) {
|
||||
Row row = new Row();
|
||||
for (int i = 1; i <= metaData.getColumnCount(); i++) {
|
||||
String columnLabel = metaData.getColumnLabel(i);
|
||||
row.put(columnLabel, normalizeValue(resultSet.getObject(i)));
|
||||
}
|
||||
records.add(row);
|
||||
}
|
||||
return records;
|
||||
}
|
||||
|
||||
protected String resolveCatalogArgument(DatacenterSource source, String catalogName) {
|
||||
return usesCatalogNamespace() ? resolveCatalogName(source, catalogName) : source.getDatabaseName();
|
||||
}
|
||||
|
||||
protected String resolveSchemaArgument(DatacenterSource source, String catalogName) {
|
||||
return usesCatalogNamespace() ? source.getSchemaName() : resolveCatalogName(source, catalogName);
|
||||
}
|
||||
|
||||
protected boolean usesCatalogNamespace() {
|
||||
return sourceType == DatacenterSourceType.MYSQL
|
||||
|| sourceType == DatacenterSourceType.PROJECT_MYSQL
|
||||
|| sourceType == DatacenterSourceType.GBASE_8A
|
||||
|| sourceType == DatacenterSourceType.GBASE_8S;
|
||||
}
|
||||
|
||||
private List<DatacenterCatalogMeta> filterConfiguredCatalogs(DatacenterSource source, List<DatacenterCatalogMeta> items) {
|
||||
if (items == null || items.isEmpty() || source == null) {
|
||||
return items;
|
||||
}
|
||||
String configuredName = usesCatalogNamespace()
|
||||
? StrUtil.trimToNull(source.getDatabaseName())
|
||||
: StrUtil.trimToNull(source.getSchemaName());
|
||||
if (StrUtil.isBlank(configuredName)) {
|
||||
return items;
|
||||
}
|
||||
List<DatacenterCatalogMeta> matched = items.stream()
|
||||
.filter(item -> configuredName.equalsIgnoreCase(item.getCatalogName()))
|
||||
.collect(Collectors.toList());
|
||||
return matched.isEmpty() ? items : matched;
|
||||
}
|
||||
|
||||
private boolean containsCatalog(List<DatacenterCatalogMeta> items, String catalogName) {
|
||||
return items.stream().anyMatch(item -> catalogName.equalsIgnoreCase(item.getCatalogName()));
|
||||
}
|
||||
|
||||
private DatacenterTableKind resolveTableKind(String tableType) {
|
||||
return "VIEW".equalsIgnoreCase(tableType) ? DatacenterTableKind.EXTERNAL_VIEW : DatacenterTableKind.EXTERNAL_TABLE;
|
||||
}
|
||||
|
||||
private boolean matchesTableName(String currentTableName, String targetTableName) {
|
||||
if (currentTableName == null || targetTableName == null) {
|
||||
return false;
|
||||
}
|
||||
return currentTableName.equals(targetTableName)
|
||||
|| currentTableName.equalsIgnoreCase(targetTableName);
|
||||
}
|
||||
|
||||
private Integer mapFieldType(int jdbcType) {
|
||||
return switch (jdbcType) {
|
||||
case Types.INTEGER, Types.TINYINT, Types.SMALLINT, Types.BIGINT -> EnumFieldType.INTEGER.getCode();
|
||||
case Types.FLOAT, Types.DOUBLE, Types.REAL, Types.NUMERIC, Types.DECIMAL -> EnumFieldType.NUMBER.getCode();
|
||||
case Types.TIMESTAMP, Types.DATE, Types.TIME -> EnumFieldType.TIME.getCode();
|
||||
case Types.BOOLEAN, Types.BIT -> EnumFieldType.BOOLEAN.getCode();
|
||||
default -> EnumFieldType.STRING.getCode();
|
||||
};
|
||||
}
|
||||
|
||||
private void appendFilter(StringBuilder sql, List<Object> params, DatacenterQueryFilter filter) {
|
||||
String operator = StrUtil.blankToDefault(filter.getOperator(), "EQ").toUpperCase(Locale.ROOT);
|
||||
String column = sqlDialect.quoteIdentifier(filter.getColumn());
|
||||
switch (operator) {
|
||||
case "EQ" -> {
|
||||
sql.append(" AND ").append(column).append(" = ?");
|
||||
params.add(filter.getValue());
|
||||
}
|
||||
case "LIKE" -> {
|
||||
sql.append(" AND ").append(column).append(" LIKE ?");
|
||||
params.add("%" + filter.getValue() + "%");
|
||||
}
|
||||
case "GT" -> {
|
||||
sql.append(" AND ").append(column).append(" > ?");
|
||||
params.add(filter.getValue());
|
||||
}
|
||||
case "GTE" -> {
|
||||
sql.append(" AND ").append(column).append(" >= ?");
|
||||
params.add(filter.getValue());
|
||||
}
|
||||
case "LT" -> {
|
||||
sql.append(" AND ").append(column).append(" < ?");
|
||||
params.add(filter.getValue());
|
||||
}
|
||||
case "LTE" -> {
|
||||
sql.append(" AND ").append(column).append(" <= ?");
|
||||
params.add(filter.getValue());
|
||||
}
|
||||
case "IN" -> {
|
||||
List<Object> values = filter.getValues() == null ? List.of() : filter.getValues();
|
||||
if (values.isEmpty()) {
|
||||
sql.append(" AND 1 = 0");
|
||||
} else {
|
||||
sql.append(" AND ").append(column).append(" IN (");
|
||||
sql.append(values.stream().map(item -> "?").collect(Collectors.joining(",")));
|
||||
sql.append(")");
|
||||
params.addAll(values);
|
||||
}
|
||||
}
|
||||
case "IS_NULL" -> sql.append(" AND ").append(column).append(" IS NULL");
|
||||
default -> throw new BusinessException("不支持的过滤操作: " + operator);
|
||||
}
|
||||
}
|
||||
|
||||
private String buildOrderClause(List<DatacenterQuerySort> sorts) {
|
||||
if (CollectionUtils.isEmpty(sorts)) {
|
||||
return "";
|
||||
}
|
||||
return " ORDER BY " + sorts.stream()
|
||||
.map(sort -> sqlDialect.quoteIdentifier(sort.getColumn()) + " " + ("DESC".equalsIgnoreCase(sort.getDirection()) ? "DESC" : "ASC"))
|
||||
.collect(Collectors.joining(", "));
|
||||
}
|
||||
|
||||
private long queryCount(Connection connection, String sql, List<Object> params) throws SQLException {
|
||||
try (PreparedStatement statement = connection.prepareStatement(sql)) {
|
||||
bindParameters(statement, params);
|
||||
try (ResultSet resultSet = statement.executeQuery()) {
|
||||
if (resultSet.next()) {
|
||||
return resultSet.getLong(1);
|
||||
}
|
||||
return 0L;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void bindParameters(PreparedStatement statement, List<Object> params) throws SQLException {
|
||||
for (int i = 0; i < params.size(); i++) {
|
||||
statement.setObject(i + 1, params.get(i));
|
||||
}
|
||||
}
|
||||
|
||||
private Object normalizeValue(Object value) {
|
||||
if (value instanceof BigDecimal || value instanceof BigInteger || value instanceof Long) {
|
||||
return value.toString();
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
private String mapSqlError(SQLException ex) {
|
||||
String state = ex.getSQLState();
|
||||
String message = ex.getMessage() == null ? "" : ex.getMessage().toLowerCase(Locale.ROOT);
|
||||
if (message.contains("access denied") || message.contains("password") || message.contains("authentication")) {
|
||||
return DatacenterConnectionErrorCode.AUTH_FAILED.name();
|
||||
}
|
||||
if (message.contains("unknown database") || message.contains("database does not exist")) {
|
||||
return DatacenterConnectionErrorCode.DATABASE_NOT_FOUND.name();
|
||||
}
|
||||
if (message.contains("schema") && message.contains("does not exist")) {
|
||||
return DatacenterConnectionErrorCode.SCHEMA_NOT_FOUND.name();
|
||||
}
|
||||
if (message.contains("permission denied") || message.contains("insufficient privilege")) {
|
||||
return DatacenterConnectionErrorCode.PERMISSION_DENIED.name();
|
||||
}
|
||||
if (state != null && state.startsWith("08")) {
|
||||
return DatacenterConnectionErrorCode.NETWORK_UNREACHABLE.name();
|
||||
}
|
||||
return DatacenterConnectionErrorCode.UNKNOWN_ERROR.name();
|
||||
}
|
||||
|
||||
@FunctionalInterface
|
||||
protected interface JdbcCallback<T> {
|
||||
T apply(Connection connection) throws Exception;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,80 @@
|
||||
package tech.easyflow.datacenter.connector.support;
|
||||
|
||||
import tech.easyflow.common.web.exceptions.BusinessException;
|
||||
|
||||
import java.net.ConnectException;
|
||||
import java.net.NoRouteToHostException;
|
||||
import java.net.SocketException;
|
||||
import java.net.SocketTimeoutException;
|
||||
import java.net.UnknownHostException;
|
||||
import java.sql.SQLException;
|
||||
import java.util.Locale;
|
||||
|
||||
public final class DatacenterConnectorExceptionSupport {
|
||||
|
||||
public static final String SOURCE_UNAVAILABLE_MESSAGE = "当前连接不可用,请检查连接配置后重试";
|
||||
|
||||
private DatacenterConnectorExceptionSupport() {
|
||||
}
|
||||
|
||||
public static BusinessException wrapAccessException(String fallbackMessage, Exception ex) {
|
||||
if (ex instanceof BusinessException businessException && !isConnectionUnavailable(ex)) {
|
||||
return businessException;
|
||||
}
|
||||
if (isConnectionUnavailable(ex)) {
|
||||
return new BusinessException(SOURCE_UNAVAILABLE_MESSAGE);
|
||||
}
|
||||
return new BusinessException(fallbackMessage);
|
||||
}
|
||||
|
||||
public static boolean isConnectionUnavailable(Throwable throwable) {
|
||||
Throwable current = throwable;
|
||||
while (current != null) {
|
||||
if (current instanceof ClassNotFoundException
|
||||
|| current instanceof ConnectException
|
||||
|| current instanceof NoRouteToHostException
|
||||
|| current instanceof SocketTimeoutException
|
||||
|| current instanceof UnknownHostException) {
|
||||
return true;
|
||||
}
|
||||
if (current instanceof SocketException socketException) {
|
||||
String socketMessage = lowerCase(socketException.getMessage());
|
||||
if (socketMessage.contains("broken pipe")
|
||||
|| socketMessage.contains("connection reset")
|
||||
|| socketMessage.contains("network is unreachable")) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
if (current instanceof SQLException sqlException) {
|
||||
String sqlState = sqlException.getSQLState();
|
||||
String message = lowerCase(sqlException.getMessage());
|
||||
if ((sqlState != null && sqlState.startsWith("08"))
|
||||
|| message.contains("communications link failure")
|
||||
|| message.contains("connection refused")
|
||||
|| message.contains("connection attempt failed")
|
||||
|| message.contains("connect timed out")
|
||||
|| message.contains("i/o error")
|
||||
|| message.contains("io error")
|
||||
|| message.contains("the network adapter could not establish the connection")
|
||||
|| message.contains("unknown host")
|
||||
|| message.contains("access denied")
|
||||
|| message.contains("authentication failed")
|
||||
|| message.contains("login failed")
|
||||
|| message.contains("password authentication failed")
|
||||
|| message.contains("unknown database")
|
||||
|| message.contains("database does not exist")
|
||||
|| (message.contains("schema") && message.contains("does not exist"))
|
||||
|| message.contains("insufficient privilege")
|
||||
|| message.contains("permission denied")) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
current = current.getCause();
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
private static String lowerCase(String message) {
|
||||
return message == null ? "" : message.toLowerCase(Locale.ROOT);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,44 @@
|
||||
package tech.easyflow.datacenter.connector.support;
|
||||
|
||||
import com.zaxxer.hikari.HikariConfig;
|
||||
import com.zaxxer.hikari.HikariDataSource;
|
||||
import org.springframework.stereotype.Component;
|
||||
import tech.easyflow.datacenter.meta.entity.DatacenterSource;
|
||||
import tech.easyflow.datacenter.security.DatacenterCredentialCipher;
|
||||
|
||||
import javax.annotation.Resource;
|
||||
import java.math.BigInteger;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
|
||||
@Component
|
||||
public class DatacenterDatasourceManager {
|
||||
|
||||
private final Map<BigInteger, HikariDataSource> externalDatasourceCache = new ConcurrentHashMap<>();
|
||||
|
||||
@Resource
|
||||
private DatacenterCredentialCipher credentialCipher;
|
||||
|
||||
public HikariDataSource createExternalDatasource(DatacenterSource source) {
|
||||
HikariConfig config = new HikariConfig();
|
||||
config.setPoolName("dc-ext-" + (source.getId() == null ? "temp" : source.getId()));
|
||||
config.setJdbcUrl(source.getJdbcUrl());
|
||||
config.setUsername(source.getUsername());
|
||||
config.setPassword(credentialCipher.decrypt(source.getCredentialCipher()));
|
||||
config.setMaximumPoolSize(3);
|
||||
config.setMinimumIdle(0);
|
||||
config.setConnectionTimeout(5000);
|
||||
config.setValidationTimeout(3000);
|
||||
if (source.getDriverClassName() != null && !source.getDriverClassName().isBlank()) {
|
||||
config.setDriverClassName(source.getDriverClassName());
|
||||
}
|
||||
return new HikariDataSource(config);
|
||||
}
|
||||
|
||||
public HikariDataSource getOrCreateExternalDatasource(DatacenterSource source) {
|
||||
if (source.getId() == null) {
|
||||
return createExternalDatasource(source);
|
||||
}
|
||||
return externalDatasourceCache.computeIfAbsent(source.getId(), key -> createExternalDatasource(source));
|
||||
}
|
||||
}
|
||||
@@ -33,6 +33,18 @@ public class DatacenterTableBase extends DateEntity implements Serializable {
|
||||
@Column(tenantId = true, comment = "租户ID")
|
||||
private BigInteger tenantId;
|
||||
|
||||
/**
|
||||
* 数据源ID
|
||||
*/
|
||||
@Column(comment = "数据源ID")
|
||||
private BigInteger sourceId;
|
||||
|
||||
/**
|
||||
* 目录ID
|
||||
*/
|
||||
@Column(comment = "目录ID")
|
||||
private BigInteger catalogId;
|
||||
|
||||
/**
|
||||
* 数据表名
|
||||
*/
|
||||
@@ -51,6 +63,30 @@ public class DatacenterTableBase extends DateEntity implements Serializable {
|
||||
@Column(comment = "物理表名")
|
||||
private String actualTable;
|
||||
|
||||
/**
|
||||
* 表类型
|
||||
*/
|
||||
@Column(comment = "表类型")
|
||||
private String tableKind;
|
||||
|
||||
/**
|
||||
* 访问模式
|
||||
*/
|
||||
@Column(comment = "访问模式")
|
||||
private String accessMode;
|
||||
|
||||
/**
|
||||
* 物化表名
|
||||
*/
|
||||
@Column(comment = "物化表名")
|
||||
private String materializedTable;
|
||||
|
||||
/**
|
||||
* 是否开启版本
|
||||
*/
|
||||
@Column(comment = "是否开启版本")
|
||||
private Integer versioningEnabled;
|
||||
|
||||
/**
|
||||
* 数据状态
|
||||
*/
|
||||
@@ -87,6 +123,12 @@ public class DatacenterTableBase extends DateEntity implements Serializable {
|
||||
@Column(typeHandler = FastjsonTypeHandler.class, comment = "扩展项")
|
||||
private Map<String, Object> options;
|
||||
|
||||
/**
|
||||
* 能力声明
|
||||
*/
|
||||
@Column(typeHandler = FastjsonTypeHandler.class, comment = "能力声明")
|
||||
private Map<String, Object> capabilitiesJson;
|
||||
|
||||
public BigInteger getId() {
|
||||
return id;
|
||||
}
|
||||
@@ -111,6 +153,22 @@ public class DatacenterTableBase extends DateEntity implements Serializable {
|
||||
this.tenantId = tenantId;
|
||||
}
|
||||
|
||||
public BigInteger getSourceId() {
|
||||
return sourceId;
|
||||
}
|
||||
|
||||
public void setSourceId(BigInteger sourceId) {
|
||||
this.sourceId = sourceId;
|
||||
}
|
||||
|
||||
public BigInteger getCatalogId() {
|
||||
return catalogId;
|
||||
}
|
||||
|
||||
public void setCatalogId(BigInteger catalogId) {
|
||||
this.catalogId = catalogId;
|
||||
}
|
||||
|
||||
public String getTableName() {
|
||||
return tableName;
|
||||
}
|
||||
@@ -135,6 +193,38 @@ public class DatacenterTableBase extends DateEntity implements Serializable {
|
||||
this.actualTable = actualTable;
|
||||
}
|
||||
|
||||
public String getTableKind() {
|
||||
return tableKind;
|
||||
}
|
||||
|
||||
public void setTableKind(String tableKind) {
|
||||
this.tableKind = tableKind;
|
||||
}
|
||||
|
||||
public String getAccessMode() {
|
||||
return accessMode;
|
||||
}
|
||||
|
||||
public void setAccessMode(String accessMode) {
|
||||
this.accessMode = accessMode;
|
||||
}
|
||||
|
||||
public String getMaterializedTable() {
|
||||
return materializedTable;
|
||||
}
|
||||
|
||||
public void setMaterializedTable(String materializedTable) {
|
||||
this.materializedTable = materializedTable;
|
||||
}
|
||||
|
||||
public Integer getVersioningEnabled() {
|
||||
return versioningEnabled;
|
||||
}
|
||||
|
||||
public void setVersioningEnabled(Integer versioningEnabled) {
|
||||
this.versioningEnabled = versioningEnabled;
|
||||
}
|
||||
|
||||
public Integer getStatus() {
|
||||
return status;
|
||||
}
|
||||
@@ -183,4 +273,12 @@ public class DatacenterTableBase extends DateEntity implements Serializable {
|
||||
this.options = options;
|
||||
}
|
||||
|
||||
public Map<String, Object> getCapabilitiesJson() {
|
||||
return capabilitiesJson;
|
||||
}
|
||||
|
||||
public void setCapabilitiesJson(Map<String, Object> capabilitiesJson) {
|
||||
this.capabilitiesJson = capabilitiesJson;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -33,6 +33,12 @@ public class DatacenterTableFieldBase extends DateEntity implements Serializable
|
||||
@Column(comment = "字段名称")
|
||||
private String fieldName;
|
||||
|
||||
/**
|
||||
* 源字段名
|
||||
*/
|
||||
@Column(comment = "源字段名")
|
||||
private String sourceColumnName;
|
||||
|
||||
/**
|
||||
* 字段描述
|
||||
*/
|
||||
@@ -45,12 +51,54 @@ public class DatacenterTableFieldBase extends DateEntity implements Serializable
|
||||
@Column(comment = "字段类型")
|
||||
private Integer fieldType;
|
||||
|
||||
/**
|
||||
* JDBC 类型
|
||||
*/
|
||||
@Column(comment = "JDBC类型")
|
||||
private String jdbcType;
|
||||
|
||||
/**
|
||||
* 精度
|
||||
*/
|
||||
@Column(comment = "精度")
|
||||
private Integer precision;
|
||||
|
||||
/**
|
||||
* 小数位
|
||||
*/
|
||||
@Column(comment = "小数位")
|
||||
private Integer scale;
|
||||
|
||||
/**
|
||||
* 是否必填
|
||||
*/
|
||||
@Column(comment = "是否必填")
|
||||
private Integer required;
|
||||
|
||||
/**
|
||||
* 可查询
|
||||
*/
|
||||
@Column(comment = "可查询")
|
||||
private Integer queryable;
|
||||
|
||||
/**
|
||||
* 可排序
|
||||
*/
|
||||
@Column(comment = "可排序")
|
||||
private Integer sortable;
|
||||
|
||||
/**
|
||||
* 可写入
|
||||
*/
|
||||
@Column(comment = "可写入")
|
||||
private Integer writable;
|
||||
|
||||
/**
|
||||
* 是否索引
|
||||
*/
|
||||
@Column(comment = "是否索引")
|
||||
private Integer indexed;
|
||||
|
||||
/**
|
||||
* 扩展项
|
||||
*/
|
||||
@@ -105,6 +153,14 @@ public class DatacenterTableFieldBase extends DateEntity implements Serializable
|
||||
this.fieldName = fieldName;
|
||||
}
|
||||
|
||||
public String getSourceColumnName() {
|
||||
return sourceColumnName;
|
||||
}
|
||||
|
||||
public void setSourceColumnName(String sourceColumnName) {
|
||||
this.sourceColumnName = sourceColumnName;
|
||||
}
|
||||
|
||||
public String getFieldDesc() {
|
||||
return fieldDesc;
|
||||
}
|
||||
@@ -121,6 +177,30 @@ public class DatacenterTableFieldBase extends DateEntity implements Serializable
|
||||
this.fieldType = fieldType;
|
||||
}
|
||||
|
||||
public String getJdbcType() {
|
||||
return jdbcType;
|
||||
}
|
||||
|
||||
public void setJdbcType(String jdbcType) {
|
||||
this.jdbcType = jdbcType;
|
||||
}
|
||||
|
||||
public Integer getPrecision() {
|
||||
return precision;
|
||||
}
|
||||
|
||||
public void setPrecision(Integer precision) {
|
||||
this.precision = precision;
|
||||
}
|
||||
|
||||
public Integer getScale() {
|
||||
return scale;
|
||||
}
|
||||
|
||||
public void setScale(Integer scale) {
|
||||
this.scale = scale;
|
||||
}
|
||||
|
||||
public Integer getRequired() {
|
||||
return required;
|
||||
}
|
||||
@@ -129,6 +209,38 @@ public class DatacenterTableFieldBase extends DateEntity implements Serializable
|
||||
this.required = required;
|
||||
}
|
||||
|
||||
public Integer getQueryable() {
|
||||
return queryable;
|
||||
}
|
||||
|
||||
public void setQueryable(Integer queryable) {
|
||||
this.queryable = queryable;
|
||||
}
|
||||
|
||||
public Integer getSortable() {
|
||||
return sortable;
|
||||
}
|
||||
|
||||
public void setSortable(Integer sortable) {
|
||||
this.sortable = sortable;
|
||||
}
|
||||
|
||||
public Integer getWritable() {
|
||||
return writable;
|
||||
}
|
||||
|
||||
public void setWritable(Integer writable) {
|
||||
this.writable = writable;
|
||||
}
|
||||
|
||||
public Integer getIndexed() {
|
||||
return indexed;
|
||||
}
|
||||
|
||||
public void setIndexed(Integer indexed) {
|
||||
this.indexed = indexed;
|
||||
}
|
||||
|
||||
public Map<String, Object> getOptions() {
|
||||
return options;
|
||||
}
|
||||
|
||||
@@ -1,107 +0,0 @@
|
||||
package tech.easyflow.datacenter.excel;
|
||||
|
||||
import cn.idev.excel.context.AnalysisContext;
|
||||
import cn.idev.excel.metadata.data.ReadCellData;
|
||||
import cn.idev.excel.read.listener.ReadListener;
|
||||
import com.alibaba.fastjson2.JSONObject;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import tech.easyflow.common.entity.LoginAccount;
|
||||
import tech.easyflow.common.util.SpringContextUtil;
|
||||
import tech.easyflow.datacenter.entity.DatacenterTableField;
|
||||
import tech.easyflow.datacenter.service.DatacenterTableService;
|
||||
|
||||
import java.math.BigInteger;
|
||||
import java.util.*;
|
||||
|
||||
public class ReadDataListener implements ReadListener<LinkedHashMap<Integer, Object>> {
|
||||
|
||||
private static final Logger log = LoggerFactory.getLogger(ReadDataListener.class);
|
||||
|
||||
private BigInteger tableId;
|
||||
|
||||
private List<DatacenterTableField> fields;
|
||||
|
||||
private LoginAccount loginAccount;
|
||||
|
||||
private final Map<String, Integer> headFieldIndex = new HashMap<>();
|
||||
|
||||
private int successCount = 0;
|
||||
private int errorCount = 0;
|
||||
private int totalCount = 0;
|
||||
|
||||
private final List<JSONObject> errorRows = new ArrayList<>();
|
||||
|
||||
public ReadDataListener() {
|
||||
}
|
||||
|
||||
public ReadDataListener(BigInteger tableId, List<DatacenterTableField> fields, LoginAccount loginAccount) {
|
||||
this.tableId = tableId;
|
||||
this.fields = fields;
|
||||
this.loginAccount = loginAccount;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void invoke(LinkedHashMap<Integer, Object> o, AnalysisContext analysisContext) {
|
||||
DatacenterTableService service = SpringContextUtil.getBean(DatacenterTableService.class);
|
||||
JSONObject obj = new JSONObject();
|
||||
for (DatacenterTableField field : fields) {
|
||||
String fieldName = field.getFieldName();
|
||||
Integer i = headFieldIndex.get(fieldName);
|
||||
if (i != null) {
|
||||
obj.put(fieldName, o.get(i));
|
||||
}
|
||||
}
|
||||
try {
|
||||
service.saveValue(tableId, obj, loginAccount);
|
||||
successCount++;
|
||||
} catch (Exception e) {
|
||||
errorCount++;
|
||||
log.error("导入数据到数据中枢失败,具体值:{}", obj, e);
|
||||
errorRows.add(obj);
|
||||
}
|
||||
totalCount++;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void invokeHead(Map<Integer, ReadCellData<?>> headMap, AnalysisContext context) {
|
||||
Set<Map.Entry<Integer, ReadCellData<?>>> entries = headMap.entrySet();
|
||||
for (Map.Entry<Integer, ReadCellData<?>> entry : entries) {
|
||||
Integer key = entry.getKey();
|
||||
String field = entry.getValue().getStringValue();
|
||||
headFieldIndex.put(field, key);
|
||||
}
|
||||
if (headFieldIndex.size() != fields.size()) {
|
||||
throw new RuntimeException("表头字段数量与表结构对应不上!");
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void doAfterAllAnalysed(AnalysisContext analysisContext) {
|
||||
|
||||
}
|
||||
|
||||
public List<DatacenterTableField> getFields() {
|
||||
return fields;
|
||||
}
|
||||
|
||||
public void setFields(List<DatacenterTableField> fields) {
|
||||
this.fields = fields;
|
||||
}
|
||||
|
||||
public int getSuccessCount() {
|
||||
return successCount;
|
||||
}
|
||||
|
||||
public int getErrorCount() {
|
||||
return errorCount;
|
||||
}
|
||||
|
||||
public int getTotalCount() {
|
||||
return totalCount;
|
||||
}
|
||||
|
||||
public List<JSONObject> getErrorRows() {
|
||||
return errorRows;
|
||||
}
|
||||
}
|
||||
@@ -1,68 +0,0 @@
|
||||
package tech.easyflow.datacenter.excel;
|
||||
|
||||
import com.alibaba.fastjson2.JSONObject;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
public class ReadResVo {
|
||||
|
||||
/**
|
||||
* 成功数
|
||||
*/
|
||||
private int successCount = 0;
|
||||
/**
|
||||
* 失败数
|
||||
*/
|
||||
private int errorCount = 0;
|
||||
/**
|
||||
* 总数
|
||||
*/
|
||||
private int totalCount = 0;
|
||||
|
||||
/**
|
||||
* 错误行
|
||||
*/
|
||||
private List<JSONObject> errorRows;
|
||||
|
||||
public ReadResVo() {
|
||||
}
|
||||
|
||||
public ReadResVo(int successCount, int errorCount, int totalCount, List<JSONObject> errorRows) {
|
||||
this.successCount = successCount;
|
||||
this.errorCount = errorCount;
|
||||
this.totalCount = totalCount;
|
||||
this.errorRows = errorRows;
|
||||
}
|
||||
|
||||
public int getSuccessCount() {
|
||||
return successCount;
|
||||
}
|
||||
|
||||
public void setSuccessCount(int successCount) {
|
||||
this.successCount = successCount;
|
||||
}
|
||||
|
||||
public int getErrorCount() {
|
||||
return errorCount;
|
||||
}
|
||||
|
||||
public void setErrorCount(int errorCount) {
|
||||
this.errorCount = errorCount;
|
||||
}
|
||||
|
||||
public int getTotalCount() {
|
||||
return totalCount;
|
||||
}
|
||||
|
||||
public void setTotalCount(int totalCount) {
|
||||
this.totalCount = totalCount;
|
||||
}
|
||||
|
||||
public List<JSONObject> getErrorRows() {
|
||||
return errorRows;
|
||||
}
|
||||
|
||||
public void setErrorRows(List<JSONObject> errorRows) {
|
||||
this.errorRows = errorRows;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,28 @@
|
||||
package tech.easyflow.datacenter.excel.model;
|
||||
|
||||
import tech.easyflow.datacenter.execution.model.DatacenterQueryFilter;
|
||||
import tech.easyflow.datacenter.execution.model.DatasetRef;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
public class DatacenterExcelDeriveRequest {
|
||||
private DatasetRef datasetRef;
|
||||
private String targetTableName;
|
||||
private List<String> selectedColumns = new ArrayList<>();
|
||||
private Map<String, String> renameMappings = new LinkedHashMap<>();
|
||||
private List<DatacenterQueryFilter> filters = new ArrayList<>();
|
||||
|
||||
public DatasetRef getDatasetRef() { return datasetRef; }
|
||||
public void setDatasetRef(DatasetRef datasetRef) { this.datasetRef = datasetRef; }
|
||||
public String getTargetTableName() { return targetTableName; }
|
||||
public void setTargetTableName(String targetTableName) { this.targetTableName = targetTableName; }
|
||||
public List<String> getSelectedColumns() { return selectedColumns; }
|
||||
public void setSelectedColumns(List<String> selectedColumns) { this.selectedColumns = selectedColumns; }
|
||||
public Map<String, String> getRenameMappings() { return renameMappings; }
|
||||
public void setRenameMappings(Map<String, String> renameMappings) { this.renameMappings = renameMappings; }
|
||||
public List<DatacenterQueryFilter> getFilters() { return filters; }
|
||||
public void setFilters(List<DatacenterQueryFilter> filters) { this.filters = filters; }
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
package tech.easyflow.datacenter.excel.model;
|
||||
|
||||
import tech.easyflow.datacenter.execution.model.DatasetRef;
|
||||
|
||||
import java.math.BigInteger;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
public class DatacenterExcelExportRequest {
|
||||
private BigInteger sourceId;
|
||||
private BigInteger catalogId;
|
||||
private List<DatasetRef> datasetRefs = new ArrayList<>();
|
||||
private String fileName;
|
||||
|
||||
public BigInteger getSourceId() { return sourceId; }
|
||||
public void setSourceId(BigInteger sourceId) { this.sourceId = sourceId; }
|
||||
public BigInteger getCatalogId() { return catalogId; }
|
||||
public void setCatalogId(BigInteger catalogId) { this.catalogId = catalogId; }
|
||||
public List<DatasetRef> getDatasetRefs() { return datasetRefs; }
|
||||
public void setDatasetRefs(List<DatasetRef> datasetRefs) { this.datasetRefs = datasetRefs; }
|
||||
public String getFileName() { return fileName; }
|
||||
public void setFileName(String fileName) { this.fileName = fileName; }
|
||||
}
|
||||
@@ -0,0 +1,22 @@
|
||||
package tech.easyflow.datacenter.excel.model;
|
||||
|
||||
import tech.easyflow.datacenter.execution.model.DatasetRef;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
public class DatacenterExcelMergeRequest {
|
||||
private List<DatasetRef> datasetRefs = new ArrayList<>();
|
||||
private String mergeMode;
|
||||
private String targetTableName;
|
||||
private String joinKey;
|
||||
|
||||
public List<DatasetRef> getDatasetRefs() { return datasetRefs; }
|
||||
public void setDatasetRefs(List<DatasetRef> datasetRefs) { this.datasetRefs = datasetRefs; }
|
||||
public String getMergeMode() { return mergeMode; }
|
||||
public void setMergeMode(String mergeMode) { this.mergeMode = mergeMode; }
|
||||
public String getTargetTableName() { return targetTableName; }
|
||||
public void setTargetTableName(String targetTableName) { this.targetTableName = targetTableName; }
|
||||
public String getJoinKey() { return joinKey; }
|
||||
public void setJoinKey(String joinKey) { this.joinKey = joinKey; }
|
||||
}
|
||||
@@ -0,0 +1,30 @@
|
||||
package tech.easyflow.datacenter.excel.model;
|
||||
|
||||
import tech.easyflow.datacenter.execution.model.DatasetRef;
|
||||
|
||||
import java.math.BigInteger;
|
||||
|
||||
public class DatacenterExcelSplitRequest {
|
||||
private BigInteger sourceId;
|
||||
private BigInteger catalogId;
|
||||
private DatasetRef datasetRef;
|
||||
private String splitMode;
|
||||
private Integer rowBatchSize;
|
||||
private String fieldName;
|
||||
private String targetNamePrefix;
|
||||
|
||||
public BigInteger getSourceId() { return sourceId; }
|
||||
public void setSourceId(BigInteger sourceId) { this.sourceId = sourceId; }
|
||||
public BigInteger getCatalogId() { return catalogId; }
|
||||
public void setCatalogId(BigInteger catalogId) { this.catalogId = catalogId; }
|
||||
public DatasetRef getDatasetRef() { return datasetRef; }
|
||||
public void setDatasetRef(DatasetRef datasetRef) { this.datasetRef = datasetRef; }
|
||||
public String getSplitMode() { return splitMode; }
|
||||
public void setSplitMode(String splitMode) { this.splitMode = splitMode; }
|
||||
public Integer getRowBatchSize() { return rowBatchSize; }
|
||||
public void setRowBatchSize(Integer rowBatchSize) { this.rowBatchSize = rowBatchSize; }
|
||||
public String getFieldName() { return fieldName; }
|
||||
public void setFieldName(String fieldName) { this.fieldName = fieldName; }
|
||||
public String getTargetNamePrefix() { return targetNamePrefix; }
|
||||
public void setTargetNamePrefix(String targetNamePrefix) { this.targetNamePrefix = targetNamePrefix; }
|
||||
}
|
||||
@@ -0,0 +1,28 @@
|
||||
package tech.easyflow.datacenter.excel.service;
|
||||
|
||||
import org.springframework.web.multipart.MultipartFile;
|
||||
import tech.easyflow.common.entity.LoginAccount;
|
||||
import tech.easyflow.datacenter.excel.model.DatacenterExcelDeriveRequest;
|
||||
import tech.easyflow.datacenter.excel.model.DatacenterExcelExportRequest;
|
||||
import tech.easyflow.datacenter.excel.model.DatacenterExcelMergeRequest;
|
||||
import tech.easyflow.datacenter.excel.model.DatacenterExcelSplitRequest;
|
||||
import tech.easyflow.datacenter.meta.entity.DatacenterImportJob;
|
||||
|
||||
import java.math.BigInteger;
|
||||
import java.util.List;
|
||||
|
||||
public interface DatacenterExcelImportService {
|
||||
DatacenterImportJob importWorkbook(MultipartFile file, LoginAccount account) throws Exception;
|
||||
|
||||
DatacenterImportJob splitWorkbook(DatacenterExcelSplitRequest request, LoginAccount account);
|
||||
|
||||
DatacenterImportJob mergeWorkbook(DatacenterExcelMergeRequest request, LoginAccount account);
|
||||
|
||||
DatacenterImportJob deriveWorkbook(DatacenterExcelDeriveRequest request, LoginAccount account);
|
||||
|
||||
DatacenterImportJob exportWorkbook(DatacenterExcelExportRequest request, LoginAccount account) throws Exception;
|
||||
|
||||
DatacenterImportJob getImportJobDetail(BigInteger jobId);
|
||||
|
||||
List<DatacenterImportJob> listJobs(BigInteger sourceId, BigInteger tableId);
|
||||
}
|
||||
@@ -0,0 +1,950 @@
|
||||
package tech.easyflow.datacenter.excel.service.impl;
|
||||
|
||||
import com.alibaba.fastjson2.JSONObject;
|
||||
import com.mybatisflex.core.keygen.impl.SnowFlakeIDKeyGenerator;
|
||||
import com.mybatisflex.core.paginate.Page;
|
||||
import com.mybatisflex.core.row.Row;
|
||||
import org.apache.poi.ss.usermodel.Cell;
|
||||
import org.apache.poi.ss.usermodel.DataFormatter;
|
||||
import org.apache.poi.ss.usermodel.Sheet;
|
||||
import org.apache.poi.ss.usermodel.Workbook;
|
||||
import org.apache.poi.ss.usermodel.WorkbookFactory;
|
||||
import org.apache.poi.xssf.streaming.SXSSFWorkbook;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.springframework.transaction.annotation.Transactional;
|
||||
import org.springframework.util.CollectionUtils;
|
||||
import org.springframework.web.multipart.MultipartFile;
|
||||
import tech.easyflow.common.constant.enums.EnumFieldType;
|
||||
import tech.easyflow.common.entity.LoginAccount;
|
||||
import tech.easyflow.common.web.exceptions.BusinessException;
|
||||
import tech.easyflow.datacenter.adapter.DbHandleManager;
|
||||
import tech.easyflow.datacenter.entity.DatacenterTable;
|
||||
import tech.easyflow.datacenter.entity.DatacenterTableField;
|
||||
import tech.easyflow.datacenter.excel.model.DatacenterExcelDeriveRequest;
|
||||
import tech.easyflow.datacenter.excel.model.DatacenterExcelExportRequest;
|
||||
import tech.easyflow.datacenter.excel.model.DatacenterExcelMergeRequest;
|
||||
import tech.easyflow.datacenter.excel.model.DatacenterExcelSplitRequest;
|
||||
import tech.easyflow.datacenter.excel.service.DatacenterExcelImportService;
|
||||
import tech.easyflow.datacenter.execution.model.DatacenterQueryFilter;
|
||||
import tech.easyflow.datacenter.execution.model.DatacenterQueryRequest;
|
||||
import tech.easyflow.datacenter.execution.model.DatasetRef;
|
||||
import tech.easyflow.datacenter.execution.service.DatacenterDatasetQueryService;
|
||||
import tech.easyflow.datacenter.mapper.DatacenterDatasetVersionMapper;
|
||||
import tech.easyflow.datacenter.mapper.DatacenterDerivedTableMapper;
|
||||
import tech.easyflow.datacenter.mapper.DatacenterImportJobMapper;
|
||||
import tech.easyflow.datacenter.meta.entity.DatacenterCatalog;
|
||||
import tech.easyflow.datacenter.meta.entity.DatacenterDatasetVersion;
|
||||
import tech.easyflow.datacenter.meta.entity.DatacenterDerivedTable;
|
||||
import tech.easyflow.datacenter.meta.entity.DatacenterImportJob;
|
||||
import tech.easyflow.datacenter.meta.entity.DatacenterSource;
|
||||
import tech.easyflow.datacenter.meta.enums.DatacenterImportStatus;
|
||||
import tech.easyflow.datacenter.meta.enums.DatacenterSourceType;
|
||||
import tech.easyflow.datacenter.meta.enums.DatacenterTableKind;
|
||||
import tech.easyflow.datacenter.meta.model.DatacenterTableDetailMeta;
|
||||
import tech.easyflow.datacenter.meta.service.DatacenterDatasetRegistryService;
|
||||
import tech.easyflow.datacenter.meta.service.DatacenterSourceService;
|
||||
|
||||
import javax.annotation.Resource;
|
||||
import java.io.File;
|
||||
import java.io.FileOutputStream;
|
||||
import java.io.InputStream;
|
||||
import java.math.BigInteger;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.time.LocalDateTime;
|
||||
import java.time.format.DateTimeFormatter;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Date;
|
||||
import java.util.HashMap;
|
||||
import java.util.HashSet;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.LinkedHashSet;
|
||||
import java.util.List;
|
||||
import java.util.Locale;
|
||||
import java.util.Map;
|
||||
import java.util.Objects;
|
||||
import java.util.Set;
|
||||
import java.util.UUID;
|
||||
|
||||
@Service
|
||||
public class DatacenterExcelImportServiceImpl implements DatacenterExcelImportService {
|
||||
|
||||
private static final long QUERY_BATCH_SIZE = 500L;
|
||||
private static final DateTimeFormatter EXPORT_TIME_FORMAT = DateTimeFormatter.ofPattern("yyyyMMddHHmmss");
|
||||
|
||||
@Resource
|
||||
private DatacenterSourceService sourceService;
|
||||
@Resource
|
||||
private DatacenterDatasetRegistryService registryService;
|
||||
@Resource
|
||||
private DatacenterImportJobMapper importJobMapper;
|
||||
@Resource
|
||||
private DatacenterDatasetVersionMapper datasetVersionMapper;
|
||||
@Resource
|
||||
private DatacenterDerivedTableMapper derivedTableMapper;
|
||||
@Resource
|
||||
private DatacenterDatasetQueryService queryService;
|
||||
@Resource
|
||||
private DbHandleManager dbHandleManager;
|
||||
|
||||
@Override
|
||||
@Transactional(rollbackFor = Exception.class)
|
||||
public DatacenterImportJob importWorkbook(MultipartFile file, LoginAccount account) throws Exception {
|
||||
if (file == null || file.isEmpty()) {
|
||||
throw new BusinessException("Excel 文件不能为空");
|
||||
}
|
||||
String workbookName = extractWorkbookName(file.getOriginalFilename());
|
||||
DatacenterSource source = new DatacenterSource();
|
||||
source.setSourceName(workbookName);
|
||||
source.setSourceCode("EXCEL_" + UUID.randomUUID());
|
||||
source.setSourceType(DatacenterSourceType.EXCEL.name());
|
||||
source.setAccessMode("READ_WRITE");
|
||||
source.setBuiltinFlag(0);
|
||||
source.setConfigJson(Map.of("originFileName", file.getOriginalFilename()));
|
||||
source = sourceService.saveSource(source, account);
|
||||
DatacenterCatalog catalog = registryService.ensureCatalog(source, workbookName, account);
|
||||
|
||||
DatacenterImportJob job = createJob("EXCEL_IMPORT", source.getId(), catalog.getId(), null,
|
||||
file.getOriginalFilename(), Map.of("operation", "import"), account);
|
||||
|
||||
long totalRows = 0L;
|
||||
long successRows = 0L;
|
||||
List<BigInteger> createdTableIds = new ArrayList<>();
|
||||
try (InputStream inputStream = file.getInputStream(); Workbook workbook = WorkbookFactory.create(inputStream)) {
|
||||
DataFormatter formatter = new DataFormatter();
|
||||
for (int sheetIndex = 0; sheetIndex < workbook.getNumberOfSheets(); sheetIndex++) {
|
||||
Sheet sheet = workbook.getSheetAt(sheetIndex);
|
||||
org.apache.poi.ss.usermodel.Row headerRow = sheet.getRow(sheet.getFirstRowNum());
|
||||
if (headerRow == null) {
|
||||
continue;
|
||||
}
|
||||
List<DatacenterTableField> fields = buildFields(headerRow, formatter);
|
||||
if (fields.isEmpty()) {
|
||||
continue;
|
||||
}
|
||||
DatacenterTable table = new DatacenterTable();
|
||||
table.setTableName(uniqueTableName(source.getId(), catalog.getId(), sheet.getSheetName()));
|
||||
table.setTableDesc(sheet.getSheetName());
|
||||
table.setActualTable(buildMaterializedTableName(source.getId(), sheetIndex));
|
||||
table.setMaterializedTable(table.getActualTable());
|
||||
table.setTableKind(DatacenterTableKind.EXCEL_MATERIALIZED.name());
|
||||
table.setAccessMode("READ_WRITE");
|
||||
table.setVersioningEnabled(1);
|
||||
table.setCapabilitiesJson(defaultExcelCapabilities());
|
||||
table.setFields(fields);
|
||||
|
||||
DatacenterTableDetailMeta detail = new DatacenterTableDetailMeta();
|
||||
detail.setTable(table);
|
||||
detail.setFields(fields);
|
||||
|
||||
dbHandleManager.getDbHandler().createTable(table);
|
||||
DatacenterTable savedTable = registryService.registerTable(source, catalog, detail, account);
|
||||
savedTable.setFields(registryService.getFields(savedTable.getId()));
|
||||
createdTableIds.add(savedTable.getId());
|
||||
|
||||
for (int rowIndex = sheet.getFirstRowNum() + 1; rowIndex <= sheet.getLastRowNum(); rowIndex++) {
|
||||
org.apache.poi.ss.usermodel.Row row = sheet.getRow(rowIndex);
|
||||
if (row == null) {
|
||||
continue;
|
||||
}
|
||||
JSONObject payload = new JSONObject();
|
||||
boolean hasValue = false;
|
||||
for (int cellIndex = 0; cellIndex < savedTable.getFields().size(); cellIndex++) {
|
||||
String value = formatter.formatCellValue(row.getCell(cellIndex));
|
||||
if (value != null && !value.isBlank()) {
|
||||
hasValue = true;
|
||||
}
|
||||
payload.put(savedTable.getFields().get(cellIndex).getFieldName(), value);
|
||||
}
|
||||
if (!hasValue) {
|
||||
continue;
|
||||
}
|
||||
dbHandleManager.getDbHandler().saveValue(savedTable, payload, account);
|
||||
totalRows++;
|
||||
successRows++;
|
||||
}
|
||||
|
||||
createVersion(savedTable, "initial-import", Map.of(
|
||||
"sheetName", sheet.getSheetName(),
|
||||
"sourceId", source.getId(),
|
||||
"originFileName", file.getOriginalFilename()
|
||||
), account);
|
||||
}
|
||||
job.setTableId(createdTableIds.isEmpty() ? null : createdTableIds.get(0));
|
||||
finishJobSuccess(job, totalRows, successRows, Map.of("tableIds", createdTableIds));
|
||||
return job;
|
||||
} catch (Exception ex) {
|
||||
finishJobFailure(job, ex);
|
||||
throw ex;
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
@Transactional(rollbackFor = Exception.class)
|
||||
public DatacenterImportJob splitWorkbook(DatacenterExcelSplitRequest request, LoginAccount account) {
|
||||
DatasetRef datasetRef = request == null ? null : request.getDatasetRef();
|
||||
DatacenterImportJob job = createJob("EXCEL_SPLIT",
|
||||
request == null ? null : request.getSourceId(),
|
||||
request == null ? null : request.getCatalogId(),
|
||||
datasetRef == null ? null : datasetRef.getTableId(),
|
||||
null,
|
||||
buildPayload("request", request),
|
||||
account);
|
||||
try {
|
||||
String splitMode = normalizeMode(request == null ? null : request.getSplitMode(), "BY_ROW_COUNT");
|
||||
return switch (splitMode) {
|
||||
case "BY_SHEET" -> doSplitBySheet(request, account, job);
|
||||
case "BY_FIELD_VALUE" -> doSplitByFieldValue(request, account, job);
|
||||
default -> doSplitByRowCount(request, account, job);
|
||||
};
|
||||
} catch (Exception ex) {
|
||||
finishJobFailure(job, ex);
|
||||
throw ex;
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
@Transactional(rollbackFor = Exception.class)
|
||||
public DatacenterImportJob mergeWorkbook(DatacenterExcelMergeRequest request, LoginAccount account) {
|
||||
if (request == null || CollectionUtils.isEmpty(request.getDatasetRefs())) {
|
||||
throw new BusinessException("合并数据集不能为空");
|
||||
}
|
||||
DatacenterImportJob job = createJob("EXCEL_MERGE", null, null, null, null, buildPayload("request", request), account);
|
||||
try {
|
||||
String mergeMode = normalizeMode(request.getMergeMode(), "VERTICAL");
|
||||
return switch (mergeMode) {
|
||||
case "HORIZONTAL" -> doHorizontalMerge(request, account, job);
|
||||
default -> doVerticalMerge(request, account, job);
|
||||
};
|
||||
} catch (Exception ex) {
|
||||
finishJobFailure(job, ex);
|
||||
throw ex;
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
@Transactional(rollbackFor = Exception.class)
|
||||
public DatacenterImportJob deriveWorkbook(DatacenterExcelDeriveRequest request, LoginAccount account) {
|
||||
if (request == null || request.getDatasetRef() == null) {
|
||||
throw new BusinessException("派生数据集不能为空");
|
||||
}
|
||||
DatacenterImportJob job = createJob("EXCEL_DERIVE",
|
||||
request.getDatasetRef().getSourceId(),
|
||||
request.getDatasetRef().getCatalogId(),
|
||||
request.getDatasetRef().getTableId(),
|
||||
null,
|
||||
buildPayload("request", request),
|
||||
account);
|
||||
try {
|
||||
DatacenterTable sourceTable = resolveTable(request.getDatasetRef());
|
||||
DatacenterSource source = registryService.getSourceRequired(sourceTable.getSourceId());
|
||||
DatacenterCatalog catalog = requireCatalog(sourceTable.getCatalogId());
|
||||
List<String> selectedColumns = resolveSelectedColumns(sourceTable, request.getSelectedColumns());
|
||||
List<DatacenterTableField> targetFields = buildDerivedFields(sourceTable, request);
|
||||
DatacenterTable targetTable = createDerivedTable(source, catalog, targetFields,
|
||||
request.getTargetTableName(), "DERIVE", Map.of("sourceTableId", sourceTable.getId()), account);
|
||||
|
||||
DatacenterQueryRequest queryRequest = new DatacenterQueryRequest();
|
||||
queryRequest.setDatasetRef(registryService.resolveDatasetRef(sourceTable.getId()));
|
||||
queryRequest.setSelectedColumns(selectedColumns);
|
||||
queryRequest.setFilters(request.getFilters());
|
||||
|
||||
long successRows = copyRows(queryRequest, rows -> mapDerivedRow(rows, selectedColumns, request), targetTable, account);
|
||||
createLineage(sourceTable.getId(), targetTable.getId(), "DERIVE", Map.of("request", request), account);
|
||||
finishJobSuccess(job, successRows, successRows, Map.of("derivedTableId", targetTable.getId()));
|
||||
job.setTableId(targetTable.getId());
|
||||
return job;
|
||||
} catch (Exception ex) {
|
||||
finishJobFailure(job, ex);
|
||||
throw ex;
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
@Transactional(rollbackFor = Exception.class)
|
||||
public DatacenterImportJob exportWorkbook(DatacenterExcelExportRequest request, LoginAccount account) throws Exception {
|
||||
List<DatacenterTable> tables = resolveExportTables(request);
|
||||
if (tables.isEmpty()) {
|
||||
throw new BusinessException("没有可导出的 Excel 数据集");
|
||||
}
|
||||
String fileName = buildExportFileName(request == null ? null : request.getFileName());
|
||||
DatacenterImportJob job = createJob("EXCEL_EXPORT",
|
||||
request == null ? null : request.getSourceId(),
|
||||
request == null ? null : request.getCatalogId(),
|
||||
null,
|
||||
fileName,
|
||||
buildPayload("request", request),
|
||||
account);
|
||||
Path exportDir = ensureExportDir();
|
||||
Path exportFile = exportDir.resolve(fileName);
|
||||
long totalRows = 0L;
|
||||
try (SXSSFWorkbook workbook = new SXSSFWorkbook(200); FileOutputStream outputStream = new FileOutputStream(exportFile.toFile())) {
|
||||
workbook.setCompressTempFiles(true);
|
||||
Set<String> usedSheetNames = new HashSet<>();
|
||||
for (DatacenterTable table : tables) {
|
||||
String sheetName = uniqueSheetName(table.getTableName(), usedSheetNames);
|
||||
org.apache.poi.ss.usermodel.Sheet sheet = workbook.createSheet(sheetName);
|
||||
writeHeaderRow(sheet, table.getFields());
|
||||
DatacenterQueryRequest queryRequest = new DatacenterQueryRequest();
|
||||
queryRequest.setDatasetRef(registryService.resolveDatasetRef(table.getId()));
|
||||
queryRequest.setSelectedColumns(table.getFields().stream().map(DatacenterTableField::getFieldName).toList());
|
||||
final int[] rowIndex = {1};
|
||||
totalRows += iterateRows(queryRequest, row -> {
|
||||
org.apache.poi.ss.usermodel.Row excelRow = sheet.createRow(rowIndex[0]++);
|
||||
for (int i = 0; i < table.getFields().size(); i++) {
|
||||
Cell cell = excelRow.createCell(i);
|
||||
Object value = row.get(table.getFields().get(i).getFieldName());
|
||||
cell.setCellValue(value == null ? "" : String.valueOf(value));
|
||||
}
|
||||
});
|
||||
}
|
||||
workbook.write(outputStream);
|
||||
workbook.dispose();
|
||||
job.setStoragePath(exportFile.toAbsolutePath().toString());
|
||||
finishJobSuccess(job, totalRows, totalRows, Map.of("storagePath", job.getStoragePath(), "fileName", fileName));
|
||||
return job;
|
||||
} catch (Exception ex) {
|
||||
finishJobFailure(job, ex);
|
||||
throw ex;
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public DatacenterImportJob getImportJobDetail(BigInteger jobId) {
|
||||
DatacenterImportJob job = importJobMapper.selectOneById(jobId);
|
||||
if (job == null) {
|
||||
throw new BusinessException("导入任务不存在: " + jobId);
|
||||
}
|
||||
return job;
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<DatacenterImportJob> listJobs(BigInteger sourceId, BigInteger tableId) {
|
||||
var wrapper = com.mybatisflex.core.query.QueryWrapper.create();
|
||||
if (sourceId != null) {
|
||||
wrapper.eq(DatacenterImportJob::getSourceId, sourceId);
|
||||
}
|
||||
if (tableId != null) {
|
||||
wrapper.eq(DatacenterImportJob::getTableId, tableId);
|
||||
}
|
||||
wrapper.orderBy("created desc");
|
||||
wrapper.limit(20L);
|
||||
return importJobMapper.selectListByQuery(wrapper);
|
||||
}
|
||||
|
||||
private DatacenterImportJob doSplitBySheet(DatacenterExcelSplitRequest request, LoginAccount account, DatacenterImportJob job) {
|
||||
BigInteger sourceId = request.getSourceId();
|
||||
BigInteger catalogId = request.getCatalogId();
|
||||
if (sourceId == null && request.getDatasetRef() != null) {
|
||||
sourceId = request.getDatasetRef().getSourceId();
|
||||
catalogId = request.getDatasetRef().getCatalogId();
|
||||
}
|
||||
if (sourceId == null) {
|
||||
throw new BusinessException("按 sheet 拆分需要 sourceId");
|
||||
}
|
||||
DatacenterSource source = registryService.getSourceRequired(sourceId);
|
||||
DatacenterCatalog catalog = requireCatalog(catalogId);
|
||||
List<DatacenterTable> sourceTables = registryService.listManagedTables(sourceId, catalogId);
|
||||
if (sourceTables.isEmpty()) {
|
||||
throw new BusinessException("当前 workbook 下没有可拆分的 sheet 表");
|
||||
}
|
||||
List<BigInteger> derivedIds = new ArrayList<>();
|
||||
long successRows = 0L;
|
||||
for (DatacenterTable sourceTable : sourceTables) {
|
||||
DatacenterTable fullTable = registryService.getTableWithFields(sourceTable.getId());
|
||||
DatacenterTable targetTable = createDerivedTable(source, catalog, cloneFields(fullTable.getFields()),
|
||||
resolveSplitPrefix(request, fullTable.getTableName()) + "_copy", "SPLIT_BY_SHEET",
|
||||
Map.of("sourceTableId", fullTable.getId()), account);
|
||||
DatacenterQueryRequest queryRequest = new DatacenterQueryRequest();
|
||||
queryRequest.setDatasetRef(registryService.resolveDatasetRef(fullTable.getId()));
|
||||
queryRequest.setSelectedColumns(fullTable.getFields().stream().map(DatacenterTableField::getFieldName).toList());
|
||||
successRows += copyRows(queryRequest, this::mapRow, targetTable, account);
|
||||
createLineage(fullTable.getId(), targetTable.getId(), "SPLIT_BY_SHEET", Map.of("sourceTableId", fullTable.getId()), account);
|
||||
derivedIds.add(targetTable.getId());
|
||||
}
|
||||
finishJobSuccess(job, successRows, successRows, Map.of("derivedTableIds", derivedIds));
|
||||
return job;
|
||||
}
|
||||
|
||||
private DatacenterImportJob doSplitByRowCount(DatacenterExcelSplitRequest request, LoginAccount account, DatacenterImportJob job) {
|
||||
if (request == null || request.getDatasetRef() == null) {
|
||||
throw new BusinessException("按行数拆分需要数据集");
|
||||
}
|
||||
int rowBatchSize = request.getRowBatchSize() == null || request.getRowBatchSize() < 1 ? 1000 : request.getRowBatchSize();
|
||||
DatacenterTable sourceTable = resolveTable(request.getDatasetRef());
|
||||
DatacenterSource source = registryService.getSourceRequired(sourceTable.getSourceId());
|
||||
DatacenterCatalog catalog = requireCatalog(sourceTable.getCatalogId());
|
||||
String baseName = resolveSplitPrefix(request, sourceTable.getTableName());
|
||||
List<BigInteger> derivedIds = new ArrayList<>();
|
||||
final Holder holder = new Holder();
|
||||
long totalRows = iterateRows(buildFullQuery(sourceTable), row -> {
|
||||
if (holder.targetTable == null || holder.currentSize >= rowBatchSize) {
|
||||
holder.batchNo++;
|
||||
holder.targetTable = createDerivedTable(source, catalog, cloneFields(sourceTable.getFields()),
|
||||
baseName + "_part_" + holder.batchNo, "SPLIT_BY_ROW_COUNT",
|
||||
Map.of("sourceTableId", sourceTable.getId(), "batchNo", holder.batchNo, "rowBatchSize", rowBatchSize), account);
|
||||
createLineage(sourceTable.getId(), holder.targetTable.getId(), "SPLIT_BY_ROW_COUNT",
|
||||
Map.of("sourceTableId", sourceTable.getId(), "batchNo", holder.batchNo), account);
|
||||
derivedIds.add(holder.targetTable.getId());
|
||||
holder.currentSize = 0;
|
||||
}
|
||||
saveToTable(holder.targetTable, mapRow(row), account);
|
||||
holder.currentSize++;
|
||||
});
|
||||
finishJobSuccess(job, totalRows, totalRows, Map.of("derivedTableIds", derivedIds));
|
||||
return job;
|
||||
}
|
||||
|
||||
private DatacenterImportJob doSplitByFieldValue(DatacenterExcelSplitRequest request, LoginAccount account, DatacenterImportJob job) {
|
||||
if (request == null || request.getDatasetRef() == null || request.getFieldName() == null || request.getFieldName().isBlank()) {
|
||||
throw new BusinessException("按字段值拆分需要数据集和字段名");
|
||||
}
|
||||
DatacenterTable sourceTable = resolveTable(request.getDatasetRef());
|
||||
DatacenterSource source = registryService.getSourceRequired(sourceTable.getSourceId());
|
||||
DatacenterCatalog catalog = requireCatalog(sourceTable.getCatalogId());
|
||||
DatacenterTableField splitField = sourceTable.getFields().stream()
|
||||
.filter(field -> request.getFieldName().equals(field.getFieldName()))
|
||||
.findFirst()
|
||||
.orElseThrow(() -> new BusinessException("拆分字段不存在: " + request.getFieldName()));
|
||||
String prefix = resolveSplitPrefix(request, sourceTable.getTableName());
|
||||
Map<String, DatacenterTable> targets = new LinkedHashMap<>();
|
||||
List<BigInteger> derivedIds = new ArrayList<>();
|
||||
long totalRows = iterateRows(buildFullQuery(sourceTable), row -> {
|
||||
String fieldValue = stringify(row.get(splitField.getFieldName()));
|
||||
String bucket = fieldValue == null || fieldValue.isBlank() ? "empty" : fieldValue;
|
||||
DatacenterTable targetTable = targets.get(bucket);
|
||||
if (targetTable == null) {
|
||||
targetTable = createDerivedTable(source, catalog, cloneFields(sourceTable.getFields()),
|
||||
prefix + "_" + normalizeIdentifier(bucket), "SPLIT_BY_FIELD_VALUE",
|
||||
Map.of("sourceTableId", sourceTable.getId(), "fieldName", splitField.getFieldName(), "fieldValue", bucket), account);
|
||||
createLineage(sourceTable.getId(), targetTable.getId(), "SPLIT_BY_FIELD_VALUE",
|
||||
Map.of("sourceTableId", sourceTable.getId(), "fieldName", splitField.getFieldName(), "fieldValue", bucket), account);
|
||||
targets.put(bucket, targetTable);
|
||||
derivedIds.add(targetTable.getId());
|
||||
}
|
||||
saveToTable(targetTable, mapRow(row), account);
|
||||
});
|
||||
finishJobSuccess(job, totalRows, totalRows, Map.of("derivedTableIds", derivedIds));
|
||||
return job;
|
||||
}
|
||||
|
||||
private DatacenterImportJob doVerticalMerge(DatacenterExcelMergeRequest request, LoginAccount account, DatacenterImportJob job) {
|
||||
List<DatacenterTable> tables = request.getDatasetRefs().stream().map(this::resolveTable).toList();
|
||||
DatacenterTable firstTable = tables.get(0);
|
||||
DatacenterSource source = registryService.getSourceRequired(firstTable.getSourceId());
|
||||
DatacenterCatalog catalog = requireCatalog(firstTable.getCatalogId());
|
||||
assertSameCatalog(tables);
|
||||
assertSameFields(tables);
|
||||
DatacenterTable targetTable = createDerivedTable(source, catalog, cloneFields(firstTable.getFields()),
|
||||
request.getTargetTableName(), "MERGE_VERTICAL", Map.of("sourceTableIds", tables.stream().map(DatacenterTable::getId).toList()), account);
|
||||
long successRows = 0L;
|
||||
for (DatacenterTable table : tables) {
|
||||
successRows += copyRows(buildFullQuery(table), this::mapRow, targetTable, account);
|
||||
createLineage(table.getId(), targetTable.getId(), "MERGE_VERTICAL", Map.of("sourceTableId", table.getId()), account);
|
||||
}
|
||||
job.setTableId(targetTable.getId());
|
||||
finishJobSuccess(job, successRows, successRows, Map.of("derivedTableId", targetTable.getId()));
|
||||
return job;
|
||||
}
|
||||
|
||||
private DatacenterImportJob doHorizontalMerge(DatacenterExcelMergeRequest request, LoginAccount account, DatacenterImportJob job) {
|
||||
if (request.getJoinKey() == null || request.getJoinKey().isBlank()) {
|
||||
throw new BusinessException("横向合并必须指定 joinKey");
|
||||
}
|
||||
List<DatacenterTable> tables = request.getDatasetRefs().stream().map(this::resolveTable).toList();
|
||||
DatacenterTable firstTable = tables.get(0);
|
||||
DatacenterSource source = registryService.getSourceRequired(firstTable.getSourceId());
|
||||
DatacenterCatalog catalog = requireCatalog(firstTable.getCatalogId());
|
||||
assertSameCatalog(tables);
|
||||
|
||||
List<DatacenterTableField> mergedFields = new ArrayList<>();
|
||||
Set<String> usedFieldNames = new LinkedHashSet<>();
|
||||
Map<BigInteger, Map<String, String>> fieldMappings = new LinkedHashMap<>();
|
||||
for (DatacenterTable table : tables) {
|
||||
Map<String, String> mapping = new LinkedHashMap<>();
|
||||
for (DatacenterTableField field : table.getFields()) {
|
||||
String targetFieldName;
|
||||
if (field.getFieldName().equals(request.getJoinKey())) {
|
||||
targetFieldName = field.getFieldName();
|
||||
} else {
|
||||
targetFieldName = field.getFieldName();
|
||||
if (usedFieldNames.contains(targetFieldName)) {
|
||||
targetFieldName = normalizeIdentifier(table.getTableName()) + "_" + targetFieldName;
|
||||
}
|
||||
}
|
||||
if (!usedFieldNames.contains(targetFieldName)) {
|
||||
usedFieldNames.add(targetFieldName);
|
||||
mergedFields.add(cloneField(field, targetFieldName, field.getFieldDesc()));
|
||||
}
|
||||
mapping.put(field.getFieldName(), targetFieldName);
|
||||
}
|
||||
fieldMappings.put(table.getId(), mapping);
|
||||
}
|
||||
|
||||
DatacenterTable targetTable = createDerivedTable(source, catalog, mergedFields,
|
||||
request.getTargetTableName(), "MERGE_HORIZONTAL", Map.of("sourceTableIds", tables.stream().map(DatacenterTable::getId).toList(), "joinKey", request.getJoinKey()), account);
|
||||
Map<String, JSONObject> mergedRows = new LinkedHashMap<>();
|
||||
for (DatacenterTable table : tables) {
|
||||
Map<String, String> mapping = fieldMappings.get(table.getId());
|
||||
iterateRows(buildFullQuery(table), row -> {
|
||||
String joinValue = stringify(row.get(request.getJoinKey()));
|
||||
if (joinValue == null || joinValue.isBlank()) {
|
||||
return;
|
||||
}
|
||||
JSONObject target = mergedRows.computeIfAbsent(joinValue, key -> new JSONObject());
|
||||
mapping.forEach((sourceField, targetField) -> target.put(targetField, row.get(sourceField)));
|
||||
});
|
||||
createLineage(table.getId(), targetTable.getId(), "MERGE_HORIZONTAL", Map.of("sourceTableId", table.getId(), "joinKey", request.getJoinKey()), account);
|
||||
}
|
||||
for (JSONObject row : mergedRows.values()) {
|
||||
saveToTable(targetTable, row, account);
|
||||
}
|
||||
job.setTableId(targetTable.getId());
|
||||
finishJobSuccess(job, (long) mergedRows.size(), (long) mergedRows.size(), Map.of("derivedTableId", targetTable.getId()));
|
||||
return job;
|
||||
}
|
||||
|
||||
private DatacenterImportJob createJob(String jobType, BigInteger sourceId, BigInteger catalogId, BigInteger tableId,
|
||||
String fileName, Map<String, Object> payload, LoginAccount account) {
|
||||
DatacenterImportJob job = new DatacenterImportJob();
|
||||
job.setSourceId(sourceId);
|
||||
job.setCatalogId(catalogId);
|
||||
job.setTableId(tableId);
|
||||
job.setTenantId(account == null || account.getTenantId() == null ? BigInteger.ZERO : account.getTenantId());
|
||||
job.setDeptId(account == null || account.getDeptId() == null ? BigInteger.ZERO : account.getDeptId());
|
||||
job.setJobType(jobType);
|
||||
job.setFileName(fileName);
|
||||
job.setStatus(DatacenterImportStatus.RUNNING.name());
|
||||
job.setPayloadJson(payload == null ? new LinkedHashMap<>() : new LinkedHashMap<>(payload));
|
||||
job.setStartedAt(new Date());
|
||||
job.setCreated(new Date());
|
||||
job.setModified(new Date());
|
||||
job.setCreatedBy(account == null ? BigInteger.ZERO : account.getId());
|
||||
job.setModifiedBy(account == null ? BigInteger.ZERO : account.getId());
|
||||
importJobMapper.insert(job);
|
||||
return job;
|
||||
}
|
||||
|
||||
private Map<String, Object> buildPayload(String key, Object value) {
|
||||
Map<String, Object> payload = new LinkedHashMap<>();
|
||||
payload.put(key, value);
|
||||
return payload;
|
||||
}
|
||||
|
||||
private void finishJobSuccess(DatacenterImportJob job, long totalRows, long successRows, Map<String, Object> payload) {
|
||||
job.setStatus(DatacenterImportStatus.SUCCESS.name());
|
||||
job.setTotalRows(totalRows);
|
||||
job.setSuccessRows(successRows);
|
||||
job.setErrorRows(Math.max(0L, totalRows - successRows));
|
||||
if (payload != null) {
|
||||
job.setPayloadJson(new LinkedHashMap<>(payload));
|
||||
}
|
||||
job.setFinishedAt(new Date());
|
||||
job.setModified(new Date());
|
||||
importJobMapper.update(job);
|
||||
}
|
||||
|
||||
private void finishJobFailure(DatacenterImportJob job, Exception ex) {
|
||||
job.setStatus(DatacenterImportStatus.FAILED.name());
|
||||
job.setErrorSummary(ex.getMessage());
|
||||
job.setFinishedAt(new Date());
|
||||
job.setModified(new Date());
|
||||
importJobMapper.update(job);
|
||||
}
|
||||
|
||||
private DatacenterTable resolveTable(DatasetRef datasetRef) {
|
||||
if (datasetRef == null || datasetRef.getTableId() == null) {
|
||||
throw new BusinessException("缺少数据集 tableId");
|
||||
}
|
||||
return registryService.getTableWithFields(datasetRef.getTableId());
|
||||
}
|
||||
|
||||
private DatacenterCatalog requireCatalog(BigInteger catalogId) {
|
||||
DatacenterCatalog catalog = registryService.getCatalogById(catalogId);
|
||||
if (catalog == null) {
|
||||
throw new BusinessException("目录不存在: " + catalogId);
|
||||
}
|
||||
return catalog;
|
||||
}
|
||||
|
||||
private DatacenterTable createDerivedTable(DatacenterSource source, DatacenterCatalog catalog, List<DatacenterTableField> fields,
|
||||
String tableName, String deriveType, Map<String, Object> config, LoginAccount account) {
|
||||
DatacenterTable table = new DatacenterTable();
|
||||
String resolvedName = uniqueTableName(source.getId(), catalog.getId(), normalizeLogicalName(tableName, deriveType));
|
||||
table.setTableName(resolvedName);
|
||||
table.setTableDesc(resolvedName);
|
||||
table.setActualTable(buildMaterializedTableName(source.getId(), Math.abs(Objects.hash(resolvedName, deriveType))));
|
||||
table.setMaterializedTable(table.getActualTable());
|
||||
table.setTableKind(DatacenterTableKind.DERIVED_TABLE.name());
|
||||
table.setAccessMode("READ_WRITE");
|
||||
table.setVersioningEnabled(1);
|
||||
table.setCapabilitiesJson(defaultExcelCapabilities());
|
||||
table.setFields(fields);
|
||||
|
||||
DatacenterTableDetailMeta detail = new DatacenterTableDetailMeta();
|
||||
detail.setTable(table);
|
||||
detail.setFields(fields);
|
||||
dbHandleManager.getDbHandler().createTable(table);
|
||||
DatacenterTable savedTable = registryService.registerTable(source, catalog, detail, account);
|
||||
savedTable.setFields(registryService.getFields(savedTable.getId()));
|
||||
createVersion(savedTable, deriveType.toLowerCase(Locale.ROOT), config, account);
|
||||
return savedTable;
|
||||
}
|
||||
|
||||
private DatacenterDatasetVersion createVersion(DatacenterTable table, String versionLabel, Map<String, Object> snapshot, LoginAccount account) {
|
||||
QueryWrapperWrapper wrapper = new QueryWrapperWrapper(table.getId());
|
||||
DatacenterDatasetVersion version = new DatacenterDatasetVersion();
|
||||
version.setTableId(table.getId());
|
||||
version.setTenantId(table.getTenantId());
|
||||
version.setDeptId(table.getDeptId());
|
||||
version.setVersionNo(wrapper.nextVersionNo(datasetVersionMapper));
|
||||
version.setVersionLabel(versionLabel);
|
||||
version.setMaterializedTable(table.getMaterializedTable());
|
||||
version.setSnapshotJson(snapshot == null ? new LinkedHashMap<>() : new LinkedHashMap<>(snapshot));
|
||||
version.setStatus(0);
|
||||
version.setCreated(new Date());
|
||||
version.setModified(new Date());
|
||||
version.setCreatedBy(account == null ? BigInteger.ZERO : account.getId());
|
||||
version.setModifiedBy(account == null ? BigInteger.ZERO : account.getId());
|
||||
datasetVersionMapper.insert(version);
|
||||
return version;
|
||||
}
|
||||
|
||||
private void createLineage(BigInteger sourceTableId, BigInteger derivedTableId, String deriveType, Map<String, Object> config, LoginAccount account) {
|
||||
DatacenterDerivedTable relation = new DatacenterDerivedTable();
|
||||
relation.setSourceTableId(sourceTableId);
|
||||
relation.setDerivedTableId(derivedTableId);
|
||||
relation.setDeriveType(deriveType);
|
||||
relation.setDeriveConfigJson(config == null ? new LinkedHashMap<>() : new LinkedHashMap<>(config));
|
||||
relation.setStatus(0);
|
||||
relation.setTenantId(account == null || account.getTenantId() == null ? BigInteger.ZERO : account.getTenantId());
|
||||
relation.setDeptId(account == null || account.getDeptId() == null ? BigInteger.ZERO : account.getDeptId());
|
||||
relation.setCreated(new Date());
|
||||
relation.setModified(new Date());
|
||||
relation.setCreatedBy(account == null ? BigInteger.ZERO : account.getId());
|
||||
relation.setModifiedBy(account == null ? BigInteger.ZERO : account.getId());
|
||||
derivedTableMapper.insert(relation);
|
||||
}
|
||||
|
||||
private long copyRows(DatacenterQueryRequest queryRequest, RowMapper mapper, DatacenterTable targetTable, LoginAccount account) {
|
||||
return iterateRows(queryRequest, row -> saveToTable(targetTable, mapper.map(row), account));
|
||||
}
|
||||
|
||||
private long iterateRows(DatacenterQueryRequest queryRequest, RowConsumer consumer) {
|
||||
long total = 0L;
|
||||
long pageNumber = 1L;
|
||||
while (true) {
|
||||
queryRequest.setPageNumber(pageNumber);
|
||||
queryRequest.setPageSize(QUERY_BATCH_SIZE);
|
||||
Page<Row> page = queryService.queryPage(queryRequest);
|
||||
if (page.getRecords() == null || page.getRecords().isEmpty()) {
|
||||
break;
|
||||
}
|
||||
for (Row row : page.getRecords()) {
|
||||
consumer.accept(row);
|
||||
total++;
|
||||
}
|
||||
if (page.getRecords().size() < QUERY_BATCH_SIZE) {
|
||||
break;
|
||||
}
|
||||
pageNumber++;
|
||||
}
|
||||
return total;
|
||||
}
|
||||
|
||||
private void saveToTable(DatacenterTable targetTable, JSONObject data, LoginAccount account) {
|
||||
dbHandleManager.getDbHandler().saveValue(targetTable, data, account);
|
||||
}
|
||||
|
||||
private DatacenterQueryRequest buildFullQuery(DatacenterTable table) {
|
||||
DatacenterQueryRequest queryRequest = new DatacenterQueryRequest();
|
||||
queryRequest.setDatasetRef(registryService.resolveDatasetRef(table.getId()));
|
||||
queryRequest.setSelectedColumns(table.getFields().stream().map(DatacenterTableField::getFieldName).toList());
|
||||
return queryRequest;
|
||||
}
|
||||
|
||||
private JSONObject mapRow(Row row) {
|
||||
JSONObject payload = new JSONObject();
|
||||
row.forEach(payload::put);
|
||||
payload.remove("id");
|
||||
payload.remove("dept_id");
|
||||
payload.remove("tenant_id");
|
||||
payload.remove("created");
|
||||
payload.remove("created_by");
|
||||
payload.remove("modified");
|
||||
payload.remove("modified_by");
|
||||
payload.remove("remark");
|
||||
return payload;
|
||||
}
|
||||
|
||||
private JSONObject mapDerivedRow(Row row, List<String> selectedColumns, DatacenterExcelDeriveRequest request) {
|
||||
JSONObject payload = new JSONObject();
|
||||
for (String column : selectedColumns) {
|
||||
String targetName = request.getRenameMappings().getOrDefault(column, column);
|
||||
payload.put(targetName, row.get(column));
|
||||
}
|
||||
return payload;
|
||||
}
|
||||
|
||||
private List<String> resolveSelectedColumns(DatacenterTable sourceTable, List<String> selectedColumns) {
|
||||
if (CollectionUtils.isEmpty(selectedColumns)) {
|
||||
return sourceTable.getFields().stream().map(DatacenterTableField::getFieldName).toList();
|
||||
}
|
||||
return selectedColumns;
|
||||
}
|
||||
|
||||
private List<DatacenterTableField> buildDerivedFields(DatacenterTable sourceTable, DatacenterExcelDeriveRequest request) {
|
||||
List<String> selectedColumns = resolveSelectedColumns(sourceTable, request.getSelectedColumns());
|
||||
Map<String, DatacenterTableField> fieldMap = new LinkedHashMap<>();
|
||||
for (DatacenterTableField field : sourceTable.getFields()) {
|
||||
fieldMap.put(field.getFieldName(), field);
|
||||
}
|
||||
List<DatacenterTableField> fields = new ArrayList<>();
|
||||
for (String column : selectedColumns) {
|
||||
DatacenterTableField sourceField = fieldMap.get(column);
|
||||
if (sourceField == null) {
|
||||
throw new BusinessException("派生字段不存在: " + column);
|
||||
}
|
||||
String targetName = request.getRenameMappings().getOrDefault(column, column);
|
||||
fields.add(cloneField(sourceField, targetName, targetName));
|
||||
}
|
||||
return fields;
|
||||
}
|
||||
|
||||
private List<DatacenterTableField> cloneFields(List<DatacenterTableField> sourceFields) {
|
||||
List<DatacenterTableField> fields = new ArrayList<>();
|
||||
for (DatacenterTableField field : sourceFields) {
|
||||
fields.add(cloneField(field, field.getFieldName(), field.getFieldDesc()));
|
||||
}
|
||||
return fields;
|
||||
}
|
||||
|
||||
private DatacenterTableField cloneField(DatacenterTableField source, String fieldName, String fieldDesc) {
|
||||
DatacenterTableField field = new DatacenterTableField();
|
||||
field.setFieldName(fieldName);
|
||||
field.setSourceColumnName(source.getSourceColumnName());
|
||||
field.setFieldDesc(fieldDesc);
|
||||
field.setFieldType(source.getFieldType());
|
||||
field.setJdbcType(source.getJdbcType());
|
||||
field.setPrecision(source.getPrecision());
|
||||
field.setScale(source.getScale());
|
||||
field.setRequired(source.getRequired());
|
||||
field.setQueryable(source.getQueryable());
|
||||
field.setSortable(source.getSortable());
|
||||
field.setWritable(source.getWritable());
|
||||
field.setIndexed(source.getIndexed());
|
||||
field.setOptions(source.getOptions());
|
||||
return field;
|
||||
}
|
||||
|
||||
private Map<String, Object> defaultExcelCapabilities() {
|
||||
return Map.of("capabilities", List.of("READ_QUERY", "WRITE_MUTATION", "MATERIALIZE", "EXPORT"));
|
||||
}
|
||||
|
||||
private void assertSameCatalog(List<DatacenterTable> tables) {
|
||||
Set<BigInteger> catalogIds = new HashSet<>();
|
||||
Set<BigInteger> sourceIds = new HashSet<>();
|
||||
for (DatacenterTable table : tables) {
|
||||
catalogIds.add(table.getCatalogId());
|
||||
sourceIds.add(table.getSourceId());
|
||||
}
|
||||
if (catalogIds.size() > 1 || sourceIds.size() > 1) {
|
||||
throw new BusinessException("Excel 操作暂只支持同一 workbook/catalog 下的数据集");
|
||||
}
|
||||
}
|
||||
|
||||
private void assertSameFields(List<DatacenterTable> tables) {
|
||||
List<String> first = tables.get(0).getFields().stream().map(DatacenterTableField::getFieldName).toList();
|
||||
for (int i = 1; i < tables.size(); i++) {
|
||||
List<String> current = tables.get(i).getFields().stream().map(DatacenterTableField::getFieldName).toList();
|
||||
if (!first.equals(current)) {
|
||||
throw new BusinessException("纵向合并仅支持同结构表");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private List<DatacenterTable> resolveExportTables(DatacenterExcelExportRequest request) {
|
||||
List<DatacenterTable> tables = new ArrayList<>();
|
||||
if (request != null && !CollectionUtils.isEmpty(request.getDatasetRefs())) {
|
||||
for (DatasetRef datasetRef : request.getDatasetRefs()) {
|
||||
tables.add(resolveTable(datasetRef));
|
||||
}
|
||||
return tables;
|
||||
}
|
||||
if (request == null || request.getSourceId() == null) {
|
||||
throw new BusinessException("导出需要 sourceId 或 datasetRefs");
|
||||
}
|
||||
tables.addAll(registryService.listManagedTables(request.getSourceId(), request.getCatalogId()));
|
||||
return tables.stream().map(table -> registryService.getTableWithFields(table.getId())).toList();
|
||||
}
|
||||
|
||||
private void writeHeaderRow(org.apache.poi.ss.usermodel.Sheet sheet, List<DatacenterTableField> fields) {
|
||||
org.apache.poi.ss.usermodel.Row headerRow = sheet.createRow(0);
|
||||
for (int i = 0; i < fields.size(); i++) {
|
||||
Cell cell = headerRow.createCell(i);
|
||||
cell.setCellValue(fields.get(i).getFieldDesc());
|
||||
}
|
||||
}
|
||||
|
||||
private Path ensureExportDir() throws Exception {
|
||||
Path dir = Path.of(System.getProperty("java.io.tmpdir"), "easyflow-datacenter", "exports");
|
||||
Files.createDirectories(dir);
|
||||
return dir;
|
||||
}
|
||||
|
||||
private String buildExportFileName(String rawFileName) {
|
||||
String baseName = rawFileName == null || rawFileName.isBlank() ? "excel_export" : extractWorkbookName(rawFileName);
|
||||
return normalizeIdentifier(baseName) + "_" + EXPORT_TIME_FORMAT.format(LocalDateTime.now()) + ".xlsx";
|
||||
}
|
||||
|
||||
private String uniqueSheetName(String rawName, Set<String> usedSheetNames) {
|
||||
String base = rawName == null || rawName.isBlank() ? "Sheet" : rawName;
|
||||
base = base.length() > 31 ? base.substring(0, 31) : base;
|
||||
String result = base;
|
||||
int suffix = 1;
|
||||
while (usedSheetNames.contains(result)) {
|
||||
String suffixText = "_" + suffix++;
|
||||
int limit = Math.max(1, 31 - suffixText.length());
|
||||
result = base.substring(0, Math.min(base.length(), limit)) + suffixText;
|
||||
}
|
||||
usedSheetNames.add(result);
|
||||
return result;
|
||||
}
|
||||
|
||||
private List<DatacenterTableField> buildFields(org.apache.poi.ss.usermodel.Row headerRow, DataFormatter formatter) {
|
||||
List<DatacenterTableField> fields = new ArrayList<>();
|
||||
Set<String> usedNames = new HashSet<>();
|
||||
short lastCellNum = headerRow.getLastCellNum();
|
||||
for (int cellIndex = 0; cellIndex < lastCellNum; cellIndex++) {
|
||||
String header = formatter.formatCellValue(headerRow.getCell(cellIndex));
|
||||
String fieldName = normalizeIdentifier(header, cellIndex, usedNames);
|
||||
DatacenterTableField field = new DatacenterTableField();
|
||||
field.setFieldName(fieldName);
|
||||
field.setSourceColumnName(header);
|
||||
field.setFieldDesc(header == null || header.isBlank() ? fieldName : header);
|
||||
field.setFieldType(EnumFieldType.STRING.getCode());
|
||||
field.setJdbcType("VARCHAR");
|
||||
field.setPrecision(255);
|
||||
field.setScale(0);
|
||||
field.setRequired(0);
|
||||
field.setQueryable(1);
|
||||
field.setSortable(1);
|
||||
field.setWritable(1);
|
||||
field.setIndexed(0);
|
||||
fields.add(field);
|
||||
}
|
||||
return fields;
|
||||
}
|
||||
|
||||
private String normalizeMode(String value, String defaultValue) {
|
||||
return value == null || value.isBlank() ? defaultValue : value.trim().toUpperCase(Locale.ROOT);
|
||||
}
|
||||
|
||||
private String resolveSplitPrefix(DatacenterExcelSplitRequest request, String fallback) {
|
||||
if (request != null && request.getTargetNamePrefix() != null && !request.getTargetNamePrefix().isBlank()) {
|
||||
return request.getTargetNamePrefix();
|
||||
}
|
||||
return fallback + "_split";
|
||||
}
|
||||
|
||||
private String normalizeLogicalName(String tableName, String deriveType) {
|
||||
if (tableName != null && !tableName.isBlank()) {
|
||||
return tableName;
|
||||
}
|
||||
return deriveType.toLowerCase(Locale.ROOT) + "_" + System.currentTimeMillis();
|
||||
}
|
||||
|
||||
private String uniqueTableName(BigInteger sourceId, BigInteger catalogId, String rawName) {
|
||||
String baseName = rawName == null || rawName.isBlank() ? "dataset" : rawName;
|
||||
baseName = baseName.trim();
|
||||
String result = baseName;
|
||||
int suffix = 1;
|
||||
while (tableNameExists(sourceId, catalogId, result)) {
|
||||
result = baseName + "_" + suffix++;
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
private boolean tableNameExists(BigInteger sourceId, BigInteger catalogId, String tableName) {
|
||||
List<DatacenterTable> tables = registryService.listManagedTables(sourceId, catalogId);
|
||||
return tables.stream().anyMatch(table -> tableName.equals(table.getTableName()));
|
||||
}
|
||||
|
||||
private String extractWorkbookName(String originalFileName) {
|
||||
if (originalFileName == null || originalFileName.isBlank()) {
|
||||
return "excel_workbook";
|
||||
}
|
||||
int index = originalFileName.lastIndexOf('.');
|
||||
return index > 0 ? originalFileName.substring(0, index) : originalFileName;
|
||||
}
|
||||
|
||||
private String buildMaterializedTableName(BigInteger sourceId, int sheetIndex) {
|
||||
long snowId = new SnowFlakeIDKeyGenerator().nextId();
|
||||
return "tb_excel_" + sourceId + "_" + sheetIndex + "_" + snowId;
|
||||
}
|
||||
|
||||
private String normalizeIdentifier(String raw) {
|
||||
if (raw == null || raw.isBlank()) {
|
||||
return "value";
|
||||
}
|
||||
String normalized = raw.trim().toLowerCase(Locale.ROOT).replaceAll("[^a-z0-9_\\u4e00-\\u9fa5]+", "_");
|
||||
normalized = normalized.replaceAll("_+", "_");
|
||||
if (normalized.isBlank()) {
|
||||
return "value";
|
||||
}
|
||||
return normalized;
|
||||
}
|
||||
|
||||
private String normalizeIdentifier(String raw, int index, Set<String> usedNames) {
|
||||
String value = normalizeIdentifier(raw);
|
||||
if (value.isBlank() || "value".equals(value)) {
|
||||
value = "col_" + (index + 1);
|
||||
}
|
||||
if (Character.isDigit(value.charAt(0))) {
|
||||
value = "col_" + value;
|
||||
}
|
||||
String result = value;
|
||||
int suffix = 1;
|
||||
while (usedNames.contains(result)) {
|
||||
result = value + "_" + suffix++;
|
||||
}
|
||||
usedNames.add(result);
|
||||
return result;
|
||||
}
|
||||
|
||||
private String stringify(Object value) {
|
||||
return value == null ? null : String.valueOf(value);
|
||||
}
|
||||
|
||||
private static final class Holder {
|
||||
private DatacenterTable targetTable;
|
||||
private int batchNo;
|
||||
private int currentSize;
|
||||
}
|
||||
|
||||
private interface RowConsumer {
|
||||
void accept(Row row);
|
||||
}
|
||||
|
||||
private interface RowMapper {
|
||||
JSONObject map(Row row);
|
||||
}
|
||||
|
||||
private static final class QueryWrapperWrapper {
|
||||
private final BigInteger tableId;
|
||||
|
||||
private QueryWrapperWrapper(BigInteger tableId) {
|
||||
this.tableId = tableId;
|
||||
}
|
||||
|
||||
private int nextVersionNo(DatacenterDatasetVersionMapper mapper) {
|
||||
return mapper.selectListByQuery(com.mybatisflex.core.query.QueryWrapper.create()
|
||||
.eq(DatacenterDatasetVersion::getTableId, tableId)
|
||||
.orderBy("version_no desc"))
|
||||
.stream()
|
||||
.findFirst()
|
||||
.map(version -> version.getVersionNo() + 1)
|
||||
.orElse(1);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,24 @@
|
||||
package tech.easyflow.datacenter.execution.model;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
public class DatacenterConnectionTestResult {
|
||||
private boolean success;
|
||||
private String errorCode;
|
||||
private String message;
|
||||
private List<String> capabilities = new ArrayList<>();
|
||||
private Map<String, Object> details;
|
||||
|
||||
public boolean isSuccess() { return success; }
|
||||
public void setSuccess(boolean success) { this.success = success; }
|
||||
public String getErrorCode() { return errorCode; }
|
||||
public void setErrorCode(String errorCode) { this.errorCode = errorCode; }
|
||||
public String getMessage() { return message; }
|
||||
public void setMessage(String message) { this.message = message; }
|
||||
public List<String> getCapabilities() { return capabilities; }
|
||||
public void setCapabilities(List<String> capabilities) { this.capabilities = capabilities; }
|
||||
public Map<String, Object> getDetails() { return details; }
|
||||
public void setDetails(Map<String, Object> details) { this.details = details; }
|
||||
}
|
||||
@@ -0,0 +1,19 @@
|
||||
package tech.easyflow.datacenter.execution.model;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
public class DatacenterQueryFilter {
|
||||
private String column;
|
||||
private String operator;
|
||||
private Object value;
|
||||
private List<Object> values;
|
||||
|
||||
public String getColumn() { return column; }
|
||||
public void setColumn(String column) { this.column = column; }
|
||||
public String getOperator() { return operator; }
|
||||
public void setOperator(String operator) { this.operator = operator; }
|
||||
public Object getValue() { return value; }
|
||||
public void setValue(Object value) { this.value = value; }
|
||||
public List<Object> getValues() { return values; }
|
||||
public void setValues(List<Object> values) { this.values = values; }
|
||||
}
|
||||
@@ -0,0 +1,29 @@
|
||||
package tech.easyflow.datacenter.execution.model;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
public class DatacenterQueryRequest {
|
||||
private DatasetRef datasetRef;
|
||||
private Long pageNumber = 1L;
|
||||
private Long pageSize = 10L;
|
||||
private List<DatacenterQueryFilter> filters = new ArrayList<>();
|
||||
private List<DatacenterQuerySort> sorts = new ArrayList<>();
|
||||
private List<String> selectedColumns = new ArrayList<>();
|
||||
private String where;
|
||||
|
||||
public DatasetRef getDatasetRef() { return datasetRef; }
|
||||
public void setDatasetRef(DatasetRef datasetRef) { this.datasetRef = datasetRef; }
|
||||
public Long getPageNumber() { return pageNumber; }
|
||||
public void setPageNumber(Long pageNumber) { this.pageNumber = pageNumber; }
|
||||
public Long getPageSize() { return pageSize; }
|
||||
public void setPageSize(Long pageSize) { this.pageSize = pageSize; }
|
||||
public List<DatacenterQueryFilter> getFilters() { return filters; }
|
||||
public void setFilters(List<DatacenterQueryFilter> filters) { this.filters = filters; }
|
||||
public List<DatacenterQuerySort> getSorts() { return sorts; }
|
||||
public void setSorts(List<DatacenterQuerySort> sorts) { this.sorts = sorts; }
|
||||
public List<String> getSelectedColumns() { return selectedColumns; }
|
||||
public void setSelectedColumns(List<String> selectedColumns) { this.selectedColumns = selectedColumns; }
|
||||
public String getWhere() { return where; }
|
||||
public void setWhere(String where) { this.where = where; }
|
||||
}
|
||||
@@ -0,0 +1,11 @@
|
||||
package tech.easyflow.datacenter.execution.model;
|
||||
|
||||
public class DatacenterQuerySort {
|
||||
private String column;
|
||||
private String direction;
|
||||
|
||||
public String getColumn() { return column; }
|
||||
public void setColumn(String column) { this.column = column; }
|
||||
public String getDirection() { return direction; }
|
||||
public void setDirection(String direction) { this.direction = direction; }
|
||||
}
|
||||
@@ -0,0 +1,39 @@
|
||||
package tech.easyflow.datacenter.execution.model;
|
||||
|
||||
import tech.easyflow.datacenter.entity.DatacenterTable;
|
||||
import tech.easyflow.datacenter.entity.DatacenterTableField;
|
||||
import tech.easyflow.datacenter.meta.entity.DatacenterCatalog;
|
||||
import tech.easyflow.datacenter.meta.entity.DatacenterDatasetVersion;
|
||||
import tech.easyflow.datacenter.meta.entity.DatacenterDerivedTable;
|
||||
import tech.easyflow.datacenter.meta.entity.DatacenterSource;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
public class DatacenterSchemaResponse {
|
||||
private DatasetRef datasetRef;
|
||||
private DatacenterSource source;
|
||||
private DatacenterCatalog catalog;
|
||||
private DatacenterTable table;
|
||||
private List<DatacenterTableField> fields = new ArrayList<>();
|
||||
private List<DatacenterDatasetVersion> versions = new ArrayList<>();
|
||||
private List<DatacenterDerivedTable> upstreamLineage = new ArrayList<>();
|
||||
private List<DatacenterDerivedTable> downstreamLineage = new ArrayList<>();
|
||||
|
||||
public DatasetRef getDatasetRef() { return datasetRef; }
|
||||
public void setDatasetRef(DatasetRef datasetRef) { this.datasetRef = datasetRef; }
|
||||
public DatacenterSource getSource() { return source; }
|
||||
public void setSource(DatacenterSource source) { this.source = source; }
|
||||
public DatacenterCatalog getCatalog() { return catalog; }
|
||||
public void setCatalog(DatacenterCatalog catalog) { this.catalog = catalog; }
|
||||
public DatacenterTable getTable() { return table; }
|
||||
public void setTable(DatacenterTable table) { this.table = table; }
|
||||
public List<DatacenterTableField> getFields() { return fields; }
|
||||
public void setFields(List<DatacenterTableField> fields) { this.fields = fields; }
|
||||
public List<DatacenterDatasetVersion> getVersions() { return versions; }
|
||||
public void setVersions(List<DatacenterDatasetVersion> versions) { this.versions = versions; }
|
||||
public List<DatacenterDerivedTable> getUpstreamLineage() { return upstreamLineage; }
|
||||
public void setUpstreamLineage(List<DatacenterDerivedTable> upstreamLineage) { this.upstreamLineage = upstreamLineage; }
|
||||
public List<DatacenterDerivedTable> getDownstreamLineage() { return downstreamLineage; }
|
||||
public void setDownstreamLineage(List<DatacenterDerivedTable> downstreamLineage) { this.downstreamLineage = downstreamLineage; }
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
package tech.easyflow.datacenter.execution.model;
|
||||
|
||||
public class DatacenterSqlQueryRequest {
|
||||
|
||||
private DatasetRef datasetRef;
|
||||
private String sql;
|
||||
|
||||
public DatasetRef getDatasetRef() {
|
||||
return datasetRef;
|
||||
}
|
||||
|
||||
public void setDatasetRef(DatasetRef datasetRef) {
|
||||
this.datasetRef = datasetRef;
|
||||
}
|
||||
|
||||
public String getSql() {
|
||||
return sql;
|
||||
}
|
||||
|
||||
public void setSql(String sql) {
|
||||
this.sql = sql;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,25 @@
|
||||
package tech.easyflow.datacenter.execution.model;
|
||||
|
||||
import java.math.BigInteger;
|
||||
|
||||
public class DatasetRef {
|
||||
private BigInteger sourceId;
|
||||
private BigInteger catalogId;
|
||||
private String catalogName;
|
||||
private BigInteger tableId;
|
||||
private String tableName;
|
||||
private BigInteger versionId;
|
||||
|
||||
public BigInteger getSourceId() { return sourceId; }
|
||||
public void setSourceId(BigInteger sourceId) { this.sourceId = sourceId; }
|
||||
public BigInteger getCatalogId() { return catalogId; }
|
||||
public void setCatalogId(BigInteger catalogId) { this.catalogId = catalogId; }
|
||||
public String getCatalogName() { return catalogName; }
|
||||
public void setCatalogName(String catalogName) { this.catalogName = catalogName; }
|
||||
public BigInteger getTableId() { return tableId; }
|
||||
public void setTableId(BigInteger tableId) { this.tableId = tableId; }
|
||||
public String getTableName() { return tableName; }
|
||||
public void setTableName(String tableName) { this.tableName = tableName; }
|
||||
public BigInteger getVersionId() { return versionId; }
|
||||
public void setVersionId(BigInteger versionId) { this.versionId = versionId; }
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
package tech.easyflow.datacenter.execution.service;
|
||||
|
||||
import com.mybatisflex.core.paginate.Page;
|
||||
import com.mybatisflex.core.row.Row;
|
||||
import tech.easyflow.datacenter.execution.model.DatacenterQueryRequest;
|
||||
import tech.easyflow.datacenter.execution.model.DatacenterSchemaResponse;
|
||||
import tech.easyflow.datacenter.execution.model.DatacenterSqlQueryRequest;
|
||||
import tech.easyflow.datacenter.execution.model.DatasetRef;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
public interface DatacenterDatasetQueryService {
|
||||
Page<Row> queryPage(DatacenterQueryRequest request);
|
||||
|
||||
List<Row> queryBySql(DatacenterSqlQueryRequest request);
|
||||
|
||||
DatacenterSchemaResponse getSchema(DatasetRef datasetRef);
|
||||
}
|
||||
@@ -0,0 +1,13 @@
|
||||
package tech.easyflow.datacenter.execution.service;
|
||||
|
||||
import com.alibaba.fastjson2.JSONObject;
|
||||
import tech.easyflow.common.entity.LoginAccount;
|
||||
import tech.easyflow.datacenter.execution.model.DatasetRef;
|
||||
|
||||
import java.math.BigInteger;
|
||||
|
||||
public interface DatacenterDatasetWriteService {
|
||||
void saveRow(DatasetRef datasetRef, JSONObject data, LoginAccount account);
|
||||
|
||||
void deleteRow(DatasetRef datasetRef, BigInteger id, LoginAccount account);
|
||||
}
|
||||
@@ -0,0 +1,311 @@
|
||||
package tech.easyflow.datacenter.execution.service.impl;
|
||||
|
||||
import com.mybatisflex.core.paginate.Page;
|
||||
import com.mybatisflex.core.query.QueryWrapper;
|
||||
import com.mybatisflex.core.row.Row;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.springframework.util.CollectionUtils;
|
||||
import org.springframework.util.StringUtils;
|
||||
import tech.easyflow.common.web.exceptions.BusinessException;
|
||||
import tech.easyflow.datacenter.connector.DatacenterConnector;
|
||||
import tech.easyflow.datacenter.connector.DatacenterConnectorRegistry;
|
||||
import tech.easyflow.datacenter.entity.DatacenterTable;
|
||||
import tech.easyflow.datacenter.entity.DatacenterTableField;
|
||||
import tech.easyflow.datacenter.execution.model.DatacenterQueryRequest;
|
||||
import tech.easyflow.datacenter.execution.model.DatacenterSchemaResponse;
|
||||
import tech.easyflow.datacenter.execution.model.DatacenterSqlQueryRequest;
|
||||
import tech.easyflow.datacenter.execution.model.DatasetRef;
|
||||
import tech.easyflow.datacenter.execution.service.DatacenterDatasetQueryService;
|
||||
import tech.easyflow.datacenter.mapper.DatacenterCatalogMapper;
|
||||
import tech.easyflow.datacenter.mapper.DatacenterDatasetVersionMapper;
|
||||
import tech.easyflow.datacenter.mapper.DatacenterDerivedTableMapper;
|
||||
import tech.easyflow.datacenter.mapper.DatacenterTableMapper;
|
||||
import tech.easyflow.datacenter.meta.entity.DatacenterCatalog;
|
||||
import tech.easyflow.datacenter.meta.entity.DatacenterDatasetVersion;
|
||||
import tech.easyflow.datacenter.meta.entity.DatacenterDerivedTable;
|
||||
import tech.easyflow.datacenter.meta.entity.DatacenterSource;
|
||||
import tech.easyflow.datacenter.meta.service.DatacenterDatasetRegistryService;
|
||||
import tech.easyflow.datacenter.utils.SqlSupportUtils;
|
||||
|
||||
import javax.annotation.Resource;
|
||||
import java.math.BigInteger;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
@Service
|
||||
public class DatacenterDatasetQueryServiceImpl implements DatacenterDatasetQueryService {
|
||||
|
||||
@Resource
|
||||
private DatacenterDatasetRegistryService registryService;
|
||||
@Resource
|
||||
private DatacenterConnectorRegistry connectorRegistry;
|
||||
@Resource
|
||||
private DatacenterTableMapper tableMapper;
|
||||
@Resource
|
||||
private DatacenterCatalogMapper catalogMapper;
|
||||
@Resource
|
||||
private DatacenterDatasetVersionMapper datasetVersionMapper;
|
||||
@Resource
|
||||
private DatacenterDerivedTableMapper derivedTableMapper;
|
||||
|
||||
@Override
|
||||
public Page<Row> queryPage(DatacenterQueryRequest request) {
|
||||
if (request == null || request.getDatasetRef() == null) {
|
||||
throw new BusinessException("datasetRef 不能为空");
|
||||
}
|
||||
normalizePage(request);
|
||||
DatacenterTable table = resolveTable(request.getDatasetRef());
|
||||
DatacenterSource source = registryService.getSourceRequired(table.getSourceId());
|
||||
DatacenterCatalog catalog = registryService.getCatalogById(table.getCatalogId());
|
||||
DatacenterTable queryTable = resolveQueryTable(table, request.getDatasetRef());
|
||||
validateRequest(queryTable, request, source);
|
||||
request.getDatasetRef().setSourceId(table.getSourceId());
|
||||
request.getDatasetRef().setCatalogId(table.getCatalogId());
|
||||
request.getDatasetRef().setTableId(table.getId());
|
||||
request.getDatasetRef().setTableName(table.getTableName());
|
||||
if (catalog != null) {
|
||||
request.getDatasetRef().setCatalogName(catalog.getCatalogName());
|
||||
}
|
||||
DatacenterConnector connector = connectorRegistry.getConnector(source.getSourceType());
|
||||
return connector.queryPage(source, queryTable, request);
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<Row> queryBySql(DatacenterSqlQueryRequest request) {
|
||||
if (request == null || request.getDatasetRef() == null) {
|
||||
throw new BusinessException("datasetRef 不能为空");
|
||||
}
|
||||
String sql = trimToNull(request.getSql());
|
||||
if (!StringUtils.hasText(sql)) {
|
||||
throw new BusinessException("SQL 不能为空");
|
||||
}
|
||||
DatasetRef datasetRef = request.getDatasetRef();
|
||||
if (datasetRef.getSourceId() == null) {
|
||||
throw new BusinessException("缺少连接服务配置");
|
||||
}
|
||||
DatacenterSource source = registryService.getSourceRequired(datasetRef.getSourceId());
|
||||
BigInteger catalogId = resolveRequestedCatalogId(datasetRef);
|
||||
List<DatacenterTable> managedTables = registryService.listManagedTables(datasetRef.getSourceId(), catalogId);
|
||||
if (CollectionUtils.isEmpty(managedTables)) {
|
||||
throw new BusinessException("当前连接下没有已接入表");
|
||||
}
|
||||
SqlSupportUtils.ResolvedSql resolvedSql = SqlSupportUtils.resolve(
|
||||
sql,
|
||||
managedTables.stream().map(this::toManagedSqlTable).toList()
|
||||
);
|
||||
DatacenterConnector connector = connectorRegistry.getConnector(source.getSourceType());
|
||||
return connector.queryBySql(source, resolvedSql.getExecutableSql());
|
||||
}
|
||||
|
||||
@Override
|
||||
public DatacenterSchemaResponse getSchema(DatasetRef datasetRef) {
|
||||
DatacenterTable table = resolveTable(datasetRef);
|
||||
DatacenterSource source = registryService.getSourceRequired(table.getSourceId());
|
||||
DatacenterCatalog catalog = registryService.getCatalogById(table.getCatalogId());
|
||||
DatacenterSchemaResponse response = new DatacenterSchemaResponse();
|
||||
response.setDatasetRef(registryService.resolveDatasetRef(table.getId()));
|
||||
response.setSource(source);
|
||||
response.setCatalog(catalog);
|
||||
response.setTable(table);
|
||||
response.setFields(table.getFields());
|
||||
response.setVersions(listVersions(table.getId()));
|
||||
response.setUpstreamLineage(listUpstream(table.getId()));
|
||||
response.setDownstreamLineage(listDownstream(table.getId()));
|
||||
return response;
|
||||
}
|
||||
|
||||
private DatacenterTable resolveTable(DatasetRef datasetRef) {
|
||||
if (datasetRef.getTableId() != null) {
|
||||
return registryService.getTableWithFields(datasetRef.getTableId());
|
||||
}
|
||||
if (datasetRef.getSourceId() == null || !StringUtils.hasText(datasetRef.getTableName())) {
|
||||
throw new BusinessException("缺少数据集定位信息");
|
||||
}
|
||||
QueryWrapper wrapper = QueryWrapper.create();
|
||||
wrapper.eq(DatacenterTable::getSourceId, datasetRef.getSourceId());
|
||||
wrapper.eq(DatacenterTable::getTableName, datasetRef.getTableName().trim());
|
||||
|
||||
boolean hasCatalogCondition = false;
|
||||
if (datasetRef.getCatalogId() != null) {
|
||||
wrapper.eq(DatacenterTable::getCatalogId, datasetRef.getCatalogId());
|
||||
hasCatalogCondition = true;
|
||||
} else if (StringUtils.hasText(datasetRef.getCatalogName())) {
|
||||
DatacenterCatalog catalog = resolveCatalog(datasetRef.getSourceId(), datasetRef.getCatalogName().trim());
|
||||
wrapper.eq(DatacenterTable::getCatalogId, catalog.getId());
|
||||
hasCatalogCondition = true;
|
||||
}
|
||||
|
||||
List<DatacenterTable> tables = tableMapper.selectListByQuery(wrapper);
|
||||
if (CollectionUtils.isEmpty(tables)) {
|
||||
throw new BusinessException("数据集不存在: " + datasetRef.getTableName());
|
||||
}
|
||||
if (!hasCatalogCondition && tables.size() > 1) {
|
||||
throw new BusinessException("数据集存在重名表,请指定库名: " + datasetRef.getTableName());
|
||||
}
|
||||
return registryService.getTableWithFields(tables.get(0).getId());
|
||||
}
|
||||
|
||||
private DatacenterCatalog resolveCatalog(java.math.BigInteger sourceId, String catalogName) {
|
||||
QueryWrapper wrapper = QueryWrapper.create();
|
||||
wrapper.eq(DatacenterCatalog::getSourceId, sourceId);
|
||||
wrapper.eq(DatacenterCatalog::getCatalogName, catalogName);
|
||||
List<DatacenterCatalog> catalogs = catalogMapper.selectListByQuery(wrapper);
|
||||
if (CollectionUtils.isEmpty(catalogs)) {
|
||||
throw new BusinessException("库不存在: " + catalogName);
|
||||
}
|
||||
if (catalogs.size() > 1) {
|
||||
throw new BusinessException("库存在重复配置,请检查: " + catalogName);
|
||||
}
|
||||
return catalogs.get(0);
|
||||
}
|
||||
|
||||
private BigInteger resolveRequestedCatalogId(DatasetRef datasetRef) {
|
||||
if (datasetRef == null) {
|
||||
return null;
|
||||
}
|
||||
if (datasetRef.getCatalogId() != null) {
|
||||
return datasetRef.getCatalogId();
|
||||
}
|
||||
if (StringUtils.hasText(datasetRef.getCatalogName())) {
|
||||
DatacenterCatalog catalog = resolveCatalog(datasetRef.getSourceId(), datasetRef.getCatalogName().trim());
|
||||
return catalog.getId();
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
private SqlSupportUtils.ManagedTable toManagedSqlTable(DatacenterTable table) {
|
||||
DatacenterCatalog catalog = registryService.getCatalogById(table.getCatalogId());
|
||||
return new SqlSupportUtils.ManagedTable(
|
||||
catalog == null ? null : catalog.getCatalogName(),
|
||||
table.getTableName(),
|
||||
resolvePhysicalTableName(table)
|
||||
);
|
||||
}
|
||||
|
||||
private String resolvePhysicalTableName(DatacenterTable table) {
|
||||
if (table == null) {
|
||||
return null;
|
||||
}
|
||||
if ("EXTERNAL_TABLE".equals(table.getTableKind()) || "EXTERNAL_VIEW".equals(table.getTableKind())) {
|
||||
return trimToNull(table.getActualTable()) == null ? table.getTableName() : table.getActualTable().trim();
|
||||
}
|
||||
String materializedTable = trimToNull(table.getMaterializedTable());
|
||||
if (materializedTable != null) {
|
||||
return materializedTable;
|
||||
}
|
||||
String actualTable = trimToNull(table.getActualTable());
|
||||
return actualTable != null ? actualTable : table.getTableName();
|
||||
}
|
||||
|
||||
private void normalizePage(DatacenterQueryRequest request) {
|
||||
if (request.getPageNumber() == null || request.getPageNumber() < 1L) {
|
||||
request.setPageNumber(1L);
|
||||
}
|
||||
if (request.getPageSize() == null || request.getPageSize() < 1L) {
|
||||
throw new BusinessException("pageSize 必须大于 0");
|
||||
}
|
||||
}
|
||||
|
||||
private DatacenterTable resolveQueryTable(DatacenterTable table, DatasetRef datasetRef) {
|
||||
if (datasetRef == null || datasetRef.getVersionId() == null) {
|
||||
return table;
|
||||
}
|
||||
DatacenterDatasetVersion version = datasetVersionMapper.selectOneById(datasetRef.getVersionId());
|
||||
if (version == null || !table.getId().equals(version.getTableId())) {
|
||||
throw new BusinessException("数据集版本不存在: " + datasetRef.getVersionId());
|
||||
}
|
||||
DatacenterTable queryTable = new DatacenterTable();
|
||||
queryTable.setId(table.getId());
|
||||
queryTable.setSourceId(table.getSourceId());
|
||||
queryTable.setCatalogId(table.getCatalogId());
|
||||
queryTable.setTableName(table.getTableName());
|
||||
queryTable.setTableDesc(table.getTableDesc());
|
||||
queryTable.setActualTable(table.getActualTable());
|
||||
queryTable.setMaterializedTable(version.getMaterializedTable());
|
||||
queryTable.setAccessMode(table.getAccessMode());
|
||||
queryTable.setTableKind(table.getTableKind());
|
||||
queryTable.setVersioningEnabled(table.getVersioningEnabled());
|
||||
queryTable.setCapabilitiesJson(table.getCapabilitiesJson());
|
||||
queryTable.setFields(table.getFields());
|
||||
return queryTable;
|
||||
}
|
||||
|
||||
private void validateRequest(DatacenterTable table, DatacenterQueryRequest request, DatacenterSource source) {
|
||||
Map<String, DatacenterTableField> fieldMap = new LinkedHashMap<>();
|
||||
for (DatacenterTableField field : table.getFields()) {
|
||||
fieldMap.put(field.getFieldName(), field);
|
||||
}
|
||||
if (!CollectionUtils.isEmpty(request.getSelectedColumns())) {
|
||||
for (String column : request.getSelectedColumns()) {
|
||||
DatacenterTableField field = fieldMap.get(column);
|
||||
if (field == null || !isEnabled(field.getQueryable())) {
|
||||
throw new BusinessException("字段不可查询: " + column);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
request.setSelectedColumns(
|
||||
table.getFields().stream()
|
||||
.filter(field -> isEnabled(field.getQueryable()))
|
||||
.map(DatacenterTableField::getFieldName)
|
||||
.toList()
|
||||
);
|
||||
}
|
||||
if (!CollectionUtils.isEmpty(request.getFilters())) {
|
||||
request.getFilters().forEach(filter -> {
|
||||
DatacenterTableField field = fieldMap.get(filter.getColumn());
|
||||
if (field == null || !isEnabled(field.getQueryable())) {
|
||||
throw new BusinessException("字段不可过滤: " + filter.getColumn());
|
||||
}
|
||||
});
|
||||
}
|
||||
if (!CollectionUtils.isEmpty(request.getSorts())) {
|
||||
request.getSorts().forEach(sort -> {
|
||||
DatacenterTableField field = fieldMap.get(sort.getColumn());
|
||||
if (field == null || !isEnabled(field.getSortable())) {
|
||||
throw new BusinessException("字段不可排序: " + sort.getColumn());
|
||||
}
|
||||
});
|
||||
}
|
||||
if (request.getWhere() != null && !request.getWhere().isBlank()) {
|
||||
boolean allowLegacyWhere = "PROJECT_MYSQL".equals(source.getSourceType())
|
||||
|| "MYSQL".equals(source.getSourceType())
|
||||
|| "POSTGRESQL".equals(source.getSourceType());
|
||||
if (!allowLegacyWhere) {
|
||||
throw new BusinessException("当前数据源仅支持结构化 DSL 查询");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private boolean isEnabled(Integer value) {
|
||||
return value == null || value == 1;
|
||||
}
|
||||
|
||||
private String trimToNull(String value) {
|
||||
if (!StringUtils.hasText(value)) {
|
||||
return null;
|
||||
}
|
||||
return value.trim();
|
||||
}
|
||||
|
||||
private List<DatacenterDatasetVersion> listVersions(java.math.BigInteger tableId) {
|
||||
QueryWrapper wrapper = QueryWrapper.create();
|
||||
wrapper.eq(DatacenterDatasetVersion::getTableId, tableId);
|
||||
wrapper.orderBy("version_no desc");
|
||||
return datasetVersionMapper.selectListByQuery(wrapper);
|
||||
}
|
||||
|
||||
private List<DatacenterDerivedTable> listUpstream(java.math.BigInteger tableId) {
|
||||
QueryWrapper wrapper = QueryWrapper.create();
|
||||
wrapper.eq(DatacenterDerivedTable::getDerivedTableId, tableId);
|
||||
wrapper.orderBy("created desc");
|
||||
return derivedTableMapper.selectListByQuery(wrapper);
|
||||
}
|
||||
|
||||
private List<DatacenterDerivedTable> listDownstream(java.math.BigInteger tableId) {
|
||||
QueryWrapper wrapper = QueryWrapper.create();
|
||||
wrapper.eq(DatacenterDerivedTable::getSourceTableId, tableId);
|
||||
wrapper.orderBy("created desc");
|
||||
return derivedTableMapper.selectListByQuery(wrapper);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,48 @@
|
||||
package tech.easyflow.datacenter.execution.service.impl;
|
||||
|
||||
import com.alibaba.fastjson2.JSONObject;
|
||||
import org.springframework.stereotype.Service;
|
||||
import tech.easyflow.common.entity.LoginAccount;
|
||||
import tech.easyflow.common.web.exceptions.BusinessException;
|
||||
import tech.easyflow.datacenter.connector.DatacenterConnector;
|
||||
import tech.easyflow.datacenter.connector.DatacenterConnectorRegistry;
|
||||
import tech.easyflow.datacenter.entity.DatacenterTable;
|
||||
import tech.easyflow.datacenter.execution.model.DatasetRef;
|
||||
import tech.easyflow.datacenter.meta.entity.DatacenterSource;
|
||||
import tech.easyflow.datacenter.meta.service.DatacenterDatasetRegistryService;
|
||||
import tech.easyflow.datacenter.execution.service.DatacenterDatasetWriteService;
|
||||
|
||||
import javax.annotation.Resource;
|
||||
import java.math.BigInteger;
|
||||
|
||||
@Service
|
||||
public class DatacenterDatasetWriteServiceImpl implements DatacenterDatasetWriteService {
|
||||
|
||||
@Resource
|
||||
private DatacenterDatasetRegistryService registryService;
|
||||
@Resource
|
||||
private DatacenterConnectorRegistry connectorRegistry;
|
||||
|
||||
@Override
|
||||
public void saveRow(DatasetRef datasetRef, JSONObject data, LoginAccount account) {
|
||||
DatacenterTable table = resolveTable(datasetRef);
|
||||
DatacenterSource source = registryService.getSourceRequired(table.getSourceId());
|
||||
DatacenterConnector connector = connectorRegistry.getConnector(source.getSourceType());
|
||||
connector.saveRow(source, table, data, account);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void deleteRow(DatasetRef datasetRef, BigInteger id, LoginAccount account) {
|
||||
DatacenterTable table = resolveTable(datasetRef);
|
||||
DatacenterSource source = registryService.getSourceRequired(table.getSourceId());
|
||||
DatacenterConnector connector = connectorRegistry.getConnector(source.getSourceType());
|
||||
connector.deleteRow(source, table, id, account);
|
||||
}
|
||||
|
||||
private DatacenterTable resolveTable(DatasetRef datasetRef) {
|
||||
if (datasetRef == null || datasetRef.getTableId() == null) {
|
||||
throw new BusinessException("缺少 tableId");
|
||||
}
|
||||
return registryService.getTableWithFields(datasetRef.getTableId());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,7 @@
|
||||
package tech.easyflow.datacenter.integration;
|
||||
|
||||
import tech.easyflow.datacenter.execution.model.DatacenterQueryRequest;
|
||||
|
||||
public interface AssistantDatacenterBridge {
|
||||
AssistantDatacenterResult queryPage(DatacenterQueryRequest request);
|
||||
}
|
||||
@@ -0,0 +1,34 @@
|
||||
package tech.easyflow.datacenter.integration;
|
||||
|
||||
import com.mybatisflex.core.row.Row;
|
||||
import tech.easyflow.datacenter.entity.DatacenterTable;
|
||||
import tech.easyflow.datacenter.meta.entity.DatacenterCatalog;
|
||||
import tech.easyflow.datacenter.meta.entity.DatacenterDatasetVersion;
|
||||
import tech.easyflow.datacenter.meta.entity.DatacenterSource;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
public class AssistantDatacenterResult {
|
||||
private List<Row> rows = new ArrayList<>();
|
||||
private DatacenterSource source;
|
||||
private DatacenterCatalog catalog;
|
||||
private DatacenterTable table;
|
||||
private DatacenterDatasetVersion version;
|
||||
private Map<String, Object> querySummary = new LinkedHashMap<>();
|
||||
|
||||
public List<Row> getRows() { return rows; }
|
||||
public void setRows(List<Row> rows) { this.rows = rows; }
|
||||
public DatacenterSource getSource() { return source; }
|
||||
public void setSource(DatacenterSource source) { this.source = source; }
|
||||
public DatacenterCatalog getCatalog() { return catalog; }
|
||||
public void setCatalog(DatacenterCatalog catalog) { this.catalog = catalog; }
|
||||
public DatacenterTable getTable() { return table; }
|
||||
public void setTable(DatacenterTable table) { this.table = table; }
|
||||
public DatacenterDatasetVersion getVersion() { return version; }
|
||||
public void setVersion(DatacenterDatasetVersion version) { this.version = version; }
|
||||
public Map<String, Object> getQuerySummary() { return querySummary; }
|
||||
public void setQuerySummary(Map<String, Object> querySummary) { this.querySummary = querySummary; }
|
||||
}
|
||||
@@ -0,0 +1,49 @@
|
||||
package tech.easyflow.datacenter.integration;
|
||||
|
||||
import org.springframework.stereotype.Component;
|
||||
import tech.easyflow.datacenter.execution.model.DatacenterQueryRequest;
|
||||
import tech.easyflow.datacenter.execution.model.DatacenterSchemaResponse;
|
||||
import tech.easyflow.datacenter.execution.service.DatacenterDatasetQueryService;
|
||||
import tech.easyflow.datacenter.meta.entity.DatacenterDatasetVersion;
|
||||
|
||||
import javax.annotation.Resource;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
|
||||
@Component
|
||||
public class DefaultAssistantDatacenterBridge implements AssistantDatacenterBridge {
|
||||
|
||||
@Resource
|
||||
private DatacenterDatasetQueryService queryService;
|
||||
|
||||
@Override
|
||||
public AssistantDatacenterResult queryPage(DatacenterQueryRequest request) {
|
||||
var page = queryService.queryPage(request);
|
||||
DatacenterSchemaResponse schema = queryService.getSchema(request.getDatasetRef());
|
||||
AssistantDatacenterResult result = new AssistantDatacenterResult();
|
||||
result.setRows(page.getRecords());
|
||||
result.setSource(schema.getSource());
|
||||
result.setCatalog(schema.getCatalog());
|
||||
result.setTable(schema.getTable());
|
||||
result.setVersion(resolveVersion(schema.getVersions(), request == null || request.getDatasetRef() == null ? null : request.getDatasetRef().getVersionId()));
|
||||
result.setQuerySummary(new LinkedHashMap<>() {{
|
||||
put("pageNumber", page.getPageNumber());
|
||||
put("pageSize", page.getPageSize());
|
||||
put("totalRows", page.getTotalRow());
|
||||
put("selectedColumns", request == null ? List.of() : request.getSelectedColumns());
|
||||
put("filterCount", request == null || request.getFilters() == null ? 0 : request.getFilters().size());
|
||||
put("sortCount", request == null || request.getSorts() == null ? 0 : request.getSorts().size());
|
||||
}});
|
||||
return result;
|
||||
}
|
||||
|
||||
private DatacenterDatasetVersion resolveVersion(List<DatacenterDatasetVersion> versions, java.math.BigInteger versionId) {
|
||||
if (versions == null || versions.isEmpty()) {
|
||||
return null;
|
||||
}
|
||||
if (versionId == null) {
|
||||
return versions.get(0);
|
||||
}
|
||||
return versions.stream().filter(version -> versionId.equals(version.getId())).findFirst().orElse(null);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,7 @@
|
||||
package tech.easyflow.datacenter.mapper;
|
||||
|
||||
import com.mybatisflex.core.BaseMapper;
|
||||
import tech.easyflow.datacenter.meta.entity.DatacenterCatalog;
|
||||
|
||||
public interface DatacenterCatalogMapper extends BaseMapper<DatacenterCatalog> {
|
||||
}
|
||||
@@ -0,0 +1,7 @@
|
||||
package tech.easyflow.datacenter.mapper;
|
||||
|
||||
import com.mybatisflex.core.BaseMapper;
|
||||
import tech.easyflow.datacenter.meta.entity.DatacenterDatasetVersion;
|
||||
|
||||
public interface DatacenterDatasetVersionMapper extends BaseMapper<DatacenterDatasetVersion> {
|
||||
}
|
||||
@@ -0,0 +1,7 @@
|
||||
package tech.easyflow.datacenter.mapper;
|
||||
|
||||
import com.mybatisflex.core.BaseMapper;
|
||||
import tech.easyflow.datacenter.meta.entity.DatacenterDerivedTable;
|
||||
|
||||
public interface DatacenterDerivedTableMapper extends BaseMapper<DatacenterDerivedTable> {
|
||||
}
|
||||
@@ -0,0 +1,7 @@
|
||||
package tech.easyflow.datacenter.mapper;
|
||||
|
||||
import com.mybatisflex.core.BaseMapper;
|
||||
import tech.easyflow.datacenter.meta.entity.DatacenterImportJob;
|
||||
|
||||
public interface DatacenterImportJobMapper extends BaseMapper<DatacenterImportJob> {
|
||||
}
|
||||
@@ -0,0 +1,7 @@
|
||||
package tech.easyflow.datacenter.mapper;
|
||||
|
||||
import com.mybatisflex.core.BaseMapper;
|
||||
import tech.easyflow.datacenter.meta.entity.DatacenterSource;
|
||||
|
||||
public interface DatacenterSourceMapper extends BaseMapper<DatacenterSource> {
|
||||
}
|
||||
@@ -0,0 +1,75 @@
|
||||
package tech.easyflow.datacenter.meta.entity;
|
||||
|
||||
import com.mybatisflex.annotation.Column;
|
||||
import com.mybatisflex.annotation.Id;
|
||||
import com.mybatisflex.annotation.KeyType;
|
||||
import com.mybatisflex.annotation.Table;
|
||||
import com.mybatisflex.core.handler.FastjsonTypeHandler;
|
||||
import tech.easyflow.common.entity.DateEntity;
|
||||
|
||||
import java.io.Serializable;
|
||||
import java.math.BigInteger;
|
||||
import java.util.Date;
|
||||
import java.util.Map;
|
||||
|
||||
@Table(value = "tb_datacenter_catalog", comment = "数据中心逻辑库/命名空间")
|
||||
public class DatacenterCatalog extends DateEntity implements Serializable {
|
||||
|
||||
@Id(keyType = KeyType.Generator, value = "snowFlakeId", comment = "主键")
|
||||
private BigInteger id;
|
||||
@Column(comment = "部门ID")
|
||||
private BigInteger deptId;
|
||||
@Column(tenantId = true, comment = "租户ID")
|
||||
private BigInteger tenantId;
|
||||
@Column(comment = "数据源ID")
|
||||
private BigInteger sourceId;
|
||||
@Column(comment = "目录名")
|
||||
private String catalogName;
|
||||
@Column(comment = "目录描述")
|
||||
private String catalogDesc;
|
||||
@Column(comment = "目录类型")
|
||||
private String catalogType;
|
||||
@Column(comment = "状态")
|
||||
private Integer status;
|
||||
@Column(typeHandler = FastjsonTypeHandler.class, comment = "扩展项")
|
||||
private Map<String, Object> options;
|
||||
@Column(comment = "创建时间")
|
||||
private Date created;
|
||||
@Column(comment = "创建人")
|
||||
private BigInteger createdBy;
|
||||
@Column(comment = "修改时间")
|
||||
private Date modified;
|
||||
@Column(comment = "修改人")
|
||||
private BigInteger modifiedBy;
|
||||
|
||||
public BigInteger getId() { return id; }
|
||||
public void setId(BigInteger id) { this.id = id; }
|
||||
public BigInteger getDeptId() { return deptId; }
|
||||
public void setDeptId(BigInteger deptId) { this.deptId = deptId; }
|
||||
public BigInteger getTenantId() { return tenantId; }
|
||||
public void setTenantId(BigInteger tenantId) { this.tenantId = tenantId; }
|
||||
public BigInteger getSourceId() { return sourceId; }
|
||||
public void setSourceId(BigInteger sourceId) { this.sourceId = sourceId; }
|
||||
public String getCatalogName() { return catalogName; }
|
||||
public void setCatalogName(String catalogName) { this.catalogName = catalogName; }
|
||||
public String getCatalogDesc() { return catalogDesc; }
|
||||
public void setCatalogDesc(String catalogDesc) { this.catalogDesc = catalogDesc; }
|
||||
public String getCatalogType() { return catalogType; }
|
||||
public void setCatalogType(String catalogType) { this.catalogType = catalogType; }
|
||||
public Integer getStatus() { return status; }
|
||||
public void setStatus(Integer status) { this.status = status; }
|
||||
public Map<String, Object> getOptions() { return options; }
|
||||
public void setOptions(Map<String, Object> options) { this.options = options; }
|
||||
@Override
|
||||
public Date getCreated() { return created; }
|
||||
@Override
|
||||
public void setCreated(Date created) { this.created = created; }
|
||||
public BigInteger getCreatedBy() { return createdBy; }
|
||||
public void setCreatedBy(BigInteger createdBy) { this.createdBy = createdBy; }
|
||||
@Override
|
||||
public Date getModified() { return modified; }
|
||||
@Override
|
||||
public void setModified(Date modified) { this.modified = modified; }
|
||||
public BigInteger getModifiedBy() { return modifiedBy; }
|
||||
public void setModifiedBy(BigInteger modifiedBy) { this.modifiedBy = modifiedBy; }
|
||||
}
|
||||
@@ -0,0 +1,70 @@
|
||||
package tech.easyflow.datacenter.meta.entity;
|
||||
|
||||
import com.mybatisflex.annotation.Column;
|
||||
import com.mybatisflex.annotation.Id;
|
||||
import com.mybatisflex.annotation.KeyType;
|
||||
import com.mybatisflex.annotation.Table;
|
||||
import com.mybatisflex.core.handler.FastjsonTypeHandler;
|
||||
import tech.easyflow.common.entity.DateEntity;
|
||||
|
||||
import java.io.Serializable;
|
||||
import java.math.BigInteger;
|
||||
import java.util.Date;
|
||||
import java.util.Map;
|
||||
|
||||
@Table(value = "tb_datacenter_dataset_version", comment = "数据集版本")
|
||||
public class DatacenterDatasetVersion extends DateEntity implements Serializable {
|
||||
@Id(keyType = KeyType.Generator, value = "snowFlakeId")
|
||||
private BigInteger id;
|
||||
@Column(comment = "部门ID")
|
||||
private BigInteger deptId;
|
||||
@Column(tenantId = true, comment = "租户ID")
|
||||
private BigInteger tenantId;
|
||||
@Column(comment = "表ID")
|
||||
private BigInteger tableId;
|
||||
@Column(comment = "版本号")
|
||||
private Integer versionNo;
|
||||
@Column(comment = "版本标签")
|
||||
private String versionLabel;
|
||||
@Column(comment = "物化表名")
|
||||
private String materializedTable;
|
||||
@Column(typeHandler = FastjsonTypeHandler.class, comment = "版本快照")
|
||||
private Map<String, Object> snapshotJson;
|
||||
@Column(comment = "状态")
|
||||
private Integer status;
|
||||
@Column(comment = "创建时间")
|
||||
private Date created;
|
||||
@Column(comment = "创建人")
|
||||
private BigInteger createdBy;
|
||||
@Column(comment = "修改时间")
|
||||
private Date modified;
|
||||
@Column(comment = "修改人")
|
||||
private BigInteger modifiedBy;
|
||||
|
||||
public BigInteger getId() { return id; }
|
||||
public void setId(BigInteger id) { this.id = id; }
|
||||
public BigInteger getDeptId() { return deptId; }
|
||||
public void setDeptId(BigInteger deptId) { this.deptId = deptId; }
|
||||
public BigInteger getTenantId() { return tenantId; }
|
||||
public void setTenantId(BigInteger tenantId) { this.tenantId = tenantId; }
|
||||
public BigInteger getTableId() { return tableId; }
|
||||
public void setTableId(BigInteger tableId) { this.tableId = tableId; }
|
||||
public Integer getVersionNo() { return versionNo; }
|
||||
public void setVersionNo(Integer versionNo) { this.versionNo = versionNo; }
|
||||
public String getVersionLabel() { return versionLabel; }
|
||||
public void setVersionLabel(String versionLabel) { this.versionLabel = versionLabel; }
|
||||
public String getMaterializedTable() { return materializedTable; }
|
||||
public void setMaterializedTable(String materializedTable) { this.materializedTable = materializedTable; }
|
||||
public Map<String, Object> getSnapshotJson() { return snapshotJson; }
|
||||
public void setSnapshotJson(Map<String, Object> snapshotJson) { this.snapshotJson = snapshotJson; }
|
||||
public Integer getStatus() { return status; }
|
||||
public void setStatus(Integer status) { this.status = status; }
|
||||
@Override public Date getCreated() { return created; }
|
||||
@Override public void setCreated(Date created) { this.created = created; }
|
||||
public BigInteger getCreatedBy() { return createdBy; }
|
||||
public void setCreatedBy(BigInteger createdBy) { this.createdBy = createdBy; }
|
||||
@Override public Date getModified() { return modified; }
|
||||
@Override public void setModified(Date modified) { this.modified = modified; }
|
||||
public BigInteger getModifiedBy() { return modifiedBy; }
|
||||
public void setModifiedBy(BigInteger modifiedBy) { this.modifiedBy = modifiedBy; }
|
||||
}
|
||||
@@ -0,0 +1,66 @@
|
||||
package tech.easyflow.datacenter.meta.entity;
|
||||
|
||||
import com.mybatisflex.annotation.Column;
|
||||
import com.mybatisflex.annotation.Id;
|
||||
import com.mybatisflex.annotation.KeyType;
|
||||
import com.mybatisflex.annotation.Table;
|
||||
import com.mybatisflex.core.handler.FastjsonTypeHandler;
|
||||
import tech.easyflow.common.entity.DateEntity;
|
||||
|
||||
import java.io.Serializable;
|
||||
import java.math.BigInteger;
|
||||
import java.util.Date;
|
||||
import java.util.Map;
|
||||
|
||||
@Table(value = "tb_datacenter_derived_table", comment = "数据中心派生表关系")
|
||||
public class DatacenterDerivedTable extends DateEntity implements Serializable {
|
||||
@Id(keyType = KeyType.Generator, value = "snowFlakeId")
|
||||
private BigInteger id;
|
||||
@Column(comment = "部门ID")
|
||||
private BigInteger deptId;
|
||||
@Column(tenantId = true, comment = "租户ID")
|
||||
private BigInteger tenantId;
|
||||
@Column(comment = "源表ID")
|
||||
private BigInteger sourceTableId;
|
||||
@Column(comment = "派生表ID")
|
||||
private BigInteger derivedTableId;
|
||||
@Column(comment = "派生类型")
|
||||
private String deriveType;
|
||||
@Column(typeHandler = FastjsonTypeHandler.class, comment = "派生配置")
|
||||
private Map<String, Object> deriveConfigJson;
|
||||
@Column(comment = "状态")
|
||||
private Integer status;
|
||||
@Column(comment = "创建时间")
|
||||
private Date created;
|
||||
@Column(comment = "创建人")
|
||||
private BigInteger createdBy;
|
||||
@Column(comment = "修改时间")
|
||||
private Date modified;
|
||||
@Column(comment = "修改人")
|
||||
private BigInteger modifiedBy;
|
||||
|
||||
public BigInteger getId() { return id; }
|
||||
public void setId(BigInteger id) { this.id = id; }
|
||||
public BigInteger getDeptId() { return deptId; }
|
||||
public void setDeptId(BigInteger deptId) { this.deptId = deptId; }
|
||||
public BigInteger getTenantId() { return tenantId; }
|
||||
public void setTenantId(BigInteger tenantId) { this.tenantId = tenantId; }
|
||||
public BigInteger getSourceTableId() { return sourceTableId; }
|
||||
public void setSourceTableId(BigInteger sourceTableId) { this.sourceTableId = sourceTableId; }
|
||||
public BigInteger getDerivedTableId() { return derivedTableId; }
|
||||
public void setDerivedTableId(BigInteger derivedTableId) { this.derivedTableId = derivedTableId; }
|
||||
public String getDeriveType() { return deriveType; }
|
||||
public void setDeriveType(String deriveType) { this.deriveType = deriveType; }
|
||||
public Map<String, Object> getDeriveConfigJson() { return deriveConfigJson; }
|
||||
public void setDeriveConfigJson(Map<String, Object> deriveConfigJson) { this.deriveConfigJson = deriveConfigJson; }
|
||||
public Integer getStatus() { return status; }
|
||||
public void setStatus(Integer status) { this.status = status; }
|
||||
@Override public Date getCreated() { return created; }
|
||||
@Override public void setCreated(Date created) { this.created = created; }
|
||||
public BigInteger getCreatedBy() { return createdBy; }
|
||||
public void setCreatedBy(BigInteger createdBy) { this.createdBy = createdBy; }
|
||||
@Override public Date getModified() { return modified; }
|
||||
@Override public void setModified(Date modified) { this.modified = modified; }
|
||||
public BigInteger getModifiedBy() { return modifiedBy; }
|
||||
public void setModifiedBy(BigInteger modifiedBy) { this.modifiedBy = modifiedBy; }
|
||||
}
|
||||
@@ -0,0 +1,102 @@
|
||||
package tech.easyflow.datacenter.meta.entity;
|
||||
|
||||
import com.mybatisflex.annotation.Column;
|
||||
import com.mybatisflex.annotation.Id;
|
||||
import com.mybatisflex.annotation.KeyType;
|
||||
import com.mybatisflex.annotation.Table;
|
||||
import com.mybatisflex.core.handler.FastjsonTypeHandler;
|
||||
import tech.easyflow.common.entity.DateEntity;
|
||||
|
||||
import java.io.Serializable;
|
||||
import java.math.BigInteger;
|
||||
import java.util.Date;
|
||||
import java.util.Map;
|
||||
|
||||
@Table(value = "tb_datacenter_import_job", comment = "数据中心导入任务")
|
||||
public class DatacenterImportJob extends DateEntity implements Serializable {
|
||||
@Id(keyType = KeyType.Generator, value = "snowFlakeId")
|
||||
private BigInteger id;
|
||||
@Column(comment = "部门ID")
|
||||
private BigInteger deptId;
|
||||
@Column(tenantId = true, comment = "租户ID")
|
||||
private BigInteger tenantId;
|
||||
@Column(comment = "数据源ID")
|
||||
private BigInteger sourceId;
|
||||
@Column(comment = "目录ID")
|
||||
private BigInteger catalogId;
|
||||
@Column(comment = "表ID")
|
||||
private BigInteger tableId;
|
||||
@Column(comment = "任务类型")
|
||||
private String jobType;
|
||||
@Column(comment = "文件名")
|
||||
private String fileName;
|
||||
@Column(comment = "文件存储路径")
|
||||
private String storagePath;
|
||||
@Column(comment = "任务状态")
|
||||
private String status;
|
||||
@Column(comment = "总行数")
|
||||
private Long totalRows;
|
||||
@Column(comment = "成功行数")
|
||||
private Long successRows;
|
||||
@Column(comment = "失败行数")
|
||||
private Long errorRows;
|
||||
@Column(comment = "错误摘要")
|
||||
private String errorSummary;
|
||||
@Column(typeHandler = FastjsonTypeHandler.class, comment = "任务载荷")
|
||||
private Map<String, Object> payloadJson;
|
||||
@Column(comment = "开始时间")
|
||||
private Date startedAt;
|
||||
@Column(comment = "结束时间")
|
||||
private Date finishedAt;
|
||||
@Column(comment = "创建时间")
|
||||
private Date created;
|
||||
@Column(comment = "创建人")
|
||||
private BigInteger createdBy;
|
||||
@Column(comment = "修改时间")
|
||||
private Date modified;
|
||||
@Column(comment = "修改人")
|
||||
private BigInteger modifiedBy;
|
||||
|
||||
public BigInteger getId() { return id; }
|
||||
public void setId(BigInteger id) { this.id = id; }
|
||||
public BigInteger getDeptId() { return deptId; }
|
||||
public void setDeptId(BigInteger deptId) { this.deptId = deptId; }
|
||||
public BigInteger getTenantId() { return tenantId; }
|
||||
public void setTenantId(BigInteger tenantId) { this.tenantId = tenantId; }
|
||||
public BigInteger getSourceId() { return sourceId; }
|
||||
public void setSourceId(BigInteger sourceId) { this.sourceId = sourceId; }
|
||||
public BigInteger getCatalogId() { return catalogId; }
|
||||
public void setCatalogId(BigInteger catalogId) { this.catalogId = catalogId; }
|
||||
public BigInteger getTableId() { return tableId; }
|
||||
public void setTableId(BigInteger tableId) { this.tableId = tableId; }
|
||||
public String getJobType() { return jobType; }
|
||||
public void setJobType(String jobType) { this.jobType = jobType; }
|
||||
public String getFileName() { return fileName; }
|
||||
public void setFileName(String fileName) { this.fileName = fileName; }
|
||||
public String getStoragePath() { return storagePath; }
|
||||
public void setStoragePath(String storagePath) { this.storagePath = storagePath; }
|
||||
public String getStatus() { return status; }
|
||||
public void setStatus(String status) { this.status = status; }
|
||||
public Long getTotalRows() { return totalRows; }
|
||||
public void setTotalRows(Long totalRows) { this.totalRows = totalRows; }
|
||||
public Long getSuccessRows() { return successRows; }
|
||||
public void setSuccessRows(Long successRows) { this.successRows = successRows; }
|
||||
public Long getErrorRows() { return errorRows; }
|
||||
public void setErrorRows(Long errorRows) { this.errorRows = errorRows; }
|
||||
public String getErrorSummary() { return errorSummary; }
|
||||
public void setErrorSummary(String errorSummary) { this.errorSummary = errorSummary; }
|
||||
public Map<String, Object> getPayloadJson() { return payloadJson; }
|
||||
public void setPayloadJson(Map<String, Object> payloadJson) { this.payloadJson = payloadJson; }
|
||||
public Date getStartedAt() { return startedAt; }
|
||||
public void setStartedAt(Date startedAt) { this.startedAt = startedAt; }
|
||||
public Date getFinishedAt() { return finishedAt; }
|
||||
public void setFinishedAt(Date finishedAt) { this.finishedAt = finishedAt; }
|
||||
@Override public Date getCreated() { return created; }
|
||||
@Override public void setCreated(Date created) { this.created = created; }
|
||||
public BigInteger getCreatedBy() { return createdBy; }
|
||||
public void setCreatedBy(BigInteger createdBy) { this.createdBy = createdBy; }
|
||||
@Override public Date getModified() { return modified; }
|
||||
@Override public void setModified(Date modified) { this.modified = modified; }
|
||||
public BigInteger getModifiedBy() { return modifiedBy; }
|
||||
public void setModifiedBy(BigInteger modifiedBy) { this.modifiedBy = modifiedBy; }
|
||||
}
|
||||
@@ -0,0 +1,131 @@
|
||||
package tech.easyflow.datacenter.meta.entity;
|
||||
|
||||
import com.mybatisflex.annotation.Column;
|
||||
import com.mybatisflex.annotation.Id;
|
||||
import com.mybatisflex.annotation.KeyType;
|
||||
import com.mybatisflex.annotation.Table;
|
||||
import com.mybatisflex.core.handler.FastjsonTypeHandler;
|
||||
import tech.easyflow.common.entity.DateEntity;
|
||||
|
||||
import java.io.Serializable;
|
||||
import java.math.BigInteger;
|
||||
import java.util.Date;
|
||||
import java.util.Map;
|
||||
|
||||
@Table(value = "tb_datacenter_source", comment = "数据中心数据源")
|
||||
public class DatacenterSource extends DateEntity implements Serializable {
|
||||
|
||||
@Id(keyType = KeyType.Generator, value = "snowFlakeId", comment = "主键")
|
||||
private BigInteger id;
|
||||
@Column(comment = "部门ID")
|
||||
private BigInteger deptId;
|
||||
@Column(tenantId = true, comment = "租户ID")
|
||||
private BigInteger tenantId;
|
||||
@Column(comment = "数据源名称")
|
||||
private String sourceName;
|
||||
@Column(comment = "数据源编码")
|
||||
private String sourceCode;
|
||||
@Column(comment = "数据源类型")
|
||||
private String sourceType;
|
||||
@Column(comment = "访问模式")
|
||||
private String accessMode;
|
||||
@Column(comment = "是否内置")
|
||||
private Integer builtinFlag;
|
||||
@Column(comment = "驱动类名")
|
||||
private String driverClassName;
|
||||
@Column(comment = "JDBC URL")
|
||||
private String jdbcUrl;
|
||||
@Column(comment = "主机")
|
||||
private String host;
|
||||
@Column(comment = "端口")
|
||||
private Integer port;
|
||||
@Column(comment = "数据库名")
|
||||
private String databaseName;
|
||||
@Column(comment = "Schema名")
|
||||
private String schemaName;
|
||||
@Column(comment = "用户名")
|
||||
private String username;
|
||||
@Column(comment = "凭据密文")
|
||||
private String credentialCipher;
|
||||
@Column(typeHandler = FastjsonTypeHandler.class, comment = "连接配置")
|
||||
private Map<String, Object> configJson;
|
||||
@Column(typeHandler = FastjsonTypeHandler.class, comment = "能力声明")
|
||||
private Map<String, Object> capabilitiesJson;
|
||||
@Column(comment = "最近测试状态")
|
||||
private String lastTestStatus;
|
||||
@Column(comment = "最近测试信息")
|
||||
private String lastTestMessage;
|
||||
@Column(comment = "最近测试时间")
|
||||
private Date lastTestedAt;
|
||||
@Column(comment = "状态")
|
||||
private Integer status;
|
||||
@Column(typeHandler = FastjsonTypeHandler.class, comment = "扩展项")
|
||||
private Map<String, Object> options;
|
||||
@Column(comment = "创建时间")
|
||||
private Date created;
|
||||
@Column(comment = "创建人")
|
||||
private BigInteger createdBy;
|
||||
@Column(comment = "修改时间")
|
||||
private Date modified;
|
||||
@Column(comment = "修改人")
|
||||
private BigInteger modifiedBy;
|
||||
|
||||
public BigInteger getId() { return id; }
|
||||
public void setId(BigInteger id) { this.id = id; }
|
||||
public BigInteger getDeptId() { return deptId; }
|
||||
public void setDeptId(BigInteger deptId) { this.deptId = deptId; }
|
||||
public BigInteger getTenantId() { return tenantId; }
|
||||
public void setTenantId(BigInteger tenantId) { this.tenantId = tenantId; }
|
||||
public String getSourceName() { return sourceName; }
|
||||
public void setSourceName(String sourceName) { this.sourceName = sourceName; }
|
||||
public String getSourceCode() { return sourceCode; }
|
||||
public void setSourceCode(String sourceCode) { this.sourceCode = sourceCode; }
|
||||
public String getSourceType() { return sourceType; }
|
||||
public void setSourceType(String sourceType) { this.sourceType = sourceType; }
|
||||
public String getAccessMode() { return accessMode; }
|
||||
public void setAccessMode(String accessMode) { this.accessMode = accessMode; }
|
||||
public Integer getBuiltinFlag() { return builtinFlag; }
|
||||
public void setBuiltinFlag(Integer builtinFlag) { this.builtinFlag = builtinFlag; }
|
||||
public String getDriverClassName() { return driverClassName; }
|
||||
public void setDriverClassName(String driverClassName) { this.driverClassName = driverClassName; }
|
||||
public String getJdbcUrl() { return jdbcUrl; }
|
||||
public void setJdbcUrl(String jdbcUrl) { this.jdbcUrl = jdbcUrl; }
|
||||
public String getHost() { return host; }
|
||||
public void setHost(String host) { this.host = host; }
|
||||
public Integer getPort() { return port; }
|
||||
public void setPort(Integer port) { this.port = port; }
|
||||
public String getDatabaseName() { return databaseName; }
|
||||
public void setDatabaseName(String databaseName) { this.databaseName = databaseName; }
|
||||
public String getSchemaName() { return schemaName; }
|
||||
public void setSchemaName(String schemaName) { this.schemaName = schemaName; }
|
||||
public String getUsername() { return username; }
|
||||
public void setUsername(String username) { this.username = username; }
|
||||
public String getCredentialCipher() { return credentialCipher; }
|
||||
public void setCredentialCipher(String credentialCipher) { this.credentialCipher = credentialCipher; }
|
||||
public Map<String, Object> getConfigJson() { return configJson; }
|
||||
public void setConfigJson(Map<String, Object> configJson) { this.configJson = configJson; }
|
||||
public Map<String, Object> getCapabilitiesJson() { return capabilitiesJson; }
|
||||
public void setCapabilitiesJson(Map<String, Object> capabilitiesJson) { this.capabilitiesJson = capabilitiesJson; }
|
||||
public String getLastTestStatus() { return lastTestStatus; }
|
||||
public void setLastTestStatus(String lastTestStatus) { this.lastTestStatus = lastTestStatus; }
|
||||
public String getLastTestMessage() { return lastTestMessage; }
|
||||
public void setLastTestMessage(String lastTestMessage) { this.lastTestMessage = lastTestMessage; }
|
||||
public Date getLastTestedAt() { return lastTestedAt; }
|
||||
public void setLastTestedAt(Date lastTestedAt) { this.lastTestedAt = lastTestedAt; }
|
||||
public Integer getStatus() { return status; }
|
||||
public void setStatus(Integer status) { this.status = status; }
|
||||
public Map<String, Object> getOptions() { return options; }
|
||||
public void setOptions(Map<String, Object> options) { this.options = options; }
|
||||
@Override
|
||||
public Date getCreated() { return created; }
|
||||
@Override
|
||||
public void setCreated(Date created) { this.created = created; }
|
||||
public BigInteger getCreatedBy() { return createdBy; }
|
||||
public void setCreatedBy(BigInteger createdBy) { this.createdBy = createdBy; }
|
||||
@Override
|
||||
public Date getModified() { return modified; }
|
||||
@Override
|
||||
public void setModified(Date modified) { this.modified = modified; }
|
||||
public BigInteger getModifiedBy() { return modifiedBy; }
|
||||
public void setModifiedBy(BigInteger modifiedBy) { this.modifiedBy = modifiedBy; }
|
||||
}
|
||||
@@ -0,0 +1,6 @@
|
||||
package tech.easyflow.datacenter.meta.enums;
|
||||
|
||||
public enum DatacenterAccessMode {
|
||||
READ_ONLY,
|
||||
READ_WRITE
|
||||
}
|
||||
@@ -0,0 +1,10 @@
|
||||
package tech.easyflow.datacenter.meta.enums;
|
||||
|
||||
public enum DatacenterCapability {
|
||||
TEST_CONNECTION,
|
||||
BROWSE_METADATA,
|
||||
READ_QUERY,
|
||||
WRITE_MUTATION,
|
||||
EXPORT,
|
||||
MATERIALIZE
|
||||
}
|
||||
@@ -0,0 +1,12 @@
|
||||
package tech.easyflow.datacenter.meta.enums;
|
||||
|
||||
public enum DatacenterConnectionErrorCode {
|
||||
INVALID_ARGUMENT,
|
||||
DRIVER_NOT_FOUND,
|
||||
NETWORK_UNREACHABLE,
|
||||
AUTH_FAILED,
|
||||
DATABASE_NOT_FOUND,
|
||||
SCHEMA_NOT_FOUND,
|
||||
PERMISSION_DENIED,
|
||||
UNKNOWN_ERROR
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
package tech.easyflow.datacenter.meta.enums;
|
||||
|
||||
public enum DatacenterImportStatus {
|
||||
PENDING,
|
||||
RUNNING,
|
||||
SUCCESS,
|
||||
FAILED,
|
||||
NOT_IMPLEMENTED
|
||||
}
|
||||
@@ -0,0 +1,13 @@
|
||||
package tech.easyflow.datacenter.meta.enums;
|
||||
|
||||
public enum DatacenterSourceType {
|
||||
PROJECT_MYSQL,
|
||||
EXCEL,
|
||||
EXCEL_MATERIALIZED,
|
||||
MYSQL,
|
||||
POSTGRESQL,
|
||||
ORACLE,
|
||||
GAUSSDB_NATIVE,
|
||||
GBASE_8A,
|
||||
GBASE_8S
|
||||
}
|
||||
@@ -0,0 +1,10 @@
|
||||
package tech.easyflow.datacenter.meta.enums;
|
||||
|
||||
public enum DatacenterTableKind {
|
||||
PROJECT_MANAGED,
|
||||
EXTERNAL_TABLE,
|
||||
EXTERNAL_VIEW,
|
||||
EXCEL_SHEET,
|
||||
EXCEL_MATERIALIZED,
|
||||
DERIVED_TABLE
|
||||
}
|
||||
@@ -0,0 +1,36 @@
|
||||
package tech.easyflow.datacenter.meta.model;
|
||||
|
||||
import java.math.BigInteger;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
public class DatacenterBatchRegisterRequest {
|
||||
|
||||
private BigInteger sourceId;
|
||||
private String catalogName;
|
||||
private List<String> tableNames = new ArrayList<>();
|
||||
|
||||
public BigInteger getSourceId() {
|
||||
return sourceId;
|
||||
}
|
||||
|
||||
public void setSourceId(BigInteger sourceId) {
|
||||
this.sourceId = sourceId;
|
||||
}
|
||||
|
||||
public String getCatalogName() {
|
||||
return catalogName;
|
||||
}
|
||||
|
||||
public void setCatalogName(String catalogName) {
|
||||
this.catalogName = catalogName;
|
||||
}
|
||||
|
||||
public List<String> getTableNames() {
|
||||
return tableNames;
|
||||
}
|
||||
|
||||
public void setTableNames(List<String> tableNames) {
|
||||
this.tableNames = tableNames;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
package tech.easyflow.datacenter.meta.model;
|
||||
|
||||
import java.math.BigInteger;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
public class DatacenterBatchRemoveRequest {
|
||||
|
||||
private List<BigInteger> tableIds = new ArrayList<>();
|
||||
|
||||
public List<BigInteger> getTableIds() {
|
||||
return tableIds;
|
||||
}
|
||||
|
||||
public void setTableIds(List<BigInteger> tableIds) {
|
||||
this.tableIds = tableIds;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,22 @@
|
||||
package tech.easyflow.datacenter.meta.model;
|
||||
|
||||
import java.math.BigInteger;
|
||||
|
||||
public class DatacenterCatalogMeta {
|
||||
private BigInteger id;
|
||||
private BigInteger sourceId;
|
||||
private String catalogName;
|
||||
private String catalogType;
|
||||
private String catalogDesc;
|
||||
|
||||
public BigInteger getId() { return id; }
|
||||
public void setId(BigInteger id) { this.id = id; }
|
||||
public BigInteger getSourceId() { return sourceId; }
|
||||
public void setSourceId(BigInteger sourceId) { this.sourceId = sourceId; }
|
||||
public String getCatalogName() { return catalogName; }
|
||||
public void setCatalogName(String catalogName) { this.catalogName = catalogName; }
|
||||
public String getCatalogType() { return catalogType; }
|
||||
public void setCatalogType(String catalogType) { this.catalogType = catalogType; }
|
||||
public String getCatalogDesc() { return catalogDesc; }
|
||||
public void setCatalogDesc(String catalogDesc) { this.catalogDesc = catalogDesc; }
|
||||
}
|
||||
@@ -0,0 +1,25 @@
|
||||
package tech.easyflow.datacenter.meta.model;
|
||||
|
||||
import java.math.BigInteger;
|
||||
|
||||
public class DatacenterFieldDescriptionUpdate {
|
||||
|
||||
private BigInteger fieldId;
|
||||
private String fieldDesc;
|
||||
|
||||
public BigInteger getFieldId() {
|
||||
return fieldId;
|
||||
}
|
||||
|
||||
public void setFieldId(BigInteger fieldId) {
|
||||
this.fieldId = fieldId;
|
||||
}
|
||||
|
||||
public String getFieldDesc() {
|
||||
return fieldDesc;
|
||||
}
|
||||
|
||||
public void setFieldDesc(String fieldDesc) {
|
||||
this.fieldDesc = fieldDesc;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,16 @@
|
||||
package tech.easyflow.datacenter.meta.model;
|
||||
|
||||
import java.math.BigInteger;
|
||||
|
||||
public class DatacenterRemoveSourceRequest {
|
||||
|
||||
private BigInteger sourceId;
|
||||
|
||||
public BigInteger getSourceId() {
|
||||
return sourceId;
|
||||
}
|
||||
|
||||
public void setSourceId(BigInteger sourceId) {
|
||||
this.sourceId = sourceId;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,36 @@
|
||||
package tech.easyflow.datacenter.meta.model;
|
||||
|
||||
import java.math.BigInteger;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
public class DatacenterSaveDescriptionsRequest {
|
||||
|
||||
private BigInteger tableId;
|
||||
private String tableDesc;
|
||||
private List<DatacenterFieldDescriptionUpdate> fields = new ArrayList<>();
|
||||
|
||||
public BigInteger getTableId() {
|
||||
return tableId;
|
||||
}
|
||||
|
||||
public void setTableId(BigInteger tableId) {
|
||||
this.tableId = tableId;
|
||||
}
|
||||
|
||||
public String getTableDesc() {
|
||||
return tableDesc;
|
||||
}
|
||||
|
||||
public void setTableDesc(String tableDesc) {
|
||||
this.tableDesc = tableDesc;
|
||||
}
|
||||
|
||||
public List<DatacenterFieldDescriptionUpdate> getFields() {
|
||||
return fields;
|
||||
}
|
||||
|
||||
public void setFields(List<DatacenterFieldDescriptionUpdate> fields) {
|
||||
this.fields = fields;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
package tech.easyflow.datacenter.meta.model;
|
||||
|
||||
import tech.easyflow.datacenter.entity.DatacenterTable;
|
||||
import tech.easyflow.datacenter.entity.DatacenterTableField;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
public class DatacenterTableDetailMeta {
|
||||
private DatacenterTable table;
|
||||
private List<DatacenterTableField> fields = new ArrayList<>();
|
||||
|
||||
public DatacenterTable getTable() { return table; }
|
||||
public void setTable(DatacenterTable table) { this.table = table; }
|
||||
public List<DatacenterTableField> getFields() { return fields; }
|
||||
public void setFields(List<DatacenterTableField> fields) { this.fields = fields; }
|
||||
}
|
||||
@@ -0,0 +1,38 @@
|
||||
package tech.easyflow.datacenter.meta.service;
|
||||
|
||||
import tech.easyflow.common.entity.LoginAccount;
|
||||
import tech.easyflow.datacenter.entity.DatacenterTable;
|
||||
import tech.easyflow.datacenter.entity.DatacenterTableField;
|
||||
import tech.easyflow.datacenter.execution.model.DatasetRef;
|
||||
import tech.easyflow.datacenter.meta.entity.DatacenterCatalog;
|
||||
import tech.easyflow.datacenter.meta.entity.DatacenterSource;
|
||||
import tech.easyflow.datacenter.meta.model.DatacenterFieldDescriptionUpdate;
|
||||
import tech.easyflow.datacenter.meta.enums.DatacenterSourceType;
|
||||
import tech.easyflow.datacenter.meta.model.DatacenterTableDetailMeta;
|
||||
|
||||
import java.math.BigInteger;
|
||||
import java.util.List;
|
||||
|
||||
public interface DatacenterDatasetRegistryService {
|
||||
DatacenterSource ensureBuiltinSource(DatacenterSourceType sourceType, LoginAccount account);
|
||||
|
||||
DatacenterCatalog ensureCatalog(DatacenterSource source, String catalogName, LoginAccount account);
|
||||
|
||||
DatacenterTable registerTable(DatacenterSource source, DatacenterCatalog catalog, DatacenterTableDetailMeta detail, LoginAccount account);
|
||||
|
||||
DatacenterTable getTableWithFields(BigInteger tableId);
|
||||
|
||||
List<DatacenterTableField> getFields(BigInteger tableId);
|
||||
|
||||
DatasetRef resolveDatasetRef(BigInteger tableId);
|
||||
|
||||
DatacenterSource getSourceRequired(BigInteger sourceId);
|
||||
|
||||
DatacenterCatalog getCatalogById(BigInteger catalogId);
|
||||
|
||||
List<DatacenterTable> listManagedTables(BigInteger sourceId, BigInteger catalogId);
|
||||
|
||||
int removeTables(List<BigInteger> tableIds);
|
||||
|
||||
DatacenterTable saveDescriptions(BigInteger tableId, String tableDesc, List<DatacenterFieldDescriptionUpdate> fields, LoginAccount account);
|
||||
}
|
||||
@@ -0,0 +1,12 @@
|
||||
package tech.easyflow.datacenter.meta.service;
|
||||
|
||||
import java.math.BigInteger;
|
||||
|
||||
public final class DatacenterMetaConstants {
|
||||
|
||||
private DatacenterMetaConstants() {
|
||||
}
|
||||
|
||||
public static final BigInteger PROJECT_SOURCE_BASE = new BigInteger("9000000000002000000");
|
||||
public static final BigInteger PROJECT_CATALOG_BASE = new BigInteger("9000000000003000000");
|
||||
}
|
||||
@@ -0,0 +1,32 @@
|
||||
package tech.easyflow.datacenter.meta.service;
|
||||
|
||||
import com.mybatisflex.core.paginate.Page;
|
||||
import com.mybatisflex.core.service.IService;
|
||||
import tech.easyflow.common.entity.LoginAccount;
|
||||
import tech.easyflow.datacenter.entity.DatacenterTable;
|
||||
import tech.easyflow.datacenter.execution.model.DatacenterConnectionTestResult;
|
||||
import tech.easyflow.datacenter.meta.entity.DatacenterSource;
|
||||
import tech.easyflow.datacenter.meta.model.DatacenterBatchRegisterRequest;
|
||||
import tech.easyflow.datacenter.meta.model.DatacenterCatalogMeta;
|
||||
import tech.easyflow.datacenter.meta.model.DatacenterTableDetailMeta;
|
||||
|
||||
import java.math.BigInteger;
|
||||
import java.util.List;
|
||||
|
||||
public interface DatacenterSourceService extends IService<DatacenterSource> {
|
||||
DatacenterSource saveSource(DatacenterSource source, LoginAccount account);
|
||||
|
||||
Page<DatacenterSource> pageSources(Long pageNumber, Long pageSize, LoginAccount account);
|
||||
|
||||
DatacenterConnectionTestResult testConnection(DatacenterSource source, LoginAccount account);
|
||||
|
||||
List<DatacenterCatalogMeta> listCatalogs(BigInteger sourceId, LoginAccount account);
|
||||
|
||||
List<DatacenterTable> listTables(BigInteger sourceId, String catalogName, LoginAccount account);
|
||||
|
||||
DatacenterTableDetailMeta getTableDetail(BigInteger sourceId, String catalogName, String tableName, boolean register, LoginAccount account);
|
||||
|
||||
List<DatacenterTable> batchRegisterTables(DatacenterBatchRegisterRequest request, LoginAccount account);
|
||||
|
||||
void removeSource(BigInteger sourceId, LoginAccount account);
|
||||
}
|
||||
@@ -0,0 +1,405 @@
|
||||
package tech.easyflow.datacenter.meta.service.impl;
|
||||
|
||||
import com.mybatisflex.core.query.QueryWrapper;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.springframework.transaction.annotation.Transactional;
|
||||
import tech.easyflow.common.entity.LoginAccount;
|
||||
import tech.easyflow.common.web.exceptions.BusinessException;
|
||||
import tech.easyflow.datacenter.adapter.DbHandleManager;
|
||||
import tech.easyflow.datacenter.entity.DatacenterTable;
|
||||
import tech.easyflow.datacenter.entity.DatacenterTableField;
|
||||
import tech.easyflow.datacenter.execution.model.DatasetRef;
|
||||
import tech.easyflow.datacenter.mapper.DatacenterCatalogMapper;
|
||||
import tech.easyflow.datacenter.mapper.DatacenterDatasetVersionMapper;
|
||||
import tech.easyflow.datacenter.mapper.DatacenterDerivedTableMapper;
|
||||
import tech.easyflow.datacenter.mapper.DatacenterImportJobMapper;
|
||||
import tech.easyflow.datacenter.mapper.DatacenterSourceMapper;
|
||||
import tech.easyflow.datacenter.mapper.DatacenterTableFieldMapper;
|
||||
import tech.easyflow.datacenter.mapper.DatacenterTableMapper;
|
||||
import tech.easyflow.datacenter.meta.entity.DatacenterCatalog;
|
||||
import tech.easyflow.datacenter.meta.entity.DatacenterSource;
|
||||
import tech.easyflow.datacenter.meta.model.DatacenterFieldDescriptionUpdate;
|
||||
import tech.easyflow.datacenter.meta.enums.DatacenterSourceType;
|
||||
import tech.easyflow.datacenter.meta.service.DatacenterDatasetRegistryService;
|
||||
import tech.easyflow.datacenter.meta.service.DatacenterMetaConstants;
|
||||
import tech.easyflow.datacenter.meta.model.DatacenterTableDetailMeta;
|
||||
import tech.easyflow.datacenter.meta.enums.DatacenterTableKind;
|
||||
|
||||
import javax.annotation.Resource;
|
||||
import java.math.BigInteger;
|
||||
import java.util.Date;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
@Service
|
||||
public class DatacenterDatasetRegistryServiceImpl implements DatacenterDatasetRegistryService {
|
||||
|
||||
@Resource
|
||||
private DatacenterSourceMapper sourceMapper;
|
||||
@Resource
|
||||
private DatacenterCatalogMapper catalogMapper;
|
||||
@Resource
|
||||
private DatacenterTableMapper tableMapper;
|
||||
@Resource
|
||||
private DatacenterTableFieldMapper tableFieldMapper;
|
||||
@Resource
|
||||
private DatacenterDatasetVersionMapper datasetVersionMapper;
|
||||
@Resource
|
||||
private DatacenterDerivedTableMapper derivedTableMapper;
|
||||
@Resource
|
||||
private DatacenterImportJobMapper importJobMapper;
|
||||
@Resource
|
||||
private DbHandleManager dbHandleManager;
|
||||
|
||||
@Override
|
||||
public DatacenterSource ensureBuiltinSource(DatacenterSourceType sourceType, LoginAccount account) {
|
||||
BigInteger tenantId = account == null || account.getTenantId() == null ? BigInteger.ZERO : account.getTenantId();
|
||||
BigInteger deptId = account == null || account.getDeptId() == null ? BigInteger.ZERO : account.getDeptId();
|
||||
BigInteger id = builtinSourceId(sourceType, tenantId);
|
||||
DatacenterSource source = sourceMapper.selectOneById(id);
|
||||
if (source != null) {
|
||||
return source;
|
||||
}
|
||||
source = new DatacenterSource();
|
||||
source.setId(id);
|
||||
source.setTenantId(tenantId);
|
||||
source.setDeptId(deptId);
|
||||
source.setSourceType(sourceType.name());
|
||||
source.setBuiltinFlag(1);
|
||||
source.setStatus(0);
|
||||
source.setCreated(new Date());
|
||||
source.setModified(new Date());
|
||||
source.setCreatedBy(account == null ? BigInteger.ZERO : account.getId());
|
||||
source.setModifiedBy(account == null ? BigInteger.ZERO : account.getId());
|
||||
source.setCapabilitiesJson(Map.of("capabilities", builtinCapabilities(sourceType)));
|
||||
if (sourceType == DatacenterSourceType.PROJECT_MYSQL) {
|
||||
source.setSourceName("项目 MySQL");
|
||||
source.setSourceCode("PROJECT_MYSQL_" + tenantId);
|
||||
source.setAccessMode("READ_WRITE");
|
||||
} else {
|
||||
throw new BusinessException("不支持的内置数据源类型: " + sourceType);
|
||||
}
|
||||
sourceMapper.insert(source);
|
||||
ensureCatalog(source, defaultCatalogName(sourceType), account);
|
||||
return source;
|
||||
}
|
||||
|
||||
@Override
|
||||
public DatacenterCatalog ensureCatalog(DatacenterSource source, String catalogName, LoginAccount account) {
|
||||
QueryWrapper wrapper = QueryWrapper.create();
|
||||
wrapper.eq(DatacenterCatalog::getSourceId, source.getId());
|
||||
wrapper.eq(DatacenterCatalog::getCatalogName, catalogName);
|
||||
DatacenterCatalog catalog = catalogMapper.selectOneByQuery(wrapper);
|
||||
if (catalog != null) {
|
||||
return catalog;
|
||||
}
|
||||
catalog = new DatacenterCatalog();
|
||||
BigInteger catalogId = builtinCatalogId(source, catalogName);
|
||||
if (catalogId != null) {
|
||||
catalog.setId(catalogId);
|
||||
}
|
||||
catalog.setSourceId(source.getId());
|
||||
catalog.setTenantId(source.getTenantId());
|
||||
catalog.setDeptId(source.getDeptId());
|
||||
catalog.setCatalogName(catalogName);
|
||||
catalog.setCatalogDesc(catalogName);
|
||||
catalog.setCatalogType("DATABASE");
|
||||
catalog.setStatus(0);
|
||||
catalog.setCreated(new Date());
|
||||
catalog.setModified(new Date());
|
||||
catalog.setCreatedBy(account == null ? BigInteger.ZERO : account.getId());
|
||||
catalog.setModifiedBy(account == null ? BigInteger.ZERO : account.getId());
|
||||
catalogMapper.insert(catalog);
|
||||
return catalog;
|
||||
}
|
||||
|
||||
@Override
|
||||
public DatacenterTable registerTable(DatacenterSource source, DatacenterCatalog catalog, DatacenterTableDetailMeta detail, LoginAccount account) {
|
||||
DatacenterTable table = detail.getTable();
|
||||
applyTableDefaults(table, source, detail);
|
||||
QueryWrapper wrapper = QueryWrapper.create();
|
||||
wrapper.eq(DatacenterTable::getSourceId, source.getId());
|
||||
wrapper.eq(DatacenterTable::getCatalogId, catalog.getId());
|
||||
wrapper.eq(DatacenterTable::getTableName, table.getTableName());
|
||||
DatacenterTable existing = tableMapper.selectOneByQuery(wrapper);
|
||||
Date now = new Date();
|
||||
if (existing == null) {
|
||||
table.setSourceId(source.getId());
|
||||
table.setCatalogId(catalog.getId());
|
||||
table.setTenantId(source.getTenantId());
|
||||
table.setDeptId(source.getDeptId());
|
||||
table.setStatus(0);
|
||||
table.setCreated(now);
|
||||
table.setModified(now);
|
||||
table.setCreatedBy(account == null ? BigInteger.ZERO : account.getId());
|
||||
table.setModifiedBy(account == null ? BigInteger.ZERO : account.getId());
|
||||
tableMapper.insert(table);
|
||||
} else {
|
||||
if (!hasText(existing.getTableDesc())) {
|
||||
existing.setTableDesc(normalizeDescription(table.getTableDesc()));
|
||||
}
|
||||
existing.setActualTable(table.getActualTable());
|
||||
existing.setMaterializedTable(table.getMaterializedTable());
|
||||
existing.setTableKind(table.getTableKind());
|
||||
existing.setAccessMode(table.getAccessMode());
|
||||
existing.setVersioningEnabled(table.getVersioningEnabled());
|
||||
existing.setCapabilitiesJson(table.getCapabilitiesJson());
|
||||
existing.setModified(now);
|
||||
existing.setModifiedBy(account == null ? BigInteger.ZERO : account.getId());
|
||||
tableMapper.update(existing);
|
||||
table = existing;
|
||||
}
|
||||
Map<String, DatacenterTableField> existingFieldMap = getFields(table.getId()).stream()
|
||||
.collect(LinkedHashMap::new, (map, field) -> map.put(field.getFieldName(), field), Map::putAll);
|
||||
QueryWrapper deleteWrapper = QueryWrapper.create();
|
||||
deleteWrapper.eq(DatacenterTableField::getTableId, table.getId());
|
||||
tableFieldMapper.deleteByQuery(deleteWrapper);
|
||||
for (DatacenterTableField field : detail.getFields()) {
|
||||
DatacenterTableField existingField = existingFieldMap.get(field.getFieldName());
|
||||
if (existingField != null && hasText(existingField.getFieldDesc())) {
|
||||
field.setFieldDesc(existingField.getFieldDesc());
|
||||
} else {
|
||||
field.setFieldDesc(normalizeDescription(field.getFieldDesc()));
|
||||
}
|
||||
field.setId(null);
|
||||
field.setTableId(table.getId());
|
||||
field.setCreated(now);
|
||||
field.setModified(now);
|
||||
field.setCreatedBy(account == null ? BigInteger.ZERO : account.getId());
|
||||
field.setModifiedBy(account == null ? BigInteger.ZERO : account.getId());
|
||||
tableFieldMapper.insert(field);
|
||||
}
|
||||
table.setFields(getFields(table.getId()));
|
||||
return table;
|
||||
}
|
||||
|
||||
@Override
|
||||
@Transactional(rollbackFor = Exception.class)
|
||||
public DatacenterTable saveDescriptions(BigInteger tableId, String tableDesc, List<DatacenterFieldDescriptionUpdate> fields, LoginAccount account) {
|
||||
if (tableId == null) {
|
||||
throw new BusinessException("缺少表 ID");
|
||||
}
|
||||
DatacenterTable table = getTableWithFields(tableId);
|
||||
Date now = new Date();
|
||||
table.setTableDesc(normalizeDescription(tableDesc));
|
||||
table.setModified(now);
|
||||
table.setModifiedBy(account == null ? BigInteger.ZERO : account.getId());
|
||||
tableMapper.update(table);
|
||||
|
||||
Map<BigInteger, DatacenterFieldDescriptionUpdate> fieldUpdateMap = new LinkedHashMap<>();
|
||||
if (fields != null) {
|
||||
for (DatacenterFieldDescriptionUpdate field : fields) {
|
||||
if (field == null || field.getFieldId() == null) {
|
||||
continue;
|
||||
}
|
||||
fieldUpdateMap.put(field.getFieldId(), field);
|
||||
}
|
||||
}
|
||||
for (DatacenterTableField field : table.getFields()) {
|
||||
DatacenterFieldDescriptionUpdate update = fieldUpdateMap.get(field.getId());
|
||||
if (update == null) {
|
||||
continue;
|
||||
}
|
||||
field.setFieldDesc(normalizeDescription(update.getFieldDesc()));
|
||||
field.setModified(now);
|
||||
field.setModifiedBy(account == null ? BigInteger.ZERO : account.getId());
|
||||
tableFieldMapper.update(field);
|
||||
}
|
||||
return getTableWithFields(tableId);
|
||||
}
|
||||
|
||||
@Override
|
||||
public DatacenterTable getTableWithFields(BigInteger tableId) {
|
||||
DatacenterTable table = tableMapper.selectOneById(tableId);
|
||||
if (table == null) {
|
||||
throw new BusinessException("数据集不存在: " + tableId);
|
||||
}
|
||||
table.setFields(getFields(tableId));
|
||||
return table;
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<DatacenterTableField> getFields(BigInteger tableId) {
|
||||
QueryWrapper wrapper = QueryWrapper.create();
|
||||
wrapper.eq(DatacenterTableField::getTableId, tableId);
|
||||
wrapper.orderBy("id");
|
||||
return tableFieldMapper.selectListByQuery(wrapper);
|
||||
}
|
||||
|
||||
@Override
|
||||
public DatasetRef resolveDatasetRef(BigInteger tableId) {
|
||||
DatacenterTable table = getTableWithFields(tableId);
|
||||
DatasetRef ref = new DatasetRef();
|
||||
ref.setTableId(tableId);
|
||||
ref.setSourceId(table.getSourceId());
|
||||
ref.setCatalogId(table.getCatalogId());
|
||||
ref.setTableName(table.getTableName());
|
||||
DatacenterCatalog catalog = getCatalogById(table.getCatalogId());
|
||||
if (catalog != null) {
|
||||
ref.setCatalogName(catalog.getCatalogName());
|
||||
}
|
||||
return ref;
|
||||
}
|
||||
|
||||
@Override
|
||||
public DatacenterSource getSourceRequired(BigInteger sourceId) {
|
||||
DatacenterSource source = sourceMapper.selectOneById(sourceId);
|
||||
if (source == null) {
|
||||
throw new BusinessException("连接不存在");
|
||||
}
|
||||
return source;
|
||||
}
|
||||
|
||||
@Override
|
||||
public DatacenterCatalog getCatalogById(BigInteger catalogId) {
|
||||
if (catalogId == null) {
|
||||
return null;
|
||||
}
|
||||
return catalogMapper.selectOneById(catalogId);
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<DatacenterTable> listManagedTables(BigInteger sourceId, BigInteger catalogId) {
|
||||
QueryWrapper wrapper = QueryWrapper.create();
|
||||
wrapper.eq(DatacenterTable::getSourceId, sourceId);
|
||||
if (catalogId != null) {
|
||||
wrapper.eq(DatacenterTable::getCatalogId, catalogId);
|
||||
}
|
||||
wrapper.orderBy("created desc");
|
||||
return tableMapper.selectListByQuery(wrapper);
|
||||
}
|
||||
|
||||
@Override
|
||||
@Transactional(rollbackFor = Exception.class)
|
||||
public int removeTables(List<BigInteger> tableIds) {
|
||||
List<BigInteger> ids = tableIds == null ? List.of() : tableIds.stream().filter(id -> id != null).distinct().toList();
|
||||
if (ids.isEmpty()) {
|
||||
return 0;
|
||||
}
|
||||
QueryWrapper physicalTableWrapper = QueryWrapper.create();
|
||||
physicalTableWrapper.in(DatacenterTable::getId, ids);
|
||||
List<DatacenterTable> tables = tableMapper.selectListByQuery(physicalTableWrapper);
|
||||
tables.forEach(this::deletePhysicalTableIfNecessary);
|
||||
|
||||
QueryWrapper fieldWrapper = QueryWrapper.create();
|
||||
fieldWrapper.in(DatacenterTableField::getTableId, ids);
|
||||
tableFieldMapper.deleteByQuery(fieldWrapper);
|
||||
|
||||
QueryWrapper versionWrapper = QueryWrapper.create();
|
||||
versionWrapper.in(tech.easyflow.datacenter.meta.entity.DatacenterDatasetVersion::getTableId, ids);
|
||||
datasetVersionMapper.deleteByQuery(versionWrapper);
|
||||
|
||||
QueryWrapper importJobWrapper = QueryWrapper.create();
|
||||
importJobWrapper.in(tech.easyflow.datacenter.meta.entity.DatacenterImportJob::getTableId, ids);
|
||||
importJobMapper.deleteByQuery(importJobWrapper);
|
||||
|
||||
QueryWrapper upstreamWrapper = QueryWrapper.create();
|
||||
upstreamWrapper.in(tech.easyflow.datacenter.meta.entity.DatacenterDerivedTable::getSourceTableId, ids);
|
||||
derivedTableMapper.deleteByQuery(upstreamWrapper);
|
||||
|
||||
QueryWrapper downstreamWrapper = QueryWrapper.create();
|
||||
downstreamWrapper.in(tech.easyflow.datacenter.meta.entity.DatacenterDerivedTable::getDerivedTableId, ids);
|
||||
derivedTableMapper.deleteByQuery(downstreamWrapper);
|
||||
|
||||
QueryWrapper tableWrapper = QueryWrapper.create();
|
||||
tableWrapper.in(DatacenterTable::getId, ids);
|
||||
return tableMapper.deleteByQuery(tableWrapper);
|
||||
}
|
||||
|
||||
private void deletePhysicalTableIfNecessary(DatacenterTable table) {
|
||||
if (table == null || !shouldDropPhysicalTable(table)) {
|
||||
return;
|
||||
}
|
||||
String physicalTableName = table.getActualTable();
|
||||
if (physicalTableName == null || physicalTableName.isBlank()) {
|
||||
physicalTableName = table.getMaterializedTable();
|
||||
}
|
||||
if (physicalTableName == null || physicalTableName.isBlank()) {
|
||||
return;
|
||||
}
|
||||
DatacenterTable physicalTable = new DatacenterTable();
|
||||
physicalTable.setActualTable(physicalTableName);
|
||||
dbHandleManager.getDbHandler().deleteTable(physicalTable);
|
||||
}
|
||||
|
||||
private boolean shouldDropPhysicalTable(DatacenterTable table) {
|
||||
String tableKind = table.getTableKind();
|
||||
if (tableKind == null || tableKind.isBlank()) {
|
||||
return false;
|
||||
}
|
||||
try {
|
||||
DatacenterTableKind kind = DatacenterTableKind.valueOf(tableKind);
|
||||
return kind == DatacenterTableKind.PROJECT_MANAGED
|
||||
|| kind == DatacenterTableKind.EXCEL_SHEET
|
||||
|| kind == DatacenterTableKind.EXCEL_MATERIALIZED
|
||||
|| kind == DatacenterTableKind.DERIVED_TABLE;
|
||||
} catch (IllegalArgumentException ignored) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
private void applyTableDefaults(DatacenterTable table, DatacenterSource source, DatacenterTableDetailMeta detail) {
|
||||
if (table.getTableKind() == null || table.getTableKind().isBlank()) {
|
||||
table.setTableKind(DatacenterTableKind.EXTERNAL_TABLE.name());
|
||||
}
|
||||
if (table.getAccessMode() == null || table.getAccessMode().isBlank()) {
|
||||
table.setAccessMode("READ_ONLY");
|
||||
}
|
||||
if (table.getVersioningEnabled() == null) {
|
||||
table.setVersioningEnabled(0);
|
||||
}
|
||||
if (table.getCapabilitiesJson() == null || table.getCapabilitiesJson().isEmpty()) {
|
||||
table.setCapabilitiesJson(Map.of(
|
||||
"capabilities",
|
||||
source.getCapabilitiesJson() == null
|
||||
? List.of()
|
||||
: source.getCapabilitiesJson().getOrDefault("capabilities", List.of())
|
||||
));
|
||||
}
|
||||
if ((table.getActualTable() == null || table.getActualTable().isBlank()) && table.getTableName() != null) {
|
||||
table.setActualTable(table.getTableName());
|
||||
}
|
||||
table.setTableDesc(normalizeDescription(table.getTableDesc()));
|
||||
if (detail.getFields() == null) {
|
||||
detail.setFields(List.of());
|
||||
}
|
||||
}
|
||||
|
||||
private String normalizeDescription(String description) {
|
||||
if (description == null) {
|
||||
return "";
|
||||
}
|
||||
return description.trim();
|
||||
}
|
||||
|
||||
private boolean hasText(String value) {
|
||||
return value != null && !value.trim().isEmpty();
|
||||
}
|
||||
|
||||
private BigInteger builtinSourceId(DatacenterSourceType sourceType, BigInteger tenantId) {
|
||||
if (sourceType == DatacenterSourceType.PROJECT_MYSQL) {
|
||||
return DatacenterMetaConstants.PROJECT_SOURCE_BASE.add(tenantId);
|
||||
}
|
||||
throw new BusinessException("不支持的内置源 ID 计算");
|
||||
}
|
||||
|
||||
private BigInteger builtinCatalogId(DatacenterSource source, String catalogName) {
|
||||
if (source.getId().compareTo(DatacenterMetaConstants.PROJECT_SOURCE_BASE) >= 0
|
||||
&& defaultCatalogName(DatacenterSourceType.PROJECT_MYSQL).equals(catalogName)) {
|
||||
return DatacenterMetaConstants.PROJECT_CATALOG_BASE.add(source.getTenantId());
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
private String defaultCatalogName(DatacenterSourceType sourceType) {
|
||||
if (sourceType == DatacenterSourceType.PROJECT_MYSQL) {
|
||||
return "project_mysql";
|
||||
}
|
||||
return sourceType.name().toLowerCase();
|
||||
}
|
||||
|
||||
private List<String> builtinCapabilities(DatacenterSourceType sourceType) {
|
||||
return List.of("TEST_CONNECTION", "BROWSE_METADATA", "READ_QUERY", "WRITE_MUTATION");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,311 @@
|
||||
package tech.easyflow.datacenter.meta.service.impl;
|
||||
|
||||
import com.mybatisflex.core.paginate.Page;
|
||||
import com.mybatisflex.core.query.QueryWrapper;
|
||||
import com.mybatisflex.spring.service.impl.ServiceImpl;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.springframework.transaction.annotation.Transactional;
|
||||
import tech.easyflow.common.entity.LoginAccount;
|
||||
import tech.easyflow.common.web.exceptions.BusinessException;
|
||||
import tech.easyflow.datacenter.connector.DatacenterConnector;
|
||||
import tech.easyflow.datacenter.connector.DatacenterConnectorRegistry;
|
||||
import tech.easyflow.datacenter.entity.DatacenterTable;
|
||||
import tech.easyflow.datacenter.mapper.DatacenterCatalogMapper;
|
||||
import tech.easyflow.datacenter.mapper.DatacenterSourceMapper;
|
||||
import tech.easyflow.datacenter.meta.entity.DatacenterCatalog;
|
||||
import tech.easyflow.datacenter.meta.entity.DatacenterSource;
|
||||
import tech.easyflow.datacenter.meta.enums.DatacenterSourceType;
|
||||
import tech.easyflow.datacenter.meta.model.DatacenterBatchRegisterRequest;
|
||||
import tech.easyflow.datacenter.meta.model.DatacenterCatalogMeta;
|
||||
import tech.easyflow.datacenter.meta.model.DatacenterTableDetailMeta;
|
||||
import tech.easyflow.datacenter.meta.service.DatacenterDatasetRegistryService;
|
||||
import tech.easyflow.datacenter.meta.service.DatacenterSourceService;
|
||||
import tech.easyflow.datacenter.meta.support.DatacenterSourceConnectionDefaults;
|
||||
import tech.easyflow.datacenter.execution.model.DatacenterConnectionTestResult;
|
||||
import tech.easyflow.datacenter.security.DatacenterCredentialCipher;
|
||||
|
||||
import javax.annotation.Resource;
|
||||
import java.math.BigInteger;
|
||||
import java.util.Date;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
@Service
|
||||
public class DatacenterSourceServiceImpl extends ServiceImpl<DatacenterSourceMapper, DatacenterSource> implements DatacenterSourceService {
|
||||
|
||||
@Resource
|
||||
private DatacenterConnectorRegistry connectorRegistry;
|
||||
@Resource
|
||||
private DatacenterDatasetRegistryService registryService;
|
||||
@Resource
|
||||
private DatacenterCredentialCipher credentialCipher;
|
||||
@Resource
|
||||
private DatacenterCatalogMapper catalogMapper;
|
||||
@Resource
|
||||
private DatacenterSourceConnectionDefaults connectionDefaults;
|
||||
|
||||
@Override
|
||||
public DatacenterSource saveSource(DatacenterSource source, LoginAccount account) {
|
||||
if (source == null || source.getSourceType() == null || source.getSourceType().isBlank()) {
|
||||
throw new BusinessException("数据源类型不能为空");
|
||||
}
|
||||
DatacenterSource existing = source.getId() == null ? null : getById(source.getId());
|
||||
DatacenterSource normalized = mergeWithExisting(existing, source);
|
||||
DatacenterConnector connector = connectorRegistry.getConnector(normalized.getSourceType());
|
||||
applyCredentialCipher(normalized, existing);
|
||||
normalized.setConfigJson(connectionDefaults.sanitizeConfig(normalized.getConfigJson()));
|
||||
connectionDefaults.normalize(normalized);
|
||||
normalized.setCapabilitiesJson(Map.of("capabilities", connector.getCapabilities().stream().map(Enum::name).toList()));
|
||||
Date now = new Date();
|
||||
if (normalized.getId() == null) {
|
||||
normalized.setCreated(now);
|
||||
normalized.setCreatedBy(account == null ? BigInteger.ZERO : account.getId());
|
||||
normalized.setTenantId(account == null ? BigInteger.ZERO : account.getTenantId());
|
||||
normalized.setDeptId(account == null ? BigInteger.ZERO : account.getDeptId());
|
||||
normalized.setStatus(normalized.getStatus() == null ? 0 : normalized.getStatus());
|
||||
normalized.setModified(now);
|
||||
normalized.setModifiedBy(account == null ? BigInteger.ZERO : account.getId());
|
||||
save(normalized);
|
||||
} else {
|
||||
if (existing == null) {
|
||||
throw new BusinessException("连接不存在");
|
||||
}
|
||||
normalized.setModified(now);
|
||||
normalized.setModifiedBy(account == null ? BigInteger.ZERO : account.getId());
|
||||
updateById(normalized);
|
||||
}
|
||||
return getById(normalized.getId());
|
||||
}
|
||||
|
||||
@Override
|
||||
public Page<DatacenterSource> pageSources(Long pageNumber, Long pageSize, LoginAccount account) {
|
||||
registryService.ensureBuiltinSource(DatacenterSourceType.PROJECT_MYSQL, account);
|
||||
return page(new Page<>(pageNumber == null ? 1L : pageNumber, pageSize == null ? 10L : pageSize), QueryWrapper.create());
|
||||
}
|
||||
|
||||
@Override
|
||||
public DatacenterConnectionTestResult testConnection(DatacenterSource source, LoginAccount account) {
|
||||
DatacenterSource existing = source != null && source.getId() != null ? getById(source.getId()) : null;
|
||||
if (source != null && source.getId() != null && existing == null) {
|
||||
throw new BusinessException("连接不存在");
|
||||
}
|
||||
DatacenterSource actual = mergeWithExisting(existing, source);
|
||||
if (DatacenterSourceType.PROJECT_MYSQL.name().equals(actual.getSourceType())) {
|
||||
actual = registryService.ensureBuiltinSource(DatacenterSourceType.PROJECT_MYSQL, account);
|
||||
}
|
||||
applyCredentialCipher(actual, existing);
|
||||
actual.setConfigJson(connectionDefaults.sanitizeConfig(actual.getConfigJson()));
|
||||
connectionDefaults.normalize(actual);
|
||||
DatacenterConnector connector = connectorRegistry.getConnector(actual.getSourceType());
|
||||
DatacenterConnectionTestResult result = connector.testConnection(actual);
|
||||
Map<String, Object> details = new LinkedHashMap<>();
|
||||
if (result.getDetails() != null) {
|
||||
details.putAll(result.getDetails());
|
||||
}
|
||||
details.put("effectiveDriverClassName", actual.getDriverClassName());
|
||||
details.put("effectiveJdbcUrl", actual.getJdbcUrl());
|
||||
details.put("effectivePort", actual.getPort());
|
||||
result.setDetails(details);
|
||||
if (actual.getId() != null) {
|
||||
DatacenterSource persisted = new DatacenterSource();
|
||||
persisted.setId(actual.getId());
|
||||
persisted.setLastTestStatus(result.isSuccess() ? "SUCCESS" : "FAILED");
|
||||
persisted.setLastTestMessage(result.getMessage());
|
||||
persisted.setLastTestedAt(new Date());
|
||||
updateById(persisted);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<DatacenterCatalogMeta> listCatalogs(BigInteger sourceId, LoginAccount account) {
|
||||
DatacenterSource source = registryService.getSourceRequired(sourceId);
|
||||
if (isManagedOnly(source.getSourceType())) {
|
||||
QueryWrapper wrapper = QueryWrapper.create();
|
||||
wrapper.eq(DatacenterCatalog::getSourceId, sourceId);
|
||||
return catalogMapper.selectListByQuery(wrapper).stream().map(this::toCatalogMeta).collect(Collectors.toList());
|
||||
}
|
||||
return connectorRegistry.getConnector(source.getSourceType()).listCatalogs(source);
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<DatacenterTable> listTables(BigInteger sourceId, String catalogName, LoginAccount account) {
|
||||
DatacenterSource source = registryService.getSourceRequired(sourceId);
|
||||
if (isManagedOnly(source.getSourceType())) {
|
||||
BigInteger catalogId = resolveCatalogId(sourceId, catalogName);
|
||||
return registryService.listManagedTables(sourceId, catalogId);
|
||||
}
|
||||
return connectorRegistry.getConnector(source.getSourceType()).listTables(source, catalogName);
|
||||
}
|
||||
|
||||
@Override
|
||||
public DatacenterTableDetailMeta getTableDetail(BigInteger sourceId, String catalogName, String tableName, boolean register, LoginAccount account) {
|
||||
DatacenterSource source = registryService.getSourceRequired(sourceId);
|
||||
if (isManagedOnly(source.getSourceType())) {
|
||||
BigInteger catalogId = resolveCatalogId(sourceId, catalogName);
|
||||
List<DatacenterTable> tables = registryService.listManagedTables(sourceId, catalogId);
|
||||
DatacenterTable target = tables.stream().filter(item -> tableName.equals(item.getTableName())).findFirst()
|
||||
.orElseThrow(() -> new BusinessException("数据集不存在: " + tableName));
|
||||
DatacenterTableDetailMeta detail = new DatacenterTableDetailMeta();
|
||||
detail.setTable(registryService.getTableWithFields(target.getId()));
|
||||
detail.setFields(detail.getTable().getFields());
|
||||
return detail;
|
||||
}
|
||||
DatacenterConnector connector = connectorRegistry.getConnector(source.getSourceType());
|
||||
DatacenterTableDetailMeta detail = connector.getTableDetail(source, catalogName, tableName);
|
||||
if (register) {
|
||||
DatacenterCatalog catalog = registryService.ensureCatalog(source, catalogName, account);
|
||||
DatacenterTable table = registryService.registerTable(source, catalog, detail, account);
|
||||
detail.setTable(table);
|
||||
detail.setFields(table.getFields());
|
||||
}
|
||||
return detail;
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<DatacenterTable> batchRegisterTables(DatacenterBatchRegisterRequest request, LoginAccount account) {
|
||||
if (request == null || request.getSourceId() == null) {
|
||||
throw new BusinessException("数据连接不能为空");
|
||||
}
|
||||
if (request.getCatalogName() == null || request.getCatalogName().isBlank()) {
|
||||
throw new BusinessException("库不能为空");
|
||||
}
|
||||
List<String> tableNames = request.getTableNames() == null
|
||||
? List.of()
|
||||
: request.getTableNames().stream().filter(name -> name != null && !name.isBlank()).distinct().toList();
|
||||
if (tableNames.isEmpty()) {
|
||||
throw new BusinessException("至少选择一张表");
|
||||
}
|
||||
DatacenterSource source = registryService.getSourceRequired(request.getSourceId());
|
||||
if (isManagedOnly(source.getSourceType())) {
|
||||
throw new BusinessException("当前数据连接不支持批量接入");
|
||||
}
|
||||
DatacenterConnector connector = connectorRegistry.getConnector(source.getSourceType());
|
||||
DatacenterCatalog catalog = registryService.ensureCatalog(source, request.getCatalogName(), account);
|
||||
return tableNames.stream().map(tableName -> {
|
||||
DatacenterTableDetailMeta detail = connector.getTableDetail(source, request.getCatalogName(), tableName);
|
||||
return registryService.registerTable(source, catalog, detail, account);
|
||||
}).collect(Collectors.toList());
|
||||
}
|
||||
|
||||
@Override
|
||||
@Transactional(rollbackFor = Exception.class)
|
||||
public void removeSource(BigInteger sourceId, LoginAccount account) {
|
||||
if (sourceId == null) {
|
||||
throw new BusinessException("数据连接不能为空");
|
||||
}
|
||||
DatacenterSource source = getById(sourceId);
|
||||
if (source == null) {
|
||||
throw new BusinessException("连接不存在");
|
||||
}
|
||||
if (Boolean.TRUE.equals(source.getBuiltinFlag())) {
|
||||
throw new BusinessException("内置数据连接不支持删除");
|
||||
}
|
||||
|
||||
List<BigInteger> tableIds = registryService.listManagedTables(sourceId, null).stream()
|
||||
.map(DatacenterTable::getId)
|
||||
.filter(id -> id != null)
|
||||
.toList();
|
||||
registryService.removeTables(tableIds);
|
||||
|
||||
QueryWrapper catalogWrapper = QueryWrapper.create();
|
||||
catalogWrapper.eq(DatacenterCatalog::getSourceId, sourceId);
|
||||
catalogMapper.deleteByQuery(catalogWrapper);
|
||||
|
||||
removeById(sourceId);
|
||||
}
|
||||
|
||||
private boolean isManagedOnly(String sourceType) {
|
||||
return DatacenterSourceType.EXCEL.name().equals(sourceType)
|
||||
|| DatacenterSourceType.EXCEL_MATERIALIZED.name().equals(sourceType);
|
||||
}
|
||||
|
||||
private DatacenterCatalogMeta toCatalogMeta(DatacenterCatalog catalog) {
|
||||
DatacenterCatalogMeta meta = new DatacenterCatalogMeta();
|
||||
meta.setId(catalog.getId());
|
||||
meta.setSourceId(catalog.getSourceId());
|
||||
meta.setCatalogName(catalog.getCatalogName());
|
||||
meta.setCatalogDesc(catalog.getCatalogDesc());
|
||||
meta.setCatalogType(catalog.getCatalogType());
|
||||
return meta;
|
||||
}
|
||||
|
||||
private BigInteger resolveCatalogId(BigInteger sourceId, String catalogName) {
|
||||
if (catalogName == null || catalogName.isBlank()) {
|
||||
return null;
|
||||
}
|
||||
QueryWrapper wrapper = QueryWrapper.create();
|
||||
wrapper.eq(DatacenterCatalog::getSourceId, sourceId);
|
||||
wrapper.eq(DatacenterCatalog::getCatalogName, catalogName);
|
||||
DatacenterCatalog catalog = catalogMapper.selectOneByQuery(wrapper);
|
||||
if (catalog == null) {
|
||||
throw new BusinessException("目录不存在: " + catalogName);
|
||||
}
|
||||
return catalog.getId();
|
||||
}
|
||||
|
||||
private void applyCredentialCipher(DatacenterSource target, DatacenterSource existing) {
|
||||
String password = extractPassword(target.getConfigJson());
|
||||
if (password != null) {
|
||||
target.setCredentialCipher(credentialCipher.encrypt(password));
|
||||
return;
|
||||
}
|
||||
if ((target.getCredentialCipher() == null || target.getCredentialCipher().isBlank()) && existing != null) {
|
||||
target.setCredentialCipher(existing.getCredentialCipher());
|
||||
}
|
||||
}
|
||||
|
||||
private String extractPassword(Map<String, Object> configJson) {
|
||||
if (configJson == null) {
|
||||
return null;
|
||||
}
|
||||
Object password = configJson.get("password");
|
||||
if (password == null) {
|
||||
return null;
|
||||
}
|
||||
String value = String.valueOf(password);
|
||||
return value.isBlank() ? null : value;
|
||||
}
|
||||
|
||||
private DatacenterSource mergeWithExisting(DatacenterSource existing, DatacenterSource incoming) {
|
||||
if (incoming == null) {
|
||||
return existing;
|
||||
}
|
||||
if (existing == null) {
|
||||
return incoming;
|
||||
}
|
||||
DatacenterSource merged = new DatacenterSource();
|
||||
merged.setId(existing.getId());
|
||||
merged.setCreated(existing.getCreated());
|
||||
merged.setCreatedBy(existing.getCreatedBy());
|
||||
merged.setTenantId(existing.getTenantId());
|
||||
merged.setDeptId(existing.getDeptId());
|
||||
merged.setBuiltinFlag(existing.getBuiltinFlag());
|
||||
merged.setStatus(existing.getStatus());
|
||||
merged.setLastTestStatus(existing.getLastTestStatus());
|
||||
merged.setLastTestMessage(existing.getLastTestMessage());
|
||||
merged.setLastTestedAt(existing.getLastTestedAt());
|
||||
merged.setOptions(existing.getOptions());
|
||||
merged.setSourceName(valueOrExisting(incoming.getSourceName(), existing.getSourceName()));
|
||||
merged.setSourceCode(valueOrExisting(incoming.getSourceCode(), existing.getSourceCode()));
|
||||
merged.setSourceType(valueOrExisting(incoming.getSourceType(), existing.getSourceType()));
|
||||
merged.setAccessMode(valueOrExisting(incoming.getAccessMode(), existing.getAccessMode()));
|
||||
merged.setDriverClassName(valueOrExisting(incoming.getDriverClassName(), existing.getDriverClassName()));
|
||||
merged.setJdbcUrl(valueOrExisting(incoming.getJdbcUrl(), existing.getJdbcUrl()));
|
||||
merged.setHost(valueOrExisting(incoming.getHost(), existing.getHost()));
|
||||
merged.setPort(incoming.getPort() != null ? incoming.getPort() : existing.getPort());
|
||||
merged.setDatabaseName(valueOrExisting(incoming.getDatabaseName(), existing.getDatabaseName()));
|
||||
merged.setSchemaName(valueOrExisting(incoming.getSchemaName(), existing.getSchemaName()));
|
||||
merged.setUsername(valueOrExisting(incoming.getUsername(), existing.getUsername()));
|
||||
merged.setCredentialCipher(valueOrExisting(incoming.getCredentialCipher(), existing.getCredentialCipher()));
|
||||
merged.setConfigJson(incoming.getConfigJson() != null ? incoming.getConfigJson() : existing.getConfigJson());
|
||||
merged.setCapabilitiesJson(existing.getCapabilitiesJson());
|
||||
return merged;
|
||||
}
|
||||
|
||||
private String valueOrExisting(String incoming, String existing) {
|
||||
return incoming != null ? incoming : existing;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,153 @@
|
||||
package tech.easyflow.datacenter.meta.support;
|
||||
|
||||
import cn.hutool.core.util.StrUtil;
|
||||
import org.springframework.stereotype.Component;
|
||||
import tech.easyflow.datacenter.meta.entity.DatacenterSource;
|
||||
import tech.easyflow.datacenter.meta.enums.DatacenterSourceType;
|
||||
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.Map;
|
||||
|
||||
@Component
|
||||
public class DatacenterSourceConnectionDefaults {
|
||||
|
||||
public boolean supportsExternalConnection(String sourceType) {
|
||||
if (StrUtil.isBlank(sourceType)) {
|
||||
return false;
|
||||
}
|
||||
DatacenterSourceType resolved = resolveSourceType(sourceType);
|
||||
if (resolved == null) {
|
||||
return false;
|
||||
}
|
||||
return switch (resolved) {
|
||||
case MYSQL, POSTGRESQL, ORACLE, GAUSSDB_NATIVE, GBASE_8A, GBASE_8S -> true;
|
||||
default -> false;
|
||||
};
|
||||
}
|
||||
|
||||
public DatacenterSource normalize(DatacenterSource source) {
|
||||
if (source == null || !supportsExternalConnection(source.getSourceType())) {
|
||||
return source;
|
||||
}
|
||||
if (source.getPort() == null || source.getPort() <= 0) {
|
||||
source.setPort(defaultPort(source.getSourceType()));
|
||||
}
|
||||
if (StrUtil.isBlank(source.getDriverClassName())) {
|
||||
source.setDriverClassName(defaultDriverClassName(source.getSourceType()));
|
||||
}
|
||||
if (StrUtil.isBlank(source.getJdbcUrl())) {
|
||||
String jdbcUrl = buildJdbcUrl(source);
|
||||
if (StrUtil.isNotBlank(jdbcUrl)) {
|
||||
source.setJdbcUrl(jdbcUrl);
|
||||
}
|
||||
}
|
||||
return source;
|
||||
}
|
||||
|
||||
public Map<String, Object> sanitizeConfig(Map<String, Object> configJson) {
|
||||
Map<String, Object> sanitized = new LinkedHashMap<>();
|
||||
if (configJson == null || configJson.isEmpty()) {
|
||||
return sanitized;
|
||||
}
|
||||
sanitized.putAll(configJson);
|
||||
sanitized.remove("password");
|
||||
return sanitized;
|
||||
}
|
||||
|
||||
public Integer defaultPort(String sourceType) {
|
||||
DatacenterSourceType resolved = requireSourceType(sourceType);
|
||||
return switch (resolved) {
|
||||
case MYSQL -> 3306;
|
||||
case POSTGRESQL, GAUSSDB_NATIVE -> 5432;
|
||||
case ORACLE -> 1521;
|
||||
case GBASE_8A -> 5258;
|
||||
case GBASE_8S -> 9088;
|
||||
default -> null;
|
||||
};
|
||||
}
|
||||
|
||||
public String defaultDriverClassName(String sourceType) {
|
||||
DatacenterSourceType resolved = requireSourceType(sourceType);
|
||||
return switch (resolved) {
|
||||
case MYSQL -> "com.mysql.cj.jdbc.Driver";
|
||||
case POSTGRESQL, GAUSSDB_NATIVE -> "org.postgresql.Driver";
|
||||
case ORACLE -> "oracle.jdbc.OracleDriver";
|
||||
case GBASE_8A -> "com.gbase.jdbc.Driver";
|
||||
case GBASE_8S -> "com.gbasedbt.jdbc.Driver";
|
||||
default -> "";
|
||||
};
|
||||
}
|
||||
|
||||
public String buildJdbcUrl(DatacenterSource source) {
|
||||
if (source == null || !supportsExternalConnection(source.getSourceType())) {
|
||||
return null;
|
||||
}
|
||||
String host = StrUtil.trimToEmpty(source.getHost());
|
||||
Integer port = source.getPort() == null || source.getPort() <= 0 ? defaultPort(source.getSourceType()) : source.getPort();
|
||||
String databaseName = resolveDatabaseName(source);
|
||||
if (StrUtil.isBlank(host) || port == null || StrUtil.isBlank(databaseName)) {
|
||||
return null;
|
||||
}
|
||||
DatacenterSourceType resolved = requireSourceType(source.getSourceType());
|
||||
return switch (resolved) {
|
||||
case MYSQL -> String.format(
|
||||
"jdbc:mysql://%s:%d/%s?useUnicode=true&characterEncoding=utf-8&serverTimezone=Asia/Shanghai&useSSL=false",
|
||||
host, port, databaseName
|
||||
);
|
||||
case POSTGRESQL -> String.format("jdbc:postgresql://%s:%d/%s", host, port, databaseName);
|
||||
case ORACLE -> String.format("jdbc:oracle:thin:@//%s:%d/%s", host, port, databaseName);
|
||||
case GAUSSDB_NATIVE -> String.format("jdbc:postgresql://%s:%d/%s", host, port, databaseName);
|
||||
case GBASE_8A -> String.format("jdbc:gbase://%s:%d/%s", host, port, databaseName);
|
||||
case GBASE_8S -> {
|
||||
String informixServer = resolveInformixServer(source);
|
||||
if (StrUtil.isBlank(informixServer)) {
|
||||
yield null;
|
||||
}
|
||||
yield String.format("jdbc:gbasedbt-sqli://%s:%d/%s:INFORMIXSERVER=%s", host, port, databaseName, informixServer);
|
||||
}
|
||||
default -> null;
|
||||
};
|
||||
}
|
||||
|
||||
private DatacenterSourceType requireSourceType(String sourceType) {
|
||||
DatacenterSourceType resolved = resolveSourceType(sourceType);
|
||||
if (resolved == null) {
|
||||
throw new IllegalArgumentException("Unsupported datacenter source type: " + sourceType);
|
||||
}
|
||||
return resolved;
|
||||
}
|
||||
|
||||
private DatacenterSourceType resolveSourceType(String sourceType) {
|
||||
try {
|
||||
return DatacenterSourceType.valueOf(sourceType);
|
||||
} catch (IllegalArgumentException ex) {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
private String resolveDatabaseName(DatacenterSource source) {
|
||||
if (StrUtil.isNotBlank(source.getDatabaseName())) {
|
||||
return source.getDatabaseName().trim();
|
||||
}
|
||||
if (source.getConfigJson() == null) {
|
||||
return null;
|
||||
}
|
||||
Object serviceName = source.getConfigJson().get("serviceName");
|
||||
if (serviceName != null && StrUtil.isNotBlank(String.valueOf(serviceName))) {
|
||||
return String.valueOf(serviceName).trim();
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
private String resolveInformixServer(DatacenterSource source) {
|
||||
if (source.getConfigJson() == null) {
|
||||
return null;
|
||||
}
|
||||
Object informixServer = source.getConfigJson().get("informixServer");
|
||||
if (informixServer == null) {
|
||||
return null;
|
||||
}
|
||||
String value = String.valueOf(informixServer).trim();
|
||||
return value.isEmpty() ? null : value;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,28 @@
|
||||
package tech.easyflow.datacenter.security;
|
||||
|
||||
import cn.hutool.crypto.SecureUtil;
|
||||
import cn.hutool.crypto.symmetric.AES;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
import java.nio.charset.StandardCharsets;
|
||||
|
||||
@Component
|
||||
public class DatacenterCredentialCipher {
|
||||
|
||||
private static final String DEFAULT_KEY = "easyflow-datacenter-phase1-key";
|
||||
private final AES aes = SecureUtil.aes(SecureUtil.sha256(DEFAULT_KEY).substring(0, 16).getBytes(StandardCharsets.UTF_8));
|
||||
|
||||
public String encrypt(String plainText) {
|
||||
if (plainText == null || plainText.isBlank()) {
|
||||
return null;
|
||||
}
|
||||
return aes.encryptHex(plainText);
|
||||
}
|
||||
|
||||
public String decrypt(String cipherText) {
|
||||
if (cipherText == null || cipherText.isBlank()) {
|
||||
return null;
|
||||
}
|
||||
return aes.decryptStr(cipherText);
|
||||
}
|
||||
}
|
||||
@@ -1,14 +0,0 @@
|
||||
package tech.easyflow.datacenter.service;
|
||||
|
||||
import com.mybatisflex.core.service.IService;
|
||||
import tech.easyflow.datacenter.entity.DatacenterTableField;
|
||||
|
||||
/**
|
||||
* 服务层。
|
||||
*
|
||||
* @author ArkLight
|
||||
* @since 2025-07-10
|
||||
*/
|
||||
public interface DatacenterTableFieldService extends IService<DatacenterTableField> {
|
||||
|
||||
}
|
||||
@@ -1,35 +0,0 @@
|
||||
package tech.easyflow.datacenter.service;
|
||||
|
||||
import com.alibaba.fastjson2.JSONObject;
|
||||
import com.mybatisflex.core.paginate.Page;
|
||||
import com.mybatisflex.core.row.Row;
|
||||
import com.mybatisflex.core.service.IService;
|
||||
import tech.easyflow.common.entity.DatacenterQuery;
|
||||
import tech.easyflow.common.entity.LoginAccount;
|
||||
import tech.easyflow.datacenter.entity.DatacenterTable;
|
||||
import tech.easyflow.datacenter.entity.DatacenterTableField;
|
||||
import tech.easyflow.datacenter.entity.vo.HeaderVo;
|
||||
|
||||
import java.math.BigInteger;
|
||||
import java.util.List;
|
||||
|
||||
public interface DatacenterTableService extends IService<DatacenterTable> {
|
||||
|
||||
void saveTable(DatacenterTable entity, LoginAccount loginUser);
|
||||
|
||||
void removeTable(BigInteger tableId);
|
||||
|
||||
Long getCount(DatacenterQuery where);
|
||||
|
||||
List<Row> getListData(DatacenterQuery where);
|
||||
|
||||
Page<Row> getPageData(DatacenterQuery where);
|
||||
|
||||
List<HeaderVo> getHeaders(BigInteger tableId);
|
||||
|
||||
void saveValue(BigInteger tableId, JSONObject object, LoginAccount account);
|
||||
|
||||
void removeValue(BigInteger tableId, BigInteger id, LoginAccount account);
|
||||
|
||||
List<DatacenterTableField> getFields(BigInteger tableId);
|
||||
}
|
||||
@@ -1,18 +0,0 @@
|
||||
package tech.easyflow.datacenter.service.impl;
|
||||
|
||||
import com.mybatisflex.spring.service.impl.ServiceImpl;
|
||||
import org.springframework.stereotype.Service;
|
||||
import tech.easyflow.datacenter.entity.DatacenterTableField;
|
||||
import tech.easyflow.datacenter.mapper.DatacenterTableFieldMapper;
|
||||
import tech.easyflow.datacenter.service.DatacenterTableFieldService;
|
||||
|
||||
/**
|
||||
* 服务层实现。
|
||||
*
|
||||
* @author ArkLight
|
||||
* @since 2025-07-10
|
||||
*/
|
||||
@Service
|
||||
public class DatacenterTableFieldServiceImpl extends ServiceImpl<DatacenterTableFieldMapper, DatacenterTableField> implements DatacenterTableFieldService {
|
||||
|
||||
}
|
||||
@@ -1,293 +0,0 @@
|
||||
package tech.easyflow.datacenter.service.impl;
|
||||
|
||||
import cn.hutool.core.collection.CollectionUtil;
|
||||
import cn.hutool.core.util.StrUtil;
|
||||
import com.alibaba.fastjson2.JSONObject;
|
||||
import com.mybatisflex.core.keygen.impl.SnowFlakeIDKeyGenerator;
|
||||
import com.mybatisflex.core.paginate.Page;
|
||||
import com.mybatisflex.core.query.QueryWrapper;
|
||||
import com.mybatisflex.core.row.Db;
|
||||
import com.mybatisflex.core.row.Row;
|
||||
import com.mybatisflex.spring.service.impl.ServiceImpl;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.springframework.transaction.annotation.Transactional;
|
||||
import tech.easyflow.common.cache.RedisLockExecutor;
|
||||
import tech.easyflow.common.entity.DatacenterQuery;
|
||||
import tech.easyflow.common.entity.LoginAccount;
|
||||
import tech.easyflow.common.web.exceptions.BusinessException;
|
||||
import tech.easyflow.datacenter.adapter.DbHandleManager;
|
||||
import tech.easyflow.datacenter.adapter.DbHandleService;
|
||||
import tech.easyflow.datacenter.entity.DatacenterTable;
|
||||
import tech.easyflow.datacenter.entity.DatacenterTableField;
|
||||
import tech.easyflow.datacenter.entity.vo.HeaderVo;
|
||||
import tech.easyflow.datacenter.mapper.DatacenterTableFieldMapper;
|
||||
import tech.easyflow.datacenter.mapper.DatacenterTableMapper;
|
||||
import tech.easyflow.datacenter.service.DatacenterTableService;
|
||||
|
||||
import javax.annotation.Resource;
|
||||
import java.math.BigDecimal;
|
||||
import java.math.BigInteger;
|
||||
import java.time.Duration;
|
||||
import java.util.*;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
@Service
|
||||
public class DatacenterTableServiceImpl extends ServiceImpl<DatacenterTableMapper, DatacenterTable> implements DatacenterTableService {
|
||||
|
||||
private static final String DATACENTER_TABLE_LOCK_KEY_PREFIX = "easyflow:lock:datacenter:table:";
|
||||
private static final String DATACENTER_TABLE_CREATE_LOCK_KEY_PREFIX = "easyflow:lock:datacenter:table:create:";
|
||||
private static final Duration LOCK_WAIT_TIMEOUT = Duration.ofSeconds(2);
|
||||
private static final Duration LOCK_LEASE_TIMEOUT = Duration.ofSeconds(15);
|
||||
|
||||
@Resource
|
||||
private DbHandleManager dbHandleManager;
|
||||
@Resource
|
||||
private DatacenterTableFieldMapper fieldsMapper;
|
||||
@Resource
|
||||
private RedisLockExecutor redisLockExecutor;
|
||||
|
||||
@Override
|
||||
@Transactional(rollbackFor = Exception.class)
|
||||
public void saveTable(DatacenterTable entity, LoginAccount loginUser) {
|
||||
String lockKey = buildTableLockKey(entity, loginUser);
|
||||
redisLockExecutor.executeWithLock(lockKey, LOCK_WAIT_TIMEOUT, LOCK_LEASE_TIMEOUT, () -> {
|
||||
doSaveTable(entity, loginUser);
|
||||
});
|
||||
}
|
||||
|
||||
private void doSaveTable(DatacenterTable entity, LoginAccount loginUser) {
|
||||
DbHandleService dbHandler = dbHandleManager.getDbHandler();
|
||||
|
||||
List<DatacenterTableField> fields = entity.getFields();
|
||||
|
||||
BigInteger tableId = entity.getId();
|
||||
|
||||
if (tableId == null) {
|
||||
long snowId = new SnowFlakeIDKeyGenerator().nextId();
|
||||
entity.setId(new BigInteger(String.valueOf(snowId)));
|
||||
|
||||
String actualTable = getActualTableName(entity);
|
||||
entity.setActualTable(actualTable);
|
||||
// 先 DDL 操作,DDL会默认提交事务,不然报错了事务不会回滚。
|
||||
dbHandler.createTable(entity);
|
||||
// 保存主表和字段表
|
||||
save(entity);
|
||||
for (DatacenterTableField field : fields) {
|
||||
// 插入
|
||||
field.setCreated(new Date());
|
||||
field.setCreatedBy(loginUser.getId());
|
||||
field.setModified(new Date());
|
||||
field.setModifiedBy(loginUser.getId());
|
||||
field.setTableId(entity.getId());
|
||||
fieldsMapper.insert(field);
|
||||
}
|
||||
} else {
|
||||
// actualTable 前端不可见,所以要设置
|
||||
DatacenterTable tableRecord = getById(tableId);
|
||||
entity.setActualTable(tableRecord.getActualTable());
|
||||
dbHandler.updateTable(entity, tableRecord);
|
||||
updateById(entity);
|
||||
// 查询所有字段
|
||||
QueryWrapper w = QueryWrapper.create();
|
||||
w.eq(DatacenterTableField::getTableId, entity.getId());
|
||||
List<DatacenterTableField> fieldRecords = fieldsMapper.selectListByQuery(w);
|
||||
|
||||
Map<BigInteger, DatacenterTableField> fieldsMap = fieldRecords.stream()
|
||||
.collect(Collectors.toMap(DatacenterTableField::getId, field -> field));
|
||||
|
||||
for (DatacenterTableField field : fields) {
|
||||
BigInteger id = field.getId();
|
||||
if (id == null) {
|
||||
// 新增字段到物理表
|
||||
dbHandler.addField(entity, field);
|
||||
// 插入
|
||||
field.setCreated(new Date());
|
||||
field.setCreatedBy(loginUser.getId());
|
||||
field.setModified(new Date());
|
||||
field.setModifiedBy(loginUser.getId());
|
||||
field.setTableId(entity.getId());
|
||||
fieldsMapper.insert(field);
|
||||
} else {
|
||||
// 删除的字段
|
||||
if (field.isHandleDelete()) {
|
||||
// 删除物理表中的字段
|
||||
dbHandler.deleteField(entity, field);
|
||||
// 删除字段
|
||||
fieldsMapper.deleteById(id);
|
||||
} else {
|
||||
// 修改物理表中的字段
|
||||
DatacenterTableField fieldRecord = fieldsMap.get(id);
|
||||
dbHandler.updateField(entity, fieldRecord, field);
|
||||
// 更新字段,字段类型不允许修改
|
||||
field.setFieldType(field.getFieldType());
|
||||
field.setModified(new Date());
|
||||
field.setModifiedBy(loginUser.getId());
|
||||
fieldsMapper.update(field);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
@Transactional(rollbackFor = Exception.class)
|
||||
public void removeTable(BigInteger tableId) {
|
||||
redisLockExecutor.executeWithLock(
|
||||
DATACENTER_TABLE_LOCK_KEY_PREFIX + tableId,
|
||||
LOCK_WAIT_TIMEOUT,
|
||||
LOCK_LEASE_TIMEOUT,
|
||||
() -> {
|
||||
DatacenterTable record = getById(tableId);
|
||||
dbHandleManager.getDbHandler().deleteTable(record);
|
||||
removeById(tableId);
|
||||
QueryWrapper wrapper = QueryWrapper.create();
|
||||
wrapper.eq(DatacenterTableField::getTableId, tableId);
|
||||
fieldsMapper.deleteByQuery(wrapper);
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Long getCount(DatacenterQuery where) {
|
||||
String actualTable = getActualTable(where.getTableId());
|
||||
QueryWrapper wrapper = QueryWrapper.create();
|
||||
buildCondition(wrapper, where);
|
||||
return Db.selectCountByQuery(actualTable, wrapper);
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<Row> getListData(DatacenterQuery where) {
|
||||
String actualTable = getActualTable(where.getTableId());
|
||||
QueryWrapper wrapper = QueryWrapper.create();
|
||||
buildCondition(wrapper, where);
|
||||
List<Row> rows = Db.selectListByQuery(actualTable, wrapper);
|
||||
handleBigNumber(rows);
|
||||
return rows;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Page<Row> getPageData(DatacenterQuery where) {
|
||||
Long pageNumber = where.getPageNumber();
|
||||
Long pageSize = where.getPageSize();
|
||||
|
||||
Long count = getCount(where);
|
||||
if (count == 0) {
|
||||
return new Page<>(new ArrayList<>(), pageNumber, pageSize, count);
|
||||
}
|
||||
|
||||
String actualTable = getActualTable(where.getTableId());
|
||||
QueryWrapper wrapper = QueryWrapper.create();
|
||||
buildCondition(wrapper, where);
|
||||
|
||||
Page<Row> page = new Page<>(pageNumber, pageSize, count);
|
||||
Page<Row> paginate = Db.paginate(actualTable, page, wrapper);
|
||||
handleBigNumber(paginate.getRecords());
|
||||
return paginate;
|
||||
}
|
||||
|
||||
private void handleBigNumber(List<Row> records) {
|
||||
for (Row record : records) {
|
||||
Map<String, Object> newMap = new LinkedHashMap<>();
|
||||
for (Map.Entry<String, Object> entry : record.entrySet()) {
|
||||
Object value = entry.getValue();
|
||||
if ((value instanceof BigInteger ||
|
||||
value instanceof BigDecimal ||
|
||||
value instanceof Long)) {
|
||||
newMap.put(entry.getKey(), value.toString());
|
||||
} else {
|
||||
newMap.put(entry.getKey(), value);
|
||||
}
|
||||
}
|
||||
record.clear();
|
||||
record.putAll(newMap);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<HeaderVo> getHeaders(BigInteger tableId) {
|
||||
QueryWrapper wrapper = QueryWrapper.create();
|
||||
wrapper.eq(DatacenterTableField::getTableId, tableId);
|
||||
wrapper.orderBy("id");
|
||||
List<DatacenterTableField> fields = fieldsMapper.selectListByQuery(wrapper);
|
||||
List<HeaderVo> headers = new ArrayList<>();
|
||||
for (DatacenterTableField field : fields) {
|
||||
HeaderVo header = new HeaderVo();
|
||||
header.setKey(field.getFieldName());
|
||||
header.setDataIndex(field.getFieldName());
|
||||
header.setTitle(field.getFieldDesc());
|
||||
header.setFieldType(field.getFieldType());
|
||||
header.setRequired(field.getRequired());
|
||||
header.setFieldId(field.getId());
|
||||
header.setTableId(field.getTableId());
|
||||
headers.add(header);
|
||||
}
|
||||
return headers;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void saveValue(BigInteger tableId, JSONObject object, LoginAccount account) {
|
||||
|
||||
DatacenterTable table = getById(tableId);
|
||||
|
||||
QueryWrapper wrapper = QueryWrapper.create();
|
||||
wrapper.eq(DatacenterTableField::getTableId, tableId);
|
||||
List<DatacenterTableField> fields = fieldsMapper.selectListByQuery(wrapper);
|
||||
|
||||
if (CollectionUtil.isEmpty(fields)) {
|
||||
throw new BusinessException("请先添加字段");
|
||||
}
|
||||
table.setFields(fields);
|
||||
Object valueId = object.get("id");
|
||||
if (valueId == null) {
|
||||
dbHandleManager.getDbHandler().saveValue(table, object, account);
|
||||
} else {
|
||||
dbHandleManager.getDbHandler().updateValue(table, object, account);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void removeValue(BigInteger tableId, BigInteger id, LoginAccount account) {
|
||||
DatacenterTable record = getById(tableId);
|
||||
dbHandleManager.getDbHandler().removeValue(record, id, account);
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<DatacenterTableField> getFields(BigInteger tableId) {
|
||||
QueryWrapper wrapper = QueryWrapper.create();
|
||||
wrapper.eq(DatacenterTableField::getTableId, tableId);
|
||||
return fieldsMapper.selectListByQuery(wrapper);
|
||||
}
|
||||
|
||||
private String getActualTable(BigInteger tableId) {
|
||||
DatacenterTable record = getById(tableId);
|
||||
return record.getActualTable();
|
||||
}
|
||||
|
||||
private String getActualTableName(DatacenterTable table) {
|
||||
String tableName = table.getTableName();
|
||||
BigInteger id = table.getId();
|
||||
return "tb_dynamic_" + tableName + "_" + id;
|
||||
}
|
||||
|
||||
private String buildTableLockKey(DatacenterTable table, LoginAccount loginUser) {
|
||||
if (table.getId() != null) {
|
||||
return DATACENTER_TABLE_LOCK_KEY_PREFIX + table.getId();
|
||||
}
|
||||
String tenant = table.getTenantId() != null
|
||||
? table.getTenantId().toString()
|
||||
: (loginUser != null && loginUser.getTenantId() != null ? loginUser.getTenantId().toString() : "0");
|
||||
String tableName = table.getTableName() == null ? "unknown" : table.getTableName();
|
||||
return DATACENTER_TABLE_CREATE_LOCK_KEY_PREFIX + tenant + ":" + tableName;
|
||||
}
|
||||
|
||||
/**
|
||||
* 构建查询条件
|
||||
*/
|
||||
private void buildCondition(QueryWrapper wrapper, DatacenterQuery where) {
|
||||
// 构建查询条件
|
||||
String condition = where.getWhere();
|
||||
if (StrUtil.isNotEmpty(condition)) {
|
||||
wrapper.where(condition);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,223 @@
|
||||
package tech.easyflow.datacenter.utils;
|
||||
|
||||
import net.sf.jsqlparser.JSQLParserException;
|
||||
import net.sf.jsqlparser.parser.CCJSqlParserUtil;
|
||||
import net.sf.jsqlparser.schema.Table;
|
||||
import net.sf.jsqlparser.statement.Statement;
|
||||
import net.sf.jsqlparser.statement.Statements;
|
||||
import net.sf.jsqlparser.statement.select.Select;
|
||||
import net.sf.jsqlparser.statement.select.WithItem;
|
||||
import net.sf.jsqlparser.util.TablesNamesFinder;
|
||||
import tech.easyflow.common.web.exceptions.BusinessException;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collection;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.LinkedHashSet;
|
||||
import java.util.List;
|
||||
import java.util.Locale;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
|
||||
public final class SqlSupportUtils {
|
||||
|
||||
private SqlSupportUtils() {
|
||||
}
|
||||
|
||||
public static ResolvedSql resolve(String sql, Collection<ManagedTable> managedTables) {
|
||||
String normalizedSql = normalizeSql(sql);
|
||||
Statement statement = parseSingleStatement(normalizedSql);
|
||||
if (!(statement instanceof Select select)) {
|
||||
throw new BusinessException("查询数据节点只支持只读 SELECT SQL");
|
||||
}
|
||||
Map<String, List<ManagedTable>> byTableName = new LinkedHashMap<>();
|
||||
Map<String, ManagedTable> byCatalogAndTable = new LinkedHashMap<>();
|
||||
for (ManagedTable managedTable : managedTables) {
|
||||
if (managedTable == null || !hasText(managedTable.getTableName())) {
|
||||
continue;
|
||||
}
|
||||
String tableKey = normalizeIdentifier(managedTable.getTableName());
|
||||
byTableName.computeIfAbsent(tableKey, key -> new ArrayList<>()).add(managedTable);
|
||||
if (hasText(managedTable.getCatalogName())) {
|
||||
byCatalogAndTable.put(
|
||||
catalogTableKey(managedTable.getCatalogName(), managedTable.getTableName()),
|
||||
managedTable
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
SqlTableCollector collector = new SqlTableCollector();
|
||||
List<Table> referencedTables = collector.collect(select);
|
||||
if (referencedTables.isEmpty()) {
|
||||
throw new BusinessException("SQL 必须引用至少一张已接入表");
|
||||
}
|
||||
Set<String> logicalTables = new LinkedHashSet<>();
|
||||
for (Table table : referencedTables) {
|
||||
ManagedTable managedTable = resolveManagedTable(table, byTableName, byCatalogAndTable);
|
||||
rewriteTable(table, managedTable);
|
||||
logicalTables.add(renderLogicalTable(managedTable));
|
||||
}
|
||||
return new ResolvedSql(select.toString(), new ArrayList<>(logicalTables));
|
||||
}
|
||||
|
||||
private static Statement parseSingleStatement(String sql) {
|
||||
try {
|
||||
Statements statements = CCJSqlParserUtil.parseStatements(sql);
|
||||
if (statements == null || statements.getStatements() == null || statements.getStatements().isEmpty()) {
|
||||
throw new BusinessException("SQL 不能为空");
|
||||
}
|
||||
if (statements.getStatements().size() != 1) {
|
||||
throw new BusinessException("查询数据节点仅支持单条 SQL");
|
||||
}
|
||||
return statements.getStatements().get(0);
|
||||
} catch (BusinessException e) {
|
||||
throw e;
|
||||
} catch (JSQLParserException e) {
|
||||
throw new BusinessException("SQL 解析失败,请检查语法");
|
||||
}
|
||||
}
|
||||
|
||||
private static ManagedTable resolveManagedTable(Table table,
|
||||
Map<String, List<ManagedTable>> byTableName,
|
||||
Map<String, ManagedTable> byCatalogAndTable) {
|
||||
String tableName = trimToNull(table.getName());
|
||||
if (!hasText(tableName)) {
|
||||
throw new BusinessException("SQL 包含无法识别的表名");
|
||||
}
|
||||
String catalogName = trimToNull(table.getSchemaName());
|
||||
if (hasText(catalogName)) {
|
||||
ManagedTable managedTable = byCatalogAndTable.get(catalogTableKey(catalogName, tableName));
|
||||
if (managedTable == null) {
|
||||
throw new BusinessException("SQL 引用了未接入表: " + catalogName + "." + tableName);
|
||||
}
|
||||
return managedTable;
|
||||
}
|
||||
List<ManagedTable> matches = byTableName.get(normalizeIdentifier(tableName));
|
||||
if (matches == null || matches.isEmpty()) {
|
||||
throw new BusinessException("SQL 引用了未接入表: " + tableName);
|
||||
}
|
||||
if (matches.size() > 1) {
|
||||
throw new BusinessException("SQL 引用了重名表,请使用 catalog.table: " + tableName);
|
||||
}
|
||||
return matches.get(0);
|
||||
}
|
||||
|
||||
private static void rewriteTable(Table table, ManagedTable managedTable) {
|
||||
table.setName(managedTable.getPhysicalTableName());
|
||||
table.setSchemaName(trimToNull(managedTable.getCatalogName()));
|
||||
}
|
||||
|
||||
private static String renderLogicalTable(ManagedTable managedTable) {
|
||||
if (!hasText(managedTable.getCatalogName())) {
|
||||
return managedTable.getTableName();
|
||||
}
|
||||
return managedTable.getCatalogName() + "." + managedTable.getTableName();
|
||||
}
|
||||
|
||||
private static String normalizeSql(String sql) {
|
||||
String normalized = trimToNull(sql);
|
||||
if (!hasText(normalized)) {
|
||||
throw new BusinessException("SQL 不能为空");
|
||||
}
|
||||
return normalized;
|
||||
}
|
||||
|
||||
private static String catalogTableKey(String catalogName, String tableName) {
|
||||
return normalizeIdentifier(catalogName) + "." + normalizeIdentifier(tableName);
|
||||
}
|
||||
|
||||
private static String normalizeIdentifier(String value) {
|
||||
String normalized = trimToNull(value);
|
||||
if (!hasText(normalized)) {
|
||||
return "";
|
||||
}
|
||||
if ((normalized.startsWith("`") && normalized.endsWith("`"))
|
||||
|| (normalized.startsWith("\"") && normalized.endsWith("\""))
|
||||
|| (normalized.startsWith("[") && normalized.endsWith("]"))) {
|
||||
normalized = normalized.substring(1, normalized.length() - 1);
|
||||
}
|
||||
return normalized.trim().toLowerCase(Locale.ROOT);
|
||||
}
|
||||
|
||||
private static boolean hasText(String value) {
|
||||
return value != null && !value.isBlank();
|
||||
}
|
||||
|
||||
private static String trimToNull(String value) {
|
||||
if (value == null) {
|
||||
return null;
|
||||
}
|
||||
String trimmed = value.trim();
|
||||
return trimmed.isEmpty() ? null : trimmed;
|
||||
}
|
||||
|
||||
public static class ManagedTable {
|
||||
private final String catalogName;
|
||||
private final String tableName;
|
||||
private final String physicalTableName;
|
||||
|
||||
public ManagedTable(String catalogName, String tableName, String physicalTableName) {
|
||||
this.catalogName = trimToNull(catalogName);
|
||||
this.tableName = trimToNull(tableName);
|
||||
this.physicalTableName = hasText(physicalTableName) ? physicalTableName.trim() : this.tableName;
|
||||
}
|
||||
|
||||
public String getCatalogName() {
|
||||
return catalogName;
|
||||
}
|
||||
|
||||
public String getTableName() {
|
||||
return tableName;
|
||||
}
|
||||
|
||||
public String getPhysicalTableName() {
|
||||
return physicalTableName;
|
||||
}
|
||||
}
|
||||
|
||||
public static class ResolvedSql {
|
||||
private final String executableSql;
|
||||
private final List<String> logicalTables;
|
||||
|
||||
public ResolvedSql(String executableSql, List<String> logicalTables) {
|
||||
this.executableSql = executableSql;
|
||||
this.logicalTables = logicalTables;
|
||||
}
|
||||
|
||||
public String getExecutableSql() {
|
||||
return executableSql;
|
||||
}
|
||||
|
||||
public List<String> getLogicalTables() {
|
||||
return logicalTables;
|
||||
}
|
||||
}
|
||||
|
||||
private static class SqlTableCollector extends TablesNamesFinder {
|
||||
private final List<Table> tables = new ArrayList<>();
|
||||
private final Set<String> withNames = new LinkedHashSet<>();
|
||||
|
||||
public List<Table> collect(Statement statement) {
|
||||
tables.clear();
|
||||
withNames.clear();
|
||||
statement.accept(this);
|
||||
return new ArrayList<>(tables);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void visit(WithItem withItem) {
|
||||
if (withItem.getAlias() != null && hasText(withItem.getAlias().getName())) {
|
||||
withNames.add(normalizeIdentifier(withItem.getAlias().getName()));
|
||||
}
|
||||
withItem.getSelect().accept((net.sf.jsqlparser.statement.select.SelectVisitor) this);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void visit(Table table) {
|
||||
String tableName = normalizeIdentifier(table.getName());
|
||||
if (!withNames.contains(tableName)) {
|
||||
tables.add(table);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user