From 798effbd5b5fb356eba5537019836a6d7db41f3f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=99=88=E5=AD=90=E9=BB=98?= <925456043@qq.com> Date: Thu, 2 Apr 2026 18:55:31 +0800 Subject: [PATCH] =?UTF-8?q?feat:=20=E9=87=8D=E6=9E=84=E6=95=B0=E6=8D=AE?= =?UTF-8?q?=E4=B8=AD=E6=9E=A2=E5=B7=A5=E4=BD=9C=E5=8F=B0=E4=B8=8E=E6=8E=A5?= =?UTF-8?q?=E5=85=A5=E7=AE=A1=E7=90=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 新增统一的数据源、目录、纳管表与 Excel 处理后端能力 - 重建管理端数据中枢工作台并替换旧表管理页面 - 补充数据中枢迁移脚本、连接器底座与说明字段支持 --- .../DatacenterDatasetController.java | 68 ++ .../datacenter/DatacenterExcelController.java | 108 ++ .../DatacenterSourceController.java | 86 ++ .../datacenter/DatacenterTableController.java | 195 ---- .../DatacenterTableFieldsController.java | 41 - .../easyflow-module-datacenter/pom.xml | 13 + .../connector/DatacenterConnector.java | 12 + .../DatacenterConnectorRegistry.java | 29 + .../connector/MetadataExplorer.java | 16 + .../datacenter/connector/QueryExecutor.java | 15 + .../connector/SourceHealthChecker.java | 8 + .../datacenter/connector/SqlDialect.java | 9 + .../datacenter/connector/WriteExecutor.java | 14 + .../connector/dialect/GaussdbSqlDialect.java | 24 + .../connector/dialect/MysqlSqlDialect.java | 24 + .../connector/dialect/OracleSqlDialect.java | 24 + .../dialect/PostgresqlSqlDialect.java | 24 + .../connector/impl/ExcelConnector.java | 22 + .../impl/ExcelMaterializedConnector.java | 22 + .../impl/GaussdbNativeConnector.java | 40 + .../connector/impl/Gbase8aConnector.java | 40 + .../connector/impl/Gbase8sConnector.java | 40 + .../connector/impl/MysqlConnector.java | 40 + .../connector/impl/OracleConnector.java | 40 + .../connector/impl/PostgresqlConnector.java | 40 + .../connector/impl/ProjectMysqlConnector.java | 135 +++ .../AbstractInternalTableConnector.java | 152 +++ .../support/AbstractJdbcConnector.java | 523 ++++++++++ .../DatacenterConnectorExceptionSupport.java | 80 ++ .../support/DatacenterDatasourceManager.java | 44 + .../entity/base/DatacenterTableBase.java | 98 ++ .../entity/base/DatacenterTableFieldBase.java | 112 +++ .../datacenter/excel/ReadDataListener.java | 107 -- .../easyflow/datacenter/excel/ReadResVo.java | 68 -- .../model/DatacenterExcelDeriveRequest.java | 28 + .../model/DatacenterExcelExportRequest.java | 23 + .../model/DatacenterExcelMergeRequest.java | 22 + .../model/DatacenterExcelSplitRequest.java | 30 + .../service/DatacenterExcelImportService.java | 28 + .../DatacenterExcelImportServiceImpl.java | 950 ++++++++++++++++++ .../model/DatacenterConnectionTestResult.java | 24 + .../model/DatacenterQueryFilter.java | 19 + .../model/DatacenterQueryRequest.java | 29 + .../execution/model/DatacenterQuerySort.java | 11 + .../model/DatacenterSchemaResponse.java | 39 + .../model/DatacenterSqlQueryRequest.java | 23 + .../execution/model/DatasetRef.java | 25 + .../DatacenterDatasetQueryService.java | 18 + .../DatacenterDatasetWriteService.java | 13 + .../DatacenterDatasetQueryServiceImpl.java | 311 ++++++ .../DatacenterDatasetWriteServiceImpl.java | 48 + .../AssistantDatacenterBridge.java | 7 + .../AssistantDatacenterResult.java | 34 + .../DefaultAssistantDatacenterBridge.java | 49 + .../mapper/DatacenterCatalogMapper.java | 7 + .../DatacenterDatasetVersionMapper.java | 7 + .../mapper/DatacenterDerivedTableMapper.java | 7 + .../mapper/DatacenterImportJobMapper.java | 7 + .../mapper/DatacenterSourceMapper.java | 7 + .../meta/entity/DatacenterCatalog.java | 75 ++ .../meta/entity/DatacenterDatasetVersion.java | 70 ++ .../meta/entity/DatacenterDerivedTable.java | 66 ++ .../meta/entity/DatacenterImportJob.java | 102 ++ .../meta/entity/DatacenterSource.java | 131 +++ .../meta/enums/DatacenterAccessMode.java | 6 + .../meta/enums/DatacenterCapability.java | 10 + .../enums/DatacenterConnectionErrorCode.java | 12 + .../meta/enums/DatacenterImportStatus.java | 9 + .../meta/enums/DatacenterSourceType.java | 13 + .../meta/enums/DatacenterTableKind.java | 10 + .../model/DatacenterBatchRegisterRequest.java | 36 + .../model/DatacenterBatchRemoveRequest.java | 18 + .../meta/model/DatacenterCatalogMeta.java | 22 + .../DatacenterFieldDescriptionUpdate.java | 25 + .../model/DatacenterRemoveSourceRequest.java | 16 + .../DatacenterSaveDescriptionsRequest.java | 36 + .../meta/model/DatacenterTableDetailMeta.java | 17 + .../DatacenterDatasetRegistryService.java | 38 + .../meta/service/DatacenterMetaConstants.java | 12 + .../meta/service/DatacenterSourceService.java | 32 + .../DatacenterDatasetRegistryServiceImpl.java | 405 ++++++++ .../impl/DatacenterSourceServiceImpl.java | 311 ++++++ .../DatacenterSourceConnectionDefaults.java | 153 +++ .../security/DatacenterCredentialCipher.java | 28 + .../service/DatacenterTableFieldService.java | 14 - .../service/DatacenterTableService.java | 35 - .../impl/DatacenterTableFieldServiceImpl.java | 18 - .../impl/DatacenterTableServiceImpl.java | 293 ------ .../datacenter/utils/SqlSupportUtils.java | 223 ++++ .../V11__datacenter_unified_access_phase1.sql | 202 ++++ .../migration/V12__datacenter_phase2_menu.sql | 86 ++ .../V13__datacenter_remove_legacy_menu.sql | 20 + .../V14__datacenter_remove_local_dynamic.sql | 101 ++ easyflow-ui-admin/app/src/api/request.ts | 29 +- .../app/src/assets/datacenter/huawei-icon.svg | 1 + .../app/src/assets/datacenter/mysql-icon.svg | 1 + .../app/src/assets/datacenter/oracle-icon.svg | 1 + .../src/assets/datacenter/postgresql-icon.svg | 1 + .../src/router/routes/modules/datacenter.ts | 33 +- .../src/views/datacenter/BatchImportModal.vue | 154 --- .../datacenter/DatacenterTableDetail.vue | 272 ----- .../views/datacenter/DatacenterTableList.vue | 204 ---- .../views/datacenter/DatacenterTableModal.vue | 272 ----- .../views/datacenter/DatacenterWorkspace.vue | 539 ++++++++++ .../app/src/views/datacenter/RecordModal.vue | 145 --- .../datacenter/components/ConnectionTree.vue | 474 +++++++++ .../components/ExcelActionDrawer.vue | 210 ++++ .../datacenter/components/SourceBrandIcon.vue | 106 ++ .../components/SourceFormDrawer.vue | 364 +++++++ .../datacenter/components/TableDetailView.vue | 422 ++++++++ .../datacenter/components/TableListView.vue | 414 ++++++++ .../composables/datacenter-constants.ts | 163 +++ .../composables/use-connection-tree.ts | 69 ++ .../composables/use-datacenter-excel.ts | 253 +++++ .../composables/use-datacenter-sources.ts | 190 ++++ .../composables/use-datacenter-tables.ts | 334 ++++++ .../datacenter/composables/use-source-form.ts | 188 ++++ 117 files changed, 9739 insertions(+), 1824 deletions(-) create mode 100644 easyflow-api/easyflow-api-admin/src/main/java/tech/easyflow/admin/controller/datacenter/DatacenterDatasetController.java create mode 100644 easyflow-api/easyflow-api-admin/src/main/java/tech/easyflow/admin/controller/datacenter/DatacenterExcelController.java create mode 100644 easyflow-api/easyflow-api-admin/src/main/java/tech/easyflow/admin/controller/datacenter/DatacenterSourceController.java delete mode 100644 easyflow-api/easyflow-api-admin/src/main/java/tech/easyflow/admin/controller/datacenter/DatacenterTableController.java delete mode 100644 easyflow-api/easyflow-api-admin/src/main/java/tech/easyflow/admin/controller/datacenter/DatacenterTableFieldsController.java create mode 100644 easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/connector/DatacenterConnector.java create mode 100644 easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/connector/DatacenterConnectorRegistry.java create mode 100644 easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/connector/MetadataExplorer.java create mode 100644 easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/connector/QueryExecutor.java create mode 100644 easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/connector/SourceHealthChecker.java create mode 100644 easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/connector/SqlDialect.java create mode 100644 easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/connector/WriteExecutor.java create mode 100644 easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/connector/dialect/GaussdbSqlDialect.java create mode 100644 easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/connector/dialect/MysqlSqlDialect.java create mode 100644 easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/connector/dialect/OracleSqlDialect.java create mode 100644 easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/connector/dialect/PostgresqlSqlDialect.java create mode 100644 easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/connector/impl/ExcelConnector.java create mode 100644 easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/connector/impl/ExcelMaterializedConnector.java create mode 100644 easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/connector/impl/GaussdbNativeConnector.java create mode 100644 easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/connector/impl/Gbase8aConnector.java create mode 100644 easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/connector/impl/Gbase8sConnector.java create mode 100644 easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/connector/impl/MysqlConnector.java create mode 100644 easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/connector/impl/OracleConnector.java create mode 100644 easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/connector/impl/PostgresqlConnector.java create mode 100644 easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/connector/impl/ProjectMysqlConnector.java create mode 100644 easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/connector/support/AbstractInternalTableConnector.java create mode 100644 easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/connector/support/AbstractJdbcConnector.java create mode 100644 easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/connector/support/DatacenterConnectorExceptionSupport.java create mode 100644 easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/connector/support/DatacenterDatasourceManager.java delete mode 100644 easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/excel/ReadDataListener.java delete mode 100644 easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/excel/ReadResVo.java create mode 100644 easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/excel/model/DatacenterExcelDeriveRequest.java create mode 100644 easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/excel/model/DatacenterExcelExportRequest.java create mode 100644 easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/excel/model/DatacenterExcelMergeRequest.java create mode 100644 easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/excel/model/DatacenterExcelSplitRequest.java create mode 100644 easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/excel/service/DatacenterExcelImportService.java create mode 100644 easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/excel/service/impl/DatacenterExcelImportServiceImpl.java create mode 100644 easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/execution/model/DatacenterConnectionTestResult.java create mode 100644 easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/execution/model/DatacenterQueryFilter.java create mode 100644 easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/execution/model/DatacenterQueryRequest.java create mode 100644 easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/execution/model/DatacenterQuerySort.java create mode 100644 easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/execution/model/DatacenterSchemaResponse.java create mode 100644 easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/execution/model/DatacenterSqlQueryRequest.java create mode 100644 easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/execution/model/DatasetRef.java create mode 100644 easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/execution/service/DatacenterDatasetQueryService.java create mode 100644 easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/execution/service/DatacenterDatasetWriteService.java create mode 100644 easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/execution/service/impl/DatacenterDatasetQueryServiceImpl.java create mode 100644 easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/execution/service/impl/DatacenterDatasetWriteServiceImpl.java create mode 100644 easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/integration/AssistantDatacenterBridge.java create mode 100644 easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/integration/AssistantDatacenterResult.java create mode 100644 easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/integration/DefaultAssistantDatacenterBridge.java create mode 100644 easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/mapper/DatacenterCatalogMapper.java create mode 100644 easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/mapper/DatacenterDatasetVersionMapper.java create mode 100644 easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/mapper/DatacenterDerivedTableMapper.java create mode 100644 easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/mapper/DatacenterImportJobMapper.java create mode 100644 easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/mapper/DatacenterSourceMapper.java create mode 100644 easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/meta/entity/DatacenterCatalog.java create mode 100644 easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/meta/entity/DatacenterDatasetVersion.java create mode 100644 easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/meta/entity/DatacenterDerivedTable.java create mode 100644 easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/meta/entity/DatacenterImportJob.java create mode 100644 easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/meta/entity/DatacenterSource.java create mode 100644 easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/meta/enums/DatacenterAccessMode.java create mode 100644 easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/meta/enums/DatacenterCapability.java create mode 100644 easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/meta/enums/DatacenterConnectionErrorCode.java create mode 100644 easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/meta/enums/DatacenterImportStatus.java create mode 100644 easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/meta/enums/DatacenterSourceType.java create mode 100644 easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/meta/enums/DatacenterTableKind.java create mode 100644 easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/meta/model/DatacenterBatchRegisterRequest.java create mode 100644 easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/meta/model/DatacenterBatchRemoveRequest.java create mode 100644 easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/meta/model/DatacenterCatalogMeta.java create mode 100644 easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/meta/model/DatacenterFieldDescriptionUpdate.java create mode 100644 easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/meta/model/DatacenterRemoveSourceRequest.java create mode 100644 easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/meta/model/DatacenterSaveDescriptionsRequest.java create mode 100644 easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/meta/model/DatacenterTableDetailMeta.java create mode 100644 easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/meta/service/DatacenterDatasetRegistryService.java create mode 100644 easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/meta/service/DatacenterMetaConstants.java create mode 100644 easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/meta/service/DatacenterSourceService.java create mode 100644 easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/meta/service/impl/DatacenterDatasetRegistryServiceImpl.java create mode 100644 easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/meta/service/impl/DatacenterSourceServiceImpl.java create mode 100644 easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/meta/support/DatacenterSourceConnectionDefaults.java create mode 100644 easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/security/DatacenterCredentialCipher.java delete mode 100644 easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/service/DatacenterTableFieldService.java delete mode 100644 easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/service/DatacenterTableService.java delete mode 100644 easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/service/impl/DatacenterTableFieldServiceImpl.java delete mode 100644 easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/service/impl/DatacenterTableServiceImpl.java create mode 100644 easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/utils/SqlSupportUtils.java create mode 100644 easyflow-starter/easyflow-starter-all/src/main/resources/db/migration/V11__datacenter_unified_access_phase1.sql create mode 100644 easyflow-starter/easyflow-starter-all/src/main/resources/db/migration/V12__datacenter_phase2_menu.sql create mode 100644 easyflow-starter/easyflow-starter-all/src/main/resources/db/migration/V13__datacenter_remove_legacy_menu.sql create mode 100644 easyflow-starter/easyflow-starter-all/src/main/resources/db/migration/V14__datacenter_remove_local_dynamic.sql create mode 100644 easyflow-ui-admin/app/src/assets/datacenter/huawei-icon.svg create mode 100644 easyflow-ui-admin/app/src/assets/datacenter/mysql-icon.svg create mode 100644 easyflow-ui-admin/app/src/assets/datacenter/oracle-icon.svg create mode 100644 easyflow-ui-admin/app/src/assets/datacenter/postgresql-icon.svg delete mode 100644 easyflow-ui-admin/app/src/views/datacenter/BatchImportModal.vue delete mode 100644 easyflow-ui-admin/app/src/views/datacenter/DatacenterTableDetail.vue delete mode 100644 easyflow-ui-admin/app/src/views/datacenter/DatacenterTableList.vue delete mode 100644 easyflow-ui-admin/app/src/views/datacenter/DatacenterTableModal.vue create mode 100644 easyflow-ui-admin/app/src/views/datacenter/DatacenterWorkspace.vue delete mode 100644 easyflow-ui-admin/app/src/views/datacenter/RecordModal.vue create mode 100644 easyflow-ui-admin/app/src/views/datacenter/components/ConnectionTree.vue create mode 100644 easyflow-ui-admin/app/src/views/datacenter/components/ExcelActionDrawer.vue create mode 100644 easyflow-ui-admin/app/src/views/datacenter/components/SourceBrandIcon.vue create mode 100644 easyflow-ui-admin/app/src/views/datacenter/components/SourceFormDrawer.vue create mode 100644 easyflow-ui-admin/app/src/views/datacenter/components/TableDetailView.vue create mode 100644 easyflow-ui-admin/app/src/views/datacenter/components/TableListView.vue create mode 100644 easyflow-ui-admin/app/src/views/datacenter/composables/datacenter-constants.ts create mode 100644 easyflow-ui-admin/app/src/views/datacenter/composables/use-connection-tree.ts create mode 100644 easyflow-ui-admin/app/src/views/datacenter/composables/use-datacenter-excel.ts create mode 100644 easyflow-ui-admin/app/src/views/datacenter/composables/use-datacenter-sources.ts create mode 100644 easyflow-ui-admin/app/src/views/datacenter/composables/use-datacenter-tables.ts create mode 100644 easyflow-ui-admin/app/src/views/datacenter/composables/use-source-form.ts diff --git a/easyflow-api/easyflow-api-admin/src/main/java/tech/easyflow/admin/controller/datacenter/DatacenterDatasetController.java b/easyflow-api/easyflow-api-admin/src/main/java/tech/easyflow/admin/controller/datacenter/DatacenterDatasetController.java new file mode 100644 index 0000000..a6ec3e4 --- /dev/null +++ b/easyflow-api/easyflow-api-admin/src/main/java/tech/easyflow/admin/controller/datacenter/DatacenterDatasetController.java @@ -0,0 +1,68 @@ +package tech.easyflow.admin.controller.datacenter; + +import cn.dev33.satoken.annotation.SaCheckPermission; +import com.mybatisflex.core.paginate.Page; +import com.mybatisflex.core.row.Row; +import org.springframework.web.bind.annotation.*; +import tech.easyflow.common.domain.Result; +import tech.easyflow.common.entity.LoginAccount; +import tech.easyflow.common.satoken.util.SaTokenUtil; +import tech.easyflow.datacenter.entity.DatacenterTable; +import tech.easyflow.datacenter.execution.model.DatacenterQueryRequest; +import tech.easyflow.datacenter.execution.model.DatacenterSchemaResponse; +import tech.easyflow.datacenter.execution.model.DatasetRef; +import tech.easyflow.datacenter.meta.model.DatacenterBatchRemoveRequest; +import tech.easyflow.datacenter.meta.model.DatacenterSaveDescriptionsRequest; +import tech.easyflow.datacenter.execution.service.DatacenterDatasetQueryService; +import tech.easyflow.datacenter.meta.service.DatacenterDatasetRegistryService; + +import javax.annotation.Resource; +import java.math.BigInteger; +import java.util.List; + +@RestController +@RequestMapping("/api/v1/datacenterDataset") +public class DatacenterDatasetController { + + @Resource + private DatacenterDatasetQueryService queryService; + @Resource + private DatacenterDatasetRegistryService registryService; + + @PostMapping("/queryPage") + @SaCheckPermission("/api/v1/datacenterSource/query") + public Result> queryPage(@RequestBody DatacenterQueryRequest request) { + return Result.ok(queryService.queryPage(request)); + } + + @GetMapping("/schema") + @SaCheckPermission("/api/v1/datacenterSource/query") + public Result schema(DatasetRef datasetRef) { + return Result.ok(queryService.getSchema(datasetRef)); + } + + @GetMapping("/managedTables") + @SaCheckPermission("/api/v1/datacenterSource/query") + public Result> managedTables(BigInteger sourceId, BigInteger catalogId) { + return Result.ok(registryService.listManagedTables(sourceId, catalogId)); + } + + @PostMapping("/removeBatch") + @SaCheckPermission("/api/v1/datacenterSource/save") + public Result removeBatch(@RequestBody DatacenterBatchRemoveRequest request) { + return Result.ok(registryService.removeTables(request == null ? List.of() : request.getTableIds())); + } + + @PostMapping("/saveDescriptions") + @SaCheckPermission("/api/v1/datacenterSource/save") + public Result saveDescriptions(@RequestBody DatacenterSaveDescriptionsRequest request) { + LoginAccount account = SaTokenUtil.getLoginAccount(); + DatacenterTable table = registryService.saveDescriptions( + request == null ? null : request.getTableId(), + request == null ? null : request.getTableDesc(), + request == null ? List.of() : request.getFields(), + account + ); + return Result.ok(queryService.getSchema(registryService.resolveDatasetRef(table.getId()))); + } +} diff --git a/easyflow-api/easyflow-api-admin/src/main/java/tech/easyflow/admin/controller/datacenter/DatacenterExcelController.java b/easyflow-api/easyflow-api-admin/src/main/java/tech/easyflow/admin/controller/datacenter/DatacenterExcelController.java new file mode 100644 index 0000000..c4d299e --- /dev/null +++ b/easyflow-api/easyflow-api-admin/src/main/java/tech/easyflow/admin/controller/datacenter/DatacenterExcelController.java @@ -0,0 +1,108 @@ +package tech.easyflow.admin.controller.datacenter; + +import cn.dev33.satoken.annotation.SaCheckPermission; +import jakarta.servlet.http.HttpServletResponse; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RestController; +import org.springframework.web.multipart.MultipartFile; +import tech.easyflow.common.domain.Result; +import tech.easyflow.common.entity.LoginAccount; +import tech.easyflow.common.satoken.util.SaTokenUtil; +import tech.easyflow.datacenter.excel.model.DatacenterExcelDeriveRequest; +import tech.easyflow.datacenter.excel.model.DatacenterExcelExportRequest; +import tech.easyflow.datacenter.excel.model.DatacenterExcelMergeRequest; +import tech.easyflow.datacenter.excel.model.DatacenterExcelSplitRequest; +import tech.easyflow.datacenter.excel.service.DatacenterExcelImportService; +import tech.easyflow.datacenter.meta.entity.DatacenterImportJob; + +import java.io.File; +import java.io.FileInputStream; +import javax.annotation.Resource; +import java.math.BigInteger; +import java.net.URLEncoder; +import java.util.List; + +@RestController +@RequestMapping("/api/v1/datacenterExcel") +public class DatacenterExcelController { + + @Resource + private DatacenterExcelImportService excelImportService; + + @PostMapping("/import") + @SaCheckPermission("/api/v1/datacenterSource/save") + public Result importWorkbook(MultipartFile file) throws Exception { + LoginAccount account = SaTokenUtil.getLoginAccount(); + return Result.ok(excelImportService.importWorkbook(file, account)); + } + + @PostMapping("/split") + @SaCheckPermission("/api/v1/datacenterSource/save") + public Result split(@RequestBody DatacenterExcelSplitRequest request) { + LoginAccount account = SaTokenUtil.getLoginAccount(); + return Result.ok(excelImportService.splitWorkbook(request, account)); + } + + @PostMapping("/merge") + @SaCheckPermission("/api/v1/datacenterSource/save") + public Result merge(@RequestBody DatacenterExcelMergeRequest request) { + LoginAccount account = SaTokenUtil.getLoginAccount(); + return Result.ok(excelImportService.mergeWorkbook(request, account)); + } + + @PostMapping("/derive") + @SaCheckPermission("/api/v1/datacenterSource/save") + public Result derive(@RequestBody DatacenterExcelDeriveRequest request) { + LoginAccount account = SaTokenUtil.getLoginAccount(); + return Result.ok(excelImportService.deriveWorkbook(request, account)); + } + + @PostMapping("/export") + @SaCheckPermission("/api/v1/datacenterSource/query") + public Result export(@RequestBody DatacenterExcelExportRequest request) throws Exception { + LoginAccount account = SaTokenUtil.getLoginAccount(); + return Result.ok(excelImportService.exportWorkbook(request, account)); + } + + @GetMapping("/importJob/detail") + @SaCheckPermission("/api/v1/datacenterSource/query") + public Result jobDetail(BigInteger jobId) { + return Result.ok(excelImportService.getImportJobDetail(jobId)); + } + + @GetMapping("/job/detail") + @SaCheckPermission("/api/v1/datacenterSource/query") + public Result newJobDetail(BigInteger jobId) { + return Result.ok(excelImportService.getImportJobDetail(jobId)); + } + + @GetMapping("/job/list") + @SaCheckPermission("/api/v1/datacenterSource/query") + public Result> jobList(BigInteger sourceId, BigInteger tableId) { + return Result.ok(excelImportService.listJobs(sourceId, tableId)); + } + + @GetMapping("/download") + @SaCheckPermission("/api/v1/datacenterSource/query") + public void download(BigInteger jobId, HttpServletResponse response) throws Exception { + DatacenterImportJob job = excelImportService.getImportJobDetail(jobId); + if (job.getStoragePath() == null || job.getStoragePath().isBlank()) { + throw new IllegalStateException("导出文件不存在"); + } + File file = new File(job.getStoragePath()); + if (!file.exists()) { + throw new IllegalStateException("导出文件不存在"); + } + response.setContentType("application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"); + response.setCharacterEncoding("utf-8"); + String fileName = URLEncoder.encode(job.getFileName(), "UTF-8").replaceAll("\\+", "%20"); + response.setHeader("Content-disposition", "attachment;filename*=utf-8''" + fileName); + try (FileInputStream inputStream = new FileInputStream(file)) { + inputStream.transferTo(response.getOutputStream()); + response.flushBuffer(); + } + } +} diff --git a/easyflow-api/easyflow-api-admin/src/main/java/tech/easyflow/admin/controller/datacenter/DatacenterSourceController.java b/easyflow-api/easyflow-api-admin/src/main/java/tech/easyflow/admin/controller/datacenter/DatacenterSourceController.java new file mode 100644 index 0000000..47d013a --- /dev/null +++ b/easyflow-api/easyflow-api-admin/src/main/java/tech/easyflow/admin/controller/datacenter/DatacenterSourceController.java @@ -0,0 +1,86 @@ +package tech.easyflow.admin.controller.datacenter; + +import cn.dev33.satoken.annotation.SaCheckPermission; +import com.mybatisflex.core.paginate.Page; +import org.springframework.web.bind.annotation.*; +import tech.easyflow.common.domain.Result; +import tech.easyflow.common.entity.LoginAccount; +import tech.easyflow.common.satoken.util.SaTokenUtil; +import tech.easyflow.datacenter.entity.DatacenterTable; +import tech.easyflow.datacenter.execution.model.DatacenterConnectionTestResult; +import tech.easyflow.datacenter.meta.entity.DatacenterSource; +import tech.easyflow.datacenter.meta.model.DatacenterBatchRegisterRequest; +import tech.easyflow.datacenter.meta.model.DatacenterCatalogMeta; +import tech.easyflow.datacenter.meta.model.DatacenterRemoveSourceRequest; +import tech.easyflow.datacenter.meta.model.DatacenterTableDetailMeta; +import tech.easyflow.datacenter.meta.service.DatacenterSourceService; + +import javax.annotation.Resource; +import java.math.BigInteger; +import java.util.List; + +@RestController +@RequestMapping("/api/v1/datacenterSource") +public class DatacenterSourceController { + + @Resource + private DatacenterSourceService sourceService; + + @PostMapping("/testConnection") + @SaCheckPermission("/api/v1/datacenterSource/query") + public Result testConnection(@RequestBody DatacenterSource source) { + LoginAccount account = SaTokenUtil.getLoginAccount(); + return Result.ok(sourceService.testConnection(source, account)); + } + + @PostMapping("/save") + @SaCheckPermission("/api/v1/datacenterSource/save") + public Result save(@RequestBody DatacenterSource source) { + LoginAccount account = SaTokenUtil.getLoginAccount(); + return Result.ok(sourceService.saveSource(source, account)); + } + + @GetMapping("/page") + @SaCheckPermission("/api/v1/datacenterSource/query") + public Result> page(Long pageNumber, Long pageSize) { + LoginAccount account = SaTokenUtil.getLoginAccount(); + return Result.ok(sourceService.pageSources(pageNumber, pageSize, account)); + } + + @GetMapping("/catalogs") + @SaCheckPermission("/api/v1/datacenterSource/query") + public Result> catalogs(BigInteger sourceId) { + LoginAccount account = SaTokenUtil.getLoginAccount(); + return Result.ok(sourceService.listCatalogs(sourceId, account)); + } + + @GetMapping("/tables") + @SaCheckPermission("/api/v1/datacenterSource/query") + public Result> tables(BigInteger sourceId, String catalogName) { + LoginAccount account = SaTokenUtil.getLoginAccount(); + return Result.ok(sourceService.listTables(sourceId, catalogName, account)); + } + + @GetMapping("/tableDetail") + @SaCheckPermission("/api/v1/datacenterSource/query") + public Result tableDetail(BigInteger sourceId, String catalogName, String tableName, + @RequestParam(defaultValue = "false") boolean register) { + LoginAccount account = SaTokenUtil.getLoginAccount(); + return Result.ok(sourceService.getTableDetail(sourceId, catalogName, tableName, register, account)); + } + + @PostMapping("/registerBatch") + @SaCheckPermission("/api/v1/datacenterSource/save") + public Result> registerBatch(@RequestBody DatacenterBatchRegisterRequest request) { + LoginAccount account = SaTokenUtil.getLoginAccount(); + return Result.ok(sourceService.batchRegisterTables(request, account)); + } + + @PostMapping("/remove") + @SaCheckPermission("/api/v1/datacenterSource/save") + public Result remove(@RequestBody DatacenterRemoveSourceRequest request) { + LoginAccount account = SaTokenUtil.getLoginAccount(); + sourceService.removeSource(request == null ? null : request.getSourceId(), account); + return Result.ok(); + } +} diff --git a/easyflow-api/easyflow-api-admin/src/main/java/tech/easyflow/admin/controller/datacenter/DatacenterTableController.java b/easyflow-api/easyflow-api-admin/src/main/java/tech/easyflow/admin/controller/datacenter/DatacenterTableController.java deleted file mode 100644 index 7550bf2..0000000 --- a/easyflow-api/easyflow-api-admin/src/main/java/tech/easyflow/admin/controller/datacenter/DatacenterTableController.java +++ /dev/null @@ -1,195 +0,0 @@ -package tech.easyflow.admin.controller.datacenter; - -import cn.dev33.satoken.annotation.SaCheckPermission; -import cn.hutool.core.collection.CollectionUtil; -import cn.idev.excel.EasyExcel; -import cn.idev.excel.FastExcel; -import com.alibaba.fastjson2.JSONObject; -import com.mybatisflex.core.paginate.Page; -import com.mybatisflex.core.query.QueryWrapper; -import com.mybatisflex.core.row.Row; -import jakarta.servlet.http.HttpServletResponse; -import org.springframework.web.bind.annotation.*; -import org.springframework.web.multipart.MultipartFile; -import tech.easyflow.common.domain.Result; -import tech.easyflow.common.entity.DatacenterQuery; -import tech.easyflow.common.entity.LoginAccount; -import tech.easyflow.common.satoken.util.SaTokenUtil; -import tech.easyflow.common.web.controller.BaseCurdController; -import tech.easyflow.datacenter.entity.DatacenterTable; -import tech.easyflow.datacenter.entity.DatacenterTableField; -import tech.easyflow.datacenter.entity.vo.HeaderVo; -import tech.easyflow.datacenter.excel.ReadDataListener; -import tech.easyflow.datacenter.excel.ReadResVo; -import tech.easyflow.datacenter.service.DatacenterTableFieldService; -import tech.easyflow.datacenter.service.DatacenterTableService; - -import javax.annotation.Resource; -import java.io.InputStream; -import java.math.BigInteger; -import java.net.URLEncoder; -import java.util.ArrayList; -import java.util.Date; -import java.util.List; -import java.util.Map; - -/** - * 数据中枢表 控制层。 - * - * @author ArkLight - * @since 2025-07-10 - */ -@RestController -@RequestMapping("/api/v1/datacenterTable") -public class DatacenterTableController extends BaseCurdController { - - @Resource - private DatacenterTableFieldService fieldsService; - - public DatacenterTableController(DatacenterTableService service) { - super(service); - } - - @Override - protected Result onSaveOrUpdateBefore(DatacenterTable entity, boolean isSave) { - LoginAccount loginUser = SaTokenUtil.getLoginAccount(); - if (isSave) { - commonFiled(entity, loginUser.getId(), loginUser.getTenantId(), loginUser.getDeptId()); - } else { - entity.setModifiedBy(loginUser.getId()); - } - return super.onSaveOrUpdateBefore(entity, isSave); - } - - @PostMapping("/saveTable") - @SaCheckPermission("/api/v1/datacenterTable/save") - public Result saveTable(@RequestBody DatacenterTable entity) { - LoginAccount loginUser = SaTokenUtil.getLoginAccount(); - List fields = entity.getFields(); - if (CollectionUtil.isEmpty(fields)) { - return Result.fail(99, "字段不能为空"); - } - BigInteger id = entity.getId(); - if (id == null) { - commonFiled(entity, loginUser.getId(), loginUser.getTenantId(), loginUser.getDeptId()); - } else { - entity.setModified(new Date()); - entity.setModifiedBy(loginUser.getId()); - } - service.saveTable(entity, loginUser); - return Result.ok(); - } - - @GetMapping("/detailInfo") - @SaCheckPermission("/api/v1/datacenterTable/query") - public Result detailInfo(BigInteger tableId) { - DatacenterTable table = service.getById(tableId); - QueryWrapper wrapper = QueryWrapper.create(); - wrapper.eq(DatacenterTableField::getTableId, tableId); - wrapper.orderBy("id"); - List fields = fieldsService.list(wrapper); - table.setFields(fields); - return Result.ok(table); - } - - @GetMapping("/removeTable") - @SaCheckPermission("/api/v1/datacenterTable/remove") - public Result removeTable(BigInteger tableId) { - service.removeTable(tableId); - return Result.ok(); - } - - @GetMapping("/getHeaders") - @SaCheckPermission("/api/v1/datacenterTable/query") - public Result> getHeaders(BigInteger tableId) { - List res = service.getHeaders(tableId); - return Result.ok(res); - } - - @GetMapping("/getPageData") - @SaCheckPermission("/api/v1/datacenterTable/query") - public Result> getPageData(DatacenterQuery where) { - Page res = service.getPageData(where); - return Result.ok(res); - } - - @PostMapping("/saveValue") - @SaCheckPermission("/api/v1/datacenterTable/save") - public Result saveValue(@RequestParam Map map) { - JSONObject object = new JSONObject(map); - BigInteger tableId = object.getBigInteger("tableId"); - LoginAccount account = SaTokenUtil.getLoginAccount(); - if (tableId == null) { - return Result.fail(99, "参数错误"); - } - service.saveValue(tableId, object, account); - return Result.ok(); - } - - @PostMapping("/removeValue") - @SaCheckPermission("/api/v1/datacenterTable/remove") - public Result removeValue(@RequestParam Map map) { - JSONObject object = new JSONObject(map); - BigInteger tableId = object.getBigInteger("tableId"); - BigInteger id = object.getBigInteger("id"); - if (tableId == null || id == null) { - return Result.fail(99, "参数错误"); - } - LoginAccount account = SaTokenUtil.getLoginAccount(); - service.removeValue(tableId, id, account); - return Result.ok(); - } - - /** - * 导入数据 - */ - @PostMapping("/importData") - @SaCheckPermission("/api/v1/datacenterTable/save") - public Result importData(MultipartFile file, @RequestParam Map map) throws Exception { - Object tableId = map.get("tableId"); - DatacenterTable record = service.getById(tableId.toString()); - if (record == null) { - throw new RuntimeException("数据表不存在"); - } - InputStream is = file.getInputStream(); - List fields = service.getFields(record.getId()); - - LoginAccount account = SaTokenUtil.getLoginAccount(); - ReadDataListener listener = new ReadDataListener(record.getId(), fields, account); - FastExcel.read(is, listener) - .sheet() - .doRead(); - int totalCount = listener.getTotalCount(); - int errorCount = listener.getErrorCount(); - int successCount = listener.getSuccessCount(); - List errorRows = listener.getErrorRows(); - return Result.ok(new ReadResVo(successCount, errorCount, totalCount, errorRows)); - } - - @GetMapping("/getTemplate") - public void getTemplate(BigInteger tableId, HttpServletResponse response) throws Exception { - List fields = service.getFields(tableId); - // 设置响应内容类型 - response.setContentType("application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"); - response.setCharacterEncoding("utf-8"); - - // 设置文件名 - String fileName = URLEncoder.encode("导入模板", "UTF-8").replaceAll("\\+", "%20"); - response.setHeader("Content-disposition", "attachment;filename*=utf-8''" + fileName + ".xlsx"); - - // 动态表头数据 - List> headList = new ArrayList<>(); - - for (DatacenterTableField field : fields) { - List head = new ArrayList<>(); - head.add(field.getFieldName()); - headList.add(head); - } - - // 写入Excel - EasyExcel.write(response.getOutputStream()) - .head(headList) - .sheet("模板") - .doWrite(new ArrayList<>()); // 写入空数据，只生成模板 - } -} \ No newline at end of file diff --git a/easyflow-api/easyflow-api-admin/src/main/java/tech/easyflow/admin/controller/datacenter/DatacenterTableFieldsController.java b/easyflow-api/easyflow-api-admin/src/main/java/tech/easyflow/admin/controller/datacenter/DatacenterTableFieldsController.java deleted file mode 100644 index 28a23eb..0000000 --- a/easyflow-api/easyflow-api-admin/src/main/java/tech/easyflow/admin/controller/datacenter/DatacenterTableFieldsController.java +++ /dev/null @@ -1,41 +0,0 @@ -package tech.easyflow.admin.controller.datacenter; - -import org.springframework.web.bind.annotation.RequestMapping; -import org.springframework.web.bind.annotation.RestController; -import tech.easyflow.common.annotation.UsePermission; -import tech.easyflow.common.domain.Result; -import tech.easyflow.common.entity.LoginAccount; -import tech.easyflow.common.satoken.util.SaTokenUtil; -import tech.easyflow.common.web.controller.BaseCurdController; -import tech.easyflow.datacenter.entity.DatacenterTableField; -import tech.easyflow.datacenter.service.DatacenterTableFieldService; - -import java.util.Date; - -/** - * 控制层。 - * - * @author ArkLight - * @since 2025-07-10 - */ -@RestController -@RequestMapping("/api/v1/datacenterTableFields") -@UsePermission(moduleName = "/api/v1/datacenterTable") -public class DatacenterTableFieldsController extends BaseCurdController { - - public DatacenterTableFieldsController(DatacenterTableFieldService service) { - super(service); - } - - @Override - protected Result onSaveOrUpdateBefore(DatacenterTableField entity, boolean isSave) { - LoginAccount loginUser = SaTokenUtil.getLoginAccount(); - if (isSave) { - commonFiled(entity, loginUser.getId(), loginUser.getTenantId(), loginUser.getDeptId()); - } else { - entity.setModified(new Date()); - entity.setModifiedBy(loginUser.getId()); - } - return null; - } -} \ No newline at end of file diff --git a/easyflow-modules/easyflow-module-datacenter/pom.xml b/easyflow-modules/easyflow-module-datacenter/pom.xml index 0dbaeb5..daa1a70 100644 --- a/easyflow-modules/easyflow-module-datacenter/pom.xml +++ b/easyflow-modules/easyflow-module-datacenter/pom.xml @@ -16,6 +16,19 @@ com.mybatis-flex mybatis-flex-spring-boot3-starter + + com.zaxxer + HikariCP + + + org.postgresql + postgresql + 42.7.3 + + + cn.hutool + hutool-crypto + tech.easyflow easyflow-common-base diff --git a/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/connector/DatacenterConnector.java b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/connector/DatacenterConnector.java new file mode 100644 index 0000000..995fd6b --- /dev/null +++ b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/connector/DatacenterConnector.java @@ -0,0 +1,12 @@ +package tech.easyflow.datacenter.connector; + +import tech.easyflow.datacenter.meta.enums.DatacenterCapability; +import tech.easyflow.datacenter.meta.enums.DatacenterSourceType; + +import java.util.Set; + +public interface DatacenterConnector extends SourceHealthChecker, MetadataExplorer, QueryExecutor, WriteExecutor { + DatacenterSourceType getSourceType(); + + Set getCapabilities(); +} diff --git a/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/connector/DatacenterConnectorRegistry.java b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/connector/DatacenterConnectorRegistry.java new file mode 100644 index 0000000..52c5a21 --- /dev/null +++ b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/connector/DatacenterConnectorRegistry.java @@ -0,0 +1,29 @@ +package tech.easyflow.datacenter.connector; + +import org.springframework.stereotype.Component; +import tech.easyflow.common.web.exceptions.BusinessException; +import tech.easyflow.datacenter.meta.enums.DatacenterSourceType; + +import java.util.List; +import java.util.Map; +import java.util.function.Function; +import java.util.stream.Collectors; + +@Component +public class DatacenterConnectorRegistry { + + private final Map connectorMap; + + public DatacenterConnectorRegistry(List connectors) { + this.connectorMap = connectors.stream().collect(Collectors.toMap(DatacenterConnector::getSourceType, Function.identity())); + } + + public DatacenterConnector getConnector(String sourceType) { + DatacenterSourceType type = DatacenterSourceType.valueOf(sourceType); + DatacenterConnector connector = connectorMap.get(type); + if (connector == null) { + throw new BusinessException("未找到连接器: " + sourceType); + } + return connector; + } +} diff --git a/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/connector/MetadataExplorer.java b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/connector/MetadataExplorer.java new file mode 100644 index 0000000..29f566d --- /dev/null +++ b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/connector/MetadataExplorer.java @@ -0,0 +1,16 @@ +package tech.easyflow.datacenter.connector; + +import tech.easyflow.datacenter.entity.DatacenterTable; +import tech.easyflow.datacenter.meta.entity.DatacenterSource; +import tech.easyflow.datacenter.meta.model.DatacenterCatalogMeta; +import tech.easyflow.datacenter.meta.model.DatacenterTableDetailMeta; + +import java.util.List; + +public interface MetadataExplorer { + List listCatalogs(DatacenterSource source); + + List listTables(DatacenterSource source, String catalogName); + + DatacenterTableDetailMeta getTableDetail(DatacenterSource source, String catalogName, String tableName); +} diff --git a/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/connector/QueryExecutor.java b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/connector/QueryExecutor.java new file mode 100644 index 0000000..09927cc --- /dev/null +++ b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/connector/QueryExecutor.java @@ -0,0 +1,15 @@ +package tech.easyflow.datacenter.connector; + +import com.mybatisflex.core.paginate.Page; +import com.mybatisflex.core.row.Row; +import tech.easyflow.datacenter.entity.DatacenterTable; +import tech.easyflow.datacenter.execution.model.DatacenterQueryRequest; +import tech.easyflow.datacenter.meta.entity.DatacenterSource; + +import java.util.List; + +public interface QueryExecutor { + Page queryPage(DatacenterSource source, DatacenterTable table, DatacenterQueryRequest request); + + List queryBySql(DatacenterSource source, String sql); +} diff --git a/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/connector/SourceHealthChecker.java b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/connector/SourceHealthChecker.java new file mode 100644 index 0000000..e097f30 --- /dev/null +++ b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/connector/SourceHealthChecker.java @@ -0,0 +1,8 @@ +package tech.easyflow.datacenter.connector; + +import tech.easyflow.datacenter.execution.model.DatacenterConnectionTestResult; +import tech.easyflow.datacenter.meta.entity.DatacenterSource; + +public interface SourceHealthChecker { + DatacenterConnectionTestResult testConnection(DatacenterSource source); +} diff --git a/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/connector/SqlDialect.java b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/connector/SqlDialect.java new file mode 100644 index 0000000..28f13ce --- /dev/null +++ b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/connector/SqlDialect.java @@ -0,0 +1,9 @@ +package tech.easyflow.datacenter.connector; + +public interface SqlDialect { + String quoteIdentifier(String identifier); + + String qualifyTable(String namespace, String tableName); + + String buildPageSql(String baseSql, long pageNumber, long pageSize); +} diff --git a/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/connector/WriteExecutor.java b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/connector/WriteExecutor.java new file mode 100644 index 0000000..8b1ac0d --- /dev/null +++ b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/connector/WriteExecutor.java @@ -0,0 +1,14 @@ +package tech.easyflow.datacenter.connector; + +import com.alibaba.fastjson2.JSONObject; +import tech.easyflow.common.entity.LoginAccount; +import tech.easyflow.datacenter.entity.DatacenterTable; +import tech.easyflow.datacenter.meta.entity.DatacenterSource; + +import java.math.BigInteger; + +public interface WriteExecutor { + void saveRow(DatacenterSource source, DatacenterTable table, JSONObject data, LoginAccount account); + + void deleteRow(DatacenterSource source, DatacenterTable table, BigInteger id, LoginAccount account); +} diff --git a/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/connector/dialect/GaussdbSqlDialect.java b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/connector/dialect/GaussdbSqlDialect.java new file mode 100644 index 0000000..a6743b2 --- /dev/null +++ b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/connector/dialect/GaussdbSqlDialect.java @@ -0,0 +1,24 @@ +package tech.easyflow.datacenter.connector.dialect; + +import cn.hutool.core.util.StrUtil; +import tech.easyflow.datacenter.connector.SqlDialect; + +public class GaussdbSqlDialect implements SqlDialect { + @Override + public String quoteIdentifier(String identifier) { + return '"' + identifier + '"'; + } + + @Override + public String qualifyTable(String namespace, String tableName) { + if (StrUtil.isBlank(namespace)) { + return quoteIdentifier(tableName); + } + return quoteIdentifier(namespace) + "." + quoteIdentifier(tableName); + } + + @Override + public String buildPageSql(String baseSql, long pageNumber, long pageSize) { + return baseSql + " OFFSET ? LIMIT ?"; + } +} diff --git a/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/connector/dialect/MysqlSqlDialect.java b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/connector/dialect/MysqlSqlDialect.java new file mode 100644 index 0000000..3fa0f5a --- /dev/null +++ b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/connector/dialect/MysqlSqlDialect.java @@ -0,0 +1,24 @@ +package tech.easyflow.datacenter.connector.dialect; + +import cn.hutool.core.util.StrUtil; +import tech.easyflow.datacenter.connector.SqlDialect; + +public class MysqlSqlDialect implements SqlDialect { + @Override + public String quoteIdentifier(String identifier) { + return "`" + identifier + "`"; + } + + @Override + public String qualifyTable(String namespace, String tableName) { + if (StrUtil.isBlank(namespace)) { + return quoteIdentifier(tableName); + } + return quoteIdentifier(namespace) + "." + quoteIdentifier(tableName); + } + + @Override + public String buildPageSql(String baseSql, long pageNumber, long pageSize) { + return baseSql + " LIMIT ?, ?"; + } +} diff --git a/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/connector/dialect/OracleSqlDialect.java b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/connector/dialect/OracleSqlDialect.java new file mode 100644 index 0000000..576699b --- /dev/null +++ b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/connector/dialect/OracleSqlDialect.java @@ -0,0 +1,24 @@ +package tech.easyflow.datacenter.connector.dialect; + +import cn.hutool.core.util.StrUtil; +import tech.easyflow.datacenter.connector.SqlDialect; + +public class OracleSqlDialect implements SqlDialect { + @Override + public String quoteIdentifier(String identifier) { + return '"' + identifier + '"'; + } + + @Override + public String qualifyTable(String namespace, String tableName) { + if (StrUtil.isBlank(namespace)) { + return quoteIdentifier(tableName); + } + return quoteIdentifier(namespace) + "." + quoteIdentifier(tableName); + } + + @Override + public String buildPageSql(String baseSql, long pageNumber, long pageSize) { + return baseSql + " OFFSET ? ROWS FETCH NEXT ? ROWS ONLY"; + } +} diff --git a/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/connector/dialect/PostgresqlSqlDialect.java b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/connector/dialect/PostgresqlSqlDialect.java new file mode 100644 index 0000000..a66fed9 --- /dev/null +++ b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/connector/dialect/PostgresqlSqlDialect.java @@ -0,0 +1,24 @@ +package tech.easyflow.datacenter.connector.dialect; + +import cn.hutool.core.util.StrUtil; +import tech.easyflow.datacenter.connector.SqlDialect; + +public class PostgresqlSqlDialect implements SqlDialect { + @Override + public String quoteIdentifier(String identifier) { + return '"' + identifier + '"'; + } + + @Override + public String qualifyTable(String namespace, String tableName) { + if (StrUtil.isBlank(namespace)) { + return quoteIdentifier(tableName); + } + return quoteIdentifier(namespace) + "." + quoteIdentifier(tableName); + } + + @Override + public String buildPageSql(String baseSql, long pageNumber, long pageSize) { + return baseSql + " OFFSET ? LIMIT ?"; + } +} diff --git a/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/connector/impl/ExcelConnector.java b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/connector/impl/ExcelConnector.java new file mode 100644 index 0000000..cd53b64 --- /dev/null +++ b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/connector/impl/ExcelConnector.java @@ -0,0 +1,22 @@ +package tech.easyflow.datacenter.connector.impl; + +import org.springframework.stereotype.Component; +import tech.easyflow.datacenter.connector.support.AbstractInternalTableConnector; +import tech.easyflow.datacenter.meta.enums.DatacenterCapability; +import tech.easyflow.datacenter.meta.enums.DatacenterSourceType; + +import java.util.EnumSet; + +@Component +public class ExcelConnector extends AbstractInternalTableConnector { + public ExcelConnector() { + super(DatacenterSourceType.EXCEL, EnumSet.of( + DatacenterCapability.TEST_CONNECTION, + DatacenterCapability.BROWSE_METADATA, + DatacenterCapability.READ_QUERY, + DatacenterCapability.WRITE_MUTATION, + DatacenterCapability.MATERIALIZE, + DatacenterCapability.EXPORT + )); + } +} diff --git a/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/connector/impl/ExcelMaterializedConnector.java b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/connector/impl/ExcelMaterializedConnector.java new file mode 100644 index 0000000..17ad5b4 --- /dev/null +++ b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/connector/impl/ExcelMaterializedConnector.java @@ -0,0 +1,22 @@ +package tech.easyflow.datacenter.connector.impl; + +import org.springframework.stereotype.Component; +import tech.easyflow.datacenter.connector.support.AbstractInternalTableConnector; +import tech.easyflow.datacenter.meta.enums.DatacenterCapability; +import tech.easyflow.datacenter.meta.enums.DatacenterSourceType; + +import java.util.EnumSet; + +@Component +public class ExcelMaterializedConnector extends AbstractInternalTableConnector { + public ExcelMaterializedConnector() { + super(DatacenterSourceType.EXCEL_MATERIALIZED, EnumSet.of( + DatacenterCapability.TEST_CONNECTION, + DatacenterCapability.BROWSE_METADATA, + DatacenterCapability.READ_QUERY, + DatacenterCapability.WRITE_MUTATION, + DatacenterCapability.MATERIALIZE, + DatacenterCapability.EXPORT + )); + } +} diff --git a/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/connector/impl/GaussdbNativeConnector.java b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/connector/impl/GaussdbNativeConnector.java new file mode 100644 index 0000000..bf08e91 --- /dev/null +++ b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/connector/impl/GaussdbNativeConnector.java @@ -0,0 +1,40 @@ +package tech.easyflow.datacenter.connector.impl; + +import com.zaxxer.hikari.HikariDataSource; +import org.springframework.stereotype.Component; +import tech.easyflow.datacenter.connector.dialect.GaussdbSqlDialect; +import tech.easyflow.datacenter.connector.support.AbstractJdbcConnector; +import tech.easyflow.datacenter.connector.support.DatacenterDatasourceManager; +import tech.easyflow.datacenter.meta.entity.DatacenterSource; +import tech.easyflow.datacenter.meta.enums.DatacenterCapability; +import tech.easyflow.datacenter.meta.enums.DatacenterSourceType; + +import java.sql.Connection; +import java.util.EnumSet; + +@Component +public class GaussdbNativeConnector extends AbstractJdbcConnector { + + private final DatacenterDatasourceManager datasourceManager; + + public GaussdbNativeConnector(DatacenterDatasourceManager datasourceManager) { + super(DatacenterSourceType.GAUSSDB_NATIVE, new GaussdbSqlDialect(), EnumSet.of( + DatacenterCapability.TEST_CONNECTION, + DatacenterCapability.BROWSE_METADATA, + DatacenterCapability.READ_QUERY + )); + this.datasourceManager = datasourceManager; + } + + @Override + protected T withConnection(DatacenterSource source, boolean cacheable, JdbcCallback callback) throws Exception { + HikariDataSource dataSource = cacheable ? datasourceManager.getOrCreateExternalDatasource(source) : datasourceManager.createExternalDatasource(source); + try (Connection connection = dataSource.getConnection()) { + return callback.apply(connection); + } finally { + if (!cacheable || source.getId() == null) { + dataSource.close(); + } + } + } +} diff --git a/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/connector/impl/Gbase8aConnector.java b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/connector/impl/Gbase8aConnector.java new file mode 100644 index 0000000..7b73d41 --- /dev/null +++ b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/connector/impl/Gbase8aConnector.java @@ -0,0 +1,40 @@ +package tech.easyflow.datacenter.connector.impl; + +import com.zaxxer.hikari.HikariDataSource; +import org.springframework.stereotype.Component; +import tech.easyflow.datacenter.connector.dialect.MysqlSqlDialect; +import tech.easyflow.datacenter.connector.support.AbstractJdbcConnector; +import tech.easyflow.datacenter.connector.support.DatacenterDatasourceManager; +import tech.easyflow.datacenter.meta.entity.DatacenterSource; +import tech.easyflow.datacenter.meta.enums.DatacenterCapability; +import tech.easyflow.datacenter.meta.enums.DatacenterSourceType; + +import java.sql.Connection; +import java.util.EnumSet; + +@Component +public class Gbase8aConnector extends AbstractJdbcConnector { + + private final DatacenterDatasourceManager datasourceManager; + + public Gbase8aConnector(DatacenterDatasourceManager datasourceManager) { + super(DatacenterSourceType.GBASE_8A, new MysqlSqlDialect(), EnumSet.of( + DatacenterCapability.TEST_CONNECTION, + DatacenterCapability.BROWSE_METADATA, + DatacenterCapability.READ_QUERY + )); + this.datasourceManager = datasourceManager; + } + + @Override + protected T withConnection(DatacenterSource source, boolean cacheable, JdbcCallback callback) throws Exception { + HikariDataSource dataSource = cacheable ? datasourceManager.getOrCreateExternalDatasource(source) : datasourceManager.createExternalDatasource(source); + try (Connection connection = dataSource.getConnection()) { + return callback.apply(connection); + } finally { + if (!cacheable || source.getId() == null) { + dataSource.close(); + } + } + } +} diff --git a/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/connector/impl/Gbase8sConnector.java b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/connector/impl/Gbase8sConnector.java new file mode 100644 index 0000000..398f51c --- /dev/null +++ b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/connector/impl/Gbase8sConnector.java @@ -0,0 +1,40 @@ +package tech.easyflow.datacenter.connector.impl; + +import com.zaxxer.hikari.HikariDataSource; +import org.springframework.stereotype.Component; +import tech.easyflow.datacenter.connector.dialect.MysqlSqlDialect; +import tech.easyflow.datacenter.connector.support.AbstractJdbcConnector; +import tech.easyflow.datacenter.connector.support.DatacenterDatasourceManager; +import tech.easyflow.datacenter.meta.entity.DatacenterSource; +import tech.easyflow.datacenter.meta.enums.DatacenterCapability; +import tech.easyflow.datacenter.meta.enums.DatacenterSourceType; + +import java.sql.Connection; +import java.util.EnumSet; + +@Component +public class Gbase8sConnector extends AbstractJdbcConnector { + + private final DatacenterDatasourceManager datasourceManager; + + public Gbase8sConnector(DatacenterDatasourceManager datasourceManager) { + super(DatacenterSourceType.GBASE_8S, new MysqlSqlDialect(), EnumSet.of( + DatacenterCapability.TEST_CONNECTION, + DatacenterCapability.BROWSE_METADATA, + DatacenterCapability.READ_QUERY + )); + this.datasourceManager = datasourceManager; + } + + @Override + protected T withConnection(DatacenterSource source, boolean cacheable, JdbcCallback callback) throws Exception { + HikariDataSource dataSource = cacheable ? datasourceManager.getOrCreateExternalDatasource(source) : datasourceManager.createExternalDatasource(source); + try (Connection connection = dataSource.getConnection()) { + return callback.apply(connection); + } finally { + if (!cacheable || source.getId() == null) { + dataSource.close(); + } + } + } +} diff --git a/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/connector/impl/MysqlConnector.java b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/connector/impl/MysqlConnector.java new file mode 100644 index 0000000..1f29b09 --- /dev/null +++ b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/connector/impl/MysqlConnector.java @@ -0,0 +1,40 @@ +package tech.easyflow.datacenter.connector.impl; + +import com.zaxxer.hikari.HikariDataSource; +import org.springframework.stereotype.Component; +import tech.easyflow.datacenter.connector.dialect.MysqlSqlDialect; +import tech.easyflow.datacenter.connector.support.AbstractJdbcConnector; +import tech.easyflow.datacenter.connector.support.DatacenterDatasourceManager; +import tech.easyflow.datacenter.meta.entity.DatacenterSource; +import tech.easyflow.datacenter.meta.enums.DatacenterCapability; +import tech.easyflow.datacenter.meta.enums.DatacenterSourceType; + +import java.sql.Connection; +import java.util.EnumSet; + +@Component +public class MysqlConnector extends AbstractJdbcConnector { + + private final DatacenterDatasourceManager datasourceManager; + + public MysqlConnector(DatacenterDatasourceManager datasourceManager) { + super(DatacenterSourceType.MYSQL, new MysqlSqlDialect(), EnumSet.of( + DatacenterCapability.TEST_CONNECTION, + DatacenterCapability.BROWSE_METADATA, + DatacenterCapability.READ_QUERY + )); + this.datasourceManager = datasourceManager; + } + + @Override + protected T withConnection(DatacenterSource source, boolean cacheable, JdbcCallback callback) throws Exception { + HikariDataSource dataSource = cacheable ? datasourceManager.getOrCreateExternalDatasource(source) : datasourceManager.createExternalDatasource(source); + try (Connection connection = dataSource.getConnection()) { + return callback.apply(connection); + } finally { + if (!cacheable || source.getId() == null) { + dataSource.close(); + } + } + } +} diff --git a/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/connector/impl/OracleConnector.java b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/connector/impl/OracleConnector.java new file mode 100644 index 0000000..99c8d46 --- /dev/null +++ b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/connector/impl/OracleConnector.java @@ -0,0 +1,40 @@ +package tech.easyflow.datacenter.connector.impl; + +import com.zaxxer.hikari.HikariDataSource; +import org.springframework.stereotype.Component; +import tech.easyflow.datacenter.connector.dialect.OracleSqlDialect; +import tech.easyflow.datacenter.connector.support.AbstractJdbcConnector; +import tech.easyflow.datacenter.connector.support.DatacenterDatasourceManager; +import tech.easyflow.datacenter.meta.entity.DatacenterSource; +import tech.easyflow.datacenter.meta.enums.DatacenterCapability; +import tech.easyflow.datacenter.meta.enums.DatacenterSourceType; + +import java.sql.Connection; +import java.util.EnumSet; + +@Component +public class OracleConnector extends AbstractJdbcConnector { + + private final DatacenterDatasourceManager datasourceManager; + + public OracleConnector(DatacenterDatasourceManager datasourceManager) { + super(DatacenterSourceType.ORACLE, new OracleSqlDialect(), EnumSet.of( + DatacenterCapability.TEST_CONNECTION, + DatacenterCapability.BROWSE_METADATA, + DatacenterCapability.READ_QUERY + )); + this.datasourceManager = datasourceManager; + } + + @Override + protected T withConnection(DatacenterSource source, boolean cacheable, JdbcCallback callback) throws Exception { + HikariDataSource dataSource = cacheable ? datasourceManager.getOrCreateExternalDatasource(source) : datasourceManager.createExternalDatasource(source); + try (Connection connection = dataSource.getConnection()) { + return callback.apply(connection); + } finally { + if (!cacheable || source.getId() == null) { + dataSource.close(); + } + } + } +} diff --git a/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/connector/impl/PostgresqlConnector.java b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/connector/impl/PostgresqlConnector.java new file mode 100644 index 0000000..8f09b12 --- /dev/null +++ b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/connector/impl/PostgresqlConnector.java @@ -0,0 +1,40 @@ +package tech.easyflow.datacenter.connector.impl; + +import com.zaxxer.hikari.HikariDataSource; +import org.springframework.stereotype.Component; +import tech.easyflow.datacenter.connector.dialect.PostgresqlSqlDialect; +import tech.easyflow.datacenter.connector.support.AbstractJdbcConnector; +import tech.easyflow.datacenter.connector.support.DatacenterDatasourceManager; +import tech.easyflow.datacenter.meta.entity.DatacenterSource; +import tech.easyflow.datacenter.meta.enums.DatacenterCapability; +import tech.easyflow.datacenter.meta.enums.DatacenterSourceType; + +import java.sql.Connection; +import java.util.EnumSet; + +@Component +public class PostgresqlConnector extends AbstractJdbcConnector { + + private final DatacenterDatasourceManager datasourceManager; + + public PostgresqlConnector(DatacenterDatasourceManager datasourceManager) { + super(DatacenterSourceType.POSTGRESQL, new PostgresqlSqlDialect(), EnumSet.of( + DatacenterCapability.TEST_CONNECTION, + DatacenterCapability.BROWSE_METADATA, + DatacenterCapability.READ_QUERY + )); + this.datasourceManager = datasourceManager; + } + + @Override + protected T withConnection(DatacenterSource source, boolean cacheable, JdbcCallback callback) throws Exception { + HikariDataSource dataSource = cacheable ? datasourceManager.getOrCreateExternalDatasource(source) : datasourceManager.createExternalDatasource(source); + try (Connection connection = dataSource.getConnection()) { + return callback.apply(connection); + } finally { + if (!cacheable || source.getId() == null) { + dataSource.close(); + } + } + } +} diff --git a/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/connector/impl/ProjectMysqlConnector.java b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/connector/impl/ProjectMysqlConnector.java new file mode 100644 index 0000000..9a8ffc2 --- /dev/null +++ b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/connector/impl/ProjectMysqlConnector.java @@ -0,0 +1,135 @@ +package tech.easyflow.datacenter.connector.impl; + +import cn.hutool.core.util.StrUtil; +import com.alibaba.fastjson2.JSONObject; +import com.mybatisflex.core.paginate.Page; +import com.mybatisflex.core.row.Row; +import org.springframework.stereotype.Component; +import tech.easyflow.common.entity.LoginAccount; +import tech.easyflow.common.web.exceptions.BusinessException; +import tech.easyflow.datacenter.connector.dialect.MysqlSqlDialect; +import tech.easyflow.datacenter.connector.support.AbstractJdbcConnector; +import tech.easyflow.datacenter.entity.DatacenterTable; +import tech.easyflow.datacenter.entity.DatacenterTableField; +import tech.easyflow.datacenter.execution.model.DatacenterQueryRequest; +import tech.easyflow.datacenter.meta.entity.DatacenterSource; +import tech.easyflow.datacenter.meta.enums.DatacenterCapability; +import tech.easyflow.datacenter.meta.enums.DatacenterSourceType; + +import javax.sql.DataSource; +import java.math.BigInteger; +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.util.ArrayList; +import java.util.EnumSet; +import java.util.List; +import java.util.stream.Collectors; + +@Component +public class ProjectMysqlConnector extends AbstractJdbcConnector { + + private final DataSource dataSource; + private final MysqlSqlDialect dialect = new MysqlSqlDialect(); + + public ProjectMysqlConnector(DataSource dataSource) { + super(DatacenterSourceType.PROJECT_MYSQL, new MysqlSqlDialect(), EnumSet.of( + DatacenterCapability.TEST_CONNECTION, + DatacenterCapability.BROWSE_METADATA, + DatacenterCapability.READ_QUERY, + DatacenterCapability.WRITE_MUTATION + )); + this.dataSource = dataSource; + } + + @Override + protected T withConnection(DatacenterSource source, boolean cacheable, JdbcCallback callback) throws Exception { + try (Connection connection = dataSource.getConnection()) { + return callback.apply(connection); + } + } + + @Override + protected boolean requiresJdbcUrl() { + return false; + } + + @Override + public Page queryPage(DatacenterSource source, DatacenterTable table, DatacenterQueryRequest request) { + return super.queryPage(source, table, request); + } + + @Override + public void saveRow(DatacenterSource source, DatacenterTable table, JSONObject data, LoginAccount account) { + List writableFields = table.getFields().stream() + .filter(field -> field.getWritable() == null || field.getWritable() == 1) + .collect(Collectors.toList()); + Object id = data.get("id"); + try (Connection connection = dataSource.getConnection()) { + if (id == null) { + List columns = new ArrayList<>(); + List values = new ArrayList<>(); + for (DatacenterTableField field : writableFields) { + Object value = data.get(field.getFieldName()); + if (value != null) { + columns.add(field.getFieldName()); + values.add(value); + } + } + if (columns.isEmpty()) { + throw new BusinessException("没有可写字段"); + } + String sql = "INSERT INTO " + dialect.qualifyTable(source.getDatabaseName(), resolvePhysicalTableName(table)) + + " (" + columns.stream().map(dialect::quoteIdentifier).collect(Collectors.joining(",")) + ") VALUES (" + + columns.stream().map(item -> "?").collect(Collectors.joining(",")) + ")"; + try (PreparedStatement statement = connection.prepareStatement(sql)) { + for (int i = 0; i < values.size(); i++) { + statement.setObject(i + 1, values.get(i)); + } + statement.executeUpdate(); + } + return; + } + List setClauses = new ArrayList<>(); + List values = new ArrayList<>(); + for (DatacenterTableField field : writableFields) { + if (!data.containsKey(field.getFieldName())) { + continue; + } + setClauses.add(dialect.quoteIdentifier(field.getFieldName()) + " = ?"); + values.add(data.get(field.getFieldName())); + } + if (setClauses.isEmpty()) { + return; + } + String sql = "UPDATE " + dialect.qualifyTable(source.getDatabaseName(), resolvePhysicalTableName(table)) + + " SET " + String.join(",", setClauses) + + " WHERE " + dialect.quoteIdentifier("id") + " = ?"; + values.add(id); + try (PreparedStatement statement = connection.prepareStatement(sql)) { + for (int i = 0; i < values.size(); i++) { + statement.setObject(i + 1, values.get(i)); + } + statement.executeUpdate(); + } + } catch (Exception ex) { + throw new BusinessException("项目 MySQL 写入失败: " + ex.getMessage()); + } + } + + @Override + public void deleteRow(DatacenterSource source, DatacenterTable table, BigInteger id, LoginAccount account) { + String sql = "DELETE FROM " + dialect.qualifyTable(source.getDatabaseName(), resolvePhysicalTableName(table)) + + " WHERE " + dialect.quoteIdentifier("id") + " = ?"; + try (Connection connection = dataSource.getConnection(); PreparedStatement statement = connection.prepareStatement(sql)) { + statement.setObject(1, id); + statement.executeUpdate(); + } catch (Exception ex) { + throw new BusinessException("项目 MySQL 删除失败: " + ex.getMessage()); + } + } + + @Override + protected String resolveCatalogName(DatacenterSource source, String requestedCatalogName) { + return StrUtil.blankToDefault(requestedCatalogName, source.getDatabaseName()); + } +} diff --git a/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/connector/support/AbstractInternalTableConnector.java b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/connector/support/AbstractInternalTableConnector.java new file mode 100644 index 0000000..e6251e0 --- /dev/null +++ b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/connector/support/AbstractInternalTableConnector.java @@ -0,0 +1,152 @@ +package tech.easyflow.datacenter.connector.support; + +import cn.hutool.core.util.StrUtil; +import com.alibaba.fastjson2.JSONObject; +import com.mybatisflex.core.paginate.Page; +import com.mybatisflex.core.query.QueryWrapper; +import com.mybatisflex.core.row.Db; +import com.mybatisflex.core.row.Row; +import com.mybatisflex.core.row.RowKey; +import org.springframework.util.CollectionUtils; +import tech.easyflow.common.entity.LoginAccount; +import tech.easyflow.common.web.exceptions.BusinessException; +import tech.easyflow.datacenter.connector.DatacenterConnector; +import tech.easyflow.datacenter.entity.DatacenterTable; +import tech.easyflow.datacenter.entity.DatacenterTableField; +import tech.easyflow.datacenter.execution.model.DatacenterConnectionTestResult; +import tech.easyflow.datacenter.execution.model.DatacenterQueryRequest; +import tech.easyflow.datacenter.meta.entity.DatacenterSource; +import tech.easyflow.datacenter.meta.enums.DatacenterCapability; +import tech.easyflow.datacenter.meta.enums.DatacenterSourceType; +import tech.easyflow.datacenter.meta.model.DatacenterCatalogMeta; +import tech.easyflow.datacenter.meta.model.DatacenterTableDetailMeta; + +import java.math.BigDecimal; +import java.math.BigInteger; +import java.util.*; + +public abstract class AbstractInternalTableConnector implements DatacenterConnector { + + private final DatacenterSourceType sourceType; + private final Set capabilities; + + protected AbstractInternalTableConnector(DatacenterSourceType sourceType, Set capabilities) { + this.sourceType = sourceType; + this.capabilities = capabilities; + } + + @Override + public DatacenterSourceType getSourceType() { + return sourceType; + } + + @Override + public Set getCapabilities() { + return capabilities; + } + + @Override + public DatacenterConnectionTestResult testConnection(DatacenterSource source) { + DatacenterConnectionTestResult result = new DatacenterConnectionTestResult(); + result.setSuccess(true); + result.setMessage("连接成功"); + result.setCapabilities(capabilities.stream().map(Enum::name).toList()); + result.setDetails(Map.of("sourceType", sourceType.name())); + return result; + } + + @Override + public List listCatalogs(DatacenterSource source) { + return Collections.emptyList(); + } + + @Override + public List listTables(DatacenterSource source, String catalogName) { + return Collections.emptyList(); + } + + @Override + public DatacenterTableDetailMeta getTableDetail(DatacenterSource source, String catalogName, String tableName) { + throw new BusinessException("内部数据源请从元数据注册表读取表结构"); + } + + @Override + public Page queryPage(DatacenterSource source, DatacenterTable table, DatacenterQueryRequest request) { + String actualTable = resolveTableName(table); + QueryWrapper wrapper = QueryWrapper.create(); + if (StrUtil.isNotBlank(request.getWhere())) { + wrapper.where(request.getWhere()); + } + long count = Db.selectCountByQuery(actualTable, wrapper); + if (count == 0) { + return new Page<>(new ArrayList<>(), request.getPageNumber(), request.getPageSize(), count); + } + Page page = Db.paginate(actualTable, new Page<>(request.getPageNumber(), request.getPageSize(), count), wrapper); + normalizeRows(page.getRecords()); + return page; + } + + @Override + public List queryBySql(DatacenterSource source, String sql) { + List rows = Db.selectListBySql(sql); + normalizeRows(rows); + return rows; + } + + @Override + public void saveRow(DatacenterSource source, DatacenterTable table, JSONObject data, LoginAccount account) { + List fields = table.getFields(); + if (CollectionUtils.isEmpty(fields)) { + throw new BusinessException("数据集字段为空，无法写入"); + } + String actualTable = resolveTableName(table); + Object id = data.get("id"); + if (id == null) { + Row row = Row.ofKey(RowKey.SNOW_FLAKE_ID); + row.put("dept_id", account.getDeptId()); + row.put("tenant_id", account.getTenantId()); + row.put("created", new Date()); + row.put("created_by", account.getId()); + row.put("modified", new Date()); + row.put("modified_by", account.getId()); + row.put("remark", data.get("remark")); + for (DatacenterTableField field : fields) { + row.put(field.getFieldName(), data.get(field.getFieldName())); + } + Db.insert(actualTable, row); + return; + } + Row row = Row.ofKey("id", id); + row.put("modified", new Date()); + row.put("modified_by", account.getId()); + for (DatacenterTableField field : fields) { + row.put(field.getFieldName(), data.get(field.getFieldName())); + } + Db.updateById(actualTable, row); + } + + @Override + public void deleteRow(DatacenterSource source, DatacenterTable table, BigInteger id, LoginAccount account) { + Db.deleteById(resolveTableName(table), Row.ofKey("id", id)); + } + + protected String resolveTableName(DatacenterTable table) { + return StrUtil.blankToDefault(table.getMaterializedTable(), table.getActualTable()); + } + + private void normalizeRows(List records) { + for (Row record : records) { + Map converted = new LinkedHashMap<>(); + for (Map.Entry entry : record.entrySet()) { + Object value = entry.getValue(); + if (value instanceof BigInteger || value instanceof BigDecimal || value instanceof Long) { + converted.put(entry.getKey(), value.toString()); + } else { + converted.put(entry.getKey(), value); + } + } + record.clear(); + record.putAll(converted); + } + } +} diff --git a/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/connector/support/AbstractJdbcConnector.java b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/connector/support/AbstractJdbcConnector.java new file mode 100644 index 0000000..7ddeb56 --- /dev/null +++ b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/connector/support/AbstractJdbcConnector.java @@ -0,0 +1,523 @@ +package tech.easyflow.datacenter.connector.support; + +import cn.hutool.core.util.StrUtil; +import com.alibaba.fastjson2.JSONObject; +import com.mybatisflex.core.paginate.Page; +import com.mybatisflex.core.row.Row; +import org.springframework.util.CollectionUtils; +import tech.easyflow.common.constant.enums.EnumFieldType; +import tech.easyflow.common.entity.LoginAccount; +import tech.easyflow.common.web.exceptions.BusinessException; +import tech.easyflow.datacenter.connector.DatacenterConnector; +import tech.easyflow.datacenter.connector.SqlDialect; +import tech.easyflow.datacenter.entity.DatacenterTable; +import tech.easyflow.datacenter.entity.DatacenterTableField; +import tech.easyflow.datacenter.execution.model.DatacenterConnectionTestResult; +import tech.easyflow.datacenter.execution.model.DatacenterQueryFilter; +import tech.easyflow.datacenter.execution.model.DatacenterQueryRequest; +import tech.easyflow.datacenter.execution.model.DatacenterQuerySort; +import tech.easyflow.datacenter.meta.entity.DatacenterSource; +import tech.easyflow.datacenter.meta.enums.DatacenterCapability; +import tech.easyflow.datacenter.meta.enums.DatacenterConnectionErrorCode; +import tech.easyflow.datacenter.meta.enums.DatacenterSourceType; +import tech.easyflow.datacenter.meta.enums.DatacenterTableKind; +import tech.easyflow.datacenter.meta.model.DatacenterCatalogMeta; +import tech.easyflow.datacenter.meta.model.DatacenterTableDetailMeta; + +import java.math.BigDecimal; +import java.math.BigInteger; +import java.sql.*; +import java.util.*; +import java.util.stream.Collectors; + +public abstract class AbstractJdbcConnector implements DatacenterConnector { + + private final DatacenterSourceType sourceType; + private final SqlDialect sqlDialect; + private final Set capabilities; + + protected AbstractJdbcConnector(DatacenterSourceType sourceType, SqlDialect sqlDialect, Set capabilities) { + this.sourceType = sourceType; + this.sqlDialect = sqlDialect; + this.capabilities = capabilities; + } + + @Override + public DatacenterSourceType getSourceType() { + return sourceType; + } + + @Override + public Set getCapabilities() { + return capabilities; + } + + protected abstract T withConnection(DatacenterSource source, boolean cacheable, JdbcCallback callback) throws Exception; + + @Override + public DatacenterConnectionTestResult testConnection(DatacenterSource source) { + DatacenterConnectionTestResult result = new DatacenterConnectionTestResult(); + result.setCapabilities(capabilities.stream().map(Enum::name).toList()); + if (StrUtil.isBlank(source.getJdbcUrl()) && requiresJdbcUrl()) { + result.setSuccess(false); + result.setErrorCode(DatacenterConnectionErrorCode.INVALID_ARGUMENT.name()); + result.setMessage("缺少 JDBC URL"); + return result; + } + try { + if (StrUtil.isNotBlank(source.getDriverClassName())) { + Class.forName(source.getDriverClassName()); + } + Map details = withConnection(source, false, connection -> { + DatabaseMetaData metaData = connection.getMetaData(); + Map map = new LinkedHashMap<>(); + map.put("databaseProductName", metaData.getDatabaseProductName()); + map.put("databaseProductVersion", metaData.getDatabaseProductVersion()); + map.put("url", metaData.getURL()); + return map; + }); + result.setSuccess(true); + result.setMessage("连接成功"); + result.setDetails(details); + return result; + } catch (ClassNotFoundException ex) { + result.setSuccess(false); + result.setErrorCode(DatacenterConnectionErrorCode.DRIVER_NOT_FOUND.name()); + result.setMessage(ex.getMessage()); + return result; + } catch (SQLException ex) { + result.setSuccess(false); + result.setErrorCode(mapSqlError(ex)); + result.setMessage(ex.getMessage()); + return result; + } catch (Exception ex) { + result.setSuccess(false); + result.setErrorCode(DatacenterConnectionErrorCode.UNKNOWN_ERROR.name()); + result.setMessage(ex.getMessage()); + return result; + } + } + + @Override + public List listCatalogs(DatacenterSource source) { + try { + return withConnection(source, true, connection -> { + List result = new ArrayList<>(); + DatabaseMetaData metaData = connection.getMetaData(); + try (ResultSet catalogs = metaData.getCatalogs()) { + while (catalogs.next()) { + String name = catalogs.getString("TABLE_CAT"); + if (StrUtil.isBlank(name)) { + continue; + } + DatacenterCatalogMeta meta = new DatacenterCatalogMeta(); + meta.setSourceId(source.getId()); + meta.setCatalogName(name); + meta.setCatalogType("CATALOG"); + result.add(meta); + } + } + try (ResultSet schemas = metaData.getSchemas()) { + while (schemas.next()) { + String name = schemas.getString("TABLE_SCHEM"); + if (StrUtil.isBlank(name) || containsCatalog(result, name)) { + continue; + } + DatacenterCatalogMeta meta = new DatacenterCatalogMeta(); + meta.setSourceId(source.getId()); + meta.setCatalogName(name); + meta.setCatalogType("SCHEMA"); + result.add(meta); + } + } + result = filterConfiguredCatalogs(source, result); + if (result.isEmpty()) { + String fallback = resolveCatalogName(source, null); + if (StrUtil.isNotBlank(fallback)) { + DatacenterCatalogMeta meta = new DatacenterCatalogMeta(); + meta.setSourceId(source.getId()); + meta.setCatalogName(fallback); + meta.setCatalogType("DEFAULT"); + result.add(meta); + } + } + return result; + }); + } catch (Exception ex) { + throw DatacenterConnectorExceptionSupport.wrapAccessException("读取目录失败", ex); + } + } + + @Override + public List listTables(DatacenterSource source, String catalogName) { + try { + return withConnection(source, true, connection -> { + DatabaseMetaData metaData = connection.getMetaData(); + List tables = new ArrayList<>(); + try (ResultSet resultSet = metaData.getTables(resolveCatalogArgument(source, catalogName), resolveSchemaArgument(source, catalogName), "%", new String[]{"TABLE", "VIEW"})) { + while (resultSet.next()) { + DatacenterTable table = new DatacenterTable(); + table.setSourceId(source.getId()); + table.setTableName(resultSet.getString("TABLE_NAME")); + table.setTableDesc(resultSet.getString("REMARKS")); + table.setActualTable(resultSet.getString("TABLE_NAME")); + table.setMaterializedTable(resultSet.getString("TABLE_NAME")); + table.setAccessMode(capabilities.contains(DatacenterCapability.WRITE_MUTATION) ? "READ_WRITE" : "READ_ONLY"); + table.setTableKind(resolveTableKind(resultSet.getString("TABLE_TYPE")).name()); + table.setCapabilitiesJson(Map.of("capabilities", capabilities.stream().map(Enum::name).toList())); + tables.add(table); + } + } + return tables; + }); + } catch (Exception ex) { + throw DatacenterConnectorExceptionSupport.wrapAccessException("读取表列表失败", ex); + } + } + + @Override + public DatacenterTableDetailMeta getTableDetail(DatacenterSource source, String catalogName, String tableName) { + try { + return withConnection(source, true, connection -> { + DatabaseMetaData metaData = connection.getMetaData(); + DatacenterTableDetailMeta detail = new DatacenterTableDetailMeta(); + DatacenterTable table = new DatacenterTable(); + table.setSourceId(source.getId()); + table.setTableName(tableName); + table.setActualTable(tableName); + table.setMaterializedTable(tableName); + table.setAccessMode(capabilities.contains(DatacenterCapability.WRITE_MUTATION) ? "READ_WRITE" : "READ_ONLY"); + table.setTableKind(DatacenterTableKind.EXTERNAL_TABLE.name()); + table.setCapabilitiesJson(Map.of("capabilities", capabilities.stream().map(Enum::name).toList())); + detail.setTable(table); + + try (ResultSet tableSet = metaData.getTables( + resolveCatalogArgument(source, catalogName), + resolveSchemaArgument(source, catalogName), + tableName, + new String[]{"TABLE", "VIEW"})) { + while (tableSet.next()) { + String currentTableName = tableSet.getString("TABLE_NAME"); + if (!matchesTableName(currentTableName, tableName)) { + continue; + } + table.setTableDesc(tableSet.getString("REMARKS")); + table.setTableKind(resolveTableKind(tableSet.getString("TABLE_TYPE")).name()); + break; + } + } + + Set primaryKeys = new HashSet<>(); + try (ResultSet pkSet = metaData.getPrimaryKeys(resolveCatalogArgument(source, catalogName), resolveSchemaArgument(source, catalogName), tableName)) { + while (pkSet.next()) { + primaryKeys.add(pkSet.getString("COLUMN_NAME")); + } + } + + List fields = new ArrayList<>(); + try (ResultSet columns = metaData.getColumns(resolveCatalogArgument(source, catalogName), resolveSchemaArgument(source, catalogName), tableName, "%")) { + while (columns.next()) { + DatacenterTableField field = new DatacenterTableField(); + field.setFieldName(columns.getString("COLUMN_NAME")); + field.setSourceColumnName(columns.getString("COLUMN_NAME")); + field.setFieldDesc(columns.getString("REMARKS")); + field.setJdbcType(columns.getString("TYPE_NAME")); + field.setPrecision(columns.getInt("COLUMN_SIZE")); + field.setScale(columns.getInt("DECIMAL_DIGITS")); + field.setRequired(columns.getInt("NULLABLE") == DatabaseMetaData.columnNoNulls ? 1 : 0); + field.setQueryable(1); + field.setSortable(1); + field.setWritable(capabilities.contains(DatacenterCapability.WRITE_MUTATION) ? 1 : 0); + field.setIndexed(primaryKeys.contains(field.getFieldName()) ? 1 : 0); + field.setFieldType(mapFieldType(columns.getInt("DATA_TYPE"))); + fields.add(field); + } + } + detail.setFields(fields); + return detail; + }); + } catch (Exception ex) { + throw DatacenterConnectorExceptionSupport.wrapAccessException("读取表详情失败", ex); + } + } + + @Override + public Page queryPage(DatacenterSource source, DatacenterTable table, DatacenterQueryRequest request) { + if (!capabilities.contains(DatacenterCapability.READ_QUERY)) { + throw new BusinessException("当前数据源暂不支持查询"); + } + try { + return withConnection(source, true, connection -> doQueryPage(connection, source, table, request)); + } catch (Exception ex) { + throw DatacenterConnectorExceptionSupport.wrapAccessException("查询失败", ex); + } + } + + @Override + public List queryBySql(DatacenterSource source, String sql) { + if (!capabilities.contains(DatacenterCapability.READ_QUERY)) { + throw new BusinessException("当前数据源暂不支持查询"); + } + try { + return withConnection(source, true, connection -> doQueryBySql(connection, sql)); + } catch (Exception ex) { + throw DatacenterConnectorExceptionSupport.wrapAccessException("SQL 查询失败", ex); + } + } + + @Override + public void saveRow(DatacenterSource source, DatacenterTable table, JSONObject data, LoginAccount account) { + throw new BusinessException("当前数据源不支持写入"); + } + + @Override + public void deleteRow(DatacenterSource source, DatacenterTable table, BigInteger id, LoginAccount account) { + throw new BusinessException("当前数据源不支持删除"); + } + + protected boolean requiresJdbcUrl() { + return true; + } + + protected Page doQueryPage(Connection connection, DatacenterSource source, DatacenterTable table, DatacenterQueryRequest request) throws SQLException { + List params = new ArrayList<>(); + String selectColumns = CollectionUtils.isEmpty(request.getSelectedColumns()) + ? "*" + : request.getSelectedColumns().stream().map(sqlDialect::quoteIdentifier).collect(Collectors.joining(", ")); + String qualifiedTable = sqlDialect.qualifyTable(resolveCatalogName(source, request.getDatasetRef() == null ? null : request.getDatasetRef().getCatalogName()), resolvePhysicalTableName(table)); + StringBuilder whereClause = new StringBuilder(); + if (StrUtil.isNotBlank(request.getWhere())) { + whereClause.append(" WHERE ").append(request.getWhere()); + } else if (!CollectionUtils.isEmpty(request.getFilters())) { + whereClause.append(" WHERE 1=1 "); + for (DatacenterQueryFilter filter : request.getFilters()) { + appendFilter(whereClause, params, filter); + } + } + String orderClause = buildOrderClause(request.getSorts()); + String baseSql = "SELECT " + selectColumns + " FROM " + qualifiedTable + whereClause + orderClause; + String countSql = "SELECT COUNT(1) FROM " + qualifiedTable + whereClause; + long total = queryCount(connection, countSql, params); + if (total == 0L) { + return new Page<>(new ArrayList<>(), request.getPageNumber(), request.getPageSize(), total); + } + String pageSql = sqlDialect.buildPageSql(baseSql, request.getPageNumber(), request.getPageSize()); + List pageParams = new ArrayList<>(params); + if (pageSql.contains("FETCH NEXT") || pageSql.toLowerCase(Locale.ROOT).contains("limit")) { + pageParams.add((request.getPageNumber() - 1) * request.getPageSize()); + pageParams.add(request.getPageSize()); + } + List records = new ArrayList<>(); + try (PreparedStatement statement = connection.prepareStatement(pageSql)) { + bindParameters(statement, pageParams); + try (ResultSet resultSet = statement.executeQuery()) { + ResultSetMetaData metaData = resultSet.getMetaData(); + while (resultSet.next()) { + Row row = new Row(); + for (int i = 1; i <= metaData.getColumnCount(); i++) { + String columnLabel = metaData.getColumnLabel(i); + row.put(columnLabel, normalizeValue(resultSet.getObject(i))); + } + records.add(row); + } + } + } + return new Page<>(records, request.getPageNumber(), request.getPageSize(), total); + } + + protected String resolvePhysicalTableName(DatacenterTable table) { + return StrUtil.blankToDefault(table.getActualTable(), table.getTableName()); + } + + protected List doQueryBySql(Connection connection, String sql) throws SQLException { + try (PreparedStatement statement = connection.prepareStatement(sql); + ResultSet resultSet = statement.executeQuery()) { + return readRows(resultSet); + } + } + + protected String resolveCatalogName(DatacenterSource source, String requestedCatalogName) { + if (StrUtil.isNotBlank(requestedCatalogName)) { + return requestedCatalogName; + } + if (usesCatalogNamespace()) { + return source.getDatabaseName(); + } + return source.getSchemaName(); + } + + protected List readRows(ResultSet resultSet) throws SQLException { + List records = new ArrayList<>(); + ResultSetMetaData metaData = resultSet.getMetaData(); + while (resultSet.next()) { + Row row = new Row(); + for (int i = 1; i <= metaData.getColumnCount(); i++) { + String columnLabel = metaData.getColumnLabel(i); + row.put(columnLabel, normalizeValue(resultSet.getObject(i))); + } + records.add(row); + } + return records; + } + + protected String resolveCatalogArgument(DatacenterSource source, String catalogName) { + return usesCatalogNamespace() ? resolveCatalogName(source, catalogName) : source.getDatabaseName(); + } + + protected String resolveSchemaArgument(DatacenterSource source, String catalogName) { + return usesCatalogNamespace() ? source.getSchemaName() : resolveCatalogName(source, catalogName); + } + + protected boolean usesCatalogNamespace() { + return sourceType == DatacenterSourceType.MYSQL + || sourceType == DatacenterSourceType.PROJECT_MYSQL + || sourceType == DatacenterSourceType.GBASE_8A + || sourceType == DatacenterSourceType.GBASE_8S; + } + + private List filterConfiguredCatalogs(DatacenterSource source, List items) { + if (items == null || items.isEmpty() || source == null) { + return items; + } + String configuredName = usesCatalogNamespace() + ? StrUtil.trimToNull(source.getDatabaseName()) + : StrUtil.trimToNull(source.getSchemaName()); + if (StrUtil.isBlank(configuredName)) { + return items; + } + List matched = items.stream() + .filter(item -> configuredName.equalsIgnoreCase(item.getCatalogName())) + .collect(Collectors.toList()); + return matched.isEmpty() ? items : matched; + } + + private boolean containsCatalog(List items, String catalogName) { + return items.stream().anyMatch(item -> catalogName.equalsIgnoreCase(item.getCatalogName())); + } + + private DatacenterTableKind resolveTableKind(String tableType) { + return "VIEW".equalsIgnoreCase(tableType) ? DatacenterTableKind.EXTERNAL_VIEW : DatacenterTableKind.EXTERNAL_TABLE; + } + + private boolean matchesTableName(String currentTableName, String targetTableName) { + if (currentTableName == null || targetTableName == null) { + return false; + } + return currentTableName.equals(targetTableName) + || currentTableName.equalsIgnoreCase(targetTableName); + } + + private Integer mapFieldType(int jdbcType) { + return switch (jdbcType) { + case Types.INTEGER, Types.TINYINT, Types.SMALLINT, Types.BIGINT -> EnumFieldType.INTEGER.getCode(); + case Types.FLOAT, Types.DOUBLE, Types.REAL, Types.NUMERIC, Types.DECIMAL -> EnumFieldType.NUMBER.getCode(); + case Types.TIMESTAMP, Types.DATE, Types.TIME -> EnumFieldType.TIME.getCode(); + case Types.BOOLEAN, Types.BIT -> EnumFieldType.BOOLEAN.getCode(); + default -> EnumFieldType.STRING.getCode(); + }; + } + + private void appendFilter(StringBuilder sql, List params, DatacenterQueryFilter filter) { + String operator = StrUtil.blankToDefault(filter.getOperator(), "EQ").toUpperCase(Locale.ROOT); + String column = sqlDialect.quoteIdentifier(filter.getColumn()); + switch (operator) { + case "EQ" -> { + sql.append(" AND ").append(column).append(" = ?"); + params.add(filter.getValue()); + } + case "LIKE" -> { + sql.append(" AND ").append(column).append(" LIKE ?"); + params.add("%" + filter.getValue() + "%"); + } + case "GT" -> { + sql.append(" AND ").append(column).append(" > ?"); + params.add(filter.getValue()); + } + case "GTE" -> { + sql.append(" AND ").append(column).append(" >= ?"); + params.add(filter.getValue()); + } + case "LT" -> { + sql.append(" AND ").append(column).append(" < ?"); + params.add(filter.getValue()); + } + case "LTE" -> { + sql.append(" AND ").append(column).append(" <= ?"); + params.add(filter.getValue()); + } + case "IN" -> { + List values = filter.getValues() == null ? List.of() : filter.getValues(); + if (values.isEmpty()) { + sql.append(" AND 1 = 0"); + } else { + sql.append(" AND ").append(column).append(" IN ("); + sql.append(values.stream().map(item -> "?").collect(Collectors.joining(","))); + sql.append(")"); + params.addAll(values); + } + } + case "IS_NULL" -> sql.append(" AND ").append(column).append(" IS NULL"); + default -> throw new BusinessException("不支持的过滤操作: " + operator); + } + } + + private String buildOrderClause(List sorts) { + if (CollectionUtils.isEmpty(sorts)) { + return ""; + } + return " ORDER BY " + sorts.stream() + .map(sort -> sqlDialect.quoteIdentifier(sort.getColumn()) + " " + ("DESC".equalsIgnoreCase(sort.getDirection()) ? "DESC" : "ASC")) + .collect(Collectors.joining(", ")); + } + + private long queryCount(Connection connection, String sql, List params) throws SQLException { + try (PreparedStatement statement = connection.prepareStatement(sql)) { + bindParameters(statement, params); + try (ResultSet resultSet = statement.executeQuery()) { + if (resultSet.next()) { + return resultSet.getLong(1); + } + return 0L; + } + } + } + + private void bindParameters(PreparedStatement statement, List params) throws SQLException { + for (int i = 0; i < params.size(); i++) { + statement.setObject(i + 1, params.get(i)); + } + } + + private Object normalizeValue(Object value) { + if (value instanceof BigDecimal || value instanceof BigInteger || value instanceof Long) { + return value.toString(); + } + return value; + } + + private String mapSqlError(SQLException ex) { + String state = ex.getSQLState(); + String message = ex.getMessage() == null ? "" : ex.getMessage().toLowerCase(Locale.ROOT); + if (message.contains("access denied") || message.contains("password") || message.contains("authentication")) { + return DatacenterConnectionErrorCode.AUTH_FAILED.name(); + } + if (message.contains("unknown database") || message.contains("database does not exist")) { + return DatacenterConnectionErrorCode.DATABASE_NOT_FOUND.name(); + } + if (message.contains("schema") && message.contains("does not exist")) { + return DatacenterConnectionErrorCode.SCHEMA_NOT_FOUND.name(); + } + if (message.contains("permission denied") || message.contains("insufficient privilege")) { + return DatacenterConnectionErrorCode.PERMISSION_DENIED.name(); + } + if (state != null && state.startsWith("08")) { + return DatacenterConnectionErrorCode.NETWORK_UNREACHABLE.name(); + } + return DatacenterConnectionErrorCode.UNKNOWN_ERROR.name(); + } + + @FunctionalInterface + protected interface JdbcCallback { + T apply(Connection connection) throws Exception; + } +} diff --git a/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/connector/support/DatacenterConnectorExceptionSupport.java b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/connector/support/DatacenterConnectorExceptionSupport.java new file mode 100644 index 0000000..1724fc2 --- /dev/null +++ b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/connector/support/DatacenterConnectorExceptionSupport.java @@ -0,0 +1,80 @@ +package tech.easyflow.datacenter.connector.support; + +import tech.easyflow.common.web.exceptions.BusinessException; + +import java.net.ConnectException; +import java.net.NoRouteToHostException; +import java.net.SocketException; +import java.net.SocketTimeoutException; +import java.net.UnknownHostException; +import java.sql.SQLException; +import java.util.Locale; + +public final class DatacenterConnectorExceptionSupport { + + public static final String SOURCE_UNAVAILABLE_MESSAGE = "当前连接不可用，请检查连接配置后重试"; + + private DatacenterConnectorExceptionSupport() { + } + + public static BusinessException wrapAccessException(String fallbackMessage, Exception ex) { + if (ex instanceof BusinessException businessException && !isConnectionUnavailable(ex)) { + return businessException; + } + if (isConnectionUnavailable(ex)) { + return new BusinessException(SOURCE_UNAVAILABLE_MESSAGE); + } + return new BusinessException(fallbackMessage); + } + + public static boolean isConnectionUnavailable(Throwable throwable) { + Throwable current = throwable; + while (current != null) { + if (current instanceof ClassNotFoundException + || current instanceof ConnectException + || current instanceof NoRouteToHostException + || current instanceof SocketTimeoutException + || current instanceof UnknownHostException) { + return true; + } + if (current instanceof SocketException socketException) { + String socketMessage = lowerCase(socketException.getMessage()); + if (socketMessage.contains("broken pipe") + || socketMessage.contains("connection reset") + || socketMessage.contains("network is unreachable")) { + return true; + } + } + if (current instanceof SQLException sqlException) { + String sqlState = sqlException.getSQLState(); + String message = lowerCase(sqlException.getMessage()); + if ((sqlState != null && sqlState.startsWith("08")) + || message.contains("communications link failure") + || message.contains("connection refused") + || message.contains("connection attempt failed") + || message.contains("connect timed out") + || message.contains("i/o error") + || message.contains("io error") + || message.contains("the network adapter could not establish the connection") + || message.contains("unknown host") + || message.contains("access denied") + || message.contains("authentication failed") + || message.contains("login failed") + || message.contains("password authentication failed") + || message.contains("unknown database") + || message.contains("database does not exist") + || (message.contains("schema") && message.contains("does not exist")) + || message.contains("insufficient privilege") + || message.contains("permission denied")) { + return true; + } + } + current = current.getCause(); + } + return false; + } + + private static String lowerCase(String message) { + return message == null ? "" : message.toLowerCase(Locale.ROOT); + } +} diff --git a/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/connector/support/DatacenterDatasourceManager.java b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/connector/support/DatacenterDatasourceManager.java new file mode 100644 index 0000000..ff0c177 --- /dev/null +++ b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/connector/support/DatacenterDatasourceManager.java @@ -0,0 +1,44 @@ +package tech.easyflow.datacenter.connector.support; + +import com.zaxxer.hikari.HikariConfig; +import com.zaxxer.hikari.HikariDataSource; +import org.springframework.stereotype.Component; +import tech.easyflow.datacenter.meta.entity.DatacenterSource; +import tech.easyflow.datacenter.security.DatacenterCredentialCipher; + +import javax.annotation.Resource; +import java.math.BigInteger; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +@Component +public class DatacenterDatasourceManager { + + private final Map externalDatasourceCache = new ConcurrentHashMap<>(); + + @Resource + private DatacenterCredentialCipher credentialCipher; + + public HikariDataSource createExternalDatasource(DatacenterSource source) { + HikariConfig config = new HikariConfig(); + config.setPoolName("dc-ext-" + (source.getId() == null ? "temp" : source.getId())); + config.setJdbcUrl(source.getJdbcUrl()); + config.setUsername(source.getUsername()); + config.setPassword(credentialCipher.decrypt(source.getCredentialCipher())); + config.setMaximumPoolSize(3); + config.setMinimumIdle(0); + config.setConnectionTimeout(5000); + config.setValidationTimeout(3000); + if (source.getDriverClassName() != null && !source.getDriverClassName().isBlank()) { + config.setDriverClassName(source.getDriverClassName()); + } + return new HikariDataSource(config); + } + + public HikariDataSource getOrCreateExternalDatasource(DatacenterSource source) { + if (source.getId() == null) { + return createExternalDatasource(source); + } + return externalDatasourceCache.computeIfAbsent(source.getId(), key -> createExternalDatasource(source)); + } +} diff --git a/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/entity/base/DatacenterTableBase.java b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/entity/base/DatacenterTableBase.java index 820491b..e5a49e7 100644 --- a/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/entity/base/DatacenterTableBase.java +++ b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/entity/base/DatacenterTableBase.java @@ -33,6 +33,18 @@ public class DatacenterTableBase extends DateEntity implements Serializable { @Column(tenantId = true, comment = "租户ID") private BigInteger tenantId; + /** + * 数据源ID + */ + @Column(comment = "数据源ID") + private BigInteger sourceId; + + /** + * 目录ID + */ + @Column(comment = "目录ID") + private BigInteger catalogId; + /** * 数据表名 */ @@ -51,6 +63,30 @@ public class DatacenterTableBase extends DateEntity implements Serializable { @Column(comment = "物理表名") private String actualTable; + /** + * 表类型 + */ + @Column(comment = "表类型") + private String tableKind; + + /** + * 访问模式 + */ + @Column(comment = "访问模式") + private String accessMode; + + /** + * 物化表名 + */ + @Column(comment = "物化表名") + private String materializedTable; + + /** + * 是否开启版本 + */ + @Column(comment = "是否开启版本") + private Integer versioningEnabled; + /** * 数据状态 */ @@ -87,6 +123,12 @@ public class DatacenterTableBase extends DateEntity implements Serializable { @Column(typeHandler = FastjsonTypeHandler.class, comment = "扩展项") private Map options; + /** + * 能力声明 + */ + @Column(typeHandler = FastjsonTypeHandler.class, comment = "能力声明") + private Map capabilitiesJson; + public BigInteger getId() { return id; } @@ -111,6 +153,22 @@ public class DatacenterTableBase extends DateEntity implements Serializable { this.tenantId = tenantId; } + public BigInteger getSourceId() { + return sourceId; + } + + public void setSourceId(BigInteger sourceId) { + this.sourceId = sourceId; + } + + public BigInteger getCatalogId() { + return catalogId; + } + + public void setCatalogId(BigInteger catalogId) { + this.catalogId = catalogId; + } + public String getTableName() { return tableName; } @@ -135,6 +193,38 @@ public class DatacenterTableBase extends DateEntity implements Serializable { this.actualTable = actualTable; } + public String getTableKind() { + return tableKind; + } + + public void setTableKind(String tableKind) { + this.tableKind = tableKind; + } + + public String getAccessMode() { + return accessMode; + } + + public void setAccessMode(String accessMode) { + this.accessMode = accessMode; + } + + public String getMaterializedTable() { + return materializedTable; + } + + public void setMaterializedTable(String materializedTable) { + this.materializedTable = materializedTable; + } + + public Integer getVersioningEnabled() { + return versioningEnabled; + } + + public void setVersioningEnabled(Integer versioningEnabled) { + this.versioningEnabled = versioningEnabled; + } + public Integer getStatus() { return status; } @@ -183,4 +273,12 @@ public class DatacenterTableBase extends DateEntity implements Serializable { this.options = options; } + public Map getCapabilitiesJson() { + return capabilitiesJson; + } + + public void setCapabilitiesJson(Map capabilitiesJson) { + this.capabilitiesJson = capabilitiesJson; + } + } diff --git a/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/entity/base/DatacenterTableFieldBase.java b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/entity/base/DatacenterTableFieldBase.java index b4926e4..beb0b4b 100644 --- a/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/entity/base/DatacenterTableFieldBase.java +++ b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/entity/base/DatacenterTableFieldBase.java @@ -33,6 +33,12 @@ public class DatacenterTableFieldBase extends DateEntity implements Serializable @Column(comment = "字段名称") private String fieldName; + /** + * 源字段名 + */ + @Column(comment = "源字段名") + private String sourceColumnName; + /** * 字段描述 */ @@ -45,12 +51,54 @@ public class DatacenterTableFieldBase extends DateEntity implements Serializable @Column(comment = "字段类型") private Integer fieldType; + /** + * JDBC 类型 + */ + @Column(comment = "JDBC类型") + private String jdbcType; + + /** + * 精度 + */ + @Column(comment = "精度") + private Integer precision; + + /** + * 小数位 + */ + @Column(comment = "小数位") + private Integer scale; + /** * 是否必填 */ @Column(comment = "是否必填") private Integer required; + /** + * 可查询 + */ + @Column(comment = "可查询") + private Integer queryable; + + /** + * 可排序 + */ + @Column(comment = "可排序") + private Integer sortable; + + /** + * 可写入 + */ + @Column(comment = "可写入") + private Integer writable; + + /** + * 是否索引 + */ + @Column(comment = "是否索引") + private Integer indexed; + /** * 扩展项 */ @@ -105,6 +153,14 @@ public class DatacenterTableFieldBase extends DateEntity implements Serializable this.fieldName = fieldName; } + public String getSourceColumnName() { + return sourceColumnName; + } + + public void setSourceColumnName(String sourceColumnName) { + this.sourceColumnName = sourceColumnName; + } + public String getFieldDesc() { return fieldDesc; } @@ -121,6 +177,30 @@ public class DatacenterTableFieldBase extends DateEntity implements Serializable this.fieldType = fieldType; } + public String getJdbcType() { + return jdbcType; + } + + public void setJdbcType(String jdbcType) { + this.jdbcType = jdbcType; + } + + public Integer getPrecision() { + return precision; + } + + public void setPrecision(Integer precision) { + this.precision = precision; + } + + public Integer getScale() { + return scale; + } + + public void setScale(Integer scale) { + this.scale = scale; + } + public Integer getRequired() { return required; } @@ -129,6 +209,38 @@ public class DatacenterTableFieldBase extends DateEntity implements Serializable this.required = required; } + public Integer getQueryable() { + return queryable; + } + + public void setQueryable(Integer queryable) { + this.queryable = queryable; + } + + public Integer getSortable() { + return sortable; + } + + public void setSortable(Integer sortable) { + this.sortable = sortable; + } + + public Integer getWritable() { + return writable; + } + + public void setWritable(Integer writable) { + this.writable = writable; + } + + public Integer getIndexed() { + return indexed; + } + + public void setIndexed(Integer indexed) { + this.indexed = indexed; + } + public Map getOptions() { return options; } diff --git a/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/excel/ReadDataListener.java b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/excel/ReadDataListener.java deleted file mode 100644 index f8d83ad..0000000 --- a/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/excel/ReadDataListener.java +++ /dev/null @@ -1,107 +0,0 @@ -package tech.easyflow.datacenter.excel; - -import cn.idev.excel.context.AnalysisContext; -import cn.idev.excel.metadata.data.ReadCellData; -import cn.idev.excel.read.listener.ReadListener; -import com.alibaba.fastjson2.JSONObject; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import tech.easyflow.common.entity.LoginAccount; -import tech.easyflow.common.util.SpringContextUtil; -import tech.easyflow.datacenter.entity.DatacenterTableField; -import tech.easyflow.datacenter.service.DatacenterTableService; - -import java.math.BigInteger; -import java.util.*; - -public class ReadDataListener implements ReadListener> { - - private static final Logger log = LoggerFactory.getLogger(ReadDataListener.class); - - private BigInteger tableId; - - private List fields; - - private LoginAccount loginAccount; - - private final Map headFieldIndex = new HashMap<>(); - - private int successCount = 0; - private int errorCount = 0; - private int totalCount = 0; - - private final List errorRows = new ArrayList<>(); - - public ReadDataListener() { - } - - public ReadDataListener(BigInteger tableId, List fields, LoginAccount loginAccount) { - this.tableId = tableId; - this.fields = fields; - this.loginAccount = loginAccount; - } - - @Override - public void invoke(LinkedHashMap o, AnalysisContext analysisContext) { - DatacenterTableService service = SpringContextUtil.getBean(DatacenterTableService.class); - JSONObject obj = new JSONObject(); - for (DatacenterTableField field : fields) { - String fieldName = field.getFieldName(); - Integer i = headFieldIndex.get(fieldName); - if (i != null) { - obj.put(fieldName, o.get(i)); - } - } - try { - service.saveValue(tableId, obj, loginAccount); - successCount++; - } catch (Exception e) { - errorCount++; - log.error("导入数据到数据中枢失败，具体值：{}", obj, e); - errorRows.add(obj); - } - totalCount++; - } - - @Override - public void invokeHead(Map> headMap, AnalysisContext context) { - Set>> entries = headMap.entrySet(); - for (Map.Entry> entry : entries) { - Integer key = entry.getKey(); - String field = entry.getValue().getStringValue(); - headFieldIndex.put(field, key); - } - if (headFieldIndex.size() != fields.size()) { - throw new RuntimeException("表头字段数量与表结构对应不上！"); - } - } - - @Override - public void doAfterAllAnalysed(AnalysisContext analysisContext) { - - } - - public List getFields() { - return fields; - } - - public void setFields(List fields) { - this.fields = fields; - } - - public int getSuccessCount() { - return successCount; - } - - public int getErrorCount() { - return errorCount; - } - - public int getTotalCount() { - return totalCount; - } - - public List getErrorRows() { - return errorRows; - } -} diff --git a/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/excel/ReadResVo.java b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/excel/ReadResVo.java deleted file mode 100644 index f4e636a..0000000 --- a/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/excel/ReadResVo.java +++ /dev/null @@ -1,68 +0,0 @@ -package tech.easyflow.datacenter.excel; - -import com.alibaba.fastjson2.JSONObject; - -import java.util.List; - -public class ReadResVo { - - /** - * 成功数 - */ - private int successCount = 0; - /** - * 失败数 - */ - private int errorCount = 0; - /** - * 总数 - */ - private int totalCount = 0; - - /** - * 错误行 - */ - private List errorRows; - - public ReadResVo() { - } - - public ReadResVo(int successCount, int errorCount, int totalCount, List errorRows) { - this.successCount = successCount; - this.errorCount = errorCount; - this.totalCount = totalCount; - this.errorRows = errorRows; - } - - public int getSuccessCount() { - return successCount; - } - - public void setSuccessCount(int successCount) { - this.successCount = successCount; - } - - public int getErrorCount() { - return errorCount; - } - - public void setErrorCount(int errorCount) { - this.errorCount = errorCount; - } - - public int getTotalCount() { - return totalCount; - } - - public void setTotalCount(int totalCount) { - this.totalCount = totalCount; - } - - public List getErrorRows() { - return errorRows; - } - - public void setErrorRows(List errorRows) { - this.errorRows = errorRows; - } -} diff --git a/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/excel/model/DatacenterExcelDeriveRequest.java b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/excel/model/DatacenterExcelDeriveRequest.java new file mode 100644 index 0000000..108d821 --- /dev/null +++ b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/excel/model/DatacenterExcelDeriveRequest.java @@ -0,0 +1,28 @@ +package tech.easyflow.datacenter.excel.model; + +import tech.easyflow.datacenter.execution.model.DatacenterQueryFilter; +import tech.easyflow.datacenter.execution.model.DatasetRef; + +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +public class DatacenterExcelDeriveRequest { + private DatasetRef datasetRef; + private String targetTableName; + private List selectedColumns = new ArrayList<>(); + private Map renameMappings = new LinkedHashMap<>(); + private List filters = new ArrayList<>(); + + public DatasetRef getDatasetRef() { return datasetRef; } + public void setDatasetRef(DatasetRef datasetRef) { this.datasetRef = datasetRef; } + public String getTargetTableName() { return targetTableName; } + public void setTargetTableName(String targetTableName) { this.targetTableName = targetTableName; } + public List getSelectedColumns() { return selectedColumns; } + public void setSelectedColumns(List selectedColumns) { this.selectedColumns = selectedColumns; } + public Map getRenameMappings() { return renameMappings; } + public void setRenameMappings(Map renameMappings) { this.renameMappings = renameMappings; } + public List getFilters() { return filters; } + public void setFilters(List filters) { this.filters = filters; } +} diff --git a/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/excel/model/DatacenterExcelExportRequest.java b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/excel/model/DatacenterExcelExportRequest.java new file mode 100644 index 0000000..43bca0b --- /dev/null +++ b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/excel/model/DatacenterExcelExportRequest.java @@ -0,0 +1,23 @@ +package tech.easyflow.datacenter.excel.model; + +import tech.easyflow.datacenter.execution.model.DatasetRef; + +import java.math.BigInteger; +import java.util.ArrayList; +import java.util.List; + +public class DatacenterExcelExportRequest { + private BigInteger sourceId; + private BigInteger catalogId; + private List datasetRefs = new ArrayList<>(); + private String fileName; + + public BigInteger getSourceId() { return sourceId; } + public void setSourceId(BigInteger sourceId) { this.sourceId = sourceId; } + public BigInteger getCatalogId() { return catalogId; } + public void setCatalogId(BigInteger catalogId) { this.catalogId = catalogId; } + public List getDatasetRefs() { return datasetRefs; } + public void setDatasetRefs(List datasetRefs) { this.datasetRefs = datasetRefs; } + public String getFileName() { return fileName; } + public void setFileName(String fileName) { this.fileName = fileName; } +} diff --git a/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/excel/model/DatacenterExcelMergeRequest.java b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/excel/model/DatacenterExcelMergeRequest.java new file mode 100644 index 0000000..b659c4a --- /dev/null +++ b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/excel/model/DatacenterExcelMergeRequest.java @@ -0,0 +1,22 @@ +package tech.easyflow.datacenter.excel.model; + +import tech.easyflow.datacenter.execution.model.DatasetRef; + +import java.util.ArrayList; +import java.util.List; + +public class DatacenterExcelMergeRequest { + private List datasetRefs = new ArrayList<>(); + private String mergeMode; + private String targetTableName; + private String joinKey; + + public List getDatasetRefs() { return datasetRefs; } + public void setDatasetRefs(List datasetRefs) { this.datasetRefs = datasetRefs; } + public String getMergeMode() { return mergeMode; } + public void setMergeMode(String mergeMode) { this.mergeMode = mergeMode; } + public String getTargetTableName() { return targetTableName; } + public void setTargetTableName(String targetTableName) { this.targetTableName = targetTableName; } + public String getJoinKey() { return joinKey; } + public void setJoinKey(String joinKey) { this.joinKey = joinKey; } +} diff --git a/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/excel/model/DatacenterExcelSplitRequest.java b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/excel/model/DatacenterExcelSplitRequest.java new file mode 100644 index 0000000..b244ff3 --- /dev/null +++ b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/excel/model/DatacenterExcelSplitRequest.java @@ -0,0 +1,30 @@ +package tech.easyflow.datacenter.excel.model; + +import tech.easyflow.datacenter.execution.model.DatasetRef; + +import java.math.BigInteger; + +public class DatacenterExcelSplitRequest { + private BigInteger sourceId; + private BigInteger catalogId; + private DatasetRef datasetRef; + private String splitMode; + private Integer rowBatchSize; + private String fieldName; + private String targetNamePrefix; + + public BigInteger getSourceId() { return sourceId; } + public void setSourceId(BigInteger sourceId) { this.sourceId = sourceId; } + public BigInteger getCatalogId() { return catalogId; } + public void setCatalogId(BigInteger catalogId) { this.catalogId = catalogId; } + public DatasetRef getDatasetRef() { return datasetRef; } + public void setDatasetRef(DatasetRef datasetRef) { this.datasetRef = datasetRef; } + public String getSplitMode() { return splitMode; } + public void setSplitMode(String splitMode) { this.splitMode = splitMode; } + public Integer getRowBatchSize() { return rowBatchSize; } + public void setRowBatchSize(Integer rowBatchSize) { this.rowBatchSize = rowBatchSize; } + public String getFieldName() { return fieldName; } + public void setFieldName(String fieldName) { this.fieldName = fieldName; } + public String getTargetNamePrefix() { return targetNamePrefix; } + public void setTargetNamePrefix(String targetNamePrefix) { this.targetNamePrefix = targetNamePrefix; } +} diff --git a/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/excel/service/DatacenterExcelImportService.java b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/excel/service/DatacenterExcelImportService.java new file mode 100644 index 0000000..7aae558 --- /dev/null +++ b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/excel/service/DatacenterExcelImportService.java @@ -0,0 +1,28 @@ +package tech.easyflow.datacenter.excel.service; + +import org.springframework.web.multipart.MultipartFile; +import tech.easyflow.common.entity.LoginAccount; +import tech.easyflow.datacenter.excel.model.DatacenterExcelDeriveRequest; +import tech.easyflow.datacenter.excel.model.DatacenterExcelExportRequest; +import tech.easyflow.datacenter.excel.model.DatacenterExcelMergeRequest; +import tech.easyflow.datacenter.excel.model.DatacenterExcelSplitRequest; +import tech.easyflow.datacenter.meta.entity.DatacenterImportJob; + +import java.math.BigInteger; +import java.util.List; + +public interface DatacenterExcelImportService { + DatacenterImportJob importWorkbook(MultipartFile file, LoginAccount account) throws Exception; + + DatacenterImportJob splitWorkbook(DatacenterExcelSplitRequest request, LoginAccount account); + + DatacenterImportJob mergeWorkbook(DatacenterExcelMergeRequest request, LoginAccount account); + + DatacenterImportJob deriveWorkbook(DatacenterExcelDeriveRequest request, LoginAccount account); + + DatacenterImportJob exportWorkbook(DatacenterExcelExportRequest request, LoginAccount account) throws Exception; + + DatacenterImportJob getImportJobDetail(BigInteger jobId); + + List listJobs(BigInteger sourceId, BigInteger tableId); +} diff --git a/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/excel/service/impl/DatacenterExcelImportServiceImpl.java b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/excel/service/impl/DatacenterExcelImportServiceImpl.java new file mode 100644 index 0000000..34a734f --- /dev/null +++ b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/excel/service/impl/DatacenterExcelImportServiceImpl.java @@ -0,0 +1,950 @@ +package tech.easyflow.datacenter.excel.service.impl; + +import com.alibaba.fastjson2.JSONObject; +import com.mybatisflex.core.keygen.impl.SnowFlakeIDKeyGenerator; +import com.mybatisflex.core.paginate.Page; +import com.mybatisflex.core.row.Row; +import org.apache.poi.ss.usermodel.Cell; +import org.apache.poi.ss.usermodel.DataFormatter; +import org.apache.poi.ss.usermodel.Sheet; +import org.apache.poi.ss.usermodel.Workbook; +import org.apache.poi.ss.usermodel.WorkbookFactory; +import org.apache.poi.xssf.streaming.SXSSFWorkbook; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; +import org.springframework.util.CollectionUtils; +import org.springframework.web.multipart.MultipartFile; +import tech.easyflow.common.constant.enums.EnumFieldType; +import tech.easyflow.common.entity.LoginAccount; +import tech.easyflow.common.web.exceptions.BusinessException; +import tech.easyflow.datacenter.adapter.DbHandleManager; +import tech.easyflow.datacenter.entity.DatacenterTable; +import tech.easyflow.datacenter.entity.DatacenterTableField; +import tech.easyflow.datacenter.excel.model.DatacenterExcelDeriveRequest; +import tech.easyflow.datacenter.excel.model.DatacenterExcelExportRequest; +import tech.easyflow.datacenter.excel.model.DatacenterExcelMergeRequest; +import tech.easyflow.datacenter.excel.model.DatacenterExcelSplitRequest; +import tech.easyflow.datacenter.excel.service.DatacenterExcelImportService; +import tech.easyflow.datacenter.execution.model.DatacenterQueryFilter; +import tech.easyflow.datacenter.execution.model.DatacenterQueryRequest; +import tech.easyflow.datacenter.execution.model.DatasetRef; +import tech.easyflow.datacenter.execution.service.DatacenterDatasetQueryService; +import tech.easyflow.datacenter.mapper.DatacenterDatasetVersionMapper; +import tech.easyflow.datacenter.mapper.DatacenterDerivedTableMapper; +import tech.easyflow.datacenter.mapper.DatacenterImportJobMapper; +import tech.easyflow.datacenter.meta.entity.DatacenterCatalog; +import tech.easyflow.datacenter.meta.entity.DatacenterDatasetVersion; +import tech.easyflow.datacenter.meta.entity.DatacenterDerivedTable; +import tech.easyflow.datacenter.meta.entity.DatacenterImportJob; +import tech.easyflow.datacenter.meta.entity.DatacenterSource; +import tech.easyflow.datacenter.meta.enums.DatacenterImportStatus; +import tech.easyflow.datacenter.meta.enums.DatacenterSourceType; +import tech.easyflow.datacenter.meta.enums.DatacenterTableKind; +import tech.easyflow.datacenter.meta.model.DatacenterTableDetailMeta; +import tech.easyflow.datacenter.meta.service.DatacenterDatasetRegistryService; +import tech.easyflow.datacenter.meta.service.DatacenterSourceService; + +import javax.annotation.Resource; +import java.io.File; +import java.io.FileOutputStream; +import java.io.InputStream; +import java.math.BigInteger; +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.LocalDateTime; +import java.time.format.DateTimeFormatter; +import java.util.ArrayList; +import java.util.Date; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.UUID; + +@Service +public class DatacenterExcelImportServiceImpl implements DatacenterExcelImportService { + + private static final long QUERY_BATCH_SIZE = 500L; + private static final DateTimeFormatter EXPORT_TIME_FORMAT = DateTimeFormatter.ofPattern("yyyyMMddHHmmss"); + + @Resource + private DatacenterSourceService sourceService; + @Resource + private DatacenterDatasetRegistryService registryService; + @Resource + private DatacenterImportJobMapper importJobMapper; + @Resource + private DatacenterDatasetVersionMapper datasetVersionMapper; + @Resource + private DatacenterDerivedTableMapper derivedTableMapper; + @Resource + private DatacenterDatasetQueryService queryService; + @Resource + private DbHandleManager dbHandleManager; + + @Override + @Transactional(rollbackFor = Exception.class) + public DatacenterImportJob importWorkbook(MultipartFile file, LoginAccount account) throws Exception { + if (file == null || file.isEmpty()) { + throw new BusinessException("Excel 文件不能为空"); + } + String workbookName = extractWorkbookName(file.getOriginalFilename()); + DatacenterSource source = new DatacenterSource(); + source.setSourceName(workbookName); + source.setSourceCode("EXCEL_" + UUID.randomUUID()); + source.setSourceType(DatacenterSourceType.EXCEL.name()); + source.setAccessMode("READ_WRITE"); + source.setBuiltinFlag(0); + source.setConfigJson(Map.of("originFileName", file.getOriginalFilename())); + source = sourceService.saveSource(source, account); + DatacenterCatalog catalog = registryService.ensureCatalog(source, workbookName, account); + + DatacenterImportJob job = createJob("EXCEL_IMPORT", source.getId(), catalog.getId(), null, + file.getOriginalFilename(), Map.of("operation", "import"), account); + + long totalRows = 0L; + long successRows = 0L; + List createdTableIds = new ArrayList<>(); + try (InputStream inputStream = file.getInputStream(); Workbook workbook = WorkbookFactory.create(inputStream)) { + DataFormatter formatter = new DataFormatter(); + for (int sheetIndex = 0; sheetIndex < workbook.getNumberOfSheets(); sheetIndex++) { + Sheet sheet = workbook.getSheetAt(sheetIndex); + org.apache.poi.ss.usermodel.Row headerRow = sheet.getRow(sheet.getFirstRowNum()); + if (headerRow == null) { + continue; + } + List fields = buildFields(headerRow, formatter); + if (fields.isEmpty()) { + continue; + } + DatacenterTable table = new DatacenterTable(); + table.setTableName(uniqueTableName(source.getId(), catalog.getId(), sheet.getSheetName())); + table.setTableDesc(sheet.getSheetName()); + table.setActualTable(buildMaterializedTableName(source.getId(), sheetIndex)); + table.setMaterializedTable(table.getActualTable()); + table.setTableKind(DatacenterTableKind.EXCEL_MATERIALIZED.name()); + table.setAccessMode("READ_WRITE"); + table.setVersioningEnabled(1); + table.setCapabilitiesJson(defaultExcelCapabilities()); + table.setFields(fields); + + DatacenterTableDetailMeta detail = new DatacenterTableDetailMeta(); + detail.setTable(table); + detail.setFields(fields); + + dbHandleManager.getDbHandler().createTable(table); + DatacenterTable savedTable = registryService.registerTable(source, catalog, detail, account); + savedTable.setFields(registryService.getFields(savedTable.getId())); + createdTableIds.add(savedTable.getId()); + + for (int rowIndex = sheet.getFirstRowNum() + 1; rowIndex <= sheet.getLastRowNum(); rowIndex++) { + org.apache.poi.ss.usermodel.Row row = sheet.getRow(rowIndex); + if (row == null) { + continue; + } + JSONObject payload = new JSONObject(); + boolean hasValue = false; + for (int cellIndex = 0; cellIndex < savedTable.getFields().size(); cellIndex++) { + String value = formatter.formatCellValue(row.getCell(cellIndex)); + if (value != null && !value.isBlank()) { + hasValue = true; + } + payload.put(savedTable.getFields().get(cellIndex).getFieldName(), value); + } + if (!hasValue) { + continue; + } + dbHandleManager.getDbHandler().saveValue(savedTable, payload, account); + totalRows++; + successRows++; + } + + createVersion(savedTable, "initial-import", Map.of( + "sheetName", sheet.getSheetName(), + "sourceId", source.getId(), + "originFileName", file.getOriginalFilename() + ), account); + } + job.setTableId(createdTableIds.isEmpty() ? null : createdTableIds.get(0)); + finishJobSuccess(job, totalRows, successRows, Map.of("tableIds", createdTableIds)); + return job; + } catch (Exception ex) { + finishJobFailure(job, ex); + throw ex; + } + } + + @Override + @Transactional(rollbackFor = Exception.class) + public DatacenterImportJob splitWorkbook(DatacenterExcelSplitRequest request, LoginAccount account) { + DatasetRef datasetRef = request == null ? null : request.getDatasetRef(); + DatacenterImportJob job = createJob("EXCEL_SPLIT", + request == null ? null : request.getSourceId(), + request == null ? null : request.getCatalogId(), + datasetRef == null ? null : datasetRef.getTableId(), + null, + buildPayload("request", request), + account); + try { + String splitMode = normalizeMode(request == null ? null : request.getSplitMode(), "BY_ROW_COUNT"); + return switch (splitMode) { + case "BY_SHEET" -> doSplitBySheet(request, account, job); + case "BY_FIELD_VALUE" -> doSplitByFieldValue(request, account, job); + default -> doSplitByRowCount(request, account, job); + }; + } catch (Exception ex) { + finishJobFailure(job, ex); + throw ex; + } + } + + @Override + @Transactional(rollbackFor = Exception.class) + public DatacenterImportJob mergeWorkbook(DatacenterExcelMergeRequest request, LoginAccount account) { + if (request == null || CollectionUtils.isEmpty(request.getDatasetRefs())) { + throw new BusinessException("合并数据集不能为空"); + } + DatacenterImportJob job = createJob("EXCEL_MERGE", null, null, null, null, buildPayload("request", request), account); + try { + String mergeMode = normalizeMode(request.getMergeMode(), "VERTICAL"); + return switch (mergeMode) { + case "HORIZONTAL" -> doHorizontalMerge(request, account, job); + default -> doVerticalMerge(request, account, job); + }; + } catch (Exception ex) { + finishJobFailure(job, ex); + throw ex; + } + } + + @Override + @Transactional(rollbackFor = Exception.class) + public DatacenterImportJob deriveWorkbook(DatacenterExcelDeriveRequest request, LoginAccount account) { + if (request == null || request.getDatasetRef() == null) { + throw new BusinessException("派生数据集不能为空"); + } + DatacenterImportJob job = createJob("EXCEL_DERIVE", + request.getDatasetRef().getSourceId(), + request.getDatasetRef().getCatalogId(), + request.getDatasetRef().getTableId(), + null, + buildPayload("request", request), + account); + try { + DatacenterTable sourceTable = resolveTable(request.getDatasetRef()); + DatacenterSource source = registryService.getSourceRequired(sourceTable.getSourceId()); + DatacenterCatalog catalog = requireCatalog(sourceTable.getCatalogId()); + List selectedColumns = resolveSelectedColumns(sourceTable, request.getSelectedColumns()); + List targetFields = buildDerivedFields(sourceTable, request); + DatacenterTable targetTable = createDerivedTable(source, catalog, targetFields, + request.getTargetTableName(), "DERIVE", Map.of("sourceTableId", sourceTable.getId()), account); + + DatacenterQueryRequest queryRequest = new DatacenterQueryRequest(); + queryRequest.setDatasetRef(registryService.resolveDatasetRef(sourceTable.getId())); + queryRequest.setSelectedColumns(selectedColumns); + queryRequest.setFilters(request.getFilters()); + + long successRows = copyRows(queryRequest, rows -> mapDerivedRow(rows, selectedColumns, request), targetTable, account); + createLineage(sourceTable.getId(), targetTable.getId(), "DERIVE", Map.of("request", request), account); + finishJobSuccess(job, successRows, successRows, Map.of("derivedTableId", targetTable.getId())); + job.setTableId(targetTable.getId()); + return job; + } catch (Exception ex) { + finishJobFailure(job, ex); + throw ex; + } + } + + @Override + @Transactional(rollbackFor = Exception.class) + public DatacenterImportJob exportWorkbook(DatacenterExcelExportRequest request, LoginAccount account) throws Exception { + List tables = resolveExportTables(request); + if (tables.isEmpty()) { + throw new BusinessException("没有可导出的 Excel 数据集"); + } + String fileName = buildExportFileName(request == null ? null : request.getFileName()); + DatacenterImportJob job = createJob("EXCEL_EXPORT", + request == null ? null : request.getSourceId(), + request == null ? null : request.getCatalogId(), + null, + fileName, + buildPayload("request", request), + account); + Path exportDir = ensureExportDir(); + Path exportFile = exportDir.resolve(fileName); + long totalRows = 0L; + try (SXSSFWorkbook workbook = new SXSSFWorkbook(200); FileOutputStream outputStream = new FileOutputStream(exportFile.toFile())) { + workbook.setCompressTempFiles(true); + Set usedSheetNames = new HashSet<>(); + for (DatacenterTable table : tables) { + String sheetName = uniqueSheetName(table.getTableName(), usedSheetNames); + org.apache.poi.ss.usermodel.Sheet sheet = workbook.createSheet(sheetName); + writeHeaderRow(sheet, table.getFields()); + DatacenterQueryRequest queryRequest = new DatacenterQueryRequest(); + queryRequest.setDatasetRef(registryService.resolveDatasetRef(table.getId())); + queryRequest.setSelectedColumns(table.getFields().stream().map(DatacenterTableField::getFieldName).toList()); + final int[] rowIndex = {1}; + totalRows += iterateRows(queryRequest, row -> { + org.apache.poi.ss.usermodel.Row excelRow = sheet.createRow(rowIndex[0]++); + for (int i = 0; i < table.getFields().size(); i++) { + Cell cell = excelRow.createCell(i); + Object value = row.get(table.getFields().get(i).getFieldName()); + cell.setCellValue(value == null ? "" : String.valueOf(value)); + } + }); + } + workbook.write(outputStream); + workbook.dispose(); + job.setStoragePath(exportFile.toAbsolutePath().toString()); + finishJobSuccess(job, totalRows, totalRows, Map.of("storagePath", job.getStoragePath(), "fileName", fileName)); + return job; + } catch (Exception ex) { + finishJobFailure(job, ex); + throw ex; + } + } + + @Override + public DatacenterImportJob getImportJobDetail(BigInteger jobId) { + DatacenterImportJob job = importJobMapper.selectOneById(jobId); + if (job == null) { + throw new BusinessException("导入任务不存在: " + jobId); + } + return job; + } + + @Override + public List listJobs(BigInteger sourceId, BigInteger tableId) { + var wrapper = com.mybatisflex.core.query.QueryWrapper.create(); + if (sourceId != null) { + wrapper.eq(DatacenterImportJob::getSourceId, sourceId); + } + if (tableId != null) { + wrapper.eq(DatacenterImportJob::getTableId, tableId); + } + wrapper.orderBy("created desc"); + wrapper.limit(20L); + return importJobMapper.selectListByQuery(wrapper); + } + + private DatacenterImportJob doSplitBySheet(DatacenterExcelSplitRequest request, LoginAccount account, DatacenterImportJob job) { + BigInteger sourceId = request.getSourceId(); + BigInteger catalogId = request.getCatalogId(); + if (sourceId == null && request.getDatasetRef() != null) { + sourceId = request.getDatasetRef().getSourceId(); + catalogId = request.getDatasetRef().getCatalogId(); + } + if (sourceId == null) { + throw new BusinessException("按 sheet 拆分需要 sourceId"); + } + DatacenterSource source = registryService.getSourceRequired(sourceId); + DatacenterCatalog catalog = requireCatalog(catalogId); + List sourceTables = registryService.listManagedTables(sourceId, catalogId); + if (sourceTables.isEmpty()) { + throw new BusinessException("当前 workbook 下没有可拆分的 sheet 表"); + } + List derivedIds = new ArrayList<>(); + long successRows = 0L; + for (DatacenterTable sourceTable : sourceTables) { + DatacenterTable fullTable = registryService.getTableWithFields(sourceTable.getId()); + DatacenterTable targetTable = createDerivedTable(source, catalog, cloneFields(fullTable.getFields()), + resolveSplitPrefix(request, fullTable.getTableName()) + "_copy", "SPLIT_BY_SHEET", + Map.of("sourceTableId", fullTable.getId()), account); + DatacenterQueryRequest queryRequest = new DatacenterQueryRequest(); + queryRequest.setDatasetRef(registryService.resolveDatasetRef(fullTable.getId())); + queryRequest.setSelectedColumns(fullTable.getFields().stream().map(DatacenterTableField::getFieldName).toList()); + successRows += copyRows(queryRequest, this::mapRow, targetTable, account); + createLineage(fullTable.getId(), targetTable.getId(), "SPLIT_BY_SHEET", Map.of("sourceTableId", fullTable.getId()), account); + derivedIds.add(targetTable.getId()); + } + finishJobSuccess(job, successRows, successRows, Map.of("derivedTableIds", derivedIds)); + return job; + } + + private DatacenterImportJob doSplitByRowCount(DatacenterExcelSplitRequest request, LoginAccount account, DatacenterImportJob job) { + if (request == null || request.getDatasetRef() == null) { + throw new BusinessException("按行数拆分需要数据集"); + } + int rowBatchSize = request.getRowBatchSize() == null || request.getRowBatchSize() < 1 ? 1000 : request.getRowBatchSize(); + DatacenterTable sourceTable = resolveTable(request.getDatasetRef()); + DatacenterSource source = registryService.getSourceRequired(sourceTable.getSourceId()); + DatacenterCatalog catalog = requireCatalog(sourceTable.getCatalogId()); + String baseName = resolveSplitPrefix(request, sourceTable.getTableName()); + List derivedIds = new ArrayList<>(); + final Holder holder = new Holder(); + long totalRows = iterateRows(buildFullQuery(sourceTable), row -> { + if (holder.targetTable == null || holder.currentSize >= rowBatchSize) { + holder.batchNo++; + holder.targetTable = createDerivedTable(source, catalog, cloneFields(sourceTable.getFields()), + baseName + "_part_" + holder.batchNo, "SPLIT_BY_ROW_COUNT", + Map.of("sourceTableId", sourceTable.getId(), "batchNo", holder.batchNo, "rowBatchSize", rowBatchSize), account); + createLineage(sourceTable.getId(), holder.targetTable.getId(), "SPLIT_BY_ROW_COUNT", + Map.of("sourceTableId", sourceTable.getId(), "batchNo", holder.batchNo), account); + derivedIds.add(holder.targetTable.getId()); + holder.currentSize = 0; + } + saveToTable(holder.targetTable, mapRow(row), account); + holder.currentSize++; + }); + finishJobSuccess(job, totalRows, totalRows, Map.of("derivedTableIds", derivedIds)); + return job; + } + + private DatacenterImportJob doSplitByFieldValue(DatacenterExcelSplitRequest request, LoginAccount account, DatacenterImportJob job) { + if (request == null || request.getDatasetRef() == null || request.getFieldName() == null || request.getFieldName().isBlank()) { + throw new BusinessException("按字段值拆分需要数据集和字段名"); + } + DatacenterTable sourceTable = resolveTable(request.getDatasetRef()); + DatacenterSource source = registryService.getSourceRequired(sourceTable.getSourceId()); + DatacenterCatalog catalog = requireCatalog(sourceTable.getCatalogId()); + DatacenterTableField splitField = sourceTable.getFields().stream() + .filter(field -> request.getFieldName().equals(field.getFieldName())) + .findFirst() + .orElseThrow(() -> new BusinessException("拆分字段不存在: " + request.getFieldName())); + String prefix = resolveSplitPrefix(request, sourceTable.getTableName()); + Map targets = new LinkedHashMap<>(); + List derivedIds = new ArrayList<>(); + long totalRows = iterateRows(buildFullQuery(sourceTable), row -> { + String fieldValue = stringify(row.get(splitField.getFieldName())); + String bucket = fieldValue == null || fieldValue.isBlank() ? "empty" : fieldValue; + DatacenterTable targetTable = targets.get(bucket); + if (targetTable == null) { + targetTable = createDerivedTable(source, catalog, cloneFields(sourceTable.getFields()), + prefix + "_" + normalizeIdentifier(bucket), "SPLIT_BY_FIELD_VALUE", + Map.of("sourceTableId", sourceTable.getId(), "fieldName", splitField.getFieldName(), "fieldValue", bucket), account); + createLineage(sourceTable.getId(), targetTable.getId(), "SPLIT_BY_FIELD_VALUE", + Map.of("sourceTableId", sourceTable.getId(), "fieldName", splitField.getFieldName(), "fieldValue", bucket), account); + targets.put(bucket, targetTable); + derivedIds.add(targetTable.getId()); + } + saveToTable(targetTable, mapRow(row), account); + }); + finishJobSuccess(job, totalRows, totalRows, Map.of("derivedTableIds", derivedIds)); + return job; + } + + private DatacenterImportJob doVerticalMerge(DatacenterExcelMergeRequest request, LoginAccount account, DatacenterImportJob job) { + List tables = request.getDatasetRefs().stream().map(this::resolveTable).toList(); + DatacenterTable firstTable = tables.get(0); + DatacenterSource source = registryService.getSourceRequired(firstTable.getSourceId()); + DatacenterCatalog catalog = requireCatalog(firstTable.getCatalogId()); + assertSameCatalog(tables); + assertSameFields(tables); + DatacenterTable targetTable = createDerivedTable(source, catalog, cloneFields(firstTable.getFields()), + request.getTargetTableName(), "MERGE_VERTICAL", Map.of("sourceTableIds", tables.stream().map(DatacenterTable::getId).toList()), account); + long successRows = 0L; + for (DatacenterTable table : tables) { + successRows += copyRows(buildFullQuery(table), this::mapRow, targetTable, account); + createLineage(table.getId(), targetTable.getId(), "MERGE_VERTICAL", Map.of("sourceTableId", table.getId()), account); + } + job.setTableId(targetTable.getId()); + finishJobSuccess(job, successRows, successRows, Map.of("derivedTableId", targetTable.getId())); + return job; + } + + private DatacenterImportJob doHorizontalMerge(DatacenterExcelMergeRequest request, LoginAccount account, DatacenterImportJob job) { + if (request.getJoinKey() == null || request.getJoinKey().isBlank()) { + throw new BusinessException("横向合并必须指定 joinKey"); + } + List tables = request.getDatasetRefs().stream().map(this::resolveTable).toList(); + DatacenterTable firstTable = tables.get(0); + DatacenterSource source = registryService.getSourceRequired(firstTable.getSourceId()); + DatacenterCatalog catalog = requireCatalog(firstTable.getCatalogId()); + assertSameCatalog(tables); + + List mergedFields = new ArrayList<>(); + Set usedFieldNames = new LinkedHashSet<>(); + Map> fieldMappings = new LinkedHashMap<>(); + for (DatacenterTable table : tables) { + Map mapping = new LinkedHashMap<>(); + for (DatacenterTableField field : table.getFields()) { + String targetFieldName; + if (field.getFieldName().equals(request.getJoinKey())) { + targetFieldName = field.getFieldName(); + } else { + targetFieldName = field.getFieldName(); + if (usedFieldNames.contains(targetFieldName)) { + targetFieldName = normalizeIdentifier(table.getTableName()) + "_" + targetFieldName; + } + } + if (!usedFieldNames.contains(targetFieldName)) { + usedFieldNames.add(targetFieldName); + mergedFields.add(cloneField(field, targetFieldName, field.getFieldDesc())); + } + mapping.put(field.getFieldName(), targetFieldName); + } + fieldMappings.put(table.getId(), mapping); + } + + DatacenterTable targetTable = createDerivedTable(source, catalog, mergedFields, + request.getTargetTableName(), "MERGE_HORIZONTAL", Map.of("sourceTableIds", tables.stream().map(DatacenterTable::getId).toList(), "joinKey", request.getJoinKey()), account); + Map mergedRows = new LinkedHashMap<>(); + for (DatacenterTable table : tables) { + Map mapping = fieldMappings.get(table.getId()); + iterateRows(buildFullQuery(table), row -> { + String joinValue = stringify(row.get(request.getJoinKey())); + if (joinValue == null || joinValue.isBlank()) { + return; + } + JSONObject target = mergedRows.computeIfAbsent(joinValue, key -> new JSONObject()); + mapping.forEach((sourceField, targetField) -> target.put(targetField, row.get(sourceField))); + }); + createLineage(table.getId(), targetTable.getId(), "MERGE_HORIZONTAL", Map.of("sourceTableId", table.getId(), "joinKey", request.getJoinKey()), account); + } + for (JSONObject row : mergedRows.values()) { + saveToTable(targetTable, row, account); + } + job.setTableId(targetTable.getId()); + finishJobSuccess(job, (long) mergedRows.size(), (long) mergedRows.size(), Map.of("derivedTableId", targetTable.getId())); + return job; + } + + private DatacenterImportJob createJob(String jobType, BigInteger sourceId, BigInteger catalogId, BigInteger tableId, + String fileName, Map payload, LoginAccount account) { + DatacenterImportJob job = new DatacenterImportJob(); + job.setSourceId(sourceId); + job.setCatalogId(catalogId); + job.setTableId(tableId); + job.setTenantId(account == null || account.getTenantId() == null ? BigInteger.ZERO : account.getTenantId()); + job.setDeptId(account == null || account.getDeptId() == null ? BigInteger.ZERO : account.getDeptId()); + job.setJobType(jobType); + job.setFileName(fileName); + job.setStatus(DatacenterImportStatus.RUNNING.name()); + job.setPayloadJson(payload == null ? new LinkedHashMap<>() : new LinkedHashMap<>(payload)); + job.setStartedAt(new Date()); + job.setCreated(new Date()); + job.setModified(new Date()); + job.setCreatedBy(account == null ? BigInteger.ZERO : account.getId()); + job.setModifiedBy(account == null ? BigInteger.ZERO : account.getId()); + importJobMapper.insert(job); + return job; + } + + private Map buildPayload(String key, Object value) { + Map payload = new LinkedHashMap<>(); + payload.put(key, value); + return payload; + } + + private void finishJobSuccess(DatacenterImportJob job, long totalRows, long successRows, Map payload) { + job.setStatus(DatacenterImportStatus.SUCCESS.name()); + job.setTotalRows(totalRows); + job.setSuccessRows(successRows); + job.setErrorRows(Math.max(0L, totalRows - successRows)); + if (payload != null) { + job.setPayloadJson(new LinkedHashMap<>(payload)); + } + job.setFinishedAt(new Date()); + job.setModified(new Date()); + importJobMapper.update(job); + } + + private void finishJobFailure(DatacenterImportJob job, Exception ex) { + job.setStatus(DatacenterImportStatus.FAILED.name()); + job.setErrorSummary(ex.getMessage()); + job.setFinishedAt(new Date()); + job.setModified(new Date()); + importJobMapper.update(job); + } + + private DatacenterTable resolveTable(DatasetRef datasetRef) { + if (datasetRef == null || datasetRef.getTableId() == null) { + throw new BusinessException("缺少数据集 tableId"); + } + return registryService.getTableWithFields(datasetRef.getTableId()); + } + + private DatacenterCatalog requireCatalog(BigInteger catalogId) { + DatacenterCatalog catalog = registryService.getCatalogById(catalogId); + if (catalog == null) { + throw new BusinessException("目录不存在: " + catalogId); + } + return catalog; + } + + private DatacenterTable createDerivedTable(DatacenterSource source, DatacenterCatalog catalog, List fields, + String tableName, String deriveType, Map config, LoginAccount account) { + DatacenterTable table = new DatacenterTable(); + String resolvedName = uniqueTableName(source.getId(), catalog.getId(), normalizeLogicalName(tableName, deriveType)); + table.setTableName(resolvedName); + table.setTableDesc(resolvedName); + table.setActualTable(buildMaterializedTableName(source.getId(), Math.abs(Objects.hash(resolvedName, deriveType)))); + table.setMaterializedTable(table.getActualTable()); + table.setTableKind(DatacenterTableKind.DERIVED_TABLE.name()); + table.setAccessMode("READ_WRITE"); + table.setVersioningEnabled(1); + table.setCapabilitiesJson(defaultExcelCapabilities()); + table.setFields(fields); + + DatacenterTableDetailMeta detail = new DatacenterTableDetailMeta(); + detail.setTable(table); + detail.setFields(fields); + dbHandleManager.getDbHandler().createTable(table); + DatacenterTable savedTable = registryService.registerTable(source, catalog, detail, account); + savedTable.setFields(registryService.getFields(savedTable.getId())); + createVersion(savedTable, deriveType.toLowerCase(Locale.ROOT), config, account); + return savedTable; + } + + private DatacenterDatasetVersion createVersion(DatacenterTable table, String versionLabel, Map snapshot, LoginAccount account) { + QueryWrapperWrapper wrapper = new QueryWrapperWrapper(table.getId()); + DatacenterDatasetVersion version = new DatacenterDatasetVersion(); + version.setTableId(table.getId()); + version.setTenantId(table.getTenantId()); + version.setDeptId(table.getDeptId()); + version.setVersionNo(wrapper.nextVersionNo(datasetVersionMapper)); + version.setVersionLabel(versionLabel); + version.setMaterializedTable(table.getMaterializedTable()); + version.setSnapshotJson(snapshot == null ? new LinkedHashMap<>() : new LinkedHashMap<>(snapshot)); + version.setStatus(0); + version.setCreated(new Date()); + version.setModified(new Date()); + version.setCreatedBy(account == null ? BigInteger.ZERO : account.getId()); + version.setModifiedBy(account == null ? BigInteger.ZERO : account.getId()); + datasetVersionMapper.insert(version); + return version; + } + + private void createLineage(BigInteger sourceTableId, BigInteger derivedTableId, String deriveType, Map config, LoginAccount account) { + DatacenterDerivedTable relation = new DatacenterDerivedTable(); + relation.setSourceTableId(sourceTableId); + relation.setDerivedTableId(derivedTableId); + relation.setDeriveType(deriveType); + relation.setDeriveConfigJson(config == null ? new LinkedHashMap<>() : new LinkedHashMap<>(config)); + relation.setStatus(0); + relation.setTenantId(account == null || account.getTenantId() == null ? BigInteger.ZERO : account.getTenantId()); + relation.setDeptId(account == null || account.getDeptId() == null ? BigInteger.ZERO : account.getDeptId()); + relation.setCreated(new Date()); + relation.setModified(new Date()); + relation.setCreatedBy(account == null ? BigInteger.ZERO : account.getId()); + relation.setModifiedBy(account == null ? BigInteger.ZERO : account.getId()); + derivedTableMapper.insert(relation); + } + + private long copyRows(DatacenterQueryRequest queryRequest, RowMapper mapper, DatacenterTable targetTable, LoginAccount account) { + return iterateRows(queryRequest, row -> saveToTable(targetTable, mapper.map(row), account)); + } + + private long iterateRows(DatacenterQueryRequest queryRequest, RowConsumer consumer) { + long total = 0L; + long pageNumber = 1L; + while (true) { + queryRequest.setPageNumber(pageNumber); + queryRequest.setPageSize(QUERY_BATCH_SIZE); + Page page = queryService.queryPage(queryRequest); + if (page.getRecords() == null || page.getRecords().isEmpty()) { + break; + } + for (Row row : page.getRecords()) { + consumer.accept(row); + total++; + } + if (page.getRecords().size() < QUERY_BATCH_SIZE) { + break; + } + pageNumber++; + } + return total; + } + + private void saveToTable(DatacenterTable targetTable, JSONObject data, LoginAccount account) { + dbHandleManager.getDbHandler().saveValue(targetTable, data, account); + } + + private DatacenterQueryRequest buildFullQuery(DatacenterTable table) { + DatacenterQueryRequest queryRequest = new DatacenterQueryRequest(); + queryRequest.setDatasetRef(registryService.resolveDatasetRef(table.getId())); + queryRequest.setSelectedColumns(table.getFields().stream().map(DatacenterTableField::getFieldName).toList()); + return queryRequest; + } + + private JSONObject mapRow(Row row) { + JSONObject payload = new JSONObject(); + row.forEach(payload::put); + payload.remove("id"); + payload.remove("dept_id"); + payload.remove("tenant_id"); + payload.remove("created"); + payload.remove("created_by"); + payload.remove("modified"); + payload.remove("modified_by"); + payload.remove("remark"); + return payload; + } + + private JSONObject mapDerivedRow(Row row, List selectedColumns, DatacenterExcelDeriveRequest request) { + JSONObject payload = new JSONObject(); + for (String column : selectedColumns) { + String targetName = request.getRenameMappings().getOrDefault(column, column); + payload.put(targetName, row.get(column)); + } + return payload; + } + + private List resolveSelectedColumns(DatacenterTable sourceTable, List selectedColumns) { + if (CollectionUtils.isEmpty(selectedColumns)) { + return sourceTable.getFields().stream().map(DatacenterTableField::getFieldName).toList(); + } + return selectedColumns; + } + + private List buildDerivedFields(DatacenterTable sourceTable, DatacenterExcelDeriveRequest request) { + List selectedColumns = resolveSelectedColumns(sourceTable, request.getSelectedColumns()); + Map fieldMap = new LinkedHashMap<>(); + for (DatacenterTableField field : sourceTable.getFields()) { + fieldMap.put(field.getFieldName(), field); + } + List fields = new ArrayList<>(); + for (String column : selectedColumns) { + DatacenterTableField sourceField = fieldMap.get(column); + if (sourceField == null) { + throw new BusinessException("派生字段不存在: " + column); + } + String targetName = request.getRenameMappings().getOrDefault(column, column); + fields.add(cloneField(sourceField, targetName, targetName)); + } + return fields; + } + + private List cloneFields(List sourceFields) { + List fields = new ArrayList<>(); + for (DatacenterTableField field : sourceFields) { + fields.add(cloneField(field, field.getFieldName(), field.getFieldDesc())); + } + return fields; + } + + private DatacenterTableField cloneField(DatacenterTableField source, String fieldName, String fieldDesc) { + DatacenterTableField field = new DatacenterTableField(); + field.setFieldName(fieldName); + field.setSourceColumnName(source.getSourceColumnName()); + field.setFieldDesc(fieldDesc); + field.setFieldType(source.getFieldType()); + field.setJdbcType(source.getJdbcType()); + field.setPrecision(source.getPrecision()); + field.setScale(source.getScale()); + field.setRequired(source.getRequired()); + field.setQueryable(source.getQueryable()); + field.setSortable(source.getSortable()); + field.setWritable(source.getWritable()); + field.setIndexed(source.getIndexed()); + field.setOptions(source.getOptions()); + return field; + } + + private Map defaultExcelCapabilities() { + return Map.of("capabilities", List.of("READ_QUERY", "WRITE_MUTATION", "MATERIALIZE", "EXPORT")); + } + + private void assertSameCatalog(List tables) { + Set catalogIds = new HashSet<>(); + Set sourceIds = new HashSet<>(); + for (DatacenterTable table : tables) { + catalogIds.add(table.getCatalogId()); + sourceIds.add(table.getSourceId()); + } + if (catalogIds.size() > 1 || sourceIds.size() > 1) { + throw new BusinessException("Excel 操作暂只支持同一 workbook/catalog 下的数据集"); + } + } + + private void assertSameFields(List tables) { + List first = tables.get(0).getFields().stream().map(DatacenterTableField::getFieldName).toList(); + for (int i = 1; i < tables.size(); i++) { + List current = tables.get(i).getFields().stream().map(DatacenterTableField::getFieldName).toList(); + if (!first.equals(current)) { + throw new BusinessException("纵向合并仅支持同结构表"); + } + } + } + + private List resolveExportTables(DatacenterExcelExportRequest request) { + List tables = new ArrayList<>(); + if (request != null && !CollectionUtils.isEmpty(request.getDatasetRefs())) { + for (DatasetRef datasetRef : request.getDatasetRefs()) { + tables.add(resolveTable(datasetRef)); + } + return tables; + } + if (request == null || request.getSourceId() == null) { + throw new BusinessException("导出需要 sourceId 或 datasetRefs"); + } + tables.addAll(registryService.listManagedTables(request.getSourceId(), request.getCatalogId())); + return tables.stream().map(table -> registryService.getTableWithFields(table.getId())).toList(); + } + + private void writeHeaderRow(org.apache.poi.ss.usermodel.Sheet sheet, List fields) { + org.apache.poi.ss.usermodel.Row headerRow = sheet.createRow(0); + for (int i = 0; i < fields.size(); i++) { + Cell cell = headerRow.createCell(i); + cell.setCellValue(fields.get(i).getFieldDesc()); + } + } + + private Path ensureExportDir() throws Exception { + Path dir = Path.of(System.getProperty("java.io.tmpdir"), "easyflow-datacenter", "exports"); + Files.createDirectories(dir); + return dir; + } + + private String buildExportFileName(String rawFileName) { + String baseName = rawFileName == null || rawFileName.isBlank() ? "excel_export" : extractWorkbookName(rawFileName); + return normalizeIdentifier(baseName) + "_" + EXPORT_TIME_FORMAT.format(LocalDateTime.now()) + ".xlsx"; + } + + private String uniqueSheetName(String rawName, Set usedSheetNames) { + String base = rawName == null || rawName.isBlank() ? "Sheet" : rawName; + base = base.length() > 31 ? base.substring(0, 31) : base; + String result = base; + int suffix = 1; + while (usedSheetNames.contains(result)) { + String suffixText = "_" + suffix++; + int limit = Math.max(1, 31 - suffixText.length()); + result = base.substring(0, Math.min(base.length(), limit)) + suffixText; + } + usedSheetNames.add(result); + return result; + } + + private List buildFields(org.apache.poi.ss.usermodel.Row headerRow, DataFormatter formatter) { + List fields = new ArrayList<>(); + Set usedNames = new HashSet<>(); + short lastCellNum = headerRow.getLastCellNum(); + for (int cellIndex = 0; cellIndex < lastCellNum; cellIndex++) { + String header = formatter.formatCellValue(headerRow.getCell(cellIndex)); + String fieldName = normalizeIdentifier(header, cellIndex, usedNames); + DatacenterTableField field = new DatacenterTableField(); + field.setFieldName(fieldName); + field.setSourceColumnName(header); + field.setFieldDesc(header == null || header.isBlank() ? fieldName : header); + field.setFieldType(EnumFieldType.STRING.getCode()); + field.setJdbcType("VARCHAR"); + field.setPrecision(255); + field.setScale(0); + field.setRequired(0); + field.setQueryable(1); + field.setSortable(1); + field.setWritable(1); + field.setIndexed(0); + fields.add(field); + } + return fields; + } + + private String normalizeMode(String value, String defaultValue) { + return value == null || value.isBlank() ? defaultValue : value.trim().toUpperCase(Locale.ROOT); + } + + private String resolveSplitPrefix(DatacenterExcelSplitRequest request, String fallback) { + if (request != null && request.getTargetNamePrefix() != null && !request.getTargetNamePrefix().isBlank()) { + return request.getTargetNamePrefix(); + } + return fallback + "_split"; + } + + private String normalizeLogicalName(String tableName, String deriveType) { + if (tableName != null && !tableName.isBlank()) { + return tableName; + } + return deriveType.toLowerCase(Locale.ROOT) + "_" + System.currentTimeMillis(); + } + + private String uniqueTableName(BigInteger sourceId, BigInteger catalogId, String rawName) { + String baseName = rawName == null || rawName.isBlank() ? "dataset" : rawName; + baseName = baseName.trim(); + String result = baseName; + int suffix = 1; + while (tableNameExists(sourceId, catalogId, result)) { + result = baseName + "_" + suffix++; + } + return result; + } + + private boolean tableNameExists(BigInteger sourceId, BigInteger catalogId, String tableName) { + List tables = registryService.listManagedTables(sourceId, catalogId); + return tables.stream().anyMatch(table -> tableName.equals(table.getTableName())); + } + + private String extractWorkbookName(String originalFileName) { + if (originalFileName == null || originalFileName.isBlank()) { + return "excel_workbook"; + } + int index = originalFileName.lastIndexOf('.'); + return index > 0 ? originalFileName.substring(0, index) : originalFileName; + } + + private String buildMaterializedTableName(BigInteger sourceId, int sheetIndex) { + long snowId = new SnowFlakeIDKeyGenerator().nextId(); + return "tb_excel_" + sourceId + "_" + sheetIndex + "_" + snowId; + } + + private String normalizeIdentifier(String raw) { + if (raw == null || raw.isBlank()) { + return "value"; + } + String normalized = raw.trim().toLowerCase(Locale.ROOT).replaceAll("[^a-z0-9_\\u4e00-\\u9fa5]+", "_"); + normalized = normalized.replaceAll("_+", "_"); + if (normalized.isBlank()) { + return "value"; + } + return normalized; + } + + private String normalizeIdentifier(String raw, int index, Set usedNames) { + String value = normalizeIdentifier(raw); + if (value.isBlank() || "value".equals(value)) { + value = "col_" + (index + 1); + } + if (Character.isDigit(value.charAt(0))) { + value = "col_" + value; + } + String result = value; + int suffix = 1; + while (usedNames.contains(result)) { + result = value + "_" + suffix++; + } + usedNames.add(result); + return result; + } + + private String stringify(Object value) { + return value == null ? null : String.valueOf(value); + } + + private static final class Holder { + private DatacenterTable targetTable; + private int batchNo; + private int currentSize; + } + + private interface RowConsumer { + void accept(Row row); + } + + private interface RowMapper { + JSONObject map(Row row); + } + + private static final class QueryWrapperWrapper { + private final BigInteger tableId; + + private QueryWrapperWrapper(BigInteger tableId) { + this.tableId = tableId; + } + + private int nextVersionNo(DatacenterDatasetVersionMapper mapper) { + return mapper.selectListByQuery(com.mybatisflex.core.query.QueryWrapper.create() + .eq(DatacenterDatasetVersion::getTableId, tableId) + .orderBy("version_no desc")) + .stream() + .findFirst() + .map(version -> version.getVersionNo() + 1) + .orElse(1); + } + } +} diff --git a/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/execution/model/DatacenterConnectionTestResult.java b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/execution/model/DatacenterConnectionTestResult.java new file mode 100644 index 0000000..be7f151 --- /dev/null +++ b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/execution/model/DatacenterConnectionTestResult.java @@ -0,0 +1,24 @@ +package tech.easyflow.datacenter.execution.model; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +public class DatacenterConnectionTestResult { + private boolean success; + private String errorCode; + private String message; + private List capabilities = new ArrayList<>(); + private Map details; + + public boolean isSuccess() { return success; } + public void setSuccess(boolean success) { this.success = success; } + public String getErrorCode() { return errorCode; } + public void setErrorCode(String errorCode) { this.errorCode = errorCode; } + public String getMessage() { return message; } + public void setMessage(String message) { this.message = message; } + public List getCapabilities() { return capabilities; } + public void setCapabilities(List capabilities) { this.capabilities = capabilities; } + public Map getDetails() { return details; } + public void setDetails(Map details) { this.details = details; } +} diff --git a/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/execution/model/DatacenterQueryFilter.java b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/execution/model/DatacenterQueryFilter.java new file mode 100644 index 0000000..eda3101 --- /dev/null +++ b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/execution/model/DatacenterQueryFilter.java @@ -0,0 +1,19 @@ +package tech.easyflow.datacenter.execution.model; + +import java.util.List; + +public class DatacenterQueryFilter { + private String column; + private String operator; + private Object value; + private List values; + + public String getColumn() { return column; } + public void setColumn(String column) { this.column = column; } + public String getOperator() { return operator; } + public void setOperator(String operator) { this.operator = operator; } + public Object getValue() { return value; } + public void setValue(Object value) { this.value = value; } + public List getValues() { return values; } + public void setValues(List values) { this.values = values; } +} diff --git a/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/execution/model/DatacenterQueryRequest.java b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/execution/model/DatacenterQueryRequest.java new file mode 100644 index 0000000..177a054 --- /dev/null +++ b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/execution/model/DatacenterQueryRequest.java @@ -0,0 +1,29 @@ +package tech.easyflow.datacenter.execution.model; + +import java.util.ArrayList; +import java.util.List; + +public class DatacenterQueryRequest { + private DatasetRef datasetRef; + private Long pageNumber = 1L; + private Long pageSize = 10L; + private List filters = new ArrayList<>(); + private List sorts = new ArrayList<>(); + private List selectedColumns = new ArrayList<>(); + private String where; + + public DatasetRef getDatasetRef() { return datasetRef; } + public void setDatasetRef(DatasetRef datasetRef) { this.datasetRef = datasetRef; } + public Long getPageNumber() { return pageNumber; } + public void setPageNumber(Long pageNumber) { this.pageNumber = pageNumber; } + public Long getPageSize() { return pageSize; } + public void setPageSize(Long pageSize) { this.pageSize = pageSize; } + public List getFilters() { return filters; } + public void setFilters(List filters) { this.filters = filters; } + public List getSorts() { return sorts; } + public void setSorts(List sorts) { this.sorts = sorts; } + public List getSelectedColumns() { return selectedColumns; } + public void setSelectedColumns(List selectedColumns) { this.selectedColumns = selectedColumns; } + public String getWhere() { return where; } + public void setWhere(String where) { this.where = where; } +} diff --git a/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/execution/model/DatacenterQuerySort.java b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/execution/model/DatacenterQuerySort.java new file mode 100644 index 0000000..149db1d --- /dev/null +++ b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/execution/model/DatacenterQuerySort.java @@ -0,0 +1,11 @@ +package tech.easyflow.datacenter.execution.model; + +public class DatacenterQuerySort { + private String column; + private String direction; + + public String getColumn() { return column; } + public void setColumn(String column) { this.column = column; } + public String getDirection() { return direction; } + public void setDirection(String direction) { this.direction = direction; } +} diff --git a/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/execution/model/DatacenterSchemaResponse.java b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/execution/model/DatacenterSchemaResponse.java new file mode 100644 index 0000000..b87765b --- /dev/null +++ b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/execution/model/DatacenterSchemaResponse.java @@ -0,0 +1,39 @@ +package tech.easyflow.datacenter.execution.model; + +import tech.easyflow.datacenter.entity.DatacenterTable; +import tech.easyflow.datacenter.entity.DatacenterTableField; +import tech.easyflow.datacenter.meta.entity.DatacenterCatalog; +import tech.easyflow.datacenter.meta.entity.DatacenterDatasetVersion; +import tech.easyflow.datacenter.meta.entity.DatacenterDerivedTable; +import tech.easyflow.datacenter.meta.entity.DatacenterSource; + +import java.util.ArrayList; +import java.util.List; + +public class DatacenterSchemaResponse { + private DatasetRef datasetRef; + private DatacenterSource source; + private DatacenterCatalog catalog; + private DatacenterTable table; + private List fields = new ArrayList<>(); + private List versions = new ArrayList<>(); + private List upstreamLineage = new ArrayList<>(); + private List downstreamLineage = new ArrayList<>(); + + public DatasetRef getDatasetRef() { return datasetRef; } + public void setDatasetRef(DatasetRef datasetRef) { this.datasetRef = datasetRef; } + public DatacenterSource getSource() { return source; } + public void setSource(DatacenterSource source) { this.source = source; } + public DatacenterCatalog getCatalog() { return catalog; } + public void setCatalog(DatacenterCatalog catalog) { this.catalog = catalog; } + public DatacenterTable getTable() { return table; } + public void setTable(DatacenterTable table) { this.table = table; } + public List getFields() { return fields; } + public void setFields(List fields) { this.fields = fields; } + public List getVersions() { return versions; } + public void setVersions(List versions) { this.versions = versions; } + public List getUpstreamLineage() { return upstreamLineage; } + public void setUpstreamLineage(List upstreamLineage) { this.upstreamLineage = upstreamLineage; } + public List getDownstreamLineage() { return downstreamLineage; } + public void setDownstreamLineage(List downstreamLineage) { this.downstreamLineage = downstreamLineage; } +} diff --git a/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/execution/model/DatacenterSqlQueryRequest.java b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/execution/model/DatacenterSqlQueryRequest.java new file mode 100644 index 0000000..e1ccf69 --- /dev/null +++ b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/execution/model/DatacenterSqlQueryRequest.java @@ -0,0 +1,23 @@ +package tech.easyflow.datacenter.execution.model; + +public class DatacenterSqlQueryRequest { + + private DatasetRef datasetRef; + private String sql; + + public DatasetRef getDatasetRef() { + return datasetRef; + } + + public void setDatasetRef(DatasetRef datasetRef) { + this.datasetRef = datasetRef; + } + + public String getSql() { + return sql; + } + + public void setSql(String sql) { + this.sql = sql; + } +} diff --git a/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/execution/model/DatasetRef.java b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/execution/model/DatasetRef.java new file mode 100644 index 0000000..8a36210 --- /dev/null +++ b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/execution/model/DatasetRef.java @@ -0,0 +1,25 @@ +package tech.easyflow.datacenter.execution.model; + +import java.math.BigInteger; + +public class DatasetRef { + private BigInteger sourceId; + private BigInteger catalogId; + private String catalogName; + private BigInteger tableId; + private String tableName; + private BigInteger versionId; + + public BigInteger getSourceId() { return sourceId; } + public void setSourceId(BigInteger sourceId) { this.sourceId = sourceId; } + public BigInteger getCatalogId() { return catalogId; } + public void setCatalogId(BigInteger catalogId) { this.catalogId = catalogId; } + public String getCatalogName() { return catalogName; } + public void setCatalogName(String catalogName) { this.catalogName = catalogName; } + public BigInteger getTableId() { return tableId; } + public void setTableId(BigInteger tableId) { this.tableId = tableId; } + public String getTableName() { return tableName; } + public void setTableName(String tableName) { this.tableName = tableName; } + public BigInteger getVersionId() { return versionId; } + public void setVersionId(BigInteger versionId) { this.versionId = versionId; } +} diff --git a/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/execution/service/DatacenterDatasetQueryService.java b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/execution/service/DatacenterDatasetQueryService.java new file mode 100644 index 0000000..62a7374 --- /dev/null +++ b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/execution/service/DatacenterDatasetQueryService.java @@ -0,0 +1,18 @@ +package tech.easyflow.datacenter.execution.service; + +import com.mybatisflex.core.paginate.Page; +import com.mybatisflex.core.row.Row; +import tech.easyflow.datacenter.execution.model.DatacenterQueryRequest; +import tech.easyflow.datacenter.execution.model.DatacenterSchemaResponse; +import tech.easyflow.datacenter.execution.model.DatacenterSqlQueryRequest; +import tech.easyflow.datacenter.execution.model.DatasetRef; + +import java.util.List; + +public interface DatacenterDatasetQueryService { + Page queryPage(DatacenterQueryRequest request); + + List queryBySql(DatacenterSqlQueryRequest request); + + DatacenterSchemaResponse getSchema(DatasetRef datasetRef); +} diff --git a/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/execution/service/DatacenterDatasetWriteService.java b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/execution/service/DatacenterDatasetWriteService.java new file mode 100644 index 0000000..7ad357f --- /dev/null +++ b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/execution/service/DatacenterDatasetWriteService.java @@ -0,0 +1,13 @@ +package tech.easyflow.datacenter.execution.service; + +import com.alibaba.fastjson2.JSONObject; +import tech.easyflow.common.entity.LoginAccount; +import tech.easyflow.datacenter.execution.model.DatasetRef; + +import java.math.BigInteger; + +public interface DatacenterDatasetWriteService { + void saveRow(DatasetRef datasetRef, JSONObject data, LoginAccount account); + + void deleteRow(DatasetRef datasetRef, BigInteger id, LoginAccount account); +} diff --git a/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/execution/service/impl/DatacenterDatasetQueryServiceImpl.java b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/execution/service/impl/DatacenterDatasetQueryServiceImpl.java new file mode 100644 index 0000000..f25bf3c --- /dev/null +++ b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/execution/service/impl/DatacenterDatasetQueryServiceImpl.java @@ -0,0 +1,311 @@ +package tech.easyflow.datacenter.execution.service.impl; + +import com.mybatisflex.core.paginate.Page; +import com.mybatisflex.core.query.QueryWrapper; +import com.mybatisflex.core.row.Row; +import org.springframework.stereotype.Service; +import org.springframework.util.CollectionUtils; +import org.springframework.util.StringUtils; +import tech.easyflow.common.web.exceptions.BusinessException; +import tech.easyflow.datacenter.connector.DatacenterConnector; +import tech.easyflow.datacenter.connector.DatacenterConnectorRegistry; +import tech.easyflow.datacenter.entity.DatacenterTable; +import tech.easyflow.datacenter.entity.DatacenterTableField; +import tech.easyflow.datacenter.execution.model.DatacenterQueryRequest; +import tech.easyflow.datacenter.execution.model.DatacenterSchemaResponse; +import tech.easyflow.datacenter.execution.model.DatacenterSqlQueryRequest; +import tech.easyflow.datacenter.execution.model.DatasetRef; +import tech.easyflow.datacenter.execution.service.DatacenterDatasetQueryService; +import tech.easyflow.datacenter.mapper.DatacenterCatalogMapper; +import tech.easyflow.datacenter.mapper.DatacenterDatasetVersionMapper; +import tech.easyflow.datacenter.mapper.DatacenterDerivedTableMapper; +import tech.easyflow.datacenter.mapper.DatacenterTableMapper; +import tech.easyflow.datacenter.meta.entity.DatacenterCatalog; +import tech.easyflow.datacenter.meta.entity.DatacenterDatasetVersion; +import tech.easyflow.datacenter.meta.entity.DatacenterDerivedTable; +import tech.easyflow.datacenter.meta.entity.DatacenterSource; +import tech.easyflow.datacenter.meta.service.DatacenterDatasetRegistryService; +import tech.easyflow.datacenter.utils.SqlSupportUtils; + +import javax.annotation.Resource; +import java.math.BigInteger; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +@Service +public class DatacenterDatasetQueryServiceImpl implements DatacenterDatasetQueryService { + + @Resource + private DatacenterDatasetRegistryService registryService; + @Resource + private DatacenterConnectorRegistry connectorRegistry; + @Resource + private DatacenterTableMapper tableMapper; + @Resource + private DatacenterCatalogMapper catalogMapper; + @Resource + private DatacenterDatasetVersionMapper datasetVersionMapper; + @Resource + private DatacenterDerivedTableMapper derivedTableMapper; + + @Override + public Page queryPage(DatacenterQueryRequest request) { + if (request == null || request.getDatasetRef() == null) { + throw new BusinessException("datasetRef 不能为空"); + } + normalizePage(request); + DatacenterTable table = resolveTable(request.getDatasetRef()); + DatacenterSource source = registryService.getSourceRequired(table.getSourceId()); + DatacenterCatalog catalog = registryService.getCatalogById(table.getCatalogId()); + DatacenterTable queryTable = resolveQueryTable(table, request.getDatasetRef()); + validateRequest(queryTable, request, source); + request.getDatasetRef().setSourceId(table.getSourceId()); + request.getDatasetRef().setCatalogId(table.getCatalogId()); + request.getDatasetRef().setTableId(table.getId()); + request.getDatasetRef().setTableName(table.getTableName()); + if (catalog != null) { + request.getDatasetRef().setCatalogName(catalog.getCatalogName()); + } + DatacenterConnector connector = connectorRegistry.getConnector(source.getSourceType()); + return connector.queryPage(source, queryTable, request); + } + + @Override + public List queryBySql(DatacenterSqlQueryRequest request) { + if (request == null || request.getDatasetRef() == null) { + throw new BusinessException("datasetRef 不能为空"); + } + String sql = trimToNull(request.getSql()); + if (!StringUtils.hasText(sql)) { + throw new BusinessException("SQL 不能为空"); + } + DatasetRef datasetRef = request.getDatasetRef(); + if (datasetRef.getSourceId() == null) { + throw new BusinessException("缺少连接服务配置"); + } + DatacenterSource source = registryService.getSourceRequired(datasetRef.getSourceId()); + BigInteger catalogId = resolveRequestedCatalogId(datasetRef); + List managedTables = registryService.listManagedTables(datasetRef.getSourceId(), catalogId); + if (CollectionUtils.isEmpty(managedTables)) { + throw new BusinessException("当前连接下没有已接入表"); + } + SqlSupportUtils.ResolvedSql resolvedSql = SqlSupportUtils.resolve( + sql, + managedTables.stream().map(this::toManagedSqlTable).toList() + ); + DatacenterConnector connector = connectorRegistry.getConnector(source.getSourceType()); + return connector.queryBySql(source, resolvedSql.getExecutableSql()); + } + + @Override + public DatacenterSchemaResponse getSchema(DatasetRef datasetRef) { + DatacenterTable table = resolveTable(datasetRef); + DatacenterSource source = registryService.getSourceRequired(table.getSourceId()); + DatacenterCatalog catalog = registryService.getCatalogById(table.getCatalogId()); + DatacenterSchemaResponse response = new DatacenterSchemaResponse(); + response.setDatasetRef(registryService.resolveDatasetRef(table.getId())); + response.setSource(source); + response.setCatalog(catalog); + response.setTable(table); + response.setFields(table.getFields()); + response.setVersions(listVersions(table.getId())); + response.setUpstreamLineage(listUpstream(table.getId())); + response.setDownstreamLineage(listDownstream(table.getId())); + return response; + } + + private DatacenterTable resolveTable(DatasetRef datasetRef) { + if (datasetRef.getTableId() != null) { + return registryService.getTableWithFields(datasetRef.getTableId()); + } + if (datasetRef.getSourceId() == null || !StringUtils.hasText(datasetRef.getTableName())) { + throw new BusinessException("缺少数据集定位信息"); + } + QueryWrapper wrapper = QueryWrapper.create(); + wrapper.eq(DatacenterTable::getSourceId, datasetRef.getSourceId()); + wrapper.eq(DatacenterTable::getTableName, datasetRef.getTableName().trim()); + + boolean hasCatalogCondition = false; + if (datasetRef.getCatalogId() != null) { + wrapper.eq(DatacenterTable::getCatalogId, datasetRef.getCatalogId()); + hasCatalogCondition = true; + } else if (StringUtils.hasText(datasetRef.getCatalogName())) { + DatacenterCatalog catalog = resolveCatalog(datasetRef.getSourceId(), datasetRef.getCatalogName().trim()); + wrapper.eq(DatacenterTable::getCatalogId, catalog.getId()); + hasCatalogCondition = true; + } + + List tables = tableMapper.selectListByQuery(wrapper); + if (CollectionUtils.isEmpty(tables)) { + throw new BusinessException("数据集不存在: " + datasetRef.getTableName()); + } + if (!hasCatalogCondition && tables.size() > 1) { + throw new BusinessException("数据集存在重名表，请指定库名: " + datasetRef.getTableName()); + } + return registryService.getTableWithFields(tables.get(0).getId()); + } + + private DatacenterCatalog resolveCatalog(java.math.BigInteger sourceId, String catalogName) { + QueryWrapper wrapper = QueryWrapper.create(); + wrapper.eq(DatacenterCatalog::getSourceId, sourceId); + wrapper.eq(DatacenterCatalog::getCatalogName, catalogName); + List catalogs = catalogMapper.selectListByQuery(wrapper); + if (CollectionUtils.isEmpty(catalogs)) { + throw new BusinessException("库不存在: " + catalogName); + } + if (catalogs.size() > 1) { + throw new BusinessException("库存在重复配置，请检查: " + catalogName); + } + return catalogs.get(0); + } + + private BigInteger resolveRequestedCatalogId(DatasetRef datasetRef) { + if (datasetRef == null) { + return null; + } + if (datasetRef.getCatalogId() != null) { + return datasetRef.getCatalogId(); + } + if (StringUtils.hasText(datasetRef.getCatalogName())) { + DatacenterCatalog catalog = resolveCatalog(datasetRef.getSourceId(), datasetRef.getCatalogName().trim()); + return catalog.getId(); + } + return null; + } + + private SqlSupportUtils.ManagedTable toManagedSqlTable(DatacenterTable table) { + DatacenterCatalog catalog = registryService.getCatalogById(table.getCatalogId()); + return new SqlSupportUtils.ManagedTable( + catalog == null ? null : catalog.getCatalogName(), + table.getTableName(), + resolvePhysicalTableName(table) + ); + } + + private String resolvePhysicalTableName(DatacenterTable table) { + if (table == null) { + return null; + } + if ("EXTERNAL_TABLE".equals(table.getTableKind()) || "EXTERNAL_VIEW".equals(table.getTableKind())) { + return trimToNull(table.getActualTable()) == null ? table.getTableName() : table.getActualTable().trim(); + } + String materializedTable = trimToNull(table.getMaterializedTable()); + if (materializedTable != null) { + return materializedTable; + } + String actualTable = trimToNull(table.getActualTable()); + return actualTable != null ? actualTable : table.getTableName(); + } + + private void normalizePage(DatacenterQueryRequest request) { + if (request.getPageNumber() == null || request.getPageNumber() < 1L) { + request.setPageNumber(1L); + } + if (request.getPageSize() == null || request.getPageSize() < 1L) { + throw new BusinessException("pageSize 必须大于 0"); + } + } + + private DatacenterTable resolveQueryTable(DatacenterTable table, DatasetRef datasetRef) { + if (datasetRef == null || datasetRef.getVersionId() == null) { + return table; + } + DatacenterDatasetVersion version = datasetVersionMapper.selectOneById(datasetRef.getVersionId()); + if (version == null || !table.getId().equals(version.getTableId())) { + throw new BusinessException("数据集版本不存在: " + datasetRef.getVersionId()); + } + DatacenterTable queryTable = new DatacenterTable(); + queryTable.setId(table.getId()); + queryTable.setSourceId(table.getSourceId()); + queryTable.setCatalogId(table.getCatalogId()); + queryTable.setTableName(table.getTableName()); + queryTable.setTableDesc(table.getTableDesc()); + queryTable.setActualTable(table.getActualTable()); + queryTable.setMaterializedTable(version.getMaterializedTable()); + queryTable.setAccessMode(table.getAccessMode()); + queryTable.setTableKind(table.getTableKind()); + queryTable.setVersioningEnabled(table.getVersioningEnabled()); + queryTable.setCapabilitiesJson(table.getCapabilitiesJson()); + queryTable.setFields(table.getFields()); + return queryTable; + } + + private void validateRequest(DatacenterTable table, DatacenterQueryRequest request, DatacenterSource source) { + Map fieldMap = new LinkedHashMap<>(); + for (DatacenterTableField field : table.getFields()) { + fieldMap.put(field.getFieldName(), field); + } + if (!CollectionUtils.isEmpty(request.getSelectedColumns())) { + for (String column : request.getSelectedColumns()) { + DatacenterTableField field = fieldMap.get(column); + if (field == null || !isEnabled(field.getQueryable())) { + throw new BusinessException("字段不可查询: " + column); + } + } + } else { + request.setSelectedColumns( + table.getFields().stream() + .filter(field -> isEnabled(field.getQueryable())) + .map(DatacenterTableField::getFieldName) + .toList() + ); + } + if (!CollectionUtils.isEmpty(request.getFilters())) { + request.getFilters().forEach(filter -> { + DatacenterTableField field = fieldMap.get(filter.getColumn()); + if (field == null || !isEnabled(field.getQueryable())) { + throw new BusinessException("字段不可过滤: " + filter.getColumn()); + } + }); + } + if (!CollectionUtils.isEmpty(request.getSorts())) { + request.getSorts().forEach(sort -> { + DatacenterTableField field = fieldMap.get(sort.getColumn()); + if (field == null || !isEnabled(field.getSortable())) { + throw new BusinessException("字段不可排序: " + sort.getColumn()); + } + }); + } + if (request.getWhere() != null && !request.getWhere().isBlank()) { + boolean allowLegacyWhere = "PROJECT_MYSQL".equals(source.getSourceType()) + || "MYSQL".equals(source.getSourceType()) + || "POSTGRESQL".equals(source.getSourceType()); + if (!allowLegacyWhere) { + throw new BusinessException("当前数据源仅支持结构化 DSL 查询"); + } + } + } + + private boolean isEnabled(Integer value) { + return value == null || value == 1; + } + + private String trimToNull(String value) { + if (!StringUtils.hasText(value)) { + return null; + } + return value.trim(); + } + + private List listVersions(java.math.BigInteger tableId) { + QueryWrapper wrapper = QueryWrapper.create(); + wrapper.eq(DatacenterDatasetVersion::getTableId, tableId); + wrapper.orderBy("version_no desc"); + return datasetVersionMapper.selectListByQuery(wrapper); + } + + private List listUpstream(java.math.BigInteger tableId) { + QueryWrapper wrapper = QueryWrapper.create(); + wrapper.eq(DatacenterDerivedTable::getDerivedTableId, tableId); + wrapper.orderBy("created desc"); + return derivedTableMapper.selectListByQuery(wrapper); + } + + private List listDownstream(java.math.BigInteger tableId) { + QueryWrapper wrapper = QueryWrapper.create(); + wrapper.eq(DatacenterDerivedTable::getSourceTableId, tableId); + wrapper.orderBy("created desc"); + return derivedTableMapper.selectListByQuery(wrapper); + } +} diff --git a/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/execution/service/impl/DatacenterDatasetWriteServiceImpl.java b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/execution/service/impl/DatacenterDatasetWriteServiceImpl.java new file mode 100644 index 0000000..46d403f --- /dev/null +++ b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/execution/service/impl/DatacenterDatasetWriteServiceImpl.java @@ -0,0 +1,48 @@ +package tech.easyflow.datacenter.execution.service.impl; + +import com.alibaba.fastjson2.JSONObject; +import org.springframework.stereotype.Service; +import tech.easyflow.common.entity.LoginAccount; +import tech.easyflow.common.web.exceptions.BusinessException; +import tech.easyflow.datacenter.connector.DatacenterConnector; +import tech.easyflow.datacenter.connector.DatacenterConnectorRegistry; +import tech.easyflow.datacenter.entity.DatacenterTable; +import tech.easyflow.datacenter.execution.model.DatasetRef; +import tech.easyflow.datacenter.meta.entity.DatacenterSource; +import tech.easyflow.datacenter.meta.service.DatacenterDatasetRegistryService; +import tech.easyflow.datacenter.execution.service.DatacenterDatasetWriteService; + +import javax.annotation.Resource; +import java.math.BigInteger; + +@Service +public class DatacenterDatasetWriteServiceImpl implements DatacenterDatasetWriteService { + + @Resource + private DatacenterDatasetRegistryService registryService; + @Resource + private DatacenterConnectorRegistry connectorRegistry; + + @Override + public void saveRow(DatasetRef datasetRef, JSONObject data, LoginAccount account) { + DatacenterTable table = resolveTable(datasetRef); + DatacenterSource source = registryService.getSourceRequired(table.getSourceId()); + DatacenterConnector connector = connectorRegistry.getConnector(source.getSourceType()); + connector.saveRow(source, table, data, account); + } + + @Override + public void deleteRow(DatasetRef datasetRef, BigInteger id, LoginAccount account) { + DatacenterTable table = resolveTable(datasetRef); + DatacenterSource source = registryService.getSourceRequired(table.getSourceId()); + DatacenterConnector connector = connectorRegistry.getConnector(source.getSourceType()); + connector.deleteRow(source, table, id, account); + } + + private DatacenterTable resolveTable(DatasetRef datasetRef) { + if (datasetRef == null || datasetRef.getTableId() == null) { + throw new BusinessException("缺少 tableId"); + } + return registryService.getTableWithFields(datasetRef.getTableId()); + } +} diff --git a/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/integration/AssistantDatacenterBridge.java b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/integration/AssistantDatacenterBridge.java new file mode 100644 index 0000000..a006f57 --- /dev/null +++ b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/integration/AssistantDatacenterBridge.java @@ -0,0 +1,7 @@ +package tech.easyflow.datacenter.integration; + +import tech.easyflow.datacenter.execution.model.DatacenterQueryRequest; + +public interface AssistantDatacenterBridge { + AssistantDatacenterResult queryPage(DatacenterQueryRequest request); +} diff --git a/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/integration/AssistantDatacenterResult.java b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/integration/AssistantDatacenterResult.java new file mode 100644 index 0000000..e3fe9d3 --- /dev/null +++ b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/integration/AssistantDatacenterResult.java @@ -0,0 +1,34 @@ +package tech.easyflow.datacenter.integration; + +import com.mybatisflex.core.row.Row; +import tech.easyflow.datacenter.entity.DatacenterTable; +import tech.easyflow.datacenter.meta.entity.DatacenterCatalog; +import tech.easyflow.datacenter.meta.entity.DatacenterDatasetVersion; +import tech.easyflow.datacenter.meta.entity.DatacenterSource; + +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +public class AssistantDatacenterResult { + private List rows = new ArrayList<>(); + private DatacenterSource source; + private DatacenterCatalog catalog; + private DatacenterTable table; + private DatacenterDatasetVersion version; + private Map querySummary = new LinkedHashMap<>(); + + public List getRows() { return rows; } + public void setRows(List rows) { this.rows = rows; } + public DatacenterSource getSource() { return source; } + public void setSource(DatacenterSource source) { this.source = source; } + public DatacenterCatalog getCatalog() { return catalog; } + public void setCatalog(DatacenterCatalog catalog) { this.catalog = catalog; } + public DatacenterTable getTable() { return table; } + public void setTable(DatacenterTable table) { this.table = table; } + public DatacenterDatasetVersion getVersion() { return version; } + public void setVersion(DatacenterDatasetVersion version) { this.version = version; } + public Map getQuerySummary() { return querySummary; } + public void setQuerySummary(Map querySummary) { this.querySummary = querySummary; } +} diff --git a/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/integration/DefaultAssistantDatacenterBridge.java b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/integration/DefaultAssistantDatacenterBridge.java new file mode 100644 index 0000000..b78e149 --- /dev/null +++ b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/integration/DefaultAssistantDatacenterBridge.java @@ -0,0 +1,49 @@ +package tech.easyflow.datacenter.integration; + +import org.springframework.stereotype.Component; +import tech.easyflow.datacenter.execution.model.DatacenterQueryRequest; +import tech.easyflow.datacenter.execution.model.DatacenterSchemaResponse; +import tech.easyflow.datacenter.execution.service.DatacenterDatasetQueryService; +import tech.easyflow.datacenter.meta.entity.DatacenterDatasetVersion; + +import javax.annotation.Resource; +import java.util.LinkedHashMap; +import java.util.List; + +@Component +public class DefaultAssistantDatacenterBridge implements AssistantDatacenterBridge { + + @Resource + private DatacenterDatasetQueryService queryService; + + @Override + public AssistantDatacenterResult queryPage(DatacenterQueryRequest request) { + var page = queryService.queryPage(request); + DatacenterSchemaResponse schema = queryService.getSchema(request.getDatasetRef()); + AssistantDatacenterResult result = new AssistantDatacenterResult(); + result.setRows(page.getRecords()); + result.setSource(schema.getSource()); + result.setCatalog(schema.getCatalog()); + result.setTable(schema.getTable()); + result.setVersion(resolveVersion(schema.getVersions(), request == null || request.getDatasetRef() == null ? null : request.getDatasetRef().getVersionId())); + result.setQuerySummary(new LinkedHashMap<>() {{ + put("pageNumber", page.getPageNumber()); + put("pageSize", page.getPageSize()); + put("totalRows", page.getTotalRow()); + put("selectedColumns", request == null ? List.of() : request.getSelectedColumns()); + put("filterCount", request == null || request.getFilters() == null ? 0 : request.getFilters().size()); + put("sortCount", request == null || request.getSorts() == null ? 0 : request.getSorts().size()); + }}); + return result; + } + + private DatacenterDatasetVersion resolveVersion(List versions, java.math.BigInteger versionId) { + if (versions == null || versions.isEmpty()) { + return null; + } + if (versionId == null) { + return versions.get(0); + } + return versions.stream().filter(version -> versionId.equals(version.getId())).findFirst().orElse(null); + } +} diff --git a/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/mapper/DatacenterCatalogMapper.java b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/mapper/DatacenterCatalogMapper.java new file mode 100644 index 0000000..ad22c15 --- /dev/null +++ b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/mapper/DatacenterCatalogMapper.java @@ -0,0 +1,7 @@ +package tech.easyflow.datacenter.mapper; + +import com.mybatisflex.core.BaseMapper; +import tech.easyflow.datacenter.meta.entity.DatacenterCatalog; + +public interface DatacenterCatalogMapper extends BaseMapper { +} diff --git a/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/mapper/DatacenterDatasetVersionMapper.java b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/mapper/DatacenterDatasetVersionMapper.java new file mode 100644 index 0000000..f7dd813 --- /dev/null +++ b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/mapper/DatacenterDatasetVersionMapper.java @@ -0,0 +1,7 @@ +package tech.easyflow.datacenter.mapper; + +import com.mybatisflex.core.BaseMapper; +import tech.easyflow.datacenter.meta.entity.DatacenterDatasetVersion; + +public interface DatacenterDatasetVersionMapper extends BaseMapper { +} diff --git a/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/mapper/DatacenterDerivedTableMapper.java b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/mapper/DatacenterDerivedTableMapper.java new file mode 100644 index 0000000..f1024c6 --- /dev/null +++ b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/mapper/DatacenterDerivedTableMapper.java @@ -0,0 +1,7 @@ +package tech.easyflow.datacenter.mapper; + +import com.mybatisflex.core.BaseMapper; +import tech.easyflow.datacenter.meta.entity.DatacenterDerivedTable; + +public interface DatacenterDerivedTableMapper extends BaseMapper { +} diff --git a/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/mapper/DatacenterImportJobMapper.java b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/mapper/DatacenterImportJobMapper.java new file mode 100644 index 0000000..ffb20b5 --- /dev/null +++ b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/mapper/DatacenterImportJobMapper.java @@ -0,0 +1,7 @@ +package tech.easyflow.datacenter.mapper; + +import com.mybatisflex.core.BaseMapper; +import tech.easyflow.datacenter.meta.entity.DatacenterImportJob; + +public interface DatacenterImportJobMapper extends BaseMapper { +} diff --git a/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/mapper/DatacenterSourceMapper.java b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/mapper/DatacenterSourceMapper.java new file mode 100644 index 0000000..e464dad --- /dev/null +++ b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/mapper/DatacenterSourceMapper.java @@ -0,0 +1,7 @@ +package tech.easyflow.datacenter.mapper; + +import com.mybatisflex.core.BaseMapper; +import tech.easyflow.datacenter.meta.entity.DatacenterSource; + +public interface DatacenterSourceMapper extends BaseMapper { +} diff --git a/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/meta/entity/DatacenterCatalog.java b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/meta/entity/DatacenterCatalog.java new file mode 100644 index 0000000..c630135 --- /dev/null +++ b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/meta/entity/DatacenterCatalog.java @@ -0,0 +1,75 @@ +package tech.easyflow.datacenter.meta.entity; + +import com.mybatisflex.annotation.Column; +import com.mybatisflex.annotation.Id; +import com.mybatisflex.annotation.KeyType; +import com.mybatisflex.annotation.Table; +import com.mybatisflex.core.handler.FastjsonTypeHandler; +import tech.easyflow.common.entity.DateEntity; + +import java.io.Serializable; +import java.math.BigInteger; +import java.util.Date; +import java.util.Map; + +@Table(value = "tb_datacenter_catalog", comment = "数据中心逻辑库/命名空间") +public class DatacenterCatalog extends DateEntity implements Serializable { + + @Id(keyType = KeyType.Generator, value = "snowFlakeId", comment = "主键") + private BigInteger id; + @Column(comment = "部门ID") + private BigInteger deptId; + @Column(tenantId = true, comment = "租户ID") + private BigInteger tenantId; + @Column(comment = "数据源ID") + private BigInteger sourceId; + @Column(comment = "目录名") + private String catalogName; + @Column(comment = "目录描述") + private String catalogDesc; + @Column(comment = "目录类型") + private String catalogType; + @Column(comment = "状态") + private Integer status; + @Column(typeHandler = FastjsonTypeHandler.class, comment = "扩展项") + private Map options; + @Column(comment = "创建时间") + private Date created; + @Column(comment = "创建人") + private BigInteger createdBy; + @Column(comment = "修改时间") + private Date modified; + @Column(comment = "修改人") + private BigInteger modifiedBy; + + public BigInteger getId() { return id; } + public void setId(BigInteger id) { this.id = id; } + public BigInteger getDeptId() { return deptId; } + public void setDeptId(BigInteger deptId) { this.deptId = deptId; } + public BigInteger getTenantId() { return tenantId; } + public void setTenantId(BigInteger tenantId) { this.tenantId = tenantId; } + public BigInteger getSourceId() { return sourceId; } + public void setSourceId(BigInteger sourceId) { this.sourceId = sourceId; } + public String getCatalogName() { return catalogName; } + public void setCatalogName(String catalogName) { this.catalogName = catalogName; } + public String getCatalogDesc() { return catalogDesc; } + public void setCatalogDesc(String catalogDesc) { this.catalogDesc = catalogDesc; } + public String getCatalogType() { return catalogType; } + public void setCatalogType(String catalogType) { this.catalogType = catalogType; } + public Integer getStatus() { return status; } + public void setStatus(Integer status) { this.status = status; } + public Map getOptions() { return options; } + public void setOptions(Map options) { this.options = options; } + @Override + public Date getCreated() { return created; } + @Override + public void setCreated(Date created) { this.created = created; } + public BigInteger getCreatedBy() { return createdBy; } + public void setCreatedBy(BigInteger createdBy) { this.createdBy = createdBy; } + @Override + public Date getModified() { return modified; } + @Override + public void setModified(Date modified) { this.modified = modified; } + public BigInteger getModifiedBy() { return modifiedBy; } + public void setModifiedBy(BigInteger modifiedBy) { this.modifiedBy = modifiedBy; } +} diff --git a/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/meta/entity/DatacenterDatasetVersion.java b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/meta/entity/DatacenterDatasetVersion.java new file mode 100644 index 0000000..72fabf9 --- /dev/null +++ b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/meta/entity/DatacenterDatasetVersion.java @@ -0,0 +1,70 @@ +package tech.easyflow.datacenter.meta.entity; + +import com.mybatisflex.annotation.Column; +import com.mybatisflex.annotation.Id; +import com.mybatisflex.annotation.KeyType; +import com.mybatisflex.annotation.Table; +import com.mybatisflex.core.handler.FastjsonTypeHandler; +import tech.easyflow.common.entity.DateEntity; + +import java.io.Serializable; +import java.math.BigInteger; +import java.util.Date; +import java.util.Map; + +@Table(value = "tb_datacenter_dataset_version", comment = "数据集版本") +public class DatacenterDatasetVersion extends DateEntity implements Serializable { + @Id(keyType = KeyType.Generator, value = "snowFlakeId") + private BigInteger id; + @Column(comment = "部门ID") + private BigInteger deptId; + @Column(tenantId = true, comment = "租户ID") + private BigInteger tenantId; + @Column(comment = "表ID") + private BigInteger tableId; + @Column(comment = "版本号") + private Integer versionNo; + @Column(comment = "版本标签") + private String versionLabel; + @Column(comment = "物化表名") + private String materializedTable; + @Column(typeHandler = FastjsonTypeHandler.class, comment = "版本快照") + private Map snapshotJson; + @Column(comment = "状态") + private Integer status; + @Column(comment = "创建时间") + private Date created; + @Column(comment = "创建人") + private BigInteger createdBy; + @Column(comment = "修改时间") + private Date modified; + @Column(comment = "修改人") + private BigInteger modifiedBy; + + public BigInteger getId() { return id; } + public void setId(BigInteger id) { this.id = id; } + public BigInteger getDeptId() { return deptId; } + public void setDeptId(BigInteger deptId) { this.deptId = deptId; } + public BigInteger getTenantId() { return tenantId; } + public void setTenantId(BigInteger tenantId) { this.tenantId = tenantId; } + public BigInteger getTableId() { return tableId; } + public void setTableId(BigInteger tableId) { this.tableId = tableId; } + public Integer getVersionNo() { return versionNo; } + public void setVersionNo(Integer versionNo) { this.versionNo = versionNo; } + public String getVersionLabel() { return versionLabel; } + public void setVersionLabel(String versionLabel) { this.versionLabel = versionLabel; } + public String getMaterializedTable() { return materializedTable; } + public void setMaterializedTable(String materializedTable) { this.materializedTable = materializedTable; } + public Map getSnapshotJson() { return snapshotJson; } + public void setSnapshotJson(Map snapshotJson) { this.snapshotJson = snapshotJson; } + public Integer getStatus() { return status; } + public void setStatus(Integer status) { this.status = status; } + @Override public Date getCreated() { return created; } + @Override public void setCreated(Date created) { this.created = created; } + public BigInteger getCreatedBy() { return createdBy; } + public void setCreatedBy(BigInteger createdBy) { this.createdBy = createdBy; } + @Override public Date getModified() { return modified; } + @Override public void setModified(Date modified) { this.modified = modified; } + public BigInteger getModifiedBy() { return modifiedBy; } + public void setModifiedBy(BigInteger modifiedBy) { this.modifiedBy = modifiedBy; } +} diff --git a/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/meta/entity/DatacenterDerivedTable.java b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/meta/entity/DatacenterDerivedTable.java new file mode 100644 index 0000000..324541c --- /dev/null +++ b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/meta/entity/DatacenterDerivedTable.java @@ -0,0 +1,66 @@ +package tech.easyflow.datacenter.meta.entity; + +import com.mybatisflex.annotation.Column; +import com.mybatisflex.annotation.Id; +import com.mybatisflex.annotation.KeyType; +import com.mybatisflex.annotation.Table; +import com.mybatisflex.core.handler.FastjsonTypeHandler; +import tech.easyflow.common.entity.DateEntity; + +import java.io.Serializable; +import java.math.BigInteger; +import java.util.Date; +import java.util.Map; + +@Table(value = "tb_datacenter_derived_table", comment = "数据中心派生表关系") +public class DatacenterDerivedTable extends DateEntity implements Serializable { + @Id(keyType = KeyType.Generator, value = "snowFlakeId") + private BigInteger id; + @Column(comment = "部门ID") + private BigInteger deptId; + @Column(tenantId = true, comment = "租户ID") + private BigInteger tenantId; + @Column(comment = "源表ID") + private BigInteger sourceTableId; + @Column(comment = "派生表ID") + private BigInteger derivedTableId; + @Column(comment = "派生类型") + private String deriveType; + @Column(typeHandler = FastjsonTypeHandler.class, comment = "派生配置") + private Map deriveConfigJson; + @Column(comment = "状态") + private Integer status; + @Column(comment = "创建时间") + private Date created; + @Column(comment = "创建人") + private BigInteger createdBy; + @Column(comment = "修改时间") + private Date modified; + @Column(comment = "修改人") + private BigInteger modifiedBy; + + public BigInteger getId() { return id; } + public void setId(BigInteger id) { this.id = id; } + public BigInteger getDeptId() { return deptId; } + public void setDeptId(BigInteger deptId) { this.deptId = deptId; } + public BigInteger getTenantId() { return tenantId; } + public void setTenantId(BigInteger tenantId) { this.tenantId = tenantId; } + public BigInteger getSourceTableId() { return sourceTableId; } + public void setSourceTableId(BigInteger sourceTableId) { this.sourceTableId = sourceTableId; } + public BigInteger getDerivedTableId() { return derivedTableId; } + public void setDerivedTableId(BigInteger derivedTableId) { this.derivedTableId = derivedTableId; } + public String getDeriveType() { return deriveType; } + public void setDeriveType(String deriveType) { this.deriveType = deriveType; } + public Map getDeriveConfigJson() { return deriveConfigJson; } + public void setDeriveConfigJson(Map deriveConfigJson) { this.deriveConfigJson = deriveConfigJson; } + public Integer getStatus() { return status; } + public void setStatus(Integer status) { this.status = status; } + @Override public Date getCreated() { return created; } + @Override public void setCreated(Date created) { this.created = created; } + public BigInteger getCreatedBy() { return createdBy; } + public void setCreatedBy(BigInteger createdBy) { this.createdBy = createdBy; } + @Override public Date getModified() { return modified; } + @Override public void setModified(Date modified) { this.modified = modified; } + public BigInteger getModifiedBy() { return modifiedBy; } + public void setModifiedBy(BigInteger modifiedBy) { this.modifiedBy = modifiedBy; } +} diff --git a/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/meta/entity/DatacenterImportJob.java b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/meta/entity/DatacenterImportJob.java new file mode 100644 index 0000000..9669663 --- /dev/null +++ b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/meta/entity/DatacenterImportJob.java @@ -0,0 +1,102 @@ +package tech.easyflow.datacenter.meta.entity; + +import com.mybatisflex.annotation.Column; +import com.mybatisflex.annotation.Id; +import com.mybatisflex.annotation.KeyType; +import com.mybatisflex.annotation.Table; +import com.mybatisflex.core.handler.FastjsonTypeHandler; +import tech.easyflow.common.entity.DateEntity; + +import java.io.Serializable; +import java.math.BigInteger; +import java.util.Date; +import java.util.Map; + +@Table(value = "tb_datacenter_import_job", comment = "数据中心导入任务") +public class DatacenterImportJob extends DateEntity implements Serializable { + @Id(keyType = KeyType.Generator, value = "snowFlakeId") + private BigInteger id; + @Column(comment = "部门ID") + private BigInteger deptId; + @Column(tenantId = true, comment = "租户ID") + private BigInteger tenantId; + @Column(comment = "数据源ID") + private BigInteger sourceId; + @Column(comment = "目录ID") + private BigInteger catalogId; + @Column(comment = "表ID") + private BigInteger tableId; + @Column(comment = "任务类型") + private String jobType; + @Column(comment = "文件名") + private String fileName; + @Column(comment = "文件存储路径") + private String storagePath; + @Column(comment = "任务状态") + private String status; + @Column(comment = "总行数") + private Long totalRows; + @Column(comment = "成功行数") + private Long successRows; + @Column(comment = "失败行数") + private Long errorRows; + @Column(comment = "错误摘要") + private String errorSummary; + @Column(typeHandler = FastjsonTypeHandler.class, comment = "任务载荷") + private Map payloadJson; + @Column(comment = "开始时间") + private Date startedAt; + @Column(comment = "结束时间") + private Date finishedAt; + @Column(comment = "创建时间") + private Date created; + @Column(comment = "创建人") + private BigInteger createdBy; + @Column(comment = "修改时间") + private Date modified; + @Column(comment = "修改人") + private BigInteger modifiedBy; + + public BigInteger getId() { return id; } + public void setId(BigInteger id) { this.id = id; } + public BigInteger getDeptId() { return deptId; } + public void setDeptId(BigInteger deptId) { this.deptId = deptId; } + public BigInteger getTenantId() { return tenantId; } + public void setTenantId(BigInteger tenantId) { this.tenantId = tenantId; } + public BigInteger getSourceId() { return sourceId; } + public void setSourceId(BigInteger sourceId) { this.sourceId = sourceId; } + public BigInteger getCatalogId() { return catalogId; } + public void setCatalogId(BigInteger catalogId) { this.catalogId = catalogId; } + public BigInteger getTableId() { return tableId; } + public void setTableId(BigInteger tableId) { this.tableId = tableId; } + public String getJobType() { return jobType; } + public void setJobType(String jobType) { this.jobType = jobType; } + public String getFileName() { return fileName; } + public void setFileName(String fileName) { this.fileName = fileName; } + public String getStoragePath() { return storagePath; } + public void setStoragePath(String storagePath) { this.storagePath = storagePath; } + public String getStatus() { return status; } + public void setStatus(String status) { this.status = status; } + public Long getTotalRows() { return totalRows; } + public void setTotalRows(Long totalRows) { this.totalRows = totalRows; } + public Long getSuccessRows() { return successRows; } + public void setSuccessRows(Long successRows) { this.successRows = successRows; } + public Long getErrorRows() { return errorRows; } + public void setErrorRows(Long errorRows) { this.errorRows = errorRows; } + public String getErrorSummary() { return errorSummary; } + public void setErrorSummary(String errorSummary) { this.errorSummary = errorSummary; } + public Map getPayloadJson() { return payloadJson; } + public void setPayloadJson(Map payloadJson) { this.payloadJson = payloadJson; } + public Date getStartedAt() { return startedAt; } + public void setStartedAt(Date startedAt) { this.startedAt = startedAt; } + public Date getFinishedAt() { return finishedAt; } + public void setFinishedAt(Date finishedAt) { this.finishedAt = finishedAt; } + @Override public Date getCreated() { return created; } + @Override public void setCreated(Date created) { this.created = created; } + public BigInteger getCreatedBy() { return createdBy; } + public void setCreatedBy(BigInteger createdBy) { this.createdBy = createdBy; } + @Override public Date getModified() { return modified; } + @Override public void setModified(Date modified) { this.modified = modified; } + public BigInteger getModifiedBy() { return modifiedBy; } + public void setModifiedBy(BigInteger modifiedBy) { this.modifiedBy = modifiedBy; } +} diff --git a/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/meta/entity/DatacenterSource.java b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/meta/entity/DatacenterSource.java new file mode 100644 index 0000000..4ae9782 --- /dev/null +++ b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/meta/entity/DatacenterSource.java @@ -0,0 +1,131 @@ +package tech.easyflow.datacenter.meta.entity; + +import com.mybatisflex.annotation.Column; +import com.mybatisflex.annotation.Id; +import com.mybatisflex.annotation.KeyType; +import com.mybatisflex.annotation.Table; +import com.mybatisflex.core.handler.FastjsonTypeHandler; +import tech.easyflow.common.entity.DateEntity; + +import java.io.Serializable; +import java.math.BigInteger; +import java.util.Date; +import java.util.Map; + +@Table(value = "tb_datacenter_source", comment = "数据中心数据源") +public class DatacenterSource extends DateEntity implements Serializable { + + @Id(keyType = KeyType.Generator, value = "snowFlakeId", comment = "主键") + private BigInteger id; + @Column(comment = "部门ID") + private BigInteger deptId; + @Column(tenantId = true, comment = "租户ID") + private BigInteger tenantId; + @Column(comment = "数据源名称") + private String sourceName; + @Column(comment = "数据源编码") + private String sourceCode; + @Column(comment = "数据源类型") + private String sourceType; + @Column(comment = "访问模式") + private String accessMode; + @Column(comment = "是否内置") + private Integer builtinFlag; + @Column(comment = "驱动类名") + private String driverClassName; + @Column(comment = "JDBC URL") + private String jdbcUrl; + @Column(comment = "主机") + private String host; + @Column(comment = "端口") + private Integer port; + @Column(comment = "数据库名") + private String databaseName; + @Column(comment = "Schema名") + private String schemaName; + @Column(comment = "用户名") + private String username; + @Column(comment = "凭据密文") + private String credentialCipher; + @Column(typeHandler = FastjsonTypeHandler.class, comment = "连接配置") + private Map configJson; + @Column(typeHandler = FastjsonTypeHandler.class, comment = "能力声明") + private Map capabilitiesJson; + @Column(comment = "最近测试状态") + private String lastTestStatus; + @Column(comment = "最近测试信息") + private String lastTestMessage; + @Column(comment = "最近测试时间") + private Date lastTestedAt; + @Column(comment = "状态") + private Integer status; + @Column(typeHandler = FastjsonTypeHandler.class, comment = "扩展项") + private Map options; + @Column(comment = "创建时间") + private Date created; + @Column(comment = "创建人") + private BigInteger createdBy; + @Column(comment = "修改时间") + private Date modified; + @Column(comment = "修改人") + private BigInteger modifiedBy; + + public BigInteger getId() { return id; } + public void setId(BigInteger id) { this.id = id; } + public BigInteger getDeptId() { return deptId; } + public void setDeptId(BigInteger deptId) { this.deptId = deptId; } + public BigInteger getTenantId() { return tenantId; } + public void setTenantId(BigInteger tenantId) { this.tenantId = tenantId; } + public String getSourceName() { return sourceName; } + public void setSourceName(String sourceName) { this.sourceName = sourceName; } + public String getSourceCode() { return sourceCode; } + public void setSourceCode(String sourceCode) { this.sourceCode = sourceCode; } + public String getSourceType() { return sourceType; } + public void setSourceType(String sourceType) { this.sourceType = sourceType; } + public String getAccessMode() { return accessMode; } + public void setAccessMode(String accessMode) { this.accessMode = accessMode; } + public Integer getBuiltinFlag() { return builtinFlag; } + public void setBuiltinFlag(Integer builtinFlag) { this.builtinFlag = builtinFlag; } + public String getDriverClassName() { return driverClassName; } + public void setDriverClassName(String driverClassName) { this.driverClassName = driverClassName; } + public String getJdbcUrl() { return jdbcUrl; } + public void setJdbcUrl(String jdbcUrl) { this.jdbcUrl = jdbcUrl; } + public String getHost() { return host; } + public void setHost(String host) { this.host = host; } + public Integer getPort() { return port; } + public void setPort(Integer port) { this.port = port; } + public String getDatabaseName() { return databaseName; } + public void setDatabaseName(String databaseName) { this.databaseName = databaseName; } + public String getSchemaName() { return schemaName; } + public void setSchemaName(String schemaName) { this.schemaName = schemaName; } + public String getUsername() { return username; } + public void setUsername(String username) { this.username = username; } + public String getCredentialCipher() { return credentialCipher; } + public void setCredentialCipher(String credentialCipher) { this.credentialCipher = credentialCipher; } + public Map getConfigJson() { return configJson; } + public void setConfigJson(Map configJson) { this.configJson = configJson; } + public Map getCapabilitiesJson() { return capabilitiesJson; } + public void setCapabilitiesJson(Map capabilitiesJson) { this.capabilitiesJson = capabilitiesJson; } + public String getLastTestStatus() { return lastTestStatus; } + public void setLastTestStatus(String lastTestStatus) { this.lastTestStatus = lastTestStatus; } + public String getLastTestMessage() { return lastTestMessage; } + public void setLastTestMessage(String lastTestMessage) { this.lastTestMessage = lastTestMessage; } + public Date getLastTestedAt() { return lastTestedAt; } + public void setLastTestedAt(Date lastTestedAt) { this.lastTestedAt = lastTestedAt; } + public Integer getStatus() { return status; } + public void setStatus(Integer status) { this.status = status; } + public Map getOptions() { return options; } + public void setOptions(Map options) { this.options = options; } + @Override + public Date getCreated() { return created; } + @Override + public void setCreated(Date created) { this.created = created; } + public BigInteger getCreatedBy() { return createdBy; } + public void setCreatedBy(BigInteger createdBy) { this.createdBy = createdBy; } + @Override + public Date getModified() { return modified; } + @Override + public void setModified(Date modified) { this.modified = modified; } + public BigInteger getModifiedBy() { return modifiedBy; } + public void setModifiedBy(BigInteger modifiedBy) { this.modifiedBy = modifiedBy; } +} diff --git a/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/meta/enums/DatacenterAccessMode.java b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/meta/enums/DatacenterAccessMode.java new file mode 100644 index 0000000..8717a14 --- /dev/null +++ b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/meta/enums/DatacenterAccessMode.java @@ -0,0 +1,6 @@ +package tech.easyflow.datacenter.meta.enums; + +public enum DatacenterAccessMode { + READ_ONLY, + READ_WRITE +} diff --git a/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/meta/enums/DatacenterCapability.java b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/meta/enums/DatacenterCapability.java new file mode 100644 index 0000000..b4a186e --- /dev/null +++ b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/meta/enums/DatacenterCapability.java @@ -0,0 +1,10 @@ +package tech.easyflow.datacenter.meta.enums; + +public enum DatacenterCapability { + TEST_CONNECTION, + BROWSE_METADATA, + READ_QUERY, + WRITE_MUTATION, + EXPORT, + MATERIALIZE +} diff --git a/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/meta/enums/DatacenterConnectionErrorCode.java b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/meta/enums/DatacenterConnectionErrorCode.java new file mode 100644 index 0000000..3ed9e60 --- /dev/null +++ b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/meta/enums/DatacenterConnectionErrorCode.java @@ -0,0 +1,12 @@ +package tech.easyflow.datacenter.meta.enums; + +public enum DatacenterConnectionErrorCode { + INVALID_ARGUMENT, + DRIVER_NOT_FOUND, + NETWORK_UNREACHABLE, + AUTH_FAILED, + DATABASE_NOT_FOUND, + SCHEMA_NOT_FOUND, + PERMISSION_DENIED, + UNKNOWN_ERROR +} diff --git a/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/meta/enums/DatacenterImportStatus.java b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/meta/enums/DatacenterImportStatus.java new file mode 100644 index 0000000..410b188 --- /dev/null +++ b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/meta/enums/DatacenterImportStatus.java @@ -0,0 +1,9 @@ +package tech.easyflow.datacenter.meta.enums; + +public enum DatacenterImportStatus { + PENDING, + RUNNING, + SUCCESS, + FAILED, + NOT_IMPLEMENTED +} diff --git a/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/meta/enums/DatacenterSourceType.java b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/meta/enums/DatacenterSourceType.java new file mode 100644 index 0000000..def856c --- /dev/null +++ b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/meta/enums/DatacenterSourceType.java @@ -0,0 +1,13 @@ +package tech.easyflow.datacenter.meta.enums; + +public enum DatacenterSourceType { + PROJECT_MYSQL, + EXCEL, + EXCEL_MATERIALIZED, + MYSQL, + POSTGRESQL, + ORACLE, + GAUSSDB_NATIVE, + GBASE_8A, + GBASE_8S +} diff --git a/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/meta/enums/DatacenterTableKind.java b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/meta/enums/DatacenterTableKind.java new file mode 100644 index 0000000..d21eecd --- /dev/null +++ b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/meta/enums/DatacenterTableKind.java @@ -0,0 +1,10 @@ +package tech.easyflow.datacenter.meta.enums; + +public enum DatacenterTableKind { + PROJECT_MANAGED, + EXTERNAL_TABLE, + EXTERNAL_VIEW, + EXCEL_SHEET, + EXCEL_MATERIALIZED, + DERIVED_TABLE +} diff --git a/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/meta/model/DatacenterBatchRegisterRequest.java b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/meta/model/DatacenterBatchRegisterRequest.java new file mode 100644 index 0000000..cafafda --- /dev/null +++ b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/meta/model/DatacenterBatchRegisterRequest.java @@ -0,0 +1,36 @@ +package tech.easyflow.datacenter.meta.model; + +import java.math.BigInteger; +import java.util.ArrayList; +import java.util.List; + +public class DatacenterBatchRegisterRequest { + + private BigInteger sourceId; + private String catalogName; + private List tableNames = new ArrayList<>(); + + public BigInteger getSourceId() { + return sourceId; + } + + public void setSourceId(BigInteger sourceId) { + this.sourceId = sourceId; + } + + public String getCatalogName() { + return catalogName; + } + + public void setCatalogName(String catalogName) { + this.catalogName = catalogName; + } + + public List getTableNames() { + return tableNames; + } + + public void setTableNames(List tableNames) { + this.tableNames = tableNames; + } +} diff --git a/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/meta/model/DatacenterBatchRemoveRequest.java b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/meta/model/DatacenterBatchRemoveRequest.java new file mode 100644 index 0000000..e076363 --- /dev/null +++ b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/meta/model/DatacenterBatchRemoveRequest.java @@ -0,0 +1,18 @@ +package tech.easyflow.datacenter.meta.model; + +import java.math.BigInteger; +import java.util.ArrayList; +import java.util.List; + +public class DatacenterBatchRemoveRequest { + + private List tableIds = new ArrayList<>(); + + public List getTableIds() { + return tableIds; + } + + public void setTableIds(List tableIds) { + this.tableIds = tableIds; + } +} diff --git a/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/meta/model/DatacenterCatalogMeta.java b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/meta/model/DatacenterCatalogMeta.java new file mode 100644 index 0000000..172887f --- /dev/null +++ b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/meta/model/DatacenterCatalogMeta.java @@ -0,0 +1,22 @@ +package tech.easyflow.datacenter.meta.model; + +import java.math.BigInteger; + +public class DatacenterCatalogMeta { + private BigInteger id; + private BigInteger sourceId; + private String catalogName; + private String catalogType; + private String catalogDesc; + + public BigInteger getId() { return id; } + public void setId(BigInteger id) { this.id = id; } + public BigInteger getSourceId() { return sourceId; } + public void setSourceId(BigInteger sourceId) { this.sourceId = sourceId; } + public String getCatalogName() { return catalogName; } + public void setCatalogName(String catalogName) { this.catalogName = catalogName; } + public String getCatalogType() { return catalogType; } + public void setCatalogType(String catalogType) { this.catalogType = catalogType; } + public String getCatalogDesc() { return catalogDesc; } + public void setCatalogDesc(String catalogDesc) { this.catalogDesc = catalogDesc; } +} diff --git a/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/meta/model/DatacenterFieldDescriptionUpdate.java b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/meta/model/DatacenterFieldDescriptionUpdate.java new file mode 100644 index 0000000..b4fb267 --- /dev/null +++ b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/meta/model/DatacenterFieldDescriptionUpdate.java @@ -0,0 +1,25 @@ +package tech.easyflow.datacenter.meta.model; + +import java.math.BigInteger; + +public class DatacenterFieldDescriptionUpdate { + + private BigInteger fieldId; + private String fieldDesc; + + public BigInteger getFieldId() { + return fieldId; + } + + public void setFieldId(BigInteger fieldId) { + this.fieldId = fieldId; + } + + public String getFieldDesc() { + return fieldDesc; + } + + public void setFieldDesc(String fieldDesc) { + this.fieldDesc = fieldDesc; + } +} diff --git a/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/meta/model/DatacenterRemoveSourceRequest.java b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/meta/model/DatacenterRemoveSourceRequest.java new file mode 100644 index 0000000..be242f8 --- /dev/null +++ b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/meta/model/DatacenterRemoveSourceRequest.java @@ -0,0 +1,16 @@ +package tech.easyflow.datacenter.meta.model; + +import java.math.BigInteger; + +public class DatacenterRemoveSourceRequest { + + private BigInteger sourceId; + + public BigInteger getSourceId() { + return sourceId; + } + + public void setSourceId(BigInteger sourceId) { + this.sourceId = sourceId; + } +} diff --git a/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/meta/model/DatacenterSaveDescriptionsRequest.java b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/meta/model/DatacenterSaveDescriptionsRequest.java new file mode 100644 index 0000000..b0968a7 --- /dev/null +++ b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/meta/model/DatacenterSaveDescriptionsRequest.java @@ -0,0 +1,36 @@ +package tech.easyflow.datacenter.meta.model; + +import java.math.BigInteger; +import java.util.ArrayList; +import java.util.List; + +public class DatacenterSaveDescriptionsRequest { + + private BigInteger tableId; + private String tableDesc; + private List fields = new ArrayList<>(); + + public BigInteger getTableId() { + return tableId; + } + + public void setTableId(BigInteger tableId) { + this.tableId = tableId; + } + + public String getTableDesc() { + return tableDesc; + } + + public void setTableDesc(String tableDesc) { + this.tableDesc = tableDesc; + } + + public List getFields() { + return fields; + } + + public void setFields(List fields) { + this.fields = fields; + } +} diff --git a/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/meta/model/DatacenterTableDetailMeta.java b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/meta/model/DatacenterTableDetailMeta.java new file mode 100644 index 0000000..572c59b --- /dev/null +++ b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/meta/model/DatacenterTableDetailMeta.java @@ -0,0 +1,17 @@ +package tech.easyflow.datacenter.meta.model; + +import tech.easyflow.datacenter.entity.DatacenterTable; +import tech.easyflow.datacenter.entity.DatacenterTableField; + +import java.util.ArrayList; +import java.util.List; + +public class DatacenterTableDetailMeta { + private DatacenterTable table; + private List fields = new ArrayList<>(); + + public DatacenterTable getTable() { return table; } + public void setTable(DatacenterTable table) { this.table = table; } + public List getFields() { return fields; } + public void setFields(List fields) { this.fields = fields; } +} diff --git a/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/meta/service/DatacenterDatasetRegistryService.java b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/meta/service/DatacenterDatasetRegistryService.java new file mode 100644 index 0000000..1d972e5 --- /dev/null +++ b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/meta/service/DatacenterDatasetRegistryService.java @@ -0,0 +1,38 @@ +package tech.easyflow.datacenter.meta.service; + +import tech.easyflow.common.entity.LoginAccount; +import tech.easyflow.datacenter.entity.DatacenterTable; +import tech.easyflow.datacenter.entity.DatacenterTableField; +import tech.easyflow.datacenter.execution.model.DatasetRef; +import tech.easyflow.datacenter.meta.entity.DatacenterCatalog; +import tech.easyflow.datacenter.meta.entity.DatacenterSource; +import tech.easyflow.datacenter.meta.model.DatacenterFieldDescriptionUpdate; +import tech.easyflow.datacenter.meta.enums.DatacenterSourceType; +import tech.easyflow.datacenter.meta.model.DatacenterTableDetailMeta; + +import java.math.BigInteger; +import java.util.List; + +public interface DatacenterDatasetRegistryService { + DatacenterSource ensureBuiltinSource(DatacenterSourceType sourceType, LoginAccount account); + + DatacenterCatalog ensureCatalog(DatacenterSource source, String catalogName, LoginAccount account); + + DatacenterTable registerTable(DatacenterSource source, DatacenterCatalog catalog, DatacenterTableDetailMeta detail, LoginAccount account); + + DatacenterTable getTableWithFields(BigInteger tableId); + + List getFields(BigInteger tableId); + + DatasetRef resolveDatasetRef(BigInteger tableId); + + DatacenterSource getSourceRequired(BigInteger sourceId); + + DatacenterCatalog getCatalogById(BigInteger catalogId); + + List listManagedTables(BigInteger sourceId, BigInteger catalogId); + + int removeTables(List tableIds); + + DatacenterTable saveDescriptions(BigInteger tableId, String tableDesc, List fields, LoginAccount account); +} diff --git a/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/meta/service/DatacenterMetaConstants.java b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/meta/service/DatacenterMetaConstants.java new file mode 100644 index 0000000..413a231 --- /dev/null +++ b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/meta/service/DatacenterMetaConstants.java @@ -0,0 +1,12 @@ +package tech.easyflow.datacenter.meta.service; + +import java.math.BigInteger; + +public final class DatacenterMetaConstants { + + private DatacenterMetaConstants() { + } + + public static final BigInteger PROJECT_SOURCE_BASE = new BigInteger("9000000000002000000"); + public static final BigInteger PROJECT_CATALOG_BASE = new BigInteger("9000000000003000000"); +} diff --git a/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/meta/service/DatacenterSourceService.java b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/meta/service/DatacenterSourceService.java new file mode 100644 index 0000000..c27bac5 --- /dev/null +++ b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/meta/service/DatacenterSourceService.java @@ -0,0 +1,32 @@ +package tech.easyflow.datacenter.meta.service; + +import com.mybatisflex.core.paginate.Page; +import com.mybatisflex.core.service.IService; +import tech.easyflow.common.entity.LoginAccount; +import tech.easyflow.datacenter.entity.DatacenterTable; +import tech.easyflow.datacenter.execution.model.DatacenterConnectionTestResult; +import tech.easyflow.datacenter.meta.entity.DatacenterSource; +import tech.easyflow.datacenter.meta.model.DatacenterBatchRegisterRequest; +import tech.easyflow.datacenter.meta.model.DatacenterCatalogMeta; +import tech.easyflow.datacenter.meta.model.DatacenterTableDetailMeta; + +import java.math.BigInteger; +import java.util.List; + +public interface DatacenterSourceService extends IService { + DatacenterSource saveSource(DatacenterSource source, LoginAccount account); + + Page pageSources(Long pageNumber, Long pageSize, LoginAccount account); + + DatacenterConnectionTestResult testConnection(DatacenterSource source, LoginAccount account); + + List listCatalogs(BigInteger sourceId, LoginAccount account); + + List listTables(BigInteger sourceId, String catalogName, LoginAccount account); + + DatacenterTableDetailMeta getTableDetail(BigInteger sourceId, String catalogName, String tableName, boolean register, LoginAccount account); + + List batchRegisterTables(DatacenterBatchRegisterRequest request, LoginAccount account); + + void removeSource(BigInteger sourceId, LoginAccount account); +} diff --git a/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/meta/service/impl/DatacenterDatasetRegistryServiceImpl.java b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/meta/service/impl/DatacenterDatasetRegistryServiceImpl.java new file mode 100644 index 0000000..c5cfd9f --- /dev/null +++ b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/meta/service/impl/DatacenterDatasetRegistryServiceImpl.java @@ -0,0 +1,405 @@ +package tech.easyflow.datacenter.meta.service.impl; + +import com.mybatisflex.core.query.QueryWrapper; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; +import tech.easyflow.common.entity.LoginAccount; +import tech.easyflow.common.web.exceptions.BusinessException; +import tech.easyflow.datacenter.adapter.DbHandleManager; +import tech.easyflow.datacenter.entity.DatacenterTable; +import tech.easyflow.datacenter.entity.DatacenterTableField; +import tech.easyflow.datacenter.execution.model.DatasetRef; +import tech.easyflow.datacenter.mapper.DatacenterCatalogMapper; +import tech.easyflow.datacenter.mapper.DatacenterDatasetVersionMapper; +import tech.easyflow.datacenter.mapper.DatacenterDerivedTableMapper; +import tech.easyflow.datacenter.mapper.DatacenterImportJobMapper; +import tech.easyflow.datacenter.mapper.DatacenterSourceMapper; +import tech.easyflow.datacenter.mapper.DatacenterTableFieldMapper; +import tech.easyflow.datacenter.mapper.DatacenterTableMapper; +import tech.easyflow.datacenter.meta.entity.DatacenterCatalog; +import tech.easyflow.datacenter.meta.entity.DatacenterSource; +import tech.easyflow.datacenter.meta.model.DatacenterFieldDescriptionUpdate; +import tech.easyflow.datacenter.meta.enums.DatacenterSourceType; +import tech.easyflow.datacenter.meta.service.DatacenterDatasetRegistryService; +import tech.easyflow.datacenter.meta.service.DatacenterMetaConstants; +import tech.easyflow.datacenter.meta.model.DatacenterTableDetailMeta; +import tech.easyflow.datacenter.meta.enums.DatacenterTableKind; + +import javax.annotation.Resource; +import java.math.BigInteger; +import java.util.Date; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +@Service +public class DatacenterDatasetRegistryServiceImpl implements DatacenterDatasetRegistryService { + + @Resource + private DatacenterSourceMapper sourceMapper; + @Resource + private DatacenterCatalogMapper catalogMapper; + @Resource + private DatacenterTableMapper tableMapper; + @Resource + private DatacenterTableFieldMapper tableFieldMapper; + @Resource + private DatacenterDatasetVersionMapper datasetVersionMapper; + @Resource + private DatacenterDerivedTableMapper derivedTableMapper; + @Resource + private DatacenterImportJobMapper importJobMapper; + @Resource + private DbHandleManager dbHandleManager; + + @Override + public DatacenterSource ensureBuiltinSource(DatacenterSourceType sourceType, LoginAccount account) { + BigInteger tenantId = account == null || account.getTenantId() == null ? BigInteger.ZERO : account.getTenantId(); + BigInteger deptId = account == null || account.getDeptId() == null ? BigInteger.ZERO : account.getDeptId(); + BigInteger id = builtinSourceId(sourceType, tenantId); + DatacenterSource source = sourceMapper.selectOneById(id); + if (source != null) { + return source; + } + source = new DatacenterSource(); + source.setId(id); + source.setTenantId(tenantId); + source.setDeptId(deptId); + source.setSourceType(sourceType.name()); + source.setBuiltinFlag(1); + source.setStatus(0); + source.setCreated(new Date()); + source.setModified(new Date()); + source.setCreatedBy(account == null ? BigInteger.ZERO : account.getId()); + source.setModifiedBy(account == null ? BigInteger.ZERO : account.getId()); + source.setCapabilitiesJson(Map.of("capabilities", builtinCapabilities(sourceType))); + if (sourceType == DatacenterSourceType.PROJECT_MYSQL) { + source.setSourceName("项目 MySQL"); + source.setSourceCode("PROJECT_MYSQL_" + tenantId); + source.setAccessMode("READ_WRITE"); + } else { + throw new BusinessException("不支持的内置数据源类型: " + sourceType); + } + sourceMapper.insert(source); + ensureCatalog(source, defaultCatalogName(sourceType), account); + return source; + } + + @Override + public DatacenterCatalog ensureCatalog(DatacenterSource source, String catalogName, LoginAccount account) { + QueryWrapper wrapper = QueryWrapper.create(); + wrapper.eq(DatacenterCatalog::getSourceId, source.getId()); + wrapper.eq(DatacenterCatalog::getCatalogName, catalogName); + DatacenterCatalog catalog = catalogMapper.selectOneByQuery(wrapper); + if (catalog != null) { + return catalog; + } + catalog = new DatacenterCatalog(); + BigInteger catalogId = builtinCatalogId(source, catalogName); + if (catalogId != null) { + catalog.setId(catalogId); + } + catalog.setSourceId(source.getId()); + catalog.setTenantId(source.getTenantId()); + catalog.setDeptId(source.getDeptId()); + catalog.setCatalogName(catalogName); + catalog.setCatalogDesc(catalogName); + catalog.setCatalogType("DATABASE"); + catalog.setStatus(0); + catalog.setCreated(new Date()); + catalog.setModified(new Date()); + catalog.setCreatedBy(account == null ? BigInteger.ZERO : account.getId()); + catalog.setModifiedBy(account == null ? BigInteger.ZERO : account.getId()); + catalogMapper.insert(catalog); + return catalog; + } + + @Override + public DatacenterTable registerTable(DatacenterSource source, DatacenterCatalog catalog, DatacenterTableDetailMeta detail, LoginAccount account) { + DatacenterTable table = detail.getTable(); + applyTableDefaults(table, source, detail); + QueryWrapper wrapper = QueryWrapper.create(); + wrapper.eq(DatacenterTable::getSourceId, source.getId()); + wrapper.eq(DatacenterTable::getCatalogId, catalog.getId()); + wrapper.eq(DatacenterTable::getTableName, table.getTableName()); + DatacenterTable existing = tableMapper.selectOneByQuery(wrapper); + Date now = new Date(); + if (existing == null) { + table.setSourceId(source.getId()); + table.setCatalogId(catalog.getId()); + table.setTenantId(source.getTenantId()); + table.setDeptId(source.getDeptId()); + table.setStatus(0); + table.setCreated(now); + table.setModified(now); + table.setCreatedBy(account == null ? BigInteger.ZERO : account.getId()); + table.setModifiedBy(account == null ? BigInteger.ZERO : account.getId()); + tableMapper.insert(table); + } else { + if (!hasText(existing.getTableDesc())) { + existing.setTableDesc(normalizeDescription(table.getTableDesc())); + } + existing.setActualTable(table.getActualTable()); + existing.setMaterializedTable(table.getMaterializedTable()); + existing.setTableKind(table.getTableKind()); + existing.setAccessMode(table.getAccessMode()); + existing.setVersioningEnabled(table.getVersioningEnabled()); + existing.setCapabilitiesJson(table.getCapabilitiesJson()); + existing.setModified(now); + existing.setModifiedBy(account == null ? BigInteger.ZERO : account.getId()); + tableMapper.update(existing); + table = existing; + } + Map existingFieldMap = getFields(table.getId()).stream() + .collect(LinkedHashMap::new, (map, field) -> map.put(field.getFieldName(), field), Map::putAll); + QueryWrapper deleteWrapper = QueryWrapper.create(); + deleteWrapper.eq(DatacenterTableField::getTableId, table.getId()); + tableFieldMapper.deleteByQuery(deleteWrapper); + for (DatacenterTableField field : detail.getFields()) { + DatacenterTableField existingField = existingFieldMap.get(field.getFieldName()); + if (existingField != null && hasText(existingField.getFieldDesc())) { + field.setFieldDesc(existingField.getFieldDesc()); + } else { + field.setFieldDesc(normalizeDescription(field.getFieldDesc())); + } + field.setId(null); + field.setTableId(table.getId()); + field.setCreated(now); + field.setModified(now); + field.setCreatedBy(account == null ? BigInteger.ZERO : account.getId()); + field.setModifiedBy(account == null ? BigInteger.ZERO : account.getId()); + tableFieldMapper.insert(field); + } + table.setFields(getFields(table.getId())); + return table; + } + + @Override + @Transactional(rollbackFor = Exception.class) + public DatacenterTable saveDescriptions(BigInteger tableId, String tableDesc, List fields, LoginAccount account) { + if (tableId == null) { + throw new BusinessException("缺少表 ID"); + } + DatacenterTable table = getTableWithFields(tableId); + Date now = new Date(); + table.setTableDesc(normalizeDescription(tableDesc)); + table.setModified(now); + table.setModifiedBy(account == null ? BigInteger.ZERO : account.getId()); + tableMapper.update(table); + + Map fieldUpdateMap = new LinkedHashMap<>(); + if (fields != null) { + for (DatacenterFieldDescriptionUpdate field : fields) { + if (field == null || field.getFieldId() == null) { + continue; + } + fieldUpdateMap.put(field.getFieldId(), field); + } + } + for (DatacenterTableField field : table.getFields()) { + DatacenterFieldDescriptionUpdate update = fieldUpdateMap.get(field.getId()); + if (update == null) { + continue; + } + field.setFieldDesc(normalizeDescription(update.getFieldDesc())); + field.setModified(now); + field.setModifiedBy(account == null ? BigInteger.ZERO : account.getId()); + tableFieldMapper.update(field); + } + return getTableWithFields(tableId); + } + + @Override + public DatacenterTable getTableWithFields(BigInteger tableId) { + DatacenterTable table = tableMapper.selectOneById(tableId); + if (table == null) { + throw new BusinessException("数据集不存在: " + tableId); + } + table.setFields(getFields(tableId)); + return table; + } + + @Override + public List getFields(BigInteger tableId) { + QueryWrapper wrapper = QueryWrapper.create(); + wrapper.eq(DatacenterTableField::getTableId, tableId); + wrapper.orderBy("id"); + return tableFieldMapper.selectListByQuery(wrapper); + } + + @Override + public DatasetRef resolveDatasetRef(BigInteger tableId) { + DatacenterTable table = getTableWithFields(tableId); + DatasetRef ref = new DatasetRef(); + ref.setTableId(tableId); + ref.setSourceId(table.getSourceId()); + ref.setCatalogId(table.getCatalogId()); + ref.setTableName(table.getTableName()); + DatacenterCatalog catalog = getCatalogById(table.getCatalogId()); + if (catalog != null) { + ref.setCatalogName(catalog.getCatalogName()); + } + return ref; + } + + @Override + public DatacenterSource getSourceRequired(BigInteger sourceId) { + DatacenterSource source = sourceMapper.selectOneById(sourceId); + if (source == null) { + throw new BusinessException("连接不存在"); + } + return source; + } + + @Override + public DatacenterCatalog getCatalogById(BigInteger catalogId) { + if (catalogId == null) { + return null; + } + return catalogMapper.selectOneById(catalogId); + } + + @Override + public List listManagedTables(BigInteger sourceId, BigInteger catalogId) { + QueryWrapper wrapper = QueryWrapper.create(); + wrapper.eq(DatacenterTable::getSourceId, sourceId); + if (catalogId != null) { + wrapper.eq(DatacenterTable::getCatalogId, catalogId); + } + wrapper.orderBy("created desc"); + return tableMapper.selectListByQuery(wrapper); + } + + @Override + @Transactional(rollbackFor = Exception.class) + public int removeTables(List tableIds) { + List ids = tableIds == null ? List.of() : tableIds.stream().filter(id -> id != null).distinct().toList(); + if (ids.isEmpty()) { + return 0; + } + QueryWrapper physicalTableWrapper = QueryWrapper.create(); + physicalTableWrapper.in(DatacenterTable::getId, ids); + List tables = tableMapper.selectListByQuery(physicalTableWrapper); + tables.forEach(this::deletePhysicalTableIfNecessary); + + QueryWrapper fieldWrapper = QueryWrapper.create(); + fieldWrapper.in(DatacenterTableField::getTableId, ids); + tableFieldMapper.deleteByQuery(fieldWrapper); + + QueryWrapper versionWrapper = QueryWrapper.create(); + versionWrapper.in(tech.easyflow.datacenter.meta.entity.DatacenterDatasetVersion::getTableId, ids); + datasetVersionMapper.deleteByQuery(versionWrapper); + + QueryWrapper importJobWrapper = QueryWrapper.create(); + importJobWrapper.in(tech.easyflow.datacenter.meta.entity.DatacenterImportJob::getTableId, ids); + importJobMapper.deleteByQuery(importJobWrapper); + + QueryWrapper upstreamWrapper = QueryWrapper.create(); + upstreamWrapper.in(tech.easyflow.datacenter.meta.entity.DatacenterDerivedTable::getSourceTableId, ids); + derivedTableMapper.deleteByQuery(upstreamWrapper); + + QueryWrapper downstreamWrapper = QueryWrapper.create(); + downstreamWrapper.in(tech.easyflow.datacenter.meta.entity.DatacenterDerivedTable::getDerivedTableId, ids); + derivedTableMapper.deleteByQuery(downstreamWrapper); + + QueryWrapper tableWrapper = QueryWrapper.create(); + tableWrapper.in(DatacenterTable::getId, ids); + return tableMapper.deleteByQuery(tableWrapper); + } + + private void deletePhysicalTableIfNecessary(DatacenterTable table) { + if (table == null || !shouldDropPhysicalTable(table)) { + return; + } + String physicalTableName = table.getActualTable(); + if (physicalTableName == null || physicalTableName.isBlank()) { + physicalTableName = table.getMaterializedTable(); + } + if (physicalTableName == null || physicalTableName.isBlank()) { + return; + } + DatacenterTable physicalTable = new DatacenterTable(); + physicalTable.setActualTable(physicalTableName); + dbHandleManager.getDbHandler().deleteTable(physicalTable); + } + + private boolean shouldDropPhysicalTable(DatacenterTable table) { + String tableKind = table.getTableKind(); + if (tableKind == null || tableKind.isBlank()) { + return false; + } + try { + DatacenterTableKind kind = DatacenterTableKind.valueOf(tableKind); + return kind == DatacenterTableKind.PROJECT_MANAGED + || kind == DatacenterTableKind.EXCEL_SHEET + || kind == DatacenterTableKind.EXCEL_MATERIALIZED + || kind == DatacenterTableKind.DERIVED_TABLE; + } catch (IllegalArgumentException ignored) { + return false; + } + } + + private void applyTableDefaults(DatacenterTable table, DatacenterSource source, DatacenterTableDetailMeta detail) { + if (table.getTableKind() == null || table.getTableKind().isBlank()) { + table.setTableKind(DatacenterTableKind.EXTERNAL_TABLE.name()); + } + if (table.getAccessMode() == null || table.getAccessMode().isBlank()) { + table.setAccessMode("READ_ONLY"); + } + if (table.getVersioningEnabled() == null) { + table.setVersioningEnabled(0); + } + if (table.getCapabilitiesJson() == null || table.getCapabilitiesJson().isEmpty()) { + table.setCapabilitiesJson(Map.of( + "capabilities", + source.getCapabilitiesJson() == null + ? List.of() + : source.getCapabilitiesJson().getOrDefault("capabilities", List.of()) + )); + } + if ((table.getActualTable() == null || table.getActualTable().isBlank()) && table.getTableName() != null) { + table.setActualTable(table.getTableName()); + } + table.setTableDesc(normalizeDescription(table.getTableDesc())); + if (detail.getFields() == null) { + detail.setFields(List.of()); + } + } + + private String normalizeDescription(String description) { + if (description == null) { + return ""; + } + return description.trim(); + } + + private boolean hasText(String value) { + return value != null && !value.trim().isEmpty(); + } + + private BigInteger builtinSourceId(DatacenterSourceType sourceType, BigInteger tenantId) { + if (sourceType == DatacenterSourceType.PROJECT_MYSQL) { + return DatacenterMetaConstants.PROJECT_SOURCE_BASE.add(tenantId); + } + throw new BusinessException("不支持的内置源 ID 计算"); + } + + private BigInteger builtinCatalogId(DatacenterSource source, String catalogName) { + if (source.getId().compareTo(DatacenterMetaConstants.PROJECT_SOURCE_BASE) >= 0 + && defaultCatalogName(DatacenterSourceType.PROJECT_MYSQL).equals(catalogName)) { + return DatacenterMetaConstants.PROJECT_CATALOG_BASE.add(source.getTenantId()); + } + return null; + } + + private String defaultCatalogName(DatacenterSourceType sourceType) { + if (sourceType == DatacenterSourceType.PROJECT_MYSQL) { + return "project_mysql"; + } + return sourceType.name().toLowerCase(); + } + + private List builtinCapabilities(DatacenterSourceType sourceType) { + return List.of("TEST_CONNECTION", "BROWSE_METADATA", "READ_QUERY", "WRITE_MUTATION"); + } +} diff --git a/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/meta/service/impl/DatacenterSourceServiceImpl.java b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/meta/service/impl/DatacenterSourceServiceImpl.java new file mode 100644 index 0000000..ffe74c9 --- /dev/null +++ b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/meta/service/impl/DatacenterSourceServiceImpl.java @@ -0,0 +1,311 @@ +package tech.easyflow.datacenter.meta.service.impl; + +import com.mybatisflex.core.paginate.Page; +import com.mybatisflex.core.query.QueryWrapper; +import com.mybatisflex.spring.service.impl.ServiceImpl; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; +import tech.easyflow.common.entity.LoginAccount; +import tech.easyflow.common.web.exceptions.BusinessException; +import tech.easyflow.datacenter.connector.DatacenterConnector; +import tech.easyflow.datacenter.connector.DatacenterConnectorRegistry; +import tech.easyflow.datacenter.entity.DatacenterTable; +import tech.easyflow.datacenter.mapper.DatacenterCatalogMapper; +import tech.easyflow.datacenter.mapper.DatacenterSourceMapper; +import tech.easyflow.datacenter.meta.entity.DatacenterCatalog; +import tech.easyflow.datacenter.meta.entity.DatacenterSource; +import tech.easyflow.datacenter.meta.enums.DatacenterSourceType; +import tech.easyflow.datacenter.meta.model.DatacenterBatchRegisterRequest; +import tech.easyflow.datacenter.meta.model.DatacenterCatalogMeta; +import tech.easyflow.datacenter.meta.model.DatacenterTableDetailMeta; +import tech.easyflow.datacenter.meta.service.DatacenterDatasetRegistryService; +import tech.easyflow.datacenter.meta.service.DatacenterSourceService; +import tech.easyflow.datacenter.meta.support.DatacenterSourceConnectionDefaults; +import tech.easyflow.datacenter.execution.model.DatacenterConnectionTestResult; +import tech.easyflow.datacenter.security.DatacenterCredentialCipher; + +import javax.annotation.Resource; +import java.math.BigInteger; +import java.util.Date; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +@Service +public class DatacenterSourceServiceImpl extends ServiceImpl implements DatacenterSourceService { + + @Resource + private DatacenterConnectorRegistry connectorRegistry; + @Resource + private DatacenterDatasetRegistryService registryService; + @Resource + private DatacenterCredentialCipher credentialCipher; + @Resource + private DatacenterCatalogMapper catalogMapper; + @Resource + private DatacenterSourceConnectionDefaults connectionDefaults; + + @Override + public DatacenterSource saveSource(DatacenterSource source, LoginAccount account) { + if (source == null || source.getSourceType() == null || source.getSourceType().isBlank()) { + throw new BusinessException("数据源类型不能为空"); + } + DatacenterSource existing = source.getId() == null ? null : getById(source.getId()); + DatacenterSource normalized = mergeWithExisting(existing, source); + DatacenterConnector connector = connectorRegistry.getConnector(normalized.getSourceType()); + applyCredentialCipher(normalized, existing); + normalized.setConfigJson(connectionDefaults.sanitizeConfig(normalized.getConfigJson())); + connectionDefaults.normalize(normalized); + normalized.setCapabilitiesJson(Map.of("capabilities", connector.getCapabilities().stream().map(Enum::name).toList())); + Date now = new Date(); + if (normalized.getId() == null) { + normalized.setCreated(now); + normalized.setCreatedBy(account == null ? BigInteger.ZERO : account.getId()); + normalized.setTenantId(account == null ? BigInteger.ZERO : account.getTenantId()); + normalized.setDeptId(account == null ? BigInteger.ZERO : account.getDeptId()); + normalized.setStatus(normalized.getStatus() == null ? 0 : normalized.getStatus()); + normalized.setModified(now); + normalized.setModifiedBy(account == null ? BigInteger.ZERO : account.getId()); + save(normalized); + } else { + if (existing == null) { + throw new BusinessException("连接不存在"); + } + normalized.setModified(now); + normalized.setModifiedBy(account == null ? BigInteger.ZERO : account.getId()); + updateById(normalized); + } + return getById(normalized.getId()); + } + + @Override + public Page pageSources(Long pageNumber, Long pageSize, LoginAccount account) { + registryService.ensureBuiltinSource(DatacenterSourceType.PROJECT_MYSQL, account); + return page(new Page<>(pageNumber == null ? 1L : pageNumber, pageSize == null ? 10L : pageSize), QueryWrapper.create()); + } + + @Override + public DatacenterConnectionTestResult testConnection(DatacenterSource source, LoginAccount account) { + DatacenterSource existing = source != null && source.getId() != null ? getById(source.getId()) : null; + if (source != null && source.getId() != null && existing == null) { + throw new BusinessException("连接不存在"); + } + DatacenterSource actual = mergeWithExisting(existing, source); + if (DatacenterSourceType.PROJECT_MYSQL.name().equals(actual.getSourceType())) { + actual = registryService.ensureBuiltinSource(DatacenterSourceType.PROJECT_MYSQL, account); + } + applyCredentialCipher(actual, existing); + actual.setConfigJson(connectionDefaults.sanitizeConfig(actual.getConfigJson())); + connectionDefaults.normalize(actual); + DatacenterConnector connector = connectorRegistry.getConnector(actual.getSourceType()); + DatacenterConnectionTestResult result = connector.testConnection(actual); + Map details = new LinkedHashMap<>(); + if (result.getDetails() != null) { + details.putAll(result.getDetails()); + } + details.put("effectiveDriverClassName", actual.getDriverClassName()); + details.put("effectiveJdbcUrl", actual.getJdbcUrl()); + details.put("effectivePort", actual.getPort()); + result.setDetails(details); + if (actual.getId() != null) { + DatacenterSource persisted = new DatacenterSource(); + persisted.setId(actual.getId()); + persisted.setLastTestStatus(result.isSuccess() ? "SUCCESS" : "FAILED"); + persisted.setLastTestMessage(result.getMessage()); + persisted.setLastTestedAt(new Date()); + updateById(persisted); + } + return result; + } + + @Override + public List listCatalogs(BigInteger sourceId, LoginAccount account) { + DatacenterSource source = registryService.getSourceRequired(sourceId); + if (isManagedOnly(source.getSourceType())) { + QueryWrapper wrapper = QueryWrapper.create(); + wrapper.eq(DatacenterCatalog::getSourceId, sourceId); + return catalogMapper.selectListByQuery(wrapper).stream().map(this::toCatalogMeta).collect(Collectors.toList()); + } + return connectorRegistry.getConnector(source.getSourceType()).listCatalogs(source); + } + + @Override + public List listTables(BigInteger sourceId, String catalogName, LoginAccount account) { + DatacenterSource source = registryService.getSourceRequired(sourceId); + if (isManagedOnly(source.getSourceType())) { + BigInteger catalogId = resolveCatalogId(sourceId, catalogName); + return registryService.listManagedTables(sourceId, catalogId); + } + return connectorRegistry.getConnector(source.getSourceType()).listTables(source, catalogName); + } + + @Override + public DatacenterTableDetailMeta getTableDetail(BigInteger sourceId, String catalogName, String tableName, boolean register, LoginAccount account) { + DatacenterSource source = registryService.getSourceRequired(sourceId); + if (isManagedOnly(source.getSourceType())) { + BigInteger catalogId = resolveCatalogId(sourceId, catalogName); + List tables = registryService.listManagedTables(sourceId, catalogId); + DatacenterTable target = tables.stream().filter(item -> tableName.equals(item.getTableName())).findFirst() + .orElseThrow(() -> new BusinessException("数据集不存在: " + tableName)); + DatacenterTableDetailMeta detail = new DatacenterTableDetailMeta(); + detail.setTable(registryService.getTableWithFields(target.getId())); + detail.setFields(detail.getTable().getFields()); + return detail; + } + DatacenterConnector connector = connectorRegistry.getConnector(source.getSourceType()); + DatacenterTableDetailMeta detail = connector.getTableDetail(source, catalogName, tableName); + if (register) { + DatacenterCatalog catalog = registryService.ensureCatalog(source, catalogName, account); + DatacenterTable table = registryService.registerTable(source, catalog, detail, account); + detail.setTable(table); + detail.setFields(table.getFields()); + } + return detail; + } + + @Override + public List batchRegisterTables(DatacenterBatchRegisterRequest request, LoginAccount account) { + if (request == null || request.getSourceId() == null) { + throw new BusinessException("数据连接不能为空"); + } + if (request.getCatalogName() == null || request.getCatalogName().isBlank()) { + throw new BusinessException("库不能为空"); + } + List tableNames = request.getTableNames() == null + ? List.of() + : request.getTableNames().stream().filter(name -> name != null && !name.isBlank()).distinct().toList(); + if (tableNames.isEmpty()) { + throw new BusinessException("至少选择一张表"); + } + DatacenterSource source = registryService.getSourceRequired(request.getSourceId()); + if (isManagedOnly(source.getSourceType())) { + throw new BusinessException("当前数据连接不支持批量接入"); + } + DatacenterConnector connector = connectorRegistry.getConnector(source.getSourceType()); + DatacenterCatalog catalog = registryService.ensureCatalog(source, request.getCatalogName(), account); + return tableNames.stream().map(tableName -> { + DatacenterTableDetailMeta detail = connector.getTableDetail(source, request.getCatalogName(), tableName); + return registryService.registerTable(source, catalog, detail, account); + }).collect(Collectors.toList()); + } + + @Override + @Transactional(rollbackFor = Exception.class) + public void removeSource(BigInteger sourceId, LoginAccount account) { + if (sourceId == null) { + throw new BusinessException("数据连接不能为空"); + } + DatacenterSource source = getById(sourceId); + if (source == null) { + throw new BusinessException("连接不存在"); + } + if (Boolean.TRUE.equals(source.getBuiltinFlag())) { + throw new BusinessException("内置数据连接不支持删除"); + } + + List tableIds = registryService.listManagedTables(sourceId, null).stream() + .map(DatacenterTable::getId) + .filter(id -> id != null) + .toList(); + registryService.removeTables(tableIds); + + QueryWrapper catalogWrapper = QueryWrapper.create(); + catalogWrapper.eq(DatacenterCatalog::getSourceId, sourceId); + catalogMapper.deleteByQuery(catalogWrapper); + + removeById(sourceId); + } + + private boolean isManagedOnly(String sourceType) { + return DatacenterSourceType.EXCEL.name().equals(sourceType) + || DatacenterSourceType.EXCEL_MATERIALIZED.name().equals(sourceType); + } + + private DatacenterCatalogMeta toCatalogMeta(DatacenterCatalog catalog) { + DatacenterCatalogMeta meta = new DatacenterCatalogMeta(); + meta.setId(catalog.getId()); + meta.setSourceId(catalog.getSourceId()); + meta.setCatalogName(catalog.getCatalogName()); + meta.setCatalogDesc(catalog.getCatalogDesc()); + meta.setCatalogType(catalog.getCatalogType()); + return meta; + } + + private BigInteger resolveCatalogId(BigInteger sourceId, String catalogName) { + if (catalogName == null || catalogName.isBlank()) { + return null; + } + QueryWrapper wrapper = QueryWrapper.create(); + wrapper.eq(DatacenterCatalog::getSourceId, sourceId); + wrapper.eq(DatacenterCatalog::getCatalogName, catalogName); + DatacenterCatalog catalog = catalogMapper.selectOneByQuery(wrapper); + if (catalog == null) { + throw new BusinessException("目录不存在: " + catalogName); + } + return catalog.getId(); + } + + private void applyCredentialCipher(DatacenterSource target, DatacenterSource existing) { + String password = extractPassword(target.getConfigJson()); + if (password != null) { + target.setCredentialCipher(credentialCipher.encrypt(password)); + return; + } + if ((target.getCredentialCipher() == null || target.getCredentialCipher().isBlank()) && existing != null) { + target.setCredentialCipher(existing.getCredentialCipher()); + } + } + + private String extractPassword(Map configJson) { + if (configJson == null) { + return null; + } + Object password = configJson.get("password"); + if (password == null) { + return null; + } + String value = String.valueOf(password); + return value.isBlank() ? null : value; + } + + private DatacenterSource mergeWithExisting(DatacenterSource existing, DatacenterSource incoming) { + if (incoming == null) { + return existing; + } + if (existing == null) { + return incoming; + } + DatacenterSource merged = new DatacenterSource(); + merged.setId(existing.getId()); + merged.setCreated(existing.getCreated()); + merged.setCreatedBy(existing.getCreatedBy()); + merged.setTenantId(existing.getTenantId()); + merged.setDeptId(existing.getDeptId()); + merged.setBuiltinFlag(existing.getBuiltinFlag()); + merged.setStatus(existing.getStatus()); + merged.setLastTestStatus(existing.getLastTestStatus()); + merged.setLastTestMessage(existing.getLastTestMessage()); + merged.setLastTestedAt(existing.getLastTestedAt()); + merged.setOptions(existing.getOptions()); + merged.setSourceName(valueOrExisting(incoming.getSourceName(), existing.getSourceName())); + merged.setSourceCode(valueOrExisting(incoming.getSourceCode(), existing.getSourceCode())); + merged.setSourceType(valueOrExisting(incoming.getSourceType(), existing.getSourceType())); + merged.setAccessMode(valueOrExisting(incoming.getAccessMode(), existing.getAccessMode())); + merged.setDriverClassName(valueOrExisting(incoming.getDriverClassName(), existing.getDriverClassName())); + merged.setJdbcUrl(valueOrExisting(incoming.getJdbcUrl(), existing.getJdbcUrl())); + merged.setHost(valueOrExisting(incoming.getHost(), existing.getHost())); + merged.setPort(incoming.getPort() != null ? incoming.getPort() : existing.getPort()); + merged.setDatabaseName(valueOrExisting(incoming.getDatabaseName(), existing.getDatabaseName())); + merged.setSchemaName(valueOrExisting(incoming.getSchemaName(), existing.getSchemaName())); + merged.setUsername(valueOrExisting(incoming.getUsername(), existing.getUsername())); + merged.setCredentialCipher(valueOrExisting(incoming.getCredentialCipher(), existing.getCredentialCipher())); + merged.setConfigJson(incoming.getConfigJson() != null ? incoming.getConfigJson() : existing.getConfigJson()); + merged.setCapabilitiesJson(existing.getCapabilitiesJson()); + return merged; + } + + private String valueOrExisting(String incoming, String existing) { + return incoming != null ? incoming : existing; + } +} diff --git a/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/meta/support/DatacenterSourceConnectionDefaults.java b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/meta/support/DatacenterSourceConnectionDefaults.java new file mode 100644 index 0000000..b0f9c93 --- /dev/null +++ b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/meta/support/DatacenterSourceConnectionDefaults.java @@ -0,0 +1,153 @@ +package tech.easyflow.datacenter.meta.support; + +import cn.hutool.core.util.StrUtil; +import org.springframework.stereotype.Component; +import tech.easyflow.datacenter.meta.entity.DatacenterSource; +import tech.easyflow.datacenter.meta.enums.DatacenterSourceType; + +import java.util.LinkedHashMap; +import java.util.Map; + +@Component +public class DatacenterSourceConnectionDefaults { + + public boolean supportsExternalConnection(String sourceType) { + if (StrUtil.isBlank(sourceType)) { + return false; + } + DatacenterSourceType resolved = resolveSourceType(sourceType); + if (resolved == null) { + return false; + } + return switch (resolved) { + case MYSQL, POSTGRESQL, ORACLE, GAUSSDB_NATIVE, GBASE_8A, GBASE_8S -> true; + default -> false; + }; + } + + public DatacenterSource normalize(DatacenterSource source) { + if (source == null || !supportsExternalConnection(source.getSourceType())) { + return source; + } + if (source.getPort() == null || source.getPort() <= 0) { + source.setPort(defaultPort(source.getSourceType())); + } + if (StrUtil.isBlank(source.getDriverClassName())) { + source.setDriverClassName(defaultDriverClassName(source.getSourceType())); + } + if (StrUtil.isBlank(source.getJdbcUrl())) { + String jdbcUrl = buildJdbcUrl(source); + if (StrUtil.isNotBlank(jdbcUrl)) { + source.setJdbcUrl(jdbcUrl); + } + } + return source; + } + + public Map sanitizeConfig(Map configJson) { + Map sanitized = new LinkedHashMap<>(); + if (configJson == null || configJson.isEmpty()) { + return sanitized; + } + sanitized.putAll(configJson); + sanitized.remove("password"); + return sanitized; + } + + public Integer defaultPort(String sourceType) { + DatacenterSourceType resolved = requireSourceType(sourceType); + return switch (resolved) { + case MYSQL -> 3306; + case POSTGRESQL, GAUSSDB_NATIVE -> 5432; + case ORACLE -> 1521; + case GBASE_8A -> 5258; + case GBASE_8S -> 9088; + default -> null; + }; + } + + public String defaultDriverClassName(String sourceType) { + DatacenterSourceType resolved = requireSourceType(sourceType); + return switch (resolved) { + case MYSQL -> "com.mysql.cj.jdbc.Driver"; + case POSTGRESQL, GAUSSDB_NATIVE -> "org.postgresql.Driver"; + case ORACLE -> "oracle.jdbc.OracleDriver"; + case GBASE_8A -> "com.gbase.jdbc.Driver"; + case GBASE_8S -> "com.gbasedbt.jdbc.Driver"; + default -> ""; + }; + } + + public String buildJdbcUrl(DatacenterSource source) { + if (source == null || !supportsExternalConnection(source.getSourceType())) { + return null; + } + String host = StrUtil.trimToEmpty(source.getHost()); + Integer port = source.getPort() == null || source.getPort() <= 0 ? defaultPort(source.getSourceType()) : source.getPort(); + String databaseName = resolveDatabaseName(source); + if (StrUtil.isBlank(host) || port == null || StrUtil.isBlank(databaseName)) { + return null; + } + DatacenterSourceType resolved = requireSourceType(source.getSourceType()); + return switch (resolved) { + case MYSQL -> String.format( + "jdbc:mysql://%s:%d/%s?useUnicode=true&characterEncoding=utf-8&serverTimezone=Asia/Shanghai&useSSL=false", + host, port, databaseName + ); + case POSTGRESQL -> String.format("jdbc:postgresql://%s:%d/%s", host, port, databaseName); + case ORACLE -> String.format("jdbc:oracle:thin:@//%s:%d/%s", host, port, databaseName); + case GAUSSDB_NATIVE -> String.format("jdbc:postgresql://%s:%d/%s", host, port, databaseName); + case GBASE_8A -> String.format("jdbc:gbase://%s:%d/%s", host, port, databaseName); + case GBASE_8S -> { + String informixServer = resolveInformixServer(source); + if (StrUtil.isBlank(informixServer)) { + yield null; + } + yield String.format("jdbc:gbasedbt-sqli://%s:%d/%s:INFORMIXSERVER=%s", host, port, databaseName, informixServer); + } + default -> null; + }; + } + + private DatacenterSourceType requireSourceType(String sourceType) { + DatacenterSourceType resolved = resolveSourceType(sourceType); + if (resolved == null) { + throw new IllegalArgumentException("Unsupported datacenter source type: " + sourceType); + } + return resolved; + } + + private DatacenterSourceType resolveSourceType(String sourceType) { + try { + return DatacenterSourceType.valueOf(sourceType); + } catch (IllegalArgumentException ex) { + return null; + } + } + + private String resolveDatabaseName(DatacenterSource source) { + if (StrUtil.isNotBlank(source.getDatabaseName())) { + return source.getDatabaseName().trim(); + } + if (source.getConfigJson() == null) { + return null; + } + Object serviceName = source.getConfigJson().get("serviceName"); + if (serviceName != null && StrUtil.isNotBlank(String.valueOf(serviceName))) { + return String.valueOf(serviceName).trim(); + } + return null; + } + + private String resolveInformixServer(DatacenterSource source) { + if (source.getConfigJson() == null) { + return null; + } + Object informixServer = source.getConfigJson().get("informixServer"); + if (informixServer == null) { + return null; + } + String value = String.valueOf(informixServer).trim(); + return value.isEmpty() ? null : value; + } +} diff --git a/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/security/DatacenterCredentialCipher.java b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/security/DatacenterCredentialCipher.java new file mode 100644 index 0000000..53db8f4 --- /dev/null +++ b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/security/DatacenterCredentialCipher.java @@ -0,0 +1,28 @@ +package tech.easyflow.datacenter.security; + +import cn.hutool.crypto.SecureUtil; +import cn.hutool.crypto.symmetric.AES; +import org.springframework.stereotype.Component; + +import java.nio.charset.StandardCharsets; + +@Component +public class DatacenterCredentialCipher { + + private static final String DEFAULT_KEY = "easyflow-datacenter-phase1-key"; + private final AES aes = SecureUtil.aes(SecureUtil.sha256(DEFAULT_KEY).substring(0, 16).getBytes(StandardCharsets.UTF_8)); + + public String encrypt(String plainText) { + if (plainText == null || plainText.isBlank()) { + return null; + } + return aes.encryptHex(plainText); + } + + public String decrypt(String cipherText) { + if (cipherText == null || cipherText.isBlank()) { + return null; + } + return aes.decryptStr(cipherText); + } +} diff --git a/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/service/DatacenterTableFieldService.java b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/service/DatacenterTableFieldService.java deleted file mode 100644 index 3b4dc41..0000000 --- a/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/service/DatacenterTableFieldService.java +++ /dev/null @@ -1,14 +0,0 @@ -package tech.easyflow.datacenter.service; - -import com.mybatisflex.core.service.IService; -import tech.easyflow.datacenter.entity.DatacenterTableField; - -/** - * 服务层。 - * - * @author ArkLight - * @since 2025-07-10 - */ -public interface DatacenterTableFieldService extends IService { - -} diff --git a/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/service/DatacenterTableService.java b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/service/DatacenterTableService.java deleted file mode 100644 index e3dec4a..0000000 --- a/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/service/DatacenterTableService.java +++ /dev/null @@ -1,35 +0,0 @@ -package tech.easyflow.datacenter.service; - -import com.alibaba.fastjson2.JSONObject; -import com.mybatisflex.core.paginate.Page; -import com.mybatisflex.core.row.Row; -import com.mybatisflex.core.service.IService; -import tech.easyflow.common.entity.DatacenterQuery; -import tech.easyflow.common.entity.LoginAccount; -import tech.easyflow.datacenter.entity.DatacenterTable; -import tech.easyflow.datacenter.entity.DatacenterTableField; -import tech.easyflow.datacenter.entity.vo.HeaderVo; - -import java.math.BigInteger; -import java.util.List; - -public interface DatacenterTableService extends IService { - - void saveTable(DatacenterTable entity, LoginAccount loginUser); - - void removeTable(BigInteger tableId); - - Long getCount(DatacenterQuery where); - - List getListData(DatacenterQuery where); - - Page getPageData(DatacenterQuery where); - - List getHeaders(BigInteger tableId); - - void saveValue(BigInteger tableId, JSONObject object, LoginAccount account); - - void removeValue(BigInteger tableId, BigInteger id, LoginAccount account); - - List getFields(BigInteger tableId); -} diff --git a/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/service/impl/DatacenterTableFieldServiceImpl.java b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/service/impl/DatacenterTableFieldServiceImpl.java deleted file mode 100644 index b89ae80..0000000 --- a/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/service/impl/DatacenterTableFieldServiceImpl.java +++ /dev/null @@ -1,18 +0,0 @@ -package tech.easyflow.datacenter.service.impl; - -import com.mybatisflex.spring.service.impl.ServiceImpl; -import org.springframework.stereotype.Service; -import tech.easyflow.datacenter.entity.DatacenterTableField; -import tech.easyflow.datacenter.mapper.DatacenterTableFieldMapper; -import tech.easyflow.datacenter.service.DatacenterTableFieldService; - -/** - * 服务层实现。 - * - * @author ArkLight - * @since 2025-07-10 - */ -@Service -public class DatacenterTableFieldServiceImpl extends ServiceImpl implements DatacenterTableFieldService { - -} diff --git a/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/service/impl/DatacenterTableServiceImpl.java b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/service/impl/DatacenterTableServiceImpl.java deleted file mode 100644 index 4b55983..0000000 --- a/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/service/impl/DatacenterTableServiceImpl.java +++ /dev/null @@ -1,293 +0,0 @@ -package tech.easyflow.datacenter.service.impl; - -import cn.hutool.core.collection.CollectionUtil; -import cn.hutool.core.util.StrUtil; -import com.alibaba.fastjson2.JSONObject; -import com.mybatisflex.core.keygen.impl.SnowFlakeIDKeyGenerator; -import com.mybatisflex.core.paginate.Page; -import com.mybatisflex.core.query.QueryWrapper; -import com.mybatisflex.core.row.Db; -import com.mybatisflex.core.row.Row; -import com.mybatisflex.spring.service.impl.ServiceImpl; -import org.springframework.stereotype.Service; -import org.springframework.transaction.annotation.Transactional; -import tech.easyflow.common.cache.RedisLockExecutor; -import tech.easyflow.common.entity.DatacenterQuery; -import tech.easyflow.common.entity.LoginAccount; -import tech.easyflow.common.web.exceptions.BusinessException; -import tech.easyflow.datacenter.adapter.DbHandleManager; -import tech.easyflow.datacenter.adapter.DbHandleService; -import tech.easyflow.datacenter.entity.DatacenterTable; -import tech.easyflow.datacenter.entity.DatacenterTableField; -import tech.easyflow.datacenter.entity.vo.HeaderVo; -import tech.easyflow.datacenter.mapper.DatacenterTableFieldMapper; -import tech.easyflow.datacenter.mapper.DatacenterTableMapper; -import tech.easyflow.datacenter.service.DatacenterTableService; - -import javax.annotation.Resource; -import java.math.BigDecimal; -import java.math.BigInteger; -import java.time.Duration; -import java.util.*; -import java.util.stream.Collectors; - -@Service -public class DatacenterTableServiceImpl extends ServiceImpl implements DatacenterTableService { - - private static final String DATACENTER_TABLE_LOCK_KEY_PREFIX = "easyflow:lock:datacenter:table:"; - private static final String DATACENTER_TABLE_CREATE_LOCK_KEY_PREFIX = "easyflow:lock:datacenter:table:create:"; - private static final Duration LOCK_WAIT_TIMEOUT = Duration.ofSeconds(2); - private static final Duration LOCK_LEASE_TIMEOUT = Duration.ofSeconds(15); - - @Resource - private DbHandleManager dbHandleManager; - @Resource - private DatacenterTableFieldMapper fieldsMapper; - @Resource - private RedisLockExecutor redisLockExecutor; - - @Override - @Transactional(rollbackFor = Exception.class) - public void saveTable(DatacenterTable entity, LoginAccount loginUser) { - String lockKey = buildTableLockKey(entity, loginUser); - redisLockExecutor.executeWithLock(lockKey, LOCK_WAIT_TIMEOUT, LOCK_LEASE_TIMEOUT, () -> { - doSaveTable(entity, loginUser); - }); - } - - private void doSaveTable(DatacenterTable entity, LoginAccount loginUser) { - DbHandleService dbHandler = dbHandleManager.getDbHandler(); - - List fields = entity.getFields(); - - BigInteger tableId = entity.getId(); - - if (tableId == null) { - long snowId = new SnowFlakeIDKeyGenerator().nextId(); - entity.setId(new BigInteger(String.valueOf(snowId))); - - String actualTable = getActualTableName(entity); - entity.setActualTable(actualTable); - // 先 DDL 操作，DDL会默认提交事务，不然报错了事务不会回滚。 - dbHandler.createTable(entity); - // 保存主表和字段表 - save(entity); - for (DatacenterTableField field : fields) { - // 插入 - field.setCreated(new Date()); - field.setCreatedBy(loginUser.getId()); - field.setModified(new Date()); - field.setModifiedBy(loginUser.getId()); - field.setTableId(entity.getId()); - fieldsMapper.insert(field); - } - } else { - // actualTable 前端不可见，所以要设置 - DatacenterTable tableRecord = getById(tableId); - entity.setActualTable(tableRecord.getActualTable()); - dbHandler.updateTable(entity, tableRecord); - updateById(entity); - // 查询所有字段 - QueryWrapper w = QueryWrapper.create(); - w.eq(DatacenterTableField::getTableId, entity.getId()); - List fieldRecords = fieldsMapper.selectListByQuery(w); - - Map fieldsMap = fieldRecords.stream() - .collect(Collectors.toMap(DatacenterTableField::getId, field -> field)); - - for (DatacenterTableField field : fields) { - BigInteger id = field.getId(); - if (id == null) { - // 新增字段到物理表 - dbHandler.addField(entity, field); - // 插入 - field.setCreated(new Date()); - field.setCreatedBy(loginUser.getId()); - field.setModified(new Date()); - field.setModifiedBy(loginUser.getId()); - field.setTableId(entity.getId()); - fieldsMapper.insert(field); - } else { - // 删除的字段 - if (field.isHandleDelete()) { - // 删除物理表中的字段 - dbHandler.deleteField(entity, field); - // 删除字段 - fieldsMapper.deleteById(id); - } else { - // 修改物理表中的字段 - DatacenterTableField fieldRecord = fieldsMap.get(id); - dbHandler.updateField(entity, fieldRecord, field); - // 更新字段，字段类型不允许修改 - field.setFieldType(field.getFieldType()); - field.setModified(new Date()); - field.setModifiedBy(loginUser.getId()); - fieldsMapper.update(field); - } - } - } - } - } - - @Override - @Transactional(rollbackFor = Exception.class) - public void removeTable(BigInteger tableId) { - redisLockExecutor.executeWithLock( - DATACENTER_TABLE_LOCK_KEY_PREFIX + tableId, - LOCK_WAIT_TIMEOUT, - LOCK_LEASE_TIMEOUT, - () -> { - DatacenterTable record = getById(tableId); - dbHandleManager.getDbHandler().deleteTable(record); - removeById(tableId); - QueryWrapper wrapper = QueryWrapper.create(); - wrapper.eq(DatacenterTableField::getTableId, tableId); - fieldsMapper.deleteByQuery(wrapper); - } - ); - } - - @Override - public Long getCount(DatacenterQuery where) { - String actualTable = getActualTable(where.getTableId()); - QueryWrapper wrapper = QueryWrapper.create(); - buildCondition(wrapper, where); - return Db.selectCountByQuery(actualTable, wrapper); - } - - @Override - public List getListData(DatacenterQuery where) { - String actualTable = getActualTable(where.getTableId()); - QueryWrapper wrapper = QueryWrapper.create(); - buildCondition(wrapper, where); - List rows = Db.selectListByQuery(actualTable, wrapper); - handleBigNumber(rows); - return rows; - } - - @Override - public Page getPageData(DatacenterQuery where) { - Long pageNumber = where.getPageNumber(); - Long pageSize = where.getPageSize(); - - Long count = getCount(where); - if (count == 0) { - return new Page<>(new ArrayList<>(), pageNumber, pageSize, count); - } - - String actualTable = getActualTable(where.getTableId()); - QueryWrapper wrapper = QueryWrapper.create(); - buildCondition(wrapper, where); - - Page page = new Page<>(pageNumber, pageSize, count); - Page paginate = Db.paginate(actualTable, page, wrapper); - handleBigNumber(paginate.getRecords()); - return paginate; - } - - private void handleBigNumber(List records) { - for (Row record : records) { - Map newMap = new LinkedHashMap<>(); - for (Map.Entry entry : record.entrySet()) { - Object value = entry.getValue(); - if ((value instanceof BigInteger || - value instanceof BigDecimal || - value instanceof Long)) { - newMap.put(entry.getKey(), value.toString()); - } else { - newMap.put(entry.getKey(), value); - } - } - record.clear(); - record.putAll(newMap); - } - } - - @Override - public List getHeaders(BigInteger tableId) { - QueryWrapper wrapper = QueryWrapper.create(); - wrapper.eq(DatacenterTableField::getTableId, tableId); - wrapper.orderBy("id"); - List fields = fieldsMapper.selectListByQuery(wrapper); - List headers = new ArrayList<>(); - for (DatacenterTableField field : fields) { - HeaderVo header = new HeaderVo(); - header.setKey(field.getFieldName()); - header.setDataIndex(field.getFieldName()); - header.setTitle(field.getFieldDesc()); - header.setFieldType(field.getFieldType()); - header.setRequired(field.getRequired()); - header.setFieldId(field.getId()); - header.setTableId(field.getTableId()); - headers.add(header); - } - return headers; - } - - @Override - public void saveValue(BigInteger tableId, JSONObject object, LoginAccount account) { - - DatacenterTable table = getById(tableId); - - QueryWrapper wrapper = QueryWrapper.create(); - wrapper.eq(DatacenterTableField::getTableId, tableId); - List fields = fieldsMapper.selectListByQuery(wrapper); - - if (CollectionUtil.isEmpty(fields)) { - throw new BusinessException("请先添加字段"); - } - table.setFields(fields); - Object valueId = object.get("id"); - if (valueId == null) { - dbHandleManager.getDbHandler().saveValue(table, object, account); - } else { - dbHandleManager.getDbHandler().updateValue(table, object, account); - } - } - - @Override - public void removeValue(BigInteger tableId, BigInteger id, LoginAccount account) { - DatacenterTable record = getById(tableId); - dbHandleManager.getDbHandler().removeValue(record, id, account); - } - - @Override - public List getFields(BigInteger tableId) { - QueryWrapper wrapper = QueryWrapper.create(); - wrapper.eq(DatacenterTableField::getTableId, tableId); - return fieldsMapper.selectListByQuery(wrapper); - } - - private String getActualTable(BigInteger tableId) { - DatacenterTable record = getById(tableId); - return record.getActualTable(); - } - - private String getActualTableName(DatacenterTable table) { - String tableName = table.getTableName(); - BigInteger id = table.getId(); - return "tb_dynamic_" + tableName + "_" + id; - } - - private String buildTableLockKey(DatacenterTable table, LoginAccount loginUser) { - if (table.getId() != null) { - return DATACENTER_TABLE_LOCK_KEY_PREFIX + table.getId(); - } - String tenant = table.getTenantId() != null - ? table.getTenantId().toString() - : (loginUser != null && loginUser.getTenantId() != null ? loginUser.getTenantId().toString() : "0"); - String tableName = table.getTableName() == null ? "unknown" : table.getTableName(); - return DATACENTER_TABLE_CREATE_LOCK_KEY_PREFIX + tenant + ":" + tableName; - } - - /** - * 构建查询条件 - */ - private void buildCondition(QueryWrapper wrapper, DatacenterQuery where) { - // 构建查询条件 - String condition = where.getWhere(); - if (StrUtil.isNotEmpty(condition)) { - wrapper.where(condition); - } - } -} diff --git a/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/utils/SqlSupportUtils.java b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/utils/SqlSupportUtils.java new file mode 100644 index 0000000..4c7e0d0 --- /dev/null +++ b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/utils/SqlSupportUtils.java @@ -0,0 +1,223 @@ +package tech.easyflow.datacenter.utils; + +import net.sf.jsqlparser.JSQLParserException; +import net.sf.jsqlparser.parser.CCJSqlParserUtil; +import net.sf.jsqlparser.schema.Table; +import net.sf.jsqlparser.statement.Statement; +import net.sf.jsqlparser.statement.Statements; +import net.sf.jsqlparser.statement.select.Select; +import net.sf.jsqlparser.statement.select.WithItem; +import net.sf.jsqlparser.util.TablesNamesFinder; +import tech.easyflow.common.web.exceptions.BusinessException; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Set; + +public final class SqlSupportUtils { + + private SqlSupportUtils() { + } + + public static ResolvedSql resolve(String sql, Collection managedTables) { + String normalizedSql = normalizeSql(sql); + Statement statement = parseSingleStatement(normalizedSql); + if (!(statement instanceof Select select)) { + throw new BusinessException("查询数据节点只支持只读 SELECT SQL"); + } + Map> byTableName = new LinkedHashMap<>(); + Map byCatalogAndTable = new LinkedHashMap<>(); + for (ManagedTable managedTable : managedTables) { + if (managedTable == null || !hasText(managedTable.getTableName())) { + continue; + } + String tableKey = normalizeIdentifier(managedTable.getTableName()); + byTableName.computeIfAbsent(tableKey, key -> new ArrayList<>()).add(managedTable); + if (hasText(managedTable.getCatalogName())) { + byCatalogAndTable.put( + catalogTableKey(managedTable.getCatalogName(), managedTable.getTableName()), + managedTable + ); + } + } + + SqlTableCollector collector = new SqlTableCollector(); + List referencedTables = collector.collect(select); + if (referencedTables.isEmpty()) { + throw new BusinessException("SQL 必须引用至少一张已接入表"); + } + Set logicalTables = new LinkedHashSet<>(); + for (Table table : referencedTables) { + ManagedTable managedTable = resolveManagedTable(table, byTableName, byCatalogAndTable); + rewriteTable(table, managedTable); + logicalTables.add(renderLogicalTable(managedTable)); + } + return new ResolvedSql(select.toString(), new ArrayList<>(logicalTables)); + } + + private static Statement parseSingleStatement(String sql) { + try { + Statements statements = CCJSqlParserUtil.parseStatements(sql); + if (statements == null || statements.getStatements() == null || statements.getStatements().isEmpty()) { + throw new BusinessException("SQL 不能为空"); + } + if (statements.getStatements().size() != 1) { + throw new BusinessException("查询数据节点仅支持单条 SQL"); + } + return statements.getStatements().get(0); + } catch (BusinessException e) { + throw e; + } catch (JSQLParserException e) { + throw new BusinessException("SQL 解析失败，请检查语法"); + } + } + + private static ManagedTable resolveManagedTable(Table table, + Map> byTableName, + Map byCatalogAndTable) { + String tableName = trimToNull(table.getName()); + if (!hasText(tableName)) { + throw new BusinessException("SQL 包含无法识别的表名"); + } + String catalogName = trimToNull(table.getSchemaName()); + if (hasText(catalogName)) { + ManagedTable managedTable = byCatalogAndTable.get(catalogTableKey(catalogName, tableName)); + if (managedTable == null) { + throw new BusinessException("SQL 引用了未接入表: " + catalogName + "." + tableName); + } + return managedTable; + } + List matches = byTableName.get(normalizeIdentifier(tableName)); + if (matches == null || matches.isEmpty()) { + throw new BusinessException("SQL 引用了未接入表: " + tableName); + } + if (matches.size() > 1) { + throw new BusinessException("SQL 引用了重名表，请使用 catalog.table: " + tableName); + } + return matches.get(0); + } + + private static void rewriteTable(Table table, ManagedTable managedTable) { + table.setName(managedTable.getPhysicalTableName()); + table.setSchemaName(trimToNull(managedTable.getCatalogName())); + } + + private static String renderLogicalTable(ManagedTable managedTable) { + if (!hasText(managedTable.getCatalogName())) { + return managedTable.getTableName(); + } + return managedTable.getCatalogName() + "." + managedTable.getTableName(); + } + + private static String normalizeSql(String sql) { + String normalized = trimToNull(sql); + if (!hasText(normalized)) { + throw new BusinessException("SQL 不能为空"); + } + return normalized; + } + + private static String catalogTableKey(String catalogName, String tableName) { + return normalizeIdentifier(catalogName) + "." + normalizeIdentifier(tableName); + } + + private static String normalizeIdentifier(String value) { + String normalized = trimToNull(value); + if (!hasText(normalized)) { + return ""; + } + if ((normalized.startsWith("`") && normalized.endsWith("`")) + || (normalized.startsWith("\"") && normalized.endsWith("\"")) + || (normalized.startsWith("[") && normalized.endsWith("]"))) { + normalized = normalized.substring(1, normalized.length() - 1); + } + return normalized.trim().toLowerCase(Locale.ROOT); + } + + private static boolean hasText(String value) { + return value != null && !value.isBlank(); + } + + private static String trimToNull(String value) { + if (value == null) { + return null; + } + String trimmed = value.trim(); + return trimmed.isEmpty() ? null : trimmed; + } + + public static class ManagedTable { + private final String catalogName; + private final String tableName; + private final String physicalTableName; + + public ManagedTable(String catalogName, String tableName, String physicalTableName) { + this.catalogName = trimToNull(catalogName); + this.tableName = trimToNull(tableName); + this.physicalTableName = hasText(physicalTableName) ? physicalTableName.trim() : this.tableName; + } + + public String getCatalogName() { + return catalogName; + } + + public String getTableName() { + return tableName; + } + + public String getPhysicalTableName() { + return physicalTableName; + } + } + + public static class ResolvedSql { + private final String executableSql; + private final List logicalTables; + + public ResolvedSql(String executableSql, List logicalTables) { + this.executableSql = executableSql; + this.logicalTables = logicalTables; + } + + public String getExecutableSql() { + return executableSql; + } + + public List getLogicalTables() { + return logicalTables; + } + } + + private static class SqlTableCollector extends TablesNamesFinder { + private final List
tables = new ArrayList<>(); + private final Set withNames = new LinkedHashSet<>(); + + public List
collect(Statement statement) { + tables.clear(); + withNames.clear(); + statement.accept(this); + return new ArrayList<>(tables); + } + + @Override + public void visit(WithItem withItem) { + if (withItem.getAlias() != null && hasText(withItem.getAlias().getName())) { + withNames.add(normalizeIdentifier(withItem.getAlias().getName())); + } + withItem.getSelect().accept((net.sf.jsqlparser.statement.select.SelectVisitor) this); + } + + @Override + public void visit(Table table) { + String tableName = normalizeIdentifier(table.getName()); + if (!withNames.contains(tableName)) { + tables.add(table); + } + } + } +} diff --git a/easyflow-starter/easyflow-starter-all/src/main/resources/db/migration/V11__datacenter_unified_access_phase1.sql b/easyflow-starter/easyflow-starter-all/src/main/resources/db/migration/V11__datacenter_unified_access_phase1.sql new file mode 100644 index 0000000..d9553be --- /dev/null +++ b/easyflow-starter/easyflow-starter-all/src/main/resources/db/migration/V11__datacenter_unified_access_phase1.sql @@ -0,0 +1,202 @@ +CREATE TABLE `tb_datacenter_source` +( + `id` bigint UNSIGNED NOT NULL COMMENT '主键', + `dept_id` bigint UNSIGNED NOT NULL DEFAULT 0 COMMENT '部门ID', + `tenant_id` bigint UNSIGNED NOT NULL DEFAULT 0 COMMENT '租户ID', + `source_name` varchar(100) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NOT NULL COMMENT '数据源名称', + `source_code` varchar(100) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NULL DEFAULT NULL COMMENT '数据源编码', + `source_type` varchar(32) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NOT NULL COMMENT '数据源类型', + `access_mode` varchar(16) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NOT NULL DEFAULT 'READ_ONLY' COMMENT '访问模式', + `builtin_flag` int NOT NULL DEFAULT 0 COMMENT '是否内置', + `driver_class_name` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NULL DEFAULT NULL COMMENT '驱动类名', + `jdbc_url` varchar(512) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NULL DEFAULT NULL COMMENT 'JDBC URL', + `host` varchar(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NULL DEFAULT NULL COMMENT '主机', + `port` int NULL DEFAULT NULL COMMENT '端口', + `database_name` varchar(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NULL DEFAULT NULL COMMENT '数据库名', + `schema_name` varchar(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NULL DEFAULT NULL COMMENT 'Schema名', + `username` varchar(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NULL DEFAULT NULL COMMENT '用户名', + `credential_cipher` text CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NULL COMMENT '凭据密文', + `config_json` text CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NULL COMMENT '连接配置', + `capabilities_json` text CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NULL COMMENT '能力声明', + `last_test_status` varchar(32) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NULL DEFAULT NULL COMMENT '最近测试状态', + `last_test_message` varchar(500) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NULL DEFAULT NULL COMMENT '最近测试信息', + `last_tested_at` datetime NULL DEFAULT NULL COMMENT '最近测试时间', + `status` int NOT NULL DEFAULT 0 COMMENT '状态', + `options` text CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NULL COMMENT '扩展项', + `created` datetime NOT NULL COMMENT '创建时间', + `created_by` bigint UNSIGNED NOT NULL DEFAULT 0 COMMENT '创建人', + `modified` datetime NOT NULL COMMENT '修改时间', + `modified_by` bigint UNSIGNED NOT NULL DEFAULT 0 COMMENT '修改人', + PRIMARY KEY (`id`) USING BTREE, + KEY `idx_datacenter_source_tenant_type` (`tenant_id`, `source_type`) USING BTREE +) ENGINE = InnoDB CHARACTER SET = utf8mb4 COLLATE = utf8mb4_0900_ai_ci COMMENT = '数据中心数据源' ROW_FORMAT = DYNAMIC; + +CREATE TABLE `tb_datacenter_catalog` +( + `id` bigint UNSIGNED NOT NULL COMMENT '主键', + `dept_id` bigint UNSIGNED NOT NULL DEFAULT 0 COMMENT '部门ID', + `tenant_id` bigint UNSIGNED NOT NULL DEFAULT 0 COMMENT '租户ID', + `source_id` bigint UNSIGNED NOT NULL COMMENT '数据源ID', + `catalog_name` varchar(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NOT NULL COMMENT '目录名', + `catalog_desc` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NULL DEFAULT NULL COMMENT '目录描述', + `catalog_type` varchar(32) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NOT NULL DEFAULT 'DATABASE' COMMENT '目录类型', + `status` int NOT NULL DEFAULT 0 COMMENT '状态', + `options` text CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NULL COMMENT '扩展项', + `created` datetime NOT NULL COMMENT '创建时间', + `created_by` bigint UNSIGNED NOT NULL DEFAULT 0 COMMENT '创建人', + `modified` datetime NOT NULL COMMENT '修改时间', + `modified_by` bigint UNSIGNED NOT NULL DEFAULT 0 COMMENT '修改人', + PRIMARY KEY (`id`) USING BTREE, + KEY `idx_datacenter_catalog_source` (`source_id`, `catalog_name`) USING BTREE +) ENGINE = InnoDB CHARACTER SET = utf8mb4 COLLATE = utf8mb4_0900_ai_ci COMMENT = '数据中心逻辑库/命名空间' ROW_FORMAT = DYNAMIC; + +ALTER TABLE `tb_datacenter_table` + ADD COLUMN `source_id` bigint UNSIGNED NULL DEFAULT NULL COMMENT '数据源ID' AFTER `tenant_id`, + ADD COLUMN `catalog_id` bigint UNSIGNED NULL DEFAULT NULL COMMENT '目录ID' AFTER `source_id`, + ADD COLUMN `table_kind` varchar(32) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NOT NULL DEFAULT 'LOCAL_DYNAMIC' COMMENT '表类型' AFTER `actual_table`, + ADD COLUMN `access_mode` varchar(16) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NOT NULL DEFAULT 'READ_WRITE' COMMENT '访问模式' AFTER `table_kind`, + ADD COLUMN `materialized_table` varchar(200) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NULL DEFAULT NULL COMMENT '物化表名' AFTER `access_mode`, + ADD COLUMN `versioning_enabled` int NOT NULL DEFAULT 0 COMMENT '是否开启版本' AFTER `materialized_table`, + ADD COLUMN `capabilities_json` text CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NULL COMMENT '能力声明' AFTER `options`; + +ALTER TABLE `tb_datacenter_table_field` + ADD COLUMN `source_column_name` varchar(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NULL DEFAULT NULL COMMENT '源字段名' AFTER `field_name`, + ADD COLUMN `jdbc_type` varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NULL DEFAULT NULL COMMENT 'JDBC类型' AFTER `field_type`, + ADD COLUMN `precision` int NULL DEFAULT NULL COMMENT '精度' AFTER `jdbc_type`, + ADD COLUMN `scale` int NULL DEFAULT NULL COMMENT '小数位' AFTER `precision`, + ADD COLUMN `queryable` int NOT NULL DEFAULT 1 COMMENT '可查询' AFTER `required`, + ADD COLUMN `sortable` int NOT NULL DEFAULT 1 COMMENT '可排序' AFTER `queryable`, + ADD COLUMN `writable` int NOT NULL DEFAULT 1 COMMENT '可写入' AFTER `sortable`, + ADD COLUMN `indexed` int NOT NULL DEFAULT 0 COMMENT '是否索引' AFTER `writable`; + +CREATE TABLE `tb_datacenter_dataset_version` +( + `id` bigint UNSIGNED NOT NULL COMMENT '主键', + `dept_id` bigint UNSIGNED NOT NULL DEFAULT 0 COMMENT '部门ID', + `tenant_id` bigint UNSIGNED NOT NULL DEFAULT 0 COMMENT '租户ID', + `table_id` bigint UNSIGNED NOT NULL COMMENT '表ID', + `version_no` int NOT NULL DEFAULT 1 COMMENT '版本号', + `version_label` varchar(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NULL DEFAULT NULL COMMENT '版本标签', + `materialized_table` varchar(200) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NULL DEFAULT NULL COMMENT '物化表名', + `snapshot_json` text CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NULL COMMENT '版本快照', + `status` int NOT NULL DEFAULT 0 COMMENT '状态', + `created` datetime NOT NULL COMMENT '创建时间', + `created_by` bigint UNSIGNED NOT NULL DEFAULT 0 COMMENT '创建人', + `modified` datetime NOT NULL COMMENT '修改时间', + `modified_by` bigint UNSIGNED NOT NULL DEFAULT 0 COMMENT '修改人', + PRIMARY KEY (`id`) USING BTREE, + KEY `idx_dataset_version_table` (`table_id`, `version_no`) USING BTREE +) ENGINE = InnoDB CHARACTER SET = utf8mb4 COLLATE = utf8mb4_0900_ai_ci COMMENT = '数据集版本' ROW_FORMAT = DYNAMIC; + +CREATE TABLE `tb_datacenter_import_job` +( + `id` bigint UNSIGNED NOT NULL COMMENT '主键', + `dept_id` bigint UNSIGNED NOT NULL DEFAULT 0 COMMENT '部门ID', + `tenant_id` bigint UNSIGNED NOT NULL DEFAULT 0 COMMENT '租户ID', + `source_id` bigint UNSIGNED NULL DEFAULT NULL COMMENT '数据源ID', + `catalog_id` bigint UNSIGNED NULL DEFAULT NULL COMMENT '目录ID', + `table_id` bigint UNSIGNED NULL DEFAULT NULL COMMENT '表ID', + `job_type` varchar(32) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NOT NULL COMMENT '任务类型', + `file_name` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NULL DEFAULT NULL COMMENT '文件名', + `storage_path` varchar(512) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NULL DEFAULT NULL COMMENT '文件存储路径', + `status` varchar(32) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NOT NULL COMMENT '任务状态', + `total_rows` bigint NULL DEFAULT NULL COMMENT '总行数', + `success_rows` bigint NULL DEFAULT NULL COMMENT '成功行数', + `error_rows` bigint NULL DEFAULT NULL COMMENT '失败行数', + `error_summary` varchar(1000) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NULL DEFAULT NULL COMMENT '错误摘要', + `payload_json` longtext CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NULL COMMENT '任务载荷', + `started_at` datetime NULL DEFAULT NULL COMMENT '开始时间', + `finished_at` datetime NULL DEFAULT NULL COMMENT '结束时间', + `created` datetime NOT NULL COMMENT '创建时间', + `created_by` bigint UNSIGNED NOT NULL DEFAULT 0 COMMENT '创建人', + `modified` datetime NOT NULL COMMENT '修改时间', + `modified_by` bigint UNSIGNED NOT NULL DEFAULT 0 COMMENT '修改人', + PRIMARY KEY (`id`) USING BTREE, + KEY `idx_import_job_source` (`source_id`, `status`) USING BTREE +) ENGINE = InnoDB CHARACTER SET = utf8mb4 COLLATE = utf8mb4_0900_ai_ci COMMENT = '数据中心导入任务' ROW_FORMAT = DYNAMIC; + +CREATE TABLE `tb_datacenter_derived_table` +( + `id` bigint UNSIGNED NOT NULL COMMENT '主键', + `dept_id` bigint UNSIGNED NOT NULL DEFAULT 0 COMMENT '部门ID', + `tenant_id` bigint UNSIGNED NOT NULL DEFAULT 0 COMMENT '租户ID', + `source_table_id` bigint UNSIGNED NOT NULL COMMENT '源表ID', + `derived_table_id` bigint UNSIGNED NOT NULL COMMENT '派生表ID', + `derive_type` varchar(32) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NOT NULL COMMENT '派生类型', + `derive_config_json` longtext CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NULL COMMENT '派生配置', + `status` int NOT NULL DEFAULT 0 COMMENT '状态', + `created` datetime NOT NULL COMMENT '创建时间', + `created_by` bigint UNSIGNED NOT NULL DEFAULT 0 COMMENT '创建人', + `modified` datetime NOT NULL COMMENT '修改时间', + `modified_by` bigint UNSIGNED NOT NULL DEFAULT 0 COMMENT '修改人', + PRIMARY KEY (`id`) USING BTREE, + KEY `idx_derived_table_source` (`source_table_id`, `derived_table_id`) USING BTREE +) ENGINE = InnoDB CHARACTER SET = utf8mb4 COLLATE = utf8mb4_0900_ai_ci COMMENT = '数据中心派生表关系' ROW_FORMAT = DYNAMIC; + +INSERT INTO `tb_datacenter_source` (`id`, `dept_id`, `tenant_id`, `source_name`, `source_code`, `source_type`, `access_mode`, `builtin_flag`, `status`, `config_json`, `capabilities_json`, `created`, `created_by`, `modified`, `modified_by`) +SELECT DISTINCT + 9000000000000000000 + `tenant_id`, + COALESCE(MIN(`dept_id`), 0), + `tenant_id`, + '本地动态表', + CONCAT('LOCAL_DYNAMIC_', `tenant_id`), + 'LOCAL_DYNAMIC', + 'READ_WRITE', + 1, + 0, + JSON_OBJECT('builtin', true), + JSON_OBJECT('capabilities', JSON_ARRAY('TEST_CONNECTION', 'BROWSE_METADATA', 'READ_QUERY', 'WRITE_MUTATION', 'MATERIALIZE')), + NOW(), + 0, + NOW(), + 0 +FROM `tb_datacenter_table` +GROUP BY `tenant_id`; + +INSERT INTO `tb_datacenter_catalog` (`id`, `dept_id`, `tenant_id`, `source_id`, `catalog_name`, `catalog_desc`, `catalog_type`, `status`, `created`, `created_by`, `modified`, `modified_by`) +SELECT DISTINCT + 9000000000001000000 + `tenant_id`, + COALESCE(MIN(`dept_id`), 0), + `tenant_id`, + 9000000000000000000 + `tenant_id`, + 'local_dynamic', + '本地动态表目录', + 'DATABASE', + 0, + NOW(), + 0, + NOW(), + 0 +FROM `tb_datacenter_table` +GROUP BY `tenant_id`; + +INSERT INTO `tb_datacenter_source` (`id`, `dept_id`, `tenant_id`, `source_name`, `source_code`, `source_type`, `access_mode`, `builtin_flag`, `status`, `config_json`, `capabilities_json`, `created`, `created_by`, `modified`, `modified_by`) +SELECT 9000000000000000000, 0, 0, '本地动态表', 'LOCAL_DYNAMIC_0', 'LOCAL_DYNAMIC', 'READ_WRITE', 1, 0, + JSON_OBJECT('builtin', true), + JSON_OBJECT('capabilities', JSON_ARRAY('TEST_CONNECTION', 'BROWSE_METADATA', 'READ_QUERY', 'WRITE_MUTATION', 'MATERIALIZE')), + NOW(), 0, NOW(), 0 +FROM dual +WHERE NOT EXISTS (SELECT 1 FROM `tb_datacenter_source` WHERE `id` = 9000000000000000000); + +INSERT INTO `tb_datacenter_catalog` (`id`, `dept_id`, `tenant_id`, `source_id`, `catalog_name`, `catalog_desc`, `catalog_type`, `status`, `created`, `created_by`, `modified`, `modified_by`) +SELECT 9000000000001000000, 0, 0, 9000000000000000000, 'local_dynamic', '本地动态表目录', 'DATABASE', 0, NOW(), 0, NOW(), 0 +FROM dual +WHERE NOT EXISTS (SELECT 1 FROM `tb_datacenter_catalog` WHERE `id` = 9000000000001000000); + +UPDATE `tb_datacenter_table` +SET `source_id` = 9000000000000000000 + `tenant_id`, + `catalog_id` = 9000000000001000000 + `tenant_id`, + `table_kind` = 'LOCAL_DYNAMIC', + `access_mode` = 'READ_WRITE', + `materialized_table` = `actual_table`, + `versioning_enabled` = 0, + `capabilities_json` = JSON_OBJECT('capabilities', JSON_ARRAY('READ_QUERY', 'WRITE_MUTATION')) +WHERE `source_id` IS NULL; + +UPDATE `tb_datacenter_table_field` +SET `source_column_name` = `field_name`, + `queryable` = 1, + `sortable` = 1, + `writable` = 1, + `indexed` = 0 +WHERE `source_column_name` IS NULL; diff --git a/easyflow-starter/easyflow-starter-all/src/main/resources/db/migration/V12__datacenter_phase2_menu.sql b/easyflow-starter/easyflow-starter-all/src/main/resources/db/migration/V12__datacenter_phase2_menu.sql new file mode 100644 index 0000000..1415233 --- /dev/null +++ b/easyflow-starter/easyflow-starter-all/src/main/resources/db/migration/V12__datacenter_phase2_menu.sql @@ -0,0 +1,86 @@ +SET NAMES utf8mb4; + +UPDATE `tb_sys_menu` +SET + `menu_url` = '/datacenter', + `component` = '', + `remark` = '数据中心统一数据接入平台' +WHERE `id` = 300817858217091072; + +INSERT INTO `tb_sys_menu` ( + `id`, `parent_id`, `menu_type`, `menu_title`, `menu_url`, `component`, + `menu_icon`, `is_show`, `permission_tag`, `sort_no`, `status`, + `created`, `created_by`, `modified`, `modified_by`, `remark` +) +SELECT + 366300000000000001, 300817858217091072, 0, 'menus.ai.datacenterDataset', + '/datacenter/dataset', '/datacenter/DatacenterDatasetManage', + 'svg:data-center', 1, '', 10, 0, + NOW(), 1, NOW(), 1, '数据中心-数据集管理' +WHERE NOT EXISTS ( + SELECT 1 FROM `tb_sys_menu` WHERE `id` = 366300000000000001 +); + +INSERT INTO `tb_sys_menu` ( + `id`, `parent_id`, `menu_type`, `menu_title`, `menu_url`, `component`, + `menu_icon`, `is_show`, `permission_tag`, `sort_no`, `status`, + `created`, `created_by`, `modified`, `modified_by`, `remark` +) +SELECT + 366300000000000002, 300817858217091072, 0, 'menus.ai.datacenterSource', + '/datacenter/source', '/datacenter/DatacenterSourceAccess', + 'svg:data-center', 1, '', 20, 0, + NOW(), 1, NOW(), 1, '数据中心-数据源接入' +WHERE NOT EXISTS ( + SELECT 1 FROM `tb_sys_menu` WHERE `id` = 366300000000000002 +); + +INSERT INTO `tb_sys_menu` ( + `id`, `parent_id`, `menu_type`, `menu_title`, `menu_url`, `component`, + `menu_icon`, `is_show`, `permission_tag`, `sort_no`, `status`, + `created`, `created_by`, `modified`, `modified_by`, `remark` +) +SELECT + 366300000000000003, 366300000000000002, 1, '查询', + '', '', '', 0, '/api/v1/datacenterSource/query', 1, 0, + NOW(), 1, NOW(), 1, '数据源接入-查询' +WHERE NOT EXISTS ( + SELECT 1 FROM `tb_sys_menu` WHERE `id` = 366300000000000003 +); + +INSERT INTO `tb_sys_menu` ( + `id`, `parent_id`, `menu_type`, `menu_title`, `menu_url`, `component`, + `menu_icon`, `is_show`, `permission_tag`, `sort_no`, `status`, + `created`, `created_by`, `modified`, `modified_by`, `remark` +) +SELECT + 366300000000000004, 366300000000000002, 1, '保存', + '', '', '', 0, '/api/v1/datacenterSource/save', 2, 0, + NOW(), 1, NOW(), 1, '数据源接入-保存' +WHERE NOT EXISTS ( + SELECT 1 FROM `tb_sys_menu` WHERE `id` = 366300000000000004 +); + +INSERT INTO `tb_sys_role_menu` (`id`, `role_id`, `menu_id`) +SELECT 366300000000000101, 1, 366300000000000001 +WHERE NOT EXISTS ( + SELECT 1 FROM `tb_sys_role_menu` WHERE `id` = 366300000000000101 +); + +INSERT INTO `tb_sys_role_menu` (`id`, `role_id`, `menu_id`) +SELECT 366300000000000102, 1, 366300000000000002 +WHERE NOT EXISTS ( + SELECT 1 FROM `tb_sys_role_menu` WHERE `id` = 366300000000000102 +); + +INSERT INTO `tb_sys_role_menu` (`id`, `role_id`, `menu_id`) +SELECT 366300000000000103, 1, 366300000000000003 +WHERE NOT EXISTS ( + SELECT 1 FROM `tb_sys_role_menu` WHERE `id` = 366300000000000103 +); + +INSERT INTO `tb_sys_role_menu` (`id`, `role_id`, `menu_id`) +SELECT 366300000000000104, 1, 366300000000000004 +WHERE NOT EXISTS ( + SELECT 1 FROM `tb_sys_role_menu` WHERE `id` = 366300000000000104 +); diff --git a/easyflow-starter/easyflow-starter-all/src/main/resources/db/migration/V13__datacenter_remove_legacy_menu.sql b/easyflow-starter/easyflow-starter-all/src/main/resources/db/migration/V13__datacenter_remove_legacy_menu.sql new file mode 100644 index 0000000..4cb3e58 --- /dev/null +++ b/easyflow-starter/easyflow-starter-all/src/main/resources/db/migration/V13__datacenter_remove_legacy_menu.sql @@ -0,0 +1,20 @@ +SET NAMES utf8mb4; + +DELETE FROM `tb_sys_role_menu` +WHERE `menu_id` IN ( + 300818298270883840, + 300818387710222336, + 300818488214134784 +); + +DELETE FROM `tb_sys_menu` +WHERE `id` IN ( + 300818298270883840, + 300818387710222336, + 300818488214134784 +) +OR `permission_tag` IN ( + '/api/v1/datacenterTable/query', + '/api/v1/datacenterTable/save', + '/api/v1/datacenterTable/remove' +); diff --git a/easyflow-starter/easyflow-starter-all/src/main/resources/db/migration/V14__datacenter_remove_local_dynamic.sql b/easyflow-starter/easyflow-starter-all/src/main/resources/db/migration/V14__datacenter_remove_local_dynamic.sql new file mode 100644 index 0000000..edc7003 --- /dev/null +++ b/easyflow-starter/easyflow-starter-all/src/main/resources/db/migration/V14__datacenter_remove_local_dynamic.sql @@ -0,0 +1,101 @@ +INSERT INTO `tb_datacenter_source` ( + `id`, `dept_id`, `tenant_id`, `source_name`, `source_code`, `source_type`, `access_mode`, + `builtin_flag`, `status`, `config_json`, `capabilities_json`, `created`, `created_by`, `modified`, `modified_by` +) +SELECT + 9000000000002000000 + t.tenant_id, + t.dept_id, + t.tenant_id, + '项目 MySQL', + CONCAT('PROJECT_MYSQL_', t.tenant_id), + 'PROJECT_MYSQL', + 'READ_WRITE', + 1, + 0, + JSON_OBJECT('builtin', true), + JSON_OBJECT('capabilities', JSON_ARRAY('TEST_CONNECTION', 'BROWSE_METADATA', 'READ_QUERY', 'WRITE_MUTATION')), + NOW(), + 0, + NOW(), + 0 +FROM ( + SELECT tenant_id, COALESCE(MIN(dept_id), 0) AS dept_id + FROM ( + SELECT `tenant_id`, `dept_id` + FROM `tb_datacenter_source` + WHERE `source_type` = 'LOCAL_DYNAMIC' + UNION ALL + SELECT `tenant_id`, `dept_id` + FROM `tb_datacenter_table` + WHERE `table_kind` = 'LOCAL_DYNAMIC' + OR `source_id` IN ( + SELECT `id` FROM `tb_datacenter_source` WHERE `source_type` = 'LOCAL_DYNAMIC' + ) + ) tenant_scope + GROUP BY tenant_id +) t +WHERE NOT EXISTS ( + SELECT 1 FROM `tb_datacenter_source` s WHERE s.`id` = 9000000000002000000 + t.tenant_id +); + +INSERT INTO `tb_datacenter_catalog` ( + `id`, `dept_id`, `tenant_id`, `source_id`, `catalog_name`, `catalog_desc`, `catalog_type`, + `status`, `created`, `created_by`, `modified`, `modified_by` +) +SELECT + 9000000000003000000 + t.tenant_id, + t.dept_id, + t.tenant_id, + 9000000000002000000 + t.tenant_id, + 'project_mysql', + '项目 MySQL', + 'DATABASE', + 0, + NOW(), + 0, + NOW(), + 0 +FROM ( + SELECT tenant_id, COALESCE(MIN(dept_id), 0) AS dept_id + FROM ( + SELECT `tenant_id`, `dept_id` + FROM `tb_datacenter_source` + WHERE `source_type` = 'LOCAL_DYNAMIC' + UNION ALL + SELECT `tenant_id`, `dept_id` + FROM `tb_datacenter_table` + WHERE `table_kind` = 'LOCAL_DYNAMIC' + OR `source_id` IN ( + SELECT `id` FROM `tb_datacenter_source` WHERE `source_type` = 'LOCAL_DYNAMIC' + ) + ) tenant_scope + GROUP BY tenant_id +) t +WHERE NOT EXISTS ( + SELECT 1 FROM `tb_datacenter_catalog` c WHERE c.`id` = 9000000000003000000 + t.tenant_id +); + +UPDATE `tb_datacenter_table` +SET `source_id` = 9000000000002000000 + `tenant_id`, + `catalog_id` = 9000000000003000000 + `tenant_id`, + `table_kind` = 'PROJECT_MANAGED', + `access_mode` = 'READ_WRITE', + `materialized_table` = COALESCE(`materialized_table`, `actual_table`), + `versioning_enabled` = COALESCE(`versioning_enabled`, 0), + `capabilities_json` = JSON_OBJECT('capabilities', JSON_ARRAY('READ_QUERY', 'WRITE_MUTATION')) +WHERE `table_kind` = 'LOCAL_DYNAMIC' + OR `source_id` IN ( + SELECT `id` FROM `tb_datacenter_source` WHERE `source_type` = 'LOCAL_DYNAMIC' + ); + +DELETE c +FROM `tb_datacenter_catalog` c +INNER JOIN `tb_datacenter_source` s ON s.`id` = c.`source_id` +WHERE s.`source_type` = 'LOCAL_DYNAMIC'; + +DELETE FROM `tb_datacenter_source` +WHERE `source_type` = 'LOCAL_DYNAMIC'; + +ALTER TABLE `tb_datacenter_table` + MODIFY COLUMN `table_kind` varchar(32) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NOT NULL DEFAULT 'EXTERNAL_TABLE' COMMENT '表类型', + MODIFY COLUMN `access_mode` varchar(16) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NOT NULL DEFAULT 'READ_ONLY' COMMENT '访问模式'; diff --git a/easyflow-ui-admin/app/src/api/request.ts b/easyflow-ui-admin/app/src/api/request.ts index ecafbe9..9f0f56e 100644 --- a/easyflow-ui-admin/app/src/api/request.ts +++ b/easyflow-ui-admin/app/src/api/request.ts @@ -23,6 +23,31 @@ import { useAuthStore } from '#/store'; import { refreshTokenApi } from './core'; const { apiURL } = useAppConfig(import.meta.env, import.meta.env.PROD); +const ERROR_MESSAGE_DEDUP_WINDOW = 800; +const SILENT_ERROR_MESSAGES = new Set([ + '当前连接不可用，请检查连接配置后重试', + '连接不存在', +]); +let lastErrorMessage = ''; +let lastErrorTimestamp = 0; + +function showErrorOnce(message?: string) { + const nextMessage = String(message || '').trim(); + if (!nextMessage) return; + if (SILENT_ERROR_MESSAGES.has(nextMessage)) { + return; + } + const now = Date.now(); + if ( + nextMessage === lastErrorMessage && + now - lastErrorTimestamp < ERROR_MESSAGE_DEDUP_WINDOW + ) { + return; + } + lastErrorMessage = nextMessage; + lastErrorTimestamp = now; + ElMessage.error(nextMessage); +} function createRequestClient(baseURL: string, options?: RequestClientOptions) { const client = new RequestClient({ @@ -80,7 +105,7 @@ function createRequestClient(baseURL: string, options?: RequestClientOptions) { codeField: 'errorCode', dataField: 'data', showErrorMessage: (message) => { - ElMessage.error(message); + showErrorOnce(message); }, successCode: 0, }), @@ -105,7 +130,7 @@ function createRequestClient(baseURL: string, options?: RequestClientOptions) { const responseData = error?.response?.data ?? {}; const errorMessage = responseData?.error ?? responseData?.message ?? ''; // 如果没有错误信息，则会根据状态码进行提示 - ElMessage.error(errorMessage || msg); + showErrorOnce(errorMessage || msg); }), ); diff --git a/easyflow-ui-admin/app/src/assets/datacenter/huawei-icon.svg b/easyflow-ui-admin/app/src/assets/datacenter/huawei-icon.svg new file mode 100644 index 0000000..a16aa03 --- /dev/null +++ b/easyflow-ui-admin/app/src/assets/datacenter/huawei-icon.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/easyflow-ui-admin/app/src/assets/datacenter/mysql-icon.svg b/easyflow-ui-admin/app/src/assets/datacenter/mysql-icon.svg new file mode 100644 index 0000000..be18116 --- /dev/null +++ b/easyflow-ui-admin/app/src/assets/datacenter/mysql-icon.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/easyflow-ui-admin/app/src/assets/datacenter/oracle-icon.svg b/easyflow-ui-admin/app/src/assets/datacenter/oracle-icon.svg new file mode 100644 index 0000000..fa63757 --- /dev/null +++ b/easyflow-ui-admin/app/src/assets/datacenter/oracle-icon.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/easyflow-ui-admin/app/src/assets/datacenter/postgresql-icon.svg b/easyflow-ui-admin/app/src/assets/datacenter/postgresql-icon.svg new file mode 100644 index 0000000..9fa5c4d --- /dev/null +++ b/easyflow-ui-admin/app/src/assets/datacenter/postgresql-icon.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/easyflow-ui-admin/app/src/router/routes/modules/datacenter.ts b/easyflow-ui-admin/app/src/router/routes/modules/datacenter.ts index 8de210a..793b831 100644 --- a/easyflow-ui-admin/app/src/router/routes/modules/datacenter.ts +++ b/easyflow-ui-admin/app/src/router/routes/modules/datacenter.ts @@ -6,14 +6,39 @@ const routes: RouteRecordRaw[] = [ { meta: { icon: 'clarity:database', - title: $t('datacenterTable.title'), + title: $t('menus.ai.datacenter'), + hideInMenu: true, + activePath: '/datacenter', + }, + name: 'DatacenterWorkspace', + path: '/datacenter', + component: () => import('#/views/datacenter/DatacenterWorkspace.vue'), + }, + { + meta: { + icon: 'clarity:database', + title: $t('menus.ai.datacenter'), hideInMenu: true, hideInTab: true, hideInBreadcrumb: true, + activePath: '/datacenter', }, - name: 'TableDetail', - path: '/datacenter/table/tableDetail', - component: () => import('#/views/datacenter/DatacenterTableDetail.vue'), + name: 'DatacenterSourceAccessLegacy', + path: '/datacenter/source', + redirect: '/datacenter', + }, + { + meta: { + icon: 'clarity:database', + title: $t('menus.ai.datacenter'), + hideInMenu: true, + hideInTab: true, + hideInBreadcrumb: true, + activePath: '/datacenter', + }, + name: 'DatacenterDatasetManageLegacy', + path: '/datacenter/dataset', + redirect: '/datacenter', }, ]; diff --git a/easyflow-ui-admin/app/src/views/datacenter/BatchImportModal.vue b/easyflow-ui-admin/app/src/views/datacenter/BatchImportModal.vue deleted file mode 100644 index 7877296..0000000 --- a/easyflow-ui-admin/app/src/views/datacenter/BatchImportModal.vue +++ /dev/null @@ -1,154 +0,0 @@ - - - - - diff --git a/easyflow-ui-admin/app/src/views/datacenter/DatacenterTableDetail.vue b/easyflow-ui-admin/app/src/views/datacenter/DatacenterTableDetail.vue deleted file mode 100644 index d2553b6..0000000 --- a/easyflow-ui-admin/app/src/views/datacenter/DatacenterTableDetail.vue +++ /dev/null @@ -1,272 +0,0 @@ - - - - - diff --git a/easyflow-ui-admin/app/src/views/datacenter/DatacenterTableList.vue b/easyflow-ui-admin/app/src/views/datacenter/DatacenterTableList.vue deleted file mode 100644 index 1be8ae7..0000000 --- a/easyflow-ui-admin/app/src/views/datacenter/DatacenterTableList.vue +++ /dev/null @@ -1,204 +0,0 @@ - - - - - diff --git a/easyflow-ui-admin/app/src/views/datacenter/DatacenterTableModal.vue b/easyflow-ui-admin/app/src/views/datacenter/DatacenterTableModal.vue deleted file mode 100644 index 2339d26..0000000 --- a/easyflow-ui-admin/app/src/views/datacenter/DatacenterTableModal.vue +++ /dev/null @@ -1,272 +0,0 @@ - - - - - diff --git a/easyflow-ui-admin/app/src/views/datacenter/DatacenterWorkspace.vue b/easyflow-ui-admin/app/src/views/datacenter/DatacenterWorkspace.vue new file mode 100644 index 0000000..3bae09c --- /dev/null +++ b/easyflow-ui-admin/app/src/views/datacenter/DatacenterWorkspace.vue @@ -0,0 +1,539 @@ + + + + + diff --git a/easyflow-ui-admin/app/src/views/datacenter/RecordModal.vue b/easyflow-ui-admin/app/src/views/datacenter/RecordModal.vue deleted file mode 100644 index e3630ad..0000000 --- a/easyflow-ui-admin/app/src/views/datacenter/RecordModal.vue +++ /dev/null @@ -1,145 +0,0 @@ - - - - - diff --git a/easyflow-ui-admin/app/src/views/datacenter/components/ConnectionTree.vue b/easyflow-ui-admin/app/src/views/datacenter/components/ConnectionTree.vue new file mode 100644 index 0000000..a2c7199 --- /dev/null +++ b/easyflow-ui-admin/app/src/views/datacenter/components/ConnectionTree.vue @@ -0,0 +1,474 @@ + + + + + diff --git a/easyflow-ui-admin/app/src/views/datacenter/components/ExcelActionDrawer.vue b/easyflow-ui-admin/app/src/views/datacenter/components/ExcelActionDrawer.vue new file mode 100644 index 0000000..02e8d6c --- /dev/null +++ b/easyflow-ui-admin/app/src/views/datacenter/components/ExcelActionDrawer.vue @@ -0,0 +1,210 @@ + + + + + diff --git a/easyflow-ui-admin/app/src/views/datacenter/components/SourceBrandIcon.vue b/easyflow-ui-admin/app/src/views/datacenter/components/SourceBrandIcon.vue new file mode 100644 index 0000000..2bd2bbe --- /dev/null +++ b/easyflow-ui-admin/app/src/views/datacenter/components/SourceBrandIcon.vue @@ -0,0 +1,106 @@ + + + + + diff --git a/easyflow-ui-admin/app/src/views/datacenter/components/SourceFormDrawer.vue b/easyflow-ui-admin/app/src/views/datacenter/components/SourceFormDrawer.vue new file mode 100644 index 0000000..8ea7d0a --- /dev/null +++ b/easyflow-ui-admin/app/src/views/datacenter/components/SourceFormDrawer.vue @@ -0,0 +1,364 @@ + + + + + diff --git a/easyflow-ui-admin/app/src/views/datacenter/components/TableDetailView.vue b/easyflow-ui-admin/app/src/views/datacenter/components/TableDetailView.vue new file mode 100644 index 0000000..f170fe3 --- /dev/null +++ b/easyflow-ui-admin/app/src/views/datacenter/components/TableDetailView.vue @@ -0,0 +1,422 @@ + + + + + diff --git a/easyflow-ui-admin/app/src/views/datacenter/components/TableListView.vue b/easyflow-ui-admin/app/src/views/datacenter/components/TableListView.vue new file mode 100644 index 0000000..7aec115 --- /dev/null +++ b/easyflow-ui-admin/app/src/views/datacenter/components/TableListView.vue @@ -0,0 +1,414 @@ + + + + + diff --git a/easyflow-ui-admin/app/src/views/datacenter/composables/datacenter-constants.ts b/easyflow-ui-admin/app/src/views/datacenter/composables/datacenter-constants.ts new file mode 100644 index 0000000..d2e3b00 --- /dev/null +++ b/easyflow-ui-admin/app/src/views/datacenter/composables/datacenter-constants.ts @@ -0,0 +1,163 @@ +export const sourceTypeOptions = [ + { label: 'Excel 文件', value: 'EXCEL' }, + { label: 'MySQL', value: 'MYSQL' }, + { label: 'PostgreSQL', value: 'POSTGRESQL' }, + { label: 'Oracle', value: 'ORACLE' }, + { label: 'GaussDB', value: 'GAUSSDB_NATIVE' }, + { label: 'GBase 8a', value: 'GBASE_8A' }, + { label: 'GBase 8s', value: 'GBASE_8S' }, +]; + +export const sourceConnectionDefaults: Record< + string, + { + buildJdbcUrl: (form: Record) => string; + defaultDriver: string; + defaultPort: number; + } +> = { + MYSQL: { + defaultPort: 3306, + defaultDriver: 'com.mysql.cj.jdbc.Driver', + buildJdbcUrl: (p) => + p.host && p.port && p.databaseName + ? `jdbc:mysql://${p.host}:${p.port}/${p.databaseName}?useUnicode=true&characterEncoding=utf-8&serverTimezone=Asia/Shanghai&useSSL=false` + : '', + }, + POSTGRESQL: { + defaultPort: 5432, + defaultDriver: 'org.postgresql.Driver', + buildJdbcUrl: (p) => + p.host && p.port && p.databaseName + ? `jdbc:postgresql://${p.host}:${p.port}/${p.databaseName}` + : '', + }, + ORACLE: { + defaultPort: 1521, + defaultDriver: 'oracle.jdbc.OracleDriver', + buildJdbcUrl: (p) => { + const serviceName = p?.configJson?.serviceName || p.databaseName; + return p.host && p.port && serviceName + ? `jdbc:oracle:thin:@//${p.host}:${p.port}/${serviceName}` + : ''; + }, + }, + GAUSSDB_NATIVE: { + defaultPort: 5432, + defaultDriver: 'org.postgresql.Driver', + buildJdbcUrl: (p) => + p.host && p.port && p.databaseName + ? `jdbc:postgresql://${p.host}:${p.port}/${p.databaseName}` + : '', + }, + GBASE_8A: { + defaultPort: 5258, + defaultDriver: 'com.gbase.jdbc.Driver', + buildJdbcUrl: (p) => + p.host && p.port && p.databaseName + ? `jdbc:gbase://${p.host}:${p.port}/${p.databaseName}` + : '', + }, + GBASE_8S: { + defaultPort: 9088, + defaultDriver: 'com.gbasedbt.jdbc.Driver', + buildJdbcUrl: (p) => { + const informixServer = p?.configJson?.informixServer || 'gbasedbt_server'; + return p.host && p.port && p.databaseName + ? `jdbc:gbasedbt-sqli://${p.host}:${p.port}/${p.databaseName}:INFORMIXSERVER=${informixServer}` + : ''; + }, + }, +}; + +export const sourceTypeLabels: Record = { + PROJECT_MYSQL: '项目 MySQL', + EXCEL: 'Excel 文件', + MYSQL: 'MySQL', + POSTGRESQL: 'PostgreSQL', + ORACLE: 'Oracle', + GAUSSDB_NATIVE: 'GaussDB', + GBASE_8A: 'GBase 8a', + GBASE_8S: 'GBase 8s', +}; + +export const accessModeLabels: Record = { + READ_ONLY: '只读', + READ_WRITE: '可编辑', +}; + +export const tableKindLabels: Record = { + SOURCE_TABLE: '表', + SOURCE_VIEW: '视图', + TABLE: '表', + VIEW: '视图', + EXCEL_SHEET: '工作表', + DERIVED_TABLE: '生成表', + MATERIALIZED_TABLE: '缓存表', +}; + +export const jobTypeLabels: Record = { + IMPORT: '导入', + SPLIT: '拆分', + MERGE: '合并', + DERIVE: '生成新表', + EXPORT: '导出', +}; + +export const jobStatusLabels: Record = { + SUCCESS: '完成', + FAILED: '失败', + RUNNING: '处理中', + PENDING: '等待中', + CREATED: '已创建', +}; + +export const relationTypeLabels: Record = { + SPLIT: '拆分生成', + MERGE: '合并生成', + DERIVE: '生成新表', +}; + +export function formatSourceType(value?: string) { + return value ? sourceTypeLabels[value] || value : '-'; +} + +export function formatAccessMode(value?: string) { + return value ? accessModeLabels[value] || value : '-'; +} + +export function formatTestStatus(value?: string) { + if (!value) return '未测试'; + return value === 'SUCCESS' ? '连接正常' : '连接失败'; +} + +export function formatTableKind(value?: string) { + return value ? tableKindLabels[value] || value : '-'; +} + +export function formatJobType(value?: string) { + return value ? jobTypeLabels[value] || value : value || '-'; +} + +export function formatJobStatus(value?: string) { + return value ? jobStatusLabels[value] || value : '-'; +} + +export function formatRelationType(value?: string) { + return value ? relationTypeLabels[value] || value : '关联'; +} + +export function formatFieldFlag(value?: null | number) { + return value === null || value === undefined || value === 1 ? '是' : '否'; +} + +export function mergeConfigJsonBySourceType( + sourceType: string, + configJson?: Record, +) { + const merged = { ...configJson }; + if (sourceType === 'GBASE_8S' && !merged.informixServer) { + merged.informixServer = 'gbasedbt_server'; + } + return merged; +} diff --git a/easyflow-ui-admin/app/src/views/datacenter/composables/use-connection-tree.ts b/easyflow-ui-admin/app/src/views/datacenter/composables/use-connection-tree.ts new file mode 100644 index 0000000..5e44a2d --- /dev/null +++ b/easyflow-ui-admin/app/src/views/datacenter/composables/use-connection-tree.ts @@ -0,0 +1,69 @@ +import type { Ref } from 'vue'; + +import { computed } from 'vue'; + +import { formatSourceType, formatTestStatus } from './datacenter-constants'; + +export interface TreeNode { + id: string; + label: string; + type: 'source' | 'table'; + icon?: string; + meta?: any; + children?: TreeNode[]; + isLeaf?: boolean; +} + +export function useConnectionTree(sources: Ref) { + const treeData = computed(() => { + return sources.value.map((source) => { + return { + id: `source-${source.id}`, + label: source.sourceName, + type: 'source' as const, + icon: getSourceIcon(source.sourceType), + meta: { + ...source, + }, + children: [], + }; + }); + }); + + function getSourceIcon(sourceType: string): string { + const iconMap: Record = { + MYSQL: 'db', + POSTGRESQL: 'db', + ORACLE: 'db', + GAUSSDB_NATIVE: 'db', + GBASE_8A: 'db', + GBASE_8S: 'db', + EXCEL: 'file', + PROJECT_MYSQL: 'db', + }; + return iconMap[sourceType] || 'db'; + } + + function parseNodeKey(key: string) { + const [type, ...idParts] = key.split('-'); + const id = idParts.join('-'); + return { + type: type as 'source' | 'table', + id: Number(id), + }; + } + + function getStatusColor(source: any): string { + if (!source.lastTestStatus) return 'muted'; + return source.lastTestStatus === 'SUCCESS' ? 'success' : 'danger'; + } + + return { + treeData, + parseNodeKey, + getSourceIcon, + getStatusColor, + formatSourceType, + formatTestStatus, + }; +} diff --git a/easyflow-ui-admin/app/src/views/datacenter/composables/use-datacenter-excel.ts b/easyflow-ui-admin/app/src/views/datacenter/composables/use-datacenter-excel.ts new file mode 100644 index 0000000..9e64b02 --- /dev/null +++ b/easyflow-ui-admin/app/src/views/datacenter/composables/use-datacenter-excel.ts @@ -0,0 +1,253 @@ +import type { Ref } from 'vue'; + +import { reactive, ref } from 'vue'; + +import { downloadFileFromBlob } from '@easyflow/utils'; + +import { ElMessage } from 'element-plus'; + +import { api } from '#/api/request'; + +export function useDatacenterExcel( + selectedSourceId: Ref, + selectedCatalogId: Ref, + selectedTableId: Ref, + reloadAll: () => Promise, +) { + const actionLoading = ref(false); + const pendingUploadFile = ref(null); + + const splitForm = reactive({ + splitMode: 'BY_ROW_COUNT', + rowBatchSize: 1000, + fieldName: '', + targetNamePrefix: '', + }); + + const mergeForm = reactive({ + mergeMode: 'VERTICAL', + tableIds: [] as number[], + targetTableName: '', + joinKey: '', + }); + + const deriveForm = reactive({ + targetTableName: '', + selectedColumnsText: '', + renameMappingsText: '{}', + filtersText: '[]', + }); + + const exportForm = reactive({ + fileName: '', + exportScope: 'TABLE', + }); + + function resetSplitForm() { + splitForm.splitMode = 'BY_ROW_COUNT'; + splitForm.rowBatchSize = 1000; + splitForm.fieldName = ''; + splitForm.targetNamePrefix = ''; + } + + function resetMergeForm() { + mergeForm.mergeMode = 'VERTICAL'; + mergeForm.tableIds = []; + mergeForm.targetTableName = ''; + mergeForm.joinKey = ''; + } + + function resetDeriveForm() { + deriveForm.targetTableName = ''; + deriveForm.selectedColumnsText = ''; + deriveForm.renameMappingsText = '{}'; + deriveForm.filtersText = '[]'; + } + + function resetExportForm() { + exportForm.fileName = ''; + exportForm.exportScope = 'TABLE'; + } + + async function handleImport() { + if (!pendingUploadFile.value) { + ElMessage.warning('请先选择 Excel 文件'); + return false; + } + actionLoading.value = true; + try { + const formData = new FormData(); + formData.append('file', pendingUploadFile.value); + await api.postFile('/api/v1/datacenterExcel/import', formData, { + timeout: 10 * 60 * 1000, + }); + ElMessage.success('Excel 已导入'); + pendingUploadFile.value = null; + await reloadAll(); + return true; + } finally { + actionLoading.value = false; + } + } + + async function handleSplit() { + if (!selectedTableId.value) return false; + if ( + splitForm.splitMode === 'BY_ROW_COUNT' && + Number(splitForm.rowBatchSize) <= 0 + ) { + ElMessage.warning('每份行数必须大于 0'); + return false; + } + if (splitForm.splitMode === 'BY_FIELD_VALUE' && !splitForm.fieldName) { + ElMessage.warning('请选择拆分字段'); + return false; + } + actionLoading.value = true; + try { + await api.post('/api/v1/datacenterExcel/split', { + datasetRef: { tableId: selectedTableId.value }, + sourceId: selectedSourceId.value, + catalogId: selectedCatalogId.value, + splitMode: splitForm.splitMode, + rowBatchSize: Number(splitForm.rowBatchSize), + fieldName: splitForm.fieldName, + targetNamePrefix: splitForm.targetNamePrefix, + }); + ElMessage.success('拆分已完成'); + await reloadAll(); + return true; + } finally { + actionLoading.value = false; + } + } + + async function handleMerge() { + if (mergeForm.tableIds.length < 2) { + ElMessage.warning('请至少选择两个已接入表'); + return false; + } + if (!mergeForm.targetTableName.trim()) { + ElMessage.warning('请输入新表名称'); + return false; + } + if (mergeForm.mergeMode === 'HORIZONTAL' && !mergeForm.joinKey.trim()) { + ElMessage.warning('横向合并需要填写关联字段'); + return false; + } + actionLoading.value = true; + try { + await api.post('/api/v1/datacenterExcel/merge', { + datasetRefs: mergeForm.tableIds.map((tableId) => ({ tableId })), + mergeMode: mergeForm.mergeMode, + targetTableName: mergeForm.targetTableName.trim(), + joinKey: mergeForm.joinKey.trim(), + }); + ElMessage.success('合并已完成'); + await reloadAll(); + return true; + } finally { + actionLoading.value = false; + } + } + + async function handleDerive() { + if (!selectedTableId.value) return false; + if (!deriveForm.targetTableName.trim()) { + ElMessage.warning('请输入新表名称'); + return false; + } + const selectedColumns = deriveForm.selectedColumnsText + ? deriveForm.selectedColumnsText + .split(',') + .map((item) => item.trim()) + .filter(Boolean) + : []; + + let renameMappings: Record; + let filters: any[]; + try { + renameMappings = deriveForm.renameMappingsText + ? JSON.parse(deriveForm.renameMappingsText) + : {}; + filters = deriveForm.filtersText + ? JSON.parse(deriveForm.filtersText) + : []; + } catch { + ElMessage.error('生成参数格式不正确，请检查输入内容'); + return false; + } + + actionLoading.value = true; + try { + await api.post('/api/v1/datacenterExcel/derive', { + datasetRef: { tableId: selectedTableId.value }, + targetTableName: deriveForm.targetTableName.trim(), + selectedColumns, + renameMappings, + filters, + }); + ElMessage.success('新表已生成'); + await reloadAll(); + return true; + } finally { + actionLoading.value = false; + } + } + + async function handleExport(loadTableRuntime: () => Promise) { + if (exportForm.exportScope === 'TABLE' && !selectedTableId.value) { + ElMessage.warning('请先选择已接入表'); + return false; + } + actionLoading.value = true; + try { + const res = await api.post('/api/v1/datacenterExcel/export', { + sourceId: + exportForm.exportScope === 'WORKBOOK' + ? selectedSourceId.value + : undefined, + catalogId: + exportForm.exportScope === 'WORKBOOK' + ? selectedCatalogId.value + : undefined, + datasetRefs: + exportForm.exportScope === 'TABLE' && selectedTableId.value + ? [{ tableId: selectedTableId.value }] + : [], + fileName: exportForm.fileName.trim(), + }); + const blob = await api.download('/api/v1/datacenterExcel/download', { + params: { jobId: res.data.id }, + }); + downloadFileFromBlob({ + fileName: + res.data.fileName || exportForm.fileName || 'datacenter-export.xlsx', + source: blob, + }); + ElMessage.success('导出文件已生成'); + await loadTableRuntime(); + return true; + } finally { + actionLoading.value = false; + } + } + + return { + actionLoading, + pendingUploadFile, + splitForm, + mergeForm, + deriveForm, + exportForm, + resetSplitForm, + resetMergeForm, + resetDeriveForm, + resetExportForm, + handleImport, + handleSplit, + handleMerge, + handleDerive, + handleExport, + }; +} diff --git a/easyflow-ui-admin/app/src/views/datacenter/composables/use-datacenter-sources.ts b/easyflow-ui-admin/app/src/views/datacenter/composables/use-datacenter-sources.ts new file mode 100644 index 0000000..f7690e3 --- /dev/null +++ b/easyflow-ui-admin/app/src/views/datacenter/composables/use-datacenter-sources.ts @@ -0,0 +1,190 @@ +import { computed, ref } from 'vue'; + +import { api } from '#/api/request'; + +import { mergeConfigJsonBySourceType } from './datacenter-constants'; + +export function useDatacenterSources() { + const sources = ref([]); + const selectedSourceId = ref(null); + const loading = ref(false); + const saving = ref(false); + const testing = ref(false); + + const selectedSource = computed( + () => + sources.value.find((item) => item.id === selectedSourceId.value) || null, + ); + + function normalizeSource(record: any, previous?: any) { + return { + ...previous, + ...record, + runtimeUnavailable: Boolean( + record?.runtimeUnavailable ?? previous?.runtimeUnavailable, + ), + }; + } + + function syncSelectedSourceAfterMutation() { + if (sources.value.length === 0) { + selectedSourceId.value = null; + return; + } + if ( + !selectedSourceId.value || + !sources.value.some((item) => item.id === selectedSourceId.value) + ) { + selectedSourceId.value = sources.value[0].id; + } + } + + function upsertSource(record?: any) { + if (!record?.id) return; + const previous = sources.value.find((item) => item.id === record.id); + const next = dedupeBuiltinSources( + sources.value.some((item) => item.id === record.id) + ? sources.value.map((item) => + item.id === record.id ? normalizeSource(record, previous) : item, + ) + : [normalizeSource(record), ...sources.value], + ); + sources.value = next; + selectedSourceId.value = record.id; + } + + function markSourceRuntimeUnavailable(sourceId?: null | number | string) { + if (!sourceId) return; + sources.value = sources.value.map((item) => + String(item.id) === String(sourceId) + ? { ...item, runtimeUnavailable: true } + : item, + ); + } + + function clearSourceRuntimeUnavailable(sourceId?: null | number | string) { + if (!sourceId) return; + sources.value = sources.value.map((item) => + String(item.id) === String(sourceId) + ? { ...item, runtimeUnavailable: false } + : item, + ); + } + + function removeSourceFromState(sourceId: number) { + sources.value = sources.value.filter((item) => item.id !== sourceId); + syncSelectedSourceAfterMutation(); + } + + function dedupeBuiltinSources(records: any[]) { + const builtinSourceTypes = new Set(['PROJECT_MYSQL']); + const seen = new Set(); + + return (records || []).filter((item) => { + const uniqueKey = `${item.sourceType}:${item.sourceName}`; + if (!item.builtinFlag || !builtinSourceTypes.has(item.sourceType)) { + return true; + } + if (seen.has(uniqueKey)) { + return false; + } + seen.add(uniqueKey); + return true; + }); + } + + async function loadSources() { + const res = await api.get('/api/v1/datacenterSource/page', { + params: { pageNumber: 1, pageSize: 200 }, + }); + const previousMap = new Map( + sources.value.map((item) => [String(item.id), item]), + ); + sources.value = dedupeBuiltinSources( + (res.data?.records || []).map((record: any) => + normalizeSource(record, previousMap.get(String(record.id))), + ), + ); + syncSelectedSourceAfterMutation(); + } + + async function saveSource(form: Record) { + saving.value = true; + try { + const payload = { + ...form, + configJson: { + ...mergeConfigJsonBySourceType(form.sourceType, form.configJson), + password: form.password || undefined, + }, + }; + const res = await api.post('/api/v1/datacenterSource/save', payload); + upsertSource(res.data); + return res.data; + } finally { + saving.value = false; + } + } + + async function testConnection(target: Record) { + testing.value = true; + try { + const payload = { + ...target, + configJson: { + ...mergeConfigJsonBySourceType(target.sourceType, target.configJson), + password: target.password || undefined, + }, + }; + const res = await api.post( + '/api/v1/datacenterSource/testConnection', + payload, + ); + return res.data; + } finally { + testing.value = false; + } + } + + async function quickTest(row: any) { + testing.value = true; + try { + const res = await api.post( + '/api/v1/datacenterSource/testConnection', + row, + ); + clearSourceRuntimeUnavailable(row?.id); + await loadSources(); + return res.data; + } catch (error) { + markSourceRuntimeUnavailable(row?.id); + throw error; + } finally { + testing.value = false; + } + } + + async function removeSource(sourceId: number) { + await api.post('/api/v1/datacenterSource/remove', { sourceId }); + removeSourceFromState(sourceId); + } + + return { + sources, + selectedSourceId, + selectedSource, + loading, + saving, + testing, + loadSources, + removeSource, + removeSourceFromState, + saveSource, + syncSelectedSourceAfterMutation, + testConnection, + quickTest, + upsertSource, + markSourceRuntimeUnavailable, + clearSourceRuntimeUnavailable, + }; +} diff --git a/easyflow-ui-admin/app/src/views/datacenter/composables/use-datacenter-tables.ts b/easyflow-ui-admin/app/src/views/datacenter/composables/use-datacenter-tables.ts new file mode 100644 index 0000000..1783c6c --- /dev/null +++ b/easyflow-ui-admin/app/src/views/datacenter/composables/use-datacenter-tables.ts @@ -0,0 +1,334 @@ +import type { Ref } from 'vue'; + +import { computed, ref } from 'vue'; + +import { ElMessage } from 'element-plus'; + +import { requestClient } from '#/api/request'; + +interface DatacenterTableRuntimeOptions { + clearSourceRuntimeUnavailable?: (sourceId?: null | number | string) => void; + markSourceRuntimeUnavailable?: (sourceId?: null | number | string) => void; +} + +export function useDatacenterTables( + selectedSourceId: Ref, + options: DatacenterTableRuntimeOptions = {}, +) { + const SOURCE_MISSING_MESSAGE = '连接不存在'; + const SOURCE_UNAVAILABLE_MESSAGE = '当前连接不可用，请检查连接配置后重试'; + const catalogs = ref([]); + const sourceTables = ref([]); + const managedTables = ref([]); + const schema = ref(null); + const previewRows = ref([]); + const jobs = ref([]); + const sourceUnavailable = ref(false); + const selectedCatalogId = ref(null); + const selectedTableId = ref(null); + const previewLoading = ref(false); + + const selectedCatalog = computed( + () => + catalogs.value.find((item) => item.id === selectedCatalogId.value) || + null, + ); + + const selectedTable = computed( + () => + managedTables.value.find((item) => item.id === selectedTableId.value) || + null, + ); + + function isSourceUnavailableError(error: any) { + const responseData = error?.response?.data ?? {}; + const message = String(responseData?.message ?? error?.message ?? ''); + return message.includes(SOURCE_UNAVAILABLE_MESSAGE); + } + + function isSourceMissingError(error: any) { + const responseData = error?.response?.data ?? {}; + const message = String(responseData?.message ?? error?.message ?? ''); + return message.includes(SOURCE_MISSING_MESSAGE); + } + + function markSourceUnavailable() { + options.markSourceRuntimeUnavailable?.(selectedSourceId.value); + catalogs.value = []; + sourceTables.value = []; + managedTables.value = []; + schema.value = null; + previewRows.value = []; + jobs.value = []; + selectedCatalogId.value = null; + selectedTableId.value = null; + sourceUnavailable.value = true; + } + + async function loadCatalogs() { + if (!selectedSourceId.value) { + catalogs.value = []; + selectedCatalogId.value = null; + sourceUnavailable.value = false; + return true; + } + try { + const data = await requestClient.get( + '/api/v1/datacenterSource/catalogs', + { + params: { sourceId: selectedSourceId.value }, + }, + ); + options.clearSourceRuntimeUnavailable?.(selectedSourceId.value); + catalogs.value = data || []; + sourceUnavailable.value = false; + if (catalogs.value.length === 0) { + selectedCatalogId.value = null; + return true; + } + if ( + !selectedCatalogId.value || + !catalogs.value.some((item) => item.id === selectedCatalogId.value) + ) { + selectedCatalogId.value = catalogs.value[0].id; + } + return true; + } catch (error) { + if (isSourceMissingError(error)) { + markSourceUnavailable(); + return false; + } + if (!isSourceUnavailableError(error)) { + throw error; + } + markSourceUnavailable(); + return false; + } + } + + async function loadSourceTables() { + if (!selectedSourceId.value) { + sourceTables.value = []; + return; + } + try { + const data = await requestClient.get('/api/v1/datacenterSource/tables', { + params: { + sourceId: selectedSourceId.value, + catalogName: selectedCatalog.value?.catalogName, + }, + }); + options.clearSourceRuntimeUnavailable?.(selectedSourceId.value); + sourceTables.value = data || []; + sourceUnavailable.value = false; + } catch (error) { + if (!isSourceUnavailableError(error) && !isSourceMissingError(error)) { + throw error; + } + markSourceUnavailable(); + } + } + + async function loadManagedTables() { + if (!selectedSourceId.value) { + managedTables.value = []; + selectedTableId.value = null; + return; + } + try { + const data = await requestClient.get( + '/api/v1/datacenterDataset/managedTables', + { + params: { + sourceId: selectedSourceId.value, + catalogId: selectedCatalogId.value, + }, + }, + ); + managedTables.value = data || []; + } catch (error) { + if (!isSourceUnavailableError(error) && !isSourceMissingError(error)) { + throw error; + } + markSourceUnavailable(); + return; + } + if (managedTables.value.length === 0) { + selectedTableId.value = null; + return; + } + if ( + !selectedTableId.value || + !managedTables.value.some((item) => item.id === selectedTableId.value) + ) { + selectedTableId.value = managedTables.value[0].id; + } + } + + async function loadTableRuntime() { + if (!selectedTableId.value) { + schema.value = null; + previewRows.value = []; + jobs.value = []; + return; + } + previewLoading.value = true; + try { + const [schemaRes, previewRes, jobsRes] = await Promise.allSettled([ + requestClient.get('/api/v1/datacenterDataset/schema', { + params: { tableId: selectedTableId.value }, + }), + requestClient.post('/api/v1/datacenterDataset/queryPage', { + datasetRef: { tableId: selectedTableId.value }, + pageNumber: 1, + pageSize: 10, + }), + requestClient.get('/api/v1/datacenterExcel/job/list', { + params: { + sourceId: selectedSourceId.value, + tableId: selectedTableId.value, + }, + }), + ]); + if (schemaRes.status === 'fulfilled') { + schema.value = schemaRes.value; + } else if ( + isSourceUnavailableError(schemaRes.reason) || + isSourceMissingError(schemaRes.reason) + ) { + markSourceUnavailable(); + return; + } else { + throw schemaRes.reason; + } + if (jobsRes.status === 'fulfilled') { + jobs.value = jobsRes.value || []; + } else if ( + isSourceUnavailableError(jobsRes.reason) || + isSourceMissingError(jobsRes.reason) + ) { + markSourceUnavailable(); + return; + } else { + throw jobsRes.reason; + } + if (previewRes.status === 'fulfilled') { + options.clearSourceRuntimeUnavailable?.(selectedSourceId.value); + previewRows.value = previewRes.value?.records || []; + sourceUnavailable.value = false; + } else if ( + isSourceUnavailableError(previewRes.reason) || + isSourceMissingError(previewRes.reason) + ) { + markSourceUnavailable(); + } else { + throw previewRes.reason; + } + } finally { + previewLoading.value = false; + } + } + + async function syncSourceContext() { + const catalogAvailable = await loadCatalogs(); + if (catalogAvailable) { + await Promise.all([loadSourceTables(), loadManagedTables()]); + } else { + sourceTables.value = []; + await loadManagedTables(); + } + await loadTableRuntime(); + } + + async function registerTable(row: any) { + const data = await requestClient.get( + '/api/v1/datacenterSource/tableDetail', + { + params: { + sourceId: selectedSourceId.value, + catalogName: selectedCatalog.value?.catalogName, + tableName: row.tableName, + register: true, + }, + }, + ); + ElMessage.success('已接入数据中心'); + await loadManagedTables(); + selectedTableId.value = data?.table?.id || selectedTableId.value; + await loadTableRuntime(); + } + + async function batchRegisterTables(rows: any[]) { + const tableNames = (rows || []) + .map((row) => row?.tableName) + .filter(Boolean); + if (tableNames.length === 0) return; + await requestClient.post('/api/v1/datacenterSource/registerBatch', { + sourceId: selectedSourceId.value, + catalogName: selectedCatalog.value?.catalogName, + tableNames, + }); + ElMessage.success(`已接入 ${tableNames.length} 张表`); + await loadManagedTables(); + } + + async function batchRemoveTables(rows: any[]) { + const tableIds = (rows || []).map((row) => row?.id).filter(Boolean); + if (tableIds.length === 0) return; + await requestClient.post('/api/v1/datacenterDataset/removeBatch', { + tableIds, + }); + ElMessage.success(`已去除 ${tableIds.length} 张表`); + if (selectedTableId.value && tableIds.includes(selectedTableId.value)) { + selectedTableId.value = null; + schema.value = null; + previewRows.value = []; + jobs.value = []; + } + await Promise.all([loadSourceTables(), loadManagedTables()]); + } + + async function saveDescriptions(payload: { + fields?: Array<{ fieldDesc: string; fieldId: number | string }>; + tableDesc?: string; + tableId: number | string; + }) { + const data = await requestClient.post( + '/api/v1/datacenterDataset/saveDescriptions', + { + fields: payload.fields || [], + tableDesc: payload.tableDesc ?? '', + tableId: payload.tableId, + }, + ); + await loadManagedTables(); + if (selectedTableId.value === payload.tableId) { + schema.value = data || null; + } + return data; + } + + return { + catalogs, + sourceTables, + managedTables, + schema, + previewRows, + jobs, + sourceUnavailable, + selectedCatalogId, + selectedTableId, + selectedCatalog, + selectedTable, + previewLoading, + loadCatalogs, + loadSourceTables, + loadManagedTables, + loadTableRuntime, + syncSourceContext, + registerTable, + batchRegisterTables, + batchRemoveTables, + saveDescriptions, + }; +} diff --git a/easyflow-ui-admin/app/src/views/datacenter/composables/use-source-form.ts b/easyflow-ui-admin/app/src/views/datacenter/composables/use-source-form.ts new file mode 100644 index 0000000..11a67a8 --- /dev/null +++ b/easyflow-ui-admin/app/src/views/datacenter/composables/use-source-form.ts @@ -0,0 +1,188 @@ +import type { FormRules } from 'element-plus'; + +import { computed, reactive, ref, watch } from 'vue'; + +import { + mergeConfigJsonBySourceType, + sourceConnectionDefaults, + sourceTypeLabels, +} from './datacenter-constants'; + +export function useSourceForm() { + const lastGeneratedJdbcUrl = ref(''); + const lastGeneratedDriverClassName = ref(''); + + const form = reactive>({ + id: undefined, + sourceName: '', + sourceCode: '', + sourceType: 'MYSQL', + accessMode: 'READ_ONLY', + driverClassName: '', + jdbcUrl: '', + host: '', + port: undefined, + databaseName: '', + schemaName: '', + username: '', + password: '', + builtinFlag: 0, + configJson: {}, + }); + + const rules: FormRules = { + sourceName: [ + { required: true, message: '请输入连接名称', trigger: 'blur' }, + ], + sourceType: [ + { required: true, message: '请选择连接类型', trigger: 'change' }, + ], + databaseName: [ + { + trigger: ['blur', 'change'], + validator: (_rule, value, callback) => { + if (!sourceConnectionDefaults[form.sourceType]) { + callback(); + return; + } + if (String(value || '').trim()) { + callback(); + return; + } + callback(new Error('请输入库名')); + }, + }, + ], + }; + + const selectedTypeDefaults = computed( + () => sourceConnectionDefaults[form.sourceType] || null, + ); + + const supportsExternalConnection = computed(() => + Boolean(selectedTypeDefaults.value), + ); + + const connectionTypeLabel = computed(() => + form.sourceType ? sourceTypeLabels[form.sourceType] || form.sourceType : '', + ); + + const connectionHelpText = computed(() => { + if (form.sourceType === 'EXCEL') { + return 'Excel 连接只需要名称，文件导入后会自动生成表。'; + } + if (!supportsExternalConnection.value) { + return '系统内置连接由平台自动维护，无需填写地址。'; + } + return `填写主机、库名和账号即可，端口与连接地址会按 ${connectionTypeLabel.value} 自动补全。`; + }); + + function generateJdbcUrl(payload: Record) { + return ( + sourceConnectionDefaults[payload.sourceType]?.buildJdbcUrl(payload) || '' + ); + } + + function syncGeneratedJdbcUrl(force = false) { + const generated = generateJdbcUrl(form); + if (!generated) { + if ( + force || + !form.jdbcUrl || + form.jdbcUrl === lastGeneratedJdbcUrl.value + ) { + form.jdbcUrl = ''; + } + lastGeneratedJdbcUrl.value = ''; + return; + } + if ( + force || + !form.jdbcUrl || + form.jdbcUrl === lastGeneratedJdbcUrl.value + ) { + form.jdbcUrl = generated; + lastGeneratedJdbcUrl.value = generated; + } else if (form.jdbcUrl === generated) { + lastGeneratedJdbcUrl.value = generated; + } + } + + function applySourceTypeDefaults(force = false) { + const defaults = selectedTypeDefaults.value; + form.configJson = mergeConfigJsonBySourceType( + form.sourceType, + form.configJson, + ); + if (!defaults) { + if (force) { + form.port = undefined; + form.driverClassName = ''; + form.jdbcUrl = ''; + lastGeneratedDriverClassName.value = ''; + lastGeneratedJdbcUrl.value = ''; + } + return; + } + if (force || !form.port) { + form.port = defaults.defaultPort; + } + if ( + force || + !form.driverClassName || + form.driverClassName === lastGeneratedDriverClassName.value + ) { + form.driverClassName = defaults.defaultDriver; + lastGeneratedDriverClassName.value = defaults.defaultDriver; + } + syncGeneratedJdbcUrl(force); + } + + function resetForm(row?: any) { + form.id = row?.id; + form.sourceName = row?.sourceName || ''; + form.sourceCode = row?.sourceCode || ''; + form.sourceType = row?.sourceType || 'MYSQL'; + form.accessMode = row?.accessMode || 'READ_ONLY'; + form.driverClassName = row?.driverClassName || ''; + form.jdbcUrl = row?.jdbcUrl || ''; + form.host = row?.host || ''; + form.port = row?.port; + form.databaseName = row?.databaseName || ''; + form.schemaName = row?.schemaName || ''; + form.username = row?.username || ''; + form.password = ''; + form.builtinFlag = row?.builtinFlag || 0; + form.configJson = mergeConfigJsonBySourceType( + form.sourceType, + row?.configJson || {}, + ); + lastGeneratedDriverClassName.value = ''; + lastGeneratedJdbcUrl.value = ''; + applySourceTypeDefaults(false); + } + + watch( + () => form.sourceType, + () => applySourceTypeDefaults(true), + ); + + watch( + () => [ + form.host, + form.port, + form.databaseName, + form.configJson?.informixServer, + form.configJson?.serviceName, + ], + () => syncGeneratedJdbcUrl(false), + ); + + return { + form, + rules, + supportsExternalConnection, + connectionHelpText, + resetForm, + }; +}