From 02b8fdd3ae9604d32fb0ea120df278186bf72398 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=99=88=E5=AD=90=E9=BB=98?= <925456043@qq.com> Date: Tue, 25 Aug 2026 01:00:49 +0800 Subject: [PATCH] =?UTF-8?q?feat:=20=E6=96=B0=E5=A2=9E=20SQL=20=E8=81=94?= =?UTF-8?q?=E9=82=A6=E6=9F=A5=E8=AF=A2=E4=B8=8E=E6=95=B0=E6=8D=AE=E5=BA=93?= =?UTF-8?q?=E9=80=82=E9=85=8D=E5=BA=95=E5=BA=A7?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 提供统一编译、逻辑表映射与单源/联邦自动路由 - 增加有界执行、查询生命周期、统计成本优化与执行分析 - 内置 MySQL 与 PostgreSQL JDBC 适配和统计采集 --- easy-agents-bom/pom.xml | 28 + easy-agents-federation-sql/README.md | 156 ++ .../pom.xml | 52 + .../adapter/jdbc/JdbcFailureClassifier.java | 54 + .../jdbc/JdbcFederationFragmentExecutor.java | 200 ++ .../jdbc/JdbcFederationFragmentExplainer.java | 307 +++ .../jdbc/JdbcFederationResultCursor.java | 446 ++++ .../JdbcFederationSqlAdapterProvider.java | 212 ++ .../JdbcFederationStatisticsCollector.java | 641 +++++ .../adapter/jdbc/JdbcSchemaDefinition.java | 38 + .../MysqlCaseInsensitiveColumnSchema.java | 308 +++ ...n.sql.adapter.FederationSqlAdapterProvider | 1 + .../jdbc/JdbcDialectSelectionTest.java | 140 ++ .../jdbc/JdbcFederatedQueryEngineTest.java | 774 ++++++ .../JdbcFederationFragmentExecutorTest.java | 756 ++++++ .../JdbcFederationFragmentExplainerTest.java | 152 ++ .../jdbc/JdbcFederationSqlEngineTest.java | 1195 +++++++++ ...bcFederationStatisticsIntegrationTest.java | 347 +++ .../MysqlCaseInsensitiveColumnSchemaTest.java | 185 ++ .../easy-agents-federation-sql-core/pom.xml | 31 + .../sql/adapter/AdapterCompatibility.java | 23 + .../adapter/AdapterCompatibilityStatus.java | 13 + .../sql/adapter/AdapterDialectContext.java | 16 + .../federation/sql/adapter/AdapterHints.java | 28 + .../sql/adapter/AdapterSchemaContext.java | 25 + .../adapter/FederationSqlAdapterProvider.java | 167 ++ .../adapter/FederationSqlAdapterRegistry.java | 94 + ...FederationStatisticsCollectionContext.java | 40 + .../FederationStatisticsCollector.java | 27 + .../sql/api/FederationCleanupMetrics.java | 35 + .../sql/api/FederationSqlEngine.java | 88 + .../sql/api/FederationSqlEngines.java | 277 +++ .../sql/api/FederationSqlErrorCode.java | 71 + .../sql/api/FederationSqlException.java | 44 + .../federation/sql/api/SqlCompletionItem.java | 31 + .../federation/sql/api/SqlCompletionKind.java | 23 + .../sql/api/SqlCompletionRequest.java | 29 + .../sql/api/SqlCompletionResult.java | 27 + .../sql/api/SqlExecutionContext.java | 46 + .../federation/sql/api/SqlQueryCommand.java | 155 ++ .../compile/FederationFragmentExplain.java | 93 + .../sql/compile/FederationSqlPlan.java | 191 ++ .../sql/compile/FederationSqlPolicy.java | 27 + .../sql/compile/SqlCompileRequest.java | 108 + .../sql/compile/SqlExplainLevel.java | 13 + .../sql/compile/SqlExplainRequest.java | 54 + .../sql/compile/SqlExplainResult.java | 165 ++ .../sql/compile/SqlPolicyContext.java | 22 + .../sql/execute/FederationColumn.java | 21 + .../sql/execute/FederationExecutionGuard.java | 67 + .../execute/FederationExecutionObserver.java | 41 + .../FederationFragmentExecutionContext.java | 122 + .../execute/FederationFragmentExecutor.java | 16 + .../FederationFragmentExplainContext.java | 74 + .../execute/FederationFragmentExplainer.java | 16 + .../execute/FederationFragmentMetrics.java | 51 + .../FederationLocalOperatorMetrics.java | 19 + .../execute/FederationPhysicalExplain.java | 59 + .../FederationQueryAdmissionController.java | 98 + .../FederationQueryMetricsSnapshot.java | 152 ++ .../sql/execute/FederationQueryPermit.java | 14 + .../sql/execute/FederationResultCursor.java | 84 + ...calFederationQueryAdmissionController.java | 160 ++ .../sql/execute/QueryAdmissionRequest.java | 49 + .../federation/sql/execute/QueryId.java | 30 + .../sql/execute/SqlExecutionOptions.java | 40 + .../federation/sql/execute/SqlParameter.java | 112 + .../sql/execute/StatementLifecycle.java | 41 + .../FederationColumnStatistics.java | 32 + .../federation/FederationCostEstimate.java | 126 + .../federation/FederationExecutionPolicy.java | 80 + .../federation/FederationFragmentPlan.java | 88 + .../federation/FederationJoinAlgorithm.java | 10 + .../FederationJoinOptimization.java | 99 + .../FederationJoinSelectionReason.java | 16 + .../FederationLogicalTableDefinition.java | 65 + .../sql/federation/FederationQueryMode.java | 11 + .../FederationQueryScopeDefinition.java | 306 +++ .../FederationSourceBindingDefinition.java | 87 + .../FederationSourceRuntimeIdentity.java | 36 + .../FederationStatisticsSnapshot.java | 235 ++ .../FederationStatisticsStatus.java | 19 + .../federation/FederationTableStatistics.java | 90 + .../FederationTableStatisticsProvider.java | 56 + .../AdapterFederationStatisticsProvider.java | 510 ++++ .../sql/runtime/BoundedPlanCache.java | 719 ++++++ .../runtime/CalciteFederationSqlCompiler.java | 2157 +++++++++++++++++ .../sql/runtime/CalciteSqlCompleter.java | 423 ++++ .../CancellationAwareFederationCursor.java | 26 + .../runtime/CompiledFederationFragment.java | 16 + .../sql/runtime/CompositeQueryResources.java | 54 + .../DefaultFederationSourceManager.java | 846 +++++++ .../runtime/DefaultFederationSqlEngine.java | 1289 ++++++++++ .../sql/runtime/DefaultFederationSqlPlan.java | 516 ++++ .../EnumerableFederationBudgetRel.java | 61 + .../sql/runtime/FederatedResultCursor.java | 275 +++ .../runtime/FederationBudgetEnumerable.java | 75 + .../sql/runtime/FederationDataContext.java | 58 + .../runtime/FederationExecutionSession.java | 516 ++++ .../sql/runtime/FederationFragmentTable.java | 70 + .../FederationQueryMetricsTracker.java | 437 ++++ .../runtime/FederationQueryScopeSnapshot.java | 228 ++ .../runtime/FederationStatisticsMetadata.java | 663 +++++ .../FederationStatisticsTableScan.java | 84 + .../sql/runtime/LogicalTableSqlResolver.java | 283 +++ .../ManagedFederationResultCursor.java | 379 +++ .../MetricsFederationResultCursor.java | 121 + .../NodeMemoryAdmissionController.java | 132 + .../federation/sql/runtime/PlanCacheKey.java | 58 + .../runtime/QueryCancellationRegistry.java | 783 ++++++ .../federation/sql/runtime/QueryDeadline.java | 127 + .../sql/runtime/SourceCatalogSnapshot.java | 20 + .../federation/sql/runtime/SourceRuntime.java | 196 ++ .../sql/runtime/TypedSqlDynamicParam.java | 47 + .../sql/source/ActiveSourceState.java | 58 + .../sql/source/ExternalSchemaDefinition.java | 42 + .../source/FederationDataSourceHandle.java | 29 + .../source/FederationDataSourceHandles.java | 68 + .../source/FederationDataSourceResolver.java | 16 + .../source/FederationSchemaDefinition.java | 24 + .../source/FederationSourceDefinition.java | 112 + .../sql/source/FederationSourceManager.java | 95 + .../sql/source/FederationSourceState.java | 30 + .../source/FederationSourceStateProvider.java | 48 + .../sql/source/FederationSourceView.java | 21 + .../sql/source/KnownJdbcDriver.java | 118 + .../sql/source/PreparedSourceRuntime.java | 23 + .../sql/source/RuntimeFingerprint.java | 45 + .../sql/source/SourceApplyOptions.java | 27 + .../sql/source/SourceApplyResult.java | 17 + .../sql/source/SourceApplyStatus.java | 15 + .../federation/sql/source/SourceId.java | 30 + .../sql/source/SourceProbeResult.java | 17 + .../sql/source/SourceRemoveResult.java | 9 + .../sql/source/SourceRuntimeStatus.java | 15 + .../sql/source/SourceSnapshotResult.java | 12 + .../sql/source/SourceStateSubscription.java | 14 + .../sql/source/SourceTombstone.java | 70 + .../api/SqlQueryCommandSerializationTest.java | 328 +++ ...ederationQueryAdmissionControllerTest.java | 61 + .../FederationStatisticsSnapshotTest.java | 158 ++ ...apterFederationStatisticsProviderTest.java | 369 +++ .../sql/runtime/BoundedPlanCacheTest.java | 741 ++++++ .../CalciteFederationSqlCompilerTest.java | 1108 +++++++++ .../DefaultFederationSourceManagerTest.java | 889 +++++++ .../FederationQueryMetricsTrackerTest.java | 93 + .../runtime/LogicalTableSqlResolverTest.java | 114 + .../ManagedFederationResultCursorTest.java | 164 ++ .../NodeMemoryAdmissionControllerTest.java | 180 ++ .../QueryCancellationRegistryTest.java | 488 ++++ .../sql/runtime/QueryDeadlineTest.java | 34 + .../FederationSourceDefinitionTest.java | 141 ++ .../sql/source/KnownJdbcDriverTest.java | 83 + easy-agents-federation-sql/pom.xml | 21 + pom.xml | 31 + 155 files changed, 27704 insertions(+) create mode 100644 easy-agents-federation-sql/README.md create mode 100644 easy-agents-federation-sql/easy-agents-federation-sql-adapter-jdbc/pom.xml create mode 100644 easy-agents-federation-sql/easy-agents-federation-sql-adapter-jdbc/src/main/java/com/easyagents/federation/sql/adapter/jdbc/JdbcFailureClassifier.java create mode 100644 easy-agents-federation-sql/easy-agents-federation-sql-adapter-jdbc/src/main/java/com/easyagents/federation/sql/adapter/jdbc/JdbcFederationFragmentExecutor.java create mode 100644 easy-agents-federation-sql/easy-agents-federation-sql-adapter-jdbc/src/main/java/com/easyagents/federation/sql/adapter/jdbc/JdbcFederationFragmentExplainer.java create mode 100644 easy-agents-federation-sql/easy-agents-federation-sql-adapter-jdbc/src/main/java/com/easyagents/federation/sql/adapter/jdbc/JdbcFederationResultCursor.java create mode 100644 easy-agents-federation-sql/easy-agents-federation-sql-adapter-jdbc/src/main/java/com/easyagents/federation/sql/adapter/jdbc/JdbcFederationSqlAdapterProvider.java create mode 100644 easy-agents-federation-sql/easy-agents-federation-sql-adapter-jdbc/src/main/java/com/easyagents/federation/sql/adapter/jdbc/JdbcFederationStatisticsCollector.java create mode 100644 easy-agents-federation-sql/easy-agents-federation-sql-adapter-jdbc/src/main/java/com/easyagents/federation/sql/adapter/jdbc/JdbcSchemaDefinition.java create mode 100644 easy-agents-federation-sql/easy-agents-federation-sql-adapter-jdbc/src/main/java/com/easyagents/federation/sql/adapter/jdbc/MysqlCaseInsensitiveColumnSchema.java create mode 100644 easy-agents-federation-sql/easy-agents-federation-sql-adapter-jdbc/src/main/resources/META-INF/services/com.easyagents.federation.sql.adapter.FederationSqlAdapterProvider create mode 100644 easy-agents-federation-sql/easy-agents-federation-sql-adapter-jdbc/src/test/java/com/easyagents/federation/sql/adapter/jdbc/JdbcDialectSelectionTest.java create mode 100644 easy-agents-federation-sql/easy-agents-federation-sql-adapter-jdbc/src/test/java/com/easyagents/federation/sql/adapter/jdbc/JdbcFederatedQueryEngineTest.java create mode 100644 easy-agents-federation-sql/easy-agents-federation-sql-adapter-jdbc/src/test/java/com/easyagents/federation/sql/adapter/jdbc/JdbcFederationFragmentExecutorTest.java create mode 100644 easy-agents-federation-sql/easy-agents-federation-sql-adapter-jdbc/src/test/java/com/easyagents/federation/sql/adapter/jdbc/JdbcFederationFragmentExplainerTest.java create mode 100644 easy-agents-federation-sql/easy-agents-federation-sql-adapter-jdbc/src/test/java/com/easyagents/federation/sql/adapter/jdbc/JdbcFederationSqlEngineTest.java create mode 100644 easy-agents-federation-sql/easy-agents-federation-sql-adapter-jdbc/src/test/java/com/easyagents/federation/sql/adapter/jdbc/JdbcFederationStatisticsIntegrationTest.java create mode 100644 easy-agents-federation-sql/easy-agents-federation-sql-adapter-jdbc/src/test/java/com/easyagents/federation/sql/adapter/jdbc/MysqlCaseInsensitiveColumnSchemaTest.java create mode 100644 easy-agents-federation-sql/easy-agents-federation-sql-core/pom.xml create mode 100644 easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/adapter/AdapterCompatibility.java create mode 100644 easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/adapter/AdapterCompatibilityStatus.java create mode 100644 easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/adapter/AdapterDialectContext.java create mode 100644 easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/adapter/AdapterHints.java create mode 100644 easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/adapter/AdapterSchemaContext.java create mode 100644 easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/adapter/FederationSqlAdapterProvider.java create mode 100644 easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/adapter/FederationSqlAdapterRegistry.java create mode 100644 easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/adapter/FederationStatisticsCollectionContext.java create mode 100644 easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/adapter/FederationStatisticsCollector.java create mode 100644 easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/api/FederationCleanupMetrics.java create mode 100644 easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/api/FederationSqlEngine.java create mode 100644 easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/api/FederationSqlEngines.java create mode 100644 easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/api/FederationSqlErrorCode.java create mode 100644 easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/api/FederationSqlException.java create mode 100644 easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/api/SqlCompletionItem.java create mode 100644 easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/api/SqlCompletionKind.java create mode 100644 easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/api/SqlCompletionRequest.java create mode 100644 easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/api/SqlCompletionResult.java create mode 100644 easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/api/SqlExecutionContext.java create mode 100644 easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/api/SqlQueryCommand.java create mode 100644 easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/compile/FederationFragmentExplain.java create mode 100644 easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/compile/FederationSqlPlan.java create mode 100644 easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/compile/FederationSqlPolicy.java create mode 100644 easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/compile/SqlCompileRequest.java create mode 100644 easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/compile/SqlExplainLevel.java create mode 100644 easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/compile/SqlExplainRequest.java create mode 100644 easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/compile/SqlExplainResult.java create mode 100644 easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/compile/SqlPolicyContext.java create mode 100644 easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/execute/FederationColumn.java create mode 100644 easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/execute/FederationExecutionGuard.java create mode 100644 easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/execute/FederationExecutionObserver.java create mode 100644 easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/execute/FederationFragmentExecutionContext.java create mode 100644 easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/execute/FederationFragmentExecutor.java create mode 100644 easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/execute/FederationFragmentExplainContext.java create mode 100644 easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/execute/FederationFragmentExplainer.java create mode 100644 easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/execute/FederationFragmentMetrics.java create mode 100644 easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/execute/FederationLocalOperatorMetrics.java create mode 100644 easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/execute/FederationPhysicalExplain.java create mode 100644 easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/execute/FederationQueryAdmissionController.java create mode 100644 easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/execute/FederationQueryMetricsSnapshot.java create mode 100644 easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/execute/FederationQueryPermit.java create mode 100644 easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/execute/FederationResultCursor.java create mode 100644 easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/execute/LocalFederationQueryAdmissionController.java create mode 100644 easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/execute/QueryAdmissionRequest.java create mode 100644 easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/execute/QueryId.java create mode 100644 easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/execute/SqlExecutionOptions.java create mode 100644 easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/execute/SqlParameter.java create mode 100644 easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/execute/StatementLifecycle.java create mode 100644 easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/federation/FederationColumnStatistics.java create mode 100644 easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/federation/FederationCostEstimate.java create mode 100644 easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/federation/FederationExecutionPolicy.java create mode 100644 easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/federation/FederationFragmentPlan.java create mode 100644 easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/federation/FederationJoinAlgorithm.java create mode 100644 easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/federation/FederationJoinOptimization.java create mode 100644 easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/federation/FederationJoinSelectionReason.java create mode 100644 easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/federation/FederationLogicalTableDefinition.java create mode 100644 easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/federation/FederationQueryMode.java create mode 100644 easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/federation/FederationQueryScopeDefinition.java create mode 100644 easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/federation/FederationSourceBindingDefinition.java create mode 100644 easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/federation/FederationSourceRuntimeIdentity.java create mode 100644 easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/federation/FederationStatisticsSnapshot.java create mode 100644 easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/federation/FederationStatisticsStatus.java create mode 100644 easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/federation/FederationTableStatistics.java create mode 100644 easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/federation/FederationTableStatisticsProvider.java create mode 100644 easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/runtime/AdapterFederationStatisticsProvider.java create mode 100644 easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/runtime/BoundedPlanCache.java create mode 100644 easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/runtime/CalciteFederationSqlCompiler.java create mode 100644 easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/runtime/CalciteSqlCompleter.java create mode 100644 easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/runtime/CancellationAwareFederationCursor.java create mode 100644 easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/runtime/CompiledFederationFragment.java create mode 100644 easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/runtime/CompositeQueryResources.java create mode 100644 easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/runtime/DefaultFederationSourceManager.java create mode 100644 easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/runtime/DefaultFederationSqlEngine.java create mode 100644 easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/runtime/DefaultFederationSqlPlan.java create mode 100644 easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/runtime/EnumerableFederationBudgetRel.java create mode 100644 easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/runtime/FederatedResultCursor.java create mode 100644 easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/runtime/FederationBudgetEnumerable.java create mode 100644 easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/runtime/FederationDataContext.java create mode 100644 easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/runtime/FederationExecutionSession.java create mode 100644 easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/runtime/FederationFragmentTable.java create mode 100644 easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/runtime/FederationQueryMetricsTracker.java create mode 100644 easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/runtime/FederationQueryScopeSnapshot.java create mode 100644 easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/runtime/FederationStatisticsMetadata.java create mode 100644 easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/runtime/FederationStatisticsTableScan.java create mode 100644 easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/runtime/LogicalTableSqlResolver.java create mode 100644 easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/runtime/ManagedFederationResultCursor.java create mode 100644 easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/runtime/MetricsFederationResultCursor.java create mode 100644 easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/runtime/NodeMemoryAdmissionController.java create mode 100644 easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/runtime/PlanCacheKey.java create mode 100644 easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/runtime/QueryCancellationRegistry.java create mode 100644 easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/runtime/QueryDeadline.java create mode 100644 easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/runtime/SourceCatalogSnapshot.java create mode 100644 easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/runtime/SourceRuntime.java create mode 100644 easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/runtime/TypedSqlDynamicParam.java create mode 100644 easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/source/ActiveSourceState.java create mode 100644 easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/source/ExternalSchemaDefinition.java create mode 100644 easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/source/FederationDataSourceHandle.java create mode 100644 easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/source/FederationDataSourceHandles.java create mode 100644 easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/source/FederationDataSourceResolver.java create mode 100644 easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/source/FederationSchemaDefinition.java create mode 100644 easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/source/FederationSourceDefinition.java create mode 100644 easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/source/FederationSourceManager.java create mode 100644 easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/source/FederationSourceState.java create mode 100644 easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/source/FederationSourceStateProvider.java create mode 100644 easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/source/FederationSourceView.java create mode 100644 easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/source/KnownJdbcDriver.java create mode 100644 easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/source/PreparedSourceRuntime.java create mode 100644 easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/source/RuntimeFingerprint.java create mode 100644 easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/source/SourceApplyOptions.java create mode 100644 easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/source/SourceApplyResult.java create mode 100644 easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/source/SourceApplyStatus.java create mode 100644 easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/source/SourceId.java create mode 100644 easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/source/SourceProbeResult.java create mode 100644 easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/source/SourceRemoveResult.java create mode 100644 easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/source/SourceRuntimeStatus.java create mode 100644 easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/source/SourceSnapshotResult.java create mode 100644 easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/source/SourceStateSubscription.java create mode 100644 easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/source/SourceTombstone.java create mode 100644 easy-agents-federation-sql/easy-agents-federation-sql-core/src/test/java/com/easyagents/federation/sql/api/SqlQueryCommandSerializationTest.java create mode 100644 easy-agents-federation-sql/easy-agents-federation-sql-core/src/test/java/com/easyagents/federation/sql/execute/LocalFederationQueryAdmissionControllerTest.java create mode 100644 easy-agents-federation-sql/easy-agents-federation-sql-core/src/test/java/com/easyagents/federation/sql/federation/FederationStatisticsSnapshotTest.java create mode 100644 easy-agents-federation-sql/easy-agents-federation-sql-core/src/test/java/com/easyagents/federation/sql/runtime/AdapterFederationStatisticsProviderTest.java create mode 100644 easy-agents-federation-sql/easy-agents-federation-sql-core/src/test/java/com/easyagents/federation/sql/runtime/BoundedPlanCacheTest.java create mode 100644 easy-agents-federation-sql/easy-agents-federation-sql-core/src/test/java/com/easyagents/federation/sql/runtime/CalciteFederationSqlCompilerTest.java create mode 100644 easy-agents-federation-sql/easy-agents-federation-sql-core/src/test/java/com/easyagents/federation/sql/runtime/DefaultFederationSourceManagerTest.java create mode 100644 easy-agents-federation-sql/easy-agents-federation-sql-core/src/test/java/com/easyagents/federation/sql/runtime/FederationQueryMetricsTrackerTest.java create mode 100644 easy-agents-federation-sql/easy-agents-federation-sql-core/src/test/java/com/easyagents/federation/sql/runtime/LogicalTableSqlResolverTest.java create mode 100644 easy-agents-federation-sql/easy-agents-federation-sql-core/src/test/java/com/easyagents/federation/sql/runtime/ManagedFederationResultCursorTest.java create mode 100644 easy-agents-federation-sql/easy-agents-federation-sql-core/src/test/java/com/easyagents/federation/sql/runtime/NodeMemoryAdmissionControllerTest.java create mode 100644 easy-agents-federation-sql/easy-agents-federation-sql-core/src/test/java/com/easyagents/federation/sql/runtime/QueryCancellationRegistryTest.java create mode 100644 easy-agents-federation-sql/easy-agents-federation-sql-core/src/test/java/com/easyagents/federation/sql/runtime/QueryDeadlineTest.java create mode 100644 easy-agents-federation-sql/easy-agents-federation-sql-core/src/test/java/com/easyagents/federation/sql/source/FederationSourceDefinitionTest.java create mode 100644 easy-agents-federation-sql/easy-agents-federation-sql-core/src/test/java/com/easyagents/federation/sql/source/KnownJdbcDriverTest.java create mode 100644 easy-agents-federation-sql/pom.xml diff --git a/easy-agents-bom/pom.xml b/easy-agents-bom/pom.xml index 254b793..504ee6f 100644 --- a/easy-agents-bom/pom.xml +++ b/easy-agents-bom/pom.xml @@ -17,6 +17,21 @@ UTF-8 + + + + com.easyagents + easy-agents-federation-sql-core + ${revision} + + + com.easyagents + easy-agents-federation-sql-adapter-jdbc + ${revision} + + + + @@ -95,6 +110,19 @@ easy-agents-rag-retrieval + + com.easyagents + easy-agents-federation-sql-core + + + + com.easyagents + easy-agents-federation-sql-adapter-jdbc + + + + + com.easyagents diff --git a/easy-agents-federation-sql/README.md b/easy-agents-federation-sql/README.md new file mode 100644 index 0000000..00b3ab4 --- /dev/null +++ b/easy-agents-federation-sql/README.md @@ -0,0 +1,156 @@ +# Easy-Agents Federation SQL + +基于 Apache Calcite 的 SQL 编译、方言转换、数据源绑定与流式 JDBC 查询底座。 + +## 模块 + +- `easy-agents-federation-sql-core`:公共 API、Calcite 编译、单源/联邦自动路由、计划缓存、数据源 Runtime、准入、指标与取消。 +- `easy-agents-federation-sql-adapter-jdbc`:默认 JDBC Adapter,也是信创数据库 Adapter 的实现示例。 + +业务项目通常只需依赖 JDBC Adapter,它会传递依赖 Core: + +```xml + + + + com.easyagents + easy-agents-bom + 1.2.0-RC + pom + import + + + + + + com.easyagents + easy-agents-federation-sql-adapter-jdbc + +``` + +## 公共入口 + +- `FederationSqlEngines.builder()`:组装 Engine、Resolver、策略与准入控制器。 +- `engine.sources()`:探测、绑定、预热、更新或移除数据源 Definition。 +- `engine.compile()` / `engine.execute()`:高级的节点本地计划模式。 +- `engine.query()`:推荐入口,在接收请求的节点完成编译或缓存命中并立即执行。 +- `engine.explain()`:显式返回 Calcite 计划;`PHYSICAL` 级别还会请求各数据库的非 `ANALYZE` Explain。 +- `engine.cancel(queryId)`:取消准入等待、JDBC 执行或游标消费中的节点本地查询;编译阶段收到取消后不会继续执行。 + +`FederationSqlPlan` 是 Engine 签发的只读接口,只能交回签发它的 Engine 执行。 + +`FederationSourceDefinition` 始终描述一个物理数据源。一次查询可见的单源或虚拟联邦范围由调用方使用 `FederationQueryScopeDefinition` 声明;Core 不持久化虚拟数据源,也不保存凭据。 + +## 最小使用示例 + +```java +SourceId sourceId = new SourceId("main"); +FederationSourceDefinition definition = new FederationSourceDefinition( + sourceId, + 1, + JdbcFederationSqlAdapterProvider.ADAPTER_ID, + List.of(new JdbcSchemaDefinition("APP", null, "public")), + Map.of() +); + +try (FederationSqlEngine engine = FederationSqlEngines.builder() + .dataSourceResolver(current -> { + HikariDataSource pool = createPool(current.sourceId()); + RuntimeFingerprint fingerprint = detectFingerprint(pool); + return FederationDataSourceHandles.owned(pool, fingerprint, pool::close); + }) + .maximumPlanCacheEntries(1024) + .maximumPlanCacheWeightBytes(64L * 1024L * 1024L) + .planCacheTimeToLive(Duration.ofMinutes(30)) + .build()) { + engine.sources().apply(definition, SourceApplyOptions.prewarmNow()); + + SqlQueryCommand command = SqlQueryCommand.of( + "SELECT NAME FROM APP.PERSON WHERE ID = ?", + sourceId, + 1, + List.of(new SqlParameter(Types.INTEGER, 1)) + ); + try (FederationResultCursor cursor = engine.query(command)) { + while (cursor.next()) { + System.out.println(cursor.row()); + } + } +} +``` + +`createPool`、凭据存储和 `detectFingerprint` 由调用方实现。Core 管理 Handle/Runtime 生命周期;连接复用、超时、泄漏检测和预热连接数由 HikariCP 等连接池负责。 + +## 虚拟联邦查询 + +调用方先分别登记 MySQL 与 PostgreSQL 的物理 `FederationSourceDefinition`,再为一次查询组装逻辑 Binding: + +```java +FederationQueryScopeDefinition scope = FederationQueryScopeDefinition.virtual( + "sales-analysis", + 7, + Map.of( + "SALES", FederationSourceBindingDefinition.of( + new SourceId("mysql-sales"), 12, Map.of("APP", "APP") + ), + "CRM", FederationSourceBindingDefinition.of( + new SourceId("pg-crm"), 5, Map.of("APP", "APP") + ) + ), + "SALES", + FederationExecutionPolicy.basic() +); + +String sql = """ + SELECT c.ID, SUM(o.AMOUNT) AS TOTAL + FROM CRM.APP.CUSTOMER c + JOIN SALES.APP.ORDER_ITEM o ON c.ID = o.CUSTOMER_ID + GROUP BY c.ID + ORDER BY TOTAL DESC + """; + +try (FederationResultCursor cursor = engine.query( + SqlQueryCommand.of(sql, scope, List.of()) +)) { + while (cursor.next()) { + System.out.println(cursor.row()); + } + FederationQueryMetricsSnapshot metrics = cursor.metrics(); +} +``` + +查询模式按 Calcite 校验后实际引用的物理 `SourceId` 数量决定。多 Binding Scope 中只引用一个源的 SQL 仍完整下推;引用多个源时,Core 生成目标方言 Fragment,并使用有界 Calcite 本地算子汇总。 + +调用方已有表列统计快照时,可以通过 `tableStatisticsProvider(...)` 注入行数、行宽、列基数、空值率和唯一键。Provider 的 `snapshot()` 必须一次性返回同时冻结版本、数据和有效期的 `FederationStatisticsSnapshot`,编译阶段不得主动执行 `COUNT(*)`;版本变化会隔离旧计划缓存,计划缓存期限也不会超过统计快照的最早失效时间。统计完整且未过期时,等值 `INNER JOIN` 会把估算搬运量较小的一侧作为本地 Hash Table 构建端;统计缺失、不完整或过期时保持稳定的保守顺序。逻辑 Explain 的每个 Fragment 会返回估算是否可用、扫描/输出行数、行宽、搬运字节、统计来源/采集时间和下推算子。 + +首批联邦算子覆盖等值 `INNER JOIN`、`LEFT JOIN`、`UNION ALL`、`COUNT/SUM/MIN/MAX/AVG`、普通 `GROUP BY`、CTE、排序和分页。非等值 Join、联邦本地字符比较/排序/分组/`MIN/MAX`、`UNION DISTINCT`、窗口函数、磁盘 Spill 与跨库事务快照会明确拒绝。字符型本地算子需要调用方先统一排序规则,后续再由 Adapter 提供可验证的 Collation 能力。Calcite 本地时间表示只保证毫秒精度;映射精度超过 3 位或运行时检测到亚毫秒值时会明确拒绝。驱动以 `ANY/OTHER` 返回的标准 JDBC 时区标量会保留纳秒并统一为 UTC Offset。 + +结果采用标准流式 Cursor 语义:Fragment 或本地算子可能在调用方已读取若干行后失败,已交付的行无法撤回。调用方只能在 `next()` 正常返回 `false` 后将本次结果视为完整成功;需要不可逆副作用时应先完整消费并自行提交,或提供补偿机制。 + +## Explain 与指标 + +普通 `query` 不会访问数据库 Optimizer。只有显式调用物理 Explain 才会产生额外数据库往返: + +```java +SqlCompileRequest compile = SqlCompileRequest.of(sql, scope); +SqlExplainResult logical = engine.explain( + new SqlExplainRequest(compile, SqlExplainLevel.LOGICAL) +); +SqlExplainResult physical = engine.explain(new SqlExplainRequest(compile)); +``` + +`physical.fragments()` 为每个 Fragment 返回目标方言 SQL、参数映射和数据库原生计划。MySQL/PostgreSQL Adapter 尽力归一化扫描方式、候选索引、选中索引、估算行数与过滤条件;数据库没有返回的字段保持空值。为避免原生计划回显敏感常量,Explain 不接受实际参数值,只按 `SqlCompileRequest` 声明的 JDBC 类型绑定 `NULL`,因此索引选择可能与真实参数计划不同。 + +`FederationResultCursor.metrics()` 可在消费过程中读取,并在耗尽或关闭后定稿,包含模式、计划缓存命中、编译、准入等待、连接获取、数据库执行、本地算子、首行与完整消费耗时,以及最终行/字节、中间搬运行/字节、截断、超时、错误分类和各 Fragment 统计。Adapter 无法安全估算字节时对应字段为 `-1`,不会用 `0` 冒充已测量值。 + +查询总时限取 Engine、Query Scope 和请求 JDBC timeout 中的最小值。硬时限会覆盖连接池等待后的 JDBC 执行和游标消费,并尝试同时 `cancel`、关闭全部活动 Statement/Cursor;连接池自身仍需配置有限的 connection timeout,以约束 Statement 创建前的连接获取阶段。 + +## Adapter 扩展 + +实现 `FederationSqlAdapterProvider` 并通过 Java `ServiceLoader` 注册。Adapter 直接提供 Calcite `Schema`、`SqlDialect`、类型系统、运算符表、Planner Rule 和参数 `SqlDataTypeSpec`,无需额外中间态。重复 `adapterId` 会在启动时拒绝。 + +## 分布式边界 + +Definition、revision 和墓碑可以由调用方存入 Redis 等共享状态系统,并通过 `FederationSourceStateProvider` 下发。连接池、Calcite Schema、计划与活动查询均为节点本地对象,不应序列化或跨节点共享。负载均衡请求应携带 `minimumRevision`,落后节点会先同步或返回明确的未就绪错误。 + +当前联邦路径以最多两个实际物理源和内存内有界汇总为基线。各源使用独立只读连接,不提供跨数据库全局快照一致性;应通过 `FederationExecutionPolicy` 为中间行数、字节数、Fragment 数和总时限设置硬上限。 diff --git a/easy-agents-federation-sql/easy-agents-federation-sql-adapter-jdbc/pom.xml b/easy-agents-federation-sql/easy-agents-federation-sql-adapter-jdbc/pom.xml new file mode 100644 index 0000000..da8e441 --- /dev/null +++ b/easy-agents-federation-sql/easy-agents-federation-sql-adapter-jdbc/pom.xml @@ -0,0 +1,52 @@ + + + 4.0.0 + + + com.easyagents + easy-agents-federation-sql + ${revision} + + + easy-agents-federation-sql-adapter-jdbc + easy-agents-federation-sql-adapter-jdbc + + + + com.easyagents + easy-agents-federation-sql-core + + + org.apache.calcite + calcite-core + + + org.slf4j + slf4j-api + + + junit + junit + test + + + com.h2database + h2 + test + + + com.mysql + mysql-connector-j + 8.4.0 + test + + + org.postgresql + postgresql + 42.7.5 + test + + + diff --git a/easy-agents-federation-sql/easy-agents-federation-sql-adapter-jdbc/src/main/java/com/easyagents/federation/sql/adapter/jdbc/JdbcFailureClassifier.java b/easy-agents-federation-sql/easy-agents-federation-sql-adapter-jdbc/src/main/java/com/easyagents/federation/sql/adapter/jdbc/JdbcFailureClassifier.java new file mode 100644 index 0000000..1f9f95d --- /dev/null +++ b/easy-agents-federation-sql/easy-agents-federation-sql-adapter-jdbc/src/main/java/com/easyagents/federation/sql/adapter/jdbc/JdbcFailureClassifier.java @@ -0,0 +1,54 @@ +package com.easyagents.federation.sql.adapter.jdbc; + +import com.easyagents.federation.sql.api.FederationSqlErrorCode; +import com.easyagents.federation.sql.api.FederationSqlException; +import com.easyagents.federation.sql.execute.StatementLifecycle; +import java.sql.SQLTimeoutException; + +/** + * 按查询注册表已经确定的终态分类 JDBC 执行与读取异常。 + */ +final class JdbcFailureClassifier { + + /** + * 工具类无需实例化。 + */ + private JdbcFailureClassifier() { + } + + /** + * 将 JDBC 异常转换为稳定的查询错误,优先保留先到达的取消或超时终态。 + * + * @param lifecycle 查询 Statement 生命周期 + * @param cause JDBC 或驱动异常 + * @param timeoutMessage 超时提示 + * @param cancellationMessage 取消提示 + * @param failureMessage 普通执行失败提示 + * @return 分类后的统一异常 + */ + static FederationSqlException classify( + StatementLifecycle lifecycle, + Throwable cause, + String timeoutMessage, + String cancellationMessage, + String failureMessage + ) { + FederationSqlErrorCode errorCode; + String message; + if (lifecycle.timeoutRequested()) { + errorCode = FederationSqlErrorCode.QUERY_TIMEOUT; + message = timeoutMessage; + } else if (lifecycle.cancellationRequested()) { + // Statement.cancel() 后部分驱动会抛 SQLTimeoutException,已登记的取消终态必须优先。 + errorCode = FederationSqlErrorCode.QUERY_CANCELLED; + message = cancellationMessage; + } else if (cause instanceof SQLTimeoutException) { + errorCode = FederationSqlErrorCode.QUERY_TIMEOUT; + message = timeoutMessage; + } else { + errorCode = FederationSqlErrorCode.EXECUTION_FAILED; + message = failureMessage; + } + return new FederationSqlException(errorCode, message, cause); + } +} diff --git a/easy-agents-federation-sql/easy-agents-federation-sql-adapter-jdbc/src/main/java/com/easyagents/federation/sql/adapter/jdbc/JdbcFederationFragmentExecutor.java b/easy-agents-federation-sql/easy-agents-federation-sql-adapter-jdbc/src/main/java/com/easyagents/federation/sql/adapter/jdbc/JdbcFederationFragmentExecutor.java new file mode 100644 index 0000000..8163c7a --- /dev/null +++ b/easy-agents-federation-sql/easy-agents-federation-sql-adapter-jdbc/src/main/java/com/easyagents/federation/sql/adapter/jdbc/JdbcFederationFragmentExecutor.java @@ -0,0 +1,200 @@ +package com.easyagents.federation.sql.adapter.jdbc; + +import com.easyagents.federation.sql.api.FederationSqlErrorCode; +import com.easyagents.federation.sql.api.FederationSqlException; +import com.easyagents.federation.sql.execute.FederationColumn; +import com.easyagents.federation.sql.execute.FederationFragmentExecutionContext; +import com.easyagents.federation.sql.execute.FederationFragmentExecutor; +import com.easyagents.federation.sql.execute.FederationResultCursor; +import com.easyagents.federation.sql.execute.SqlParameter; +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.ResultSetMetaData; +import java.sql.SQLException; +import java.sql.SQLTimeoutException; +import java.sql.SQLTransientConnectionException; +import java.util.ArrayList; +import java.util.List; +import java.util.Locale; + +/** + * 直接使用 PreparedStatement 执行目标数据库 SQL 的流式 JDBC 执行器。 + */ +final class JdbcFederationFragmentExecutor implements FederationFragmentExecutor { + + /** + * 获取连接、应用只读限制、绑定参数并返回持有全部资源的流式游标。 + * + * @param context 执行上下文 + * @return 流式游标 + */ + @Override + public FederationResultCursor execute(FederationFragmentExecutionContext context) { + Connection connection = null; + PreparedStatement statement = null; + boolean registered = false; + boolean connectionAcquired = false; + try { + context.executionGuard().ensureAllowed(); + long connectionStarted = System.nanoTime(); + connection = context.dataSource().getConnection(); + connectionAcquired = true; + context.observer().connectionAcquired(System.nanoTime() - connectionStarted); + context.executionGuard().ensureAllowed(); + configureConnection(connection, context); + statement = connection.prepareStatement( + context.sql(), + ResultSet.TYPE_FORWARD_ONLY, + ResultSet.CONCUR_READ_ONLY + ); + applyOptions(statement, context); + bindParameters(statement, context.parameters()); + context.statementLifecycle().register(statement); + registered = true; + context.executionGuard().ensureAllowed(); + long executionStarted = System.nanoTime(); + ResultSet resultSet = statement.executeQuery(); + context.executionGuard().ensureAllowed(); + context.observer().databaseExecutionCompleted(System.nanoTime() - executionStarted); + List columns = readColumns(resultSet.getMetaData()); + return new JdbcFederationResultCursor( + context.queryId(), + columns, + resultSet, + statement, + connection, + context.statementLifecycle(), + context.executionGuard(), + context.observer() + ); + } catch (SQLException | RuntimeException exception) { + if (!connectionAcquired) { + // 连接池等待可能跨过统一截止时间;总超时或显式取消应保持为查询终态。 + context.executionGuard().ensureAllowed(); + } + if (registered) { + context.statementLifecycle().unregister(statement); + } + closeAfterFailure(statement, connection, exception); + if (exception instanceof FederationSqlException federationSqlException) { + throw federationSqlException; + } + boolean connectionTimedOut = !connectionAcquired + && (exception instanceof SQLTransientConnectionException + || exception instanceof SQLTimeoutException); + if (connectionTimedOut) { + throw new FederationSqlException( + FederationSqlErrorCode.CONNECTION_ACQUISITION_TIMEOUT, + "timed out while acquiring a JDBC connection", + exception + ); + } + if (!connectionAcquired) { + throw new FederationSqlException( + FederationSqlErrorCode.CONNECTION_ACQUISITION_FAILED, + "failed to acquire a JDBC connection", + exception + ); + } + throw JdbcFailureClassifier.classify( + context.statementLifecycle(), + exception, + "JDBC query timed out", + "JDBC query was cancelled", + "JDBC query execution failed" + ); + } + } + + private static void applyOptions( + PreparedStatement statement, + FederationFragmentExecutionContext context + ) throws SQLException { + int fetchSize = effectiveFetchSize(context); + if (fetchSize != 0) { + statement.setFetchSize(fetchSize); + } + if (context.options().maxRows() > 0) { + statement.setMaxRows(context.options().maxRows()); + } + int queryTimeout = context.executionGuard().boundedQueryTimeoutSeconds( + context.options().queryTimeoutSeconds() + ); + if (queryTimeout > 0) { + statement.setQueryTimeout(queryTimeout); + } + } + + private static void configureConnection( + Connection connection, + FederationFragmentExecutionContext context + ) throws SQLException { + if (!connection.isReadOnly()) { + connection.setReadOnly(true); + } + String product = context.compatibility().databaseProduct().toLowerCase(Locale.ROOT); + // PostgreSQL 只有在事务模式下才会按正 fetchSize 使用服务端游标。 + if (product.contains("postgres") && context.options().fetchSize() > 0 + && connection.getAutoCommit()) { + connection.setAutoCommit(false); + } + } + + private static int effectiveFetchSize(FederationFragmentExecutionContext context) { + String product = context.compatibility().databaseProduct().toLowerCase(Locale.ROOT); + if (product.contains("mysql") + && "legacy".equalsIgnoreCase(context.adapterOptions().get("mysqlStreamingMode"))) { + // Connector/J 旧式逐行流需要显式 MIN_VALUE;默认仍使用正 fetchSize + useCursorFetch。 + return Integer.MIN_VALUE; + } + return context.options().fetchSize(); + } + + private static void bindParameters(PreparedStatement statement, List parameters) + throws SQLException { + for (int index = 0; index < parameters.size(); index++) { + SqlParameter parameter = parameters.get(index); + int jdbcIndex = index + 1; + if (parameter.value() == null) { + statement.setNull(jdbcIndex, parameter.jdbcType()); + } else { + statement.setObject(jdbcIndex, parameter.value(), parameter.jdbcType()); + } + } + } + + private static List readColumns(ResultSetMetaData metadata) throws SQLException { + List columns = new ArrayList<>(metadata.getColumnCount()); + for (int index = 1; index <= metadata.getColumnCount(); index++) { + columns.add(new FederationColumn( + index, + metadata.getColumnLabel(index), + metadata.getColumnType(index), + metadata.getColumnTypeName(index), + metadata.isNullable(index) != ResultSetMetaData.columnNoNulls + )); + } + return List.copyOf(columns); + } + + private static void closeAfterFailure( + PreparedStatement statement, + Connection connection, + Throwable original + ) { + closeAndSuppress(statement, original); + closeAndSuppress(connection, original); + } + + private static void closeAndSuppress(AutoCloseable closeable, Throwable original) { + if (closeable == null) { + return; + } + try { + closeable.close(); + } catch (Exception closeException) { + original.addSuppressed(closeException); + } + } +} diff --git a/easy-agents-federation-sql/easy-agents-federation-sql-adapter-jdbc/src/main/java/com/easyagents/federation/sql/adapter/jdbc/JdbcFederationFragmentExplainer.java b/easy-agents-federation-sql/easy-agents-federation-sql-adapter-jdbc/src/main/java/com/easyagents/federation/sql/adapter/jdbc/JdbcFederationFragmentExplainer.java new file mode 100644 index 0000000..17347da --- /dev/null +++ b/easy-agents-federation-sql/easy-agents-federation-sql-adapter-jdbc/src/main/java/com/easyagents/federation/sql/adapter/jdbc/JdbcFederationFragmentExplainer.java @@ -0,0 +1,307 @@ +package com.easyagents.federation.sql.adapter.jdbc; + +import com.easyagents.federation.sql.api.FederationSqlErrorCode; +import com.easyagents.federation.sql.api.FederationSqlException; +import com.easyagents.federation.sql.execute.FederationFragmentExplainContext; +import com.easyagents.federation.sql.execute.FederationFragmentExplainer; +import com.easyagents.federation.sql.execute.FederationPhysicalExplain; +import com.easyagents.federation.sql.execute.SqlParameter; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.ResultSetMetaData; +import java.sql.SQLException; +import java.sql.SQLTimeoutException; +import java.sql.SQLTransientConnectionException; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; +import java.util.Locale; + +/** + * MySQL、PostgreSQL 和 H2 的非 ANALYZE 物理 Explain 实现。 + */ +final class JdbcFederationFragmentExplainer implements FederationFragmentExplainer { + + private static final ObjectMapper JSON = new ObjectMapper(); + + /** {@inheritDoc} */ + @Override + public FederationPhysicalExplain explain(FederationFragmentExplainContext context) { + String product = normalize(context.compatibility().databaseProduct()); + String explainSql = explainSql(product, context.sql()); + if (explainSql == null) { + return FederationPhysicalExplain.unavailable( + "physical Explain is not implemented for " + + context.compatibility().databaseProduct() + ); + } + Connection acquired = acquireConnection(context); + try (Connection connection = acquired) { + context.executionGuard().ensureAllowed(); + if (!connection.isReadOnly()) { + connection.setReadOnly(true); + } + try (PreparedStatement statement = connection.prepareStatement(explainSql)) { + int queryTimeout = context.executionGuard().boundedQueryTimeoutSeconds( + context.queryTimeoutSeconds() + ); + if (queryTimeout > 0) { + statement.setQueryTimeout(queryTimeout); + } + bind(statement, context.parameters()); + context.executionGuard().ensureAllowed(); + try (ResultSet resultSet = statement.executeQuery()) { + context.executionGuard().ensureAllowed(); + String nativePlan = readPlan(resultSet); + return normalizePlan(product, nativePlan); + } + } + } catch (SQLException | RuntimeException exception) { + if (exception instanceof FederationSqlException federationSqlException) { + throw federationSqlException; + } + throw new FederationSqlException( + FederationSqlErrorCode.EXPLAIN_FAILED, + "physical database Explain failed", + exception + ); + } + } + + /** + * 在统一截止时间约束下获取物理 Explain 连接。 + * + * @param context 分片 Explain 上下文 + * @return 已获取连接 + * @throws FederationSqlException 获取超时、失败或查询已终止时抛出 + */ + private static Connection acquireConnection(FederationFragmentExplainContext context) { + try { + context.executionGuard().ensureAllowed(); + return context.dataSource().getConnection(); + } catch (SQLException | RuntimeException exception) { + if (exception instanceof FederationSqlException federationSqlException) { + throw federationSqlException; + } + context.executionGuard().ensureAllowed(); + FederationSqlErrorCode code = exception instanceof SQLTimeoutException + || exception instanceof SQLTransientConnectionException + ? FederationSqlErrorCode.CONNECTION_ACQUISITION_TIMEOUT + : FederationSqlErrorCode.CONNECTION_ACQUISITION_FAILED; + String message = code == FederationSqlErrorCode.CONNECTION_ACQUISITION_TIMEOUT + ? "timed out while acquiring a JDBC connection for physical Explain" + : "failed to acquire a JDBC connection for physical Explain"; + throw new FederationSqlException(code, message, exception); + } + } + + private static String explainSql(String product, String sql) { + if (product.contains("mysql")) { + return "EXPLAIN FORMAT=JSON " + sql; + } + if (product.contains("postgres")) { + return "EXPLAIN (FORMAT JSON, ANALYZE FALSE, COSTS TRUE, VERBOSE FALSE, BUFFERS FALSE) " + + sql; + } + if (product.equals("h2")) { + return "EXPLAIN " + sql; + } + return null; + } + + private static void bind(PreparedStatement statement, List parameters) + throws SQLException { + for (int index = 0; index < parameters.size(); index++) { + SqlParameter parameter = parameters.get(index); + if (parameter.value() == null) { + statement.setNull(index + 1, parameter.jdbcType()); + } else { + statement.setObject(index + 1, parameter.value(), parameter.jdbcType()); + } + } + } + + private static String readPlan(ResultSet resultSet) throws SQLException { + StringBuilder plan = new StringBuilder(); + ResultSetMetaData metadata = resultSet.getMetaData(); + while (resultSet.next()) { + if (!plan.isEmpty()) { + plan.append('\n'); + } + for (int column = 1; column <= metadata.getColumnCount(); column++) { + if (column > 1) { + plan.append('\t'); + } + Object value = resultSet.getObject(column); + if (value != null) { + plan.append(value); + } + } + } + return plan.toString(); + } + + private static FederationPhysicalExplain normalizePlan(String product, String nativePlan) { + if (!nativePlan.isBlank() && (product.contains("mysql") || product.contains("postgres"))) { + try { + JsonNode root = JSON.readTree(nativePlan); + return product.contains("mysql") + ? normalizeMysql(root, nativePlan) + : normalizePostgresql(root, nativePlan); + } catch (Exception ignored) { + // 原生计划仍可用;归一化失败不会伪造索引结论。 + } + } + return new FederationPhysicalExplain( + true, + nativePlan, + null, + null, + List.of(), + null, + null, + null, + "native plan is available; normalized index fields are unavailable" + ); + } + + private static FederationPhysicalExplain normalizeMysql(JsonNode root, String nativePlan) { + JsonNode table = findObjectWithField(root, "access_type"); + if (table == null) { + return nativeOnly(nativePlan, "MySQL plan contains no normalized table access node"); + } + List candidates = stringValues(table.get("possible_keys")); + return new FederationPhysicalExplain( + true, + nativePlan, + "table", + text(table, "access_type"), + candidates, + text(table, "key"), + longValue(table, "rows_examined_per_scan", "rows"), + firstText(table, "attached_condition", "index_condition"), + "normalized from MySQL JSON Explain" + ); + } + + private static FederationPhysicalExplain normalizePostgresql(JsonNode root, String nativePlan) { + JsonNode plan = root.isArray() && !root.isEmpty() ? root.get(0).get("Plan") : root.get("Plan"); + JsonNode scan = findObjectWithField(plan, "Index Name"); + if (scan == null) { + scan = findObjectWithTextSuffix(plan, "Node Type", "Scan"); + } + if (scan == null) { + return nativeOnly(nativePlan, "PostgreSQL plan contains no normalized plan node"); + } + return new FederationPhysicalExplain( + true, + nativePlan, + text(scan, "Node Type"), + text(scan, "Node Type"), + List.of(), + text(scan, "Index Name"), + longValue(scan, "Plan Rows"), + firstText(scan, "Index Cond", "Filter", "Join Filter"), + "normalized from PostgreSQL JSON Explain" + ); + } + + private static FederationPhysicalExplain nativeOnly(String nativePlan, String diagnostic) { + return new FederationPhysicalExplain( + true, + nativePlan, + null, + null, + List.of(), + null, + null, + null, + diagnostic + ); + } + + private static JsonNode findObjectWithField(JsonNode node, String field) { + if (node == null) { + return null; + } + if (node.isObject() && node.has(field)) { + return node; + } + Iterator children = node.elements(); + while (children.hasNext()) { + JsonNode found = findObjectWithField(children.next(), field); + if (found != null) { + return found; + } + } + return null; + } + + private static JsonNode findObjectWithTextSuffix( + JsonNode node, + String field, + String suffix + ) { + if (node == null) { + return null; + } + if (node.isObject()) { + String value = text(node, field); + if (value != null && value.endsWith(suffix)) { + return node; + } + } + Iterator children = node.elements(); + while (children.hasNext()) { + JsonNode found = findObjectWithTextSuffix(children.next(), field, suffix); + if (found != null) { + return found; + } + } + return null; + } + + private static List stringValues(JsonNode node) { + if (node == null || node.isNull()) { + return List.of(); + } + if (node.isArray()) { + List values = new ArrayList<>(); + node.forEach(value -> values.add(value.asText())); + return List.copyOf(values); + } + return List.of(node.asText()); + } + + private static String firstText(JsonNode node, String... fields) { + for (String field : fields) { + String value = text(node, field); + if (value != null) { + return value; + } + } + return null; + } + + private static String text(JsonNode node, String field) { + JsonNode value = node == null ? null : node.get(field); + return value == null || value.isNull() ? null : value.asText(); + } + + private static Long longValue(JsonNode node, String... fields) { + for (String field : fields) { + JsonNode value = node == null ? null : node.get(field); + if (value != null && value.isNumber()) { + return value.longValue(); + } + } + return null; + } + + private static String normalize(String product) { + return product == null ? "" : product.trim().toLowerCase(Locale.ROOT); + } +} diff --git a/easy-agents-federation-sql/easy-agents-federation-sql-adapter-jdbc/src/main/java/com/easyagents/federation/sql/adapter/jdbc/JdbcFederationResultCursor.java b/easy-agents-federation-sql/easy-agents-federation-sql-adapter-jdbc/src/main/java/com/easyagents/federation/sql/adapter/jdbc/JdbcFederationResultCursor.java new file mode 100644 index 0000000..e3359d8 --- /dev/null +++ b/easy-agents-federation-sql/easy-agents-federation-sql-adapter-jdbc/src/main/java/com/easyagents/federation/sql/adapter/jdbc/JdbcFederationResultCursor.java @@ -0,0 +1,446 @@ +package com.easyagents.federation.sql.adapter.jdbc; + +import com.easyagents.federation.sql.api.FederationSqlErrorCode; +import com.easyagents.federation.sql.api.FederationSqlException; +import com.easyagents.federation.sql.execute.FederationColumn; +import com.easyagents.federation.sql.execute.FederationExecutionGuard; +import com.easyagents.federation.sql.execute.FederationExecutionObserver; +import com.easyagents.federation.sql.execute.FederationResultCursor; +import com.easyagents.federation.sql.execute.QueryId; +import com.easyagents.federation.sql.execute.StatementLifecycle; +import java.io.FilterInputStream; +import java.io.FilterReader; +import java.io.IOException; +import java.io.InputStream; +import java.io.Reader; +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.atomic.AtomicBoolean; + +/** + * 持有 ResultSet、Statement、Connection 和 Engine 资源的 JDBC 流式游标。 + */ +final class JdbcFederationResultCursor implements FederationResultCursor { + + private final QueryId queryId; + private final List columns; + private final ResultSet resultSet; + private final PreparedStatement statement; + private final Connection connection; + private final StatementLifecycle statementLifecycle; + private final FederationExecutionGuard executionGuard; + private final FederationExecutionObserver observer; + private final AtomicBoolean closed = new AtomicBoolean(); + private final AtomicBoolean firstRowObserved = new AtomicBoolean(); + private final long resultSetCreatedNanos = System.nanoTime(); + + /** + * 创建 JDBC 流式游标。 + * + * @param queryId 查询标识 + * @param columns 结果列 + * @param resultSet JDBC ResultSet + * @param statement JDBC Statement + * @param connection JDBC Connection + * @param statementLifecycle Statement 生命周期回调 + * @param executionGuard 查询取消与截止时间检查器 + * @param observer Fragment 执行阶段观察器 + */ + JdbcFederationResultCursor( + QueryId queryId, + List columns, + ResultSet resultSet, + PreparedStatement statement, + Connection connection, + StatementLifecycle statementLifecycle, + FederationExecutionGuard executionGuard, + FederationExecutionObserver observer + ) { + this.queryId = queryId; + this.columns = List.copyOf(columns); + this.resultSet = resultSet; + this.statement = statement; + this.connection = connection; + this.statementLifecycle = statementLifecycle; + this.executionGuard = executionGuard; + this.observer = observer; + } + + /** + * 创建不采集阶段指标的兼容 JDBC 游标。 + * + * @param queryId 查询标识 + * @param columns 结果列 + * @param resultSet JDBC ResultSet + * @param statement JDBC Statement + * @param connection JDBC Connection + * @param statementLifecycle Statement 生命周期 + */ + JdbcFederationResultCursor( + QueryId queryId, + List columns, + ResultSet resultSet, + PreparedStatement statement, + Connection connection, + StatementLifecycle statementLifecycle + ) { + this( + queryId, + columns, + resultSet, + statement, + connection, + statementLifecycle, + FederationExecutionGuard.none(), + FederationExecutionObserver.none() + ); + } + + /** + * 返回查询标识。 + * + * @return 查询标识 + */ + @Override + public QueryId queryId() { + return queryId; + } + + /** + * 返回结果列。 + * + * @return 结果列 + */ + @Override + public List columns() { + return columns; + } + + /** + * 移动到下一行;读取结束时保留资源直至调用方关闭游标。 + * + * @return 是否存在下一行 + */ + @Override + public boolean next() { + ensureOpen(); + try { + boolean present = resultSet.next(); + ensureAllowedAfterRead(); + if (present && firstRowObserved.compareAndSet(false, true)) { + observer.firstRowAvailable(System.nanoTime() - resultSetCreatedNanos); + } + return present; + } catch (SQLException exception) { + closeWithSuppressed(exception); + throw JdbcFailureClassifier.classify( + statementLifecycle, + exception, + "JDBC result read timed out", + "JDBC query was cancelled", + "failed to advance JDBC result cursor" + ); + } + } + + /** + * 读取当前行指定列。 + * + * @param columnIndex 从 1 开始的列序号 + * @return 列值 + */ + @Override + public Object getObject(int columnIndex) { + ensureOpen(); + try { + Object value = resultSet.getObject(columnIndex); + ensureAllowedAfterRead(); + return value; + } catch (SQLException exception) { + closeWithSuppressed(exception); + throw JdbcFailureClassifier.classify( + statementLifecycle, + exception, + "JDBC result read timed out", + "JDBC query was cancelled", + "failed to read JDBC result column " + columnIndex + ); + } + } + + /** + * 以 JDBC 流读取二进制列。 + * + * @param columnIndex 从 1 开始的列序号 + * @return 二进制流;SQL NULL 返回 null + */ + @Override + public InputStream getBinaryStream(int columnIndex) { + ensureOpen(); + try { + InputStream stream = resultSet.getBinaryStream(columnIndex); + ensureAllowedAfterRead(); + return stream == null ? null : new GuardedInputStream(stream, columnIndex); + } catch (SQLException exception) { + throw readFailure(columnIndex, exception); + } + } + + /** + * 以 JDBC 流读取字符列。 + * + * @param columnIndex 从 1 开始的列序号 + * @return 字符流;SQL NULL 返回 null + */ + @Override + public Reader getCharacterStream(int columnIndex) { + ensureOpen(); + try { + Reader reader = resultSet.getCharacterStream(columnIndex); + ensureAllowedAfterRead(); + return reader == null ? null : new GuardedReader(reader, columnIndex); + } catch (SQLException exception) { + throw readFailure(columnIndex, exception); + } + } + + /** + * 复制当前行;Engine 不缓存返回行。 + * + * @return 当前行列值 + */ + @Override + public List row() { + ensureOpen(); + List row = new ArrayList<>(columns.size()); + for (int index = 1; index <= columns.size(); index++) { + row.add(getObject(index)); + } + return Collections.unmodifiableList(row); + } + + private void ensureOpen() { + // 异步关闭可能先于消费线程到达,优先保留取消或超时终态语义。 + try { + executionGuard.ensureAllowed(); + } catch (RuntimeException exception) { + closeWithSuppressed(exception); + throw exception; + } + if (closed.get()) { + throw new FederationSqlException( + FederationSqlErrorCode.EXECUTION_FAILED, + "result cursor is closed" + ); + } + } + + private void ensureAllowedAfterRead() { + try { + executionGuard.ensureAllowed(); + } catch (RuntimeException exception) { + closeWithSuppressed(exception); + throw exception; + } + } + + /** + * 幂等关闭 JDBC 资源并最终释放准入许可与 Runtime lease。 + */ + @Override + public void close() { + if (!closed.compareAndSet(false, true)) { + return; + } + FederationSqlException failure = null; + try { + resultSet.close(); + } catch (SQLException exception) { + failure = closeFailure("ResultSet", exception); + } + try { + statementLifecycle.unregister(statement); + } catch (RuntimeException exception) { + failure = append(failure, closeFailure("Statement lifecycle", exception)); + } + try { + statement.close(); + } catch (SQLException exception) { + failure = append(failure, closeFailure("PreparedStatement", exception)); + } + try { + connection.close(); + } catch (SQLException exception) { + failure = append(failure, closeFailure("Connection", exception)); + } + if (failure != null) { + throw failure; + } + } + + private void closeWithSuppressed(Throwable original) { + try { + close(); + } catch (RuntimeException closeException) { + original.addSuppressed(closeException); + } + } + + /** + * 将流式列读取异常映射为统一错误并确定性关闭 JDBC 资源。 + * + * @param columnIndex 列序号 + * @param exception JDBC 或流读取异常 + * @return 统一 Federation 异常 + */ + private FederationSqlException readFailure(int columnIndex, Throwable exception) { + closeWithSuppressed(exception); + return JdbcFailureClassifier.classify( + statementLifecycle, + exception, + "JDBC result read timed out", + "JDBC query was cancelled", + "failed to stream JDBC result column " + columnIndex + ); + } + + /** + * 对二进制列的每次实际读取执行查询终态检查。 + */ + private final class GuardedInputStream extends FilterInputStream { + + private final int columnIndex; + + /** + * 创建受查询生命周期保护的二进制流。 + * + * @param delegate JDBC 驱动流 + * @param columnIndex 列序号 + */ + private GuardedInputStream(InputStream delegate, int columnIndex) { + super(delegate); + this.columnIndex = columnIndex; + } + + /** {@inheritDoc} */ + @Override + public int read() throws IOException { + ensureOpen(); + try { + int value = super.read(); + ensureAllowedAfterRead(); + return value; + } catch (IOException exception) { + throw readFailure(columnIndex, exception); + } + } + + /** {@inheritDoc} */ + @Override + public int read(byte[] buffer, int offset, int length) throws IOException { + ensureOpen(); + try { + int read = super.read(buffer, offset, length); + ensureAllowedAfterRead(); + return read; + } catch (IOException exception) { + throw readFailure(columnIndex, exception); + } + } + + /** {@inheritDoc} */ + @Override + public long skip(long count) throws IOException { + ensureOpen(); + try { + long skipped = super.skip(count); + ensureAllowedAfterRead(); + return skipped; + } catch (IOException exception) { + throw readFailure(columnIndex, exception); + } + } + } + + /** + * 对字符列的每次实际读取执行查询终态检查。 + */ + private final class GuardedReader extends FilterReader { + + private final int columnIndex; + + /** + * 创建受查询生命周期保护的字符流。 + * + * @param delegate JDBC 驱动 Reader + * @param columnIndex 列序号 + */ + private GuardedReader(Reader delegate, int columnIndex) { + super(delegate); + this.columnIndex = columnIndex; + } + + /** {@inheritDoc} */ + @Override + public int read() throws IOException { + ensureOpen(); + try { + int value = super.read(); + ensureAllowedAfterRead(); + return value; + } catch (IOException exception) { + throw readFailure(columnIndex, exception); + } + } + + /** {@inheritDoc} */ + @Override + public int read(char[] buffer, int offset, int length) throws IOException { + ensureOpen(); + try { + int read = super.read(buffer, offset, length); + ensureAllowedAfterRead(); + return read; + } catch (IOException exception) { + throw readFailure(columnIndex, exception); + } + } + + /** {@inheritDoc} */ + @Override + public long skip(long count) throws IOException { + ensureOpen(); + try { + long skipped = super.skip(count); + ensureAllowedAfterRead(); + return skipped; + } catch (IOException exception) { + throw readFailure(columnIndex, exception); + } + } + } + + private static FederationSqlException closeFailure(String resource, Exception cause) { + return new FederationSqlException( + FederationSqlErrorCode.RESOURCE_CLOSE_FAILED, + "failed to close JDBC " + resource, + cause + ); + } + + private static FederationSqlException append( + FederationSqlException failure, + FederationSqlException next + ) { + if (failure == null) { + return next; + } + failure.addSuppressed(next); + return failure; + } +} diff --git a/easy-agents-federation-sql/easy-agents-federation-sql-adapter-jdbc/src/main/java/com/easyagents/federation/sql/adapter/jdbc/JdbcFederationSqlAdapterProvider.java b/easy-agents-federation-sql/easy-agents-federation-sql-adapter-jdbc/src/main/java/com/easyagents/federation/sql/adapter/jdbc/JdbcFederationSqlAdapterProvider.java new file mode 100644 index 0000000..4cb00b4 --- /dev/null +++ b/easy-agents-federation-sql/easy-agents-federation-sql-adapter-jdbc/src/main/java/com/easyagents/federation/sql/adapter/jdbc/JdbcFederationSqlAdapterProvider.java @@ -0,0 +1,212 @@ +package com.easyagents.federation.sql.adapter.jdbc; + +import com.easyagents.federation.sql.adapter.AdapterCompatibility; +import com.easyagents.federation.sql.adapter.AdapterCompatibilityStatus; +import com.easyagents.federation.sql.adapter.AdapterDialectContext; +import com.easyagents.federation.sql.adapter.AdapterHints; +import com.easyagents.federation.sql.adapter.AdapterSchemaContext; +import com.easyagents.federation.sql.adapter.FederationSqlAdapterProvider; +import com.easyagents.federation.sql.adapter.FederationStatisticsCollector; +import com.easyagents.federation.sql.api.FederationSqlErrorCode; +import com.easyagents.federation.sql.api.FederationSqlException; +import com.easyagents.federation.sql.execute.FederationFragmentExecutor; +import com.easyagents.federation.sql.execute.FederationFragmentExplainer; +import java.sql.DatabaseMetaData; +import java.sql.SQLException; +import java.util.Locale; +import java.util.Optional; +import java.util.Set; +import org.apache.calcite.adapter.jdbc.JdbcConvention; +import org.apache.calcite.adapter.jdbc.JdbcSchema; +import org.apache.calcite.schema.Schema; +import org.apache.calcite.schema.Schemas; +import org.apache.calcite.sql.SqlDialect; +import org.apache.calcite.sql.SqlDialectFactoryImpl; +import org.apache.calcite.sql.dialect.AnsiSqlDialect; +import org.apache.calcite.sql.dialect.MysqlSqlDialect; + +/** + * MySQL、PostgreSQL、Oracle 及显式实验 ANSI 数据库的默认 JDBC Adapter。 + */ +public final class JdbcFederationSqlAdapterProvider implements FederationSqlAdapterProvider { + + /** 默认 JDBC Adapter 标识。 */ + public static final String ADAPTER_ID = "jdbc"; + /** 允许未知数据库采用实验 ANSI 方言的 Definition 选项。 */ + public static final String EXPERIMENTAL_ANSI_OPTION = "experimentalAnsi"; + + private static final Set SUPPORTED_PRODUCTS = Set.of( + "mysql", + "postgresql", + "oracle", + "h2" + ); + + private final FederationFragmentExecutor executor = new JdbcFederationFragmentExecutor(); + private final FederationFragmentExplainer explainer = new JdbcFederationFragmentExplainer(); + private final FederationStatisticsCollector statisticsCollector = + new JdbcFederationStatisticsCollector(); + + /** + * 创建默认 JDBC Adapter Provider。 + */ + public JdbcFederationSqlAdapterProvider() { + } + + /** + * 返回默认 Adapter 标识。 + * + * @return {@value #ADAPTER_ID} + */ + @Override + public String adapterId() { + return ADAPTER_ID; + } + + /** + * 基于数据库产品名判断内建或实验 ANSI 支持。 + * + * @param metadata JDBC 元数据 + * @param hints Adapter 提示 + * @return 是否支持 + * @throws SQLException 元数据读取失败 + */ + @Override + public boolean supports(DatabaseMetaData metadata, AdapterHints hints) throws SQLException { + return SUPPORTED_PRODUCTS.contains(normalize(metadata.getDatabaseProductName())) + || hints.enabled(EXPERIMENTAL_ANSI_OPTION); + } + + /** + * 返回与实际验证证据一致的兼容性状态。 + * + * @param metadata JDBC 元数据 + * @param hints Adapter 提示 + * @return 兼容性说明 + * @throws SQLException 元数据读取失败 + */ + @Override + public AdapterCompatibility compatibility(DatabaseMetaData metadata, AdapterHints hints) throws SQLException { + String product = metadata.getDatabaseProductName(); + String normalized = normalize(product); + AdapterCompatibilityStatus status; + String diagnostic; + if ("h2".equals(normalized)) { + status = AdapterCompatibilityStatus.VERIFIED; + diagnostic = "verified by module-level H2 integration tests"; + } else if (SUPPORTED_PRODUCTS.contains(normalized)) { + status = AdapterCompatibilityStatus.CODE_SUPPORTED_UNVERIFIED; + diagnostic = "dialect is supported by code; verify against the target database version before production"; + } else if (hints.enabled(EXPERIMENTAL_ANSI_OPTION)) { + status = AdapterCompatibilityStatus.CODE_SUPPORTED_UNVERIFIED; + diagnostic = "experimental ANSI mode is enabled for an unrecognized database"; + } else { + status = AdapterCompatibilityStatus.UNSUPPORTED; + diagnostic = "database product is not recognized"; + } + return new AdapterCompatibility( + status, + product, + metadata.getDatabaseProductVersion(), + metadata.getDriverName(), + metadata.getDriverVersion(), + diagnostic + ); + } + + /** + * 创建复用已探测 Dialect 和调用方 DataSource 的 JdbcSchema。 + * + * @param context Schema 上下文 + * @return Calcite JdbcSchema + */ + @Override + public Schema createSchema(AdapterSchemaContext context) { + if (!(context.schemaDefinition() instanceof JdbcSchemaDefinition definition)) { + throw new FederationSqlException( + FederationSqlErrorCode.INVALID_ARGUMENT, + "jdbc adapter requires JdbcSchemaDefinition" + ); + } + JdbcConvention convention = JdbcConvention.of( + context.dialect(), + Schemas.subSchemaExpression( + context.parentSchema(), + definition.logicalName(), + JdbcSchema.class + ), + context.sourceDefinition().sourceId().value() + "." + definition.logicalName() + ); + Schema schema = new JdbcSchema( + context.handle().dataSource(), + context.dialect(), + convention, + definition.catalog(), + definition.physicalSchema() + ); + // MySQL 表名可区分大小写而列名始终不区分大小写,需分别建模。 + return context.dialect() instanceof MysqlSqlDialect + ? new MysqlCaseInsensitiveColumnSchema(schema) + : schema; + } + + /** + * 使用 Calcite 官方 DialectFactory 选择方言,未知数据库仅在显式 ANSI 模式下放行。 + * + * @param context 方言上下文 + * @return SqlDialect + * @throws SQLException 元数据读取失败 + */ + @Override + public SqlDialect createDialect(AdapterDialectContext context) throws SQLException { + String product = normalize(context.metadata().getDatabaseProductName()); + if (!SUPPORTED_PRODUCTS.contains(product)) { + AdapterHints hints = new AdapterHints(context.sourceDefinition().adapterOptions()); + if (hints.enabled(EXPERIMENTAL_ANSI_OPTION)) { + return AnsiSqlDialect.DEFAULT; + } + throw new FederationSqlException( + FederationSqlErrorCode.ADAPTER_UNSUPPORTED, + "database product is not supported by jdbc adapter: " + + context.metadata().getDatabaseProductName() + ); + } + return SqlDialectFactoryImpl.INSTANCE.create(context.metadata()); + } + + /** + * 返回直接 JDBC 流式执行器。 + * + * @return Fragment 执行器 + */ + @Override + public FederationFragmentExecutor fragmentExecutor() { + return executor; + } + + /** + * 返回 MySQL、PostgreSQL 和 H2 的显式物理 Explain 实现。 + * + * @return JDBC 物理 Explain SPI + */ + @Override + public Optional fragmentExplainer() { + return Optional.of(explainer); + } + + /** + * 返回 MySQL 与 PostgreSQL 的内建目录统计采集器。 + * + *

Oracle、H2 和实验 ANSI 数据库当前返回空统计,由引擎使用默认成本估算。

+ * + * @return JDBC 统计采集 SPI + */ + @Override + public Optional statisticsCollector() { + return Optional.of(statisticsCollector); + } + + private static String normalize(String productName) { + return productName == null ? "" : productName.trim().toLowerCase(Locale.ROOT); + } +} diff --git a/easy-agents-federation-sql/easy-agents-federation-sql-adapter-jdbc/src/main/java/com/easyagents/federation/sql/adapter/jdbc/JdbcFederationStatisticsCollector.java b/easy-agents-federation-sql/easy-agents-federation-sql-adapter-jdbc/src/main/java/com/easyagents/federation/sql/adapter/jdbc/JdbcFederationStatisticsCollector.java new file mode 100644 index 0000000..a5babcc --- /dev/null +++ b/easy-agents-federation-sql/easy-agents-federation-sql-adapter-jdbc/src/main/java/com/easyagents/federation/sql/adapter/jdbc/JdbcFederationStatisticsCollector.java @@ -0,0 +1,641 @@ +package com.easyagents.federation.sql.adapter.jdbc; + +import com.easyagents.federation.sql.adapter.FederationStatisticsCollectionContext; +import com.easyagents.federation.sql.adapter.FederationStatisticsCollector; +import com.easyagents.federation.sql.api.FederationSqlErrorCode; +import com.easyagents.federation.sql.api.FederationSqlException; +import com.easyagents.federation.sql.federation.FederationColumnStatistics; +import com.easyagents.federation.sql.federation.FederationStatisticsSnapshot; +import com.easyagents.federation.sql.federation.FederationStatisticsStatus; +import com.easyagents.federation.sql.federation.FederationTableStatistics; +import com.easyagents.federation.sql.source.FederationSchemaDefinition; +import java.sql.Connection; +import java.sql.DatabaseMetaData; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Types; +import java.util.ArrayList; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Set; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * MySQL 与 PostgreSQL 的批量 JDBC 目录统计采集器。 + */ +final class JdbcFederationStatisticsCollector implements FederationStatisticsCollector { + + private static final Logger LOG = LoggerFactory.getLogger( + JdbcFederationStatisticsCollector.class + ); + + /** + * 根据 JDBC 数据库产品分派内建统计采集逻辑。 + * + * @param context 统计采集上下文 + * @return 表统计映射 + * @throws SQLException 目录或 JDBC 元数据读取失败 + */ + @Override + public Map collect( + FederationStatisticsCollectionContext context + ) throws SQLException { + String product = normalize( + context.connection().getMetaData().getDatabaseProductName() + ); + List schemas = jdbcSchemas(context); + return switch (product) { + case "mysql" -> collectMysql(context, schemas); + case "postgresql" -> collectPostgresql(context, schemas); + default -> Map.of(); + }; + } + + /** + * 批量读取 MySQL INFORMATION_SCHEMA 表统计和主键。 + * + * @param context 采集上下文 + * @param schemas JDBC Schema 映射 + * @return MySQL 表统计 + * @throws SQLException 目录读取失败 + */ + private Map collectMysql( + FederationStatisticsCollectionContext context, + List schemas + ) throws SQLException { + Map statistics = + new LinkedHashMap<>(); + for (JdbcSchemaDefinition schema : schemas) { + String catalog = textOr(schema.catalog(), context.connection().getCatalog()); + if (catalog == null || catalog.isBlank()) { + continue; + } + Map layouts = readColumnLayouts( + context.connection().getMetaData(), + catalog, + schema.physicalSchema() + ); + Map> primaryKeys = readMysqlPrimaryKeys( + context, + catalog + ); + String sql = "SELECT TABLE_NAME, TABLE_ROWS, AVG_ROW_LENGTH " + + "FROM INFORMATION_SCHEMA.TABLES " + + "WHERE TABLE_SCHEMA = ? AND TABLE_TYPE = 'BASE TABLE'"; + try (PreparedStatement statement = context.connection().prepareStatement(sql)) { + statement.setQueryTimeout(context.queryTimeoutSeconds()); + statement.setString(1, catalog); + try (ResultSet result = statement.executeQuery()) { + while (result.next()) { + String table = result.getString("TABLE_NAME"); + ColumnLayout layout = layouts.getOrDefault( + normalize(table), + ColumnLayout.empty() + ); + long averageWidth = result.getLong("AVG_ROW_LENGTH"); + if (averageWidth <= 0L) { + averageWidth = layout.fallbackWidthBytes(); + } + FederationStatisticsSnapshot.TableKey key = + new FederationStatisticsSnapshot.TableKey( + context.sourceDefinition().sourceId(), + schema.logicalName(), + table + ); + statistics.put(key, new FederationTableStatistics( + Math.max(0D, result.getDouble("TABLE_ROWS")), + Math.max(1L, averageWidth), + context.collectedAt(), + "database-catalog:mysql", + Map.of(), + uniqueKey(primaryKeys.get(normalize(table))), + context.expiresAt(), + FederationStatisticsStatus.PARTIAL + )); + } + } + } + } + return Map.copyOf(statistics); + } + + /** + * 批量读取 PostgreSQL 表行数、列分布和主键统计。 + * + * @param context 采集上下文 + * @param schemas JDBC Schema 映射 + * @return PostgreSQL 表统计 + * @throws SQLException 表级目录读取失败 + */ + private Map + collectPostgresql( + FederationStatisticsCollectionContext context, + List schemas + ) throws SQLException { + Map schemasByPhysical = new LinkedHashMap<>(); + Map layouts = new LinkedHashMap<>(); + for (JdbcSchemaDefinition schema : schemas) { + String physical = textOr(schema.physicalSchema(), context.connection().getSchema()); + if (physical == null || physical.isBlank()) { + physical = "public"; + } + schemasByPhysical.putIfAbsent(normalize(physical), schema); + Map schemaLayouts = readColumnLayouts( + context.connection().getMetaData(), + schema.catalog(), + physical + ); + String resolvedPhysical = physical; + schemaLayouts.forEach((table, layout) -> layouts.put( + tableKey(resolvedPhysical, table), + layout + )); + } + if (schemasByPhysical.isEmpty()) { + return Map.of(); + } + Map estimates = readPostgresqlTableEstimates( + context, + schemasByPhysical.keySet() + ); + Map> columns; + try { + columns = readPostgresqlColumnStatistics( + context, + schemasByPhysical.keySet(), + estimates + ); + } catch (SQLException exception) { + LOG.warn( + "PostgreSQL column statistics are unavailable; retaining table estimates, sourceId={}", + context.sourceDefinition().sourceId(), + exception + ); + columns = Map.of(); + } + Map> primaryKeys; + try { + primaryKeys = readPostgresqlPrimaryKeys( + context, + schemasByPhysical.keySet() + ); + } catch (SQLException exception) { + LOG.warn( + "PostgreSQL primary-key statistics are unavailable, sourceId={}", + context.sourceDefinition().sourceId(), + exception + ); + primaryKeys = Map.of(); + } + + Map statistics = + new LinkedHashMap<>(); + for (Map.Entry entry : estimates.entrySet()) { + TableEstimate estimate = entry.getValue(); + JdbcSchemaDefinition schema = schemasByPhysical.get(normalize(estimate.schema())); + if (schema == null) { + continue; + } + ColumnLayout layout = layouts.getOrDefault(entry.getKey(), ColumnLayout.empty()); + Map tableColumns = columns.getOrDefault( + entry.getKey(), + Map.of() + ); + long averageWidth = Math.max( + layout.fallbackWidthBytes(), + averageColumnWidth(tableColumns) + ); + boolean complete = !layout.columns().isEmpty() + && containsAllIgnoreCase(tableColumns.keySet(), layout.columns()); + FederationStatisticsSnapshot.TableKey key = + new FederationStatisticsSnapshot.TableKey( + context.sourceDefinition().sourceId(), + schema.logicalName(), + estimate.table() + ); + statistics.put(key, new FederationTableStatistics( + estimate.estimatedRows(), + Math.max(1L, averageWidth), + context.collectedAt(), + "database-catalog:postgresql", + tableColumns, + uniqueKey(primaryKeys.get(entry.getKey())), + context.expiresAt(), + complete + ? FederationStatisticsStatus.COMPLETE + : FederationStatisticsStatus.PARTIAL + )); + } + return Map.copyOf(statistics); + } + + /** + * 读取 Definition 中的 JDBC Schema 映射并拒绝不匹配的定义类型。 + * + * @param context 采集上下文 + * @return JDBC Schema 定义 + */ + private List jdbcSchemas( + FederationStatisticsCollectionContext context + ) { + List schemas = new ArrayList<>(); + for (FederationSchemaDefinition schema : context.sourceDefinition().schemas()) { + if (!(schema instanceof JdbcSchemaDefinition jdbcSchema)) { + throw new FederationSqlException( + FederationSqlErrorCode.INVALID_ARGUMENT, + "jdbc statistics collector requires JdbcSchemaDefinition" + ); + } + schemas.add(jdbcSchema); + } + return List.copyOf(schemas); + } + + /** + * 通过 JDBC 元数据按 Schema 批量读取字段布局。 + * + * @param metadata JDBC 元数据 + * @param catalog 物理 Catalog + * @param schema 物理 Schema + * @return 按规范化表名索引的字段布局 + * @throws SQLException 元数据读取失败 + */ + private Map readColumnLayouts( + DatabaseMetaData metadata, + String catalog, + String schema + ) throws SQLException { + Map layouts = new LinkedHashMap<>(); + try (ResultSet result = metadata.getColumns(catalog, schema, "%", "%")) { + while (result.next()) { + String table = normalize(result.getString("TABLE_NAME")); + MutableColumnLayout layout = layouts.computeIfAbsent( + table, + ignored -> new MutableColumnLayout() + ); + layout.columns.add(result.getString("COLUMN_NAME")); + layout.fallbackWidthBytes = saturatedAdd( + layout.fallbackWidthBytes, + estimatedJdbcWidth(result.getInt("DATA_TYPE")) + ); + } + } + Map frozen = new LinkedHashMap<>(); + layouts.forEach((table, layout) -> frozen.put( + table, + new ColumnLayout( + Math.max(1L, layout.fallbackWidthBytes), + Set.copyOf(layout.columns) + ) + )); + return Map.copyOf(frozen); + } + + /** + * 一次查询一个 MySQL Catalog 的全部主键字段。 + * + * @param context 采集上下文 + * @param catalog 物理 Catalog + * @return 按规范化表名索引的有序主键 + * @throws SQLException 目录读取失败 + */ + private Map> readMysqlPrimaryKeys( + FederationStatisticsCollectionContext context, + String catalog + ) throws SQLException { + String sql = "SELECT TABLE_NAME, COLUMN_NAME " + + "FROM INFORMATION_SCHEMA.KEY_COLUMN_USAGE " + + "WHERE TABLE_SCHEMA = ? AND CONSTRAINT_NAME = 'PRIMARY' " + + "ORDER BY TABLE_NAME, ORDINAL_POSITION"; + Map> keys = new LinkedHashMap<>(); + try (PreparedStatement statement = context.connection().prepareStatement(sql)) { + statement.setQueryTimeout(context.queryTimeoutSeconds()); + statement.setString(1, catalog); + try (ResultSet result = statement.executeQuery()) { + while (result.next()) { + keys.computeIfAbsent( + normalize(result.getString("TABLE_NAME")), + ignored -> new ArrayList<>() + ).add(result.getString("COLUMN_NAME")); + } + } + } + return freezeLists(keys); + } + + /** + * 读取 PostgreSQL 表级近似行数。 + * + * @param context 采集上下文 + * @param schemas 物理 Schema + * @return 表级估算 + * @throws SQLException 目录读取失败 + */ + private Map readPostgresqlTableEstimates( + FederationStatisticsCollectionContext context, + Set schemas + ) throws SQLException { + String sql = "SELECT n.nspname AS schema_name, c.relname AS table_name, " + + "GREATEST(c.reltuples, 0)::double precision AS estimated_rows " + + "FROM pg_catalog.pg_class c " + + "JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace " + + "WHERE c.relkind IN ('r', 'p') AND lower(n.nspname) IN (" + + placeholders(schemas.size()) + ")"; + Map estimates = new LinkedHashMap<>(); + try (PreparedStatement statement = context.connection().prepareStatement(sql)) { + statement.setQueryTimeout(context.queryTimeoutSeconds()); + bind(statement, schemas); + try (ResultSet result = statement.executeQuery()) { + while (result.next()) { + String schema = result.getString("schema_name"); + String table = result.getString("table_name"); + estimates.put( + tableKey(schema, table), + new TableEstimate( + schema, + table, + Math.max(0D, result.getDouble("estimated_rows")) + ) + ); + } + } + } + return Map.copyOf(estimates); + } + + /** + * 读取 PostgreSQL 列分布统计。 + * + * @param context 采集上下文 + * @param schemas 物理 Schema + * @param estimates 已读取的表级估算 + * @return 按物理表索引的列统计 + * @throws SQLException 目录读取失败 + */ + private Map> + readPostgresqlColumnStatistics( + FederationStatisticsCollectionContext context, + Set schemas, + Map estimates + ) throws SQLException { + String sql = "SELECT schemaname, tablename, attname, null_frac, n_distinct, avg_width " + + "FROM pg_catalog.pg_stats WHERE lower(schemaname) IN (" + + placeholders(schemas.size()) + ")"; + Map> columns = new LinkedHashMap<>(); + try (PreparedStatement statement = context.connection().prepareStatement(sql)) { + statement.setQueryTimeout(context.queryTimeoutSeconds()); + bind(statement, schemas); + try (ResultSet result = statement.executeQuery()) { + while (result.next()) { + String key = tableKey( + result.getString("schemaname"), + result.getString("tablename") + ); + TableEstimate table = estimates.get(key); + if (table == null) { + continue; + } + double rawDistinct = result.getDouble("n_distinct"); + double distinct = rawDistinct < 0D + ? Math.abs(rawDistinct) * table.estimatedRows() + : rawDistinct; + if (!Double.isFinite(distinct)) { + distinct = 0D; + } + columns.computeIfAbsent(key, ignored -> new LinkedHashMap<>()).put( + result.getString("attname"), + new FederationColumnStatistics( + Math.max(0D, distinct), + Math.max(0D, Math.min(1D, result.getDouble("null_frac"))), + Math.max(0L, result.getLong("avg_width")) + ) + ); + } + } + } + Map> frozen = new LinkedHashMap<>(); + columns.forEach((table, values) -> frozen.put(table, Map.copyOf(values))); + return Map.copyOf(frozen); + } + + /** + * 一次读取多个 PostgreSQL Schema 的主键字段。 + * + * @param context 采集上下文 + * @param schemas 物理 Schema + * @return 按物理表索引的主键字段 + * @throws SQLException 目录读取失败 + */ + private Map> readPostgresqlPrimaryKeys( + FederationStatisticsCollectionContext context, + Set schemas + ) throws SQLException { + String sql = "SELECT n.nspname AS schema_name, c.relname AS table_name, " + + "a.attname AS column_name " + + "FROM pg_catalog.pg_index i " + + "JOIN pg_catalog.pg_class c ON c.oid = i.indrelid " + + "JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace " + + "JOIN pg_catalog.pg_attribute a " + + "ON a.attrelid = c.oid AND a.attnum = ANY(i.indkey) " + + "WHERE i.indisprimary AND lower(n.nspname) IN (" + + placeholders(schemas.size()) + ") " + + "ORDER BY n.nspname, c.relname, a.attnum"; + Map> keys = new LinkedHashMap<>(); + try (PreparedStatement statement = context.connection().prepareStatement(sql)) { + statement.setQueryTimeout(context.queryTimeoutSeconds()); + bind(statement, schemas); + try (ResultSet result = statement.executeQuery()) { + while (result.next()) { + keys.computeIfAbsent( + tableKey( + result.getString("schema_name"), + result.getString("table_name") + ), + ignored -> new ArrayList<>() + ).add(result.getString("column_name")); + } + } + } + return freezeLists(keys); + } + + /** + * 将可选主键转换为唯一键列表。 + * + * @param primaryKey 主键字段 + * @return 零个或一个唯一键 + */ + private List> uniqueKey(List primaryKey) { + return primaryKey == null || primaryKey.isEmpty() + ? List.of() + : List.of(List.copyOf(primaryKey)); + } + + /** + * 汇总列平均宽度并防止 long 溢出。 + * + * @param columns 列统计 + * @return 至少为 1 的平均宽度 + */ + private long averageColumnWidth(Map columns) { + long width = 0L; + for (FederationColumnStatistics column : columns.values()) { + width = saturatedAdd(width, column.averageWidthBytes()); + } + return Math.max(1L, width); + } + + /** + * 判断列统计是否覆盖全部字段。 + * + * @param available 已有列统计名称 + * @param required JDBC 字段名称 + * @return 完整覆盖时为 true + */ + private boolean containsAllIgnoreCase(Set available, Set required) { + Set normalized = new LinkedHashSet<>(); + available.forEach(value -> normalized.add(normalize(value))); + return required.stream().map(this::normalize).allMatch(normalized::contains); + } + + /** + * 估算 JDBC 类型的保守内存宽度。 + * + * @param jdbcType JDBC 类型 + * @return 估算字节数 + */ + private long estimatedJdbcWidth(int jdbcType) { + return switch (jdbcType) { + case Types.BOOLEAN, Types.BIT, Types.TINYINT -> 1L; + case Types.SMALLINT -> 2L; + case Types.INTEGER, Types.REAL, Types.FLOAT, Types.DATE -> 4L; + case Types.BIGINT, Types.DOUBLE, Types.TIMESTAMP, + Types.TIMESTAMP_WITH_TIMEZONE, Types.TIME, + Types.TIME_WITH_TIMEZONE -> 8L; + case Types.DECIMAL, Types.NUMERIC -> 16L; + case Types.BINARY, Types.VARBINARY, Types.LONGVARBINARY, + Types.BLOB, Types.CLOB, Types.NCLOB, + Types.LONGVARCHAR, Types.LONGNVARCHAR -> 64L; + default -> 32L; + }; + } + + /** + * 生成固定数量的 PreparedStatement 占位符。 + * + * @param size 占位符数量 + * @return 逗号分隔占位符 + */ + private String placeholders(int size) { + return String.join(", ", Collections.nCopies(size, "?")); + } + + /** + * 按稳定顺序绑定规范化 Schema。 + * + * @param statement PreparedStatement + * @param schemas Schema 集合 + * @throws SQLException 参数绑定失败 + */ + private void bind(PreparedStatement statement, Set schemas) throws SQLException { + int index = 1; + for (String schema : schemas) { + statement.setString(index++, normalize(schema)); + } + } + + /** + * 冻结可变列表映射。 + * + * @param source 可变列表映射 + * @return 不可变列表映射 + */ + private Map> freezeLists(Map> source) { + Map> frozen = new LinkedHashMap<>(); + source.forEach((key, value) -> frozen.put(key, List.copyOf(value))); + return Map.copyOf(frozen); + } + + /** + * 生成大小写不敏感的物理表索引键。 + * + * @param schema 物理 Schema + * @param table 物理表 + * @return 稳定索引键 + */ + private String tableKey(String schema, String table) { + return normalize(schema) + '\u0000' + normalize(table); + } + + /** + * 返回首个非空文本。 + * + * @param primary 首选值 + * @param fallback 备用值 + * @return 可空结果 + */ + private String textOr(String primary, String fallback) { + return primary == null || primary.isBlank() ? fallback : primary; + } + + /** + * 规范化数据库产品名或标识符。 + * + * @param value 原始值 + * @return 小写非空值 + */ + private String normalize(String value) { + return value == null ? "" : value.trim().toLowerCase(Locale.ROOT); + } + + /** + * 饱和 long 加法。 + * + * @param left 左值 + * @param right 右值 + * @return 不溢出的和 + */ + private long saturatedAdd(long left, long right) { + return Long.MAX_VALUE - left < right ? Long.MAX_VALUE : left + right; + } + + /** + * 单表字段布局。 + * + * @param fallbackWidthBytes JDBC 类型估算行宽 + * @param columns 字段名称 + */ + private record ColumnLayout(long fallbackWidthBytes, Set columns) { + + /** + * 创建空布局。 + * + * @return 保守空布局 + */ + private static ColumnLayout empty() { + return new ColumnLayout(1L, Set.of()); + } + } + + /** 可变字段布局构造器。 */ + private static final class MutableColumnLayout { + private long fallbackWidthBytes; + private final Set columns = new LinkedHashSet<>(); + } + + /** + * PostgreSQL 表级估算。 + * + * @param schema 物理 Schema + * @param table 物理表 + * @param estimatedRows 估算行数 + */ + private record TableEstimate(String schema, String table, double estimatedRows) { + } +} diff --git a/easy-agents-federation-sql/easy-agents-federation-sql-adapter-jdbc/src/main/java/com/easyagents/federation/sql/adapter/jdbc/JdbcSchemaDefinition.java b/easy-agents-federation-sql/easy-agents-federation-sql-adapter-jdbc/src/main/java/com/easyagents/federation/sql/adapter/jdbc/JdbcSchemaDefinition.java new file mode 100644 index 0000000..556eae9 --- /dev/null +++ b/easy-agents-federation-sql/easy-agents-federation-sql-adapter-jdbc/src/main/java/com/easyagents/federation/sql/adapter/jdbc/JdbcSchemaDefinition.java @@ -0,0 +1,38 @@ +package com.easyagents.federation.sql.adapter.jdbc; + +import com.easyagents.federation.sql.source.FederationSchemaDefinition; +import java.util.Arrays; +import java.util.List; + +/** + * JDBC Catalog/Schema 到逻辑 Schema 的映射定义。 + * + * @param logicalName SQL 中使用的逻辑 Schema 名称 + * @param catalog 物理 Catalog,可为空 + * @param physicalSchema 物理 Schema,可为空 + */ +public record JdbcSchemaDefinition( + String logicalName, + String catalog, + String physicalSchema +) implements FederationSchemaDefinition { + + /** + * 校验逻辑名称并保留可空物理 Catalog/Schema。 + */ + public JdbcSchemaDefinition { + if (logicalName == null || logicalName.isBlank()) { + throw new IllegalArgumentException("logicalName must not be blank"); + } + } + + /** + * 返回 JDBC Schema 映射的稳定校验和材料。 + * + * @return 稳定材料 + */ + @Override + public List checksumFields() { + return Arrays.asList(catalog, physicalSchema); + } +} diff --git a/easy-agents-federation-sql/easy-agents-federation-sql-adapter-jdbc/src/main/java/com/easyagents/federation/sql/adapter/jdbc/MysqlCaseInsensitiveColumnSchema.java b/easy-agents-federation-sql/easy-agents-federation-sql-adapter-jdbc/src/main/java/com/easyagents/federation/sql/adapter/jdbc/MysqlCaseInsensitiveColumnSchema.java new file mode 100644 index 0000000..6b1cc33 --- /dev/null +++ b/easy-agents-federation-sql/easy-agents-federation-sql-adapter-jdbc/src/main/java/com/easyagents/federation/sql/adapter/jdbc/MysqlCaseInsensitiveColumnSchema.java @@ -0,0 +1,308 @@ +package com.easyagents.federation.sql.adapter.jdbc; + +import java.sql.Connection; +import java.sql.SQLException; +import java.util.Objects; +import java.util.Set; +import java.util.stream.Collectors; +import org.apache.calcite.adapter.jdbc.JdbcSchema; +import org.apache.calcite.adapter.jdbc.JdbcTable; +import org.apache.calcite.config.CalciteConnectionConfig; +import org.apache.calcite.plan.RelOptTable; +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.rel.type.RelDataTypeFactory; +import org.apache.calcite.rel.type.RelDataTypeField; +import org.apache.calcite.rel.type.RelRecordType; +import org.apache.calcite.schema.Schema; +import org.apache.calcite.schema.SchemaVersion; +import org.apache.calcite.schema.Statistic; +import org.apache.calcite.schema.Table; +import org.apache.calcite.schema.TranslatableTable; +import org.apache.calcite.schema.Wrapper; +import org.apache.calcite.schema.impl.DelegatingSchema; +import org.apache.calcite.schema.lookup.IgnoreCaseLookup; +import org.apache.calcite.schema.lookup.LikePattern; +import org.apache.calcite.schema.lookup.Lookup; +import org.apache.calcite.sql.SqlCall; +import org.apache.calcite.sql.SqlNode; + +/** + * 保留 MySQL 表名精确匹配,同时让列名遵循 MySQL 的大小写不敏感语义。 + */ +final class MysqlCaseInsensitiveColumnSchema extends DelegatingSchema { + + private final Lookup tableLookup; + + /** + * 创建 MySQL 列名语义包装器。 + * + * @param schema 原始 JDBC Schema + */ + MysqlCaseInsensitiveColumnSchema(Schema schema) { + super(Objects.requireNonNull(schema, "schema")); + Lookup
sourceLookup = schema.tables(); + if (schema instanceof JdbcSchema jdbcSchema) { + sourceLookup = new ExactJdbcTableLookup(jdbcSchema, sourceLookup); + } + this.tableLookup = sourceLookup.map((table, ignoredName) -> wrap(table)); + } + + /** + * 返回保持原始表名 Lookup 规则的包装表集合。 + * + * @return 包装后的表 Lookup + */ + @Override + public Lookup
tables() { + return tableLookup; + } + + /** + * 按原始 Schema 规则精确获取表,再包装列类型。 + * + * @param name 表名 + * @return 包装表;不存在时返回 null + */ + @Override + public Table getTable(String name) { + return tableLookup.get(name); + } + + /** + * 为 Schema 快照保留相同的列名语义。 + * + * @param version Schema 版本 + * @return 包装后的快照 + */ + @Override + public Schema snapshot(SchemaVersion version) { + return new MysqlCaseInsensitiveColumnSchema(schema.snapshot(version)); + } + + private static Table wrap(Table table) { + return table instanceof MysqlCaseInsensitiveColumnTable + ? table + : new MysqlCaseInsensitiveColumnTable(table); + } + + /** + * 将 Calcite 的表名查找收紧为 JDBC 元数据层面的精确查找。 + */ + private static final class ExactJdbcTableLookup extends IgnoreCaseLookup
{ + + private final JdbcSchema jdbcSchema; + private final Lookup
delegate; + private volatile boolean searchEscapeLoaded; + private String searchEscape; + + private ExactJdbcTableLookup(JdbcSchema jdbcSchema, Lookup
delegate) { + this.jdbcSchema = Objects.requireNonNull(jdbcSchema, "jdbcSchema"); + this.delegate = Objects.requireNonNull(delegate, "delegate"); + } + + /** + * 转义 JDBC LIKE 通配字符后读取,并校验驱动返回的真实物理表名。 + * + * @param name 精确表名 + * @return 精确匹配的表;不存在时返回 null + */ + @Override + public Table get(String name) { + Table table = delegate.get(escapePattern(name)); + if (table == null) { + return null; + } + JdbcTable jdbcTable = table instanceof JdbcTable direct + ? direct + : table instanceof Wrapper wrapper + ? wrapper.unwrap(JdbcTable.class) + : null; + return jdbcTable != null && name.equals(jdbcTable.jdbcTableName) ? table : null; + } + + /** + * 返回符合 Calcite LIKE 语义的表名,过滤 JDBC 对下划线的额外通配匹配。 + * + * @param pattern 表名模式 + * @return 匹配名称集合 + */ + @Override + public Set getNames(LikePattern pattern) { + return delegate.getNames(pattern).stream() + .filter(pattern.matcher()::apply) + .collect(Collectors.toUnmodifiableSet()); + } + + private String escapePattern(String name) { + String escape = searchEscape(); + if (escape == null || escape.isEmpty()) { + if (name.indexOf('_') >= 0 || name.indexOf('%') >= 0) { + throw new IllegalStateException( + "MySQL JDBC driver does not expose a metadata search escape" + ); + } + return name; + } + return name + .replace(escape, escape + escape) + .replace("_", escape + "_") + .replace("%", escape + "%"); + } + + private String searchEscape() { + if (searchEscapeLoaded) { + return searchEscape; + } + synchronized (this) { + if (!searchEscapeLoaded) { + try (Connection connection = jdbcSchema.getDataSource().getConnection()) { + searchEscape = connection.getMetaData().getSearchStringEscape(); + searchEscapeLoaded = true; + } catch (SQLException exception) { + throw new IllegalStateException( + "Failed to read MySQL JDBC metadata search escape", + exception + ); + } + } + return searchEscape; + } + } + } + + /** + * 仅调整行类型的字段查找规则,关系转换继续交由原始 JDBC Table 完成。 + */ + private static final class MysqlCaseInsensitiveColumnTable + implements TranslatableTable, Wrapper { + + private final Table delegate; + + private MysqlCaseInsensitiveColumnTable(Table delegate) { + this.delegate = Objects.requireNonNull(delegate, "delegate"); + } + + /** + * 返回列名大小写不敏感的结构类型。 + * + * @param typeFactory Calcite 类型工厂 + * @return 包装后的结构类型 + */ + @Override + public RelDataType getRowType(RelDataTypeFactory typeFactory) { + return new CaseInsensitiveRelRecordType(delegate.getRowType(typeFactory)); + } + + /** + * 复用原始表统计信息。 + * + * @return 表统计信息 + */ + @Override + public Statistic getStatistic() { + return delegate.getStatistic(); + } + + /** + * 复用原始 JDBC 表类型。 + * + * @return JDBC 表类型 + */ + @Override + public Schema.TableType getJdbcTableType() { + return delegate.getJdbcTableType(); + } + + /** + * 判断列是否为预聚合列。 + * + * @param column 列名 + * @return 原始表判断结果 + */ + @Override + public boolean isRolledUp(String column) { + return delegate.isRolledUp(column); + } + + /** + * 判断预聚合列能否用于聚合表达式。 + * + * @param column 列名 + * @param call SQL 调用 + * @param parent 父节点 + * @param config Calcite 连接配置 + * @return 原始表判断结果 + */ + @Override + public boolean rolledUpColumnValidInsideAgg( + String column, + SqlCall call, + SqlNode parent, + CalciteConnectionConfig config + ) { + return delegate.rolledUpColumnValidInsideAgg(column, call, parent, config); + } + + /** + * 交由原始 JDBC Table 生成关系节点,保留 JDBC Convention 与 SQL 下推。 + * + * @param context 关系转换上下文 + * @param relOptTable 规划器表 + * @return 关系节点 + * @throws IllegalStateException 原始表不支持关系转换 + */ + @Override + public RelNode toRel(RelOptTable.ToRelContext context, RelOptTable relOptTable) { + if (!(delegate instanceof TranslatableTable translatableTable)) { + throw new IllegalStateException("MySQL JDBC table does not support relational translation"); + } + return translatableTable.toRel(context, relOptTable); + } + + /** + * 解包包装器或原始 JDBC Table 能力。 + * + * @param type 目标类型 + * @param 目标类型参数 + * @return 匹配实例;不存在时返回 null + */ + @Override + public C unwrap(Class type) { + if (type.isInstance(this)) { + return type.cast(this); + } + if (type.isInstance(delegate)) { + return type.cast(delegate); + } + return delegate instanceof Wrapper wrapper ? wrapper.unwrap(type) : null; + } + } + + /** + * 始终以大小写不敏感方式解析 MySQL 列名的记录类型。 + */ + private static final class CaseInsensitiveRelRecordType extends RelRecordType { + + private CaseInsensitiveRelRecordType(RelDataType delegate) { + super(delegate.getStructKind(), delegate.getFieldList(), delegate.isNullable()); + } + + /** + * 按 MySQL 规则查找字段。 + * + * @param fieldName 字段名 + * @param caseSensitive Calcite 请求的匹配规则;MySQL 列名语义下忽略 + * @param elideRecord 是否递归省略嵌套记录层级 + * @return 匹配字段;不存在时返回 null + */ + @Override + public RelDataTypeField getField( + String fieldName, + boolean caseSensitive, + boolean elideRecord + ) { + return super.getField(fieldName, false, elideRecord); + } + } +} diff --git a/easy-agents-federation-sql/easy-agents-federation-sql-adapter-jdbc/src/main/resources/META-INF/services/com.easyagents.federation.sql.adapter.FederationSqlAdapterProvider b/easy-agents-federation-sql/easy-agents-federation-sql-adapter-jdbc/src/main/resources/META-INF/services/com.easyagents.federation.sql.adapter.FederationSqlAdapterProvider new file mode 100644 index 0000000..59e3f21 --- /dev/null +++ b/easy-agents-federation-sql/easy-agents-federation-sql-adapter-jdbc/src/main/resources/META-INF/services/com.easyagents.federation.sql.adapter.FederationSqlAdapterProvider @@ -0,0 +1 @@ +com.easyagents.federation.sql.adapter.jdbc.JdbcFederationSqlAdapterProvider diff --git a/easy-agents-federation-sql/easy-agents-federation-sql-adapter-jdbc/src/test/java/com/easyagents/federation/sql/adapter/jdbc/JdbcDialectSelectionTest.java b/easy-agents-federation-sql/easy-agents-federation-sql-adapter-jdbc/src/test/java/com/easyagents/federation/sql/adapter/jdbc/JdbcDialectSelectionTest.java new file mode 100644 index 0000000..3ea3e04 --- /dev/null +++ b/easy-agents-federation-sql/easy-agents-federation-sql-adapter-jdbc/src/test/java/com/easyagents/federation/sql/adapter/jdbc/JdbcDialectSelectionTest.java @@ -0,0 +1,140 @@ +package com.easyagents.federation.sql.adapter.jdbc; + +import com.easyagents.federation.sql.adapter.AdapterCompatibilityStatus; +import com.easyagents.federation.sql.adapter.AdapterDialectContext; +import com.easyagents.federation.sql.adapter.AdapterHints; +import com.easyagents.federation.sql.source.FederationSourceDefinition; +import com.easyagents.federation.sql.source.SourceId; +import java.lang.reflect.Proxy; +import java.sql.DatabaseMetaData; +import java.util.List; +import java.util.Map; +import org.apache.calcite.sql.SqlDialect; +import org.apache.calcite.sql.dialect.MysqlSqlDialect; +import org.apache.calcite.sql.dialect.OracleSqlDialect; +import org.apache.calcite.sql.dialect.PostgresqlSqlDialect; +import org.junit.Assert; +import org.junit.Test; + +/** + * 默认 JDBC Adapter 的数据库识别与 Calcite 方言选择契约测试。 + */ +public class JdbcDialectSelectionTest { + + /** + * 验证 MySQL、PostgreSQL 和 Oracle 使用对应 Calcite 官方方言。 + * + * @throws Exception 元数据读取失败 + */ + @Test + public void shouldSelectBuiltInCalciteDialects() throws Exception { + JdbcFederationSqlAdapterProvider adapter = new JdbcFederationSqlAdapterProvider(); + SqlDialect mysql = assertDialect(adapter, "MySQL", "`", MysqlSqlDialect.class); + SqlDialect postgresql = assertDialect(adapter, "PostgreSQL", "\"", PostgresqlSqlDialect.class); + assertDialect(adapter, "Oracle", "\"", OracleSqlDialect.class); + Assert.assertEquals(mysql.isCaseSensitive(), adapter.parserConfig(mysql).caseSensitive()); + Assert.assertTrue(adapter.parserConfig(postgresql).caseSensitive()); + } + + /** + * 验证未知数据库默认拒绝,显式 ANSI 模式才以未验证状态放行。 + * + * @throws Exception 元数据读取失败 + */ + @Test + public void shouldRequireExplicitAnsiModeForUnknownDatabase() throws Exception { + JdbcFederationSqlAdapterProvider adapter = new JdbcFederationSqlAdapterProvider(); + DatabaseMetaData metadata = metadata("UnknownDB", "\""); + Assert.assertFalse(adapter.supports(metadata, new AdapterHints(Map.of()))); + AdapterHints experimental = new AdapterHints(Map.of( + JdbcFederationSqlAdapterProvider.EXPERIMENTAL_ANSI_OPTION, + "true" + )); + Assert.assertTrue(adapter.supports(metadata, experimental)); + Assert.assertEquals( + AdapterCompatibilityStatus.CODE_SUPPORTED_UNVERIFIED, + adapter.compatibility(metadata, experimental).status() + ); + } + + private static SqlDialect assertDialect( + JdbcFederationSqlAdapterProvider adapter, + String product, + String quote, + Class expectedType + ) throws Exception { + DatabaseMetaData metadata = metadata(product, quote); + Assert.assertTrue(adapter.supports(metadata, new AdapterHints(Map.of()))); + SqlDialect dialect = adapter.createDialect(new AdapterDialectContext(metadata, definition(Map.of()))); + Assert.assertTrue(expectedType.isInstance(dialect)); + Assert.assertEquals( + AdapterCompatibilityStatus.CODE_SUPPORTED_UNVERIFIED, + adapter.compatibility(metadata, new AdapterHints(Map.of())).status() + ); + return dialect; + } + + private static FederationSourceDefinition definition(Map options) { + return new FederationSourceDefinition( + new SourceId("source"), + 1, + JdbcFederationSqlAdapterProvider.ADAPTER_ID, + List.of(new JdbcSchemaDefinition("app", null, null)), + options + ); + } + + private static DatabaseMetaData metadata(String product, String quote) { + return (DatabaseMetaData) Proxy.newProxyInstance( + JdbcDialectSelectionTest.class.getClassLoader(), + new Class[] {DatabaseMetaData.class}, + (proxy, method, arguments) -> switch (method.getName()) { + case "getDatabaseProductName" -> product; + case "getDatabaseProductVersion" -> "test-version"; + case "getDatabaseMajorVersion" -> 1; + case "getDatabaseMinorVersion" -> 0; + case "getDriverName" -> "test-driver"; + case "getDriverVersion" -> "1"; + case "getIdentifierQuoteString" -> quote; + case "nullsAreSortedHigh" -> true; + case "nullsAreSortedAtEnd", "nullsAreSortedAtStart", "nullsAreSortedLow" -> false; + case "storesUpperCaseIdentifiers", "storesUpperCaseQuotedIdentifiers" -> false; + case "storesLowerCaseIdentifiers", "storesLowerCaseQuotedIdentifiers" -> false; + case "storesMixedCaseIdentifiers", "storesMixedCaseQuotedIdentifiers" -> true; + case "supportsMixedCaseIdentifiers", "supportsMixedCaseQuotedIdentifiers" -> true; + default -> defaultValue(method.getReturnType()); + } + ); + } + + private static Object defaultValue(Class type) { + if (!type.isPrimitive()) { + return null; + } + if (type == boolean.class) { + return false; + } + if (type == int.class) { + return 0; + } + if (type == long.class) { + return 0L; + } + if (type == short.class) { + return (short) 0; + } + if (type == byte.class) { + return (byte) 0; + } + if (type == float.class) { + return 0F; + } + if (type == double.class) { + return 0D; + } + if (type == char.class) { + return '\0'; + } + return null; + } +} diff --git a/easy-agents-federation-sql/easy-agents-federation-sql-adapter-jdbc/src/test/java/com/easyagents/federation/sql/adapter/jdbc/JdbcFederatedQueryEngineTest.java b/easy-agents-federation-sql/easy-agents-federation-sql-adapter-jdbc/src/test/java/com/easyagents/federation/sql/adapter/jdbc/JdbcFederatedQueryEngineTest.java new file mode 100644 index 0000000..eac235d --- /dev/null +++ b/easy-agents-federation-sql/easy-agents-federation-sql-adapter-jdbc/src/test/java/com/easyagents/federation/sql/adapter/jdbc/JdbcFederatedQueryEngineTest.java @@ -0,0 +1,774 @@ +package com.easyagents.federation.sql.adapter.jdbc; + +import com.easyagents.federation.sql.api.FederationSqlEngine; +import com.easyagents.federation.sql.api.FederationSqlEngines; +import com.easyagents.federation.sql.api.FederationSqlErrorCode; +import com.easyagents.federation.sql.api.FederationSqlException; +import com.easyagents.federation.sql.api.SqlQueryCommand; +import com.easyagents.federation.sql.compile.FederationSqlPlan; +import com.easyagents.federation.sql.compile.SqlCompileRequest; +import com.easyagents.federation.sql.compile.SqlExplainLevel; +import com.easyagents.federation.sql.compile.SqlExplainRequest; +import com.easyagents.federation.sql.compile.SqlExplainResult; +import com.easyagents.federation.sql.execute.FederationQueryMetricsSnapshot; +import com.easyagents.federation.sql.execute.FederationResultCursor; +import com.easyagents.federation.sql.execute.SqlParameter; +import com.easyagents.federation.sql.federation.FederationExecutionPolicy; +import com.easyagents.federation.sql.federation.FederationLogicalTableDefinition; +import com.easyagents.federation.sql.federation.FederationQueryMode; +import com.easyagents.federation.sql.federation.FederationQueryScopeDefinition; +import com.easyagents.federation.sql.federation.FederationSourceBindingDefinition; +import com.easyagents.federation.sql.federation.FederationStatisticsSnapshot; +import com.easyagents.federation.sql.federation.FederationTableStatistics; +import com.easyagents.federation.sql.federation.FederationTableStatisticsProvider; +import com.easyagents.federation.sql.source.FederationDataSourceHandles; +import com.easyagents.federation.sql.source.FederationSourceDefinition; +import com.easyagents.federation.sql.source.RuntimeFingerprint; +import com.easyagents.federation.sql.source.SourceApplyOptions; +import com.easyagents.federation.sql.source.SourceId; +import java.math.BigDecimal; +import java.time.Instant; +import java.sql.Connection; +import java.sql.Statement; +import java.sql.Types; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.atomic.AtomicReference; +import org.h2.jdbcx.JdbcDataSource; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +/** + * 多个独立 JDBC DataSource 的基础联邦查询集成测试。 + */ +public class JdbcFederatedQueryEngineTest { + + private static final SourceId SALES_SOURCE = new SourceId("sales-source"); + private static final SourceId BILLING_SOURCE = new SourceId("billing-source"); + private static final SourceId REGION_SOURCE = new SourceId("region-source"); + + private JdbcDataSource sales; + private JdbcDataSource billing; + private JdbcDataSource region; + private FederationSqlEngine engine; + private FederationQueryScopeDefinition scope; + private final AtomicReference statisticsVersion = + new AtomicReference<>("stats-v1"); + private final AtomicLong salesRowCount = new AtomicLong(3); + + /** + * 创建三个独立 H2 数据库并登记物理数据源。 + * + * @throws Exception 数据库初始化失败 + */ + @Before + public void setUp() throws Exception { + Instant statisticsCollectedAt = Instant.now(); + sales = dataSource("sales"); + billing = dataSource("billing"); + region = dataSource("region"); + execute(sales, + "CREATE TABLE CUSTOMER (ID INT PRIMARY KEY, NAME VARCHAR(64) NOT NULL)", + "INSERT INTO CUSTOMER VALUES (1, 'Alice'), (2, 'Bob'), (3, 'Carol')", + "CREATE TABLE TIME_EVENT (ID INT PRIMARY KEY, EVENT_TIME TIME(6) WITH TIME ZONE, " + + "EVENT_AT TIMESTAMP(6) WITH TIME ZONE)", + "INSERT INTO TIME_EVENT VALUES (1, TIME WITH TIME ZONE '12:00:00.123456+08:00', " + + "TIMESTAMP WITH TIME ZONE '2026-08-21 12:00:00.123456+08:00')", + "CREATE TABLE PRECISE_EVENT (ID INT PRIMARY KEY, EVENT_AT TIMESTAMP(6))", + "INSERT INTO PRECISE_EVENT VALUES " + + "(1, TIMESTAMP '2026-08-21 12:00:00.123456')"); + execute(billing, + "CREATE TABLE ORDER_ITEM (ID INT PRIMARY KEY, CUSTOMER_ID INT NOT NULL, AMOUNT DECIMAL(12,2), CODE VARCHAR(64))", + "INSERT INTO ORDER_ITEM VALUES (10, 1, 30.00, 'Alice'), (11, 1, 20.00, 'Alice'), (12, 2, 80.00, 'Bob')", + "CREATE TABLE TIME_EVENT (ID INT PRIMARY KEY, EVENT_TIME TIME(6) WITH TIME ZONE, " + + "EVENT_AT TIMESTAMP(6) WITH TIME ZONE)", + "INSERT INTO TIME_EVENT VALUES (2, TIME WITH TIME ZONE '06:00:00.123456+02:00', " + + "TIMESTAMP WITH TIME ZONE '2026-08-21 06:00:00.123456+02:00')", + "CREATE TABLE PRECISE_EVENT (ID INT PRIMARY KEY, EVENT_AT TIMESTAMP(6))", + "INSERT INTO PRECISE_EVENT VALUES " + + "(2, TIMESTAMP '2026-08-21 08:30:00.654321')"); + execute(region, + "CREATE TABLE CUSTOMER_REGION (CUSTOMER_ID INT PRIMARY KEY, REGION VARCHAR(64))", + "INSERT INTO CUSTOMER_REGION VALUES (1, 'North'), (2, 'South'), (3, 'West')"); + + RuntimeFingerprint fingerprint = new RuntimeFingerprint( + "H2", "2", "H2 JDBC Driver", "2", "1" + ); + engine = FederationSqlEngines.builder() + .dataSourceResolver(definition -> FederationDataSourceHandles.shared( + dataSourceFor(definition.sourceId()), + fingerprint + )) + .tableStatisticsProvider(() -> new FederationStatisticsSnapshot( + statisticsVersion.get(), + Map.of( + new FederationStatisticsSnapshot.TableKey( + SALES_SOURCE, "APP", "CUSTOMER" + ), + new FederationTableStatistics( + salesRowCount.get(), + 48, + statisticsCollectedAt, + "test-catalog" + ), + new FederationStatisticsSnapshot.TableKey( + BILLING_SOURCE, "APP", "ORDER_ITEM" + ), + new FederationTableStatistics( + 3, + 64, + statisticsCollectedAt, + "test-catalog" + ), + new FederationStatisticsSnapshot.TableKey( + REGION_SOURCE, "APP", "CUSTOMER_REGION" + ), + new FederationTableStatistics( + 3, + 32, + statisticsCollectedAt, + "test-catalog" + ) + ) + )) + .federationExecutionPolicy(threeSourcePolicy()) + .maximumPlanCacheEntries(32) + .build(); + engine.sources().apply(definition(SALES_SOURCE), SourceApplyOptions.prewarmNow()); + engine.sources().apply(definition(BILLING_SOURCE), SourceApplyOptions.prewarmNow()); + engine.sources().apply(definition(REGION_SOURCE), SourceApplyOptions.prewarmNow()); + + scope = new FederationQueryScopeDefinition( + "sales-billing", + 1, + Map.of( + "SALES", + FederationSourceBindingDefinition.of(SALES_SOURCE, 1), + "BILLING", + FederationSourceBindingDefinition.of(BILLING_SOURCE, 1) + ), + "SALES", + FederationExecutionPolicy.basic() + ); + } + + /** + * 关闭 Engine。 + */ + @After + public void tearDown() { + if (engine != null) { + engine.close(); + } + } + + /** + * 验证同一 Query Scope 中实际只引用一个源时保持完整单源下推。 + */ + @Test + public void shouldRouteSingleReferencedSourceToDirectExecution() { + FederationSqlPlan plan = engine.compile(SqlCompileRequest.of( + "SELECT NAME FROM SALES.APP.CUSTOMER ORDER BY ID", + scope + )); + + Assert.assertEquals(FederationQueryMode.SINGLE_SOURCE, plan.queryMode()); + Assert.assertEquals(1, plan.fragments().size()); + Assert.assertEquals(java.util.Set.of(SALES_SOURCE), plan.referencedSources()); + } + + /** + * 验证短逻辑表名、三段逻辑表名和跨源逻辑表 Join 共用底层映射。 + */ + @Test + public void shouldExecuteLogicalTableNamesAcrossSources() { + FederationQueryScopeDefinition logicalScope = + FederationQueryScopeDefinition.virtual( + "logical-sales-billing", + 2, + scope.bindings(), + "SALES", + List.of( + FederationLogicalTableDefinition.of( + "customers", "SALES", "APP", "CUSTOMER" + ), + FederationLogicalTableDefinition.of( + "order_lines", "BILLING", "APP", "ORDER_ITEM" + ) + ), + FederationExecutionPolicy.basic() + ); + + try (FederationResultCursor cursor = engine.query(SqlQueryCommand.of( + "SELECT customers.NAME FROM customers ORDER BY customers.ID", + logicalScope, + List.of() + ))) { + Assert.assertTrue(cursor.next()); + Assert.assertEquals("Alice", cursor.getObject(1)); + } + + try (FederationResultCursor cursor = engine.query(SqlQueryCommand.of( + "SELECT customers.NAME FROM SALES.APP.customers " + + "ORDER BY SALES.APP.customers.ID", + logicalScope, + List.of() + ))) { + Assert.assertTrue(cursor.next()); + Assert.assertEquals("Alice", cursor.getObject(1)); + } + + try (FederationResultCursor cursor = engine.query(SqlQueryCommand.of( + "SELECT c.NAME, o.AMOUNT FROM customers c " + + "JOIN order_lines o ON c.ID = o.CUSTOMER_ID " + + "ORDER BY o.ID", + logicalScope, + List.of() + ))) { + Assert.assertTrue(cursor.next()); + Assert.assertEquals(List.of("Alice", new BigDecimal("30.00")), cursor.row()); + } + } + + /** + * 验证全限定 SQL 不依赖未引用默认 Binding 的运行状态。 + */ + @Test + public void shouldNotAcquireUnavailableDefaultBindingWhenOnlyAnotherSourceIsReferenced() { + FederationQueryScopeDefinition unavailableDefault = + FederationQueryScopeDefinition.virtual( + "unavailable-default", + 1, + Map.of( + "OFFLINE", FederationSourceBindingDefinition.of( + new SourceId("offline-source"), 1 + ), + "BILLING", FederationSourceBindingDefinition.of(BILLING_SOURCE, 1) + ), + "OFFLINE", + FederationExecutionPolicy.basic() + ); + + FederationSqlPlan plan = engine.compile(SqlCompileRequest.of( + "SELECT AMOUNT FROM BILLING.APP.ORDER_ITEM", + unavailableDefault + )); + + Assert.assertEquals(FederationQueryMode.SINGLE_SOURCE, plan.queryMode()); + Assert.assertEquals(java.util.Set.of(BILLING_SOURCE), plan.referencedSources()); + + FederationSqlPlan ctePlan = engine.compile(SqlCompileRequest.of( + "WITH x AS (SELECT AMOUNT FROM BILLING.APP.ORDER_ITEM) SELECT * FROM x", + unavailableDefault + )); + Assert.assertEquals(FederationQueryMode.SINGLE_SOURCE, ctePlan.queryMode()); + Assert.assertEquals(java.util.Set.of(BILLING_SOURCE), ctePlan.referencedSources()); + } + + /** + * 验证跨源等值 Join、聚合、全局排序和查询指标。 + */ + @Test + public void shouldJoinAggregateAndSortAcrossTwoSources() { + String sql = "SELECT c.ID, SUM(o.AMOUNT) AS TOTAL " + + "FROM SALES.APP.CUSTOMER c " + + "JOIN BILLING.APP.ORDER_ITEM o ON c.ID = o.CUSTOMER_ID " + + "GROUP BY c.ID ORDER BY TOTAL DESC"; + FederationSqlPlan plan = engine.compile(SqlCompileRequest.of(sql, scope)); + Assert.assertEquals(FederationQueryMode.FEDERATED, plan.queryMode()); + Assert.assertEquals(2, plan.fragments().size()); + Assert.assertEquals( + "the smaller estimated input should become the Hash Join build side", + SALES_SOURCE, + plan.fragments().get(1).sourceId() + ); + + List> rows = new ArrayList<>(); + FederationQueryMetricsSnapshot finalMetrics; + try (FederationResultCursor cursor = engine.query( + SqlQueryCommand.of(sql, scope, List.of()) + )) { + while (cursor.next()) { + rows.add(cursor.row()); + } + finalMetrics = cursor.metrics(); + } + + Assert.assertEquals(2, rows.size()); + Assert.assertEquals(2, rows.get(0).get(0)); + Assert.assertEquals(new BigDecimal("80.00"), rows.get(0).get(1)); + Assert.assertEquals(1, rows.get(1).get(0)); + Assert.assertEquals(new BigDecimal("50.00"), rows.get(1).get(1)); + Assert.assertTrue(finalMetrics.complete()); + Assert.assertEquals(2, finalMetrics.returnedRows()); + Assert.assertTrue(finalMetrics.intermediateRows() >= 6); + Assert.assertEquals(2, finalMetrics.fragments().size()); + } + + /** + * 验证三个独立 JDBC 数据源可由同一计划完成 Join 并返回稳定结果。 + */ + @Test + public void shouldExecuteJoinAcrossThreeSources() { + FederationQueryScopeDefinition threeSourceScope = + new FederationQueryScopeDefinition( + "sales-billing-region", + 1, + Map.of( + "SALES", FederationSourceBindingDefinition.of(SALES_SOURCE, 1), + "BILLING", FederationSourceBindingDefinition.of(BILLING_SOURCE, 1), + "REGION", FederationSourceBindingDefinition.of(REGION_SOURCE, 1) + ), + "SALES", + threeSourcePolicy() + ); + String sql = "SELECT c.NAME, o.AMOUNT, r.REGION " + + "FROM SALES.APP.CUSTOMER c " + + "JOIN BILLING.APP.ORDER_ITEM o ON c.ID = o.CUSTOMER_ID " + + "JOIN REGION.APP.CUSTOMER_REGION r ON c.ID = r.CUSTOMER_ID " + + "ORDER BY o.ID"; + + FederationSqlPlan plan = engine.compile(SqlCompileRequest.of(sql, threeSourceScope)); + Assert.assertEquals(FederationQueryMode.FEDERATED, plan.queryMode()); + Assert.assertEquals(3, plan.referencedSources().size()); + Assert.assertEquals(3, plan.fragments().size()); + + List> rows = new ArrayList<>(); + try (FederationResultCursor cursor = engine.query( + SqlQueryCommand.of(sql, threeSourceScope, List.of()) + )) { + while (cursor.next()) { + rows.add(cursor.row()); + } + } + Assert.assertEquals(3, rows.size()); + Assert.assertEquals( + List.of("Alice", new BigDecimal("30.00"), "North"), + rows.get(0) + ); + Assert.assertEquals( + List.of("Bob", new BigDecimal("80.00"), "South"), + rows.get(2) + ); + } + + /** + * 验证两个分片分别绑定原始查询中的动态参数。 + */ + @Test + public void shouldMapParametersIntoDifferentFragments() { + String sql = "SELECT c.NAME, o.AMOUNT " + + "FROM SALES.APP.CUSTOMER c " + + "JOIN BILLING.APP.ORDER_ITEM o ON c.ID = o.CUSTOMER_ID " + + "WHERE c.ID > ? AND o.AMOUNT > ? ORDER BY o.AMOUNT"; + FederationSqlPlan plan = engine.compile(new SqlCompileRequest( + sql, + scope, + List.of(Types.INTEGER, Types.DECIMAL), + "default" + )); + Assert.assertEquals(2, plan.fragments().size()); + Assert.assertEquals(List.of(0), plan.fragments().get(0).parameterMapping()); + Assert.assertEquals(List.of(1), plan.fragments().get(1).parameterMapping()); + Assert.assertTrue(plan.fragments().stream() + .allMatch(fragment -> fragment.executableSql().toUpperCase().contains("WHERE"))); + try (FederationResultCursor cursor = engine.query(SqlQueryCommand.of( + sql, + scope, + List.of( + new SqlParameter(Types.INTEGER, 0), + new SqlParameter(Types.DECIMAL, new BigDecimal("25.00")) + ) + ))) { + Assert.assertTrue(cursor.next()); + Assert.assertEquals(List.of("Alice", new BigDecimal("30.00")), cursor.row()); + Assert.assertTrue(cursor.next()); + Assert.assertEquals(List.of("Bob", new BigDecimal("80.00")), cursor.row()); + Assert.assertFalse(cursor.next()); + } + + try (FederationResultCursor cursor = engine.query(SqlQueryCommand.of( + "SELECT NAME FROM SALES.APP.CUSTOMER ORDER BY ID " + + "OFFSET ? ROWS FETCH NEXT ? ROWS ONLY", + scope, + List.of( + new SqlParameter(Types.INTEGER, 1), + new SqlParameter(Types.INTEGER, 1) + ) + ))) { + Assert.assertTrue(cursor.next()); + Assert.assertEquals("Bob", cursor.getObject(1)); + Assert.assertFalse(cursor.next()); + } + } + + /** + * 验证 LEFT JOIN、UNION ALL、CTE 和全局分页使用同一联邦执行入口。 + */ + @Test + public void shouldExecuteBasicFederatedOperators() { + List> leftRows = query( + "SELECT c.NAME, o.AMOUNT FROM SALES.APP.CUSTOMER c " + + "LEFT JOIN BILLING.APP.ORDER_ITEM o ON c.ID = o.CUSTOMER_ID " + + "ORDER BY c.ID, o.ID" + ); + Assert.assertEquals(4, leftRows.size()); + Assert.assertEquals("Carol", leftRows.get(3).get(0)); + Assert.assertNull(leftRows.get(3).get(1)); + + List> unionRows = query( + "SELECT ID FROM SALES.APP.CUSTOMER " + + "UNION ALL SELECT CUSTOMER_ID FROM BILLING.APP.ORDER_ITEM ORDER BY ID" + ); + Assert.assertEquals(List.of(1, 1, 1, 2, 2, 3), + unionRows.stream().map(row -> row.get(0)).toList()); + + List> cteRows = query( + "WITH large_orders AS (" + + "SELECT CUSTOMER_ID, AMOUNT FROM BILLING.APP.ORDER_ITEM WHERE AMOUNT >= 50" + + ") SELECT c.NAME, o.AMOUNT FROM SALES.APP.CUSTOMER c " + + "JOIN large_orders o ON c.ID = o.CUSTOMER_ID " + + "ORDER BY o.AMOUNT DESC FETCH NEXT 1 ROWS ONLY" + ); + Assert.assertEquals(List.of(List.of("Bob", new BigDecimal("80.00"))), cteRows); + + List> residualRows = query( + "SELECT c.ID, c.NAME FROM SALES.APP.CUSTOMER c " + + "JOIN BILLING.APP.ORDER_ITEM o ON c.ID = o.CUSTOMER_ID " + + "WHERE c.ID < o.ID ORDER BY c.ID, o.ID" + ); + Assert.assertEquals(List.of("Alice", "Alice", "Bob"), + residualRows.stream().map(row -> row.get(1)).toList()); + } + + /** + * 验证本地联邦算子以 UTC Offset 类型返回 JDBC 4.2 时区值。 + */ + @Test + public void shouldPreserveTimezoneSemanticsAcrossFederatedUnion() { + List> rows = query( + "SELECT ID, EVENT_TIME, EVENT_AT FROM SALES.APP.TIME_EVENT " + + "UNION ALL SELECT ID, EVENT_TIME, EVENT_AT FROM BILLING.APP.TIME_EVENT " + + "ORDER BY ID" + ); + + Assert.assertEquals(2, rows.size()); + Assert.assertEquals( + java.time.OffsetTime.parse("04:00:00.123456Z"), + rows.get(0).get(1) + ); + Assert.assertEquals( + java.time.OffsetDateTime.parse("2026-08-21T04:00:00.123456Z"), + rows.get(0).get(2) + ); + Assert.assertEquals( + java.time.OffsetTime.parse("04:00:00.123456Z"), + rows.get(1).get(1) + ); + Assert.assertEquals( + java.time.OffsetDateTime.parse("2026-08-21T04:00:00.123456Z"), + rows.get(1).get(2) + ); + + List> joined = query( + "SELECT s.ID, b.ID FROM SALES.APP.TIME_EVENT s " + + "JOIN BILLING.APP.TIME_EVENT b ON s.EVENT_AT = b.EVENT_AT" + ); + Assert.assertEquals(List.of(List.of(1, 2)), joined); + } + + /** + * 验证逻辑 Explain 不访问数据库 Optimizer,物理 Explain 显式返回每个分片原生计划。 + */ + @Test + public void shouldExplainLogicalAndPhysicalFederatedPlans() { + String sql = "SELECT c.NAME, o.AMOUNT FROM SALES.APP.CUSTOMER c " + + "JOIN BILLING.APP.ORDER_ITEM o ON c.ID = o.CUSTOMER_ID"; + SqlCompileRequest compileRequest = SqlCompileRequest.of(sql, scope); + + SqlExplainResult logical = engine.explain(new SqlExplainRequest( + compileRequest, + SqlExplainLevel.LOGICAL + )); + Assert.assertEquals(FederationQueryMode.FEDERATED, logical.queryMode()); + Assert.assertEquals(2, logical.fragments().size()); + Assert.assertTrue(logical.fragments().stream() + .allMatch(fragment -> fragment.physicalExplain() == null)); + Assert.assertTrue(logical.fragments().stream().allMatch(fragment -> + fragment.costEstimate().estimatedRows() >= 0 + && fragment.costEstimate().estimatedRowWidthBytes() > 0 + && fragment.costEstimate().estimatedTransferBytes() >= 0 + && fragment.costEstimate().statisticsSource().contains("test-catalog") + && "stats-v1".equals( + fragment.costEstimate().statisticsSnapshotVersion() + ) + && !fragment.pushedDownOperators().isEmpty() + )); + + SqlExplainResult physical = engine.explain(new SqlExplainRequest(compileRequest)); + Assert.assertEquals(SqlExplainLevel.PHYSICAL, physical.level()); + Assert.assertEquals(2, physical.fragments().size()); + Assert.assertTrue(physical.fragments().stream().allMatch(fragment -> + fragment.physicalExplain() != null + && fragment.physicalExplain().available() + && !fragment.physicalExplain().nativePlan().isBlank() + )); + } + + /** + * 验证纯快照版本变化不扰动计划,实际引用表统计变化才触发重编译。 + */ + @Test + public void shouldRecompileWhenStatisticsSnapshotChanges() { + SqlCompileRequest request = SqlCompileRequest.of( + "SELECT NAME FROM SALES.APP.CUSTOMER ORDER BY ID", + scope + ); + FederationSqlPlan first = engine.compile(request); + FederationSqlPlan cacheHit = engine.compile(request); + Assert.assertSame(first.relRoot(), cacheHit.relRoot()); + + statisticsVersion.set("stats-v2"); + FederationSqlPlan versionOnly = engine.compile(request); + Assert.assertSame(first.relRoot(), versionOnly.relRoot()); + + salesRowCount.set(4); + FederationSqlPlan refreshed = engine.compile(request); + + Assert.assertNotSame(first.relRoot(), refreshed.relRoot()); + Assert.assertEquals( + "stats-v2", + refreshed.fragments().get(0).costEstimate().statisticsSnapshotVersion() + ); + } + + /** + * 验证不支持的跨源算子和中间结果预算超限均返回稳定错误。 + */ + @Test + public void shouldRejectUnsupportedOperatorsAndExceededBudget() { + assertCompileError( + "SELECT c.NAME FROM SALES.APP.CUSTOMER c " + + "JOIN BILLING.APP.ORDER_ITEM o ON c.ID < o.CUSTOMER_ID", + FederationSqlErrorCode.FEDERATION_OPERATOR_UNSUPPORTED + ); + assertCompileError( + "SELECT ID FROM SALES.APP.CUSTOMER " + + "UNION SELECT CUSTOMER_ID FROM BILLING.APP.ORDER_ITEM", + FederationSqlErrorCode.FEDERATION_OPERATOR_UNSUPPORTED + ); + assertCompileError( + "SELECT c.ID FROM SALES.APP.CUSTOMER c " + + "JOIN BILLING.APP.ORDER_ITEM o ON c.NAME = o.CODE", + FederationSqlErrorCode.FEDERATION_OPERATOR_UNSUPPORTED + ); + assertCompileError( + "SELECT c.ID FROM SALES.APP.CUSTOMER c " + + "JOIN BILLING.APP.ORDER_ITEM o ON c.ID = o.CUSTOMER_ID " + + "AND c.NAME < o.CODE", + FederationSqlErrorCode.FEDERATION_OPERATOR_UNSUPPORTED + ); + assertCompileError( + "SELECT c.ID FROM SALES.APP.CUSTOMER c " + + "JOIN BILLING.APP.ORDER_ITEM o ON c.ID = o.CUSTOMER_ID " + + "AND c.ID < o.ID", + FederationSqlErrorCode.FEDERATION_OPERATOR_UNSUPPORTED + ); + assertCompileError( + "SELECT c.ID FROM SALES.APP.CUSTOMER c " + + "JOIN BILLING.APP.ORDER_ITEM o ON c.ID = o.CUSTOMER_ID " + + "AND c.NAME LIKE o.CODE", + FederationSqlErrorCode.FEDERATION_OPERATOR_UNSUPPORTED + ); + assertCompileError( + "SELECT c.NAME FROM SALES.APP.CUSTOMER c " + + "JOIN BILLING.APP.ORDER_ITEM o ON c.ID = o.CUSTOMER_ID " + + "ORDER BY c.NAME", + FederationSqlErrorCode.FEDERATION_OPERATOR_UNSUPPORTED + ); + assertCompileError( + "SELECT c.NAME, COUNT(*) FROM SALES.APP.CUSTOMER c " + + "JOIN BILLING.APP.ORDER_ITEM o ON c.ID = o.CUSTOMER_ID " + + "GROUP BY c.NAME", + FederationSqlErrorCode.FEDERATION_OPERATOR_UNSUPPORTED + ); + assertQueryError( + "SELECT EVENT_AT FROM SALES.APP.PRECISE_EVENT " + + "UNION ALL SELECT EVENT_AT FROM BILLING.APP.PRECISE_EVENT", + FederationSqlErrorCode.FEDERATION_OPERATOR_UNSUPPORTED + ); + + FederationQueryScopeDefinition strictScope = FederationQueryScopeDefinition.virtual( + "strict-budget", + 2, + scope.bindings(), + scope.defaultBinding(), + new FederationExecutionPolicy(2, 8, 2, 1, 1024, 60_000) + ); + try (FederationResultCursor cursor = engine.query(SqlQueryCommand.of( + "SELECT c.NAME, o.AMOUNT FROM SALES.APP.CUSTOMER c " + + "JOIN BILLING.APP.ORDER_ITEM o ON c.ID = o.CUSTOMER_ID", + strictScope, + List.of() + ))) { + cursor.next(); + Assert.fail("intermediate row budget should reject the query"); + } catch (FederationSqlException exception) { + Assert.assertEquals( + FederationSqlErrorCode.FEDERATION_RESOURCE_LIMIT_EXCEEDED, + exception.errorCode() + ); + } + + FederationQueryScopeDefinition localExpansionScope = + FederationQueryScopeDefinition.virtual( + "local-expansion-budget", + 3, + scope.bindings(), + scope.defaultBinding(), + new FederationExecutionPolicy(2, 8, 2, 7, 64_000, 60_000) + ); + try (FederationResultCursor cursor = engine.query(SqlQueryCommand.of( + "SELECT COUNT(*) FROM SALES.APP.CUSTOMER c " + + "JOIN BILLING.APP.ORDER_ITEM o ON c.ID = o.CUSTOMER_ID", + localExpansionScope, + List.of() + ))) { + cursor.next(); + Assert.fail("local join expansion should consume the intermediate row budget"); + } catch (FederationSqlException exception) { + Assert.assertEquals( + FederationSqlErrorCode.FEDERATION_RESOURCE_LIMIT_EXCEEDED, + exception.errorCode() + ); + } + + Assert.assertFalse(query( + "SELECT c.NAME FROM SALES.APP.CUSTOMER c " + + "JOIN BILLING.APP.ORDER_ITEM o ON c.ID = o.CUSTOMER_ID" + ).isEmpty()); + + FederationQueryScopeDefinition singleFragmentSlot = + FederationQueryScopeDefinition.virtual( + "single-fragment-slot", + 3, + scope.bindings(), + scope.defaultBinding(), + new FederationExecutionPolicy(2, 8, 1, 100, 64_000, 2_000) + ); + try (FederationResultCursor cursor = engine.query(SqlQueryCommand.of( + "SELECT c.NAME FROM SALES.APP.CUSTOMER c " + + "JOIN BILLING.APP.ORDER_ITEM o ON c.ID = o.CUSTOMER_ID", + singleFragmentSlot, + List.of() + ))) { + Assert.assertTrue(cursor.next()); + } + } + + private List> query(String sql) { + List> rows = new ArrayList<>(); + try (FederationResultCursor cursor = engine.query( + SqlQueryCommand.of(sql, scope, List.of()) + )) { + while (cursor.next()) { + rows.add(cursor.row()); + } + } + return rows; + } + + private void assertCompileError(String sql, FederationSqlErrorCode expected) { + try { + engine.compile(SqlCompileRequest.of(sql, scope)); + Assert.fail("SQL should have been rejected: " + sql); + } catch (FederationSqlException exception) { + Assert.assertEquals(expected, exception.errorCode()); + } + } + + /** + * 断言联邦查询在执行阶段返回指定稳定错误。 + * + * @param sql 待执行 SQL + * @param expected 预期错误码 + */ + private void assertQueryError(String sql, FederationSqlErrorCode expected) { + try (FederationResultCursor cursor = engine.query( + SqlQueryCommand.of(sql, scope, List.of()) + )) { + cursor.next(); + Assert.fail("SQL execution should have been rejected: " + sql); + } catch (FederationSqlException exception) { + Assert.assertEquals(expected, exception.errorCode()); + } + } + + private static JdbcDataSource dataSource(String name) { + JdbcDataSource dataSource = new JdbcDataSource(); + dataSource.setURL( + "jdbc:h2:mem:federation_" + name + '_' + System.nanoTime() + + ";DB_CLOSE_DELAY=-1" + ); + return dataSource; + } + + /** + * 根据物理源选择测试数据库。 + * + * @param sourceId 物理数据源标识 + * @return 对应测试数据源 + */ + private JdbcDataSource dataSourceFor(SourceId sourceId) { + if (sourceId.equals(SALES_SOURCE)) { + return sales; + } + if (sourceId.equals(BILLING_SOURCE)) { + return billing; + } + if (sourceId.equals(REGION_SOURCE)) { + return region; + } + throw new IllegalArgumentException("unknown test source: " + sourceId.value()); + } + + /** + * 返回允许三源执行且限制并发分片数的测试策略。 + * + * @return 三源执行策略 + */ + private static FederationExecutionPolicy threeSourcePolicy() { + return new FederationExecutionPolicy( + 3, + 8, + 2, + 100_000, + 64L * 1024L * 1024L, + 60_000 + ); + } + + private static void execute(JdbcDataSource dataSource, String... statements) + throws Exception { + try (Connection connection = dataSource.getConnection(); + Statement statement = connection.createStatement()) { + for (String sql : statements) { + statement.execute(sql); + } + } + } + + private static FederationSourceDefinition definition(SourceId sourceId) { + return new FederationSourceDefinition( + sourceId, + 1, + JdbcFederationSqlAdapterProvider.ADAPTER_ID, + List.of(new JdbcSchemaDefinition("APP", null, "PUBLIC")), + Map.of() + ); + } +} diff --git a/easy-agents-federation-sql/easy-agents-federation-sql-adapter-jdbc/src/test/java/com/easyagents/federation/sql/adapter/jdbc/JdbcFederationFragmentExecutorTest.java b/easy-agents-federation-sql/easy-agents-federation-sql-adapter-jdbc/src/test/java/com/easyagents/federation/sql/adapter/jdbc/JdbcFederationFragmentExecutorTest.java new file mode 100644 index 0000000..7ad7d35 --- /dev/null +++ b/easy-agents-federation-sql/easy-agents-federation-sql-adapter-jdbc/src/test/java/com/easyagents/federation/sql/adapter/jdbc/JdbcFederationFragmentExecutorTest.java @@ -0,0 +1,756 @@ +package com.easyagents.federation.sql.adapter.jdbc; + +import com.easyagents.federation.sql.adapter.AdapterCompatibility; +import com.easyagents.federation.sql.adapter.AdapterCompatibilityStatus; +import com.easyagents.federation.sql.api.FederationSqlErrorCode; +import com.easyagents.federation.sql.api.FederationSqlException; +import com.easyagents.federation.sql.execute.FederationExecutionGuard; +import com.easyagents.federation.sql.execute.FederationExecutionObserver; +import com.easyagents.federation.sql.execute.FederationFragmentExecutionContext; +import com.easyagents.federation.sql.execute.QueryId; +import com.easyagents.federation.sql.execute.SqlExecutionOptions; +import com.easyagents.federation.sql.execute.StatementLifecycle; +import java.io.ByteArrayInputStream; +import java.io.InputStream; +import java.io.PrintWriter; +import java.io.Reader; +import java.io.StringReader; +import java.lang.reflect.Proxy; +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.SQLTimeoutException; +import java.sql.SQLTransientConnectionException; +import java.sql.Statement; +import java.util.List; +import java.util.Map; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; +import java.util.logging.Logger; +import javax.sql.DataSource; +import org.junit.Assert; +import org.junit.Test; + +/** + * JDBC 分片执行器的连接获取错误边界测试。 + */ +public class JdbcFederationFragmentExecutorTest { + + /** + * 验证连接池等待跨过查询截止时间时保留统一查询超时错误。 + */ + @Test + public void shouldPreferQueryDeadlineOverConnectionPoolTimeout() { + AtomicInteger checks = new AtomicInteger(); + FederationExecutionGuard guard = new FederationExecutionGuard() { + @Override + public void ensureAllowed() { + if (checks.incrementAndGet() > 1) { + throw new FederationSqlException( + FederationSqlErrorCode.QUERY_TIMEOUT, + "query deadline reached" + ); + } + } + + @Override + public long remainingNanos() { + return 1L; + } + }; + + FederationSqlException failure = expectFailure(context(guard)); + + Assert.assertEquals(FederationSqlErrorCode.QUERY_TIMEOUT, failure.errorCode()); + } + + /** + * 验证截止时间仍有效时保留连接获取超时分类。 + */ + @Test + public void shouldReportConnectionAcquisitionTimeoutBeforeQueryDeadline() { + FederationSqlException failure = expectFailure( + context(FederationExecutionGuard.none()) + ); + + Assert.assertEquals( + FederationSqlErrorCode.CONNECTION_ACQUISITION_TIMEOUT, + failure.errorCode() + ); + } + + /** + * 验证显式取消先到达时,驱动的 SQLTimeoutException 不会覆盖取消终态。 + */ + @Test + public void shouldPreserveCancellationWhenDriverReportsExecutionTimeout() { + TerminalLifecycle lifecycle = new TerminalLifecycle(false, true); + AtomicBoolean statementClosed = new AtomicBoolean(); + AtomicBoolean connectionClosed = new AtomicBoolean(); + + FederationSqlException failure = expectFailure(context( + FederationExecutionGuard.none(), + executionTimeoutDataSource(statementClosed, connectionClosed), + lifecycle + )); + + Assert.assertEquals(FederationSqlErrorCode.QUERY_CANCELLED, failure.errorCode()); + Assert.assertTrue(lifecycle.unregistered.get()); + Assert.assertTrue(statementClosed.get()); + Assert.assertTrue(connectionClosed.get()); + } + + /** + * 验证结果读取阶段同样保留已经先到达的显式取消终态。 + */ + @Test + public void shouldPreserveCancellationWhenDriverReportsResultTimeout() { + TerminalLifecycle lifecycle = new TerminalLifecycle(false, true); + JdbcFederationResultCursor cursor = failingCursor( + lifecycle, + new SQLTimeoutException("driver reported timeout after cancel") + ); + + FederationSqlException failure = expectCursorFailure(cursor); + + Assert.assertEquals(FederationSqlErrorCode.QUERY_CANCELLED, failure.errorCode()); + Assert.assertTrue(lifecycle.unregistered.get()); + } + + /** + * 验证统一超时先到达时,驱动普通异常仍保持超时终态。 + */ + @Test + public void shouldPreserveTimeoutWhenDriverReportsGenericReadFailure() { + TerminalLifecycle lifecycle = new TerminalLifecycle(true, true); + JdbcFederationResultCursor cursor = failingCursor( + lifecycle, + new SQLException("statement was closed by timeout task") + ); + + FederationSqlException failure = expectCursorFailure(cursor); + + Assert.assertEquals(FederationSqlErrorCode.QUERY_TIMEOUT, failure.errorCode()); + Assert.assertTrue(lifecycle.unregistered.get()); + } + + /** + * 验证二进制流取得后发生取消时,后续流读取立即终止并释放 JDBC 资源。 + * + * @throws Exception 流读取失败 + */ + @Test + public void shouldStopBinaryStreamReadAfterCancellation() throws Exception { + AtomicBoolean cancelled = new AtomicBoolean(); + TerminalLifecycle lifecycle = new TerminalLifecycle(false, false); + JdbcFederationResultCursor cursor = streamingCursor( + lifecycle, + cancellationGuard(cancelled) + ); + + InputStream stream = cursor.getBinaryStream(1); + cancelled.set(true); + FederationSqlException failure = expectStreamFailure(stream); + + Assert.assertEquals(FederationSqlErrorCode.QUERY_CANCELLED, failure.errorCode()); + Assert.assertTrue(lifecycle.unregistered.get()); + } + + /** + * 验证字符流取得后发生超时时,后续流读取立即终止并释放 JDBC 资源。 + * + * @throws Exception 流读取失败 + */ + @Test + public void shouldStopCharacterStreamReadAfterTimeout() throws Exception { + AtomicBoolean timedOut = new AtomicBoolean(); + TerminalLifecycle lifecycle = new TerminalLifecycle(false, false); + JdbcFederationResultCursor cursor = streamingCursor( + lifecycle, + timeoutGuard(timedOut) + ); + + Reader reader = cursor.getCharacterStream(2); + timedOut.set(true); + FederationSqlException failure = expectStreamFailure(reader); + + Assert.assertEquals(FederationSqlErrorCode.QUERY_TIMEOUT, failure.errorCode()); + Assert.assertTrue(lifecycle.unregistered.get()); + } + + /** + * 验证阻塞的结果读取可由 Statement.cancel 解阻,并确定性释放全部 JDBC 资源。 + * + * @throws Exception 并发测试等待失败 + */ + @Test + public void shouldCancelBlockingResultReadAndCloseAllResources() throws Exception { + CountDownLatch readStarted = new CountDownLatch(1); + CountDownLatch cancelSignal = new CountDownLatch(1); + AtomicBoolean resultSetClosed = new AtomicBoolean(); + AtomicBoolean statementClosed = new AtomicBoolean(); + AtomicBoolean connectionClosed = new AtomicBoolean(); + CancellableLifecycle lifecycle = new CancellableLifecycle(); + DataSource dataSource = blockingReadDataSource( + readStarted, + cancelSignal, + resultSetClosed, + statementClosed, + connectionClosed + ); + JdbcFederationResultCursor cursor = (JdbcFederationResultCursor) + new JdbcFederationFragmentExecutor().execute(context( + FederationExecutionGuard.none(), + dataSource, + lifecycle + )); + ExecutorService executor = Executors.newSingleThreadExecutor(); + try { + Future read = executor.submit( + () -> expectCursorFailure(cursor) + ); + Assert.assertTrue(readStarted.await(2, TimeUnit.SECONDS)); + + lifecycle.requestCancellation(); + FederationSqlException failure = read.get(2, TimeUnit.SECONDS); + + Assert.assertEquals( + FederationSqlErrorCode.QUERY_CANCELLED, + failure.errorCode() + ); + Assert.assertTrue(lifecycle.unregistered.get()); + Assert.assertTrue(resultSetClosed.get()); + Assert.assertTrue(statementClosed.get()); + Assert.assertTrue(connectionClosed.get()); + } finally { + executor.shutdownNow(); + executor.awaitTermination(2, TimeUnit.SECONDS); + } + } + + private static FederationSqlException expectFailure( + FederationFragmentExecutionContext context + ) { + try { + new JdbcFederationFragmentExecutor().execute(context); + Assert.fail("expected connection acquisition to fail"); + return null; + } catch (FederationSqlException exception) { + return exception; + } + } + + private static FederationFragmentExecutionContext context( + FederationExecutionGuard guard + ) { + return context(guard, new FailingDataSource(), new TerminalLifecycle(false, false)); + } + + private static FederationFragmentExecutionContext context( + FederationExecutionGuard guard, + DataSource dataSource, + StatementLifecycle lifecycle + ) { + return new FederationFragmentExecutionContext( + QueryId.create(), + "SELECT 1", + List.of(), + SqlExecutionOptions.defaults(), + dataSource, + new AdapterCompatibility( + AdapterCompatibilityStatus.VERIFIED, + "test", + "1", + "test", + "1", + "test" + ), + Map.of(), + lifecycle, + null, + guard + ); + } + + private static DataSource executionTimeoutDataSource( + AtomicBoolean statementClosed, + AtomicBoolean connectionClosed + ) { + PreparedStatement statement = (PreparedStatement) Proxy.newProxyInstance( + JdbcFederationFragmentExecutorTest.class.getClassLoader(), + new Class[] {PreparedStatement.class}, + (proxy, method, arguments) -> { + if ("executeQuery".equals(method.getName())) { + throw new SQLTimeoutException("driver reported timeout after cancel"); + } + if ("close".equals(method.getName())) { + statementClosed.set(true); + } + return defaultValue(method.getReturnType()); + } + ); + Connection connection = (Connection) Proxy.newProxyInstance( + JdbcFederationFragmentExecutorTest.class.getClassLoader(), + new Class[] {Connection.class}, + (proxy, method, arguments) -> { + if ("prepareStatement".equals(method.getName())) { + return statement; + } + if ("close".equals(method.getName())) { + connectionClosed.set(true); + } + return defaultValue(method.getReturnType()); + } + ); + return dataSource(connection); + } + + /** + * 创建在 ResultSet.next 中等待 Statement.cancel 的 JDBC 代理。 + * + * @param readStarted 结果读取已开始信号 + * @param cancelSignal Statement 已取消信号 + * @param resultSetClosed ResultSet 关闭标记 + * @param statementClosed Statement 关闭标记 + * @param connectionClosed Connection 关闭标记 + * @return 可执行阻塞读取的 DataSource + */ + private static DataSource blockingReadDataSource( + CountDownLatch readStarted, + CountDownLatch cancelSignal, + AtomicBoolean resultSetClosed, + AtomicBoolean statementClosed, + AtomicBoolean connectionClosed + ) { + Object metadata = Proxy.newProxyInstance( + JdbcFederationFragmentExecutorTest.class.getClassLoader(), + new Class[] {java.sql.ResultSetMetaData.class}, + (proxy, method, arguments) -> defaultValue(method.getReturnType()) + ); + ResultSet resultSet = (ResultSet) Proxy.newProxyInstance( + JdbcFederationFragmentExecutorTest.class.getClassLoader(), + new Class[] {ResultSet.class}, + (proxy, method, arguments) -> { + if ("getMetaData".equals(method.getName())) { + return metadata; + } + if ("next".equals(method.getName())) { + readStarted.countDown(); + try { + if (!cancelSignal.await(2, TimeUnit.SECONDS)) { + throw new SQLException("test cancellation did not arrive"); + } + } catch (InterruptedException exception) { + Thread.currentThread().interrupt(); + throw new SQLException("blocking read was interrupted", exception); + } + throw new SQLException("driver read cancelled"); + } + if ("close".equals(method.getName())) { + resultSetClosed.set(true); + } + return defaultValue(method.getReturnType()); + } + ); + PreparedStatement statement = (PreparedStatement) Proxy.newProxyInstance( + JdbcFederationFragmentExecutorTest.class.getClassLoader(), + new Class[] {PreparedStatement.class}, + (proxy, method, arguments) -> { + if ("executeQuery".equals(method.getName())) { + return resultSet; + } + if ("cancel".equals(method.getName())) { + cancelSignal.countDown(); + } + if ("close".equals(method.getName())) { + statementClosed.set(true); + } + return defaultValue(method.getReturnType()); + } + ); + Connection connection = (Connection) Proxy.newProxyInstance( + JdbcFederationFragmentExecutorTest.class.getClassLoader(), + new Class[] {Connection.class}, + (proxy, method, arguments) -> { + if ("isReadOnly".equals(method.getName())) { + return true; + } + if ("prepareStatement".equals(method.getName())) { + return statement; + } + if ("close".equals(method.getName())) { + connectionClosed.set(true); + } + return defaultValue(method.getReturnType()); + } + ); + return dataSource(connection); + } + + private static JdbcFederationResultCursor failingCursor( + StatementLifecycle lifecycle, + SQLException readFailure + ) { + ResultSet resultSet = (ResultSet) Proxy.newProxyInstance( + JdbcFederationFragmentExecutorTest.class.getClassLoader(), + new Class[] {ResultSet.class}, + (proxy, method, arguments) -> { + if ("next".equals(method.getName())) { + throw readFailure; + } + return defaultValue(method.getReturnType()); + } + ); + PreparedStatement statement = (PreparedStatement) Proxy.newProxyInstance( + JdbcFederationFragmentExecutorTest.class.getClassLoader(), + new Class[] {PreparedStatement.class}, + (proxy, method, arguments) -> defaultValue(method.getReturnType()) + ); + Connection connection = (Connection) Proxy.newProxyInstance( + JdbcFederationFragmentExecutorTest.class.getClassLoader(), + new Class[] {Connection.class}, + (proxy, method, arguments) -> defaultValue(method.getReturnType()) + ); + return new JdbcFederationResultCursor( + QueryId.create(), + List.of(), + resultSet, + statement, + connection, + lifecycle + ); + } + + /** + * 创建可返回二进制流和字符流的测试游标。 + * + * @param lifecycle Statement 生命周期 + * @param guard 查询终态检查器 + * @return 测试游标 + */ + private static JdbcFederationResultCursor streamingCursor( + StatementLifecycle lifecycle, + FederationExecutionGuard guard + ) { + ResultSet resultSet = (ResultSet) Proxy.newProxyInstance( + JdbcFederationFragmentExecutorTest.class.getClassLoader(), + new Class[] {ResultSet.class}, + (proxy, method, arguments) -> { + if ("getBinaryStream".equals(method.getName())) { + return new ByteArrayInputStream(new byte[] {1, 2, 3}); + } + if ("getCharacterStream".equals(method.getName())) { + return new StringReader("streamed value"); + } + return defaultValue(method.getReturnType()); + } + ); + PreparedStatement statement = (PreparedStatement) Proxy.newProxyInstance( + JdbcFederationFragmentExecutorTest.class.getClassLoader(), + new Class[] {PreparedStatement.class}, + (proxy, method, arguments) -> defaultValue(method.getReturnType()) + ); + Connection connection = (Connection) Proxy.newProxyInstance( + JdbcFederationFragmentExecutorTest.class.getClassLoader(), + new Class[] {Connection.class}, + (proxy, method, arguments) -> defaultValue(method.getReturnType()) + ); + return new JdbcFederationResultCursor( + QueryId.create(), + List.of(), + resultSet, + statement, + connection, + lifecycle, + guard, + FederationExecutionObserver.none() + ); + } + + /** + * 创建由布尔终态驱动的取消检查器。 + * + * @param cancelled 是否已取消 + * @return 取消检查器 + */ + private static FederationExecutionGuard cancellationGuard(AtomicBoolean cancelled) { + return terminalGuard( + cancelled, + FederationSqlErrorCode.QUERY_CANCELLED, + "query was cancelled" + ); + } + + /** + * 创建由布尔终态驱动的超时检查器。 + * + * @param timedOut 是否已超时 + * @return 超时检查器 + */ + private static FederationExecutionGuard timeoutGuard(AtomicBoolean timedOut) { + return terminalGuard( + timedOut, + FederationSqlErrorCode.QUERY_TIMEOUT, + "query deadline reached" + ); + } + + /** + * 创建固定错误语义的查询终态检查器。 + * + * @param terminal 是否进入终态 + * @param errorCode 终态错误码 + * @param message 错误消息 + * @return 查询终态检查器 + */ + private static FederationExecutionGuard terminalGuard( + AtomicBoolean terminal, + FederationSqlErrorCode errorCode, + String message + ) { + return new FederationExecutionGuard() { + @Override + public void ensureAllowed() { + if (terminal.get()) { + throw new FederationSqlException(errorCode, message); + } + } + + @Override + public long remainingNanos() { + return Long.MAX_VALUE; + } + }; + } + + /** + * 读取二进制流并捕获预期的统一异常。 + * + * @param stream 测试流 + * @return 捕获的统一异常 + * @throws Exception 非预期读取错误 + */ + private static FederationSqlException expectStreamFailure(InputStream stream) + throws Exception { + try { + stream.read(); + Assert.fail("expected binary stream read to fail"); + return null; + } catch (FederationSqlException exception) { + return exception; + } + } + + /** + * 读取字符流并捕获预期的统一异常。 + * + * @param reader 测试 Reader + * @return 捕获的统一异常 + * @throws Exception 非预期读取错误 + */ + private static FederationSqlException expectStreamFailure(Reader reader) + throws Exception { + try { + reader.read(); + Assert.fail("expected character stream read to fail"); + return null; + } catch (FederationSqlException exception) { + return exception; + } + } + + private static FederationSqlException expectCursorFailure( + JdbcFederationResultCursor cursor + ) { + try { + cursor.next(); + Assert.fail("expected result read to fail"); + return null; + } catch (FederationSqlException exception) { + return exception; + } + } + + private static DataSource dataSource(Connection connection) { + return (DataSource) Proxy.newProxyInstance( + JdbcFederationFragmentExecutorTest.class.getClassLoader(), + new Class[] {DataSource.class}, + (proxy, method, arguments) -> { + if ("getConnection".equals(method.getName())) { + return connection; + } + if ("getParentLogger".equals(method.getName())) { + return Logger.getGlobal(); + } + return defaultValue(method.getReturnType()); + } + ); + } + + private static Object defaultValue(Class type) { + if (!type.isPrimitive()) { + return null; + } + if (type == boolean.class) { + return false; + } + if (type == byte.class) { + return (byte) 0; + } + if (type == short.class) { + return (short) 0; + } + if (type == int.class) { + return 0; + } + if (type == long.class) { + return 0L; + } + if (type == float.class) { + return 0F; + } + if (type == double.class) { + return 0D; + } + if (type == char.class) { + return '\0'; + } + return null; + } + + /** 记录测试所需的查询终态和注销动作。 */ + private static final class TerminalLifecycle implements StatementLifecycle { + + private final boolean timedOut; + private final boolean cancelled; + private final AtomicBoolean unregistered = new AtomicBoolean(); + + /** + * 创建固定终态的生命周期。 + * + * @param timedOut 是否已超时 + * @param cancelled 是否已取消 + */ + private TerminalLifecycle(boolean timedOut, boolean cancelled) { + this.timedOut = timedOut; + this.cancelled = cancelled; + } + + /** {@inheritDoc} */ + @Override + public void register(Statement statement) { + } + + /** {@inheritDoc} */ + @Override + public void unregister(Statement statement) { + unregistered.set(true); + } + + /** {@inheritDoc} */ + @Override + public boolean cancellationRequested() { + return cancelled; + } + + /** {@inheritDoc} */ + @Override + public boolean timeoutRequested() { + return timedOut; + } + } + + /** 可从测试线程触发 Statement.cancel 的生命周期。 */ + private static final class CancellableLifecycle implements StatementLifecycle { + + private final AtomicReference statement = new AtomicReference<>(); + private final AtomicBoolean cancelled = new AtomicBoolean(); + private final AtomicBoolean unregistered = new AtomicBoolean(); + + /** {@inheritDoc} */ + @Override + public void register(Statement candidate) { + statement.set(candidate); + } + + /** {@inheritDoc} */ + @Override + public void unregister(Statement candidate) { + statement.compareAndSet(candidate, null); + unregistered.set(true); + } + + /** {@inheritDoc} */ + @Override + public boolean cancellationRequested() { + return cancelled.get(); + } + + /** + * 标记查询取消并调用已登记 Statement 的取消入口。 + * + * @throws SQLException JDBC 取消失败 + */ + private void requestCancellation() throws SQLException { + cancelled.set(true); + Statement active = statement.get(); + if (active != null) { + active.cancel(); + } + } + } + + /** 始终返回瞬时连接池超时的测试 DataSource。 */ + private static final class FailingDataSource implements DataSource { + + @Override + public Connection getConnection() throws SQLException { + throw new SQLTransientConnectionException("pool timeout"); + } + + @Override + public Connection getConnection(String username, String password) throws SQLException { + throw new SQLTransientConnectionException("pool timeout"); + } + + @Override + public PrintWriter getLogWriter() { + return null; + } + + @Override + public void setLogWriter(PrintWriter out) { + } + + @Override + public void setLoginTimeout(int seconds) { + } + + @Override + public int getLoginTimeout() { + return 0; + } + + @Override + public Logger getParentLogger() { + return Logger.getGlobal(); + } + + @Override + public T unwrap(Class iface) throws SQLException { + throw new SQLException("unsupported"); + } + + @Override + public boolean isWrapperFor(Class iface) { + return false; + } + } +} diff --git a/easy-agents-federation-sql/easy-agents-federation-sql-adapter-jdbc/src/test/java/com/easyagents/federation/sql/adapter/jdbc/JdbcFederationFragmentExplainerTest.java b/easy-agents-federation-sql/easy-agents-federation-sql-adapter-jdbc/src/test/java/com/easyagents/federation/sql/adapter/jdbc/JdbcFederationFragmentExplainerTest.java new file mode 100644 index 0000000..be6ce99 --- /dev/null +++ b/easy-agents-federation-sql/easy-agents-federation-sql-adapter-jdbc/src/test/java/com/easyagents/federation/sql/adapter/jdbc/JdbcFederationFragmentExplainerTest.java @@ -0,0 +1,152 @@ +package com.easyagents.federation.sql.adapter.jdbc; + +import com.easyagents.federation.sql.adapter.AdapterCompatibility; +import com.easyagents.federation.sql.adapter.AdapterCompatibilityStatus; +import com.easyagents.federation.sql.api.FederationSqlErrorCode; +import com.easyagents.federation.sql.api.FederationSqlException; +import com.easyagents.federation.sql.execute.FederationExecutionGuard; +import com.easyagents.federation.sql.execute.FederationFragmentExplainContext; +import java.io.PrintWriter; +import java.sql.Connection; +import java.sql.SQLException; +import java.util.List; +import java.util.Map; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.logging.Logger; +import javax.sql.DataSource; +import org.junit.Assert; +import org.junit.Test; + +/** + * JDBC 物理 Explain 的连接获取错误边界测试。 + */ +public class JdbcFederationFragmentExplainerTest { + + /** + * 验证连接池以运行时异常拒绝连接时返回精确连接获取失败错误。 + */ + @Test + public void shouldMapRuntimeConnectionFailure() { + FederationSqlException failure = expectFailure(context( + FederationExecutionGuard.none(), + new IllegalStateException("pool is closed") + )); + + Assert.assertEquals( + FederationSqlErrorCode.CONNECTION_ACQUISITION_FAILED, + failure.errorCode() + ); + } + + /** + * 验证连接池失败返回时,已经到达的统一截止时间优先于连接错误。 + */ + @Test + public void shouldPreferDeadlineOverRuntimeConnectionFailure() { + AtomicInteger checks = new AtomicInteger(); + FederationExecutionGuard guard = new FederationExecutionGuard() { + @Override + public void ensureAllowed() { + if (checks.incrementAndGet() > 1) { + throw new FederationSqlException( + FederationSqlErrorCode.QUERY_TIMEOUT, + "query deadline reached" + ); + } + } + + @Override + public long remainingNanos() { + return 1L; + } + }; + + FederationSqlException failure = expectFailure(context( + guard, + new IllegalStateException("pool is closed") + )); + + Assert.assertEquals(FederationSqlErrorCode.QUERY_TIMEOUT, failure.errorCode()); + } + + private static FederationSqlException expectFailure( + FederationFragmentExplainContext context + ) { + try { + new JdbcFederationFragmentExplainer().explain(context); + Assert.fail("physical Explain should fail"); + return null; + } catch (FederationSqlException exception) { + return exception; + } + } + + private static FederationFragmentExplainContext context( + FederationExecutionGuard guard, + RuntimeException failure + ) { + return new FederationFragmentExplainContext( + "SELECT 1", + List.of(), + failingDataSource(failure), + new AdapterCompatibility( + AdapterCompatibilityStatus.VERIFIED, + "MySQL", + "8", + "test-driver", + "1", + "test" + ), + Map.of(), + 5, + guard + ); + } + + private static DataSource failingDataSource(RuntimeException failure) { + return new DataSource() { + @Override + public Connection getConnection() { + throw failure; + } + + @Override + public Connection getConnection(String username, String password) { + throw failure; + } + + @Override + public T unwrap(Class iface) throws SQLException { + throw new SQLException("not a wrapper"); + } + + @Override + public boolean isWrapperFor(Class iface) { + return false; + } + + @Override + public PrintWriter getLogWriter() { + return null; + } + + @Override + public void setLogWriter(PrintWriter out) { + } + + @Override + public void setLoginTimeout(int seconds) { + } + + @Override + public int getLoginTimeout() { + return 0; + } + + @Override + public Logger getParentLogger() { + return Logger.getGlobal(); + } + }; + } +} diff --git a/easy-agents-federation-sql/easy-agents-federation-sql-adapter-jdbc/src/test/java/com/easyagents/federation/sql/adapter/jdbc/JdbcFederationSqlEngineTest.java b/easy-agents-federation-sql/easy-agents-federation-sql-adapter-jdbc/src/test/java/com/easyagents/federation/sql/adapter/jdbc/JdbcFederationSqlEngineTest.java new file mode 100644 index 0000000..8201b28 --- /dev/null +++ b/easy-agents-federation-sql/easy-agents-federation-sql-adapter-jdbc/src/test/java/com/easyagents/federation/sql/adapter/jdbc/JdbcFederationSqlEngineTest.java @@ -0,0 +1,1195 @@ +package com.easyagents.federation.sql.adapter.jdbc; + +import com.easyagents.federation.sql.api.FederationSqlEngine; +import com.easyagents.federation.sql.api.FederationSqlEngines; +import com.easyagents.federation.sql.api.FederationSqlErrorCode; +import com.easyagents.federation.sql.api.FederationSqlException; +import com.easyagents.federation.sql.api.SqlExecutionContext; +import com.easyagents.federation.sql.api.SqlQueryCommand; +import com.easyagents.federation.sql.compile.FederationSqlPlan; +import com.easyagents.federation.sql.compile.SqlCompileRequest; +import com.easyagents.federation.sql.compile.SqlExplainRequest; +import com.easyagents.federation.sql.execute.FederationResultCursor; +import com.easyagents.federation.sql.execute.FederationColumn; +import com.easyagents.federation.sql.execute.FederationQueryAdmissionController; +import com.easyagents.federation.sql.execute.FederationQueryMetricsSnapshot; +import com.easyagents.federation.sql.execute.FederationQueryPermit; +import com.easyagents.federation.sql.execute.LocalFederationQueryAdmissionController; +import com.easyagents.federation.sql.execute.QueryId; +import com.easyagents.federation.sql.execute.SqlExecutionOptions; +import com.easyagents.federation.sql.execute.SqlParameter; +import com.easyagents.federation.sql.execute.StatementLifecycle; +import com.easyagents.federation.sql.federation.FederationExecutionPolicy; +import com.easyagents.federation.sql.federation.FederationQueryScopeDefinition; +import com.easyagents.federation.sql.federation.FederationSourceBindingDefinition; +import com.easyagents.federation.sql.source.FederationDataSourceHandles; +import com.easyagents.federation.sql.source.FederationSourceDefinition; +import com.easyagents.federation.sql.source.RuntimeFingerprint; +import com.easyagents.federation.sql.source.SourceApplyOptions; +import com.easyagents.federation.sql.source.SourceId; +import java.io.PrintWriter; +import java.io.InputStream; +import java.io.Reader; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Proxy; +import java.math.BigDecimal; +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.SQLTimeoutException; +import java.sql.Statement; +import java.sql.Types; +import java.time.Duration; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.BooleanSupplier; +import java.util.logging.Logger; +import javax.sql.DataSource; +import org.h2.jdbcx.JdbcDataSource; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +/** + * Calcite 编译、ServiceLoader Adapter 和直接 JDBC 流式执行集成测试。 + */ +public class JdbcFederationSqlEngineTest { + + private static final SourceId SOURCE_ID = new SourceId("main"); + + private TrackingDataSource dataSource; + private FederationSqlEngine engine; + + /** + * 创建隔离 H2 数据库与预热的数据源 Runtime。 + * + * @throws SQLException 初始化失败 + */ + @Before + public void setUp() throws SQLException { + JdbcDataSource h2 = new JdbcDataSource(); + h2.setURL("jdbc:h2:mem:federation_" + System.nanoTime() + ";DB_CLOSE_DELAY=-1"); + dataSource = new TrackingDataSource(h2); + try (Connection connection = dataSource.getConnection(); Statement statement = connection.createStatement()) { + statement.execute("CREATE TABLE DEPARTMENT (ID INT PRIMARY KEY, NAME VARCHAR(64) NOT NULL)"); + statement.execute( + "CREATE TABLE PERSON (ID INT PRIMARY KEY, NAME VARCHAR(64) NOT NULL, DEPARTMENT_ID INT NOT NULL)" + ); + statement.execute("INSERT INTO DEPARTMENT VALUES (10, 'Engineering'), (20, 'Finance')"); + statement.execute( + "INSERT INTO PERSON VALUES (1, 'Alice', 10), (2, 'Bob', 10), (3, 'Carol', 20)" + ); + } + + RuntimeFingerprint fingerprint = new RuntimeFingerprint("H2", "2", "H2 JDBC Driver", "2", "1"); + engine = FederationSqlEngines.builder() + .dataSourceResolver(definition -> FederationDataSourceHandles.shared(dataSource, fingerprint)) + .maximumPlanCacheEntries(32) + .build(); + engine.sources().apply(definition(1), SourceApplyOptions.prewarmNow()); + Assert.assertEquals(0, dataSource.activeConnections()); + } + + /** + * 关闭 Engine。 + */ + @After + public void tearDown() { + if (engine != null) { + engine.close(); + } + } + + /** + * 验证参数、同源 Join、排序、分页、Explain 和流式资源释放。 + */ + @Test + public void shouldCompileAndStreamSameSourceJoin() { + String sql = "SELECT p.NAME, d.NAME AS DEPARTMENT_NAME " + + "FROM PERSON p JOIN DEPARTMENT d ON p.DEPARTMENT_ID = d.ID " + + "WHERE p.ID > ? ORDER BY p.ID FETCH NEXT 2 ROWS ONLY"; + SqlQueryCommand command = SqlQueryCommand.of( + sql, + SOURCE_ID, + 1, + List.of(new SqlParameter(Types.INTEGER, 1)) + ); + + try (FederationResultCursor cursor = engine.query(command)) { + Assert.assertEquals(1, dataSource.activeConnections()); + Assert.assertEquals(2, cursor.columns().size()); + Assert.assertTrue(cursor.next()); + Assert.assertEquals(List.of("Bob", "Engineering"), cursor.row()); + Assert.assertTrue(cursor.next()); + Assert.assertEquals(List.of("Carol", "Finance"), cursor.row()); + Assert.assertFalse(cursor.next()); + } + Assert.assertEquals(0, dataSource.activeConnections()); + + FederationSqlPlan plan = engine.compile(new SqlCompileRequest( + sql, + SOURCE_ID, + 1, + List.of(Types.INTEGER), + "policy-v1" + )); + Assert.assertTrue(plan.executable()); + Assert.assertEquals(1, plan.parameterCount()); + Assert.assertEquals(List.of(0), plan.parameterMapping()); + Assert.assertTrue(plan.executableSql().contains("?")); + Assert.assertTrue(plan.executableSql().toUpperCase().contains("PUBLIC")); + Assert.assertFalse(plan.executableSql().toUpperCase().contains("APP")); + Assert.assertFalse(plan.executableSql().contains("Bob")); + Assert.assertFalse( + engine.explain(new SqlExplainRequest(SqlCompileRequest.of( + "SELECT COUNT(*) FROM PERSON", + SOURCE_ID, + 1 + ))).relationalPlan().isBlank() + ); + } + + /** + * 验证缓存计划的每次签发保留各自的编译耗时与命中状态。 + */ + @Test + public void shouldKeepPlanningMetricsOnEachIssuedPlan() { + SqlCompileRequest request = SqlCompileRequest.of( + "SELECT ID FROM PERSON WHERE ID >= 1 ORDER BY ID", + SOURCE_ID, + 1 + ); + FederationSqlPlan coldPlan = engine.compile(request); + FederationSqlPlan hotPlan = engine.compile(request); + + Assert.assertNotSame(coldPlan, hotPlan); + FederationQueryMetricsSnapshot coldMetrics; + try (FederationResultCursor cursor = engine.execute( + coldPlan, + SqlExecutionContext.of(List.of()) + )) { + while (cursor.next()) { + cursor.row(); + } + coldMetrics = cursor.metrics(); + } + FederationQueryMetricsSnapshot hotMetrics; + try (FederationResultCursor cursor = engine.execute( + hotPlan, + SqlExecutionContext.of(List.of()) + )) { + while (cursor.next()) { + cursor.row(); + } + hotMetrics = cursor.metrics(); + } + + Assert.assertFalse(coldMetrics.planCacheHit()); + Assert.assertTrue(hotMetrics.planCacheHit()); + } + + /** + * 验证 DML 在获取执行连接前被只读基线拒绝。 + */ + @Test + public void shouldRejectDataModificationSql() { + try { + engine.compile(SqlCompileRequest.of("DELETE FROM PERSON", SOURCE_ID, 1)); + Assert.fail("DML should be rejected"); + } catch (FederationSqlException exception) { + Assert.assertEquals(FederationSqlErrorCode.SQL_NOT_READ_ONLY, exception.errorCode()); + } + Assert.assertEquals(0, dataSource.activeConnections()); + } + + /** + * 验证参数数量错误具有稳定错误码且不会借出执行连接。 + */ + @Test + public void shouldRejectParameterCountMismatch() { + try { + engine.query(SqlQueryCommand.of( + "SELECT NAME FROM PERSON WHERE ID = ?", + SOURCE_ID, + 1, + List.of() + )); + Assert.fail("parameter mismatch should be rejected"); + } catch (FederationSqlException exception) { + Assert.assertEquals(FederationSqlErrorCode.PARAMETER_COUNT_MISMATCH, exception.errorCode()); + } + + FederationSqlPlan integerPlan = engine.compile(new SqlCompileRequest( + "SELECT NAME FROM PERSON WHERE ID = ?", + SOURCE_ID, + 1, + List.of(Types.INTEGER), + "policy-v1" + )); + try { + engine.execute( + integerPlan, + SqlExecutionContext.of(List.of(new SqlParameter(Types.VARCHAR, "1"))) + ); + Assert.fail("parameter type mismatch should be rejected"); + } catch (FederationSqlException exception) { + Assert.assertEquals(FederationSqlErrorCode.PARAMETER_COUNT_MISMATCH, exception.errorCode()); + } + Assert.assertEquals(0, dataSource.activeConnections()); + } + + /** + * 验证当前行复制允许数据库 NULL,且提前关闭游标立即归还连接。 + */ + @Test + public void shouldReturnNullValuesAndReleaseConnectionOnEarlyClose() { + FederationResultCursor cursor = engine.query(SqlQueryCommand.of( + "SELECT CAST(NULL AS VARCHAR) AS OPTIONAL_VALUE FROM PERSON ORDER BY ID", + SOURCE_ID, + 1, + List.of() + )); + Assert.assertEquals(1, dataSource.activeConnections()); + Assert.assertTrue(cursor.next()); + Assert.assertEquals(Arrays.asList((Object) null), cursor.row()); + cursor.close(); + Assert.assertEquals(0, dataSource.activeConnections()); + } + + /** + * 验证大文本和二进制列可通过流读取,并在关闭游标后归还连接。 + * + * @throws Exception 流读取失败 + */ + @Test + public void shouldExposeCharacterAndBinaryStreams() throws Exception { + try (FederationResultCursor cursor = engine.query(SqlQueryCommand.of( + "SELECT CAST('hello' AS VARCHAR), CAST(X'0102' AS VARBINARY) " + + "FROM PERSON FETCH NEXT 1 ROWS ONLY", + SOURCE_ID, + 1, + List.of() + ))) { + Assert.assertTrue(cursor.next()); + try (Reader reader = cursor.getCharacterStream(1); + InputStream input = cursor.getBinaryStream(2)) { + char[] text = new char[5]; + Assert.assertEquals(5, reader.read(text)); + Assert.assertEquals("hello", new String(text)); + Assert.assertArrayEquals(new byte[] {1, 2}, input.readAllBytes()); + } + } + Assert.assertEquals(0, dataSource.activeConnections()); + } + + /** + * 验证计划缓存命中后的编译门面 p95 保持在 5ms 性能预算内。 + */ + @Test + public void shouldKeepHotCompileFacadeWithinBudget() { + SqlCompileRequest request = SqlCompileRequest.of( + "SELECT ID, NAME FROM PERSON WHERE ID > 0 ORDER BY ID", + SOURCE_ID, + 1 + ); + for (int index = 0; index < 10_000; index++) { + engine.compile(request); + } + + long[] durations = new long[10_000]; + for (int index = 0; index < durations.length; index++) { + long started = System.nanoTime(); + engine.compile(request); + durations[index] = System.nanoTime() - started; + } + Arrays.sort(durations); + long p95Nanos = durations[(int) (durations.length * 0.95)]; + Assert.assertTrue( + "hot compile p95 was " + p95Nanos + "ns", + p95Nanos <= 5_000_000L + ); + } + + /** + * 验证 NULL 必须携带真实 JDBC 类型,并能通过 PreparedStatement 正确绑定。 + */ + @Test + public void shouldRequireAndExecuteExplicitlyTypedNullParameter() { + try { + SqlParameter.of(null); + Assert.fail("untyped NULL should be rejected"); + } catch (IllegalArgumentException expected) { + Assert.assertTrue(expected.getMessage().contains("jdbcType")); + } + + try (FederationResultCursor cursor = engine.query(SqlQueryCommand.of( + "SELECT COALESCE(CAST(? AS VARCHAR), NAME) FROM PERSON WHERE ID = 1", + SOURCE_ID, + 1, + List.of(new SqlParameter(Types.VARCHAR, null)) + ))) { + Assert.assertTrue(cursor.next()); + Assert.assertEquals("Alice", cursor.getObject(1)); + Assert.assertFalse(cursor.next()); + } + Assert.assertEquals(0, dataSource.activeConnections()); + } + + /** + * 验证声明的 JDBC 类型通过 Calcite CAST 参与动态参数校验和结果类型推导。 + */ + @Test + public void shouldApplyDeclaredParameterTypeDuringCompilation() { + String sql = "SELECT ? AS TYPED_VALUE FROM PERSON FETCH NEXT 1 ROWS ONLY"; + FederationSqlPlan plan = engine.compile(new SqlCompileRequest( + sql, + SOURCE_ID, + 1, + List.of(Types.INTEGER), + "default" + )); + + Assert.assertEquals(Types.INTEGER, plan.columns().get(0).jdbcType()); + Assert.assertTrue(plan.normalizedSql().toUpperCase().contains("CAST(? AS INTEGER)")); + Assert.assertFalse( + plan.executableSql(), + plan.executableSql().toUpperCase().contains("CAST(? AS INTEGER)") + ); + try (FederationResultCursor cursor = engine.query(SqlQueryCommand.of( + sql, + SOURCE_ID, + 1, + List.of(new SqlParameter(Types.INTEGER, 7)) + ))) { + Assert.assertTrue(cursor.next()); + Assert.assertEquals(7, cursor.getObject(1)); + Assert.assertFalse(cursor.next()); + } + Assert.assertEquals(0, dataSource.activeConnections()); + } + + /** + * 验证 JDBC 4.2 时区参数和 UUID 参数能参与 Calcite 推导并正常执行。 + */ + @Test + public void shouldCompileAndExecuteJdbc42ScalarParameters() { + java.time.OffsetTime offsetTime = java.time.OffsetTime.parse("12:00:00+08:00"); + java.time.OffsetDateTime offsetTimestamp = java.time.OffsetDateTime.parse( + "2026-08-21T12:00:00+08:00" + ); + java.util.UUID uuid = java.util.UUID.randomUUID(); + String sql = "SELECT ? AS OFFSET_TIME_VALUE, ? AS OFFSET_TIMESTAMP_VALUE, " + + "? AS UUID_VALUE FROM PERSON FETCH NEXT 1 ROWS ONLY"; + SqlQueryCommand command = SqlQueryCommand.of( + sql, + SOURCE_ID, + 1, + List.of( + SqlParameter.of(offsetTime), + SqlParameter.of(offsetTimestamp), + SqlParameter.of(uuid) + ) + ); + + FederationSqlPlan plan = engine.compile(new SqlCompileRequest( + sql, + SOURCE_ID, + 1, + List.of( + Types.TIME_WITH_TIMEZONE, + Types.TIMESTAMP_WITH_TIMEZONE, + Types.OTHER + ), + "default" + )); + Assert.assertEquals(Types.TIME_WITH_TIMEZONE, plan.columns().get(0).jdbcType()); + Assert.assertEquals(Types.TIMESTAMP_WITH_TIMEZONE, plan.columns().get(1).jdbcType()); + Assert.assertEquals(Types.OTHER, plan.columns().get(2).jdbcType()); + + try (FederationResultCursor cursor = engine.query(command)) { + Assert.assertTrue(cursor.next()); + Assert.assertEquals(offsetTime, cursor.getObject(1)); + Assert.assertEquals(offsetTimestamp, cursor.getObject(2)); + Assert.assertEquals(uuid, cursor.getObject(3)); + Assert.assertFalse(cursor.next()); + } + Assert.assertEquals(0, dataSource.activeConnections()); + } + + /** + * 验证仅用于 Calcite 推导的类型 CAST 不会下推并改变小数参数语义。 + */ + @Test + public void shouldPreserveDecimalParameterValueDuringExecution() { + BigDecimal value = new BigDecimal("123.45"); + String sql = "SELECT ? AS DECIMAL_VALUE FROM PERSON FETCH NEXT 1 ROWS ONLY"; + FederationSqlPlan plan = engine.compile(new SqlCompileRequest( + sql, + SOURCE_ID, + 1, + List.of(Types.DECIMAL), + "default" + )); + Assert.assertFalse( + plan.executableSql(), + plan.executableSql().toUpperCase().contains("CAST(? AS DECIMAL)") + ); + SqlQueryCommand command = SqlQueryCommand.of( + sql, + SOURCE_ID, + 1, + List.of(SqlParameter.of(value)) + ); + + try (FederationResultCursor cursor = engine.query(command)) { + Assert.assertTrue(cursor.next()); + Assert.assertEquals(0, value.compareTo((BigDecimal) cursor.getObject(1))); + Assert.assertFalse(cursor.next()); + } + Assert.assertEquals(0, dataSource.activeConnections()); + } + + /** + * 验证公开执行选项不能关闭正式查询的 JDBC 只读防线。 + */ + @Test(expected = IllegalArgumentException.class) + public void shouldRejectDisabledReadOnlyExecution() { + new SqlExecutionOptions(100, 0, 30, false); + } + + /** + * 验证缓存命中和高级 execute 均会重新执行动态策略校验。 + */ + @Test + public void shouldRevalidatePolicyOnCacheHitAndExecute() { + AtomicBoolean allowed = new AtomicBoolean(true); + AtomicInteger evaluations = new AtomicInteger(); + RuntimeFingerprint fingerprint = new RuntimeFingerprint("H2", "2", "H2 JDBC Driver", "2", "1"); + FederationSqlEngine policyEngine = FederationSqlEngines.builder() + .dataSourceResolver(definition -> FederationDataSourceHandles.shared(dataSource, fingerprint)) + .policy(context -> { + evaluations.incrementAndGet(); + if (!allowed.get()) { + throw new FederationSqlException( + FederationSqlErrorCode.INVALID_ARGUMENT, + "query is denied by the current policy" + ); + } + }) + .build(); + try { + policyEngine.sources().apply(definition(1), SourceApplyOptions.prewarmNow()); + SqlCompileRequest request = SqlCompileRequest.of("SELECT ID FROM PERSON", SOURCE_ID, 1); + FederationSqlPlan plan = policyEngine.compile(request); + FederationSqlPlan cached = policyEngine.compile(request); + Assert.assertNotSame(plan, cached); + Assert.assertSame(plan.relRoot(), cached.relRoot()); + Assert.assertEquals(2, evaluations.get()); + + allowed.set(false); + assertPolicyDenied(() -> policyEngine.compile(request)); + assertPolicyDenied(() -> policyEngine.execute(plan, SqlExecutionContext.of(List.of()))); + Assert.assertEquals(4, evaluations.get()); + } finally { + policyEngine.close(); + } + Assert.assertEquals(0, dataSource.activeConnections()); + } + + /** + * 验证实际 Runtime 相同时,不同最小 revision 的计划仍保留各自策略上下文。 + */ + @Test + public void shouldIsolatePolicyContextByMinimumRevision() { + AtomicBoolean denyRevisionOne = new AtomicBoolean(); + RuntimeFingerprint fingerprint = new RuntimeFingerprint("H2", "2", "H2 JDBC Driver", "2", "1"); + FederationSqlEngine policyEngine = FederationSqlEngines.builder() + .dataSourceResolver(definition -> FederationDataSourceHandles.shared(dataSource, fingerprint)) + .policy(context -> { + if (denyRevisionOne.get() && context.request().minimumRevision() == 1) { + throw new FederationSqlException( + FederationSqlErrorCode.INVALID_ARGUMENT, + "minimum revision 1 is denied" + ); + } + }) + .build(); + try { + policyEngine.sources().apply(definition(2), SourceApplyOptions.prewarmNow()); + String sql = "SELECT ID FROM PERSON WHERE ID = 1"; + FederationSqlPlan revisionOnePlan = policyEngine.compile( + SqlCompileRequest.of(sql, SOURCE_ID, 1) + ); + FederationSqlPlan revisionTwoPlan = policyEngine.compile( + SqlCompileRequest.of(sql, SOURCE_ID, 2) + ); + Assert.assertNotSame(revisionOnePlan, revisionTwoPlan); + + denyRevisionOne.set(true); + assertPolicyDenied(() -> policyEngine.execute( + revisionOnePlan, + SqlExecutionContext.of(List.of()) + )); + try (FederationResultCursor cursor = policyEngine.execute( + revisionTwoPlan, + SqlExecutionContext.of(List.of()) + )) { + Assert.assertTrue(cursor.next()); + Assert.assertEquals(1, cursor.getObject(1)); + Assert.assertFalse(cursor.next()); + } + } finally { + policyEngine.close(); + } + Assert.assertEquals(0, dataSource.activeConnections()); + } + + /** + * 验证调用方实现的 Plan 接口无法绕过当前 Engine 的签发与只读策略。 + */ + @Test + public void shouldRejectPlanNotAuthorizedByCurrentEngine() { + FederationSqlPlan compiled = engine.compile(SqlCompileRequest.of( + "SELECT ID FROM PERSON", + SOURCE_ID, + 1 + )); + FederationSqlPlan forged = (FederationSqlPlan) Proxy.newProxyInstance( + getClass().getClassLoader(), + new Class[] {FederationSqlPlan.class}, + (proxy, method, arguments) -> { + if ("executableSql".equals(method.getName())) { + return "DELETE FROM PERSON"; + } + try { + return method.invoke(compiled, arguments); + } catch (InvocationTargetException exception) { + throw exception.getCause(); + } + } + ); + + try { + engine.execute(forged, SqlExecutionContext.of(List.of())); + Assert.fail("forged plan should be rejected"); + } catch (FederationSqlException exception) { + Assert.assertEquals( + exception.getMessage(), + FederationSqlErrorCode.INVALID_ARGUMENT, + exception.errorCode() + ); + } + Assert.assertEquals(0, dataSource.activeConnections()); + } + + /** + * 验证查询范围未变化时,无关 Source 新增或 revision 更新都不会冲掉热计划。 + */ + @Test + public void shouldKeepPlanWhenUnrelatedSourceChanges() { + RuntimeFingerprint fingerprint = new RuntimeFingerprint("H2", "2", "H2 JDBC Driver", "2", "1"); + FederationSqlEngine cacheEngine = FederationSqlEngines.builder() + .dataSourceResolver(definition -> FederationDataSourceHandles.shared(dataSource, fingerprint)) + .maximumPlanCacheEntries(32) + .build(); + try { + cacheEngine.sources().apply(definition(SOURCE_ID, 1), SourceApplyOptions.prewarmNow()); + SqlCompileRequest request = SqlCompileRequest.of("SELECT ID FROM PERSON", SOURCE_ID, 1); + FederationSqlPlan initial = cacheEngine.compile(request); + FederationSqlPlan initialHit = cacheEngine.compile(request); + Assert.assertNotSame(initial, initialHit); + Assert.assertSame(initial.relRoot(), initialHit.relRoot()); + + SourceId other = new SourceId("other"); + cacheEngine.sources().apply(definition(other, 1)); + FederationSqlPlan membershipChanged = cacheEngine.compile(request); + Assert.assertSame(initial.relRoot(), membershipChanged.relRoot()); + + cacheEngine.sources().apply(definition(other, 2)); + FederationSqlPlan revisionChanged = cacheEngine.compile(request); + Assert.assertSame(membershipChanged.relRoot(), revisionChanged.relRoot()); + } finally { + cacheEngine.close(); + } + Assert.assertEquals(0, dataSource.activeConnections()); + } + + /** + * 验证取消已返回的游标会同步归还 JDBC 连接和 Core 查询资源。 + */ + @Test + public void shouldReleaseReturnedCursorResourcesOnCancel() { + FederationResultCursor cursor = engine.query(SqlQueryCommand.of( + "SELECT ID, NAME FROM PERSON ORDER BY ID", + SOURCE_ID, + 1, + List.of() + )); + Assert.assertEquals(1, dataSource.activeConnections()); + Assert.assertTrue(engine.cancel(cursor.queryId())); + Assert.assertTrue(cursor.metrics().cancelled()); + Assert.assertFalse(cursor.metrics().complete()); + long frozenExecutionNanos = cursor.metrics().executionNanos(); + long frozenFragmentNanos = cursor.metrics().fragments().get(0).elapsedNanos(); + Assert.assertEquals(-1, cursor.metrics().returnedBytes()); + java.util.concurrent.locks.LockSupport.parkNanos( + java.util.concurrent.TimeUnit.MILLISECONDS.toNanos(2) + ); + Assert.assertEquals(frozenExecutionNanos, cursor.metrics().executionNanos()); + Assert.assertEquals( + frozenFragmentNanos, + cursor.metrics().fragments().get(0).elapsedNanos() + ); + Assert.assertEquals(0, dataSource.activeConnections()); + cursor.close(); + } + + /** + * 验证 Statement 登记后、驱动真正执行前到达的取消会关闭 Statement 并阻止执行。 + * + * @throws Exception 并发执行失败 + */ + @Test + public void shouldCancelBetweenStatementRegistrationAndDriverExecution() + throws Exception { + FederationSqlPlan plan = engine.compile(SqlCompileRequest.of( + "SELECT ID FROM PERSON ORDER BY ID", + SOURCE_ID, + 1 + )); + QueryId queryId = new QueryId("cancel-before-driver-execute"); + ExecutionGate gate = dataSource.gateNextExecution(); + ExecutorService executor = Executors.newSingleThreadExecutor(); + try { + Future query = executor.submit(() -> { + try (FederationResultCursor ignored = engine.execute( + plan, + new SqlExecutionContext( + queryId, + List.of(), + SqlExecutionOptions.defaults(), + Duration.ofSeconds(5) + ) + )) { + return null; + } catch (FederationSqlException exception) { + return exception.errorCode(); + } + }); + + Assert.assertTrue(gate.awaitExecution()); + Assert.assertTrue(engine.cancel(queryId)); + Assert.assertEquals( + FederationSqlErrorCode.QUERY_CANCELLED, + query.get(2, TimeUnit.SECONDS) + ); + Assert.assertTrue(gate.statementClosed()); + Assert.assertEquals(0, dataSource.activeConnections()); + } finally { + gate.release(); + executor.shutdownNow(); + } + } + + /** + * 验证 Scope 总时限可以关闭阻塞在 JDBC 执行边界的 Statement,并返回 QUERY_TIMEOUT。 + * + * @throws Exception 并发执行失败 + */ + @Test + public void shouldEnforceHardExecutionDeadlineAcrossBlockingJdbc() throws Exception { + FederationQueryScopeDefinition deadlineScope = FederationQueryScopeDefinition.virtual( + "deadline-scope", + 1, + Map.of( + "MAIN", + FederationSourceBindingDefinition.of(SOURCE_ID, 1) + ), + "MAIN", + new FederationExecutionPolicy(1, 1, 1, 1_000, 64_000, 100) + ); + FederationSqlPlan plan = engine.compile(SqlCompileRequest.of( + "SELECT ID FROM PERSON ORDER BY ID", + deadlineScope + )); + ExecutionGate gate = dataSource.gateNextExecution(); + ExecutorService executor = Executors.newSingleThreadExecutor(); + long started = System.nanoTime(); + try { + Future query = executor.submit(() -> { + try (FederationResultCursor ignored = engine.execute( + plan, + new SqlExecutionContext( + new QueryId("hard-deadline"), + List.of(), + SqlExecutionOptions.defaults(), + Duration.ofSeconds(5) + ) + )) { + return null; + } catch (FederationSqlException exception) { + return exception.errorCode(); + } + }); + + Assert.assertTrue(gate.awaitExecution()); + Assert.assertEquals( + FederationSqlErrorCode.QUERY_TIMEOUT, + query.get(2, TimeUnit.SECONDS) + ); + Assert.assertTrue(gate.statementClosed()); + Assert.assertTrue( + TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - started) < 2_000 + ); + Assert.assertEquals(0, dataSource.activeConnections()); + } finally { + gate.release(); + executor.shutdownNow(); + } + } + + /** + * 验证 QueryId 在准入等待前已经登记,取消可以及时中断默认本地准入。 + * + * @throws Exception 并发等待失败 + */ + @Test + public void shouldCancelQueryWaitingForAdmission() throws Exception { + LocalFederationQueryAdmissionController delegate = + new LocalFederationQueryAdmissionController(1); + CountDownLatch secondAdmission = new CountDownLatch(1); + AtomicInteger admissionCalls = new AtomicInteger(); + FederationQueryAdmissionController admission = new FederationQueryAdmissionController() { + @Override + public FederationQueryPermit acquire( + SourceId sourceId, + QueryId queryId, + Duration timeout + ) { + return delegate.acquire(sourceId, queryId, timeout); + } + + @Override + public FederationQueryPermit acquire( + SourceId sourceId, + QueryId queryId, + Duration timeout, + BooleanSupplier cancellationRequested + ) { + if (admissionCalls.incrementAndGet() == 2) { + secondAdmission.countDown(); + } + return delegate.acquire(sourceId, queryId, timeout, cancellationRequested); + } + + @Override + public void close() { + delegate.close(); + } + }; + RuntimeFingerprint fingerprint = new RuntimeFingerprint("H2", "2", "H2 JDBC Driver", "2", "1"); + FederationSqlEngine admissionEngine = FederationSqlEngines.builder() + .dataSourceResolver(definition -> FederationDataSourceHandles.shared(dataSource, fingerprint)) + .admissionController(admission) + .build(); + ExecutorService executor = Executors.newSingleThreadExecutor(); + FederationResultCursor first = null; + try { + admissionEngine.sources().apply(definition(1), SourceApplyOptions.prewarmNow()); + first = admissionEngine.query(SqlQueryCommand.of( + "SELECT ID FROM PERSON ORDER BY ID", + SOURCE_ID, + 1, + List.of() + )); + Assert.assertEquals(1, dataSource.activeConnections()); + + QueryId waitingId = new QueryId("waiting-for-admission"); + SqlQueryCommand waitingCommand = new SqlQueryCommand( + waitingId, + "SELECT ID FROM PERSON ORDER BY ID", + SOURCE_ID, + 1, + List.of(), + SqlExecutionOptions.defaults(), + Duration.ofSeconds(5).toMillis(), + "default" + ); + Future waiting = executor.submit(() -> { + try (FederationResultCursor ignored = admissionEngine.query(waitingCommand)) { + return null; + } catch (FederationSqlException exception) { + return exception.errorCode(); + } + }); + Assert.assertTrue(secondAdmission.await(2, TimeUnit.SECONDS)); + Assert.assertTrue(admissionEngine.cancel(waitingId)); + Assert.assertEquals( + FederationSqlErrorCode.QUERY_CANCELLED, + waiting.get(2, TimeUnit.SECONDS) + ); + Assert.assertEquals(1, dataSource.activeConnections()); + } finally { + if (first != null) { + first.close(); + } + executor.shutdownNow(); + admissionEngine.close(); + } + Assert.assertEquals(0, dataSource.activeConnections()); + } + + /** + * 验证结果消费超时具有稳定错误码,并关闭全部 JDBC 资源。 + */ + @Test + public void shouldClassifyResultTimeoutAndCloseResources() { + AtomicBoolean resultSetClosed = new AtomicBoolean(); + AtomicBoolean statementClosed = new AtomicBoolean(); + AtomicBoolean connectionClosed = new AtomicBoolean(); + AtomicBoolean unregistered = new AtomicBoolean(); + ResultSet resultSet = (ResultSet) Proxy.newProxyInstance( + getClass().getClassLoader(), + new Class[] {ResultSet.class}, + (proxy, method, arguments) -> { + if ("next".equals(method.getName())) { + throw new SQLTimeoutException("simulated read timeout"); + } + if ("close".equals(method.getName())) { + resultSetClosed.set(true); + } + return null; + } + ); + PreparedStatement statement = (PreparedStatement) Proxy.newProxyInstance( + getClass().getClassLoader(), + new Class[] {PreparedStatement.class}, + (proxy, method, arguments) -> { + if ("close".equals(method.getName())) { + statementClosed.set(true); + } + return null; + } + ); + Connection connection = (Connection) Proxy.newProxyInstance( + getClass().getClassLoader(), + new Class[] {Connection.class}, + (proxy, method, arguments) -> { + if ("close".equals(method.getName())) { + connectionClosed.set(true); + } + return null; + } + ); + StatementLifecycle lifecycle = new StatementLifecycle() { + @Override + public void register(Statement registeredStatement) { + } + + @Override + public void unregister(Statement registeredStatement) { + unregistered.set(true); + } + }; + JdbcFederationResultCursor cursor = new JdbcFederationResultCursor( + new QueryId("timeout-query"), + List.of(new FederationColumn(1, "ID", Types.INTEGER, "INTEGER", false)), + resultSet, + statement, + connection, + lifecycle + ); + + try { + cursor.next(); + Assert.fail("timeout should be reported"); + } catch (FederationSqlException exception) { + Assert.assertEquals(FederationSqlErrorCode.QUERY_TIMEOUT, exception.errorCode()); + } + Assert.assertTrue(resultSetClosed.get()); + Assert.assertTrue(statementClosed.get()); + Assert.assertTrue(connectionClosed.get()); + Assert.assertTrue(unregistered.get()); + } + + /** + * 验证 SELECT 列中的 schema.table.column 不会因 SourceId 同名而被误判为跨源。 + */ + @Test + public void shouldDetectCrossSourceOnlyFromRelationReferences() { + RuntimeFingerprint fingerprint = new RuntimeFingerprint("H2", "2", "H2 JDBC Driver", "2", "1"); + FederationSqlEngine localEngine = FederationSqlEngines.builder() + .dataSourceResolver(definition -> FederationDataSourceHandles.shared(dataSource, fingerprint)) + .crossSourceEnabled(false) + .build(); + try { + localEngine.sources().apply(new FederationSourceDefinition( + SOURCE_ID, + 1, + JdbcFederationSqlAdapterProvider.ADAPTER_ID, + List.of( + new JdbcSchemaDefinition("APP", null, "PUBLIC"), + new JdbcSchemaDefinition("AUX", null, "PUBLIC") + ), + Map.of() + ), SourceApplyOptions.prewarmNow()); + localEngine.sources().apply(definition(new SourceId("APP"), 1)); + + FederationSqlPlan plan = localEngine.compile(SqlCompileRequest.of( + "SELECT APP.PERSON.ID FROM APP.PERSON", + SOURCE_ID, + 1 + )); + Assert.assertTrue(plan.executable()); + Assert.assertEquals(Set.of(SOURCE_ID), plan.referencedSources()); + try { + FederationQueryScopeDefinition scope = new FederationQueryScopeDefinition( + "cross-source-detection", + 1, + Map.of( + "MAIN", + FederationSourceBindingDefinition.of(SOURCE_ID, 1), + "REMOTE", + FederationSourceBindingDefinition.of(new SourceId("APP"), 1) + ), + "MAIN", + FederationExecutionPolicy.basic() + ); + localEngine.compile(SqlCompileRequest.of( + "SELECT p.ID FROM MAIN.APP.PERSON p " + + "JOIN REMOTE.APP.PERSON r ON p.ID = r.ID", + scope + )); + Assert.fail("qualified relation should be detected as cross-source"); + } catch (FederationSqlException exception) { + Assert.assertEquals(FederationSqlErrorCode.CROSS_SOURCE_DISABLED, exception.errorCode()); + } + } finally { + localEngine.close(); + } + } + + /** + * 断言策略拒绝使用稳定的参数错误码返回。 + * + * @param action 待执行动作 + */ + private static void assertPolicyDenied(Runnable action) { + try { + action.run(); + Assert.fail("policy should reject the query"); + } catch (FederationSqlException exception) { + Assert.assertEquals(FederationSqlErrorCode.INVALID_ARGUMENT, exception.errorCode()); + } + } + + private static FederationSourceDefinition definition(long revision) { + return definition(SOURCE_ID, revision); + } + + private static FederationSourceDefinition definition(SourceId sourceId, long revision) { + return new FederationSourceDefinition( + sourceId, + revision, + JdbcFederationSqlAdapterProvider.ADAPTER_ID, + List.of(new JdbcSchemaDefinition("APP", null, "PUBLIC")), + Map.of() + ); + } + + /** + * 统计借出连接且保持 DataSource 其余行为透明的测试包装器。 + */ + private static final class TrackingDataSource implements DataSource { + + private final DataSource delegate; + private final AtomicInteger activeConnections = new AtomicInteger(); + private final AtomicReference nextExecutionGate = + new AtomicReference<>(); + + private TrackingDataSource(DataSource delegate) { + this.delegate = delegate; + } + + private int activeConnections() { + return activeConnections.get(); + } + + /** + * 为下一条 PreparedStatement 安装执行门闩。 + * + * @return 执行门闩 + */ + private ExecutionGate gateNextExecution() { + ExecutionGate gate = new ExecutionGate(); + if (!nextExecutionGate.compareAndSet(null, gate)) { + throw new IllegalStateException("an execution gate is already pending"); + } + return gate; + } + + @Override + public Connection getConnection() throws SQLException { + return track(delegate.getConnection()); + } + + @Override + public Connection getConnection(String username, String password) throws SQLException { + return track(delegate.getConnection(username, password)); + } + + private Connection track(Connection connection) { + activeConnections.incrementAndGet(); + AtomicBoolean closed = new AtomicBoolean(); + return (Connection) Proxy.newProxyInstance( + getClass().getClassLoader(), + new Class[] {Connection.class}, + (proxy, method, arguments) -> { + if ("close".equals(method.getName())) { + if (closed.compareAndSet(false, true)) { + try { + connection.close(); + } finally { + activeConnections.decrementAndGet(); + } + } + return null; + } + try { + Object result = method.invoke(connection, arguments); + if (result instanceof PreparedStatement preparedStatement) { + ExecutionGate gate = nextExecutionGate.getAndSet(null); + return gate == null + ? preparedStatement + : gate.wrap(preparedStatement); + } + return result; + } catch (InvocationTargetException exception) { + throw exception.getCause(); + } + } + ); + } + + @Override + public PrintWriter getLogWriter() throws SQLException { + return delegate.getLogWriter(); + } + + @Override + public void setLogWriter(PrintWriter out) throws SQLException { + delegate.setLogWriter(out); + } + + @Override + public void setLoginTimeout(int seconds) throws SQLException { + delegate.setLoginTimeout(seconds); + } + + @Override + public int getLoginTimeout() throws SQLException { + return delegate.getLoginTimeout(); + } + + @Override + public Logger getParentLogger() { + return Logger.getGlobal(); + } + + @Override + public T unwrap(Class iface) throws SQLException { + return delegate.unwrap(iface); + } + + @Override + public boolean isWrapperFor(Class iface) throws SQLException { + return delegate.isWrapperFor(iface); + } + } + + /** + * 将 PreparedStatement 暂停在驱动执行边界,用于复现取消空窗。 + */ + private static final class ExecutionGate { + + private final CountDownLatch executionEntered = new CountDownLatch(1); + private final CountDownLatch executionReleased = new CountDownLatch(1); + private final AtomicBoolean statementClosed = new AtomicBoolean(); + + /** + * 包装待执行 Statement。 + * + * @param delegate 原始 Statement + * @return 受门闩控制的 Statement + */ + private PreparedStatement wrap(PreparedStatement delegate) { + return (PreparedStatement) Proxy.newProxyInstance( + getClass().getClassLoader(), + new Class[] {PreparedStatement.class}, + (proxy, method, arguments) -> { + if ("executeQuery".equals(method.getName())) { + executionEntered.countDown(); + try { + if (!executionReleased.await(2, TimeUnit.SECONDS)) { + throw new SQLException("execution gate timed out"); + } + } catch (InterruptedException exception) { + Thread.currentThread().interrupt(); + throw new SQLException("execution gate interrupted", exception); + } + if (statementClosed.get()) { + throw new SQLException("statement closed before execution"); + } + } + if ("close".equals(method.getName())) { + statementClosed.set(true); + executionReleased.countDown(); + } + try { + return method.invoke(delegate, arguments); + } catch (InvocationTargetException exception) { + throw exception.getCause(); + } + } + ); + } + + /** + * 等待查询抵达驱动执行边界。 + * + * @return 是否在超时前到达 + * @throws InterruptedException 等待被中断 + */ + private boolean awaitExecution() throws InterruptedException { + return executionEntered.await(2, TimeUnit.SECONDS); + } + + /** + * 返回 Statement 是否已由取消路径关闭。 + * + * @return 是否关闭 + */ + private boolean statementClosed() { + return statementClosed.get(); + } + + /** + * 释放驱动执行门闩。 + */ + private void release() { + executionReleased.countDown(); + } + } +} diff --git a/easy-agents-federation-sql/easy-agents-federation-sql-adapter-jdbc/src/test/java/com/easyagents/federation/sql/adapter/jdbc/JdbcFederationStatisticsIntegrationTest.java b/easy-agents-federation-sql/easy-agents-federation-sql-adapter-jdbc/src/test/java/com/easyagents/federation/sql/adapter/jdbc/JdbcFederationStatisticsIntegrationTest.java new file mode 100644 index 0000000..58f9d33 --- /dev/null +++ b/easy-agents-federation-sql/easy-agents-federation-sql-adapter-jdbc/src/test/java/com/easyagents/federation/sql/adapter/jdbc/JdbcFederationStatisticsIntegrationTest.java @@ -0,0 +1,347 @@ +package com.easyagents.federation.sql.adapter.jdbc; + +import com.easyagents.federation.sql.adapter.FederationStatisticsCollectionContext; +import com.easyagents.federation.sql.api.FederationSqlEngine; +import com.easyagents.federation.sql.api.FederationSqlEngines; +import com.easyagents.federation.sql.compile.SqlCompileRequest; +import com.easyagents.federation.sql.compile.SqlExplainLevel; +import com.easyagents.federation.sql.compile.SqlExplainRequest; +import com.easyagents.federation.sql.compile.SqlExplainResult; +import com.easyagents.federation.sql.federation.FederationExecutionPolicy; +import com.easyagents.federation.sql.federation.FederationQueryScopeDefinition; +import com.easyagents.federation.sql.federation.FederationSourceBindingDefinition; +import com.easyagents.federation.sql.federation.FederationStatisticsSnapshot; +import com.easyagents.federation.sql.federation.FederationTableStatistics; +import com.easyagents.federation.sql.source.FederationDataSourceHandles; +import com.easyagents.federation.sql.source.FederationSourceDefinition; +import com.easyagents.federation.sql.source.RuntimeFingerprint; +import com.easyagents.federation.sql.source.SourceApplyOptions; +import com.easyagents.federation.sql.source.SourceId; +import com.mysql.cj.jdbc.MysqlDataSource; +import java.sql.Connection; +import java.sql.DriverManager; +import java.time.Duration; +import java.time.Instant; +import java.util.List; +import java.util.Map; +import javax.sql.DataSource; +import org.postgresql.ds.PGSimpleDataSource; +import org.junit.Assert; +import org.junit.Assume; +import org.junit.Test; + +/** + * 本机 MySQL 与 PostgreSQL 的 Adapter 内建统计采集验证。 + */ +public class JdbcFederationStatisticsIntegrationTest { + + private static final Duration STATISTICS_TTL = Duration.ofMinutes(30); + private static final SourceId MYSQL_SOURCE = new SourceId("mysql-statistics"); + private static final SourceId POSTGRESQL_SOURCE = new SourceId( + "postgresql-statistics" + ); + + /** + * 验证 MySQL 目录统计可由 JDBC Adapter 自动采集。 + * + * @throws Exception JDBC 连接或目录读取失败 + */ + @Test + public void shouldCollectMysqlStatistics() throws Exception { + Assume.assumeTrue(Boolean.getBoolean("federation.integration.enabled")); + String database = System.getProperty( + "federation.mysql.database", + "data-sheet" + ); + String url = "jdbc:mysql://" + + System.getProperty("federation.mysql.host", "127.0.0.1") + + ':' + Integer.getInteger("federation.mysql.port", 33306) + + '/' + database + + "?useSSL=false&allowPublicKeyRetrieval=true&serverTimezone=Asia/Shanghai"; + FederationSourceDefinition definition = definition( + "mysql-statistics", + "MAIN", + database, + null + ); + + try (Connection connection = DriverManager.getConnection( + url, + System.getProperty("federation.mysql.username", "root"), + System.getProperty("federation.mysql.password", "root") + )) { + Map statistics = + collect(definition, connection); + + FederationTableStatistics outlet = statistics.get( + new FederationStatisticsSnapshot.TableKey( + definition.sourceId(), + "MAIN", + "outlet" + ) + ); + Assert.assertNotNull(outlet); + assertUsable(outlet, "database-catalog:mysql"); + } + } + + /** + * 验证 PostgreSQL 目录统计可由 JDBC Adapter 自动采集。 + * + * @throws Exception JDBC 连接或目录读取失败 + */ + @Test + public void shouldCollectPostgresqlStatistics() throws Exception { + Assume.assumeTrue(Boolean.getBoolean("federation.integration.enabled")); + String database = System.getProperty( + "federation.pg.database", + "harmony_adapter" + ); + String url = "jdbc:postgresql://" + + System.getProperty("federation.pg.host", "127.0.0.1") + + ':' + Integer.getInteger("federation.pg.port", 54329) + + '/' + database; + FederationSourceDefinition definition = definition( + "postgresql-statistics", + "MAIN", + database, + "public" + ); + + try (Connection connection = DriverManager.getConnection( + url, + System.getProperty("federation.pg.username", "harmony"), + System.getProperty("federation.pg.password", "harmony") + )) { + Map statistics = + collect(definition, connection); + + Assert.assertFalse(statistics.isEmpty()); + Assert.assertTrue(statistics.keySet().stream().allMatch(key -> + definition.sourceId().equals(key.sourceId()) + && "main".equals(key.schema()) + )); + statistics.values().forEach(value -> + assertUsable(value, "database-catalog:postgresql") + ); + } + } + + /** + * 验证 Engine 默认启用 Adapter 统计,并将结果交给 Explain 成本估算。 + */ + @Test + public void shouldExposeAutomaticallyCollectedStatisticsThroughExplain() { + Assume.assumeTrue(Boolean.getBoolean("federation.integration.enabled")); + String mysqlDatabase = System.getProperty( + "federation.mysql.database", + "data-sheet" + ); + String postgresqlDatabase = System.getProperty( + "federation.pg.database", + "harmony_adapter" + ); + FederationSourceDefinition mysqlDefinition = definition( + MYSQL_SOURCE.value(), + "main", + mysqlDatabase, + null + ); + FederationSourceDefinition postgresqlDefinition = definition( + POSTGRESQL_SOURCE.value(), + "main", + postgresqlDatabase, + "public" + ); + Map dataSources = Map.of( + MYSQL_SOURCE, + mysqlDataSource(mysqlDatabase), + POSTGRESQL_SOURCE, + postgresqlDataSource(postgresqlDatabase) + ); + + try (FederationSqlEngine engine = FederationSqlEngines.builder() + .dataSourceResolver(definition -> FederationDataSourceHandles.shared( + dataSources.get(definition.sourceId()), + new RuntimeFingerprint("test", "1", "jdbc", "1", "1") + )) + .build()) { + engine.sources().apply(mysqlDefinition, SourceApplyOptions.prewarmNow()); + engine.sources().apply( + postgresqlDefinition, + SourceApplyOptions.prewarmNow() + ); + FederationQueryScopeDefinition scope = FederationQueryScopeDefinition.virtual( + "automatic-statistics", + 1, + Map.of( + "mysql", + FederationSourceBindingDefinition.of(MYSQL_SOURCE, 1), + "pg", + FederationSourceBindingDefinition.of(POSTGRESQL_SOURCE, 1) + ), + "mysql", + FederationExecutionPolicy.basic() + ); + + SqlExplainResult mysqlExplain = explain( + engine, + scope, + "SELECT * FROM mysql.main.outlet" + ); + SqlExplainResult postgresqlExplain = explain( + engine, + scope, + "SELECT * FROM pg.main.artifact" + ); + + assertExplainStatistics(mysqlExplain, "database-catalog:mysql"); + assertExplainStatistics( + postgresqlExplain, + "database-catalog:postgresql" + ); + } + } + + /** + * 使用 Adapter 统计采集器读取当前连接。 + * + * @param definition 物理源定义 + * @param connection JDBC 连接 + * @return 不可变表统计 + * @throws Exception 目录读取失败 + */ + private Map collect( + FederationSourceDefinition definition, + Connection connection + ) throws Exception { + Instant collectedAt = Instant.now(); + return new JdbcFederationStatisticsCollector().collect( + new FederationStatisticsCollectionContext( + definition, + connection, + collectedAt, + collectedAt.plus(STATISTICS_TTL), + 5 + ) + ); + } + + /** + * 创建单 Schema JDBC 数据源定义。 + * + * @param sourceId 物理源标识 + * @param logicalSchema 逻辑 Schema + * @param catalog 物理 Catalog + * @param physicalSchema 物理 Schema + * @return 数据源定义 + */ + private FederationSourceDefinition definition( + String sourceId, + String logicalSchema, + String catalog, + String physicalSchema + ) { + return new FederationSourceDefinition( + new SourceId(sourceId), + 1, + JdbcFederationSqlAdapterProvider.ADAPTER_ID, + List.of(new JdbcSchemaDefinition( + logicalSchema, + catalog, + physicalSchema + )), + Map.of() + ); + } + + /** + * 创建本机 MySQL 测试 DataSource。 + * + * @param database 数据库名称 + * @return MySQL DataSource + */ + private DataSource mysqlDataSource(String database) { + MysqlDataSource dataSource = new MysqlDataSource(); + dataSource.setUrl( + "jdbc:mysql://" + + System.getProperty("federation.mysql.host", "127.0.0.1") + + ':' + Integer.getInteger("federation.mysql.port", 33306) + + '/' + database + + "?useSSL=false&allowPublicKeyRetrieval=true&serverTimezone=Asia/Shanghai" + ); + dataSource.setUser(System.getProperty("federation.mysql.username", "root")); + dataSource.setPassword(System.getProperty("federation.mysql.password", "root")); + return dataSource; + } + + /** + * 创建本机 PostgreSQL 测试 DataSource。 + * + * @param database 数据库名称 + * @return PostgreSQL DataSource + */ + private DataSource postgresqlDataSource(String database) { + PGSimpleDataSource dataSource = new PGSimpleDataSource(); + dataSource.setServerNames(new String[]{ + System.getProperty("federation.pg.host", "127.0.0.1") + }); + dataSource.setPortNumbers(new int[]{ + Integer.getInteger("federation.pg.port", 54329) + }); + dataSource.setDatabaseName(database); + dataSource.setUser(System.getProperty("federation.pg.username", "harmony")); + dataSource.setPassword(System.getProperty("federation.pg.password", "harmony")); + return dataSource; + } + + /** + * 执行逻辑 Explain。 + * + * @param engine 联邦 SQL Engine + * @param scope 查询范围 + * @param sql SQL + * @return Explain 结果 + */ + private SqlExplainResult explain( + FederationSqlEngine engine, + FederationQueryScopeDefinition scope, + String sql + ) { + return engine.explain(new SqlExplainRequest( + SqlCompileRequest.of(sql, scope), + SqlExplainLevel.LOGICAL + )); + } + + /** + * 断言 Explain 已使用自动采集的数据库统计。 + * + * @param explain Explain 结果 + * @param source 预期统计来源 + */ + private void assertExplainStatistics(SqlExplainResult explain, String source) { + Assert.assertEquals(1, explain.fragments().size()); + Assert.assertFalse(explain.fragments().get(0).costEstimate().statisticsMissing()); + Assert.assertTrue( + explain.fragments().get(0).costEstimate().statisticsSource().contains(source) + ); + Assert.assertTrue( + explain.fragments().get(0).costEstimate().estimatedRowWidthBytes() > 0L + ); + } + + /** + * 断言采集结果包含优化器可使用的基础统计。 + * + * @param statistics 表统计 + * @param source 预期统计来源 + */ + private void assertUsable(FederationTableStatistics statistics, String source) { + Assert.assertTrue(statistics.estimatedRows() >= 0D); + Assert.assertTrue(statistics.averageRowWidthBytes() > 0L); + Assert.assertEquals(source, statistics.source()); + Assert.assertNotNull(statistics.collectedAt()); + Assert.assertTrue(statistics.expiresAt().isAfter(statistics.collectedAt())); + } +} diff --git a/easy-agents-federation-sql/easy-agents-federation-sql-adapter-jdbc/src/test/java/com/easyagents/federation/sql/adapter/jdbc/MysqlCaseInsensitiveColumnSchemaTest.java b/easy-agents-federation-sql/easy-agents-federation-sql-adapter-jdbc/src/test/java/com/easyagents/federation/sql/adapter/jdbc/MysqlCaseInsensitiveColumnSchemaTest.java new file mode 100644 index 0000000..bf8dbd7 --- /dev/null +++ b/easy-agents-federation-sql/easy-agents-federation-sql-adapter-jdbc/src/test/java/com/easyagents/federation/sql/adapter/jdbc/MysqlCaseInsensitiveColumnSchemaTest.java @@ -0,0 +1,185 @@ +package com.easyagents.federation.sql.adapter.jdbc; + +import java.lang.reflect.Proxy; +import java.sql.Connection; +import java.sql.DatabaseMetaData; +import java.sql.ResultSet; +import java.util.List; +import java.util.Map; +import javax.sql.DataSource; +import org.apache.calcite.adapter.jdbc.JdbcSchema; +import org.apache.calcite.adapter.jdbc.JdbcTable; +import org.apache.calcite.jdbc.JavaTypeFactoryImpl; +import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.rel.type.RelDataTypeFactory; +import org.apache.calcite.schema.Schema; +import org.apache.calcite.schema.Table; +import org.apache.calcite.schema.Wrapper; +import org.apache.calcite.schema.impl.AbstractSchema; +import org.apache.calcite.schema.impl.AbstractTable; +import org.apache.calcite.sql.dialect.MysqlSqlDialect; +import org.apache.calcite.sql.type.SqlTypeName; +import org.junit.Assert; +import org.junit.Test; + +/** + * MySQL 表名与列名大小写语义包装器测试。 + */ +public class MysqlCaseInsensitiveColumnSchemaTest { + + /** + * 验证 JDBC 元数据 LIKE 模式不会把下划线表名解析到近似表。 + */ + @Test + public void shouldEscapeJdbcMetadataPatternAndRequireExactPhysicalTable() { + DataSource dataSource = metadataDataSource(); + JdbcSchema jdbcSchema = new JdbcSchema( + dataSource, + MysqlSqlDialect.DEFAULT, + null, + "app", + null + ); + JdbcTable rawTable = ((Wrapper) jdbcSchema.tables().get("order_item")) + .unwrap(JdbcTable.class); + Assert.assertEquals("order0item", rawTable.jdbcTableName); + + Schema schema = new MysqlCaseInsensitiveColumnSchema(jdbcSchema); + JdbcTable exactTable = ((Wrapper) schema.getTable("order_item")) + .unwrap(JdbcTable.class); + Assert.assertEquals("order_item", exactTable.jdbcTableName); + } + + /** + * 验证大小写不同的表保持独立,同时列名可忽略大小写查找。 + */ + @Test + public void shouldKeepExactTableNamesAndMatchColumnsIgnoringCase() { + Schema schema = new MysqlCaseInsensitiveColumnSchema(new AbstractSchema() { + @Override + protected Map getTableMap() { + return Map.of( + "orders", table("id"), + "Orders", table("different_column") + ); + } + }); + RelDataTypeFactory typeFactory = new JavaTypeFactoryImpl(); + + Table lowerCaseTable = schema.getTable("orders"); + Table upperCaseTable = schema.getTable("Orders"); + Assert.assertNotNull(lowerCaseTable); + Assert.assertNotNull(upperCaseTable); + Assert.assertNull(schema.getTable("ORDERS")); + + RelDataType lowerCaseRow = lowerCaseTable.getRowType(typeFactory); + RelDataType upperCaseRow = upperCaseTable.getRowType(typeFactory); + Assert.assertNotNull(lowerCaseRow.getField("ID", true, false)); + Assert.assertEquals("id", lowerCaseRow.getField("ID", true, false).getName()); + Assert.assertNotNull(upperCaseRow.getField("DIFFERENT_COLUMN", true, false)); + Assert.assertNull(upperCaseRow.getField("ID", true, false)); + } + + private static Table table(String columnName) { + return new AbstractTable() { + @Override + public RelDataType getRowType(RelDataTypeFactory typeFactory) { + return typeFactory.builder() + .add(columnName, SqlTypeName.INTEGER) + .build(); + } + }; + } + + private static DataSource metadataDataSource() { + DatabaseMetaData metadata = (DatabaseMetaData) Proxy.newProxyInstance( + MysqlCaseInsensitiveColumnSchemaTest.class.getClassLoader(), + new Class[] {DatabaseMetaData.class}, + (proxy, method, arguments) -> switch (method.getName()) { + case "getSearchStringEscape" -> "\\"; + case "getJDBCMajorVersion" -> 4; + case "getJDBCMinorVersion" -> 2; + case "getDatabaseProductName" -> "MySQL"; + case "getTables" -> tableResultSet((String) arguments[2]); + default -> defaultValue(method.getReturnType()); + } + ); + Connection connection = (Connection) Proxy.newProxyInstance( + MysqlCaseInsensitiveColumnSchemaTest.class.getClassLoader(), + new Class[] {Connection.class}, + (proxy, method, arguments) -> switch (method.getName()) { + case "getMetaData" -> metadata; + case "getCatalog" -> "app"; + case "getSchema" -> null; + case "close" -> null; + case "isClosed" -> false; + default -> defaultValue(method.getReturnType()); + } + ); + return (DataSource) Proxy.newProxyInstance( + MysqlCaseInsensitiveColumnSchemaTest.class.getClassLoader(), + new Class[] {DataSource.class}, + (proxy, method, arguments) -> switch (method.getName()) { + case "getConnection" -> connection; + default -> defaultValue(method.getReturnType()); + } + ); + } + + private static ResultSet tableResultSet(String pattern) { + List tableNames = switch (pattern) { + case "order_item" -> List.of("order0item", "order_item"); + case "order\\_item" -> List.of("order_item"); + case "%" -> List.of("order0item", "order_item"); + default -> List.of(); + }; + int[] cursor = {-1}; + return (ResultSet) Proxy.newProxyInstance( + MysqlCaseInsensitiveColumnSchemaTest.class.getClassLoader(), + new Class[] {ResultSet.class}, + (proxy, method, arguments) -> switch (method.getName()) { + case "next" -> ++cursor[0] < tableNames.size(); + case "getString" -> switch ((Integer) arguments[0]) { + case 1 -> "app"; + case 2 -> null; + case 3 -> tableNames.get(cursor[0]); + case 4 -> "TABLE"; + default -> null; + }; + case "close" -> null; + default -> defaultValue(method.getReturnType()); + } + ); + } + + private static Object defaultValue(Class type) { + if (!type.isPrimitive()) { + return null; + } + if (type == boolean.class) { + return false; + } + if (type == int.class) { + return 0; + } + if (type == long.class) { + return 0L; + } + if (type == short.class) { + return (short) 0; + } + if (type == byte.class) { + return (byte) 0; + } + if (type == float.class) { + return 0F; + } + if (type == double.class) { + return 0D; + } + if (type == char.class) { + return '\0'; + } + return null; + } +} diff --git a/easy-agents-federation-sql/easy-agents-federation-sql-core/pom.xml b/easy-agents-federation-sql/easy-agents-federation-sql-core/pom.xml new file mode 100644 index 0000000..7572e41 --- /dev/null +++ b/easy-agents-federation-sql/easy-agents-federation-sql-core/pom.xml @@ -0,0 +1,31 @@ + + + 4.0.0 + + + com.easyagents + easy-agents-federation-sql + ${revision} + + + easy-agents-federation-sql-core + easy-agents-federation-sql-core + + + + org.apache.calcite + calcite-core + + + org.slf4j + slf4j-api + + + junit + junit + test + + + diff --git a/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/adapter/AdapterCompatibility.java b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/adapter/AdapterCompatibility.java new file mode 100644 index 0000000..eb6f589 --- /dev/null +++ b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/adapter/AdapterCompatibility.java @@ -0,0 +1,23 @@ +package com.easyagents.federation.sql.adapter; + +import java.io.Serializable; + +/** + * Adapter 对当前数据库与驱动的兼容性说明。 + * + * @param status 兼容性状态 + * @param databaseProduct 数据库产品 + * @param databaseVersion 数据库版本 + * @param driverName 驱动名称 + * @param driverVersion 驱动版本 + * @param diagnostic 不含敏感信息的说明 + */ +public record AdapterCompatibility( + AdapterCompatibilityStatus status, + String databaseProduct, + String databaseVersion, + String driverName, + String driverVersion, + String diagnostic +) implements Serializable { +} diff --git a/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/adapter/AdapterCompatibilityStatus.java b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/adapter/AdapterCompatibilityStatus.java new file mode 100644 index 0000000..2c75524 --- /dev/null +++ b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/adapter/AdapterCompatibilityStatus.java @@ -0,0 +1,13 @@ +package com.easyagents.federation.sql.adapter; + +/** + * 数据库 Adapter 兼容性证据状态。 + */ +public enum AdapterCompatibilityStatus { + /** 已通过目标数据库真实集成验证。 */ + VERIFIED, + /** 代码与契约已支持,缺少目标环境验证。 */ + CODE_SUPPORTED_UNVERIFIED, + /** 当前 Adapter 明确不支持。 */ + UNSUPPORTED +} diff --git a/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/adapter/AdapterDialectContext.java b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/adapter/AdapterDialectContext.java new file mode 100644 index 0000000..3ab722d --- /dev/null +++ b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/adapter/AdapterDialectContext.java @@ -0,0 +1,16 @@ +package com.easyagents.federation.sql.adapter; + +import com.easyagents.federation.sql.source.FederationSourceDefinition; +import java.sql.DatabaseMetaData; + +/** + * Adapter 选择数据库方言的上下文。 + * + * @param metadata JDBC 元数据 + * @param sourceDefinition 数据源定义 + */ +public record AdapterDialectContext( + DatabaseMetaData metadata, + FederationSourceDefinition sourceDefinition +) { +} diff --git a/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/adapter/AdapterHints.java b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/adapter/AdapterHints.java new file mode 100644 index 0000000..be52188 --- /dev/null +++ b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/adapter/AdapterHints.java @@ -0,0 +1,28 @@ +package com.easyagents.federation.sql.adapter; + +import java.util.Map; + +/** + * Adapter 探测与编译的非敏感提示。 + * + * @param options Definition 中的 Adapter 选项 + */ +public record AdapterHints(Map options) { + + /** + * 防御性复制提示选项。 + */ + public AdapterHints { + options = Map.copyOf(options == null ? Map.of() : options); + } + + /** + * 判断指定布尔选项是否开启。 + * + * @param key 选项名 + * @return 是否开启 + */ + public boolean enabled(String key) { + return Boolean.parseBoolean(options.getOrDefault(key, "false")); + } +} diff --git a/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/adapter/AdapterSchemaContext.java b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/adapter/AdapterSchemaContext.java new file mode 100644 index 0000000..3cc2916 --- /dev/null +++ b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/adapter/AdapterSchemaContext.java @@ -0,0 +1,25 @@ +package com.easyagents.federation.sql.adapter; + +import com.easyagents.federation.sql.source.FederationDataSourceHandle; +import com.easyagents.federation.sql.source.FederationSchemaDefinition; +import com.easyagents.federation.sql.source.FederationSourceDefinition; +import org.apache.calcite.schema.SchemaPlus; +import org.apache.calcite.sql.SqlDialect; + +/** + * Adapter 创建 Calcite Schema 时所需的节点本地上下文。 + * + * @param parentSchema Calcite 父 Schema + * @param sourceDefinition 数据源定义 + * @param schemaDefinition 当前 Schema 定义 + * @param handle DataSource 句柄 + * @param dialect 已探测方言 + */ +public record AdapterSchemaContext( + SchemaPlus parentSchema, + FederationSourceDefinition sourceDefinition, + FederationSchemaDefinition schemaDefinition, + FederationDataSourceHandle handle, + SqlDialect dialect +) { +} diff --git a/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/adapter/FederationSqlAdapterProvider.java b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/adapter/FederationSqlAdapterProvider.java new file mode 100644 index 0000000..83ec008 --- /dev/null +++ b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/adapter/FederationSqlAdapterProvider.java @@ -0,0 +1,167 @@ +package com.easyagents.federation.sql.adapter; + +import com.easyagents.federation.sql.execute.FederationFragmentExecutor; +import com.easyagents.federation.sql.execute.FederationFragmentExplainer; +import java.sql.DatabaseMetaData; +import java.sql.SQLException; +import java.sql.Types; +import java.util.List; +import java.util.Optional; +import org.apache.calcite.plan.RelOptRule; +import org.apache.calcite.rel.type.RelDataTypeSystem; +import org.apache.calcite.schema.Schema; +import org.apache.calcite.sql.SqlBasicTypeNameSpec; +import org.apache.calcite.sql.SqlDataTypeSpec; +import org.apache.calcite.sql.SqlDialect; +import org.apache.calcite.sql.SqlOperatorTable; +import org.apache.calcite.sql.fun.SqlStdOperatorTable; +import org.apache.calcite.sql.parser.SqlParserPos; +import org.apache.calcite.sql.parser.SqlParser; +import org.apache.calcite.sql.type.SqlTypeName; + +/** + * 直接扩展 Calcite Schema、Dialect、类型和规则的数据库 Adapter SPI。 + */ +public interface FederationSqlAdapterProvider { + + /** + * 返回全局唯一 Adapter 标识。 + * + * @return Adapter 标识 + */ + String adapterId(); + + /** + * 判断当前数据库与驱动是否受支持。 + * + * @param metadata JDBC 元数据 + * @param hints 非敏感提示 + * @return 是否受支持 + * @throws SQLException 元数据读取失败 + */ + boolean supports(DatabaseMetaData metadata, AdapterHints hints) throws SQLException; + + /** + * 返回当前数据库的兼容性证据状态。 + * + * @param metadata JDBC 元数据 + * @param hints 非敏感提示 + * @return 兼容性说明 + * @throws SQLException 元数据读取失败 + */ + AdapterCompatibility compatibility(DatabaseMetaData metadata, AdapterHints hints) throws SQLException; + + /** + * 创建当前 Definition 对应的 Calcite Schema。 + * + * @param context Schema 上下文 + * @return Calcite Schema + */ + Schema createSchema(AdapterSchemaContext context); + + /** + * 选择目标数据库 SqlDialect。 + * + * @param context 方言上下文 + * @return Calcite SqlDialect + * @throws SQLException 元数据读取失败 + */ + SqlDialect createDialect(AdapterDialectContext context) throws SQLException; + + /** + * 返回数据库类型系统。 + * + * @return Calcite 类型系统 + */ + default RelDataTypeSystem typeSystem() { + return RelDataTypeSystem.DEFAULT; + } + + /** + * 返回数据库运算符表。 + * + * @return Calcite 运算符表 + */ + default SqlOperatorTable operatorTable() { + return SqlStdOperatorTable.instance(); + } + + /** + * 返回 Adapter 附加的 Calcite Planner 规则。 + * + * @return Planner 规则 + */ + default List plannerRules() { + return List.of(); + } + + /** + * 创建保留 ANSI 双引号输入、继承目标方言大小写语义的解析配置。 + * + *

数据库若对表名与列名采用不同的大小写规则,可以在 Adapter 中覆盖。

+ * + * @param dialect 目标数据库方言 + * @return Calcite 解析配置 + */ + default SqlParser.Config parserConfig(SqlDialect dialect) { + return SqlParser.config() + .withQuotedCasing(dialect.getQuotedCasing()) + .withUnquotedCasing(dialect.getUnquotedCasing()) + .withCaseSensitive(dialect.isCaseSensitive()); + } + + /** + * 将 JDBC 参数类型映射为 Calcite 类型声明,供动态参数参与校验和类型推导。 + * + *

默认实现补齐 JDBC 4.2 时区类型,并将 {@link Types#OTHER} 解释为 UUID。 + * 厂商 Adapter 可以覆盖此方法,直接返回带精度、长度或专有类型名的 + * Calcite 类型声明。

+ * + * @param jdbcType {@link java.sql.Types} 类型值 + * @param parserPosition 动态参数的解析位置 + * @return Calcite 类型声明;无法映射时返回 null + */ + default SqlDataTypeSpec parameterTypeSpec(int jdbcType, SqlParserPos parserPosition) { + SqlTypeName typeName = switch (jdbcType) { + case Types.TIME_WITH_TIMEZONE -> SqlTypeName.TIME_TZ; + case Types.TIMESTAMP_WITH_TIMEZONE -> SqlTypeName.TIMESTAMP_TZ; + case Types.OTHER -> SqlTypeName.UUID; + default -> SqlTypeName.getNameForJdbcType(jdbcType); + }; + if (typeName == null || typeName.isSpecial() || !typeName.allowsNoPrecNoScale()) { + return null; + } + return new SqlDataTypeSpec( + new SqlBasicTypeNameSpec(typeName, parserPosition), + parserPosition + ); + } + + /** + * 返回目标数据库 Fragment 执行器。 + * + * @return Fragment 执行器 + */ + FederationFragmentExecutor fragmentExecutor(); + + /** + * 返回可选的数据库物理 Explain 实现。 + * + * @return 物理 Explain SPI + */ + default Optional fragmentExplainer() { + return Optional.empty(); + } + + /** + * 返回可选的数据库目录统计采集器。 + * + *

统计采集由引擎管理缓存、并发合并、失效和失败降级,Adapter 只负责 + * 当前数据库的目录语义。

+ * + * @return 统计采集 SPI;未适配时为空 + */ + default Optional statisticsCollector() { + return Optional.empty(); + } +} diff --git a/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/adapter/FederationSqlAdapterRegistry.java b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/adapter/FederationSqlAdapterRegistry.java new file mode 100644 index 0000000..06375b2 --- /dev/null +++ b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/adapter/FederationSqlAdapterRegistry.java @@ -0,0 +1,94 @@ +package com.easyagents.federation.sql.adapter; + +import com.easyagents.federation.sql.api.FederationSqlErrorCode; +import com.easyagents.federation.sql.api.FederationSqlException; +import java.util.Collection; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.ServiceLoader; +import java.util.Set; + +/** + * 支持显式注册与 ServiceLoader 的 Adapter 注册表。 + */ +public final class FederationSqlAdapterRegistry { + + private final Map providers; + + /** + * 创建注册表;显式 Provider 优先于 ServiceLoader Provider。 + * + * @param explicitProviders 显式 Provider + * @param classLoader ServiceLoader 使用的类加载器 + */ + public FederationSqlAdapterRegistry( + Collection explicitProviders, + ClassLoader classLoader + ) { + Map loaded = new LinkedHashMap<>(); + ServiceLoader.load(FederationSqlAdapterProvider.class, classLoader) + .forEach(provider -> putUnique(loaded, provider)); + if (explicitProviders != null) { + Set explicitIds = new HashSet<>(); + for (FederationSqlAdapterProvider provider : explicitProviders) { + Objects.requireNonNull(provider, "adapter provider must not be null"); + if (!explicitIds.add(provider.adapterId())) { + throw new FederationSqlException( + FederationSqlErrorCode.SOURCE_DEFINITION_CONFLICT, + "duplicate explicitly registered adapter id: " + provider.adapterId() + ); + } + loaded.put(provider.adapterId(), provider); + } + } + this.providers = Map.copyOf(loaded); + } + + private static void putUnique( + Map providers, + FederationSqlAdapterProvider provider + ) { + FederationSqlAdapterProvider previous = providers.putIfAbsent(provider.adapterId(), provider); + if (previous != null && !previous.getClass().equals(provider.getClass())) { + throw new FederationSqlException( + FederationSqlErrorCode.SOURCE_DEFINITION_CONFLICT, + "duplicate adapter id from ServiceLoader: " + provider.adapterId() + ); + } + } + + /** + * 查找 Adapter Provider。 + * + * @param adapterId Adapter 标识 + * @return 可选 Provider + */ + public Optional find(String adapterId) { + return Optional.ofNullable(providers.get(adapterId)); + } + + /** + * 返回 Adapter Provider,缺失时抛出稳定错误。 + * + * @param adapterId Adapter 标识 + * @return Provider + */ + public FederationSqlAdapterProvider require(String adapterId) { + return find(adapterId).orElseThrow(() -> new FederationSqlException( + FederationSqlErrorCode.ADAPTER_NOT_FOUND, + "adapter is not registered: " + adapterId + )); + } + + /** + * 返回不可变 Provider 视图。 + * + * @return Provider 映射 + */ + public Map providers() { + return providers; + } +} diff --git a/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/adapter/FederationStatisticsCollectionContext.java b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/adapter/FederationStatisticsCollectionContext.java new file mode 100644 index 0000000..ab7f808 --- /dev/null +++ b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/adapter/FederationStatisticsCollectionContext.java @@ -0,0 +1,40 @@ +package com.easyagents.federation.sql.adapter; + +import com.easyagents.federation.sql.source.FederationSourceDefinition; +import java.sql.Connection; +import java.time.Instant; +import java.util.Objects; + +/** + * Adapter 采集数据库目录统计时使用的只读上下文。 + * + * @param sourceDefinition 当前物理数据源定义 + * @param connection 已从运行时连接池借出的 JDBC 连接 + * @param collectedAt 本轮统计采集时间 + * @param expiresAt 本轮统计默认失效时间 + * @param queryTimeoutSeconds 单条目录查询超时秒数 + */ +public record FederationStatisticsCollectionContext( + FederationSourceDefinition sourceDefinition, + Connection connection, + Instant collectedAt, + Instant expiresAt, + int queryTimeoutSeconds +) { + + /** + * 校验统计采集上下文。 + */ + public FederationStatisticsCollectionContext { + sourceDefinition = Objects.requireNonNull(sourceDefinition, "sourceDefinition"); + connection = Objects.requireNonNull(connection, "connection"); + collectedAt = Objects.requireNonNull(collectedAt, "collectedAt"); + expiresAt = Objects.requireNonNull(expiresAt, "expiresAt"); + if (!expiresAt.isAfter(collectedAt)) { + throw new IllegalArgumentException("expiresAt must be after collectedAt"); + } + if (queryTimeoutSeconds <= 0) { + throw new IllegalArgumentException("queryTimeoutSeconds must be positive"); + } + } +} diff --git a/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/adapter/FederationStatisticsCollector.java b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/adapter/FederationStatisticsCollector.java new file mode 100644 index 0000000..68a7b35 --- /dev/null +++ b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/adapter/FederationStatisticsCollector.java @@ -0,0 +1,27 @@ +package com.easyagents.federation.sql.adapter; + +import com.easyagents.federation.sql.federation.FederationStatisticsSnapshot; +import com.easyagents.federation.sql.federation.FederationTableStatistics; +import java.sql.SQLException; +import java.util.Map; + +/** + * 数据库 Adapter 提供的轻量目录统计采集 SPI。 + * + *

实现应使用数据库系统目录或 JDBC 元数据批量采集,禁止执行逐表 + * {@code COUNT(*)}。采集异常由引擎统一降级,不应在实现中伪造成功结果。

+ */ +@FunctionalInterface +public interface FederationStatisticsCollector { + + /** + * 采集一个物理数据源当前 revision 的表统计。 + * + * @param context 统计采集上下文 + * @return 按逻辑 Schema 和物理表索引的不可变统计;不支持时返回空映射 + * @throws SQLException 数据库目录或 JDBC 元数据读取失败 + */ + Map collect( + FederationStatisticsCollectionContext context + ) throws SQLException; +} diff --git a/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/api/FederationCleanupMetrics.java b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/api/FederationCleanupMetrics.java new file mode 100644 index 0000000..de2ab07 --- /dev/null +++ b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/api/FederationCleanupMetrics.java @@ -0,0 +1,35 @@ +package com.easyagents.federation.sql.api; + +import java.io.Serializable; + +/** + * 节点本地 JDBC 终止与游标清理通道的累计观测指标。 + * + * @param overflowFallbacks 主清理队列拒绝后转入隔离通道的次数 + * @param deferredRetries 隔离通道拒绝后进入有界延期重试队列的次数 + * @param unresolvedCleanups 延期队列溢出或 Engine 有界关闭后仍未完成的清理数 + * @param deferredQueueDepth 当前等待重试的清理数 + */ +public record FederationCleanupMetrics( + long overflowFallbacks, + long deferredRetries, + long unresolvedCleanups, + int deferredQueueDepth +) implements Serializable { + + private static final FederationCleanupMetrics EMPTY = new FederationCleanupMetrics( + 0L, + 0L, + 0L, + 0 + ); + + /** + * 返回无清理压力的空指标。 + * + * @return 空指标 + */ + public static FederationCleanupMetrics empty() { + return EMPTY; + } +} diff --git a/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/api/FederationSqlEngine.java b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/api/FederationSqlEngine.java new file mode 100644 index 0000000..229a74e --- /dev/null +++ b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/api/FederationSqlEngine.java @@ -0,0 +1,88 @@ +package com.easyagents.federation.sql.api; + +import com.easyagents.federation.sql.compile.FederationSqlPlan; +import com.easyagents.federation.sql.compile.SqlCompileRequest; +import com.easyagents.federation.sql.compile.SqlExplainRequest; +import com.easyagents.federation.sql.compile.SqlExplainResult; +import com.easyagents.federation.sql.execute.FederationResultCursor; +import com.easyagents.federation.sql.execute.QueryId; +import com.easyagents.federation.sql.source.FederationSourceManager; + +/** + * SQL 编译、补全、查询、Explain、取消和数据源管理的统一公共入口。 + */ +public interface FederationSqlEngine extends AutoCloseable { + + /** + * 返回数据源管理入口。 + * + * @return 数据源管理器 + */ + FederationSourceManager sources(); + + /** + * 编译节点本地计划。 + * + * @param request 编译请求 + * @return 节点本地计划 + */ + FederationSqlPlan compile(SqlCompileRequest request); + + /** + * 执行节点本地计划。 + * + * @param plan 编译计划 + * @param context 执行上下文 + * @return 流式游标 + */ + FederationResultCursor execute(FederationSqlPlan plan, SqlExecutionContext context); + + /** + * 在当前节点完成编译或缓存命中并立即执行。 + * + * @param command 可跨节点查询命令 + * @return 流式游标 + */ + FederationResultCursor query(SqlQueryCommand command); + + /** + * 返回不含运行对象的 Explain 结果。 + * + * @param request Explain 请求 + * @return Explain 结果 + */ + SqlExplainResult explain(SqlExplainRequest request); + + /** + * 根据当前查询范围返回 Calcite SQL 上下文补全候选。 + * + * @param request 补全请求 + * @return 替换区间与候选列表 + */ + SqlCompletionResult complete(SqlCompletionRequest request); + + /** + * 尝试取消当前节点正在执行的查询。 + * + * @param queryId 查询标识 + * @return 是否找到并发起取消 + */ + boolean cancel(QueryId queryId); + + /** + * 返回节点本地 JDBC 终止与游标清理通道的累计指标。 + * + *

自定义 Engine 未提供资源治理指标时返回空快照。

+ * + * @return 清理通道指标 + */ + default FederationCleanupMetrics cleanupMetrics() { + return FederationCleanupMetrics.empty(); + } + + /** + * 关闭 Engine、订阅、Runtime 和独占句柄。 + */ + @Override + void close(); +} diff --git a/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/api/FederationSqlEngines.java b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/api/FederationSqlEngines.java new file mode 100644 index 0000000..570fb87 --- /dev/null +++ b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/api/FederationSqlEngines.java @@ -0,0 +1,277 @@ +package com.easyagents.federation.sql.api; + +import com.easyagents.federation.sql.adapter.FederationSqlAdapterProvider; +import com.easyagents.federation.sql.adapter.FederationSqlAdapterRegistry; +import com.easyagents.federation.sql.compile.FederationSqlPolicy; +import com.easyagents.federation.sql.execute.FederationQueryAdmissionController; +import com.easyagents.federation.sql.execute.LocalFederationQueryAdmissionController; +import com.easyagents.federation.sql.federation.FederationExecutionPolicy; +import com.easyagents.federation.sql.federation.FederationTableStatisticsProvider; +import com.easyagents.federation.sql.runtime.DefaultFederationSqlEngine; +import com.easyagents.federation.sql.runtime.DefaultFederationSourceManager; +import com.easyagents.federation.sql.source.FederationDataSourceResolver; +import com.easyagents.federation.sql.source.FederationSourceStateProvider; +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; +import java.time.Duration; + +/** + * 使用显式依赖构建独立 FederationSqlEngine 的入口。 + */ +public final class FederationSqlEngines { + + private FederationSqlEngines() { + } + + /** + * 创建 Engine Builder。 + * + * @return Builder + */ + public static Builder builder() { + return new Builder(); + } + + /** + * FederationSqlEngine 的轻量配置 Builder。 + */ + public static final class Builder { + + private final List adapters = new ArrayList<>(); + private final List policies = new ArrayList<>(); + private FederationDataSourceResolver resolver; + private FederationSourceStateProvider stateProvider = FederationSourceStateProvider.none(); + private FederationQueryAdmissionController admissionController = + new LocalFederationQueryAdmissionController(64); + private int maximumPlanCacheEntries = 1024; + private long maximumPlanCacheWeightBytes = 64L * 1024L * 1024L; + private Duration planCacheTimeToLive = Duration.ofMinutes(30); + private int maximumConcurrentCompilations = Math.max( + 1, + Math.min(8, Runtime.getRuntime().availableProcessors()) + ); + private boolean crossSourceEnabled = true; + private FederationExecutionPolicy executionPolicy = FederationExecutionPolicy.basic(); + private long maximumNodeIntermediateBytes = 512L * 1024L * 1024L; + private FederationTableStatisticsProvider statisticsProvider; + private ClassLoader classLoader = Thread.currentThread().getContextClassLoader(); + + private Builder() { + } + + /** + * 设置调用方 DataSource Resolver。 + * + * @param resolver Resolver + * @return 当前 Builder + */ + public Builder dataSourceResolver(FederationDataSourceResolver resolver) { + this.resolver = Objects.requireNonNull(resolver, "resolver must not be null"); + return this; + } + + /** + * 显式注册 Adapter;同 id 时覆盖 ServiceLoader 实现。 + * + * @param adapter Adapter Provider + * @return 当前 Builder + */ + public Builder adapter(FederationSqlAdapterProvider adapter) { + this.adapters.add(Objects.requireNonNull(adapter, "adapter must not be null")); + return this; + } + + /** + * 增加 SQL 策略。 + * + * @param policy 策略 + * @return 当前 Builder + */ + public Builder policy(FederationSqlPolicy policy) { + this.policies.add(Objects.requireNonNull(policy, "policy must not be null")); + return this; + } + + /** + * 设置共享状态 Provider。 + * + * @param stateProvider 状态 Provider + * @return 当前 Builder + */ + public Builder stateProvider(FederationSourceStateProvider stateProvider) { + this.stateProvider = Objects.requireNonNull(stateProvider, "stateProvider must not be null"); + return this; + } + + /** + * 设置查询准入控制器。 + * + * @param admissionController 准入控制器 + * @return 当前 Builder + */ + public Builder admissionController(FederationQueryAdmissionController admissionController) { + this.admissionController = Objects.requireNonNull( + admissionController, + "admissionController must not be null" + ); + return this; + } + + /** + * 设置计划缓存最大条目数。 + * + * @param maximumPlanCacheEntries 最大条目数 + * @return 当前 Builder + */ + public Builder maximumPlanCacheEntries(int maximumPlanCacheEntries) { + if (maximumPlanCacheEntries <= 0) { + throw new IllegalArgumentException("maximumPlanCacheEntries must be positive"); + } + this.maximumPlanCacheEntries = maximumPlanCacheEntries; + return this; + } + + /** + * 设置计划缓存最大估算权重。 + * + * @param maximumPlanCacheWeightBytes 最大估算字节数 + * @return 当前 Builder + */ + public Builder maximumPlanCacheWeightBytes(long maximumPlanCacheWeightBytes) { + if (maximumPlanCacheWeightBytes <= 0) { + throw new IllegalArgumentException("maximumPlanCacheWeightBytes must be positive"); + } + this.maximumPlanCacheWeightBytes = maximumPlanCacheWeightBytes; + return this; + } + + /** + * 设置计划缓存条目存活时间。 + * + * @param planCacheTimeToLive 存活时间 + * @return 当前 Builder + */ + public Builder planCacheTimeToLive(Duration planCacheTimeToLive) { + if (planCacheTimeToLive == null + || planCacheTimeToLive.isZero() + || planCacheTimeToLive.isNegative()) { + throw new IllegalArgumentException("planCacheTimeToLive must be positive"); + } + this.planCacheTimeToLive = planCacheTimeToLive; + return this; + } + + /** + * 设置 Calcite 冷编译最大并发数。 + * + * @param maximumConcurrentCompilations 最大并发冷编译数 + * @return 当前 Builder + */ + public Builder maximumConcurrentCompilations(int maximumConcurrentCompilations) { + if (maximumConcurrentCompilations <= 0) { + throw new IllegalArgumentException("maximumConcurrentCompilations must be positive"); + } + this.maximumConcurrentCompilations = maximumConcurrentCompilations; + return this; + } + + /** + * 设置联邦执行开关。 + * + * @param enabled 是否开启 + * @return 当前 Builder + */ + public Builder crossSourceEnabled(boolean enabled) { + this.crossSourceEnabled = enabled; + return this; + } + + /** + * 设置 Engine 级联邦资源硬上限。 + * + * @param executionPolicy 资源策略 + * @return 当前 Builder + */ + public Builder federationExecutionPolicy(FederationExecutionPolicy executionPolicy) { + this.executionPolicy = Objects.requireNonNull( + executionPolicy, + "executionPolicy must not be null" + ); + return this; + } + + /** + * 设置节点同时预留的联邦中间结果内存总上限。 + * + * @param maximumNodeIntermediateBytes 节点内存上限 + * @return 当前 Builder + */ + public Builder maximumNodeIntermediateBytes(long maximumNodeIntermediateBytes) { + if (maximumNodeIntermediateBytes <= 0L) { + throw new IllegalArgumentException( + "maximumNodeIntermediateBytes must be positive" + ); + } + this.maximumNodeIntermediateBytes = maximumNodeIntermediateBytes; + return this; + } + + /** + * 设置联邦表统计 Provider,覆盖引擎内建的 Adapter 自动采集能力。 + * + * @param statisticsProvider 调用方完全托管的只读统计快照 Provider + * @return 当前 Builder + */ + public Builder tableStatisticsProvider( + FederationTableStatisticsProvider statisticsProvider + ) { + this.statisticsProvider = Objects.requireNonNull( + statisticsProvider, + "statisticsProvider must not be null" + ); + return this; + } + + /** + * 设置 ServiceLoader 类加载器。 + * + * @param classLoader 类加载器 + * @return 当前 Builder + */ + public Builder classLoader(ClassLoader classLoader) { + this.classLoader = Objects.requireNonNull(classLoader, "classLoader must not be null"); + return this; + } + + /** + * 构建独立 Engine。 + * + * @return Engine + */ + public FederationSqlEngine build() { + if (resolver == null) { + throw new IllegalStateException("dataSourceResolver must be configured"); + } + FederationSqlAdapterRegistry registry = new FederationSqlAdapterRegistry(adapters, classLoader); + DefaultFederationSourceManager sourceManager = new DefaultFederationSourceManager( + resolver, + registry, + stateProvider + ); + return new DefaultFederationSqlEngine( + sourceManager, + admissionController, + policies, + maximumPlanCacheEntries, + maximumConcurrentCompilations, + crossSourceEnabled, + executionPolicy, + maximumPlanCacheWeightBytes, + planCacheTimeToLive, + statisticsProvider, + maximumNodeIntermediateBytes + ); + } + } +} diff --git a/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/api/FederationSqlErrorCode.java b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/api/FederationSqlErrorCode.java new file mode 100644 index 0000000..821bf80 --- /dev/null +++ b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/api/FederationSqlErrorCode.java @@ -0,0 +1,71 @@ +package com.easyagents.federation.sql.api; + +/** + * SQL 联邦查询稳定错误码。 + */ +public enum FederationSqlErrorCode { + /** 公共参数不合法。 */ + INVALID_ARGUMENT, + /** Engine 或 SourceManager 已关闭。 */ + ENGINE_CLOSED, + /** SQL 解析失败。 */ + SQL_PARSE_FAILED, + /** SQL 校验失败。 */ + SQL_VALIDATION_FAILED, + /** SQL 超出只读查询基线。 */ + SQL_NOT_READ_ONLY, + /** SQL 编译或关系转换失败。 */ + SQL_COMPILE_FAILED, + /** SQL 冷编译等待或编译过程超过统一时限。 */ + SQL_COMPILE_TIMEOUT, + /** SQL 编辑器补全失败。 */ + SQL_COMPLETION_FAILED, + /** 单源计划仍含不可执行的本地残余算子。 */ + SQL_NOT_FULLY_PUSHDOWN, + /** 跨数据源能力未开启。 */ + CROSS_SOURCE_DISABLED, + /** 跨数据源执行在当前阶段未实现。 */ + CROSS_SOURCE_EXECUTION_UNSUPPORTED, + /** 查询范围或 Binding 声明不合法。 */ + INVALID_QUERY_SCOPE, + /** 联邦本地执行暂不支持当前关系算子。 */ + FEDERATION_OPERATOR_UNSUPPORTED, + /** 联邦中间结果行数、字节数或执行时间超过限制。 */ + FEDERATION_RESOURCE_LIMIT_EXCEEDED, + /** 节点本地计划绑定的 Runtime 身份已经失效。 */ + PLAN_STALE, + /** 数据源未登记且无法从共享状态恢复。 */ + SOURCE_NOT_FOUND, + /** 数据源已被墓碑删除。 */ + SOURCE_REMOVED, + /** 节点本地数据源版本不满足请求。 */ + SOURCE_REVISION_NOT_READY, + /** 同 revision 出现不同 Definition 校验和。 */ + SOURCE_DEFINITION_CONFLICT, + /** 数据源 Runtime 初始化失败。 */ + SOURCE_INITIALIZATION_FAILED, + /** Adapter 未注册。 */ + ADAPTER_NOT_FOUND, + /** Adapter 不支持当前数据库。 */ + ADAPTER_UNSUPPORTED, + /** SQL 动态参数数量不匹配。 */ + PARAMETER_COUNT_MISMATCH, + /** 查询准入等待超时或被中断。 */ + QUERY_ADMISSION_TIMEOUT, + /** 节点本地联邦中间结果内存准入超时。 */ + NODE_MEMORY_ADMISSION_TIMEOUT, + /** JDBC 连接池获取连接达到超时。 */ + CONNECTION_ACQUISITION_TIMEOUT, + /** JDBC 连接获取因网络、认证或连接池关闭等原因失败。 */ + CONNECTION_ACQUISITION_FAILED, + /** 查询被主动取消。 */ + QUERY_CANCELLED, + /** JDBC 查询或结果读取达到驱动超时。 */ + QUERY_TIMEOUT, + /** JDBC 查询执行失败。 */ + EXECUTION_FAILED, + /** 物理数据库 Explain 执行失败。 */ + EXPLAIN_FAILED, + /** JDBC 或 Runtime 资源关闭失败。 */ + RESOURCE_CLOSE_FAILED +} diff --git a/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/api/FederationSqlException.java b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/api/FederationSqlException.java new file mode 100644 index 0000000..d4e7af7 --- /dev/null +++ b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/api/FederationSqlException.java @@ -0,0 +1,44 @@ +package com.easyagents.federation.sql.api; + +import java.util.Objects; + +/** + * SQL 联邦查询异常,携带稳定错误码供调用方分类处理。 + */ +public class FederationSqlException extends RuntimeException { + + /** 稳定错误码。 */ + private final FederationSqlErrorCode errorCode; + + /** + * 创建异常。 + * + * @param errorCode 稳定错误码 + * @param message 可安全返回的错误说明 + */ + public FederationSqlException(FederationSqlErrorCode errorCode, String message) { + super(message); + this.errorCode = Objects.requireNonNull(errorCode, "errorCode must not be null"); + } + + /** + * 创建带原始原因的异常。 + * + * @param errorCode 稳定错误码 + * @param message 可安全返回的错误说明 + * @param cause 原始异常 + */ + public FederationSqlException(FederationSqlErrorCode errorCode, String message, Throwable cause) { + super(message, cause); + this.errorCode = Objects.requireNonNull(errorCode, "errorCode must not be null"); + } + + /** + * 返回稳定错误码。 + * + * @return 错误码 + */ + public FederationSqlErrorCode errorCode() { + return errorCode; + } +} diff --git a/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/api/SqlCompletionItem.java b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/api/SqlCompletionItem.java new file mode 100644 index 0000000..271588d --- /dev/null +++ b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/api/SqlCompletionItem.java @@ -0,0 +1,31 @@ +package com.easyagents.federation.sql.api; + +import java.util.List; + +/** + * 一个可插入 SQL 编辑器的补全候选。 + * + * @param label 面向用户展示的短名称 + * @param insertText Calcite 生成的替换文本 + * @param kind 候选类型 + * @param qualifiedName 候选的完整限定名称 + */ +public record SqlCompletionItem( + String label, + String insertText, + SqlCompletionKind kind, + List qualifiedName +) { + + /** + * 校验并防御性复制候选信息。 + */ + public SqlCompletionItem { + if (label == null || label.isBlank() || insertText == null || kind == null) { + throw new IllegalArgumentException( + "completion label, insertText and kind must be provided" + ); + } + qualifiedName = List.copyOf(qualifiedName == null ? List.of() : qualifiedName); + } +} diff --git a/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/api/SqlCompletionKind.java b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/api/SqlCompletionKind.java new file mode 100644 index 0000000..4640a24 --- /dev/null +++ b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/api/SqlCompletionKind.java @@ -0,0 +1,23 @@ +package com.easyagents.federation.sql.api; + +/** + * SQL 补全候选类型。 + */ +public enum SqlCompletionKind { + /** SQL 关键字。 */ + KEYWORD, + /** SQL 函数。 */ + FUNCTION, + /** 逻辑表。 */ + TABLE, + /** 逻辑视图。 */ + VIEW, + /** Schema。 */ + SCHEMA, + /** Catalog。 */ + CATALOG, + /** 字段。 */ + COLUMN, + /** 无法进一步分类的候选。 */ + OTHER +} diff --git a/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/api/SqlCompletionRequest.java b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/api/SqlCompletionRequest.java new file mode 100644 index 0000000..d83a150 --- /dev/null +++ b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/api/SqlCompletionRequest.java @@ -0,0 +1,29 @@ +package com.easyagents.federation.sql.api; + +import com.easyagents.federation.sql.federation.FederationQueryScopeDefinition; + +/** + * SQL 编辑器补全请求。 + * + * @param queryScope 当前编辑器可见的查询范围 + * @param sql 允许不完整的 SQL 文本 + * @param cursorOffset 光标 UTF-16 字符偏移 + */ +public record SqlCompletionRequest( + FederationQueryScopeDefinition queryScope, + String sql, + int cursorOffset +) { + + /** + * 校验补全请求。 + */ + public SqlCompletionRequest { + if (queryScope == null || sql == null) { + throw new IllegalArgumentException("queryScope and sql must be provided"); + } + if (cursorOffset < 0 || cursorOffset > sql.length()) { + throw new IllegalArgumentException("cursorOffset is outside the SQL text"); + } + } +} diff --git a/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/api/SqlCompletionResult.java b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/api/SqlCompletionResult.java new file mode 100644 index 0000000..5adbc35 --- /dev/null +++ b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/api/SqlCompletionResult.java @@ -0,0 +1,27 @@ +package com.easyagents.federation.sql.api; + +import java.util.List; + +/** + * SQL 补全结果。 + * + * @param replaceStart 建议替换区间起点,使用 UTF-16 字符偏移 + * @param replaceEnd 建议替换区间终点,使用 UTF-16 字符偏移 + * @param items 补全候选 + */ +public record SqlCompletionResult( + int replaceStart, + int replaceEnd, + List items +) { + + /** + * 校验并防御性复制补全结果。 + */ + public SqlCompletionResult { + if (replaceStart < 0 || replaceEnd < replaceStart) { + throw new IllegalArgumentException("completion replacement range is invalid"); + } + items = List.copyOf(items == null ? List.of() : items); + } +} diff --git a/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/api/SqlExecutionContext.java b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/api/SqlExecutionContext.java new file mode 100644 index 0000000..022370c --- /dev/null +++ b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/api/SqlExecutionContext.java @@ -0,0 +1,46 @@ +package com.easyagents.federation.sql.api; + +import com.easyagents.federation.sql.execute.QueryId; +import com.easyagents.federation.sql.execute.SqlExecutionOptions; +import com.easyagents.federation.sql.execute.SqlParameter; +import java.time.Duration; +import java.util.List; + +/** + * 执行节点本地编译计划的上下文。 + * + * @param queryId 查询标识 + * @param parameters 参数值 + * @param options JDBC 执行限制 + * @param admissionTimeout 查询准入等待上限 + */ +public record SqlExecutionContext( + QueryId queryId, + List parameters, + SqlExecutionOptions options, + Duration admissionTimeout +) { + + /** + * 校验并防御性复制执行上下文。 + */ + public SqlExecutionContext { + queryId = queryId == null ? QueryId.create() : queryId; + parameters = List.copyOf(parameters == null ? List.of() : parameters); + options = options == null ? SqlExecutionOptions.defaults() : options; + admissionTimeout = admissionTimeout == null ? Duration.ofSeconds(5) : admissionTimeout; + if (admissionTimeout.isNegative()) { + throw new IllegalArgumentException("admissionTimeout must not be negative"); + } + } + + /** + * 创建默认执行上下文。 + * + * @param parameters 参数值 + * @return 执行上下文 + */ + public static SqlExecutionContext of(List parameters) { + return new SqlExecutionContext(null, parameters, null, null); + } +} diff --git a/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/api/SqlQueryCommand.java b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/api/SqlQueryCommand.java new file mode 100644 index 0000000..e7049fb --- /dev/null +++ b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/api/SqlQueryCommand.java @@ -0,0 +1,155 @@ +package com.easyagents.federation.sql.api; + +import com.easyagents.federation.sql.execute.QueryId; +import com.easyagents.federation.sql.execute.SqlExecutionOptions; +import com.easyagents.federation.sql.execute.SqlParameter; +import com.easyagents.federation.sql.federation.FederationQueryScopeDefinition; +import com.easyagents.federation.sql.federation.FederationSourceBindingDefinition; +import com.easyagents.federation.sql.source.SourceId; +import java.io.Serializable; +import java.time.Duration; +import java.util.List; + +/** + * 可持久化或跨节点传递的一体化查询命令。 + * + * @param queryId 查询标识,可为空并在构造命令时生成 + * @param sql 单条只读 SQL + * @param queryScope 查询可见的数据源范围 + * @param parameters 参数值 + * @param options JDBC 执行限制 + * @param admissionTimeoutMillis 查询准入等待毫秒数 + * @param policyVersion 策略版本 + */ +public record SqlQueryCommand( + QueryId queryId, + String sql, + FederationQueryScopeDefinition queryScope, + List parameters, + SqlExecutionOptions options, + long admissionTimeoutMillis, + String policyVersion +) implements Serializable { + + /** + * 校验并防御性复制查询命令。 + */ + public SqlQueryCommand { + if (sql == null || sql.isBlank() || queryScope == null) { + throw new IllegalArgumentException("sql and queryScope must be provided"); + } + if (admissionTimeoutMillis < 0) { + throw new IllegalArgumentException("timeout must not be negative"); + } + queryId = queryId == null ? QueryId.create() : queryId; + parameters = List.copyOf(parameters == null ? List.of() : parameters); + options = options == null ? SqlExecutionOptions.defaults() : options; + policyVersion = policyVersion == null || policyVersion.isBlank() ? "default" : policyVersion; + } + + /** + * 使用单物理数据源创建兼容查询命令。 + * + * @param queryId 查询标识 + * @param sql 单条只读 SQL + * @param sourceId 默认数据源 + * @param minimumRevision 最低数据源版本 + * @param parameters 参数值 + * @param options JDBC 执行限制 + * @param admissionTimeoutMillis 准入等待毫秒数 + * @param policyVersion 策略版本 + */ + public SqlQueryCommand( + QueryId queryId, + String sql, + SourceId sourceId, + long minimumRevision, + List parameters, + SqlExecutionOptions options, + long admissionTimeoutMillis, + String policyVersion + ) { + this( + queryId, + sql, + FederationQueryScopeDefinition.single(sourceId, minimumRevision), + parameters, + options, + admissionTimeoutMillis, + policyVersion + ); + } + + /** + * 返回默认 Binding 的物理数据源,供单源调用方兼容读取。 + * + * @return 默认物理数据源 + */ + public SourceId sourceId() { + return defaultBinding().sourceId(); + } + + /** + * 返回默认 Binding 的最低物理 Definition 版本。 + * + * @return 最低版本 + */ + public long minimumRevision() { + return defaultBinding().minimumRevision(); + } + + private FederationSourceBindingDefinition defaultBinding() { + return queryScope.defaultBindingDefinition(); + } + + /** + * 创建使用默认执行限制的查询命令。 + * + * @param sql 单条只读 SQL + * @param sourceId 数据源标识 + * @param minimumRevision 最低数据源版本 + * @param parameters 参数 + * @return 查询命令 + */ + public static SqlQueryCommand of( + String sql, + SourceId sourceId, + long minimumRevision, + List parameters + ) { + return new SqlQueryCommand( + null, + sql, + sourceId, + minimumRevision, + parameters, + SqlExecutionOptions.defaults(), + Duration.ofSeconds(5).toMillis(), + "default" + ); + } + + /** + * 创建使用默认执行限制的查询范围命令。 + * + * @param sql 单条只读 SQL + * @param queryScope 查询范围 + * @param parameters 参数 + * @return 查询命令 + */ + public static SqlQueryCommand of( + String sql, + FederationQueryScopeDefinition queryScope, + List parameters + ) { + return new SqlQueryCommand( + null, + sql, + queryScope, + parameters, + SqlExecutionOptions.defaults(), + Duration.ofSeconds(5).toMillis(), + "default" + ); + } +} diff --git a/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/compile/FederationFragmentExplain.java b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/compile/FederationFragmentExplain.java new file mode 100644 index 0000000..50f4726 --- /dev/null +++ b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/compile/FederationFragmentExplain.java @@ -0,0 +1,93 @@ +package com.easyagents.federation.sql.compile; + +import com.easyagents.federation.sql.execute.FederationColumn; +import com.easyagents.federation.sql.execute.FederationPhysicalExplain; +import com.easyagents.federation.sql.federation.FederationCostEstimate; +import com.easyagents.federation.sql.source.SourceId; +import java.io.Serializable; +import java.util.List; + +/** + * Explain 中一个目标数据库分片的纯数据视图。 + * + * @param fragmentId 分片标识 + * @param bindingName 查询范围 Binding 名称 + * @param sourceId 物理数据源 + * @param adapterId Adapter 标识 + * @param executableSql 目标方言参数化 SQL + * @param parameterMapping 分片参数到原查询参数的映射 + * @param columns 分片输出列 + * @param costEstimate 分片搬运成本估算 + * @param pushedDownOperators 已下推算子 + * @param physicalExplain 显式物理 Explain;逻辑级别时为空 + */ +public record FederationFragmentExplain( + String fragmentId, + String bindingName, + SourceId sourceId, + String adapterId, + String executableSql, + List parameterMapping, + List columns, + FederationCostEstimate costEstimate, + List pushedDownOperators, + FederationPhysicalExplain physicalExplain +) implements Serializable { + + /** + * 防御性复制集合字段。 + */ + public FederationFragmentExplain { + parameterMapping = List.copyOf(parameterMapping == null ? List.of() : parameterMapping); + columns = List.copyOf(columns == null ? List.of() : columns); + pushedDownOperators = List.copyOf( + pushedDownOperators == null ? List.of() : pushedDownOperators + ); + } + + /** + * 创建旧字段集合的兼容 Explain 分片。 + * + * @param fragmentId 分片标识 + * @param bindingName Binding 名称 + * @param sourceId 物理源 + * @param adapterId Adapter 标识 + * @param executableSql 目标 SQL + * @param parameterMapping 参数映射 + * @param columns 输出列 + * @param physicalExplain 物理 Explain + */ + public FederationFragmentExplain( + String fragmentId, + String bindingName, + SourceId sourceId, + String adapterId, + String executableSql, + List parameterMapping, + List columns, + FederationPhysicalExplain physicalExplain + ) { + this( + fragmentId, + bindingName, + sourceId, + adapterId, + executableSql, + parameterMapping, + columns, + new FederationCostEstimate( + 0, + 0, + 0, + "calcite-default", + "none", + java.time.Instant.EPOCH, + true, + com.easyagents.federation.sql.federation.FederationStatisticsStatus.MISSING, + false + ), + List.of(), + physicalExplain + ); + } +} diff --git a/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/compile/FederationSqlPlan.java b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/compile/FederationSqlPlan.java new file mode 100644 index 0000000..7d2f6f9 --- /dev/null +++ b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/compile/FederationSqlPlan.java @@ -0,0 +1,191 @@ +package com.easyagents.federation.sql.compile; + +import com.easyagents.federation.sql.adapter.AdapterCompatibility; +import com.easyagents.federation.sql.execute.FederationColumn; +import com.easyagents.federation.sql.federation.FederationFragmentPlan; +import com.easyagents.federation.sql.federation.FederationJoinOptimization; +import com.easyagents.federation.sql.federation.FederationQueryMode; +import com.easyagents.federation.sql.federation.FederationQueryScopeDefinition; +import com.easyagents.federation.sql.federation.FederationSourceRuntimeIdentity; +import com.easyagents.federation.sql.source.SourceId; +import java.time.Instant; +import java.util.List; +import java.util.Set; +import org.apache.calcite.rel.RelRoot; +import org.apache.calcite.sql.SqlNode; + +/** + * Engine 签发的节点本地 SQL 编译计划。 + * + *

该接口仅用于读取编译事实。调用方不能自行创建可执行计划,且计划只能交回 + * 签发它的 Engine 实例执行。

+ */ +public interface FederationSqlPlan { + + /** + * 返回主数据源。 + * + * @return 主数据源 + */ + SourceId sourceId(); + + /** + * 返回编译时使用的不可变查询范围。 + * + * @return 查询范围 + */ + FederationQueryScopeDefinition queryScope(); + + /** + * 返回根据实际引用源确定的查询模式。 + * + * @return 查询模式 + */ + FederationQueryMode queryMode(); + + /** + * 返回数据源版本。 + * + * @return 数据源版本 + */ + long sourceRevision(); + + /** + * 返回 Calcite 规范化 SQL。 + * + * @return Calcite 规范化 SQL + */ + String normalizedSql(); + + /** + * 返回单源目标数据库参数化 SQL。 + * + *

联邦计划应读取 {@link #fragments()};本兼容视图不代表任一数据库可执行 SQL。

+ * + * @return 单源目标 SQL,或联邦调用方原始 SQL 兼容视图 + */ + String executableSql(); + + /** + * 返回 Calcite 已校验 SQL 节点。 + * + * @return Calcite 已校验 SQL 节点 + */ + SqlNode sqlNode(); + + /** + * 返回 Calcite 关系计划。 + * + * @return Calcite 关系计划 + */ + RelRoot relRoot(); + + /** + * 返回动态参数数量。 + * + * @return 动态参数数量 + */ + int parameterCount(); + + /** + * 返回编译时声明的原始 JDBC 参数类型。 + * + * @return JDBC 参数类型;未显式声明时为空 + */ + List parameterJdbcTypes(); + + /** + * 返回目标 SQL 占位符到原始参数的零基索引映射。 + * + * @return 参数映射 + */ + List parameterMapping(); + + /** + * 返回物理数据源分片;单源计划也包含一个分片。 + * + * @return 分片列表 + */ + List fragments(); + + /** + * 返回跨源 Join 的优化选择。 + * + * @return 不可变 Join 优化列表 + */ + default List joinOptimizations() { + return List.of(); + } + + /** + * 返回实际引用 Binding 对应的节点本地运行身份。 + * + * @return 运行身份列表 + */ + List sourceRuntimeIdentities(); + + /** + * 返回查询范围的稳定校验和。 + * + * @return Scope 校验和 + */ + String scopeChecksum(); + + /** + * 返回结果列。 + * + * @return 结果列 + */ + List columns(); + + /** + * 返回引用的数据源集合。 + * + * @return 引用的数据源集合 + */ + Set referencedSources(); + + /** + * 返回 Adapter 兼容性。 + * + * @return Adapter 兼容性 + */ + AdapterCompatibility compatibility(); + + /** + * 返回是否允许直接执行。 + * + * @return 是否允许直接执行 + */ + boolean executable(); + + /** + * 返回编译时的数据源 Definition 校验和。 + * + * @return Definition 校验和 + */ + String sourceChecksum(); + + /** + * 返回编译时的 Adapter 标识。 + * + * @return Adapter 标识 + */ + String adapterId(); + + /** + * 返回编译时的数据库与驱动指纹。 + * + * @return 运行指纹摘要 + */ + String runtimeFingerprint(); + + /** + * 返回该计划所依赖统计快照的最早失效时间。 + * + * @return 最早失效时间;未使用有期限统计时为 {@link Instant#MAX} + */ + default Instant statisticsValidUntil() { + return Instant.MAX; + } +} diff --git a/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/compile/FederationSqlPolicy.java b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/compile/FederationSqlPolicy.java new file mode 100644 index 0000000..f1845bd --- /dev/null +++ b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/compile/FederationSqlPolicy.java @@ -0,0 +1,27 @@ +package com.easyagents.federation.sql.compile; + +/** + * 调用方在 SQL 已校验并转换为 RelRoot 后执行的策略 SPI。 + */ +@FunctionalInterface +public interface FederationSqlPolicy { + + /** + * 返回策略实现的稳定版本,用于隔离计划缓存。 + * + *

策略规则发生变化时应同步更新版本。默认版本适用于 Engine 生命周期内 + * 逻辑不变的无状态策略。

+ * + * @return 稳定策略版本 + */ + default String version() { + return "1"; + } + + /** + * 校验已编译 SQL;拒绝时应抛出 FederationSqlException。 + * + * @param context 策略上下文 + */ + void validate(SqlPolicyContext context); +} diff --git a/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/compile/SqlCompileRequest.java b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/compile/SqlCompileRequest.java new file mode 100644 index 0000000..fdee17e --- /dev/null +++ b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/compile/SqlCompileRequest.java @@ -0,0 +1,108 @@ +package com.easyagents.federation.sql.compile; + +import com.easyagents.federation.sql.federation.FederationQueryScopeDefinition; +import com.easyagents.federation.sql.federation.FederationSourceBindingDefinition; +import com.easyagents.federation.sql.source.SourceId; +import java.util.List; + +/** + * 节点本地 SQL 编译请求。 + * + * @param sql 单条只读 SQL + * @param queryScope 查询可见的数据源范围 + * @param parameterJdbcTypes 参数 JDBC 类型列表 + * @param policyVersion 调用方策略版本,用于隔离计划缓存 + */ +public record SqlCompileRequest( + String sql, + FederationQueryScopeDefinition queryScope, + List parameterJdbcTypes, + String policyVersion +) { + + /** + * 校验并防御性复制编译请求。 + */ + public SqlCompileRequest { + if (sql == null || sql.isBlank()) { + throw new IllegalArgumentException("sql must not be blank"); + } + if (queryScope == null) { + throw new IllegalArgumentException("queryScope must not be null"); + } + parameterJdbcTypes = List.copyOf(parameterJdbcTypes == null ? List.of() : parameterJdbcTypes); + policyVersion = policyVersion == null || policyVersion.isBlank() ? "default" : policyVersion; + } + + /** + * 使用单物理数据源创建兼容编译请求。 + * + * @param sql 单条只读 SQL + * @param sourceId 默认数据源 + * @param minimumRevision 最低数据源版本 + * @param parameterJdbcTypes 参数 JDBC 类型 + * @param policyVersion 调用方策略版本 + */ + public SqlCompileRequest( + String sql, + SourceId sourceId, + long minimumRevision, + List parameterJdbcTypes, + String policyVersion + ) { + this( + sql, + FederationQueryScopeDefinition.single(sourceId, minimumRevision), + parameterJdbcTypes, + policyVersion + ); + } + + /** + * 返回默认 Binding 的物理数据源,供单源调用方兼容读取。 + * + * @return 默认物理数据源 + */ + public SourceId sourceId() { + return defaultBinding().sourceId(); + } + + /** + * 返回默认 Binding 的最低物理 Definition 版本。 + * + * @return 最低版本 + */ + public long minimumRevision() { + return defaultBinding().minimumRevision(); + } + + private FederationSourceBindingDefinition defaultBinding() { + return queryScope.defaultBindingDefinition(); + } + + /** + * 创建无参数的默认编译请求。 + * + * @param sql 单条只读 SQL + * @param sourceId 默认数据源 + * @param minimumRevision 最低数据源版本 + * @return 编译请求 + */ + public static SqlCompileRequest of(String sql, SourceId sourceId, long minimumRevision) { + return new SqlCompileRequest(sql, sourceId, minimumRevision, List.of(), "default"); + } + + /** + * 创建无参数的查询范围编译请求。 + * + * @param sql 单条只读 SQL + * @param queryScope 查询范围 + * @return 编译请求 + */ + public static SqlCompileRequest of( + String sql, + FederationQueryScopeDefinition queryScope + ) { + return new SqlCompileRequest(sql, queryScope, List.of(), "default"); + } +} diff --git a/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/compile/SqlExplainLevel.java b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/compile/SqlExplainLevel.java new file mode 100644 index 0000000..4d331b4 --- /dev/null +++ b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/compile/SqlExplainLevel.java @@ -0,0 +1,13 @@ +package com.easyagents.federation.sql.compile; + +/** + * Explain 深度。 + */ +public enum SqlExplainLevel { + + /** 只生成 Calcite 逻辑计划和物理分片 SQL,不访问数据库 Optimizer。 */ + LOGICAL, + + /** 在逻辑计划基础上显式请求各物理数据库的非 ANALYZE Explain。 */ + PHYSICAL +} diff --git a/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/compile/SqlExplainRequest.java b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/compile/SqlExplainRequest.java new file mode 100644 index 0000000..93fe1fd --- /dev/null +++ b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/compile/SqlExplainRequest.java @@ -0,0 +1,54 @@ +package com.easyagents.federation.sql.compile; + +import com.easyagents.federation.sql.execute.SqlParameter; +import java.util.List; + +/** + * SQL Explain 请求。 + * + * @param compileRequest 编译请求 + * @param level Explain 深度 + * @param parameters 保留的兼容字段;为避免数据库计划回显敏感值,只允许为空 + */ +public record SqlExplainRequest( + SqlCompileRequest compileRequest, + SqlExplainLevel level, + List parameters +) { + + /** + * 校验 Explain 请求。 + */ + public SqlExplainRequest { + if (compileRequest == null) { + throw new IllegalArgumentException("compileRequest must not be null"); + } + level = level == null ? SqlExplainLevel.PHYSICAL : level; + parameters = List.copyOf(parameters == null ? List.of() : parameters); + if (!parameters.isEmpty()) { + throw new IllegalArgumentException( + "physical Explain does not accept parameter values; " + + "declare JDBC types in compileRequest" + ); + } + } + + /** + * 创建默认物理 Explain 请求,不提供实际参数值。 + * + * @param compileRequest 编译请求 + */ + public SqlExplainRequest(SqlCompileRequest compileRequest) { + this(compileRequest, SqlExplainLevel.PHYSICAL, List.of()); + } + + /** + * 创建指定深度的 Explain 请求。 + * + * @param compileRequest 编译请求 + * @param level Explain 深度 + */ + public SqlExplainRequest(SqlCompileRequest compileRequest, SqlExplainLevel level) { + this(compileRequest, level, List.of()); + } +} diff --git a/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/compile/SqlExplainResult.java b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/compile/SqlExplainResult.java new file mode 100644 index 0000000..07b052b --- /dev/null +++ b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/compile/SqlExplainResult.java @@ -0,0 +1,165 @@ +package com.easyagents.federation.sql.compile; + +import com.easyagents.federation.sql.adapter.AdapterCompatibility; +import com.easyagents.federation.sql.federation.FederationQueryMode; +import com.easyagents.federation.sql.federation.FederationJoinOptimization; +import com.easyagents.federation.sql.federation.FederationStatisticsStatus; +import com.easyagents.federation.sql.source.SourceId; +import java.io.Serializable; +import java.util.List; +import java.util.Set; + +/** + * 不含节点本地 Calcite/JDBC 对象的 Explain 结果。 + * + * @param level Explain 深度 + * @param queryMode 实际查询模式 + * @param statisticsStatus 成本统计完整性与时效状态 + * @param estimateAvailable 聚合成本数值是否为有效估算 + * @param estimatedTransferBytes 预计从物理源搬运的总字节数 + * @param estimatedLocalMemoryBytes 本地 Join 构建侧估算内存字节数 + * @param joinOptimizations 跨源 Join 优化选择 + * @param normalizedSql Calcite 规范化 SQL + * @param executableSql 单源目标方言 SQL;联邦计划仅为兼容视图,应读取 fragments + * @param relationalPlan 关系计划文本 + * @param executionPlan 实际单源下推或联邦本地执行计划文本 + * @param fragments 物理分片与可选数据库计划 + * @param referencedSources 引用的数据源 + * @param compatibility Adapter 兼容性 + * @param executable 是否允许执行 + * @param planCacheHit 是否命中节点本地计划缓存 + * @param diagnostic 诊断说明 + */ +public record SqlExplainResult( + SqlExplainLevel level, + FederationQueryMode queryMode, + FederationStatisticsStatus statisticsStatus, + boolean estimateAvailable, + double estimatedTransferBytes, + long estimatedLocalMemoryBytes, + List joinOptimizations, + String normalizedSql, + String executableSql, + String relationalPlan, + String executionPlan, + List fragments, + Set referencedSources, + AdapterCompatibility compatibility, + boolean executable, + boolean planCacheHit, + String diagnostic +) implements Serializable { + + /** + * 防御性复制引用集合。 + */ + public SqlExplainResult { + level = level == null ? SqlExplainLevel.LOGICAL : level; + queryMode = queryMode == null ? FederationQueryMode.SINGLE_SOURCE : queryMode; + statisticsStatus = statisticsStatus == null + ? FederationStatisticsStatus.MISSING + : statisticsStatus; + if (!Double.isFinite(estimatedTransferBytes) || estimatedTransferBytes < 0 + || estimatedLocalMemoryBytes < 0) { + throw new IllegalArgumentException("Explain cost values must be non-negative"); + } + joinOptimizations = List.copyOf( + joinOptimizations == null ? List.of() : joinOptimizations + ); + fragments = List.copyOf(fragments == null ? List.of() : fragments); + referencedSources = Set.copyOf(referencedSources); + diagnostic = diagnostic == null ? "" : diagnostic; + } + + /** + * 创建未包含聚合成本字段的兼容 Explain 结果。 + * + * @param level Explain 深度 + * @param queryMode 查询模式 + * @param normalizedSql 规范化 SQL + * @param executableSql 可执行 SQL + * @param relationalPlan 关系计划 + * @param executionPlan 执行计划 + * @param fragments 分片计划 + * @param referencedSources 引用源 + * @param compatibility Adapter 兼容性 + * @param executable 是否可执行 + * @param planCacheHit 是否命中缓存 + * @param diagnostic 诊断信息 + */ + public SqlExplainResult( + SqlExplainLevel level, + FederationQueryMode queryMode, + String normalizedSql, + String executableSql, + String relationalPlan, + String executionPlan, + List fragments, + Set referencedSources, + AdapterCompatibility compatibility, + boolean executable, + boolean planCacheHit, + String diagnostic + ) { + this( + level, + queryMode, + FederationStatisticsStatus.MISSING, + false, + 0D, + 0L, + List.of(), + normalizedSql, + executableSql, + relationalPlan, + executionPlan, + fragments, + referencedSources, + compatibility, + executable, + planCacheHit, + diagnostic + ); + } + + /** + * 创建旧单源字段视图的兼容 Explain 结果。 + * + * @param normalizedSql Calcite 规范化 SQL + * @param executableSql 目标方言 SQL + * @param relationalPlan 关系计划文本 + * @param referencedSources 引用的数据源 + * @param compatibility Adapter 兼容性 + * @param executable 是否允许执行 + * @param diagnostic 诊断说明 + */ + public SqlExplainResult( + String normalizedSql, + String executableSql, + String relationalPlan, + Set referencedSources, + AdapterCompatibility compatibility, + boolean executable, + String diagnostic + ) { + this( + SqlExplainLevel.LOGICAL, + FederationQueryMode.SINGLE_SOURCE, + FederationStatisticsStatus.MISSING, + false, + 0D, + 0L, + List.of(), + normalizedSql, + executableSql, + relationalPlan, + relationalPlan, + List.of(), + referencedSources, + compatibility, + executable, + false, + diagnostic + ); + } +} diff --git a/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/compile/SqlPolicyContext.java b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/compile/SqlPolicyContext.java new file mode 100644 index 0000000..d63a162 --- /dev/null +++ b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/compile/SqlPolicyContext.java @@ -0,0 +1,22 @@ +package com.easyagents.federation.sql.compile; + +import com.easyagents.federation.sql.source.SourceId; +import java.util.Set; +import org.apache.calcite.rel.RelRoot; +import org.apache.calcite.sql.SqlNode; + +/** + * SQL 策略直接读取 Calcite 事实对象的上下文。 + * + * @param request 原始编译请求 + * @param validatedSql 已校验 SqlNode + * @param relRoot 关系计划 + * @param referencedSources 引用的数据源 + */ +public record SqlPolicyContext( + SqlCompileRequest request, + SqlNode validatedSql, + RelRoot relRoot, + Set referencedSources +) { +} diff --git a/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/execute/FederationColumn.java b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/execute/FederationColumn.java new file mode 100644 index 0000000..1f878a6 --- /dev/null +++ b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/execute/FederationColumn.java @@ -0,0 +1,21 @@ +package com.easyagents.federation.sql.execute; + +import java.io.Serializable; + +/** + * 查询结果列元数据。 + * + * @param index 从 1 开始的列序号 + * @param label 列标签 + * @param jdbcType JDBC 类型 + * @param typeName 数据库类型名 + * @param nullable 是否允许空值 + */ +public record FederationColumn( + int index, + String label, + int jdbcType, + String typeName, + boolean nullable +) implements Serializable { +} diff --git a/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/execute/FederationExecutionGuard.java b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/execute/FederationExecutionGuard.java new file mode 100644 index 0000000..8289dd5 --- /dev/null +++ b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/execute/FederationExecutionGuard.java @@ -0,0 +1,67 @@ +package com.easyagents.federation.sql.execute; + +import java.util.concurrent.TimeUnit; + +/** + * 贯穿连接获取、Statement 执行和结果读取的查询终态检查器。 + */ +public interface FederationExecutionGuard { + + /** + * 检查查询是否仍允许继续执行。 + * + * @throws RuntimeException 查询取消或超时时抛出稳定异常 + */ + void ensureAllowed(); + + /** + * 返回查询剩余时限。 + * + * @return 剩余纳秒数;无限制时返回 {@link Long#MAX_VALUE} + */ + long remainingNanos(); + + /** + * 将调用方 JDBC 秒级超时收敛到统一剩余时限。 + * + * @param requestedSeconds 调用方超时,0 表示未指定 + * @return 至少 1 秒的 JDBC 超时;无限制且未指定时返回 0 + */ + default int boundedQueryTimeoutSeconds(int requestedSeconds) { + if (remainingNanos() == Long.MAX_VALUE) { + return requestedSeconds; + } + long remainingSeconds = Math.max( + 1L, + TimeUnit.NANOSECONDS.toSeconds(Math.max(1L, remainingNanos())) + ); + int bounded = (int) Math.min(Integer.MAX_VALUE, remainingSeconds); + return requestedSeconds == 0 ? bounded : Math.min(requestedSeconds, bounded); + } + + /** + * 返回无限制检查器,供旧 Adapter 调用兼容使用。 + * + * @return 无限制检查器 + */ + static FederationExecutionGuard none() { + return NoopHolder.INSTANCE; + } + + /** 无状态实例持有者。 */ + final class NoopHolder { + private static final FederationExecutionGuard INSTANCE = new FederationExecutionGuard() { + @Override + public void ensureAllowed() { + } + + @Override + public long remainingNanos() { + return Long.MAX_VALUE; + } + }; + + private NoopHolder() { + } + } +} diff --git a/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/execute/FederationExecutionObserver.java b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/execute/FederationExecutionObserver.java new file mode 100644 index 0000000..9f37b74 --- /dev/null +++ b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/execute/FederationExecutionObserver.java @@ -0,0 +1,41 @@ +package com.easyagents.federation.sql.execute; + +/** + * Adapter 向 Core 回传 Fragment 执行阶段耗时的轻量观察器。 + */ +public interface FederationExecutionObserver { + + /** + * 记录获取物理连接的耗时。 + * + * @param elapsedNanos 获取连接耗时 + */ + default void connectionAcquired(long elapsedNanos) { + } + + /** + * 记录数据库完成 Statement 执行并返回 ResultSet 的耗时。 + * + * @param elapsedNanos 数据库执行耗时 + */ + default void databaseExecutionCompleted(long elapsedNanos) { + } + + /** + * 记录 ResultSet 返回首行的耗时。 + * + * @param elapsedNanos 从 ResultSet 创建到首行可用的耗时 + */ + default void firstRowAvailable(long elapsedNanos) { + } + + /** + * 返回不采集指标的观察器。 + * + * @return 空观察器 + */ + static FederationExecutionObserver none() { + return new FederationExecutionObserver() { + }; + } +} diff --git a/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/execute/FederationFragmentExecutionContext.java b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/execute/FederationFragmentExecutionContext.java new file mode 100644 index 0000000..b0e456c --- /dev/null +++ b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/execute/FederationFragmentExecutionContext.java @@ -0,0 +1,122 @@ +package com.easyagents.federation.sql.execute; + +import com.easyagents.federation.sql.adapter.AdapterCompatibility; +import java.util.List; +import java.util.Map; +import javax.sql.DataSource; + +/** + * 单数据源 SQL Fragment 的执行上下文。 + * + * @param queryId 查询标识 + * @param sql 已按目标方言生成的参数化 SQL + * @param parameters JDBC 参数 + * @param options 强制执行限制 + * @param dataSource 调用方提供的 DataSource + * @param compatibility 当前数据库与驱动兼容性信息 + * @param adapterOptions 不含凭据的 Adapter 执行选项 + * @param statementLifecycle Statement 取消登记回调 + * @param observer Fragment 执行阶段观察器 + * @param executionGuard 查询取消与统一截止时间检查器 + */ +public record FederationFragmentExecutionContext( + QueryId queryId, + String sql, + List parameters, + SqlExecutionOptions options, + DataSource dataSource, + AdapterCompatibility compatibility, + Map adapterOptions, + StatementLifecycle statementLifecycle, + FederationExecutionObserver observer, + FederationExecutionGuard executionGuard +) { + + /** + * 防御性复制参数并校验必需字段。 + */ + public FederationFragmentExecutionContext { + if (queryId == null || sql == null || sql.isBlank() || options == null + || dataSource == null || compatibility == null || statementLifecycle == null) { + throw new IllegalArgumentException("fragment execution context contains null or blank values"); + } + parameters = List.copyOf(parameters == null ? List.of() : parameters); + adapterOptions = Map.copyOf(adapterOptions == null ? Map.of() : adapterOptions); + observer = observer == null ? FederationExecutionObserver.none() : observer; + executionGuard = executionGuard == null ? FederationExecutionGuard.none() : executionGuard; + } + + /** + * 创建不采集 Adapter 阶段指标的兼容执行上下文。 + * + * @param queryId 查询标识 + * @param sql 参数化 SQL + * @param parameters JDBC 参数 + * @param options 执行限制 + * @param dataSource 数据源 + * @param compatibility 数据库兼容信息 + * @param adapterOptions Adapter 选项 + * @param statementLifecycle Statement 生命周期 + */ + public FederationFragmentExecutionContext( + QueryId queryId, + String sql, + List parameters, + SqlExecutionOptions options, + DataSource dataSource, + AdapterCompatibility compatibility, + Map adapterOptions, + StatementLifecycle statementLifecycle + ) { + this( + queryId, + sql, + parameters, + options, + dataSource, + compatibility, + adapterOptions, + statementLifecycle, + FederationExecutionObserver.none(), + FederationExecutionGuard.none() + ); + } + + /** + * 创建带阶段观察器的旧调用兼容上下文。 + * + * @param queryId 查询标识 + * @param sql 参数化 SQL + * @param parameters JDBC 参数 + * @param options 执行限制 + * @param dataSource 数据源 + * @param compatibility 数据库兼容信息 + * @param adapterOptions Adapter 选项 + * @param statementLifecycle Statement 生命周期 + * @param observer Fragment 观察器 + */ + public FederationFragmentExecutionContext( + QueryId queryId, + String sql, + List parameters, + SqlExecutionOptions options, + DataSource dataSource, + AdapterCompatibility compatibility, + Map adapterOptions, + StatementLifecycle statementLifecycle, + FederationExecutionObserver observer + ) { + this( + queryId, + sql, + parameters, + options, + dataSource, + compatibility, + adapterOptions, + statementLifecycle, + observer, + FederationExecutionGuard.none() + ); + } +} diff --git a/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/execute/FederationFragmentExecutor.java b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/execute/FederationFragmentExecutor.java new file mode 100644 index 0000000..5d0fb8a --- /dev/null +++ b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/execute/FederationFragmentExecutor.java @@ -0,0 +1,16 @@ +package com.easyagents.federation.sql.execute; + +/** + * Adapter 提供的单数据源 SQL Fragment 执行器。 + */ +@FunctionalInterface +public interface FederationFragmentExecutor { + + /** + * 执行参数化 SQL 并返回流式游标。 + * + * @param context 执行上下文 + * @return 流式游标 + */ + FederationResultCursor execute(FederationFragmentExecutionContext context); +} diff --git a/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/execute/FederationFragmentExplainContext.java b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/execute/FederationFragmentExplainContext.java new file mode 100644 index 0000000..50b0492 --- /dev/null +++ b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/execute/FederationFragmentExplainContext.java @@ -0,0 +1,74 @@ +package com.easyagents.federation.sql.execute; + +import com.easyagents.federation.sql.adapter.AdapterCompatibility; +import java.util.List; +import java.util.Map; +import javax.sql.DataSource; + +/** + * Adapter 执行单个物理分片 Explain 的上下文。 + * + * @param sql 目标数据库方言参数化 SQL + * @param parameters 已按分片参数映射排序的参数 + * @param dataSource 调用方提供的 DataSource + * @param compatibility 数据库与驱动兼容性 + * @param adapterOptions 不含凭据的 Adapter 选项 + * @param queryTimeoutSeconds Explain 超时秒数 + * @param executionGuard 统一查询终态与截止时间检查器 + */ +public record FederationFragmentExplainContext( + String sql, + List parameters, + DataSource dataSource, + AdapterCompatibility compatibility, + Map adapterOptions, + int queryTimeoutSeconds, + FederationExecutionGuard executionGuard +) { + + /** + * 校验并创建不可变 Explain 上下文。 + */ + public FederationFragmentExplainContext { + if (sql == null || sql.isBlank() || dataSource == null || compatibility == null) { + throw new IllegalArgumentException("fragment Explain context is incomplete"); + } + if (queryTimeoutSeconds < 0) { + throw new IllegalArgumentException("queryTimeoutSeconds must not be negative"); + } + parameters = List.copyOf(parameters == null ? List.of() : parameters); + adapterOptions = Map.copyOf(adapterOptions == null ? Map.of() : adapterOptions); + executionGuard = executionGuard == null + ? FederationExecutionGuard.none() + : executionGuard; + } + + /** + * 保留旧 Adapter 与调用方的兼容构造器。 + * + * @param sql 目标数据库 SQL + * @param parameters 参数 + * @param dataSource 数据源 + * @param compatibility 兼容性信息 + * @param adapterOptions Adapter 选项 + * @param queryTimeoutSeconds Explain 超时秒数 + */ + public FederationFragmentExplainContext( + String sql, + List parameters, + DataSource dataSource, + AdapterCompatibility compatibility, + Map adapterOptions, + int queryTimeoutSeconds + ) { + this( + sql, + parameters, + dataSource, + compatibility, + adapterOptions, + queryTimeoutSeconds, + FederationExecutionGuard.none() + ); + } +} diff --git a/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/execute/FederationFragmentExplainer.java b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/execute/FederationFragmentExplainer.java new file mode 100644 index 0000000..2c8123d --- /dev/null +++ b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/execute/FederationFragmentExplainer.java @@ -0,0 +1,16 @@ +package com.easyagents.federation.sql.execute; + +/** + * Adapter 可选的物理数据库 Explain SPI。 + */ +@FunctionalInterface +public interface FederationFragmentExplainer { + + /** + * 执行不会运行真实数据查询的物理 Explain。 + * + * @param context 分片 Explain 上下文 + * @return 物理计划 + */ + FederationPhysicalExplain explain(FederationFragmentExplainContext context); +} diff --git a/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/execute/FederationFragmentMetrics.java b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/execute/FederationFragmentMetrics.java new file mode 100644 index 0000000..a56da5f --- /dev/null +++ b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/execute/FederationFragmentMetrics.java @@ -0,0 +1,51 @@ +package com.easyagents.federation.sql.execute; + +import com.easyagents.federation.sql.source.SourceId; +import java.io.Serializable; + +/** + * 单个物理分片的查询消耗快照。 + * + * @param fragmentId 分片标识 + * @param sourceId 物理数据源标识 + * @param rowsRead 已读取行数 + * @param bytesRead 已读取估算字节数;Adapter 未安全提供时为 -1 + * @param elapsedNanos 当前或最终耗时 + * @param connectionAcquireNanos 获取连接耗时 + * @param databaseExecutionNanos Statement 返回 ResultSet 的耗时 + * @param firstRowNanos ResultSet 创建到首行可用的耗时;不可用时为 -1 + * @param complete 是否已完成或关闭 + */ +public record FederationFragmentMetrics( + String fragmentId, + SourceId sourceId, + long rowsRead, + long bytesRead, + long elapsedNanos, + long connectionAcquireNanos, + long databaseExecutionNanos, + long firstRowNanos, + boolean complete +) implements Serializable { + + /** + * 创建只包含旧基础字段的兼容分片指标。 + * + * @param fragmentId 分片标识 + * @param sourceId 数据源 + * @param rowsRead 读取行数 + * @param bytesRead 读取字节数 + * @param elapsedNanos 耗时 + * @param complete 是否完成 + */ + public FederationFragmentMetrics( + String fragmentId, + SourceId sourceId, + long rowsRead, + long bytesRead, + long elapsedNanos, + boolean complete + ) { + this(fragmentId, sourceId, rowsRead, bytesRead, elapsedNanos, 0, 0, -1, complete); + } +} diff --git a/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/execute/FederationLocalOperatorMetrics.java b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/execute/FederationLocalOperatorMetrics.java new file mode 100644 index 0000000..6421478 --- /dev/null +++ b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/execute/FederationLocalOperatorMetrics.java @@ -0,0 +1,19 @@ +package com.easyagents.federation.sql.execute; + +import java.io.Serializable; + +/** + * Calcite 本地联邦算子的累计执行指标。 + * + * @param operatorName 算子名称 + * @param outputRows 交给下游的输出行数 + * @param outputBytes 输出估算字节数 + * @param executionNanos 算子产生输出的累计耗时 + */ +public record FederationLocalOperatorMetrics( + String operatorName, + long outputRows, + long outputBytes, + long executionNanos +) implements Serializable { +} diff --git a/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/execute/FederationPhysicalExplain.java b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/execute/FederationPhysicalExplain.java new file mode 100644 index 0000000..1fc9da7 --- /dev/null +++ b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/execute/FederationPhysicalExplain.java @@ -0,0 +1,59 @@ +package com.easyagents.federation.sql.execute; + +import java.io.Serializable; +import java.util.List; + +/** + * 物理数据库 Optimizer 的非 ANALYZE Explain 结果。 + * + * @param available 是否获得物理计划 + * @param nativePlan 数据库原生计划文本 + * @param nodeType 首个主要计划节点类型 + * @param scanType 扫描或访问方式 + * @param candidateIndexes 数据库返回的候选索引 + * @param chosenIndex 数据库选择的索引 + * @param estimatedRows 数据库估算行数 + * @param extraCondition 额外过滤或索引条件 + * @param diagnostic 不含凭据和参数值的诊断 + */ +public record FederationPhysicalExplain( + boolean available, + String nativePlan, + String nodeType, + String scanType, + List candidateIndexes, + String chosenIndex, + Long estimatedRows, + String extraCondition, + String diagnostic +) implements Serializable { + + /** + * 防御性复制候选索引。 + */ + public FederationPhysicalExplain { + candidateIndexes = List.copyOf(candidateIndexes == null ? List.of() : candidateIndexes); + nativePlan = nativePlan == null ? "" : nativePlan; + diagnostic = diagnostic == null ? "" : diagnostic; + } + + /** + * 创建数据库不支持或未能提供物理计划的结果。 + * + * @param diagnostic 诊断说明 + * @return 不可用结果 + */ + public static FederationPhysicalExplain unavailable(String diagnostic) { + return new FederationPhysicalExplain( + false, + "", + null, + null, + List.of(), + null, + null, + null, + diagnostic + ); + } +} diff --git a/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/execute/FederationQueryAdmissionController.java b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/execute/FederationQueryAdmissionController.java new file mode 100644 index 0000000..90d3139 --- /dev/null +++ b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/execute/FederationQueryAdmissionController.java @@ -0,0 +1,98 @@ +package com.easyagents.federation.sql.execute; + +import com.easyagents.federation.sql.api.FederationSqlErrorCode; +import com.easyagents.federation.sql.api.FederationSqlException; +import com.easyagents.federation.sql.source.SourceId; +import java.time.Duration; +import java.util.function.BooleanSupplier; + +/** + * 可替换的查询并发准入控制器。 + */ +@FunctionalInterface +public interface FederationQueryAdmissionController extends AutoCloseable { + + /** + * 获取查询许可。 + * + * @param sourceId 数据源标识 + * @param queryId 查询标识 + * @param timeout 最大等待时间 + * @return 查询许可 + */ + FederationQueryPermit acquire(SourceId sourceId, QueryId queryId, Duration timeout); + + /** + * 获取支持查询级取消的许可。 + * + *

自定义实现可以覆盖此方法及时中断分布式或远程准入等待。

+ * + * @param sourceId 数据源标识 + * @param queryId 查询标识 + * @param timeout 最大等待时间 + * @param cancellationRequested 取消状态 + * @return 查询许可 + */ + default FederationQueryPermit acquire( + SourceId sourceId, + QueryId queryId, + Duration timeout, + BooleanSupplier cancellationRequested + ) { + return acquire(sourceId, queryId, timeout); + } + + /** + * 为一次单源或联邦查询获取一份查询级许可。 + * + *

兼容实现只接受单源请求。联邦查询必须由实现方明确覆盖本方法,避免其余 + * 物理源静默绕过源级配额。

+ * + * @param request 查询级准入请求 + * @return 查询许可 + */ + default FederationQueryPermit acquire(QueryAdmissionRequest request) { + if (request.sourceIds().size() != 1) { + throw new FederationSqlException( + FederationSqlErrorCode.INVALID_QUERY_SCOPE, + "admission controller does not declare multi-source query support" + ); + } + return acquire( + request.primarySourceId(), + request.queryId(), + request.timeout(), + request.cancellationRequested() + ); + } + + /** + * 关闭控制器;默认无额外资源。 + */ + @Override + default void close() { + } + + /** + * 返回无并发限制的控制器。 + * + * @return 无限制控制器 + */ + static FederationQueryAdmissionController unlimited() { + return new FederationQueryAdmissionController() { + @Override + public FederationQueryPermit acquire( + SourceId sourceId, + QueryId queryId, + Duration timeout + ) { + return () -> { }; + } + + @Override + public FederationQueryPermit acquire(QueryAdmissionRequest request) { + return () -> { }; + } + }; + } +} diff --git a/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/execute/FederationQueryMetricsSnapshot.java b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/execute/FederationQueryMetricsSnapshot.java new file mode 100644 index 0000000..6a19ecd --- /dev/null +++ b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/execute/FederationQueryMetricsSnapshot.java @@ -0,0 +1,152 @@ +package com.easyagents.federation.sql.execute; + +import com.easyagents.federation.sql.federation.FederationQueryMode; +import java.io.Serializable; +import java.util.List; + +/** + * 查询游标当前或关闭后的不可变消耗指标快照。 + * + * @param queryId 查询标识;不可用快照可为空 + * @param queryMode 查询模式 + * @param planCacheHit 是否命中计划缓存 + * @param planningNanos 编译阶段耗时 + * @param admissionWaitNanos 准入等待耗时 + * @param connectionAcquireNanos 全部分片获取连接累计耗时 + * @param databaseExecutionNanos 全部分片执行 Statement 累计耗时 + * @param localExecutionNanos 本地算子累计耗时 + * @param executionNanos 从执行开始到当前或结束的耗时 + * @param firstRowNanos 从执行开始到首行的耗时;尚未返回首行时为 -1 + * @param returnedRows 调用方已消费的最终结果行数 + * @param returnedBytes 最终结果估算字节数;Adapter 未安全提供时为 -1 + * @param intermediateRows 全部分片读取的中间结果行数 + * @param intermediateBytes 全部分片读取的中间结果估算字节数;Adapter 未安全提供时为 -1 + * @param complete 查询是否正常消费完成 + * @param cancelled 查询是否因取消结束 + * @param timedOut 查询是否因统一执行时限结束 + * @param truncated 是否因最终行数上限停止继续消费 + * @param terminalErrorCode 失败终态错误码;成功或尚未失败时为空 + * @param fragments 分片指标 + * @param localOperators 本地算子指标 + */ +public record FederationQueryMetricsSnapshot( + QueryId queryId, + FederationQueryMode queryMode, + boolean planCacheHit, + long planningNanos, + long admissionWaitNanos, + long connectionAcquireNanos, + long databaseExecutionNanos, + long localExecutionNanos, + long executionNanos, + long firstRowNanos, + long returnedRows, + long returnedBytes, + long intermediateRows, + long intermediateBytes, + boolean complete, + boolean cancelled, + boolean timedOut, + boolean truncated, + String terminalErrorCode, + List fragments, + List localOperators +) implements Serializable { + + /** + * 防御性复制分片指标。 + */ + public FederationQueryMetricsSnapshot { + fragments = List.copyOf(fragments == null ? List.of() : fragments); + localOperators = List.copyOf(localOperators == null ? List.of() : localOperators); + terminalErrorCode = terminalErrorCode == null ? "" : terminalErrorCode; + } + + /** + * 创建旧基础字段视图的兼容指标快照。 + * + * @param queryId 查询标识 + * @param queryMode 查询模式 + * @param planCacheHit 是否命中计划缓存 + * @param planningNanos 编译耗时 + * @param executionNanos 执行耗时 + * @param firstRowNanos 首行耗时 + * @param returnedRows 返回行数 + * @param returnedBytes 返回字节数 + * @param intermediateRows 中间行数 + * @param intermediateBytes 中间字节数 + * @param complete 是否完成 + * @param cancelled 是否取消 + * @param fragments 分片指标 + */ + public FederationQueryMetricsSnapshot( + QueryId queryId, + FederationQueryMode queryMode, + boolean planCacheHit, + long planningNanos, + long executionNanos, + long firstRowNanos, + long returnedRows, + long returnedBytes, + long intermediateRows, + long intermediateBytes, + boolean complete, + boolean cancelled, + List fragments + ) { + this( + queryId, + queryMode, + planCacheHit, + planningNanos, + 0, + 0, + 0, + 0, + executionNanos, + firstRowNanos, + returnedRows, + returnedBytes, + intermediateRows, + intermediateBytes, + complete, + cancelled, + false, + false, + "", + fragments, + List.of() + ); + } + + /** + * 返回第三方 Adapter 尚未接入指标时的空快照。 + * + * @return 空快照 + */ + public static FederationQueryMetricsSnapshot unavailable() { + return new FederationQueryMetricsSnapshot( + null, + FederationQueryMode.SINGLE_SOURCE, + false, + 0, + 0, + 0, + 0, + 0, + 0, + -1, + 0, + -1, + 0, + -1, + false, + false, + false, + false, + "", + List.of(), + List.of() + ); + } +} diff --git a/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/execute/FederationQueryPermit.java b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/execute/FederationQueryPermit.java new file mode 100644 index 0000000..9010a62 --- /dev/null +++ b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/execute/FederationQueryPermit.java @@ -0,0 +1,14 @@ +package com.easyagents.federation.sql.execute; + +/** + * 查询准入许可,关闭时释放并发配额。 + */ +@FunctionalInterface +public interface FederationQueryPermit extends AutoCloseable { + + /** + * 释放准入许可。 + */ + @Override + void close(); +} diff --git a/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/execute/FederationResultCursor.java b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/execute/FederationResultCursor.java new file mode 100644 index 0000000..bffa59e --- /dev/null +++ b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/execute/FederationResultCursor.java @@ -0,0 +1,84 @@ +package com.easyagents.federation.sql.execute; + +import java.io.InputStream; +import java.io.Reader; +import java.util.List; + +/** + * 按行消费且必须关闭的流式结果游标。 + */ +public interface FederationResultCursor extends AutoCloseable { + + /** + * 返回查询标识。 + * + * @return 查询标识 + */ + QueryId queryId(); + + /** + * 返回结果列元数据。 + * + * @return 不可变列列表 + */ + List columns(); + + /** + * 返回查询当前或关闭后的消耗指标快照。 + * + * @return 不可变指标快照 + */ + default FederationQueryMetricsSnapshot metrics() { + return FederationQueryMetricsSnapshot.unavailable(); + } + + /** + * 移动至下一行。 + * + * @return 是否存在下一行 + */ + boolean next(); + + /** + * 按 JDBC 列序号读取当前行。 + * + * @param columnIndex 从 1 开始的列序号 + * @return 列值 + */ + Object getObject(int columnIndex); + + /** + * 以流方式读取二进制列,避免调用方为大字段一次性分配完整字节数组。 + * + * @param columnIndex 从 1 开始的列序号 + * @return 二进制流;SQL NULL 返回 null + * @throws UnsupportedOperationException Adapter 不支持流式列读取 + */ + default InputStream getBinaryStream(int columnIndex) { + throw new UnsupportedOperationException("binary stream access is not supported by this adapter"); + } + + /** + * 以流方式读取字符列,避免调用方为大字段一次性分配完整字符串。 + * + * @param columnIndex 从 1 开始的列序号 + * @return 字符流;SQL NULL 返回 null + * @throws UnsupportedOperationException Adapter 不支持流式列读取 + */ + default Reader getCharacterStream(int columnIndex) { + throw new UnsupportedOperationException("character stream access is not supported by this adapter"); + } + + /** + * 将当前行复制为不可变列表。 + * + * @return 当前行列值 + */ + List row(); + + /** + * 关闭结果集、Statement、Connection、准入许可和 Runtime lease。 + */ + @Override + void close(); +} diff --git a/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/execute/LocalFederationQueryAdmissionController.java b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/execute/LocalFederationQueryAdmissionController.java new file mode 100644 index 0000000..0076b82 --- /dev/null +++ b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/execute/LocalFederationQueryAdmissionController.java @@ -0,0 +1,160 @@ +package com.easyagents.federation.sql.execute; + +import com.easyagents.federation.sql.api.FederationSqlErrorCode; +import com.easyagents.federation.sql.api.FederationSqlException; +import com.easyagents.federation.sql.source.SourceId; +import java.time.Duration; +import java.util.concurrent.Semaphore; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.function.BooleanSupplier; + +/** + * 使用公平信号量限制单节点查询并发的默认控制器。 + */ +public final class LocalFederationQueryAdmissionController implements FederationQueryAdmissionController { + + private final int maxConcurrentQueries; + private final Semaphore permits; + private final AtomicBoolean closed = new AtomicBoolean(); + + /** + * 创建本地准入控制器。 + * + * @param maxConcurrentQueries 最大并发查询数 + */ + public LocalFederationQueryAdmissionController(int maxConcurrentQueries) { + if (maxConcurrentQueries <= 0) { + throw new IllegalArgumentException("maxConcurrentQueries must be positive"); + } + this.maxConcurrentQueries = maxConcurrentQueries; + this.permits = new Semaphore(maxConcurrentQueries, true); + } + + /** + * 在指定时间内获取本地许可。 + * + * @param sourceId 数据源标识 + * @param queryId 查询标识 + * @param timeout 最大等待时间 + * @return 可幂等关闭的许可 + */ + @Override + public FederationQueryPermit acquire(SourceId sourceId, QueryId queryId, Duration timeout) { + return acquire(sourceId, queryId, timeout, () -> false); + } + + /** + * 在等待本地许可期间轮询查询取消状态。 + * + * @param sourceId 数据源标识 + * @param queryId 查询标识 + * @param timeout 最大等待时间 + * @param cancellationRequested 取消状态 + * @return 可幂等关闭的许可 + */ + @Override + public FederationQueryPermit acquire( + SourceId sourceId, + QueryId queryId, + Duration timeout, + BooleanSupplier cancellationRequested + ) { + ensureOpen(); + if (cancellationRequested.getAsBoolean()) { + throw cancelled(queryId); + } + boolean acquired = false; + try { + long timeoutNanos = timeout.toNanos(); + if (timeoutNanos == 0) { + acquired = permits.tryAcquire(); + } else { + long started = System.nanoTime(); + long remaining = timeoutNanos; + long pollNanos = TimeUnit.MILLISECONDS.toNanos(50); + while (!acquired && remaining > 0) { + acquired = permits.tryAcquire(Math.min(remaining, pollNanos), TimeUnit.NANOSECONDS); + if (!acquired && cancellationRequested.getAsBoolean()) { + throw cancelled(queryId); + } + remaining = timeoutNanos - (System.nanoTime() - started); + } + } + } catch (InterruptedException exception) { + Thread.currentThread().interrupt(); + throw new FederationSqlException( + FederationSqlErrorCode.QUERY_ADMISSION_TIMEOUT, + "query admission was interrupted for source " + sourceId, + exception + ); + } + if (cancellationRequested.getAsBoolean()) { + if (acquired) { + permits.release(); + } + throw cancelled(queryId); + } + if (!acquired) { + throw new FederationSqlException( + FederationSqlErrorCode.QUERY_ADMISSION_TIMEOUT, + "query admission timed out for source " + sourceId + ); + } + if (closed.get()) { + permits.release(); + throw new FederationSqlException( + FederationSqlErrorCode.ENGINE_CLOSED, + "query admission controller is closed" + ); + } + AtomicBoolean released = new AtomicBoolean(); + return () -> { + if (released.compareAndSet(false, true)) { + permits.release(); + } + }; + } + + /** + * 为单源或联邦查询获取一份节点级本地许可。 + * + * @param request 查询级准入请求 + * @return 可幂等关闭的节点许可 + */ + @Override + public FederationQueryPermit acquire(QueryAdmissionRequest request) { + return acquire( + request.primarySourceId(), + request.queryId(), + request.timeout(), + request.cancellationRequested() + ); + } + + /** + * 关闭控制器并唤醒等待准入的线程。 + */ + @Override + public void close() { + if (closed.compareAndSet(false, true)) { + permits.release(maxConcurrentQueries); + } + } + + private void ensureOpen() { + if (closed.get()) { + throw new FederationSqlException( + FederationSqlErrorCode.ENGINE_CLOSED, + "query admission controller is closed" + ); + } + } + + private static FederationSqlException cancelled(QueryId queryId) { + return new FederationSqlException( + FederationSqlErrorCode.QUERY_CANCELLED, + "query admission was cancelled: " + queryId.value() + ); + } +} diff --git a/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/execute/QueryAdmissionRequest.java b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/execute/QueryAdmissionRequest.java new file mode 100644 index 0000000..a498003 --- /dev/null +++ b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/execute/QueryAdmissionRequest.java @@ -0,0 +1,49 @@ +package com.easyagents.federation.sql.execute; + +import com.easyagents.federation.sql.source.SourceId; +import java.time.Duration; +import java.util.Comparator; +import java.util.List; +import java.util.function.BooleanSupplier; + +/** + * 单次单源或联邦查询的准入请求。 + * + * @param sourceIds 查询实际引用的去重物理数据源,按稳定顺序排列 + * @param queryId 查询标识 + * @param timeout 最大等待时间 + * @param cancellationRequested 查询取消状态 + */ +public record QueryAdmissionRequest( + List sourceIds, + QueryId queryId, + Duration timeout, + BooleanSupplier cancellationRequested +) { + + /** + * 校验并创建不可变准入请求。 + */ + public QueryAdmissionRequest { + if (sourceIds == null || sourceIds.isEmpty() || queryId == null) { + throw new IllegalArgumentException("sourceIds and queryId must be provided"); + } + sourceIds = sourceIds.stream() + .distinct() + .sorted(Comparator.comparing(SourceId::value)) + .toList(); + timeout = timeout == null ? Duration.ZERO : timeout; + cancellationRequested = cancellationRequested == null + ? () -> false + : cancellationRequested; + } + + /** + * 返回兼容单源准入实现使用的首个数据源。 + * + * @return 稳定排序后的首个数据源 + */ + public SourceId primarySourceId() { + return sourceIds.get(0); + } +} diff --git a/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/execute/QueryId.java b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/execute/QueryId.java new file mode 100644 index 0000000..259512a --- /dev/null +++ b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/execute/QueryId.java @@ -0,0 +1,30 @@ +package com.easyagents.federation.sql.execute; + +import java.io.Serializable; +import java.util.UUID; + +/** + * 节点本地查询标识。 + * + * @param value 查询标识文本 + */ +public record QueryId(String value) implements Serializable { + + /** + * 校验查询标识。 + */ + public QueryId { + if (value == null || value.isBlank()) { + throw new IllegalArgumentException("query id must not be blank"); + } + } + + /** + * 创建随机查询标识。 + * + * @return 查询标识 + */ + public static QueryId create() { + return new QueryId(UUID.randomUUID().toString()); + } +} diff --git a/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/execute/SqlExecutionOptions.java b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/execute/SqlExecutionOptions.java new file mode 100644 index 0000000..379016b --- /dev/null +++ b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/execute/SqlExecutionOptions.java @@ -0,0 +1,40 @@ +package com.easyagents.federation.sql.execute; + +import java.io.Serializable; + +/** + * 不可由 SQL 文本覆盖的 JDBC 执行限制。 + * + * @param fetchSize 驱动抓取批次,0 表示驱动默认 + * @param maxRows 最大返回行数,0 表示不额外限制 + * @param queryTimeoutSeconds 查询超时秒数,0 表示驱动默认 + * @param readOnly 是否强制只读连接;正式 JDBC 路径必须为 true + */ +public record SqlExecutionOptions( + int fetchSize, + int maxRows, + int queryTimeoutSeconds, + boolean readOnly +) implements Serializable { + + /** + * 校验执行限制。 + */ + public SqlExecutionOptions { + if (fetchSize < 0 || maxRows < 0 || queryTimeoutSeconds < 0) { + throw new IllegalArgumentException("execution limits must not be negative"); + } + if (!readOnly) { + throw new IllegalArgumentException("federation SQL execution must remain read-only"); + } + } + + /** + * 返回适合普通只读流式查询的默认配置。 + * + * @return 默认配置 + */ + public static SqlExecutionOptions defaults() { + return new SqlExecutionOptions(500, 0, 30, true); + } +} diff --git a/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/execute/SqlParameter.java b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/execute/SqlParameter.java new file mode 100644 index 0000000..2c4c21e --- /dev/null +++ b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/execute/SqlParameter.java @@ -0,0 +1,112 @@ +package com.easyagents.federation.sql.execute; + +import java.io.Serializable; +import java.sql.Types; + +/** + * 可序列化边界内的 JDBC 标量参数值与显式类型。 + * + *

默认 Adapter 将 {@link Types#OTHER} 解释为 UUID;厂商专有 OTHER 类型应由 + * 对应 Adapter 覆盖参数类型映射。

+ * + * @param jdbcType {@link Types} 类型值 + * @param value 参数值 + */ +public record SqlParameter(int jdbcType, Object value) implements Serializable { + + /** + * 校验跨节点参数值属于稳定、无嵌套对象图的 JDBC 标量类型。 + */ + public SqlParameter { + if (jdbcType == Types.NULL) { + throw new IllegalArgumentException("an explicit JDBC type is required for NULL parameters"); + } + if (value != null && !isSupportedScalar(value)) { + throw new IllegalArgumentException( + "SQL parameter value must be a supported serializable JDBC scalar" + ); + } + } + + /** + * 根据常用 Java 值推断 JDBC 类型。 + * + * @param value 参数值 + * @return 参数 + */ + public static SqlParameter of(Object value) { + return new SqlParameter(inferType(value), value); + } + + private static int inferType(Object value) { + if (value == null) { + throw new IllegalArgumentException("use new SqlParameter(jdbcType, null) for NULL values"); + } + if (value instanceof Integer || value instanceof Short || value instanceof Byte) { + return Types.INTEGER; + } + if (value instanceof Long) { + return Types.BIGINT; + } + if (value instanceof Float) { + return Types.REAL; + } + if (value instanceof Double) { + return Types.DOUBLE; + } + if (value instanceof java.math.BigDecimal || value instanceof java.math.BigInteger) { + return Types.DECIMAL; + } + if (value instanceof Boolean) { + return Types.BOOLEAN; + } + if (value instanceof java.sql.Date || value instanceof java.time.LocalDate) { + return Types.DATE; + } + if (value instanceof java.sql.Time || value instanceof java.time.LocalTime) { + return Types.TIME; + } + if (value instanceof java.time.OffsetTime) { + return Types.TIME_WITH_TIMEZONE; + } + if (value instanceof java.time.OffsetDateTime) { + return Types.TIMESTAMP_WITH_TIMEZONE; + } + if (value instanceof java.sql.Timestamp || value instanceof java.time.LocalDateTime + || value instanceof java.time.Instant) { + return Types.TIMESTAMP; + } + if (value instanceof byte[]) { + return Types.VARBINARY; + } + if (value instanceof java.util.UUID) { + return Types.OTHER; + } + return Types.VARCHAR; + } + + private static boolean isSupportedScalar(Object value) { + return value instanceof String + || value instanceof Character + || value instanceof Boolean + || value instanceof Byte + || value instanceof Short + || value instanceof Integer + || value instanceof Long + || value instanceof Float + || value instanceof Double + || value instanceof java.math.BigDecimal + || value instanceof java.math.BigInteger + || value instanceof byte[] + || value instanceof java.sql.Date + || value instanceof java.sql.Time + || value instanceof java.sql.Timestamp + || value instanceof java.time.LocalDate + || value instanceof java.time.LocalTime + || value instanceof java.time.LocalDateTime + || value instanceof java.time.OffsetTime + || value instanceof java.time.OffsetDateTime + || value instanceof java.time.Instant + || value instanceof java.util.UUID; + } +} diff --git a/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/execute/StatementLifecycle.java b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/execute/StatementLifecycle.java new file mode 100644 index 0000000..38b3e37 --- /dev/null +++ b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/execute/StatementLifecycle.java @@ -0,0 +1,41 @@ +package com.easyagents.federation.sql.execute; + +import java.sql.Statement; + +/** + * Adapter 用于登记和清理可取消 Statement 的回调。 + */ +public interface StatementLifecycle { + + /** + * 登记正在执行的 Statement。 + * + * @param statement JDBC Statement + */ + void register(Statement statement); + + /** + * 清除已结束的 Statement。 + * + * @param statement JDBC Statement + */ + void unregister(Statement statement); + + /** + * 返回当前查询是否已收到主动取消请求。 + * + * @return 是否已请求取消 + */ + default boolean cancellationRequested() { + return false; + } + + /** + * 返回当前查询是否已经超过统一执行时限。 + * + * @return 是否已超时 + */ + default boolean timeoutRequested() { + return false; + } +} diff --git a/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/federation/FederationColumnStatistics.java b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/federation/FederationColumnStatistics.java new file mode 100644 index 0000000..824e62a --- /dev/null +++ b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/federation/FederationColumnStatistics.java @@ -0,0 +1,32 @@ +package com.easyagents.federation.sql.federation; + +import java.io.Serializable; + +/** + * 一列用于成本估算的轻量统计。 + * + * @param distinctCount 估算不同值数量;未知时为 0 + * @param nullFraction 空值比例,范围为 0 到 1 + * @param averageWidthBytes 平均列宽字节数;未知时为 0 + */ +public record FederationColumnStatistics( + double distinctCount, + double nullFraction, + long averageWidthBytes +) implements Serializable { + + /** + * 校验列统计。 + */ + public FederationColumnStatistics { + if (!Double.isFinite(distinctCount) || distinctCount < 0) { + throw new IllegalArgumentException("distinctCount must be finite and non-negative"); + } + if (!Double.isFinite(nullFraction) || nullFraction < 0 || nullFraction > 1) { + throw new IllegalArgumentException("nullFraction must be between 0 and 1"); + } + if (averageWidthBytes < 0) { + throw new IllegalArgumentException("averageWidthBytes must be non-negative"); + } + } +} diff --git a/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/federation/FederationCostEstimate.java b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/federation/FederationCostEstimate.java new file mode 100644 index 0000000..2f6022f --- /dev/null +++ b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/federation/FederationCostEstimate.java @@ -0,0 +1,126 @@ +package com.easyagents.federation.sql.federation; + +import java.io.Serializable; +import java.time.Instant; + +/** + * 一个物理分片的轻量搬运成本估算。 + * + * @param estimatedRows 分片输出估算行数 + * @param estimatedRowWidthBytes 分片输出估算行宽 + * @param estimatedTransferBytes 分片到本地执行器的估算搬运字节数 + * @param statisticsSource 统计来源 + * @param statisticsSnapshotVersion 统计快照版本 + * @param statisticsCollectedAt 外部统计采集时间;缺失时为 epoch + * @param statisticsMissing 是否完全使用 Calcite 默认估算 + * @param statisticsStatus 统计完整性与时效状态 + * @param estimateAvailable 当前数值是否为有效估算;兼容旧计划缺少估算时为 false + */ +public record FederationCostEstimate( + double estimatedRows, + long estimatedRowWidthBytes, + double estimatedTransferBytes, + String statisticsSource, + String statisticsSnapshotVersion, + Instant statisticsCollectedAt, + boolean statisticsMissing, + FederationStatisticsStatus statisticsStatus, + boolean estimateAvailable +) implements Serializable { + + /** + * 校验并规范化成本估算。 + */ + public FederationCostEstimate { + if (!Double.isFinite(estimatedRows) || estimatedRows < 0 + || estimatedRowWidthBytes < 0 + || !Double.isFinite(estimatedTransferBytes) + || estimatedTransferBytes < 0) { + throw new IllegalArgumentException("cost estimate values must be finite and non-negative"); + } + statisticsSource = statisticsSource == null || statisticsSource.isBlank() + ? "calcite-default" + : statisticsSource; + statisticsSnapshotVersion = statisticsSnapshotVersion == null + ? "none" + : statisticsSnapshotVersion; + statisticsCollectedAt = statisticsCollectedAt == null + ? Instant.EPOCH + : statisticsCollectedAt; + statisticsStatus = statisticsStatus == null + ? statisticsMissing + ? FederationStatisticsStatus.MISSING + : FederationStatisticsStatus.COMPLETE + : statisticsStatus; + } + + /** + * 创建带显式统计状态的有效成本估算。 + * + * @param estimatedRows 估算行数 + * @param estimatedRowWidthBytes 估算行宽 + * @param estimatedTransferBytes 估算搬运字节 + * @param statisticsSource 统计来源 + * @param statisticsSnapshotVersion 统计版本 + * @param statisticsCollectedAt 采集时间 + * @param statisticsMissing 是否缺失统计 + * @param statisticsStatus 统计状态 + */ + public FederationCostEstimate( + double estimatedRows, + long estimatedRowWidthBytes, + double estimatedTransferBytes, + String statisticsSource, + String statisticsSnapshotVersion, + Instant statisticsCollectedAt, + boolean statisticsMissing, + FederationStatisticsStatus statisticsStatus + ) { + this( + estimatedRows, + estimatedRowWidthBytes, + estimatedTransferBytes, + statisticsSource, + statisticsSnapshotVersion, + statisticsCollectedAt, + statisticsMissing, + statisticsStatus, + true + ); + } + + /** + * 创建旧字段集合的兼容成本估算。 + * + * @param estimatedRows 估算行数 + * @param estimatedRowWidthBytes 估算行宽 + * @param estimatedTransferBytes 估算搬运字节 + * @param statisticsSource 统计来源 + * @param statisticsSnapshotVersion 统计版本 + * @param statisticsCollectedAt 采集时间 + * @param statisticsMissing 是否缺失统计 + */ + public FederationCostEstimate( + double estimatedRows, + long estimatedRowWidthBytes, + double estimatedTransferBytes, + String statisticsSource, + String statisticsSnapshotVersion, + Instant statisticsCollectedAt, + boolean statisticsMissing + ) { + this( + estimatedRows, + estimatedRowWidthBytes, + estimatedTransferBytes, + statisticsSource, + statisticsSnapshotVersion, + statisticsCollectedAt, + statisticsMissing, + statisticsMissing + ? FederationStatisticsStatus.MISSING + : FederationStatisticsStatus.COMPLETE, + true + ); + } +} diff --git a/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/federation/FederationExecutionPolicy.java b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/federation/FederationExecutionPolicy.java new file mode 100644 index 0000000..cf81e36 --- /dev/null +++ b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/federation/FederationExecutionPolicy.java @@ -0,0 +1,80 @@ +package com.easyagents.federation.sql.federation; + +import java.io.Serializable; + +/** + * 联邦查询的调用方资源上限;Engine 会与自己的硬上限取更严格值。 + * + * @param maximumReferencedSources 单条 SQL 最多实际引用的数据源数 + * @param maximumFragments 最多物理查询分片数 + * @param maximumConcurrentFragments 最大并发分片数 + * @param maximumIntermediateRows 最多读取的中间结果行数 + * @param maximumIntermediateBytes 最多读取的中间结果估算字节数 + * @param maximumExecutionTimeMillis 联邦执行总时限 + */ +public record FederationExecutionPolicy( + int maximumReferencedSources, + int maximumFragments, + int maximumConcurrentFragments, + long maximumIntermediateRows, + long maximumIntermediateBytes, + long maximumExecutionTimeMillis +) implements Serializable { + + private static final FederationExecutionPolicy BASIC = new FederationExecutionPolicy( + 2, + 8, + 2, + 100_000, + 64L * 1024L * 1024L, + 60_000 + ); + + /** + * 校验资源上限。 + */ + public FederationExecutionPolicy { + if (maximumReferencedSources <= 0 + || maximumFragments <= 0 + || maximumConcurrentFragments <= 0 + || maximumIntermediateRows <= 0 + || maximumIntermediateBytes <= 0 + || maximumExecutionTimeMillis <= 0) { + throw new IllegalArgumentException("federation execution limits must be positive"); + } + if (maximumConcurrentFragments > maximumFragments) { + throw new IllegalArgumentException( + "maximumConcurrentFragments must not exceed maximumFragments" + ); + } + } + + /** + * 返回适合首批联邦查询的保守默认策略。 + * + * @return 默认策略 + */ + public static FederationExecutionPolicy basic() { + return BASIC; + } + + /** + * 将两个策略收敛为逐项更严格的有效策略。 + * + * @param other 另一个策略 + * @return 有效策略 + */ + public FederationExecutionPolicy intersect(FederationExecutionPolicy other) { + if (other == null) { + return this; + } + return new FederationExecutionPolicy( + Math.min(maximumReferencedSources, other.maximumReferencedSources), + Math.min(maximumFragments, other.maximumFragments), + Math.min(maximumConcurrentFragments, other.maximumConcurrentFragments), + Math.min(maximumIntermediateRows, other.maximumIntermediateRows), + Math.min(maximumIntermediateBytes, other.maximumIntermediateBytes), + Math.min(maximumExecutionTimeMillis, other.maximumExecutionTimeMillis) + ); + } +} diff --git a/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/federation/FederationFragmentPlan.java b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/federation/FederationFragmentPlan.java new file mode 100644 index 0000000..0622480 --- /dev/null +++ b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/federation/FederationFragmentPlan.java @@ -0,0 +1,88 @@ +package com.easyagents.federation.sql.federation; + +import com.easyagents.federation.sql.execute.FederationColumn; +import com.easyagents.federation.sql.source.SourceId; +import java.util.List; + +/** + * 一个可交给物理数据源 Adapter 执行的查询分片。 + * + * @param fragmentId 计划内唯一分片标识 + * @param bindingName 查询范围 Binding 名称 + * @param sourceId 物理数据源标识 + * @param executableSql 目标数据库方言 SQL + * @param parameterMapping 分片占位符到原始查询参数的零基索引映射 + * @param columns 分片输出列 + * @param costEstimate 分片搬运成本估算 + * @param pushedDownOperators 已下推至物理源的关系算子 + */ +public record FederationFragmentPlan( + String fragmentId, + String bindingName, + SourceId sourceId, + String executableSql, + List parameterMapping, + List columns, + FederationCostEstimate costEstimate, + List pushedDownOperators +) { + + /** + * 校验并创建不可变分片计划。 + */ + public FederationFragmentPlan { + if (fragmentId == null || fragmentId.isBlank() + || bindingName == null || bindingName.isBlank() + || sourceId == null || executableSql == null || executableSql.isBlank()) { + throw new IllegalArgumentException("fragment identity and SQL must be provided"); + } + parameterMapping = List.copyOf(parameterMapping == null ? List.of() : parameterMapping); + columns = List.copyOf(columns == null ? List.of() : columns); + if (costEstimate == null) { + throw new IllegalArgumentException("costEstimate must be provided"); + } + pushedDownOperators = List.copyOf( + pushedDownOperators == null ? List.of() : pushedDownOperators + ); + } + + /** + * 创建不携带显式成本输入的兼容分片计划。 + * + * @param fragmentId 计划内唯一分片标识 + * @param bindingName 查询范围 Binding 名称 + * @param sourceId 物理数据源标识 + * @param executableSql 目标数据库方言 SQL + * @param parameterMapping 参数映射 + * @param columns 输出列 + */ + public FederationFragmentPlan( + String fragmentId, + String bindingName, + SourceId sourceId, + String executableSql, + List parameterMapping, + List columns + ) { + this( + fragmentId, + bindingName, + sourceId, + executableSql, + parameterMapping, + columns, + new FederationCostEstimate( + 0, + 0, + 0, + "calcite-default", + "none", + java.time.Instant.EPOCH, + true, + FederationStatisticsStatus.MISSING, + false + ), + List.of() + ); + } +} diff --git a/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/federation/FederationJoinAlgorithm.java b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/federation/FederationJoinAlgorithm.java new file mode 100644 index 0000000..bceeda9 --- /dev/null +++ b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/federation/FederationJoinAlgorithm.java @@ -0,0 +1,10 @@ +package com.easyagents.federation.sql.federation; + +/** + * 联邦本地 Join 的执行算法。 + */ +public enum FederationJoinAlgorithm { + + /** 在构建侧建立哈希表后探测。 */ + HASH_JOIN +} diff --git a/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/federation/FederationJoinOptimization.java b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/federation/FederationJoinOptimization.java new file mode 100644 index 0000000..2d044fb --- /dev/null +++ b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/federation/FederationJoinOptimization.java @@ -0,0 +1,99 @@ +package com.easyagents.federation.sql.federation; + +import java.io.Serializable; +import java.util.List; + +/** + * 一次跨源 Join 的优化结果。 + * + * @param stageIndex 执行阶段序号,从 1 开始 + * @param leftBindings 左输入包含的 Binding 集合 + * @param rightBindings 右输入包含的 Binding 集合 + * @param buildBinding 哈希表构建侧 Binding + * @param algorithm Join 算法 + * @param reason 选择原因 + * @param estimatedBuildBytes 构建侧估算字节数 + */ +public record FederationJoinOptimization( + int stageIndex, + List leftBindings, + List rightBindings, + String buildBinding, + FederationJoinAlgorithm algorithm, + FederationJoinSelectionReason reason, + double estimatedBuildBytes +) implements Serializable { + + /** + * 校验并规范化优化结果。 + */ + public FederationJoinOptimization { + if (stageIndex <= 0) { + throw new IllegalArgumentException("stageIndex must be positive"); + } + leftBindings = List.copyOf(leftBindings == null ? List.of() : leftBindings); + rightBindings = List.copyOf(rightBindings == null ? List.of() : rightBindings); + if (leftBindings.isEmpty() || rightBindings.isEmpty() + || leftBindings.stream().anyMatch(value -> value == null || value.isBlank()) + || rightBindings.stream().anyMatch(value -> value == null || value.isBlank()) + || buildBinding == null || buildBinding.isBlank()) { + throw new IllegalArgumentException("join binding names must not be blank"); + } + if (!Double.isFinite(estimatedBuildBytes) || estimatedBuildBytes < 0) { + throw new IllegalArgumentException( + "estimatedBuildBytes must be finite and non-negative" + ); + } + algorithm = algorithm == null ? FederationJoinAlgorithm.HASH_JOIN : algorithm; + reason = reason == null + ? FederationJoinSelectionReason.INCOMPLETE_STATISTICS + : reason; + } + + /** + * 创建兼容的两输入单阶段优化结果。 + * + * @param leftBinding 左输入 Binding + * @param rightBinding 右输入 Binding + * @param buildBinding 哈希构建侧 Binding + * @param algorithm Join 算法 + * @param reason 选择原因 + * @param estimatedBuildBytes 预计构建字节数 + */ + public FederationJoinOptimization( + String leftBinding, + String rightBinding, + String buildBinding, + FederationJoinAlgorithm algorithm, + FederationJoinSelectionReason reason, + double estimatedBuildBytes + ) { + this( + 1, + List.of(leftBinding), + List.of(rightBinding), + buildBinding, + algorithm, + reason, + estimatedBuildBytes + ); + } + + /** + * 返回左输入的紧凑展示名称。 + * + * @return 单个 Binding 或多个 Binding 的组合名称 + */ + public String leftBinding() { + return String.join(" + ", leftBindings); + } + + /** + * 返回右输入的紧凑展示名称。 + * + * @return 单个 Binding 或多个 Binding 的组合名称 + */ + public String rightBinding() { + return String.join(" + ", rightBindings); + } +} diff --git a/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/federation/FederationJoinSelectionReason.java b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/federation/FederationJoinSelectionReason.java new file mode 100644 index 0000000..0e5c8c0 --- /dev/null +++ b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/federation/FederationJoinSelectionReason.java @@ -0,0 +1,16 @@ +package com.easyagents.federation.sql.federation; + +/** + * Join 构建侧的选择原因。 + */ +public enum FederationJoinSelectionReason { + + /** 可信表级统计表明当前构建侧搬运量更小。 */ + SMALLER_BUILD_SIDE, + + /** 外连接语义要求保留输入顺序。 */ + JOIN_SEMANTICS, + + /** 统计不完整,保留稳定默认顺序。 */ + INCOMPLETE_STATISTICS +} diff --git a/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/federation/FederationLogicalTableDefinition.java b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/federation/FederationLogicalTableDefinition.java new file mode 100644 index 0000000..92ae49e --- /dev/null +++ b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/federation/FederationLogicalTableDefinition.java @@ -0,0 +1,65 @@ +package com.easyagents.federation.sql.federation; + +import java.io.Serializable; + +/** + * 查询范围内一张逻辑表到物理 Binding 表的不可变映射。 + * + * @param logicalName 查询方可见的全局唯一逻辑表名 + * @param bindingName 物理数据源 Binding 名称 + * @param schemaName Binding 内的查询逻辑 Schema 名称 + * @param sourceTableName 数据源 Definition 中的实际表名 + */ +public record FederationLogicalTableDefinition( + String logicalName, + String bindingName, + String schemaName, + String sourceTableName +) implements Serializable { + + /** + * 校验逻辑表映射的必填字段。 + */ + public FederationLogicalTableDefinition { + requireText(logicalName, "logicalName"); + requireText(bindingName, "bindingName"); + requireText(schemaName, "schemaName"); + requireText(sourceTableName, "sourceTableName"); + } + + /** + * 创建逻辑表映射。 + * + * @param logicalName 查询方可见逻辑表名 + * @param bindingName 物理数据源 Binding 名称 + * @param schemaName 查询逻辑 Schema 名称 + * @param sourceTableName 数据源中的实际表名 + * @return 逻辑表映射 + */ + public static FederationLogicalTableDefinition of( + String logicalName, + String bindingName, + String schemaName, + String sourceTableName + ) { + return new FederationLogicalTableDefinition( + logicalName, + bindingName, + schemaName, + sourceTableName + ); + } + + /** + * 校验映射字段包含有效文本。 + * + * @param value 字段值 + * @param field 字段名称 + * @throws IllegalArgumentException 字段为空时抛出 + */ + private static void requireText(String value, String field) { + if (value == null || value.isBlank()) { + throw new IllegalArgumentException(field + " must not be blank"); + } + } +} diff --git a/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/federation/FederationQueryMode.java b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/federation/FederationQueryMode.java new file mode 100644 index 0000000..d3cfb83 --- /dev/null +++ b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/federation/FederationQueryMode.java @@ -0,0 +1,11 @@ +package com.easyagents.federation.sql.federation; + +/** + * 根据 SQL 实际引用物理数据源数量确定的查询模式。 + */ +public enum FederationQueryMode { + /** 单一物理数据源完整下推。 */ + SINGLE_SOURCE, + /** 多物理数据源分片下推并由 Core 合并。 */ + FEDERATED +} diff --git a/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/federation/FederationQueryScopeDefinition.java b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/federation/FederationQueryScopeDefinition.java new file mode 100644 index 0000000..0c90553 --- /dev/null +++ b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/federation/FederationQueryScopeDefinition.java @@ -0,0 +1,306 @@ +package com.easyagents.federation.sql.federation; + +import com.easyagents.federation.sql.source.SourceId; +import java.io.Serializable; +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.HexFormat; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.HashSet; +import java.util.Set; + +/** + * 调用方传入的不可变查询范围,描述 SQL 可见的物理数据源 Binding。 + * + *

该定义不保存 DataSource、连接池或凭据。Core 只在编译和执行期间解析它, + * 虚拟数据源的持久化、发布和分布式一致性由调用方负责。

+ * + * @param definitionId 调用方定义标识 + * @param revision 查询范围版本 + * @param bindings Binding 名称到物理数据源定义的映射 + * @param defaultBinding 默认 Binding 名称 + * @param logicalTables 查询方可见的逻辑表映射;空列表保留原始物理表解析语义 + * @param executionPolicy 调用方联邦资源上限 + */ +public record FederationQueryScopeDefinition( + String definitionId, + long revision, + Map bindings, + String defaultBinding, + List logicalTables, + FederationExecutionPolicy executionPolicy +) implements Serializable { + + /** + * 校验并创建不可变查询范围。 + */ + public FederationQueryScopeDefinition { + if (definitionId == null || definitionId.isBlank()) { + throw new IllegalArgumentException("definitionId must not be blank"); + } + if (revision < 0) { + throw new IllegalArgumentException("scope revision must not be negative"); + } + if (bindings == null || bindings.isEmpty()) { + throw new IllegalArgumentException("scope bindings must not be empty"); + } + LinkedHashMap copied = new LinkedHashMap<>(); + Set normalizedBindingNames = new HashSet<>(); + bindings.forEach((bindingName, binding) -> { + if (bindingName == null || bindingName.isBlank() || binding == null) { + throw new IllegalArgumentException("binding name and definition must be provided"); + } + if (!normalizedBindingNames.add(bindingName.toUpperCase(Locale.ROOT))) { + throw new IllegalArgumentException( + "binding names must be unique ignoring unquoted identifier case" + ); + } + copied.put(bindingName, binding); + }); + bindings = Collections.unmodifiableMap(copied); + if (defaultBinding == null || !bindings.containsKey(defaultBinding)) { + throw new IllegalArgumentException("defaultBinding must reference a declared binding"); + } + List copiedTables = logicalTables == null + ? List.of() + : List.copyOf(logicalTables); + Set normalizedLogicalNames = new HashSet<>(); + for (FederationLogicalTableDefinition table : copiedTables) { + if (table == null) { + throw new IllegalArgumentException("logical table definition must not be null"); + } + if (!bindings.containsKey(table.bindingName())) { + throw new IllegalArgumentException( + "logical table binding must reference a declared binding: " + + table.bindingName() + ); + } + if (!normalizedLogicalNames.add(table.logicalName().toUpperCase(Locale.ROOT))) { + throw new IllegalArgumentException( + "logical table names must be unique ignoring unquoted identifier case" + ); + } + } + logicalTables = copiedTables; + executionPolicy = executionPolicy == null + ? FederationExecutionPolicy.basic() + : executionPolicy; + } + + /** + * 创建不声明逻辑表映射的兼容查询范围。 + * + * @param definitionId 调用方定义标识 + * @param revision 查询范围版本 + * @param bindings Binding 名称到物理数据源定义的映射 + * @param defaultBinding 默认 Binding 名称 + * @param executionPolicy 调用方联邦资源上限 + */ + public FederationQueryScopeDefinition( + String definitionId, + long revision, + Map bindings, + String defaultBinding, + FederationExecutionPolicy executionPolicy + ) { + this( + definitionId, + revision, + bindings, + defaultBinding, + List.of(), + executionPolicy + ); + } + + /** + * 创建单物理数据源查询范围。 + * + * @param sourceId 物理数据源标识,同时作为默认 Binding 名称 + * @param minimumRevision 最低 Definition 版本 + * @return 单源查询范围 + */ + public static FederationQueryScopeDefinition single( + SourceId sourceId, + long minimumRevision + ) { + if (sourceId == null) { + throw new IllegalArgumentException("sourceId must not be null"); + } + return single( + "source:" + sourceId.value(), + minimumRevision, + sourceId.value(), + sourceId, + minimumRevision + ); + } + + /** + * 创建调用方管理的虚拟联邦查询范围。 + * + * @param definitionId 查询范围标识 + * @param revision 查询范围版本 + * @param bindings SQL 逻辑 Binding 到物理数据源的映射 + * @param defaultBinding 默认 Binding + * @param executionPolicy 调用方资源上限 + * @return 虚拟联邦查询范围 + */ + public static FederationQueryScopeDefinition virtual( + String definitionId, + long revision, + Map bindings, + String defaultBinding, + FederationExecutionPolicy executionPolicy + ) { + return new FederationQueryScopeDefinition( + definitionId, + revision, + bindings, + defaultBinding, + List.of(), + executionPolicy + ); + } + + /** + * 创建带逻辑表映射的虚拟联邦查询范围。 + * + * @param definitionId 查询范围标识 + * @param revision 查询范围版本 + * @param bindings SQL 逻辑 Binding 到物理数据源的映射 + * @param defaultBinding 默认 Binding + * @param logicalTables 查询方可见的逻辑表映射 + * @param executionPolicy 调用方资源上限 + * @return 虚拟联邦查询范围 + */ + public static FederationQueryScopeDefinition virtual( + String definitionId, + long revision, + Map bindings, + String defaultBinding, + List logicalTables, + FederationExecutionPolicy executionPolicy + ) { + return new FederationQueryScopeDefinition( + definitionId, + revision, + bindings, + defaultBinding, + logicalTables, + executionPolicy + ); + } + + /** + * 创建带独立范围版本的单物理数据源查询范围。 + * + * @param definitionId 查询范围标识 + * @param scopeRevision 查询范围版本 + * @param bindingName 默认 Binding 名称 + * @param sourceId 物理数据源标识 + * @param minimumRevision 最低 Definition 版本 + * @return 单源查询范围 + */ + public static FederationQueryScopeDefinition single( + String definitionId, + long scopeRevision, + String bindingName, + SourceId sourceId, + long minimumRevision + ) { + return new FederationQueryScopeDefinition( + definitionId, + scopeRevision, + Map.of(bindingName, FederationSourceBindingDefinition.of(sourceId, minimumRevision)), + bindingName, + List.of(), + FederationExecutionPolicy.basic() + ); + } + + /** + * 返回默认 Binding。 + * + * @return 默认 Binding 定义 + */ + public FederationSourceBindingDefinition defaultBindingDefinition() { + return bindings.get(defaultBinding); + } + + /** + * 返回用于计划缓存与跨节点一致性判断的稳定摘要。 + * + * @return SHA-256 十六进制摘要 + */ + public String checksum() { + try { + MessageDigest digest = MessageDigest.getInstance("SHA-256"); + update(digest, "federation-query-scope-v2"); + update(digest, definitionId); + update(digest, Long.toString(revision)); + update(digest, defaultBinding); + List> orderedBindings = + new ArrayList<>(bindings.entrySet()); + orderedBindings.sort(Map.Entry.comparingByKey()); + updateCount(digest, orderedBindings.size()); + for (Map.Entry entry : orderedBindings) { + update(digest, "binding"); + update(digest, entry.getKey()); + FederationSourceBindingDefinition binding = entry.getValue(); + update(digest, binding.sourceId().value()); + update(digest, Long.toString(binding.minimumRevision())); + List> mappings = + new ArrayList<>(binding.schemaMappings().entrySet()); + mappings.sort(Comparator.comparing(Map.Entry::getKey)); + updateCount(digest, mappings.size()); + for (Map.Entry mapping : mappings) { + update(digest, "schema-mapping"); + update(digest, mapping.getKey()); + update(digest, mapping.getValue()); + } + } + List orderedTables = + new ArrayList<>(logicalTables); + orderedTables.sort(Comparator.comparing( + table -> table.logicalName().toUpperCase(Locale.ROOT) + )); + updateCount(digest, orderedTables.size()); + for (FederationLogicalTableDefinition table : orderedTables) { + update(digest, "logical-table"); + update(digest, table.logicalName()); + update(digest, table.bindingName()); + update(digest, table.schemaName()); + update(digest, table.sourceTableName()); + } + update(digest, "execution-policy"); + update(digest, Integer.toString(executionPolicy.maximumReferencedSources())); + update(digest, Integer.toString(executionPolicy.maximumFragments())); + update(digest, Integer.toString(executionPolicy.maximumConcurrentFragments())); + update(digest, Long.toString(executionPolicy.maximumIntermediateRows())); + update(digest, Long.toString(executionPolicy.maximumIntermediateBytes())); + update(digest, Long.toString(executionPolicy.maximumExecutionTimeMillis())); + return HexFormat.of().formatHex(digest.digest()); + } catch (NoSuchAlgorithmException exception) { + throw new IllegalStateException("SHA-256 is not available", exception); + } + } + + private static void update(MessageDigest digest, String value) { + byte[] bytes = value.getBytes(StandardCharsets.UTF_8); + updateCount(digest, bytes.length); + digest.update(bytes); + } + + private static void updateCount(MessageDigest digest, int value) { + digest.update(ByteBuffer.allocate(Integer.BYTES).putInt(value).array()); + } +} diff --git a/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/federation/FederationSourceBindingDefinition.java b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/federation/FederationSourceBindingDefinition.java new file mode 100644 index 0000000..633f366 --- /dev/null +++ b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/federation/FederationSourceBindingDefinition.java @@ -0,0 +1,87 @@ +package com.easyagents.federation.sql.federation; + +import com.easyagents.federation.sql.source.SourceId; +import java.io.Serializable; +import java.util.Collections; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.Locale; +import java.util.Map; +import java.util.Set; + +/** + * 将查询范围内的一个 Binding 绑定到已经登记的物理数据源。 + * + * @param sourceId 物理数据源标识 + * @param minimumRevision 查询要求的最低 Definition 版本 + * @param schemaMappings 查询逻辑 Schema 到物理 Definition 逻辑 Schema 的映射;空映射表示同名暴露全部 Schema + */ +public record FederationSourceBindingDefinition( + SourceId sourceId, + long minimumRevision, + Map schemaMappings +) implements Serializable { + + /** + * 校验并创建不可变 Binding 定义。 + */ + public FederationSourceBindingDefinition { + if (sourceId == null) { + throw new IllegalArgumentException("sourceId must not be null"); + } + if (minimumRevision < 0) { + throw new IllegalArgumentException("minimumRevision must not be negative"); + } + LinkedHashMap copied = new LinkedHashMap<>(); + Set normalizedSchemaNames = new HashSet<>(); + if (schemaMappings != null) { + schemaMappings.forEach((querySchema, sourceSchema) -> { + if (querySchema == null || querySchema.isBlank() + || sourceSchema == null || sourceSchema.isBlank()) { + throw new IllegalArgumentException("schema mapping names must not be blank"); + } + if (!normalizedSchemaNames.add(querySchema.toUpperCase(Locale.ROOT))) { + throw new IllegalArgumentException( + "query schema names must be unique ignoring unquoted identifier case" + ); + } + copied.put(querySchema, sourceSchema); + }); + } + schemaMappings = Collections.unmodifiableMap(copied); + } + + /** + * 创建不改写 Schema 名称的物理数据源 Binding。 + * + * @param sourceId 物理数据源标识 + * @param minimumRevision 最低 Definition 版本 + * @return Binding 定义 + */ + public static FederationSourceBindingDefinition of( + SourceId sourceId, + long minimumRevision + ) { + return new FederationSourceBindingDefinition(sourceId, minimumRevision, Map.of()); + } + + /** + * 创建显式映射查询 Schema 的物理数据源 Binding。 + * + * @param sourceId 物理数据源标识 + * @param minimumRevision 最低 Definition 版本 + * @param schemaMappings 查询逻辑 Schema 到 Definition 逻辑 Schema 的映射 + * @return Binding 定义 + */ + public static FederationSourceBindingDefinition of( + SourceId sourceId, + long minimumRevision, + Map schemaMappings + ) { + return new FederationSourceBindingDefinition( + sourceId, + minimumRevision, + schemaMappings + ); + } +} diff --git a/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/federation/FederationSourceRuntimeIdentity.java b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/federation/FederationSourceRuntimeIdentity.java new file mode 100644 index 0000000..902013c --- /dev/null +++ b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/federation/FederationSourceRuntimeIdentity.java @@ -0,0 +1,36 @@ +package com.easyagents.federation.sql.federation; + +import com.easyagents.federation.sql.source.SourceId; + +/** + * 编译计划绑定的节点本地物理数据源运行身份。 + * + * @param bindingName 查询范围 Binding 名称 + * @param sourceId 物理数据源标识 + * @param sourceRevision Definition 版本 + * @param sourceChecksum Definition 校验和 + * @param adapterId Adapter 标识 + * @param runtimeFingerprint 数据库与驱动运行指纹 + */ +public record FederationSourceRuntimeIdentity( + String bindingName, + SourceId sourceId, + long sourceRevision, + String sourceChecksum, + String adapterId, + String runtimeFingerprint +) { + + /** + * 校验运行身份字段。 + */ + public FederationSourceRuntimeIdentity { + if (bindingName == null || bindingName.isBlank() + || sourceId == null || sourceRevision < 0 + || sourceChecksum == null || sourceChecksum.isBlank() + || adapterId == null || adapterId.isBlank() + || runtimeFingerprint == null || runtimeFingerprint.isBlank()) { + throw new IllegalArgumentException("source runtime identity is incomplete"); + } + } +} diff --git a/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/federation/FederationStatisticsSnapshot.java b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/federation/FederationStatisticsSnapshot.java new file mode 100644 index 0000000..0f76f16 --- /dev/null +++ b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/federation/FederationStatisticsSnapshot.java @@ -0,0 +1,235 @@ +package com.easyagents.federation.sql.federation; + +import com.easyagents.federation.sql.source.SourceId; +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.time.Instant; +import java.util.Comparator; +import java.util.HexFormat; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Objects; +import java.util.Set; + +/** + * 一次编译期间不可变引用的统计快照。 + * + *

构造时会复制完整统计映射,确保版本、数据与有效期属于同一个冻结视图。

+ */ +public final class FederationStatisticsSnapshot { + + private final String version; + private final Instant capturedAt; + private final Map statisticsByTable; + private final Instant validUntil; + + /** + * 创建统计快照。 + * + * @param version 稳定快照版本 + * @param statisticsByTable 按物理表索引的统计映射 + */ + public FederationStatisticsSnapshot( + String version, + Map statisticsByTable + ) { + this(version, Instant.now(), statisticsByTable); + } + + /** + * 创建带固定评估时刻的统计快照。 + * + * @param version 稳定快照版本 + * @param capturedAt 快照捕获和有效期评估时刻 + * @param statisticsByTable 按物理表索引的统计映射 + */ + public FederationStatisticsSnapshot( + String version, + Instant capturedAt, + Map statisticsByTable + ) { + this.version = version == null || version.isBlank() ? "none" : version; + this.capturedAt = Objects.requireNonNull(capturedAt, "capturedAt"); + this.statisticsByTable = Map.copyOf(new LinkedHashMap<>( + statisticsByTable == null ? Map.of() : statisticsByTable + )); + this.validUntil = this.statisticsByTable.values().stream() + .filter(statistics -> { + FederationStatisticsStatus status = statistics.effectiveStatus(capturedAt); + return status == FederationStatisticsStatus.COMPLETE + || status == FederationStatisticsStatus.PARTIAL; + }) + .map(FederationTableStatistics::expiresAt) + .min(Instant::compareTo) + .orElse(Instant.MAX); + } + + /** + * 返回空统计快照。 + * + * @return 空快照 + */ + public static FederationStatisticsSnapshot empty() { + return new FederationStatisticsSnapshot("none", Instant.now(), Map.of()); + } + + /** + * 返回快照版本。 + * + * @return 稳定版本 + */ + public String version() { + return version; + } + + /** + * 返回本次编译统一使用的统计有效期评估时刻。 + * + * @return 快照捕获时刻 + */ + public Instant capturedAt() { + return capturedAt; + } + + /** + * 返回该快照内最早的统计失效时间。 + * + * @return 最早失效时间;空快照为 {@link Instant#MAX} + */ + public Instant validUntil() { + return validUntil; + } + + /** + * 查询冻结快照中的表统计。 + * + * @param sourceId 物理源 + * @param schema Source Definition 暴露的逻辑 Schema 名称 + * @param table 表名称 + * @return 表统计;缺失时为 {@code null} + */ + public FederationTableStatistics statistics( + SourceId sourceId, + String schema, + String table + ) { + return statisticsByTable.get(new TableKey(sourceId, schema, table)); + } + + /** + * 截取指定物理表的稳定统计身份与最早失效时间。 + * + *

该结果只依赖查询实际引用的表。未引用表的刷新不会使已有计划失效。

+ * + * @param tables 查询实际引用的物理表键 + * @return 查询级统计选择结果 + */ + public Selection select(Set tables) { + List ordered = (tables == null ? Set.of() : tables).stream() + .sorted(Comparator + .comparing((TableKey key) -> key.sourceId().value()) + .thenComparing(TableKey::schema) + .thenComparing(TableKey::table)) + .toList(); + MessageDigest digest = sha256(); + Instant selectedValidUntil = Instant.MAX; + for (TableKey key : ordered) { + update(digest, key.sourceId().value()); + update(digest, key.schema()); + update(digest, key.table()); + FederationTableStatistics statistics = statisticsByTable.get(key); + if (statistics == null) { + update(digest, "missing"); + continue; + } + FederationStatisticsStatus effectiveStatus = statistics.effectiveStatus(capturedAt); + update(digest, effectiveStatus.name()); + update(digest, Double.toString(statistics.estimatedRows())); + update(digest, Long.toString(statistics.averageRowWidthBytes())); + update(digest, statistics.collectedAt().toString()); + update(digest, statistics.source()); + update(digest, statistics.expiresAt().toString()); + statistics.columns().entrySet().stream() + .sorted(Map.Entry.comparingByKey(String.CASE_INSENSITIVE_ORDER)) + .forEach(entry -> { + update(digest, entry.getKey().toLowerCase(Locale.ROOT)); + update(digest, entry.getValue().toString()); + }); + statistics.uniqueKeys().stream() + .map(columns -> columns.stream() + .map(value -> value.toLowerCase(Locale.ROOT)) + .sorted() + .toList()) + .map(columns -> String.join("\u0001", columns)) + .sorted() + .forEach(value -> update(digest, value)); + if ((effectiveStatus == FederationStatisticsStatus.COMPLETE + || effectiveStatus == FederationStatisticsStatus.PARTIAL) + && statistics.expiresAt().isBefore(selectedValidUntil)) { + selectedValidUntil = statistics.expiresAt(); + } + } + return new Selection( + HexFormat.of().formatHex(digest.digest()), + selectedValidUntil + ); + } + + /** + * 查询级统计选择结果。 + * + * @param fingerprint 实际引用表统计的稳定指纹 + * @param validUntil 实际引用表统计的最早失效时间 + */ + public record Selection(String fingerprint, Instant validUntil) { + + /** 校验查询级统计选择结果。 */ + public Selection { + fingerprint = Objects.requireNonNull(fingerprint, "fingerprint"); + validUntil = validUntil == null ? Instant.MAX : validUntil; + } + } + + private static MessageDigest sha256() { + try { + return MessageDigest.getInstance("SHA-256"); + } catch (NoSuchAlgorithmException exception) { + throw new IllegalStateException("SHA-256 is unavailable", exception); + } + } + + private static void update(MessageDigest digest, String value) { + byte[] bytes = value.getBytes(StandardCharsets.UTF_8); + digest.update((byte) (bytes.length >>> 24)); + digest.update((byte) (bytes.length >>> 16)); + digest.update((byte) (bytes.length >>> 8)); + digest.update((byte) bytes.length); + digest.update(bytes); + } + + /** + * 数据源表统计键。 + * + * @param sourceId 物理源 + * @param schema Source Definition 暴露的逻辑 Schema 名称 + * @param table 表名称 + */ + public record TableKey(SourceId sourceId, String schema, String table) { + + /** + * 规范化物理表键,保证常见数据库标识符大小写差异不影响命中。 + */ + public TableKey { + sourceId = Objects.requireNonNull(sourceId, "sourceId"); + schema = normalize(schema); + table = normalize(table); + } + + private static String normalize(String value) { + return value == null ? "" : value.toLowerCase(Locale.ROOT); + } + } +} diff --git a/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/federation/FederationStatisticsStatus.java b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/federation/FederationStatisticsStatus.java new file mode 100644 index 0000000..a9a8cae --- /dev/null +++ b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/federation/FederationStatisticsStatus.java @@ -0,0 +1,19 @@ +package com.easyagents.federation.sql.federation; + +/** + * 联邦查询成本统计的可用状态。 + */ +public enum FederationStatisticsStatus { + + /** 所需表均具有当前可信统计。 */ + COMPLETE, + + /** 仅部分表或字段具有可信统计。 */ + PARTIAL, + + /** 没有可用的外部统计。 */ + MISSING, + + /** 统计存在但已超过调用方声明的有效期。 */ + STALE +} diff --git a/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/federation/FederationTableStatistics.java b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/federation/FederationTableStatistics.java new file mode 100644 index 0000000..ca770dc --- /dev/null +++ b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/federation/FederationTableStatistics.java @@ -0,0 +1,90 @@ +package com.easyagents.federation.sql.federation; + +import java.io.Serializable; +import java.time.Instant; +import java.util.List; +import java.util.Map; + +/** + * 调用方或 Adapter 提供的一张物理表的轻量统计快照。 + * + * @param estimatedRows 估算总行数 + * @param averageRowWidthBytes 平均物理行宽字节数 + * @param collectedAt 统计采集时间 + * @param source 统计来源,例如 catalog、adapter 或业务统计服务 + * @param columns 按物理列名索引的列统计 + * @param uniqueKeys 唯一键列集合 + * @param expiresAt 统计失效时间;不失效时为 {@link Instant#MAX} + * @param status Provider 声明的统计完整状态 + */ +public record FederationTableStatistics( + double estimatedRows, + long averageRowWidthBytes, + Instant collectedAt, + String source, + Map columns, + List> uniqueKeys, + Instant expiresAt, + FederationStatisticsStatus status +) implements Serializable { + + /** + * 校验并规范化统计值。 + */ + public FederationTableStatistics { + if (!Double.isFinite(estimatedRows) || estimatedRows < 0) { + throw new IllegalArgumentException("estimatedRows must be finite and non-negative"); + } + if (averageRowWidthBytes <= 0) { + throw new IllegalArgumentException("averageRowWidthBytes must be positive"); + } + collectedAt = collectedAt == null ? Instant.EPOCH : collectedAt; + source = source == null || source.isBlank() ? "unspecified" : source; + columns = Map.copyOf(columns == null ? Map.of() : columns); + uniqueKeys = (uniqueKeys == null ? List.>of() : uniqueKeys).stream() + .map(List::copyOf) + .toList(); + expiresAt = expiresAt == null ? Instant.MAX : expiresAt; + status = status == null ? FederationStatisticsStatus.COMPLETE : status; + } + + /** + * 创建仅包含表级统计的兼容快照。 + * + * @param estimatedRows 估算总行数 + * @param averageRowWidthBytes 平均行宽 + * @param collectedAt 采集时间 + * @param source 统计来源 + */ + public FederationTableStatistics( + double estimatedRows, + long averageRowWidthBytes, + Instant collectedAt, + String source + ) { + this( + estimatedRows, + averageRowWidthBytes, + collectedAt, + source, + Map.of(), + List.of(), + Instant.MAX, + FederationStatisticsStatus.COMPLETE + ); + } + + /** + * 返回指定时刻的有效状态。 + * + * @param now 当前时间 + * @return 计入有效期后的状态 + */ + public FederationStatisticsStatus effectiveStatus(Instant now) { + if (status != FederationStatisticsStatus.MISSING + && !expiresAt.isAfter(now == null ? Instant.now() : now)) { + return FederationStatisticsStatus.STALE; + } + return status; + } +} diff --git a/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/federation/FederationTableStatisticsProvider.java b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/federation/FederationTableStatisticsProvider.java new file mode 100644 index 0000000..0f6dc11 --- /dev/null +++ b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/federation/FederationTableStatisticsProvider.java @@ -0,0 +1,56 @@ +package com.easyagents.federation.sql.federation; + +import com.easyagents.federation.sql.source.SourceId; + +/** + * 为联邦编译提供轻量、无数据库查询副作用的表统计快照。 + * + *

实现必须直接返回已冻结快照,不得在编译热路径执行 {@code COUNT(*)}。统计数据变化时, + * 快照版本必须同步变化,使节点本地计划缓存自然隔离旧成本计划。

+ */ +@FunctionalInterface +public interface FederationTableStatisticsProvider { + + /** + * 捕获一次编译使用的不可变统计快照。 + * + * @return 同时冻结版本、数据与有效期的快照 + */ + FederationStatisticsSnapshot snapshot(); + + /** + * 查询当前快照中的一张物理表统计。 + * + *

编译器会先捕获一次 {@link #snapshot()} 并复用,调用方仅应将本方法用于诊断读取。

+ * + * @param sourceId 物理数据源标识 + * @param schema 物理 Schema 名称 + * @param table 物理表名称 + * @return 统计快照;没有可信统计时返回 {@code null} + */ + default FederationTableStatistics statistics( + SourceId sourceId, + String schema, + String table + ) { + return snapshot().statistics(sourceId, schema, table); + } + + /** + * 返回当前统计快照版本。 + * + * @return 非空版本字符串 + */ + default String snapshotVersion() { + return snapshot().version(); + } + + /** + * 返回不提供外部统计的实现。 + * + * @return 空统计 Provider + */ + static FederationTableStatisticsProvider none() { + return FederationStatisticsSnapshot::empty; + } +} diff --git a/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/runtime/AdapterFederationStatisticsProvider.java b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/runtime/AdapterFederationStatisticsProvider.java new file mode 100644 index 0000000..f3de8c6 --- /dev/null +++ b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/runtime/AdapterFederationStatisticsProvider.java @@ -0,0 +1,510 @@ +package com.easyagents.federation.sql.runtime; + +import com.easyagents.federation.sql.adapter.FederationStatisticsCollectionContext; +import com.easyagents.federation.sql.adapter.FederationStatisticsCollector; +import com.easyagents.federation.sql.api.FederationSqlErrorCode; +import com.easyagents.federation.sql.api.FederationSqlException; +import com.easyagents.federation.sql.federation.FederationStatisticsSnapshot; +import com.easyagents.federation.sql.federation.FederationTableStatistics; +import com.easyagents.federation.sql.federation.FederationTableStatisticsProvider; +import com.easyagents.federation.sql.source.SourceId; +import java.sql.Connection; +import java.sql.SQLException; +import java.time.Duration; +import java.time.Instant; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ArrayBlockingQueue; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.RejectedExecutionException; +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * 使用数据库 Adapter 自动采集、缓存并降级联邦查询统计。 + */ +final class AdapterFederationStatisticsProvider + implements FederationTableStatisticsProvider, AutoCloseable { + + private static final Logger LOG = LoggerFactory.getLogger( + AdapterFederationStatisticsProvider.class + ); + private static final Duration DEFAULT_TTL = Duration.ofMinutes(30); + private static final Duration DEFAULT_FAILURE_RETRY_DELAY = Duration.ofMinutes(1); + private static final int DEFAULT_QUERY_TIMEOUT_SECONDS = 5; + private static final int MAXIMUM_PARALLEL_REFRESHES = 4; + private static final int MAXIMUM_PENDING_REFRESHES = 64; + + private final Duration ttl; + private final Duration failureRetryDelay; + private final int queryTimeoutSeconds; + private final ConcurrentMap sourceLocks = new ConcurrentHashMap<>(); + private final ConcurrentMap> inFlight = + new ConcurrentHashMap<>(); + private final ExecutorService refreshExecutor; + private final AtomicBoolean closed = new AtomicBoolean(); + private volatile RegistryState state = RegistryState.empty(); + private long versionSequence; + + /** + * 使用默认有效期、失败限频和有界并行度创建自动统计 Provider。 + */ + AdapterFederationStatisticsProvider() { + this( + DEFAULT_TTL, + DEFAULT_FAILURE_RETRY_DELAY, + DEFAULT_QUERY_TIMEOUT_SECONDS, + Math.min( + MAXIMUM_PARALLEL_REFRESHES, + Math.max(1, Runtime.getRuntime().availableProcessors()) + ) + ); + } + + /** + * 创建可测试的自动统计 Provider。 + * + * @param ttl 统计有效期 + * @param failureRetryDelay 失败后的最短重试间隔 + * @param queryTimeoutSeconds 单条目录查询超时 + * @param parallelism 不同物理源的最大并行采集数 + */ + AdapterFederationStatisticsProvider( + Duration ttl, + Duration failureRetryDelay, + int queryTimeoutSeconds, + int parallelism + ) { + this.ttl = positive(ttl, "ttl"); + this.failureRetryDelay = positive(failureRetryDelay, "failureRetryDelay"); + if (queryTimeoutSeconds <= 0) { + throw new IllegalArgumentException("queryTimeoutSeconds must be positive"); + } + if (parallelism <= 0) { + throw new IllegalArgumentException("parallelism must be positive"); + } + this.queryTimeoutSeconds = queryTimeoutSeconds; + this.refreshExecutor = new ThreadPoolExecutor( + parallelism, + parallelism, + 0L, + TimeUnit.MILLISECONDS, + new ArrayBlockingQueue<>(MAXIMUM_PENDING_REFRESHES), + daemonThreadFactory() + ); + } + + /** + * 返回当前冻结统计快照。 + * + * @return 无数据库访问副作用的不可变快照 + */ + @Override + public FederationStatisticsSnapshot snapshot() { + return state.snapshot(); + } + + /** + * 异步刷新查询实际涉及的物理源统计。 + * + *

同一 source revision 的并发刷新合并为一个任务,不同物理源最多按固定 + * 并行度同时读取目录。该方法不阻塞调用线程;需要等待统计的 Explain 调用可 + * 在自身截止时间内等待返回的 Future。失败时保留旧统计并限频。

+ * + * @param querySnapshot 已持有 Runtime lease 的查询范围快照 + * @return 所有已安排刷新完成时结束的 Future + */ + CompletableFuture refreshIfNeeded(FederationQueryScopeSnapshot querySnapshot) { + if (closed.get()) { + return CompletableFuture.completedFuture(null); + } + Map runtimes = new LinkedHashMap<>(); + querySnapshot.runtimesByBinding().values().forEach(runtime -> + runtimes.putIfAbsent(runtime.definition().sourceId(), runtime) + ); + Instant now = Instant.now(); + List> refreshes = new ArrayList<>(); + for (SourceRuntime runtime : runtimes.values()) { + if (!fresh(state.sources().get(runtime.definition().sourceId()), runtime, now)) { + refreshes.add(refreshAsync(runtime)); + } + } + return refreshes.isEmpty() + ? CompletableFuture.completedFuture(null) + : CompletableFuture.allOf(refreshes.toArray(CompletableFuture[]::new)); + } + + /** + * 在 Runtime 最终关闭后清除同 revision 的节点本地统计。 + * + * @param runtime 已关闭 Runtime + */ + void invalidate(SourceRuntime runtime) { + if (closed.get()) { + return; + } + SourceId sourceId = runtime.definition().sourceId(); + Object lock = sourceLocks.computeIfAbsent(sourceId, ignored -> new Object()); + synchronized (lock) { + SourceStatistics current = state.sources().get(sourceId); + if (current != null && current.revision() == runtime.definition().revision()) { + replace(sourceId, null); + } + } + // 锁对象必须覆盖数据源的完整生命周期,避免旧 revision 关闭与新 revision + // 刷新交错时为同一数据源创建两个临界区。锁会在 Provider 关闭时统一释放。 + } + + /** + * 关闭后台采集器并释放节点本地统计。 + */ + @Override + public void close() { + if (!closed.compareAndSet(false, true)) { + return; + } + refreshExecutor.shutdownNow(); + try { + if (!refreshExecutor.awaitTermination( + queryTimeoutSeconds + 1L, + TimeUnit.SECONDS + )) { + LOG.warn( + "federation statistics refresh did not stop within the bounded shutdown window" + ); + } + } catch (InterruptedException exception) { + Thread.currentThread().interrupt(); + LOG.warn("interrupted while closing federation statistics refresh", exception); + } + inFlight.clear(); + sourceLocks.clear(); + synchronized (this) { + state = RegistryState.empty(); + } + } + + /** + * 合并同一 source revision 的并发刷新。 + * + * @param runtime 待刷新 Runtime + * @return 可等待的共享刷新任务 + */ + private CompletableFuture refreshAsync(SourceRuntime runtime) { + SourceRevision key = new SourceRevision( + runtime.definition().sourceId(), + runtime.definition().revision() + ); + CompletableFuture created = new CompletableFuture<>(); + CompletableFuture existing = inFlight.putIfAbsent(key, created); + if (existing != null) { + return existing; + } + Runnable task = () -> { + try { + try (SourceRuntime.RuntimeLease lease = runtime.acquire()) { + // 后台采集必须持有独立租约,避免查询快照释放后旧 Runtime + // 在目录连接仍被使用时关闭连接池。 + refresh(lease.runtime()); + } + // 租约释放可能触发旧 Runtime 关闭,完成信号必须晚于资源收口。 + created.complete(null); + } catch (FederationSqlException exception) { + if (exception.errorCode() == FederationSqlErrorCode.SOURCE_REVISION_NOT_READY) { + // 已退休 revision 的统计没有继续采集的价值,保留现有估算即可。 + created.complete(null); + } else { + created.completeExceptionally(exception); + } + } catch (RuntimeException exception) { + created.completeExceptionally(exception); + } finally { + inFlight.remove(key, created); + } + }; + try { + refreshExecutor.execute(task); + } catch (RejectedExecutionException exception) { + // 查询线程不能在统计线程池饱和时退化为同步目录查询。 + created.complete(null); + inFlight.remove(key, created); + if (!closed.get()) { + LOG.warn( + "federation statistics refresh rejected; using existing or default estimates, sourceId={}, revision={}", + runtime.definition().sourceId(), + runtime.definition().revision() + ); + } + } + return created; + } + + /** + * 在源级临界区读取数据库目录并发布统计。 + * + * @param runtime 数据源 Runtime + */ + private void refresh(SourceRuntime runtime) { + SourceId sourceId = runtime.definition().sourceId(); + long revision = runtime.definition().revision(); + Object lock = sourceLocks.computeIfAbsent(sourceId, ignored -> new Object()); + synchronized (lock) { + Instant now = Instant.now(); + SourceStatistics current = state.sources().get(sourceId); + if (current != null && current.revision() > revision) { + return; + } + if (fresh(current, runtime, now)) { + return; + } + try { + Map tables = + collect(runtime, now); + if (closed.get()) { + return; + } + replace(sourceId, new SourceStatistics( + revision, + validateSource(sourceId, tables), + now.plus(ttl) + )); + } catch (SQLException | RuntimeException exception) { + if (!closed.get()) { + throttleAfterFailure(sourceId, revision, now); + } + LOG.warn( + "federation statistics collection failed; using existing or default estimates, sourceId={}, revision={}", + sourceId, + revision, + exception + ); + } + } + } + + /** + * 借用运行时连接并调用 Adapter 统计采集器。 + * + * @param runtime 数据源 Runtime + * @param collectedAt 采集时间 + * @return 表统计映射 + * @throws SQLException 获取连接或读取目录失败 + */ + private Map collect( + SourceRuntime runtime, + Instant collectedAt + ) throws SQLException { + FederationStatisticsCollector collector = runtime.adapter().statisticsCollector() + .orElse(null); + if (collector == null) { + return Map.of(); + } + try (Connection connection = runtime.handle().dataSource().getConnection()) { + markReadOnly(connection); + Map collected = + collector.collect(new FederationStatisticsCollectionContext( + runtime.definition(), + connection, + collectedAt, + collectedAt.plus(ttl), + queryTimeoutSeconds + )); + return Map.copyOf(collected == null ? Map.of() : collected); + } + } + + /** + * 尽力将目录连接标记为只读;驱动不支持时仍由只读 SQL 保证安全。 + * + * @param connection JDBC 连接 + */ + private void markReadOnly(Connection connection) { + try { + if (!connection.isReadOnly()) { + connection.setReadOnly(true); + } + } catch (SQLException exception) { + LOG.debug("JDBC driver does not support setting statistics connection read-only", exception); + } + } + + /** + * 校验 Adapter 只能发布当前物理源的统计。 + * + * @param sourceId 当前物理源 + * @param tables Adapter 统计 + * @return 冻结统计映射 + */ + private Map validateSource( + SourceId sourceId, + Map tables + ) { + for (FederationStatisticsSnapshot.TableKey key : tables.keySet()) { + if (!sourceId.equals(key.sourceId())) { + throw new IllegalArgumentException( + "statistics collector returned a table for another source: " + key.sourceId() + ); + } + } + return Map.copyOf(tables); + } + + /** + * 判断当前 revision 的统计是否仍在刷新窗口内。 + * + * @param statistics 已缓存源统计 + * @param runtime 当前 Runtime + * @param now 当前时间 + * @return 可直接使用时为 true + */ + private boolean fresh(SourceStatistics statistics, SourceRuntime runtime, Instant now) { + return statistics != null + && statistics.revision() == runtime.definition().revision() + && statistics.refreshAfter().isAfter(now); + } + + /** + * 采集失败时保留同 revision 旧值并限制后续重试频率。 + * + * @param sourceId 物理源 + * @param revision 当前 revision + * @param failedAt 失败时间 + */ + private void throttleAfterFailure(SourceId sourceId, long revision, Instant failedAt) { + SourceStatistics current = state.sources().get(sourceId); + Map tables = + current != null && current.revision() == revision ? current.tables() : Map.of(); + replace(sourceId, new SourceStatistics( + revision, + tables, + failedAt.plus(failureRetryDelay) + )); + } + + /** + * 原子替换单源统计并重建聚合快照。 + * + * @param sourceId 物理源 + * @param statistics 新统计;null 表示移除 + */ + private synchronized void replace(SourceId sourceId, SourceStatistics statistics) { + Map sources = new LinkedHashMap<>(state.sources()); + if (statistics == null) { + sources.remove(sourceId); + } else { + sources.put(sourceId, statistics); + } + Map tables = + new LinkedHashMap<>(); + sources.values().forEach(value -> tables.putAll(value.tables())); + versionSequence++; + state = new RegistryState( + Map.copyOf(sources), + new FederationStatisticsSnapshot( + "adapter-" + versionSequence, + Instant.now(), + tables + ) + ); + } + + /** + * 校验正 Duration。 + * + * @param value Duration + * @param name 参数名 + * @return 原值 + */ + private static Duration positive(Duration value, String name) { + if (value == null || value.isZero() || value.isNegative()) { + throw new IllegalArgumentException(name + " must be positive"); + } + return value; + } + + /** + * 创建统计采集守护线程工厂。 + * + * @return 守护线程工厂 + */ + private static ThreadFactory daemonThreadFactory() { + AtomicInteger sequence = new AtomicInteger(); + return task -> { + Thread thread = new Thread( + task, + "federation-statistics-" + sequence.incrementAndGet() + ); + thread.setDaemon(true); + return thread; + }; + } + + /** + * 单物理源统计及刷新边界。 + * + * @param revision Definition revision + * @param tables 表统计 + * @param refreshAfter 下次允许刷新时间 + */ + private record SourceStatistics( + long revision, + Map tables, + Instant refreshAfter + ) { + + /** + * 冻结源统计。 + */ + private SourceStatistics { + tables = Map.copyOf(tables == null ? Map.of() : tables); + refreshAfter = refreshAfter == null ? Instant.EPOCH : refreshAfter; + } + } + + /** + * 正在采集的物理源 revision。 + * + * @param sourceId 物理源 + * @param revision Definition revision + */ + private record SourceRevision(SourceId sourceId, long revision) { + } + + /** + * 节点当前聚合统计状态。 + * + * @param sources 按物理源索引的统计 + * @param snapshot 冻结聚合快照 + */ + private record RegistryState( + Map sources, + FederationStatisticsSnapshot snapshot + ) { + + /** + * 创建空统计状态。 + * + * @return 空状态 + */ + private static RegistryState empty() { + return new RegistryState(Map.of(), FederationStatisticsSnapshot.empty()); + } + + /** + * 冻结聚合状态。 + */ + private RegistryState { + sources = Map.copyOf(sources == null ? Map.of() : sources); + } + } +} diff --git a/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/runtime/BoundedPlanCache.java b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/runtime/BoundedPlanCache.java new file mode 100644 index 0000000..38d6a89 --- /dev/null +++ b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/runtime/BoundedPlanCache.java @@ -0,0 +1,719 @@ +package com.easyagents.federation.sql.runtime; + +import com.easyagents.federation.sql.api.FederationSqlErrorCode; +import com.easyagents.federation.sql.api.FederationSqlException; +import com.easyagents.federation.sql.compile.FederationSqlPlan; +import java.time.Duration; +import java.time.Instant; +import java.util.Iterator; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionException; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Semaphore; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.locks.LockSupport; +import java.util.function.Predicate; +import java.util.function.LongSupplier; +import java.util.function.Supplier; +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.RelVisitor; + +/** + * 有界 LRU 计划缓存,并对相同冷键执行 single-flight 编译。 + */ +final class BoundedPlanCache implements AutoCloseable { + + private final int maximumEntries; + private final int maximumConcurrentCompilations; + private final long maximumWeightBytes; + private final long ttlNanos; + private final LongSupplier nanoTime; + private final Map entries; + private final ConcurrentHashMap> inFlight = + new ConcurrentHashMap<>(); + private final Semaphore compileSlots; + private final AtomicBoolean closed = new AtomicBoolean(); + private long currentWeightBytes; + + /** + * 创建有界计划缓存。 + * + * @param maximumEntries 最大条目数 + */ + BoundedPlanCache(int maximumEntries) { + this( + maximumEntries, + Math.min(maximumEntries, defaultCompilationConcurrency()), + defaultMaximumWeight(maximumEntries), + Duration.ofMinutes(30), + System::nanoTime + ); + } + + /** + * 创建同时限制缓存容量与冷编译并发的计划缓存。 + * + * @param maximumEntries 最大条目数 + * @param maximumConcurrentCompilations 最大并发冷编译数 + */ + BoundedPlanCache(int maximumEntries, int maximumConcurrentCompilations) { + this( + maximumEntries, + maximumConcurrentCompilations, + defaultMaximumWeight(maximumEntries), + Duration.ofMinutes(30), + System::nanoTime + ); + } + + /** + * 创建同时限制条目、估算权重、存活时间和冷编译并发的计划缓存。 + * + * @param maximumEntries 最大条目数 + * @param maximumConcurrentCompilations 最大并发冷编译数 + * @param maximumWeightBytes 最大估算权重 + * @param timeToLive 条目存活时间 + */ + BoundedPlanCache( + int maximumEntries, + int maximumConcurrentCompilations, + long maximumWeightBytes, + Duration timeToLive + ) { + this( + maximumEntries, + maximumConcurrentCompilations, + maximumWeightBytes, + timeToLive, + System::nanoTime + ); + } + + /** + * 创建使用指定单调时钟的计划缓存,供回绕边界测试使用。 + * + * @param maximumEntries 最大条目数 + * @param maximumConcurrentCompilations 最大并发冷编译数 + * @param maximumWeightBytes 最大估算权重 + * @param timeToLive 条目存活时间 + * @param nanoTime 单调时钟 + */ + BoundedPlanCache( + int maximumEntries, + int maximumConcurrentCompilations, + long maximumWeightBytes, + Duration timeToLive, + LongSupplier nanoTime + ) { + if (maximumEntries <= 0) { + throw new IllegalArgumentException("maximumEntries must be positive"); + } + if (maximumConcurrentCompilations <= 0) { + throw new IllegalArgumentException("maximumConcurrentCompilations must be positive"); + } + if (maximumWeightBytes <= 0) { + throw new IllegalArgumentException("maximumWeightBytes must be positive"); + } + if (timeToLive == null || timeToLive.isZero() || timeToLive.isNegative()) { + throw new IllegalArgumentException("timeToLive must be positive"); + } + this.maximumEntries = maximumEntries; + this.maximumConcurrentCompilations = maximumConcurrentCompilations; + this.maximumWeightBytes = maximumWeightBytes; + this.ttlNanos = saturatingNanos(timeToLive); + this.nanoTime = java.util.Objects.requireNonNull(nanoTime, "nanoTime"); + this.compileSlots = new Semaphore(maximumConcurrentCompilations, true); + this.entries = new LinkedHashMap<>(16, 0.75f, true); + } + + /** + * 命中计划或由单个线程完成冷编译。 + * + * @param key 缓存键 + * @param compiler 冷编译函数 + * @return 编译计划 + */ + FederationSqlPlan getOrCompile(PlanCacheKey key, Supplier compiler) { + return getOrCompileWithStatus(key, compiler, WaitGuard.unbounded()).plan(); + } + + /** + * 命中计划或完成冷编译,并返回是否复用了缓存或并发 single-flight。 + * + * @param key 缓存键 + * @param compiler 冷编译函数 + * @return 缓存查询结果 + */ + LookupResult getOrCompileWithStatus( + PlanCacheKey key, + Supplier compiler + ) { + return getOrCompileWithStatus(key, compiler, WaitGuard.unbounded()); + } + + /** + * 命中计划或完成冷编译,并让 single-flight 与编译槽等待响应查询终态。 + * + * @param key 缓存键 + * @param compiler 冷编译函数 + * @param waitGuard 等待终态检查器 + * @return 缓存查询结果 + */ + LookupResult getOrCompileWithStatus( + PlanCacheKey key, + Supplier compiler, + WaitGuard waitGuard + ) { + return getOrCompileWithStatus(key, compiler, waitGuard, plan -> true); + } + + /** + * 命中计划或完成冷编译,并按调用方当前上下文校验缓存计划。 + * + * @param key 缓存键 + * @param compiler 冷编译函数 + * @param waitGuard 等待终态检查器 + * @param cachedPlanValidator 缓存计划是否仍适用于当前上下文 + * @return 缓存查询结果 + */ + LookupResult getOrCompileWithStatus( + PlanCacheKey key, + Supplier compiler, + WaitGuard waitGuard, + Predicate cachedPlanValidator + ) { + return getOrCompileWithStatus( + key, + compiler, + waitGuard, + cachedPlanValidator, + Instant.MIN + ); + } + + /** + * 命中计划或完成冷编译,并阻止较旧上下文淘汰或覆盖较新的缓存计划。 + * + * @param key 缓存键 + * @param compiler 冷编译函数 + * @param waitGuard 等待终态检查器 + * @param cachedPlanValidator 缓存计划是否仍适用于当前上下文 + * @param contextGeneration 当前上下文的可比较代际 + * @return 缓存查询结果 + */ + LookupResult getOrCompileWithStatus( + PlanCacheKey key, + Supplier compiler, + WaitGuard waitGuard, + Predicate cachedPlanValidator, + Instant contextGeneration + ) { + Instant generation = java.util.Objects.requireNonNull( + contextGeneration, + "contextGeneration" + ); + while (true) { + ensureOpen(); + waitGuard.ensureAllowed(); + CacheEntry cached; + synchronized (entries) { + cached = cachedEntry(key, nanoTime.getAsLong()); + } + if (cached != null && cachedPlanValidator.test(cached.plan())) { + return new LookupResult(cached.plan(), true); + } + if (cached != null) { + synchronized (entries) { + removeEntryIfNotNewer(key, cached.plan(), generation); + } + } + + CompletableFuture existing = inFlight.get(key); + if (existing != null) { + FederationSqlPlan sharedPlan = await(existing, waitGuard); + if (cachedPlanValidator.test(sharedPlan)) { + return new LookupResult(sharedPlan, true); + } + synchronized (entries) { + removeEntryIfNotNewer(key, sharedPlan, generation); + } + awaitInFlightRemoval(key, existing, waitGuard); + continue; + } + + acquireCompileSlot(waitGuard); + CompletableFuture ownFuture = new CompletableFuture<>(); + try { + ensureOpen(); + CacheEntry cachedAfterPermit; + synchronized (entries) { + cachedAfterPermit = cachedEntry(key, nanoTime.getAsLong()); + } + if (cachedAfterPermit != null + && cachedPlanValidator.test(cachedAfterPermit.plan())) { + compileSlots.release(); + return new LookupResult(cachedAfterPermit.plan(), true); + } + if (cachedAfterPermit != null) { + synchronized (entries) { + removeEntryIfNotNewer( + key, + cachedAfterPermit.plan(), + generation + ); + } + } + synchronized (entries) { + // 与 close 使用同一线性化锁,禁止关闭后登记新的编译任务。 + ensureOpen(); + existing = inFlight.putIfAbsent(key, ownFuture); + } + } catch (Throwable throwable) { + compileSlots.release(); + throw throwable; + } + if (existing != null) { + // 相同键只短暂占用配额完成握手,等待期间不阻塞其他冷键。 + compileSlots.release(); + FederationSqlPlan sharedPlan = await(existing, waitGuard); + if (cachedPlanValidator.test(sharedPlan)) { + return new LookupResult(sharedPlan, true); + } + synchronized (entries) { + removeEntryIfNotNewer(key, sharedPlan, generation); + } + awaitInFlightRemoval(key, existing, waitGuard); + continue; + } + try { + waitGuard.ensureAllowed(); + FederationSqlPlan plan = compiler.get(); + waitGuard.ensureAllowed(); + long weight = estimateWeight(plan); + boolean cacheable = weight <= maximumWeightBytes + && cachedPlanValidator.test(plan); + synchronized (entries) { + // 缓存写入、成功发布与 close 共享线性化点,避免关闭后的迟到发布。 + ensureOpen(); + if (cacheable) { + if (putEntry(key, plan, weight, nanoTime.getAsLong(), generation)) { + evictToBounds(); + } + } + ownFuture.complete(plan); + } + return new LookupResult(plan, false); + } catch (Throwable throwable) { + ownFuture.completeExceptionally(throwable); + throw throwable; + } finally { + inFlight.remove(key, ownFuture); + compileSlots.release(); + } + } + } + + private void awaitInFlightRemoval( + PlanCacheKey key, + CompletableFuture completed, + WaitGuard waitGuard + ) { + while (inFlight.get(key) == completed) { + ensureOpen(); + waitGuard.ensureAllowed(); + LockSupport.parkNanos(Math.min( + boundedWaitNanos(waitGuard.remainingNanos()), + TimeUnit.MILLISECONDS.toNanos(1) + )); + if (Thread.interrupted()) { + Thread.currentThread().interrupt(); + throw new FederationSqlException( + FederationSqlErrorCode.SQL_COMPILE_FAILED, + "waiting for a stale in-flight SQL compilation was interrupted" + ); + } + } + } + + /** + * 计划缓存查询结果。 + * + * @param plan 计划 + * @param cacheHit 是否复用缓存或同键 single-flight + */ + record LookupResult(FederationSqlPlan plan, boolean cacheHit) { + } + + private static int defaultCompilationConcurrency() { + return Math.max(1, Math.min(8, Runtime.getRuntime().availableProcessors())); + } + + private static long defaultMaximumWeight(int maximumEntries) { + return Math.max(16L * 1024L * 1024L, Math.min( + 256L * 1024L * 1024L, + maximumEntries * 128L * 1024L + )); + } + + private void acquireCompileSlot(WaitGuard waitGuard) { + while (true) { + ensureOpen(); + waitGuard.ensureAllowed(); + long waitNanos = boundedWaitNanos(waitGuard.remainingNanos()); + try { + if (compileSlots.tryAcquire(waitNanos, TimeUnit.NANOSECONDS)) { + try { + ensureOpen(); + } catch (RuntimeException exception) { + compileSlots.release(); + throw exception; + } + return; + } + } catch (InterruptedException exception) { + Thread.currentThread().interrupt(); + throw new FederationSqlException( + FederationSqlErrorCode.SQL_COMPILE_FAILED, + "waiting for a SQL compile slot was interrupted", + exception + ); + } + } + } + + private void ensureOpen() { + if (closed.get()) { + throw new FederationSqlException( + FederationSqlErrorCode.ENGINE_CLOSED, + "plan cache is closed" + ); + } + } + + private static FederationSqlPlan await( + CompletableFuture future, + WaitGuard waitGuard + ) { + while (true) { + waitGuard.ensureAllowed(); + try { + return future.get( + boundedWaitNanos(waitGuard.remainingNanos()), + TimeUnit.NANOSECONDS + ); + } catch (TimeoutException ignored) { + // 短轮询使同键等待能及时响应取消和统一截止时间。 + } catch (InterruptedException exception) { + Thread.currentThread().interrupt(); + throw new FederationSqlException( + FederationSqlErrorCode.SQL_COMPILE_FAILED, + "waiting for an in-flight SQL compilation was interrupted", + exception + ); + } catch (ExecutionException exception) { + if (exception.getCause() instanceof RuntimeException runtimeException) { + throw runtimeException; + } + throw new CompletionException(exception.getCause()); + } + } + } + + private static long boundedWaitNanos(long remainingNanos) { + return Math.max( + 1L, + Math.min(remainingNanos, TimeUnit.MILLISECONDS.toNanos(50)) + ); + } + + /** + * 冷编译等待检查器。 + */ + interface WaitGuard { + + /** 检查等待是否仍允许继续。 */ + void ensureAllowed(); + + /** + * 返回剩余等待时间。 + * + * @return 剩余纳秒 + */ + long remainingNanos(); + + /** + * 返回无限制检查器。 + * + * @return 无限制检查器 + */ + static WaitGuard unbounded() { + return UnboundedHolder.INSTANCE; + } + + /** 无限制实例持有者。 */ + final class UnboundedHolder { + private static final WaitGuard INSTANCE = new WaitGuard() { + @Override + public void ensureAllowed() { + } + + @Override + public long remainingNanos() { + return Long.MAX_VALUE; + } + }; + + private UnboundedHolder() { + } + } + } + + /** + * 返回当前缓存条目数,供测试与指标桥接读取。 + * + * @return 条目数 + */ + int size() { + synchronized (entries) { + removeExpired(nanoTime.getAsLong()); + return entries.size(); + } + } + + /** + * 返回当前缓存估算权重,供测试和监控读取。 + * + * @return 当前估算权重 + */ + long weightBytes() { + synchronized (entries) { + removeExpired(nanoTime.getAsLong()); + return currentWeightBytes; + } + } + + /** + * 返回当前可用冷编译槽,供关闭竞态测试确认许可没有泄漏。 + * + * @return 可用冷编译槽 + */ + int availableCompileSlots() { + return compileSlots.availablePermits(); + } + + /** + * 精确移除引用已关闭 Runtime 身份的计划。 + * + * @param runtime 已关闭 Runtime + */ + void invalidateRuntime(SourceRuntime runtime) { + invalidateRuntimeIdentity( + runtime.definition().sourceId(), + runtime.definition().revision(), + runtime.sourceChecksum(), + runtime.runtimeFingerprint() + ); + } + + /** + * 精确移除引用指定 Runtime 身份的计划,供生命周期回调与回归测试复用。 + * + * @param sourceId 物理源标识 + * @param revision Definition 版本 + * @param checksum Definition 校验和 + * @param runtimeFingerprint 数据库与驱动指纹 + */ + void invalidateRuntimeIdentity( + com.easyagents.federation.sql.source.SourceId sourceId, + long revision, + String checksum, + String runtimeFingerprint + ) { + synchronized (entries) { + Iterator> iterator = entries.entrySet().iterator(); + while (iterator.hasNext()) { + CacheEntry entry = iterator.next().getValue(); + boolean matches = entry.plan().sourceRuntimeIdentities().stream().anyMatch(identity -> + identity.sourceId().equals(sourceId) + && identity.sourceRevision() == revision + && identity.sourceChecksum().equals(checksum) + && identity.runtimeFingerprint().equals(runtimeFingerprint) + ); + if (matches) { + currentWeightBytes -= entry.weightBytes(); + iterator.remove(); + } + } + } + } + + /** + * 清空计划和进行中的编译记录。 + */ + @Override + public void close() { + synchronized (entries) { + if (!closed.compareAndSet(false, true)) { + return; + } + entries.clear(); + currentWeightBytes = 0; + FederationSqlException failure = new FederationSqlException( + FederationSqlErrorCode.ENGINE_CLOSED, + "plan cache was closed during compilation" + ); + inFlight.values().forEach(future -> future.completeExceptionally(failure)); + inFlight.clear(); + } + } + + private CacheEntry cachedEntry(PlanCacheKey key, long now) { + CacheEntry cached = entries.get(key); + if (cached == null) { + return null; + } + if (isExpired(cached, now)) { + removeEntryIfSame(key, cached.plan()); + return null; + } + return cached; + } + + private void removeEntryIfSame(PlanCacheKey key, FederationSqlPlan plan) { + CacheEntry current = entries.get(key); + if (current != null && current.plan() == plan) { + entries.remove(key); + currentWeightBytes -= current.weightBytes(); + } + } + + private void removeEntryIfNotNewer( + PlanCacheKey key, + FederationSqlPlan plan, + Instant contextGeneration + ) { + CacheEntry current = entries.get(key); + if (current != null + && current.plan() == plan + && !current.contextGeneration().isAfter(contextGeneration)) { + entries.remove(key); + currentWeightBytes -= current.weightBytes(); + } + } + + private boolean putEntry( + PlanCacheKey key, + FederationSqlPlan plan, + long weightBytes, + long now, + Instant contextGeneration + ) { + long statisticsTtlNanos = remainingStatisticsTtlNanos( + plan.statisticsValidUntil(), + Instant.now() + ); + long effectiveTtlNanos = Math.min(ttlNanos, statisticsTtlNanos); + if (effectiveTtlNanos <= 0L) { + return false; + } + CacheEntry current = entries.get(key); + if (current != null && current.contextGeneration().isAfter(contextGeneration)) { + return false; + } + CacheEntry previous = entries.put( + key, + new CacheEntry( + plan, + weightBytes, + now, + effectiveTtlNanos, + contextGeneration + ) + ); + if (previous != null) { + currentWeightBytes -= previous.weightBytes(); + } + currentWeightBytes += weightBytes; + return true; + } + + private void evictToBounds() { + Iterator> iterator = entries.entrySet().iterator(); + while ((entries.size() > maximumEntries || currentWeightBytes > maximumWeightBytes) + && iterator.hasNext()) { + CacheEntry eldest = iterator.next().getValue(); + currentWeightBytes -= eldest.weightBytes(); + iterator.remove(); + } + } + + private void removeExpired(long now) { + Iterator> iterator = entries.entrySet().iterator(); + while (iterator.hasNext()) { + CacheEntry entry = iterator.next().getValue(); + if (isExpired(entry, now)) { + currentWeightBytes -= entry.weightBytes(); + iterator.remove(); + } + } + } + + private static long estimateWeight(FederationSqlPlan plan) { + long[] nodes = {0}; + if (plan.relRoot() != null && plan.relRoot().rel != null) { + new RelVisitor() { + @Override + public void visit(RelNode node, int ordinal, RelNode parent) { + nodes[0]++; + super.visit(node, ordinal, parent); + } + }.go(plan.relRoot().rel); + } + long characters = plan.normalizedSql().length() + plan.executableSql().length(); + for (var fragment : plan.fragments()) { + characters += fragment.executableSql().length(); + } + return 4_096L + + nodes[0] * 1_024L + + characters * 2L + + plan.columns().size() * 256L + + plan.fragments().size() * 512L; + } + + private static long saturatingNanos(Duration duration) { + try { + return duration.toNanos(); + } catch (ArithmeticException ignored) { + return Long.MAX_VALUE; + } + } + + private static long remainingStatisticsTtlNanos(Instant validUntil, Instant now) { + if (validUntil == null || Instant.MAX.equals(validUntil)) { + return Long.MAX_VALUE; + } + if (!validUntil.isAfter(now)) { + return 0L; + } + return saturatingNanos(Duration.between(now, validUntil)); + } + + private static boolean isExpired(CacheEntry entry, long now) { + return entry.ttlNanos() != Long.MAX_VALUE + && now - entry.createdAtNanos() >= entry.ttlNanos(); + } + + private record CacheEntry( + FederationSqlPlan plan, + long weightBytes, + long createdAtNanos, + long ttlNanos, + Instant contextGeneration + ) { + } +} diff --git a/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/runtime/CalciteFederationSqlCompiler.java b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/runtime/CalciteFederationSqlCompiler.java new file mode 100644 index 0000000..e1ba68b --- /dev/null +++ b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/runtime/CalciteFederationSqlCompiler.java @@ -0,0 +1,2157 @@ +package com.easyagents.federation.sql.runtime; + +import com.easyagents.federation.sql.api.FederationSqlErrorCode; +import com.easyagents.federation.sql.api.FederationSqlException; +import com.easyagents.federation.sql.compile.FederationSqlPlan; +import com.easyagents.federation.sql.compile.FederationSqlPolicy; +import com.easyagents.federation.sql.compile.SqlCompileRequest; +import com.easyagents.federation.sql.compile.SqlPolicyContext; +import com.easyagents.federation.sql.execute.FederationColumn; +import com.easyagents.federation.sql.federation.FederationExecutionPolicy; +import com.easyagents.federation.sql.federation.FederationCostEstimate; +import com.easyagents.federation.sql.federation.FederationFragmentPlan; +import com.easyagents.federation.sql.federation.FederationJoinAlgorithm; +import com.easyagents.federation.sql.federation.FederationJoinOptimization; +import com.easyagents.federation.sql.federation.FederationJoinSelectionReason; +import com.easyagents.federation.sql.federation.FederationQueryMode; +import com.easyagents.federation.sql.federation.FederationQueryScopeDefinition; +import com.easyagents.federation.sql.federation.FederationSourceBindingDefinition; +import com.easyagents.federation.sql.federation.FederationSourceRuntimeIdentity; +import com.easyagents.federation.sql.federation.FederationStatisticsSnapshot; +import com.easyagents.federation.sql.federation.FederationStatisticsStatus; +import com.easyagents.federation.sql.federation.FederationTableStatistics; +import com.easyagents.federation.sql.federation.FederationTableStatisticsProvider; +import com.easyagents.federation.sql.source.FederationSchemaDefinition; +import com.easyagents.federation.sql.source.SourceId; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.IdentityHashMap; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.time.Instant; +import java.util.concurrent.atomic.AtomicReference; +import java.util.stream.Collectors; +import org.apache.calcite.adapter.enumerable.EnumerableConvention; +import org.apache.calcite.adapter.enumerable.EnumerableInterpretable; +import org.apache.calcite.adapter.enumerable.EnumerableRel; +import org.apache.calcite.adapter.enumerable.EnumerableRules; +import org.apache.calcite.config.Lex; +import org.apache.calcite.jdbc.CalcitePrepare; +import org.apache.calcite.jdbc.CalciteSchema; +import org.apache.calcite.plan.hep.HepMatchOrder; +import org.apache.calcite.plan.hep.HepPlanner; +import org.apache.calcite.plan.hep.HepProgramBuilder; +import org.apache.calcite.plan.RelOptRule; +import org.apache.calcite.plan.RelOptUtil; +import org.apache.calcite.prepare.RelOptTableImpl; +import org.apache.calcite.rel.RelHomogeneousShuttle; +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.RelRoot; +import org.apache.calcite.rel.RelVisitor; +import org.apache.calcite.rel.core.Aggregate; +import org.apache.calcite.rel.core.AggregateCall; +import org.apache.calcite.rel.core.Filter; +import org.apache.calcite.rel.core.Join; +import org.apache.calcite.rel.core.JoinRelType; +import org.apache.calcite.rel.core.Project; +import org.apache.calcite.rel.core.Sort; +import org.apache.calcite.rel.core.TableScan; +import org.apache.calcite.rel.core.Union; +import org.apache.calcite.rel.core.Values; +import org.apache.calcite.rel.logical.LogicalTableScan; +import org.apache.calcite.rel.rel2sql.RelToSqlConverter; +import org.apache.calcite.rel.rules.CoreRules; +import org.apache.calcite.rel.rules.HyperGraph; +import org.apache.calcite.rel.rules.JoinCommuteRule; +import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.rel.type.RelDataTypeField; +import org.apache.calcite.rex.RexCall; +import org.apache.calcite.rex.RexDynamicParam; +import org.apache.calcite.rex.RexNode; +import org.apache.calcite.rex.RexShuttle; +import org.apache.calcite.rex.RexVisitorImpl; +import org.apache.calcite.runtime.Bindable; +import org.apache.calcite.schema.Schema; +import org.apache.calcite.schema.SchemaPlus; +import org.apache.calcite.schema.Schemas; +import org.apache.calcite.schema.impl.AbstractSchema; +import org.apache.calcite.sql.SqlCall; +import org.apache.calcite.sql.SqlDataTypeSpec; +import org.apache.calcite.sql.SqlDynamicParam; +import org.apache.calcite.sql.SqlIdentifier; +import org.apache.calcite.sql.SqlJoin; +import org.apache.calcite.sql.SqlKind; +import org.apache.calcite.sql.SqlNode; +import org.apache.calcite.sql.SqlOperatorTable; +import org.apache.calcite.sql.SqlOrderBy; +import org.apache.calcite.sql.SqlSelect; +import org.apache.calcite.sql.SqlWith; +import org.apache.calcite.sql.SqlWithItem; +import org.apache.calcite.sql.fun.SqlStdOperatorTable; +import org.apache.calcite.sql.parser.SqlParseException; +import org.apache.calcite.sql.type.SqlTypeUtil; +import org.apache.calcite.sql.type.SqlTypeName; +import org.apache.calcite.sql.util.SqlBasicVisitor; +import org.apache.calcite.sql.util.SqlOperatorTables; +import org.apache.calcite.sql.util.SqlShuttle; +import org.apache.calcite.sql.util.SqlString; +import org.apache.calcite.tools.FrameworkConfig; +import org.apache.calcite.tools.Frameworks; +import org.apache.calcite.tools.Planner; +import org.apache.calcite.tools.Programs; +import org.apache.calcite.tools.RelConversionException; +import org.apache.calcite.tools.ValidationException; + +/** + * 为每次冷编译创建独立 Planner,并直接以 Calcite RelNode 生成单源或联邦计划。 + */ +final class CalciteFederationSqlCompiler { + + private static final List LOCAL_ENUMERABLE_RULES = List.of( + EnumerableRules.ENUMERABLE_TABLE_SCAN_RULE, + EnumerableRules.ENUMERABLE_CALC_RULE, + EnumerableRules.ENUMERABLE_JOIN_RULE, + EnumerableRules.ENUMERABLE_AGGREGATE_RULE, + EnumerableRules.ENUMERABLE_SORT_RULE, + EnumerableRules.ENUMERABLE_LIMIT_SORT_RULE, + EnumerableRules.ENUMERABLE_LIMIT_RULE, + EnumerableRules.ENUMERABLE_UNION_RULE, + EnumerableRules.ENUMERABLE_VALUES_RULE + ); + + private final List policies; + private final String policyFingerprint; + private final boolean crossSourceEnabled; + private final FederationExecutionPolicy enginePolicy; + private final FederationTableStatisticsProvider statisticsProvider; + + /** + * 创建 Calcite 编译器。 + * + * @param policies SQL 策略 + * @param crossSourceEnabled 是否允许联邦执行 + * @param enginePolicy Engine 资源硬上限 + */ + CalciteFederationSqlCompiler( + List policies, + boolean crossSourceEnabled, + FederationExecutionPolicy enginePolicy, + FederationTableStatisticsProvider statisticsProvider + ) { + this.policies = List.copyOf(policies); + this.policyFingerprint = policyFingerprint(this.policies); + this.crossSourceEnabled = crossSourceEnabled; + this.enginePolicy = enginePolicy == null ? FederationExecutionPolicy.basic() : enginePolicy; + this.statisticsProvider = statisticsProvider == null + ? FederationTableStatisticsProvider.none() + : statisticsProvider; + } + + /** + * 创建不注入外部统计的兼容编译器。 + * + * @param policies SQL 策略 + * @param crossSourceEnabled 是否允许联邦执行 + * @param enginePolicy Engine 资源硬上限 + */ + CalciteFederationSqlCompiler( + List policies, + boolean crossSourceEnabled, + FederationExecutionPolicy enginePolicy + ) { + this( + policies, + crossSourceEnabled, + enginePolicy, + FederationTableStatisticsProvider.none() + ); + } + + /** + * 创建使用默认资源上限的兼容编译器。 + * + * @param policies SQL 策略 + * @param crossSourceEnabled 是否允许联邦执行 + */ + CalciteFederationSqlCompiler( + List policies, + boolean crossSourceEnabled + ) { + this(policies, crossSourceEnabled, FederationExecutionPolicy.basic()); + } + + /** + * 使用已构建单源 Runtime 的兼容编译入口。 + * + * @param request 编译请求 + * @param runtime 单源 Runtime + * @param knownSources 旧接口已知源集合;单源范围内不再使用 + * @return 编译计划 + */ + FederationSqlPlan compile( + SqlCompileRequest request, + SourceRuntime runtime, + Set knownSources + ) { + try (FederationQueryScopeSnapshot snapshot = + FederationQueryScopeSnapshot.borrowedSingle(request.queryScope(), runtime)) { + return compile(request, snapshot); + } + } + + /** + * 返回当前 Engine 策略实现与版本的稳定指纹。 + * + * @return 策略指纹 + */ + String policyFingerprint() { + return policyFingerprint; + } + + /** + * 返回当前外部统计快照版本,用于隔离成本计划缓存。 + * + * @return 稳定版本字符串 + */ + FederationStatisticsSnapshot statisticsSnapshot() { + FederationStatisticsSnapshot snapshot = statisticsProvider.snapshot(); + return snapshot == null + ? FederationStatisticsSnapshot.empty() + : snapshot; + } + + /** + * 返回 Engine 与调用方 Scope 收敛后的有效资源策略。 + * + * @param scope 查询范围 + * @return 有效策略 + */ + FederationExecutionPolicy effectivePolicy(FederationQueryScopeDefinition scope) { + return enginePolicy.intersect(scope.executionPolicy()); + } + + /** + * 使用默认 Binding 方言预解析 SQL,确定需要加载的候选 Binding。 + * + *

最终来源仍以校验后的 TableScan 为准;该结果只用于减少无关 Runtime 初始化。

+ * + * @param request 编译请求 + * @return 候选 Binding 名称 + */ + Set discoverBindings(SqlCompileRequest request) { + try { + SqlNode parsed = org.apache.calcite.sql.parser.SqlParser.create( + request.sql(), + org.apache.calcite.sql.parser.SqlParser.config() + .withLex(Lex.ORACLE) + ).parseQuery(); + requireReadOnly(parsed); + Set bindings = new LinkedHashSet<>(); + discoverBindings(parsed, request.queryScope(), Set.of(), bindings); + if (bindings.isEmpty()) { + bindings.add(request.queryScope().defaultBinding()); + } + return Set.copyOf(bindings); + } catch (SqlParseException exception) { + throw new FederationSqlException( + FederationSqlErrorCode.SQL_PARSE_FAILED, + "SQL parsing failed", + exception + ); + } + } + + /** + * 对已编译计划执行当前策略校验;缓存命中和执行前均会调用。 + * + * @param request 原始编译请求 + * @param plan 已编译计划 + */ + void validatePolicies(SqlCompileRequest request, FederationSqlPlan plan) { + SqlPolicyContext context = new SqlPolicyContext( + request, + plan.sqlNode(), + plan.relRoot(), + plan.referencedSources() + ); + for (FederationSqlPolicy policy : policies) { + policy.validate(context); + } + } + + /** + * 编译单源完整下推计划或两源基础联邦计划。 + * + * @param request 编译请求 + * @param snapshot 候选 Runtime 快照 + * @return 编译计划 + */ + FederationSqlPlan compile( + SqlCompileRequest request, + FederationQueryScopeSnapshot snapshot + ) { + return compile(request, snapshot, statisticsSnapshot()); + } + + /** + * 使用调用方已捕获的统计快照编译,确保计划诊断、成本数据与缓存键一致。 + * + * @param request 编译请求 + * @param snapshot 候选 Runtime 快照 + * @param statisticsSnapshot 冻结的统计快照 + * @return 编译计划 + */ + FederationSqlPlan compile( + SqlCompileRequest request, + FederationQueryScopeSnapshot snapshot, + FederationStatisticsSnapshot statisticsSnapshot + ) { + SourceRuntime plannerRuntime = plannerRuntime(request.queryScope(), snapshot); + UnifiedSchema unifiedSchema = buildUnifiedSchema(request.queryScope(), snapshot); + List plannerRules = uniquePhysicalSources(snapshot) == 1 + ? plannerRuntime.adapter().plannerRules() + : List.of(); + Frameworks.ConfigBuilder configBuilder = Frameworks.newConfigBuilder() + .defaultSchema(unifiedSchema.defaultSchema()) + .operatorTable(operatorTable(snapshot)) + .parserConfig(plannerRuntime.adapter().parserConfig(plannerRuntime.dialect())) + .typeSystem(plannerRuntime.adapter().typeSystem()); + if (!plannerRules.isEmpty()) { + configBuilder.programs(Programs.ofRules(plannerRules)); + } + FrameworkConfig config = configBuilder.build(); + + try (Planner planner = Frameworks.getPlanner(config)) { + SqlNode parsed = planner.parse(request.sql()); + requireReadOnly(parsed); + int dynamicParameterCount = countDynamicParameters(parsed); + requireMatchingParameterTypes(request, dynamicParameterCount); + SqlNode resolved = LogicalTableSqlResolver.resolve(parsed, request.queryScope()); + SqlNode typedSql = applyDeclaredParameterTypes(resolved, request, plannerRuntime); + SqlNode validated = planner.validate(typedSql); + RelRoot relRoot = planner.rel(validated); + if (!request.parameterJdbcTypes().isEmpty()) { + relRoot = relRoot.withRel(stripSyntheticParameterCasts(relRoot.rel)); + } + if (!plannerRules.isEmpty()) { + relRoot = relRoot.withRel(planner.transform( + 0, + relRoot.rel.getTraitSet(), + relRoot.rel + )); + } + + RelNode queryRoot = applyStatistics( + relRoot.project(), + request.queryScope(), + snapshot, + statisticsSnapshot + ); + FederationStatisticsMetadata.install(queryRoot.getCluster()); + RelSourceAnalysis initialSourceAnalysis = analyzeSources( + queryRoot, + request.queryScope(), + snapshot + ); + validateCrossSourceJoinConditions(queryRoot, initialSourceAnalysis); + queryRoot = optimizeForFederation(queryRoot); + RelSourceAnalysis sourceAnalysis = analyzeSources( + queryRoot, + request.queryScope(), + snapshot + ); + if (sourceAnalysis.sources(queryRoot).size() > 1) { + queryRoot = optimizeJoinBuildSides( + queryRoot, + sourceAnalysis, + request.queryScope(), + snapshot, + statisticsSnapshot + ); + // Join 交换会生成新的 Join/Project 节点,来源身份需按新树重新计算。 + sourceAnalysis = analyzeSources(queryRoot, request.queryScope(), snapshot); + } + Set referencedSources = sourceAnalysis.sources(queryRoot); + if (referencedSources.isEmpty()) { + referencedSources = Set.of(request.sourceId()); + } + Set statisticsTables = + referencedStatisticsTables( + queryRoot, + request.queryScope(), + snapshot + ); + FederationStatisticsSnapshot.Selection statisticsSelection = + statisticsSnapshot.select(statisticsTables); + FederationExecutionPolicy effectivePolicy = effectivePolicy(request.queryScope()); + if (referencedSources.size() > effectivePolicy.maximumReferencedSources()) { + throw new FederationSqlException( + FederationSqlErrorCode.FEDERATION_RESOURCE_LIMIT_EXCEEDED, + "query references " + referencedSources.size() + + " physical sources but the limit is " + + effectivePolicy.maximumReferencedSources() + ); + } + if (referencedSources.size() == 1) { + return compileSingleSource( + request, + snapshot, + validated, + relRoot, + queryRoot, + dynamicParameterCount, + referencedSources, + sourceAnalysis, + statisticsSnapshot, + statisticsTables, + statisticsSelection + ); + } + if (!crossSourceEnabled) { + throw new FederationSqlException( + FederationSqlErrorCode.CROSS_SOURCE_DISABLED, + "cross-source SQL execution is disabled" + ); + } + return compileFederated( + request, + snapshot, + plannerRuntime, + validated, + relRoot, + queryRoot, + dynamicParameterCount, + referencedSources, + sourceAnalysis, + effectivePolicy, + statisticsSnapshot, + statisticsTables, + statisticsSelection + ); + } catch (FederationSqlException exception) { + throw exception; + } catch (SqlParseException exception) { + throw new FederationSqlException( + FederationSqlErrorCode.SQL_PARSE_FAILED, + "SQL parsing failed", + exception + ); + } catch (ValidationException exception) { + throw new FederationSqlException( + FederationSqlErrorCode.SQL_VALIDATION_FAILED, + "SQL validation failed", + exception + ); + } catch (RelConversionException exception) { + throw new FederationSqlException( + FederationSqlErrorCode.SQL_COMPILE_FAILED, + "SQL relational conversion failed", + exception + ); + } catch (RuntimeException exception) { + throw new FederationSqlException( + FederationSqlErrorCode.SQL_COMPILE_FAILED, + "SQL compilation failed", + exception + ); + } + } + + private FederationSqlPlan compileSingleSource( + SqlCompileRequest request, + FederationQueryScopeSnapshot snapshot, + SqlNode validated, + RelRoot relRoot, + RelNode queryRoot, + int parameterCount, + Set referencedSources, + RelSourceAnalysis sourceAnalysis, + FederationStatisticsSnapshot statisticsSnapshot, + Set statisticsTables, + FederationStatisticsSnapshot.Selection statisticsSelection + ) { + ensureNoEnumerableResidual(queryRoot); + SourceId sourceId = referencedSources.iterator().next(); + String bindingName = sourceAnalysis.bindingForSource(sourceId); + SourceRuntime runtime = snapshot.runtime(bindingName); + SqlString targetSql = toSql(queryRoot, runtime); + List mapping = parameterMapping(targetSql, parameterCount, true); + FederationFragmentPlan fragment = new FederationFragmentPlan( + "fragment-1", + bindingName, + sourceId, + targetSql.getSql(), + mapping, + columns(queryRoot.getRowType()), + estimateCost( + queryRoot, + sourceId, + bindingName, + request.queryScope(), + statisticsSnapshot + ), + pushedDownOperators(queryRoot) + ); + CompiledFederationFragment compiled = new CompiledFederationFragment( + fragment, + queryRoot.getRowType() + ); + List identities = identitiesForBindings( + snapshot, + Set.of(bindingName) + ); + return new DefaultFederationSqlPlan( + request.queryScope(), + FederationQueryMode.SINGLE_SOURCE, + validated.toSqlString(runtime.dialect()).getSql(), + targetSql.getSql(), + validated, + relRoot.withRel(queryRoot), + parameterCount, + request.parameterJdbcTypes(), + mapping, + List.of(fragment), + List.of(), + identities, + columns(relRoot.validatedRowType), + referencedSources, + runtime.compatibility(), + true, + Map.of(fragment.fragmentId(), compiled), + RelOptUtil.toString(queryRoot), + null, + null, + statisticsTables, + statisticsSelection.fingerprint(), + statisticsSelection.validUntil() + ); + } + + private FederationSqlPlan compileFederated( + SqlCompileRequest request, + FederationQueryScopeSnapshot snapshot, + SourceRuntime plannerRuntime, + SqlNode validated, + RelRoot relRoot, + RelNode queryRoot, + int parameterCount, + Set referencedSources, + RelSourceAnalysis sourceAnalysis, + FederationExecutionPolicy effectivePolicy, + FederationStatisticsSnapshot statisticsSnapshot, + Set statisticsTables, + FederationStatisticsSnapshot.Selection statisticsSelection + ) { + List fragmentRoots = maximalSingleSourceFragments(queryRoot, sourceAnalysis); + if (fragmentRoots.size() > effectivePolicy.maximumFragments()) { + throw new FederationSqlException( + FederationSqlErrorCode.FEDERATION_RESOURCE_LIMIT_EXCEEDED, + "query requires " + fragmentRoots.size() + " fragments but the limit is " + + effectivePolicy.maximumFragments() + ); + } + IdentityHashMap fragmentsByRoot = + new IdentityHashMap<>(); + LinkedHashMap compiledById = new LinkedHashMap<>(); + int sequence = 1; + for (RelNode fragmentRoot : fragmentRoots) { + SourceId sourceId = sourceAnalysis.sources(fragmentRoot).iterator().next(); + String bindingName = sourceAnalysis.bindingForSource(sourceId); + SourceRuntime runtime = snapshot.runtime(bindingName); + SqlString fragmentSql = toSql(fragmentRoot, runtime); + List mapping = parameterMapping(fragmentSql, parameterCount, false); + FederationFragmentPlan fragmentPlan = new FederationFragmentPlan( + "fragment-" + sequence++, + bindingName, + sourceId, + fragmentSql.getSql(), + mapping, + columns(fragmentRoot.getRowType()), + estimateCost( + fragmentRoot, + sourceId, + bindingName, + request.queryScope(), + statisticsSnapshot + ), + pushedDownOperators(fragmentRoot) + ); + CompiledFederationFragment compiled = new CompiledFederationFragment( + fragmentPlan, + fragmentRoot.getRowType() + ); + fragmentsByRoot.put(fragmentRoot, compiled); + compiledById.put(fragmentPlan.fragmentId(), compiled); + } + LocalLogicalPlan localPlan = replaceFragments(queryRoot, fragmentsByRoot); + validateLocalOperators(localPlan.root()); + Bindable bindable = compileLocalBindable(localPlan.root()); + List publicFragments = compiledById.values().stream() + .map(CompiledFederationFragment::plan) + .toList(); + List joinOptimizations = joinOptimizations( + queryRoot, + sourceAnalysis, + request.queryScope(), + snapshot, + statisticsSnapshot + ); + Set actualBindings = publicFragments.stream() + .map(FederationFragmentPlan::bindingName) + .collect(Collectors.toCollection(LinkedHashSet::new)); + return new DefaultFederationSqlPlan( + request.queryScope(), + FederationQueryMode.FEDERATED, + validated.toSqlString(plannerRuntime.dialect()).getSql(), + request.sql(), + validated, + relRoot.withRel(queryRoot), + parameterCount, + request.parameterJdbcTypes(), + List.of(), + publicFragments, + joinOptimizations, + identitiesForBindings(snapshot, actualBindings), + columns(relRoot.validatedRowType), + referencedSources, + plannerRuntime.compatibility(), + true, + compiledById, + RelOptUtil.toString(localPlan.root()), + bindable, + localPlan.rootSchema(), + statisticsTables, + statisticsSelection.fingerprint(), + statisticsSelection.validUntil() + ); + } + + private static UnifiedSchema buildUnifiedSchema( + FederationQueryScopeDefinition scope, + FederationQueryScopeSnapshot snapshot + ) { + SchemaPlus root = CalciteSchema.createRootSchema(true, false).plus(); + SchemaPlus defaultSchema = root; + for (Map.Entry entry : snapshot.runtimesByBinding().entrySet()) { + String bindingName = entry.getKey(); + SourceRuntime runtime = entry.getValue(); + FederationSourceBindingDefinition binding = scope.bindings().get(bindingName); + SchemaPlus bindingSchema = root.add(bindingName, new AbstractSchema()); + Map mappings = binding.schemaMappings(); + if (mappings.isEmpty()) { + LinkedHashMap sameName = new LinkedHashMap<>(); + for (FederationSchemaDefinition definition : runtime.definition().schemas()) { + sameName.put(definition.logicalName(), definition.logicalName()); + } + mappings = sameName; + } + SchemaPlus onlySchema = null; + for (Map.Entry mapping : mappings.entrySet()) { + SchemaPlus sourceSchema = runtime.rootSchema() + .getSubSchema(runtime.definition().sourceId().value()); + sourceSchema = sourceSchema == null + ? null + : sourceSchema.getSubSchema(mapping.getValue()); + if (sourceSchema == null) { + throw new FederationSqlException( + FederationSqlErrorCode.INVALID_QUERY_SCOPE, + "binding " + bindingName + " maps an unknown source schema: " + + mapping.getValue() + ); + } + Schema mounted = sourceSchema.unwrap(Schema.class); + onlySchema = bindingSchema.add(mapping.getKey(), mounted); + } + if (bindingName.equals(scope.defaultBinding())) { + defaultSchema = mappings.size() == 1 ? onlySchema : bindingSchema; + } + } + return new UnifiedSchema(defaultSchema); + } + + private static SqlOperatorTable operatorTable(FederationQueryScopeSnapshot snapshot) { + List tables = snapshot.runtimesByBinding().values().stream() + .map(runtime -> runtime.adapter().operatorTable()) + .distinct() + .toList(); + return tables.isEmpty() + ? SqlStdOperatorTable.instance() + : SqlOperatorTables.chain(tables); + } + + private static SourceRuntime plannerRuntime( + FederationQueryScopeDefinition scope, + FederationQueryScopeSnapshot snapshot + ) { + SourceRuntime defaultRuntime = snapshot.runtimesByBinding().get(scope.defaultBinding()); + return defaultRuntime == null + ? snapshot.runtimesByBinding().values().iterator().next() + : defaultRuntime; + } + + private static long uniquePhysicalSources(FederationQueryScopeSnapshot snapshot) { + return snapshot.runtimesByBinding().values().stream() + .map(runtime -> runtime.definition().sourceId()) + .distinct() + .count(); + } + + private static RelSourceAnalysis analyzeSources( + RelNode root, + FederationQueryScopeDefinition scope, + FederationQueryScopeSnapshot snapshot + ) { + IdentityHashMap> sources = new IdentityHashMap<>(); + Map bindingsBySource = new LinkedHashMap<>(); + snapshot.runtimesByBinding().forEach((binding, runtime) -> + bindingsBySource.putIfAbsent(runtime.definition().sourceId(), binding) + ); + analyzeNode(root, scope, snapshot, sources, bindingsBySource); + return new RelSourceAnalysis(sources, bindingsBySource); + } + + private static Set analyzeNode( + RelNode node, + FederationQueryScopeDefinition scope, + FederationQueryScopeSnapshot snapshot, + IdentityHashMap> cache, + Map bindingsBySource + ) { + Set cached = cache.get(node); + if (cached != null) { + return cached; + } + LinkedHashSet found = new LinkedHashSet<>(); + if (node instanceof TableScan scan) { + String bindingName = bindingFromQualifiedName( + scan.getTable().getQualifiedName(), + scope, + snapshot + ); + SourceRuntime runtime = snapshot.runtime(bindingName); + found.add(runtime.definition().sourceId()); + bindingsBySource.putIfAbsent(runtime.definition().sourceId(), bindingName); + } else { + for (RelNode input : node.getInputs()) { + found.addAll(analyzeNode(input, scope, snapshot, cache, bindingsBySource)); + } + } + Set immutable = Set.copyOf(found); + cache.put(node, immutable); + return immutable; + } + + private FederationCostEstimate estimateCost( + RelNode fragmentRoot, + SourceId sourceId, + String bindingName, + FederationQueryScopeDefinition scope, + FederationStatisticsSnapshot statisticsSnapshot + ) { + List scans = new ArrayList<>(); + new RelVisitor() { + @Override + public void visit(RelNode node, int ordinal, RelNode parent) { + if (node instanceof TableScan scan) { + scans.add(scan); + } + super.visit(node, ordinal, parent); + } + }.go(fragmentRoot); + + var metadata = fragmentRoot.getCluster().getMetadataQuery(); + double calciteRows = safeEstimate(metadata.getRowCount(fragmentRoot), 100D); + double scale = 1D; + long providedWidth = 0; + int providedFields = 0; + int totalScanFields = 0; + int missingStatistics = 0; + int partialStatistics = 0; + boolean staleStatistics = false; + Instant oldestCollection = null; + LinkedHashSet sources = new LinkedHashSet<>(); + for (TableScan scan : scans) { + TableReference reference = tableReference(scan, bindingName, scope); + FederationTableStatistics statistics = statisticsSnapshot.statistics( + sourceId, + reference.schema(), + reference.table() + ); + totalScanFields += scan.getRowType().getFieldCount(); + if (statistics == null) { + missingStatistics++; + continue; + } + FederationStatisticsStatus status = statistics.effectiveStatus( + statisticsSnapshot.capturedAt() + ); + if (status == FederationStatisticsStatus.MISSING) { + missingStatistics++; + continue; + } + if (status == FederationStatisticsStatus.STALE) { + staleStatistics = true; + missingStatistics++; + sources.add(statistics.source()); + if (oldestCollection == null + || statistics.collectedAt().isBefore(oldestCollection)) { + oldestCollection = statistics.collectedAt(); + } + // 过期值只用于诊断,不参与行数、行宽或 Join 构建侧决策。 + continue; + } else if (status == FederationStatisticsStatus.PARTIAL) { + partialStatistics++; + } + double defaultTableRows = Math.max( + 1D, + safeEstimate(metadata.getRowCount(scan), 100D) + ); + scale = saturatingMultiply(scale, statistics.estimatedRows() / defaultTableRows); + providedWidth += statistics.averageRowWidthBytes(); + providedFields += scan.getRowType().getFieldCount(); + sources.add(statistics.source()); + if (oldestCollection == null || statistics.collectedAt().isBefore(oldestCollection)) { + oldestCollection = statistics.collectedAt(); + } + } + + double estimatedRows = Math.max(0D, saturatingMultiply(calciteRows, scale)); + Double calciteAverageRowSize = metadata.getAverageRowSize(fragmentRoot); + long rowWidth = calciteAverageRowSize == null || !Double.isFinite(calciteAverageRowSize) + ? Math.max(1L, fragmentRoot.getRowType().getFieldCount() * 16L) + : Math.max(1L, (long) Math.ceil(calciteAverageRowSize)); + if (providedFields > 0 && totalScanFields > 0) { + double projectedRatio = Math.min( + 1D, + (double) fragmentRoot.getRowType().getFieldCount() / totalScanFields + ); + long providedEstimate = Math.max( + 1L, + (long) Math.ceil(providedWidth * projectedRatio) + ); + rowWidth = missingStatistics > 0 + ? Math.max(rowWidth, providedEstimate) + : providedEstimate; + } + double transferBytes = saturatingMultiply(estimatedRows, rowWidth); + boolean statisticsMissing = scans.isEmpty() || missingStatistics > 0 || sources.isEmpty(); + FederationStatisticsStatus statisticsStatus; + if (staleStatistics) { + statisticsStatus = FederationStatisticsStatus.STALE; + } else if (sources.isEmpty()) { + statisticsStatus = FederationStatisticsStatus.MISSING; + } else if (missingStatistics > 0 || partialStatistics > 0) { + statisticsStatus = FederationStatisticsStatus.PARTIAL; + } else { + statisticsStatus = FederationStatisticsStatus.COMPLETE; + } + return new FederationCostEstimate( + estimatedRows, + rowWidth, + transferBytes, + sources.isEmpty() ? "calcite-default" : String.join(",", sources), + statisticsSnapshot.version(), + oldestCollection == null ? Instant.EPOCH : oldestCollection, + statisticsMissing, + statisticsStatus + ); + } + + private RelNode applyStatistics( + RelNode root, + FederationQueryScopeDefinition scope, + FederationQueryScopeSnapshot snapshot, + FederationStatisticsSnapshot statisticsSnapshot + ) { + return root.accept(new RelHomogeneousShuttle() { + @Override + public RelNode visit(TableScan scan) { + if (!(scan instanceof LogicalTableScan)) { + return scan; + } + String bindingName = bindingFromQualifiedName( + scan.getTable().getQualifiedName(), + scope, + snapshot + ); + SourceRuntime runtime = snapshot.runtime(bindingName); + TableReference reference = tableReference(scan, bindingName, scope); + FederationTableStatistics statistics = statisticsSnapshot.statistics( + runtime.definition().sourceId(), + reference.schema(), + reference.table() + ); + FederationStatisticsStatus status = statistics == null + ? FederationStatisticsStatus.MISSING + : statistics.effectiveStatus(statisticsSnapshot.capturedAt()); + if (status == FederationStatisticsStatus.MISSING + || status == FederationStatisticsStatus.STALE) { + return scan; + } + return new FederationStatisticsTableScan( + scan.getCluster(), + scan.getTraitSet(), + scan.getHints(), + scan.getTable(), + statistics, + runtime.definition().sourceId() + ); + } + }); + } + + private static TableReference tableReference( + TableScan scan, + String bindingName, + FederationQueryScopeDefinition scope + ) { + List qualifiedName = scan.getTable().getQualifiedName(); + String table = qualifiedName.isEmpty() + ? "" + : qualifiedName.get(qualifiedName.size() - 1); + String logicalSchema = qualifiedName.size() < 2 + ? "" + : qualifiedName.get(qualifiedName.size() - 2); + FederationSourceBindingDefinition binding = scope.bindings().get(bindingName); + String physicalSchema = binding == null + ? logicalSchema + : binding.schemaMappings().getOrDefault(logicalSchema, logicalSchema); + return new TableReference(physicalSchema, table); + } + + private static Set referencedStatisticsTables( + RelNode root, + FederationQueryScopeDefinition scope, + FederationQueryScopeSnapshot snapshot + ) { + LinkedHashSet tables = new LinkedHashSet<>(); + new RelVisitor() { + @Override + public void visit(RelNode node, int ordinal, RelNode parent) { + if (node instanceof TableScan scan) { + String binding = bindingFromQualifiedName( + scan.getTable().getQualifiedName(), + scope, + snapshot + ); + TableReference reference = tableReference(scan, binding, scope); + tables.add(new FederationStatisticsSnapshot.TableKey( + snapshot.runtime(binding).definition().sourceId(), + reference.schema(), + reference.table() + )); + } + super.visit(node, ordinal, parent); + } + }.go(root); + return Set.copyOf(tables); + } + + private static List pushedDownOperators(RelNode root) { + LinkedHashSet operators = new LinkedHashSet<>(); + new RelVisitor() { + @Override + public void visit(RelNode node, int ordinal, RelNode parent) { + operators.add(node.getRelTypeName()); + super.visit(node, ordinal, parent); + } + }.go(root); + return List.copyOf(operators); + } + + private static double safeEstimate(Double value, double fallback) { + return value == null || !Double.isFinite(value) || value < 0 ? fallback : value; + } + + private static double saturatingMultiply(double left, double right) { + if (left == 0D || right == 0D) { + return 0D; + } + if (!Double.isFinite(left) || !Double.isFinite(right) + || left > Double.MAX_VALUE / right) { + return Double.MAX_VALUE; + } + return left * right; + } + + private RelNode optimizeJoinBuildSides( + RelNode node, + RelSourceAnalysis analysis, + FederationQueryScopeDefinition scope, + FederationQueryScopeSnapshot snapshot, + FederationStatisticsSnapshot statisticsSnapshot + ) { + List rewrittenInputs = node.getInputs().stream() + .map(input -> optimizeJoinBuildSides( + input, + analysis, + scope, + snapshot, + statisticsSnapshot + )) + .toList(); + RelNode rewritten = rewrittenInputs.equals(node.getInputs()) + ? node + : node.copy(node.getTraitSet(), rewrittenInputs); + if (!(node instanceof Join originalJoin) + || !(rewritten instanceof Join rewrittenJoin) + || rewrittenJoin.getJoinType() != JoinRelType.INNER + || rewrittenJoin.analyzeCondition().pairs().isEmpty()) { + return rewritten; + } + Set leftSources = analysis.sources(originalJoin.getLeft()); + Set rightSources = analysis.sources(originalJoin.getRight()); + LinkedHashSet joinedSources = new LinkedHashSet<>(leftSources); + joinedSources.addAll(rightSources); + if (leftSources.isEmpty() || rightSources.isEmpty() || joinedSources.size() <= 1) { + return rewritten; + } + FederationCostEstimate leftCost = estimateJoinInputCost( + rewrittenJoin.getLeft(), + leftSources, + analysis, + scope, + snapshot, + statisticsSnapshot + ); + FederationCostEstimate rightCost = estimateJoinInputCost( + rewrittenJoin.getRight(), + rightSources, + analysis, + scope, + snapshot, + statisticsSnapshot + ); + // EnumerableHashJoin 构建右侧 Hash Table;表级估算可信且收益明确时才交换。 + if (!hasUsableBuildEstimate(leftCost) + || !hasUsableBuildEstimate(rightCost) + || leftCost.estimatedTransferBytes() >= rightCost.estimatedTransferBytes()) { + return rewritten; + } + RelNode swapped = JoinCommuteRule.swap(rewrittenJoin, true); + return swapped == null ? rewritten : swapped; + } + + private List joinOptimizations( + RelNode root, + RelSourceAnalysis analysis, + FederationQueryScopeDefinition scope, + FederationQueryScopeSnapshot snapshot, + FederationStatisticsSnapshot statisticsSnapshot + ) { + List optimizations = new ArrayList<>(); + collectJoinOptimizations( + root, + analysis, + scope, + snapshot, + statisticsSnapshot, + optimizations + ); + return List.copyOf(optimizations); + } + + private void collectJoinOptimizations( + RelNode node, + RelSourceAnalysis analysis, + FederationQueryScopeDefinition scope, + FederationQueryScopeSnapshot snapshot, + FederationStatisticsSnapshot statisticsSnapshot, + List target + ) { + for (RelNode input : node.getInputs()) { + collectJoinOptimizations( + input, + analysis, + scope, + snapshot, + statisticsSnapshot, + target + ); + } + if (node instanceof Join join && !join.analyzeCondition().pairs().isEmpty()) { + appendJoinOptimization( + join, + analysis, + scope, + snapshot, + statisticsSnapshot, + target + ); + } + } + + private void appendJoinOptimization( + Join join, + RelSourceAnalysis analysis, + FederationQueryScopeDefinition scope, + FederationQueryScopeSnapshot snapshot, + FederationStatisticsSnapshot statisticsSnapshot, + List target + ) { + Set leftSources = analysis.sources(join.getLeft()); + Set rightSources = analysis.sources(join.getRight()); + LinkedHashSet joinedSources = new LinkedHashSet<>(leftSources); + joinedSources.addAll(rightSources); + if (leftSources.isEmpty() || rightSources.isEmpty() || joinedSources.size() <= 1) { + return; + } + List leftBindings = bindingsForSources(leftSources, analysis); + List rightBindings = bindingsForSources(rightSources, analysis); + FederationCostEstimate leftCost = estimateJoinInputCost( + join.getLeft(), + leftSources, + analysis, + scope, + snapshot, + statisticsSnapshot + ); + FederationCostEstimate rightCost = estimateJoinInputCost( + join.getRight(), + rightSources, + analysis, + scope, + snapshot, + statisticsSnapshot + ); + FederationJoinSelectionReason reason; + if (join.getJoinType() != JoinRelType.INNER) { + reason = FederationJoinSelectionReason.JOIN_SEMANTICS; + } else if (hasUsableBuildEstimate(leftCost) + && hasUsableBuildEstimate(rightCost)) { + reason = FederationJoinSelectionReason.SMALLER_BUILD_SIDE; + } else { + reason = FederationJoinSelectionReason.INCOMPLETE_STATISTICS; + } + target.add(new FederationJoinOptimization( + target.size() + 1, + leftBindings, + rightBindings, + String.join(" + ", rightBindings), + FederationJoinAlgorithm.HASH_JOIN, + reason, + rightCost.estimatedTransferBytes() + )); + } + + private FederationCostEstimate estimateJoinInputCost( + RelNode input, + Set sources, + RelSourceAnalysis analysis, + FederationQueryScopeDefinition scope, + FederationQueryScopeSnapshot snapshot, + FederationStatisticsSnapshot statisticsSnapshot + ) { + if (sources.size() == 1) { + SourceId sourceId = sources.iterator().next(); + return estimateCost( + input, + sourceId, + analysis.bindingForSource(sourceId), + scope, + statisticsSnapshot + ); + } + List scanCosts = new ArrayList<>(); + new RelVisitor() { + @Override + public void visit(RelNode node, int ordinal, RelNode parent) { + if (node instanceof TableScan scan) { + String binding = bindingFromQualifiedName( + scan.getTable().getQualifiedName(), + scope, + snapshot + ); + scanCosts.add(estimateCost( + scan, + snapshot.runtime(binding).definition().sourceId(), + binding, + scope, + statisticsSnapshot + )); + } + super.visit(node, ordinal, parent); + } + }.go(input); + var metadata = input.getCluster().getMetadataQuery(); + double rows = safeEstimate(metadata.getRowCount(input), 100D); + Double averageSize = metadata.getAverageRowSize(input); + long rowWidth = averageSize == null || !Double.isFinite(averageSize) + ? Math.max(1L, input.getRowType().getFieldCount() * 16L) + : Math.max(1L, (long) Math.ceil(averageSize)); + FederationStatisticsStatus status = aggregateStatisticsStatus(scanCosts); + Instant oldestCollection = scanCosts.stream() + .map(FederationCostEstimate::statisticsCollectedAt) + .filter(value -> !Instant.EPOCH.equals(value)) + .min(Instant::compareTo) + .orElse(Instant.EPOCH); + String source = scanCosts.stream() + .map(FederationCostEstimate::statisticsSource) + .distinct() + .sorted() + .collect(Collectors.joining(",")); + return new FederationCostEstimate( + rows, + rowWidth, + saturatingMultiply(rows, rowWidth), + source.isBlank() ? "calcite-default" : source, + statisticsSnapshot.version(), + oldestCollection, + scanCosts.stream().anyMatch(FederationCostEstimate::statisticsMissing), + status, + scanCosts.stream().allMatch(FederationCostEstimate::estimateAvailable) + ); + } + + /** + * 判断成本是否包含可用于 Hash Join 构建侧选择的表级估算。 + * + *

数据库仅提供表行数和平均行宽时状态为 PARTIAL,但只要实际扫描表均有 + * 有效估算,就足以比较两侧搬运量。缺失或过期统计仍保留稳定输入顺序。

+ * + * @param cost 输入成本 + * @return 可安全比较搬运量时为 true + */ + private static boolean hasUsableBuildEstimate(FederationCostEstimate cost) { + return cost.estimateAvailable() + && !cost.statisticsMissing() + && (cost.statisticsStatus() == FederationStatisticsStatus.COMPLETE + || cost.statisticsStatus() == FederationStatisticsStatus.PARTIAL); + } + + private static FederationStatisticsStatus aggregateStatisticsStatus( + List costs + ) { + if (costs.isEmpty()) { + return FederationStatisticsStatus.MISSING; + } + if (costs.stream().anyMatch(cost -> + cost.statisticsStatus() == FederationStatisticsStatus.STALE)) { + return FederationStatisticsStatus.STALE; + } + boolean allComplete = costs.stream().allMatch(cost -> + cost.statisticsStatus() == FederationStatisticsStatus.COMPLETE); + if (allComplete) { + return FederationStatisticsStatus.COMPLETE; + } + boolean allMissing = costs.stream().allMatch(cost -> + cost.statisticsStatus() == FederationStatisticsStatus.MISSING); + return allMissing + ? FederationStatisticsStatus.MISSING + : FederationStatisticsStatus.PARTIAL; + } + + private static List bindingsForSources( + Set sources, + RelSourceAnalysis analysis + ) { + return sources.stream() + .map(analysis::bindingForSource) + .distinct() + .sorted(String.CASE_INSENSITIVE_ORDER) + .toList(); + } + + private static String bindingFromQualifiedName( + List qualifiedName, + FederationQueryScopeDefinition scope, + FederationQueryScopeSnapshot snapshot + ) { + for (String component : qualifiedName) { + for (String binding : snapshot.runtimesByBinding().keySet()) { + if (binding.equals(component) || binding.equalsIgnoreCase(component)) { + return binding; + } + } + } + if (snapshot.runtimesByBinding().size() == 1) { + return snapshot.runtimesByBinding().keySet().iterator().next(); + } + if (snapshot.runtimesByBinding().containsKey(scope.defaultBinding())) { + return scope.defaultBinding(); + } + throw new FederationSqlException( + FederationSqlErrorCode.INVALID_QUERY_SCOPE, + "validated table cannot be assigned to a query binding: " + qualifiedName + ); + } + + private static List maximalSingleSourceFragments( + RelNode root, + RelSourceAnalysis analysis + ) { + List fragments = new ArrayList<>(); + collectFragments(root, Set.of(), analysis, fragments); + return List.copyOf(fragments); + } + + private static void collectFragments( + RelNode node, + Set parentSources, + RelSourceAnalysis analysis, + List fragments + ) { + Set current = analysis.sources(node); + if (current.size() == 1 && parentSources.size() > 1) { + fragments.add(node); + return; + } + for (RelNode input : node.getInputs()) { + collectFragments(input, current, analysis, fragments); + } + } + + private static LocalLogicalPlan replaceFragments( + RelNode root, + IdentityHashMap fragments + ) { + SchemaPlus rootSchema = CalciteSchema.createRootSchema(true, false).plus(); + SchemaPlus fragmentSchema = rootSchema.add("__fragments", new AbstractSchema()); + IdentityHashMap replacements = new IdentityHashMap<>(); + fragments.forEach((fragmentRoot, compiled) -> { + FederationFragmentTable table = new FederationFragmentTable( + compiled.plan().fragmentId(), + compiled.rowType(), + compiled.plan().costEstimate().estimatedRows() + ); + fragmentSchema.add(compiled.plan().fragmentId(), table); + RelOptTableImpl relOptTable = RelOptTableImpl.create( + null, + compiled.rowType(), + List.of("__fragments", compiled.plan().fragmentId()), + table, + requestedClass -> Schemas.getTableExpression( + fragmentSchema, + compiled.plan().fragmentId(), + table, + requestedClass + ) + ); + replacements.put( + fragmentRoot, + LogicalTableScan.create(fragmentRoot.getCluster(), relOptTable, List.of()) + ); + }); + RelNode replaced = root.accept(new RelHomogeneousShuttle() { + @Override + public RelNode visit(RelNode node) { + RelNode replacement = replacements.get(node); + return replacement == null ? super.visit(node) : replacement; + } + }); + return new LocalLogicalPlan(replaced, rootSchema); + } + + private static void validateLocalOperators(RelNode root) { + AtomicReference failure = new AtomicReference<>(); + new RelVisitor() { + @Override + public void visit(RelNode node, int ordinal, RelNode parent) { + if (failure.get() != null) { + return; + } + if (hasTemporalPrecisionBeyondMilliseconds(node.getRowType())) { + failure.set(unsupported( + "local federation supports TIME/TIMESTAMP precision up to 3 digits" + )); + return; + } + if (node instanceof Join join) { + org.apache.calcite.rel.core.JoinInfo joinInfo = join.analyzeCondition(); + if ((join.getJoinType() != JoinRelType.INNER + && join.getJoinType() != JoinRelType.LEFT) + || joinInfo.pairs().isEmpty()) { + failure.set(unsupported("only equi INNER and LEFT JOIN are supported")); + return; + } + if (hasCharacterJoinKey(join, joinInfo)) { + failure.set(unsupported( + "character join keys require an explicit cross-source collation adapter" + )); + return; + } + if (hasCharacterComparison(join.getCondition())) { + failure.set(unsupported( + "local character comparisons require an explicit " + + "cross-source collation adapter" + )); + return; + } + } else if (node instanceof Union union) { + if (!union.all) { + failure.set(unsupported("only UNION ALL is supported")); + return; + } + } else if (node instanceof Aggregate aggregate) { + if (hasCharacterGroupingOrAggregate(aggregate)) { + failure.set(unsupported( + "local character grouping and aggregates require an explicit " + + "cross-source collation adapter" + )); + return; + } + if (aggregate.getGroupSets().size() != 1) { + failure.set(unsupported("GROUPING SETS, ROLLUP and CUBE are not supported")); + return; + } + for (AggregateCall call : aggregate.getAggCallList()) { + SqlKind kind = call.getAggregation().getKind(); + if (call.isDistinct() || !(kind == SqlKind.COUNT + || kind == SqlKind.SUM + || kind == SqlKind.SUM0 + || kind == SqlKind.MIN + || kind == SqlKind.MAX + || kind == SqlKind.AVG)) { + failure.set(unsupported( + "aggregate is not supported: " + call.getAggregation().getName() + )); + return; + } + } + } else if (node instanceof Sort sort) { + if (hasCharacterSortKey(sort)) { + failure.set(unsupported( + "local character sorting requires an explicit " + + "cross-source collation adapter" + )); + return; + } + } else if (node instanceof Filter filter) { + if (hasCharacterComparison(filter.getCondition())) { + failure.set(unsupported( + "local character comparisons require an explicit " + + "cross-source collation adapter" + )); + return; + } + } else if (node instanceof Project project) { + if (project.getProjects().stream() + .anyMatch(CalciteFederationSqlCompiler::hasCharacterComparison)) { + failure.set(unsupported( + "local character comparisons require an explicit " + + "cross-source collation adapter" + )); + return; + } + } else if (!(node instanceof Project) + && !(node instanceof Filter) + && !(node instanceof Values) + && !(node instanceof TableScan)) { + failure.set(unsupported( + "local relational operator is not supported: " + node.getRelTypeName() + )); + return; + } + super.visit(node, ordinal, parent); + } + }.go(root); + if (failure.get() != null) { + throw failure.get(); + } + } + + /** + * 在优化器改写前校验跨源 Join 条件,避免 WHERE 残余谓词被合并进 Join 后误判。 + * + * @param root 优化前关系树 + * @param analysis 关系树来源分析 + * @throws FederationSqlException 跨源 Join 使用非纯等值 ON 条件时抛出 + */ + private static void validateCrossSourceJoinConditions( + RelNode root, + RelSourceAnalysis analysis + ) { + AtomicReference failure = new AtomicReference<>(); + new RelVisitor() { + @Override + public void visit(RelNode node, int ordinal, RelNode parent) { + if (failure.get() != null) { + return; + } + if (node instanceof Join join) { + Set leftSources = analysis.sources(join.getLeft()); + Set rightSources = analysis.sources(join.getRight()); + LinkedHashSet joinedSources = + new LinkedHashSet<>(leftSources); + joinedSources.addAll(rightSources); + boolean crossesSources = !leftSources.isEmpty() + && !rightSources.isEmpty() + && joinedSources.size() > 1; + if (crossesSources && !join.analyzeCondition().isEqui()) { + failure.set(unsupported( + "cross-source JOIN ON supports equality predicates only" + )); + return; + } + } + super.visit(node, ordinal, parent); + } + }.go(root); + if (failure.get() != null) { + throw failure.get(); + } + } + + /** + * 判断本地 Enumerable 是否会截断时间字段的亚毫秒精度。 + * + * @param rowType 本地算子行类型 + * @return 是否存在超过毫秒的时间精度 + */ + private static boolean hasTemporalPrecisionBeyondMilliseconds(RelDataType rowType) { + return rowType.getFieldList().stream().anyMatch(field -> { + RelDataType type = field.getType(); + return switch (type.getSqlTypeName()) { + case TIME, TIME_WITH_LOCAL_TIME_ZONE, TIME_TZ, + TIMESTAMP, TIMESTAMP_WITH_LOCAL_TIME_ZONE, TIMESTAMP_TZ -> + type.getPrecision() > 3; + default -> false; + }; + }); + } + + private static boolean hasCharacterJoinKey( + Join join, + org.apache.calcite.rel.core.JoinInfo joinInfo + ) { + for (int index = 0; index < joinInfo.leftKeys.size(); index++) { + RelDataType leftType = join.getLeft().getRowType().getFieldList() + .get(joinInfo.leftKeys.getInt(index)).getType(); + RelDataType rightType = join.getRight().getRowType().getFieldList() + .get(joinInfo.rightKeys.getInt(index)).getType(); + if (SqlTypeUtil.isCharacter(leftType) || SqlTypeUtil.isCharacter(rightType)) { + return true; + } + } + return false; + } + + private static boolean hasCharacterComparison(RexNode expression) { + AtomicReference found = new AtomicReference<>(false); + expression.accept(new RexVisitorImpl(true) { + @Override + public Void visitCall(RexCall call) { + if (isCharacterSemanticOperation(call.getKind()) + && call.getOperands().stream() + .anyMatch(operand -> SqlTypeUtil.isCharacter(operand.getType()))) { + found.set(true); + return null; + } + return super.visitCall(call); + } + }); + return found.get(); + } + + private static boolean isCharacterSemanticOperation(SqlKind kind) { + return SqlKind.COMPARISON.contains(kind) + || kind == SqlKind.LIKE + || kind == SqlKind.RLIKE + || kind == SqlKind.SIMILAR + || kind == SqlKind.POSIX_REGEX_CASE_SENSITIVE + || kind == SqlKind.POSIX_REGEX_CASE_INSENSITIVE + || kind == SqlKind.SEARCH + || kind == SqlKind.IN + || kind == SqlKind.NOT_IN + || kind == SqlKind.BETWEEN + || kind == SqlKind.STARTS_WITH + || kind == SqlKind.ENDS_WITH + || kind == SqlKind.CONTAINS_SUBSTR; + } + + private static boolean hasCharacterSortKey(Sort sort) { + return sort.getCollation().getFieldCollations().stream() + .map(field -> sort.getInput().getRowType().getFieldList() + .get(field.getFieldIndex()).getType()) + .anyMatch(SqlTypeUtil::isCharacter); + } + + private static boolean hasCharacterGroupingOrAggregate(Aggregate aggregate) { + List fields = aggregate.getInput().getRowType().getFieldList(); + if (aggregate.getGroupSet().asList().stream() + .map(index -> fields.get(index).getType()) + .anyMatch(SqlTypeUtil::isCharacter)) { + return true; + } + return aggregate.getAggCallList().stream().anyMatch(call -> + (call.getAggregation().getKind() == SqlKind.MIN + || call.getAggregation().getKind() == SqlKind.MAX) + && call.getArgList().stream() + .map(index -> fields.get(index).getType()) + .anyMatch(SqlTypeUtil::isCharacter) + ); + } + + private static RelNode optimizeForFederation(RelNode root) { + HepPlanner normalizationPlanner = new HepPlanner( + new HepProgramBuilder() + .addRuleInstance(CoreRules.FILTER_INTO_JOIN) + .addRuleInstance(CoreRules.PROJECT_JOIN_TRANSPOSE) + .addRuleInstance(CoreRules.FILTER_PROJECT_TRANSPOSE) + .addRuleInstance(CoreRules.FILTER_MERGE) + .addRuleInstance(CoreRules.PROJECT_MERGE) + .addRuleInstance(CoreRules.PROJECT_REMOVE) + .build() + ); + normalizationPlanner.setRoot(root); + RelNode normalized = normalizationPlanner.findBestExp(); + if (RelOptUtil.countJoins(normalized) < 2 || containsNonInnerJoin(normalized)) { + return normalized; + } + + if (supportsStatisticsDrivenJoinSearch(normalized)) { + RelNode optimized = optimizeWithHyperGraph(normalized); + if (optimized != null) { + return optimized; + } + } + return optimizeWithMultiJoin(normalized); + } + + /** + * 判断计划是否适合有界的统计驱动 Join 搜索。 + * + *

首版仅处理三个不同物理源、两个简单 INNER 等值 Join,避免搜索空间失控, + * 统计缺失时则沿用稳定的基础规则。

+ * + * @param root 规范化计划 + * @return 是否允许启用 DPHyp 搜索 + */ + private static boolean supportsStatisticsDrivenJoinSearch(RelNode root) { + List scans = new ArrayList<>(); + AtomicReference eligible = new AtomicReference<>(true); + new RelVisitor() { + @Override + public void visit(RelNode node, int ordinal, RelNode parent) { + if (node instanceof Join join + && (join.getJoinType() != JoinRelType.INNER + || !join.analyzeCondition().isEqui() + || join.analyzeCondition().pairs().isEmpty())) { + eligible.set(false); + return; + } + if (node instanceof TableScan scan) { + if (!(scan instanceof FederationStatisticsTableScan statisticsScan) + || statisticsScan.statistics().status() + != FederationStatisticsStatus.COMPLETE) { + eligible.set(false); + return; + } + scans.add(statisticsScan); + } + super.visit(node, ordinal, parent); + } + }.go(root); + return eligible.get() + && RelOptUtil.countJoins(root) == 2 + && scans.size() == 3 + && scans.stream().map(FederationStatisticsTableScan::sourceId) + .collect(Collectors.toSet()).size() == 3; + } + + /** + * 使用 Calcite DPHyp 在受控三表范围内按累计联邦成本选择结合顺序。 + * + * @param root 规范化计划 + * @return 优化计划;规则无法完整消解 HyperGraph 时返回 null + */ + private static RelNode optimizeWithHyperGraph(RelNode root) { + HepPlanner graphPlanner = new HepPlanner( + new HepProgramBuilder() + .addMatchOrder(HepMatchOrder.BOTTOM_UP) + .addRuleInstance(CoreRules.JOIN_TO_HYPER_GRAPH) + .build() + ); + graphPlanner.setRoot(root); + RelNode graph = graphPlanner.findBestExp(); + if (!containsHyperGraph(graph)) { + return null; + } + HepPlanner orderPlanner = new HepPlanner( + new HepProgramBuilder() + .addRuleInstance(CoreRules.HYPER_GRAPH_OPTIMIZE) + .build() + ); + orderPlanner.setRoot(graph); + RelNode optimized = orderPlanner.findBestExp(); + return containsHyperGraph(optimized) ? null : optimized; + } + + /** + * 使用基础 MultiJoin 规则生成稳定的 bushy 计划。 + * + * @param root 规范化计划 + * @return Join 重排后的计划 + */ + private static RelNode optimizeWithMultiJoin(RelNode root) { + HepPlanner joinPlanner = new HepPlanner( + new HepProgramBuilder() + .addMatchOrder(HepMatchOrder.BOTTOM_UP) + .addRuleInstance(CoreRules.JOIN_TO_MULTI_JOIN) + .build() + ); + joinPlanner.setRoot(root); + RelNode joinGraph = joinPlanner.findBestExp(); + HepPlanner joinOrderPlanner = new HepPlanner( + new HepProgramBuilder() + .addRuleInstance(CoreRules.MULTI_JOIN_OPTIMIZE_BUSHY) + .build() + ); + joinOrderPlanner.setRoot(joinGraph); + return joinOrderPlanner.findBestExp(); + } + + /** + * 判断计划中是否仍含 DPHyp 的临时 HyperGraph 节点。 + * + * @param root 计划根节点 + * @return 是否包含临时节点 + */ + private static boolean containsHyperGraph(RelNode root) { + AtomicReference found = new AtomicReference<>(false); + new RelVisitor() { + @Override + public void visit(RelNode node, int ordinal, RelNode parent) { + if (node instanceof HyperGraph) { + found.set(true); + return; + } + super.visit(node, ordinal, parent); + } + }.go(root); + return found.get(); + } + + /** + * 判断计划是否包含不能安全改变结合顺序的外连接。 + * + * @param root 关系计划根节点 + * @return 是否包含非 INNER Join + */ + private static boolean containsNonInnerJoin(RelNode root) { + AtomicReference found = new AtomicReference<>(false); + new RelVisitor() { + @Override + public void visit(RelNode node, int ordinal, RelNode parent) { + if (node instanceof Join join && join.getJoinType() != JoinRelType.INNER) { + found.set(true); + return; + } + super.visit(node, ordinal, parent); + } + }.go(root); + return found.get(); + } + + private static Bindable compileLocalBindable(RelNode localLogical) { + HepPlanner normalizationPlanner = new HepPlanner( + new HepProgramBuilder() + .addRuleInstance(CoreRules.FILTER_TO_CALC) + .addRuleInstance(CoreRules.PROJECT_TO_CALC) + .addRuleInstance(CoreRules.FILTER_CALC_MERGE) + .addRuleInstance(CoreRules.PROJECT_CALC_MERGE) + .addRuleInstance(CoreRules.CALC_MERGE) + .build() + ); + normalizationPlanner.setRoot(localLogical); + RelNode normalized = normalizationPlanner.findBestExp(); + RelNode converted = Programs.ofRules(LOCAL_ENUMERABLE_RULES).run( + normalized.getCluster().getPlanner(), + normalized, + normalized.getTraitSet().replace(EnumerableConvention.INSTANCE), + List.of(), + List.of() + ); + RelNode guarded = addLocalBudgetGuards(converted); + if (!(guarded instanceof EnumerableRel enumerable)) { + throw unsupported( + "Calcite could not convert the local plan to Enumerable convention" + ); + } + return EnumerableInterpretable.toBindable( + Map.of(), + (CalcitePrepare.SparkHandler) null, + enumerable, + EnumerableRel.Prefer.ARRAY + ); + } + + private static RelNode addLocalBudgetGuards(RelNode node) { + List inputs = node.getInputs(); + List guardedInputs = inputs.stream() + .map(CalciteFederationSqlCompiler::addLocalBudgetGuards) + .toList(); + RelNode guardedNode = inputs.equals(guardedInputs) + ? node + : node.copy(node.getTraitSet(), guardedInputs); + if (guardedNode instanceof Join + || guardedNode instanceof Aggregate + || guardedNode instanceof Sort + || guardedNode instanceof Union + || guardedNode instanceof org.apache.calcite.rel.core.Calc) { + return new EnumerableFederationBudgetRel( + guardedNode, + guardedNode.getRelTypeName() + ); + } + return guardedNode; + } + + private static FederationSqlException unsupported(String message) { + return new FederationSqlException( + FederationSqlErrorCode.FEDERATION_OPERATOR_UNSUPPORTED, + message + ); + } + + private static SqlString toSql(RelNode relNode, SourceRuntime runtime) { + return new RelToSqlConverter(runtime.dialect()) + .visitRoot(relNode) + .asStatement() + .toSqlString(runtime.dialect()); + } + + private static List identitiesForBindings( + FederationQueryScopeSnapshot snapshot, + Set bindings + ) { + return snapshot.identities().stream() + .filter(identity -> bindings.contains(identity.bindingName())) + .toList(); + } + + private static void discoverBindings( + SqlNode node, + FederationQueryScopeDefinition scope, + Set visibleCteNames, + Set bindings + ) { + if (node == null) { + return; + } + if (node instanceof SqlWith with) { + Set withScope = new HashSet<>(visibleCteNames); + for (SqlNode entry : with.withList) { + SqlWithItem item = (SqlWithItem) entry; + Set itemScope = new HashSet<>(withScope); + if (item.recursive != null && item.recursive.booleanValue()) { + // 只有递归 CTE 的查询体可以引用自身。 + itemScope.add(normalizeCteName(item.name)); + } + discoverBindings(item.query, scope, itemScope, bindings); + withScope.add(normalizeCteName(item.name)); + } + discoverBindings(with.body, scope, withScope, bindings); + return; + } + if (node instanceof SqlSelect select) { + collectFromBindings(select.getFrom(), scope, visibleCteNames, bindings); + } + if (node instanceof SqlCall call) { + for (SqlNode operand : call.getOperandList()) { + discoverBindings(operand, scope, visibleCteNames, bindings); + } + return; + } + if (node instanceof org.apache.calcite.sql.SqlNodeList list) { + for (SqlNode child : list) { + discoverBindings(child, scope, visibleCteNames, bindings); + } + } + } + + private static String normalizeCteName(SqlIdentifier identifier) { + // 预解析使用 Lex.ORACLE:未引号名称已转大写,引号名称保留原始大小写。 + return identifier.getSimple(); + } + + private static void collectFromBindings( + SqlNode from, + FederationQueryScopeDefinition scope, + Set visibleCteNames, + Set bindings + ) { + if (from == null || from instanceof SqlSelect) { + return; + } + if (from instanceof SqlIdentifier identifier) { + if (identifier.names.size() == 1 + && visibleCteNames.contains(normalizeCteName(identifier))) { + return; + } + if (identifier.names.size() == 1) { + String logicalBinding = LogicalTableSqlResolver.bindingForShortName( + identifier.getSimple(), + scope + ); + bindings.add(logicalBinding == null ? scope.defaultBinding() : logicalBinding); + } else if (identifier.names.size() >= 3) { + String first = identifier.names.get(0); + scope.bindings().keySet().stream() + .filter(binding -> identifier.isComponentQuoted(0) + ? binding.equals(first) + : binding.equalsIgnoreCase(first)) + .findFirst() + .ifPresentOrElse(bindings::add, () -> bindings.add(scope.defaultBinding())); + } else { + bindings.add(scope.defaultBinding()); + } + return; + } + if (from instanceof SqlJoin join) { + collectFromBindings(join.getLeft(), scope, visibleCteNames, bindings); + collectFromBindings(join.getRight(), scope, visibleCteNames, bindings); + return; + } + if (from instanceof SqlCall call && (call.getKind() == SqlKind.AS + || call.getKind() == SqlKind.LATERAL + || call.getKind() == SqlKind.TABLESAMPLE + || call.getKind() == SqlKind.SNAPSHOT)) { + List operands = call.getOperandList(); + if (!operands.isEmpty()) { + collectFromBindings(operands.get(0), scope, visibleCteNames, bindings); + } + } + } + + private static void requireReadOnly(SqlNode parsed) { + if (!SqlKind.QUERY.contains(parsed.getKind())) { + throw new FederationSqlException( + FederationSqlErrorCode.SQL_NOT_READ_ONLY, + "only one read-only query is allowed" + ); + } + } + + private static int countDynamicParameters(SqlNode node) { + Set parameterIndexes = new HashSet<>(); + node.accept(new SqlBasicVisitor() { + @Override + public Void visit(SqlDynamicParam parameter) { + parameterIndexes.add(parameter.getIndex()); + return null; + } + }); + return parameterIndexes.size(); + } + + private static void requireMatchingParameterTypes( + SqlCompileRequest request, + int dynamicParameterCount + ) { + if (!request.parameterJdbcTypes().isEmpty() + && request.parameterJdbcTypes().size() != dynamicParameterCount) { + throw new FederationSqlException( + FederationSqlErrorCode.PARAMETER_COUNT_MISMATCH, + "SQL expects " + dynamicParameterCount + " parameters but received " + + request.parameterJdbcTypes().size() + " types" + ); + } + } + + private static SqlNode applyDeclaredParameterTypes( + SqlNode parsed, + SqlCompileRequest request, + SourceRuntime runtime + ) { + if (request.parameterJdbcTypes().isEmpty()) { + return parsed; + } + Set paginationParameters = paginationParameterIndexes(parsed); + return parsed.accept(new SqlShuttle() { + @Override + public SqlNode visit(SqlDynamicParam parameter) { + int jdbcType = request.parameterJdbcTypes().get(parameter.getIndex()); + SqlDataTypeSpec typeSpec = runtime.adapter().parameterTypeSpec( + jdbcType, + parameter.getParserPosition() + ); + if (typeSpec == null) { + throw new FederationSqlException( + FederationSqlErrorCode.INVALID_ARGUMENT, + "JDBC parameter type " + jdbcType + " has no scalar Calcite mapping" + ); + } + if (!paginationParameters.contains(parameter.getIndex())) { + return SqlStdOperatorTable.CAST.createCall( + parameter.getParserPosition(), + parameter, + typeSpec + ); + } + return new TypedSqlDynamicParam( + parameter.getIndex(), + parameter.getParserPosition(), + typeSpec + ); + } + }); + } + + private static Set paginationParameterIndexes(SqlNode parsed) { + Set indexes = new HashSet<>(); + parsed.accept(new SqlBasicVisitor() { + @Override + public Void visit(SqlCall call) { + if (call instanceof SqlSelect select) { + addDynamicParameterIndex(select.getOffset(), indexes); + addDynamicParameterIndex(select.getFetch(), indexes); + } else if (call instanceof SqlOrderBy orderBy) { + addDynamicParameterIndex(orderBy.offset, indexes); + addDynamicParameterIndex(orderBy.fetch, indexes); + } + return super.visit(call); + } + }); + return Set.copyOf(indexes); + } + + private static void addDynamicParameterIndex(SqlNode node, Set indexes) { + if (node instanceof SqlDynamicParam parameter) { + indexes.add(parameter.getIndex()); + } + } + + private static RelNode stripSyntheticParameterCasts(RelNode root) { + RexShuttle castStripper = new RexShuttle() { + @Override + public RexNode visitCall(RexCall call) { + if (call.getKind() == SqlKind.CAST + && call.getOperands().size() == 1 + && call.getOperands().get(0) instanceof RexDynamicParam) { + return call.getOperands().get(0); + } + return super.visitCall(call); + } + }; + return root.accept(new RelHomogeneousShuttle() { + @Override + public RelNode visit(RelNode node) { + RelNode withRewrittenInputs = super.visit(node); + return withRewrittenInputs.accept(castStripper); + } + }); + } + + private static String policyFingerprint(List policies) { + StringBuilder fingerprint = new StringBuilder(); + fingerprint.append(policies.size()).append(':'); + for (FederationSqlPolicy policy : policies) { + String version = policy.version(); + if (version == null || version.isBlank()) { + throw new IllegalArgumentException( + "federation SQL policy version must not be blank" + ); + } + appendFingerprintField(fingerprint, policy.getClass().getName()); + appendFingerprintField(fingerprint, version); + } + return fingerprint.toString(); + } + + private static void appendFingerprintField(StringBuilder target, String value) { + target.append(value.length()).append(':').append(value); + } + + private static List parameterMapping( + SqlString targetSql, + int parameterCount, + boolean singleSource + ) { + List mapping = targetSql.getDynamicParameters(); + if ((mapping == null || mapping.isEmpty()) && parameterCount > 0 && singleSource) { + List identity = new ArrayList<>(parameterCount); + for (int index = 0; index < parameterCount; index++) { + identity.add(index); + } + return List.copyOf(identity); + } + if (mapping == null) { + return List.of(); + } + for (Integer sourceIndex : mapping) { + if (sourceIndex == null || sourceIndex < 0 || sourceIndex >= parameterCount) { + throw new FederationSqlException( + FederationSqlErrorCode.SQL_COMPILE_FAILED, + "Calcite returned an invalid dynamic parameter mapping" + ); + } + } + return List.copyOf(mapping); + } + + private static List columns(RelDataType rowType) { + List columns = new ArrayList<>(); + for (RelDataTypeField field : rowType.getFieldList()) { + SqlTypeName sqlTypeName = field.getType().getSqlTypeName(); + int jdbcType = switch (sqlTypeName) { + case TIME_TZ -> java.sql.Types.TIME_WITH_TIMEZONE; + case TIMESTAMP_TZ -> java.sql.Types.TIMESTAMP_WITH_TIMEZONE; + default -> sqlTypeName.getJdbcOrdinal(); + }; + columns.add(new FederationColumn( + field.getIndex() + 1, + field.getName(), + jdbcType, + sqlTypeName.getName(), + field.getType().isNullable() + )); + } + return List.copyOf(columns); + } + + private static void ensureNoEnumerableResidual(RelNode root) { + AtomicReference enumerableNode = new AtomicReference<>(); + new RelVisitor() { + @Override + public void visit(RelNode node, int ordinal, RelNode parent) { + if (node.getConvention() == EnumerableConvention.INSTANCE) { + enumerableNode.compareAndSet(null, node); + return; + } + super.visit(node, ordinal, parent); + } + }.go(root); + RelNode residual = enumerableNode.get(); + if (residual != null) { + throw new FederationSqlException( + FederationSqlErrorCode.SQL_NOT_FULLY_PUSHDOWN, + "single-source plan contains an Enumerable residual: " + + residual.getRelTypeName() + ); + } + } + + private record UnifiedSchema(SchemaPlus defaultSchema) { + } + + private record LocalLogicalPlan(RelNode root, SchemaPlus rootSchema) { + } + + private record TableReference(String schema, String table) { + } + + private record RelSourceAnalysis( + IdentityHashMap> sourcesByNode, + Map bindingsBySource + ) { + + private Set sources(RelNode node) { + return sourcesByNode.getOrDefault(node, Set.of()); + } + + private String bindingForSource(SourceId sourceId) { + String binding = bindingsBySource.get(sourceId); + if (binding == null) { + throw new FederationSqlException( + FederationSqlErrorCode.INVALID_QUERY_SCOPE, + "no query binding resolved for source: " + sourceId + ); + } + return binding; + } + } +} diff --git a/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/runtime/CalciteSqlCompleter.java b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/runtime/CalciteSqlCompleter.java new file mode 100644 index 0000000..d1c8c84 --- /dev/null +++ b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/runtime/CalciteSqlCompleter.java @@ -0,0 +1,423 @@ +package com.easyagents.federation.sql.runtime; + +import com.easyagents.federation.sql.api.FederationSqlErrorCode; +import com.easyagents.federation.sql.api.FederationSqlException; +import com.easyagents.federation.sql.api.SqlCompletionItem; +import com.easyagents.federation.sql.api.SqlCompletionKind; +import com.easyagents.federation.sql.api.SqlCompletionRequest; +import com.easyagents.federation.sql.api.SqlCompletionResult; +import com.easyagents.federation.sql.federation.FederationLogicalTableDefinition; +import com.easyagents.federation.sql.federation.FederationQueryScopeDefinition; +import com.easyagents.federation.sql.federation.FederationSourceBindingDefinition; +import com.easyagents.federation.sql.source.FederationSchemaDefinition; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Properties; +import org.apache.calcite.config.CalciteConnectionConfigImpl; +import org.apache.calcite.config.CalciteConnectionProperty; +import org.apache.calcite.jdbc.CalciteSchema; +import org.apache.calcite.jdbc.JavaTypeFactoryImpl; +import org.apache.calcite.prepare.CalciteCatalogReader; +import org.apache.calcite.schema.Schema; +import org.apache.calcite.schema.SchemaPlus; +import org.apache.calcite.schema.Table; +import org.apache.calcite.schema.impl.AbstractSchema; +import org.apache.calcite.sql.SqlFunction; +import org.apache.calcite.sql.SqlOperatorTable; +import org.apache.calcite.sql.advise.SqlAdvisor; +import org.apache.calcite.sql.advise.SqlAdvisorValidator; +import org.apache.calcite.sql.fun.SqlStdOperatorTable; +import org.apache.calcite.sql.parser.SqlParser; +import org.apache.calcite.sql.util.SqlOperatorTables; +import org.apache.calcite.sql.validate.SqlMoniker; +import org.apache.calcite.sql.validate.SqlMonikerImpl; +import org.apache.calcite.sql.validate.SqlMonikerType; +import org.apache.calcite.sql.validate.SqlValidator; + +/** + * 基于 Calcite Advisor 的请求级 SQL 补全器。 + */ +final class CalciteSqlCompleter { + + /** + * 在当前 Runtime 快照内生成上下文补全候选。 + * + * @param request 补全请求 + * @param snapshot 查询范围 Runtime 快照 + * @return 替换区间与候选列表 + */ + SqlCompletionResult complete( + SqlCompletionRequest request, + FederationQueryScopeSnapshot snapshot + ) { + try { + CompletionCatalog catalog = buildCatalog(request.queryScope(), snapshot); + SourceRuntime plannerRuntime = plannerRuntime(request.queryScope(), snapshot); + SqlParser.Config parserConfig = plannerRuntime.adapter() + .parserConfig(plannerRuntime.dialect()); + JavaTypeFactoryImpl typeFactory = new JavaTypeFactoryImpl( + plannerRuntime.adapter().typeSystem() + ); + CalciteCatalogReader catalogReader = new CalciteCatalogReader( + CalciteSchema.from(catalog.root()), + catalog.defaultPath(), + typeFactory, + connectionConfig(parserConfig) + ); + SqlOperatorTable operatorTable = operatorTable(snapshot); + SqlAdvisorValidator validator = new SqlAdvisorValidator( + operatorTable, + catalogReader, + typeFactory, + SqlValidator.Config.DEFAULT + .withConformance(parserConfig.conformance()) + .withIdentifierExpansion(true) + .withLenientOperatorLookup(true) + ); + SqlAdvisor advisor = new SqlAdvisor(validator, parserConfig); + String[] replaced = {""}; + List hints = new ArrayList<>(advisor.getCompletionHints( + request.sql(), + request.cursorOffset(), + replaced + )); + // Advisor 在语句首部仅有半截关键字时可能没有候选,直接复用当前 + // Calcite Parser 的期望 Token 补齐,避免维护一份易漂移的关键字表。 + hints.addAll(statementStartHints( + advisor, + request.sql(), + request.cursorOffset(), + replaced[0] + )); + // SqlAdvisor 对末尾三段 Schema 路径可能不返回候选,继续使用同一 Calcite + // CatalogReader 补齐该路径,避免实现第二套 SQL 元数据目录。 + hints.addAll(qualifiedCatalogHints( + catalogReader, + request.sql(), + request.cursorOffset(), + replaced[0] + )); + int replaceStart = request.cursorOffset() - replaced[0].length(); + return new SqlCompletionResult( + replaceStart, + request.cursorOffset(), + completionItems(advisor, operatorTable, hints, replaced[0]) + ); + } catch (FederationSqlException exception) { + throw exception; + } catch (RuntimeException exception) { + throw new FederationSqlException( + FederationSqlErrorCode.SQL_COMPLETION_FAILED, + "Calcite SQL completion failed", + exception + ); + } + } + + private static CompletionCatalog buildCatalog( + FederationQueryScopeDefinition scope, + FederationQueryScopeSnapshot snapshot + ) { + return scope.logicalTables().isEmpty() + ? buildPhysicalCatalog(scope, snapshot) + : buildLogicalCatalog(scope, snapshot); + } + + private static CompletionCatalog buildLogicalCatalog( + FederationQueryScopeDefinition scope, + FederationQueryScopeSnapshot snapshot + ) { + SchemaPlus root = CalciteSchema.createRootSchema(true, false).plus(); + Map bindingSchemas = new LinkedHashMap<>(); + Map logicalSchemas = new LinkedHashMap<>(); + for (FederationLogicalTableDefinition definition : scope.logicalTables()) { + SourceRuntime runtime = snapshot.runtime(definition.bindingName()); + Table sourceTable = sourceTable(scope, runtime, definition); + SchemaPlus bindingSchema = bindingSchemas.computeIfAbsent( + definition.bindingName(), + name -> root.add(name, new AbstractSchema()) + ); + String schemaKey = definition.bindingName() + '\u0000' + definition.schemaName(); + SchemaPlus logicalSchema = logicalSchemas.computeIfAbsent( + schemaKey, + ignored -> bindingSchema.add(definition.schemaName(), new AbstractSchema()) + ); + // 逻辑目录只挂载显式授权别名,避免 Advisor 泄露同 Schema 的其他物理表。 + root.add(definition.logicalName(), sourceTable); + logicalSchema.add(definition.logicalName(), sourceTable); + } + return new CompletionCatalog(root, List.of()); + } + + private static CompletionCatalog buildPhysicalCatalog( + FederationQueryScopeDefinition scope, + FederationQueryScopeSnapshot snapshot + ) { + SchemaPlus root = CalciteSchema.createRootSchema(true, false).plus(); + List defaultPath = List.of(); + for (Map.Entry entry : snapshot.runtimesByBinding().entrySet()) { + String bindingName = entry.getKey(); + SourceRuntime runtime = entry.getValue(); + FederationSourceBindingDefinition binding = scope.bindings().get(bindingName); + SchemaPlus bindingSchema = root.add(bindingName, new AbstractSchema()); + Map mappings = schemaMappings(binding, runtime); + for (Map.Entry mapping : mappings.entrySet()) { + SchemaPlus sourceSchema = sourceSchema(runtime, mapping.getValue()); + Schema mounted = sourceSchema.unwrap(Schema.class); + bindingSchema.add(mapping.getKey(), mounted); + } + if (bindingName.equals(scope.defaultBinding())) { + defaultPath = mappings.size() == 1 + ? List.of(bindingName, mappings.keySet().iterator().next()) + : List.of(bindingName); + } + } + return new CompletionCatalog(root, defaultPath); + } + + private static Table sourceTable( + FederationQueryScopeDefinition scope, + SourceRuntime runtime, + FederationLogicalTableDefinition definition + ) { + FederationSourceBindingDefinition binding = scope.bindings() + .get(definition.bindingName()); + String physicalSchema = mappedSchema(binding, definition.schemaName()); + SchemaPlus schema = sourceSchema(runtime, physicalSchema); + Table table = schema.getTable(definition.sourceTableName()); + if (table == null) { + String actualName = schema.getTableNames().stream() + .filter(name -> name.equalsIgnoreCase(definition.sourceTableName())) + .findFirst() + .orElse(null); + table = actualName == null ? null : schema.getTable(actualName); + } + if (table == null) { + throw new FederationSqlException( + FederationSqlErrorCode.INVALID_QUERY_SCOPE, + "logical table maps an unknown source table: " + definition.logicalName() + ); + } + return table; + } + + private static SchemaPlus sourceSchema(SourceRuntime runtime, String schemaName) { + SchemaPlus sourceRoot = runtime.rootSchema() + .getSubSchema(runtime.definition().sourceId().value()); + SchemaPlus sourceSchema = sourceRoot == null ? null : sourceRoot.getSubSchema(schemaName); + if (sourceSchema == null && sourceRoot != null) { + String actualName = sourceRoot.getSubSchemaNames().stream() + .filter(name -> name.equalsIgnoreCase(schemaName)) + .findFirst() + .orElse(null); + sourceSchema = actualName == null ? null : sourceRoot.getSubSchema(actualName); + } + if (sourceSchema == null) { + throw new FederationSqlException( + FederationSqlErrorCode.INVALID_QUERY_SCOPE, + "query scope maps an unknown source schema: " + schemaName + ); + } + return sourceSchema; + } + + private static String mappedSchema( + FederationSourceBindingDefinition binding, + String logicalSchema + ) { + if (binding == null || binding.schemaMappings().isEmpty()) { + return logicalSchema; + } + return binding.schemaMappings().entrySet().stream() + .filter(entry -> entry.getKey().equalsIgnoreCase(logicalSchema)) + .map(Map.Entry::getValue) + .findFirst() + .orElseThrow(() -> new FederationSqlException( + FederationSqlErrorCode.INVALID_QUERY_SCOPE, + "logical table references an unknown mapped schema: " + logicalSchema + )); + } + + private static Map schemaMappings( + FederationSourceBindingDefinition binding, + SourceRuntime runtime + ) { + if (binding != null && !binding.schemaMappings().isEmpty()) { + return binding.schemaMappings(); + } + LinkedHashMap mappings = new LinkedHashMap<>(); + for (FederationSchemaDefinition definition : runtime.definition().schemas()) { + mappings.put(definition.logicalName(), definition.logicalName()); + } + return mappings; + } + + private static CalciteConnectionConfigImpl connectionConfig( + SqlParser.Config parserConfig + ) { + return new CalciteConnectionConfigImpl(new Properties()) + .set( + CalciteConnectionProperty.CASE_SENSITIVE, + Boolean.toString(parserConfig.caseSensitive()) + ) + .set(CalciteConnectionProperty.QUOTING, parserConfig.quoting().name()) + .set( + CalciteConnectionProperty.UNQUOTED_CASING, + parserConfig.unquotedCasing().name() + ) + .set( + CalciteConnectionProperty.QUOTED_CASING, + parserConfig.quotedCasing().name() + ) + .set( + CalciteConnectionProperty.CONFORMANCE, + parserConfig.conformance().toString() + ); + } + + private static SqlOperatorTable operatorTable(FederationQueryScopeSnapshot snapshot) { + List adapterTables = snapshot.runtimesByBinding().values().stream() + .map(runtime -> runtime.adapter().operatorTable()) + .distinct() + .toList(); + List tables = new ArrayList<>(adapterTables.size() + 1); + // Adapter 只声明厂商扩展;标准 SQL 函数始终由 Calcite 标准表提供。 + tables.add(SqlStdOperatorTable.instance()); + tables.addAll(adapterTables); + return SqlOperatorTables.chain(tables); + } + + private static SourceRuntime plannerRuntime( + FederationQueryScopeDefinition scope, + FederationQueryScopeSnapshot snapshot + ) { + SourceRuntime runtime = snapshot.runtimesByBinding().get(scope.defaultBinding()); + return runtime == null + ? snapshot.runtimesByBinding().values().iterator().next() + : runtime; + } + + private static List completionItems( + SqlAdvisor advisor, + SqlOperatorTable operatorTable, + List hints, + String replacedWord + ) { + LinkedHashMap distinct = new LinkedHashMap<>(); + for (SqlMoniker hint : hints) { + List qualifiedName = hint.getFullyQualifiedNames(); + String label = qualifiedName.isEmpty() + ? hint.id() + : qualifiedName.get(qualifiedName.size() - 1); + String insertText = advisor.getReplacement(hint, replacedWord); + SqlCompletionKind kind = completionKind( + hint.getType(), + label, + operatorTable + ); + SqlCompletionItem item = new SqlCompletionItem( + label, + insertText, + kind, + qualifiedName + ); + distinct.putIfAbsent(kind + "\u0000" + insertText + "\u0000" + qualifiedName, item); + } + return new ArrayList<>(distinct.values()); + } + + private static List qualifiedCatalogHints( + CalciteCatalogReader catalogReader, + String sql, + int cursorOffset, + String replacedWord + ) { + int tokenStart = cursorOffset; + while (tokenStart > 0) { + char current = sql.charAt(tokenStart - 1); + if (current != '.' && !Character.isJavaIdentifierPart(current)) { + break; + } + tokenStart--; + } + String token = sql.substring(tokenStart, cursorOffset); + int lastDot = token.lastIndexOf('.'); + if (lastDot <= 0) { + return List.of(); + } + String qualifier = token.substring(0, lastDot); + List path = List.of(qualifier.split("\\.")); + return catalogReader.getAllSchemaObjectNames(path).stream() + .filter(moniker -> { + List names = moniker.getFullyQualifiedNames(); + String name = names.isEmpty() ? moniker.id() : names.get(names.size() - 1); + return replacedWord.isEmpty() + || name.regionMatches(true, 0, replacedWord, 0, replacedWord.length()); + }) + .toList(); + } + + private static List statementStartHints( + SqlAdvisor advisor, + String sql, + int cursorOffset, + String replacedWord + ) { + int wordStart = cursorOffset - replacedWord.length(); + if (wordStart < 0 || !sql.substring(0, wordStart).isBlank()) { + return List.of(); + } + return advisor.getReservedAndKeyWords().stream() + .filter(keyword -> keyword.regionMatches( + true, + 0, + replacedWord, + 0, + replacedWord.length() + )) + .map(keyword -> (SqlMoniker) new SqlMonikerImpl( + List.of(keyword), + SqlMonikerType.KEYWORD + )) + .distinct() + .toList(); + } + + private static SqlCompletionKind completionKind( + SqlMonikerType type, + String label, + SqlOperatorTable operatorTable + ) { + // SqlAdvisor 会把 COUNT 等可直接输入的标准函数标记为 KEYWORD,继续以 + // Calcite OperatorTable 校正分类,避免前端维护函数名称清单。 + if (type == SqlMonikerType.FUNCTION || operatorTable.getOperatorList().stream() + .anyMatch(operator -> operator instanceof SqlFunction + && operator.getName().equalsIgnoreCase(label))) { + return SqlCompletionKind.FUNCTION; + } + if (type == SqlMonikerType.KEYWORD) { + return SqlCompletionKind.KEYWORD; + } + if (type == SqlMonikerType.TABLE) { + return SqlCompletionKind.TABLE; + } + if (type == SqlMonikerType.VIEW) { + return SqlCompletionKind.VIEW; + } + if (type == SqlMonikerType.SCHEMA) { + return SqlCompletionKind.SCHEMA; + } + if (type == SqlMonikerType.CATALOG) { + return SqlCompletionKind.CATALOG; + } + if (type == SqlMonikerType.COLUMN) { + return SqlCompletionKind.COLUMN; + } + return SqlCompletionKind.OTHER; + } + + private record CompletionCatalog(SchemaPlus root, List defaultPath) { + } +} diff --git a/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/runtime/CancellationAwareFederationCursor.java b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/runtime/CancellationAwareFederationCursor.java new file mode 100644 index 0000000..f0ac472 --- /dev/null +++ b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/runtime/CancellationAwareFederationCursor.java @@ -0,0 +1,26 @@ +package com.easyagents.federation.sql.runtime; + +import com.easyagents.federation.sql.api.FederationSqlErrorCode; + +/** + * Core 内部用于在主动取消并关闭游标前定稿指标的回调。 + */ +interface CancellationAwareFederationCursor { + + /** + * 标记查询已收到主动取消请求。 + */ + void markCancelled(); + + /** + * 标记查询因统一执行时限结束。 + */ + void markTimedOut(); + + /** + * 标记查询失败的稳定错误码。 + * + * @param errorCode 错误码 + */ + void markFailed(FederationSqlErrorCode errorCode); +} diff --git a/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/runtime/CompiledFederationFragment.java b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/runtime/CompiledFederationFragment.java new file mode 100644 index 0000000..d703db9 --- /dev/null +++ b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/runtime/CompiledFederationFragment.java @@ -0,0 +1,16 @@ +package com.easyagents.federation.sql.runtime; + +import com.easyagents.federation.sql.federation.FederationFragmentPlan; +import org.apache.calcite.rel.type.RelDataType; + +/** + * Core 节点本地保存的分片执行元数据。 + * + * @param plan 公共分片计划 + * @param rowType Calcite 输出行类型 + */ +record CompiledFederationFragment( + FederationFragmentPlan plan, + RelDataType rowType +) { +} diff --git a/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/runtime/CompositeQueryResources.java b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/runtime/CompositeQueryResources.java new file mode 100644 index 0000000..23a19dd --- /dev/null +++ b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/runtime/CompositeQueryResources.java @@ -0,0 +1,54 @@ +package com.easyagents.federation.sql.runtime; + +import com.easyagents.federation.sql.api.FederationSqlErrorCode; +import com.easyagents.federation.sql.api.FederationSqlException; +import java.util.List; +import java.util.concurrent.atomic.AtomicBoolean; + +/** + * 按逆序幂等关闭准入许可与 Runtime lease 的资源组合。 + */ +final class CompositeQueryResources implements AutoCloseable { + + private final List resources; + private final AtomicBoolean closed = new AtomicBoolean(); + + /** + * 创建资源组合。 + * + * @param resources 按获取顺序排列的资源 + */ + CompositeQueryResources(AutoCloseable... resources) { + this.resources = List.of(resources); + } + + /** + * 逆序关闭资源并保留全部失败原因。 + */ + @Override + public void close() { + if (!closed.compareAndSet(false, true)) { + return; + } + FederationSqlException failure = null; + for (int index = resources.size() - 1; index >= 0; index--) { + try { + resources.get(index).close(); + } catch (Exception exception) { + FederationSqlException wrapped = new FederationSqlException( + FederationSqlErrorCode.RESOURCE_CLOSE_FAILED, + "failed to close query resource", + exception + ); + if (failure == null) { + failure = wrapped; + } else { + failure.addSuppressed(wrapped); + } + } + } + if (failure != null) { + throw failure; + } + } +} diff --git a/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/runtime/DefaultFederationSourceManager.java b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/runtime/DefaultFederationSourceManager.java new file mode 100644 index 0000000..a7f4b0e --- /dev/null +++ b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/runtime/DefaultFederationSourceManager.java @@ -0,0 +1,846 @@ +package com.easyagents.federation.sql.runtime; + +import com.easyagents.federation.sql.adapter.AdapterDialectContext; +import com.easyagents.federation.sql.adapter.AdapterHints; +import com.easyagents.federation.sql.adapter.AdapterSchemaContext; +import com.easyagents.federation.sql.adapter.FederationSqlAdapterProvider; +import com.easyagents.federation.sql.adapter.FederationSqlAdapterRegistry; +import com.easyagents.federation.sql.api.FederationSqlErrorCode; +import com.easyagents.federation.sql.api.FederationSqlException; +import com.easyagents.federation.sql.source.ActiveSourceState; +import com.easyagents.federation.sql.source.FederationDataSourceHandle; +import com.easyagents.federation.sql.source.FederationDataSourceResolver; +import com.easyagents.federation.sql.source.FederationSchemaDefinition; +import com.easyagents.federation.sql.source.FederationSourceDefinition; +import com.easyagents.federation.sql.source.FederationSourceManager; +import com.easyagents.federation.sql.source.FederationSourceState; +import com.easyagents.federation.sql.source.FederationSourceStateProvider; +import com.easyagents.federation.sql.source.FederationSourceView; +import com.easyagents.federation.sql.source.PreparedSourceRuntime; +import com.easyagents.federation.sql.source.SourceApplyOptions; +import com.easyagents.federation.sql.source.SourceApplyResult; +import com.easyagents.federation.sql.source.SourceApplyStatus; +import com.easyagents.federation.sql.source.SourceId; +import com.easyagents.federation.sql.source.SourceProbeResult; +import com.easyagents.federation.sql.source.SourceRemoveResult; +import com.easyagents.federation.sql.source.SourceRuntimeStatus; +import com.easyagents.federation.sql.source.SourceSnapshotResult; +import com.easyagents.federation.sql.source.SourceStateSubscription; +import com.easyagents.federation.sql.source.SourceTombstone; +import java.sql.Connection; +import java.sql.DatabaseMetaData; +import java.sql.SQLException; +import java.util.Collection; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.function.Consumer; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReentrantReadWriteLock; +import org.apache.calcite.jdbc.CalciteSchema; +import org.apache.calcite.schema.Schema; +import org.apache.calcite.schema.SchemaPlus; +import org.apache.calcite.schema.impl.AbstractSchema; +import org.apache.calcite.sql.SqlDialect; + +/** + * 以 revision、single-flight 初始化和 Runtime lease 管理数据源的默认实现。 + */ +public final class DefaultFederationSourceManager implements FederationSourceManager, AutoCloseable { + + private final FederationDataSourceResolver resolver; + private final FederationSqlAdapterRegistry adapters; + private final FederationSourceStateProvider stateProvider; + private final Map slots = new ConcurrentHashMap<>(); + private final Set preparedRuntimes = ConcurrentHashMap.newKeySet(); + private final Set retiringRuntimes = ConcurrentHashMap.newKeySet(); + private final List> runtimeClosedListeners = + new CopyOnWriteArrayList<>(); + private final Set pendingHandleClosures = + ConcurrentHashMap.newKeySet(); + private final Object sourceCatalogLock = new Object(); + private volatile SourceCatalogSnapshot sourceCatalogSnapshot = new SourceCatalogSnapshot(0, Set.of()); + private final ReentrantReadWriteLock lifecycleLock = new ReentrantReadWriteLock(); + private final Lock operationLock = lifecycleLock.readLock(); + private final Lock closeLock = lifecycleLock.writeLock(); + private final SourceStateSubscription subscription; + private final AtomicBoolean closed = new AtomicBoolean(); + private final AtomicBoolean subscriptionClosed = new AtomicBoolean(); + private final Object subscriptionCloseLock = new Object(); + + /** + * 创建数据源管理器并恢复共享快照、订阅变更提示。 + * + * @param resolver DataSource 解析器 + * @param adapters Adapter 注册表 + * @param stateProvider 共享状态 Provider + */ + public DefaultFederationSourceManager( + FederationDataSourceResolver resolver, + FederationSqlAdapterRegistry adapters, + FederationSourceStateProvider stateProvider + ) { + this.resolver = resolver; + this.adapters = adapters; + this.stateProvider = stateProvider; + applySnapshot(stateProvider.loadSnapshot()); + this.subscription = stateProvider.subscribe(this::applySharedState); + } + + /** + * 使用 Resolver 探测数据源。 + * + * @param definition 数据源定义 + * @return 探测结果 + */ + @Override + public SourceProbeResult probe(FederationSourceDefinition definition) { + operationLock.lock(); + try { + ensureOpen(); + return probeLocked(definition, resolver.resolve(definition)); + } finally { + operationLock.unlock(); + } + } + + /** + * 使用临时句柄探测并确定性关闭全部临时资源。 + * + * @param definition 数据源定义 + * @param temporaryHandle 临时句柄 + * @return 探测结果 + */ + @Override + public SourceProbeResult probe( + FederationSourceDefinition definition, + FederationDataSourceHandle temporaryHandle + ) { + operationLock.lock(); + try { + return probeLocked(definition, temporaryHandle); + } finally { + operationLock.unlock(); + } + } + + private SourceProbeResult probeLocked( + FederationSourceDefinition definition, + FederationDataSourceHandle temporaryHandle + ) { + SourceRuntime runtime = null; + boolean buildAttempted = false; + try { + ensureOpen(); + buildAttempted = true; + runtime = buildRuntime(definition, temporaryHandle); + return new SourceProbeResult( + definition.sourceId(), + true, + temporaryHandle.fingerprint(), + runtime.compatibility().diagnostic() + ); + } catch (RuntimeException exception) { + if (!buildAttempted) { + closeHandleAndSuppress(temporaryHandle, exception); + } + throw exception; + } finally { + if (runtime != null) { + forceCloseRuntime(runtime); + } + } + } + + /** + * 应用 Definition,并可选择立即预热。 + * + * @param definition 数据源定义 + * @param options 应用策略 + * @return 应用结果 + */ + @Override + public SourceApplyResult apply(FederationSourceDefinition definition, SourceApplyOptions options) { + operationLock.lock(); + try { + ensureOpen(); + SourceApplyResult result = applyState(ActiveSourceState.of(definition)); + if (options.prewarm() && result.status() != SourceApplyStatus.IGNORED_STALE + && result.status() != SourceApplyStatus.CONFLICT) { + ensureReadyInternal(definition.sourceId(), definition.revision()); + } + return result; + } finally { + operationLock.unlock(); + } + } + + /** + * 构建与共享 Slot 隔离的 Runtime,供调用方在外部状态 CAS 前验证全部本地资源。 + * + * @param definition 即将发布的数据源 Definition + * @return 预构建 Runtime + */ + @Override + public PreparedSourceRuntime prepare(FederationSourceDefinition definition) { + operationLock.lock(); + try { + ensureOpen(); + FederationDataSourceHandle handle = resolver.resolve(definition); + SourceRuntime runtime = buildRuntime(definition, handle); + PreparedRuntime prepared = new PreparedRuntime(runtime); + preparedRuntimes.add(prepared); + return prepared; + } finally { + operationLock.unlock(); + } + } + + /** + * 原子接管并发布由当前管理器创建的预构建 Runtime。 + * + * @param prepared 预构建 Runtime + * @return Definition 应用结果 + */ + @Override + public SourceApplyResult commit(PreparedSourceRuntime prepared) { + operationLock.lock(); + SourceRuntime candidate = null; + boolean installed = false; + try { + ensureOpen(); + if (!(prepared instanceof PreparedRuntime preparedRuntime) + || !preparedRuntimes.contains(preparedRuntime)) { + throw new IllegalArgumentException( + "prepared runtime was not created by this source manager or is no longer open" + ); + } + candidate = preparedRuntime.take(); + preparedRuntimes.remove(preparedRuntime); + PreparedCommitOutcome outcome = commitPreparedRuntime(candidate); + installed = outcome.installed(); + return outcome.result(); + } finally { + if (candidate != null && !installed) { + forceCloseRuntime(candidate); + } + operationLock.unlock(); + } + } + + /** + * 应用共享状态快照并按结果分类计数。 + * + * @param states 状态集合 + * @return 应用统计 + */ + @Override + public SourceSnapshotResult applySnapshot(Collection states) { + operationLock.lock(); + try { + ensureOpen(); + int applied = 0; + int idempotent = 0; + int stale = 0; + int conflicts = 0; + if (states == null) { + return new SourceSnapshotResult(0, 0, 0, 0); + } + for (FederationSourceState state : states) { + SourceApplyResult result = applyState(state); + switch (result.status()) { + case APPLIED -> applied++; + case IDEMPOTENT -> idempotent++; + case IGNORED_STALE -> stale++; + case CONFLICT -> conflicts++; + } + } + return new SourceSnapshotResult(applied, idempotent, stale, conflicts); + } finally { + operationLock.unlock(); + } + } + + /** + * 应用删除墓碑。 + * + * @param tombstone 删除墓碑 + * @return 删除结果 + */ + @Override + public SourceRemoveResult remove(SourceTombstone tombstone) { + operationLock.lock(); + try { + ensureOpen(); + return new SourceRemoveResult(applyState(tombstone)); + } finally { + operationLock.unlock(); + } + } + + /** + * 返回节点本地数据源视图。 + * + * @param sourceId 数据源标识 + * @return 可选视图 + */ + @Override + public Optional view(SourceId sourceId) { + operationLock.lock(); + try { + ensureOpen(); + SourceSlot slot = slots.get(sourceId); + if (slot == null) { + return Optional.empty(); + } + synchronized (slot) { + return Optional.of(toView(slot)); + } + } finally { + operationLock.unlock(); + } + } + + /** + * 确保节点本地 Runtime 达到最低 revision;同一 Slot 内初始化天然 single-flight。 + * + * @param sourceId 数据源标识 + * @param minimumRevision 最低版本 + * @return 就绪视图 + */ + @Override + public FederationSourceView ensureReady(SourceId sourceId, long minimumRevision) { + operationLock.lock(); + try { + ensureOpen(); + return ensureReadyInternal(sourceId, minimumRevision); + } finally { + operationLock.unlock(); + } + } + + private FederationSourceView ensureReadyInternal(SourceId sourceId, long minimumRevision) { + SourceSlot slot = refreshSlotIfNeeded(sourceId, minimumRevision); + synchronized (slot) { + ensureReadyLocked(slot, sourceId, minimumRevision); + return toView(slot); + } + } + + /** + * 原子确保并获取 Runtime lease,避免 revision 切换与执行获取之间的竞态。 + * + * @param sourceId 数据源标识 + * @param minimumRevision 最低版本 + * @return Runtime lease + */ + public SourceRuntime.RuntimeLease acquireRuntime(SourceId sourceId, long minimumRevision) { + operationLock.lock(); + try { + ensureOpen(); + SourceSlot slot = refreshSlotIfNeeded(sourceId, minimumRevision); + synchronized (slot) { + ensureReadyLocked(slot, sourceId, minimumRevision); + return slot.current.acquire(); + } + } finally { + operationLock.unlock(); + } + } + + /** + * 原子返回注册表代次与 SourceId 集合,供计划缓存和编译共享同一语义快照。 + * + * @return 注册表快照 + */ + SourceCatalogSnapshot catalogSnapshot() { + operationLock.lock(); + try { + ensureOpen(); + return sourceCatalogSnapshot; + } finally { + operationLock.unlock(); + } + } + + private SourceSlot refreshSlotIfNeeded(SourceId sourceId, long minimumRevision) { + SourceSlot slot = slots.get(sourceId); + if (slot == null || slot.desired.revision() < minimumRevision) { + stateProvider.find(sourceId).ifPresent(this::applySharedState); + slot = slots.get(sourceId); + } + if (slot == null) { + throw new FederationSqlException( + FederationSqlErrorCode.SOURCE_NOT_FOUND, + "source is not defined: " + sourceId + ); + } + return slot; + } + + private void ensureReadyLocked(SourceSlot slot, SourceId sourceId, long minimumRevision) { + if (slot.desired instanceof SourceTombstone) { + throw new FederationSqlException( + FederationSqlErrorCode.SOURCE_REMOVED, + "source has been removed: " + sourceId + ); + } + ActiveSourceState active = (ActiveSourceState) slot.desired; + if (active.revision() < minimumRevision) { + throw new FederationSqlException( + FederationSqlErrorCode.SOURCE_REVISION_NOT_READY, + "source revision " + active.revision() + " is lower than required " + minimumRevision + ); + } + if (slot.current != null && slot.current.definition().revision() >= minimumRevision + && slot.current.definition().revision() == active.revision()) { + return; + } + + SourceRuntime candidate; + try { + FederationDataSourceHandle handle = resolver.resolve(active.definition()); + candidate = buildRuntime(active.definition(), handle); + } catch (FederationSqlException exception) { + slot.lastFailure = exception.getMessage(); + if (slot.current != null && slot.current.definition().revision() >= minimumRevision) { + return; + } + throw exception; + } catch (RuntimeException exception) { + slot.lastFailure = exception.getMessage(); + if (slot.current != null && slot.current.definition().revision() >= minimumRevision) { + return; + } + throw new FederationSqlException( + FederationSqlErrorCode.SOURCE_INITIALIZATION_FAILED, + "failed to initialize source " + sourceId, + exception + ); + } + + SourceRuntime previous = slot.current; + slot.current = candidate; + slot.lastFailure = null; + if (previous != null) { + retireRuntime(previous); + } + } + + private SourceRuntime buildRuntime( + FederationSourceDefinition definition, + FederationDataSourceHandle handle + ) { + try { + FederationSqlAdapterProvider adapter = adapters.require(definition.adapterId()); + try (Connection connection = handle.dataSource().getConnection()) { + DatabaseMetaData metadata = connection.getMetaData(); + AdapterHints hints = new AdapterHints(definition.adapterOptions()); + if (!adapter.supports(metadata, hints)) { + throw new FederationSqlException( + FederationSqlErrorCode.ADAPTER_UNSUPPORTED, + "adapter " + adapter.adapterId() + " does not support database " + + metadata.getDatabaseProductName() + ); + } + SqlDialect dialect = adapter.createDialect(new AdapterDialectContext(metadata, definition)); + // 动态 JDBC 元数据必须在冷编译时可见;有界计划缓存负责热查询性能。 + SchemaPlus root = CalciteSchema.createRootSchema(true, false).plus(); + SchemaPlus sourceSchema = root.add(definition.sourceId().value(), new AbstractSchema()); + SchemaPlus onlySchema = null; + for (FederationSchemaDefinition schemaDefinition : definition.schemas()) { + Schema schema = adapter.createSchema(new AdapterSchemaContext( + sourceSchema, + definition, + schemaDefinition, + handle, + dialect + )); + SchemaPlus added = sourceSchema.add(schemaDefinition.logicalName(), schema); + onlySchema = definition.schemas().size() == 1 ? added : null; + } + SchemaPlus defaultSchema = onlySchema == null ? sourceSchema : onlySchema; + return new SourceRuntime( + definition, + handle, + adapter, + dialect, + root, + defaultSchema, + adapter.compatibility(metadata, hints), + this::runtimeClosed + ); + } + } catch (SQLException exception) { + FederationSqlException failure = new FederationSqlException( + FederationSqlErrorCode.SOURCE_INITIALIZATION_FAILED, + "failed to inspect source " + definition.sourceId(), + exception + ); + closeHandleAndSuppress(handle, failure); + throw failure; + } catch (RuntimeException exception) { + closeHandleAndSuppress(handle, exception); + throw exception; + } + } + + private void closeHandleAndSuppress( + FederationDataSourceHandle handle, + RuntimeException original + ) { + try { + handle.close(); + } catch (RuntimeException closeException) { + pendingHandleClosures.add(handle); + original.addSuppressed(closeException); + } + } + + private void applySharedState(FederationSourceState state) { + operationLock.lock(); + try { + if (!closed.get()) { + applyState(state); + } + } finally { + operationLock.unlock(); + } + } + + private SourceApplyResult applyState(FederationSourceState state) { + synchronized (sourceCatalogLock) { + SourceSlot newSlot = new SourceSlot(state); + SourceSlot existing = slots.putIfAbsent(state.sourceId(), newSlot); + if (existing == null) { + if (state instanceof ActiveSourceState) { + publishSourceCatalogSnapshot(); + } + return result(state, state.revision(), SourceApplyStatus.APPLIED); + } + SourceSlot slot = existing; + synchronized (slot) { + FederationSourceState current = slot.desired; + if (state.revision() < current.revision()) { + return result(state, current.revision(), SourceApplyStatus.IGNORED_STALE); + } + if (state.revision() == current.revision()) { + SourceApplyStatus status = state.checksum().equals(current.checksum()) + ? SourceApplyStatus.IDEMPOTENT + : SourceApplyStatus.CONFLICT; + return result(state, current.revision(), status); + } + slot.desired = state; + if ((current instanceof ActiveSourceState) != (state instanceof ActiveSourceState)) { + publishSourceCatalogSnapshot(); + } + if (state instanceof SourceTombstone && slot.current != null) { + retireRuntime(slot.current); + slot.current = null; + } + return result(state, state.revision(), SourceApplyStatus.APPLIED); + } + } + } + + private PreparedCommitOutcome commitPreparedRuntime(SourceRuntime candidate) { + ActiveSourceState state = ActiveSourceState.of(candidate.definition()); + synchronized (sourceCatalogLock) { + SourceSlot newSlot = new SourceSlot(state); + newSlot.current = candidate; + SourceSlot existing = slots.putIfAbsent(state.sourceId(), newSlot); + if (existing == null) { + publishSourceCatalogSnapshot(); + return new PreparedCommitOutcome( + result(state, state.revision(), SourceApplyStatus.APPLIED), true + ); + } + SourceSlot slot = existing; + synchronized (slot) { + FederationSourceState current = slot.desired; + if (state.revision() < current.revision()) { + return new PreparedCommitOutcome( + result(state, current.revision(), SourceApplyStatus.IGNORED_STALE), false + ); + } + SourceApplyStatus status; + if (state.revision() == current.revision()) { + if (!state.checksum().equals(current.checksum())) { + return new PreparedCommitOutcome( + result(state, current.revision(), SourceApplyStatus.CONFLICT), false + ); + } + status = SourceApplyStatus.IDEMPOTENT; + } else { + status = SourceApplyStatus.APPLIED; + slot.desired = state; + if (current instanceof SourceTombstone) { + publishSourceCatalogSnapshot(); + } + } + if (slot.current != null + && slot.current.definition().revision() == state.revision() + && slot.current.sourceChecksum().equals(state.checksum())) { + return new PreparedCommitOutcome( + result(state, state.revision(), status), false + ); + } + SourceRuntime previous = slot.current; + slot.current = candidate; + slot.lastFailure = null; + if (previous != null) { + try { + retireRuntime(previous); + } catch (RuntimeException exception) { + // 新 Runtime 已原子接管;记录旧 Handle 关闭错误且不回滚可用的新版本。 + slot.lastFailure = "previous runtime retirement failed: " + exception.getMessage(); + } + } + return new PreparedCommitOutcome( + result(state, state.revision(), status), true + ); + } + } + } + + private Set activeSourceIds() { + Set sourceIds = new HashSet<>(); + slots.forEach((sourceId, slot) -> { + if (slot.desired instanceof ActiveSourceState) { + sourceIds.add(sourceId); + } + }); + return Set.copyOf(sourceIds); + } + + private void publishSourceCatalogSnapshot() { + sourceCatalogSnapshot = new SourceCatalogSnapshot( + sourceCatalogSnapshot.generation() + 1, + activeSourceIds() + ); + } + + private static SourceApplyResult result( + FederationSourceState requested, + long effectiveRevision, + SourceApplyStatus status + ) { + return new SourceApplyResult( + requested.sourceId(), + requested.revision(), + effectiveRevision, + status + ); + } + + private static FederationSourceView toView(SourceSlot slot) { + SourceRuntimeStatus status; + if (slot.desired instanceof SourceTombstone) { + status = SourceRuntimeStatus.REMOVED; + } else if (slot.current != null) { + status = SourceRuntimeStatus.READY; + } else if (slot.lastFailure != null) { + status = SourceRuntimeStatus.FAILED; + } else { + status = SourceRuntimeStatus.DEFINED; + } + return new FederationSourceView( + slot.desired.sourceId(), + slot.desired.revision(), + slot.current == null ? -1 : slot.current.definition().revision(), + status, + slot.desired.checksum(), + slot.lastFailure + ); + } + + private void ensureOpen() { + if (closed.get()) { + throw new FederationSqlException(FederationSqlErrorCode.ENGINE_CLOSED, "source manager is closed"); + } + } + + /** + * 登记旧 Runtime 并按租约排空语义关闭。 + * + * @param runtime 待退役 Runtime + */ + private void retireRuntime(SourceRuntime runtime) { + retiringRuntimes.add(runtime); + runtime.retire(); + } + + /** + * 增加节点本地 Runtime 成功关闭后的监听器。 + * + * @param listener 关闭监听器 + */ + void addRuntimeClosedListener(Consumer listener) { + runtimeClosedListeners.add(java.util.Objects.requireNonNull(listener, "listener")); + } + + private void runtimeClosed(SourceRuntime runtime) { + retiringRuntimes.remove(runtime); + RuntimeException failure = null; + for (Consumer listener : runtimeClosedListeners) { + try { + listener.accept(runtime); + } catch (RuntimeException exception) { + if (failure == null) { + failure = exception; + } else { + failure.addSuppressed(exception); + } + } + } + if (failure != null) { + throw failure; + } + } + + /** + * 登记候选 Runtime 并立即关闭其 Handle。 + * + * @param runtime 待关闭 Runtime + */ + private void forceCloseRuntime(SourceRuntime runtime) { + retiringRuntimes.add(runtime); + runtime.forceClose(); + } + + /** + * 关闭订阅并让所有 Runtime 进入排空;无活跃 lease 的 Handle 会立即关闭。 + */ + @Override + public void close() { + RuntimeException failure = null; + closeLock.lock(); + try { + // write lock 排除了并发 build/commit,快照包含进入关闭前的全部失败资源。 + Set retryRuntimes = Set.copyOf(retiringRuntimes); + Set retryHandles = + Set.copyOf(pendingHandleClosures); + if (closed.compareAndSet(false, true)) { + for (SourceSlot slot : slots.values()) { + synchronized (slot) { + if (slot.current != null) { + try { + retireRuntime(slot.current); + } catch (RuntimeException exception) { + failure = append(failure, exception); + } + slot.current = null; + } + } + } + for (PreparedRuntime prepared : Set.copyOf(preparedRuntimes)) { + try { + prepared.close(); + } catch (RuntimeException exception) { + failure = append(failure, exception); + } + } + } + // 上一次释放失败的资源在后续 close 调用中重试,成功后由回调移出集合。 + for (SourceRuntime runtime : retryRuntimes) { + try { + runtime.retire(); + } catch (RuntimeException exception) { + failure = append(failure, exception); + } + } + for (FederationDataSourceHandle handle : retryHandles) { + try { + handle.close(); + pendingHandleClosures.remove(handle); + } catch (RuntimeException exception) { + failure = append(failure, exception); + } + } + } finally { + closeLock.unlock(); + } + synchronized (subscriptionCloseLock) { + try { + if (!subscriptionClosed.get()) { + subscription.close(); + subscriptionClosed.set(true); + } + } catch (RuntimeException exception) { + failure = append(failure, exception); + } + } + if (failure != null) { + throw failure; + } + } + + private static RuntimeException append(RuntimeException failure, RuntimeException next) { + if (failure == null) { + return next; + } + failure.addSuppressed(next); + return failure; + } + + private static final class SourceSlot { + + private FederationSourceState desired; + private SourceRuntime current; + private String lastFailure; + + private SourceSlot(FederationSourceState desired) { + this.desired = desired; + } + } + + private record PreparedCommitOutcome( + SourceApplyResult result, + boolean installed + ) { + } + + private final class PreparedRuntime implements PreparedSourceRuntime { + + private SourceRuntime runtime; + + private PreparedRuntime(SourceRuntime runtime) { + this.runtime = runtime; + } + + @Override + public synchronized FederationSourceDefinition definition() { + if (runtime == null) { + throw new IllegalStateException("prepared runtime is no longer open"); + } + return runtime.definition(); + } + + private synchronized SourceRuntime take() { + if (runtime == null) { + throw new IllegalStateException("prepared runtime is no longer open"); + } + SourceRuntime claimed = runtime; + runtime = null; + return claimed; + } + + @Override + public void close() { + SourceRuntime discarded; + synchronized (this) { + discarded = runtime; + runtime = null; + } + preparedRuntimes.remove(this); + if (discarded != null) { + forceCloseRuntime(discarded); + } + } + } +} diff --git a/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/runtime/DefaultFederationSqlEngine.java b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/runtime/DefaultFederationSqlEngine.java new file mode 100644 index 0000000..7d2abf8 --- /dev/null +++ b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/runtime/DefaultFederationSqlEngine.java @@ -0,0 +1,1289 @@ +package com.easyagents.federation.sql.runtime; + +import com.easyagents.federation.sql.api.FederationSqlEngine; +import com.easyagents.federation.sql.api.FederationSqlErrorCode; +import com.easyagents.federation.sql.api.FederationSqlException; +import com.easyagents.federation.sql.api.FederationCleanupMetrics; +import com.easyagents.federation.sql.api.SqlExecutionContext; +import com.easyagents.federation.sql.api.SqlCompletionRequest; +import com.easyagents.federation.sql.api.SqlCompletionResult; +import com.easyagents.federation.sql.api.SqlQueryCommand; +import com.easyagents.federation.sql.compile.FederationSqlPlan; +import com.easyagents.federation.sql.compile.FederationFragmentExplain; +import com.easyagents.federation.sql.compile.FederationSqlPolicy; +import com.easyagents.federation.sql.compile.SqlCompileRequest; +import com.easyagents.federation.sql.compile.SqlExplainRequest; +import com.easyagents.federation.sql.compile.SqlExplainResult; +import com.easyagents.federation.sql.compile.SqlExplainLevel; +import com.easyagents.federation.sql.execute.FederationFragmentExplainContext; +import com.easyagents.federation.sql.execute.FederationFragmentExecutionContext; +import com.easyagents.federation.sql.execute.FederationQueryAdmissionController; +import com.easyagents.federation.sql.execute.FederationQueryPermit; +import com.easyagents.federation.sql.execute.FederationResultCursor; +import com.easyagents.federation.sql.execute.FederationPhysicalExplain; +import com.easyagents.federation.sql.execute.QueryId; +import com.easyagents.federation.sql.execute.QueryAdmissionRequest; +import com.easyagents.federation.sql.execute.SqlParameter; +import com.easyagents.federation.sql.federation.FederationExecutionPolicy; +import com.easyagents.federation.sql.federation.FederationFragmentPlan; +import com.easyagents.federation.sql.federation.FederationJoinOptimization; +import com.easyagents.federation.sql.federation.FederationQueryMode; +import com.easyagents.federation.sql.federation.FederationSourceRuntimeIdentity; +import com.easyagents.federation.sql.federation.FederationStatisticsSnapshot; +import com.easyagents.federation.sql.federation.FederationStatisticsStatus; +import com.easyagents.federation.sql.federation.FederationTableStatisticsProvider; +import com.easyagents.federation.sql.source.FederationSourceManager; +import java.time.Duration; +import java.util.ArrayList; +import java.util.Collections; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.WeakHashMap; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.ScheduledThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicReference; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReentrantReadWriteLock; +import java.util.stream.Collectors; +import org.apache.calcite.plan.RelOptUtil; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * 默认 FederationSqlEngine,实现 Query Scope 编译、单源/联邦自动路由和资源治理。 + */ +public final class DefaultFederationSqlEngine implements FederationSqlEngine { + + private static final Logger LOG = LoggerFactory.getLogger(DefaultFederationSqlEngine.class); + + private final DefaultFederationSourceManager sourceManager; + private final FederationQueryAdmissionController admissionController; + private final BoundedPlanCache planCache; + private final CalciteFederationSqlCompiler compiler; + private final AdapterFederationStatisticsProvider automaticStatisticsProvider; + private final NodeMemoryAdmissionController nodeMemoryAdmission; + private final CalciteSqlCompleter completer = new CalciteSqlCompleter(); + private final QueryCancellationRegistry cancellations = new QueryCancellationRegistry(); + private final ScheduledThreadPoolExecutor deadlineScheduler = deadlineScheduler(); + private final Map authorizedPlans = + Collections.synchronizedMap(new WeakHashMap<>()); + private final Map openCursors = + new ConcurrentHashMap<>(); + private final AtomicBoolean closed = new AtomicBoolean(); + private final ReentrantReadWriteLock lifecycleLock = new ReentrantReadWriteLock(); + private final Lock operationLock = lifecycleLock.readLock(); + private final Lock closeLock = lifecycleLock.writeLock(); + + /** + * 创建使用默认冷编译并发和联邦资源上限的 Engine。 + * + * @param sourceManager 数据源管理器 + * @param admissionController 查询准入控制器 + * @param policies SQL 策略 + * @param maximumPlanCacheEntries 计划缓存上限 + * @param crossSourceEnabled 是否允许联邦执行 + */ + public DefaultFederationSqlEngine( + DefaultFederationSourceManager sourceManager, + FederationQueryAdmissionController admissionController, + List policies, + int maximumPlanCacheEntries, + boolean crossSourceEnabled + ) { + this( + sourceManager, + admissionController, + policies, + maximumPlanCacheEntries, + Math.min( + maximumPlanCacheEntries, + Math.max(1, Math.min(8, Runtime.getRuntime().availableProcessors())) + ), + crossSourceEnabled, + FederationExecutionPolicy.basic(), + 64L * 1024L * 1024L, + Duration.ofMinutes(30), + null, + 512L * 1024L * 1024L + ); + } + + /** + * 创建可独立限制计划容量与冷编译并发的 Engine。 + * + * @param sourceManager 数据源管理器 + * @param admissionController 查询准入控制器 + * @param policies SQL 策略 + * @param maximumPlanCacheEntries 计划缓存上限 + * @param maximumConcurrentCompilations 最大并发冷编译数 + * @param crossSourceEnabled 是否允许联邦执行 + */ + public DefaultFederationSqlEngine( + DefaultFederationSourceManager sourceManager, + FederationQueryAdmissionController admissionController, + List policies, + int maximumPlanCacheEntries, + int maximumConcurrentCompilations, + boolean crossSourceEnabled + ) { + this( + sourceManager, + admissionController, + policies, + maximumPlanCacheEntries, + maximumConcurrentCompilations, + crossSourceEnabled, + FederationExecutionPolicy.basic(), + 64L * 1024L * 1024L, + Duration.ofMinutes(30), + null, + 512L * 1024L * 1024L + ); + } + + /** + * 创建完整配置的 Engine。 + * + * @param sourceManager 数据源管理器 + * @param admissionController 查询准入控制器 + * @param policies SQL 策略 + * @param maximumPlanCacheEntries 计划缓存上限 + * @param maximumConcurrentCompilations 最大并发冷编译数 + * @param crossSourceEnabled 是否允许联邦执行 + * @param executionPolicy Engine 联邦资源硬上限 + */ + public DefaultFederationSqlEngine( + DefaultFederationSourceManager sourceManager, + FederationQueryAdmissionController admissionController, + List policies, + int maximumPlanCacheEntries, + int maximumConcurrentCompilations, + boolean crossSourceEnabled, + FederationExecutionPolicy executionPolicy + ) { + this( + sourceManager, + admissionController, + policies, + maximumPlanCacheEntries, + maximumConcurrentCompilations, + crossSourceEnabled, + executionPolicy, + 64L * 1024L * 1024L, + Duration.ofMinutes(30), + null, + 512L * 1024L * 1024L + ); + } + + /** + * 创建包含计划缓存权重与 TTL 的完整配置 Engine。 + * + * @param sourceManager 数据源管理器 + * @param admissionController 查询准入控制器 + * @param policies SQL 策略 + * @param maximumPlanCacheEntries 计划缓存条目上限 + * @param maximumConcurrentCompilations 最大并发冷编译数 + * @param crossSourceEnabled 是否允许联邦执行 + * @param executionPolicy Engine 联邦资源硬上限 + * @param maximumPlanCacheWeightBytes 计划缓存估算权重上限 + * @param planCacheTimeToLive 计划缓存条目存活时间 + */ + public DefaultFederationSqlEngine( + DefaultFederationSourceManager sourceManager, + FederationQueryAdmissionController admissionController, + List policies, + int maximumPlanCacheEntries, + int maximumConcurrentCompilations, + boolean crossSourceEnabled, + FederationExecutionPolicy executionPolicy, + long maximumPlanCacheWeightBytes, + Duration planCacheTimeToLive + ) { + this( + sourceManager, + admissionController, + policies, + maximumPlanCacheEntries, + maximumConcurrentCompilations, + crossSourceEnabled, + executionPolicy, + maximumPlanCacheWeightBytes, + planCacheTimeToLive, + null, + 512L * 1024L * 1024L + ); + } + + /** + * 创建包含计划缓存与外部统计 SPI 的完整配置 Engine。 + * + * @param sourceManager 数据源管理器 + * @param admissionController 查询准入控制器 + * @param policies SQL 策略 + * @param maximumPlanCacheEntries 计划缓存条目上限 + * @param maximumConcurrentCompilations 最大并发冷编译数 + * @param crossSourceEnabled 是否允许联邦执行 + * @param executionPolicy Engine 联邦资源硬上限 + * @param maximumPlanCacheWeightBytes 计划缓存估算权重上限 + * @param planCacheTimeToLive 计划缓存条目存活时间 + * @param statisticsProvider 调用方托管的联邦表统计 Provider;null 时启用 Adapter 自动采集 + */ + public DefaultFederationSqlEngine( + DefaultFederationSourceManager sourceManager, + FederationQueryAdmissionController admissionController, + List policies, + int maximumPlanCacheEntries, + int maximumConcurrentCompilations, + boolean crossSourceEnabled, + FederationExecutionPolicy executionPolicy, + long maximumPlanCacheWeightBytes, + Duration planCacheTimeToLive, + FederationTableStatisticsProvider statisticsProvider + ) { + this( + sourceManager, + admissionController, + policies, + maximumPlanCacheEntries, + maximumConcurrentCompilations, + crossSourceEnabled, + executionPolicy, + maximumPlanCacheWeightBytes, + planCacheTimeToLive, + statisticsProvider, + 512L * 1024L * 1024L + ); + } + + /** + * 创建包含节点中间结果内存准入的完整配置 Engine。 + * + * @param sourceManager 数据源管理器 + * @param admissionController 查询准入控制器 + * @param policies SQL 策略 + * @param maximumPlanCacheEntries 计划缓存条目上限 + * @param maximumConcurrentCompilations 最大并发冷编译数 + * @param crossSourceEnabled 是否允许联邦执行 + * @param executionPolicy Engine 联邦资源硬上限 + * @param maximumPlanCacheWeightBytes 计划缓存估算权重上限 + * @param planCacheTimeToLive 计划缓存条目存活时间 + * @param statisticsProvider 调用方托管的联邦表统计 Provider;null 时启用 Adapter 自动采集 + * @param maximumNodeIntermediateBytes 节点本地中间结果总预留上限 + */ + public DefaultFederationSqlEngine( + DefaultFederationSourceManager sourceManager, + FederationQueryAdmissionController admissionController, + List policies, + int maximumPlanCacheEntries, + int maximumConcurrentCompilations, + boolean crossSourceEnabled, + FederationExecutionPolicy executionPolicy, + long maximumPlanCacheWeightBytes, + Duration planCacheTimeToLive, + FederationTableStatisticsProvider statisticsProvider, + long maximumNodeIntermediateBytes + ) { + this.sourceManager = sourceManager; + this.admissionController = admissionController; + this.nodeMemoryAdmission = new NodeMemoryAdmissionController( + maximumNodeIntermediateBytes + ); + this.planCache = new BoundedPlanCache( + maximumPlanCacheEntries, + maximumConcurrentCompilations, + maximumPlanCacheWeightBytes, + planCacheTimeToLive + ); + this.sourceManager.addRuntimeClosedListener(planCache::invalidateRuntime); + this.automaticStatisticsProvider = statisticsProvider == null + ? new AdapterFederationStatisticsProvider() + : null; + if (automaticStatisticsProvider != null) { + this.sourceManager.addRuntimeClosedListener(automaticStatisticsProvider::invalidate); + } + FederationTableStatisticsProvider effectiveStatisticsProvider = + automaticStatisticsProvider == null + ? statisticsProvider + : automaticStatisticsProvider; + this.compiler = new CalciteFederationSqlCompiler( + policies, + crossSourceEnabled, + executionPolicy, + effectiveStatisticsProvider + ); + } + + /** {@inheritDoc} */ + @Override + public FederationSourceManager sources() { + operationLock.lock(); + try { + ensureOpen(); + return sourceManager; + } finally { + operationLock.unlock(); + } + } + + /** {@inheritDoc} */ + @Override + public SqlCompletionResult complete(SqlCompletionRequest request) { + if (request == null) { + throw new FederationSqlException( + FederationSqlErrorCode.INVALID_ARGUMENT, + "completion request must be provided" + ); + } + ensureOpen(); + Set bindings = new LinkedHashSet<>(request.queryScope().bindings().keySet()); + try (FederationQueryScopeSnapshot snapshot = FederationQueryScopeSnapshot.acquire( + sourceManager, + request.queryScope(), + bindings + )) { + return completer.complete(request, snapshot); + } + } + + /** {@inheritDoc} */ + @Override + public FederationSqlPlan compile(SqlCompileRequest request) { + requireCompileRequest(request); + QueryDeadline deadline = QueryDeadline.compile(Duration.ofMillis( + compiler.effectivePolicy(request.queryScope()).maximumExecutionTimeMillis() + )); + return compileInternal(request, deadline, false).plan(); + } + + /** + * 编译并登记一个授权计划。 + * + * @param request 编译请求 + * @param deadline 编译或查询共享截止时间 + * @param waitForStatistics 是否在截止时间内等待自动统计刷新 + * @return 编译结果及授权信息 + */ + private CompilationResult compileInternal( + SqlCompileRequest request, + QueryDeadline deadline, + boolean waitForStatistics + ) { + requireCompileRequest(request); + long started = System.nanoTime(); + ensureOpen(); + deadline.ensureAllowed(); + Set candidateBindings = compiler.discoverBindings(request); + deadline.ensureAllowed(); + try (FederationQueryScopeSnapshot snapshot = FederationQueryScopeSnapshot.acquire( + sourceManager, + request.queryScope(), + candidateBindings + )) { + if (automaticStatisticsProvider != null) { + CompletableFuture refresh = + automaticStatisticsProvider.refreshIfNeeded(snapshot); + if (waitForStatistics) { + awaitStatistics(refresh, deadline); + } + } + FederationStatisticsSnapshot statisticsSnapshot = compiler.statisticsSnapshot(); + PlanCacheKey key = new PlanCacheKey( + request.sql(), + request.queryScope().checksum(), + request.parameterJdbcTypes(), + runtimeIdentityFingerprint(snapshot.identities()), + compiler.policyFingerprint(), + "table-scoped-v2", + request.policyVersion() + ); + BoundedPlanCache.LookupResult lookup = planCache.getOrCompileWithStatus( + key, + () -> compiler.compile(request, snapshot, statisticsSnapshot), + deadline, + cached -> statisticsCompatible(cached, statisticsSnapshot), + statisticsSnapshot.capturedAt() + ); + if (!(lookup.plan() instanceof DefaultFederationSqlPlan cachedPlan)) { + throw new FederationSqlException( + FederationSqlErrorCode.EXECUTION_FAILED, + "plan cache returned an unsupported plan implementation" + ); + } + DefaultFederationSqlPlan plan = cachedPlan.issuedCopy(); + deadline.ensureAllowed(); + compiler.validatePolicies(request, plan); + AuthorizedPlan authorized = new AuthorizedPlan( + request, + lookup.cacheHit(), + System.nanoTime() - started + ); + operationLock.lock(); + try { + ensureOpen(); + authorizedPlans.put(plan, authorized); + return new CompilationResult(plan, authorized); + } finally { + operationLock.unlock(); + } + } + } + + /** + * 在请求截止时间内等待显式 Explain 所需的自动统计。 + * + * @param refresh 自动统计刷新任务 + * @param deadline 请求截止时间 + * @throws FederationSqlException 等待超时或线程被中断时抛出 + */ + private void awaitStatistics( + CompletableFuture refresh, + QueryDeadline deadline + ) { + deadline.ensureAllowed(); + try { + refresh.get(Math.max(1L, deadline.remainingNanos()), TimeUnit.NANOSECONDS); + deadline.ensureAllowed(); + } catch (TimeoutException exception) { + deadline.ensureAllowed(); + throw new FederationSqlException( + FederationSqlErrorCode.SQL_COMPILE_TIMEOUT, + "statistics refresh exceeded the SQL compilation deadline", + exception + ); + } catch (InterruptedException exception) { + Thread.currentThread().interrupt(); + throw new FederationSqlException( + FederationSqlErrorCode.EXPLAIN_FAILED, + "statistics refresh was interrupted", + exception + ); + } catch (ExecutionException exception) { + // 自动统计是优化提示,异常时继续使用已冻结统计或默认估算。 + LOG.warn("automatic federation statistics refresh failed", exception.getCause()); + } + } + + /** + * 判断缓存计划引用的统计是否仍与当前快照一致。 + * + * @param cached 缓存计划 + * @param current 当前统计快照 + * @return 统计指纹一致时为 true + */ + private static boolean statisticsCompatible( + FederationSqlPlan cached, + FederationStatisticsSnapshot current + ) { + if (!(cached instanceof DefaultFederationSqlPlan plan)) { + return false; + } + return current.select(plan.statisticsTables()).fingerprint() + .equals(plan.statisticsFingerprint()); + } + + /** {@inheritDoc} */ + @Override + public FederationResultCursor execute( + FederationSqlPlan plan, + SqlExecutionContext context + ) { + if (plan == null || context == null) { + throw new FederationSqlException( + FederationSqlErrorCode.INVALID_ARGUMENT, + "plan and execution context must be provided" + ); + } + QueryCancellationRegistry.QueryRegistration registration = + cancellations.begin(context.queryId()); + QueryDeadline deadline = QueryDeadline.query( + QueryDeadline.deadlineAfter(Duration.ofMillis(effectiveExecutionTimeoutMillis( + compiler.effectivePolicy(plan.queryScope()), + context + ))), + registration + ); + try { + return executeRegistered( + plan, + context, + registration, + requireAuthorizedPlan(plan), + deadline + ); + } catch (RuntimeException exception) { + closeAndSuppress(registration, exception); + throw exception; + } + } + + private FederationResultCursor executeRegistered( + FederationSqlPlan publicPlan, + SqlExecutionContext context, + QueryCancellationRegistry.QueryRegistration registration, + AuthorizedPlan authorized, + QueryDeadline queryDeadline + ) { + FederationQueryScopeSnapshot runtimeSnapshot = null; + FederationQueryPermit permit = null; + NodeMemoryAdmissionController.Permit memoryPermit = null; + CompositeQueryResources resources = null; + AutoCloseable deadline = null; + FederationExecutionSession session = null; + FederationResultCursor queryCursor = null; + try { + if (!(publicPlan instanceof DefaultFederationSqlPlan plan)) { + throw new FederationSqlException( + FederationSqlErrorCode.INVALID_ARGUMENT, + "plan was not issued by the federation SQL engine" + ); + } + registration.ensureNotCancelled(); + queryDeadline.ensureAllowed(); + compiler.validatePolicies(authorized.request(), publicPlan); + registration.ensureNotCancelled(); + if (!plan.executable()) { + throw new FederationSqlException( + FederationSqlErrorCode.CROSS_SOURCE_EXECUTION_UNSUPPORTED, + "the compiled plan is explain-only" + ); + } + if (plan.parameterCount() != context.parameters().size()) { + throw new FederationSqlException( + FederationSqlErrorCode.PARAMETER_COUNT_MISMATCH, + "plan expects " + plan.parameterCount() + " parameters but received " + + context.parameters().size() + ); + } + requireMatchingParameterTypes(plan.parameterJdbcTypes(), context.parameters()); + + Set actualBindings = plan.sourceRuntimeIdentities().stream() + .map(FederationSourceRuntimeIdentity::bindingName) + .collect(Collectors.toCollection(LinkedHashSet::new)); + runtimeSnapshot = FederationQueryScopeSnapshot.acquire( + sourceManager, + plan.queryScope(), + actualBindings + ); + registration.ensureNotCancelled(); + requireMatchingRuntimes(plan, runtimeSnapshot); + FederationQueryMetricsTracker metrics = new FederationQueryMetricsTracker( + context.queryId(), + plan.queryMode(), + authorized.planCacheHit(), + authorized.planningNanos(), + plan.fragments() + ); + long admissionStarted = System.nanoTime(); + Duration admissionTimeout = boundedAdmissionTimeout( + context.admissionTimeout(), + queryDeadline + ); + try { + permit = admissionController.acquire(new QueryAdmissionRequest( + plan.referencedSources().stream().toList(), + context.queryId(), + admissionTimeout, + registration::cancellationRequested + )); + } catch (FederationSqlException exception) { + if (exception.errorCode() == FederationSqlErrorCode.QUERY_ADMISSION_TIMEOUT) { + // 总查询时限先到时,终态应稳定为 QUERY_TIMEOUT。 + queryDeadline.ensureAllowed(); + } + throw exception; + } + metrics.recordAdmissionWait(System.nanoTime() - admissionStarted); + registration.ensureNotCancelled(); + queryDeadline.ensureAllowed(); + + FederationQueryScopeSnapshot executionSnapshot = runtimeSnapshot; + FederationExecutionPolicy effectivePolicy = compiler.effectivePolicy( + plan.queryScope() + ); + memoryPermit = plan.queryMode() == FederationQueryMode.FEDERATED + ? nodeMemoryAdmission.acquire( + effectivePolicy.maximumIntermediateBytes(), + queryDeadline + ) + : NodeMemoryAdmissionController.none(); + deadline = scheduleExecutionDeadline( + queryDeadline, + context.queryId(), + metrics, + queryDeadline.remainingNanos() + ); + resources = new CompositeQueryResources( + registration, + runtimeSnapshot, + permit, + memoryPermit, + deadline + ); + runtimeSnapshot = null; + permit = null; + memoryPermit = null; + deadline = null; + if (plan.queryMode() == FederationQueryMode.SINGLE_SOURCE) { + queryCursor = executeSingleSource( + plan, + context, + executionSnapshot, + registration, + metrics, + queryDeadline + ); + } else { + // 资源组合已经持有同一快照;会话只借用它并在游标关闭时清理分片。 + session = new FederationExecutionSession( + plan, + context, + executionSnapshot, + registration, + compiler.effectivePolicy(plan.queryScope()), + metrics, + queryDeadline + ); + queryCursor = new FederatedResultCursor( + plan, + context.queryId(), + session, + metrics, + context.options().maxRows() + ); + session = null; + } + return manageCursor(queryCursor, resources, registration, context.queryId()); + } catch (RuntimeException exception) { + closeAndSuppress(queryCursor, exception); + closeAndSuppress(session, exception); + closeAndSuppress(resources, exception); + closeAndSuppress(deadline, exception); + closeAndSuppress(memoryPermit, exception); + closeAndSuppress(permit, exception); + closeAndSuppress(runtimeSnapshot, exception); + closeAndSuppress(registration, exception); + throw exception; + } + } + + private FederationResultCursor executeSingleSource( + DefaultFederationSqlPlan plan, + SqlExecutionContext context, + FederationQueryScopeSnapshot runtimeSnapshot, + QueryCancellationRegistry.QueryRegistration registration, + FederationQueryMetricsTracker metrics, + QueryDeadline queryDeadline + ) { + FederationFragmentPlan fragment = plan.fragments().get(0); + SourceRuntime runtime = runtimeSnapshot.runtime(fragment.bindingName()); + requireMatchingRuntime(plan.sourceRuntimeIdentities().get(0), runtime); + FederationResultCursor adapterCursor = runtime.adapter().fragmentExecutor().execute( + new FederationFragmentExecutionContext( + context.queryId(), + fragment.executableSql(), + remapParameters(fragment.parameterMapping(), context.parameters()), + context.options(), + runtime.handle().dataSource(), + runtime.compatibility(), + runtime.definition().adapterOptions(), + registration.statementLifecycle(), + metrics.fragmentObserver(fragment.fragmentId()), + queryDeadline + ) + ); + if (adapterCursor == null || !context.queryId().equals(adapterCursor.queryId())) { + FederationSqlException invalid = new FederationSqlException( + FederationSqlErrorCode.EXECUTION_FAILED, + "adapter returned an invalid query cursor" + ); + closeAndSuppress(adapterCursor, invalid); + throw invalid; + } + return new MetricsFederationResultCursor(adapterCursor, metrics); + } + + private FederationResultCursor manageCursor( + FederationResultCursor queryCursor, + CompositeQueryResources resources, + QueryCancellationRegistry.QueryRegistration registration, + QueryId queryId + ) { + operationLock.lock(); + try { + ensureOpen(); + AtomicReference reference = new AtomicReference<>(); + ManagedFederationResultCursor managed = new ManagedFederationResultCursor( + queryCursor, + resources, + () -> openCursors.remove(queryId, reference.get()) + ); + reference.set(managed); + ManagedFederationResultCursor previous = openCursors.putIfAbsent(queryId, managed); + if (previous != null) { + throw new FederationSqlException( + FederationSqlErrorCode.EXECUTION_FAILED, + "query id is already active: " + queryId.value() + ); + } + if (registration.cancellationRequested()) { + try { + registration.ensureNotCancelled(); + } catch (FederationSqlException terminal) { + if (registration.timeoutRequested()) { + managed.markTimedOut(); + } else { + managed.markCancelled(); + } + closeAndSuppress(managed, terminal); + throw terminal; + } + } + return managed; + } finally { + operationLock.unlock(); + } + } + + /** {@inheritDoc} */ + @Override + public FederationResultCursor query(SqlQueryCommand command) { + if (command == null) { + throw new FederationSqlException( + FederationSqlErrorCode.INVALID_ARGUMENT, + "query command must be provided" + ); + } + ensureOpen(); + List parameterTypes = command.parameters().stream() + .map(SqlParameter::jdbcType) + .toList(); + SqlCompileRequest compileRequest = new SqlCompileRequest( + command.sql(), + command.queryScope(), + parameterTypes, + command.policyVersion() + ); + SqlExecutionContext executionContext = new SqlExecutionContext( + command.queryId(), + command.parameters(), + command.options(), + Duration.ofMillis(command.admissionTimeoutMillis()) + ); + long absoluteDeadline = QueryDeadline.deadlineAfter(Duration.ofMillis( + effectiveExecutionTimeoutMillis( + compiler.effectivePolicy(command.queryScope()), + executionContext + ) + )); + for (int attempt = 0; attempt < 2; attempt++) { + QueryCancellationRegistry.QueryRegistration registration = + cancellations.begin(command.queryId()); + QueryDeadline deadline = QueryDeadline.query(absoluteDeadline, registration); + try { + CompilationResult compilation = compileInternal( + compileRequest, + deadline, + false + ); + registration.ensureNotCancelled(); + return executeRegistered( + compilation.plan(), + executionContext, + registration, + compilation.authorized(), + deadline + ); + } catch (RuntimeException exception) { + closeAndSuppress(registration, exception); + if (attempt == 0 && isPlanStale(exception)) { + continue; + } + throw exception; + } + } + throw new IllegalStateException("query retry loop completed unexpectedly"); + } + + private static boolean isPlanStale(RuntimeException exception) { + return exception instanceof FederationSqlException federationException + && federationException.errorCode() == FederationSqlErrorCode.PLAN_STALE; + } + + /** {@inheritDoc} */ + @Override + public SqlExplainResult explain(SqlExplainRequest request) { + if (request == null) { + throw new FederationSqlException( + FederationSqlErrorCode.INVALID_ARGUMENT, + "Explain request must be provided" + ); + } + QueryDeadline deadline = QueryDeadline.compile(Duration.ofMillis( + compiler.effectivePolicy(request.compileRequest().queryScope()) + .maximumExecutionTimeMillis() + )); + CompilationResult compilation = compileInternal( + request.compileRequest(), + deadline, + true + ); + FederationSqlPlan plan = compilation.plan(); + List fragments = request.level() == SqlExplainLevel.PHYSICAL + ? explainPhysicalFragments(request, plan, deadline) + : explainLogicalFragments(plan); + return new SqlExplainResult( + request.level(), + plan.queryMode(), + explainStatisticsStatus(plan.fragments()), + plan.fragments().stream() + .allMatch(fragment -> fragment.costEstimate().estimateAvailable()), + estimatedTransferBytes(plan.fragments()), + estimatedLocalMemoryBytes(plan.joinOptimizations()), + plan.joinOptimizations(), + plan.normalizedSql(), + plan.executableSql(), + RelOptUtil.toString(plan.relRoot().rel), + ((DefaultFederationSqlPlan) plan).executionRelationalPlan(), + fragments, + plan.referencedSources(), + plan.compatibility(), + plan.executable(), + compilation.authorized().planCacheHit(), + plan.queryMode() == FederationQueryMode.SINGLE_SOURCE + ? "single-source plan is executable" + : "federated plan is executable with " + plan.fragments().size() + " fragments" + ); + } + + private static FederationStatisticsStatus explainStatisticsStatus( + List fragments + ) { + boolean anyComplete = false; + boolean anyMissing = false; + boolean anyPartial = false; + for (FederationFragmentPlan fragment : fragments) { + FederationStatisticsStatus status = fragment.costEstimate().statisticsStatus(); + if (status == FederationStatisticsStatus.STALE) { + return FederationStatisticsStatus.STALE; + } + anyComplete |= status == FederationStatisticsStatus.COMPLETE; + anyMissing |= status == FederationStatisticsStatus.MISSING; + anyPartial |= status == FederationStatisticsStatus.PARTIAL; + } + if (anyPartial || anyComplete && anyMissing) { + return FederationStatisticsStatus.PARTIAL; + } + return anyComplete ? FederationStatisticsStatus.COMPLETE : FederationStatisticsStatus.MISSING; + } + + private static double estimatedTransferBytes(List fragments) { + double total = 0D; + for (FederationFragmentPlan fragment : fragments) { + double next = fragment.costEstimate().estimatedTransferBytes(); + total = total > Double.MAX_VALUE - next ? Double.MAX_VALUE : total + next; + } + return total; + } + + private static long estimatedLocalMemoryBytes( + List optimizations + ) { + long peak = 0L; + for (FederationJoinOptimization optimization : optimizations) { + long next = (long) Math.min( + Long.MAX_VALUE, + Math.ceil(optimization.estimatedBuildBytes()) + ); + peak = Math.max(peak, next); + } + return peak; + } + + private List explainLogicalFragments(FederationSqlPlan plan) { + return plan.fragments().stream() + .map(fragment -> fragmentExplain(plan, fragment, null)) + .toList(); + } + + private List explainPhysicalFragments( + SqlExplainRequest request, + FederationSqlPlan publicPlan, + QueryDeadline deadline + ) { + if (!(publicPlan instanceof DefaultFederationSqlPlan plan)) { + throw new FederationSqlException( + FederationSqlErrorCode.INVALID_ARGUMENT, + "plan was not issued by the federation SQL engine" + ); + } + Set bindings = plan.sourceRuntimeIdentities().stream() + .map(FederationSourceRuntimeIdentity::bindingName) + .collect(Collectors.toCollection(LinkedHashSet::new)); + QueryId explainQueryId = QueryId.create(); + List originalParameters = explainParameters(request); + FederationQueryPermit permit; + Duration admissionTimeout = boundedAdmissionTimeout( + Duration.ofSeconds(5), + deadline + ); + try { + permit = admissionController.acquire(new QueryAdmissionRequest( + plan.referencedSources().stream().toList(), + explainQueryId, + admissionTimeout, + () -> deadline.remainingNanos() <= 0L + )); + } catch (FederationSqlException exception) { + if (exception.errorCode() == FederationSqlErrorCode.QUERY_ADMISSION_TIMEOUT + || exception.errorCode() == FederationSqlErrorCode.QUERY_CANCELLED) { + deadline.ensureAllowed(); + } + throw exception; + } + try (FederationQueryScopeSnapshot snapshot = FederationQueryScopeSnapshot.acquire( + sourceManager, + plan.queryScope(), + bindings + ); + FederationQueryPermit ignored = permit) { + deadline.ensureAllowed(); + requireMatchingRuntimes(plan, snapshot); + List results = new ArrayList<>(plan.fragments().size()); + for (FederationFragmentPlan fragment : plan.fragments()) { + SourceRuntime runtime = snapshot.runtime(fragment.bindingName()); + FederationPhysicalExplain physical = runtime.adapter().fragmentExplainer() + .map(explainer -> explainer.explain(new FederationFragmentExplainContext( + fragment.executableSql(), + remapParameters(fragment.parameterMapping(), originalParameters), + runtime.handle().dataSource(), + runtime.compatibility(), + runtime.definition().adapterOptions(), + 15, + deadline + ))) + .orElseGet(() -> FederationPhysicalExplain.unavailable( + "adapter does not provide physical Explain" + )); + results.add(fragmentExplain(plan, fragment, physical)); + } + return List.copyOf(results); + } + } + + private static FederationFragmentExplain fragmentExplain( + FederationSqlPlan plan, + FederationFragmentPlan fragment, + FederationPhysicalExplain physical + ) { + String adapterId = plan.sourceRuntimeIdentities().stream() + .filter(identity -> identity.bindingName().equals(fragment.bindingName())) + .map(FederationSourceRuntimeIdentity::adapterId) + .findFirst() + .orElse(""); + return new FederationFragmentExplain( + fragment.fragmentId(), + fragment.bindingName(), + fragment.sourceId(), + adapterId, + fragment.executableSql(), + fragment.parameterMapping(), + fragment.columns(), + fragment.costEstimate(), + fragment.pushedDownOperators(), + physical + ); + } + + private static List explainParameters(SqlExplainRequest request) { + return request.compileRequest().parameterJdbcTypes().stream() + .map(jdbcType -> new SqlParameter(jdbcType, null)) + .toList(); + } + + private static void requireMatchingParameterTypes( + List expectedTypes, + List parameters + ) { + if (expectedTypes.isEmpty()) { + return; + } + for (int index = 0; index < expectedTypes.size(); index++) { + if (expectedTypes.get(index) != parameters.get(index).jdbcType()) { + throw new FederationSqlException( + FederationSqlErrorCode.PARAMETER_COUNT_MISMATCH, + "parameter " + index + " JDBC type does not match the compiled plan" + ); + } + } + } + + /** {@inheritDoc} */ + @Override + public boolean cancel(QueryId queryId) { + ensureOpen(); + QueryCancellationRegistry.CancellationOutcome outcome = + cancellations.cancelOutcome(queryId); + boolean found = outcome.found(); + ManagedFederationResultCursor cursor = openCursors.get(queryId); + if (cursor != null) { + found = true; + if (outcome.reason() == QueryCancellationRegistry.TerminationReason.TIMED_OUT) { + cursor.markTimedOut(); + } else { + cursor.markCancelled(); + } + cancellations.submitCleanup(cursor::close); + } + return found; + } + + /** {@inheritDoc} */ + @Override + public FederationCleanupMetrics cleanupMetrics() { + return cancellations.cleanupMetrics(); + } + + private void ensureOpen() { + if (closed.get()) { + throw new FederationSqlException( + FederationSqlErrorCode.ENGINE_CLOSED, + "federation SQL engine is closed" + ); + } + } + + /** + * 有界提交活动游标清理,再关闭取消通道、数据源 Runtime、计划缓存和准入控制器。 + */ + @Override + public void close() { + closeLock.lock(); + try { + closed.compareAndSet(false, true); + RuntimeException failure = null; + deadlineScheduler.shutdownNow(); + for (ManagedFederationResultCursor cursor : new ArrayList<>(openCursors.values())) { + cursor.markCancelled(); + cancellations.submitCleanup(cursor::close); + } + try { + cancellations.close(); + } catch (RuntimeException exception) { + failure = exception; + } + try { + sourceManager.close(); + } catch (RuntimeException exception) { + failure = append(failure, exception); + } + if (automaticStatisticsProvider != null) { + try { + automaticStatisticsProvider.close(); + } catch (RuntimeException exception) { + failure = append(failure, exception); + } + } + planCache.close(); + authorizedPlans.clear(); + try { + admissionController.close(); + } catch (RuntimeException exception) { + failure = append(failure, exception); + } + if (failure != null) { + throw failure; + } + } finally { + closeLock.unlock(); + } + } + + private AuthorizedPlan requireAuthorizedPlan(FederationSqlPlan plan) { + if (!(plan instanceof DefaultFederationSqlPlan)) { + throw new FederationSqlException( + FederationSqlErrorCode.INVALID_ARGUMENT, + "plan was not issued by the federation SQL engine" + ); + } + AuthorizedPlan authorized = authorizedPlans.get(plan); + if (authorized == null) { + throw new FederationSqlException( + FederationSqlErrorCode.INVALID_ARGUMENT, + "plan was not compiled by this engine instance" + ); + } + return authorized; + } + + private static void requireMatchingRuntimes( + DefaultFederationSqlPlan plan, + FederationQueryScopeSnapshot snapshot + ) { + for (FederationSourceRuntimeIdentity identity : plan.sourceRuntimeIdentities()) { + requireMatchingRuntime(identity, snapshot.runtime(identity.bindingName())); + } + } + + private static void requireMatchingRuntime( + FederationSourceRuntimeIdentity identity, + SourceRuntime runtime + ) { + boolean matches = runtime.definition().sourceId().equals(identity.sourceId()) + && runtime.definition().revision() == identity.sourceRevision() + && runtime.sourceChecksum().equals(identity.sourceChecksum()) + && runtime.adapter().adapterId().equals(identity.adapterId()) + && runtime.runtimeFingerprint().equals(identity.runtimeFingerprint()); + if (!matches) { + throw new FederationSqlException( + FederationSqlErrorCode.PLAN_STALE, + "plan runtime identity is stale; recompile on the current node" + ); + } + } + + private static List remapParameters( + List mapping, + List parameters + ) { + if (mapping.isEmpty()) { + return List.of(); + } + List remapped = new ArrayList<>(mapping.size()); + for (Integer sourceIndex : mapping) { + remapped.add(parameters.get(sourceIndex)); + } + return List.copyOf(remapped); + } + + private static String runtimeIdentityFingerprint( + List identities + ) { + StringBuilder value = new StringBuilder(); + identities.stream() + .sorted(java.util.Comparator.comparing(FederationSourceRuntimeIdentity::bindingName)) + .forEach(identity -> { + appendField(value, identity.bindingName()); + appendField(value, identity.sourceId().value()); + appendField(value, Long.toString(identity.sourceRevision())); + appendField(value, identity.sourceChecksum()); + appendField(value, identity.adapterId()); + appendField(value, identity.runtimeFingerprint()); + }); + return value.toString(); + } + + private static void appendField(StringBuilder target, String value) { + target.append(value.length()).append(':').append(value); + } + + private AutoCloseable scheduleExecutionDeadline( + QueryDeadline deadline, + QueryId queryId, + FederationQueryMetricsTracker metrics, + long timeoutNanos + ) { + ScheduledFuture future = deadlineScheduler.schedule(() -> { + QueryCancellationRegistry.TerminationReason reason = deadline.requestTimeout(); + if (reason != QueryCancellationRegistry.TerminationReason.TIMED_OUT) { + return; + } + metrics.markTimedOut(); + ManagedFederationResultCursor cursor = openCursors.get(queryId); + if (cursor != null) { + cursor.markTimedOut(); + cancellations.submitCleanup(cursor::close); + } + }, Math.max(1L, timeoutNanos), TimeUnit.NANOSECONDS); + return () -> future.cancel(false); + } + + private static Duration boundedAdmissionTimeout( + Duration requested, + QueryDeadline deadline + ) { + deadline.ensureAllowed(); + long requestedNanos; + try { + requestedNanos = requested.toNanos(); + } catch (ArithmeticException exception) { + requestedNanos = Long.MAX_VALUE; + } + return Duration.ofNanos(Math.max( + 1L, + Math.min(requestedNanos, deadline.remainingNanos()) + )); + } + + private static long effectiveExecutionTimeoutMillis( + FederationExecutionPolicy policy, + SqlExecutionContext context + ) { + long policyMillis = policy.maximumExecutionTimeMillis(); + int requestSeconds = context.options().queryTimeoutSeconds(); + if (requestSeconds == 0) { + return policyMillis; + } + long requestMillis = TimeUnit.SECONDS.toMillis(requestSeconds); + return Math.max(1L, Math.min(policyMillis, requestMillis)); + } + + private static void requireCompileRequest(SqlCompileRequest request) { + if (request == null) { + throw new FederationSqlException( + FederationSqlErrorCode.INVALID_ARGUMENT, + "compile request must be provided" + ); + } + } + + private static ScheduledThreadPoolExecutor deadlineScheduler() { + ScheduledThreadPoolExecutor scheduler = new ScheduledThreadPoolExecutor(1, runnable -> { + Thread thread = new Thread(runnable, "easy-agents-federation-deadline"); + thread.setDaemon(true); + return thread; + }); + scheduler.setRemoveOnCancelPolicy(true); + return scheduler; + } + + private static void closeAndSuppress(AutoCloseable closeable, RuntimeException original) { + if (closeable == null) { + return; + } + try { + closeable.close(); + } catch (Exception closeException) { + original.addSuppressed(closeException); + } + } + + private static RuntimeException append(RuntimeException failure, RuntimeException next) { + if (failure == null) { + return next; + } + failure.addSuppressed(next); + return failure; + } + + private record AuthorizedPlan( + SqlCompileRequest request, + boolean planCacheHit, + long planningNanos + ) { + } + + private record CompilationResult( + FederationSqlPlan plan, + AuthorizedPlan authorized + ) { + } +} diff --git a/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/runtime/DefaultFederationSqlPlan.java b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/runtime/DefaultFederationSqlPlan.java new file mode 100644 index 0000000..e1a8f35 --- /dev/null +++ b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/runtime/DefaultFederationSqlPlan.java @@ -0,0 +1,516 @@ +package com.easyagents.federation.sql.runtime; + +import com.easyagents.federation.sql.adapter.AdapterCompatibility; +import com.easyagents.federation.sql.compile.FederationSqlPlan; +import com.easyagents.federation.sql.execute.FederationColumn; +import com.easyagents.federation.sql.federation.FederationFragmentPlan; +import com.easyagents.federation.sql.federation.FederationJoinOptimization; +import com.easyagents.federation.sql.federation.FederationQueryMode; +import com.easyagents.federation.sql.federation.FederationQueryScopeDefinition; +import com.easyagents.federation.sql.federation.FederationSourceRuntimeIdentity; +import com.easyagents.federation.sql.federation.FederationStatisticsSnapshot; +import com.easyagents.federation.sql.source.SourceId; +import java.time.Instant; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import org.apache.calcite.rel.RelRoot; +import org.apache.calcite.runtime.Bindable; +import org.apache.calcite.schema.SchemaPlus; +import org.apache.calcite.sql.SqlNode; + +/** + * Core 内部的计划实现。 + */ +final class DefaultFederationSqlPlan implements FederationSqlPlan { + + private final SourceId sourceId; + private final FederationQueryScopeDefinition queryScope; + private final FederationQueryMode queryMode; + private final long sourceRevision; + private final String normalizedSql; + private final String executableSql; + private final SqlNode sqlNode; + private final RelRoot relRoot; + private final int parameterCount; + private final List parameterJdbcTypes; + private final List parameterMapping; + private final List fragments; + private final List joinOptimizations; + private final List sourceRuntimeIdentities; + private final String scopeChecksum; + private final List columns; + private final Set referencedSources; + private final AdapterCompatibility compatibility; + private final boolean executable; + private final String sourceChecksum; + private final String adapterId; + private final String runtimeFingerprint; + private final Set statisticsTables; + private final String statisticsFingerprint; + private final Instant statisticsValidUntil; + private final Map compiledFragments; + private final String executionRelationalPlan; + private final Bindable localBindable; + private final SchemaPlus localRootSchema; + + /** + * 创建 Engine 内部计划。 + * + * @param sourceId 主数据源 + * @param queryScope 查询范围 + * @param queryMode 查询模式 + * @param normalizedSql Calcite 规范化 SQL + * @param executableSql 目标数据库参数化 SQL + * @param sqlNode Calcite 已校验 SQL 节点 + * @param relRoot Calcite 关系计划 + * @param parameterCount 动态参数数量 + * @param parameterJdbcTypes 编译时声明的 JDBC 参数类型 + * @param parameterMapping 单源目标 SQL 占位符映射 + * @param fragments 物理分片 + * @param joinOptimizations 跨源 Join 优化选择 + * @param sourceRuntimeIdentities 实际引用源运行身份 + * @param columns 结果列 + * @param referencedSources 引用的数据源集合 + * @param compatibility Adapter 兼容性 + * @param executable 是否允许执行 + * @param sourceChecksum 数据源 Definition 校验和 + * @param adapterId Adapter 标识 + * @param compiledFragments 内部分片元数据 + * @param executionRelationalPlan 实际单源下推或联邦本地关系计划文本 + * @param localBindable 联邦本地可执行计划;单源为 null + * @param localRootSchema 分片占位表根 Schema;单源为 null + * @param statisticsTables 查询实际引用的统计表键 + * @param statisticsFingerprint 查询级统计指纹 + * @param statisticsValidUntil 查询级统计最早失效时间 + */ + DefaultFederationSqlPlan( + FederationQueryScopeDefinition queryScope, + FederationQueryMode queryMode, + String normalizedSql, + String executableSql, + SqlNode sqlNode, + RelRoot relRoot, + int parameterCount, + List parameterJdbcTypes, + List parameterMapping, + List fragments, + List joinOptimizations, + List sourceRuntimeIdentities, + List columns, + Set referencedSources, + AdapterCompatibility compatibility, + boolean executable, + Map compiledFragments, + String executionRelationalPlan, + Bindable localBindable, + SchemaPlus localRootSchema, + Set statisticsTables, + String statisticsFingerprint, + Instant statisticsValidUntil + ) { + if (fragments == null || fragments.isEmpty() + || sourceRuntimeIdentities == null || sourceRuntimeIdentities.isEmpty()) { + throw new IllegalArgumentException("plan fragments and runtime identities must not be empty"); + } + FederationSourceRuntimeIdentity primary = sourceRuntimeIdentities.stream() + .filter(identity -> identity.bindingName().equals(queryScope.defaultBinding())) + .findFirst() + .orElse(sourceRuntimeIdentities.get(0)); + this.sourceId = primary.sourceId(); + this.sourceRevision = primary.sourceRevision(); + this.queryScope = queryScope; + this.queryMode = queryMode; + this.normalizedSql = normalizedSql; + this.executableSql = executableSql; + this.sqlNode = sqlNode; + this.relRoot = relRoot; + this.parameterCount = parameterCount; + this.parameterJdbcTypes = List.copyOf(parameterJdbcTypes); + this.parameterMapping = List.copyOf(parameterMapping); + this.fragments = List.copyOf(fragments); + this.joinOptimizations = List.copyOf( + joinOptimizations == null ? List.of() : joinOptimizations + ); + this.sourceRuntimeIdentities = List.copyOf(sourceRuntimeIdentities); + this.scopeChecksum = queryScope.checksum(); + this.columns = List.copyOf(columns); + this.referencedSources = Set.copyOf(referencedSources); + this.compatibility = compatibility; + this.executable = executable; + this.sourceChecksum = primary.sourceChecksum(); + this.adapterId = primary.adapterId(); + this.runtimeFingerprint = primary.runtimeFingerprint(); + this.statisticsTables = Set.copyOf( + statisticsTables == null ? Set.of() : statisticsTables + ); + this.statisticsFingerprint = statisticsFingerprint == null + ? "" + : statisticsFingerprint; + this.statisticsValidUntil = statisticsValidUntil == null + ? Instant.MAX + : statisticsValidUntil; + this.compiledFragments = Map.copyOf(new LinkedHashMap<>(compiledFragments)); + this.executionRelationalPlan = executionRelationalPlan; + this.localBindable = localBindable; + this.localRootSchema = localRootSchema; + } + + /** + * 保留 Core 内部缓存测试使用的单源兼容构造器。 + */ + DefaultFederationSqlPlan( + SourceId sourceId, + long sourceRevision, + String normalizedSql, + String executableSql, + SqlNode sqlNode, + RelRoot relRoot, + int parameterCount, + List parameterMapping, + List columns, + Set referencedSources, + AdapterCompatibility compatibility, + boolean executable, + String sourceChecksum, + String adapterId, + String runtimeFingerprint + ) { + this( + sourceId, + sourceRevision, + normalizedSql, + executableSql, + sqlNode, + relRoot, + parameterCount, + parameterMapping, + columns, + referencedSources, + compatibility, + executable, + sourceChecksum, + adapterId, + runtimeFingerprint, + Instant.MAX + ); + } + + /** + * 保留 Core 内部统计缓存有效期测试使用的单源兼容构造器。 + * + * @param sourceId 主数据源 + * @param sourceRevision 数据源版本 + * @param normalizedSql 规范化 SQL + * @param executableSql 可执行 SQL + * @param sqlNode SQL 节点 + * @param relRoot 关系计划 + * @param parameterCount 参数数量 + * @param parameterMapping 参数映射 + * @param columns 结果列 + * @param referencedSources 引用源 + * @param compatibility Adapter 兼容性 + * @param executable 是否可执行 + * @param sourceChecksum 源校验和 + * @param adapterId Adapter 标识 + * @param runtimeFingerprint 运行指纹 + * @param statisticsValidUntil 统计最早失效时间 + */ + DefaultFederationSqlPlan( + SourceId sourceId, + long sourceRevision, + String normalizedSql, + String executableSql, + SqlNode sqlNode, + RelRoot relRoot, + int parameterCount, + List parameterMapping, + List columns, + Set referencedSources, + AdapterCompatibility compatibility, + boolean executable, + String sourceChecksum, + String adapterId, + String runtimeFingerprint, + Instant statisticsValidUntil + ) { + this( + FederationQueryScopeDefinition.single(sourceId, sourceRevision), + FederationQueryMode.SINGLE_SOURCE, + normalizedSql, + executableSql, + sqlNode, + relRoot, + parameterCount, + List.of(), + parameterMapping, + List.of(new FederationFragmentPlan( + "fragment-1", + sourceId.value(), + sourceId, + executableSql, + parameterMapping, + columns + )), + List.of(), + List.of(new FederationSourceRuntimeIdentity( + sourceId.value(), + sourceId, + sourceRevision, + sourceChecksum, + adapterId, + runtimeFingerprint + )), + columns, + referencedSources, + compatibility, + executable, + Map.of(), + "", + null, + null, + Set.of(), + "", + statisticsValidUntil + ); + } + + /** + * 为一次公开 compile 调用创建独立签发实例,并共享不可变编译产物。 + * + * @return 独立计划签发实例 + */ + DefaultFederationSqlPlan issuedCopy() { + return new DefaultFederationSqlPlan(this); + } + + /** + * 复制缓存计划的不可变引用,避免热命中重复计算摘要或复制集合。 + * + * @param cached 缓存中的编译计划 + */ + private DefaultFederationSqlPlan(DefaultFederationSqlPlan cached) { + this.sourceId = cached.sourceId; + this.queryScope = cached.queryScope; + this.queryMode = cached.queryMode; + this.sourceRevision = cached.sourceRevision; + this.normalizedSql = cached.normalizedSql; + this.executableSql = cached.executableSql; + this.sqlNode = cached.sqlNode; + this.relRoot = cached.relRoot; + this.parameterCount = cached.parameterCount; + this.parameterJdbcTypes = cached.parameterJdbcTypes; + this.parameterMapping = cached.parameterMapping; + this.fragments = cached.fragments; + this.joinOptimizations = cached.joinOptimizations; + this.sourceRuntimeIdentities = cached.sourceRuntimeIdentities; + this.scopeChecksum = cached.scopeChecksum; + this.columns = cached.columns; + this.referencedSources = cached.referencedSources; + this.compatibility = cached.compatibility; + this.executable = cached.executable; + this.sourceChecksum = cached.sourceChecksum; + this.adapterId = cached.adapterId; + this.runtimeFingerprint = cached.runtimeFingerprint; + this.statisticsTables = cached.statisticsTables; + this.statisticsFingerprint = cached.statisticsFingerprint; + this.statisticsValidUntil = cached.statisticsValidUntil; + this.compiledFragments = cached.compiledFragments; + this.executionRelationalPlan = cached.executionRelationalPlan; + this.localBindable = cached.localBindable; + this.localRootSchema = cached.localRootSchema; + } + + /** {@inheritDoc} */ + @Override + public SourceId sourceId() { + return sourceId; + } + + /** {@inheritDoc} */ + @Override + public FederationQueryScopeDefinition queryScope() { + return queryScope; + } + + /** {@inheritDoc} */ + @Override + public FederationQueryMode queryMode() { + return queryMode; + } + + /** {@inheritDoc} */ + @Override + public long sourceRevision() { + return sourceRevision; + } + + /** {@inheritDoc} */ + @Override + public String normalizedSql() { + return normalizedSql; + } + + /** {@inheritDoc} */ + @Override + public String executableSql() { + return executableSql; + } + + /** {@inheritDoc} */ + @Override + public SqlNode sqlNode() { + return sqlNode; + } + + /** {@inheritDoc} */ + @Override + public RelRoot relRoot() { + return relRoot; + } + + /** {@inheritDoc} */ + @Override + public int parameterCount() { + return parameterCount; + } + + /** {@inheritDoc} */ + @Override + public List parameterJdbcTypes() { + return parameterJdbcTypes; + } + + /** {@inheritDoc} */ + @Override + public List parameterMapping() { + return parameterMapping; + } + + /** {@inheritDoc} */ + @Override + public List fragments() { + return fragments; + } + + /** {@inheritDoc} */ + @Override + public List joinOptimizations() { + return joinOptimizations; + } + + /** {@inheritDoc} */ + @Override + public List sourceRuntimeIdentities() { + return sourceRuntimeIdentities; + } + + /** {@inheritDoc} */ + @Override + public String scopeChecksum() { + return scopeChecksum; + } + + /** {@inheritDoc} */ + @Override + public List columns() { + return columns; + } + + /** {@inheritDoc} */ + @Override + public Set referencedSources() { + return referencedSources; + } + + /** {@inheritDoc} */ + @Override + public AdapterCompatibility compatibility() { + return compatibility; + } + + /** {@inheritDoc} */ + @Override + public boolean executable() { + return executable; + } + + /** {@inheritDoc} */ + @Override + public String sourceChecksum() { + return sourceChecksum; + } + + /** {@inheritDoc} */ + @Override + public String adapterId() { + return adapterId; + } + + /** {@inheritDoc} */ + @Override + public String runtimeFingerprint() { + return runtimeFingerprint; + } + + /** {@inheritDoc} */ + @Override + public Instant statisticsValidUntil() { + return statisticsValidUntil; + } + + /** + * 返回该计划实际引用的统计表键。 + * + * @return 不可变统计表键集合 + */ + Set statisticsTables() { + return statisticsTables; + } + + /** + * 返回编译时查询级统计指纹。 + * + * @return 统计指纹 + */ + String statisticsFingerprint() { + return statisticsFingerprint; + } + + /** + * 返回内部编译分片。 + * + * @return 分片映射 + */ + Map compiledFragments() { + return compiledFragments; + } + + /** + * 返回实际执行的关系计划文本。 + * + * @return 单源下推或联邦本地计划 + */ + String executionRelationalPlan() { + return executionRelationalPlan; + } + + /** + * 返回联邦本地 Bindable。 + * + * @return Bindable;单源计划返回 null + */ + Bindable localBindable() { + return localBindable; + } + + /** + * 返回联邦本地分片占位表根 Schema。 + * + * @return 根 Schema;单源计划返回 null + */ + SchemaPlus localRootSchema() { + return localRootSchema; + } +} diff --git a/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/runtime/EnumerableFederationBudgetRel.java b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/runtime/EnumerableFederationBudgetRel.java new file mode 100644 index 0000000..63c2ffa --- /dev/null +++ b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/runtime/EnumerableFederationBudgetRel.java @@ -0,0 +1,61 @@ +package com.easyagents.federation.sql.runtime; + +import java.util.List; +import org.apache.calcite.DataContext; +import org.apache.calcite.adapter.enumerable.EnumerableConvention; +import org.apache.calcite.adapter.enumerable.EnumerableRel; +import org.apache.calcite.adapter.enumerable.EnumerableRelImplementor; +import org.apache.calcite.linq4j.tree.BlockBuilder; +import org.apache.calcite.linq4j.tree.Expression; +import org.apache.calcite.linq4j.tree.Expressions; +import org.apache.calcite.plan.RelTraitSet; +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.SingleRel; + +/** + * 在 Calcite Enumerable 本地算子输出边界插入查询预算检查。 + */ +final class EnumerableFederationBudgetRel extends SingleRel implements EnumerableRel { + + private final String operatorName; + + /** + * 创建预算边界。 + * + * @param input 已转换为 Enumerable Convention 的输入 + * @param operatorName 被计量的本地算子名称 + */ + EnumerableFederationBudgetRel(RelNode input, String operatorName) { + super(input.getCluster(), input.getTraitSet(), input); + if (input.getConvention() != EnumerableConvention.INSTANCE) { + throw new IllegalArgumentException("budget input must use Enumerable convention"); + } + this.operatorName = operatorName; + } + + /** {@inheritDoc} */ + @Override + public EnumerableFederationBudgetRel copy( + RelTraitSet traitSet, + List inputs + ) { + return new EnumerableFederationBudgetRel(sole(inputs), operatorName); + } + + /** {@inheritDoc} */ + @Override + public Result implement(EnumerableRelImplementor implementor, Prefer pref) { + BlockBuilder builder = new BlockBuilder(); + Result child = implementor.visitChild(this, 0, (EnumerableRel) getInput(), pref); + Expression enumerable = builder.append("budgetInput", child.block); + Expression guarded = Expressions.call( + FederationBudgetEnumerable.class, + "wrap", + enumerable, + DataContext.ROOT, + Expressions.constant(operatorName) + ); + builder.add(Expressions.return_(null, guarded)); + return implementor.result(child.physType, builder.toBlock()); + } +} diff --git a/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/runtime/FederatedResultCursor.java b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/runtime/FederatedResultCursor.java new file mode 100644 index 0000000..82f80ee --- /dev/null +++ b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/runtime/FederatedResultCursor.java @@ -0,0 +1,275 @@ +package com.easyagents.federation.sql.runtime; + +import com.easyagents.federation.sql.api.FederationSqlErrorCode; +import com.easyagents.federation.sql.api.FederationSqlException; +import com.easyagents.federation.sql.execute.FederationColumn; +import com.easyagents.federation.sql.execute.FederationQueryMetricsSnapshot; +import com.easyagents.federation.sql.execute.FederationResultCursor; +import com.easyagents.federation.sql.execute.QueryId; +import java.io.ByteArrayInputStream; +import java.io.InputStream; +import java.io.Reader; +import java.io.StringReader; +import java.sql.Date; +import java.sql.Time; +import java.sql.Timestamp; +import java.time.Instant; +import java.time.LocalTime; +import java.time.OffsetDateTime; +import java.time.OffsetTime; +import java.time.ZoneOffset; +import java.util.Arrays; +import java.util.List; +import java.util.concurrent.atomic.AtomicBoolean; +import org.apache.calcite.avatica.util.ByteString; +import org.apache.calcite.adapter.java.JavaTypeFactory; +import org.apache.calcite.linq4j.Enumerator; + +/** + * 将 Calcite 本地 Bindable 暴露为统一流式结果游标。 + */ +final class FederatedResultCursor + implements FederationResultCursor, CancellationAwareFederationCursor { + + private final DefaultFederationSqlPlan plan; + private final QueryId queryId; + private final FederationExecutionSession session; + private final FederationQueryMetricsTracker metrics; + private final int maximumRows; + private final Enumerator enumerator; + private final AtomicBoolean closed = new AtomicBoolean(); + private long rows; + private Object[] current; + + /** + * 创建联邦结果游标并绑定本次查询 DataContext。 + * + * @param plan 联邦计划 + * @param queryId 查询标识 + * @param session 执行会话 + * @param metrics 指标跟踪器 + * @param maximumRows 最终结果上限,0 表示不额外限制 + */ + FederatedResultCursor( + DefaultFederationSqlPlan plan, + QueryId queryId, + FederationExecutionSession session, + FederationQueryMetricsTracker metrics, + int maximumRows + ) { + this.plan = plan; + this.queryId = queryId; + this.session = session; + this.metrics = metrics; + this.maximumRows = maximumRows; + this.enumerator = plan.localBindable() + .bind(new FederationDataContext( + plan.localRootSchema(), + (JavaTypeFactory) plan.relRoot().rel.getCluster().getTypeFactory(), + session.dataContextValues() + )) + .enumerator(); + } + + /** {@inheritDoc} */ + @Override + public QueryId queryId() { + return queryId; + } + + /** {@inheritDoc} */ + @Override + public List columns() { + return plan.columns(); + } + + /** {@inheritDoc} */ + @Override + public FederationQueryMetricsSnapshot metrics() { + return metrics.snapshot(); + } + + /** {@inheritDoc} */ + @Override + public void markCancelled() { + metrics.markCancelled(); + } + + /** {@inheritDoc} */ + @Override + public void markTimedOut() { + metrics.markTimedOut(); + } + + /** {@inheritDoc} */ + @Override + public void markFailed(FederationSqlErrorCode errorCode) { + metrics.markFailed(errorCode); + } + + /** {@inheritDoc} */ + @Override + public boolean next() { + ensureOpen(); + session.ensureExecutionAllowed(); + if (maximumRows > 0 && rows >= maximumRows) { + metrics.markTruncated(); + finish(); + return false; + } + try { + if (!enumerator.moveNext()) { + finish(); + return false; + } + Object value = enumerator.current(); + Object[] row = value instanceof Object[] values ? values : new Object[] {value}; + current = denormalize(row, plan.columns()); + rows++; + metrics.recordOutput(current); + return true; + } catch (RuntimeException exception) { + if (exception instanceof FederationSqlException sqlException + && sqlException.errorCode() == FederationSqlErrorCode.QUERY_CANCELLED) { + metrics.markCancelled(); + } + closeAndSuppress(exception); + throw exception; + } + } + + /** {@inheritDoc} */ + @Override + public Object getObject(int columnIndex) { + ensureCurrent(columnIndex); + return current[columnIndex - 1]; + } + + /** {@inheritDoc} */ + @Override + public InputStream getBinaryStream(int columnIndex) { + Object value = getObject(columnIndex); + if (value == null) { + return null; + } + if (value instanceof byte[] bytes) { + return new ByteArrayInputStream(bytes); + } + throw new UnsupportedOperationException("column is not a binary value"); + } + + /** {@inheritDoc} */ + @Override + public Reader getCharacterStream(int columnIndex) { + Object value = getObject(columnIndex); + return value == null ? null : new StringReader(value.toString()); + } + + /** {@inheritDoc} */ + @Override + public List row() { + if (current == null) { + throw new IllegalStateException("cursor is not positioned on a row"); + } + return java.util.Collections.unmodifiableList(Arrays.asList(current.clone())); + } + + /** + * 关闭本地 Enumerator 和全部分片游标。 + */ + @Override + public void close() { + if (!closed.compareAndSet(false, true)) { + return; + } + RuntimeException failure = null; + try { + try { + enumerator.close(); + } catch (RuntimeException exception) { + failure = exception; + } + try { + session.close(); + } catch (RuntimeException exception) { + if (failure == null) { + failure = exception; + } else { + failure.addSuppressed(exception); + } + } + } finally { + metrics.finishClosed(); + } + if (failure != null) { + throw failure; + } + } + + private void finish() { + metrics.finishSuccessfully(); + close(); + } + + private void ensureOpen() { + if (closed.get()) { + throw new IllegalStateException("cursor is closed"); + } + } + + private void ensureCurrent(int columnIndex) { + if (current == null) { + throw new IllegalStateException("cursor is not positioned on a row"); + } + if (columnIndex <= 0 || columnIndex > current.length) { + throw new IndexOutOfBoundsException("column index out of range: " + columnIndex); + } + } + + private void closeAndSuppress(RuntimeException original) { + try { + close(); + } catch (RuntimeException closeFailure) { + original.addSuppressed(closeFailure); + } + } + + private static Object[] denormalize( + Object[] values, + List columns + ) { + Object[] converted = values.clone(); + for (int index = 0; index < converted.length; index++) { + Object value = converted[index]; + if (value == null) { + continue; + } + converted[index] = switch (columns.get(index).jdbcType()) { + case java.sql.Types.DATE -> value instanceof Integer days + ? Date.valueOf(java.time.LocalDate.ofEpochDay(days)) + : value; + case java.sql.Types.TIME -> value instanceof Integer millis + ? Time.valueOf(LocalTime.ofNanoOfDay(millis * 1_000_000L)) + : value; + case java.sql.Types.TIME_WITH_TIMEZONE -> value instanceof Integer millis + ? OffsetTime.of( + LocalTime.ofNanoOfDay(millis * 1_000_000L), + ZoneOffset.UTC + ) + : value; + case java.sql.Types.TIMESTAMP -> value instanceof Long millis + ? new Timestamp(millis) + : value; + case java.sql.Types.TIMESTAMP_WITH_TIMEZONE -> value instanceof Long millis + ? OffsetDateTime.ofInstant(Instant.ofEpochMilli(millis), ZoneOffset.UTC) + : value; + case java.sql.Types.BINARY, java.sql.Types.VARBINARY, + java.sql.Types.LONGVARBINARY -> value instanceof ByteString bytes + ? bytes.getBytes() + : value; + default -> value; + }; + } + return converted; + } +} diff --git a/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/runtime/FederationBudgetEnumerable.java b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/runtime/FederationBudgetEnumerable.java new file mode 100644 index 0000000..59eb6ee --- /dev/null +++ b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/runtime/FederationBudgetEnumerable.java @@ -0,0 +1,75 @@ +package com.easyagents.federation.sql.runtime; + +import org.apache.calcite.DataContext; +import org.apache.calcite.linq4j.AbstractEnumerable; +import org.apache.calcite.linq4j.Enumerable; +import org.apache.calcite.linq4j.Enumerator; + +/** + * 为生成的 Calcite Enumerable 提供运行期本地算子预算检查。 + */ +public final class FederationBudgetEnumerable { + + /** + * 禁止实例化运行期桥接工具。 + */ + private FederationBudgetEnumerable() { + } + + /** + * 包装一个本地算子输出,在每行交给下游前计量资源并检查取消与时限。 + * + * @param input Calcite 算子输出 + * @param context 当前查询 DataContext + * @param operatorName 本地算子名称 + * @param Calcite 内部行类型 + * @return 带预算检查的 Enumerable + */ + public static Enumerable wrap( + Enumerable input, + DataContext context, + String operatorName + ) { + Object value = context.get(FederationExecutionSession.DATA_CONTEXT_KEY); + if (!(value instanceof FederationExecutionSession session)) { + throw new IllegalStateException("federation execution session is missing"); + } + return new AbstractEnumerable<>() { + @Override + public Enumerator enumerator() { + Enumerator delegate = input.enumerator(); + return new Enumerator<>() { + @Override + public T current() { + return delegate.current(); + } + + @Override + public boolean moveNext() { + session.ensureExecutionAllowed(); + long started = System.nanoTime(); + if (!delegate.moveNext()) { + return false; + } + session.recordLocalIntermediate( + operatorName, + delegate.current(), + System.nanoTime() - started + ); + return true; + } + + @Override + public void reset() { + delegate.reset(); + } + + @Override + public void close() { + delegate.close(); + } + }; + } + }; + } +} diff --git a/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/runtime/FederationDataContext.java b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/runtime/FederationDataContext.java new file mode 100644 index 0000000..f6b1faa --- /dev/null +++ b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/runtime/FederationDataContext.java @@ -0,0 +1,58 @@ +package com.easyagents.federation.sql.runtime; + +import java.util.Map; +import org.apache.calcite.DataContext; +import org.apache.calcite.adapter.java.JavaTypeFactory; +import org.apache.calcite.linq4j.QueryProvider; +import org.apache.calcite.schema.SchemaPlus; + +/** + * 为节点本地 Bindable 提供分片 Schema、动态参数和执行会话。 + */ +final class FederationDataContext implements DataContext { + + private final SchemaPlus rootSchema; + private final JavaTypeFactory typeFactory; + private final Map values; + + /** + * 创建联邦 DataContext。 + * + * @param rootSchema 分片占位表所在根 Schema + * @param typeFactory Calcite Java 类型工厂 + * @param values 查询参数和执行会话 + */ + FederationDataContext( + SchemaPlus rootSchema, + JavaTypeFactory typeFactory, + Map values + ) { + this.rootSchema = rootSchema; + this.typeFactory = typeFactory; + this.values = values; + } + + /** {@inheritDoc} */ + @Override + public SchemaPlus getRootSchema() { + return rootSchema; + } + + /** {@inheritDoc} */ + @Override + public JavaTypeFactory getTypeFactory() { + return typeFactory; + } + + /** {@inheritDoc} */ + @Override + public QueryProvider getQueryProvider() { + throw new UnsupportedOperationException("query provider is not required for fragment scans"); + } + + /** {@inheritDoc} */ + @Override + public Object get(String name) { + return values.get(name); + } +} diff --git a/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/runtime/FederationExecutionSession.java b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/runtime/FederationExecutionSession.java new file mode 100644 index 0000000..2f9a58a --- /dev/null +++ b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/runtime/FederationExecutionSession.java @@ -0,0 +1,516 @@ +package com.easyagents.federation.sql.runtime; + +import com.easyagents.federation.sql.api.FederationSqlErrorCode; +import com.easyagents.federation.sql.api.FederationSqlException; +import com.easyagents.federation.sql.api.SqlExecutionContext; +import com.easyagents.federation.sql.execute.FederationFragmentExecutionContext; +import com.easyagents.federation.sql.execute.FederationResultCursor; +import com.easyagents.federation.sql.execute.SqlExecutionOptions; +import com.easyagents.federation.sql.execute.SqlParameter; +import com.easyagents.federation.sql.federation.FederationExecutionPolicy; +import com.easyagents.federation.sql.federation.FederationFragmentPlan; +import java.math.BigDecimal; +import java.sql.Date; +import java.sql.Time; +import java.sql.Timestamp; +import java.sql.Types; +import java.time.Instant; +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.time.LocalTime; +import java.time.OffsetDateTime; +import java.time.ZoneOffset; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.Semaphore; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import org.apache.calcite.avatica.util.ByteString; +import org.apache.calcite.linq4j.AbstractEnumerable; +import org.apache.calcite.linq4j.Enumerable; +import org.apache.calcite.linq4j.Enumerator; +import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.rel.type.RelDataTypeField; +import org.apache.calcite.sql.type.SqlTypeName; + +/** + * 一次联邦查询的分片执行、类型桥接、预算和取消上下文。 + */ +final class FederationExecutionSession implements AutoCloseable { + + static final String DATA_CONTEXT_KEY = "easyagents.federation.session"; + + private final DefaultFederationSqlPlan plan; + private final SqlExecutionContext context; + private final FederationQueryScopeSnapshot runtimeSnapshot; + private final QueryCancellationRegistry.QueryRegistration registration; + private final FederationExecutionPolicy policy; + private final FederationQueryMetricsTracker metrics; + private final QueryDeadline deadline; + private final Semaphore fragmentSlots; + private final Set openFragmentCursors = ConcurrentHashMap.newKeySet(); + private final AtomicBoolean closed = new AtomicBoolean(); + + /** + * 创建查询执行会话。 + * + * @param plan 联邦计划 + * @param context 执行上下文 + * @param runtimeSnapshot 实际引用源 Runtime 快照 + * @param registration 查询取消登记 + * @param policy 有效资源策略 + * @param metrics 指标跟踪器 + * @param deadline 请求级统一截止时间 + */ + FederationExecutionSession( + DefaultFederationSqlPlan plan, + SqlExecutionContext context, + FederationQueryScopeSnapshot runtimeSnapshot, + QueryCancellationRegistry.QueryRegistration registration, + FederationExecutionPolicy policy, + FederationQueryMetricsTracker metrics, + QueryDeadline deadline + ) { + this.plan = plan; + this.context = context; + this.runtimeSnapshot = runtimeSnapshot; + this.registration = registration; + this.policy = policy; + this.metrics = metrics; + this.deadline = deadline; + this.fragmentSlots = new Semaphore(policy.maximumConcurrentFragments(), true); + } + + /** + * 为 Calcite ScannableTable 创建一个按需执行物理分片的 Enumerable。 + * + * @param fragmentId 分片标识 + * @return 分片行 Enumerable + */ + Enumerable scan(String fragmentId) { + CompiledFederationFragment compiled = plan.compiledFragments().get(fragmentId); + if (compiled == null) { + throw new FederationSqlException( + FederationSqlErrorCode.EXECUTION_FAILED, + "compiled fragment is missing: " + fragmentId + ); + } + return new AbstractEnumerable<>() { + @Override + public Enumerator enumerator() { + return openFragment(compiled); + } + }; + } + + /** + * 返回 Calcite DataContext 使用的查询参数值。 + * + * @return 参数键值 + */ + Map dataContextValues() { + Map values = new java.util.LinkedHashMap<>(); + values.put(DATA_CONTEXT_KEY, this); + for (int index = 0; index < context.parameters().size(); index++) { + values.put("?" + index, normalizeParameter(context.parameters().get(index))); + } + return java.util.Collections.unmodifiableMap(values); + } + + /** + * 在本地算子或分片读取前检查取消、关闭和总时限。 + */ + void ensureExecutionAllowed() { + if (closed.get()) { + throw new FederationSqlException( + FederationSqlErrorCode.EXECUTION_FAILED, + "federation execution session is closed" + ); + } + if (registration.cancellationRequested()) { + if (registration.timeoutRequested()) { + metrics.markTimedOut(); + } else { + metrics.markCancelled(); + } + registration.ensureNotCancelled(); + } + deadline.ensureAllowed(); + } + + /** + * 计量一个 Calcite 本地算子输出行并执行统一硬预算检查。 + * + * @param operatorName 本地算子名称 + * @param value Calcite 内部行值 + * @param elapsedNanos 算子产生当前行的耗时 + */ + void recordLocalIntermediate(String operatorName, Object value, long elapsedNanos) { + ensureExecutionAllowed(); + Object[] row = value instanceof Object[] values ? values : new Object[] {value}; + FederationQueryMetricsTracker.IntermediateUsage usage = + metrics.recordLocalIntermediate(operatorName, row, elapsedNanos); + if (usage.rows() > policy.maximumIntermediateRows() + || usage.bytes() > policy.maximumIntermediateBytes()) { + throw new FederationSqlException( + FederationSqlErrorCode.FEDERATION_RESOURCE_LIMIT_EXCEEDED, + "federation local operator " + operatorName + + " exceeded the intermediate result limit" + ); + } + } + + private Enumerator openFragment(CompiledFederationFragment compiled) { + ensureExecutionAllowed(); + acquireFragmentSlot(); + FederationFragmentPlan fragment = compiled.plan(); + SourceRuntime runtime = runtimeSnapshot.runtime(fragment.bindingName()); + try { + FederationResultCursor cursor = runtime.adapter().fragmentExecutor().execute( + new FederationFragmentExecutionContext( + context.queryId(), + fragment.executableSql(), + remapParameters(fragment, context.parameters()), + fragmentOptions(context.options()), + runtime.handle().dataSource(), + runtime.compatibility(), + runtime.definition().adapterOptions(), + registration.statementLifecycle(), + metrics.fragmentObserver(fragment.fragmentId()), + deadline + ) + ); + if (cursor == null || !context.queryId().equals(cursor.queryId())) { + closeQuietly(cursor); + throw new FederationSqlException( + FederationSqlErrorCode.EXECUTION_FAILED, + "adapter returned an invalid fragment cursor" + ); + } + openFragmentCursors.add(cursor); + return new FragmentEnumerator(compiled, cursor); + } catch (RuntimeException exception) { + fragmentSlots.release(); + throw exception; + } + } + + private void acquireFragmentSlot() { + while (true) { + ensureExecutionAllowed(); + long remaining = deadline.remainingNanos(); + if (remaining <= 0L) { + ensureExecutionAllowed(); + } + try { + if (fragmentSlots.tryAcquire( + Math.min(remaining, TimeUnit.MILLISECONDS.toNanos(50)), + TimeUnit.NANOSECONDS + )) { + return; + } + } catch (InterruptedException exception) { + Thread.currentThread().interrupt(); + throw new FederationSqlException( + FederationSqlErrorCode.EXECUTION_FAILED, + "waiting for a federation fragment slot was interrupted", + exception + ); + } + } + } + + private SqlExecutionOptions fragmentOptions(SqlExecutionOptions requested) { + // 最终 maxRows 不能截断 Join/Aggregate 输入,否则会改变查询结果。 + return new SqlExecutionOptions( + requested.fetchSize(), + 0, + effectiveJdbcTimeoutSeconds(requested.queryTimeoutSeconds()), + requested.readOnly() + ); + } + + private int effectiveJdbcTimeoutSeconds(int requestedSeconds) { + return deadline.boundedQueryTimeoutSeconds(requestedSeconds); + } + + private static List remapParameters( + FederationFragmentPlan fragment, + List parameters + ) { + if (fragment.parameterMapping().isEmpty()) { + return List.of(); + } + List remapped = new ArrayList<>(fragment.parameterMapping().size()); + for (Integer index : fragment.parameterMapping()) { + remapped.add(parameters.get(index)); + } + return List.copyOf(remapped); + } + + /** + * 关闭所有仍活动的物理分片游标。 + */ + @Override + public void close() { + if (!closed.compareAndSet(false, true)) { + return; + } + RuntimeException failure = null; + for (FederationResultCursor cursor : List.copyOf(openFragmentCursors)) { + try { + cursor.close(); + } catch (RuntimeException exception) { + if (failure == null) { + failure = exception; + } else { + failure.addSuppressed(exception); + } + } finally { + openFragmentCursors.remove(cursor); + } + } + if (failure != null) { + throw failure; + } + } + + private final class FragmentEnumerator implements Enumerator { + + private final CompiledFederationFragment compiled; + private final FederationResultCursor cursor; + private final AtomicBoolean released = new AtomicBoolean(); + private Object[] current; + + private FragmentEnumerator( + CompiledFederationFragment compiled, + FederationResultCursor cursor + ) { + this.compiled = compiled; + this.cursor = cursor; + } + + /** {@inheritDoc} */ + @Override + public Object[] current() { + return current; + } + + /** {@inheritDoc} */ + @Override + public boolean moveNext() { + ensureExecutionAllowed(); + try { + if (!cursor.next()) { + release(true); + return false; + } + Object[] row = normalizeRow(cursor, compiled.rowType()); + FederationQueryMetricsTracker.IntermediateUsage usage = + metrics.recordIntermediate(compiled.plan().fragmentId(), row); + if (usage.rows() > policy.maximumIntermediateRows() + || usage.bytes() > policy.maximumIntermediateBytes()) { + throw new FederationSqlException( + FederationSqlErrorCode.FEDERATION_RESOURCE_LIMIT_EXCEEDED, + "federation intermediate result limit was exceeded" + ); + } + current = row; + return true; + } catch (RuntimeException exception) { + release(false); + throw exception; + } + } + + /** {@inheritDoc} */ + @Override + public void reset() { + throw new UnsupportedOperationException("fragment cursor cannot be reset"); + } + + /** {@inheritDoc} */ + @Override + public void close() { + release(false); + } + + private void release(boolean exhausted) { + if (!released.compareAndSet(false, true)) { + return; + } + openFragmentCursors.remove(cursor); + try { + cursor.close(); + } finally { + fragmentSlots.release(); + if (exhausted) { + metrics.finishFragment(compiled.plan().fragmentId()); + } + } + } + } + + private static Object[] normalizeRow( + FederationResultCursor cursor, + RelDataType rowType + ) { + List fields = rowType.getFieldList(); + Object[] values = new Object[fields.size()]; + for (int index = 0; index < fields.size(); index++) { + values[index] = normalizeValue( + cursor.getObject(index + 1), + fields.get(index).getType().getSqlTypeName() + ); + } + return values; + } + + private static Object normalizeValue(Object value, SqlTypeName typeName) { + if (value == null) { + return null; + } + return switch (typeName) { + case BOOLEAN -> (Boolean) value; + case TINYINT -> ((Number) value).byteValue(); + case SMALLINT -> ((Number) value).shortValue(); + case INTEGER -> ((Number) value).intValue(); + case BIGINT -> ((Number) value).longValue(); + case REAL -> ((Number) value).floatValue(); + case FLOAT, DOUBLE -> ((Number) value).doubleValue(); + case DECIMAL -> value instanceof BigDecimal + ? value + : new BigDecimal(value.toString()); + case DATE -> normalizeDate(value); + case TIME, TIME_WITH_LOCAL_TIME_ZONE, TIME_TZ -> normalizeTime(value); + case TIMESTAMP, TIMESTAMP_WITH_LOCAL_TIME_ZONE, TIMESTAMP_TZ -> + normalizeTimestamp(value); + case BINARY, VARBINARY -> value instanceof ByteString + ? value + : new ByteString((byte[]) value); + case CHAR, VARCHAR -> value.toString(); + case UUID -> value instanceof java.util.UUID + ? value + : java.util.UUID.fromString(value.toString()); + case ANY, OTHER -> normalizeKnownUnknownScalar(value); + case ARRAY, MULTISET, MAP, ROW -> throw new FederationSqlException( + FederationSqlErrorCode.FEDERATION_OPERATOR_UNSUPPORTED, + "complex JDBC values are not supported by local federation operators" + ); + default -> throw new FederationSqlException( + FederationSqlErrorCode.FEDERATION_OPERATOR_UNSUPPORTED, + "JDBC type cannot be safely normalized for local federation: " + typeName + ); + }; + } + + /** + * 接受驱动元数据无法精确声明、但仍可稳定序列化和计量的标准 JDBC 标量。 + * + * @param value JDBC 返回值 + * @return 可供本地 Enumerable 透传的值 + * @throws FederationSqlException 值不在安全白名单内 + */ + private static Object normalizeKnownUnknownScalar(Object value) { + if (value instanceof java.time.OffsetTime offsetTime) { + return offsetTime.withOffsetSameInstant(ZoneOffset.UTC); + } + if (value instanceof java.time.OffsetDateTime offsetDateTime) { + return offsetDateTime.withOffsetSameInstant(ZoneOffset.UTC); + } + if (value instanceof java.time.LocalDate + || value instanceof java.time.LocalTime + || value instanceof java.time.LocalDateTime + || value instanceof java.time.Instant + || value instanceof java.sql.Date + || value instanceof java.sql.Time + || value instanceof java.sql.Timestamp + || value instanceof java.util.UUID) { + return value; + } + throw new FederationSqlException( + FederationSqlErrorCode.FEDERATION_OPERATOR_UNSUPPORTED, + "JDBC value cannot be safely normalized for local federation: " + + value.getClass().getName() + ); + } + + private static Integer normalizeDate(Object value) { + LocalDate date = value instanceof Date sqlDate + ? sqlDate.toLocalDate() + : (LocalDate) value; + return Math.toIntExact(date.toEpochDay()); + } + + private static Integer normalizeTime(Object value) { + LocalTime time; + if (value instanceof Time sqlTime) { + time = sqlTime.toLocalTime(); + } else if (value instanceof java.time.OffsetTime offsetTime) { + time = offsetTime.withOffsetSameInstant(ZoneOffset.UTC).toLocalTime(); + } else { + time = (LocalTime) value; + } + requireMillisecondPrecision(time.getNano(), "TIME"); + return Math.toIntExact(time.toNanoOfDay() / 1_000_000L); + } + + private static Long normalizeTimestamp(Object value) { + if (value instanceof Timestamp timestamp) { + requireMillisecondPrecision(timestamp.getNanos(), "TIMESTAMP"); + return timestamp.getTime(); + } + if (value instanceof Instant instant) { + requireMillisecondPrecision(instant.getNano(), "TIMESTAMP"); + return instant.toEpochMilli(); + } + if (value instanceof OffsetDateTime offset) { + requireMillisecondPrecision(offset.getNano(), "TIMESTAMP"); + return offset.toInstant().toEpochMilli(); + } + LocalDateTime local = (LocalDateTime) value; + requireMillisecondPrecision(local.getNano(), "TIMESTAMP"); + return local.toInstant(ZoneOffset.UTC).toEpochMilli(); + } + + /** + * 拒绝 Calcite 毫秒内部表示无法保真的时间值。 + * + * @param nanos 秒内纳秒值 + * @param typeName SQL 时间类型名 + * @throws FederationSqlException 时间值包含亚毫秒精度 + */ + private static void requireMillisecondPrecision(int nanos, String typeName) { + if (nanos % 1_000_000 != 0) { + throw new FederationSqlException( + FederationSqlErrorCode.FEDERATION_OPERATOR_UNSUPPORTED, + typeName + " values with precision above milliseconds are not supported " + + "by local federation" + ); + } + } + + private static Object normalizeParameter(SqlParameter parameter) { + Object value = parameter.value(); + if (value == null) { + return null; + } + return switch (parameter.jdbcType()) { + case Types.DATE -> normalizeDate(value); + case Types.TIME, Types.TIME_WITH_TIMEZONE -> normalizeTime(value); + case Types.TIMESTAMP, Types.TIMESTAMP_WITH_TIMEZONE -> normalizeTimestamp(value); + case Types.BINARY, Types.VARBINARY, Types.LONGVARBINARY -> + value instanceof ByteString ? value : new ByteString((byte[]) value); + default -> value; + }; + } + + private static void closeQuietly(FederationResultCursor cursor) { + if (cursor != null) { + cursor.close(); + } + } +} diff --git a/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/runtime/FederationFragmentTable.java b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/runtime/FederationFragmentTable.java new file mode 100644 index 0000000..a425567 --- /dev/null +++ b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/runtime/FederationFragmentTable.java @@ -0,0 +1,70 @@ +package com.easyagents.federation.sql.runtime; + +import com.easyagents.federation.sql.api.FederationSqlErrorCode; +import com.easyagents.federation.sql.api.FederationSqlException; +import org.apache.calcite.DataContext; +import org.apache.calcite.linq4j.Enumerable; +import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.rel.type.RelDataTypeFactory; +import org.apache.calcite.schema.ScannableTable; +import org.apache.calcite.schema.Statistic; +import org.apache.calcite.schema.Statistics; +import org.apache.calcite.schema.impl.AbstractTable; + +/** + * Calcite 本地计划中的无状态分片输入表。 + */ +public final class FederationFragmentTable extends AbstractTable implements ScannableTable { + + private final String fragmentId; + private final RelDataType rowType; + private final double estimatedRows; + + /** + * 创建分片输入表。 + * + * @param fragmentId 分片标识 + * @param rowType Calcite 行类型 + * @param estimatedRows 分片输出估算行数 + */ + FederationFragmentTable(String fragmentId, RelDataType rowType, double estimatedRows) { + this.fragmentId = fragmentId; + this.rowType = rowType; + this.estimatedRows = Math.max(0D, estimatedRows); + } + + /** + * 创建缺少成本统计的兼容分片表。 + * + * @param fragmentId 分片标识 + * @param rowType Calcite 行类型 + */ + FederationFragmentTable(String fragmentId, RelDataType rowType) { + this(fragmentId, rowType, 100D); + } + + /** {@inheritDoc} */ + @Override + public RelDataType getRowType(RelDataTypeFactory typeFactory) { + return rowType; + } + + /** {@inheritDoc} */ + @Override + public Statistic getStatistic() { + return Statistics.of(estimatedRows, java.util.List.of()); + } + + /** {@inheritDoc} */ + @Override + public Enumerable scan(DataContext root) { + Object value = root.get(FederationExecutionSession.DATA_CONTEXT_KEY); + if (!(value instanceof FederationExecutionSession session)) { + throw new FederationSqlException( + FederationSqlErrorCode.EXECUTION_FAILED, + "federation execution session is missing from Calcite DataContext" + ); + } + return session.scan(fragmentId); + } +} diff --git a/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/runtime/FederationQueryMetricsTracker.java b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/runtime/FederationQueryMetricsTracker.java new file mode 100644 index 0000000..f3e93c5 --- /dev/null +++ b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/runtime/FederationQueryMetricsTracker.java @@ -0,0 +1,437 @@ +package com.easyagents.federation.sql.runtime; + +import com.easyagents.federation.sql.api.FederationSqlErrorCode; +import com.easyagents.federation.sql.api.FederationSqlException; +import com.easyagents.federation.sql.execute.FederationFragmentMetrics; +import com.easyagents.federation.sql.execute.FederationExecutionObserver; +import com.easyagents.federation.sql.execute.FederationLocalOperatorMetrics; +import com.easyagents.federation.sql.execute.FederationQueryMetricsSnapshot; +import com.easyagents.federation.sql.execute.QueryId; +import com.easyagents.federation.sql.federation.FederationFragmentPlan; +import com.easyagents.federation.sql.federation.FederationQueryMode; +import java.math.BigDecimal; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.atomic.AtomicReference; +import org.apache.calcite.avatica.util.ByteString; + +/** + * 维护查询级和分片级有界、低开销指标。 + */ +final class FederationQueryMetricsTracker { + + private final QueryId queryId; + private final FederationQueryMode queryMode; + private final boolean planCacheHit; + private final long planningNanos; + private final long executionStartedNanos = System.nanoTime(); + private final AtomicLong executionFinishedNanos = new AtomicLong(-1); + private final AtomicLong firstRowNanos = new AtomicLong(-1); + private final AtomicLong returnedRows = new AtomicLong(); + private final AtomicLong returnedBytes; + private final AtomicLong intermediateRows = new AtomicLong(); + private final AtomicLong intermediateBytes; + private final AtomicLong admissionWaitNanos = new AtomicLong(); + private final AtomicLong connectionAcquireNanos = new AtomicLong(); + private final AtomicLong databaseExecutionNanos = new AtomicLong(); + private final AtomicLong localExecutionNanos = new AtomicLong(); + private final AtomicBoolean complete = new AtomicBoolean(); + private final AtomicBoolean cancelled = new AtomicBoolean(); + private final AtomicBoolean timedOut = new AtomicBoolean(); + private final AtomicBoolean truncated = new AtomicBoolean(); + private final AtomicReference terminalErrorCode = new AtomicReference<>(""); + private final Map fragments = new LinkedHashMap<>(); + private final Map localOperators = new LinkedHashMap<>(); + + /** + * 创建指标跟踪器。 + * + * @param queryId 查询标识 + * @param queryMode 查询模式 + * @param planCacheHit 是否命中计划缓存 + * @param planningNanos 编译耗时 + * @param fragmentPlans 分片计划 + */ + FederationQueryMetricsTracker( + QueryId queryId, + FederationQueryMode queryMode, + boolean planCacheHit, + long planningNanos, + List fragmentPlans + ) { + this.queryId = queryId; + this.queryMode = queryMode; + this.planCacheHit = planCacheHit; + this.planningNanos = planningNanos; + boolean byteMetricsAvailable = queryMode != FederationQueryMode.SINGLE_SOURCE; + this.returnedBytes = new AtomicLong(byteMetricsAvailable ? 0 : -1); + this.intermediateBytes = new AtomicLong(byteMetricsAvailable ? 0 : -1); + fragmentPlans.forEach(fragment -> fragments.put( + fragment.fragmentId(), + new FragmentState(fragment, byteMetricsAvailable) + )); + } + + /** + * 记录调用方消费的一行最终结果。 + * + * @param row 行值 + */ + void recordOutput(Object[] row) { + firstRowNanos.compareAndSet(-1, System.nanoTime() - executionStartedNanos); + returnedRows.incrementAndGet(); + returnedBytes.addAndGet(estimateRowBytes(row)); + } + + /** + * 记录单源流式游标的一行,避免指标采集主动读取尚未被调用方访问的大字段。 + */ + void recordSingleSourceRow() { + firstRowNanos.compareAndSet(-1, System.nanoTime() - executionStartedNanos); + returnedRows.incrementAndGet(); + intermediateRows.incrementAndGet(); + fragment("fragment-1").rows.incrementAndGet(); + } + + /** + * 记录查询级准入等待耗时。 + * + * @param elapsedNanos 等待耗时 + */ + void recordAdmissionWait(long elapsedNanos) { + admissionWaitNanos.addAndGet(Math.max(0, elapsedNanos)); + } + + /** + * 返回指定分片的 Adapter 阶段观察器。 + * + * @param fragmentId 分片标识 + * @return 阶段观察器 + */ + FederationExecutionObserver fragmentObserver(String fragmentId) { + FragmentState state = fragment(fragmentId); + state.start(); + return new FederationExecutionObserver() { + @Override + public void connectionAcquired(long elapsedNanos) { + long safe = Math.max(0, elapsedNanos); + state.connectionAcquireNanos.addAndGet(safe); + connectionAcquireNanos.addAndGet(safe); + } + + @Override + public void databaseExecutionCompleted(long elapsedNanos) { + long safe = Math.max(0, elapsedNanos); + state.databaseExecutionNanos.addAndGet(safe); + databaseExecutionNanos.addAndGet(safe); + } + + @Override + public void firstRowAvailable(long elapsedNanos) { + state.firstRowNanos.compareAndSet(-1, Math.max(0, elapsedNanos)); + } + }; + } + + /** + * 记录物理分片读取的一行并返回累计中间结果计数。 + * + * @param fragmentId 分片标识 + * @param row 行值 + * @return 当前累计行数和字节数 + */ + IntermediateUsage recordIntermediate(String fragmentId, Object[] row) { + long bytes = estimateRowBytes(row); + long rows = intermediateRows.incrementAndGet(); + long totalBytes = intermediateBytes.addAndGet(bytes); + FragmentState fragment = fragment(fragmentId); + fragment.rows.incrementAndGet(); + fragment.bytes.addAndGet(bytes); + return new IntermediateUsage(rows, totalBytes); + } + + /** + * 记录 Calcite 本地算子交给下游的一行。 + * + * @param row Calcite 内部行 + * @return 当前全局中间资源累计值 + */ + synchronized IntermediateUsage recordLocalIntermediate( + String operatorName, + Object[] row, + long elapsedNanos + ) { + long bytes = estimateRowBytes(row); + LocalOperatorState operator = localOperators.computeIfAbsent( + operatorName, + ignored -> new LocalOperatorState(operatorName) + ); + operator.rows.incrementAndGet(); + operator.bytes.addAndGet(bytes); + long safeElapsed = Math.max(0, elapsedNanos); + operator.executionNanos.addAndGet(safeElapsed); + localExecutionNanos.addAndGet(safeElapsed); + return new IntermediateUsage( + intermediateRows.incrementAndGet(), + intermediateBytes.addAndGet(bytes) + ); + } + + /** + * 标记一个分片完成或关闭。 + * + * @param fragmentId 分片标识 + */ + void finishFragment(String fragmentId) { + fragment(fragmentId).finish(); + } + + /** + * 标记最终结果已正常耗尽。 + */ + void finishSuccessfully() { + complete.set(true); + finishClosed(); + } + + /** + * 标记查询收到取消请求。 + */ + void markCancelled() { + if (terminalErrorCode.compareAndSet( + "", + FederationSqlErrorCode.QUERY_CANCELLED.name() + )) { + cancelled.set(true); + finishClosed(); + } + } + + /** + * 标记查询因统一执行时限结束。 + */ + void markTimedOut() { + if (terminalErrorCode.compareAndSet( + "", + FederationSqlErrorCode.QUERY_TIMEOUT.name() + )) { + timedOut.set(true); + finishClosed(); + } + } + + /** + * 标记查询因最终结果行数上限停止继续消费。 + */ + void markTruncated() { + truncated.set(true); + } + + /** + * 记录查询失败的稳定错误码。 + * + * @param errorCode 错误码 + */ + void markFailed(FederationSqlErrorCode errorCode) { + if (errorCode == FederationSqlErrorCode.QUERY_CANCELLED) { + markCancelled(); + return; + } + if (errorCode == FederationSqlErrorCode.QUERY_TIMEOUT) { + markTimedOut(); + return; + } + if (terminalErrorCode.compareAndSet("", errorCode.name())) { + finishClosed(); + } + } + + /** + * 冻结查询和全部分片耗时;提前关闭不会标记查询正常完成。 + */ + void finishClosed() { + long now = System.nanoTime(); + executionFinishedNanos.compareAndSet(-1, now); + synchronized (this) { + fragments.values().forEach(fragment -> fragment.finish(now)); + } + } + + /** + * 返回当前不可变指标快照。 + * + * @return 指标快照 + */ + synchronized FederationQueryMetricsSnapshot snapshot() { + long now = System.nanoTime(); + long executionEnd = executionFinishedNanos.get(); + long effectiveExecutionEnd = executionEnd < 0 ? now : executionEnd; + List fragmentSnapshots = new ArrayList<>(fragments.size()); + fragments.values().forEach(fragment -> fragmentSnapshots.add( + new FederationFragmentMetrics( + fragment.plan.fragmentId(), + fragment.plan.sourceId(), + fragment.rows.get(), + fragment.bytes.get(), + fragment.elapsedNanos(now), + fragment.connectionAcquireNanos.get(), + fragment.databaseExecutionNanos.get(), + fragment.firstRowNanos.get(), + fragment.complete.get() + ) + )); + List operatorSnapshots = new ArrayList<>( + localOperators.size() + ); + localOperators.values().forEach(operator -> operatorSnapshots.add( + new FederationLocalOperatorMetrics( + operator.name, + operator.rows.get(), + operator.bytes.get(), + operator.executionNanos.get() + ) + )); + return new FederationQueryMetricsSnapshot( + queryId, + queryMode, + planCacheHit, + planningNanos, + admissionWaitNanos.get(), + connectionAcquireNanos.get(), + databaseExecutionNanos.get(), + localExecutionNanos.get(), + effectiveExecutionEnd - executionStartedNanos, + firstRowNanos.get(), + returnedRows.get(), + returnedBytes.get(), + intermediateRows.get(), + intermediateBytes.get(), + complete.get(), + cancelled.get(), + timedOut.get(), + truncated.get(), + terminalErrorCode.get(), + fragmentSnapshots, + operatorSnapshots + ); + } + + private synchronized FragmentState fragment(String fragmentId) { + FragmentState state = fragments.get(fragmentId); + if (state == null) { + throw new IllegalArgumentException("unknown fragment: " + fragmentId); + } + return state; + } + + /** + * 对常见 JDBC 标量进行保守、常量时间的内存估算。 + */ + static long estimateRowBytes(Object[] row) { + long bytes = 16L + 8L * row.length; + for (Object value : row) { + if (value == null) { + continue; + } + if (value instanceof CharSequence text) { + bytes += 40L + 2L * text.length(); + } else if (value instanceof byte[] binary) { + bytes += 24L + binary.length; + } else if (value instanceof ByteBuffer binary) { + bytes += 24L + binary.remaining(); + } else if (value instanceof ByteString binary) { + // Calcite Enumerable 使用 ByteString 承载 BINARY/VARBINARY。 + bytes += 24L + binary.length(); + } else if (value instanceof BigDecimal decimal) { + bytes += 48L + decimal.precision() / 2L; + } else if (value instanceof Number + || value instanceof Boolean + || value instanceof java.util.UUID + || value instanceof java.time.OffsetTime + || value instanceof java.time.OffsetDateTime + || value instanceof java.time.LocalDate + || value instanceof java.time.LocalTime + || value instanceof java.time.LocalDateTime + || value instanceof java.time.Instant + || value instanceof java.sql.Date + || value instanceof java.sql.Time + || value instanceof java.sql.Timestamp) { + bytes += 32L; + } else { + throw new FederationSqlException( + FederationSqlErrorCode.FEDERATION_OPERATOR_UNSUPPORTED, + "value cannot be safely measured for local federation: " + + value.getClass().getName() + ); + } + } + return bytes; + } + + /** + * 累计中间结果资源用量。 + * + * @param rows 行数 + * @param bytes 估算字节数 + */ + record IntermediateUsage(long rows, long bytes) { + } + + private static final class FragmentState { + + private final FederationFragmentPlan plan; + private final AtomicLong startedNanos = new AtomicLong(-1); + private final AtomicLong finishedNanos = new AtomicLong(-1); + private final AtomicLong rows = new AtomicLong(); + private final AtomicLong bytes; + private final AtomicLong connectionAcquireNanos = new AtomicLong(); + private final AtomicLong databaseExecutionNanos = new AtomicLong(); + private final AtomicLong firstRowNanos = new AtomicLong(-1); + private final AtomicBoolean complete = new AtomicBoolean(); + + private FragmentState(FederationFragmentPlan plan, boolean byteMetricsAvailable) { + this.plan = plan; + this.bytes = new AtomicLong(byteMetricsAvailable ? 0 : -1); + } + + private void finish() { + finish(System.nanoTime()); + } + + private void start() { + startedNanos.compareAndSet(-1, System.nanoTime()); + } + + private void finish(long now) { + start(); + complete.set(true); + finishedNanos.compareAndSet(-1, now); + } + + private long elapsedNanos(long now) { + long started = startedNanos.get(); + if (started < 0) { + return 0; + } + long finished = finishedNanos.get(); + return (finished < 0 ? now : finished) - started; + } + } + + private static final class LocalOperatorState { + + private final String name; + private final AtomicLong rows = new AtomicLong(); + private final AtomicLong bytes = new AtomicLong(); + private final AtomicLong executionNanos = new AtomicLong(); + + private LocalOperatorState(String name) { + this.name = name; + } + } +} diff --git a/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/runtime/FederationQueryScopeSnapshot.java b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/runtime/FederationQueryScopeSnapshot.java new file mode 100644 index 0000000..a31245f --- /dev/null +++ b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/runtime/FederationQueryScopeSnapshot.java @@ -0,0 +1,228 @@ +package com.easyagents.federation.sql.runtime; + +import com.easyagents.federation.sql.api.FederationSqlErrorCode; +import com.easyagents.federation.sql.api.FederationSqlException; +import com.easyagents.federation.sql.federation.FederationQueryScopeDefinition; +import com.easyagents.federation.sql.federation.FederationSourceBindingDefinition; +import com.easyagents.federation.sql.federation.FederationSourceRuntimeIdentity; +import com.easyagents.federation.sql.source.SourceId; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.atomic.AtomicBoolean; + +/** + * 编译或执行期间持有的查询范围 Runtime 快照。 + */ +final class FederationQueryScopeSnapshot implements AutoCloseable { + + private final FederationQueryScopeDefinition definition; + private final Map runtimesByBinding; + private final List leases; + private final AtomicBoolean closed = new AtomicBoolean(); + + private FederationQueryScopeSnapshot( + FederationQueryScopeDefinition definition, + Map runtimesByBinding, + List leases + ) { + this.definition = definition; + this.runtimesByBinding = Collections.unmodifiableMap( + new LinkedHashMap<>(runtimesByBinding) + ); + this.leases = List.copyOf(leases); + } + + /** + * 仅供同包编译器兼容测试借用既有单源 Runtime,不接管其生命周期。 + */ + static FederationQueryScopeSnapshot borrowedSingle( + FederationQueryScopeDefinition definition, + SourceRuntime runtime + ) { + return borrowed(definition, Map.of(definition.defaultBinding(), runtime)); + } + + /** + * 仅供同包编译器契约测试借用既有多源 Runtime,不接管其生命周期。 + * + * @param definition 查询范围 + * @param runtimesByBinding Binding 到 Runtime 的完整映射 + * @return 不持有 Runtime lease 的快照 + */ + static FederationQueryScopeSnapshot borrowed( + FederationQueryScopeDefinition definition, + Map runtimesByBinding + ) { + if (definition == null || runtimesByBinding == null + || !runtimesByBinding.keySet().containsAll(definition.bindings().keySet())) { + throw new IllegalArgumentException( + "borrowed snapshot must provide every declared binding runtime" + ); + } + return new FederationQueryScopeSnapshot(definition, runtimesByBinding, List.of()); + } + + /** + * 按稳定 SourceId 顺序获取指定 Binding 的 Runtime,并合并重复物理源。 + * + * @param sourceManager 数据源管理器 + * @param definition 查询范围 + * @param bindingNames 需要加载的 Binding 名称 + * @return Runtime 快照 + */ + static FederationQueryScopeSnapshot acquire( + DefaultFederationSourceManager sourceManager, + FederationQueryScopeDefinition definition, + Set bindingNames + ) { + if (bindingNames == null || bindingNames.isEmpty()) { + throw new FederationSqlException( + FederationSqlErrorCode.INVALID_ARGUMENT, + "at least one query binding must be selected" + ); + } + Map revisions = new LinkedHashMap<>(); + for (String bindingName : bindingNames) { + FederationSourceBindingDefinition binding = definition.bindings().get(bindingName); + if (binding == null) { + throw new FederationSqlException( + FederationSqlErrorCode.INVALID_ARGUMENT, + "query references an unknown binding: " + bindingName + ); + } + revisions.merge(binding.sourceId(), binding.minimumRevision(), Math::max); + } + List> orderedSources = new ArrayList<>(revisions.entrySet()); + orderedSources.sort(Comparator.comparing(entry -> entry.getKey().value())); + + List acquired = new ArrayList<>(); + Map runtimesBySource = new LinkedHashMap<>(); + try { + for (Map.Entry source : orderedSources) { + SourceRuntime.RuntimeLease lease = sourceManager.acquireRuntime( + source.getKey(), + source.getValue() + ); + acquired.add(lease); + runtimesBySource.put(source.getKey(), lease.runtime()); + } + Map runtimesByBinding = new LinkedHashMap<>(); + bindingNames.stream().sorted().forEach(bindingName -> { + FederationSourceBindingDefinition binding = definition.bindings().get(bindingName); + runtimesByBinding.put(bindingName, runtimesBySource.get(binding.sourceId())); + }); + return new FederationQueryScopeSnapshot(definition, runtimesByBinding, acquired); + } catch (RuntimeException exception) { + closeReverseAndSuppress(acquired, exception); + throw exception; + } + } + + /** + * 返回查询范围定义。 + * + * @return 查询范围 + */ + FederationQueryScopeDefinition definition() { + return definition; + } + + /** + * 返回指定 Binding 的 Runtime。 + * + * @param bindingName Binding 名称 + * @return Runtime + */ + SourceRuntime runtime(String bindingName) { + SourceRuntime runtime = runtimesByBinding.get(bindingName); + if (runtime == null) { + throw new FederationSqlException( + FederationSqlErrorCode.INVALID_ARGUMENT, + "binding runtime is not loaded: " + bindingName + ); + } + return runtime; + } + + /** + * 返回已加载的 Binding 到 Runtime 映射。 + * + * @return 不可变映射 + */ + Map runtimesByBinding() { + return runtimesByBinding; + } + + /** + * 返回全部 Binding 的运行身份。 + * + * @return 运行身份列表 + */ + List identities() { + return runtimesByBinding.entrySet().stream() + .map(entry -> identity(entry.getKey(), entry.getValue())) + .toList(); + } + + private static FederationSourceRuntimeIdentity identity( + String bindingName, + SourceRuntime runtime + ) { + return new FederationSourceRuntimeIdentity( + bindingName, + runtime.definition().sourceId(), + runtime.definition().revision(), + runtime.sourceChecksum(), + runtime.adapter().adapterId(), + runtime.runtimeFingerprint() + ); + } + + /** + * 逆序释放所有唯一物理 Runtime lease。 + */ + @Override + public void close() { + if (!closed.compareAndSet(false, true)) { + return; + } + FederationSqlException failure = null; + for (int index = leases.size() - 1; index >= 0; index--) { + try { + leases.get(index).close(); + } catch (RuntimeException exception) { + FederationSqlException wrapped = new FederationSqlException( + FederationSqlErrorCode.RESOURCE_CLOSE_FAILED, + "failed to close query scope runtime lease", + exception + ); + if (failure == null) { + failure = wrapped; + } else { + failure.addSuppressed(wrapped); + } + } + } + if (failure != null) { + throw failure; + } + } + + private static void closeReverseAndSuppress( + List acquired, + RuntimeException original + ) { + for (int index = acquired.size() - 1; index >= 0; index--) { + try { + acquired.get(index).close(); + } catch (RuntimeException closeFailure) { + original.addSuppressed(closeFailure); + } + } + } +} diff --git a/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/runtime/FederationStatisticsMetadata.java b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/runtime/FederationStatisticsMetadata.java new file mode 100644 index 0000000..f9fa27f --- /dev/null +++ b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/runtime/FederationStatisticsMetadata.java @@ -0,0 +1,663 @@ +package com.easyagents.federation.sql.runtime; + +import com.easyagents.federation.sql.federation.FederationColumnStatistics; +import com.easyagents.federation.sql.federation.FederationStatisticsStatus; +import com.easyagents.federation.sql.federation.FederationTableStatistics; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Set; +import org.apache.calcite.plan.RelOptCost; +import org.apache.calcite.plan.RelOptCostImpl; +import org.apache.calcite.plan.RelOptCluster; +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.core.Join; +import org.apache.calcite.rel.metadata.BuiltInMetadata; +import org.apache.calcite.rel.metadata.ChainedRelMetadataProvider; +import org.apache.calcite.rel.metadata.MetadataDef; +import org.apache.calcite.rel.metadata.MetadataHandler; +import org.apache.calcite.rel.metadata.RelMetadataProvider; +import org.apache.calcite.rel.metadata.RelMetadataQuery; +import org.apache.calcite.rel.metadata.ReflectiveRelMetadataProvider; +import org.apache.calcite.rex.RexCall; +import org.apache.calcite.rex.RexInputRef; +import org.apache.calcite.rex.RexLiteral; +import org.apache.calcite.rex.RexNode; +import org.apache.calcite.rex.RexUtil; +import org.apache.calcite.sql.SqlKind; +import org.apache.calcite.util.ImmutableBitSet; + +/** + * 将调用方冻结的表列统计接入 Calcite 元数据查询链路。 + * + *

该类型仅因 Calcite 元数据代码生成要求公开可访问,调用方无需直接使用。

+ */ +public final class FederationStatisticsMetadata { + + private static final RelMetadataProvider SOURCE = ChainedRelMetadataProvider.of(List.of( + ReflectiveRelMetadataProvider.reflectiveSource( + new CumulativeCostHandler(), + BuiltInMetadata.CumulativeCost.Handler.class + ), + ReflectiveRelMetadataProvider.reflectiveSource( + new DistinctRowCountHandler(), + BuiltInMetadata.DistinctRowCount.Handler.class + ), + ReflectiveRelMetadataProvider.reflectiveSource( + new SelectivityHandler(), + BuiltInMetadata.Selectivity.Handler.class + ), + ReflectiveRelMetadataProvider.reflectiveSource( + new UniqueKeysHandler(), + BuiltInMetadata.UniqueKeys.Handler.class + ), + ReflectiveRelMetadataProvider.reflectiveSource( + new SizeHandler(), + BuiltInMetadata.Size.Handler.class + ) + )); + + /** + * 禁止实例化静态元数据桥接器。 + */ + private FederationStatisticsMetadata() { } + + /** + * 将联邦统计提供器置于现有元数据链路之前,并清理已经计算的元数据。 + * + * @param cluster 关系表达式集群 + */ + static void install(RelOptCluster cluster) { + cluster.setMetadataProvider(ChainedRelMetadataProvider.of(List.of( + SOURCE, + cluster.getMetadataProvider() + ))); + cluster.invalidateMetadataQuery(); + } + + /** + * 按字段序号查找列统计,并兼容数据库返回的字段名大小写差异。 + * + * @param scan 表扫描 + * @param index 字段序号 + * @return 列统计;字段或统计不存在时返回 null + */ + private static FederationColumnStatistics columnStatistics( + FederationStatisticsTableScan scan, + int index + ) { + if (index < 0 || index >= scan.getRowType().getFieldCount()) { + return null; + } + String fieldName = scan.getRowType().getFieldList().get(index).getName(); + Map columns = scan.statistics().columns(); + FederationColumnStatistics direct = columns.get(fieldName); + if (direct != null) { + return direct; + } + String normalized = fieldName.toLowerCase(Locale.ROOT); + return columns.entrySet().stream() + .filter(entry -> entry.getKey().toLowerCase(Locale.ROOT).equals(normalized)) + .map(Map.Entry::getValue) + .findFirst() + .orElse(null); + } + + /** + * 提取输入字段序号,同时剥离不改变字段来源的 CAST。 + * + * @param node 表达式节点 + * @return 输入字段序号;非字段引用时返回 -1 + */ + private static int inputIndex(RexNode node) { + RexNode current = node; + while (current instanceof RexCall call + && call.getKind() == SqlKind.CAST + && RexUtil.isLosslessCast(call)) { + current = call.getOperands().get(0); + } + return current instanceof RexInputRef inputRef ? inputRef.getIndex() : -1; + } + + /** + * 将概率值限制在有效区间。 + * + * @param value 原始概率 + * @return 位于 0 到 1 之间的概率 + */ + private static double clampProbability(double value) { + return Math.max(0D, Math.min(1D, value)); + } + + /** + * 将成本值限制在可比较的有限区间。 + * + * @param value 原始成本 + * @return 有限非负成本 + */ + private static double finiteCost(double value) { + if (Double.isNaN(value) || value <= 0D) { + return 0D; + } + return Math.min(value, Double.MAX_VALUE / 16D); + } + + /** + * 安全累加成本,避免极端统计导致无穷值破坏计划比较。 + * + * @param left 左成本 + * @param right 右成本 + * @return 饱和后的成本 + */ + private static double addCost(double left, double right) { + double normalizedLeft = finiteCost(left); + double normalizedRight = finiteCost(right); + double maximum = Double.MAX_VALUE / 16D; + return normalizedLeft >= maximum - normalizedRight + ? maximum + : normalizedLeft + normalizedRight; + } + + /** + * 安全计算乘积成本。 + * + * @param left 左因子 + * @param right 右因子 + * @return 饱和后的成本 + */ + private static double multiplyCost(double left, double right) { + double normalizedLeft = finiteCost(left); + double normalizedRight = finiteCost(right); + double maximum = Double.MAX_VALUE / 16D; + if (normalizedLeft == 0D || normalizedRight == 0D) { + return 0D; + } + return normalizedLeft >= maximum / normalizedRight + ? maximum + : normalizedLeft * normalizedRight; + } + + /** + * 将 Calcite 多维成本折算成稳定标量。 + * + * @param cost 原始成本 + * @return 非负有限成本 + */ + private static double scalarCost(RelOptCost cost) { + if (cost == null || cost.isInfinite()) { + return Double.MAX_VALUE / 16D; + } + return addCost(addCost(cost.getRows(), cost.getCpu()), cost.getIo()); + } + + /** + * 为联邦 Join 搜索提供包含数据搬运和本地构建开销的累计成本。 + */ + public static final class CumulativeCostHandler + implements MetadataHandler { + + private static final double HASH_BUILD_WEIGHT = 2D; + + /** + * 创建累计联邦成本处理器。 + */ + private CumulativeCostHandler() { } + + /** {@inheritDoc} */ + @Override + public MetadataDef getDef() { + return BuiltInMetadata.CumulativeCost.DEF; + } + + /** + * 计算通用节点的自身成本和全部输入累计成本。 + * + * @param node 关系节点 + * @param metadataQuery 元数据查询 + * @return 累计成本 + */ + public RelOptCost getCumulativeCost( + RelNode node, + RelMetadataQuery metadataQuery + ) { + double total = scalarCost(metadataQuery.getNonCumulativeCost(node)); + for (RelNode input : node.getInputs()) { + total = addCost(total, scalarCost(metadataQuery.getCumulativeCost(input))); + } + return new RelOptCostImpl(total); + } + + /** + * 使用冻结行数与行宽计算物理源数据搬运成本。 + * + * @param scan 表扫描 + * @param metadataQuery 元数据查询 + * @return 扫描累计成本 + */ + public RelOptCost getCumulativeCost( + FederationStatisticsTableScan scan, + RelMetadataQuery metadataQuery + ) { + FederationTableStatistics statistics = scan.statistics(); + double transfer = multiplyCost( + statistics.estimatedRows(), + statistics.averageRowWidthBytes() + ); + return new RelOptCostImpl(addCost(statistics.estimatedRows(), transfer)); + } + + /** + * 计算 Join 输入、输出与右侧 Hash 构建的累计成本。 + * + * @param join Join 节点 + * @param metadataQuery 元数据查询 + * @return Join 累计成本 + */ + public RelOptCost getCumulativeCost( + Join join, + RelMetadataQuery metadataQuery + ) { + double leftCost = scalarCost(metadataQuery.getCumulativeCost(join.getLeft())); + double rightCost = scalarCost(metadataQuery.getCumulativeCost(join.getRight())); + double outputRows = finiteCost(metadataQuery.getRowCount(join)); + Double averageRowSize = metadataQuery.getAverageRowSize(join); + double outputWidth = averageRowSize == null ? 16D : finiteCost(averageRowSize); + double outputCost = addCost(outputRows, multiplyCost(outputRows, outputWidth)); + double rightRows = finiteCost(metadataQuery.getRowCount(join.getRight())); + Double rightRowSize = metadataQuery.getAverageRowSize(join.getRight()); + double rightWidth = rightRowSize == null ? 16D : finiteCost(rightRowSize); + double buildCost = multiplyCost( + multiplyCost(rightRows, rightWidth), + HASH_BUILD_WEIGHT + ); + return new RelOptCostImpl(addCost( + addCost(leftCost, rightCost), + addCost(outputCost, buildCost) + )); + } + } + + /** + * 使用冻结列统计估算基础谓词选择率。 + * + * @param scan 表扫描 + * @param predicate 谓词表达式 + * @return 选择率;无法可靠估算时返回 null + */ + private static Double predicateSelectivity( + FederationStatisticsTableScan scan, + RexNode predicate + ) { + if (predicate == null) { + return 1D; + } + if (!(predicate instanceof RexCall call)) { + return null; + } + return switch (call.getKind()) { + case AND -> combineAnd(scan, call.getOperands()); + case OR -> combineOr(scan, call.getOperands()); + case EQUALS -> equalitySelectivity(scan, call, false); + case IS_NOT_DISTINCT_FROM -> equalitySelectivity(scan, call, true); + case IS_NULL -> nullSelectivity(scan, call, true); + case IS_NOT_NULL -> nullSelectivity(scan, call, false); + default -> null; + }; + } + + /** + * 合并 AND 子谓词选择率。 + * + * @param scan 表扫描 + * @param operands 子谓词 + * @return 合并选择率;任一子谓词不可估算时返回 null + */ + private static Double combineAnd( + FederationStatisticsTableScan scan, + List operands + ) { + double result = 1D; + for (RexNode operand : operands) { + Double selectivity = predicateSelectivity(scan, operand); + if (selectivity == null) { + return null; + } + result *= selectivity; + } + return clampProbability(result); + } + + /** + * 合并 OR 子谓词选择率。 + * + * @param scan 表扫描 + * @param operands 子谓词 + * @return 合并选择率;任一子谓词不可估算时返回 null + */ + private static Double combineOr( + FederationStatisticsTableScan scan, + List operands + ) { + double noneSelected = 1D; + for (RexNode operand : operands) { + Double selectivity = predicateSelectivity(scan, operand); + if (selectivity == null) { + return null; + } + noneSelected *= 1D - selectivity; + } + return clampProbability(1D - noneSelected); + } + + /** + * 根据参与等值比较列的基数和空值比例估算选择率。 + * + * @param scan 表扫描 + * @param call 等值表达式 + * @param nullSafe 是否采用 NULL-safe 等值语义 + * @return 选择率;统计不足时返回 null + */ + private static Double equalitySelectivity( + FederationStatisticsTableScan scan, + RexCall call, + boolean nullSafe + ) { + if (call.getOperands().size() != 2) { + return null; + } + int leftIndex = inputIndex(call.getOperands().get(0)); + int rightIndex = inputIndex(call.getOperands().get(1)); + if (leftIndex < 0 && rightIndex < 0) { + return null; + } + FederationColumnStatistics left = columnStatistics(scan, leftIndex); + FederationColumnStatistics right = columnStatistics(scan, rightIndex); + if (nullSafe && isNullLiteral(call.getOperands().get(0)) && right != null) { + return right.nullFraction(); + } + if (nullSafe && isNullLiteral(call.getOperands().get(1)) && left != null) { + return left.nullFraction(); + } + if (leftIndex >= 0 && leftIndex == rightIndex && left != null) { + // 同一字段与自身比较时,普通等值仅排除 NULL,NULL-safe 等值恒为真。 + return nullSafe ? 1D : 1D - left.nullFraction(); + } + double distinctCount = Math.max( + left == null ? 0D : left.distinctCount(), + right == null ? 0D : right.distinctCount() + ); + if (distinctCount <= 0D) { + return null; + } + double nonNullFraction = 1D; + if (left != null) { + nonNullFraction *= 1D - left.nullFraction(); + } + if (right != null) { + nonNullFraction *= 1D - right.nullFraction(); + } + double selectivity = nonNullFraction / distinctCount; + if (nullSafe && left != null && right != null) { + // NULL-safe 等值还会命中两侧同时为 NULL 的行。 + selectivity += left.nullFraction() * right.nullFraction(); + } + return clampProbability(selectivity); + } + + /** + * 判断表达式是否为 NULL 字面量,仅剥离不改变 NULL 语义的 CAST。 + * + * @param node 表达式节点 + * @return 是否为 NULL 字面量 + */ + private static boolean isNullLiteral(RexNode node) { + RexNode current = node; + while (current instanceof RexCall call && call.getKind() == SqlKind.CAST) { + current = call.getOperands().get(0); + } + return current instanceof RexLiteral literal && literal.isNull(); + } + + /** + * 估算 IS NULL 或 IS NOT NULL 谓词选择率。 + * + * @param scan 表扫描 + * @param call 空值判断表达式 + * @param nullOnly true 表示 IS NULL,false 表示 IS NOT NULL + * @return 选择率;统计不足时返回 null + */ + private static Double nullSelectivity( + FederationStatisticsTableScan scan, + RexCall call, + boolean nullOnly + ) { + if (call.getOperands().size() != 1) { + return null; + } + FederationColumnStatistics column = columnStatistics( + scan, + inputIndex(call.getOperands().get(0)) + ); + if (column == null) { + return null; + } + return nullOnly ? column.nullFraction() : 1D - column.nullFraction(); + } + + /** + * 基于列基数估算分组后的不同值数量。 + */ + public static final class DistinctRowCountHandler + implements MetadataHandler { + + /** + * 创建不同值行数元数据处理器。 + */ + private DistinctRowCountHandler() { } + + /** {@inheritDoc} */ + @Override + public MetadataDef getDef() { + return BuiltInMetadata.DistinctRowCount.DEF; + } + + /** + * 返回冻结统计中的不同值估算。 + * + * @param scan 表扫描 + * @param metadataQuery 元数据查询 + * @param groupKey 分组列 + * @param predicate 可选过滤条件 + * @return 不同值估算;统计不足时返回 null 交给默认提供器 + */ + public Double getDistinctRowCount( + FederationStatisticsTableScan scan, + RelMetadataQuery metadataQuery, + ImmutableBitSet groupKey, + RexNode predicate + ) { + if (groupKey.isEmpty()) { + return null; + } + double distinctRows = 1D; + for (int index : groupKey) { + FederationColumnStatistics column = columnStatistics(scan, index); + if (column == null || column.distinctCount() <= 0D) { + return null; + } + distinctRows = Math.min( + scan.statistics().estimatedRows(), + distinctRows * column.distinctCount() + ); + } + Double selectivity = predicateSelectivity(scan, predicate); + double selectedRows = selectivity == null + ? scan.statistics().estimatedRows() + : scan.statistics().estimatedRows() * selectivity; + return Math.min(distinctRows, selectedRows); + } + } + + /** + * 基于列基数和空值比例估算基础谓词选择率。 + */ + public static final class SelectivityHandler + implements MetadataHandler { + + /** + * 创建选择率元数据处理器。 + */ + private SelectivityHandler() { } + + /** {@inheritDoc} */ + @Override + public MetadataDef getDef() { + return BuiltInMetadata.Selectivity.DEF; + } + + /** + * 返回基础谓词的选择率。 + * + * @param scan 表扫描 + * @param metadataQuery 元数据查询 + * @param predicate 过滤条件 + * @return 选择率;无法可靠估算时返回 null 交给默认提供器 + */ + public Double getSelectivity( + FederationStatisticsTableScan scan, + RelMetadataQuery metadataQuery, + RexNode predicate + ) { + return predicateSelectivity(scan, predicate); + } + } + + /** + * 将调用方声明的唯一键转换为 Calcite 列位集合。 + */ + public static final class UniqueKeysHandler + implements MetadataHandler { + + /** + * 创建唯一键元数据处理器。 + */ + private UniqueKeysHandler() { } + + /** {@inheritDoc} */ + @Override + public MetadataDef getDef() { + return BuiltInMetadata.UniqueKeys.DEF; + } + + /** + * 返回冻结统计中的唯一键。 + * + * @param scan 表扫描 + * @param metadataQuery 元数据查询 + * @param ignoreNulls 是否忽略空值 + * @return 唯一键集合;部分统计且无键时返回 null + */ + public Set getUniqueKeys( + FederationStatisticsTableScan scan, + RelMetadataQuery metadataQuery, + boolean ignoreNulls + ) { + FederationTableStatistics statistics = scan.statistics(); + Set keys = new HashSet<>(); + List fields = scan.getRowType().getFieldNames(); + for (List key : statistics.uniqueKeys()) { + ImmutableBitSet.Builder builder = ImmutableBitSet.builder(); + boolean complete = true; + for (String columnName : key) { + int index = findField(fields, columnName); + if (index < 0) { + complete = false; + break; + } + builder.set(index); + } + if (complete && !key.isEmpty()) { + keys.add(builder.build()); + } + } + if (keys.isEmpty() && statistics.status() != FederationStatisticsStatus.COMPLETE) { + return null; + } + return Set.copyOf(keys); + } + + /** + * 按不区分大小写的方式定位字段。 + * + * @param fields 字段名称列表 + * @param expected 待查找字段 + * @return 字段序号;未找到时返回 -1 + */ + private static int findField(List fields, String expected) { + for (int index = 0; index < fields.size(); index++) { + if (fields.get(index).equalsIgnoreCase(expected)) { + return index; + } + } + return -1; + } + } + + /** + * 将平均行宽与列宽接入 Calcite Size 元数据。 + */ + public static final class SizeHandler implements MetadataHandler { + + /** + * 创建行列宽度元数据处理器。 + */ + private SizeHandler() { } + + /** {@inheritDoc} */ + @Override + public MetadataDef getDef() { + return BuiltInMetadata.Size.DEF; + } + + /** + * 返回平均行宽。 + * + * @param scan 表扫描 + * @param metadataQuery 元数据查询 + * @return 平均行宽字节数 + */ + public Double averageRowSize( + FederationStatisticsTableScan scan, + RelMetadataQuery metadataQuery + ) { + return (double) scan.statistics().averageRowWidthBytes(); + } + + /** + * 返回各列平均宽度。 + * + * @param scan 表扫描 + * @param metadataQuery 元数据查询 + * @return 列宽列表;没有任何列宽统计时返回 null + */ + public List averageColumnSizes( + FederationStatisticsTableScan scan, + RelMetadataQuery metadataQuery + ) { + List widths = new ArrayList<>(scan.getRowType().getFieldCount()); + boolean known = false; + for (int index = 0; index < scan.getRowType().getFieldCount(); index++) { + FederationColumnStatistics column = columnStatistics(scan, index); + Double width = column == null || column.averageWidthBytes() <= 0 + ? null + : (double) column.averageWidthBytes(); + widths.add(width); + known |= width != null; + } + return known ? Collections.unmodifiableList(widths) : null; + } + } +} diff --git a/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/runtime/FederationStatisticsTableScan.java b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/runtime/FederationStatisticsTableScan.java new file mode 100644 index 0000000..9117215 --- /dev/null +++ b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/runtime/FederationStatisticsTableScan.java @@ -0,0 +1,84 @@ +package com.easyagents.federation.sql.runtime; + +import com.easyagents.federation.sql.federation.FederationTableStatistics; +import com.easyagents.federation.sql.source.SourceId; +import java.util.List; +import org.apache.calcite.plan.RelOptCluster; +import org.apache.calcite.plan.RelOptTable; +import org.apache.calcite.plan.RelTraitSet; +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.core.TableScan; +import org.apache.calcite.rel.hint.RelHint; +import org.apache.calcite.rel.metadata.RelMetadataQuery; + +/** + * 将调用方冻结统计中的表行数暴露给 Calcite 元数据查询。 + */ +public final class FederationStatisticsTableScan extends TableScan { + + private final FederationTableStatistics statistics; + private final SourceId sourceId; + + /** + * 创建带稳定行数估算的逻辑表扫描。 + * + * @param cluster 关系表达式集群 + * @param traitSet 关系特征 + * @param hints 扫描提示 + * @param table Calcite 表定义 + * @param statistics 冻结的表统计 + * @param sourceId 物理数据源标识 + */ + FederationStatisticsTableScan( + RelOptCluster cluster, + RelTraitSet traitSet, + List hints, + RelOptTable table, + FederationTableStatistics statistics, + SourceId sourceId + ) { + super(cluster, traitSet, hints, table); + this.statistics = statistics; + this.sourceId = sourceId; + } + + /** {@inheritDoc} */ + @Override + public double estimateRowCount(RelMetadataQuery metadataQuery) { + return statistics.estimatedRows(); + } + + /** + * 返回本次编译冻结的表统计。 + * + * @return 表统计快照 + */ + FederationTableStatistics statistics() { + return statistics; + } + + /** + * 返回物理数据源标识,用于限制跨源 Join 搜索范围。 + * + * @return 物理数据源标识 + */ + SourceId sourceId() { + return sourceId; + } + + /** {@inheritDoc} */ + @Override + public RelNode copy(RelTraitSet traitSet, List inputs) { + if (!inputs.isEmpty()) { + throw new IllegalArgumentException("table scan must not have inputs"); + } + return new FederationStatisticsTableScan( + getCluster(), + traitSet, + getHints(), + table, + statistics, + sourceId + ); + } +} diff --git a/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/runtime/LogicalTableSqlResolver.java b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/runtime/LogicalTableSqlResolver.java new file mode 100644 index 0000000..e7bc32c --- /dev/null +++ b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/runtime/LogicalTableSqlResolver.java @@ -0,0 +1,283 @@ +package com.easyagents.federation.sql.runtime; + +import com.easyagents.federation.sql.api.FederationSqlErrorCode; +import com.easyagents.federation.sql.api.FederationSqlException; +import com.easyagents.federation.sql.federation.FederationLogicalTableDefinition; +import com.easyagents.federation.sql.federation.FederationQueryScopeDefinition; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Set; +import org.apache.calcite.sql.SqlCall; +import org.apache.calcite.sql.SqlIdentifier; +import org.apache.calcite.sql.SqlJoin; +import org.apache.calcite.sql.SqlKind; +import org.apache.calcite.sql.SqlNode; +import org.apache.calcite.sql.SqlSelect; +import org.apache.calcite.sql.SqlWith; +import org.apache.calcite.sql.SqlWithItem; +import org.apache.calcite.sql.fun.SqlStdOperatorTable; +import org.apache.calcite.sql.util.SqlShuttle; + +/** + * 使用 Calcite AST 将查询范围逻辑表名解析为物理 Binding 表路径。 + */ +final class LogicalTableSqlResolver { + + private final Map tablesByName; + private final Map tablesByPath; + + private LogicalTableSqlResolver(List tables) { + Map byName = new HashMap<>(); + Map byPath = new HashMap<>(); + for (FederationLogicalTableDefinition table : tables) { + byName.put(normalize(table.logicalName()), table); + byPath.put(pathKey( + table.bindingName(), + table.schemaName(), + table.logicalName() + ), table); + } + this.tablesByName = Map.copyOf(byName); + this.tablesByPath = Map.copyOf(byPath); + } + + /** + * 解析查询范围中声明的逻辑表;未声明映射时原样返回。 + * + * @param parsed Calcite 已解析 SQL AST + * @param scope 查询范围 + * @return 可交给 Calcite Validator 的物理表 AST + */ + static SqlNode resolve(SqlNode parsed, FederationQueryScopeDefinition scope) { + if (scope.logicalTables().isEmpty()) { + return parsed; + } + LogicalTableSqlResolver resolver = new LogicalTableSqlResolver(scope.logicalTables()); + return parsed.accept(resolver.new ResolverShuttle(Set.of())); + } + + /** + * 返回短逻辑表实际绑定的物理数据源。 + * + * @param logicalName 逻辑表名 + * @param scope 查询范围 + * @return Binding 名称;不存在时返回 {@code null} + */ + static String bindingForShortName( + String logicalName, + FederationQueryScopeDefinition scope + ) { + if (scope.logicalTables().isEmpty()) { + return null; + } + return scope.logicalTables().stream() + .filter(table -> table.logicalName().equalsIgnoreCase(logicalName)) + .map(FederationLogicalTableDefinition::bindingName) + .findFirst() + .orElse(null); + } + + /** + * 创建大小写不敏感且无分隔歧义的三段路径键。 + * + * @param binding Binding 名称 + * @param schema Schema 名称 + * @param table 逻辑表名 + * @return 路径键 + */ + private static String pathKey(String binding, String schema, String table) { + return normalize(binding) + '\u0000' + normalize(schema) + '\u0000' + normalize(table); + } + + /** + * 按 Calcite 未加引号标识符规则归一化名称。 + * + * @param value 原始名称 + * @return 大写名称 + */ + private static String normalize(String value) { + return value.toUpperCase(Locale.ROOT); + } + + /** + * 创建未知逻辑表校验异常。 + * + * @param identifier 未命中的表标识符 + * @return SQL 校验异常 + */ + private FederationSqlException unknownTable(SqlIdentifier identifier) { + return new FederationSqlException( + FederationSqlErrorCode.SQL_VALIDATION_FAILED, + "logical table is not declared in the query scope: " + + String.join(".", identifier.names) + ); + } + + /** + * 仅在关系来源位置解析表名,并维护 CTE 词法作用域。 + */ + private final class ResolverShuttle extends SqlShuttle { + + private final Set visibleCtes; + + private ResolverShuttle(Set visibleCtes) { + this.visibleCtes = visibleCtes; + } + + /** {@inheritDoc} */ + @Override + public SqlNode visit(SqlCall call) { + if (call instanceof SqlWith with) { + return rewriteWith(with); + } + if (call instanceof SqlSelect) { + SqlSelect rewritten = (SqlSelect) super.visit(call); + rewritten.setFrom(rewriteFrom(rewritten.getFrom(), true)); + return rewritten; + } + return super.visit(call); + } + + /** + * 将四段逻辑列限定名转换为隐式逻辑表别名限定名。 + * + * @param identifier SQL 标识符 + * @return 原标识符或两段逻辑列限定名 + */ + @Override + public SqlNode visit(SqlIdentifier identifier) { + if (identifier.names.size() != 4) { + return identifier; + } + FederationLogicalTableDefinition table = tablesByPath.get(pathKey( + identifier.names.get(0), + identifier.names.get(1), + identifier.names.get(2) + )); + if (table == null) { + return identifier; + } + return new SqlIdentifier( + List.of(normalize(table.logicalName()), identifier.names.get(3)), + identifier.getParserPosition() + ); + } + + /** + * 按声明顺序重写 CTE,并维护其词法可见范围。 + * + * @param with CTE 节点 + * @return 重写后的 CTE + */ + private SqlNode rewriteWith(SqlWith with) { + Set withScope = new HashSet<>(visibleCtes); + for (SqlNode node : with.withList) { + SqlWithItem item = (SqlWithItem) node; + Set itemScope = new HashSet<>(withScope); + if (item.recursive != null && item.recursive.booleanValue()) { + itemScope.add(normalize(item.name.getSimple())); + } + item.query = item.query.accept(new ResolverShuttle(itemScope)); + withScope.add(normalize(item.name.getSimple())); + } + with.body = with.body.accept(new ResolverShuttle(withScope)); + return with; + } + + /** + * 递归重写 FROM 关系来源中的逻辑表。 + * + * @param from 关系来源 + * @param addImplicitAlias 是否补充逻辑表隐式别名 + * @return 重写后的关系来源 + */ + private SqlNode rewriteFrom(SqlNode from, boolean addImplicitAlias) { + if (from == null) { + return null; + } + if (from instanceof SqlIdentifier identifier) { + return resolveIdentifier(identifier, addImplicitAlias); + } + if (from instanceof SqlJoin join) { + join.setLeft(rewriteFrom(join.getLeft(), true)); + join.setRight(rewriteFrom(join.getRight(), true)); + return join; + } + if (from instanceof SqlSelect || from instanceof SqlWith) { + return from.accept(new ResolverShuttle(visibleCtes)); + } + if (from instanceof SqlCall call && isRelationWrapper(call.getKind())) { + if (call.operandCount() > 0) { + boolean wrappedByExplicitAlias = call.getKind() == SqlKind.AS; + call.setOperand(0, rewriteFrom(call.operand(0), !wrappedByExplicitAlias)); + } + return call; + } + return from; + } + + /** + * 将短名或三段逻辑表名解析为物理 Binding 路径。 + * + * @param identifier 逻辑表标识符 + * @param addImplicitAlias 是否补充逻辑表隐式别名 + * @return 物理表路径或带别名的关系节点 + * @throws FederationSqlException 逻辑表未声明时抛出 + */ + private SqlNode resolveIdentifier( + SqlIdentifier identifier, + boolean addImplicitAlias + ) { + FederationLogicalTableDefinition table; + if (identifier.isSimple()) { + String name = identifier.getSimple(); + if (visibleCtes.contains(normalize(name))) { + return identifier; + } + table = tablesByName.get(normalize(name)); + } else if (identifier.names.size() == 3) { + table = tablesByPath.get(pathKey( + identifier.names.get(0), + identifier.names.get(1), + identifier.names.get(2) + )); + } else { + throw unknownTable(identifier); + } + if (table == null) { + throw unknownTable(identifier); + } + SqlIdentifier physical = new SqlIdentifier( + List.of(table.bindingName(), table.schemaName(), table.sourceTableName()), + identifier.getParserPosition() + ); + if (!addImplicitAlias) { + return physical; + } + return SqlStdOperatorTable.AS.createCall( + identifier.getParserPosition(), + physical, + new SqlIdentifier( + normalize(table.logicalName()), + identifier.getParserPosition() + ) + ); + } + + /** + * 判断调用节点是否仅包装一个关系来源。 + * + * @param kind Calcite 节点类型 + * @return 是否为关系包装节点 + */ + private boolean isRelationWrapper(SqlKind kind) { + return kind == SqlKind.AS + || kind == SqlKind.LATERAL + || kind == SqlKind.TABLESAMPLE + || kind == SqlKind.SNAPSHOT; + } + } +} diff --git a/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/runtime/ManagedFederationResultCursor.java b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/runtime/ManagedFederationResultCursor.java new file mode 100644 index 0000000..57d75c1 --- /dev/null +++ b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/runtime/ManagedFederationResultCursor.java @@ -0,0 +1,379 @@ +package com.easyagents.federation.sql.runtime; + +import com.easyagents.federation.sql.api.FederationSqlErrorCode; +import com.easyagents.federation.sql.api.FederationSqlException; +import com.easyagents.federation.sql.execute.FederationColumn; +import com.easyagents.federation.sql.execute.FederationQueryMetricsSnapshot; +import com.easyagents.federation.sql.execute.FederationResultCursor; +import com.easyagents.federation.sql.execute.QueryId; +import java.io.FilterInputStream; +import java.io.FilterReader; +import java.io.IOException; +import java.io.InputStream; +import java.io.Reader; +import java.util.List; +import java.util.concurrent.atomic.AtomicBoolean; + +/** + * 由 Core 统一释放 Runtime lease、准入许可并追踪活动游标的包装器。 + */ +final class ManagedFederationResultCursor implements FederationResultCursor { + + private final FederationResultCursor delegate; + private final AutoCloseable resources; + private final Runnable onClose; + private final AtomicBoolean closed = new AtomicBoolean(); + + /** + * 创建 Core 托管游标。 + * + * @param delegate Adapter 游标 + * @param resources Core 资源 + * @param onClose 关闭后的追踪清理动作 + */ + ManagedFederationResultCursor( + FederationResultCursor delegate, + AutoCloseable resources, + Runnable onClose + ) { + this.delegate = delegate; + this.resources = resources; + this.onClose = onClose; + } + + /** + * 返回查询标识。 + * + * @return 查询标识 + */ + @Override + public QueryId queryId() { + return delegate.queryId(); + } + + /** + * 返回结果列。 + * + * @return 结果列 + */ + @Override + public List columns() { + return delegate.columns(); + } + + /** {@inheritDoc} */ + @Override + public FederationQueryMetricsSnapshot metrics() { + return delegate.metrics(); + } + + /** + * 在 Engine 主动取消后、资源关闭前定稿取消指标。 + */ + void markCancelled() { + if (delegate instanceof CancellationAwareFederationCursor aware) { + aware.markCancelled(); + } + } + + /** + * 在 Engine 执行时限到达后、资源关闭前定稿超时指标。 + */ + void markTimedOut() { + if (delegate instanceof CancellationAwareFederationCursor aware) { + aware.markTimedOut(); + } + } + + /** + * 移动至下一行,Adapter 异常时同步释放 Core 资源。 + * + * @return 是否存在下一行 + */ + @Override + public boolean next() { + try { + return delegate.next(); + } catch (RuntimeException exception) { + markFailure(exception); + closeAndSuppress(exception); + throw exception; + } + } + + /** + * 读取当前行列值,Adapter 异常时同步释放 Core 资源。 + * + * @param columnIndex 从 1 开始的列序号 + * @return 列值 + */ + @Override + public Object getObject(int columnIndex) { + try { + return delegate.getObject(columnIndex); + } catch (RuntimeException exception) { + markFailure(exception); + closeAndSuppress(exception); + throw exception; + } + } + + /** + * 委托流式读取二进制列,并在 Adapter 异常时同步释放 Core 资源。 + * + * @param columnIndex 从 1 开始的列序号 + * @return 二进制流 + */ + @Override + public InputStream getBinaryStream(int columnIndex) { + try { + InputStream stream = delegate.getBinaryStream(columnIndex); + return stream == null ? null : managed(stream); + } catch (RuntimeException exception) { + markFailure(exception); + closeAndSuppress(exception); + throw exception; + } + } + + /** + * 委托流式读取字符列,并在 Adapter 异常时同步释放 Core 资源。 + * + * @param columnIndex 从 1 开始的列序号 + * @return 字符流 + */ + @Override + public Reader getCharacterStream(int columnIndex) { + try { + Reader reader = delegate.getCharacterStream(columnIndex); + return reader == null ? null : managed(reader); + } catch (RuntimeException exception) { + markFailure(exception); + closeAndSuppress(exception); + throw exception; + } + } + + /** + * 复制当前行,Adapter 异常时同步释放 Core 资源。 + * + * @return 当前行值 + */ + @Override + public List row() { + try { + return delegate.row(); + } catch (RuntimeException exception) { + markFailure(exception); + closeAndSuppress(exception); + throw exception; + } + } + + /** + * 先摘除 Engine 游标追踪,再关闭 Adapter 游标并确定性释放 Core 资源。 + */ + @Override + public void close() { + if (!closed.compareAndSet(false, true)) { + return; + } + RuntimeException failure = null; + // 追踪移除必须早于 QueryRegistration 释放,避免并发 cancel 写入伪预取消状态。 + try { + onClose.run(); + } catch (RuntimeException exception) { + failure = exception; + } + try { + delegate.close(); + } catch (RuntimeException exception) { + if (failure == null) { + failure = exception; + } else { + failure.addSuppressed(exception); + } + } + try { + resources.close(); + } catch (Exception exception) { + RuntimeException wrapped = exception instanceof RuntimeException runtimeException + ? runtimeException + : new IllegalStateException("failed to close managed query resources", exception); + if (failure == null) { + failure = wrapped; + } else { + failure.addSuppressed(wrapped); + } + } + if (failure != null) { + throw failure; + } + } + + private InputStream managed(InputStream stream) { + return new FilterInputStream(stream) { + @Override + public int read() throws IOException { + try { + return super.read(); + } catch (IOException | RuntimeException exception) { + failStream(exception); + throw exception; + } + } + + @Override + public int read(byte[] bytes, int offset, int length) throws IOException { + try { + return super.read(bytes, offset, length); + } catch (IOException | RuntimeException exception) { + failStream(exception); + throw exception; + } + } + + @Override + public long skip(long count) throws IOException { + try { + return super.skip(count); + } catch (IOException | RuntimeException exception) { + failStream(exception); + throw exception; + } + } + + @Override + public int available() throws IOException { + try { + return super.available(); + } catch (IOException | RuntimeException exception) { + failStream(exception); + throw exception; + } + } + + @Override + public void reset() throws IOException { + try { + super.reset(); + } catch (IOException | RuntimeException exception) { + failStream(exception); + throw exception; + } + } + + @Override + public void close() throws IOException { + try { + super.close(); + } catch (IOException | RuntimeException exception) { + failStream(exception); + throw exception; + } + } + }; + } + + private Reader managed(Reader reader) { + return new FilterReader(reader) { + @Override + public int read() throws IOException { + try { + return super.read(); + } catch (IOException | RuntimeException exception) { + failStream(exception); + throw exception; + } + } + + @Override + public int read(char[] characters, int offset, int length) throws IOException { + try { + return super.read(characters, offset, length); + } catch (IOException | RuntimeException exception) { + failStream(exception); + throw exception; + } + } + + @Override + public long skip(long count) throws IOException { + try { + return super.skip(count); + } catch (IOException | RuntimeException exception) { + failStream(exception); + throw exception; + } + } + + @Override + public boolean ready() throws IOException { + try { + return super.ready(); + } catch (IOException | RuntimeException exception) { + failStream(exception); + throw exception; + } + } + + @Override + public void mark(int readAheadLimit) throws IOException { + try { + super.mark(readAheadLimit); + } catch (IOException | RuntimeException exception) { + failStream(exception); + throw exception; + } + } + + @Override + public void reset() throws IOException { + try { + super.reset(); + } catch (IOException | RuntimeException exception) { + failStream(exception); + throw exception; + } + } + + @Override + public void close() throws IOException { + try { + super.close(); + } catch (IOException | RuntimeException exception) { + failStream(exception); + throw exception; + } + } + }; + } + + private void failStream(Throwable exception) { + markFailure(exception); + closeAndSuppress(exception); + } + + private void closeAndSuppress(Throwable original) { + try { + close(); + } catch (RuntimeException closeException) { + original.addSuppressed(closeException); + } + } + + private void markFailure(Throwable exception) { + if (delegate instanceof CancellationAwareFederationCursor aware) { + FederationSqlErrorCode errorCode = exception instanceof FederationSqlException sqlException + ? sqlException.errorCode() + : FederationSqlErrorCode.EXECUTION_FAILED; + if (errorCode == FederationSqlErrorCode.QUERY_TIMEOUT) { + aware.markTimedOut(); + } else if (errorCode == FederationSqlErrorCode.QUERY_CANCELLED) { + aware.markCancelled(); + } else { + aware.markFailed(errorCode); + } + } + } +} diff --git a/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/runtime/MetricsFederationResultCursor.java b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/runtime/MetricsFederationResultCursor.java new file mode 100644 index 0000000..21222ee --- /dev/null +++ b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/runtime/MetricsFederationResultCursor.java @@ -0,0 +1,121 @@ +package com.easyagents.federation.sql.runtime; + +import com.easyagents.federation.sql.api.FederationSqlErrorCode; +import com.easyagents.federation.sql.execute.FederationColumn; +import com.easyagents.federation.sql.execute.FederationQueryMetricsSnapshot; +import com.easyagents.federation.sql.execute.FederationResultCursor; +import com.easyagents.federation.sql.execute.QueryId; +import java.io.InputStream; +import java.io.Reader; +import java.util.List; +import java.util.concurrent.atomic.AtomicBoolean; + +/** + * 为单源 Adapter 游标补充统一查询指标。 + */ +final class MetricsFederationResultCursor + implements FederationResultCursor, CancellationAwareFederationCursor { + + private final FederationResultCursor delegate; + private final FederationQueryMetricsTracker metrics; + private final AtomicBoolean closed = new AtomicBoolean(); + + /** + * 创建指标包装游标。 + * + * @param delegate Adapter 游标 + * @param metrics 指标跟踪器 + */ + MetricsFederationResultCursor( + FederationResultCursor delegate, + FederationQueryMetricsTracker metrics + ) { + this.delegate = delegate; + this.metrics = metrics; + } + + /** {@inheritDoc} */ + @Override + public QueryId queryId() { + return delegate.queryId(); + } + + /** {@inheritDoc} */ + @Override + public List columns() { + return delegate.columns(); + } + + /** {@inheritDoc} */ + @Override + public FederationQueryMetricsSnapshot metrics() { + return metrics.snapshot(); + } + + /** {@inheritDoc} */ + @Override + public void markCancelled() { + metrics.markCancelled(); + } + + /** {@inheritDoc} */ + @Override + public void markTimedOut() { + metrics.markTimedOut(); + } + + /** {@inheritDoc} */ + @Override + public void markFailed(FederationSqlErrorCode errorCode) { + metrics.markFailed(errorCode); + } + + /** {@inheritDoc} */ + @Override + public boolean next() { + boolean present = delegate.next(); + if (present) { + metrics.recordSingleSourceRow(); + } else { + metrics.finishFragment("fragment-1"); + metrics.finishSuccessfully(); + } + return present; + } + + /** {@inheritDoc} */ + @Override + public Object getObject(int columnIndex) { + return delegate.getObject(columnIndex); + } + + /** {@inheritDoc} */ + @Override + public InputStream getBinaryStream(int columnIndex) { + return delegate.getBinaryStream(columnIndex); + } + + /** {@inheritDoc} */ + @Override + public Reader getCharacterStream(int columnIndex) { + return delegate.getCharacterStream(columnIndex); + } + + /** {@inheritDoc} */ + @Override + public List row() { + return delegate.row(); + } + + /** {@inheritDoc} */ + @Override + public void close() { + if (closed.compareAndSet(false, true)) { + try { + delegate.close(); + } finally { + metrics.finishClosed(); + } + } + } +} diff --git a/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/runtime/NodeMemoryAdmissionController.java b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/runtime/NodeMemoryAdmissionController.java new file mode 100644 index 0000000..bb60637 --- /dev/null +++ b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/runtime/NodeMemoryAdmissionController.java @@ -0,0 +1,132 @@ +package com.easyagents.federation.sql.runtime; + +import com.easyagents.federation.sql.api.FederationSqlErrorCode; +import com.easyagents.federation.sql.api.FederationSqlException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.locks.Condition; +import java.util.concurrent.locks.ReentrantLock; + +/** + * 节点本地联邦中间结果内存的保守准入控制器。 + */ +final class NodeMemoryAdmissionController { + + private final long maximumBytes; + private final ReentrantLock lock = new ReentrantLock(true); + private final Condition released = lock.newCondition(); + private long reservedBytes; + + /** + * 创建节点内存准入控制器。 + * + * @param maximumBytes 节点允许同时预留的最大字节数 + */ + NodeMemoryAdmissionController(long maximumBytes) { + if (maximumBytes <= 0L) { + throw new IllegalArgumentException("maximumBytes must be positive"); + } + this.maximumBytes = maximumBytes; + } + + /** + * 为一次联邦查询预留最坏情况下的本地中间结果预算。 + * + * @param requestedBytes 查询预算 + * @param deadline 统一查询截止时间 + * @return 内存许可 + */ + Permit acquire(long requestedBytes, QueryDeadline deadline) { + if (requestedBytes <= 0L || requestedBytes > maximumBytes) { + throw new FederationSqlException( + FederationSqlErrorCode.FEDERATION_RESOURCE_LIMIT_EXCEEDED, + "federation intermediate memory budget exceeds the node limit" + ); + } + lock.lock(); + try { + while (maximumBytes - reservedBytes < requestedBytes) { + ensureAdmissionAllowed(deadline); + long waitNanos = Math.max( + 1L, + Math.min(deadline.remainingNanos(), TimeUnit.MILLISECONDS.toNanos(50)) + ); + try { + released.awaitNanos(waitNanos); + } catch (InterruptedException exception) { + Thread.currentThread().interrupt(); + throw new FederationSqlException( + FederationSqlErrorCode.EXECUTION_FAILED, + "waiting for federation node memory admission was interrupted", + exception + ); + } + } + ensureAdmissionAllowed(deadline); + reservedBytes += requestedBytes; + return new Permit(requestedBytes, this); + } finally { + lock.unlock(); + } + } + + private static void ensureAdmissionAllowed(QueryDeadline deadline) { + try { + deadline.ensureAllowed(); + } catch (FederationSqlException exception) { + if (exception.errorCode() == FederationSqlErrorCode.QUERY_CANCELLED) { + throw exception; + } + if (exception.errorCode() == FederationSqlErrorCode.QUERY_TIMEOUT + || exception.errorCode() == FederationSqlErrorCode.SQL_COMPILE_TIMEOUT) { + throw new FederationSqlException( + FederationSqlErrorCode.NODE_MEMORY_ADMISSION_TIMEOUT, + "timed out while waiting for federation node memory admission", + exception + ); + } + throw exception; + } + } + + /** + * 返回无需释放资源的许可。 + * + * @return 空许可 + */ + static Permit none() { + return new Permit(0L, null); + } + + /** + * 节点内存预留许可。 + */ + static final class Permit implements AutoCloseable { + + private final long bytes; + private final NodeMemoryAdmissionController owner; + private final AtomicBoolean closed = new AtomicBoolean(); + + private Permit(long bytes, NodeMemoryAdmissionController owner) { + this.bytes = bytes; + this.owner = owner; + } + + /** + * 释放节点内存预算。 + */ + @Override + public void close() { + if (owner == null || !closed.compareAndSet(false, true)) { + return; + } + owner.lock.lock(); + try { + owner.reservedBytes -= bytes; + owner.released.signalAll(); + } finally { + owner.lock.unlock(); + } + } + } +} diff --git a/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/runtime/PlanCacheKey.java b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/runtime/PlanCacheKey.java new file mode 100644 index 0000000..4c0c944 --- /dev/null +++ b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/runtime/PlanCacheKey.java @@ -0,0 +1,58 @@ +package com.easyagents.federation.sql.runtime; + +import java.util.List; + +/** + * 使用精确 SQL、参数类型、revision、运行指纹和策略版本隔离计划的缓存键。 + * + * @param sql 精确 SQL;精确匹配会减少命中,但不会产生错误碰撞 + * @param scopeChecksum 查询范围稳定校验和 + * @param parameterTypes 参数 JDBC 类型 + * @param runtimeIdentityFingerprint 实际候选 Runtime 身份指纹 + * @param enginePolicyFingerprint Engine 策略实现与版本指纹 + * @param statisticsSnapshotVersion 查询级统计缓存协议版本;实际表统计由计划指纹校验 + * @param requestPolicyVersion 请求级策略上下文版本 + */ +record PlanCacheKey( + String sql, + String scopeChecksum, + List parameterTypes, + String runtimeIdentityFingerprint, + String enginePolicyFingerprint, + String statisticsSnapshotVersion, + String requestPolicyVersion +) { + + /** + * 保留旧单源测试与内部调用的兼容构造器。 + */ + PlanCacheKey( + String sql, + com.easyagents.federation.sql.source.SourceId sourceId, + long minimumRevision, + long sourceRevision, + List parameterTypes, + String adapterId, + String runtimeFingerprint, + String enginePolicyFingerprint, + String requestPolicyVersion, + long sourceGeneration + ) { + this( + sql, + "single:" + sourceId.value() + ':' + minimumRevision, + parameterTypes, + sourceRevision + ":" + adapterId + ':' + runtimeFingerprint + ':' + sourceGeneration, + enginePolicyFingerprint, + "none", + requestPolicyVersion + ); + } + + /** + * 防御性复制参数类型。 + */ + PlanCacheKey { + parameterTypes = List.copyOf(parameterTypes); + } +} diff --git a/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/runtime/QueryCancellationRegistry.java b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/runtime/QueryCancellationRegistry.java new file mode 100644 index 0000000..876b3b3 --- /dev/null +++ b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/runtime/QueryCancellationRegistry.java @@ -0,0 +1,783 @@ +package com.easyagents.federation.sql.runtime; + +import com.easyagents.federation.sql.api.FederationSqlErrorCode; +import com.easyagents.federation.sql.api.FederationSqlException; +import com.easyagents.federation.sql.api.FederationCleanupMetrics; +import com.easyagents.federation.sql.execute.QueryId; +import com.easyagents.federation.sql.execute.StatementLifecycle; +import java.sql.SQLException; +import java.sql.Statement; +import java.time.Duration; +import java.util.Iterator; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ArrayBlockingQueue; +import java.util.concurrent.RejectedExecutionException; +import java.util.concurrent.ScheduledThreadPoolExecutor; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.atomic.AtomicReference; + +/** + * 从 QueryId 建立到 Statement、Cursor 结束的节点本地取消状态。 + */ +final class QueryCancellationRegistry implements AutoCloseable { + + private static final System.Logger LOGGER = System.getLogger( + QueryCancellationRegistry.class.getName() + ); + private static final int MAX_PENDING_CANCELLATIONS = 4_096; + private static final int MAX_DEFERRED_CLEANUPS = 4_096; + private static final long DEFERRED_RETRY_DELAY_MILLIS = 10L; + private static final long PENDING_CANCELLATION_TTL_NANOS = + Duration.ofMinutes(10).toNanos(); + + private final Map queries = new ConcurrentHashMap<>(); + private final Object lifecycleLock = new Object(); + private final LinkedHashMap pendingCancellations = + new LinkedHashMap<>(); + private final AtomicBoolean closed = new AtomicBoolean(); + private final AtomicBoolean acceptingDeferredCleanups = new AtomicBoolean(true); + private final AtomicBoolean retryScheduled = new AtomicBoolean(); + private final AtomicLong overflowFallbacks = new AtomicLong(); + private final AtomicLong deferredRetries = new AtomicLong(); + private final AtomicLong unscheduledCleanups = new AtomicLong(); + private final Object deferredLock = new Object(); + private final ArrayBlockingQueue deferredCleanups = + new ArrayBlockingQueue<>(MAX_DEFERRED_CLEANUPS); + private final ThreadPoolExecutor terminationExecutor; + private final ThreadPoolExecutor cleanupExecutor; + private final ThreadPoolExecutor overflowExecutor; + private final ScheduledThreadPoolExecutor retryExecutor; + + /** + * 创建使用独立 Statement 终止与游标清理队列的取消登记器。 + */ + QueryCancellationRegistry() { + this( + executor("easy-agents-federation-jdbc-cancel"), + executor("easy-agents-federation-cursor-cleanup"), + executor("easy-agents-federation-resource-overflow") + ); + } + + /** + * 创建使用指定执行器的取消登记器,供确定性并发测试使用。 + * + * @param terminationExecutor Statement 终止执行器 + * @param cleanupExecutor 游标清理执行器 + */ + QueryCancellationRegistry( + ThreadPoolExecutor terminationExecutor, + ThreadPoolExecutor cleanupExecutor + ) { + this( + terminationExecutor, + cleanupExecutor, + executor("easy-agents-federation-resource-overflow") + ); + } + + /** + * 创建使用指定主执行器和过载隔离执行器的取消登记器。 + * + * @param terminationExecutor Statement 终止执行器 + * @param cleanupExecutor 游标清理执行器 + * @param overflowExecutor 主队列饱和后的独立隔离执行器 + */ + QueryCancellationRegistry( + ThreadPoolExecutor terminationExecutor, + ThreadPoolExecutor cleanupExecutor, + ThreadPoolExecutor overflowExecutor + ) { + this.terminationExecutor = Objects.requireNonNull( + terminationExecutor, + "terminationExecutor must not be null" + ); + this.cleanupExecutor = Objects.requireNonNull( + cleanupExecutor, + "cleanupExecutor must not be null" + ); + this.overflowExecutor = Objects.requireNonNull( + overflowExecutor, + "overflowExecutor must not be null" + ); + this.retryExecutor = retryExecutor(); + } + + /** + * 在查询进入编译、准入或执行前登记取消状态。 + * + * @param queryId 查询标识 + * @return 查询级登记 + */ + QueryRegistration begin(QueryId queryId) { + QueryState state; + synchronized (lifecycleLock) { + ensureOpen(); + if (consumePendingCancellation(queryId)) { + throw cancelled(queryId, null); + } + state = new QueryState(queryId); + QueryState previous = queries.putIfAbsent(queryId, state); + if (previous != null) { + throw new FederationSqlException( + FederationSqlErrorCode.EXECUTION_FAILED, + "query id is already active: " + queryId.value() + ); + } + } + return new QueryRegistration(queryId, state); + } + + /** + * 取消指定查询。 + * + * @param queryId 查询标识 + * @return 是否找到活动查询;未找到时仍保留有界的短期预取消状态 + */ + boolean cancel(QueryId queryId) { + return cancelOutcome(queryId).found(); + } + + /** + * 取消指定查询并返回原子确定的终止原因。 + * + * @param queryId 查询标识 + * @return 取消观察结果 + */ + CancellationOutcome cancelOutcome(QueryId queryId) { + QueryState state; + synchronized (lifecycleLock) { + state = queries.get(queryId); + if (state == null) { + rememberPendingCancellation(queryId); + return new CancellationOutcome(false, TerminationReason.NONE); + } + } + return new CancellationOutcome(true, state.requestCancellation()); + } + + /** + * 在隔离线程中执行可能被 JDBC 驱动阻塞的游标清理。 + * + * @param cleanup 清理动作 + * @return 是否成功提交 + */ + boolean submitCleanup(Runnable cleanup) { + if (cleanup == null) { + return false; + } + return submit( + cleanupExecutor, + cleanup, + "query cursor cleanup failed", + "cursor cleanup" + ); + } + + /** + * 返回主队列饱和后转交隔离执行器的保护性降级次数。 + * + * @return 隔离降级次数 + */ + long overflowFallbackCount() { + return overflowFallbacks.get(); + } + + /** + * 返回所有隔离队列均饱和后未能调度的清理次数。 + * + * @return 未调度清理次数 + */ + long unscheduledCleanupCount() { + return unscheduledCleanups.get(); + } + + /** + * 返回清理通道的不可变观测快照。 + * + * @return 清理通道指标 + */ + FederationCleanupMetrics cleanupMetrics() { + return new FederationCleanupMetrics( + overflowFallbacks.get(), + deferredRetries.get(), + unscheduledCleanups.get(), + deferredCleanups.size() + ); + } + + /** + * 尝试取消当前节点所有查询,并保留第一个错误。 + */ + @Override + public void close() { + synchronized (lifecycleLock) { + if (!closed.compareAndSet(false, true)) { + return; + } + pendingCancellations.clear(); + } + for (QueryState state : queries.values()) { + state.requestCancellation(); + } + transferQueuedOnShutdown(terminationExecutor, "JDBC statement termination"); + transferQueuedOnShutdown(cleanupExecutor, "cursor cleanup"); + drainDeferredCleanups(); + acceptingDeferredCleanups.set(false); + retryExecutor.shutdownNow(); + shutdownOverflow(); + } + + private void ensureOpen() { + if (closed.get()) { + throw new FederationSqlException( + FederationSqlErrorCode.ENGINE_CLOSED, + "query cancellation registry is closed" + ); + } + } + + private boolean consumePendingCancellation(QueryId queryId) { + cleanupPendingCancellations(System.nanoTime()); + return pendingCancellations.remove(queryId) != null; + } + + private void rememberPendingCancellation(QueryId queryId) { + long now = System.nanoTime(); + cleanupPendingCancellations(now); + pendingCancellations.put( + queryId, + now + PENDING_CANCELLATION_TTL_NANOS + ); + while (pendingCancellations.size() > MAX_PENDING_CANCELLATIONS) { + Iterator iterator = pendingCancellations.keySet().iterator(); + iterator.next(); + iterator.remove(); + } + } + + private void cleanupPendingCancellations(long now) { + pendingCancellations.entrySet().removeIf( + entry -> entry.getValue() - now <= 0L + ); + } + + /** + * 单次查询的取消登记,直到 Cursor 和 Core 资源完成关闭后才清理终态。 + */ + final class QueryRegistration implements AutoCloseable { + + private final QueryId queryId; + private final QueryState state; + private final AtomicBoolean released = new AtomicBoolean(); + + private QueryRegistration(QueryId queryId, QueryState state) { + this.queryId = queryId; + this.state = state; + } + + /** + * 返回 Adapter 用于登记 Statement 的生命周期回调。 + * + * @return Statement 生命周期 + */ + StatementLifecycle statementLifecycle() { + return state.statementLifecycle(); + } + + /** + * 返回查询是否已经收到取消请求。 + * + * @return 是否已取消 + */ + boolean cancellationRequested() { + return state.cancellationRequested(); + } + + /** + * 已取消时立即抛出稳定错误。 + */ + void ensureNotCancelled() { + state.ensureActive(); + } + + /** + * 将查询推进为超时终态,并关闭当前已登记的全部 Statement。 + */ + void requestTimeout() { + state.requestTimeout(); + } + + /** + * 将查询推进为超时并返回原子确定的终止原因。 + * + * @return 已确定的终止原因 + */ + TerminationReason requestTimeoutOutcome() { + return state.requestTimeout(); + } + + /** + * 返回查询是否已经超过统一执行时限。 + * + * @return 是否已超时 + */ + boolean timeoutRequested() { + return state.timeoutRequested(); + } + + /** + * 清理活动 QueryId;状态对象仍供并发 JDBC 消费线程判断取消原因。 + */ + @Override + public void close() { + if (released.compareAndSet(false, true)) { + queries.remove(queryId, state); + state.releaseStatementReference(); + } + } + } + + private final class QueryState { + + private final QueryId queryId; + private final Map statements = new ConcurrentHashMap<>(); + private final AtomicReference terminationReason = + new AtomicReference<>(TerminationReason.NONE); + private final StatementLifecycle statementLifecycle = new StatementLifecycle() { + @Override + public void register(Statement candidate) { + if (candidate == null) { + throw new IllegalArgumentException("statement must not be null"); + } + ensureActive(); + StatementTermination termination = new StatementTermination(candidate); + StatementTermination previous = statements.putIfAbsent(candidate, termination); + if (previous != null && previous.statement != candidate) { + throw new FederationSqlException( + FederationSqlErrorCode.EXECUTION_FAILED, + "query id already registered a different statement instance" + ); + } + if (terminationRequested()) { + StatementTermination active = previous == null ? termination : previous; + submitTermination( + () -> active.terminate(queryId), + "late JDBC statement termination failed" + ); + ensureActive(); + } + } + + @Override + public void unregister(Statement candidate) { + statements.remove(candidate); + } + + @Override + public boolean cancellationRequested() { + return QueryState.this.terminationRequested(); + } + + @Override + public boolean timeoutRequested() { + return QueryState.this.timeoutRequested(); + } + }; + + private QueryState(QueryId queryId) { + this.queryId = queryId; + } + + private StatementLifecycle statementLifecycle() { + return statementLifecycle; + } + + private boolean cancellationRequested() { + return terminationRequested(); + } + + private boolean terminationRequested() { + return terminationReason.get() != TerminationReason.NONE; + } + + private boolean timeoutRequested() { + return terminationReason.get() == TerminationReason.TIMED_OUT; + } + + private void ensureActive() { + TerminationReason reason = terminationReason.get(); + if (reason == TerminationReason.TIMED_OUT) { + throw timedOut(queryId, null); + } + if (reason == TerminationReason.CANCELLED) { + throw cancelled(queryId, null); + } + } + + private TerminationReason requestCancellation() { + if (terminationReason.compareAndSet( + TerminationReason.NONE, + TerminationReason.CANCELLED + )) { + terminateStatements(); + } + return terminationReason.get(); + } + + private TerminationReason requestTimeout() { + if (terminationReason.compareAndSet( + TerminationReason.NONE, + TerminationReason.TIMED_OUT + )) { + terminateStatements(); + } + return terminationReason.get(); + } + + private void terminateStatements() { + for (StatementTermination termination : List.copyOf(statements.values())) { + submitTermination( + () -> termination.terminate(queryId), + "JDBC statement termination failed" + ); + } + } + + private void releaseStatementReference() { + statements.clear(); + } + } + + enum TerminationReason { + NONE, + CANCELLED, + TIMED_OUT + } + + /** + * 一次取消请求观察到的活动状态和原子终态。 + * + * @param found 是否找到活动查询 + * @param reason 已确定的终止原因 + */ + record CancellationOutcome(boolean found, TerminationReason reason) { + } + + /** + * 每个 Statement 独立维护取消和关闭幂等状态。 + */ + private static final class StatementTermination { + + private final Statement statement; + private final AtomicBoolean cancelIssued = new AtomicBoolean(); + private final AtomicBoolean closeIssued = new AtomicBoolean(); + + private StatementTermination(Statement statement) { + this.statement = statement; + } + + private void terminate(QueryId queryId) { + SQLException failure = null; + if (cancelIssued.compareAndSet(false, true)) { + try { + statement.cancel(); + } catch (SQLException exception) { + cancelIssued.set(false); + failure = exception; + } + } + // close() 同时覆盖 register 与 executeQuery 之间的 JDBC 取消空窗。 + if (closeIssued.compareAndSet(false, true)) { + try { + statement.close(); + } catch (SQLException exception) { + closeIssued.set(false); + if (failure == null) { + failure = exception; + } else { + failure.addSuppressed(exception); + } + } + } + if (failure != null) { + throw new FederationSqlException( + FederationSqlErrorCode.EXECUTION_FAILED, + "failed to cancel query " + queryId.value(), + failure + ); + } + } + } + + private static FederationSqlException cancelled(QueryId queryId, Throwable cause) { + String message = "query was cancelled: " + queryId.value(); + return cause == null + ? new FederationSqlException(FederationSqlErrorCode.QUERY_CANCELLED, message) + : new FederationSqlException(FederationSqlErrorCode.QUERY_CANCELLED, message, cause); + } + + private static FederationSqlException timedOut(QueryId queryId, Throwable cause) { + String message = "query execution timed out: " + queryId.value(); + return cause == null + ? new FederationSqlException(FederationSqlErrorCode.QUERY_TIMEOUT, message) + : new FederationSqlException(FederationSqlErrorCode.QUERY_TIMEOUT, message, cause); + } + + private boolean submitTermination(Runnable action, String failureMessage) { + return submit( + terminationExecutor, + action, + failureMessage, + "JDBC statement termination" + ); + } + + private boolean submit( + ThreadPoolExecutor executor, + Runnable action, + String failureMessage, + String actionLabel + ) { + Runnable guarded = () -> { + try { + runGuarded(action, failureMessage); + } finally { + drainDeferredCleanups(); + } + }; + try { + executor.execute(guarded); + return true; + } catch (RejectedExecutionException exception) { + overflowFallbacks.incrementAndGet(); + LOGGER.log( + System.Logger.Level.WARNING, + actionLabel + + " queue is unavailable; handing cleanup to the overflow isolator: " + + exception.getMessage() + ); + submitOverflow(guarded, actionLabel); + return false; + } + } + + private void submitOverflow(Runnable guarded, String actionLabel) { + try { + overflowExecutor.execute(guarded); + } catch (RejectedExecutionException overflow) { + deferCleanup(new DeferredCleanup(guarded, actionLabel), overflow); + } + } + + private void deferCleanup(DeferredCleanup cleanup, RejectedExecutionException cause) { + boolean accepted; + synchronized (deferredLock) { + accepted = acceptingDeferredCleanups.get() && deferredCleanups.offer(cleanup); + } + if (!accepted) { + recordUnresolved( + 1L, + cleanup.actionLabel() + + " could not be retained because all bounded cleanup channels are saturated", + cause + ); + return; + } + deferredRetries.incrementAndGet(); + LOGGER.log( + System.Logger.Level.WARNING, + cleanup.actionLabel() + + " entered the bounded deferred cleanup queue after both executors rejected it" + ); + scheduleDeferredRetry(); + } + + private void scheduleDeferredRetry() { + if (!acceptingDeferredCleanups.get() + || !retryScheduled.compareAndSet(false, true)) { + return; + } + try { + retryExecutor.schedule( + this::retryDeferredCleanups, + DEFERRED_RETRY_DELAY_MILLIS, + TimeUnit.MILLISECONDS + ); + } catch (RejectedExecutionException exception) { + retryScheduled.set(false); + if (acceptingDeferredCleanups.get()) { + LOGGER.log( + System.Logger.Level.ERROR, + "deferred JDBC cleanup retry scheduler is unavailable", + exception + ); + } + } + } + + private void retryDeferredCleanups() { + retryScheduled.set(false); + drainDeferredCleanups(); + if (!deferredCleanups.isEmpty()) { + scheduleDeferredRetry(); + } + } + + private void drainDeferredCleanups() { + while (acceptingDeferredCleanups.get()) { + DeferredCleanup cleanup; + synchronized (deferredLock) { + cleanup = deferredCleanups.poll(); + } + if (cleanup == null) { + return; + } + try { + overflowExecutor.execute(cleanup.action()); + } catch (RejectedExecutionException exception) { + boolean restored; + synchronized (deferredLock) { + restored = acceptingDeferredCleanups.get() + && deferredCleanups.offer(cleanup); + } + if (!restored) { + recordUnresolved( + 1L, + cleanup.actionLabel() + + " could not be retained while retrying deferred cleanup", + exception + ); + } else { + scheduleDeferredRetry(); + } + return; + } + } + } + + private static void runGuarded(Runnable action, String failureMessage) { + try { + action.run(); + } catch (RuntimeException exception) { + LOGGER.log(System.Logger.Level.ERROR, failureMessage, exception); + } + } + + private void transferQueuedOnShutdown( + ThreadPoolExecutor executor, + String actionLabel + ) { + executor.shutdown(); + boolean interrupted = false; + List queued = List.of(); + boolean terminated = false; + try { + if (!executor.awaitTermination(200, TimeUnit.MILLISECONDS)) { + queued = executor.shutdownNow(); + } else { + terminated = true; + } + } catch (InterruptedException exception) { + interrupted = true; + queued = executor.shutdownNow(); + } + // 关闭线程只做有界等待;尚未开始的阻塞清理转交隔离执行器。 + queued.forEach(task -> submitOverflow(task, actionLabel)); + if (!terminated && executor.getActiveCount() > 0) { + recordUnresolved( + executor.getActiveCount(), + actionLabel + " remained active after the bounded engine shutdown window", + null + ); + } + if (interrupted) { + Thread.currentThread().interrupt(); + } + } + + private void shutdownOverflow() { + overflowExecutor.shutdown(); + boolean interrupted = false; + List discarded = List.of(); + try { + if (!overflowExecutor.awaitTermination(200, TimeUnit.MILLISECONDS)) { + discarded = overflowExecutor.shutdownNow(); + } + } catch (InterruptedException exception) { + interrupted = true; + discarded = overflowExecutor.shutdownNow(); + } + int deferred; + synchronized (deferredLock) { + deferred = deferredCleanups.size(); + deferredCleanups.clear(); + } + long unresolved = (long) discarded.size() + + overflowExecutor.getActiveCount() + + deferred; + if (unresolved > 0L) { + recordUnresolved( + unresolved, + "JDBC cleanup tasks remained after the bounded engine shutdown window", + null + ); + } + if (interrupted) { + Thread.currentThread().interrupt(); + } + } + + private static ThreadPoolExecutor executor(String threadName) { + return new ThreadPoolExecutor( + 2, + 2, + 0L, + TimeUnit.MILLISECONDS, + new ArrayBlockingQueue<>(1_024), + runnable -> { + Thread thread = new Thread(runnable, threadName); + thread.setDaemon(true); + return thread; + }, + new ThreadPoolExecutor.AbortPolicy() + ); + } + + private static ScheduledThreadPoolExecutor retryExecutor() { + ScheduledThreadPoolExecutor executor = new ScheduledThreadPoolExecutor( + 1, + runnable -> { + Thread thread = new Thread( + runnable, + "easy-agents-federation-resource-retry" + ); + thread.setDaemon(true); + return thread; + }, + new ThreadPoolExecutor.AbortPolicy() + ); + executor.setExecuteExistingDelayedTasksAfterShutdownPolicy(false); + executor.setRemoveOnCancelPolicy(true); + return executor; + } + + private void recordUnresolved(long count, String message, Throwable cause) { + unscheduledCleanups.addAndGet(count); + if (cause == null) { + LOGGER.log(System.Logger.Level.ERROR, count + " " + message); + } else { + LOGGER.log(System.Logger.Level.ERROR, count + " " + message, cause); + } + } + + private record DeferredCleanup(Runnable action, String actionLabel) { + } +} diff --git a/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/runtime/QueryDeadline.java b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/runtime/QueryDeadline.java new file mode 100644 index 0000000..2d747a3 --- /dev/null +++ b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/runtime/QueryDeadline.java @@ -0,0 +1,127 @@ +package com.easyagents.federation.sql.runtime; + +import com.easyagents.federation.sql.api.FederationSqlErrorCode; +import com.easyagents.federation.sql.api.FederationSqlException; +import com.easyagents.federation.sql.execute.FederationExecutionGuard; +import java.time.Duration; + +/** + * 一次查询从编译到游标关闭共享的绝对截止时间。 + */ +final class QueryDeadline implements FederationExecutionGuard, BoundedPlanCache.WaitGuard { + + private final long deadlineNanos; + private final QueryCancellationRegistry.QueryRegistration registration; + private final FederationSqlErrorCode timeoutCode; + private final String timeoutMessage; + + private QueryDeadline( + long deadlineNanos, + QueryCancellationRegistry.QueryRegistration registration, + FederationSqlErrorCode timeoutCode, + String timeoutMessage + ) { + this.deadlineNanos = deadlineNanos; + this.registration = registration; + this.timeoutCode = timeoutCode; + this.timeoutMessage = timeoutMessage; + } + + /** + * 计算从当前时刻开始的饱和绝对截止时间。 + * + * @param duration 最大持续时间 + * @return 绝对截止纳秒 + */ + static long deadlineAfter(Duration duration) { + return deadlineAfter(System.nanoTime(), duration); + } + + /** + * 基于指定单调时钟值计算截止时间,允许 {@link System#nanoTime()} 自然回绕。 + * + * @param now 当前单调时钟值 + * @param duration 最大持续时间 + * @return 绝对截止纳秒;持续时间无法转换时返回无限时限标记 + */ + static long deadlineAfter(long now, Duration duration) { + long nanos; + try { + nanos = duration.toNanos(); + } catch (ArithmeticException exception) { + return Long.MAX_VALUE; + } + if (nanos == Long.MAX_VALUE) { + return Long.MAX_VALUE; + } + // nanoTime 只保证差值语义;直接相加并允许补码回绕,剩余时间差仍然正确。 + return now + nanos; + } + + /** + * 创建仅约束公开编译调用的截止时间。 + * + * @param duration 最大编译时间 + * @return 编译截止时间 + */ + static QueryDeadline compile(Duration duration) { + return new QueryDeadline( + deadlineAfter(duration), + null, + FederationSqlErrorCode.SQL_COMPILE_TIMEOUT, + "SQL compilation exceeded the configured deadline" + ); + } + + /** + * 创建绑定查询取消终态的截止时间。 + * + * @param deadlineNanos 请求级绝对截止时间 + * @param registration 查询登记 + * @return 查询截止时间 + */ + static QueryDeadline query( + long deadlineNanos, + QueryCancellationRegistry.QueryRegistration registration + ) { + return new QueryDeadline( + deadlineNanos, + registration, + FederationSqlErrorCode.QUERY_TIMEOUT, + "query exceeded the configured deadline" + ); + } + + /** {@inheritDoc} */ + @Override + public void ensureAllowed() { + if (registration != null) { + registration.ensureNotCancelled(); + } + if (remainingNanos() <= 0L) { + requestTimeout(); + if (registration != null) { + registration.ensureNotCancelled(); + } + throw new FederationSqlException(timeoutCode, timeoutMessage); + } + } + + /** {@inheritDoc} */ + @Override + public long remainingNanos() { + if (deadlineNanos == Long.MAX_VALUE) { + return Long.MAX_VALUE; + } + return deadlineNanos - System.nanoTime(); + } + + /** + * 将绑定查询推进为超时终态。 + */ + QueryCancellationRegistry.TerminationReason requestTimeout() { + return registration == null + ? QueryCancellationRegistry.TerminationReason.NONE + : registration.requestTimeoutOutcome(); + } +} diff --git a/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/runtime/SourceCatalogSnapshot.java b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/runtime/SourceCatalogSnapshot.java new file mode 100644 index 0000000..c229947 --- /dev/null +++ b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/runtime/SourceCatalogSnapshot.java @@ -0,0 +1,20 @@ +package com.easyagents.federation.sql.runtime; + +import com.easyagents.federation.sql.source.SourceId; +import java.util.Set; + +/** + * 原子读取的数据源注册表代次与标识快照。 + * + * @param generation 注册表代次 + * @param sourceIds 当前已知数据源标识 + */ +record SourceCatalogSnapshot(long generation, Set sourceIds) { + + /** + * 防御性复制数据源集合。 + */ + SourceCatalogSnapshot { + sourceIds = Set.copyOf(sourceIds); + } +} diff --git a/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/runtime/SourceRuntime.java b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/runtime/SourceRuntime.java new file mode 100644 index 0000000..185d5bc --- /dev/null +++ b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/runtime/SourceRuntime.java @@ -0,0 +1,196 @@ +package com.easyagents.federation.sql.runtime; + +import com.easyagents.federation.sql.adapter.AdapterCompatibility; +import com.easyagents.federation.sql.adapter.FederationSqlAdapterProvider; +import com.easyagents.federation.sql.api.FederationSqlErrorCode; +import com.easyagents.federation.sql.api.FederationSqlException; +import com.easyagents.federation.sql.source.FederationDataSourceHandle; +import com.easyagents.federation.sql.source.FederationSourceDefinition; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.Consumer; +import org.apache.calcite.schema.SchemaPlus; +import org.apache.calcite.sql.SqlDialect; + +/** + * 单个 sourceId + revision 的节点本地 Calcite 与 DataSource 运行对象。 + */ +final class SourceRuntime { + + private final FederationSourceDefinition definition; + private final String sourceChecksum; + private final String runtimeFingerprint; + private final FederationDataSourceHandle handle; + private final FederationSqlAdapterProvider adapter; + private final SqlDialect dialect; + private final SchemaPlus rootSchema; + private final SchemaPlus defaultSchema; + private final AdapterCompatibility compatibility; + private final Consumer closedListener; + private final AtomicInteger leases = new AtomicInteger(); + private final AtomicBoolean retiring = new AtomicBoolean(); + private final AtomicBoolean closed = new AtomicBoolean(); + + SourceRuntime( + FederationSourceDefinition definition, + FederationDataSourceHandle handle, + FederationSqlAdapterProvider adapter, + SqlDialect dialect, + SchemaPlus rootSchema, + SchemaPlus defaultSchema, + AdapterCompatibility compatibility, + Consumer closedListener + ) { + this.definition = definition; + this.sourceChecksum = definition.checksum(); + this.runtimeFingerprint = handle.fingerprint().cacheKey(); + this.handle = handle; + this.adapter = adapter; + this.dialect = dialect; + this.rootSchema = rootSchema; + this.defaultSchema = defaultSchema; + this.compatibility = compatibility; + this.closedListener = closedListener; + } + + /** + * 返回不可变数据源定义。 + * + * @return 数据源定义 + */ + FederationSourceDefinition definition() { + return definition; + } + + /** + * 返回 Runtime 构建时缓存的 Definition 校验和。 + * + * @return Definition 校验和 + */ + String sourceChecksum() { + return sourceChecksum; + } + + /** + * 返回 Runtime 构建时缓存的数据库与驱动指纹。 + * + * @return Runtime 指纹 + */ + String runtimeFingerprint() { + return runtimeFingerprint; + } + + FederationDataSourceHandle handle() { + return handle; + } + + FederationSqlAdapterProvider adapter() { + return adapter; + } + + SqlDialect dialect() { + return dialect; + } + + SchemaPlus rootSchema() { + return rootSchema; + } + + SchemaPlus defaultSchema() { + return defaultSchema; + } + + AdapterCompatibility compatibility() { + return compatibility; + } + + RuntimeLease acquire() { + while (true) { + if (retiring.get() || closed.get()) { + throw new FederationSqlException( + FederationSqlErrorCode.SOURCE_REVISION_NOT_READY, + "source runtime is retiring: " + definition.sourceId() + ); + } + leases.incrementAndGet(); + if (!retiring.get() && !closed.get()) { + return new RuntimeLease(this); + } + release(); + } + } + + void retire() { + retiring.set(true); + closeWhenUnused(); + } + + void forceClose() { + retiring.set(true); + closeHandle(); + } + + private void release() { + int remaining = leases.decrementAndGet(); + if (remaining < 0) { + throw new IllegalStateException("runtime lease count became negative"); + } + closeWhenUnused(); + } + + private void closeWhenUnused() { + if (retiring.get() && leases.get() == 0) { + closeHandle(); + } + } + + private synchronized void closeHandle() { + if (closed.get()) { + closedListener.accept(this); + return; + } + RuntimeException failure = null; + for (int attempt = 0; attempt < 2; attempt++) { + try { + handle.close(); + closed.set(true); + closedListener.accept(this); + return; + } catch (RuntimeException exception) { + if (failure == null) { + failure = exception; + } else { + failure.addSuppressed(exception); + } + } + } + throw failure; + } + + /** + * 对旧 Runtime 的引用租约,防止流式游标消费期间提前关闭 Handle。 + */ + static final class RuntimeLease implements AutoCloseable { + + private final SourceRuntime runtime; + private final AtomicBoolean released = new AtomicBoolean(); + + private RuntimeLease(SourceRuntime runtime) { + this.runtime = runtime; + } + + SourceRuntime runtime() { + return runtime; + } + + /** + * 幂等释放 Runtime 租约。 + */ + @Override + public void close() { + if (released.compareAndSet(false, true)) { + runtime.release(); + } + } + } +} diff --git a/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/runtime/TypedSqlDynamicParam.java b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/runtime/TypedSqlDynamicParam.java new file mode 100644 index 0000000..0c03199 --- /dev/null +++ b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/runtime/TypedSqlDynamicParam.java @@ -0,0 +1,47 @@ +package com.easyagents.federation.sql.runtime; + +import org.apache.calcite.sql.SqlDataTypeSpec; +import org.apache.calcite.sql.SqlDynamicParam; +import org.apache.calcite.sql.SqlNode; +import org.apache.calcite.sql.parser.SqlParserPos; +import org.apache.calcite.sql.validate.SqlValidator; +import org.apache.calcite.sql.validate.SqlValidatorScope; + +/** + * 将调用方声明的 JDBC 参数类型直接交给 Calcite Validator 的动态参数节点。 + * + *

该节点仍按普通 {@code ?} 输出,不向目标数据库注入额外 CAST,且可覆盖 + * Calcite 对 OFFSET/FETCH 动态参数无法自行推导类型的路径。

+ */ +final class TypedSqlDynamicParam extends SqlDynamicParam { + + private final SqlDataTypeSpec typeSpec; + + /** + * 创建带声明类型的动态参数。 + * + * @param index 原 SQL 中的零基参数索引 + * @param position 参数语法位置 + * @param typeSpec Adapter 提供的 Calcite 类型声明 + */ + TypedSqlDynamicParam(int index, SqlParserPos position, SqlDataTypeSpec typeSpec) { + super(index, position); + if (typeSpec == null) { + throw new IllegalArgumentException("typeSpec must not be null"); + } + this.typeSpec = typeSpec; + } + + /** {@inheritDoc} */ + @Override + public void validate(SqlValidator validator, SqlValidatorScope scope) { + validator.validateDynamicParam(this); + validator.setValidatedNodeType(this, typeSpec.deriveType(validator)); + } + + /** {@inheritDoc} */ + @Override + public SqlNode clone(SqlParserPos position) { + return new TypedSqlDynamicParam(getIndex(), position, typeSpec); + } +} diff --git a/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/source/ActiveSourceState.java b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/source/ActiveSourceState.java new file mode 100644 index 0000000..291072e --- /dev/null +++ b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/source/ActiveSourceState.java @@ -0,0 +1,58 @@ +package com.easyagents.federation.sql.source; + +/** + * 已启用的数据源共享状态。 + * + * @param definition 数据源定义 + * @param checksum 定义校验和 + */ +public record ActiveSourceState( + FederationSourceDefinition definition, + String checksum +) implements FederationSourceState { + + /** + * 校验启用状态。 + */ + public ActiveSourceState { + if (definition == null) { + throw new IllegalArgumentException("definition must not be null"); + } + if (checksum == null || checksum.isBlank()) { + throw new IllegalArgumentException("checksum must not be blank"); + } + if (!checksum.equals(definition.checksum())) { + throw new IllegalArgumentException("checksum does not match source definition"); + } + } + + /** + * 从 Definition 创建启用状态。 + * + * @param definition 数据源定义 + * @return 启用状态 + */ + public static ActiveSourceState of(FederationSourceDefinition definition) { + return new ActiveSourceState(definition, definition.checksum()); + } + + /** + * 返回数据源标识。 + * + * @return 数据源标识 + */ + @Override + public SourceId sourceId() { + return definition.sourceId(); + } + + /** + * 返回定义版本。 + * + * @return 定义版本 + */ + @Override + public long revision() { + return definition.revision(); + } +} diff --git a/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/source/ExternalSchemaDefinition.java b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/source/ExternalSchemaDefinition.java new file mode 100644 index 0000000..eb6a076 --- /dev/null +++ b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/source/ExternalSchemaDefinition.java @@ -0,0 +1,42 @@ +package com.easyagents.federation.sql.source; + +import java.util.List; + +/** + * 调用方管理的外部 Schema 快照引用。 + * + * @param logicalName SQL 中的逻辑 Schema 名称 + * @param schemaRef 外部 Schema 引用 + * @param schemaRevision 外部 Schema 版本 + */ +public record ExternalSchemaDefinition( + String logicalName, + String schemaRef, + long schemaRevision +) implements FederationSchemaDefinition { + + /** + * 校验外部 Schema 定义。 + */ + public ExternalSchemaDefinition { + if (logicalName == null || logicalName.isBlank()) { + throw new IllegalArgumentException("logicalName must not be blank"); + } + if (schemaRef == null || schemaRef.isBlank()) { + throw new IllegalArgumentException("schemaRef must not be blank"); + } + if (schemaRevision < 0) { + throw new IllegalArgumentException("schemaRevision must not be negative"); + } + } + + /** + * 返回外部 Schema 的稳定校验和材料。 + * + * @return 稳定材料 + */ + @Override + public List checksumFields() { + return List.of(schemaRef, Long.toString(schemaRevision)); + } +} diff --git a/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/source/FederationDataSourceHandle.java b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/source/FederationDataSourceHandle.java new file mode 100644 index 0000000..925d4ff --- /dev/null +++ b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/source/FederationDataSourceHandle.java @@ -0,0 +1,29 @@ +package com.easyagents.federation.sql.source; + +import javax.sql.DataSource; + +/** + * 调用方提供的数据源运行句柄,明确表达连接池所有权与关闭语义。 + */ +public interface FederationDataSourceHandle extends AutoCloseable { + + /** + * 返回已配置的数据源或连接池。 + * + * @return JDBC DataSource + */ + DataSource dataSource(); + + /** + * 返回不含敏感信息的运行指纹。 + * + * @return 运行指纹 + */ + RuntimeFingerprint fingerprint(); + + /** + * 按调用方声明的所有权规则关闭句柄。 + */ + @Override + void close(); +} diff --git a/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/source/FederationDataSourceHandles.java b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/source/FederationDataSourceHandles.java new file mode 100644 index 0000000..eba8394 --- /dev/null +++ b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/source/FederationDataSourceHandles.java @@ -0,0 +1,68 @@ +package com.easyagents.federation.sql.source; + +import java.util.Objects; +import javax.sql.DataSource; + +/** + * 创建共享或独占 DataSource Handle 的便捷方法。 + */ +public final class FederationDataSourceHandles { + + private FederationDataSourceHandles() { + } + + /** + * 创建不关闭底层共享池的 Handle。 + * + * @param dataSource 共享 DataSource + * @param fingerprint 运行指纹 + * @return 共享 Handle + */ + public static FederationDataSourceHandle shared( + DataSource dataSource, + RuntimeFingerprint fingerprint + ) { + return owned(dataSource, fingerprint, () -> { }); + } + + /** + * 创建关闭时执行指定回调的独占 Handle。 + * + * @param dataSource DataSource 或连接池 + * @param fingerprint 运行指纹 + * @param closeAction 独占资源关闭动作 + * @return 独占 Handle + */ + public static FederationDataSourceHandle owned( + DataSource dataSource, + RuntimeFingerprint fingerprint, + Runnable closeAction + ) { + Objects.requireNonNull(dataSource, "dataSource must not be null"); + Objects.requireNonNull(fingerprint, "fingerprint must not be null"); + Objects.requireNonNull(closeAction, "closeAction must not be null"); + return new FederationDataSourceHandle() { + private boolean closed; + + @Override + public DataSource dataSource() { + return dataSource; + } + + @Override + public RuntimeFingerprint fingerprint() { + return fingerprint; + } + + @Override + public synchronized void close() { + if (closed) { + return; + } + // 仅在底层资源确实释放后提交关闭状态,允许瞬时失败后安全重试。 + closeAction.run(); + closed = true; + } + }; + } +} diff --git a/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/source/FederationDataSourceResolver.java b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/source/FederationDataSourceResolver.java new file mode 100644 index 0000000..7e8a544 --- /dev/null +++ b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/source/FederationDataSourceResolver.java @@ -0,0 +1,16 @@ +package com.easyagents.federation.sql.source; + +/** + * 将无凭据 Definition 解析为节点本地 DataSource 句柄。 + */ +@FunctionalInterface +public interface FederationDataSourceResolver { + + /** + * 解析指定数据源版本。 + * + * @param definition 数据源定义 + * @return 节点本地句柄 + */ + FederationDataSourceHandle resolve(FederationSourceDefinition definition); +} diff --git a/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/source/FederationSchemaDefinition.java b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/source/FederationSchemaDefinition.java new file mode 100644 index 0000000..532f4a8 --- /dev/null +++ b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/source/FederationSchemaDefinition.java @@ -0,0 +1,24 @@ +package com.easyagents.federation.sql.source; + +import java.io.Serializable; +import java.util.List; + +/** + * Adapter 可扩展的逻辑 Schema 定义,不承载连接凭据或运行时对象。 + */ +public interface FederationSchemaDefinition extends Serializable { + + /** + * 返回 SQL 中使用的逻辑 Schema 名称。 + * + * @return 逻辑名称 + */ + String logicalName(); + + /** + * 返回逻辑名称之外的跨节点稳定校验和字段;实现不得包含对象 identity 或敏感信息。 + * + * @return 稳定校验和字段列表,字段值允许为 null + */ + List checksumFields(); +} diff --git a/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/source/FederationSourceDefinition.java b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/source/FederationSourceDefinition.java new file mode 100644 index 0000000..c8441d1 --- /dev/null +++ b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/source/FederationSourceDefinition.java @@ -0,0 +1,112 @@ +package com.easyagents.federation.sql.source; + +import java.io.Serializable; +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.HashSet; +import java.util.HexFormat; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.TreeMap; + +/** + * 可持久化、可跨节点传递的数据源定义。 + * + * @param sourceId 逻辑数据源标识 + * @param revision 单调递增版本 + * @param adapterId Adapter 标识 + * @param schemas Schema 定义 + * @param adapterOptions 不含凭据的 Adapter 选项 + */ +public record FederationSourceDefinition( + SourceId sourceId, + long revision, + String adapterId, + List schemas, + Map adapterOptions +) implements Serializable { + + /** + * 校验并防御性复制数据源定义。 + */ + public FederationSourceDefinition { + if (sourceId == null) { + throw new IllegalArgumentException("sourceId must not be null"); + } + if (revision < 0) { + throw new IllegalArgumentException("revision must not be negative"); + } + if (adapterId == null || adapterId.isBlank()) { + throw new IllegalArgumentException("adapterId must not be blank"); + } + schemas = List.copyOf(schemas == null ? List.of() : schemas); + if (schemas.isEmpty()) { + throw new IllegalArgumentException("schemas must not be empty"); + } + Set logicalNames = new HashSet<>(); + for (FederationSchemaDefinition schema : schemas) { + if (schema == null || !logicalNames.add(schema.logicalName())) { + throw new IllegalArgumentException("schema definitions must be non-null and uniquely named"); + } + } + adapterOptions = Map.copyOf(adapterOptions == null ? Map.of() : adapterOptions); + } + + /** + * 计算不包含凭据的稳定定义校验和。 + * + * @return SHA-256 十六进制校验和 + */ + public String checksum() { + try { + MessageDigest digest = MessageDigest.getInstance("SHA-256"); + updateField(digest, sourceId.value()); + updateField(digest, Long.toString(revision)); + updateField(digest, adapterId); + List orderedSchemas = new ArrayList<>(schemas); + orderedSchemas.sort(Comparator.comparing(FederationSchemaDefinition::logicalName)); + updateCount(digest, orderedSchemas.size()); + for (FederationSchemaDefinition schema : orderedSchemas) { + updateField(digest, schema.getClass().getName()); + updateField(digest, schema.logicalName()); + List checksumFields = schema.checksumFields(); + updateCount(digest, checksumFields.size()); + for (String field : checksumFields) { + updateNullableField(digest, field); + } + } + Map orderedOptions = new TreeMap<>(adapterOptions); + updateCount(digest, orderedOptions.size()); + orderedOptions.forEach((key, value) -> { + updateField(digest, key); + updateField(digest, value); + }); + return HexFormat.of().formatHex(digest.digest()); + } catch (NoSuchAlgorithmException exception) { + throw new IllegalStateException("SHA-256 is not available", exception); + } + } + + private static void updateField(MessageDigest digest, String value) { + byte[] bytes = value.getBytes(StandardCharsets.UTF_8); + updateCount(digest, bytes.length); + digest.update(bytes); + } + + private static void updateCount(MessageDigest digest, int value) { + digest.update(ByteBuffer.allocate(Integer.BYTES).putInt(value).array()); + } + + private static void updateNullableField(MessageDigest digest, String value) { + if (value == null) { + updateCount(digest, -1); + return; + } + updateField(digest, value); + } +} diff --git a/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/source/FederationSourceManager.java b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/source/FederationSourceManager.java new file mode 100644 index 0000000..303cf2d --- /dev/null +++ b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/source/FederationSourceManager.java @@ -0,0 +1,95 @@ +package com.easyagents.federation.sql.source; + +import java.util.Collection; +import java.util.Optional; + +/** + * 数据源 Definition、revision 与节点本地 Runtime 的管理入口。 + */ +public interface FederationSourceManager { + + /** + * 使用 Resolver 临时探测数据源,并在结束后关闭临时句柄。 + * + * @param definition 数据源定义 + * @return 探测结果 + */ + SourceProbeResult probe(FederationSourceDefinition definition); + + /** + * 使用调用方提供的临时句柄探测数据源,并在结束后关闭句柄。 + * + * @param definition 数据源定义 + * @param temporaryHandle 临时句柄 + * @return 探测结果 + */ + SourceProbeResult probe(FederationSourceDefinition definition, FederationDataSourceHandle temporaryHandle); + + /** + * 以默认懒加载策略应用 Definition。 + * + * @param definition 数据源定义 + * @return 应用结果 + */ + default SourceApplyResult apply(FederationSourceDefinition definition) { + return apply(definition, SourceApplyOptions.lazy()); + } + + /** + * 应用 Definition 并选择懒加载或预热。 + * + * @param definition 数据源定义 + * @param options 应用策略 + * @return 应用结果 + */ + SourceApplyResult apply(FederationSourceDefinition definition, SourceApplyOptions options); + + /** + * 预构建一个不进入共享 Source Slot 的节点本地 Runtime。 + * + * @param definition 即将发布的数据源 Definition + * @return 由调用方暂时持有的预构建 Runtime + */ + PreparedSourceRuntime prepare(FederationSourceDefinition definition); + + /** + * 原子发布预构建 Runtime,并切换对应 Source Slot 的 desired state。 + * + * @param prepared 由当前 SourceManager 创建的预构建 Runtime + * @return Definition 应用结果 + */ + SourceApplyResult commit(PreparedSourceRuntime prepared); + + /** + * 应用共享状态快照。 + * + * @param states 状态集合 + * @return 应用统计 + */ + SourceSnapshotResult applySnapshot(Collection states); + + /** + * 应用递增版本墓碑并禁止新查询。 + * + * @param tombstone 删除墓碑 + * @return 删除结果 + */ + SourceRemoveResult remove(SourceTombstone tombstone); + + /** + * 返回节点本地数据源视图。 + * + * @param sourceId 数据源标识 + * @return 可选视图 + */ + Optional view(SourceId sourceId); + + /** + * 确保节点本地 Runtime 至少达到指定 revision。 + * + * @param sourceId 数据源标识 + * @param minimumRevision 最低版本 + * @return 就绪视图 + */ + FederationSourceView ensureReady(SourceId sourceId, long minimumRevision); +} diff --git a/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/source/FederationSourceState.java b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/source/FederationSourceState.java new file mode 100644 index 0000000..cda9ef6 --- /dev/null +++ b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/source/FederationSourceState.java @@ -0,0 +1,30 @@ +package com.easyagents.federation.sql.source; + +import java.io.Serializable; + +/** + * 可共享的数据源期望状态。 + */ +public interface FederationSourceState extends Serializable { + + /** + * 返回数据源标识。 + * + * @return 数据源标识 + */ + SourceId sourceId(); + + /** + * 返回状态版本。 + * + * @return 单调递增版本 + */ + long revision(); + + /** + * 返回状态校验和。 + * + * @return 校验和 + */ + String checksum(); +} diff --git a/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/source/FederationSourceStateProvider.java b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/source/FederationSourceStateProvider.java new file mode 100644 index 0000000..f31829b --- /dev/null +++ b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/source/FederationSourceStateProvider.java @@ -0,0 +1,48 @@ +package com.easyagents.federation.sql.source; + +import java.util.Collection; +import java.util.List; +import java.util.Optional; +import java.util.function.Consumer; + +/** + * 调用方可实现的共享状态读取与变更提示 SPI,例如 Redis 实现。 + */ +public interface FederationSourceStateProvider { + + /** + * 按需读取某个数据源的最新共享状态。 + * + * @param sourceId 数据源标识 + * @return 最新状态 + */ + Optional find(SourceId sourceId); + + /** + * 加载节点启动时的状态快照。 + * + * @return 状态快照 + */ + default Collection loadSnapshot() { + return List.of(); + } + + /** + * 订阅状态变更提示;可靠性仍由按需读取与快照恢复保证。 + * + * @param consumer 状态消费者 + * @return 可关闭订阅 + */ + default SourceStateSubscription subscribe(Consumer consumer) { + return () -> { }; + } + + /** + * 返回不访问共享存储的空实现。 + * + * @return 空状态 Provider + */ + static FederationSourceStateProvider none() { + return sourceId -> Optional.empty(); + } +} diff --git a/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/source/FederationSourceView.java b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/source/FederationSourceView.java new file mode 100644 index 0000000..2a36122 --- /dev/null +++ b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/source/FederationSourceView.java @@ -0,0 +1,21 @@ +package com.easyagents.federation.sql.source; + +/** + * 不暴露 DataSource 或 Calcite 对象的数据源运行视图。 + * + * @param sourceId 数据源标识 + * @param desiredRevision 期望版本 + * @param readyRevision 本地就绪版本,未就绪时为 -1 + * @param status 运行状态 + * @param checksum 当前期望状态校验和 + * @param diagnostic 不含敏感信息的诊断说明 + */ +public record FederationSourceView( + SourceId sourceId, + long desiredRevision, + long readyRevision, + SourceRuntimeStatus status, + String checksum, + String diagnostic +) { +} diff --git a/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/source/KnownJdbcDriver.java b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/source/KnownJdbcDriver.java new file mode 100644 index 0000000..e1b19c9 --- /dev/null +++ b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/source/KnownJdbcDriver.java @@ -0,0 +1,118 @@ +package com.easyagents.federation.sql.source; + +import java.util.Optional; + +/** + * 常见数据库 JDBC 驱动元数据目录。 + * + *

该枚举仅提供驱动类名、JDBC URL 前缀和可选 Maven 坐标,不引入厂商驱动, + * 也不表示对应数据库已经通过 Federation Adapter 兼容性验证。

+ */ +public enum KnownJdbcDriver { + + /** MySQL Connector/J。 */ + MYSQL( + "com.mysql.cj.jdbc.Driver", + "jdbc:mysql:", + "com.mysql:mysql-connector-j" + ), + /** PostgreSQL pgJDBC。 */ + POSTGRESQL( + "org.postgresql.Driver", + "jdbc:postgresql:", + "org.postgresql:postgresql" + ), + /** Oracle JDBC Thin Driver。 */ + ORACLE( + "oracle.jdbc.OracleDriver", + "jdbc:oracle:thin:", + "com.oracle.database.jdbc:ojdbc11" + ), + /** Microsoft JDBC Driver for SQL Server。 */ + SQL_SERVER( + "com.microsoft.sqlserver.jdbc.SQLServerDriver", + "jdbc:sqlserver:", + "com.microsoft.sqlserver:mssql-jdbc" + ), + /** 华为 GaussDB 原生 JDBC 驱动。 */ + GAUSSDB( + "com.huawei.gaussdb.jdbc.Driver", + "jdbc:gaussdb:", + null + ), + /** 达梦 DM8 JDBC 驱动。 */ + DM8( + "dm.jdbc.driver.DmDriver", + "jdbc:dm:", + "com.dameng:DmJdbcDriver8" + ), + /** 南大通用 GBase 8a JDBC 驱动。 */ + GBASE_8A( + "com.gbase.jdbc.Driver", + "jdbc:gbase:", + null + ), + /** 南大通用 GBase 8s JDBC 驱动。 */ + GBASE_8S( + "com.gbasedbt.jdbc.Driver", + "jdbc:gbasedbt-sqli:", + null + ), + /** OceanBase Connector/J,可识别 MySQL 与 Oracle 两种租户模式。 */ + OCEANBASE( + "com.oceanbase.jdbc.Driver", + "jdbc:oceanbase:", + "com.oceanbase:oceanbase-client" + ); + + /** JDBC 驱动实现类的全限定名称。 */ + private final String driverClassName; + /** 驱动接受的 JDBC URL 前缀。 */ + private final String jdbcUrlPrefix; + /** 不包含版本号的 Maven groupId:artifactId,可为空。 */ + private final String mavenCoordinate; + + /** + * 创建 JDBC 驱动元数据项。 + * + * @param driverClassName JDBC 驱动实现类的全限定名称 + * @param jdbcUrlPrefix JDBC URL 前缀 + * @param mavenCoordinate 不包含版本号的 Maven 坐标,不公开时传入 {@code null} + */ + KnownJdbcDriver( + String driverClassName, + String jdbcUrlPrefix, + String mavenCoordinate + ) { + this.driverClassName = driverClassName; + this.jdbcUrlPrefix = jdbcUrlPrefix; + this.mavenCoordinate = mavenCoordinate; + } + + /** + * 返回 JDBC 驱动实现类的全限定名称。 + * + * @return 驱动类名 + */ + public String driverClassName() { + return driverClassName; + } + + /** + * 返回该驱动接受的 JDBC URL 前缀。 + * + * @return JDBC URL 前缀 + */ + public String jdbcUrlPrefix() { + return jdbcUrlPrefix; + } + + /** + * 返回不包含版本号的 Maven 坐标。 + * + * @return 公开 Maven 坐标;厂商驱动未提供公开坐标时为空 + */ + public Optional mavenCoordinate() { + return Optional.ofNullable(mavenCoordinate); + } +} diff --git a/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/source/PreparedSourceRuntime.java b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/source/PreparedSourceRuntime.java new file mode 100644 index 0000000..e0eba9a --- /dev/null +++ b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/source/PreparedSourceRuntime.java @@ -0,0 +1,23 @@ +package com.easyagents.federation.sql.source; + +/** + * 尚未发布到节点本地 Source Slot 的预构建 Runtime。 + * + *

调用方应使用 try-with-resources 持有该对象。发布成功后由 SourceManager 接管资源; + * 放弃发布或发生异常时,{@link #close()} 会关闭预构建的 DataSource Handle。

+ */ +public interface PreparedSourceRuntime extends AutoCloseable { + + /** + * 返回该 Runtime 对应的不可变 Definition。 + * + * @return 数据源 Definition + */ + FederationSourceDefinition definition(); + + /** + * 放弃尚未提交的 Runtime,并确定性关闭其资源。 + */ + @Override + void close(); +} diff --git a/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/source/RuntimeFingerprint.java b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/source/RuntimeFingerprint.java new file mode 100644 index 0000000..f23c935 --- /dev/null +++ b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/source/RuntimeFingerprint.java @@ -0,0 +1,45 @@ +package com.easyagents.federation.sql.source; + +import java.io.Serializable; + +/** + * 不含敏感信息的数据库与驱动运行指纹。 + * + * @param databaseProduct 数据库产品名 + * @param databaseVersion 数据库版本 + * @param driverName 驱动名 + * @param driverVersion 驱动版本 + * @param adapterVersion Adapter 版本 + */ +public record RuntimeFingerprint( + String databaseProduct, + String databaseVersion, + String driverName, + String driverVersion, + String adapterVersion +) implements Serializable { + + /** + * 防止指纹字段为空。 + */ + public RuntimeFingerprint { + databaseProduct = safe(databaseProduct); + databaseVersion = safe(databaseVersion); + driverName = safe(driverName); + driverVersion = safe(driverVersion); + adapterVersion = safe(adapterVersion); + } + + private static String safe(String value) { + return value == null ? "unknown" : value; + } + + /** + * 返回适合缓存键的稳定摘要。 + * + * @return 指纹摘要 + */ + public String cacheKey() { + return String.join("|", databaseProduct, databaseVersion, driverName, driverVersion, adapterVersion); + } +} diff --git a/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/source/SourceApplyOptions.java b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/source/SourceApplyOptions.java new file mode 100644 index 0000000..57f8279 --- /dev/null +++ b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/source/SourceApplyOptions.java @@ -0,0 +1,27 @@ +package com.easyagents.federation.sql.source; + +/** + * 数据源定义应用策略。 + * + * @param prewarm 应用后是否立即初始化本地 Runtime + */ +public record SourceApplyOptions(boolean prewarm) { + + /** + * 返回默认懒加载策略。 + * + * @return 懒加载选项 + */ + public static SourceApplyOptions lazy() { + return new SourceApplyOptions(false); + } + + /** + * 返回立即预热策略。 + * + * @return 预热选项 + */ + public static SourceApplyOptions prewarmNow() { + return new SourceApplyOptions(true); + } +} diff --git a/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/source/SourceApplyResult.java b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/source/SourceApplyResult.java new file mode 100644 index 0000000..6b8ae99 --- /dev/null +++ b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/source/SourceApplyResult.java @@ -0,0 +1,17 @@ +package com.easyagents.federation.sql.source; + +/** + * 数据源状态应用结果。 + * + * @param sourceId 数据源标识 + * @param requestedRevision 请求版本 + * @param effectiveRevision 当前生效版本 + * @param status 应用状态 + */ +public record SourceApplyResult( + SourceId sourceId, + long requestedRevision, + long effectiveRevision, + SourceApplyStatus status +) { +} diff --git a/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/source/SourceApplyStatus.java b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/source/SourceApplyStatus.java new file mode 100644 index 0000000..641b09b --- /dev/null +++ b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/source/SourceApplyStatus.java @@ -0,0 +1,15 @@ +package com.easyagents.federation.sql.source; + +/** + * Definition 或墓碑应用结果。 + */ +public enum SourceApplyStatus { + /** 首次登记或更高 revision 已应用。 */ + APPLIED, + /** 同 revision、同校验和的幂等重放。 */ + IDEMPOTENT, + /** 低 revision 状态已忽略。 */ + IGNORED_STALE, + /** 同 revision、不同校验和冲突。 */ + CONFLICT +} diff --git a/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/source/SourceId.java b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/source/SourceId.java new file mode 100644 index 0000000..eed38c4 --- /dev/null +++ b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/source/SourceId.java @@ -0,0 +1,30 @@ +package com.easyagents.federation.sql.source; + +import java.io.Serializable; + +/** + * 节点间可传递的逻辑数据源标识。 + * + * @param value 非空逻辑标识 + */ +public record SourceId(String value) implements Serializable { + + /** + * 校验并创建数据源标识。 + */ + public SourceId { + if (value == null || value.isBlank()) { + throw new IllegalArgumentException("source id must not be blank"); + } + } + + /** + * 返回标识文本。 + * + * @return 标识文本 + */ + @Override + public String toString() { + return value; + } +} diff --git a/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/source/SourceProbeResult.java b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/source/SourceProbeResult.java new file mode 100644 index 0000000..6e94238 --- /dev/null +++ b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/source/SourceProbeResult.java @@ -0,0 +1,17 @@ +package com.easyagents.federation.sql.source; + +/** + * 数据源连接与 Adapter 探测结果。 + * + * @param sourceId 数据源标识 + * @param supported 是否受支持 + * @param fingerprint 数据库与驱动指纹 + * @param diagnostic 不含敏感信息的诊断说明 + */ +public record SourceProbeResult( + SourceId sourceId, + boolean supported, + RuntimeFingerprint fingerprint, + String diagnostic +) { +} diff --git a/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/source/SourceRemoveResult.java b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/source/SourceRemoveResult.java new file mode 100644 index 0000000..1cb0b26 --- /dev/null +++ b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/source/SourceRemoveResult.java @@ -0,0 +1,9 @@ +package com.easyagents.federation.sql.source; + +/** + * 数据源删除结果。 + * + * @param result 状态应用结果 + */ +public record SourceRemoveResult(SourceApplyResult result) { +} diff --git a/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/source/SourceRuntimeStatus.java b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/source/SourceRuntimeStatus.java new file mode 100644 index 0000000..1fb3777 --- /dev/null +++ b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/source/SourceRuntimeStatus.java @@ -0,0 +1,15 @@ +package com.easyagents.federation.sql.source; + +/** + * 节点本地数据源运行状态。 + */ +public enum SourceRuntimeStatus { + /** Definition 已登记,Runtime 尚未初始化。 */ + DEFINED, + /** 节点本地 Runtime 已就绪。 */ + READY, + /** 数据源已被墓碑删除。 */ + REMOVED, + /** 最近一次初始化失败且无就绪 Runtime。 */ + FAILED +} diff --git a/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/source/SourceSnapshotResult.java b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/source/SourceSnapshotResult.java new file mode 100644 index 0000000..6d5c7a9 --- /dev/null +++ b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/source/SourceSnapshotResult.java @@ -0,0 +1,12 @@ +package com.easyagents.federation.sql.source; + +/** + * 共享状态快照应用统计。 + * + * @param applied 已应用数量 + * @param idempotent 幂等数量 + * @param stale 已忽略旧版本数量 + * @param conflicts 冲突数量 + */ +public record SourceSnapshotResult(int applied, int idempotent, int stale, int conflicts) { +} diff --git a/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/source/SourceStateSubscription.java b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/source/SourceStateSubscription.java new file mode 100644 index 0000000..99f2080 --- /dev/null +++ b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/source/SourceStateSubscription.java @@ -0,0 +1,14 @@ +package com.easyagents.federation.sql.source; + +/** + * 共享状态变更提示订阅。 + */ +@FunctionalInterface +public interface SourceStateSubscription extends AutoCloseable { + + /** + * 关闭订阅并释放资源。 + */ + @Override + void close(); +} diff --git a/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/source/SourceTombstone.java b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/source/SourceTombstone.java new file mode 100644 index 0000000..df113e3 --- /dev/null +++ b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/main/java/com/easyagents/federation/sql/source/SourceTombstone.java @@ -0,0 +1,70 @@ +package com.easyagents.federation.sql.source; + +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.HexFormat; + +/** + * 防止延迟 Definition 复活的数据源删除墓碑。 + * + * @param sourceId 数据源标识 + * @param revision 删除版本 + * @param checksum 墓碑校验和 + */ +public record SourceTombstone( + SourceId sourceId, + long revision, + String checksum +) implements FederationSourceState { + + /** + * 校验墓碑状态。 + */ + public SourceTombstone { + if (sourceId == null) { + throw new IllegalArgumentException("sourceId must not be null"); + } + if (revision < 0) { + throw new IllegalArgumentException("revision must not be negative"); + } + if (checksum == null || checksum.isBlank()) { + throw new IllegalArgumentException("checksum must not be blank"); + } + if (!checksum.equals(expectedChecksum(sourceId, revision))) { + throw new IllegalArgumentException("checksum does not match source tombstone"); + } + } + + /** + * 创建具有稳定校验和的墓碑。 + * + * @param sourceId 数据源标识 + * @param revision 删除版本 + * @return 墓碑 + */ + public static SourceTombstone of(SourceId sourceId, long revision) { + if (sourceId == null) { + throw new IllegalArgumentException("sourceId must not be null"); + } + if (revision < 0) { + throw new IllegalArgumentException("revision must not be negative"); + } + return new SourceTombstone(sourceId, revision, expectedChecksum(sourceId, revision)); + } + + private static String expectedChecksum(SourceId sourceId, long revision) { + try { + MessageDigest digest = MessageDigest.getInstance("SHA-256"); + byte[] sourceBytes = sourceId.value().getBytes(StandardCharsets.UTF_8); + digest.update("federation-source-tombstone-v1".getBytes(StandardCharsets.UTF_8)); + digest.update(ByteBuffer.allocate(Integer.BYTES).putInt(sourceBytes.length).array()); + digest.update(sourceBytes); + digest.update(ByteBuffer.allocate(Long.BYTES).putLong(revision).array()); + return HexFormat.of().formatHex(digest.digest()); + } catch (NoSuchAlgorithmException exception) { + throw new IllegalStateException("SHA-256 is not available", exception); + } + } +} diff --git a/easy-agents-federation-sql/easy-agents-federation-sql-core/src/test/java/com/easyagents/federation/sql/api/SqlQueryCommandSerializationTest.java b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/test/java/com/easyagents/federation/sql/api/SqlQueryCommandSerializationTest.java new file mode 100644 index 0000000..a953352 --- /dev/null +++ b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/test/java/com/easyagents/federation/sql/api/SqlQueryCommandSerializationTest.java @@ -0,0 +1,328 @@ +package com.easyagents.federation.sql.api; + +import com.easyagents.federation.sql.adapter.AdapterCompatibility; +import com.easyagents.federation.sql.adapter.AdapterCompatibilityStatus; +import com.easyagents.federation.sql.compile.FederationFragmentExplain; +import com.easyagents.federation.sql.compile.SqlCompileRequest; +import com.easyagents.federation.sql.compile.SqlExplainLevel; +import com.easyagents.federation.sql.compile.SqlExplainRequest; +import com.easyagents.federation.sql.compile.SqlExplainResult; +import com.easyagents.federation.sql.execute.FederationColumn; +import com.easyagents.federation.sql.execute.FederationFragmentMetrics; +import com.easyagents.federation.sql.execute.FederationPhysicalExplain; +import com.easyagents.federation.sql.execute.FederationQueryMetricsSnapshot; +import com.easyagents.federation.sql.execute.QueryId; +import com.easyagents.federation.sql.execute.SqlParameter; +import com.easyagents.federation.sql.federation.FederationExecutionPolicy; +import com.easyagents.federation.sql.federation.FederationQueryMode; +import com.easyagents.federation.sql.federation.FederationQueryScopeDefinition; +import com.easyagents.federation.sql.federation.FederationSourceBindingDefinition; +import com.easyagents.federation.sql.source.SourceId; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.ObjectInputStream; +import java.io.ObjectOutputStream; +import java.io.Serializable; +import java.sql.Types; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import org.junit.Assert; +import org.junit.Test; + +/** + * 跨节点查询命令序列化边界测试。 + */ +public class SqlQueryCommandSerializationTest { + + /** + * 验证命令可以序列化并在目标节点恢复。 + * + * @throws Exception 序列化失败 + */ + @Test + public void shouldRoundTripSerializableCommand() throws Exception { + SqlQueryCommand command = SqlQueryCommand.of( + "SELECT name FROM person WHERE id = ?", + new SourceId("main"), + 3, + List.of(SqlParameter.of(7)) + ); + ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + try (ObjectOutputStream output = new ObjectOutputStream(bytes)) { + output.writeObject(command); + } + SqlQueryCommand restored; + try (ObjectInputStream input = new ObjectInputStream(new ByteArrayInputStream(bytes.toByteArray()))) { + restored = (SqlQueryCommand) input.readObject(); + } + Assert.assertEquals(command, restored); + } + + /** + * 验证非序列化参数在进入跨节点命令前被拒绝。 + */ + @Test(expected = IllegalArgumentException.class) + public void shouldRejectNonSerializableParameterValue() { + SqlParameter.of(new Object()); + } + + /** + * 验证仅带 Serializable 标记的任意对象不能进入跨节点参数边界。 + */ + @Test(expected = IllegalArgumentException.class) + public void shouldRejectUnsupportedSerializableObjectGraph() { + SqlParameter.of(new BrokenSerializableValue()); + } + + /** + * 验证常见浮点、时间和时区时间值映射为精确 JDBC 类型。 + */ + @Test + public void shouldInferPreciseJdbcScalarTypes() { + Assert.assertEquals(Types.REAL, SqlParameter.of(1.5F).jdbcType()); + Assert.assertEquals(Types.DOUBLE, SqlParameter.of(1.5D).jdbcType()); + Assert.assertEquals( + Types.TIME, + SqlParameter.of(java.time.LocalTime.NOON).jdbcType() + ); + Assert.assertEquals( + Types.TIME_WITH_TIMEZONE, + SqlParameter.of(java.time.OffsetTime.parse("12:00:00+08:00")).jdbcType() + ); + Assert.assertEquals( + Types.TIMESTAMP_WITH_TIMEZONE, + SqlParameter.of(java.time.OffsetDateTime.parse( + "2026-08-21T12:00:00+08:00" + )).jdbcType() + ); + Assert.assertEquals( + Types.OTHER, + SqlParameter.of(java.util.UUID.randomUUID()).jdbcType() + ); + } + + /** + * 验证虚拟查询范围可跨节点序列化,且 Binding 顺序不影响稳定摘要。 + * + * @throws Exception 序列化失败 + */ + @Test + public void shouldSerializeVirtualScopeWithStableChecksum() throws Exception { + SourceId first = new SourceId("first"); + SourceId second = new SourceId("second"); + FederationSourceBindingDefinition firstBinding = + FederationSourceBindingDefinition.of(first, 2, Map.of("APP", "MAIN")); + FederationSourceBindingDefinition secondBinding = + FederationSourceBindingDefinition.of(second, 4); + FederationQueryScopeDefinition scope = FederationQueryScopeDefinition.virtual( + "virtual", + 9, + Map.of("A", firstBinding, "B", secondBinding), + "A", + FederationExecutionPolicy.basic() + ); + FederationQueryScopeDefinition reordered = FederationQueryScopeDefinition.virtual( + "virtual", + 9, + new java.util.LinkedHashMap<>(Map.of("B", secondBinding, "A", firstBinding)), + "A", + FederationExecutionPolicy.basic() + ); + Assert.assertEquals(scope.checksum(), reordered.checksum()); + + SqlQueryCommand command = SqlQueryCommand.of( + "SELECT * FROM A.APP.T JOIN B.APP.T USING (ID)", + scope, + List.of() + ); + ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + try (ObjectOutputStream output = new ObjectOutputStream(bytes)) { + output.writeObject(command); + } + try (ObjectInputStream input = new ObjectInputStream( + new ByteArrayInputStream(bytes.toByteArray()))) { + SqlQueryCommand restored = (SqlQueryCommand) input.readObject(); + Assert.assertEquals(scope, restored.queryScope()); + Assert.assertEquals(scope.checksum(), restored.queryScope().checksum()); + } + } + + /** + * 验证 Binding 与 Schema Mapping 的层级边界会进入稳定摘要。 + */ + @Test + public void shouldSeparateBindingAndSchemaMappingChecksumStructure() { + FederationQueryScopeDefinition mappingScope = FederationQueryScopeDefinition.virtual( + "collision-regression", + 1, + Map.of("0", FederationSourceBindingDefinition.of( + new SourceId("s1"), + 1, + Map.of("1", "s2", "2", "b3", "3", "4") + )), + "0", + FederationExecutionPolicy.basic() + ); + FederationQueryScopeDefinition bindingScope = FederationQueryScopeDefinition.virtual( + "collision-regression", + 1, + Map.of( + "0", FederationSourceBindingDefinition.of(new SourceId("s1"), 1), + "1", FederationSourceBindingDefinition.of(new SourceId("s2"), 2), + "b3", FederationSourceBindingDefinition.of(new SourceId("3"), 4) + ), + "0", + FederationExecutionPolicy.basic() + ); + + Assert.assertNotEquals(mappingScope.checksum(), bindingScope.checksum()); + } + + /** + * 验证未加引号标识符的大小写歧义在定义阶段直接拒绝。 + */ + @Test + public void shouldRejectCaseInsensitiveScopeNameAmbiguity() { + LinkedHashMap bindings = new LinkedHashMap<>(); + bindings.put("Sales", FederationSourceBindingDefinition.of(new SourceId("first"), 1)); + bindings.put("SALES", FederationSourceBindingDefinition.of(new SourceId("second"), 1)); + try { + FederationQueryScopeDefinition.virtual( + "ambiguous-bindings", + 1, + bindings, + "Sales", + FederationExecutionPolicy.basic() + ); + Assert.fail("case-insensitive duplicate bindings should be rejected"); + } catch (IllegalArgumentException expected) { + Assert.assertTrue(expected.getMessage().contains("binding names")); + } + + LinkedHashMap mappings = new LinkedHashMap<>(); + mappings.put("App", "MAIN"); + mappings.put("APP", "OTHER"); + try { + FederationSourceBindingDefinition.of(new SourceId("first"), 1, mappings); + Assert.fail("case-insensitive duplicate schema mappings should be rejected"); + } catch (IllegalArgumentException expected) { + Assert.assertTrue(expected.getMessage().contains("query schema names")); + } + } + + /** + * 验证 Explain 与指标纯数据视图可完整跨节点序列化。 + * + * @throws Exception 序列化失败 + */ + @Test + public void shouldSerializeExplainAndMetricsValueGraphs() throws Exception { + SourceId sourceId = new SourceId("main"); + FederationColumn column = new FederationColumn(1, "ID", Types.INTEGER, "INTEGER", false); + FederationPhysicalExplain physical = new FederationPhysicalExplain( + true, + "index scan", + "Index Scan", + "index", + List.of("IDX_ID"), + "IDX_ID", + 1L, + null, + "safe" + ); + FederationFragmentExplain fragment = new FederationFragmentExplain( + "fragment-1", + "main", + sourceId, + "jdbc", + "SELECT ID FROM T", + List.of(), + List.of(column), + physical + ); + AdapterCompatibility compatibility = new AdapterCompatibility( + AdapterCompatibilityStatus.CODE_SUPPORTED_UNVERIFIED, + "H2", + "2", + "H2 JDBC Driver", + "2", + "supported" + ); + SqlExplainResult explain = new SqlExplainResult( + SqlExplainLevel.PHYSICAL, + FederationQueryMode.SINGLE_SOURCE, + "SELECT ID FROM T", + "SELECT ID FROM T", + "LogicalProject", + "JdbcProject", + List.of(fragment), + Set.of(sourceId), + compatibility, + true, + false, + "executable" + ); + Assert.assertEquals(explain, roundTrip(explain)); + + FederationQueryMetricsSnapshot metrics = new FederationQueryMetricsSnapshot( + QueryId.create(), + FederationQueryMode.FEDERATED, + true, + 10, + 20, + 5, + 1, + 8, + 2, + 16, + true, + false, + List.of(new FederationFragmentMetrics("fragment-1", sourceId, 2, 16, 12, true)) + ); + Assert.assertEquals(metrics, roundTrip(metrics)); + } + + /** + * 验证物理 Explain 拒绝实际参数值,避免原生计划泄露敏感常量。 + */ + @Test + public void shouldRejectActualExplainParameterValues() { + SqlCompileRequest compileRequest = new SqlCompileRequest( + "SELECT NAME FROM PERSON WHERE ID = ?", + new SourceId("main"), + 1, + List.of(Types.INTEGER), + "policy-v1" + ); + try { + new SqlExplainRequest( + compileRequest, + SqlExplainLevel.PHYSICAL, + List.of(new SqlParameter(Types.INTEGER, 7)) + ); + Assert.fail("Explain parameter values should be rejected"); + } catch (IllegalArgumentException expected) { + Assert.assertTrue(expected.getMessage().contains("does not accept parameter values")); + } + } + + private static T roundTrip(T value) throws Exception { + ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + try (ObjectOutputStream output = new ObjectOutputStream(bytes)) { + output.writeObject(value); + } + try (ObjectInputStream input = new ObjectInputStream( + new ByteArrayInputStream(bytes.toByteArray()))) { + @SuppressWarnings("unchecked") + T restored = (T) input.readObject(); + return restored; + } + } + + private static final class BrokenSerializableValue implements Serializable { + + private final Object nested = new Object(); + } +} diff --git a/easy-agents-federation-sql/easy-agents-federation-sql-core/src/test/java/com/easyagents/federation/sql/execute/LocalFederationQueryAdmissionControllerTest.java b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/test/java/com/easyagents/federation/sql/execute/LocalFederationQueryAdmissionControllerTest.java new file mode 100644 index 0000000..4c8f87f --- /dev/null +++ b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/test/java/com/easyagents/federation/sql/execute/LocalFederationQueryAdmissionControllerTest.java @@ -0,0 +1,61 @@ +package com.easyagents.federation.sql.execute; + +import com.easyagents.federation.sql.api.FederationSqlErrorCode; +import com.easyagents.federation.sql.api.FederationSqlException; +import com.easyagents.federation.sql.source.SourceId; +import java.time.Duration; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import org.junit.Assert; +import org.junit.Test; + +/** + * 本地查询准入控制器关闭与等待唤醒测试。 + */ +public class LocalFederationQueryAdmissionControllerTest { + + /** + * 验证关闭控制器会唤醒等待线程,并拒绝后续准入。 + * + * @throws Exception 并发测试失败 + */ + @Test + public void shouldWakeAndRejectWaitersAfterClose() throws Exception { + LocalFederationQueryAdmissionController controller = + new LocalFederationQueryAdmissionController(1); + FederationQueryPermit first = controller.acquire( + new SourceId("main"), + new QueryId("first"), + Duration.ZERO + ); + ExecutorService executor = Executors.newSingleThreadExecutor(); + try { + Future waiting = executor.submit(() -> { + try { + controller.acquire( + new SourceId("main"), + new QueryId("waiting"), + Duration.ofSeconds(5) + ); + return null; + } catch (FederationSqlException exception) { + return exception.errorCode(); + } + }); + controller.close(); + Assert.assertEquals(FederationSqlErrorCode.ENGINE_CLOSED, waiting.get(2, TimeUnit.SECONDS)); + try { + controller.acquire(new SourceId("main"), new QueryId("late"), Duration.ZERO); + Assert.fail("closed controller should reject admission"); + } catch (FederationSqlException exception) { + Assert.assertEquals(FederationSqlErrorCode.ENGINE_CLOSED, exception.errorCode()); + } + } finally { + first.close(); + controller.close(); + executor.shutdownNow(); + } + } +} diff --git a/easy-agents-federation-sql/easy-agents-federation-sql-core/src/test/java/com/easyagents/federation/sql/federation/FederationStatisticsSnapshotTest.java b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/test/java/com/easyagents/federation/sql/federation/FederationStatisticsSnapshotTest.java new file mode 100644 index 0000000..44dcb6b --- /dev/null +++ b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/test/java/com/easyagents/federation/sql/federation/FederationStatisticsSnapshotTest.java @@ -0,0 +1,158 @@ +package com.easyagents.federation.sql.federation; + +import com.easyagents.federation.sql.source.SourceId; +import java.time.Instant; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.Set; +import org.junit.Assert; +import org.junit.Test; + +/** + * 统计快照冻结语义测试。 + */ +public class FederationStatisticsSnapshotTest { + + /** + * 验证快照复制输入映射并采用最早统计有效期。 + */ + @Test + public void shouldFreezeStatisticsAndExposeEarliestExpiry() { + SourceId sourceId = new SourceId("source"); + Instant firstExpiry = Instant.parse("2026-08-23T10:00:00Z"); + Map mutable = + new LinkedHashMap<>(); + mutable.put( + new FederationStatisticsSnapshot.TableKey(sourceId, "PUBLIC", "ORDERS"), + statistics(firstExpiry) + ); + FederationStatisticsSnapshot snapshot = new FederationStatisticsSnapshot( + "snapshot-1", + Instant.parse("2026-08-23T09:00:00Z"), + mutable + ); + mutable.clear(); + + Assert.assertNotNull(snapshot.statistics(sourceId, "public", "orders")); + Assert.assertEquals(firstExpiry, snapshot.validUntil()); + } + + /** + * 验证查询级统计指纹不会被未引用表的刷新影响。 + */ + @Test + public void shouldFingerprintOnlySelectedTables() { + SourceId sourceId = new SourceId("source"); + Instant capturedAt = Instant.parse("2026-08-23T09:00:00Z"); + Instant selectedExpiry = Instant.parse("2026-08-23T10:00:00Z"); + FederationStatisticsSnapshot.TableKey selected = + new FederationStatisticsSnapshot.TableKey(sourceId, "public", "orders"); + FederationStatisticsSnapshot.TableKey unrelated = + new FederationStatisticsSnapshot.TableKey(sourceId, "public", "customers"); + FederationStatisticsSnapshot first = new FederationStatisticsSnapshot( + "global-v1", + capturedAt, + Map.of( + selected, statistics(selectedExpiry), + unrelated, statistics(capturedAt.plusSeconds(30)) + ) + ); + FederationStatisticsSnapshot second = new FederationStatisticsSnapshot( + "global-v2", + capturedAt, + Map.of( + selected, statistics(selectedExpiry), + unrelated, statistics(capturedAt.plusSeconds(300)) + ) + ); + + FederationStatisticsSnapshot.Selection firstSelection = first.select(Set.of(selected)); + FederationStatisticsSnapshot.Selection secondSelection = second.select(Set.of(selected)); + + Assert.assertEquals(firstSelection.fingerprint(), secondSelection.fingerprint()); + Assert.assertEquals(selectedExpiry, firstSelection.validUntil()); + Assert.assertEquals(selectedExpiry, secondSelection.validUntil()); + } + + /** + * 验证被查询表的统计变化会生成新的查询级指纹。 + */ + @Test + public void shouldChangeFingerprintWhenSelectedTableStatisticsChange() { + SourceId sourceId = new SourceId("source"); + Instant capturedAt = Instant.parse("2026-08-23T09:00:00Z"); + FederationStatisticsSnapshot.TableKey selected = + new FederationStatisticsSnapshot.TableKey(sourceId, "public", "orders"); + FederationStatisticsSnapshot first = new FederationStatisticsSnapshot( + "global-v1", + capturedAt, + Map.of(selected, statistics(capturedAt.plusSeconds(60))) + ); + FederationStatisticsSnapshot second = new FederationStatisticsSnapshot( + "global-v2", + capturedAt, + Map.of(selected, new FederationTableStatistics( + 200, + 64, + Instant.parse("2026-08-23T08:00:00Z"), + "catalog", + Map.of(), + java.util.List.of(), + capturedAt.plusSeconds(60), + FederationStatisticsStatus.COMPLETE + )) + ); + + Assert.assertNotEquals( + first.select(Set.of(selected)).fingerprint(), + second.select(Set.of(selected)).fingerprint() + ); + } + + /** + * 验证统计在失效边界时立即按过期处理。 + */ + @Test + public void shouldTreatExactExpiryBoundaryAsStale() { + Instant boundary = Instant.parse("2026-08-23T10:00:00Z"); + + Assert.assertEquals( + FederationStatisticsStatus.STALE, + statistics(boundary).effectiveStatus(boundary) + ); + } + + /** + * 验证已过期统计仍参与稳定指纹,但不会让降级计划在生成时立即失效。 + */ + @Test + public void shouldKeepStaleFallbackSelectionCacheable() { + Instant capturedAt = Instant.parse("2026-08-23T10:00:00Z"); + SourceId sourceId = new SourceId("source"); + FederationStatisticsSnapshot.TableKey table = + new FederationStatisticsSnapshot.TableKey(sourceId, "public", "orders"); + FederationStatisticsSnapshot snapshot = new FederationStatisticsSnapshot( + "stale-snapshot", + capturedAt, + Map.of(table, statistics(capturedAt.minusSeconds(1))) + ); + + FederationStatisticsSnapshot.Selection selection = snapshot.select(Set.of(table)); + + Assert.assertEquals(Instant.MAX, selection.validUntil()); + Assert.assertFalse(selection.fingerprint().isBlank()); + } + + private static FederationTableStatistics statistics(Instant expiresAt) { + return new FederationTableStatistics( + 100, + 64, + Instant.parse("2026-08-23T08:00:00Z"), + "catalog", + Map.of(), + java.util.List.of(), + expiresAt, + FederationStatisticsStatus.COMPLETE + ); + } +} diff --git a/easy-agents-federation-sql/easy-agents-federation-sql-core/src/test/java/com/easyagents/federation/sql/runtime/AdapterFederationStatisticsProviderTest.java b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/test/java/com/easyagents/federation/sql/runtime/AdapterFederationStatisticsProviderTest.java new file mode 100644 index 0000000..e862d76 --- /dev/null +++ b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/test/java/com/easyagents/federation/sql/runtime/AdapterFederationStatisticsProviderTest.java @@ -0,0 +1,369 @@ +package com.easyagents.federation.sql.runtime; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; + +import com.easyagents.federation.sql.adapter.AdapterCompatibility; +import com.easyagents.federation.sql.adapter.AdapterDialectContext; +import com.easyagents.federation.sql.adapter.AdapterHints; +import com.easyagents.federation.sql.adapter.AdapterSchemaContext; +import com.easyagents.federation.sql.adapter.FederationSqlAdapterProvider; +import com.easyagents.federation.sql.adapter.FederationStatisticsCollector; +import com.easyagents.federation.sql.execute.FederationFragmentExecutor; +import com.easyagents.federation.sql.federation.FederationQueryScopeDefinition; +import com.easyagents.federation.sql.federation.FederationStatisticsSnapshot; +import com.easyagents.federation.sql.federation.FederationStatisticsStatus; +import com.easyagents.federation.sql.federation.FederationTableStatistics; +import com.easyagents.federation.sql.source.ExternalSchemaDefinition; +import com.easyagents.federation.sql.source.FederationDataSourceHandles; +import com.easyagents.federation.sql.source.FederationSourceDefinition; +import com.easyagents.federation.sql.source.RuntimeFingerprint; +import com.easyagents.federation.sql.source.SourceId; +import java.lang.reflect.Proxy; +import java.sql.Connection; +import java.sql.DatabaseMetaData; +import java.time.Duration; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import javax.sql.DataSource; +import org.apache.calcite.schema.Schema; +import org.apache.calcite.schema.impl.AbstractSchema; +import org.apache.calcite.sql.SqlDialect; +import org.junit.Test; + +/** + * Adapter 自动统计缓存的并发合并和失败降级测试。 + */ +public class AdapterFederationStatisticsProviderTest { + + /** + * 验证同一物理源 revision 的并发刷新只触发一次目录采集且不阻塞调用线程。 + * + * @throws Exception 并发等待超时 + */ + @Test + public void shouldMergeConcurrentRefreshForSameRevision() throws Exception { + AtomicInteger collections = new AtomicInteger(); + CountDownLatch started = new CountDownLatch(1); + CountDownLatch release = new CountDownLatch(1); + FederationStatisticsCollector collector = context -> { + collections.incrementAndGet(); + started.countDown(); + try { + if (!release.await(5, TimeUnit.SECONDS)) { + throw new IllegalStateException("statistics test collector timed out"); + } + } catch (InterruptedException exception) { + Thread.currentThread().interrupt(); + throw new IllegalStateException("statistics test collector interrupted", exception); + } + FederationStatisticsSnapshot.TableKey key = + new FederationStatisticsSnapshot.TableKey( + context.sourceDefinition().sourceId(), + "main", + "orders" + ); + return Map.of(key, new FederationTableStatistics( + 42D, + 64L, + context.collectedAt(), + "test", + Map.of(), + List.of(), + context.expiresAt(), + FederationStatisticsStatus.PARTIAL + )); + }; + AdapterFederationStatisticsProvider provider = provider(); + SourceRuntime runtime = runtime(1L, collector); + FederationQueryScopeSnapshot snapshot = snapshot(runtime); + try { + CompletableFuture first = provider.refreshIfNeeded(snapshot); + if (!started.await(5, TimeUnit.SECONDS)) { + throw new AssertionError("statistics collection did not start"); + } + assertFalse("refresh should still be waiting on the collector", first.isDone()); + CompletableFuture second = provider.refreshIfNeeded(snapshot); + release.countDown(); + CompletableFuture.allOf(first, second).get(5, TimeUnit.SECONDS); + + assertEquals(1, collections.get()); + assertNotNull(provider.snapshot().statistics( + runtime.definition().sourceId(), + "main", + "orders" + )); + } finally { + release.countDown(); + snapshot.close(); + provider.close(); + } + } + + /** + * 验证目录读取失败不阻断查询,并在失败限频窗口内避免重复访问数据库。 + */ + @Test + public void shouldThrottleCollectionFailureWithoutFailingQuery() throws Exception { + AtomicInteger collections = new AtomicInteger(); + FederationStatisticsCollector collector = context -> { + collections.incrementAndGet(); + throw new java.sql.SQLException("catalog unavailable"); + }; + AdapterFederationStatisticsProvider provider = provider(); + SourceRuntime runtime = runtime(1L, collector); + FederationQueryScopeSnapshot snapshot = snapshot(runtime); + try { + provider.refreshIfNeeded(snapshot).get(5, TimeUnit.SECONDS); + provider.refreshIfNeeded(snapshot).get(5, TimeUnit.SECONDS); + + assertEquals(1, collections.get()); + assertEquals(null, provider.snapshot().statistics( + runtime.definition().sourceId(), + "main", + "orders" + )); + } finally { + snapshot.close(); + provider.close(); + } + } + + /** + * 验证异步采集持有独立 Runtime 租约,退休过程不会提前关闭正在使用的数据源。 + * + * @throws Exception 并发等待超时 + */ + @Test + public void shouldKeepRuntimeAliveDuringAsynchronousRefresh() throws Exception { + CountDownLatch started = new CountDownLatch(1); + CountDownLatch release = new CountDownLatch(1); + AtomicInteger closedRuntimes = new AtomicInteger(); + FederationStatisticsCollector collector = context -> { + started.countDown(); + try { + if (!release.await(5, TimeUnit.SECONDS)) { + throw new IllegalStateException("statistics test collector timed out"); + } + } catch (InterruptedException exception) { + Thread.currentThread().interrupt(); + throw new IllegalStateException("statistics test collector interrupted", exception); + } + return Map.of(); + }; + AdapterFederationStatisticsProvider provider = provider(); + SourceRuntime runtime = runtime(1L, collector, ignored -> + closedRuntimes.incrementAndGet() + ); + FederationQueryScopeSnapshot snapshot = snapshot(runtime); + try { + CompletableFuture refresh = provider.refreshIfNeeded(snapshot); + if (!started.await(5, TimeUnit.SECONDS)) { + throw new AssertionError("statistics collection did not start"); + } + + runtime.retire(); + assertEquals(0, closedRuntimes.get()); + + release.countDown(); + refresh.get(5, TimeUnit.SECONDS); + assertEquals(1, closedRuntimes.get()); + } finally { + release.countDown(); + snapshot.close(); + provider.close(); + } + } + + /** + * 创建短有效期的测试 Provider。 + * + * @return 自动统计 Provider + */ + private AdapterFederationStatisticsProvider provider() { + return new AdapterFederationStatisticsProvider( + Duration.ofMinutes(5), + Duration.ofMinutes(1), + 1, + 2 + ); + } + + /** + * 创建仅用于统计采集的最小 Source Runtime。 + * + * @param revision Definition revision + * @param collector 测试统计采集器 + * @return Source Runtime + */ + private SourceRuntime runtime(long revision, FederationStatisticsCollector collector) { + return runtime(revision, collector, ignored -> { }); + } + + /** + * 创建可观察关闭事件的最小 Source Runtime。 + * + * @param revision Definition revision + * @param collector 测试统计采集器 + * @param closedListener Runtime 关闭监听器 + * @return Source Runtime + */ + private SourceRuntime runtime( + long revision, + FederationStatisticsCollector collector, + java.util.function.Consumer closedListener + ) { + SourceId sourceId = new SourceId("source-a"); + FederationSourceDefinition definition = new FederationSourceDefinition( + sourceId, + revision, + "test-adapter", + List.of(new ExternalSchemaDefinition("main", "schema", 1)), + Map.of() + ); + return new SourceRuntime( + definition, + FederationDataSourceHandles.shared( + dataSource(), + new RuntimeFingerprint("test", "1", "test", "1", "1") + ), + new TestAdapter(collector), + SqlDialect.DatabaseProduct.CALCITE.getDialect(), + null, + null, + null, + closedListener + ); + } + + /** + * 创建借用 Runtime 的单源查询快照。 + * + * @param runtime Source Runtime + * @return 查询快照 + */ + private FederationQueryScopeSnapshot snapshot(SourceRuntime runtime) { + FederationQueryScopeDefinition scope = FederationQueryScopeDefinition.single( + runtime.definition().sourceId(), + runtime.definition().revision() + ); + return FederationQueryScopeSnapshot.borrowedSingle(scope, runtime); + } + + /** + * 创建支持只读标记和关闭的最小 DataSource。 + * + * @return 测试 DataSource + */ + private DataSource dataSource() { + Connection connection = (Connection) Proxy.newProxyInstance( + getClass().getClassLoader(), + new Class[] {Connection.class}, + (proxy, method, arguments) -> switch (method.getName()) { + case "isReadOnly" -> false; + case "setReadOnly", "close" -> null; + case "isClosed" -> false; + case "isWrapperFor" -> false; + case "unwrap" -> null; + default -> defaultValue(method.getReturnType()); + } + ); + return (DataSource) Proxy.newProxyInstance( + getClass().getClassLoader(), + new Class[] {DataSource.class}, + (proxy, method, arguments) -> switch (method.getName()) { + case "getConnection" -> connection; + case "isWrapperFor" -> false; + case "unwrap" -> null; + default -> defaultValue(method.getReturnType()); + } + ); + } + + /** + * 返回代理方法原始类型的默认值。 + * + * @param type 返回类型 + * @return 默认值 + */ + private Object defaultValue(Class type) { + if (!type.isPrimitive()) { + return null; + } + if (type == boolean.class) { + return false; + } + if (type == char.class) { + return '\0'; + } + return 0; + } + + /** + * 暴露测试统计采集器的最小 Adapter。 + */ + private static final class TestAdapter implements FederationSqlAdapterProvider { + + private final FederationStatisticsCollector collector; + + /** + * 创建测试 Adapter。 + * + * @param collector 统计采集器 + */ + private TestAdapter(FederationStatisticsCollector collector) { + this.collector = collector; + } + + /** {@inheritDoc} */ + @Override + public String adapterId() { + return "test-adapter"; + } + + /** {@inheritDoc} */ + @Override + public boolean supports(DatabaseMetaData metadata, AdapterHints hints) { + return true; + } + + /** {@inheritDoc} */ + @Override + public AdapterCompatibility compatibility( + DatabaseMetaData metadata, + AdapterHints hints + ) { + throw new UnsupportedOperationException(); + } + + /** {@inheritDoc} */ + @Override + public Schema createSchema(AdapterSchemaContext context) { + return new AbstractSchema(); + } + + /** {@inheritDoc} */ + @Override + public SqlDialect createDialect(AdapterDialectContext context) { + return SqlDialect.DatabaseProduct.CALCITE.getDialect(); + } + + /** {@inheritDoc} */ + @Override + public FederationFragmentExecutor fragmentExecutor() { + throw new UnsupportedOperationException(); + } + + /** {@inheritDoc} */ + @Override + public Optional statisticsCollector() { + return Optional.of(collector); + } + } +} diff --git a/easy-agents-federation-sql/easy-agents-federation-sql-core/src/test/java/com/easyagents/federation/sql/runtime/BoundedPlanCacheTest.java b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/test/java/com/easyagents/federation/sql/runtime/BoundedPlanCacheTest.java new file mode 100644 index 0000000..2b47b22 --- /dev/null +++ b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/test/java/com/easyagents/federation/sql/runtime/BoundedPlanCacheTest.java @@ -0,0 +1,741 @@ +package com.easyagents.federation.sql.runtime; + +import com.easyagents.federation.sql.adapter.AdapterCompatibility; +import com.easyagents.federation.sql.adapter.AdapterCompatibilityStatus; +import com.easyagents.federation.sql.api.FederationSqlErrorCode; +import com.easyagents.federation.sql.api.FederationSqlException; +import com.easyagents.federation.sql.compile.FederationSqlPlan; +import com.easyagents.federation.sql.source.SourceId; +import java.util.ArrayList; +import java.util.List; +import java.util.Set; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; +import java.time.Duration; +import java.time.Instant; +import org.junit.Assert; +import org.junit.Test; + +/** + * 有界计划缓存容量与 single-flight 测试。 + */ +public class BoundedPlanCacheTest { + + /** + * 验证唯一 SQL 不会让缓存无界增长。 + */ + @Test + public void shouldRemainBounded() { + BoundedPlanCache cache = new BoundedPlanCache(4); + for (int index = 0; index < 20; index++) { + PlanCacheKey key = key("select " + index); + cache.getOrCompile(key, BoundedPlanCacheTest::plan); + } + Assert.assertEquals(4, cache.size()); + cache.close(); + } + + /** + * 验证相同冷键的并发请求只执行一次编译。 + * + * @throws Exception 并发测试失败 + */ + @Test + public void shouldCompileSameColdKeyOnce() throws Exception { + BoundedPlanCache cache = new BoundedPlanCache(4); + PlanCacheKey key = key("select 1"); + AtomicInteger compilations = new AtomicInteger(); + int concurrency = 20; + ExecutorService executor = Executors.newFixedThreadPool(8); + CountDownLatch start = new CountDownLatch(1); + CountDownLatch done = new CountDownLatch(concurrency); + for (int index = 0; index < concurrency; index++) { + executor.execute(() -> { + try { + start.await(); + cache.getOrCompile(key, () -> { + compilations.incrementAndGet(); + return plan(); + }); + } catch (InterruptedException exception) { + Thread.currentThread().interrupt(); + } finally { + done.countDown(); + } + }); + } + start.countDown(); + Assert.assertTrue(done.await(5, TimeUnit.SECONDS)); + executor.shutdownNow(); + Assert.assertEquals(1, compilations.get()); + cache.close(); + } + + /** + * 验证关闭期间的冷编译不会在 close 返回后重新写入缓存。 + * + * @throws Exception 并发测试失败 + */ + @Test + public void shouldRejectCompilationCompletedAfterClose() throws Exception { + BoundedPlanCache cache = new BoundedPlanCache(2); + CountDownLatch compiling = new CountDownLatch(1); + CountDownLatch release = new CountDownLatch(1); + ExecutorService executor = Executors.newSingleThreadExecutor(); + try { + Future future = executor.submit(() -> { + try { + cache.getOrCompile(key("select slow"), () -> { + compiling.countDown(); + try { + release.await(); + } catch (InterruptedException exception) { + Thread.currentThread().interrupt(); + throw new IllegalStateException(exception); + } + return plan(); + }); + return null; + } catch (FederationSqlException exception) { + return exception.errorCode(); + } + }); + Assert.assertTrue(compiling.await(2, TimeUnit.SECONDS)); + cache.close(); + release.countDown(); + Assert.assertEquals(FederationSqlErrorCode.ENGINE_CLOSED, future.get(2, TimeUnit.SECONDS)); + Assert.assertEquals(0, cache.size()); + } finally { + release.countDown(); + executor.shutdownNow(); + cache.close(); + } + } + + /** + * 验证冷编译并发上限与缓存容量相互独立。 + * + * @throws Exception 并发测试失败 + */ + @Test + public void shouldLimitConcurrentColdCompilations() throws Exception { + BoundedPlanCache cache = new BoundedPlanCache(16, 2); + AtomicInteger active = new AtomicInteger(); + AtomicInteger maximumActive = new AtomicInteger(); + CountDownLatch twoCompilersEntered = new CountDownLatch(2); + CountDownLatch release = new CountDownLatch(1); + ExecutorService executor = Executors.newFixedThreadPool(6); + List> futures = new ArrayList<>(); + try { + for (int index = 0; index < 6; index++) { + int sqlIndex = index; + futures.add(executor.submit(() -> cache.getOrCompile(key("select " + sqlIndex), () -> { + int current = active.incrementAndGet(); + maximumActive.accumulateAndGet(current, Math::max); + twoCompilersEntered.countDown(); + try { + release.await(); + return plan(); + } catch (InterruptedException exception) { + Thread.currentThread().interrupt(); + throw new IllegalStateException(exception); + } finally { + active.decrementAndGet(); + } + }))); + } + Assert.assertTrue(twoCompilersEntered.await(2, TimeUnit.SECONDS)); + Assert.assertEquals(2, maximumActive.get()); + release.countDown(); + for (Future future : futures) { + Assert.assertNotNull(future.get(2, TimeUnit.SECONDS)); + } + Assert.assertEquals(2, maximumActive.get()); + } finally { + release.countDown(); + executor.shutdownNow(); + cache.close(); + } + } + + /** + * 验证超出权重的计划不进入缓存,条目权重始终保持有界。 + */ + @Test + public void shouldRejectOversizedPlanFromCache() { + BoundedPlanCache cache = new BoundedPlanCache(4, 2, 1, Duration.ofMinutes(1)); + AtomicInteger compilations = new AtomicInteger(); + PlanCacheKey key = key("select weighted"); + + cache.getOrCompile(key, () -> { + compilations.incrementAndGet(); + return plan(); + }); + cache.getOrCompile(key, () -> { + compilations.incrementAndGet(); + return plan(); + }); + + Assert.assertEquals(2, compilations.get()); + Assert.assertEquals(0, cache.size()); + Assert.assertEquals(0, cache.weightBytes()); + cache.close(); + } + + /** + * 验证 TTL 到期后旧计划不会继续命中。 + * + * @throws Exception 休眠被中断 + */ + @Test + public void shouldRecompileExpiredPlan() throws Exception { + BoundedPlanCache cache = new BoundedPlanCache( + 4, + 2, + 1_000_000, + Duration.ofMillis(10) + ); + AtomicInteger compilations = new AtomicInteger(); + PlanCacheKey key = key("select expiring"); + cache.getOrCompile(key, () -> { + compilations.incrementAndGet(); + return plan(); + }); + Thread.sleep(30); + cache.getOrCompile(key, () -> { + compilations.incrementAndGet(); + return plan(); + }); + + Assert.assertEquals(2, compilations.get()); + Assert.assertEquals(1, cache.size()); + cache.close(); + } + + /** + * 验证单调时钟跨越 long 回绕边界时,TTL 仍按经过时长判断。 + */ + @Test + public void shouldExpireByElapsedTimeAcrossNanoTimeWrap() { + AtomicLong clock = new AtomicLong(Long.MAX_VALUE - 5L); + BoundedPlanCache cache = new BoundedPlanCache( + 4, + 2, + 1_000_000, + Duration.ofNanos(10L), + clock::get + ); + AtomicInteger compilations = new AtomicInteger(); + PlanCacheKey key = key("select wrap"); + + cache.getOrCompile(key, () -> { + compilations.incrementAndGet(); + return plan(); + }); + clock.set(Long.MIN_VALUE + 1L); + cache.getOrCompile(key, () -> { + compilations.incrementAndGet(); + return plan(); + }); + Assert.assertEquals(1, compilations.get()); + + clock.set(Long.MIN_VALUE + 6L); + cache.getOrCompile(key, () -> { + compilations.incrementAndGet(); + return plan(); + }); + Assert.assertEquals(2, compilations.get()); + cache.close(); + } + + /** + * 验证统计快照先于缓存 TTL 失效时旧成本计划会重新编译。 + * + * @throws Exception 休眠被中断 + */ + @Test + public void shouldNotCacheBeyondStatisticsValidity() throws Exception { + BoundedPlanCache cache = new BoundedPlanCache( + 4, + 2, + 1_000_000, + Duration.ofMinutes(1) + ); + AtomicInteger compilations = new AtomicInteger(); + PlanCacheKey key = key("select statistics expiring"); + cache.getOrCompile(key, () -> { + compilations.incrementAndGet(); + return plan(Instant.now().plusMillis(10)); + }); + Thread.sleep(30); + cache.getOrCompile(key, () -> { + compilations.incrementAndGet(); + return plan(Instant.MAX); + }); + + Assert.assertEquals(2, compilations.get()); + Assert.assertEquals(1, cache.size()); + cache.close(); + } + + /** + * 验证关闭 Runtime 时只移除引用其完整身份的计划。 + */ + @Test + public void shouldInvalidateExactRuntimeIdentity() { + BoundedPlanCache cache = new BoundedPlanCache(4); + cache.getOrCompile(key("select exact"), BoundedPlanCacheTest::plan); + Assert.assertEquals(1, cache.size()); + + cache.invalidateRuntimeIdentity( + new SourceId("source"), + 1, + "checksum", + "fingerprint" + ); + + Assert.assertEquals(0, cache.size()); + cache.close(); + } + + /** + * 验证调用方判定查询级统计已变化时,缓存会丢弃旧计划并重新编译。 + */ + @Test + public void shouldRecompileWhenContextValidatorRejectsCachedPlan() { + BoundedPlanCache cache = new BoundedPlanCache(4); + PlanCacheKey key = key("select context scoped statistics"); + AtomicInteger compilations = new AtomicInteger(); + FederationSqlPlan first = cache.getOrCompileWithStatus( + key, + () -> { + compilations.incrementAndGet(); + return plan(); + }, + BoundedPlanCache.WaitGuard.unbounded(), + ignored -> true + ).plan(); + + BoundedPlanCache.LookupResult refreshed = cache.getOrCompileWithStatus( + key, + () -> { + compilations.incrementAndGet(); + return plan(); + }, + BoundedPlanCache.WaitGuard.unbounded(), + candidate -> candidate != first + ); + + Assert.assertEquals(2, compilations.get()); + Assert.assertFalse(refreshed.cacheHit()); + Assert.assertNotSame(first, refreshed.plan()); + cache.close(); + } + + /** + * 验证迟到的旧统计请求不会淘汰或覆盖已经缓存的新代计划。 + * + * @throws Exception 并发测试失败 + */ + @Test + public void shouldKeepNewerPlanWhenOlderGenerationFinishesLate() throws Exception { + BoundedPlanCache cache = new BoundedPlanCache(4, 2); + PlanCacheKey key = key("select generation protected"); + Instant olderGeneration = Instant.parse("2026-08-23T00:00:00Z"); + Instant newerGeneration = olderGeneration.plusSeconds(1); + FederationSqlPlan olderPlan = plan(); + FederationSqlPlan newerPlan = plan(); + cache.getOrCompileWithStatus( + key, + () -> newerPlan, + BoundedPlanCache.WaitGuard.unbounded(), + candidate -> candidate == newerPlan, + newerGeneration + ); + CountDownLatch olderCompiling = new CountDownLatch(1); + CountDownLatch releaseOlder = new CountDownLatch(1); + ExecutorService executor = Executors.newSingleThreadExecutor(); + try { + Future olderLookup = executor.submit(() -> + cache.getOrCompileWithStatus( + key, + () -> { + olderCompiling.countDown(); + awaitLatch(releaseOlder); + return olderPlan; + }, + BoundedPlanCache.WaitGuard.unbounded(), + candidate -> candidate == olderPlan, + olderGeneration + ) + ); + Assert.assertTrue(olderCompiling.await(2, TimeUnit.SECONDS)); + + BoundedPlanCache.LookupResult concurrentNewerHit = cache.getOrCompileWithStatus( + key, + () -> { + throw new AssertionError("newer plan should remain cached"); + }, + BoundedPlanCache.WaitGuard.unbounded(), + candidate -> candidate == newerPlan, + newerGeneration + ); + Assert.assertTrue(concurrentNewerHit.cacheHit()); + Assert.assertSame(newerPlan, concurrentNewerHit.plan()); + + releaseOlder.countDown(); + Assert.assertSame(olderPlan, olderLookup.get(2, TimeUnit.SECONDS).plan()); + BoundedPlanCache.LookupResult finalNewerHit = cache.getOrCompileWithStatus( + key, + () -> { + throw new AssertionError("older plan must not replace newer generation"); + }, + BoundedPlanCache.WaitGuard.unbounded(), + candidate -> candidate == newerPlan, + newerGeneration + ); + Assert.assertTrue(finalNewerHit.cacheHit()); + Assert.assertSame(newerPlan, finalNewerHit.plan()); + Assert.assertEquals(1, cache.size()); + } finally { + releaseOlder.countDown(); + executor.shutdownNow(); + cache.close(); + } + } + + /** + * 验证编译完成但尚未发布时关闭缓存,迟到任务不能发布成功计划。 + * + * @throws Exception 并发测试失败 + */ + @Test + public void shouldRejectLateInFlightPublicationAfterClose() throws Exception { + BoundedPlanCache cache = new BoundedPlanCache(2, 1); + CountDownLatch readyToPublish = new CountDownLatch(1); + CountDownLatch releasePublication = new CountDownLatch(1); + ExecutorService executor = Executors.newSingleThreadExecutor(); + try { + Future publisher = executor.submit(() -> { + try { + cache.getOrCompileWithStatus( + key("select late publication"), + BoundedPlanCacheTest::plan, + BoundedPlanCache.WaitGuard.unbounded(), + candidate -> { + readyToPublish.countDown(); + awaitLatch(releasePublication); + return true; + }, + Instant.parse("2026-08-23T00:00:00Z") + ); + return null; + } catch (FederationSqlException exception) { + return exception.errorCode(); + } + }); + Assert.assertTrue(readyToPublish.await(2, TimeUnit.SECONDS)); + + cache.close(); + releasePublication.countDown(); + + Assert.assertEquals( + FederationSqlErrorCode.ENGINE_CLOSED, + publisher.get(2, TimeUnit.SECONDS) + ); + Assert.assertEquals(0, cache.size()); + Assert.assertEquals(1, cache.availableCompileSlots()); + } finally { + releasePublication.countDown(); + executor.shutdownNow(); + cache.close(); + } + } + + /** + * 验证可能读取宽表统计的上下文校验不会占用缓存全局锁。 + * + * @throws Exception 并发测试失败 + */ + @Test + public void shouldValidateCachedPlanOutsideGlobalLock() throws Exception { + BoundedPlanCache cache = new BoundedPlanCache(4); + PlanCacheKey key = key("select validation outside lock"); + cache.getOrCompile(key, BoundedPlanCacheTest::plan); + CountDownLatch validating = new CountDownLatch(1); + CountDownLatch release = new CountDownLatch(1); + ExecutorService executor = Executors.newFixedThreadPool(2); + try { + Future lookup = executor.submit(() -> + cache.getOrCompileWithStatus( + key, + BoundedPlanCacheTest::plan, + BoundedPlanCache.WaitGuard.unbounded(), + candidate -> { + validating.countDown(); + try { + release.await(); + } catch (InterruptedException exception) { + Thread.currentThread().interrupt(); + throw new IllegalStateException(exception); + } + return true; + } + ) + ); + Assert.assertTrue(validating.await(2, TimeUnit.SECONDS)); + + Future size = executor.submit(cache::size); + Assert.assertEquals(Integer.valueOf(1), size.get(500, TimeUnit.MILLISECONDS)); + + release.countDown(); + Assert.assertTrue(lookup.get(2, TimeUnit.SECONDS).cacheHit()); + } finally { + release.countDown(); + executor.shutdownNow(); + cache.close(); + } + } + + /** + * 验证同键 single-flight 等待会响应统一编译截止时间。 + * + * @throws Exception 并发测试失败 + */ + @Test + public void shouldStopWaitingForSameKeyWhenCompileDeadlineExpires() throws Exception { + BoundedPlanCache cache = new BoundedPlanCache(4, 1); + CountDownLatch compiling = new CountDownLatch(1); + CountDownLatch release = new CountDownLatch(1); + ExecutorService executor = Executors.newSingleThreadExecutor(); + try { + Future compiler = executor.submit(() -> cache.getOrCompile( + key("select guarded"), + () -> { + compiling.countDown(); + try { + release.await(); + } catch (InterruptedException exception) { + Thread.currentThread().interrupt(); + throw new IllegalStateException(exception); + } + return plan(); + } + )); + Assert.assertTrue(compiling.await(2, TimeUnit.SECONDS)); + + long deadline = System.nanoTime() + TimeUnit.MILLISECONDS.toNanos(30); + try { + cache.getOrCompileWithStatus( + key("select guarded"), + BoundedPlanCacheTest::plan, + deadlineGuard(deadline) + ); + Assert.fail("single-flight waiter should respect compile deadline"); + } catch (FederationSqlException exception) { + Assert.assertEquals( + FederationSqlErrorCode.SQL_COMPILE_TIMEOUT, + exception.errorCode() + ); + } + + release.countDown(); + Assert.assertNotNull(compiler.get(2, TimeUnit.SECONDS)); + } finally { + release.countDown(); + executor.shutdownNow(); + cache.close(); + } + } + + /** + * 验证冷编译槽等待同样受查询截止时间约束。 + * + * @throws Exception 并发测试失败 + */ + @Test + public void shouldStopWaitingForCompileSlotWhenDeadlineExpires() throws Exception { + BoundedPlanCache cache = new BoundedPlanCache(4, 1); + CountDownLatch compiling = new CountDownLatch(1); + CountDownLatch release = new CountDownLatch(1); + ExecutorService executor = Executors.newSingleThreadExecutor(); + try { + Future compiler = executor.submit(() -> cache.getOrCompile( + key("select first"), + () -> { + compiling.countDown(); + try { + release.await(); + } catch (InterruptedException exception) { + Thread.currentThread().interrupt(); + throw new IllegalStateException(exception); + } + return plan(); + } + )); + Assert.assertTrue(compiling.await(2, TimeUnit.SECONDS)); + + long deadline = System.nanoTime() + TimeUnit.MILLISECONDS.toNanos(30); + try { + cache.getOrCompileWithStatus( + key("select second"), + BoundedPlanCacheTest::plan, + deadlineGuard(deadline) + ); + Assert.fail("compile-slot waiter should respect compile deadline"); + } catch (FederationSqlException exception) { + Assert.assertEquals( + FederationSqlErrorCode.SQL_COMPILE_TIMEOUT, + exception.errorCode() + ); + } + + release.countDown(); + Assert.assertNotNull(compiler.get(2, TimeUnit.SECONDS)); + } finally { + release.countDown(); + executor.shutdownNow(); + cache.close(); + } + } + + /** + * 验证等待冷编译槽的线程遇到 close 后退出,随后归还的许可不会泄漏。 + * + * @throws Exception 并发测试失败 + */ + @Test + public void shouldReleaseCompileSlotWhenCacheClosesWhileAnotherThreadWaits() throws Exception { + BoundedPlanCache cache = new BoundedPlanCache(4, 1); + CountDownLatch compiling = new CountDownLatch(1); + CountDownLatch releaseCompiler = new CountDownLatch(1); + CountDownLatch waiterStarted = new CountDownLatch(1); + ExecutorService executor = Executors.newFixedThreadPool(2); + try { + Future compiler = executor.submit(() -> { + try { + cache.getOrCompile(key("select holding slot"), () -> { + compiling.countDown(); + try { + releaseCompiler.await(); + } catch (InterruptedException exception) { + Thread.currentThread().interrupt(); + throw new IllegalStateException(exception); + } + return plan(); + }); + return null; + } catch (FederationSqlException exception) { + return exception.errorCode(); + } + }); + Assert.assertTrue(compiling.await(2, TimeUnit.SECONDS)); + Future waiter = executor.submit(() -> { + waiterStarted.countDown(); + try { + cache.getOrCompile(key("select waiting for slot"), BoundedPlanCacheTest::plan); + return null; + } catch (FederationSqlException exception) { + return exception.errorCode(); + } + }); + Assert.assertTrue(waiterStarted.await(2, TimeUnit.SECONDS)); + + cache.close(); + releaseCompiler.countDown(); + + Assert.assertEquals(FederationSqlErrorCode.ENGINE_CLOSED, waiter.get(2, TimeUnit.SECONDS)); + Assert.assertEquals(FederationSqlErrorCode.ENGINE_CLOSED, compiler.get(2, TimeUnit.SECONDS)); + Assert.assertEquals(1, cache.availableCompileSlots()); + } finally { + releaseCompiler.countDown(); + executor.shutdownNow(); + cache.close(); + } + } + + private static BoundedPlanCache.WaitGuard deadlineGuard(long deadlineNanos) { + return new BoundedPlanCache.WaitGuard() { + @Override + public void ensureAllowed() { + if (remainingNanos() <= 0L) { + throw new FederationSqlException( + FederationSqlErrorCode.SQL_COMPILE_TIMEOUT, + "test compile deadline expired" + ); + } + } + + @Override + public long remainingNanos() { + return Math.max(0L, deadlineNanos - System.nanoTime()); + } + }; + } + + /** + * 等待测试闩锁,并把中断转换为测试线程可见的失败。 + * + * @param latch 测试闩锁 + */ + private static void awaitLatch(CountDownLatch latch) { + try { + latch.await(); + } catch (InterruptedException exception) { + Thread.currentThread().interrupt(); + throw new IllegalStateException(exception); + } + } + + private static PlanCacheKey key(String sql) { + return new PlanCacheKey( + sql, + new SourceId("source"), + 1, + 1, + List.of(), + "fake", + "fingerprint", + "engine-policy", + "default", + 1 + ); + } + + private static FederationSqlPlan plan() { + return plan(Instant.MAX); + } + + private static FederationSqlPlan plan(Instant statisticsValidUntil) { + return new DefaultFederationSqlPlan( + new SourceId("source"), + 1, + "SELECT 1", + "SELECT 1", + null, + null, + 0, + List.of(), + List.of(), + Set.of(new SourceId("source")), + new AdapterCompatibility( + AdapterCompatibilityStatus.VERIFIED, + "fake", + "1", + "fake-driver", + "1", + "test" + ), + true, + "checksum", + "fake", + "fingerprint", + statisticsValidUntil + ); + } +} diff --git a/easy-agents-federation-sql/easy-agents-federation-sql-core/src/test/java/com/easyagents/federation/sql/runtime/CalciteFederationSqlCompilerTest.java b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/test/java/com/easyagents/federation/sql/runtime/CalciteFederationSqlCompilerTest.java new file mode 100644 index 0000000..7aee8a1 --- /dev/null +++ b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/test/java/com/easyagents/federation/sql/runtime/CalciteFederationSqlCompilerTest.java @@ -0,0 +1,1108 @@ +package com.easyagents.federation.sql.runtime; + +import com.easyagents.federation.sql.adapter.AdapterCompatibility; +import com.easyagents.federation.sql.adapter.AdapterCompatibilityStatus; +import com.easyagents.federation.sql.adapter.AdapterDialectContext; +import com.easyagents.federation.sql.adapter.AdapterHints; +import com.easyagents.federation.sql.adapter.AdapterSchemaContext; +import com.easyagents.federation.sql.adapter.FederationSqlAdapterProvider; +import com.easyagents.federation.sql.api.SqlCompletionItem; +import com.easyagents.federation.sql.api.SqlCompletionKind; +import com.easyagents.federation.sql.api.SqlCompletionRequest; +import com.easyagents.federation.sql.api.SqlCompletionResult; +import com.easyagents.federation.sql.compile.FederationSqlPlan; +import com.easyagents.federation.sql.compile.SqlCompileRequest; +import com.easyagents.federation.sql.execute.FederationFragmentExecutor; +import com.easyagents.federation.sql.federation.FederationColumnStatistics; +import com.easyagents.federation.sql.federation.FederationExecutionPolicy; +import com.easyagents.federation.sql.federation.FederationJoinOptimization; +import com.easyagents.federation.sql.federation.FederationJoinSelectionReason; +import com.easyagents.federation.sql.federation.FederationLogicalTableDefinition; +import com.easyagents.federation.sql.federation.FederationQueryScopeDefinition; +import com.easyagents.federation.sql.federation.FederationQueryMode; +import com.easyagents.federation.sql.federation.FederationStatisticsSnapshot; +import com.easyagents.federation.sql.federation.FederationStatisticsStatus; +import com.easyagents.federation.sql.federation.FederationSourceBindingDefinition; +import com.easyagents.federation.sql.federation.FederationTableStatistics; +import com.easyagents.federation.sql.federation.FederationTableStatisticsProvider; +import com.easyagents.federation.sql.source.ExternalSchemaDefinition; +import com.easyagents.federation.sql.source.FederationDataSourceHandles; +import com.easyagents.federation.sql.source.FederationSourceDefinition; +import com.easyagents.federation.sql.source.RuntimeFingerprint; +import com.easyagents.federation.sql.source.SourceId; +import java.lang.reflect.Proxy; +import java.sql.DatabaseMetaData; +import java.time.Instant; +import java.util.List; +import java.util.Map; +import java.util.Set; +import javax.sql.DataSource; +import org.apache.calcite.avatica.util.Casing; +import org.apache.calcite.jdbc.CalciteSchema; +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.RelVisitor; +import org.apache.calcite.rel.metadata.RelMetadataQuery; +import org.apache.calcite.rel.rules.HyperGraph; +import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.rel.type.RelDataTypeFactory; +import org.apache.calcite.schema.Schema; +import org.apache.calcite.schema.SchemaPlus; +import org.apache.calcite.schema.Table; +import org.apache.calcite.schema.impl.AbstractSchema; +import org.apache.calcite.schema.impl.AbstractTable; +import org.apache.calcite.sql.SqlDialect; +import org.apache.calcite.sql.SqlIdentifier; +import org.apache.calcite.sql.SqlDialect.DatabaseProduct; +import org.apache.calcite.sql.fun.SqlStdOperatorTable; +import org.apache.calcite.sql.type.SqlTypeName; +import org.apache.calcite.util.ImmutableBitSet; +import org.junit.Assert; +import org.junit.Test; + +/** + * Calcite 编译器的目标方言解析契约测试。 + */ +public class CalciteFederationSqlCompilerTest { + + /** + * 验证未加引号名称遵循目标数据库大小写,同时保留 ANSI 双引号输入。 + */ + @Test + public void shouldUseTargetDialectIdentifierCasingWithAnsiQuotes() { + SqlDialect dialect = new SqlDialect(SqlDialect.EMPTY_CONTEXT + .withDatabaseProduct(DatabaseProduct.MYSQL) + .withIdentifierQuoteString("`") + .withQuotedCasing(Casing.UNCHANGED) + .withUnquotedCasing(Casing.UNCHANGED) + .withCaseSensitive(true)); + SchemaPlus rootSchema = CalciteSchema.createRootSchema(true, false).plus(); + SchemaPlus sourceSchema = rootSchema.add("main", new AbstractSchema()); + SchemaPlus defaultSchema = sourceSchema.add("main", lowerCaseSchema()); + FederationSourceDefinition definition = new FederationSourceDefinition( + new SourceId("main"), + 1, + "test-adapter", + List.of(new ExternalSchemaDefinition("main", "test", 1)), + Map.of() + ); + DataSource dataSource = (DataSource) Proxy.newProxyInstance( + getClass().getClassLoader(), + new Class[] {DataSource.class}, + (proxy, method, arguments) -> { + throw new UnsupportedOperationException(method.getName()); + } + ); + SourceRuntime runtime = new SourceRuntime( + definition, + FederationDataSourceHandles.shared( + dataSource, + new RuntimeFingerprint("test", "1", "test", "1", "1") + ), + new TestAdapter(dialect), + dialect, + rootSchema, + defaultSchema, + new AdapterCompatibility( + AdapterCompatibilityStatus.VERIFIED, + "test", + "1", + "test", + "1", + "test" + ), + ignored -> { } + ); + CalciteFederationSqlCompiler compiler = new CalciteFederationSqlCompiler(List.of(), false); + Assert.assertTrue(runtime.adapter().parserConfig(dialect).caseSensitive()); + + FederationSqlPlan unquoted = compiler.compile( + SqlCompileRequest.of("SELECT id FROM orders", definition.sourceId(), 1), + runtime, + Set.of(definition.sourceId()) + ); + FederationSqlPlan quoted = compiler.compile( + SqlCompileRequest.of("SELECT \"id\" FROM \"orders\"", definition.sourceId(), 1), + runtime, + Set.of(definition.sourceId()) + ); + + Assert.assertTrue(unquoted.executable()); + Assert.assertTrue(quoted.executable()); + Assert.assertTrue(unquoted.executableSql().contains("`orders`")); + Assert.assertTrue(quoted.executableSql().contains("`orders`")); + } + + /** + * 验证嵌套 CTE 名称不会污染外层物理表的 Binding 发现作用域。 + */ + @Test + public void shouldDiscoverBindingsWithLexicallyScopedCteNames() { + FederationQueryScopeDefinition scope = FederationQueryScopeDefinition.virtual( + "cte-scope", + 1, + Map.of( + "DEFAULT", FederationSourceBindingDefinition.of(new SourceId("default"), 1), + "REMOTE", FederationSourceBindingDefinition.of(new SourceId("remote"), 1) + ), + "DEFAULT", + FederationExecutionPolicy.basic() + ); + SqlCompileRequest request = SqlCompileRequest.of( + "SELECT * FROM x WHERE EXISTS (" + + "WITH x AS (SELECT * FROM REMOTE.APP.T) SELECT * FROM x)", + scope + ); + + Set bindings = new CalciteFederationSqlCompiler(List.of(), true) + .discoverBindings(request); + + Assert.assertEquals(Set.of("DEFAULT", "REMOTE"), bindings); + + SqlCompileRequest nonRecursive = SqlCompileRequest.of( + "WITH x AS (SELECT d.ID FROM x d JOIN REMOTE.APP.T r ON d.ID = r.ID) " + + "SELECT * FROM x", + scope + ); + Assert.assertEquals( + Set.of("DEFAULT", "REMOTE"), + new CalciteFederationSqlCompiler(List.of(), true) + .discoverBindings(nonRecursive) + ); + + SqlCompileRequest quoted = SqlCompileRequest.of( + "WITH \"x\" AS (SELECT * FROM REMOTE.APP.T) SELECT * FROM \"X\"", + scope + ); + org.apache.calcite.sql.SqlWith parsedQuoted; + try { + parsedQuoted = (org.apache.calcite.sql.SqlWith) + org.apache.calcite.sql.parser.SqlParser.create( + quoted.sql(), + org.apache.calcite.sql.parser.SqlParser.config() + .withLex(org.apache.calcite.config.Lex.ORACLE) + ).parseQuery(); + } catch (org.apache.calcite.sql.parser.SqlParseException exception) { + throw new AssertionError(exception); + } + SqlIdentifier declared = ((org.apache.calcite.sql.SqlWithItem) + parsedQuoted.withList.get(0)).name; + SqlIdentifier referenced = (SqlIdentifier) + ((org.apache.calcite.sql.SqlSelect) parsedQuoted.body).getFrom(); + Assert.assertEquals("x", declared.getSimple()); + Assert.assertEquals("X", referenced.getSimple()); + Assert.assertEquals( + Set.of("DEFAULT", "REMOTE"), + new CalciteFederationSqlCompiler(List.of(), true).discoverBindings(quoted) + ); + } + + /** + * 验证短逻辑表名按映射选择候选 Binding,避免错误加载默认物理源。 + */ + @Test + public void shouldDiscoverBindingsFromLogicalTableMappings() { + FederationQueryScopeDefinition scope = FederationQueryScopeDefinition.virtual( + "logical-binding-scope", + 1, + Map.of( + "MYSQL_1", FederationSourceBindingDefinition.of(new SourceId("mysql"), 1), + "PG_1", FederationSourceBindingDefinition.of(new SourceId("postgres"), 1) + ), + "MYSQL_1", + List.of( + FederationLogicalTableDefinition.of( + "outlet", "MYSQL_1", "MAIN", "physical_outlet" + ), + FederationLogicalTableDefinition.of( + "outlet_region", "PG_1", "PUBLIC", "physical_region" + ) + ), + FederationExecutionPolicy.basic() + ); + SqlCompileRequest request = SqlCompileRequest.of( + "SELECT * FROM outlet o JOIN outlet_region r ON o.ID = r.ID", + scope + ); + + Assert.assertEquals( + Set.of("MYSQL_1", "PG_1"), + new CalciteFederationSqlCompiler(List.of(), true).discoverBindings(request) + ); + } + + /** + * 验证 Calcite Advisor 只暴露查询范围逻辑表,并支持短名、三段名与字段补全。 + */ + @Test + public void shouldCompleteLogicalTablesAndColumnsWithoutLeakingPhysicalNames() { + SourceId sourceId = new SourceId("source"); + SchemaPlus rootSchema = CalciteSchema.createRootSchema(true, false).plus(); + SchemaPlus sourceRoot = rootSchema.add(sourceId.value(), new AbstractSchema()); + SchemaPlus defaultSchema = sourceRoot.add("main", new AbstractSchema() { + @Override + protected Map getTableMap() { + return Map.of("physical_orders", table("customer_id")); + } + }); + FederationSourceDefinition definition = new FederationSourceDefinition( + sourceId, + 1, + "test-adapter", + List.of(new ExternalSchemaDefinition("main", "test", 1)), + Map.of() + ); + SourceRuntime runtime = runtime(definition, rootSchema, defaultSchema); + FederationQueryScopeDefinition scope = FederationQueryScopeDefinition.virtual( + "completion-scope", + 1, + Map.of( + "sales", + FederationSourceBindingDefinition.of(sourceId, 1, Map.of("main", "main")) + ), + "sales", + List.of(FederationLogicalTableDefinition.of( + "orders", "sales", "main", "physical_orders" + )), + FederationExecutionPolicy.basic() + ); + + try (FederationQueryScopeSnapshot snapshot = + FederationQueryScopeSnapshot.borrowedSingle(scope, runtime)) { + CalciteSqlCompleter completer = new CalciteSqlCompleter(); + SqlCompletionResult shortName = completer.complete( + request(scope, "SELECT * FROM ord"), snapshot + ); + SqlCompletionResult qualified = completer.complete( + request(scope, "SELECT * FROM sales.main.o"), snapshot + ); + String columnSql = "SELECT orders. FROM orders"; + SqlCompletionResult column = completer.complete( + new SqlCompletionRequest(scope, columnSql, columnSql.indexOf('.') + 1), + snapshot + ); + SqlCompletionResult keyword = completer.complete( + request(scope, "SEL"), snapshot + ); + SqlCompletionResult function = completer.complete( + request(scope, "SELECT COU"), snapshot + ); + + Assert.assertTrue(has(shortName, "orders", SqlCompletionKind.TABLE)); + Assert.assertTrue(qualified.toString(), has(qualified, "orders", SqlCompletionKind.TABLE)); + Assert.assertTrue(column.toString(), has(column, "customer_id", SqlCompletionKind.COLUMN)); + Assert.assertTrue(keyword.toString(), has(keyword, "SELECT", SqlCompletionKind.KEYWORD)); + Assert.assertTrue(function.toString(), has(function, "COUNT", SqlCompletionKind.FUNCTION)); + Assert.assertFalse(shortName.items().stream() + .map(SqlCompletionItem::label) + .anyMatch("physical_orders"::equalsIgnoreCase)); + } + } + + /** + * 验证冻结统计进入 Calcite 元数据与分片成本,并让等值 INNER JOIN 选择较小构建侧。 + */ + @Test + public void shouldUseFrozenStatisticsForFragmentCostsAndJoinBuildSide() { + SourceId smallSource = new SourceId("small-source"); + SourceId largeSource = new SourceId("large-source"); + SourceRuntime smallRuntime = runtime( + definition(smallSource, "small_table"), + rootSchema(smallSource, "small_table"), + null + ); + SourceRuntime largeRuntime = runtime( + definition(largeSource, "large_table"), + rootSchema(largeSource, "large_table"), + null + ); + FederationQueryScopeDefinition scope = FederationQueryScopeDefinition.virtual( + "statistics-scope", + 1, + Map.of( + "SMALL_DB", FederationSourceBindingDefinition.of( + smallSource, 1, Map.of("main", "main") + ), + "LARGE_DB", FederationSourceBindingDefinition.of( + largeSource, 1, Map.of("main", "main") + ) + ), + "LARGE_DB", + FederationExecutionPolicy.basic() + ); + Instant collectedAt = Instant.parse("2026-08-23T00:00:00Z"); + FederationStatisticsSnapshot frozen = new FederationStatisticsSnapshot( + "statistics-v1", + Map.of( + new FederationStatisticsSnapshot.TableKey( + smallSource, "main", "small_table" + ), + statistics(10, 32, 10, collectedAt), + new FederationStatisticsSnapshot.TableKey( + largeSource, "main", "large_table" + ), + statistics(10_000, 64, 8_000, collectedAt) + ) + ); + FederationTableStatisticsProvider provider = () -> frozen; + CalciteFederationSqlCompiler compiler = new CalciteFederationSqlCompiler( + List.of(), + true, + FederationExecutionPolicy.basic(), + provider + ); + SqlCompileRequest request = SqlCompileRequest.of( + "SELECT * FROM SMALL_DB.main.small_table s " + + "JOIN LARGE_DB.main.large_table l ON s.id = l.id", + scope + ); + + try (FederationQueryScopeSnapshot snapshot = FederationQueryScopeSnapshot.borrowed( + scope, + Map.of("SMALL_DB", smallRuntime, "LARGE_DB", largeRuntime) + )) { + FederationSqlPlan plan = compiler.compile(request, snapshot, frozen); + + Assert.assertEquals(FederationQueryMode.FEDERATED, plan.queryMode()); + Assert.assertEquals(2, plan.fragments().size()); + Assert.assertTrue(plan.fragments().stream().allMatch(fragment -> + fragment.costEstimate().statisticsStatus() + == FederationStatisticsStatus.COMPLETE + && fragment.costEstimate().statisticsSnapshotVersion() + .equals("statistics-v1") + )); + Assert.assertEquals(1, plan.joinOptimizations().size()); + Assert.assertEquals( + "SMALL_DB", + plan.joinOptimizations().get(0).buildBinding() + ); + Assert.assertEquals( + 320D, + plan.joinOptimizations().get(0).estimatedBuildBytes(), + 0.001D + ); + assertFrozenMetadata(plan); + } + } + + /** + * 验证仅含表行数和行宽的 MySQL 级 PARTIAL 统计仍可选择较小构建侧。 + */ + @Test + public void shouldUsePartialTableStatisticsForJoinBuildSide() { + SourceId smallSource = new SourceId("partial-small-source"); + SourceId largeSource = new SourceId("complete-large-source"); + SourceRuntime smallRuntime = runtime( + definition(smallSource, "small_table"), + rootSchema(smallSource, "small_table"), + null + ); + SourceRuntime largeRuntime = runtime( + definition(largeSource, "large_table"), + rootSchema(largeSource, "large_table"), + null + ); + FederationQueryScopeDefinition scope = FederationQueryScopeDefinition.virtual( + "partial-statistics-scope", + 1, + Map.of( + "SMALL_DB", FederationSourceBindingDefinition.of( + smallSource, 1, Map.of("main", "main") + ), + "LARGE_DB", FederationSourceBindingDefinition.of( + largeSource, 1, Map.of("main", "main") + ) + ), + "LARGE_DB", + FederationExecutionPolicy.basic() + ); + Instant collectedAt = Instant.parse("2026-08-23T00:00:00Z"); + FederationStatisticsSnapshot frozen = new FederationStatisticsSnapshot( + "partial-statistics-v1", + Map.of( + new FederationStatisticsSnapshot.TableKey( + smallSource, "main", "small_table" + ), + partialStatistics(10, 32, collectedAt), + new FederationStatisticsSnapshot.TableKey( + largeSource, "main", "large_table" + ), + statistics(10_000, 64, 8_000, collectedAt) + ) + ); + CalciteFederationSqlCompiler compiler = new CalciteFederationSqlCompiler( + List.of(), + true, + FederationExecutionPolicy.basic(), + () -> frozen + ); + SqlCompileRequest request = SqlCompileRequest.of( + "SELECT * FROM SMALL_DB.main.small_table s " + + "JOIN LARGE_DB.main.large_table l ON s.id = l.id", + scope + ); + + try (FederationQueryScopeSnapshot snapshot = FederationQueryScopeSnapshot.borrowed( + scope, + Map.of("SMALL_DB", smallRuntime, "LARGE_DB", largeRuntime) + )) { + FederationSqlPlan plan = compiler.compile(request, snapshot, frozen); + + Assert.assertTrue(plan.fragments().stream().anyMatch(fragment -> + fragment.bindingName().equals("SMALL_DB") + && fragment.costEstimate().statisticsStatus() + == FederationStatisticsStatus.PARTIAL + && !fragment.costEstimate().statisticsMissing() + )); + Assert.assertEquals("SMALL_DB", plan.joinOptimizations().get(0).buildBinding()); + Assert.assertEquals( + FederationJoinSelectionReason.SMALLER_BUILD_SIDE, + plan.joinOptimizations().get(0).reason() + ); + Assert.assertEquals( + 320D, + plan.joinOptimizations().get(0).estimatedBuildBytes(), + 0.001D + ); + } + } + + /** + * 验证三数据源 Join 生成两个有序阶段,并保留每阶段完整输入集合。 + */ + @Test + public void shouldDescribeEveryJoinStageForThreeSources() { + FederationExecutionPolicy threeSourcePolicy = new FederationExecutionPolicy( + 3, + 8, + 2, + 100_000, + 64L * 1024L * 1024L, + 60_000 + ); + SourceId firstSource = new SourceId("first-source"); + SourceId secondSource = new SourceId("second-source"); + SourceId thirdSource = new SourceId("third-source"); + SourceRuntime firstRuntime = runtime( + definition(firstSource, "first_table"), + rootSchema(firstSource, "first_table"), + null + ); + SourceRuntime secondRuntime = runtime( + definition(secondSource, "second_table"), + rootSchema(secondSource, "second_table"), + null + ); + SourceRuntime thirdRuntime = runtime( + definition(thirdSource, "third_table"), + rootSchema(thirdSource, "third_table"), + null + ); + FederationQueryScopeDefinition scope = FederationQueryScopeDefinition.virtual( + "three-source-statistics-scope", + 1, + Map.of( + "FIRST_DB", FederationSourceBindingDefinition.of( + firstSource, 1, Map.of("main", "main") + ), + "SECOND_DB", FederationSourceBindingDefinition.of( + secondSource, 1, Map.of("main", "main") + ), + "THIRD_DB", FederationSourceBindingDefinition.of( + thirdSource, 1, Map.of("main", "main") + ) + ), + "FIRST_DB", + threeSourcePolicy + ); + Instant collectedAt = Instant.parse("2026-08-23T00:00:00Z"); + FederationStatisticsSnapshot frozen = new FederationStatisticsSnapshot( + "three-source-v1", + Map.of( + new FederationStatisticsSnapshot.TableKey( + firstSource, "main", "first_table" + ), + statistics(10_000, 64, 8_000, collectedAt), + new FederationStatisticsSnapshot.TableKey( + secondSource, "main", "second_table" + ), + statistics(100, 32, 100, collectedAt), + new FederationStatisticsSnapshot.TableKey( + thirdSource, "main", "third_table" + ), + statistics(10, 24, 10, collectedAt) + ) + ); + CalciteFederationSqlCompiler compiler = new CalciteFederationSqlCompiler( + List.of(), + true, + threeSourcePolicy, + () -> frozen + ); + SqlCompileRequest request = SqlCompileRequest.of( + "SELECT * FROM FIRST_DB.main.first_table a " + + "JOIN SECOND_DB.main.second_table b ON a.id = b.id " + + "JOIN THIRD_DB.main.third_table c ON b.id = c.id", + scope + ); + + try (FederationQueryScopeSnapshot snapshot = FederationQueryScopeSnapshot.borrowed( + scope, + Map.of( + "FIRST_DB", firstRuntime, + "SECOND_DB", secondRuntime, + "THIRD_DB", thirdRuntime + ) + )) { + FederationSqlPlan plan = compiler.compile(request, snapshot, frozen); + + Assert.assertEquals(FederationQueryMode.FEDERATED, plan.queryMode()); + assertNoPlannerPlaceholder(plan); + Assert.assertEquals(3, plan.fragments().size()); + Assert.assertEquals(2, plan.joinOptimizations().size()); + Assert.assertEquals(1, plan.joinOptimizations().get(0).stageIndex()); + Assert.assertEquals(2, plan.joinOptimizations().get(1).stageIndex()); + var firstStage = plan.joinOptimizations().get(0); + java.util.LinkedHashSet firstInputs = new java.util.LinkedHashSet<>( + firstStage.leftBindings() + ); + firstInputs.addAll(firstStage.rightBindings()); + Assert.assertEquals(Set.of("SECOND_DB", "THIRD_DB"), firstInputs); + var finalStage = plan.joinOptimizations().get(1); + java.util.LinkedHashSet finalInputs = new java.util.LinkedHashSet<>( + finalStage.leftBindings() + ); + finalInputs.addAll(finalStage.rightBindings()); + Assert.assertEquals(Set.of("FIRST_DB", "SECOND_DB", "THIRD_DB"), finalInputs); + Assert.assertTrue( + finalStage.leftBindings().size() > 1 || finalStage.rightBindings().size() > 1 + ); + Assert.assertTrue(plan.joinOptimizations().stream().allMatch(optimization -> + optimization.estimatedBuildBytes() >= 0D + && Double.isFinite(optimization.estimatedBuildBytes()) + )); + } + } + + /** + * 验证中间结果行数更少但字节更大的候选不会击败低搬运成本 Join 顺序。 + */ + @Test + public void shouldPreferLowerTransferBytesOverLowerIntermediateRowCount() { + FederationExecutionPolicy threeSourcePolicy = new FederationExecutionPolicy( + 3, + 8, + 2, + 100_000, + 64L * 1024L * 1024L, + 60_000 + ); + SourceId wideSource = new SourceId("wide-source"); + SourceId bridgeSource = new SourceId("bridge-source"); + SourceId narrowSource = new SourceId("narrow-source"); + SourceRuntime wideRuntime = runtime( + definition(wideSource, "wide_table"), + rootSchema(wideSource, "wide_table"), + null + ); + SourceRuntime bridgeRuntime = runtime( + definition(bridgeSource, "bridge_table"), + rootSchema(bridgeSource, "bridge_table"), + null + ); + SourceRuntime narrowRuntime = runtime( + definition(narrowSource, "narrow_table"), + rootSchema(narrowSource, "narrow_table"), + null + ); + FederationQueryScopeDefinition scope = FederationQueryScopeDefinition.virtual( + "transfer-cost-scope", + 1, + Map.of( + "WIDE_DB", FederationSourceBindingDefinition.of( + wideSource, 1, Map.of("main", "main") + ), + "BRIDGE_DB", FederationSourceBindingDefinition.of( + bridgeSource, 1, Map.of("main", "main") + ), + "NARROW_DB", FederationSourceBindingDefinition.of( + narrowSource, 1, Map.of("main", "main") + ) + ), + "WIDE_DB", + threeSourcePolicy + ); + Instant collectedAt = Instant.parse("2026-08-23T00:00:00Z"); + FederationStatisticsSnapshot frozen = new FederationStatisticsSnapshot( + "transfer-cost-v1", + Map.of( + new FederationStatisticsSnapshot.TableKey( + wideSource, "main", "wide_table" + ), + singleColumnStatistics(1, 1_000_000, 1, collectedAt), + new FederationStatisticsSnapshot.TableKey( + bridgeSource, "main", "bridge_table" + ), + singleColumnStatistics(100, 16, 100, collectedAt), + new FederationStatisticsSnapshot.TableKey( + narrowSource, "main", "narrow_table" + ), + singleColumnStatistics(10_000, 16, 10_000, collectedAt) + ) + ); + CalciteFederationSqlCompiler compiler = new CalciteFederationSqlCompiler( + List.of(), + true, + threeSourcePolicy, + () -> frozen + ); + SqlCompileRequest request = SqlCompileRequest.of( + "SELECT * FROM WIDE_DB.main.wide_table a " + + "JOIN BRIDGE_DB.main.bridge_table b ON a.id = b.id " + + "JOIN NARROW_DB.main.narrow_table c ON b.id = c.id", + scope + ); + + try (FederationQueryScopeSnapshot snapshot = FederationQueryScopeSnapshot.borrowed( + scope, + Map.of( + "WIDE_DB", wideRuntime, + "BRIDGE_DB", bridgeRuntime, + "NARROW_DB", narrowRuntime + ) + )) { + FederationSqlPlan plan = compiler.compile(request, snapshot, frozen); + + Assert.assertEquals(2, plan.joinOptimizations().size()); + FederationJoinOptimization firstStage = plan.joinOptimizations().get(0); + java.util.LinkedHashSet firstInputs = new java.util.LinkedHashSet<>( + firstStage.leftBindings() + ); + firstInputs.addAll(firstStage.rightBindings()); + Assert.assertEquals(Set.of("BRIDGE_DB", "NARROW_DB"), firstInputs); + Assert.assertTrue(firstStage.estimatedBuildBytes() < 10_000D); + } + } + + /** + * 断言优化器内部占位节点已经完全转换为可执行关系节点。 + * + * @param plan 编译计划 + */ + private static void assertNoPlannerPlaceholder(FederationSqlPlan plan) { + new RelVisitor() { + @Override + public void visit(RelNode node, int ordinal, RelNode parent) { + Assert.assertFalse(node instanceof HyperGraph); + super.visit(node, ordinal, parent); + } + }.go(plan.relRoot().rel); + } + + /** + * 创建包含表级和列级信息的完整统计。 + * + * @param rows 表行数 + * @param rowWidth 平均行宽 + * @param distinctIds ID 基数 + * @param collectedAt 采集时间 + * @return 完整统计快照 + */ + private static FederationTableStatistics statistics( + double rows, + long rowWidth, + double distinctIds, + Instant collectedAt + ) { + return new FederationTableStatistics( + rows, + rowWidth, + collectedAt, + "catalog", + Map.of("id", new FederationColumnStatistics(distinctIds, 0.1D, 8)), + List.of(List.of("id")), + Instant.MAX, + FederationStatisticsStatus.COMPLETE + ); + } + + /** + * 创建仅包含表级行数和平均行宽的部分统计。 + * + * @param rows 表行数 + * @param rowWidth 平均行宽 + * @param collectedAt 采集时间 + * @return 部分统计快照 + */ + private static FederationTableStatistics partialStatistics( + double rows, + long rowWidth, + Instant collectedAt + ) { + return new FederationTableStatistics( + rows, + rowWidth, + collectedAt, + "catalog", + Map.of(), + List.of(), + Instant.MAX, + FederationStatisticsStatus.PARTIAL + ); + } + + /** + * 创建单列测试表统计,使列宽与整行宽度一致以判别字节成本。 + * + * @param rows 表行数 + * @param rowWidth 平均行宽及唯一列宽 + * @param distinctIds ID 基数 + * @param collectedAt 采集时间 + * @return 完整统计快照 + */ + private static FederationTableStatistics singleColumnStatistics( + double rows, + long rowWidth, + double distinctIds, + Instant collectedAt + ) { + return new FederationTableStatistics( + rows, + rowWidth, + collectedAt, + "catalog", + Map.of("id", new FederationColumnStatistics(distinctIds, 0.1D, rowWidth)), + List.of(List.of("id")), + Instant.MAX, + FederationStatisticsStatus.COMPLETE + ); + } + + /** + * 断言冻结统计可由 Calcite 元数据查询读取。 + * + * @param plan 编译计划 + */ + private static void assertFrozenMetadata(FederationSqlPlan plan) { + List scans = new java.util.ArrayList<>(); + new RelVisitor() { + @Override + public void visit(RelNode node, int ordinal, RelNode parent) { + if (node instanceof FederationStatisticsTableScan scan) { + scans.add(scan); + } + super.visit(node, ordinal, parent); + } + }.go(plan.relRoot().rel); + Assert.assertEquals(2, scans.size()); + + FederationStatisticsTableScan smallScan = scans.stream() + .filter(scan -> scan.statistics().estimatedRows() == 10D) + .findFirst() + .orElseThrow(); + RelMetadataQuery metadataQuery = smallScan.getCluster().getMetadataQuery(); + Assert.assertEquals(10D, metadataQuery.getRowCount(smallScan), 0.001D); + Assert.assertEquals(32D, metadataQuery.getAverageRowSize(smallScan), 0.001D); + Assert.assertEquals( + Double.valueOf(8D), + metadataQuery.getAverageColumnSizes(smallScan).get(0) + ); + Assert.assertEquals( + 10D, + metadataQuery.getDistinctRowCount(smallScan, ImmutableBitSet.of(0), null), + 0.001D + ); + Assert.assertTrue(metadataQuery.getUniqueKeys(smallScan).contains(ImmutableBitSet.of(0))); + Assert.assertEquals( + 0.09D, + metadataQuery.getSelectivity( + smallScan, + smallScan.getCluster().getRexBuilder().makeCall( + SqlStdOperatorTable.EQUALS, + smallScan.getCluster().getRexBuilder().makeInputRef(smallScan, 0), + smallScan.getCluster().getRexBuilder().makeExactLiteral( + java.math.BigDecimal.ONE + ) + ) + ), + 0.001D + ); + Assert.assertEquals( + 1D, + metadataQuery.getSelectivity( + smallScan, + smallScan.getCluster().getRexBuilder().makeCall( + SqlStdOperatorTable.IS_NOT_DISTINCT_FROM, + smallScan.getCluster().getRexBuilder().makeInputRef(smallScan, 0), + smallScan.getCluster().getRexBuilder().makeInputRef(smallScan, 0) + ) + ), + 0.001D + ); + var nullLiteral = smallScan.getCluster().getRexBuilder().makeNullLiteral( + smallScan.getRowType().getFieldList().get(0).getType() + ); + var inputReference = smallScan.getCluster().getRexBuilder().makeInputRef(smallScan, 0); + Assert.assertEquals( + 0.1D, + metadataQuery.getSelectivity( + smallScan, + smallScan.getCluster().getRexBuilder().makeCall( + SqlStdOperatorTable.IS_NOT_DISTINCT_FROM, + inputReference, + nullLiteral + ) + ), + 0.001D + ); + Assert.assertEquals( + 0.1D, + metadataQuery.getSelectivity( + smallScan, + smallScan.getCluster().getRexBuilder().makeCall( + SqlStdOperatorTable.IS_NOT_DISTINCT_FROM, + nullLiteral, + inputReference + ) + ), + 0.001D + ); + } + + /** + * 验证同一编译只使用快照捕获时刻判断有效期,过期统计不会驱动 Join 交换。 + */ + @Test + public void shouldKeepJoinOrderWhenStatisticsAreStaleAtSnapshotTime() { + SourceId smallSource = new SourceId("stale-small-source"); + SourceId largeSource = new SourceId("stale-large-source"); + SourceRuntime smallRuntime = runtime( + definition(smallSource, "small_table"), + rootSchema(smallSource, "small_table"), + null + ); + SourceRuntime largeRuntime = runtime( + definition(largeSource, "large_table"), + rootSchema(largeSource, "large_table"), + null + ); + FederationQueryScopeDefinition scope = FederationQueryScopeDefinition.virtual( + "stale-statistics-scope", + 1, + Map.of( + "SMALL_DB", FederationSourceBindingDefinition.of( + smallSource, 1, Map.of("main", "main") + ), + "LARGE_DB", FederationSourceBindingDefinition.of( + largeSource, 1, Map.of("main", "main") + ) + ), + "SMALL_DB", + FederationExecutionPolicy.basic() + ); + Instant capturedAt = Instant.parse("2026-08-23T12:00:00Z"); + Instant expiresAt = capturedAt.minusSeconds(1); + FederationStatisticsSnapshot frozen = new FederationStatisticsSnapshot( + "stale-v1", + capturedAt, + Map.of( + new FederationStatisticsSnapshot.TableKey( + smallSource, "main", "small_table" + ), + new FederationTableStatistics( + 10, + 32, + capturedAt.minusSeconds(60), + "catalog", + Map.of(), + List.of(), + expiresAt, + FederationStatisticsStatus.COMPLETE + ), + new FederationStatisticsSnapshot.TableKey( + largeSource, "main", "large_table" + ), + new FederationTableStatistics( + 10_000, + 64, + capturedAt.minusSeconds(60), + "catalog", + Map.of(), + List.of(), + expiresAt, + FederationStatisticsStatus.COMPLETE + ) + ) + ); + CalciteFederationSqlCompiler compiler = new CalciteFederationSqlCompiler( + List.of(), + true, + FederationExecutionPolicy.basic(), + () -> frozen + ); + SqlCompileRequest request = SqlCompileRequest.of( + "SELECT * FROM SMALL_DB.main.small_table s " + + "JOIN LARGE_DB.main.large_table l ON s.id = l.id", + scope + ); + + try (FederationQueryScopeSnapshot snapshot = FederationQueryScopeSnapshot.borrowed( + scope, + Map.of("SMALL_DB", smallRuntime, "LARGE_DB", largeRuntime) + )) { + FederationSqlPlan plan = compiler.compile(request, snapshot, frozen); + + Assert.assertTrue(plan.fragments().stream().allMatch(fragment -> + fragment.costEstimate().statisticsStatus() + == FederationStatisticsStatus.STALE + )); + Assert.assertEquals(1, plan.joinOptimizations().size()); + Assert.assertEquals( + "LARGE_DB", + plan.joinOptimizations().get(0).buildBinding() + ); + } + } + + private static SqlCompletionRequest request( + FederationQueryScopeDefinition scope, + String sql + ) { + return new SqlCompletionRequest(scope, sql, sql.length()); + } + + private static boolean has( + SqlCompletionResult result, + String label, + SqlCompletionKind kind + ) { + return result.items().stream().anyMatch(item -> + item.kind() == kind && item.label().equalsIgnoreCase(label) + ); + } + + private SourceRuntime runtime( + FederationSourceDefinition definition, + SchemaPlus rootSchema, + SchemaPlus defaultSchema + ) { + SqlDialect dialect = new SqlDialect(SqlDialect.EMPTY_CONTEXT + .withDatabaseProduct(DatabaseProduct.CALCITE) + .withQuotedCasing(Casing.UNCHANGED) + .withUnquotedCasing(Casing.UNCHANGED) + .withCaseSensitive(false)); + DataSource dataSource = (DataSource) Proxy.newProxyInstance( + getClass().getClassLoader(), + new Class[] {DataSource.class}, + (proxy, method, arguments) -> { + throw new UnsupportedOperationException(method.getName()); + } + ); + SchemaPlus resolvedDefaultSchema = defaultSchema == null + ? rootSchema.getSubSchema(definition.sourceId().value()).getSubSchema("main") + : defaultSchema; + return new SourceRuntime( + definition, + FederationDataSourceHandles.shared( + dataSource, + new RuntimeFingerprint("test", "1", "test", "1", "1") + ), + new TestAdapter(dialect), + dialect, + rootSchema, + resolvedDefaultSchema, + new AdapterCompatibility( + AdapterCompatibilityStatus.VERIFIED, + "test", + "1", + "test", + "1", + "test" + ), + ignored -> { } + ); + } + + private static FederationSourceDefinition definition( + SourceId sourceId, + String tableName + ) { + return new FederationSourceDefinition( + sourceId, + 1, + "test-adapter", + List.of(new ExternalSchemaDefinition("main", "test", 1)), + Map.of("table", tableName) + ); + } + + private static SchemaPlus rootSchema(SourceId sourceId, String tableName) { + SchemaPlus root = CalciteSchema.createRootSchema(true, false).plus(); + SchemaPlus sourceRoot = root.add(sourceId.value(), new AbstractSchema()); + sourceRoot.add("main", new AbstractSchema() { + @Override + protected Map getTableMap() { + return Map.of(tableName, table("id")); + } + }); + return root; + } + + private static Schema lowerCaseSchema() { + return new AbstractSchema() { + @Override + protected Map getTableMap() { + return Map.of("orders", table("id")); + } + }; + } + + private static Table table(String columnName) { + return new AbstractTable() { + @Override + public RelDataType getRowType(RelDataTypeFactory typeFactory) { + return typeFactory.builder() + .add(columnName, SqlTypeName.INTEGER) + .build(); + } + }; + } + + private static final class TestAdapter implements FederationSqlAdapterProvider { + + private final SqlDialect dialect; + + private TestAdapter(SqlDialect dialect) { + this.dialect = dialect; + } + + @Override + public String adapterId() { + return "test-adapter"; + } + + @Override + public boolean supports(DatabaseMetaData metadata, AdapterHints hints) { + return true; + } + + @Override + public AdapterCompatibility compatibility(DatabaseMetaData metadata, AdapterHints hints) { + throw new UnsupportedOperationException(); + } + + @Override + public Schema createSchema(AdapterSchemaContext context) { + return lowerCaseSchema(); + } + + @Override + public SqlDialect createDialect(AdapterDialectContext context) { + return dialect; + } + + @Override + public FederationFragmentExecutor fragmentExecutor() { + throw new UnsupportedOperationException(); + } + } +} diff --git a/easy-agents-federation-sql/easy-agents-federation-sql-core/src/test/java/com/easyagents/federation/sql/runtime/DefaultFederationSourceManagerTest.java b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/test/java/com/easyagents/federation/sql/runtime/DefaultFederationSourceManagerTest.java new file mode 100644 index 0000000..a8396ed --- /dev/null +++ b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/test/java/com/easyagents/federation/sql/runtime/DefaultFederationSourceManagerTest.java @@ -0,0 +1,889 @@ +package com.easyagents.federation.sql.runtime; + +import com.easyagents.federation.sql.adapter.AdapterCompatibility; +import com.easyagents.federation.sql.adapter.AdapterCompatibilityStatus; +import com.easyagents.federation.sql.adapter.AdapterDialectContext; +import com.easyagents.federation.sql.adapter.AdapterHints; +import com.easyagents.federation.sql.adapter.AdapterSchemaContext; +import com.easyagents.federation.sql.adapter.FederationSqlAdapterProvider; +import com.easyagents.federation.sql.adapter.FederationSqlAdapterRegistry; +import com.easyagents.federation.sql.api.FederationSqlErrorCode; +import com.easyagents.federation.sql.api.FederationSqlException; +import com.easyagents.federation.sql.api.SqlExecutionContext; +import com.easyagents.federation.sql.compile.FederationSqlPlan; +import com.easyagents.federation.sql.compile.SqlCompileRequest; +import com.easyagents.federation.sql.compile.SqlExplainRequest; +import com.easyagents.federation.sql.execute.FederationFragmentExecutor; +import com.easyagents.federation.sql.execute.FederationQueryAdmissionController; +import com.easyagents.federation.sql.execute.FederationQueryPermit; +import com.easyagents.federation.sql.execute.FederationResultCursor; +import com.easyagents.federation.sql.execute.LocalFederationQueryAdmissionController; +import com.easyagents.federation.sql.execute.QueryAdmissionRequest; +import com.easyagents.federation.sql.execute.QueryId; +import com.easyagents.federation.sql.federation.FederationExecutionPolicy; +import com.easyagents.federation.sql.source.ExternalSchemaDefinition; +import com.easyagents.federation.sql.source.FederationDataSourceHandles; +import com.easyagents.federation.sql.source.FederationSourceDefinition; +import com.easyagents.federation.sql.source.FederationSourceState; +import com.easyagents.federation.sql.source.FederationSourceStateProvider; +import com.easyagents.federation.sql.source.PreparedSourceRuntime; +import com.easyagents.federation.sql.source.RuntimeFingerprint; +import com.easyagents.federation.sql.source.SourceApplyOptions; +import com.easyagents.federation.sql.source.SourceApplyStatus; +import com.easyagents.federation.sql.source.SourceId; +import com.easyagents.federation.sql.source.SourceStateSubscription; +import com.easyagents.federation.sql.source.SourceTombstone; +import java.io.PrintWriter; +import java.lang.reflect.Proxy; +import java.sql.Connection; +import java.sql.DatabaseMetaData; +import java.sql.SQLException; +import java.sql.Statement; +import java.time.Duration; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.Consumer; +import java.util.logging.Logger; +import javax.sql.DataSource; +import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.rel.type.RelDataTypeFactory; +import org.apache.calcite.schema.Schema; +import org.apache.calcite.schema.Table; +import org.apache.calcite.schema.impl.AbstractSchema; +import org.apache.calcite.sql.SqlDialect; +import org.apache.calcite.sql.dialect.AnsiSqlDialect; +import org.apache.calcite.sql.type.SqlTypeName; +import org.junit.Assert; +import org.junit.Test; + +/** + * 数据源 revision、single-flight 和 Runtime lease 生命周期测试。 + */ +public class DefaultFederationSourceManagerTest { + + /** + * 验证 Adapter Schema 不缓存物理元数据,确保 scope-only 接入新表后冷编译可见。 + */ + @Test + public void shouldExposeNewTablesWithoutRebuildingRuntime() { + Map tables = new HashMap<>(); + Schema mutableSchema = new AbstractSchema() { + @Override + protected Map getTableMap() { + return tables; + } + }; + DefaultFederationSourceManager manager = new DefaultFederationSourceManager( + definition -> FederationDataSourceHandles.shared( + fakeDataSource(), + new RuntimeFingerprint("fake", "1", "fake-driver", "1", "1") + ), + new FederationSqlAdapterRegistry( + List.of(new FakeAdapter(mutableSchema)), + getClass().getClassLoader() + ), + FederationSourceStateProvider.none() + ); + manager.apply(definition(1, Map.of()), SourceApplyOptions.prewarmNow()); + + try (SourceRuntime.RuntimeLease lease = manager.acquireRuntime(new SourceId("orders"), 1)) { + Assert.assertFalse(lease.runtime().defaultSchema().isCacheEnabled()); + Assert.assertNull(lease.runtime().defaultSchema().getTable("new_orders")); + tables.put("new_orders", integerTable()); + Assert.assertNotNull(lease.runtime().defaultSchema().getTable("new_orders")); + } finally { + manager.close(); + } + } + + /** + * 创建仅含整数主键的测试表。 + * + * @return Calcite 测试表 + */ + private static Table integerTable() { + return new org.apache.calcite.schema.impl.AbstractTable() { + @Override + public RelDataType getRowType(RelDataTypeFactory typeFactory) { + return typeFactory.builder().add("ID", SqlTypeName.INTEGER).build(); + } + }; + } + + /** + * 验证并发预热只解析一次 Handle,并覆盖版本冲突、切换和墓碑语义。 + * + * @throws Exception 并发测试失败 + */ + @Test + public void shouldInitializeOnceAndDrainOldRuntime() throws Exception { + AtomicInteger resolves = new AtomicInteger(); + AtomicInteger closes = new AtomicInteger(); + DataSource dataSource = fakeDataSource(); + FederationSqlAdapterRegistry adapters = new FederationSqlAdapterRegistry( + List.of(new FakeAdapter()), + getClass().getClassLoader() + ); + DefaultFederationSourceManager manager = new DefaultFederationSourceManager( + definition -> { + resolves.incrementAndGet(); + return FederationDataSourceHandles.owned( + dataSource, + new RuntimeFingerprint("fake", "1", "fake-driver", "1", "1"), + closes::incrementAndGet + ); + }, + adapters, + FederationSourceStateProvider.none() + ); + + FederationSourceDefinition revisionOne = definition(1, Map.of()); + Assert.assertEquals(SourceApplyStatus.APPLIED, manager.apply(revisionOne).status()); + Assert.assertEquals(0, resolves.get()); + + int concurrency = 32; + ExecutorService executor = Executors.newFixedThreadPool(8); + CountDownLatch start = new CountDownLatch(1); + CountDownLatch done = new CountDownLatch(concurrency); + for (int index = 0; index < concurrency; index++) { + executor.execute(() -> { + try { + start.await(); + manager.ensureReady(new SourceId("orders"), 1); + } catch (InterruptedException exception) { + Thread.currentThread().interrupt(); + } finally { + done.countDown(); + } + }); + } + start.countDown(); + Assert.assertTrue(done.await(5, TimeUnit.SECONDS)); + executor.shutdownNow(); + Assert.assertEquals(1, resolves.get()); + + Assert.assertEquals(SourceApplyStatus.IDEMPOTENT, manager.apply(revisionOne).status()); + Assert.assertEquals( + SourceApplyStatus.CONFLICT, + manager.apply(definition(1, Map.of("changed", "true"))).status() + ); + Assert.assertEquals(SourceApplyStatus.IGNORED_STALE, manager.apply(definition(0, Map.of())).status()); + + SourceRuntime.RuntimeLease oldLease = manager.acquireRuntime(new SourceId("orders"), 1); + Assert.assertEquals( + SourceApplyStatus.APPLIED, + manager.apply(definition(2, Map.of()), SourceApplyOptions.prewarmNow()).status() + ); + Assert.assertEquals(2, resolves.get()); + Assert.assertEquals(0, closes.get()); + oldLease.close(); + Assert.assertEquals(1, closes.get()); + + Assert.assertEquals( + SourceApplyStatus.APPLIED, + manager.remove(SourceTombstone.of(new SourceId("orders"), 3)).result().status() + ); + Assert.assertEquals(2, closes.get()); + Assert.assertEquals(SourceApplyStatus.IGNORED_STALE, manager.apply(definition(2, Map.of())).status()); + manager.close(); + } + + /** + * 验证新 revision 初始化失败时,满足最低 revision 的旧 Runtime 仍可服务查询。 + */ + @Test + public void shouldKeepLastReadyRuntimeWhenNewRevisionFails() { + DataSource dataSource = fakeDataSource(); + AtomicInteger closes = new AtomicInteger(); + DefaultFederationSourceManager manager = new DefaultFederationSourceManager( + definition -> { + if (definition.revision() == 2) { + throw new FederationSqlException( + FederationSqlErrorCode.SOURCE_INITIALIZATION_FAILED, + "simulated revision two failure" + ); + } + return FederationDataSourceHandles.owned( + dataSource, + new RuntimeFingerprint("fake", "1", "fake-driver", "1", "1"), + closes::incrementAndGet + ); + }, + new FederationSqlAdapterRegistry(List.of(new FakeAdapter()), getClass().getClassLoader()), + FederationSourceStateProvider.none() + ); + manager.apply(definition(1, Map.of()), SourceApplyOptions.prewarmNow()); + manager.apply(definition(2, Map.of())); + + Assert.assertEquals(1, manager.ensureReady(new SourceId("orders"), 1).readyRevision()); + Assert.assertEquals(2, manager.view(new SourceId("orders")).orElseThrow().desiredRevision()); + try { + manager.ensureReady(new SourceId("orders"), 2); + Assert.fail("minimum revision two should fail"); + } catch (FederationSqlException exception) { + Assert.assertEquals(FederationSqlErrorCode.SOURCE_INITIALIZATION_FAILED, exception.errorCode()); + } + manager.close(); + Assert.assertEquals(1, closes.get()); + } + + /** + * 验证预构建 Runtime 在 commit 前不改变共享 Slot,commit 后再原子替换旧版本。 + */ + @Test + public void shouldPrepareWithoutPublishingAndCommitAtomically() { + DataSource dataSource = fakeDataSource(); + AtomicInteger closes = new AtomicInteger(); + DefaultFederationSourceManager manager = new DefaultFederationSourceManager( + definition -> FederationDataSourceHandles.owned( + dataSource, + new RuntimeFingerprint("fake", "1", "fake-driver", "1", "1"), + closes::incrementAndGet + ), + new FederationSqlAdapterRegistry(List.of(new FakeAdapter()), getClass().getClassLoader()), + FederationSourceStateProvider.none() + ); + manager.apply(definition(1, Map.of()), SourceApplyOptions.prewarmNow()); + + try (PreparedSourceRuntime prepared = manager.prepare(definition(2, Map.of()))) { + Assert.assertEquals(1L, manager.view(new SourceId("orders")) + .orElseThrow().desiredRevision()); + Assert.assertEquals(1L, manager.view(new SourceId("orders")) + .orElseThrow().readyRevision()); + + Assert.assertEquals(SourceApplyStatus.APPLIED, manager.commit(prepared).status()); + } + + Assert.assertEquals(2L, manager.view(new SourceId("orders")) + .orElseThrow().desiredRevision()); + Assert.assertEquals(2L, manager.view(new SourceId("orders")) + .orElseThrow().readyRevision()); + Assert.assertEquals(1, closes.get()); + manager.close(); + Assert.assertEquals(2, closes.get()); + } + + /** + * 验证放弃预构建 Runtime 会关闭候选 Handle,且不改变当前版本。 + */ + @Test + public void shouldDiscardPreparedRuntimeWithoutChangingSlot() { + DataSource dataSource = fakeDataSource(); + AtomicInteger closes = new AtomicInteger(); + DefaultFederationSourceManager manager = new DefaultFederationSourceManager( + definition -> FederationDataSourceHandles.owned( + dataSource, + new RuntimeFingerprint("fake", "1", "fake-driver", "1", "1"), + closes::incrementAndGet + ), + new FederationSqlAdapterRegistry(List.of(new FakeAdapter()), getClass().getClassLoader()), + FederationSourceStateProvider.none() + ); + manager.apply(definition(1, Map.of()), SourceApplyOptions.prewarmNow()); + + try (PreparedSourceRuntime ignored = manager.prepare(definition(2, Map.of()))) { + Assert.assertEquals(0, closes.get()); + } + + Assert.assertEquals(1, closes.get()); + Assert.assertEquals(1L, manager.view(new SourceId("orders")) + .orElseThrow().desiredRevision()); + Assert.assertEquals(1L, manager.view(new SourceId("orders")) + .orElseThrow().readyRevision()); + manager.close(); + Assert.assertEquals(2, closes.get()); + } + + /** + * 验证连接池关闭连续失败后仍保留可重试引用,且成功后保持幂等。 + */ + @Test + public void shouldRetryRuntimeClosureAfterTransientFailures() { + AtomicInteger closeAttempts = new AtomicInteger(); + DefaultFederationSourceManager manager = new DefaultFederationSourceManager( + definition -> FederationDataSourceHandles.owned( + fakeDataSource(), + new RuntimeFingerprint("fake", "1", "fake-driver", "1", "1"), + () -> { + if (closeAttempts.incrementAndGet() <= 2) { + throw new IllegalStateException("simulated transient close failure"); + } + } + ), + new FederationSqlAdapterRegistry(List.of(new FakeAdapter()), getClass().getClassLoader()), + FederationSourceStateProvider.none() + ); + manager.apply(definition(1, Map.of()), SourceApplyOptions.prewarmNow()); + + try { + manager.close(); + Assert.fail("first manager close should report exhausted bounded retries"); + } catch (IllegalStateException expected) { + Assert.assertEquals(2, closeAttempts.get()); + } + + manager.close(); + Assert.assertEquals(3, closeAttempts.get()); + manager.close(); + Assert.assertEquals(3, closeAttempts.get()); + } + + /** + * 验证共享状态订阅关闭失败后,重复关闭管理器会继续释放订阅。 + */ + @Test + public void shouldRetrySubscriptionClosureAfterFailure() { + AtomicInteger closeAttempts = new AtomicInteger(); + FederationSourceStateProvider provider = new FederationSourceStateProvider() { + @Override + public Optional find(SourceId sourceId) { + return Optional.empty(); + } + + @Override + public SourceStateSubscription subscribe( + Consumer consumer + ) { + return () -> { + if (closeAttempts.incrementAndGet() == 1) { + throw new IllegalStateException( + "simulated subscription close failure" + ); + } + }; + } + }; + DefaultFederationSourceManager manager = new DefaultFederationSourceManager( + definition -> FederationDataSourceHandles.shared( + fakeDataSource(), + new RuntimeFingerprint("fake", "1", "fake-driver", "1", "1") + ), + new FederationSqlAdapterRegistry(List.of(new FakeAdapter()), getClass().getClassLoader()), + provider + ); + + try { + manager.close(); + Assert.fail("first subscription close should fail"); + } catch (IllegalStateException expected) { + Assert.assertEquals(1, closeAttempts.get()); + } + manager.close(); + manager.close(); + Assert.assertEquals(2, closeAttempts.get()); + } + + /** + * 验证注册表代次只跟随启用 SourceId 集合变化,避免无关 revision 冲掉热计划。 + */ + @Test + public void shouldAdvanceCatalogGenerationOnlyWhenActiveMembershipChanges() { + DefaultFederationSourceManager manager = new DefaultFederationSourceManager( + definition -> FederationDataSourceHandles.shared( + fakeDataSource(), + new RuntimeFingerprint("fake", "1", "fake-driver", "1", "1") + ), + new FederationSqlAdapterRegistry(List.of(new FakeAdapter()), getClass().getClassLoader()), + FederationSourceStateProvider.none() + ); + + manager.apply(definition(1, Map.of())); + SourceCatalogSnapshot first = manager.catalogSnapshot(); + Assert.assertEquals(Set.of(new SourceId("orders")), first.sourceIds()); + + manager.apply(definition(2, Map.of())); + SourceCatalogSnapshot revised = manager.catalogSnapshot(); + Assert.assertEquals(first.generation(), revised.generation()); + + manager.remove(SourceTombstone.of(new SourceId("orders"), 3)); + SourceCatalogSnapshot removed = manager.catalogSnapshot(); + Assert.assertTrue(removed.sourceIds().isEmpty()); + Assert.assertEquals(first.generation() + 1, removed.generation()); + manager.close(); + } + + /** + * 验证显式注册重复 AdapterId 会直接报告冲突。 + */ + @Test + public void shouldRejectDuplicateExplicitAdapterIds() { + try { + new FederationSqlAdapterRegistry( + List.of(new FakeAdapter(), new FakeAdapter()), + getClass().getClassLoader() + ); + Assert.fail("duplicate adapter ids should be rejected"); + } catch (FederationSqlException exception) { + Assert.assertEquals(FederationSqlErrorCode.SOURCE_DEFINITION_CONFLICT, exception.errorCode()); + } + } + + /** + * 验证关闭 Engine 会及时取消已登记查询,且不会等待阻塞的 Adapter 执行返回。 + * + * @throws Exception 并发测试失败 + */ + @Test + public void shouldCancelBlockingQueryDuringEngineClose() throws Exception { + CountDownLatch registered = new CountDownLatch(1); + CountDownLatch cancelled = new CountDownLatch(1); + AtomicInteger cancelCalls = new AtomicInteger(); + Statement statement = (Statement) Proxy.newProxyInstance( + getClass().getClassLoader(), + new Class[] {Statement.class}, + (proxy, method, args) -> { + if ("cancel".equals(method.getName())) { + cancelCalls.incrementAndGet(); + cancelled.countDown(); + return null; + } + return defaultValue(method.getReturnType()); + } + ); + FederationFragmentExecutor blockingExecutor = context -> { + context.statementLifecycle().register(statement); + registered.countDown(); + try { + if (!cancelled.await(5, TimeUnit.SECONDS)) { + throw new FederationSqlException( + FederationSqlErrorCode.EXECUTION_FAILED, + "test query was not cancelled" + ); + } + throw new FederationSqlException( + context.statementLifecycle().cancellationRequested() + ? FederationSqlErrorCode.QUERY_CANCELLED + : FederationSqlErrorCode.EXECUTION_FAILED, + "test query stopped" + ); + } catch (InterruptedException exception) { + Thread.currentThread().interrupt(); + throw new FederationSqlException( + FederationSqlErrorCode.EXECUTION_FAILED, + "test query was interrupted", + exception + ); + } finally { + context.statementLifecycle().unregister(statement); + } + }; + DefaultFederationSourceManager manager = new DefaultFederationSourceManager( + definition -> FederationDataSourceHandles.shared( + fakeDataSource(), + new RuntimeFingerprint("fake", "1", "fake-driver", "1", "1") + ), + new FederationSqlAdapterRegistry( + List.of(new FakeAdapter(blockingExecutor)), + getClass().getClassLoader() + ), + FederationSourceStateProvider.none() + ); + DefaultFederationSqlEngine engine = new DefaultFederationSqlEngine( + manager, + new LocalFederationQueryAdmissionController(4), + List.of(), + 16, + false + ); + manager.apply(definition(1, Map.of()), SourceApplyOptions.prewarmNow()); + FederationSqlPlan plan = engine.compile(SqlCompileRequest.of( + "VALUES (1)", + new SourceId("orders"), + 1 + )); + + ExecutorService executor = Executors.newFixedThreadPool(2); + try { + Future query = executor.submit(() -> { + try { + engine.execute(plan, SqlExecutionContext.of(List.of())); + return null; + } catch (FederationSqlException exception) { + return exception.errorCode(); + } + }); + Assert.assertTrue(registered.await(2, TimeUnit.SECONDS)); + Future closing = executor.submit(engine::close); + closing.get(2, TimeUnit.SECONDS); + Assert.assertEquals(1, cancelCalls.get()); + Assert.assertEquals(FederationSqlErrorCode.QUERY_CANCELLED, query.get(2, TimeUnit.SECONDS)); + } finally { + executor.shutdownNow(); + engine.close(); + } + } + + /** + * 验证活动 JDBC 游标永久阻塞关闭时,Engine 关闭仍保持有界并暴露未完成清理。 + * + * @throws Exception 并发测试失败 + */ + @Test + public void shouldCloseEngineWithinBoundWhenCursorCloseBlocks() throws Exception { + CountDownLatch closeStarted = new CountDownLatch(1); + CountDownLatch releaseClose = new CountDownLatch(1); + CountDownLatch closeFinished = new CountDownLatch(1); + FederationFragmentExecutor blockingCursorExecutor = context -> new FederationResultCursor() { + @Override + public QueryId queryId() { + return context.queryId(); + } + + @Override + public List columns() { + return List.of(); + } + + @Override + public boolean next() { + return false; + } + + @Override + public Object getObject(int columnIndex) { + return null; + } + + @Override + public List row() { + return List.of(); + } + + @Override + public void close() { + closeStarted.countDown(); + try { + while (releaseClose.getCount() > 0L) { + try { + releaseClose.await(); + } catch (InterruptedException ignored) { + // 模拟忽略线程中断且永久阻塞 close() 的 JDBC Driver。 + } + } + } finally { + closeFinished.countDown(); + } + } + }; + Schema tableSchema = new AbstractSchema() { + @Override + protected Map getTableMap() { + return Map.of("ORDERS", integerTable()); + } + }; + DefaultFederationSourceManager manager = sourceManager( + new FakeAdapter(blockingCursorExecutor, tableSchema) + ); + DefaultFederationSqlEngine engine = new DefaultFederationSqlEngine( + manager, + new LocalFederationQueryAdmissionController(4), + List.of(), + 16, + false + ); + manager.apply(definition(1, Map.of()), SourceApplyOptions.prewarmNow()); + FederationSqlPlan plan = engine.compile(SqlCompileRequest.of( + "SELECT id FROM orders", + new SourceId("orders"), + 1 + )); + FederationResultCursor cursor = engine.execute( + plan, + SqlExecutionContext.of(List.of()) + ); + ExecutorService executor = Executors.newSingleThreadExecutor(); + try { + Future closing = executor.submit(engine::close); + Assert.assertTrue(closeStarted.await(2, TimeUnit.SECONDS)); + closing.get(2, TimeUnit.SECONDS); + Assert.assertTrue(engine.cleanupMetrics().unresolvedCleanups() > 0L); + } finally { + releaseClose.countDown(); + Assert.assertTrue(closeFinished.await(2, TimeUnit.SECONDS)); + cursor.close(); + executor.shutdownNow(); + engine.close(); + } + } + + /** + * 验证 Explain 准入等待受统一编译截止时间约束,并返回精确超时错误。 + */ + @Test + public void shouldApplyUnifiedDeadlineToExplainAdmission() { + CountDownLatch admissionStarted = new CountDownLatch(1); + FederationQueryAdmissionController blockingAdmission = + new FederationQueryAdmissionController() { + @Override + public FederationQueryPermit acquire( + SourceId sourceId, + QueryId queryId, + Duration timeout + ) { + throw new AssertionError("query-level admission request is required"); + } + + @Override + public FederationQueryPermit acquire(QueryAdmissionRequest request) { + admissionStarted.countDown(); + while (!request.cancellationRequested().getAsBoolean()) { + try { + Thread.sleep(5L); + } catch (InterruptedException exception) { + Thread.currentThread().interrupt(); + throw new FederationSqlException( + FederationSqlErrorCode.QUERY_ADMISSION_TIMEOUT, + "Explain admission was interrupted", + exception + ); + } + } + throw new FederationSqlException( + FederationSqlErrorCode.QUERY_ADMISSION_TIMEOUT, + "Explain admission exceeded its deadline" + ); + } + }; + Schema tableSchema = new AbstractSchema() { + @Override + protected Map getTableMap() { + return Map.of("ORDERS", integerTable()); + } + }; + DefaultFederationSourceManager manager = sourceManager(new FakeAdapter(tableSchema)); + DefaultFederationSqlEngine engine = new DefaultFederationSqlEngine( + manager, + blockingAdmission, + List.of(), + 16, + 4, + false, + new FederationExecutionPolicy( + 2, + 8, + 2, + 100_000, + 64L * 1024L * 1024L, + 1_000 + ) + ); + manager.apply(definition(1, Map.of()), SourceApplyOptions.prewarmNow()); + long startedAt = System.nanoTime(); + try { + engine.explain(new SqlExplainRequest(SqlCompileRequest.of( + "SELECT id FROM orders", + new SourceId("orders"), + 1 + ))); + Assert.fail("Explain admission should respect the unified deadline"); + } catch (FederationSqlException exception) { + Assert.assertEquals( + FederationSqlErrorCode.SQL_COMPILE_TIMEOUT, + exception.errorCode() + ); + Assert.assertEquals(0L, admissionStarted.getCount()); + long elapsedMillis = TimeUnit.NANOSECONDS.toMillis( + System.nanoTime() - startedAt + ); + Assert.assertTrue("Explain took " + elapsedMillis + "ms", elapsedMillis < 2_000L); + } finally { + engine.close(); + } + } + + private DefaultFederationSourceManager sourceManager(FakeAdapter adapter) { + return new DefaultFederationSourceManager( + definition -> FederationDataSourceHandles.shared( + fakeDataSource(), + new RuntimeFingerprint("fake", "1", "fake-driver", "1", "1") + ), + new FederationSqlAdapterRegistry( + List.of(adapter), + getClass().getClassLoader() + ), + FederationSourceStateProvider.none() + ); + } + + private static FederationSourceDefinition definition(long revision, Map options) { + return new FederationSourceDefinition( + new SourceId("orders"), + revision, + "fake", + List.of(new ExternalSchemaDefinition("app", "snapshot", revision)), + options + ); + } + + private static DataSource fakeDataSource() { + DatabaseMetaData metadata = (DatabaseMetaData) Proxy.newProxyInstance( + DefaultFederationSourceManagerTest.class.getClassLoader(), + new Class[] {DatabaseMetaData.class}, + (proxy, method, args) -> switch (method.getName()) { + case "getDatabaseProductName" -> "fake"; + case "getDatabaseProductVersion" -> "1"; + case "getDriverName" -> "fake-driver"; + case "getDriverVersion" -> "1"; + default -> defaultValue(method.getReturnType()); + } + ); + Connection connection = (Connection) Proxy.newProxyInstance( + DefaultFederationSourceManagerTest.class.getClassLoader(), + new Class[] {Connection.class}, + (proxy, method, args) -> switch (method.getName()) { + case "getMetaData" -> metadata; + case "close" -> null; + case "isClosed" -> false; + default -> defaultValue(method.getReturnType()); + } + ); + return new DataSource() { + @Override + public Connection getConnection() { + return connection; + } + + @Override + public Connection getConnection(String username, String password) { + return connection; + } + + @Override + public T unwrap(Class iface) throws SQLException { + throw new SQLException("not a wrapper"); + } + + @Override + public boolean isWrapperFor(Class iface) { + return false; + } + + @Override + public PrintWriter getLogWriter() { + return null; + } + + @Override + public void setLogWriter(PrintWriter out) { + } + + @Override + public void setLoginTimeout(int seconds) { + } + + @Override + public int getLoginTimeout() { + return 0; + } + + @Override + public Logger getParentLogger() { + return Logger.getGlobal(); + } + }; + } + + private static Object defaultValue(Class type) { + if (!type.isPrimitive()) { + return null; + } + if (type == boolean.class) { + return false; + } + if (type == byte.class) { + return (byte) 0; + } + if (type == short.class) { + return (short) 0; + } + if (type == int.class) { + return 0; + } + if (type == long.class) { + return 0L; + } + if (type == float.class) { + return 0F; + } + if (type == double.class) { + return 0D; + } + if (type == char.class) { + return '\0'; + } + return null; + } + + private static final class FakeAdapter implements FederationSqlAdapterProvider { + + private final FederationFragmentExecutor fragmentExecutor; + private final Schema schema; + + private FakeAdapter() { + this(context -> { + throw new UnsupportedOperationException("not required by lifecycle test"); + }, new AbstractSchema()); + } + + private FakeAdapter(Schema schema) { + this(context -> { + throw new UnsupportedOperationException("not required by lifecycle test"); + }, schema); + } + + private FakeAdapter(FederationFragmentExecutor fragmentExecutor) { + this(fragmentExecutor, new AbstractSchema()); + } + + private FakeAdapter(FederationFragmentExecutor fragmentExecutor, Schema schema) { + this.fragmentExecutor = fragmentExecutor; + this.schema = schema; + } + + @Override + public String adapterId() { + return "fake"; + } + + @Override + public boolean supports(DatabaseMetaData metadata, AdapterHints hints) { + return true; + } + + @Override + public AdapterCompatibility compatibility(DatabaseMetaData metadata, AdapterHints hints) { + return new AdapterCompatibility( + AdapterCompatibilityStatus.VERIFIED, + "fake", + "1", + "fake-driver", + "1", + "test adapter" + ); + } + + @Override + public Schema createSchema(AdapterSchemaContext context) { + return schema; + } + + @Override + public SqlDialect createDialect(AdapterDialectContext context) { + return AnsiSqlDialect.DEFAULT; + } + + @Override + public FederationFragmentExecutor fragmentExecutor() { + return fragmentExecutor; + } + } +} diff --git a/easy-agents-federation-sql/easy-agents-federation-sql-core/src/test/java/com/easyagents/federation/sql/runtime/FederationQueryMetricsTrackerTest.java b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/test/java/com/easyagents/federation/sql/runtime/FederationQueryMetricsTrackerTest.java new file mode 100644 index 0000000..c935a00 --- /dev/null +++ b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/test/java/com/easyagents/federation/sql/runtime/FederationQueryMetricsTrackerTest.java @@ -0,0 +1,93 @@ +package com.easyagents.federation.sql.runtime; + +import com.easyagents.federation.sql.api.FederationSqlErrorCode; +import com.easyagents.federation.sql.api.FederationSqlException; +import org.apache.calcite.avatica.util.ByteString; +import com.easyagents.federation.sql.execute.FederationQueryMetricsSnapshot; +import com.easyagents.federation.sql.execute.QueryId; +import com.easyagents.federation.sql.federation.FederationFragmentPlan; +import com.easyagents.federation.sql.federation.FederationQueryMode; +import com.easyagents.federation.sql.source.SourceId; +import java.util.List; +import org.junit.Assert; +import org.junit.Test; + +/** + * 联邦查询指标估算边界测试。 + */ +public class FederationQueryMetricsTrackerTest { + + /** + * 验证 Calcite 二进制值按实际长度计入中间字节预算。 + */ + @Test + public void shouldMeasureCalciteByteStringByActualLength() { + long small = FederationQueryMetricsTracker.estimateRowBytes( + new Object[] {new ByteString(new byte[8])} + ); + long large = FederationQueryMetricsTracker.estimateRowBytes( + new Object[] {new ByteString(new byte[4096])} + ); + + Assert.assertEquals(4096 - 8, large - small); + } + + /** + * 验证无法保守计量的专有对象不会按固定小值放过预算。 + */ + @Test + public void shouldRejectUnmeasurableIntermediateValue() { + try { + FederationQueryMetricsTracker.estimateRowBytes(new Object[] {new Object()}); + Assert.fail("unknown values must fail closed"); + } catch (FederationSqlException exception) { + Assert.assertEquals( + FederationSqlErrorCode.FEDERATION_OPERATOR_UNSUPPORTED, + exception.errorCode() + ); + } + } + + /** + * 验证查询、分片和本地算子阶段指标会在终态冻结。 + */ + @Test + public void shouldPublishDetailedTerminalMetrics() { + FederationQueryMetricsTracker tracker = new FederationQueryMetricsTracker( + new QueryId("metrics-query"), + FederationQueryMode.FEDERATED, + true, + 10, + List.of(new FederationFragmentPlan( + "fragment-1", + "SOURCE", + new SourceId("source"), + "SELECT 1", + List.of(), + List.of() + )) + ); + tracker.recordAdmissionWait(11); + tracker.fragmentObserver("fragment-1").connectionAcquired(12); + tracker.fragmentObserver("fragment-1").databaseExecutionCompleted(13); + tracker.recordIntermediate("fragment-1", new Object[] {1}); + tracker.recordLocalIntermediate("EnumerableHashJoin", new Object[] {1}, 14); + tracker.recordOutput(new Object[] {1}); + tracker.finishFragment("fragment-1"); + tracker.finishSuccessfully(); + + FederationQueryMetricsSnapshot snapshot = tracker.snapshot(); + long frozenExecution = snapshot.executionNanos(); + Assert.assertTrue(snapshot.complete()); + Assert.assertTrue(snapshot.planCacheHit()); + Assert.assertEquals(11, snapshot.admissionWaitNanos()); + Assert.assertEquals(12, snapshot.connectionAcquireNanos()); + Assert.assertEquals(13, snapshot.databaseExecutionNanos()); + Assert.assertEquals(14, snapshot.localExecutionNanos()); + Assert.assertEquals(1, snapshot.localOperators().size()); + Assert.assertEquals(1, snapshot.fragments().size()); + Assert.assertEquals(12, snapshot.fragments().get(0).connectionAcquireNanos()); + Assert.assertEquals(13, snapshot.fragments().get(0).databaseExecutionNanos()); + Assert.assertEquals(frozenExecution, tracker.snapshot().executionNanos()); + } +} diff --git a/easy-agents-federation-sql/easy-agents-federation-sql-core/src/test/java/com/easyagents/federation/sql/runtime/LogicalTableSqlResolverTest.java b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/test/java/com/easyagents/federation/sql/runtime/LogicalTableSqlResolverTest.java new file mode 100644 index 0000000..682dcdc --- /dev/null +++ b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/test/java/com/easyagents/federation/sql/runtime/LogicalTableSqlResolverTest.java @@ -0,0 +1,114 @@ +package com.easyagents.federation.sql.runtime; + +import com.easyagents.federation.sql.federation.FederationExecutionPolicy; +import com.easyagents.federation.sql.federation.FederationLogicalTableDefinition; +import com.easyagents.federation.sql.federation.FederationQueryScopeDefinition; +import com.easyagents.federation.sql.federation.FederationSourceBindingDefinition; +import com.easyagents.federation.sql.source.SourceId; +import java.util.List; +import java.util.Map; +import org.apache.calcite.config.Lex; +import org.apache.calcite.sql.SqlNode; +import org.apache.calcite.sql.parser.SqlParseException; +import org.apache.calcite.sql.parser.SqlParser; +import org.junit.Assert; +import org.junit.Test; + +/** + * 逻辑表到物理 Binding 表路径的 Calcite AST 解析测试。 + */ +public class LogicalTableSqlResolverTest { + + /** + * 验证短逻辑表名保留为查询限定别名,并指向实际物理表。 + */ + @Test + public void shouldResolveShortNameAndPreserveLogicalQualifier() { + String sql = resolve("SELECT outlet.ID FROM outlet"); + + Assert.assertTrue(sql, sql.contains("MYSQL_1.MAIN.physical_outlet AS OUTLET")); + Assert.assertTrue(sql, sql.toUpperCase().contains("OUTLET.ID")); + } + + /** + * 验证三段逻辑表名与四段列限定名映射到同一物理表。 + */ + @Test + public void shouldResolveQualifiedLogicalTableName() { + String sql = resolve( + "SELECT MYSQL_1.MAIN.outlet.ID FROM MYSQL_1.MAIN.outlet" + ); + + Assert.assertTrue(sql, sql.contains("MYSQL_1.MAIN.physical_outlet AS OUTLET")); + Assert.assertTrue(sql, sql.toUpperCase().contains("OUTLET.ID")); + } + + /** + * 验证用户显式 SQL 别名不会与内置逻辑别名形成嵌套 AS。 + */ + @Test + public void shouldPreserveExplicitAlias() { + String sql = resolve("SELECT o.ID FROM outlet AS o"); + + Assert.assertTrue( + sql, + sql.toUpperCase().contains("MYSQL_1.MAIN.PHYSICAL_OUTLET AS O") + ); + Assert.assertFalse(sql, sql.contains("AS outlet AS o")); + } + + /** + * 验证逻辑表映射变化会使查询范围 checksum 失效。 + */ + @Test + public void shouldIncludeLogicalTablesInScopeChecksum() { + FederationQueryScopeDefinition original = scope("physical_outlet"); + FederationQueryScopeDefinition renamedPhysical = scope("physical_outlet_v2"); + + Assert.assertNotEquals(original.checksum(), renamedPhysical.checksum()); + } + + /** + * 解析并重写测试 SQL。 + * + * @param sql 测试 SQL + * @return 重写后的 SQL + */ + private String resolve(String sql) { + try { + SqlNode parsed = SqlParser.create( + sql, + SqlParser.config().withLex(Lex.ORACLE) + ).parseQuery(); + SqlNode resolved = LogicalTableSqlResolver.resolve(parsed, scope("physical_outlet")); + return resolved.toSqlString(config -> config.withQuoteAllIdentifiers(false)).getSql(); + } catch (SqlParseException exception) { + throw new AssertionError(exception); + } + } + + /** + * 创建包含单张逻辑表的测试查询范围。 + * + * @param sourceTableName 物理表名 + * @return 测试查询范围 + */ + private FederationQueryScopeDefinition scope(String sourceTableName) { + return FederationQueryScopeDefinition.virtual( + "logical-table-test", + 1, + Map.of( + "MYSQL_1", + FederationSourceBindingDefinition.of(new SourceId("mysql"), 1) + ), + "MYSQL_1", + List.of(FederationLogicalTableDefinition.of( + "outlet", + "MYSQL_1", + "MAIN", + sourceTableName + )), + FederationExecutionPolicy.basic() + ); + } +} diff --git a/easy-agents-federation-sql/easy-agents-federation-sql-core/src/test/java/com/easyagents/federation/sql/runtime/ManagedFederationResultCursorTest.java b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/test/java/com/easyagents/federation/sql/runtime/ManagedFederationResultCursorTest.java new file mode 100644 index 0000000..f04cf32 --- /dev/null +++ b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/test/java/com/easyagents/federation/sql/runtime/ManagedFederationResultCursorTest.java @@ -0,0 +1,164 @@ +package com.easyagents.federation.sql.runtime; + +import com.easyagents.federation.sql.api.FederationSqlErrorCode; +import com.easyagents.federation.sql.execute.FederationColumn; +import com.easyagents.federation.sql.execute.FederationResultCursor; +import com.easyagents.federation.sql.execute.QueryId; +import java.io.IOException; +import java.io.InputStream; +import java.io.Reader; +import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; +import org.junit.Assert; +import org.junit.Test; + +/** + * Core 托管游标的大字段流异常资源释放测试。 + */ +public class ManagedFederationResultCursorTest { + + /** + * 验证二进制流在获取后的读取失败会释放 Adapter 与 Core 资源。 + * + * @throws Exception 测试流读取失败 + */ + @Test + public void shouldReleaseResourcesWhenBinaryStreamReadFails() throws Exception { + assertStreamFailure(true); + } + + /** + * 验证字符流在获取后的读取失败会释放 Adapter 与 Core 资源。 + * + * @throws Exception 测试流读取失败 + */ + @Test + public void shouldReleaseResourcesWhenCharacterStreamReadFails() throws Exception { + assertStreamFailure(false); + } + + /** + * 验证关闭游标与取消并发时不会为复用的 QueryId 留下伪预取消状态。 + */ + @Test + public void shouldRemoveCursorTrackingBeforeReleasingRegistration() { + QueryCancellationRegistry registry = new QueryCancellationRegistry(); + QueryId queryId = new QueryId("reusable-query-id"); + QueryCancellationRegistry.QueryRegistration registration = registry.begin(queryId); + ManagedFederationResultCursor cursor = new ManagedFederationResultCursor( + new TestCursor(), + registration, + () -> Assert.assertTrue(registry.cancel(queryId)) + ); + + cursor.close(); + + QueryCancellationRegistry.QueryRegistration reused = registry.begin(queryId); + reused.close(); + registry.close(); + } + + private static void assertStreamFailure(boolean binary) throws Exception { + TestCursor delegate = new TestCursor(); + AtomicInteger resourceCloses = new AtomicInteger(); + AtomicInteger trackingCloses = new AtomicInteger(); + ManagedFederationResultCursor cursor = new ManagedFederationResultCursor( + delegate, + resourceCloses::incrementAndGet, + trackingCloses::incrementAndGet + ); + + try { + if (binary) { + cursor.getBinaryStream(1).read(); + } else { + cursor.getCharacterStream(1).read(); + } + Assert.fail("LOB stream read should fail"); + } catch (IOException exception) { + Assert.assertEquals("stream failed", exception.getMessage()); + } + + Assert.assertEquals(1, delegate.closeCalls.get()); + Assert.assertEquals(1, resourceCloses.get()); + Assert.assertEquals(1, trackingCloses.get()); + Assert.assertEquals(FederationSqlErrorCode.EXECUTION_FAILED, delegate.failure.get()); + cursor.close(); + Assert.assertEquals(1, delegate.closeCalls.get()); + } + + private static final class TestCursor + implements FederationResultCursor, CancellationAwareFederationCursor { + + private final AtomicInteger closeCalls = new AtomicInteger(); + private final AtomicReference failure = new AtomicReference<>(); + + @Override + public QueryId queryId() { + return new QueryId("lob-stream-test"); + } + + @Override + public List columns() { + return List.of(); + } + + @Override + public boolean next() { + return false; + } + + @Override + public Object getObject(int columnIndex) { + return null; + } + + @Override + public InputStream getBinaryStream(int columnIndex) { + return new InputStream() { + @Override + public int read() throws IOException { + throw new IOException("stream failed"); + } + }; + } + + @Override + public Reader getCharacterStream(int columnIndex) { + return new Reader() { + @Override + public int read(char[] characters, int offset, int length) throws IOException { + throw new IOException("stream failed"); + } + + @Override + public void close() { + } + }; + } + + @Override + public List row() { + return List.of(); + } + + @Override + public void close() { + closeCalls.incrementAndGet(); + } + + @Override + public void markCancelled() { + } + + @Override + public void markTimedOut() { + } + + @Override + public void markFailed(FederationSqlErrorCode errorCode) { + failure.set(errorCode); + } + } +} diff --git a/easy-agents-federation-sql/easy-agents-federation-sql-core/src/test/java/com/easyagents/federation/sql/runtime/NodeMemoryAdmissionControllerTest.java b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/test/java/com/easyagents/federation/sql/runtime/NodeMemoryAdmissionControllerTest.java new file mode 100644 index 0000000..99535d5 --- /dev/null +++ b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/test/java/com/easyagents/federation/sql/runtime/NodeMemoryAdmissionControllerTest.java @@ -0,0 +1,180 @@ +package com.easyagents.federation.sql.runtime; + +import com.easyagents.federation.sql.api.FederationSqlErrorCode; +import com.easyagents.federation.sql.api.FederationSqlException; +import java.time.Duration; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; +import org.junit.Assert; +import org.junit.Test; + +/** + * 节点联邦内存准入的并发与释放契约测试。 + */ +public class NodeMemoryAdmissionControllerTest { + + /** + * 验证单个查询预算超过节点上限时立即拒绝,不进入等待队列。 + */ + @Test + public void shouldRejectRequestLargerThanNodeBudget() { + NodeMemoryAdmissionController controller = new NodeMemoryAdmissionController(10L); + + FederationSqlException failure = expectFailure(() -> controller.acquire( + 11L, + QueryDeadline.compile(Duration.ofSeconds(1)) + )); + + Assert.assertEquals( + FederationSqlErrorCode.FEDERATION_RESOURCE_LIMIT_EXCEEDED, + failure.errorCode() + ); + } + + /** + * 验证聚合预留达到节点上限后,等待者受同一个绝对截止时间约束。 + */ + @Test + public void shouldRespectDeadlineWhileWaitingForAggregateBudget() { + NodeMemoryAdmissionController controller = new NodeMemoryAdmissionController(10L); + try (NodeMemoryAdmissionController.Permit ignored = controller.acquire( + 6L, + QueryDeadline.compile(Duration.ofSeconds(1)) + )) { + FederationSqlException failure = expectFailure(() -> controller.acquire( + 5L, + QueryDeadline.compile(Duration.ofMillis(20)) + )); + + Assert.assertEquals( + FederationSqlErrorCode.NODE_MEMORY_ADMISSION_TIMEOUT, + failure.errorCode() + ); + } + } + + /** + * 验证准入等待收到取消时保留 QUERY_CANCELLED,不误报资源等待超时。 + */ + @Test + public void shouldPreserveCancellationDuringAdmissionWait() { + NodeMemoryAdmissionController controller = new NodeMemoryAdmissionController(10L); + QueryCancellationRegistry registry = new QueryCancellationRegistry(); + var registration = registry.begin(new com.easyagents.federation.sql.execute.QueryId( + "memory-admission-cancel" + )); + try (NodeMemoryAdmissionController.Permit ignored = controller.acquire( + 10L, + QueryDeadline.compile(Duration.ofSeconds(1)) + )) { + registry.cancel(new com.easyagents.federation.sql.execute.QueryId( + "memory-admission-cancel" + )); + FederationSqlException failure = expectFailure(() -> controller.acquire( + 1L, + QueryDeadline.query( + QueryDeadline.deadlineAfter(Duration.ofSeconds(1)), + registration + ) + )); + + Assert.assertEquals(FederationSqlErrorCode.QUERY_CANCELLED, failure.errorCode()); + } finally { + registration.close(); + registry.close(); + } + } + + /** + * 验证重复关闭不会增加许可,等待者只在真实预算释放后获得准入。 + * + * @throws Exception 并发测试失败 + */ + @Test + public void shouldReleaseMemoryPermitExactlyOnce() throws Exception { + NodeMemoryAdmissionController controller = new NodeMemoryAdmissionController(10L); + ExecutorService executor = Executors.newSingleThreadExecutor(); + try { + NodeMemoryAdmissionController.Permit first = controller.acquire( + 10L, + QueryDeadline.compile(Duration.ofSeconds(2)) + ); + first.close(); + first.close(); + + NodeMemoryAdmissionController.Permit second = controller.acquire( + 10L, + QueryDeadline.compile(Duration.ofSeconds(2)) + ); + Future waiting = executor.submit(() -> + controller.acquire(10L, QueryDeadline.compile(Duration.ofSeconds(2))) + ); + + TimeUnit.MILLISECONDS.sleep(75L); + Assert.assertFalse(waiting.isDone()); + second.close(); + + NodeMemoryAdmissionController.Permit third = waiting.get(1, TimeUnit.SECONDS); + third.close(); + } finally { + executor.shutdownNow(); + } + } + + /** + * 验证线程中断与资源等待超时采用不同错误码,并保留线程中断标记。 + * + * @throws Exception 并发测试失败 + */ + @Test + public void shouldReportExecutionFailureWhenAdmissionWaitIsInterrupted() throws Exception { + NodeMemoryAdmissionController controller = new NodeMemoryAdmissionController(10L); + AtomicReference failure = new AtomicReference<>(); + AtomicReference interrupted = new AtomicReference<>(false); + Thread waiter; + try (NodeMemoryAdmissionController.Permit ignored = controller.acquire( + 10L, + QueryDeadline.compile(Duration.ofSeconds(2)) + )) { + waiter = new Thread(() -> { + try { + controller.acquire(1L, QueryDeadline.compile(Duration.ofSeconds(2))); + } catch (FederationSqlException exception) { + failure.set(exception); + interrupted.set(Thread.currentThread().isInterrupted()); + } + }); + waiter.start(); + TimeUnit.MILLISECONDS.sleep(50L); + waiter.interrupt(); + waiter.join(1_000L); + } + + Assert.assertFalse(waiter.isAlive()); + Assert.assertNotNull(failure.get()); + Assert.assertEquals( + FederationSqlErrorCode.EXECUTION_FAILED, + failure.get().errorCode() + ); + Assert.assertTrue(interrupted.get()); + } + + /** + * 执行预期失败的内存准入动作。 + * + * @param action 测试动作 + * @return 捕获的统一异常 + */ + private static FederationSqlException expectFailure(Runnable action) { + try { + action.run(); + Assert.fail("expected memory admission to fail"); + return null; + } catch (FederationSqlException exception) { + return exception; + } + } +} diff --git a/easy-agents-federation-sql/easy-agents-federation-sql-core/src/test/java/com/easyagents/federation/sql/runtime/QueryCancellationRegistryTest.java b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/test/java/com/easyagents/federation/sql/runtime/QueryCancellationRegistryTest.java new file mode 100644 index 0000000..74dbf82 --- /dev/null +++ b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/test/java/com/easyagents/federation/sql/runtime/QueryCancellationRegistryTest.java @@ -0,0 +1,488 @@ +package com.easyagents.federation.sql.runtime; + +import com.easyagents.federation.sql.api.FederationSqlErrorCode; +import com.easyagents.federation.sql.api.FederationSqlException; +import com.easyagents.federation.sql.execute.QueryId; +import com.easyagents.federation.sql.execute.StatementLifecycle; +import java.lang.reflect.Proxy; +import java.sql.Statement; +import java.util.concurrent.ArrayBlockingQueue; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import org.junit.Assert; +import org.junit.Test; + +/** + * 查询全生命周期取消登记与终态保留测试。 + */ +public class QueryCancellationRegistryTest { + + /** + * 验证 Statement 注销和 QueryId 清理后,消费线程仍能读取取消终态。 + */ + @Test + public void shouldRetainCancellationStateForConcurrentConsumers() throws Exception { + QueryCancellationRegistry registry = new QueryCancellationRegistry(); + QueryId queryId = new QueryId("cancelled-query"); + QueryCancellationRegistry.QueryRegistration registration = registry.begin(queryId); + StatementLifecycle lifecycle = registration.statementLifecycle(); + AtomicInteger cancelCalls = new AtomicInteger(); + AtomicInteger closeCalls = new AtomicInteger(); + Statement statement = statement(cancelCalls, closeCalls); + + lifecycle.register(statement); + Assert.assertTrue(registry.cancel(queryId)); + lifecycle.unregister(statement); + registration.close(); + + awaitTermination(cancelCalls, closeCalls, 1); + Assert.assertTrue(lifecycle.cancellationRequested()); + Assert.assertFalse(registry.cancel(queryId)); + registry.close(); + } + + /** + * 验证在 Statement 登记前发出的取消会阻止后续 JDBC 执行。 + */ + @Test + public void shouldRejectStatementRegisteredAfterPendingCancellation() { + QueryCancellationRegistry registry = new QueryCancellationRegistry(); + QueryId queryId = new QueryId("pending-cancel"); + QueryCancellationRegistry.QueryRegistration registration = registry.begin(queryId); + Assert.assertTrue(registry.cancel(queryId)); + + try { + registration.statementLifecycle().register(statement( + new AtomicInteger(), + new AtomicInteger() + )); + Assert.fail("cancelled query should reject a late Statement"); + } catch (FederationSqlException exception) { + Assert.assertEquals(FederationSqlErrorCode.QUERY_CANCELLED, exception.errorCode()); + } finally { + registration.close(); + registry.close(); + } + } + + /** + * 验证 Core 登记前到达的取消不会被随后创建的 QueryState 覆盖。 + */ + @Test + public void shouldRejectQueryBegunAfterPreCancellation() { + QueryCancellationRegistry registry = new QueryCancellationRegistry(); + QueryId queryId = new QueryId("pre-cancelled-query"); + + Assert.assertFalse(registry.cancel(queryId)); + try { + registry.begin(queryId); + Assert.fail("pre-cancelled query should not begin"); + } catch (FederationSqlException exception) { + Assert.assertEquals( + FederationSqlErrorCode.QUERY_CANCELLED, + exception.errorCode() + ); + } finally { + registry.close(); + } + } + + /** + * 验证联邦查询登记的多个分片 Statement 会被同一次取消全部关闭。 + */ + @Test + public void shouldCancelEveryRegisteredFragmentStatement() throws Exception { + QueryCancellationRegistry registry = new QueryCancellationRegistry(); + QueryId queryId = new QueryId("federated-cancel"); + QueryCancellationRegistry.QueryRegistration registration = registry.begin(queryId); + AtomicInteger firstCancel = new AtomicInteger(); + AtomicInteger firstClose = new AtomicInteger(); + AtomicInteger secondCancel = new AtomicInteger(); + AtomicInteger secondClose = new AtomicInteger(); + registration.statementLifecycle().register(statement(firstCancel, firstClose)); + registration.statementLifecycle().register(statement(secondCancel, secondClose)); + + Assert.assertTrue(registry.cancel(queryId)); + awaitTermination(firstCancel, firstClose, 1); + awaitTermination(secondCancel, secondClose, 1); + + registration.close(); + registry.close(); + } + + /** + * 验证统一执行时限关闭全部 Statement,并保留 QUERY_TIMEOUT 终态。 + */ + @Test + public void shouldRetainTimeoutReasonWhileTerminatingStatements() throws Exception { + QueryCancellationRegistry registry = new QueryCancellationRegistry(); + QueryId queryId = new QueryId("timed-out-query"); + QueryCancellationRegistry.QueryRegistration registration = registry.begin(queryId); + AtomicInteger cancelCalls = new AtomicInteger(); + AtomicInteger closeCalls = new AtomicInteger(); + StatementLifecycle lifecycle = registration.statementLifecycle(); + lifecycle.register(statement(cancelCalls, closeCalls)); + + registration.requestTimeout(); + + Assert.assertTrue(lifecycle.timeoutRequested()); + Assert.assertTrue(lifecycle.cancellationRequested()); + awaitTermination(cancelCalls, closeCalls, 1); + try { + registration.ensureNotCancelled(); + Assert.fail("timed out query must not continue"); + } catch (FederationSqlException exception) { + Assert.assertEquals(FederationSqlErrorCode.QUERY_TIMEOUT, exception.errorCode()); + } finally { + registration.close(); + registry.close(); + } + } + + /** + * 验证取消与超时并发到达时,查询只保留第一个成功写入的终止原因。 + * + * @throws Exception 并发执行失败 + */ + @Test + public void shouldKeepFirstTerminationReasonUnderCancelTimeoutRace() throws Exception { + QueryCancellationRegistry registry = new QueryCancellationRegistry(); + QueryId queryId = new QueryId("termination-race"); + QueryCancellationRegistry.QueryRegistration registration = registry.begin(queryId); + CountDownLatch start = new CountDownLatch(1); + ExecutorService executor = Executors.newFixedThreadPool(2); + try { + var cancel = executor.submit(() -> { + start.await(); + return registry.cancelOutcome(queryId).reason(); + }); + var timeout = executor.submit(() -> { + start.await(); + return registration.requestTimeoutOutcome(); + }); + + start.countDown(); + QueryCancellationRegistry.TerminationReason cancelObserved = + cancel.get(2, TimeUnit.SECONDS); + QueryCancellationRegistry.TerminationReason timeoutObserved = + timeout.get(2, TimeUnit.SECONDS); + + Assert.assertEquals(cancelObserved, timeoutObserved); + Assert.assertTrue( + cancelObserved == QueryCancellationRegistry.TerminationReason.CANCELLED + || cancelObserved == QueryCancellationRegistry.TerminationReason.TIMED_OUT + ); + try { + registration.ensureNotCancelled(); + Assert.fail("terminated query must not continue"); + } catch (FederationSqlException exception) { + FederationSqlErrorCode expected = cancelObserved + == QueryCancellationRegistry.TerminationReason.TIMED_OUT + ? FederationSqlErrorCode.QUERY_TIMEOUT + : FederationSqlErrorCode.QUERY_CANCELLED; + Assert.assertEquals(expected, exception.errorCode()); + } + } finally { + registration.close(); + executor.shutdownNow(); + registry.close(); + } + } + + /** + * 验证终止队列饱和时转交隔离执行器,取消线程不会被 JDBC 清理反向阻塞。 + * + * @throws Exception 并发测试失败 + */ + @Test + public void shouldIsolateStatementTerminationWhenPrimaryQueueIsFull() throws Exception { + ThreadPoolExecutor terminationExecutor = executor("cancel-overflow-test"); + ThreadPoolExecutor cleanupExecutor = executor("cleanup-overflow-test"); + ThreadPoolExecutor overflowExecutor = executor("resource-overflow-test"); + QueryCancellationRegistry registry = new QueryCancellationRegistry( + terminationExecutor, + cleanupExecutor, + overflowExecutor + ); + CountDownLatch workerStarted = new CountDownLatch(1); + CountDownLatch releaseWorker = new CountDownLatch(1); + terminationExecutor.execute(() -> { + workerStarted.countDown(); + try { + releaseWorker.await(); + } catch (InterruptedException exception) { + Thread.currentThread().interrupt(); + } + }); + Assert.assertTrue(workerStarted.await(2, TimeUnit.SECONDS)); + terminationExecutor.execute(() -> { + }); + + QueryId queryId = new QueryId("cancel-queue-overflow"); + QueryCancellationRegistry.QueryRegistration registration = registry.begin(queryId); + AtomicInteger cancelCalls = new AtomicInteger(); + AtomicInteger closeCalls = new AtomicInteger(); + registration.statementLifecycle().register(statement(cancelCalls, closeCalls)); + + long startedAt = System.nanoTime(); + Assert.assertTrue(registry.cancel(queryId)); + Assert.assertTrue( + TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startedAt) < 200L + ); + awaitTermination(cancelCalls, closeCalls, 1); + Assert.assertEquals(1L, registry.overflowFallbackCount()); + Assert.assertEquals(0L, registry.unscheduledCleanupCount()); + + releaseWorker.countDown(); + registration.close(); + registry.close(); + } + + /** + * 验证游标清理队列饱和时转交隔离执行器,并明确返回主队列未接收。 + * + * @throws Exception 并发测试失败 + */ + @Test + public void shouldIsolateCursorCleanupWhenPrimaryQueueIsFull() throws Exception { + ThreadPoolExecutor terminationExecutor = executor("cancel-cleanup-test"); + ThreadPoolExecutor cleanupExecutor = executor("cleanup-overflow-test"); + ThreadPoolExecutor overflowExecutor = executor("resource-overflow-test"); + QueryCancellationRegistry registry = new QueryCancellationRegistry( + terminationExecutor, + cleanupExecutor, + overflowExecutor + ); + CountDownLatch workerStarted = new CountDownLatch(1); + CountDownLatch releaseWorker = new CountDownLatch(1); + cleanupExecutor.execute(() -> { + workerStarted.countDown(); + try { + releaseWorker.await(); + } catch (InterruptedException exception) { + Thread.currentThread().interrupt(); + } + }); + Assert.assertTrue(workerStarted.await(2, TimeUnit.SECONDS)); + cleanupExecutor.execute(() -> { + }); + AtomicInteger cleanupCalls = new AtomicInteger(); + + Assert.assertFalse(registry.submitCleanup(cleanupCalls::incrementAndGet)); + awaitValue(cleanupCalls, 1); + Assert.assertEquals(1L, registry.overflowFallbackCount()); + Assert.assertEquals(0L, registry.unscheduledCleanupCount()); + + releaseWorker.countDown(); + registry.close(); + } + + /** + * 验证主队列和隔离队列同时饱和时,清理进入有界延期队列并在容量恢复后重试。 + * + * @throws Exception 并发测试失败 + */ + @Test + public void shouldRetryDeferredCleanupAfterBothExecutorsRecover() throws Exception { + ThreadPoolExecutor terminationExecutor = executor("cancel-deferred-test"); + ThreadPoolExecutor cleanupExecutor = executor("cleanup-deferred-test"); + ThreadPoolExecutor overflowExecutor = executor("overflow-deferred-test"); + QueryCancellationRegistry registry = new QueryCancellationRegistry( + terminationExecutor, + cleanupExecutor, + overflowExecutor + ); + CountDownLatch cleanupStarted = new CountDownLatch(1); + CountDownLatch overflowStarted = new CountDownLatch(1); + CountDownLatch releaseCleanup = new CountDownLatch(1); + CountDownLatch releaseOverflow = new CountDownLatch(1); + blockExecutor(cleanupExecutor, cleanupStarted, releaseCleanup); + blockExecutor(overflowExecutor, overflowStarted, releaseOverflow); + Assert.assertTrue(cleanupStarted.await(2, TimeUnit.SECONDS)); + Assert.assertTrue(overflowStarted.await(2, TimeUnit.SECONDS)); + cleanupExecutor.execute(() -> { }); + overflowExecutor.execute(() -> { }); + AtomicInteger cleanupCalls = new AtomicInteger(); + + Assert.assertFalse(registry.submitCleanup(cleanupCalls::incrementAndGet)); + Assert.assertEquals(1L, registry.cleanupMetrics().deferredRetries()); + Assert.assertEquals(1, registry.cleanupMetrics().deferredQueueDepth()); + + releaseOverflow.countDown(); + awaitValue(cleanupCalls, 1); + Assert.assertEquals(0, registry.cleanupMetrics().deferredQueueDepth()); + Assert.assertEquals(0L, registry.cleanupMetrics().unresolvedCleanups()); + + releaseCleanup.countDown(); + registry.close(); + } + + /** + * 验证 Driver 清理忽略中断时,登记器关闭仍保持有界且不会卡死调用线程。 + * + * @throws Exception 并发测试失败 + */ + @Test + public void shouldCloseWithinBoundWhenDriverCleanupBlocks() throws Exception { + ThreadPoolExecutor terminationExecutor = executor("cancel-close-test"); + ThreadPoolExecutor cleanupExecutor = executor("cleanup-close-test"); + ThreadPoolExecutor overflowExecutor = executor("overflow-close-test"); + QueryCancellationRegistry registry = new QueryCancellationRegistry( + terminationExecutor, + cleanupExecutor, + overflowExecutor + ); + CountDownLatch releaseDriver = new CountDownLatch(1); + QueryCancellationRegistry.QueryRegistration first = registry.begin( + new QueryId("blocked-driver-1") + ); + QueryCancellationRegistry.QueryRegistration second = registry.begin( + new QueryId("blocked-driver-2") + ); + first.statementLifecycle().register(blockingStatement(releaseDriver)); + second.statementLifecycle().register(blockingStatement(releaseDriver)); + registry.cancel(new QueryId("blocked-driver-1")); + registry.cancel(new QueryId("blocked-driver-2")); + + long startedAt = System.nanoTime(); + try { + registry.close(); + long elapsedMillis = TimeUnit.NANOSECONDS.toMillis( + System.nanoTime() - startedAt + ); + Assert.assertTrue("close took " + elapsedMillis + "ms", elapsedMillis < 1_000L); + Assert.assertTrue(registry.cleanupMetrics().unresolvedCleanups() > 0L); + } finally { + releaseDriver.countDown(); + first.close(); + second.close(); + } + } + + private static void blockExecutor( + ThreadPoolExecutor executor, + CountDownLatch started, + CountDownLatch release + ) { + executor.execute(() -> { + started.countDown(); + try { + release.await(); + } catch (InterruptedException exception) { + Thread.currentThread().interrupt(); + } + }); + } + + private static void awaitTermination( + AtomicInteger cancelCalls, + AtomicInteger closeCalls, + int expected + ) throws InterruptedException { + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(2); + while ((cancelCalls.get() != expected || closeCalls.get() != expected) + && System.nanoTime() - deadline < 0L) { + Thread.sleep(5L); + } + Assert.assertEquals(expected, cancelCalls.get()); + Assert.assertEquals(expected, closeCalls.get()); + } + + private static void awaitValue(AtomicInteger value, int expected) + throws InterruptedException { + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(2); + while (value.get() != expected && System.nanoTime() - deadline < 0L) { + Thread.sleep(5L); + } + Assert.assertEquals(expected, value.get()); + } + + private static Statement blockingStatement(CountDownLatch releaseDriver) { + return (Statement) Proxy.newProxyInstance( + QueryCancellationRegistryTest.class.getClassLoader(), + new Class[] {Statement.class}, + (proxy, method, arguments) -> { + if ("cancel".equals(method.getName())) { + while (releaseDriver.getCount() > 0L) { + try { + releaseDriver.await(); + } catch (InterruptedException ignored) { + // 模拟忽略线程中断的 JDBC Driver。 + } + } + } + return defaultValue(method.getReturnType()); + } + ); + } + + private static Statement statement( + AtomicInteger cancelCalls, + AtomicInteger closeCalls + ) { + return (Statement) Proxy.newProxyInstance( + QueryCancellationRegistryTest.class.getClassLoader(), + new Class[] {Statement.class}, + (proxy, method, arguments) -> { + if ("cancel".equals(method.getName())) { + cancelCalls.incrementAndGet(); + } + if ("close".equals(method.getName())) { + closeCalls.incrementAndGet(); + } + return defaultValue(method.getReturnType()); + } + ); + } + + private static ThreadPoolExecutor executor(String threadName) { + return new ThreadPoolExecutor( + 1, + 1, + 0L, + TimeUnit.MILLISECONDS, + new ArrayBlockingQueue<>(1), + runnable -> { + Thread thread = new Thread(runnable, threadName); + thread.setDaemon(true); + return thread; + }, + new ThreadPoolExecutor.AbortPolicy() + ); + } + + private static Object defaultValue(Class type) { + if (!type.isPrimitive()) { + return null; + } + if (type == boolean.class) { + return false; + } + if (type == byte.class) { + return (byte) 0; + } + if (type == short.class) { + return (short) 0; + } + if (type == int.class) { + return 0; + } + if (type == long.class) { + return 0L; + } + if (type == float.class) { + return 0F; + } + if (type == double.class) { + return 0D; + } + if (type == char.class) { + return '\0'; + } + return null; + } +} diff --git a/easy-agents-federation-sql/easy-agents-federation-sql-core/src/test/java/com/easyagents/federation/sql/runtime/QueryDeadlineTest.java b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/test/java/com/easyagents/federation/sql/runtime/QueryDeadlineTest.java new file mode 100644 index 0000000..d9e5b9b --- /dev/null +++ b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/test/java/com/easyagents/federation/sql/runtime/QueryDeadlineTest.java @@ -0,0 +1,34 @@ +package com.easyagents.federation.sql.runtime; + +import java.time.Duration; +import org.junit.Assert; +import org.junit.Test; + +/** + * 查询统一截止时间的单调时钟边界测试。 + */ +public class QueryDeadlineTest { + + /** + * 验证 nanoTime 为负时仍保留有限查询时限。 + */ + @Test + public void shouldKeepFiniteDeadlineWhenNanoTimeIsNegative() { + Assert.assertEquals( + 15L, + QueryDeadline.deadlineAfter(-5L, Duration.ofNanos(20L)) + ); + } + + /** + * 验证单调时钟跨 Long 上界时按补码回绕,差值语义保持正确。 + */ + @Test + public void shouldAllowNanoTimeDeadlineToWrap() { + long now = Long.MAX_VALUE - 5L; + long deadline = QueryDeadline.deadlineAfter(now, Duration.ofNanos(10L)); + + Assert.assertEquals(Long.MIN_VALUE + 4L, deadline); + Assert.assertEquals(10L, deadline - now); + } +} diff --git a/easy-agents-federation-sql/easy-agents-federation-sql-core/src/test/java/com/easyagents/federation/sql/source/FederationSourceDefinitionTest.java b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/test/java/com/easyagents/federation/sql/source/FederationSourceDefinitionTest.java new file mode 100644 index 0000000..385c9a6 --- /dev/null +++ b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/test/java/com/easyagents/federation/sql/source/FederationSourceDefinitionTest.java @@ -0,0 +1,141 @@ +package com.easyagents.federation.sql.source; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import org.junit.Assert; +import org.junit.Test; + +/** + * FederationSourceDefinition 不可变性与校验和测试。 + */ +public class FederationSourceDefinitionTest { + + /** + * 验证构造后修改外部集合不会改变 Definition 与校验和。 + */ + @Test + public void shouldDefensivelyCopyNestedCollections() { + List schemas = new ArrayList<>(); + schemas.add(new ExternalSchemaDefinition("app", "schema-ref", 1)); + Map options = new HashMap<>(); + options.put("mode", "safe"); + + FederationSourceDefinition definition = new FederationSourceDefinition( + new SourceId("source-a"), + 7, + "fake", + schemas, + options + ); + String checksum = definition.checksum(); + + schemas.clear(); + options.put("mode", "changed"); + + Assert.assertEquals(1, definition.schemas().size()); + Assert.assertEquals("safe", definition.adapterOptions().get("mode")); + Assert.assertEquals(checksum, definition.checksum()); + } + + /** + * 验证同名 Schema 会在进入 Runtime 前被拒绝。 + */ + @Test(expected = IllegalArgumentException.class) + public void shouldRejectDuplicateLogicalSchemaNames() { + new FederationSourceDefinition( + new SourceId("source-a"), + 1, + "fake", + List.of( + new ExternalSchemaDefinition("app", "one", 1), + new ExternalSchemaDefinition("app", "two", 1) + ), + Map.of() + ); + } + + /** + * 验证可空字段、分隔符字符和 Adapter 选项边界不会产生规范化碰撞。 + */ + @Test + public void shouldUseUnambiguousCanonicalChecksumFields() { + FederationSourceDefinition nullField = customDefinition( + new ChecksumSchema("app", null, "x\0y"), + Map.of("a", "bc") + ); + FederationSourceDefinition literalField = customDefinition( + new ChecksumSchema("app", "", "x\0y"), + Map.of("a", "bc") + ); + FederationSourceDefinition shiftedBoundary = customDefinition( + new ChecksumSchema("app", null, "x\0", "y"), + Map.of("a", "bc") + ); + FederationSourceDefinition optionBoundary = customDefinition( + new ChecksumSchema("app", null, "x\0y"), + Map.of("ab", "c") + ); + + Assert.assertNotEquals(nullField.checksum(), literalField.checksum()); + Assert.assertNotEquals(nullField.checksum(), shiftedBoundary.checksum()); + Assert.assertNotEquals(nullField.checksum(), optionBoundary.checksum()); + } + + /** + * 验证共享状态不能携带伪造的 Definition 或墓碑校验和。 + */ + @Test + public void shouldRejectForgedSharedStateChecksums() { + FederationSourceDefinition definition = customDefinition( + new ChecksumSchema("app", "catalog", "schema"), + Map.of() + ); + try { + new ActiveSourceState(definition, "forged"); + Assert.fail("forged active checksum should be rejected"); + } catch (IllegalArgumentException expected) { + Assert.assertTrue(expected.getMessage().contains("checksum")); + } + try { + new SourceTombstone(definition.sourceId(), definition.revision() + 1, "forged"); + Assert.fail("forged tombstone checksum should be rejected"); + } catch (IllegalArgumentException expected) { + Assert.assertTrue(expected.getMessage().contains("checksum")); + } + } + + private static FederationSourceDefinition customDefinition( + FederationSchemaDefinition schema, + Map options + ) { + return new FederationSourceDefinition( + new SourceId("source-a"), + 7, + "fake", + List.of(schema), + options + ); + } + + /** + * 测试用可空、可含分隔符的 Schema 校验和字段。 + * + * @param logicalName 逻辑名称 + * @param fields 校验和字段 + */ + private record ChecksumSchema(String logicalName, List fields) + implements FederationSchemaDefinition { + + private ChecksumSchema(String logicalName, String... fields) { + this(logicalName, Arrays.asList(fields)); + } + + @Override + public List checksumFields() { + return fields; + } + } +} diff --git a/easy-agents-federation-sql/easy-agents-federation-sql-core/src/test/java/com/easyagents/federation/sql/source/KnownJdbcDriverTest.java b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/test/java/com/easyagents/federation/sql/source/KnownJdbcDriverTest.java new file mode 100644 index 0000000..23b0534 --- /dev/null +++ b/easy-agents-federation-sql/easy-agents-federation-sql-core/src/test/java/com/easyagents/federation/sql/source/KnownJdbcDriverTest.java @@ -0,0 +1,83 @@ +package com.easyagents.federation.sql.source; + +import org.junit.Assert; +import org.junit.Test; + +/** + * KnownJdbcDriver 公共元数据契约测试。 + */ +public class KnownJdbcDriverTest { + + /** + * 验证计划内常见数据库使用标准驱动类名和 URL 前缀。 + */ + @Test + public void shouldExposeMainstreamDriverMetadata() { + assertDriver(KnownJdbcDriver.MYSQL, "com.mysql.cj.jdbc.Driver", "jdbc:mysql:"); + assertDriver(KnownJdbcDriver.POSTGRESQL, "org.postgresql.Driver", "jdbc:postgresql:"); + assertDriver(KnownJdbcDriver.ORACLE, "oracle.jdbc.OracleDriver", "jdbc:oracle:thin:"); + assertDriver( + KnownJdbcDriver.SQL_SERVER, + "com.microsoft.sqlserver.jdbc.SQLServerDriver", + "jdbc:sqlserver:" + ); + } + + /** + * 验证计划内信创数据库使用已确认的厂商驱动类名和 URL 前缀。 + */ + @Test + public void shouldExposePlannedDomesticDriverMetadata() { + assertDriver( + KnownJdbcDriver.GAUSSDB, + "com.huawei.gaussdb.jdbc.Driver", + "jdbc:gaussdb:" + ); + assertDriver(KnownJdbcDriver.DM8, "dm.jdbc.driver.DmDriver", "jdbc:dm:"); + assertDriver(KnownJdbcDriver.GBASE_8A, "com.gbase.jdbc.Driver", "jdbc:gbase:"); + assertDriver( + KnownJdbcDriver.GBASE_8S, + "com.gbasedbt.jdbc.Driver", + "jdbc:gbasedbt-sqli:" + ); + assertDriver( + KnownJdbcDriver.OCEANBASE, + "com.oceanbase.jdbc.Driver", + "jdbc:oceanbase:" + ); + } + + /** + * 验证公开 Maven 坐标与厂商包获取方式能够明确区分。 + */ + @Test + public void shouldExposeOnlyKnownPublicMavenCoordinates() { + Assert.assertEquals( + "com.dameng:DmJdbcDriver8", + KnownJdbcDriver.DM8.mavenCoordinate().orElseThrow() + ); + Assert.assertEquals( + "com.oceanbase:oceanbase-client", + KnownJdbcDriver.OCEANBASE.mavenCoordinate().orElseThrow() + ); + Assert.assertTrue(KnownJdbcDriver.GAUSSDB.mavenCoordinate().isEmpty()); + Assert.assertTrue(KnownJdbcDriver.GBASE_8A.mavenCoordinate().isEmpty()); + Assert.assertTrue(KnownJdbcDriver.GBASE_8S.mavenCoordinate().isEmpty()); + } + + /** + * 断言驱动类名和 JDBC URL 前缀。 + * + * @param driver 待验证驱动元数据 + * @param expectedClassName 预期驱动类名 + * @param expectedUrlPrefix 预期 JDBC URL 前缀 + */ + private static void assertDriver( + KnownJdbcDriver driver, + String expectedClassName, + String expectedUrlPrefix + ) { + Assert.assertEquals(expectedClassName, driver.driverClassName()); + Assert.assertEquals(expectedUrlPrefix, driver.jdbcUrlPrefix()); + } +} diff --git a/easy-agents-federation-sql/pom.xml b/easy-agents-federation-sql/pom.xml new file mode 100644 index 0000000..5167710 --- /dev/null +++ b/easy-agents-federation-sql/pom.xml @@ -0,0 +1,21 @@ + + + 4.0.0 + + + com.easyagents + easy-agents + ${revision} + + + easy-agents-federation-sql + pom + easy-agents-federation-sql + + + easy-agents-federation-sql-core + easy-agents-federation-sql-adapter-jdbc + + diff --git a/pom.xml b/pom.xml index 97406ff..5d77001 100644 --- a/pom.xml +++ b/pom.xml @@ -31,6 +31,7 @@ easy-agents-agent-runtime easy-agents-agui easy-agents-flow + easy-agents-federation-sql easy-agents-support @@ -49,6 +50,8 @@ 1.0.12 2.6 1.28.0 + 1.42.0 + 2.3.232 @@ -129,6 +132,19 @@ ${commons-compress.version} + + org.apache.calcite + calcite-core + ${calcite.version} + + + + com.h2database + h2 + ${h2.version} + + + com.easyagents @@ -202,6 +218,21 @@ ${revision} + + com.easyagents + easy-agents-federation-sql-core + ${revision} + + + + com.easyagents + easy-agents-federation-sql-adapter-jdbc + ${revision} + + + + +