perf: 优化多源联邦分片调度

- 增加 Engine 级有界并行调度与资源收口

- 补齐调度饱和、失败快速传播及六源基准测试
This commit is contained in:
2026-09-01 17:02:08 +08:00
parent 1b36067e6c
commit c1fe64cefa
9 changed files with 1137 additions and 48 deletions

View File

@@ -0,0 +1,129 @@
package com.easyagents.federation.sql.adapter.jdbc;
import com.easyagents.federation.sql.api.FederationSqlEngine;
import com.easyagents.federation.sql.api.FederationSqlEngines;
import com.easyagents.federation.sql.api.SqlQueryCommand;
import com.easyagents.federation.sql.execute.FederationResultCursor;
import com.easyagents.federation.sql.federation.FederationExecutionPolicy;
import com.easyagents.federation.sql.federation.FederationQueryScopeDefinition;
import com.easyagents.federation.sql.federation.FederationSourceBindingDefinition;
import com.easyagents.federation.sql.source.FederationDataSourceHandles;
import com.easyagents.federation.sql.source.FederationSourceDefinition;
import com.easyagents.federation.sql.source.RuntimeFingerprint;
import com.easyagents.federation.sql.source.SourceApplyOptions;
import com.easyagents.federation.sql.source.SourceId;
import com.mysql.cj.jdbc.MysqlDataSource;
import org.junit.Assert;
import org.junit.Assume;
import org.junit.Test;
import java.math.BigDecimal;
import java.sql.Connection;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.List;
import java.util.Map;
/**
* 亿级数据集 100,000 行 Cursor 手工基准;默认跳过,必须显式启用。
*/
public class Jdbc100mCursorManualBenchmarkTest {
/**
* 从支付库顺序消费恰好 100,000 行,验证 Cursor 流式消费和资源上限。
*/
@Test
public void shouldConsumeOneHundredThousandRows() {
Assume.assumeTrue(Boolean.getBoolean("federation.100m.enabled"));
String host = System.getProperty("federation.100m.mysql.host", "127.0.0.1");
int port = Integer.getInteger("federation.100m.mysql.port", 33307);
String database = System.getProperty(
"federation.100m.mysql.database", "ef_bank_payment_perf");
String username = System.getProperty("federation.100m.mysql.username", "ef_bench_ro");
String password = System.getProperty("federation.100m.mysql.password");
Assert.assertNotNull("必须通过系统属性提供只读密码", password);
MysqlDataSource dataSource = new MysqlDataSource();
dataSource.setUrl("jdbc:mysql://" + host + ':' + port + '/' + database
+ "?useSSL=false&allowPublicKeyRetrieval=true&serverTimezone=Asia/Shanghai"
+ "&useCursorFetch=true&defaultFetchSize=1000");
dataSource.setUser(username);
dataSource.setPassword(password);
SourceId sourceId = new SourceId("payment-100m");
FederationSourceDefinition definition = new FederationSourceDefinition(
sourceId, 1, JdbcFederationSqlAdapterProvider.ADAPTER_ID,
List.of(new JdbcSchemaDefinition("MAIN", database, null)), Map.of());
FederationExecutionPolicy policy = new FederationExecutionPolicy(
6, 12, 2, 100_000, 64L * 1024L * 1024L, 60_000);
FederationQueryScopeDefinition scope = FederationQueryScopeDefinition.virtual(
"payment-cursor-100m", 1,
Map.of("PAY", FederationSourceBindingDefinition.of(sourceId, 1)),
"PAY", policy);
CursorResult nativeResult = consumeNative(dataSource);
CursorResult federationResult;
long federationStarted = System.nanoTime();
try (FederationSqlEngine engine = FederationSqlEngines.builder()
.dataSourceResolver(ignored -> FederationDataSourceHandles.shared(
dataSource, new RuntimeFingerprint("MySQL", "8.0", "JDBC", "8.4", "1")))
.adapter(new JdbcFederationSqlAdapterProvider())
.federationExecutionPolicy(policy)
.build()) {
engine.sources().apply(definition, SourceApplyOptions.prewarmNow());
String sql = "SELECT payment_id, amount FROM PAY.MAIN.payment_order "
+ "WHERE payment_id BETWEEN 1 AND 100000 ORDER BY payment_id";
int rows = 0;
long amountCents = 0;
try (FederationResultCursor cursor = engine.query(
SqlQueryCommand.of(sql, scope, List.of()))) {
while (cursor.next()) {
rows++;
amountCents += ((BigDecimal) cursor.getObject(2))
.movePointRight(2).longValue();
}
}
federationResult = new CursorResult(rows, amountCents,
elapsedMillis(federationStarted));
}
Assert.assertEquals(100_000, nativeResult.rows());
Assert.assertEquals(100_000, federationResult.rows());
Assert.assertEquals(nativeResult.amountCents(), federationResult.amountCents());
Assert.assertTrue(federationResult.amountCents() > 0L);
System.out.println("BENCH_CURSOR_100K rows=" + federationResult.rows()
+ " nativeMillis=" + nativeResult.elapsedMillis()
+ " federationMillis=" + federationResult.elapsedMillis()
+ " amountCents=" + federationResult.amountCents());
}
private CursorResult consumeNative(MysqlDataSource dataSource) {
String sql = "SELECT payment_id, amount FROM payment_order "
+ "WHERE payment_id BETWEEN 1 AND 100000 ORDER BY payment_id";
long started = System.nanoTime();
int rows = 0;
long amountCents = 0;
try (Connection connection = dataSource.getConnection();
PreparedStatement statement = connection.prepareStatement(sql,
ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY)) {
statement.setFetchSize(1000);
try (ResultSet resultSet = statement.executeQuery()) {
while (resultSet.next()) {
rows++;
amountCents += resultSet.getBigDecimal(2)
.movePointRight(2).longValue();
}
}
} catch (SQLException exception) {
throw new IllegalStateException("原生 JDBC Cursor 基准执行失败", exception);
}
return new CursorResult(rows, amountCents, elapsedMillis(started));
}
private long elapsedMillis(long started) {
return (System.nanoTime() - started) / 1_000_000L;
}
private record CursorResult(int rows, long amountCents, long elapsedMillis) {
}
}

View File

@@ -27,6 +27,7 @@ import com.easyagents.federation.sql.source.RuntimeFingerprint;
import com.easyagents.federation.sql.source.SourceApplyOptions;
import com.easyagents.federation.sql.source.SourceId;
import java.math.BigDecimal;
import java.sql.SQLException;
import java.time.Instant;
import java.sql.Connection;
import java.sql.Statement;
@@ -34,6 +35,9 @@ import java.sql.Types;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicLong;
import java.util.concurrent.atomic.AtomicReference;
import org.h2.jdbcx.JdbcDataSource;
@@ -50,6 +54,13 @@ public class JdbcFederatedQueryEngineTest {
private static final SourceId SALES_SOURCE = new SourceId("sales-source");
private static final SourceId BILLING_SOURCE = new SourceId("billing-source");
private static final SourceId REGION_SOURCE = new SourceId("region-source");
private static final AtomicInteger ACTIVE_FRAGMENTS = new AtomicInteger();
private static final AtomicInteger MAXIMUM_ACTIVE_FRAGMENTS = new AtomicInteger();
private static final AtomicInteger FAILING_TRACKED_VALUE = new AtomicInteger(
Integer.MIN_VALUE
);
private static final AtomicReference<CountDownLatch> FRAGMENT_START_BARRIER =
new AtomicReference<>(new CountDownLatch(0));
private JdbcDataSource sales;
private JdbcDataSource billing;
@@ -80,7 +91,10 @@ public class JdbcFederatedQueryEngineTest {
+ "TIMESTAMP WITH TIME ZONE '2026-08-21 12:00:00.123456+08:00')",
"CREATE TABLE PRECISE_EVENT (ID INT PRIMARY KEY, EVENT_AT TIMESTAMP(6))",
"INSERT INTO PRECISE_EVENT VALUES "
+ "(1, TIMESTAMP '2026-08-21 12:00:00.123456')");
+ "(1, TIMESTAMP '2026-08-21 12:00:00.123456')",
fragmentTrackingAlias(),
"CREATE VIEW TRACKED_VALUE AS SELECT TRACK_FRAGMENT(ID) AS TRACKED_KEY "
+ "FROM CUSTOMER WHERE ID <= 2");
execute(billing,
"CREATE TABLE ORDER_ITEM (ID INT PRIMARY KEY, CUSTOMER_ID INT NOT NULL, AMOUNT DECIMAL(12,2), CODE VARCHAR(64))",
"INSERT INTO ORDER_ITEM VALUES (10, 1, 30.00, 'Alice'), (11, 1, 20.00, 'Alice'), (12, 2, 80.00, 'Bob')",
@@ -90,10 +104,16 @@ public class JdbcFederatedQueryEngineTest {
+ "TIMESTAMP WITH TIME ZONE '2026-08-21 06:00:00.123456+02:00')",
"CREATE TABLE PRECISE_EVENT (ID INT PRIMARY KEY, EVENT_AT TIMESTAMP(6))",
"INSERT INTO PRECISE_EVENT VALUES "
+ "(2, TIMESTAMP '2026-08-21 08:30:00.654321')");
+ "(2, TIMESTAMP '2026-08-21 08:30:00.654321')",
fragmentTrackingAlias(),
"CREATE VIEW TRACKED_VALUE AS SELECT TRACK_FRAGMENT(ID) AS TRACKED_KEY "
+ "FROM ORDER_ITEM WHERE ID <= 11");
execute(region,
"CREATE TABLE CUSTOMER_REGION (CUSTOMER_ID INT PRIMARY KEY, REGION VARCHAR(64))",
"INSERT INTO CUSTOMER_REGION VALUES (1, 'North'), (2, 'South'), (3, 'West')");
"INSERT INTO CUSTOMER_REGION VALUES (1, 'North'), (2, 'South'), (3, 'West')",
fragmentTrackingAlias(),
"CREATE VIEW TRACKED_VALUE AS SELECT TRACK_FRAGMENT(CUSTOMER_ID) AS TRACKED_KEY "
+ "FROM CUSTOMER_REGION WHERE CUSTOMER_ID <= 2");
RuntimeFingerprint fingerprint = new RuntimeFingerprint(
"H2", "2", "H2 JDBC Driver", "2", "1"
@@ -356,6 +376,91 @@ public class JdbcFederatedQueryEngineTest {
);
}
/**
* 验证真实 JDBC Fragment 执行会重叠,且始终受查询级并发上限约束。
*/
@Test
public void shouldBoundConcurrentJdbcFragments() {
ACTIVE_FRAGMENTS.set(0);
MAXIMUM_ACTIVE_FRAGMENTS.set(0);
FRAGMENT_START_BARRIER.set(new CountDownLatch(2));
FederationQueryScopeDefinition threeSourceScope =
new FederationQueryScopeDefinition(
"tracked-fragments",
1,
Map.of(
"SALES", FederationSourceBindingDefinition.of(SALES_SOURCE, 1),
"BILLING", FederationSourceBindingDefinition.of(BILLING_SOURCE, 1),
"REGION", FederationSourceBindingDefinition.of(REGION_SOURCE, 1)
),
"SALES",
threeSourcePolicy()
);
try (FederationResultCursor cursor = engine.query(SqlQueryCommand.of(
"SELECT COUNT(*) FROM SALES.APP.TRACKED_VALUE s "
+ "JOIN BILLING.APP.TRACKED_VALUE b ON s.TRACKED_KEY = b.TRACKED_KEY "
+ "JOIN REGION.APP.TRACKED_VALUE r ON s.TRACKED_KEY = r.TRACKED_KEY",
threeSourceScope,
List.of()
))) {
Assert.assertTrue(cursor.next());
Assert.assertEquals(8L, cursor.getObject(1));
Assert.assertFalse(cursor.next());
}
Assert.assertEquals(2, MAXIMUM_ACTIVE_FRAGMENTS.get());
Assert.assertEquals(0, ACTIVE_FRAGMENTS.get());
}
/**
* 验证任一并行 Fragment 失败后立即传播原始执行错误,不等待未启动任务超时。
*/
@Test
public void shouldFailFastWhenConcurrentFragmentFails() {
ACTIVE_FRAGMENTS.set(0);
MAXIMUM_ACTIVE_FRAGMENTS.set(0);
FRAGMENT_START_BARRIER.set(new CountDownLatch(0));
FAILING_TRACKED_VALUE.set(10);
FederationQueryScopeDefinition threeSourceScope =
new FederationQueryScopeDefinition(
"tracked-fragment-failure",
1,
Map.of(
"SALES", FederationSourceBindingDefinition.of(SALES_SOURCE, 1),
"BILLING", FederationSourceBindingDefinition.of(BILLING_SOURCE, 1),
"REGION", FederationSourceBindingDefinition.of(REGION_SOURCE, 1)
),
"SALES",
new FederationExecutionPolicy(
3,
8,
2,
100_000,
64L * 1024L * 1024L,
1_000
)
);
try (FederationResultCursor cursor = engine.query(SqlQueryCommand.of(
"SELECT COUNT(*) FROM SALES.APP.TRACKED_VALUE s "
+ "JOIN BILLING.APP.TRACKED_VALUE b ON s.TRACKED_KEY = b.TRACKED_KEY "
+ "JOIN REGION.APP.TRACKED_VALUE r ON s.TRACKED_KEY = r.TRACKED_KEY",
threeSourceScope,
List.of()
))) {
cursor.next();
Assert.fail("failing fragment should terminate the federated query");
} catch (FederationSqlException exception) {
Assert.assertEquals(
FederationSqlErrorCode.EXECUTION_FAILED,
exception.errorCode()
);
} finally {
FAILING_TRACKED_VALUE.set(Integer.MIN_VALUE);
}
}
/**
* 验证两个分片分别绑定原始查询中的动态参数。
*/
@@ -717,6 +822,41 @@ public class JdbcFederatedQueryEngineTest {
return dataSource;
}
/**
* H2 视图调用的并发跟踪函数。
*
* @param value 原始列值
* @return 固定键值
* @throws SQLException 分片未能在时限内重叠启动
*/
public static int trackFragment(int value) throws SQLException {
int active = ACTIVE_FRAGMENTS.incrementAndGet();
MAXIMUM_ACTIVE_FRAGMENTS.accumulateAndGet(active, Math::max);
CountDownLatch barrier = FRAGMENT_START_BARRIER.get();
barrier.countDown();
try {
if (value == FAILING_TRACKED_VALUE.get()) {
throw new SQLException("simulated federation fragment failure");
}
if (!barrier.await(2, TimeUnit.SECONDS)) {
throw new SQLException("federation fragments did not overlap");
}
Thread.sleep(100L);
return 1;
} catch (InterruptedException exception) {
Thread.currentThread().interrupt();
throw new SQLException("fragment tracking was interrupted", exception);
} finally {
ACTIVE_FRAGMENTS.decrementAndGet();
}
}
private static String fragmentTrackingAlias() {
return "CREATE ALIAS TRACK_FRAGMENT FOR \""
+ JdbcFederatedQueryEngineTest.class.getName()
+ ".trackFragment\"";
}
/**
* 根据物理源选择测试数据库。
*

View File

@@ -0,0 +1,185 @@
package com.easyagents.federation.sql.adapter.jdbc;
import com.easyagents.federation.sql.api.FederationSqlEngine;
import com.easyagents.federation.sql.api.FederationSqlEngines;
import com.easyagents.federation.sql.api.FederationSqlErrorCode;
import com.easyagents.federation.sql.api.FederationSqlException;
import com.easyagents.federation.sql.api.SqlQueryCommand;
import com.easyagents.federation.sql.compile.FederationSqlPlan;
import com.easyagents.federation.sql.compile.SqlCompileRequest;
import com.easyagents.federation.sql.execute.FederationResultCursor;
import com.easyagents.federation.sql.federation.FederationExecutionPolicy;
import com.easyagents.federation.sql.federation.FederationQueryMode;
import com.easyagents.federation.sql.federation.FederationQueryScopeDefinition;
import com.easyagents.federation.sql.federation.FederationSourceBindingDefinition;
import com.easyagents.federation.sql.source.FederationDataSourceHandles;
import com.easyagents.federation.sql.source.FederationSourceDefinition;
import com.easyagents.federation.sql.source.RuntimeFingerprint;
import com.easyagents.federation.sql.source.SourceApplyOptions;
import com.easyagents.federation.sql.source.SourceId;
import org.h2.jdbcx.JdbcDataSource;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import java.sql.Connection;
import java.sql.Statement;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
/**
* 四源和六源 JDBC 多阶段 Join 集成测试。
*/
public class JdbcSixSourceQueryEngineTest {
private static final FederationExecutionPolicy SIX_SOURCE_POLICY =
new FederationExecutionPolicy(6, 12, 2, 100_000,
64L * 1024L * 1024L, 60_000);
private final Map<SourceId, JdbcDataSource> dataSources = new LinkedHashMap<>();
private final Map<String, FederationSourceBindingDefinition> bindings = new LinkedHashMap<>();
private FederationSqlEngine engine;
/**
* 创建六个独立 H2 JDBC 数据源。
*/
@Before
public void setUp() throws Exception {
for (int index = 1; index <= 6; index++) {
SourceId sourceId = new SourceId("source-" + index);
JdbcDataSource dataSource = dataSource("six-source-" + index);
execute(dataSource,
"DROP TABLE IF EXISTS SUBJECT_METRIC",
"CREATE TABLE SUBJECT_METRIC (SUBJECT_ID INT PRIMARY KEY, METRIC_VALUE INT NOT NULL)",
"INSERT INTO SUBJECT_METRIC VALUES (1, " + (index * 10)
+ "), (2, " + (index * 100) + ")");
dataSources.put(sourceId, dataSource);
bindings.put("S" + index, FederationSourceBindingDefinition.of(sourceId, 1));
}
RuntimeFingerprint fingerprint = new RuntimeFingerprint(
"H2", "2", "H2 JDBC Driver", "2", "1");
engine = FederationSqlEngines.builder()
.dataSourceResolver(definition -> FederationDataSourceHandles.shared(
dataSources.get(definition.sourceId()), fingerprint))
.adapter(new JdbcFederationSqlAdapterProvider())
.federationExecutionPolicy(SIX_SOURCE_POLICY)
.maximumPlanCacheEntries(32)
.build();
for (SourceId sourceId : dataSources.keySet()) {
engine.sources().apply(definition(sourceId), SourceApplyOptions.prewarmNow());
}
}
/**
* 关闭 Engine。
*/
@After
public void tearDown() {
if (engine != null) {
engine.close();
}
}
/**
* 验证同一六源 Scope 中只引用四源时生成四个 Fragment。
*/
@Test
public void shouldCompileFourReferencedSources() {
FederationSqlPlan plan = engine.compile(SqlCompileRequest.of(
selectSql(4), scope(SIX_SOURCE_POLICY)));
Assert.assertEquals(FederationQueryMode.FEDERATED, plan.queryMode());
Assert.assertEquals(4, plan.referencedSources().size());
Assert.assertEquals(4, plan.fragments().size());
Assert.assertEquals(3, plan.joinOptimizations().size());
}
/**
* 验证六个独立 JDBC 数据源可完成五阶段 Join 并返回稳定结果。
*/
@Test
public void shouldExecuteJoinAcrossSixSources() {
String sql = selectSql(6);
FederationSqlPlan plan = engine.compile(SqlCompileRequest.of(
sql, scope(SIX_SOURCE_POLICY)));
Assert.assertEquals(6, plan.referencedSources().size());
Assert.assertEquals(6, plan.fragments().size());
Assert.assertEquals(5, plan.joinOptimizations().size());
try (FederationResultCursor cursor = engine.query(
SqlQueryCommand.of(sql, scope(SIX_SOURCE_POLICY), List.of()))) {
Assert.assertTrue(cursor.next());
Assert.assertEquals(1, cursor.getObject(1));
Assert.assertEquals(10, cursor.getObject(2));
Assert.assertEquals(60, cursor.getObject(7));
Assert.assertTrue(cursor.next());
Assert.assertEquals(2, cursor.getObject(1));
Assert.assertEquals(600, cursor.getObject(7));
Assert.assertFalse(cursor.next());
Assert.assertEquals(6, cursor.metrics().fragments().size());
}
}
/**
* 验证 Scope 收紧为四源时,六源 SQL 在编译期稳定拒绝。
*/
@Test
public void shouldRejectSixSourcesWhenScopeLimitIsFour() {
FederationExecutionPolicy fourSourcePolicy = new FederationExecutionPolicy(
4, 12, 2, 100_000, 64L * 1024L * 1024L, 60_000);
try {
engine.compile(SqlCompileRequest.of(selectSql(6), scope(fourSourcePolicy)));
Assert.fail("expected source limit rejection");
} catch (FederationSqlException exception) {
Assert.assertEquals(FederationSqlErrorCode.FEDERATION_RESOURCE_LIMIT_EXCEEDED,
exception.errorCode());
}
}
private FederationQueryScopeDefinition scope(FederationExecutionPolicy policy) {
return new FederationQueryScopeDefinition(
"six-source-scope", 1, bindings, "S1", policy);
}
private static String selectSql(int sourceCount) {
StringBuilder sql = new StringBuilder("SELECT s1.SUBJECT_ID");
for (int index = 1; index <= sourceCount; index++) {
sql.append(", s").append(index).append(".METRIC_VALUE");
}
sql.append(" FROM S1.APP.SUBJECT_METRIC s1");
for (int index = 2; index <= sourceCount; index++) {
sql.append(" JOIN S").append(index).append(".APP.SUBJECT_METRIC s")
.append(index).append(" ON s").append(index)
.append(".SUBJECT_ID = s1.SUBJECT_ID");
}
return sql.append(" ORDER BY s1.SUBJECT_ID").toString();
}
private static JdbcDataSource dataSource(String name) {
JdbcDataSource dataSource = new JdbcDataSource();
dataSource.setURL("jdbc:h2:mem:" + name + ";DB_CLOSE_DELAY=-1");
dataSource.setUser("sa");
dataSource.setPassword("");
return dataSource;
}
private static void execute(JdbcDataSource dataSource, String... statements)
throws Exception {
try (Connection connection = dataSource.getConnection();
Statement statement = connection.createStatement()) {
for (String sql : statements) {
statement.execute(sql);
}
}
}
private static FederationSourceDefinition definition(SourceId sourceId) {
return new FederationSourceDefinition(
sourceId, 1, JdbcFederationSqlAdapterProvider.ADAPTER_ID,
List.of(new JdbcSchemaDefinition("APP", null, "PUBLIC")), Map.of());
}
}