EasyFlow-Tech-Dev/EasyFlow
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

perf: 收敛后端资源与健康检查开销

Browse Source 
- 缩小模块扫描范围并显式注册各业务模块自动配置

- 增加可配置线程池、MQ 连接池与消费线程池，降低默认资源占用

- 将 RAG 与分析库中间件探活下沉到健康检查并增加短缓存

- 补齐文档向量库生命周期释放与 SSE 断连清理
This commit is contained in:
陈子默
2026-05-28 11:22:14 +08:00
parent 72df00f25b
commit 11e595b088
43 changed files with 1343 additions and 288 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
easyflow-commons/easyflow-common-analytical-db/pom.xml
View File

@@ -25,6 +25,11 @@
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-autoconfigure</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-actuator</artifactId>
<version>${spring-boot.version}</version>
</dependency>
<dependency>
<groupId>com.clickhouse</groupId>
<artifactId>clickhouse-jdbc</artifactId>

41
easyflow-commons/easyflow-common-analytical-db/src/main/java/tech/easyflow/common/analyticaldb/support/AnalyticalDBHealthIndicator.java Normal file
View File

@@ -0,0 +1,41 @@
package tech.easyflow.common.analyticaldb.support;
import org.springframework.boot.actuate.health.Health;
import org.springframework.boot.actuate.health.HealthIndicator;
import org.springframework.stereotype.Component;
/**
* 分析数据库健康检查。
*/
@Component("analyticalDbHealthIndicator")
public class AnalyticalDBHealthIndicator implements HealthIndicator {
private final AnalyticalDBHealthSupport healthSupport;
/**
* 创建分析数据库健康检查器。
*
* @param healthSupport 分析数据库健康检查支持
*/
public AnalyticalDBHealthIndicator(AnalyticalDBHealthSupport healthSupport) {
this.healthSupport = healthSupport;
}
/**
* 检查分析数据库是否可用。
*
* @return 健康状态
*/
@Override
public Health health() {
if (!healthSupport.enabled()) {
return Health.up().withDetail("enabled", false).build();
}
try {
healthSupport.selfCheck();
return Health.up().withDetail("enabled", true).build();
} catch (Exception e) {
return Health.down(e).withDetail("enabled", true).build();
}
}
}

30
easyflow-commons/easyflow-common-audio/src/main/java/tech/easyflow/common/audio/config/AudioThreadPoolProperties.java Normal file
View File

@@ -0,0 +1,30 @@
package tech.easyflow.common.audio.config;
import org.springframework.boot.context.properties.ConfigurationProperties;
/**
* 音频模块线程池配置。
*/
@ConfigurationProperties(prefix = "easyflow.thread-pool.scheduler")
public class AudioThreadPoolProperties {
private int poolSize = 4;
/**
* 获取调度线程池大小。
*
* @return 调度线程池大小
*/
public int getPoolSize() {
return poolSize;
}
/**
* 设置调度线程池大小。
*
* @param poolSize 调度线程池大小
*/
public void setPoolSize(int poolSize) {
this.poolSize = poolSize;
}
}

21
easyflow-commons/easyflow-common-audio/src/main/java/tech/easyflow/common/audio/socket/SchedulingConfig.java
View File

@@ -1,19 +1,38 @@
package tech.easyflow.common.audio.socket;
import org.springframework.boot.context.properties.EnableConfigurationProperties;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.scheduling.TaskScheduler;
import org.springframework.scheduling.annotation.EnableScheduling;
import org.springframework.scheduling.concurrent.ThreadPoolTaskScheduler;
import tech.easyflow.common.audio.config.AudioThreadPoolProperties;
@Configuration
@EnableScheduling
@EnableConfigurationProperties(AudioThreadPoolProperties.class)
public class SchedulingConfig {
private final AudioThreadPoolProperties properties;
/**
* 创建音频调度配置。
*
* @param properties 音频调度线程池配置
*/
public SchedulingConfig(AudioThreadPoolProperties properties) {
this.properties = properties;
}
/**
* 创建调度线程池。
*
* @return 调度线程池
*/
@Bean
public TaskScheduler taskScheduler() {
ThreadPoolTaskScheduler scheduler = new ThreadPoolTaskScheduler();
scheduler.setPoolSize(10);
scheduler.setPoolSize(properties.getPoolSize());
scheduler.setThreadNamePrefix("scheduled-task-");
scheduler.setDaemon(true);
scheduler.initialize();

5
easyflow-commons/easyflow-common-mq/pom.xml
View File

@@ -22,5 +22,10 @@
<artifactId>jackson-databind</artifactId>
<version>${jackson.version}</version>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-pool2</artifactId>
<version>2.11.1</version>
</dependency>
</dependencies>
</project>

24
easyflow-commons/easyflow-common-mq/src/main/java/tech/easyflow/common/mq/config/MQConfiguration.java
View File

@@ -9,7 +9,9 @@ import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.data.redis.connection.RedisPassword;
import org.springframework.data.redis.connection.RedisStandaloneConfiguration;
import org.springframework.data.redis.connection.lettuce.LettuceClientConfiguration;
import org.springframework.data.redis.connection.lettuce.LettuceConnectionFactory;
import org.springframework.data.redis.connection.lettuce.LettucePoolingClientConfiguration;
import org.springframework.data.redis.core.StringRedisTemplate;
import tech.easyflow.common.mq.core.MQConsumerContainer;
import tech.easyflow.common.mq.core.MQConsumerHandler;
@@ -24,6 +26,10 @@ import tech.easyflow.common.mq.redis.RedisMQProducer;
import tech.easyflow.common.mq.redis.RedisStreamKeySupport;
import tech.easyflow.common.mq.support.MQHealthSupport;
import org.apache.commons.pool2.impl.GenericObjectPoolConfig;
import io.lettuce.core.api.StatefulConnection;
import java.util.List;
@Configuration
@@ -43,11 +49,27 @@ public class MQConfiguration {
if (redisProperties.getPassword() != null) {
configuration.setPassword(RedisPassword.of(redisProperties.getPassword()));
}
LettuceConnectionFactory connectionFactory = new LettuceConnectionFactory(configuration);
LettuceClientConfiguration clientConfiguration = createClientConfiguration(redisProperties, mqProperties);
LettuceConnectionFactory connectionFactory = new LettuceConnectionFactory(configuration, clientConfiguration);
connectionFactory.afterPropertiesSet();
return new MQRedisResources(connectionFactory, new StringRedisTemplate(connectionFactory));
}
private LettuceClientConfiguration createClientConfiguration(RedisProperties redisProperties,
MQProperties mqProperties) {
MQProperties.Redis.Pool pool = mqProperties.getRedis().getPool();
GenericObjectPoolConfig<StatefulConnection<?, ?>> poolConfig = new GenericObjectPoolConfig<>();
poolConfig.setMaxTotal(pool.getMaxActive());
poolConfig.setMaxIdle(pool.getMaxIdle());
poolConfig.setMinIdle(pool.getMinIdle());
LettucePoolingClientConfiguration.LettucePoolingClientConfigurationBuilder builder =
LettucePoolingClientConfiguration.builder().poolConfig(poolConfig);
if (redisProperties.getTimeout() != null) {
builder.commandTimeout(redisProperties.getTimeout());
}
return builder.build();
}
@Bean(name = "mqRedisConnectionFactory", autowireCandidate = false, defaultCandidate = false)
@ConditionalOnProperty(prefix = "easyflow.mq", name = "enabled", havingValue = "true", matchIfMissing = true)
public LettuceConnectionFactory mqRedisConnectionFactory(MQRedisResources mqRedisResources) {

95
easyflow-commons/easyflow-common-mq/src/main/java/tech/easyflow/common/mq/config/MQProperties.java
View File

@@ -40,6 +40,8 @@ public class MQProperties {
private Duration consumerBlockTimeout = Duration.ofMillis(2000);
private Duration pendingClaimIdle = Duration.ofMillis(60000);
private int maxRetry = 16;
private ConsumerExecutor consumerExecutor = new ConsumerExecutor();
private Pool pool = new Pool();
public int getDatabase() {
return database;
@@ -96,5 +98,98 @@ public class MQProperties {
public void setMaxRetry(int maxRetry) {
this.maxRetry = maxRetry;
}
public ConsumerExecutor getConsumerExecutor() {
return consumerExecutor;
}
public void setConsumerExecutor(ConsumerExecutor consumerExecutor) {
this.consumerExecutor = consumerExecutor;
}
public Pool getPool() {
return pool;
}
public void setPool(Pool pool) {
this.pool = pool;
}
/**
* Redis MQ 消费线程池配置。
*/
public static class ConsumerExecutor {
private int coreSize = 4;
private int maxSize = 12;
private int queueCapacity = 64;
private int keepAliveSeconds = 60;
public int getCoreSize() {
return coreSize;
}
public void setCoreSize(int coreSize) {
this.coreSize = coreSize;
}
public int getMaxSize() {
return maxSize;
}
public void setMaxSize(int maxSize) {
this.maxSize = maxSize;
}
public int getQueueCapacity() {
return queueCapacity;
}
public void setQueueCapacity(int queueCapacity) {
this.queueCapacity = queueCapacity;
}
public int getKeepAliveSeconds() {
return keepAliveSeconds;
}
public void setKeepAliveSeconds(int keepAliveSeconds) {
this.keepAliveSeconds = keepAliveSeconds;
}
}
/**
* Redis MQ 连接池配置。
*/
public static class Pool {
private int maxActive = 12;
private int maxIdle = 8;
private int minIdle = 1;
public int getMaxActive() {
return maxActive;
}
public void setMaxActive(int maxActive) {
this.maxActive = maxActive;
}
public int getMaxIdle() {
return maxIdle;
}
public void setMaxIdle(int maxIdle) {
this.maxIdle = maxIdle;
}
public int getMinIdle() {
return minIdle;
}
public void setMinIdle(int minIdle) {
this.minIdle = minIdle;
}
}
}
}

50
easyflow-commons/easyflow-common-mq/src/main/java/tech/easyflow/common/mq/redis/RedisMQConsumerContainer.java
View File

@@ -30,9 +30,11 @@ import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
public class RedisMQConsumerContainer implements MQConsumerContainer, SmartLifecycle {
@@ -45,7 +47,7 @@ public class RedisMQConsumerContainer implements MQConsumerContainer, SmartLifec
private final MQDeadLetterService deadLetterService;
private final RedisStreamKeySupport keySupport;
private final List<MQConsumerHandler> handlers;
private final ExecutorService executorService = Executors.newCachedThreadPool();
private final ExecutorService executorService;
private volatile boolean running;
@@ -63,6 +65,7 @@ public class RedisMQConsumerContainer implements MQConsumerContainer, SmartLifec
this.deadLetterService = deadLetterService;
this.keySupport = keySupport;
this.handlers = handlers;
this.executorService = createExecutor(properties, handlers);
}
@Override
@@ -77,7 +80,12 @@ public class RedisMQConsumerContainer implements MQConsumerContainer, SmartLifec
int currentShard = shard;
LOG.info("启动 MQ 消费线程: topic={}, group={}, shard={}, handler={}",
subscription.getTopic(), subscription.getConsumerGroup(), currentShard, handler.getClass().getSimpleName());
executorService.submit(() -> consumeLoop(handler, subscription, currentShard));
try {
executorService.submit(() -> consumeLoop(handler, subscription, currentShard));
} catch (RuntimeException e) {
running = false;
throw e;
}
}
}
}
@@ -108,6 +116,42 @@ public class RedisMQConsumerContainer implements MQConsumerContainer, SmartLifec
stop();
}
private ExecutorService createExecutor(MQProperties properties, List<MQConsumerHandler> handlers) {
MQProperties.Redis.ConsumerExecutor config = properties.getRedis().getConsumerExecutor();
int consumerTaskCount = handlers.stream()
.map(MQConsumerHandler::subscription)
.filter(Objects::nonNull)
.mapToInt(subscription -> Math.max(subscription.getShardCount(), 1))
.sum();
if (config.getCoreSize() > config.getMaxSize()) {
throw new IllegalStateException("Redis MQ 消费线程池配置错误：core-size 不能大于 max-size");
}
if (consumerTaskCount > config.getMaxSize()) {
throw new IllegalStateException("Redis MQ 消费线程池配置错误：max-size="
+ config.getMaxSize() + " 小于消费循环数 " + consumerTaskCount
+ "，请调大 easyflow.mq.redis.consumer-executor.max-size");
}
int coreSize = Math.max(config.getCoreSize(), consumerTaskCount);
int maxSize = config.getMaxSize();
AtomicInteger threadIndex = new AtomicInteger(1);
ThreadPoolExecutor executor = new ThreadPoolExecutor(
coreSize,
maxSize,
config.getKeepAliveSeconds(),
TimeUnit.SECONDS,
new ArrayBlockingQueue<>(config.getQueueCapacity()),
task -> {
Thread thread = new Thread(task);
thread.setName("redis-mq-consumer-" + threadIndex.getAndIncrement());
thread.setDaemon(false);
return thread;
},
new ThreadPoolExecutor.AbortPolicy()
);
executor.allowCoreThreadTimeOut(true);
return executor;
}
private void consumeLoop(MQConsumerHandler handler, MQSubscription subscription, int shard) {
String streamKey = keySupport.streamKey(subscription.getTopic(), shard);
String consumerName = subscription.getConsumerGroup() + "-" + shard;