feat: 增强多实例分布式部署兼容
- 增加定时任务分布式锁并覆盖 chatlog、文档导入和 Agent HITL 过期扫描 - 增强 Redis MQ 多实例 consumer 标识、pending reclaim 和单条处理能力 - 增加文档导入状态 Redis 广播和 Agent HITL 跨节点路由确认
This commit is contained in:
@@ -5,11 +5,13 @@ import org.springframework.boot.autoconfigure.AutoConfiguration;
|
||||
import org.springframework.boot.context.properties.EnableConfigurationProperties;
|
||||
import org.springframework.context.annotation.ComponentScan;
|
||||
import tech.easyflow.ai.documentimport.task.DocumentImportParseMonitorProperties;
|
||||
import tech.easyflow.ai.documentimport.task.DocumentImportStatusBroadcastProperties;
|
||||
|
||||
@MapperScan("tech.easyflow.ai.mapper")
|
||||
@ComponentScan("tech.easyflow.ai")
|
||||
@EnableConfigurationProperties({
|
||||
DocumentImportParseMonitorProperties.class,
|
||||
DocumentImportStatusBroadcastProperties.class,
|
||||
RagHealthProperties.class
|
||||
})
|
||||
@AutoConfiguration
|
||||
|
||||
@@ -2,6 +2,7 @@ package tech.easyflow.ai.documentimport.task;
|
||||
|
||||
import org.springframework.scheduling.annotation.Scheduled;
|
||||
import org.springframework.stereotype.Component;
|
||||
import tech.easyflow.common.cache.DistributedScheduledLock;
|
||||
|
||||
/**
|
||||
* 知识库文档解析任务收敛器。
|
||||
@@ -27,6 +28,7 @@ public class DocumentImportParseMonitor {
|
||||
fixedDelayString = "${easyflow.ai.document-import.parse-monitor.fixed-delay:10000}",
|
||||
initialDelayString = "${easyflow.ai.document-import.parse-monitor.initial-delay:10000}"
|
||||
)
|
||||
@DistributedScheduledLock(key = "easyflow:schedule:document-import:parse-monitor", leaseSeconds = 300L)
|
||||
public void reconcileRunningParseTasks() {
|
||||
appService.monitorRunningParseTasks();
|
||||
}
|
||||
|
||||
@@ -0,0 +1,79 @@
|
||||
package tech.easyflow.ai.documentimport.task;
|
||||
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.context.annotation.Bean;
|
||||
import org.springframework.context.annotation.Configuration;
|
||||
import org.springframework.data.redis.connection.Message;
|
||||
import org.springframework.data.redis.connection.MessageListener;
|
||||
import org.springframework.data.redis.connection.RedisConnectionFactory;
|
||||
import org.springframework.data.redis.listener.ChannelTopic;
|
||||
import org.springframework.data.redis.listener.RedisMessageListenerContainer;
|
||||
|
||||
import java.math.BigInteger;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
|
||||
/**
|
||||
* 文档导入状态 Redis 广播配置。
|
||||
*/
|
||||
@Configuration
|
||||
public class DocumentImportStatusBroadcastConfig {
|
||||
|
||||
private static final Logger LOG = LoggerFactory.getLogger(DocumentImportStatusBroadcastConfig.class);
|
||||
|
||||
/**
|
||||
* 创建文档导入状态广播监听容器。
|
||||
*
|
||||
* @param connectionFactory Redis 连接工厂
|
||||
* @param streamService 文档导入状态流服务
|
||||
* @param properties 文档导入监控配置
|
||||
* @return Redis 消息监听容器
|
||||
*/
|
||||
@Bean
|
||||
public RedisMessageListenerContainer documentImportStatusListenerContainer(
|
||||
RedisConnectionFactory connectionFactory,
|
||||
DocumentImportTaskStatusStreamService streamService,
|
||||
DocumentImportStatusBroadcastProperties properties
|
||||
) {
|
||||
RedisMessageListenerContainer container = new RedisMessageListenerContainer();
|
||||
container.setConnectionFactory(connectionFactory);
|
||||
container.addMessageListener(
|
||||
new DocumentImportStatusMessageListener(streamService),
|
||||
new ChannelTopic(properties.getStatusBroadcastChannel())
|
||||
);
|
||||
return container;
|
||||
}
|
||||
|
||||
/**
|
||||
* 文档导入状态广播监听器。
|
||||
*/
|
||||
private static final class DocumentImportStatusMessageListener implements MessageListener {
|
||||
|
||||
private final DocumentImportTaskStatusStreamService streamService;
|
||||
|
||||
/**
|
||||
* 创建监听器。
|
||||
*
|
||||
* @param streamService 文档导入状态流服务
|
||||
*/
|
||||
private DocumentImportStatusMessageListener(DocumentImportTaskStatusStreamService streamService) {
|
||||
this.streamService = streamService;
|
||||
}
|
||||
|
||||
/**
|
||||
* 处理 Redis 广播消息。
|
||||
*
|
||||
* @param message 消息
|
||||
* @param pattern 订阅模式
|
||||
*/
|
||||
@Override
|
||||
public void onMessage(Message message, byte[] pattern) {
|
||||
String payload = new String(message.getBody(), StandardCharsets.UTF_8);
|
||||
try {
|
||||
streamService.publishLocal(new BigInteger(payload));
|
||||
} catch (RuntimeException e) {
|
||||
LOG.warn("处理文档导入状态广播失败: payload={}", payload, e);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,34 @@
|
||||
package tech.easyflow.ai.documentimport.task;
|
||||
|
||||
import org.springframework.boot.context.properties.ConfigurationProperties;
|
||||
|
||||
/**
|
||||
* 文档导入状态广播配置。
|
||||
*/
|
||||
@ConfigurationProperties(prefix = "easyflow.ai.document-import")
|
||||
public class DocumentImportStatusBroadcastProperties {
|
||||
|
||||
private String statusBroadcastChannel = "easyflow:document-import:status";
|
||||
|
||||
/**
|
||||
* 获取文档导入状态广播通道。
|
||||
*
|
||||
* @return Redis 广播通道
|
||||
*/
|
||||
public String getStatusBroadcastChannel() {
|
||||
return statusBroadcastChannel;
|
||||
}
|
||||
|
||||
/**
|
||||
* 设置文档导入状态广播通道。
|
||||
*
|
||||
* @param statusBroadcastChannel Redis 广播通道
|
||||
*/
|
||||
public void setStatusBroadcastChannel(String statusBroadcastChannel) {
|
||||
if (statusBroadcastChannel == null || statusBroadcastChannel.trim().isEmpty()) {
|
||||
this.statusBroadcastChannel = "easyflow:document-import:status";
|
||||
return;
|
||||
}
|
||||
this.statusBroadcastChannel = statusBroadcastChannel.trim();
|
||||
}
|
||||
}
|
||||
@@ -1,6 +1,7 @@
|
||||
package tech.easyflow.ai.documentimport.task;
|
||||
|
||||
import org.springframework.http.MediaType;
|
||||
import org.springframework.data.redis.core.StringRedisTemplate;
|
||||
import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.springframework.transaction.support.TransactionSynchronization;
|
||||
@@ -43,6 +44,12 @@ public class DocumentImportTaskStatusStreamService {
|
||||
@Resource(name = "sseThreadPool")
|
||||
private ThreadPoolTaskExecutor sseThreadPool;
|
||||
|
||||
@Resource
|
||||
private StringRedisTemplate stringRedisTemplate;
|
||||
|
||||
@Resource
|
||||
private DocumentImportStatusBroadcastProperties statusBroadcastProperties;
|
||||
|
||||
/**
|
||||
* 订阅知识库文档任务状态流。
|
||||
*
|
||||
@@ -75,7 +82,7 @@ public class DocumentImportTaskStatusStreamService {
|
||||
if (documentId == null) {
|
||||
return;
|
||||
}
|
||||
Runnable publishAction = () -> publishNow(documentId);
|
||||
Runnable publishAction = () -> publishStatusChange(documentId);
|
||||
if (TransactionSynchronizationManager.isSynchronizationActive()
|
||||
&& TransactionSynchronizationManager.isActualTransactionActive()) {
|
||||
TransactionSynchronizationManager.registerSynchronization(new TransactionSynchronization() {
|
||||
@@ -89,7 +96,22 @@ public class DocumentImportTaskStatusStreamService {
|
||||
publishAction.run();
|
||||
}
|
||||
|
||||
private void publishNow(BigInteger documentId) {
|
||||
/**
|
||||
* 处理 Redis 广播收到的文档状态变更。
|
||||
*
|
||||
* @param documentId 文档 ID
|
||||
*/
|
||||
public void publishLocal(BigInteger documentId) {
|
||||
publishNow(documentId);
|
||||
}
|
||||
|
||||
private void publishStatusChange(BigInteger documentId) {
|
||||
// 先推送本机连接,降低单机部署和广播链路延迟。
|
||||
publishNow(documentId);
|
||||
stringRedisTemplate.convertAndSend(statusBroadcastProperties.getStatusBroadcastChannel(), documentId.toString());
|
||||
}
|
||||
|
||||
void publishNow(BigInteger documentId) {
|
||||
Document document = documentMapper.selectOneById(documentId);
|
||||
if (document == null || document.getCollectionId() == null) {
|
||||
return;
|
||||
|
||||
Reference in New Issue
Block a user