feat: 增强多实例分布式部署兼容
- 增加定时任务分布式锁并覆盖 chatlog、文档导入和 Agent HITL 过期扫描 - 增强 Redis MQ 多实例 consumer 标识、pending reclaim 和单条处理能力 - 增加文档导入状态 Redis 广播和 Agent HITL 跨节点路由确认
This commit is contained in:
@@ -0,0 +1,35 @@
|
||||
package tech.easyflow.common.cache;
|
||||
|
||||
import java.lang.annotation.ElementType;
|
||||
import java.lang.annotation.Retention;
|
||||
import java.lang.annotation.RetentionPolicy;
|
||||
import java.lang.annotation.Target;
|
||||
|
||||
/**
|
||||
* Spring 定时任务 Redis 分布式锁。
|
||||
*/
|
||||
@Target(ElementType.METHOD)
|
||||
@Retention(RetentionPolicy.RUNTIME)
|
||||
public @interface DistributedScheduledLock {
|
||||
|
||||
/**
|
||||
* 获取锁使用的 Redis key。
|
||||
*
|
||||
* @return Redis 锁 key
|
||||
*/
|
||||
String key();
|
||||
|
||||
/**
|
||||
* 等待锁的秒数。
|
||||
*
|
||||
* @return 等待锁的秒数
|
||||
*/
|
||||
long waitSeconds() default 0L;
|
||||
|
||||
/**
|
||||
* 锁租约秒数。
|
||||
*
|
||||
* @return 锁租约秒数
|
||||
*/
|
||||
long leaseSeconds() default 300L;
|
||||
}
|
||||
@@ -0,0 +1,111 @@
|
||||
package tech.easyflow.common.cache;
|
||||
|
||||
import org.aspectj.lang.ProceedingJoinPoint;
|
||||
import org.aspectj.lang.annotation.Around;
|
||||
import org.aspectj.lang.annotation.Aspect;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import jakarta.annotation.PreDestroy;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
import java.time.Duration;
|
||||
import java.util.concurrent.Executors;
|
||||
import java.util.concurrent.ScheduledExecutorService;
|
||||
import java.util.concurrent.ScheduledFuture;
|
||||
import java.util.concurrent.ThreadFactory;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
|
||||
/**
|
||||
* 定时任务分布式锁切面。
|
||||
*/
|
||||
@Aspect
|
||||
@Component
|
||||
public class DistributedScheduledLockAspect {
|
||||
|
||||
private static final Logger LOG = LoggerFactory.getLogger(DistributedScheduledLockAspect.class);
|
||||
|
||||
private final RedisLockExecutor redisLockExecutor;
|
||||
private final ScheduledExecutorService renewExecutor;
|
||||
|
||||
/**
|
||||
* 创建定时任务分布式锁切面。
|
||||
*
|
||||
* @param redisLockExecutor Redis 分布式锁执行器
|
||||
*/
|
||||
public DistributedScheduledLockAspect(RedisLockExecutor redisLockExecutor) {
|
||||
this.redisLockExecutor = redisLockExecutor;
|
||||
this.renewExecutor = Executors.newScheduledThreadPool(
|
||||
1,
|
||||
new DistributedScheduledLockThreadFactory()
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* 拦截带分布式调度锁的定时任务。
|
||||
*
|
||||
* @param joinPoint 切点
|
||||
* @param lock 锁注解
|
||||
* @return 原方法返回值；未抢到锁时返回 null
|
||||
* @throws Throwable 原方法执行异常或 Redis 访问异常
|
||||
*/
|
||||
@Around("@annotation(lock)")
|
||||
public Object around(ProceedingJoinPoint joinPoint, DistributedScheduledLock lock) throws Throwable {
|
||||
Duration waitTimeout = Duration.ofSeconds(Math.max(lock.waitSeconds(), 0L));
|
||||
Duration leaseTimeout = Duration.ofSeconds(Math.max(lock.leaseSeconds(), 1L));
|
||||
RedisLockExecutor.LockHandle handle = redisLockExecutor.tryAcquire(lock.key(), waitTimeout, leaseTimeout);
|
||||
if (handle == null) {
|
||||
LOG.info("定时任务分布式锁已被其他实例持有，跳过本轮执行: lockKey={}, method={}",
|
||||
lock.key(), joinPoint.getSignature().toShortString());
|
||||
return null;
|
||||
}
|
||||
ScheduledFuture<?> renewTask = scheduleRenew(lock.key(), handle, leaseTimeout);
|
||||
try {
|
||||
return joinPoint.proceed();
|
||||
} finally {
|
||||
renewTask.cancel(false);
|
||||
handle.release();
|
||||
}
|
||||
}
|
||||
|
||||
private ScheduledFuture<?> scheduleRenew(String lockKey,
|
||||
RedisLockExecutor.LockHandle handle,
|
||||
Duration leaseTimeout) {
|
||||
long renewIntervalMillis = Math.max(leaseTimeout.toMillis() / 3L, 1000L);
|
||||
return renewExecutor.scheduleWithFixedDelay(() -> {
|
||||
if (!handle.renew()) {
|
||||
LOG.warn("定时任务分布式锁续期失败: lockKey={}", lockKey);
|
||||
}
|
||||
}, renewIntervalMillis, renewIntervalMillis, TimeUnit.MILLISECONDS);
|
||||
}
|
||||
|
||||
/**
|
||||
* 关闭调度锁续期线程池。
|
||||
*/
|
||||
@PreDestroy
|
||||
public void destroy() {
|
||||
renewExecutor.shutdownNow();
|
||||
}
|
||||
|
||||
/**
|
||||
* 调度锁续期线程工厂。
|
||||
*/
|
||||
private static final class DistributedScheduledLockThreadFactory implements ThreadFactory {
|
||||
|
||||
private final AtomicInteger index = new AtomicInteger(1);
|
||||
|
||||
/**
|
||||
* 创建续期线程。
|
||||
*
|
||||
* @param runnable 线程任务
|
||||
* @return 续期线程
|
||||
*/
|
||||
@Override
|
||||
public Thread newThread(Runnable runnable) {
|
||||
Thread thread = new Thread(runnable);
|
||||
thread.setName("distributed-scheduled-lock-renew-" + index.getAndIncrement());
|
||||
thread.setDaemon(true);
|
||||
return thread;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -12,6 +12,9 @@ import java.util.Collections;
|
||||
import java.util.UUID;
|
||||
import java.util.function.Supplier;
|
||||
|
||||
/**
|
||||
* Redis 分布式锁执行器。
|
||||
*/
|
||||
@Component
|
||||
public class RedisLockExecutor {
|
||||
|
||||
@@ -42,6 +45,14 @@ public class RedisLockExecutor {
|
||||
@Autowired
|
||||
private StringRedisTemplate stringRedisTemplate;
|
||||
|
||||
/**
|
||||
* 在分布式锁保护下执行无返回任务。
|
||||
*
|
||||
* @param lockKey 锁 key
|
||||
* @param waitTimeout 等待锁的最大时间
|
||||
* @param leaseTimeout 锁租约时间
|
||||
* @param task 业务任务
|
||||
*/
|
||||
public void executeWithLock(String lockKey, Duration waitTimeout, Duration leaseTimeout, Runnable task) {
|
||||
executeWithLock(lockKey, waitTimeout, leaseTimeout, () -> {
|
||||
task.run();
|
||||
@@ -49,6 +60,16 @@ public class RedisLockExecutor {
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* 在分布式锁保护下执行有返回任务。
|
||||
*
|
||||
* @param lockKey 锁 key
|
||||
* @param waitTimeout 等待锁的最大时间
|
||||
* @param leaseTimeout 锁租约时间
|
||||
* @param task 业务任务
|
||||
* @param <T> 返回类型
|
||||
* @return 任务返回值
|
||||
*/
|
||||
public <T> T executeWithLock(String lockKey, Duration waitTimeout, Duration leaseTimeout, Supplier<T> task) {
|
||||
LockHandle handle = acquire(lockKey, waitTimeout, leaseTimeout);
|
||||
try {
|
||||
@@ -70,24 +91,46 @@ public class RedisLockExecutor {
|
||||
* @return 锁句柄
|
||||
*/
|
||||
public LockHandle acquire(String lockKey, Duration waitTimeout, Duration leaseTimeout) {
|
||||
LockHandle handle = tryAcquire(lockKey, waitTimeout, leaseTimeout);
|
||||
if (handle == null) {
|
||||
throw new IllegalStateException("获取分布式锁失败，请稍后重试，lockKey=" + lockKey);
|
||||
}
|
||||
return handle;
|
||||
}
|
||||
|
||||
/**
|
||||
* 尝试获取显式释放的分布式锁句柄。
|
||||
*
|
||||
* <p>返回 {@code null} 表示锁当前被其他节点持有。Redis 访问失败或等待过程被中断仍会抛出异常，
|
||||
* 调用方可据此区分“正常跳过”和“基础设施异常”。</p>
|
||||
*
|
||||
* @param lockKey 锁 key
|
||||
* @param waitTimeout 等待时间
|
||||
* @param leaseTimeout 租约时间
|
||||
* @return 获取成功时返回锁句柄，否则返回 null
|
||||
*/
|
||||
public LockHandle tryAcquire(String lockKey, Duration waitTimeout, Duration leaseTimeout) {
|
||||
String lockValue = UUID.randomUUID().toString();
|
||||
boolean acquired = false;
|
||||
long deadline = System.nanoTime() + waitTimeout.toNanos();
|
||||
try {
|
||||
while (System.nanoTime() <= deadline) {
|
||||
do {
|
||||
Boolean success = stringRedisTemplate.opsForValue().setIfAbsent(lockKey, lockValue, leaseTimeout);
|
||||
if (Boolean.TRUE.equals(success)) {
|
||||
acquired = true;
|
||||
break;
|
||||
}
|
||||
if (System.nanoTime() >= deadline) {
|
||||
break;
|
||||
}
|
||||
Thread.sleep(RETRY_INTERVAL_MILLIS);
|
||||
}
|
||||
} while (System.nanoTime() <= deadline);
|
||||
} catch (InterruptedException e) {
|
||||
Thread.currentThread().interrupt();
|
||||
throw new IllegalStateException("等待分布式锁被中断，lockKey=" + lockKey, e);
|
||||
}
|
||||
if (!acquired) {
|
||||
throw new IllegalStateException("获取分布式锁失败，请稍后重试，lockKey=" + lockKey);
|
||||
return null;
|
||||
}
|
||||
return new LockHandle(lockKey, lockValue, leaseTimeout);
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user