feat: 切换定时任务至分布式调度底座

- 以执行账本和有界 Worker 承载重负载任务与故障接管

- 接入统一调度 Starter 并增加 MySQL 迁移、指标和回归测试
This commit is contained in:
2026-08-31 14:57:08 +08:00
parent 17ef189862
commit 8c174e5c02
66 changed files with 5348 additions and 545 deletions

View File

@@ -2,7 +2,7 @@ package tech.easyflow.ai.service;
import org.springframework.stereotype.Service;
import tech.easyflow.ai.entity.Workflow;
import tech.easyflow.common.constant.enums.EnumDataStatus;
import tech.easyflow.ai.enums.PublishStatus;
import tech.easyflow.common.entity.LoginAccount;
import tech.easyflow.common.web.exceptions.BusinessException;
import tech.easyflow.system.enums.CategoryResourceType;
@@ -15,7 +15,7 @@ import java.util.Objects;
/**
* 工作流使用权限校验服务。
*
* <p>统一封装工作流存在性、租户、启用状态和资源使用权限校验,供页面能力和后台任务复用。</p>
* <p>统一封装工作流存在性、租户、发布快照和资源使用权限校验,供页面能力和后台任务复用。</p>
*/
@Service
public class WorkflowUsageAuthorizationService {
@@ -40,20 +40,20 @@ public class WorkflowUsageAuthorizationService {
}
/**
* 获取当前账号可使用的启用工作流。
* 获取当前账号可使用的已发布工作流视图
*
* @param workflowId 工作流 ID
* @param account 使用工作流的账号
* @param denyMessage 校验失败提示
* @return 可使用的工作流
* @throws BusinessException 工作流不存在、未启用、跨租户或无使用权限时抛出
* @return 可使用的工作流发布视图
* @throws BusinessException 工作流不存在、未发布、缺少发布快照、跨租户或无使用权限时抛出
*/
public Workflow requireUsableWorkflow(
BigInteger workflowId,
LoginAccount account,
String denyMessage) {
String message = denyMessage == null || denyMessage.isBlank()
? "工作流不存在、已禁用或无权使用"
? "工作流不存在、未发布或无权使用"
: denyMessage;
if (workflowId == null || account == null || account.getId() == null
|| account.getTenantId() == null) {
@@ -62,7 +62,9 @@ public class WorkflowUsageAuthorizationService {
Workflow workflow = workflowService.getById(workflowId);
boolean usable = workflow != null
&& Objects.equals(workflow.getTenantId(), account.getTenantId())
&& EnumDataStatus.AVAILABLE.getCode().equals(workflow.getStatus())
&& PublishStatus.PUBLISHED.getCode().equals(workflow.getPublishStatus())
&& workflow.getPublishedSnapshotJson() != null
&& !workflow.getPublishedSnapshotJson().isEmpty()
&& resourceAccessService.canAccess(
account,
CategoryResourceType.WORKFLOW,
@@ -71,6 +73,10 @@ public class WorkflowUsageAuthorizationService {
if (!usable) {
throw new BusinessException(403, 403, message);
}
return workflow;
Workflow published = workflowService.toPublishedView(workflow);
if (published == null) {
throw new BusinessException(403, 403, message);
}
return published;
}
}

View File

@@ -3,7 +3,7 @@ package tech.easyflow.ai.service;
import org.junit.Assert;
import org.junit.Test;
import tech.easyflow.ai.entity.Workflow;
import tech.easyflow.common.constant.enums.EnumDataStatus;
import tech.easyflow.ai.enums.PublishStatus;
import tech.easyflow.common.entity.LoginAccount;
import tech.easyflow.common.web.exceptions.BusinessException;
import tech.easyflow.system.enums.CategoryResourceType;
@@ -11,6 +11,7 @@ import tech.easyflow.system.enums.ResourceAction;
import tech.easyflow.system.service.ResourceAccessService;
import java.math.BigInteger;
import java.util.Map;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
@@ -21,14 +22,14 @@ import static org.mockito.Mockito.when;
public class WorkflowUsageAuthorizationServiceTest {
/**
* 验证禁用工作流即使资源权限允许也不能被使用。
* 验证未发布工作流即使资源权限允许也不能被使用。
*/
@Test
public void shouldRejectDisabledWorkflow() {
public void shouldRejectUnpublishedWorkflow() {
BigInteger workflowId = BigInteger.valueOf(101);
WorkflowService workflowService = mock(WorkflowService.class);
ResourceAccessService resourceAccessService = mock(ResourceAccessService.class);
Workflow workflow = workflow(workflowId, BigInteger.TEN, EnumDataStatus.UNAVAILABLE.getCode());
Workflow workflow = workflow(workflowId, BigInteger.TEN, PublishStatus.DRAFT, Map.of());
LoginAccount account = account(BigInteger.ONE, BigInteger.TEN);
when(workflowService.getById(workflowId)).thenReturn(workflow);
when(resourceAccessService.canAccess(
@@ -58,7 +59,8 @@ public class WorkflowUsageAuthorizationServiceTest {
Workflow workflow = workflow(
workflowId,
BigInteger.valueOf(20),
EnumDataStatus.AVAILABLE.getCode());
PublishStatus.PUBLISHED,
Map.of("title", "published"));
LoginAccount account = account(BigInteger.ONE, BigInteger.TEN);
when(workflowService.getById(workflowId)).thenReturn(workflow);
WorkflowUsageAuthorizationService service =
@@ -71,17 +73,50 @@ public class WorkflowUsageAuthorizationServiceTest {
}
/**
* 验证启用、同租户且具有使用权限的工作流可以返回。
* 验证已发布、同租户且具有使用权限的工作流返回发布视图
*/
@Test
public void shouldReturnUsableWorkflow() {
public void shouldReturnPublishedWorkflowView() {
BigInteger workflowId = BigInteger.valueOf(103);
WorkflowService workflowService = mock(WorkflowService.class);
ResourceAccessService resourceAccessService = mock(ResourceAccessService.class);
Workflow workflow = workflow(
workflowId,
BigInteger.TEN,
EnumDataStatus.AVAILABLE.getCode());
PublishStatus.PUBLISHED,
Map.of("title", "published"));
Workflow published = new Workflow();
published.setId(workflowId);
published.setTitle("发布版");
LoginAccount account = account(BigInteger.ONE, BigInteger.TEN);
when(workflowService.getById(workflowId)).thenReturn(workflow);
when(resourceAccessService.canAccess(
account,
CategoryResourceType.WORKFLOW,
workflow,
ResourceAction.USE)).thenReturn(true);
when(workflowService.toPublishedView(workflow)).thenReturn(published);
WorkflowUsageAuthorizationService service =
new WorkflowUsageAuthorizationService(workflowService, resourceAccessService);
Workflow result = service.requireUsableWorkflow(workflowId, account, "工作流不可用");
Assert.assertSame(result, published);
}
/**
* 验证发布状态异常但缺少快照的工作流不可被后台任务使用。
*/
@Test
public void shouldRejectPublishedWorkflowWithoutSnapshot() {
BigInteger workflowId = BigInteger.valueOf(104);
WorkflowService workflowService = mock(WorkflowService.class);
ResourceAccessService resourceAccessService = mock(ResourceAccessService.class);
Workflow workflow = workflow(
workflowId,
BigInteger.TEN,
PublishStatus.PUBLISHED,
Map.of());
LoginAccount account = account(BigInteger.ONE, BigInteger.TEN);
when(workflowService.getById(workflowId)).thenReturn(workflow);
when(resourceAccessService.canAccess(
@@ -92,9 +127,10 @@ public class WorkflowUsageAuthorizationServiceTest {
WorkflowUsageAuthorizationService service =
new WorkflowUsageAuthorizationService(workflowService, resourceAccessService);
Workflow result = service.requireUsableWorkflow(workflowId, account, "工作流不可用");
Assert.assertSame(result, workflow);
Assert.assertThrows(
BusinessException.class,
() -> service.requireUsableWorkflow(workflowId, account, "工作流不可用")
);
}
/**
@@ -102,14 +138,18 @@ public class WorkflowUsageAuthorizationServiceTest {
*
* @param id 工作流 ID
* @param tenantId 租户 ID
* @param status 工作流状态
* @param publishStatus 工作流发布状态
* @param snapshot 工作流发布快照
* @return 工作流
*/
private Workflow workflow(BigInteger id, BigInteger tenantId, Integer status) {
private Workflow workflow(BigInteger id, BigInteger tenantId,
PublishStatus publishStatus,
Map<String, Object> snapshot) {
Workflow workflow = new Workflow();
workflow.setId(id);
workflow.setTenantId(tenantId);
workflow.setStatus(status);
workflow.setPublishStatus(publishStatus.getCode());
workflow.setPublishedSnapshotJson(snapshot);
return workflow;
}

View File

@@ -11,13 +11,18 @@
<dependencies>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-quartz</artifactId>
<groupId>com.easyagents</groupId>
<artifactId>easy-agents-scheduler-core</artifactId>
</dependency>
<dependency>
<groupId>com.mybatis-flex</groupId>
<artifactId>mybatis-flex-spring-boot3-starter</artifactId>
</dependency>
<dependency>
<groupId>io.micrometer</groupId>
<artifactId>micrometer-core</artifactId>
<version>1.15.7</version>
</dependency>
<dependency>
<groupId>tech.easyflow</groupId>
<artifactId>easyflow-common-base</artifactId>
@@ -48,6 +53,11 @@
<version>5.12.0</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>com.mysql</groupId>
<artifactId>mysql-connector-j</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>

View File

@@ -2,11 +2,13 @@ package tech.easyflow.job.config;
import org.mybatis.spring.annotation.MapperScan;
import org.springframework.boot.autoconfigure.AutoConfiguration;
import org.springframework.boot.context.properties.EnableConfigurationProperties;
import org.springframework.context.annotation.ComponentScan;
@AutoConfiguration
@MapperScan("tech.easyflow.job.mapper")
@ComponentScan("tech.easyflow.job")
@EnableConfigurationProperties(SysJobExecutionProperties.class)
public class JobModuleConfig {
public JobModuleConfig() {

View File

@@ -0,0 +1,49 @@
package tech.easyflow.job.config;
import org.springframework.beans.factory.InitializingBean;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
/** 防止调度、Worker 与管理 Saga 形成连接池级等待环。 */
@Component
public class SysJobConnectionCapacityValidator implements InitializingBean {
private final SysJobExecutionProperties properties;
private final int poolSize;
private final int quartzThreadCount;
private final boolean schedulerEnabled;
public SysJobConnectionCapacityValidator(
SysJobExecutionProperties properties,
@Value("${spring.datasource.hikari.maximum-pool-size:10}") int poolSize,
@Value("${easy-agents.scheduler.quartz.thread-count:10}") int quartzThreadCount,
@Value("${easy-agents.scheduler.enabled:false}") boolean schedulerEnabled) {
this.properties = properties;
this.poolSize = poolSize;
this.quartzThreadCount = quartzThreadCount;
this.schedulerEnabled = schedulerEnabled;
}
@Override
public void afterPropertiesSet() {
if (!schedulerEnabled || !properties.isEnabled()) return;
int required = requiredPoolSize(
properties.getManagementCommandConcurrency(),
quartzThreadCount,
properties.getWorkerCount(),
properties.getBusinessConnectionReserve());
if (poolSize < required) {
throw new IllegalStateException(
"定时任务连接池容量不足: spring.datasource.hikari.maximum-pool-size="
+ poolSize + ", 至少需要 " + required
+ " (2*management-command-concurrency + quartz.thread-count"
+ " + worker-count + business-connection-reserve)");
}
}
static int requiredPoolSize(int managementConcurrency, int quartzThreads,
int workers, int businessReserve) {
return Math.addExact(Math.multiplyExact(managementConcurrency, 2),
Math.addExact(quartzThreads, Math.addExact(workers, businessReserve)));
}
}

View File

@@ -0,0 +1,80 @@
package tech.easyflow.job.config;
import org.springframework.boot.context.properties.ConfigurationProperties;
import java.time.Duration;
/** 定时任务持久执行器配置。 */
@ConfigurationProperties(prefix = "easyflow.job.execution")
public class SysJobExecutionProperties {
private boolean enabled = true;
private int workerCount = 4;
private int managementCommandConcurrency = 4;
private int businessConnectionReserve = 4;
private Duration pollInterval = Duration.ofMillis(500);
private Duration leaseDuration = Duration.ofMinutes(2);
private Duration heartbeatInterval = Duration.ofSeconds(30);
private Duration retryBackoff = Duration.ofSeconds(5);
private Duration shutdownWaitTimeout = Duration.ofSeconds(30);
private int infrastructureRetryLimit = 16;
private int registrationMaxAttempts = 3;
private Duration registrationRetryDelay = Duration.ofMillis(100);
private int registrationQuartzRefireLimit;
private Duration registrationQuartzRefireDelay = Duration.ofMillis(250);
public boolean isEnabled() { return enabled; }
public void setEnabled(boolean enabled) { this.enabled = enabled; }
public int getWorkerCount() { return workerCount; }
public void setWorkerCount(int value) { this.workerCount = positive(value, "workerCount"); }
public int getManagementCommandConcurrency() { return managementCommandConcurrency; }
public void setManagementCommandConcurrency(int value) {
this.managementCommandConcurrency = positive(value, "managementCommandConcurrency");
}
public int getBusinessConnectionReserve() { return businessConnectionReserve; }
public void setBusinessConnectionReserve(int value) {
this.businessConnectionReserve = positive(value, "businessConnectionReserve");
}
public Duration getPollInterval() { return pollInterval; }
public void setPollInterval(Duration value) { this.pollInterval = positive(value, "pollInterval"); }
public Duration getLeaseDuration() { return leaseDuration; }
public void setLeaseDuration(Duration value) { this.leaseDuration = positive(value, "leaseDuration"); }
public Duration getHeartbeatInterval() { return heartbeatInterval; }
public void setHeartbeatInterval(Duration value) { this.heartbeatInterval = positive(value, "heartbeatInterval"); }
public Duration getRetryBackoff() { return retryBackoff; }
public void setRetryBackoff(Duration value) { this.retryBackoff = positive(value, "retryBackoff"); }
public Duration getShutdownWaitTimeout() { return shutdownWaitTimeout; }
public void setShutdownWaitTimeout(Duration value) { this.shutdownWaitTimeout = positive(value, "shutdownWaitTimeout"); }
public int getInfrastructureRetryLimit() { return infrastructureRetryLimit; }
public void setInfrastructureRetryLimit(int value) { this.infrastructureRetryLimit = positive(value, "infrastructureRetryLimit"); }
public int getRegistrationMaxAttempts() { return registrationMaxAttempts; }
public void setRegistrationMaxAttempts(int value) { this.registrationMaxAttempts = positive(value, "registrationMaxAttempts"); }
public Duration getRegistrationRetryDelay() { return registrationRetryDelay; }
public void setRegistrationRetryDelay(Duration value) { this.registrationRetryDelay = positive(value, "registrationRetryDelay"); }
public int getRegistrationQuartzRefireLimit() { return registrationQuartzRefireLimit; }
public void setRegistrationQuartzRefireLimit(int value) {
if (value < 0) throw new IllegalArgumentException("registrationQuartzRefireLimit must not be negative");
this.registrationQuartzRefireLimit = value;
}
public Duration getRegistrationQuartzRefireDelay() { return registrationQuartzRefireDelay; }
public void setRegistrationQuartzRefireDelay(Duration value) { this.registrationQuartzRefireDelay = positive(value, "registrationQuartzRefireDelay"); }
public void validate() {
Duration minimumLease = heartbeatInterval.multipliedBy(3);
if (leaseDuration.compareTo(minimumLease) <= 0) {
throw new IllegalStateException(
"leaseDuration must be greater than three times heartbeatInterval");
}
}
private static int positive(int value, String name) {
if (value < 1) throw new IllegalArgumentException(name + " must be positive");
return value;
}
private static Duration positive(Duration value, String name) {
if (value == null || value.isZero() || value.isNegative()) {
throw new IllegalArgumentException(name + " must be positive");
}
return value;
}
}

View File

@@ -81,6 +81,12 @@ public class SysJobBase extends DateEntity implements Serializable {
@Column(comment = "数据状态")
private Integer status;
/**
* 调度定义代际。每次修改定义或从停止进入运行时递增,用于隔离旧 Quartz fire。
*/
@Column(comment = "调度定义代际")
private Long scheduleGeneration;
/**
* 创建时间
*/
@@ -199,6 +205,14 @@ public class SysJobBase extends DateEntity implements Serializable {
this.status = status;
}
public Long getScheduleGeneration() {
return scheduleGeneration;
}
public void setScheduleGeneration(Long scheduleGeneration) {
this.scheduleGeneration = scheduleGeneration;
}
public Date getCreated() {
return created;
}

View File

@@ -20,24 +20,86 @@ public class SysJobLogBase implements Serializable {
@Id(keyType = KeyType.Generator, value = "snowFlakeId", comment = "主键")
private BigInteger id;
/** 执行幂等键的 SHA-256。 */
@Column(comment = "执行幂等键SHA-256")
private String executionKey;
/**
* 任务ID
*/
@Column(comment = "任务ID")
private BigInteger jobId;
/** 触发所属任务代际。 */
@Column(comment = "触发所属任务代际")
private Long jobGeneration;
@Column(tenantId = true, comment = "租户ID")
private BigInteger tenantId;
@Column(comment = "部门ID")
private BigInteger deptId;
/**
* 任务名称
*/
@Column(comment = "任务名称")
private String jobName;
@Column(comment = "任务类型快照")
private Integer jobType;
/**
* 任务参数
*/
@Column(typeHandler = FastjsonTypeHandler.class, comment = "任务参数")
private Map<String, Object> jobParams;
@Column(typeHandler = FastjsonTypeHandler.class, comment = "任务扩展配置快照")
private Map<String, Object> jobOptions;
@Column(comment = "是否允许并发执行")
private Integer allowConcurrent;
@Column(comment = "触发来源")
private String triggerSource;
@Column(comment = "立即触发调用标识")
private String invocationId;
@Column(comment = "计划触发时间")
private Date scheduledFireTime;
@Column(comment = "实际触发时间")
private Date actualFireTime;
@Column(comment = "Quartz物理触发实例")
private String fireInstanceId;
@Column(comment = "是否为Quartz故障恢复")
private Integer recovering;
@Column(comment = "执行节点")
private String leaseOwner;
@Column(comment = "本轮执行令牌")
private String executionToken;
@Column(comment = "租约到期时间")
private Date leaseUntil;
@Column(comment = "最近续租时间")
private Date heartbeatTime;
@Column(comment = "基础设施恢复次数")
private Integer attemptCount;
@Column(comment = "下次恢复时间")
private Date nextRetryTime;
@Column(comment = "状态版本")
private Long version;
/**
* 执行结果
*/
@@ -88,6 +150,9 @@ public class SysJobLogBase implements Serializable {
this.id = id;
}
public String getExecutionKey() { return executionKey; }
public void setExecutionKey(String executionKey) { this.executionKey = executionKey; }
public BigInteger getJobId() {
return jobId;
}
@@ -96,6 +161,14 @@ public class SysJobLogBase implements Serializable {
this.jobId = jobId;
}
public Long getJobGeneration() { return jobGeneration; }
public void setJobGeneration(Long jobGeneration) { this.jobGeneration = jobGeneration; }
public BigInteger getTenantId() { return tenantId; }
public void setTenantId(BigInteger tenantId) { this.tenantId = tenantId; }
public BigInteger getDeptId() { return deptId; }
public void setDeptId(BigInteger deptId) { this.deptId = deptId; }
public String getJobName() {
return jobName;
}
@@ -104,6 +177,9 @@ public class SysJobLogBase implements Serializable {
this.jobName = jobName;
}
public Integer getJobType() { return jobType; }
public void setJobType(Integer jobType) { this.jobType = jobType; }
public Map<String, Object> getJobParams() {
return jobParams;
}
@@ -112,6 +188,37 @@ public class SysJobLogBase implements Serializable {
this.jobParams = jobParams;
}
public Map<String, Object> getJobOptions() { return jobOptions; }
public void setJobOptions(Map<String, Object> jobOptions) { this.jobOptions = jobOptions; }
public Integer getAllowConcurrent() { return allowConcurrent; }
public void setAllowConcurrent(Integer allowConcurrent) { this.allowConcurrent = allowConcurrent; }
public String getTriggerSource() { return triggerSource; }
public void setTriggerSource(String triggerSource) { this.triggerSource = triggerSource; }
public String getInvocationId() { return invocationId; }
public void setInvocationId(String invocationId) { this.invocationId = invocationId; }
public Date getScheduledFireTime() { return scheduledFireTime; }
public void setScheduledFireTime(Date scheduledFireTime) { this.scheduledFireTime = scheduledFireTime; }
public Date getActualFireTime() { return actualFireTime; }
public void setActualFireTime(Date actualFireTime) { this.actualFireTime = actualFireTime; }
public String getFireInstanceId() { return fireInstanceId; }
public void setFireInstanceId(String fireInstanceId) { this.fireInstanceId = fireInstanceId; }
public Integer getRecovering() { return recovering; }
public void setRecovering(Integer recovering) { this.recovering = recovering; }
public String getLeaseOwner() { return leaseOwner; }
public void setLeaseOwner(String leaseOwner) { this.leaseOwner = leaseOwner; }
public String getExecutionToken() { return executionToken; }
public void setExecutionToken(String executionToken) { this.executionToken = executionToken; }
public Date getLeaseUntil() { return leaseUntil; }
public void setLeaseUntil(Date leaseUntil) { this.leaseUntil = leaseUntil; }
public Date getHeartbeatTime() { return heartbeatTime; }
public void setHeartbeatTime(Date heartbeatTime) { this.heartbeatTime = heartbeatTime; }
public Integer getAttemptCount() { return attemptCount; }
public void setAttemptCount(Integer attemptCount) { this.attemptCount = attemptCount; }
public Date getNextRetryTime() { return nextRetryTime; }
public void setNextRetryTime(Date nextRetryTime) { this.nextRetryTime = nextRetryTime; }
public Long getVersion() { return version; }
public void setVersion(Long version) { this.version = version; }
public String getJobResult() {
return jobResult;
}

View File

@@ -0,0 +1,7 @@
package tech.easyflow.job.execution;
import tech.easyflow.job.entity.SysJobLog;
/** 当前节点持有租约的一次任务执行。 */
public record ClaimedSysJobExecution(SysJobLog execution, String owner, String token) {
}

View File

@@ -0,0 +1,87 @@
package tech.easyflow.job.execution;
import com.easyagents.scheduler.ScheduleFireContext;
import com.easyagents.scheduler.ScheduleHandler;
import com.easyagents.scheduler.ScheduleRefireException;
import org.springframework.dao.DataAccessException;
import org.springframework.dao.DataAccessResourceFailureException;
import org.springframework.dao.RecoverableDataAccessException;
import org.springframework.dao.TransientDataAccessException;
import org.springframework.stereotype.Component;
import org.springframework.transaction.TransactionException;
import tech.easyflow.job.config.SysJobExecutionProperties;
import java.util.concurrent.locks.LockSupport;
/** Quartz 短 Handler只负责把触发写入数据库执行账本。 */
@Component
public class EasyFlowScheduleHandler implements ScheduleHandler {
public static final String CODE = "easyflow.job.execution";
private final SysJobExecutionRegistrar registrar;
private final SysJobExecutionProperties properties;
private final SysJobExecutionMetrics metrics;
public EasyFlowScheduleHandler(SysJobExecutionRegistrar registrar,
SysJobExecutionProperties properties,
SysJobExecutionMetrics metrics) {
this.registrar = registrar;
this.properties = properties;
this.metrics = metrics;
}
@Override
public String code() {
return CODE;
}
@Override
public void execute(ScheduleFireContext context) {
long startedAt = System.nanoTime();
boolean success = false;
try {
executeWithRetry(context);
success = true;
} finally {
metrics.recordRegistration(System.nanoTime() - startedAt, success);
}
}
private void executeWithRetry(ScheduleFireContext context) {
for (int attempt = 1; ; attempt++) {
try {
registrar.register(context);
return;
} catch (RuntimeException exception) {
if (!isRetryable(exception)) {
throw exception;
}
if (attempt >= properties.getRegistrationMaxAttempts()) {
throw refire(exception);
}
LockSupport.parkNanos(properties.getRegistrationRetryDelay().toNanos());
if (Thread.currentThread().isInterrupted()) {
Thread.currentThread().interrupt();
throw refire(exception);
}
}
}
}
private ScheduleRefireException refire(RuntimeException exception) {
return new ScheduleRefireException(
"定时任务触发登记失败,请求 Quartz 保留本次触发并重新执行",
exception,
properties.getRegistrationQuartzRefireLimit(),
properties.getRegistrationQuartzRefireDelay());
}
private static boolean isRetryable(RuntimeException exception) {
if (exception instanceof TransactionException) return true;
if (!(exception instanceof DataAccessException dataAccessException)) return false;
return dataAccessException instanceof TransientDataAccessException
|| dataAccessException instanceof RecoverableDataAccessException
|| dataAccessException instanceof DataAccessResourceFailureException;
}
}

View File

@@ -0,0 +1,8 @@
package tech.easyflow.job.execution;
/** 任务在领取后、业务执行前已不再满足运行条件。 */
public class SysJobCancelledException extends RuntimeException {
public SysJobCancelledException(String message) {
super(message);
}
}

View File

@@ -0,0 +1,16 @@
package tech.easyflow.job.execution;
import java.math.BigInteger;
import java.time.Instant;
/** 可供三类业务执行器读取的稳定幂等上下文。 */
public record SysJobExecutionContext(
BigInteger executionId,
String executionKey,
BigInteger tenantId,
int attempt,
String triggerSource,
String invocationId,
Instant scheduledFireTime
) {
}

View File

@@ -0,0 +1,28 @@
package tech.easyflow.job.execution;
import java.util.Optional;
/** 当前 Worker 线程的任务幂等上下文。 */
public final class SysJobExecutionContextHolder {
private static final ThreadLocal<SysJobExecutionContext> CURRENT = new ThreadLocal<>();
private SysJobExecutionContextHolder() {
}
/** 返回当前执行上下文;异步派生线程需要由业务显式传递。 */
public static Optional<SysJobExecutionContext> current() {
return Optional.ofNullable(CURRENT.get());
}
static void set(SysJobExecutionContext context) {
if (CURRENT.get() != null) {
throw new IllegalStateException("定时任务执行上下文不允许嵌套");
}
CURRENT.set(context);
}
static void clear() {
CURRENT.remove();
}
}

View File

@@ -0,0 +1,105 @@
package tech.easyflow.job.execution;
import io.micrometer.core.instrument.Counter;
import io.micrometer.core.instrument.Gauge;
import io.micrometer.core.instrument.MeterRegistry;
import io.micrometer.core.instrument.Timer;
import org.springframework.stereotype.Component;
import java.util.Map;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicLong;
/** 定时任务登记、队列、租约和执行指标。 */
@Component
public class SysJobExecutionMetrics {
private final Timer registrationTimer;
private final Timer executionTimer;
private final Counter registrationFailures;
private final Counter claimConflicts;
private final Counter leaseTakeovers;
private final Counter lostLeases;
private final Map<String, Counter> terminalCounters;
private final AtomicLong pending = new AtomicLong();
private final AtomicLong running = new AtomicLong();
private final AtomicLong localActive = new AtomicLong();
private final AtomicLong oldestBacklogMillis = new AtomicLong();
public SysJobExecutionMetrics(MeterRegistry registry) {
registrationTimer = Timer.builder("easyflow.job.registration.duration")
.description("Quartz trigger ledger registration duration")
.register(registry);
executionTimer = Timer.builder("easyflow.job.execution.duration")
.description("Business job execution duration")
.register(registry);
registrationFailures = counter(registry, "easyflow.job.registration.failures");
claimConflicts = counter(registry, "easyflow.job.claim.conflicts");
leaseTakeovers = counter(registry, "easyflow.job.lease.takeovers");
lostLeases = counter(registry, "easyflow.job.lease.lost");
terminalCounters = Map.of(
"success", terminalCounter(registry, "success"),
"failure", terminalCounter(registry, "failure"),
"dead", terminalCounter(registry, "dead"),
"cancelled", terminalCounter(registry, "cancelled"));
gauge(registry, "easyflow.job.queue.pending", pending);
gauge(registry, "easyflow.job.queue.running", running);
gauge(registry, "easyflow.job.execution.local_active", localActive);
Gauge.builder("easyflow.job.queue.oldest_backlog_seconds", oldestBacklogMillis,
value -> value.get() / 1_000.0D)
.register(registry);
}
public void recordRegistration(long durationNanos, boolean success) {
registrationTimer.record(durationNanos, TimeUnit.NANOSECONDS);
if (!success) registrationFailures.increment();
}
public void recordClaimConflict() {
claimConflicts.increment();
}
public void recordLeaseTakeover() {
leaseTakeovers.increment();
}
public void recordLostLease() {
lostLeases.increment();
}
public void recordDead() {
terminalCounters.get("dead").increment();
}
public void executionStarted() {
localActive.incrementAndGet();
}
public void executionFinished(String status, long durationNanos) {
localActive.updateAndGet(value -> Math.max(0L, value - 1L));
executionTimer.record(durationNanos, TimeUnit.NANOSECONDS);
// 失租、fencing 拒绝或关闭中断时没有本节点可确认的终态status 合法为 null。
Counter counter = status == null ? null : terminalCounters.get(status);
if (counter != null) counter.increment();
}
public void updateQueue(long pendingCount, long runningCount, long oldestMillis) {
pending.set(pendingCount);
running.set(runningCount);
oldestBacklogMillis.set(Math.max(0L, oldestMillis));
}
private static Counter counter(MeterRegistry registry, String name) {
return Counter.builder(name).register(registry);
}
private static Counter terminalCounter(MeterRegistry registry, String status) {
return Counter.builder("easyflow.job.execution.terminal")
.tag("status", status)
.register(registry);
}
private static void gauge(MeterRegistry registry, String name, AtomicLong value) {
Gauge.builder(name, value, AtomicLong::get).register(registry);
}
}

View File

@@ -0,0 +1,8 @@
package tech.easyflow.job.execution;
import com.easyagents.scheduler.ScheduleFireContext;
/** 将调度触发幂等登记到持久执行账本。 */
public interface SysJobExecutionRegistrar {
void register(ScheduleFireContext context);
}

View File

@@ -0,0 +1,332 @@
package tech.easyflow.job.execution;
import com.easyagents.scheduler.ScheduleFireContext;
import com.mybatisflex.core.tenant.TenantManager;
import org.springframework.dao.ConcurrencyFailureException;
import org.springframework.dao.DuplicateKeyException;
import org.springframework.stereotype.Repository;
import org.springframework.transaction.PlatformTransactionManager;
import org.springframework.transaction.TransactionDefinition;
import org.springframework.transaction.support.TransactionTemplate;
import tech.easyflow.common.constant.enums.EnumJobResult;
import tech.easyflow.common.constant.enums.EnumJobStatus;
import tech.easyflow.job.config.SysJobExecutionProperties;
import tech.easyflow.job.entity.SysJob;
import tech.easyflow.job.entity.SysJobLog;
import tech.easyflow.job.mapper.SysJobLogMapper;
import tech.easyflow.job.mapper.SysJobMapper;
import tech.easyflow.job.job.JobConstant;
import java.math.BigInteger;
import java.nio.charset.StandardCharsets;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.time.Duration;
import java.util.Date;
import java.util.HexFormat;
import java.util.Objects;
import java.util.Optional;
import java.util.UUID;
import java.util.concurrent.ThreadLocalRandom;
import java.util.concurrent.locks.LockSupport;
/** `tb_sys_job_log` 执行账本及跨节点租约协议。 */
@Repository
public class SysJobExecutionStore implements SysJobExecutionRegistrar {
/** 单次认领最多清退的终态候选数,兼顾队首疏通和短事务边界。 */
private static final int MAX_TERMINAL_ROWS_PER_CLAIM = 64;
/** 登记与领取锁序竞争时的短事务重试上限。 */
private static final int MAX_CLAIM_CONCURRENCY_ATTEMPTS = 3;
private final SysJobMapper jobMapper;
private final SysJobLogMapper logMapper;
private final SysJobExecutionProperties properties;
private final SysJobExecutionMetrics metrics;
private final TransactionTemplate requiresNew;
public SysJobExecutionStore(SysJobMapper jobMapper,
SysJobLogMapper logMapper,
SysJobExecutionProperties properties,
SysJobExecutionMetrics metrics,
PlatformTransactionManager transactionManager) {
this.jobMapper = jobMapper;
this.logMapper = logMapper;
this.properties = properties;
this.metrics = metrics;
this.requiresNew = new TransactionTemplate(transactionManager);
this.requiresNew.setPropagationBehavior(TransactionDefinition.PROPAGATION_REQUIRES_NEW);
this.requiresNew.setIsolationLevel(TransactionDefinition.ISOLATION_READ_COMMITTED);
}
@Override
public void register(ScheduleFireContext context) {
BigInteger jobId = parseJobId(context);
long fireGeneration = parseGeneration(context);
String executionKey = executionKey(jobId, fireGeneration, context);
requiresNew.execute(status -> TenantManager.withoutTenantCondition(() -> {
// 管理事务会先锁定任务行再改变启停状态。这里使用同一行锁,确保 Quartz
// 即刻触发时等待管理事务提交,避免读取旧 STOP 后把本次 fire 当成功吞掉。
SysJob job = jobMapper.selectByIdForUpdate(jobId);
if (job == null
|| !Integer.valueOf(EnumJobStatus.RUNNING.getCode()).equals(job.getStatus())
|| !Objects.equals(job.getScheduleGeneration(), fireGeneration)) {
return null;
}
SysJobLog execution = snapshot(job, fireGeneration, context, executionKey);
try {
logMapper.insert(execution);
} catch (DuplicateKeyException duplicate) {
// 只有 execution_key 命中才是可吞掉的幂等重复;主键或其他
// 唯一约束冲突必须上抛,让 Quartz 保留 fire 并进行持久化重试。
if (logMapper.selectIdByExecutionKey(executionKey) == null) {
throw duplicate;
}
}
return null;
}));
}
public Optional<ClaimedSysJobExecution> claimOne(String owner) {
return TenantManager.withoutTenantCondition(() -> {
for (int index = 0; index < MAX_TERMINAL_ROWS_PER_CLAIM; index++) {
ClaimAttempt attempt = executeClaimTransaction(owner);
if (attempt == null) {
return Optional.empty();
}
if (attempt.claim() != null) {
return Optional.of(attempt.claim());
}
if (!attempt.retryImmediately()) {
return Optional.empty();
}
}
return Optional.empty();
});
}
private ClaimAttempt executeClaimTransaction(String owner) {
for (int attempt = 1; attempt <= MAX_CLAIM_CONCURRENCY_ATTEMPTS; attempt++) {
try {
ClaimAttempt expired = claimExpiredIfPresent(owner);
if (expired != null) return expired;
return requiresNew.execute(status -> claimInTransaction(owner, false));
} catch (ConcurrencyFailureException exception) {
if (attempt == MAX_CLAIM_CONCURRENCY_ATTEMPTS) throw exception;
// 短暂抖动后用全新事务重试,不改变 execution也不把可恢复的锁冲突
// 上抛为一次业务失败。
LockSupport.parkNanos(ThreadLocalRandom.current().nextLong(
1_000_000L, 5_000_001L));
}
}
throw new IllegalStateException("定时任务认领重试状态异常");
}
private ClaimAttempt claimExpiredIfPresent(String owner) {
// 空范围 FOR UPDATE 会保留状态索引的末端间隙锁;同一事务再把 PENDING
// 更新为 RUNNING 时会与其他 Worker 互锁,因此只在确认有候选后加锁。
if (logMapper.selectExpiredClaimCandidateIdWithoutLock() == null) return null;
return requiresNew.execute(status -> claimInTransaction(owner, true));
}
private ClaimAttempt claimInTransaction(String owner, boolean expired) {
SysJobLog candidate = expired
? logMapper.selectExpiredClaimCandidate()
: logMapper.selectPendingClaimCandidate();
// 无锁探测到的过期行可能已被其他 Worker 锁住。SKIP LOCKED 此时返回空,
// 不能结束整轮领取,否则一条受阻的过期记录会让 PENDING 队列反复空转。
if (candidate == null) return expired ? null : ClaimAttempt.stop();
if (candidate.getAttemptCount() != null
&& candidate.getAttemptCount() >= properties.getInfrastructureRetryLimit()) {
int marked = expired
? logMapper.markExpiredDead(
candidate.getId(), "执行节点多次失联,已停止自动接管")
: logMapper.markPendingDead(
candidate.getId(), "基础设施多次故障,已停止自动重试");
if (marked == 1) {
metrics.recordDead();
}
return marked == 1 ? ClaimAttempt.retry() : ClaimAttempt.stop();
}
SysJob job = jobMapper.selectByIdForUpdate(candidate.getJobId());
if (job == null
|| !Integer.valueOf(EnumJobStatus.RUNNING.getCode()).equals(job.getStatus())) {
return claimAndCancel(candidate, owner, expired, "任务已停止或删除")
? ClaimAttempt.retry() : ClaimAttempt.stop();
}
if (!Integer.valueOf(1).equals(candidate.getAllowConcurrent())
&& logMapper.selectOtherActiveForUpdate(
candidate.getJobId(), candidate.getId()) != null) {
if (expired) {
logMapper.releaseExpiredForRetry(candidate.getId(),
micros(properties.getRetryBackoff()));
} else {
logMapper.deferPending(candidate.getId(), micros(properties.getRetryBackoff()));
}
metrics.recordClaimConflict();
return ClaimAttempt.stop();
}
String token = UUID.randomUUID().toString();
int claimed = expired
? logMapper.claimExpired(candidate.getId(), owner, token,
micros(properties.getLeaseDuration()))
: logMapper.claimPending(candidate.getId(), owner, token,
micros(properties.getLeaseDuration()));
if (claimed != 1) {
metrics.recordClaimConflict();
return ClaimAttempt.stop();
}
if (expired) metrics.recordLeaseTakeover();
candidate.setLeaseOwner(owner);
candidate.setExecutionToken(token);
candidate.setStatus(EnumJobResult.RUNNING.getCode());
return ClaimAttempt.claimed(new ClaimedSysJobExecution(candidate, owner, token));
}
private boolean claimAndCancel(SysJobLog candidate, String owner, boolean expired, String reason) {
String token = UUID.randomUUID().toString();
int claimed = expired
? logMapper.claimExpired(candidate.getId(), owner, token,
micros(properties.getLeaseDuration()))
: logMapper.claimPending(candidate.getId(), owner, token,
micros(properties.getLeaseDuration()));
if (claimed == 1) {
logMapper.finishOwned(candidate.getId(), owner, token,
EnumJobResult.CANCELLED.getCode(), null, reason);
}
return claimed == 1;
}
public boolean heartbeat(ClaimedSysJobExecution claim) {
return TenantManager.withoutTenantCondition(() -> logMapper.renewLease(
claim.execution().getId(), claim.owner(), claim.token(),
micros(properties.getLeaseDuration())) == 1);
}
public boolean finish(ClaimedSysJobExecution claim, int status, String result, String error) {
return TenantManager.withoutTenantCondition(() -> logMapper.finishOwned(
claim.execution().getId(), claim.owner(), claim.token(), status,
truncate(result, 15000), truncate(error, 4000)) == 1);
}
public boolean releaseForInfrastructureRetry(ClaimedSysJobExecution claim, String error) {
return TenantManager.withoutTenantCondition(() -> logMapper.releaseOwnedForRetry(
claim.execution().getId(), claim.owner(), claim.token(),
micros(properties.getRetryBackoff()), truncate(error, 4000)) == 1);
}
public int cancelPending(BigInteger jobId, String reason) {
Integer cancelled = requiresNew.execute(status -> TenantManager.withoutTenantCondition(() -> {
// 先锁待清理账本,再锁任务行,保持与 Worker 一致的 log -> job 锁序。
logMapper.selectPendingIdsByJobIdForUpdate(jobId);
// 旧 STOP/删除命令失锁后可能与新 START 重叠。锁定并重新读取当前任务状态,
// 只有最终仍为 STOP或已删除时才清理避免误取消新一代待执行记录。
SysJob job = jobMapper.selectByIdForUpdate(jobId);
if (job != null
&& Integer.valueOf(EnumJobStatus.RUNNING.getCode()).equals(job.getStatus())) {
return 0;
}
return logMapper.cancelPendingByJobId(jobId, truncate(reason, 4000));
}));
return cancelled == null ? 0 : cancelled;
}
public SysJobQueueSnapshot queueSnapshot() {
return TenantManager.withoutTenantCondition(() -> new SysJobQueueSnapshot(
logMapper.countPending(),
logMapper.countRunning(),
logMapper.selectOldestPendingMillis()));
}
private SysJobLog snapshot(SysJob job, long fireGeneration, ScheduleFireContext context,
String executionKey) {
SysJobLog log = new SysJobLog();
String source = context.invocationId() == null ? "SCHEDULED" : "MANUAL";
log.setExecutionKey(executionKey);
log.setJobId(job.getId());
log.setJobGeneration(fireGeneration);
log.setTenantId(job.getTenantId());
log.setDeptId(job.getDeptId());
log.setJobName(job.getJobName());
log.setJobType(job.getJobType());
log.setJobParams(job.getJobParams());
log.setJobOptions(job.getOptions());
log.setAllowConcurrent(job.getAllowConcurrent());
log.setTriggerSource(source);
log.setInvocationId(context.invocationId());
log.setScheduledFireTime(Date.from(context.scheduledFireTime()));
log.setActualFireTime(Date.from(context.actualFireTime()));
log.setFireInstanceId(context.fireInstanceId());
log.setRecovering(context.recovering() ? 1 : 0);
log.setAttemptCount(0);
log.setNextRetryTime(Date.from(context.actualFireTime()));
log.setStatus(EnumJobResult.PENDING.getCode());
log.setVersion(0L);
log.setCreated(new Date());
return log;
}
private static String executionKey(BigInteger jobId, long fireGeneration,
ScheduleFireContext context) {
String source = context.invocationId() == null ? "SCHEDULED" : "MANUAL";
String canonical = source + '|' + jobId + '|' + fireGeneration + '|'
+ (context.invocationId() == null
? context.scheduledFireTime().toEpochMilli()
: context.invocationId());
return sha256(canonical);
}
private static BigInteger parseJobId(ScheduleFireContext context) {
try {
return new BigInteger(context.scheduleId().name());
} catch (NumberFormatException exception) {
throw new IllegalArgumentException("非法 EasyFlow 定时任务标识: " + context.scheduleId(), exception);
}
}
private static long parseGeneration(ScheduleFireContext context) {
String value = context.parameters().get(JobConstant.SCHEDULE_GENERATION);
try {
long generation = Long.parseLong(value);
if (generation < 0L) throw new NumberFormatException("negative generation");
return generation;
} catch (NumberFormatException | NullPointerException exception) {
throw new IllegalArgumentException(
"非法 EasyFlow 定时任务调度代际: " + value, exception);
}
}
private static String sha256(String value) {
try {
byte[] digest = MessageDigest.getInstance("SHA-256")
.digest(value.getBytes(StandardCharsets.UTF_8));
return HexFormat.of().formatHex(digest);
} catch (NoSuchAlgorithmException exception) {
throw new IllegalStateException("JDK 缺少 SHA-256", exception);
}
}
private static String truncate(String value, int maxLength) {
return value == null || value.length() <= maxLength ? value : value.substring(0, maxLength);
}
private static long micros(Duration duration) {
return Math.max(1L, duration.toNanos() / 1_000L);
}
private record ClaimAttempt(ClaimedSysJobExecution claim, boolean retryImmediately) {
private static ClaimAttempt claimed(ClaimedSysJobExecution claim) {
return new ClaimAttempt(claim, false);
}
private static ClaimAttempt retry() {
return new ClaimAttempt(null, true);
}
private static ClaimAttempt stop() {
return new ClaimAttempt(null, false);
}
}
}

View File

@@ -0,0 +1,372 @@
package tech.easyflow.job.execution;
import cn.hutool.core.exceptions.ExceptionUtil;
import com.alibaba.fastjson2.JSON;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.context.SmartLifecycle;
import org.springframework.stereotype.Component;
import tech.easyflow.common.constant.enums.EnumJobResult;
import tech.easyflow.job.config.SysJobExecutionProperties;
import tech.easyflow.job.entity.SysJob;
import tech.easyflow.job.entity.SysJobLog;
import java.lang.management.ManagementFactory;
import java.util.Map;
import java.util.Optional;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.ThreadLocalRandom;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.locks.LockSupport;
/** 固定并发、数据库驱动的重任务 Worker。 */
@Component
public class SysJobExecutionWorker implements SmartLifecycle {
private static final Logger log = LoggerFactory.getLogger(SysJobExecutionWorker.class);
private final SysJobExecutionStore store;
private final SysJobInvoker invoker;
private final SysJobExecutionProperties properties;
private final SysJobExecutionMetrics metrics;
private final String owner = buildOwner();
private final AtomicBoolean running = new AtomicBoolean();
private final Map<String, ActiveExecution> active = new ConcurrentHashMap<>();
private ExecutorService workers;
private ScheduledExecutorService heartbeat;
public SysJobExecutionWorker(SysJobExecutionStore store,
SysJobInvoker invoker,
SysJobExecutionProperties properties,
SysJobExecutionMetrics metrics) {
this.store = store;
this.invoker = invoker;
this.properties = properties;
this.metrics = metrics;
}
@Override
public synchronized void start() {
if (!properties.isEnabled()) return;
properties.validate();
if (!running.compareAndSet(false, true)) return;
AtomicInteger workerNumber = new AtomicInteger();
workers = Executors.newFixedThreadPool(properties.getWorkerCount(), runnable -> {
Thread thread = new Thread(runnable,
"easyflow-job-worker-" + workerNumber.incrementAndGet());
// 正常关闭仍等待在途任务;超时且业务代码忽略中断时不阻塞 JVM 退出,
// 未完成记录由数据库租约交给其他节点接管。
thread.setDaemon(true);
return thread;
});
heartbeat = Executors.newSingleThreadScheduledExecutor(runnable -> {
Thread thread = new Thread(runnable, "easyflow-job-heartbeat");
thread.setDaemon(true);
return thread;
});
for (int i = 0; i < properties.getWorkerCount(); i++) workers.submit(this::workerLoop);
refreshQueueMetrics();
heartbeat.scheduleWithFixedDelay(this::heartbeatAll,
properties.getHeartbeatInterval().toMillis(),
properties.getHeartbeatInterval().toMillis(), TimeUnit.MILLISECONDS);
log.info("定时任务 Worker 已启动: owner={}, workers={}", owner, properties.getWorkerCount());
}
private void workerLoop() {
int consecutiveFailures = 0;
while (running.get()) {
try {
Optional<ClaimedSysJobExecution> claim = store.claimOne(owner);
consecutiveFailures = 0;
if (claim.isPresent()) {
execute(claim.get());
if (running.get()) Thread.interrupted();
}
else idle();
} catch (RuntimeException exception) {
log.error("定时任务 Worker 领取执行记录失败", exception);
failureBackoff(++consecutiveFailures);
}
}
}
private void execute(ClaimedSysJobExecution claim) {
ActiveExecution activeExecution = new ActiveExecution(claim, Thread.currentThread());
active.put(claim.token(), activeExecution);
long startedAt = System.nanoTime();
String terminalMetric = null;
boolean metricStarted = false;
try {
SysJobExecutionContextHolder.set(toExecutionContext(claim.execution()));
metricStarted = recordExecutionStarted(claim.execution().getId());
terminalMetric = invokeAndFinish(claim, activeExecution);
} finally {
// 上下文和活动租约是 Worker 的正确性状态,必须先于可观测性收尾清理。
// 否则指标实现一旦抛错,线程复用后会永久残留上下文,且旧租约会被持续续期。
SysJobExecutionContextHolder.clear();
synchronized (activeExecution) {
active.remove(claim.token(), activeExecution);
}
if (metricStarted) {
recordExecutionFinished(claim.execution().getId(), terminalMetric,
System.nanoTime() - startedAt);
}
}
}
private String invokeAndFinish(ClaimedSysJobExecution claim,
ActiveExecution activeExecution) {
String terminalMetric = null;
try {
Object result = invoker.execute(toSnapshot(claim.execution()));
String serialized = serializeResult(result, claim.execution().getId());
if (finishOwned(claim, EnumJobResult.SUCCESS.getCode(), serialized, null)) {
terminalMetric = "success";
}
} catch (SysJobCancelledException exception) {
if (finishOwned(claim, EnumJobResult.CANCELLED.getCode(), null,
exception.getMessage())) {
terminalMetric = "cancelled";
}
} catch (SysJobInfrastructureException exception) {
String message = ExceptionUtil.getRootCauseMessage(exception);
try {
if (!store.releaseForInfrastructureRetry(claim, message)) {
activeExecution.abandon().set(true);
metrics.recordLostLease();
log.warn("基础设施故障记录释放被 fencing 拒绝: executionId={}",
claim.execution().getId());
}
} catch (RuntimeException releaseFailure) {
activeExecution.abandon().set(true);
metrics.recordLostLease();
log.error("基础设施故障记录释放失败,保留租约等待接管: executionId={}",
claim.execution().getId(), releaseFailure);
}
log.warn("定时任务执行前基础设施故障,已安排重试: executionId={}, jobId={}",
claim.execution().getId(), claim.execution().getJobId(), exception);
} catch (Exception exception) {
terminalMetric = handleBusinessFailure(claim, activeExecution, exception);
} catch (Error error) {
if (error instanceof VirtualMachineError || error instanceof ThreadDeath) {
throw error;
}
// FutureTask 会吞掉 Error 并结束整个永久轮询任务;非 VM Error 必须
// 记账后继续,让固定 workerCount 不因单个业务实现错误永久缩水。
terminalMetric = handleBusinessFailure(claim, activeExecution, error);
}
return terminalMetric;
}
private boolean recordExecutionStarted(java.math.BigInteger executionId) {
try {
metrics.executionStarted();
return true;
} catch (RuntimeException exception) {
log.warn("记录定时任务开始指标失败,不影响业务执行: executionId={}",
executionId, exception);
return false;
}
}
private void recordExecutionFinished(java.math.BigInteger executionId,
String status, long durationNanos) {
try {
metrics.executionFinished(status, durationNanos);
} catch (RuntimeException exception) {
log.warn("记录定时任务完成指标失败,不影响业务终态: executionId={}",
executionId, exception);
}
}
private String handleBusinessFailure(ClaimedSysJobExecution claim,
ActiveExecution activeExecution,
Throwable failure) {
if (activeExecution.abandon().get()) {
log.warn("定时任务因失租或关闭中断,保留租约等待其他节点接管: "
+ "executionId={}, jobId={}",
claim.execution().getId(), claim.execution().getJobId(), failure);
} else {
log.error("定时任务业务执行失败: executionId={}, jobId={}",
claim.execution().getId(), claim.execution().getJobId(), failure);
String message = ExceptionUtil.getRootCauseMessage(failure);
if (finishOwned(claim, EnumJobResult.FAIL.getCode(), null, message)) {
return "failure";
}
}
return null;
}
private void heartbeatAll() {
for (ActiveExecution execution : active.values()) {
if (execution.abandon().get()) continue;
try {
if (!store.heartbeat(execution.claim())) {
if (!abandonCurrent(execution)) continue;
log.warn("定时任务已失去租约,中断旧 Worker: executionId={}",
execution.claim().execution().getId());
metrics.recordLostLease();
}
} catch (RuntimeException exception) {
if (!abandonCurrent(execution)) continue;
log.error("定时任务续租失败,中断执行以降低重复副作用风险: executionId={}",
execution.claim().execution().getId(), exception);
metrics.recordLostLease();
}
}
refreshQueueMetrics();
}
/**
* 仅当执行仍是当前活动项时中断其线程。
*
* <p>Worker 线程会被执行池复用;这里必须与 {@link #execute} 的 finally
* 使用同一监视器,避免旧心跳在活动项移除后误伤同一线程上的下一任务。</p>
*/
private boolean abandonCurrent(ActiveExecution execution) {
synchronized (execution) {
if (!active.remove(execution.claim().token(), execution)) return false;
execution.abandon().set(true);
execution.thread().interrupt();
return true;
}
}
private void idle() {
LockSupport.parkNanos(properties.getPollInterval().toNanos());
}
private void failureBackoff(int consecutiveFailures) {
long base = properties.getPollInterval().toNanos();
int shift = Math.min(6, Math.max(0, consecutiveFailures - 1));
long capped = Math.min(TimeUnit.SECONDS.toNanos(30), base << shift);
long jitter = ThreadLocalRandom.current().nextLong(Math.max(1L, base));
LockSupport.parkNanos(Math.min(TimeUnit.SECONDS.toNanos(30), capped + jitter));
}
private static SysJob toSnapshot(SysJobLog execution) {
SysJob job = new SysJob();
job.setId(execution.getJobId());
job.setTenantId(execution.getTenantId());
job.setDeptId(execution.getDeptId());
job.setJobName(execution.getJobName());
job.setJobType(execution.getJobType());
job.setJobParams(execution.getJobParams());
job.setOptions(execution.getJobOptions());
job.setAllowConcurrent(execution.getAllowConcurrent());
return job;
}
private boolean finishOwned(ClaimedSysJobExecution claim, int status,
String result, String error) {
try {
if (store.finish(claim, status, result, error)) return true;
log.warn("定时任务终态提交被 fencing 拒绝: executionId={}, status={}",
claim.execution().getId(), status);
} catch (RuntimeException exception) {
log.error("定时任务终态提交失败,保留租约等待恢复: executionId={}, status={}",
claim.execution().getId(), status, exception);
}
metrics.recordLostLease();
return false;
}
private static String serializeResult(Object result, java.math.BigInteger executionId) {
if (result == null) return null;
try {
return JSON.toJSONString(result);
} catch (RuntimeException exception) {
log.warn("定时任务结果无法序列化,仅保留成功状态: executionId={}",
executionId, exception);
return null;
}
}
private void refreshQueueMetrics() {
try {
SysJobQueueSnapshot snapshot = store.queueSnapshot();
metrics.updateQueue(snapshot.pending(), snapshot.running(),
snapshot.oldestBacklogMillis());
} catch (RuntimeException exception) {
log.warn("刷新定时任务队列指标失败", exception);
}
}
private static SysJobExecutionContext toExecutionContext(SysJobLog execution) {
return new SysJobExecutionContext(
execution.getId(),
execution.getExecutionKey(),
execution.getTenantId(),
execution.getAttemptCount() == null ? 1 : execution.getAttemptCount() + 1,
execution.getTriggerSource(),
execution.getInvocationId(),
execution.getScheduledFireTime() == null
? null : execution.getScheduledFireTime().toInstant());
}
@Override
public synchronized void stop() {
if (!running.compareAndSet(true, false)) return;
workers.shutdown();
try {
if (!workers.awaitTermination(properties.getShutdownWaitTimeout().toMillis(),
TimeUnit.MILLISECONDS)) {
abandonActiveExecutions();
workers.shutdownNow();
awaitForcedWorkerShutdown();
}
} catch (InterruptedException exception) {
Thread.currentThread().interrupt();
abandonActiveExecutions();
workers.shutdownNow();
awaitForcedWorkerShutdown();
} finally {
heartbeat.shutdownNow();
}
log.info("定时任务 Worker 已停止: owner={}", owner);
}
@Override
public boolean isRunning() {
return running.get();
}
@Override
public int getPhase() {
return Integer.MAX_VALUE - 100;
}
private static String buildOwner() {
String runtime = ManagementFactory.getRuntimeMXBean().getName();
String value = runtime + '-' + Integer.toHexString(System.identityHashCode(SysJobExecutionWorker.class));
return value.substring(0, Math.min(190, value.length()));
}
private void abandonActiveExecutions() {
active.values().forEach(execution -> execution.abandon().set(true));
}
private void awaitForcedWorkerShutdown() {
try {
if (!workers.awaitTermination(
Math.min(1_000L, properties.getShutdownWaitTimeout().toMillis()),
TimeUnit.MILLISECONDS)) {
log.warn("部分定时任务业务代码忽略中断,将由守护线程退出和租约接管收口: owner={}", owner);
}
} catch (InterruptedException exception) {
Thread.currentThread().interrupt();
}
}
private record ActiveExecution(ClaimedSysJobExecution claim, Thread thread,
AtomicBoolean abandon) {
private ActiveExecution(ClaimedSysJobExecution claim, Thread thread) {
this(claim, thread, new AtomicBoolean());
}
}
}

View File

@@ -0,0 +1,11 @@
package tech.easyflow.job.execution;
/**
* 业务副作用开始前发生的可恢复基础设施故障。
*/
public class SysJobInfrastructureException extends RuntimeException {
public SysJobInfrastructureException(String message, Throwable cause) {
super(message, cause);
}
}

View File

@@ -0,0 +1,195 @@
package tech.easyflow.job.execution;
import cn.hutool.core.util.ArrayUtil;
import cn.hutool.core.util.StrUtil;
import com.mybatisflex.core.tenant.TenantManager;
import org.springframework.aop.support.AopUtils;
import org.springframework.context.ApplicationContext;
import org.springframework.dao.DataAccessException;
import org.springframework.dao.DataAccessResourceFailureException;
import org.springframework.dao.RecoverableDataAccessException;
import org.springframework.dao.TransientDataAccessException;
import org.springframework.stereotype.Component;
import tech.easyflow.common.constant.enums.EnumJobStatus;
import tech.easyflow.common.constant.enums.EnumJobType;
import tech.easyflow.job.entity.SysJob;
import tech.easyflow.job.job.JobConstant;
import tech.easyflow.job.mapper.SysJobMapper;
import tech.easyflow.job.service.WorkflowJobExecutionService;
import tech.easyflow.system.entity.SysAccount;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
import java.lang.reflect.Modifier;
import java.util.Map;
import java.util.Objects;
/** 在 Worker 线程中执行一种 EasyFlow 业务任务。 */
@Component
public class SysJobInvoker {
private final SysJobMapper jobMapper;
private final ApplicationContext applicationContext;
private final WorkflowJobExecutionService workflowExecutionService;
private final SysJobOwnerValidator ownerValidator;
public SysJobInvoker(SysJobMapper jobMapper,
ApplicationContext applicationContext,
WorkflowJobExecutionService workflowExecutionService,
SysJobOwnerValidator ownerValidator) {
this.jobMapper = jobMapper;
this.applicationContext = applicationContext;
this.workflowExecutionService = workflowExecutionService;
this.ownerValidator = ownerValidator;
}
public Object execute(SysJob snapshot) throws Exception {
SysJob current;
SysAccount owner;
try {
current = TenantManager.withoutTenantCondition(
() -> jobMapper.selectOneById(snapshot.getId()));
validateCurrent(snapshot, current);
owner = ownerValidator.requireAvailableOwner(current);
} catch (DataAccessException exception) {
throw classifyPreparationFailure("准备定时任务执行上下文失败", exception);
}
if (Integer.valueOf(EnumJobType.TINY_FLOW.getCode()).equals(snapshot.getJobType())) {
return workflowExecutionService.execute(snapshot, current, owner);
}
if (Integer.valueOf(EnumJobType.SPRING_BEAN.getCode()).equals(snapshot.getJobType())) {
return executeSpringBean(snapshot);
}
if (Integer.valueOf(EnumJobType.JAVA_CLASS.getCode()).equals(snapshot.getJobType())) {
return executeJavaClass(snapshot);
}
throw new IllegalArgumentException("不支持的定时任务类型: " + snapshot.getJobType());
}
private static RuntimeException classifyPreparationFailure(
String message, DataAccessException exception) {
if (exception instanceof TransientDataAccessException
|| exception instanceof RecoverableDataAccessException
|| exception instanceof DataAccessResourceFailureException) {
return new SysJobInfrastructureException(message, exception);
}
return exception;
}
private void validateCurrent(SysJob snapshot, SysJob current) {
if (current == null) {
throw new SysJobCancelledException("定时任务已删除id=" + snapshot.getId());
}
if (!Objects.equals(snapshot.getTenantId(), current.getTenantId())) {
throw new SysJobCancelledException("定时任务租户已变化id=" + snapshot.getId());
}
if (!Integer.valueOf(EnumJobStatus.RUNNING.getCode()).equals(current.getStatus())) {
throw new SysJobCancelledException("定时任务已停止id=" + snapshot.getId());
}
if (!Objects.equals(snapshot.getJobType(), current.getJobType())) {
throw new SysJobCancelledException("定时任务类型已变化id=" + snapshot.getId());
}
}
private Object executeSpringBean(SysJob job) throws Exception {
String expression = requiredExpression(job, JobConstant.BEAN_METHOD_KEY);
MethodCall call = parse(expression);
Object bean = applicationContext.getBean(call.owner());
Class<?> targetClass = AopUtils.getTargetClass(bean);
Method targetMethod = publicMethod(targetClass, call, false);
Method invocableMethod = AopUtils.selectInvocableMethod(targetMethod, bean.getClass());
return invoke(bean, invocableMethod, call.arguments());
}
private Object executeJavaClass(SysJob job) throws Exception {
String expression = requiredExpression(job, JobConstant.JAVA_METHOD_KEY);
MethodCall call = parse(expression);
Class<?> type = Class.forName(call.owner());
Object target = type.getDeclaredConstructor().newInstance();
return invoke(target, publicMethod(type, call, true), call.arguments());
}
private static Method publicMethod(Class<?> type, MethodCall call,
boolean declaredOnly) throws NoSuchMethodException {
Method method = declaredOnly
? type.getDeclaredMethod(call.method(), call.parameterTypes())
: type.getMethod(call.method(), call.parameterTypes());
if (!Modifier.isPublic(method.getModifiers())) {
throw new NoSuchMethodException("仅允许调用 public 方法: "
+ type.getName() + '.' + call.method());
}
return method;
}
private static Object invoke(Object target, Method method, Object[] arguments)
throws Exception {
try {
return method.invoke(target, arguments);
} catch (InvocationTargetException exception) {
Throwable cause = exception.getTargetException();
if (cause instanceof Exception checked) throw checked;
if (cause instanceof Error error) throw error;
throw exception;
}
}
private static String requiredExpression(SysJob job, String key) {
Map<String, Object> params = job.getJobParams();
Object value = params == null ? null : params.get(key);
if (value == null || !StrUtil.isNotBlank(value.toString())) {
throw new IllegalArgumentException("定时任务缺少执行表达式: " + key);
}
return value.toString().trim();
}
private static MethodCall parse(String expression) {
String before = StrUtil.subBefore(expression, "(", false);
int separator = before.lastIndexOf('.');
if (separator < 1 || !expression.endsWith(")")) {
throw new IllegalArgumentException("非法方法表达式: " + expression);
}
String owner = before.substring(0, separator);
String method = before.substring(separator + 1);
String params = StrUtil.subBetween(expression, "(", ")");
Object[] parsed = parseParams(params);
return new MethodCall(owner, method, (Class<?>[]) parsed[0], (Object[]) parsed[1]);
}
private static Object[] parseParams(String params) {
if (StrUtil.isEmpty(params)) return new Object[]{new Class<?>[0], new Object[0]};
String[] values = params.split(",");
Object[] arguments = new Object[values.length];
Class<?>[] types = new Class<?>[values.length];
for (int i = 0; i < values.length; i++) {
String value = values[i].trim();
if (value.startsWith("\"") && value.endsWith("\"") && value.length() >= 2) {
arguments[i] = value.substring(1, value.length() - 1);
types[i] = String.class;
} else if ("true".equals(value) || "false".equals(value)) {
arguments[i] = Boolean.valueOf(value);
types[i] = Boolean.class;
} else if (value.endsWith("L")) {
arguments[i] = Long.valueOf(value.substring(0, value.length() - 1));
types[i] = Long.class;
} else if (value.endsWith("D")) {
arguments[i] = Double.valueOf(value.substring(0, value.length() - 1));
types[i] = Double.class;
} else if (value.endsWith("F")) {
arguments[i] = Float.valueOf(value.substring(0, value.length() - 1));
types[i] = Float.class;
} else {
arguments[i] = Integer.valueOf(value);
types[i] = Integer.class;
}
}
return new Object[]{types, arguments};
}
private record MethodCall(String owner, String method,
Class<?>[] parameterTypes, Object[] arguments) {
private MethodCall {
parameterTypes = ArrayUtil.isEmpty(parameterTypes) ? new Class<?>[0] : parameterTypes;
arguments = ArrayUtil.isEmpty(arguments) ? new Object[0] : arguments;
}
}
}

View File

@@ -0,0 +1,47 @@
package tech.easyflow.job.execution;
import com.mybatisflex.core.tenant.TenantManager;
import org.springframework.stereotype.Component;
import tech.easyflow.common.constant.enums.EnumDataStatus;
import tech.easyflow.job.entity.SysJob;
import tech.easyflow.system.entity.SysAccount;
import tech.easyflow.system.service.SysAccountService;
import java.math.BigInteger;
import java.util.Objects;
/** 定时任务服务端归属账号校验器。 */
@Component
public class SysJobOwnerValidator {
private final SysAccountService accountService;
public SysJobOwnerValidator(SysAccountService accountService) {
this.accountService = accountService;
}
/**
* 按数据库当前状态校验任务归属账号。
*
* <p>调度 Worker 没有登录租户上下文,因此关闭 ORM 租户条件读取账号,
* 再显式比较任务与账号租户,避免跨租户账号被用于高权限任务执行。</p>
*/
public SysAccount requireAvailableOwner(SysJob job) {
BigInteger accountId = job.getCreatedBy();
if (accountId == null) {
throw new IllegalStateException("定时任务缺少服务端归属账号id=" + job.getId());
}
SysAccount account = TenantManager.withoutTenantCondition(
() -> accountService.getById(accountId));
if (account == null) {
throw new IllegalStateException("定时任务归属账号不存在id=" + accountId);
}
if (!EnumDataStatus.AVAILABLE.getCode().equals(account.getStatus())) {
throw new IllegalStateException("定时任务归属账号未启用id=" + accountId);
}
if (!Objects.equals(job.getTenantId(), account.getTenantId())) {
throw new IllegalStateException("定时任务与归属账号租户不一致id=" + job.getId());
}
return account;
}
}

View File

@@ -0,0 +1,5 @@
package tech.easyflow.job.execution;
/** 当前数据库执行队列的全局状态。 */
public record SysJobQueueSnapshot(long pending, long running, long oldestBacklogMillis) {
}

View File

@@ -1,72 +0,0 @@
package tech.easyflow.job.job;
import cn.hutool.core.exceptions.ExceptionUtil;
import com.alibaba.fastjson.JSON;
import org.quartz.Job;
import org.quartz.JobExecutionContext;
import org.quartz.JobExecutionException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import tech.easyflow.common.constant.enums.EnumJobExecStatus;
import tech.easyflow.common.util.SpringContextUtil;
import tech.easyflow.job.entity.SysJob;
import tech.easyflow.job.entity.SysJobLog;
import tech.easyflow.job.service.SysJobLogService;
import java.util.Date;
public abstract class BaseQuartzJob implements Job {
protected Logger log = LoggerFactory.getLogger(getClass());
private final static ThreadLocal<Date> TIME_RECORD = new ThreadLocal<>();
@Override
public void execute(JobExecutionContext ctx) throws JobExecutionException {
SysJob job = (SysJob) ctx.getMergedJobDataMap().get(JobConstant.JOB_MAP_BEAN_NAME);
try {
beforeExecute(ctx, job);
Object result = doExecute(ctx, job);
afterExecute(ctx, job, result, null);
} catch (Exception e) {
log.error("quartz 任务执行报错:", e);
afterExecute(ctx, job, null, e);
}
}
protected void beforeExecute(JobExecutionContext ctx, SysJob job) {
TIME_RECORD.set(new Date());
}
protected void afterExecute(JobExecutionContext ctx, SysJob job, Object result, Exception e) {
Date startTime = TIME_RECORD.get();
TIME_RECORD.remove();
Date endTime = new Date();
SysJobLog sysJobLog = new SysJobLog();
sysJobLog.setJobId(job.getId());
sysJobLog.setJobName(job.getJobName());
sysJobLog.setStatus(EnumJobExecStatus.SUCCESS.getCode());
sysJobLog.setJobParams(job.getJobParams());
if (result != null) {
sysJobLog.setJobResult(JSON.toJSONString(result));
}
if (e != null) {
String message = ExceptionUtil.getRootCauseMessage(e);
if (message.length() > 1000) {
message = message.substring(0, 1000);
}
sysJobLog.setErrorInfo(message);
sysJobLog.setStatus(EnumJobExecStatus.FAIL.getCode());
}
sysJobLog.setStartTime(startTime);
sysJobLog.setEndTime(endTime);
sysJobLog.setCreated(new Date());
SysJobLogService service = SpringContextUtil.getBean(SysJobLogService.class);
service.save(sysJobLog);
}
protected abstract Object doExecute(JobExecutionContext ctx, SysJob job) throws Exception;
}

View File

@@ -2,12 +2,15 @@ package tech.easyflow.job.job;
public interface JobConstant {
String JOB_GROUP = "easyflow";
String JOB_MAP_BEAN_NAME = "jobMapBean";
String BEAN_METHOD_KEY = "beanMethod";
String JAVA_METHOD_KEY = "javaMethod";
String WORKFLOW_KEY = "workflowId";
String WORKFLOW_PARAMS_KEY = "workflowParams";
/** 注入工作流运行参数的定时任务业务幂等键。 */
String EXECUTION_KEY = "_easyflowJobExecutionKey";
/** Quartz JobData 中用于隔离旧定义触发的任务代际。 */
String SCHEDULE_GENERATION = "scheduleGeneration";
String ACCOUNT_ID = "accountId";
}

View File

@@ -1,16 +0,0 @@
package tech.easyflow.job.job;
import org.quartz.JobExecutionContext;
import tech.easyflow.job.entity.SysJob;
import tech.easyflow.job.util.JobUtil;
/**
* 可并发执行
*/
public class QuartzJob extends BaseQuartzJob {
@Override
protected Object doExecute(JobExecutionContext ctx, SysJob job) throws Exception {
return JobUtil.execute(job);
}
}

View File

@@ -1,18 +0,0 @@
package tech.easyflow.job.job;
import org.quartz.DisallowConcurrentExecution;
import org.quartz.JobExecutionContext;
import tech.easyflow.job.entity.SysJob;
import tech.easyflow.job.util.JobUtil;
/**
* 禁止并发执行
*/
@DisallowConcurrentExecution
public class QuartzJobNoConcurrent extends BaseQuartzJob {
@Override
protected Object doExecute(JobExecutionContext ctx, SysJob job) throws Exception {
return JobUtil.execute(job);
}
}

View File

@@ -1,8 +1,14 @@
package tech.easyflow.job.mapper;
import com.mybatisflex.core.BaseMapper;
import org.apache.ibatis.annotations.Delete;
import org.apache.ibatis.annotations.Param;
import org.apache.ibatis.annotations.Select;
import org.apache.ibatis.annotations.Update;
import tech.easyflow.job.entity.SysJobLog;
import java.math.BigInteger;
/**
* 系统任务日志 映射层。
*
@@ -11,4 +17,158 @@ import tech.easyflow.job.entity.SysJobLog;
*/
public interface SysJobLogMapper extends BaseMapper<SysJobLog> {
@Select("SELECT id FROM tb_sys_job_log WHERE execution_key=#{executionKey} LIMIT 1")
BigInteger selectIdByExecutionKey(@Param("executionKey") String executionKey);
@Select("SELECT id FROM tb_sys_job_log WHERE job_id=#{jobId} AND status=2 "
+ "ORDER BY id FOR UPDATE")
java.util.List<BigInteger> selectPendingIdsByJobIdForUpdate(
@Param("jobId") BigInteger jobId);
@Select("SELECT id FROM tb_sys_job_log WHERE status=3 "
+ "AND lease_until<=CURRENT_TIMESTAMP(3) "
+ "ORDER BY lease_until, id LIMIT 1 FOR UPDATE SKIP LOCKED")
BigInteger selectExpiredClaimCandidateId();
/**
* 无锁探测是否存在已过期执行。过期范围为空时若直接使用 {@code FOR UPDATE}
* MySQL 仍可能锁住状态索引的末端间隙,随后 PENDING -> RUNNING 的索引迁移会在
* 多 Worker 间形成 insert-intention 死锁。
*/
@Select("SELECT id FROM tb_sys_job_log WHERE status=3 "
+ "AND lease_until<=CURRENT_TIMESTAMP(3) "
+ "ORDER BY lease_until, id LIMIT 1")
BigInteger selectExpiredClaimCandidateIdWithoutLock();
@Select("SELECT id FROM tb_sys_job_log WHERE status=2 "
+ "AND next_retry_time<=CURRENT_TIMESTAMP(3) "
+ "ORDER BY next_retry_time, id LIMIT 1 FOR UPDATE SKIP LOCKED")
BigInteger selectPendingClaimCandidateId();
/**
* 先用最小投影锁定候选行,再通过 BaseMapper 实体 ResultMap 读取快照。
* 这样既保留 MySQL {@code SKIP LOCKED},又避免注解式 {@code SELECT *}
* 丢失 {@code job_id}、JSON 快照等下划线字段映射。
*/
default SysJobLog selectExpiredClaimCandidate() {
BigInteger id = selectExpiredClaimCandidateId();
return id == null ? null : selectOneById(id);
}
/** 同 {@link #selectExpiredClaimCandidate()}。 */
default SysJobLog selectPendingClaimCandidate() {
BigInteger id = selectPendingClaimCandidateId();
return id == null ? null : selectOneById(id);
}
@Select("SELECT id FROM tb_sys_job_log WHERE job_id=#{jobId} AND id<>#{excludedId} "
+ "AND status=3 AND lease_until>CURRENT_TIMESTAMP(3) ORDER BY id LIMIT 1 FOR UPDATE")
BigInteger selectOtherActiveForUpdate(@Param("jobId") BigInteger jobId,
@Param("excludedId") BigInteger excludedId);
@Update("UPDATE tb_sys_job_log SET status=3, lease_owner=#{owner}, "
+ "execution_token=#{token}, "
+ "lease_until=TIMESTAMPADD(MICROSECOND, #{leaseMicros}, CURRENT_TIMESTAMP(3)), "
+ "heartbeat_time=CURRENT_TIMESTAMP(3), "
+ "attempt_count=attempt_count+1, next_retry_time=NULL, "
+ "start_time=COALESCE(start_time, CURRENT_TIMESTAMP(3)), end_time=NULL, "
+ "error_info=NULL, version=version+1 "
+ "WHERE id=#{id} AND status=2 AND next_retry_time<=CURRENT_TIMESTAMP(3)")
int claimPending(@Param("id") BigInteger id,
@Param("owner") String owner,
@Param("token") String token,
@Param("leaseMicros") long leaseMicros);
@Update("UPDATE tb_sys_job_log SET lease_owner=#{owner}, execution_token=#{token}, "
+ "lease_until=TIMESTAMPADD(MICROSECOND, #{leaseMicros}, CURRENT_TIMESTAMP(3)), "
+ "heartbeat_time=CURRENT_TIMESTAMP(3), attempt_count=attempt_count+1, "
+ "next_retry_time=NULL, error_info=NULL, version=version+1 "
+ "WHERE id=#{id} AND status=3 AND lease_until<=CURRENT_TIMESTAMP(3)")
int claimExpired(@Param("id") BigInteger id,
@Param("owner") String owner,
@Param("token") String token,
@Param("leaseMicros") long leaseMicros);
@Update("UPDATE tb_sys_job_log SET "
+ "next_retry_time=TIMESTAMPADD(MICROSECOND, #{retryMicros}, CURRENT_TIMESTAMP(3)) "
+ "WHERE id=#{id} AND status=2")
int deferPending(@Param("id") BigInteger id,
@Param("retryMicros") long retryMicros);
@Update("UPDATE tb_sys_job_log SET status=2, lease_owner=NULL, execution_token=NULL, "
+ "lease_until=NULL, heartbeat_time=CURRENT_TIMESTAMP(3), "
+ "next_retry_time=TIMESTAMPADD(MICROSECOND, #{retryMicros}, CURRENT_TIMESTAMP(3)), "
+ "version=version+1 WHERE id=#{id} AND status=3 "
+ "AND lease_until<=CURRENT_TIMESTAMP(3)")
int releaseExpiredForRetry(@Param("id") BigInteger id,
@Param("retryMicros") long retryMicros);
@Update("UPDATE tb_sys_job_log SET status=2, lease_owner=NULL, execution_token=NULL, "
+ "lease_until=NULL, heartbeat_time=CURRENT_TIMESTAMP(3), "
+ "next_retry_time=TIMESTAMPADD(MICROSECOND, #{retryMicros}, CURRENT_TIMESTAMP(3)), "
+ "error_info=#{errorInfo}, version=version+1 WHERE id=#{id} AND status=3 "
+ "AND lease_owner=#{owner} AND execution_token=#{token}")
int releaseOwnedForRetry(@Param("id") BigInteger id,
@Param("owner") String owner,
@Param("token") String token,
@Param("retryMicros") long retryMicros,
@Param("errorInfo") String errorInfo);
@Update("UPDATE tb_sys_job_log SET "
+ "lease_until=TIMESTAMPADD(MICROSECOND, #{leaseMicros}, CURRENT_TIMESTAMP(3)), "
+ "heartbeat_time=CURRENT_TIMESTAMP(3) "
+ "WHERE id=#{id} AND status=3 AND lease_owner=#{owner} AND execution_token=#{token}")
int renewLease(@Param("id") BigInteger id,
@Param("owner") String owner,
@Param("token") String token,
@Param("leaseMicros") long leaseMicros);
@Update("UPDATE tb_sys_job_log SET status=#{status}, job_result=#{jobResult}, "
+ "error_info=#{errorInfo}, end_time=CURRENT_TIMESTAMP(3), lease_owner=NULL, "
+ "execution_token=NULL, lease_until=NULL, heartbeat_time=CURRENT_TIMESTAMP(3), "
+ "version=version+1 "
+ "WHERE id=#{id} AND status=3 AND lease_owner=#{owner} AND execution_token=#{token}")
int finishOwned(@Param("id") BigInteger id,
@Param("owner") String owner,
@Param("token") String token,
@Param("status") int status,
@Param("jobResult") String jobResult,
@Param("errorInfo") String errorInfo);
@Update("UPDATE tb_sys_job_log SET status=4, error_info=#{errorInfo}, "
+ "end_time=CURRENT_TIMESTAMP(3), lease_owner=NULL, execution_token=NULL, "
+ "lease_until=NULL, heartbeat_time=CURRENT_TIMESTAMP(3), version=version+1 "
+ "WHERE id=#{id} AND status=3 AND lease_until<=CURRENT_TIMESTAMP(3)")
int markExpiredDead(@Param("id") BigInteger id,
@Param("errorInfo") String errorInfo);
@Update("UPDATE tb_sys_job_log SET status=4, error_info=#{errorInfo}, "
+ "end_time=CURRENT_TIMESTAMP(3), next_retry_time=NULL, "
+ "heartbeat_time=CURRENT_TIMESTAMP(3), version=version+1 "
+ "WHERE id=#{id} AND status=2")
int markPendingDead(@Param("id") BigInteger id,
@Param("errorInfo") String errorInfo);
@Update("UPDATE tb_sys_job_log SET status=5, error_info=#{reason}, "
+ "end_time=CURRENT_TIMESTAMP(3), "
+ "next_retry_time=NULL, version=version+1 WHERE job_id=#{jobId} AND status=2")
int cancelPendingByJobId(@Param("jobId") BigInteger jobId,
@Param("reason") String reason);
@Select("SELECT COUNT(1) FROM tb_sys_job_log WHERE status=2")
long countPending();
@Select("SELECT COUNT(1) FROM tb_sys_job_log WHERE status=3")
long countRunning();
@Select("SELECT COALESCE(TIMESTAMPDIFF(MICROSECOND, MIN(scheduled_fire_time), "
+ "CURRENT_TIMESTAMP(3)) DIV 1000, 0) FROM tb_sys_job_log WHERE status=2")
long selectOldestPendingMillis();
@Select("SELECT COUNT(1) FROM tb_sys_job_log WHERE id=#{id} AND status IN (2,3)")
int countActiveById(@Param("id") BigInteger id);
@Delete("DELETE FROM tb_sys_job_log WHERE id=#{id} AND status NOT IN (2,3)")
int deleteTerminalById(@Param("id") BigInteger id);
}

View File

@@ -1,8 +1,14 @@
package tech.easyflow.job.mapper;
import com.mybatisflex.core.BaseMapper;
import com.mybatisflex.core.query.QueryWrapper;
import org.apache.ibatis.annotations.Param;
import org.apache.ibatis.annotations.Update;
import tech.easyflow.job.entity.SysJob;
import java.math.BigInteger;
import java.util.List;
/**
* 系统任务表 映射层。
*
@@ -11,4 +17,31 @@ import tech.easyflow.job.entity.SysJob;
*/
public interface SysJobMapper extends BaseMapper<SysJob> {
/**
* 使用 BaseMapper 的实体 ResultMap 读取并锁定任务。
*
* <p>不能用注解式 {@code SELECT *}:新增下划线列时它不会复用
* MyBatis-Flex 生成的实体映射,曾导致 {@code schedule_generation}
* 在运行态被读取为 {@code null}。</p>
*/
default SysJob selectByIdForUpdate(BigInteger id) {
return selectOneByQuery(QueryWrapper.create()
.eq(SysJob::getId, id)
.forUpdate());
}
@Update("UPDATE tb_sys_job SET status=#{runningStatus}, "
+ "schedule_generation=schedule_generation+1 "
+ "WHERE id=#{id} AND status<>#{runningStatus}")
int startNextGeneration(@Param("id") BigInteger id,
@Param("runningStatus") int runningStatus);
/** 使用实体 ResultMap 做 keyset 分页,确保所有调度字段完整映射。 */
default List<SysJob> selectReconciliationPage(BigInteger lastId, int limit) {
return selectListByQuery(QueryWrapper.create()
.gt(SysJob::getId, lastId)
.orderBy(SysJob::getId, true)
.limit(limit));
}
}

View File

@@ -0,0 +1,82 @@
package tech.easyflow.job.schedule;
import com.easyagents.scheduler.ConcurrencyPolicy;
import com.easyagents.scheduler.CronSchedulePlan;
import com.easyagents.scheduler.MisfirePolicy;
import com.easyagents.scheduler.ScheduleDefinition;
import com.easyagents.scheduler.ScheduleId;
import com.easyagents.scheduler.ScheduleService;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
import tech.easyflow.common.constant.enums.EnumMisfirePolicy;
import tech.easyflow.job.entity.SysJob;
import tech.easyflow.job.execution.EasyFlowScheduleHandler;
import tech.easyflow.job.job.JobConstant;
import java.time.ZoneId;
import java.util.Map;
import java.util.UUID;
/** EasyFlow 任务定义到 easy-agents 调度模型的唯一映射入口。 */
@Component
public class SysJobScheduleAdapter {
static final String NAMESPACE = "easyflow.job";
private final ScheduleService scheduleService;
private final ZoneId zoneId;
public SysJobScheduleAdapter(ScheduleService scheduleService,
@Value("${easyflow.job.timezone:Asia/Shanghai}") String zoneId) {
this.scheduleService = scheduleService;
this.zoneId = ZoneId.of(zoneId);
}
public void replace(SysJob job) {
scheduleService.replace(toDefinition(job));
}
public void delete(java.math.BigInteger jobId) {
scheduleService.delete(scheduleId(jobId));
}
public String triggerNow(java.math.BigInteger jobId) {
String invocationId = UUID.randomUUID().toString();
scheduleService.triggerNow(scheduleId(jobId), invocationId, Map.of());
return invocationId;
}
ScheduleDefinition toDefinition(SysJob job) {
Long generation = job.getScheduleGeneration();
if (generation == null || generation < 0L) {
throw new IllegalArgumentException("任务调度代际非法: " + generation);
}
return new ScheduleDefinition(
scheduleId(job.getId()),
EasyFlowScheduleHandler.CODE,
new CronSchedulePlan(job.getCronExpression(), zoneId),
toMisfirePolicy(job.getMisfirePolicy()),
Integer.valueOf(1).equals(job.getAllowConcurrent())
? ConcurrencyPolicy.ALLOW : ConcurrencyPolicy.DISALLOW,
true,
Map.of(
"jobId", job.getId().toString(),
JobConstant.SCHEDULE_GENERATION, generation.toString()),
job.getJobName());
}
private static MisfirePolicy toMisfirePolicy(Integer policy) {
if (Integer.valueOf(EnumMisfirePolicy.FIRE_ONCE_NOW.getCode()).equals(policy)) {
return MisfirePolicy.FIRE_ONCE_NOW;
}
if (Integer.valueOf(EnumMisfirePolicy.SKIP.getCode()).equals(policy)) {
return MisfirePolicy.SKIP;
}
throw new IllegalArgumentException("不支持的 Misfire 策略: " + policy);
}
private static ScheduleId scheduleId(java.math.BigInteger jobId) {
if (jobId == null) throw new IllegalArgumentException("jobId must not be null");
return new ScheduleId(NAMESPACE, jobId.toString());
}
}

View File

@@ -0,0 +1,62 @@
package tech.easyflow.job.schedule;
import com.mybatisflex.core.tenant.TenantManager;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.boot.context.event.ApplicationReadyEvent;
import org.springframework.context.event.EventListener;
import org.springframework.stereotype.Component;
import tech.easyflow.job.config.SysJobExecutionProperties;
import tech.easyflow.job.entity.SysJob;
import tech.easyflow.job.mapper.SysJobMapper;
import tech.easyflow.job.service.SysJobService;
import java.math.BigInteger;
import java.util.List;
/** 启动时一次性修复业务定义与 Quartz 投影的跨事务漂移。 */
@Component
public class SysJobScheduleReconciler {
private static final Logger log = LoggerFactory.getLogger(SysJobScheduleReconciler.class);
private static final int PAGE_SIZE = 200;
private final SysJobMapper mapper;
private final SysJobService jobService;
private final SysJobExecutionProperties properties;
public SysJobScheduleReconciler(SysJobMapper mapper,
SysJobService jobService,
SysJobExecutionProperties properties) {
this.mapper = mapper;
this.jobService = jobService;
this.properties = properties;
}
@EventListener(ApplicationReadyEvent.class)
public void reconcile() {
if (!properties.isEnabled()) return;
BigInteger lastId = BigInteger.ZERO;
int reconciled = 0;
int failed = 0;
while (true) {
BigInteger pageStartId = lastId;
List<SysJob> jobs = TenantManager.withoutTenantCondition(
() -> mapper.selectReconciliationPage(pageStartId, PAGE_SIZE));
for (SysJob job : jobs) {
try {
// syncJob 会按 jobId 加分布式锁,并在锁内重新读取最新定义。
jobService.syncJob(job.getId());
reconciled++;
} catch (RuntimeException exception) {
failed++;
log.error("定时任务启动对账失败: jobId={}", job.getId(), exception);
}
}
if (jobs.size() < PAGE_SIZE) break;
// Keyset 分页不依赖已处理行仍然存在,避免对账过程中并发删除导致跳行。
lastId = jobs.get(jobs.size() - 1).getId();
}
log.info("定时任务启动对账完成: succeeded={}, failed={}", reconciled, failed);
}
}

View File

@@ -3,6 +3,9 @@ package tech.easyflow.job.service;
import com.mybatisflex.core.service.IService;
import tech.easyflow.job.entity.SysJobLog;
import java.io.Serializable;
import java.util.Collection;
/**
* 系统任务日志 服务层。
*
@@ -11,4 +14,5 @@ import tech.easyflow.job.entity.SysJobLog;
*/
public interface SysJobLogService extends IService<SysJobLog> {
void requireTerminal(Collection<Serializable> ids);
}

View File

@@ -6,6 +6,7 @@ import tech.easyflow.job.entity.SysJob;
import java.io.Serializable;
import java.math.BigInteger;
import java.util.Collection;
import java.util.Date;
import java.util.List;
/**
@@ -22,12 +23,20 @@ public interface SysJobService extends IService<SysJob> {
void addJob(SysJob job);
void syncJob(BigInteger id);
void updateJobDefinition(SysJob job);
void deleteJob(Collection<Serializable> ids);
void startJob(BigInteger id);
void stopJob(BigInteger id);
String triggerNow(BigInteger id);
List<Date> nextFireTimes(String cronExpression, int limit);
/**
* 查询引用指定工作流的定时任务。
*

View File

@@ -3,18 +3,23 @@ package tech.easyflow.job.service;
import cn.hutool.core.bean.BeanUtil;
import com.alibaba.fastjson2.JSONObject;
import com.easyagents.flow.core.chain.runtime.ChainExecutor;
import com.mybatisflex.core.tenant.TenantManager;
import org.springframework.stereotype.Service;
import org.springframework.dao.DataAccessException;
import org.springframework.dao.DataAccessResourceFailureException;
import org.springframework.dao.RecoverableDataAccessException;
import org.springframework.dao.TransientDataAccessException;
import tech.easyflow.ai.easyagentsflow.support.PublishedWorkflowDefinitionIds;
import tech.easyflow.ai.service.WorkflowUsageAuthorizationService;
import tech.easyflow.common.constant.Constants;
import tech.easyflow.common.constant.enums.EnumDataStatus;
import tech.easyflow.common.constant.enums.EnumJobStatus;
import tech.easyflow.common.entity.LoginAccount;
import tech.easyflow.job.entity.SysJob;
import tech.easyflow.job.execution.SysJobExecutionContextHolder;
import tech.easyflow.job.execution.SysJobInfrastructureException;
import tech.easyflow.job.job.JobConstant;
import tech.easyflow.job.support.SysJobWorkflowReferenceSupport;
import tech.easyflow.system.entity.SysAccount;
import tech.easyflow.system.service.SysAccountService;
import java.math.BigInteger;
import java.util.Map;
@@ -23,17 +28,13 @@ import java.util.Objects;
/**
* 工作流定时任务执行服务。
*
* <p>每次触发都重新加载任务账号和工作流,并按服务端记录恢复执行主体及重新授权。</p>
* <p>任务入口已按数据库当前状态复核任务账号;本服务继续检查工作流权限。
* 工作流引用与运行参数使用触发登记时的账本快照,避免积压期间的普通编辑
* 改变既有 execution。</p>
*/
@Service
public class WorkflowJobExecutionService {
/** 定时任务服务。 */
private final SysJobService sysJobService;
/** 系统账号服务。 */
private final SysAccountService sysAccountService;
/** 工作流使用权限校验服务。 */
private final WorkflowUsageAuthorizationService workflowUsageAuthorizationService;
@@ -43,18 +44,12 @@ public class WorkflowJobExecutionService {
/**
* 创建工作流定时任务执行服务。
*
* @param sysJobService 定时任务服务
* @param sysAccountService 系统账号服务
* @param workflowUsageAuthorizationService 工作流使用权限校验服务
* @param chainExecutor 工作流执行器
*/
public WorkflowJobExecutionService(
SysJobService sysJobService,
SysAccountService sysAccountService,
WorkflowUsageAuthorizationService workflowUsageAuthorizationService,
ChainExecutor chainExecutor) {
this.sysJobService = sysJobService;
this.sysAccountService = sysAccountService;
this.workflowUsageAuthorizationService = workflowUsageAuthorizationService;
this.chainExecutor = chainExecutor;
}
@@ -63,31 +58,45 @@ public class WorkflowJobExecutionService {
* 使用当前数据库状态执行工作流定时任务。
*
* @param scheduledJob Quartz 中保存的任务快照
* @param currentJob 数据库当前任务定义
* @param owner 已完成实时复核的任务归属账号
* @return 工作流执行结果
* @throws IllegalStateException 任务、账号或租户状态非法时抛出
*/
public Object execute(SysJob scheduledJob) {
public Object execute(SysJob scheduledJob, SysJob currentJob, SysAccount owner) {
if (scheduledJob == null || scheduledJob.getId() == null) {
throw new IllegalStateException("定时任务不存在或缺少ID");
}
return TenantManager.withoutTenantCondition(
() -> executeWithoutTenantCondition(
scheduledJob.getId(),
scheduledJob.getTenantId()));
PreparedWorkflowExecution prepared;
try {
prepared = prepare(scheduledJob, currentJob, owner);
} catch (DataAccessException exception) {
if (exception instanceof TransientDataAccessException
|| exception instanceof RecoverableDataAccessException
|| exception instanceof DataAccessResourceFailureException) {
throw new SysJobInfrastructureException(
"准备工作流定时任务执行上下文失败", exception);
}
throw exception;
}
return chainExecutor.execute(
PublishedWorkflowDefinitionIds.published(prepared.workflowId().toString()),
prepared.parameters());
}
/**
* 在已关闭 ORM 租户条件的作用域中执行任务,并显式完成租户边界校验
* 组装执行参数,并再次验证入口传入的当前状态与租户边界。
*
* @param jobId 定时任务 ID
* @param scheduledTenantId Quartz 任务快照中的租户 ID
* @param scheduledJob 触发登记时保存的任务快照
* @param job 数据库当前任务定义
* @param owner 已完成实时复核的任务归属账号
* @return 工作流执行结果
* @throws IllegalStateException 任务、账号或租户状态非法时抛出
*/
private Object executeWithoutTenantCondition(
BigInteger jobId,
BigInteger scheduledTenantId) {
SysJob job = sysJobService.getById(jobId);
private PreparedWorkflowExecution prepare(
SysJob scheduledJob, SysJob job, SysAccount owner) {
BigInteger jobId = scheduledJob.getId();
BigInteger scheduledTenantId = scheduledJob.getTenantId();
if (job == null) {
throw new IllegalStateException("定时任务不存在或已删除id=" + jobId);
}
@@ -102,44 +111,38 @@ public class WorkflowJobExecutionService {
if (!SysJobWorkflowReferenceSupport.isWorkflowJob(job)) {
throw new IllegalStateException("定时任务类型已变更id=" + jobId);
}
if (!SysJobWorkflowReferenceSupport.isWorkflowJob(scheduledJob)) {
throw new IllegalStateException("定时任务快照类型非法id=" + jobId);
}
SysAccount account = requireAvailableOwner(job);
SysAccount account = requirePreparedOwner(job, owner);
LoginAccount loginAccount = new LoginAccount();
BeanUtil.copyProperties(account, loginAccount);
BigInteger workflowId = SysJobWorkflowReferenceSupport.requireWorkflowId(job);
BigInteger workflowId = SysJobWorkflowReferenceSupport.requireWorkflowId(scheduledJob);
workflowUsageAuthorizationService.requireUsableWorkflow(
workflowId,
loginAccount,
"定时任务关联的工作流不存在、已禁用或无权运行");
"定时任务关联的工作流不存在、未发布或无权运行");
JSONObject workflowParams = resolveWorkflowParams(job.getJobParams());
JSONObject workflowParams = resolveWorkflowParams(scheduledJob.getJobParams());
SysJobExecutionContextHolder.current().ifPresent(context ->
workflowParams.put(JobConstant.EXECUTION_KEY, context.executionKey()));
workflowParams.put(Constants.LOGIN_USER_KEY, loginAccount);
return chainExecutor.execute(workflowId.toString(), workflowParams);
return new PreparedWorkflowExecution(workflowId, workflowParams);
}
/**
* 获取任务创建账号并校验账号仍可用于执行任务。
*
* @param job 当前数据库中的定时任务
* @return 可用的任务创建账号
* @throws IllegalStateException 创建账号缺失、禁用或跨租户时抛出
*/
private SysAccount requireAvailableOwner(SysJob job) {
BigInteger accountId = job.getCreatedBy();
if (accountId == null) {
throw new IllegalStateException("定时任务缺少服务端归属账号id=" + job.getId());
/** 验证调用方传入的是当前任务已完成数据库复核的归属账号。 */
private static SysAccount requirePreparedOwner(SysJob job, SysAccount owner) {
if (owner == null || !Objects.equals(job.getCreatedBy(), owner.getId())) {
throw new IllegalStateException("定时任务归属账号校验结果无效id=" + job.getId());
}
SysAccount account = sysAccountService.getById(accountId);
if (account == null) {
throw new IllegalStateException("定时任务归属账号不存在id=" + accountId);
if (!EnumDataStatus.AVAILABLE.getCode().equals(owner.getStatus())) {
throw new IllegalStateException("定时任务归属账号未启用id=" + owner.getId());
}
if (!EnumDataStatus.AVAILABLE.getCode().equals(account.getStatus())) {
throw new IllegalStateException("定时任务归属账号未启用id=" + accountId);
}
if (!Objects.equals(job.getTenantId(), account.getTenantId())) {
if (!Objects.equals(job.getTenantId(), owner.getTenantId())) {
throw new IllegalStateException("定时任务与归属账号租户不一致id=" + job.getId());
}
return account;
return owner;
}
/**
@@ -156,4 +159,7 @@ public class WorkflowJobExecutionService {
.getJSONObject(JobConstant.WORKFLOW_PARAMS_KEY);
return params == null ? new JSONObject() : new JSONObject(params);
}
private record PreparedWorkflowExecution(BigInteger workflowId, JSONObject parameters) {
}
}

View File

@@ -6,6 +6,10 @@ import tech.easyflow.job.entity.SysJobLog;
import tech.easyflow.job.mapper.SysJobLogMapper;
import tech.easyflow.job.service.SysJobLogService;
import java.io.Serializable;
import java.math.BigInteger;
import java.util.Collection;
/**
* 系统任务日志 服务层实现。
*
@@ -15,4 +19,13 @@ import tech.easyflow.job.service.SysJobLogService;
@Service
public class SysJobLogServiceImpl extends ServiceImpl<SysJobLogMapper, SysJobLog> implements SysJobLogService {
@Override
public void requireTerminal(Collection<Serializable> ids) {
if (ids == null) return;
for (Serializable id : ids) {
if (mapper.countActiveById(new BigInteger(id.toString())) > 0) {
throw new IllegalStateException("等待执行或执行中的任务记录不能删除id=" + id);
}
}
}
}

View File

@@ -1,175 +1,368 @@
package tech.easyflow.job.service.impl;
import com.easyagents.scheduler.CronSchedulePlan;
import com.easyagents.scheduler.ScheduleService;
import com.mybatisflex.core.query.QueryWrapper;
import com.mybatisflex.spring.service.impl.ServiceImpl;
import org.quartz.*;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.transaction.PlatformTransactionManager;
import org.springframework.transaction.TransactionDefinition;
import org.springframework.transaction.support.TransactionTemplate;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.stereotype.Service;
import tech.easyflow.common.constant.enums.EnumJobType;
import tech.easyflow.common.constant.enums.EnumMisfirePolicy;
import tech.easyflow.common.constant.enums.EnumJobStatus;
import tech.easyflow.common.cache.RedisLockExecutor;
import tech.easyflow.common.constant.enums.EnumJobStatus;
import tech.easyflow.common.constant.enums.EnumJobType;
import tech.easyflow.common.web.exceptions.BusinessException;
import tech.easyflow.job.config.SysJobExecutionProperties;
import tech.easyflow.job.entity.SysJob;
import tech.easyflow.job.job.JobConstant;
import tech.easyflow.job.job.QuartzJob;
import tech.easyflow.job.job.QuartzJobNoConcurrent;
import tech.easyflow.job.execution.SysJobExecutionStore;
import tech.easyflow.job.mapper.SysJobMapper;
import tech.easyflow.job.schedule.SysJobScheduleAdapter;
import tech.easyflow.job.service.SysJobService;
import tech.easyflow.job.support.SysJobWorkflowReferenceSupport;
import tech.easyflow.job.util.JobUtil;
import javax.annotation.Resource;
import java.io.Serializable;
import java.math.BigInteger;
import java.time.Duration;
import java.time.ZoneId;
import java.util.Collection;
import java.util.Collections;
import java.util.Date;
import java.util.List;
import java.util.concurrent.Semaphore;
import java.util.concurrent.TimeUnit;
import java.util.function.Supplier;
/**
* 系统任务表 服务层实现。
*
* @author xiaoma
* @since 2025-05-20
*/
/** 系统任务管理及 Quartz 投影协调服务。 */
@Service
public class SysJobServiceImpl extends ServiceImpl<SysJobMapper, SysJob> implements SysJobService {
public class SysJobServiceImpl extends ServiceImpl<SysJobMapper, SysJob> implements SysJobService {
private static final Logger log = LoggerFactory.getLogger(SysJobServiceImpl.class);
private static final String JOB_LOCK_KEY_PREFIX = "easyflow:lock:job:";
private static final Duration LOCK_WAIT_TIMEOUT = Duration.ofSeconds(2);
private static final Duration LOCK_LEASE_TIMEOUT = Duration.ofSeconds(10);
protected Logger log = LoggerFactory.getLogger(SysJobServiceImpl.class);
private final SysJobScheduleAdapter scheduleAdapter;
private final SysJobExecutionStore executionStore;
private final ScheduleService scheduleService;
private final RedisLockExecutor redisLockExecutor;
private final TransactionTemplate managementTransaction;
private final Semaphore managementPermits;
private final ZoneId zoneId;
@Resource
private Scheduler scheduler;
@Resource
private RedisLockExecutor redisLockExecutor;
public SysJobServiceImpl(SysJobScheduleAdapter scheduleAdapter,
SysJobExecutionStore executionStore,
ScheduleService scheduleService,
RedisLockExecutor redisLockExecutor,
PlatformTransactionManager transactionManager,
SysJobExecutionProperties executionProperties,
@Value("${easyflow.job.timezone:Asia/Shanghai}") String zoneId) {
this.scheduleAdapter = scheduleAdapter;
this.executionStore = executionStore;
this.scheduleService = scheduleService;
this.redisLockExecutor = redisLockExecutor;
this.managementTransaction = new TransactionTemplate(transactionManager);
this.managementTransaction.setIsolationLevel(TransactionDefinition.ISOLATION_READ_COMMITTED);
this.managementPermits = new Semaphore(
executionProperties.getManagementCommandConcurrency(), true);
this.zoneId = ZoneId.of(zoneId);
}
@Override
public void test() {
System.out.println("java bean 动态执行");
// 供 Spring Bean 类型定时任务进行开发环境验证。
}
@Override
public void testParam(String a, Boolean b, Integer c) {
System.out.println("动态执行spring bean,执行参数:" + "a="+ a + ",b="+ b + ",c="+ c);
// 供 Spring Bean 类型定时任务进行参数绑定验证。
}
@Override
public void addJob(SysJob job) {
Integer allowConcurrent = job.getAllowConcurrent();
Class<? extends Job> jobClass = allowConcurrent == 1 ? QuartzJob.class : QuartzJobNoConcurrent.class;
JobDetail jobDetail = JobBuilder.newJob(jobClass)
.withIdentity(JobUtil.getJobKey(job))
.build();
jobDetail.getJobDataMap().put(JobConstant.JOB_MAP_BEAN_NAME, job);
CronScheduleBuilder cron = CronScheduleBuilder.cronSchedule(job.getCronExpression());
Integer misfirePolicy = job.getMisfirePolicy();
if (EnumMisfirePolicy.MISFIRE_DO_NOTHING.getCode() == misfirePolicy) {
cron.withMisfireHandlingInstructionDoNothing();
}
if (EnumMisfirePolicy.MISFIRE_FIRE_AND_PROCEED.getCode() == misfirePolicy) {
cron.withMisfireHandlingInstructionFireAndProceed();
}
if (EnumMisfirePolicy.MISFIRE_IGNORE_MISFIRES.getCode() == misfirePolicy) {
cron.withMisfireHandlingInstructionIgnoreMisfires();
if (job == null || job.getId() == null) {
throw new IllegalArgumentException("任务及任务ID不能为空");
}
syncJob(job.getId());
}
Trigger trigger = TriggerBuilder.newTrigger()
.withIdentity(JobUtil.getTriggerKey(job))
.withSchedule(cron).build();
@Override
public void syncJob(BigInteger id) {
withJobCommandLock(id, () -> {
if (!projectCurrentStateOrStop(id)) {
cancelPendingBestEffort(id, "任务已停止");
}
return null;
});
}
try {
scheduler.scheduleJob(jobDetail,trigger);
} catch (SchedulerException e) {
log.error("启动任务失败:", e);
throw new RuntimeException(e);
@Override
public void updateJobDefinition(SysJob update) {
if (update == null || update.getId() == null) {
throw new IllegalArgumentException("任务及任务ID不能为空");
}
BigInteger id = update.getId();
withJobCommandLock(id, () -> {
inManagementTransaction(() -> {
SysJob current = requireJobForUpdate(id);
long generation = nextScheduleGeneration(current);
// 状态、归属和代际都是服务端维护字段。必须以行锁下的最终值覆盖
// 请求快照,避免普通编辑越过 start/stop 或把并发 STOP 写回 RUNNING。
update.setTenantId(current.getTenantId());
update.setDeptId(current.getDeptId());
update.setCreated(current.getCreated());
update.setCreatedBy(current.getCreatedBy());
update.setStatus(current.getStatus());
update.setScheduleGeneration(generation);
if (!updateById(update)) {
throw new IllegalStateException("定时任务更新失败id=" + id);
}
return null;
});
projectCurrentStateOrStop(id);
return null;
});
}
@Override
public void deleteJob(Collection<Serializable> ids) {
try {
for (Serializable id : ids) {
SysJob sysJob = new SysJob();
sysJob.setId(new BigInteger(id.toString()));
scheduler.deleteJob(JobUtil.getJobKey(sysJob));
}
} catch (SchedulerException e) {
log.error("删除任务失败:", e);
throw new RuntimeException(e);
if (ids == null) return;
List<BigInteger> jobIds = ids.stream()
.map(id -> new BigInteger(id.toString()))
.toList();
// 在产生任何 Quartz 或跨租户账本副作用前,先按当前租户完整校验全部任务。
jobIds.forEach(this::requireJob);
for (BigInteger jobId : jobIds) {
withJobCommandLock(jobId, () -> {
// 分三段收口:业务先 STOPQuartz 投影删除成功后再物理删除。
// 任一阶段失败都只会留下可重试的 STOP 行,不会留下 RUNNING 无投影。
inManagementTransaction(() -> {
SysJob job = requireJobForUpdate(jobId);
if (!Integer.valueOf(EnumJobStatus.STOP.getCode()).equals(job.getStatus())) {
SysJob stopped = new SysJob();
stopped.setId(jobId);
stopped.setStatus(EnumJobStatus.STOP.getCode());
if (!updateById(stopped)) {
throw new IllegalStateException("定时任务停止失败id=" + jobId);
}
}
return null;
});
try {
projectCurrentStateOrStop(jobId);
inManagementTransaction(() -> {
SysJob current = requireJobForUpdate(jobId);
if (!Integer.valueOf(EnumJobStatus.STOP.getCode())
.equals(current.getStatus())) {
throw new IllegalStateException(
"定时任务状态已变化取消删除id=" + jobId);
}
if (!removeById(jobId)) {
throw new IllegalStateException("定时任务删除失败id=" + jobId);
}
return null;
});
} finally {
cancelPendingBestEffort(jobId, "任务已删除或已停止");
}
return null;
});
}
}
@Override
public void startJob(BigInteger id) {
redisLockExecutor.executeWithLock(JOB_LOCK_KEY_PREFIX + id, LOCK_WAIT_TIMEOUT, LOCK_LEASE_TIMEOUT, () -> {
SysJob sysJob = this.getById(id);
if (sysJob == null) {
throw new IllegalStateException("任务不存在id=" + id);
}
try {
JobKey jobKey = JobUtil.getJobKey(sysJob);
if (!scheduler.checkExists(jobKey)) {
addJob(sysJob);
withJobCommandLock(id, () -> {
// STOP 期间可能因数据库瞬时故障遗留上一代 PENDING。重新进入 RUNNING
// 前必须强制清理;失败则拒绝启动,避免旧 fire 被新一代状态放行。
requireJob(id);
executionStore.cancelPending(id, "任务重新启动,取消上一代待执行记录");
inManagementTransaction(() -> {
SysJob current = requireJobForUpdate(id);
if (!Integer.valueOf(EnumJobStatus.RUNNING.getCode()).equals(current.getStatus())) {
if (getMapper().startNextGeneration(
id, EnumJobStatus.RUNNING.getCode()) != 1) {
throw new IllegalStateException("定时任务启动失败id=" + id);
}
}
if (!Integer.valueOf(EnumJobStatus.RUNNING.getCode()).equals(sysJob.getStatus())) {
SysJob update = new SysJob();
update.setId(id);
update.setStatus(EnumJobStatus.RUNNING.getCode());
this.updateById(update);
}
} catch (SchedulerException e) {
log.error("启动任务失败id={}", id, e);
throw new RuntimeException(e);
}
return null;
});
projectCurrentStateOrStop(id);
return null;
});
}
@Override
public void stopJob(BigInteger id) {
redisLockExecutor.executeWithLock(JOB_LOCK_KEY_PREFIX + id, LOCK_WAIT_TIMEOUT, LOCK_LEASE_TIMEOUT, () -> {
SysJob sysJob = this.getById(id);
if (sysJob == null) {
throw new IllegalStateException("任务不存在id=" + id);
}
withJobCommandLock(id, () -> {
inManagementTransaction(() -> {
SysJob job = requireJobForUpdate(id);
if (!Integer.valueOf(EnumJobStatus.STOP.getCode()).equals(job.getStatus())) {
SysJob stopped = new SysJob();
stopped.setId(id);
stopped.setStatus(EnumJobStatus.STOP.getCode());
if (!updateById(stopped)) {
throw new IllegalStateException("定时任务停止失败id=" + id);
}
}
return null;
});
try {
JobKey jobKey = JobUtil.getJobKey(sysJob);
if (scheduler.checkExists(jobKey)) {
deleteJob(Collections.singletonList(id));
}
if (!Integer.valueOf(EnumJobStatus.STOP.getCode()).equals(sysJob.getStatus())) {
SysJob update = new SysJob();
update.setId(id);
update.setStatus(EnumJobStatus.STOP.getCode());
this.updateById(update);
}
} catch (SchedulerException e) {
log.error("停止任务失败id={}", id, e);
throw new RuntimeException(e);
projectCurrentStateOrStop(id);
} finally {
cancelPendingBestEffort(id, "任务已停止");
}
return null;
});
}
/**
* {@inheritDoc}
*/
@Override
public String triggerNow(BigInteger id) {
return withJobCommandLock(id, () -> inManagementTransaction(() -> {
SysJob job = requireJobForUpdate(id);
if (!Integer.valueOf(EnumJobStatus.RUNNING.getCode()).equals(job.getStatus())) {
throw new BusinessException(409, 409, "任务未处于运行状态,请先启动任务");
}
return scheduleAdapter.triggerNow(id);
}));
}
@Override
public List<Date> nextFireTimes(String cronExpression, int limit) {
return scheduleService.nextFireTimes(new CronSchedulePlan(cronExpression, zoneId), limit)
.stream().map(Date::from).toList();
}
@Override
public List<SysJob> listWorkflowJobsByWorkflowId(BigInteger workflowId) {
if (workflowId == null) {
return List.of();
}
if (workflowId == null) return List.of();
QueryWrapper queryWrapper = QueryWrapper.create()
.eq(SysJob::getJobType, EnumJobType.TINY_FLOW.getCode());
.eq(SysJob::getJobType, EnumJobType.TINY_FLOW.getCode());
return list(queryWrapper).stream()
.filter(job -> workflowId.equals(SysJobWorkflowReferenceSupport.resolveWorkflowId(job)))
.toList();
.filter(job -> workflowId.equals(SysJobWorkflowReferenceSupport.resolveWorkflowId(job)))
.toList();
}
private SysJob requireJob(BigInteger id) {
SysJob job = getById(id);
if (job == null) throw new IllegalStateException("任务不存在id=" + id);
return job;
}
private boolean projectCurrentStateOrStop(BigInteger jobId) {
// 第二段持任务行锁完成 JobStoreTX 投影。Provider 失败必须在同一行锁下把
// 业务状态改为 STOP事务基础设施或 commit 失败则直接抛出,不做破坏性清理。
ProjectionOutcome outcome = inManagementTransaction(() -> {
SysJob current = getMapper().selectByIdForUpdate(jobId);
boolean running = current != null
&& Integer.valueOf(EnumJobStatus.RUNNING.getCode()).equals(current.getStatus());
try {
if (running) {
scheduleAdapter.replace(current);
} else {
scheduleAdapter.delete(jobId);
}
return new ProjectionOutcome(running, null);
} catch (RuntimeException providerFailure) {
if (current != null
&& !Integer.valueOf(EnumJobStatus.STOP.getCode())
.equals(current.getStatus())) {
SysJob stopped = new SysJob();
stopped.setId(jobId);
stopped.setStatus(EnumJobStatus.STOP.getCode());
if (!updateById(stopped)) {
throw new IllegalStateException(
"调度投影失败后无法停止任务id=" + jobId,
providerFailure);
}
}
return new ProjectionOutcome(false, providerFailure);
}
});
if (outcome.failure() == null) return outcome.running();
RuntimeException providerFailure = outcome.failure();
try {
// STOP 已确认提交后再重试删除投影,并再次锁行检查;若更新命令已重新
// 启动任务则跳过旧清理,避免失锁窗口中的 stale delete。
inManagementTransaction(() -> {
SysJob current = getMapper().selectByIdForUpdate(jobId);
if (current == null
|| Integer.valueOf(EnumJobStatus.STOP.getCode())
.equals(current.getStatus())) {
scheduleAdapter.delete(jobId);
}
return null;
});
} catch (RuntimeException cleanupFailure) {
if (cleanupFailure != providerFailure) {
providerFailure.addSuppressed(cleanupFailure);
}
}
cancelPendingBestEffort(jobId, "调度投影同步失败,任务已停止");
throw providerFailure;
}
private SysJob requireJobForUpdate(BigInteger id) {
SysJob job = getMapper().selectByIdForUpdate(id);
if (job == null) throw new IllegalStateException("任务不存在id=" + id);
return job;
}
private <T> T withJobCommandLock(BigInteger id, Supplier<T> command) {
return withManagementPermit(() -> redisLockExecutor.executeWithRenewingLock(
lockKey(id), LOCK_WAIT_TIMEOUT, LOCK_LEASE_TIMEOUT, command));
}
private <T> T inManagementTransaction(Supplier<T> command) {
return managementTransaction.execute(status -> command.get());
}
private <T> T withManagementPermit(Supplier<T> command) {
boolean acquired = false;
try {
acquired = managementPermits.tryAcquire(
LOCK_WAIT_TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
if (!acquired) {
throw new IllegalStateException("定时任务管理命令繁忙,请稍后重试");
}
return command.get();
} catch (InterruptedException exception) {
Thread.currentThread().interrupt();
throw new IllegalStateException("等待定时任务管理命令资源时被中断", exception);
} finally {
if (acquired) managementPermits.release();
}
}
private void cancelPendingBestEffort(BigInteger jobId, String reason) {
try {
// 与 Worker 一致按日志行 -> 任务行加锁,并在管理状态提交后复核最终状态。
executionStore.cancelPending(jobId, reason);
} catch (RuntimeException cleanupFailure) {
// Worker 领取遗留记录时仍会根据 STOP/已删除状态安全取消。
log.warn("定时任务状态已收口,但待执行记录延迟清理: jobId={}",
jobId, cleanupFailure);
}
}
private static String lockKey(BigInteger id) {
if (id == null) throw new IllegalArgumentException("任务ID不能为空");
return JOB_LOCK_KEY_PREFIX + id;
}
private static long nextScheduleGeneration(SysJob job) {
Long generation = job.getScheduleGeneration();
if (generation == null || generation < 0L || generation == Long.MAX_VALUE) {
throw new IllegalStateException(
"定时任务调度代际非法id=" + job.getId() + ", generation=" + generation);
}
return generation + 1L;
}
private record ProjectionOutcome(boolean running, RuntimeException failure) {
}
}

View File

@@ -1,146 +1,18 @@
package tech.easyflow.job.util;
import cn.hutool.core.util.ArrayUtil;
import cn.hutool.core.util.StrUtil;
import org.quartz.JobKey;
import org.quartz.TriggerKey;
import tech.easyflow.common.constant.enums.EnumJobType;
import tech.easyflow.common.util.SpringContextUtil;
import tech.easyflow.job.entity.SysJob;
import tech.easyflow.job.job.JobConstant;
import tech.easyflow.job.service.WorkflowJobExecutionService;
import java.lang.reflect.Method;
import java.util.Arrays;
import java.util.Map;
/** Java 类定时任务的开发环境示例。 */
public class JobUtil {
/**
* sysJobService.test()
*/
public static Object execSpringBean(SysJob job) {
Map<String, Object> jobParams = job.getJobParams();
if (jobParams != null) {
String beanMethod = jobParams.get(JobConstant.BEAN_METHOD_KEY).toString();
String[] strings = StrUtil.subBefore(beanMethod, "(", false).split("\\.");
Object bean = SpringContextUtil.getBean(strings[0]);
String param = StrUtil.subBetween(beanMethod, "(", ")");
try {
// 调用方法并传递参数
return invoke(bean, strings[1], getParams(param));
} catch (Exception e) {
throw new RuntimeException("执行 beanMethod 报错:", e);
}
}
return null;
public String execTest(String a, Integer b, Double c, Long d) {
return "a=" + a + ",b=" + b + ",c=" + c + ",d=" + d;
}
/**
* tech.easyflow.job.util.JobUtil.execTest("test",1,0.52D,100L)
* @param job
*/
public static Object execJavaClass(SysJob job) {
Map<String, Object> jobParams = job.getJobParams();
if (jobParams != null) {
try {
String javaMethod = jobParams.get(JobConstant.JAVA_METHOD_KEY).toString();
String before = StrUtil.subBefore(javaMethod, "(", false);
String[] strings = before.split("\\.");
String className = String.join(".", Arrays.copyOf(strings, strings.length - 1));
String methodName = strings[strings.length - 1];
String param = StrUtil.subBetween(javaMethod, "(", ")");
Object obj = Class.forName(className).getDeclaredConstructor().newInstance();;
return invoke(obj, methodName, getParams(param));
} catch (Exception e) {
throw new RuntimeException("执行 javaMethod 报错: ",e);
}
/** 可用于开发环境容量与并发验收的阻塞型模拟任务。 */
public String sleepTask(String label, Long millis) throws InterruptedException {
if (millis == null || millis < 0 || millis > 300_000) {
throw new IllegalArgumentException("millis must be between 0 and 300000");
}
return null;
}
/**
* 通过任务模块的受控执行服务运行工作流。
*
* @param job Quartz 中保存的任务快照
* @return 工作流执行结果
*/
public static Object execWorkFlow(SysJob job) {
WorkflowJobExecutionService executionService =
SpringContextUtil.getBean(WorkflowJobExecutionService.class);
return executionService.execute(job);
}
public static Object execute(SysJob job) {
Object res = null;
Integer jobType = job.getJobType();
if (EnumJobType.TINY_FLOW.getCode() == jobType) {
res = execWorkFlow(job);
}
if (EnumJobType.SPRING_BEAN.getCode() == jobType) {
res = execSpringBean(job);
}
if (EnumJobType.JAVA_CLASS.getCode() == jobType) {
res = execJavaClass(job);
}
return res;
}
public void execTest(String a,Integer b,Double c,Long d) {
System.out.println("动态执行方法,执行参数:" + "a="+ a + ",b="+ b + ",c="+ c + ",d="+ d);
}
private static Object[] getParams(String param) {
if (StrUtil.isEmpty(param)) {
return new Object[]{new Class<?>[]{}, new Object[]{}};
}
String[] splits = param.split(",");
Object[] res = new Object[2];
Object[] params = new Object[splits.length];
Class<?>[] paramTypes = new Class[splits.length];
for (int i = 0; i < splits.length; i++) {
String split = splits[i].trim();
if (split.startsWith("\"")) {
params[i] = split.substring(1, split.length() - 1);
paramTypes[i] = String.class;
} else if ("true".equals(split) || "false".equals(split)) {
params[i] = Boolean.valueOf(split);
paramTypes[i] = Boolean.class;
} else if (split.endsWith("L")) {
params[i] = Long.valueOf(split.substring(0, split.length() - 1));
paramTypes[i] = Long.class;
} else if (split.endsWith("D")) {
params[i] = Double.valueOf(split.substring(0, split.length() - 1));
paramTypes[i] = Double.class;
} else if (split.endsWith("F")) {
params[i] = Float.valueOf(split.substring(0, split.length() - 1));
paramTypes[i] = Float.class;
} else {
params[i] = Integer.valueOf(split);
paramTypes[i] = Integer.class;
}
}
res[0] = paramTypes;
res[1] = params;
return res;
}
private static Object invoke(Object bean, String methodName, Object[] params) throws Exception {
Object[] args = (Object[]) params[1];
if (ArrayUtil.isEmpty(params[1])) {
Method method = bean.getClass().getDeclaredMethod(methodName);
return method.invoke(bean);
} else {
Method method = bean.getClass().getDeclaredMethod(methodName, (Class<?>[]) params[0]);
return method.invoke(bean, args);
}
}
public static JobKey getJobKey(SysJob job) {
return JobKey.jobKey(job.getId().toString(), JobConstant.JOB_GROUP);
}
public static TriggerKey getTriggerKey(SysJob job) {
return TriggerKey.triggerKey(job.getId().toString(), JobConstant.JOB_GROUP);
Thread.sleep(millis);
return label;
}
}

View File

@@ -0,0 +1,23 @@
package tech.easyflow.job.config;
import org.junit.Assert;
import org.junit.Test;
public class SysJobConnectionCapacityValidatorTest {
@Test
public void configuredBaselineMustReserveAllSchedulerConnections() {
Assert.assertEquals(24,
SysJobConnectionCapacityValidator.requiredPoolSize(4, 8, 4, 4));
SysJobExecutionProperties properties = new SysJobExecutionProperties();
new SysJobConnectionCapacityValidator(properties, 24, 8, true)
.afterPropertiesSet();
}
@Test(expected = IllegalStateException.class)
public void insufficientPoolMustFailFast() {
SysJobExecutionProperties properties = new SysJobExecutionProperties();
new SysJobConnectionCapacityValidator(properties, 23, 8, true)
.afterPropertiesSet();
}
}

View File

@@ -0,0 +1,148 @@
package tech.easyflow.job.execution;
import com.easyagents.scheduler.ScheduleFireContext;
import com.easyagents.scheduler.ScheduleId;
import com.easyagents.scheduler.ScheduleRefireException;
import org.junit.Test;
import org.springframework.dao.DataAccessResourceFailureException;
import org.springframework.dao.DataIntegrityViolationException;
import org.springframework.dao.TransientDataAccessResourceException;
import org.springframework.transaction.CannotCreateTransactionException;
import org.springframework.transaction.TransactionSystemException;
import tech.easyflow.job.config.SysJobExecutionProperties;
import java.time.Duration;
import java.time.Instant;
import java.util.Map;
import static org.mockito.Mockito.doNothing;
import static org.mockito.Mockito.doThrow;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
public class EasyFlowScheduleHandlerTest {
@Test
public void shouldRetryOnlyBoundedTransientRegistrationFailures() {
SysJobExecutionRegistrar registrar = mock(SysJobExecutionRegistrar.class);
ScheduleFireContext context = context();
doThrow(new TransientDataAccessResourceException("temporary"))
.doThrow(new TransientDataAccessResourceException("temporary"))
.doNothing().when(registrar).register(context);
SysJobExecutionProperties properties = new SysJobExecutionProperties();
properties.setRegistrationMaxAttempts(3);
properties.setRegistrationRetryDelay(Duration.ofNanos(1));
new EasyFlowScheduleHandler(registrar, properties,
mock(SysJobExecutionMetrics.class)).execute(context);
verify(registrar, times(3)).register(context);
}
@Test(expected = IllegalStateException.class)
public void shouldNotRetryBusinessFailure() {
SysJobExecutionRegistrar registrar = mock(SysJobExecutionRegistrar.class);
ScheduleFireContext context = context();
doThrow(new IllegalStateException("invalid job")).when(registrar).register(context);
new EasyFlowScheduleHandler(registrar, new SysJobExecutionProperties(),
mock(SysJobExecutionMetrics.class)).execute(context);
}
@Test(expected = ScheduleRefireException.class)
public void shouldRequestQuartzRefireAfterTransientRetriesAreExhausted() {
SysJobExecutionRegistrar registrar = mock(SysJobExecutionRegistrar.class);
ScheduleFireContext context = context();
doThrow(new TransientDataAccessResourceException("temporary"))
.when(registrar).register(context);
SysJobExecutionProperties properties = new SysJobExecutionProperties();
properties.setRegistrationMaxAttempts(2);
properties.setRegistrationRetryDelay(Duration.ofNanos(1));
new EasyFlowScheduleHandler(registrar, properties,
mock(SysJobExecutionMetrics.class)).execute(context);
}
@Test(expected = ScheduleRefireException.class)
public void shouldRequestQuartzRefireWhenDatabaseConnectionIsUnavailable() {
SysJobExecutionRegistrar registrar = mock(SysJobExecutionRegistrar.class);
ScheduleFireContext context = context();
doThrow(new DataAccessResourceFailureException("connection unavailable"))
.when(registrar).register(context);
SysJobExecutionProperties properties = new SysJobExecutionProperties();
properties.setRegistrationMaxAttempts(2);
properties.setRegistrationRetryDelay(Duration.ofNanos(1));
try {
new EasyFlowScheduleHandler(registrar, properties,
mock(SysJobExecutionMetrics.class)).execute(context);
} finally {
verify(registrar, times(2)).register(context);
}
}
@Test(expected = ScheduleRefireException.class)
public void transactionBeginFailureMustRequestQuartzRefire() {
assertTransactionFailureRequestsRefire(
new CannotCreateTransactionException("pool exhausted"));
}
@Test(expected = ScheduleRefireException.class)
public void transactionCommitFailureMustRequestQuartzRefire() {
assertTransactionFailureRequestsRefire(
new TransactionSystemException("commit failed"));
}
@Test(expected = ScheduleRefireException.class)
public void interruptedLocalRetryMustStillRequestDurableQuartzRefire() {
SysJobExecutionRegistrar registrar = mock(SysJobExecutionRegistrar.class);
ScheduleFireContext context = context();
doThrow(new TransientDataAccessResourceException("temporary"))
.when(registrar).register(context);
SysJobExecutionProperties properties = new SysJobExecutionProperties();
properties.setRegistrationMaxAttempts(3);
properties.setRegistrationRetryDelay(Duration.ofSeconds(1));
Thread.currentThread().interrupt();
try {
new EasyFlowScheduleHandler(registrar, properties,
mock(SysJobExecutionMetrics.class)).execute(context);
} finally {
Thread.interrupted();
}
}
@Test(expected = DataIntegrityViolationException.class)
public void shouldNotRetryNonRecoverableDataError() {
SysJobExecutionRegistrar registrar = mock(SysJobExecutionRegistrar.class);
ScheduleFireContext context = context();
doThrow(new DataIntegrityViolationException("invalid row"))
.when(registrar).register(context);
try {
new EasyFlowScheduleHandler(registrar, new SysJobExecutionProperties(),
mock(SysJobExecutionMetrics.class)).execute(context);
} finally {
verify(registrar, times(1)).register(context);
}
}
private static void assertTransactionFailureRequestsRefire(RuntimeException failure) {
SysJobExecutionRegistrar registrar = mock(SysJobExecutionRegistrar.class);
ScheduleFireContext context = context();
doThrow(failure).when(registrar).register(context);
SysJobExecutionProperties properties = new SysJobExecutionProperties();
properties.setRegistrationMaxAttempts(2);
properties.setRegistrationRetryDelay(Duration.ofNanos(1));
new EasyFlowScheduleHandler(registrar, properties,
mock(SysJobExecutionMetrics.class)).execute(context);
}
private static ScheduleFireContext context() {
Instant now = Instant.now();
return new ScheduleFireContext(new ScheduleId("easyflow.job", "1"),
EasyFlowScheduleHandler.CODE, now, now, "fire-1", null, false, Map.of());
}
}

View File

@@ -0,0 +1,83 @@
package tech.easyflow.job.execution;
import org.apache.ibatis.annotations.Select;
import org.apache.ibatis.annotations.Update;
import org.junit.Assert;
import org.junit.Test;
import tech.easyflow.job.mapper.SysJobLogMapper;
import java.lang.reflect.Method;
import java.util.Locale;
public class SysJobExecutionMapperContractTest {
@Test
public void claimShouldUseSkipLockedAndOwnedTerminalCas() throws Exception {
String expiredSql = sql(SysJobLogMapper.class.getMethod(
"selectExpiredClaimCandidateId").getAnnotation(Select.class).value());
String expiredProbeSql = sql(SysJobLogMapper.class.getMethod(
"selectExpiredClaimCandidateIdWithoutLock").getAnnotation(Select.class).value());
String pendingSql = sql(SysJobLogMapper.class.getMethod(
"selectPendingClaimCandidateId").getAnnotation(Select.class).value());
String finishSql = sql(SysJobLogMapper.class.getMethod("finishOwned",
java.math.BigInteger.class, String.class, String.class, int.class,
String.class, String.class).getAnnotation(Update.class).value());
Assert.assertTrue(expiredSql.contains("FOR UPDATE SKIP LOCKED"));
Assert.assertFalse(expiredProbeSql.contains("FOR UPDATE"));
Assert.assertTrue(expiredProbeSql.contains("STATUS=3"));
Assert.assertTrue(pendingSql.contains("FOR UPDATE SKIP LOCKED"));
Assert.assertTrue(expiredSql.startsWith("SELECT ID "));
Assert.assertTrue(pendingSql.startsWith("SELECT ID "));
Assert.assertFalse(expiredSql.contains("SELECT *"));
Assert.assertFalse(pendingSql.contains("SELECT *"));
Assert.assertFalse(expiredSql.contains(" OR "));
Assert.assertFalse(pendingSql.contains(" OR "));
Assert.assertTrue(finishSql.contains("STATUS=3"));
Assert.assertTrue(finishSql.contains("LEASE_OWNER=#{OWNER}"));
Assert.assertTrue(finishSql.contains("EXECUTION_TOKEN=#{TOKEN}"));
}
@Test
public void heartbeatMustNotAdvanceFencingVersion() throws Exception {
Method method = SysJobLogMapper.class.getMethod("renewLease",
java.math.BigInteger.class, String.class, String.class,
long.class);
String heartbeatSql = sql(method.getAnnotation(Update.class).value());
Assert.assertFalse(heartbeatSql.contains("VERSION"));
Assert.assertTrue(heartbeatSql.contains("LEASE_OWNER=#{OWNER}"));
Assert.assertTrue(heartbeatSql.contains("EXECUTION_TOKEN=#{TOKEN}"));
Assert.assertTrue(heartbeatSql.contains("CURRENT_TIMESTAMP(3)"));
}
@Test
public void exhaustedPendingInfrastructureRetryMustLeaveClaimQueue() throws Exception {
String sql = sql(SysJobLogMapper.class.getMethod("markPendingDead",
java.math.BigInteger.class, String.class).getAnnotation(Update.class).value());
Assert.assertTrue(sql.contains("STATUS=4"));
Assert.assertTrue(sql.contains("WHERE ID=#{ID} AND STATUS=2"));
Assert.assertTrue(sql.contains("NEXT_RETRY_TIME=NULL"));
}
@Test
public void cancellationUpdateMustOnlyTouchPendingLedgerRows() throws Exception {
String lockSql = sql(SysJobLogMapper.class.getMethod(
"selectPendingIdsByJobIdForUpdate", java.math.BigInteger.class)
.getAnnotation(org.apache.ibatis.annotations.Select.class).value());
String sql = sql(SysJobLogMapper.class.getMethod("cancelPendingByJobId",
java.math.BigInteger.class, String.class)
.getAnnotation(Update.class).value());
Assert.assertTrue(lockSql.contains("WHERE JOB_ID=#{JOBID} AND STATUS=2"));
Assert.assertTrue(lockSql.contains("ORDER BY ID FOR UPDATE"));
Assert.assertTrue(sql.contains("WHERE JOB_ID=#{JOBID} AND STATUS=2"));
Assert.assertFalse(sql.contains("FROM TB_SYS_JOB"));
}
private static String sql(String[] fragments) {
return String.join(" ", fragments).replaceAll("\\s+", " ")
.toUpperCase(Locale.ROOT);
}
}

View File

@@ -0,0 +1,28 @@
package tech.easyflow.job.execution;
import io.micrometer.core.instrument.simple.SimpleMeterRegistry;
import org.junit.Assert;
import org.junit.Test;
public class SysJobExecutionMetricsTest {
@Test
public void completionWithoutOwnedTerminalStatusMustBeAccepted() {
SimpleMeterRegistry registry = new SimpleMeterRegistry();
SysJobExecutionMetrics metrics = new SysJobExecutionMetrics(registry);
metrics.executionStarted();
metrics.executionFinished(null, 1L);
Assert.assertEquals(0.0D,
registry.get("easyflow.job.execution.local_active").gauge().value(), 0.0D);
Assert.assertEquals(1L,
registry.get("easyflow.job.execution.duration").timer().count());
Assert.assertEquals(0.0D,
registry.get("easyflow.job.execution.terminal")
.tag("status", "success").counter().count(), 0.0D);
Assert.assertEquals(0.0D,
registry.get("easyflow.job.execution.terminal")
.tag("status", "failure").counter().count(), 0.0D);
}
}

View File

@@ -0,0 +1,210 @@
package tech.easyflow.job.execution;
import com.easyagents.scheduler.ScheduleFireContext;
import com.easyagents.scheduler.ScheduleId;
import org.junit.Assert;
import org.junit.Test;
import org.springframework.dao.CannotAcquireLockException;
import org.springframework.dao.DuplicateKeyException;
import org.springframework.transaction.PlatformTransactionManager;
import org.springframework.transaction.TransactionStatus;
import tech.easyflow.common.constant.enums.EnumJobStatus;
import tech.easyflow.job.config.SysJobExecutionProperties;
import tech.easyflow.job.entity.SysJob;
import tech.easyflow.job.entity.SysJobLog;
import tech.easyflow.job.mapper.SysJobLogMapper;
import tech.easyflow.job.mapper.SysJobMapper;
import tech.easyflow.job.job.JobConstant;
import java.math.BigInteger;
import java.sql.SQLException;
import java.time.Instant;
import java.util.Map;
import java.util.Optional;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyLong;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.doThrow;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
public class SysJobExecutionStoreTest {
@Test
public void executionKeyDuplicateMustBeAcceptedAsIdempotentRegistration() {
RegistrationFixture fixture = registrationFixture();
doThrow(new DuplicateKeyException("execution key duplicate"))
.when(fixture.logMapper()).insert(any(SysJobLog.class));
when(fixture.logMapper().selectIdByExecutionKey(anyString()))
.thenReturn(BigInteger.valueOf(999L));
fixture.store().register(fixture.context());
verify(fixture.logMapper()).selectIdByExecutionKey(anyString());
}
@Test
public void unrelatedUniqueConstraintViolationMustNotLoseFire() {
RegistrationFixture fixture = registrationFixture();
DuplicateKeyException duplicate = new DuplicateKeyException("primary key duplicate");
doThrow(duplicate).when(fixture.logMapper()).insert(any(SysJobLog.class));
when(fixture.logMapper().selectIdByExecutionKey(anyString())).thenReturn(null);
DuplicateKeyException thrown = Assert.assertThrows(
DuplicateKeyException.class,
() -> fixture.store().register(fixture.context()));
Assert.assertSame(duplicate, thrown);
verify(fixture.logMapper()).selectIdByExecutionKey(anyString());
}
@Test
public void exhaustedHeadRowsMustNotDelayFollowingRunnableExecution() {
SysJobMapper jobMapper = mock(SysJobMapper.class);
SysJobLogMapper logMapper = mock(SysJobLogMapper.class);
SysJobExecutionMetrics metrics = mock(SysJobExecutionMetrics.class);
PlatformTransactionManager transactionManager = mock(PlatformTransactionManager.class);
when(transactionManager.getTransaction(any())).thenReturn(mock(TransactionStatus.class));
SysJobExecutionProperties properties = new SysJobExecutionProperties();
SysJobLog exhausted1 = execution(101L, 1_001L,
properties.getInfrastructureRetryLimit());
SysJobLog exhausted2 = execution(102L, 1_002L,
properties.getInfrastructureRetryLimit());
SysJobLog runnable = execution(103L, 1_003L, 0);
when(logMapper.selectExpiredClaimCandidate()).thenReturn(null);
when(logMapper.selectPendingClaimCandidate())
.thenReturn(exhausted1, exhausted2, runnable);
when(logMapper.markPendingDead(any(), anyString())).thenReturn(1);
when(logMapper.claimPending(eq(runnable.getId()), eq("node-a"),
anyString(), anyLong())).thenReturn(1);
SysJob activeJob = new SysJob();
activeJob.setId(runnable.getJobId());
activeJob.setStatus(EnumJobStatus.RUNNING.getCode());
when(jobMapper.selectByIdForUpdate(runnable.getJobId())).thenReturn(activeJob);
SysJobExecutionStore store = new SysJobExecutionStore(
jobMapper, logMapper, properties, metrics, transactionManager);
Optional<ClaimedSysJobExecution> claimed = store.claimOne("node-a");
Assert.assertTrue(claimed.isPresent());
Assert.assertEquals(runnable.getId(), claimed.get().execution().getId());
verify(logMapper, times(2)).markPendingDead(any(), anyString());
verify(metrics, times(2)).recordDead();
verify(transactionManager, times(3)).commit(any());
}
@Test
public void transientClaimDeadlockMustRetryInANewTransaction() {
SysJobMapper jobMapper = mock(SysJobMapper.class);
SysJobLogMapper logMapper = mock(SysJobLogMapper.class);
SysJobExecutionMetrics metrics = mock(SysJobExecutionMetrics.class);
PlatformTransactionManager transactionManager = mock(PlatformTransactionManager.class);
when(transactionManager.getTransaction(any())).thenReturn(mock(TransactionStatus.class));
SysJobExecutionProperties properties = new SysJobExecutionProperties();
SysJobLog runnable = execution(201L, 2_001L, 0);
when(logMapper.selectExpiredClaimCandidate()).thenReturn(null);
when(logMapper.selectPendingClaimCandidate())
.thenThrow(new CannotAcquireLockException(
"simulated deadlock", new SQLException("deadlock")))
.thenReturn(runnable);
SysJob activeJob = new SysJob();
activeJob.setId(runnable.getJobId());
activeJob.setStatus(EnumJobStatus.RUNNING.getCode());
when(jobMapper.selectByIdForUpdate(runnable.getJobId())).thenReturn(activeJob);
when(logMapper.claimPending(eq(runnable.getId()), eq("node-a"),
anyString(), anyLong())).thenReturn(1);
SysJobExecutionStore store = new SysJobExecutionStore(
jobMapper, logMapper, properties, metrics, transactionManager);
Optional<ClaimedSysJobExecution> claimed = store.claimOne("node-a");
Assert.assertTrue(claimed.isPresent());
Assert.assertEquals(runnable.getId(), claimed.get().execution().getId());
verify(transactionManager, times(2)).getTransaction(any());
verify(transactionManager, times(1)).rollback(any());
verify(transactionManager, times(1)).commit(any());
verify(logMapper, never()).markPendingDead(any(), anyString());
}
@Test
public void lockedExpiredCandidateMustFallBackToPendingExecution() {
SysJobMapper jobMapper = mock(SysJobMapper.class);
SysJobLogMapper logMapper = mock(SysJobLogMapper.class);
SysJobExecutionMetrics metrics = mock(SysJobExecutionMetrics.class);
PlatformTransactionManager transactionManager = mock(PlatformTransactionManager.class);
when(transactionManager.getTransaction(any())).thenReturn(mock(TransactionStatus.class));
SysJobExecutionProperties properties = new SysJobExecutionProperties();
SysJobLog runnable = execution(301L, 3_001L, 0);
when(logMapper.selectExpiredClaimCandidateIdWithoutLock())
.thenReturn(BigInteger.valueOf(300L));
// 过期候选已被其他 Worker 锁定SKIP LOCKED 后必须继续领取 PENDING。
when(logMapper.selectExpiredClaimCandidate()).thenReturn(null);
when(logMapper.selectPendingClaimCandidate()).thenReturn(runnable);
SysJob activeJob = new SysJob();
activeJob.setId(runnable.getJobId());
activeJob.setStatus(EnumJobStatus.RUNNING.getCode());
when(jobMapper.selectByIdForUpdate(runnable.getJobId())).thenReturn(activeJob);
when(logMapper.claimPending(eq(runnable.getId()), eq("node-a"),
anyString(), anyLong())).thenReturn(1);
SysJobExecutionStore store = new SysJobExecutionStore(
jobMapper, logMapper, properties, metrics, transactionManager);
Optional<ClaimedSysJobExecution> claimed = store.claimOne("node-a");
Assert.assertTrue(claimed.isPresent());
Assert.assertEquals(runnable.getId(), claimed.get().execution().getId());
verify(logMapper).selectExpiredClaimCandidate();
verify(logMapper).selectPendingClaimCandidate();
verify(transactionManager, times(2)).commit(any());
}
private static SysJobLog execution(long id, long jobId, int attempts) {
SysJobLog execution = new SysJobLog();
execution.setId(BigInteger.valueOf(id));
execution.setJobId(BigInteger.valueOf(jobId));
execution.setAttemptCount(attempts);
return execution;
}
private static RegistrationFixture registrationFixture() {
SysJobMapper jobMapper = mock(SysJobMapper.class);
SysJobLogMapper logMapper = mock(SysJobLogMapper.class);
SysJobExecutionMetrics metrics = mock(SysJobExecutionMetrics.class);
PlatformTransactionManager transactionManager = mock(PlatformTransactionManager.class);
when(transactionManager.getTransaction(any())).thenReturn(mock(TransactionStatus.class));
SysJob job = new SysJob();
job.setId(BigInteger.ONE);
job.setTenantId(BigInteger.TWO);
job.setStatus(EnumJobStatus.RUNNING.getCode());
job.setScheduleGeneration(0L);
when(jobMapper.selectByIdForUpdate(BigInteger.ONE)).thenReturn(job);
Instant now = Instant.now();
ScheduleFireContext context = new ScheduleFireContext(
new ScheduleId("easyflow.job", "1"),
EasyFlowScheduleHandler.CODE,
now,
now,
"fire-1",
null,
false,
Map.of(JobConstant.SCHEDULE_GENERATION, "0"));
return new RegistrationFixture(
new SysJobExecutionStore(jobMapper, logMapper,
new SysJobExecutionProperties(), metrics, transactionManager),
logMapper,
context);
}
private record RegistrationFixture(SysJobExecutionStore store,
SysJobLogMapper logMapper,
ScheduleFireContext context) {
}
}

View File

@@ -0,0 +1,256 @@
package tech.easyflow.job.execution;
import io.micrometer.core.instrument.simple.SimpleMeterRegistry;
import org.junit.Assert;
import org.junit.Test;
import tech.easyflow.job.config.SysJobExecutionProperties;
import tech.easyflow.job.entity.SysJobLog;
import java.math.BigInteger;
import java.time.Duration;
import java.util.Optional;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyInt;
import static org.mockito.ArgumentMatchers.anyLong;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.ArgumentMatchers.nullable;
import static org.mockito.Mockito.doAnswer;
import static org.mockito.Mockito.doNothing;
import static org.mockito.Mockito.doThrow;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.timeout;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
public class SysJobExecutionWorkerTest {
@Test
public void nonVmErrorMustNotPermanentlyReduceWorkerCapacity() throws Exception {
SysJobExecutionStore store = mock(SysJobExecutionStore.class);
SysJobInvoker invoker = mock(SysJobInvoker.class);
SysJobExecutionMetrics metrics = mock(SysJobExecutionMetrics.class);
SysJobExecutionProperties properties = properties();
ClaimedSysJobExecution first = claim(101L, "token-1");
ClaimedSysJobExecution second = claim(102L, "token-2");
when(store.queueSnapshot()).thenReturn(new SysJobQueueSnapshot(0, 0, 0));
when(store.claimOne(any())).thenReturn(
Optional.of(first), Optional.of(second), Optional.empty());
when(store.finish(any(), anyInt(), nullable(String.class), nullable(String.class)))
.thenReturn(true);
CountDownLatch secondCompleted = new CountDownLatch(1);
AtomicInteger calls = new AtomicInteger();
doAnswer(invocation -> {
if (calls.incrementAndGet() == 1) {
throw new AssertionError("broken business handler");
}
secondCompleted.countDown();
return "ok";
}).when(invoker).execute(any());
SysJobExecutionWorker worker = new SysJobExecutionWorker(
store, invoker, properties, metrics);
worker.start();
try {
Assert.assertTrue("worker 未在非 VM Error 后继续领取任务",
secondCompleted.await(3, TimeUnit.SECONDS));
verify(store, timeout(1_000)).finish(
first, 0, null, "AssertionError: broken business handler");
verify(store, timeout(1_000)).finish(second, 1, "\"ok\"", null);
} finally {
worker.stop();
}
}
@Test
public void preparationInfrastructureFailureMustReleaseForRetry() throws Exception {
SysJobExecutionStore store = mock(SysJobExecutionStore.class);
SysJobInvoker invoker = mock(SysJobInvoker.class);
SysJobExecutionMetrics metrics = mock(SysJobExecutionMetrics.class);
ClaimedSysJobExecution claim = claim(201L, "token-infra");
when(store.queueSnapshot()).thenReturn(new SysJobQueueSnapshot(0, 0, 0));
when(store.claimOne(any())).thenReturn(Optional.of(claim), Optional.empty());
CountDownLatch released = new CountDownLatch(1);
when(store.releaseForInfrastructureRetry(any(), any())).thenAnswer(invocation -> {
released.countDown();
return true;
});
doAnswer(invocation -> {
throw new SysJobInfrastructureException("database unavailable",
new IllegalStateException("connection lost"));
}).when(invoker).execute(any());
SysJobExecutionWorker worker = new SysJobExecutionWorker(
store, invoker, properties(), metrics);
worker.start();
try {
Assert.assertTrue(released.await(3, TimeUnit.SECONDS));
verify(store).releaseForInfrastructureRetry(
claim, "IllegalStateException: connection lost");
} finally {
worker.stop();
}
}
@Test
public void metricsFailureMustNotLeakContextOrActiveExecution() throws Exception {
SysJobExecutionStore store = mock(SysJobExecutionStore.class);
SysJobInvoker invoker = mock(SysJobInvoker.class);
SysJobExecutionMetrics metrics = mock(SysJobExecutionMetrics.class);
ClaimedSysJobExecution first = claim(251L, "token-metrics-1");
ClaimedSysJobExecution second = claim(252L, "token-metrics-2");
when(store.queueSnapshot()).thenReturn(new SysJobQueueSnapshot(0, 0, 0));
when(store.claimOne(any())).thenReturn(
Optional.of(first), Optional.of(second), Optional.empty());
when(store.finish(any(), anyInt(), nullable(String.class), nullable(String.class)))
.thenReturn(true);
doThrow(new IllegalStateException("meter unavailable"))
.doNothing()
.when(metrics).executionFinished(nullable(String.class), anyLong());
CountDownLatch secondCompleted = new CountDownLatch(1);
AtomicInteger calls = new AtomicInteger();
doAnswer(invocation -> {
if (calls.incrementAndGet() == 2) secondCompleted.countDown();
return "ok";
}).when(invoker).execute(any());
SysJobExecutionWorker worker = new SysJobExecutionWorker(
store, invoker, properties(), metrics);
worker.start();
try {
Assert.assertTrue("指标收尾异常后 Worker 未继续执行下一任务",
secondCompleted.await(3, TimeUnit.SECONDS));
verify(store, timeout(1_000)).finish(first, 1, "\"ok\"", null);
verify(store, timeout(1_000)).finish(second, 1, "\"ok\"", null);
} finally {
worker.stop();
}
}
@Test
public void fencingRejectionMustNotPolluteNextExecution() throws Exception {
SysJobExecutionStore store = mock(SysJobExecutionStore.class);
SysJobInvoker invoker = mock(SysJobInvoker.class);
SysJobExecutionMetrics metrics = new SysJobExecutionMetrics(
new SimpleMeterRegistry());
ClaimedSysJobExecution first = claim(271L, "token-fenced");
ClaimedSysJobExecution second = claim(272L, "token-next");
when(store.queueSnapshot()).thenReturn(new SysJobQueueSnapshot(0, 0, 0));
when(store.claimOne(any())).thenReturn(
Optional.of(first), Optional.of(second), Optional.empty());
when(store.finish(any(), anyInt(), nullable(String.class), nullable(String.class)))
.thenReturn(false, true);
CountDownLatch secondCompleted = new CountDownLatch(1);
AtomicInteger calls = new AtomicInteger();
doAnswer(invocation -> {
if (calls.incrementAndGet() == 2) secondCompleted.countDown();
return "ok";
}).when(invoker).execute(any());
SysJobExecutionWorker worker = new SysJobExecutionWorker(
store, invoker, properties(), metrics);
worker.start();
try {
Assert.assertTrue("fencing 拒绝后 Worker 未继续执行下一任务",
secondCompleted.await(3, TimeUnit.SECONDS));
verify(store, timeout(1_000)).finish(first, 1, "\"ok\"", null);
verify(store, timeout(1_000)).finish(second, 1, "\"ok\"", null);
} finally {
worker.stop();
}
}
@Test
public void staleHeartbeatMustNotInterruptNextExecutionOnReusedWorkerThread()
throws Exception {
SysJobExecutionStore store = mock(SysJobExecutionStore.class);
SysJobInvoker invoker = mock(SysJobInvoker.class);
SysJobExecutionMetrics metrics = mock(SysJobExecutionMetrics.class);
SysJobExecutionProperties properties = properties();
properties.setHeartbeatInterval(Duration.ofMillis(20));
ClaimedSysJobExecution first = claim(301L, "token-stale");
ClaimedSysJobExecution second = claim(302L, "token-current");
when(store.queueSnapshot()).thenReturn(new SysJobQueueSnapshot(0, 0, 0));
when(store.claimOne(any())).thenReturn(
Optional.of(first), Optional.of(second), Optional.empty());
when(store.finish(any(), anyInt(), nullable(String.class), nullable(String.class)))
.thenReturn(true);
when(store.heartbeat(eq(second))).thenReturn(true);
CountDownLatch firstStarted = new CountDownLatch(1);
CountDownLatch releaseFirst = new CountDownLatch(1);
CountDownLatch staleHeartbeatStarted = new CountDownLatch(1);
CountDownLatch lostLeaseMetricStarted = new CountDownLatch(1);
CountDownLatch releaseLostLeaseMetric = new CountDownLatch(1);
CountDownLatch secondStarted = new CountDownLatch(1);
CountDownLatch releaseSecond = new CountDownLatch(1);
when(store.heartbeat(eq(first))).thenAnswer(invocation -> {
staleHeartbeatStarted.countDown();
return false;
});
doAnswer(invocation -> {
lostLeaseMetricStarted.countDown();
releaseLostLeaseMetric.await(3, TimeUnit.SECONDS);
return null;
}).when(metrics).recordLostLease();
AtomicInteger calls = new AtomicInteger();
doAnswer(invocation -> {
if (calls.incrementAndGet() == 1) {
firstStarted.countDown();
releaseFirst.await(3, TimeUnit.SECONDS);
return "first";
}
secondStarted.countDown();
releaseSecond.await(3, TimeUnit.SECONDS);
return "second";
}).when(invoker).execute(any());
SysJobExecutionWorker worker = new SysJobExecutionWorker(
store, invoker, properties, metrics);
worker.start();
try {
Assert.assertTrue(firstStarted.await(1, TimeUnit.SECONDS));
Assert.assertTrue(staleHeartbeatStarted.await(1, TimeUnit.SECONDS));
Assert.assertTrue(lostLeaseMetricStarted.await(1, TimeUnit.SECONDS));
releaseFirst.countDown();
Assert.assertTrue(secondStarted.await(1, TimeUnit.SECONDS));
releaseLostLeaseMetric.countDown();
Thread.sleep(100L);
releaseSecond.countDown();
verify(store, timeout(1_000)).finish(second, 1, "\"second\"", null);
verify(store, never()).finish(eq(second), eq(0),
nullable(String.class), nullable(String.class));
verify(metrics).recordLostLease();
} finally {
releaseFirst.countDown();
releaseLostLeaseMetric.countDown();
releaseSecond.countDown();
worker.stop();
}
}
private static SysJobExecutionProperties properties() {
SysJobExecutionProperties properties = new SysJobExecutionProperties();
properties.setWorkerCount(1);
properties.setPollInterval(Duration.ofMillis(10));
properties.setLeaseDuration(Duration.ofSeconds(3));
properties.setHeartbeatInterval(Duration.ofMillis(500));
properties.setShutdownWaitTimeout(Duration.ofSeconds(1));
return properties;
}
private static ClaimedSysJobExecution claim(long id, String token) {
SysJobLog execution = new SysJobLog();
execution.setId(BigInteger.valueOf(id));
execution.setJobId(BigInteger.valueOf(id + 1_000L));
execution.setTenantId(BigInteger.ONE);
execution.setDeptId(BigInteger.ONE);
execution.setJobType(3);
execution.setAttemptCount(1);
return new ClaimedSysJobExecution(execution, "node-a", token);
}
}

View File

@@ -0,0 +1,140 @@
package tech.easyflow.job.execution;
import org.junit.Assert;
import org.junit.Test;
import org.springframework.aop.framework.ProxyFactory;
import org.springframework.context.ApplicationContext;
import tech.easyflow.common.constant.enums.EnumJobStatus;
import tech.easyflow.common.constant.enums.EnumJobType;
import tech.easyflow.job.entity.SysJob;
import tech.easyflow.job.job.JobConstant;
import tech.easyflow.job.mapper.SysJobMapper;
import tech.easyflow.job.service.WorkflowJobExecutionService;
import tech.easyflow.system.entity.SysAccount;
import java.math.BigInteger;
import java.util.Map;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
public class SysJobInvokerTest {
@Test
public void springBeanInvocationMustUseJdkProxyPublicMethod() throws Exception {
Fixture fixture = fixture(EnumJobType.SPRING_BEAN);
EchoService target = value -> "echo:" + value;
ProxyFactory proxyFactory = new ProxyFactory(target);
proxyFactory.setInterfaces(EchoService.class);
when(fixture.applicationContext().getBean("echoService"))
.thenReturn(proxyFactory.getProxy());
fixture.snapshot().setJobParams(Map.of(
JobConstant.BEAN_METHOD_KEY, "echoService.echo(\"hello\")"));
Object result = fixture.invoker().execute(fixture.snapshot());
Assert.assertEquals("echo:hello", result);
verify(fixture.ownerValidator()).requireAvailableOwner(fixture.current());
}
@Test
public void springBeanInvocationMustUseCglibProxyPublicMethod() throws Exception {
Fixture fixture = fixture(EnumJobType.SPRING_BEAN);
ProxyFactory proxyFactory = new ProxyFactory(new EchoServiceImpl());
proxyFactory.setProxyTargetClass(true);
when(fixture.applicationContext().getBean("echoService"))
.thenReturn(proxyFactory.getProxy());
fixture.snapshot().setJobParams(Map.of(
JobConstant.BEAN_METHOD_KEY, "echoService.echo(\"hello\")"));
Object result = fixture.invoker().execute(fixture.snapshot());
Assert.assertEquals("echo:hello", result);
}
@Test
public void javaClassInvocationMustRejectPrivateMethod() {
Fixture fixture = fixture(EnumJobType.JAVA_CLASS);
fixture.snapshot().setJobParams(Map.of(
JobConstant.JAVA_METHOD_KEY,
PrivateJavaTask.class.getName() + ".hidden()"));
NoSuchMethodException exception = Assert.assertThrows(
NoSuchMethodException.class,
() -> fixture.invoker().execute(fixture.snapshot()));
Assert.assertTrue(exception.getMessage().contains("hidden"));
}
@Test
public void invalidOwnerMustBlockSpringBeanBeforeLookup() {
Fixture fixture = fixture(EnumJobType.SPRING_BEAN);
fixture.snapshot().setJobParams(Map.of(
JobConstant.BEAN_METHOD_KEY, "echoService.echo(\"hello\")"));
when(fixture.ownerValidator().requireAvailableOwner(fixture.current()))
.thenThrow(new IllegalStateException("owner disabled"));
IllegalStateException exception = Assert.assertThrows(
IllegalStateException.class,
() -> fixture.invoker().execute(fixture.snapshot()));
Assert.assertEquals("owner disabled", exception.getMessage());
verify(fixture.applicationContext(), never()).getBean(any(String.class));
}
private static Fixture fixture(EnumJobType type) {
SysJobMapper jobMapper = mock(SysJobMapper.class);
ApplicationContext applicationContext = mock(ApplicationContext.class);
WorkflowJobExecutionService workflowExecutionService =
mock(WorkflowJobExecutionService.class);
SysJobOwnerValidator ownerValidator = mock(SysJobOwnerValidator.class);
SysJob snapshot = job(101L, type);
SysJob current = job(101L, type);
current.setStatus(EnumJobStatus.RUNNING.getCode());
current.setCreatedBy(BigInteger.valueOf(301L));
SysAccount owner = new SysAccount();
owner.setId(current.getCreatedBy());
owner.setTenantId(current.getTenantId());
when(jobMapper.selectOneById(snapshot.getId())).thenReturn(current);
when(ownerValidator.requireAvailableOwner(current)).thenReturn(owner);
return new Fixture(
new SysJobInvoker(jobMapper, applicationContext,
workflowExecutionService, ownerValidator),
applicationContext, ownerValidator, snapshot, current);
}
private static SysJob job(long id, EnumJobType type) {
SysJob job = new SysJob();
job.setId(BigInteger.valueOf(id));
job.setTenantId(BigInteger.valueOf(201L));
job.setJobType(type.getCode());
return job;
}
public interface EchoService {
String echo(String value);
}
public static class EchoServiceImpl implements EchoService {
@Override
public String echo(String value) {
return "echo:" + value;
}
}
public static class PrivateJavaTask {
private String hidden() {
return "hidden";
}
}
private record Fixture(SysJobInvoker invoker,
ApplicationContext applicationContext,
SysJobOwnerValidator ownerValidator,
SysJob snapshot,
SysJob current) {
}
}

View File

@@ -0,0 +1,778 @@
package tech.easyflow.job.execution;
import org.junit.After;
import org.junit.Assert;
import org.junit.Assume;
import org.junit.Before;
import org.junit.Test;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
import java.util.UUID;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
/**
* 在显式提供的 MySQL 8 测试库上验证执行账本的真实事务协议。
*/
public class SysJobLedgerMySqlIntegrationTest {
private String jdbcUrl;
private String username;
private String password;
private long jobId;
private long firstExecutionId;
private long secondExecutionId;
private long registrationExecutionId;
private long generationExecutionId;
@Before
public void setUp() throws Exception {
jdbcUrl = System.getenv("EASYFLOW_JOB_IT_JDBC_URL");
username = System.getenv("EASYFLOW_JOB_IT_USERNAME");
password = System.getenv("EASYFLOW_JOB_IT_PASSWORD");
Assume.assumeTrue("未配置真实 MySQL 集成测试环境",
jdbcUrl != null && username != null && password != null);
long suffix = Math.abs(UUID.randomUUID().getMostSignificantBits() % 100_000_000L);
jobId = 8_100_000_000_000_000L + suffix * 10L;
firstExecutionId = jobId + 1L;
secondExecutionId = jobId + 2L;
registrationExecutionId = jobId + 3L;
generationExecutionId = jobId + 4L;
try (Connection connection = open(); Statement statement = connection.createStatement()) {
try (ResultSet result = statement.executeQuery("SELECT VERSION()")) {
Assert.assertTrue(result.next());
Assert.assertTrue("需要 MySQL 8实际为 " + result.getString(1),
result.getString(1).startsWith("8."));
}
insertFixtures(connection);
}
}
@After
public void tearDown() throws Exception {
if (jdbcUrl == null || jobId == 0L) return;
try (Connection connection = open(); PreparedStatement logs = connection.prepareStatement(
"DELETE FROM tb_sys_job_log WHERE job_id=?");
PreparedStatement job = connection.prepareStatement(
"DELETE FROM tb_sys_job WHERE id=?")) {
logs.setLong(1, jobId);
logs.executeUpdate();
job.setLong(1, jobId);
job.executeUpdate();
}
}
@Test
public void shouldEnforceSkipLockedNonConcurrencyAndFencing() throws Exception {
verifySkipLockedClaim();
resetExecutions();
verifyUnlockedExpiredProbeDoesNotDeadlockPendingClaims();
resetExecutions();
verifyLockedExpiredCandidateDoesNotStarvePendingClaim();
resetExecutions();
verifyNonConcurrentJobRowLock();
verifyOwnerTokenFencing();
verifyManagementCommandRowLock();
verifyManagementCommitsBeforeLedgerCleanup();
verifyCleanupUsesLedgerThenJobLock();
verifyRegistrationWaitsForStartCommit();
verifyStaleCleanupWaitsForStartCommit();
verifyDefinitionEditCannotResurrectConcurrentStop();
verifyRestartClearsOldPendingGeneration();
verifyOldFireCannotCrossRestartGeneration();
verifyOldFireCannotCrossDefinitionUpdate();
verifyRegisteredExecutionSurvivesDefinitionGeneration();
verifyExpiredRegisteredExecutionSurvivesDefinitionGeneration();
verifyDuplicateExecutionKeyCanBeReadInSameTransaction();
}
private void verifySkipLockedClaim() throws Exception {
try (Connection first = transactional(); Connection second = transactional()) {
Assert.assertEquals(firstExecutionId, selectPendingForUpdate(first));
Assert.assertEquals(secondExecutionId, selectPendingForUpdate(second));
Assert.assertEquals(1, claim(first, firstExecutionId, "node-a", "token-a"));
first.commit();
Assert.assertEquals(0, claim(second, firstExecutionId, "node-b", "token-b"));
second.rollback();
}
}
private void verifyUnlockedExpiredProbeDoesNotDeadlockPendingClaims() throws Exception {
ExecutorService executor = Executors.newFixedThreadPool(2);
CountDownLatch ready = new CountDownLatch(2);
CountDownLatch start = new CountDownLatch(1);
try (Connection first = transactional(); Connection second = transactional()) {
Assert.assertEquals(0L, selectExpiredWithoutLock(first));
Assert.assertEquals(0L, selectExpiredWithoutLock(second));
Assert.assertEquals(firstExecutionId, selectPendingForUpdate(first));
Assert.assertEquals(secondExecutionId, selectPendingForUpdate(second));
Future<Integer> firstClaim = executor.submit(() -> {
ready.countDown();
start.await(5, TimeUnit.SECONDS);
int claimed = claim(first, firstExecutionId, "node-a", "probe-a");
first.commit();
return claimed;
});
Future<Integer> secondClaim = executor.submit(() -> {
ready.countDown();
start.await(5, TimeUnit.SECONDS);
int claimed = claim(second, secondExecutionId, "node-b", "probe-b");
second.commit();
return claimed;
});
Assert.assertTrue(ready.await(5, TimeUnit.SECONDS));
start.countDown();
Assert.assertEquals(Integer.valueOf(1), firstClaim.get(5, TimeUnit.SECONDS));
Assert.assertEquals(Integer.valueOf(1), secondClaim.get(5, TimeUnit.SECONDS));
} finally {
executor.shutdownNow();
}
}
private void verifyLockedExpiredCandidateDoesNotStarvePendingClaim() throws Exception {
try (Connection connection = open(); PreparedStatement statement = connection.prepareStatement(
"UPDATE tb_sys_job_log SET status=3,lease_owner='locked-owner',"
+ "execution_token='locked-token',"
+ "lease_until=TIMESTAMPADD(SECOND,-1,CURRENT_TIMESTAMP(3)) WHERE id=?")) {
statement.setLong(1, firstExecutionId);
Assert.assertEquals(1, statement.executeUpdate());
}
try (Connection lockOwner = transactional(); Connection claimant = transactional()) {
lockExecution(lockOwner, firstExecutionId);
Assert.assertEquals("无锁探测应仍能看见已提交的过期记录",
firstExecutionId, selectExpiredWithoutLock(claimant));
Assert.assertEquals("SKIP LOCKED 应跳过其他 Worker 持有的过期记录",
0L, selectExpiredForUpdate(claimant));
Assert.assertEquals("受阻的过期记录不得阻塞 PENDING 队列",
secondExecutionId, selectPendingForUpdate(claimant));
Assert.assertEquals(1,
claim(claimant, secondExecutionId, "pending-node", "pending-token"));
claimant.commit();
lockOwner.rollback();
}
}
private void verifyNonConcurrentJobRowLock() throws Exception {
ExecutorService executor = Executors.newSingleThreadExecutor();
try (Connection first = transactional()) {
lockJob(first);
Assert.assertEquals(1, claim(first, firstExecutionId, "node-a", "token-a"));
Future<Integer> secondNode = executor.submit(() -> {
try (Connection second = transactional()) {
lockJob(second);
int active = countOtherActive(second, secondExecutionId);
second.commit();
return active;
}
});
Thread.sleep(200L);
Assert.assertFalse("第二节点不应越过同任务行锁", secondNode.isDone());
first.commit();
Assert.assertEquals(Integer.valueOf(1), secondNode.get(5, TimeUnit.SECONDS));
} finally {
executor.shutdownNow();
}
}
private void verifyOwnerTokenFencing() throws Exception {
try (Connection connection = open()) {
try (PreparedStatement takeover = connection.prepareStatement(
"UPDATE tb_sys_job_log SET status=3, lease_owner='new-owner', "
+ "execution_token='new-token', lease_until=TIMESTAMPADD(SECOND,60,CURRENT_TIMESTAMP(3)) "
+ "WHERE id=?")) {
takeover.setLong(1, firstExecutionId);
Assert.assertEquals(1, takeover.executeUpdate());
}
Assert.assertEquals(0, finish(connection, "old-owner", "old-token"));
Assert.assertEquals(1, finish(connection, "new-owner", "new-token"));
}
}
private void verifyManagementCommandRowLock() throws Exception {
ExecutorService executor = Executors.newSingleThreadExecutor();
try (Connection first = transactional()) {
lockJob(first);
updateJobStatus(first, 0);
Future<Integer> secondCommand = executor.submit(() -> {
try (Connection second = transactional()) {
lockJob(second);
int observed = readJobStatus(second);
updateJobStatus(second, 1);
second.commit();
return observed;
}
});
Thread.sleep(200L);
Assert.assertFalse("并发管理命令不应在前一事务提交前执行", secondCommand.isDone());
first.commit();
Assert.assertEquals("后一命令必须读取前一命令已提交状态",
Integer.valueOf(0), secondCommand.get(5, TimeUnit.SECONDS));
} finally {
executor.shutdownNow();
}
}
private void verifyManagementCommitsBeforeLedgerCleanup() throws Exception {
resetExecutions();
ExecutorService executor = Executors.newSingleThreadExecutor();
try (Connection worker = transactional()) {
Assert.assertEquals(firstExecutionId, selectPendingForUpdate(worker));
Future<?> stopCommand = executor.submit(() -> {
try (Connection management = transactional()) {
lockJob(management);
updateJobStatus(management, 0);
management.commit();
return null;
}
});
// 管理事务不再反向触碰已被 Worker 锁定的日志行,因此应能先提交。
stopCommand.get(5, TimeUnit.SECONDS);
lockJob(worker);
Assert.assertEquals(0, readJobStatus(worker));
worker.rollback();
} finally {
executor.shutdownNow();
}
try (Connection connection = open()) {
updateJobStatus(connection, 1);
}
try (Connection cleanup = transactional()) {
Assert.assertEquals("RUNNING 任务不得清理待执行记录", 0,
cancelPendingIfStopped(cleanup));
cleanup.commit();
}
try (Connection connection = open()) {
updateJobStatus(connection, 0);
}
try (Connection cleanup = transactional()) {
Assert.assertEquals(2, cancelPendingIfStopped(cleanup));
cleanup.commit();
}
try (Connection connection = open()) {
updateJobStatus(connection, 1);
}
}
private void verifyRegistrationWaitsForStartCommit() throws Exception {
try (Connection connection = open()) {
updateJobStatus(connection, 0);
}
ExecutorService executor = Executors.newSingleThreadExecutor();
try (Connection start = transactional()) {
lockJob(start);
updateJobStatus(start, 1);
Future<Integer> registration = executor.submit(() -> {
try (Connection fire = transactional()) {
lockJob(fire);
int observed = readJobStatus(fire);
if (observed == 1) insertExecution(fire, registrationExecutionId);
fire.commit();
return observed;
}
});
Thread.sleep(200L);
Assert.assertFalse("Quartz 触发登记必须等待 START 事务提交", registration.isDone());
start.commit();
Assert.assertEquals("登记必须读取 START 提交后的 RUNNING 状态",
Integer.valueOf(1), registration.get(5, TimeUnit.SECONDS));
} finally {
executor.shutdownNow();
}
try (Connection connection = open(); PreparedStatement statement = connection.prepareStatement(
"SELECT COUNT(*) FROM tb_sys_job_log WHERE id=? AND status=2")) {
statement.setLong(1, registrationExecutionId);
try (ResultSet result = statement.executeQuery()) {
Assert.assertTrue(result.next());
Assert.assertEquals(1, result.getInt(1));
}
}
}
private void verifyCleanupUsesLedgerThenJobLock() throws Exception {
resetExecutions();
try (Connection connection = open()) {
updateJobStatus(connection, 0);
}
ExecutorService executor = Executors.newSingleThreadExecutor();
try (Connection worker = transactional()) {
Assert.assertEquals(firstExecutionId, selectPendingForUpdate(worker));
Future<Integer> cleanup = executor.submit(() -> {
try (Connection connection = transactional()) {
int cancelled = cancelPendingIfStopped(connection);
connection.commit();
return cancelled;
}
});
Thread.sleep(200L);
Assert.assertFalse("清理应先等待 Worker 持有的账本行", cleanup.isDone());
// 清理尚未持有任务行,因此 Worker 可继续按 log -> job 顺序完成。
lockJob(worker);
worker.rollback();
Assert.assertEquals(Integer.valueOf(2), cleanup.get(5, TimeUnit.SECONDS));
} finally {
executor.shutdownNow();
}
}
private void verifyStaleCleanupWaitsForStartCommit() throws Exception {
resetExecutions();
try (Connection connection = open()) {
updateJobStatus(connection, 0);
}
ExecutorService executor = Executors.newSingleThreadExecutor();
try (Connection start = transactional()) {
lockJob(start);
updateJobStatus(start, 1);
Future<Integer> staleCleanup = executor.submit(() -> {
try (Connection cleanup = transactional()) {
lockJob(cleanup);
int cancelled = cancelPendingIfStopped(cleanup);
cleanup.commit();
return cancelled;
}
});
Thread.sleep(200L);
Assert.assertFalse("旧清理命令必须等待并发 START 事务提交", staleCleanup.isDone());
start.commit();
Assert.assertEquals("旧清理命令不得取消新一代 RUNNING 记录",
Integer.valueOf(0), staleCleanup.get(5, TimeUnit.SECONDS));
} finally {
executor.shutdownNow();
}
}
private void verifyRestartClearsOldPendingGeneration() throws Exception {
resetExecutions();
try (Connection connection = open()) {
updateJobStatus(connection, 0);
}
try (Connection cleanup = transactional()) {
Assert.assertTrue("重新启动前应清理上一代全部 PENDING",
cancelPendingIfStopped(cleanup) >= 2);
cleanup.commit();
}
try (Connection start = transactional()) {
lockJob(start);
startNextGeneration(start);
start.commit();
}
try (Connection connection = open(); PreparedStatement statement = connection.prepareStatement(
"SELECT COUNT(*) FROM tb_sys_job_log WHERE job_id=? AND status=2")) {
statement.setLong(1, jobId);
try (ResultSet result = statement.executeQuery()) {
Assert.assertTrue(result.next());
Assert.assertEquals("旧 PENDING 不得被新 RUNNING 代际重新放行",
0, result.getInt(1));
}
}
}
private void verifyDefinitionEditCannotResurrectConcurrentStop() throws Exception {
try (Connection connection = open()) {
updateJobStatus(connection, 1);
}
ExecutorService executor = Executors.newSingleThreadExecutor();
try (Connection stop = transactional()) {
lockJob(stop);
updateJobStatus(stop, 0);
Future<Integer> edit = executor.submit(() -> {
try (Connection connection = transactional()) {
lockJob(connection);
int observedStatus = readJobStatus(connection);
advanceDefinitionGeneration(connection);
connection.commit();
return observedStatus;
}
});
Thread.sleep(200L);
Assert.assertFalse("定义编辑必须等待并发 STOP 提交", edit.isDone());
stop.commit();
Assert.assertEquals("编辑只能保留行锁下的最终 STOP 状态",
Integer.valueOf(0), edit.get(5, TimeUnit.SECONDS));
} finally {
executor.shutdownNow();
}
try (Connection connection = open()) {
Assert.assertEquals("普通编辑不得复活已停止任务", 0, readJobStatus(connection));
updateJobStatus(connection, 1);
}
}
private void verifyOldFireCannotCrossRestartGeneration() throws Exception {
long oldGeneration;
try (Connection connection = open()) {
oldGeneration = readJobGeneration(connection);
updateJobStatus(connection, 0);
}
try (Connection start = transactional()) {
lockJob(start);
startNextGeneration(start);
start.commit();
}
try (Connection fire = transactional()) {
Assert.assertEquals("STOP 前取得的 Quartz fire 不得登记到新启停代际", 0,
registerIfGenerationMatches(fire, generationExecutionId, oldGeneration));
fire.commit();
}
}
private void verifyOldFireCannotCrossDefinitionUpdate() throws Exception {
long oldGeneration;
try (Connection edit = transactional()) {
lockJob(edit);
oldGeneration = readJobGeneration(edit);
advanceDefinitionGeneration(edit);
edit.commit();
}
try (Connection oldFire = transactional()) {
Assert.assertEquals("编辑提交前取得的 Quartz fire 不得读取新定义并登记", 0,
registerIfGenerationMatches(oldFire, generationExecutionId + 1L, oldGeneration));
oldFire.commit();
}
try (Connection currentFire = transactional()) {
long currentGeneration = readJobGeneration(currentFire);
Assert.assertEquals(oldGeneration + 1L, currentGeneration);
Assert.assertEquals("当前定义代际的 fire 应可正常登记", 1,
registerIfGenerationMatches(
currentFire, generationExecutionId + 2L, currentGeneration));
currentFire.commit();
}
}
private void verifyRegisteredExecutionSurvivesDefinitionGeneration() throws Exception {
long oldGeneration;
try (Connection connection = open()) {
oldGeneration = readJobGeneration(connection);
insertExecution(connection, generationExecutionId + 3L, oldGeneration);
}
try (Connection edit = transactional()) {
lockJob(edit);
advanceDefinitionGeneration(edit);
edit.commit();
}
try (Connection connection = open()) {
Assert.assertTrue(readJobGeneration(connection) > oldGeneration);
Assert.assertEquals("已登记快照不得因后续编辑失去领取资格", 1,
claim(connection, generationExecutionId + 3L,
"generation-node", "generation-token"));
}
}
private void verifyExpiredRegisteredExecutionSurvivesDefinitionGeneration() throws Exception {
long executionId = generationExecutionId + 4L;
long oldGeneration;
try (Connection connection = open()) {
oldGeneration = readJobGeneration(connection);
insertExecution(connection, executionId, oldGeneration);
try (PreparedStatement statement = connection.prepareStatement(
"UPDATE tb_sys_job_log SET status=3,lease_owner='expired-owner',"
+ "execution_token='expired-token',"
+ "lease_until=TIMESTAMPADD(SECOND,-1,CURRENT_TIMESTAMP(3)) WHERE id=?")) {
statement.setLong(1, executionId);
Assert.assertEquals(1, statement.executeUpdate());
}
}
try (Connection edit = transactional()) {
lockJob(edit);
advanceDefinitionGeneration(edit);
edit.commit();
}
try (Connection takeover = transactional()) {
Assert.assertEquals("已登记的过期 RUNNING 不得因定义代际变化失去接管资格",
executionId, selectExpiredForUpdate(takeover));
Assert.assertEquals(1, claimExpired(
takeover, executionId, "takeover-owner", "takeover-token"));
takeover.commit();
}
try (Connection connection = open()) {
Assert.assertEquals("旧租约持有者不得覆盖接管后的终态", 0,
finish(connection, executionId, "expired-owner", "expired-token"));
Assert.assertEquals(1,
finish(connection, executionId, "takeover-owner", "takeover-token"));
}
}
private void verifyDuplicateExecutionKeyCanBeReadInSameTransaction() throws Exception {
long originalId = generationExecutionId + 10L;
long conflictingId = generationExecutionId + 11L;
String executionKey = String.format("%064x", originalId);
try (Connection connection = open()) {
insertExecution(connection, originalId, readJobGeneration(connection), executionKey);
}
try (Connection connection = transactional()) {
SQLException duplicate = Assert.assertThrows(
SQLException.class,
() -> insertExecution(
connection, conflictingId, readJobGeneration(connection), executionKey));
Assert.assertEquals("23000", duplicate.getSQLState());
try (PreparedStatement statement = connection.prepareStatement(
"SELECT id FROM tb_sys_job_log WHERE execution_key=?")) {
statement.setString(1, executionKey);
try (ResultSet result = statement.executeQuery()) {
Assert.assertTrue("唯一键冲突后同一事务仍应能复查 execution_key",
result.next());
Assert.assertEquals(originalId, result.getLong(1));
}
}
connection.rollback();
}
}
private void insertFixtures(Connection connection) throws SQLException {
try (PreparedStatement job = connection.prepareStatement(
"INSERT INTO tb_sys_job(id,dept_id,tenant_id,job_name,job_type,job_params,"
+ "cron_expression,allow_concurrent,misfire_policy,options,status,created,"
+ "schedule_generation,created_by,modified,modified_by,remark) "
+ "VALUES(?,1,1,'L25 MySQL IT',3,'{}','0 0 0 1 1 ? 2099',0,3,'{}',1,NOW(),0,1,NOW(),1,'')")) {
job.setLong(1, jobId);
job.executeUpdate();
}
insertExecution(connection, firstExecutionId, 0L);
insertExecution(connection, secondExecutionId, 0L);
}
private void insertExecution(Connection connection, long id) throws SQLException {
insertExecution(connection, id, readJobGeneration(connection));
}
private void insertExecution(Connection connection, long id, long generation) throws SQLException {
insertExecution(connection, id, generation, String.format("%064x", id));
}
private void insertExecution(Connection connection, long id, long generation,
String executionKey) throws SQLException {
try (PreparedStatement execution = connection.prepareStatement(
"INSERT INTO tb_sys_job_log(id,execution_key,job_id,job_generation,tenant_id,dept_id,job_name,"
+ "job_type,job_params,job_options,allow_concurrent,trigger_source,"
+ "scheduled_fire_time,actual_fire_time,fire_instance_id,recovering,"
+ "attempt_count,next_retry_time,version,status,created,remark) "
+ "VALUES(?,?,?,?,1,1,'L25 MySQL IT',3,'{}','{}',0,'MANUAL',"
+ "CURRENT_TIMESTAMP(3),CURRENT_TIMESTAMP(3),?,0,0,CURRENT_TIMESTAMP(3),0,2,NOW(),'')")) {
execution.setLong(1, id);
execution.setString(2, executionKey);
execution.setLong(3, jobId);
execution.setLong(4, generation);
execution.setString(5, "fire-" + id);
execution.executeUpdate();
}
}
private int registerIfGenerationMatches(Connection connection, long executionId,
long fireGeneration) throws SQLException {
lockJob(connection);
if (readJobStatus(connection) != 1
|| readJobGeneration(connection) != fireGeneration) {
return 0;
}
insertExecution(connection, executionId, fireGeneration);
return 1;
}
private void startNextGeneration(Connection connection) throws SQLException {
try (PreparedStatement statement = connection.prepareStatement(
"UPDATE tb_sys_job SET status=1,schedule_generation=schedule_generation+1 "
+ "WHERE id=? AND status<>1")) {
statement.setLong(1, jobId);
Assert.assertEquals(1, statement.executeUpdate());
}
}
private void advanceDefinitionGeneration(Connection connection) throws SQLException {
try (PreparedStatement statement = connection.prepareStatement(
"UPDATE tb_sys_job SET schedule_generation=schedule_generation+1 WHERE id=?")) {
statement.setLong(1, jobId);
Assert.assertEquals(1, statement.executeUpdate());
}
}
private void resetExecutions() throws SQLException {
try (Connection connection = open(); PreparedStatement statement = connection.prepareStatement(
"UPDATE tb_sys_job_log SET status=2,lease_owner=NULL,execution_token=NULL,"
+ "lease_until=NULL,next_retry_time=CURRENT_TIMESTAMP(3) WHERE job_id=?")) {
statement.setLong(1, jobId);
statement.executeUpdate();
}
}
private long selectPendingForUpdate(Connection connection) throws SQLException {
try (PreparedStatement statement = connection.prepareStatement(
"SELECT id FROM tb_sys_job_log FORCE INDEX(PRIMARY) WHERE id IN (?,?) "
+ "AND job_id=? AND status=2 "
+ "ORDER BY id LIMIT 1 FOR UPDATE SKIP LOCKED")) {
statement.setLong(1, firstExecutionId);
statement.setLong(2, secondExecutionId);
statement.setLong(3, jobId);
try (ResultSet result = statement.executeQuery()) {
Assert.assertTrue(result.next());
return result.getLong(1);
}
}
}
private long selectExpiredForUpdate(Connection connection) throws SQLException {
try (PreparedStatement statement = connection.prepareStatement(
"SELECT id FROM tb_sys_job_log WHERE status=3 "
+ "AND lease_until<=CURRENT_TIMESTAMP(3) "
+ "ORDER BY lease_until,id LIMIT 1 FOR UPDATE SKIP LOCKED")) {
try (ResultSet result = statement.executeQuery()) {
return result.next() ? result.getLong(1) : 0L;
}
}
}
private void lockExecution(Connection connection, long executionId) throws SQLException {
try (PreparedStatement statement = connection.prepareStatement(
"SELECT id FROM tb_sys_job_log WHERE id=? FOR UPDATE")) {
statement.setLong(1, executionId);
try (ResultSet result = statement.executeQuery()) {
Assert.assertTrue(result.next());
}
}
}
private long selectExpiredWithoutLock(Connection connection) throws SQLException {
try (PreparedStatement statement = connection.prepareStatement(
"SELECT id FROM tb_sys_job_log WHERE status=3 "
+ "AND lease_until<=CURRENT_TIMESTAMP(3) "
+ "ORDER BY lease_until,id LIMIT 1")) {
try (ResultSet result = statement.executeQuery()) {
return result.next() ? result.getLong(1) : 0L;
}
}
}
private int claim(Connection connection, long id, String owner, String token) throws SQLException {
try (PreparedStatement statement = connection.prepareStatement(
"UPDATE tb_sys_job_log SET status=3,lease_owner=?,execution_token=?,"
+ "lease_until=TIMESTAMPADD(SECOND,60,CURRENT_TIMESTAMP(3)) "
+ "WHERE id=? AND status=2")) {
statement.setString(1, owner);
statement.setString(2, token);
statement.setLong(3, id);
return statement.executeUpdate();
}
}
private int claimExpired(Connection connection, long id, String owner,
String token) throws SQLException {
try (PreparedStatement statement = connection.prepareStatement(
"UPDATE tb_sys_job_log SET lease_owner=?,execution_token=?,"
+ "lease_until=TIMESTAMPADD(SECOND,60,CURRENT_TIMESTAMP(3)),"
+ "attempt_count=attempt_count+1 WHERE id=? AND status=3 "
+ "AND lease_until<=CURRENT_TIMESTAMP(3)")) {
statement.setString(1, owner);
statement.setString(2, token);
statement.setLong(3, id);
return statement.executeUpdate();
}
}
private void lockJob(Connection connection) throws SQLException {
try (PreparedStatement statement = connection.prepareStatement(
"SELECT id FROM tb_sys_job WHERE id=? FOR UPDATE")) {
statement.setLong(1, jobId);
try (ResultSet result = statement.executeQuery()) {
Assert.assertTrue(result.next());
}
}
}
private int countOtherActive(Connection connection, long excludedId) throws SQLException {
try (PreparedStatement statement = connection.prepareStatement(
"SELECT COUNT(*) FROM tb_sys_job_log WHERE job_id=? AND id<>? AND status=3 "
+ "AND lease_until>CURRENT_TIMESTAMP(3) FOR UPDATE")) {
statement.setLong(1, jobId);
statement.setLong(2, excludedId);
try (ResultSet result = statement.executeQuery()) {
Assert.assertTrue(result.next());
return result.getInt(1);
}
}
}
private int finish(Connection connection, String owner, String token) throws SQLException {
return finish(connection, firstExecutionId, owner, token);
}
private int finish(Connection connection, long executionId,
String owner, String token) throws SQLException {
try (PreparedStatement statement = connection.prepareStatement(
"UPDATE tb_sys_job_log SET status=1,end_time=CURRENT_TIMESTAMP(3) WHERE id=? "
+ "AND status=3 AND lease_owner=? AND execution_token=?")) {
statement.setLong(1, executionId);
statement.setString(2, owner);
statement.setString(3, token);
return statement.executeUpdate();
}
}
private int cancelPendingIfStopped(Connection connection) throws SQLException {
try (PreparedStatement statement = connection.prepareStatement(
"SELECT id FROM tb_sys_job_log WHERE job_id=? AND status=2 "
+ "ORDER BY id FOR UPDATE")) {
statement.setLong(1, jobId);
try (ResultSet ignored = statement.executeQuery()) {
while (ignored.next()) {
// 读取完整结果集并保持行锁到事务结束。
}
}
}
lockJob(connection);
if (readJobStatus(connection) != 0) return 0;
try (PreparedStatement statement = connection.prepareStatement(
"UPDATE tb_sys_job_log SET status=5 WHERE job_id=? AND status=2")) {
statement.setLong(1, jobId);
return statement.executeUpdate();
}
}
private void updateJobStatus(Connection connection, int status) throws SQLException {
try (PreparedStatement statement = connection.prepareStatement(
"UPDATE tb_sys_job SET status=? WHERE id=?")) {
statement.setInt(1, status);
statement.setLong(2, jobId);
Assert.assertEquals(1, statement.executeUpdate());
}
}
private int readJobStatus(Connection connection) throws SQLException {
try (PreparedStatement statement = connection.prepareStatement(
"SELECT status FROM tb_sys_job WHERE id=?")) {
statement.setLong(1, jobId);
try (ResultSet result = statement.executeQuery()) {
Assert.assertTrue(result.next());
return result.getInt(1);
}
}
}
private long readJobGeneration(Connection connection) throws SQLException {
try (PreparedStatement statement = connection.prepareStatement(
"SELECT schedule_generation FROM tb_sys_job WHERE id=?")) {
statement.setLong(1, jobId);
try (ResultSet result = statement.executeQuery()) {
Assert.assertTrue(result.next());
return result.getLong(1);
}
}
}
private Connection transactional() throws SQLException {
Connection connection = open();
connection.setAutoCommit(false);
connection.setTransactionIsolation(Connection.TRANSACTION_READ_COMMITTED);
return connection;
}
private Connection open() throws SQLException {
return DriverManager.getConnection(jdbcUrl, username, password);
}
}

View File

@@ -0,0 +1,75 @@
package tech.easyflow.job.execution;
import org.junit.Assert;
import org.junit.Test;
import tech.easyflow.common.constant.enums.EnumDataStatus;
import tech.easyflow.job.entity.SysJob;
import tech.easyflow.system.entity.SysAccount;
import tech.easyflow.system.service.SysAccountService;
import java.math.BigInteger;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
public class SysJobOwnerValidatorTest {
@Test
public void missingOwnerMustBeRejected() {
SysAccountService accountService = mock(SysAccountService.class);
SysJob job = job();
when(accountService.getById(job.getCreatedBy())).thenReturn(null);
IllegalStateException exception = Assert.assertThrows(
IllegalStateException.class,
() -> new SysJobOwnerValidator(accountService).requireAvailableOwner(job));
Assert.assertTrue(exception.getMessage().contains("不存在"));
}
@Test
public void disabledOwnerMustBeRejected() {
SysAccountService accountService = mock(SysAccountService.class);
SysJob job = job();
SysAccount account = account(job.getCreatedBy(), job.getTenantId(), -1);
when(accountService.getById(job.getCreatedBy())).thenReturn(account);
IllegalStateException exception = Assert.assertThrows(
IllegalStateException.class,
() -> new SysJobOwnerValidator(accountService).requireAvailableOwner(job));
Assert.assertTrue(exception.getMessage().contains("未启用"));
}
@Test
public void crossTenantOwnerMustBeRejected() {
SysAccountService accountService = mock(SysAccountService.class);
SysJob job = job();
SysAccount account = account(
job.getCreatedBy(), BigInteger.valueOf(999L),
EnumDataStatus.AVAILABLE.getCode());
when(accountService.getById(job.getCreatedBy())).thenReturn(account);
IllegalStateException exception = Assert.assertThrows(
IllegalStateException.class,
() -> new SysJobOwnerValidator(accountService).requireAvailableOwner(job));
Assert.assertTrue(exception.getMessage().contains("租户"));
}
private static SysJob job() {
SysJob job = new SysJob();
job.setId(BigInteger.valueOf(101L));
job.setTenantId(BigInteger.valueOf(201L));
job.setCreatedBy(BigInteger.valueOf(301L));
return job;
}
private static SysAccount account(BigInteger id, BigInteger tenantId, int status) {
SysAccount account = new SysAccount();
account.setId(id);
account.setTenantId(tenantId);
account.setStatus(status);
return account;
}
}

View File

@@ -0,0 +1,54 @@
package tech.easyflow.job.mapper;
import org.junit.Assert;
import org.junit.Test;
import tech.easyflow.job.entity.SysJobLog;
import java.math.BigInteger;
import static org.mockito.Mockito.CALLS_REAL_METHODS;
import static org.mockito.Mockito.doReturn;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.verify;
public class SysJobLogMapperTest {
@Test
public void pendingClaimShouldReadLockedIdThroughEntityResultMap() {
SysJobLogMapper mapper = mock(SysJobLogMapper.class, CALLS_REAL_METHODS);
BigInteger id = BigInteger.valueOf(31L);
SysJobLog expected = execution(id);
doReturn(id).when(mapper).selectPendingClaimCandidateId();
doReturn(expected).when(mapper).selectOneById(id);
Assert.assertSame(expected, mapper.selectPendingClaimCandidate());
verify(mapper).selectOneById(id);
}
@Test
public void expiredClaimShouldReadLockedIdThroughEntityResultMap() {
SysJobLogMapper mapper = mock(SysJobLogMapper.class, CALLS_REAL_METHODS);
BigInteger id = BigInteger.valueOf(32L);
SysJobLog expected = execution(id);
doReturn(id).when(mapper).selectExpiredClaimCandidateId();
doReturn(expected).when(mapper).selectOneById(id);
Assert.assertSame(expected, mapper.selectExpiredClaimCandidate());
verify(mapper).selectOneById(id);
}
@Test
public void emptyClaimShouldNotIssueEntityRead() {
SysJobLogMapper mapper = mock(SysJobLogMapper.class, CALLS_REAL_METHODS);
doReturn(null).when(mapper).selectPendingClaimCandidateId();
Assert.assertNull(mapper.selectPendingClaimCandidate());
}
private static SysJobLog execution(BigInteger id) {
SysJobLog execution = new SysJobLog();
execution.setId(id);
execution.setJobId(BigInteger.TEN);
return execution;
}
}

View File

@@ -0,0 +1,65 @@
package tech.easyflow.job.mapper;
import com.mybatisflex.core.query.QueryWrapper;
import org.junit.Assert;
import org.junit.Test;
import org.mockito.ArgumentCaptor;
import tech.easyflow.job.entity.SysJob;
import java.math.BigInteger;
import java.util.List;
import java.util.Locale;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.CALLS_REAL_METHODS;
import static org.mockito.Mockito.doReturn;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.verify;
public class SysJobMapperTest {
@Test
public void lockQueryShouldDelegateToEntityMappedBaseMapper() {
SysJobMapper mapper = mock(SysJobMapper.class, CALLS_REAL_METHODS);
SysJob expected = job(17L);
doReturn(expected).when(mapper).selectOneByQuery(any(QueryWrapper.class));
SysJob actual = mapper.selectByIdForUpdate(BigInteger.valueOf(17L));
Assert.assertSame(expected, actual);
ArgumentCaptor<QueryWrapper> query = ArgumentCaptor.forClass(QueryWrapper.class);
verify(mapper).selectOneByQuery(query.capture());
String sql = normalizedSql(query.getValue());
Assert.assertTrue(sql.contains("ID = 17"));
Assert.assertTrue(sql.contains("FOR UPDATE"));
}
@Test
public void reconciliationShouldUseEntityMappingAndKeysetPage() {
SysJobMapper mapper = mock(SysJobMapper.class, CALLS_REAL_METHODS);
List<SysJob> expected = List.of(job(18L));
doReturn(expected).when(mapper).selectListByQuery(any(QueryWrapper.class));
List<SysJob> actual = mapper.selectReconciliationPage(
BigInteger.valueOf(17L), 200);
Assert.assertSame(expected, actual);
ArgumentCaptor<QueryWrapper> query = ArgumentCaptor.forClass(QueryWrapper.class);
verify(mapper).selectListByQuery(query.capture());
String sql = normalizedSql(query.getValue());
Assert.assertTrue(sql.contains("ID > 17"));
Assert.assertTrue(sql.contains("ORDER BY ID"));
Assert.assertTrue(sql.contains("LIMIT 200"));
}
private static SysJob job(long id) {
SysJob job = new SysJob();
job.setId(BigInteger.valueOf(id));
job.setScheduleGeneration(1L);
return job;
}
private static String normalizedSql(QueryWrapper query) {
return query.toSQL().replace("`", "").toUpperCase(Locale.ROOT);
}
}

View File

@@ -0,0 +1,84 @@
package tech.easyflow.job.schedule;
import com.easyagents.scheduler.ConcurrencyPolicy;
import com.easyagents.scheduler.MisfirePolicy;
import com.easyagents.scheduler.ScheduleDefinition;
import com.easyagents.scheduler.ScheduleService;
import org.junit.Assert;
import org.junit.Test;
import tech.easyflow.common.constant.enums.EnumMisfirePolicy;
import tech.easyflow.job.entity.SysJob;
import tech.easyflow.job.job.JobConstant;
import java.math.BigInteger;
import static org.mockito.Mockito.mock;
public class SysJobScheduleAdapterTest {
@Test
public void shouldMapBusinessDefinitionToStableSchedule() {
SysJob job = new SysJob();
job.setId(BigInteger.valueOf(42));
job.setJobName("daily-report");
job.setCronExpression("0 0 2 * * ?");
job.setMisfirePolicy(EnumMisfirePolicy.FIRE_ONCE_NOW.getCode());
job.setAllowConcurrent(0);
job.setScheduleGeneration(7L);
ScheduleDefinition definition = new SysJobScheduleAdapter(
mock(ScheduleService.class), "Asia/Shanghai").toDefinition(job);
Assert.assertEquals("easyflow.job", definition.id().namespace());
Assert.assertEquals("42", definition.id().name());
Assert.assertEquals("easyflow.job.execution", definition.handlerCode());
Assert.assertEquals(MisfirePolicy.FIRE_ONCE_NOW, definition.misfirePolicy());
Assert.assertEquals(ConcurrencyPolicy.DISALLOW, definition.concurrencyPolicy());
Assert.assertTrue(definition.recoverOnNodeFailure());
Assert.assertEquals("42", definition.parameters().get("jobId"));
Assert.assertEquals("7",
definition.parameters().get(JobConstant.SCHEDULE_GENERATION));
}
@Test
public void shouldMapSkipAndConcurrentPolicies() {
SysJob job = new SysJob();
job.setId(BigInteger.ONE);
job.setJobName("parallel");
job.setCronExpression("0/5 * * * * ?");
job.setMisfirePolicy(EnumMisfirePolicy.SKIP.getCode());
job.setAllowConcurrent(1);
job.setScheduleGeneration(0L);
ScheduleDefinition definition = new SysJobScheduleAdapter(
mock(ScheduleService.class), "UTC").toDefinition(job);
Assert.assertEquals(MisfirePolicy.SKIP, definition.misfirePolicy());
Assert.assertEquals(ConcurrencyPolicy.ALLOW, definition.concurrencyPolicy());
}
@Test(expected = IllegalArgumentException.class)
public void shouldRejectUnsupportedMisfirePolicy() {
SysJob job = new SysJob();
job.setId(BigInteger.ONE);
job.setJobName("invalid-policy");
job.setCronExpression("0/5 * * * * ?");
job.setMisfirePolicy(99);
job.setAllowConcurrent(0);
job.setScheduleGeneration(0L);
new SysJobScheduleAdapter(mock(ScheduleService.class), "UTC").toDefinition(job);
}
@Test(expected = IllegalArgumentException.class)
public void shouldRejectMissingScheduleGeneration() {
SysJob job = new SysJob();
job.setId(BigInteger.ONE);
job.setJobName("missing-generation");
job.setCronExpression("0/5 * * * * ?");
job.setMisfirePolicy(EnumMisfirePolicy.SKIP.getCode());
job.setAllowConcurrent(0);
new SysJobScheduleAdapter(mock(ScheduleService.class), "UTC").toDefinition(job);
}
}

View File

@@ -0,0 +1,46 @@
package tech.easyflow.job.schedule;
import org.junit.Test;
import tech.easyflow.job.config.SysJobExecutionProperties;
import tech.easyflow.job.entity.SysJob;
import tech.easyflow.job.mapper.SysJobMapper;
import tech.easyflow.job.service.SysJobService;
import java.math.BigInteger;
import java.util.ArrayList;
import java.util.List;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
public class SysJobScheduleReconcilerTest {
@Test
public void reconciliationMustUseLastSeenIdInsteadOfMutableOffset() {
SysJobMapper mapper = mock(SysJobMapper.class);
SysJobService service = mock(SysJobService.class);
List<SysJob> firstPage = jobs(1, 200);
List<SysJob> secondPage = jobs(201, 1);
when(mapper.selectReconciliationPage(BigInteger.ZERO, 200)).thenReturn(firstPage);
when(mapper.selectReconciliationPage(BigInteger.valueOf(200), 200))
.thenReturn(secondPage);
new SysJobScheduleReconciler(
mapper, service, new SysJobExecutionProperties()).reconcile();
verify(mapper).selectReconciliationPage(BigInteger.ZERO, 200);
verify(mapper).selectReconciliationPage(BigInteger.valueOf(200), 200);
verify(service).syncJob(BigInteger.valueOf(201));
}
private static List<SysJob> jobs(long firstId, int count) {
List<SysJob> jobs = new ArrayList<>(count);
for (long id = firstId; id < firstId + count; id++) {
SysJob job = new SysJob();
job.setId(BigInteger.valueOf(id));
jobs.add(job);
}
return jobs;
}
}

View File

@@ -4,6 +4,7 @@ import com.easyagents.flow.core.chain.runtime.ChainExecutor;
import org.junit.Assert;
import org.junit.Test;
import org.mockito.ArgumentCaptor;
import tech.easyflow.ai.easyagentsflow.support.PublishedWorkflowDefinitionIds;
import tech.easyflow.ai.service.WorkflowUsageAuthorizationService;
import tech.easyflow.common.constant.Constants;
import tech.easyflow.common.constant.enums.EnumDataStatus;
@@ -14,7 +15,6 @@ import tech.easyflow.common.web.exceptions.BusinessException;
import tech.easyflow.job.entity.SysJob;
import tech.easyflow.job.job.JobConstant;
import tech.easyflow.system.entity.SysAccount;
import tech.easyflow.system.service.SysAccountService;
import java.math.BigInteger;
import java.util.Map;
@@ -42,25 +42,23 @@ public class WorkflowJobExecutionServiceTest {
BigInteger tenantId = BigInteger.valueOf(201);
BigInteger accountId = BigInteger.valueOf(301);
BigInteger workflowId = BigInteger.valueOf(401);
SysJobService jobService = mock(SysJobService.class);
SysAccountService accountService = mock(SysAccountService.class);
WorkflowUsageAuthorizationService authorizationService =
mock(WorkflowUsageAuthorizationService.class);
ChainExecutor chainExecutor = mock(ChainExecutor.class);
SysJob currentJob = workflowJob(jobId, tenantId, accountId, workflowId);
SysAccount account = account(accountId, tenantId, EnumDataStatus.AVAILABLE.getCode());
when(jobService.getById(jobId)).thenReturn(currentJob);
when(accountService.getById(accountId)).thenReturn(account);
Map<String, Object> executionResult = Map.of("status", "done");
when(chainExecutor.execute(eq(workflowId.toString()), anyMap()))
String publishedWorkflowId = PublishedWorkflowDefinitionIds.published(workflowId.toString());
when(chainExecutor.execute(eq(publishedWorkflowId), anyMap()))
.thenReturn(executionResult);
WorkflowJobExecutionService service = new WorkflowJobExecutionService(
jobService,
accountService,
authorizationService,
chainExecutor);
Object result = service.execute(scheduledJob(jobId, tenantId));
Object result = service.execute(
scheduledJob(jobId, tenantId, workflowId, Map.of("question", "hello")),
currentJob,
account);
Assert.assertSame(executionResult, result);
ArgumentCaptor<LoginAccount> accountCaptor =
@@ -75,7 +73,7 @@ public class WorkflowJobExecutionServiceTest {
@SuppressWarnings("unchecked")
ArgumentCaptor<Map<String, Object>> paramsCaptor =
ArgumentCaptor.forClass(Map.class);
verify(chainExecutor).execute(eq(workflowId.toString()), paramsCaptor.capture());
verify(chainExecutor).execute(eq(publishedWorkflowId), paramsCaptor.capture());
Object loginUser = paramsCaptor.getValue().get(Constants.LOGIN_USER_KEY);
Assert.assertTrue(loginUser instanceof LoginAccount);
Assert.assertEquals(((LoginAccount) loginUser).getId(), accountId);
@@ -90,29 +88,28 @@ public class WorkflowJobExecutionServiceTest {
BigInteger tenantId = BigInteger.valueOf(202);
BigInteger accountId = BigInteger.valueOf(302);
BigInteger workflowId = BigInteger.valueOf(402);
SysJobService jobService = mock(SysJobService.class);
SysAccountService accountService = mock(SysAccountService.class);
WorkflowUsageAuthorizationService authorizationService =
mock(WorkflowUsageAuthorizationService.class);
ChainExecutor chainExecutor = mock(ChainExecutor.class);
when(jobService.getById(jobId))
.thenReturn(workflowJob(jobId, tenantId, accountId, workflowId));
when(accountService.getById(accountId))
.thenReturn(account(accountId, tenantId, EnumDataStatus.AVAILABLE.getCode()));
SysJob currentJob = workflowJob(jobId, tenantId, accountId, workflowId);
SysAccount owner = account(
accountId, tenantId, EnumDataStatus.AVAILABLE.getCode());
when(authorizationService.requireUsableWorkflow(
eq(workflowId),
any(LoginAccount.class),
anyString()))
.thenThrow(new BusinessException("工作流权限已撤销"));
WorkflowJobExecutionService service = new WorkflowJobExecutionService(
jobService,
accountService,
authorizationService,
chainExecutor);
BusinessException exception = Assert.assertThrows(
BusinessException.class,
() -> service.execute(scheduledJob(jobId, tenantId))
() -> service.execute(
scheduledJob(jobId, tenantId, workflowId,
Map.of("question", "hello")),
currentJob,
owner)
);
Assert.assertTrue(exception.getMessage().contains("权限已撤销"));
@@ -126,25 +123,23 @@ public class WorkflowJobExecutionServiceTest {
public void shouldRejectCrossTenantJobSnapshot() {
BigInteger jobId = BigInteger.valueOf(103);
BigInteger tenantId = BigInteger.valueOf(203);
SysJobService jobService = mock(SysJobService.class);
SysAccountService accountService = mock(SysAccountService.class);
WorkflowUsageAuthorizationService authorizationService =
mock(WorkflowUsageAuthorizationService.class);
ChainExecutor chainExecutor = mock(ChainExecutor.class);
when(jobService.getById(jobId)).thenReturn(workflowJob(
jobId,
tenantId,
BigInteger.valueOf(303),
BigInteger.valueOf(403)));
BigInteger accountId = BigInteger.valueOf(303);
SysJob currentJob = workflowJob(
jobId, tenantId, accountId, BigInteger.valueOf(403));
WorkflowJobExecutionService service = new WorkflowJobExecutionService(
jobService,
accountService,
authorizationService,
chainExecutor);
IllegalStateException exception = Assert.assertThrows(
IllegalStateException.class,
() -> service.execute(scheduledJob(jobId, BigInteger.valueOf(999)))
() -> service.execute(
scheduledJob(jobId, BigInteger.valueOf(999),
BigInteger.valueOf(403), Map.of()),
currentJob,
account(accountId, tenantId, EnumDataStatus.AVAILABLE.getCode()))
);
Assert.assertTrue(exception.getMessage().contains("租户"));
@@ -154,6 +149,39 @@ public class WorkflowJobExecutionServiceTest {
anyString());
}
@Test
public void shouldExecuteRegisteredWorkflowSnapshotAfterDefinitionEdit() {
BigInteger jobId = BigInteger.valueOf(104);
BigInteger tenantId = BigInteger.valueOf(204);
BigInteger accountId = BigInteger.valueOf(304);
BigInteger oldWorkflowId = BigInteger.valueOf(404);
BigInteger newWorkflowId = BigInteger.valueOf(405);
WorkflowUsageAuthorizationService authorizationService =
mock(WorkflowUsageAuthorizationService.class);
ChainExecutor chainExecutor = mock(ChainExecutor.class);
SysJob currentJob = workflowJob(
jobId, tenantId, accountId, newWorkflowId);
SysAccount owner = account(
accountId, tenantId, EnumDataStatus.AVAILABLE.getCode());
WorkflowJobExecutionService service = new WorkflowJobExecutionService(
authorizationService, chainExecutor);
service.execute(
scheduledJob(jobId, tenantId, oldWorkflowId,
Map.of("question", "old-ledger-value")),
currentJob,
owner);
verify(authorizationService).requireUsableWorkflow(
eq(oldWorkflowId), any(LoginAccount.class), anyString());
@SuppressWarnings("unchecked")
ArgumentCaptor<Map<String, Object>> parameters = ArgumentCaptor.forClass(Map.class);
verify(chainExecutor).execute(
eq(PublishedWorkflowDefinitionIds.published(oldWorkflowId.toString())),
parameters.capture());
Assert.assertEquals(parameters.getValue().get("question"), "old-ledger-value");
}
/**
* 创建 Quartz 任务快照。
*
@@ -161,11 +189,16 @@ public class WorkflowJobExecutionServiceTest {
* @param tenantId 租户 ID
* @return 任务快照
*/
private SysJob scheduledJob(BigInteger jobId, BigInteger tenantId) {
private SysJob scheduledJob(BigInteger jobId, BigInteger tenantId,
BigInteger workflowId,
Map<String, Object> workflowParams) {
SysJob job = new SysJob();
job.setId(jobId);
job.setTenantId(tenantId);
job.setJobType(EnumJobType.TINY_FLOW.getCode());
job.setJobParams(Map.of(
JobConstant.WORKFLOW_KEY, workflowId.toString(),
JobConstant.WORKFLOW_PARAMS_KEY, workflowParams));
return job;
}

View File

@@ -0,0 +1,204 @@
package tech.easyflow.job.service.impl;
import com.easyagents.scheduler.ScheduleService;
import org.junit.Assert;
import org.junit.Test;
import org.mockito.ArgumentCaptor;
import org.springframework.dao.DataAccessResourceFailureException;
import org.springframework.transaction.CannotCreateTransactionException;
import org.springframework.transaction.PlatformTransactionManager;
import org.springframework.transaction.TransactionStatus;
import org.springframework.transaction.TransactionSystemException;
import tech.easyflow.common.cache.RedisLockExecutor;
import tech.easyflow.common.constant.enums.EnumJobStatus;
import tech.easyflow.common.web.exceptions.BusinessException;
import tech.easyflow.job.config.SysJobExecutionProperties;
import tech.easyflow.job.entity.SysJob;
import tech.easyflow.job.execution.SysJobExecutionStore;
import tech.easyflow.job.mapper.SysJobMapper;
import tech.easyflow.job.schedule.SysJobScheduleAdapter;
import java.math.BigInteger;
import java.time.Duration;
import java.util.List;
import java.util.function.Supplier;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyInt;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.atLeastOnce;
import static org.mockito.Mockito.doAnswer;
import static org.mockito.Mockito.doReturn;
import static org.mockito.Mockito.doThrow;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.spy;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
public class SysJobServiceImplTest {
@Test
public void providerFailureMustStopJobAndRetryProjectionCleanup() {
Fixture fixture = fixture();
SysJob running = job(EnumJobStatus.RUNNING);
SysJob stopped = job(EnumJobStatus.STOP);
when(fixture.mapper().selectByIdForUpdate(running.getId()))
.thenReturn(running, stopped);
doReturn(true).when(fixture.service()).updateById(any(SysJob.class));
DataAccessResourceFailureException providerFailure =
new DataAccessResourceFailureException("quartz unavailable");
doThrow(providerFailure).when(fixture.scheduleAdapter()).replace(running);
RuntimeException thrown = Assert.assertThrows(
RuntimeException.class,
() -> fixture.service().syncJob(running.getId()));
Assert.assertSame(providerFailure, thrown);
ArgumentCaptor<SysJob> update = ArgumentCaptor.forClass(SysJob.class);
verify(fixture.service()).updateById(update.capture());
Assert.assertEquals(
Integer.valueOf(EnumJobStatus.STOP.getCode()), update.getValue().getStatus());
verify(fixture.scheduleAdapter()).delete(running.getId());
verify(fixture.executionStore()).cancelPending(
running.getId(), "调度投影同步失败,任务已停止");
}
@Test
public void transactionBeginFailureMustNotDeleteExistingProjection() {
Fixture fixture = fixture(new CannotCreateTransactionException("pool exhausted"));
Assert.assertThrows(
CannotCreateTransactionException.class,
() -> fixture.service().syncJob(BigInteger.ONE));
verify(fixture.scheduleAdapter(), never()).replace(any(SysJob.class));
verify(fixture.scheduleAdapter(), never()).delete(any(BigInteger.class));
}
@Test
public void transactionCommitFailureMustNotDeleteExistingProjection() {
Fixture fixture = fixture();
SysJob running = job(EnumJobStatus.RUNNING);
when(fixture.mapper().selectByIdForUpdate(running.getId())).thenReturn(running);
doThrow(new TransactionSystemException("commit failed"))
.when(fixture.transactionManager()).commit(any(TransactionStatus.class));
Assert.assertThrows(
TransactionSystemException.class,
() -> fixture.service().syncJob(running.getId()));
verify(fixture.scheduleAdapter()).replace(running);
verify(fixture.scheduleAdapter(), never()).delete(any(BigInteger.class));
verify(fixture.executionStore(), never()).cancelPending(any(BigInteger.class), anyString());
}
@Test
public void startMustAbortBeforeStateChangeWhenOldPendingCleanupFails() {
Fixture fixture = fixture();
SysJob stopped = job(EnumJobStatus.STOP);
doReturn(stopped).when(fixture.service()).getById(stopped.getId());
doThrow(new DataAccessResourceFailureException("ledger unavailable"))
.when(fixture.executionStore()).cancelPending(any(BigInteger.class), anyString());
Assert.assertThrows(
DataAccessResourceFailureException.class,
() -> fixture.service().startJob(stopped.getId()));
verify(fixture.mapper(), never()).startNextGeneration(any(BigInteger.class), anyInt());
verify(fixture.scheduleAdapter(), never()).replace(any(SysJob.class));
}
@Test
public void deleteProjectionFailureMustLeaveStoppedBusinessRow() {
Fixture fixture = fixture();
SysJob stopped = job(EnumJobStatus.STOP);
doReturn(stopped).when(fixture.service()).getById(stopped.getId());
when(fixture.mapper().selectByIdForUpdate(stopped.getId()))
.thenReturn(stopped, stopped, stopped);
doThrow(new DataAccessResourceFailureException("delete failed"))
.when(fixture.scheduleAdapter()).delete(stopped.getId());
Assert.assertThrows(
DataAccessResourceFailureException.class,
() -> fixture.service().deleteJob(List.of(stopped.getId())));
verify(fixture.service(), never()).removeById(stopped.getId());
verify(fixture.executionStore(), atLeastOnce()).cancelPending(
eq(stopped.getId()), anyString());
}
@Test
public void triggerNowMustRejectStoppedTaskWithoutCallingProvider() {
Fixture fixture = fixture();
SysJob stopped = job(EnumJobStatus.STOP);
when(fixture.mapper().selectByIdForUpdate(stopped.getId())).thenReturn(stopped);
Assert.assertThrows(
BusinessException.class,
() -> fixture.service().triggerNow(stopped.getId()));
verify(fixture.scheduleAdapter(), never()).triggerNow(any(BigInteger.class));
}
private static Fixture fixture() {
return fixture(null);
}
@SuppressWarnings({"unchecked", "rawtypes"})
private static Fixture fixture(RuntimeException transactionBeginFailure) {
SysJobMapper mapper = mock(SysJobMapper.class);
SysJobScheduleAdapter scheduleAdapter = mock(SysJobScheduleAdapter.class);
SysJobExecutionStore executionStore = mock(SysJobExecutionStore.class);
ScheduleService scheduleService = mock(ScheduleService.class);
RedisLockExecutor lockExecutor = mock(RedisLockExecutor.class);
PlatformTransactionManager transactionManager = mock(PlatformTransactionManager.class);
if (transactionBeginFailure == null) {
when(transactionManager.getTransaction(any()))
.thenReturn(mock(TransactionStatus.class));
} else {
when(transactionManager.getTransaction(any())).thenThrow(transactionBeginFailure);
}
doAnswer(invocation -> ((Supplier) invocation.getArgument(3)).get())
.when(lockExecutor).executeWithRenewingLock(
anyString(), any(Duration.class), any(Duration.class), any(Supplier.class));
SysJobExecutionProperties properties = new SysJobExecutionProperties();
TestService service = spy(new TestService(
mapper, scheduleAdapter, executionStore, scheduleService,
lockExecutor, transactionManager, properties));
return new Fixture(
service, mapper, scheduleAdapter, executionStore, transactionManager);
}
private static SysJob job(EnumJobStatus status) {
SysJob job = new SysJob();
job.setId(BigInteger.ONE);
job.setTenantId(BigInteger.TWO);
job.setStatus(status.getCode());
job.setScheduleGeneration(0L);
return job;
}
private static final class TestService extends SysJobServiceImpl {
private TestService(SysJobMapper mapper,
SysJobScheduleAdapter scheduleAdapter,
SysJobExecutionStore executionStore,
ScheduleService scheduleService,
RedisLockExecutor lockExecutor,
PlatformTransactionManager transactionManager,
SysJobExecutionProperties properties) {
super(scheduleAdapter, executionStore, scheduleService, lockExecutor,
transactionManager, properties, "Asia/Shanghai");
this.mapper = mapper;
}
}
private record Fixture(TestService service,
SysJobMapper mapper,
SysJobScheduleAdapter scheduleAdapter,
SysJobExecutionStore executionStore,
PlatformTransactionManager transactionManager) {
}
}