feat: 切换定时任务至分布式调度底座
- 以执行账本和有界 Worker 承载重负载任务与故障接管 - 接入统一调度 Starter 并增加 MySQL 迁移、指标和回归测试
This commit is contained in:
@@ -2,7 +2,7 @@ package tech.easyflow.ai.service;
|
||||
|
||||
import org.springframework.stereotype.Service;
|
||||
import tech.easyflow.ai.entity.Workflow;
|
||||
import tech.easyflow.common.constant.enums.EnumDataStatus;
|
||||
import tech.easyflow.ai.enums.PublishStatus;
|
||||
import tech.easyflow.common.entity.LoginAccount;
|
||||
import tech.easyflow.common.web.exceptions.BusinessException;
|
||||
import tech.easyflow.system.enums.CategoryResourceType;
|
||||
@@ -15,7 +15,7 @@ import java.util.Objects;
|
||||
/**
|
||||
* 工作流使用权限校验服务。
|
||||
*
|
||||
* <p>统一封装工作流存在性、租户、启用状态和资源使用权限校验,供页面能力和后台任务复用。</p>
|
||||
* <p>统一封装工作流存在性、租户、发布快照和资源使用权限校验,供页面能力和后台任务复用。</p>
|
||||
*/
|
||||
@Service
|
||||
public class WorkflowUsageAuthorizationService {
|
||||
@@ -40,20 +40,20 @@ public class WorkflowUsageAuthorizationService {
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取当前账号可使用的启用工作流。
|
||||
* 获取当前账号可使用的已发布工作流视图。
|
||||
*
|
||||
* @param workflowId 工作流 ID
|
||||
* @param account 使用工作流的账号
|
||||
* @param denyMessage 校验失败提示
|
||||
* @return 可使用的工作流
|
||||
* @throws BusinessException 工作流不存在、未启用、跨租户或无使用权限时抛出
|
||||
* @return 可使用的工作流发布视图
|
||||
* @throws BusinessException 工作流不存在、未发布、缺少发布快照、跨租户或无使用权限时抛出
|
||||
*/
|
||||
public Workflow requireUsableWorkflow(
|
||||
BigInteger workflowId,
|
||||
LoginAccount account,
|
||||
String denyMessage) {
|
||||
String message = denyMessage == null || denyMessage.isBlank()
|
||||
? "工作流不存在、已禁用或无权使用"
|
||||
? "工作流不存在、未发布或无权使用"
|
||||
: denyMessage;
|
||||
if (workflowId == null || account == null || account.getId() == null
|
||||
|| account.getTenantId() == null) {
|
||||
@@ -62,7 +62,9 @@ public class WorkflowUsageAuthorizationService {
|
||||
Workflow workflow = workflowService.getById(workflowId);
|
||||
boolean usable = workflow != null
|
||||
&& Objects.equals(workflow.getTenantId(), account.getTenantId())
|
||||
&& EnumDataStatus.AVAILABLE.getCode().equals(workflow.getStatus())
|
||||
&& PublishStatus.PUBLISHED.getCode().equals(workflow.getPublishStatus())
|
||||
&& workflow.getPublishedSnapshotJson() != null
|
||||
&& !workflow.getPublishedSnapshotJson().isEmpty()
|
||||
&& resourceAccessService.canAccess(
|
||||
account,
|
||||
CategoryResourceType.WORKFLOW,
|
||||
@@ -71,6 +73,10 @@ public class WorkflowUsageAuthorizationService {
|
||||
if (!usable) {
|
||||
throw new BusinessException(403, 403, message);
|
||||
}
|
||||
return workflow;
|
||||
Workflow published = workflowService.toPublishedView(workflow);
|
||||
if (published == null) {
|
||||
throw new BusinessException(403, 403, message);
|
||||
}
|
||||
return published;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3,7 +3,7 @@ package tech.easyflow.ai.service;
|
||||
import org.junit.Assert;
|
||||
import org.junit.Test;
|
||||
import tech.easyflow.ai.entity.Workflow;
|
||||
import tech.easyflow.common.constant.enums.EnumDataStatus;
|
||||
import tech.easyflow.ai.enums.PublishStatus;
|
||||
import tech.easyflow.common.entity.LoginAccount;
|
||||
import tech.easyflow.common.web.exceptions.BusinessException;
|
||||
import tech.easyflow.system.enums.CategoryResourceType;
|
||||
@@ -11,6 +11,7 @@ import tech.easyflow.system.enums.ResourceAction;
|
||||
import tech.easyflow.system.service.ResourceAccessService;
|
||||
|
||||
import java.math.BigInteger;
|
||||
import java.util.Map;
|
||||
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.when;
|
||||
@@ -21,14 +22,14 @@ import static org.mockito.Mockito.when;
|
||||
public class WorkflowUsageAuthorizationServiceTest {
|
||||
|
||||
/**
|
||||
* 验证禁用工作流即使资源权限允许也不能被使用。
|
||||
* 验证未发布工作流即使资源权限允许也不能被使用。
|
||||
*/
|
||||
@Test
|
||||
public void shouldRejectDisabledWorkflow() {
|
||||
public void shouldRejectUnpublishedWorkflow() {
|
||||
BigInteger workflowId = BigInteger.valueOf(101);
|
||||
WorkflowService workflowService = mock(WorkflowService.class);
|
||||
ResourceAccessService resourceAccessService = mock(ResourceAccessService.class);
|
||||
Workflow workflow = workflow(workflowId, BigInteger.TEN, EnumDataStatus.UNAVAILABLE.getCode());
|
||||
Workflow workflow = workflow(workflowId, BigInteger.TEN, PublishStatus.DRAFT, Map.of());
|
||||
LoginAccount account = account(BigInteger.ONE, BigInteger.TEN);
|
||||
when(workflowService.getById(workflowId)).thenReturn(workflow);
|
||||
when(resourceAccessService.canAccess(
|
||||
@@ -58,7 +59,8 @@ public class WorkflowUsageAuthorizationServiceTest {
|
||||
Workflow workflow = workflow(
|
||||
workflowId,
|
||||
BigInteger.valueOf(20),
|
||||
EnumDataStatus.AVAILABLE.getCode());
|
||||
PublishStatus.PUBLISHED,
|
||||
Map.of("title", "published"));
|
||||
LoginAccount account = account(BigInteger.ONE, BigInteger.TEN);
|
||||
when(workflowService.getById(workflowId)).thenReturn(workflow);
|
||||
WorkflowUsageAuthorizationService service =
|
||||
@@ -71,17 +73,50 @@ public class WorkflowUsageAuthorizationServiceTest {
|
||||
}
|
||||
|
||||
/**
|
||||
* 验证启用、同租户且具有使用权限的工作流可以返回。
|
||||
* 验证已发布、同租户且具有使用权限的工作流返回发布视图。
|
||||
*/
|
||||
@Test
|
||||
public void shouldReturnUsableWorkflow() {
|
||||
public void shouldReturnPublishedWorkflowView() {
|
||||
BigInteger workflowId = BigInteger.valueOf(103);
|
||||
WorkflowService workflowService = mock(WorkflowService.class);
|
||||
ResourceAccessService resourceAccessService = mock(ResourceAccessService.class);
|
||||
Workflow workflow = workflow(
|
||||
workflowId,
|
||||
BigInteger.TEN,
|
||||
EnumDataStatus.AVAILABLE.getCode());
|
||||
PublishStatus.PUBLISHED,
|
||||
Map.of("title", "published"));
|
||||
Workflow published = new Workflow();
|
||||
published.setId(workflowId);
|
||||
published.setTitle("发布版");
|
||||
LoginAccount account = account(BigInteger.ONE, BigInteger.TEN);
|
||||
when(workflowService.getById(workflowId)).thenReturn(workflow);
|
||||
when(resourceAccessService.canAccess(
|
||||
account,
|
||||
CategoryResourceType.WORKFLOW,
|
||||
workflow,
|
||||
ResourceAction.USE)).thenReturn(true);
|
||||
when(workflowService.toPublishedView(workflow)).thenReturn(published);
|
||||
WorkflowUsageAuthorizationService service =
|
||||
new WorkflowUsageAuthorizationService(workflowService, resourceAccessService);
|
||||
|
||||
Workflow result = service.requireUsableWorkflow(workflowId, account, "工作流不可用");
|
||||
|
||||
Assert.assertSame(result, published);
|
||||
}
|
||||
|
||||
/**
|
||||
* 验证发布状态异常但缺少快照的工作流不可被后台任务使用。
|
||||
*/
|
||||
@Test
|
||||
public void shouldRejectPublishedWorkflowWithoutSnapshot() {
|
||||
BigInteger workflowId = BigInteger.valueOf(104);
|
||||
WorkflowService workflowService = mock(WorkflowService.class);
|
||||
ResourceAccessService resourceAccessService = mock(ResourceAccessService.class);
|
||||
Workflow workflow = workflow(
|
||||
workflowId,
|
||||
BigInteger.TEN,
|
||||
PublishStatus.PUBLISHED,
|
||||
Map.of());
|
||||
LoginAccount account = account(BigInteger.ONE, BigInteger.TEN);
|
||||
when(workflowService.getById(workflowId)).thenReturn(workflow);
|
||||
when(resourceAccessService.canAccess(
|
||||
@@ -92,9 +127,10 @@ public class WorkflowUsageAuthorizationServiceTest {
|
||||
WorkflowUsageAuthorizationService service =
|
||||
new WorkflowUsageAuthorizationService(workflowService, resourceAccessService);
|
||||
|
||||
Workflow result = service.requireUsableWorkflow(workflowId, account, "工作流不可用");
|
||||
|
||||
Assert.assertSame(result, workflow);
|
||||
Assert.assertThrows(
|
||||
BusinessException.class,
|
||||
() -> service.requireUsableWorkflow(workflowId, account, "工作流不可用")
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -102,14 +138,18 @@ public class WorkflowUsageAuthorizationServiceTest {
|
||||
*
|
||||
* @param id 工作流 ID
|
||||
* @param tenantId 租户 ID
|
||||
* @param status 工作流状态
|
||||
* @param publishStatus 工作流发布状态
|
||||
* @param snapshot 工作流发布快照
|
||||
* @return 工作流
|
||||
*/
|
||||
private Workflow workflow(BigInteger id, BigInteger tenantId, Integer status) {
|
||||
private Workflow workflow(BigInteger id, BigInteger tenantId,
|
||||
PublishStatus publishStatus,
|
||||
Map<String, Object> snapshot) {
|
||||
Workflow workflow = new Workflow();
|
||||
workflow.setId(id);
|
||||
workflow.setTenantId(tenantId);
|
||||
workflow.setStatus(status);
|
||||
workflow.setPublishStatus(publishStatus.getCode());
|
||||
workflow.setPublishedSnapshotJson(snapshot);
|
||||
return workflow;
|
||||
}
|
||||
|
||||
|
||||
@@ -11,13 +11,18 @@
|
||||
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>org.springframework.boot</groupId>
|
||||
<artifactId>spring-boot-starter-quartz</artifactId>
|
||||
<groupId>com.easyagents</groupId>
|
||||
<artifactId>easy-agents-scheduler-core</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>com.mybatis-flex</groupId>
|
||||
<artifactId>mybatis-flex-spring-boot3-starter</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>io.micrometer</groupId>
|
||||
<artifactId>micrometer-core</artifactId>
|
||||
<version>1.15.7</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>tech.easyflow</groupId>
|
||||
<artifactId>easyflow-common-base</artifactId>
|
||||
@@ -48,6 +53,11 @@
|
||||
<version>5.12.0</version>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>com.mysql</groupId>
|
||||
<artifactId>mysql-connector-j</artifactId>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>junit</groupId>
|
||||
<artifactId>junit</artifactId>
|
||||
|
||||
@@ -2,11 +2,13 @@ package tech.easyflow.job.config;
|
||||
|
||||
import org.mybatis.spring.annotation.MapperScan;
|
||||
import org.springframework.boot.autoconfigure.AutoConfiguration;
|
||||
import org.springframework.boot.context.properties.EnableConfigurationProperties;
|
||||
import org.springframework.context.annotation.ComponentScan;
|
||||
|
||||
@AutoConfiguration
|
||||
@MapperScan("tech.easyflow.job.mapper")
|
||||
@ComponentScan("tech.easyflow.job")
|
||||
@EnableConfigurationProperties(SysJobExecutionProperties.class)
|
||||
public class JobModuleConfig {
|
||||
|
||||
public JobModuleConfig() {
|
||||
|
||||
@@ -0,0 +1,49 @@
|
||||
package tech.easyflow.job.config;
|
||||
|
||||
import org.springframework.beans.factory.InitializingBean;
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
/** 防止调度、Worker 与管理 Saga 形成连接池级等待环。 */
|
||||
@Component
|
||||
public class SysJobConnectionCapacityValidator implements InitializingBean {
|
||||
|
||||
private final SysJobExecutionProperties properties;
|
||||
private final int poolSize;
|
||||
private final int quartzThreadCount;
|
||||
private final boolean schedulerEnabled;
|
||||
|
||||
public SysJobConnectionCapacityValidator(
|
||||
SysJobExecutionProperties properties,
|
||||
@Value("${spring.datasource.hikari.maximum-pool-size:10}") int poolSize,
|
||||
@Value("${easy-agents.scheduler.quartz.thread-count:10}") int quartzThreadCount,
|
||||
@Value("${easy-agents.scheduler.enabled:false}") boolean schedulerEnabled) {
|
||||
this.properties = properties;
|
||||
this.poolSize = poolSize;
|
||||
this.quartzThreadCount = quartzThreadCount;
|
||||
this.schedulerEnabled = schedulerEnabled;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void afterPropertiesSet() {
|
||||
if (!schedulerEnabled || !properties.isEnabled()) return;
|
||||
int required = requiredPoolSize(
|
||||
properties.getManagementCommandConcurrency(),
|
||||
quartzThreadCount,
|
||||
properties.getWorkerCount(),
|
||||
properties.getBusinessConnectionReserve());
|
||||
if (poolSize < required) {
|
||||
throw new IllegalStateException(
|
||||
"定时任务连接池容量不足: spring.datasource.hikari.maximum-pool-size="
|
||||
+ poolSize + ", 至少需要 " + required
|
||||
+ " (2*management-command-concurrency + quartz.thread-count"
|
||||
+ " + worker-count + business-connection-reserve)");
|
||||
}
|
||||
}
|
||||
|
||||
static int requiredPoolSize(int managementConcurrency, int quartzThreads,
|
||||
int workers, int businessReserve) {
|
||||
return Math.addExact(Math.multiplyExact(managementConcurrency, 2),
|
||||
Math.addExact(quartzThreads, Math.addExact(workers, businessReserve)));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,80 @@
|
||||
package tech.easyflow.job.config;
|
||||
|
||||
import org.springframework.boot.context.properties.ConfigurationProperties;
|
||||
|
||||
import java.time.Duration;
|
||||
|
||||
/** 定时任务持久执行器配置。 */
|
||||
@ConfigurationProperties(prefix = "easyflow.job.execution")
|
||||
public class SysJobExecutionProperties {
|
||||
private boolean enabled = true;
|
||||
private int workerCount = 4;
|
||||
private int managementCommandConcurrency = 4;
|
||||
private int businessConnectionReserve = 4;
|
||||
private Duration pollInterval = Duration.ofMillis(500);
|
||||
private Duration leaseDuration = Duration.ofMinutes(2);
|
||||
private Duration heartbeatInterval = Duration.ofSeconds(30);
|
||||
private Duration retryBackoff = Duration.ofSeconds(5);
|
||||
private Duration shutdownWaitTimeout = Duration.ofSeconds(30);
|
||||
private int infrastructureRetryLimit = 16;
|
||||
private int registrationMaxAttempts = 3;
|
||||
private Duration registrationRetryDelay = Duration.ofMillis(100);
|
||||
private int registrationQuartzRefireLimit;
|
||||
private Duration registrationQuartzRefireDelay = Duration.ofMillis(250);
|
||||
|
||||
public boolean isEnabled() { return enabled; }
|
||||
public void setEnabled(boolean enabled) { this.enabled = enabled; }
|
||||
public int getWorkerCount() { return workerCount; }
|
||||
public void setWorkerCount(int value) { this.workerCount = positive(value, "workerCount"); }
|
||||
public int getManagementCommandConcurrency() { return managementCommandConcurrency; }
|
||||
public void setManagementCommandConcurrency(int value) {
|
||||
this.managementCommandConcurrency = positive(value, "managementCommandConcurrency");
|
||||
}
|
||||
public int getBusinessConnectionReserve() { return businessConnectionReserve; }
|
||||
public void setBusinessConnectionReserve(int value) {
|
||||
this.businessConnectionReserve = positive(value, "businessConnectionReserve");
|
||||
}
|
||||
public Duration getPollInterval() { return pollInterval; }
|
||||
public void setPollInterval(Duration value) { this.pollInterval = positive(value, "pollInterval"); }
|
||||
public Duration getLeaseDuration() { return leaseDuration; }
|
||||
public void setLeaseDuration(Duration value) { this.leaseDuration = positive(value, "leaseDuration"); }
|
||||
public Duration getHeartbeatInterval() { return heartbeatInterval; }
|
||||
public void setHeartbeatInterval(Duration value) { this.heartbeatInterval = positive(value, "heartbeatInterval"); }
|
||||
public Duration getRetryBackoff() { return retryBackoff; }
|
||||
public void setRetryBackoff(Duration value) { this.retryBackoff = positive(value, "retryBackoff"); }
|
||||
public Duration getShutdownWaitTimeout() { return shutdownWaitTimeout; }
|
||||
public void setShutdownWaitTimeout(Duration value) { this.shutdownWaitTimeout = positive(value, "shutdownWaitTimeout"); }
|
||||
public int getInfrastructureRetryLimit() { return infrastructureRetryLimit; }
|
||||
public void setInfrastructureRetryLimit(int value) { this.infrastructureRetryLimit = positive(value, "infrastructureRetryLimit"); }
|
||||
public int getRegistrationMaxAttempts() { return registrationMaxAttempts; }
|
||||
public void setRegistrationMaxAttempts(int value) { this.registrationMaxAttempts = positive(value, "registrationMaxAttempts"); }
|
||||
public Duration getRegistrationRetryDelay() { return registrationRetryDelay; }
|
||||
public void setRegistrationRetryDelay(Duration value) { this.registrationRetryDelay = positive(value, "registrationRetryDelay"); }
|
||||
public int getRegistrationQuartzRefireLimit() { return registrationQuartzRefireLimit; }
|
||||
public void setRegistrationQuartzRefireLimit(int value) {
|
||||
if (value < 0) throw new IllegalArgumentException("registrationQuartzRefireLimit must not be negative");
|
||||
this.registrationQuartzRefireLimit = value;
|
||||
}
|
||||
public Duration getRegistrationQuartzRefireDelay() { return registrationQuartzRefireDelay; }
|
||||
public void setRegistrationQuartzRefireDelay(Duration value) { this.registrationQuartzRefireDelay = positive(value, "registrationQuartzRefireDelay"); }
|
||||
|
||||
public void validate() {
|
||||
Duration minimumLease = heartbeatInterval.multipliedBy(3);
|
||||
if (leaseDuration.compareTo(minimumLease) <= 0) {
|
||||
throw new IllegalStateException(
|
||||
"leaseDuration must be greater than three times heartbeatInterval");
|
||||
}
|
||||
}
|
||||
|
||||
private static int positive(int value, String name) {
|
||||
if (value < 1) throw new IllegalArgumentException(name + " must be positive");
|
||||
return value;
|
||||
}
|
||||
|
||||
private static Duration positive(Duration value, String name) {
|
||||
if (value == null || value.isZero() || value.isNegative()) {
|
||||
throw new IllegalArgumentException(name + " must be positive");
|
||||
}
|
||||
return value;
|
||||
}
|
||||
}
|
||||
@@ -81,6 +81,12 @@ public class SysJobBase extends DateEntity implements Serializable {
|
||||
@Column(comment = "数据状态")
|
||||
private Integer status;
|
||||
|
||||
/**
|
||||
* 调度定义代际。每次修改定义或从停止进入运行时递增,用于隔离旧 Quartz fire。
|
||||
*/
|
||||
@Column(comment = "调度定义代际")
|
||||
private Long scheduleGeneration;
|
||||
|
||||
/**
|
||||
* 创建时间
|
||||
*/
|
||||
@@ -199,6 +205,14 @@ public class SysJobBase extends DateEntity implements Serializable {
|
||||
this.status = status;
|
||||
}
|
||||
|
||||
public Long getScheduleGeneration() {
|
||||
return scheduleGeneration;
|
||||
}
|
||||
|
||||
public void setScheduleGeneration(Long scheduleGeneration) {
|
||||
this.scheduleGeneration = scheduleGeneration;
|
||||
}
|
||||
|
||||
public Date getCreated() {
|
||||
return created;
|
||||
}
|
||||
|
||||
@@ -20,24 +20,86 @@ public class SysJobLogBase implements Serializable {
|
||||
@Id(keyType = KeyType.Generator, value = "snowFlakeId", comment = "主键")
|
||||
private BigInteger id;
|
||||
|
||||
/** 执行幂等键的 SHA-256。 */
|
||||
@Column(comment = "执行幂等键SHA-256")
|
||||
private String executionKey;
|
||||
|
||||
/**
|
||||
* 任务ID
|
||||
*/
|
||||
@Column(comment = "任务ID")
|
||||
private BigInteger jobId;
|
||||
|
||||
/** 触发所属任务代际。 */
|
||||
@Column(comment = "触发所属任务代际")
|
||||
private Long jobGeneration;
|
||||
|
||||
@Column(tenantId = true, comment = "租户ID")
|
||||
private BigInteger tenantId;
|
||||
|
||||
@Column(comment = "部门ID")
|
||||
private BigInteger deptId;
|
||||
|
||||
/**
|
||||
* 任务名称
|
||||
*/
|
||||
@Column(comment = "任务名称")
|
||||
private String jobName;
|
||||
|
||||
@Column(comment = "任务类型快照")
|
||||
private Integer jobType;
|
||||
|
||||
/**
|
||||
* 任务参数
|
||||
*/
|
||||
@Column(typeHandler = FastjsonTypeHandler.class, comment = "任务参数")
|
||||
private Map<String, Object> jobParams;
|
||||
|
||||
@Column(typeHandler = FastjsonTypeHandler.class, comment = "任务扩展配置快照")
|
||||
private Map<String, Object> jobOptions;
|
||||
|
||||
@Column(comment = "是否允许并发执行")
|
||||
private Integer allowConcurrent;
|
||||
|
||||
@Column(comment = "触发来源")
|
||||
private String triggerSource;
|
||||
|
||||
@Column(comment = "立即触发调用标识")
|
||||
private String invocationId;
|
||||
|
||||
@Column(comment = "计划触发时间")
|
||||
private Date scheduledFireTime;
|
||||
|
||||
@Column(comment = "实际触发时间")
|
||||
private Date actualFireTime;
|
||||
|
||||
@Column(comment = "Quartz物理触发实例")
|
||||
private String fireInstanceId;
|
||||
|
||||
@Column(comment = "是否为Quartz故障恢复")
|
||||
private Integer recovering;
|
||||
|
||||
@Column(comment = "执行节点")
|
||||
private String leaseOwner;
|
||||
|
||||
@Column(comment = "本轮执行令牌")
|
||||
private String executionToken;
|
||||
|
||||
@Column(comment = "租约到期时间")
|
||||
private Date leaseUntil;
|
||||
|
||||
@Column(comment = "最近续租时间")
|
||||
private Date heartbeatTime;
|
||||
|
||||
@Column(comment = "基础设施恢复次数")
|
||||
private Integer attemptCount;
|
||||
|
||||
@Column(comment = "下次恢复时间")
|
||||
private Date nextRetryTime;
|
||||
|
||||
@Column(comment = "状态版本")
|
||||
private Long version;
|
||||
|
||||
/**
|
||||
* 执行结果
|
||||
*/
|
||||
@@ -88,6 +150,9 @@ public class SysJobLogBase implements Serializable {
|
||||
this.id = id;
|
||||
}
|
||||
|
||||
public String getExecutionKey() { return executionKey; }
|
||||
public void setExecutionKey(String executionKey) { this.executionKey = executionKey; }
|
||||
|
||||
public BigInteger getJobId() {
|
||||
return jobId;
|
||||
}
|
||||
@@ -96,6 +161,14 @@ public class SysJobLogBase implements Serializable {
|
||||
this.jobId = jobId;
|
||||
}
|
||||
|
||||
public Long getJobGeneration() { return jobGeneration; }
|
||||
public void setJobGeneration(Long jobGeneration) { this.jobGeneration = jobGeneration; }
|
||||
|
||||
public BigInteger getTenantId() { return tenantId; }
|
||||
public void setTenantId(BigInteger tenantId) { this.tenantId = tenantId; }
|
||||
public BigInteger getDeptId() { return deptId; }
|
||||
public void setDeptId(BigInteger deptId) { this.deptId = deptId; }
|
||||
|
||||
public String getJobName() {
|
||||
return jobName;
|
||||
}
|
||||
@@ -104,6 +177,9 @@ public class SysJobLogBase implements Serializable {
|
||||
this.jobName = jobName;
|
||||
}
|
||||
|
||||
public Integer getJobType() { return jobType; }
|
||||
public void setJobType(Integer jobType) { this.jobType = jobType; }
|
||||
|
||||
public Map<String, Object> getJobParams() {
|
||||
return jobParams;
|
||||
}
|
||||
@@ -112,6 +188,37 @@ public class SysJobLogBase implements Serializable {
|
||||
this.jobParams = jobParams;
|
||||
}
|
||||
|
||||
public Map<String, Object> getJobOptions() { return jobOptions; }
|
||||
public void setJobOptions(Map<String, Object> jobOptions) { this.jobOptions = jobOptions; }
|
||||
public Integer getAllowConcurrent() { return allowConcurrent; }
|
||||
public void setAllowConcurrent(Integer allowConcurrent) { this.allowConcurrent = allowConcurrent; }
|
||||
public String getTriggerSource() { return triggerSource; }
|
||||
public void setTriggerSource(String triggerSource) { this.triggerSource = triggerSource; }
|
||||
public String getInvocationId() { return invocationId; }
|
||||
public void setInvocationId(String invocationId) { this.invocationId = invocationId; }
|
||||
public Date getScheduledFireTime() { return scheduledFireTime; }
|
||||
public void setScheduledFireTime(Date scheduledFireTime) { this.scheduledFireTime = scheduledFireTime; }
|
||||
public Date getActualFireTime() { return actualFireTime; }
|
||||
public void setActualFireTime(Date actualFireTime) { this.actualFireTime = actualFireTime; }
|
||||
public String getFireInstanceId() { return fireInstanceId; }
|
||||
public void setFireInstanceId(String fireInstanceId) { this.fireInstanceId = fireInstanceId; }
|
||||
public Integer getRecovering() { return recovering; }
|
||||
public void setRecovering(Integer recovering) { this.recovering = recovering; }
|
||||
public String getLeaseOwner() { return leaseOwner; }
|
||||
public void setLeaseOwner(String leaseOwner) { this.leaseOwner = leaseOwner; }
|
||||
public String getExecutionToken() { return executionToken; }
|
||||
public void setExecutionToken(String executionToken) { this.executionToken = executionToken; }
|
||||
public Date getLeaseUntil() { return leaseUntil; }
|
||||
public void setLeaseUntil(Date leaseUntil) { this.leaseUntil = leaseUntil; }
|
||||
public Date getHeartbeatTime() { return heartbeatTime; }
|
||||
public void setHeartbeatTime(Date heartbeatTime) { this.heartbeatTime = heartbeatTime; }
|
||||
public Integer getAttemptCount() { return attemptCount; }
|
||||
public void setAttemptCount(Integer attemptCount) { this.attemptCount = attemptCount; }
|
||||
public Date getNextRetryTime() { return nextRetryTime; }
|
||||
public void setNextRetryTime(Date nextRetryTime) { this.nextRetryTime = nextRetryTime; }
|
||||
public Long getVersion() { return version; }
|
||||
public void setVersion(Long version) { this.version = version; }
|
||||
|
||||
public String getJobResult() {
|
||||
return jobResult;
|
||||
}
|
||||
|
||||
@@ -0,0 +1,7 @@
|
||||
package tech.easyflow.job.execution;
|
||||
|
||||
import tech.easyflow.job.entity.SysJobLog;
|
||||
|
||||
/** 当前节点持有租约的一次任务执行。 */
|
||||
public record ClaimedSysJobExecution(SysJobLog execution, String owner, String token) {
|
||||
}
|
||||
@@ -0,0 +1,87 @@
|
||||
package tech.easyflow.job.execution;
|
||||
|
||||
import com.easyagents.scheduler.ScheduleFireContext;
|
||||
import com.easyagents.scheduler.ScheduleHandler;
|
||||
import com.easyagents.scheduler.ScheduleRefireException;
|
||||
import org.springframework.dao.DataAccessException;
|
||||
import org.springframework.dao.DataAccessResourceFailureException;
|
||||
import org.springframework.dao.RecoverableDataAccessException;
|
||||
import org.springframework.dao.TransientDataAccessException;
|
||||
import org.springframework.stereotype.Component;
|
||||
import org.springframework.transaction.TransactionException;
|
||||
import tech.easyflow.job.config.SysJobExecutionProperties;
|
||||
|
||||
import java.util.concurrent.locks.LockSupport;
|
||||
|
||||
/** Quartz 短 Handler:只负责把触发写入数据库执行账本。 */
|
||||
@Component
|
||||
public class EasyFlowScheduleHandler implements ScheduleHandler {
|
||||
|
||||
public static final String CODE = "easyflow.job.execution";
|
||||
|
||||
private final SysJobExecutionRegistrar registrar;
|
||||
private final SysJobExecutionProperties properties;
|
||||
private final SysJobExecutionMetrics metrics;
|
||||
|
||||
public EasyFlowScheduleHandler(SysJobExecutionRegistrar registrar,
|
||||
SysJobExecutionProperties properties,
|
||||
SysJobExecutionMetrics metrics) {
|
||||
this.registrar = registrar;
|
||||
this.properties = properties;
|
||||
this.metrics = metrics;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String code() {
|
||||
return CODE;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void execute(ScheduleFireContext context) {
|
||||
long startedAt = System.nanoTime();
|
||||
boolean success = false;
|
||||
try {
|
||||
executeWithRetry(context);
|
||||
success = true;
|
||||
} finally {
|
||||
metrics.recordRegistration(System.nanoTime() - startedAt, success);
|
||||
}
|
||||
}
|
||||
|
||||
private void executeWithRetry(ScheduleFireContext context) {
|
||||
for (int attempt = 1; ; attempt++) {
|
||||
try {
|
||||
registrar.register(context);
|
||||
return;
|
||||
} catch (RuntimeException exception) {
|
||||
if (!isRetryable(exception)) {
|
||||
throw exception;
|
||||
}
|
||||
if (attempt >= properties.getRegistrationMaxAttempts()) {
|
||||
throw refire(exception);
|
||||
}
|
||||
LockSupport.parkNanos(properties.getRegistrationRetryDelay().toNanos());
|
||||
if (Thread.currentThread().isInterrupted()) {
|
||||
Thread.currentThread().interrupt();
|
||||
throw refire(exception);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private ScheduleRefireException refire(RuntimeException exception) {
|
||||
return new ScheduleRefireException(
|
||||
"定时任务触发登记失败,请求 Quartz 保留本次触发并重新执行",
|
||||
exception,
|
||||
properties.getRegistrationQuartzRefireLimit(),
|
||||
properties.getRegistrationQuartzRefireDelay());
|
||||
}
|
||||
|
||||
private static boolean isRetryable(RuntimeException exception) {
|
||||
if (exception instanceof TransactionException) return true;
|
||||
if (!(exception instanceof DataAccessException dataAccessException)) return false;
|
||||
return dataAccessException instanceof TransientDataAccessException
|
||||
|| dataAccessException instanceof RecoverableDataAccessException
|
||||
|| dataAccessException instanceof DataAccessResourceFailureException;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,8 @@
|
||||
package tech.easyflow.job.execution;
|
||||
|
||||
/** 任务在领取后、业务执行前已不再满足运行条件。 */
|
||||
public class SysJobCancelledException extends RuntimeException {
|
||||
public SysJobCancelledException(String message) {
|
||||
super(message);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,16 @@
|
||||
package tech.easyflow.job.execution;
|
||||
|
||||
import java.math.BigInteger;
|
||||
import java.time.Instant;
|
||||
|
||||
/** 可供三类业务执行器读取的稳定幂等上下文。 */
|
||||
public record SysJobExecutionContext(
|
||||
BigInteger executionId,
|
||||
String executionKey,
|
||||
BigInteger tenantId,
|
||||
int attempt,
|
||||
String triggerSource,
|
||||
String invocationId,
|
||||
Instant scheduledFireTime
|
||||
) {
|
||||
}
|
||||
@@ -0,0 +1,28 @@
|
||||
package tech.easyflow.job.execution;
|
||||
|
||||
import java.util.Optional;
|
||||
|
||||
/** 当前 Worker 线程的任务幂等上下文。 */
|
||||
public final class SysJobExecutionContextHolder {
|
||||
|
||||
private static final ThreadLocal<SysJobExecutionContext> CURRENT = new ThreadLocal<>();
|
||||
|
||||
private SysJobExecutionContextHolder() {
|
||||
}
|
||||
|
||||
/** 返回当前执行上下文;异步派生线程需要由业务显式传递。 */
|
||||
public static Optional<SysJobExecutionContext> current() {
|
||||
return Optional.ofNullable(CURRENT.get());
|
||||
}
|
||||
|
||||
static void set(SysJobExecutionContext context) {
|
||||
if (CURRENT.get() != null) {
|
||||
throw new IllegalStateException("定时任务执行上下文不允许嵌套");
|
||||
}
|
||||
CURRENT.set(context);
|
||||
}
|
||||
|
||||
static void clear() {
|
||||
CURRENT.remove();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,105 @@
|
||||
package tech.easyflow.job.execution;
|
||||
|
||||
import io.micrometer.core.instrument.Counter;
|
||||
import io.micrometer.core.instrument.Gauge;
|
||||
import io.micrometer.core.instrument.MeterRegistry;
|
||||
import io.micrometer.core.instrument.Timer;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.concurrent.atomic.AtomicLong;
|
||||
|
||||
/** 定时任务登记、队列、租约和执行指标。 */
|
||||
@Component
|
||||
public class SysJobExecutionMetrics {
|
||||
|
||||
private final Timer registrationTimer;
|
||||
private final Timer executionTimer;
|
||||
private final Counter registrationFailures;
|
||||
private final Counter claimConflicts;
|
||||
private final Counter leaseTakeovers;
|
||||
private final Counter lostLeases;
|
||||
private final Map<String, Counter> terminalCounters;
|
||||
private final AtomicLong pending = new AtomicLong();
|
||||
private final AtomicLong running = new AtomicLong();
|
||||
private final AtomicLong localActive = new AtomicLong();
|
||||
private final AtomicLong oldestBacklogMillis = new AtomicLong();
|
||||
|
||||
public SysJobExecutionMetrics(MeterRegistry registry) {
|
||||
registrationTimer = Timer.builder("easyflow.job.registration.duration")
|
||||
.description("Quartz trigger ledger registration duration")
|
||||
.register(registry);
|
||||
executionTimer = Timer.builder("easyflow.job.execution.duration")
|
||||
.description("Business job execution duration")
|
||||
.register(registry);
|
||||
registrationFailures = counter(registry, "easyflow.job.registration.failures");
|
||||
claimConflicts = counter(registry, "easyflow.job.claim.conflicts");
|
||||
leaseTakeovers = counter(registry, "easyflow.job.lease.takeovers");
|
||||
lostLeases = counter(registry, "easyflow.job.lease.lost");
|
||||
terminalCounters = Map.of(
|
||||
"success", terminalCounter(registry, "success"),
|
||||
"failure", terminalCounter(registry, "failure"),
|
||||
"dead", terminalCounter(registry, "dead"),
|
||||
"cancelled", terminalCounter(registry, "cancelled"));
|
||||
gauge(registry, "easyflow.job.queue.pending", pending);
|
||||
gauge(registry, "easyflow.job.queue.running", running);
|
||||
gauge(registry, "easyflow.job.execution.local_active", localActive);
|
||||
Gauge.builder("easyflow.job.queue.oldest_backlog_seconds", oldestBacklogMillis,
|
||||
value -> value.get() / 1_000.0D)
|
||||
.register(registry);
|
||||
}
|
||||
|
||||
public void recordRegistration(long durationNanos, boolean success) {
|
||||
registrationTimer.record(durationNanos, TimeUnit.NANOSECONDS);
|
||||
if (!success) registrationFailures.increment();
|
||||
}
|
||||
|
||||
public void recordClaimConflict() {
|
||||
claimConflicts.increment();
|
||||
}
|
||||
|
||||
public void recordLeaseTakeover() {
|
||||
leaseTakeovers.increment();
|
||||
}
|
||||
|
||||
public void recordLostLease() {
|
||||
lostLeases.increment();
|
||||
}
|
||||
|
||||
public void recordDead() {
|
||||
terminalCounters.get("dead").increment();
|
||||
}
|
||||
|
||||
public void executionStarted() {
|
||||
localActive.incrementAndGet();
|
||||
}
|
||||
|
||||
public void executionFinished(String status, long durationNanos) {
|
||||
localActive.updateAndGet(value -> Math.max(0L, value - 1L));
|
||||
executionTimer.record(durationNanos, TimeUnit.NANOSECONDS);
|
||||
// 失租、fencing 拒绝或关闭中断时没有本节点可确认的终态,status 合法为 null。
|
||||
Counter counter = status == null ? null : terminalCounters.get(status);
|
||||
if (counter != null) counter.increment();
|
||||
}
|
||||
|
||||
public void updateQueue(long pendingCount, long runningCount, long oldestMillis) {
|
||||
pending.set(pendingCount);
|
||||
running.set(runningCount);
|
||||
oldestBacklogMillis.set(Math.max(0L, oldestMillis));
|
||||
}
|
||||
|
||||
private static Counter counter(MeterRegistry registry, String name) {
|
||||
return Counter.builder(name).register(registry);
|
||||
}
|
||||
|
||||
private static Counter terminalCounter(MeterRegistry registry, String status) {
|
||||
return Counter.builder("easyflow.job.execution.terminal")
|
||||
.tag("status", status)
|
||||
.register(registry);
|
||||
}
|
||||
|
||||
private static void gauge(MeterRegistry registry, String name, AtomicLong value) {
|
||||
Gauge.builder(name, value, AtomicLong::get).register(registry);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,8 @@
|
||||
package tech.easyflow.job.execution;
|
||||
|
||||
import com.easyagents.scheduler.ScheduleFireContext;
|
||||
|
||||
/** 将调度触发幂等登记到持久执行账本。 */
|
||||
public interface SysJobExecutionRegistrar {
|
||||
void register(ScheduleFireContext context);
|
||||
}
|
||||
@@ -0,0 +1,332 @@
|
||||
package tech.easyflow.job.execution;
|
||||
|
||||
import com.easyagents.scheduler.ScheduleFireContext;
|
||||
import com.mybatisflex.core.tenant.TenantManager;
|
||||
import org.springframework.dao.ConcurrencyFailureException;
|
||||
import org.springframework.dao.DuplicateKeyException;
|
||||
import org.springframework.stereotype.Repository;
|
||||
import org.springframework.transaction.PlatformTransactionManager;
|
||||
import org.springframework.transaction.TransactionDefinition;
|
||||
import org.springframework.transaction.support.TransactionTemplate;
|
||||
import tech.easyflow.common.constant.enums.EnumJobResult;
|
||||
import tech.easyflow.common.constant.enums.EnumJobStatus;
|
||||
import tech.easyflow.job.config.SysJobExecutionProperties;
|
||||
import tech.easyflow.job.entity.SysJob;
|
||||
import tech.easyflow.job.entity.SysJobLog;
|
||||
import tech.easyflow.job.mapper.SysJobLogMapper;
|
||||
import tech.easyflow.job.mapper.SysJobMapper;
|
||||
import tech.easyflow.job.job.JobConstant;
|
||||
|
||||
import java.math.BigInteger;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.security.MessageDigest;
|
||||
import java.security.NoSuchAlgorithmException;
|
||||
import java.time.Duration;
|
||||
import java.util.Date;
|
||||
import java.util.HexFormat;
|
||||
import java.util.Objects;
|
||||
import java.util.Optional;
|
||||
import java.util.UUID;
|
||||
import java.util.concurrent.ThreadLocalRandom;
|
||||
import java.util.concurrent.locks.LockSupport;
|
||||
|
||||
/** `tb_sys_job_log` 执行账本及跨节点租约协议。 */
|
||||
@Repository
|
||||
public class SysJobExecutionStore implements SysJobExecutionRegistrar {
|
||||
|
||||
/** 单次认领最多清退的终态候选数,兼顾队首疏通和短事务边界。 */
|
||||
private static final int MAX_TERMINAL_ROWS_PER_CLAIM = 64;
|
||||
|
||||
/** 登记与领取锁序竞争时的短事务重试上限。 */
|
||||
private static final int MAX_CLAIM_CONCURRENCY_ATTEMPTS = 3;
|
||||
|
||||
private final SysJobMapper jobMapper;
|
||||
private final SysJobLogMapper logMapper;
|
||||
private final SysJobExecutionProperties properties;
|
||||
private final SysJobExecutionMetrics metrics;
|
||||
private final TransactionTemplate requiresNew;
|
||||
|
||||
public SysJobExecutionStore(SysJobMapper jobMapper,
|
||||
SysJobLogMapper logMapper,
|
||||
SysJobExecutionProperties properties,
|
||||
SysJobExecutionMetrics metrics,
|
||||
PlatformTransactionManager transactionManager) {
|
||||
this.jobMapper = jobMapper;
|
||||
this.logMapper = logMapper;
|
||||
this.properties = properties;
|
||||
this.metrics = metrics;
|
||||
this.requiresNew = new TransactionTemplate(transactionManager);
|
||||
this.requiresNew.setPropagationBehavior(TransactionDefinition.PROPAGATION_REQUIRES_NEW);
|
||||
this.requiresNew.setIsolationLevel(TransactionDefinition.ISOLATION_READ_COMMITTED);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void register(ScheduleFireContext context) {
|
||||
BigInteger jobId = parseJobId(context);
|
||||
long fireGeneration = parseGeneration(context);
|
||||
String executionKey = executionKey(jobId, fireGeneration, context);
|
||||
requiresNew.execute(status -> TenantManager.withoutTenantCondition(() -> {
|
||||
// 管理事务会先锁定任务行再改变启停状态。这里使用同一行锁,确保 Quartz
|
||||
// 即刻触发时等待管理事务提交,避免读取旧 STOP 后把本次 fire 当成功吞掉。
|
||||
SysJob job = jobMapper.selectByIdForUpdate(jobId);
|
||||
if (job == null
|
||||
|| !Integer.valueOf(EnumJobStatus.RUNNING.getCode()).equals(job.getStatus())
|
||||
|| !Objects.equals(job.getScheduleGeneration(), fireGeneration)) {
|
||||
return null;
|
||||
}
|
||||
SysJobLog execution = snapshot(job, fireGeneration, context, executionKey);
|
||||
try {
|
||||
logMapper.insert(execution);
|
||||
} catch (DuplicateKeyException duplicate) {
|
||||
// 只有 execution_key 命中才是可吞掉的幂等重复;主键或其他
|
||||
// 唯一约束冲突必须上抛,让 Quartz 保留 fire 并进行持久化重试。
|
||||
if (logMapper.selectIdByExecutionKey(executionKey) == null) {
|
||||
throw duplicate;
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}));
|
||||
}
|
||||
|
||||
public Optional<ClaimedSysJobExecution> claimOne(String owner) {
|
||||
return TenantManager.withoutTenantCondition(() -> {
|
||||
for (int index = 0; index < MAX_TERMINAL_ROWS_PER_CLAIM; index++) {
|
||||
ClaimAttempt attempt = executeClaimTransaction(owner);
|
||||
if (attempt == null) {
|
||||
return Optional.empty();
|
||||
}
|
||||
if (attempt.claim() != null) {
|
||||
return Optional.of(attempt.claim());
|
||||
}
|
||||
if (!attempt.retryImmediately()) {
|
||||
return Optional.empty();
|
||||
}
|
||||
}
|
||||
return Optional.empty();
|
||||
});
|
||||
}
|
||||
|
||||
private ClaimAttempt executeClaimTransaction(String owner) {
|
||||
for (int attempt = 1; attempt <= MAX_CLAIM_CONCURRENCY_ATTEMPTS; attempt++) {
|
||||
try {
|
||||
ClaimAttempt expired = claimExpiredIfPresent(owner);
|
||||
if (expired != null) return expired;
|
||||
return requiresNew.execute(status -> claimInTransaction(owner, false));
|
||||
} catch (ConcurrencyFailureException exception) {
|
||||
if (attempt == MAX_CLAIM_CONCURRENCY_ATTEMPTS) throw exception;
|
||||
// 短暂抖动后用全新事务重试,不改变 execution,也不把可恢复的锁冲突
|
||||
// 上抛为一次业务失败。
|
||||
LockSupport.parkNanos(ThreadLocalRandom.current().nextLong(
|
||||
1_000_000L, 5_000_001L));
|
||||
}
|
||||
}
|
||||
throw new IllegalStateException("定时任务认领重试状态异常");
|
||||
}
|
||||
|
||||
private ClaimAttempt claimExpiredIfPresent(String owner) {
|
||||
// 空范围 FOR UPDATE 会保留状态索引的末端间隙锁;同一事务再把 PENDING
|
||||
// 更新为 RUNNING 时会与其他 Worker 互锁,因此只在确认有候选后加锁。
|
||||
if (logMapper.selectExpiredClaimCandidateIdWithoutLock() == null) return null;
|
||||
return requiresNew.execute(status -> claimInTransaction(owner, true));
|
||||
}
|
||||
|
||||
private ClaimAttempt claimInTransaction(String owner, boolean expired) {
|
||||
SysJobLog candidate = expired
|
||||
? logMapper.selectExpiredClaimCandidate()
|
||||
: logMapper.selectPendingClaimCandidate();
|
||||
// 无锁探测到的过期行可能已被其他 Worker 锁住。SKIP LOCKED 此时返回空,
|
||||
// 不能结束整轮领取,否则一条受阻的过期记录会让 PENDING 队列反复空转。
|
||||
if (candidate == null) return expired ? null : ClaimAttempt.stop();
|
||||
if (candidate.getAttemptCount() != null
|
||||
&& candidate.getAttemptCount() >= properties.getInfrastructureRetryLimit()) {
|
||||
int marked = expired
|
||||
? logMapper.markExpiredDead(
|
||||
candidate.getId(), "执行节点多次失联,已停止自动接管")
|
||||
: logMapper.markPendingDead(
|
||||
candidate.getId(), "基础设施多次故障,已停止自动重试");
|
||||
if (marked == 1) {
|
||||
metrics.recordDead();
|
||||
}
|
||||
return marked == 1 ? ClaimAttempt.retry() : ClaimAttempt.stop();
|
||||
}
|
||||
SysJob job = jobMapper.selectByIdForUpdate(candidate.getJobId());
|
||||
if (job == null
|
||||
|| !Integer.valueOf(EnumJobStatus.RUNNING.getCode()).equals(job.getStatus())) {
|
||||
return claimAndCancel(candidate, owner, expired, "任务已停止或删除")
|
||||
? ClaimAttempt.retry() : ClaimAttempt.stop();
|
||||
}
|
||||
if (!Integer.valueOf(1).equals(candidate.getAllowConcurrent())
|
||||
&& logMapper.selectOtherActiveForUpdate(
|
||||
candidate.getJobId(), candidate.getId()) != null) {
|
||||
if (expired) {
|
||||
logMapper.releaseExpiredForRetry(candidate.getId(),
|
||||
micros(properties.getRetryBackoff()));
|
||||
} else {
|
||||
logMapper.deferPending(candidate.getId(), micros(properties.getRetryBackoff()));
|
||||
}
|
||||
metrics.recordClaimConflict();
|
||||
return ClaimAttempt.stop();
|
||||
}
|
||||
String token = UUID.randomUUID().toString();
|
||||
int claimed = expired
|
||||
? logMapper.claimExpired(candidate.getId(), owner, token,
|
||||
micros(properties.getLeaseDuration()))
|
||||
: logMapper.claimPending(candidate.getId(), owner, token,
|
||||
micros(properties.getLeaseDuration()));
|
||||
if (claimed != 1) {
|
||||
metrics.recordClaimConflict();
|
||||
return ClaimAttempt.stop();
|
||||
}
|
||||
if (expired) metrics.recordLeaseTakeover();
|
||||
candidate.setLeaseOwner(owner);
|
||||
candidate.setExecutionToken(token);
|
||||
candidate.setStatus(EnumJobResult.RUNNING.getCode());
|
||||
return ClaimAttempt.claimed(new ClaimedSysJobExecution(candidate, owner, token));
|
||||
}
|
||||
|
||||
private boolean claimAndCancel(SysJobLog candidate, String owner, boolean expired, String reason) {
|
||||
String token = UUID.randomUUID().toString();
|
||||
int claimed = expired
|
||||
? logMapper.claimExpired(candidate.getId(), owner, token,
|
||||
micros(properties.getLeaseDuration()))
|
||||
: logMapper.claimPending(candidate.getId(), owner, token,
|
||||
micros(properties.getLeaseDuration()));
|
||||
if (claimed == 1) {
|
||||
logMapper.finishOwned(candidate.getId(), owner, token,
|
||||
EnumJobResult.CANCELLED.getCode(), null, reason);
|
||||
}
|
||||
return claimed == 1;
|
||||
}
|
||||
|
||||
public boolean heartbeat(ClaimedSysJobExecution claim) {
|
||||
return TenantManager.withoutTenantCondition(() -> logMapper.renewLease(
|
||||
claim.execution().getId(), claim.owner(), claim.token(),
|
||||
micros(properties.getLeaseDuration())) == 1);
|
||||
}
|
||||
|
||||
public boolean finish(ClaimedSysJobExecution claim, int status, String result, String error) {
|
||||
return TenantManager.withoutTenantCondition(() -> logMapper.finishOwned(
|
||||
claim.execution().getId(), claim.owner(), claim.token(), status,
|
||||
truncate(result, 15000), truncate(error, 4000)) == 1);
|
||||
}
|
||||
|
||||
public boolean releaseForInfrastructureRetry(ClaimedSysJobExecution claim, String error) {
|
||||
return TenantManager.withoutTenantCondition(() -> logMapper.releaseOwnedForRetry(
|
||||
claim.execution().getId(), claim.owner(), claim.token(),
|
||||
micros(properties.getRetryBackoff()), truncate(error, 4000)) == 1);
|
||||
}
|
||||
|
||||
public int cancelPending(BigInteger jobId, String reason) {
|
||||
Integer cancelled = requiresNew.execute(status -> TenantManager.withoutTenantCondition(() -> {
|
||||
// 先锁待清理账本,再锁任务行,保持与 Worker 一致的 log -> job 锁序。
|
||||
logMapper.selectPendingIdsByJobIdForUpdate(jobId);
|
||||
// 旧 STOP/删除命令失锁后可能与新 START 重叠。锁定并重新读取当前任务状态,
|
||||
// 只有最终仍为 STOP(或已删除)时才清理,避免误取消新一代待执行记录。
|
||||
SysJob job = jobMapper.selectByIdForUpdate(jobId);
|
||||
if (job != null
|
||||
&& Integer.valueOf(EnumJobStatus.RUNNING.getCode()).equals(job.getStatus())) {
|
||||
return 0;
|
||||
}
|
||||
return logMapper.cancelPendingByJobId(jobId, truncate(reason, 4000));
|
||||
}));
|
||||
return cancelled == null ? 0 : cancelled;
|
||||
}
|
||||
|
||||
public SysJobQueueSnapshot queueSnapshot() {
|
||||
return TenantManager.withoutTenantCondition(() -> new SysJobQueueSnapshot(
|
||||
logMapper.countPending(),
|
||||
logMapper.countRunning(),
|
||||
logMapper.selectOldestPendingMillis()));
|
||||
}
|
||||
|
||||
private SysJobLog snapshot(SysJob job, long fireGeneration, ScheduleFireContext context,
|
||||
String executionKey) {
|
||||
SysJobLog log = new SysJobLog();
|
||||
String source = context.invocationId() == null ? "SCHEDULED" : "MANUAL";
|
||||
log.setExecutionKey(executionKey);
|
||||
log.setJobId(job.getId());
|
||||
log.setJobGeneration(fireGeneration);
|
||||
log.setTenantId(job.getTenantId());
|
||||
log.setDeptId(job.getDeptId());
|
||||
log.setJobName(job.getJobName());
|
||||
log.setJobType(job.getJobType());
|
||||
log.setJobParams(job.getJobParams());
|
||||
log.setJobOptions(job.getOptions());
|
||||
log.setAllowConcurrent(job.getAllowConcurrent());
|
||||
log.setTriggerSource(source);
|
||||
log.setInvocationId(context.invocationId());
|
||||
log.setScheduledFireTime(Date.from(context.scheduledFireTime()));
|
||||
log.setActualFireTime(Date.from(context.actualFireTime()));
|
||||
log.setFireInstanceId(context.fireInstanceId());
|
||||
log.setRecovering(context.recovering() ? 1 : 0);
|
||||
log.setAttemptCount(0);
|
||||
log.setNextRetryTime(Date.from(context.actualFireTime()));
|
||||
log.setStatus(EnumJobResult.PENDING.getCode());
|
||||
log.setVersion(0L);
|
||||
log.setCreated(new Date());
|
||||
return log;
|
||||
}
|
||||
|
||||
private static String executionKey(BigInteger jobId, long fireGeneration,
|
||||
ScheduleFireContext context) {
|
||||
String source = context.invocationId() == null ? "SCHEDULED" : "MANUAL";
|
||||
String canonical = source + '|' + jobId + '|' + fireGeneration + '|'
|
||||
+ (context.invocationId() == null
|
||||
? context.scheduledFireTime().toEpochMilli()
|
||||
: context.invocationId());
|
||||
return sha256(canonical);
|
||||
}
|
||||
|
||||
private static BigInteger parseJobId(ScheduleFireContext context) {
|
||||
try {
|
||||
return new BigInteger(context.scheduleId().name());
|
||||
} catch (NumberFormatException exception) {
|
||||
throw new IllegalArgumentException("非法 EasyFlow 定时任务标识: " + context.scheduleId(), exception);
|
||||
}
|
||||
}
|
||||
|
||||
private static long parseGeneration(ScheduleFireContext context) {
|
||||
String value = context.parameters().get(JobConstant.SCHEDULE_GENERATION);
|
||||
try {
|
||||
long generation = Long.parseLong(value);
|
||||
if (generation < 0L) throw new NumberFormatException("negative generation");
|
||||
return generation;
|
||||
} catch (NumberFormatException | NullPointerException exception) {
|
||||
throw new IllegalArgumentException(
|
||||
"非法 EasyFlow 定时任务调度代际: " + value, exception);
|
||||
}
|
||||
}
|
||||
|
||||
private static String sha256(String value) {
|
||||
try {
|
||||
byte[] digest = MessageDigest.getInstance("SHA-256")
|
||||
.digest(value.getBytes(StandardCharsets.UTF_8));
|
||||
return HexFormat.of().formatHex(digest);
|
||||
} catch (NoSuchAlgorithmException exception) {
|
||||
throw new IllegalStateException("JDK 缺少 SHA-256", exception);
|
||||
}
|
||||
}
|
||||
|
||||
private static String truncate(String value, int maxLength) {
|
||||
return value == null || value.length() <= maxLength ? value : value.substring(0, maxLength);
|
||||
}
|
||||
|
||||
private static long micros(Duration duration) {
|
||||
return Math.max(1L, duration.toNanos() / 1_000L);
|
||||
}
|
||||
|
||||
private record ClaimAttempt(ClaimedSysJobExecution claim, boolean retryImmediately) {
|
||||
|
||||
private static ClaimAttempt claimed(ClaimedSysJobExecution claim) {
|
||||
return new ClaimAttempt(claim, false);
|
||||
}
|
||||
|
||||
private static ClaimAttempt retry() {
|
||||
return new ClaimAttempt(null, true);
|
||||
}
|
||||
|
||||
private static ClaimAttempt stop() {
|
||||
return new ClaimAttempt(null, false);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,372 @@
|
||||
package tech.easyflow.job.execution;
|
||||
|
||||
import cn.hutool.core.exceptions.ExceptionUtil;
|
||||
import com.alibaba.fastjson2.JSON;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.context.SmartLifecycle;
|
||||
import org.springframework.stereotype.Component;
|
||||
import tech.easyflow.common.constant.enums.EnumJobResult;
|
||||
import tech.easyflow.job.config.SysJobExecutionProperties;
|
||||
import tech.easyflow.job.entity.SysJob;
|
||||
import tech.easyflow.job.entity.SysJobLog;
|
||||
|
||||
import java.lang.management.ManagementFactory;
|
||||
import java.util.Map;
|
||||
import java.util.Optional;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
import java.util.concurrent.ExecutorService;
|
||||
import java.util.concurrent.Executors;
|
||||
import java.util.concurrent.ScheduledExecutorService;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.concurrent.ThreadLocalRandom;
|
||||
import java.util.concurrent.atomic.AtomicBoolean;
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
import java.util.concurrent.locks.LockSupport;
|
||||
|
||||
/** 固定并发、数据库驱动的重任务 Worker。 */
|
||||
@Component
|
||||
public class SysJobExecutionWorker implements SmartLifecycle {
|
||||
|
||||
private static final Logger log = LoggerFactory.getLogger(SysJobExecutionWorker.class);
|
||||
|
||||
private final SysJobExecutionStore store;
|
||||
private final SysJobInvoker invoker;
|
||||
private final SysJobExecutionProperties properties;
|
||||
private final SysJobExecutionMetrics metrics;
|
||||
private final String owner = buildOwner();
|
||||
private final AtomicBoolean running = new AtomicBoolean();
|
||||
private final Map<String, ActiveExecution> active = new ConcurrentHashMap<>();
|
||||
private ExecutorService workers;
|
||||
private ScheduledExecutorService heartbeat;
|
||||
|
||||
public SysJobExecutionWorker(SysJobExecutionStore store,
|
||||
SysJobInvoker invoker,
|
||||
SysJobExecutionProperties properties,
|
||||
SysJobExecutionMetrics metrics) {
|
||||
this.store = store;
|
||||
this.invoker = invoker;
|
||||
this.properties = properties;
|
||||
this.metrics = metrics;
|
||||
}
|
||||
|
||||
@Override
|
||||
public synchronized void start() {
|
||||
if (!properties.isEnabled()) return;
|
||||
properties.validate();
|
||||
if (!running.compareAndSet(false, true)) return;
|
||||
AtomicInteger workerNumber = new AtomicInteger();
|
||||
workers = Executors.newFixedThreadPool(properties.getWorkerCount(), runnable -> {
|
||||
Thread thread = new Thread(runnable,
|
||||
"easyflow-job-worker-" + workerNumber.incrementAndGet());
|
||||
// 正常关闭仍等待在途任务;超时且业务代码忽略中断时不阻塞 JVM 退出,
|
||||
// 未完成记录由数据库租约交给其他节点接管。
|
||||
thread.setDaemon(true);
|
||||
return thread;
|
||||
});
|
||||
heartbeat = Executors.newSingleThreadScheduledExecutor(runnable -> {
|
||||
Thread thread = new Thread(runnable, "easyflow-job-heartbeat");
|
||||
thread.setDaemon(true);
|
||||
return thread;
|
||||
});
|
||||
for (int i = 0; i < properties.getWorkerCount(); i++) workers.submit(this::workerLoop);
|
||||
refreshQueueMetrics();
|
||||
heartbeat.scheduleWithFixedDelay(this::heartbeatAll,
|
||||
properties.getHeartbeatInterval().toMillis(),
|
||||
properties.getHeartbeatInterval().toMillis(), TimeUnit.MILLISECONDS);
|
||||
log.info("定时任务 Worker 已启动: owner={}, workers={}", owner, properties.getWorkerCount());
|
||||
}
|
||||
|
||||
private void workerLoop() {
|
||||
int consecutiveFailures = 0;
|
||||
while (running.get()) {
|
||||
try {
|
||||
Optional<ClaimedSysJobExecution> claim = store.claimOne(owner);
|
||||
consecutiveFailures = 0;
|
||||
if (claim.isPresent()) {
|
||||
execute(claim.get());
|
||||
if (running.get()) Thread.interrupted();
|
||||
}
|
||||
else idle();
|
||||
} catch (RuntimeException exception) {
|
||||
log.error("定时任务 Worker 领取执行记录失败", exception);
|
||||
failureBackoff(++consecutiveFailures);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void execute(ClaimedSysJobExecution claim) {
|
||||
ActiveExecution activeExecution = new ActiveExecution(claim, Thread.currentThread());
|
||||
active.put(claim.token(), activeExecution);
|
||||
long startedAt = System.nanoTime();
|
||||
String terminalMetric = null;
|
||||
boolean metricStarted = false;
|
||||
try {
|
||||
SysJobExecutionContextHolder.set(toExecutionContext(claim.execution()));
|
||||
metricStarted = recordExecutionStarted(claim.execution().getId());
|
||||
terminalMetric = invokeAndFinish(claim, activeExecution);
|
||||
} finally {
|
||||
// 上下文和活动租约是 Worker 的正确性状态,必须先于可观测性收尾清理。
|
||||
// 否则指标实现一旦抛错,线程复用后会永久残留上下文,且旧租约会被持续续期。
|
||||
SysJobExecutionContextHolder.clear();
|
||||
synchronized (activeExecution) {
|
||||
active.remove(claim.token(), activeExecution);
|
||||
}
|
||||
if (metricStarted) {
|
||||
recordExecutionFinished(claim.execution().getId(), terminalMetric,
|
||||
System.nanoTime() - startedAt);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private String invokeAndFinish(ClaimedSysJobExecution claim,
|
||||
ActiveExecution activeExecution) {
|
||||
String terminalMetric = null;
|
||||
try {
|
||||
Object result = invoker.execute(toSnapshot(claim.execution()));
|
||||
String serialized = serializeResult(result, claim.execution().getId());
|
||||
if (finishOwned(claim, EnumJobResult.SUCCESS.getCode(), serialized, null)) {
|
||||
terminalMetric = "success";
|
||||
}
|
||||
} catch (SysJobCancelledException exception) {
|
||||
if (finishOwned(claim, EnumJobResult.CANCELLED.getCode(), null,
|
||||
exception.getMessage())) {
|
||||
terminalMetric = "cancelled";
|
||||
}
|
||||
} catch (SysJobInfrastructureException exception) {
|
||||
String message = ExceptionUtil.getRootCauseMessage(exception);
|
||||
try {
|
||||
if (!store.releaseForInfrastructureRetry(claim, message)) {
|
||||
activeExecution.abandon().set(true);
|
||||
metrics.recordLostLease();
|
||||
log.warn("基础设施故障记录释放被 fencing 拒绝: executionId={}",
|
||||
claim.execution().getId());
|
||||
}
|
||||
} catch (RuntimeException releaseFailure) {
|
||||
activeExecution.abandon().set(true);
|
||||
metrics.recordLostLease();
|
||||
log.error("基础设施故障记录释放失败,保留租约等待接管: executionId={}",
|
||||
claim.execution().getId(), releaseFailure);
|
||||
}
|
||||
log.warn("定时任务执行前基础设施故障,已安排重试: executionId={}, jobId={}",
|
||||
claim.execution().getId(), claim.execution().getJobId(), exception);
|
||||
} catch (Exception exception) {
|
||||
terminalMetric = handleBusinessFailure(claim, activeExecution, exception);
|
||||
} catch (Error error) {
|
||||
if (error instanceof VirtualMachineError || error instanceof ThreadDeath) {
|
||||
throw error;
|
||||
}
|
||||
// FutureTask 会吞掉 Error 并结束整个永久轮询任务;非 VM Error 必须
|
||||
// 记账后继续,让固定 workerCount 不因单个业务实现错误永久缩水。
|
||||
terminalMetric = handleBusinessFailure(claim, activeExecution, error);
|
||||
}
|
||||
return terminalMetric;
|
||||
}
|
||||
|
||||
private boolean recordExecutionStarted(java.math.BigInteger executionId) {
|
||||
try {
|
||||
metrics.executionStarted();
|
||||
return true;
|
||||
} catch (RuntimeException exception) {
|
||||
log.warn("记录定时任务开始指标失败,不影响业务执行: executionId={}",
|
||||
executionId, exception);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
private void recordExecutionFinished(java.math.BigInteger executionId,
|
||||
String status, long durationNanos) {
|
||||
try {
|
||||
metrics.executionFinished(status, durationNanos);
|
||||
} catch (RuntimeException exception) {
|
||||
log.warn("记录定时任务完成指标失败,不影响业务终态: executionId={}",
|
||||
executionId, exception);
|
||||
}
|
||||
}
|
||||
|
||||
private String handleBusinessFailure(ClaimedSysJobExecution claim,
|
||||
ActiveExecution activeExecution,
|
||||
Throwable failure) {
|
||||
if (activeExecution.abandon().get()) {
|
||||
log.warn("定时任务因失租或关闭中断,保留租约等待其他节点接管: "
|
||||
+ "executionId={}, jobId={}",
|
||||
claim.execution().getId(), claim.execution().getJobId(), failure);
|
||||
} else {
|
||||
log.error("定时任务业务执行失败: executionId={}, jobId={}",
|
||||
claim.execution().getId(), claim.execution().getJobId(), failure);
|
||||
String message = ExceptionUtil.getRootCauseMessage(failure);
|
||||
if (finishOwned(claim, EnumJobResult.FAIL.getCode(), null, message)) {
|
||||
return "failure";
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
private void heartbeatAll() {
|
||||
for (ActiveExecution execution : active.values()) {
|
||||
if (execution.abandon().get()) continue;
|
||||
try {
|
||||
if (!store.heartbeat(execution.claim())) {
|
||||
if (!abandonCurrent(execution)) continue;
|
||||
log.warn("定时任务已失去租约,中断旧 Worker: executionId={}",
|
||||
execution.claim().execution().getId());
|
||||
metrics.recordLostLease();
|
||||
}
|
||||
} catch (RuntimeException exception) {
|
||||
if (!abandonCurrent(execution)) continue;
|
||||
log.error("定时任务续租失败,中断执行以降低重复副作用风险: executionId={}",
|
||||
execution.claim().execution().getId(), exception);
|
||||
metrics.recordLostLease();
|
||||
}
|
||||
}
|
||||
refreshQueueMetrics();
|
||||
}
|
||||
|
||||
/**
|
||||
* 仅当执行仍是当前活动项时中断其线程。
|
||||
*
|
||||
* <p>Worker 线程会被执行池复用;这里必须与 {@link #execute} 的 finally
|
||||
* 使用同一监视器,避免旧心跳在活动项移除后误伤同一线程上的下一任务。</p>
|
||||
*/
|
||||
private boolean abandonCurrent(ActiveExecution execution) {
|
||||
synchronized (execution) {
|
||||
if (!active.remove(execution.claim().token(), execution)) return false;
|
||||
execution.abandon().set(true);
|
||||
execution.thread().interrupt();
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
private void idle() {
|
||||
LockSupport.parkNanos(properties.getPollInterval().toNanos());
|
||||
}
|
||||
|
||||
private void failureBackoff(int consecutiveFailures) {
|
||||
long base = properties.getPollInterval().toNanos();
|
||||
int shift = Math.min(6, Math.max(0, consecutiveFailures - 1));
|
||||
long capped = Math.min(TimeUnit.SECONDS.toNanos(30), base << shift);
|
||||
long jitter = ThreadLocalRandom.current().nextLong(Math.max(1L, base));
|
||||
LockSupport.parkNanos(Math.min(TimeUnit.SECONDS.toNanos(30), capped + jitter));
|
||||
}
|
||||
|
||||
private static SysJob toSnapshot(SysJobLog execution) {
|
||||
SysJob job = new SysJob();
|
||||
job.setId(execution.getJobId());
|
||||
job.setTenantId(execution.getTenantId());
|
||||
job.setDeptId(execution.getDeptId());
|
||||
job.setJobName(execution.getJobName());
|
||||
job.setJobType(execution.getJobType());
|
||||
job.setJobParams(execution.getJobParams());
|
||||
job.setOptions(execution.getJobOptions());
|
||||
job.setAllowConcurrent(execution.getAllowConcurrent());
|
||||
return job;
|
||||
}
|
||||
|
||||
private boolean finishOwned(ClaimedSysJobExecution claim, int status,
|
||||
String result, String error) {
|
||||
try {
|
||||
if (store.finish(claim, status, result, error)) return true;
|
||||
log.warn("定时任务终态提交被 fencing 拒绝: executionId={}, status={}",
|
||||
claim.execution().getId(), status);
|
||||
} catch (RuntimeException exception) {
|
||||
log.error("定时任务终态提交失败,保留租约等待恢复: executionId={}, status={}",
|
||||
claim.execution().getId(), status, exception);
|
||||
}
|
||||
metrics.recordLostLease();
|
||||
return false;
|
||||
}
|
||||
|
||||
private static String serializeResult(Object result, java.math.BigInteger executionId) {
|
||||
if (result == null) return null;
|
||||
try {
|
||||
return JSON.toJSONString(result);
|
||||
} catch (RuntimeException exception) {
|
||||
log.warn("定时任务结果无法序列化,仅保留成功状态: executionId={}",
|
||||
executionId, exception);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
private void refreshQueueMetrics() {
|
||||
try {
|
||||
SysJobQueueSnapshot snapshot = store.queueSnapshot();
|
||||
metrics.updateQueue(snapshot.pending(), snapshot.running(),
|
||||
snapshot.oldestBacklogMillis());
|
||||
} catch (RuntimeException exception) {
|
||||
log.warn("刷新定时任务队列指标失败", exception);
|
||||
}
|
||||
}
|
||||
|
||||
private static SysJobExecutionContext toExecutionContext(SysJobLog execution) {
|
||||
return new SysJobExecutionContext(
|
||||
execution.getId(),
|
||||
execution.getExecutionKey(),
|
||||
execution.getTenantId(),
|
||||
execution.getAttemptCount() == null ? 1 : execution.getAttemptCount() + 1,
|
||||
execution.getTriggerSource(),
|
||||
execution.getInvocationId(),
|
||||
execution.getScheduledFireTime() == null
|
||||
? null : execution.getScheduledFireTime().toInstant());
|
||||
}
|
||||
|
||||
@Override
|
||||
public synchronized void stop() {
|
||||
if (!running.compareAndSet(true, false)) return;
|
||||
workers.shutdown();
|
||||
try {
|
||||
if (!workers.awaitTermination(properties.getShutdownWaitTimeout().toMillis(),
|
||||
TimeUnit.MILLISECONDS)) {
|
||||
abandonActiveExecutions();
|
||||
workers.shutdownNow();
|
||||
awaitForcedWorkerShutdown();
|
||||
}
|
||||
} catch (InterruptedException exception) {
|
||||
Thread.currentThread().interrupt();
|
||||
abandonActiveExecutions();
|
||||
workers.shutdownNow();
|
||||
awaitForcedWorkerShutdown();
|
||||
} finally {
|
||||
heartbeat.shutdownNow();
|
||||
}
|
||||
log.info("定时任务 Worker 已停止: owner={}", owner);
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean isRunning() {
|
||||
return running.get();
|
||||
}
|
||||
|
||||
@Override
|
||||
public int getPhase() {
|
||||
return Integer.MAX_VALUE - 100;
|
||||
}
|
||||
|
||||
private static String buildOwner() {
|
||||
String runtime = ManagementFactory.getRuntimeMXBean().getName();
|
||||
String value = runtime + '-' + Integer.toHexString(System.identityHashCode(SysJobExecutionWorker.class));
|
||||
return value.substring(0, Math.min(190, value.length()));
|
||||
}
|
||||
|
||||
private void abandonActiveExecutions() {
|
||||
active.values().forEach(execution -> execution.abandon().set(true));
|
||||
}
|
||||
|
||||
private void awaitForcedWorkerShutdown() {
|
||||
try {
|
||||
if (!workers.awaitTermination(
|
||||
Math.min(1_000L, properties.getShutdownWaitTimeout().toMillis()),
|
||||
TimeUnit.MILLISECONDS)) {
|
||||
log.warn("部分定时任务业务代码忽略中断,将由守护线程退出和租约接管收口: owner={}", owner);
|
||||
}
|
||||
} catch (InterruptedException exception) {
|
||||
Thread.currentThread().interrupt();
|
||||
}
|
||||
}
|
||||
|
||||
private record ActiveExecution(ClaimedSysJobExecution claim, Thread thread,
|
||||
AtomicBoolean abandon) {
|
||||
private ActiveExecution(ClaimedSysJobExecution claim, Thread thread) {
|
||||
this(claim, thread, new AtomicBoolean());
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,11 @@
|
||||
package tech.easyflow.job.execution;
|
||||
|
||||
/**
|
||||
* 业务副作用开始前发生的可恢复基础设施故障。
|
||||
*/
|
||||
public class SysJobInfrastructureException extends RuntimeException {
|
||||
|
||||
public SysJobInfrastructureException(String message, Throwable cause) {
|
||||
super(message, cause);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,195 @@
|
||||
package tech.easyflow.job.execution;
|
||||
|
||||
import cn.hutool.core.util.ArrayUtil;
|
||||
import cn.hutool.core.util.StrUtil;
|
||||
import com.mybatisflex.core.tenant.TenantManager;
|
||||
import org.springframework.aop.support.AopUtils;
|
||||
import org.springframework.context.ApplicationContext;
|
||||
import org.springframework.dao.DataAccessException;
|
||||
import org.springframework.dao.DataAccessResourceFailureException;
|
||||
import org.springframework.dao.RecoverableDataAccessException;
|
||||
import org.springframework.dao.TransientDataAccessException;
|
||||
import org.springframework.stereotype.Component;
|
||||
import tech.easyflow.common.constant.enums.EnumJobStatus;
|
||||
import tech.easyflow.common.constant.enums.EnumJobType;
|
||||
import tech.easyflow.job.entity.SysJob;
|
||||
import tech.easyflow.job.job.JobConstant;
|
||||
import tech.easyflow.job.mapper.SysJobMapper;
|
||||
import tech.easyflow.job.service.WorkflowJobExecutionService;
|
||||
import tech.easyflow.system.entity.SysAccount;
|
||||
|
||||
import java.lang.reflect.InvocationTargetException;
|
||||
import java.lang.reflect.Method;
|
||||
import java.lang.reflect.Modifier;
|
||||
import java.util.Map;
|
||||
import java.util.Objects;
|
||||
|
||||
/** 在 Worker 线程中执行一种 EasyFlow 业务任务。 */
|
||||
@Component
|
||||
public class SysJobInvoker {
|
||||
|
||||
private final SysJobMapper jobMapper;
|
||||
private final ApplicationContext applicationContext;
|
||||
private final WorkflowJobExecutionService workflowExecutionService;
|
||||
private final SysJobOwnerValidator ownerValidator;
|
||||
|
||||
public SysJobInvoker(SysJobMapper jobMapper,
|
||||
ApplicationContext applicationContext,
|
||||
WorkflowJobExecutionService workflowExecutionService,
|
||||
SysJobOwnerValidator ownerValidator) {
|
||||
this.jobMapper = jobMapper;
|
||||
this.applicationContext = applicationContext;
|
||||
this.workflowExecutionService = workflowExecutionService;
|
||||
this.ownerValidator = ownerValidator;
|
||||
}
|
||||
|
||||
public Object execute(SysJob snapshot) throws Exception {
|
||||
SysJob current;
|
||||
SysAccount owner;
|
||||
try {
|
||||
current = TenantManager.withoutTenantCondition(
|
||||
() -> jobMapper.selectOneById(snapshot.getId()));
|
||||
validateCurrent(snapshot, current);
|
||||
owner = ownerValidator.requireAvailableOwner(current);
|
||||
} catch (DataAccessException exception) {
|
||||
throw classifyPreparationFailure("准备定时任务执行上下文失败", exception);
|
||||
}
|
||||
if (Integer.valueOf(EnumJobType.TINY_FLOW.getCode()).equals(snapshot.getJobType())) {
|
||||
return workflowExecutionService.execute(snapshot, current, owner);
|
||||
}
|
||||
if (Integer.valueOf(EnumJobType.SPRING_BEAN.getCode()).equals(snapshot.getJobType())) {
|
||||
return executeSpringBean(snapshot);
|
||||
}
|
||||
if (Integer.valueOf(EnumJobType.JAVA_CLASS.getCode()).equals(snapshot.getJobType())) {
|
||||
return executeJavaClass(snapshot);
|
||||
}
|
||||
throw new IllegalArgumentException("不支持的定时任务类型: " + snapshot.getJobType());
|
||||
}
|
||||
|
||||
private static RuntimeException classifyPreparationFailure(
|
||||
String message, DataAccessException exception) {
|
||||
if (exception instanceof TransientDataAccessException
|
||||
|| exception instanceof RecoverableDataAccessException
|
||||
|| exception instanceof DataAccessResourceFailureException) {
|
||||
return new SysJobInfrastructureException(message, exception);
|
||||
}
|
||||
return exception;
|
||||
}
|
||||
|
||||
private void validateCurrent(SysJob snapshot, SysJob current) {
|
||||
if (current == null) {
|
||||
throw new SysJobCancelledException("定时任务已删除,id=" + snapshot.getId());
|
||||
}
|
||||
if (!Objects.equals(snapshot.getTenantId(), current.getTenantId())) {
|
||||
throw new SysJobCancelledException("定时任务租户已变化,id=" + snapshot.getId());
|
||||
}
|
||||
if (!Integer.valueOf(EnumJobStatus.RUNNING.getCode()).equals(current.getStatus())) {
|
||||
throw new SysJobCancelledException("定时任务已停止,id=" + snapshot.getId());
|
||||
}
|
||||
if (!Objects.equals(snapshot.getJobType(), current.getJobType())) {
|
||||
throw new SysJobCancelledException("定时任务类型已变化,id=" + snapshot.getId());
|
||||
}
|
||||
}
|
||||
|
||||
private Object executeSpringBean(SysJob job) throws Exception {
|
||||
String expression = requiredExpression(job, JobConstant.BEAN_METHOD_KEY);
|
||||
MethodCall call = parse(expression);
|
||||
Object bean = applicationContext.getBean(call.owner());
|
||||
Class<?> targetClass = AopUtils.getTargetClass(bean);
|
||||
Method targetMethod = publicMethod(targetClass, call, false);
|
||||
Method invocableMethod = AopUtils.selectInvocableMethod(targetMethod, bean.getClass());
|
||||
return invoke(bean, invocableMethod, call.arguments());
|
||||
}
|
||||
|
||||
private Object executeJavaClass(SysJob job) throws Exception {
|
||||
String expression = requiredExpression(job, JobConstant.JAVA_METHOD_KEY);
|
||||
MethodCall call = parse(expression);
|
||||
Class<?> type = Class.forName(call.owner());
|
||||
Object target = type.getDeclaredConstructor().newInstance();
|
||||
return invoke(target, publicMethod(type, call, true), call.arguments());
|
||||
}
|
||||
|
||||
private static Method publicMethod(Class<?> type, MethodCall call,
|
||||
boolean declaredOnly) throws NoSuchMethodException {
|
||||
Method method = declaredOnly
|
||||
? type.getDeclaredMethod(call.method(), call.parameterTypes())
|
||||
: type.getMethod(call.method(), call.parameterTypes());
|
||||
if (!Modifier.isPublic(method.getModifiers())) {
|
||||
throw new NoSuchMethodException("仅允许调用 public 方法: "
|
||||
+ type.getName() + '.' + call.method());
|
||||
}
|
||||
return method;
|
||||
}
|
||||
|
||||
private static Object invoke(Object target, Method method, Object[] arguments)
|
||||
throws Exception {
|
||||
try {
|
||||
return method.invoke(target, arguments);
|
||||
} catch (InvocationTargetException exception) {
|
||||
Throwable cause = exception.getTargetException();
|
||||
if (cause instanceof Exception checked) throw checked;
|
||||
if (cause instanceof Error error) throw error;
|
||||
throw exception;
|
||||
}
|
||||
}
|
||||
|
||||
private static String requiredExpression(SysJob job, String key) {
|
||||
Map<String, Object> params = job.getJobParams();
|
||||
Object value = params == null ? null : params.get(key);
|
||||
if (value == null || !StrUtil.isNotBlank(value.toString())) {
|
||||
throw new IllegalArgumentException("定时任务缺少执行表达式: " + key);
|
||||
}
|
||||
return value.toString().trim();
|
||||
}
|
||||
|
||||
private static MethodCall parse(String expression) {
|
||||
String before = StrUtil.subBefore(expression, "(", false);
|
||||
int separator = before.lastIndexOf('.');
|
||||
if (separator < 1 || !expression.endsWith(")")) {
|
||||
throw new IllegalArgumentException("非法方法表达式: " + expression);
|
||||
}
|
||||
String owner = before.substring(0, separator);
|
||||
String method = before.substring(separator + 1);
|
||||
String params = StrUtil.subBetween(expression, "(", ")");
|
||||
Object[] parsed = parseParams(params);
|
||||
return new MethodCall(owner, method, (Class<?>[]) parsed[0], (Object[]) parsed[1]);
|
||||
}
|
||||
|
||||
private static Object[] parseParams(String params) {
|
||||
if (StrUtil.isEmpty(params)) return new Object[]{new Class<?>[0], new Object[0]};
|
||||
String[] values = params.split(",");
|
||||
Object[] arguments = new Object[values.length];
|
||||
Class<?>[] types = new Class<?>[values.length];
|
||||
for (int i = 0; i < values.length; i++) {
|
||||
String value = values[i].trim();
|
||||
if (value.startsWith("\"") && value.endsWith("\"") && value.length() >= 2) {
|
||||
arguments[i] = value.substring(1, value.length() - 1);
|
||||
types[i] = String.class;
|
||||
} else if ("true".equals(value) || "false".equals(value)) {
|
||||
arguments[i] = Boolean.valueOf(value);
|
||||
types[i] = Boolean.class;
|
||||
} else if (value.endsWith("L")) {
|
||||
arguments[i] = Long.valueOf(value.substring(0, value.length() - 1));
|
||||
types[i] = Long.class;
|
||||
} else if (value.endsWith("D")) {
|
||||
arguments[i] = Double.valueOf(value.substring(0, value.length() - 1));
|
||||
types[i] = Double.class;
|
||||
} else if (value.endsWith("F")) {
|
||||
arguments[i] = Float.valueOf(value.substring(0, value.length() - 1));
|
||||
types[i] = Float.class;
|
||||
} else {
|
||||
arguments[i] = Integer.valueOf(value);
|
||||
types[i] = Integer.class;
|
||||
}
|
||||
}
|
||||
return new Object[]{types, arguments};
|
||||
}
|
||||
|
||||
private record MethodCall(String owner, String method,
|
||||
Class<?>[] parameterTypes, Object[] arguments) {
|
||||
private MethodCall {
|
||||
parameterTypes = ArrayUtil.isEmpty(parameterTypes) ? new Class<?>[0] : parameterTypes;
|
||||
arguments = ArrayUtil.isEmpty(arguments) ? new Object[0] : arguments;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,47 @@
|
||||
package tech.easyflow.job.execution;
|
||||
|
||||
import com.mybatisflex.core.tenant.TenantManager;
|
||||
import org.springframework.stereotype.Component;
|
||||
import tech.easyflow.common.constant.enums.EnumDataStatus;
|
||||
import tech.easyflow.job.entity.SysJob;
|
||||
import tech.easyflow.system.entity.SysAccount;
|
||||
import tech.easyflow.system.service.SysAccountService;
|
||||
|
||||
import java.math.BigInteger;
|
||||
import java.util.Objects;
|
||||
|
||||
/** 定时任务服务端归属账号校验器。 */
|
||||
@Component
|
||||
public class SysJobOwnerValidator {
|
||||
|
||||
private final SysAccountService accountService;
|
||||
|
||||
public SysJobOwnerValidator(SysAccountService accountService) {
|
||||
this.accountService = accountService;
|
||||
}
|
||||
|
||||
/**
|
||||
* 按数据库当前状态校验任务归属账号。
|
||||
*
|
||||
* <p>调度 Worker 没有登录租户上下文,因此关闭 ORM 租户条件读取账号,
|
||||
* 再显式比较任务与账号租户,避免跨租户账号被用于高权限任务执行。</p>
|
||||
*/
|
||||
public SysAccount requireAvailableOwner(SysJob job) {
|
||||
BigInteger accountId = job.getCreatedBy();
|
||||
if (accountId == null) {
|
||||
throw new IllegalStateException("定时任务缺少服务端归属账号,id=" + job.getId());
|
||||
}
|
||||
SysAccount account = TenantManager.withoutTenantCondition(
|
||||
() -> accountService.getById(accountId));
|
||||
if (account == null) {
|
||||
throw new IllegalStateException("定时任务归属账号不存在,id=" + accountId);
|
||||
}
|
||||
if (!EnumDataStatus.AVAILABLE.getCode().equals(account.getStatus())) {
|
||||
throw new IllegalStateException("定时任务归属账号未启用,id=" + accountId);
|
||||
}
|
||||
if (!Objects.equals(job.getTenantId(), account.getTenantId())) {
|
||||
throw new IllegalStateException("定时任务与归属账号租户不一致,id=" + job.getId());
|
||||
}
|
||||
return account;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,5 @@
|
||||
package tech.easyflow.job.execution;
|
||||
|
||||
/** 当前数据库执行队列的全局状态。 */
|
||||
public record SysJobQueueSnapshot(long pending, long running, long oldestBacklogMillis) {
|
||||
}
|
||||
@@ -1,72 +0,0 @@
|
||||
package tech.easyflow.job.job;
|
||||
|
||||
import cn.hutool.core.exceptions.ExceptionUtil;
|
||||
import com.alibaba.fastjson.JSON;
|
||||
import org.quartz.Job;
|
||||
import org.quartz.JobExecutionContext;
|
||||
import org.quartz.JobExecutionException;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import tech.easyflow.common.constant.enums.EnumJobExecStatus;
|
||||
import tech.easyflow.common.util.SpringContextUtil;
|
||||
import tech.easyflow.job.entity.SysJob;
|
||||
import tech.easyflow.job.entity.SysJobLog;
|
||||
import tech.easyflow.job.service.SysJobLogService;
|
||||
|
||||
import java.util.Date;
|
||||
|
||||
public abstract class BaseQuartzJob implements Job {
|
||||
|
||||
protected Logger log = LoggerFactory.getLogger(getClass());
|
||||
|
||||
private final static ThreadLocal<Date> TIME_RECORD = new ThreadLocal<>();
|
||||
|
||||
@Override
|
||||
public void execute(JobExecutionContext ctx) throws JobExecutionException {
|
||||
|
||||
SysJob job = (SysJob) ctx.getMergedJobDataMap().get(JobConstant.JOB_MAP_BEAN_NAME);
|
||||
|
||||
try {
|
||||
beforeExecute(ctx, job);
|
||||
Object result = doExecute(ctx, job);
|
||||
afterExecute(ctx, job, result, null);
|
||||
} catch (Exception e) {
|
||||
log.error("quartz 任务执行报错:", e);
|
||||
afterExecute(ctx, job, null, e);
|
||||
}
|
||||
}
|
||||
|
||||
protected void beforeExecute(JobExecutionContext ctx, SysJob job) {
|
||||
TIME_RECORD.set(new Date());
|
||||
}
|
||||
|
||||
protected void afterExecute(JobExecutionContext ctx, SysJob job, Object result, Exception e) {
|
||||
Date startTime = TIME_RECORD.get();
|
||||
TIME_RECORD.remove();
|
||||
Date endTime = new Date();
|
||||
|
||||
SysJobLog sysJobLog = new SysJobLog();
|
||||
sysJobLog.setJobId(job.getId());
|
||||
sysJobLog.setJobName(job.getJobName());
|
||||
sysJobLog.setStatus(EnumJobExecStatus.SUCCESS.getCode());
|
||||
sysJobLog.setJobParams(job.getJobParams());
|
||||
if (result != null) {
|
||||
sysJobLog.setJobResult(JSON.toJSONString(result));
|
||||
}
|
||||
if (e != null) {
|
||||
String message = ExceptionUtil.getRootCauseMessage(e);
|
||||
if (message.length() > 1000) {
|
||||
message = message.substring(0, 1000);
|
||||
}
|
||||
sysJobLog.setErrorInfo(message);
|
||||
sysJobLog.setStatus(EnumJobExecStatus.FAIL.getCode());
|
||||
}
|
||||
sysJobLog.setStartTime(startTime);
|
||||
sysJobLog.setEndTime(endTime);
|
||||
sysJobLog.setCreated(new Date());
|
||||
SysJobLogService service = SpringContextUtil.getBean(SysJobLogService.class);
|
||||
service.save(sysJobLog);
|
||||
}
|
||||
|
||||
protected abstract Object doExecute(JobExecutionContext ctx, SysJob job) throws Exception;
|
||||
}
|
||||
@@ -2,12 +2,15 @@ package tech.easyflow.job.job;
|
||||
|
||||
public interface JobConstant {
|
||||
|
||||
String JOB_GROUP = "easyflow";
|
||||
String JOB_MAP_BEAN_NAME = "jobMapBean";
|
||||
|
||||
String BEAN_METHOD_KEY = "beanMethod";
|
||||
String JAVA_METHOD_KEY = "javaMethod";
|
||||
String WORKFLOW_KEY = "workflowId";
|
||||
String WORKFLOW_PARAMS_KEY = "workflowParams";
|
||||
|
||||
/** 注入工作流运行参数的定时任务业务幂等键。 */
|
||||
String EXECUTION_KEY = "_easyflowJobExecutionKey";
|
||||
|
||||
/** Quartz JobData 中用于隔离旧定义触发的任务代际。 */
|
||||
String SCHEDULE_GENERATION = "scheduleGeneration";
|
||||
String ACCOUNT_ID = "accountId";
|
||||
}
|
||||
|
||||
@@ -1,16 +0,0 @@
|
||||
package tech.easyflow.job.job;
|
||||
|
||||
import org.quartz.JobExecutionContext;
|
||||
import tech.easyflow.job.entity.SysJob;
|
||||
import tech.easyflow.job.util.JobUtil;
|
||||
|
||||
/**
|
||||
* 可并发执行
|
||||
*/
|
||||
public class QuartzJob extends BaseQuartzJob {
|
||||
|
||||
@Override
|
||||
protected Object doExecute(JobExecutionContext ctx, SysJob job) throws Exception {
|
||||
return JobUtil.execute(job);
|
||||
}
|
||||
}
|
||||
@@ -1,18 +0,0 @@
|
||||
package tech.easyflow.job.job;
|
||||
|
||||
import org.quartz.DisallowConcurrentExecution;
|
||||
import org.quartz.JobExecutionContext;
|
||||
import tech.easyflow.job.entity.SysJob;
|
||||
import tech.easyflow.job.util.JobUtil;
|
||||
|
||||
/**
|
||||
* 禁止并发执行
|
||||
*/
|
||||
@DisallowConcurrentExecution
|
||||
public class QuartzJobNoConcurrent extends BaseQuartzJob {
|
||||
|
||||
@Override
|
||||
protected Object doExecute(JobExecutionContext ctx, SysJob job) throws Exception {
|
||||
return JobUtil.execute(job);
|
||||
}
|
||||
}
|
||||
@@ -1,8 +1,14 @@
|
||||
package tech.easyflow.job.mapper;
|
||||
|
||||
import com.mybatisflex.core.BaseMapper;
|
||||
import org.apache.ibatis.annotations.Delete;
|
||||
import org.apache.ibatis.annotations.Param;
|
||||
import org.apache.ibatis.annotations.Select;
|
||||
import org.apache.ibatis.annotations.Update;
|
||||
import tech.easyflow.job.entity.SysJobLog;
|
||||
|
||||
import java.math.BigInteger;
|
||||
|
||||
/**
|
||||
* 系统任务日志 映射层。
|
||||
*
|
||||
@@ -11,4 +17,158 @@ import tech.easyflow.job.entity.SysJobLog;
|
||||
*/
|
||||
public interface SysJobLogMapper extends BaseMapper<SysJobLog> {
|
||||
|
||||
@Select("SELECT id FROM tb_sys_job_log WHERE execution_key=#{executionKey} LIMIT 1")
|
||||
BigInteger selectIdByExecutionKey(@Param("executionKey") String executionKey);
|
||||
|
||||
@Select("SELECT id FROM tb_sys_job_log WHERE job_id=#{jobId} AND status=2 "
|
||||
+ "ORDER BY id FOR UPDATE")
|
||||
java.util.List<BigInteger> selectPendingIdsByJobIdForUpdate(
|
||||
@Param("jobId") BigInteger jobId);
|
||||
|
||||
@Select("SELECT id FROM tb_sys_job_log WHERE status=3 "
|
||||
+ "AND lease_until<=CURRENT_TIMESTAMP(3) "
|
||||
+ "ORDER BY lease_until, id LIMIT 1 FOR UPDATE SKIP LOCKED")
|
||||
BigInteger selectExpiredClaimCandidateId();
|
||||
|
||||
/**
|
||||
* 无锁探测是否存在已过期执行。过期范围为空时若直接使用 {@code FOR UPDATE},
|
||||
* MySQL 仍可能锁住状态索引的末端间隙,随后 PENDING -> RUNNING 的索引迁移会在
|
||||
* 多 Worker 间形成 insert-intention 死锁。
|
||||
*/
|
||||
@Select("SELECT id FROM tb_sys_job_log WHERE status=3 "
|
||||
+ "AND lease_until<=CURRENT_TIMESTAMP(3) "
|
||||
+ "ORDER BY lease_until, id LIMIT 1")
|
||||
BigInteger selectExpiredClaimCandidateIdWithoutLock();
|
||||
|
||||
@Select("SELECT id FROM tb_sys_job_log WHERE status=2 "
|
||||
+ "AND next_retry_time<=CURRENT_TIMESTAMP(3) "
|
||||
+ "ORDER BY next_retry_time, id LIMIT 1 FOR UPDATE SKIP LOCKED")
|
||||
BigInteger selectPendingClaimCandidateId();
|
||||
|
||||
/**
|
||||
* 先用最小投影锁定候选行,再通过 BaseMapper 实体 ResultMap 读取快照。
|
||||
* 这样既保留 MySQL {@code SKIP LOCKED},又避免注解式 {@code SELECT *}
|
||||
* 丢失 {@code job_id}、JSON 快照等下划线字段映射。
|
||||
*/
|
||||
default SysJobLog selectExpiredClaimCandidate() {
|
||||
BigInteger id = selectExpiredClaimCandidateId();
|
||||
return id == null ? null : selectOneById(id);
|
||||
}
|
||||
|
||||
/** 同 {@link #selectExpiredClaimCandidate()}。 */
|
||||
default SysJobLog selectPendingClaimCandidate() {
|
||||
BigInteger id = selectPendingClaimCandidateId();
|
||||
return id == null ? null : selectOneById(id);
|
||||
}
|
||||
|
||||
@Select("SELECT id FROM tb_sys_job_log WHERE job_id=#{jobId} AND id<>#{excludedId} "
|
||||
+ "AND status=3 AND lease_until>CURRENT_TIMESTAMP(3) ORDER BY id LIMIT 1 FOR UPDATE")
|
||||
BigInteger selectOtherActiveForUpdate(@Param("jobId") BigInteger jobId,
|
||||
@Param("excludedId") BigInteger excludedId);
|
||||
|
||||
@Update("UPDATE tb_sys_job_log SET status=3, lease_owner=#{owner}, "
|
||||
+ "execution_token=#{token}, "
|
||||
+ "lease_until=TIMESTAMPADD(MICROSECOND, #{leaseMicros}, CURRENT_TIMESTAMP(3)), "
|
||||
+ "heartbeat_time=CURRENT_TIMESTAMP(3), "
|
||||
+ "attempt_count=attempt_count+1, next_retry_time=NULL, "
|
||||
+ "start_time=COALESCE(start_time, CURRENT_TIMESTAMP(3)), end_time=NULL, "
|
||||
+ "error_info=NULL, version=version+1 "
|
||||
+ "WHERE id=#{id} AND status=2 AND next_retry_time<=CURRENT_TIMESTAMP(3)")
|
||||
int claimPending(@Param("id") BigInteger id,
|
||||
@Param("owner") String owner,
|
||||
@Param("token") String token,
|
||||
@Param("leaseMicros") long leaseMicros);
|
||||
|
||||
@Update("UPDATE tb_sys_job_log SET lease_owner=#{owner}, execution_token=#{token}, "
|
||||
+ "lease_until=TIMESTAMPADD(MICROSECOND, #{leaseMicros}, CURRENT_TIMESTAMP(3)), "
|
||||
+ "heartbeat_time=CURRENT_TIMESTAMP(3), attempt_count=attempt_count+1, "
|
||||
+ "next_retry_time=NULL, error_info=NULL, version=version+1 "
|
||||
+ "WHERE id=#{id} AND status=3 AND lease_until<=CURRENT_TIMESTAMP(3)")
|
||||
int claimExpired(@Param("id") BigInteger id,
|
||||
@Param("owner") String owner,
|
||||
@Param("token") String token,
|
||||
@Param("leaseMicros") long leaseMicros);
|
||||
|
||||
@Update("UPDATE tb_sys_job_log SET "
|
||||
+ "next_retry_time=TIMESTAMPADD(MICROSECOND, #{retryMicros}, CURRENT_TIMESTAMP(3)) "
|
||||
+ "WHERE id=#{id} AND status=2")
|
||||
int deferPending(@Param("id") BigInteger id,
|
||||
@Param("retryMicros") long retryMicros);
|
||||
|
||||
@Update("UPDATE tb_sys_job_log SET status=2, lease_owner=NULL, execution_token=NULL, "
|
||||
+ "lease_until=NULL, heartbeat_time=CURRENT_TIMESTAMP(3), "
|
||||
+ "next_retry_time=TIMESTAMPADD(MICROSECOND, #{retryMicros}, CURRENT_TIMESTAMP(3)), "
|
||||
+ "version=version+1 WHERE id=#{id} AND status=3 "
|
||||
+ "AND lease_until<=CURRENT_TIMESTAMP(3)")
|
||||
int releaseExpiredForRetry(@Param("id") BigInteger id,
|
||||
@Param("retryMicros") long retryMicros);
|
||||
|
||||
@Update("UPDATE tb_sys_job_log SET status=2, lease_owner=NULL, execution_token=NULL, "
|
||||
+ "lease_until=NULL, heartbeat_time=CURRENT_TIMESTAMP(3), "
|
||||
+ "next_retry_time=TIMESTAMPADD(MICROSECOND, #{retryMicros}, CURRENT_TIMESTAMP(3)), "
|
||||
+ "error_info=#{errorInfo}, version=version+1 WHERE id=#{id} AND status=3 "
|
||||
+ "AND lease_owner=#{owner} AND execution_token=#{token}")
|
||||
int releaseOwnedForRetry(@Param("id") BigInteger id,
|
||||
@Param("owner") String owner,
|
||||
@Param("token") String token,
|
||||
@Param("retryMicros") long retryMicros,
|
||||
@Param("errorInfo") String errorInfo);
|
||||
|
||||
@Update("UPDATE tb_sys_job_log SET "
|
||||
+ "lease_until=TIMESTAMPADD(MICROSECOND, #{leaseMicros}, CURRENT_TIMESTAMP(3)), "
|
||||
+ "heartbeat_time=CURRENT_TIMESTAMP(3) "
|
||||
+ "WHERE id=#{id} AND status=3 AND lease_owner=#{owner} AND execution_token=#{token}")
|
||||
int renewLease(@Param("id") BigInteger id,
|
||||
@Param("owner") String owner,
|
||||
@Param("token") String token,
|
||||
@Param("leaseMicros") long leaseMicros);
|
||||
|
||||
@Update("UPDATE tb_sys_job_log SET status=#{status}, job_result=#{jobResult}, "
|
||||
+ "error_info=#{errorInfo}, end_time=CURRENT_TIMESTAMP(3), lease_owner=NULL, "
|
||||
+ "execution_token=NULL, lease_until=NULL, heartbeat_time=CURRENT_TIMESTAMP(3), "
|
||||
+ "version=version+1 "
|
||||
+ "WHERE id=#{id} AND status=3 AND lease_owner=#{owner} AND execution_token=#{token}")
|
||||
int finishOwned(@Param("id") BigInteger id,
|
||||
@Param("owner") String owner,
|
||||
@Param("token") String token,
|
||||
@Param("status") int status,
|
||||
@Param("jobResult") String jobResult,
|
||||
@Param("errorInfo") String errorInfo);
|
||||
|
||||
@Update("UPDATE tb_sys_job_log SET status=4, error_info=#{errorInfo}, "
|
||||
+ "end_time=CURRENT_TIMESTAMP(3), lease_owner=NULL, execution_token=NULL, "
|
||||
+ "lease_until=NULL, heartbeat_time=CURRENT_TIMESTAMP(3), version=version+1 "
|
||||
+ "WHERE id=#{id} AND status=3 AND lease_until<=CURRENT_TIMESTAMP(3)")
|
||||
int markExpiredDead(@Param("id") BigInteger id,
|
||||
@Param("errorInfo") String errorInfo);
|
||||
|
||||
@Update("UPDATE tb_sys_job_log SET status=4, error_info=#{errorInfo}, "
|
||||
+ "end_time=CURRENT_TIMESTAMP(3), next_retry_time=NULL, "
|
||||
+ "heartbeat_time=CURRENT_TIMESTAMP(3), version=version+1 "
|
||||
+ "WHERE id=#{id} AND status=2")
|
||||
int markPendingDead(@Param("id") BigInteger id,
|
||||
@Param("errorInfo") String errorInfo);
|
||||
|
||||
@Update("UPDATE tb_sys_job_log SET status=5, error_info=#{reason}, "
|
||||
+ "end_time=CURRENT_TIMESTAMP(3), "
|
||||
+ "next_retry_time=NULL, version=version+1 WHERE job_id=#{jobId} AND status=2")
|
||||
int cancelPendingByJobId(@Param("jobId") BigInteger jobId,
|
||||
@Param("reason") String reason);
|
||||
|
||||
@Select("SELECT COUNT(1) FROM tb_sys_job_log WHERE status=2")
|
||||
long countPending();
|
||||
|
||||
@Select("SELECT COUNT(1) FROM tb_sys_job_log WHERE status=3")
|
||||
long countRunning();
|
||||
|
||||
@Select("SELECT COALESCE(TIMESTAMPDIFF(MICROSECOND, MIN(scheduled_fire_time), "
|
||||
+ "CURRENT_TIMESTAMP(3)) DIV 1000, 0) FROM tb_sys_job_log WHERE status=2")
|
||||
long selectOldestPendingMillis();
|
||||
|
||||
@Select("SELECT COUNT(1) FROM tb_sys_job_log WHERE id=#{id} AND status IN (2,3)")
|
||||
int countActiveById(@Param("id") BigInteger id);
|
||||
|
||||
@Delete("DELETE FROM tb_sys_job_log WHERE id=#{id} AND status NOT IN (2,3)")
|
||||
int deleteTerminalById(@Param("id") BigInteger id);
|
||||
|
||||
}
|
||||
|
||||
@@ -1,8 +1,14 @@
|
||||
package tech.easyflow.job.mapper;
|
||||
|
||||
import com.mybatisflex.core.BaseMapper;
|
||||
import com.mybatisflex.core.query.QueryWrapper;
|
||||
import org.apache.ibatis.annotations.Param;
|
||||
import org.apache.ibatis.annotations.Update;
|
||||
import tech.easyflow.job.entity.SysJob;
|
||||
|
||||
import java.math.BigInteger;
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* 系统任务表 映射层。
|
||||
*
|
||||
@@ -11,4 +17,31 @@ import tech.easyflow.job.entity.SysJob;
|
||||
*/
|
||||
public interface SysJobMapper extends BaseMapper<SysJob> {
|
||||
|
||||
/**
|
||||
* 使用 BaseMapper 的实体 ResultMap 读取并锁定任务。
|
||||
*
|
||||
* <p>不能用注解式 {@code SELECT *}:新增下划线列时它不会复用
|
||||
* MyBatis-Flex 生成的实体映射,曾导致 {@code schedule_generation}
|
||||
* 在运行态被读取为 {@code null}。</p>
|
||||
*/
|
||||
default SysJob selectByIdForUpdate(BigInteger id) {
|
||||
return selectOneByQuery(QueryWrapper.create()
|
||||
.eq(SysJob::getId, id)
|
||||
.forUpdate());
|
||||
}
|
||||
|
||||
@Update("UPDATE tb_sys_job SET status=#{runningStatus}, "
|
||||
+ "schedule_generation=schedule_generation+1 "
|
||||
+ "WHERE id=#{id} AND status<>#{runningStatus}")
|
||||
int startNextGeneration(@Param("id") BigInteger id,
|
||||
@Param("runningStatus") int runningStatus);
|
||||
|
||||
/** 使用实体 ResultMap 做 keyset 分页,确保所有调度字段完整映射。 */
|
||||
default List<SysJob> selectReconciliationPage(BigInteger lastId, int limit) {
|
||||
return selectListByQuery(QueryWrapper.create()
|
||||
.gt(SysJob::getId, lastId)
|
||||
.orderBy(SysJob::getId, true)
|
||||
.limit(limit));
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -0,0 +1,82 @@
|
||||
package tech.easyflow.job.schedule;
|
||||
|
||||
import com.easyagents.scheduler.ConcurrencyPolicy;
|
||||
import com.easyagents.scheduler.CronSchedulePlan;
|
||||
import com.easyagents.scheduler.MisfirePolicy;
|
||||
import com.easyagents.scheduler.ScheduleDefinition;
|
||||
import com.easyagents.scheduler.ScheduleId;
|
||||
import com.easyagents.scheduler.ScheduleService;
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
import org.springframework.stereotype.Component;
|
||||
import tech.easyflow.common.constant.enums.EnumMisfirePolicy;
|
||||
import tech.easyflow.job.entity.SysJob;
|
||||
import tech.easyflow.job.execution.EasyFlowScheduleHandler;
|
||||
import tech.easyflow.job.job.JobConstant;
|
||||
|
||||
import java.time.ZoneId;
|
||||
import java.util.Map;
|
||||
import java.util.UUID;
|
||||
|
||||
/** EasyFlow 任务定义到 easy-agents 调度模型的唯一映射入口。 */
|
||||
@Component
|
||||
public class SysJobScheduleAdapter {
|
||||
|
||||
static final String NAMESPACE = "easyflow.job";
|
||||
|
||||
private final ScheduleService scheduleService;
|
||||
private final ZoneId zoneId;
|
||||
|
||||
public SysJobScheduleAdapter(ScheduleService scheduleService,
|
||||
@Value("${easyflow.job.timezone:Asia/Shanghai}") String zoneId) {
|
||||
this.scheduleService = scheduleService;
|
||||
this.zoneId = ZoneId.of(zoneId);
|
||||
}
|
||||
|
||||
public void replace(SysJob job) {
|
||||
scheduleService.replace(toDefinition(job));
|
||||
}
|
||||
|
||||
public void delete(java.math.BigInteger jobId) {
|
||||
scheduleService.delete(scheduleId(jobId));
|
||||
}
|
||||
|
||||
public String triggerNow(java.math.BigInteger jobId) {
|
||||
String invocationId = UUID.randomUUID().toString();
|
||||
scheduleService.triggerNow(scheduleId(jobId), invocationId, Map.of());
|
||||
return invocationId;
|
||||
}
|
||||
|
||||
ScheduleDefinition toDefinition(SysJob job) {
|
||||
Long generation = job.getScheduleGeneration();
|
||||
if (generation == null || generation < 0L) {
|
||||
throw new IllegalArgumentException("任务调度代际非法: " + generation);
|
||||
}
|
||||
return new ScheduleDefinition(
|
||||
scheduleId(job.getId()),
|
||||
EasyFlowScheduleHandler.CODE,
|
||||
new CronSchedulePlan(job.getCronExpression(), zoneId),
|
||||
toMisfirePolicy(job.getMisfirePolicy()),
|
||||
Integer.valueOf(1).equals(job.getAllowConcurrent())
|
||||
? ConcurrencyPolicy.ALLOW : ConcurrencyPolicy.DISALLOW,
|
||||
true,
|
||||
Map.of(
|
||||
"jobId", job.getId().toString(),
|
||||
JobConstant.SCHEDULE_GENERATION, generation.toString()),
|
||||
job.getJobName());
|
||||
}
|
||||
|
||||
private static MisfirePolicy toMisfirePolicy(Integer policy) {
|
||||
if (Integer.valueOf(EnumMisfirePolicy.FIRE_ONCE_NOW.getCode()).equals(policy)) {
|
||||
return MisfirePolicy.FIRE_ONCE_NOW;
|
||||
}
|
||||
if (Integer.valueOf(EnumMisfirePolicy.SKIP.getCode()).equals(policy)) {
|
||||
return MisfirePolicy.SKIP;
|
||||
}
|
||||
throw new IllegalArgumentException("不支持的 Misfire 策略: " + policy);
|
||||
}
|
||||
|
||||
private static ScheduleId scheduleId(java.math.BigInteger jobId) {
|
||||
if (jobId == null) throw new IllegalArgumentException("jobId must not be null");
|
||||
return new ScheduleId(NAMESPACE, jobId.toString());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,62 @@
|
||||
package tech.easyflow.job.schedule;
|
||||
|
||||
import com.mybatisflex.core.tenant.TenantManager;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.boot.context.event.ApplicationReadyEvent;
|
||||
import org.springframework.context.event.EventListener;
|
||||
import org.springframework.stereotype.Component;
|
||||
import tech.easyflow.job.config.SysJobExecutionProperties;
|
||||
import tech.easyflow.job.entity.SysJob;
|
||||
import tech.easyflow.job.mapper.SysJobMapper;
|
||||
import tech.easyflow.job.service.SysJobService;
|
||||
|
||||
import java.math.BigInteger;
|
||||
import java.util.List;
|
||||
|
||||
/** 启动时一次性修复业务定义与 Quartz 投影的跨事务漂移。 */
|
||||
@Component
|
||||
public class SysJobScheduleReconciler {
|
||||
|
||||
private static final Logger log = LoggerFactory.getLogger(SysJobScheduleReconciler.class);
|
||||
private static final int PAGE_SIZE = 200;
|
||||
|
||||
private final SysJobMapper mapper;
|
||||
private final SysJobService jobService;
|
||||
private final SysJobExecutionProperties properties;
|
||||
|
||||
public SysJobScheduleReconciler(SysJobMapper mapper,
|
||||
SysJobService jobService,
|
||||
SysJobExecutionProperties properties) {
|
||||
this.mapper = mapper;
|
||||
this.jobService = jobService;
|
||||
this.properties = properties;
|
||||
}
|
||||
|
||||
@EventListener(ApplicationReadyEvent.class)
|
||||
public void reconcile() {
|
||||
if (!properties.isEnabled()) return;
|
||||
BigInteger lastId = BigInteger.ZERO;
|
||||
int reconciled = 0;
|
||||
int failed = 0;
|
||||
while (true) {
|
||||
BigInteger pageStartId = lastId;
|
||||
List<SysJob> jobs = TenantManager.withoutTenantCondition(
|
||||
() -> mapper.selectReconciliationPage(pageStartId, PAGE_SIZE));
|
||||
for (SysJob job : jobs) {
|
||||
try {
|
||||
// syncJob 会按 jobId 加分布式锁,并在锁内重新读取最新定义。
|
||||
jobService.syncJob(job.getId());
|
||||
reconciled++;
|
||||
} catch (RuntimeException exception) {
|
||||
failed++;
|
||||
log.error("定时任务启动对账失败: jobId={}", job.getId(), exception);
|
||||
}
|
||||
}
|
||||
if (jobs.size() < PAGE_SIZE) break;
|
||||
// Keyset 分页不依赖已处理行仍然存在,避免对账过程中并发删除导致跳行。
|
||||
lastId = jobs.get(jobs.size() - 1).getId();
|
||||
}
|
||||
log.info("定时任务启动对账完成: succeeded={}, failed={}", reconciled, failed);
|
||||
}
|
||||
}
|
||||
@@ -3,6 +3,9 @@ package tech.easyflow.job.service;
|
||||
import com.mybatisflex.core.service.IService;
|
||||
import tech.easyflow.job.entity.SysJobLog;
|
||||
|
||||
import java.io.Serializable;
|
||||
import java.util.Collection;
|
||||
|
||||
/**
|
||||
* 系统任务日志 服务层。
|
||||
*
|
||||
@@ -11,4 +14,5 @@ import tech.easyflow.job.entity.SysJobLog;
|
||||
*/
|
||||
public interface SysJobLogService extends IService<SysJobLog> {
|
||||
|
||||
void requireTerminal(Collection<Serializable> ids);
|
||||
}
|
||||
|
||||
@@ -6,6 +6,7 @@ import tech.easyflow.job.entity.SysJob;
|
||||
import java.io.Serializable;
|
||||
import java.math.BigInteger;
|
||||
import java.util.Collection;
|
||||
import java.util.Date;
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
@@ -22,12 +23,20 @@ public interface SysJobService extends IService<SysJob> {
|
||||
|
||||
void addJob(SysJob job);
|
||||
|
||||
void syncJob(BigInteger id);
|
||||
|
||||
void updateJobDefinition(SysJob job);
|
||||
|
||||
void deleteJob(Collection<Serializable> ids);
|
||||
|
||||
void startJob(BigInteger id);
|
||||
|
||||
void stopJob(BigInteger id);
|
||||
|
||||
String triggerNow(BigInteger id);
|
||||
|
||||
List<Date> nextFireTimes(String cronExpression, int limit);
|
||||
|
||||
/**
|
||||
* 查询引用指定工作流的定时任务。
|
||||
*
|
||||
|
||||
@@ -3,18 +3,23 @@ package tech.easyflow.job.service;
|
||||
import cn.hutool.core.bean.BeanUtil;
|
||||
import com.alibaba.fastjson2.JSONObject;
|
||||
import com.easyagents.flow.core.chain.runtime.ChainExecutor;
|
||||
import com.mybatisflex.core.tenant.TenantManager;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.springframework.dao.DataAccessException;
|
||||
import org.springframework.dao.DataAccessResourceFailureException;
|
||||
import org.springframework.dao.RecoverableDataAccessException;
|
||||
import org.springframework.dao.TransientDataAccessException;
|
||||
import tech.easyflow.ai.easyagentsflow.support.PublishedWorkflowDefinitionIds;
|
||||
import tech.easyflow.ai.service.WorkflowUsageAuthorizationService;
|
||||
import tech.easyflow.common.constant.Constants;
|
||||
import tech.easyflow.common.constant.enums.EnumDataStatus;
|
||||
import tech.easyflow.common.constant.enums.EnumJobStatus;
|
||||
import tech.easyflow.common.entity.LoginAccount;
|
||||
import tech.easyflow.job.entity.SysJob;
|
||||
import tech.easyflow.job.execution.SysJobExecutionContextHolder;
|
||||
import tech.easyflow.job.execution.SysJobInfrastructureException;
|
||||
import tech.easyflow.job.job.JobConstant;
|
||||
import tech.easyflow.job.support.SysJobWorkflowReferenceSupport;
|
||||
import tech.easyflow.system.entity.SysAccount;
|
||||
import tech.easyflow.system.service.SysAccountService;
|
||||
|
||||
import java.math.BigInteger;
|
||||
import java.util.Map;
|
||||
@@ -23,17 +28,13 @@ import java.util.Objects;
|
||||
/**
|
||||
* 工作流定时任务执行服务。
|
||||
*
|
||||
* <p>每次触发都重新加载任务、账号和工作流,并按服务端记录恢复执行主体及重新授权。</p>
|
||||
* <p>任务入口已按数据库当前状态复核任务和账号;本服务继续检查工作流权限。
|
||||
* 工作流引用与运行参数使用触发登记时的账本快照,避免积压期间的普通编辑
|
||||
* 改变既有 execution。</p>
|
||||
*/
|
||||
@Service
|
||||
public class WorkflowJobExecutionService {
|
||||
|
||||
/** 定时任务服务。 */
|
||||
private final SysJobService sysJobService;
|
||||
|
||||
/** 系统账号服务。 */
|
||||
private final SysAccountService sysAccountService;
|
||||
|
||||
/** 工作流使用权限校验服务。 */
|
||||
private final WorkflowUsageAuthorizationService workflowUsageAuthorizationService;
|
||||
|
||||
@@ -43,18 +44,12 @@ public class WorkflowJobExecutionService {
|
||||
/**
|
||||
* 创建工作流定时任务执行服务。
|
||||
*
|
||||
* @param sysJobService 定时任务服务
|
||||
* @param sysAccountService 系统账号服务
|
||||
* @param workflowUsageAuthorizationService 工作流使用权限校验服务
|
||||
* @param chainExecutor 工作流执行器
|
||||
*/
|
||||
public WorkflowJobExecutionService(
|
||||
SysJobService sysJobService,
|
||||
SysAccountService sysAccountService,
|
||||
WorkflowUsageAuthorizationService workflowUsageAuthorizationService,
|
||||
ChainExecutor chainExecutor) {
|
||||
this.sysJobService = sysJobService;
|
||||
this.sysAccountService = sysAccountService;
|
||||
this.workflowUsageAuthorizationService = workflowUsageAuthorizationService;
|
||||
this.chainExecutor = chainExecutor;
|
||||
}
|
||||
@@ -63,31 +58,45 @@ public class WorkflowJobExecutionService {
|
||||
* 使用当前数据库状态执行工作流定时任务。
|
||||
*
|
||||
* @param scheduledJob Quartz 中保存的任务快照
|
||||
* @param currentJob 数据库当前任务定义
|
||||
* @param owner 已完成实时复核的任务归属账号
|
||||
* @return 工作流执行结果
|
||||
* @throws IllegalStateException 任务、账号或租户状态非法时抛出
|
||||
*/
|
||||
public Object execute(SysJob scheduledJob) {
|
||||
public Object execute(SysJob scheduledJob, SysJob currentJob, SysAccount owner) {
|
||||
if (scheduledJob == null || scheduledJob.getId() == null) {
|
||||
throw new IllegalStateException("定时任务不存在或缺少ID");
|
||||
}
|
||||
return TenantManager.withoutTenantCondition(
|
||||
() -> executeWithoutTenantCondition(
|
||||
scheduledJob.getId(),
|
||||
scheduledJob.getTenantId()));
|
||||
PreparedWorkflowExecution prepared;
|
||||
try {
|
||||
prepared = prepare(scheduledJob, currentJob, owner);
|
||||
} catch (DataAccessException exception) {
|
||||
if (exception instanceof TransientDataAccessException
|
||||
|| exception instanceof RecoverableDataAccessException
|
||||
|| exception instanceof DataAccessResourceFailureException) {
|
||||
throw new SysJobInfrastructureException(
|
||||
"准备工作流定时任务执行上下文失败", exception);
|
||||
}
|
||||
throw exception;
|
||||
}
|
||||
return chainExecutor.execute(
|
||||
PublishedWorkflowDefinitionIds.published(prepared.workflowId().toString()),
|
||||
prepared.parameters());
|
||||
}
|
||||
|
||||
/**
|
||||
* 在已关闭 ORM 租户条件的作用域中执行任务,并显式完成租户边界校验。
|
||||
* 组装执行参数,并再次验证入口传入的当前状态与租户边界。
|
||||
*
|
||||
* @param jobId 定时任务 ID
|
||||
* @param scheduledTenantId Quartz 任务快照中的租户 ID
|
||||
* @param scheduledJob 触发登记时保存的任务快照
|
||||
* @param job 数据库当前任务定义
|
||||
* @param owner 已完成实时复核的任务归属账号
|
||||
* @return 工作流执行结果
|
||||
* @throws IllegalStateException 任务、账号或租户状态非法时抛出
|
||||
*/
|
||||
private Object executeWithoutTenantCondition(
|
||||
BigInteger jobId,
|
||||
BigInteger scheduledTenantId) {
|
||||
SysJob job = sysJobService.getById(jobId);
|
||||
private PreparedWorkflowExecution prepare(
|
||||
SysJob scheduledJob, SysJob job, SysAccount owner) {
|
||||
BigInteger jobId = scheduledJob.getId();
|
||||
BigInteger scheduledTenantId = scheduledJob.getTenantId();
|
||||
if (job == null) {
|
||||
throw new IllegalStateException("定时任务不存在或已删除,id=" + jobId);
|
||||
}
|
||||
@@ -102,44 +111,38 @@ public class WorkflowJobExecutionService {
|
||||
if (!SysJobWorkflowReferenceSupport.isWorkflowJob(job)) {
|
||||
throw new IllegalStateException("定时任务类型已变更,id=" + jobId);
|
||||
}
|
||||
if (!SysJobWorkflowReferenceSupport.isWorkflowJob(scheduledJob)) {
|
||||
throw new IllegalStateException("定时任务快照类型非法,id=" + jobId);
|
||||
}
|
||||
|
||||
SysAccount account = requireAvailableOwner(job);
|
||||
SysAccount account = requirePreparedOwner(job, owner);
|
||||
LoginAccount loginAccount = new LoginAccount();
|
||||
BeanUtil.copyProperties(account, loginAccount);
|
||||
BigInteger workflowId = SysJobWorkflowReferenceSupport.requireWorkflowId(job);
|
||||
BigInteger workflowId = SysJobWorkflowReferenceSupport.requireWorkflowId(scheduledJob);
|
||||
workflowUsageAuthorizationService.requireUsableWorkflow(
|
||||
workflowId,
|
||||
loginAccount,
|
||||
"定时任务关联的工作流不存在、已禁用或无权运行");
|
||||
"定时任务关联的工作流不存在、未发布或无权运行");
|
||||
|
||||
JSONObject workflowParams = resolveWorkflowParams(job.getJobParams());
|
||||
JSONObject workflowParams = resolveWorkflowParams(scheduledJob.getJobParams());
|
||||
SysJobExecutionContextHolder.current().ifPresent(context ->
|
||||
workflowParams.put(JobConstant.EXECUTION_KEY, context.executionKey()));
|
||||
workflowParams.put(Constants.LOGIN_USER_KEY, loginAccount);
|
||||
return chainExecutor.execute(workflowId.toString(), workflowParams);
|
||||
return new PreparedWorkflowExecution(workflowId, workflowParams);
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取任务创建账号并校验账号仍可用于执行任务。
|
||||
*
|
||||
* @param job 当前数据库中的定时任务
|
||||
* @return 可用的任务创建账号
|
||||
* @throws IllegalStateException 创建账号缺失、禁用或跨租户时抛出
|
||||
*/
|
||||
private SysAccount requireAvailableOwner(SysJob job) {
|
||||
BigInteger accountId = job.getCreatedBy();
|
||||
if (accountId == null) {
|
||||
throw new IllegalStateException("定时任务缺少服务端归属账号,id=" + job.getId());
|
||||
/** 验证调用方传入的是当前任务已完成数据库复核的归属账号。 */
|
||||
private static SysAccount requirePreparedOwner(SysJob job, SysAccount owner) {
|
||||
if (owner == null || !Objects.equals(job.getCreatedBy(), owner.getId())) {
|
||||
throw new IllegalStateException("定时任务归属账号校验结果无效,id=" + job.getId());
|
||||
}
|
||||
SysAccount account = sysAccountService.getById(accountId);
|
||||
if (account == null) {
|
||||
throw new IllegalStateException("定时任务归属账号不存在,id=" + accountId);
|
||||
if (!EnumDataStatus.AVAILABLE.getCode().equals(owner.getStatus())) {
|
||||
throw new IllegalStateException("定时任务归属账号未启用,id=" + owner.getId());
|
||||
}
|
||||
if (!EnumDataStatus.AVAILABLE.getCode().equals(account.getStatus())) {
|
||||
throw new IllegalStateException("定时任务归属账号未启用,id=" + accountId);
|
||||
}
|
||||
if (!Objects.equals(job.getTenantId(), account.getTenantId())) {
|
||||
if (!Objects.equals(job.getTenantId(), owner.getTenantId())) {
|
||||
throw new IllegalStateException("定时任务与归属账号租户不一致,id=" + job.getId());
|
||||
}
|
||||
return account;
|
||||
return owner;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -156,4 +159,7 @@ public class WorkflowJobExecutionService {
|
||||
.getJSONObject(JobConstant.WORKFLOW_PARAMS_KEY);
|
||||
return params == null ? new JSONObject() : new JSONObject(params);
|
||||
}
|
||||
|
||||
private record PreparedWorkflowExecution(BigInteger workflowId, JSONObject parameters) {
|
||||
}
|
||||
}
|
||||
|
||||
@@ -6,6 +6,10 @@ import tech.easyflow.job.entity.SysJobLog;
|
||||
import tech.easyflow.job.mapper.SysJobLogMapper;
|
||||
import tech.easyflow.job.service.SysJobLogService;
|
||||
|
||||
import java.io.Serializable;
|
||||
import java.math.BigInteger;
|
||||
import java.util.Collection;
|
||||
|
||||
/**
|
||||
* 系统任务日志 服务层实现。
|
||||
*
|
||||
@@ -15,4 +19,13 @@ import tech.easyflow.job.service.SysJobLogService;
|
||||
@Service
|
||||
public class SysJobLogServiceImpl extends ServiceImpl<SysJobLogMapper, SysJobLog> implements SysJobLogService {
|
||||
|
||||
@Override
|
||||
public void requireTerminal(Collection<Serializable> ids) {
|
||||
if (ids == null) return;
|
||||
for (Serializable id : ids) {
|
||||
if (mapper.countActiveById(new BigInteger(id.toString())) > 0) {
|
||||
throw new IllegalStateException("等待执行或执行中的任务记录不能删除,id=" + id);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,175 +1,368 @@
|
||||
package tech.easyflow.job.service.impl;
|
||||
|
||||
import com.easyagents.scheduler.CronSchedulePlan;
|
||||
import com.easyagents.scheduler.ScheduleService;
|
||||
import com.mybatisflex.core.query.QueryWrapper;
|
||||
import com.mybatisflex.spring.service.impl.ServiceImpl;
|
||||
import org.quartz.*;
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
import org.springframework.transaction.PlatformTransactionManager;
|
||||
import org.springframework.transaction.TransactionDefinition;
|
||||
import org.springframework.transaction.support.TransactionTemplate;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.stereotype.Service;
|
||||
import tech.easyflow.common.constant.enums.EnumJobType;
|
||||
import tech.easyflow.common.constant.enums.EnumMisfirePolicy;
|
||||
import tech.easyflow.common.constant.enums.EnumJobStatus;
|
||||
import tech.easyflow.common.cache.RedisLockExecutor;
|
||||
import tech.easyflow.common.constant.enums.EnumJobStatus;
|
||||
import tech.easyflow.common.constant.enums.EnumJobType;
|
||||
import tech.easyflow.common.web.exceptions.BusinessException;
|
||||
import tech.easyflow.job.config.SysJobExecutionProperties;
|
||||
import tech.easyflow.job.entity.SysJob;
|
||||
import tech.easyflow.job.job.JobConstant;
|
||||
import tech.easyflow.job.job.QuartzJob;
|
||||
import tech.easyflow.job.job.QuartzJobNoConcurrent;
|
||||
import tech.easyflow.job.execution.SysJobExecutionStore;
|
||||
import tech.easyflow.job.mapper.SysJobMapper;
|
||||
import tech.easyflow.job.schedule.SysJobScheduleAdapter;
|
||||
import tech.easyflow.job.service.SysJobService;
|
||||
import tech.easyflow.job.support.SysJobWorkflowReferenceSupport;
|
||||
import tech.easyflow.job.util.JobUtil;
|
||||
|
||||
import javax.annotation.Resource;
|
||||
import java.io.Serializable;
|
||||
import java.math.BigInteger;
|
||||
import java.time.Duration;
|
||||
import java.time.ZoneId;
|
||||
import java.util.Collection;
|
||||
import java.util.Collections;
|
||||
import java.util.Date;
|
||||
import java.util.List;
|
||||
import java.util.concurrent.Semaphore;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.function.Supplier;
|
||||
|
||||
/**
|
||||
* 系统任务表 服务层实现。
|
||||
*
|
||||
* @author xiaoma
|
||||
* @since 2025-05-20
|
||||
*/
|
||||
/** 系统任务管理及 Quartz 投影协调服务。 */
|
||||
@Service
|
||||
public class SysJobServiceImpl extends ServiceImpl<SysJobMapper, SysJob> implements SysJobService {
|
||||
public class SysJobServiceImpl extends ServiceImpl<SysJobMapper, SysJob> implements SysJobService {
|
||||
|
||||
private static final Logger log = LoggerFactory.getLogger(SysJobServiceImpl.class);
|
||||
private static final String JOB_LOCK_KEY_PREFIX = "easyflow:lock:job:";
|
||||
private static final Duration LOCK_WAIT_TIMEOUT = Duration.ofSeconds(2);
|
||||
private static final Duration LOCK_LEASE_TIMEOUT = Duration.ofSeconds(10);
|
||||
|
||||
protected Logger log = LoggerFactory.getLogger(SysJobServiceImpl.class);
|
||||
private final SysJobScheduleAdapter scheduleAdapter;
|
||||
private final SysJobExecutionStore executionStore;
|
||||
private final ScheduleService scheduleService;
|
||||
private final RedisLockExecutor redisLockExecutor;
|
||||
private final TransactionTemplate managementTransaction;
|
||||
private final Semaphore managementPermits;
|
||||
private final ZoneId zoneId;
|
||||
|
||||
@Resource
|
||||
private Scheduler scheduler;
|
||||
|
||||
@Resource
|
||||
private RedisLockExecutor redisLockExecutor;
|
||||
public SysJobServiceImpl(SysJobScheduleAdapter scheduleAdapter,
|
||||
SysJobExecutionStore executionStore,
|
||||
ScheduleService scheduleService,
|
||||
RedisLockExecutor redisLockExecutor,
|
||||
PlatformTransactionManager transactionManager,
|
||||
SysJobExecutionProperties executionProperties,
|
||||
@Value("${easyflow.job.timezone:Asia/Shanghai}") String zoneId) {
|
||||
this.scheduleAdapter = scheduleAdapter;
|
||||
this.executionStore = executionStore;
|
||||
this.scheduleService = scheduleService;
|
||||
this.redisLockExecutor = redisLockExecutor;
|
||||
this.managementTransaction = new TransactionTemplate(transactionManager);
|
||||
this.managementTransaction.setIsolationLevel(TransactionDefinition.ISOLATION_READ_COMMITTED);
|
||||
this.managementPermits = new Semaphore(
|
||||
executionProperties.getManagementCommandConcurrency(), true);
|
||||
this.zoneId = ZoneId.of(zoneId);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void test() {
|
||||
System.out.println("java bean 动态执行");
|
||||
// 供 Spring Bean 类型定时任务进行开发环境验证。
|
||||
}
|
||||
|
||||
@Override
|
||||
public void testParam(String a, Boolean b, Integer c) {
|
||||
System.out.println("动态执行spring bean,执行参数:" + "a="+ a + ",b="+ b + ",c="+ c);
|
||||
// 供 Spring Bean 类型定时任务进行参数绑定验证。
|
||||
}
|
||||
|
||||
@Override
|
||||
public void addJob(SysJob job) {
|
||||
Integer allowConcurrent = job.getAllowConcurrent();
|
||||
Class<? extends Job> jobClass = allowConcurrent == 1 ? QuartzJob.class : QuartzJobNoConcurrent.class;
|
||||
|
||||
JobDetail jobDetail = JobBuilder.newJob(jobClass)
|
||||
.withIdentity(JobUtil.getJobKey(job))
|
||||
.build();
|
||||
|
||||
jobDetail.getJobDataMap().put(JobConstant.JOB_MAP_BEAN_NAME, job);
|
||||
|
||||
CronScheduleBuilder cron = CronScheduleBuilder.cronSchedule(job.getCronExpression());
|
||||
|
||||
Integer misfirePolicy = job.getMisfirePolicy();
|
||||
if (EnumMisfirePolicy.MISFIRE_DO_NOTHING.getCode() == misfirePolicy) {
|
||||
cron.withMisfireHandlingInstructionDoNothing();
|
||||
}
|
||||
if (EnumMisfirePolicy.MISFIRE_FIRE_AND_PROCEED.getCode() == misfirePolicy) {
|
||||
cron.withMisfireHandlingInstructionFireAndProceed();
|
||||
}
|
||||
if (EnumMisfirePolicy.MISFIRE_IGNORE_MISFIRES.getCode() == misfirePolicy) {
|
||||
cron.withMisfireHandlingInstructionIgnoreMisfires();
|
||||
if (job == null || job.getId() == null) {
|
||||
throw new IllegalArgumentException("任务及任务ID不能为空");
|
||||
}
|
||||
syncJob(job.getId());
|
||||
}
|
||||
|
||||
Trigger trigger = TriggerBuilder.newTrigger()
|
||||
.withIdentity(JobUtil.getTriggerKey(job))
|
||||
.withSchedule(cron).build();
|
||||
@Override
|
||||
public void syncJob(BigInteger id) {
|
||||
withJobCommandLock(id, () -> {
|
||||
if (!projectCurrentStateOrStop(id)) {
|
||||
cancelPendingBestEffort(id, "任务已停止");
|
||||
}
|
||||
return null;
|
||||
});
|
||||
}
|
||||
|
||||
try {
|
||||
scheduler.scheduleJob(jobDetail,trigger);
|
||||
} catch (SchedulerException e) {
|
||||
log.error("启动任务失败:", e);
|
||||
throw new RuntimeException(e);
|
||||
@Override
|
||||
public void updateJobDefinition(SysJob update) {
|
||||
if (update == null || update.getId() == null) {
|
||||
throw new IllegalArgumentException("任务及任务ID不能为空");
|
||||
}
|
||||
BigInteger id = update.getId();
|
||||
withJobCommandLock(id, () -> {
|
||||
inManagementTransaction(() -> {
|
||||
SysJob current = requireJobForUpdate(id);
|
||||
long generation = nextScheduleGeneration(current);
|
||||
|
||||
// 状态、归属和代际都是服务端维护字段。必须以行锁下的最终值覆盖
|
||||
// 请求快照,避免普通编辑越过 start/stop 或把并发 STOP 写回 RUNNING。
|
||||
update.setTenantId(current.getTenantId());
|
||||
update.setDeptId(current.getDeptId());
|
||||
update.setCreated(current.getCreated());
|
||||
update.setCreatedBy(current.getCreatedBy());
|
||||
update.setStatus(current.getStatus());
|
||||
update.setScheduleGeneration(generation);
|
||||
if (!updateById(update)) {
|
||||
throw new IllegalStateException("定时任务更新失败,id=" + id);
|
||||
}
|
||||
return null;
|
||||
});
|
||||
projectCurrentStateOrStop(id);
|
||||
return null;
|
||||
});
|
||||
}
|
||||
|
||||
@Override
|
||||
public void deleteJob(Collection<Serializable> ids) {
|
||||
try {
|
||||
for (Serializable id : ids) {
|
||||
SysJob sysJob = new SysJob();
|
||||
sysJob.setId(new BigInteger(id.toString()));
|
||||
scheduler.deleteJob(JobUtil.getJobKey(sysJob));
|
||||
}
|
||||
} catch (SchedulerException e) {
|
||||
log.error("删除任务失败:", e);
|
||||
throw new RuntimeException(e);
|
||||
if (ids == null) return;
|
||||
List<BigInteger> jobIds = ids.stream()
|
||||
.map(id -> new BigInteger(id.toString()))
|
||||
.toList();
|
||||
// 在产生任何 Quartz 或跨租户账本副作用前,先按当前租户完整校验全部任务。
|
||||
jobIds.forEach(this::requireJob);
|
||||
for (BigInteger jobId : jobIds) {
|
||||
withJobCommandLock(jobId, () -> {
|
||||
// 分三段收口:业务先 STOP,Quartz 投影删除成功后再物理删除。
|
||||
// 任一阶段失败都只会留下可重试的 STOP 行,不会留下 RUNNING 无投影。
|
||||
inManagementTransaction(() -> {
|
||||
SysJob job = requireJobForUpdate(jobId);
|
||||
if (!Integer.valueOf(EnumJobStatus.STOP.getCode()).equals(job.getStatus())) {
|
||||
SysJob stopped = new SysJob();
|
||||
stopped.setId(jobId);
|
||||
stopped.setStatus(EnumJobStatus.STOP.getCode());
|
||||
if (!updateById(stopped)) {
|
||||
throw new IllegalStateException("定时任务停止失败,id=" + jobId);
|
||||
}
|
||||
}
|
||||
return null;
|
||||
});
|
||||
try {
|
||||
projectCurrentStateOrStop(jobId);
|
||||
inManagementTransaction(() -> {
|
||||
SysJob current = requireJobForUpdate(jobId);
|
||||
if (!Integer.valueOf(EnumJobStatus.STOP.getCode())
|
||||
.equals(current.getStatus())) {
|
||||
throw new IllegalStateException(
|
||||
"定时任务状态已变化,取消删除,id=" + jobId);
|
||||
}
|
||||
if (!removeById(jobId)) {
|
||||
throw new IllegalStateException("定时任务删除失败,id=" + jobId);
|
||||
}
|
||||
return null;
|
||||
});
|
||||
} finally {
|
||||
cancelPendingBestEffort(jobId, "任务已删除或已停止");
|
||||
}
|
||||
return null;
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void startJob(BigInteger id) {
|
||||
redisLockExecutor.executeWithLock(JOB_LOCK_KEY_PREFIX + id, LOCK_WAIT_TIMEOUT, LOCK_LEASE_TIMEOUT, () -> {
|
||||
SysJob sysJob = this.getById(id);
|
||||
if (sysJob == null) {
|
||||
throw new IllegalStateException("任务不存在,id=" + id);
|
||||
}
|
||||
try {
|
||||
JobKey jobKey = JobUtil.getJobKey(sysJob);
|
||||
if (!scheduler.checkExists(jobKey)) {
|
||||
addJob(sysJob);
|
||||
withJobCommandLock(id, () -> {
|
||||
// STOP 期间可能因数据库瞬时故障遗留上一代 PENDING。重新进入 RUNNING
|
||||
// 前必须强制清理;失败则拒绝启动,避免旧 fire 被新一代状态放行。
|
||||
requireJob(id);
|
||||
executionStore.cancelPending(id, "任务重新启动,取消上一代待执行记录");
|
||||
inManagementTransaction(() -> {
|
||||
SysJob current = requireJobForUpdate(id);
|
||||
if (!Integer.valueOf(EnumJobStatus.RUNNING.getCode()).equals(current.getStatus())) {
|
||||
if (getMapper().startNextGeneration(
|
||||
id, EnumJobStatus.RUNNING.getCode()) != 1) {
|
||||
throw new IllegalStateException("定时任务启动失败,id=" + id);
|
||||
}
|
||||
}
|
||||
if (!Integer.valueOf(EnumJobStatus.RUNNING.getCode()).equals(sysJob.getStatus())) {
|
||||
SysJob update = new SysJob();
|
||||
update.setId(id);
|
||||
update.setStatus(EnumJobStatus.RUNNING.getCode());
|
||||
this.updateById(update);
|
||||
}
|
||||
} catch (SchedulerException e) {
|
||||
log.error("启动任务失败:id={}", id, e);
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
return null;
|
||||
});
|
||||
projectCurrentStateOrStop(id);
|
||||
return null;
|
||||
});
|
||||
}
|
||||
|
||||
@Override
|
||||
public void stopJob(BigInteger id) {
|
||||
redisLockExecutor.executeWithLock(JOB_LOCK_KEY_PREFIX + id, LOCK_WAIT_TIMEOUT, LOCK_LEASE_TIMEOUT, () -> {
|
||||
SysJob sysJob = this.getById(id);
|
||||
if (sysJob == null) {
|
||||
throw new IllegalStateException("任务不存在,id=" + id);
|
||||
}
|
||||
withJobCommandLock(id, () -> {
|
||||
inManagementTransaction(() -> {
|
||||
SysJob job = requireJobForUpdate(id);
|
||||
if (!Integer.valueOf(EnumJobStatus.STOP.getCode()).equals(job.getStatus())) {
|
||||
SysJob stopped = new SysJob();
|
||||
stopped.setId(id);
|
||||
stopped.setStatus(EnumJobStatus.STOP.getCode());
|
||||
if (!updateById(stopped)) {
|
||||
throw new IllegalStateException("定时任务停止失败,id=" + id);
|
||||
}
|
||||
}
|
||||
return null;
|
||||
});
|
||||
try {
|
||||
JobKey jobKey = JobUtil.getJobKey(sysJob);
|
||||
if (scheduler.checkExists(jobKey)) {
|
||||
deleteJob(Collections.singletonList(id));
|
||||
}
|
||||
if (!Integer.valueOf(EnumJobStatus.STOP.getCode()).equals(sysJob.getStatus())) {
|
||||
SysJob update = new SysJob();
|
||||
update.setId(id);
|
||||
update.setStatus(EnumJobStatus.STOP.getCode());
|
||||
this.updateById(update);
|
||||
}
|
||||
} catch (SchedulerException e) {
|
||||
log.error("停止任务失败:id={}", id, e);
|
||||
throw new RuntimeException(e);
|
||||
projectCurrentStateOrStop(id);
|
||||
} finally {
|
||||
cancelPendingBestEffort(id, "任务已停止");
|
||||
}
|
||||
return null;
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* {@inheritDoc}
|
||||
*/
|
||||
@Override
|
||||
public String triggerNow(BigInteger id) {
|
||||
return withJobCommandLock(id, () -> inManagementTransaction(() -> {
|
||||
SysJob job = requireJobForUpdate(id);
|
||||
if (!Integer.valueOf(EnumJobStatus.RUNNING.getCode()).equals(job.getStatus())) {
|
||||
throw new BusinessException(409, 409, "任务未处于运行状态,请先启动任务");
|
||||
}
|
||||
return scheduleAdapter.triggerNow(id);
|
||||
}));
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<Date> nextFireTimes(String cronExpression, int limit) {
|
||||
return scheduleService.nextFireTimes(new CronSchedulePlan(cronExpression, zoneId), limit)
|
||||
.stream().map(Date::from).toList();
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<SysJob> listWorkflowJobsByWorkflowId(BigInteger workflowId) {
|
||||
if (workflowId == null) {
|
||||
return List.of();
|
||||
}
|
||||
if (workflowId == null) return List.of();
|
||||
QueryWrapper queryWrapper = QueryWrapper.create()
|
||||
.eq(SysJob::getJobType, EnumJobType.TINY_FLOW.getCode());
|
||||
.eq(SysJob::getJobType, EnumJobType.TINY_FLOW.getCode());
|
||||
return list(queryWrapper).stream()
|
||||
.filter(job -> workflowId.equals(SysJobWorkflowReferenceSupport.resolveWorkflowId(job)))
|
||||
.toList();
|
||||
.filter(job -> workflowId.equals(SysJobWorkflowReferenceSupport.resolveWorkflowId(job)))
|
||||
.toList();
|
||||
}
|
||||
|
||||
private SysJob requireJob(BigInteger id) {
|
||||
SysJob job = getById(id);
|
||||
if (job == null) throw new IllegalStateException("任务不存在,id=" + id);
|
||||
return job;
|
||||
}
|
||||
|
||||
private boolean projectCurrentStateOrStop(BigInteger jobId) {
|
||||
// 第二段持任务行锁完成 JobStoreTX 投影。Provider 失败必须在同一行锁下把
|
||||
// 业务状态改为 STOP;事务基础设施或 commit 失败则直接抛出,不做破坏性清理。
|
||||
ProjectionOutcome outcome = inManagementTransaction(() -> {
|
||||
SysJob current = getMapper().selectByIdForUpdate(jobId);
|
||||
boolean running = current != null
|
||||
&& Integer.valueOf(EnumJobStatus.RUNNING.getCode()).equals(current.getStatus());
|
||||
try {
|
||||
if (running) {
|
||||
scheduleAdapter.replace(current);
|
||||
} else {
|
||||
scheduleAdapter.delete(jobId);
|
||||
}
|
||||
return new ProjectionOutcome(running, null);
|
||||
} catch (RuntimeException providerFailure) {
|
||||
if (current != null
|
||||
&& !Integer.valueOf(EnumJobStatus.STOP.getCode())
|
||||
.equals(current.getStatus())) {
|
||||
SysJob stopped = new SysJob();
|
||||
stopped.setId(jobId);
|
||||
stopped.setStatus(EnumJobStatus.STOP.getCode());
|
||||
if (!updateById(stopped)) {
|
||||
throw new IllegalStateException(
|
||||
"调度投影失败后无法停止任务,id=" + jobId,
|
||||
providerFailure);
|
||||
}
|
||||
}
|
||||
return new ProjectionOutcome(false, providerFailure);
|
||||
}
|
||||
});
|
||||
if (outcome.failure() == null) return outcome.running();
|
||||
|
||||
RuntimeException providerFailure = outcome.failure();
|
||||
try {
|
||||
// STOP 已确认提交后再重试删除投影,并再次锁行检查;若更新命令已重新
|
||||
// 启动任务则跳过旧清理,避免失锁窗口中的 stale delete。
|
||||
inManagementTransaction(() -> {
|
||||
SysJob current = getMapper().selectByIdForUpdate(jobId);
|
||||
if (current == null
|
||||
|| Integer.valueOf(EnumJobStatus.STOP.getCode())
|
||||
.equals(current.getStatus())) {
|
||||
scheduleAdapter.delete(jobId);
|
||||
}
|
||||
return null;
|
||||
});
|
||||
} catch (RuntimeException cleanupFailure) {
|
||||
if (cleanupFailure != providerFailure) {
|
||||
providerFailure.addSuppressed(cleanupFailure);
|
||||
}
|
||||
}
|
||||
cancelPendingBestEffort(jobId, "调度投影同步失败,任务已停止");
|
||||
throw providerFailure;
|
||||
}
|
||||
|
||||
private SysJob requireJobForUpdate(BigInteger id) {
|
||||
SysJob job = getMapper().selectByIdForUpdate(id);
|
||||
if (job == null) throw new IllegalStateException("任务不存在,id=" + id);
|
||||
return job;
|
||||
}
|
||||
|
||||
private <T> T withJobCommandLock(BigInteger id, Supplier<T> command) {
|
||||
return withManagementPermit(() -> redisLockExecutor.executeWithRenewingLock(
|
||||
lockKey(id), LOCK_WAIT_TIMEOUT, LOCK_LEASE_TIMEOUT, command));
|
||||
}
|
||||
|
||||
private <T> T inManagementTransaction(Supplier<T> command) {
|
||||
return managementTransaction.execute(status -> command.get());
|
||||
}
|
||||
|
||||
private <T> T withManagementPermit(Supplier<T> command) {
|
||||
boolean acquired = false;
|
||||
try {
|
||||
acquired = managementPermits.tryAcquire(
|
||||
LOCK_WAIT_TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
|
||||
if (!acquired) {
|
||||
throw new IllegalStateException("定时任务管理命令繁忙,请稍后重试");
|
||||
}
|
||||
return command.get();
|
||||
} catch (InterruptedException exception) {
|
||||
Thread.currentThread().interrupt();
|
||||
throw new IllegalStateException("等待定时任务管理命令资源时被中断", exception);
|
||||
} finally {
|
||||
if (acquired) managementPermits.release();
|
||||
}
|
||||
}
|
||||
|
||||
private void cancelPendingBestEffort(BigInteger jobId, String reason) {
|
||||
try {
|
||||
// 与 Worker 一致按日志行 -> 任务行加锁,并在管理状态提交后复核最终状态。
|
||||
executionStore.cancelPending(jobId, reason);
|
||||
} catch (RuntimeException cleanupFailure) {
|
||||
// Worker 领取遗留记录时仍会根据 STOP/已删除状态安全取消。
|
||||
log.warn("定时任务状态已收口,但待执行记录延迟清理: jobId={}",
|
||||
jobId, cleanupFailure);
|
||||
}
|
||||
}
|
||||
|
||||
private static String lockKey(BigInteger id) {
|
||||
if (id == null) throw new IllegalArgumentException("任务ID不能为空");
|
||||
return JOB_LOCK_KEY_PREFIX + id;
|
||||
}
|
||||
|
||||
private static long nextScheduleGeneration(SysJob job) {
|
||||
Long generation = job.getScheduleGeneration();
|
||||
if (generation == null || generation < 0L || generation == Long.MAX_VALUE) {
|
||||
throw new IllegalStateException(
|
||||
"定时任务调度代际非法,id=" + job.getId() + ", generation=" + generation);
|
||||
}
|
||||
return generation + 1L;
|
||||
}
|
||||
|
||||
private record ProjectionOutcome(boolean running, RuntimeException failure) {
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -1,146 +1,18 @@
|
||||
package tech.easyflow.job.util;
|
||||
|
||||
import cn.hutool.core.util.ArrayUtil;
|
||||
import cn.hutool.core.util.StrUtil;
|
||||
import org.quartz.JobKey;
|
||||
import org.quartz.TriggerKey;
|
||||
import tech.easyflow.common.constant.enums.EnumJobType;
|
||||
import tech.easyflow.common.util.SpringContextUtil;
|
||||
import tech.easyflow.job.entity.SysJob;
|
||||
import tech.easyflow.job.job.JobConstant;
|
||||
import tech.easyflow.job.service.WorkflowJobExecutionService;
|
||||
|
||||
import java.lang.reflect.Method;
|
||||
import java.util.Arrays;
|
||||
import java.util.Map;
|
||||
|
||||
/** Java 类定时任务的开发环境示例。 */
|
||||
public class JobUtil {
|
||||
|
||||
/**
|
||||
* sysJobService.test()
|
||||
*/
|
||||
public static Object execSpringBean(SysJob job) {
|
||||
Map<String, Object> jobParams = job.getJobParams();
|
||||
if (jobParams != null) {
|
||||
String beanMethod = jobParams.get(JobConstant.BEAN_METHOD_KEY).toString();
|
||||
String[] strings = StrUtil.subBefore(beanMethod, "(", false).split("\\.");
|
||||
Object bean = SpringContextUtil.getBean(strings[0]);
|
||||
String param = StrUtil.subBetween(beanMethod, "(", ")");
|
||||
try {
|
||||
// 调用方法并传递参数
|
||||
return invoke(bean, strings[1], getParams(param));
|
||||
} catch (Exception e) {
|
||||
throw new RuntimeException("执行 beanMethod 报错:", e);
|
||||
}
|
||||
}
|
||||
return null;
|
||||
public String execTest(String a, Integer b, Double c, Long d) {
|
||||
return "a=" + a + ",b=" + b + ",c=" + c + ",d=" + d;
|
||||
}
|
||||
|
||||
/**
|
||||
* tech.easyflow.job.util.JobUtil.execTest("test",1,0.52D,100L)
|
||||
* @param job
|
||||
*/
|
||||
public static Object execJavaClass(SysJob job) {
|
||||
Map<String, Object> jobParams = job.getJobParams();
|
||||
if (jobParams != null) {
|
||||
try {
|
||||
String javaMethod = jobParams.get(JobConstant.JAVA_METHOD_KEY).toString();
|
||||
String before = StrUtil.subBefore(javaMethod, "(", false);
|
||||
String[] strings = before.split("\\.");
|
||||
String className = String.join(".", Arrays.copyOf(strings, strings.length - 1));
|
||||
String methodName = strings[strings.length - 1];
|
||||
String param = StrUtil.subBetween(javaMethod, "(", ")");
|
||||
Object obj = Class.forName(className).getDeclaredConstructor().newInstance();;
|
||||
return invoke(obj, methodName, getParams(param));
|
||||
} catch (Exception e) {
|
||||
throw new RuntimeException("执行 javaMethod 报错: ",e);
|
||||
}
|
||||
/** 可用于开发环境容量与并发验收的阻塞型模拟任务。 */
|
||||
public String sleepTask(String label, Long millis) throws InterruptedException {
|
||||
if (millis == null || millis < 0 || millis > 300_000) {
|
||||
throw new IllegalArgumentException("millis must be between 0 and 300000");
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* 通过任务模块的受控执行服务运行工作流。
|
||||
*
|
||||
* @param job Quartz 中保存的任务快照
|
||||
* @return 工作流执行结果
|
||||
*/
|
||||
public static Object execWorkFlow(SysJob job) {
|
||||
WorkflowJobExecutionService executionService =
|
||||
SpringContextUtil.getBean(WorkflowJobExecutionService.class);
|
||||
return executionService.execute(job);
|
||||
}
|
||||
|
||||
public static Object execute(SysJob job) {
|
||||
Object res = null;
|
||||
Integer jobType = job.getJobType();
|
||||
if (EnumJobType.TINY_FLOW.getCode() == jobType) {
|
||||
res = execWorkFlow(job);
|
||||
}
|
||||
if (EnumJobType.SPRING_BEAN.getCode() == jobType) {
|
||||
res = execSpringBean(job);
|
||||
}
|
||||
if (EnumJobType.JAVA_CLASS.getCode() == jobType) {
|
||||
res = execJavaClass(job);
|
||||
}
|
||||
return res;
|
||||
}
|
||||
|
||||
public void execTest(String a,Integer b,Double c,Long d) {
|
||||
System.out.println("动态执行方法,执行参数:" + "a="+ a + ",b="+ b + ",c="+ c + ",d="+ d);
|
||||
}
|
||||
|
||||
private static Object[] getParams(String param) {
|
||||
if (StrUtil.isEmpty(param)) {
|
||||
return new Object[]{new Class<?>[]{}, new Object[]{}};
|
||||
}
|
||||
String[] splits = param.split(",");
|
||||
Object[] res = new Object[2];
|
||||
Object[] params = new Object[splits.length];
|
||||
Class<?>[] paramTypes = new Class[splits.length];
|
||||
for (int i = 0; i < splits.length; i++) {
|
||||
String split = splits[i].trim();
|
||||
if (split.startsWith("\"")) {
|
||||
params[i] = split.substring(1, split.length() - 1);
|
||||
paramTypes[i] = String.class;
|
||||
} else if ("true".equals(split) || "false".equals(split)) {
|
||||
params[i] = Boolean.valueOf(split);
|
||||
paramTypes[i] = Boolean.class;
|
||||
} else if (split.endsWith("L")) {
|
||||
params[i] = Long.valueOf(split.substring(0, split.length() - 1));
|
||||
paramTypes[i] = Long.class;
|
||||
} else if (split.endsWith("D")) {
|
||||
params[i] = Double.valueOf(split.substring(0, split.length() - 1));
|
||||
paramTypes[i] = Double.class;
|
||||
} else if (split.endsWith("F")) {
|
||||
params[i] = Float.valueOf(split.substring(0, split.length() - 1));
|
||||
paramTypes[i] = Float.class;
|
||||
} else {
|
||||
params[i] = Integer.valueOf(split);
|
||||
paramTypes[i] = Integer.class;
|
||||
}
|
||||
}
|
||||
res[0] = paramTypes;
|
||||
res[1] = params;
|
||||
return res;
|
||||
}
|
||||
|
||||
private static Object invoke(Object bean, String methodName, Object[] params) throws Exception {
|
||||
Object[] args = (Object[]) params[1];
|
||||
if (ArrayUtil.isEmpty(params[1])) {
|
||||
Method method = bean.getClass().getDeclaredMethod(methodName);
|
||||
return method.invoke(bean);
|
||||
} else {
|
||||
Method method = bean.getClass().getDeclaredMethod(methodName, (Class<?>[]) params[0]);
|
||||
return method.invoke(bean, args);
|
||||
}
|
||||
}
|
||||
|
||||
public static JobKey getJobKey(SysJob job) {
|
||||
return JobKey.jobKey(job.getId().toString(), JobConstant.JOB_GROUP);
|
||||
}
|
||||
|
||||
public static TriggerKey getTriggerKey(SysJob job) {
|
||||
return TriggerKey.triggerKey(job.getId().toString(), JobConstant.JOB_GROUP);
|
||||
Thread.sleep(millis);
|
||||
return label;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,23 @@
|
||||
package tech.easyflow.job.config;
|
||||
|
||||
import org.junit.Assert;
|
||||
import org.junit.Test;
|
||||
|
||||
public class SysJobConnectionCapacityValidatorTest {
|
||||
|
||||
@Test
|
||||
public void configuredBaselineMustReserveAllSchedulerConnections() {
|
||||
Assert.assertEquals(24,
|
||||
SysJobConnectionCapacityValidator.requiredPoolSize(4, 8, 4, 4));
|
||||
SysJobExecutionProperties properties = new SysJobExecutionProperties();
|
||||
new SysJobConnectionCapacityValidator(properties, 24, 8, true)
|
||||
.afterPropertiesSet();
|
||||
}
|
||||
|
||||
@Test(expected = IllegalStateException.class)
|
||||
public void insufficientPoolMustFailFast() {
|
||||
SysJobExecutionProperties properties = new SysJobExecutionProperties();
|
||||
new SysJobConnectionCapacityValidator(properties, 23, 8, true)
|
||||
.afterPropertiesSet();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,148 @@
|
||||
package tech.easyflow.job.execution;
|
||||
|
||||
import com.easyagents.scheduler.ScheduleFireContext;
|
||||
import com.easyagents.scheduler.ScheduleId;
|
||||
import com.easyagents.scheduler.ScheduleRefireException;
|
||||
import org.junit.Test;
|
||||
import org.springframework.dao.DataAccessResourceFailureException;
|
||||
import org.springframework.dao.DataIntegrityViolationException;
|
||||
import org.springframework.dao.TransientDataAccessResourceException;
|
||||
import org.springframework.transaction.CannotCreateTransactionException;
|
||||
import org.springframework.transaction.TransactionSystemException;
|
||||
import tech.easyflow.job.config.SysJobExecutionProperties;
|
||||
|
||||
import java.time.Duration;
|
||||
import java.time.Instant;
|
||||
import java.util.Map;
|
||||
|
||||
import static org.mockito.Mockito.doNothing;
|
||||
import static org.mockito.Mockito.doThrow;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.times;
|
||||
import static org.mockito.Mockito.verify;
|
||||
|
||||
public class EasyFlowScheduleHandlerTest {
|
||||
|
||||
@Test
|
||||
public void shouldRetryOnlyBoundedTransientRegistrationFailures() {
|
||||
SysJobExecutionRegistrar registrar = mock(SysJobExecutionRegistrar.class);
|
||||
ScheduleFireContext context = context();
|
||||
doThrow(new TransientDataAccessResourceException("temporary"))
|
||||
.doThrow(new TransientDataAccessResourceException("temporary"))
|
||||
.doNothing().when(registrar).register(context);
|
||||
SysJobExecutionProperties properties = new SysJobExecutionProperties();
|
||||
properties.setRegistrationMaxAttempts(3);
|
||||
properties.setRegistrationRetryDelay(Duration.ofNanos(1));
|
||||
|
||||
new EasyFlowScheduleHandler(registrar, properties,
|
||||
mock(SysJobExecutionMetrics.class)).execute(context);
|
||||
|
||||
verify(registrar, times(3)).register(context);
|
||||
}
|
||||
|
||||
@Test(expected = IllegalStateException.class)
|
||||
public void shouldNotRetryBusinessFailure() {
|
||||
SysJobExecutionRegistrar registrar = mock(SysJobExecutionRegistrar.class);
|
||||
ScheduleFireContext context = context();
|
||||
doThrow(new IllegalStateException("invalid job")).when(registrar).register(context);
|
||||
|
||||
new EasyFlowScheduleHandler(registrar, new SysJobExecutionProperties(),
|
||||
mock(SysJobExecutionMetrics.class)).execute(context);
|
||||
}
|
||||
|
||||
@Test(expected = ScheduleRefireException.class)
|
||||
public void shouldRequestQuartzRefireAfterTransientRetriesAreExhausted() {
|
||||
SysJobExecutionRegistrar registrar = mock(SysJobExecutionRegistrar.class);
|
||||
ScheduleFireContext context = context();
|
||||
doThrow(new TransientDataAccessResourceException("temporary"))
|
||||
.when(registrar).register(context);
|
||||
SysJobExecutionProperties properties = new SysJobExecutionProperties();
|
||||
properties.setRegistrationMaxAttempts(2);
|
||||
properties.setRegistrationRetryDelay(Duration.ofNanos(1));
|
||||
|
||||
new EasyFlowScheduleHandler(registrar, properties,
|
||||
mock(SysJobExecutionMetrics.class)).execute(context);
|
||||
}
|
||||
|
||||
@Test(expected = ScheduleRefireException.class)
|
||||
public void shouldRequestQuartzRefireWhenDatabaseConnectionIsUnavailable() {
|
||||
SysJobExecutionRegistrar registrar = mock(SysJobExecutionRegistrar.class);
|
||||
ScheduleFireContext context = context();
|
||||
doThrow(new DataAccessResourceFailureException("connection unavailable"))
|
||||
.when(registrar).register(context);
|
||||
SysJobExecutionProperties properties = new SysJobExecutionProperties();
|
||||
properties.setRegistrationMaxAttempts(2);
|
||||
properties.setRegistrationRetryDelay(Duration.ofNanos(1));
|
||||
|
||||
try {
|
||||
new EasyFlowScheduleHandler(registrar, properties,
|
||||
mock(SysJobExecutionMetrics.class)).execute(context);
|
||||
} finally {
|
||||
verify(registrar, times(2)).register(context);
|
||||
}
|
||||
}
|
||||
|
||||
@Test(expected = ScheduleRefireException.class)
|
||||
public void transactionBeginFailureMustRequestQuartzRefire() {
|
||||
assertTransactionFailureRequestsRefire(
|
||||
new CannotCreateTransactionException("pool exhausted"));
|
||||
}
|
||||
|
||||
@Test(expected = ScheduleRefireException.class)
|
||||
public void transactionCommitFailureMustRequestQuartzRefire() {
|
||||
assertTransactionFailureRequestsRefire(
|
||||
new TransactionSystemException("commit failed"));
|
||||
}
|
||||
|
||||
@Test(expected = ScheduleRefireException.class)
|
||||
public void interruptedLocalRetryMustStillRequestDurableQuartzRefire() {
|
||||
SysJobExecutionRegistrar registrar = mock(SysJobExecutionRegistrar.class);
|
||||
ScheduleFireContext context = context();
|
||||
doThrow(new TransientDataAccessResourceException("temporary"))
|
||||
.when(registrar).register(context);
|
||||
SysJobExecutionProperties properties = new SysJobExecutionProperties();
|
||||
properties.setRegistrationMaxAttempts(3);
|
||||
properties.setRegistrationRetryDelay(Duration.ofSeconds(1));
|
||||
|
||||
Thread.currentThread().interrupt();
|
||||
try {
|
||||
new EasyFlowScheduleHandler(registrar, properties,
|
||||
mock(SysJobExecutionMetrics.class)).execute(context);
|
||||
} finally {
|
||||
Thread.interrupted();
|
||||
}
|
||||
}
|
||||
|
||||
@Test(expected = DataIntegrityViolationException.class)
|
||||
public void shouldNotRetryNonRecoverableDataError() {
|
||||
SysJobExecutionRegistrar registrar = mock(SysJobExecutionRegistrar.class);
|
||||
ScheduleFireContext context = context();
|
||||
doThrow(new DataIntegrityViolationException("invalid row"))
|
||||
.when(registrar).register(context);
|
||||
|
||||
try {
|
||||
new EasyFlowScheduleHandler(registrar, new SysJobExecutionProperties(),
|
||||
mock(SysJobExecutionMetrics.class)).execute(context);
|
||||
} finally {
|
||||
verify(registrar, times(1)).register(context);
|
||||
}
|
||||
}
|
||||
|
||||
private static void assertTransactionFailureRequestsRefire(RuntimeException failure) {
|
||||
SysJobExecutionRegistrar registrar = mock(SysJobExecutionRegistrar.class);
|
||||
ScheduleFireContext context = context();
|
||||
doThrow(failure).when(registrar).register(context);
|
||||
SysJobExecutionProperties properties = new SysJobExecutionProperties();
|
||||
properties.setRegistrationMaxAttempts(2);
|
||||
properties.setRegistrationRetryDelay(Duration.ofNanos(1));
|
||||
|
||||
new EasyFlowScheduleHandler(registrar, properties,
|
||||
mock(SysJobExecutionMetrics.class)).execute(context);
|
||||
}
|
||||
|
||||
private static ScheduleFireContext context() {
|
||||
Instant now = Instant.now();
|
||||
return new ScheduleFireContext(new ScheduleId("easyflow.job", "1"),
|
||||
EasyFlowScheduleHandler.CODE, now, now, "fire-1", null, false, Map.of());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,83 @@
|
||||
package tech.easyflow.job.execution;
|
||||
|
||||
import org.apache.ibatis.annotations.Select;
|
||||
import org.apache.ibatis.annotations.Update;
|
||||
import org.junit.Assert;
|
||||
import org.junit.Test;
|
||||
import tech.easyflow.job.mapper.SysJobLogMapper;
|
||||
|
||||
import java.lang.reflect.Method;
|
||||
import java.util.Locale;
|
||||
|
||||
public class SysJobExecutionMapperContractTest {
|
||||
|
||||
@Test
|
||||
public void claimShouldUseSkipLockedAndOwnedTerminalCas() throws Exception {
|
||||
String expiredSql = sql(SysJobLogMapper.class.getMethod(
|
||||
"selectExpiredClaimCandidateId").getAnnotation(Select.class).value());
|
||||
String expiredProbeSql = sql(SysJobLogMapper.class.getMethod(
|
||||
"selectExpiredClaimCandidateIdWithoutLock").getAnnotation(Select.class).value());
|
||||
String pendingSql = sql(SysJobLogMapper.class.getMethod(
|
||||
"selectPendingClaimCandidateId").getAnnotation(Select.class).value());
|
||||
String finishSql = sql(SysJobLogMapper.class.getMethod("finishOwned",
|
||||
java.math.BigInteger.class, String.class, String.class, int.class,
|
||||
String.class, String.class).getAnnotation(Update.class).value());
|
||||
|
||||
Assert.assertTrue(expiredSql.contains("FOR UPDATE SKIP LOCKED"));
|
||||
Assert.assertFalse(expiredProbeSql.contains("FOR UPDATE"));
|
||||
Assert.assertTrue(expiredProbeSql.contains("STATUS=3"));
|
||||
Assert.assertTrue(pendingSql.contains("FOR UPDATE SKIP LOCKED"));
|
||||
Assert.assertTrue(expiredSql.startsWith("SELECT ID "));
|
||||
Assert.assertTrue(pendingSql.startsWith("SELECT ID "));
|
||||
Assert.assertFalse(expiredSql.contains("SELECT *"));
|
||||
Assert.assertFalse(pendingSql.contains("SELECT *"));
|
||||
Assert.assertFalse(expiredSql.contains(" OR "));
|
||||
Assert.assertFalse(pendingSql.contains(" OR "));
|
||||
Assert.assertTrue(finishSql.contains("STATUS=3"));
|
||||
Assert.assertTrue(finishSql.contains("LEASE_OWNER=#{OWNER}"));
|
||||
Assert.assertTrue(finishSql.contains("EXECUTION_TOKEN=#{TOKEN}"));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void heartbeatMustNotAdvanceFencingVersion() throws Exception {
|
||||
Method method = SysJobLogMapper.class.getMethod("renewLease",
|
||||
java.math.BigInteger.class, String.class, String.class,
|
||||
long.class);
|
||||
String heartbeatSql = sql(method.getAnnotation(Update.class).value());
|
||||
|
||||
Assert.assertFalse(heartbeatSql.contains("VERSION"));
|
||||
Assert.assertTrue(heartbeatSql.contains("LEASE_OWNER=#{OWNER}"));
|
||||
Assert.assertTrue(heartbeatSql.contains("EXECUTION_TOKEN=#{TOKEN}"));
|
||||
Assert.assertTrue(heartbeatSql.contains("CURRENT_TIMESTAMP(3)"));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void exhaustedPendingInfrastructureRetryMustLeaveClaimQueue() throws Exception {
|
||||
String sql = sql(SysJobLogMapper.class.getMethod("markPendingDead",
|
||||
java.math.BigInteger.class, String.class).getAnnotation(Update.class).value());
|
||||
|
||||
Assert.assertTrue(sql.contains("STATUS=4"));
|
||||
Assert.assertTrue(sql.contains("WHERE ID=#{ID} AND STATUS=2"));
|
||||
Assert.assertTrue(sql.contains("NEXT_RETRY_TIME=NULL"));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void cancellationUpdateMustOnlyTouchPendingLedgerRows() throws Exception {
|
||||
String lockSql = sql(SysJobLogMapper.class.getMethod(
|
||||
"selectPendingIdsByJobIdForUpdate", java.math.BigInteger.class)
|
||||
.getAnnotation(org.apache.ibatis.annotations.Select.class).value());
|
||||
String sql = sql(SysJobLogMapper.class.getMethod("cancelPendingByJobId",
|
||||
java.math.BigInteger.class, String.class)
|
||||
.getAnnotation(Update.class).value());
|
||||
|
||||
Assert.assertTrue(lockSql.contains("WHERE JOB_ID=#{JOBID} AND STATUS=2"));
|
||||
Assert.assertTrue(lockSql.contains("ORDER BY ID FOR UPDATE"));
|
||||
Assert.assertTrue(sql.contains("WHERE JOB_ID=#{JOBID} AND STATUS=2"));
|
||||
Assert.assertFalse(sql.contains("FROM TB_SYS_JOB"));
|
||||
}
|
||||
|
||||
private static String sql(String[] fragments) {
|
||||
return String.join(" ", fragments).replaceAll("\\s+", " ")
|
||||
.toUpperCase(Locale.ROOT);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,28 @@
|
||||
package tech.easyflow.job.execution;
|
||||
|
||||
import io.micrometer.core.instrument.simple.SimpleMeterRegistry;
|
||||
import org.junit.Assert;
|
||||
import org.junit.Test;
|
||||
|
||||
public class SysJobExecutionMetricsTest {
|
||||
|
||||
@Test
|
||||
public void completionWithoutOwnedTerminalStatusMustBeAccepted() {
|
||||
SimpleMeterRegistry registry = new SimpleMeterRegistry();
|
||||
SysJobExecutionMetrics metrics = new SysJobExecutionMetrics(registry);
|
||||
|
||||
metrics.executionStarted();
|
||||
metrics.executionFinished(null, 1L);
|
||||
|
||||
Assert.assertEquals(0.0D,
|
||||
registry.get("easyflow.job.execution.local_active").gauge().value(), 0.0D);
|
||||
Assert.assertEquals(1L,
|
||||
registry.get("easyflow.job.execution.duration").timer().count());
|
||||
Assert.assertEquals(0.0D,
|
||||
registry.get("easyflow.job.execution.terminal")
|
||||
.tag("status", "success").counter().count(), 0.0D);
|
||||
Assert.assertEquals(0.0D,
|
||||
registry.get("easyflow.job.execution.terminal")
|
||||
.tag("status", "failure").counter().count(), 0.0D);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,210 @@
|
||||
package tech.easyflow.job.execution;
|
||||
|
||||
import com.easyagents.scheduler.ScheduleFireContext;
|
||||
import com.easyagents.scheduler.ScheduleId;
|
||||
import org.junit.Assert;
|
||||
import org.junit.Test;
|
||||
import org.springframework.dao.CannotAcquireLockException;
|
||||
import org.springframework.dao.DuplicateKeyException;
|
||||
import org.springframework.transaction.PlatformTransactionManager;
|
||||
import org.springframework.transaction.TransactionStatus;
|
||||
import tech.easyflow.common.constant.enums.EnumJobStatus;
|
||||
import tech.easyflow.job.config.SysJobExecutionProperties;
|
||||
import tech.easyflow.job.entity.SysJob;
|
||||
import tech.easyflow.job.entity.SysJobLog;
|
||||
import tech.easyflow.job.mapper.SysJobLogMapper;
|
||||
import tech.easyflow.job.mapper.SysJobMapper;
|
||||
import tech.easyflow.job.job.JobConstant;
|
||||
|
||||
import java.math.BigInteger;
|
||||
import java.sql.SQLException;
|
||||
import java.time.Instant;
|
||||
import java.util.Map;
|
||||
import java.util.Optional;
|
||||
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.ArgumentMatchers.anyLong;
|
||||
import static org.mockito.ArgumentMatchers.anyString;
|
||||
import static org.mockito.ArgumentMatchers.eq;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.never;
|
||||
import static org.mockito.Mockito.doThrow;
|
||||
import static org.mockito.Mockito.times;
|
||||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
public class SysJobExecutionStoreTest {
|
||||
|
||||
@Test
|
||||
public void executionKeyDuplicateMustBeAcceptedAsIdempotentRegistration() {
|
||||
RegistrationFixture fixture = registrationFixture();
|
||||
doThrow(new DuplicateKeyException("execution key duplicate"))
|
||||
.when(fixture.logMapper()).insert(any(SysJobLog.class));
|
||||
when(fixture.logMapper().selectIdByExecutionKey(anyString()))
|
||||
.thenReturn(BigInteger.valueOf(999L));
|
||||
|
||||
fixture.store().register(fixture.context());
|
||||
|
||||
verify(fixture.logMapper()).selectIdByExecutionKey(anyString());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void unrelatedUniqueConstraintViolationMustNotLoseFire() {
|
||||
RegistrationFixture fixture = registrationFixture();
|
||||
DuplicateKeyException duplicate = new DuplicateKeyException("primary key duplicate");
|
||||
doThrow(duplicate).when(fixture.logMapper()).insert(any(SysJobLog.class));
|
||||
when(fixture.logMapper().selectIdByExecutionKey(anyString())).thenReturn(null);
|
||||
|
||||
DuplicateKeyException thrown = Assert.assertThrows(
|
||||
DuplicateKeyException.class,
|
||||
() -> fixture.store().register(fixture.context()));
|
||||
|
||||
Assert.assertSame(duplicate, thrown);
|
||||
verify(fixture.logMapper()).selectIdByExecutionKey(anyString());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void exhaustedHeadRowsMustNotDelayFollowingRunnableExecution() {
|
||||
SysJobMapper jobMapper = mock(SysJobMapper.class);
|
||||
SysJobLogMapper logMapper = mock(SysJobLogMapper.class);
|
||||
SysJobExecutionMetrics metrics = mock(SysJobExecutionMetrics.class);
|
||||
PlatformTransactionManager transactionManager = mock(PlatformTransactionManager.class);
|
||||
when(transactionManager.getTransaction(any())).thenReturn(mock(TransactionStatus.class));
|
||||
|
||||
SysJobExecutionProperties properties = new SysJobExecutionProperties();
|
||||
SysJobLog exhausted1 = execution(101L, 1_001L,
|
||||
properties.getInfrastructureRetryLimit());
|
||||
SysJobLog exhausted2 = execution(102L, 1_002L,
|
||||
properties.getInfrastructureRetryLimit());
|
||||
SysJobLog runnable = execution(103L, 1_003L, 0);
|
||||
when(logMapper.selectExpiredClaimCandidate()).thenReturn(null);
|
||||
when(logMapper.selectPendingClaimCandidate())
|
||||
.thenReturn(exhausted1, exhausted2, runnable);
|
||||
when(logMapper.markPendingDead(any(), anyString())).thenReturn(1);
|
||||
when(logMapper.claimPending(eq(runnable.getId()), eq("node-a"),
|
||||
anyString(), anyLong())).thenReturn(1);
|
||||
SysJob activeJob = new SysJob();
|
||||
activeJob.setId(runnable.getJobId());
|
||||
activeJob.setStatus(EnumJobStatus.RUNNING.getCode());
|
||||
when(jobMapper.selectByIdForUpdate(runnable.getJobId())).thenReturn(activeJob);
|
||||
SysJobExecutionStore store = new SysJobExecutionStore(
|
||||
jobMapper, logMapper, properties, metrics, transactionManager);
|
||||
|
||||
Optional<ClaimedSysJobExecution> claimed = store.claimOne("node-a");
|
||||
|
||||
Assert.assertTrue(claimed.isPresent());
|
||||
Assert.assertEquals(runnable.getId(), claimed.get().execution().getId());
|
||||
verify(logMapper, times(2)).markPendingDead(any(), anyString());
|
||||
verify(metrics, times(2)).recordDead();
|
||||
verify(transactionManager, times(3)).commit(any());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void transientClaimDeadlockMustRetryInANewTransaction() {
|
||||
SysJobMapper jobMapper = mock(SysJobMapper.class);
|
||||
SysJobLogMapper logMapper = mock(SysJobLogMapper.class);
|
||||
SysJobExecutionMetrics metrics = mock(SysJobExecutionMetrics.class);
|
||||
PlatformTransactionManager transactionManager = mock(PlatformTransactionManager.class);
|
||||
when(transactionManager.getTransaction(any())).thenReturn(mock(TransactionStatus.class));
|
||||
|
||||
SysJobExecutionProperties properties = new SysJobExecutionProperties();
|
||||
SysJobLog runnable = execution(201L, 2_001L, 0);
|
||||
when(logMapper.selectExpiredClaimCandidate()).thenReturn(null);
|
||||
when(logMapper.selectPendingClaimCandidate())
|
||||
.thenThrow(new CannotAcquireLockException(
|
||||
"simulated deadlock", new SQLException("deadlock")))
|
||||
.thenReturn(runnable);
|
||||
SysJob activeJob = new SysJob();
|
||||
activeJob.setId(runnable.getJobId());
|
||||
activeJob.setStatus(EnumJobStatus.RUNNING.getCode());
|
||||
when(jobMapper.selectByIdForUpdate(runnable.getJobId())).thenReturn(activeJob);
|
||||
when(logMapper.claimPending(eq(runnable.getId()), eq("node-a"),
|
||||
anyString(), anyLong())).thenReturn(1);
|
||||
SysJobExecutionStore store = new SysJobExecutionStore(
|
||||
jobMapper, logMapper, properties, metrics, transactionManager);
|
||||
|
||||
Optional<ClaimedSysJobExecution> claimed = store.claimOne("node-a");
|
||||
|
||||
Assert.assertTrue(claimed.isPresent());
|
||||
Assert.assertEquals(runnable.getId(), claimed.get().execution().getId());
|
||||
verify(transactionManager, times(2)).getTransaction(any());
|
||||
verify(transactionManager, times(1)).rollback(any());
|
||||
verify(transactionManager, times(1)).commit(any());
|
||||
verify(logMapper, never()).markPendingDead(any(), anyString());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void lockedExpiredCandidateMustFallBackToPendingExecution() {
|
||||
SysJobMapper jobMapper = mock(SysJobMapper.class);
|
||||
SysJobLogMapper logMapper = mock(SysJobLogMapper.class);
|
||||
SysJobExecutionMetrics metrics = mock(SysJobExecutionMetrics.class);
|
||||
PlatformTransactionManager transactionManager = mock(PlatformTransactionManager.class);
|
||||
when(transactionManager.getTransaction(any())).thenReturn(mock(TransactionStatus.class));
|
||||
|
||||
SysJobExecutionProperties properties = new SysJobExecutionProperties();
|
||||
SysJobLog runnable = execution(301L, 3_001L, 0);
|
||||
when(logMapper.selectExpiredClaimCandidateIdWithoutLock())
|
||||
.thenReturn(BigInteger.valueOf(300L));
|
||||
// 过期候选已被其他 Worker 锁定,SKIP LOCKED 后必须继续领取 PENDING。
|
||||
when(logMapper.selectExpiredClaimCandidate()).thenReturn(null);
|
||||
when(logMapper.selectPendingClaimCandidate()).thenReturn(runnable);
|
||||
SysJob activeJob = new SysJob();
|
||||
activeJob.setId(runnable.getJobId());
|
||||
activeJob.setStatus(EnumJobStatus.RUNNING.getCode());
|
||||
when(jobMapper.selectByIdForUpdate(runnable.getJobId())).thenReturn(activeJob);
|
||||
when(logMapper.claimPending(eq(runnable.getId()), eq("node-a"),
|
||||
anyString(), anyLong())).thenReturn(1);
|
||||
SysJobExecutionStore store = new SysJobExecutionStore(
|
||||
jobMapper, logMapper, properties, metrics, transactionManager);
|
||||
|
||||
Optional<ClaimedSysJobExecution> claimed = store.claimOne("node-a");
|
||||
|
||||
Assert.assertTrue(claimed.isPresent());
|
||||
Assert.assertEquals(runnable.getId(), claimed.get().execution().getId());
|
||||
verify(logMapper).selectExpiredClaimCandidate();
|
||||
verify(logMapper).selectPendingClaimCandidate();
|
||||
verify(transactionManager, times(2)).commit(any());
|
||||
}
|
||||
|
||||
private static SysJobLog execution(long id, long jobId, int attempts) {
|
||||
SysJobLog execution = new SysJobLog();
|
||||
execution.setId(BigInteger.valueOf(id));
|
||||
execution.setJobId(BigInteger.valueOf(jobId));
|
||||
execution.setAttemptCount(attempts);
|
||||
return execution;
|
||||
}
|
||||
|
||||
private static RegistrationFixture registrationFixture() {
|
||||
SysJobMapper jobMapper = mock(SysJobMapper.class);
|
||||
SysJobLogMapper logMapper = mock(SysJobLogMapper.class);
|
||||
SysJobExecutionMetrics metrics = mock(SysJobExecutionMetrics.class);
|
||||
PlatformTransactionManager transactionManager = mock(PlatformTransactionManager.class);
|
||||
when(transactionManager.getTransaction(any())).thenReturn(mock(TransactionStatus.class));
|
||||
SysJob job = new SysJob();
|
||||
job.setId(BigInteger.ONE);
|
||||
job.setTenantId(BigInteger.TWO);
|
||||
job.setStatus(EnumJobStatus.RUNNING.getCode());
|
||||
job.setScheduleGeneration(0L);
|
||||
when(jobMapper.selectByIdForUpdate(BigInteger.ONE)).thenReturn(job);
|
||||
Instant now = Instant.now();
|
||||
ScheduleFireContext context = new ScheduleFireContext(
|
||||
new ScheduleId("easyflow.job", "1"),
|
||||
EasyFlowScheduleHandler.CODE,
|
||||
now,
|
||||
now,
|
||||
"fire-1",
|
||||
null,
|
||||
false,
|
||||
Map.of(JobConstant.SCHEDULE_GENERATION, "0"));
|
||||
return new RegistrationFixture(
|
||||
new SysJobExecutionStore(jobMapper, logMapper,
|
||||
new SysJobExecutionProperties(), metrics, transactionManager),
|
||||
logMapper,
|
||||
context);
|
||||
}
|
||||
|
||||
private record RegistrationFixture(SysJobExecutionStore store,
|
||||
SysJobLogMapper logMapper,
|
||||
ScheduleFireContext context) {
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,256 @@
|
||||
package tech.easyflow.job.execution;
|
||||
|
||||
import io.micrometer.core.instrument.simple.SimpleMeterRegistry;
|
||||
import org.junit.Assert;
|
||||
import org.junit.Test;
|
||||
import tech.easyflow.job.config.SysJobExecutionProperties;
|
||||
import tech.easyflow.job.entity.SysJobLog;
|
||||
|
||||
import java.math.BigInteger;
|
||||
import java.time.Duration;
|
||||
import java.util.Optional;
|
||||
import java.util.concurrent.CountDownLatch;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.ArgumentMatchers.anyInt;
|
||||
import static org.mockito.ArgumentMatchers.anyLong;
|
||||
import static org.mockito.ArgumentMatchers.eq;
|
||||
import static org.mockito.ArgumentMatchers.nullable;
|
||||
import static org.mockito.Mockito.doAnswer;
|
||||
import static org.mockito.Mockito.doNothing;
|
||||
import static org.mockito.Mockito.doThrow;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.never;
|
||||
import static org.mockito.Mockito.timeout;
|
||||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
public class SysJobExecutionWorkerTest {
|
||||
|
||||
@Test
|
||||
public void nonVmErrorMustNotPermanentlyReduceWorkerCapacity() throws Exception {
|
||||
SysJobExecutionStore store = mock(SysJobExecutionStore.class);
|
||||
SysJobInvoker invoker = mock(SysJobInvoker.class);
|
||||
SysJobExecutionMetrics metrics = mock(SysJobExecutionMetrics.class);
|
||||
SysJobExecutionProperties properties = properties();
|
||||
ClaimedSysJobExecution first = claim(101L, "token-1");
|
||||
ClaimedSysJobExecution second = claim(102L, "token-2");
|
||||
when(store.queueSnapshot()).thenReturn(new SysJobQueueSnapshot(0, 0, 0));
|
||||
when(store.claimOne(any())).thenReturn(
|
||||
Optional.of(first), Optional.of(second), Optional.empty());
|
||||
when(store.finish(any(), anyInt(), nullable(String.class), nullable(String.class)))
|
||||
.thenReturn(true);
|
||||
CountDownLatch secondCompleted = new CountDownLatch(1);
|
||||
AtomicInteger calls = new AtomicInteger();
|
||||
doAnswer(invocation -> {
|
||||
if (calls.incrementAndGet() == 1) {
|
||||
throw new AssertionError("broken business handler");
|
||||
}
|
||||
secondCompleted.countDown();
|
||||
return "ok";
|
||||
}).when(invoker).execute(any());
|
||||
SysJobExecutionWorker worker = new SysJobExecutionWorker(
|
||||
store, invoker, properties, metrics);
|
||||
|
||||
worker.start();
|
||||
try {
|
||||
Assert.assertTrue("worker 未在非 VM Error 后继续领取任务",
|
||||
secondCompleted.await(3, TimeUnit.SECONDS));
|
||||
verify(store, timeout(1_000)).finish(
|
||||
first, 0, null, "AssertionError: broken business handler");
|
||||
verify(store, timeout(1_000)).finish(second, 1, "\"ok\"", null);
|
||||
} finally {
|
||||
worker.stop();
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void preparationInfrastructureFailureMustReleaseForRetry() throws Exception {
|
||||
SysJobExecutionStore store = mock(SysJobExecutionStore.class);
|
||||
SysJobInvoker invoker = mock(SysJobInvoker.class);
|
||||
SysJobExecutionMetrics metrics = mock(SysJobExecutionMetrics.class);
|
||||
ClaimedSysJobExecution claim = claim(201L, "token-infra");
|
||||
when(store.queueSnapshot()).thenReturn(new SysJobQueueSnapshot(0, 0, 0));
|
||||
when(store.claimOne(any())).thenReturn(Optional.of(claim), Optional.empty());
|
||||
CountDownLatch released = new CountDownLatch(1);
|
||||
when(store.releaseForInfrastructureRetry(any(), any())).thenAnswer(invocation -> {
|
||||
released.countDown();
|
||||
return true;
|
||||
});
|
||||
doAnswer(invocation -> {
|
||||
throw new SysJobInfrastructureException("database unavailable",
|
||||
new IllegalStateException("connection lost"));
|
||||
}).when(invoker).execute(any());
|
||||
SysJobExecutionWorker worker = new SysJobExecutionWorker(
|
||||
store, invoker, properties(), metrics);
|
||||
|
||||
worker.start();
|
||||
try {
|
||||
Assert.assertTrue(released.await(3, TimeUnit.SECONDS));
|
||||
verify(store).releaseForInfrastructureRetry(
|
||||
claim, "IllegalStateException: connection lost");
|
||||
} finally {
|
||||
worker.stop();
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void metricsFailureMustNotLeakContextOrActiveExecution() throws Exception {
|
||||
SysJobExecutionStore store = mock(SysJobExecutionStore.class);
|
||||
SysJobInvoker invoker = mock(SysJobInvoker.class);
|
||||
SysJobExecutionMetrics metrics = mock(SysJobExecutionMetrics.class);
|
||||
ClaimedSysJobExecution first = claim(251L, "token-metrics-1");
|
||||
ClaimedSysJobExecution second = claim(252L, "token-metrics-2");
|
||||
when(store.queueSnapshot()).thenReturn(new SysJobQueueSnapshot(0, 0, 0));
|
||||
when(store.claimOne(any())).thenReturn(
|
||||
Optional.of(first), Optional.of(second), Optional.empty());
|
||||
when(store.finish(any(), anyInt(), nullable(String.class), nullable(String.class)))
|
||||
.thenReturn(true);
|
||||
doThrow(new IllegalStateException("meter unavailable"))
|
||||
.doNothing()
|
||||
.when(metrics).executionFinished(nullable(String.class), anyLong());
|
||||
CountDownLatch secondCompleted = new CountDownLatch(1);
|
||||
AtomicInteger calls = new AtomicInteger();
|
||||
doAnswer(invocation -> {
|
||||
if (calls.incrementAndGet() == 2) secondCompleted.countDown();
|
||||
return "ok";
|
||||
}).when(invoker).execute(any());
|
||||
SysJobExecutionWorker worker = new SysJobExecutionWorker(
|
||||
store, invoker, properties(), metrics);
|
||||
|
||||
worker.start();
|
||||
try {
|
||||
Assert.assertTrue("指标收尾异常后 Worker 未继续执行下一任务",
|
||||
secondCompleted.await(3, TimeUnit.SECONDS));
|
||||
verify(store, timeout(1_000)).finish(first, 1, "\"ok\"", null);
|
||||
verify(store, timeout(1_000)).finish(second, 1, "\"ok\"", null);
|
||||
} finally {
|
||||
worker.stop();
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void fencingRejectionMustNotPolluteNextExecution() throws Exception {
|
||||
SysJobExecutionStore store = mock(SysJobExecutionStore.class);
|
||||
SysJobInvoker invoker = mock(SysJobInvoker.class);
|
||||
SysJobExecutionMetrics metrics = new SysJobExecutionMetrics(
|
||||
new SimpleMeterRegistry());
|
||||
ClaimedSysJobExecution first = claim(271L, "token-fenced");
|
||||
ClaimedSysJobExecution second = claim(272L, "token-next");
|
||||
when(store.queueSnapshot()).thenReturn(new SysJobQueueSnapshot(0, 0, 0));
|
||||
when(store.claimOne(any())).thenReturn(
|
||||
Optional.of(first), Optional.of(second), Optional.empty());
|
||||
when(store.finish(any(), anyInt(), nullable(String.class), nullable(String.class)))
|
||||
.thenReturn(false, true);
|
||||
CountDownLatch secondCompleted = new CountDownLatch(1);
|
||||
AtomicInteger calls = new AtomicInteger();
|
||||
doAnswer(invocation -> {
|
||||
if (calls.incrementAndGet() == 2) secondCompleted.countDown();
|
||||
return "ok";
|
||||
}).when(invoker).execute(any());
|
||||
SysJobExecutionWorker worker = new SysJobExecutionWorker(
|
||||
store, invoker, properties(), metrics);
|
||||
|
||||
worker.start();
|
||||
try {
|
||||
Assert.assertTrue("fencing 拒绝后 Worker 未继续执行下一任务",
|
||||
secondCompleted.await(3, TimeUnit.SECONDS));
|
||||
verify(store, timeout(1_000)).finish(first, 1, "\"ok\"", null);
|
||||
verify(store, timeout(1_000)).finish(second, 1, "\"ok\"", null);
|
||||
} finally {
|
||||
worker.stop();
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void staleHeartbeatMustNotInterruptNextExecutionOnReusedWorkerThread()
|
||||
throws Exception {
|
||||
SysJobExecutionStore store = mock(SysJobExecutionStore.class);
|
||||
SysJobInvoker invoker = mock(SysJobInvoker.class);
|
||||
SysJobExecutionMetrics metrics = mock(SysJobExecutionMetrics.class);
|
||||
SysJobExecutionProperties properties = properties();
|
||||
properties.setHeartbeatInterval(Duration.ofMillis(20));
|
||||
ClaimedSysJobExecution first = claim(301L, "token-stale");
|
||||
ClaimedSysJobExecution second = claim(302L, "token-current");
|
||||
when(store.queueSnapshot()).thenReturn(new SysJobQueueSnapshot(0, 0, 0));
|
||||
when(store.claimOne(any())).thenReturn(
|
||||
Optional.of(first), Optional.of(second), Optional.empty());
|
||||
when(store.finish(any(), anyInt(), nullable(String.class), nullable(String.class)))
|
||||
.thenReturn(true);
|
||||
when(store.heartbeat(eq(second))).thenReturn(true);
|
||||
|
||||
CountDownLatch firstStarted = new CountDownLatch(1);
|
||||
CountDownLatch releaseFirst = new CountDownLatch(1);
|
||||
CountDownLatch staleHeartbeatStarted = new CountDownLatch(1);
|
||||
CountDownLatch lostLeaseMetricStarted = new CountDownLatch(1);
|
||||
CountDownLatch releaseLostLeaseMetric = new CountDownLatch(1);
|
||||
CountDownLatch secondStarted = new CountDownLatch(1);
|
||||
CountDownLatch releaseSecond = new CountDownLatch(1);
|
||||
when(store.heartbeat(eq(first))).thenAnswer(invocation -> {
|
||||
staleHeartbeatStarted.countDown();
|
||||
return false;
|
||||
});
|
||||
doAnswer(invocation -> {
|
||||
lostLeaseMetricStarted.countDown();
|
||||
releaseLostLeaseMetric.await(3, TimeUnit.SECONDS);
|
||||
return null;
|
||||
}).when(metrics).recordLostLease();
|
||||
AtomicInteger calls = new AtomicInteger();
|
||||
doAnswer(invocation -> {
|
||||
if (calls.incrementAndGet() == 1) {
|
||||
firstStarted.countDown();
|
||||
releaseFirst.await(3, TimeUnit.SECONDS);
|
||||
return "first";
|
||||
}
|
||||
secondStarted.countDown();
|
||||
releaseSecond.await(3, TimeUnit.SECONDS);
|
||||
return "second";
|
||||
}).when(invoker).execute(any());
|
||||
SysJobExecutionWorker worker = new SysJobExecutionWorker(
|
||||
store, invoker, properties, metrics);
|
||||
|
||||
worker.start();
|
||||
try {
|
||||
Assert.assertTrue(firstStarted.await(1, TimeUnit.SECONDS));
|
||||
Assert.assertTrue(staleHeartbeatStarted.await(1, TimeUnit.SECONDS));
|
||||
Assert.assertTrue(lostLeaseMetricStarted.await(1, TimeUnit.SECONDS));
|
||||
releaseFirst.countDown();
|
||||
Assert.assertTrue(secondStarted.await(1, TimeUnit.SECONDS));
|
||||
releaseLostLeaseMetric.countDown();
|
||||
Thread.sleep(100L);
|
||||
releaseSecond.countDown();
|
||||
verify(store, timeout(1_000)).finish(second, 1, "\"second\"", null);
|
||||
verify(store, never()).finish(eq(second), eq(0),
|
||||
nullable(String.class), nullable(String.class));
|
||||
verify(metrics).recordLostLease();
|
||||
} finally {
|
||||
releaseFirst.countDown();
|
||||
releaseLostLeaseMetric.countDown();
|
||||
releaseSecond.countDown();
|
||||
worker.stop();
|
||||
}
|
||||
}
|
||||
|
||||
private static SysJobExecutionProperties properties() {
|
||||
SysJobExecutionProperties properties = new SysJobExecutionProperties();
|
||||
properties.setWorkerCount(1);
|
||||
properties.setPollInterval(Duration.ofMillis(10));
|
||||
properties.setLeaseDuration(Duration.ofSeconds(3));
|
||||
properties.setHeartbeatInterval(Duration.ofMillis(500));
|
||||
properties.setShutdownWaitTimeout(Duration.ofSeconds(1));
|
||||
return properties;
|
||||
}
|
||||
|
||||
private static ClaimedSysJobExecution claim(long id, String token) {
|
||||
SysJobLog execution = new SysJobLog();
|
||||
execution.setId(BigInteger.valueOf(id));
|
||||
execution.setJobId(BigInteger.valueOf(id + 1_000L));
|
||||
execution.setTenantId(BigInteger.ONE);
|
||||
execution.setDeptId(BigInteger.ONE);
|
||||
execution.setJobType(3);
|
||||
execution.setAttemptCount(1);
|
||||
return new ClaimedSysJobExecution(execution, "node-a", token);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,140 @@
|
||||
package tech.easyflow.job.execution;
|
||||
|
||||
import org.junit.Assert;
|
||||
import org.junit.Test;
|
||||
import org.springframework.aop.framework.ProxyFactory;
|
||||
import org.springframework.context.ApplicationContext;
|
||||
import tech.easyflow.common.constant.enums.EnumJobStatus;
|
||||
import tech.easyflow.common.constant.enums.EnumJobType;
|
||||
import tech.easyflow.job.entity.SysJob;
|
||||
import tech.easyflow.job.job.JobConstant;
|
||||
import tech.easyflow.job.mapper.SysJobMapper;
|
||||
import tech.easyflow.job.service.WorkflowJobExecutionService;
|
||||
import tech.easyflow.system.entity.SysAccount;
|
||||
|
||||
import java.math.BigInteger;
|
||||
import java.util.Map;
|
||||
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.never;
|
||||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
public class SysJobInvokerTest {
|
||||
|
||||
@Test
|
||||
public void springBeanInvocationMustUseJdkProxyPublicMethod() throws Exception {
|
||||
Fixture fixture = fixture(EnumJobType.SPRING_BEAN);
|
||||
EchoService target = value -> "echo:" + value;
|
||||
ProxyFactory proxyFactory = new ProxyFactory(target);
|
||||
proxyFactory.setInterfaces(EchoService.class);
|
||||
when(fixture.applicationContext().getBean("echoService"))
|
||||
.thenReturn(proxyFactory.getProxy());
|
||||
fixture.snapshot().setJobParams(Map.of(
|
||||
JobConstant.BEAN_METHOD_KEY, "echoService.echo(\"hello\")"));
|
||||
|
||||
Object result = fixture.invoker().execute(fixture.snapshot());
|
||||
|
||||
Assert.assertEquals("echo:hello", result);
|
||||
verify(fixture.ownerValidator()).requireAvailableOwner(fixture.current());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void springBeanInvocationMustUseCglibProxyPublicMethod() throws Exception {
|
||||
Fixture fixture = fixture(EnumJobType.SPRING_BEAN);
|
||||
ProxyFactory proxyFactory = new ProxyFactory(new EchoServiceImpl());
|
||||
proxyFactory.setProxyTargetClass(true);
|
||||
when(fixture.applicationContext().getBean("echoService"))
|
||||
.thenReturn(proxyFactory.getProxy());
|
||||
fixture.snapshot().setJobParams(Map.of(
|
||||
JobConstant.BEAN_METHOD_KEY, "echoService.echo(\"hello\")"));
|
||||
|
||||
Object result = fixture.invoker().execute(fixture.snapshot());
|
||||
|
||||
Assert.assertEquals("echo:hello", result);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void javaClassInvocationMustRejectPrivateMethod() {
|
||||
Fixture fixture = fixture(EnumJobType.JAVA_CLASS);
|
||||
fixture.snapshot().setJobParams(Map.of(
|
||||
JobConstant.JAVA_METHOD_KEY,
|
||||
PrivateJavaTask.class.getName() + ".hidden()"));
|
||||
|
||||
NoSuchMethodException exception = Assert.assertThrows(
|
||||
NoSuchMethodException.class,
|
||||
() -> fixture.invoker().execute(fixture.snapshot()));
|
||||
|
||||
Assert.assertTrue(exception.getMessage().contains("hidden"));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void invalidOwnerMustBlockSpringBeanBeforeLookup() {
|
||||
Fixture fixture = fixture(EnumJobType.SPRING_BEAN);
|
||||
fixture.snapshot().setJobParams(Map.of(
|
||||
JobConstant.BEAN_METHOD_KEY, "echoService.echo(\"hello\")"));
|
||||
when(fixture.ownerValidator().requireAvailableOwner(fixture.current()))
|
||||
.thenThrow(new IllegalStateException("owner disabled"));
|
||||
|
||||
IllegalStateException exception = Assert.assertThrows(
|
||||
IllegalStateException.class,
|
||||
() -> fixture.invoker().execute(fixture.snapshot()));
|
||||
|
||||
Assert.assertEquals("owner disabled", exception.getMessage());
|
||||
verify(fixture.applicationContext(), never()).getBean(any(String.class));
|
||||
}
|
||||
|
||||
private static Fixture fixture(EnumJobType type) {
|
||||
SysJobMapper jobMapper = mock(SysJobMapper.class);
|
||||
ApplicationContext applicationContext = mock(ApplicationContext.class);
|
||||
WorkflowJobExecutionService workflowExecutionService =
|
||||
mock(WorkflowJobExecutionService.class);
|
||||
SysJobOwnerValidator ownerValidator = mock(SysJobOwnerValidator.class);
|
||||
SysJob snapshot = job(101L, type);
|
||||
SysJob current = job(101L, type);
|
||||
current.setStatus(EnumJobStatus.RUNNING.getCode());
|
||||
current.setCreatedBy(BigInteger.valueOf(301L));
|
||||
SysAccount owner = new SysAccount();
|
||||
owner.setId(current.getCreatedBy());
|
||||
owner.setTenantId(current.getTenantId());
|
||||
when(jobMapper.selectOneById(snapshot.getId())).thenReturn(current);
|
||||
when(ownerValidator.requireAvailableOwner(current)).thenReturn(owner);
|
||||
return new Fixture(
|
||||
new SysJobInvoker(jobMapper, applicationContext,
|
||||
workflowExecutionService, ownerValidator),
|
||||
applicationContext, ownerValidator, snapshot, current);
|
||||
}
|
||||
|
||||
private static SysJob job(long id, EnumJobType type) {
|
||||
SysJob job = new SysJob();
|
||||
job.setId(BigInteger.valueOf(id));
|
||||
job.setTenantId(BigInteger.valueOf(201L));
|
||||
job.setJobType(type.getCode());
|
||||
return job;
|
||||
}
|
||||
|
||||
public interface EchoService {
|
||||
String echo(String value);
|
||||
}
|
||||
|
||||
public static class EchoServiceImpl implements EchoService {
|
||||
@Override
|
||||
public String echo(String value) {
|
||||
return "echo:" + value;
|
||||
}
|
||||
}
|
||||
|
||||
public static class PrivateJavaTask {
|
||||
private String hidden() {
|
||||
return "hidden";
|
||||
}
|
||||
}
|
||||
|
||||
private record Fixture(SysJobInvoker invoker,
|
||||
ApplicationContext applicationContext,
|
||||
SysJobOwnerValidator ownerValidator,
|
||||
SysJob snapshot,
|
||||
SysJob current) {
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,778 @@
|
||||
package tech.easyflow.job.execution;
|
||||
|
||||
import org.junit.After;
|
||||
import org.junit.Assert;
|
||||
import org.junit.Assume;
|
||||
import org.junit.Before;
|
||||
import org.junit.Test;
|
||||
|
||||
import java.sql.Connection;
|
||||
import java.sql.DriverManager;
|
||||
import java.sql.PreparedStatement;
|
||||
import java.sql.ResultSet;
|
||||
import java.sql.SQLException;
|
||||
import java.sql.Statement;
|
||||
import java.util.UUID;
|
||||
import java.util.concurrent.CountDownLatch;
|
||||
import java.util.concurrent.ExecutorService;
|
||||
import java.util.concurrent.Executors;
|
||||
import java.util.concurrent.Future;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
|
||||
/**
|
||||
* 在显式提供的 MySQL 8 测试库上验证执行账本的真实事务协议。
|
||||
*/
|
||||
public class SysJobLedgerMySqlIntegrationTest {
|
||||
|
||||
private String jdbcUrl;
|
||||
private String username;
|
||||
private String password;
|
||||
private long jobId;
|
||||
private long firstExecutionId;
|
||||
private long secondExecutionId;
|
||||
private long registrationExecutionId;
|
||||
private long generationExecutionId;
|
||||
|
||||
@Before
|
||||
public void setUp() throws Exception {
|
||||
jdbcUrl = System.getenv("EASYFLOW_JOB_IT_JDBC_URL");
|
||||
username = System.getenv("EASYFLOW_JOB_IT_USERNAME");
|
||||
password = System.getenv("EASYFLOW_JOB_IT_PASSWORD");
|
||||
Assume.assumeTrue("未配置真实 MySQL 集成测试环境",
|
||||
jdbcUrl != null && username != null && password != null);
|
||||
long suffix = Math.abs(UUID.randomUUID().getMostSignificantBits() % 100_000_000L);
|
||||
jobId = 8_100_000_000_000_000L + suffix * 10L;
|
||||
firstExecutionId = jobId + 1L;
|
||||
secondExecutionId = jobId + 2L;
|
||||
registrationExecutionId = jobId + 3L;
|
||||
generationExecutionId = jobId + 4L;
|
||||
try (Connection connection = open(); Statement statement = connection.createStatement()) {
|
||||
try (ResultSet result = statement.executeQuery("SELECT VERSION()")) {
|
||||
Assert.assertTrue(result.next());
|
||||
Assert.assertTrue("需要 MySQL 8,实际为 " + result.getString(1),
|
||||
result.getString(1).startsWith("8."));
|
||||
}
|
||||
insertFixtures(connection);
|
||||
}
|
||||
}
|
||||
|
||||
@After
|
||||
public void tearDown() throws Exception {
|
||||
if (jdbcUrl == null || jobId == 0L) return;
|
||||
try (Connection connection = open(); PreparedStatement logs = connection.prepareStatement(
|
||||
"DELETE FROM tb_sys_job_log WHERE job_id=?");
|
||||
PreparedStatement job = connection.prepareStatement(
|
||||
"DELETE FROM tb_sys_job WHERE id=?")) {
|
||||
logs.setLong(1, jobId);
|
||||
logs.executeUpdate();
|
||||
job.setLong(1, jobId);
|
||||
job.executeUpdate();
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void shouldEnforceSkipLockedNonConcurrencyAndFencing() throws Exception {
|
||||
verifySkipLockedClaim();
|
||||
resetExecutions();
|
||||
verifyUnlockedExpiredProbeDoesNotDeadlockPendingClaims();
|
||||
resetExecutions();
|
||||
verifyLockedExpiredCandidateDoesNotStarvePendingClaim();
|
||||
resetExecutions();
|
||||
verifyNonConcurrentJobRowLock();
|
||||
verifyOwnerTokenFencing();
|
||||
verifyManagementCommandRowLock();
|
||||
verifyManagementCommitsBeforeLedgerCleanup();
|
||||
verifyCleanupUsesLedgerThenJobLock();
|
||||
verifyRegistrationWaitsForStartCommit();
|
||||
verifyStaleCleanupWaitsForStartCommit();
|
||||
verifyDefinitionEditCannotResurrectConcurrentStop();
|
||||
verifyRestartClearsOldPendingGeneration();
|
||||
verifyOldFireCannotCrossRestartGeneration();
|
||||
verifyOldFireCannotCrossDefinitionUpdate();
|
||||
verifyRegisteredExecutionSurvivesDefinitionGeneration();
|
||||
verifyExpiredRegisteredExecutionSurvivesDefinitionGeneration();
|
||||
verifyDuplicateExecutionKeyCanBeReadInSameTransaction();
|
||||
}
|
||||
|
||||
private void verifySkipLockedClaim() throws Exception {
|
||||
try (Connection first = transactional(); Connection second = transactional()) {
|
||||
Assert.assertEquals(firstExecutionId, selectPendingForUpdate(first));
|
||||
Assert.assertEquals(secondExecutionId, selectPendingForUpdate(second));
|
||||
Assert.assertEquals(1, claim(first, firstExecutionId, "node-a", "token-a"));
|
||||
first.commit();
|
||||
Assert.assertEquals(0, claim(second, firstExecutionId, "node-b", "token-b"));
|
||||
second.rollback();
|
||||
}
|
||||
}
|
||||
|
||||
private void verifyUnlockedExpiredProbeDoesNotDeadlockPendingClaims() throws Exception {
|
||||
ExecutorService executor = Executors.newFixedThreadPool(2);
|
||||
CountDownLatch ready = new CountDownLatch(2);
|
||||
CountDownLatch start = new CountDownLatch(1);
|
||||
try (Connection first = transactional(); Connection second = transactional()) {
|
||||
Assert.assertEquals(0L, selectExpiredWithoutLock(first));
|
||||
Assert.assertEquals(0L, selectExpiredWithoutLock(second));
|
||||
Assert.assertEquals(firstExecutionId, selectPendingForUpdate(first));
|
||||
Assert.assertEquals(secondExecutionId, selectPendingForUpdate(second));
|
||||
Future<Integer> firstClaim = executor.submit(() -> {
|
||||
ready.countDown();
|
||||
start.await(5, TimeUnit.SECONDS);
|
||||
int claimed = claim(first, firstExecutionId, "node-a", "probe-a");
|
||||
first.commit();
|
||||
return claimed;
|
||||
});
|
||||
Future<Integer> secondClaim = executor.submit(() -> {
|
||||
ready.countDown();
|
||||
start.await(5, TimeUnit.SECONDS);
|
||||
int claimed = claim(second, secondExecutionId, "node-b", "probe-b");
|
||||
second.commit();
|
||||
return claimed;
|
||||
});
|
||||
Assert.assertTrue(ready.await(5, TimeUnit.SECONDS));
|
||||
start.countDown();
|
||||
Assert.assertEquals(Integer.valueOf(1), firstClaim.get(5, TimeUnit.SECONDS));
|
||||
Assert.assertEquals(Integer.valueOf(1), secondClaim.get(5, TimeUnit.SECONDS));
|
||||
} finally {
|
||||
executor.shutdownNow();
|
||||
}
|
||||
}
|
||||
|
||||
private void verifyLockedExpiredCandidateDoesNotStarvePendingClaim() throws Exception {
|
||||
try (Connection connection = open(); PreparedStatement statement = connection.prepareStatement(
|
||||
"UPDATE tb_sys_job_log SET status=3,lease_owner='locked-owner',"
|
||||
+ "execution_token='locked-token',"
|
||||
+ "lease_until=TIMESTAMPADD(SECOND,-1,CURRENT_TIMESTAMP(3)) WHERE id=?")) {
|
||||
statement.setLong(1, firstExecutionId);
|
||||
Assert.assertEquals(1, statement.executeUpdate());
|
||||
}
|
||||
try (Connection lockOwner = transactional(); Connection claimant = transactional()) {
|
||||
lockExecution(lockOwner, firstExecutionId);
|
||||
Assert.assertEquals("无锁探测应仍能看见已提交的过期记录",
|
||||
firstExecutionId, selectExpiredWithoutLock(claimant));
|
||||
Assert.assertEquals("SKIP LOCKED 应跳过其他 Worker 持有的过期记录",
|
||||
0L, selectExpiredForUpdate(claimant));
|
||||
Assert.assertEquals("受阻的过期记录不得阻塞 PENDING 队列",
|
||||
secondExecutionId, selectPendingForUpdate(claimant));
|
||||
Assert.assertEquals(1,
|
||||
claim(claimant, secondExecutionId, "pending-node", "pending-token"));
|
||||
claimant.commit();
|
||||
lockOwner.rollback();
|
||||
}
|
||||
}
|
||||
|
||||
private void verifyNonConcurrentJobRowLock() throws Exception {
|
||||
ExecutorService executor = Executors.newSingleThreadExecutor();
|
||||
try (Connection first = transactional()) {
|
||||
lockJob(first);
|
||||
Assert.assertEquals(1, claim(first, firstExecutionId, "node-a", "token-a"));
|
||||
Future<Integer> secondNode = executor.submit(() -> {
|
||||
try (Connection second = transactional()) {
|
||||
lockJob(second);
|
||||
int active = countOtherActive(second, secondExecutionId);
|
||||
second.commit();
|
||||
return active;
|
||||
}
|
||||
});
|
||||
Thread.sleep(200L);
|
||||
Assert.assertFalse("第二节点不应越过同任务行锁", secondNode.isDone());
|
||||
first.commit();
|
||||
Assert.assertEquals(Integer.valueOf(1), secondNode.get(5, TimeUnit.SECONDS));
|
||||
} finally {
|
||||
executor.shutdownNow();
|
||||
}
|
||||
}
|
||||
|
||||
private void verifyOwnerTokenFencing() throws Exception {
|
||||
try (Connection connection = open()) {
|
||||
try (PreparedStatement takeover = connection.prepareStatement(
|
||||
"UPDATE tb_sys_job_log SET status=3, lease_owner='new-owner', "
|
||||
+ "execution_token='new-token', lease_until=TIMESTAMPADD(SECOND,60,CURRENT_TIMESTAMP(3)) "
|
||||
+ "WHERE id=?")) {
|
||||
takeover.setLong(1, firstExecutionId);
|
||||
Assert.assertEquals(1, takeover.executeUpdate());
|
||||
}
|
||||
Assert.assertEquals(0, finish(connection, "old-owner", "old-token"));
|
||||
Assert.assertEquals(1, finish(connection, "new-owner", "new-token"));
|
||||
}
|
||||
}
|
||||
|
||||
private void verifyManagementCommandRowLock() throws Exception {
|
||||
ExecutorService executor = Executors.newSingleThreadExecutor();
|
||||
try (Connection first = transactional()) {
|
||||
lockJob(first);
|
||||
updateJobStatus(first, 0);
|
||||
Future<Integer> secondCommand = executor.submit(() -> {
|
||||
try (Connection second = transactional()) {
|
||||
lockJob(second);
|
||||
int observed = readJobStatus(second);
|
||||
updateJobStatus(second, 1);
|
||||
second.commit();
|
||||
return observed;
|
||||
}
|
||||
});
|
||||
Thread.sleep(200L);
|
||||
Assert.assertFalse("并发管理命令不应在前一事务提交前执行", secondCommand.isDone());
|
||||
first.commit();
|
||||
Assert.assertEquals("后一命令必须读取前一命令已提交状态",
|
||||
Integer.valueOf(0), secondCommand.get(5, TimeUnit.SECONDS));
|
||||
} finally {
|
||||
executor.shutdownNow();
|
||||
}
|
||||
}
|
||||
|
||||
private void verifyManagementCommitsBeforeLedgerCleanup() throws Exception {
|
||||
resetExecutions();
|
||||
ExecutorService executor = Executors.newSingleThreadExecutor();
|
||||
try (Connection worker = transactional()) {
|
||||
Assert.assertEquals(firstExecutionId, selectPendingForUpdate(worker));
|
||||
Future<?> stopCommand = executor.submit(() -> {
|
||||
try (Connection management = transactional()) {
|
||||
lockJob(management);
|
||||
updateJobStatus(management, 0);
|
||||
management.commit();
|
||||
return null;
|
||||
}
|
||||
});
|
||||
// 管理事务不再反向触碰已被 Worker 锁定的日志行,因此应能先提交。
|
||||
stopCommand.get(5, TimeUnit.SECONDS);
|
||||
lockJob(worker);
|
||||
Assert.assertEquals(0, readJobStatus(worker));
|
||||
worker.rollback();
|
||||
} finally {
|
||||
executor.shutdownNow();
|
||||
}
|
||||
try (Connection connection = open()) {
|
||||
updateJobStatus(connection, 1);
|
||||
}
|
||||
try (Connection cleanup = transactional()) {
|
||||
Assert.assertEquals("RUNNING 任务不得清理待执行记录", 0,
|
||||
cancelPendingIfStopped(cleanup));
|
||||
cleanup.commit();
|
||||
}
|
||||
try (Connection connection = open()) {
|
||||
updateJobStatus(connection, 0);
|
||||
}
|
||||
try (Connection cleanup = transactional()) {
|
||||
Assert.assertEquals(2, cancelPendingIfStopped(cleanup));
|
||||
cleanup.commit();
|
||||
}
|
||||
try (Connection connection = open()) {
|
||||
updateJobStatus(connection, 1);
|
||||
}
|
||||
}
|
||||
|
||||
private void verifyRegistrationWaitsForStartCommit() throws Exception {
|
||||
try (Connection connection = open()) {
|
||||
updateJobStatus(connection, 0);
|
||||
}
|
||||
ExecutorService executor = Executors.newSingleThreadExecutor();
|
||||
try (Connection start = transactional()) {
|
||||
lockJob(start);
|
||||
updateJobStatus(start, 1);
|
||||
Future<Integer> registration = executor.submit(() -> {
|
||||
try (Connection fire = transactional()) {
|
||||
lockJob(fire);
|
||||
int observed = readJobStatus(fire);
|
||||
if (observed == 1) insertExecution(fire, registrationExecutionId);
|
||||
fire.commit();
|
||||
return observed;
|
||||
}
|
||||
});
|
||||
Thread.sleep(200L);
|
||||
Assert.assertFalse("Quartz 触发登记必须等待 START 事务提交", registration.isDone());
|
||||
start.commit();
|
||||
Assert.assertEquals("登记必须读取 START 提交后的 RUNNING 状态",
|
||||
Integer.valueOf(1), registration.get(5, TimeUnit.SECONDS));
|
||||
} finally {
|
||||
executor.shutdownNow();
|
||||
}
|
||||
try (Connection connection = open(); PreparedStatement statement = connection.prepareStatement(
|
||||
"SELECT COUNT(*) FROM tb_sys_job_log WHERE id=? AND status=2")) {
|
||||
statement.setLong(1, registrationExecutionId);
|
||||
try (ResultSet result = statement.executeQuery()) {
|
||||
Assert.assertTrue(result.next());
|
||||
Assert.assertEquals(1, result.getInt(1));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void verifyCleanupUsesLedgerThenJobLock() throws Exception {
|
||||
resetExecutions();
|
||||
try (Connection connection = open()) {
|
||||
updateJobStatus(connection, 0);
|
||||
}
|
||||
ExecutorService executor = Executors.newSingleThreadExecutor();
|
||||
try (Connection worker = transactional()) {
|
||||
Assert.assertEquals(firstExecutionId, selectPendingForUpdate(worker));
|
||||
Future<Integer> cleanup = executor.submit(() -> {
|
||||
try (Connection connection = transactional()) {
|
||||
int cancelled = cancelPendingIfStopped(connection);
|
||||
connection.commit();
|
||||
return cancelled;
|
||||
}
|
||||
});
|
||||
Thread.sleep(200L);
|
||||
Assert.assertFalse("清理应先等待 Worker 持有的账本行", cleanup.isDone());
|
||||
// 清理尚未持有任务行,因此 Worker 可继续按 log -> job 顺序完成。
|
||||
lockJob(worker);
|
||||
worker.rollback();
|
||||
Assert.assertEquals(Integer.valueOf(2), cleanup.get(5, TimeUnit.SECONDS));
|
||||
} finally {
|
||||
executor.shutdownNow();
|
||||
}
|
||||
}
|
||||
|
||||
private void verifyStaleCleanupWaitsForStartCommit() throws Exception {
|
||||
resetExecutions();
|
||||
try (Connection connection = open()) {
|
||||
updateJobStatus(connection, 0);
|
||||
}
|
||||
ExecutorService executor = Executors.newSingleThreadExecutor();
|
||||
try (Connection start = transactional()) {
|
||||
lockJob(start);
|
||||
updateJobStatus(start, 1);
|
||||
Future<Integer> staleCleanup = executor.submit(() -> {
|
||||
try (Connection cleanup = transactional()) {
|
||||
lockJob(cleanup);
|
||||
int cancelled = cancelPendingIfStopped(cleanup);
|
||||
cleanup.commit();
|
||||
return cancelled;
|
||||
}
|
||||
});
|
||||
Thread.sleep(200L);
|
||||
Assert.assertFalse("旧清理命令必须等待并发 START 事务提交", staleCleanup.isDone());
|
||||
start.commit();
|
||||
Assert.assertEquals("旧清理命令不得取消新一代 RUNNING 记录",
|
||||
Integer.valueOf(0), staleCleanup.get(5, TimeUnit.SECONDS));
|
||||
} finally {
|
||||
executor.shutdownNow();
|
||||
}
|
||||
}
|
||||
|
||||
private void verifyRestartClearsOldPendingGeneration() throws Exception {
|
||||
resetExecutions();
|
||||
try (Connection connection = open()) {
|
||||
updateJobStatus(connection, 0);
|
||||
}
|
||||
try (Connection cleanup = transactional()) {
|
||||
Assert.assertTrue("重新启动前应清理上一代全部 PENDING",
|
||||
cancelPendingIfStopped(cleanup) >= 2);
|
||||
cleanup.commit();
|
||||
}
|
||||
try (Connection start = transactional()) {
|
||||
lockJob(start);
|
||||
startNextGeneration(start);
|
||||
start.commit();
|
||||
}
|
||||
try (Connection connection = open(); PreparedStatement statement = connection.prepareStatement(
|
||||
"SELECT COUNT(*) FROM tb_sys_job_log WHERE job_id=? AND status=2")) {
|
||||
statement.setLong(1, jobId);
|
||||
try (ResultSet result = statement.executeQuery()) {
|
||||
Assert.assertTrue(result.next());
|
||||
Assert.assertEquals("旧 PENDING 不得被新 RUNNING 代际重新放行",
|
||||
0, result.getInt(1));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void verifyDefinitionEditCannotResurrectConcurrentStop() throws Exception {
|
||||
try (Connection connection = open()) {
|
||||
updateJobStatus(connection, 1);
|
||||
}
|
||||
ExecutorService executor = Executors.newSingleThreadExecutor();
|
||||
try (Connection stop = transactional()) {
|
||||
lockJob(stop);
|
||||
updateJobStatus(stop, 0);
|
||||
Future<Integer> edit = executor.submit(() -> {
|
||||
try (Connection connection = transactional()) {
|
||||
lockJob(connection);
|
||||
int observedStatus = readJobStatus(connection);
|
||||
advanceDefinitionGeneration(connection);
|
||||
connection.commit();
|
||||
return observedStatus;
|
||||
}
|
||||
});
|
||||
Thread.sleep(200L);
|
||||
Assert.assertFalse("定义编辑必须等待并发 STOP 提交", edit.isDone());
|
||||
stop.commit();
|
||||
Assert.assertEquals("编辑只能保留行锁下的最终 STOP 状态",
|
||||
Integer.valueOf(0), edit.get(5, TimeUnit.SECONDS));
|
||||
} finally {
|
||||
executor.shutdownNow();
|
||||
}
|
||||
try (Connection connection = open()) {
|
||||
Assert.assertEquals("普通编辑不得复活已停止任务", 0, readJobStatus(connection));
|
||||
updateJobStatus(connection, 1);
|
||||
}
|
||||
}
|
||||
|
||||
private void verifyOldFireCannotCrossRestartGeneration() throws Exception {
|
||||
long oldGeneration;
|
||||
try (Connection connection = open()) {
|
||||
oldGeneration = readJobGeneration(connection);
|
||||
updateJobStatus(connection, 0);
|
||||
}
|
||||
try (Connection start = transactional()) {
|
||||
lockJob(start);
|
||||
startNextGeneration(start);
|
||||
start.commit();
|
||||
}
|
||||
try (Connection fire = transactional()) {
|
||||
Assert.assertEquals("STOP 前取得的 Quartz fire 不得登记到新启停代际", 0,
|
||||
registerIfGenerationMatches(fire, generationExecutionId, oldGeneration));
|
||||
fire.commit();
|
||||
}
|
||||
}
|
||||
|
||||
private void verifyOldFireCannotCrossDefinitionUpdate() throws Exception {
|
||||
long oldGeneration;
|
||||
try (Connection edit = transactional()) {
|
||||
lockJob(edit);
|
||||
oldGeneration = readJobGeneration(edit);
|
||||
advanceDefinitionGeneration(edit);
|
||||
edit.commit();
|
||||
}
|
||||
try (Connection oldFire = transactional()) {
|
||||
Assert.assertEquals("编辑提交前取得的 Quartz fire 不得读取新定义并登记", 0,
|
||||
registerIfGenerationMatches(oldFire, generationExecutionId + 1L, oldGeneration));
|
||||
oldFire.commit();
|
||||
}
|
||||
try (Connection currentFire = transactional()) {
|
||||
long currentGeneration = readJobGeneration(currentFire);
|
||||
Assert.assertEquals(oldGeneration + 1L, currentGeneration);
|
||||
Assert.assertEquals("当前定义代际的 fire 应可正常登记", 1,
|
||||
registerIfGenerationMatches(
|
||||
currentFire, generationExecutionId + 2L, currentGeneration));
|
||||
currentFire.commit();
|
||||
}
|
||||
}
|
||||
|
||||
private void verifyRegisteredExecutionSurvivesDefinitionGeneration() throws Exception {
|
||||
long oldGeneration;
|
||||
try (Connection connection = open()) {
|
||||
oldGeneration = readJobGeneration(connection);
|
||||
insertExecution(connection, generationExecutionId + 3L, oldGeneration);
|
||||
}
|
||||
try (Connection edit = transactional()) {
|
||||
lockJob(edit);
|
||||
advanceDefinitionGeneration(edit);
|
||||
edit.commit();
|
||||
}
|
||||
try (Connection connection = open()) {
|
||||
Assert.assertTrue(readJobGeneration(connection) > oldGeneration);
|
||||
Assert.assertEquals("已登记快照不得因后续编辑失去领取资格", 1,
|
||||
claim(connection, generationExecutionId + 3L,
|
||||
"generation-node", "generation-token"));
|
||||
}
|
||||
}
|
||||
|
||||
private void verifyExpiredRegisteredExecutionSurvivesDefinitionGeneration() throws Exception {
|
||||
long executionId = generationExecutionId + 4L;
|
||||
long oldGeneration;
|
||||
try (Connection connection = open()) {
|
||||
oldGeneration = readJobGeneration(connection);
|
||||
insertExecution(connection, executionId, oldGeneration);
|
||||
try (PreparedStatement statement = connection.prepareStatement(
|
||||
"UPDATE tb_sys_job_log SET status=3,lease_owner='expired-owner',"
|
||||
+ "execution_token='expired-token',"
|
||||
+ "lease_until=TIMESTAMPADD(SECOND,-1,CURRENT_TIMESTAMP(3)) WHERE id=?")) {
|
||||
statement.setLong(1, executionId);
|
||||
Assert.assertEquals(1, statement.executeUpdate());
|
||||
}
|
||||
}
|
||||
try (Connection edit = transactional()) {
|
||||
lockJob(edit);
|
||||
advanceDefinitionGeneration(edit);
|
||||
edit.commit();
|
||||
}
|
||||
try (Connection takeover = transactional()) {
|
||||
Assert.assertEquals("已登记的过期 RUNNING 不得因定义代际变化失去接管资格",
|
||||
executionId, selectExpiredForUpdate(takeover));
|
||||
Assert.assertEquals(1, claimExpired(
|
||||
takeover, executionId, "takeover-owner", "takeover-token"));
|
||||
takeover.commit();
|
||||
}
|
||||
try (Connection connection = open()) {
|
||||
Assert.assertEquals("旧租约持有者不得覆盖接管后的终态", 0,
|
||||
finish(connection, executionId, "expired-owner", "expired-token"));
|
||||
Assert.assertEquals(1,
|
||||
finish(connection, executionId, "takeover-owner", "takeover-token"));
|
||||
}
|
||||
}
|
||||
|
||||
private void verifyDuplicateExecutionKeyCanBeReadInSameTransaction() throws Exception {
|
||||
long originalId = generationExecutionId + 10L;
|
||||
long conflictingId = generationExecutionId + 11L;
|
||||
String executionKey = String.format("%064x", originalId);
|
||||
try (Connection connection = open()) {
|
||||
insertExecution(connection, originalId, readJobGeneration(connection), executionKey);
|
||||
}
|
||||
try (Connection connection = transactional()) {
|
||||
SQLException duplicate = Assert.assertThrows(
|
||||
SQLException.class,
|
||||
() -> insertExecution(
|
||||
connection, conflictingId, readJobGeneration(connection), executionKey));
|
||||
Assert.assertEquals("23000", duplicate.getSQLState());
|
||||
try (PreparedStatement statement = connection.prepareStatement(
|
||||
"SELECT id FROM tb_sys_job_log WHERE execution_key=?")) {
|
||||
statement.setString(1, executionKey);
|
||||
try (ResultSet result = statement.executeQuery()) {
|
||||
Assert.assertTrue("唯一键冲突后同一事务仍应能复查 execution_key",
|
||||
result.next());
|
||||
Assert.assertEquals(originalId, result.getLong(1));
|
||||
}
|
||||
}
|
||||
connection.rollback();
|
||||
}
|
||||
}
|
||||
|
||||
private void insertFixtures(Connection connection) throws SQLException {
|
||||
try (PreparedStatement job = connection.prepareStatement(
|
||||
"INSERT INTO tb_sys_job(id,dept_id,tenant_id,job_name,job_type,job_params,"
|
||||
+ "cron_expression,allow_concurrent,misfire_policy,options,status,created,"
|
||||
+ "schedule_generation,created_by,modified,modified_by,remark) "
|
||||
+ "VALUES(?,1,1,'L25 MySQL IT',3,'{}','0 0 0 1 1 ? 2099',0,3,'{}',1,NOW(),0,1,NOW(),1,'')")) {
|
||||
job.setLong(1, jobId);
|
||||
job.executeUpdate();
|
||||
}
|
||||
insertExecution(connection, firstExecutionId, 0L);
|
||||
insertExecution(connection, secondExecutionId, 0L);
|
||||
}
|
||||
|
||||
private void insertExecution(Connection connection, long id) throws SQLException {
|
||||
insertExecution(connection, id, readJobGeneration(connection));
|
||||
}
|
||||
|
||||
private void insertExecution(Connection connection, long id, long generation) throws SQLException {
|
||||
insertExecution(connection, id, generation, String.format("%064x", id));
|
||||
}
|
||||
|
||||
private void insertExecution(Connection connection, long id, long generation,
|
||||
String executionKey) throws SQLException {
|
||||
try (PreparedStatement execution = connection.prepareStatement(
|
||||
"INSERT INTO tb_sys_job_log(id,execution_key,job_id,job_generation,tenant_id,dept_id,job_name,"
|
||||
+ "job_type,job_params,job_options,allow_concurrent,trigger_source,"
|
||||
+ "scheduled_fire_time,actual_fire_time,fire_instance_id,recovering,"
|
||||
+ "attempt_count,next_retry_time,version,status,created,remark) "
|
||||
+ "VALUES(?,?,?,?,1,1,'L25 MySQL IT',3,'{}','{}',0,'MANUAL',"
|
||||
+ "CURRENT_TIMESTAMP(3),CURRENT_TIMESTAMP(3),?,0,0,CURRENT_TIMESTAMP(3),0,2,NOW(),'')")) {
|
||||
execution.setLong(1, id);
|
||||
execution.setString(2, executionKey);
|
||||
execution.setLong(3, jobId);
|
||||
execution.setLong(4, generation);
|
||||
execution.setString(5, "fire-" + id);
|
||||
execution.executeUpdate();
|
||||
}
|
||||
}
|
||||
|
||||
private int registerIfGenerationMatches(Connection connection, long executionId,
|
||||
long fireGeneration) throws SQLException {
|
||||
lockJob(connection);
|
||||
if (readJobStatus(connection) != 1
|
||||
|| readJobGeneration(connection) != fireGeneration) {
|
||||
return 0;
|
||||
}
|
||||
insertExecution(connection, executionId, fireGeneration);
|
||||
return 1;
|
||||
}
|
||||
|
||||
private void startNextGeneration(Connection connection) throws SQLException {
|
||||
try (PreparedStatement statement = connection.prepareStatement(
|
||||
"UPDATE tb_sys_job SET status=1,schedule_generation=schedule_generation+1 "
|
||||
+ "WHERE id=? AND status<>1")) {
|
||||
statement.setLong(1, jobId);
|
||||
Assert.assertEquals(1, statement.executeUpdate());
|
||||
}
|
||||
}
|
||||
|
||||
private void advanceDefinitionGeneration(Connection connection) throws SQLException {
|
||||
try (PreparedStatement statement = connection.prepareStatement(
|
||||
"UPDATE tb_sys_job SET schedule_generation=schedule_generation+1 WHERE id=?")) {
|
||||
statement.setLong(1, jobId);
|
||||
Assert.assertEquals(1, statement.executeUpdate());
|
||||
}
|
||||
}
|
||||
|
||||
private void resetExecutions() throws SQLException {
|
||||
try (Connection connection = open(); PreparedStatement statement = connection.prepareStatement(
|
||||
"UPDATE tb_sys_job_log SET status=2,lease_owner=NULL,execution_token=NULL,"
|
||||
+ "lease_until=NULL,next_retry_time=CURRENT_TIMESTAMP(3) WHERE job_id=?")) {
|
||||
statement.setLong(1, jobId);
|
||||
statement.executeUpdate();
|
||||
}
|
||||
}
|
||||
|
||||
private long selectPendingForUpdate(Connection connection) throws SQLException {
|
||||
try (PreparedStatement statement = connection.prepareStatement(
|
||||
"SELECT id FROM tb_sys_job_log FORCE INDEX(PRIMARY) WHERE id IN (?,?) "
|
||||
+ "AND job_id=? AND status=2 "
|
||||
+ "ORDER BY id LIMIT 1 FOR UPDATE SKIP LOCKED")) {
|
||||
statement.setLong(1, firstExecutionId);
|
||||
statement.setLong(2, secondExecutionId);
|
||||
statement.setLong(3, jobId);
|
||||
try (ResultSet result = statement.executeQuery()) {
|
||||
Assert.assertTrue(result.next());
|
||||
return result.getLong(1);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private long selectExpiredForUpdate(Connection connection) throws SQLException {
|
||||
try (PreparedStatement statement = connection.prepareStatement(
|
||||
"SELECT id FROM tb_sys_job_log WHERE status=3 "
|
||||
+ "AND lease_until<=CURRENT_TIMESTAMP(3) "
|
||||
+ "ORDER BY lease_until,id LIMIT 1 FOR UPDATE SKIP LOCKED")) {
|
||||
try (ResultSet result = statement.executeQuery()) {
|
||||
return result.next() ? result.getLong(1) : 0L;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void lockExecution(Connection connection, long executionId) throws SQLException {
|
||||
try (PreparedStatement statement = connection.prepareStatement(
|
||||
"SELECT id FROM tb_sys_job_log WHERE id=? FOR UPDATE")) {
|
||||
statement.setLong(1, executionId);
|
||||
try (ResultSet result = statement.executeQuery()) {
|
||||
Assert.assertTrue(result.next());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private long selectExpiredWithoutLock(Connection connection) throws SQLException {
|
||||
try (PreparedStatement statement = connection.prepareStatement(
|
||||
"SELECT id FROM tb_sys_job_log WHERE status=3 "
|
||||
+ "AND lease_until<=CURRENT_TIMESTAMP(3) "
|
||||
+ "ORDER BY lease_until,id LIMIT 1")) {
|
||||
try (ResultSet result = statement.executeQuery()) {
|
||||
return result.next() ? result.getLong(1) : 0L;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private int claim(Connection connection, long id, String owner, String token) throws SQLException {
|
||||
try (PreparedStatement statement = connection.prepareStatement(
|
||||
"UPDATE tb_sys_job_log SET status=3,lease_owner=?,execution_token=?,"
|
||||
+ "lease_until=TIMESTAMPADD(SECOND,60,CURRENT_TIMESTAMP(3)) "
|
||||
+ "WHERE id=? AND status=2")) {
|
||||
statement.setString(1, owner);
|
||||
statement.setString(2, token);
|
||||
statement.setLong(3, id);
|
||||
return statement.executeUpdate();
|
||||
}
|
||||
}
|
||||
|
||||
private int claimExpired(Connection connection, long id, String owner,
|
||||
String token) throws SQLException {
|
||||
try (PreparedStatement statement = connection.prepareStatement(
|
||||
"UPDATE tb_sys_job_log SET lease_owner=?,execution_token=?,"
|
||||
+ "lease_until=TIMESTAMPADD(SECOND,60,CURRENT_TIMESTAMP(3)),"
|
||||
+ "attempt_count=attempt_count+1 WHERE id=? AND status=3 "
|
||||
+ "AND lease_until<=CURRENT_TIMESTAMP(3)")) {
|
||||
statement.setString(1, owner);
|
||||
statement.setString(2, token);
|
||||
statement.setLong(3, id);
|
||||
return statement.executeUpdate();
|
||||
}
|
||||
}
|
||||
|
||||
private void lockJob(Connection connection) throws SQLException {
|
||||
try (PreparedStatement statement = connection.prepareStatement(
|
||||
"SELECT id FROM tb_sys_job WHERE id=? FOR UPDATE")) {
|
||||
statement.setLong(1, jobId);
|
||||
try (ResultSet result = statement.executeQuery()) {
|
||||
Assert.assertTrue(result.next());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private int countOtherActive(Connection connection, long excludedId) throws SQLException {
|
||||
try (PreparedStatement statement = connection.prepareStatement(
|
||||
"SELECT COUNT(*) FROM tb_sys_job_log WHERE job_id=? AND id<>? AND status=3 "
|
||||
+ "AND lease_until>CURRENT_TIMESTAMP(3) FOR UPDATE")) {
|
||||
statement.setLong(1, jobId);
|
||||
statement.setLong(2, excludedId);
|
||||
try (ResultSet result = statement.executeQuery()) {
|
||||
Assert.assertTrue(result.next());
|
||||
return result.getInt(1);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private int finish(Connection connection, String owner, String token) throws SQLException {
|
||||
return finish(connection, firstExecutionId, owner, token);
|
||||
}
|
||||
|
||||
private int finish(Connection connection, long executionId,
|
||||
String owner, String token) throws SQLException {
|
||||
try (PreparedStatement statement = connection.prepareStatement(
|
||||
"UPDATE tb_sys_job_log SET status=1,end_time=CURRENT_TIMESTAMP(3) WHERE id=? "
|
||||
+ "AND status=3 AND lease_owner=? AND execution_token=?")) {
|
||||
statement.setLong(1, executionId);
|
||||
statement.setString(2, owner);
|
||||
statement.setString(3, token);
|
||||
return statement.executeUpdate();
|
||||
}
|
||||
}
|
||||
|
||||
private int cancelPendingIfStopped(Connection connection) throws SQLException {
|
||||
try (PreparedStatement statement = connection.prepareStatement(
|
||||
"SELECT id FROM tb_sys_job_log WHERE job_id=? AND status=2 "
|
||||
+ "ORDER BY id FOR UPDATE")) {
|
||||
statement.setLong(1, jobId);
|
||||
try (ResultSet ignored = statement.executeQuery()) {
|
||||
while (ignored.next()) {
|
||||
// 读取完整结果集并保持行锁到事务结束。
|
||||
}
|
||||
}
|
||||
}
|
||||
lockJob(connection);
|
||||
if (readJobStatus(connection) != 0) return 0;
|
||||
try (PreparedStatement statement = connection.prepareStatement(
|
||||
"UPDATE tb_sys_job_log SET status=5 WHERE job_id=? AND status=2")) {
|
||||
statement.setLong(1, jobId);
|
||||
return statement.executeUpdate();
|
||||
}
|
||||
}
|
||||
|
||||
private void updateJobStatus(Connection connection, int status) throws SQLException {
|
||||
try (PreparedStatement statement = connection.prepareStatement(
|
||||
"UPDATE tb_sys_job SET status=? WHERE id=?")) {
|
||||
statement.setInt(1, status);
|
||||
statement.setLong(2, jobId);
|
||||
Assert.assertEquals(1, statement.executeUpdate());
|
||||
}
|
||||
}
|
||||
|
||||
private int readJobStatus(Connection connection) throws SQLException {
|
||||
try (PreparedStatement statement = connection.prepareStatement(
|
||||
"SELECT status FROM tb_sys_job WHERE id=?")) {
|
||||
statement.setLong(1, jobId);
|
||||
try (ResultSet result = statement.executeQuery()) {
|
||||
Assert.assertTrue(result.next());
|
||||
return result.getInt(1);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private long readJobGeneration(Connection connection) throws SQLException {
|
||||
try (PreparedStatement statement = connection.prepareStatement(
|
||||
"SELECT schedule_generation FROM tb_sys_job WHERE id=?")) {
|
||||
statement.setLong(1, jobId);
|
||||
try (ResultSet result = statement.executeQuery()) {
|
||||
Assert.assertTrue(result.next());
|
||||
return result.getLong(1);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private Connection transactional() throws SQLException {
|
||||
Connection connection = open();
|
||||
connection.setAutoCommit(false);
|
||||
connection.setTransactionIsolation(Connection.TRANSACTION_READ_COMMITTED);
|
||||
return connection;
|
||||
}
|
||||
|
||||
private Connection open() throws SQLException {
|
||||
return DriverManager.getConnection(jdbcUrl, username, password);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,75 @@
|
||||
package tech.easyflow.job.execution;
|
||||
|
||||
import org.junit.Assert;
|
||||
import org.junit.Test;
|
||||
import tech.easyflow.common.constant.enums.EnumDataStatus;
|
||||
import tech.easyflow.job.entity.SysJob;
|
||||
import tech.easyflow.system.entity.SysAccount;
|
||||
import tech.easyflow.system.service.SysAccountService;
|
||||
|
||||
import java.math.BigInteger;
|
||||
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
public class SysJobOwnerValidatorTest {
|
||||
|
||||
@Test
|
||||
public void missingOwnerMustBeRejected() {
|
||||
SysAccountService accountService = mock(SysAccountService.class);
|
||||
SysJob job = job();
|
||||
when(accountService.getById(job.getCreatedBy())).thenReturn(null);
|
||||
|
||||
IllegalStateException exception = Assert.assertThrows(
|
||||
IllegalStateException.class,
|
||||
() -> new SysJobOwnerValidator(accountService).requireAvailableOwner(job));
|
||||
|
||||
Assert.assertTrue(exception.getMessage().contains("不存在"));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void disabledOwnerMustBeRejected() {
|
||||
SysAccountService accountService = mock(SysAccountService.class);
|
||||
SysJob job = job();
|
||||
SysAccount account = account(job.getCreatedBy(), job.getTenantId(), -1);
|
||||
when(accountService.getById(job.getCreatedBy())).thenReturn(account);
|
||||
|
||||
IllegalStateException exception = Assert.assertThrows(
|
||||
IllegalStateException.class,
|
||||
() -> new SysJobOwnerValidator(accountService).requireAvailableOwner(job));
|
||||
|
||||
Assert.assertTrue(exception.getMessage().contains("未启用"));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void crossTenantOwnerMustBeRejected() {
|
||||
SysAccountService accountService = mock(SysAccountService.class);
|
||||
SysJob job = job();
|
||||
SysAccount account = account(
|
||||
job.getCreatedBy(), BigInteger.valueOf(999L),
|
||||
EnumDataStatus.AVAILABLE.getCode());
|
||||
when(accountService.getById(job.getCreatedBy())).thenReturn(account);
|
||||
|
||||
IllegalStateException exception = Assert.assertThrows(
|
||||
IllegalStateException.class,
|
||||
() -> new SysJobOwnerValidator(accountService).requireAvailableOwner(job));
|
||||
|
||||
Assert.assertTrue(exception.getMessage().contains("租户"));
|
||||
}
|
||||
|
||||
private static SysJob job() {
|
||||
SysJob job = new SysJob();
|
||||
job.setId(BigInteger.valueOf(101L));
|
||||
job.setTenantId(BigInteger.valueOf(201L));
|
||||
job.setCreatedBy(BigInteger.valueOf(301L));
|
||||
return job;
|
||||
}
|
||||
|
||||
private static SysAccount account(BigInteger id, BigInteger tenantId, int status) {
|
||||
SysAccount account = new SysAccount();
|
||||
account.setId(id);
|
||||
account.setTenantId(tenantId);
|
||||
account.setStatus(status);
|
||||
return account;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,54 @@
|
||||
package tech.easyflow.job.mapper;
|
||||
|
||||
import org.junit.Assert;
|
||||
import org.junit.Test;
|
||||
import tech.easyflow.job.entity.SysJobLog;
|
||||
|
||||
import java.math.BigInteger;
|
||||
|
||||
import static org.mockito.Mockito.CALLS_REAL_METHODS;
|
||||
import static org.mockito.Mockito.doReturn;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.verify;
|
||||
|
||||
public class SysJobLogMapperTest {
|
||||
|
||||
@Test
|
||||
public void pendingClaimShouldReadLockedIdThroughEntityResultMap() {
|
||||
SysJobLogMapper mapper = mock(SysJobLogMapper.class, CALLS_REAL_METHODS);
|
||||
BigInteger id = BigInteger.valueOf(31L);
|
||||
SysJobLog expected = execution(id);
|
||||
doReturn(id).when(mapper).selectPendingClaimCandidateId();
|
||||
doReturn(expected).when(mapper).selectOneById(id);
|
||||
|
||||
Assert.assertSame(expected, mapper.selectPendingClaimCandidate());
|
||||
verify(mapper).selectOneById(id);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void expiredClaimShouldReadLockedIdThroughEntityResultMap() {
|
||||
SysJobLogMapper mapper = mock(SysJobLogMapper.class, CALLS_REAL_METHODS);
|
||||
BigInteger id = BigInteger.valueOf(32L);
|
||||
SysJobLog expected = execution(id);
|
||||
doReturn(id).when(mapper).selectExpiredClaimCandidateId();
|
||||
doReturn(expected).when(mapper).selectOneById(id);
|
||||
|
||||
Assert.assertSame(expected, mapper.selectExpiredClaimCandidate());
|
||||
verify(mapper).selectOneById(id);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void emptyClaimShouldNotIssueEntityRead() {
|
||||
SysJobLogMapper mapper = mock(SysJobLogMapper.class, CALLS_REAL_METHODS);
|
||||
doReturn(null).when(mapper).selectPendingClaimCandidateId();
|
||||
|
||||
Assert.assertNull(mapper.selectPendingClaimCandidate());
|
||||
}
|
||||
|
||||
private static SysJobLog execution(BigInteger id) {
|
||||
SysJobLog execution = new SysJobLog();
|
||||
execution.setId(id);
|
||||
execution.setJobId(BigInteger.TEN);
|
||||
return execution;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,65 @@
|
||||
package tech.easyflow.job.mapper;
|
||||
|
||||
import com.mybatisflex.core.query.QueryWrapper;
|
||||
import org.junit.Assert;
|
||||
import org.junit.Test;
|
||||
import org.mockito.ArgumentCaptor;
|
||||
import tech.easyflow.job.entity.SysJob;
|
||||
|
||||
import java.math.BigInteger;
|
||||
import java.util.List;
|
||||
import java.util.Locale;
|
||||
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.Mockito.CALLS_REAL_METHODS;
|
||||
import static org.mockito.Mockito.doReturn;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.verify;
|
||||
|
||||
public class SysJobMapperTest {
|
||||
|
||||
@Test
|
||||
public void lockQueryShouldDelegateToEntityMappedBaseMapper() {
|
||||
SysJobMapper mapper = mock(SysJobMapper.class, CALLS_REAL_METHODS);
|
||||
SysJob expected = job(17L);
|
||||
doReturn(expected).when(mapper).selectOneByQuery(any(QueryWrapper.class));
|
||||
|
||||
SysJob actual = mapper.selectByIdForUpdate(BigInteger.valueOf(17L));
|
||||
|
||||
Assert.assertSame(expected, actual);
|
||||
ArgumentCaptor<QueryWrapper> query = ArgumentCaptor.forClass(QueryWrapper.class);
|
||||
verify(mapper).selectOneByQuery(query.capture());
|
||||
String sql = normalizedSql(query.getValue());
|
||||
Assert.assertTrue(sql.contains("ID = 17"));
|
||||
Assert.assertTrue(sql.contains("FOR UPDATE"));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void reconciliationShouldUseEntityMappingAndKeysetPage() {
|
||||
SysJobMapper mapper = mock(SysJobMapper.class, CALLS_REAL_METHODS);
|
||||
List<SysJob> expected = List.of(job(18L));
|
||||
doReturn(expected).when(mapper).selectListByQuery(any(QueryWrapper.class));
|
||||
|
||||
List<SysJob> actual = mapper.selectReconciliationPage(
|
||||
BigInteger.valueOf(17L), 200);
|
||||
|
||||
Assert.assertSame(expected, actual);
|
||||
ArgumentCaptor<QueryWrapper> query = ArgumentCaptor.forClass(QueryWrapper.class);
|
||||
verify(mapper).selectListByQuery(query.capture());
|
||||
String sql = normalizedSql(query.getValue());
|
||||
Assert.assertTrue(sql.contains("ID > 17"));
|
||||
Assert.assertTrue(sql.contains("ORDER BY ID"));
|
||||
Assert.assertTrue(sql.contains("LIMIT 200"));
|
||||
}
|
||||
|
||||
private static SysJob job(long id) {
|
||||
SysJob job = new SysJob();
|
||||
job.setId(BigInteger.valueOf(id));
|
||||
job.setScheduleGeneration(1L);
|
||||
return job;
|
||||
}
|
||||
|
||||
private static String normalizedSql(QueryWrapper query) {
|
||||
return query.toSQL().replace("`", "").toUpperCase(Locale.ROOT);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,84 @@
|
||||
package tech.easyflow.job.schedule;
|
||||
|
||||
import com.easyagents.scheduler.ConcurrencyPolicy;
|
||||
import com.easyagents.scheduler.MisfirePolicy;
|
||||
import com.easyagents.scheduler.ScheduleDefinition;
|
||||
import com.easyagents.scheduler.ScheduleService;
|
||||
import org.junit.Assert;
|
||||
import org.junit.Test;
|
||||
import tech.easyflow.common.constant.enums.EnumMisfirePolicy;
|
||||
import tech.easyflow.job.entity.SysJob;
|
||||
import tech.easyflow.job.job.JobConstant;
|
||||
|
||||
import java.math.BigInteger;
|
||||
|
||||
import static org.mockito.Mockito.mock;
|
||||
|
||||
public class SysJobScheduleAdapterTest {
|
||||
|
||||
@Test
|
||||
public void shouldMapBusinessDefinitionToStableSchedule() {
|
||||
SysJob job = new SysJob();
|
||||
job.setId(BigInteger.valueOf(42));
|
||||
job.setJobName("daily-report");
|
||||
job.setCronExpression("0 0 2 * * ?");
|
||||
job.setMisfirePolicy(EnumMisfirePolicy.FIRE_ONCE_NOW.getCode());
|
||||
job.setAllowConcurrent(0);
|
||||
job.setScheduleGeneration(7L);
|
||||
|
||||
ScheduleDefinition definition = new SysJobScheduleAdapter(
|
||||
mock(ScheduleService.class), "Asia/Shanghai").toDefinition(job);
|
||||
|
||||
Assert.assertEquals("easyflow.job", definition.id().namespace());
|
||||
Assert.assertEquals("42", definition.id().name());
|
||||
Assert.assertEquals("easyflow.job.execution", definition.handlerCode());
|
||||
Assert.assertEquals(MisfirePolicy.FIRE_ONCE_NOW, definition.misfirePolicy());
|
||||
Assert.assertEquals(ConcurrencyPolicy.DISALLOW, definition.concurrencyPolicy());
|
||||
Assert.assertTrue(definition.recoverOnNodeFailure());
|
||||
Assert.assertEquals("42", definition.parameters().get("jobId"));
|
||||
Assert.assertEquals("7",
|
||||
definition.parameters().get(JobConstant.SCHEDULE_GENERATION));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void shouldMapSkipAndConcurrentPolicies() {
|
||||
SysJob job = new SysJob();
|
||||
job.setId(BigInteger.ONE);
|
||||
job.setJobName("parallel");
|
||||
job.setCronExpression("0/5 * * * * ?");
|
||||
job.setMisfirePolicy(EnumMisfirePolicy.SKIP.getCode());
|
||||
job.setAllowConcurrent(1);
|
||||
job.setScheduleGeneration(0L);
|
||||
|
||||
ScheduleDefinition definition = new SysJobScheduleAdapter(
|
||||
mock(ScheduleService.class), "UTC").toDefinition(job);
|
||||
|
||||
Assert.assertEquals(MisfirePolicy.SKIP, definition.misfirePolicy());
|
||||
Assert.assertEquals(ConcurrencyPolicy.ALLOW, definition.concurrencyPolicy());
|
||||
}
|
||||
|
||||
@Test(expected = IllegalArgumentException.class)
|
||||
public void shouldRejectUnsupportedMisfirePolicy() {
|
||||
SysJob job = new SysJob();
|
||||
job.setId(BigInteger.ONE);
|
||||
job.setJobName("invalid-policy");
|
||||
job.setCronExpression("0/5 * * * * ?");
|
||||
job.setMisfirePolicy(99);
|
||||
job.setAllowConcurrent(0);
|
||||
job.setScheduleGeneration(0L);
|
||||
|
||||
new SysJobScheduleAdapter(mock(ScheduleService.class), "UTC").toDefinition(job);
|
||||
}
|
||||
|
||||
@Test(expected = IllegalArgumentException.class)
|
||||
public void shouldRejectMissingScheduleGeneration() {
|
||||
SysJob job = new SysJob();
|
||||
job.setId(BigInteger.ONE);
|
||||
job.setJobName("missing-generation");
|
||||
job.setCronExpression("0/5 * * * * ?");
|
||||
job.setMisfirePolicy(EnumMisfirePolicy.SKIP.getCode());
|
||||
job.setAllowConcurrent(0);
|
||||
|
||||
new SysJobScheduleAdapter(mock(ScheduleService.class), "UTC").toDefinition(job);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,46 @@
|
||||
package tech.easyflow.job.schedule;
|
||||
|
||||
import org.junit.Test;
|
||||
import tech.easyflow.job.config.SysJobExecutionProperties;
|
||||
import tech.easyflow.job.entity.SysJob;
|
||||
import tech.easyflow.job.mapper.SysJobMapper;
|
||||
import tech.easyflow.job.service.SysJobService;
|
||||
|
||||
import java.math.BigInteger;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
public class SysJobScheduleReconcilerTest {
|
||||
|
||||
@Test
|
||||
public void reconciliationMustUseLastSeenIdInsteadOfMutableOffset() {
|
||||
SysJobMapper mapper = mock(SysJobMapper.class);
|
||||
SysJobService service = mock(SysJobService.class);
|
||||
List<SysJob> firstPage = jobs(1, 200);
|
||||
List<SysJob> secondPage = jobs(201, 1);
|
||||
when(mapper.selectReconciliationPage(BigInteger.ZERO, 200)).thenReturn(firstPage);
|
||||
when(mapper.selectReconciliationPage(BigInteger.valueOf(200), 200))
|
||||
.thenReturn(secondPage);
|
||||
|
||||
new SysJobScheduleReconciler(
|
||||
mapper, service, new SysJobExecutionProperties()).reconcile();
|
||||
|
||||
verify(mapper).selectReconciliationPage(BigInteger.ZERO, 200);
|
||||
verify(mapper).selectReconciliationPage(BigInteger.valueOf(200), 200);
|
||||
verify(service).syncJob(BigInteger.valueOf(201));
|
||||
}
|
||||
|
||||
private static List<SysJob> jobs(long firstId, int count) {
|
||||
List<SysJob> jobs = new ArrayList<>(count);
|
||||
for (long id = firstId; id < firstId + count; id++) {
|
||||
SysJob job = new SysJob();
|
||||
job.setId(BigInteger.valueOf(id));
|
||||
jobs.add(job);
|
||||
}
|
||||
return jobs;
|
||||
}
|
||||
}
|
||||
@@ -4,6 +4,7 @@ import com.easyagents.flow.core.chain.runtime.ChainExecutor;
|
||||
import org.junit.Assert;
|
||||
import org.junit.Test;
|
||||
import org.mockito.ArgumentCaptor;
|
||||
import tech.easyflow.ai.easyagentsflow.support.PublishedWorkflowDefinitionIds;
|
||||
import tech.easyflow.ai.service.WorkflowUsageAuthorizationService;
|
||||
import tech.easyflow.common.constant.Constants;
|
||||
import tech.easyflow.common.constant.enums.EnumDataStatus;
|
||||
@@ -14,7 +15,6 @@ import tech.easyflow.common.web.exceptions.BusinessException;
|
||||
import tech.easyflow.job.entity.SysJob;
|
||||
import tech.easyflow.job.job.JobConstant;
|
||||
import tech.easyflow.system.entity.SysAccount;
|
||||
import tech.easyflow.system.service.SysAccountService;
|
||||
|
||||
import java.math.BigInteger;
|
||||
import java.util.Map;
|
||||
@@ -42,25 +42,23 @@ public class WorkflowJobExecutionServiceTest {
|
||||
BigInteger tenantId = BigInteger.valueOf(201);
|
||||
BigInteger accountId = BigInteger.valueOf(301);
|
||||
BigInteger workflowId = BigInteger.valueOf(401);
|
||||
SysJobService jobService = mock(SysJobService.class);
|
||||
SysAccountService accountService = mock(SysAccountService.class);
|
||||
WorkflowUsageAuthorizationService authorizationService =
|
||||
mock(WorkflowUsageAuthorizationService.class);
|
||||
ChainExecutor chainExecutor = mock(ChainExecutor.class);
|
||||
SysJob currentJob = workflowJob(jobId, tenantId, accountId, workflowId);
|
||||
SysAccount account = account(accountId, tenantId, EnumDataStatus.AVAILABLE.getCode());
|
||||
when(jobService.getById(jobId)).thenReturn(currentJob);
|
||||
when(accountService.getById(accountId)).thenReturn(account);
|
||||
Map<String, Object> executionResult = Map.of("status", "done");
|
||||
when(chainExecutor.execute(eq(workflowId.toString()), anyMap()))
|
||||
String publishedWorkflowId = PublishedWorkflowDefinitionIds.published(workflowId.toString());
|
||||
when(chainExecutor.execute(eq(publishedWorkflowId), anyMap()))
|
||||
.thenReturn(executionResult);
|
||||
WorkflowJobExecutionService service = new WorkflowJobExecutionService(
|
||||
jobService,
|
||||
accountService,
|
||||
authorizationService,
|
||||
chainExecutor);
|
||||
|
||||
Object result = service.execute(scheduledJob(jobId, tenantId));
|
||||
Object result = service.execute(
|
||||
scheduledJob(jobId, tenantId, workflowId, Map.of("question", "hello")),
|
||||
currentJob,
|
||||
account);
|
||||
|
||||
Assert.assertSame(executionResult, result);
|
||||
ArgumentCaptor<LoginAccount> accountCaptor =
|
||||
@@ -75,7 +73,7 @@ public class WorkflowJobExecutionServiceTest {
|
||||
@SuppressWarnings("unchecked")
|
||||
ArgumentCaptor<Map<String, Object>> paramsCaptor =
|
||||
ArgumentCaptor.forClass(Map.class);
|
||||
verify(chainExecutor).execute(eq(workflowId.toString()), paramsCaptor.capture());
|
||||
verify(chainExecutor).execute(eq(publishedWorkflowId), paramsCaptor.capture());
|
||||
Object loginUser = paramsCaptor.getValue().get(Constants.LOGIN_USER_KEY);
|
||||
Assert.assertTrue(loginUser instanceof LoginAccount);
|
||||
Assert.assertEquals(((LoginAccount) loginUser).getId(), accountId);
|
||||
@@ -90,29 +88,28 @@ public class WorkflowJobExecutionServiceTest {
|
||||
BigInteger tenantId = BigInteger.valueOf(202);
|
||||
BigInteger accountId = BigInteger.valueOf(302);
|
||||
BigInteger workflowId = BigInteger.valueOf(402);
|
||||
SysJobService jobService = mock(SysJobService.class);
|
||||
SysAccountService accountService = mock(SysAccountService.class);
|
||||
WorkflowUsageAuthorizationService authorizationService =
|
||||
mock(WorkflowUsageAuthorizationService.class);
|
||||
ChainExecutor chainExecutor = mock(ChainExecutor.class);
|
||||
when(jobService.getById(jobId))
|
||||
.thenReturn(workflowJob(jobId, tenantId, accountId, workflowId));
|
||||
when(accountService.getById(accountId))
|
||||
.thenReturn(account(accountId, tenantId, EnumDataStatus.AVAILABLE.getCode()));
|
||||
SysJob currentJob = workflowJob(jobId, tenantId, accountId, workflowId);
|
||||
SysAccount owner = account(
|
||||
accountId, tenantId, EnumDataStatus.AVAILABLE.getCode());
|
||||
when(authorizationService.requireUsableWorkflow(
|
||||
eq(workflowId),
|
||||
any(LoginAccount.class),
|
||||
anyString()))
|
||||
.thenThrow(new BusinessException("工作流权限已撤销"));
|
||||
WorkflowJobExecutionService service = new WorkflowJobExecutionService(
|
||||
jobService,
|
||||
accountService,
|
||||
authorizationService,
|
||||
chainExecutor);
|
||||
|
||||
BusinessException exception = Assert.assertThrows(
|
||||
BusinessException.class,
|
||||
() -> service.execute(scheduledJob(jobId, tenantId))
|
||||
() -> service.execute(
|
||||
scheduledJob(jobId, tenantId, workflowId,
|
||||
Map.of("question", "hello")),
|
||||
currentJob,
|
||||
owner)
|
||||
);
|
||||
|
||||
Assert.assertTrue(exception.getMessage().contains("权限已撤销"));
|
||||
@@ -126,25 +123,23 @@ public class WorkflowJobExecutionServiceTest {
|
||||
public void shouldRejectCrossTenantJobSnapshot() {
|
||||
BigInteger jobId = BigInteger.valueOf(103);
|
||||
BigInteger tenantId = BigInteger.valueOf(203);
|
||||
SysJobService jobService = mock(SysJobService.class);
|
||||
SysAccountService accountService = mock(SysAccountService.class);
|
||||
WorkflowUsageAuthorizationService authorizationService =
|
||||
mock(WorkflowUsageAuthorizationService.class);
|
||||
ChainExecutor chainExecutor = mock(ChainExecutor.class);
|
||||
when(jobService.getById(jobId)).thenReturn(workflowJob(
|
||||
jobId,
|
||||
tenantId,
|
||||
BigInteger.valueOf(303),
|
||||
BigInteger.valueOf(403)));
|
||||
BigInteger accountId = BigInteger.valueOf(303);
|
||||
SysJob currentJob = workflowJob(
|
||||
jobId, tenantId, accountId, BigInteger.valueOf(403));
|
||||
WorkflowJobExecutionService service = new WorkflowJobExecutionService(
|
||||
jobService,
|
||||
accountService,
|
||||
authorizationService,
|
||||
chainExecutor);
|
||||
|
||||
IllegalStateException exception = Assert.assertThrows(
|
||||
IllegalStateException.class,
|
||||
() -> service.execute(scheduledJob(jobId, BigInteger.valueOf(999)))
|
||||
() -> service.execute(
|
||||
scheduledJob(jobId, BigInteger.valueOf(999),
|
||||
BigInteger.valueOf(403), Map.of()),
|
||||
currentJob,
|
||||
account(accountId, tenantId, EnumDataStatus.AVAILABLE.getCode()))
|
||||
);
|
||||
|
||||
Assert.assertTrue(exception.getMessage().contains("租户"));
|
||||
@@ -154,6 +149,39 @@ public class WorkflowJobExecutionServiceTest {
|
||||
anyString());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void shouldExecuteRegisteredWorkflowSnapshotAfterDefinitionEdit() {
|
||||
BigInteger jobId = BigInteger.valueOf(104);
|
||||
BigInteger tenantId = BigInteger.valueOf(204);
|
||||
BigInteger accountId = BigInteger.valueOf(304);
|
||||
BigInteger oldWorkflowId = BigInteger.valueOf(404);
|
||||
BigInteger newWorkflowId = BigInteger.valueOf(405);
|
||||
WorkflowUsageAuthorizationService authorizationService =
|
||||
mock(WorkflowUsageAuthorizationService.class);
|
||||
ChainExecutor chainExecutor = mock(ChainExecutor.class);
|
||||
SysJob currentJob = workflowJob(
|
||||
jobId, tenantId, accountId, newWorkflowId);
|
||||
SysAccount owner = account(
|
||||
accountId, tenantId, EnumDataStatus.AVAILABLE.getCode());
|
||||
WorkflowJobExecutionService service = new WorkflowJobExecutionService(
|
||||
authorizationService, chainExecutor);
|
||||
|
||||
service.execute(
|
||||
scheduledJob(jobId, tenantId, oldWorkflowId,
|
||||
Map.of("question", "old-ledger-value")),
|
||||
currentJob,
|
||||
owner);
|
||||
|
||||
verify(authorizationService).requireUsableWorkflow(
|
||||
eq(oldWorkflowId), any(LoginAccount.class), anyString());
|
||||
@SuppressWarnings("unchecked")
|
||||
ArgumentCaptor<Map<String, Object>> parameters = ArgumentCaptor.forClass(Map.class);
|
||||
verify(chainExecutor).execute(
|
||||
eq(PublishedWorkflowDefinitionIds.published(oldWorkflowId.toString())),
|
||||
parameters.capture());
|
||||
Assert.assertEquals(parameters.getValue().get("question"), "old-ledger-value");
|
||||
}
|
||||
|
||||
/**
|
||||
* 创建 Quartz 任务快照。
|
||||
*
|
||||
@@ -161,11 +189,16 @@ public class WorkflowJobExecutionServiceTest {
|
||||
* @param tenantId 租户 ID
|
||||
* @return 任务快照
|
||||
*/
|
||||
private SysJob scheduledJob(BigInteger jobId, BigInteger tenantId) {
|
||||
private SysJob scheduledJob(BigInteger jobId, BigInteger tenantId,
|
||||
BigInteger workflowId,
|
||||
Map<String, Object> workflowParams) {
|
||||
SysJob job = new SysJob();
|
||||
job.setId(jobId);
|
||||
job.setTenantId(tenantId);
|
||||
job.setJobType(EnumJobType.TINY_FLOW.getCode());
|
||||
job.setJobParams(Map.of(
|
||||
JobConstant.WORKFLOW_KEY, workflowId.toString(),
|
||||
JobConstant.WORKFLOW_PARAMS_KEY, workflowParams));
|
||||
return job;
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,204 @@
|
||||
package tech.easyflow.job.service.impl;
|
||||
|
||||
import com.easyagents.scheduler.ScheduleService;
|
||||
import org.junit.Assert;
|
||||
import org.junit.Test;
|
||||
import org.mockito.ArgumentCaptor;
|
||||
import org.springframework.dao.DataAccessResourceFailureException;
|
||||
import org.springframework.transaction.CannotCreateTransactionException;
|
||||
import org.springframework.transaction.PlatformTransactionManager;
|
||||
import org.springframework.transaction.TransactionStatus;
|
||||
import org.springframework.transaction.TransactionSystemException;
|
||||
import tech.easyflow.common.cache.RedisLockExecutor;
|
||||
import tech.easyflow.common.constant.enums.EnumJobStatus;
|
||||
import tech.easyflow.common.web.exceptions.BusinessException;
|
||||
import tech.easyflow.job.config.SysJobExecutionProperties;
|
||||
import tech.easyflow.job.entity.SysJob;
|
||||
import tech.easyflow.job.execution.SysJobExecutionStore;
|
||||
import tech.easyflow.job.mapper.SysJobMapper;
|
||||
import tech.easyflow.job.schedule.SysJobScheduleAdapter;
|
||||
|
||||
import java.math.BigInteger;
|
||||
import java.time.Duration;
|
||||
import java.util.List;
|
||||
import java.util.function.Supplier;
|
||||
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.ArgumentMatchers.anyInt;
|
||||
import static org.mockito.ArgumentMatchers.anyString;
|
||||
import static org.mockito.ArgumentMatchers.eq;
|
||||
import static org.mockito.Mockito.atLeastOnce;
|
||||
import static org.mockito.Mockito.doAnswer;
|
||||
import static org.mockito.Mockito.doReturn;
|
||||
import static org.mockito.Mockito.doThrow;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.never;
|
||||
import static org.mockito.Mockito.spy;
|
||||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
public class SysJobServiceImplTest {
|
||||
|
||||
@Test
|
||||
public void providerFailureMustStopJobAndRetryProjectionCleanup() {
|
||||
Fixture fixture = fixture();
|
||||
SysJob running = job(EnumJobStatus.RUNNING);
|
||||
SysJob stopped = job(EnumJobStatus.STOP);
|
||||
when(fixture.mapper().selectByIdForUpdate(running.getId()))
|
||||
.thenReturn(running, stopped);
|
||||
doReturn(true).when(fixture.service()).updateById(any(SysJob.class));
|
||||
DataAccessResourceFailureException providerFailure =
|
||||
new DataAccessResourceFailureException("quartz unavailable");
|
||||
doThrow(providerFailure).when(fixture.scheduleAdapter()).replace(running);
|
||||
|
||||
RuntimeException thrown = Assert.assertThrows(
|
||||
RuntimeException.class,
|
||||
() -> fixture.service().syncJob(running.getId()));
|
||||
|
||||
Assert.assertSame(providerFailure, thrown);
|
||||
ArgumentCaptor<SysJob> update = ArgumentCaptor.forClass(SysJob.class);
|
||||
verify(fixture.service()).updateById(update.capture());
|
||||
Assert.assertEquals(
|
||||
Integer.valueOf(EnumJobStatus.STOP.getCode()), update.getValue().getStatus());
|
||||
verify(fixture.scheduleAdapter()).delete(running.getId());
|
||||
verify(fixture.executionStore()).cancelPending(
|
||||
running.getId(), "调度投影同步失败,任务已停止");
|
||||
}
|
||||
|
||||
@Test
|
||||
public void transactionBeginFailureMustNotDeleteExistingProjection() {
|
||||
Fixture fixture = fixture(new CannotCreateTransactionException("pool exhausted"));
|
||||
|
||||
Assert.assertThrows(
|
||||
CannotCreateTransactionException.class,
|
||||
() -> fixture.service().syncJob(BigInteger.ONE));
|
||||
|
||||
verify(fixture.scheduleAdapter(), never()).replace(any(SysJob.class));
|
||||
verify(fixture.scheduleAdapter(), never()).delete(any(BigInteger.class));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void transactionCommitFailureMustNotDeleteExistingProjection() {
|
||||
Fixture fixture = fixture();
|
||||
SysJob running = job(EnumJobStatus.RUNNING);
|
||||
when(fixture.mapper().selectByIdForUpdate(running.getId())).thenReturn(running);
|
||||
doThrow(new TransactionSystemException("commit failed"))
|
||||
.when(fixture.transactionManager()).commit(any(TransactionStatus.class));
|
||||
|
||||
Assert.assertThrows(
|
||||
TransactionSystemException.class,
|
||||
() -> fixture.service().syncJob(running.getId()));
|
||||
|
||||
verify(fixture.scheduleAdapter()).replace(running);
|
||||
verify(fixture.scheduleAdapter(), never()).delete(any(BigInteger.class));
|
||||
verify(fixture.executionStore(), never()).cancelPending(any(BigInteger.class), anyString());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void startMustAbortBeforeStateChangeWhenOldPendingCleanupFails() {
|
||||
Fixture fixture = fixture();
|
||||
SysJob stopped = job(EnumJobStatus.STOP);
|
||||
doReturn(stopped).when(fixture.service()).getById(stopped.getId());
|
||||
doThrow(new DataAccessResourceFailureException("ledger unavailable"))
|
||||
.when(fixture.executionStore()).cancelPending(any(BigInteger.class), anyString());
|
||||
|
||||
Assert.assertThrows(
|
||||
DataAccessResourceFailureException.class,
|
||||
() -> fixture.service().startJob(stopped.getId()));
|
||||
|
||||
verify(fixture.mapper(), never()).startNextGeneration(any(BigInteger.class), anyInt());
|
||||
verify(fixture.scheduleAdapter(), never()).replace(any(SysJob.class));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void deleteProjectionFailureMustLeaveStoppedBusinessRow() {
|
||||
Fixture fixture = fixture();
|
||||
SysJob stopped = job(EnumJobStatus.STOP);
|
||||
doReturn(stopped).when(fixture.service()).getById(stopped.getId());
|
||||
when(fixture.mapper().selectByIdForUpdate(stopped.getId()))
|
||||
.thenReturn(stopped, stopped, stopped);
|
||||
doThrow(new DataAccessResourceFailureException("delete failed"))
|
||||
.when(fixture.scheduleAdapter()).delete(stopped.getId());
|
||||
|
||||
Assert.assertThrows(
|
||||
DataAccessResourceFailureException.class,
|
||||
() -> fixture.service().deleteJob(List.of(stopped.getId())));
|
||||
|
||||
verify(fixture.service(), never()).removeById(stopped.getId());
|
||||
verify(fixture.executionStore(), atLeastOnce()).cancelPending(
|
||||
eq(stopped.getId()), anyString());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void triggerNowMustRejectStoppedTaskWithoutCallingProvider() {
|
||||
Fixture fixture = fixture();
|
||||
SysJob stopped = job(EnumJobStatus.STOP);
|
||||
when(fixture.mapper().selectByIdForUpdate(stopped.getId())).thenReturn(stopped);
|
||||
|
||||
Assert.assertThrows(
|
||||
BusinessException.class,
|
||||
() -> fixture.service().triggerNow(stopped.getId()));
|
||||
|
||||
verify(fixture.scheduleAdapter(), never()).triggerNow(any(BigInteger.class));
|
||||
}
|
||||
|
||||
private static Fixture fixture() {
|
||||
return fixture(null);
|
||||
}
|
||||
|
||||
@SuppressWarnings({"unchecked", "rawtypes"})
|
||||
private static Fixture fixture(RuntimeException transactionBeginFailure) {
|
||||
SysJobMapper mapper = mock(SysJobMapper.class);
|
||||
SysJobScheduleAdapter scheduleAdapter = mock(SysJobScheduleAdapter.class);
|
||||
SysJobExecutionStore executionStore = mock(SysJobExecutionStore.class);
|
||||
ScheduleService scheduleService = mock(ScheduleService.class);
|
||||
RedisLockExecutor lockExecutor = mock(RedisLockExecutor.class);
|
||||
PlatformTransactionManager transactionManager = mock(PlatformTransactionManager.class);
|
||||
if (transactionBeginFailure == null) {
|
||||
when(transactionManager.getTransaction(any()))
|
||||
.thenReturn(mock(TransactionStatus.class));
|
||||
} else {
|
||||
when(transactionManager.getTransaction(any())).thenThrow(transactionBeginFailure);
|
||||
}
|
||||
doAnswer(invocation -> ((Supplier) invocation.getArgument(3)).get())
|
||||
.when(lockExecutor).executeWithRenewingLock(
|
||||
anyString(), any(Duration.class), any(Duration.class), any(Supplier.class));
|
||||
SysJobExecutionProperties properties = new SysJobExecutionProperties();
|
||||
TestService service = spy(new TestService(
|
||||
mapper, scheduleAdapter, executionStore, scheduleService,
|
||||
lockExecutor, transactionManager, properties));
|
||||
return new Fixture(
|
||||
service, mapper, scheduleAdapter, executionStore, transactionManager);
|
||||
}
|
||||
|
||||
private static SysJob job(EnumJobStatus status) {
|
||||
SysJob job = new SysJob();
|
||||
job.setId(BigInteger.ONE);
|
||||
job.setTenantId(BigInteger.TWO);
|
||||
job.setStatus(status.getCode());
|
||||
job.setScheduleGeneration(0L);
|
||||
return job;
|
||||
}
|
||||
|
||||
private static final class TestService extends SysJobServiceImpl {
|
||||
|
||||
private TestService(SysJobMapper mapper,
|
||||
SysJobScheduleAdapter scheduleAdapter,
|
||||
SysJobExecutionStore executionStore,
|
||||
ScheduleService scheduleService,
|
||||
RedisLockExecutor lockExecutor,
|
||||
PlatformTransactionManager transactionManager,
|
||||
SysJobExecutionProperties properties) {
|
||||
super(scheduleAdapter, executionStore, scheduleService, lockExecutor,
|
||||
transactionManager, properties, "Asia/Shanghai");
|
||||
this.mapper = mapper;
|
||||
}
|
||||
}
|
||||
|
||||
private record Fixture(TestService service,
|
||||
SysJobMapper mapper,
|
||||
SysJobScheduleAdapter scheduleAdapter,
|
||||
SysJobExecutionStore executionStore,
|
||||
PlatformTransactionManager transactionManager) {
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user