perf: 优化日志保留与分页查询

- 降低普通只读请求的日志写入并保留敏感 GET 审计

- 增加数据库分批清理、时间索引和文件滚动容量上限

- 增加近 30 天筛选、稳定倒序和分页大小限制
This commit is contained in:
2026-07-30 14:21:36 +08:00
parent c78074a969
commit 03f45212ef
21 changed files with 732 additions and 19 deletions

View File

@@ -1,7 +1,11 @@
package tech.easyflow.system.mapper;
import tech.easyflow.system.entity.SysLog;
import com.mybatisflex.core.BaseMapper;
import org.apache.ibatis.annotations.Delete;
import org.apache.ibatis.annotations.Param;
import tech.easyflow.system.entity.SysLog;
import java.util.Date;
/**
* 映射层。
@@ -11,4 +15,19 @@ import com.mybatisflex.core.BaseMapper;
*/
public interface SysLogMapper extends BaseMapper<SysLog> {
/**
* 按操作时间删除一批过期日志。
*
* @param cutoff 过期时间边界,不包含该时间
* @param batchSize 单批最大删除行数
* @return 实际删除行数
*/
@Delete("""
DELETE FROM tb_sys_log
WHERE created < #{cutoff}
ORDER BY created
LIMIT #{batchSize}
""")
int deleteExpiredBatch(@Param("cutoff") Date cutoff,
@Param("batchSize") int batchSize);
}

View File

@@ -0,0 +1,86 @@
package tech.easyflow.system.schedule;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Component;
import tech.easyflow.common.cache.DistributedScheduledLock;
import tech.easyflow.log.LogRecordProperties;
import tech.easyflow.system.mapper.SysLogMapper;
import java.time.Instant;
import java.time.temporal.ChronoUnit;
import java.util.Date;
/**
* 分批清理超过在线保留期的数据库操作日志。
*/
@Component
@ConditionalOnProperty(
prefix = "easyflow.log-record.retention",
name = "enabled",
havingValue = "true",
matchIfMissing = true)
public class SysLogCleanupJob {
private static final Logger LOG = LoggerFactory.getLogger(SysLogCleanupJob.class);
private final SysLogMapper sysLogMapper;
private final LogRecordProperties.Retention retention;
/**
* 创建操作日志清理任务。
*
* @param sysLogMapper 操作日志 Mapper
* @param properties 操作日志配置
*/
public SysLogCleanupJob(SysLogMapper sysLogMapper,
LogRecordProperties properties) {
this.sysLogMapper = sysLogMapper;
this.retention = properties.getRetention();
}
/**
* 每天低峰期在单个集群节点上清理过期操作日志。
*/
@Scheduled(cron = "${easyflow.log-record.retention.cron:0 30 2 * * *}")
@DistributedScheduledLock(
key = "easyflow:schedule:sys-log-cleanup",
leaseSeconds = 600L)
public void cleanup() {
Date cutoff = Date.from(
Instant.now().minus(retention.getDays(), ChronoUnit.DAYS));
try {
int totalDeleted = cleanupExpired(cutoff);
if (totalDeleted > 0) {
LOG.info("已清理 {} 条过期操作日志,保留天数={}",
totalDeleted, retention.getDays());
}
} catch (RuntimeException error) {
LOG.error("清理过期操作日志失败", error);
throw error;
}
}
/**
* 按配置的批次上限清理指定时间之前的日志。
*
* @param cutoff 过期时间边界
* @return 实际删除总行数
*/
int cleanupExpired(Date cutoff) {
int totalDeleted = 0;
int batchSize = retention.getBatchSize();
for (int batch = 0; batch < retention.getMaxBatches(); batch++) {
int deleted = sysLogMapper.deleteExpiredBatch(cutoff, batchSize);
totalDeleted += deleted;
if (deleted < batchSize) {
return totalDeleted;
}
}
LOG.warn("操作日志单次清理达到批次上限,后续调度将继续处理,已清理={}",
totalDeleted);
return totalDeleted;
}
}

View File

@@ -0,0 +1,67 @@
package tech.easyflow.system.schedule;
import org.junit.Test;
import tech.easyflow.log.LogRecordProperties;
import tech.easyflow.system.mapper.SysLogMapper;
import java.util.Date;
import static org.junit.Assert.assertEquals;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
/**
* {@link SysLogCleanupJob} 分批清理行为测试。
*/
public class SysLogCleanupJobTest {
/**
* 验证清理任务在最后一个不满批次后停止。
*/
@Test
public void cleanupShouldStopAfterPartialBatch() {
SysLogMapper mapper = mock(SysLogMapper.class);
LogRecordProperties properties = properties(5_000, 10);
Date cutoff = new Date();
when(mapper.deleteExpiredBatch(cutoff, 5_000))
.thenReturn(5_000)
.thenReturn(1_200);
SysLogCleanupJob job = new SysLogCleanupJob(mapper, properties);
assertEquals(6_200, job.cleanupExpired(cutoff));
verify(mapper, times(2)).deleteExpiredBatch(cutoff, 5_000);
}
/**
* 验证清理任务遵守单次调度最大批次数。
*/
@Test
public void cleanupShouldRespectMaxBatches() {
SysLogMapper mapper = mock(SysLogMapper.class);
LogRecordProperties properties = properties(2_000, 3);
Date cutoff = new Date();
when(mapper.deleteExpiredBatch(cutoff, 2_000)).thenReturn(2_000);
SysLogCleanupJob job = new SysLogCleanupJob(mapper, properties);
assertEquals(6_000, job.cleanupExpired(cutoff));
verify(mapper, times(3)).deleteExpiredBatch(cutoff, 2_000);
}
/**
* 创建测试日志配置。
*
* @param batchSize 单批行数
* @param maxBatches 最大批次数
* @return 日志配置
*/
private LogRecordProperties properties(int batchSize, int maxBatches) {
LogRecordProperties properties = new LogRecordProperties();
properties.getRetention().setBatchSize(batchSize);
properties.getRetention().setMaxBatches(maxBatches);
return properties;
}
}