perf: 优化日志保留与分页查询
- 降低普通只读请求的日志写入并保留敏感 GET 审计 - 增加数据库分批清理、时间索引和文件滚动容量上限 - 增加近 30 天筛选、稳定倒序和分页大小限制
This commit is contained in:
@@ -1,7 +1,11 @@
|
||||
package tech.easyflow.system.mapper;
|
||||
|
||||
import tech.easyflow.system.entity.SysLog;
|
||||
import com.mybatisflex.core.BaseMapper;
|
||||
import org.apache.ibatis.annotations.Delete;
|
||||
import org.apache.ibatis.annotations.Param;
|
||||
import tech.easyflow.system.entity.SysLog;
|
||||
|
||||
import java.util.Date;
|
||||
|
||||
/**
|
||||
* 映射层。
|
||||
@@ -11,4 +15,19 @@ import com.mybatisflex.core.BaseMapper;
|
||||
*/
|
||||
public interface SysLogMapper extends BaseMapper<SysLog> {
|
||||
|
||||
/**
|
||||
* 按操作时间删除一批过期日志。
|
||||
*
|
||||
* @param cutoff 过期时间边界,不包含该时间
|
||||
* @param batchSize 单批最大删除行数
|
||||
* @return 实际删除行数
|
||||
*/
|
||||
@Delete("""
|
||||
DELETE FROM tb_sys_log
|
||||
WHERE created < #{cutoff}
|
||||
ORDER BY created
|
||||
LIMIT #{batchSize}
|
||||
""")
|
||||
int deleteExpiredBatch(@Param("cutoff") Date cutoff,
|
||||
@Param("batchSize") int batchSize);
|
||||
}
|
||||
|
||||
@@ -0,0 +1,86 @@
|
||||
package tech.easyflow.system.schedule;
|
||||
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
|
||||
import org.springframework.scheduling.annotation.Scheduled;
|
||||
import org.springframework.stereotype.Component;
|
||||
import tech.easyflow.common.cache.DistributedScheduledLock;
|
||||
import tech.easyflow.log.LogRecordProperties;
|
||||
import tech.easyflow.system.mapper.SysLogMapper;
|
||||
|
||||
import java.time.Instant;
|
||||
import java.time.temporal.ChronoUnit;
|
||||
import java.util.Date;
|
||||
|
||||
/**
|
||||
* 分批清理超过在线保留期的数据库操作日志。
|
||||
*/
|
||||
@Component
|
||||
@ConditionalOnProperty(
|
||||
prefix = "easyflow.log-record.retention",
|
||||
name = "enabled",
|
||||
havingValue = "true",
|
||||
matchIfMissing = true)
|
||||
public class SysLogCleanupJob {
|
||||
|
||||
private static final Logger LOG = LoggerFactory.getLogger(SysLogCleanupJob.class);
|
||||
|
||||
private final SysLogMapper sysLogMapper;
|
||||
private final LogRecordProperties.Retention retention;
|
||||
|
||||
/**
|
||||
* 创建操作日志清理任务。
|
||||
*
|
||||
* @param sysLogMapper 操作日志 Mapper
|
||||
* @param properties 操作日志配置
|
||||
*/
|
||||
public SysLogCleanupJob(SysLogMapper sysLogMapper,
|
||||
LogRecordProperties properties) {
|
||||
this.sysLogMapper = sysLogMapper;
|
||||
this.retention = properties.getRetention();
|
||||
}
|
||||
|
||||
/**
|
||||
* 每天低峰期在单个集群节点上清理过期操作日志。
|
||||
*/
|
||||
@Scheduled(cron = "${easyflow.log-record.retention.cron:0 30 2 * * *}")
|
||||
@DistributedScheduledLock(
|
||||
key = "easyflow:schedule:sys-log-cleanup",
|
||||
leaseSeconds = 600L)
|
||||
public void cleanup() {
|
||||
Date cutoff = Date.from(
|
||||
Instant.now().minus(retention.getDays(), ChronoUnit.DAYS));
|
||||
try {
|
||||
int totalDeleted = cleanupExpired(cutoff);
|
||||
if (totalDeleted > 0) {
|
||||
LOG.info("已清理 {} 条过期操作日志,保留天数={}",
|
||||
totalDeleted, retention.getDays());
|
||||
}
|
||||
} catch (RuntimeException error) {
|
||||
LOG.error("清理过期操作日志失败", error);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 按配置的批次上限清理指定时间之前的日志。
|
||||
*
|
||||
* @param cutoff 过期时间边界
|
||||
* @return 实际删除总行数
|
||||
*/
|
||||
int cleanupExpired(Date cutoff) {
|
||||
int totalDeleted = 0;
|
||||
int batchSize = retention.getBatchSize();
|
||||
for (int batch = 0; batch < retention.getMaxBatches(); batch++) {
|
||||
int deleted = sysLogMapper.deleteExpiredBatch(cutoff, batchSize);
|
||||
totalDeleted += deleted;
|
||||
if (deleted < batchSize) {
|
||||
return totalDeleted;
|
||||
}
|
||||
}
|
||||
LOG.warn("操作日志单次清理达到批次上限,后续调度将继续处理,已清理={}",
|
||||
totalDeleted);
|
||||
return totalDeleted;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,67 @@
|
||||
package tech.easyflow.system.schedule;
|
||||
|
||||
import org.junit.Test;
|
||||
import tech.easyflow.log.LogRecordProperties;
|
||||
import tech.easyflow.system.mapper.SysLogMapper;
|
||||
|
||||
import java.util.Date;
|
||||
|
||||
import static org.junit.Assert.assertEquals;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.times;
|
||||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
/**
|
||||
* {@link SysLogCleanupJob} 分批清理行为测试。
|
||||
*/
|
||||
public class SysLogCleanupJobTest {
|
||||
|
||||
/**
|
||||
* 验证清理任务在最后一个不满批次后停止。
|
||||
*/
|
||||
@Test
|
||||
public void cleanupShouldStopAfterPartialBatch() {
|
||||
SysLogMapper mapper = mock(SysLogMapper.class);
|
||||
LogRecordProperties properties = properties(5_000, 10);
|
||||
Date cutoff = new Date();
|
||||
when(mapper.deleteExpiredBatch(cutoff, 5_000))
|
||||
.thenReturn(5_000)
|
||||
.thenReturn(1_200);
|
||||
|
||||
SysLogCleanupJob job = new SysLogCleanupJob(mapper, properties);
|
||||
|
||||
assertEquals(6_200, job.cleanupExpired(cutoff));
|
||||
verify(mapper, times(2)).deleteExpiredBatch(cutoff, 5_000);
|
||||
}
|
||||
|
||||
/**
|
||||
* 验证清理任务遵守单次调度最大批次数。
|
||||
*/
|
||||
@Test
|
||||
public void cleanupShouldRespectMaxBatches() {
|
||||
SysLogMapper mapper = mock(SysLogMapper.class);
|
||||
LogRecordProperties properties = properties(2_000, 3);
|
||||
Date cutoff = new Date();
|
||||
when(mapper.deleteExpiredBatch(cutoff, 2_000)).thenReturn(2_000);
|
||||
|
||||
SysLogCleanupJob job = new SysLogCleanupJob(mapper, properties);
|
||||
|
||||
assertEquals(6_000, job.cleanupExpired(cutoff));
|
||||
verify(mapper, times(3)).deleteExpiredBatch(cutoff, 2_000);
|
||||
}
|
||||
|
||||
/**
|
||||
* 创建测试日志配置。
|
||||
*
|
||||
* @param batchSize 单批行数
|
||||
* @param maxBatches 最大批次数
|
||||
* @return 日志配置
|
||||
*/
|
||||
private LogRecordProperties properties(int batchSize, int maxBatches) {
|
||||
LogRecordProperties properties = new LogRecordProperties();
|
||||
properties.getRetention().setBatchSize(batchSize);
|
||||
properties.getRetention().setMaxBatches(maxBatches);
|
||||
return properties;
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user