perf: 优化日志保留与分页查询

- 降低普通只读请求的日志写入并保留敏感 GET 审计 - 增加数据库分批清理、时间索引和文件滚动容量上限 - 增加近 30 天筛选、稳定倒序和分页大小限制
2026-07-30 14:21:36 +08:00
parent c78074a969
commit 03f45212ef
21 changed files with 732 additions and 19 deletions
--- a/easyflow-modules/easyflow-module-log/src/main/java/tech/easyflow/log/LogAspect.java
+++ b/easyflow-modules/easyflow-module-log/src/main/java/tech/easyflow/log/LogAspect.java
@@ -26,7 +26,9 @@ import java.util.Date;
 import java.util.Enumeration;
 import java.util.LinkedHashMap;
 import java.util.List;
+import java.util.Locale;
 import java.util.Map;
+import java.util.Set;

 /**
 * 记录管理端 Controller 操作结果及最小必要请求摘要。
@@ -43,20 +45,38 @@ public class LogAspect {
    private static final String SKILL_API_PREFIX = "/api/v1/skill";
    private static final List<String> SKILL_IDENTIFIER_KEYS =
            List.of("id", "resourceId", "skillId", "sourceId");
+    private static final Set<String> READ_ONLY_HTTP_METHODS =
+            Set.of("GET", "HEAD", "OPTIONS");

    private final WriteLogMapper logService;
    private final LogRecordProperties config;

+    /**
+     * 创建操作日志切面。
+     *
+     * @param logService 操作日志写入 Mapper
+     * @param config 操作日志配置
+     */
    public LogAspect(WriteLogMapper logService, LogRecordProperties config) {
        this.logService = logService;
        this.config = config;
    }

+    /**
+     * 匹配管理端控制器与通用增删改查控制器方法。
+     */
    @Pointcut("within(@org.springframework.web.bind.annotation.RestController *) " +
            "|| execution(* tech.easyflow.common.web.controller.BaseCurdController.*(..))")
    public void pointcut() {
    }

+    /**
+     * 执行业务方法并按配置持久化操作日志。
+     *
+     * @param proceedingJoinPoint 当前连接点
+     * @return 业务方法返回值
+     * @throws Throwable 业务方法或日志写入异常
+     */
    @Around("pointcut()")
    public Object doAround(ProceedingJoinPoint proceedingJoinPoint) throws Throwable {
        ServletRequestAttributes attributes = (ServletRequestAttributes) RequestContextHolder.getRequestAttributes();
@@ -72,6 +92,10 @@ public class LogAspect {
        MethodSignature signature = (MethodSignature) proceedingJoinPoint.getSignature();
        Class<?> controllerClass = signature.getDeclaringType();
        Method method = signature.getMethod();
+        LogRecord logRecord = method.getAnnotation(LogRecord.class);
+        if (!shouldRecordRequest(request, logRecord)) {
+            return proceedingJoinPoint.proceed();
+        }
        String params = getRequestParamsString(request);
        Throwable actionFailure = null;

@@ -82,7 +106,6 @@ public class LogAspect {
            throw throwable;
        } finally {
            WriteLog sysLog = new WriteLog();
-            LogRecord logRecord = method.getAnnotation(LogRecord.class);
            if (StpUtil.isLogin()) {

                BigInteger accountId = SaTokenUtil.getLoginAccount().getId();
@@ -103,6 +126,22 @@ public class LogAspect {
        }
    }

+    /**
+     * 判断当前请求是否需要写入操作日志。
+     *
+     * @param request 当前 HTTP 请求
+     * @param logRecord 方法上的显式日志标记
+     * @return 需要记录返回 {@code true}
+     */
+    private boolean shouldRecordRequest(HttpServletRequest request, LogRecord logRecord) {
+        if (config.isRecordReadActions() || logRecord != null) {
+            return true;
+        }
+        String httpMethod = request.getMethod();
+        return httpMethod == null
+                || !READ_ONLY_HTTP_METHODS.contains(httpMethod.toUpperCase(Locale.ROOT));
+    }
+
    /**
     * 持久化操作日志；业务动作已失败时，日志异常不得覆盖原始异常。
     *
@@ -123,6 +162,13 @@ public class LogAspect {
        }
    }

+    /**
+     * 解析日志动作名称。
+     *
+     * @param logRecord 显式日志标记
+     * @param method 当前控制器方法
+     * @return 日志动作名称
+     */
    private String buildActionName(LogRecord logRecord, Method method) {
        if (logRecord != null && StringUtil.hasText(logRecord.value())) {
            return logRecord.value();
@@ -199,6 +245,12 @@ public class LogAspect {
        return null;
    }

+    /**
+     * 提取并限制请求查询参数，避免超长内容进入数据库。
+     *
+     * @param request 当前 HTTP 请求
+     * @return 有界查询参数文本
+     */
    private String getRequestParamsString(HttpServletRequest request) {
        StringBuilder sb = new StringBuilder();
        Enumeration<String> e = request.getParameterNames();
--- a/easyflow-modules/easyflow-module-log/src/main/java/tech/easyflow/log/LogRecordProperties.java
+++ b/easyflow-modules/easyflow-module-log/src/main/java/tech/easyflow/log/LogRecordProperties.java
@@ -3,17 +3,182 @@ package tech.easyflow.log;
 import org.springframework.boot.context.properties.ConfigurationProperties;
 import org.springframework.context.annotation.Configuration;

+/**
+ * 操作日志记录与保留策略配置。
+ */
@Configuration
@ConfigurationProperties(prefix = "easyflow.log-record")
 public class LogRecordProperties {

+    /**
+     * 仅记录此前缀下的请求；为空时不限制请求路径。
+     */
    private String recordActionPrefix;

+    /**
+     * 是否记录未显式标注的只读请求。
+     */
+    private boolean recordReadActions;
+
+    /**
+     * 数据库操作日志保留策略。
+     */
+    private final Retention retention = new Retention();
+
+    /**
+     * 获取请求路径前缀。
+     *
+     * @return 请求路径前缀
+     */
    public String getRecordActionPrefix() {
        return recordActionPrefix;
    }

+    /**
+     * 设置请求路径前缀。
+     *
+     * @param recordActionPrefix 请求路径前缀
+     */
    public void setRecordActionPrefix(String recordActionPrefix) {
        this.recordActionPrefix = recordActionPrefix;
    }
+
+    /**
+     * 判断是否记录未显式标注的只读请求。
+     *
+     * @return 是否记录只读请求
+     */
+    public boolean isRecordReadActions() {
+        return recordReadActions;
+    }
+
+    /**
+     * 设置是否记录未显式标注的只读请求。
+     *
+     * @param recordReadActions 是否记录只读请求
+     */
+    public void setRecordReadActions(boolean recordReadActions) {
+        this.recordReadActions = recordReadActions;
+    }
+
+    /**
+     * 获取数据库操作日志保留策略。
+     *
+     * @return 保留策略
+     */
+    public Retention getRetention() {
+        return retention;
+    }
+
+    /**
+     * 数据库操作日志保留策略。
+     */
+    public static class Retention {
+
+        private static final int MAX_BATCH_SIZE = 10_000;
+
+        /**
+         * 是否启用自动清理。
+         */
+        private boolean enabled = true;
+
+        /**
+         * 在线日志保留天数。
+         */
+        private int days = 90;
+
+        /**
+         * 单批清理行数。
+         */
+        private int batchSize = 5_000;
+
+        /**
+         * 单次调度最多清理批次。
+         */
+        private int maxBatches = 20;
+
+        /**
+         * 判断是否启用自动清理。
+         *
+         * @return 是否启用
+         */
+        public boolean isEnabled() {
+            return enabled;
+        }
+
+        /**
+         * 设置是否启用自动清理。
+         *
+         * @param enabled 是否启用
+         */
+        public void setEnabled(boolean enabled) {
+            this.enabled = enabled;
+        }
+
+        /**
+         * 获取在线日志保留天数。
+         *
+         * @return 保留天数
+         */
+        public int getDays() {
+            return days;
+        }
+
+        /**
+         * 设置在线日志保留天数。
+         *
+         * @param days 保留天数，必须大于零
+         * @throws IllegalArgumentException 保留天数不合法时抛出
+         */
+        public void setDays(int days) {
+            if (days < 1) {
+                throw new IllegalArgumentException("日志保留天数必须大于零");
+            }
+            this.days = days;
+        }
+
+        /**
+         * 获取单批清理行数。
+         *
+         * @return 单批清理行数
+         */
+        public int getBatchSize() {
+            return batchSize;
+        }
+
+        /**
+         * 设置单批清理行数。
+         *
+         * @param batchSize 单批清理行数
+         * @throws IllegalArgumentException 批次大小不在允许范围内时抛出
+         */
+        public void setBatchSize(int batchSize) {
+            if (batchSize < 1 || batchSize > MAX_BATCH_SIZE) {
+                throw new IllegalArgumentException("日志清理批次大小必须在 1 到 10000 之间");
+            }
+            this.batchSize = batchSize;
+        }
+
+        /**
+         * 获取单次调度最多清理批次。
+         *
+         * @return 最大批次数
+         */
+        public int getMaxBatches() {
+            return maxBatches;
+        }
+
+        /**
+         * 设置单次调度最多清理批次。
+         *
+         * @param maxBatches 最大批次数，必须大于零
+         * @throws IllegalArgumentException 最大批次数不合法时抛出
+         */
+        public void setMaxBatches(int maxBatches) {
+            if (maxBatches < 1) {
+                throw new IllegalArgumentException("日志清理最大批次数必须大于零");
+            }
+            this.maxBatches = maxBatches;
+        }
+    }
 }
--- a/easyflow-modules/easyflow-module-log/src/test/java/tech/easyflow/log/LogAspectTest.java
+++ b/easyflow-modules/easyflow-module-log/src/test/java/tech/easyflow/log/LogAspectTest.java
@@ -13,6 +13,7 @@ import org.mockito.MockedStatic;
 import org.mockito.Mockito;
 import org.springframework.web.context.request.RequestContextHolder;
 import org.springframework.web.context.request.ServletRequestAttributes;
+import tech.easyflow.log.annotation.LogRecord;
 import tech.easyflow.log.entity.WriteLog;
 import tech.easyflow.log.mapper.WriteLogMapper;

@@ -28,6 +29,7 @@ import static org.junit.Assert.assertSame;
 import static org.mockito.ArgumentMatchers.any;
 import static org.mockito.Mockito.doThrow;
 import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.never;
 import static org.mockito.Mockito.verify;
 import static org.mockito.Mockito.when;

@@ -158,6 +160,47 @@ public class LogAspectTest {
        assertNull(captor.getValue().getActionBody());
    }

+    /**
+     * 验证未显式标注的 GET 请求不会产生操作日志。
+     *
+     * @throws Throwable 切面执行异常
+     */
+    @Test
+    public void unannotatedGetShouldNotWriteLog() throws Throwable {
+        WriteLogMapper mapper = mock(WriteLogMapper.class);
+        LogAspect aspect = new LogAspect(mapper, properties());
+        request("{}", "GET");
+        ProceedingJoinPoint joinPoint = joinPoint("read");
+        when(joinPoint.proceed()).thenReturn("ok");
+
+        assertEquals("ok", aspect.doAround(joinPoint));
+
+        verify(mapper, never()).insert(any(WriteLog.class));
+    }
+
+    /**
+     * 验证显式标注的 GET 请求仍会产生操作日志。
+     *
+     * @throws Throwable 切面执行异常
+     */
+    @Test
+    public void annotatedGetShouldWriteLog() throws Throwable {
+        WriteLogMapper mapper = mock(WriteLogMapper.class);
+        LogAspect aspect = new LogAspect(mapper, properties());
+        request("{}", "GET");
+        ProceedingJoinPoint joinPoint = joinPoint("auditedRead");
+        when(joinPoint.proceed()).thenReturn("ok");
+
+        try (MockedStatic<StpUtil> stp = Mockito.mockStatic(StpUtil.class)) {
+            stp.when(StpUtil::isLogin).thenReturn(false);
+            assertEquals("ok", aspect.doAround(joinPoint));
+        }
+
+        ArgumentCaptor<WriteLog> captor = ArgumentCaptor.forClass(WriteLog.class);
+        verify(mapper).insert(captor.capture());
+        assertEquals("审计读取", captor.getValue().getActionName());
+    }
+
    /**
     * 创建测试日志配置。
     *
@@ -177,8 +220,21 @@ public class LogAspectTest {
     * @throws IOException 输入流创建失败
     */
    private HttpServletRequest request(String body) throws IOException {
+        return request(body, "POST");
+    }
+
+    /**
+     * 创建并绑定指定 HTTP 方法的测试请求。
+     *
+     * @param body JSON body
+     * @param httpMethod HTTP 方法
+     * @return 测试请求
+     * @throws IOException 输入流创建失败
+     */
+    private HttpServletRequest request(String body, String httpMethod) throws IOException {
        HttpServletRequest request = mock(HttpServletRequest.class);
        when(request.getServletPath()).thenReturn("/api/v1/skill/update");
+        when(request.getMethod()).thenReturn(httpMethod);
        when(request.getContentType()).thenReturn("application/json;charset=UTF-8");
        when(request.getCharacterEncoding()).thenReturn(StandardCharsets.UTF_8.name());
        when(request.getParameterNames()).thenReturn(Collections.emptyEnumeration());
@@ -196,9 +252,20 @@ public class LogAspectTest {
     * @throws NoSuchMethodException 测试方法不存在
     */
    private ProceedingJoinPoint joinPoint() throws NoSuchMethodException {
+        return joinPoint("update");
+    }
+
+    /**
+     * 创建指定控制器方法的测试连接点。
+     *
+     * @param methodName 控制器方法名
+     * @return 测试连接点
+     * @throws NoSuchMethodException 测试方法不存在
+     */
+    private ProceedingJoinPoint joinPoint(String methodName) throws NoSuchMethodException {
        ProceedingJoinPoint joinPoint = mock(ProceedingJoinPoint.class);
        MethodSignature signature = mock(MethodSignature.class);
-        Method method = TestController.class.getMethod("update");
+        Method method = TestController.class.getMethod(methodName);
        when(joinPoint.getSignature()).thenReturn(signature);
        when(signature.getDeclaringType()).thenReturn(TestController.class);
        when(signature.getMethod()).thenReturn(method);
@@ -249,5 +316,24 @@ public class LogAspectTest {
        public Object update() {
            return null;
        }
+
+        /**
+         * 模拟普通读取入口。
+         *
+         * @return 空结果
+         */
+        public Object read() {
+            return null;
+        }
+
+        /**
+         * 模拟需要审计的读取入口。
+         *
+         * @return 空结果
+         */
+        @LogRecord("审计读取")
+        public Object auditedRead() {
+            return null;
+        }
    }
 }
--- a/easyflow-modules/easyflow-module-system/src/main/java/tech/easyflow/system/mapper/SysLogMapper.java
+++ b/easyflow-modules/easyflow-module-system/src/main/java/tech/easyflow/system/mapper/SysLogMapper.java
@@ -1,7 +1,11 @@
 package tech.easyflow.system.mapper;

-import tech.easyflow.system.entity.SysLog;
 import com.mybatisflex.core.BaseMapper;
+import org.apache.ibatis.annotations.Delete;
+import org.apache.ibatis.annotations.Param;
+import tech.easyflow.system.entity.SysLog;
+
+import java.util.Date;

 /**
 *  映射层。
@@ -11,4 +15,19 @@ import com.mybatisflex.core.BaseMapper;
 */
 public interface SysLogMapper extends BaseMapper<SysLog> {

+    /**
+     * 按操作时间删除一批过期日志。
+     *
+     * @param cutoff 过期时间边界，不包含该时间
+     * @param batchSize 单批最大删除行数
+     * @return 实际删除行数
+     */
+    @Delete("""
+            DELETE FROM tb_sys_log
+            WHERE created < #{cutoff}
+            ORDER BY created
+            LIMIT #{batchSize}
+            """)
+    int deleteExpiredBatch(@Param("cutoff") Date cutoff,
+                           @Param("batchSize") int batchSize);
 }
--- a/easyflow-modules/easyflow-module-system/src/main/java/tech/easyflow/system/schedule/SysLogCleanupJob.java
+++ b/easyflow-modules/easyflow-module-system/src/main/java/tech/easyflow/system/schedule/SysLogCleanupJob.java
@@ -0,0 +1,86 @@
+package tech.easyflow.system.schedule;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
+import org.springframework.scheduling.annotation.Scheduled;
+import org.springframework.stereotype.Component;
+import tech.easyflow.common.cache.DistributedScheduledLock;
+import tech.easyflow.log.LogRecordProperties;
+import tech.easyflow.system.mapper.SysLogMapper;
+
+import java.time.Instant;
+import java.time.temporal.ChronoUnit;
+import java.util.Date;
+
+/**
+ * 分批清理超过在线保留期的数据库操作日志。
+ */
+@Component
+@ConditionalOnProperty(
+        prefix = "easyflow.log-record.retention",
+        name = "enabled",
+        havingValue = "true",
+        matchIfMissing = true)
+public class SysLogCleanupJob {
+
+    private static final Logger LOG = LoggerFactory.getLogger(SysLogCleanupJob.class);
+
+    private final SysLogMapper sysLogMapper;
+    private final LogRecordProperties.Retention retention;
+
+    /**
+     * 创建操作日志清理任务。
+     *
+     * @param sysLogMapper 操作日志 Mapper
+     * @param properties 操作日志配置
+     */
+    public SysLogCleanupJob(SysLogMapper sysLogMapper,
+                            LogRecordProperties properties) {
+        this.sysLogMapper = sysLogMapper;
+        this.retention = properties.getRetention();
+    }
+
+    /**
+     * 每天低峰期在单个集群节点上清理过期操作日志。
+     */
+    @Scheduled(cron = "${easyflow.log-record.retention.cron:0 30 2 * * *}")
+    @DistributedScheduledLock(
+            key = "easyflow:schedule:sys-log-cleanup",
+            leaseSeconds = 600L)
+    public void cleanup() {
+        Date cutoff = Date.from(
+                Instant.now().minus(retention.getDays(), ChronoUnit.DAYS));
+        try {
+            int totalDeleted = cleanupExpired(cutoff);
+            if (totalDeleted > 0) {
+                LOG.info("已清理 {} 条过期操作日志，保留天数={}",
+                        totalDeleted, retention.getDays());
+            }
+        } catch (RuntimeException error) {
+            LOG.error("清理过期操作日志失败", error);
+            throw error;
+        }
+    }
+
+    /**
+     * 按配置的批次上限清理指定时间之前的日志。
+     *
+     * @param cutoff 过期时间边界
+     * @return 实际删除总行数
+     */
+    int cleanupExpired(Date cutoff) {
+        int totalDeleted = 0;
+        int batchSize = retention.getBatchSize();
+        for (int batch = 0; batch < retention.getMaxBatches(); batch++) {
+            int deleted = sysLogMapper.deleteExpiredBatch(cutoff, batchSize);
+            totalDeleted += deleted;
+            if (deleted < batchSize) {
+                return totalDeleted;
+            }
+        }
+        LOG.warn("操作日志单次清理达到批次上限，后续调度将继续处理，已清理={}",
+                totalDeleted);
+        return totalDeleted;
+    }
+}
--- a/easyflow-modules/easyflow-module-system/src/test/java/tech/easyflow/system/schedule/SysLogCleanupJobTest.java
+++ b/easyflow-modules/easyflow-module-system/src/test/java/tech/easyflow/system/schedule/SysLogCleanupJobTest.java
@@ -0,0 +1,67 @@
+package tech.easyflow.system.schedule;
+
+import org.junit.Test;
+import tech.easyflow.log.LogRecordProperties;
+import tech.easyflow.system.mapper.SysLogMapper;
+
+import java.util.Date;
+
+import static org.junit.Assert.assertEquals;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+/**
+ * {@link SysLogCleanupJob} 分批清理行为测试。
+ */
+public class SysLogCleanupJobTest {
+
+    /**
+     * 验证清理任务在最后一个不满批次后停止。
+     */
+    @Test
+    public void cleanupShouldStopAfterPartialBatch() {
+        SysLogMapper mapper = mock(SysLogMapper.class);
+        LogRecordProperties properties = properties(5_000, 10);
+        Date cutoff = new Date();
+        when(mapper.deleteExpiredBatch(cutoff, 5_000))
+                .thenReturn(5_000)
+                .thenReturn(1_200);
+
+        SysLogCleanupJob job = new SysLogCleanupJob(mapper, properties);
+
+        assertEquals(6_200, job.cleanupExpired(cutoff));
+        verify(mapper, times(2)).deleteExpiredBatch(cutoff, 5_000);
+    }
+
+    /**
+     * 验证清理任务遵守单次调度最大批次数。
+     */
+    @Test
+    public void cleanupShouldRespectMaxBatches() {
+        SysLogMapper mapper = mock(SysLogMapper.class);
+        LogRecordProperties properties = properties(2_000, 3);
+        Date cutoff = new Date();
+        when(mapper.deleteExpiredBatch(cutoff, 2_000)).thenReturn(2_000);
+
+        SysLogCleanupJob job = new SysLogCleanupJob(mapper, properties);
+
+        assertEquals(6_000, job.cleanupExpired(cutoff));
+        verify(mapper, times(3)).deleteExpiredBatch(cutoff, 2_000);
+    }
+
+    /**
+     * 创建测试日志配置。
+     *
+     * @param batchSize 单批行数
+     * @param maxBatches 最大批次数
+     * @return 日志配置
+     */
+    private LogRecordProperties properties(int batchSize, int maxBatches) {
+        LogRecordProperties properties = new LogRecordProperties();
+        properties.getRetention().setBatchSize(batchSize);
+        properties.getRetention().setMaxBatches(maxBatches);
+        return properties;
+    }
+}