feat: 落地聊天记录异步持久化基础设施

- 新增 chatlog 模块、AnalyticalDB 公共层与 common-mq Redis Streams 实现 - 建立 Redis 热态、MySQL 热数据、AnalyticalDB 历史查询与同步链路 - 收紧聊天记录幂等、摘要时序与持久化失败语义
2026-04-05 11:35:05 +08:00
parent 1ecc28e498
commit 25e80433a5
105 changed files with 8050 additions and 2 deletions
--- a/easyflow-commons/easyflow-common-analytical-db/pom.xml
+++ b/easyflow-commons/easyflow-common-analytical-db/pom.xml
@@ -0,0 +1,48 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+    <parent>
+        <groupId>tech.easyflow</groupId>
+        <artifactId>easyflow-commons</artifactId>
+        <version>${revision}</version>
+    </parent>
+
+    <name>easyflow-common-analytical-db</name>
+    <artifactId>easyflow-common-analytical-db</artifactId>
+
+    <dependencies>
+        <dependency>
+            <groupId>tech.easyflow</groupId>
+            <artifactId>easyflow-common-base</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>org.springframework.boot</groupId>
+            <artifactId>spring-boot-starter-jdbc</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>org.springframework.boot</groupId>
+            <artifactId>spring-boot-autoconfigure</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>com.clickhouse</groupId>
+            <artifactId>clickhouse-jdbc</artifactId>
+            <classifier>all</classifier>
+        </dependency>
+        <dependency>
+            <groupId>org.flywaydb</groupId>
+            <artifactId>flyway-core</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>org.flywaydb</groupId>
+            <artifactId>flyway-database-clickhouse</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>junit</groupId>
+            <artifactId>junit</artifactId>
+            <version>${junit.version}</version>
+            <scope>test</scope>
+        </dependency>
+    </dependencies>
+</project>
--- a/easyflow-commons/easyflow-common-analytical-db/src/main/java/tech/easyflow/common/analyticaldb/config/AnalyticalDBConfiguration.java
+++ b/easyflow-commons/easyflow-common-analytical-db/src/main/java/tech/easyflow/common/analyticaldb/config/AnalyticalDBConfiguration.java
@@ -0,0 +1,72 @@
+package tech.easyflow.common.analyticaldb.config;
+
+import com.zaxxer.hikari.HikariConfig;
+import com.zaxxer.hikari.HikariDataSource;
+import org.flywaydb.core.Flyway;
+import org.springframework.beans.factory.InitializingBean;
+import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.jdbc.core.JdbcTemplate;
+import org.springframework.util.StringUtils;
+import tech.easyflow.common.analyticaldb.core.AnalyticalDBOperations;
+import tech.easyflow.common.analyticaldb.core.DefaultAnalyticalDBOperations;
+
+@Configuration
+public class AnalyticalDBConfiguration {
+
+    @Bean(destroyMethod = "close")
+    @ConditionalOnProperty(prefix = "easyflow.analytical-db", name = "enabled", havingValue = "true")
+    public AnalyticalDBResources analyticalDBResources(AnalyticalDBProperties properties) {
+        HikariConfig config = new HikariConfig();
+        config.setPoolName("easyflow-analytical-db");
+        config.setDriverClassName(properties.getDriverClassName());
+        config.setJdbcUrl(properties.getUrl());
+        config.setUsername(properties.getUsername());
+        config.setPassword(properties.getPassword());
+        config.setMaximumPoolSize(properties.getPool().getMaxPoolSize());
+        config.setMinimumIdle(properties.getPool().getMinIdle());
+        config.setConnectionTimeout(properties.getPool().getConnectionTimeout());
+        config.setValidationTimeout(properties.getPool().getValidationTimeout());
+        config.setIdleTimeout(properties.getPool().getIdleTimeout());
+        config.setMaxLifetime(properties.getPool().getMaxLifetime());
+        config.setInitializationFailTimeout(-1L);
+        HikariDataSource dataSource = new HikariDataSource(config);
+        return new AnalyticalDBResources(dataSource, new JdbcTemplate(dataSource));
+    }
+
+    @Bean
+    @ConditionalOnProperty(prefix = "easyflow.analytical-db", name = "enabled", havingValue = "true")
+    public AnalyticalDBOperations analyticalDBOperations(AnalyticalDBResources resources) {
+        return new DefaultAnalyticalDBOperations(resources.jdbcTemplate());
+    }
+
+    @Bean
+    @ConditionalOnProperty(prefix = "easyflow.analytical-db", name = "enabled", havingValue = "true")
+    public InitializingBean analyticalDBFlywayInitializer(AnalyticalDBResources resources,
+                                                          AnalyticalDBFlywayProperties properties) {
+        return () -> Flyway.configure()
+                .dataSource(resources.dataSource())
+                .locations(splitLocations(properties.getLocations()))
+                .table(properties.getTable())
+                .baselineOnMigrate(properties.isBaselineOnMigrate())
+                .validateOnMigrate(properties.isValidateOnMigrate())
+                .load()
+                .migrate();
+    }
+
+    private String[] splitLocations(String locations) {
+        if (!StringUtils.hasText(locations)) {
+            return new String[]{"classpath:db/migration/analyticaldb"};
+        }
+        return StringUtils.commaDelimitedListToStringArray(locations);
+    }
+
+    public record AnalyticalDBResources(HikariDataSource dataSource, JdbcTemplate jdbcTemplate) implements AutoCloseable {
+
+        @Override
+        public void close() {
+            dataSource.close();
+        }
+    }
+}
--- a/easyflow-commons/easyflow-common-analytical-db/src/main/java/tech/easyflow/common/analyticaldb/config/AnalyticalDBFlywayProperties.java
+++ b/easyflow-commons/easyflow-common-analytical-db/src/main/java/tech/easyflow/common/analyticaldb/config/AnalyticalDBFlywayProperties.java
@@ -0,0 +1,46 @@
+package tech.easyflow.common.analyticaldb.config;
+
+import org.springframework.boot.context.properties.ConfigurationProperties;
+import org.springframework.stereotype.Component;
+
+@Component
+@ConfigurationProperties(prefix = "easyflow.flyway.analytical-db")
+public class AnalyticalDBFlywayProperties {
+
+    private String locations = "classpath:db/migration/analyticaldb";
+    private String table = "flyway_schema_history_analytical_db";
+    private boolean baselineOnMigrate = false;
+    private boolean validateOnMigrate = true;
+
+    public String getLocations() {
+        return locations;
+    }
+
+    public void setLocations(String locations) {
+        this.locations = locations;
+    }
+
+    public String getTable() {
+        return table;
+    }
+
+    public void setTable(String table) {
+        this.table = table;
+    }
+
+    public boolean isBaselineOnMigrate() {
+        return baselineOnMigrate;
+    }
+
+    public void setBaselineOnMigrate(boolean baselineOnMigrate) {
+        this.baselineOnMigrate = baselineOnMigrate;
+    }
+
+    public boolean isValidateOnMigrate() {
+        return validateOnMigrate;
+    }
+
+    public void setValidateOnMigrate(boolean validateOnMigrate) {
+        this.validateOnMigrate = validateOnMigrate;
+    }
+}
--- a/easyflow-commons/easyflow-common-analytical-db/src/main/java/tech/easyflow/common/analyticaldb/config/AnalyticalDBProperties.java
+++ b/easyflow-commons/easyflow-common-analytical-db/src/main/java/tech/easyflow/common/analyticaldb/config/AnalyticalDBProperties.java
@@ -0,0 +1,122 @@
+package tech.easyflow.common.analyticaldb.config;
+
+import org.springframework.boot.context.properties.ConfigurationProperties;
+import org.springframework.stereotype.Component;
+
+@Component
+@ConfigurationProperties(prefix = "easyflow.analytical-db")
+public class AnalyticalDBProperties {
+
+    private boolean enabled = false;
+    private String url;
+    private String username;
+    private String password;
+    private String driverClassName = "com.clickhouse.jdbc.ClickHouseDriver";
+    private Pool pool = new Pool();
+
+    public boolean isEnabled() {
+        return enabled;
+    }
+
+    public void setEnabled(boolean enabled) {
+        this.enabled = enabled;
+    }
+
+    public String getUrl() {
+        return url;
+    }
+
+    public void setUrl(String url) {
+        this.url = url;
+    }
+
+    public String getUsername() {
+        return username;
+    }
+
+    public void setUsername(String username) {
+        this.username = username;
+    }
+
+    public String getPassword() {
+        return password;
+    }
+
+    public void setPassword(String password) {
+        this.password = password;
+    }
+
+    public String getDriverClassName() {
+        return driverClassName;
+    }
+
+    public void setDriverClassName(String driverClassName) {
+        this.driverClassName = driverClassName;
+    }
+
+    public Pool getPool() {
+        return pool;
+    }
+
+    public void setPool(Pool pool) {
+        this.pool = pool;
+    }
+
+    public static class Pool {
+
+        private int maxPoolSize = 10;
+        private int minIdle = 1;
+        private long connectionTimeout = 5000L;
+        private long validationTimeout = 3000L;
+        private long idleTimeout = 600000L;
+        private long maxLifetime = 1800000L;
+
+        public int getMaxPoolSize() {
+            return maxPoolSize;
+        }
+
+        public void setMaxPoolSize(int maxPoolSize) {
+            this.maxPoolSize = maxPoolSize;
+        }
+
+        public int getMinIdle() {
+            return minIdle;
+        }
+
+        public void setMinIdle(int minIdle) {
+            this.minIdle = minIdle;
+        }
+
+        public long getConnectionTimeout() {
+            return connectionTimeout;
+        }
+
+        public void setConnectionTimeout(long connectionTimeout) {
+            this.connectionTimeout = connectionTimeout;
+        }
+
+        public long getValidationTimeout() {
+            return validationTimeout;
+        }
+
+        public void setValidationTimeout(long validationTimeout) {
+            this.validationTimeout = validationTimeout;
+        }
+
+        public long getIdleTimeout() {
+            return idleTimeout;
+        }
+
+        public void setIdleTimeout(long idleTimeout) {
+            this.idleTimeout = idleTimeout;
+        }
+
+        public long getMaxLifetime() {
+            return maxLifetime;
+        }
+
+        public void setMaxLifetime(long maxLifetime) {
+            this.maxLifetime = maxLifetime;
+        }
+    }
+}
--- a/easyflow-commons/easyflow-common-analytical-db/src/main/java/tech/easyflow/common/analyticaldb/core/AnalyticalDBOperations.java
+++ b/easyflow-commons/easyflow-common-analytical-db/src/main/java/tech/easyflow/common/analyticaldb/core/AnalyticalDBOperations.java
@@ -0,0 +1,34 @@
+package tech.easyflow.common.analyticaldb.core;
+
+import org.springframework.jdbc.core.ParameterizedPreparedStatementSetter;
+import org.springframework.jdbc.core.RowMapper;
+import tech.easyflow.common.analyticaldb.page.AnalyticalDBPageRequest;
+import tech.easyflow.common.analyticaldb.page.AnalyticalDBPageResult;
+
+import java.util.List;
+
+public interface AnalyticalDBOperations {
+
+    boolean available();
+
+    void assertAvailable();
+
+    <T> List<T> query(String sql, RowMapper<T> rowMapper, Object... args);
+
+    <T> T queryOne(String sql, Class<T> requiredType, Object... args);
+
+    <T> T queryOne(String sql, RowMapper<T> rowMapper, Object... args);
+
+    <T> List<T> queryForList(String sql, Class<T> elementType, Object... args);
+
+    int update(String sql, Object... args);
+
+    <T> int[][] batchUpdate(String sql, List<T> items, int batchSize, ParameterizedPreparedStatementSetter<T> setter);
+
+    <T> AnalyticalDBPageResult<T> page(String countSql,
+                                       Object[] countArgs,
+                                       String dataSql,
+                                       Object[] dataArgs,
+                                       AnalyticalDBPageRequest pageRequest,
+                                       RowMapper<T> rowMapper);
+}
--- a/easyflow-commons/easyflow-common-analytical-db/src/main/java/tech/easyflow/common/analyticaldb/core/DefaultAnalyticalDBOperations.java
+++ b/easyflow-commons/easyflow-common-analytical-db/src/main/java/tech/easyflow/common/analyticaldb/core/DefaultAnalyticalDBOperations.java
@@ -0,0 +1,113 @@
+package tech.easyflow.common.analyticaldb.core;
+
+import org.springframework.dao.EmptyResultDataAccessException;
+import org.springframework.jdbc.core.JdbcTemplate;
+import org.springframework.jdbc.core.ParameterizedPreparedStatementSetter;
+import org.springframework.jdbc.core.RowMapper;
+import tech.easyflow.common.analyticaldb.exception.AnalyticalDBException;
+import tech.easyflow.common.analyticaldb.page.AnalyticalDBPageRequest;
+import tech.easyflow.common.analyticaldb.page.AnalyticalDBPageResult;
+
+import java.util.ArrayList;
+import java.util.List;
+
+public class DefaultAnalyticalDBOperations implements AnalyticalDBOperations {
+
+    private final JdbcTemplate jdbcTemplate;
+
+    public DefaultAnalyticalDBOperations(JdbcTemplate jdbcTemplate) {
+        this.jdbcTemplate = jdbcTemplate;
+    }
+
+    @Override
+    public boolean available() {
+        return jdbcTemplate != null;
+    }
+
+    @Override
+    public void assertAvailable() {
+        if (!available()) {
+            throw new AnalyticalDBException("AnalyticalDB 数据源未启用");
+        }
+    }
+
+    @Override
+    public <T> List<T> query(String sql, RowMapper<T> rowMapper, Object... args) {
+        assertAvailable();
+        return jdbcTemplate.query(sql, rowMapper, args);
+    }
+
+    @Override
+    public <T> T queryOne(String sql, Class<T> requiredType, Object... args) {
+        assertAvailable();
+        try {
+            return jdbcTemplate.queryForObject(sql, requiredType, args);
+        } catch (EmptyResultDataAccessException ex) {
+            return null;
+        }
+    }
+
+    @Override
+    public <T> T queryOne(String sql, RowMapper<T> rowMapper, Object... args) {
+        assertAvailable();
+        List<T> rows = jdbcTemplate.query(sql, rowMapper, args);
+        return rows.isEmpty() ? null : rows.get(0);
+    }
+
+    @Override
+    public <T> List<T> queryForList(String sql, Class<T> elementType, Object... args) {
+        assertAvailable();
+        return jdbcTemplate.queryForList(sql, elementType, args);
+    }
+
+    @Override
+    public int update(String sql, Object... args) {
+        assertAvailable();
+        return jdbcTemplate.update(sql, args);
+    }
+
+    @Override
+    public <T> int[][] batchUpdate(String sql, List<T> items, int batchSize, ParameterizedPreparedStatementSetter<T> setter) {
+        assertAvailable();
+        if (items == null || items.isEmpty()) {
+            return new int[0][0];
+        }
+        return jdbcTemplate.batchUpdate(sql, items, Math.max(batchSize, 1), setter);
+    }
+
+    @Override
+    public <T> AnalyticalDBPageResult<T> page(String countSql,
+                                              Object[] countArgs,
+                                              String dataSql,
+                                              Object[] dataArgs,
+                                              AnalyticalDBPageRequest pageRequest,
+                                              RowMapper<T> rowMapper) {
+        assertAvailable();
+        AnalyticalDBPageRequest request = pageRequest == null ? new AnalyticalDBPageRequest() : pageRequest;
+        Long total = jdbcTemplate.queryForObject(countSql, Long.class, nullSafeArgs(countArgs));
+        List<T> records = new ArrayList<>();
+        if (total != null && total > 0) {
+            Object[] argsWithPage = appendPageArgs(dataArgs, request.getSafePageSize(), request.getOffset());
+            records = jdbcTemplate.query(dataSql + " LIMIT ? OFFSET ?", rowMapper, argsWithPage);
+        }
+        AnalyticalDBPageResult<T> result = new AnalyticalDBPageResult<>();
+        result.setTotal(total == null ? 0L : total);
+        result.setPageNumber(request.getPageNumber());
+        result.setPageSize(request.getSafePageSize());
+        result.setRecords(records);
+        return result;
+    }
+
+    private Object[] appendPageArgs(Object[] args, int limit, int offset) {
+        Object[] safeArgs = nullSafeArgs(args);
+        Object[] merged = new Object[safeArgs.length + 2];
+        System.arraycopy(safeArgs, 0, merged, 0, safeArgs.length);
+        merged[safeArgs.length] = limit;
+        merged[safeArgs.length + 1] = offset;
+        return merged;
+    }
+
+    private Object[] nullSafeArgs(Object[] args) {
+        return args == null ? new Object[0] : args;
+    }
+}
--- a/easyflow-commons/easyflow-common-analytical-db/src/main/java/tech/easyflow/common/analyticaldb/exception/AnalyticalDBException.java
+++ b/easyflow-commons/easyflow-common-analytical-db/src/main/java/tech/easyflow/common/analyticaldb/exception/AnalyticalDBException.java
@@ -0,0 +1,12 @@
+package tech.easyflow.common.analyticaldb.exception;
+
+public class AnalyticalDBException extends RuntimeException {
+
+    public AnalyticalDBException(String message) {
+        super(message);
+    }
+
+    public AnalyticalDBException(String message, Throwable cause) {
+        super(message, cause);
+    }
+}
--- a/easyflow-commons/easyflow-common-analytical-db/src/main/java/tech/easyflow/common/analyticaldb/page/AnalyticalDBPageRequest.java
+++ b/easyflow-commons/easyflow-common-analytical-db/src/main/java/tech/easyflow/common/analyticaldb/page/AnalyticalDBPageRequest.java
@@ -0,0 +1,40 @@
+package tech.easyflow.common.analyticaldb.page;
+
+public class AnalyticalDBPageRequest {
+
+    private int pageNumber = 1;
+    private int pageSize = 20;
+
+    public AnalyticalDBPageRequest() {
+    }
+
+    public AnalyticalDBPageRequest(int pageNumber, int pageSize) {
+        this.pageNumber = pageNumber;
+        this.pageSize = pageSize;
+    }
+
+    public int getPageNumber() {
+        return pageNumber;
+    }
+
+    public void setPageNumber(int pageNumber) {
+        this.pageNumber = pageNumber;
+    }
+
+    public int getPageSize() {
+        return pageSize;
+    }
+
+    public void setPageSize(int pageSize) {
+        this.pageSize = pageSize;
+    }
+
+    public int getOffset() {
+        int safePageNumber = Math.max(pageNumber, 1);
+        return (safePageNumber - 1) * getSafePageSize();
+    }
+
+    public int getSafePageSize() {
+        return Math.max(pageSize, 1);
+    }
+}
--- a/easyflow-commons/easyflow-common-analytical-db/src/main/java/tech/easyflow/common/analyticaldb/page/AnalyticalDBPageResult.java
+++ b/easyflow-commons/easyflow-common-analytical-db/src/main/java/tech/easyflow/common/analyticaldb/page/AnalyticalDBPageResult.java
@@ -0,0 +1,44 @@
+package tech.easyflow.common.analyticaldb.page;
+
+import java.util.ArrayList;
+import java.util.List;
+
+public class AnalyticalDBPageResult<T> {
+
+    private long total;
+    private int pageNumber;
+    private int pageSize;
+    private List<T> records = new ArrayList<>();
+
+    public long getTotal() {
+        return total;
+    }
+
+    public void setTotal(long total) {
+        this.total = total;
+    }
+
+    public int getPageNumber() {
+        return pageNumber;
+    }
+
+    public void setPageNumber(int pageNumber) {
+        this.pageNumber = pageNumber;
+    }
+
+    public int getPageSize() {
+        return pageSize;
+    }
+
+    public void setPageSize(int pageSize) {
+        this.pageSize = pageSize;
+    }
+
+    public List<T> getRecords() {
+        return records;
+    }
+
+    public void setRecords(List<T> records) {
+        this.records = records;
+    }
+}
--- a/easyflow-commons/easyflow-common-analytical-db/src/main/java/tech/easyflow/common/analyticaldb/support/AnalyticalDBHealthSupport.java
+++ b/easyflow-commons/easyflow-common-analytical-db/src/main/java/tech/easyflow/common/analyticaldb/support/AnalyticalDBHealthSupport.java
@@ -0,0 +1,32 @@
+package tech.easyflow.common.analyticaldb.support;
+
+import org.springframework.beans.factory.ObjectProvider;
+import org.springframework.stereotype.Component;
+import tech.easyflow.common.analyticaldb.config.AnalyticalDBFlywayProperties;
+import tech.easyflow.common.analyticaldb.core.AnalyticalDBOperations;
+
+@Component
+public class AnalyticalDBHealthSupport {
+
+    private final ObjectProvider<AnalyticalDBOperations> analyticalDBOperationsProvider;
+    private final AnalyticalDBFlywayProperties flywayProperties;
+
+    public AnalyticalDBHealthSupport(ObjectProvider<AnalyticalDBOperations> analyticalDBOperationsProvider,
+                                     AnalyticalDBFlywayProperties flywayProperties) {
+        this.analyticalDBOperationsProvider = analyticalDBOperationsProvider;
+        this.flywayProperties = flywayProperties;
+    }
+
+    public boolean enabled() {
+        return analyticalDBOperationsProvider.getIfAvailable() != null;
+    }
+
+    public void selfCheck() {
+        AnalyticalDBOperations operations = analyticalDBOperationsProvider.getIfAvailable();
+        if (operations == null) {
+            return;
+        }
+        operations.queryOne("SELECT 1", Integer.class);
+        operations.queryOne("SELECT COUNT(1) FROM " + flywayProperties.getTable(), Long.class);
+    }
+}