feat: 落地聊天记录异步持久化基础设施
- 新增 chatlog 模块、AnalyticalDB 公共层与 common-mq Redis Streams 实现 - 建立 Redis 热态、MySQL 热数据、AnalyticalDB 历史查询与同步链路 - 收紧聊天记录幂等、摘要时序与持久化失败语义
This commit is contained in:
@@ -11,6 +11,10 @@
|
||||
<artifactId>easyflow-common-all</artifactId>
|
||||
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>tech.easyflow</groupId>
|
||||
<artifactId>easyflow-common-analytical-db</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>tech.easyflow</groupId>
|
||||
<artifactId>easyflow-common-ai</artifactId>
|
||||
@@ -23,6 +27,10 @@
|
||||
<groupId>tech.easyflow</groupId>
|
||||
<artifactId>easyflow-common-cache</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>tech.easyflow</groupId>
|
||||
<artifactId>easyflow-common-mq</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>tech.easyflow</groupId>
|
||||
<artifactId>easyflow-common-file-storage</artifactId>
|
||||
|
||||
48
easyflow-commons/easyflow-common-analytical-db/pom.xml
Normal file
48
easyflow-commons/easyflow-common-analytical-db/pom.xml
Normal file
@@ -0,0 +1,48 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<project xmlns="http://maven.apache.org/POM/4.0.0"
|
||||
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
<parent>
|
||||
<groupId>tech.easyflow</groupId>
|
||||
<artifactId>easyflow-commons</artifactId>
|
||||
<version>${revision}</version>
|
||||
</parent>
|
||||
|
||||
<name>easyflow-common-analytical-db</name>
|
||||
<artifactId>easyflow-common-analytical-db</artifactId>
|
||||
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>tech.easyflow</groupId>
|
||||
<artifactId>easyflow-common-base</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.springframework.boot</groupId>
|
||||
<artifactId>spring-boot-starter-jdbc</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.springframework.boot</groupId>
|
||||
<artifactId>spring-boot-autoconfigure</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>com.clickhouse</groupId>
|
||||
<artifactId>clickhouse-jdbc</artifactId>
|
||||
<classifier>all</classifier>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.flywaydb</groupId>
|
||||
<artifactId>flyway-core</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.flywaydb</groupId>
|
||||
<artifactId>flyway-database-clickhouse</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>junit</groupId>
|
||||
<artifactId>junit</artifactId>
|
||||
<version>${junit.version}</version>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
</project>
|
||||
@@ -0,0 +1,72 @@
|
||||
package tech.easyflow.common.analyticaldb.config;
|
||||
|
||||
import com.zaxxer.hikari.HikariConfig;
|
||||
import com.zaxxer.hikari.HikariDataSource;
|
||||
import org.flywaydb.core.Flyway;
|
||||
import org.springframework.beans.factory.InitializingBean;
|
||||
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
|
||||
import org.springframework.context.annotation.Bean;
|
||||
import org.springframework.context.annotation.Configuration;
|
||||
import org.springframework.jdbc.core.JdbcTemplate;
|
||||
import org.springframework.util.StringUtils;
|
||||
import tech.easyflow.common.analyticaldb.core.AnalyticalDBOperations;
|
||||
import tech.easyflow.common.analyticaldb.core.DefaultAnalyticalDBOperations;
|
||||
|
||||
@Configuration
|
||||
public class AnalyticalDBConfiguration {
|
||||
|
||||
@Bean(destroyMethod = "close")
|
||||
@ConditionalOnProperty(prefix = "easyflow.analytical-db", name = "enabled", havingValue = "true")
|
||||
public AnalyticalDBResources analyticalDBResources(AnalyticalDBProperties properties) {
|
||||
HikariConfig config = new HikariConfig();
|
||||
config.setPoolName("easyflow-analytical-db");
|
||||
config.setDriverClassName(properties.getDriverClassName());
|
||||
config.setJdbcUrl(properties.getUrl());
|
||||
config.setUsername(properties.getUsername());
|
||||
config.setPassword(properties.getPassword());
|
||||
config.setMaximumPoolSize(properties.getPool().getMaxPoolSize());
|
||||
config.setMinimumIdle(properties.getPool().getMinIdle());
|
||||
config.setConnectionTimeout(properties.getPool().getConnectionTimeout());
|
||||
config.setValidationTimeout(properties.getPool().getValidationTimeout());
|
||||
config.setIdleTimeout(properties.getPool().getIdleTimeout());
|
||||
config.setMaxLifetime(properties.getPool().getMaxLifetime());
|
||||
config.setInitializationFailTimeout(-1L);
|
||||
HikariDataSource dataSource = new HikariDataSource(config);
|
||||
return new AnalyticalDBResources(dataSource, new JdbcTemplate(dataSource));
|
||||
}
|
||||
|
||||
@Bean
|
||||
@ConditionalOnProperty(prefix = "easyflow.analytical-db", name = "enabled", havingValue = "true")
|
||||
public AnalyticalDBOperations analyticalDBOperations(AnalyticalDBResources resources) {
|
||||
return new DefaultAnalyticalDBOperations(resources.jdbcTemplate());
|
||||
}
|
||||
|
||||
@Bean
|
||||
@ConditionalOnProperty(prefix = "easyflow.analytical-db", name = "enabled", havingValue = "true")
|
||||
public InitializingBean analyticalDBFlywayInitializer(AnalyticalDBResources resources,
|
||||
AnalyticalDBFlywayProperties properties) {
|
||||
return () -> Flyway.configure()
|
||||
.dataSource(resources.dataSource())
|
||||
.locations(splitLocations(properties.getLocations()))
|
||||
.table(properties.getTable())
|
||||
.baselineOnMigrate(properties.isBaselineOnMigrate())
|
||||
.validateOnMigrate(properties.isValidateOnMigrate())
|
||||
.load()
|
||||
.migrate();
|
||||
}
|
||||
|
||||
private String[] splitLocations(String locations) {
|
||||
if (!StringUtils.hasText(locations)) {
|
||||
return new String[]{"classpath:db/migration/analyticaldb"};
|
||||
}
|
||||
return StringUtils.commaDelimitedListToStringArray(locations);
|
||||
}
|
||||
|
||||
public record AnalyticalDBResources(HikariDataSource dataSource, JdbcTemplate jdbcTemplate) implements AutoCloseable {
|
||||
|
||||
@Override
|
||||
public void close() {
|
||||
dataSource.close();
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,46 @@
|
||||
package tech.easyflow.common.analyticaldb.config;
|
||||
|
||||
import org.springframework.boot.context.properties.ConfigurationProperties;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
@Component
|
||||
@ConfigurationProperties(prefix = "easyflow.flyway.analytical-db")
|
||||
public class AnalyticalDBFlywayProperties {
|
||||
|
||||
private String locations = "classpath:db/migration/analyticaldb";
|
||||
private String table = "flyway_schema_history_analytical_db";
|
||||
private boolean baselineOnMigrate = false;
|
||||
private boolean validateOnMigrate = true;
|
||||
|
||||
public String getLocations() {
|
||||
return locations;
|
||||
}
|
||||
|
||||
public void setLocations(String locations) {
|
||||
this.locations = locations;
|
||||
}
|
||||
|
||||
public String getTable() {
|
||||
return table;
|
||||
}
|
||||
|
||||
public void setTable(String table) {
|
||||
this.table = table;
|
||||
}
|
||||
|
||||
public boolean isBaselineOnMigrate() {
|
||||
return baselineOnMigrate;
|
||||
}
|
||||
|
||||
public void setBaselineOnMigrate(boolean baselineOnMigrate) {
|
||||
this.baselineOnMigrate = baselineOnMigrate;
|
||||
}
|
||||
|
||||
public boolean isValidateOnMigrate() {
|
||||
return validateOnMigrate;
|
||||
}
|
||||
|
||||
public void setValidateOnMigrate(boolean validateOnMigrate) {
|
||||
this.validateOnMigrate = validateOnMigrate;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,122 @@
|
||||
package tech.easyflow.common.analyticaldb.config;
|
||||
|
||||
import org.springframework.boot.context.properties.ConfigurationProperties;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
@Component
|
||||
@ConfigurationProperties(prefix = "easyflow.analytical-db")
|
||||
public class AnalyticalDBProperties {
|
||||
|
||||
private boolean enabled = false;
|
||||
private String url;
|
||||
private String username;
|
||||
private String password;
|
||||
private String driverClassName = "com.clickhouse.jdbc.ClickHouseDriver";
|
||||
private Pool pool = new Pool();
|
||||
|
||||
public boolean isEnabled() {
|
||||
return enabled;
|
||||
}
|
||||
|
||||
public void setEnabled(boolean enabled) {
|
||||
this.enabled = enabled;
|
||||
}
|
||||
|
||||
public String getUrl() {
|
||||
return url;
|
||||
}
|
||||
|
||||
public void setUrl(String url) {
|
||||
this.url = url;
|
||||
}
|
||||
|
||||
public String getUsername() {
|
||||
return username;
|
||||
}
|
||||
|
||||
public void setUsername(String username) {
|
||||
this.username = username;
|
||||
}
|
||||
|
||||
public String getPassword() {
|
||||
return password;
|
||||
}
|
||||
|
||||
public void setPassword(String password) {
|
||||
this.password = password;
|
||||
}
|
||||
|
||||
public String getDriverClassName() {
|
||||
return driverClassName;
|
||||
}
|
||||
|
||||
public void setDriverClassName(String driverClassName) {
|
||||
this.driverClassName = driverClassName;
|
||||
}
|
||||
|
||||
public Pool getPool() {
|
||||
return pool;
|
||||
}
|
||||
|
||||
public void setPool(Pool pool) {
|
||||
this.pool = pool;
|
||||
}
|
||||
|
||||
public static class Pool {
|
||||
|
||||
private int maxPoolSize = 10;
|
||||
private int minIdle = 1;
|
||||
private long connectionTimeout = 5000L;
|
||||
private long validationTimeout = 3000L;
|
||||
private long idleTimeout = 600000L;
|
||||
private long maxLifetime = 1800000L;
|
||||
|
||||
public int getMaxPoolSize() {
|
||||
return maxPoolSize;
|
||||
}
|
||||
|
||||
public void setMaxPoolSize(int maxPoolSize) {
|
||||
this.maxPoolSize = maxPoolSize;
|
||||
}
|
||||
|
||||
public int getMinIdle() {
|
||||
return minIdle;
|
||||
}
|
||||
|
||||
public void setMinIdle(int minIdle) {
|
||||
this.minIdle = minIdle;
|
||||
}
|
||||
|
||||
public long getConnectionTimeout() {
|
||||
return connectionTimeout;
|
||||
}
|
||||
|
||||
public void setConnectionTimeout(long connectionTimeout) {
|
||||
this.connectionTimeout = connectionTimeout;
|
||||
}
|
||||
|
||||
public long getValidationTimeout() {
|
||||
return validationTimeout;
|
||||
}
|
||||
|
||||
public void setValidationTimeout(long validationTimeout) {
|
||||
this.validationTimeout = validationTimeout;
|
||||
}
|
||||
|
||||
public long getIdleTimeout() {
|
||||
return idleTimeout;
|
||||
}
|
||||
|
||||
public void setIdleTimeout(long idleTimeout) {
|
||||
this.idleTimeout = idleTimeout;
|
||||
}
|
||||
|
||||
public long getMaxLifetime() {
|
||||
return maxLifetime;
|
||||
}
|
||||
|
||||
public void setMaxLifetime(long maxLifetime) {
|
||||
this.maxLifetime = maxLifetime;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,34 @@
|
||||
package tech.easyflow.common.analyticaldb.core;
|
||||
|
||||
import org.springframework.jdbc.core.ParameterizedPreparedStatementSetter;
|
||||
import org.springframework.jdbc.core.RowMapper;
|
||||
import tech.easyflow.common.analyticaldb.page.AnalyticalDBPageRequest;
|
||||
import tech.easyflow.common.analyticaldb.page.AnalyticalDBPageResult;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
public interface AnalyticalDBOperations {
|
||||
|
||||
boolean available();
|
||||
|
||||
void assertAvailable();
|
||||
|
||||
<T> List<T> query(String sql, RowMapper<T> rowMapper, Object... args);
|
||||
|
||||
<T> T queryOne(String sql, Class<T> requiredType, Object... args);
|
||||
|
||||
<T> T queryOne(String sql, RowMapper<T> rowMapper, Object... args);
|
||||
|
||||
<T> List<T> queryForList(String sql, Class<T> elementType, Object... args);
|
||||
|
||||
int update(String sql, Object... args);
|
||||
|
||||
<T> int[][] batchUpdate(String sql, List<T> items, int batchSize, ParameterizedPreparedStatementSetter<T> setter);
|
||||
|
||||
<T> AnalyticalDBPageResult<T> page(String countSql,
|
||||
Object[] countArgs,
|
||||
String dataSql,
|
||||
Object[] dataArgs,
|
||||
AnalyticalDBPageRequest pageRequest,
|
||||
RowMapper<T> rowMapper);
|
||||
}
|
||||
@@ -0,0 +1,113 @@
|
||||
package tech.easyflow.common.analyticaldb.core;
|
||||
|
||||
import org.springframework.dao.EmptyResultDataAccessException;
|
||||
import org.springframework.jdbc.core.JdbcTemplate;
|
||||
import org.springframework.jdbc.core.ParameterizedPreparedStatementSetter;
|
||||
import org.springframework.jdbc.core.RowMapper;
|
||||
import tech.easyflow.common.analyticaldb.exception.AnalyticalDBException;
|
||||
import tech.easyflow.common.analyticaldb.page.AnalyticalDBPageRequest;
|
||||
import tech.easyflow.common.analyticaldb.page.AnalyticalDBPageResult;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
public class DefaultAnalyticalDBOperations implements AnalyticalDBOperations {
|
||||
|
||||
private final JdbcTemplate jdbcTemplate;
|
||||
|
||||
public DefaultAnalyticalDBOperations(JdbcTemplate jdbcTemplate) {
|
||||
this.jdbcTemplate = jdbcTemplate;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean available() {
|
||||
return jdbcTemplate != null;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void assertAvailable() {
|
||||
if (!available()) {
|
||||
throw new AnalyticalDBException("AnalyticalDB 数据源未启用");
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public <T> List<T> query(String sql, RowMapper<T> rowMapper, Object... args) {
|
||||
assertAvailable();
|
||||
return jdbcTemplate.query(sql, rowMapper, args);
|
||||
}
|
||||
|
||||
@Override
|
||||
public <T> T queryOne(String sql, Class<T> requiredType, Object... args) {
|
||||
assertAvailable();
|
||||
try {
|
||||
return jdbcTemplate.queryForObject(sql, requiredType, args);
|
||||
} catch (EmptyResultDataAccessException ex) {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public <T> T queryOne(String sql, RowMapper<T> rowMapper, Object... args) {
|
||||
assertAvailable();
|
||||
List<T> rows = jdbcTemplate.query(sql, rowMapper, args);
|
||||
return rows.isEmpty() ? null : rows.get(0);
|
||||
}
|
||||
|
||||
@Override
|
||||
public <T> List<T> queryForList(String sql, Class<T> elementType, Object... args) {
|
||||
assertAvailable();
|
||||
return jdbcTemplate.queryForList(sql, elementType, args);
|
||||
}
|
||||
|
||||
@Override
|
||||
public int update(String sql, Object... args) {
|
||||
assertAvailable();
|
||||
return jdbcTemplate.update(sql, args);
|
||||
}
|
||||
|
||||
@Override
|
||||
public <T> int[][] batchUpdate(String sql, List<T> items, int batchSize, ParameterizedPreparedStatementSetter<T> setter) {
|
||||
assertAvailable();
|
||||
if (items == null || items.isEmpty()) {
|
||||
return new int[0][0];
|
||||
}
|
||||
return jdbcTemplate.batchUpdate(sql, items, Math.max(batchSize, 1), setter);
|
||||
}
|
||||
|
||||
@Override
|
||||
public <T> AnalyticalDBPageResult<T> page(String countSql,
|
||||
Object[] countArgs,
|
||||
String dataSql,
|
||||
Object[] dataArgs,
|
||||
AnalyticalDBPageRequest pageRequest,
|
||||
RowMapper<T> rowMapper) {
|
||||
assertAvailable();
|
||||
AnalyticalDBPageRequest request = pageRequest == null ? new AnalyticalDBPageRequest() : pageRequest;
|
||||
Long total = jdbcTemplate.queryForObject(countSql, Long.class, nullSafeArgs(countArgs));
|
||||
List<T> records = new ArrayList<>();
|
||||
if (total != null && total > 0) {
|
||||
Object[] argsWithPage = appendPageArgs(dataArgs, request.getSafePageSize(), request.getOffset());
|
||||
records = jdbcTemplate.query(dataSql + " LIMIT ? OFFSET ?", rowMapper, argsWithPage);
|
||||
}
|
||||
AnalyticalDBPageResult<T> result = new AnalyticalDBPageResult<>();
|
||||
result.setTotal(total == null ? 0L : total);
|
||||
result.setPageNumber(request.getPageNumber());
|
||||
result.setPageSize(request.getSafePageSize());
|
||||
result.setRecords(records);
|
||||
return result;
|
||||
}
|
||||
|
||||
private Object[] appendPageArgs(Object[] args, int limit, int offset) {
|
||||
Object[] safeArgs = nullSafeArgs(args);
|
||||
Object[] merged = new Object[safeArgs.length + 2];
|
||||
System.arraycopy(safeArgs, 0, merged, 0, safeArgs.length);
|
||||
merged[safeArgs.length] = limit;
|
||||
merged[safeArgs.length + 1] = offset;
|
||||
return merged;
|
||||
}
|
||||
|
||||
private Object[] nullSafeArgs(Object[] args) {
|
||||
return args == null ? new Object[0] : args;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,12 @@
|
||||
package tech.easyflow.common.analyticaldb.exception;
|
||||
|
||||
public class AnalyticalDBException extends RuntimeException {
|
||||
|
||||
public AnalyticalDBException(String message) {
|
||||
super(message);
|
||||
}
|
||||
|
||||
public AnalyticalDBException(String message, Throwable cause) {
|
||||
super(message, cause);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,40 @@
|
||||
package tech.easyflow.common.analyticaldb.page;
|
||||
|
||||
public class AnalyticalDBPageRequest {
|
||||
|
||||
private int pageNumber = 1;
|
||||
private int pageSize = 20;
|
||||
|
||||
public AnalyticalDBPageRequest() {
|
||||
}
|
||||
|
||||
public AnalyticalDBPageRequest(int pageNumber, int pageSize) {
|
||||
this.pageNumber = pageNumber;
|
||||
this.pageSize = pageSize;
|
||||
}
|
||||
|
||||
public int getPageNumber() {
|
||||
return pageNumber;
|
||||
}
|
||||
|
||||
public void setPageNumber(int pageNumber) {
|
||||
this.pageNumber = pageNumber;
|
||||
}
|
||||
|
||||
public int getPageSize() {
|
||||
return pageSize;
|
||||
}
|
||||
|
||||
public void setPageSize(int pageSize) {
|
||||
this.pageSize = pageSize;
|
||||
}
|
||||
|
||||
public int getOffset() {
|
||||
int safePageNumber = Math.max(pageNumber, 1);
|
||||
return (safePageNumber - 1) * getSafePageSize();
|
||||
}
|
||||
|
||||
public int getSafePageSize() {
|
||||
return Math.max(pageSize, 1);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,44 @@
|
||||
package tech.easyflow.common.analyticaldb.page;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
public class AnalyticalDBPageResult<T> {
|
||||
|
||||
private long total;
|
||||
private int pageNumber;
|
||||
private int pageSize;
|
||||
private List<T> records = new ArrayList<>();
|
||||
|
||||
public long getTotal() {
|
||||
return total;
|
||||
}
|
||||
|
||||
public void setTotal(long total) {
|
||||
this.total = total;
|
||||
}
|
||||
|
||||
public int getPageNumber() {
|
||||
return pageNumber;
|
||||
}
|
||||
|
||||
public void setPageNumber(int pageNumber) {
|
||||
this.pageNumber = pageNumber;
|
||||
}
|
||||
|
||||
public int getPageSize() {
|
||||
return pageSize;
|
||||
}
|
||||
|
||||
public void setPageSize(int pageSize) {
|
||||
this.pageSize = pageSize;
|
||||
}
|
||||
|
||||
public List<T> getRecords() {
|
||||
return records;
|
||||
}
|
||||
|
||||
public void setRecords(List<T> records) {
|
||||
this.records = records;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,32 @@
|
||||
package tech.easyflow.common.analyticaldb.support;
|
||||
|
||||
import org.springframework.beans.factory.ObjectProvider;
|
||||
import org.springframework.stereotype.Component;
|
||||
import tech.easyflow.common.analyticaldb.config.AnalyticalDBFlywayProperties;
|
||||
import tech.easyflow.common.analyticaldb.core.AnalyticalDBOperations;
|
||||
|
||||
@Component
|
||||
public class AnalyticalDBHealthSupport {
|
||||
|
||||
private final ObjectProvider<AnalyticalDBOperations> analyticalDBOperationsProvider;
|
||||
private final AnalyticalDBFlywayProperties flywayProperties;
|
||||
|
||||
public AnalyticalDBHealthSupport(ObjectProvider<AnalyticalDBOperations> analyticalDBOperationsProvider,
|
||||
AnalyticalDBFlywayProperties flywayProperties) {
|
||||
this.analyticalDBOperationsProvider = analyticalDBOperationsProvider;
|
||||
this.flywayProperties = flywayProperties;
|
||||
}
|
||||
|
||||
public boolean enabled() {
|
||||
return analyticalDBOperationsProvider.getIfAvailable() != null;
|
||||
}
|
||||
|
||||
public void selfCheck() {
|
||||
AnalyticalDBOperations operations = analyticalDBOperationsProvider.getIfAvailable();
|
||||
if (operations == null) {
|
||||
return;
|
||||
}
|
||||
operations.queryOne("SELECT 1", Integer.class);
|
||||
operations.queryOne("SELECT COUNT(1) FROM " + flywayProperties.getTable(), Long.class);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,74 @@
|
||||
package tech.easyflow.core.runtime;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
public class ChatAssistantAccumulator {
|
||||
|
||||
private final StringBuilder content = new StringBuilder();
|
||||
private final StringBuilder reasoning = new StringBuilder();
|
||||
private final List<Map<String, Object>> chains = new ArrayList<>();
|
||||
|
||||
public void appendContent(String delta) {
|
||||
if (delta != null && !delta.isEmpty()) {
|
||||
content.append(delta);
|
||||
}
|
||||
}
|
||||
|
||||
public void appendReasoning(String delta) {
|
||||
if (delta != null && !delta.isEmpty()) {
|
||||
reasoning.append(delta);
|
||||
}
|
||||
}
|
||||
|
||||
public void appendToolCall(String id, String name, Object arguments) {
|
||||
Map<String, Object> chain = findToolChain(id, name);
|
||||
chain.put("status", "TOOL_CALL");
|
||||
chain.put("result", arguments);
|
||||
}
|
||||
|
||||
public void appendToolResult(String id, String name, Object result) {
|
||||
Map<String, Object> chain = findToolChain(id, name);
|
||||
chain.put("status", "TOOL_RESULT");
|
||||
chain.put("result", result);
|
||||
}
|
||||
|
||||
public String getContent() {
|
||||
return content.toString();
|
||||
}
|
||||
|
||||
public Map<String, Object> buildPayload() {
|
||||
Map<String, Object> payload = new LinkedHashMap<>();
|
||||
List<Map<String, Object>> payloadChains = new ArrayList<>();
|
||||
if (reasoning.length() > 0) {
|
||||
Map<String, Object> think = new LinkedHashMap<>();
|
||||
think.put("reasoning_content", reasoning.toString());
|
||||
think.put("thinkingStatus", "end");
|
||||
think.put("thinlCollapse", Boolean.TRUE);
|
||||
payloadChains.add(think);
|
||||
}
|
||||
payloadChains.addAll(chains);
|
||||
if (!payloadChains.isEmpty()) {
|
||||
payload.put("chains", payloadChains);
|
||||
}
|
||||
return payload;
|
||||
}
|
||||
|
||||
private Map<String, Object> findToolChain(String id, String name) {
|
||||
for (Map<String, Object> chain : chains) {
|
||||
if (String.valueOf(chain.get("id")).equals(id)) {
|
||||
if (name != null && !name.isEmpty()) {
|
||||
chain.put("name", name);
|
||||
}
|
||||
return chain;
|
||||
}
|
||||
}
|
||||
Map<String, Object> chain = new LinkedHashMap<>();
|
||||
chain.put("id", id);
|
||||
chain.put("name", name);
|
||||
chains.add(chain);
|
||||
return chain;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,8 @@
|
||||
package tech.easyflow.core.runtime;
|
||||
|
||||
public enum ChatChannel {
|
||||
|
||||
ADMIN,
|
||||
USER_CENTER,
|
||||
PUBLIC_API
|
||||
}
|
||||
@@ -0,0 +1,138 @@
|
||||
package tech.easyflow.core.runtime;
|
||||
|
||||
import java.io.Serializable;
|
||||
import java.math.BigInteger;
|
||||
import java.util.ArrayList;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
public class ChatRuntimeContext implements Serializable {
|
||||
|
||||
private ChatChannel channel;
|
||||
private BigInteger sessionId;
|
||||
private BigInteger tenantId;
|
||||
private BigInteger deptId;
|
||||
private BigInteger userId;
|
||||
private String userAccount;
|
||||
private String userName;
|
||||
private BigInteger assistantId;
|
||||
private String assistantCode;
|
||||
private String assistantName;
|
||||
private String sessionTitle;
|
||||
private boolean anonymous;
|
||||
private List<String> attachments = new ArrayList<>();
|
||||
private Map<String, Object> ext = new LinkedHashMap<>();
|
||||
|
||||
public ChatChannel getChannel() {
|
||||
return channel;
|
||||
}
|
||||
|
||||
public void setChannel(ChatChannel channel) {
|
||||
this.channel = channel;
|
||||
}
|
||||
|
||||
public BigInteger getSessionId() {
|
||||
return sessionId;
|
||||
}
|
||||
|
||||
public void setSessionId(BigInteger sessionId) {
|
||||
this.sessionId = sessionId;
|
||||
}
|
||||
|
||||
public BigInteger getTenantId() {
|
||||
return tenantId;
|
||||
}
|
||||
|
||||
public void setTenantId(BigInteger tenantId) {
|
||||
this.tenantId = tenantId;
|
||||
}
|
||||
|
||||
public BigInteger getDeptId() {
|
||||
return deptId;
|
||||
}
|
||||
|
||||
public void setDeptId(BigInteger deptId) {
|
||||
this.deptId = deptId;
|
||||
}
|
||||
|
||||
public BigInteger getUserId() {
|
||||
return userId;
|
||||
}
|
||||
|
||||
public void setUserId(BigInteger userId) {
|
||||
this.userId = userId;
|
||||
}
|
||||
|
||||
public String getUserAccount() {
|
||||
return userAccount;
|
||||
}
|
||||
|
||||
public void setUserAccount(String userAccount) {
|
||||
this.userAccount = userAccount;
|
||||
}
|
||||
|
||||
public String getUserName() {
|
||||
return userName;
|
||||
}
|
||||
|
||||
public void setUserName(String userName) {
|
||||
this.userName = userName;
|
||||
}
|
||||
|
||||
public BigInteger getAssistantId() {
|
||||
return assistantId;
|
||||
}
|
||||
|
||||
public void setAssistantId(BigInteger assistantId) {
|
||||
this.assistantId = assistantId;
|
||||
}
|
||||
|
||||
public String getAssistantCode() {
|
||||
return assistantCode;
|
||||
}
|
||||
|
||||
public void setAssistantCode(String assistantCode) {
|
||||
this.assistantCode = assistantCode;
|
||||
}
|
||||
|
||||
public String getAssistantName() {
|
||||
return assistantName;
|
||||
}
|
||||
|
||||
public void setAssistantName(String assistantName) {
|
||||
this.assistantName = assistantName;
|
||||
}
|
||||
|
||||
public String getSessionTitle() {
|
||||
return sessionTitle;
|
||||
}
|
||||
|
||||
public void setSessionTitle(String sessionTitle) {
|
||||
this.sessionTitle = sessionTitle;
|
||||
}
|
||||
|
||||
public boolean isAnonymous() {
|
||||
return anonymous;
|
||||
}
|
||||
|
||||
public void setAnonymous(boolean anonymous) {
|
||||
this.anonymous = anonymous;
|
||||
}
|
||||
|
||||
public List<String> getAttachments() {
|
||||
return attachments;
|
||||
}
|
||||
|
||||
public void setAttachments(List<String> attachments) {
|
||||
this.attachments = attachments == null ? new ArrayList<>() : attachments;
|
||||
}
|
||||
|
||||
public Map<String, Object> getExt() {
|
||||
return ext;
|
||||
}
|
||||
|
||||
public void setExt(Map<String, Object> ext) {
|
||||
this.ext = ext == null ? new LinkedHashMap<>() : ext;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,29 @@
|
||||
package tech.easyflow.core.runtime;
|
||||
|
||||
import java.util.Collections;
|
||||
import java.util.List;
|
||||
|
||||
public interface ChatRuntimeListener {
|
||||
|
||||
default void onSessionPrepared(ChatRuntimeContext context) {
|
||||
}
|
||||
|
||||
default void onUserMessage(ChatRuntimeContext context, ChatRuntimeMessage message) {
|
||||
}
|
||||
|
||||
default void onAssistantDelta(ChatRuntimeContext context, ChatRuntimeMessage message) {
|
||||
}
|
||||
|
||||
default void onAssistantCompleted(ChatRuntimeContext context, ChatRuntimeMessage message) {
|
||||
}
|
||||
|
||||
default void onChatFailed(ChatRuntimeContext context, Throwable throwable) {
|
||||
}
|
||||
|
||||
default void onChatCompleted(ChatRuntimeContext context) {
|
||||
}
|
||||
|
||||
default List<ChatRuntimeMessage> loadMessages(ChatRuntimeContext context, int limit) {
|
||||
return Collections.emptyList();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,20 @@
|
||||
package tech.easyflow.core.runtime;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
public interface ChatRuntimeManager {
|
||||
|
||||
void prepareSession(ChatRuntimeContext context);
|
||||
|
||||
void recordUserMessage(ChatRuntimeContext context, ChatRuntimeMessage message);
|
||||
|
||||
void recordAssistantDelta(ChatRuntimeContext context, ChatRuntimeMessage message);
|
||||
|
||||
void recordAssistantCompleted(ChatRuntimeContext context, ChatRuntimeMessage message);
|
||||
|
||||
void recordFailure(ChatRuntimeContext context, Throwable throwable);
|
||||
|
||||
void recordCompleted(ChatRuntimeContext context);
|
||||
|
||||
List<ChatRuntimeMessage> loadMessages(ChatRuntimeContext context, int limit);
|
||||
}
|
||||
@@ -0,0 +1,83 @@
|
||||
package tech.easyflow.core.runtime;
|
||||
|
||||
import java.io.Serializable;
|
||||
import java.math.BigInteger;
|
||||
import java.util.Date;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.Map;
|
||||
|
||||
public class ChatRuntimeMessage implements Serializable {
|
||||
|
||||
private BigInteger messageId;
|
||||
private String role;
|
||||
private String contentType = "TEXT";
|
||||
private String contentText;
|
||||
private Map<String, Object> contentPayload = new LinkedHashMap<>();
|
||||
private Date createdAt = new Date();
|
||||
private BigInteger senderId;
|
||||
private String senderName;
|
||||
|
||||
public BigInteger getMessageId() {
|
||||
return messageId;
|
||||
}
|
||||
|
||||
public void setMessageId(BigInteger messageId) {
|
||||
this.messageId = messageId;
|
||||
}
|
||||
|
||||
public String getRole() {
|
||||
return role;
|
||||
}
|
||||
|
||||
public void setRole(String role) {
|
||||
this.role = role;
|
||||
}
|
||||
|
||||
public String getContentType() {
|
||||
return contentType;
|
||||
}
|
||||
|
||||
public void setContentType(String contentType) {
|
||||
this.contentType = contentType;
|
||||
}
|
||||
|
||||
public String getContentText() {
|
||||
return contentText;
|
||||
}
|
||||
|
||||
public void setContentText(String contentText) {
|
||||
this.contentText = contentText;
|
||||
}
|
||||
|
||||
public Map<String, Object> getContentPayload() {
|
||||
return contentPayload;
|
||||
}
|
||||
|
||||
public void setContentPayload(Map<String, Object> contentPayload) {
|
||||
this.contentPayload = contentPayload == null ? new LinkedHashMap<>() : contentPayload;
|
||||
}
|
||||
|
||||
public Date getCreatedAt() {
|
||||
return createdAt;
|
||||
}
|
||||
|
||||
public void setCreatedAt(Date createdAt) {
|
||||
this.createdAt = createdAt;
|
||||
}
|
||||
|
||||
public BigInteger getSenderId() {
|
||||
return senderId;
|
||||
}
|
||||
|
||||
public void setSenderId(BigInteger senderId) {
|
||||
this.senderId = senderId;
|
||||
}
|
||||
|
||||
public String getSenderName() {
|
||||
return senderName;
|
||||
}
|
||||
|
||||
public void setSenderName(String senderName) {
|
||||
this.senderName = senderName;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,93 @@
|
||||
package tech.easyflow.core.runtime;
|
||||
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.beans.factory.ObjectProvider;
|
||||
import org.springframework.stereotype.Component;
|
||||
import tech.easyflow.common.web.exceptions.BusinessException;
|
||||
|
||||
import java.util.Collections;
|
||||
import java.util.List;
|
||||
|
||||
@Component
|
||||
public class CompositeChatRuntimeManager implements ChatRuntimeManager {
|
||||
|
||||
private static final Logger log = LoggerFactory.getLogger(CompositeChatRuntimeManager.class);
|
||||
|
||||
private final ObjectProvider<ChatRuntimeListener> listenerProvider;
|
||||
|
||||
public CompositeChatRuntimeManager(ObjectProvider<ChatRuntimeListener> listenerProvider) {
|
||||
this.listenerProvider = listenerProvider;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void prepareSession(ChatRuntimeContext context) {
|
||||
forEach(listener -> listener.onSessionPrepared(context), "prepareSession", context);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void recordUserMessage(ChatRuntimeContext context, ChatRuntimeMessage message) {
|
||||
forEach(listener -> listener.onUserMessage(context, message), "recordUserMessage", context);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void recordAssistantDelta(ChatRuntimeContext context, ChatRuntimeMessage message) {
|
||||
forEach(listener -> listener.onAssistantDelta(context, message), "recordAssistantDelta", context);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void recordAssistantCompleted(ChatRuntimeContext context, ChatRuntimeMessage message) {
|
||||
forEach(listener -> listener.onAssistantCompleted(context, message), "recordAssistantCompleted", context);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void recordFailure(ChatRuntimeContext context, Throwable throwable) {
|
||||
forEach(listener -> listener.onChatFailed(context, throwable), "recordFailure", context);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void recordCompleted(ChatRuntimeContext context) {
|
||||
forEach(listener -> listener.onChatCompleted(context), "recordCompleted", context);
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<ChatRuntimeMessage> loadMessages(ChatRuntimeContext context, int limit) {
|
||||
for (ChatRuntimeListener listener : listenerProvider.orderedStream().toList()) {
|
||||
try {
|
||||
List<ChatRuntimeMessage> messages = listener.loadMessages(context, limit);
|
||||
if (messages != null && !messages.isEmpty()) {
|
||||
return messages;
|
||||
}
|
||||
} catch (Exception ex) {
|
||||
log.warn("chat runtime loadMessages failed, channel={}, sessionId={}, listener={}",
|
||||
context == null || context.getChannel() == null ? null : context.getChannel().name(),
|
||||
context == null ? null : context.getSessionId(),
|
||||
listener.getClass().getName(),
|
||||
ex);
|
||||
}
|
||||
}
|
||||
return Collections.emptyList();
|
||||
}
|
||||
|
||||
private void forEach(ListenerConsumer consumer, String action, ChatRuntimeContext context) {
|
||||
for (ChatRuntimeListener listener : listenerProvider.orderedStream().toList()) {
|
||||
try {
|
||||
consumer.accept(listener);
|
||||
} catch (BusinessException ex) {
|
||||
throw ex;
|
||||
} catch (Exception ex) {
|
||||
log.warn("chat runtime {} failed, channel={}, sessionId={}, listener={}",
|
||||
action,
|
||||
context == null || context.getChannel() == null ? null : context.getChannel().name(),
|
||||
context == null ? null : context.getSessionId(),
|
||||
listener.getClass().getName(),
|
||||
ex);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@FunctionalInterface
|
||||
private interface ListenerConsumer {
|
||||
void accept(ChatRuntimeListener listener);
|
||||
}
|
||||
}
|
||||
26
easyflow-commons/easyflow-common-mq/pom.xml
Normal file
26
easyflow-commons/easyflow-common-mq/pom.xml
Normal file
@@ -0,0 +1,26 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<project xmlns="http://maven.apache.org/POM/4.0.0"
|
||||
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
<parent>
|
||||
<groupId>tech.easyflow</groupId>
|
||||
<artifactId>easyflow-commons</artifactId>
|
||||
<version>${revision}</version>
|
||||
</parent>
|
||||
|
||||
<name>easyflow-common-mq</name>
|
||||
<artifactId>easyflow-common-mq</artifactId>
|
||||
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>org.springframework.boot</groupId>
|
||||
<artifactId>spring-boot-starter-data-redis</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>com.fasterxml.jackson.core</groupId>
|
||||
<artifactId>jackson-databind</artifactId>
|
||||
<version>${jackson.version}</version>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
</project>
|
||||
@@ -0,0 +1,129 @@
|
||||
package tech.easyflow.common.mq.config;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import org.springframework.beans.factory.ObjectProvider;
|
||||
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
|
||||
import org.springframework.boot.autoconfigure.data.redis.RedisProperties;
|
||||
import org.springframework.boot.context.properties.EnableConfigurationProperties;
|
||||
import org.springframework.context.annotation.Bean;
|
||||
import org.springframework.context.annotation.Configuration;
|
||||
import org.springframework.data.redis.connection.RedisPassword;
|
||||
import org.springframework.data.redis.connection.RedisStandaloneConfiguration;
|
||||
import org.springframework.data.redis.connection.lettuce.LettuceConnectionFactory;
|
||||
import org.springframework.data.redis.core.StringRedisTemplate;
|
||||
import tech.easyflow.common.mq.core.MQConsumerContainer;
|
||||
import tech.easyflow.common.mq.core.MQConsumerHandler;
|
||||
import tech.easyflow.common.mq.core.MQDeadLetterHandler;
|
||||
import tech.easyflow.common.mq.core.MQDeadLetterService;
|
||||
import tech.easyflow.common.mq.core.MQMessageConverter;
|
||||
import tech.easyflow.common.mq.core.MQProducer;
|
||||
import tech.easyflow.common.mq.redis.JacksonMQMessageConverter;
|
||||
import tech.easyflow.common.mq.redis.RedisMQConsumerContainer;
|
||||
import tech.easyflow.common.mq.redis.RedisMQDeadLetterService;
|
||||
import tech.easyflow.common.mq.redis.RedisMQProducer;
|
||||
import tech.easyflow.common.mq.redis.RedisStreamKeySupport;
|
||||
import tech.easyflow.common.mq.support.MQHealthSupport;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
@Configuration
|
||||
@EnableConfigurationProperties(MQProperties.class)
|
||||
public class MQConfiguration {
|
||||
|
||||
@Bean(destroyMethod = "close")
|
||||
@ConditionalOnProperty(prefix = "easyflow.mq", name = "enabled", havingValue = "true", matchIfMissing = true)
|
||||
public MQRedisResources mqRedisResources(RedisProperties redisProperties, MQProperties mqProperties) {
|
||||
RedisStandaloneConfiguration configuration = new RedisStandaloneConfiguration();
|
||||
configuration.setHostName(redisProperties.getHost());
|
||||
configuration.setPort(redisProperties.getPort());
|
||||
configuration.setDatabase(mqProperties.getRedis().getDatabase());
|
||||
if (redisProperties.getUsername() != null) {
|
||||
configuration.setUsername(redisProperties.getUsername());
|
||||
}
|
||||
if (redisProperties.getPassword() != null) {
|
||||
configuration.setPassword(RedisPassword.of(redisProperties.getPassword()));
|
||||
}
|
||||
LettuceConnectionFactory connectionFactory = new LettuceConnectionFactory(configuration);
|
||||
connectionFactory.afterPropertiesSet();
|
||||
return new MQRedisResources(connectionFactory, new StringRedisTemplate(connectionFactory));
|
||||
}
|
||||
|
||||
@Bean(name = "mqRedisConnectionFactory", autowireCandidate = false, defaultCandidate = false)
|
||||
@ConditionalOnProperty(prefix = "easyflow.mq", name = "enabled", havingValue = "true", matchIfMissing = true)
|
||||
public LettuceConnectionFactory mqRedisConnectionFactory(MQRedisResources mqRedisResources) {
|
||||
return mqRedisResources.connectionFactory();
|
||||
}
|
||||
|
||||
@Bean(name = "mqStringRedisTemplate", autowireCandidate = false, defaultCandidate = false)
|
||||
@ConditionalOnProperty(prefix = "easyflow.mq", name = "enabled", havingValue = "true", matchIfMissing = true)
|
||||
public StringRedisTemplate mqStringRedisTemplate(MQRedisResources mqRedisResources) {
|
||||
return mqRedisResources.stringRedisTemplate();
|
||||
}
|
||||
|
||||
@Bean
|
||||
@ConditionalOnProperty(prefix = "easyflow.mq", name = "enabled", havingValue = "true", matchIfMissing = true)
|
||||
public RedisStreamKeySupport redisStreamKeySupport(MQProperties mqProperties) {
|
||||
return new RedisStreamKeySupport(mqProperties);
|
||||
}
|
||||
|
||||
@Bean
|
||||
@ConditionalOnProperty(prefix = "easyflow.mq", name = "enabled", havingValue = "true", matchIfMissing = true)
|
||||
public MQMessageConverter mqMessageConverter(ObjectMapper objectMapper) {
|
||||
return new JacksonMQMessageConverter(objectMapper);
|
||||
}
|
||||
|
||||
@Bean
|
||||
@ConditionalOnProperty(prefix = "easyflow.mq", name = "enabled", havingValue = "true", matchIfMissing = true)
|
||||
public MQDeadLetterService mqDeadLetterService(MQRedisResources mqRedisResources,
|
||||
MQMessageConverter mqMessageConverter,
|
||||
RedisStreamKeySupport redisStreamKeySupport,
|
||||
ObjectProvider<MQDeadLetterHandler> handlersProvider) {
|
||||
List<MQDeadLetterHandler> handlers = handlersProvider.orderedStream().toList();
|
||||
return new RedisMQDeadLetterService(
|
||||
mqRedisResources.stringRedisTemplate(),
|
||||
mqMessageConverter,
|
||||
redisStreamKeySupport,
|
||||
handlers
|
||||
);
|
||||
}
|
||||
|
||||
@Bean
|
||||
@ConditionalOnProperty(prefix = "easyflow.mq", name = "enabled", havingValue = "true", matchIfMissing = true)
|
||||
public MQProducer mqProducer(MQRedisResources mqRedisResources,
|
||||
MQProperties mqProperties,
|
||||
MQMessageConverter mqMessageConverter,
|
||||
RedisStreamKeySupport redisStreamKeySupport) {
|
||||
return new RedisMQProducer(
|
||||
mqRedisResources.stringRedisTemplate(),
|
||||
mqProperties,
|
||||
mqMessageConverter,
|
||||
redisStreamKeySupport
|
||||
);
|
||||
}
|
||||
|
||||
@Bean
|
||||
@ConditionalOnProperty(prefix = "easyflow.mq", name = "enabled", havingValue = "true", matchIfMissing = true)
|
||||
public MQHealthSupport mqHealthSupport(MQRedisResources mqRedisResources) {
|
||||
return new MQHealthSupport(mqRedisResources.connectionFactory());
|
||||
}
|
||||
|
||||
@Bean
|
||||
@ConditionalOnProperty(prefix = "easyflow.mq", name = "enabled", havingValue = "true", matchIfMissing = true)
|
||||
public MQConsumerContainer mqConsumerContainer(MQRedisResources mqRedisResources,
|
||||
MQProperties mqProperties,
|
||||
MQMessageConverter mqMessageConverter,
|
||||
MQDeadLetterService mqDeadLetterService,
|
||||
RedisStreamKeySupport redisStreamKeySupport,
|
||||
ObjectProvider<MQConsumerHandler> handlersProvider) {
|
||||
List<MQConsumerHandler> handlers = handlersProvider.orderedStream().toList();
|
||||
return new RedisMQConsumerContainer(
|
||||
mqRedisResources.connectionFactory(),
|
||||
mqRedisResources.stringRedisTemplate(),
|
||||
mqProperties,
|
||||
mqMessageConverter,
|
||||
mqDeadLetterService,
|
||||
redisStreamKeySupport,
|
||||
handlers
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,100 @@
|
||||
package tech.easyflow.common.mq.config;
|
||||
|
||||
import org.springframework.boot.context.properties.ConfigurationProperties;
|
||||
|
||||
import java.time.Duration;
|
||||
|
||||
@ConfigurationProperties(prefix = "easyflow.mq")
|
||||
public class MQProperties {
|
||||
|
||||
private boolean enabled = true;
|
||||
private String type = "redis";
|
||||
private final Redis redis = new Redis();
|
||||
|
||||
public boolean isEnabled() {
|
||||
return enabled;
|
||||
}
|
||||
|
||||
public void setEnabled(boolean enabled) {
|
||||
this.enabled = enabled;
|
||||
}
|
||||
|
||||
public String getType() {
|
||||
return type;
|
||||
}
|
||||
|
||||
public void setType(String type) {
|
||||
this.type = type;
|
||||
}
|
||||
|
||||
public Redis getRedis() {
|
||||
return redis;
|
||||
}
|
||||
|
||||
public static class Redis {
|
||||
|
||||
private int database = 1;
|
||||
private String streamPrefix = "easyflow:mq";
|
||||
private int chatPersistShardCount = 4;
|
||||
private int consumerBatchSize = 200;
|
||||
private Duration consumerBlockTimeout = Duration.ofMillis(2000);
|
||||
private Duration pendingClaimIdle = Duration.ofMillis(60000);
|
||||
private int maxRetry = 16;
|
||||
|
||||
public int getDatabase() {
|
||||
return database;
|
||||
}
|
||||
|
||||
public void setDatabase(int database) {
|
||||
this.database = database;
|
||||
}
|
||||
|
||||
public String getStreamPrefix() {
|
||||
return streamPrefix;
|
||||
}
|
||||
|
||||
public void setStreamPrefix(String streamPrefix) {
|
||||
this.streamPrefix = streamPrefix;
|
||||
}
|
||||
|
||||
public int getChatPersistShardCount() {
|
||||
return chatPersistShardCount;
|
||||
}
|
||||
|
||||
public void setChatPersistShardCount(int chatPersistShardCount) {
|
||||
this.chatPersistShardCount = chatPersistShardCount;
|
||||
}
|
||||
|
||||
public int getConsumerBatchSize() {
|
||||
return consumerBatchSize;
|
||||
}
|
||||
|
||||
public void setConsumerBatchSize(int consumerBatchSize) {
|
||||
this.consumerBatchSize = consumerBatchSize;
|
||||
}
|
||||
|
||||
public Duration getConsumerBlockTimeout() {
|
||||
return consumerBlockTimeout;
|
||||
}
|
||||
|
||||
public void setConsumerBlockTimeout(Duration consumerBlockTimeout) {
|
||||
this.consumerBlockTimeout = consumerBlockTimeout;
|
||||
}
|
||||
|
||||
public Duration getPendingClaimIdle() {
|
||||
return pendingClaimIdle;
|
||||
}
|
||||
|
||||
public void setPendingClaimIdle(Duration pendingClaimIdle) {
|
||||
this.pendingClaimIdle = pendingClaimIdle;
|
||||
}
|
||||
|
||||
public int getMaxRetry() {
|
||||
return maxRetry;
|
||||
}
|
||||
|
||||
public void setMaxRetry(int maxRetry) {
|
||||
this.maxRetry = maxRetry;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,13 @@
|
||||
package tech.easyflow.common.mq.config;
|
||||
|
||||
import org.springframework.data.redis.connection.lettuce.LettuceConnectionFactory;
|
||||
import org.springframework.data.redis.core.StringRedisTemplate;
|
||||
|
||||
public record MQRedisResources(LettuceConnectionFactory connectionFactory,
|
||||
StringRedisTemplate stringRedisTemplate) implements AutoCloseable {
|
||||
|
||||
@Override
|
||||
public void close() {
|
||||
connectionFactory.destroy();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,8 @@
|
||||
package tech.easyflow.common.mq.core;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
public interface MQAcknowledger {
|
||||
|
||||
void acknowledge(List<MQMessage> messages);
|
||||
}
|
||||
@@ -0,0 +1,6 @@
|
||||
package tech.easyflow.common.mq.core;
|
||||
|
||||
public interface MQConsumerContainer {
|
||||
|
||||
boolean isRunning();
|
||||
}
|
||||
@@ -0,0 +1,10 @@
|
||||
package tech.easyflow.common.mq.core;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
public interface MQConsumerHandler {
|
||||
|
||||
MQSubscription subscription();
|
||||
|
||||
void handle(List<MQMessage> messages) throws Exception;
|
||||
}
|
||||
@@ -0,0 +1,8 @@
|
||||
package tech.easyflow.common.mq.core;
|
||||
|
||||
public interface MQDeadLetterHandler {
|
||||
|
||||
boolean supports(String topic);
|
||||
|
||||
void handle(MQMessage message, String reason);
|
||||
}
|
||||
@@ -0,0 +1,6 @@
|
||||
package tech.easyflow.common.mq.core;
|
||||
|
||||
public interface MQDeadLetterService {
|
||||
|
||||
void deadLetter(MQMessage message, String reason);
|
||||
}
|
||||
@@ -0,0 +1,91 @@
|
||||
package tech.easyflow.common.mq.core;
|
||||
|
||||
import java.io.Serializable;
|
||||
import java.util.Date;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.Map;
|
||||
|
||||
public class MQMessage implements Serializable {
|
||||
|
||||
private String messageId;
|
||||
private String topic;
|
||||
private String key;
|
||||
private String body;
|
||||
private Date createdAt;
|
||||
private int retryCount;
|
||||
private String streamKey;
|
||||
private String streamMessageId;
|
||||
private Map<String, String> headers = new LinkedHashMap<>();
|
||||
|
||||
public String getMessageId() {
|
||||
return messageId;
|
||||
}
|
||||
|
||||
public void setMessageId(String messageId) {
|
||||
this.messageId = messageId;
|
||||
}
|
||||
|
||||
public String getTopic() {
|
||||
return topic;
|
||||
}
|
||||
|
||||
public void setTopic(String topic) {
|
||||
this.topic = topic;
|
||||
}
|
||||
|
||||
public String getKey() {
|
||||
return key;
|
||||
}
|
||||
|
||||
public void setKey(String key) {
|
||||
this.key = key;
|
||||
}
|
||||
|
||||
public String getBody() {
|
||||
return body;
|
||||
}
|
||||
|
||||
public void setBody(String body) {
|
||||
this.body = body;
|
||||
}
|
||||
|
||||
public Date getCreatedAt() {
|
||||
return createdAt;
|
||||
}
|
||||
|
||||
public void setCreatedAt(Date createdAt) {
|
||||
this.createdAt = createdAt;
|
||||
}
|
||||
|
||||
public int getRetryCount() {
|
||||
return retryCount;
|
||||
}
|
||||
|
||||
public void setRetryCount(int retryCount) {
|
||||
this.retryCount = retryCount;
|
||||
}
|
||||
|
||||
public String getStreamKey() {
|
||||
return streamKey;
|
||||
}
|
||||
|
||||
public void setStreamKey(String streamKey) {
|
||||
this.streamKey = streamKey;
|
||||
}
|
||||
|
||||
public String getStreamMessageId() {
|
||||
return streamMessageId;
|
||||
}
|
||||
|
||||
public void setStreamMessageId(String streamMessageId) {
|
||||
this.streamMessageId = streamMessageId;
|
||||
}
|
||||
|
||||
public Map<String, String> getHeaders() {
|
||||
return headers;
|
||||
}
|
||||
|
||||
public void setHeaders(Map<String, String> headers) {
|
||||
this.headers = headers == null ? new LinkedHashMap<>() : new LinkedHashMap<>(headers);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,8 @@
|
||||
package tech.easyflow.common.mq.core;
|
||||
|
||||
public interface MQMessageConverter {
|
||||
|
||||
String serialize(MQMessage message);
|
||||
|
||||
MQMessage deserialize(String payload);
|
||||
}
|
||||
@@ -0,0 +1,6 @@
|
||||
package tech.easyflow.common.mq.core;
|
||||
|
||||
public interface MQProducer {
|
||||
|
||||
String send(MQMessage message);
|
||||
}
|
||||
@@ -0,0 +1,32 @@
|
||||
package tech.easyflow.common.mq.core;
|
||||
|
||||
public class MQSubscription {
|
||||
|
||||
private String topic;
|
||||
private String consumerGroup;
|
||||
private int shardCount;
|
||||
|
||||
public String getTopic() {
|
||||
return topic;
|
||||
}
|
||||
|
||||
public void setTopic(String topic) {
|
||||
this.topic = topic;
|
||||
}
|
||||
|
||||
public String getConsumerGroup() {
|
||||
return consumerGroup;
|
||||
}
|
||||
|
||||
public void setConsumerGroup(String consumerGroup) {
|
||||
this.consumerGroup = consumerGroup;
|
||||
}
|
||||
|
||||
public int getShardCount() {
|
||||
return shardCount;
|
||||
}
|
||||
|
||||
public void setShardCount(int shardCount) {
|
||||
this.shardCount = shardCount;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,12 @@
|
||||
package tech.easyflow.common.mq.exception;
|
||||
|
||||
public class MQException extends RuntimeException {
|
||||
|
||||
public MQException(String message) {
|
||||
super(message);
|
||||
}
|
||||
|
||||
public MQException(String message, Throwable cause) {
|
||||
super(message, cause);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,34 @@
|
||||
package tech.easyflow.common.mq.redis;
|
||||
|
||||
import com.fasterxml.jackson.core.JsonProcessingException;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import tech.easyflow.common.mq.core.MQMessage;
|
||||
import tech.easyflow.common.mq.core.MQMessageConverter;
|
||||
import tech.easyflow.common.mq.exception.MQException;
|
||||
|
||||
public class JacksonMQMessageConverter implements MQMessageConverter {
|
||||
|
||||
private final ObjectMapper objectMapper;
|
||||
|
||||
public JacksonMQMessageConverter(ObjectMapper objectMapper) {
|
||||
this.objectMapper = objectMapper;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String serialize(MQMessage message) {
|
||||
try {
|
||||
return objectMapper.writeValueAsString(message);
|
||||
} catch (JsonProcessingException e) {
|
||||
throw new MQException("MQ 消息序列化失败", e);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public MQMessage deserialize(String payload) {
|
||||
try {
|
||||
return objectMapper.readValue(payload, MQMessage.class);
|
||||
} catch (JsonProcessingException e) {
|
||||
throw new MQException("MQ 消息反序列化失败", e);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,259 @@
|
||||
package tech.easyflow.common.mq.redis;
|
||||
|
||||
import jakarta.annotation.PreDestroy;
|
||||
import org.springframework.context.SmartLifecycle;
|
||||
import org.springframework.data.domain.Range;
|
||||
import org.springframework.data.redis.connection.RedisConnection;
|
||||
import org.springframework.data.redis.connection.RedisConnectionFactory;
|
||||
import org.springframework.data.redis.connection.RedisStreamCommands;
|
||||
import org.springframework.data.redis.connection.stream.Consumer;
|
||||
import org.springframework.data.redis.connection.stream.MapRecord;
|
||||
import org.springframework.data.redis.connection.stream.PendingMessage;
|
||||
import org.springframework.data.redis.connection.stream.RecordId;
|
||||
import org.springframework.data.redis.connection.stream.StreamOffset;
|
||||
import org.springframework.data.redis.connection.stream.StreamReadOptions;
|
||||
import org.springframework.data.redis.core.StringRedisTemplate;
|
||||
import tech.easyflow.common.mq.config.MQProperties;
|
||||
import tech.easyflow.common.mq.core.MQAcknowledger;
|
||||
import tech.easyflow.common.mq.core.MQConsumerContainer;
|
||||
import tech.easyflow.common.mq.core.MQConsumerHandler;
|
||||
import tech.easyflow.common.mq.core.MQDeadLetterService;
|
||||
import tech.easyflow.common.mq.core.MQMessage;
|
||||
import tech.easyflow.common.mq.core.MQMessageConverter;
|
||||
import tech.easyflow.common.mq.core.MQSubscription;
|
||||
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.time.Duration;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Objects;
|
||||
import java.util.concurrent.ExecutorService;
|
||||
import java.util.concurrent.Executors;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
|
||||
public class RedisMQConsumerContainer implements MQConsumerContainer, SmartLifecycle {
|
||||
|
||||
private final RedisConnectionFactory redisConnectionFactory;
|
||||
private final StringRedisTemplate stringRedisTemplate;
|
||||
private final MQProperties properties;
|
||||
private final MQMessageConverter messageConverter;
|
||||
private final MQDeadLetterService deadLetterService;
|
||||
private final RedisStreamKeySupport keySupport;
|
||||
private final List<MQConsumerHandler> handlers;
|
||||
private final ExecutorService executorService = Executors.newCachedThreadPool();
|
||||
|
||||
private volatile boolean running;
|
||||
|
||||
public RedisMQConsumerContainer(RedisConnectionFactory redisConnectionFactory,
|
||||
StringRedisTemplate stringRedisTemplate,
|
||||
MQProperties properties,
|
||||
MQMessageConverter messageConverter,
|
||||
MQDeadLetterService deadLetterService,
|
||||
RedisStreamKeySupport keySupport,
|
||||
List<MQConsumerHandler> handlers) {
|
||||
this.redisConnectionFactory = redisConnectionFactory;
|
||||
this.stringRedisTemplate = stringRedisTemplate;
|
||||
this.properties = properties;
|
||||
this.messageConverter = messageConverter;
|
||||
this.deadLetterService = deadLetterService;
|
||||
this.keySupport = keySupport;
|
||||
this.handlers = handlers;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void start() {
|
||||
if (running) {
|
||||
return;
|
||||
}
|
||||
running = true;
|
||||
for (MQConsumerHandler handler : handlers) {
|
||||
MQSubscription subscription = handler.subscription();
|
||||
for (int shard = 0; shard < Math.max(subscription.getShardCount(), 1); shard++) {
|
||||
int currentShard = shard;
|
||||
executorService.submit(() -> consumeLoop(handler, subscription, currentShard));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void stop() {
|
||||
running = false;
|
||||
executorService.shutdownNow();
|
||||
try {
|
||||
executorService.awaitTermination(5, TimeUnit.SECONDS);
|
||||
} catch (InterruptedException ignored) {
|
||||
Thread.currentThread().interrupt();
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean isRunning() {
|
||||
return running;
|
||||
}
|
||||
|
||||
@Override
|
||||
public int getPhase() {
|
||||
return Integer.MAX_VALUE;
|
||||
}
|
||||
|
||||
@PreDestroy
|
||||
public void destroy() {
|
||||
stop();
|
||||
}
|
||||
|
||||
private void consumeLoop(MQConsumerHandler handler, MQSubscription subscription, int shard) {
|
||||
String streamKey = keySupport.streamKey(subscription.getTopic(), shard);
|
||||
String consumerName = subscription.getConsumerGroup() + "-" + shard;
|
||||
ensureConsumerGroup(streamKey, subscription.getConsumerGroup());
|
||||
while (running) {
|
||||
try {
|
||||
reclaimPending(streamKey, subscription.getConsumerGroup(), consumerName);
|
||||
List<MapRecord<String, Object, Object>> records = stringRedisTemplate.opsForStream().read(
|
||||
Consumer.from(subscription.getConsumerGroup(), consumerName),
|
||||
StreamReadOptions.empty()
|
||||
.count(properties.getRedis().getConsumerBatchSize())
|
||||
.block(properties.getRedis().getConsumerBlockTimeout()),
|
||||
StreamOffset.create(streamKey, org.springframework.data.redis.connection.stream.ReadOffset.lastConsumed())
|
||||
);
|
||||
if (records == null || records.isEmpty()) {
|
||||
continue;
|
||||
}
|
||||
List<MQMessage> messages = toMessages(streamKey, records);
|
||||
if (messages.isEmpty()) {
|
||||
continue;
|
||||
}
|
||||
handleMessages(handler, streamKey, subscription.getConsumerGroup(), messages);
|
||||
} catch (Exception ignored) {
|
||||
sleepSilently(1000L);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void reclaimPending(String streamKey, String group, String consumerName) {
|
||||
Duration idle = properties.getRedis().getPendingClaimIdle();
|
||||
try (RedisConnection connection = redisConnectionFactory.getConnection()) {
|
||||
RedisStreamCommands.XPendingOptions options = RedisStreamCommands.XPendingOptions
|
||||
.range(Range.unbounded(), (long) properties.getRedis().getConsumerBatchSize());
|
||||
var pendingMessages = connection.streamCommands()
|
||||
.xPending(streamKey.getBytes(StandardCharsets.UTF_8), group, options);
|
||||
if (pendingMessages == null || pendingMessages.isEmpty()) {
|
||||
return;
|
||||
}
|
||||
List<RecordId> ids = new ArrayList<>();
|
||||
for (PendingMessage pendingMessage : pendingMessages) {
|
||||
if (pendingMessage.getElapsedTimeSinceLastDelivery().compareTo(idle) >= 0) {
|
||||
ids.add(pendingMessage.getId());
|
||||
}
|
||||
}
|
||||
if (ids.isEmpty()) {
|
||||
return;
|
||||
}
|
||||
stringRedisTemplate.opsForStream().claim(
|
||||
streamKey,
|
||||
group,
|
||||
consumerName,
|
||||
idle,
|
||||
ids.toArray(new RecordId[0])
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
private void ensureConsumerGroup(String streamKey, String group) {
|
||||
try (RedisConnection connection = redisConnectionFactory.getConnection()) {
|
||||
connection.streamCommands().xGroupCreate(
|
||||
streamKey.getBytes(StandardCharsets.UTF_8),
|
||||
group,
|
||||
org.springframework.data.redis.connection.stream.ReadOffset.latest(),
|
||||
true
|
||||
);
|
||||
} catch (Exception ignored) {
|
||||
}
|
||||
}
|
||||
|
||||
private List<MQMessage> toMessages(String streamKey, List<MapRecord<String, Object, Object>> records) {
|
||||
List<MQMessage> messages = new ArrayList<>(records.size());
|
||||
for (MapRecord<String, Object, Object> record : records) {
|
||||
Object payload = record.getValue().get("payload");
|
||||
if (payload == null) {
|
||||
continue;
|
||||
}
|
||||
MQMessage message = messageConverter.deserialize(String.valueOf(payload));
|
||||
message.setStreamKey(streamKey);
|
||||
message.setStreamMessageId(record.getId().getValue());
|
||||
messages.add(message);
|
||||
}
|
||||
return messages;
|
||||
}
|
||||
|
||||
private void retryOrDeadLetter(List<MQMessage> messages, String reason) {
|
||||
for (MQMessage message : messages) {
|
||||
int retryCount = message.getRetryCount() + 1;
|
||||
message.setRetryCount(retryCount);
|
||||
message.getHeaders().put("lastError", reason == null ? "" : reason);
|
||||
if (retryCount > properties.getRedis().getMaxRetry()) {
|
||||
deadLetterService.deadLetter(message, reason);
|
||||
} else {
|
||||
stringRedisTemplate.opsForStream().add(
|
||||
org.springframework.data.redis.connection.stream.StreamRecords.string(
|
||||
Map.of("payload", messageConverter.serialize(message))
|
||||
).withStreamKey(message.getStreamKey())
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void handleMessages(MQConsumerHandler handler, String streamKey, String group, List<MQMessage> messages) throws Exception {
|
||||
try {
|
||||
handler.handle(messages);
|
||||
acknowledge(streamKey, group, messages);
|
||||
return;
|
||||
} catch (Exception batchEx) {
|
||||
if (messages.size() == 1) {
|
||||
retryOrDeadLetter(messages, resolveReason(batchEx));
|
||||
acknowledge(streamKey, group, messages);
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
for (MQMessage message : messages) {
|
||||
try {
|
||||
handler.handle(List.of(message));
|
||||
} catch (Exception singleEx) {
|
||||
retryOrDeadLetter(List.of(message), resolveReason(singleEx));
|
||||
} finally {
|
||||
acknowledge(streamKey, group, List.of(message));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void acknowledge(String streamKey, String group, List<MQMessage> messages) {
|
||||
if (messages == null || messages.isEmpty()) {
|
||||
return;
|
||||
}
|
||||
String[] ids = messages.stream()
|
||||
.map(MQMessage::getStreamMessageId)
|
||||
.filter(Objects::nonNull)
|
||||
.toArray(String[]::new);
|
||||
if (ids.length == 0) {
|
||||
return;
|
||||
}
|
||||
MQAcknowledger acknowledger = records -> stringRedisTemplate.opsForStream().acknowledge(streamKey, group, ids);
|
||||
acknowledger.acknowledge(messages);
|
||||
}
|
||||
|
||||
private String resolveReason(Exception exception) {
|
||||
if (exception == null || exception.getMessage() == null || exception.getMessage().isBlank()) {
|
||||
return "消费失败";
|
||||
}
|
||||
return exception.getMessage();
|
||||
}
|
||||
|
||||
private void sleepSilently(long millis) {
|
||||
try {
|
||||
Thread.sleep(millis);
|
||||
} catch (InterruptedException ignored) {
|
||||
Thread.currentThread().interrupt();
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,49 @@
|
||||
package tech.easyflow.common.mq.redis;
|
||||
|
||||
import org.springframework.data.redis.connection.stream.StreamRecords;
|
||||
import org.springframework.data.redis.core.StringRedisTemplate;
|
||||
import tech.easyflow.common.mq.core.MQDeadLetterHandler;
|
||||
import tech.easyflow.common.mq.core.MQDeadLetterService;
|
||||
import tech.easyflow.common.mq.core.MQMessage;
|
||||
import tech.easyflow.common.mq.core.MQMessageConverter;
|
||||
import tech.easyflow.common.mq.exception.MQException;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
public class RedisMQDeadLetterService implements MQDeadLetterService {
|
||||
|
||||
private final StringRedisTemplate stringRedisTemplate;
|
||||
private final MQMessageConverter messageConverter;
|
||||
private final RedisStreamKeySupport keySupport;
|
||||
private final List<MQDeadLetterHandler> handlers;
|
||||
|
||||
public RedisMQDeadLetterService(StringRedisTemplate stringRedisTemplate,
|
||||
MQMessageConverter messageConverter,
|
||||
RedisStreamKeySupport keySupport,
|
||||
List<MQDeadLetterHandler> handlers) {
|
||||
this.stringRedisTemplate = stringRedisTemplate;
|
||||
this.messageConverter = messageConverter;
|
||||
this.keySupport = keySupport;
|
||||
this.handlers = handlers;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void deadLetter(MQMessage message, String reason) {
|
||||
if (message == null) {
|
||||
return;
|
||||
}
|
||||
message.getHeaders().put("deadLetterReason", reason == null ? "" : reason);
|
||||
String deadLetterKey = keySupport.deadLetterKey(message.getTopic());
|
||||
if (stringRedisTemplate.opsForStream().add(
|
||||
StreamRecords.string(Map.of("payload", messageConverter.serialize(message))).withStreamKey(deadLetterKey)
|
||||
) == null) {
|
||||
throw new MQException("写入死信流失败");
|
||||
}
|
||||
for (MQDeadLetterHandler handler : handlers) {
|
||||
if (handler.supports(message.getTopic())) {
|
||||
handler.handle(message, reason);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,58 @@
|
||||
package tech.easyflow.common.mq.redis;
|
||||
|
||||
import org.springframework.data.redis.connection.stream.RecordId;
|
||||
import org.springframework.data.redis.connection.stream.StreamRecords;
|
||||
import org.springframework.data.redis.core.StringRedisTemplate;
|
||||
import tech.easyflow.common.mq.config.MQProperties;
|
||||
import tech.easyflow.common.mq.core.MQMessage;
|
||||
import tech.easyflow.common.mq.core.MQProducer;
|
||||
import tech.easyflow.common.mq.core.MQMessageConverter;
|
||||
import tech.easyflow.common.mq.exception.MQException;
|
||||
|
||||
import java.util.Date;
|
||||
import java.util.Map;
|
||||
import java.util.UUID;
|
||||
|
||||
public class RedisMQProducer implements MQProducer {
|
||||
|
||||
private final StringRedisTemplate stringRedisTemplate;
|
||||
private final MQProperties properties;
|
||||
private final MQMessageConverter messageConverter;
|
||||
private final RedisStreamKeySupport keySupport;
|
||||
|
||||
public RedisMQProducer(StringRedisTemplate stringRedisTemplate,
|
||||
MQProperties properties,
|
||||
MQMessageConverter messageConverter,
|
||||
RedisStreamKeySupport keySupport) {
|
||||
this.stringRedisTemplate = stringRedisTemplate;
|
||||
this.properties = properties;
|
||||
this.messageConverter = messageConverter;
|
||||
this.keySupport = keySupport;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String send(MQMessage message) {
|
||||
if (message == null) {
|
||||
throw new MQException("MQ 消息不能为空");
|
||||
}
|
||||
if (message.getTopic() == null || message.getTopic().isBlank()) {
|
||||
throw new MQException("MQ topic 不能为空");
|
||||
}
|
||||
if (message.getMessageId() == null || message.getMessageId().isBlank()) {
|
||||
message.setMessageId(UUID.randomUUID().toString());
|
||||
}
|
||||
if (message.getCreatedAt() == null) {
|
||||
message.setCreatedAt(new Date());
|
||||
}
|
||||
int shardCount = Math.max(properties.getRedis().getChatPersistShardCount(), 1);
|
||||
int shard = keySupport.resolveShard(message.getKey(), shardCount);
|
||||
String streamKey = keySupport.streamKey(message.getTopic(), shard);
|
||||
RecordId recordId = stringRedisTemplate.opsForStream().add(
|
||||
StreamRecords.string(Map.of("payload", messageConverter.serialize(message))).withStreamKey(streamKey)
|
||||
);
|
||||
if (recordId == null) {
|
||||
throw new MQException("MQ 消息投递失败");
|
||||
}
|
||||
return recordId.getValue();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,27 @@
|
||||
package tech.easyflow.common.mq.redis;
|
||||
|
||||
import tech.easyflow.common.mq.config.MQProperties;
|
||||
|
||||
public class RedisStreamKeySupport {
|
||||
|
||||
private final MQProperties properties;
|
||||
|
||||
public RedisStreamKeySupport(MQProperties properties) {
|
||||
this.properties = properties;
|
||||
}
|
||||
|
||||
public String streamKey(String topic, int shard) {
|
||||
return properties.getRedis().getStreamPrefix() + ":" + topic + ":" + String.format("%02d", shard);
|
||||
}
|
||||
|
||||
public String deadLetterKey(String topic) {
|
||||
return properties.getRedis().getStreamPrefix() + ":dead-letter:" + topic;
|
||||
}
|
||||
|
||||
public int resolveShard(String key, int shardCount) {
|
||||
if (shardCount <= 0) {
|
||||
return 0;
|
||||
}
|
||||
return Math.floorMod(key == null ? 0 : key.hashCode(), shardCount);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,28 @@
|
||||
package tech.easyflow.common.mq.support;
|
||||
|
||||
import org.springframework.data.redis.connection.RedisConnectionFactory;
|
||||
import tech.easyflow.common.mq.exception.MQException;
|
||||
|
||||
public class MQHealthSupport {
|
||||
|
||||
private final RedisConnectionFactory redisConnectionFactory;
|
||||
|
||||
public MQHealthSupport(RedisConnectionFactory redisConnectionFactory) {
|
||||
this.redisConnectionFactory = redisConnectionFactory;
|
||||
}
|
||||
|
||||
public boolean available() {
|
||||
try (var connection = redisConnectionFactory.getConnection()) {
|
||||
String pong = connection.ping();
|
||||
return pong != null;
|
||||
} catch (Exception e) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
public void assertAvailable() {
|
||||
if (!available()) {
|
||||
throw new MQException("MQ Redis 不可用");
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -13,6 +13,8 @@
|
||||
|
||||
<modules>
|
||||
<module>easyflow-common-all</module>
|
||||
<module>easyflow-common-analytical-db</module>
|
||||
<module>easyflow-common-mq</module>
|
||||
<module>easyflow-common-web</module>
|
||||
<module>easyflow-common-captcha</module>
|
||||
<module>easyflow-common-base</module>
|
||||
|
||||
Reference in New Issue
Block a user