EasyFlow-Tech-Dev/EasyFlow
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

初始化

Browse Source
This commit is contained in:
陈子默
2026-02-22 18:56:10 +08:00
commit 26677972a6
3112 changed files with 255972 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

63
easyflow-commons/easyflow-common-audio/pom.xml Normal file
View File

@@ -0,0 +1,63 @@
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>tech.easyflow</groupId>
<artifactId>easyflow-commons</artifactId>
<version>${revision}</version>
</parent>
<artifactId>easyflow-common-audio</artifactId>
<dependencies>
<dependency>
<groupId>io.netty</groupId>
<artifactId>netty-all</artifactId>
<version>${netty.version}</version>
</dependency>
<dependency>
<groupId>com.alibaba.nls</groupId>
<artifactId>nls-sdk-tts</artifactId>
<exclusions>
<exclusion>
<groupId>io.netty</groupId>
<artifactId>*</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>com.alibaba.nls</groupId>
<artifactId>nls-sdk-common</artifactId>
<exclusions>
<exclusion>
<groupId>io.netty</groupId>
<artifactId>*</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>tech.easyflow</groupId>
<artifactId>easyflow-common-base</artifactId>
<scope>compile</scope>
</dependency>
<dependency>
<groupId>tech.easyflow</groupId>
<artifactId>easyflow-common-web</artifactId>
</dependency>
<dependency>
<groupId>org.java-websocket</groupId>
<artifactId>Java-WebSocket</artifactId>
</dependency>
<dependency>
<groupId>tech.easyflow</groupId>
<artifactId>easyflow-common-satoken</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-websocket</artifactId>
</dependency>
</dependencies>
</project>

36
easyflow-commons/easyflow-common-audio/src/main/java/tech/easyflow/common/audio/TestAudioController.java Normal file
View File

@@ -0,0 +1,36 @@
package tech.easyflow.common.audio;
import cn.dev33.satoken.annotation.SaIgnore;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
import tech.easyflow.common.audio.core.AudioServiceManager;
import tech.easyflow.common.web.controller.BaseController;
import javax.annotation.Resource;
@SaIgnore
@RequestMapping("/tts")
@RestController
public class TestAudioController extends BaseController {
@Resource
private AudioServiceManager manager;
@GetMapping("/test2")
public String test2() throws Exception {
return "2";
}
@GetMapping("/test1")
public String test1() throws Exception {
return "1";
}
@GetMapping("/test")
public String test() throws Exception {
return "hello world";
}
}

59
easyflow-commons/easyflow-common-audio/src/main/java/tech/easyflow/common/audio/config/AliConfig.java Normal file
View File

@@ -0,0 +1,59 @@
package tech.easyflow.common.audio.config;
import com.alibaba.nls.client.AccessToken;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.context.annotation.Configuration;
import java.io.IOException;
@Configuration
@ConfigurationProperties(prefix = "easyflow.audio.ali")
public class AliConfig {
private String accessKeyId;
private String accessKeySecret;
private String appKey;
private String voice;
public String getAccessKeyId() {
return accessKeyId;
}
public void setAccessKeyId(String accessKeyId) {
this.accessKeyId = accessKeyId;
}
public String getAccessKeySecret() {
return accessKeySecret;
}
public void setAccessKeySecret(String accessKeySecret) {
this.accessKeySecret = accessKeySecret;
}
public String getAppKey() {
return appKey;
}
public void setAppKey(String appKey) {
this.appKey = appKey;
}
public String getVoice() {
return voice;
}
public void setVoice(String voice) {
this.voice = voice;
}
public String createToken() {
AccessToken accessToken = new AccessToken(accessKeyId, accessKeySecret);
try {
accessToken.apply();
return accessToken.getToken();
} catch (IOException e) {
throw new RuntimeException(e);
}
}
}

24
easyflow-commons/easyflow-common-audio/src/main/java/tech/easyflow/common/audio/config/AudioConfig.java Normal file
View File

@@ -0,0 +1,24 @@
package tech.easyflow.common.audio.config;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.context.annotation.Configuration;
import tech.easyflow.common.util.SpringContextUtil;
@Configuration
@ConfigurationProperties(prefix = "easyflow.audio")
public class AudioConfig {
private String type;
public String getType() {
return type;
}
public void setType(String type) {
this.type = type;
}
public static AudioConfig getInstance() {
return SpringContextUtil.getBean(AudioConfig.class);
}
}

19
easyflow-commons/easyflow-common-audio/src/main/java/tech/easyflow/common/audio/core/AudioService.java Normal file
View File

@@ -0,0 +1,19 @@
package tech.easyflow.common.audio.core;
import java.io.InputStream;
public interface AudioService {
/**
* 文字转语音 - 流式
*/
void textToVoiceStream(BaseAudioClient client, String sessionId, String messageId, String text);
/**
* 语音转文字 - 同步
*
* @param is 输入流
* @return 文本
*/
String audioToText(InputStream is);
}

48
easyflow-commons/easyflow-common-audio/src/main/java/tech/easyflow/common/audio/core/AudioServiceManager.java Normal file
View File

@@ -0,0 +1,48 @@
package tech.easyflow.common.audio.core;
import org.springframework.scheduling.TaskScheduler;
import org.springframework.stereotype.Component;
import tech.easyflow.common.audio.config.AliConfig;
import tech.easyflow.common.audio.config.AudioConfig;
import tech.easyflow.common.audio.impl.ali.AliAudioClient;
import tech.easyflow.common.util.SpringContextUtil;
import javax.annotation.Resource;
import java.io.InputStream;
@Component
public class AudioServiceManager implements AudioService {
@Resource
private AliConfig aliConfig;
@Resource(name = "taskScheduler")
private TaskScheduler scheduler;
@Override
public void textToVoiceStream(BaseAudioClient client, String sessionId,String messageId, String text) {
getService().textToVoiceStream(client, sessionId,messageId, text);
}
@Override
public String audioToText(InputStream is) {
return getService().audioToText(is);
}
public BaseAudioClient getClient() {
String type = AudioConfig.getInstance().getType();
if ("aliAudioService".equals(type)) {
return new AliAudioClient(aliConfig);
}
return null;
}
private AudioService getService() {
String type = AudioConfig.getInstance().getType();
return SpringContextUtil.getBean(type);
}
public AliConfig getAliConfig() {
return aliConfig;
}
}

20
easyflow-commons/easyflow-common-audio/src/main/java/tech/easyflow/common/audio/core/BaseAudioClient.java Normal file
View File

@@ -0,0 +1,20 @@
package tech.easyflow.common.audio.core;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import tech.easyflow.common.audio.listener.AudioMessageListener;
public abstract class BaseAudioClient {
protected Logger log = LoggerFactory.getLogger(getClass());
public abstract void beforeSend(String sessionId, String messageId);
public abstract void send(String sessionId, String messageId, String text);
public abstract void afterSend(String sessionId, String messageId);
public abstract void addListener(String sessionId, String messageId, AudioMessageListener listener);
public abstract void close();
}

119
easyflow-commons/easyflow-common-audio/src/main/java/tech/easyflow/common/audio/impl/ali/AliAudioClient.java Normal file
View File

@@ -0,0 +1,119 @@
package tech.easyflow.common.audio.impl.ali;
import cn.hutool.core.util.StrUtil;
import com.alibaba.nls.client.protocol.NlsClient;
import com.alibaba.nls.client.protocol.OutputFormatEnum;
import com.alibaba.nls.client.protocol.SampleRateEnum;
import com.alibaba.nls.client.protocol.SpeechReqProtocol;
import com.alibaba.nls.client.protocol.tts.FlowingSpeechSynthesizer;
import tech.easyflow.common.audio.config.AliConfig;
import tech.easyflow.common.audio.core.BaseAudioClient;
import tech.easyflow.common.audio.listener.AudioMessageListener;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
public class AliAudioClient extends BaseAudioClient {
private final AliConfig aliConfig;
private NlsClient nlsClient;
private final Map<String, FlowingSpeechSynthesizer> execClients = new ConcurrentHashMap<>();
public AliAudioClient(AliConfig aliConfig) {
this.aliConfig = aliConfig;
initClient();
}
private void initClient() {
String token = aliConfig.createToken();
this.nlsClient = new NlsClient(token);
}
@Override
public void beforeSend(String sessionId, String messageId) {
FlowingSpeechSynthesizer execClient = getExecClient(sessionId, messageId);
try {
execClient.start();
} catch (Exception e) {
throw new RuntimeException(e);
}
}
@Override
public void send(String sessionId, String messageId, String text) {
FlowingSpeechSynthesizer execClient = getExecClient(sessionId, messageId);
if (StrUtil.isNotEmpty(text)) {
execClient.send(text);
} else {
execClient.getConnection().sendPing();
}
}
@Override
public void afterSend(String sessionId, String messageId) {
FlowingSpeechSynthesizer execClient = getExecClient(sessionId, messageId);
try {
execClient.stop();
} catch (Exception e) {
log.error("【阿里云nls】客户端结束时发生异常：{}", e.getMessage());
} finally {
execClients.remove(sessionId + messageId);
try {
execClient.close();
} catch (Exception e) {
log.error("【阿里云nls】关闭客户端时发生异常: {}", e.getMessage());
}
}
}
@Override
public void addListener(String sessionId, String messageId, AudioMessageListener listener) {
FlowingSpeechSynthesizer execClient = getExecClient(sessionId, messageId);
AliListener aliListener = (AliListener) execClient.getStreamTTSListener();
aliListener.addListener(listener);
}
@Override
public void close() {
execClients.forEach((sessionIdMessageId, execClient) -> {
try {
execClient.stop();
SpeechReqProtocol.State state = execClient.getState();
} catch (Exception e) {
log.error("【阿里云nls】客户端结束时发生异常：{}", e.getMessage());
} finally {
execClient.close();
}
});
execClients.clear();
nlsClient.shutdown();
}
private FlowingSpeechSynthesizer getExecClient(String sessionId, String messageId) {
FlowingSpeechSynthesizer synthesizer = execClients.get(sessionId + messageId);
if (synthesizer == null) {
//创建实例，建立连接。
try {
synthesizer = new FlowingSpeechSynthesizer(nlsClient, new AliListener(sessionId, messageId));
} catch (Exception e) {
throw new RuntimeException(e);
}
synthesizer.setAppKey(aliConfig.getAppKey());
//设置返回音频的编码格式。
synthesizer.setFormat(OutputFormatEnum.MP3);
//设置返回音频的采样率。
synthesizer.setSampleRate(SampleRateEnum.SAMPLE_RATE_16K);
//发音人。注意Java SDK不支持调用超高清场景对应的发音人（例如"zhiqi"），如需调用请使用restfulAPI方式。
synthesizer.setVoice(aliConfig.getVoice());
//音量，范围是0~100，可选，默认50。
synthesizer.setVolume(50);
//语调，范围是-500~500，可选，默认是0。
synthesizer.setPitchRate(0);
//语速，范围是-500~500，默认是0。
synthesizer.setSpeechRate(0);
execClients.put(sessionId + messageId, synthesizer);
}
return synthesizer;
}
}

92
easyflow-commons/easyflow-common-audio/src/main/java/tech/easyflow/common/audio/impl/ali/AliAudioService.java Normal file
View File

@@ -0,0 +1,92 @@
package tech.easyflow.common.audio.impl.ali;
import cn.hutool.core.io.IoUtil;
import cn.hutool.core.util.IdUtil;
import cn.hutool.http.HttpRequest;
import cn.hutool.http.HttpResponse;
import cn.hutool.http.HttpUtil;
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.stereotype.Component;
import tech.easyflow.common.audio.config.AliConfig;
import tech.easyflow.common.audio.core.AudioService;
import tech.easyflow.common.audio.core.BaseAudioClient;
import tech.easyflow.common.web.exceptions.BusinessException;
import javax.annotation.Resource;
import java.io.InputStream;
import java.util.HashMap;
import java.util.Map;
/**
* 阿里云
*/
@Component("aliAudioService")
public class AliAudioService implements AudioService {
private static final Logger log = LoggerFactory.getLogger(AliAudioService.class);
@Resource
private AliConfig aliConfig;
@Override
public void textToVoiceStream(BaseAudioClient client, String sessionId, String messageId, String text) {
client.send(sessionId, messageId, text);
}
@Override
public String audioToText(InputStream is) {
String token = aliConfig.createToken();
/**
* 设置HTTPS REST POST请求
* 1.使用http协议
* 2.语音识别服务域名：nls-gateway-cn-shanghai.aliyuncs.com
* 3.语音识别接口请求路径：/stream/v1/FlashRecognizer
* 4.设置必须请求参数：appkey、token、format、sample_rate
*/
String url = "https://nls-gateway-cn-shanghai.aliyuncs.com/stream/v1/FlashRecognizer";
String request = url;
request = request + "?appkey=" + aliConfig.getAppKey();
request = request + "&token=" + token;
request = request + "&format=" + "MP3";
request = request + "&sample_rate=" + 16000;
/**
* 设置HTTPS头部字段
* 1.Content-Type：application/octet-stream
*/
Map<String, String> headers = new HashMap<>();
headers.put("Content-Type", "application/octet-stream");
HttpRequest post = HttpUtil.createPost(request);
post.headerMap(headers, true);
byte[] bytes = IoUtil.readBytes(is);
post.body(bytes);
StringBuilder sb = new StringBuilder();
try (HttpResponse execute = post.execute()) {
String body = execute.body();
JSONObject obj = JSON.parseObject(body);
Integer status = obj.getInteger("status");
String message = obj.getString("message");
if (20000000 != status) {
log.error("语音识别失败：{}", obj);
throw new BusinessException(message);
}
JSONArray sentences = obj.getJSONObject("flash_result")
.getJSONArray("sentences");
for (Object sentence : sentences) {
JSONObject json = (JSONObject) sentence;
String text = json.getString("text");
sb.append(text);
}
}
return sb.toString();
}
}

115
easyflow-commons/easyflow-common-audio/src/main/java/tech/easyflow/common/audio/impl/ali/AliListener.java Normal file
View File

@@ -0,0 +1,115 @@
package tech.easyflow.common.audio.impl.ali;
import com.alibaba.fastjson.JSON;
import com.alibaba.nls.client.protocol.tts.FlowingSpeechSynthesizerListener;
import com.alibaba.nls.client.protocol.tts.FlowingSpeechSynthesizerResponse;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import tech.easyflow.common.audio.listener.AudioMessageListener;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.List;
public class AliListener extends FlowingSpeechSynthesizerListener {
private final static Logger log = LoggerFactory.getLogger(AliListener.class);
private final String sessionId;
private final String messageId;
private final List<AudioMessageListener> listeners = new ArrayList<>();
private boolean firstRecvBinary = true;
private final ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
public AliListener(String sessionId, String messageId) {
this.sessionId = sessionId;
this.messageId = messageId;
}
public void addListener(AudioMessageListener listener) {
listeners.add(listener);
}
//流入语音合成开始
public void onSynthesisStart(FlowingSpeechSynthesizerResponse response) {
//log.info("【阿里云nls】流入语音合成开始 ---> name: {}，status: {}", response.getName(), response.getStatus());
}
//服务端检测到了一句话的开始
public void onSentenceBegin(FlowingSpeechSynthesizerResponse response) {
//log.info("【阿里云nls】服务端检测到了一句话的开始 ---> name: {}，status: {}", response.getName(), response.getStatus());
}
//服务端检测到了一句话的结束，获得这句话的起止位置和所有时间戳
public void onSentenceEnd(FlowingSpeechSynthesizerResponse response) {
//log.info("【阿里云nls】服务端检测到了一句话的结束 ---> name: {}，status: {}，subtitles: {}", response.getName(), response.getStatus(), response.getObject("subtitles"));
}
//流入语音合成结束
@Override
public void onSynthesisComplete(FlowingSpeechSynthesizerResponse response) {
// 调用onSynthesisComplete时，表示所有TTS数据已经接收完成，所有文本都已经合成音频并返回。
//log.info("【阿里云nls】流入语音合成结束 ---> name: {}，status: {}", response.getName(), response.getStatus());
for (AudioMessageListener listener : listeners) {
listener.onFinished(sessionId, messageId, outputStream.toByteArray());
}
}
//收到语音合成的语音二进制数据
@Override
public void onAudioData(ByteBuffer message) {
//log.info("【阿里云nls】收到语音合成的语音二进制数据。");
if (firstRecvBinary) {
// 此处计算首包语音流的延迟，收到第一包语音流时，即可以进行语音播放，以提升响应速度（特别是实时交互场景下）。
firstRecvBinary = false;
}
byte[] bytesArray = new byte[message.remaining()];
message.get(bytesArray, 0, bytesArray.length);
try {
outputStream.write(bytesArray);
} catch (IOException e) {
log.error("【阿里云nls】写入二进制数据失败", e);
throw new RuntimeException(e);
}
for (AudioMessageListener listener : listeners) {
listener.onMessageReceived(sessionId, messageId, bytesArray);
}
}
//收到语音合成的增量音频时间戳
@Override
public void onSentenceSynthesis(FlowingSpeechSynthesizerResponse response) {
//log.info("【阿里云nls】收到语音合成的增量音频时间戳 ---> name: {}，status: {}，subtitles: {}", response.getName(), response.getStatus(), response.getObject("subtitles"));
}
@Override
public void onFail(FlowingSpeechSynthesizerResponse response) {
// task_id是调用方和服务端通信的唯一标识，当遇到问题时，需要提供此task_id以便排查。
int status = response.getStatus();
if (status != 40000004) {
log.error("【阿里云nls】合成失败 ---> 会话id：{}，消息id：{} session_id: {}，task_id: {}，status: {}，status_text: {}",
sessionId,
messageId,
getFlowingSpeechSynthesizer().getCurrentSessionId(),
response.getTaskId(),
status,
response.getStatusText());
for (AudioMessageListener listener : listeners) {
listener.onError(sessionId, messageId, new Exception(JSON.toJSONString(response)));
}
}
}
public String getSessionId() {
return sessionId;
}
public String getMessageId() {
return messageId;
}
}

22
easyflow-commons/easyflow-common-audio/src/main/java/tech/easyflow/common/audio/impl/tencent/TencentAudioService.java Normal file
View File

@@ -0,0 +1,22 @@
package tech.easyflow.common.audio.impl.tencent;
import tech.easyflow.common.audio.core.AudioService;
import tech.easyflow.common.audio.core.BaseAudioClient;
import java.io.InputStream;
/**
* 腾讯云
*/
public class TencentAudioService implements AudioService {
@Override
public void textToVoiceStream(BaseAudioClient client, String sessionId, String messageId, String text) {
}
@Override
public String audioToText(InputStream is) {
return "";
}
}

22
easyflow-commons/easyflow-common-audio/src/main/java/tech/easyflow/common/audio/impl/volc/VolcanoEngineAudioService.java Normal file
View File

@@ -0,0 +1,22 @@
package tech.easyflow.common.audio.impl.volc;
import tech.easyflow.common.audio.core.AudioService;
import tech.easyflow.common.audio.core.BaseAudioClient;
import java.io.InputStream;
/**
* 火山引擎
*/
public class VolcanoEngineAudioService implements AudioService {
@Override
public void textToVoiceStream(BaseAudioClient client, String sessionId, String messageId, String text) {
}
@Override
public String audioToText(InputStream is) {
return "";
}
}

10
easyflow-commons/easyflow-common-audio/src/main/java/tech/easyflow/common/audio/listener/AudioMessageListener.java Normal file
View File

@@ -0,0 +1,10 @@
package tech.easyflow.common.audio.listener;
public interface AudioMessageListener {
void onMessageReceived(String sessionId, String messageId, byte[] message);
void onFinished(String sessionId, String messageId, byte[] fullMessage);
void onError(String sessionId, String messageId, Exception e);
}

26
easyflow-commons/easyflow-common-audio/src/main/java/tech/easyflow/common/audio/socket/AudioSocketConfig.java Normal file
View File

@@ -0,0 +1,26 @@
package tech.easyflow.common.audio.socket;
import org.springframework.context.annotation.Configuration;
import org.springframework.web.socket.config.annotation.EnableWebSocket;
import org.springframework.web.socket.config.annotation.WebSocketConfigurer;
import org.springframework.web.socket.config.annotation.WebSocketHandlerRegistry;
import tech.easyflow.common.audio.core.AudioServiceManager;
import javax.annotation.Resource;
@Configuration
@EnableWebSocket
public class AudioSocketConfig implements WebSocketConfigurer {
@Resource
private AudioServiceManager manager;
@Override
public void registerWebSocketHandlers(WebSocketHandlerRegistry registry) {
AudioSocketHandler handler = new AudioSocketHandler(manager);
registry.addHandler(handler, "/api/v1/bot/ws/audio")
.addInterceptors(new AudioSocketInterceptor())
.setAllowedOrigins("*");
}
}

143
easyflow-commons/easyflow-common-audio/src/main/java/tech/easyflow/common/audio/socket/AudioSocketHandler.java Normal file
View File

@@ -0,0 +1,143 @@
package tech.easyflow.common.audio.socket;
import cn.hutool.core.codec.Base64;
import cn.hutool.core.io.FileUtil;
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONObject;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.web.socket.CloseStatus;
import org.springframework.web.socket.TextMessage;
import org.springframework.web.socket.WebSocketSession;
import org.springframework.web.socket.handler.TextWebSocketHandler;
import tech.easyflow.common.audio.core.AudioServiceManager;
import tech.easyflow.common.audio.core.BaseAudioClient;
import tech.easyflow.common.audio.listener.AudioMessageListener;
import java.io.IOException;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
public class AudioSocketHandler extends TextWebSocketHandler {
public static final String SESSION_ID = "sessionId";
public static final String MESSAGE_ID = "messageId";
public static final String TYPE = "type";
public static final String CONTENT = "content";
public static final String DATA = "_data_";
public static final String END = "_end_";
public static final String ERROR = "_error_";
public static final String START = "_start_";
public static Map<String, SocketEntity> sessionMap = new ConcurrentHashMap<>();
private final AudioServiceManager manager;
private final static Logger log = LoggerFactory.getLogger(AudioSocketHandler.class);
public AudioSocketHandler(AudioServiceManager manager) {
this.manager = manager;
}
@Override
public void afterConnectionEstablished(WebSocketSession session) throws Exception {
String sessionId = session.getAttributes().get(SESSION_ID).toString();
log.info("连接建立：{}", sessionId);
BaseAudioClient client = manager.getClient();
SocketEntity entity = new SocketEntity();
entity.setSessionId(sessionId);
entity.setSession(session);
entity.setClient(client);
sessionMap.put(sessionId, entity);
}
@Override
protected void handleTextMessage(WebSocketSession session, TextMessage message) throws Exception {
String sessionId = session.getAttributes().get(SESSION_ID).toString();
SocketEntity entity = getSocketEntity(sessionId);
BaseAudioClient client = entity.getClient();
String msg = message.getPayload();
JSONObject obj = JSON.parseObject(msg);
String messageId = obj.getString(MESSAGE_ID);
String type = obj.getString(TYPE);
String content = obj.getString(CONTENT);
if (START.equals(type)) {
log.info("文本转语音开始：sessionId：{}，messageId：{}", sessionId, messageId);
handleStartMessage(entity, client, messageId);
}
if (END.equals(type)) {
log.info("文本转语音结束：sessionId：{}，messageId：{}", sessionId, messageId);
handleEndMessage(entity, client, messageId);
}
if (DATA.equals(type)) {
handleDataMessage(entity, client, messageId, content);
}
}
private void handleStartMessage(SocketEntity entity, BaseAudioClient client, String messageId) {
client.addListener(entity.getSessionId(), messageId, new AudioMessageListener() {
@Override
public void onMessageReceived(String sessionId, String messageId, byte[] message) {
String encode = Base64.encode(message);
AudioSocketHandler.sendJsonVoiceMessage(sessionId, messageId, DATA, encode);
}
@Override
public void onFinished(String sessionId, String messageId, byte[] fullMessage) {
String encode = Base64.encode(fullMessage);
/*long l = System.currentTimeMillis();
FileUtil.writeBytes(fullMessage, "D:\\system\\desktop\\"+l+".mp3");*/
AudioSocketHandler.sendJsonVoiceMessage(sessionId, messageId, END, encode);
}
@Override
public void onError(String sessionId, String messageId, Exception e) {
AudioSocketHandler.sendJsonVoiceMessage(sessionId, messageId, ERROR, e.getLocalizedMessage());
}
});
client.beforeSend(entity.getSessionId(), messageId);
AudioSocketHandler.sendJsonVoiceMessage(entity.getSessionId(), messageId, START, "");
}
private void handleEndMessage(SocketEntity entity, BaseAudioClient client, String messageId) {
client.afterSend(entity.getSessionId(), messageId);
}
private void handleDataMessage(SocketEntity entity, BaseAudioClient client, String messageId, String content) {
client.send(entity.getSessionId(), messageId, content);
}
@Override
public void afterConnectionClosed(WebSocketSession session, CloseStatus status) throws Exception {
String sessionId = session.getAttributes().get(SESSION_ID).toString();
log.info("{} -> 断开连接：{},{}", sessionId, status.getCode(), status.getReason());
SocketEntity entity = getSocketEntity(sessionId);
entity.getClient().close();
}
public static void sendJsonVoiceMessage(String sessionId, String messageId, String msgType, String content) {
WebSocketSession session = getSocketEntity(sessionId).getSession();
try {
JSONObject obj = new JSONObject();
obj.put(MESSAGE_ID, messageId);
obj.put(TYPE, msgType);
obj.put(CONTENT, content);
String msg = obj.toJSONString();
session.sendMessage(new TextMessage(msg));
} catch (IOException e) {
log.error("发送语音消息失败", e);
throw new RuntimeException(e);
}
}
public static SocketEntity getSocketEntity(String sessionId) {
SocketEntity socket = sessionMap.get(sessionId);
if (socket == null || !socket.getSession().isOpen()) {
log.error("获取Socket失败：WebSocket 连接为空或连接已关闭");
}
return socket;
}
}

44
easyflow-commons/easyflow-common-audio/src/main/java/tech/easyflow/common/audio/socket/AudioSocketInterceptor.java Normal file
View File

@@ -0,0 +1,44 @@
package tech.easyflow.common.audio.socket;
import cn.dev33.satoken.stp.StpUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.http.server.ServerHttpRequest;
import org.springframework.http.server.ServerHttpResponse;
import org.springframework.http.server.ServletServerHttpRequest;
import org.springframework.util.StringUtils;
import org.springframework.web.socket.WebSocketHandler;
import org.springframework.web.socket.server.HandshakeInterceptor;
import java.util.Map;
public class AudioSocketInterceptor implements HandshakeInterceptor {
private static final Logger log = LoggerFactory.getLogger(AudioSocketInterceptor.class);
@Override
public boolean beforeHandshake(ServerHttpRequest request, ServerHttpResponse response, WebSocketHandler wsHandler, Map<String, Object> attributes) throws Exception {
if (request instanceof ServletServerHttpRequest) {
ServletServerHttpRequest servletRequest = (ServletServerHttpRequest) request;
String sessionId = servletRequest.getServletRequest().getParameter("sessionId");
String token = servletRequest.getServletRequest().getParameter("token");
Object loginIdByToken = StpUtil.getLoginIdByToken(token);
if (loginIdByToken == null) {
response.setStatusCode(org.springframework.http.HttpStatus.UNAUTHORIZED);
return false;
}
if (!StringUtils.hasLength(sessionId)) {
return false;
}
attributes.put("sessionId", sessionId);
return true;
}
return false;
}
@Override
public void afterHandshake(ServerHttpRequest request, ServerHttpResponse response, WebSocketHandler wsHandler, Exception exception) {
}
}

22
easyflow-commons/easyflow-common-audio/src/main/java/tech/easyflow/common/audio/socket/SchedulingConfig.java Normal file
View File

@@ -0,0 +1,22 @@
package tech.easyflow.common.audio.socket;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.scheduling.TaskScheduler;
import org.springframework.scheduling.annotation.EnableScheduling;
import org.springframework.scheduling.concurrent.ThreadPoolTaskScheduler;
@Configuration
@EnableScheduling
public class SchedulingConfig {
@Bean
public TaskScheduler taskScheduler() {
ThreadPoolTaskScheduler scheduler = new ThreadPoolTaskScheduler();
scheduler.setPoolSize(10);
scheduler.setThreadNamePrefix("scheduled-task-");
scheduler.setDaemon(true);
scheduler.initialize();
return scheduler;
}
}

35
easyflow-commons/easyflow-common-audio/src/main/java/tech/easyflow/common/audio/socket/SocketEntity.java Normal file
View File

@@ -0,0 +1,35 @@
package tech.easyflow.common.audio.socket;
import org.springframework.web.socket.WebSocketSession;
import tech.easyflow.common.audio.core.BaseAudioClient;
public class SocketEntity {
private String sessionId;
private WebSocketSession session;
private BaseAudioClient client;
public String getSessionId() {
return sessionId;
}
public void setSessionId(String sessionId) {
this.sessionId = sessionId;
}
public WebSocketSession getSession() {
return session;
}
public void setSession(WebSocketSession session) {
this.session = session;
}
public BaseAudioClient getClient() {
return client;
}
public void setClient(BaseAudioClient client) {
this.client = client;
}
}