EasyFlow-Tech-Dev/EasyFlow
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

发布 v1.10 #5

Merged
czm merged 147 commits from develop into main 2026-08-20 11:36:27 +08:00
Conversation 0 Commits 147 Files Changed 1488 +1399 -82
26 changed files with 1399 additions and 82 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Showing only changes of commit 19dac5146c - Show all commits

15
easyflow-api/easyflow-api-admin/src/main/java/tech/easyflow/admin/controller/datacenter/DatacenterExcelController.java
View File

@@ -6,6 +6,7 @@ import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.RestController;
import org.springframework.web.multipart.MultipartFile;
import tech.easyflow.common.domain.Result;
@@ -32,11 +33,21 @@ public class DatacenterExcelController {
@Resource
private DatacenterExcelImportService excelImportService;
/**
* 上传并导入 Excel 工作簿。
*
* @param file Excel 工作簿
* @param sourceName 数据源名称，留空时使用文件名
* @return 导入任务及新建数据源标识
* @throws Exception 文件解析或数据写入失败时抛出
*/
@PostMapping("/import")
@SaCheckPermission("/api/v1/datacenterSource/save")
public Result<DatacenterImportJob> importWorkbook(MultipartFile file) throws Exception {
public Result<DatacenterImportJob> importWorkbook(
@RequestParam("file") MultipartFile file,
@RequestParam(value = "sourceName", required = false) String sourceName) throws Exception {
LoginAccount account = SaTokenUtil.getLoginAccount();
return Result.ok(excelImportService.importWorkbook(file, account));
return Result.ok(excelImportService.importWorkbook(file, sourceName, account));
}
@PostMapping("/split")

37
easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/easyagentsflow/service/WorkflowDatacenterContentService.java
View File

@@ -37,6 +37,7 @@ public class WorkflowDatacenterContentService {
public static final String SEARCH_SQL_MISSING_MESSAGE = "查询数据节点未设置 SQL";
public static final String SAVE_EXPIRED_MESSAGE = "写入数据节点配置已过期，请重新选择已接入表";
public static final String INVALID_QUERY_CONTEXT_MESSAGE = "查询上下文配置无效，请重新选择查询数据节点";
private static final String QUERY_DATA_CONTEXT_PLACEHOLDER = "{{" + QUERY_DATA_CONTEXT + "}}";
private static final String QUERY_CONTEXT_PROMPT = """
你是为工作流中的查询数据节点生成只读 SQL 的生成器，你的职责是返回可直接执行的 SQL，并且你只能输出 SQL。
@@ -47,7 +48,9 @@ public class WorkflowDatacenterContentService {
4. 只能生成只读 SELECT SQL，允许 WITH、JOIN、子查询、聚合、分组、排序。
5. 不要生成 INSERT、UPDATE、DELETE、DDL、多语句、存储过程调用。
6. 优先使用逻辑表名和逻辑字段名，不要输出物理表名、JDBC、驱动信息。
7. 如果存在重名表，请使用 catalog.table 形式消除歧义。
7. 表名和字段名默认不要加引号，禁止使用双引号包裹标识符。
8. 只有摘要中存在重名表时才使用 catalog.table；没有重名时只输出 tableName，不要添加 sourceName 或 catalogName 前缀。
9. 当字段名无法体现业务含义、字段描述提供了明确含义时，在 SELECT 中为该字段添加简短清晰的英文 snake_case 别名，例如 token AS input_price；别名不要加引号。
以下是可用的连接摘要：
""";
@@ -211,6 +214,7 @@ public class WorkflowDatacenterContentService {
JSONArray nodeIds = data.getJSONArray("queryContextNodeIds");
if (nodeIds == null || nodeIds.isEmpty()) {
removeQueryDataContextParameter(data);
removeQueryDataContextPlaceholder(data);
return;
}
Map<BigInteger, JSONObject> sourceSummaries = new LinkedHashMap<>();
@@ -229,6 +233,37 @@ public class WorkflowDatacenterContentService {
}
String contextValue = QUERY_CONTEXT_PROMPT + "\n" + JSON.toJSONString(new ArrayList<>(sourceSummaries.values()));
upsertQueryDataContextParameter(data, contextValue);
appendQueryDataContextPlaceholder(data);
}
/**
* 将查询上下文参数追加到大模型系统提示词，避免覆盖用户配置的提示词。
*
* @param data 大模型节点数据
*/
private void appendQueryDataContextPlaceholder(JSONObject data) {
String systemPrompt = data.getString("systemPrompt");
if (StringUtils.hasText(systemPrompt) && systemPrompt.contains(QUERY_DATA_CONTEXT_PLACEHOLDER)) {
return;
}
if (!StringUtils.hasText(systemPrompt)) {
data.put("systemPrompt", QUERY_DATA_CONTEXT_PLACEHOLDER);
return;
}
data.put("systemPrompt", systemPrompt.stripTrailing() + "\n\n" + QUERY_DATA_CONTEXT_PLACEHOLDER);
}
/**
* 查询上下文关闭后移除自动绑定的提示词占位符。
*
* @param data 大模型节点数据
*/
private void removeQueryDataContextPlaceholder(JSONObject data) {
String systemPrompt = data.getString("systemPrompt");
if (!StringUtils.hasText(systemPrompt) || !systemPrompt.contains(QUERY_DATA_CONTEXT_PLACEHOLDER)) {
return;
}
data.put("systemPrompt", systemPrompt.replace(QUERY_DATA_CONTEXT_PLACEHOLDER, "").trim());
}
private String resolveFieldType(DatacenterTableField field) {

20
easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/node/SearchDatasetNode.java
View File

@@ -26,6 +26,9 @@ public class SearchDatasetNode extends BaseNode {
private static final Pattern PARAM_PATTERN = Pattern.compile("\\{\\{(.+?)\\}\\}");
private static final Pattern SQL_CODE_BLOCK_PATTERN = Pattern.compile(
"\\A```(?:sql)?\\s*([\\s\\S]*?)\\s*```\\z",
Pattern.CASE_INSENSITIVE);
private static final int QUERY_PAGE_SIZE = Math.max(
1,
Integer.getInteger(
@@ -101,13 +104,28 @@ public class SearchDatasetNode extends BaseNode {
}
private String resolveQuerySql(Map<String, Object> params) {
String sql = resolveTemplateString(querySql, params);
String sql = normalizeSqlCodeBlock(resolveTemplateString(querySql, params));
if (!StringUtil.hasText(sql)) {
throw new BusinessException("查询数据节点未设置 SQL");
}
return sql.trim();
}
/**
* 去除完整单个 SQL Markdown 代码块的边界标记。
*
* @param sql 原始 SQL 文本
* @return 可交给 SQL 解析器处理的文本
*/
private String normalizeSqlCodeBlock(String sql) {
if (!StringUtil.hasText(sql)) {
return sql;
}
String trimmed = sql.trim();
Matcher matcher = SQL_CODE_BLOCK_PATTERN.matcher(trimmed);
return matcher.matches() ? matcher.group(1).trim() : trimmed;
}
private DatasetRef copyDatasetRef() {
DatasetRef copy = new DatasetRef();
copy.setSourceId(datasetRef == null ? null : datasetRef.getSourceId());

217
easyflow-modules/easyflow-module-ai/src/test/java/tech/easyflow/ai/easyagentsflow/service/WorkflowDatacenterContentServiceTest.java Normal file
View File

@@ -0,0 +1,217 @@
package tech.easyflow.ai.easyagentsflow.service;
import com.alibaba.fastjson2.JSONArray;
import com.alibaba.fastjson2.JSONObject;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import org.mockito.Mockito;
import tech.easyflow.datacenter.entity.DatacenterTable;
import tech.easyflow.datacenter.entity.DatacenterTableField;
import tech.easyflow.datacenter.meta.entity.DatacenterSource;
import tech.easyflow.datacenter.meta.service.DatacenterDatasetRegistryService;
import java.lang.reflect.Field;
import java.math.BigInteger;
import java.util.ArrayList;
import java.util.List;
/**
* 工作流数据中枢内容准备服务测试。
*/
public class WorkflowDatacenterContentServiceTest {
private static final BigInteger SOURCE_ID = BigInteger.valueOf(1001L);
private static final BigInteger TABLE_ID = BigInteger.valueOf(2001L);
private WorkflowDatacenterContentService service;
private DatacenterDatasetRegistryService registryService;
/**
* 初始化服务及数据源元数据桩。
*
* @throws Exception 注入测试依赖失败时抛出
*/
@Before
public void setUp() throws Exception {
service = new WorkflowDatacenterContentService();
registryService = Mockito.mock(DatacenterDatasetRegistryService.class);
injectField(service, "registryService", registryService);
DatacenterSource source = Mockito.mock(DatacenterSource.class);
Mockito.when(source.getSourceName()).thenReturn("ama 实验基线模型预算");
Mockito.when(source.getSourceType()).thenReturn("EXCEL");
DatacenterTableField modelId = mockField("col_id", "模型ID", "VARCHAR");
DatacenterTableField inputPrice = mockField("token", "输入价格", "DECIMAL");
DatacenterTable table = Mockito.mock(DatacenterTable.class);
Mockito.when(table.getId()).thenReturn(TABLE_ID);
Mockito.when(table.getTableName()).thenReturn("Sheet1");
Mockito.when(table.getFields()).thenReturn(List.of(modelId, inputPrice));
Mockito.when(registryService.getSourceRequired(SOURCE_ID)).thenReturn(source);
Mockito.when(registryService.listManagedTables(SOURCE_ID, null))
.thenReturn(new ArrayList<>(List.of(table)));
Mockito.when(registryService.getTableWithFields(TABLE_ID)).thenReturn(table);
}
/**
* 验证查询上下文会自动且幂等地绑定到系统提示词。
*/
@Test
public void testPrepareRootShouldBindQueryContextToSystemPromptIdempotently() {
JSONObject root = buildWorkflowRoot();
JSONObject llmData = root.getJSONArray("nodes").getJSONObject(1).getJSONObject("data");
service.prepareRoot(root);
service.prepareRoot(root);
String systemPrompt = llmData.getString("systemPrompt");
Assert.assertEquals("请根据问题生成查询语句\n\n{{queryDataContext}}", systemPrompt);
Assert.assertEquals(1, countOccurrences(systemPrompt, "{{queryDataContext}}"));
JSONObject contextParameter = findParameter(llmData, "queryDataContext");
Assert.assertNotNull(contextParameter);
String contextValue = contextParameter.getString("value");
Assert.assertTrue(contextValue.contains("只输出 SQL"));
Assert.assertTrue(contextValue.contains("Sheet1"));
Assert.assertTrue(contextValue.contains("col_id"));
Assert.assertTrue(contextValue.contains("模型ID"));
Assert.assertTrue(contextValue.contains("token AS input_price"));
}
/**
* 验证关闭查询上下文后会同步清理参数和系统提示词占位符。
*/
@Test
public void testPrepareRootShouldRemoveQueryContextBindingWhenDisabled() {
JSONObject root = buildWorkflowRoot();
JSONObject llmData = root.getJSONArray("nodes").getJSONObject(1).getJSONObject("data");
service.prepareRoot(root);
llmData.put("queryContextNodeIds", new JSONArray());
service.prepareRoot(root);
Assert.assertEquals("请根据问题生成查询语句", llmData.getString("systemPrompt"));
Assert.assertNull(findParameter(llmData, "queryDataContext"));
}
/**
* 构造包含查询节点和大模型节点的最小工作流。
*
* @return 工作流根对象
*/
private JSONObject buildWorkflowRoot() {
JSONObject datasetRef = new JSONObject();
datasetRef.put("sourceId", SOURCE_ID);
JSONObject queryData = new JSONObject();
queryData.put("datasetRef", datasetRef);
queryData.put("querySql", "{{query}}");
JSONObject queryNode = buildNode(
"query-node",
WorkflowDatacenterContentService.SEARCH_NODE_TYPE,
queryData);
JSONArray queryContextNodeIds = new JSONArray();
queryContextNodeIds.add("query-node");
JSONObject llmData = new JSONObject();
llmData.put("systemPrompt", "请根据问题生成查询语句");
llmData.put("queryContextNodeIds", queryContextNodeIds);
llmData.put("parameters", new JSONArray());
JSONObject llmNode = buildNode(
"llm-node",
WorkflowDatacenterContentService.LLM_NODE_TYPE,
llmData);
JSONArray nodes = new JSONArray();
nodes.add(queryNode);
nodes.add(llmNode);
JSONObject root = new JSONObject();
root.put("nodes", nodes);
return root;
}
/**
* 构造工作流节点。
*
* @param id 节点标识
* @param type 节点类型
* @param data 节点数据
* @return 工作流节点
*/
private JSONObject buildNode(String id, String type, JSONObject data) {
JSONObject node = new JSONObject();
node.put("id", id);
node.put("type", type);
node.put("data", data);
return node;
}
/**
* 构造字段元数据桩。
*
* @param fieldName 字段名
* @param fieldDesc 字段描述
* @param jdbcType JDBC 类型
* @return 字段元数据
*/
private DatacenterTableField mockField(String fieldName, String fieldDesc, String jdbcType) {
DatacenterTableField field = Mockito.mock(DatacenterTableField.class);
Mockito.when(field.getFieldName()).thenReturn(fieldName);
Mockito.when(field.getFieldDesc()).thenReturn(fieldDesc);
Mockito.when(field.getJdbcType()).thenReturn(jdbcType);
return field;
}
/**
* 查找指定名称的节点参数。
*
* @param data 节点数据
* @param name 参数名
* @return 参数对象，不存在时返回 {@code null}
*/
private JSONObject findParameter(JSONObject data, String name) {
JSONArray parameters = data.getJSONArray("parameters");
if (parameters == null) {
return null;
}
for (int i = 0; i < parameters.size(); i++) {
JSONObject parameter = parameters.getJSONObject(i);
if (parameter != null && name.equals(parameter.getString("name"))) {
return parameter;
}
}
return null;
}
/**
* 统计子串出现次数。
*
* @param value 原始文本
* @param target 目标子串
* @return 出现次数
*/
private int countOccurrences(String value, String target) {
int count = 0;
int index = 0;
while ((index = value.indexOf(target, index)) >= 0) {
count++;
index += target.length();
}
return count;
}
/**
* 注入服务私有依赖。
*
* @param target 目标对象
* @param fieldName 字段名
* @param value 字段值
* @throws Exception 反射注入失败时抛出
*/
private void injectField(Object target, String fieldName, Object value) throws Exception {
Field field = target.getClass().getDeclaredField(fieldName);
field.setAccessible(true);
field.set(target, value);
}
}

37
easyflow-modules/easyflow-module-ai/src/test/java/tech/easyflow/ai/node/SearchDatasetNodeTest.java
View File

@@ -65,6 +65,43 @@ public class SearchDatasetNodeTest {
}
}
/**
* 验证查询节点可接收大模型返回的单个 SQL Markdown 代码块。
*
* @throws Exception 反射调用失败时抛出
*/
@Test
public void testResolveQuerySqlShouldUnwrapMarkdownSqlCodeBlock() throws Exception {
DatasetRef datasetRef = new DatasetRef();
datasetRef.setSourceId(BigInteger.valueOf(4004L));
SearchDatasetNode node = new SearchDatasetNode(datasetRef, "{{query}}");
Map<String, Object> params = new HashMap<>();
params.put("query", "```sql SELECT token FROM Sheet1 WHERE col_id = 'deepseek-v4-pro'; ```");
String sql = invokeResolveQuerySql(node, params);
Assert.assertEquals("SELECT token FROM Sheet1 WHERE col_id = 'deepseek-v4-pro';", sql);
}
/**
* 验证包含说明文本的模型输出不会被宽松清洗，从而继续由 SQL 解析器拒绝。
*
* @throws Exception 反射调用失败时抛出
*/
@Test
public void testResolveQuerySqlShouldKeepTextOutsideSqlCodeBlock() throws Exception {
DatasetRef datasetRef = new DatasetRef();
datasetRef.setSourceId(BigInteger.valueOf(5005L));
SearchDatasetNode node = new SearchDatasetNode(datasetRef, "{{query}}");
Map<String, Object> params = new HashMap<>();
String generated = "查询语句如下：\n```sql\nSELECT token FROM Sheet1;\n```";
params.put("query", generated);
String sql = invokeResolveQuerySql(node, params);
Assert.assertEquals(generated, sql);
}
private String invokeResolveQuerySql(SearchDatasetNode node, Map<String, Object> params) throws Exception {
Method method = SearchDatasetNode.class.getDeclaredMethod("resolveQuerySql", Map.class);
method.setAccessible(true);

41
easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/adapter/DefaultDbHandleService.java
View File

@@ -16,20 +16,25 @@ import tech.easyflow.datacenter.utils.SqlInjectionUtils;
import java.math.BigInteger;
import java.util.Date;
import java.util.List;
import java.util.Objects;
@Component("defaultDbHandleService")
public class DefaultDbHandleService extends DbHandleService {
private static final Logger log = LoggerFactory.getLogger(DefaultDbHandleService.class);
/**
* 创建指定的物理数据表。
*
* @param table 包含表结构和字段定义的数据表
*/
@Override
public void createTable(DatacenterTable table) {
// 设置为 [tb_dynamic_表名_tableId] 的格式
String actualTable = table.getActualTable();
SqlInjectionUtils.checkIdentifier(actualTable);
// 表注释
String tableDesc = table.getTableDesc();
SqlInjectionUtils.checkComment(tableDesc);
String tableDesc = SqlInjectionUtils.checkComment(table.getTableDesc());
List<DatacenterTableField> fields = table.getFields();
StringBuilder sql = new StringBuilder("CREATE TABLE " + actualTable + " (");
@@ -56,15 +61,21 @@ public class DefaultDbHandleService extends DbHandleService {
Db.selectObject(sql.toString());
}
/**
* 更新数据表的可修改元数据。
*
* @param table 待更新的数据表
* @param record 已持久化的数据表记录
*/
@Override
public void updateTable(DatacenterTable table, DatacenterTable record) {
String tableDesc = table.getTableDesc();
SqlInjectionUtils.checkComment(tableDesc);
String escapedTableDesc = SqlInjectionUtils.checkComment(tableDesc);
String actualTable = record.getActualTable();
// 只允许改表备注
if (!tableDesc.equals(record.getTableDesc())) {
if (!Objects.equals(tableDesc, record.getTableDesc())) {
String sql = "ALTER TABLE `" + actualTable + "` "
+ "COMMENT '" + tableDesc + "';";
+ "COMMENT '" + escapedTableDesc + "';";
log.info("修改表备注语句 >>> {}", sql);
Db.selectObject(sql);
}
@@ -95,13 +106,18 @@ public class DefaultDbHandleService extends DbHandleService {
return "text";
}
/**
* 为物理数据表新增字段。
*
* @param entity 目标数据表
* @param field 待新增字段
*/
@Override
public void addField(DatacenterTable entity, DatacenterTableField field) {
String fieldName = field.getFieldName();
SqlInjectionUtils.checkIdentifier(fieldName);
String fieldDesc = field.getFieldDesc();
SqlInjectionUtils.checkComment(fieldDesc);
String fieldDesc = SqlInjectionUtils.checkComment(field.getFieldDesc());
Integer fieldType = field.getFieldType();
Integer required = field.getRequired();
@@ -125,6 +141,13 @@ public class DefaultDbHandleService extends DbHandleService {
Db.selectObject(sql);
}
/**
* 更新物理数据表中的字段定义。
*
* @param entity 目标数据表
* @param fieldRecord 已持久化的字段记录
* @param field 待更新字段
*/
@Override
public void updateField(DatacenterTable entity, DatacenterTableField fieldRecord, DatacenterTableField field) {
String actualTable = entity.getActualTable();
@@ -135,10 +158,10 @@ public class DefaultDbHandleService extends DbHandleService {
SqlInjectionUtils.checkIdentifier(fieldName);
// 字段描述
String fieldDesc = field.getFieldDesc();
SqlInjectionUtils.checkComment(fieldDesc);
String escapedFieldDesc = SqlInjectionUtils.checkComment(fieldDesc);
String nullable = required == 1 ? "NOT NULL " : "NULL ";
String desc = "COMMENT '" + fieldDesc + "';";
String desc = "COMMENT '" + escapedFieldDesc + "';";
boolean isUpdate = false;
String handleType = "MODIFY COLUMN `" + fieldRecord.getFieldName() + "` ";

41
easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/connector/support/AbstractInternalTableConnector.java
View File

@@ -86,19 +86,34 @@ public abstract class AbstractInternalTableConnector implements DatacenterConnec
@Override
public Page<Row> queryPage(DatacenterSource source, DatacenterTable table, DatacenterQueryRequest request) {
String actualTable = resolveTableName(table);
QueryWrapper wrapper = QueryWrapper.create();
if (StrUtil.isNotBlank(request.getWhere())) {
wrapper.where(request.getWhere());
}
long count = Db.selectCountByQuery(actualTable, wrapper);
long count = Db.selectCountByQuery(
actualTable, createQueryWrapper(request.getWhere()));
if (count == 0) {
return new Page<>(new ArrayList<>(), request.getPageNumber(), request.getPageSize(), count);
}
Page<Row> page = Db.paginate(actualTable, new Page<>(request.getPageNumber(), request.getPageSize(), count), wrapper);
// selectCountByQuery 会把无投影的 QueryWrapper 改为 COUNT(*)，分页查询必须使用独立实例。
Page<Row> page = Db.paginate(
actualTable,
new Page<>(request.getPageNumber(), request.getPageSize(), count),
createQueryWrapper(request.getWhere()));
normalizeRows(page.getRecords());
return page;
}
/**
* 创建用于动态表查询的独立条件包装器。
*
* @param where 已校验的筛选表达式
* @return 新建的查询条件包装器
*/
static QueryWrapper createQueryWrapper(String where) {
QueryWrapper wrapper = QueryWrapper.create();
if (StrUtil.isNotBlank(where)) {
wrapper.where(where);
}
return wrapper;
}
@Override
public List<Row> queryBySql(DatacenterSource source, String sql) {
List<Row> rows = Db.selectListBySql(sql);
@@ -527,12 +542,20 @@ public abstract class AbstractInternalTableConnector implements DatacenterConnec
return StrUtil.blankToDefault(table.getMaterializedTable(), table.getActualTable());
}
private void normalizeRows(List<Row> records) {
/**
* 统一内部表查询结果的字段名与值类型。
*
* <p>MySQL 驱动在部分查询路径会返回大写列标签，而数据集元数据中的物理字段名
* 始终为小写。这里在序列化前收敛为小写，确保预览、SQL 消费和导出按同一字段名取值。</p>
*
* @param records 待规范化的数据行
*/
static void normalizeRows(List<Row> records) {
for (Row record : records) {
Map<String, Object> converted = new LinkedHashMap<>();
for (Map.Entry<String, Object> entry : record.entrySet()) {
converted.put(
entry.getKey(),
entry.getKey().toLowerCase(Locale.ROOT),
normalizeValue(entry.getValue()));
}
record.clear();
@@ -546,7 +569,7 @@ public abstract class AbstractInternalTableConnector implements DatacenterConnec
* @param value JDBC 原始值
* @return 兼容既有查询接口的值
*/
private Object normalizeValue(Object value) {
private static Object normalizeValue(Object value) {
if (value instanceof BigInteger
|| value instanceof BigDecimal
|| value instanceof Long) {

14
easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/connector/support/DatacenterConnectorExceptionSupport.java
View File

@@ -10,6 +10,9 @@ import java.net.UnknownHostException;
import java.sql.SQLException;
import java.util.Locale;
/**
* 数据连接器访问异常分类与安全包装工具。
*/
public final class DatacenterConnectorExceptionSupport {
public static final String SOURCE_UNAVAILABLE_MESSAGE = "当前连接不可用，请检查连接配置后重试";
@@ -17,14 +20,21 @@ public final class DatacenterConnectorExceptionSupport {
private DatacenterConnectorExceptionSupport() {
}
/**
* 将连接器异常转换为可安全展示的业务异常，并保留原始异常供日志追踪。
*
* @param fallbackMessage 默认业务错误
* @param ex 原始异常
* @return 包装后的业务异常
*/
public static BusinessException wrapAccessException(String fallbackMessage, Exception ex) {
if (ex instanceof BusinessException businessException && !isConnectionUnavailable(ex)) {
return businessException;
}
if (isConnectionUnavailable(ex)) {
return new BusinessException(SOURCE_UNAVAILABLE_MESSAGE);
return new BusinessException(400, 1, SOURCE_UNAVAILABLE_MESSAGE, ex);
}
return new BusinessException(fallbackMessage);
return new BusinessException(400, 1, fallbackMessage, ex);
}
public static boolean isConnectionUnavailable(Throwable throwable) {

5
easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/entity/base/DatacenterTableBase.java
View File

@@ -1,5 +1,7 @@
package tech.easyflow.datacenter.entity.base;
import com.fasterxml.jackson.databind.annotation.JsonSerialize;
import com.fasterxml.jackson.databind.ser.std.ToStringSerializer;
import com.mybatisflex.annotation.Column;
import com.mybatisflex.annotation.Id;
import com.mybatisflex.annotation.KeyType;
@@ -129,6 +131,7 @@ public class DatacenterTableBase extends DateEntity implements Serializable {
@Column(typeHandler = FastjsonTypeHandler.class, comment = "能力声明")
private Map<String, Object> capabilitiesJson;
@JsonSerialize(using = ToStringSerializer.class)
public BigInteger getId() {
return id;
}
@@ -153,6 +156,7 @@ public class DatacenterTableBase extends DateEntity implements Serializable {
this.tenantId = tenantId;
}
@JsonSerialize(using = ToStringSerializer.class)
public BigInteger getSourceId() {
return sourceId;
}
@@ -161,6 +165,7 @@ public class DatacenterTableBase extends DateEntity implements Serializable {
this.sourceId = sourceId;
}
@JsonSerialize(using = ToStringSerializer.class)
public BigInteger getCatalogId() {
return catalogId;
}

4
easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/entity/base/DatacenterTableFieldBase.java
View File

@@ -1,5 +1,7 @@
package tech.easyflow.datacenter.entity.base;
import com.fasterxml.jackson.databind.annotation.JsonSerialize;
import com.fasterxml.jackson.databind.ser.std.ToStringSerializer;
import com.mybatisflex.annotation.Column;
import com.mybatisflex.annotation.Id;
import com.mybatisflex.annotation.KeyType;
@@ -129,6 +131,7 @@ public class DatacenterTableFieldBase extends DateEntity implements Serializable
@Column(comment = "修改者")
private BigInteger modifiedBy;
@JsonSerialize(using = ToStringSerializer.class)
public BigInteger getId() {
return id;
}
@@ -137,6 +140,7 @@ public class DatacenterTableFieldBase extends DateEntity implements Serializable
this.id = id;
}
@JsonSerialize(using = ToStringSerializer.class)
public BigInteger getTableId() {
return tableId;
}

11
easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/excel/service/DatacenterExcelImportService.java
View File

@@ -12,7 +12,16 @@ import java.math.BigInteger;
import java.util.List;
public interface DatacenterExcelImportService {
DatacenterImportJob importWorkbook(MultipartFile file, LoginAccount account) throws Exception;
/**
* 导入 Excel 工作簿并创建对应的数据源、目录和物化表。
*
* @param file Excel 工作簿文件
* @param sourceName 用户指定的数据源名称，留空时使用文件名
* @param account 当前登录账号
* @return 已完成的导入任务
* @throws Exception 文件读取或数据写入失败时抛出
*/
DatacenterImportJob importWorkbook(MultipartFile file, String sourceName, LoginAccount account) throws Exception;
DatacenterImportJob splitWorkbook(DatacenterExcelSplitRequest request, LoginAccount account);

275
easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/excel/service/impl/DatacenterExcelImportServiceImpl.java
View File

@@ -6,6 +6,7 @@ import com.mybatisflex.core.paginate.Page;
import com.mybatisflex.core.row.Row;
import org.apache.poi.ss.usermodel.Cell;
import org.apache.poi.ss.usermodel.DataFormatter;
import org.apache.poi.ss.usermodel.FormulaEvaluator;
import org.apache.poi.ss.usermodel.Sheet;
import org.apache.poi.ss.usermodel.Workbook;
import org.apache.poi.ss.usermodel.WorkbookFactory;
@@ -41,6 +42,7 @@ import tech.easyflow.datacenter.meta.enums.DatacenterImportStatus;
import tech.easyflow.datacenter.meta.enums.DatacenterSourceType;
import tech.easyflow.datacenter.meta.enums.DatacenterTableKind;
import tech.easyflow.datacenter.meta.model.DatacenterTableDetailMeta;
import tech.easyflow.datacenter.utils.SqlInjectionUtils;
import tech.easyflow.datacenter.meta.service.DatacenterDatasetRegistryService;
import tech.easyflow.datacenter.meta.service.DatacenterSourceService;
@@ -70,6 +72,14 @@ import java.util.UUID;
public class DatacenterExcelImportServiceImpl implements DatacenterExcelImportService {
private static final long QUERY_BATCH_SIZE = 500L;
private static final int MAX_IDENTIFIER_LENGTH = 64;
private static final int MAX_SOURCE_NAME_LENGTH = 100;
private static final int MAX_IMPORT_JOB_ERROR_SUMMARY_LENGTH = 1000;
private static final String ERROR_SUMMARY_TRUNCATION_MARKER = "…（错误摘要已截断）…";
private static final Set<String> SYSTEM_FIELD_NAMES = Set.of(
"id", "dept_id", "tenant_id", "created", "created_by",
"modified", "modified_by", "remark"
);
private static final DateTimeFormatter EXPORT_TIME_FORMAT = DateTimeFormatter.ofPattern("yyyyMMddHHmmss");
@Resource
@@ -89,11 +99,58 @@ public class DatacenterExcelImportServiceImpl implements DatacenterExcelImportSe
@Override
@Transactional(rollbackFor = Exception.class)
public DatacenterImportJob importWorkbook(MultipartFile file, LoginAccount account) throws Exception {
public DatacenterImportJob importWorkbook(
MultipartFile file,
String sourceName,
LoginAccount account) throws Exception {
if (file == null || file.isEmpty()) {
throw new BusinessException("Excel 文件不能为空");
}
String workbookName = extractWorkbookName(file.getOriginalFilename());
if (!isSupportedWorkbookFileName(file.getOriginalFilename())) {
throw new BusinessException("仅支持 .xls 和 .xlsx 格式的 Excel 文件");
}
try (InputStream inputStream = file.getInputStream()) {
Workbook parsedWorkbook;
try {
parsedWorkbook = WorkbookFactory.create(inputStream);
} catch (Exception ex) {
throw new BusinessException(
400,
1,
"Excel 文件无法解析，请确认文件未损坏且未加密",
ex
);
}
try (Workbook workbook = parsedWorkbook) {
return importParsedWorkbook(file, sourceName, account, workbook);
}
}
}
/**
* 将已解析并校验的工作簿写入数据中枢。
*
* @param file 原始上传文件
* @param sourceName 用户指定的数据源名称
* @param account 当前登录账号
* @param workbook 已解析的工作簿
* @return 已完成的导入任务
*/
private DatacenterImportJob importParsedWorkbook(
MultipartFile file,
String sourceName,
LoginAccount account,
Workbook workbook) {
DataFormatter formatter = new DataFormatter();
FormulaEvaluator formulaEvaluator = workbook.getCreationHelper().createFormulaEvaluator();
if (!hasImportableSheet(workbook, formatter)) {
throw new BusinessException(
"未检测到可导入的数据，请确认至少有一个工作表，首行包含表头且下方有数据"
);
}
String workbookName = resolveImportSourceName(sourceName, file.getOriginalFilename());
DatacenterSource source = new DatacenterSource();
source.setSourceName(workbookName);
source.setSourceCode("EXCEL_" + UUID.randomUUID());
@@ -110,18 +167,14 @@ public class DatacenterExcelImportServiceImpl implements DatacenterExcelImportSe
long totalRows = 0L;
long successRows = 0L;
List<BigInteger> createdTableIds = new ArrayList<>();
try (InputStream inputStream = file.getInputStream(); Workbook workbook = WorkbookFactory.create(inputStream)) {
DataFormatter formatter = new DataFormatter();
try {
for (int sheetIndex = 0; sheetIndex < workbook.getNumberOfSheets(); sheetIndex++) {
Sheet sheet = workbook.getSheetAt(sheetIndex);
org.apache.poi.ss.usermodel.Row headerRow = sheet.getRow(sheet.getFirstRowNum());
if (headerRow == null) {
if (!isImportableSheet(sheet, headerRow, formatter)) {
continue;
}
List<DatacenterTableField> fields = buildFields(headerRow, formatter);
if (fields.isEmpty()) {
continue;
}
DatacenterTable table = new DatacenterTable();
table.setTableName(uniqueTableName(source.getId(), catalog.getId(), sheet.getSheetName()));
table.setTableDesc(sheet.getSheetName());
@@ -150,7 +203,11 @@ public class DatacenterExcelImportServiceImpl implements DatacenterExcelImportSe
JSONObject payload = new JSONObject();
boolean hasValue = false;
for (int cellIndex = 0; cellIndex < savedTable.getFields().size(); cellIndex++) {
String value = formatter.formatCellValue(row.getCell(cellIndex));
String value = formatCellValue(
row.getCell(cellIndex),
formatter,
formulaEvaluator
);
if (value != null && !value.isBlank()) {
hasValue = true;
}
@@ -544,14 +601,59 @@ public class DatacenterExcelImportServiceImpl implements DatacenterExcelImportSe
importJobMapper.update(job);
}
/**
* 将导入任务标记为失败，并保存长度受限的错误摘要。
*
* @param job 待更新的导入任务
* @param ex 导入过程中抛出的异常
*/
private void finishJobFailure(DatacenterImportJob job, Exception ex) {
job.setStatus(DatacenterImportStatus.FAILED.name());
job.setErrorSummary(ex.getMessage());
job.setErrorSummary(summarizeImportError(ex));
job.setFinishedAt(new Date());
job.setModified(new Date());
importJobMapper.update(job);
}
/**
* 生成可写入导入任务错误摘要列的错误信息。
*
* @param ex 导入过程中抛出的异常
* @return 不超过数据库字段长度的错误摘要
*/
static String summarizeImportError(Exception ex) {
String message = ex == null ? null : ex.getMessage();
if (message == null || message.isBlank()) {
message = ex == null ? "Excel 导入失败" : ex.getClass().getSimpleName();
}
int messageLength = message.codePointCount(0, message.length());
if (messageLength <= MAX_IMPORT_JOB_ERROR_SUMMARY_LENGTH) {
return message;
}
int markerLength = ERROR_SUMMARY_TRUNCATION_MARKER.codePointCount(0, ERROR_SUMMARY_TRUNCATION_MARKER.length());
int availableLength = MAX_IMPORT_JOB_ERROR_SUMMARY_LENGTH - markerLength;
int prefixLength = availableLength * 2 / 3;
int suffixLength = availableLength - prefixLength;
return substringByCodePoint(message, 0, prefixLength)
+ ERROR_SUMMARY_TRUNCATION_MARKER
+ substringByCodePoint(message, messageLength - suffixLength, messageLength);
}
/**
* 按 Unicode 码点截取字符串，避免截断代理对字符。
*
* @param value 原始字符串
* @param beginCodePoint 起始码点索引（包含）
* @param endCodePoint 结束码点索引（不包含）
* @return 截取后的字符串
*/
private static String substringByCodePoint(String value, int beginCodePoint, int endCodePoint) {
int beginIndex = value.offsetByCodePoints(0, beginCodePoint);
int endIndex = value.offsetByCodePoints(0, endCodePoint);
return value.substring(beginIndex, endIndex);
}
private DatacenterTable resolveTable(DatasetRef datasetRef) {
if (datasetRef == null || datasetRef.getTableId() == null) {
throw new BusinessException("缺少数据集 tableId");
@@ -870,7 +972,7 @@ public class DatacenterExcelImportServiceImpl implements DatacenterExcelImportSe
return tables.stream().anyMatch(table -> tableName.equals(table.getTableName()));
}
private String extractWorkbookName(String originalFileName) {
private static String extractWorkbookName(String originalFileName) {
if (originalFileName == null || originalFileName.isBlank()) {
return "excel_workbook";
}
@@ -878,35 +980,166 @@ public class DatacenterExcelImportServiceImpl implements DatacenterExcelImportSe
return index > 0 ? originalFileName.substring(0, index) : originalFileName;
}
/**
* 解析 Excel 导入后使用的数据源名称。
*
* @param requestedSourceName 用户填写的数据源名称
* @param originalFileName 原始文件名
* @return 去除首尾空白且长度合法的数据源名称
* @throws BusinessException 名称超过数据库字段长度时抛出
*/
static String resolveImportSourceName(String requestedSourceName, String originalFileName) {
String resolved = requestedSourceName == null || requestedSourceName.isBlank()
? extractWorkbookName(originalFileName).trim()
: requestedSourceName.trim();
if (resolved.codePointCount(0, resolved.length()) > MAX_SOURCE_NAME_LENGTH) {
throw new BusinessException("连接名称不能超过 100 个字符");
}
return resolved;
}
/**
* 判断工作簿是否至少包含一个带表头和数据行的工作表。
*
* @param workbook 待检查的工作簿
* @param formatter 单元格格式化器
* @return 存在可导入工作表时返回 {@code true}
*/
static boolean hasImportableSheet(Workbook workbook, DataFormatter formatter) {
if (workbook == null) {
return false;
}
for (int sheetIndex = 0; sheetIndex < workbook.getNumberOfSheets(); sheetIndex++) {
Sheet sheet = workbook.getSheetAt(sheetIndex);
org.apache.poi.ss.usermodel.Row headerRow = sheet.getRow(sheet.getFirstRowNum());
if (isImportableSheet(sheet, headerRow, formatter)) {
return true;
}
}
return false;
}
/**
* 判断单个工作表是否包含有效表头和至少一行数据。
*
* @param sheet 待检查的工作表
* @param headerRow 首行表头
* @param formatter 单元格格式化器
* @return 工作表可导入时返回 {@code true}
*/
private static boolean isImportableSheet(
Sheet sheet,
org.apache.poi.ss.usermodel.Row headerRow,
DataFormatter formatter) {
if (sheet == null || headerRow == null || headerRow.getLastCellNum() <= 0) {
return false;
}
boolean hasHeader = false;
for (int cellIndex = 0; cellIndex < headerRow.getLastCellNum(); cellIndex++) {
if (!formatter.formatCellValue(headerRow.getCell(cellIndex)).isBlank()) {
hasHeader = true;
break;
}
}
if (!hasHeader) {
return false;
}
for (int rowIndex = sheet.getFirstRowNum() + 1; rowIndex <= sheet.getLastRowNum(); rowIndex++) {
org.apache.poi.ss.usermodel.Row row = sheet.getRow(rowIndex);
if (row == null) {
continue;
}
for (int cellIndex = 0; cellIndex < headerRow.getLastCellNum(); cellIndex++) {
if (!formatter.formatCellValue(row.getCell(cellIndex)).isBlank()) {
return true;
}
}
}
return false;
}
/**
* 获取单元格最终显示值，公式单元格返回计算结果。
*
* @param cell 单元格
* @param formatter 单元格格式化器
* @param formulaEvaluator 公式计算器
* @return 单元格显示值；空单元格返回空字符串
*/
static String formatCellValue(
Cell cell,
DataFormatter formatter,
FormulaEvaluator formulaEvaluator) {
if (cell == null) {
return "";
}
return formatter.formatCellValue(cell, formulaEvaluator);
}
/**
* 判断文件名是否为支持的 Excel 工作簿格式。
*
* @param originalFileName 原始上传文件名
* @return 文件扩展名为 {@code .xls} 或 {@code .xlsx} 时返回 {@code true}
*/
static boolean isSupportedWorkbookFileName(String originalFileName) {
if (originalFileName == null || originalFileName.isBlank()) {
return false;
}
String fileName = originalFileName.trim().toLowerCase(Locale.ROOT);
return fileName.endsWith(".xls") || fileName.endsWith(".xlsx");
}
private String buildMaterializedTableName(BigInteger sourceId, int sheetIndex) {
long snowId = new SnowFlakeIDKeyGenerator().nextId();
return "tb_excel_" + sourceId + "_" + sheetIndex + "_" + snowId;
}
private String normalizeIdentifier(String raw) {
/**
* 将 Excel 表头转换为可用于数据库物理字段的安全标识符。
*
* @param raw 原始表头
* @return 仅包含 ASCII 字母、数字和下划线的候选字段名
*/
static String normalizeIdentifier(String raw) {
if (raw == null || raw.isBlank()) {
return "value";
return "";
}
String normalized = raw.trim().toLowerCase(Locale.ROOT).replaceAll("[^a-z0-9_\\u4e00-\\u9fa5]+", "_");
normalized = normalized.replaceAll("_+", "_");
if (normalized.isBlank()) {
return "value";
}
return normalized;
return raw.trim()
.toLowerCase(Locale.ROOT)
.replaceAll("[^a-z0-9]+", "_")
.replaceAll("_+", "_")
.replaceAll("^_+|_+$", "");
}
private String normalizeIdentifier(String raw, int index, Set<String> usedNames) {
/**
* 为 Excel 表头生成唯一且安全的物理字段名。
*
* @param raw 原始表头
* @param index 表头从零开始的列索引
* @param usedNames 当前已使用的物理字段名
* @return 唯一的安全物理字段名，且不会与系统保留列冲突
*/
static String normalizeIdentifier(String raw, int index, Set<String> usedNames) {
String value = normalizeIdentifier(raw);
if (value.isBlank() || "value".equals(value)) {
if (value.isBlank()) {
value = "col_" + (index + 1);
}
if (Character.isDigit(value.charAt(0))) {
value = "col_" + value;
}
if (SqlInjectionUtils.isSqlKeyword(value) || SYSTEM_FIELD_NAMES.contains(value)) {
value = "col_" + value;
}
value = value.substring(0, Math.min(value.length(), MAX_IDENTIFIER_LENGTH));
String result = value;
int suffix = 1;
while (usedNames.contains(result)) {
result = value + "_" + suffix++;
if (result.length() > MAX_IDENTIFIER_LENGTH) {
String suffixText = "_" + (suffix - 1);
result = value.substring(0, MAX_IDENTIFIER_LENGTH - suffixText.length()) + suffixText;
}
}
usedNames.add(result);
return result;

17
easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/execution/service/impl/DatacenterDatasetQueryServiceImpl.java
View File

@@ -9,6 +9,7 @@ import org.springframework.util.StringUtils;
import tech.easyflow.common.web.exceptions.BusinessException;
import tech.easyflow.datacenter.connector.DatacenterConnector;
import tech.easyflow.datacenter.connector.DatacenterConnectorRegistry;
import tech.easyflow.datacenter.connector.support.AbstractInternalTableConnector;
import tech.easyflow.datacenter.entity.DatacenterTable;
import tech.easyflow.datacenter.entity.DatacenterTableField;
import tech.easyflow.datacenter.execution.model.DatacenterQueryRequest;
@@ -189,14 +190,16 @@ public class DatacenterDatasetQueryServiceImpl implements DatacenterDatasetQuery
}
Map<BigInteger, DatacenterCatalog> catalogsById =
loadCatalogsById(managedTables);
SqlSupportUtils.ResolvedSql resolvedSql = SqlSupportUtils.resolve(
sql,
managedTables.stream()
.map(table -> toManagedSqlTable(
table, catalogsById))
.toList()
);
DatacenterConnector connector = connectorRegistry.getConnector(source.getSourceType());
List<SqlSupportUtils.ManagedTable> sqlTables = managedTables.stream()
.map(table -> toManagedSqlTable(
table, catalogsById))
.toList();
// 内部连接的 catalog 是逻辑命名空间，底层项目 MySQL 只执行物理表名。
SqlSupportUtils.ResolvedSql resolvedSql =
connector instanceof AbstractInternalTableConnector
? SqlSupportUtils.resolveInternalMysql(sql, sqlTables)
: SqlSupportUtils.resolve(sql, sqlTables);
return new ResolvedSqlQuery(
source,
connector,

3
easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/meta/entity/DatacenterCatalog.java
View File

@@ -1,5 +1,7 @@
package tech.easyflow.datacenter.meta.entity;
import com.fasterxml.jackson.databind.annotation.JsonSerialize;
import com.fasterxml.jackson.databind.ser.std.ToStringSerializer;
import com.mybatisflex.annotation.Column;
import com.mybatisflex.annotation.Id;
import com.mybatisflex.annotation.KeyType;
@@ -42,6 +44,7 @@ public class DatacenterCatalog extends DateEntity implements Serializable {
@Column(comment = "修改人")
private BigInteger modifiedBy;
@JsonSerialize(using = ToStringSerializer.class)
public BigInteger getId() { return id; }
public void setId(BigInteger id) { this.id = id; }
public BigInteger getDeptId() { return deptId; }

6
easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/meta/entity/DatacenterImportJob.java
View File

@@ -1,5 +1,7 @@
package tech.easyflow.datacenter.meta.entity;
import com.fasterxml.jackson.databind.annotation.JsonSerialize;
import com.fasterxml.jackson.databind.ser.std.ToStringSerializer;
import com.mybatisflex.annotation.Column;
import com.mybatisflex.annotation.Id;
import com.mybatisflex.annotation.KeyType;
@@ -57,16 +59,20 @@ public class DatacenterImportJob extends DateEntity implements Serializable {
@Column(comment = "修改人")
private BigInteger modifiedBy;
@JsonSerialize(using = ToStringSerializer.class)
public BigInteger getId() { return id; }
public void setId(BigInteger id) { this.id = id; }
public BigInteger getDeptId() { return deptId; }
public void setDeptId(BigInteger deptId) { this.deptId = deptId; }
public BigInteger getTenantId() { return tenantId; }
public void setTenantId(BigInteger tenantId) { this.tenantId = tenantId; }
@JsonSerialize(using = ToStringSerializer.class)
public BigInteger getSourceId() { return sourceId; }
public void setSourceId(BigInteger sourceId) { this.sourceId = sourceId; }
@JsonSerialize(using = ToStringSerializer.class)
public BigInteger getCatalogId() { return catalogId; }
public void setCatalogId(BigInteger catalogId) { this.catalogId = catalogId; }
@JsonSerialize(using = ToStringSerializer.class)
public BigInteger getTableId() { return tableId; }
public void setTableId(BigInteger tableId) { this.tableId = tableId; }
public String getJobType() { return jobType; }

3
easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/meta/entity/DatacenterSource.java
View File

@@ -1,5 +1,7 @@
package tech.easyflow.datacenter.meta.entity;
import com.fasterxml.jackson.databind.annotation.JsonSerialize;
import com.fasterxml.jackson.databind.ser.std.ToStringSerializer;
import com.mybatisflex.annotation.Column;
import com.mybatisflex.annotation.Id;
import com.mybatisflex.annotation.KeyType;
@@ -70,6 +72,7 @@ public class DatacenterSource extends DateEntity implements Serializable {
@Column(comment = "修改人")
private BigInteger modifiedBy;
@JsonSerialize(using = ToStringSerializer.class)
public BigInteger getId() { return id; }
public void setId(BigInteger id) { this.id = id; }
public BigInteger getDeptId() { return deptId; }

8
easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/meta/model/DatacenterCatalogMeta.java
View File

@@ -1,7 +1,13 @@
package tech.easyflow.datacenter.meta.model;
import com.fasterxml.jackson.databind.annotation.JsonSerialize;
import com.fasterxml.jackson.databind.ser.std.ToStringSerializer;
import java.math.BigInteger;
/**
* 数据中枢目录元数据。
*/
public class DatacenterCatalogMeta {
private BigInteger id;
private BigInteger sourceId;
@@ -9,8 +15,10 @@ public class DatacenterCatalogMeta {
private String catalogType;
private String catalogDesc;
@JsonSerialize(using = ToStringSerializer.class)
public BigInteger getId() { return id; }
public void setId(BigInteger id) { this.id = id; }
@JsonSerialize(using = ToStringSerializer.class)
public BigInteger getSourceId() { return sourceId; }
public void setSourceId(BigInteger sourceId) { this.sourceId = sourceId; }
public String getCatalogName() { return catalogName; }

67
easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/utils/SqlInjectionUtils.java
View File

@@ -2,20 +2,35 @@ package tech.easyflow.datacenter.utils;
import tech.easyflow.common.web.exceptions.BusinessException;
import java.util.Arrays;
import java.util.HashSet;
import java.util.Locale;
import java.util.Set;
public class SqlInjectionUtils {
/**
* 数据中心动态 SQL 的安全校验与字面量转义工具。
*/
public final class SqlInjectionUtils {
private static final Set<String> SQL_KEYWORDS = new HashSet<>(Arrays.asList(
private static final Set<String> SQL_KEYWORDS = Set.of(
"select", "insert", "update", "delete", "drop", "alter", "create",
"table", "where", "from", "join", "union", "truncate", "execute",
"grant", "revoke", "commit", "rollback"
));
"grant", "revoke", "commit", "rollback", "order", "group", "by",
"having", "limit", "offset", "as", "on", "into", "values", "index",
"key", "primary", "constraint", "references", "distinct", "case",
"when", "then", "else", "end", "and", "or", "not", "null", "like",
"in", "is", "exists", "between", "procedure", "function", "trigger",
"view", "database", "schema", "column", "add", "rename", "replace",
"show", "describe", "explain", "use", "lock", "unlock"
);
private SqlInjectionUtils() {
}
/**
* 校验字段或表名
* 校验动态 SQL 中的字段或表标识符。
*
* @param identifier 待校验的标识符
* @return 已校验的标识符
* @throws BusinessException 标识符为空、过长、包含非法字符或为 SQL 关键字时抛出
*/
public static String checkIdentifier(String identifier) {
if (identifier == null || identifier.isEmpty()) {
@@ -35,34 +50,36 @@ public class SqlInjectionUtils {
}
/**
* 校验注释
* 允许的字符包括以下 Unicode 类别或符号：
* \p{L}：任何语言的字母（包括中文、英文、日文等）。
* \p{N}：任何数字（包括阿拉伯数字 0-9 或其他语言的数字符号）。
* \p{Zs}：空白分隔符（如空格，但不包括换行符、制表符等）。
* 标点符号：. , - : ? !（基础标点）。
* 校验并转义 SQL 字符串字面量中的表或字段备注。
*
* <p>Excel 表头和工作表名称可包含常见的中英文标点、单位符号和 Unicode 字符。换行和制表符会
* 规范为单个空格；其余控制字符会被拒绝。单引号和反斜杠会被转义，避免备注内容破坏动态 DDL 语句。</p>
*
* @param comment 原始备注内容
* @return 可安全拼入单引号 SQL 字符串字面量的备注内容
* @throws BusinessException 备注过长或包含控制字符时抛出
*/
public static String checkComment(String comment) {
if (comment == null) {
if (comment == null || comment.isEmpty()) {
return "";
}
if (comment.length() > 255) {
throw new BusinessException("注释过长");
}
if (!comment.matches("^[\\p{L}\\p{N}\\p{Zs}\\.\\,\\-\\:\\?\\!]+$")) {
throw new BusinessException("包含非法字符");
String normalizedComment = comment.replaceAll("[\\r\\n\\t]+", " ");
if (normalizedComment.codePoints().anyMatch(Character::isISOControl)) {
throw new BusinessException("备注不能包含控制字符");
}
if (comment.contains("--")) {
throw new BusinessException("包含非法字符！");
}
if (comment.chars().anyMatch(c -> c <= 31 || c == 127)) {
throw new BusinessException("存在非法字符");
}
return comment;
return normalizedComment.replace("\\", "\\\\").replace("'", "''");
}
// 检查是否是数据库关键字
/**
* 判断指定单词是否为受限 SQL 关键字。
*
* @param word 待判断的单词
* @return 是 SQL 关键字时返回 {@code true}
*/
public static boolean isSqlKeyword(String word) {
return SQL_KEYWORDS.contains(word.toLowerCase());
return word != null && SQL_KEYWORDS.contains(word.toLowerCase(Locale.ROOT));
}
}

106
easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/utils/SqlSupportUtils.java
View File

@@ -25,6 +25,36 @@ public final class SqlSupportUtils {
}
public static ResolvedSql resolve(String sql, Collection<ManagedTable> managedTables) {
return resolve(sql, managedTables, true, false);
}
/**
* 解析内部物化表 SQL，逻辑目录仅用于白名单匹配，不进入项目 MySQL 的执行 SQL。
*
* @param sql 逻辑 SQL
* @param managedTables 已接入表
* @return 已解析的项目 MySQL SQL
*/
public static ResolvedSql resolveInternalMysql(
String sql,
Collection<ManagedTable> managedTables) {
return resolve(sql, managedTables, false, true);
}
/**
* 解析并重写只读 SQL。
*
* @param sql 逻辑 SQL
* @param managedTables 已接入表
* @param retainCatalog 是否保留目录限定符
* @param mysqlIdentifierQuotes 是否转换为 MySQL 标识符引号
* @return 已解析 SQL
*/
private static ResolvedSql resolve(
String sql,
Collection<ManagedTable> managedTables,
boolean retainCatalog,
boolean mysqlIdentifierQuotes) {
String normalizedSql = normalizeSql(sql);
Statement statement = parseSingleStatement(normalizedSql);
if (!(statement instanceof Select select)) {
@@ -54,10 +84,14 @@ public final class SqlSupportUtils {
Set<String> logicalTables = new LinkedHashSet<>();
for (Table table : referencedTables) {
ManagedTable managedTable = resolveManagedTable(table, byTableName, byCatalogAndTable);
rewriteTable(table, managedTable);
rewriteTable(table, managedTable, retainCatalog);
logicalTables.add(renderLogicalTable(managedTable));
}
return new ResolvedSql(select.toString(), new ArrayList<>(logicalTables));
String executableSql = select.toString();
if (mysqlIdentifierQuotes) {
executableSql = normalizeMysqlIdentifierQuotes(executableSql);
}
return new ResolvedSql(executableSql, new ArrayList<>(logicalTables));
}
private static Statement parseSingleStatement(String sql) {
@@ -102,9 +136,73 @@ public final class SqlSupportUtils {
return matches.get(0);
}
private static void rewriteTable(Table table, ManagedTable managedTable) {
/**
* 将逻辑表替换为物理表。
*
* @param table SQL 表节点
* @param managedTable 已接入表
* @param retainCatalog 是否保留目录限定符
*/
private static void rewriteTable(
Table table,
ManagedTable managedTable,
boolean retainCatalog) {
table.setName(managedTable.getPhysicalTableName());
table.setSchemaName(trimToNull(managedTable.getCatalogName()));
table.setSchemaName(retainCatalog
? trimToNull(managedTable.getCatalogName())
: null);
}
/**
* 将标准 SQL 双引号标识符转换为 MySQL 反引号，同时保留字符串字面量内容。
*
* @param sql 已解析 SQL
* @return MySQL 可执行 SQL
*/
private static String normalizeMysqlIdentifierQuotes(String sql) {
StringBuilder normalized = new StringBuilder(sql.length());
boolean singleQuoted = false;
boolean doubleQuotedIdentifier = false;
for (int index = 0; index < sql.length(); index++) {
char current = sql.charAt(index);
if (singleQuoted) {
normalized.append(current);
if (current == '\\' && index + 1 < sql.length()) {
normalized.append(sql.charAt(++index));
} else if (current == '\'' && index + 1 < sql.length()
&& sql.charAt(index + 1) == '\'') {
normalized.append(sql.charAt(++index));
} else if (current == '\'') {
singleQuoted = false;
}
continue;
}
if (doubleQuotedIdentifier) {
if (current == '"' && index + 1 < sql.length()
&& sql.charAt(index + 1) == '"') {
normalized.append('"');
index++;
} else if (current == '"') {
normalized.append('`');
doubleQuotedIdentifier = false;
} else if (current == '`') {
normalized.append("``");
} else {
normalized.append(current);
}
continue;
}
if (current == '\'') {
singleQuoted = true;
normalized.append(current);
} else if (current == '"') {
doubleQuotedIdentifier = true;
normalized.append('`');
} else {
normalized.append(current);
}
}
return normalized.toString();
}
private static String renderLogicalTable(ManagedTable managedTable) {

98
easyflow-modules/easyflow-module-datacenter/src/test/java/tech/easyflow/datacenter/connector/support/AbstractInternalTableConnectorTest.java Normal file
View File

@@ -0,0 +1,98 @@
package tech.easyflow.datacenter.connector.support;
import com.mybatisflex.core.row.Row;
import com.mybatisflex.core.paginate.Page;
import com.mybatisflex.core.query.QueryWrapper;
import com.mybatisflex.core.row.Db;
import org.junit.Assert;
import org.junit.Test;
import org.mockito.MockedStatic;
import org.mockito.Mockito;
import tech.easyflow.datacenter.entity.DatacenterTable;
import tech.easyflow.datacenter.execution.model.DatacenterQueryRequest;
import tech.easyflow.datacenter.meta.enums.DatacenterSourceType;
import java.math.BigInteger;
import java.util.List;
import java.util.Set;
import java.util.concurrent.atomic.AtomicReference;
/**
* {@link AbstractInternalTableConnector} 内部查询结果规范化测试。
*/
public class AbstractInternalTableConnectorTest {
/**
* 验证 JDBC 返回的大写列标签会转换为数据集字段使用的小写名称。
*/
@Test
public void shouldNormalizeMysqlColumnLabelsToLowerCase() {
Row row = new Row();
row.put("COL_ID", "deepseek-v4-pro");
row.put("TOKEN", BigInteger.valueOf(3));
AbstractInternalTableConnector.normalizeRows(List.of(row));
Assert.assertEquals(
List.of("col_id", "token"),
List.copyOf(row.keySet()));
Assert.assertEquals("deepseek-v4-pro", row.get("col_id"));
Assert.assertEquals("3", row.get("token"));
}
/**
* 验证统计查询修改投影后，不会污染后续的分页查询包装器。
*/
@Test
public void shouldUseIndependentQueryWrapperForPaginationAfterCount() {
DatacenterTable table = new DatacenterTable();
table.setActualTable("preview_table");
DatacenterQueryRequest request = new DatacenterQueryRequest();
request.setPageNumber(1L);
request.setPageSize(10L);
Row record = new Row();
record.put("MODEL_ID", "deepseek-v4-pro");
AtomicReference<QueryWrapper> countWrapper = new AtomicReference<>();
AtomicReference<QueryWrapper> pageWrapper = new AtomicReference<>();
try (MockedStatic<Db> db = Mockito.mockStatic(Db.class)) {
db.when(() -> Db.selectCountByQuery(
Mockito.eq("preview_table"),
Mockito.any(QueryWrapper.class)))
.thenAnswer(invocation -> {
QueryWrapper wrapper = invocation.getArgument(1);
countWrapper.set(wrapper);
// 模拟 MyBatis-Flex 为 count 查询自动追加 COUNT(*) 投影的行为。
wrapper.select("COUNT(*)");
return 1L;
});
db.when(() -> Db.paginate(
Mockito.eq("preview_table"),
Mockito.any(Page.class),
Mockito.any(QueryWrapper.class)))
.thenAnswer(invocation -> {
pageWrapper.set(invocation.getArgument(2));
return new Page<>(List.of(record), 1L, 10L, 1L);
});
Page<Row> page = new TestInternalTableConnector().queryPage(
null, table, request);
Assert.assertNotSame(countWrapper.get(), pageWrapper.get());
Assert.assertEquals("deepseek-v4-pro", page.getRecords().get(0).get("model_id"));
}
}
/**
* 供内部动态表查询测试使用的最小连接器实现。
*/
private static class TestInternalTableConnector extends AbstractInternalTableConnector {
/**
* 创建测试连接器。
*/
private TestInternalTableConnector() {
super(DatacenterSourceType.EXCEL, Set.of(), null);
}
}
}

51
easyflow-modules/easyflow-module-datacenter/src/test/java/tech/easyflow/datacenter/connector/support/DatacenterConnectorExceptionSupportTest.java Normal file
View File

@@ -0,0 +1,51 @@
package tech.easyflow.datacenter.connector.support;
import org.junit.Assert;
import org.junit.Test;
import tech.easyflow.common.web.exceptions.BusinessException;
import java.sql.SQLException;
/**
* {@link DatacenterConnectorExceptionSupport} 异常包装测试。
*/
public class DatacenterConnectorExceptionSupportTest {
/**
* 验证通用访问错误保留底层 SQL 异常。
*/
@Test
public void wrapAccessExceptionShouldPreserveSqlCause() {
SQLException cause = new SQLException(
"You have an error in your SQL syntax",
"42000");
BusinessException wrapped =
DatacenterConnectorExceptionSupport.wrapAccessException(
"SQL 流式查询失败",
cause);
Assert.assertEquals("SQL 流式查询失败", wrapped.getMessage());
Assert.assertSame(cause, wrapped.getCause());
}
/**
* 验证连接不可用错误保留底层 SQL 异常并返回安全文案。
*/
@Test
public void wrapAccessExceptionShouldPreserveUnavailableCause() {
SQLException cause = new SQLException(
"Unknown database 'missing'",
"42000");
BusinessException wrapped =
DatacenterConnectorExceptionSupport.wrapAccessException(
"SQL 流式查询失败",
cause);
Assert.assertEquals(
DatacenterConnectorExceptionSupport.SOURCE_UNAVAILABLE_MESSAGE,
wrapped.getMessage());
Assert.assertSame(cause, wrapped.getCause());
}
}

177
easyflow-modules/easyflow-module-datacenter/src/test/java/tech/easyflow/datacenter/excel/service/impl/DatacenterExcelImportServiceImplTest.java Normal file
View File

@@ -0,0 +1,177 @@
package tech.easyflow.datacenter.excel.service.impl;
import org.junit.Assert;
import org.junit.Test;
import org.apache.poi.hssf.usermodel.HSSFWorkbook;
import org.apache.poi.ss.usermodel.DataFormatter;
import org.apache.poi.ss.usermodel.Workbook;
import org.apache.poi.ss.usermodel.WorkbookFactory;
import org.apache.poi.xssf.usermodel.XSSFWorkbook;
import tech.easyflow.common.web.exceptions.BusinessException;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.util.HashSet;
import java.util.Set;
/**
* {@link DatacenterExcelImportServiceImpl} 的 Excel 表头字段名处理测试。
*/
public class DatacenterExcelImportServiceImplTest {
/**
* 验证包含中文和常见标点的表头会转换为安全的物理字段名。
*/
@Test
public void shouldNormalizeExcelHeaderToSafeIdentifier() {
Assert.assertEquals("", DatacenterExcelImportServiceImpl.normalizeIdentifier("金额（万元）"));
Assert.assertEquals("amount_cny", DatacenterExcelImportServiceImpl.normalizeIdentifier("Amount（CNY）"));
Assert.assertEquals("model_version", DatacenterExcelImportServiceImpl.normalizeIdentifier("模型 / Model Version"));
}
/**
* 验证非 ASCII 表头、SQL 关键字和重复表头均会生成有效且唯一的字段名。
*/
@Test
public void shouldGenerateSafeUniqueExcelFieldNames() {
Set<String> usedNames = new HashSet<>();
Assert.assertEquals("col_1", DatacenterExcelImportServiceImpl.normalizeIdentifier("金额（万元）", 0, usedNames));
Assert.assertEquals("col_select", DatacenterExcelImportServiceImpl.normalizeIdentifier("select", 1, usedNames));
Assert.assertEquals("col_id", DatacenterExcelImportServiceImpl.normalizeIdentifier("模型ID", 2, usedNames));
Assert.assertEquals("col_id_1", DatacenterExcelImportServiceImpl.normalizeIdentifier("部门ID", 3, usedNames));
Assert.assertEquals("amount", DatacenterExcelImportServiceImpl.normalizeIdentifier("Amount", 4, usedNames));
Assert.assertEquals("amount_1", DatacenterExcelImportServiceImpl.normalizeIdentifier("Amount", 5, usedNames));
Assert.assertEquals("col_order", DatacenterExcelImportServiceImpl.normalizeIdentifier("Order", 6, usedNames));
Assert.assertEquals("col_group", DatacenterExcelImportServiceImpl.normalizeIdentifier("Group", 7, usedNames));
}
/**
* 验证超长异常摘要会保留首尾信息并限制在数据库字段长度内。
*/
@Test
public void shouldTruncateLongImportErrorSummary() {
String message = "起始原因:" + "a".repeat(1200) + ":最终原因";
String summary = DatacenterExcelImportServiceImpl.summarizeImportError(new IllegalArgumentException(message));
Assert.assertEquals(1000, summary.codePointCount(0, summary.length()));
Assert.assertTrue(summary.startsWith("起始原因:"));
Assert.assertTrue(summary.contains("…（错误摘要已截断）…"));
Assert.assertTrue(summary.endsWith(":最终原因"));
}
/**
* 验证仅允许 xls 和 xlsx 文件名通过服务端扩展名校验。
*/
@Test
public void shouldAcceptOnlySupportedWorkbookExtensions() {
Assert.assertTrue(DatacenterExcelImportServiceImpl.isSupportedWorkbookFileName("预算表.XLS"));
Assert.assertTrue(DatacenterExcelImportServiceImpl.isSupportedWorkbookFileName("预算表.xlsx"));
Assert.assertFalse(DatacenterExcelImportServiceImpl.isSupportedWorkbookFileName("预算表.xlsm"));
Assert.assertFalse(DatacenterExcelImportServiceImpl.isSupportedWorkbookFileName("预算表.csv"));
}
/**
* 验证当前 WorkbookFactory 能解析 xls 和 xlsx 工作簿。
*
* @throws IOException 工作簿序列化或读取失败时抛出
*/
@Test
public void shouldParseXlsAndXlsxWorkbooks() throws IOException {
assertWorkbookCanBeParsed(new HSSFWorkbook());
assertWorkbookCanBeParsed(new XSSFWorkbook());
}
/**
* 验证自定义连接名称优先使用，空名称回退到文件名。
*/
@Test
public void shouldResolveImportSourceName() {
Assert.assertEquals(
"模型预算",
DatacenterExcelImportServiceImpl.resolveImportSourceName(" 模型预算 ", "原文件.xlsx")
);
Assert.assertEquals(
"原文件",
DatacenterExcelImportServiceImpl.resolveImportSourceName(" ", "原文件.xlsx")
);
}
/**
* 验证连接名称超过数据库字段长度时给出业务错误。
*/
@Test
public void shouldRejectTooLongImportSourceName() {
BusinessException error = Assert.assertThrows(
BusinessException.class,
() -> DatacenterExcelImportServiceImpl.resolveImportSourceName("长".repeat(101), "原文件.xlsx")
);
Assert.assertEquals("连接名称不能超过 100 个字符", error.getMessage());
}
/**
* 验证空工作簿、无表头和只有表头的工作表均不可导入。
*/
@Test
public void shouldRejectWorkbookWithoutHeaderAndData() {
DataFormatter formatter = new DataFormatter();
try (Workbook workbook = new XSSFWorkbook()) {
Assert.assertFalse(DatacenterExcelImportServiceImpl.hasImportableSheet(workbook, formatter));
var sheet = workbook.createSheet("空表");
sheet.createRow(0).createCell(0).setCellValue("");
Assert.assertFalse(DatacenterExcelImportServiceImpl.hasImportableSheet(workbook, formatter));
sheet.getRow(0).getCell(0).setCellValue("模型ID");
Assert.assertFalse(DatacenterExcelImportServiceImpl.hasImportableSheet(workbook, formatter));
sheet.createRow(1).createCell(0).setCellValue("deepseek-v4-pro");
Assert.assertTrue(DatacenterExcelImportServiceImpl.hasImportableSheet(workbook, formatter));
} catch (IOException ex) {
Assert.fail(ex.getMessage());
}
}
/**
* 验证公式单元格导入计算结果，不暴露公式文本。
*/
@Test
public void shouldImportCalculatedFormulaValue() {
DataFormatter formatter = new DataFormatter();
try (Workbook workbook = new XSSFWorkbook()) {
var row = workbook.createSheet("公式").createRow(0);
row.createCell(0).setCellValue(3);
var formulaCell = row.createCell(1);
formulaCell.setCellFormula("A1*2");
String value = DatacenterExcelImportServiceImpl.formatCellValue(
formulaCell,
formatter,
workbook.getCreationHelper().createFormulaEvaluator()
);
Assert.assertEquals("6", value);
} catch (IOException ex) {
Assert.fail(ex.getMessage());
}
}
/**
* 将工作簿序列化后再交给导入使用的解析器读取。
*
* @param workbook 待验证的工作簿
* @throws IOException 工作簿序列化或读取失败时抛出
*/
private void assertWorkbookCanBeParsed(Workbook workbook) throws IOException {
workbook.createSheet("Sheet1");
try (workbook; ByteArrayOutputStream outputStream = new ByteArrayOutputStream()) {
workbook.write(outputStream);
try (Workbook parsedWorkbook = WorkbookFactory.create(new ByteArrayInputStream(outputStream.toByteArray()))) {
Assert.assertEquals(1, parsedWorkbook.getNumberOfSheets());
}
}
}
}

74
easyflow-modules/easyflow-module-datacenter/src/test/java/tech/easyflow/datacenter/execution/service/impl/DatacenterDatasetQueryServiceImplTest.java
View File

@@ -3,13 +3,17 @@ package tech.easyflow.datacenter.execution.service.impl;
import com.mybatisflex.core.row.Row;
import org.junit.Assert;
import org.junit.Test;
import org.mockito.ArgumentCaptor;
import org.mockito.ArgumentMatchers;
import org.mockito.Mockito;
import tech.easyflow.datacenter.connector.DatacenterConnector;
import tech.easyflow.datacenter.connector.DatacenterConnectorRegistry;
import tech.easyflow.datacenter.connector.support.AbstractInternalTableConnector;
import tech.easyflow.datacenter.entity.DatacenterTable;
import tech.easyflow.datacenter.execution.model.DatacenterSqlQueryRequest;
import tech.easyflow.datacenter.execution.model.DatasetRef;
import tech.easyflow.datacenter.mapper.DatacenterCatalogMapper;
import tech.easyflow.datacenter.meta.entity.DatacenterCatalog;
import tech.easyflow.datacenter.meta.entity.DatacenterSource;
import tech.easyflow.datacenter.meta.service.DatacenterDatasetRegistryService;
@@ -101,6 +105,76 @@ public class DatacenterDatasetQueryServiceImplTest {
ArgumentMatchers.any());
}
/**
* 验证内部 Excel 查询移除逻辑目录并转换 MySQL 标识符引号。
*
* @throws Exception 测试依赖注入失败时抛出
*/
@Test
public void queryBySqlShouldNormalizeInternalMysqlSql()
throws Exception {
BigInteger sourceId = BigInteger.valueOf(3001L);
BigInteger catalogId = BigInteger.valueOf(3002L);
DatacenterSource source = new DatacenterSource();
source.setId(sourceId);
source.setSourceType("EXCEL");
DatacenterTable table = new DatacenterTable();
table.setId(BigInteger.valueOf(3003L));
table.setSourceId(sourceId);
table.setCatalogId(catalogId);
table.setTableName("Sheet1");
table.setMaterializedTable("tb_excel_budget");
DatacenterCatalog catalog = new DatacenterCatalog();
catalog.setId(catalogId);
catalog.setCatalogName("ama 实验基线模型预算");
DatacenterDatasetRegistryService registry =
Mockito.mock(DatacenterDatasetRegistryService.class);
Mockito.when(registry.getSourceRequired(sourceId))
.thenReturn(source);
Mockito.when(registry.listManagedTables(sourceId, null))
.thenReturn(List.of(table));
DatacenterCatalogMapper catalogMapper =
Mockito.mock(DatacenterCatalogMapper.class);
Mockito.when(catalogMapper.selectListByQuery(
ArgumentMatchers.any()))
.thenReturn(List.of(catalog));
AbstractInternalTableConnector connector =
Mockito.mock(AbstractInternalTableConnector.class);
DatacenterConnectorRegistry connectors =
Mockito.mock(DatacenterConnectorRegistry.class);
Mockito.when(connectors.getConnector("EXCEL"))
.thenReturn(connector);
DatacenterDatasetQueryServiceImpl service =
new DatacenterDatasetQueryServiceImpl();
setField(service, "registryService", registry);
setField(service, "connectorRegistry", connectors);
setField(service, "catalogMapper", catalogMapper);
DatacenterSqlQueryRequest request =
new DatacenterSqlQueryRequest();
DatasetRef datasetRef = new DatasetRef();
datasetRef.setSourceId(sourceId);
request.setDatasetRef(datasetRef);
request.setSql("""
SELECT "token", "token_1"
FROM "ama 实验基线模型预算"."Sheet1"
WHERE "col_id" = 'deepseek-v4-pro'
""");
service.queryBySql(request);
ArgumentCaptor<String> sqlCaptor =
ArgumentCaptor.forClass(String.class);
Mockito.verify(connector).queryBySql(
ArgumentMatchers.eq(source),
sqlCaptor.capture());
Assert.assertEquals(
"SELECT `token`, `token_1` FROM tb_excel_budget "
+ "WHERE `col_id` = 'deepseek-v4-pro'",
sqlCaptor.getValue());
}
/**
* 创建测试数据行。
*

62
easyflow-modules/easyflow-module-datacenter/src/test/java/tech/easyflow/datacenter/meta/entity/DatacenterIdJsonSerializationTest.java Normal file
View File

@@ -0,0 +1,62 @@
package tech.easyflow.datacenter.meta.entity;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import org.junit.Assert;
import org.junit.Test;
import tech.easyflow.datacenter.entity.DatacenterTable;
import tech.easyflow.datacenter.meta.model.DatacenterCatalogMeta;
import java.math.BigInteger;
/**
* 数据中枢雪花 ID 的 HTTP JSON 序列化测试。
*/
public class DatacenterIdJsonSerializationTest {
private static final BigInteger UNSAFE_JAVASCRIPT_INTEGER = new BigInteger("9007199254740993");
private final ObjectMapper objectMapper = new ObjectMapper();
/**
* 验证连接、目录和表的雪花 ID 会输出为字符串，避免 JavaScript 精度丢失。
*
* @throws Exception JSON 序列化失败时抛出
*/
@Test
public void shouldSerializeDatacenterSnowflakeIdsAsStrings() throws Exception {
DatacenterSource source = new DatacenterSource();
source.setId(UNSAFE_JAVASCRIPT_INTEGER);
assertTextualId(source, "id");
DatacenterCatalog catalog = new DatacenterCatalog();
catalog.setId(UNSAFE_JAVASCRIPT_INTEGER);
assertTextualId(catalog, "id");
DatacenterCatalogMeta catalogMeta = new DatacenterCatalogMeta();
catalogMeta.setId(UNSAFE_JAVASCRIPT_INTEGER);
catalogMeta.setSourceId(UNSAFE_JAVASCRIPT_INTEGER);
assertTextualId(catalogMeta, "id");
assertTextualId(catalogMeta, "sourceId");
DatacenterTable table = new DatacenterTable();
table.setId(UNSAFE_JAVASCRIPT_INTEGER);
table.setSourceId(UNSAFE_JAVASCRIPT_INTEGER);
table.setCatalogId(UNSAFE_JAVASCRIPT_INTEGER);
assertTextualId(table, "id");
assertTextualId(table, "sourceId");
assertTextualId(table, "catalogId");
}
/**
* 验证给定属性被输出为精确的文本 ID。
*
* @param value 待序列化对象
* @param fieldName ID 属性名
* @throws Exception JSON 序列化失败时抛出
*/
private void assertTextualId(Object value, String fieldName) throws Exception {
JsonNode node = objectMapper.readTree(objectMapper.writeValueAsString(value)).path(fieldName);
Assert.assertTrue(fieldName + " 应为 JSON 字符串", node.isTextual());
Assert.assertEquals(UNSAFE_JAVASCRIPT_INTEGER.toString(), node.asText());
}
}

41
easyflow-modules/easyflow-module-datacenter/src/test/java/tech/easyflow/datacenter/utils/SqlInjectionUtilsTest.java Normal file
View File

@@ -0,0 +1,41 @@
package tech.easyflow.datacenter.utils;
import org.junit.Assert;
import org.junit.Test;
import tech.easyflow.common.web.exceptions.BusinessException;
/**
* {@link SqlInjectionUtils} 的单元测试。
*/
public class SqlInjectionUtilsTest {
/**
* 验证常见 Excel 表头标点能够作为数据库备注使用。
*/
@Test
public void shouldEscapeCommonExcelHeaderCharacters() {
String comment = "金额（万元）/预算\n50% \\ '含税'";
Assert.assertEquals("金额（万元）/预算 50% \\\\ ''含税''", SqlInjectionUtils.checkComment(comment));
}
/**
* 验证控制字符仍会被拒绝，避免动态 DDL 出现不可见内容。
*/
@Test(expected = BusinessException.class)
public void shouldRejectControlCharacterInComment() {
SqlInjectionUtils.checkComment("金额\u0000");
}
/**
* 验证常见查询保留字能够被大小写无关地识别。
*/
@Test
public void shouldRecognizeCommonQueryKeywords() {
Assert.assertTrue(SqlInjectionUtils.isSqlKeyword("Order"));
Assert.assertTrue(SqlInjectionUtils.isSqlKeyword("GROUP"));
Assert.assertTrue(SqlInjectionUtils.isSqlKeyword("limit"));
Assert.assertFalse(SqlInjectionUtils.isSqlKeyword("model_name"));
Assert.assertFalse(SqlInjectionUtils.isSqlKeyword(null));
}
}

51
easyflow-modules/easyflow-module-datacenter/src/test/java/tech/easyflow/datacenter/utils/SqlSupportUtilsTest.java Normal file
View File

@@ -0,0 +1,51 @@
package tech.easyflow.datacenter.utils;
import org.junit.Assert;
import org.junit.Test;
import java.util.List;
/**
* {@link SqlSupportUtils} 内部 MySQL SQL 重写测试。
*/
public class SqlSupportUtilsTest {
/**
* 验证逻辑目录只用于表匹配，执行 SQL 使用内部物理表。
*/
@Test
public void resolveInternalMysqlShouldDropLogicalCatalog() {
SqlSupportUtils.ResolvedSql resolved = SqlSupportUtils.resolveInternalMysql(
"""
SELECT "token", "token_1"
FROM "ama 实验基线模型预算"."Sheet1"
WHERE "col_id" = 'deepseek-v4-pro'
""",
List.of(new SqlSupportUtils.ManagedTable(
"ama 实验基线模型预算",
"Sheet1",
"tb_excel_budget")));
Assert.assertEquals(
"SELECT `token`, `token_1` FROM tb_excel_budget "
+ "WHERE `col_id` = 'deepseek-v4-pro'",
resolved.getExecutableSql());
}
/**
* 验证字符串字面量中的双引号不会被当成标识符转换。
*/
@Test
public void resolveInternalMysqlShouldPreserveStringLiteralQuotes() {
SqlSupportUtils.ResolvedSql resolved = SqlSupportUtils.resolveInternalMysql(
"SELECT \"token\" FROM \"Sheet1\" WHERE \"col_id\" = 'deep\"seek'",
List.of(new SqlSupportUtils.ManagedTable(
null,
"Sheet1",
"tb_excel_budget")));
Assert.assertEquals(
"SELECT `token` FROM tb_excel_budget WHERE `col_id` = 'deep\"seek'",
resolved.getExecutableSql());
}
}