diff --git a/easyflow-api/easyflow-api-admin/src/main/java/tech/easyflow/admin/controller/datacenter/DatacenterExcelController.java b/easyflow-api/easyflow-api-admin/src/main/java/tech/easyflow/admin/controller/datacenter/DatacenterExcelController.java index c4d299ed..1751c86e 100644 --- a/easyflow-api/easyflow-api-admin/src/main/java/tech/easyflow/admin/controller/datacenter/DatacenterExcelController.java +++ b/easyflow-api/easyflow-api-admin/src/main/java/tech/easyflow/admin/controller/datacenter/DatacenterExcelController.java @@ -6,6 +6,7 @@ import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.PostMapping; import org.springframework.web.bind.annotation.RequestBody; import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.RestController; import org.springframework.web.multipart.MultipartFile; import tech.easyflow.common.domain.Result; @@ -32,11 +33,21 @@ public class DatacenterExcelController { @Resource private DatacenterExcelImportService excelImportService; + /** + * 上传并导入 Excel 工作簿。 + * + * @param file Excel 工作簿 + * @param sourceName 数据源名称，留空时使用文件名 + * @return 导入任务及新建数据源标识 + * @throws Exception 文件解析或数据写入失败时抛出 + */ @PostMapping("/import") @SaCheckPermission("/api/v1/datacenterSource/save") - public Result importWorkbook(MultipartFile file) throws Exception { + public Result importWorkbook( + @RequestParam("file") MultipartFile file, + @RequestParam(value = "sourceName", required = false) String sourceName) throws Exception { LoginAccount account = SaTokenUtil.getLoginAccount(); - return Result.ok(excelImportService.importWorkbook(file, account)); + return Result.ok(excelImportService.importWorkbook(file, sourceName, account)); } @PostMapping("/split") diff --git a/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/easyagentsflow/service/WorkflowDatacenterContentService.java b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/easyagentsflow/service/WorkflowDatacenterContentService.java index 221b9bde..e35d242e 100644 --- a/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/easyagentsflow/service/WorkflowDatacenterContentService.java +++ b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/easyagentsflow/service/WorkflowDatacenterContentService.java @@ -37,6 +37,7 @@ public class WorkflowDatacenterContentService { public static final String SEARCH_SQL_MISSING_MESSAGE = "查询数据节点未设置 SQL"; public static final String SAVE_EXPIRED_MESSAGE = "写入数据节点配置已过期，请重新选择已接入表"; public static final String INVALID_QUERY_CONTEXT_MESSAGE = "查询上下文配置无效，请重新选择查询数据节点"; + private static final String QUERY_DATA_CONTEXT_PLACEHOLDER = "{{" + QUERY_DATA_CONTEXT + "}}"; private static final String QUERY_CONTEXT_PROMPT = """ 你是为工作流中的查询数据节点生成只读 SQL 的生成器，你的职责是返回可直接执行的 SQL，并且你只能输出 SQL。 @@ -47,7 +48,9 @@ public class WorkflowDatacenterContentService { 4. 只能生成只读 SELECT SQL，允许 WITH、JOIN、子查询、聚合、分组、排序。 5. 不要生成 INSERT、UPDATE、DELETE、DDL、多语句、存储过程调用。 6. 优先使用逻辑表名和逻辑字段名，不要输出物理表名、JDBC、驱动信息。 - 7. 如果存在重名表，请使用 catalog.table 形式消除歧义。 + 7. 表名和字段名默认不要加引号，禁止使用双引号包裹标识符。 + 8. 只有摘要中存在重名表时才使用 catalog.table；没有重名时只输出 tableName，不要添加 sourceName 或 catalogName 前缀。 + 9. 当字段名无法体现业务含义、字段描述提供了明确含义时，在 SELECT 中为该字段添加简短清晰的英文 snake_case 别名，例如 token AS input_price；别名不要加引号。 以下是可用的连接摘要： """; @@ -211,6 +214,7 @@ public class WorkflowDatacenterContentService { JSONArray nodeIds = data.getJSONArray("queryContextNodeIds"); if (nodeIds == null || nodeIds.isEmpty()) { removeQueryDataContextParameter(data); + removeQueryDataContextPlaceholder(data); return; } Map sourceSummaries = new LinkedHashMap<>(); @@ -229,6 +233,37 @@ public class WorkflowDatacenterContentService { } String contextValue = QUERY_CONTEXT_PROMPT + "\n" + JSON.toJSONString(new ArrayList<>(sourceSummaries.values())); upsertQueryDataContextParameter(data, contextValue); + appendQueryDataContextPlaceholder(data); + } + + /** + * 将查询上下文参数追加到大模型系统提示词，避免覆盖用户配置的提示词。 + * + * @param data 大模型节点数据 + */ + private void appendQueryDataContextPlaceholder(JSONObject data) { + String systemPrompt = data.getString("systemPrompt"); + if (StringUtils.hasText(systemPrompt) && systemPrompt.contains(QUERY_DATA_CONTEXT_PLACEHOLDER)) { + return; + } + if (!StringUtils.hasText(systemPrompt)) { + data.put("systemPrompt", QUERY_DATA_CONTEXT_PLACEHOLDER); + return; + } + data.put("systemPrompt", systemPrompt.stripTrailing() + "\n\n" + QUERY_DATA_CONTEXT_PLACEHOLDER); + } + + /** + * 查询上下文关闭后移除自动绑定的提示词占位符。 + * + * @param data 大模型节点数据 + */ + private void removeQueryDataContextPlaceholder(JSONObject data) { + String systemPrompt = data.getString("systemPrompt"); + if (!StringUtils.hasText(systemPrompt) || !systemPrompt.contains(QUERY_DATA_CONTEXT_PLACEHOLDER)) { + return; + } + data.put("systemPrompt", systemPrompt.replace(QUERY_DATA_CONTEXT_PLACEHOLDER, "").trim()); } private String resolveFieldType(DatacenterTableField field) { diff --git a/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/node/SearchDatasetNode.java b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/node/SearchDatasetNode.java index f5c00320..742954a3 100644 --- a/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/node/SearchDatasetNode.java +++ b/easyflow-modules/easyflow-module-ai/src/main/java/tech/easyflow/ai/node/SearchDatasetNode.java @@ -26,6 +26,9 @@ public class SearchDatasetNode extends BaseNode { private static final Pattern PARAM_PATTERN = Pattern.compile("\\{\\{(.+?)\\}\\}"); + private static final Pattern SQL_CODE_BLOCK_PATTERN = Pattern.compile( + "\\A```(?:sql)?\\s*([\\s\\S]*?)\\s*```\\z", + Pattern.CASE_INSENSITIVE); private static final int QUERY_PAGE_SIZE = Math.max( 1, Integer.getInteger( @@ -101,13 +104,28 @@ public class SearchDatasetNode extends BaseNode { } private String resolveQuerySql(Map params) { - String sql = resolveTemplateString(querySql, params); + String sql = normalizeSqlCodeBlock(resolveTemplateString(querySql, params)); if (!StringUtil.hasText(sql)) { throw new BusinessException("查询数据节点未设置 SQL"); } return sql.trim(); } + /** + * 去除完整单个 SQL Markdown 代码块的边界标记。 + * + * @param sql 原始 SQL 文本 + * @return 可交给 SQL 解析器处理的文本 + */ + private String normalizeSqlCodeBlock(String sql) { + if (!StringUtil.hasText(sql)) { + return sql; + } + String trimmed = sql.trim(); + Matcher matcher = SQL_CODE_BLOCK_PATTERN.matcher(trimmed); + return matcher.matches() ? matcher.group(1).trim() : trimmed; + } + private DatasetRef copyDatasetRef() { DatasetRef copy = new DatasetRef(); copy.setSourceId(datasetRef == null ? null : datasetRef.getSourceId()); diff --git a/easyflow-modules/easyflow-module-ai/src/test/java/tech/easyflow/ai/easyagentsflow/service/WorkflowDatacenterContentServiceTest.java b/easyflow-modules/easyflow-module-ai/src/test/java/tech/easyflow/ai/easyagentsflow/service/WorkflowDatacenterContentServiceTest.java new file mode 100644 index 00000000..fb9a1ffc --- /dev/null +++ b/easyflow-modules/easyflow-module-ai/src/test/java/tech/easyflow/ai/easyagentsflow/service/WorkflowDatacenterContentServiceTest.java @@ -0,0 +1,217 @@ +package tech.easyflow.ai.easyagentsflow.service; + +import com.alibaba.fastjson2.JSONArray; +import com.alibaba.fastjson2.JSONObject; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.mockito.Mockito; +import tech.easyflow.datacenter.entity.DatacenterTable; +import tech.easyflow.datacenter.entity.DatacenterTableField; +import tech.easyflow.datacenter.meta.entity.DatacenterSource; +import tech.easyflow.datacenter.meta.service.DatacenterDatasetRegistryService; + +import java.lang.reflect.Field; +import java.math.BigInteger; +import java.util.ArrayList; +import java.util.List; + +/** + * 工作流数据中枢内容准备服务测试。 + */ +public class WorkflowDatacenterContentServiceTest { + + private static final BigInteger SOURCE_ID = BigInteger.valueOf(1001L); + private static final BigInteger TABLE_ID = BigInteger.valueOf(2001L); + + private WorkflowDatacenterContentService service; + private DatacenterDatasetRegistryService registryService; + + /** + * 初始化服务及数据源元数据桩。 + * + * @throws Exception 注入测试依赖失败时抛出 + */ + @Before + public void setUp() throws Exception { + service = new WorkflowDatacenterContentService(); + registryService = Mockito.mock(DatacenterDatasetRegistryService.class); + injectField(service, "registryService", registryService); + + DatacenterSource source = Mockito.mock(DatacenterSource.class); + Mockito.when(source.getSourceName()).thenReturn("ama 实验基线模型预算"); + Mockito.when(source.getSourceType()).thenReturn("EXCEL"); + + DatacenterTableField modelId = mockField("col_id", "模型ID", "VARCHAR"); + DatacenterTableField inputPrice = mockField("token", "输入价格", "DECIMAL"); + DatacenterTable table = Mockito.mock(DatacenterTable.class); + Mockito.when(table.getId()).thenReturn(TABLE_ID); + Mockito.when(table.getTableName()).thenReturn("Sheet1"); + Mockito.when(table.getFields()).thenReturn(List.of(modelId, inputPrice)); + + Mockito.when(registryService.getSourceRequired(SOURCE_ID)).thenReturn(source); + Mockito.when(registryService.listManagedTables(SOURCE_ID, null)) + .thenReturn(new ArrayList<>(List.of(table))); + Mockito.when(registryService.getTableWithFields(TABLE_ID)).thenReturn(table); + } + + /** + * 验证查询上下文会自动且幂等地绑定到系统提示词。 + */ + @Test + public void testPrepareRootShouldBindQueryContextToSystemPromptIdempotently() { + JSONObject root = buildWorkflowRoot(); + JSONObject llmData = root.getJSONArray("nodes").getJSONObject(1).getJSONObject("data"); + + service.prepareRoot(root); + service.prepareRoot(root); + + String systemPrompt = llmData.getString("systemPrompt"); + Assert.assertEquals("请根据问题生成查询语句\n\n{{queryDataContext}}", systemPrompt); + Assert.assertEquals(1, countOccurrences(systemPrompt, "{{queryDataContext}}")); + JSONObject contextParameter = findParameter(llmData, "queryDataContext"); + Assert.assertNotNull(contextParameter); + String contextValue = contextParameter.getString("value"); + Assert.assertTrue(contextValue.contains("只输出 SQL")); + Assert.assertTrue(contextValue.contains("Sheet1")); + Assert.assertTrue(contextValue.contains("col_id")); + Assert.assertTrue(contextValue.contains("模型ID")); + Assert.assertTrue(contextValue.contains("token AS input_price")); + } + + /** + * 验证关闭查询上下文后会同步清理参数和系统提示词占位符。 + */ + @Test + public void testPrepareRootShouldRemoveQueryContextBindingWhenDisabled() { + JSONObject root = buildWorkflowRoot(); + JSONObject llmData = root.getJSONArray("nodes").getJSONObject(1).getJSONObject("data"); + service.prepareRoot(root); + llmData.put("queryContextNodeIds", new JSONArray()); + + service.prepareRoot(root); + + Assert.assertEquals("请根据问题生成查询语句", llmData.getString("systemPrompt")); + Assert.assertNull(findParameter(llmData, "queryDataContext")); + } + + /** + * 构造包含查询节点和大模型节点的最小工作流。 + * + * @return 工作流根对象 + */ + private JSONObject buildWorkflowRoot() { + JSONObject datasetRef = new JSONObject(); + datasetRef.put("sourceId", SOURCE_ID); + + JSONObject queryData = new JSONObject(); + queryData.put("datasetRef", datasetRef); + queryData.put("querySql", "{{query}}"); + JSONObject queryNode = buildNode( + "query-node", + WorkflowDatacenterContentService.SEARCH_NODE_TYPE, + queryData); + + JSONArray queryContextNodeIds = new JSONArray(); + queryContextNodeIds.add("query-node"); + JSONObject llmData = new JSONObject(); + llmData.put("systemPrompt", "请根据问题生成查询语句"); + llmData.put("queryContextNodeIds", queryContextNodeIds); + llmData.put("parameters", new JSONArray()); + JSONObject llmNode = buildNode( + "llm-node", + WorkflowDatacenterContentService.LLM_NODE_TYPE, + llmData); + + JSONArray nodes = new JSONArray(); + nodes.add(queryNode); + nodes.add(llmNode); + JSONObject root = new JSONObject(); + root.put("nodes", nodes); + return root; + } + + /** + * 构造工作流节点。 + * + * @param id 节点标识 + * @param type 节点类型 + * @param data 节点数据 + * @return 工作流节点 + */ + private JSONObject buildNode(String id, String type, JSONObject data) { + JSONObject node = new JSONObject(); + node.put("id", id); + node.put("type", type); + node.put("data", data); + return node; + } + + /** + * 构造字段元数据桩。 + * + * @param fieldName 字段名 + * @param fieldDesc 字段描述 + * @param jdbcType JDBC 类型 + * @return 字段元数据 + */ + private DatacenterTableField mockField(String fieldName, String fieldDesc, String jdbcType) { + DatacenterTableField field = Mockito.mock(DatacenterTableField.class); + Mockito.when(field.getFieldName()).thenReturn(fieldName); + Mockito.when(field.getFieldDesc()).thenReturn(fieldDesc); + Mockito.when(field.getJdbcType()).thenReturn(jdbcType); + return field; + } + + /** + * 查找指定名称的节点参数。 + * + * @param data 节点数据 + * @param name 参数名 + * @return 参数对象，不存在时返回 {@code null} + */ + private JSONObject findParameter(JSONObject data, String name) { + JSONArray parameters = data.getJSONArray("parameters"); + if (parameters == null) { + return null; + } + for (int i = 0; i < parameters.size(); i++) { + JSONObject parameter = parameters.getJSONObject(i); + if (parameter != null && name.equals(parameter.getString("name"))) { + return parameter; + } + } + return null; + } + + /** + * 统计子串出现次数。 + * + * @param value 原始文本 + * @param target 目标子串 + * @return 出现次数 + */ + private int countOccurrences(String value, String target) { + int count = 0; + int index = 0; + while ((index = value.indexOf(target, index)) >= 0) { + count++; + index += target.length(); + } + return count; + } + + /** + * 注入服务私有依赖。 + * + * @param target 目标对象 + * @param fieldName 字段名 + * @param value 字段值 + * @throws Exception 反射注入失败时抛出 + */ + private void injectField(Object target, String fieldName, Object value) throws Exception { + Field field = target.getClass().getDeclaredField(fieldName); + field.setAccessible(true); + field.set(target, value); + } +} diff --git a/easyflow-modules/easyflow-module-ai/src/test/java/tech/easyflow/ai/node/SearchDatasetNodeTest.java b/easyflow-modules/easyflow-module-ai/src/test/java/tech/easyflow/ai/node/SearchDatasetNodeTest.java index 7460f67d..59cd2eae 100644 --- a/easyflow-modules/easyflow-module-ai/src/test/java/tech/easyflow/ai/node/SearchDatasetNodeTest.java +++ b/easyflow-modules/easyflow-module-ai/src/test/java/tech/easyflow/ai/node/SearchDatasetNodeTest.java @@ -65,6 +65,43 @@ public class SearchDatasetNodeTest { } } + /** + * 验证查询节点可接收大模型返回的单个 SQL Markdown 代码块。 + * + * @throws Exception 反射调用失败时抛出 + */ + @Test + public void testResolveQuerySqlShouldUnwrapMarkdownSqlCodeBlock() throws Exception { + DatasetRef datasetRef = new DatasetRef(); + datasetRef.setSourceId(BigInteger.valueOf(4004L)); + SearchDatasetNode node = new SearchDatasetNode(datasetRef, "{{query}}"); + Map params = new HashMap<>(); + params.put("query", "```sql SELECT token FROM Sheet1 WHERE col_id = 'deepseek-v4-pro'; ```"); + + String sql = invokeResolveQuerySql(node, params); + + Assert.assertEquals("SELECT token FROM Sheet1 WHERE col_id = 'deepseek-v4-pro';", sql); + } + + /** + * 验证包含说明文本的模型输出不会被宽松清洗，从而继续由 SQL 解析器拒绝。 + * + * @throws Exception 反射调用失败时抛出 + */ + @Test + public void testResolveQuerySqlShouldKeepTextOutsideSqlCodeBlock() throws Exception { + DatasetRef datasetRef = new DatasetRef(); + datasetRef.setSourceId(BigInteger.valueOf(5005L)); + SearchDatasetNode node = new SearchDatasetNode(datasetRef, "{{query}}"); + Map params = new HashMap<>(); + String generated = "查询语句如下：\n```sql\nSELECT token FROM Sheet1;\n```"; + params.put("query", generated); + + String sql = invokeResolveQuerySql(node, params); + + Assert.assertEquals(generated, sql); + } + private String invokeResolveQuerySql(SearchDatasetNode node, Map params) throws Exception { Method method = SearchDatasetNode.class.getDeclaredMethod("resolveQuerySql", Map.class); method.setAccessible(true); diff --git a/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/adapter/DefaultDbHandleService.java b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/adapter/DefaultDbHandleService.java index 57a4cd40..6897fb16 100644 --- a/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/adapter/DefaultDbHandleService.java +++ b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/adapter/DefaultDbHandleService.java @@ -16,20 +16,25 @@ import tech.easyflow.datacenter.utils.SqlInjectionUtils; import java.math.BigInteger; import java.util.Date; import java.util.List; +import java.util.Objects; @Component("defaultDbHandleService") public class DefaultDbHandleService extends DbHandleService { private static final Logger log = LoggerFactory.getLogger(DefaultDbHandleService.class); + /** + * 创建指定的物理数据表。 + * + * @param table 包含表结构和字段定义的数据表 + */ @Override public void createTable(DatacenterTable table) { // 设置为 [tb_dynamic_表名_tableId] 的格式 String actualTable = table.getActualTable(); SqlInjectionUtils.checkIdentifier(actualTable); // 表注释 - String tableDesc = table.getTableDesc(); - SqlInjectionUtils.checkComment(tableDesc); + String tableDesc = SqlInjectionUtils.checkComment(table.getTableDesc()); List fields = table.getFields(); StringBuilder sql = new StringBuilder("CREATE TABLE " + actualTable + " ("); @@ -56,15 +61,21 @@ public class DefaultDbHandleService extends DbHandleService { Db.selectObject(sql.toString()); } + /** + * 更新数据表的可修改元数据。 + * + * @param table 待更新的数据表 + * @param record 已持久化的数据表记录 + */ @Override public void updateTable(DatacenterTable table, DatacenterTable record) { String tableDesc = table.getTableDesc(); - SqlInjectionUtils.checkComment(tableDesc); + String escapedTableDesc = SqlInjectionUtils.checkComment(tableDesc); String actualTable = record.getActualTable(); // 只允许改表备注 - if (!tableDesc.equals(record.getTableDesc())) { + if (!Objects.equals(tableDesc, record.getTableDesc())) { String sql = "ALTER TABLE `" + actualTable + "` " - + "COMMENT '" + tableDesc + "';"; + + "COMMENT '" + escapedTableDesc + "';"; log.info("修改表备注语句 >>> {}", sql); Db.selectObject(sql); } @@ -95,13 +106,18 @@ public class DefaultDbHandleService extends DbHandleService { return "text"; } + /** + * 为物理数据表新增字段。 + * + * @param entity 目标数据表 + * @param field 待新增字段 + */ @Override public void addField(DatacenterTable entity, DatacenterTableField field) { String fieldName = field.getFieldName(); SqlInjectionUtils.checkIdentifier(fieldName); - String fieldDesc = field.getFieldDesc(); - SqlInjectionUtils.checkComment(fieldDesc); + String fieldDesc = SqlInjectionUtils.checkComment(field.getFieldDesc()); Integer fieldType = field.getFieldType(); Integer required = field.getRequired(); @@ -125,6 +141,13 @@ public class DefaultDbHandleService extends DbHandleService { Db.selectObject(sql); } + /** + * 更新物理数据表中的字段定义。 + * + * @param entity 目标数据表 + * @param fieldRecord 已持久化的字段记录 + * @param field 待更新字段 + */ @Override public void updateField(DatacenterTable entity, DatacenterTableField fieldRecord, DatacenterTableField field) { String actualTable = entity.getActualTable(); @@ -135,10 +158,10 @@ public class DefaultDbHandleService extends DbHandleService { SqlInjectionUtils.checkIdentifier(fieldName); // 字段描述 String fieldDesc = field.getFieldDesc(); - SqlInjectionUtils.checkComment(fieldDesc); + String escapedFieldDesc = SqlInjectionUtils.checkComment(fieldDesc); String nullable = required == 1 ? "NOT NULL " : "NULL "; - String desc = "COMMENT '" + fieldDesc + "';"; + String desc = "COMMENT '" + escapedFieldDesc + "';"; boolean isUpdate = false; String handleType = "MODIFY COLUMN `" + fieldRecord.getFieldName() + "` "; diff --git a/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/connector/support/AbstractInternalTableConnector.java b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/connector/support/AbstractInternalTableConnector.java index f65d130c..79316aed 100644 --- a/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/connector/support/AbstractInternalTableConnector.java +++ b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/connector/support/AbstractInternalTableConnector.java @@ -86,19 +86,34 @@ public abstract class AbstractInternalTableConnector implements DatacenterConnec @Override public Page queryPage(DatacenterSource source, DatacenterTable table, DatacenterQueryRequest request) { String actualTable = resolveTableName(table); - QueryWrapper wrapper = QueryWrapper.create(); - if (StrUtil.isNotBlank(request.getWhere())) { - wrapper.where(request.getWhere()); - } - long count = Db.selectCountByQuery(actualTable, wrapper); + long count = Db.selectCountByQuery( + actualTable, createQueryWrapper(request.getWhere())); if (count == 0) { return new Page<>(new ArrayList<>(), request.getPageNumber(), request.getPageSize(), count); } - Page page = Db.paginate(actualTable, new Page<>(request.getPageNumber(), request.getPageSize(), count), wrapper); + // selectCountByQuery 会把无投影的 QueryWrapper 改为 COUNT(*)，分页查询必须使用独立实例。 + Page page = Db.paginate( + actualTable, + new Page<>(request.getPageNumber(), request.getPageSize(), count), + createQueryWrapper(request.getWhere())); normalizeRows(page.getRecords()); return page; } + /** + * 创建用于动态表查询的独立条件包装器。 + * + * @param where 已校验的筛选表达式 + * @return 新建的查询条件包装器 + */ + static QueryWrapper createQueryWrapper(String where) { + QueryWrapper wrapper = QueryWrapper.create(); + if (StrUtil.isNotBlank(where)) { + wrapper.where(where); + } + return wrapper; + } + @Override public List queryBySql(DatacenterSource source, String sql) { List rows = Db.selectListBySql(sql); @@ -527,12 +542,20 @@ public abstract class AbstractInternalTableConnector implements DatacenterConnec return StrUtil.blankToDefault(table.getMaterializedTable(), table.getActualTable()); } - private void normalizeRows(List records) { + /** + * 统一内部表查询结果的字段名与值类型。 + * + *

MySQL 驱动在部分查询路径会返回大写列标签，而数据集元数据中的物理字段名 + * 始终为小写。这里在序列化前收敛为小写，确保预览、SQL 消费和导出按同一字段名取值。

+ * + * @param records 待规范化的数据行 + */ + static void normalizeRows(List records) { for (Row record : records) { Map converted = new LinkedHashMap<>(); for (Map.Entry entry : record.entrySet()) { converted.put( - entry.getKey(), + entry.getKey().toLowerCase(Locale.ROOT), normalizeValue(entry.getValue())); } record.clear(); @@ -546,7 +569,7 @@ public abstract class AbstractInternalTableConnector implements DatacenterConnec * @param value JDBC 原始值 * @return 兼容既有查询接口的值 */ - private Object normalizeValue(Object value) { + private static Object normalizeValue(Object value) { if (value instanceof BigInteger || value instanceof BigDecimal || value instanceof Long) { diff --git a/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/connector/support/DatacenterConnectorExceptionSupport.java b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/connector/support/DatacenterConnectorExceptionSupport.java index 1724fc2b..b0447f26 100644 --- a/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/connector/support/DatacenterConnectorExceptionSupport.java +++ b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/connector/support/DatacenterConnectorExceptionSupport.java @@ -10,6 +10,9 @@ import java.net.UnknownHostException; import java.sql.SQLException; import java.util.Locale; +/** + * 数据连接器访问异常分类与安全包装工具。 + */ public final class DatacenterConnectorExceptionSupport { public static final String SOURCE_UNAVAILABLE_MESSAGE = "当前连接不可用，请检查连接配置后重试"; @@ -17,14 +20,21 @@ public final class DatacenterConnectorExceptionSupport { private DatacenterConnectorExceptionSupport() { } + /** + * 将连接器异常转换为可安全展示的业务异常，并保留原始异常供日志追踪。 + * + * @param fallbackMessage 默认业务错误 + * @param ex 原始异常 + * @return 包装后的业务异常 + */ public static BusinessException wrapAccessException(String fallbackMessage, Exception ex) { if (ex instanceof BusinessException businessException && !isConnectionUnavailable(ex)) { return businessException; } if (isConnectionUnavailable(ex)) { - return new BusinessException(SOURCE_UNAVAILABLE_MESSAGE); + return new BusinessException(400, 1, SOURCE_UNAVAILABLE_MESSAGE, ex); } - return new BusinessException(fallbackMessage); + return new BusinessException(400, 1, fallbackMessage, ex); } public static boolean isConnectionUnavailable(Throwable throwable) { diff --git a/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/entity/base/DatacenterTableBase.java b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/entity/base/DatacenterTableBase.java index e5a49e79..9c2c6d83 100644 --- a/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/entity/base/DatacenterTableBase.java +++ b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/entity/base/DatacenterTableBase.java @@ -1,5 +1,7 @@ package tech.easyflow.datacenter.entity.base; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; +import com.fasterxml.jackson.databind.ser.std.ToStringSerializer; import com.mybatisflex.annotation.Column; import com.mybatisflex.annotation.Id; import com.mybatisflex.annotation.KeyType; @@ -129,6 +131,7 @@ public class DatacenterTableBase extends DateEntity implements Serializable { @Column(typeHandler = FastjsonTypeHandler.class, comment = "能力声明") private Map capabilitiesJson; + @JsonSerialize(using = ToStringSerializer.class) public BigInteger getId() { return id; } @@ -153,6 +156,7 @@ public class DatacenterTableBase extends DateEntity implements Serializable { this.tenantId = tenantId; } + @JsonSerialize(using = ToStringSerializer.class) public BigInteger getSourceId() { return sourceId; } @@ -161,6 +165,7 @@ public class DatacenterTableBase extends DateEntity implements Serializable { this.sourceId = sourceId; } + @JsonSerialize(using = ToStringSerializer.class) public BigInteger getCatalogId() { return catalogId; } diff --git a/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/entity/base/DatacenterTableFieldBase.java b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/entity/base/DatacenterTableFieldBase.java index beb0b4b9..beed28c4 100644 --- a/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/entity/base/DatacenterTableFieldBase.java +++ b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/entity/base/DatacenterTableFieldBase.java @@ -1,5 +1,7 @@ package tech.easyflow.datacenter.entity.base; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; +import com.fasterxml.jackson.databind.ser.std.ToStringSerializer; import com.mybatisflex.annotation.Column; import com.mybatisflex.annotation.Id; import com.mybatisflex.annotation.KeyType; @@ -129,6 +131,7 @@ public class DatacenterTableFieldBase extends DateEntity implements Serializable @Column(comment = "修改者") private BigInteger modifiedBy; + @JsonSerialize(using = ToStringSerializer.class) public BigInteger getId() { return id; } @@ -137,6 +140,7 @@ public class DatacenterTableFieldBase extends DateEntity implements Serializable this.id = id; } + @JsonSerialize(using = ToStringSerializer.class) public BigInteger getTableId() { return tableId; } diff --git a/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/excel/service/DatacenterExcelImportService.java b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/excel/service/DatacenterExcelImportService.java index 7aae5583..5f0910e8 100644 --- a/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/excel/service/DatacenterExcelImportService.java +++ b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/excel/service/DatacenterExcelImportService.java @@ -12,7 +12,16 @@ import java.math.BigInteger; import java.util.List; public interface DatacenterExcelImportService { - DatacenterImportJob importWorkbook(MultipartFile file, LoginAccount account) throws Exception; + /** + * 导入 Excel 工作簿并创建对应的数据源、目录和物化表。 + * + * @param file Excel 工作簿文件 + * @param sourceName 用户指定的数据源名称，留空时使用文件名 + * @param account 当前登录账号 + * @return 已完成的导入任务 + * @throws Exception 文件读取或数据写入失败时抛出 + */ + DatacenterImportJob importWorkbook(MultipartFile file, String sourceName, LoginAccount account) throws Exception; DatacenterImportJob splitWorkbook(DatacenterExcelSplitRequest request, LoginAccount account); diff --git a/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/excel/service/impl/DatacenterExcelImportServiceImpl.java b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/excel/service/impl/DatacenterExcelImportServiceImpl.java index 34a734f8..84751ce4 100644 --- a/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/excel/service/impl/DatacenterExcelImportServiceImpl.java +++ b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/excel/service/impl/DatacenterExcelImportServiceImpl.java @@ -6,6 +6,7 @@ import com.mybatisflex.core.paginate.Page; import com.mybatisflex.core.row.Row; import org.apache.poi.ss.usermodel.Cell; import org.apache.poi.ss.usermodel.DataFormatter; +import org.apache.poi.ss.usermodel.FormulaEvaluator; import org.apache.poi.ss.usermodel.Sheet; import org.apache.poi.ss.usermodel.Workbook; import org.apache.poi.ss.usermodel.WorkbookFactory; @@ -41,6 +42,7 @@ import tech.easyflow.datacenter.meta.enums.DatacenterImportStatus; import tech.easyflow.datacenter.meta.enums.DatacenterSourceType; import tech.easyflow.datacenter.meta.enums.DatacenterTableKind; import tech.easyflow.datacenter.meta.model.DatacenterTableDetailMeta; +import tech.easyflow.datacenter.utils.SqlInjectionUtils; import tech.easyflow.datacenter.meta.service.DatacenterDatasetRegistryService; import tech.easyflow.datacenter.meta.service.DatacenterSourceService; @@ -70,6 +72,14 @@ import java.util.UUID; public class DatacenterExcelImportServiceImpl implements DatacenterExcelImportService { private static final long QUERY_BATCH_SIZE = 500L; + private static final int MAX_IDENTIFIER_LENGTH = 64; + private static final int MAX_SOURCE_NAME_LENGTH = 100; + private static final int MAX_IMPORT_JOB_ERROR_SUMMARY_LENGTH = 1000; + private static final String ERROR_SUMMARY_TRUNCATION_MARKER = "…（错误摘要已截断）…"; + private static final Set SYSTEM_FIELD_NAMES = Set.of( + "id", "dept_id", "tenant_id", "created", "created_by", + "modified", "modified_by", "remark" + ); private static final DateTimeFormatter EXPORT_TIME_FORMAT = DateTimeFormatter.ofPattern("yyyyMMddHHmmss"); @Resource @@ -89,11 +99,58 @@ public class DatacenterExcelImportServiceImpl implements DatacenterExcelImportSe @Override @Transactional(rollbackFor = Exception.class) - public DatacenterImportJob importWorkbook(MultipartFile file, LoginAccount account) throws Exception { + public DatacenterImportJob importWorkbook( + MultipartFile file, + String sourceName, + LoginAccount account) throws Exception { if (file == null || file.isEmpty()) { throw new BusinessException("Excel 文件不能为空"); } - String workbookName = extractWorkbookName(file.getOriginalFilename()); + if (!isSupportedWorkbookFileName(file.getOriginalFilename())) { + throw new BusinessException("仅支持 .xls 和 .xlsx 格式的 Excel 文件"); + } + + try (InputStream inputStream = file.getInputStream()) { + Workbook parsedWorkbook; + try { + parsedWorkbook = WorkbookFactory.create(inputStream); + } catch (Exception ex) { + throw new BusinessException( + 400, + 1, + "Excel 文件无法解析，请确认文件未损坏且未加密", + ex + ); + } + try (Workbook workbook = parsedWorkbook) { + return importParsedWorkbook(file, sourceName, account, workbook); + } + } + } + + /** + * 将已解析并校验的工作簿写入数据中枢。 + * + * @param file 原始上传文件 + * @param sourceName 用户指定的数据源名称 + * @param account 当前登录账号 + * @param workbook 已解析的工作簿 + * @return 已完成的导入任务 + */ + private DatacenterImportJob importParsedWorkbook( + MultipartFile file, + String sourceName, + LoginAccount account, + Workbook workbook) { + DataFormatter formatter = new DataFormatter(); + FormulaEvaluator formulaEvaluator = workbook.getCreationHelper().createFormulaEvaluator(); + if (!hasImportableSheet(workbook, formatter)) { + throw new BusinessException( + "未检测到可导入的数据，请确认至少有一个工作表，首行包含表头且下方有数据" + ); + } + + String workbookName = resolveImportSourceName(sourceName, file.getOriginalFilename()); DatacenterSource source = new DatacenterSource(); source.setSourceName(workbookName); source.setSourceCode("EXCEL_" + UUID.randomUUID()); @@ -110,18 +167,14 @@ public class DatacenterExcelImportServiceImpl implements DatacenterExcelImportSe long totalRows = 0L; long successRows = 0L; List createdTableIds = new ArrayList<>(); - try (InputStream inputStream = file.getInputStream(); Workbook workbook = WorkbookFactory.create(inputStream)) { - DataFormatter formatter = new DataFormatter(); + try { for (int sheetIndex = 0; sheetIndex < workbook.getNumberOfSheets(); sheetIndex++) { Sheet sheet = workbook.getSheetAt(sheetIndex); org.apache.poi.ss.usermodel.Row headerRow = sheet.getRow(sheet.getFirstRowNum()); - if (headerRow == null) { + if (!isImportableSheet(sheet, headerRow, formatter)) { continue; } List fields = buildFields(headerRow, formatter); - if (fields.isEmpty()) { - continue; - } DatacenterTable table = new DatacenterTable(); table.setTableName(uniqueTableName(source.getId(), catalog.getId(), sheet.getSheetName())); table.setTableDesc(sheet.getSheetName()); @@ -150,7 +203,11 @@ public class DatacenterExcelImportServiceImpl implements DatacenterExcelImportSe JSONObject payload = new JSONObject(); boolean hasValue = false; for (int cellIndex = 0; cellIndex < savedTable.getFields().size(); cellIndex++) { - String value = formatter.formatCellValue(row.getCell(cellIndex)); + String value = formatCellValue( + row.getCell(cellIndex), + formatter, + formulaEvaluator + ); if (value != null && !value.isBlank()) { hasValue = true; } @@ -544,14 +601,59 @@ public class DatacenterExcelImportServiceImpl implements DatacenterExcelImportSe importJobMapper.update(job); } + /** + * 将导入任务标记为失败，并保存长度受限的错误摘要。 + * + * @param job 待更新的导入任务 + * @param ex 导入过程中抛出的异常 + */ private void finishJobFailure(DatacenterImportJob job, Exception ex) { job.setStatus(DatacenterImportStatus.FAILED.name()); - job.setErrorSummary(ex.getMessage()); + job.setErrorSummary(summarizeImportError(ex)); job.setFinishedAt(new Date()); job.setModified(new Date()); importJobMapper.update(job); } + /** + * 生成可写入导入任务错误摘要列的错误信息。 + * + * @param ex 导入过程中抛出的异常 + * @return 不超过数据库字段长度的错误摘要 + */ + static String summarizeImportError(Exception ex) { + String message = ex == null ? null : ex.getMessage(); + if (message == null || message.isBlank()) { + message = ex == null ? "Excel 导入失败" : ex.getClass().getSimpleName(); + } + int messageLength = message.codePointCount(0, message.length()); + if (messageLength <= MAX_IMPORT_JOB_ERROR_SUMMARY_LENGTH) { + return message; + } + + int markerLength = ERROR_SUMMARY_TRUNCATION_MARKER.codePointCount(0, ERROR_SUMMARY_TRUNCATION_MARKER.length()); + int availableLength = MAX_IMPORT_JOB_ERROR_SUMMARY_LENGTH - markerLength; + int prefixLength = availableLength * 2 / 3; + int suffixLength = availableLength - prefixLength; + return substringByCodePoint(message, 0, prefixLength) + + ERROR_SUMMARY_TRUNCATION_MARKER + + substringByCodePoint(message, messageLength - suffixLength, messageLength); + } + + /** + * 按 Unicode 码点截取字符串，避免截断代理对字符。 + * + * @param value 原始字符串 + * @param beginCodePoint 起始码点索引（包含） + * @param endCodePoint 结束码点索引（不包含） + * @return 截取后的字符串 + */ + private static String substringByCodePoint(String value, int beginCodePoint, int endCodePoint) { + int beginIndex = value.offsetByCodePoints(0, beginCodePoint); + int endIndex = value.offsetByCodePoints(0, endCodePoint); + return value.substring(beginIndex, endIndex); + } + private DatacenterTable resolveTable(DatasetRef datasetRef) { if (datasetRef == null || datasetRef.getTableId() == null) { throw new BusinessException("缺少数据集 tableId"); @@ -870,7 +972,7 @@ public class DatacenterExcelImportServiceImpl implements DatacenterExcelImportSe return tables.stream().anyMatch(table -> tableName.equals(table.getTableName())); } - private String extractWorkbookName(String originalFileName) { + private static String extractWorkbookName(String originalFileName) { if (originalFileName == null || originalFileName.isBlank()) { return "excel_workbook"; } @@ -878,35 +980,166 @@ public class DatacenterExcelImportServiceImpl implements DatacenterExcelImportSe return index > 0 ? originalFileName.substring(0, index) : originalFileName; } + /** + * 解析 Excel 导入后使用的数据源名称。 + * + * @param requestedSourceName 用户填写的数据源名称 + * @param originalFileName 原始文件名 + * @return 去除首尾空白且长度合法的数据源名称 + * @throws BusinessException 名称超过数据库字段长度时抛出 + */ + static String resolveImportSourceName(String requestedSourceName, String originalFileName) { + String resolved = requestedSourceName == null || requestedSourceName.isBlank() + ? extractWorkbookName(originalFileName).trim() + : requestedSourceName.trim(); + if (resolved.codePointCount(0, resolved.length()) > MAX_SOURCE_NAME_LENGTH) { + throw new BusinessException("连接名称不能超过 100 个字符"); + } + return resolved; + } + + /** + * 判断工作簿是否至少包含一个带表头和数据行的工作表。 + * + * @param workbook 待检查的工作簿 + * @param formatter 单元格格式化器 + * @return 存在可导入工作表时返回 {@code true} + */ + static boolean hasImportableSheet(Workbook workbook, DataFormatter formatter) { + if (workbook == null) { + return false; + } + for (int sheetIndex = 0; sheetIndex < workbook.getNumberOfSheets(); sheetIndex++) { + Sheet sheet = workbook.getSheetAt(sheetIndex); + org.apache.poi.ss.usermodel.Row headerRow = sheet.getRow(sheet.getFirstRowNum()); + if (isImportableSheet(sheet, headerRow, formatter)) { + return true; + } + } + return false; + } + + /** + * 判断单个工作表是否包含有效表头和至少一行数据。 + * + * @param sheet 待检查的工作表 + * @param headerRow 首行表头 + * @param formatter 单元格格式化器 + * @return 工作表可导入时返回 {@code true} + */ + private static boolean isImportableSheet( + Sheet sheet, + org.apache.poi.ss.usermodel.Row headerRow, + DataFormatter formatter) { + if (sheet == null || headerRow == null || headerRow.getLastCellNum() <= 0) { + return false; + } + boolean hasHeader = false; + for (int cellIndex = 0; cellIndex < headerRow.getLastCellNum(); cellIndex++) { + if (!formatter.formatCellValue(headerRow.getCell(cellIndex)).isBlank()) { + hasHeader = true; + break; + } + } + if (!hasHeader) { + return false; + } + for (int rowIndex = sheet.getFirstRowNum() + 1; rowIndex <= sheet.getLastRowNum(); rowIndex++) { + org.apache.poi.ss.usermodel.Row row = sheet.getRow(rowIndex); + if (row == null) { + continue; + } + for (int cellIndex = 0; cellIndex < headerRow.getLastCellNum(); cellIndex++) { + if (!formatter.formatCellValue(row.getCell(cellIndex)).isBlank()) { + return true; + } + } + } + return false; + } + + /** + * 获取单元格最终显示值，公式单元格返回计算结果。 + * + * @param cell 单元格 + * @param formatter 单元格格式化器 + * @param formulaEvaluator 公式计算器 + * @return 单元格显示值；空单元格返回空字符串 + */ + static String formatCellValue( + Cell cell, + DataFormatter formatter, + FormulaEvaluator formulaEvaluator) { + if (cell == null) { + return ""; + } + return formatter.formatCellValue(cell, formulaEvaluator); + } + + /** + * 判断文件名是否为支持的 Excel 工作簿格式。 + * + * @param originalFileName 原始上传文件名 + * @return 文件扩展名为 {@code .xls} 或 {@code .xlsx} 时返回 {@code true} + */ + static boolean isSupportedWorkbookFileName(String originalFileName) { + if (originalFileName == null || originalFileName.isBlank()) { + return false; + } + String fileName = originalFileName.trim().toLowerCase(Locale.ROOT); + return fileName.endsWith(".xls") || fileName.endsWith(".xlsx"); + } + private String buildMaterializedTableName(BigInteger sourceId, int sheetIndex) { long snowId = new SnowFlakeIDKeyGenerator().nextId(); return "tb_excel_" + sourceId + "_" + sheetIndex + "_" + snowId; } - private String normalizeIdentifier(String raw) { + /** + * 将 Excel 表头转换为可用于数据库物理字段的安全标识符。 + * + * @param raw 原始表头 + * @return 仅包含 ASCII 字母、数字和下划线的候选字段名 + */ + static String normalizeIdentifier(String raw) { if (raw == null || raw.isBlank()) { - return "value"; + return ""; } - String normalized = raw.trim().toLowerCase(Locale.ROOT).replaceAll("[^a-z0-9_\\u4e00-\\u9fa5]+", "_"); - normalized = normalized.replaceAll("_+", "_"); - if (normalized.isBlank()) { - return "value"; - } - return normalized; + return raw.trim() + .toLowerCase(Locale.ROOT) + .replaceAll("[^a-z0-9]+", "_") + .replaceAll("_+", "_") + .replaceAll("^_+|_+$", ""); } - private String normalizeIdentifier(String raw, int index, Set usedNames) { + /** + * 为 Excel 表头生成唯一且安全的物理字段名。 + * + * @param raw 原始表头 + * @param index 表头从零开始的列索引 + * @param usedNames 当前已使用的物理字段名 + * @return 唯一的安全物理字段名，且不会与系统保留列冲突 + */ + static String normalizeIdentifier(String raw, int index, Set usedNames) { String value = normalizeIdentifier(raw); - if (value.isBlank() || "value".equals(value)) { + if (value.isBlank()) { value = "col_" + (index + 1); } if (Character.isDigit(value.charAt(0))) { value = "col_" + value; } + if (SqlInjectionUtils.isSqlKeyword(value) || SYSTEM_FIELD_NAMES.contains(value)) { + value = "col_" + value; + } + value = value.substring(0, Math.min(value.length(), MAX_IDENTIFIER_LENGTH)); String result = value; int suffix = 1; while (usedNames.contains(result)) { result = value + "_" + suffix++; + if (result.length() > MAX_IDENTIFIER_LENGTH) { + String suffixText = "_" + (suffix - 1); + result = value.substring(0, MAX_IDENTIFIER_LENGTH - suffixText.length()) + suffixText; + } } usedNames.add(result); return result; diff --git a/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/execution/service/impl/DatacenterDatasetQueryServiceImpl.java b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/execution/service/impl/DatacenterDatasetQueryServiceImpl.java index eea5b8a2..fad8cadf 100644 --- a/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/execution/service/impl/DatacenterDatasetQueryServiceImpl.java +++ b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/execution/service/impl/DatacenterDatasetQueryServiceImpl.java @@ -9,6 +9,7 @@ import org.springframework.util.StringUtils; import tech.easyflow.common.web.exceptions.BusinessException; import tech.easyflow.datacenter.connector.DatacenterConnector; import tech.easyflow.datacenter.connector.DatacenterConnectorRegistry; +import tech.easyflow.datacenter.connector.support.AbstractInternalTableConnector; import tech.easyflow.datacenter.entity.DatacenterTable; import tech.easyflow.datacenter.entity.DatacenterTableField; import tech.easyflow.datacenter.execution.model.DatacenterQueryRequest; @@ -189,14 +190,16 @@ public class DatacenterDatasetQueryServiceImpl implements DatacenterDatasetQuery } Map catalogsById = loadCatalogsById(managedTables); - SqlSupportUtils.ResolvedSql resolvedSql = SqlSupportUtils.resolve( - sql, - managedTables.stream() - .map(table -> toManagedSqlTable( - table, catalogsById)) - .toList() - ); DatacenterConnector connector = connectorRegistry.getConnector(source.getSourceType()); + List sqlTables = managedTables.stream() + .map(table -> toManagedSqlTable( + table, catalogsById)) + .toList(); + // 内部连接的 catalog 是逻辑命名空间，底层项目 MySQL 只执行物理表名。 + SqlSupportUtils.ResolvedSql resolvedSql = + connector instanceof AbstractInternalTableConnector + ? SqlSupportUtils.resolveInternalMysql(sql, sqlTables) + : SqlSupportUtils.resolve(sql, sqlTables); return new ResolvedSqlQuery( source, connector, diff --git a/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/meta/entity/DatacenterCatalog.java b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/meta/entity/DatacenterCatalog.java index c6301357..45b838f4 100644 --- a/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/meta/entity/DatacenterCatalog.java +++ b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/meta/entity/DatacenterCatalog.java @@ -1,5 +1,7 @@ package tech.easyflow.datacenter.meta.entity; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; +import com.fasterxml.jackson.databind.ser.std.ToStringSerializer; import com.mybatisflex.annotation.Column; import com.mybatisflex.annotation.Id; import com.mybatisflex.annotation.KeyType; @@ -42,6 +44,7 @@ public class DatacenterCatalog extends DateEntity implements Serializable { @Column(comment = "修改人") private BigInteger modifiedBy; + @JsonSerialize(using = ToStringSerializer.class) public BigInteger getId() { return id; } public void setId(BigInteger id) { this.id = id; } public BigInteger getDeptId() { return deptId; } diff --git a/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/meta/entity/DatacenterImportJob.java b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/meta/entity/DatacenterImportJob.java index 9669663b..c05d18c1 100644 --- a/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/meta/entity/DatacenterImportJob.java +++ b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/meta/entity/DatacenterImportJob.java @@ -1,5 +1,7 @@ package tech.easyflow.datacenter.meta.entity; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; +import com.fasterxml.jackson.databind.ser.std.ToStringSerializer; import com.mybatisflex.annotation.Column; import com.mybatisflex.annotation.Id; import com.mybatisflex.annotation.KeyType; @@ -57,16 +59,20 @@ public class DatacenterImportJob extends DateEntity implements Serializable { @Column(comment = "修改人") private BigInteger modifiedBy; + @JsonSerialize(using = ToStringSerializer.class) public BigInteger getId() { return id; } public void setId(BigInteger id) { this.id = id; } public BigInteger getDeptId() { return deptId; } public void setDeptId(BigInteger deptId) { this.deptId = deptId; } public BigInteger getTenantId() { return tenantId; } public void setTenantId(BigInteger tenantId) { this.tenantId = tenantId; } + @JsonSerialize(using = ToStringSerializer.class) public BigInteger getSourceId() { return sourceId; } public void setSourceId(BigInteger sourceId) { this.sourceId = sourceId; } + @JsonSerialize(using = ToStringSerializer.class) public BigInteger getCatalogId() { return catalogId; } public void setCatalogId(BigInteger catalogId) { this.catalogId = catalogId; } + @JsonSerialize(using = ToStringSerializer.class) public BigInteger getTableId() { return tableId; } public void setTableId(BigInteger tableId) { this.tableId = tableId; } public String getJobType() { return jobType; } diff --git a/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/meta/entity/DatacenterSource.java b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/meta/entity/DatacenterSource.java index 4ae9782c..01261e5b 100644 --- a/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/meta/entity/DatacenterSource.java +++ b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/meta/entity/DatacenterSource.java @@ -1,5 +1,7 @@ package tech.easyflow.datacenter.meta.entity; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; +import com.fasterxml.jackson.databind.ser.std.ToStringSerializer; import com.mybatisflex.annotation.Column; import com.mybatisflex.annotation.Id; import com.mybatisflex.annotation.KeyType; @@ -70,6 +72,7 @@ public class DatacenterSource extends DateEntity implements Serializable { @Column(comment = "修改人") private BigInteger modifiedBy; + @JsonSerialize(using = ToStringSerializer.class) public BigInteger getId() { return id; } public void setId(BigInteger id) { this.id = id; } public BigInteger getDeptId() { return deptId; } diff --git a/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/meta/model/DatacenterCatalogMeta.java b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/meta/model/DatacenterCatalogMeta.java index 172887f7..e1fe9a3a 100644 --- a/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/meta/model/DatacenterCatalogMeta.java +++ b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/meta/model/DatacenterCatalogMeta.java @@ -1,7 +1,13 @@ package tech.easyflow.datacenter.meta.model; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; +import com.fasterxml.jackson.databind.ser.std.ToStringSerializer; + import java.math.BigInteger; +/** + * 数据中枢目录元数据。 + */ public class DatacenterCatalogMeta { private BigInteger id; private BigInteger sourceId; @@ -9,8 +15,10 @@ public class DatacenterCatalogMeta { private String catalogType; private String catalogDesc; + @JsonSerialize(using = ToStringSerializer.class) public BigInteger getId() { return id; } public void setId(BigInteger id) { this.id = id; } + @JsonSerialize(using = ToStringSerializer.class) public BigInteger getSourceId() { return sourceId; } public void setSourceId(BigInteger sourceId) { this.sourceId = sourceId; } public String getCatalogName() { return catalogName; } diff --git a/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/utils/SqlInjectionUtils.java b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/utils/SqlInjectionUtils.java index 2007f17c..7db3d5dd 100644 --- a/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/utils/SqlInjectionUtils.java +++ b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/utils/SqlInjectionUtils.java @@ -2,20 +2,35 @@ package tech.easyflow.datacenter.utils; import tech.easyflow.common.web.exceptions.BusinessException; -import java.util.Arrays; -import java.util.HashSet; +import java.util.Locale; import java.util.Set; -public class SqlInjectionUtils { +/** + * 数据中心动态 SQL 的安全校验与字面量转义工具。 + */ +public final class SqlInjectionUtils { - private static final Set SQL_KEYWORDS = new HashSet<>(Arrays.asList( + private static final Set SQL_KEYWORDS = Set.of( "select", "insert", "update", "delete", "drop", "alter", "create", "table", "where", "from", "join", "union", "truncate", "execute", - "grant", "revoke", "commit", "rollback" - )); + "grant", "revoke", "commit", "rollback", "order", "group", "by", + "having", "limit", "offset", "as", "on", "into", "values", "index", + "key", "primary", "constraint", "references", "distinct", "case", + "when", "then", "else", "end", "and", "or", "not", "null", "like", + "in", "is", "exists", "between", "procedure", "function", "trigger", + "view", "database", "schema", "column", "add", "rename", "replace", + "show", "describe", "explain", "use", "lock", "unlock" + ); + + private SqlInjectionUtils() { + } /** - * 校验字段或表名 + * 校验动态 SQL 中的字段或表标识符。 + * + * @param identifier 待校验的标识符 + * @return 已校验的标识符 + * @throws BusinessException 标识符为空、过长、包含非法字符或为 SQL 关键字时抛出 */ public static String checkIdentifier(String identifier) { if (identifier == null || identifier.isEmpty()) { @@ -35,34 +50,36 @@ public class SqlInjectionUtils { } /** - * 校验注释 - * 允许的字符包括以下 Unicode 类别或符号： - * \p{L}：任何语言的字母（包括中文、英文、日文等）。 - * \p{N}：任何数字（包括阿拉伯数字 0-9 或其他语言的数字符号）。 - * \p{Zs}：空白分隔符（如空格，但不包括换行符、制表符等）。 - * 标点符号：. , - : ? !（基础标点）。 + * 校验并转义 SQL 字符串字面量中的表或字段备注。 + * + *

Excel 表头和工作表名称可包含常见的中英文标点、单位符号和 Unicode 字符。换行和制表符会 + * 规范为单个空格；其余控制字符会被拒绝。单引号和反斜杠会被转义，避免备注内容破坏动态 DDL 语句。

+ * + * @param comment 原始备注内容 + * @return 可安全拼入单引号 SQL 字符串字面量的备注内容 + * @throws BusinessException 备注过长或包含控制字符时抛出 */ public static String checkComment(String comment) { - if (comment == null) { + if (comment == null || comment.isEmpty()) { return ""; } if (comment.length() > 255) { throw new BusinessException("注释过长"); } - if (!comment.matches("^[\\p{L}\\p{N}\\p{Zs}\\.\\,\\-\\:\\?\\!]+$")) { - throw new BusinessException("包含非法字符"); + String normalizedComment = comment.replaceAll("[\\r\\n\\t]+", " "); + if (normalizedComment.codePoints().anyMatch(Character::isISOControl)) { + throw new BusinessException("备注不能包含控制字符"); } - if (comment.contains("--")) { - throw new BusinessException("包含非法字符！"); - } - if (comment.chars().anyMatch(c -> c <= 31 || c == 127)) { - throw new BusinessException("存在非法字符"); - } - return comment; + return normalizedComment.replace("\\", "\\\\").replace("'", "''"); } - // 检查是否是数据库关键字 + /** + * 判断指定单词是否为受限 SQL 关键字。 + * + * @param word 待判断的单词 + * @return 是 SQL 关键字时返回 {@code true} + */ public static boolean isSqlKeyword(String word) { - return SQL_KEYWORDS.contains(word.toLowerCase()); + return word != null && SQL_KEYWORDS.contains(word.toLowerCase(Locale.ROOT)); } } diff --git a/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/utils/SqlSupportUtils.java b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/utils/SqlSupportUtils.java index 4c7e0d07..e6965f27 100644 --- a/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/utils/SqlSupportUtils.java +++ b/easyflow-modules/easyflow-module-datacenter/src/main/java/tech/easyflow/datacenter/utils/SqlSupportUtils.java @@ -25,6 +25,36 @@ public final class SqlSupportUtils { } public static ResolvedSql resolve(String sql, Collection managedTables) { + return resolve(sql, managedTables, true, false); + } + + /** + * 解析内部物化表 SQL，逻辑目录仅用于白名单匹配，不进入项目 MySQL 的执行 SQL。 + * + * @param sql 逻辑 SQL + * @param managedTables 已接入表 + * @return 已解析的项目 MySQL SQL + */ + public static ResolvedSql resolveInternalMysql( + String sql, + Collection managedTables) { + return resolve(sql, managedTables, false, true); + } + + /** + * 解析并重写只读 SQL。 + * + * @param sql 逻辑 SQL + * @param managedTables 已接入表 + * @param retainCatalog 是否保留目录限定符 + * @param mysqlIdentifierQuotes 是否转换为 MySQL 标识符引号 + * @return 已解析 SQL + */ + private static ResolvedSql resolve( + String sql, + Collection managedTables, + boolean retainCatalog, + boolean mysqlIdentifierQuotes) { String normalizedSql = normalizeSql(sql); Statement statement = parseSingleStatement(normalizedSql); if (!(statement instanceof Select select)) { @@ -54,10 +84,14 @@ public final class SqlSupportUtils { Set logicalTables = new LinkedHashSet<>(); for (Table table : referencedTables) { ManagedTable managedTable = resolveManagedTable(table, byTableName, byCatalogAndTable); - rewriteTable(table, managedTable); + rewriteTable(table, managedTable, retainCatalog); logicalTables.add(renderLogicalTable(managedTable)); } - return new ResolvedSql(select.toString(), new ArrayList<>(logicalTables)); + String executableSql = select.toString(); + if (mysqlIdentifierQuotes) { + executableSql = normalizeMysqlIdentifierQuotes(executableSql); + } + return new ResolvedSql(executableSql, new ArrayList<>(logicalTables)); } private static Statement parseSingleStatement(String sql) { @@ -102,9 +136,73 @@ public final class SqlSupportUtils { return matches.get(0); } - private static void rewriteTable(Table table, ManagedTable managedTable) { + /** + * 将逻辑表替换为物理表。 + * + * @param table SQL 表节点 + * @param managedTable 已接入表 + * @param retainCatalog 是否保留目录限定符 + */ + private static void rewriteTable( + Table table, + ManagedTable managedTable, + boolean retainCatalog) { table.setName(managedTable.getPhysicalTableName()); - table.setSchemaName(trimToNull(managedTable.getCatalogName())); + table.setSchemaName(retainCatalog + ? trimToNull(managedTable.getCatalogName()) + : null); + } + + /** + * 将标准 SQL 双引号标识符转换为 MySQL 反引号，同时保留字符串字面量内容。 + * + * @param sql 已解析 SQL + * @return MySQL 可执行 SQL + */ + private static String normalizeMysqlIdentifierQuotes(String sql) { + StringBuilder normalized = new StringBuilder(sql.length()); + boolean singleQuoted = false; + boolean doubleQuotedIdentifier = false; + for (int index = 0; index < sql.length(); index++) { + char current = sql.charAt(index); + if (singleQuoted) { + normalized.append(current); + if (current == '\\' && index + 1 < sql.length()) { + normalized.append(sql.charAt(++index)); + } else if (current == '\'' && index + 1 < sql.length() + && sql.charAt(index + 1) == '\'') { + normalized.append(sql.charAt(++index)); + } else if (current == '\'') { + singleQuoted = false; + } + continue; + } + if (doubleQuotedIdentifier) { + if (current == '"' && index + 1 < sql.length() + && sql.charAt(index + 1) == '"') { + normalized.append('"'); + index++; + } else if (current == '"') { + normalized.append('`'); + doubleQuotedIdentifier = false; + } else if (current == '`') { + normalized.append("``"); + } else { + normalized.append(current); + } + continue; + } + if (current == '\'') { + singleQuoted = true; + normalized.append(current); + } else if (current == '"') { + doubleQuotedIdentifier = true; + normalized.append('`'); + } else { + normalized.append(current); + } + } + return normalized.toString(); } private static String renderLogicalTable(ManagedTable managedTable) { diff --git a/easyflow-modules/easyflow-module-datacenter/src/test/java/tech/easyflow/datacenter/connector/support/AbstractInternalTableConnectorTest.java b/easyflow-modules/easyflow-module-datacenter/src/test/java/tech/easyflow/datacenter/connector/support/AbstractInternalTableConnectorTest.java new file mode 100644 index 00000000..ed95ed26 --- /dev/null +++ b/easyflow-modules/easyflow-module-datacenter/src/test/java/tech/easyflow/datacenter/connector/support/AbstractInternalTableConnectorTest.java @@ -0,0 +1,98 @@ +package tech.easyflow.datacenter.connector.support; + +import com.mybatisflex.core.row.Row; +import com.mybatisflex.core.paginate.Page; +import com.mybatisflex.core.query.QueryWrapper; +import com.mybatisflex.core.row.Db; +import org.junit.Assert; +import org.junit.Test; +import org.mockito.MockedStatic; +import org.mockito.Mockito; +import tech.easyflow.datacenter.entity.DatacenterTable; +import tech.easyflow.datacenter.execution.model.DatacenterQueryRequest; +import tech.easyflow.datacenter.meta.enums.DatacenterSourceType; + +import java.math.BigInteger; +import java.util.List; +import java.util.Set; +import java.util.concurrent.atomic.AtomicReference; + +/** + * {@link AbstractInternalTableConnector} 内部查询结果规范化测试。 + */ +public class AbstractInternalTableConnectorTest { + + /** + * 验证 JDBC 返回的大写列标签会转换为数据集字段使用的小写名称。 + */ + @Test + public void shouldNormalizeMysqlColumnLabelsToLowerCase() { + Row row = new Row(); + row.put("COL_ID", "deepseek-v4-pro"); + row.put("TOKEN", BigInteger.valueOf(3)); + + AbstractInternalTableConnector.normalizeRows(List.of(row)); + + Assert.assertEquals( + List.of("col_id", "token"), + List.copyOf(row.keySet())); + Assert.assertEquals("deepseek-v4-pro", row.get("col_id")); + Assert.assertEquals("3", row.get("token")); + } + + /** + * 验证统计查询修改投影后，不会污染后续的分页查询包装器。 + */ + @Test + public void shouldUseIndependentQueryWrapperForPaginationAfterCount() { + DatacenterTable table = new DatacenterTable(); + table.setActualTable("preview_table"); + DatacenterQueryRequest request = new DatacenterQueryRequest(); + request.setPageNumber(1L); + request.setPageSize(10L); + + Row record = new Row(); + record.put("MODEL_ID", "deepseek-v4-pro"); + AtomicReference countWrapper = new AtomicReference<>(); + AtomicReference pageWrapper = new AtomicReference<>(); + try (MockedStatic db = Mockito.mockStatic(Db.class)) { + db.when(() -> Db.selectCountByQuery( + Mockito.eq("preview_table"), + Mockito.any(QueryWrapper.class))) + .thenAnswer(invocation -> { + QueryWrapper wrapper = invocation.getArgument(1); + countWrapper.set(wrapper); + // 模拟 MyBatis-Flex 为 count 查询自动追加 COUNT(*) 投影的行为。 + wrapper.select("COUNT(*)"); + return 1L; + }); + db.when(() -> Db.paginate( + Mockito.eq("preview_table"), + Mockito.any(Page.class), + Mockito.any(QueryWrapper.class))) + .thenAnswer(invocation -> { + pageWrapper.set(invocation.getArgument(2)); + return new Page<>(List.of(record), 1L, 10L, 1L); + }); + + Page page = new TestInternalTableConnector().queryPage( + null, table, request); + + Assert.assertNotSame(countWrapper.get(), pageWrapper.get()); + Assert.assertEquals("deepseek-v4-pro", page.getRecords().get(0).get("model_id")); + } + } + + /** + * 供内部动态表查询测试使用的最小连接器实现。 + */ + private static class TestInternalTableConnector extends AbstractInternalTableConnector { + + /** + * 创建测试连接器。 + */ + private TestInternalTableConnector() { + super(DatacenterSourceType.EXCEL, Set.of(), null); + } + } +} diff --git a/easyflow-modules/easyflow-module-datacenter/src/test/java/tech/easyflow/datacenter/connector/support/DatacenterConnectorExceptionSupportTest.java b/easyflow-modules/easyflow-module-datacenter/src/test/java/tech/easyflow/datacenter/connector/support/DatacenterConnectorExceptionSupportTest.java new file mode 100644 index 00000000..d60c1a25 --- /dev/null +++ b/easyflow-modules/easyflow-module-datacenter/src/test/java/tech/easyflow/datacenter/connector/support/DatacenterConnectorExceptionSupportTest.java @@ -0,0 +1,51 @@ +package tech.easyflow.datacenter.connector.support; + +import org.junit.Assert; +import org.junit.Test; +import tech.easyflow.common.web.exceptions.BusinessException; + +import java.sql.SQLException; + +/** + * {@link DatacenterConnectorExceptionSupport} 异常包装测试。 + */ +public class DatacenterConnectorExceptionSupportTest { + + /** + * 验证通用访问错误保留底层 SQL 异常。 + */ + @Test + public void wrapAccessExceptionShouldPreserveSqlCause() { + SQLException cause = new SQLException( + "You have an error in your SQL syntax", + "42000"); + + BusinessException wrapped = + DatacenterConnectorExceptionSupport.wrapAccessException( + "SQL 流式查询失败", + cause); + + Assert.assertEquals("SQL 流式查询失败", wrapped.getMessage()); + Assert.assertSame(cause, wrapped.getCause()); + } + + /** + * 验证连接不可用错误保留底层 SQL 异常并返回安全文案。 + */ + @Test + public void wrapAccessExceptionShouldPreserveUnavailableCause() { + SQLException cause = new SQLException( + "Unknown database 'missing'", + "42000"); + + BusinessException wrapped = + DatacenterConnectorExceptionSupport.wrapAccessException( + "SQL 流式查询失败", + cause); + + Assert.assertEquals( + DatacenterConnectorExceptionSupport.SOURCE_UNAVAILABLE_MESSAGE, + wrapped.getMessage()); + Assert.assertSame(cause, wrapped.getCause()); + } +} diff --git a/easyflow-modules/easyflow-module-datacenter/src/test/java/tech/easyflow/datacenter/excel/service/impl/DatacenterExcelImportServiceImplTest.java b/easyflow-modules/easyflow-module-datacenter/src/test/java/tech/easyflow/datacenter/excel/service/impl/DatacenterExcelImportServiceImplTest.java new file mode 100644 index 00000000..f378f73e --- /dev/null +++ b/easyflow-modules/easyflow-module-datacenter/src/test/java/tech/easyflow/datacenter/excel/service/impl/DatacenterExcelImportServiceImplTest.java @@ -0,0 +1,177 @@ +package tech.easyflow.datacenter.excel.service.impl; + +import org.junit.Assert; +import org.junit.Test; +import org.apache.poi.hssf.usermodel.HSSFWorkbook; +import org.apache.poi.ss.usermodel.DataFormatter; +import org.apache.poi.ss.usermodel.Workbook; +import org.apache.poi.ss.usermodel.WorkbookFactory; +import org.apache.poi.xssf.usermodel.XSSFWorkbook; +import tech.easyflow.common.web.exceptions.BusinessException; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.util.HashSet; +import java.util.Set; + +/** + * {@link DatacenterExcelImportServiceImpl} 的 Excel 表头字段名处理测试。 + */ +public class DatacenterExcelImportServiceImplTest { + + /** + * 验证包含中文和常见标点的表头会转换为安全的物理字段名。 + */ + @Test + public void shouldNormalizeExcelHeaderToSafeIdentifier() { + Assert.assertEquals("", DatacenterExcelImportServiceImpl.normalizeIdentifier("金额（万元）")); + Assert.assertEquals("amount_cny", DatacenterExcelImportServiceImpl.normalizeIdentifier("Amount（CNY）")); + Assert.assertEquals("model_version", DatacenterExcelImportServiceImpl.normalizeIdentifier("模型 / Model Version")); + } + + /** + * 验证非 ASCII 表头、SQL 关键字和重复表头均会生成有效且唯一的字段名。 + */ + @Test + public void shouldGenerateSafeUniqueExcelFieldNames() { + Set usedNames = new HashSet<>(); + + Assert.assertEquals("col_1", DatacenterExcelImportServiceImpl.normalizeIdentifier("金额（万元）", 0, usedNames)); + Assert.assertEquals("col_select", DatacenterExcelImportServiceImpl.normalizeIdentifier("select", 1, usedNames)); + Assert.assertEquals("col_id", DatacenterExcelImportServiceImpl.normalizeIdentifier("模型ID", 2, usedNames)); + Assert.assertEquals("col_id_1", DatacenterExcelImportServiceImpl.normalizeIdentifier("部门ID", 3, usedNames)); + Assert.assertEquals("amount", DatacenterExcelImportServiceImpl.normalizeIdentifier("Amount", 4, usedNames)); + Assert.assertEquals("amount_1", DatacenterExcelImportServiceImpl.normalizeIdentifier("Amount", 5, usedNames)); + Assert.assertEquals("col_order", DatacenterExcelImportServiceImpl.normalizeIdentifier("Order", 6, usedNames)); + Assert.assertEquals("col_group", DatacenterExcelImportServiceImpl.normalizeIdentifier("Group", 7, usedNames)); + } + + /** + * 验证超长异常摘要会保留首尾信息并限制在数据库字段长度内。 + */ + @Test + public void shouldTruncateLongImportErrorSummary() { + String message = "起始原因:" + "a".repeat(1200) + ":最终原因"; + + String summary = DatacenterExcelImportServiceImpl.summarizeImportError(new IllegalArgumentException(message)); + + Assert.assertEquals(1000, summary.codePointCount(0, summary.length())); + Assert.assertTrue(summary.startsWith("起始原因:")); + Assert.assertTrue(summary.contains("…（错误摘要已截断）…")); + Assert.assertTrue(summary.endsWith(":最终原因")); + } + + /** + * 验证仅允许 xls 和 xlsx 文件名通过服务端扩展名校验。 + */ + @Test + public void shouldAcceptOnlySupportedWorkbookExtensions() { + Assert.assertTrue(DatacenterExcelImportServiceImpl.isSupportedWorkbookFileName("预算表.XLS")); + Assert.assertTrue(DatacenterExcelImportServiceImpl.isSupportedWorkbookFileName("预算表.xlsx")); + Assert.assertFalse(DatacenterExcelImportServiceImpl.isSupportedWorkbookFileName("预算表.xlsm")); + Assert.assertFalse(DatacenterExcelImportServiceImpl.isSupportedWorkbookFileName("预算表.csv")); + } + + /** + * 验证当前 WorkbookFactory 能解析 xls 和 xlsx 工作簿。 + * + * @throws IOException 工作簿序列化或读取失败时抛出 + */ + @Test + public void shouldParseXlsAndXlsxWorkbooks() throws IOException { + assertWorkbookCanBeParsed(new HSSFWorkbook()); + assertWorkbookCanBeParsed(new XSSFWorkbook()); + } + + /** + * 验证自定义连接名称优先使用，空名称回退到文件名。 + */ + @Test + public void shouldResolveImportSourceName() { + Assert.assertEquals( + "模型预算", + DatacenterExcelImportServiceImpl.resolveImportSourceName(" 模型预算 ", "原文件.xlsx") + ); + Assert.assertEquals( + "原文件", + DatacenterExcelImportServiceImpl.resolveImportSourceName(" ", "原文件.xlsx") + ); + } + + /** + * 验证连接名称超过数据库字段长度时给出业务错误。 + */ + @Test + public void shouldRejectTooLongImportSourceName() { + BusinessException error = Assert.assertThrows( + BusinessException.class, + () -> DatacenterExcelImportServiceImpl.resolveImportSourceName("长".repeat(101), "原文件.xlsx") + ); + + Assert.assertEquals("连接名称不能超过 100 个字符", error.getMessage()); + } + + /** + * 验证空工作簿、无表头和只有表头的工作表均不可导入。 + */ + @Test + public void shouldRejectWorkbookWithoutHeaderAndData() { + DataFormatter formatter = new DataFormatter(); + try (Workbook workbook = new XSSFWorkbook()) { + Assert.assertFalse(DatacenterExcelImportServiceImpl.hasImportableSheet(workbook, formatter)); + + var sheet = workbook.createSheet("空表"); + sheet.createRow(0).createCell(0).setCellValue(""); + Assert.assertFalse(DatacenterExcelImportServiceImpl.hasImportableSheet(workbook, formatter)); + + sheet.getRow(0).getCell(0).setCellValue("模型ID"); + Assert.assertFalse(DatacenterExcelImportServiceImpl.hasImportableSheet(workbook, formatter)); + + sheet.createRow(1).createCell(0).setCellValue("deepseek-v4-pro"); + Assert.assertTrue(DatacenterExcelImportServiceImpl.hasImportableSheet(workbook, formatter)); + } catch (IOException ex) { + Assert.fail(ex.getMessage()); + } + } + + /** + * 验证公式单元格导入计算结果，不暴露公式文本。 + */ + @Test + public void shouldImportCalculatedFormulaValue() { + DataFormatter formatter = new DataFormatter(); + try (Workbook workbook = new XSSFWorkbook()) { + var row = workbook.createSheet("公式").createRow(0); + row.createCell(0).setCellValue(3); + var formulaCell = row.createCell(1); + formulaCell.setCellFormula("A1*2"); + + String value = DatacenterExcelImportServiceImpl.formatCellValue( + formulaCell, + formatter, + workbook.getCreationHelper().createFormulaEvaluator() + ); + + Assert.assertEquals("6", value); + } catch (IOException ex) { + Assert.fail(ex.getMessage()); + } + } + + /** + * 将工作簿序列化后再交给导入使用的解析器读取。 + * + * @param workbook 待验证的工作簿 + * @throws IOException 工作簿序列化或读取失败时抛出 + */ + private void assertWorkbookCanBeParsed(Workbook workbook) throws IOException { + workbook.createSheet("Sheet1"); + try (workbook; ByteArrayOutputStream outputStream = new ByteArrayOutputStream()) { + workbook.write(outputStream); + try (Workbook parsedWorkbook = WorkbookFactory.create(new ByteArrayInputStream(outputStream.toByteArray()))) { + Assert.assertEquals(1, parsedWorkbook.getNumberOfSheets()); + } + } + } +} diff --git a/easyflow-modules/easyflow-module-datacenter/src/test/java/tech/easyflow/datacenter/execution/service/impl/DatacenterDatasetQueryServiceImplTest.java b/easyflow-modules/easyflow-module-datacenter/src/test/java/tech/easyflow/datacenter/execution/service/impl/DatacenterDatasetQueryServiceImplTest.java index 2dabfbea..86db4fb0 100644 --- a/easyflow-modules/easyflow-module-datacenter/src/test/java/tech/easyflow/datacenter/execution/service/impl/DatacenterDatasetQueryServiceImplTest.java +++ b/easyflow-modules/easyflow-module-datacenter/src/test/java/tech/easyflow/datacenter/execution/service/impl/DatacenterDatasetQueryServiceImplTest.java @@ -3,13 +3,17 @@ package tech.easyflow.datacenter.execution.service.impl; import com.mybatisflex.core.row.Row; import org.junit.Assert; import org.junit.Test; +import org.mockito.ArgumentCaptor; import org.mockito.ArgumentMatchers; import org.mockito.Mockito; import tech.easyflow.datacenter.connector.DatacenterConnector; import tech.easyflow.datacenter.connector.DatacenterConnectorRegistry; +import tech.easyflow.datacenter.connector.support.AbstractInternalTableConnector; import tech.easyflow.datacenter.entity.DatacenterTable; import tech.easyflow.datacenter.execution.model.DatacenterSqlQueryRequest; import tech.easyflow.datacenter.execution.model.DatasetRef; +import tech.easyflow.datacenter.mapper.DatacenterCatalogMapper; +import tech.easyflow.datacenter.meta.entity.DatacenterCatalog; import tech.easyflow.datacenter.meta.entity.DatacenterSource; import tech.easyflow.datacenter.meta.service.DatacenterDatasetRegistryService; @@ -101,6 +105,76 @@ public class DatacenterDatasetQueryServiceImplTest { ArgumentMatchers.any()); } + /** + * 验证内部 Excel 查询移除逻辑目录并转换 MySQL 标识符引号。 + * + * @throws Exception 测试依赖注入失败时抛出 + */ + @Test + public void queryBySqlShouldNormalizeInternalMysqlSql() + throws Exception { + BigInteger sourceId = BigInteger.valueOf(3001L); + BigInteger catalogId = BigInteger.valueOf(3002L); + DatacenterSource source = new DatacenterSource(); + source.setId(sourceId); + source.setSourceType("EXCEL"); + DatacenterTable table = new DatacenterTable(); + table.setId(BigInteger.valueOf(3003L)); + table.setSourceId(sourceId); + table.setCatalogId(catalogId); + table.setTableName("Sheet1"); + table.setMaterializedTable("tb_excel_budget"); + + DatacenterCatalog catalog = new DatacenterCatalog(); + catalog.setId(catalogId); + catalog.setCatalogName("ama 实验基线模型预算"); + DatacenterDatasetRegistryService registry = + Mockito.mock(DatacenterDatasetRegistryService.class); + Mockito.when(registry.getSourceRequired(sourceId)) + .thenReturn(source); + Mockito.when(registry.listManagedTables(sourceId, null)) + .thenReturn(List.of(table)); + DatacenterCatalogMapper catalogMapper = + Mockito.mock(DatacenterCatalogMapper.class); + Mockito.when(catalogMapper.selectListByQuery( + ArgumentMatchers.any())) + .thenReturn(List.of(catalog)); + AbstractInternalTableConnector connector = + Mockito.mock(AbstractInternalTableConnector.class); + DatacenterConnectorRegistry connectors = + Mockito.mock(DatacenterConnectorRegistry.class); + Mockito.when(connectors.getConnector("EXCEL")) + .thenReturn(connector); + + DatacenterDatasetQueryServiceImpl service = + new DatacenterDatasetQueryServiceImpl(); + setField(service, "registryService", registry); + setField(service, "connectorRegistry", connectors); + setField(service, "catalogMapper", catalogMapper); + DatacenterSqlQueryRequest request = + new DatacenterSqlQueryRequest(); + DatasetRef datasetRef = new DatasetRef(); + datasetRef.setSourceId(sourceId); + request.setDatasetRef(datasetRef); + request.setSql(""" + SELECT "token", "token_1" + FROM "ama 实验基线模型预算"."Sheet1" + WHERE "col_id" = 'deepseek-v4-pro' + """); + + service.queryBySql(request); + + ArgumentCaptor sqlCaptor = + ArgumentCaptor.forClass(String.class); + Mockito.verify(connector).queryBySql( + ArgumentMatchers.eq(source), + sqlCaptor.capture()); + Assert.assertEquals( + "SELECT `token`, `token_1` FROM tb_excel_budget " + + "WHERE `col_id` = 'deepseek-v4-pro'", + sqlCaptor.getValue()); + } + /** * 创建测试数据行。 * diff --git a/easyflow-modules/easyflow-module-datacenter/src/test/java/tech/easyflow/datacenter/meta/entity/DatacenterIdJsonSerializationTest.java b/easyflow-modules/easyflow-module-datacenter/src/test/java/tech/easyflow/datacenter/meta/entity/DatacenterIdJsonSerializationTest.java new file mode 100644 index 00000000..6e8de6cf --- /dev/null +++ b/easyflow-modules/easyflow-module-datacenter/src/test/java/tech/easyflow/datacenter/meta/entity/DatacenterIdJsonSerializationTest.java @@ -0,0 +1,62 @@ +package tech.easyflow.datacenter.meta.entity; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.junit.Assert; +import org.junit.Test; +import tech.easyflow.datacenter.entity.DatacenterTable; +import tech.easyflow.datacenter.meta.model.DatacenterCatalogMeta; + +import java.math.BigInteger; + +/** + * 数据中枢雪花 ID 的 HTTP JSON 序列化测试。 + */ +public class DatacenterIdJsonSerializationTest { + + private static final BigInteger UNSAFE_JAVASCRIPT_INTEGER = new BigInteger("9007199254740993"); + private final ObjectMapper objectMapper = new ObjectMapper(); + + /** + * 验证连接、目录和表的雪花 ID 会输出为字符串，避免 JavaScript 精度丢失。 + * + * @throws Exception JSON 序列化失败时抛出 + */ + @Test + public void shouldSerializeDatacenterSnowflakeIdsAsStrings() throws Exception { + DatacenterSource source = new DatacenterSource(); + source.setId(UNSAFE_JAVASCRIPT_INTEGER); + assertTextualId(source, "id"); + + DatacenterCatalog catalog = new DatacenterCatalog(); + catalog.setId(UNSAFE_JAVASCRIPT_INTEGER); + assertTextualId(catalog, "id"); + + DatacenterCatalogMeta catalogMeta = new DatacenterCatalogMeta(); + catalogMeta.setId(UNSAFE_JAVASCRIPT_INTEGER); + catalogMeta.setSourceId(UNSAFE_JAVASCRIPT_INTEGER); + assertTextualId(catalogMeta, "id"); + assertTextualId(catalogMeta, "sourceId"); + + DatacenterTable table = new DatacenterTable(); + table.setId(UNSAFE_JAVASCRIPT_INTEGER); + table.setSourceId(UNSAFE_JAVASCRIPT_INTEGER); + table.setCatalogId(UNSAFE_JAVASCRIPT_INTEGER); + assertTextualId(table, "id"); + assertTextualId(table, "sourceId"); + assertTextualId(table, "catalogId"); + } + + /** + * 验证给定属性被输出为精确的文本 ID。 + * + * @param value 待序列化对象 + * @param fieldName ID 属性名 + * @throws Exception JSON 序列化失败时抛出 + */ + private void assertTextualId(Object value, String fieldName) throws Exception { + JsonNode node = objectMapper.readTree(objectMapper.writeValueAsString(value)).path(fieldName); + Assert.assertTrue(fieldName + " 应为 JSON 字符串", node.isTextual()); + Assert.assertEquals(UNSAFE_JAVASCRIPT_INTEGER.toString(), node.asText()); + } +} diff --git a/easyflow-modules/easyflow-module-datacenter/src/test/java/tech/easyflow/datacenter/utils/SqlInjectionUtilsTest.java b/easyflow-modules/easyflow-module-datacenter/src/test/java/tech/easyflow/datacenter/utils/SqlInjectionUtilsTest.java new file mode 100644 index 00000000..7174f975 --- /dev/null +++ b/easyflow-modules/easyflow-module-datacenter/src/test/java/tech/easyflow/datacenter/utils/SqlInjectionUtilsTest.java @@ -0,0 +1,41 @@ +package tech.easyflow.datacenter.utils; + +import org.junit.Assert; +import org.junit.Test; +import tech.easyflow.common.web.exceptions.BusinessException; + +/** + * {@link SqlInjectionUtils} 的单元测试。 + */ +public class SqlInjectionUtilsTest { + + /** + * 验证常见 Excel 表头标点能够作为数据库备注使用。 + */ + @Test + public void shouldEscapeCommonExcelHeaderCharacters() { + String comment = "金额（万元）/预算\n50% \\ '含税'"; + + Assert.assertEquals("金额（万元）/预算 50% \\\\ ''含税''", SqlInjectionUtils.checkComment(comment)); + } + + /** + * 验证控制字符仍会被拒绝，避免动态 DDL 出现不可见内容。 + */ + @Test(expected = BusinessException.class) + public void shouldRejectControlCharacterInComment() { + SqlInjectionUtils.checkComment("金额\u0000"); + } + + /** + * 验证常见查询保留字能够被大小写无关地识别。 + */ + @Test + public void shouldRecognizeCommonQueryKeywords() { + Assert.assertTrue(SqlInjectionUtils.isSqlKeyword("Order")); + Assert.assertTrue(SqlInjectionUtils.isSqlKeyword("GROUP")); + Assert.assertTrue(SqlInjectionUtils.isSqlKeyword("limit")); + Assert.assertFalse(SqlInjectionUtils.isSqlKeyword("model_name")); + Assert.assertFalse(SqlInjectionUtils.isSqlKeyword(null)); + } +} diff --git a/easyflow-modules/easyflow-module-datacenter/src/test/java/tech/easyflow/datacenter/utils/SqlSupportUtilsTest.java b/easyflow-modules/easyflow-module-datacenter/src/test/java/tech/easyflow/datacenter/utils/SqlSupportUtilsTest.java new file mode 100644 index 00000000..2673cb13 --- /dev/null +++ b/easyflow-modules/easyflow-module-datacenter/src/test/java/tech/easyflow/datacenter/utils/SqlSupportUtilsTest.java @@ -0,0 +1,51 @@ +package tech.easyflow.datacenter.utils; + +import org.junit.Assert; +import org.junit.Test; + +import java.util.List; + +/** + * {@link SqlSupportUtils} 内部 MySQL SQL 重写测试。 + */ +public class SqlSupportUtilsTest { + + /** + * 验证逻辑目录只用于表匹配，执行 SQL 使用内部物理表。 + */ + @Test + public void resolveInternalMysqlShouldDropLogicalCatalog() { + SqlSupportUtils.ResolvedSql resolved = SqlSupportUtils.resolveInternalMysql( + """ + SELECT "token", "token_1" + FROM "ama 实验基线模型预算"."Sheet1" + WHERE "col_id" = 'deepseek-v4-pro' + """, + List.of(new SqlSupportUtils.ManagedTable( + "ama 实验基线模型预算", + "Sheet1", + "tb_excel_budget"))); + + Assert.assertEquals( + "SELECT `token`, `token_1` FROM tb_excel_budget " + + "WHERE `col_id` = 'deepseek-v4-pro'", + resolved.getExecutableSql()); + } + + /** + * 验证字符串字面量中的双引号不会被当成标识符转换。 + */ + @Test + public void resolveInternalMysqlShouldPreserveStringLiteralQuotes() { + SqlSupportUtils.ResolvedSql resolved = SqlSupportUtils.resolveInternalMysql( + "SELECT \"token\" FROM \"Sheet1\" WHERE \"col_id\" = 'deep\"seek'", + List.of(new SqlSupportUtils.ManagedTable( + null, + "Sheet1", + "tb_excel_budget"))); + + Assert.assertEquals( + "SELECT `token` FROM tb_excel_budget WHERE `col_id` = 'deep\"seek'", + resolved.getExecutableSql()); + } +}