fix: 完善数据中枢导入与查询链路

- 支持常见 Excel 表头、工作簿校验及数据可靠落库

- 修复大整数 ID 精度和逻辑表 SQL 解析问题

- 为查询数据节点注入结构化上下文并兼容 SQL 代码块
This commit is contained in:
2026-08-03 11:39:14 +08:00
parent e3228837f4
commit 19dac5146c
26 changed files with 1399 additions and 82 deletions

View File

@@ -16,20 +16,25 @@ import tech.easyflow.datacenter.utils.SqlInjectionUtils;
import java.math.BigInteger;
import java.util.Date;
import java.util.List;
import java.util.Objects;
@Component("defaultDbHandleService")
public class DefaultDbHandleService extends DbHandleService {
private static final Logger log = LoggerFactory.getLogger(DefaultDbHandleService.class);
/**
* 创建指定的物理数据表。
*
* @param table 包含表结构和字段定义的数据表
*/
@Override
public void createTable(DatacenterTable table) {
// 设置为 [tb_dynamic_表名_tableId] 的格式
String actualTable = table.getActualTable();
SqlInjectionUtils.checkIdentifier(actualTable);
// 表注释
String tableDesc = table.getTableDesc();
SqlInjectionUtils.checkComment(tableDesc);
String tableDesc = SqlInjectionUtils.checkComment(table.getTableDesc());
List<DatacenterTableField> fields = table.getFields();
StringBuilder sql = new StringBuilder("CREATE TABLE " + actualTable + " (");
@@ -56,15 +61,21 @@ public class DefaultDbHandleService extends DbHandleService {
Db.selectObject(sql.toString());
}
/**
* 更新数据表的可修改元数据。
*
* @param table 待更新的数据表
* @param record 已持久化的数据表记录
*/
@Override
public void updateTable(DatacenterTable table, DatacenterTable record) {
String tableDesc = table.getTableDesc();
SqlInjectionUtils.checkComment(tableDesc);
String escapedTableDesc = SqlInjectionUtils.checkComment(tableDesc);
String actualTable = record.getActualTable();
// 只允许改表备注
if (!tableDesc.equals(record.getTableDesc())) {
if (!Objects.equals(tableDesc, record.getTableDesc())) {
String sql = "ALTER TABLE `" + actualTable + "` "
+ "COMMENT '" + tableDesc + "';";
+ "COMMENT '" + escapedTableDesc + "';";
log.info("修改表备注语句 >>> {}", sql);
Db.selectObject(sql);
}
@@ -95,13 +106,18 @@ public class DefaultDbHandleService extends DbHandleService {
return "text";
}
/**
* 为物理数据表新增字段。
*
* @param entity 目标数据表
* @param field 待新增字段
*/
@Override
public void addField(DatacenterTable entity, DatacenterTableField field) {
String fieldName = field.getFieldName();
SqlInjectionUtils.checkIdentifier(fieldName);
String fieldDesc = field.getFieldDesc();
SqlInjectionUtils.checkComment(fieldDesc);
String fieldDesc = SqlInjectionUtils.checkComment(field.getFieldDesc());
Integer fieldType = field.getFieldType();
Integer required = field.getRequired();
@@ -125,6 +141,13 @@ public class DefaultDbHandleService extends DbHandleService {
Db.selectObject(sql);
}
/**
* 更新物理数据表中的字段定义。
*
* @param entity 目标数据表
* @param fieldRecord 已持久化的字段记录
* @param field 待更新字段
*/
@Override
public void updateField(DatacenterTable entity, DatacenterTableField fieldRecord, DatacenterTableField field) {
String actualTable = entity.getActualTable();
@@ -135,10 +158,10 @@ public class DefaultDbHandleService extends DbHandleService {
SqlInjectionUtils.checkIdentifier(fieldName);
// 字段描述
String fieldDesc = field.getFieldDesc();
SqlInjectionUtils.checkComment(fieldDesc);
String escapedFieldDesc = SqlInjectionUtils.checkComment(fieldDesc);
String nullable = required == 1 ? "NOT NULL " : "NULL ";
String desc = "COMMENT '" + fieldDesc + "';";
String desc = "COMMENT '" + escapedFieldDesc + "';";
boolean isUpdate = false;
String handleType = "MODIFY COLUMN `" + fieldRecord.getFieldName() + "` ";

View File

@@ -86,19 +86,34 @@ public abstract class AbstractInternalTableConnector implements DatacenterConnec
@Override
public Page<Row> queryPage(DatacenterSource source, DatacenterTable table, DatacenterQueryRequest request) {
String actualTable = resolveTableName(table);
QueryWrapper wrapper = QueryWrapper.create();
if (StrUtil.isNotBlank(request.getWhere())) {
wrapper.where(request.getWhere());
}
long count = Db.selectCountByQuery(actualTable, wrapper);
long count = Db.selectCountByQuery(
actualTable, createQueryWrapper(request.getWhere()));
if (count == 0) {
return new Page<>(new ArrayList<>(), request.getPageNumber(), request.getPageSize(), count);
}
Page<Row> page = Db.paginate(actualTable, new Page<>(request.getPageNumber(), request.getPageSize(), count), wrapper);
// selectCountByQuery 会把无投影的 QueryWrapper 改为 COUNT(*),分页查询必须使用独立实例。
Page<Row> page = Db.paginate(
actualTable,
new Page<>(request.getPageNumber(), request.getPageSize(), count),
createQueryWrapper(request.getWhere()));
normalizeRows(page.getRecords());
return page;
}
/**
* 创建用于动态表查询的独立条件包装器。
*
* @param where 已校验的筛选表达式
* @return 新建的查询条件包装器
*/
static QueryWrapper createQueryWrapper(String where) {
QueryWrapper wrapper = QueryWrapper.create();
if (StrUtil.isNotBlank(where)) {
wrapper.where(where);
}
return wrapper;
}
@Override
public List<Row> queryBySql(DatacenterSource source, String sql) {
List<Row> rows = Db.selectListBySql(sql);
@@ -527,12 +542,20 @@ public abstract class AbstractInternalTableConnector implements DatacenterConnec
return StrUtil.blankToDefault(table.getMaterializedTable(), table.getActualTable());
}
private void normalizeRows(List<Row> records) {
/**
* 统一内部表查询结果的字段名与值类型。
*
* <p>MySQL 驱动在部分查询路径会返回大写列标签,而数据集元数据中的物理字段名
* 始终为小写。这里在序列化前收敛为小写确保预览、SQL 消费和导出按同一字段名取值。</p>
*
* @param records 待规范化的数据行
*/
static void normalizeRows(List<Row> records) {
for (Row record : records) {
Map<String, Object> converted = new LinkedHashMap<>();
for (Map.Entry<String, Object> entry : record.entrySet()) {
converted.put(
entry.getKey(),
entry.getKey().toLowerCase(Locale.ROOT),
normalizeValue(entry.getValue()));
}
record.clear();
@@ -546,7 +569,7 @@ public abstract class AbstractInternalTableConnector implements DatacenterConnec
* @param value JDBC 原始值
* @return 兼容既有查询接口的值
*/
private Object normalizeValue(Object value) {
private static Object normalizeValue(Object value) {
if (value instanceof BigInteger
|| value instanceof BigDecimal
|| value instanceof Long) {

View File

@@ -10,6 +10,9 @@ import java.net.UnknownHostException;
import java.sql.SQLException;
import java.util.Locale;
/**
* 数据连接器访问异常分类与安全包装工具。
*/
public final class DatacenterConnectorExceptionSupport {
public static final String SOURCE_UNAVAILABLE_MESSAGE = "当前连接不可用,请检查连接配置后重试";
@@ -17,14 +20,21 @@ public final class DatacenterConnectorExceptionSupport {
private DatacenterConnectorExceptionSupport() {
}
/**
* 将连接器异常转换为可安全展示的业务异常,并保留原始异常供日志追踪。
*
* @param fallbackMessage 默认业务错误
* @param ex 原始异常
* @return 包装后的业务异常
*/
public static BusinessException wrapAccessException(String fallbackMessage, Exception ex) {
if (ex instanceof BusinessException businessException && !isConnectionUnavailable(ex)) {
return businessException;
}
if (isConnectionUnavailable(ex)) {
return new BusinessException(SOURCE_UNAVAILABLE_MESSAGE);
return new BusinessException(400, 1, SOURCE_UNAVAILABLE_MESSAGE, ex);
}
return new BusinessException(fallbackMessage);
return new BusinessException(400, 1, fallbackMessage, ex);
}
public static boolean isConnectionUnavailable(Throwable throwable) {

View File

@@ -1,5 +1,7 @@
package tech.easyflow.datacenter.entity.base;
import com.fasterxml.jackson.databind.annotation.JsonSerialize;
import com.fasterxml.jackson.databind.ser.std.ToStringSerializer;
import com.mybatisflex.annotation.Column;
import com.mybatisflex.annotation.Id;
import com.mybatisflex.annotation.KeyType;
@@ -129,6 +131,7 @@ public class DatacenterTableBase extends DateEntity implements Serializable {
@Column(typeHandler = FastjsonTypeHandler.class, comment = "能力声明")
private Map<String, Object> capabilitiesJson;
@JsonSerialize(using = ToStringSerializer.class)
public BigInteger getId() {
return id;
}
@@ -153,6 +156,7 @@ public class DatacenterTableBase extends DateEntity implements Serializable {
this.tenantId = tenantId;
}
@JsonSerialize(using = ToStringSerializer.class)
public BigInteger getSourceId() {
return sourceId;
}
@@ -161,6 +165,7 @@ public class DatacenterTableBase extends DateEntity implements Serializable {
this.sourceId = sourceId;
}
@JsonSerialize(using = ToStringSerializer.class)
public BigInteger getCatalogId() {
return catalogId;
}

View File

@@ -1,5 +1,7 @@
package tech.easyflow.datacenter.entity.base;
import com.fasterxml.jackson.databind.annotation.JsonSerialize;
import com.fasterxml.jackson.databind.ser.std.ToStringSerializer;
import com.mybatisflex.annotation.Column;
import com.mybatisflex.annotation.Id;
import com.mybatisflex.annotation.KeyType;
@@ -129,6 +131,7 @@ public class DatacenterTableFieldBase extends DateEntity implements Serializable
@Column(comment = "修改者")
private BigInteger modifiedBy;
@JsonSerialize(using = ToStringSerializer.class)
public BigInteger getId() {
return id;
}
@@ -137,6 +140,7 @@ public class DatacenterTableFieldBase extends DateEntity implements Serializable
this.id = id;
}
@JsonSerialize(using = ToStringSerializer.class)
public BigInteger getTableId() {
return tableId;
}

View File

@@ -12,7 +12,16 @@ import java.math.BigInteger;
import java.util.List;
public interface DatacenterExcelImportService {
DatacenterImportJob importWorkbook(MultipartFile file, LoginAccount account) throws Exception;
/**
* 导入 Excel 工作簿并创建对应的数据源、目录和物化表。
*
* @param file Excel 工作簿文件
* @param sourceName 用户指定的数据源名称,留空时使用文件名
* @param account 当前登录账号
* @return 已完成的导入任务
* @throws Exception 文件读取或数据写入失败时抛出
*/
DatacenterImportJob importWorkbook(MultipartFile file, String sourceName, LoginAccount account) throws Exception;
DatacenterImportJob splitWorkbook(DatacenterExcelSplitRequest request, LoginAccount account);

View File

@@ -6,6 +6,7 @@ import com.mybatisflex.core.paginate.Page;
import com.mybatisflex.core.row.Row;
import org.apache.poi.ss.usermodel.Cell;
import org.apache.poi.ss.usermodel.DataFormatter;
import org.apache.poi.ss.usermodel.FormulaEvaluator;
import org.apache.poi.ss.usermodel.Sheet;
import org.apache.poi.ss.usermodel.Workbook;
import org.apache.poi.ss.usermodel.WorkbookFactory;
@@ -41,6 +42,7 @@ import tech.easyflow.datacenter.meta.enums.DatacenterImportStatus;
import tech.easyflow.datacenter.meta.enums.DatacenterSourceType;
import tech.easyflow.datacenter.meta.enums.DatacenterTableKind;
import tech.easyflow.datacenter.meta.model.DatacenterTableDetailMeta;
import tech.easyflow.datacenter.utils.SqlInjectionUtils;
import tech.easyflow.datacenter.meta.service.DatacenterDatasetRegistryService;
import tech.easyflow.datacenter.meta.service.DatacenterSourceService;
@@ -70,6 +72,14 @@ import java.util.UUID;
public class DatacenterExcelImportServiceImpl implements DatacenterExcelImportService {
private static final long QUERY_BATCH_SIZE = 500L;
private static final int MAX_IDENTIFIER_LENGTH = 64;
private static final int MAX_SOURCE_NAME_LENGTH = 100;
private static final int MAX_IMPORT_JOB_ERROR_SUMMARY_LENGTH = 1000;
private static final String ERROR_SUMMARY_TRUNCATION_MARKER = "…(错误摘要已截断)…";
private static final Set<String> SYSTEM_FIELD_NAMES = Set.of(
"id", "dept_id", "tenant_id", "created", "created_by",
"modified", "modified_by", "remark"
);
private static final DateTimeFormatter EXPORT_TIME_FORMAT = DateTimeFormatter.ofPattern("yyyyMMddHHmmss");
@Resource
@@ -89,11 +99,58 @@ public class DatacenterExcelImportServiceImpl implements DatacenterExcelImportSe
@Override
@Transactional(rollbackFor = Exception.class)
public DatacenterImportJob importWorkbook(MultipartFile file, LoginAccount account) throws Exception {
public DatacenterImportJob importWorkbook(
MultipartFile file,
String sourceName,
LoginAccount account) throws Exception {
if (file == null || file.isEmpty()) {
throw new BusinessException("Excel 文件不能为空");
}
String workbookName = extractWorkbookName(file.getOriginalFilename());
if (!isSupportedWorkbookFileName(file.getOriginalFilename())) {
throw new BusinessException("仅支持 .xls 和 .xlsx 格式的 Excel 文件");
}
try (InputStream inputStream = file.getInputStream()) {
Workbook parsedWorkbook;
try {
parsedWorkbook = WorkbookFactory.create(inputStream);
} catch (Exception ex) {
throw new BusinessException(
400,
1,
"Excel 文件无法解析,请确认文件未损坏且未加密",
ex
);
}
try (Workbook workbook = parsedWorkbook) {
return importParsedWorkbook(file, sourceName, account, workbook);
}
}
}
/**
* 将已解析并校验的工作簿写入数据中枢。
*
* @param file 原始上传文件
* @param sourceName 用户指定的数据源名称
* @param account 当前登录账号
* @param workbook 已解析的工作簿
* @return 已完成的导入任务
*/
private DatacenterImportJob importParsedWorkbook(
MultipartFile file,
String sourceName,
LoginAccount account,
Workbook workbook) {
DataFormatter formatter = new DataFormatter();
FormulaEvaluator formulaEvaluator = workbook.getCreationHelper().createFormulaEvaluator();
if (!hasImportableSheet(workbook, formatter)) {
throw new BusinessException(
"未检测到可导入的数据,请确认至少有一个工作表,首行包含表头且下方有数据"
);
}
String workbookName = resolveImportSourceName(sourceName, file.getOriginalFilename());
DatacenterSource source = new DatacenterSource();
source.setSourceName(workbookName);
source.setSourceCode("EXCEL_" + UUID.randomUUID());
@@ -110,18 +167,14 @@ public class DatacenterExcelImportServiceImpl implements DatacenterExcelImportSe
long totalRows = 0L;
long successRows = 0L;
List<BigInteger> createdTableIds = new ArrayList<>();
try (InputStream inputStream = file.getInputStream(); Workbook workbook = WorkbookFactory.create(inputStream)) {
DataFormatter formatter = new DataFormatter();
try {
for (int sheetIndex = 0; sheetIndex < workbook.getNumberOfSheets(); sheetIndex++) {
Sheet sheet = workbook.getSheetAt(sheetIndex);
org.apache.poi.ss.usermodel.Row headerRow = sheet.getRow(sheet.getFirstRowNum());
if (headerRow == null) {
if (!isImportableSheet(sheet, headerRow, formatter)) {
continue;
}
List<DatacenterTableField> fields = buildFields(headerRow, formatter);
if (fields.isEmpty()) {
continue;
}
DatacenterTable table = new DatacenterTable();
table.setTableName(uniqueTableName(source.getId(), catalog.getId(), sheet.getSheetName()));
table.setTableDesc(sheet.getSheetName());
@@ -150,7 +203,11 @@ public class DatacenterExcelImportServiceImpl implements DatacenterExcelImportSe
JSONObject payload = new JSONObject();
boolean hasValue = false;
for (int cellIndex = 0; cellIndex < savedTable.getFields().size(); cellIndex++) {
String value = formatter.formatCellValue(row.getCell(cellIndex));
String value = formatCellValue(
row.getCell(cellIndex),
formatter,
formulaEvaluator
);
if (value != null && !value.isBlank()) {
hasValue = true;
}
@@ -544,14 +601,59 @@ public class DatacenterExcelImportServiceImpl implements DatacenterExcelImportSe
importJobMapper.update(job);
}
/**
* 将导入任务标记为失败,并保存长度受限的错误摘要。
*
* @param job 待更新的导入任务
* @param ex 导入过程中抛出的异常
*/
private void finishJobFailure(DatacenterImportJob job, Exception ex) {
job.setStatus(DatacenterImportStatus.FAILED.name());
job.setErrorSummary(ex.getMessage());
job.setErrorSummary(summarizeImportError(ex));
job.setFinishedAt(new Date());
job.setModified(new Date());
importJobMapper.update(job);
}
/**
* 生成可写入导入任务错误摘要列的错误信息。
*
* @param ex 导入过程中抛出的异常
* @return 不超过数据库字段长度的错误摘要
*/
static String summarizeImportError(Exception ex) {
String message = ex == null ? null : ex.getMessage();
if (message == null || message.isBlank()) {
message = ex == null ? "Excel 导入失败" : ex.getClass().getSimpleName();
}
int messageLength = message.codePointCount(0, message.length());
if (messageLength <= MAX_IMPORT_JOB_ERROR_SUMMARY_LENGTH) {
return message;
}
int markerLength = ERROR_SUMMARY_TRUNCATION_MARKER.codePointCount(0, ERROR_SUMMARY_TRUNCATION_MARKER.length());
int availableLength = MAX_IMPORT_JOB_ERROR_SUMMARY_LENGTH - markerLength;
int prefixLength = availableLength * 2 / 3;
int suffixLength = availableLength - prefixLength;
return substringByCodePoint(message, 0, prefixLength)
+ ERROR_SUMMARY_TRUNCATION_MARKER
+ substringByCodePoint(message, messageLength - suffixLength, messageLength);
}
/**
* 按 Unicode 码点截取字符串,避免截断代理对字符。
*
* @param value 原始字符串
* @param beginCodePoint 起始码点索引(包含)
* @param endCodePoint 结束码点索引(不包含)
* @return 截取后的字符串
*/
private static String substringByCodePoint(String value, int beginCodePoint, int endCodePoint) {
int beginIndex = value.offsetByCodePoints(0, beginCodePoint);
int endIndex = value.offsetByCodePoints(0, endCodePoint);
return value.substring(beginIndex, endIndex);
}
private DatacenterTable resolveTable(DatasetRef datasetRef) {
if (datasetRef == null || datasetRef.getTableId() == null) {
throw new BusinessException("缺少数据集 tableId");
@@ -870,7 +972,7 @@ public class DatacenterExcelImportServiceImpl implements DatacenterExcelImportSe
return tables.stream().anyMatch(table -> tableName.equals(table.getTableName()));
}
private String extractWorkbookName(String originalFileName) {
private static String extractWorkbookName(String originalFileName) {
if (originalFileName == null || originalFileName.isBlank()) {
return "excel_workbook";
}
@@ -878,35 +980,166 @@ public class DatacenterExcelImportServiceImpl implements DatacenterExcelImportSe
return index > 0 ? originalFileName.substring(0, index) : originalFileName;
}
/**
* 解析 Excel 导入后使用的数据源名称。
*
* @param requestedSourceName 用户填写的数据源名称
* @param originalFileName 原始文件名
* @return 去除首尾空白且长度合法的数据源名称
* @throws BusinessException 名称超过数据库字段长度时抛出
*/
static String resolveImportSourceName(String requestedSourceName, String originalFileName) {
String resolved = requestedSourceName == null || requestedSourceName.isBlank()
? extractWorkbookName(originalFileName).trim()
: requestedSourceName.trim();
if (resolved.codePointCount(0, resolved.length()) > MAX_SOURCE_NAME_LENGTH) {
throw new BusinessException("连接名称不能超过 100 个字符");
}
return resolved;
}
/**
* 判断工作簿是否至少包含一个带表头和数据行的工作表。
*
* @param workbook 待检查的工作簿
* @param formatter 单元格格式化器
* @return 存在可导入工作表时返回 {@code true}
*/
static boolean hasImportableSheet(Workbook workbook, DataFormatter formatter) {
if (workbook == null) {
return false;
}
for (int sheetIndex = 0; sheetIndex < workbook.getNumberOfSheets(); sheetIndex++) {
Sheet sheet = workbook.getSheetAt(sheetIndex);
org.apache.poi.ss.usermodel.Row headerRow = sheet.getRow(sheet.getFirstRowNum());
if (isImportableSheet(sheet, headerRow, formatter)) {
return true;
}
}
return false;
}
/**
* 判断单个工作表是否包含有效表头和至少一行数据。
*
* @param sheet 待检查的工作表
* @param headerRow 首行表头
* @param formatter 单元格格式化器
* @return 工作表可导入时返回 {@code true}
*/
private static boolean isImportableSheet(
Sheet sheet,
org.apache.poi.ss.usermodel.Row headerRow,
DataFormatter formatter) {
if (sheet == null || headerRow == null || headerRow.getLastCellNum() <= 0) {
return false;
}
boolean hasHeader = false;
for (int cellIndex = 0; cellIndex < headerRow.getLastCellNum(); cellIndex++) {
if (!formatter.formatCellValue(headerRow.getCell(cellIndex)).isBlank()) {
hasHeader = true;
break;
}
}
if (!hasHeader) {
return false;
}
for (int rowIndex = sheet.getFirstRowNum() + 1; rowIndex <= sheet.getLastRowNum(); rowIndex++) {
org.apache.poi.ss.usermodel.Row row = sheet.getRow(rowIndex);
if (row == null) {
continue;
}
for (int cellIndex = 0; cellIndex < headerRow.getLastCellNum(); cellIndex++) {
if (!formatter.formatCellValue(row.getCell(cellIndex)).isBlank()) {
return true;
}
}
}
return false;
}
/**
* 获取单元格最终显示值,公式单元格返回计算结果。
*
* @param cell 单元格
* @param formatter 单元格格式化器
* @param formulaEvaluator 公式计算器
* @return 单元格显示值;空单元格返回空字符串
*/
static String formatCellValue(
Cell cell,
DataFormatter formatter,
FormulaEvaluator formulaEvaluator) {
if (cell == null) {
return "";
}
return formatter.formatCellValue(cell, formulaEvaluator);
}
/**
* 判断文件名是否为支持的 Excel 工作簿格式。
*
* @param originalFileName 原始上传文件名
* @return 文件扩展名为 {@code .xls} 或 {@code .xlsx} 时返回 {@code true}
*/
static boolean isSupportedWorkbookFileName(String originalFileName) {
if (originalFileName == null || originalFileName.isBlank()) {
return false;
}
String fileName = originalFileName.trim().toLowerCase(Locale.ROOT);
return fileName.endsWith(".xls") || fileName.endsWith(".xlsx");
}
private String buildMaterializedTableName(BigInteger sourceId, int sheetIndex) {
long snowId = new SnowFlakeIDKeyGenerator().nextId();
return "tb_excel_" + sourceId + "_" + sheetIndex + "_" + snowId;
}
private String normalizeIdentifier(String raw) {
/**
* 将 Excel 表头转换为可用于数据库物理字段的安全标识符。
*
* @param raw 原始表头
* @return 仅包含 ASCII 字母、数字和下划线的候选字段名
*/
static String normalizeIdentifier(String raw) {
if (raw == null || raw.isBlank()) {
return "value";
return "";
}
String normalized = raw.trim().toLowerCase(Locale.ROOT).replaceAll("[^a-z0-9_\\u4e00-\\u9fa5]+", "_");
normalized = normalized.replaceAll("_+", "_");
if (normalized.isBlank()) {
return "value";
}
return normalized;
return raw.trim()
.toLowerCase(Locale.ROOT)
.replaceAll("[^a-z0-9]+", "_")
.replaceAll("_+", "_")
.replaceAll("^_+|_+$", "");
}
private String normalizeIdentifier(String raw, int index, Set<String> usedNames) {
/**
* 为 Excel 表头生成唯一且安全的物理字段名。
*
* @param raw 原始表头
* @param index 表头从零开始的列索引
* @param usedNames 当前已使用的物理字段名
* @return 唯一的安全物理字段名,且不会与系统保留列冲突
*/
static String normalizeIdentifier(String raw, int index, Set<String> usedNames) {
String value = normalizeIdentifier(raw);
if (value.isBlank() || "value".equals(value)) {
if (value.isBlank()) {
value = "col_" + (index + 1);
}
if (Character.isDigit(value.charAt(0))) {
value = "col_" + value;
}
if (SqlInjectionUtils.isSqlKeyword(value) || SYSTEM_FIELD_NAMES.contains(value)) {
value = "col_" + value;
}
value = value.substring(0, Math.min(value.length(), MAX_IDENTIFIER_LENGTH));
String result = value;
int suffix = 1;
while (usedNames.contains(result)) {
result = value + "_" + suffix++;
if (result.length() > MAX_IDENTIFIER_LENGTH) {
String suffixText = "_" + (suffix - 1);
result = value.substring(0, MAX_IDENTIFIER_LENGTH - suffixText.length()) + suffixText;
}
}
usedNames.add(result);
return result;

View File

@@ -9,6 +9,7 @@ import org.springframework.util.StringUtils;
import tech.easyflow.common.web.exceptions.BusinessException;
import tech.easyflow.datacenter.connector.DatacenterConnector;
import tech.easyflow.datacenter.connector.DatacenterConnectorRegistry;
import tech.easyflow.datacenter.connector.support.AbstractInternalTableConnector;
import tech.easyflow.datacenter.entity.DatacenterTable;
import tech.easyflow.datacenter.entity.DatacenterTableField;
import tech.easyflow.datacenter.execution.model.DatacenterQueryRequest;
@@ -189,14 +190,16 @@ public class DatacenterDatasetQueryServiceImpl implements DatacenterDatasetQuery
}
Map<BigInteger, DatacenterCatalog> catalogsById =
loadCatalogsById(managedTables);
SqlSupportUtils.ResolvedSql resolvedSql = SqlSupportUtils.resolve(
sql,
managedTables.stream()
.map(table -> toManagedSqlTable(
table, catalogsById))
.toList()
);
DatacenterConnector connector = connectorRegistry.getConnector(source.getSourceType());
List<SqlSupportUtils.ManagedTable> sqlTables = managedTables.stream()
.map(table -> toManagedSqlTable(
table, catalogsById))
.toList();
// 内部连接的 catalog 是逻辑命名空间,底层项目 MySQL 只执行物理表名。
SqlSupportUtils.ResolvedSql resolvedSql =
connector instanceof AbstractInternalTableConnector
? SqlSupportUtils.resolveInternalMysql(sql, sqlTables)
: SqlSupportUtils.resolve(sql, sqlTables);
return new ResolvedSqlQuery(
source,
connector,

View File

@@ -1,5 +1,7 @@
package tech.easyflow.datacenter.meta.entity;
import com.fasterxml.jackson.databind.annotation.JsonSerialize;
import com.fasterxml.jackson.databind.ser.std.ToStringSerializer;
import com.mybatisflex.annotation.Column;
import com.mybatisflex.annotation.Id;
import com.mybatisflex.annotation.KeyType;
@@ -42,6 +44,7 @@ public class DatacenterCatalog extends DateEntity implements Serializable {
@Column(comment = "修改人")
private BigInteger modifiedBy;
@JsonSerialize(using = ToStringSerializer.class)
public BigInteger getId() { return id; }
public void setId(BigInteger id) { this.id = id; }
public BigInteger getDeptId() { return deptId; }

View File

@@ -1,5 +1,7 @@
package tech.easyflow.datacenter.meta.entity;
import com.fasterxml.jackson.databind.annotation.JsonSerialize;
import com.fasterxml.jackson.databind.ser.std.ToStringSerializer;
import com.mybatisflex.annotation.Column;
import com.mybatisflex.annotation.Id;
import com.mybatisflex.annotation.KeyType;
@@ -57,16 +59,20 @@ public class DatacenterImportJob extends DateEntity implements Serializable {
@Column(comment = "修改人")
private BigInteger modifiedBy;
@JsonSerialize(using = ToStringSerializer.class)
public BigInteger getId() { return id; }
public void setId(BigInteger id) { this.id = id; }
public BigInteger getDeptId() { return deptId; }
public void setDeptId(BigInteger deptId) { this.deptId = deptId; }
public BigInteger getTenantId() { return tenantId; }
public void setTenantId(BigInteger tenantId) { this.tenantId = tenantId; }
@JsonSerialize(using = ToStringSerializer.class)
public BigInteger getSourceId() { return sourceId; }
public void setSourceId(BigInteger sourceId) { this.sourceId = sourceId; }
@JsonSerialize(using = ToStringSerializer.class)
public BigInteger getCatalogId() { return catalogId; }
public void setCatalogId(BigInteger catalogId) { this.catalogId = catalogId; }
@JsonSerialize(using = ToStringSerializer.class)
public BigInteger getTableId() { return tableId; }
public void setTableId(BigInteger tableId) { this.tableId = tableId; }
public String getJobType() { return jobType; }

View File

@@ -1,5 +1,7 @@
package tech.easyflow.datacenter.meta.entity;
import com.fasterxml.jackson.databind.annotation.JsonSerialize;
import com.fasterxml.jackson.databind.ser.std.ToStringSerializer;
import com.mybatisflex.annotation.Column;
import com.mybatisflex.annotation.Id;
import com.mybatisflex.annotation.KeyType;
@@ -70,6 +72,7 @@ public class DatacenterSource extends DateEntity implements Serializable {
@Column(comment = "修改人")
private BigInteger modifiedBy;
@JsonSerialize(using = ToStringSerializer.class)
public BigInteger getId() { return id; }
public void setId(BigInteger id) { this.id = id; }
public BigInteger getDeptId() { return deptId; }

View File

@@ -1,7 +1,13 @@
package tech.easyflow.datacenter.meta.model;
import com.fasterxml.jackson.databind.annotation.JsonSerialize;
import com.fasterxml.jackson.databind.ser.std.ToStringSerializer;
import java.math.BigInteger;
/**
* 数据中枢目录元数据。
*/
public class DatacenterCatalogMeta {
private BigInteger id;
private BigInteger sourceId;
@@ -9,8 +15,10 @@ public class DatacenterCatalogMeta {
private String catalogType;
private String catalogDesc;
@JsonSerialize(using = ToStringSerializer.class)
public BigInteger getId() { return id; }
public void setId(BigInteger id) { this.id = id; }
@JsonSerialize(using = ToStringSerializer.class)
public BigInteger getSourceId() { return sourceId; }
public void setSourceId(BigInteger sourceId) { this.sourceId = sourceId; }
public String getCatalogName() { return catalogName; }

View File

@@ -2,20 +2,35 @@ package tech.easyflow.datacenter.utils;
import tech.easyflow.common.web.exceptions.BusinessException;
import java.util.Arrays;
import java.util.HashSet;
import java.util.Locale;
import java.util.Set;
public class SqlInjectionUtils {
/**
* 数据中心动态 SQL 的安全校验与字面量转义工具。
*/
public final class SqlInjectionUtils {
private static final Set<String> SQL_KEYWORDS = new HashSet<>(Arrays.asList(
private static final Set<String> SQL_KEYWORDS = Set.of(
"select", "insert", "update", "delete", "drop", "alter", "create",
"table", "where", "from", "join", "union", "truncate", "execute",
"grant", "revoke", "commit", "rollback"
));
"grant", "revoke", "commit", "rollback", "order", "group", "by",
"having", "limit", "offset", "as", "on", "into", "values", "index",
"key", "primary", "constraint", "references", "distinct", "case",
"when", "then", "else", "end", "and", "or", "not", "null", "like",
"in", "is", "exists", "between", "procedure", "function", "trigger",
"view", "database", "schema", "column", "add", "rename", "replace",
"show", "describe", "explain", "use", "lock", "unlock"
);
private SqlInjectionUtils() {
}
/**
* 校验字段或表名
* 校验动态 SQL 中的字段或表标识符。
*
* @param identifier 待校验的标识符
* @return 已校验的标识符
* @throws BusinessException 标识符为空、过长、包含非法字符或为 SQL 关键字时抛出
*/
public static String checkIdentifier(String identifier) {
if (identifier == null || identifier.isEmpty()) {
@@ -35,34 +50,36 @@ public class SqlInjectionUtils {
}
/**
* 校验注释
* 允许的字符包括以下 Unicode 类别或符号:
* \p{L}:任何语言的字母(包括中文、英文、日文等)。
* \p{N}:任何数字(包括阿拉伯数字 0-9 或其他语言的数字符号)。
* \p{Zs}:空白分隔符(如空格,但不包括换行符、制表符等)。
* 标点符号:. , - : ? !(基础标点)。
* 校验并转义 SQL 字符串字面量中的表或字段备注。
*
* <p>Excel 表头和工作表名称可包含常见的中英文标点、单位符号和 Unicode 字符。换行和制表符会
* 规范为单个空格;其余控制字符会被拒绝。单引号和反斜杠会被转义,避免备注内容破坏动态 DDL 语句。</p>
*
* @param comment 原始备注内容
* @return 可安全拼入单引号 SQL 字符串字面量的备注内容
* @throws BusinessException 备注过长或包含控制字符时抛出
*/
public static String checkComment(String comment) {
if (comment == null) {
if (comment == null || comment.isEmpty()) {
return "";
}
if (comment.length() > 255) {
throw new BusinessException("注释过长");
}
if (!comment.matches("^[\\p{L}\\p{N}\\p{Zs}\\.\\,\\-\\:\\?\\!]+$")) {
throw new BusinessException("包含非法字符");
String normalizedComment = comment.replaceAll("[\\r\\n\\t]+", " ");
if (normalizedComment.codePoints().anyMatch(Character::isISOControl)) {
throw new BusinessException("备注不能包含控制字符");
}
if (comment.contains("--")) {
throw new BusinessException("包含非法字符!");
}
if (comment.chars().anyMatch(c -> c <= 31 || c == 127)) {
throw new BusinessException("存在非法字符");
}
return comment;
return normalizedComment.replace("\\", "\\\\").replace("'", "''");
}
// 检查是否是数据库关键字
/**
* 判断指定单词是否为受限 SQL 关键字。
*
* @param word 待判断的单词
* @return 是 SQL 关键字时返回 {@code true}
*/
public static boolean isSqlKeyword(String word) {
return SQL_KEYWORDS.contains(word.toLowerCase());
return word != null && SQL_KEYWORDS.contains(word.toLowerCase(Locale.ROOT));
}
}

View File

@@ -25,6 +25,36 @@ public final class SqlSupportUtils {
}
public static ResolvedSql resolve(String sql, Collection<ManagedTable> managedTables) {
return resolve(sql, managedTables, true, false);
}
/**
* 解析内部物化表 SQL逻辑目录仅用于白名单匹配不进入项目 MySQL 的执行 SQL。
*
* @param sql 逻辑 SQL
* @param managedTables 已接入表
* @return 已解析的项目 MySQL SQL
*/
public static ResolvedSql resolveInternalMysql(
String sql,
Collection<ManagedTable> managedTables) {
return resolve(sql, managedTables, false, true);
}
/**
* 解析并重写只读 SQL。
*
* @param sql 逻辑 SQL
* @param managedTables 已接入表
* @param retainCatalog 是否保留目录限定符
* @param mysqlIdentifierQuotes 是否转换为 MySQL 标识符引号
* @return 已解析 SQL
*/
private static ResolvedSql resolve(
String sql,
Collection<ManagedTable> managedTables,
boolean retainCatalog,
boolean mysqlIdentifierQuotes) {
String normalizedSql = normalizeSql(sql);
Statement statement = parseSingleStatement(normalizedSql);
if (!(statement instanceof Select select)) {
@@ -54,10 +84,14 @@ public final class SqlSupportUtils {
Set<String> logicalTables = new LinkedHashSet<>();
for (Table table : referencedTables) {
ManagedTable managedTable = resolveManagedTable(table, byTableName, byCatalogAndTable);
rewriteTable(table, managedTable);
rewriteTable(table, managedTable, retainCatalog);
logicalTables.add(renderLogicalTable(managedTable));
}
return new ResolvedSql(select.toString(), new ArrayList<>(logicalTables));
String executableSql = select.toString();
if (mysqlIdentifierQuotes) {
executableSql = normalizeMysqlIdentifierQuotes(executableSql);
}
return new ResolvedSql(executableSql, new ArrayList<>(logicalTables));
}
private static Statement parseSingleStatement(String sql) {
@@ -102,9 +136,73 @@ public final class SqlSupportUtils {
return matches.get(0);
}
private static void rewriteTable(Table table, ManagedTable managedTable) {
/**
* 将逻辑表替换为物理表。
*
* @param table SQL 表节点
* @param managedTable 已接入表
* @param retainCatalog 是否保留目录限定符
*/
private static void rewriteTable(
Table table,
ManagedTable managedTable,
boolean retainCatalog) {
table.setName(managedTable.getPhysicalTableName());
table.setSchemaName(trimToNull(managedTable.getCatalogName()));
table.setSchemaName(retainCatalog
? trimToNull(managedTable.getCatalogName())
: null);
}
/**
* 将标准 SQL 双引号标识符转换为 MySQL 反引号,同时保留字符串字面量内容。
*
* @param sql 已解析 SQL
* @return MySQL 可执行 SQL
*/
private static String normalizeMysqlIdentifierQuotes(String sql) {
StringBuilder normalized = new StringBuilder(sql.length());
boolean singleQuoted = false;
boolean doubleQuotedIdentifier = false;
for (int index = 0; index < sql.length(); index++) {
char current = sql.charAt(index);
if (singleQuoted) {
normalized.append(current);
if (current == '\\' && index + 1 < sql.length()) {
normalized.append(sql.charAt(++index));
} else if (current == '\'' && index + 1 < sql.length()
&& sql.charAt(index + 1) == '\'') {
normalized.append(sql.charAt(++index));
} else if (current == '\'') {
singleQuoted = false;
}
continue;
}
if (doubleQuotedIdentifier) {
if (current == '"' && index + 1 < sql.length()
&& sql.charAt(index + 1) == '"') {
normalized.append('"');
index++;
} else if (current == '"') {
normalized.append('`');
doubleQuotedIdentifier = false;
} else if (current == '`') {
normalized.append("``");
} else {
normalized.append(current);
}
continue;
}
if (current == '\'') {
singleQuoted = true;
normalized.append(current);
} else if (current == '"') {
doubleQuotedIdentifier = true;
normalized.append('`');
} else {
normalized.append(current);
}
}
return normalized.toString();
}
private static String renderLogicalTable(ManagedTable managedTable) {