Compare commits

...

2 Commits

Author SHA1 Message Date
1ecc28e498 feat: 工作流适配数据中枢查询节点
- 新增查询数据与写入数据节点并移除旧数据中心节点入口

- 将查询数据节点切换为连接服务加 SQL 的执行模型

- 同步更新工作流校验、提示词上下文与设计器交互
2026-04-02 18:56:34 +08:00
798effbd5b feat: 重构数据中枢工作台与接入管理
- 新增统一的数据源、目录、纳管表与 Excel 处理后端能力

- 重建管理端数据中枢工作台并替换旧表管理页面

- 补充数据中枢迁移脚本、连接器底座与说明字段支持
2026-04-02 18:55:31 +08:00
157 changed files with 11712 additions and 2516 deletions

View File

@@ -13,6 +13,7 @@ import com.easyagents.flow.core.parser.ChainParser;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
import tech.easyflow.ai.easyagentsflow.service.WorkflowDatacenterContentService;
import tech.easyflow.ai.entity.Workflow;
import tech.easyflow.ai.service.WorkflowService;
import tech.easyflow.common.domain.Result;
@@ -29,6 +30,8 @@ public class WorkFlowNodeController {
private WorkflowService workflowService;
@Resource
private ChainParser chainParser;
@Resource
private WorkflowDatacenterContentService workflowDatacenterContentService;
@GetMapping("/getChainParams")
public Result<?> getChainParams(String currentId, String workflowId) {
@@ -43,7 +46,7 @@ public class WorkFlowNodeController {
nodeData.put("workflowId", workflow.getId());
nodeData.put("workflowName", workflow.getTitle());
ChainDefinition definition = chainParser.parse(workflow.getContent());
ChainDefinition definition = chainParser.parse(workflowDatacenterContentService.prepareContent(workflow.getContent()));
List<Node> nodes = definition.getNodes();
JSONArray inputs = new JSONArray();
JSONArray outputs = new JSONArray();

View File

@@ -21,6 +21,7 @@ import tech.easyflow.ai.easyagentsflow.entity.WorkflowCheckStage;
import tech.easyflow.ai.easyagentsflow.service.CodeEngineCapabilityService;
import tech.easyflow.ai.easyagentsflow.service.TinyFlowService;
import tech.easyflow.ai.easyagentsflow.service.WorkflowCheckService;
import tech.easyflow.ai.easyagentsflow.service.WorkflowDatacenterContentService;
import tech.easyflow.ai.entity.Workflow;
import tech.easyflow.ai.service.BotWorkflowService;
import tech.easyflow.ai.service.ModelService;
@@ -76,6 +77,8 @@ public class WorkflowController extends BaseCurdController<WorkflowService, Work
@Resource
private WorkflowCheckService workflowCheckService;
@Resource
private WorkflowDatacenterContentService workflowDatacenterContentService;
@Resource
private ResourceAccessService resourceAccessService;
@Resource
private WorkflowVisibilityQueryHelper workflowVisibilityQueryHelper;
@@ -223,7 +226,7 @@ public class WorkflowController extends BaseCurdController<WorkflowService, Work
}
workflowCheckService.checkOrThrow(workflow.getContent(), WorkflowCheckStage.PRE_EXECUTE, workflow.getId());
ChainDefinition definition = chainParser.parse(workflow.getContent());
ChainDefinition definition = chainParser.parse(workflowDatacenterContentService.prepareContent(workflow.getContent()));
if (definition == null) {
return Result.fail(2, "节点配置错误,请检查! ");
}

View File

@@ -0,0 +1,68 @@
package tech.easyflow.admin.controller.datacenter;
import cn.dev33.satoken.annotation.SaCheckPermission;
import com.mybatisflex.core.paginate.Page;
import com.mybatisflex.core.row.Row;
import org.springframework.web.bind.annotation.*;
import tech.easyflow.common.domain.Result;
import tech.easyflow.common.entity.LoginAccount;
import tech.easyflow.common.satoken.util.SaTokenUtil;
import tech.easyflow.datacenter.entity.DatacenterTable;
import tech.easyflow.datacenter.execution.model.DatacenterQueryRequest;
import tech.easyflow.datacenter.execution.model.DatacenterSchemaResponse;
import tech.easyflow.datacenter.execution.model.DatasetRef;
import tech.easyflow.datacenter.meta.model.DatacenterBatchRemoveRequest;
import tech.easyflow.datacenter.meta.model.DatacenterSaveDescriptionsRequest;
import tech.easyflow.datacenter.execution.service.DatacenterDatasetQueryService;
import tech.easyflow.datacenter.meta.service.DatacenterDatasetRegistryService;
import javax.annotation.Resource;
import java.math.BigInteger;
import java.util.List;
@RestController
@RequestMapping("/api/v1/datacenterDataset")
public class DatacenterDatasetController {
@Resource
private DatacenterDatasetQueryService queryService;
@Resource
private DatacenterDatasetRegistryService registryService;
@PostMapping("/queryPage")
@SaCheckPermission("/api/v1/datacenterSource/query")
public Result<Page<Row>> queryPage(@RequestBody DatacenterQueryRequest request) {
return Result.ok(queryService.queryPage(request));
}
@GetMapping("/schema")
@SaCheckPermission("/api/v1/datacenterSource/query")
public Result<DatacenterSchemaResponse> schema(DatasetRef datasetRef) {
return Result.ok(queryService.getSchema(datasetRef));
}
@GetMapping("/managedTables")
@SaCheckPermission("/api/v1/datacenterSource/query")
public Result<List<DatacenterTable>> managedTables(BigInteger sourceId, BigInteger catalogId) {
return Result.ok(registryService.listManagedTables(sourceId, catalogId));
}
@PostMapping("/removeBatch")
@SaCheckPermission("/api/v1/datacenterSource/save")
public Result<Integer> removeBatch(@RequestBody DatacenterBatchRemoveRequest request) {
return Result.ok(registryService.removeTables(request == null ? List.of() : request.getTableIds()));
}
@PostMapping("/saveDescriptions")
@SaCheckPermission("/api/v1/datacenterSource/save")
public Result<DatacenterSchemaResponse> saveDescriptions(@RequestBody DatacenterSaveDescriptionsRequest request) {
LoginAccount account = SaTokenUtil.getLoginAccount();
DatacenterTable table = registryService.saveDescriptions(
request == null ? null : request.getTableId(),
request == null ? null : request.getTableDesc(),
request == null ? List.of() : request.getFields(),
account
);
return Result.ok(queryService.getSchema(registryService.resolveDatasetRef(table.getId())));
}
}

View File

@@ -0,0 +1,108 @@
package tech.easyflow.admin.controller.datacenter;
import cn.dev33.satoken.annotation.SaCheckPermission;
import jakarta.servlet.http.HttpServletResponse;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
import org.springframework.web.multipart.MultipartFile;
import tech.easyflow.common.domain.Result;
import tech.easyflow.common.entity.LoginAccount;
import tech.easyflow.common.satoken.util.SaTokenUtil;
import tech.easyflow.datacenter.excel.model.DatacenterExcelDeriveRequest;
import tech.easyflow.datacenter.excel.model.DatacenterExcelExportRequest;
import tech.easyflow.datacenter.excel.model.DatacenterExcelMergeRequest;
import tech.easyflow.datacenter.excel.model.DatacenterExcelSplitRequest;
import tech.easyflow.datacenter.excel.service.DatacenterExcelImportService;
import tech.easyflow.datacenter.meta.entity.DatacenterImportJob;
import java.io.File;
import java.io.FileInputStream;
import javax.annotation.Resource;
import java.math.BigInteger;
import java.net.URLEncoder;
import java.util.List;
@RestController
@RequestMapping("/api/v1/datacenterExcel")
public class DatacenterExcelController {
@Resource
private DatacenterExcelImportService excelImportService;
@PostMapping("/import")
@SaCheckPermission("/api/v1/datacenterSource/save")
public Result<DatacenterImportJob> importWorkbook(MultipartFile file) throws Exception {
LoginAccount account = SaTokenUtil.getLoginAccount();
return Result.ok(excelImportService.importWorkbook(file, account));
}
@PostMapping("/split")
@SaCheckPermission("/api/v1/datacenterSource/save")
public Result<DatacenterImportJob> split(@RequestBody DatacenterExcelSplitRequest request) {
LoginAccount account = SaTokenUtil.getLoginAccount();
return Result.ok(excelImportService.splitWorkbook(request, account));
}
@PostMapping("/merge")
@SaCheckPermission("/api/v1/datacenterSource/save")
public Result<DatacenterImportJob> merge(@RequestBody DatacenterExcelMergeRequest request) {
LoginAccount account = SaTokenUtil.getLoginAccount();
return Result.ok(excelImportService.mergeWorkbook(request, account));
}
@PostMapping("/derive")
@SaCheckPermission("/api/v1/datacenterSource/save")
public Result<DatacenterImportJob> derive(@RequestBody DatacenterExcelDeriveRequest request) {
LoginAccount account = SaTokenUtil.getLoginAccount();
return Result.ok(excelImportService.deriveWorkbook(request, account));
}
@PostMapping("/export")
@SaCheckPermission("/api/v1/datacenterSource/query")
public Result<DatacenterImportJob> export(@RequestBody DatacenterExcelExportRequest request) throws Exception {
LoginAccount account = SaTokenUtil.getLoginAccount();
return Result.ok(excelImportService.exportWorkbook(request, account));
}
@GetMapping("/importJob/detail")
@SaCheckPermission("/api/v1/datacenterSource/query")
public Result<DatacenterImportJob> jobDetail(BigInteger jobId) {
return Result.ok(excelImportService.getImportJobDetail(jobId));
}
@GetMapping("/job/detail")
@SaCheckPermission("/api/v1/datacenterSource/query")
public Result<DatacenterImportJob> newJobDetail(BigInteger jobId) {
return Result.ok(excelImportService.getImportJobDetail(jobId));
}
@GetMapping("/job/list")
@SaCheckPermission("/api/v1/datacenterSource/query")
public Result<List<DatacenterImportJob>> jobList(BigInteger sourceId, BigInteger tableId) {
return Result.ok(excelImportService.listJobs(sourceId, tableId));
}
@GetMapping("/download")
@SaCheckPermission("/api/v1/datacenterSource/query")
public void download(BigInteger jobId, HttpServletResponse response) throws Exception {
DatacenterImportJob job = excelImportService.getImportJobDetail(jobId);
if (job.getStoragePath() == null || job.getStoragePath().isBlank()) {
throw new IllegalStateException("导出文件不存在");
}
File file = new File(job.getStoragePath());
if (!file.exists()) {
throw new IllegalStateException("导出文件不存在");
}
response.setContentType("application/vnd.openxmlformats-officedocument.spreadsheetml.sheet");
response.setCharacterEncoding("utf-8");
String fileName = URLEncoder.encode(job.getFileName(), "UTF-8").replaceAll("\\+", "%20");
response.setHeader("Content-disposition", "attachment;filename*=utf-8''" + fileName);
try (FileInputStream inputStream = new FileInputStream(file)) {
inputStream.transferTo(response.getOutputStream());
response.flushBuffer();
}
}
}

View File

@@ -0,0 +1,86 @@
package tech.easyflow.admin.controller.datacenter;
import cn.dev33.satoken.annotation.SaCheckPermission;
import com.mybatisflex.core.paginate.Page;
import org.springframework.web.bind.annotation.*;
import tech.easyflow.common.domain.Result;
import tech.easyflow.common.entity.LoginAccount;
import tech.easyflow.common.satoken.util.SaTokenUtil;
import tech.easyflow.datacenter.entity.DatacenterTable;
import tech.easyflow.datacenter.execution.model.DatacenterConnectionTestResult;
import tech.easyflow.datacenter.meta.entity.DatacenterSource;
import tech.easyflow.datacenter.meta.model.DatacenterBatchRegisterRequest;
import tech.easyflow.datacenter.meta.model.DatacenterCatalogMeta;
import tech.easyflow.datacenter.meta.model.DatacenterRemoveSourceRequest;
import tech.easyflow.datacenter.meta.model.DatacenterTableDetailMeta;
import tech.easyflow.datacenter.meta.service.DatacenterSourceService;
import javax.annotation.Resource;
import java.math.BigInteger;
import java.util.List;
@RestController
@RequestMapping("/api/v1/datacenterSource")
public class DatacenterSourceController {
@Resource
private DatacenterSourceService sourceService;
@PostMapping("/testConnection")
@SaCheckPermission("/api/v1/datacenterSource/query")
public Result<DatacenterConnectionTestResult> testConnection(@RequestBody DatacenterSource source) {
LoginAccount account = SaTokenUtil.getLoginAccount();
return Result.ok(sourceService.testConnection(source, account));
}
@PostMapping("/save")
@SaCheckPermission("/api/v1/datacenterSource/save")
public Result<DatacenterSource> save(@RequestBody DatacenterSource source) {
LoginAccount account = SaTokenUtil.getLoginAccount();
return Result.ok(sourceService.saveSource(source, account));
}
@GetMapping("/page")
@SaCheckPermission("/api/v1/datacenterSource/query")
public Result<Page<DatacenterSource>> page(Long pageNumber, Long pageSize) {
LoginAccount account = SaTokenUtil.getLoginAccount();
return Result.ok(sourceService.pageSources(pageNumber, pageSize, account));
}
@GetMapping("/catalogs")
@SaCheckPermission("/api/v1/datacenterSource/query")
public Result<List<DatacenterCatalogMeta>> catalogs(BigInteger sourceId) {
LoginAccount account = SaTokenUtil.getLoginAccount();
return Result.ok(sourceService.listCatalogs(sourceId, account));
}
@GetMapping("/tables")
@SaCheckPermission("/api/v1/datacenterSource/query")
public Result<List<DatacenterTable>> tables(BigInteger sourceId, String catalogName) {
LoginAccount account = SaTokenUtil.getLoginAccount();
return Result.ok(sourceService.listTables(sourceId, catalogName, account));
}
@GetMapping("/tableDetail")
@SaCheckPermission("/api/v1/datacenterSource/query")
public Result<DatacenterTableDetailMeta> tableDetail(BigInteger sourceId, String catalogName, String tableName,
@RequestParam(defaultValue = "false") boolean register) {
LoginAccount account = SaTokenUtil.getLoginAccount();
return Result.ok(sourceService.getTableDetail(sourceId, catalogName, tableName, register, account));
}
@PostMapping("/registerBatch")
@SaCheckPermission("/api/v1/datacenterSource/save")
public Result<List<DatacenterTable>> registerBatch(@RequestBody DatacenterBatchRegisterRequest request) {
LoginAccount account = SaTokenUtil.getLoginAccount();
return Result.ok(sourceService.batchRegisterTables(request, account));
}
@PostMapping("/remove")
@SaCheckPermission("/api/v1/datacenterSource/save")
public Result<Void> remove(@RequestBody DatacenterRemoveSourceRequest request) {
LoginAccount account = SaTokenUtil.getLoginAccount();
sourceService.removeSource(request == null ? null : request.getSourceId(), account);
return Result.ok();
}
}

View File

@@ -1,195 +0,0 @@
package tech.easyflow.admin.controller.datacenter;
import cn.dev33.satoken.annotation.SaCheckPermission;
import cn.hutool.core.collection.CollectionUtil;
import cn.idev.excel.EasyExcel;
import cn.idev.excel.FastExcel;
import com.alibaba.fastjson2.JSONObject;
import com.mybatisflex.core.paginate.Page;
import com.mybatisflex.core.query.QueryWrapper;
import com.mybatisflex.core.row.Row;
import jakarta.servlet.http.HttpServletResponse;
import org.springframework.web.bind.annotation.*;
import org.springframework.web.multipart.MultipartFile;
import tech.easyflow.common.domain.Result;
import tech.easyflow.common.entity.DatacenterQuery;
import tech.easyflow.common.entity.LoginAccount;
import tech.easyflow.common.satoken.util.SaTokenUtil;
import tech.easyflow.common.web.controller.BaseCurdController;
import tech.easyflow.datacenter.entity.DatacenterTable;
import tech.easyflow.datacenter.entity.DatacenterTableField;
import tech.easyflow.datacenter.entity.vo.HeaderVo;
import tech.easyflow.datacenter.excel.ReadDataListener;
import tech.easyflow.datacenter.excel.ReadResVo;
import tech.easyflow.datacenter.service.DatacenterTableFieldService;
import tech.easyflow.datacenter.service.DatacenterTableService;
import javax.annotation.Resource;
import java.io.InputStream;
import java.math.BigInteger;
import java.net.URLEncoder;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import java.util.Map;
/**
* 数据中枢表 控制层。
*
* @author ArkLight
* @since 2025-07-10
*/
@RestController
@RequestMapping("/api/v1/datacenterTable")
public class DatacenterTableController extends BaseCurdController<DatacenterTableService, DatacenterTable> {
@Resource
private DatacenterTableFieldService fieldsService;
public DatacenterTableController(DatacenterTableService service) {
super(service);
}
@Override
protected Result onSaveOrUpdateBefore(DatacenterTable entity, boolean isSave) {
LoginAccount loginUser = SaTokenUtil.getLoginAccount();
if (isSave) {
commonFiled(entity, loginUser.getId(), loginUser.getTenantId(), loginUser.getDeptId());
} else {
entity.setModifiedBy(loginUser.getId());
}
return super.onSaveOrUpdateBefore(entity, isSave);
}
@PostMapping("/saveTable")
@SaCheckPermission("/api/v1/datacenterTable/save")
public Result<Void> saveTable(@RequestBody DatacenterTable entity) {
LoginAccount loginUser = SaTokenUtil.getLoginAccount();
List<DatacenterTableField> fields = entity.getFields();
if (CollectionUtil.isEmpty(fields)) {
return Result.fail(99, "字段不能为空");
}
BigInteger id = entity.getId();
if (id == null) {
commonFiled(entity, loginUser.getId(), loginUser.getTenantId(), loginUser.getDeptId());
} else {
entity.setModified(new Date());
entity.setModifiedBy(loginUser.getId());
}
service.saveTable(entity, loginUser);
return Result.ok();
}
@GetMapping("/detailInfo")
@SaCheckPermission("/api/v1/datacenterTable/query")
public Result<DatacenterTable> detailInfo(BigInteger tableId) {
DatacenterTable table = service.getById(tableId);
QueryWrapper wrapper = QueryWrapper.create();
wrapper.eq(DatacenterTableField::getTableId, tableId);
wrapper.orderBy("id");
List<DatacenterTableField> fields = fieldsService.list(wrapper);
table.setFields(fields);
return Result.ok(table);
}
@GetMapping("/removeTable")
@SaCheckPermission("/api/v1/datacenterTable/remove")
public Result<Void> removeTable(BigInteger tableId) {
service.removeTable(tableId);
return Result.ok();
}
@GetMapping("/getHeaders")
@SaCheckPermission("/api/v1/datacenterTable/query")
public Result<List<HeaderVo>> getHeaders(BigInteger tableId) {
List<HeaderVo> res = service.getHeaders(tableId);
return Result.ok(res);
}
@GetMapping("/getPageData")
@SaCheckPermission("/api/v1/datacenterTable/query")
public Result<Page<Row>> getPageData(DatacenterQuery where) {
Page<Row> res = service.getPageData(where);
return Result.ok(res);
}
@PostMapping("/saveValue")
@SaCheckPermission("/api/v1/datacenterTable/save")
public Result<Void> saveValue(@RequestParam Map<String, Object> map) {
JSONObject object = new JSONObject(map);
BigInteger tableId = object.getBigInteger("tableId");
LoginAccount account = SaTokenUtil.getLoginAccount();
if (tableId == null) {
return Result.fail(99, "参数错误");
}
service.saveValue(tableId, object, account);
return Result.ok();
}
@PostMapping("/removeValue")
@SaCheckPermission("/api/v1/datacenterTable/remove")
public Result<Void> removeValue(@RequestParam Map<String, Object> map) {
JSONObject object = new JSONObject(map);
BigInteger tableId = object.getBigInteger("tableId");
BigInteger id = object.getBigInteger("id");
if (tableId == null || id == null) {
return Result.fail(99, "参数错误");
}
LoginAccount account = SaTokenUtil.getLoginAccount();
service.removeValue(tableId, id, account);
return Result.ok();
}
/**
* 导入数据
*/
@PostMapping("/importData")
@SaCheckPermission("/api/v1/datacenterTable/save")
public Result<ReadResVo> importData(MultipartFile file, @RequestParam Map<String, Object> map) throws Exception {
Object tableId = map.get("tableId");
DatacenterTable record = service.getById(tableId.toString());
if (record == null) {
throw new RuntimeException("数据表不存在");
}
InputStream is = file.getInputStream();
List<DatacenterTableField> fields = service.getFields(record.getId());
LoginAccount account = SaTokenUtil.getLoginAccount();
ReadDataListener listener = new ReadDataListener(record.getId(), fields, account);
FastExcel.read(is, listener)
.sheet()
.doRead();
int totalCount = listener.getTotalCount();
int errorCount = listener.getErrorCount();
int successCount = listener.getSuccessCount();
List<JSONObject> errorRows = listener.getErrorRows();
return Result.ok(new ReadResVo(successCount, errorCount, totalCount, errorRows));
}
@GetMapping("/getTemplate")
public void getTemplate(BigInteger tableId, HttpServletResponse response) throws Exception {
List<DatacenterTableField> fields = service.getFields(tableId);
// 设置响应内容类型
response.setContentType("application/vnd.openxmlformats-officedocument.spreadsheetml.sheet");
response.setCharacterEncoding("utf-8");
// 设置文件名
String fileName = URLEncoder.encode("导入模板", "UTF-8").replaceAll("\\+", "%20");
response.setHeader("Content-disposition", "attachment;filename*=utf-8''" + fileName + ".xlsx");
// 动态表头数据
List<List<String>> headList = new ArrayList<>();
for (DatacenterTableField field : fields) {
List<String> head = new ArrayList<>();
head.add(field.getFieldName());
headList.add(head);
}
// 写入Excel
EasyExcel.write(response.getOutputStream())
.head(headList)
.sheet("模板")
.doWrite(new ArrayList<>()); // 写入空数据,只生成模板
}
}

View File

@@ -1,41 +0,0 @@
package tech.easyflow.admin.controller.datacenter;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
import tech.easyflow.common.annotation.UsePermission;
import tech.easyflow.common.domain.Result;
import tech.easyflow.common.entity.LoginAccount;
import tech.easyflow.common.satoken.util.SaTokenUtil;
import tech.easyflow.common.web.controller.BaseCurdController;
import tech.easyflow.datacenter.entity.DatacenterTableField;
import tech.easyflow.datacenter.service.DatacenterTableFieldService;
import java.util.Date;
/**
* 控制层。
*
* @author ArkLight
* @since 2025-07-10
*/
@RestController
@RequestMapping("/api/v1/datacenterTableFields")
@UsePermission(moduleName = "/api/v1/datacenterTable")
public class DatacenterTableFieldsController extends BaseCurdController<DatacenterTableFieldService, DatacenterTableField> {
public DatacenterTableFieldsController(DatacenterTableFieldService service) {
super(service);
}
@Override
protected Result onSaveOrUpdateBefore(DatacenterTableField entity, boolean isSave) {
LoginAccount loginUser = SaTokenUtil.getLoginAccount();
if (isSave) {
commonFiled(entity, loginUser.getId(), loginUser.getTenantId(), loginUser.getDeptId());
} else {
entity.setModified(new Date());
entity.setModifiedBy(loginUser.getId());
}
return null;
}
}

View File

@@ -13,6 +13,7 @@ import tech.easyflow.ai.easyagentsflow.entity.NodeInfo;
import tech.easyflow.ai.easyagentsflow.entity.WorkflowCheckStage;
import tech.easyflow.ai.easyagentsflow.service.TinyFlowService;
import tech.easyflow.ai.easyagentsflow.service.WorkflowCheckService;
import tech.easyflow.ai.easyagentsflow.service.WorkflowDatacenterContentService;
import tech.easyflow.ai.entity.Workflow;
import tech.easyflow.ai.service.WorkflowService;
import tech.easyflow.common.domain.Result;
@@ -40,6 +41,8 @@ public class PublicWorkflowController {
private TinyFlowService tinyFlowService;
@Resource
private WorkflowCheckService workflowCheckService;
@Resource
private WorkflowDatacenterContentService workflowDatacenterContentService;
/**
* 通过id或别名获取工作流详情
@@ -123,7 +126,7 @@ public class PublicWorkflowController {
}
workflowCheckService.checkOrThrow(workflow.getContent(), WorkflowCheckStage.PRE_EXECUTE, workflow.getId());
ChainDefinition definition = chainParser.parse(workflow.getContent());
ChainDefinition definition = chainParser.parse(workflowDatacenterContentService.prepareContent(workflow.getContent()));
if (definition == null) {
return Result.fail(2, "节点配置错误,请检查! ");
}

View File

@@ -13,6 +13,7 @@ import tech.easyflow.ai.permission.WorkflowVisibilityQueryHelper;
import tech.easyflow.ai.easyagentsflow.entity.ChainInfo;
import tech.easyflow.ai.easyagentsflow.entity.NodeInfo;
import tech.easyflow.ai.easyagentsflow.entity.WorkflowCheckStage;
import tech.easyflow.ai.easyagentsflow.service.WorkflowDatacenterContentService;
import tech.easyflow.ai.easyagentsflow.service.TinyFlowService;
import tech.easyflow.ai.easyagentsflow.service.WorkflowCheckService;
import tech.easyflow.ai.entity.Workflow;
@@ -53,6 +54,8 @@ public class UcWorkflowController extends BaseCurdController<WorkflowService, Wo
@Resource
private WorkflowCheckService workflowCheckService;
@Resource
private WorkflowDatacenterContentService workflowDatacenterContentService;
@Resource
private WorkflowVisibilityQueryHelper workflowVisibilityQueryHelper;
public UcWorkflowController(WorkflowService service) {
@@ -168,7 +171,7 @@ public class UcWorkflowController extends BaseCurdController<WorkflowService, Wo
}
workflowCheckService.checkOrThrow(workflow.getContent(), WorkflowCheckStage.PRE_EXECUTE, workflow.getId());
ChainDefinition definition = chainParser.parse(workflow.getContent());
ChainDefinition definition = chainParser.parse(workflowDatacenterContentService.prepareContent(workflow.getContent()));
if (definition == null) {
return Result.fail(2, "节点配置错误,请检查! ");
}

View File

@@ -4,6 +4,7 @@ import com.easyagents.flow.core.chain.ChainDefinition;
import com.easyagents.flow.core.chain.repository.ChainDefinitionRepository;
import com.easyagents.flow.core.parser.ChainParser;
import org.springframework.stereotype.Component;
import tech.easyflow.ai.easyagentsflow.service.WorkflowDatacenterContentService;
import tech.easyflow.ai.entity.Workflow;
import tech.easyflow.ai.service.WorkflowService;
@@ -16,11 +17,13 @@ public class ChainDefinitionRepositoryImpl implements ChainDefinitionRepository
private WorkflowService workflowService;
@Resource
private ChainParser chainParser;
@Resource
private WorkflowDatacenterContentService workflowDatacenterContentService;
@Override
public ChainDefinition getChainDefinitionById(String id) {
Workflow workflow = workflowService.getById(id);
String json = workflow.getContent();
String json = workflowDatacenterContentService.prepareContent(workflow.getContent());
ChainDefinition chainDefinition = chainParser.parse(json);
chainDefinition.setId(workflow.getId().toString());
chainDefinition.setName(workflow.getEnglishName());

View File

@@ -65,14 +65,12 @@ public class TinyFlowConfigService {
MakeFileNodeParser makeFileNodeParser = new MakeFileNodeParser();
// 插件
PluginToolNodeParser pluginToolNodeParser = new PluginToolNodeParser();
// SQL查询
SqlNodeParser sqlNodeParser = new SqlNodeParser();
// 下载文件节点
DownloadNodeParser downloadNodeParser = new DownloadNodeParser();
// 保存数据节点
SaveToDatacenterNodeParser saveDaveParser = new SaveToDatacenterNodeParser();
SaveDatasetNodeParser saveDatasetNodeParser = new SaveDatasetNodeParser();
// 查询数据节点
SearchDatacenterNodeParser searchDatacenterNodeParser = new SearchDatacenterNodeParser();
SearchDatasetNodeParser searchDatasetNodeParser = new SearchDatasetNodeParser();
// 工作流节点
WorkflowNodeParser workflowNodeParser = new WorkflowNodeParser();
// 条件判断节点
@@ -81,10 +79,9 @@ public class TinyFlowConfigService {
chainParser.addNodeParser(docNodeParser.getNodeName(), docNodeParser);
chainParser.addNodeParser(makeFileNodeParser.getNodeName(), makeFileNodeParser);
chainParser.addNodeParser(pluginToolNodeParser.getNodeName(), pluginToolNodeParser);
chainParser.addNodeParser(sqlNodeParser.getNodeName(), sqlNodeParser);
chainParser.addNodeParser(downloadNodeParser.getNodeName(), downloadNodeParser);
chainParser.addNodeParser(saveDaveParser.getNodeName(), saveDaveParser);
chainParser.addNodeParser(searchDatacenterNodeParser.getNodeName(), searchDatacenterNodeParser);
chainParser.addNodeParser(saveDatasetNodeParser.getNodeName(), saveDatasetNodeParser);
chainParser.addNodeParser(searchDatasetNodeParser.getNodeName(), searchDatasetNodeParser);
chainParser.addNodeParser(workflowNodeParser.getNodeName(), workflowNodeParser);
chainParser.addNodeParser(conditionNodeParser.getNodeName(), conditionNodeParser);
}

View File

@@ -12,15 +12,30 @@ import tech.easyflow.ai.easyagentsflow.entity.WorkflowCheckStage;
import tech.easyflow.ai.entity.Workflow;
import tech.easyflow.ai.service.WorkflowService;
import tech.easyflow.common.web.exceptions.BusinessException;
import tech.easyflow.datacenter.execution.model.DatacenterSchemaResponse;
import javax.annotation.Resource;
import java.math.BigInteger;
import java.util.*;
import java.util.ArrayDeque;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.Deque;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedHashMap;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.stream.Collectors;
@Service
public class WorkflowCheckService {
private static final String LEVEL_ERROR = "ERROR";
private static final String LEVEL_WARNING = "WARNING";
private static final String TYPE_START = "startNode";
private static final String TYPE_END = "endNode";
private static final String TYPE_LOOP = "loopNode";
@@ -30,6 +45,8 @@ public class WorkflowCheckService {
private WorkflowService workflowService;
@Resource
private ChainParser chainParser;
@Resource
private WorkflowDatacenterContentService workflowDatacenterContentService;
public WorkflowCheckResult checkWorkflow(BigInteger workflowId, WorkflowCheckStage stage) {
if (workflowId == null) {
@@ -177,9 +194,91 @@ public class WorkflowCheckService {
parsedWorkflow.nodes = nodes;
parsedWorkflow.edges = edges;
parsedWorkflow.nodeMap = nodeMap;
checkDatacenterNodes(parsedWorkflow, issues, issueKeys);
return parsedWorkflow;
}
private void checkDatacenterNodes(ParsedWorkflow parsed, List<WorkflowCheckIssue> issues, Set<String> issueKeys) {
for (NodeView node : parsed.nodes) {
if (node == null) {
continue;
}
if (workflowDatacenterContentService.isSearchDatasetNode(node.type)) {
checkSearchDatasetNode(node, issues, issueKeys);
continue;
}
if (workflowDatacenterContentService.isSaveDatasetNode(node.type)) {
checkSaveDatasetNode(node, issues, issueKeys);
continue;
}
if (workflowDatacenterContentService.isLlmNode(node.type)) {
checkLlmQueryContext(node, parsed, issues, issueKeys);
}
}
}
private void checkSearchDatasetNode(NodeView node,
List<WorkflowCheckIssue> issues,
Set<String> issueKeys) {
try {
workflowDatacenterContentService.requireSearchDatasetRef(node.data);
} catch (BusinessException e) {
addIssue(issues, issueKeys, "SEARCH_DATASET_INVALID", e.getMessage(), node.id, null, node.name);
} catch (Exception e) {
addIssue(issues, issueKeys, "SEARCH_DATASET_INVALID",
"查询数据节点校验失败: " + shortError(e), node.id, null, node.name);
}
}
private void checkSaveDatasetNode(NodeView node,
List<WorkflowCheckIssue> issues,
Set<String> issueKeys) {
try {
workflowDatacenterContentService.requireSaveDatasetRef(node.data);
} catch (BusinessException e) {
addIssue(issues, issueKeys, "SAVE_DATASET_INVALID", e.getMessage(), node.id, null, node.name);
} catch (Exception e) {
addIssue(issues, issueKeys, "SAVE_DATASET_INVALID",
"写入数据节点校验失败: " + shortError(e), node.id, null, node.name);
}
}
private void checkLlmQueryContext(NodeView node,
ParsedWorkflow parsed,
List<WorkflowCheckIssue> issues,
Set<String> issueKeys) {
if (node.data == null) {
return;
}
Object rawNodeIds = node.data.get("queryContextNodeIds");
if (rawNodeIds == null) {
return;
}
if (!(rawNodeIds instanceof JSONArray nodeIds)) {
addIssue(issues, issueKeys, "LLM_QUERY_CONTEXT_INVALID",
"查询上下文配置无效,请重新选择查询数据节点", node.id, null, node.name);
return;
}
for (int i = 0; i < nodeIds.size(); i++) {
String refNodeId = trimToNull(nodeIds.getString(i));
if (!StringUtils.hasText(refNodeId)) {
addIssue(issues, issueKeys, "LLM_QUERY_CONTEXT_EMPTY",
"查询上下文配置无效,请重新选择查询数据节点", node.id, null, node.name);
continue;
}
NodeView refNode = parsed.nodeMap.get(refNodeId);
if (refNode == null) {
addIssue(issues, issueKeys, "LLM_QUERY_CONTEXT_NOT_FOUND",
"查询上下文节点不存在: " + refNodeId, node.id, null, node.name);
continue;
}
if (!workflowDatacenterContentService.isSearchDatasetNode(refNode.type)) {
addIssue(issues, issueKeys, "LLM_QUERY_CONTEXT_TYPE_INVALID",
"查询上下文只能选择查询数据节点", node.id, null, node.name);
}
}
}
private void runStrictChecks(String content, ParsedWorkflow parsed, BigInteger currentWorkflowId,
List<WorkflowCheckIssue> issues, Set<String> issueKeys) {
if (parsed.nodes.isEmpty()) {
@@ -190,7 +289,8 @@ public class WorkflowCheckService {
}
try {
Object definition = chainParser.parse(content);
String preparedContent = workflowDatacenterContentService.prepareContent(content);
Object definition = chainParser.parse(preparedContent);
if (definition == null) {
addIssue(issues, issueKeys, "PARSE_NULL", "预执行校验失败:节点配置错误,请检查", null, null, null);
}
@@ -606,31 +706,47 @@ public class WorkflowCheckService {
result.setStage(stage);
result.setIssues(issues);
result.setIssueCount(issues.size());
result.setPassed(issues.isEmpty());
result.setPassed(issues.stream().noneMatch(issue -> LEVEL_ERROR.equalsIgnoreCase(issue.getLevel())));
return result;
}
private void throwIfFailed(WorkflowCheckResult result) {
if (result == null || result.isPassed()) {
if (result == null) {
return;
}
String summary = result.getIssues().stream()
List<WorkflowCheckIssue> errorIssues = result.getIssues().stream()
.filter(issue -> LEVEL_ERROR.equalsIgnoreCase(issue.getLevel()))
.collect(Collectors.toList());
if (errorIssues.isEmpty()) {
return;
}
String summary = errorIssues.stream()
.limit(5)
.map(WorkflowCheckIssue::getMessage)
.collect(Collectors.joining(""));
if (result.getIssueCount() > 5) {
if (errorIssues.size() > 5) {
summary = summary + ";等";
}
throw new BusinessException("工作流校验未通过(" + result.getStage() + "),共 " + result.getIssueCount() + " 项:" + summary);
throw new BusinessException("工作流校验未通过(" + result.getStage() + "),共 " + errorIssues.size() + " 项:" + summary);
}
private void addIssue(List<WorkflowCheckIssue> issues, Set<String> issueKeys, String code,
String message, String nodeId, String edgeId, String nodeName) {
addIssue(issues, issueKeys, code, LEVEL_ERROR, message, nodeId, edgeId, nodeName);
}
private void addWarning(List<WorkflowCheckIssue> issues, Set<String> issueKeys, String code,
String message, String nodeId, String edgeId, String nodeName) {
addIssue(issues, issueKeys, code, LEVEL_WARNING, message, nodeId, edgeId, nodeName);
}
private void addIssue(List<WorkflowCheckIssue> issues, Set<String> issueKeys, String code, String level,
String message, String nodeId, String edgeId, String nodeName) {
String key = code + "|" + safe(nodeId) + "|" + safe(edgeId) + "|" + message;
if (!issueKeys.add(key)) {
return;
}
issues.add(new WorkflowCheckIssue(code, LEVEL_ERROR, message, nodeId, edgeId, nodeName));
issues.add(new WorkflowCheckIssue(code, level, message, nodeId, edgeId, nodeName));
}
private String extractNodeName(JSONObject nodeJson, JSONObject data, String fallback) {

View File

@@ -0,0 +1,349 @@
package tech.easyflow.ai.easyagentsflow.service;
import com.alibaba.fastjson2.JSON;
import com.alibaba.fastjson2.JSONArray;
import com.alibaba.fastjson2.JSONObject;
import org.springframework.stereotype.Service;
import org.springframework.util.StringUtils;
import tech.easyflow.common.constant.enums.EnumFieldType;
import tech.easyflow.common.web.exceptions.BusinessException;
import tech.easyflow.datacenter.entity.DatacenterTable;
import tech.easyflow.datacenter.entity.DatacenterTableField;
import tech.easyflow.datacenter.execution.model.DatacenterSchemaResponse;
import tech.easyflow.datacenter.execution.model.DatasetRef;
import tech.easyflow.datacenter.execution.service.DatacenterDatasetQueryService;
import tech.easyflow.datacenter.meta.entity.DatacenterCatalog;
import tech.easyflow.datacenter.meta.entity.DatacenterSource;
import tech.easyflow.datacenter.meta.service.DatacenterDatasetRegistryService;
import javax.annotation.Resource;
import java.math.BigInteger;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.LinkedHashMap;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
@Service
public class WorkflowDatacenterContentService {
public static final String SEARCH_NODE_TYPE = "search-dataset-node";
public static final String SAVE_NODE_TYPE = "save-dataset-node";
public static final String LLM_NODE_TYPE = "llmNode";
public static final String QUERY_DATA_CONTEXT = "queryDataContext";
public static final String SEARCH_SOURCE_MISSING_MESSAGE = "查询数据节点未选择连接服务";
public static final String SEARCH_SQL_MISSING_MESSAGE = "查询数据节点未设置 SQL";
public static final String SAVE_EXPIRED_MESSAGE = "写入数据节点配置已过期,请重新选择已接入表";
public static final String INVALID_QUERY_CONTEXT_MESSAGE = "查询上下文配置无效,请重新选择查询数据节点";
private static final String QUERY_CONTEXT_PROMPT = """
你是为工作流中的查询数据节点生成只读 SQL 的生成器,你的职责是返回可直接执行的 SQL并且你只能输出 SQL。
必须严格遵守以下规则:
1. 只能从下面给出的连接摘要中选择最合适的表。
2. 只能使用摘要中给出的字段名,不要虚构表名或字段名。
3. 只输出 SQL不要输出解释、注释、Markdown、JSON 或多余文本。
4. 只能生成只读 SELECT SQL允许 WITH、JOIN、子查询、聚合、分组、排序。
5. 不要生成 INSERT、UPDATE、DELETE、DDL、多语句、存储过程调用。
6. 优先使用逻辑表名和逻辑字段名不要输出物理表名、JDBC、驱动信息。
7. 如果存在重名表,请使用 catalog.table 形式消除歧义。
以下是可用的连接摘要:
""";
@Resource
private DatacenterDatasetQueryService queryService;
@Resource
private DatacenterDatasetRegistryService registryService;
public boolean isSearchDatasetNode(String nodeType) {
return SEARCH_NODE_TYPE.equals(nodeType);
}
public boolean isSaveDatasetNode(String nodeType) {
return SAVE_NODE_TYPE.equals(nodeType);
}
public boolean isLlmNode(String nodeType) {
return LLM_NODE_TYPE.equals(nodeType);
}
public String prepareContent(String content) {
if (!StringUtils.hasText(content)) {
throw new BusinessException("工作流内容不能为空");
}
Object parsed;
try {
parsed = JSON.parse(content);
} catch (Exception e) {
throw new BusinessException("工作流内容不是合法JSON: " + shortError(e));
}
if (!(parsed instanceof JSONObject root)) {
throw new BusinessException("工作流内容必须是JSON对象");
}
JSONObject copy = JSON.parseObject(root.toJSONString());
prepareRoot(copy);
return copy.toJSONString();
}
public JSONObject prepareRoot(JSONObject root) {
if (root == null) {
throw new BusinessException("工作流内容不能为空");
}
JSONArray nodes = root.getJSONArray("nodes");
if (nodes == null || nodes.isEmpty()) {
return root;
}
Map<String, JSONObject> nodeMap = buildNodeMap(nodes);
for (int i = 0; i < nodes.size(); i++) {
JSONObject node = nodes.getJSONObject(i);
if (node == null) {
continue;
}
String nodeType = node.getString("type");
JSONObject data = node.getJSONObject("data");
if (isSearchDatasetNode(nodeType)) {
requireSearchDatasetRef(data);
} else if (isSaveDatasetNode(nodeType)) {
requireSaveDatasetRef(data);
}
}
for (int i = 0; i < nodes.size(); i++) {
JSONObject node = nodes.getJSONObject(i);
if (node == null || !isLlmNode(node.getString("type"))) {
continue;
}
injectQueryDataContext(node.getJSONObject("data"), nodeMap);
}
return root;
}
public boolean hasSaveLegacyFields(JSONObject data) {
return hasAnyField(data, "tableId", "sourceId", "catalogId", "versionId");
}
public DatasetRef readDatasetRef(JSONObject data) {
return data == null ? null : data.getObject("datasetRef", DatasetRef.class);
}
public DatasetRef requireSearchDatasetRef(JSONObject data) {
DatasetRef datasetRef = readDatasetRef(data);
if (datasetRef == null || datasetRef.getSourceId() == null) {
throw new BusinessException(SEARCH_SOURCE_MISSING_MESSAGE);
}
String querySql = data == null ? null : trimToNull(data.getString("querySql"));
if (!StringUtils.hasText(querySql)) {
throw new BusinessException(SEARCH_SQL_MISSING_MESSAGE);
}
return datasetRef;
}
public DatasetRef requireSaveDatasetRef(JSONObject data) {
if (hasSaveLegacyFields(data)) {
throw new BusinessException(SAVE_EXPIRED_MESSAGE);
}
DatasetRef datasetRef = readDatasetRef(data);
if (datasetRef == null || datasetRef.getTableId() == null) {
throw new BusinessException(SAVE_EXPIRED_MESSAGE);
}
return datasetRef;
}
public DatacenterSchemaResponse loadSchema(DatasetRef datasetRef, Map<BigInteger, DatacenterSchemaResponse> schemaCache) {
if (datasetRef == null || datasetRef.getTableId() == null) {
throw new BusinessException("缺少已接入表配置");
}
BigInteger tableId = datasetRef.getTableId();
if (schemaCache != null && schemaCache.containsKey(tableId)) {
return schemaCache.get(tableId);
}
DatacenterSchemaResponse schema = queryService.getSchema(copyDatasetRef(datasetRef));
if (schemaCache != null) {
schemaCache.put(tableId, schema);
}
return schema;
}
public JSONObject buildSourceSummary(DatasetRef datasetRef) {
if (datasetRef == null || datasetRef.getSourceId() == null) {
throw new BusinessException(SEARCH_SOURCE_MISSING_MESSAGE);
}
DatacenterSource source = registryService.getSourceRequired(datasetRef.getSourceId());
List<DatacenterTable> managedTables = registryService.listManagedTables(datasetRef.getSourceId(), datasetRef.getCatalogId());
managedTables.sort(Comparator.comparing(table -> table.getTableName() == null ? "" : table.getTableName()));
JSONObject sourceSummary = new JSONObject();
sourceSummary.put("sourceName", source.getSourceName());
sourceSummary.put("sourceType", source.getSourceType());
JSONArray tables = new JSONArray();
for (DatacenterTable table : managedTables) {
DatacenterTable fullTable = registryService.getTableWithFields(table.getId());
DatacenterCatalog catalog = registryService.getCatalogById(fullTable.getCatalogId());
if (StringUtils.hasText(datasetRef.getCatalogName())
&& (catalog == null || !datasetRef.getCatalogName().equals(catalog.getCatalogName()))) {
continue;
}
JSONObject tableSummary = new JSONObject();
tableSummary.put("catalogName", catalog == null ? null : catalog.getCatalogName());
tableSummary.put("tableName", fullTable.getTableName());
tableSummary.put("tableDesc", fullTable.getTableDesc());
JSONArray fields = new JSONArray();
if (fullTable.getFields() != null) {
for (DatacenterTableField field : fullTable.getFields()) {
JSONObject fieldSummary = new JSONObject();
fieldSummary.put("fieldName", field.getFieldName());
fieldSummary.put("fieldDesc", field.getFieldDesc());
fieldSummary.put("fieldType", resolveFieldType(field));
fields.add(fieldSummary);
}
}
tableSummary.put("fields", fields);
tables.add(tableSummary);
}
sourceSummary.put("tables", tables);
return sourceSummary;
}
private void injectQueryDataContext(JSONObject data, Map<String, JSONObject> nodeMap) {
if (data == null) {
return;
}
JSONArray nodeIds = data.getJSONArray("queryContextNodeIds");
if (nodeIds == null || nodeIds.isEmpty()) {
removeQueryDataContextParameter(data);
return;
}
Map<BigInteger, JSONObject> sourceSummaries = new LinkedHashMap<>();
Set<String> visitedNodeIds = new LinkedHashSet<>();
for (int i = 0; i < nodeIds.size(); i++) {
String nodeId = trimToNull(nodeIds.getString(i));
if (!StringUtils.hasText(nodeId) || !visitedNodeIds.add(nodeId)) {
continue;
}
JSONObject targetNode = nodeMap.get(nodeId);
if (targetNode == null || !isSearchDatasetNode(targetNode.getString("type"))) {
throw new BusinessException(INVALID_QUERY_CONTEXT_MESSAGE);
}
DatasetRef datasetRef = requireSearchDatasetRef(targetNode.getJSONObject("data"));
sourceSummaries.putIfAbsent(datasetRef.getSourceId(), buildSourceSummary(datasetRef));
}
String contextValue = QUERY_CONTEXT_PROMPT + "\n" + JSON.toJSONString(new ArrayList<>(sourceSummaries.values()));
upsertQueryDataContextParameter(data, contextValue);
}
private String resolveFieldType(DatacenterTableField field) {
if (field == null) {
return null;
}
if (StringUtils.hasText(field.getJdbcType())) {
return field.getJdbcType();
}
try {
EnumFieldType enumFieldType = EnumFieldType.getByCode(field.getFieldType());
if (enumFieldType != null) {
return enumFieldType.getText();
}
} catch (Exception ignored) {
}
return field.getFieldType() == null ? null : String.valueOf(field.getFieldType());
}
private void upsertQueryDataContextParameter(JSONObject data, String contextValue) {
JSONArray parameters = data.getJSONArray("parameters");
if (parameters == null) {
parameters = new JSONArray();
data.put("parameters", parameters);
}
for (int i = parameters.size() - 1; i >= 0; i--) {
JSONObject parameter = parameters.getJSONObject(i);
if (parameter != null && QUERY_DATA_CONTEXT.equals(parameter.getString("name"))) {
parameters.remove(i);
}
}
JSONObject parameter = new JSONObject();
parameter.put("id", QUERY_DATA_CONTEXT);
parameter.put("name", QUERY_DATA_CONTEXT);
parameter.put("title", "查询上下文");
parameter.put("description", "数据查询规则与连接表摘要");
parameter.put("dataType", "String");
parameter.put("refType", "fixed");
parameter.put("value", contextValue);
parameter.put("required", false);
parameter.put("nameDisabled", true);
parameter.put("dataTypeDisabled", true);
parameter.put("deleteDisabled", true);
parameters.add(parameter);
}
private void removeQueryDataContextParameter(JSONObject data) {
if (data == null) {
return;
}
JSONArray parameters = data.getJSONArray("parameters");
if (parameters == null || parameters.isEmpty()) {
return;
}
for (int i = parameters.size() - 1; i >= 0; i--) {
JSONObject parameter = parameters.getJSONObject(i);
if (parameter != null && QUERY_DATA_CONTEXT.equals(parameter.getString("name"))) {
parameters.remove(i);
}
}
}
private Map<String, JSONObject> buildNodeMap(JSONArray nodes) {
Map<String, JSONObject> nodeMap = new LinkedHashMap<>();
for (int i = 0; i < nodes.size(); i++) {
JSONObject node = nodes.getJSONObject(i);
if (node == null) {
continue;
}
String nodeId = trimToNull(node.getString("id"));
if (StringUtils.hasText(nodeId)) {
nodeMap.put(nodeId, node);
}
}
return nodeMap;
}
private DatasetRef copyDatasetRef(DatasetRef datasetRef) {
DatasetRef copy = new DatasetRef();
copy.setSourceId(datasetRef.getSourceId());
copy.setCatalogId(datasetRef.getCatalogId());
copy.setCatalogName(datasetRef.getCatalogName());
copy.setTableId(datasetRef.getTableId());
copy.setTableName(datasetRef.getTableName());
copy.setVersionId(datasetRef.getVersionId());
return copy;
}
private boolean hasAnyField(JSONObject data, String... fieldNames) {
if (data == null || fieldNames == null) {
return false;
}
for (String fieldName : fieldNames) {
if (data.containsKey(fieldName)) {
return true;
}
}
return false;
}
private String trimToNull(String value) {
if (!StringUtils.hasText(value)) {
return null;
}
return value.trim();
}
private String shortError(Throwable throwable) {
if (throwable == null) {
return "unknown";
}
String message = throwable.getMessage();
if (!StringUtils.hasText(message)) {
return throwable.getClass().getSimpleName();
}
return message.length() > 120 ? message.substring(0, 120) + "..." : message;
}
}

View File

@@ -0,0 +1,75 @@
package tech.easyflow.ai.node;
import com.alibaba.fastjson2.JSONArray;
import com.alibaba.fastjson2.JSONObject;
import com.easyagents.flow.core.chain.Chain;
import com.easyagents.flow.core.node.BaseNode;
import com.mybatisflex.core.tenant.TenantManager;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import tech.easyflow.ai.utils.WorkFlowUtil;
import tech.easyflow.common.entity.LoginAccount;
import tech.easyflow.common.util.SpringContextUtil;
import tech.easyflow.datacenter.execution.model.DatasetRef;
import tech.easyflow.datacenter.execution.service.DatacenterDatasetQueryService;
import tech.easyflow.datacenter.execution.service.DatacenterDatasetWriteService;
import java.util.HashMap;
import java.util.Map;
public class SaveDatasetNode extends BaseNode {
private static final Logger log = LoggerFactory.getLogger(SaveDatasetNode.class);
private DatasetRef datasetRef;
public SaveDatasetNode() {
}
public SaveDatasetNode(DatasetRef datasetRef) {
this.datasetRef = datasetRef;
}
@Override
public Map<String, Object> execute(Chain chain) {
Map<String, Object> state = chain.getState().resolveParameters(this);
JSONObject payload = new JSONObject(state);
JSONArray saveList = payload.getJSONArray("saveList");
if (saveList == null || saveList.isEmpty()) {
throw new RuntimeException("saveList 不能为空");
}
LoginAccount account = WorkFlowUtil.getOperator(chain);
DatacenterDatasetWriteService writeService = SpringContextUtil.getBean(DatacenterDatasetWriteService.class);
DatacenterDatasetQueryService queryService = SpringContextUtil.getBean(DatacenterDatasetQueryService.class);
int successRows = 0;
try {
TenantManager.ignoreTenantCondition();
for (Object item : saveList) {
JSONObject row = item instanceof JSONObject json ? json : JSONObject.from(item);
writeService.saveRow(datasetRef, row, account);
successRows++;
}
var schema = queryService.getSchema(datasetRef);
Map<String, Object> result = new HashMap<>();
result.put("successRows", successRows);
result.put("source", schema.getSource());
result.put("catalog", schema.getCatalog());
result.put("table", schema.getTable());
result.put("version", datasetRef.getVersionId());
return result;
} catch (Exception ex) {
log.error("工作流保存数据到统一数据集失败datasetRef={}", datasetRef, ex);
throw ex;
} finally {
TenantManager.restoreTenantCondition();
}
}
public DatasetRef getDatasetRef() {
return datasetRef;
}
public void setDatasetRef(DatasetRef datasetRef) {
this.datasetRef = datasetRef;
}
}

View File

@@ -0,0 +1,41 @@
package tech.easyflow.ai.node;
import com.alibaba.fastjson.JSONObject;
import com.easyagents.flow.core.node.BaseNode;
import com.easyagents.flow.core.parser.BaseNodeParser;
import tech.easyflow.common.web.exceptions.BusinessException;
import tech.easyflow.datacenter.execution.model.DatasetRef;
public class SaveDatasetNodeParser extends BaseNodeParser {
private static final String EXPIRED_MESSAGE = "写入数据节点配置已过期,请重新选择已接入表";
@Override
protected BaseNode doParse(JSONObject root, JSONObject data, JSONObject tinyflow) {
if (data == null) {
throw new BusinessException(EXPIRED_MESSAGE);
}
if (hasLegacyFields(data)) {
throw new BusinessException(EXPIRED_MESSAGE);
}
DatasetRef datasetRef = data.getObject("datasetRef", DatasetRef.class);
if (datasetRef == null || datasetRef.getTableId() == null) {
throw new BusinessException(EXPIRED_MESSAGE);
}
return new SaveDatasetNode(datasetRef);
}
public String getNodeName() {
return "save-dataset-node";
}
private boolean hasLegacyFields(JSONObject data) {
if (data == null) {
return false;
}
return data.containsKey("tableId")
|| data.containsKey("sourceId")
|| data.containsKey("catalogId")
|| data.containsKey("versionId");
}
}

View File

@@ -1,75 +0,0 @@
package tech.easyflow.ai.node;
import com.alibaba.fastjson2.JSONArray;
import com.alibaba.fastjson2.JSONObject;
import com.mybatisflex.core.tenant.TenantManager;
import com.easyagents.flow.core.chain.Chain;
import com.easyagents.flow.core.node.BaseNode;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import tech.easyflow.ai.utils.WorkFlowUtil;
import tech.easyflow.common.entity.LoginAccount;
import tech.easyflow.common.util.SpringContextUtil;
import tech.easyflow.datacenter.service.DatacenterTableService;
import java.math.BigInteger;
import java.util.HashMap;
import java.util.Map;
public class SaveToDatacenterNode extends BaseNode {
private static final Logger log = LoggerFactory.getLogger(SaveToDatacenterNode.class);
private BigInteger tableId;
public SaveToDatacenterNode() {
}
public SaveToDatacenterNode(BigInteger tableId) {
this.tableId = tableId;
}
@Override
public Map<String, Object> execute(Chain chain) {
Map<String, Object> map = chain.getState().resolveParameters(this);
JSONObject json = new JSONObject(map);
Map<String, Object> res = new HashMap<>();
// 默认为未知来源
LoginAccount account = WorkFlowUtil.getOperator(chain);
DatacenterTableService service = SpringContextUtil.getBean(DatacenterTableService.class);
JSONArray saveList = json.getJSONArray("saveList");
int successRows = 0;
for (Object object : saveList) {
JSONObject obj = new JSONObject((com.alibaba.fastjson.JSONObject) object);
obj.put("table_id", tableId);
try {
TenantManager.ignoreTenantCondition();
service.saveValue(tableId, obj, account);
} catch (Exception e) {
log.error("工作流保存数据到数据中枢失败表ID{},具体值:{}", tableId, obj, e);
throw e;
} finally {
TenantManager.restoreTenantCondition();
}
successRows++;
}
res.put("successRows", successRows);
return res;
}
public BigInteger getTableId() {
return tableId;
}
public void setTableId(BigInteger tableId) {
this.tableId = tableId;
}
}

View File

@@ -1,23 +0,0 @@
package tech.easyflow.ai.node;
import com.alibaba.fastjson.JSONObject;
import com.easyagents.flow.core.node.BaseNode;
import com.easyagents.flow.core.parser.BaseNodeParser;
import java.math.BigInteger;
public class SaveToDatacenterNodeParser extends BaseNodeParser {
@Override
protected BaseNode doParse(JSONObject root, JSONObject data, JSONObject tinyflow) {
BigInteger tableId = data.getBigInteger("tableId");
if (tableId == null) {
throw new RuntimeException("请选择数据表");
}
return new SaveToDatacenterNode(tableId);
}
public String getNodeName() {
return "save-to-datacenter-node";
}
}

View File

@@ -1,143 +0,0 @@
package tech.easyflow.ai.node;
import com.easyagents.core.util.StringUtil;
import com.mybatisflex.core.paginate.Page;
import com.mybatisflex.core.row.Row;
import com.mybatisflex.core.tenant.TenantManager;
import com.easyagents.flow.core.chain.Chain;
import com.easyagents.flow.core.chain.Parameter;
import com.easyagents.flow.core.node.BaseNode;
import net.sf.jsqlparser.expression.Expression;
import net.sf.jsqlparser.parser.CCJSqlParserUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import tech.easyflow.common.entity.DatacenterQuery;
import tech.easyflow.common.util.SpringContextUtil;
import tech.easyflow.datacenter.entity.DatacenterTableField;
import tech.easyflow.datacenter.service.DatacenterTableService;
import tech.easyflow.datacenter.utils.WhereConditionSecurityChecker;
import java.math.BigInteger;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
public class SearchDatacenterNode extends BaseNode {
private static final Logger log = LoggerFactory.getLogger(SearchDatacenterNode.class);
private BigInteger tableId;
private String where;
private Long limit;
public SearchDatacenterNode() {
}
public SearchDatacenterNode(BigInteger tableId, String where, Long limit) {
this.tableId = tableId;
this.where = where;
this.limit = limit;
}
@Override
public Map<String, Object> execute(Chain chain) {
Map<String, Object> map = chain.getState().resolveParameters(this);
Map<String, Object> res = new HashMap<>();
long limitNum = 10;
if (limit != null) {
limitNum = Long.parseLong(limit.toString());
}
DatacenterTableService service = SpringContextUtil.getBean(DatacenterTableService.class);
DatacenterQuery condition = new DatacenterQuery();
condition.setTableId(tableId);
condition.setPageNumber(1L);
condition.setPageSize(limitNum);
// 组合查询条件
if (where != null) {
setCondition(where, condition, map);
}
try {
TenantManager.ignoreTenantCondition();
Page<Row> pageData = service.getPageData(condition);
String key = "rows";
List<Parameter> outputDefs = getOutputDefs();
if (outputDefs != null && !outputDefs.isEmpty()) {
String defName = outputDefs.get(0).getName();
if (StringUtil.hasText(defName)) key = defName;
}
res.put(key, pageData.getRecords());
} finally {
TenantManager.restoreTenantCondition();
}
return res;
}
public BigInteger getTableId() {
return tableId;
}
public void setTableId(BigInteger tableId) {
this.tableId = tableId;
}
public String getWhere() {
return where;
}
public void setWhere(String where) {
this.where = where;
}
public Long getLimit() {
return limit;
}
public void setLimit(Long limit) {
this.limit = limit;
}
private void setCondition(String where, DatacenterQuery condition, Map<String, Object> params) {
// 条件封装
Pattern pattern = Pattern.compile("\\{\\{(.+?)\\}\\}");
Matcher matcher = pattern.matcher(where);
StringBuffer result = new StringBuffer();
while (matcher.find()) {
String key = matcher.group(1);
Object value = params.get(key);
if (value == null) {
throw new RuntimeException("参数" + key + "不存在");
}
String replacement = value.toString();
matcher.appendReplacement(result, "'" + replacement + "'");
}
matcher.appendTail(result);
try {
Expression expression = CCJSqlParserUtil.parseCondExpression(result.toString());
if (expression != null) {
WhereConditionSecurityChecker checker = new WhereConditionSecurityChecker();
DatacenterTableService service = SpringContextUtil.getBean(DatacenterTableService.class);
List<DatacenterTableField> fields = service.getFields(tableId);
Set<String> columns = fields.stream().map(DatacenterTableField::getFieldName).collect(Collectors.toSet());
columns.add("id");
columns.add("created");
columns.add("modified");
columns.add("created_by");
columns.add("modified_by");
checker.checkConditionSafety(expression, columns);
condition.setWhere(expression.toString());
}
} catch (Exception e) {
log.error("WHERE SQL解析错误", e);
throw new RuntimeException(e);
}
}
}

View File

@@ -1,25 +0,0 @@
package tech.easyflow.ai.node;
import com.alibaba.fastjson.JSONObject;
import com.easyagents.flow.core.node.BaseNode;
import com.easyagents.flow.core.parser.BaseNodeParser;
import java.math.BigInteger;
public class SearchDatacenterNodeParser extends BaseNodeParser {
@Override
protected BaseNode doParse(JSONObject root, JSONObject data, JSONObject tinyflow) {
BigInteger tableId = data.getBigInteger("tableId");
String where = data.getString("where");
Long limit = data.getLong("limit");
if (tableId == null) {
throw new RuntimeException("请选择数据表");
}
return new SearchDatacenterNode(tableId,where,limit);
}
public String getNodeName() {
return "search-datacenter-node";
}
}

View File

@@ -0,0 +1,120 @@
package tech.easyflow.ai.node;
import com.easyagents.core.util.StringUtil;
import com.easyagents.flow.core.chain.Chain;
import com.easyagents.flow.core.chain.Parameter;
import com.easyagents.flow.core.node.BaseNode;
import com.mybatisflex.core.row.Row;
import com.mybatisflex.core.tenant.TenantManager;
import tech.easyflow.common.util.SpringContextUtil;
import tech.easyflow.common.web.exceptions.BusinessException;
import tech.easyflow.datacenter.execution.model.DatacenterSqlQueryRequest;
import tech.easyflow.datacenter.execution.model.DatasetRef;
import tech.easyflow.datacenter.execution.service.DatacenterDatasetQueryService;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
public class SearchDatasetNode extends BaseNode {
private static final Pattern PARAM_PATTERN = Pattern.compile("\\{\\{(.+?)\\}\\}");
private DatasetRef datasetRef;
private String querySql;
public SearchDatasetNode() {
}
public SearchDatasetNode(DatasetRef datasetRef) {
this.datasetRef = datasetRef;
}
public SearchDatasetNode(DatasetRef datasetRef, String querySql) {
this.datasetRef = datasetRef;
this.querySql = querySql;
}
@Override
public Map<String, Object> execute(Chain chain) {
Map<String, Object> params = chain.getState().resolveParameters(this);
DatacenterDatasetQueryService queryService = SpringContextUtil.getBean(DatacenterDatasetQueryService.class);
DatacenterSqlQueryRequest request = buildRuntimeRequest(params);
Map<String, Object> result = new HashMap<>();
try {
TenantManager.ignoreTenantCondition();
List<Row> rows = queryService.queryBySql(request);
result.put(resolveOutputKey("data"), rows);
return result;
} finally {
TenantManager.restoreTenantCondition();
}
}
private DatacenterSqlQueryRequest buildRuntimeRequest(Map<String, Object> params) {
DatacenterSqlQueryRequest request = new DatacenterSqlQueryRequest();
request.setDatasetRef(copyDatasetRef());
request.setSql(resolveQuerySql(params));
return request;
}
private String resolveQuerySql(Map<String, Object> params) {
String sql = resolveTemplateString(querySql, params);
if (!StringUtil.hasText(sql)) {
throw new BusinessException("查询数据节点未设置 SQL");
}
return sql.trim();
}
private DatasetRef copyDatasetRef() {
DatasetRef copy = new DatasetRef();
copy.setSourceId(datasetRef == null ? null : datasetRef.getSourceId());
copy.setCatalogId(datasetRef == null ? null : datasetRef.getCatalogId());
copy.setCatalogName(datasetRef == null ? null : datasetRef.getCatalogName());
copy.setTableId(null);
copy.setTableName(null);
copy.setVersionId(null);
return copy;
}
private String resolveTemplateString(String text, Map<String, Object> params) {
if (!StringUtil.hasText(text) || params == null || params.isEmpty()) {
return text;
}
Matcher matcher = PARAM_PATTERN.matcher(text);
StringBuffer buffer = new StringBuffer();
while (matcher.find()) {
Object replacement = params.get(matcher.group(1));
matcher.appendReplacement(buffer, replacement == null ? "" : Matcher.quoteReplacement(String.valueOf(replacement)));
}
matcher.appendTail(buffer);
return buffer.toString();
}
private String resolveOutputKey(String defaultName) {
List<Parameter> outputDefs = getOutputDefs();
if (outputDefs == null || outputDefs.isEmpty()) {
return defaultName;
}
String name = outputDefs.get(0).getName();
return StringUtil.hasText(name) ? name : defaultName;
}
public DatasetRef getDatasetRef() {
return datasetRef;
}
public void setDatasetRef(DatasetRef datasetRef) {
this.datasetRef = datasetRef;
}
public String getQuerySql() {
return querySql;
}
public void setQuerySql(String querySql) {
this.querySql = querySql;
}
}

View File

@@ -0,0 +1,33 @@
package tech.easyflow.ai.node;
import com.alibaba.fastjson.JSONObject;
import com.easyagents.flow.core.node.BaseNode;
import com.easyagents.flow.core.parser.BaseNodeParser;
import tech.easyflow.common.web.exceptions.BusinessException;
import tech.easyflow.datacenter.execution.model.DatasetRef;
public class SearchDatasetNodeParser extends BaseNodeParser {
private static final String SOURCE_MISSING_MESSAGE = "查询数据节点未选择连接服务";
private static final String SQL_MISSING_MESSAGE = "查询数据节点未设置 SQL";
@Override
protected BaseNode doParse(JSONObject root, JSONObject data, JSONObject tinyflow) {
if (data == null) {
throw new BusinessException(SOURCE_MISSING_MESSAGE);
}
DatasetRef datasetRef = data.getObject("datasetRef", DatasetRef.class);
if (datasetRef == null || datasetRef.getSourceId() == null) {
throw new BusinessException(SOURCE_MISSING_MESSAGE);
}
String querySql = data.getString("querySql");
if (querySql == null || querySql.isBlank()) {
throw new BusinessException(SQL_MISSING_MESSAGE);
}
return new SearchDatasetNode(datasetRef, querySql);
}
public String getNodeName() {
return "search-dataset-node";
}
}

View File

@@ -1,149 +0,0 @@
package tech.easyflow.ai.node;
import com.mybatisflex.core.row.Db;
import com.mybatisflex.core.row.Row;
import com.easyagents.flow.core.chain.Chain;
import com.easyagents.flow.core.node.BaseNode;
import net.sf.jsqlparser.JSQLParserException;
import net.sf.jsqlparser.parser.CCJSqlParserUtil;
import net.sf.jsqlparser.statement.Statement;
import net.sf.jsqlparser.statement.select.Select;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.util.StringUtils;
import tech.easyflow.common.web.exceptions.BusinessException;
import java.util.*;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
/**
* SQL查询节点
*
* @author tao
* @date 2025-05-21
*/
public class SqlNode extends BaseNode {
private String sql;
private static final Logger logger = LoggerFactory.getLogger(SqlNode.class);
public SqlNode() {
}
public SqlNode(String sql) {
this.sql = sql;
}
@Override
public Map<String, Object> execute(Chain chain) {
Map<String, Object> map = chain.getState().resolveParameters(this);
Map<String, Object> res = new HashMap<>();
Map<String, Object> formatSqlMap = formatSql(sql, map);
String formatSql = (String) formatSqlMap.get("replacedSql");
Statement statement = null;
try {
statement = CCJSqlParserUtil.parse(formatSql);
} catch (JSQLParserException e) {
logger.error("sql 解析报错:", e);
throw new BusinessException("SQL解析失败请确认SQL语法无误");
}
if (!(statement instanceof Select)) {
logger.error("sql 解析报错statement instanceof Select 结果为false");
throw new BusinessException("仅支持查询语句!");
}
List<String> paramNames = (List<String>) formatSqlMap.get("paramNames");
List<Object> paramValues = new ArrayList<>();
paramNames.forEach(paramName -> {
Object o = map.get(paramName);
paramValues.add(o);
});
List<Row> rows = Db.selectListBySql(formatSql, paramValues.toArray());
if (rows == null || rows.isEmpty()) {
return Collections.emptyMap();
}
res.put("queryData", rows);
return res;
}
private Map<String, Object> formatSql(String rawSql, Map<String, Object> paramMap) {
if (!StringUtils.hasLength(rawSql)) {
logger.error("sql解析报错sql为空");
throw new BusinessException("sql 不能为空!");
}
// 匹配 {{?...}} 表示可用占位符的参数
Pattern paramPattern = Pattern.compile("\\{\\{\\?([^}]+)}}");
// 匹配 {{...}} 表示直接替换的参数(非占位符)
Pattern directPattern = Pattern.compile("\\{\\{([^}?][^}]*)}}");
List<String> paramNames = new ArrayList<>();
StringBuffer sqlBuffer = new StringBuffer();
// 替换 {{?...}} -> ?
Matcher paramMatcher = paramPattern.matcher(rawSql);
while (paramMatcher.find()) {
String paramName = paramMatcher.group(1).trim();
paramNames.add(paramName);
paramMatcher.appendReplacement(sqlBuffer, "?");
}
paramMatcher.appendTail(sqlBuffer);
String intermediateSql = sqlBuffer.toString();
// 替换 {{...}} -> 实际值(用于表名/列名等)
sqlBuffer = new StringBuffer(); // 清空 buffer 重新处理
Matcher directMatcher = directPattern.matcher(intermediateSql);
while (directMatcher.find()) {
String key = directMatcher.group(1).trim();
Object value = paramMap.get(key);
if (value == null) {
logger.error("未找到参数:" + key);
throw new BusinessException("sql解析失败请确保sql语法正确");
}
String safeValue = value.toString();
directMatcher.appendReplacement(sqlBuffer, Matcher.quoteReplacement(safeValue));
}
directMatcher.appendTail(sqlBuffer);
String finalSql = sqlBuffer.toString().trim();
// 清理末尾分号与中文引号
if (finalSql.endsWith(";") || finalSql.endsWith("")) {
finalSql = finalSql.substring(0, finalSql.length() - 1);
}
finalSql = finalSql.replace("", "\"").replace("", "\"");
logger.info("Final SQL: {}", finalSql);
logger.info("Param names: {}", paramNames);
Map<String, Object> result = new HashMap<>();
result.put("replacedSql", finalSql);
result.put("paramNames", paramNames);
return result;
}
public String getSql() {
return sql;
}
public void setSql(String sql) {
this.sql = sql;
}
}

View File

@@ -1,25 +0,0 @@
package tech.easyflow.ai.node;
import com.alibaba.fastjson.JSONObject;
import com.easyagents.flow.core.node.BaseNode;
import com.easyagents.flow.core.parser.BaseNodeParser;
/**
* Sql查询节点解析
*
* @author tao
* @date 2025-05-21
*/
public class SqlNodeParser extends BaseNodeParser {
@Override
public BaseNode doParse(JSONObject root, JSONObject data, JSONObject tinyflow) {
String sql = data.getString("sql");
return new SqlNode(sql);
}
public String getNodeName() {
return "sql-node";
}
}

View File

@@ -8,6 +8,7 @@ import org.junit.Test;
import tech.easyflow.ai.easyagentsflow.entity.WorkflowCheckResult;
import tech.easyflow.ai.easyagentsflow.entity.WorkflowCheckStage;
import tech.easyflow.ai.entity.Workflow;
import tech.easyflow.ai.node.SearchDatasetNodeParser;
import tech.easyflow.ai.node.WorkflowNodeParser;
import tech.easyflow.ai.service.WorkflowService;
@@ -69,6 +70,24 @@ public class WorkflowCheckServiceTest {
assertHasCode(result, "EDGE_TARGET_NOT_FOUND");
}
@Test
public void testSaveShouldBlockSearchDatasetWithoutSql() throws Exception {
WorkflowCheckService service = newService(new HashMap<>());
JSONObject searchData = data("查询数据");
JSONObject datasetRef = new JSONObject();
datasetRef.put("sourceId", "1001");
datasetRef.put("tableId", "2001");
searchData.put("datasetRef", datasetRef);
String content = workflowJson(
array(node("search-1", "search-dataset-node", null, searchData)),
new JSONArray()
);
WorkflowCheckResult result = service.checkContent(content, WorkflowCheckStage.SAVE, null);
Assert.assertFalse(result.isPassed());
assertHasCode(result, "SEARCH_DATASET_INVALID");
}
@Test
public void testPreExecuteShouldBlockMissingStartOrEnd() throws Exception {
WorkflowCheckService service = newService(new HashMap<>());
@@ -86,6 +105,26 @@ public class WorkflowCheckServiceTest {
assertHasCode(result, "END_NODE_MISSING");
}
@Test
public void testPreExecuteShouldPassForSourceOnlySearchDatasetNode() throws Exception {
WorkflowCheckService service = newService(new HashMap<>());
String content = workflowJson(
array(
node("s1", "startNode", null, data("开始")),
searchDatasetNode("q1", null, "1001"),
node("e1", "endNode", null, data("结束"))
),
array(
edge("e1", "s1", "q1"),
edge("e2", "q1", "e1")
)
);
WorkflowCheckResult result = service.checkContent(content, WorkflowCheckStage.PRE_EXECUTE, BigInteger.ONE);
Assert.assertTrue(result.isPassed());
Assert.assertEquals(0, result.getIssueCount());
}
@Test
public void testPreExecuteShouldBlockRootEntryNotStart() throws Exception {
WorkflowCheckService service = newService(new HashMap<>());
@@ -203,8 +242,10 @@ public class WorkflowCheckServiceTest {
.withDefaultParsers(true)
.build();
parser.addNodeParser("workflow-node", new WorkflowNodeParser());
parser.addNodeParser("search-dataset-node", new SearchDatasetNodeParser());
setField(service, "chainParser", parser);
setField(service, "workflowService", mockWorkflowService(workflowStore));
setField(service, "workflowDatacenterContentService", new WorkflowDatacenterContentService());
return service;
}
@@ -294,6 +335,15 @@ public class WorkflowCheckServiceTest {
return node(id, "workflow-node", parentId, data);
}
private static JSONObject searchDatasetNode(String id, String parentId, String sourceId) {
JSONObject data = data("查询数据");
JSONObject datasetRef = new JSONObject();
datasetRef.put("sourceId", sourceId);
data.put("datasetRef", datasetRef);
data.put("querySql", "SELECT 1");
return node(id, "search-dataset-node", parentId, data);
}
private static JSONObject data(String title) {
JSONObject data = new JSONObject();
data.put("title", title);

View File

@@ -0,0 +1,73 @@
package tech.easyflow.ai.node;
import org.junit.Assert;
import org.junit.Test;
import tech.easyflow.common.web.exceptions.BusinessException;
import tech.easyflow.datacenter.execution.model.DatasetRef;
import java.lang.reflect.Method;
import java.math.BigInteger;
import java.util.HashMap;
import java.util.Map;
public class SearchDatasetNodeTest {
@Test
public void testResolveQuerySqlShouldUseNodeQuerySqlAndResolveTemplate() throws Exception {
DatasetRef datasetRef = new DatasetRef();
datasetRef.setSourceId(BigInteger.valueOf(1001L));
SearchDatasetNode node = new SearchDatasetNode(datasetRef, """
SELECT id, name
FROM orders_{{biz}}
WHERE name LIKE '%{{keyword}}%'
ORDER BY created_at {{direction}}
""");
Map<String, Object> params = new HashMap<>();
params.put("biz", "prod");
params.put("keyword", "vip");
params.put("direction", "DESC");
String sql = invokeResolveQuerySql(node, params);
Assert.assertEquals("""
SELECT id, name
FROM orders_prod
WHERE name LIKE '%vip%'
ORDER BY created_at DESC
""".trim(), sql);
}
@Test
public void testResolveQuerySqlShouldUseNodeQuerySqlWhenParamsDoNotContainSql() throws Exception {
DatasetRef datasetRef = new DatasetRef();
datasetRef.setSourceId(BigInteger.valueOf(2002L));
SearchDatasetNode node = new SearchDatasetNode(datasetRef, "SELECT * FROM fallback_table");
Map<String, Object> params = new HashMap<>();
String sql = invokeResolveQuerySql(node, params);
Assert.assertEquals("SELECT * FROM fallback_table", sql);
}
@Test
public void testResolveQuerySqlShouldRejectBlankSql() throws Exception {
DatasetRef datasetRef = new DatasetRef();
datasetRef.setSourceId(BigInteger.valueOf(3003L));
SearchDatasetNode node = new SearchDatasetNode(datasetRef, " ");
try {
invokeResolveQuerySql(node, new HashMap<>());
Assert.fail("expected BusinessException");
} catch (Exception e) {
Throwable cause = e.getCause() == null ? e : e.getCause();
Assert.assertTrue(cause instanceof BusinessException);
Assert.assertEquals("查询数据节点未设置 SQL", cause.getMessage());
}
}
private String invokeResolveQuerySql(SearchDatasetNode node, Map<String, Object> params) throws Exception {
Method method = SearchDatasetNode.class.getDeclaredMethod("resolveQuerySql", Map.class);
method.setAccessible(true);
return (String) method.invoke(node, params);
}
}

View File

@@ -16,6 +16,19 @@
<groupId>com.mybatis-flex</groupId>
<artifactId>mybatis-flex-spring-boot3-starter</artifactId>
</dependency>
<dependency>
<groupId>com.zaxxer</groupId>
<artifactId>HikariCP</artifactId>
</dependency>
<dependency>
<groupId>org.postgresql</groupId>
<artifactId>postgresql</artifactId>
<version>42.7.3</version>
</dependency>
<dependency>
<groupId>cn.hutool</groupId>
<artifactId>hutool-crypto</artifactId>
</dependency>
<dependency>
<groupId>tech.easyflow</groupId>
<artifactId>easyflow-common-base</artifactId>

View File

@@ -0,0 +1,12 @@
package tech.easyflow.datacenter.connector;
import tech.easyflow.datacenter.meta.enums.DatacenterCapability;
import tech.easyflow.datacenter.meta.enums.DatacenterSourceType;
import java.util.Set;
public interface DatacenterConnector extends SourceHealthChecker, MetadataExplorer, QueryExecutor, WriteExecutor {
DatacenterSourceType getSourceType();
Set<DatacenterCapability> getCapabilities();
}

View File

@@ -0,0 +1,29 @@
package tech.easyflow.datacenter.connector;
import org.springframework.stereotype.Component;
import tech.easyflow.common.web.exceptions.BusinessException;
import tech.easyflow.datacenter.meta.enums.DatacenterSourceType;
import java.util.List;
import java.util.Map;
import java.util.function.Function;
import java.util.stream.Collectors;
@Component
public class DatacenterConnectorRegistry {
private final Map<DatacenterSourceType, DatacenterConnector> connectorMap;
public DatacenterConnectorRegistry(List<DatacenterConnector> connectors) {
this.connectorMap = connectors.stream().collect(Collectors.toMap(DatacenterConnector::getSourceType, Function.identity()));
}
public DatacenterConnector getConnector(String sourceType) {
DatacenterSourceType type = DatacenterSourceType.valueOf(sourceType);
DatacenterConnector connector = connectorMap.get(type);
if (connector == null) {
throw new BusinessException("未找到连接器: " + sourceType);
}
return connector;
}
}

View File

@@ -0,0 +1,16 @@
package tech.easyflow.datacenter.connector;
import tech.easyflow.datacenter.entity.DatacenterTable;
import tech.easyflow.datacenter.meta.entity.DatacenterSource;
import tech.easyflow.datacenter.meta.model.DatacenterCatalogMeta;
import tech.easyflow.datacenter.meta.model.DatacenterTableDetailMeta;
import java.util.List;
public interface MetadataExplorer {
List<DatacenterCatalogMeta> listCatalogs(DatacenterSource source);
List<DatacenterTable> listTables(DatacenterSource source, String catalogName);
DatacenterTableDetailMeta getTableDetail(DatacenterSource source, String catalogName, String tableName);
}

View File

@@ -0,0 +1,15 @@
package tech.easyflow.datacenter.connector;
import com.mybatisflex.core.paginate.Page;
import com.mybatisflex.core.row.Row;
import tech.easyflow.datacenter.entity.DatacenterTable;
import tech.easyflow.datacenter.execution.model.DatacenterQueryRequest;
import tech.easyflow.datacenter.meta.entity.DatacenterSource;
import java.util.List;
public interface QueryExecutor {
Page<Row> queryPage(DatacenterSource source, DatacenterTable table, DatacenterQueryRequest request);
List<Row> queryBySql(DatacenterSource source, String sql);
}

View File

@@ -0,0 +1,8 @@
package tech.easyflow.datacenter.connector;
import tech.easyflow.datacenter.execution.model.DatacenterConnectionTestResult;
import tech.easyflow.datacenter.meta.entity.DatacenterSource;
public interface SourceHealthChecker {
DatacenterConnectionTestResult testConnection(DatacenterSource source);
}

View File

@@ -0,0 +1,9 @@
package tech.easyflow.datacenter.connector;
public interface SqlDialect {
String quoteIdentifier(String identifier);
String qualifyTable(String namespace, String tableName);
String buildPageSql(String baseSql, long pageNumber, long pageSize);
}

View File

@@ -0,0 +1,14 @@
package tech.easyflow.datacenter.connector;
import com.alibaba.fastjson2.JSONObject;
import tech.easyflow.common.entity.LoginAccount;
import tech.easyflow.datacenter.entity.DatacenterTable;
import tech.easyflow.datacenter.meta.entity.DatacenterSource;
import java.math.BigInteger;
public interface WriteExecutor {
void saveRow(DatacenterSource source, DatacenterTable table, JSONObject data, LoginAccount account);
void deleteRow(DatacenterSource source, DatacenterTable table, BigInteger id, LoginAccount account);
}

View File

@@ -0,0 +1,24 @@
package tech.easyflow.datacenter.connector.dialect;
import cn.hutool.core.util.StrUtil;
import tech.easyflow.datacenter.connector.SqlDialect;
public class GaussdbSqlDialect implements SqlDialect {
@Override
public String quoteIdentifier(String identifier) {
return '"' + identifier + '"';
}
@Override
public String qualifyTable(String namespace, String tableName) {
if (StrUtil.isBlank(namespace)) {
return quoteIdentifier(tableName);
}
return quoteIdentifier(namespace) + "." + quoteIdentifier(tableName);
}
@Override
public String buildPageSql(String baseSql, long pageNumber, long pageSize) {
return baseSql + " OFFSET ? LIMIT ?";
}
}

View File

@@ -0,0 +1,24 @@
package tech.easyflow.datacenter.connector.dialect;
import cn.hutool.core.util.StrUtil;
import tech.easyflow.datacenter.connector.SqlDialect;
public class MysqlSqlDialect implements SqlDialect {
@Override
public String quoteIdentifier(String identifier) {
return "`" + identifier + "`";
}
@Override
public String qualifyTable(String namespace, String tableName) {
if (StrUtil.isBlank(namespace)) {
return quoteIdentifier(tableName);
}
return quoteIdentifier(namespace) + "." + quoteIdentifier(tableName);
}
@Override
public String buildPageSql(String baseSql, long pageNumber, long pageSize) {
return baseSql + " LIMIT ?, ?";
}
}

View File

@@ -0,0 +1,24 @@
package tech.easyflow.datacenter.connector.dialect;
import cn.hutool.core.util.StrUtil;
import tech.easyflow.datacenter.connector.SqlDialect;
public class OracleSqlDialect implements SqlDialect {
@Override
public String quoteIdentifier(String identifier) {
return '"' + identifier + '"';
}
@Override
public String qualifyTable(String namespace, String tableName) {
if (StrUtil.isBlank(namespace)) {
return quoteIdentifier(tableName);
}
return quoteIdentifier(namespace) + "." + quoteIdentifier(tableName);
}
@Override
public String buildPageSql(String baseSql, long pageNumber, long pageSize) {
return baseSql + " OFFSET ? ROWS FETCH NEXT ? ROWS ONLY";
}
}

View File

@@ -0,0 +1,24 @@
package tech.easyflow.datacenter.connector.dialect;
import cn.hutool.core.util.StrUtil;
import tech.easyflow.datacenter.connector.SqlDialect;
public class PostgresqlSqlDialect implements SqlDialect {
@Override
public String quoteIdentifier(String identifier) {
return '"' + identifier + '"';
}
@Override
public String qualifyTable(String namespace, String tableName) {
if (StrUtil.isBlank(namespace)) {
return quoteIdentifier(tableName);
}
return quoteIdentifier(namespace) + "." + quoteIdentifier(tableName);
}
@Override
public String buildPageSql(String baseSql, long pageNumber, long pageSize) {
return baseSql + " OFFSET ? LIMIT ?";
}
}

View File

@@ -0,0 +1,22 @@
package tech.easyflow.datacenter.connector.impl;
import org.springframework.stereotype.Component;
import tech.easyflow.datacenter.connector.support.AbstractInternalTableConnector;
import tech.easyflow.datacenter.meta.enums.DatacenterCapability;
import tech.easyflow.datacenter.meta.enums.DatacenterSourceType;
import java.util.EnumSet;
@Component
public class ExcelConnector extends AbstractInternalTableConnector {
public ExcelConnector() {
super(DatacenterSourceType.EXCEL, EnumSet.of(
DatacenterCapability.TEST_CONNECTION,
DatacenterCapability.BROWSE_METADATA,
DatacenterCapability.READ_QUERY,
DatacenterCapability.WRITE_MUTATION,
DatacenterCapability.MATERIALIZE,
DatacenterCapability.EXPORT
));
}
}

View File

@@ -0,0 +1,22 @@
package tech.easyflow.datacenter.connector.impl;
import org.springframework.stereotype.Component;
import tech.easyflow.datacenter.connector.support.AbstractInternalTableConnector;
import tech.easyflow.datacenter.meta.enums.DatacenterCapability;
import tech.easyflow.datacenter.meta.enums.DatacenterSourceType;
import java.util.EnumSet;
@Component
public class ExcelMaterializedConnector extends AbstractInternalTableConnector {
public ExcelMaterializedConnector() {
super(DatacenterSourceType.EXCEL_MATERIALIZED, EnumSet.of(
DatacenterCapability.TEST_CONNECTION,
DatacenterCapability.BROWSE_METADATA,
DatacenterCapability.READ_QUERY,
DatacenterCapability.WRITE_MUTATION,
DatacenterCapability.MATERIALIZE,
DatacenterCapability.EXPORT
));
}
}

View File

@@ -0,0 +1,40 @@
package tech.easyflow.datacenter.connector.impl;
import com.zaxxer.hikari.HikariDataSource;
import org.springframework.stereotype.Component;
import tech.easyflow.datacenter.connector.dialect.GaussdbSqlDialect;
import tech.easyflow.datacenter.connector.support.AbstractJdbcConnector;
import tech.easyflow.datacenter.connector.support.DatacenterDatasourceManager;
import tech.easyflow.datacenter.meta.entity.DatacenterSource;
import tech.easyflow.datacenter.meta.enums.DatacenterCapability;
import tech.easyflow.datacenter.meta.enums.DatacenterSourceType;
import java.sql.Connection;
import java.util.EnumSet;
@Component
public class GaussdbNativeConnector extends AbstractJdbcConnector {
private final DatacenterDatasourceManager datasourceManager;
public GaussdbNativeConnector(DatacenterDatasourceManager datasourceManager) {
super(DatacenterSourceType.GAUSSDB_NATIVE, new GaussdbSqlDialect(), EnumSet.of(
DatacenterCapability.TEST_CONNECTION,
DatacenterCapability.BROWSE_METADATA,
DatacenterCapability.READ_QUERY
));
this.datasourceManager = datasourceManager;
}
@Override
protected <T> T withConnection(DatacenterSource source, boolean cacheable, JdbcCallback<T> callback) throws Exception {
HikariDataSource dataSource = cacheable ? datasourceManager.getOrCreateExternalDatasource(source) : datasourceManager.createExternalDatasource(source);
try (Connection connection = dataSource.getConnection()) {
return callback.apply(connection);
} finally {
if (!cacheable || source.getId() == null) {
dataSource.close();
}
}
}
}

View File

@@ -0,0 +1,40 @@
package tech.easyflow.datacenter.connector.impl;
import com.zaxxer.hikari.HikariDataSource;
import org.springframework.stereotype.Component;
import tech.easyflow.datacenter.connector.dialect.MysqlSqlDialect;
import tech.easyflow.datacenter.connector.support.AbstractJdbcConnector;
import tech.easyflow.datacenter.connector.support.DatacenterDatasourceManager;
import tech.easyflow.datacenter.meta.entity.DatacenterSource;
import tech.easyflow.datacenter.meta.enums.DatacenterCapability;
import tech.easyflow.datacenter.meta.enums.DatacenterSourceType;
import java.sql.Connection;
import java.util.EnumSet;
@Component
public class Gbase8aConnector extends AbstractJdbcConnector {
private final DatacenterDatasourceManager datasourceManager;
public Gbase8aConnector(DatacenterDatasourceManager datasourceManager) {
super(DatacenterSourceType.GBASE_8A, new MysqlSqlDialect(), EnumSet.of(
DatacenterCapability.TEST_CONNECTION,
DatacenterCapability.BROWSE_METADATA,
DatacenterCapability.READ_QUERY
));
this.datasourceManager = datasourceManager;
}
@Override
protected <T> T withConnection(DatacenterSource source, boolean cacheable, JdbcCallback<T> callback) throws Exception {
HikariDataSource dataSource = cacheable ? datasourceManager.getOrCreateExternalDatasource(source) : datasourceManager.createExternalDatasource(source);
try (Connection connection = dataSource.getConnection()) {
return callback.apply(connection);
} finally {
if (!cacheable || source.getId() == null) {
dataSource.close();
}
}
}
}

View File

@@ -0,0 +1,40 @@
package tech.easyflow.datacenter.connector.impl;
import com.zaxxer.hikari.HikariDataSource;
import org.springframework.stereotype.Component;
import tech.easyflow.datacenter.connector.dialect.MysqlSqlDialect;
import tech.easyflow.datacenter.connector.support.AbstractJdbcConnector;
import tech.easyflow.datacenter.connector.support.DatacenterDatasourceManager;
import tech.easyflow.datacenter.meta.entity.DatacenterSource;
import tech.easyflow.datacenter.meta.enums.DatacenterCapability;
import tech.easyflow.datacenter.meta.enums.DatacenterSourceType;
import java.sql.Connection;
import java.util.EnumSet;
@Component
public class Gbase8sConnector extends AbstractJdbcConnector {
private final DatacenterDatasourceManager datasourceManager;
public Gbase8sConnector(DatacenterDatasourceManager datasourceManager) {
super(DatacenterSourceType.GBASE_8S, new MysqlSqlDialect(), EnumSet.of(
DatacenterCapability.TEST_CONNECTION,
DatacenterCapability.BROWSE_METADATA,
DatacenterCapability.READ_QUERY
));
this.datasourceManager = datasourceManager;
}
@Override
protected <T> T withConnection(DatacenterSource source, boolean cacheable, JdbcCallback<T> callback) throws Exception {
HikariDataSource dataSource = cacheable ? datasourceManager.getOrCreateExternalDatasource(source) : datasourceManager.createExternalDatasource(source);
try (Connection connection = dataSource.getConnection()) {
return callback.apply(connection);
} finally {
if (!cacheable || source.getId() == null) {
dataSource.close();
}
}
}
}

View File

@@ -0,0 +1,40 @@
package tech.easyflow.datacenter.connector.impl;
import com.zaxxer.hikari.HikariDataSource;
import org.springframework.stereotype.Component;
import tech.easyflow.datacenter.connector.dialect.MysqlSqlDialect;
import tech.easyflow.datacenter.connector.support.AbstractJdbcConnector;
import tech.easyflow.datacenter.connector.support.DatacenterDatasourceManager;
import tech.easyflow.datacenter.meta.entity.DatacenterSource;
import tech.easyflow.datacenter.meta.enums.DatacenterCapability;
import tech.easyflow.datacenter.meta.enums.DatacenterSourceType;
import java.sql.Connection;
import java.util.EnumSet;
@Component
public class MysqlConnector extends AbstractJdbcConnector {
private final DatacenterDatasourceManager datasourceManager;
public MysqlConnector(DatacenterDatasourceManager datasourceManager) {
super(DatacenterSourceType.MYSQL, new MysqlSqlDialect(), EnumSet.of(
DatacenterCapability.TEST_CONNECTION,
DatacenterCapability.BROWSE_METADATA,
DatacenterCapability.READ_QUERY
));
this.datasourceManager = datasourceManager;
}
@Override
protected <T> T withConnection(DatacenterSource source, boolean cacheable, JdbcCallback<T> callback) throws Exception {
HikariDataSource dataSource = cacheable ? datasourceManager.getOrCreateExternalDatasource(source) : datasourceManager.createExternalDatasource(source);
try (Connection connection = dataSource.getConnection()) {
return callback.apply(connection);
} finally {
if (!cacheable || source.getId() == null) {
dataSource.close();
}
}
}
}

View File

@@ -0,0 +1,40 @@
package tech.easyflow.datacenter.connector.impl;
import com.zaxxer.hikari.HikariDataSource;
import org.springframework.stereotype.Component;
import tech.easyflow.datacenter.connector.dialect.OracleSqlDialect;
import tech.easyflow.datacenter.connector.support.AbstractJdbcConnector;
import tech.easyflow.datacenter.connector.support.DatacenterDatasourceManager;
import tech.easyflow.datacenter.meta.entity.DatacenterSource;
import tech.easyflow.datacenter.meta.enums.DatacenterCapability;
import tech.easyflow.datacenter.meta.enums.DatacenterSourceType;
import java.sql.Connection;
import java.util.EnumSet;
@Component
public class OracleConnector extends AbstractJdbcConnector {
private final DatacenterDatasourceManager datasourceManager;
public OracleConnector(DatacenterDatasourceManager datasourceManager) {
super(DatacenterSourceType.ORACLE, new OracleSqlDialect(), EnumSet.of(
DatacenterCapability.TEST_CONNECTION,
DatacenterCapability.BROWSE_METADATA,
DatacenterCapability.READ_QUERY
));
this.datasourceManager = datasourceManager;
}
@Override
protected <T> T withConnection(DatacenterSource source, boolean cacheable, JdbcCallback<T> callback) throws Exception {
HikariDataSource dataSource = cacheable ? datasourceManager.getOrCreateExternalDatasource(source) : datasourceManager.createExternalDatasource(source);
try (Connection connection = dataSource.getConnection()) {
return callback.apply(connection);
} finally {
if (!cacheable || source.getId() == null) {
dataSource.close();
}
}
}
}

View File

@@ -0,0 +1,40 @@
package tech.easyflow.datacenter.connector.impl;
import com.zaxxer.hikari.HikariDataSource;
import org.springframework.stereotype.Component;
import tech.easyflow.datacenter.connector.dialect.PostgresqlSqlDialect;
import tech.easyflow.datacenter.connector.support.AbstractJdbcConnector;
import tech.easyflow.datacenter.connector.support.DatacenterDatasourceManager;
import tech.easyflow.datacenter.meta.entity.DatacenterSource;
import tech.easyflow.datacenter.meta.enums.DatacenterCapability;
import tech.easyflow.datacenter.meta.enums.DatacenterSourceType;
import java.sql.Connection;
import java.util.EnumSet;
@Component
public class PostgresqlConnector extends AbstractJdbcConnector {
private final DatacenterDatasourceManager datasourceManager;
public PostgresqlConnector(DatacenterDatasourceManager datasourceManager) {
super(DatacenterSourceType.POSTGRESQL, new PostgresqlSqlDialect(), EnumSet.of(
DatacenterCapability.TEST_CONNECTION,
DatacenterCapability.BROWSE_METADATA,
DatacenterCapability.READ_QUERY
));
this.datasourceManager = datasourceManager;
}
@Override
protected <T> T withConnection(DatacenterSource source, boolean cacheable, JdbcCallback<T> callback) throws Exception {
HikariDataSource dataSource = cacheable ? datasourceManager.getOrCreateExternalDatasource(source) : datasourceManager.createExternalDatasource(source);
try (Connection connection = dataSource.getConnection()) {
return callback.apply(connection);
} finally {
if (!cacheable || source.getId() == null) {
dataSource.close();
}
}
}
}

View File

@@ -0,0 +1,135 @@
package tech.easyflow.datacenter.connector.impl;
import cn.hutool.core.util.StrUtil;
import com.alibaba.fastjson2.JSONObject;
import com.mybatisflex.core.paginate.Page;
import com.mybatisflex.core.row.Row;
import org.springframework.stereotype.Component;
import tech.easyflow.common.entity.LoginAccount;
import tech.easyflow.common.web.exceptions.BusinessException;
import tech.easyflow.datacenter.connector.dialect.MysqlSqlDialect;
import tech.easyflow.datacenter.connector.support.AbstractJdbcConnector;
import tech.easyflow.datacenter.entity.DatacenterTable;
import tech.easyflow.datacenter.entity.DatacenterTableField;
import tech.easyflow.datacenter.execution.model.DatacenterQueryRequest;
import tech.easyflow.datacenter.meta.entity.DatacenterSource;
import tech.easyflow.datacenter.meta.enums.DatacenterCapability;
import tech.easyflow.datacenter.meta.enums.DatacenterSourceType;
import javax.sql.DataSource;
import java.math.BigInteger;
import java.sql.Connection;
import java.sql.PreparedStatement;
import java.util.ArrayList;
import java.util.EnumSet;
import java.util.List;
import java.util.stream.Collectors;
@Component
public class ProjectMysqlConnector extends AbstractJdbcConnector {
private final DataSource dataSource;
private final MysqlSqlDialect dialect = new MysqlSqlDialect();
public ProjectMysqlConnector(DataSource dataSource) {
super(DatacenterSourceType.PROJECT_MYSQL, new MysqlSqlDialect(), EnumSet.of(
DatacenterCapability.TEST_CONNECTION,
DatacenterCapability.BROWSE_METADATA,
DatacenterCapability.READ_QUERY,
DatacenterCapability.WRITE_MUTATION
));
this.dataSource = dataSource;
}
@Override
protected <T> T withConnection(DatacenterSource source, boolean cacheable, JdbcCallback<T> callback) throws Exception {
try (Connection connection = dataSource.getConnection()) {
return callback.apply(connection);
}
}
@Override
protected boolean requiresJdbcUrl() {
return false;
}
@Override
public Page<Row> queryPage(DatacenterSource source, DatacenterTable table, DatacenterQueryRequest request) {
return super.queryPage(source, table, request);
}
@Override
public void saveRow(DatacenterSource source, DatacenterTable table, JSONObject data, LoginAccount account) {
List<DatacenterTableField> writableFields = table.getFields().stream()
.filter(field -> field.getWritable() == null || field.getWritable() == 1)
.collect(Collectors.toList());
Object id = data.get("id");
try (Connection connection = dataSource.getConnection()) {
if (id == null) {
List<String> columns = new ArrayList<>();
List<Object> values = new ArrayList<>();
for (DatacenterTableField field : writableFields) {
Object value = data.get(field.getFieldName());
if (value != null) {
columns.add(field.getFieldName());
values.add(value);
}
}
if (columns.isEmpty()) {
throw new BusinessException("没有可写字段");
}
String sql = "INSERT INTO " + dialect.qualifyTable(source.getDatabaseName(), resolvePhysicalTableName(table))
+ " (" + columns.stream().map(dialect::quoteIdentifier).collect(Collectors.joining(",")) + ") VALUES ("
+ columns.stream().map(item -> "?").collect(Collectors.joining(",")) + ")";
try (PreparedStatement statement = connection.prepareStatement(sql)) {
for (int i = 0; i < values.size(); i++) {
statement.setObject(i + 1, values.get(i));
}
statement.executeUpdate();
}
return;
}
List<String> setClauses = new ArrayList<>();
List<Object> values = new ArrayList<>();
for (DatacenterTableField field : writableFields) {
if (!data.containsKey(field.getFieldName())) {
continue;
}
setClauses.add(dialect.quoteIdentifier(field.getFieldName()) + " = ?");
values.add(data.get(field.getFieldName()));
}
if (setClauses.isEmpty()) {
return;
}
String sql = "UPDATE " + dialect.qualifyTable(source.getDatabaseName(), resolvePhysicalTableName(table))
+ " SET " + String.join(",", setClauses)
+ " WHERE " + dialect.quoteIdentifier("id") + " = ?";
values.add(id);
try (PreparedStatement statement = connection.prepareStatement(sql)) {
for (int i = 0; i < values.size(); i++) {
statement.setObject(i + 1, values.get(i));
}
statement.executeUpdate();
}
} catch (Exception ex) {
throw new BusinessException("项目 MySQL 写入失败: " + ex.getMessage());
}
}
@Override
public void deleteRow(DatacenterSource source, DatacenterTable table, BigInteger id, LoginAccount account) {
String sql = "DELETE FROM " + dialect.qualifyTable(source.getDatabaseName(), resolvePhysicalTableName(table))
+ " WHERE " + dialect.quoteIdentifier("id") + " = ?";
try (Connection connection = dataSource.getConnection(); PreparedStatement statement = connection.prepareStatement(sql)) {
statement.setObject(1, id);
statement.executeUpdate();
} catch (Exception ex) {
throw new BusinessException("项目 MySQL 删除失败: " + ex.getMessage());
}
}
@Override
protected String resolveCatalogName(DatacenterSource source, String requestedCatalogName) {
return StrUtil.blankToDefault(requestedCatalogName, source.getDatabaseName());
}
}

View File

@@ -0,0 +1,152 @@
package tech.easyflow.datacenter.connector.support;
import cn.hutool.core.util.StrUtil;
import com.alibaba.fastjson2.JSONObject;
import com.mybatisflex.core.paginate.Page;
import com.mybatisflex.core.query.QueryWrapper;
import com.mybatisflex.core.row.Db;
import com.mybatisflex.core.row.Row;
import com.mybatisflex.core.row.RowKey;
import org.springframework.util.CollectionUtils;
import tech.easyflow.common.entity.LoginAccount;
import tech.easyflow.common.web.exceptions.BusinessException;
import tech.easyflow.datacenter.connector.DatacenterConnector;
import tech.easyflow.datacenter.entity.DatacenterTable;
import tech.easyflow.datacenter.entity.DatacenterTableField;
import tech.easyflow.datacenter.execution.model.DatacenterConnectionTestResult;
import tech.easyflow.datacenter.execution.model.DatacenterQueryRequest;
import tech.easyflow.datacenter.meta.entity.DatacenterSource;
import tech.easyflow.datacenter.meta.enums.DatacenterCapability;
import tech.easyflow.datacenter.meta.enums.DatacenterSourceType;
import tech.easyflow.datacenter.meta.model.DatacenterCatalogMeta;
import tech.easyflow.datacenter.meta.model.DatacenterTableDetailMeta;
import java.math.BigDecimal;
import java.math.BigInteger;
import java.util.*;
public abstract class AbstractInternalTableConnector implements DatacenterConnector {
private final DatacenterSourceType sourceType;
private final Set<DatacenterCapability> capabilities;
protected AbstractInternalTableConnector(DatacenterSourceType sourceType, Set<DatacenterCapability> capabilities) {
this.sourceType = sourceType;
this.capabilities = capabilities;
}
@Override
public DatacenterSourceType getSourceType() {
return sourceType;
}
@Override
public Set<DatacenterCapability> getCapabilities() {
return capabilities;
}
@Override
public DatacenterConnectionTestResult testConnection(DatacenterSource source) {
DatacenterConnectionTestResult result = new DatacenterConnectionTestResult();
result.setSuccess(true);
result.setMessage("连接成功");
result.setCapabilities(capabilities.stream().map(Enum::name).toList());
result.setDetails(Map.of("sourceType", sourceType.name()));
return result;
}
@Override
public List<DatacenterCatalogMeta> listCatalogs(DatacenterSource source) {
return Collections.emptyList();
}
@Override
public List<DatacenterTable> listTables(DatacenterSource source, String catalogName) {
return Collections.emptyList();
}
@Override
public DatacenterTableDetailMeta getTableDetail(DatacenterSource source, String catalogName, String tableName) {
throw new BusinessException("内部数据源请从元数据注册表读取表结构");
}
@Override
public Page<Row> queryPage(DatacenterSource source, DatacenterTable table, DatacenterQueryRequest request) {
String actualTable = resolveTableName(table);
QueryWrapper wrapper = QueryWrapper.create();
if (StrUtil.isNotBlank(request.getWhere())) {
wrapper.where(request.getWhere());
}
long count = Db.selectCountByQuery(actualTable, wrapper);
if (count == 0) {
return new Page<>(new ArrayList<>(), request.getPageNumber(), request.getPageSize(), count);
}
Page<Row> page = Db.paginate(actualTable, new Page<>(request.getPageNumber(), request.getPageSize(), count), wrapper);
normalizeRows(page.getRecords());
return page;
}
@Override
public List<Row> queryBySql(DatacenterSource source, String sql) {
List<Row> rows = Db.selectListBySql(sql);
normalizeRows(rows);
return rows;
}
@Override
public void saveRow(DatacenterSource source, DatacenterTable table, JSONObject data, LoginAccount account) {
List<DatacenterTableField> fields = table.getFields();
if (CollectionUtils.isEmpty(fields)) {
throw new BusinessException("数据集字段为空,无法写入");
}
String actualTable = resolveTableName(table);
Object id = data.get("id");
if (id == null) {
Row row = Row.ofKey(RowKey.SNOW_FLAKE_ID);
row.put("dept_id", account.getDeptId());
row.put("tenant_id", account.getTenantId());
row.put("created", new Date());
row.put("created_by", account.getId());
row.put("modified", new Date());
row.put("modified_by", account.getId());
row.put("remark", data.get("remark"));
for (DatacenterTableField field : fields) {
row.put(field.getFieldName(), data.get(field.getFieldName()));
}
Db.insert(actualTable, row);
return;
}
Row row = Row.ofKey("id", id);
row.put("modified", new Date());
row.put("modified_by", account.getId());
for (DatacenterTableField field : fields) {
row.put(field.getFieldName(), data.get(field.getFieldName()));
}
Db.updateById(actualTable, row);
}
@Override
public void deleteRow(DatacenterSource source, DatacenterTable table, BigInteger id, LoginAccount account) {
Db.deleteById(resolveTableName(table), Row.ofKey("id", id));
}
protected String resolveTableName(DatacenterTable table) {
return StrUtil.blankToDefault(table.getMaterializedTable(), table.getActualTable());
}
private void normalizeRows(List<Row> records) {
for (Row record : records) {
Map<String, Object> converted = new LinkedHashMap<>();
for (Map.Entry<String, Object> entry : record.entrySet()) {
Object value = entry.getValue();
if (value instanceof BigInteger || value instanceof BigDecimal || value instanceof Long) {
converted.put(entry.getKey(), value.toString());
} else {
converted.put(entry.getKey(), value);
}
}
record.clear();
record.putAll(converted);
}
}
}

View File

@@ -0,0 +1,523 @@
package tech.easyflow.datacenter.connector.support;
import cn.hutool.core.util.StrUtil;
import com.alibaba.fastjson2.JSONObject;
import com.mybatisflex.core.paginate.Page;
import com.mybatisflex.core.row.Row;
import org.springframework.util.CollectionUtils;
import tech.easyflow.common.constant.enums.EnumFieldType;
import tech.easyflow.common.entity.LoginAccount;
import tech.easyflow.common.web.exceptions.BusinessException;
import tech.easyflow.datacenter.connector.DatacenterConnector;
import tech.easyflow.datacenter.connector.SqlDialect;
import tech.easyflow.datacenter.entity.DatacenterTable;
import tech.easyflow.datacenter.entity.DatacenterTableField;
import tech.easyflow.datacenter.execution.model.DatacenterConnectionTestResult;
import tech.easyflow.datacenter.execution.model.DatacenterQueryFilter;
import tech.easyflow.datacenter.execution.model.DatacenterQueryRequest;
import tech.easyflow.datacenter.execution.model.DatacenterQuerySort;
import tech.easyflow.datacenter.meta.entity.DatacenterSource;
import tech.easyflow.datacenter.meta.enums.DatacenterCapability;
import tech.easyflow.datacenter.meta.enums.DatacenterConnectionErrorCode;
import tech.easyflow.datacenter.meta.enums.DatacenterSourceType;
import tech.easyflow.datacenter.meta.enums.DatacenterTableKind;
import tech.easyflow.datacenter.meta.model.DatacenterCatalogMeta;
import tech.easyflow.datacenter.meta.model.DatacenterTableDetailMeta;
import java.math.BigDecimal;
import java.math.BigInteger;
import java.sql.*;
import java.util.*;
import java.util.stream.Collectors;
public abstract class AbstractJdbcConnector implements DatacenterConnector {
private final DatacenterSourceType sourceType;
private final SqlDialect sqlDialect;
private final Set<DatacenterCapability> capabilities;
protected AbstractJdbcConnector(DatacenterSourceType sourceType, SqlDialect sqlDialect, Set<DatacenterCapability> capabilities) {
this.sourceType = sourceType;
this.sqlDialect = sqlDialect;
this.capabilities = capabilities;
}
@Override
public DatacenterSourceType getSourceType() {
return sourceType;
}
@Override
public Set<DatacenterCapability> getCapabilities() {
return capabilities;
}
protected abstract <T> T withConnection(DatacenterSource source, boolean cacheable, JdbcCallback<T> callback) throws Exception;
@Override
public DatacenterConnectionTestResult testConnection(DatacenterSource source) {
DatacenterConnectionTestResult result = new DatacenterConnectionTestResult();
result.setCapabilities(capabilities.stream().map(Enum::name).toList());
if (StrUtil.isBlank(source.getJdbcUrl()) && requiresJdbcUrl()) {
result.setSuccess(false);
result.setErrorCode(DatacenterConnectionErrorCode.INVALID_ARGUMENT.name());
result.setMessage("缺少 JDBC URL");
return result;
}
try {
if (StrUtil.isNotBlank(source.getDriverClassName())) {
Class.forName(source.getDriverClassName());
}
Map<String, Object> details = withConnection(source, false, connection -> {
DatabaseMetaData metaData = connection.getMetaData();
Map<String, Object> map = new LinkedHashMap<>();
map.put("databaseProductName", metaData.getDatabaseProductName());
map.put("databaseProductVersion", metaData.getDatabaseProductVersion());
map.put("url", metaData.getURL());
return map;
});
result.setSuccess(true);
result.setMessage("连接成功");
result.setDetails(details);
return result;
} catch (ClassNotFoundException ex) {
result.setSuccess(false);
result.setErrorCode(DatacenterConnectionErrorCode.DRIVER_NOT_FOUND.name());
result.setMessage(ex.getMessage());
return result;
} catch (SQLException ex) {
result.setSuccess(false);
result.setErrorCode(mapSqlError(ex));
result.setMessage(ex.getMessage());
return result;
} catch (Exception ex) {
result.setSuccess(false);
result.setErrorCode(DatacenterConnectionErrorCode.UNKNOWN_ERROR.name());
result.setMessage(ex.getMessage());
return result;
}
}
@Override
public List<DatacenterCatalogMeta> listCatalogs(DatacenterSource source) {
try {
return withConnection(source, true, connection -> {
List<DatacenterCatalogMeta> result = new ArrayList<>();
DatabaseMetaData metaData = connection.getMetaData();
try (ResultSet catalogs = metaData.getCatalogs()) {
while (catalogs.next()) {
String name = catalogs.getString("TABLE_CAT");
if (StrUtil.isBlank(name)) {
continue;
}
DatacenterCatalogMeta meta = new DatacenterCatalogMeta();
meta.setSourceId(source.getId());
meta.setCatalogName(name);
meta.setCatalogType("CATALOG");
result.add(meta);
}
}
try (ResultSet schemas = metaData.getSchemas()) {
while (schemas.next()) {
String name = schemas.getString("TABLE_SCHEM");
if (StrUtil.isBlank(name) || containsCatalog(result, name)) {
continue;
}
DatacenterCatalogMeta meta = new DatacenterCatalogMeta();
meta.setSourceId(source.getId());
meta.setCatalogName(name);
meta.setCatalogType("SCHEMA");
result.add(meta);
}
}
result = filterConfiguredCatalogs(source, result);
if (result.isEmpty()) {
String fallback = resolveCatalogName(source, null);
if (StrUtil.isNotBlank(fallback)) {
DatacenterCatalogMeta meta = new DatacenterCatalogMeta();
meta.setSourceId(source.getId());
meta.setCatalogName(fallback);
meta.setCatalogType("DEFAULT");
result.add(meta);
}
}
return result;
});
} catch (Exception ex) {
throw DatacenterConnectorExceptionSupport.wrapAccessException("读取目录失败", ex);
}
}
@Override
public List<DatacenterTable> listTables(DatacenterSource source, String catalogName) {
try {
return withConnection(source, true, connection -> {
DatabaseMetaData metaData = connection.getMetaData();
List<DatacenterTable> tables = new ArrayList<>();
try (ResultSet resultSet = metaData.getTables(resolveCatalogArgument(source, catalogName), resolveSchemaArgument(source, catalogName), "%", new String[]{"TABLE", "VIEW"})) {
while (resultSet.next()) {
DatacenterTable table = new DatacenterTable();
table.setSourceId(source.getId());
table.setTableName(resultSet.getString("TABLE_NAME"));
table.setTableDesc(resultSet.getString("REMARKS"));
table.setActualTable(resultSet.getString("TABLE_NAME"));
table.setMaterializedTable(resultSet.getString("TABLE_NAME"));
table.setAccessMode(capabilities.contains(DatacenterCapability.WRITE_MUTATION) ? "READ_WRITE" : "READ_ONLY");
table.setTableKind(resolveTableKind(resultSet.getString("TABLE_TYPE")).name());
table.setCapabilitiesJson(Map.of("capabilities", capabilities.stream().map(Enum::name).toList()));
tables.add(table);
}
}
return tables;
});
} catch (Exception ex) {
throw DatacenterConnectorExceptionSupport.wrapAccessException("读取表列表失败", ex);
}
}
@Override
public DatacenterTableDetailMeta getTableDetail(DatacenterSource source, String catalogName, String tableName) {
try {
return withConnection(source, true, connection -> {
DatabaseMetaData metaData = connection.getMetaData();
DatacenterTableDetailMeta detail = new DatacenterTableDetailMeta();
DatacenterTable table = new DatacenterTable();
table.setSourceId(source.getId());
table.setTableName(tableName);
table.setActualTable(tableName);
table.setMaterializedTable(tableName);
table.setAccessMode(capabilities.contains(DatacenterCapability.WRITE_MUTATION) ? "READ_WRITE" : "READ_ONLY");
table.setTableKind(DatacenterTableKind.EXTERNAL_TABLE.name());
table.setCapabilitiesJson(Map.of("capabilities", capabilities.stream().map(Enum::name).toList()));
detail.setTable(table);
try (ResultSet tableSet = metaData.getTables(
resolveCatalogArgument(source, catalogName),
resolveSchemaArgument(source, catalogName),
tableName,
new String[]{"TABLE", "VIEW"})) {
while (tableSet.next()) {
String currentTableName = tableSet.getString("TABLE_NAME");
if (!matchesTableName(currentTableName, tableName)) {
continue;
}
table.setTableDesc(tableSet.getString("REMARKS"));
table.setTableKind(resolveTableKind(tableSet.getString("TABLE_TYPE")).name());
break;
}
}
Set<String> primaryKeys = new HashSet<>();
try (ResultSet pkSet = metaData.getPrimaryKeys(resolveCatalogArgument(source, catalogName), resolveSchemaArgument(source, catalogName), tableName)) {
while (pkSet.next()) {
primaryKeys.add(pkSet.getString("COLUMN_NAME"));
}
}
List<DatacenterTableField> fields = new ArrayList<>();
try (ResultSet columns = metaData.getColumns(resolveCatalogArgument(source, catalogName), resolveSchemaArgument(source, catalogName), tableName, "%")) {
while (columns.next()) {
DatacenterTableField field = new DatacenterTableField();
field.setFieldName(columns.getString("COLUMN_NAME"));
field.setSourceColumnName(columns.getString("COLUMN_NAME"));
field.setFieldDesc(columns.getString("REMARKS"));
field.setJdbcType(columns.getString("TYPE_NAME"));
field.setPrecision(columns.getInt("COLUMN_SIZE"));
field.setScale(columns.getInt("DECIMAL_DIGITS"));
field.setRequired(columns.getInt("NULLABLE") == DatabaseMetaData.columnNoNulls ? 1 : 0);
field.setQueryable(1);
field.setSortable(1);
field.setWritable(capabilities.contains(DatacenterCapability.WRITE_MUTATION) ? 1 : 0);
field.setIndexed(primaryKeys.contains(field.getFieldName()) ? 1 : 0);
field.setFieldType(mapFieldType(columns.getInt("DATA_TYPE")));
fields.add(field);
}
}
detail.setFields(fields);
return detail;
});
} catch (Exception ex) {
throw DatacenterConnectorExceptionSupport.wrapAccessException("读取表详情失败", ex);
}
}
@Override
public Page<Row> queryPage(DatacenterSource source, DatacenterTable table, DatacenterQueryRequest request) {
if (!capabilities.contains(DatacenterCapability.READ_QUERY)) {
throw new BusinessException("当前数据源暂不支持查询");
}
try {
return withConnection(source, true, connection -> doQueryPage(connection, source, table, request));
} catch (Exception ex) {
throw DatacenterConnectorExceptionSupport.wrapAccessException("查询失败", ex);
}
}
@Override
public List<Row> queryBySql(DatacenterSource source, String sql) {
if (!capabilities.contains(DatacenterCapability.READ_QUERY)) {
throw new BusinessException("当前数据源暂不支持查询");
}
try {
return withConnection(source, true, connection -> doQueryBySql(connection, sql));
} catch (Exception ex) {
throw DatacenterConnectorExceptionSupport.wrapAccessException("SQL 查询失败", ex);
}
}
@Override
public void saveRow(DatacenterSource source, DatacenterTable table, JSONObject data, LoginAccount account) {
throw new BusinessException("当前数据源不支持写入");
}
@Override
public void deleteRow(DatacenterSource source, DatacenterTable table, BigInteger id, LoginAccount account) {
throw new BusinessException("当前数据源不支持删除");
}
protected boolean requiresJdbcUrl() {
return true;
}
protected Page<Row> doQueryPage(Connection connection, DatacenterSource source, DatacenterTable table, DatacenterQueryRequest request) throws SQLException {
List<Object> params = new ArrayList<>();
String selectColumns = CollectionUtils.isEmpty(request.getSelectedColumns())
? "*"
: request.getSelectedColumns().stream().map(sqlDialect::quoteIdentifier).collect(Collectors.joining(", "));
String qualifiedTable = sqlDialect.qualifyTable(resolveCatalogName(source, request.getDatasetRef() == null ? null : request.getDatasetRef().getCatalogName()), resolvePhysicalTableName(table));
StringBuilder whereClause = new StringBuilder();
if (StrUtil.isNotBlank(request.getWhere())) {
whereClause.append(" WHERE ").append(request.getWhere());
} else if (!CollectionUtils.isEmpty(request.getFilters())) {
whereClause.append(" WHERE 1=1 ");
for (DatacenterQueryFilter filter : request.getFilters()) {
appendFilter(whereClause, params, filter);
}
}
String orderClause = buildOrderClause(request.getSorts());
String baseSql = "SELECT " + selectColumns + " FROM " + qualifiedTable + whereClause + orderClause;
String countSql = "SELECT COUNT(1) FROM " + qualifiedTable + whereClause;
long total = queryCount(connection, countSql, params);
if (total == 0L) {
return new Page<>(new ArrayList<>(), request.getPageNumber(), request.getPageSize(), total);
}
String pageSql = sqlDialect.buildPageSql(baseSql, request.getPageNumber(), request.getPageSize());
List<Object> pageParams = new ArrayList<>(params);
if (pageSql.contains("FETCH NEXT") || pageSql.toLowerCase(Locale.ROOT).contains("limit")) {
pageParams.add((request.getPageNumber() - 1) * request.getPageSize());
pageParams.add(request.getPageSize());
}
List<Row> records = new ArrayList<>();
try (PreparedStatement statement = connection.prepareStatement(pageSql)) {
bindParameters(statement, pageParams);
try (ResultSet resultSet = statement.executeQuery()) {
ResultSetMetaData metaData = resultSet.getMetaData();
while (resultSet.next()) {
Row row = new Row();
for (int i = 1; i <= metaData.getColumnCount(); i++) {
String columnLabel = metaData.getColumnLabel(i);
row.put(columnLabel, normalizeValue(resultSet.getObject(i)));
}
records.add(row);
}
}
}
return new Page<>(records, request.getPageNumber(), request.getPageSize(), total);
}
protected String resolvePhysicalTableName(DatacenterTable table) {
return StrUtil.blankToDefault(table.getActualTable(), table.getTableName());
}
protected List<Row> doQueryBySql(Connection connection, String sql) throws SQLException {
try (PreparedStatement statement = connection.prepareStatement(sql);
ResultSet resultSet = statement.executeQuery()) {
return readRows(resultSet);
}
}
protected String resolveCatalogName(DatacenterSource source, String requestedCatalogName) {
if (StrUtil.isNotBlank(requestedCatalogName)) {
return requestedCatalogName;
}
if (usesCatalogNamespace()) {
return source.getDatabaseName();
}
return source.getSchemaName();
}
protected List<Row> readRows(ResultSet resultSet) throws SQLException {
List<Row> records = new ArrayList<>();
ResultSetMetaData metaData = resultSet.getMetaData();
while (resultSet.next()) {
Row row = new Row();
for (int i = 1; i <= metaData.getColumnCount(); i++) {
String columnLabel = metaData.getColumnLabel(i);
row.put(columnLabel, normalizeValue(resultSet.getObject(i)));
}
records.add(row);
}
return records;
}
protected String resolveCatalogArgument(DatacenterSource source, String catalogName) {
return usesCatalogNamespace() ? resolveCatalogName(source, catalogName) : source.getDatabaseName();
}
protected String resolveSchemaArgument(DatacenterSource source, String catalogName) {
return usesCatalogNamespace() ? source.getSchemaName() : resolveCatalogName(source, catalogName);
}
protected boolean usesCatalogNamespace() {
return sourceType == DatacenterSourceType.MYSQL
|| sourceType == DatacenterSourceType.PROJECT_MYSQL
|| sourceType == DatacenterSourceType.GBASE_8A
|| sourceType == DatacenterSourceType.GBASE_8S;
}
private List<DatacenterCatalogMeta> filterConfiguredCatalogs(DatacenterSource source, List<DatacenterCatalogMeta> items) {
if (items == null || items.isEmpty() || source == null) {
return items;
}
String configuredName = usesCatalogNamespace()
? StrUtil.trimToNull(source.getDatabaseName())
: StrUtil.trimToNull(source.getSchemaName());
if (StrUtil.isBlank(configuredName)) {
return items;
}
List<DatacenterCatalogMeta> matched = items.stream()
.filter(item -> configuredName.equalsIgnoreCase(item.getCatalogName()))
.collect(Collectors.toList());
return matched.isEmpty() ? items : matched;
}
private boolean containsCatalog(List<DatacenterCatalogMeta> items, String catalogName) {
return items.stream().anyMatch(item -> catalogName.equalsIgnoreCase(item.getCatalogName()));
}
private DatacenterTableKind resolveTableKind(String tableType) {
return "VIEW".equalsIgnoreCase(tableType) ? DatacenterTableKind.EXTERNAL_VIEW : DatacenterTableKind.EXTERNAL_TABLE;
}
private boolean matchesTableName(String currentTableName, String targetTableName) {
if (currentTableName == null || targetTableName == null) {
return false;
}
return currentTableName.equals(targetTableName)
|| currentTableName.equalsIgnoreCase(targetTableName);
}
private Integer mapFieldType(int jdbcType) {
return switch (jdbcType) {
case Types.INTEGER, Types.TINYINT, Types.SMALLINT, Types.BIGINT -> EnumFieldType.INTEGER.getCode();
case Types.FLOAT, Types.DOUBLE, Types.REAL, Types.NUMERIC, Types.DECIMAL -> EnumFieldType.NUMBER.getCode();
case Types.TIMESTAMP, Types.DATE, Types.TIME -> EnumFieldType.TIME.getCode();
case Types.BOOLEAN, Types.BIT -> EnumFieldType.BOOLEAN.getCode();
default -> EnumFieldType.STRING.getCode();
};
}
private void appendFilter(StringBuilder sql, List<Object> params, DatacenterQueryFilter filter) {
String operator = StrUtil.blankToDefault(filter.getOperator(), "EQ").toUpperCase(Locale.ROOT);
String column = sqlDialect.quoteIdentifier(filter.getColumn());
switch (operator) {
case "EQ" -> {
sql.append(" AND ").append(column).append(" = ?");
params.add(filter.getValue());
}
case "LIKE" -> {
sql.append(" AND ").append(column).append(" LIKE ?");
params.add("%" + filter.getValue() + "%");
}
case "GT" -> {
sql.append(" AND ").append(column).append(" > ?");
params.add(filter.getValue());
}
case "GTE" -> {
sql.append(" AND ").append(column).append(" >= ?");
params.add(filter.getValue());
}
case "LT" -> {
sql.append(" AND ").append(column).append(" < ?");
params.add(filter.getValue());
}
case "LTE" -> {
sql.append(" AND ").append(column).append(" <= ?");
params.add(filter.getValue());
}
case "IN" -> {
List<Object> values = filter.getValues() == null ? List.of() : filter.getValues();
if (values.isEmpty()) {
sql.append(" AND 1 = 0");
} else {
sql.append(" AND ").append(column).append(" IN (");
sql.append(values.stream().map(item -> "?").collect(Collectors.joining(",")));
sql.append(")");
params.addAll(values);
}
}
case "IS_NULL" -> sql.append(" AND ").append(column).append(" IS NULL");
default -> throw new BusinessException("不支持的过滤操作: " + operator);
}
}
private String buildOrderClause(List<DatacenterQuerySort> sorts) {
if (CollectionUtils.isEmpty(sorts)) {
return "";
}
return " ORDER BY " + sorts.stream()
.map(sort -> sqlDialect.quoteIdentifier(sort.getColumn()) + " " + ("DESC".equalsIgnoreCase(sort.getDirection()) ? "DESC" : "ASC"))
.collect(Collectors.joining(", "));
}
private long queryCount(Connection connection, String sql, List<Object> params) throws SQLException {
try (PreparedStatement statement = connection.prepareStatement(sql)) {
bindParameters(statement, params);
try (ResultSet resultSet = statement.executeQuery()) {
if (resultSet.next()) {
return resultSet.getLong(1);
}
return 0L;
}
}
}
private void bindParameters(PreparedStatement statement, List<Object> params) throws SQLException {
for (int i = 0; i < params.size(); i++) {
statement.setObject(i + 1, params.get(i));
}
}
private Object normalizeValue(Object value) {
if (value instanceof BigDecimal || value instanceof BigInteger || value instanceof Long) {
return value.toString();
}
return value;
}
private String mapSqlError(SQLException ex) {
String state = ex.getSQLState();
String message = ex.getMessage() == null ? "" : ex.getMessage().toLowerCase(Locale.ROOT);
if (message.contains("access denied") || message.contains("password") || message.contains("authentication")) {
return DatacenterConnectionErrorCode.AUTH_FAILED.name();
}
if (message.contains("unknown database") || message.contains("database does not exist")) {
return DatacenterConnectionErrorCode.DATABASE_NOT_FOUND.name();
}
if (message.contains("schema") && message.contains("does not exist")) {
return DatacenterConnectionErrorCode.SCHEMA_NOT_FOUND.name();
}
if (message.contains("permission denied") || message.contains("insufficient privilege")) {
return DatacenterConnectionErrorCode.PERMISSION_DENIED.name();
}
if (state != null && state.startsWith("08")) {
return DatacenterConnectionErrorCode.NETWORK_UNREACHABLE.name();
}
return DatacenterConnectionErrorCode.UNKNOWN_ERROR.name();
}
@FunctionalInterface
protected interface JdbcCallback<T> {
T apply(Connection connection) throws Exception;
}
}

View File

@@ -0,0 +1,80 @@
package tech.easyflow.datacenter.connector.support;
import tech.easyflow.common.web.exceptions.BusinessException;
import java.net.ConnectException;
import java.net.NoRouteToHostException;
import java.net.SocketException;
import java.net.SocketTimeoutException;
import java.net.UnknownHostException;
import java.sql.SQLException;
import java.util.Locale;
public final class DatacenterConnectorExceptionSupport {
public static final String SOURCE_UNAVAILABLE_MESSAGE = "当前连接不可用,请检查连接配置后重试";
private DatacenterConnectorExceptionSupport() {
}
public static BusinessException wrapAccessException(String fallbackMessage, Exception ex) {
if (ex instanceof BusinessException businessException && !isConnectionUnavailable(ex)) {
return businessException;
}
if (isConnectionUnavailable(ex)) {
return new BusinessException(SOURCE_UNAVAILABLE_MESSAGE);
}
return new BusinessException(fallbackMessage);
}
public static boolean isConnectionUnavailable(Throwable throwable) {
Throwable current = throwable;
while (current != null) {
if (current instanceof ClassNotFoundException
|| current instanceof ConnectException
|| current instanceof NoRouteToHostException
|| current instanceof SocketTimeoutException
|| current instanceof UnknownHostException) {
return true;
}
if (current instanceof SocketException socketException) {
String socketMessage = lowerCase(socketException.getMessage());
if (socketMessage.contains("broken pipe")
|| socketMessage.contains("connection reset")
|| socketMessage.contains("network is unreachable")) {
return true;
}
}
if (current instanceof SQLException sqlException) {
String sqlState = sqlException.getSQLState();
String message = lowerCase(sqlException.getMessage());
if ((sqlState != null && sqlState.startsWith("08"))
|| message.contains("communications link failure")
|| message.contains("connection refused")
|| message.contains("connection attempt failed")
|| message.contains("connect timed out")
|| message.contains("i/o error")
|| message.contains("io error")
|| message.contains("the network adapter could not establish the connection")
|| message.contains("unknown host")
|| message.contains("access denied")
|| message.contains("authentication failed")
|| message.contains("login failed")
|| message.contains("password authentication failed")
|| message.contains("unknown database")
|| message.contains("database does not exist")
|| (message.contains("schema") && message.contains("does not exist"))
|| message.contains("insufficient privilege")
|| message.contains("permission denied")) {
return true;
}
}
current = current.getCause();
}
return false;
}
private static String lowerCase(String message) {
return message == null ? "" : message.toLowerCase(Locale.ROOT);
}
}

View File

@@ -0,0 +1,44 @@
package tech.easyflow.datacenter.connector.support;
import com.zaxxer.hikari.HikariConfig;
import com.zaxxer.hikari.HikariDataSource;
import org.springframework.stereotype.Component;
import tech.easyflow.datacenter.meta.entity.DatacenterSource;
import tech.easyflow.datacenter.security.DatacenterCredentialCipher;
import javax.annotation.Resource;
import java.math.BigInteger;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
@Component
public class DatacenterDatasourceManager {
private final Map<BigInteger, HikariDataSource> externalDatasourceCache = new ConcurrentHashMap<>();
@Resource
private DatacenterCredentialCipher credentialCipher;
public HikariDataSource createExternalDatasource(DatacenterSource source) {
HikariConfig config = new HikariConfig();
config.setPoolName("dc-ext-" + (source.getId() == null ? "temp" : source.getId()));
config.setJdbcUrl(source.getJdbcUrl());
config.setUsername(source.getUsername());
config.setPassword(credentialCipher.decrypt(source.getCredentialCipher()));
config.setMaximumPoolSize(3);
config.setMinimumIdle(0);
config.setConnectionTimeout(5000);
config.setValidationTimeout(3000);
if (source.getDriverClassName() != null && !source.getDriverClassName().isBlank()) {
config.setDriverClassName(source.getDriverClassName());
}
return new HikariDataSource(config);
}
public HikariDataSource getOrCreateExternalDatasource(DatacenterSource source) {
if (source.getId() == null) {
return createExternalDatasource(source);
}
return externalDatasourceCache.computeIfAbsent(source.getId(), key -> createExternalDatasource(source));
}
}

View File

@@ -33,6 +33,18 @@ public class DatacenterTableBase extends DateEntity implements Serializable {
@Column(tenantId = true, comment = "租户ID")
private BigInteger tenantId;
/**
* 数据源ID
*/
@Column(comment = "数据源ID")
private BigInteger sourceId;
/**
* 目录ID
*/
@Column(comment = "目录ID")
private BigInteger catalogId;
/**
* 数据表名
*/
@@ -51,6 +63,30 @@ public class DatacenterTableBase extends DateEntity implements Serializable {
@Column(comment = "物理表名")
private String actualTable;
/**
* 表类型
*/
@Column(comment = "表类型")
private String tableKind;
/**
* 访问模式
*/
@Column(comment = "访问模式")
private String accessMode;
/**
* 物化表名
*/
@Column(comment = "物化表名")
private String materializedTable;
/**
* 是否开启版本
*/
@Column(comment = "是否开启版本")
private Integer versioningEnabled;
/**
* 数据状态
*/
@@ -87,6 +123,12 @@ public class DatacenterTableBase extends DateEntity implements Serializable {
@Column(typeHandler = FastjsonTypeHandler.class, comment = "扩展项")
private Map<String, Object> options;
/**
* 能力声明
*/
@Column(typeHandler = FastjsonTypeHandler.class, comment = "能力声明")
private Map<String, Object> capabilitiesJson;
public BigInteger getId() {
return id;
}
@@ -111,6 +153,22 @@ public class DatacenterTableBase extends DateEntity implements Serializable {
this.tenantId = tenantId;
}
public BigInteger getSourceId() {
return sourceId;
}
public void setSourceId(BigInteger sourceId) {
this.sourceId = sourceId;
}
public BigInteger getCatalogId() {
return catalogId;
}
public void setCatalogId(BigInteger catalogId) {
this.catalogId = catalogId;
}
public String getTableName() {
return tableName;
}
@@ -135,6 +193,38 @@ public class DatacenterTableBase extends DateEntity implements Serializable {
this.actualTable = actualTable;
}
public String getTableKind() {
return tableKind;
}
public void setTableKind(String tableKind) {
this.tableKind = tableKind;
}
public String getAccessMode() {
return accessMode;
}
public void setAccessMode(String accessMode) {
this.accessMode = accessMode;
}
public String getMaterializedTable() {
return materializedTable;
}
public void setMaterializedTable(String materializedTable) {
this.materializedTable = materializedTable;
}
public Integer getVersioningEnabled() {
return versioningEnabled;
}
public void setVersioningEnabled(Integer versioningEnabled) {
this.versioningEnabled = versioningEnabled;
}
public Integer getStatus() {
return status;
}
@@ -183,4 +273,12 @@ public class DatacenterTableBase extends DateEntity implements Serializable {
this.options = options;
}
public Map<String, Object> getCapabilitiesJson() {
return capabilitiesJson;
}
public void setCapabilitiesJson(Map<String, Object> capabilitiesJson) {
this.capabilitiesJson = capabilitiesJson;
}
}

View File

@@ -33,6 +33,12 @@ public class DatacenterTableFieldBase extends DateEntity implements Serializable
@Column(comment = "字段名称")
private String fieldName;
/**
* 源字段名
*/
@Column(comment = "源字段名")
private String sourceColumnName;
/**
* 字段描述
*/
@@ -45,12 +51,54 @@ public class DatacenterTableFieldBase extends DateEntity implements Serializable
@Column(comment = "字段类型")
private Integer fieldType;
/**
* JDBC 类型
*/
@Column(comment = "JDBC类型")
private String jdbcType;
/**
* 精度
*/
@Column(comment = "精度")
private Integer precision;
/**
* 小数位
*/
@Column(comment = "小数位")
private Integer scale;
/**
* 是否必填
*/
@Column(comment = "是否必填")
private Integer required;
/**
* 可查询
*/
@Column(comment = "可查询")
private Integer queryable;
/**
* 可排序
*/
@Column(comment = "可排序")
private Integer sortable;
/**
* 可写入
*/
@Column(comment = "可写入")
private Integer writable;
/**
* 是否索引
*/
@Column(comment = "是否索引")
private Integer indexed;
/**
* 扩展项
*/
@@ -105,6 +153,14 @@ public class DatacenterTableFieldBase extends DateEntity implements Serializable
this.fieldName = fieldName;
}
public String getSourceColumnName() {
return sourceColumnName;
}
public void setSourceColumnName(String sourceColumnName) {
this.sourceColumnName = sourceColumnName;
}
public String getFieldDesc() {
return fieldDesc;
}
@@ -121,6 +177,30 @@ public class DatacenterTableFieldBase extends DateEntity implements Serializable
this.fieldType = fieldType;
}
public String getJdbcType() {
return jdbcType;
}
public void setJdbcType(String jdbcType) {
this.jdbcType = jdbcType;
}
public Integer getPrecision() {
return precision;
}
public void setPrecision(Integer precision) {
this.precision = precision;
}
public Integer getScale() {
return scale;
}
public void setScale(Integer scale) {
this.scale = scale;
}
public Integer getRequired() {
return required;
}
@@ -129,6 +209,38 @@ public class DatacenterTableFieldBase extends DateEntity implements Serializable
this.required = required;
}
public Integer getQueryable() {
return queryable;
}
public void setQueryable(Integer queryable) {
this.queryable = queryable;
}
public Integer getSortable() {
return sortable;
}
public void setSortable(Integer sortable) {
this.sortable = sortable;
}
public Integer getWritable() {
return writable;
}
public void setWritable(Integer writable) {
this.writable = writable;
}
public Integer getIndexed() {
return indexed;
}
public void setIndexed(Integer indexed) {
this.indexed = indexed;
}
public Map<String, Object> getOptions() {
return options;
}

View File

@@ -1,107 +0,0 @@
package tech.easyflow.datacenter.excel;
import cn.idev.excel.context.AnalysisContext;
import cn.idev.excel.metadata.data.ReadCellData;
import cn.idev.excel.read.listener.ReadListener;
import com.alibaba.fastjson2.JSONObject;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import tech.easyflow.common.entity.LoginAccount;
import tech.easyflow.common.util.SpringContextUtil;
import tech.easyflow.datacenter.entity.DatacenterTableField;
import tech.easyflow.datacenter.service.DatacenterTableService;
import java.math.BigInteger;
import java.util.*;
public class ReadDataListener implements ReadListener<LinkedHashMap<Integer, Object>> {
private static final Logger log = LoggerFactory.getLogger(ReadDataListener.class);
private BigInteger tableId;
private List<DatacenterTableField> fields;
private LoginAccount loginAccount;
private final Map<String, Integer> headFieldIndex = new HashMap<>();
private int successCount = 0;
private int errorCount = 0;
private int totalCount = 0;
private final List<JSONObject> errorRows = new ArrayList<>();
public ReadDataListener() {
}
public ReadDataListener(BigInteger tableId, List<DatacenterTableField> fields, LoginAccount loginAccount) {
this.tableId = tableId;
this.fields = fields;
this.loginAccount = loginAccount;
}
@Override
public void invoke(LinkedHashMap<Integer, Object> o, AnalysisContext analysisContext) {
DatacenterTableService service = SpringContextUtil.getBean(DatacenterTableService.class);
JSONObject obj = new JSONObject();
for (DatacenterTableField field : fields) {
String fieldName = field.getFieldName();
Integer i = headFieldIndex.get(fieldName);
if (i != null) {
obj.put(fieldName, o.get(i));
}
}
try {
service.saveValue(tableId, obj, loginAccount);
successCount++;
} catch (Exception e) {
errorCount++;
log.error("导入数据到数据中枢失败,具体值:{}", obj, e);
errorRows.add(obj);
}
totalCount++;
}
@Override
public void invokeHead(Map<Integer, ReadCellData<?>> headMap, AnalysisContext context) {
Set<Map.Entry<Integer, ReadCellData<?>>> entries = headMap.entrySet();
for (Map.Entry<Integer, ReadCellData<?>> entry : entries) {
Integer key = entry.getKey();
String field = entry.getValue().getStringValue();
headFieldIndex.put(field, key);
}
if (headFieldIndex.size() != fields.size()) {
throw new RuntimeException("表头字段数量与表结构对应不上!");
}
}
@Override
public void doAfterAllAnalysed(AnalysisContext analysisContext) {
}
public List<DatacenterTableField> getFields() {
return fields;
}
public void setFields(List<DatacenterTableField> fields) {
this.fields = fields;
}
public int getSuccessCount() {
return successCount;
}
public int getErrorCount() {
return errorCount;
}
public int getTotalCount() {
return totalCount;
}
public List<JSONObject> getErrorRows() {
return errorRows;
}
}

View File

@@ -1,68 +0,0 @@
package tech.easyflow.datacenter.excel;
import com.alibaba.fastjson2.JSONObject;
import java.util.List;
public class ReadResVo {
/**
* 成功数
*/
private int successCount = 0;
/**
* 失败数
*/
private int errorCount = 0;
/**
* 总数
*/
private int totalCount = 0;
/**
* 错误行
*/
private List<JSONObject> errorRows;
public ReadResVo() {
}
public ReadResVo(int successCount, int errorCount, int totalCount, List<JSONObject> errorRows) {
this.successCount = successCount;
this.errorCount = errorCount;
this.totalCount = totalCount;
this.errorRows = errorRows;
}
public int getSuccessCount() {
return successCount;
}
public void setSuccessCount(int successCount) {
this.successCount = successCount;
}
public int getErrorCount() {
return errorCount;
}
public void setErrorCount(int errorCount) {
this.errorCount = errorCount;
}
public int getTotalCount() {
return totalCount;
}
public void setTotalCount(int totalCount) {
this.totalCount = totalCount;
}
public List<JSONObject> getErrorRows() {
return errorRows;
}
public void setErrorRows(List<JSONObject> errorRows) {
this.errorRows = errorRows;
}
}

View File

@@ -0,0 +1,28 @@
package tech.easyflow.datacenter.excel.model;
import tech.easyflow.datacenter.execution.model.DatacenterQueryFilter;
import tech.easyflow.datacenter.execution.model.DatasetRef;
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
public class DatacenterExcelDeriveRequest {
private DatasetRef datasetRef;
private String targetTableName;
private List<String> selectedColumns = new ArrayList<>();
private Map<String, String> renameMappings = new LinkedHashMap<>();
private List<DatacenterQueryFilter> filters = new ArrayList<>();
public DatasetRef getDatasetRef() { return datasetRef; }
public void setDatasetRef(DatasetRef datasetRef) { this.datasetRef = datasetRef; }
public String getTargetTableName() { return targetTableName; }
public void setTargetTableName(String targetTableName) { this.targetTableName = targetTableName; }
public List<String> getSelectedColumns() { return selectedColumns; }
public void setSelectedColumns(List<String> selectedColumns) { this.selectedColumns = selectedColumns; }
public Map<String, String> getRenameMappings() { return renameMappings; }
public void setRenameMappings(Map<String, String> renameMappings) { this.renameMappings = renameMappings; }
public List<DatacenterQueryFilter> getFilters() { return filters; }
public void setFilters(List<DatacenterQueryFilter> filters) { this.filters = filters; }
}

View File

@@ -0,0 +1,23 @@
package tech.easyflow.datacenter.excel.model;
import tech.easyflow.datacenter.execution.model.DatasetRef;
import java.math.BigInteger;
import java.util.ArrayList;
import java.util.List;
public class DatacenterExcelExportRequest {
private BigInteger sourceId;
private BigInteger catalogId;
private List<DatasetRef> datasetRefs = new ArrayList<>();
private String fileName;
public BigInteger getSourceId() { return sourceId; }
public void setSourceId(BigInteger sourceId) { this.sourceId = sourceId; }
public BigInteger getCatalogId() { return catalogId; }
public void setCatalogId(BigInteger catalogId) { this.catalogId = catalogId; }
public List<DatasetRef> getDatasetRefs() { return datasetRefs; }
public void setDatasetRefs(List<DatasetRef> datasetRefs) { this.datasetRefs = datasetRefs; }
public String getFileName() { return fileName; }
public void setFileName(String fileName) { this.fileName = fileName; }
}

View File

@@ -0,0 +1,22 @@
package tech.easyflow.datacenter.excel.model;
import tech.easyflow.datacenter.execution.model.DatasetRef;
import java.util.ArrayList;
import java.util.List;
public class DatacenterExcelMergeRequest {
private List<DatasetRef> datasetRefs = new ArrayList<>();
private String mergeMode;
private String targetTableName;
private String joinKey;
public List<DatasetRef> getDatasetRefs() { return datasetRefs; }
public void setDatasetRefs(List<DatasetRef> datasetRefs) { this.datasetRefs = datasetRefs; }
public String getMergeMode() { return mergeMode; }
public void setMergeMode(String mergeMode) { this.mergeMode = mergeMode; }
public String getTargetTableName() { return targetTableName; }
public void setTargetTableName(String targetTableName) { this.targetTableName = targetTableName; }
public String getJoinKey() { return joinKey; }
public void setJoinKey(String joinKey) { this.joinKey = joinKey; }
}

View File

@@ -0,0 +1,30 @@
package tech.easyflow.datacenter.excel.model;
import tech.easyflow.datacenter.execution.model.DatasetRef;
import java.math.BigInteger;
public class DatacenterExcelSplitRequest {
private BigInteger sourceId;
private BigInteger catalogId;
private DatasetRef datasetRef;
private String splitMode;
private Integer rowBatchSize;
private String fieldName;
private String targetNamePrefix;
public BigInteger getSourceId() { return sourceId; }
public void setSourceId(BigInteger sourceId) { this.sourceId = sourceId; }
public BigInteger getCatalogId() { return catalogId; }
public void setCatalogId(BigInteger catalogId) { this.catalogId = catalogId; }
public DatasetRef getDatasetRef() { return datasetRef; }
public void setDatasetRef(DatasetRef datasetRef) { this.datasetRef = datasetRef; }
public String getSplitMode() { return splitMode; }
public void setSplitMode(String splitMode) { this.splitMode = splitMode; }
public Integer getRowBatchSize() { return rowBatchSize; }
public void setRowBatchSize(Integer rowBatchSize) { this.rowBatchSize = rowBatchSize; }
public String getFieldName() { return fieldName; }
public void setFieldName(String fieldName) { this.fieldName = fieldName; }
public String getTargetNamePrefix() { return targetNamePrefix; }
public void setTargetNamePrefix(String targetNamePrefix) { this.targetNamePrefix = targetNamePrefix; }
}

View File

@@ -0,0 +1,28 @@
package tech.easyflow.datacenter.excel.service;
import org.springframework.web.multipart.MultipartFile;
import tech.easyflow.common.entity.LoginAccount;
import tech.easyflow.datacenter.excel.model.DatacenterExcelDeriveRequest;
import tech.easyflow.datacenter.excel.model.DatacenterExcelExportRequest;
import tech.easyflow.datacenter.excel.model.DatacenterExcelMergeRequest;
import tech.easyflow.datacenter.excel.model.DatacenterExcelSplitRequest;
import tech.easyflow.datacenter.meta.entity.DatacenterImportJob;
import java.math.BigInteger;
import java.util.List;
public interface DatacenterExcelImportService {
DatacenterImportJob importWorkbook(MultipartFile file, LoginAccount account) throws Exception;
DatacenterImportJob splitWorkbook(DatacenterExcelSplitRequest request, LoginAccount account);
DatacenterImportJob mergeWorkbook(DatacenterExcelMergeRequest request, LoginAccount account);
DatacenterImportJob deriveWorkbook(DatacenterExcelDeriveRequest request, LoginAccount account);
DatacenterImportJob exportWorkbook(DatacenterExcelExportRequest request, LoginAccount account) throws Exception;
DatacenterImportJob getImportJobDetail(BigInteger jobId);
List<DatacenterImportJob> listJobs(BigInteger sourceId, BigInteger tableId);
}

View File

@@ -0,0 +1,950 @@
package tech.easyflow.datacenter.excel.service.impl;
import com.alibaba.fastjson2.JSONObject;
import com.mybatisflex.core.keygen.impl.SnowFlakeIDKeyGenerator;
import com.mybatisflex.core.paginate.Page;
import com.mybatisflex.core.row.Row;
import org.apache.poi.ss.usermodel.Cell;
import org.apache.poi.ss.usermodel.DataFormatter;
import org.apache.poi.ss.usermodel.Sheet;
import org.apache.poi.ss.usermodel.Workbook;
import org.apache.poi.ss.usermodel.WorkbookFactory;
import org.apache.poi.xssf.streaming.SXSSFWorkbook;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import org.springframework.util.CollectionUtils;
import org.springframework.web.multipart.MultipartFile;
import tech.easyflow.common.constant.enums.EnumFieldType;
import tech.easyflow.common.entity.LoginAccount;
import tech.easyflow.common.web.exceptions.BusinessException;
import tech.easyflow.datacenter.adapter.DbHandleManager;
import tech.easyflow.datacenter.entity.DatacenterTable;
import tech.easyflow.datacenter.entity.DatacenterTableField;
import tech.easyflow.datacenter.excel.model.DatacenterExcelDeriveRequest;
import tech.easyflow.datacenter.excel.model.DatacenterExcelExportRequest;
import tech.easyflow.datacenter.excel.model.DatacenterExcelMergeRequest;
import tech.easyflow.datacenter.excel.model.DatacenterExcelSplitRequest;
import tech.easyflow.datacenter.excel.service.DatacenterExcelImportService;
import tech.easyflow.datacenter.execution.model.DatacenterQueryFilter;
import tech.easyflow.datacenter.execution.model.DatacenterQueryRequest;
import tech.easyflow.datacenter.execution.model.DatasetRef;
import tech.easyflow.datacenter.execution.service.DatacenterDatasetQueryService;
import tech.easyflow.datacenter.mapper.DatacenterDatasetVersionMapper;
import tech.easyflow.datacenter.mapper.DatacenterDerivedTableMapper;
import tech.easyflow.datacenter.mapper.DatacenterImportJobMapper;
import tech.easyflow.datacenter.meta.entity.DatacenterCatalog;
import tech.easyflow.datacenter.meta.entity.DatacenterDatasetVersion;
import tech.easyflow.datacenter.meta.entity.DatacenterDerivedTable;
import tech.easyflow.datacenter.meta.entity.DatacenterImportJob;
import tech.easyflow.datacenter.meta.entity.DatacenterSource;
import tech.easyflow.datacenter.meta.enums.DatacenterImportStatus;
import tech.easyflow.datacenter.meta.enums.DatacenterSourceType;
import tech.easyflow.datacenter.meta.enums.DatacenterTableKind;
import tech.easyflow.datacenter.meta.model.DatacenterTableDetailMeta;
import tech.easyflow.datacenter.meta.service.DatacenterDatasetRegistryService;
import tech.easyflow.datacenter.meta.service.DatacenterSourceService;
import javax.annotation.Resource;
import java.io.File;
import java.io.FileOutputStream;
import java.io.InputStream;
import java.math.BigInteger;
import java.nio.file.Files;
import java.nio.file.Path;
import java.time.LocalDateTime;
import java.time.format.DateTimeFormatter;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedHashMap;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.UUID;
@Service
public class DatacenterExcelImportServiceImpl implements DatacenterExcelImportService {
private static final long QUERY_BATCH_SIZE = 500L;
private static final DateTimeFormatter EXPORT_TIME_FORMAT = DateTimeFormatter.ofPattern("yyyyMMddHHmmss");
@Resource
private DatacenterSourceService sourceService;
@Resource
private DatacenterDatasetRegistryService registryService;
@Resource
private DatacenterImportJobMapper importJobMapper;
@Resource
private DatacenterDatasetVersionMapper datasetVersionMapper;
@Resource
private DatacenterDerivedTableMapper derivedTableMapper;
@Resource
private DatacenterDatasetQueryService queryService;
@Resource
private DbHandleManager dbHandleManager;
@Override
@Transactional(rollbackFor = Exception.class)
public DatacenterImportJob importWorkbook(MultipartFile file, LoginAccount account) throws Exception {
if (file == null || file.isEmpty()) {
throw new BusinessException("Excel 文件不能为空");
}
String workbookName = extractWorkbookName(file.getOriginalFilename());
DatacenterSource source = new DatacenterSource();
source.setSourceName(workbookName);
source.setSourceCode("EXCEL_" + UUID.randomUUID());
source.setSourceType(DatacenterSourceType.EXCEL.name());
source.setAccessMode("READ_WRITE");
source.setBuiltinFlag(0);
source.setConfigJson(Map.of("originFileName", file.getOriginalFilename()));
source = sourceService.saveSource(source, account);
DatacenterCatalog catalog = registryService.ensureCatalog(source, workbookName, account);
DatacenterImportJob job = createJob("EXCEL_IMPORT", source.getId(), catalog.getId(), null,
file.getOriginalFilename(), Map.of("operation", "import"), account);
long totalRows = 0L;
long successRows = 0L;
List<BigInteger> createdTableIds = new ArrayList<>();
try (InputStream inputStream = file.getInputStream(); Workbook workbook = WorkbookFactory.create(inputStream)) {
DataFormatter formatter = new DataFormatter();
for (int sheetIndex = 0; sheetIndex < workbook.getNumberOfSheets(); sheetIndex++) {
Sheet sheet = workbook.getSheetAt(sheetIndex);
org.apache.poi.ss.usermodel.Row headerRow = sheet.getRow(sheet.getFirstRowNum());
if (headerRow == null) {
continue;
}
List<DatacenterTableField> fields = buildFields(headerRow, formatter);
if (fields.isEmpty()) {
continue;
}
DatacenterTable table = new DatacenterTable();
table.setTableName(uniqueTableName(source.getId(), catalog.getId(), sheet.getSheetName()));
table.setTableDesc(sheet.getSheetName());
table.setActualTable(buildMaterializedTableName(source.getId(), sheetIndex));
table.setMaterializedTable(table.getActualTable());
table.setTableKind(DatacenterTableKind.EXCEL_MATERIALIZED.name());
table.setAccessMode("READ_WRITE");
table.setVersioningEnabled(1);
table.setCapabilitiesJson(defaultExcelCapabilities());
table.setFields(fields);
DatacenterTableDetailMeta detail = new DatacenterTableDetailMeta();
detail.setTable(table);
detail.setFields(fields);
dbHandleManager.getDbHandler().createTable(table);
DatacenterTable savedTable = registryService.registerTable(source, catalog, detail, account);
savedTable.setFields(registryService.getFields(savedTable.getId()));
createdTableIds.add(savedTable.getId());
for (int rowIndex = sheet.getFirstRowNum() + 1; rowIndex <= sheet.getLastRowNum(); rowIndex++) {
org.apache.poi.ss.usermodel.Row row = sheet.getRow(rowIndex);
if (row == null) {
continue;
}
JSONObject payload = new JSONObject();
boolean hasValue = false;
for (int cellIndex = 0; cellIndex < savedTable.getFields().size(); cellIndex++) {
String value = formatter.formatCellValue(row.getCell(cellIndex));
if (value != null && !value.isBlank()) {
hasValue = true;
}
payload.put(savedTable.getFields().get(cellIndex).getFieldName(), value);
}
if (!hasValue) {
continue;
}
dbHandleManager.getDbHandler().saveValue(savedTable, payload, account);
totalRows++;
successRows++;
}
createVersion(savedTable, "initial-import", Map.of(
"sheetName", sheet.getSheetName(),
"sourceId", source.getId(),
"originFileName", file.getOriginalFilename()
), account);
}
job.setTableId(createdTableIds.isEmpty() ? null : createdTableIds.get(0));
finishJobSuccess(job, totalRows, successRows, Map.of("tableIds", createdTableIds));
return job;
} catch (Exception ex) {
finishJobFailure(job, ex);
throw ex;
}
}
@Override
@Transactional(rollbackFor = Exception.class)
public DatacenterImportJob splitWorkbook(DatacenterExcelSplitRequest request, LoginAccount account) {
DatasetRef datasetRef = request == null ? null : request.getDatasetRef();
DatacenterImportJob job = createJob("EXCEL_SPLIT",
request == null ? null : request.getSourceId(),
request == null ? null : request.getCatalogId(),
datasetRef == null ? null : datasetRef.getTableId(),
null,
buildPayload("request", request),
account);
try {
String splitMode = normalizeMode(request == null ? null : request.getSplitMode(), "BY_ROW_COUNT");
return switch (splitMode) {
case "BY_SHEET" -> doSplitBySheet(request, account, job);
case "BY_FIELD_VALUE" -> doSplitByFieldValue(request, account, job);
default -> doSplitByRowCount(request, account, job);
};
} catch (Exception ex) {
finishJobFailure(job, ex);
throw ex;
}
}
@Override
@Transactional(rollbackFor = Exception.class)
public DatacenterImportJob mergeWorkbook(DatacenterExcelMergeRequest request, LoginAccount account) {
if (request == null || CollectionUtils.isEmpty(request.getDatasetRefs())) {
throw new BusinessException("合并数据集不能为空");
}
DatacenterImportJob job = createJob("EXCEL_MERGE", null, null, null, null, buildPayload("request", request), account);
try {
String mergeMode = normalizeMode(request.getMergeMode(), "VERTICAL");
return switch (mergeMode) {
case "HORIZONTAL" -> doHorizontalMerge(request, account, job);
default -> doVerticalMerge(request, account, job);
};
} catch (Exception ex) {
finishJobFailure(job, ex);
throw ex;
}
}
@Override
@Transactional(rollbackFor = Exception.class)
public DatacenterImportJob deriveWorkbook(DatacenterExcelDeriveRequest request, LoginAccount account) {
if (request == null || request.getDatasetRef() == null) {
throw new BusinessException("派生数据集不能为空");
}
DatacenterImportJob job = createJob("EXCEL_DERIVE",
request.getDatasetRef().getSourceId(),
request.getDatasetRef().getCatalogId(),
request.getDatasetRef().getTableId(),
null,
buildPayload("request", request),
account);
try {
DatacenterTable sourceTable = resolveTable(request.getDatasetRef());
DatacenterSource source = registryService.getSourceRequired(sourceTable.getSourceId());
DatacenterCatalog catalog = requireCatalog(sourceTable.getCatalogId());
List<String> selectedColumns = resolveSelectedColumns(sourceTable, request.getSelectedColumns());
List<DatacenterTableField> targetFields = buildDerivedFields(sourceTable, request);
DatacenterTable targetTable = createDerivedTable(source, catalog, targetFields,
request.getTargetTableName(), "DERIVE", Map.of("sourceTableId", sourceTable.getId()), account);
DatacenterQueryRequest queryRequest = new DatacenterQueryRequest();
queryRequest.setDatasetRef(registryService.resolveDatasetRef(sourceTable.getId()));
queryRequest.setSelectedColumns(selectedColumns);
queryRequest.setFilters(request.getFilters());
long successRows = copyRows(queryRequest, rows -> mapDerivedRow(rows, selectedColumns, request), targetTable, account);
createLineage(sourceTable.getId(), targetTable.getId(), "DERIVE", Map.of("request", request), account);
finishJobSuccess(job, successRows, successRows, Map.of("derivedTableId", targetTable.getId()));
job.setTableId(targetTable.getId());
return job;
} catch (Exception ex) {
finishJobFailure(job, ex);
throw ex;
}
}
@Override
@Transactional(rollbackFor = Exception.class)
public DatacenterImportJob exportWorkbook(DatacenterExcelExportRequest request, LoginAccount account) throws Exception {
List<DatacenterTable> tables = resolveExportTables(request);
if (tables.isEmpty()) {
throw new BusinessException("没有可导出的 Excel 数据集");
}
String fileName = buildExportFileName(request == null ? null : request.getFileName());
DatacenterImportJob job = createJob("EXCEL_EXPORT",
request == null ? null : request.getSourceId(),
request == null ? null : request.getCatalogId(),
null,
fileName,
buildPayload("request", request),
account);
Path exportDir = ensureExportDir();
Path exportFile = exportDir.resolve(fileName);
long totalRows = 0L;
try (SXSSFWorkbook workbook = new SXSSFWorkbook(200); FileOutputStream outputStream = new FileOutputStream(exportFile.toFile())) {
workbook.setCompressTempFiles(true);
Set<String> usedSheetNames = new HashSet<>();
for (DatacenterTable table : tables) {
String sheetName = uniqueSheetName(table.getTableName(), usedSheetNames);
org.apache.poi.ss.usermodel.Sheet sheet = workbook.createSheet(sheetName);
writeHeaderRow(sheet, table.getFields());
DatacenterQueryRequest queryRequest = new DatacenterQueryRequest();
queryRequest.setDatasetRef(registryService.resolveDatasetRef(table.getId()));
queryRequest.setSelectedColumns(table.getFields().stream().map(DatacenterTableField::getFieldName).toList());
final int[] rowIndex = {1};
totalRows += iterateRows(queryRequest, row -> {
org.apache.poi.ss.usermodel.Row excelRow = sheet.createRow(rowIndex[0]++);
for (int i = 0; i < table.getFields().size(); i++) {
Cell cell = excelRow.createCell(i);
Object value = row.get(table.getFields().get(i).getFieldName());
cell.setCellValue(value == null ? "" : String.valueOf(value));
}
});
}
workbook.write(outputStream);
workbook.dispose();
job.setStoragePath(exportFile.toAbsolutePath().toString());
finishJobSuccess(job, totalRows, totalRows, Map.of("storagePath", job.getStoragePath(), "fileName", fileName));
return job;
} catch (Exception ex) {
finishJobFailure(job, ex);
throw ex;
}
}
@Override
public DatacenterImportJob getImportJobDetail(BigInteger jobId) {
DatacenterImportJob job = importJobMapper.selectOneById(jobId);
if (job == null) {
throw new BusinessException("导入任务不存在: " + jobId);
}
return job;
}
@Override
public List<DatacenterImportJob> listJobs(BigInteger sourceId, BigInteger tableId) {
var wrapper = com.mybatisflex.core.query.QueryWrapper.create();
if (sourceId != null) {
wrapper.eq(DatacenterImportJob::getSourceId, sourceId);
}
if (tableId != null) {
wrapper.eq(DatacenterImportJob::getTableId, tableId);
}
wrapper.orderBy("created desc");
wrapper.limit(20L);
return importJobMapper.selectListByQuery(wrapper);
}
private DatacenterImportJob doSplitBySheet(DatacenterExcelSplitRequest request, LoginAccount account, DatacenterImportJob job) {
BigInteger sourceId = request.getSourceId();
BigInteger catalogId = request.getCatalogId();
if (sourceId == null && request.getDatasetRef() != null) {
sourceId = request.getDatasetRef().getSourceId();
catalogId = request.getDatasetRef().getCatalogId();
}
if (sourceId == null) {
throw new BusinessException("按 sheet 拆分需要 sourceId");
}
DatacenterSource source = registryService.getSourceRequired(sourceId);
DatacenterCatalog catalog = requireCatalog(catalogId);
List<DatacenterTable> sourceTables = registryService.listManagedTables(sourceId, catalogId);
if (sourceTables.isEmpty()) {
throw new BusinessException("当前 workbook 下没有可拆分的 sheet 表");
}
List<BigInteger> derivedIds = new ArrayList<>();
long successRows = 0L;
for (DatacenterTable sourceTable : sourceTables) {
DatacenterTable fullTable = registryService.getTableWithFields(sourceTable.getId());
DatacenterTable targetTable = createDerivedTable(source, catalog, cloneFields(fullTable.getFields()),
resolveSplitPrefix(request, fullTable.getTableName()) + "_copy", "SPLIT_BY_SHEET",
Map.of("sourceTableId", fullTable.getId()), account);
DatacenterQueryRequest queryRequest = new DatacenterQueryRequest();
queryRequest.setDatasetRef(registryService.resolveDatasetRef(fullTable.getId()));
queryRequest.setSelectedColumns(fullTable.getFields().stream().map(DatacenterTableField::getFieldName).toList());
successRows += copyRows(queryRequest, this::mapRow, targetTable, account);
createLineage(fullTable.getId(), targetTable.getId(), "SPLIT_BY_SHEET", Map.of("sourceTableId", fullTable.getId()), account);
derivedIds.add(targetTable.getId());
}
finishJobSuccess(job, successRows, successRows, Map.of("derivedTableIds", derivedIds));
return job;
}
private DatacenterImportJob doSplitByRowCount(DatacenterExcelSplitRequest request, LoginAccount account, DatacenterImportJob job) {
if (request == null || request.getDatasetRef() == null) {
throw new BusinessException("按行数拆分需要数据集");
}
int rowBatchSize = request.getRowBatchSize() == null || request.getRowBatchSize() < 1 ? 1000 : request.getRowBatchSize();
DatacenterTable sourceTable = resolveTable(request.getDatasetRef());
DatacenterSource source = registryService.getSourceRequired(sourceTable.getSourceId());
DatacenterCatalog catalog = requireCatalog(sourceTable.getCatalogId());
String baseName = resolveSplitPrefix(request, sourceTable.getTableName());
List<BigInteger> derivedIds = new ArrayList<>();
final Holder holder = new Holder();
long totalRows = iterateRows(buildFullQuery(sourceTable), row -> {
if (holder.targetTable == null || holder.currentSize >= rowBatchSize) {
holder.batchNo++;
holder.targetTable = createDerivedTable(source, catalog, cloneFields(sourceTable.getFields()),
baseName + "_part_" + holder.batchNo, "SPLIT_BY_ROW_COUNT",
Map.of("sourceTableId", sourceTable.getId(), "batchNo", holder.batchNo, "rowBatchSize", rowBatchSize), account);
createLineage(sourceTable.getId(), holder.targetTable.getId(), "SPLIT_BY_ROW_COUNT",
Map.of("sourceTableId", sourceTable.getId(), "batchNo", holder.batchNo), account);
derivedIds.add(holder.targetTable.getId());
holder.currentSize = 0;
}
saveToTable(holder.targetTable, mapRow(row), account);
holder.currentSize++;
});
finishJobSuccess(job, totalRows, totalRows, Map.of("derivedTableIds", derivedIds));
return job;
}
private DatacenterImportJob doSplitByFieldValue(DatacenterExcelSplitRequest request, LoginAccount account, DatacenterImportJob job) {
if (request == null || request.getDatasetRef() == null || request.getFieldName() == null || request.getFieldName().isBlank()) {
throw new BusinessException("按字段值拆分需要数据集和字段名");
}
DatacenterTable sourceTable = resolveTable(request.getDatasetRef());
DatacenterSource source = registryService.getSourceRequired(sourceTable.getSourceId());
DatacenterCatalog catalog = requireCatalog(sourceTable.getCatalogId());
DatacenterTableField splitField = sourceTable.getFields().stream()
.filter(field -> request.getFieldName().equals(field.getFieldName()))
.findFirst()
.orElseThrow(() -> new BusinessException("拆分字段不存在: " + request.getFieldName()));
String prefix = resolveSplitPrefix(request, sourceTable.getTableName());
Map<String, DatacenterTable> targets = new LinkedHashMap<>();
List<BigInteger> derivedIds = new ArrayList<>();
long totalRows = iterateRows(buildFullQuery(sourceTable), row -> {
String fieldValue = stringify(row.get(splitField.getFieldName()));
String bucket = fieldValue == null || fieldValue.isBlank() ? "empty" : fieldValue;
DatacenterTable targetTable = targets.get(bucket);
if (targetTable == null) {
targetTable = createDerivedTable(source, catalog, cloneFields(sourceTable.getFields()),
prefix + "_" + normalizeIdentifier(bucket), "SPLIT_BY_FIELD_VALUE",
Map.of("sourceTableId", sourceTable.getId(), "fieldName", splitField.getFieldName(), "fieldValue", bucket), account);
createLineage(sourceTable.getId(), targetTable.getId(), "SPLIT_BY_FIELD_VALUE",
Map.of("sourceTableId", sourceTable.getId(), "fieldName", splitField.getFieldName(), "fieldValue", bucket), account);
targets.put(bucket, targetTable);
derivedIds.add(targetTable.getId());
}
saveToTable(targetTable, mapRow(row), account);
});
finishJobSuccess(job, totalRows, totalRows, Map.of("derivedTableIds", derivedIds));
return job;
}
private DatacenterImportJob doVerticalMerge(DatacenterExcelMergeRequest request, LoginAccount account, DatacenterImportJob job) {
List<DatacenterTable> tables = request.getDatasetRefs().stream().map(this::resolveTable).toList();
DatacenterTable firstTable = tables.get(0);
DatacenterSource source = registryService.getSourceRequired(firstTable.getSourceId());
DatacenterCatalog catalog = requireCatalog(firstTable.getCatalogId());
assertSameCatalog(tables);
assertSameFields(tables);
DatacenterTable targetTable = createDerivedTable(source, catalog, cloneFields(firstTable.getFields()),
request.getTargetTableName(), "MERGE_VERTICAL", Map.of("sourceTableIds", tables.stream().map(DatacenterTable::getId).toList()), account);
long successRows = 0L;
for (DatacenterTable table : tables) {
successRows += copyRows(buildFullQuery(table), this::mapRow, targetTable, account);
createLineage(table.getId(), targetTable.getId(), "MERGE_VERTICAL", Map.of("sourceTableId", table.getId()), account);
}
job.setTableId(targetTable.getId());
finishJobSuccess(job, successRows, successRows, Map.of("derivedTableId", targetTable.getId()));
return job;
}
private DatacenterImportJob doHorizontalMerge(DatacenterExcelMergeRequest request, LoginAccount account, DatacenterImportJob job) {
if (request.getJoinKey() == null || request.getJoinKey().isBlank()) {
throw new BusinessException("横向合并必须指定 joinKey");
}
List<DatacenterTable> tables = request.getDatasetRefs().stream().map(this::resolveTable).toList();
DatacenterTable firstTable = tables.get(0);
DatacenterSource source = registryService.getSourceRequired(firstTable.getSourceId());
DatacenterCatalog catalog = requireCatalog(firstTable.getCatalogId());
assertSameCatalog(tables);
List<DatacenterTableField> mergedFields = new ArrayList<>();
Set<String> usedFieldNames = new LinkedHashSet<>();
Map<BigInteger, Map<String, String>> fieldMappings = new LinkedHashMap<>();
for (DatacenterTable table : tables) {
Map<String, String> mapping = new LinkedHashMap<>();
for (DatacenterTableField field : table.getFields()) {
String targetFieldName;
if (field.getFieldName().equals(request.getJoinKey())) {
targetFieldName = field.getFieldName();
} else {
targetFieldName = field.getFieldName();
if (usedFieldNames.contains(targetFieldName)) {
targetFieldName = normalizeIdentifier(table.getTableName()) + "_" + targetFieldName;
}
}
if (!usedFieldNames.contains(targetFieldName)) {
usedFieldNames.add(targetFieldName);
mergedFields.add(cloneField(field, targetFieldName, field.getFieldDesc()));
}
mapping.put(field.getFieldName(), targetFieldName);
}
fieldMappings.put(table.getId(), mapping);
}
DatacenterTable targetTable = createDerivedTable(source, catalog, mergedFields,
request.getTargetTableName(), "MERGE_HORIZONTAL", Map.of("sourceTableIds", tables.stream().map(DatacenterTable::getId).toList(), "joinKey", request.getJoinKey()), account);
Map<String, JSONObject> mergedRows = new LinkedHashMap<>();
for (DatacenterTable table : tables) {
Map<String, String> mapping = fieldMappings.get(table.getId());
iterateRows(buildFullQuery(table), row -> {
String joinValue = stringify(row.get(request.getJoinKey()));
if (joinValue == null || joinValue.isBlank()) {
return;
}
JSONObject target = mergedRows.computeIfAbsent(joinValue, key -> new JSONObject());
mapping.forEach((sourceField, targetField) -> target.put(targetField, row.get(sourceField)));
});
createLineage(table.getId(), targetTable.getId(), "MERGE_HORIZONTAL", Map.of("sourceTableId", table.getId(), "joinKey", request.getJoinKey()), account);
}
for (JSONObject row : mergedRows.values()) {
saveToTable(targetTable, row, account);
}
job.setTableId(targetTable.getId());
finishJobSuccess(job, (long) mergedRows.size(), (long) mergedRows.size(), Map.of("derivedTableId", targetTable.getId()));
return job;
}
private DatacenterImportJob createJob(String jobType, BigInteger sourceId, BigInteger catalogId, BigInteger tableId,
String fileName, Map<String, Object> payload, LoginAccount account) {
DatacenterImportJob job = new DatacenterImportJob();
job.setSourceId(sourceId);
job.setCatalogId(catalogId);
job.setTableId(tableId);
job.setTenantId(account == null || account.getTenantId() == null ? BigInteger.ZERO : account.getTenantId());
job.setDeptId(account == null || account.getDeptId() == null ? BigInteger.ZERO : account.getDeptId());
job.setJobType(jobType);
job.setFileName(fileName);
job.setStatus(DatacenterImportStatus.RUNNING.name());
job.setPayloadJson(payload == null ? new LinkedHashMap<>() : new LinkedHashMap<>(payload));
job.setStartedAt(new Date());
job.setCreated(new Date());
job.setModified(new Date());
job.setCreatedBy(account == null ? BigInteger.ZERO : account.getId());
job.setModifiedBy(account == null ? BigInteger.ZERO : account.getId());
importJobMapper.insert(job);
return job;
}
private Map<String, Object> buildPayload(String key, Object value) {
Map<String, Object> payload = new LinkedHashMap<>();
payload.put(key, value);
return payload;
}
private void finishJobSuccess(DatacenterImportJob job, long totalRows, long successRows, Map<String, Object> payload) {
job.setStatus(DatacenterImportStatus.SUCCESS.name());
job.setTotalRows(totalRows);
job.setSuccessRows(successRows);
job.setErrorRows(Math.max(0L, totalRows - successRows));
if (payload != null) {
job.setPayloadJson(new LinkedHashMap<>(payload));
}
job.setFinishedAt(new Date());
job.setModified(new Date());
importJobMapper.update(job);
}
private void finishJobFailure(DatacenterImportJob job, Exception ex) {
job.setStatus(DatacenterImportStatus.FAILED.name());
job.setErrorSummary(ex.getMessage());
job.setFinishedAt(new Date());
job.setModified(new Date());
importJobMapper.update(job);
}
private DatacenterTable resolveTable(DatasetRef datasetRef) {
if (datasetRef == null || datasetRef.getTableId() == null) {
throw new BusinessException("缺少数据集 tableId");
}
return registryService.getTableWithFields(datasetRef.getTableId());
}
private DatacenterCatalog requireCatalog(BigInteger catalogId) {
DatacenterCatalog catalog = registryService.getCatalogById(catalogId);
if (catalog == null) {
throw new BusinessException("目录不存在: " + catalogId);
}
return catalog;
}
private DatacenterTable createDerivedTable(DatacenterSource source, DatacenterCatalog catalog, List<DatacenterTableField> fields,
String tableName, String deriveType, Map<String, Object> config, LoginAccount account) {
DatacenterTable table = new DatacenterTable();
String resolvedName = uniqueTableName(source.getId(), catalog.getId(), normalizeLogicalName(tableName, deriveType));
table.setTableName(resolvedName);
table.setTableDesc(resolvedName);
table.setActualTable(buildMaterializedTableName(source.getId(), Math.abs(Objects.hash(resolvedName, deriveType))));
table.setMaterializedTable(table.getActualTable());
table.setTableKind(DatacenterTableKind.DERIVED_TABLE.name());
table.setAccessMode("READ_WRITE");
table.setVersioningEnabled(1);
table.setCapabilitiesJson(defaultExcelCapabilities());
table.setFields(fields);
DatacenterTableDetailMeta detail = new DatacenterTableDetailMeta();
detail.setTable(table);
detail.setFields(fields);
dbHandleManager.getDbHandler().createTable(table);
DatacenterTable savedTable = registryService.registerTable(source, catalog, detail, account);
savedTable.setFields(registryService.getFields(savedTable.getId()));
createVersion(savedTable, deriveType.toLowerCase(Locale.ROOT), config, account);
return savedTable;
}
private DatacenterDatasetVersion createVersion(DatacenterTable table, String versionLabel, Map<String, Object> snapshot, LoginAccount account) {
QueryWrapperWrapper wrapper = new QueryWrapperWrapper(table.getId());
DatacenterDatasetVersion version = new DatacenterDatasetVersion();
version.setTableId(table.getId());
version.setTenantId(table.getTenantId());
version.setDeptId(table.getDeptId());
version.setVersionNo(wrapper.nextVersionNo(datasetVersionMapper));
version.setVersionLabel(versionLabel);
version.setMaterializedTable(table.getMaterializedTable());
version.setSnapshotJson(snapshot == null ? new LinkedHashMap<>() : new LinkedHashMap<>(snapshot));
version.setStatus(0);
version.setCreated(new Date());
version.setModified(new Date());
version.setCreatedBy(account == null ? BigInteger.ZERO : account.getId());
version.setModifiedBy(account == null ? BigInteger.ZERO : account.getId());
datasetVersionMapper.insert(version);
return version;
}
private void createLineage(BigInteger sourceTableId, BigInteger derivedTableId, String deriveType, Map<String, Object> config, LoginAccount account) {
DatacenterDerivedTable relation = new DatacenterDerivedTable();
relation.setSourceTableId(sourceTableId);
relation.setDerivedTableId(derivedTableId);
relation.setDeriveType(deriveType);
relation.setDeriveConfigJson(config == null ? new LinkedHashMap<>() : new LinkedHashMap<>(config));
relation.setStatus(0);
relation.setTenantId(account == null || account.getTenantId() == null ? BigInteger.ZERO : account.getTenantId());
relation.setDeptId(account == null || account.getDeptId() == null ? BigInteger.ZERO : account.getDeptId());
relation.setCreated(new Date());
relation.setModified(new Date());
relation.setCreatedBy(account == null ? BigInteger.ZERO : account.getId());
relation.setModifiedBy(account == null ? BigInteger.ZERO : account.getId());
derivedTableMapper.insert(relation);
}
private long copyRows(DatacenterQueryRequest queryRequest, RowMapper mapper, DatacenterTable targetTable, LoginAccount account) {
return iterateRows(queryRequest, row -> saveToTable(targetTable, mapper.map(row), account));
}
private long iterateRows(DatacenterQueryRequest queryRequest, RowConsumer consumer) {
long total = 0L;
long pageNumber = 1L;
while (true) {
queryRequest.setPageNumber(pageNumber);
queryRequest.setPageSize(QUERY_BATCH_SIZE);
Page<Row> page = queryService.queryPage(queryRequest);
if (page.getRecords() == null || page.getRecords().isEmpty()) {
break;
}
for (Row row : page.getRecords()) {
consumer.accept(row);
total++;
}
if (page.getRecords().size() < QUERY_BATCH_SIZE) {
break;
}
pageNumber++;
}
return total;
}
private void saveToTable(DatacenterTable targetTable, JSONObject data, LoginAccount account) {
dbHandleManager.getDbHandler().saveValue(targetTable, data, account);
}
private DatacenterQueryRequest buildFullQuery(DatacenterTable table) {
DatacenterQueryRequest queryRequest = new DatacenterQueryRequest();
queryRequest.setDatasetRef(registryService.resolveDatasetRef(table.getId()));
queryRequest.setSelectedColumns(table.getFields().stream().map(DatacenterTableField::getFieldName).toList());
return queryRequest;
}
private JSONObject mapRow(Row row) {
JSONObject payload = new JSONObject();
row.forEach(payload::put);
payload.remove("id");
payload.remove("dept_id");
payload.remove("tenant_id");
payload.remove("created");
payload.remove("created_by");
payload.remove("modified");
payload.remove("modified_by");
payload.remove("remark");
return payload;
}
private JSONObject mapDerivedRow(Row row, List<String> selectedColumns, DatacenterExcelDeriveRequest request) {
JSONObject payload = new JSONObject();
for (String column : selectedColumns) {
String targetName = request.getRenameMappings().getOrDefault(column, column);
payload.put(targetName, row.get(column));
}
return payload;
}
private List<String> resolveSelectedColumns(DatacenterTable sourceTable, List<String> selectedColumns) {
if (CollectionUtils.isEmpty(selectedColumns)) {
return sourceTable.getFields().stream().map(DatacenterTableField::getFieldName).toList();
}
return selectedColumns;
}
private List<DatacenterTableField> buildDerivedFields(DatacenterTable sourceTable, DatacenterExcelDeriveRequest request) {
List<String> selectedColumns = resolveSelectedColumns(sourceTable, request.getSelectedColumns());
Map<String, DatacenterTableField> fieldMap = new LinkedHashMap<>();
for (DatacenterTableField field : sourceTable.getFields()) {
fieldMap.put(field.getFieldName(), field);
}
List<DatacenterTableField> fields = new ArrayList<>();
for (String column : selectedColumns) {
DatacenterTableField sourceField = fieldMap.get(column);
if (sourceField == null) {
throw new BusinessException("派生字段不存在: " + column);
}
String targetName = request.getRenameMappings().getOrDefault(column, column);
fields.add(cloneField(sourceField, targetName, targetName));
}
return fields;
}
private List<DatacenterTableField> cloneFields(List<DatacenterTableField> sourceFields) {
List<DatacenterTableField> fields = new ArrayList<>();
for (DatacenterTableField field : sourceFields) {
fields.add(cloneField(field, field.getFieldName(), field.getFieldDesc()));
}
return fields;
}
private DatacenterTableField cloneField(DatacenterTableField source, String fieldName, String fieldDesc) {
DatacenterTableField field = new DatacenterTableField();
field.setFieldName(fieldName);
field.setSourceColumnName(source.getSourceColumnName());
field.setFieldDesc(fieldDesc);
field.setFieldType(source.getFieldType());
field.setJdbcType(source.getJdbcType());
field.setPrecision(source.getPrecision());
field.setScale(source.getScale());
field.setRequired(source.getRequired());
field.setQueryable(source.getQueryable());
field.setSortable(source.getSortable());
field.setWritable(source.getWritable());
field.setIndexed(source.getIndexed());
field.setOptions(source.getOptions());
return field;
}
private Map<String, Object> defaultExcelCapabilities() {
return Map.of("capabilities", List.of("READ_QUERY", "WRITE_MUTATION", "MATERIALIZE", "EXPORT"));
}
private void assertSameCatalog(List<DatacenterTable> tables) {
Set<BigInteger> catalogIds = new HashSet<>();
Set<BigInteger> sourceIds = new HashSet<>();
for (DatacenterTable table : tables) {
catalogIds.add(table.getCatalogId());
sourceIds.add(table.getSourceId());
}
if (catalogIds.size() > 1 || sourceIds.size() > 1) {
throw new BusinessException("Excel 操作暂只支持同一 workbook/catalog 下的数据集");
}
}
private void assertSameFields(List<DatacenterTable> tables) {
List<String> first = tables.get(0).getFields().stream().map(DatacenterTableField::getFieldName).toList();
for (int i = 1; i < tables.size(); i++) {
List<String> current = tables.get(i).getFields().stream().map(DatacenterTableField::getFieldName).toList();
if (!first.equals(current)) {
throw new BusinessException("纵向合并仅支持同结构表");
}
}
}
private List<DatacenterTable> resolveExportTables(DatacenterExcelExportRequest request) {
List<DatacenterTable> tables = new ArrayList<>();
if (request != null && !CollectionUtils.isEmpty(request.getDatasetRefs())) {
for (DatasetRef datasetRef : request.getDatasetRefs()) {
tables.add(resolveTable(datasetRef));
}
return tables;
}
if (request == null || request.getSourceId() == null) {
throw new BusinessException("导出需要 sourceId 或 datasetRefs");
}
tables.addAll(registryService.listManagedTables(request.getSourceId(), request.getCatalogId()));
return tables.stream().map(table -> registryService.getTableWithFields(table.getId())).toList();
}
private void writeHeaderRow(org.apache.poi.ss.usermodel.Sheet sheet, List<DatacenterTableField> fields) {
org.apache.poi.ss.usermodel.Row headerRow = sheet.createRow(0);
for (int i = 0; i < fields.size(); i++) {
Cell cell = headerRow.createCell(i);
cell.setCellValue(fields.get(i).getFieldDesc());
}
}
private Path ensureExportDir() throws Exception {
Path dir = Path.of(System.getProperty("java.io.tmpdir"), "easyflow-datacenter", "exports");
Files.createDirectories(dir);
return dir;
}
private String buildExportFileName(String rawFileName) {
String baseName = rawFileName == null || rawFileName.isBlank() ? "excel_export" : extractWorkbookName(rawFileName);
return normalizeIdentifier(baseName) + "_" + EXPORT_TIME_FORMAT.format(LocalDateTime.now()) + ".xlsx";
}
private String uniqueSheetName(String rawName, Set<String> usedSheetNames) {
String base = rawName == null || rawName.isBlank() ? "Sheet" : rawName;
base = base.length() > 31 ? base.substring(0, 31) : base;
String result = base;
int suffix = 1;
while (usedSheetNames.contains(result)) {
String suffixText = "_" + suffix++;
int limit = Math.max(1, 31 - suffixText.length());
result = base.substring(0, Math.min(base.length(), limit)) + suffixText;
}
usedSheetNames.add(result);
return result;
}
private List<DatacenterTableField> buildFields(org.apache.poi.ss.usermodel.Row headerRow, DataFormatter formatter) {
List<DatacenterTableField> fields = new ArrayList<>();
Set<String> usedNames = new HashSet<>();
short lastCellNum = headerRow.getLastCellNum();
for (int cellIndex = 0; cellIndex < lastCellNum; cellIndex++) {
String header = formatter.formatCellValue(headerRow.getCell(cellIndex));
String fieldName = normalizeIdentifier(header, cellIndex, usedNames);
DatacenterTableField field = new DatacenterTableField();
field.setFieldName(fieldName);
field.setSourceColumnName(header);
field.setFieldDesc(header == null || header.isBlank() ? fieldName : header);
field.setFieldType(EnumFieldType.STRING.getCode());
field.setJdbcType("VARCHAR");
field.setPrecision(255);
field.setScale(0);
field.setRequired(0);
field.setQueryable(1);
field.setSortable(1);
field.setWritable(1);
field.setIndexed(0);
fields.add(field);
}
return fields;
}
private String normalizeMode(String value, String defaultValue) {
return value == null || value.isBlank() ? defaultValue : value.trim().toUpperCase(Locale.ROOT);
}
private String resolveSplitPrefix(DatacenterExcelSplitRequest request, String fallback) {
if (request != null && request.getTargetNamePrefix() != null && !request.getTargetNamePrefix().isBlank()) {
return request.getTargetNamePrefix();
}
return fallback + "_split";
}
private String normalizeLogicalName(String tableName, String deriveType) {
if (tableName != null && !tableName.isBlank()) {
return tableName;
}
return deriveType.toLowerCase(Locale.ROOT) + "_" + System.currentTimeMillis();
}
private String uniqueTableName(BigInteger sourceId, BigInteger catalogId, String rawName) {
String baseName = rawName == null || rawName.isBlank() ? "dataset" : rawName;
baseName = baseName.trim();
String result = baseName;
int suffix = 1;
while (tableNameExists(sourceId, catalogId, result)) {
result = baseName + "_" + suffix++;
}
return result;
}
private boolean tableNameExists(BigInteger sourceId, BigInteger catalogId, String tableName) {
List<DatacenterTable> tables = registryService.listManagedTables(sourceId, catalogId);
return tables.stream().anyMatch(table -> tableName.equals(table.getTableName()));
}
private String extractWorkbookName(String originalFileName) {
if (originalFileName == null || originalFileName.isBlank()) {
return "excel_workbook";
}
int index = originalFileName.lastIndexOf('.');
return index > 0 ? originalFileName.substring(0, index) : originalFileName;
}
private String buildMaterializedTableName(BigInteger sourceId, int sheetIndex) {
long snowId = new SnowFlakeIDKeyGenerator().nextId();
return "tb_excel_" + sourceId + "_" + sheetIndex + "_" + snowId;
}
private String normalizeIdentifier(String raw) {
if (raw == null || raw.isBlank()) {
return "value";
}
String normalized = raw.trim().toLowerCase(Locale.ROOT).replaceAll("[^a-z0-9_\\u4e00-\\u9fa5]+", "_");
normalized = normalized.replaceAll("_+", "_");
if (normalized.isBlank()) {
return "value";
}
return normalized;
}
private String normalizeIdentifier(String raw, int index, Set<String> usedNames) {
String value = normalizeIdentifier(raw);
if (value.isBlank() || "value".equals(value)) {
value = "col_" + (index + 1);
}
if (Character.isDigit(value.charAt(0))) {
value = "col_" + value;
}
String result = value;
int suffix = 1;
while (usedNames.contains(result)) {
result = value + "_" + suffix++;
}
usedNames.add(result);
return result;
}
private String stringify(Object value) {
return value == null ? null : String.valueOf(value);
}
private static final class Holder {
private DatacenterTable targetTable;
private int batchNo;
private int currentSize;
}
private interface RowConsumer {
void accept(Row row);
}
private interface RowMapper {
JSONObject map(Row row);
}
private static final class QueryWrapperWrapper {
private final BigInteger tableId;
private QueryWrapperWrapper(BigInteger tableId) {
this.tableId = tableId;
}
private int nextVersionNo(DatacenterDatasetVersionMapper mapper) {
return mapper.selectListByQuery(com.mybatisflex.core.query.QueryWrapper.create()
.eq(DatacenterDatasetVersion::getTableId, tableId)
.orderBy("version_no desc"))
.stream()
.findFirst()
.map(version -> version.getVersionNo() + 1)
.orElse(1);
}
}
}

View File

@@ -0,0 +1,24 @@
package tech.easyflow.datacenter.execution.model;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
public class DatacenterConnectionTestResult {
private boolean success;
private String errorCode;
private String message;
private List<String> capabilities = new ArrayList<>();
private Map<String, Object> details;
public boolean isSuccess() { return success; }
public void setSuccess(boolean success) { this.success = success; }
public String getErrorCode() { return errorCode; }
public void setErrorCode(String errorCode) { this.errorCode = errorCode; }
public String getMessage() { return message; }
public void setMessage(String message) { this.message = message; }
public List<String> getCapabilities() { return capabilities; }
public void setCapabilities(List<String> capabilities) { this.capabilities = capabilities; }
public Map<String, Object> getDetails() { return details; }
public void setDetails(Map<String, Object> details) { this.details = details; }
}

View File

@@ -0,0 +1,19 @@
package tech.easyflow.datacenter.execution.model;
import java.util.List;
public class DatacenterQueryFilter {
private String column;
private String operator;
private Object value;
private List<Object> values;
public String getColumn() { return column; }
public void setColumn(String column) { this.column = column; }
public String getOperator() { return operator; }
public void setOperator(String operator) { this.operator = operator; }
public Object getValue() { return value; }
public void setValue(Object value) { this.value = value; }
public List<Object> getValues() { return values; }
public void setValues(List<Object> values) { this.values = values; }
}

View File

@@ -0,0 +1,29 @@
package tech.easyflow.datacenter.execution.model;
import java.util.ArrayList;
import java.util.List;
public class DatacenterQueryRequest {
private DatasetRef datasetRef;
private Long pageNumber = 1L;
private Long pageSize = 10L;
private List<DatacenterQueryFilter> filters = new ArrayList<>();
private List<DatacenterQuerySort> sorts = new ArrayList<>();
private List<String> selectedColumns = new ArrayList<>();
private String where;
public DatasetRef getDatasetRef() { return datasetRef; }
public void setDatasetRef(DatasetRef datasetRef) { this.datasetRef = datasetRef; }
public Long getPageNumber() { return pageNumber; }
public void setPageNumber(Long pageNumber) { this.pageNumber = pageNumber; }
public Long getPageSize() { return pageSize; }
public void setPageSize(Long pageSize) { this.pageSize = pageSize; }
public List<DatacenterQueryFilter> getFilters() { return filters; }
public void setFilters(List<DatacenterQueryFilter> filters) { this.filters = filters; }
public List<DatacenterQuerySort> getSorts() { return sorts; }
public void setSorts(List<DatacenterQuerySort> sorts) { this.sorts = sorts; }
public List<String> getSelectedColumns() { return selectedColumns; }
public void setSelectedColumns(List<String> selectedColumns) { this.selectedColumns = selectedColumns; }
public String getWhere() { return where; }
public void setWhere(String where) { this.where = where; }
}

View File

@@ -0,0 +1,11 @@
package tech.easyflow.datacenter.execution.model;
public class DatacenterQuerySort {
private String column;
private String direction;
public String getColumn() { return column; }
public void setColumn(String column) { this.column = column; }
public String getDirection() { return direction; }
public void setDirection(String direction) { this.direction = direction; }
}

View File

@@ -0,0 +1,39 @@
package tech.easyflow.datacenter.execution.model;
import tech.easyflow.datacenter.entity.DatacenterTable;
import tech.easyflow.datacenter.entity.DatacenterTableField;
import tech.easyflow.datacenter.meta.entity.DatacenterCatalog;
import tech.easyflow.datacenter.meta.entity.DatacenterDatasetVersion;
import tech.easyflow.datacenter.meta.entity.DatacenterDerivedTable;
import tech.easyflow.datacenter.meta.entity.DatacenterSource;
import java.util.ArrayList;
import java.util.List;
public class DatacenterSchemaResponse {
private DatasetRef datasetRef;
private DatacenterSource source;
private DatacenterCatalog catalog;
private DatacenterTable table;
private List<DatacenterTableField> fields = new ArrayList<>();
private List<DatacenterDatasetVersion> versions = new ArrayList<>();
private List<DatacenterDerivedTable> upstreamLineage = new ArrayList<>();
private List<DatacenterDerivedTable> downstreamLineage = new ArrayList<>();
public DatasetRef getDatasetRef() { return datasetRef; }
public void setDatasetRef(DatasetRef datasetRef) { this.datasetRef = datasetRef; }
public DatacenterSource getSource() { return source; }
public void setSource(DatacenterSource source) { this.source = source; }
public DatacenterCatalog getCatalog() { return catalog; }
public void setCatalog(DatacenterCatalog catalog) { this.catalog = catalog; }
public DatacenterTable getTable() { return table; }
public void setTable(DatacenterTable table) { this.table = table; }
public List<DatacenterTableField> getFields() { return fields; }
public void setFields(List<DatacenterTableField> fields) { this.fields = fields; }
public List<DatacenterDatasetVersion> getVersions() { return versions; }
public void setVersions(List<DatacenterDatasetVersion> versions) { this.versions = versions; }
public List<DatacenterDerivedTable> getUpstreamLineage() { return upstreamLineage; }
public void setUpstreamLineage(List<DatacenterDerivedTable> upstreamLineage) { this.upstreamLineage = upstreamLineage; }
public List<DatacenterDerivedTable> getDownstreamLineage() { return downstreamLineage; }
public void setDownstreamLineage(List<DatacenterDerivedTable> downstreamLineage) { this.downstreamLineage = downstreamLineage; }
}

View File

@@ -0,0 +1,23 @@
package tech.easyflow.datacenter.execution.model;
public class DatacenterSqlQueryRequest {
private DatasetRef datasetRef;
private String sql;
public DatasetRef getDatasetRef() {
return datasetRef;
}
public void setDatasetRef(DatasetRef datasetRef) {
this.datasetRef = datasetRef;
}
public String getSql() {
return sql;
}
public void setSql(String sql) {
this.sql = sql;
}
}

View File

@@ -0,0 +1,25 @@
package tech.easyflow.datacenter.execution.model;
import java.math.BigInteger;
public class DatasetRef {
private BigInteger sourceId;
private BigInteger catalogId;
private String catalogName;
private BigInteger tableId;
private String tableName;
private BigInteger versionId;
public BigInteger getSourceId() { return sourceId; }
public void setSourceId(BigInteger sourceId) { this.sourceId = sourceId; }
public BigInteger getCatalogId() { return catalogId; }
public void setCatalogId(BigInteger catalogId) { this.catalogId = catalogId; }
public String getCatalogName() { return catalogName; }
public void setCatalogName(String catalogName) { this.catalogName = catalogName; }
public BigInteger getTableId() { return tableId; }
public void setTableId(BigInteger tableId) { this.tableId = tableId; }
public String getTableName() { return tableName; }
public void setTableName(String tableName) { this.tableName = tableName; }
public BigInteger getVersionId() { return versionId; }
public void setVersionId(BigInteger versionId) { this.versionId = versionId; }
}

View File

@@ -0,0 +1,18 @@
package tech.easyflow.datacenter.execution.service;
import com.mybatisflex.core.paginate.Page;
import com.mybatisflex.core.row.Row;
import tech.easyflow.datacenter.execution.model.DatacenterQueryRequest;
import tech.easyflow.datacenter.execution.model.DatacenterSchemaResponse;
import tech.easyflow.datacenter.execution.model.DatacenterSqlQueryRequest;
import tech.easyflow.datacenter.execution.model.DatasetRef;
import java.util.List;
public interface DatacenterDatasetQueryService {
Page<Row> queryPage(DatacenterQueryRequest request);
List<Row> queryBySql(DatacenterSqlQueryRequest request);
DatacenterSchemaResponse getSchema(DatasetRef datasetRef);
}

View File

@@ -0,0 +1,13 @@
package tech.easyflow.datacenter.execution.service;
import com.alibaba.fastjson2.JSONObject;
import tech.easyflow.common.entity.LoginAccount;
import tech.easyflow.datacenter.execution.model.DatasetRef;
import java.math.BigInteger;
public interface DatacenterDatasetWriteService {
void saveRow(DatasetRef datasetRef, JSONObject data, LoginAccount account);
void deleteRow(DatasetRef datasetRef, BigInteger id, LoginAccount account);
}

View File

@@ -0,0 +1,311 @@
package tech.easyflow.datacenter.execution.service.impl;
import com.mybatisflex.core.paginate.Page;
import com.mybatisflex.core.query.QueryWrapper;
import com.mybatisflex.core.row.Row;
import org.springframework.stereotype.Service;
import org.springframework.util.CollectionUtils;
import org.springframework.util.StringUtils;
import tech.easyflow.common.web.exceptions.BusinessException;
import tech.easyflow.datacenter.connector.DatacenterConnector;
import tech.easyflow.datacenter.connector.DatacenterConnectorRegistry;
import tech.easyflow.datacenter.entity.DatacenterTable;
import tech.easyflow.datacenter.entity.DatacenterTableField;
import tech.easyflow.datacenter.execution.model.DatacenterQueryRequest;
import tech.easyflow.datacenter.execution.model.DatacenterSchemaResponse;
import tech.easyflow.datacenter.execution.model.DatacenterSqlQueryRequest;
import tech.easyflow.datacenter.execution.model.DatasetRef;
import tech.easyflow.datacenter.execution.service.DatacenterDatasetQueryService;
import tech.easyflow.datacenter.mapper.DatacenterCatalogMapper;
import tech.easyflow.datacenter.mapper.DatacenterDatasetVersionMapper;
import tech.easyflow.datacenter.mapper.DatacenterDerivedTableMapper;
import tech.easyflow.datacenter.mapper.DatacenterTableMapper;
import tech.easyflow.datacenter.meta.entity.DatacenterCatalog;
import tech.easyflow.datacenter.meta.entity.DatacenterDatasetVersion;
import tech.easyflow.datacenter.meta.entity.DatacenterDerivedTable;
import tech.easyflow.datacenter.meta.entity.DatacenterSource;
import tech.easyflow.datacenter.meta.service.DatacenterDatasetRegistryService;
import tech.easyflow.datacenter.utils.SqlSupportUtils;
import javax.annotation.Resource;
import java.math.BigInteger;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
@Service
public class DatacenterDatasetQueryServiceImpl implements DatacenterDatasetQueryService {
@Resource
private DatacenterDatasetRegistryService registryService;
@Resource
private DatacenterConnectorRegistry connectorRegistry;
@Resource
private DatacenterTableMapper tableMapper;
@Resource
private DatacenterCatalogMapper catalogMapper;
@Resource
private DatacenterDatasetVersionMapper datasetVersionMapper;
@Resource
private DatacenterDerivedTableMapper derivedTableMapper;
@Override
public Page<Row> queryPage(DatacenterQueryRequest request) {
if (request == null || request.getDatasetRef() == null) {
throw new BusinessException("datasetRef 不能为空");
}
normalizePage(request);
DatacenterTable table = resolveTable(request.getDatasetRef());
DatacenterSource source = registryService.getSourceRequired(table.getSourceId());
DatacenterCatalog catalog = registryService.getCatalogById(table.getCatalogId());
DatacenterTable queryTable = resolveQueryTable(table, request.getDatasetRef());
validateRequest(queryTable, request, source);
request.getDatasetRef().setSourceId(table.getSourceId());
request.getDatasetRef().setCatalogId(table.getCatalogId());
request.getDatasetRef().setTableId(table.getId());
request.getDatasetRef().setTableName(table.getTableName());
if (catalog != null) {
request.getDatasetRef().setCatalogName(catalog.getCatalogName());
}
DatacenterConnector connector = connectorRegistry.getConnector(source.getSourceType());
return connector.queryPage(source, queryTable, request);
}
@Override
public List<Row> queryBySql(DatacenterSqlQueryRequest request) {
if (request == null || request.getDatasetRef() == null) {
throw new BusinessException("datasetRef 不能为空");
}
String sql = trimToNull(request.getSql());
if (!StringUtils.hasText(sql)) {
throw new BusinessException("SQL 不能为空");
}
DatasetRef datasetRef = request.getDatasetRef();
if (datasetRef.getSourceId() == null) {
throw new BusinessException("缺少连接服务配置");
}
DatacenterSource source = registryService.getSourceRequired(datasetRef.getSourceId());
BigInteger catalogId = resolveRequestedCatalogId(datasetRef);
List<DatacenterTable> managedTables = registryService.listManagedTables(datasetRef.getSourceId(), catalogId);
if (CollectionUtils.isEmpty(managedTables)) {
throw new BusinessException("当前连接下没有已接入表");
}
SqlSupportUtils.ResolvedSql resolvedSql = SqlSupportUtils.resolve(
sql,
managedTables.stream().map(this::toManagedSqlTable).toList()
);
DatacenterConnector connector = connectorRegistry.getConnector(source.getSourceType());
return connector.queryBySql(source, resolvedSql.getExecutableSql());
}
@Override
public DatacenterSchemaResponse getSchema(DatasetRef datasetRef) {
DatacenterTable table = resolveTable(datasetRef);
DatacenterSource source = registryService.getSourceRequired(table.getSourceId());
DatacenterCatalog catalog = registryService.getCatalogById(table.getCatalogId());
DatacenterSchemaResponse response = new DatacenterSchemaResponse();
response.setDatasetRef(registryService.resolveDatasetRef(table.getId()));
response.setSource(source);
response.setCatalog(catalog);
response.setTable(table);
response.setFields(table.getFields());
response.setVersions(listVersions(table.getId()));
response.setUpstreamLineage(listUpstream(table.getId()));
response.setDownstreamLineage(listDownstream(table.getId()));
return response;
}
private DatacenterTable resolveTable(DatasetRef datasetRef) {
if (datasetRef.getTableId() != null) {
return registryService.getTableWithFields(datasetRef.getTableId());
}
if (datasetRef.getSourceId() == null || !StringUtils.hasText(datasetRef.getTableName())) {
throw new BusinessException("缺少数据集定位信息");
}
QueryWrapper wrapper = QueryWrapper.create();
wrapper.eq(DatacenterTable::getSourceId, datasetRef.getSourceId());
wrapper.eq(DatacenterTable::getTableName, datasetRef.getTableName().trim());
boolean hasCatalogCondition = false;
if (datasetRef.getCatalogId() != null) {
wrapper.eq(DatacenterTable::getCatalogId, datasetRef.getCatalogId());
hasCatalogCondition = true;
} else if (StringUtils.hasText(datasetRef.getCatalogName())) {
DatacenterCatalog catalog = resolveCatalog(datasetRef.getSourceId(), datasetRef.getCatalogName().trim());
wrapper.eq(DatacenterTable::getCatalogId, catalog.getId());
hasCatalogCondition = true;
}
List<DatacenterTable> tables = tableMapper.selectListByQuery(wrapper);
if (CollectionUtils.isEmpty(tables)) {
throw new BusinessException("数据集不存在: " + datasetRef.getTableName());
}
if (!hasCatalogCondition && tables.size() > 1) {
throw new BusinessException("数据集存在重名表,请指定库名: " + datasetRef.getTableName());
}
return registryService.getTableWithFields(tables.get(0).getId());
}
private DatacenterCatalog resolveCatalog(java.math.BigInteger sourceId, String catalogName) {
QueryWrapper wrapper = QueryWrapper.create();
wrapper.eq(DatacenterCatalog::getSourceId, sourceId);
wrapper.eq(DatacenterCatalog::getCatalogName, catalogName);
List<DatacenterCatalog> catalogs = catalogMapper.selectListByQuery(wrapper);
if (CollectionUtils.isEmpty(catalogs)) {
throw new BusinessException("库不存在: " + catalogName);
}
if (catalogs.size() > 1) {
throw new BusinessException("库存在重复配置,请检查: " + catalogName);
}
return catalogs.get(0);
}
private BigInteger resolveRequestedCatalogId(DatasetRef datasetRef) {
if (datasetRef == null) {
return null;
}
if (datasetRef.getCatalogId() != null) {
return datasetRef.getCatalogId();
}
if (StringUtils.hasText(datasetRef.getCatalogName())) {
DatacenterCatalog catalog = resolveCatalog(datasetRef.getSourceId(), datasetRef.getCatalogName().trim());
return catalog.getId();
}
return null;
}
private SqlSupportUtils.ManagedTable toManagedSqlTable(DatacenterTable table) {
DatacenterCatalog catalog = registryService.getCatalogById(table.getCatalogId());
return new SqlSupportUtils.ManagedTable(
catalog == null ? null : catalog.getCatalogName(),
table.getTableName(),
resolvePhysicalTableName(table)
);
}
private String resolvePhysicalTableName(DatacenterTable table) {
if (table == null) {
return null;
}
if ("EXTERNAL_TABLE".equals(table.getTableKind()) || "EXTERNAL_VIEW".equals(table.getTableKind())) {
return trimToNull(table.getActualTable()) == null ? table.getTableName() : table.getActualTable().trim();
}
String materializedTable = trimToNull(table.getMaterializedTable());
if (materializedTable != null) {
return materializedTable;
}
String actualTable = trimToNull(table.getActualTable());
return actualTable != null ? actualTable : table.getTableName();
}
private void normalizePage(DatacenterQueryRequest request) {
if (request.getPageNumber() == null || request.getPageNumber() < 1L) {
request.setPageNumber(1L);
}
if (request.getPageSize() == null || request.getPageSize() < 1L) {
throw new BusinessException("pageSize 必须大于 0");
}
}
private DatacenterTable resolveQueryTable(DatacenterTable table, DatasetRef datasetRef) {
if (datasetRef == null || datasetRef.getVersionId() == null) {
return table;
}
DatacenterDatasetVersion version = datasetVersionMapper.selectOneById(datasetRef.getVersionId());
if (version == null || !table.getId().equals(version.getTableId())) {
throw new BusinessException("数据集版本不存在: " + datasetRef.getVersionId());
}
DatacenterTable queryTable = new DatacenterTable();
queryTable.setId(table.getId());
queryTable.setSourceId(table.getSourceId());
queryTable.setCatalogId(table.getCatalogId());
queryTable.setTableName(table.getTableName());
queryTable.setTableDesc(table.getTableDesc());
queryTable.setActualTable(table.getActualTable());
queryTable.setMaterializedTable(version.getMaterializedTable());
queryTable.setAccessMode(table.getAccessMode());
queryTable.setTableKind(table.getTableKind());
queryTable.setVersioningEnabled(table.getVersioningEnabled());
queryTable.setCapabilitiesJson(table.getCapabilitiesJson());
queryTable.setFields(table.getFields());
return queryTable;
}
private void validateRequest(DatacenterTable table, DatacenterQueryRequest request, DatacenterSource source) {
Map<String, DatacenterTableField> fieldMap = new LinkedHashMap<>();
for (DatacenterTableField field : table.getFields()) {
fieldMap.put(field.getFieldName(), field);
}
if (!CollectionUtils.isEmpty(request.getSelectedColumns())) {
for (String column : request.getSelectedColumns()) {
DatacenterTableField field = fieldMap.get(column);
if (field == null || !isEnabled(field.getQueryable())) {
throw new BusinessException("字段不可查询: " + column);
}
}
} else {
request.setSelectedColumns(
table.getFields().stream()
.filter(field -> isEnabled(field.getQueryable()))
.map(DatacenterTableField::getFieldName)
.toList()
);
}
if (!CollectionUtils.isEmpty(request.getFilters())) {
request.getFilters().forEach(filter -> {
DatacenterTableField field = fieldMap.get(filter.getColumn());
if (field == null || !isEnabled(field.getQueryable())) {
throw new BusinessException("字段不可过滤: " + filter.getColumn());
}
});
}
if (!CollectionUtils.isEmpty(request.getSorts())) {
request.getSorts().forEach(sort -> {
DatacenterTableField field = fieldMap.get(sort.getColumn());
if (field == null || !isEnabled(field.getSortable())) {
throw new BusinessException("字段不可排序: " + sort.getColumn());
}
});
}
if (request.getWhere() != null && !request.getWhere().isBlank()) {
boolean allowLegacyWhere = "PROJECT_MYSQL".equals(source.getSourceType())
|| "MYSQL".equals(source.getSourceType())
|| "POSTGRESQL".equals(source.getSourceType());
if (!allowLegacyWhere) {
throw new BusinessException("当前数据源仅支持结构化 DSL 查询");
}
}
}
private boolean isEnabled(Integer value) {
return value == null || value == 1;
}
private String trimToNull(String value) {
if (!StringUtils.hasText(value)) {
return null;
}
return value.trim();
}
private List<DatacenterDatasetVersion> listVersions(java.math.BigInteger tableId) {
QueryWrapper wrapper = QueryWrapper.create();
wrapper.eq(DatacenterDatasetVersion::getTableId, tableId);
wrapper.orderBy("version_no desc");
return datasetVersionMapper.selectListByQuery(wrapper);
}
private List<DatacenterDerivedTable> listUpstream(java.math.BigInteger tableId) {
QueryWrapper wrapper = QueryWrapper.create();
wrapper.eq(DatacenterDerivedTable::getDerivedTableId, tableId);
wrapper.orderBy("created desc");
return derivedTableMapper.selectListByQuery(wrapper);
}
private List<DatacenterDerivedTable> listDownstream(java.math.BigInteger tableId) {
QueryWrapper wrapper = QueryWrapper.create();
wrapper.eq(DatacenterDerivedTable::getSourceTableId, tableId);
wrapper.orderBy("created desc");
return derivedTableMapper.selectListByQuery(wrapper);
}
}

View File

@@ -0,0 +1,48 @@
package tech.easyflow.datacenter.execution.service.impl;
import com.alibaba.fastjson2.JSONObject;
import org.springframework.stereotype.Service;
import tech.easyflow.common.entity.LoginAccount;
import tech.easyflow.common.web.exceptions.BusinessException;
import tech.easyflow.datacenter.connector.DatacenterConnector;
import tech.easyflow.datacenter.connector.DatacenterConnectorRegistry;
import tech.easyflow.datacenter.entity.DatacenterTable;
import tech.easyflow.datacenter.execution.model.DatasetRef;
import tech.easyflow.datacenter.meta.entity.DatacenterSource;
import tech.easyflow.datacenter.meta.service.DatacenterDatasetRegistryService;
import tech.easyflow.datacenter.execution.service.DatacenterDatasetWriteService;
import javax.annotation.Resource;
import java.math.BigInteger;
@Service
public class DatacenterDatasetWriteServiceImpl implements DatacenterDatasetWriteService {
@Resource
private DatacenterDatasetRegistryService registryService;
@Resource
private DatacenterConnectorRegistry connectorRegistry;
@Override
public void saveRow(DatasetRef datasetRef, JSONObject data, LoginAccount account) {
DatacenterTable table = resolveTable(datasetRef);
DatacenterSource source = registryService.getSourceRequired(table.getSourceId());
DatacenterConnector connector = connectorRegistry.getConnector(source.getSourceType());
connector.saveRow(source, table, data, account);
}
@Override
public void deleteRow(DatasetRef datasetRef, BigInteger id, LoginAccount account) {
DatacenterTable table = resolveTable(datasetRef);
DatacenterSource source = registryService.getSourceRequired(table.getSourceId());
DatacenterConnector connector = connectorRegistry.getConnector(source.getSourceType());
connector.deleteRow(source, table, id, account);
}
private DatacenterTable resolveTable(DatasetRef datasetRef) {
if (datasetRef == null || datasetRef.getTableId() == null) {
throw new BusinessException("缺少 tableId");
}
return registryService.getTableWithFields(datasetRef.getTableId());
}
}

View File

@@ -0,0 +1,7 @@
package tech.easyflow.datacenter.integration;
import tech.easyflow.datacenter.execution.model.DatacenterQueryRequest;
public interface AssistantDatacenterBridge {
AssistantDatacenterResult queryPage(DatacenterQueryRequest request);
}

View File

@@ -0,0 +1,34 @@
package tech.easyflow.datacenter.integration;
import com.mybatisflex.core.row.Row;
import tech.easyflow.datacenter.entity.DatacenterTable;
import tech.easyflow.datacenter.meta.entity.DatacenterCatalog;
import tech.easyflow.datacenter.meta.entity.DatacenterDatasetVersion;
import tech.easyflow.datacenter.meta.entity.DatacenterSource;
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
public class AssistantDatacenterResult {
private List<Row> rows = new ArrayList<>();
private DatacenterSource source;
private DatacenterCatalog catalog;
private DatacenterTable table;
private DatacenterDatasetVersion version;
private Map<String, Object> querySummary = new LinkedHashMap<>();
public List<Row> getRows() { return rows; }
public void setRows(List<Row> rows) { this.rows = rows; }
public DatacenterSource getSource() { return source; }
public void setSource(DatacenterSource source) { this.source = source; }
public DatacenterCatalog getCatalog() { return catalog; }
public void setCatalog(DatacenterCatalog catalog) { this.catalog = catalog; }
public DatacenterTable getTable() { return table; }
public void setTable(DatacenterTable table) { this.table = table; }
public DatacenterDatasetVersion getVersion() { return version; }
public void setVersion(DatacenterDatasetVersion version) { this.version = version; }
public Map<String, Object> getQuerySummary() { return querySummary; }
public void setQuerySummary(Map<String, Object> querySummary) { this.querySummary = querySummary; }
}

View File

@@ -0,0 +1,49 @@
package tech.easyflow.datacenter.integration;
import org.springframework.stereotype.Component;
import tech.easyflow.datacenter.execution.model.DatacenterQueryRequest;
import tech.easyflow.datacenter.execution.model.DatacenterSchemaResponse;
import tech.easyflow.datacenter.execution.service.DatacenterDatasetQueryService;
import tech.easyflow.datacenter.meta.entity.DatacenterDatasetVersion;
import javax.annotation.Resource;
import java.util.LinkedHashMap;
import java.util.List;
@Component
public class DefaultAssistantDatacenterBridge implements AssistantDatacenterBridge {
@Resource
private DatacenterDatasetQueryService queryService;
@Override
public AssistantDatacenterResult queryPage(DatacenterQueryRequest request) {
var page = queryService.queryPage(request);
DatacenterSchemaResponse schema = queryService.getSchema(request.getDatasetRef());
AssistantDatacenterResult result = new AssistantDatacenterResult();
result.setRows(page.getRecords());
result.setSource(schema.getSource());
result.setCatalog(schema.getCatalog());
result.setTable(schema.getTable());
result.setVersion(resolveVersion(schema.getVersions(), request == null || request.getDatasetRef() == null ? null : request.getDatasetRef().getVersionId()));
result.setQuerySummary(new LinkedHashMap<>() {{
put("pageNumber", page.getPageNumber());
put("pageSize", page.getPageSize());
put("totalRows", page.getTotalRow());
put("selectedColumns", request == null ? List.of() : request.getSelectedColumns());
put("filterCount", request == null || request.getFilters() == null ? 0 : request.getFilters().size());
put("sortCount", request == null || request.getSorts() == null ? 0 : request.getSorts().size());
}});
return result;
}
private DatacenterDatasetVersion resolveVersion(List<DatacenterDatasetVersion> versions, java.math.BigInteger versionId) {
if (versions == null || versions.isEmpty()) {
return null;
}
if (versionId == null) {
return versions.get(0);
}
return versions.stream().filter(version -> versionId.equals(version.getId())).findFirst().orElse(null);
}
}

View File

@@ -0,0 +1,7 @@
package tech.easyflow.datacenter.mapper;
import com.mybatisflex.core.BaseMapper;
import tech.easyflow.datacenter.meta.entity.DatacenterCatalog;
public interface DatacenterCatalogMapper extends BaseMapper<DatacenterCatalog> {
}

View File

@@ -0,0 +1,7 @@
package tech.easyflow.datacenter.mapper;
import com.mybatisflex.core.BaseMapper;
import tech.easyflow.datacenter.meta.entity.DatacenterDatasetVersion;
public interface DatacenterDatasetVersionMapper extends BaseMapper<DatacenterDatasetVersion> {
}

View File

@@ -0,0 +1,7 @@
package tech.easyflow.datacenter.mapper;
import com.mybatisflex.core.BaseMapper;
import tech.easyflow.datacenter.meta.entity.DatacenterDerivedTable;
public interface DatacenterDerivedTableMapper extends BaseMapper<DatacenterDerivedTable> {
}

View File

@@ -0,0 +1,7 @@
package tech.easyflow.datacenter.mapper;
import com.mybatisflex.core.BaseMapper;
import tech.easyflow.datacenter.meta.entity.DatacenterImportJob;
public interface DatacenterImportJobMapper extends BaseMapper<DatacenterImportJob> {
}

View File

@@ -0,0 +1,7 @@
package tech.easyflow.datacenter.mapper;
import com.mybatisflex.core.BaseMapper;
import tech.easyflow.datacenter.meta.entity.DatacenterSource;
public interface DatacenterSourceMapper extends BaseMapper<DatacenterSource> {
}

View File

@@ -0,0 +1,75 @@
package tech.easyflow.datacenter.meta.entity;
import com.mybatisflex.annotation.Column;
import com.mybatisflex.annotation.Id;
import com.mybatisflex.annotation.KeyType;
import com.mybatisflex.annotation.Table;
import com.mybatisflex.core.handler.FastjsonTypeHandler;
import tech.easyflow.common.entity.DateEntity;
import java.io.Serializable;
import java.math.BigInteger;
import java.util.Date;
import java.util.Map;
@Table(value = "tb_datacenter_catalog", comment = "数据中心逻辑库/命名空间")
public class DatacenterCatalog extends DateEntity implements Serializable {
@Id(keyType = KeyType.Generator, value = "snowFlakeId", comment = "主键")
private BigInteger id;
@Column(comment = "部门ID")
private BigInteger deptId;
@Column(tenantId = true, comment = "租户ID")
private BigInteger tenantId;
@Column(comment = "数据源ID")
private BigInteger sourceId;
@Column(comment = "目录名")
private String catalogName;
@Column(comment = "目录描述")
private String catalogDesc;
@Column(comment = "目录类型")
private String catalogType;
@Column(comment = "状态")
private Integer status;
@Column(typeHandler = FastjsonTypeHandler.class, comment = "扩展项")
private Map<String, Object> options;
@Column(comment = "创建时间")
private Date created;
@Column(comment = "创建人")
private BigInteger createdBy;
@Column(comment = "修改时间")
private Date modified;
@Column(comment = "修改人")
private BigInteger modifiedBy;
public BigInteger getId() { return id; }
public void setId(BigInteger id) { this.id = id; }
public BigInteger getDeptId() { return deptId; }
public void setDeptId(BigInteger deptId) { this.deptId = deptId; }
public BigInteger getTenantId() { return tenantId; }
public void setTenantId(BigInteger tenantId) { this.tenantId = tenantId; }
public BigInteger getSourceId() { return sourceId; }
public void setSourceId(BigInteger sourceId) { this.sourceId = sourceId; }
public String getCatalogName() { return catalogName; }
public void setCatalogName(String catalogName) { this.catalogName = catalogName; }
public String getCatalogDesc() { return catalogDesc; }
public void setCatalogDesc(String catalogDesc) { this.catalogDesc = catalogDesc; }
public String getCatalogType() { return catalogType; }
public void setCatalogType(String catalogType) { this.catalogType = catalogType; }
public Integer getStatus() { return status; }
public void setStatus(Integer status) { this.status = status; }
public Map<String, Object> getOptions() { return options; }
public void setOptions(Map<String, Object> options) { this.options = options; }
@Override
public Date getCreated() { return created; }
@Override
public void setCreated(Date created) { this.created = created; }
public BigInteger getCreatedBy() { return createdBy; }
public void setCreatedBy(BigInteger createdBy) { this.createdBy = createdBy; }
@Override
public Date getModified() { return modified; }
@Override
public void setModified(Date modified) { this.modified = modified; }
public BigInteger getModifiedBy() { return modifiedBy; }
public void setModifiedBy(BigInteger modifiedBy) { this.modifiedBy = modifiedBy; }
}

View File

@@ -0,0 +1,70 @@
package tech.easyflow.datacenter.meta.entity;
import com.mybatisflex.annotation.Column;
import com.mybatisflex.annotation.Id;
import com.mybatisflex.annotation.KeyType;
import com.mybatisflex.annotation.Table;
import com.mybatisflex.core.handler.FastjsonTypeHandler;
import tech.easyflow.common.entity.DateEntity;
import java.io.Serializable;
import java.math.BigInteger;
import java.util.Date;
import java.util.Map;
@Table(value = "tb_datacenter_dataset_version", comment = "数据集版本")
public class DatacenterDatasetVersion extends DateEntity implements Serializable {
@Id(keyType = KeyType.Generator, value = "snowFlakeId")
private BigInteger id;
@Column(comment = "部门ID")
private BigInteger deptId;
@Column(tenantId = true, comment = "租户ID")
private BigInteger tenantId;
@Column(comment = "表ID")
private BigInteger tableId;
@Column(comment = "版本号")
private Integer versionNo;
@Column(comment = "版本标签")
private String versionLabel;
@Column(comment = "物化表名")
private String materializedTable;
@Column(typeHandler = FastjsonTypeHandler.class, comment = "版本快照")
private Map<String, Object> snapshotJson;
@Column(comment = "状态")
private Integer status;
@Column(comment = "创建时间")
private Date created;
@Column(comment = "创建人")
private BigInteger createdBy;
@Column(comment = "修改时间")
private Date modified;
@Column(comment = "修改人")
private BigInteger modifiedBy;
public BigInteger getId() { return id; }
public void setId(BigInteger id) { this.id = id; }
public BigInteger getDeptId() { return deptId; }
public void setDeptId(BigInteger deptId) { this.deptId = deptId; }
public BigInteger getTenantId() { return tenantId; }
public void setTenantId(BigInteger tenantId) { this.tenantId = tenantId; }
public BigInteger getTableId() { return tableId; }
public void setTableId(BigInteger tableId) { this.tableId = tableId; }
public Integer getVersionNo() { return versionNo; }
public void setVersionNo(Integer versionNo) { this.versionNo = versionNo; }
public String getVersionLabel() { return versionLabel; }
public void setVersionLabel(String versionLabel) { this.versionLabel = versionLabel; }
public String getMaterializedTable() { return materializedTable; }
public void setMaterializedTable(String materializedTable) { this.materializedTable = materializedTable; }
public Map<String, Object> getSnapshotJson() { return snapshotJson; }
public void setSnapshotJson(Map<String, Object> snapshotJson) { this.snapshotJson = snapshotJson; }
public Integer getStatus() { return status; }
public void setStatus(Integer status) { this.status = status; }
@Override public Date getCreated() { return created; }
@Override public void setCreated(Date created) { this.created = created; }
public BigInteger getCreatedBy() { return createdBy; }
public void setCreatedBy(BigInteger createdBy) { this.createdBy = createdBy; }
@Override public Date getModified() { return modified; }
@Override public void setModified(Date modified) { this.modified = modified; }
public BigInteger getModifiedBy() { return modifiedBy; }
public void setModifiedBy(BigInteger modifiedBy) { this.modifiedBy = modifiedBy; }
}

View File

@@ -0,0 +1,66 @@
package tech.easyflow.datacenter.meta.entity;
import com.mybatisflex.annotation.Column;
import com.mybatisflex.annotation.Id;
import com.mybatisflex.annotation.KeyType;
import com.mybatisflex.annotation.Table;
import com.mybatisflex.core.handler.FastjsonTypeHandler;
import tech.easyflow.common.entity.DateEntity;
import java.io.Serializable;
import java.math.BigInteger;
import java.util.Date;
import java.util.Map;
@Table(value = "tb_datacenter_derived_table", comment = "数据中心派生表关系")
public class DatacenterDerivedTable extends DateEntity implements Serializable {
@Id(keyType = KeyType.Generator, value = "snowFlakeId")
private BigInteger id;
@Column(comment = "部门ID")
private BigInteger deptId;
@Column(tenantId = true, comment = "租户ID")
private BigInteger tenantId;
@Column(comment = "源表ID")
private BigInteger sourceTableId;
@Column(comment = "派生表ID")
private BigInteger derivedTableId;
@Column(comment = "派生类型")
private String deriveType;
@Column(typeHandler = FastjsonTypeHandler.class, comment = "派生配置")
private Map<String, Object> deriveConfigJson;
@Column(comment = "状态")
private Integer status;
@Column(comment = "创建时间")
private Date created;
@Column(comment = "创建人")
private BigInteger createdBy;
@Column(comment = "修改时间")
private Date modified;
@Column(comment = "修改人")
private BigInteger modifiedBy;
public BigInteger getId() { return id; }
public void setId(BigInteger id) { this.id = id; }
public BigInteger getDeptId() { return deptId; }
public void setDeptId(BigInteger deptId) { this.deptId = deptId; }
public BigInteger getTenantId() { return tenantId; }
public void setTenantId(BigInteger tenantId) { this.tenantId = tenantId; }
public BigInteger getSourceTableId() { return sourceTableId; }
public void setSourceTableId(BigInteger sourceTableId) { this.sourceTableId = sourceTableId; }
public BigInteger getDerivedTableId() { return derivedTableId; }
public void setDerivedTableId(BigInteger derivedTableId) { this.derivedTableId = derivedTableId; }
public String getDeriveType() { return deriveType; }
public void setDeriveType(String deriveType) { this.deriveType = deriveType; }
public Map<String, Object> getDeriveConfigJson() { return deriveConfigJson; }
public void setDeriveConfigJson(Map<String, Object> deriveConfigJson) { this.deriveConfigJson = deriveConfigJson; }
public Integer getStatus() { return status; }
public void setStatus(Integer status) { this.status = status; }
@Override public Date getCreated() { return created; }
@Override public void setCreated(Date created) { this.created = created; }
public BigInteger getCreatedBy() { return createdBy; }
public void setCreatedBy(BigInteger createdBy) { this.createdBy = createdBy; }
@Override public Date getModified() { return modified; }
@Override public void setModified(Date modified) { this.modified = modified; }
public BigInteger getModifiedBy() { return modifiedBy; }
public void setModifiedBy(BigInteger modifiedBy) { this.modifiedBy = modifiedBy; }
}

View File

@@ -0,0 +1,102 @@
package tech.easyflow.datacenter.meta.entity;
import com.mybatisflex.annotation.Column;
import com.mybatisflex.annotation.Id;
import com.mybatisflex.annotation.KeyType;
import com.mybatisflex.annotation.Table;
import com.mybatisflex.core.handler.FastjsonTypeHandler;
import tech.easyflow.common.entity.DateEntity;
import java.io.Serializable;
import java.math.BigInteger;
import java.util.Date;
import java.util.Map;
@Table(value = "tb_datacenter_import_job", comment = "数据中心导入任务")
public class DatacenterImportJob extends DateEntity implements Serializable {
@Id(keyType = KeyType.Generator, value = "snowFlakeId")
private BigInteger id;
@Column(comment = "部门ID")
private BigInteger deptId;
@Column(tenantId = true, comment = "租户ID")
private BigInteger tenantId;
@Column(comment = "数据源ID")
private BigInteger sourceId;
@Column(comment = "目录ID")
private BigInteger catalogId;
@Column(comment = "表ID")
private BigInteger tableId;
@Column(comment = "任务类型")
private String jobType;
@Column(comment = "文件名")
private String fileName;
@Column(comment = "文件存储路径")
private String storagePath;
@Column(comment = "任务状态")
private String status;
@Column(comment = "总行数")
private Long totalRows;
@Column(comment = "成功行数")
private Long successRows;
@Column(comment = "失败行数")
private Long errorRows;
@Column(comment = "错误摘要")
private String errorSummary;
@Column(typeHandler = FastjsonTypeHandler.class, comment = "任务载荷")
private Map<String, Object> payloadJson;
@Column(comment = "开始时间")
private Date startedAt;
@Column(comment = "结束时间")
private Date finishedAt;
@Column(comment = "创建时间")
private Date created;
@Column(comment = "创建人")
private BigInteger createdBy;
@Column(comment = "修改时间")
private Date modified;
@Column(comment = "修改人")
private BigInteger modifiedBy;
public BigInteger getId() { return id; }
public void setId(BigInteger id) { this.id = id; }
public BigInteger getDeptId() { return deptId; }
public void setDeptId(BigInteger deptId) { this.deptId = deptId; }
public BigInteger getTenantId() { return tenantId; }
public void setTenantId(BigInteger tenantId) { this.tenantId = tenantId; }
public BigInteger getSourceId() { return sourceId; }
public void setSourceId(BigInteger sourceId) { this.sourceId = sourceId; }
public BigInteger getCatalogId() { return catalogId; }
public void setCatalogId(BigInteger catalogId) { this.catalogId = catalogId; }
public BigInteger getTableId() { return tableId; }
public void setTableId(BigInteger tableId) { this.tableId = tableId; }
public String getJobType() { return jobType; }
public void setJobType(String jobType) { this.jobType = jobType; }
public String getFileName() { return fileName; }
public void setFileName(String fileName) { this.fileName = fileName; }
public String getStoragePath() { return storagePath; }
public void setStoragePath(String storagePath) { this.storagePath = storagePath; }
public String getStatus() { return status; }
public void setStatus(String status) { this.status = status; }
public Long getTotalRows() { return totalRows; }
public void setTotalRows(Long totalRows) { this.totalRows = totalRows; }
public Long getSuccessRows() { return successRows; }
public void setSuccessRows(Long successRows) { this.successRows = successRows; }
public Long getErrorRows() { return errorRows; }
public void setErrorRows(Long errorRows) { this.errorRows = errorRows; }
public String getErrorSummary() { return errorSummary; }
public void setErrorSummary(String errorSummary) { this.errorSummary = errorSummary; }
public Map<String, Object> getPayloadJson() { return payloadJson; }
public void setPayloadJson(Map<String, Object> payloadJson) { this.payloadJson = payloadJson; }
public Date getStartedAt() { return startedAt; }
public void setStartedAt(Date startedAt) { this.startedAt = startedAt; }
public Date getFinishedAt() { return finishedAt; }
public void setFinishedAt(Date finishedAt) { this.finishedAt = finishedAt; }
@Override public Date getCreated() { return created; }
@Override public void setCreated(Date created) { this.created = created; }
public BigInteger getCreatedBy() { return createdBy; }
public void setCreatedBy(BigInteger createdBy) { this.createdBy = createdBy; }
@Override public Date getModified() { return modified; }
@Override public void setModified(Date modified) { this.modified = modified; }
public BigInteger getModifiedBy() { return modifiedBy; }
public void setModifiedBy(BigInteger modifiedBy) { this.modifiedBy = modifiedBy; }
}

View File

@@ -0,0 +1,131 @@
package tech.easyflow.datacenter.meta.entity;
import com.mybatisflex.annotation.Column;
import com.mybatisflex.annotation.Id;
import com.mybatisflex.annotation.KeyType;
import com.mybatisflex.annotation.Table;
import com.mybatisflex.core.handler.FastjsonTypeHandler;
import tech.easyflow.common.entity.DateEntity;
import java.io.Serializable;
import java.math.BigInteger;
import java.util.Date;
import java.util.Map;
@Table(value = "tb_datacenter_source", comment = "数据中心数据源")
public class DatacenterSource extends DateEntity implements Serializable {
@Id(keyType = KeyType.Generator, value = "snowFlakeId", comment = "主键")
private BigInteger id;
@Column(comment = "部门ID")
private BigInteger deptId;
@Column(tenantId = true, comment = "租户ID")
private BigInteger tenantId;
@Column(comment = "数据源名称")
private String sourceName;
@Column(comment = "数据源编码")
private String sourceCode;
@Column(comment = "数据源类型")
private String sourceType;
@Column(comment = "访问模式")
private String accessMode;
@Column(comment = "是否内置")
private Integer builtinFlag;
@Column(comment = "驱动类名")
private String driverClassName;
@Column(comment = "JDBC URL")
private String jdbcUrl;
@Column(comment = "主机")
private String host;
@Column(comment = "端口")
private Integer port;
@Column(comment = "数据库名")
private String databaseName;
@Column(comment = "Schema名")
private String schemaName;
@Column(comment = "用户名")
private String username;
@Column(comment = "凭据密文")
private String credentialCipher;
@Column(typeHandler = FastjsonTypeHandler.class, comment = "连接配置")
private Map<String, Object> configJson;
@Column(typeHandler = FastjsonTypeHandler.class, comment = "能力声明")
private Map<String, Object> capabilitiesJson;
@Column(comment = "最近测试状态")
private String lastTestStatus;
@Column(comment = "最近测试信息")
private String lastTestMessage;
@Column(comment = "最近测试时间")
private Date lastTestedAt;
@Column(comment = "状态")
private Integer status;
@Column(typeHandler = FastjsonTypeHandler.class, comment = "扩展项")
private Map<String, Object> options;
@Column(comment = "创建时间")
private Date created;
@Column(comment = "创建人")
private BigInteger createdBy;
@Column(comment = "修改时间")
private Date modified;
@Column(comment = "修改人")
private BigInteger modifiedBy;
public BigInteger getId() { return id; }
public void setId(BigInteger id) { this.id = id; }
public BigInteger getDeptId() { return deptId; }
public void setDeptId(BigInteger deptId) { this.deptId = deptId; }
public BigInteger getTenantId() { return tenantId; }
public void setTenantId(BigInteger tenantId) { this.tenantId = tenantId; }
public String getSourceName() { return sourceName; }
public void setSourceName(String sourceName) { this.sourceName = sourceName; }
public String getSourceCode() { return sourceCode; }
public void setSourceCode(String sourceCode) { this.sourceCode = sourceCode; }
public String getSourceType() { return sourceType; }
public void setSourceType(String sourceType) { this.sourceType = sourceType; }
public String getAccessMode() { return accessMode; }
public void setAccessMode(String accessMode) { this.accessMode = accessMode; }
public Integer getBuiltinFlag() { return builtinFlag; }
public void setBuiltinFlag(Integer builtinFlag) { this.builtinFlag = builtinFlag; }
public String getDriverClassName() { return driverClassName; }
public void setDriverClassName(String driverClassName) { this.driverClassName = driverClassName; }
public String getJdbcUrl() { return jdbcUrl; }
public void setJdbcUrl(String jdbcUrl) { this.jdbcUrl = jdbcUrl; }
public String getHost() { return host; }
public void setHost(String host) { this.host = host; }
public Integer getPort() { return port; }
public void setPort(Integer port) { this.port = port; }
public String getDatabaseName() { return databaseName; }
public void setDatabaseName(String databaseName) { this.databaseName = databaseName; }
public String getSchemaName() { return schemaName; }
public void setSchemaName(String schemaName) { this.schemaName = schemaName; }
public String getUsername() { return username; }
public void setUsername(String username) { this.username = username; }
public String getCredentialCipher() { return credentialCipher; }
public void setCredentialCipher(String credentialCipher) { this.credentialCipher = credentialCipher; }
public Map<String, Object> getConfigJson() { return configJson; }
public void setConfigJson(Map<String, Object> configJson) { this.configJson = configJson; }
public Map<String, Object> getCapabilitiesJson() { return capabilitiesJson; }
public void setCapabilitiesJson(Map<String, Object> capabilitiesJson) { this.capabilitiesJson = capabilitiesJson; }
public String getLastTestStatus() { return lastTestStatus; }
public void setLastTestStatus(String lastTestStatus) { this.lastTestStatus = lastTestStatus; }
public String getLastTestMessage() { return lastTestMessage; }
public void setLastTestMessage(String lastTestMessage) { this.lastTestMessage = lastTestMessage; }
public Date getLastTestedAt() { return lastTestedAt; }
public void setLastTestedAt(Date lastTestedAt) { this.lastTestedAt = lastTestedAt; }
public Integer getStatus() { return status; }
public void setStatus(Integer status) { this.status = status; }
public Map<String, Object> getOptions() { return options; }
public void setOptions(Map<String, Object> options) { this.options = options; }
@Override
public Date getCreated() { return created; }
@Override
public void setCreated(Date created) { this.created = created; }
public BigInteger getCreatedBy() { return createdBy; }
public void setCreatedBy(BigInteger createdBy) { this.createdBy = createdBy; }
@Override
public Date getModified() { return modified; }
@Override
public void setModified(Date modified) { this.modified = modified; }
public BigInteger getModifiedBy() { return modifiedBy; }
public void setModifiedBy(BigInteger modifiedBy) { this.modifiedBy = modifiedBy; }
}

View File

@@ -0,0 +1,6 @@
package tech.easyflow.datacenter.meta.enums;
public enum DatacenterAccessMode {
READ_ONLY,
READ_WRITE
}

View File

@@ -0,0 +1,10 @@
package tech.easyflow.datacenter.meta.enums;
public enum DatacenterCapability {
TEST_CONNECTION,
BROWSE_METADATA,
READ_QUERY,
WRITE_MUTATION,
EXPORT,
MATERIALIZE
}

View File

@@ -0,0 +1,12 @@
package tech.easyflow.datacenter.meta.enums;
public enum DatacenterConnectionErrorCode {
INVALID_ARGUMENT,
DRIVER_NOT_FOUND,
NETWORK_UNREACHABLE,
AUTH_FAILED,
DATABASE_NOT_FOUND,
SCHEMA_NOT_FOUND,
PERMISSION_DENIED,
UNKNOWN_ERROR
}

View File

@@ -0,0 +1,9 @@
package tech.easyflow.datacenter.meta.enums;
public enum DatacenterImportStatus {
PENDING,
RUNNING,
SUCCESS,
FAILED,
NOT_IMPLEMENTED
}

View File

@@ -0,0 +1,13 @@
package tech.easyflow.datacenter.meta.enums;
public enum DatacenterSourceType {
PROJECT_MYSQL,
EXCEL,
EXCEL_MATERIALIZED,
MYSQL,
POSTGRESQL,
ORACLE,
GAUSSDB_NATIVE,
GBASE_8A,
GBASE_8S
}

View File

@@ -0,0 +1,10 @@
package tech.easyflow.datacenter.meta.enums;
public enum DatacenterTableKind {
PROJECT_MANAGED,
EXTERNAL_TABLE,
EXTERNAL_VIEW,
EXCEL_SHEET,
EXCEL_MATERIALIZED,
DERIVED_TABLE
}

View File

@@ -0,0 +1,36 @@
package tech.easyflow.datacenter.meta.model;
import java.math.BigInteger;
import java.util.ArrayList;
import java.util.List;
public class DatacenterBatchRegisterRequest {
private BigInteger sourceId;
private String catalogName;
private List<String> tableNames = new ArrayList<>();
public BigInteger getSourceId() {
return sourceId;
}
public void setSourceId(BigInteger sourceId) {
this.sourceId = sourceId;
}
public String getCatalogName() {
return catalogName;
}
public void setCatalogName(String catalogName) {
this.catalogName = catalogName;
}
public List<String> getTableNames() {
return tableNames;
}
public void setTableNames(List<String> tableNames) {
this.tableNames = tableNames;
}
}

View File

@@ -0,0 +1,18 @@
package tech.easyflow.datacenter.meta.model;
import java.math.BigInteger;
import java.util.ArrayList;
import java.util.List;
public class DatacenterBatchRemoveRequest {
private List<BigInteger> tableIds = new ArrayList<>();
public List<BigInteger> getTableIds() {
return tableIds;
}
public void setTableIds(List<BigInteger> tableIds) {
this.tableIds = tableIds;
}
}

View File

@@ -0,0 +1,22 @@
package tech.easyflow.datacenter.meta.model;
import java.math.BigInteger;
public class DatacenterCatalogMeta {
private BigInteger id;
private BigInteger sourceId;
private String catalogName;
private String catalogType;
private String catalogDesc;
public BigInteger getId() { return id; }
public void setId(BigInteger id) { this.id = id; }
public BigInteger getSourceId() { return sourceId; }
public void setSourceId(BigInteger sourceId) { this.sourceId = sourceId; }
public String getCatalogName() { return catalogName; }
public void setCatalogName(String catalogName) { this.catalogName = catalogName; }
public String getCatalogType() { return catalogType; }
public void setCatalogType(String catalogType) { this.catalogType = catalogType; }
public String getCatalogDesc() { return catalogDesc; }
public void setCatalogDesc(String catalogDesc) { this.catalogDesc = catalogDesc; }
}

View File

@@ -0,0 +1,25 @@
package tech.easyflow.datacenter.meta.model;
import java.math.BigInteger;
public class DatacenterFieldDescriptionUpdate {
private BigInteger fieldId;
private String fieldDesc;
public BigInteger getFieldId() {
return fieldId;
}
public void setFieldId(BigInteger fieldId) {
this.fieldId = fieldId;
}
public String getFieldDesc() {
return fieldDesc;
}
public void setFieldDesc(String fieldDesc) {
this.fieldDesc = fieldDesc;
}
}

View File

@@ -0,0 +1,16 @@
package tech.easyflow.datacenter.meta.model;
import java.math.BigInteger;
public class DatacenterRemoveSourceRequest {
private BigInteger sourceId;
public BigInteger getSourceId() {
return sourceId;
}
public void setSourceId(BigInteger sourceId) {
this.sourceId = sourceId;
}
}

View File

@@ -0,0 +1,36 @@
package tech.easyflow.datacenter.meta.model;
import java.math.BigInteger;
import java.util.ArrayList;
import java.util.List;
public class DatacenterSaveDescriptionsRequest {
private BigInteger tableId;
private String tableDesc;
private List<DatacenterFieldDescriptionUpdate> fields = new ArrayList<>();
public BigInteger getTableId() {
return tableId;
}
public void setTableId(BigInteger tableId) {
this.tableId = tableId;
}
public String getTableDesc() {
return tableDesc;
}
public void setTableDesc(String tableDesc) {
this.tableDesc = tableDesc;
}
public List<DatacenterFieldDescriptionUpdate> getFields() {
return fields;
}
public void setFields(List<DatacenterFieldDescriptionUpdate> fields) {
this.fields = fields;
}
}

View File

@@ -0,0 +1,17 @@
package tech.easyflow.datacenter.meta.model;
import tech.easyflow.datacenter.entity.DatacenterTable;
import tech.easyflow.datacenter.entity.DatacenterTableField;
import java.util.ArrayList;
import java.util.List;
public class DatacenterTableDetailMeta {
private DatacenterTable table;
private List<DatacenterTableField> fields = new ArrayList<>();
public DatacenterTable getTable() { return table; }
public void setTable(DatacenterTable table) { this.table = table; }
public List<DatacenterTableField> getFields() { return fields; }
public void setFields(List<DatacenterTableField> fields) { this.fields = fields; }
}

View File

@@ -0,0 +1,38 @@
package tech.easyflow.datacenter.meta.service;
import tech.easyflow.common.entity.LoginAccount;
import tech.easyflow.datacenter.entity.DatacenterTable;
import tech.easyflow.datacenter.entity.DatacenterTableField;
import tech.easyflow.datacenter.execution.model.DatasetRef;
import tech.easyflow.datacenter.meta.entity.DatacenterCatalog;
import tech.easyflow.datacenter.meta.entity.DatacenterSource;
import tech.easyflow.datacenter.meta.model.DatacenterFieldDescriptionUpdate;
import tech.easyflow.datacenter.meta.enums.DatacenterSourceType;
import tech.easyflow.datacenter.meta.model.DatacenterTableDetailMeta;
import java.math.BigInteger;
import java.util.List;
public interface DatacenterDatasetRegistryService {
DatacenterSource ensureBuiltinSource(DatacenterSourceType sourceType, LoginAccount account);
DatacenterCatalog ensureCatalog(DatacenterSource source, String catalogName, LoginAccount account);
DatacenterTable registerTable(DatacenterSource source, DatacenterCatalog catalog, DatacenterTableDetailMeta detail, LoginAccount account);
DatacenterTable getTableWithFields(BigInteger tableId);
List<DatacenterTableField> getFields(BigInteger tableId);
DatasetRef resolveDatasetRef(BigInteger tableId);
DatacenterSource getSourceRequired(BigInteger sourceId);
DatacenterCatalog getCatalogById(BigInteger catalogId);
List<DatacenterTable> listManagedTables(BigInteger sourceId, BigInteger catalogId);
int removeTables(List<BigInteger> tableIds);
DatacenterTable saveDescriptions(BigInteger tableId, String tableDesc, List<DatacenterFieldDescriptionUpdate> fields, LoginAccount account);
}

View File

@@ -0,0 +1,12 @@
package tech.easyflow.datacenter.meta.service;
import java.math.BigInteger;
public final class DatacenterMetaConstants {
private DatacenterMetaConstants() {
}
public static final BigInteger PROJECT_SOURCE_BASE = new BigInteger("9000000000002000000");
public static final BigInteger PROJECT_CATALOG_BASE = new BigInteger("9000000000003000000");
}

View File

@@ -0,0 +1,32 @@
package tech.easyflow.datacenter.meta.service;
import com.mybatisflex.core.paginate.Page;
import com.mybatisflex.core.service.IService;
import tech.easyflow.common.entity.LoginAccount;
import tech.easyflow.datacenter.entity.DatacenterTable;
import tech.easyflow.datacenter.execution.model.DatacenterConnectionTestResult;
import tech.easyflow.datacenter.meta.entity.DatacenterSource;
import tech.easyflow.datacenter.meta.model.DatacenterBatchRegisterRequest;
import tech.easyflow.datacenter.meta.model.DatacenterCatalogMeta;
import tech.easyflow.datacenter.meta.model.DatacenterTableDetailMeta;
import java.math.BigInteger;
import java.util.List;
public interface DatacenterSourceService extends IService<DatacenterSource> {
DatacenterSource saveSource(DatacenterSource source, LoginAccount account);
Page<DatacenterSource> pageSources(Long pageNumber, Long pageSize, LoginAccount account);
DatacenterConnectionTestResult testConnection(DatacenterSource source, LoginAccount account);
List<DatacenterCatalogMeta> listCatalogs(BigInteger sourceId, LoginAccount account);
List<DatacenterTable> listTables(BigInteger sourceId, String catalogName, LoginAccount account);
DatacenterTableDetailMeta getTableDetail(BigInteger sourceId, String catalogName, String tableName, boolean register, LoginAccount account);
List<DatacenterTable> batchRegisterTables(DatacenterBatchRegisterRequest request, LoginAccount account);
void removeSource(BigInteger sourceId, LoginAccount account);
}

Some files were not shown because too many files have changed in this diff Show More