Compare commits
2 Commits
b6213d0933
...
1ecc28e498
| Author | SHA1 | Date | |
|---|---|---|---|
| 1ecc28e498 | |||
| 798effbd5b |
@@ -13,6 +13,7 @@ import com.easyagents.flow.core.parser.ChainParser;
|
|||||||
import org.springframework.web.bind.annotation.GetMapping;
|
import org.springframework.web.bind.annotation.GetMapping;
|
||||||
import org.springframework.web.bind.annotation.RequestMapping;
|
import org.springframework.web.bind.annotation.RequestMapping;
|
||||||
import org.springframework.web.bind.annotation.RestController;
|
import org.springframework.web.bind.annotation.RestController;
|
||||||
|
import tech.easyflow.ai.easyagentsflow.service.WorkflowDatacenterContentService;
|
||||||
import tech.easyflow.ai.entity.Workflow;
|
import tech.easyflow.ai.entity.Workflow;
|
||||||
import tech.easyflow.ai.service.WorkflowService;
|
import tech.easyflow.ai.service.WorkflowService;
|
||||||
import tech.easyflow.common.domain.Result;
|
import tech.easyflow.common.domain.Result;
|
||||||
@@ -29,6 +30,8 @@ public class WorkFlowNodeController {
|
|||||||
private WorkflowService workflowService;
|
private WorkflowService workflowService;
|
||||||
@Resource
|
@Resource
|
||||||
private ChainParser chainParser;
|
private ChainParser chainParser;
|
||||||
|
@Resource
|
||||||
|
private WorkflowDatacenterContentService workflowDatacenterContentService;
|
||||||
|
|
||||||
@GetMapping("/getChainParams")
|
@GetMapping("/getChainParams")
|
||||||
public Result<?> getChainParams(String currentId, String workflowId) {
|
public Result<?> getChainParams(String currentId, String workflowId) {
|
||||||
@@ -43,7 +46,7 @@ public class WorkFlowNodeController {
|
|||||||
nodeData.put("workflowId", workflow.getId());
|
nodeData.put("workflowId", workflow.getId());
|
||||||
nodeData.put("workflowName", workflow.getTitle());
|
nodeData.put("workflowName", workflow.getTitle());
|
||||||
|
|
||||||
ChainDefinition definition = chainParser.parse(workflow.getContent());
|
ChainDefinition definition = chainParser.parse(workflowDatacenterContentService.prepareContent(workflow.getContent()));
|
||||||
List<Node> nodes = definition.getNodes();
|
List<Node> nodes = definition.getNodes();
|
||||||
JSONArray inputs = new JSONArray();
|
JSONArray inputs = new JSONArray();
|
||||||
JSONArray outputs = new JSONArray();
|
JSONArray outputs = new JSONArray();
|
||||||
|
|||||||
@@ -21,6 +21,7 @@ import tech.easyflow.ai.easyagentsflow.entity.WorkflowCheckStage;
|
|||||||
import tech.easyflow.ai.easyagentsflow.service.CodeEngineCapabilityService;
|
import tech.easyflow.ai.easyagentsflow.service.CodeEngineCapabilityService;
|
||||||
import tech.easyflow.ai.easyagentsflow.service.TinyFlowService;
|
import tech.easyflow.ai.easyagentsflow.service.TinyFlowService;
|
||||||
import tech.easyflow.ai.easyagentsflow.service.WorkflowCheckService;
|
import tech.easyflow.ai.easyagentsflow.service.WorkflowCheckService;
|
||||||
|
import tech.easyflow.ai.easyagentsflow.service.WorkflowDatacenterContentService;
|
||||||
import tech.easyflow.ai.entity.Workflow;
|
import tech.easyflow.ai.entity.Workflow;
|
||||||
import tech.easyflow.ai.service.BotWorkflowService;
|
import tech.easyflow.ai.service.BotWorkflowService;
|
||||||
import tech.easyflow.ai.service.ModelService;
|
import tech.easyflow.ai.service.ModelService;
|
||||||
@@ -76,6 +77,8 @@ public class WorkflowController extends BaseCurdController<WorkflowService, Work
|
|||||||
@Resource
|
@Resource
|
||||||
private WorkflowCheckService workflowCheckService;
|
private WorkflowCheckService workflowCheckService;
|
||||||
@Resource
|
@Resource
|
||||||
|
private WorkflowDatacenterContentService workflowDatacenterContentService;
|
||||||
|
@Resource
|
||||||
private ResourceAccessService resourceAccessService;
|
private ResourceAccessService resourceAccessService;
|
||||||
@Resource
|
@Resource
|
||||||
private WorkflowVisibilityQueryHelper workflowVisibilityQueryHelper;
|
private WorkflowVisibilityQueryHelper workflowVisibilityQueryHelper;
|
||||||
@@ -223,7 +226,7 @@ public class WorkflowController extends BaseCurdController<WorkflowService, Work
|
|||||||
}
|
}
|
||||||
workflowCheckService.checkOrThrow(workflow.getContent(), WorkflowCheckStage.PRE_EXECUTE, workflow.getId());
|
workflowCheckService.checkOrThrow(workflow.getContent(), WorkflowCheckStage.PRE_EXECUTE, workflow.getId());
|
||||||
|
|
||||||
ChainDefinition definition = chainParser.parse(workflow.getContent());
|
ChainDefinition definition = chainParser.parse(workflowDatacenterContentService.prepareContent(workflow.getContent()));
|
||||||
if (definition == null) {
|
if (definition == null) {
|
||||||
return Result.fail(2, "节点配置错误,请检查! ");
|
return Result.fail(2, "节点配置错误,请检查! ");
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,68 @@
|
|||||||
|
package tech.easyflow.admin.controller.datacenter;
|
||||||
|
|
||||||
|
import cn.dev33.satoken.annotation.SaCheckPermission;
|
||||||
|
import com.mybatisflex.core.paginate.Page;
|
||||||
|
import com.mybatisflex.core.row.Row;
|
||||||
|
import org.springframework.web.bind.annotation.*;
|
||||||
|
import tech.easyflow.common.domain.Result;
|
||||||
|
import tech.easyflow.common.entity.LoginAccount;
|
||||||
|
import tech.easyflow.common.satoken.util.SaTokenUtil;
|
||||||
|
import tech.easyflow.datacenter.entity.DatacenterTable;
|
||||||
|
import tech.easyflow.datacenter.execution.model.DatacenterQueryRequest;
|
||||||
|
import tech.easyflow.datacenter.execution.model.DatacenterSchemaResponse;
|
||||||
|
import tech.easyflow.datacenter.execution.model.DatasetRef;
|
||||||
|
import tech.easyflow.datacenter.meta.model.DatacenterBatchRemoveRequest;
|
||||||
|
import tech.easyflow.datacenter.meta.model.DatacenterSaveDescriptionsRequest;
|
||||||
|
import tech.easyflow.datacenter.execution.service.DatacenterDatasetQueryService;
|
||||||
|
import tech.easyflow.datacenter.meta.service.DatacenterDatasetRegistryService;
|
||||||
|
|
||||||
|
import javax.annotation.Resource;
|
||||||
|
import java.math.BigInteger;
|
||||||
|
import java.util.List;
|
||||||
|
|
||||||
|
@RestController
|
||||||
|
@RequestMapping("/api/v1/datacenterDataset")
|
||||||
|
public class DatacenterDatasetController {
|
||||||
|
|
||||||
|
@Resource
|
||||||
|
private DatacenterDatasetQueryService queryService;
|
||||||
|
@Resource
|
||||||
|
private DatacenterDatasetRegistryService registryService;
|
||||||
|
|
||||||
|
@PostMapping("/queryPage")
|
||||||
|
@SaCheckPermission("/api/v1/datacenterSource/query")
|
||||||
|
public Result<Page<Row>> queryPage(@RequestBody DatacenterQueryRequest request) {
|
||||||
|
return Result.ok(queryService.queryPage(request));
|
||||||
|
}
|
||||||
|
|
||||||
|
@GetMapping("/schema")
|
||||||
|
@SaCheckPermission("/api/v1/datacenterSource/query")
|
||||||
|
public Result<DatacenterSchemaResponse> schema(DatasetRef datasetRef) {
|
||||||
|
return Result.ok(queryService.getSchema(datasetRef));
|
||||||
|
}
|
||||||
|
|
||||||
|
@GetMapping("/managedTables")
|
||||||
|
@SaCheckPermission("/api/v1/datacenterSource/query")
|
||||||
|
public Result<List<DatacenterTable>> managedTables(BigInteger sourceId, BigInteger catalogId) {
|
||||||
|
return Result.ok(registryService.listManagedTables(sourceId, catalogId));
|
||||||
|
}
|
||||||
|
|
||||||
|
@PostMapping("/removeBatch")
|
||||||
|
@SaCheckPermission("/api/v1/datacenterSource/save")
|
||||||
|
public Result<Integer> removeBatch(@RequestBody DatacenterBatchRemoveRequest request) {
|
||||||
|
return Result.ok(registryService.removeTables(request == null ? List.of() : request.getTableIds()));
|
||||||
|
}
|
||||||
|
|
||||||
|
@PostMapping("/saveDescriptions")
|
||||||
|
@SaCheckPermission("/api/v1/datacenterSource/save")
|
||||||
|
public Result<DatacenterSchemaResponse> saveDescriptions(@RequestBody DatacenterSaveDescriptionsRequest request) {
|
||||||
|
LoginAccount account = SaTokenUtil.getLoginAccount();
|
||||||
|
DatacenterTable table = registryService.saveDescriptions(
|
||||||
|
request == null ? null : request.getTableId(),
|
||||||
|
request == null ? null : request.getTableDesc(),
|
||||||
|
request == null ? List.of() : request.getFields(),
|
||||||
|
account
|
||||||
|
);
|
||||||
|
return Result.ok(queryService.getSchema(registryService.resolveDatasetRef(table.getId())));
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,108 @@
|
|||||||
|
package tech.easyflow.admin.controller.datacenter;
|
||||||
|
|
||||||
|
import cn.dev33.satoken.annotation.SaCheckPermission;
|
||||||
|
import jakarta.servlet.http.HttpServletResponse;
|
||||||
|
import org.springframework.web.bind.annotation.GetMapping;
|
||||||
|
import org.springframework.web.bind.annotation.PostMapping;
|
||||||
|
import org.springframework.web.bind.annotation.RequestBody;
|
||||||
|
import org.springframework.web.bind.annotation.RequestMapping;
|
||||||
|
import org.springframework.web.bind.annotation.RestController;
|
||||||
|
import org.springframework.web.multipart.MultipartFile;
|
||||||
|
import tech.easyflow.common.domain.Result;
|
||||||
|
import tech.easyflow.common.entity.LoginAccount;
|
||||||
|
import tech.easyflow.common.satoken.util.SaTokenUtil;
|
||||||
|
import tech.easyflow.datacenter.excel.model.DatacenterExcelDeriveRequest;
|
||||||
|
import tech.easyflow.datacenter.excel.model.DatacenterExcelExportRequest;
|
||||||
|
import tech.easyflow.datacenter.excel.model.DatacenterExcelMergeRequest;
|
||||||
|
import tech.easyflow.datacenter.excel.model.DatacenterExcelSplitRequest;
|
||||||
|
import tech.easyflow.datacenter.excel.service.DatacenterExcelImportService;
|
||||||
|
import tech.easyflow.datacenter.meta.entity.DatacenterImportJob;
|
||||||
|
|
||||||
|
import java.io.File;
|
||||||
|
import java.io.FileInputStream;
|
||||||
|
import javax.annotation.Resource;
|
||||||
|
import java.math.BigInteger;
|
||||||
|
import java.net.URLEncoder;
|
||||||
|
import java.util.List;
|
||||||
|
|
||||||
|
@RestController
|
||||||
|
@RequestMapping("/api/v1/datacenterExcel")
|
||||||
|
public class DatacenterExcelController {
|
||||||
|
|
||||||
|
@Resource
|
||||||
|
private DatacenterExcelImportService excelImportService;
|
||||||
|
|
||||||
|
@PostMapping("/import")
|
||||||
|
@SaCheckPermission("/api/v1/datacenterSource/save")
|
||||||
|
public Result<DatacenterImportJob> importWorkbook(MultipartFile file) throws Exception {
|
||||||
|
LoginAccount account = SaTokenUtil.getLoginAccount();
|
||||||
|
return Result.ok(excelImportService.importWorkbook(file, account));
|
||||||
|
}
|
||||||
|
|
||||||
|
@PostMapping("/split")
|
||||||
|
@SaCheckPermission("/api/v1/datacenterSource/save")
|
||||||
|
public Result<DatacenterImportJob> split(@RequestBody DatacenterExcelSplitRequest request) {
|
||||||
|
LoginAccount account = SaTokenUtil.getLoginAccount();
|
||||||
|
return Result.ok(excelImportService.splitWorkbook(request, account));
|
||||||
|
}
|
||||||
|
|
||||||
|
@PostMapping("/merge")
|
||||||
|
@SaCheckPermission("/api/v1/datacenterSource/save")
|
||||||
|
public Result<DatacenterImportJob> merge(@RequestBody DatacenterExcelMergeRequest request) {
|
||||||
|
LoginAccount account = SaTokenUtil.getLoginAccount();
|
||||||
|
return Result.ok(excelImportService.mergeWorkbook(request, account));
|
||||||
|
}
|
||||||
|
|
||||||
|
@PostMapping("/derive")
|
||||||
|
@SaCheckPermission("/api/v1/datacenterSource/save")
|
||||||
|
public Result<DatacenterImportJob> derive(@RequestBody DatacenterExcelDeriveRequest request) {
|
||||||
|
LoginAccount account = SaTokenUtil.getLoginAccount();
|
||||||
|
return Result.ok(excelImportService.deriveWorkbook(request, account));
|
||||||
|
}
|
||||||
|
|
||||||
|
@PostMapping("/export")
|
||||||
|
@SaCheckPermission("/api/v1/datacenterSource/query")
|
||||||
|
public Result<DatacenterImportJob> export(@RequestBody DatacenterExcelExportRequest request) throws Exception {
|
||||||
|
LoginAccount account = SaTokenUtil.getLoginAccount();
|
||||||
|
return Result.ok(excelImportService.exportWorkbook(request, account));
|
||||||
|
}
|
||||||
|
|
||||||
|
@GetMapping("/importJob/detail")
|
||||||
|
@SaCheckPermission("/api/v1/datacenterSource/query")
|
||||||
|
public Result<DatacenterImportJob> jobDetail(BigInteger jobId) {
|
||||||
|
return Result.ok(excelImportService.getImportJobDetail(jobId));
|
||||||
|
}
|
||||||
|
|
||||||
|
@GetMapping("/job/detail")
|
||||||
|
@SaCheckPermission("/api/v1/datacenterSource/query")
|
||||||
|
public Result<DatacenterImportJob> newJobDetail(BigInteger jobId) {
|
||||||
|
return Result.ok(excelImportService.getImportJobDetail(jobId));
|
||||||
|
}
|
||||||
|
|
||||||
|
@GetMapping("/job/list")
|
||||||
|
@SaCheckPermission("/api/v1/datacenterSource/query")
|
||||||
|
public Result<List<DatacenterImportJob>> jobList(BigInteger sourceId, BigInteger tableId) {
|
||||||
|
return Result.ok(excelImportService.listJobs(sourceId, tableId));
|
||||||
|
}
|
||||||
|
|
||||||
|
@GetMapping("/download")
|
||||||
|
@SaCheckPermission("/api/v1/datacenterSource/query")
|
||||||
|
public void download(BigInteger jobId, HttpServletResponse response) throws Exception {
|
||||||
|
DatacenterImportJob job = excelImportService.getImportJobDetail(jobId);
|
||||||
|
if (job.getStoragePath() == null || job.getStoragePath().isBlank()) {
|
||||||
|
throw new IllegalStateException("导出文件不存在");
|
||||||
|
}
|
||||||
|
File file = new File(job.getStoragePath());
|
||||||
|
if (!file.exists()) {
|
||||||
|
throw new IllegalStateException("导出文件不存在");
|
||||||
|
}
|
||||||
|
response.setContentType("application/vnd.openxmlformats-officedocument.spreadsheetml.sheet");
|
||||||
|
response.setCharacterEncoding("utf-8");
|
||||||
|
String fileName = URLEncoder.encode(job.getFileName(), "UTF-8").replaceAll("\\+", "%20");
|
||||||
|
response.setHeader("Content-disposition", "attachment;filename*=utf-8''" + fileName);
|
||||||
|
try (FileInputStream inputStream = new FileInputStream(file)) {
|
||||||
|
inputStream.transferTo(response.getOutputStream());
|
||||||
|
response.flushBuffer();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,86 @@
|
|||||||
|
package tech.easyflow.admin.controller.datacenter;
|
||||||
|
|
||||||
|
import cn.dev33.satoken.annotation.SaCheckPermission;
|
||||||
|
import com.mybatisflex.core.paginate.Page;
|
||||||
|
import org.springframework.web.bind.annotation.*;
|
||||||
|
import tech.easyflow.common.domain.Result;
|
||||||
|
import tech.easyflow.common.entity.LoginAccount;
|
||||||
|
import tech.easyflow.common.satoken.util.SaTokenUtil;
|
||||||
|
import tech.easyflow.datacenter.entity.DatacenterTable;
|
||||||
|
import tech.easyflow.datacenter.execution.model.DatacenterConnectionTestResult;
|
||||||
|
import tech.easyflow.datacenter.meta.entity.DatacenterSource;
|
||||||
|
import tech.easyflow.datacenter.meta.model.DatacenterBatchRegisterRequest;
|
||||||
|
import tech.easyflow.datacenter.meta.model.DatacenterCatalogMeta;
|
||||||
|
import tech.easyflow.datacenter.meta.model.DatacenterRemoveSourceRequest;
|
||||||
|
import tech.easyflow.datacenter.meta.model.DatacenterTableDetailMeta;
|
||||||
|
import tech.easyflow.datacenter.meta.service.DatacenterSourceService;
|
||||||
|
|
||||||
|
import javax.annotation.Resource;
|
||||||
|
import java.math.BigInteger;
|
||||||
|
import java.util.List;
|
||||||
|
|
||||||
|
@RestController
|
||||||
|
@RequestMapping("/api/v1/datacenterSource")
|
||||||
|
public class DatacenterSourceController {
|
||||||
|
|
||||||
|
@Resource
|
||||||
|
private DatacenterSourceService sourceService;
|
||||||
|
|
||||||
|
@PostMapping("/testConnection")
|
||||||
|
@SaCheckPermission("/api/v1/datacenterSource/query")
|
||||||
|
public Result<DatacenterConnectionTestResult> testConnection(@RequestBody DatacenterSource source) {
|
||||||
|
LoginAccount account = SaTokenUtil.getLoginAccount();
|
||||||
|
return Result.ok(sourceService.testConnection(source, account));
|
||||||
|
}
|
||||||
|
|
||||||
|
@PostMapping("/save")
|
||||||
|
@SaCheckPermission("/api/v1/datacenterSource/save")
|
||||||
|
public Result<DatacenterSource> save(@RequestBody DatacenterSource source) {
|
||||||
|
LoginAccount account = SaTokenUtil.getLoginAccount();
|
||||||
|
return Result.ok(sourceService.saveSource(source, account));
|
||||||
|
}
|
||||||
|
|
||||||
|
@GetMapping("/page")
|
||||||
|
@SaCheckPermission("/api/v1/datacenterSource/query")
|
||||||
|
public Result<Page<DatacenterSource>> page(Long pageNumber, Long pageSize) {
|
||||||
|
LoginAccount account = SaTokenUtil.getLoginAccount();
|
||||||
|
return Result.ok(sourceService.pageSources(pageNumber, pageSize, account));
|
||||||
|
}
|
||||||
|
|
||||||
|
@GetMapping("/catalogs")
|
||||||
|
@SaCheckPermission("/api/v1/datacenterSource/query")
|
||||||
|
public Result<List<DatacenterCatalogMeta>> catalogs(BigInteger sourceId) {
|
||||||
|
LoginAccount account = SaTokenUtil.getLoginAccount();
|
||||||
|
return Result.ok(sourceService.listCatalogs(sourceId, account));
|
||||||
|
}
|
||||||
|
|
||||||
|
@GetMapping("/tables")
|
||||||
|
@SaCheckPermission("/api/v1/datacenterSource/query")
|
||||||
|
public Result<List<DatacenterTable>> tables(BigInteger sourceId, String catalogName) {
|
||||||
|
LoginAccount account = SaTokenUtil.getLoginAccount();
|
||||||
|
return Result.ok(sourceService.listTables(sourceId, catalogName, account));
|
||||||
|
}
|
||||||
|
|
||||||
|
@GetMapping("/tableDetail")
|
||||||
|
@SaCheckPermission("/api/v1/datacenterSource/query")
|
||||||
|
public Result<DatacenterTableDetailMeta> tableDetail(BigInteger sourceId, String catalogName, String tableName,
|
||||||
|
@RequestParam(defaultValue = "false") boolean register) {
|
||||||
|
LoginAccount account = SaTokenUtil.getLoginAccount();
|
||||||
|
return Result.ok(sourceService.getTableDetail(sourceId, catalogName, tableName, register, account));
|
||||||
|
}
|
||||||
|
|
||||||
|
@PostMapping("/registerBatch")
|
||||||
|
@SaCheckPermission("/api/v1/datacenterSource/save")
|
||||||
|
public Result<List<DatacenterTable>> registerBatch(@RequestBody DatacenterBatchRegisterRequest request) {
|
||||||
|
LoginAccount account = SaTokenUtil.getLoginAccount();
|
||||||
|
return Result.ok(sourceService.batchRegisterTables(request, account));
|
||||||
|
}
|
||||||
|
|
||||||
|
@PostMapping("/remove")
|
||||||
|
@SaCheckPermission("/api/v1/datacenterSource/save")
|
||||||
|
public Result<Void> remove(@RequestBody DatacenterRemoveSourceRequest request) {
|
||||||
|
LoginAccount account = SaTokenUtil.getLoginAccount();
|
||||||
|
sourceService.removeSource(request == null ? null : request.getSourceId(), account);
|
||||||
|
return Result.ok();
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -1,195 +0,0 @@
|
|||||||
package tech.easyflow.admin.controller.datacenter;
|
|
||||||
|
|
||||||
import cn.dev33.satoken.annotation.SaCheckPermission;
|
|
||||||
import cn.hutool.core.collection.CollectionUtil;
|
|
||||||
import cn.idev.excel.EasyExcel;
|
|
||||||
import cn.idev.excel.FastExcel;
|
|
||||||
import com.alibaba.fastjson2.JSONObject;
|
|
||||||
import com.mybatisflex.core.paginate.Page;
|
|
||||||
import com.mybatisflex.core.query.QueryWrapper;
|
|
||||||
import com.mybatisflex.core.row.Row;
|
|
||||||
import jakarta.servlet.http.HttpServletResponse;
|
|
||||||
import org.springframework.web.bind.annotation.*;
|
|
||||||
import org.springframework.web.multipart.MultipartFile;
|
|
||||||
import tech.easyflow.common.domain.Result;
|
|
||||||
import tech.easyflow.common.entity.DatacenterQuery;
|
|
||||||
import tech.easyflow.common.entity.LoginAccount;
|
|
||||||
import tech.easyflow.common.satoken.util.SaTokenUtil;
|
|
||||||
import tech.easyflow.common.web.controller.BaseCurdController;
|
|
||||||
import tech.easyflow.datacenter.entity.DatacenterTable;
|
|
||||||
import tech.easyflow.datacenter.entity.DatacenterTableField;
|
|
||||||
import tech.easyflow.datacenter.entity.vo.HeaderVo;
|
|
||||||
import tech.easyflow.datacenter.excel.ReadDataListener;
|
|
||||||
import tech.easyflow.datacenter.excel.ReadResVo;
|
|
||||||
import tech.easyflow.datacenter.service.DatacenterTableFieldService;
|
|
||||||
import tech.easyflow.datacenter.service.DatacenterTableService;
|
|
||||||
|
|
||||||
import javax.annotation.Resource;
|
|
||||||
import java.io.InputStream;
|
|
||||||
import java.math.BigInteger;
|
|
||||||
import java.net.URLEncoder;
|
|
||||||
import java.util.ArrayList;
|
|
||||||
import java.util.Date;
|
|
||||||
import java.util.List;
|
|
||||||
import java.util.Map;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 数据中枢表 控制层。
|
|
||||||
*
|
|
||||||
* @author ArkLight
|
|
||||||
* @since 2025-07-10
|
|
||||||
*/
|
|
||||||
@RestController
|
|
||||||
@RequestMapping("/api/v1/datacenterTable")
|
|
||||||
public class DatacenterTableController extends BaseCurdController<DatacenterTableService, DatacenterTable> {
|
|
||||||
|
|
||||||
@Resource
|
|
||||||
private DatacenterTableFieldService fieldsService;
|
|
||||||
|
|
||||||
public DatacenterTableController(DatacenterTableService service) {
|
|
||||||
super(service);
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
protected Result onSaveOrUpdateBefore(DatacenterTable entity, boolean isSave) {
|
|
||||||
LoginAccount loginUser = SaTokenUtil.getLoginAccount();
|
|
||||||
if (isSave) {
|
|
||||||
commonFiled(entity, loginUser.getId(), loginUser.getTenantId(), loginUser.getDeptId());
|
|
||||||
} else {
|
|
||||||
entity.setModifiedBy(loginUser.getId());
|
|
||||||
}
|
|
||||||
return super.onSaveOrUpdateBefore(entity, isSave);
|
|
||||||
}
|
|
||||||
|
|
||||||
@PostMapping("/saveTable")
|
|
||||||
@SaCheckPermission("/api/v1/datacenterTable/save")
|
|
||||||
public Result<Void> saveTable(@RequestBody DatacenterTable entity) {
|
|
||||||
LoginAccount loginUser = SaTokenUtil.getLoginAccount();
|
|
||||||
List<DatacenterTableField> fields = entity.getFields();
|
|
||||||
if (CollectionUtil.isEmpty(fields)) {
|
|
||||||
return Result.fail(99, "字段不能为空");
|
|
||||||
}
|
|
||||||
BigInteger id = entity.getId();
|
|
||||||
if (id == null) {
|
|
||||||
commonFiled(entity, loginUser.getId(), loginUser.getTenantId(), loginUser.getDeptId());
|
|
||||||
} else {
|
|
||||||
entity.setModified(new Date());
|
|
||||||
entity.setModifiedBy(loginUser.getId());
|
|
||||||
}
|
|
||||||
service.saveTable(entity, loginUser);
|
|
||||||
return Result.ok();
|
|
||||||
}
|
|
||||||
|
|
||||||
@GetMapping("/detailInfo")
|
|
||||||
@SaCheckPermission("/api/v1/datacenterTable/query")
|
|
||||||
public Result<DatacenterTable> detailInfo(BigInteger tableId) {
|
|
||||||
DatacenterTable table = service.getById(tableId);
|
|
||||||
QueryWrapper wrapper = QueryWrapper.create();
|
|
||||||
wrapper.eq(DatacenterTableField::getTableId, tableId);
|
|
||||||
wrapper.orderBy("id");
|
|
||||||
List<DatacenterTableField> fields = fieldsService.list(wrapper);
|
|
||||||
table.setFields(fields);
|
|
||||||
return Result.ok(table);
|
|
||||||
}
|
|
||||||
|
|
||||||
@GetMapping("/removeTable")
|
|
||||||
@SaCheckPermission("/api/v1/datacenterTable/remove")
|
|
||||||
public Result<Void> removeTable(BigInteger tableId) {
|
|
||||||
service.removeTable(tableId);
|
|
||||||
return Result.ok();
|
|
||||||
}
|
|
||||||
|
|
||||||
@GetMapping("/getHeaders")
|
|
||||||
@SaCheckPermission("/api/v1/datacenterTable/query")
|
|
||||||
public Result<List<HeaderVo>> getHeaders(BigInteger tableId) {
|
|
||||||
List<HeaderVo> res = service.getHeaders(tableId);
|
|
||||||
return Result.ok(res);
|
|
||||||
}
|
|
||||||
|
|
||||||
@GetMapping("/getPageData")
|
|
||||||
@SaCheckPermission("/api/v1/datacenterTable/query")
|
|
||||||
public Result<Page<Row>> getPageData(DatacenterQuery where) {
|
|
||||||
Page<Row> res = service.getPageData(where);
|
|
||||||
return Result.ok(res);
|
|
||||||
}
|
|
||||||
|
|
||||||
@PostMapping("/saveValue")
|
|
||||||
@SaCheckPermission("/api/v1/datacenterTable/save")
|
|
||||||
public Result<Void> saveValue(@RequestParam Map<String, Object> map) {
|
|
||||||
JSONObject object = new JSONObject(map);
|
|
||||||
BigInteger tableId = object.getBigInteger("tableId");
|
|
||||||
LoginAccount account = SaTokenUtil.getLoginAccount();
|
|
||||||
if (tableId == null) {
|
|
||||||
return Result.fail(99, "参数错误");
|
|
||||||
}
|
|
||||||
service.saveValue(tableId, object, account);
|
|
||||||
return Result.ok();
|
|
||||||
}
|
|
||||||
|
|
||||||
@PostMapping("/removeValue")
|
|
||||||
@SaCheckPermission("/api/v1/datacenterTable/remove")
|
|
||||||
public Result<Void> removeValue(@RequestParam Map<String, Object> map) {
|
|
||||||
JSONObject object = new JSONObject(map);
|
|
||||||
BigInteger tableId = object.getBigInteger("tableId");
|
|
||||||
BigInteger id = object.getBigInteger("id");
|
|
||||||
if (tableId == null || id == null) {
|
|
||||||
return Result.fail(99, "参数错误");
|
|
||||||
}
|
|
||||||
LoginAccount account = SaTokenUtil.getLoginAccount();
|
|
||||||
service.removeValue(tableId, id, account);
|
|
||||||
return Result.ok();
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 导入数据
|
|
||||||
*/
|
|
||||||
@PostMapping("/importData")
|
|
||||||
@SaCheckPermission("/api/v1/datacenterTable/save")
|
|
||||||
public Result<ReadResVo> importData(MultipartFile file, @RequestParam Map<String, Object> map) throws Exception {
|
|
||||||
Object tableId = map.get("tableId");
|
|
||||||
DatacenterTable record = service.getById(tableId.toString());
|
|
||||||
if (record == null) {
|
|
||||||
throw new RuntimeException("数据表不存在");
|
|
||||||
}
|
|
||||||
InputStream is = file.getInputStream();
|
|
||||||
List<DatacenterTableField> fields = service.getFields(record.getId());
|
|
||||||
|
|
||||||
LoginAccount account = SaTokenUtil.getLoginAccount();
|
|
||||||
ReadDataListener listener = new ReadDataListener(record.getId(), fields, account);
|
|
||||||
FastExcel.read(is, listener)
|
|
||||||
.sheet()
|
|
||||||
.doRead();
|
|
||||||
int totalCount = listener.getTotalCount();
|
|
||||||
int errorCount = listener.getErrorCount();
|
|
||||||
int successCount = listener.getSuccessCount();
|
|
||||||
List<JSONObject> errorRows = listener.getErrorRows();
|
|
||||||
return Result.ok(new ReadResVo(successCount, errorCount, totalCount, errorRows));
|
|
||||||
}
|
|
||||||
|
|
||||||
@GetMapping("/getTemplate")
|
|
||||||
public void getTemplate(BigInteger tableId, HttpServletResponse response) throws Exception {
|
|
||||||
List<DatacenterTableField> fields = service.getFields(tableId);
|
|
||||||
// 设置响应内容类型
|
|
||||||
response.setContentType("application/vnd.openxmlformats-officedocument.spreadsheetml.sheet");
|
|
||||||
response.setCharacterEncoding("utf-8");
|
|
||||||
|
|
||||||
// 设置文件名
|
|
||||||
String fileName = URLEncoder.encode("导入模板", "UTF-8").replaceAll("\\+", "%20");
|
|
||||||
response.setHeader("Content-disposition", "attachment;filename*=utf-8''" + fileName + ".xlsx");
|
|
||||||
|
|
||||||
// 动态表头数据
|
|
||||||
List<List<String>> headList = new ArrayList<>();
|
|
||||||
|
|
||||||
for (DatacenterTableField field : fields) {
|
|
||||||
List<String> head = new ArrayList<>();
|
|
||||||
head.add(field.getFieldName());
|
|
||||||
headList.add(head);
|
|
||||||
}
|
|
||||||
|
|
||||||
// 写入Excel
|
|
||||||
EasyExcel.write(response.getOutputStream())
|
|
||||||
.head(headList)
|
|
||||||
.sheet("模板")
|
|
||||||
.doWrite(new ArrayList<>()); // 写入空数据,只生成模板
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,41 +0,0 @@
|
|||||||
package tech.easyflow.admin.controller.datacenter;
|
|
||||||
|
|
||||||
import org.springframework.web.bind.annotation.RequestMapping;
|
|
||||||
import org.springframework.web.bind.annotation.RestController;
|
|
||||||
import tech.easyflow.common.annotation.UsePermission;
|
|
||||||
import tech.easyflow.common.domain.Result;
|
|
||||||
import tech.easyflow.common.entity.LoginAccount;
|
|
||||||
import tech.easyflow.common.satoken.util.SaTokenUtil;
|
|
||||||
import tech.easyflow.common.web.controller.BaseCurdController;
|
|
||||||
import tech.easyflow.datacenter.entity.DatacenterTableField;
|
|
||||||
import tech.easyflow.datacenter.service.DatacenterTableFieldService;
|
|
||||||
|
|
||||||
import java.util.Date;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 控制层。
|
|
||||||
*
|
|
||||||
* @author ArkLight
|
|
||||||
* @since 2025-07-10
|
|
||||||
*/
|
|
||||||
@RestController
|
|
||||||
@RequestMapping("/api/v1/datacenterTableFields")
|
|
||||||
@UsePermission(moduleName = "/api/v1/datacenterTable")
|
|
||||||
public class DatacenterTableFieldsController extends BaseCurdController<DatacenterTableFieldService, DatacenterTableField> {
|
|
||||||
|
|
||||||
public DatacenterTableFieldsController(DatacenterTableFieldService service) {
|
|
||||||
super(service);
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
protected Result onSaveOrUpdateBefore(DatacenterTableField entity, boolean isSave) {
|
|
||||||
LoginAccount loginUser = SaTokenUtil.getLoginAccount();
|
|
||||||
if (isSave) {
|
|
||||||
commonFiled(entity, loginUser.getId(), loginUser.getTenantId(), loginUser.getDeptId());
|
|
||||||
} else {
|
|
||||||
entity.setModified(new Date());
|
|
||||||
entity.setModifiedBy(loginUser.getId());
|
|
||||||
}
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -13,6 +13,7 @@ import tech.easyflow.ai.easyagentsflow.entity.NodeInfo;
|
|||||||
import tech.easyflow.ai.easyagentsflow.entity.WorkflowCheckStage;
|
import tech.easyflow.ai.easyagentsflow.entity.WorkflowCheckStage;
|
||||||
import tech.easyflow.ai.easyagentsflow.service.TinyFlowService;
|
import tech.easyflow.ai.easyagentsflow.service.TinyFlowService;
|
||||||
import tech.easyflow.ai.easyagentsflow.service.WorkflowCheckService;
|
import tech.easyflow.ai.easyagentsflow.service.WorkflowCheckService;
|
||||||
|
import tech.easyflow.ai.easyagentsflow.service.WorkflowDatacenterContentService;
|
||||||
import tech.easyflow.ai.entity.Workflow;
|
import tech.easyflow.ai.entity.Workflow;
|
||||||
import tech.easyflow.ai.service.WorkflowService;
|
import tech.easyflow.ai.service.WorkflowService;
|
||||||
import tech.easyflow.common.domain.Result;
|
import tech.easyflow.common.domain.Result;
|
||||||
@@ -40,6 +41,8 @@ public class PublicWorkflowController {
|
|||||||
private TinyFlowService tinyFlowService;
|
private TinyFlowService tinyFlowService;
|
||||||
@Resource
|
@Resource
|
||||||
private WorkflowCheckService workflowCheckService;
|
private WorkflowCheckService workflowCheckService;
|
||||||
|
@Resource
|
||||||
|
private WorkflowDatacenterContentService workflowDatacenterContentService;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 通过id或别名获取工作流详情
|
* 通过id或别名获取工作流详情
|
||||||
@@ -123,7 +126,7 @@ public class PublicWorkflowController {
|
|||||||
}
|
}
|
||||||
workflowCheckService.checkOrThrow(workflow.getContent(), WorkflowCheckStage.PRE_EXECUTE, workflow.getId());
|
workflowCheckService.checkOrThrow(workflow.getContent(), WorkflowCheckStage.PRE_EXECUTE, workflow.getId());
|
||||||
|
|
||||||
ChainDefinition definition = chainParser.parse(workflow.getContent());
|
ChainDefinition definition = chainParser.parse(workflowDatacenterContentService.prepareContent(workflow.getContent()));
|
||||||
if (definition == null) {
|
if (definition == null) {
|
||||||
return Result.fail(2, "节点配置错误,请检查! ");
|
return Result.fail(2, "节点配置错误,请检查! ");
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -13,6 +13,7 @@ import tech.easyflow.ai.permission.WorkflowVisibilityQueryHelper;
|
|||||||
import tech.easyflow.ai.easyagentsflow.entity.ChainInfo;
|
import tech.easyflow.ai.easyagentsflow.entity.ChainInfo;
|
||||||
import tech.easyflow.ai.easyagentsflow.entity.NodeInfo;
|
import tech.easyflow.ai.easyagentsflow.entity.NodeInfo;
|
||||||
import tech.easyflow.ai.easyagentsflow.entity.WorkflowCheckStage;
|
import tech.easyflow.ai.easyagentsflow.entity.WorkflowCheckStage;
|
||||||
|
import tech.easyflow.ai.easyagentsflow.service.WorkflowDatacenterContentService;
|
||||||
import tech.easyflow.ai.easyagentsflow.service.TinyFlowService;
|
import tech.easyflow.ai.easyagentsflow.service.TinyFlowService;
|
||||||
import tech.easyflow.ai.easyagentsflow.service.WorkflowCheckService;
|
import tech.easyflow.ai.easyagentsflow.service.WorkflowCheckService;
|
||||||
import tech.easyflow.ai.entity.Workflow;
|
import tech.easyflow.ai.entity.Workflow;
|
||||||
@@ -53,6 +54,8 @@ public class UcWorkflowController extends BaseCurdController<WorkflowService, Wo
|
|||||||
@Resource
|
@Resource
|
||||||
private WorkflowCheckService workflowCheckService;
|
private WorkflowCheckService workflowCheckService;
|
||||||
@Resource
|
@Resource
|
||||||
|
private WorkflowDatacenterContentService workflowDatacenterContentService;
|
||||||
|
@Resource
|
||||||
private WorkflowVisibilityQueryHelper workflowVisibilityQueryHelper;
|
private WorkflowVisibilityQueryHelper workflowVisibilityQueryHelper;
|
||||||
|
|
||||||
public UcWorkflowController(WorkflowService service) {
|
public UcWorkflowController(WorkflowService service) {
|
||||||
@@ -168,7 +171,7 @@ public class UcWorkflowController extends BaseCurdController<WorkflowService, Wo
|
|||||||
}
|
}
|
||||||
workflowCheckService.checkOrThrow(workflow.getContent(), WorkflowCheckStage.PRE_EXECUTE, workflow.getId());
|
workflowCheckService.checkOrThrow(workflow.getContent(), WorkflowCheckStage.PRE_EXECUTE, workflow.getId());
|
||||||
|
|
||||||
ChainDefinition definition = chainParser.parse(workflow.getContent());
|
ChainDefinition definition = chainParser.parse(workflowDatacenterContentService.prepareContent(workflow.getContent()));
|
||||||
if (definition == null) {
|
if (definition == null) {
|
||||||
return Result.fail(2, "节点配置错误,请检查! ");
|
return Result.fail(2, "节点配置错误,请检查! ");
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -4,6 +4,7 @@ import com.easyagents.flow.core.chain.ChainDefinition;
|
|||||||
import com.easyagents.flow.core.chain.repository.ChainDefinitionRepository;
|
import com.easyagents.flow.core.chain.repository.ChainDefinitionRepository;
|
||||||
import com.easyagents.flow.core.parser.ChainParser;
|
import com.easyagents.flow.core.parser.ChainParser;
|
||||||
import org.springframework.stereotype.Component;
|
import org.springframework.stereotype.Component;
|
||||||
|
import tech.easyflow.ai.easyagentsflow.service.WorkflowDatacenterContentService;
|
||||||
import tech.easyflow.ai.entity.Workflow;
|
import tech.easyflow.ai.entity.Workflow;
|
||||||
import tech.easyflow.ai.service.WorkflowService;
|
import tech.easyflow.ai.service.WorkflowService;
|
||||||
|
|
||||||
@@ -16,11 +17,13 @@ public class ChainDefinitionRepositoryImpl implements ChainDefinitionRepository
|
|||||||
private WorkflowService workflowService;
|
private WorkflowService workflowService;
|
||||||
@Resource
|
@Resource
|
||||||
private ChainParser chainParser;
|
private ChainParser chainParser;
|
||||||
|
@Resource
|
||||||
|
private WorkflowDatacenterContentService workflowDatacenterContentService;
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public ChainDefinition getChainDefinitionById(String id) {
|
public ChainDefinition getChainDefinitionById(String id) {
|
||||||
Workflow workflow = workflowService.getById(id);
|
Workflow workflow = workflowService.getById(id);
|
||||||
String json = workflow.getContent();
|
String json = workflowDatacenterContentService.prepareContent(workflow.getContent());
|
||||||
ChainDefinition chainDefinition = chainParser.parse(json);
|
ChainDefinition chainDefinition = chainParser.parse(json);
|
||||||
chainDefinition.setId(workflow.getId().toString());
|
chainDefinition.setId(workflow.getId().toString());
|
||||||
chainDefinition.setName(workflow.getEnglishName());
|
chainDefinition.setName(workflow.getEnglishName());
|
||||||
|
|||||||
@@ -65,14 +65,12 @@ public class TinyFlowConfigService {
|
|||||||
MakeFileNodeParser makeFileNodeParser = new MakeFileNodeParser();
|
MakeFileNodeParser makeFileNodeParser = new MakeFileNodeParser();
|
||||||
// 插件
|
// 插件
|
||||||
PluginToolNodeParser pluginToolNodeParser = new PluginToolNodeParser();
|
PluginToolNodeParser pluginToolNodeParser = new PluginToolNodeParser();
|
||||||
// SQL查询
|
|
||||||
SqlNodeParser sqlNodeParser = new SqlNodeParser();
|
|
||||||
// 下载文件节点
|
// 下载文件节点
|
||||||
DownloadNodeParser downloadNodeParser = new DownloadNodeParser();
|
DownloadNodeParser downloadNodeParser = new DownloadNodeParser();
|
||||||
// 保存数据节点
|
// 保存数据节点
|
||||||
SaveToDatacenterNodeParser saveDaveParser = new SaveToDatacenterNodeParser();
|
SaveDatasetNodeParser saveDatasetNodeParser = new SaveDatasetNodeParser();
|
||||||
// 查询数据节点
|
// 查询数据节点
|
||||||
SearchDatacenterNodeParser searchDatacenterNodeParser = new SearchDatacenterNodeParser();
|
SearchDatasetNodeParser searchDatasetNodeParser = new SearchDatasetNodeParser();
|
||||||
// 工作流节点
|
// 工作流节点
|
||||||
WorkflowNodeParser workflowNodeParser = new WorkflowNodeParser();
|
WorkflowNodeParser workflowNodeParser = new WorkflowNodeParser();
|
||||||
// 条件判断节点
|
// 条件判断节点
|
||||||
@@ -81,10 +79,9 @@ public class TinyFlowConfigService {
|
|||||||
chainParser.addNodeParser(docNodeParser.getNodeName(), docNodeParser);
|
chainParser.addNodeParser(docNodeParser.getNodeName(), docNodeParser);
|
||||||
chainParser.addNodeParser(makeFileNodeParser.getNodeName(), makeFileNodeParser);
|
chainParser.addNodeParser(makeFileNodeParser.getNodeName(), makeFileNodeParser);
|
||||||
chainParser.addNodeParser(pluginToolNodeParser.getNodeName(), pluginToolNodeParser);
|
chainParser.addNodeParser(pluginToolNodeParser.getNodeName(), pluginToolNodeParser);
|
||||||
chainParser.addNodeParser(sqlNodeParser.getNodeName(), sqlNodeParser);
|
|
||||||
chainParser.addNodeParser(downloadNodeParser.getNodeName(), downloadNodeParser);
|
chainParser.addNodeParser(downloadNodeParser.getNodeName(), downloadNodeParser);
|
||||||
chainParser.addNodeParser(saveDaveParser.getNodeName(), saveDaveParser);
|
chainParser.addNodeParser(saveDatasetNodeParser.getNodeName(), saveDatasetNodeParser);
|
||||||
chainParser.addNodeParser(searchDatacenterNodeParser.getNodeName(), searchDatacenterNodeParser);
|
chainParser.addNodeParser(searchDatasetNodeParser.getNodeName(), searchDatasetNodeParser);
|
||||||
chainParser.addNodeParser(workflowNodeParser.getNodeName(), workflowNodeParser);
|
chainParser.addNodeParser(workflowNodeParser.getNodeName(), workflowNodeParser);
|
||||||
chainParser.addNodeParser(conditionNodeParser.getNodeName(), conditionNodeParser);
|
chainParser.addNodeParser(conditionNodeParser.getNodeName(), conditionNodeParser);
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -12,15 +12,30 @@ import tech.easyflow.ai.easyagentsflow.entity.WorkflowCheckStage;
|
|||||||
import tech.easyflow.ai.entity.Workflow;
|
import tech.easyflow.ai.entity.Workflow;
|
||||||
import tech.easyflow.ai.service.WorkflowService;
|
import tech.easyflow.ai.service.WorkflowService;
|
||||||
import tech.easyflow.common.web.exceptions.BusinessException;
|
import tech.easyflow.common.web.exceptions.BusinessException;
|
||||||
|
import tech.easyflow.datacenter.execution.model.DatacenterSchemaResponse;
|
||||||
|
|
||||||
import javax.annotation.Resource;
|
import javax.annotation.Resource;
|
||||||
import java.math.BigInteger;
|
import java.math.BigInteger;
|
||||||
import java.util.*;
|
import java.util.ArrayDeque;
|
||||||
|
import java.util.ArrayList;
|
||||||
|
import java.util.Arrays;
|
||||||
|
import java.util.Collection;
|
||||||
|
import java.util.Collections;
|
||||||
|
import java.util.Deque;
|
||||||
|
import java.util.HashMap;
|
||||||
|
import java.util.HashSet;
|
||||||
|
import java.util.LinkedHashMap;
|
||||||
|
import java.util.LinkedHashSet;
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.Map;
|
||||||
|
import java.util.Objects;
|
||||||
|
import java.util.Set;
|
||||||
import java.util.stream.Collectors;
|
import java.util.stream.Collectors;
|
||||||
|
|
||||||
@Service
|
@Service
|
||||||
public class WorkflowCheckService {
|
public class WorkflowCheckService {
|
||||||
private static final String LEVEL_ERROR = "ERROR";
|
private static final String LEVEL_ERROR = "ERROR";
|
||||||
|
private static final String LEVEL_WARNING = "WARNING";
|
||||||
private static final String TYPE_START = "startNode";
|
private static final String TYPE_START = "startNode";
|
||||||
private static final String TYPE_END = "endNode";
|
private static final String TYPE_END = "endNode";
|
||||||
private static final String TYPE_LOOP = "loopNode";
|
private static final String TYPE_LOOP = "loopNode";
|
||||||
@@ -30,6 +45,8 @@ public class WorkflowCheckService {
|
|||||||
private WorkflowService workflowService;
|
private WorkflowService workflowService;
|
||||||
@Resource
|
@Resource
|
||||||
private ChainParser chainParser;
|
private ChainParser chainParser;
|
||||||
|
@Resource
|
||||||
|
private WorkflowDatacenterContentService workflowDatacenterContentService;
|
||||||
|
|
||||||
public WorkflowCheckResult checkWorkflow(BigInteger workflowId, WorkflowCheckStage stage) {
|
public WorkflowCheckResult checkWorkflow(BigInteger workflowId, WorkflowCheckStage stage) {
|
||||||
if (workflowId == null) {
|
if (workflowId == null) {
|
||||||
@@ -177,9 +194,91 @@ public class WorkflowCheckService {
|
|||||||
parsedWorkflow.nodes = nodes;
|
parsedWorkflow.nodes = nodes;
|
||||||
parsedWorkflow.edges = edges;
|
parsedWorkflow.edges = edges;
|
||||||
parsedWorkflow.nodeMap = nodeMap;
|
parsedWorkflow.nodeMap = nodeMap;
|
||||||
|
checkDatacenterNodes(parsedWorkflow, issues, issueKeys);
|
||||||
return parsedWorkflow;
|
return parsedWorkflow;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private void checkDatacenterNodes(ParsedWorkflow parsed, List<WorkflowCheckIssue> issues, Set<String> issueKeys) {
|
||||||
|
for (NodeView node : parsed.nodes) {
|
||||||
|
if (node == null) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if (workflowDatacenterContentService.isSearchDatasetNode(node.type)) {
|
||||||
|
checkSearchDatasetNode(node, issues, issueKeys);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if (workflowDatacenterContentService.isSaveDatasetNode(node.type)) {
|
||||||
|
checkSaveDatasetNode(node, issues, issueKeys);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if (workflowDatacenterContentService.isLlmNode(node.type)) {
|
||||||
|
checkLlmQueryContext(node, parsed, issues, issueKeys);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private void checkSearchDatasetNode(NodeView node,
|
||||||
|
List<WorkflowCheckIssue> issues,
|
||||||
|
Set<String> issueKeys) {
|
||||||
|
try {
|
||||||
|
workflowDatacenterContentService.requireSearchDatasetRef(node.data);
|
||||||
|
} catch (BusinessException e) {
|
||||||
|
addIssue(issues, issueKeys, "SEARCH_DATASET_INVALID", e.getMessage(), node.id, null, node.name);
|
||||||
|
} catch (Exception e) {
|
||||||
|
addIssue(issues, issueKeys, "SEARCH_DATASET_INVALID",
|
||||||
|
"查询数据节点校验失败: " + shortError(e), node.id, null, node.name);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private void checkSaveDatasetNode(NodeView node,
|
||||||
|
List<WorkflowCheckIssue> issues,
|
||||||
|
Set<String> issueKeys) {
|
||||||
|
try {
|
||||||
|
workflowDatacenterContentService.requireSaveDatasetRef(node.data);
|
||||||
|
} catch (BusinessException e) {
|
||||||
|
addIssue(issues, issueKeys, "SAVE_DATASET_INVALID", e.getMessage(), node.id, null, node.name);
|
||||||
|
} catch (Exception e) {
|
||||||
|
addIssue(issues, issueKeys, "SAVE_DATASET_INVALID",
|
||||||
|
"写入数据节点校验失败: " + shortError(e), node.id, null, node.name);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private void checkLlmQueryContext(NodeView node,
|
||||||
|
ParsedWorkflow parsed,
|
||||||
|
List<WorkflowCheckIssue> issues,
|
||||||
|
Set<String> issueKeys) {
|
||||||
|
if (node.data == null) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
Object rawNodeIds = node.data.get("queryContextNodeIds");
|
||||||
|
if (rawNodeIds == null) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (!(rawNodeIds instanceof JSONArray nodeIds)) {
|
||||||
|
addIssue(issues, issueKeys, "LLM_QUERY_CONTEXT_INVALID",
|
||||||
|
"查询上下文配置无效,请重新选择查询数据节点", node.id, null, node.name);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
for (int i = 0; i < nodeIds.size(); i++) {
|
||||||
|
String refNodeId = trimToNull(nodeIds.getString(i));
|
||||||
|
if (!StringUtils.hasText(refNodeId)) {
|
||||||
|
addIssue(issues, issueKeys, "LLM_QUERY_CONTEXT_EMPTY",
|
||||||
|
"查询上下文配置无效,请重新选择查询数据节点", node.id, null, node.name);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
NodeView refNode = parsed.nodeMap.get(refNodeId);
|
||||||
|
if (refNode == null) {
|
||||||
|
addIssue(issues, issueKeys, "LLM_QUERY_CONTEXT_NOT_FOUND",
|
||||||
|
"查询上下文节点不存在: " + refNodeId, node.id, null, node.name);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if (!workflowDatacenterContentService.isSearchDatasetNode(refNode.type)) {
|
||||||
|
addIssue(issues, issueKeys, "LLM_QUERY_CONTEXT_TYPE_INVALID",
|
||||||
|
"查询上下文只能选择查询数据节点", node.id, null, node.name);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
private void runStrictChecks(String content, ParsedWorkflow parsed, BigInteger currentWorkflowId,
|
private void runStrictChecks(String content, ParsedWorkflow parsed, BigInteger currentWorkflowId,
|
||||||
List<WorkflowCheckIssue> issues, Set<String> issueKeys) {
|
List<WorkflowCheckIssue> issues, Set<String> issueKeys) {
|
||||||
if (parsed.nodes.isEmpty()) {
|
if (parsed.nodes.isEmpty()) {
|
||||||
@@ -190,7 +289,8 @@ public class WorkflowCheckService {
|
|||||||
}
|
}
|
||||||
|
|
||||||
try {
|
try {
|
||||||
Object definition = chainParser.parse(content);
|
String preparedContent = workflowDatacenterContentService.prepareContent(content);
|
||||||
|
Object definition = chainParser.parse(preparedContent);
|
||||||
if (definition == null) {
|
if (definition == null) {
|
||||||
addIssue(issues, issueKeys, "PARSE_NULL", "预执行校验失败:节点配置错误,请检查", null, null, null);
|
addIssue(issues, issueKeys, "PARSE_NULL", "预执行校验失败:节点配置错误,请检查", null, null, null);
|
||||||
}
|
}
|
||||||
@@ -606,31 +706,47 @@ public class WorkflowCheckService {
|
|||||||
result.setStage(stage);
|
result.setStage(stage);
|
||||||
result.setIssues(issues);
|
result.setIssues(issues);
|
||||||
result.setIssueCount(issues.size());
|
result.setIssueCount(issues.size());
|
||||||
result.setPassed(issues.isEmpty());
|
result.setPassed(issues.stream().noneMatch(issue -> LEVEL_ERROR.equalsIgnoreCase(issue.getLevel())));
|
||||||
return result;
|
return result;
|
||||||
}
|
}
|
||||||
|
|
||||||
private void throwIfFailed(WorkflowCheckResult result) {
|
private void throwIfFailed(WorkflowCheckResult result) {
|
||||||
if (result == null || result.isPassed()) {
|
if (result == null) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
String summary = result.getIssues().stream()
|
List<WorkflowCheckIssue> errorIssues = result.getIssues().stream()
|
||||||
|
.filter(issue -> LEVEL_ERROR.equalsIgnoreCase(issue.getLevel()))
|
||||||
|
.collect(Collectors.toList());
|
||||||
|
if (errorIssues.isEmpty()) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
String summary = errorIssues.stream()
|
||||||
.limit(5)
|
.limit(5)
|
||||||
.map(WorkflowCheckIssue::getMessage)
|
.map(WorkflowCheckIssue::getMessage)
|
||||||
.collect(Collectors.joining(";"));
|
.collect(Collectors.joining(";"));
|
||||||
if (result.getIssueCount() > 5) {
|
if (errorIssues.size() > 5) {
|
||||||
summary = summary + ";等";
|
summary = summary + ";等";
|
||||||
}
|
}
|
||||||
throw new BusinessException("工作流校验未通过(" + result.getStage() + "),共 " + result.getIssueCount() + " 项:" + summary);
|
throw new BusinessException("工作流校验未通过(" + result.getStage() + "),共 " + errorIssues.size() + " 项:" + summary);
|
||||||
}
|
}
|
||||||
|
|
||||||
private void addIssue(List<WorkflowCheckIssue> issues, Set<String> issueKeys, String code,
|
private void addIssue(List<WorkflowCheckIssue> issues, Set<String> issueKeys, String code,
|
||||||
String message, String nodeId, String edgeId, String nodeName) {
|
String message, String nodeId, String edgeId, String nodeName) {
|
||||||
|
addIssue(issues, issueKeys, code, LEVEL_ERROR, message, nodeId, edgeId, nodeName);
|
||||||
|
}
|
||||||
|
|
||||||
|
private void addWarning(List<WorkflowCheckIssue> issues, Set<String> issueKeys, String code,
|
||||||
|
String message, String nodeId, String edgeId, String nodeName) {
|
||||||
|
addIssue(issues, issueKeys, code, LEVEL_WARNING, message, nodeId, edgeId, nodeName);
|
||||||
|
}
|
||||||
|
|
||||||
|
private void addIssue(List<WorkflowCheckIssue> issues, Set<String> issueKeys, String code, String level,
|
||||||
|
String message, String nodeId, String edgeId, String nodeName) {
|
||||||
String key = code + "|" + safe(nodeId) + "|" + safe(edgeId) + "|" + message;
|
String key = code + "|" + safe(nodeId) + "|" + safe(edgeId) + "|" + message;
|
||||||
if (!issueKeys.add(key)) {
|
if (!issueKeys.add(key)) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
issues.add(new WorkflowCheckIssue(code, LEVEL_ERROR, message, nodeId, edgeId, nodeName));
|
issues.add(new WorkflowCheckIssue(code, level, message, nodeId, edgeId, nodeName));
|
||||||
}
|
}
|
||||||
|
|
||||||
private String extractNodeName(JSONObject nodeJson, JSONObject data, String fallback) {
|
private String extractNodeName(JSONObject nodeJson, JSONObject data, String fallback) {
|
||||||
|
|||||||
@@ -0,0 +1,349 @@
|
|||||||
|
package tech.easyflow.ai.easyagentsflow.service;
|
||||||
|
|
||||||
|
import com.alibaba.fastjson2.JSON;
|
||||||
|
import com.alibaba.fastjson2.JSONArray;
|
||||||
|
import com.alibaba.fastjson2.JSONObject;
|
||||||
|
import org.springframework.stereotype.Service;
|
||||||
|
import org.springframework.util.StringUtils;
|
||||||
|
import tech.easyflow.common.constant.enums.EnumFieldType;
|
||||||
|
import tech.easyflow.common.web.exceptions.BusinessException;
|
||||||
|
import tech.easyflow.datacenter.entity.DatacenterTable;
|
||||||
|
import tech.easyflow.datacenter.entity.DatacenterTableField;
|
||||||
|
import tech.easyflow.datacenter.execution.model.DatacenterSchemaResponse;
|
||||||
|
import tech.easyflow.datacenter.execution.model.DatasetRef;
|
||||||
|
import tech.easyflow.datacenter.execution.service.DatacenterDatasetQueryService;
|
||||||
|
import tech.easyflow.datacenter.meta.entity.DatacenterCatalog;
|
||||||
|
import tech.easyflow.datacenter.meta.entity.DatacenterSource;
|
||||||
|
import tech.easyflow.datacenter.meta.service.DatacenterDatasetRegistryService;
|
||||||
|
|
||||||
|
import javax.annotation.Resource;
|
||||||
|
import java.math.BigInteger;
|
||||||
|
import java.util.ArrayList;
|
||||||
|
import java.util.Comparator;
|
||||||
|
import java.util.LinkedHashMap;
|
||||||
|
import java.util.LinkedHashSet;
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.Map;
|
||||||
|
import java.util.Set;
|
||||||
|
|
||||||
|
@Service
|
||||||
|
public class WorkflowDatacenterContentService {
|
||||||
|
|
||||||
|
public static final String SEARCH_NODE_TYPE = "search-dataset-node";
|
||||||
|
public static final String SAVE_NODE_TYPE = "save-dataset-node";
|
||||||
|
public static final String LLM_NODE_TYPE = "llmNode";
|
||||||
|
public static final String QUERY_DATA_CONTEXT = "queryDataContext";
|
||||||
|
public static final String SEARCH_SOURCE_MISSING_MESSAGE = "查询数据节点未选择连接服务";
|
||||||
|
public static final String SEARCH_SQL_MISSING_MESSAGE = "查询数据节点未设置 SQL";
|
||||||
|
public static final String SAVE_EXPIRED_MESSAGE = "写入数据节点配置已过期,请重新选择已接入表";
|
||||||
|
public static final String INVALID_QUERY_CONTEXT_MESSAGE = "查询上下文配置无效,请重新选择查询数据节点";
|
||||||
|
private static final String QUERY_CONTEXT_PROMPT = """
|
||||||
|
你是为工作流中的查询数据节点生成只读 SQL 的生成器,你的职责是返回可直接执行的 SQL,并且你只能输出 SQL。
|
||||||
|
|
||||||
|
必须严格遵守以下规则:
|
||||||
|
1. 只能从下面给出的连接摘要中选择最合适的表。
|
||||||
|
2. 只能使用摘要中给出的字段名,不要虚构表名或字段名。
|
||||||
|
3. 只输出 SQL,不要输出解释、注释、Markdown、JSON 或多余文本。
|
||||||
|
4. 只能生成只读 SELECT SQL,允许 WITH、JOIN、子查询、聚合、分组、排序。
|
||||||
|
5. 不要生成 INSERT、UPDATE、DELETE、DDL、多语句、存储过程调用。
|
||||||
|
6. 优先使用逻辑表名和逻辑字段名,不要输出物理表名、JDBC、驱动信息。
|
||||||
|
7. 如果存在重名表,请使用 catalog.table 形式消除歧义。
|
||||||
|
|
||||||
|
以下是可用的连接摘要:
|
||||||
|
""";
|
||||||
|
|
||||||
|
@Resource
|
||||||
|
private DatacenterDatasetQueryService queryService;
|
||||||
|
@Resource
|
||||||
|
private DatacenterDatasetRegistryService registryService;
|
||||||
|
|
||||||
|
public boolean isSearchDatasetNode(String nodeType) {
|
||||||
|
return SEARCH_NODE_TYPE.equals(nodeType);
|
||||||
|
}
|
||||||
|
|
||||||
|
public boolean isSaveDatasetNode(String nodeType) {
|
||||||
|
return SAVE_NODE_TYPE.equals(nodeType);
|
||||||
|
}
|
||||||
|
|
||||||
|
public boolean isLlmNode(String nodeType) {
|
||||||
|
return LLM_NODE_TYPE.equals(nodeType);
|
||||||
|
}
|
||||||
|
|
||||||
|
public String prepareContent(String content) {
|
||||||
|
if (!StringUtils.hasText(content)) {
|
||||||
|
throw new BusinessException("工作流内容不能为空");
|
||||||
|
}
|
||||||
|
Object parsed;
|
||||||
|
try {
|
||||||
|
parsed = JSON.parse(content);
|
||||||
|
} catch (Exception e) {
|
||||||
|
throw new BusinessException("工作流内容不是合法JSON: " + shortError(e));
|
||||||
|
}
|
||||||
|
if (!(parsed instanceof JSONObject root)) {
|
||||||
|
throw new BusinessException("工作流内容必须是JSON对象");
|
||||||
|
}
|
||||||
|
JSONObject copy = JSON.parseObject(root.toJSONString());
|
||||||
|
prepareRoot(copy);
|
||||||
|
return copy.toJSONString();
|
||||||
|
}
|
||||||
|
|
||||||
|
public JSONObject prepareRoot(JSONObject root) {
|
||||||
|
if (root == null) {
|
||||||
|
throw new BusinessException("工作流内容不能为空");
|
||||||
|
}
|
||||||
|
JSONArray nodes = root.getJSONArray("nodes");
|
||||||
|
if (nodes == null || nodes.isEmpty()) {
|
||||||
|
return root;
|
||||||
|
}
|
||||||
|
Map<String, JSONObject> nodeMap = buildNodeMap(nodes);
|
||||||
|
for (int i = 0; i < nodes.size(); i++) {
|
||||||
|
JSONObject node = nodes.getJSONObject(i);
|
||||||
|
if (node == null) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
String nodeType = node.getString("type");
|
||||||
|
JSONObject data = node.getJSONObject("data");
|
||||||
|
if (isSearchDatasetNode(nodeType)) {
|
||||||
|
requireSearchDatasetRef(data);
|
||||||
|
} else if (isSaveDatasetNode(nodeType)) {
|
||||||
|
requireSaveDatasetRef(data);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for (int i = 0; i < nodes.size(); i++) {
|
||||||
|
JSONObject node = nodes.getJSONObject(i);
|
||||||
|
if (node == null || !isLlmNode(node.getString("type"))) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
injectQueryDataContext(node.getJSONObject("data"), nodeMap);
|
||||||
|
}
|
||||||
|
return root;
|
||||||
|
}
|
||||||
|
|
||||||
|
public boolean hasSaveLegacyFields(JSONObject data) {
|
||||||
|
return hasAnyField(data, "tableId", "sourceId", "catalogId", "versionId");
|
||||||
|
}
|
||||||
|
|
||||||
|
public DatasetRef readDatasetRef(JSONObject data) {
|
||||||
|
return data == null ? null : data.getObject("datasetRef", DatasetRef.class);
|
||||||
|
}
|
||||||
|
|
||||||
|
public DatasetRef requireSearchDatasetRef(JSONObject data) {
|
||||||
|
DatasetRef datasetRef = readDatasetRef(data);
|
||||||
|
if (datasetRef == null || datasetRef.getSourceId() == null) {
|
||||||
|
throw new BusinessException(SEARCH_SOURCE_MISSING_MESSAGE);
|
||||||
|
}
|
||||||
|
String querySql = data == null ? null : trimToNull(data.getString("querySql"));
|
||||||
|
if (!StringUtils.hasText(querySql)) {
|
||||||
|
throw new BusinessException(SEARCH_SQL_MISSING_MESSAGE);
|
||||||
|
}
|
||||||
|
return datasetRef;
|
||||||
|
}
|
||||||
|
|
||||||
|
public DatasetRef requireSaveDatasetRef(JSONObject data) {
|
||||||
|
if (hasSaveLegacyFields(data)) {
|
||||||
|
throw new BusinessException(SAVE_EXPIRED_MESSAGE);
|
||||||
|
}
|
||||||
|
DatasetRef datasetRef = readDatasetRef(data);
|
||||||
|
if (datasetRef == null || datasetRef.getTableId() == null) {
|
||||||
|
throw new BusinessException(SAVE_EXPIRED_MESSAGE);
|
||||||
|
}
|
||||||
|
return datasetRef;
|
||||||
|
}
|
||||||
|
|
||||||
|
public DatacenterSchemaResponse loadSchema(DatasetRef datasetRef, Map<BigInteger, DatacenterSchemaResponse> schemaCache) {
|
||||||
|
if (datasetRef == null || datasetRef.getTableId() == null) {
|
||||||
|
throw new BusinessException("缺少已接入表配置");
|
||||||
|
}
|
||||||
|
BigInteger tableId = datasetRef.getTableId();
|
||||||
|
if (schemaCache != null && schemaCache.containsKey(tableId)) {
|
||||||
|
return schemaCache.get(tableId);
|
||||||
|
}
|
||||||
|
DatacenterSchemaResponse schema = queryService.getSchema(copyDatasetRef(datasetRef));
|
||||||
|
if (schemaCache != null) {
|
||||||
|
schemaCache.put(tableId, schema);
|
||||||
|
}
|
||||||
|
return schema;
|
||||||
|
}
|
||||||
|
|
||||||
|
public JSONObject buildSourceSummary(DatasetRef datasetRef) {
|
||||||
|
if (datasetRef == null || datasetRef.getSourceId() == null) {
|
||||||
|
throw new BusinessException(SEARCH_SOURCE_MISSING_MESSAGE);
|
||||||
|
}
|
||||||
|
DatacenterSource source = registryService.getSourceRequired(datasetRef.getSourceId());
|
||||||
|
List<DatacenterTable> managedTables = registryService.listManagedTables(datasetRef.getSourceId(), datasetRef.getCatalogId());
|
||||||
|
managedTables.sort(Comparator.comparing(table -> table.getTableName() == null ? "" : table.getTableName()));
|
||||||
|
JSONObject sourceSummary = new JSONObject();
|
||||||
|
sourceSummary.put("sourceName", source.getSourceName());
|
||||||
|
sourceSummary.put("sourceType", source.getSourceType());
|
||||||
|
JSONArray tables = new JSONArray();
|
||||||
|
for (DatacenterTable table : managedTables) {
|
||||||
|
DatacenterTable fullTable = registryService.getTableWithFields(table.getId());
|
||||||
|
DatacenterCatalog catalog = registryService.getCatalogById(fullTable.getCatalogId());
|
||||||
|
if (StringUtils.hasText(datasetRef.getCatalogName())
|
||||||
|
&& (catalog == null || !datasetRef.getCatalogName().equals(catalog.getCatalogName()))) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
JSONObject tableSummary = new JSONObject();
|
||||||
|
tableSummary.put("catalogName", catalog == null ? null : catalog.getCatalogName());
|
||||||
|
tableSummary.put("tableName", fullTable.getTableName());
|
||||||
|
tableSummary.put("tableDesc", fullTable.getTableDesc());
|
||||||
|
JSONArray fields = new JSONArray();
|
||||||
|
if (fullTable.getFields() != null) {
|
||||||
|
for (DatacenterTableField field : fullTable.getFields()) {
|
||||||
|
JSONObject fieldSummary = new JSONObject();
|
||||||
|
fieldSummary.put("fieldName", field.getFieldName());
|
||||||
|
fieldSummary.put("fieldDesc", field.getFieldDesc());
|
||||||
|
fieldSummary.put("fieldType", resolveFieldType(field));
|
||||||
|
fields.add(fieldSummary);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
tableSummary.put("fields", fields);
|
||||||
|
tables.add(tableSummary);
|
||||||
|
}
|
||||||
|
sourceSummary.put("tables", tables);
|
||||||
|
return sourceSummary;
|
||||||
|
}
|
||||||
|
|
||||||
|
private void injectQueryDataContext(JSONObject data, Map<String, JSONObject> nodeMap) {
|
||||||
|
if (data == null) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
JSONArray nodeIds = data.getJSONArray("queryContextNodeIds");
|
||||||
|
if (nodeIds == null || nodeIds.isEmpty()) {
|
||||||
|
removeQueryDataContextParameter(data);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
Map<BigInteger, JSONObject> sourceSummaries = new LinkedHashMap<>();
|
||||||
|
Set<String> visitedNodeIds = new LinkedHashSet<>();
|
||||||
|
for (int i = 0; i < nodeIds.size(); i++) {
|
||||||
|
String nodeId = trimToNull(nodeIds.getString(i));
|
||||||
|
if (!StringUtils.hasText(nodeId) || !visitedNodeIds.add(nodeId)) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
JSONObject targetNode = nodeMap.get(nodeId);
|
||||||
|
if (targetNode == null || !isSearchDatasetNode(targetNode.getString("type"))) {
|
||||||
|
throw new BusinessException(INVALID_QUERY_CONTEXT_MESSAGE);
|
||||||
|
}
|
||||||
|
DatasetRef datasetRef = requireSearchDatasetRef(targetNode.getJSONObject("data"));
|
||||||
|
sourceSummaries.putIfAbsent(datasetRef.getSourceId(), buildSourceSummary(datasetRef));
|
||||||
|
}
|
||||||
|
String contextValue = QUERY_CONTEXT_PROMPT + "\n" + JSON.toJSONString(new ArrayList<>(sourceSummaries.values()));
|
||||||
|
upsertQueryDataContextParameter(data, contextValue);
|
||||||
|
}
|
||||||
|
|
||||||
|
private String resolveFieldType(DatacenterTableField field) {
|
||||||
|
if (field == null) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
if (StringUtils.hasText(field.getJdbcType())) {
|
||||||
|
return field.getJdbcType();
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
EnumFieldType enumFieldType = EnumFieldType.getByCode(field.getFieldType());
|
||||||
|
if (enumFieldType != null) {
|
||||||
|
return enumFieldType.getText();
|
||||||
|
}
|
||||||
|
} catch (Exception ignored) {
|
||||||
|
}
|
||||||
|
return field.getFieldType() == null ? null : String.valueOf(field.getFieldType());
|
||||||
|
}
|
||||||
|
|
||||||
|
private void upsertQueryDataContextParameter(JSONObject data, String contextValue) {
|
||||||
|
JSONArray parameters = data.getJSONArray("parameters");
|
||||||
|
if (parameters == null) {
|
||||||
|
parameters = new JSONArray();
|
||||||
|
data.put("parameters", parameters);
|
||||||
|
}
|
||||||
|
for (int i = parameters.size() - 1; i >= 0; i--) {
|
||||||
|
JSONObject parameter = parameters.getJSONObject(i);
|
||||||
|
if (parameter != null && QUERY_DATA_CONTEXT.equals(parameter.getString("name"))) {
|
||||||
|
parameters.remove(i);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
JSONObject parameter = new JSONObject();
|
||||||
|
parameter.put("id", QUERY_DATA_CONTEXT);
|
||||||
|
parameter.put("name", QUERY_DATA_CONTEXT);
|
||||||
|
parameter.put("title", "查询上下文");
|
||||||
|
parameter.put("description", "数据查询规则与连接表摘要");
|
||||||
|
parameter.put("dataType", "String");
|
||||||
|
parameter.put("refType", "fixed");
|
||||||
|
parameter.put("value", contextValue);
|
||||||
|
parameter.put("required", false);
|
||||||
|
parameter.put("nameDisabled", true);
|
||||||
|
parameter.put("dataTypeDisabled", true);
|
||||||
|
parameter.put("deleteDisabled", true);
|
||||||
|
parameters.add(parameter);
|
||||||
|
}
|
||||||
|
|
||||||
|
private void removeQueryDataContextParameter(JSONObject data) {
|
||||||
|
if (data == null) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
JSONArray parameters = data.getJSONArray("parameters");
|
||||||
|
if (parameters == null || parameters.isEmpty()) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
for (int i = parameters.size() - 1; i >= 0; i--) {
|
||||||
|
JSONObject parameter = parameters.getJSONObject(i);
|
||||||
|
if (parameter != null && QUERY_DATA_CONTEXT.equals(parameter.getString("name"))) {
|
||||||
|
parameters.remove(i);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private Map<String, JSONObject> buildNodeMap(JSONArray nodes) {
|
||||||
|
Map<String, JSONObject> nodeMap = new LinkedHashMap<>();
|
||||||
|
for (int i = 0; i < nodes.size(); i++) {
|
||||||
|
JSONObject node = nodes.getJSONObject(i);
|
||||||
|
if (node == null) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
String nodeId = trimToNull(node.getString("id"));
|
||||||
|
if (StringUtils.hasText(nodeId)) {
|
||||||
|
nodeMap.put(nodeId, node);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nodeMap;
|
||||||
|
}
|
||||||
|
|
||||||
|
private DatasetRef copyDatasetRef(DatasetRef datasetRef) {
|
||||||
|
DatasetRef copy = new DatasetRef();
|
||||||
|
copy.setSourceId(datasetRef.getSourceId());
|
||||||
|
copy.setCatalogId(datasetRef.getCatalogId());
|
||||||
|
copy.setCatalogName(datasetRef.getCatalogName());
|
||||||
|
copy.setTableId(datasetRef.getTableId());
|
||||||
|
copy.setTableName(datasetRef.getTableName());
|
||||||
|
copy.setVersionId(datasetRef.getVersionId());
|
||||||
|
return copy;
|
||||||
|
}
|
||||||
|
|
||||||
|
private boolean hasAnyField(JSONObject data, String... fieldNames) {
|
||||||
|
if (data == null || fieldNames == null) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
for (String fieldName : fieldNames) {
|
||||||
|
if (data.containsKey(fieldName)) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
private String trimToNull(String value) {
|
||||||
|
if (!StringUtils.hasText(value)) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
return value.trim();
|
||||||
|
}
|
||||||
|
|
||||||
|
private String shortError(Throwable throwable) {
|
||||||
|
if (throwable == null) {
|
||||||
|
return "unknown";
|
||||||
|
}
|
||||||
|
String message = throwable.getMessage();
|
||||||
|
if (!StringUtils.hasText(message)) {
|
||||||
|
return throwable.getClass().getSimpleName();
|
||||||
|
}
|
||||||
|
return message.length() > 120 ? message.substring(0, 120) + "..." : message;
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,75 @@
|
|||||||
|
package tech.easyflow.ai.node;
|
||||||
|
|
||||||
|
import com.alibaba.fastjson2.JSONArray;
|
||||||
|
import com.alibaba.fastjson2.JSONObject;
|
||||||
|
import com.easyagents.flow.core.chain.Chain;
|
||||||
|
import com.easyagents.flow.core.node.BaseNode;
|
||||||
|
import com.mybatisflex.core.tenant.TenantManager;
|
||||||
|
import org.slf4j.Logger;
|
||||||
|
import org.slf4j.LoggerFactory;
|
||||||
|
import tech.easyflow.ai.utils.WorkFlowUtil;
|
||||||
|
import tech.easyflow.common.entity.LoginAccount;
|
||||||
|
import tech.easyflow.common.util.SpringContextUtil;
|
||||||
|
import tech.easyflow.datacenter.execution.model.DatasetRef;
|
||||||
|
import tech.easyflow.datacenter.execution.service.DatacenterDatasetQueryService;
|
||||||
|
import tech.easyflow.datacenter.execution.service.DatacenterDatasetWriteService;
|
||||||
|
|
||||||
|
import java.util.HashMap;
|
||||||
|
import java.util.Map;
|
||||||
|
|
||||||
|
public class SaveDatasetNode extends BaseNode {
|
||||||
|
|
||||||
|
private static final Logger log = LoggerFactory.getLogger(SaveDatasetNode.class);
|
||||||
|
|
||||||
|
private DatasetRef datasetRef;
|
||||||
|
|
||||||
|
public SaveDatasetNode() {
|
||||||
|
}
|
||||||
|
|
||||||
|
public SaveDatasetNode(DatasetRef datasetRef) {
|
||||||
|
this.datasetRef = datasetRef;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public Map<String, Object> execute(Chain chain) {
|
||||||
|
Map<String, Object> state = chain.getState().resolveParameters(this);
|
||||||
|
JSONObject payload = new JSONObject(state);
|
||||||
|
JSONArray saveList = payload.getJSONArray("saveList");
|
||||||
|
if (saveList == null || saveList.isEmpty()) {
|
||||||
|
throw new RuntimeException("saveList 不能为空");
|
||||||
|
}
|
||||||
|
LoginAccount account = WorkFlowUtil.getOperator(chain);
|
||||||
|
DatacenterDatasetWriteService writeService = SpringContextUtil.getBean(DatacenterDatasetWriteService.class);
|
||||||
|
DatacenterDatasetQueryService queryService = SpringContextUtil.getBean(DatacenterDatasetQueryService.class);
|
||||||
|
int successRows = 0;
|
||||||
|
try {
|
||||||
|
TenantManager.ignoreTenantCondition();
|
||||||
|
for (Object item : saveList) {
|
||||||
|
JSONObject row = item instanceof JSONObject json ? json : JSONObject.from(item);
|
||||||
|
writeService.saveRow(datasetRef, row, account);
|
||||||
|
successRows++;
|
||||||
|
}
|
||||||
|
var schema = queryService.getSchema(datasetRef);
|
||||||
|
Map<String, Object> result = new HashMap<>();
|
||||||
|
result.put("successRows", successRows);
|
||||||
|
result.put("source", schema.getSource());
|
||||||
|
result.put("catalog", schema.getCatalog());
|
||||||
|
result.put("table", schema.getTable());
|
||||||
|
result.put("version", datasetRef.getVersionId());
|
||||||
|
return result;
|
||||||
|
} catch (Exception ex) {
|
||||||
|
log.error("工作流保存数据到统一数据集失败,datasetRef={}", datasetRef, ex);
|
||||||
|
throw ex;
|
||||||
|
} finally {
|
||||||
|
TenantManager.restoreTenantCondition();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public DatasetRef getDatasetRef() {
|
||||||
|
return datasetRef;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setDatasetRef(DatasetRef datasetRef) {
|
||||||
|
this.datasetRef = datasetRef;
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,41 @@
|
|||||||
|
package tech.easyflow.ai.node;
|
||||||
|
|
||||||
|
import com.alibaba.fastjson.JSONObject;
|
||||||
|
import com.easyagents.flow.core.node.BaseNode;
|
||||||
|
import com.easyagents.flow.core.parser.BaseNodeParser;
|
||||||
|
import tech.easyflow.common.web.exceptions.BusinessException;
|
||||||
|
import tech.easyflow.datacenter.execution.model.DatasetRef;
|
||||||
|
|
||||||
|
public class SaveDatasetNodeParser extends BaseNodeParser {
|
||||||
|
|
||||||
|
private static final String EXPIRED_MESSAGE = "写入数据节点配置已过期,请重新选择已接入表";
|
||||||
|
|
||||||
|
@Override
|
||||||
|
protected BaseNode doParse(JSONObject root, JSONObject data, JSONObject tinyflow) {
|
||||||
|
if (data == null) {
|
||||||
|
throw new BusinessException(EXPIRED_MESSAGE);
|
||||||
|
}
|
||||||
|
if (hasLegacyFields(data)) {
|
||||||
|
throw new BusinessException(EXPIRED_MESSAGE);
|
||||||
|
}
|
||||||
|
DatasetRef datasetRef = data.getObject("datasetRef", DatasetRef.class);
|
||||||
|
if (datasetRef == null || datasetRef.getTableId() == null) {
|
||||||
|
throw new BusinessException(EXPIRED_MESSAGE);
|
||||||
|
}
|
||||||
|
return new SaveDatasetNode(datasetRef);
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getNodeName() {
|
||||||
|
return "save-dataset-node";
|
||||||
|
}
|
||||||
|
|
||||||
|
private boolean hasLegacyFields(JSONObject data) {
|
||||||
|
if (data == null) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
return data.containsKey("tableId")
|
||||||
|
|| data.containsKey("sourceId")
|
||||||
|
|| data.containsKey("catalogId")
|
||||||
|
|| data.containsKey("versionId");
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -1,75 +0,0 @@
|
|||||||
package tech.easyflow.ai.node;
|
|
||||||
|
|
||||||
import com.alibaba.fastjson2.JSONArray;
|
|
||||||
import com.alibaba.fastjson2.JSONObject;
|
|
||||||
import com.mybatisflex.core.tenant.TenantManager;
|
|
||||||
import com.easyagents.flow.core.chain.Chain;
|
|
||||||
import com.easyagents.flow.core.node.BaseNode;
|
|
||||||
import org.slf4j.Logger;
|
|
||||||
import org.slf4j.LoggerFactory;
|
|
||||||
import tech.easyflow.ai.utils.WorkFlowUtil;
|
|
||||||
import tech.easyflow.common.entity.LoginAccount;
|
|
||||||
import tech.easyflow.common.util.SpringContextUtil;
|
|
||||||
import tech.easyflow.datacenter.service.DatacenterTableService;
|
|
||||||
|
|
||||||
import java.math.BigInteger;
|
|
||||||
import java.util.HashMap;
|
|
||||||
import java.util.Map;
|
|
||||||
|
|
||||||
public class SaveToDatacenterNode extends BaseNode {
|
|
||||||
|
|
||||||
private static final Logger log = LoggerFactory.getLogger(SaveToDatacenterNode.class);
|
|
||||||
|
|
||||||
private BigInteger tableId;
|
|
||||||
|
|
||||||
public SaveToDatacenterNode() {
|
|
||||||
}
|
|
||||||
|
|
||||||
public SaveToDatacenterNode(BigInteger tableId) {
|
|
||||||
this.tableId = tableId;
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public Map<String, Object> execute(Chain chain) {
|
|
||||||
|
|
||||||
Map<String, Object> map = chain.getState().resolveParameters(this);
|
|
||||||
JSONObject json = new JSONObject(map);
|
|
||||||
|
|
||||||
Map<String, Object> res = new HashMap<>();
|
|
||||||
|
|
||||||
// 默认为未知来源
|
|
||||||
LoginAccount account = WorkFlowUtil.getOperator(chain);
|
|
||||||
|
|
||||||
DatacenterTableService service = SpringContextUtil.getBean(DatacenterTableService.class);
|
|
||||||
|
|
||||||
JSONArray saveList = json.getJSONArray("saveList");
|
|
||||||
|
|
||||||
int successRows = 0;
|
|
||||||
for (Object object : saveList) {
|
|
||||||
JSONObject obj = new JSONObject((com.alibaba.fastjson.JSONObject) object);
|
|
||||||
obj.put("table_id", tableId);
|
|
||||||
try {
|
|
||||||
TenantManager.ignoreTenantCondition();
|
|
||||||
service.saveValue(tableId, obj, account);
|
|
||||||
} catch (Exception e) {
|
|
||||||
log.error("工作流保存数据到数据中枢失败,表ID:{},具体值:{}", tableId, obj, e);
|
|
||||||
throw e;
|
|
||||||
} finally {
|
|
||||||
TenantManager.restoreTenantCondition();
|
|
||||||
}
|
|
||||||
successRows++;
|
|
||||||
}
|
|
||||||
|
|
||||||
res.put("successRows", successRows);
|
|
||||||
return res;
|
|
||||||
}
|
|
||||||
|
|
||||||
public BigInteger getTableId() {
|
|
||||||
return tableId;
|
|
||||||
}
|
|
||||||
|
|
||||||
public void setTableId(BigInteger tableId) {
|
|
||||||
this.tableId = tableId;
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
||||||
@@ -1,23 +0,0 @@
|
|||||||
package tech.easyflow.ai.node;
|
|
||||||
|
|
||||||
import com.alibaba.fastjson.JSONObject;
|
|
||||||
import com.easyagents.flow.core.node.BaseNode;
|
|
||||||
import com.easyagents.flow.core.parser.BaseNodeParser;
|
|
||||||
|
|
||||||
import java.math.BigInteger;
|
|
||||||
|
|
||||||
public class SaveToDatacenterNodeParser extends BaseNodeParser {
|
|
||||||
|
|
||||||
@Override
|
|
||||||
protected BaseNode doParse(JSONObject root, JSONObject data, JSONObject tinyflow) {
|
|
||||||
BigInteger tableId = data.getBigInteger("tableId");
|
|
||||||
if (tableId == null) {
|
|
||||||
throw new RuntimeException("请选择数据表");
|
|
||||||
}
|
|
||||||
return new SaveToDatacenterNode(tableId);
|
|
||||||
}
|
|
||||||
|
|
||||||
public String getNodeName() {
|
|
||||||
return "save-to-datacenter-node";
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,143 +0,0 @@
|
|||||||
package tech.easyflow.ai.node;
|
|
||||||
|
|
||||||
import com.easyagents.core.util.StringUtil;
|
|
||||||
import com.mybatisflex.core.paginate.Page;
|
|
||||||
import com.mybatisflex.core.row.Row;
|
|
||||||
import com.mybatisflex.core.tenant.TenantManager;
|
|
||||||
import com.easyagents.flow.core.chain.Chain;
|
|
||||||
import com.easyagents.flow.core.chain.Parameter;
|
|
||||||
import com.easyagents.flow.core.node.BaseNode;
|
|
||||||
import net.sf.jsqlparser.expression.Expression;
|
|
||||||
import net.sf.jsqlparser.parser.CCJSqlParserUtil;
|
|
||||||
import org.slf4j.Logger;
|
|
||||||
import org.slf4j.LoggerFactory;
|
|
||||||
import tech.easyflow.common.entity.DatacenterQuery;
|
|
||||||
import tech.easyflow.common.util.SpringContextUtil;
|
|
||||||
import tech.easyflow.datacenter.entity.DatacenterTableField;
|
|
||||||
import tech.easyflow.datacenter.service.DatacenterTableService;
|
|
||||||
import tech.easyflow.datacenter.utils.WhereConditionSecurityChecker;
|
|
||||||
|
|
||||||
import java.math.BigInteger;
|
|
||||||
import java.util.HashMap;
|
|
||||||
import java.util.List;
|
|
||||||
import java.util.Map;
|
|
||||||
import java.util.Set;
|
|
||||||
import java.util.regex.Matcher;
|
|
||||||
import java.util.regex.Pattern;
|
|
||||||
import java.util.stream.Collectors;
|
|
||||||
|
|
||||||
public class SearchDatacenterNode extends BaseNode {
|
|
||||||
|
|
||||||
private static final Logger log = LoggerFactory.getLogger(SearchDatacenterNode.class);
|
|
||||||
private BigInteger tableId;
|
|
||||||
private String where;
|
|
||||||
private Long limit;
|
|
||||||
|
|
||||||
public SearchDatacenterNode() {
|
|
||||||
}
|
|
||||||
|
|
||||||
public SearchDatacenterNode(BigInteger tableId, String where, Long limit) {
|
|
||||||
this.tableId = tableId;
|
|
||||||
this.where = where;
|
|
||||||
this.limit = limit;
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public Map<String, Object> execute(Chain chain) {
|
|
||||||
|
|
||||||
Map<String, Object> map = chain.getState().resolveParameters(this);
|
|
||||||
Map<String, Object> res = new HashMap<>();
|
|
||||||
long limitNum = 10;
|
|
||||||
if (limit != null) {
|
|
||||||
limitNum = Long.parseLong(limit.toString());
|
|
||||||
}
|
|
||||||
|
|
||||||
DatacenterTableService service = SpringContextUtil.getBean(DatacenterTableService.class);
|
|
||||||
|
|
||||||
DatacenterQuery condition = new DatacenterQuery();
|
|
||||||
condition.setTableId(tableId);
|
|
||||||
condition.setPageNumber(1L);
|
|
||||||
condition.setPageSize(limitNum);
|
|
||||||
// 组合查询条件
|
|
||||||
if (where != null) {
|
|
||||||
setCondition(where, condition, map);
|
|
||||||
}
|
|
||||||
try {
|
|
||||||
TenantManager.ignoreTenantCondition();
|
|
||||||
Page<Row> pageData = service.getPageData(condition);
|
|
||||||
|
|
||||||
String key = "rows";
|
|
||||||
List<Parameter> outputDefs = getOutputDefs();
|
|
||||||
if (outputDefs != null && !outputDefs.isEmpty()) {
|
|
||||||
String defName = outputDefs.get(0).getName();
|
|
||||||
if (StringUtil.hasText(defName)) key = defName;
|
|
||||||
}
|
|
||||||
res.put(key, pageData.getRecords());
|
|
||||||
} finally {
|
|
||||||
TenantManager.restoreTenantCondition();
|
|
||||||
}
|
|
||||||
return res;
|
|
||||||
}
|
|
||||||
|
|
||||||
public BigInteger getTableId() {
|
|
||||||
return tableId;
|
|
||||||
}
|
|
||||||
|
|
||||||
public void setTableId(BigInteger tableId) {
|
|
||||||
this.tableId = tableId;
|
|
||||||
}
|
|
||||||
|
|
||||||
public String getWhere() {
|
|
||||||
return where;
|
|
||||||
}
|
|
||||||
|
|
||||||
public void setWhere(String where) {
|
|
||||||
this.where = where;
|
|
||||||
}
|
|
||||||
|
|
||||||
public Long getLimit() {
|
|
||||||
return limit;
|
|
||||||
}
|
|
||||||
|
|
||||||
public void setLimit(Long limit) {
|
|
||||||
this.limit = limit;
|
|
||||||
}
|
|
||||||
|
|
||||||
private void setCondition(String where, DatacenterQuery condition, Map<String, Object> params) {
|
|
||||||
// 条件封装
|
|
||||||
Pattern pattern = Pattern.compile("\\{\\{(.+?)\\}\\}");
|
|
||||||
Matcher matcher = pattern.matcher(where);
|
|
||||||
|
|
||||||
StringBuffer result = new StringBuffer();
|
|
||||||
while (matcher.find()) {
|
|
||||||
String key = matcher.group(1);
|
|
||||||
Object value = params.get(key);
|
|
||||||
if (value == null) {
|
|
||||||
throw new RuntimeException("参数" + key + "不存在");
|
|
||||||
}
|
|
||||||
String replacement = value.toString();
|
|
||||||
matcher.appendReplacement(result, "'" + replacement + "'");
|
|
||||||
}
|
|
||||||
matcher.appendTail(result);
|
|
||||||
|
|
||||||
try {
|
|
||||||
Expression expression = CCJSqlParserUtil.parseCondExpression(result.toString());
|
|
||||||
if (expression != null) {
|
|
||||||
WhereConditionSecurityChecker checker = new WhereConditionSecurityChecker();
|
|
||||||
DatacenterTableService service = SpringContextUtil.getBean(DatacenterTableService.class);
|
|
||||||
List<DatacenterTableField> fields = service.getFields(tableId);
|
|
||||||
Set<String> columns = fields.stream().map(DatacenterTableField::getFieldName).collect(Collectors.toSet());
|
|
||||||
columns.add("id");
|
|
||||||
columns.add("created");
|
|
||||||
columns.add("modified");
|
|
||||||
columns.add("created_by");
|
|
||||||
columns.add("modified_by");
|
|
||||||
checker.checkConditionSafety(expression, columns);
|
|
||||||
condition.setWhere(expression.toString());
|
|
||||||
}
|
|
||||||
} catch (Exception e) {
|
|
||||||
log.error("WHERE SQL解析错误:", e);
|
|
||||||
throw new RuntimeException(e);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,25 +0,0 @@
|
|||||||
package tech.easyflow.ai.node;
|
|
||||||
|
|
||||||
import com.alibaba.fastjson.JSONObject;
|
|
||||||
import com.easyagents.flow.core.node.BaseNode;
|
|
||||||
import com.easyagents.flow.core.parser.BaseNodeParser;
|
|
||||||
|
|
||||||
import java.math.BigInteger;
|
|
||||||
|
|
||||||
public class SearchDatacenterNodeParser extends BaseNodeParser {
|
|
||||||
|
|
||||||
@Override
|
|
||||||
protected BaseNode doParse(JSONObject root, JSONObject data, JSONObject tinyflow) {
|
|
||||||
BigInteger tableId = data.getBigInteger("tableId");
|
|
||||||
String where = data.getString("where");
|
|
||||||
Long limit = data.getLong("limit");
|
|
||||||
if (tableId == null) {
|
|
||||||
throw new RuntimeException("请选择数据表");
|
|
||||||
}
|
|
||||||
return new SearchDatacenterNode(tableId,where,limit);
|
|
||||||
}
|
|
||||||
|
|
||||||
public String getNodeName() {
|
|
||||||
return "search-datacenter-node";
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -0,0 +1,120 @@
|
|||||||
|
package tech.easyflow.ai.node;
|
||||||
|
|
||||||
|
import com.easyagents.core.util.StringUtil;
|
||||||
|
import com.easyagents.flow.core.chain.Chain;
|
||||||
|
import com.easyagents.flow.core.chain.Parameter;
|
||||||
|
import com.easyagents.flow.core.node.BaseNode;
|
||||||
|
import com.mybatisflex.core.row.Row;
|
||||||
|
import com.mybatisflex.core.tenant.TenantManager;
|
||||||
|
import tech.easyflow.common.util.SpringContextUtil;
|
||||||
|
import tech.easyflow.common.web.exceptions.BusinessException;
|
||||||
|
import tech.easyflow.datacenter.execution.model.DatacenterSqlQueryRequest;
|
||||||
|
import tech.easyflow.datacenter.execution.model.DatasetRef;
|
||||||
|
import tech.easyflow.datacenter.execution.service.DatacenterDatasetQueryService;
|
||||||
|
|
||||||
|
import java.util.HashMap;
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.Map;
|
||||||
|
import java.util.regex.Matcher;
|
||||||
|
import java.util.regex.Pattern;
|
||||||
|
|
||||||
|
public class SearchDatasetNode extends BaseNode {
|
||||||
|
|
||||||
|
private static final Pattern PARAM_PATTERN = Pattern.compile("\\{\\{(.+?)\\}\\}");
|
||||||
|
|
||||||
|
private DatasetRef datasetRef;
|
||||||
|
private String querySql;
|
||||||
|
|
||||||
|
public SearchDatasetNode() {
|
||||||
|
}
|
||||||
|
|
||||||
|
public SearchDatasetNode(DatasetRef datasetRef) {
|
||||||
|
this.datasetRef = datasetRef;
|
||||||
|
}
|
||||||
|
|
||||||
|
public SearchDatasetNode(DatasetRef datasetRef, String querySql) {
|
||||||
|
this.datasetRef = datasetRef;
|
||||||
|
this.querySql = querySql;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public Map<String, Object> execute(Chain chain) {
|
||||||
|
Map<String, Object> params = chain.getState().resolveParameters(this);
|
||||||
|
DatacenterDatasetQueryService queryService = SpringContextUtil.getBean(DatacenterDatasetQueryService.class);
|
||||||
|
DatacenterSqlQueryRequest request = buildRuntimeRequest(params);
|
||||||
|
Map<String, Object> result = new HashMap<>();
|
||||||
|
try {
|
||||||
|
TenantManager.ignoreTenantCondition();
|
||||||
|
List<Row> rows = queryService.queryBySql(request);
|
||||||
|
result.put(resolveOutputKey("data"), rows);
|
||||||
|
return result;
|
||||||
|
} finally {
|
||||||
|
TenantManager.restoreTenantCondition();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private DatacenterSqlQueryRequest buildRuntimeRequest(Map<String, Object> params) {
|
||||||
|
DatacenterSqlQueryRequest request = new DatacenterSqlQueryRequest();
|
||||||
|
request.setDatasetRef(copyDatasetRef());
|
||||||
|
request.setSql(resolveQuerySql(params));
|
||||||
|
return request;
|
||||||
|
}
|
||||||
|
|
||||||
|
private String resolveQuerySql(Map<String, Object> params) {
|
||||||
|
String sql = resolveTemplateString(querySql, params);
|
||||||
|
if (!StringUtil.hasText(sql)) {
|
||||||
|
throw new BusinessException("查询数据节点未设置 SQL");
|
||||||
|
}
|
||||||
|
return sql.trim();
|
||||||
|
}
|
||||||
|
|
||||||
|
private DatasetRef copyDatasetRef() {
|
||||||
|
DatasetRef copy = new DatasetRef();
|
||||||
|
copy.setSourceId(datasetRef == null ? null : datasetRef.getSourceId());
|
||||||
|
copy.setCatalogId(datasetRef == null ? null : datasetRef.getCatalogId());
|
||||||
|
copy.setCatalogName(datasetRef == null ? null : datasetRef.getCatalogName());
|
||||||
|
copy.setTableId(null);
|
||||||
|
copy.setTableName(null);
|
||||||
|
copy.setVersionId(null);
|
||||||
|
return copy;
|
||||||
|
}
|
||||||
|
|
||||||
|
private String resolveTemplateString(String text, Map<String, Object> params) {
|
||||||
|
if (!StringUtil.hasText(text) || params == null || params.isEmpty()) {
|
||||||
|
return text;
|
||||||
|
}
|
||||||
|
Matcher matcher = PARAM_PATTERN.matcher(text);
|
||||||
|
StringBuffer buffer = new StringBuffer();
|
||||||
|
while (matcher.find()) {
|
||||||
|
Object replacement = params.get(matcher.group(1));
|
||||||
|
matcher.appendReplacement(buffer, replacement == null ? "" : Matcher.quoteReplacement(String.valueOf(replacement)));
|
||||||
|
}
|
||||||
|
matcher.appendTail(buffer);
|
||||||
|
return buffer.toString();
|
||||||
|
}
|
||||||
|
|
||||||
|
private String resolveOutputKey(String defaultName) {
|
||||||
|
List<Parameter> outputDefs = getOutputDefs();
|
||||||
|
if (outputDefs == null || outputDefs.isEmpty()) {
|
||||||
|
return defaultName;
|
||||||
|
}
|
||||||
|
String name = outputDefs.get(0).getName();
|
||||||
|
return StringUtil.hasText(name) ? name : defaultName;
|
||||||
|
}
|
||||||
|
|
||||||
|
public DatasetRef getDatasetRef() {
|
||||||
|
return datasetRef;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setDatasetRef(DatasetRef datasetRef) {
|
||||||
|
this.datasetRef = datasetRef;
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getQuerySql() {
|
||||||
|
return querySql;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setQuerySql(String querySql) {
|
||||||
|
this.querySql = querySql;
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,33 @@
|
|||||||
|
package tech.easyflow.ai.node;
|
||||||
|
|
||||||
|
import com.alibaba.fastjson.JSONObject;
|
||||||
|
import com.easyagents.flow.core.node.BaseNode;
|
||||||
|
import com.easyagents.flow.core.parser.BaseNodeParser;
|
||||||
|
import tech.easyflow.common.web.exceptions.BusinessException;
|
||||||
|
import tech.easyflow.datacenter.execution.model.DatasetRef;
|
||||||
|
|
||||||
|
public class SearchDatasetNodeParser extends BaseNodeParser {
|
||||||
|
|
||||||
|
private static final String SOURCE_MISSING_MESSAGE = "查询数据节点未选择连接服务";
|
||||||
|
private static final String SQL_MISSING_MESSAGE = "查询数据节点未设置 SQL";
|
||||||
|
|
||||||
|
@Override
|
||||||
|
protected BaseNode doParse(JSONObject root, JSONObject data, JSONObject tinyflow) {
|
||||||
|
if (data == null) {
|
||||||
|
throw new BusinessException(SOURCE_MISSING_MESSAGE);
|
||||||
|
}
|
||||||
|
DatasetRef datasetRef = data.getObject("datasetRef", DatasetRef.class);
|
||||||
|
if (datasetRef == null || datasetRef.getSourceId() == null) {
|
||||||
|
throw new BusinessException(SOURCE_MISSING_MESSAGE);
|
||||||
|
}
|
||||||
|
String querySql = data.getString("querySql");
|
||||||
|
if (querySql == null || querySql.isBlank()) {
|
||||||
|
throw new BusinessException(SQL_MISSING_MESSAGE);
|
||||||
|
}
|
||||||
|
return new SearchDatasetNode(datasetRef, querySql);
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getNodeName() {
|
||||||
|
return "search-dataset-node";
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -1,149 +0,0 @@
|
|||||||
package tech.easyflow.ai.node;
|
|
||||||
|
|
||||||
import com.mybatisflex.core.row.Db;
|
|
||||||
import com.mybatisflex.core.row.Row;
|
|
||||||
import com.easyagents.flow.core.chain.Chain;
|
|
||||||
import com.easyagents.flow.core.node.BaseNode;
|
|
||||||
import net.sf.jsqlparser.JSQLParserException;
|
|
||||||
import net.sf.jsqlparser.parser.CCJSqlParserUtil;
|
|
||||||
import net.sf.jsqlparser.statement.Statement;
|
|
||||||
import net.sf.jsqlparser.statement.select.Select;
|
|
||||||
import org.slf4j.Logger;
|
|
||||||
import org.slf4j.LoggerFactory;
|
|
||||||
import org.springframework.util.StringUtils;
|
|
||||||
import tech.easyflow.common.web.exceptions.BusinessException;
|
|
||||||
|
|
||||||
import java.util.*;
|
|
||||||
import java.util.regex.Matcher;
|
|
||||||
import java.util.regex.Pattern;
|
|
||||||
|
|
||||||
|
|
||||||
/**
|
|
||||||
* SQL查询节点
|
|
||||||
*
|
|
||||||
* @author tao
|
|
||||||
* @date 2025-05-21
|
|
||||||
*/
|
|
||||||
public class SqlNode extends BaseNode {
|
|
||||||
|
|
||||||
private String sql;
|
|
||||||
|
|
||||||
private static final Logger logger = LoggerFactory.getLogger(SqlNode.class);
|
|
||||||
|
|
||||||
public SqlNode() {
|
|
||||||
}
|
|
||||||
|
|
||||||
public SqlNode(String sql) {
|
|
||||||
this.sql = sql;
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public Map<String, Object> execute(Chain chain) {
|
|
||||||
|
|
||||||
Map<String, Object> map = chain.getState().resolveParameters(this);
|
|
||||||
Map<String, Object> res = new HashMap<>();
|
|
||||||
|
|
||||||
|
|
||||||
Map<String, Object> formatSqlMap = formatSql(sql, map);
|
|
||||||
String formatSql = (String) formatSqlMap.get("replacedSql");
|
|
||||||
|
|
||||||
Statement statement = null;
|
|
||||||
try {
|
|
||||||
statement = CCJSqlParserUtil.parse(formatSql);
|
|
||||||
|
|
||||||
} catch (JSQLParserException e) {
|
|
||||||
logger.error("sql 解析报错:", e);
|
|
||||||
throw new BusinessException("SQL解析失败,请确认SQL语法无误");
|
|
||||||
}
|
|
||||||
|
|
||||||
if (!(statement instanceof Select)) {
|
|
||||||
logger.error("sql 解析报错:statement instanceof Select 结果为false");
|
|
||||||
throw new BusinessException("仅支持查询语句!");
|
|
||||||
}
|
|
||||||
|
|
||||||
List<String> paramNames = (List<String>) formatSqlMap.get("paramNames");
|
|
||||||
|
|
||||||
List<Object> paramValues = new ArrayList<>();
|
|
||||||
paramNames.forEach(paramName -> {
|
|
||||||
Object o = map.get(paramName);
|
|
||||||
paramValues.add(o);
|
|
||||||
});
|
|
||||||
|
|
||||||
List<Row> rows = Db.selectListBySql(formatSql, paramValues.toArray());
|
|
||||||
|
|
||||||
if (rows == null || rows.isEmpty()) {
|
|
||||||
return Collections.emptyMap();
|
|
||||||
}
|
|
||||||
|
|
||||||
res.put("queryData", rows);
|
|
||||||
return res;
|
|
||||||
}
|
|
||||||
|
|
||||||
private Map<String, Object> formatSql(String rawSql, Map<String, Object> paramMap) {
|
|
||||||
|
|
||||||
if (!StringUtils.hasLength(rawSql)) {
|
|
||||||
logger.error("sql解析报错:sql为空");
|
|
||||||
throw new BusinessException("sql 不能为空!");
|
|
||||||
}
|
|
||||||
|
|
||||||
// 匹配 {{?...}} 表示可用占位符的参数
|
|
||||||
Pattern paramPattern = Pattern.compile("\\{\\{\\?([^}]+)}}");
|
|
||||||
|
|
||||||
// 匹配 {{...}} 表示直接替换的参数(非占位符)
|
|
||||||
Pattern directPattern = Pattern.compile("\\{\\{([^}?][^}]*)}}");
|
|
||||||
|
|
||||||
List<String> paramNames = new ArrayList<>();
|
|
||||||
StringBuffer sqlBuffer = new StringBuffer();
|
|
||||||
|
|
||||||
// 替换 {{?...}} -> ?
|
|
||||||
Matcher paramMatcher = paramPattern.matcher(rawSql);
|
|
||||||
while (paramMatcher.find()) {
|
|
||||||
String paramName = paramMatcher.group(1).trim();
|
|
||||||
paramNames.add(paramName);
|
|
||||||
paramMatcher.appendReplacement(sqlBuffer, "?");
|
|
||||||
}
|
|
||||||
paramMatcher.appendTail(sqlBuffer);
|
|
||||||
String intermediateSql = sqlBuffer.toString();
|
|
||||||
|
|
||||||
// 替换 {{...}} -> 实际值(用于表名/列名等)
|
|
||||||
sqlBuffer = new StringBuffer(); // 清空 buffer 重新处理
|
|
||||||
Matcher directMatcher = directPattern.matcher(intermediateSql);
|
|
||||||
while (directMatcher.find()) {
|
|
||||||
String key = directMatcher.group(1).trim();
|
|
||||||
Object value = paramMap.get(key);
|
|
||||||
if (value == null) {
|
|
||||||
logger.error("未找到参数:" + key);
|
|
||||||
throw new BusinessException("sql解析失败,请确保sql语法正确!");
|
|
||||||
}
|
|
||||||
|
|
||||||
String safeValue = value.toString();
|
|
||||||
|
|
||||||
directMatcher.appendReplacement(sqlBuffer, Matcher.quoteReplacement(safeValue));
|
|
||||||
}
|
|
||||||
directMatcher.appendTail(sqlBuffer);
|
|
||||||
|
|
||||||
String finalSql = sqlBuffer.toString().trim();
|
|
||||||
|
|
||||||
// 清理末尾分号与中文引号
|
|
||||||
if (finalSql.endsWith(";") || finalSql.endsWith(";")) {
|
|
||||||
finalSql = finalSql.substring(0, finalSql.length() - 1);
|
|
||||||
}
|
|
||||||
finalSql = finalSql.replace("“", "\"").replace("”", "\"");
|
|
||||||
|
|
||||||
logger.info("Final SQL: {}", finalSql);
|
|
||||||
logger.info("Param names: {}", paramNames);
|
|
||||||
|
|
||||||
Map<String, Object> result = new HashMap<>();
|
|
||||||
result.put("replacedSql", finalSql);
|
|
||||||
result.put("paramNames", paramNames);
|
|
||||||
return result;
|
|
||||||
}
|
|
||||||
|
|
||||||
public String getSql() {
|
|
||||||
return sql;
|
|
||||||
}
|
|
||||||
|
|
||||||
public void setSql(String sql) {
|
|
||||||
this.sql = sql;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,25 +0,0 @@
|
|||||||
package tech.easyflow.ai.node;
|
|
||||||
|
|
||||||
import com.alibaba.fastjson.JSONObject;
|
|
||||||
import com.easyagents.flow.core.node.BaseNode;
|
|
||||||
import com.easyagents.flow.core.parser.BaseNodeParser;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Sql查询节点解析
|
|
||||||
*
|
|
||||||
* @author tao
|
|
||||||
* @date 2025-05-21
|
|
||||||
*/
|
|
||||||
public class SqlNodeParser extends BaseNodeParser {
|
|
||||||
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public BaseNode doParse(JSONObject root, JSONObject data, JSONObject tinyflow) {
|
|
||||||
String sql = data.getString("sql");
|
|
||||||
return new SqlNode(sql);
|
|
||||||
}
|
|
||||||
|
|
||||||
public String getNodeName() {
|
|
||||||
return "sql-node";
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -8,6 +8,7 @@ import org.junit.Test;
|
|||||||
import tech.easyflow.ai.easyagentsflow.entity.WorkflowCheckResult;
|
import tech.easyflow.ai.easyagentsflow.entity.WorkflowCheckResult;
|
||||||
import tech.easyflow.ai.easyagentsflow.entity.WorkflowCheckStage;
|
import tech.easyflow.ai.easyagentsflow.entity.WorkflowCheckStage;
|
||||||
import tech.easyflow.ai.entity.Workflow;
|
import tech.easyflow.ai.entity.Workflow;
|
||||||
|
import tech.easyflow.ai.node.SearchDatasetNodeParser;
|
||||||
import tech.easyflow.ai.node.WorkflowNodeParser;
|
import tech.easyflow.ai.node.WorkflowNodeParser;
|
||||||
import tech.easyflow.ai.service.WorkflowService;
|
import tech.easyflow.ai.service.WorkflowService;
|
||||||
|
|
||||||
@@ -69,6 +70,24 @@ public class WorkflowCheckServiceTest {
|
|||||||
assertHasCode(result, "EDGE_TARGET_NOT_FOUND");
|
assertHasCode(result, "EDGE_TARGET_NOT_FOUND");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testSaveShouldBlockSearchDatasetWithoutSql() throws Exception {
|
||||||
|
WorkflowCheckService service = newService(new HashMap<>());
|
||||||
|
JSONObject searchData = data("查询数据");
|
||||||
|
JSONObject datasetRef = new JSONObject();
|
||||||
|
datasetRef.put("sourceId", "1001");
|
||||||
|
datasetRef.put("tableId", "2001");
|
||||||
|
searchData.put("datasetRef", datasetRef);
|
||||||
|
String content = workflowJson(
|
||||||
|
array(node("search-1", "search-dataset-node", null, searchData)),
|
||||||
|
new JSONArray()
|
||||||
|
);
|
||||||
|
|
||||||
|
WorkflowCheckResult result = service.checkContent(content, WorkflowCheckStage.SAVE, null);
|
||||||
|
Assert.assertFalse(result.isPassed());
|
||||||
|
assertHasCode(result, "SEARCH_DATASET_INVALID");
|
||||||
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void testPreExecuteShouldBlockMissingStartOrEnd() throws Exception {
|
public void testPreExecuteShouldBlockMissingStartOrEnd() throws Exception {
|
||||||
WorkflowCheckService service = newService(new HashMap<>());
|
WorkflowCheckService service = newService(new HashMap<>());
|
||||||
@@ -86,6 +105,26 @@ public class WorkflowCheckServiceTest {
|
|||||||
assertHasCode(result, "END_NODE_MISSING");
|
assertHasCode(result, "END_NODE_MISSING");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testPreExecuteShouldPassForSourceOnlySearchDatasetNode() throws Exception {
|
||||||
|
WorkflowCheckService service = newService(new HashMap<>());
|
||||||
|
String content = workflowJson(
|
||||||
|
array(
|
||||||
|
node("s1", "startNode", null, data("开始")),
|
||||||
|
searchDatasetNode("q1", null, "1001"),
|
||||||
|
node("e1", "endNode", null, data("结束"))
|
||||||
|
),
|
||||||
|
array(
|
||||||
|
edge("e1", "s1", "q1"),
|
||||||
|
edge("e2", "q1", "e1")
|
||||||
|
)
|
||||||
|
);
|
||||||
|
|
||||||
|
WorkflowCheckResult result = service.checkContent(content, WorkflowCheckStage.PRE_EXECUTE, BigInteger.ONE);
|
||||||
|
Assert.assertTrue(result.isPassed());
|
||||||
|
Assert.assertEquals(0, result.getIssueCount());
|
||||||
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void testPreExecuteShouldBlockRootEntryNotStart() throws Exception {
|
public void testPreExecuteShouldBlockRootEntryNotStart() throws Exception {
|
||||||
WorkflowCheckService service = newService(new HashMap<>());
|
WorkflowCheckService service = newService(new HashMap<>());
|
||||||
@@ -203,8 +242,10 @@ public class WorkflowCheckServiceTest {
|
|||||||
.withDefaultParsers(true)
|
.withDefaultParsers(true)
|
||||||
.build();
|
.build();
|
||||||
parser.addNodeParser("workflow-node", new WorkflowNodeParser());
|
parser.addNodeParser("workflow-node", new WorkflowNodeParser());
|
||||||
|
parser.addNodeParser("search-dataset-node", new SearchDatasetNodeParser());
|
||||||
setField(service, "chainParser", parser);
|
setField(service, "chainParser", parser);
|
||||||
setField(service, "workflowService", mockWorkflowService(workflowStore));
|
setField(service, "workflowService", mockWorkflowService(workflowStore));
|
||||||
|
setField(service, "workflowDatacenterContentService", new WorkflowDatacenterContentService());
|
||||||
return service;
|
return service;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -294,6 +335,15 @@ public class WorkflowCheckServiceTest {
|
|||||||
return node(id, "workflow-node", parentId, data);
|
return node(id, "workflow-node", parentId, data);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private static JSONObject searchDatasetNode(String id, String parentId, String sourceId) {
|
||||||
|
JSONObject data = data("查询数据");
|
||||||
|
JSONObject datasetRef = new JSONObject();
|
||||||
|
datasetRef.put("sourceId", sourceId);
|
||||||
|
data.put("datasetRef", datasetRef);
|
||||||
|
data.put("querySql", "SELECT 1");
|
||||||
|
return node(id, "search-dataset-node", parentId, data);
|
||||||
|
}
|
||||||
|
|
||||||
private static JSONObject data(String title) {
|
private static JSONObject data(String title) {
|
||||||
JSONObject data = new JSONObject();
|
JSONObject data = new JSONObject();
|
||||||
data.put("title", title);
|
data.put("title", title);
|
||||||
|
|||||||
@@ -0,0 +1,73 @@
|
|||||||
|
package tech.easyflow.ai.node;
|
||||||
|
|
||||||
|
import org.junit.Assert;
|
||||||
|
import org.junit.Test;
|
||||||
|
import tech.easyflow.common.web.exceptions.BusinessException;
|
||||||
|
import tech.easyflow.datacenter.execution.model.DatasetRef;
|
||||||
|
|
||||||
|
import java.lang.reflect.Method;
|
||||||
|
import java.math.BigInteger;
|
||||||
|
import java.util.HashMap;
|
||||||
|
import java.util.Map;
|
||||||
|
|
||||||
|
public class SearchDatasetNodeTest {
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testResolveQuerySqlShouldUseNodeQuerySqlAndResolveTemplate() throws Exception {
|
||||||
|
DatasetRef datasetRef = new DatasetRef();
|
||||||
|
datasetRef.setSourceId(BigInteger.valueOf(1001L));
|
||||||
|
SearchDatasetNode node = new SearchDatasetNode(datasetRef, """
|
||||||
|
SELECT id, name
|
||||||
|
FROM orders_{{biz}}
|
||||||
|
WHERE name LIKE '%{{keyword}}%'
|
||||||
|
ORDER BY created_at {{direction}}
|
||||||
|
""");
|
||||||
|
|
||||||
|
Map<String, Object> params = new HashMap<>();
|
||||||
|
params.put("biz", "prod");
|
||||||
|
params.put("keyword", "vip");
|
||||||
|
params.put("direction", "DESC");
|
||||||
|
|
||||||
|
String sql = invokeResolveQuerySql(node, params);
|
||||||
|
Assert.assertEquals("""
|
||||||
|
SELECT id, name
|
||||||
|
FROM orders_prod
|
||||||
|
WHERE name LIKE '%vip%'
|
||||||
|
ORDER BY created_at DESC
|
||||||
|
""".trim(), sql);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testResolveQuerySqlShouldUseNodeQuerySqlWhenParamsDoNotContainSql() throws Exception {
|
||||||
|
DatasetRef datasetRef = new DatasetRef();
|
||||||
|
datasetRef.setSourceId(BigInteger.valueOf(2002L));
|
||||||
|
SearchDatasetNode node = new SearchDatasetNode(datasetRef, "SELECT * FROM fallback_table");
|
||||||
|
|
||||||
|
Map<String, Object> params = new HashMap<>();
|
||||||
|
|
||||||
|
String sql = invokeResolveQuerySql(node, params);
|
||||||
|
Assert.assertEquals("SELECT * FROM fallback_table", sql);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testResolveQuerySqlShouldRejectBlankSql() throws Exception {
|
||||||
|
DatasetRef datasetRef = new DatasetRef();
|
||||||
|
datasetRef.setSourceId(BigInteger.valueOf(3003L));
|
||||||
|
SearchDatasetNode node = new SearchDatasetNode(datasetRef, " ");
|
||||||
|
|
||||||
|
try {
|
||||||
|
invokeResolveQuerySql(node, new HashMap<>());
|
||||||
|
Assert.fail("expected BusinessException");
|
||||||
|
} catch (Exception e) {
|
||||||
|
Throwable cause = e.getCause() == null ? e : e.getCause();
|
||||||
|
Assert.assertTrue(cause instanceof BusinessException);
|
||||||
|
Assert.assertEquals("查询数据节点未设置 SQL", cause.getMessage());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private String invokeResolveQuerySql(SearchDatasetNode node, Map<String, Object> params) throws Exception {
|
||||||
|
Method method = SearchDatasetNode.class.getDeclaredMethod("resolveQuerySql", Map.class);
|
||||||
|
method.setAccessible(true);
|
||||||
|
return (String) method.invoke(node, params);
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -16,6 +16,19 @@
|
|||||||
<groupId>com.mybatis-flex</groupId>
|
<groupId>com.mybatis-flex</groupId>
|
||||||
<artifactId>mybatis-flex-spring-boot3-starter</artifactId>
|
<artifactId>mybatis-flex-spring-boot3-starter</artifactId>
|
||||||
</dependency>
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>com.zaxxer</groupId>
|
||||||
|
<artifactId>HikariCP</artifactId>
|
||||||
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.postgresql</groupId>
|
||||||
|
<artifactId>postgresql</artifactId>
|
||||||
|
<version>42.7.3</version>
|
||||||
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>cn.hutool</groupId>
|
||||||
|
<artifactId>hutool-crypto</artifactId>
|
||||||
|
</dependency>
|
||||||
<dependency>
|
<dependency>
|
||||||
<groupId>tech.easyflow</groupId>
|
<groupId>tech.easyflow</groupId>
|
||||||
<artifactId>easyflow-common-base</artifactId>
|
<artifactId>easyflow-common-base</artifactId>
|
||||||
|
|||||||
@@ -0,0 +1,12 @@
|
|||||||
|
package tech.easyflow.datacenter.connector;
|
||||||
|
|
||||||
|
import tech.easyflow.datacenter.meta.enums.DatacenterCapability;
|
||||||
|
import tech.easyflow.datacenter.meta.enums.DatacenterSourceType;
|
||||||
|
|
||||||
|
import java.util.Set;
|
||||||
|
|
||||||
|
public interface DatacenterConnector extends SourceHealthChecker, MetadataExplorer, QueryExecutor, WriteExecutor {
|
||||||
|
DatacenterSourceType getSourceType();
|
||||||
|
|
||||||
|
Set<DatacenterCapability> getCapabilities();
|
||||||
|
}
|
||||||
@@ -0,0 +1,29 @@
|
|||||||
|
package tech.easyflow.datacenter.connector;
|
||||||
|
|
||||||
|
import org.springframework.stereotype.Component;
|
||||||
|
import tech.easyflow.common.web.exceptions.BusinessException;
|
||||||
|
import tech.easyflow.datacenter.meta.enums.DatacenterSourceType;
|
||||||
|
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.Map;
|
||||||
|
import java.util.function.Function;
|
||||||
|
import java.util.stream.Collectors;
|
||||||
|
|
||||||
|
@Component
|
||||||
|
public class DatacenterConnectorRegistry {
|
||||||
|
|
||||||
|
private final Map<DatacenterSourceType, DatacenterConnector> connectorMap;
|
||||||
|
|
||||||
|
public DatacenterConnectorRegistry(List<DatacenterConnector> connectors) {
|
||||||
|
this.connectorMap = connectors.stream().collect(Collectors.toMap(DatacenterConnector::getSourceType, Function.identity()));
|
||||||
|
}
|
||||||
|
|
||||||
|
public DatacenterConnector getConnector(String sourceType) {
|
||||||
|
DatacenterSourceType type = DatacenterSourceType.valueOf(sourceType);
|
||||||
|
DatacenterConnector connector = connectorMap.get(type);
|
||||||
|
if (connector == null) {
|
||||||
|
throw new BusinessException("未找到连接器: " + sourceType);
|
||||||
|
}
|
||||||
|
return connector;
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,16 @@
|
|||||||
|
package tech.easyflow.datacenter.connector;
|
||||||
|
|
||||||
|
import tech.easyflow.datacenter.entity.DatacenterTable;
|
||||||
|
import tech.easyflow.datacenter.meta.entity.DatacenterSource;
|
||||||
|
import tech.easyflow.datacenter.meta.model.DatacenterCatalogMeta;
|
||||||
|
import tech.easyflow.datacenter.meta.model.DatacenterTableDetailMeta;
|
||||||
|
|
||||||
|
import java.util.List;
|
||||||
|
|
||||||
|
public interface MetadataExplorer {
|
||||||
|
List<DatacenterCatalogMeta> listCatalogs(DatacenterSource source);
|
||||||
|
|
||||||
|
List<DatacenterTable> listTables(DatacenterSource source, String catalogName);
|
||||||
|
|
||||||
|
DatacenterTableDetailMeta getTableDetail(DatacenterSource source, String catalogName, String tableName);
|
||||||
|
}
|
||||||
@@ -0,0 +1,15 @@
|
|||||||
|
package tech.easyflow.datacenter.connector;
|
||||||
|
|
||||||
|
import com.mybatisflex.core.paginate.Page;
|
||||||
|
import com.mybatisflex.core.row.Row;
|
||||||
|
import tech.easyflow.datacenter.entity.DatacenterTable;
|
||||||
|
import tech.easyflow.datacenter.execution.model.DatacenterQueryRequest;
|
||||||
|
import tech.easyflow.datacenter.meta.entity.DatacenterSource;
|
||||||
|
|
||||||
|
import java.util.List;
|
||||||
|
|
||||||
|
public interface QueryExecutor {
|
||||||
|
Page<Row> queryPage(DatacenterSource source, DatacenterTable table, DatacenterQueryRequest request);
|
||||||
|
|
||||||
|
List<Row> queryBySql(DatacenterSource source, String sql);
|
||||||
|
}
|
||||||
@@ -0,0 +1,8 @@
|
|||||||
|
package tech.easyflow.datacenter.connector;
|
||||||
|
|
||||||
|
import tech.easyflow.datacenter.execution.model.DatacenterConnectionTestResult;
|
||||||
|
import tech.easyflow.datacenter.meta.entity.DatacenterSource;
|
||||||
|
|
||||||
|
public interface SourceHealthChecker {
|
||||||
|
DatacenterConnectionTestResult testConnection(DatacenterSource source);
|
||||||
|
}
|
||||||
@@ -0,0 +1,9 @@
|
|||||||
|
package tech.easyflow.datacenter.connector;
|
||||||
|
|
||||||
|
public interface SqlDialect {
|
||||||
|
String quoteIdentifier(String identifier);
|
||||||
|
|
||||||
|
String qualifyTable(String namespace, String tableName);
|
||||||
|
|
||||||
|
String buildPageSql(String baseSql, long pageNumber, long pageSize);
|
||||||
|
}
|
||||||
@@ -0,0 +1,14 @@
|
|||||||
|
package tech.easyflow.datacenter.connector;
|
||||||
|
|
||||||
|
import com.alibaba.fastjson2.JSONObject;
|
||||||
|
import tech.easyflow.common.entity.LoginAccount;
|
||||||
|
import tech.easyflow.datacenter.entity.DatacenterTable;
|
||||||
|
import tech.easyflow.datacenter.meta.entity.DatacenterSource;
|
||||||
|
|
||||||
|
import java.math.BigInteger;
|
||||||
|
|
||||||
|
public interface WriteExecutor {
|
||||||
|
void saveRow(DatacenterSource source, DatacenterTable table, JSONObject data, LoginAccount account);
|
||||||
|
|
||||||
|
void deleteRow(DatacenterSource source, DatacenterTable table, BigInteger id, LoginAccount account);
|
||||||
|
}
|
||||||
@@ -0,0 +1,24 @@
|
|||||||
|
package tech.easyflow.datacenter.connector.dialect;
|
||||||
|
|
||||||
|
import cn.hutool.core.util.StrUtil;
|
||||||
|
import tech.easyflow.datacenter.connector.SqlDialect;
|
||||||
|
|
||||||
|
public class GaussdbSqlDialect implements SqlDialect {
|
||||||
|
@Override
|
||||||
|
public String quoteIdentifier(String identifier) {
|
||||||
|
return '"' + identifier + '"';
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public String qualifyTable(String namespace, String tableName) {
|
||||||
|
if (StrUtil.isBlank(namespace)) {
|
||||||
|
return quoteIdentifier(tableName);
|
||||||
|
}
|
||||||
|
return quoteIdentifier(namespace) + "." + quoteIdentifier(tableName);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public String buildPageSql(String baseSql, long pageNumber, long pageSize) {
|
||||||
|
return baseSql + " OFFSET ? LIMIT ?";
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,24 @@
|
|||||||
|
package tech.easyflow.datacenter.connector.dialect;
|
||||||
|
|
||||||
|
import cn.hutool.core.util.StrUtil;
|
||||||
|
import tech.easyflow.datacenter.connector.SqlDialect;
|
||||||
|
|
||||||
|
public class MysqlSqlDialect implements SqlDialect {
|
||||||
|
@Override
|
||||||
|
public String quoteIdentifier(String identifier) {
|
||||||
|
return "`" + identifier + "`";
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public String qualifyTable(String namespace, String tableName) {
|
||||||
|
if (StrUtil.isBlank(namespace)) {
|
||||||
|
return quoteIdentifier(tableName);
|
||||||
|
}
|
||||||
|
return quoteIdentifier(namespace) + "." + quoteIdentifier(tableName);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public String buildPageSql(String baseSql, long pageNumber, long pageSize) {
|
||||||
|
return baseSql + " LIMIT ?, ?";
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,24 @@
|
|||||||
|
package tech.easyflow.datacenter.connector.dialect;
|
||||||
|
|
||||||
|
import cn.hutool.core.util.StrUtil;
|
||||||
|
import tech.easyflow.datacenter.connector.SqlDialect;
|
||||||
|
|
||||||
|
public class OracleSqlDialect implements SqlDialect {
|
||||||
|
@Override
|
||||||
|
public String quoteIdentifier(String identifier) {
|
||||||
|
return '"' + identifier + '"';
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public String qualifyTable(String namespace, String tableName) {
|
||||||
|
if (StrUtil.isBlank(namespace)) {
|
||||||
|
return quoteIdentifier(tableName);
|
||||||
|
}
|
||||||
|
return quoteIdentifier(namespace) + "." + quoteIdentifier(tableName);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public String buildPageSql(String baseSql, long pageNumber, long pageSize) {
|
||||||
|
return baseSql + " OFFSET ? ROWS FETCH NEXT ? ROWS ONLY";
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,24 @@
|
|||||||
|
package tech.easyflow.datacenter.connector.dialect;
|
||||||
|
|
||||||
|
import cn.hutool.core.util.StrUtil;
|
||||||
|
import tech.easyflow.datacenter.connector.SqlDialect;
|
||||||
|
|
||||||
|
public class PostgresqlSqlDialect implements SqlDialect {
|
||||||
|
@Override
|
||||||
|
public String quoteIdentifier(String identifier) {
|
||||||
|
return '"' + identifier + '"';
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public String qualifyTable(String namespace, String tableName) {
|
||||||
|
if (StrUtil.isBlank(namespace)) {
|
||||||
|
return quoteIdentifier(tableName);
|
||||||
|
}
|
||||||
|
return quoteIdentifier(namespace) + "." + quoteIdentifier(tableName);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public String buildPageSql(String baseSql, long pageNumber, long pageSize) {
|
||||||
|
return baseSql + " OFFSET ? LIMIT ?";
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,22 @@
|
|||||||
|
package tech.easyflow.datacenter.connector.impl;
|
||||||
|
|
||||||
|
import org.springframework.stereotype.Component;
|
||||||
|
import tech.easyflow.datacenter.connector.support.AbstractInternalTableConnector;
|
||||||
|
import tech.easyflow.datacenter.meta.enums.DatacenterCapability;
|
||||||
|
import tech.easyflow.datacenter.meta.enums.DatacenterSourceType;
|
||||||
|
|
||||||
|
import java.util.EnumSet;
|
||||||
|
|
||||||
|
@Component
|
||||||
|
public class ExcelConnector extends AbstractInternalTableConnector {
|
||||||
|
public ExcelConnector() {
|
||||||
|
super(DatacenterSourceType.EXCEL, EnumSet.of(
|
||||||
|
DatacenterCapability.TEST_CONNECTION,
|
||||||
|
DatacenterCapability.BROWSE_METADATA,
|
||||||
|
DatacenterCapability.READ_QUERY,
|
||||||
|
DatacenterCapability.WRITE_MUTATION,
|
||||||
|
DatacenterCapability.MATERIALIZE,
|
||||||
|
DatacenterCapability.EXPORT
|
||||||
|
));
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,22 @@
|
|||||||
|
package tech.easyflow.datacenter.connector.impl;
|
||||||
|
|
||||||
|
import org.springframework.stereotype.Component;
|
||||||
|
import tech.easyflow.datacenter.connector.support.AbstractInternalTableConnector;
|
||||||
|
import tech.easyflow.datacenter.meta.enums.DatacenterCapability;
|
||||||
|
import tech.easyflow.datacenter.meta.enums.DatacenterSourceType;
|
||||||
|
|
||||||
|
import java.util.EnumSet;
|
||||||
|
|
||||||
|
@Component
|
||||||
|
public class ExcelMaterializedConnector extends AbstractInternalTableConnector {
|
||||||
|
public ExcelMaterializedConnector() {
|
||||||
|
super(DatacenterSourceType.EXCEL_MATERIALIZED, EnumSet.of(
|
||||||
|
DatacenterCapability.TEST_CONNECTION,
|
||||||
|
DatacenterCapability.BROWSE_METADATA,
|
||||||
|
DatacenterCapability.READ_QUERY,
|
||||||
|
DatacenterCapability.WRITE_MUTATION,
|
||||||
|
DatacenterCapability.MATERIALIZE,
|
||||||
|
DatacenterCapability.EXPORT
|
||||||
|
));
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,40 @@
|
|||||||
|
package tech.easyflow.datacenter.connector.impl;
|
||||||
|
|
||||||
|
import com.zaxxer.hikari.HikariDataSource;
|
||||||
|
import org.springframework.stereotype.Component;
|
||||||
|
import tech.easyflow.datacenter.connector.dialect.GaussdbSqlDialect;
|
||||||
|
import tech.easyflow.datacenter.connector.support.AbstractJdbcConnector;
|
||||||
|
import tech.easyflow.datacenter.connector.support.DatacenterDatasourceManager;
|
||||||
|
import tech.easyflow.datacenter.meta.entity.DatacenterSource;
|
||||||
|
import tech.easyflow.datacenter.meta.enums.DatacenterCapability;
|
||||||
|
import tech.easyflow.datacenter.meta.enums.DatacenterSourceType;
|
||||||
|
|
||||||
|
import java.sql.Connection;
|
||||||
|
import java.util.EnumSet;
|
||||||
|
|
||||||
|
@Component
|
||||||
|
public class GaussdbNativeConnector extends AbstractJdbcConnector {
|
||||||
|
|
||||||
|
private final DatacenterDatasourceManager datasourceManager;
|
||||||
|
|
||||||
|
public GaussdbNativeConnector(DatacenterDatasourceManager datasourceManager) {
|
||||||
|
super(DatacenterSourceType.GAUSSDB_NATIVE, new GaussdbSqlDialect(), EnumSet.of(
|
||||||
|
DatacenterCapability.TEST_CONNECTION,
|
||||||
|
DatacenterCapability.BROWSE_METADATA,
|
||||||
|
DatacenterCapability.READ_QUERY
|
||||||
|
));
|
||||||
|
this.datasourceManager = datasourceManager;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
protected <T> T withConnection(DatacenterSource source, boolean cacheable, JdbcCallback<T> callback) throws Exception {
|
||||||
|
HikariDataSource dataSource = cacheable ? datasourceManager.getOrCreateExternalDatasource(source) : datasourceManager.createExternalDatasource(source);
|
||||||
|
try (Connection connection = dataSource.getConnection()) {
|
||||||
|
return callback.apply(connection);
|
||||||
|
} finally {
|
||||||
|
if (!cacheable || source.getId() == null) {
|
||||||
|
dataSource.close();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,40 @@
|
|||||||
|
package tech.easyflow.datacenter.connector.impl;
|
||||||
|
|
||||||
|
import com.zaxxer.hikari.HikariDataSource;
|
||||||
|
import org.springframework.stereotype.Component;
|
||||||
|
import tech.easyflow.datacenter.connector.dialect.MysqlSqlDialect;
|
||||||
|
import tech.easyflow.datacenter.connector.support.AbstractJdbcConnector;
|
||||||
|
import tech.easyflow.datacenter.connector.support.DatacenterDatasourceManager;
|
||||||
|
import tech.easyflow.datacenter.meta.entity.DatacenterSource;
|
||||||
|
import tech.easyflow.datacenter.meta.enums.DatacenterCapability;
|
||||||
|
import tech.easyflow.datacenter.meta.enums.DatacenterSourceType;
|
||||||
|
|
||||||
|
import java.sql.Connection;
|
||||||
|
import java.util.EnumSet;
|
||||||
|
|
||||||
|
@Component
|
||||||
|
public class Gbase8aConnector extends AbstractJdbcConnector {
|
||||||
|
|
||||||
|
private final DatacenterDatasourceManager datasourceManager;
|
||||||
|
|
||||||
|
public Gbase8aConnector(DatacenterDatasourceManager datasourceManager) {
|
||||||
|
super(DatacenterSourceType.GBASE_8A, new MysqlSqlDialect(), EnumSet.of(
|
||||||
|
DatacenterCapability.TEST_CONNECTION,
|
||||||
|
DatacenterCapability.BROWSE_METADATA,
|
||||||
|
DatacenterCapability.READ_QUERY
|
||||||
|
));
|
||||||
|
this.datasourceManager = datasourceManager;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
protected <T> T withConnection(DatacenterSource source, boolean cacheable, JdbcCallback<T> callback) throws Exception {
|
||||||
|
HikariDataSource dataSource = cacheable ? datasourceManager.getOrCreateExternalDatasource(source) : datasourceManager.createExternalDatasource(source);
|
||||||
|
try (Connection connection = dataSource.getConnection()) {
|
||||||
|
return callback.apply(connection);
|
||||||
|
} finally {
|
||||||
|
if (!cacheable || source.getId() == null) {
|
||||||
|
dataSource.close();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,40 @@
|
|||||||
|
package tech.easyflow.datacenter.connector.impl;
|
||||||
|
|
||||||
|
import com.zaxxer.hikari.HikariDataSource;
|
||||||
|
import org.springframework.stereotype.Component;
|
||||||
|
import tech.easyflow.datacenter.connector.dialect.MysqlSqlDialect;
|
||||||
|
import tech.easyflow.datacenter.connector.support.AbstractJdbcConnector;
|
||||||
|
import tech.easyflow.datacenter.connector.support.DatacenterDatasourceManager;
|
||||||
|
import tech.easyflow.datacenter.meta.entity.DatacenterSource;
|
||||||
|
import tech.easyflow.datacenter.meta.enums.DatacenterCapability;
|
||||||
|
import tech.easyflow.datacenter.meta.enums.DatacenterSourceType;
|
||||||
|
|
||||||
|
import java.sql.Connection;
|
||||||
|
import java.util.EnumSet;
|
||||||
|
|
||||||
|
@Component
|
||||||
|
public class Gbase8sConnector extends AbstractJdbcConnector {
|
||||||
|
|
||||||
|
private final DatacenterDatasourceManager datasourceManager;
|
||||||
|
|
||||||
|
public Gbase8sConnector(DatacenterDatasourceManager datasourceManager) {
|
||||||
|
super(DatacenterSourceType.GBASE_8S, new MysqlSqlDialect(), EnumSet.of(
|
||||||
|
DatacenterCapability.TEST_CONNECTION,
|
||||||
|
DatacenterCapability.BROWSE_METADATA,
|
||||||
|
DatacenterCapability.READ_QUERY
|
||||||
|
));
|
||||||
|
this.datasourceManager = datasourceManager;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
protected <T> T withConnection(DatacenterSource source, boolean cacheable, JdbcCallback<T> callback) throws Exception {
|
||||||
|
HikariDataSource dataSource = cacheable ? datasourceManager.getOrCreateExternalDatasource(source) : datasourceManager.createExternalDatasource(source);
|
||||||
|
try (Connection connection = dataSource.getConnection()) {
|
||||||
|
return callback.apply(connection);
|
||||||
|
} finally {
|
||||||
|
if (!cacheable || source.getId() == null) {
|
||||||
|
dataSource.close();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,40 @@
|
|||||||
|
package tech.easyflow.datacenter.connector.impl;
|
||||||
|
|
||||||
|
import com.zaxxer.hikari.HikariDataSource;
|
||||||
|
import org.springframework.stereotype.Component;
|
||||||
|
import tech.easyflow.datacenter.connector.dialect.MysqlSqlDialect;
|
||||||
|
import tech.easyflow.datacenter.connector.support.AbstractJdbcConnector;
|
||||||
|
import tech.easyflow.datacenter.connector.support.DatacenterDatasourceManager;
|
||||||
|
import tech.easyflow.datacenter.meta.entity.DatacenterSource;
|
||||||
|
import tech.easyflow.datacenter.meta.enums.DatacenterCapability;
|
||||||
|
import tech.easyflow.datacenter.meta.enums.DatacenterSourceType;
|
||||||
|
|
||||||
|
import java.sql.Connection;
|
||||||
|
import java.util.EnumSet;
|
||||||
|
|
||||||
|
@Component
|
||||||
|
public class MysqlConnector extends AbstractJdbcConnector {
|
||||||
|
|
||||||
|
private final DatacenterDatasourceManager datasourceManager;
|
||||||
|
|
||||||
|
public MysqlConnector(DatacenterDatasourceManager datasourceManager) {
|
||||||
|
super(DatacenterSourceType.MYSQL, new MysqlSqlDialect(), EnumSet.of(
|
||||||
|
DatacenterCapability.TEST_CONNECTION,
|
||||||
|
DatacenterCapability.BROWSE_METADATA,
|
||||||
|
DatacenterCapability.READ_QUERY
|
||||||
|
));
|
||||||
|
this.datasourceManager = datasourceManager;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
protected <T> T withConnection(DatacenterSource source, boolean cacheable, JdbcCallback<T> callback) throws Exception {
|
||||||
|
HikariDataSource dataSource = cacheable ? datasourceManager.getOrCreateExternalDatasource(source) : datasourceManager.createExternalDatasource(source);
|
||||||
|
try (Connection connection = dataSource.getConnection()) {
|
||||||
|
return callback.apply(connection);
|
||||||
|
} finally {
|
||||||
|
if (!cacheable || source.getId() == null) {
|
||||||
|
dataSource.close();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,40 @@
|
|||||||
|
package tech.easyflow.datacenter.connector.impl;
|
||||||
|
|
||||||
|
import com.zaxxer.hikari.HikariDataSource;
|
||||||
|
import org.springframework.stereotype.Component;
|
||||||
|
import tech.easyflow.datacenter.connector.dialect.OracleSqlDialect;
|
||||||
|
import tech.easyflow.datacenter.connector.support.AbstractJdbcConnector;
|
||||||
|
import tech.easyflow.datacenter.connector.support.DatacenterDatasourceManager;
|
||||||
|
import tech.easyflow.datacenter.meta.entity.DatacenterSource;
|
||||||
|
import tech.easyflow.datacenter.meta.enums.DatacenterCapability;
|
||||||
|
import tech.easyflow.datacenter.meta.enums.DatacenterSourceType;
|
||||||
|
|
||||||
|
import java.sql.Connection;
|
||||||
|
import java.util.EnumSet;
|
||||||
|
|
||||||
|
@Component
|
||||||
|
public class OracleConnector extends AbstractJdbcConnector {
|
||||||
|
|
||||||
|
private final DatacenterDatasourceManager datasourceManager;
|
||||||
|
|
||||||
|
public OracleConnector(DatacenterDatasourceManager datasourceManager) {
|
||||||
|
super(DatacenterSourceType.ORACLE, new OracleSqlDialect(), EnumSet.of(
|
||||||
|
DatacenterCapability.TEST_CONNECTION,
|
||||||
|
DatacenterCapability.BROWSE_METADATA,
|
||||||
|
DatacenterCapability.READ_QUERY
|
||||||
|
));
|
||||||
|
this.datasourceManager = datasourceManager;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
protected <T> T withConnection(DatacenterSource source, boolean cacheable, JdbcCallback<T> callback) throws Exception {
|
||||||
|
HikariDataSource dataSource = cacheable ? datasourceManager.getOrCreateExternalDatasource(source) : datasourceManager.createExternalDatasource(source);
|
||||||
|
try (Connection connection = dataSource.getConnection()) {
|
||||||
|
return callback.apply(connection);
|
||||||
|
} finally {
|
||||||
|
if (!cacheable || source.getId() == null) {
|
||||||
|
dataSource.close();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,40 @@
|
|||||||
|
package tech.easyflow.datacenter.connector.impl;
|
||||||
|
|
||||||
|
import com.zaxxer.hikari.HikariDataSource;
|
||||||
|
import org.springframework.stereotype.Component;
|
||||||
|
import tech.easyflow.datacenter.connector.dialect.PostgresqlSqlDialect;
|
||||||
|
import tech.easyflow.datacenter.connector.support.AbstractJdbcConnector;
|
||||||
|
import tech.easyflow.datacenter.connector.support.DatacenterDatasourceManager;
|
||||||
|
import tech.easyflow.datacenter.meta.entity.DatacenterSource;
|
||||||
|
import tech.easyflow.datacenter.meta.enums.DatacenterCapability;
|
||||||
|
import tech.easyflow.datacenter.meta.enums.DatacenterSourceType;
|
||||||
|
|
||||||
|
import java.sql.Connection;
|
||||||
|
import java.util.EnumSet;
|
||||||
|
|
||||||
|
@Component
|
||||||
|
public class PostgresqlConnector extends AbstractJdbcConnector {
|
||||||
|
|
||||||
|
private final DatacenterDatasourceManager datasourceManager;
|
||||||
|
|
||||||
|
public PostgresqlConnector(DatacenterDatasourceManager datasourceManager) {
|
||||||
|
super(DatacenterSourceType.POSTGRESQL, new PostgresqlSqlDialect(), EnumSet.of(
|
||||||
|
DatacenterCapability.TEST_CONNECTION,
|
||||||
|
DatacenterCapability.BROWSE_METADATA,
|
||||||
|
DatacenterCapability.READ_QUERY
|
||||||
|
));
|
||||||
|
this.datasourceManager = datasourceManager;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
protected <T> T withConnection(DatacenterSource source, boolean cacheable, JdbcCallback<T> callback) throws Exception {
|
||||||
|
HikariDataSource dataSource = cacheable ? datasourceManager.getOrCreateExternalDatasource(source) : datasourceManager.createExternalDatasource(source);
|
||||||
|
try (Connection connection = dataSource.getConnection()) {
|
||||||
|
return callback.apply(connection);
|
||||||
|
} finally {
|
||||||
|
if (!cacheable || source.getId() == null) {
|
||||||
|
dataSource.close();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,135 @@
|
|||||||
|
package tech.easyflow.datacenter.connector.impl;
|
||||||
|
|
||||||
|
import cn.hutool.core.util.StrUtil;
|
||||||
|
import com.alibaba.fastjson2.JSONObject;
|
||||||
|
import com.mybatisflex.core.paginate.Page;
|
||||||
|
import com.mybatisflex.core.row.Row;
|
||||||
|
import org.springframework.stereotype.Component;
|
||||||
|
import tech.easyflow.common.entity.LoginAccount;
|
||||||
|
import tech.easyflow.common.web.exceptions.BusinessException;
|
||||||
|
import tech.easyflow.datacenter.connector.dialect.MysqlSqlDialect;
|
||||||
|
import tech.easyflow.datacenter.connector.support.AbstractJdbcConnector;
|
||||||
|
import tech.easyflow.datacenter.entity.DatacenterTable;
|
||||||
|
import tech.easyflow.datacenter.entity.DatacenterTableField;
|
||||||
|
import tech.easyflow.datacenter.execution.model.DatacenterQueryRequest;
|
||||||
|
import tech.easyflow.datacenter.meta.entity.DatacenterSource;
|
||||||
|
import tech.easyflow.datacenter.meta.enums.DatacenterCapability;
|
||||||
|
import tech.easyflow.datacenter.meta.enums.DatacenterSourceType;
|
||||||
|
|
||||||
|
import javax.sql.DataSource;
|
||||||
|
import java.math.BigInteger;
|
||||||
|
import java.sql.Connection;
|
||||||
|
import java.sql.PreparedStatement;
|
||||||
|
import java.util.ArrayList;
|
||||||
|
import java.util.EnumSet;
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.stream.Collectors;
|
||||||
|
|
||||||
|
@Component
|
||||||
|
public class ProjectMysqlConnector extends AbstractJdbcConnector {
|
||||||
|
|
||||||
|
private final DataSource dataSource;
|
||||||
|
private final MysqlSqlDialect dialect = new MysqlSqlDialect();
|
||||||
|
|
||||||
|
public ProjectMysqlConnector(DataSource dataSource) {
|
||||||
|
super(DatacenterSourceType.PROJECT_MYSQL, new MysqlSqlDialect(), EnumSet.of(
|
||||||
|
DatacenterCapability.TEST_CONNECTION,
|
||||||
|
DatacenterCapability.BROWSE_METADATA,
|
||||||
|
DatacenterCapability.READ_QUERY,
|
||||||
|
DatacenterCapability.WRITE_MUTATION
|
||||||
|
));
|
||||||
|
this.dataSource = dataSource;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
protected <T> T withConnection(DatacenterSource source, boolean cacheable, JdbcCallback<T> callback) throws Exception {
|
||||||
|
try (Connection connection = dataSource.getConnection()) {
|
||||||
|
return callback.apply(connection);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
protected boolean requiresJdbcUrl() {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public Page<Row> queryPage(DatacenterSource source, DatacenterTable table, DatacenterQueryRequest request) {
|
||||||
|
return super.queryPage(source, table, request);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void saveRow(DatacenterSource source, DatacenterTable table, JSONObject data, LoginAccount account) {
|
||||||
|
List<DatacenterTableField> writableFields = table.getFields().stream()
|
||||||
|
.filter(field -> field.getWritable() == null || field.getWritable() == 1)
|
||||||
|
.collect(Collectors.toList());
|
||||||
|
Object id = data.get("id");
|
||||||
|
try (Connection connection = dataSource.getConnection()) {
|
||||||
|
if (id == null) {
|
||||||
|
List<String> columns = new ArrayList<>();
|
||||||
|
List<Object> values = new ArrayList<>();
|
||||||
|
for (DatacenterTableField field : writableFields) {
|
||||||
|
Object value = data.get(field.getFieldName());
|
||||||
|
if (value != null) {
|
||||||
|
columns.add(field.getFieldName());
|
||||||
|
values.add(value);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (columns.isEmpty()) {
|
||||||
|
throw new BusinessException("没有可写字段");
|
||||||
|
}
|
||||||
|
String sql = "INSERT INTO " + dialect.qualifyTable(source.getDatabaseName(), resolvePhysicalTableName(table))
|
||||||
|
+ " (" + columns.stream().map(dialect::quoteIdentifier).collect(Collectors.joining(",")) + ") VALUES ("
|
||||||
|
+ columns.stream().map(item -> "?").collect(Collectors.joining(",")) + ")";
|
||||||
|
try (PreparedStatement statement = connection.prepareStatement(sql)) {
|
||||||
|
for (int i = 0; i < values.size(); i++) {
|
||||||
|
statement.setObject(i + 1, values.get(i));
|
||||||
|
}
|
||||||
|
statement.executeUpdate();
|
||||||
|
}
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
List<String> setClauses = new ArrayList<>();
|
||||||
|
List<Object> values = new ArrayList<>();
|
||||||
|
for (DatacenterTableField field : writableFields) {
|
||||||
|
if (!data.containsKey(field.getFieldName())) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
setClauses.add(dialect.quoteIdentifier(field.getFieldName()) + " = ?");
|
||||||
|
values.add(data.get(field.getFieldName()));
|
||||||
|
}
|
||||||
|
if (setClauses.isEmpty()) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
String sql = "UPDATE " + dialect.qualifyTable(source.getDatabaseName(), resolvePhysicalTableName(table))
|
||||||
|
+ " SET " + String.join(",", setClauses)
|
||||||
|
+ " WHERE " + dialect.quoteIdentifier("id") + " = ?";
|
||||||
|
values.add(id);
|
||||||
|
try (PreparedStatement statement = connection.prepareStatement(sql)) {
|
||||||
|
for (int i = 0; i < values.size(); i++) {
|
||||||
|
statement.setObject(i + 1, values.get(i));
|
||||||
|
}
|
||||||
|
statement.executeUpdate();
|
||||||
|
}
|
||||||
|
} catch (Exception ex) {
|
||||||
|
throw new BusinessException("项目 MySQL 写入失败: " + ex.getMessage());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void deleteRow(DatacenterSource source, DatacenterTable table, BigInteger id, LoginAccount account) {
|
||||||
|
String sql = "DELETE FROM " + dialect.qualifyTable(source.getDatabaseName(), resolvePhysicalTableName(table))
|
||||||
|
+ " WHERE " + dialect.quoteIdentifier("id") + " = ?";
|
||||||
|
try (Connection connection = dataSource.getConnection(); PreparedStatement statement = connection.prepareStatement(sql)) {
|
||||||
|
statement.setObject(1, id);
|
||||||
|
statement.executeUpdate();
|
||||||
|
} catch (Exception ex) {
|
||||||
|
throw new BusinessException("项目 MySQL 删除失败: " + ex.getMessage());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
protected String resolveCatalogName(DatacenterSource source, String requestedCatalogName) {
|
||||||
|
return StrUtil.blankToDefault(requestedCatalogName, source.getDatabaseName());
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,152 @@
|
|||||||
|
package tech.easyflow.datacenter.connector.support;
|
||||||
|
|
||||||
|
import cn.hutool.core.util.StrUtil;
|
||||||
|
import com.alibaba.fastjson2.JSONObject;
|
||||||
|
import com.mybatisflex.core.paginate.Page;
|
||||||
|
import com.mybatisflex.core.query.QueryWrapper;
|
||||||
|
import com.mybatisflex.core.row.Db;
|
||||||
|
import com.mybatisflex.core.row.Row;
|
||||||
|
import com.mybatisflex.core.row.RowKey;
|
||||||
|
import org.springframework.util.CollectionUtils;
|
||||||
|
import tech.easyflow.common.entity.LoginAccount;
|
||||||
|
import tech.easyflow.common.web.exceptions.BusinessException;
|
||||||
|
import tech.easyflow.datacenter.connector.DatacenterConnector;
|
||||||
|
import tech.easyflow.datacenter.entity.DatacenterTable;
|
||||||
|
import tech.easyflow.datacenter.entity.DatacenterTableField;
|
||||||
|
import tech.easyflow.datacenter.execution.model.DatacenterConnectionTestResult;
|
||||||
|
import tech.easyflow.datacenter.execution.model.DatacenterQueryRequest;
|
||||||
|
import tech.easyflow.datacenter.meta.entity.DatacenterSource;
|
||||||
|
import tech.easyflow.datacenter.meta.enums.DatacenterCapability;
|
||||||
|
import tech.easyflow.datacenter.meta.enums.DatacenterSourceType;
|
||||||
|
import tech.easyflow.datacenter.meta.model.DatacenterCatalogMeta;
|
||||||
|
import tech.easyflow.datacenter.meta.model.DatacenterTableDetailMeta;
|
||||||
|
|
||||||
|
import java.math.BigDecimal;
|
||||||
|
import java.math.BigInteger;
|
||||||
|
import java.util.*;
|
||||||
|
|
||||||
|
public abstract class AbstractInternalTableConnector implements DatacenterConnector {
|
||||||
|
|
||||||
|
private final DatacenterSourceType sourceType;
|
||||||
|
private final Set<DatacenterCapability> capabilities;
|
||||||
|
|
||||||
|
protected AbstractInternalTableConnector(DatacenterSourceType sourceType, Set<DatacenterCapability> capabilities) {
|
||||||
|
this.sourceType = sourceType;
|
||||||
|
this.capabilities = capabilities;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public DatacenterSourceType getSourceType() {
|
||||||
|
return sourceType;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public Set<DatacenterCapability> getCapabilities() {
|
||||||
|
return capabilities;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public DatacenterConnectionTestResult testConnection(DatacenterSource source) {
|
||||||
|
DatacenterConnectionTestResult result = new DatacenterConnectionTestResult();
|
||||||
|
result.setSuccess(true);
|
||||||
|
result.setMessage("连接成功");
|
||||||
|
result.setCapabilities(capabilities.stream().map(Enum::name).toList());
|
||||||
|
result.setDetails(Map.of("sourceType", sourceType.name()));
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public List<DatacenterCatalogMeta> listCatalogs(DatacenterSource source) {
|
||||||
|
return Collections.emptyList();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public List<DatacenterTable> listTables(DatacenterSource source, String catalogName) {
|
||||||
|
return Collections.emptyList();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public DatacenterTableDetailMeta getTableDetail(DatacenterSource source, String catalogName, String tableName) {
|
||||||
|
throw new BusinessException("内部数据源请从元数据注册表读取表结构");
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public Page<Row> queryPage(DatacenterSource source, DatacenterTable table, DatacenterQueryRequest request) {
|
||||||
|
String actualTable = resolveTableName(table);
|
||||||
|
QueryWrapper wrapper = QueryWrapper.create();
|
||||||
|
if (StrUtil.isNotBlank(request.getWhere())) {
|
||||||
|
wrapper.where(request.getWhere());
|
||||||
|
}
|
||||||
|
long count = Db.selectCountByQuery(actualTable, wrapper);
|
||||||
|
if (count == 0) {
|
||||||
|
return new Page<>(new ArrayList<>(), request.getPageNumber(), request.getPageSize(), count);
|
||||||
|
}
|
||||||
|
Page<Row> page = Db.paginate(actualTable, new Page<>(request.getPageNumber(), request.getPageSize(), count), wrapper);
|
||||||
|
normalizeRows(page.getRecords());
|
||||||
|
return page;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public List<Row> queryBySql(DatacenterSource source, String sql) {
|
||||||
|
List<Row> rows = Db.selectListBySql(sql);
|
||||||
|
normalizeRows(rows);
|
||||||
|
return rows;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void saveRow(DatacenterSource source, DatacenterTable table, JSONObject data, LoginAccount account) {
|
||||||
|
List<DatacenterTableField> fields = table.getFields();
|
||||||
|
if (CollectionUtils.isEmpty(fields)) {
|
||||||
|
throw new BusinessException("数据集字段为空,无法写入");
|
||||||
|
}
|
||||||
|
String actualTable = resolveTableName(table);
|
||||||
|
Object id = data.get("id");
|
||||||
|
if (id == null) {
|
||||||
|
Row row = Row.ofKey(RowKey.SNOW_FLAKE_ID);
|
||||||
|
row.put("dept_id", account.getDeptId());
|
||||||
|
row.put("tenant_id", account.getTenantId());
|
||||||
|
row.put("created", new Date());
|
||||||
|
row.put("created_by", account.getId());
|
||||||
|
row.put("modified", new Date());
|
||||||
|
row.put("modified_by", account.getId());
|
||||||
|
row.put("remark", data.get("remark"));
|
||||||
|
for (DatacenterTableField field : fields) {
|
||||||
|
row.put(field.getFieldName(), data.get(field.getFieldName()));
|
||||||
|
}
|
||||||
|
Db.insert(actualTable, row);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
Row row = Row.ofKey("id", id);
|
||||||
|
row.put("modified", new Date());
|
||||||
|
row.put("modified_by", account.getId());
|
||||||
|
for (DatacenterTableField field : fields) {
|
||||||
|
row.put(field.getFieldName(), data.get(field.getFieldName()));
|
||||||
|
}
|
||||||
|
Db.updateById(actualTable, row);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void deleteRow(DatacenterSource source, DatacenterTable table, BigInteger id, LoginAccount account) {
|
||||||
|
Db.deleteById(resolveTableName(table), Row.ofKey("id", id));
|
||||||
|
}
|
||||||
|
|
||||||
|
protected String resolveTableName(DatacenterTable table) {
|
||||||
|
return StrUtil.blankToDefault(table.getMaterializedTable(), table.getActualTable());
|
||||||
|
}
|
||||||
|
|
||||||
|
private void normalizeRows(List<Row> records) {
|
||||||
|
for (Row record : records) {
|
||||||
|
Map<String, Object> converted = new LinkedHashMap<>();
|
||||||
|
for (Map.Entry<String, Object> entry : record.entrySet()) {
|
||||||
|
Object value = entry.getValue();
|
||||||
|
if (value instanceof BigInteger || value instanceof BigDecimal || value instanceof Long) {
|
||||||
|
converted.put(entry.getKey(), value.toString());
|
||||||
|
} else {
|
||||||
|
converted.put(entry.getKey(), value);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
record.clear();
|
||||||
|
record.putAll(converted);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,523 @@
|
|||||||
|
package tech.easyflow.datacenter.connector.support;
|
||||||
|
|
||||||
|
import cn.hutool.core.util.StrUtil;
|
||||||
|
import com.alibaba.fastjson2.JSONObject;
|
||||||
|
import com.mybatisflex.core.paginate.Page;
|
||||||
|
import com.mybatisflex.core.row.Row;
|
||||||
|
import org.springframework.util.CollectionUtils;
|
||||||
|
import tech.easyflow.common.constant.enums.EnumFieldType;
|
||||||
|
import tech.easyflow.common.entity.LoginAccount;
|
||||||
|
import tech.easyflow.common.web.exceptions.BusinessException;
|
||||||
|
import tech.easyflow.datacenter.connector.DatacenterConnector;
|
||||||
|
import tech.easyflow.datacenter.connector.SqlDialect;
|
||||||
|
import tech.easyflow.datacenter.entity.DatacenterTable;
|
||||||
|
import tech.easyflow.datacenter.entity.DatacenterTableField;
|
||||||
|
import tech.easyflow.datacenter.execution.model.DatacenterConnectionTestResult;
|
||||||
|
import tech.easyflow.datacenter.execution.model.DatacenterQueryFilter;
|
||||||
|
import tech.easyflow.datacenter.execution.model.DatacenterQueryRequest;
|
||||||
|
import tech.easyflow.datacenter.execution.model.DatacenterQuerySort;
|
||||||
|
import tech.easyflow.datacenter.meta.entity.DatacenterSource;
|
||||||
|
import tech.easyflow.datacenter.meta.enums.DatacenterCapability;
|
||||||
|
import tech.easyflow.datacenter.meta.enums.DatacenterConnectionErrorCode;
|
||||||
|
import tech.easyflow.datacenter.meta.enums.DatacenterSourceType;
|
||||||
|
import tech.easyflow.datacenter.meta.enums.DatacenterTableKind;
|
||||||
|
import tech.easyflow.datacenter.meta.model.DatacenterCatalogMeta;
|
||||||
|
import tech.easyflow.datacenter.meta.model.DatacenterTableDetailMeta;
|
||||||
|
|
||||||
|
import java.math.BigDecimal;
|
||||||
|
import java.math.BigInteger;
|
||||||
|
import java.sql.*;
|
||||||
|
import java.util.*;
|
||||||
|
import java.util.stream.Collectors;
|
||||||
|
|
||||||
|
public abstract class AbstractJdbcConnector implements DatacenterConnector {
|
||||||
|
|
||||||
|
private final DatacenterSourceType sourceType;
|
||||||
|
private final SqlDialect sqlDialect;
|
||||||
|
private final Set<DatacenterCapability> capabilities;
|
||||||
|
|
||||||
|
protected AbstractJdbcConnector(DatacenterSourceType sourceType, SqlDialect sqlDialect, Set<DatacenterCapability> capabilities) {
|
||||||
|
this.sourceType = sourceType;
|
||||||
|
this.sqlDialect = sqlDialect;
|
||||||
|
this.capabilities = capabilities;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public DatacenterSourceType getSourceType() {
|
||||||
|
return sourceType;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public Set<DatacenterCapability> getCapabilities() {
|
||||||
|
return capabilities;
|
||||||
|
}
|
||||||
|
|
||||||
|
protected abstract <T> T withConnection(DatacenterSource source, boolean cacheable, JdbcCallback<T> callback) throws Exception;
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public DatacenterConnectionTestResult testConnection(DatacenterSource source) {
|
||||||
|
DatacenterConnectionTestResult result = new DatacenterConnectionTestResult();
|
||||||
|
result.setCapabilities(capabilities.stream().map(Enum::name).toList());
|
||||||
|
if (StrUtil.isBlank(source.getJdbcUrl()) && requiresJdbcUrl()) {
|
||||||
|
result.setSuccess(false);
|
||||||
|
result.setErrorCode(DatacenterConnectionErrorCode.INVALID_ARGUMENT.name());
|
||||||
|
result.setMessage("缺少 JDBC URL");
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
if (StrUtil.isNotBlank(source.getDriverClassName())) {
|
||||||
|
Class.forName(source.getDriverClassName());
|
||||||
|
}
|
||||||
|
Map<String, Object> details = withConnection(source, false, connection -> {
|
||||||
|
DatabaseMetaData metaData = connection.getMetaData();
|
||||||
|
Map<String, Object> map = new LinkedHashMap<>();
|
||||||
|
map.put("databaseProductName", metaData.getDatabaseProductName());
|
||||||
|
map.put("databaseProductVersion", metaData.getDatabaseProductVersion());
|
||||||
|
map.put("url", metaData.getURL());
|
||||||
|
return map;
|
||||||
|
});
|
||||||
|
result.setSuccess(true);
|
||||||
|
result.setMessage("连接成功");
|
||||||
|
result.setDetails(details);
|
||||||
|
return result;
|
||||||
|
} catch (ClassNotFoundException ex) {
|
||||||
|
result.setSuccess(false);
|
||||||
|
result.setErrorCode(DatacenterConnectionErrorCode.DRIVER_NOT_FOUND.name());
|
||||||
|
result.setMessage(ex.getMessage());
|
||||||
|
return result;
|
||||||
|
} catch (SQLException ex) {
|
||||||
|
result.setSuccess(false);
|
||||||
|
result.setErrorCode(mapSqlError(ex));
|
||||||
|
result.setMessage(ex.getMessage());
|
||||||
|
return result;
|
||||||
|
} catch (Exception ex) {
|
||||||
|
result.setSuccess(false);
|
||||||
|
result.setErrorCode(DatacenterConnectionErrorCode.UNKNOWN_ERROR.name());
|
||||||
|
result.setMessage(ex.getMessage());
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public List<DatacenterCatalogMeta> listCatalogs(DatacenterSource source) {
|
||||||
|
try {
|
||||||
|
return withConnection(source, true, connection -> {
|
||||||
|
List<DatacenterCatalogMeta> result = new ArrayList<>();
|
||||||
|
DatabaseMetaData metaData = connection.getMetaData();
|
||||||
|
try (ResultSet catalogs = metaData.getCatalogs()) {
|
||||||
|
while (catalogs.next()) {
|
||||||
|
String name = catalogs.getString("TABLE_CAT");
|
||||||
|
if (StrUtil.isBlank(name)) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
DatacenterCatalogMeta meta = new DatacenterCatalogMeta();
|
||||||
|
meta.setSourceId(source.getId());
|
||||||
|
meta.setCatalogName(name);
|
||||||
|
meta.setCatalogType("CATALOG");
|
||||||
|
result.add(meta);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
try (ResultSet schemas = metaData.getSchemas()) {
|
||||||
|
while (schemas.next()) {
|
||||||
|
String name = schemas.getString("TABLE_SCHEM");
|
||||||
|
if (StrUtil.isBlank(name) || containsCatalog(result, name)) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
DatacenterCatalogMeta meta = new DatacenterCatalogMeta();
|
||||||
|
meta.setSourceId(source.getId());
|
||||||
|
meta.setCatalogName(name);
|
||||||
|
meta.setCatalogType("SCHEMA");
|
||||||
|
result.add(meta);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
result = filterConfiguredCatalogs(source, result);
|
||||||
|
if (result.isEmpty()) {
|
||||||
|
String fallback = resolveCatalogName(source, null);
|
||||||
|
if (StrUtil.isNotBlank(fallback)) {
|
||||||
|
DatacenterCatalogMeta meta = new DatacenterCatalogMeta();
|
||||||
|
meta.setSourceId(source.getId());
|
||||||
|
meta.setCatalogName(fallback);
|
||||||
|
meta.setCatalogType("DEFAULT");
|
||||||
|
result.add(meta);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return result;
|
||||||
|
});
|
||||||
|
} catch (Exception ex) {
|
||||||
|
throw DatacenterConnectorExceptionSupport.wrapAccessException("读取目录失败", ex);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public List<DatacenterTable> listTables(DatacenterSource source, String catalogName) {
|
||||||
|
try {
|
||||||
|
return withConnection(source, true, connection -> {
|
||||||
|
DatabaseMetaData metaData = connection.getMetaData();
|
||||||
|
List<DatacenterTable> tables = new ArrayList<>();
|
||||||
|
try (ResultSet resultSet = metaData.getTables(resolveCatalogArgument(source, catalogName), resolveSchemaArgument(source, catalogName), "%", new String[]{"TABLE", "VIEW"})) {
|
||||||
|
while (resultSet.next()) {
|
||||||
|
DatacenterTable table = new DatacenterTable();
|
||||||
|
table.setSourceId(source.getId());
|
||||||
|
table.setTableName(resultSet.getString("TABLE_NAME"));
|
||||||
|
table.setTableDesc(resultSet.getString("REMARKS"));
|
||||||
|
table.setActualTable(resultSet.getString("TABLE_NAME"));
|
||||||
|
table.setMaterializedTable(resultSet.getString("TABLE_NAME"));
|
||||||
|
table.setAccessMode(capabilities.contains(DatacenterCapability.WRITE_MUTATION) ? "READ_WRITE" : "READ_ONLY");
|
||||||
|
table.setTableKind(resolveTableKind(resultSet.getString("TABLE_TYPE")).name());
|
||||||
|
table.setCapabilitiesJson(Map.of("capabilities", capabilities.stream().map(Enum::name).toList()));
|
||||||
|
tables.add(table);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return tables;
|
||||||
|
});
|
||||||
|
} catch (Exception ex) {
|
||||||
|
throw DatacenterConnectorExceptionSupport.wrapAccessException("读取表列表失败", ex);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public DatacenterTableDetailMeta getTableDetail(DatacenterSource source, String catalogName, String tableName) {
|
||||||
|
try {
|
||||||
|
return withConnection(source, true, connection -> {
|
||||||
|
DatabaseMetaData metaData = connection.getMetaData();
|
||||||
|
DatacenterTableDetailMeta detail = new DatacenterTableDetailMeta();
|
||||||
|
DatacenterTable table = new DatacenterTable();
|
||||||
|
table.setSourceId(source.getId());
|
||||||
|
table.setTableName(tableName);
|
||||||
|
table.setActualTable(tableName);
|
||||||
|
table.setMaterializedTable(tableName);
|
||||||
|
table.setAccessMode(capabilities.contains(DatacenterCapability.WRITE_MUTATION) ? "READ_WRITE" : "READ_ONLY");
|
||||||
|
table.setTableKind(DatacenterTableKind.EXTERNAL_TABLE.name());
|
||||||
|
table.setCapabilitiesJson(Map.of("capabilities", capabilities.stream().map(Enum::name).toList()));
|
||||||
|
detail.setTable(table);
|
||||||
|
|
||||||
|
try (ResultSet tableSet = metaData.getTables(
|
||||||
|
resolveCatalogArgument(source, catalogName),
|
||||||
|
resolveSchemaArgument(source, catalogName),
|
||||||
|
tableName,
|
||||||
|
new String[]{"TABLE", "VIEW"})) {
|
||||||
|
while (tableSet.next()) {
|
||||||
|
String currentTableName = tableSet.getString("TABLE_NAME");
|
||||||
|
if (!matchesTableName(currentTableName, tableName)) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
table.setTableDesc(tableSet.getString("REMARKS"));
|
||||||
|
table.setTableKind(resolveTableKind(tableSet.getString("TABLE_TYPE")).name());
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
Set<String> primaryKeys = new HashSet<>();
|
||||||
|
try (ResultSet pkSet = metaData.getPrimaryKeys(resolveCatalogArgument(source, catalogName), resolveSchemaArgument(source, catalogName), tableName)) {
|
||||||
|
while (pkSet.next()) {
|
||||||
|
primaryKeys.add(pkSet.getString("COLUMN_NAME"));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
List<DatacenterTableField> fields = new ArrayList<>();
|
||||||
|
try (ResultSet columns = metaData.getColumns(resolveCatalogArgument(source, catalogName), resolveSchemaArgument(source, catalogName), tableName, "%")) {
|
||||||
|
while (columns.next()) {
|
||||||
|
DatacenterTableField field = new DatacenterTableField();
|
||||||
|
field.setFieldName(columns.getString("COLUMN_NAME"));
|
||||||
|
field.setSourceColumnName(columns.getString("COLUMN_NAME"));
|
||||||
|
field.setFieldDesc(columns.getString("REMARKS"));
|
||||||
|
field.setJdbcType(columns.getString("TYPE_NAME"));
|
||||||
|
field.setPrecision(columns.getInt("COLUMN_SIZE"));
|
||||||
|
field.setScale(columns.getInt("DECIMAL_DIGITS"));
|
||||||
|
field.setRequired(columns.getInt("NULLABLE") == DatabaseMetaData.columnNoNulls ? 1 : 0);
|
||||||
|
field.setQueryable(1);
|
||||||
|
field.setSortable(1);
|
||||||
|
field.setWritable(capabilities.contains(DatacenterCapability.WRITE_MUTATION) ? 1 : 0);
|
||||||
|
field.setIndexed(primaryKeys.contains(field.getFieldName()) ? 1 : 0);
|
||||||
|
field.setFieldType(mapFieldType(columns.getInt("DATA_TYPE")));
|
||||||
|
fields.add(field);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
detail.setFields(fields);
|
||||||
|
return detail;
|
||||||
|
});
|
||||||
|
} catch (Exception ex) {
|
||||||
|
throw DatacenterConnectorExceptionSupport.wrapAccessException("读取表详情失败", ex);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public Page<Row> queryPage(DatacenterSource source, DatacenterTable table, DatacenterQueryRequest request) {
|
||||||
|
if (!capabilities.contains(DatacenterCapability.READ_QUERY)) {
|
||||||
|
throw new BusinessException("当前数据源暂不支持查询");
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
return withConnection(source, true, connection -> doQueryPage(connection, source, table, request));
|
||||||
|
} catch (Exception ex) {
|
||||||
|
throw DatacenterConnectorExceptionSupport.wrapAccessException("查询失败", ex);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public List<Row> queryBySql(DatacenterSource source, String sql) {
|
||||||
|
if (!capabilities.contains(DatacenterCapability.READ_QUERY)) {
|
||||||
|
throw new BusinessException("当前数据源暂不支持查询");
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
return withConnection(source, true, connection -> doQueryBySql(connection, sql));
|
||||||
|
} catch (Exception ex) {
|
||||||
|
throw DatacenterConnectorExceptionSupport.wrapAccessException("SQL 查询失败", ex);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void saveRow(DatacenterSource source, DatacenterTable table, JSONObject data, LoginAccount account) {
|
||||||
|
throw new BusinessException("当前数据源不支持写入");
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void deleteRow(DatacenterSource source, DatacenterTable table, BigInteger id, LoginAccount account) {
|
||||||
|
throw new BusinessException("当前数据源不支持删除");
|
||||||
|
}
|
||||||
|
|
||||||
|
protected boolean requiresJdbcUrl() {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
protected Page<Row> doQueryPage(Connection connection, DatacenterSource source, DatacenterTable table, DatacenterQueryRequest request) throws SQLException {
|
||||||
|
List<Object> params = new ArrayList<>();
|
||||||
|
String selectColumns = CollectionUtils.isEmpty(request.getSelectedColumns())
|
||||||
|
? "*"
|
||||||
|
: request.getSelectedColumns().stream().map(sqlDialect::quoteIdentifier).collect(Collectors.joining(", "));
|
||||||
|
String qualifiedTable = sqlDialect.qualifyTable(resolveCatalogName(source, request.getDatasetRef() == null ? null : request.getDatasetRef().getCatalogName()), resolvePhysicalTableName(table));
|
||||||
|
StringBuilder whereClause = new StringBuilder();
|
||||||
|
if (StrUtil.isNotBlank(request.getWhere())) {
|
||||||
|
whereClause.append(" WHERE ").append(request.getWhere());
|
||||||
|
} else if (!CollectionUtils.isEmpty(request.getFilters())) {
|
||||||
|
whereClause.append(" WHERE 1=1 ");
|
||||||
|
for (DatacenterQueryFilter filter : request.getFilters()) {
|
||||||
|
appendFilter(whereClause, params, filter);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
String orderClause = buildOrderClause(request.getSorts());
|
||||||
|
String baseSql = "SELECT " + selectColumns + " FROM " + qualifiedTable + whereClause + orderClause;
|
||||||
|
String countSql = "SELECT COUNT(1) FROM " + qualifiedTable + whereClause;
|
||||||
|
long total = queryCount(connection, countSql, params);
|
||||||
|
if (total == 0L) {
|
||||||
|
return new Page<>(new ArrayList<>(), request.getPageNumber(), request.getPageSize(), total);
|
||||||
|
}
|
||||||
|
String pageSql = sqlDialect.buildPageSql(baseSql, request.getPageNumber(), request.getPageSize());
|
||||||
|
List<Object> pageParams = new ArrayList<>(params);
|
||||||
|
if (pageSql.contains("FETCH NEXT") || pageSql.toLowerCase(Locale.ROOT).contains("limit")) {
|
||||||
|
pageParams.add((request.getPageNumber() - 1) * request.getPageSize());
|
||||||
|
pageParams.add(request.getPageSize());
|
||||||
|
}
|
||||||
|
List<Row> records = new ArrayList<>();
|
||||||
|
try (PreparedStatement statement = connection.prepareStatement(pageSql)) {
|
||||||
|
bindParameters(statement, pageParams);
|
||||||
|
try (ResultSet resultSet = statement.executeQuery()) {
|
||||||
|
ResultSetMetaData metaData = resultSet.getMetaData();
|
||||||
|
while (resultSet.next()) {
|
||||||
|
Row row = new Row();
|
||||||
|
for (int i = 1; i <= metaData.getColumnCount(); i++) {
|
||||||
|
String columnLabel = metaData.getColumnLabel(i);
|
||||||
|
row.put(columnLabel, normalizeValue(resultSet.getObject(i)));
|
||||||
|
}
|
||||||
|
records.add(row);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return new Page<>(records, request.getPageNumber(), request.getPageSize(), total);
|
||||||
|
}
|
||||||
|
|
||||||
|
protected String resolvePhysicalTableName(DatacenterTable table) {
|
||||||
|
return StrUtil.blankToDefault(table.getActualTable(), table.getTableName());
|
||||||
|
}
|
||||||
|
|
||||||
|
protected List<Row> doQueryBySql(Connection connection, String sql) throws SQLException {
|
||||||
|
try (PreparedStatement statement = connection.prepareStatement(sql);
|
||||||
|
ResultSet resultSet = statement.executeQuery()) {
|
||||||
|
return readRows(resultSet);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
protected String resolveCatalogName(DatacenterSource source, String requestedCatalogName) {
|
||||||
|
if (StrUtil.isNotBlank(requestedCatalogName)) {
|
||||||
|
return requestedCatalogName;
|
||||||
|
}
|
||||||
|
if (usesCatalogNamespace()) {
|
||||||
|
return source.getDatabaseName();
|
||||||
|
}
|
||||||
|
return source.getSchemaName();
|
||||||
|
}
|
||||||
|
|
||||||
|
protected List<Row> readRows(ResultSet resultSet) throws SQLException {
|
||||||
|
List<Row> records = new ArrayList<>();
|
||||||
|
ResultSetMetaData metaData = resultSet.getMetaData();
|
||||||
|
while (resultSet.next()) {
|
||||||
|
Row row = new Row();
|
||||||
|
for (int i = 1; i <= metaData.getColumnCount(); i++) {
|
||||||
|
String columnLabel = metaData.getColumnLabel(i);
|
||||||
|
row.put(columnLabel, normalizeValue(resultSet.getObject(i)));
|
||||||
|
}
|
||||||
|
records.add(row);
|
||||||
|
}
|
||||||
|
return records;
|
||||||
|
}
|
||||||
|
|
||||||
|
protected String resolveCatalogArgument(DatacenterSource source, String catalogName) {
|
||||||
|
return usesCatalogNamespace() ? resolveCatalogName(source, catalogName) : source.getDatabaseName();
|
||||||
|
}
|
||||||
|
|
||||||
|
protected String resolveSchemaArgument(DatacenterSource source, String catalogName) {
|
||||||
|
return usesCatalogNamespace() ? source.getSchemaName() : resolveCatalogName(source, catalogName);
|
||||||
|
}
|
||||||
|
|
||||||
|
protected boolean usesCatalogNamespace() {
|
||||||
|
return sourceType == DatacenterSourceType.MYSQL
|
||||||
|
|| sourceType == DatacenterSourceType.PROJECT_MYSQL
|
||||||
|
|| sourceType == DatacenterSourceType.GBASE_8A
|
||||||
|
|| sourceType == DatacenterSourceType.GBASE_8S;
|
||||||
|
}
|
||||||
|
|
||||||
|
private List<DatacenterCatalogMeta> filterConfiguredCatalogs(DatacenterSource source, List<DatacenterCatalogMeta> items) {
|
||||||
|
if (items == null || items.isEmpty() || source == null) {
|
||||||
|
return items;
|
||||||
|
}
|
||||||
|
String configuredName = usesCatalogNamespace()
|
||||||
|
? StrUtil.trimToNull(source.getDatabaseName())
|
||||||
|
: StrUtil.trimToNull(source.getSchemaName());
|
||||||
|
if (StrUtil.isBlank(configuredName)) {
|
||||||
|
return items;
|
||||||
|
}
|
||||||
|
List<DatacenterCatalogMeta> matched = items.stream()
|
||||||
|
.filter(item -> configuredName.equalsIgnoreCase(item.getCatalogName()))
|
||||||
|
.collect(Collectors.toList());
|
||||||
|
return matched.isEmpty() ? items : matched;
|
||||||
|
}
|
||||||
|
|
||||||
|
private boolean containsCatalog(List<DatacenterCatalogMeta> items, String catalogName) {
|
||||||
|
return items.stream().anyMatch(item -> catalogName.equalsIgnoreCase(item.getCatalogName()));
|
||||||
|
}
|
||||||
|
|
||||||
|
private DatacenterTableKind resolveTableKind(String tableType) {
|
||||||
|
return "VIEW".equalsIgnoreCase(tableType) ? DatacenterTableKind.EXTERNAL_VIEW : DatacenterTableKind.EXTERNAL_TABLE;
|
||||||
|
}
|
||||||
|
|
||||||
|
private boolean matchesTableName(String currentTableName, String targetTableName) {
|
||||||
|
if (currentTableName == null || targetTableName == null) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
return currentTableName.equals(targetTableName)
|
||||||
|
|| currentTableName.equalsIgnoreCase(targetTableName);
|
||||||
|
}
|
||||||
|
|
||||||
|
private Integer mapFieldType(int jdbcType) {
|
||||||
|
return switch (jdbcType) {
|
||||||
|
case Types.INTEGER, Types.TINYINT, Types.SMALLINT, Types.BIGINT -> EnumFieldType.INTEGER.getCode();
|
||||||
|
case Types.FLOAT, Types.DOUBLE, Types.REAL, Types.NUMERIC, Types.DECIMAL -> EnumFieldType.NUMBER.getCode();
|
||||||
|
case Types.TIMESTAMP, Types.DATE, Types.TIME -> EnumFieldType.TIME.getCode();
|
||||||
|
case Types.BOOLEAN, Types.BIT -> EnumFieldType.BOOLEAN.getCode();
|
||||||
|
default -> EnumFieldType.STRING.getCode();
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
private void appendFilter(StringBuilder sql, List<Object> params, DatacenterQueryFilter filter) {
|
||||||
|
String operator = StrUtil.blankToDefault(filter.getOperator(), "EQ").toUpperCase(Locale.ROOT);
|
||||||
|
String column = sqlDialect.quoteIdentifier(filter.getColumn());
|
||||||
|
switch (operator) {
|
||||||
|
case "EQ" -> {
|
||||||
|
sql.append(" AND ").append(column).append(" = ?");
|
||||||
|
params.add(filter.getValue());
|
||||||
|
}
|
||||||
|
case "LIKE" -> {
|
||||||
|
sql.append(" AND ").append(column).append(" LIKE ?");
|
||||||
|
params.add("%" + filter.getValue() + "%");
|
||||||
|
}
|
||||||
|
case "GT" -> {
|
||||||
|
sql.append(" AND ").append(column).append(" > ?");
|
||||||
|
params.add(filter.getValue());
|
||||||
|
}
|
||||||
|
case "GTE" -> {
|
||||||
|
sql.append(" AND ").append(column).append(" >= ?");
|
||||||
|
params.add(filter.getValue());
|
||||||
|
}
|
||||||
|
case "LT" -> {
|
||||||
|
sql.append(" AND ").append(column).append(" < ?");
|
||||||
|
params.add(filter.getValue());
|
||||||
|
}
|
||||||
|
case "LTE" -> {
|
||||||
|
sql.append(" AND ").append(column).append(" <= ?");
|
||||||
|
params.add(filter.getValue());
|
||||||
|
}
|
||||||
|
case "IN" -> {
|
||||||
|
List<Object> values = filter.getValues() == null ? List.of() : filter.getValues();
|
||||||
|
if (values.isEmpty()) {
|
||||||
|
sql.append(" AND 1 = 0");
|
||||||
|
} else {
|
||||||
|
sql.append(" AND ").append(column).append(" IN (");
|
||||||
|
sql.append(values.stream().map(item -> "?").collect(Collectors.joining(",")));
|
||||||
|
sql.append(")");
|
||||||
|
params.addAll(values);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
case "IS_NULL" -> sql.append(" AND ").append(column).append(" IS NULL");
|
||||||
|
default -> throw new BusinessException("不支持的过滤操作: " + operator);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private String buildOrderClause(List<DatacenterQuerySort> sorts) {
|
||||||
|
if (CollectionUtils.isEmpty(sorts)) {
|
||||||
|
return "";
|
||||||
|
}
|
||||||
|
return " ORDER BY " + sorts.stream()
|
||||||
|
.map(sort -> sqlDialect.quoteIdentifier(sort.getColumn()) + " " + ("DESC".equalsIgnoreCase(sort.getDirection()) ? "DESC" : "ASC"))
|
||||||
|
.collect(Collectors.joining(", "));
|
||||||
|
}
|
||||||
|
|
||||||
|
private long queryCount(Connection connection, String sql, List<Object> params) throws SQLException {
|
||||||
|
try (PreparedStatement statement = connection.prepareStatement(sql)) {
|
||||||
|
bindParameters(statement, params);
|
||||||
|
try (ResultSet resultSet = statement.executeQuery()) {
|
||||||
|
if (resultSet.next()) {
|
||||||
|
return resultSet.getLong(1);
|
||||||
|
}
|
||||||
|
return 0L;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private void bindParameters(PreparedStatement statement, List<Object> params) throws SQLException {
|
||||||
|
for (int i = 0; i < params.size(); i++) {
|
||||||
|
statement.setObject(i + 1, params.get(i));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private Object normalizeValue(Object value) {
|
||||||
|
if (value instanceof BigDecimal || value instanceof BigInteger || value instanceof Long) {
|
||||||
|
return value.toString();
|
||||||
|
}
|
||||||
|
return value;
|
||||||
|
}
|
||||||
|
|
||||||
|
private String mapSqlError(SQLException ex) {
|
||||||
|
String state = ex.getSQLState();
|
||||||
|
String message = ex.getMessage() == null ? "" : ex.getMessage().toLowerCase(Locale.ROOT);
|
||||||
|
if (message.contains("access denied") || message.contains("password") || message.contains("authentication")) {
|
||||||
|
return DatacenterConnectionErrorCode.AUTH_FAILED.name();
|
||||||
|
}
|
||||||
|
if (message.contains("unknown database") || message.contains("database does not exist")) {
|
||||||
|
return DatacenterConnectionErrorCode.DATABASE_NOT_FOUND.name();
|
||||||
|
}
|
||||||
|
if (message.contains("schema") && message.contains("does not exist")) {
|
||||||
|
return DatacenterConnectionErrorCode.SCHEMA_NOT_FOUND.name();
|
||||||
|
}
|
||||||
|
if (message.contains("permission denied") || message.contains("insufficient privilege")) {
|
||||||
|
return DatacenterConnectionErrorCode.PERMISSION_DENIED.name();
|
||||||
|
}
|
||||||
|
if (state != null && state.startsWith("08")) {
|
||||||
|
return DatacenterConnectionErrorCode.NETWORK_UNREACHABLE.name();
|
||||||
|
}
|
||||||
|
return DatacenterConnectionErrorCode.UNKNOWN_ERROR.name();
|
||||||
|
}
|
||||||
|
|
||||||
|
@FunctionalInterface
|
||||||
|
protected interface JdbcCallback<T> {
|
||||||
|
T apply(Connection connection) throws Exception;
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,80 @@
|
|||||||
|
package tech.easyflow.datacenter.connector.support;
|
||||||
|
|
||||||
|
import tech.easyflow.common.web.exceptions.BusinessException;
|
||||||
|
|
||||||
|
import java.net.ConnectException;
|
||||||
|
import java.net.NoRouteToHostException;
|
||||||
|
import java.net.SocketException;
|
||||||
|
import java.net.SocketTimeoutException;
|
||||||
|
import java.net.UnknownHostException;
|
||||||
|
import java.sql.SQLException;
|
||||||
|
import java.util.Locale;
|
||||||
|
|
||||||
|
public final class DatacenterConnectorExceptionSupport {
|
||||||
|
|
||||||
|
public static final String SOURCE_UNAVAILABLE_MESSAGE = "当前连接不可用,请检查连接配置后重试";
|
||||||
|
|
||||||
|
private DatacenterConnectorExceptionSupport() {
|
||||||
|
}
|
||||||
|
|
||||||
|
public static BusinessException wrapAccessException(String fallbackMessage, Exception ex) {
|
||||||
|
if (ex instanceof BusinessException businessException && !isConnectionUnavailable(ex)) {
|
||||||
|
return businessException;
|
||||||
|
}
|
||||||
|
if (isConnectionUnavailable(ex)) {
|
||||||
|
return new BusinessException(SOURCE_UNAVAILABLE_MESSAGE);
|
||||||
|
}
|
||||||
|
return new BusinessException(fallbackMessage);
|
||||||
|
}
|
||||||
|
|
||||||
|
public static boolean isConnectionUnavailable(Throwable throwable) {
|
||||||
|
Throwable current = throwable;
|
||||||
|
while (current != null) {
|
||||||
|
if (current instanceof ClassNotFoundException
|
||||||
|
|| current instanceof ConnectException
|
||||||
|
|| current instanceof NoRouteToHostException
|
||||||
|
|| current instanceof SocketTimeoutException
|
||||||
|
|| current instanceof UnknownHostException) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
if (current instanceof SocketException socketException) {
|
||||||
|
String socketMessage = lowerCase(socketException.getMessage());
|
||||||
|
if (socketMessage.contains("broken pipe")
|
||||||
|
|| socketMessage.contains("connection reset")
|
||||||
|
|| socketMessage.contains("network is unreachable")) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (current instanceof SQLException sqlException) {
|
||||||
|
String sqlState = sqlException.getSQLState();
|
||||||
|
String message = lowerCase(sqlException.getMessage());
|
||||||
|
if ((sqlState != null && sqlState.startsWith("08"))
|
||||||
|
|| message.contains("communications link failure")
|
||||||
|
|| message.contains("connection refused")
|
||||||
|
|| message.contains("connection attempt failed")
|
||||||
|
|| message.contains("connect timed out")
|
||||||
|
|| message.contains("i/o error")
|
||||||
|
|| message.contains("io error")
|
||||||
|
|| message.contains("the network adapter could not establish the connection")
|
||||||
|
|| message.contains("unknown host")
|
||||||
|
|| message.contains("access denied")
|
||||||
|
|| message.contains("authentication failed")
|
||||||
|
|| message.contains("login failed")
|
||||||
|
|| message.contains("password authentication failed")
|
||||||
|
|| message.contains("unknown database")
|
||||||
|
|| message.contains("database does not exist")
|
||||||
|
|| (message.contains("schema") && message.contains("does not exist"))
|
||||||
|
|| message.contains("insufficient privilege")
|
||||||
|
|| message.contains("permission denied")) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
current = current.getCause();
|
||||||
|
}
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
private static String lowerCase(String message) {
|
||||||
|
return message == null ? "" : message.toLowerCase(Locale.ROOT);
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,44 @@
|
|||||||
|
package tech.easyflow.datacenter.connector.support;
|
||||||
|
|
||||||
|
import com.zaxxer.hikari.HikariConfig;
|
||||||
|
import com.zaxxer.hikari.HikariDataSource;
|
||||||
|
import org.springframework.stereotype.Component;
|
||||||
|
import tech.easyflow.datacenter.meta.entity.DatacenterSource;
|
||||||
|
import tech.easyflow.datacenter.security.DatacenterCredentialCipher;
|
||||||
|
|
||||||
|
import javax.annotation.Resource;
|
||||||
|
import java.math.BigInteger;
|
||||||
|
import java.util.Map;
|
||||||
|
import java.util.concurrent.ConcurrentHashMap;
|
||||||
|
|
||||||
|
@Component
|
||||||
|
public class DatacenterDatasourceManager {
|
||||||
|
|
||||||
|
private final Map<BigInteger, HikariDataSource> externalDatasourceCache = new ConcurrentHashMap<>();
|
||||||
|
|
||||||
|
@Resource
|
||||||
|
private DatacenterCredentialCipher credentialCipher;
|
||||||
|
|
||||||
|
public HikariDataSource createExternalDatasource(DatacenterSource source) {
|
||||||
|
HikariConfig config = new HikariConfig();
|
||||||
|
config.setPoolName("dc-ext-" + (source.getId() == null ? "temp" : source.getId()));
|
||||||
|
config.setJdbcUrl(source.getJdbcUrl());
|
||||||
|
config.setUsername(source.getUsername());
|
||||||
|
config.setPassword(credentialCipher.decrypt(source.getCredentialCipher()));
|
||||||
|
config.setMaximumPoolSize(3);
|
||||||
|
config.setMinimumIdle(0);
|
||||||
|
config.setConnectionTimeout(5000);
|
||||||
|
config.setValidationTimeout(3000);
|
||||||
|
if (source.getDriverClassName() != null && !source.getDriverClassName().isBlank()) {
|
||||||
|
config.setDriverClassName(source.getDriverClassName());
|
||||||
|
}
|
||||||
|
return new HikariDataSource(config);
|
||||||
|
}
|
||||||
|
|
||||||
|
public HikariDataSource getOrCreateExternalDatasource(DatacenterSource source) {
|
||||||
|
if (source.getId() == null) {
|
||||||
|
return createExternalDatasource(source);
|
||||||
|
}
|
||||||
|
return externalDatasourceCache.computeIfAbsent(source.getId(), key -> createExternalDatasource(source));
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -33,6 +33,18 @@ public class DatacenterTableBase extends DateEntity implements Serializable {
|
|||||||
@Column(tenantId = true, comment = "租户ID")
|
@Column(tenantId = true, comment = "租户ID")
|
||||||
private BigInteger tenantId;
|
private BigInteger tenantId;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 数据源ID
|
||||||
|
*/
|
||||||
|
@Column(comment = "数据源ID")
|
||||||
|
private BigInteger sourceId;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 目录ID
|
||||||
|
*/
|
||||||
|
@Column(comment = "目录ID")
|
||||||
|
private BigInteger catalogId;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 数据表名
|
* 数据表名
|
||||||
*/
|
*/
|
||||||
@@ -51,6 +63,30 @@ public class DatacenterTableBase extends DateEntity implements Serializable {
|
|||||||
@Column(comment = "物理表名")
|
@Column(comment = "物理表名")
|
||||||
private String actualTable;
|
private String actualTable;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 表类型
|
||||||
|
*/
|
||||||
|
@Column(comment = "表类型")
|
||||||
|
private String tableKind;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 访问模式
|
||||||
|
*/
|
||||||
|
@Column(comment = "访问模式")
|
||||||
|
private String accessMode;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 物化表名
|
||||||
|
*/
|
||||||
|
@Column(comment = "物化表名")
|
||||||
|
private String materializedTable;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 是否开启版本
|
||||||
|
*/
|
||||||
|
@Column(comment = "是否开启版本")
|
||||||
|
private Integer versioningEnabled;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 数据状态
|
* 数据状态
|
||||||
*/
|
*/
|
||||||
@@ -87,6 +123,12 @@ public class DatacenterTableBase extends DateEntity implements Serializable {
|
|||||||
@Column(typeHandler = FastjsonTypeHandler.class, comment = "扩展项")
|
@Column(typeHandler = FastjsonTypeHandler.class, comment = "扩展项")
|
||||||
private Map<String, Object> options;
|
private Map<String, Object> options;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 能力声明
|
||||||
|
*/
|
||||||
|
@Column(typeHandler = FastjsonTypeHandler.class, comment = "能力声明")
|
||||||
|
private Map<String, Object> capabilitiesJson;
|
||||||
|
|
||||||
public BigInteger getId() {
|
public BigInteger getId() {
|
||||||
return id;
|
return id;
|
||||||
}
|
}
|
||||||
@@ -111,6 +153,22 @@ public class DatacenterTableBase extends DateEntity implements Serializable {
|
|||||||
this.tenantId = tenantId;
|
this.tenantId = tenantId;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public BigInteger getSourceId() {
|
||||||
|
return sourceId;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setSourceId(BigInteger sourceId) {
|
||||||
|
this.sourceId = sourceId;
|
||||||
|
}
|
||||||
|
|
||||||
|
public BigInteger getCatalogId() {
|
||||||
|
return catalogId;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setCatalogId(BigInteger catalogId) {
|
||||||
|
this.catalogId = catalogId;
|
||||||
|
}
|
||||||
|
|
||||||
public String getTableName() {
|
public String getTableName() {
|
||||||
return tableName;
|
return tableName;
|
||||||
}
|
}
|
||||||
@@ -135,6 +193,38 @@ public class DatacenterTableBase extends DateEntity implements Serializable {
|
|||||||
this.actualTable = actualTable;
|
this.actualTable = actualTable;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public String getTableKind() {
|
||||||
|
return tableKind;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setTableKind(String tableKind) {
|
||||||
|
this.tableKind = tableKind;
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getAccessMode() {
|
||||||
|
return accessMode;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setAccessMode(String accessMode) {
|
||||||
|
this.accessMode = accessMode;
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getMaterializedTable() {
|
||||||
|
return materializedTable;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setMaterializedTable(String materializedTable) {
|
||||||
|
this.materializedTable = materializedTable;
|
||||||
|
}
|
||||||
|
|
||||||
|
public Integer getVersioningEnabled() {
|
||||||
|
return versioningEnabled;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setVersioningEnabled(Integer versioningEnabled) {
|
||||||
|
this.versioningEnabled = versioningEnabled;
|
||||||
|
}
|
||||||
|
|
||||||
public Integer getStatus() {
|
public Integer getStatus() {
|
||||||
return status;
|
return status;
|
||||||
}
|
}
|
||||||
@@ -183,4 +273,12 @@ public class DatacenterTableBase extends DateEntity implements Serializable {
|
|||||||
this.options = options;
|
this.options = options;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public Map<String, Object> getCapabilitiesJson() {
|
||||||
|
return capabilitiesJson;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setCapabilitiesJson(Map<String, Object> capabilitiesJson) {
|
||||||
|
this.capabilitiesJson = capabilitiesJson;
|
||||||
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -33,6 +33,12 @@ public class DatacenterTableFieldBase extends DateEntity implements Serializable
|
|||||||
@Column(comment = "字段名称")
|
@Column(comment = "字段名称")
|
||||||
private String fieldName;
|
private String fieldName;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 源字段名
|
||||||
|
*/
|
||||||
|
@Column(comment = "源字段名")
|
||||||
|
private String sourceColumnName;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 字段描述
|
* 字段描述
|
||||||
*/
|
*/
|
||||||
@@ -45,12 +51,54 @@ public class DatacenterTableFieldBase extends DateEntity implements Serializable
|
|||||||
@Column(comment = "字段类型")
|
@Column(comment = "字段类型")
|
||||||
private Integer fieldType;
|
private Integer fieldType;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* JDBC 类型
|
||||||
|
*/
|
||||||
|
@Column(comment = "JDBC类型")
|
||||||
|
private String jdbcType;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 精度
|
||||||
|
*/
|
||||||
|
@Column(comment = "精度")
|
||||||
|
private Integer precision;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 小数位
|
||||||
|
*/
|
||||||
|
@Column(comment = "小数位")
|
||||||
|
private Integer scale;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 是否必填
|
* 是否必填
|
||||||
*/
|
*/
|
||||||
@Column(comment = "是否必填")
|
@Column(comment = "是否必填")
|
||||||
private Integer required;
|
private Integer required;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 可查询
|
||||||
|
*/
|
||||||
|
@Column(comment = "可查询")
|
||||||
|
private Integer queryable;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 可排序
|
||||||
|
*/
|
||||||
|
@Column(comment = "可排序")
|
||||||
|
private Integer sortable;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 可写入
|
||||||
|
*/
|
||||||
|
@Column(comment = "可写入")
|
||||||
|
private Integer writable;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 是否索引
|
||||||
|
*/
|
||||||
|
@Column(comment = "是否索引")
|
||||||
|
private Integer indexed;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 扩展项
|
* 扩展项
|
||||||
*/
|
*/
|
||||||
@@ -105,6 +153,14 @@ public class DatacenterTableFieldBase extends DateEntity implements Serializable
|
|||||||
this.fieldName = fieldName;
|
this.fieldName = fieldName;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public String getSourceColumnName() {
|
||||||
|
return sourceColumnName;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setSourceColumnName(String sourceColumnName) {
|
||||||
|
this.sourceColumnName = sourceColumnName;
|
||||||
|
}
|
||||||
|
|
||||||
public String getFieldDesc() {
|
public String getFieldDesc() {
|
||||||
return fieldDesc;
|
return fieldDesc;
|
||||||
}
|
}
|
||||||
@@ -121,6 +177,30 @@ public class DatacenterTableFieldBase extends DateEntity implements Serializable
|
|||||||
this.fieldType = fieldType;
|
this.fieldType = fieldType;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public String getJdbcType() {
|
||||||
|
return jdbcType;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setJdbcType(String jdbcType) {
|
||||||
|
this.jdbcType = jdbcType;
|
||||||
|
}
|
||||||
|
|
||||||
|
public Integer getPrecision() {
|
||||||
|
return precision;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setPrecision(Integer precision) {
|
||||||
|
this.precision = precision;
|
||||||
|
}
|
||||||
|
|
||||||
|
public Integer getScale() {
|
||||||
|
return scale;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setScale(Integer scale) {
|
||||||
|
this.scale = scale;
|
||||||
|
}
|
||||||
|
|
||||||
public Integer getRequired() {
|
public Integer getRequired() {
|
||||||
return required;
|
return required;
|
||||||
}
|
}
|
||||||
@@ -129,6 +209,38 @@ public class DatacenterTableFieldBase extends DateEntity implements Serializable
|
|||||||
this.required = required;
|
this.required = required;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public Integer getQueryable() {
|
||||||
|
return queryable;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setQueryable(Integer queryable) {
|
||||||
|
this.queryable = queryable;
|
||||||
|
}
|
||||||
|
|
||||||
|
public Integer getSortable() {
|
||||||
|
return sortable;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setSortable(Integer sortable) {
|
||||||
|
this.sortable = sortable;
|
||||||
|
}
|
||||||
|
|
||||||
|
public Integer getWritable() {
|
||||||
|
return writable;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setWritable(Integer writable) {
|
||||||
|
this.writable = writable;
|
||||||
|
}
|
||||||
|
|
||||||
|
public Integer getIndexed() {
|
||||||
|
return indexed;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setIndexed(Integer indexed) {
|
||||||
|
this.indexed = indexed;
|
||||||
|
}
|
||||||
|
|
||||||
public Map<String, Object> getOptions() {
|
public Map<String, Object> getOptions() {
|
||||||
return options;
|
return options;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,107 +0,0 @@
|
|||||||
package tech.easyflow.datacenter.excel;
|
|
||||||
|
|
||||||
import cn.idev.excel.context.AnalysisContext;
|
|
||||||
import cn.idev.excel.metadata.data.ReadCellData;
|
|
||||||
import cn.idev.excel.read.listener.ReadListener;
|
|
||||||
import com.alibaba.fastjson2.JSONObject;
|
|
||||||
import org.slf4j.Logger;
|
|
||||||
import org.slf4j.LoggerFactory;
|
|
||||||
import tech.easyflow.common.entity.LoginAccount;
|
|
||||||
import tech.easyflow.common.util.SpringContextUtil;
|
|
||||||
import tech.easyflow.datacenter.entity.DatacenterTableField;
|
|
||||||
import tech.easyflow.datacenter.service.DatacenterTableService;
|
|
||||||
|
|
||||||
import java.math.BigInteger;
|
|
||||||
import java.util.*;
|
|
||||||
|
|
||||||
public class ReadDataListener implements ReadListener<LinkedHashMap<Integer, Object>> {
|
|
||||||
|
|
||||||
private static final Logger log = LoggerFactory.getLogger(ReadDataListener.class);
|
|
||||||
|
|
||||||
private BigInteger tableId;
|
|
||||||
|
|
||||||
private List<DatacenterTableField> fields;
|
|
||||||
|
|
||||||
private LoginAccount loginAccount;
|
|
||||||
|
|
||||||
private final Map<String, Integer> headFieldIndex = new HashMap<>();
|
|
||||||
|
|
||||||
private int successCount = 0;
|
|
||||||
private int errorCount = 0;
|
|
||||||
private int totalCount = 0;
|
|
||||||
|
|
||||||
private final List<JSONObject> errorRows = new ArrayList<>();
|
|
||||||
|
|
||||||
public ReadDataListener() {
|
|
||||||
}
|
|
||||||
|
|
||||||
public ReadDataListener(BigInteger tableId, List<DatacenterTableField> fields, LoginAccount loginAccount) {
|
|
||||||
this.tableId = tableId;
|
|
||||||
this.fields = fields;
|
|
||||||
this.loginAccount = loginAccount;
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public void invoke(LinkedHashMap<Integer, Object> o, AnalysisContext analysisContext) {
|
|
||||||
DatacenterTableService service = SpringContextUtil.getBean(DatacenterTableService.class);
|
|
||||||
JSONObject obj = new JSONObject();
|
|
||||||
for (DatacenterTableField field : fields) {
|
|
||||||
String fieldName = field.getFieldName();
|
|
||||||
Integer i = headFieldIndex.get(fieldName);
|
|
||||||
if (i != null) {
|
|
||||||
obj.put(fieldName, o.get(i));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
try {
|
|
||||||
service.saveValue(tableId, obj, loginAccount);
|
|
||||||
successCount++;
|
|
||||||
} catch (Exception e) {
|
|
||||||
errorCount++;
|
|
||||||
log.error("导入数据到数据中枢失败,具体值:{}", obj, e);
|
|
||||||
errorRows.add(obj);
|
|
||||||
}
|
|
||||||
totalCount++;
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public void invokeHead(Map<Integer, ReadCellData<?>> headMap, AnalysisContext context) {
|
|
||||||
Set<Map.Entry<Integer, ReadCellData<?>>> entries = headMap.entrySet();
|
|
||||||
for (Map.Entry<Integer, ReadCellData<?>> entry : entries) {
|
|
||||||
Integer key = entry.getKey();
|
|
||||||
String field = entry.getValue().getStringValue();
|
|
||||||
headFieldIndex.put(field, key);
|
|
||||||
}
|
|
||||||
if (headFieldIndex.size() != fields.size()) {
|
|
||||||
throw new RuntimeException("表头字段数量与表结构对应不上!");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public void doAfterAllAnalysed(AnalysisContext analysisContext) {
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
public List<DatacenterTableField> getFields() {
|
|
||||||
return fields;
|
|
||||||
}
|
|
||||||
|
|
||||||
public void setFields(List<DatacenterTableField> fields) {
|
|
||||||
this.fields = fields;
|
|
||||||
}
|
|
||||||
|
|
||||||
public int getSuccessCount() {
|
|
||||||
return successCount;
|
|
||||||
}
|
|
||||||
|
|
||||||
public int getErrorCount() {
|
|
||||||
return errorCount;
|
|
||||||
}
|
|
||||||
|
|
||||||
public int getTotalCount() {
|
|
||||||
return totalCount;
|
|
||||||
}
|
|
||||||
|
|
||||||
public List<JSONObject> getErrorRows() {
|
|
||||||
return errorRows;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,68 +0,0 @@
|
|||||||
package tech.easyflow.datacenter.excel;
|
|
||||||
|
|
||||||
import com.alibaba.fastjson2.JSONObject;
|
|
||||||
|
|
||||||
import java.util.List;
|
|
||||||
|
|
||||||
public class ReadResVo {
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 成功数
|
|
||||||
*/
|
|
||||||
private int successCount = 0;
|
|
||||||
/**
|
|
||||||
* 失败数
|
|
||||||
*/
|
|
||||||
private int errorCount = 0;
|
|
||||||
/**
|
|
||||||
* 总数
|
|
||||||
*/
|
|
||||||
private int totalCount = 0;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 错误行
|
|
||||||
*/
|
|
||||||
private List<JSONObject> errorRows;
|
|
||||||
|
|
||||||
public ReadResVo() {
|
|
||||||
}
|
|
||||||
|
|
||||||
public ReadResVo(int successCount, int errorCount, int totalCount, List<JSONObject> errorRows) {
|
|
||||||
this.successCount = successCount;
|
|
||||||
this.errorCount = errorCount;
|
|
||||||
this.totalCount = totalCount;
|
|
||||||
this.errorRows = errorRows;
|
|
||||||
}
|
|
||||||
|
|
||||||
public int getSuccessCount() {
|
|
||||||
return successCount;
|
|
||||||
}
|
|
||||||
|
|
||||||
public void setSuccessCount(int successCount) {
|
|
||||||
this.successCount = successCount;
|
|
||||||
}
|
|
||||||
|
|
||||||
public int getErrorCount() {
|
|
||||||
return errorCount;
|
|
||||||
}
|
|
||||||
|
|
||||||
public void setErrorCount(int errorCount) {
|
|
||||||
this.errorCount = errorCount;
|
|
||||||
}
|
|
||||||
|
|
||||||
public int getTotalCount() {
|
|
||||||
return totalCount;
|
|
||||||
}
|
|
||||||
|
|
||||||
public void setTotalCount(int totalCount) {
|
|
||||||
this.totalCount = totalCount;
|
|
||||||
}
|
|
||||||
|
|
||||||
public List<JSONObject> getErrorRows() {
|
|
||||||
return errorRows;
|
|
||||||
}
|
|
||||||
|
|
||||||
public void setErrorRows(List<JSONObject> errorRows) {
|
|
||||||
this.errorRows = errorRows;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -0,0 +1,28 @@
|
|||||||
|
package tech.easyflow.datacenter.excel.model;
|
||||||
|
|
||||||
|
import tech.easyflow.datacenter.execution.model.DatacenterQueryFilter;
|
||||||
|
import tech.easyflow.datacenter.execution.model.DatasetRef;
|
||||||
|
|
||||||
|
import java.util.ArrayList;
|
||||||
|
import java.util.LinkedHashMap;
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.Map;
|
||||||
|
|
||||||
|
public class DatacenterExcelDeriveRequest {
|
||||||
|
private DatasetRef datasetRef;
|
||||||
|
private String targetTableName;
|
||||||
|
private List<String> selectedColumns = new ArrayList<>();
|
||||||
|
private Map<String, String> renameMappings = new LinkedHashMap<>();
|
||||||
|
private List<DatacenterQueryFilter> filters = new ArrayList<>();
|
||||||
|
|
||||||
|
public DatasetRef getDatasetRef() { return datasetRef; }
|
||||||
|
public void setDatasetRef(DatasetRef datasetRef) { this.datasetRef = datasetRef; }
|
||||||
|
public String getTargetTableName() { return targetTableName; }
|
||||||
|
public void setTargetTableName(String targetTableName) { this.targetTableName = targetTableName; }
|
||||||
|
public List<String> getSelectedColumns() { return selectedColumns; }
|
||||||
|
public void setSelectedColumns(List<String> selectedColumns) { this.selectedColumns = selectedColumns; }
|
||||||
|
public Map<String, String> getRenameMappings() { return renameMappings; }
|
||||||
|
public void setRenameMappings(Map<String, String> renameMappings) { this.renameMappings = renameMappings; }
|
||||||
|
public List<DatacenterQueryFilter> getFilters() { return filters; }
|
||||||
|
public void setFilters(List<DatacenterQueryFilter> filters) { this.filters = filters; }
|
||||||
|
}
|
||||||
@@ -0,0 +1,23 @@
|
|||||||
|
package tech.easyflow.datacenter.excel.model;
|
||||||
|
|
||||||
|
import tech.easyflow.datacenter.execution.model.DatasetRef;
|
||||||
|
|
||||||
|
import java.math.BigInteger;
|
||||||
|
import java.util.ArrayList;
|
||||||
|
import java.util.List;
|
||||||
|
|
||||||
|
public class DatacenterExcelExportRequest {
|
||||||
|
private BigInteger sourceId;
|
||||||
|
private BigInteger catalogId;
|
||||||
|
private List<DatasetRef> datasetRefs = new ArrayList<>();
|
||||||
|
private String fileName;
|
||||||
|
|
||||||
|
public BigInteger getSourceId() { return sourceId; }
|
||||||
|
public void setSourceId(BigInteger sourceId) { this.sourceId = sourceId; }
|
||||||
|
public BigInteger getCatalogId() { return catalogId; }
|
||||||
|
public void setCatalogId(BigInteger catalogId) { this.catalogId = catalogId; }
|
||||||
|
public List<DatasetRef> getDatasetRefs() { return datasetRefs; }
|
||||||
|
public void setDatasetRefs(List<DatasetRef> datasetRefs) { this.datasetRefs = datasetRefs; }
|
||||||
|
public String getFileName() { return fileName; }
|
||||||
|
public void setFileName(String fileName) { this.fileName = fileName; }
|
||||||
|
}
|
||||||
@@ -0,0 +1,22 @@
|
|||||||
|
package tech.easyflow.datacenter.excel.model;
|
||||||
|
|
||||||
|
import tech.easyflow.datacenter.execution.model.DatasetRef;
|
||||||
|
|
||||||
|
import java.util.ArrayList;
|
||||||
|
import java.util.List;
|
||||||
|
|
||||||
|
public class DatacenterExcelMergeRequest {
|
||||||
|
private List<DatasetRef> datasetRefs = new ArrayList<>();
|
||||||
|
private String mergeMode;
|
||||||
|
private String targetTableName;
|
||||||
|
private String joinKey;
|
||||||
|
|
||||||
|
public List<DatasetRef> getDatasetRefs() { return datasetRefs; }
|
||||||
|
public void setDatasetRefs(List<DatasetRef> datasetRefs) { this.datasetRefs = datasetRefs; }
|
||||||
|
public String getMergeMode() { return mergeMode; }
|
||||||
|
public void setMergeMode(String mergeMode) { this.mergeMode = mergeMode; }
|
||||||
|
public String getTargetTableName() { return targetTableName; }
|
||||||
|
public void setTargetTableName(String targetTableName) { this.targetTableName = targetTableName; }
|
||||||
|
public String getJoinKey() { return joinKey; }
|
||||||
|
public void setJoinKey(String joinKey) { this.joinKey = joinKey; }
|
||||||
|
}
|
||||||
@@ -0,0 +1,30 @@
|
|||||||
|
package tech.easyflow.datacenter.excel.model;
|
||||||
|
|
||||||
|
import tech.easyflow.datacenter.execution.model.DatasetRef;
|
||||||
|
|
||||||
|
import java.math.BigInteger;
|
||||||
|
|
||||||
|
public class DatacenterExcelSplitRequest {
|
||||||
|
private BigInteger sourceId;
|
||||||
|
private BigInteger catalogId;
|
||||||
|
private DatasetRef datasetRef;
|
||||||
|
private String splitMode;
|
||||||
|
private Integer rowBatchSize;
|
||||||
|
private String fieldName;
|
||||||
|
private String targetNamePrefix;
|
||||||
|
|
||||||
|
public BigInteger getSourceId() { return sourceId; }
|
||||||
|
public void setSourceId(BigInteger sourceId) { this.sourceId = sourceId; }
|
||||||
|
public BigInteger getCatalogId() { return catalogId; }
|
||||||
|
public void setCatalogId(BigInteger catalogId) { this.catalogId = catalogId; }
|
||||||
|
public DatasetRef getDatasetRef() { return datasetRef; }
|
||||||
|
public void setDatasetRef(DatasetRef datasetRef) { this.datasetRef = datasetRef; }
|
||||||
|
public String getSplitMode() { return splitMode; }
|
||||||
|
public void setSplitMode(String splitMode) { this.splitMode = splitMode; }
|
||||||
|
public Integer getRowBatchSize() { return rowBatchSize; }
|
||||||
|
public void setRowBatchSize(Integer rowBatchSize) { this.rowBatchSize = rowBatchSize; }
|
||||||
|
public String getFieldName() { return fieldName; }
|
||||||
|
public void setFieldName(String fieldName) { this.fieldName = fieldName; }
|
||||||
|
public String getTargetNamePrefix() { return targetNamePrefix; }
|
||||||
|
public void setTargetNamePrefix(String targetNamePrefix) { this.targetNamePrefix = targetNamePrefix; }
|
||||||
|
}
|
||||||
@@ -0,0 +1,28 @@
|
|||||||
|
package tech.easyflow.datacenter.excel.service;
|
||||||
|
|
||||||
|
import org.springframework.web.multipart.MultipartFile;
|
||||||
|
import tech.easyflow.common.entity.LoginAccount;
|
||||||
|
import tech.easyflow.datacenter.excel.model.DatacenterExcelDeriveRequest;
|
||||||
|
import tech.easyflow.datacenter.excel.model.DatacenterExcelExportRequest;
|
||||||
|
import tech.easyflow.datacenter.excel.model.DatacenterExcelMergeRequest;
|
||||||
|
import tech.easyflow.datacenter.excel.model.DatacenterExcelSplitRequest;
|
||||||
|
import tech.easyflow.datacenter.meta.entity.DatacenterImportJob;
|
||||||
|
|
||||||
|
import java.math.BigInteger;
|
||||||
|
import java.util.List;
|
||||||
|
|
||||||
|
public interface DatacenterExcelImportService {
|
||||||
|
DatacenterImportJob importWorkbook(MultipartFile file, LoginAccount account) throws Exception;
|
||||||
|
|
||||||
|
DatacenterImportJob splitWorkbook(DatacenterExcelSplitRequest request, LoginAccount account);
|
||||||
|
|
||||||
|
DatacenterImportJob mergeWorkbook(DatacenterExcelMergeRequest request, LoginAccount account);
|
||||||
|
|
||||||
|
DatacenterImportJob deriveWorkbook(DatacenterExcelDeriveRequest request, LoginAccount account);
|
||||||
|
|
||||||
|
DatacenterImportJob exportWorkbook(DatacenterExcelExportRequest request, LoginAccount account) throws Exception;
|
||||||
|
|
||||||
|
DatacenterImportJob getImportJobDetail(BigInteger jobId);
|
||||||
|
|
||||||
|
List<DatacenterImportJob> listJobs(BigInteger sourceId, BigInteger tableId);
|
||||||
|
}
|
||||||
@@ -0,0 +1,950 @@
|
|||||||
|
package tech.easyflow.datacenter.excel.service.impl;
|
||||||
|
|
||||||
|
import com.alibaba.fastjson2.JSONObject;
|
||||||
|
import com.mybatisflex.core.keygen.impl.SnowFlakeIDKeyGenerator;
|
||||||
|
import com.mybatisflex.core.paginate.Page;
|
||||||
|
import com.mybatisflex.core.row.Row;
|
||||||
|
import org.apache.poi.ss.usermodel.Cell;
|
||||||
|
import org.apache.poi.ss.usermodel.DataFormatter;
|
||||||
|
import org.apache.poi.ss.usermodel.Sheet;
|
||||||
|
import org.apache.poi.ss.usermodel.Workbook;
|
||||||
|
import org.apache.poi.ss.usermodel.WorkbookFactory;
|
||||||
|
import org.apache.poi.xssf.streaming.SXSSFWorkbook;
|
||||||
|
import org.springframework.stereotype.Service;
|
||||||
|
import org.springframework.transaction.annotation.Transactional;
|
||||||
|
import org.springframework.util.CollectionUtils;
|
||||||
|
import org.springframework.web.multipart.MultipartFile;
|
||||||
|
import tech.easyflow.common.constant.enums.EnumFieldType;
|
||||||
|
import tech.easyflow.common.entity.LoginAccount;
|
||||||
|
import tech.easyflow.common.web.exceptions.BusinessException;
|
||||||
|
import tech.easyflow.datacenter.adapter.DbHandleManager;
|
||||||
|
import tech.easyflow.datacenter.entity.DatacenterTable;
|
||||||
|
import tech.easyflow.datacenter.entity.DatacenterTableField;
|
||||||
|
import tech.easyflow.datacenter.excel.model.DatacenterExcelDeriveRequest;
|
||||||
|
import tech.easyflow.datacenter.excel.model.DatacenterExcelExportRequest;
|
||||||
|
import tech.easyflow.datacenter.excel.model.DatacenterExcelMergeRequest;
|
||||||
|
import tech.easyflow.datacenter.excel.model.DatacenterExcelSplitRequest;
|
||||||
|
import tech.easyflow.datacenter.excel.service.DatacenterExcelImportService;
|
||||||
|
import tech.easyflow.datacenter.execution.model.DatacenterQueryFilter;
|
||||||
|
import tech.easyflow.datacenter.execution.model.DatacenterQueryRequest;
|
||||||
|
import tech.easyflow.datacenter.execution.model.DatasetRef;
|
||||||
|
import tech.easyflow.datacenter.execution.service.DatacenterDatasetQueryService;
|
||||||
|
import tech.easyflow.datacenter.mapper.DatacenterDatasetVersionMapper;
|
||||||
|
import tech.easyflow.datacenter.mapper.DatacenterDerivedTableMapper;
|
||||||
|
import tech.easyflow.datacenter.mapper.DatacenterImportJobMapper;
|
||||||
|
import tech.easyflow.datacenter.meta.entity.DatacenterCatalog;
|
||||||
|
import tech.easyflow.datacenter.meta.entity.DatacenterDatasetVersion;
|
||||||
|
import tech.easyflow.datacenter.meta.entity.DatacenterDerivedTable;
|
||||||
|
import tech.easyflow.datacenter.meta.entity.DatacenterImportJob;
|
||||||
|
import tech.easyflow.datacenter.meta.entity.DatacenterSource;
|
||||||
|
import tech.easyflow.datacenter.meta.enums.DatacenterImportStatus;
|
||||||
|
import tech.easyflow.datacenter.meta.enums.DatacenterSourceType;
|
||||||
|
import tech.easyflow.datacenter.meta.enums.DatacenterTableKind;
|
||||||
|
import tech.easyflow.datacenter.meta.model.DatacenterTableDetailMeta;
|
||||||
|
import tech.easyflow.datacenter.meta.service.DatacenterDatasetRegistryService;
|
||||||
|
import tech.easyflow.datacenter.meta.service.DatacenterSourceService;
|
||||||
|
|
||||||
|
import javax.annotation.Resource;
|
||||||
|
import java.io.File;
|
||||||
|
import java.io.FileOutputStream;
|
||||||
|
import java.io.InputStream;
|
||||||
|
import java.math.BigInteger;
|
||||||
|
import java.nio.file.Files;
|
||||||
|
import java.nio.file.Path;
|
||||||
|
import java.time.LocalDateTime;
|
||||||
|
import java.time.format.DateTimeFormatter;
|
||||||
|
import java.util.ArrayList;
|
||||||
|
import java.util.Date;
|
||||||
|
import java.util.HashMap;
|
||||||
|
import java.util.HashSet;
|
||||||
|
import java.util.LinkedHashMap;
|
||||||
|
import java.util.LinkedHashSet;
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.Locale;
|
||||||
|
import java.util.Map;
|
||||||
|
import java.util.Objects;
|
||||||
|
import java.util.Set;
|
||||||
|
import java.util.UUID;
|
||||||
|
|
||||||
|
@Service
|
||||||
|
public class DatacenterExcelImportServiceImpl implements DatacenterExcelImportService {
|
||||||
|
|
||||||
|
private static final long QUERY_BATCH_SIZE = 500L;
|
||||||
|
private static final DateTimeFormatter EXPORT_TIME_FORMAT = DateTimeFormatter.ofPattern("yyyyMMddHHmmss");
|
||||||
|
|
||||||
|
@Resource
|
||||||
|
private DatacenterSourceService sourceService;
|
||||||
|
@Resource
|
||||||
|
private DatacenterDatasetRegistryService registryService;
|
||||||
|
@Resource
|
||||||
|
private DatacenterImportJobMapper importJobMapper;
|
||||||
|
@Resource
|
||||||
|
private DatacenterDatasetVersionMapper datasetVersionMapper;
|
||||||
|
@Resource
|
||||||
|
private DatacenterDerivedTableMapper derivedTableMapper;
|
||||||
|
@Resource
|
||||||
|
private DatacenterDatasetQueryService queryService;
|
||||||
|
@Resource
|
||||||
|
private DbHandleManager dbHandleManager;
|
||||||
|
|
||||||
|
@Override
|
||||||
|
@Transactional(rollbackFor = Exception.class)
|
||||||
|
public DatacenterImportJob importWorkbook(MultipartFile file, LoginAccount account) throws Exception {
|
||||||
|
if (file == null || file.isEmpty()) {
|
||||||
|
throw new BusinessException("Excel 文件不能为空");
|
||||||
|
}
|
||||||
|
String workbookName = extractWorkbookName(file.getOriginalFilename());
|
||||||
|
DatacenterSource source = new DatacenterSource();
|
||||||
|
source.setSourceName(workbookName);
|
||||||
|
source.setSourceCode("EXCEL_" + UUID.randomUUID());
|
||||||
|
source.setSourceType(DatacenterSourceType.EXCEL.name());
|
||||||
|
source.setAccessMode("READ_WRITE");
|
||||||
|
source.setBuiltinFlag(0);
|
||||||
|
source.setConfigJson(Map.of("originFileName", file.getOriginalFilename()));
|
||||||
|
source = sourceService.saveSource(source, account);
|
||||||
|
DatacenterCatalog catalog = registryService.ensureCatalog(source, workbookName, account);
|
||||||
|
|
||||||
|
DatacenterImportJob job = createJob("EXCEL_IMPORT", source.getId(), catalog.getId(), null,
|
||||||
|
file.getOriginalFilename(), Map.of("operation", "import"), account);
|
||||||
|
|
||||||
|
long totalRows = 0L;
|
||||||
|
long successRows = 0L;
|
||||||
|
List<BigInteger> createdTableIds = new ArrayList<>();
|
||||||
|
try (InputStream inputStream = file.getInputStream(); Workbook workbook = WorkbookFactory.create(inputStream)) {
|
||||||
|
DataFormatter formatter = new DataFormatter();
|
||||||
|
for (int sheetIndex = 0; sheetIndex < workbook.getNumberOfSheets(); sheetIndex++) {
|
||||||
|
Sheet sheet = workbook.getSheetAt(sheetIndex);
|
||||||
|
org.apache.poi.ss.usermodel.Row headerRow = sheet.getRow(sheet.getFirstRowNum());
|
||||||
|
if (headerRow == null) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
List<DatacenterTableField> fields = buildFields(headerRow, formatter);
|
||||||
|
if (fields.isEmpty()) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
DatacenterTable table = new DatacenterTable();
|
||||||
|
table.setTableName(uniqueTableName(source.getId(), catalog.getId(), sheet.getSheetName()));
|
||||||
|
table.setTableDesc(sheet.getSheetName());
|
||||||
|
table.setActualTable(buildMaterializedTableName(source.getId(), sheetIndex));
|
||||||
|
table.setMaterializedTable(table.getActualTable());
|
||||||
|
table.setTableKind(DatacenterTableKind.EXCEL_MATERIALIZED.name());
|
||||||
|
table.setAccessMode("READ_WRITE");
|
||||||
|
table.setVersioningEnabled(1);
|
||||||
|
table.setCapabilitiesJson(defaultExcelCapabilities());
|
||||||
|
table.setFields(fields);
|
||||||
|
|
||||||
|
DatacenterTableDetailMeta detail = new DatacenterTableDetailMeta();
|
||||||
|
detail.setTable(table);
|
||||||
|
detail.setFields(fields);
|
||||||
|
|
||||||
|
dbHandleManager.getDbHandler().createTable(table);
|
||||||
|
DatacenterTable savedTable = registryService.registerTable(source, catalog, detail, account);
|
||||||
|
savedTable.setFields(registryService.getFields(savedTable.getId()));
|
||||||
|
createdTableIds.add(savedTable.getId());
|
||||||
|
|
||||||
|
for (int rowIndex = sheet.getFirstRowNum() + 1; rowIndex <= sheet.getLastRowNum(); rowIndex++) {
|
||||||
|
org.apache.poi.ss.usermodel.Row row = sheet.getRow(rowIndex);
|
||||||
|
if (row == null) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
JSONObject payload = new JSONObject();
|
||||||
|
boolean hasValue = false;
|
||||||
|
for (int cellIndex = 0; cellIndex < savedTable.getFields().size(); cellIndex++) {
|
||||||
|
String value = formatter.formatCellValue(row.getCell(cellIndex));
|
||||||
|
if (value != null && !value.isBlank()) {
|
||||||
|
hasValue = true;
|
||||||
|
}
|
||||||
|
payload.put(savedTable.getFields().get(cellIndex).getFieldName(), value);
|
||||||
|
}
|
||||||
|
if (!hasValue) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
dbHandleManager.getDbHandler().saveValue(savedTable, payload, account);
|
||||||
|
totalRows++;
|
||||||
|
successRows++;
|
||||||
|
}
|
||||||
|
|
||||||
|
createVersion(savedTable, "initial-import", Map.of(
|
||||||
|
"sheetName", sheet.getSheetName(),
|
||||||
|
"sourceId", source.getId(),
|
||||||
|
"originFileName", file.getOriginalFilename()
|
||||||
|
), account);
|
||||||
|
}
|
||||||
|
job.setTableId(createdTableIds.isEmpty() ? null : createdTableIds.get(0));
|
||||||
|
finishJobSuccess(job, totalRows, successRows, Map.of("tableIds", createdTableIds));
|
||||||
|
return job;
|
||||||
|
} catch (Exception ex) {
|
||||||
|
finishJobFailure(job, ex);
|
||||||
|
throw ex;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
@Transactional(rollbackFor = Exception.class)
|
||||||
|
public DatacenterImportJob splitWorkbook(DatacenterExcelSplitRequest request, LoginAccount account) {
|
||||||
|
DatasetRef datasetRef = request == null ? null : request.getDatasetRef();
|
||||||
|
DatacenterImportJob job = createJob("EXCEL_SPLIT",
|
||||||
|
request == null ? null : request.getSourceId(),
|
||||||
|
request == null ? null : request.getCatalogId(),
|
||||||
|
datasetRef == null ? null : datasetRef.getTableId(),
|
||||||
|
null,
|
||||||
|
buildPayload("request", request),
|
||||||
|
account);
|
||||||
|
try {
|
||||||
|
String splitMode = normalizeMode(request == null ? null : request.getSplitMode(), "BY_ROW_COUNT");
|
||||||
|
return switch (splitMode) {
|
||||||
|
case "BY_SHEET" -> doSplitBySheet(request, account, job);
|
||||||
|
case "BY_FIELD_VALUE" -> doSplitByFieldValue(request, account, job);
|
||||||
|
default -> doSplitByRowCount(request, account, job);
|
||||||
|
};
|
||||||
|
} catch (Exception ex) {
|
||||||
|
finishJobFailure(job, ex);
|
||||||
|
throw ex;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
@Transactional(rollbackFor = Exception.class)
|
||||||
|
public DatacenterImportJob mergeWorkbook(DatacenterExcelMergeRequest request, LoginAccount account) {
|
||||||
|
if (request == null || CollectionUtils.isEmpty(request.getDatasetRefs())) {
|
||||||
|
throw new BusinessException("合并数据集不能为空");
|
||||||
|
}
|
||||||
|
DatacenterImportJob job = createJob("EXCEL_MERGE", null, null, null, null, buildPayload("request", request), account);
|
||||||
|
try {
|
||||||
|
String mergeMode = normalizeMode(request.getMergeMode(), "VERTICAL");
|
||||||
|
return switch (mergeMode) {
|
||||||
|
case "HORIZONTAL" -> doHorizontalMerge(request, account, job);
|
||||||
|
default -> doVerticalMerge(request, account, job);
|
||||||
|
};
|
||||||
|
} catch (Exception ex) {
|
||||||
|
finishJobFailure(job, ex);
|
||||||
|
throw ex;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
@Transactional(rollbackFor = Exception.class)
|
||||||
|
public DatacenterImportJob deriveWorkbook(DatacenterExcelDeriveRequest request, LoginAccount account) {
|
||||||
|
if (request == null || request.getDatasetRef() == null) {
|
||||||
|
throw new BusinessException("派生数据集不能为空");
|
||||||
|
}
|
||||||
|
DatacenterImportJob job = createJob("EXCEL_DERIVE",
|
||||||
|
request.getDatasetRef().getSourceId(),
|
||||||
|
request.getDatasetRef().getCatalogId(),
|
||||||
|
request.getDatasetRef().getTableId(),
|
||||||
|
null,
|
||||||
|
buildPayload("request", request),
|
||||||
|
account);
|
||||||
|
try {
|
||||||
|
DatacenterTable sourceTable = resolveTable(request.getDatasetRef());
|
||||||
|
DatacenterSource source = registryService.getSourceRequired(sourceTable.getSourceId());
|
||||||
|
DatacenterCatalog catalog = requireCatalog(sourceTable.getCatalogId());
|
||||||
|
List<String> selectedColumns = resolveSelectedColumns(sourceTable, request.getSelectedColumns());
|
||||||
|
List<DatacenterTableField> targetFields = buildDerivedFields(sourceTable, request);
|
||||||
|
DatacenterTable targetTable = createDerivedTable(source, catalog, targetFields,
|
||||||
|
request.getTargetTableName(), "DERIVE", Map.of("sourceTableId", sourceTable.getId()), account);
|
||||||
|
|
||||||
|
DatacenterQueryRequest queryRequest = new DatacenterQueryRequest();
|
||||||
|
queryRequest.setDatasetRef(registryService.resolveDatasetRef(sourceTable.getId()));
|
||||||
|
queryRequest.setSelectedColumns(selectedColumns);
|
||||||
|
queryRequest.setFilters(request.getFilters());
|
||||||
|
|
||||||
|
long successRows = copyRows(queryRequest, rows -> mapDerivedRow(rows, selectedColumns, request), targetTable, account);
|
||||||
|
createLineage(sourceTable.getId(), targetTable.getId(), "DERIVE", Map.of("request", request), account);
|
||||||
|
finishJobSuccess(job, successRows, successRows, Map.of("derivedTableId", targetTable.getId()));
|
||||||
|
job.setTableId(targetTable.getId());
|
||||||
|
return job;
|
||||||
|
} catch (Exception ex) {
|
||||||
|
finishJobFailure(job, ex);
|
||||||
|
throw ex;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
@Transactional(rollbackFor = Exception.class)
|
||||||
|
public DatacenterImportJob exportWorkbook(DatacenterExcelExportRequest request, LoginAccount account) throws Exception {
|
||||||
|
List<DatacenterTable> tables = resolveExportTables(request);
|
||||||
|
if (tables.isEmpty()) {
|
||||||
|
throw new BusinessException("没有可导出的 Excel 数据集");
|
||||||
|
}
|
||||||
|
String fileName = buildExportFileName(request == null ? null : request.getFileName());
|
||||||
|
DatacenterImportJob job = createJob("EXCEL_EXPORT",
|
||||||
|
request == null ? null : request.getSourceId(),
|
||||||
|
request == null ? null : request.getCatalogId(),
|
||||||
|
null,
|
||||||
|
fileName,
|
||||||
|
buildPayload("request", request),
|
||||||
|
account);
|
||||||
|
Path exportDir = ensureExportDir();
|
||||||
|
Path exportFile = exportDir.resolve(fileName);
|
||||||
|
long totalRows = 0L;
|
||||||
|
try (SXSSFWorkbook workbook = new SXSSFWorkbook(200); FileOutputStream outputStream = new FileOutputStream(exportFile.toFile())) {
|
||||||
|
workbook.setCompressTempFiles(true);
|
||||||
|
Set<String> usedSheetNames = new HashSet<>();
|
||||||
|
for (DatacenterTable table : tables) {
|
||||||
|
String sheetName = uniqueSheetName(table.getTableName(), usedSheetNames);
|
||||||
|
org.apache.poi.ss.usermodel.Sheet sheet = workbook.createSheet(sheetName);
|
||||||
|
writeHeaderRow(sheet, table.getFields());
|
||||||
|
DatacenterQueryRequest queryRequest = new DatacenterQueryRequest();
|
||||||
|
queryRequest.setDatasetRef(registryService.resolveDatasetRef(table.getId()));
|
||||||
|
queryRequest.setSelectedColumns(table.getFields().stream().map(DatacenterTableField::getFieldName).toList());
|
||||||
|
final int[] rowIndex = {1};
|
||||||
|
totalRows += iterateRows(queryRequest, row -> {
|
||||||
|
org.apache.poi.ss.usermodel.Row excelRow = sheet.createRow(rowIndex[0]++);
|
||||||
|
for (int i = 0; i < table.getFields().size(); i++) {
|
||||||
|
Cell cell = excelRow.createCell(i);
|
||||||
|
Object value = row.get(table.getFields().get(i).getFieldName());
|
||||||
|
cell.setCellValue(value == null ? "" : String.valueOf(value));
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
workbook.write(outputStream);
|
||||||
|
workbook.dispose();
|
||||||
|
job.setStoragePath(exportFile.toAbsolutePath().toString());
|
||||||
|
finishJobSuccess(job, totalRows, totalRows, Map.of("storagePath", job.getStoragePath(), "fileName", fileName));
|
||||||
|
return job;
|
||||||
|
} catch (Exception ex) {
|
||||||
|
finishJobFailure(job, ex);
|
||||||
|
throw ex;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public DatacenterImportJob getImportJobDetail(BigInteger jobId) {
|
||||||
|
DatacenterImportJob job = importJobMapper.selectOneById(jobId);
|
||||||
|
if (job == null) {
|
||||||
|
throw new BusinessException("导入任务不存在: " + jobId);
|
||||||
|
}
|
||||||
|
return job;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public List<DatacenterImportJob> listJobs(BigInteger sourceId, BigInteger tableId) {
|
||||||
|
var wrapper = com.mybatisflex.core.query.QueryWrapper.create();
|
||||||
|
if (sourceId != null) {
|
||||||
|
wrapper.eq(DatacenterImportJob::getSourceId, sourceId);
|
||||||
|
}
|
||||||
|
if (tableId != null) {
|
||||||
|
wrapper.eq(DatacenterImportJob::getTableId, tableId);
|
||||||
|
}
|
||||||
|
wrapper.orderBy("created desc");
|
||||||
|
wrapper.limit(20L);
|
||||||
|
return importJobMapper.selectListByQuery(wrapper);
|
||||||
|
}
|
||||||
|
|
||||||
|
private DatacenterImportJob doSplitBySheet(DatacenterExcelSplitRequest request, LoginAccount account, DatacenterImportJob job) {
|
||||||
|
BigInteger sourceId = request.getSourceId();
|
||||||
|
BigInteger catalogId = request.getCatalogId();
|
||||||
|
if (sourceId == null && request.getDatasetRef() != null) {
|
||||||
|
sourceId = request.getDatasetRef().getSourceId();
|
||||||
|
catalogId = request.getDatasetRef().getCatalogId();
|
||||||
|
}
|
||||||
|
if (sourceId == null) {
|
||||||
|
throw new BusinessException("按 sheet 拆分需要 sourceId");
|
||||||
|
}
|
||||||
|
DatacenterSource source = registryService.getSourceRequired(sourceId);
|
||||||
|
DatacenterCatalog catalog = requireCatalog(catalogId);
|
||||||
|
List<DatacenterTable> sourceTables = registryService.listManagedTables(sourceId, catalogId);
|
||||||
|
if (sourceTables.isEmpty()) {
|
||||||
|
throw new BusinessException("当前 workbook 下没有可拆分的 sheet 表");
|
||||||
|
}
|
||||||
|
List<BigInteger> derivedIds = new ArrayList<>();
|
||||||
|
long successRows = 0L;
|
||||||
|
for (DatacenterTable sourceTable : sourceTables) {
|
||||||
|
DatacenterTable fullTable = registryService.getTableWithFields(sourceTable.getId());
|
||||||
|
DatacenterTable targetTable = createDerivedTable(source, catalog, cloneFields(fullTable.getFields()),
|
||||||
|
resolveSplitPrefix(request, fullTable.getTableName()) + "_copy", "SPLIT_BY_SHEET",
|
||||||
|
Map.of("sourceTableId", fullTable.getId()), account);
|
||||||
|
DatacenterQueryRequest queryRequest = new DatacenterQueryRequest();
|
||||||
|
queryRequest.setDatasetRef(registryService.resolveDatasetRef(fullTable.getId()));
|
||||||
|
queryRequest.setSelectedColumns(fullTable.getFields().stream().map(DatacenterTableField::getFieldName).toList());
|
||||||
|
successRows += copyRows(queryRequest, this::mapRow, targetTable, account);
|
||||||
|
createLineage(fullTable.getId(), targetTable.getId(), "SPLIT_BY_SHEET", Map.of("sourceTableId", fullTable.getId()), account);
|
||||||
|
derivedIds.add(targetTable.getId());
|
||||||
|
}
|
||||||
|
finishJobSuccess(job, successRows, successRows, Map.of("derivedTableIds", derivedIds));
|
||||||
|
return job;
|
||||||
|
}
|
||||||
|
|
||||||
|
private DatacenterImportJob doSplitByRowCount(DatacenterExcelSplitRequest request, LoginAccount account, DatacenterImportJob job) {
|
||||||
|
if (request == null || request.getDatasetRef() == null) {
|
||||||
|
throw new BusinessException("按行数拆分需要数据集");
|
||||||
|
}
|
||||||
|
int rowBatchSize = request.getRowBatchSize() == null || request.getRowBatchSize() < 1 ? 1000 : request.getRowBatchSize();
|
||||||
|
DatacenterTable sourceTable = resolveTable(request.getDatasetRef());
|
||||||
|
DatacenterSource source = registryService.getSourceRequired(sourceTable.getSourceId());
|
||||||
|
DatacenterCatalog catalog = requireCatalog(sourceTable.getCatalogId());
|
||||||
|
String baseName = resolveSplitPrefix(request, sourceTable.getTableName());
|
||||||
|
List<BigInteger> derivedIds = new ArrayList<>();
|
||||||
|
final Holder holder = new Holder();
|
||||||
|
long totalRows = iterateRows(buildFullQuery(sourceTable), row -> {
|
||||||
|
if (holder.targetTable == null || holder.currentSize >= rowBatchSize) {
|
||||||
|
holder.batchNo++;
|
||||||
|
holder.targetTable = createDerivedTable(source, catalog, cloneFields(sourceTable.getFields()),
|
||||||
|
baseName + "_part_" + holder.batchNo, "SPLIT_BY_ROW_COUNT",
|
||||||
|
Map.of("sourceTableId", sourceTable.getId(), "batchNo", holder.batchNo, "rowBatchSize", rowBatchSize), account);
|
||||||
|
createLineage(sourceTable.getId(), holder.targetTable.getId(), "SPLIT_BY_ROW_COUNT",
|
||||||
|
Map.of("sourceTableId", sourceTable.getId(), "batchNo", holder.batchNo), account);
|
||||||
|
derivedIds.add(holder.targetTable.getId());
|
||||||
|
holder.currentSize = 0;
|
||||||
|
}
|
||||||
|
saveToTable(holder.targetTable, mapRow(row), account);
|
||||||
|
holder.currentSize++;
|
||||||
|
});
|
||||||
|
finishJobSuccess(job, totalRows, totalRows, Map.of("derivedTableIds", derivedIds));
|
||||||
|
return job;
|
||||||
|
}
|
||||||
|
|
||||||
|
private DatacenterImportJob doSplitByFieldValue(DatacenterExcelSplitRequest request, LoginAccount account, DatacenterImportJob job) {
|
||||||
|
if (request == null || request.getDatasetRef() == null || request.getFieldName() == null || request.getFieldName().isBlank()) {
|
||||||
|
throw new BusinessException("按字段值拆分需要数据集和字段名");
|
||||||
|
}
|
||||||
|
DatacenterTable sourceTable = resolveTable(request.getDatasetRef());
|
||||||
|
DatacenterSource source = registryService.getSourceRequired(sourceTable.getSourceId());
|
||||||
|
DatacenterCatalog catalog = requireCatalog(sourceTable.getCatalogId());
|
||||||
|
DatacenterTableField splitField = sourceTable.getFields().stream()
|
||||||
|
.filter(field -> request.getFieldName().equals(field.getFieldName()))
|
||||||
|
.findFirst()
|
||||||
|
.orElseThrow(() -> new BusinessException("拆分字段不存在: " + request.getFieldName()));
|
||||||
|
String prefix = resolveSplitPrefix(request, sourceTable.getTableName());
|
||||||
|
Map<String, DatacenterTable> targets = new LinkedHashMap<>();
|
||||||
|
List<BigInteger> derivedIds = new ArrayList<>();
|
||||||
|
long totalRows = iterateRows(buildFullQuery(sourceTable), row -> {
|
||||||
|
String fieldValue = stringify(row.get(splitField.getFieldName()));
|
||||||
|
String bucket = fieldValue == null || fieldValue.isBlank() ? "empty" : fieldValue;
|
||||||
|
DatacenterTable targetTable = targets.get(bucket);
|
||||||
|
if (targetTable == null) {
|
||||||
|
targetTable = createDerivedTable(source, catalog, cloneFields(sourceTable.getFields()),
|
||||||
|
prefix + "_" + normalizeIdentifier(bucket), "SPLIT_BY_FIELD_VALUE",
|
||||||
|
Map.of("sourceTableId", sourceTable.getId(), "fieldName", splitField.getFieldName(), "fieldValue", bucket), account);
|
||||||
|
createLineage(sourceTable.getId(), targetTable.getId(), "SPLIT_BY_FIELD_VALUE",
|
||||||
|
Map.of("sourceTableId", sourceTable.getId(), "fieldName", splitField.getFieldName(), "fieldValue", bucket), account);
|
||||||
|
targets.put(bucket, targetTable);
|
||||||
|
derivedIds.add(targetTable.getId());
|
||||||
|
}
|
||||||
|
saveToTable(targetTable, mapRow(row), account);
|
||||||
|
});
|
||||||
|
finishJobSuccess(job, totalRows, totalRows, Map.of("derivedTableIds", derivedIds));
|
||||||
|
return job;
|
||||||
|
}
|
||||||
|
|
||||||
|
private DatacenterImportJob doVerticalMerge(DatacenterExcelMergeRequest request, LoginAccount account, DatacenterImportJob job) {
|
||||||
|
List<DatacenterTable> tables = request.getDatasetRefs().stream().map(this::resolveTable).toList();
|
||||||
|
DatacenterTable firstTable = tables.get(0);
|
||||||
|
DatacenterSource source = registryService.getSourceRequired(firstTable.getSourceId());
|
||||||
|
DatacenterCatalog catalog = requireCatalog(firstTable.getCatalogId());
|
||||||
|
assertSameCatalog(tables);
|
||||||
|
assertSameFields(tables);
|
||||||
|
DatacenterTable targetTable = createDerivedTable(source, catalog, cloneFields(firstTable.getFields()),
|
||||||
|
request.getTargetTableName(), "MERGE_VERTICAL", Map.of("sourceTableIds", tables.stream().map(DatacenterTable::getId).toList()), account);
|
||||||
|
long successRows = 0L;
|
||||||
|
for (DatacenterTable table : tables) {
|
||||||
|
successRows += copyRows(buildFullQuery(table), this::mapRow, targetTable, account);
|
||||||
|
createLineage(table.getId(), targetTable.getId(), "MERGE_VERTICAL", Map.of("sourceTableId", table.getId()), account);
|
||||||
|
}
|
||||||
|
job.setTableId(targetTable.getId());
|
||||||
|
finishJobSuccess(job, successRows, successRows, Map.of("derivedTableId", targetTable.getId()));
|
||||||
|
return job;
|
||||||
|
}
|
||||||
|
|
||||||
|
private DatacenterImportJob doHorizontalMerge(DatacenterExcelMergeRequest request, LoginAccount account, DatacenterImportJob job) {
|
||||||
|
if (request.getJoinKey() == null || request.getJoinKey().isBlank()) {
|
||||||
|
throw new BusinessException("横向合并必须指定 joinKey");
|
||||||
|
}
|
||||||
|
List<DatacenterTable> tables = request.getDatasetRefs().stream().map(this::resolveTable).toList();
|
||||||
|
DatacenterTable firstTable = tables.get(0);
|
||||||
|
DatacenterSource source = registryService.getSourceRequired(firstTable.getSourceId());
|
||||||
|
DatacenterCatalog catalog = requireCatalog(firstTable.getCatalogId());
|
||||||
|
assertSameCatalog(tables);
|
||||||
|
|
||||||
|
List<DatacenterTableField> mergedFields = new ArrayList<>();
|
||||||
|
Set<String> usedFieldNames = new LinkedHashSet<>();
|
||||||
|
Map<BigInteger, Map<String, String>> fieldMappings = new LinkedHashMap<>();
|
||||||
|
for (DatacenterTable table : tables) {
|
||||||
|
Map<String, String> mapping = new LinkedHashMap<>();
|
||||||
|
for (DatacenterTableField field : table.getFields()) {
|
||||||
|
String targetFieldName;
|
||||||
|
if (field.getFieldName().equals(request.getJoinKey())) {
|
||||||
|
targetFieldName = field.getFieldName();
|
||||||
|
} else {
|
||||||
|
targetFieldName = field.getFieldName();
|
||||||
|
if (usedFieldNames.contains(targetFieldName)) {
|
||||||
|
targetFieldName = normalizeIdentifier(table.getTableName()) + "_" + targetFieldName;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (!usedFieldNames.contains(targetFieldName)) {
|
||||||
|
usedFieldNames.add(targetFieldName);
|
||||||
|
mergedFields.add(cloneField(field, targetFieldName, field.getFieldDesc()));
|
||||||
|
}
|
||||||
|
mapping.put(field.getFieldName(), targetFieldName);
|
||||||
|
}
|
||||||
|
fieldMappings.put(table.getId(), mapping);
|
||||||
|
}
|
||||||
|
|
||||||
|
DatacenterTable targetTable = createDerivedTable(source, catalog, mergedFields,
|
||||||
|
request.getTargetTableName(), "MERGE_HORIZONTAL", Map.of("sourceTableIds", tables.stream().map(DatacenterTable::getId).toList(), "joinKey", request.getJoinKey()), account);
|
||||||
|
Map<String, JSONObject> mergedRows = new LinkedHashMap<>();
|
||||||
|
for (DatacenterTable table : tables) {
|
||||||
|
Map<String, String> mapping = fieldMappings.get(table.getId());
|
||||||
|
iterateRows(buildFullQuery(table), row -> {
|
||||||
|
String joinValue = stringify(row.get(request.getJoinKey()));
|
||||||
|
if (joinValue == null || joinValue.isBlank()) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
JSONObject target = mergedRows.computeIfAbsent(joinValue, key -> new JSONObject());
|
||||||
|
mapping.forEach((sourceField, targetField) -> target.put(targetField, row.get(sourceField)));
|
||||||
|
});
|
||||||
|
createLineage(table.getId(), targetTable.getId(), "MERGE_HORIZONTAL", Map.of("sourceTableId", table.getId(), "joinKey", request.getJoinKey()), account);
|
||||||
|
}
|
||||||
|
for (JSONObject row : mergedRows.values()) {
|
||||||
|
saveToTable(targetTable, row, account);
|
||||||
|
}
|
||||||
|
job.setTableId(targetTable.getId());
|
||||||
|
finishJobSuccess(job, (long) mergedRows.size(), (long) mergedRows.size(), Map.of("derivedTableId", targetTable.getId()));
|
||||||
|
return job;
|
||||||
|
}
|
||||||
|
|
||||||
|
private DatacenterImportJob createJob(String jobType, BigInteger sourceId, BigInteger catalogId, BigInteger tableId,
|
||||||
|
String fileName, Map<String, Object> payload, LoginAccount account) {
|
||||||
|
DatacenterImportJob job = new DatacenterImportJob();
|
||||||
|
job.setSourceId(sourceId);
|
||||||
|
job.setCatalogId(catalogId);
|
||||||
|
job.setTableId(tableId);
|
||||||
|
job.setTenantId(account == null || account.getTenantId() == null ? BigInteger.ZERO : account.getTenantId());
|
||||||
|
job.setDeptId(account == null || account.getDeptId() == null ? BigInteger.ZERO : account.getDeptId());
|
||||||
|
job.setJobType(jobType);
|
||||||
|
job.setFileName(fileName);
|
||||||
|
job.setStatus(DatacenterImportStatus.RUNNING.name());
|
||||||
|
job.setPayloadJson(payload == null ? new LinkedHashMap<>() : new LinkedHashMap<>(payload));
|
||||||
|
job.setStartedAt(new Date());
|
||||||
|
job.setCreated(new Date());
|
||||||
|
job.setModified(new Date());
|
||||||
|
job.setCreatedBy(account == null ? BigInteger.ZERO : account.getId());
|
||||||
|
job.setModifiedBy(account == null ? BigInteger.ZERO : account.getId());
|
||||||
|
importJobMapper.insert(job);
|
||||||
|
return job;
|
||||||
|
}
|
||||||
|
|
||||||
|
private Map<String, Object> buildPayload(String key, Object value) {
|
||||||
|
Map<String, Object> payload = new LinkedHashMap<>();
|
||||||
|
payload.put(key, value);
|
||||||
|
return payload;
|
||||||
|
}
|
||||||
|
|
||||||
|
private void finishJobSuccess(DatacenterImportJob job, long totalRows, long successRows, Map<String, Object> payload) {
|
||||||
|
job.setStatus(DatacenterImportStatus.SUCCESS.name());
|
||||||
|
job.setTotalRows(totalRows);
|
||||||
|
job.setSuccessRows(successRows);
|
||||||
|
job.setErrorRows(Math.max(0L, totalRows - successRows));
|
||||||
|
if (payload != null) {
|
||||||
|
job.setPayloadJson(new LinkedHashMap<>(payload));
|
||||||
|
}
|
||||||
|
job.setFinishedAt(new Date());
|
||||||
|
job.setModified(new Date());
|
||||||
|
importJobMapper.update(job);
|
||||||
|
}
|
||||||
|
|
||||||
|
private void finishJobFailure(DatacenterImportJob job, Exception ex) {
|
||||||
|
job.setStatus(DatacenterImportStatus.FAILED.name());
|
||||||
|
job.setErrorSummary(ex.getMessage());
|
||||||
|
job.setFinishedAt(new Date());
|
||||||
|
job.setModified(new Date());
|
||||||
|
importJobMapper.update(job);
|
||||||
|
}
|
||||||
|
|
||||||
|
private DatacenterTable resolveTable(DatasetRef datasetRef) {
|
||||||
|
if (datasetRef == null || datasetRef.getTableId() == null) {
|
||||||
|
throw new BusinessException("缺少数据集 tableId");
|
||||||
|
}
|
||||||
|
return registryService.getTableWithFields(datasetRef.getTableId());
|
||||||
|
}
|
||||||
|
|
||||||
|
private DatacenterCatalog requireCatalog(BigInteger catalogId) {
|
||||||
|
DatacenterCatalog catalog = registryService.getCatalogById(catalogId);
|
||||||
|
if (catalog == null) {
|
||||||
|
throw new BusinessException("目录不存在: " + catalogId);
|
||||||
|
}
|
||||||
|
return catalog;
|
||||||
|
}
|
||||||
|
|
||||||
|
private DatacenterTable createDerivedTable(DatacenterSource source, DatacenterCatalog catalog, List<DatacenterTableField> fields,
|
||||||
|
String tableName, String deriveType, Map<String, Object> config, LoginAccount account) {
|
||||||
|
DatacenterTable table = new DatacenterTable();
|
||||||
|
String resolvedName = uniqueTableName(source.getId(), catalog.getId(), normalizeLogicalName(tableName, deriveType));
|
||||||
|
table.setTableName(resolvedName);
|
||||||
|
table.setTableDesc(resolvedName);
|
||||||
|
table.setActualTable(buildMaterializedTableName(source.getId(), Math.abs(Objects.hash(resolvedName, deriveType))));
|
||||||
|
table.setMaterializedTable(table.getActualTable());
|
||||||
|
table.setTableKind(DatacenterTableKind.DERIVED_TABLE.name());
|
||||||
|
table.setAccessMode("READ_WRITE");
|
||||||
|
table.setVersioningEnabled(1);
|
||||||
|
table.setCapabilitiesJson(defaultExcelCapabilities());
|
||||||
|
table.setFields(fields);
|
||||||
|
|
||||||
|
DatacenterTableDetailMeta detail = new DatacenterTableDetailMeta();
|
||||||
|
detail.setTable(table);
|
||||||
|
detail.setFields(fields);
|
||||||
|
dbHandleManager.getDbHandler().createTable(table);
|
||||||
|
DatacenterTable savedTable = registryService.registerTable(source, catalog, detail, account);
|
||||||
|
savedTable.setFields(registryService.getFields(savedTable.getId()));
|
||||||
|
createVersion(savedTable, deriveType.toLowerCase(Locale.ROOT), config, account);
|
||||||
|
return savedTable;
|
||||||
|
}
|
||||||
|
|
||||||
|
private DatacenterDatasetVersion createVersion(DatacenterTable table, String versionLabel, Map<String, Object> snapshot, LoginAccount account) {
|
||||||
|
QueryWrapperWrapper wrapper = new QueryWrapperWrapper(table.getId());
|
||||||
|
DatacenterDatasetVersion version = new DatacenterDatasetVersion();
|
||||||
|
version.setTableId(table.getId());
|
||||||
|
version.setTenantId(table.getTenantId());
|
||||||
|
version.setDeptId(table.getDeptId());
|
||||||
|
version.setVersionNo(wrapper.nextVersionNo(datasetVersionMapper));
|
||||||
|
version.setVersionLabel(versionLabel);
|
||||||
|
version.setMaterializedTable(table.getMaterializedTable());
|
||||||
|
version.setSnapshotJson(snapshot == null ? new LinkedHashMap<>() : new LinkedHashMap<>(snapshot));
|
||||||
|
version.setStatus(0);
|
||||||
|
version.setCreated(new Date());
|
||||||
|
version.setModified(new Date());
|
||||||
|
version.setCreatedBy(account == null ? BigInteger.ZERO : account.getId());
|
||||||
|
version.setModifiedBy(account == null ? BigInteger.ZERO : account.getId());
|
||||||
|
datasetVersionMapper.insert(version);
|
||||||
|
return version;
|
||||||
|
}
|
||||||
|
|
||||||
|
private void createLineage(BigInteger sourceTableId, BigInteger derivedTableId, String deriveType, Map<String, Object> config, LoginAccount account) {
|
||||||
|
DatacenterDerivedTable relation = new DatacenterDerivedTable();
|
||||||
|
relation.setSourceTableId(sourceTableId);
|
||||||
|
relation.setDerivedTableId(derivedTableId);
|
||||||
|
relation.setDeriveType(deriveType);
|
||||||
|
relation.setDeriveConfigJson(config == null ? new LinkedHashMap<>() : new LinkedHashMap<>(config));
|
||||||
|
relation.setStatus(0);
|
||||||
|
relation.setTenantId(account == null || account.getTenantId() == null ? BigInteger.ZERO : account.getTenantId());
|
||||||
|
relation.setDeptId(account == null || account.getDeptId() == null ? BigInteger.ZERO : account.getDeptId());
|
||||||
|
relation.setCreated(new Date());
|
||||||
|
relation.setModified(new Date());
|
||||||
|
relation.setCreatedBy(account == null ? BigInteger.ZERO : account.getId());
|
||||||
|
relation.setModifiedBy(account == null ? BigInteger.ZERO : account.getId());
|
||||||
|
derivedTableMapper.insert(relation);
|
||||||
|
}
|
||||||
|
|
||||||
|
private long copyRows(DatacenterQueryRequest queryRequest, RowMapper mapper, DatacenterTable targetTable, LoginAccount account) {
|
||||||
|
return iterateRows(queryRequest, row -> saveToTable(targetTable, mapper.map(row), account));
|
||||||
|
}
|
||||||
|
|
||||||
|
private long iterateRows(DatacenterQueryRequest queryRequest, RowConsumer consumer) {
|
||||||
|
long total = 0L;
|
||||||
|
long pageNumber = 1L;
|
||||||
|
while (true) {
|
||||||
|
queryRequest.setPageNumber(pageNumber);
|
||||||
|
queryRequest.setPageSize(QUERY_BATCH_SIZE);
|
||||||
|
Page<Row> page = queryService.queryPage(queryRequest);
|
||||||
|
if (page.getRecords() == null || page.getRecords().isEmpty()) {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
for (Row row : page.getRecords()) {
|
||||||
|
consumer.accept(row);
|
||||||
|
total++;
|
||||||
|
}
|
||||||
|
if (page.getRecords().size() < QUERY_BATCH_SIZE) {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
pageNumber++;
|
||||||
|
}
|
||||||
|
return total;
|
||||||
|
}
|
||||||
|
|
||||||
|
private void saveToTable(DatacenterTable targetTable, JSONObject data, LoginAccount account) {
|
||||||
|
dbHandleManager.getDbHandler().saveValue(targetTable, data, account);
|
||||||
|
}
|
||||||
|
|
||||||
|
private DatacenterQueryRequest buildFullQuery(DatacenterTable table) {
|
||||||
|
DatacenterQueryRequest queryRequest = new DatacenterQueryRequest();
|
||||||
|
queryRequest.setDatasetRef(registryService.resolveDatasetRef(table.getId()));
|
||||||
|
queryRequest.setSelectedColumns(table.getFields().stream().map(DatacenterTableField::getFieldName).toList());
|
||||||
|
return queryRequest;
|
||||||
|
}
|
||||||
|
|
||||||
|
private JSONObject mapRow(Row row) {
|
||||||
|
JSONObject payload = new JSONObject();
|
||||||
|
row.forEach(payload::put);
|
||||||
|
payload.remove("id");
|
||||||
|
payload.remove("dept_id");
|
||||||
|
payload.remove("tenant_id");
|
||||||
|
payload.remove("created");
|
||||||
|
payload.remove("created_by");
|
||||||
|
payload.remove("modified");
|
||||||
|
payload.remove("modified_by");
|
||||||
|
payload.remove("remark");
|
||||||
|
return payload;
|
||||||
|
}
|
||||||
|
|
||||||
|
private JSONObject mapDerivedRow(Row row, List<String> selectedColumns, DatacenterExcelDeriveRequest request) {
|
||||||
|
JSONObject payload = new JSONObject();
|
||||||
|
for (String column : selectedColumns) {
|
||||||
|
String targetName = request.getRenameMappings().getOrDefault(column, column);
|
||||||
|
payload.put(targetName, row.get(column));
|
||||||
|
}
|
||||||
|
return payload;
|
||||||
|
}
|
||||||
|
|
||||||
|
private List<String> resolveSelectedColumns(DatacenterTable sourceTable, List<String> selectedColumns) {
|
||||||
|
if (CollectionUtils.isEmpty(selectedColumns)) {
|
||||||
|
return sourceTable.getFields().stream().map(DatacenterTableField::getFieldName).toList();
|
||||||
|
}
|
||||||
|
return selectedColumns;
|
||||||
|
}
|
||||||
|
|
||||||
|
private List<DatacenterTableField> buildDerivedFields(DatacenterTable sourceTable, DatacenterExcelDeriveRequest request) {
|
||||||
|
List<String> selectedColumns = resolveSelectedColumns(sourceTable, request.getSelectedColumns());
|
||||||
|
Map<String, DatacenterTableField> fieldMap = new LinkedHashMap<>();
|
||||||
|
for (DatacenterTableField field : sourceTable.getFields()) {
|
||||||
|
fieldMap.put(field.getFieldName(), field);
|
||||||
|
}
|
||||||
|
List<DatacenterTableField> fields = new ArrayList<>();
|
||||||
|
for (String column : selectedColumns) {
|
||||||
|
DatacenterTableField sourceField = fieldMap.get(column);
|
||||||
|
if (sourceField == null) {
|
||||||
|
throw new BusinessException("派生字段不存在: " + column);
|
||||||
|
}
|
||||||
|
String targetName = request.getRenameMappings().getOrDefault(column, column);
|
||||||
|
fields.add(cloneField(sourceField, targetName, targetName));
|
||||||
|
}
|
||||||
|
return fields;
|
||||||
|
}
|
||||||
|
|
||||||
|
private List<DatacenterTableField> cloneFields(List<DatacenterTableField> sourceFields) {
|
||||||
|
List<DatacenterTableField> fields = new ArrayList<>();
|
||||||
|
for (DatacenterTableField field : sourceFields) {
|
||||||
|
fields.add(cloneField(field, field.getFieldName(), field.getFieldDesc()));
|
||||||
|
}
|
||||||
|
return fields;
|
||||||
|
}
|
||||||
|
|
||||||
|
private DatacenterTableField cloneField(DatacenterTableField source, String fieldName, String fieldDesc) {
|
||||||
|
DatacenterTableField field = new DatacenterTableField();
|
||||||
|
field.setFieldName(fieldName);
|
||||||
|
field.setSourceColumnName(source.getSourceColumnName());
|
||||||
|
field.setFieldDesc(fieldDesc);
|
||||||
|
field.setFieldType(source.getFieldType());
|
||||||
|
field.setJdbcType(source.getJdbcType());
|
||||||
|
field.setPrecision(source.getPrecision());
|
||||||
|
field.setScale(source.getScale());
|
||||||
|
field.setRequired(source.getRequired());
|
||||||
|
field.setQueryable(source.getQueryable());
|
||||||
|
field.setSortable(source.getSortable());
|
||||||
|
field.setWritable(source.getWritable());
|
||||||
|
field.setIndexed(source.getIndexed());
|
||||||
|
field.setOptions(source.getOptions());
|
||||||
|
return field;
|
||||||
|
}
|
||||||
|
|
||||||
|
private Map<String, Object> defaultExcelCapabilities() {
|
||||||
|
return Map.of("capabilities", List.of("READ_QUERY", "WRITE_MUTATION", "MATERIALIZE", "EXPORT"));
|
||||||
|
}
|
||||||
|
|
||||||
|
private void assertSameCatalog(List<DatacenterTable> tables) {
|
||||||
|
Set<BigInteger> catalogIds = new HashSet<>();
|
||||||
|
Set<BigInteger> sourceIds = new HashSet<>();
|
||||||
|
for (DatacenterTable table : tables) {
|
||||||
|
catalogIds.add(table.getCatalogId());
|
||||||
|
sourceIds.add(table.getSourceId());
|
||||||
|
}
|
||||||
|
if (catalogIds.size() > 1 || sourceIds.size() > 1) {
|
||||||
|
throw new BusinessException("Excel 操作暂只支持同一 workbook/catalog 下的数据集");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private void assertSameFields(List<DatacenterTable> tables) {
|
||||||
|
List<String> first = tables.get(0).getFields().stream().map(DatacenterTableField::getFieldName).toList();
|
||||||
|
for (int i = 1; i < tables.size(); i++) {
|
||||||
|
List<String> current = tables.get(i).getFields().stream().map(DatacenterTableField::getFieldName).toList();
|
||||||
|
if (!first.equals(current)) {
|
||||||
|
throw new BusinessException("纵向合并仅支持同结构表");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private List<DatacenterTable> resolveExportTables(DatacenterExcelExportRequest request) {
|
||||||
|
List<DatacenterTable> tables = new ArrayList<>();
|
||||||
|
if (request != null && !CollectionUtils.isEmpty(request.getDatasetRefs())) {
|
||||||
|
for (DatasetRef datasetRef : request.getDatasetRefs()) {
|
||||||
|
tables.add(resolveTable(datasetRef));
|
||||||
|
}
|
||||||
|
return tables;
|
||||||
|
}
|
||||||
|
if (request == null || request.getSourceId() == null) {
|
||||||
|
throw new BusinessException("导出需要 sourceId 或 datasetRefs");
|
||||||
|
}
|
||||||
|
tables.addAll(registryService.listManagedTables(request.getSourceId(), request.getCatalogId()));
|
||||||
|
return tables.stream().map(table -> registryService.getTableWithFields(table.getId())).toList();
|
||||||
|
}
|
||||||
|
|
||||||
|
private void writeHeaderRow(org.apache.poi.ss.usermodel.Sheet sheet, List<DatacenterTableField> fields) {
|
||||||
|
org.apache.poi.ss.usermodel.Row headerRow = sheet.createRow(0);
|
||||||
|
for (int i = 0; i < fields.size(); i++) {
|
||||||
|
Cell cell = headerRow.createCell(i);
|
||||||
|
cell.setCellValue(fields.get(i).getFieldDesc());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private Path ensureExportDir() throws Exception {
|
||||||
|
Path dir = Path.of(System.getProperty("java.io.tmpdir"), "easyflow-datacenter", "exports");
|
||||||
|
Files.createDirectories(dir);
|
||||||
|
return dir;
|
||||||
|
}
|
||||||
|
|
||||||
|
private String buildExportFileName(String rawFileName) {
|
||||||
|
String baseName = rawFileName == null || rawFileName.isBlank() ? "excel_export" : extractWorkbookName(rawFileName);
|
||||||
|
return normalizeIdentifier(baseName) + "_" + EXPORT_TIME_FORMAT.format(LocalDateTime.now()) + ".xlsx";
|
||||||
|
}
|
||||||
|
|
||||||
|
private String uniqueSheetName(String rawName, Set<String> usedSheetNames) {
|
||||||
|
String base = rawName == null || rawName.isBlank() ? "Sheet" : rawName;
|
||||||
|
base = base.length() > 31 ? base.substring(0, 31) : base;
|
||||||
|
String result = base;
|
||||||
|
int suffix = 1;
|
||||||
|
while (usedSheetNames.contains(result)) {
|
||||||
|
String suffixText = "_" + suffix++;
|
||||||
|
int limit = Math.max(1, 31 - suffixText.length());
|
||||||
|
result = base.substring(0, Math.min(base.length(), limit)) + suffixText;
|
||||||
|
}
|
||||||
|
usedSheetNames.add(result);
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
|
private List<DatacenterTableField> buildFields(org.apache.poi.ss.usermodel.Row headerRow, DataFormatter formatter) {
|
||||||
|
List<DatacenterTableField> fields = new ArrayList<>();
|
||||||
|
Set<String> usedNames = new HashSet<>();
|
||||||
|
short lastCellNum = headerRow.getLastCellNum();
|
||||||
|
for (int cellIndex = 0; cellIndex < lastCellNum; cellIndex++) {
|
||||||
|
String header = formatter.formatCellValue(headerRow.getCell(cellIndex));
|
||||||
|
String fieldName = normalizeIdentifier(header, cellIndex, usedNames);
|
||||||
|
DatacenterTableField field = new DatacenterTableField();
|
||||||
|
field.setFieldName(fieldName);
|
||||||
|
field.setSourceColumnName(header);
|
||||||
|
field.setFieldDesc(header == null || header.isBlank() ? fieldName : header);
|
||||||
|
field.setFieldType(EnumFieldType.STRING.getCode());
|
||||||
|
field.setJdbcType("VARCHAR");
|
||||||
|
field.setPrecision(255);
|
||||||
|
field.setScale(0);
|
||||||
|
field.setRequired(0);
|
||||||
|
field.setQueryable(1);
|
||||||
|
field.setSortable(1);
|
||||||
|
field.setWritable(1);
|
||||||
|
field.setIndexed(0);
|
||||||
|
fields.add(field);
|
||||||
|
}
|
||||||
|
return fields;
|
||||||
|
}
|
||||||
|
|
||||||
|
private String normalizeMode(String value, String defaultValue) {
|
||||||
|
return value == null || value.isBlank() ? defaultValue : value.trim().toUpperCase(Locale.ROOT);
|
||||||
|
}
|
||||||
|
|
||||||
|
private String resolveSplitPrefix(DatacenterExcelSplitRequest request, String fallback) {
|
||||||
|
if (request != null && request.getTargetNamePrefix() != null && !request.getTargetNamePrefix().isBlank()) {
|
||||||
|
return request.getTargetNamePrefix();
|
||||||
|
}
|
||||||
|
return fallback + "_split";
|
||||||
|
}
|
||||||
|
|
||||||
|
private String normalizeLogicalName(String tableName, String deriveType) {
|
||||||
|
if (tableName != null && !tableName.isBlank()) {
|
||||||
|
return tableName;
|
||||||
|
}
|
||||||
|
return deriveType.toLowerCase(Locale.ROOT) + "_" + System.currentTimeMillis();
|
||||||
|
}
|
||||||
|
|
||||||
|
private String uniqueTableName(BigInteger sourceId, BigInteger catalogId, String rawName) {
|
||||||
|
String baseName = rawName == null || rawName.isBlank() ? "dataset" : rawName;
|
||||||
|
baseName = baseName.trim();
|
||||||
|
String result = baseName;
|
||||||
|
int suffix = 1;
|
||||||
|
while (tableNameExists(sourceId, catalogId, result)) {
|
||||||
|
result = baseName + "_" + suffix++;
|
||||||
|
}
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
|
private boolean tableNameExists(BigInteger sourceId, BigInteger catalogId, String tableName) {
|
||||||
|
List<DatacenterTable> tables = registryService.listManagedTables(sourceId, catalogId);
|
||||||
|
return tables.stream().anyMatch(table -> tableName.equals(table.getTableName()));
|
||||||
|
}
|
||||||
|
|
||||||
|
private String extractWorkbookName(String originalFileName) {
|
||||||
|
if (originalFileName == null || originalFileName.isBlank()) {
|
||||||
|
return "excel_workbook";
|
||||||
|
}
|
||||||
|
int index = originalFileName.lastIndexOf('.');
|
||||||
|
return index > 0 ? originalFileName.substring(0, index) : originalFileName;
|
||||||
|
}
|
||||||
|
|
||||||
|
private String buildMaterializedTableName(BigInteger sourceId, int sheetIndex) {
|
||||||
|
long snowId = new SnowFlakeIDKeyGenerator().nextId();
|
||||||
|
return "tb_excel_" + sourceId + "_" + sheetIndex + "_" + snowId;
|
||||||
|
}
|
||||||
|
|
||||||
|
private String normalizeIdentifier(String raw) {
|
||||||
|
if (raw == null || raw.isBlank()) {
|
||||||
|
return "value";
|
||||||
|
}
|
||||||
|
String normalized = raw.trim().toLowerCase(Locale.ROOT).replaceAll("[^a-z0-9_\\u4e00-\\u9fa5]+", "_");
|
||||||
|
normalized = normalized.replaceAll("_+", "_");
|
||||||
|
if (normalized.isBlank()) {
|
||||||
|
return "value";
|
||||||
|
}
|
||||||
|
return normalized;
|
||||||
|
}
|
||||||
|
|
||||||
|
private String normalizeIdentifier(String raw, int index, Set<String> usedNames) {
|
||||||
|
String value = normalizeIdentifier(raw);
|
||||||
|
if (value.isBlank() || "value".equals(value)) {
|
||||||
|
value = "col_" + (index + 1);
|
||||||
|
}
|
||||||
|
if (Character.isDigit(value.charAt(0))) {
|
||||||
|
value = "col_" + value;
|
||||||
|
}
|
||||||
|
String result = value;
|
||||||
|
int suffix = 1;
|
||||||
|
while (usedNames.contains(result)) {
|
||||||
|
result = value + "_" + suffix++;
|
||||||
|
}
|
||||||
|
usedNames.add(result);
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
|
private String stringify(Object value) {
|
||||||
|
return value == null ? null : String.valueOf(value);
|
||||||
|
}
|
||||||
|
|
||||||
|
private static final class Holder {
|
||||||
|
private DatacenterTable targetTable;
|
||||||
|
private int batchNo;
|
||||||
|
private int currentSize;
|
||||||
|
}
|
||||||
|
|
||||||
|
private interface RowConsumer {
|
||||||
|
void accept(Row row);
|
||||||
|
}
|
||||||
|
|
||||||
|
private interface RowMapper {
|
||||||
|
JSONObject map(Row row);
|
||||||
|
}
|
||||||
|
|
||||||
|
private static final class QueryWrapperWrapper {
|
||||||
|
private final BigInteger tableId;
|
||||||
|
|
||||||
|
private QueryWrapperWrapper(BigInteger tableId) {
|
||||||
|
this.tableId = tableId;
|
||||||
|
}
|
||||||
|
|
||||||
|
private int nextVersionNo(DatacenterDatasetVersionMapper mapper) {
|
||||||
|
return mapper.selectListByQuery(com.mybatisflex.core.query.QueryWrapper.create()
|
||||||
|
.eq(DatacenterDatasetVersion::getTableId, tableId)
|
||||||
|
.orderBy("version_no desc"))
|
||||||
|
.stream()
|
||||||
|
.findFirst()
|
||||||
|
.map(version -> version.getVersionNo() + 1)
|
||||||
|
.orElse(1);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,24 @@
|
|||||||
|
package tech.easyflow.datacenter.execution.model;
|
||||||
|
|
||||||
|
import java.util.ArrayList;
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.Map;
|
||||||
|
|
||||||
|
public class DatacenterConnectionTestResult {
|
||||||
|
private boolean success;
|
||||||
|
private String errorCode;
|
||||||
|
private String message;
|
||||||
|
private List<String> capabilities = new ArrayList<>();
|
||||||
|
private Map<String, Object> details;
|
||||||
|
|
||||||
|
public boolean isSuccess() { return success; }
|
||||||
|
public void setSuccess(boolean success) { this.success = success; }
|
||||||
|
public String getErrorCode() { return errorCode; }
|
||||||
|
public void setErrorCode(String errorCode) { this.errorCode = errorCode; }
|
||||||
|
public String getMessage() { return message; }
|
||||||
|
public void setMessage(String message) { this.message = message; }
|
||||||
|
public List<String> getCapabilities() { return capabilities; }
|
||||||
|
public void setCapabilities(List<String> capabilities) { this.capabilities = capabilities; }
|
||||||
|
public Map<String, Object> getDetails() { return details; }
|
||||||
|
public void setDetails(Map<String, Object> details) { this.details = details; }
|
||||||
|
}
|
||||||
@@ -0,0 +1,19 @@
|
|||||||
|
package tech.easyflow.datacenter.execution.model;
|
||||||
|
|
||||||
|
import java.util.List;
|
||||||
|
|
||||||
|
public class DatacenterQueryFilter {
|
||||||
|
private String column;
|
||||||
|
private String operator;
|
||||||
|
private Object value;
|
||||||
|
private List<Object> values;
|
||||||
|
|
||||||
|
public String getColumn() { return column; }
|
||||||
|
public void setColumn(String column) { this.column = column; }
|
||||||
|
public String getOperator() { return operator; }
|
||||||
|
public void setOperator(String operator) { this.operator = operator; }
|
||||||
|
public Object getValue() { return value; }
|
||||||
|
public void setValue(Object value) { this.value = value; }
|
||||||
|
public List<Object> getValues() { return values; }
|
||||||
|
public void setValues(List<Object> values) { this.values = values; }
|
||||||
|
}
|
||||||
@@ -0,0 +1,29 @@
|
|||||||
|
package tech.easyflow.datacenter.execution.model;
|
||||||
|
|
||||||
|
import java.util.ArrayList;
|
||||||
|
import java.util.List;
|
||||||
|
|
||||||
|
public class DatacenterQueryRequest {
|
||||||
|
private DatasetRef datasetRef;
|
||||||
|
private Long pageNumber = 1L;
|
||||||
|
private Long pageSize = 10L;
|
||||||
|
private List<DatacenterQueryFilter> filters = new ArrayList<>();
|
||||||
|
private List<DatacenterQuerySort> sorts = new ArrayList<>();
|
||||||
|
private List<String> selectedColumns = new ArrayList<>();
|
||||||
|
private String where;
|
||||||
|
|
||||||
|
public DatasetRef getDatasetRef() { return datasetRef; }
|
||||||
|
public void setDatasetRef(DatasetRef datasetRef) { this.datasetRef = datasetRef; }
|
||||||
|
public Long getPageNumber() { return pageNumber; }
|
||||||
|
public void setPageNumber(Long pageNumber) { this.pageNumber = pageNumber; }
|
||||||
|
public Long getPageSize() { return pageSize; }
|
||||||
|
public void setPageSize(Long pageSize) { this.pageSize = pageSize; }
|
||||||
|
public List<DatacenterQueryFilter> getFilters() { return filters; }
|
||||||
|
public void setFilters(List<DatacenterQueryFilter> filters) { this.filters = filters; }
|
||||||
|
public List<DatacenterQuerySort> getSorts() { return sorts; }
|
||||||
|
public void setSorts(List<DatacenterQuerySort> sorts) { this.sorts = sorts; }
|
||||||
|
public List<String> getSelectedColumns() { return selectedColumns; }
|
||||||
|
public void setSelectedColumns(List<String> selectedColumns) { this.selectedColumns = selectedColumns; }
|
||||||
|
public String getWhere() { return where; }
|
||||||
|
public void setWhere(String where) { this.where = where; }
|
||||||
|
}
|
||||||
@@ -0,0 +1,11 @@
|
|||||||
|
package tech.easyflow.datacenter.execution.model;
|
||||||
|
|
||||||
|
public class DatacenterQuerySort {
|
||||||
|
private String column;
|
||||||
|
private String direction;
|
||||||
|
|
||||||
|
public String getColumn() { return column; }
|
||||||
|
public void setColumn(String column) { this.column = column; }
|
||||||
|
public String getDirection() { return direction; }
|
||||||
|
public void setDirection(String direction) { this.direction = direction; }
|
||||||
|
}
|
||||||
@@ -0,0 +1,39 @@
|
|||||||
|
package tech.easyflow.datacenter.execution.model;
|
||||||
|
|
||||||
|
import tech.easyflow.datacenter.entity.DatacenterTable;
|
||||||
|
import tech.easyflow.datacenter.entity.DatacenterTableField;
|
||||||
|
import tech.easyflow.datacenter.meta.entity.DatacenterCatalog;
|
||||||
|
import tech.easyflow.datacenter.meta.entity.DatacenterDatasetVersion;
|
||||||
|
import tech.easyflow.datacenter.meta.entity.DatacenterDerivedTable;
|
||||||
|
import tech.easyflow.datacenter.meta.entity.DatacenterSource;
|
||||||
|
|
||||||
|
import java.util.ArrayList;
|
||||||
|
import java.util.List;
|
||||||
|
|
||||||
|
public class DatacenterSchemaResponse {
|
||||||
|
private DatasetRef datasetRef;
|
||||||
|
private DatacenterSource source;
|
||||||
|
private DatacenterCatalog catalog;
|
||||||
|
private DatacenterTable table;
|
||||||
|
private List<DatacenterTableField> fields = new ArrayList<>();
|
||||||
|
private List<DatacenterDatasetVersion> versions = new ArrayList<>();
|
||||||
|
private List<DatacenterDerivedTable> upstreamLineage = new ArrayList<>();
|
||||||
|
private List<DatacenterDerivedTable> downstreamLineage = new ArrayList<>();
|
||||||
|
|
||||||
|
public DatasetRef getDatasetRef() { return datasetRef; }
|
||||||
|
public void setDatasetRef(DatasetRef datasetRef) { this.datasetRef = datasetRef; }
|
||||||
|
public DatacenterSource getSource() { return source; }
|
||||||
|
public void setSource(DatacenterSource source) { this.source = source; }
|
||||||
|
public DatacenterCatalog getCatalog() { return catalog; }
|
||||||
|
public void setCatalog(DatacenterCatalog catalog) { this.catalog = catalog; }
|
||||||
|
public DatacenterTable getTable() { return table; }
|
||||||
|
public void setTable(DatacenterTable table) { this.table = table; }
|
||||||
|
public List<DatacenterTableField> getFields() { return fields; }
|
||||||
|
public void setFields(List<DatacenterTableField> fields) { this.fields = fields; }
|
||||||
|
public List<DatacenterDatasetVersion> getVersions() { return versions; }
|
||||||
|
public void setVersions(List<DatacenterDatasetVersion> versions) { this.versions = versions; }
|
||||||
|
public List<DatacenterDerivedTable> getUpstreamLineage() { return upstreamLineage; }
|
||||||
|
public void setUpstreamLineage(List<DatacenterDerivedTable> upstreamLineage) { this.upstreamLineage = upstreamLineage; }
|
||||||
|
public List<DatacenterDerivedTable> getDownstreamLineage() { return downstreamLineage; }
|
||||||
|
public void setDownstreamLineage(List<DatacenterDerivedTable> downstreamLineage) { this.downstreamLineage = downstreamLineage; }
|
||||||
|
}
|
||||||
@@ -0,0 +1,23 @@
|
|||||||
|
package tech.easyflow.datacenter.execution.model;
|
||||||
|
|
||||||
|
public class DatacenterSqlQueryRequest {
|
||||||
|
|
||||||
|
private DatasetRef datasetRef;
|
||||||
|
private String sql;
|
||||||
|
|
||||||
|
public DatasetRef getDatasetRef() {
|
||||||
|
return datasetRef;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setDatasetRef(DatasetRef datasetRef) {
|
||||||
|
this.datasetRef = datasetRef;
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getSql() {
|
||||||
|
return sql;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setSql(String sql) {
|
||||||
|
this.sql = sql;
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,25 @@
|
|||||||
|
package tech.easyflow.datacenter.execution.model;
|
||||||
|
|
||||||
|
import java.math.BigInteger;
|
||||||
|
|
||||||
|
public class DatasetRef {
|
||||||
|
private BigInteger sourceId;
|
||||||
|
private BigInteger catalogId;
|
||||||
|
private String catalogName;
|
||||||
|
private BigInteger tableId;
|
||||||
|
private String tableName;
|
||||||
|
private BigInteger versionId;
|
||||||
|
|
||||||
|
public BigInteger getSourceId() { return sourceId; }
|
||||||
|
public void setSourceId(BigInteger sourceId) { this.sourceId = sourceId; }
|
||||||
|
public BigInteger getCatalogId() { return catalogId; }
|
||||||
|
public void setCatalogId(BigInteger catalogId) { this.catalogId = catalogId; }
|
||||||
|
public String getCatalogName() { return catalogName; }
|
||||||
|
public void setCatalogName(String catalogName) { this.catalogName = catalogName; }
|
||||||
|
public BigInteger getTableId() { return tableId; }
|
||||||
|
public void setTableId(BigInteger tableId) { this.tableId = tableId; }
|
||||||
|
public String getTableName() { return tableName; }
|
||||||
|
public void setTableName(String tableName) { this.tableName = tableName; }
|
||||||
|
public BigInteger getVersionId() { return versionId; }
|
||||||
|
public void setVersionId(BigInteger versionId) { this.versionId = versionId; }
|
||||||
|
}
|
||||||
@@ -0,0 +1,18 @@
|
|||||||
|
package tech.easyflow.datacenter.execution.service;
|
||||||
|
|
||||||
|
import com.mybatisflex.core.paginate.Page;
|
||||||
|
import com.mybatisflex.core.row.Row;
|
||||||
|
import tech.easyflow.datacenter.execution.model.DatacenterQueryRequest;
|
||||||
|
import tech.easyflow.datacenter.execution.model.DatacenterSchemaResponse;
|
||||||
|
import tech.easyflow.datacenter.execution.model.DatacenterSqlQueryRequest;
|
||||||
|
import tech.easyflow.datacenter.execution.model.DatasetRef;
|
||||||
|
|
||||||
|
import java.util.List;
|
||||||
|
|
||||||
|
public interface DatacenterDatasetQueryService {
|
||||||
|
Page<Row> queryPage(DatacenterQueryRequest request);
|
||||||
|
|
||||||
|
List<Row> queryBySql(DatacenterSqlQueryRequest request);
|
||||||
|
|
||||||
|
DatacenterSchemaResponse getSchema(DatasetRef datasetRef);
|
||||||
|
}
|
||||||
@@ -0,0 +1,13 @@
|
|||||||
|
package tech.easyflow.datacenter.execution.service;
|
||||||
|
|
||||||
|
import com.alibaba.fastjson2.JSONObject;
|
||||||
|
import tech.easyflow.common.entity.LoginAccount;
|
||||||
|
import tech.easyflow.datacenter.execution.model.DatasetRef;
|
||||||
|
|
||||||
|
import java.math.BigInteger;
|
||||||
|
|
||||||
|
public interface DatacenterDatasetWriteService {
|
||||||
|
void saveRow(DatasetRef datasetRef, JSONObject data, LoginAccount account);
|
||||||
|
|
||||||
|
void deleteRow(DatasetRef datasetRef, BigInteger id, LoginAccount account);
|
||||||
|
}
|
||||||
@@ -0,0 +1,311 @@
|
|||||||
|
package tech.easyflow.datacenter.execution.service.impl;
|
||||||
|
|
||||||
|
import com.mybatisflex.core.paginate.Page;
|
||||||
|
import com.mybatisflex.core.query.QueryWrapper;
|
||||||
|
import com.mybatisflex.core.row.Row;
|
||||||
|
import org.springframework.stereotype.Service;
|
||||||
|
import org.springframework.util.CollectionUtils;
|
||||||
|
import org.springframework.util.StringUtils;
|
||||||
|
import tech.easyflow.common.web.exceptions.BusinessException;
|
||||||
|
import tech.easyflow.datacenter.connector.DatacenterConnector;
|
||||||
|
import tech.easyflow.datacenter.connector.DatacenterConnectorRegistry;
|
||||||
|
import tech.easyflow.datacenter.entity.DatacenterTable;
|
||||||
|
import tech.easyflow.datacenter.entity.DatacenterTableField;
|
||||||
|
import tech.easyflow.datacenter.execution.model.DatacenterQueryRequest;
|
||||||
|
import tech.easyflow.datacenter.execution.model.DatacenterSchemaResponse;
|
||||||
|
import tech.easyflow.datacenter.execution.model.DatacenterSqlQueryRequest;
|
||||||
|
import tech.easyflow.datacenter.execution.model.DatasetRef;
|
||||||
|
import tech.easyflow.datacenter.execution.service.DatacenterDatasetQueryService;
|
||||||
|
import tech.easyflow.datacenter.mapper.DatacenterCatalogMapper;
|
||||||
|
import tech.easyflow.datacenter.mapper.DatacenterDatasetVersionMapper;
|
||||||
|
import tech.easyflow.datacenter.mapper.DatacenterDerivedTableMapper;
|
||||||
|
import tech.easyflow.datacenter.mapper.DatacenterTableMapper;
|
||||||
|
import tech.easyflow.datacenter.meta.entity.DatacenterCatalog;
|
||||||
|
import tech.easyflow.datacenter.meta.entity.DatacenterDatasetVersion;
|
||||||
|
import tech.easyflow.datacenter.meta.entity.DatacenterDerivedTable;
|
||||||
|
import tech.easyflow.datacenter.meta.entity.DatacenterSource;
|
||||||
|
import tech.easyflow.datacenter.meta.service.DatacenterDatasetRegistryService;
|
||||||
|
import tech.easyflow.datacenter.utils.SqlSupportUtils;
|
||||||
|
|
||||||
|
import javax.annotation.Resource;
|
||||||
|
import java.math.BigInteger;
|
||||||
|
import java.util.LinkedHashMap;
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.Map;
|
||||||
|
|
||||||
|
@Service
|
||||||
|
public class DatacenterDatasetQueryServiceImpl implements DatacenterDatasetQueryService {
|
||||||
|
|
||||||
|
@Resource
|
||||||
|
private DatacenterDatasetRegistryService registryService;
|
||||||
|
@Resource
|
||||||
|
private DatacenterConnectorRegistry connectorRegistry;
|
||||||
|
@Resource
|
||||||
|
private DatacenterTableMapper tableMapper;
|
||||||
|
@Resource
|
||||||
|
private DatacenterCatalogMapper catalogMapper;
|
||||||
|
@Resource
|
||||||
|
private DatacenterDatasetVersionMapper datasetVersionMapper;
|
||||||
|
@Resource
|
||||||
|
private DatacenterDerivedTableMapper derivedTableMapper;
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public Page<Row> queryPage(DatacenterQueryRequest request) {
|
||||||
|
if (request == null || request.getDatasetRef() == null) {
|
||||||
|
throw new BusinessException("datasetRef 不能为空");
|
||||||
|
}
|
||||||
|
normalizePage(request);
|
||||||
|
DatacenterTable table = resolveTable(request.getDatasetRef());
|
||||||
|
DatacenterSource source = registryService.getSourceRequired(table.getSourceId());
|
||||||
|
DatacenterCatalog catalog = registryService.getCatalogById(table.getCatalogId());
|
||||||
|
DatacenterTable queryTable = resolveQueryTable(table, request.getDatasetRef());
|
||||||
|
validateRequest(queryTable, request, source);
|
||||||
|
request.getDatasetRef().setSourceId(table.getSourceId());
|
||||||
|
request.getDatasetRef().setCatalogId(table.getCatalogId());
|
||||||
|
request.getDatasetRef().setTableId(table.getId());
|
||||||
|
request.getDatasetRef().setTableName(table.getTableName());
|
||||||
|
if (catalog != null) {
|
||||||
|
request.getDatasetRef().setCatalogName(catalog.getCatalogName());
|
||||||
|
}
|
||||||
|
DatacenterConnector connector = connectorRegistry.getConnector(source.getSourceType());
|
||||||
|
return connector.queryPage(source, queryTable, request);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public List<Row> queryBySql(DatacenterSqlQueryRequest request) {
|
||||||
|
if (request == null || request.getDatasetRef() == null) {
|
||||||
|
throw new BusinessException("datasetRef 不能为空");
|
||||||
|
}
|
||||||
|
String sql = trimToNull(request.getSql());
|
||||||
|
if (!StringUtils.hasText(sql)) {
|
||||||
|
throw new BusinessException("SQL 不能为空");
|
||||||
|
}
|
||||||
|
DatasetRef datasetRef = request.getDatasetRef();
|
||||||
|
if (datasetRef.getSourceId() == null) {
|
||||||
|
throw new BusinessException("缺少连接服务配置");
|
||||||
|
}
|
||||||
|
DatacenterSource source = registryService.getSourceRequired(datasetRef.getSourceId());
|
||||||
|
BigInteger catalogId = resolveRequestedCatalogId(datasetRef);
|
||||||
|
List<DatacenterTable> managedTables = registryService.listManagedTables(datasetRef.getSourceId(), catalogId);
|
||||||
|
if (CollectionUtils.isEmpty(managedTables)) {
|
||||||
|
throw new BusinessException("当前连接下没有已接入表");
|
||||||
|
}
|
||||||
|
SqlSupportUtils.ResolvedSql resolvedSql = SqlSupportUtils.resolve(
|
||||||
|
sql,
|
||||||
|
managedTables.stream().map(this::toManagedSqlTable).toList()
|
||||||
|
);
|
||||||
|
DatacenterConnector connector = connectorRegistry.getConnector(source.getSourceType());
|
||||||
|
return connector.queryBySql(source, resolvedSql.getExecutableSql());
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public DatacenterSchemaResponse getSchema(DatasetRef datasetRef) {
|
||||||
|
DatacenterTable table = resolveTable(datasetRef);
|
||||||
|
DatacenterSource source = registryService.getSourceRequired(table.getSourceId());
|
||||||
|
DatacenterCatalog catalog = registryService.getCatalogById(table.getCatalogId());
|
||||||
|
DatacenterSchemaResponse response = new DatacenterSchemaResponse();
|
||||||
|
response.setDatasetRef(registryService.resolveDatasetRef(table.getId()));
|
||||||
|
response.setSource(source);
|
||||||
|
response.setCatalog(catalog);
|
||||||
|
response.setTable(table);
|
||||||
|
response.setFields(table.getFields());
|
||||||
|
response.setVersions(listVersions(table.getId()));
|
||||||
|
response.setUpstreamLineage(listUpstream(table.getId()));
|
||||||
|
response.setDownstreamLineage(listDownstream(table.getId()));
|
||||||
|
return response;
|
||||||
|
}
|
||||||
|
|
||||||
|
private DatacenterTable resolveTable(DatasetRef datasetRef) {
|
||||||
|
if (datasetRef.getTableId() != null) {
|
||||||
|
return registryService.getTableWithFields(datasetRef.getTableId());
|
||||||
|
}
|
||||||
|
if (datasetRef.getSourceId() == null || !StringUtils.hasText(datasetRef.getTableName())) {
|
||||||
|
throw new BusinessException("缺少数据集定位信息");
|
||||||
|
}
|
||||||
|
QueryWrapper wrapper = QueryWrapper.create();
|
||||||
|
wrapper.eq(DatacenterTable::getSourceId, datasetRef.getSourceId());
|
||||||
|
wrapper.eq(DatacenterTable::getTableName, datasetRef.getTableName().trim());
|
||||||
|
|
||||||
|
boolean hasCatalogCondition = false;
|
||||||
|
if (datasetRef.getCatalogId() != null) {
|
||||||
|
wrapper.eq(DatacenterTable::getCatalogId, datasetRef.getCatalogId());
|
||||||
|
hasCatalogCondition = true;
|
||||||
|
} else if (StringUtils.hasText(datasetRef.getCatalogName())) {
|
||||||
|
DatacenterCatalog catalog = resolveCatalog(datasetRef.getSourceId(), datasetRef.getCatalogName().trim());
|
||||||
|
wrapper.eq(DatacenterTable::getCatalogId, catalog.getId());
|
||||||
|
hasCatalogCondition = true;
|
||||||
|
}
|
||||||
|
|
||||||
|
List<DatacenterTable> tables = tableMapper.selectListByQuery(wrapper);
|
||||||
|
if (CollectionUtils.isEmpty(tables)) {
|
||||||
|
throw new BusinessException("数据集不存在: " + datasetRef.getTableName());
|
||||||
|
}
|
||||||
|
if (!hasCatalogCondition && tables.size() > 1) {
|
||||||
|
throw new BusinessException("数据集存在重名表,请指定库名: " + datasetRef.getTableName());
|
||||||
|
}
|
||||||
|
return registryService.getTableWithFields(tables.get(0).getId());
|
||||||
|
}
|
||||||
|
|
||||||
|
private DatacenterCatalog resolveCatalog(java.math.BigInteger sourceId, String catalogName) {
|
||||||
|
QueryWrapper wrapper = QueryWrapper.create();
|
||||||
|
wrapper.eq(DatacenterCatalog::getSourceId, sourceId);
|
||||||
|
wrapper.eq(DatacenterCatalog::getCatalogName, catalogName);
|
||||||
|
List<DatacenterCatalog> catalogs = catalogMapper.selectListByQuery(wrapper);
|
||||||
|
if (CollectionUtils.isEmpty(catalogs)) {
|
||||||
|
throw new BusinessException("库不存在: " + catalogName);
|
||||||
|
}
|
||||||
|
if (catalogs.size() > 1) {
|
||||||
|
throw new BusinessException("库存在重复配置,请检查: " + catalogName);
|
||||||
|
}
|
||||||
|
return catalogs.get(0);
|
||||||
|
}
|
||||||
|
|
||||||
|
private BigInteger resolveRequestedCatalogId(DatasetRef datasetRef) {
|
||||||
|
if (datasetRef == null) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
if (datasetRef.getCatalogId() != null) {
|
||||||
|
return datasetRef.getCatalogId();
|
||||||
|
}
|
||||||
|
if (StringUtils.hasText(datasetRef.getCatalogName())) {
|
||||||
|
DatacenterCatalog catalog = resolveCatalog(datasetRef.getSourceId(), datasetRef.getCatalogName().trim());
|
||||||
|
return catalog.getId();
|
||||||
|
}
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
private SqlSupportUtils.ManagedTable toManagedSqlTable(DatacenterTable table) {
|
||||||
|
DatacenterCatalog catalog = registryService.getCatalogById(table.getCatalogId());
|
||||||
|
return new SqlSupportUtils.ManagedTable(
|
||||||
|
catalog == null ? null : catalog.getCatalogName(),
|
||||||
|
table.getTableName(),
|
||||||
|
resolvePhysicalTableName(table)
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
private String resolvePhysicalTableName(DatacenterTable table) {
|
||||||
|
if (table == null) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
if ("EXTERNAL_TABLE".equals(table.getTableKind()) || "EXTERNAL_VIEW".equals(table.getTableKind())) {
|
||||||
|
return trimToNull(table.getActualTable()) == null ? table.getTableName() : table.getActualTable().trim();
|
||||||
|
}
|
||||||
|
String materializedTable = trimToNull(table.getMaterializedTable());
|
||||||
|
if (materializedTable != null) {
|
||||||
|
return materializedTable;
|
||||||
|
}
|
||||||
|
String actualTable = trimToNull(table.getActualTable());
|
||||||
|
return actualTable != null ? actualTable : table.getTableName();
|
||||||
|
}
|
||||||
|
|
||||||
|
private void normalizePage(DatacenterQueryRequest request) {
|
||||||
|
if (request.getPageNumber() == null || request.getPageNumber() < 1L) {
|
||||||
|
request.setPageNumber(1L);
|
||||||
|
}
|
||||||
|
if (request.getPageSize() == null || request.getPageSize() < 1L) {
|
||||||
|
throw new BusinessException("pageSize 必须大于 0");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private DatacenterTable resolveQueryTable(DatacenterTable table, DatasetRef datasetRef) {
|
||||||
|
if (datasetRef == null || datasetRef.getVersionId() == null) {
|
||||||
|
return table;
|
||||||
|
}
|
||||||
|
DatacenterDatasetVersion version = datasetVersionMapper.selectOneById(datasetRef.getVersionId());
|
||||||
|
if (version == null || !table.getId().equals(version.getTableId())) {
|
||||||
|
throw new BusinessException("数据集版本不存在: " + datasetRef.getVersionId());
|
||||||
|
}
|
||||||
|
DatacenterTable queryTable = new DatacenterTable();
|
||||||
|
queryTable.setId(table.getId());
|
||||||
|
queryTable.setSourceId(table.getSourceId());
|
||||||
|
queryTable.setCatalogId(table.getCatalogId());
|
||||||
|
queryTable.setTableName(table.getTableName());
|
||||||
|
queryTable.setTableDesc(table.getTableDesc());
|
||||||
|
queryTable.setActualTable(table.getActualTable());
|
||||||
|
queryTable.setMaterializedTable(version.getMaterializedTable());
|
||||||
|
queryTable.setAccessMode(table.getAccessMode());
|
||||||
|
queryTable.setTableKind(table.getTableKind());
|
||||||
|
queryTable.setVersioningEnabled(table.getVersioningEnabled());
|
||||||
|
queryTable.setCapabilitiesJson(table.getCapabilitiesJson());
|
||||||
|
queryTable.setFields(table.getFields());
|
||||||
|
return queryTable;
|
||||||
|
}
|
||||||
|
|
||||||
|
private void validateRequest(DatacenterTable table, DatacenterQueryRequest request, DatacenterSource source) {
|
||||||
|
Map<String, DatacenterTableField> fieldMap = new LinkedHashMap<>();
|
||||||
|
for (DatacenterTableField field : table.getFields()) {
|
||||||
|
fieldMap.put(field.getFieldName(), field);
|
||||||
|
}
|
||||||
|
if (!CollectionUtils.isEmpty(request.getSelectedColumns())) {
|
||||||
|
for (String column : request.getSelectedColumns()) {
|
||||||
|
DatacenterTableField field = fieldMap.get(column);
|
||||||
|
if (field == null || !isEnabled(field.getQueryable())) {
|
||||||
|
throw new BusinessException("字段不可查询: " + column);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
request.setSelectedColumns(
|
||||||
|
table.getFields().stream()
|
||||||
|
.filter(field -> isEnabled(field.getQueryable()))
|
||||||
|
.map(DatacenterTableField::getFieldName)
|
||||||
|
.toList()
|
||||||
|
);
|
||||||
|
}
|
||||||
|
if (!CollectionUtils.isEmpty(request.getFilters())) {
|
||||||
|
request.getFilters().forEach(filter -> {
|
||||||
|
DatacenterTableField field = fieldMap.get(filter.getColumn());
|
||||||
|
if (field == null || !isEnabled(field.getQueryable())) {
|
||||||
|
throw new BusinessException("字段不可过滤: " + filter.getColumn());
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
if (!CollectionUtils.isEmpty(request.getSorts())) {
|
||||||
|
request.getSorts().forEach(sort -> {
|
||||||
|
DatacenterTableField field = fieldMap.get(sort.getColumn());
|
||||||
|
if (field == null || !isEnabled(field.getSortable())) {
|
||||||
|
throw new BusinessException("字段不可排序: " + sort.getColumn());
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
if (request.getWhere() != null && !request.getWhere().isBlank()) {
|
||||||
|
boolean allowLegacyWhere = "PROJECT_MYSQL".equals(source.getSourceType())
|
||||||
|
|| "MYSQL".equals(source.getSourceType())
|
||||||
|
|| "POSTGRESQL".equals(source.getSourceType());
|
||||||
|
if (!allowLegacyWhere) {
|
||||||
|
throw new BusinessException("当前数据源仅支持结构化 DSL 查询");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private boolean isEnabled(Integer value) {
|
||||||
|
return value == null || value == 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
private String trimToNull(String value) {
|
||||||
|
if (!StringUtils.hasText(value)) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
return value.trim();
|
||||||
|
}
|
||||||
|
|
||||||
|
private List<DatacenterDatasetVersion> listVersions(java.math.BigInteger tableId) {
|
||||||
|
QueryWrapper wrapper = QueryWrapper.create();
|
||||||
|
wrapper.eq(DatacenterDatasetVersion::getTableId, tableId);
|
||||||
|
wrapper.orderBy("version_no desc");
|
||||||
|
return datasetVersionMapper.selectListByQuery(wrapper);
|
||||||
|
}
|
||||||
|
|
||||||
|
private List<DatacenterDerivedTable> listUpstream(java.math.BigInteger tableId) {
|
||||||
|
QueryWrapper wrapper = QueryWrapper.create();
|
||||||
|
wrapper.eq(DatacenterDerivedTable::getDerivedTableId, tableId);
|
||||||
|
wrapper.orderBy("created desc");
|
||||||
|
return derivedTableMapper.selectListByQuery(wrapper);
|
||||||
|
}
|
||||||
|
|
||||||
|
private List<DatacenterDerivedTable> listDownstream(java.math.BigInteger tableId) {
|
||||||
|
QueryWrapper wrapper = QueryWrapper.create();
|
||||||
|
wrapper.eq(DatacenterDerivedTable::getSourceTableId, tableId);
|
||||||
|
wrapper.orderBy("created desc");
|
||||||
|
return derivedTableMapper.selectListByQuery(wrapper);
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,48 @@
|
|||||||
|
package tech.easyflow.datacenter.execution.service.impl;
|
||||||
|
|
||||||
|
import com.alibaba.fastjson2.JSONObject;
|
||||||
|
import org.springframework.stereotype.Service;
|
||||||
|
import tech.easyflow.common.entity.LoginAccount;
|
||||||
|
import tech.easyflow.common.web.exceptions.BusinessException;
|
||||||
|
import tech.easyflow.datacenter.connector.DatacenterConnector;
|
||||||
|
import tech.easyflow.datacenter.connector.DatacenterConnectorRegistry;
|
||||||
|
import tech.easyflow.datacenter.entity.DatacenterTable;
|
||||||
|
import tech.easyflow.datacenter.execution.model.DatasetRef;
|
||||||
|
import tech.easyflow.datacenter.meta.entity.DatacenterSource;
|
||||||
|
import tech.easyflow.datacenter.meta.service.DatacenterDatasetRegistryService;
|
||||||
|
import tech.easyflow.datacenter.execution.service.DatacenterDatasetWriteService;
|
||||||
|
|
||||||
|
import javax.annotation.Resource;
|
||||||
|
import java.math.BigInteger;
|
||||||
|
|
||||||
|
@Service
|
||||||
|
public class DatacenterDatasetWriteServiceImpl implements DatacenterDatasetWriteService {
|
||||||
|
|
||||||
|
@Resource
|
||||||
|
private DatacenterDatasetRegistryService registryService;
|
||||||
|
@Resource
|
||||||
|
private DatacenterConnectorRegistry connectorRegistry;
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void saveRow(DatasetRef datasetRef, JSONObject data, LoginAccount account) {
|
||||||
|
DatacenterTable table = resolveTable(datasetRef);
|
||||||
|
DatacenterSource source = registryService.getSourceRequired(table.getSourceId());
|
||||||
|
DatacenterConnector connector = connectorRegistry.getConnector(source.getSourceType());
|
||||||
|
connector.saveRow(source, table, data, account);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void deleteRow(DatasetRef datasetRef, BigInteger id, LoginAccount account) {
|
||||||
|
DatacenterTable table = resolveTable(datasetRef);
|
||||||
|
DatacenterSource source = registryService.getSourceRequired(table.getSourceId());
|
||||||
|
DatacenterConnector connector = connectorRegistry.getConnector(source.getSourceType());
|
||||||
|
connector.deleteRow(source, table, id, account);
|
||||||
|
}
|
||||||
|
|
||||||
|
private DatacenterTable resolveTable(DatasetRef datasetRef) {
|
||||||
|
if (datasetRef == null || datasetRef.getTableId() == null) {
|
||||||
|
throw new BusinessException("缺少 tableId");
|
||||||
|
}
|
||||||
|
return registryService.getTableWithFields(datasetRef.getTableId());
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,7 @@
|
|||||||
|
package tech.easyflow.datacenter.integration;
|
||||||
|
|
||||||
|
import tech.easyflow.datacenter.execution.model.DatacenterQueryRequest;
|
||||||
|
|
||||||
|
public interface AssistantDatacenterBridge {
|
||||||
|
AssistantDatacenterResult queryPage(DatacenterQueryRequest request);
|
||||||
|
}
|
||||||
@@ -0,0 +1,34 @@
|
|||||||
|
package tech.easyflow.datacenter.integration;
|
||||||
|
|
||||||
|
import com.mybatisflex.core.row.Row;
|
||||||
|
import tech.easyflow.datacenter.entity.DatacenterTable;
|
||||||
|
import tech.easyflow.datacenter.meta.entity.DatacenterCatalog;
|
||||||
|
import tech.easyflow.datacenter.meta.entity.DatacenterDatasetVersion;
|
||||||
|
import tech.easyflow.datacenter.meta.entity.DatacenterSource;
|
||||||
|
|
||||||
|
import java.util.ArrayList;
|
||||||
|
import java.util.LinkedHashMap;
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.Map;
|
||||||
|
|
||||||
|
public class AssistantDatacenterResult {
|
||||||
|
private List<Row> rows = new ArrayList<>();
|
||||||
|
private DatacenterSource source;
|
||||||
|
private DatacenterCatalog catalog;
|
||||||
|
private DatacenterTable table;
|
||||||
|
private DatacenterDatasetVersion version;
|
||||||
|
private Map<String, Object> querySummary = new LinkedHashMap<>();
|
||||||
|
|
||||||
|
public List<Row> getRows() { return rows; }
|
||||||
|
public void setRows(List<Row> rows) { this.rows = rows; }
|
||||||
|
public DatacenterSource getSource() { return source; }
|
||||||
|
public void setSource(DatacenterSource source) { this.source = source; }
|
||||||
|
public DatacenterCatalog getCatalog() { return catalog; }
|
||||||
|
public void setCatalog(DatacenterCatalog catalog) { this.catalog = catalog; }
|
||||||
|
public DatacenterTable getTable() { return table; }
|
||||||
|
public void setTable(DatacenterTable table) { this.table = table; }
|
||||||
|
public DatacenterDatasetVersion getVersion() { return version; }
|
||||||
|
public void setVersion(DatacenterDatasetVersion version) { this.version = version; }
|
||||||
|
public Map<String, Object> getQuerySummary() { return querySummary; }
|
||||||
|
public void setQuerySummary(Map<String, Object> querySummary) { this.querySummary = querySummary; }
|
||||||
|
}
|
||||||
@@ -0,0 +1,49 @@
|
|||||||
|
package tech.easyflow.datacenter.integration;
|
||||||
|
|
||||||
|
import org.springframework.stereotype.Component;
|
||||||
|
import tech.easyflow.datacenter.execution.model.DatacenterQueryRequest;
|
||||||
|
import tech.easyflow.datacenter.execution.model.DatacenterSchemaResponse;
|
||||||
|
import tech.easyflow.datacenter.execution.service.DatacenterDatasetQueryService;
|
||||||
|
import tech.easyflow.datacenter.meta.entity.DatacenterDatasetVersion;
|
||||||
|
|
||||||
|
import javax.annotation.Resource;
|
||||||
|
import java.util.LinkedHashMap;
|
||||||
|
import java.util.List;
|
||||||
|
|
||||||
|
@Component
|
||||||
|
public class DefaultAssistantDatacenterBridge implements AssistantDatacenterBridge {
|
||||||
|
|
||||||
|
@Resource
|
||||||
|
private DatacenterDatasetQueryService queryService;
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public AssistantDatacenterResult queryPage(DatacenterQueryRequest request) {
|
||||||
|
var page = queryService.queryPage(request);
|
||||||
|
DatacenterSchemaResponse schema = queryService.getSchema(request.getDatasetRef());
|
||||||
|
AssistantDatacenterResult result = new AssistantDatacenterResult();
|
||||||
|
result.setRows(page.getRecords());
|
||||||
|
result.setSource(schema.getSource());
|
||||||
|
result.setCatalog(schema.getCatalog());
|
||||||
|
result.setTable(schema.getTable());
|
||||||
|
result.setVersion(resolveVersion(schema.getVersions(), request == null || request.getDatasetRef() == null ? null : request.getDatasetRef().getVersionId()));
|
||||||
|
result.setQuerySummary(new LinkedHashMap<>() {{
|
||||||
|
put("pageNumber", page.getPageNumber());
|
||||||
|
put("pageSize", page.getPageSize());
|
||||||
|
put("totalRows", page.getTotalRow());
|
||||||
|
put("selectedColumns", request == null ? List.of() : request.getSelectedColumns());
|
||||||
|
put("filterCount", request == null || request.getFilters() == null ? 0 : request.getFilters().size());
|
||||||
|
put("sortCount", request == null || request.getSorts() == null ? 0 : request.getSorts().size());
|
||||||
|
}});
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
|
private DatacenterDatasetVersion resolveVersion(List<DatacenterDatasetVersion> versions, java.math.BigInteger versionId) {
|
||||||
|
if (versions == null || versions.isEmpty()) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
if (versionId == null) {
|
||||||
|
return versions.get(0);
|
||||||
|
}
|
||||||
|
return versions.stream().filter(version -> versionId.equals(version.getId())).findFirst().orElse(null);
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,7 @@
|
|||||||
|
package tech.easyflow.datacenter.mapper;
|
||||||
|
|
||||||
|
import com.mybatisflex.core.BaseMapper;
|
||||||
|
import tech.easyflow.datacenter.meta.entity.DatacenterCatalog;
|
||||||
|
|
||||||
|
public interface DatacenterCatalogMapper extends BaseMapper<DatacenterCatalog> {
|
||||||
|
}
|
||||||
@@ -0,0 +1,7 @@
|
|||||||
|
package tech.easyflow.datacenter.mapper;
|
||||||
|
|
||||||
|
import com.mybatisflex.core.BaseMapper;
|
||||||
|
import tech.easyflow.datacenter.meta.entity.DatacenterDatasetVersion;
|
||||||
|
|
||||||
|
public interface DatacenterDatasetVersionMapper extends BaseMapper<DatacenterDatasetVersion> {
|
||||||
|
}
|
||||||
@@ -0,0 +1,7 @@
|
|||||||
|
package tech.easyflow.datacenter.mapper;
|
||||||
|
|
||||||
|
import com.mybatisflex.core.BaseMapper;
|
||||||
|
import tech.easyflow.datacenter.meta.entity.DatacenterDerivedTable;
|
||||||
|
|
||||||
|
public interface DatacenterDerivedTableMapper extends BaseMapper<DatacenterDerivedTable> {
|
||||||
|
}
|
||||||
@@ -0,0 +1,7 @@
|
|||||||
|
package tech.easyflow.datacenter.mapper;
|
||||||
|
|
||||||
|
import com.mybatisflex.core.BaseMapper;
|
||||||
|
import tech.easyflow.datacenter.meta.entity.DatacenterImportJob;
|
||||||
|
|
||||||
|
public interface DatacenterImportJobMapper extends BaseMapper<DatacenterImportJob> {
|
||||||
|
}
|
||||||
@@ -0,0 +1,7 @@
|
|||||||
|
package tech.easyflow.datacenter.mapper;
|
||||||
|
|
||||||
|
import com.mybatisflex.core.BaseMapper;
|
||||||
|
import tech.easyflow.datacenter.meta.entity.DatacenterSource;
|
||||||
|
|
||||||
|
public interface DatacenterSourceMapper extends BaseMapper<DatacenterSource> {
|
||||||
|
}
|
||||||
@@ -0,0 +1,75 @@
|
|||||||
|
package tech.easyflow.datacenter.meta.entity;
|
||||||
|
|
||||||
|
import com.mybatisflex.annotation.Column;
|
||||||
|
import com.mybatisflex.annotation.Id;
|
||||||
|
import com.mybatisflex.annotation.KeyType;
|
||||||
|
import com.mybatisflex.annotation.Table;
|
||||||
|
import com.mybatisflex.core.handler.FastjsonTypeHandler;
|
||||||
|
import tech.easyflow.common.entity.DateEntity;
|
||||||
|
|
||||||
|
import java.io.Serializable;
|
||||||
|
import java.math.BigInteger;
|
||||||
|
import java.util.Date;
|
||||||
|
import java.util.Map;
|
||||||
|
|
||||||
|
@Table(value = "tb_datacenter_catalog", comment = "数据中心逻辑库/命名空间")
|
||||||
|
public class DatacenterCatalog extends DateEntity implements Serializable {
|
||||||
|
|
||||||
|
@Id(keyType = KeyType.Generator, value = "snowFlakeId", comment = "主键")
|
||||||
|
private BigInteger id;
|
||||||
|
@Column(comment = "部门ID")
|
||||||
|
private BigInteger deptId;
|
||||||
|
@Column(tenantId = true, comment = "租户ID")
|
||||||
|
private BigInteger tenantId;
|
||||||
|
@Column(comment = "数据源ID")
|
||||||
|
private BigInteger sourceId;
|
||||||
|
@Column(comment = "目录名")
|
||||||
|
private String catalogName;
|
||||||
|
@Column(comment = "目录描述")
|
||||||
|
private String catalogDesc;
|
||||||
|
@Column(comment = "目录类型")
|
||||||
|
private String catalogType;
|
||||||
|
@Column(comment = "状态")
|
||||||
|
private Integer status;
|
||||||
|
@Column(typeHandler = FastjsonTypeHandler.class, comment = "扩展项")
|
||||||
|
private Map<String, Object> options;
|
||||||
|
@Column(comment = "创建时间")
|
||||||
|
private Date created;
|
||||||
|
@Column(comment = "创建人")
|
||||||
|
private BigInteger createdBy;
|
||||||
|
@Column(comment = "修改时间")
|
||||||
|
private Date modified;
|
||||||
|
@Column(comment = "修改人")
|
||||||
|
private BigInteger modifiedBy;
|
||||||
|
|
||||||
|
public BigInteger getId() { return id; }
|
||||||
|
public void setId(BigInteger id) { this.id = id; }
|
||||||
|
public BigInteger getDeptId() { return deptId; }
|
||||||
|
public void setDeptId(BigInteger deptId) { this.deptId = deptId; }
|
||||||
|
public BigInteger getTenantId() { return tenantId; }
|
||||||
|
public void setTenantId(BigInteger tenantId) { this.tenantId = tenantId; }
|
||||||
|
public BigInteger getSourceId() { return sourceId; }
|
||||||
|
public void setSourceId(BigInteger sourceId) { this.sourceId = sourceId; }
|
||||||
|
public String getCatalogName() { return catalogName; }
|
||||||
|
public void setCatalogName(String catalogName) { this.catalogName = catalogName; }
|
||||||
|
public String getCatalogDesc() { return catalogDesc; }
|
||||||
|
public void setCatalogDesc(String catalogDesc) { this.catalogDesc = catalogDesc; }
|
||||||
|
public String getCatalogType() { return catalogType; }
|
||||||
|
public void setCatalogType(String catalogType) { this.catalogType = catalogType; }
|
||||||
|
public Integer getStatus() { return status; }
|
||||||
|
public void setStatus(Integer status) { this.status = status; }
|
||||||
|
public Map<String, Object> getOptions() { return options; }
|
||||||
|
public void setOptions(Map<String, Object> options) { this.options = options; }
|
||||||
|
@Override
|
||||||
|
public Date getCreated() { return created; }
|
||||||
|
@Override
|
||||||
|
public void setCreated(Date created) { this.created = created; }
|
||||||
|
public BigInteger getCreatedBy() { return createdBy; }
|
||||||
|
public void setCreatedBy(BigInteger createdBy) { this.createdBy = createdBy; }
|
||||||
|
@Override
|
||||||
|
public Date getModified() { return modified; }
|
||||||
|
@Override
|
||||||
|
public void setModified(Date modified) { this.modified = modified; }
|
||||||
|
public BigInteger getModifiedBy() { return modifiedBy; }
|
||||||
|
public void setModifiedBy(BigInteger modifiedBy) { this.modifiedBy = modifiedBy; }
|
||||||
|
}
|
||||||
@@ -0,0 +1,70 @@
|
|||||||
|
package tech.easyflow.datacenter.meta.entity;
|
||||||
|
|
||||||
|
import com.mybatisflex.annotation.Column;
|
||||||
|
import com.mybatisflex.annotation.Id;
|
||||||
|
import com.mybatisflex.annotation.KeyType;
|
||||||
|
import com.mybatisflex.annotation.Table;
|
||||||
|
import com.mybatisflex.core.handler.FastjsonTypeHandler;
|
||||||
|
import tech.easyflow.common.entity.DateEntity;
|
||||||
|
|
||||||
|
import java.io.Serializable;
|
||||||
|
import java.math.BigInteger;
|
||||||
|
import java.util.Date;
|
||||||
|
import java.util.Map;
|
||||||
|
|
||||||
|
@Table(value = "tb_datacenter_dataset_version", comment = "数据集版本")
|
||||||
|
public class DatacenterDatasetVersion extends DateEntity implements Serializable {
|
||||||
|
@Id(keyType = KeyType.Generator, value = "snowFlakeId")
|
||||||
|
private BigInteger id;
|
||||||
|
@Column(comment = "部门ID")
|
||||||
|
private BigInteger deptId;
|
||||||
|
@Column(tenantId = true, comment = "租户ID")
|
||||||
|
private BigInteger tenantId;
|
||||||
|
@Column(comment = "表ID")
|
||||||
|
private BigInteger tableId;
|
||||||
|
@Column(comment = "版本号")
|
||||||
|
private Integer versionNo;
|
||||||
|
@Column(comment = "版本标签")
|
||||||
|
private String versionLabel;
|
||||||
|
@Column(comment = "物化表名")
|
||||||
|
private String materializedTable;
|
||||||
|
@Column(typeHandler = FastjsonTypeHandler.class, comment = "版本快照")
|
||||||
|
private Map<String, Object> snapshotJson;
|
||||||
|
@Column(comment = "状态")
|
||||||
|
private Integer status;
|
||||||
|
@Column(comment = "创建时间")
|
||||||
|
private Date created;
|
||||||
|
@Column(comment = "创建人")
|
||||||
|
private BigInteger createdBy;
|
||||||
|
@Column(comment = "修改时间")
|
||||||
|
private Date modified;
|
||||||
|
@Column(comment = "修改人")
|
||||||
|
private BigInteger modifiedBy;
|
||||||
|
|
||||||
|
public BigInteger getId() { return id; }
|
||||||
|
public void setId(BigInteger id) { this.id = id; }
|
||||||
|
public BigInteger getDeptId() { return deptId; }
|
||||||
|
public void setDeptId(BigInteger deptId) { this.deptId = deptId; }
|
||||||
|
public BigInteger getTenantId() { return tenantId; }
|
||||||
|
public void setTenantId(BigInteger tenantId) { this.tenantId = tenantId; }
|
||||||
|
public BigInteger getTableId() { return tableId; }
|
||||||
|
public void setTableId(BigInteger tableId) { this.tableId = tableId; }
|
||||||
|
public Integer getVersionNo() { return versionNo; }
|
||||||
|
public void setVersionNo(Integer versionNo) { this.versionNo = versionNo; }
|
||||||
|
public String getVersionLabel() { return versionLabel; }
|
||||||
|
public void setVersionLabel(String versionLabel) { this.versionLabel = versionLabel; }
|
||||||
|
public String getMaterializedTable() { return materializedTable; }
|
||||||
|
public void setMaterializedTable(String materializedTable) { this.materializedTable = materializedTable; }
|
||||||
|
public Map<String, Object> getSnapshotJson() { return snapshotJson; }
|
||||||
|
public void setSnapshotJson(Map<String, Object> snapshotJson) { this.snapshotJson = snapshotJson; }
|
||||||
|
public Integer getStatus() { return status; }
|
||||||
|
public void setStatus(Integer status) { this.status = status; }
|
||||||
|
@Override public Date getCreated() { return created; }
|
||||||
|
@Override public void setCreated(Date created) { this.created = created; }
|
||||||
|
public BigInteger getCreatedBy() { return createdBy; }
|
||||||
|
public void setCreatedBy(BigInteger createdBy) { this.createdBy = createdBy; }
|
||||||
|
@Override public Date getModified() { return modified; }
|
||||||
|
@Override public void setModified(Date modified) { this.modified = modified; }
|
||||||
|
public BigInteger getModifiedBy() { return modifiedBy; }
|
||||||
|
public void setModifiedBy(BigInteger modifiedBy) { this.modifiedBy = modifiedBy; }
|
||||||
|
}
|
||||||
@@ -0,0 +1,66 @@
|
|||||||
|
package tech.easyflow.datacenter.meta.entity;
|
||||||
|
|
||||||
|
import com.mybatisflex.annotation.Column;
|
||||||
|
import com.mybatisflex.annotation.Id;
|
||||||
|
import com.mybatisflex.annotation.KeyType;
|
||||||
|
import com.mybatisflex.annotation.Table;
|
||||||
|
import com.mybatisflex.core.handler.FastjsonTypeHandler;
|
||||||
|
import tech.easyflow.common.entity.DateEntity;
|
||||||
|
|
||||||
|
import java.io.Serializable;
|
||||||
|
import java.math.BigInteger;
|
||||||
|
import java.util.Date;
|
||||||
|
import java.util.Map;
|
||||||
|
|
||||||
|
@Table(value = "tb_datacenter_derived_table", comment = "数据中心派生表关系")
|
||||||
|
public class DatacenterDerivedTable extends DateEntity implements Serializable {
|
||||||
|
@Id(keyType = KeyType.Generator, value = "snowFlakeId")
|
||||||
|
private BigInteger id;
|
||||||
|
@Column(comment = "部门ID")
|
||||||
|
private BigInteger deptId;
|
||||||
|
@Column(tenantId = true, comment = "租户ID")
|
||||||
|
private BigInteger tenantId;
|
||||||
|
@Column(comment = "源表ID")
|
||||||
|
private BigInteger sourceTableId;
|
||||||
|
@Column(comment = "派生表ID")
|
||||||
|
private BigInteger derivedTableId;
|
||||||
|
@Column(comment = "派生类型")
|
||||||
|
private String deriveType;
|
||||||
|
@Column(typeHandler = FastjsonTypeHandler.class, comment = "派生配置")
|
||||||
|
private Map<String, Object> deriveConfigJson;
|
||||||
|
@Column(comment = "状态")
|
||||||
|
private Integer status;
|
||||||
|
@Column(comment = "创建时间")
|
||||||
|
private Date created;
|
||||||
|
@Column(comment = "创建人")
|
||||||
|
private BigInteger createdBy;
|
||||||
|
@Column(comment = "修改时间")
|
||||||
|
private Date modified;
|
||||||
|
@Column(comment = "修改人")
|
||||||
|
private BigInteger modifiedBy;
|
||||||
|
|
||||||
|
public BigInteger getId() { return id; }
|
||||||
|
public void setId(BigInteger id) { this.id = id; }
|
||||||
|
public BigInteger getDeptId() { return deptId; }
|
||||||
|
public void setDeptId(BigInteger deptId) { this.deptId = deptId; }
|
||||||
|
public BigInteger getTenantId() { return tenantId; }
|
||||||
|
public void setTenantId(BigInteger tenantId) { this.tenantId = tenantId; }
|
||||||
|
public BigInteger getSourceTableId() { return sourceTableId; }
|
||||||
|
public void setSourceTableId(BigInteger sourceTableId) { this.sourceTableId = sourceTableId; }
|
||||||
|
public BigInteger getDerivedTableId() { return derivedTableId; }
|
||||||
|
public void setDerivedTableId(BigInteger derivedTableId) { this.derivedTableId = derivedTableId; }
|
||||||
|
public String getDeriveType() { return deriveType; }
|
||||||
|
public void setDeriveType(String deriveType) { this.deriveType = deriveType; }
|
||||||
|
public Map<String, Object> getDeriveConfigJson() { return deriveConfigJson; }
|
||||||
|
public void setDeriveConfigJson(Map<String, Object> deriveConfigJson) { this.deriveConfigJson = deriveConfigJson; }
|
||||||
|
public Integer getStatus() { return status; }
|
||||||
|
public void setStatus(Integer status) { this.status = status; }
|
||||||
|
@Override public Date getCreated() { return created; }
|
||||||
|
@Override public void setCreated(Date created) { this.created = created; }
|
||||||
|
public BigInteger getCreatedBy() { return createdBy; }
|
||||||
|
public void setCreatedBy(BigInteger createdBy) { this.createdBy = createdBy; }
|
||||||
|
@Override public Date getModified() { return modified; }
|
||||||
|
@Override public void setModified(Date modified) { this.modified = modified; }
|
||||||
|
public BigInteger getModifiedBy() { return modifiedBy; }
|
||||||
|
public void setModifiedBy(BigInteger modifiedBy) { this.modifiedBy = modifiedBy; }
|
||||||
|
}
|
||||||
@@ -0,0 +1,102 @@
|
|||||||
|
package tech.easyflow.datacenter.meta.entity;
|
||||||
|
|
||||||
|
import com.mybatisflex.annotation.Column;
|
||||||
|
import com.mybatisflex.annotation.Id;
|
||||||
|
import com.mybatisflex.annotation.KeyType;
|
||||||
|
import com.mybatisflex.annotation.Table;
|
||||||
|
import com.mybatisflex.core.handler.FastjsonTypeHandler;
|
||||||
|
import tech.easyflow.common.entity.DateEntity;
|
||||||
|
|
||||||
|
import java.io.Serializable;
|
||||||
|
import java.math.BigInteger;
|
||||||
|
import java.util.Date;
|
||||||
|
import java.util.Map;
|
||||||
|
|
||||||
|
@Table(value = "tb_datacenter_import_job", comment = "数据中心导入任务")
|
||||||
|
public class DatacenterImportJob extends DateEntity implements Serializable {
|
||||||
|
@Id(keyType = KeyType.Generator, value = "snowFlakeId")
|
||||||
|
private BigInteger id;
|
||||||
|
@Column(comment = "部门ID")
|
||||||
|
private BigInteger deptId;
|
||||||
|
@Column(tenantId = true, comment = "租户ID")
|
||||||
|
private BigInteger tenantId;
|
||||||
|
@Column(comment = "数据源ID")
|
||||||
|
private BigInteger sourceId;
|
||||||
|
@Column(comment = "目录ID")
|
||||||
|
private BigInteger catalogId;
|
||||||
|
@Column(comment = "表ID")
|
||||||
|
private BigInteger tableId;
|
||||||
|
@Column(comment = "任务类型")
|
||||||
|
private String jobType;
|
||||||
|
@Column(comment = "文件名")
|
||||||
|
private String fileName;
|
||||||
|
@Column(comment = "文件存储路径")
|
||||||
|
private String storagePath;
|
||||||
|
@Column(comment = "任务状态")
|
||||||
|
private String status;
|
||||||
|
@Column(comment = "总行数")
|
||||||
|
private Long totalRows;
|
||||||
|
@Column(comment = "成功行数")
|
||||||
|
private Long successRows;
|
||||||
|
@Column(comment = "失败行数")
|
||||||
|
private Long errorRows;
|
||||||
|
@Column(comment = "错误摘要")
|
||||||
|
private String errorSummary;
|
||||||
|
@Column(typeHandler = FastjsonTypeHandler.class, comment = "任务载荷")
|
||||||
|
private Map<String, Object> payloadJson;
|
||||||
|
@Column(comment = "开始时间")
|
||||||
|
private Date startedAt;
|
||||||
|
@Column(comment = "结束时间")
|
||||||
|
private Date finishedAt;
|
||||||
|
@Column(comment = "创建时间")
|
||||||
|
private Date created;
|
||||||
|
@Column(comment = "创建人")
|
||||||
|
private BigInteger createdBy;
|
||||||
|
@Column(comment = "修改时间")
|
||||||
|
private Date modified;
|
||||||
|
@Column(comment = "修改人")
|
||||||
|
private BigInteger modifiedBy;
|
||||||
|
|
||||||
|
public BigInteger getId() { return id; }
|
||||||
|
public void setId(BigInteger id) { this.id = id; }
|
||||||
|
public BigInteger getDeptId() { return deptId; }
|
||||||
|
public void setDeptId(BigInteger deptId) { this.deptId = deptId; }
|
||||||
|
public BigInteger getTenantId() { return tenantId; }
|
||||||
|
public void setTenantId(BigInteger tenantId) { this.tenantId = tenantId; }
|
||||||
|
public BigInteger getSourceId() { return sourceId; }
|
||||||
|
public void setSourceId(BigInteger sourceId) { this.sourceId = sourceId; }
|
||||||
|
public BigInteger getCatalogId() { return catalogId; }
|
||||||
|
public void setCatalogId(BigInteger catalogId) { this.catalogId = catalogId; }
|
||||||
|
public BigInteger getTableId() { return tableId; }
|
||||||
|
public void setTableId(BigInteger tableId) { this.tableId = tableId; }
|
||||||
|
public String getJobType() { return jobType; }
|
||||||
|
public void setJobType(String jobType) { this.jobType = jobType; }
|
||||||
|
public String getFileName() { return fileName; }
|
||||||
|
public void setFileName(String fileName) { this.fileName = fileName; }
|
||||||
|
public String getStoragePath() { return storagePath; }
|
||||||
|
public void setStoragePath(String storagePath) { this.storagePath = storagePath; }
|
||||||
|
public String getStatus() { return status; }
|
||||||
|
public void setStatus(String status) { this.status = status; }
|
||||||
|
public Long getTotalRows() { return totalRows; }
|
||||||
|
public void setTotalRows(Long totalRows) { this.totalRows = totalRows; }
|
||||||
|
public Long getSuccessRows() { return successRows; }
|
||||||
|
public void setSuccessRows(Long successRows) { this.successRows = successRows; }
|
||||||
|
public Long getErrorRows() { return errorRows; }
|
||||||
|
public void setErrorRows(Long errorRows) { this.errorRows = errorRows; }
|
||||||
|
public String getErrorSummary() { return errorSummary; }
|
||||||
|
public void setErrorSummary(String errorSummary) { this.errorSummary = errorSummary; }
|
||||||
|
public Map<String, Object> getPayloadJson() { return payloadJson; }
|
||||||
|
public void setPayloadJson(Map<String, Object> payloadJson) { this.payloadJson = payloadJson; }
|
||||||
|
public Date getStartedAt() { return startedAt; }
|
||||||
|
public void setStartedAt(Date startedAt) { this.startedAt = startedAt; }
|
||||||
|
public Date getFinishedAt() { return finishedAt; }
|
||||||
|
public void setFinishedAt(Date finishedAt) { this.finishedAt = finishedAt; }
|
||||||
|
@Override public Date getCreated() { return created; }
|
||||||
|
@Override public void setCreated(Date created) { this.created = created; }
|
||||||
|
public BigInteger getCreatedBy() { return createdBy; }
|
||||||
|
public void setCreatedBy(BigInteger createdBy) { this.createdBy = createdBy; }
|
||||||
|
@Override public Date getModified() { return modified; }
|
||||||
|
@Override public void setModified(Date modified) { this.modified = modified; }
|
||||||
|
public BigInteger getModifiedBy() { return modifiedBy; }
|
||||||
|
public void setModifiedBy(BigInteger modifiedBy) { this.modifiedBy = modifiedBy; }
|
||||||
|
}
|
||||||
@@ -0,0 +1,131 @@
|
|||||||
|
package tech.easyflow.datacenter.meta.entity;
|
||||||
|
|
||||||
|
import com.mybatisflex.annotation.Column;
|
||||||
|
import com.mybatisflex.annotation.Id;
|
||||||
|
import com.mybatisflex.annotation.KeyType;
|
||||||
|
import com.mybatisflex.annotation.Table;
|
||||||
|
import com.mybatisflex.core.handler.FastjsonTypeHandler;
|
||||||
|
import tech.easyflow.common.entity.DateEntity;
|
||||||
|
|
||||||
|
import java.io.Serializable;
|
||||||
|
import java.math.BigInteger;
|
||||||
|
import java.util.Date;
|
||||||
|
import java.util.Map;
|
||||||
|
|
||||||
|
@Table(value = "tb_datacenter_source", comment = "数据中心数据源")
|
||||||
|
public class DatacenterSource extends DateEntity implements Serializable {
|
||||||
|
|
||||||
|
@Id(keyType = KeyType.Generator, value = "snowFlakeId", comment = "主键")
|
||||||
|
private BigInteger id;
|
||||||
|
@Column(comment = "部门ID")
|
||||||
|
private BigInteger deptId;
|
||||||
|
@Column(tenantId = true, comment = "租户ID")
|
||||||
|
private BigInteger tenantId;
|
||||||
|
@Column(comment = "数据源名称")
|
||||||
|
private String sourceName;
|
||||||
|
@Column(comment = "数据源编码")
|
||||||
|
private String sourceCode;
|
||||||
|
@Column(comment = "数据源类型")
|
||||||
|
private String sourceType;
|
||||||
|
@Column(comment = "访问模式")
|
||||||
|
private String accessMode;
|
||||||
|
@Column(comment = "是否内置")
|
||||||
|
private Integer builtinFlag;
|
||||||
|
@Column(comment = "驱动类名")
|
||||||
|
private String driverClassName;
|
||||||
|
@Column(comment = "JDBC URL")
|
||||||
|
private String jdbcUrl;
|
||||||
|
@Column(comment = "主机")
|
||||||
|
private String host;
|
||||||
|
@Column(comment = "端口")
|
||||||
|
private Integer port;
|
||||||
|
@Column(comment = "数据库名")
|
||||||
|
private String databaseName;
|
||||||
|
@Column(comment = "Schema名")
|
||||||
|
private String schemaName;
|
||||||
|
@Column(comment = "用户名")
|
||||||
|
private String username;
|
||||||
|
@Column(comment = "凭据密文")
|
||||||
|
private String credentialCipher;
|
||||||
|
@Column(typeHandler = FastjsonTypeHandler.class, comment = "连接配置")
|
||||||
|
private Map<String, Object> configJson;
|
||||||
|
@Column(typeHandler = FastjsonTypeHandler.class, comment = "能力声明")
|
||||||
|
private Map<String, Object> capabilitiesJson;
|
||||||
|
@Column(comment = "最近测试状态")
|
||||||
|
private String lastTestStatus;
|
||||||
|
@Column(comment = "最近测试信息")
|
||||||
|
private String lastTestMessage;
|
||||||
|
@Column(comment = "最近测试时间")
|
||||||
|
private Date lastTestedAt;
|
||||||
|
@Column(comment = "状态")
|
||||||
|
private Integer status;
|
||||||
|
@Column(typeHandler = FastjsonTypeHandler.class, comment = "扩展项")
|
||||||
|
private Map<String, Object> options;
|
||||||
|
@Column(comment = "创建时间")
|
||||||
|
private Date created;
|
||||||
|
@Column(comment = "创建人")
|
||||||
|
private BigInteger createdBy;
|
||||||
|
@Column(comment = "修改时间")
|
||||||
|
private Date modified;
|
||||||
|
@Column(comment = "修改人")
|
||||||
|
private BigInteger modifiedBy;
|
||||||
|
|
||||||
|
public BigInteger getId() { return id; }
|
||||||
|
public void setId(BigInteger id) { this.id = id; }
|
||||||
|
public BigInteger getDeptId() { return deptId; }
|
||||||
|
public void setDeptId(BigInteger deptId) { this.deptId = deptId; }
|
||||||
|
public BigInteger getTenantId() { return tenantId; }
|
||||||
|
public void setTenantId(BigInteger tenantId) { this.tenantId = tenantId; }
|
||||||
|
public String getSourceName() { return sourceName; }
|
||||||
|
public void setSourceName(String sourceName) { this.sourceName = sourceName; }
|
||||||
|
public String getSourceCode() { return sourceCode; }
|
||||||
|
public void setSourceCode(String sourceCode) { this.sourceCode = sourceCode; }
|
||||||
|
public String getSourceType() { return sourceType; }
|
||||||
|
public void setSourceType(String sourceType) { this.sourceType = sourceType; }
|
||||||
|
public String getAccessMode() { return accessMode; }
|
||||||
|
public void setAccessMode(String accessMode) { this.accessMode = accessMode; }
|
||||||
|
public Integer getBuiltinFlag() { return builtinFlag; }
|
||||||
|
public void setBuiltinFlag(Integer builtinFlag) { this.builtinFlag = builtinFlag; }
|
||||||
|
public String getDriverClassName() { return driverClassName; }
|
||||||
|
public void setDriverClassName(String driverClassName) { this.driverClassName = driverClassName; }
|
||||||
|
public String getJdbcUrl() { return jdbcUrl; }
|
||||||
|
public void setJdbcUrl(String jdbcUrl) { this.jdbcUrl = jdbcUrl; }
|
||||||
|
public String getHost() { return host; }
|
||||||
|
public void setHost(String host) { this.host = host; }
|
||||||
|
public Integer getPort() { return port; }
|
||||||
|
public void setPort(Integer port) { this.port = port; }
|
||||||
|
public String getDatabaseName() { return databaseName; }
|
||||||
|
public void setDatabaseName(String databaseName) { this.databaseName = databaseName; }
|
||||||
|
public String getSchemaName() { return schemaName; }
|
||||||
|
public void setSchemaName(String schemaName) { this.schemaName = schemaName; }
|
||||||
|
public String getUsername() { return username; }
|
||||||
|
public void setUsername(String username) { this.username = username; }
|
||||||
|
public String getCredentialCipher() { return credentialCipher; }
|
||||||
|
public void setCredentialCipher(String credentialCipher) { this.credentialCipher = credentialCipher; }
|
||||||
|
public Map<String, Object> getConfigJson() { return configJson; }
|
||||||
|
public void setConfigJson(Map<String, Object> configJson) { this.configJson = configJson; }
|
||||||
|
public Map<String, Object> getCapabilitiesJson() { return capabilitiesJson; }
|
||||||
|
public void setCapabilitiesJson(Map<String, Object> capabilitiesJson) { this.capabilitiesJson = capabilitiesJson; }
|
||||||
|
public String getLastTestStatus() { return lastTestStatus; }
|
||||||
|
public void setLastTestStatus(String lastTestStatus) { this.lastTestStatus = lastTestStatus; }
|
||||||
|
public String getLastTestMessage() { return lastTestMessage; }
|
||||||
|
public void setLastTestMessage(String lastTestMessage) { this.lastTestMessage = lastTestMessage; }
|
||||||
|
public Date getLastTestedAt() { return lastTestedAt; }
|
||||||
|
public void setLastTestedAt(Date lastTestedAt) { this.lastTestedAt = lastTestedAt; }
|
||||||
|
public Integer getStatus() { return status; }
|
||||||
|
public void setStatus(Integer status) { this.status = status; }
|
||||||
|
public Map<String, Object> getOptions() { return options; }
|
||||||
|
public void setOptions(Map<String, Object> options) { this.options = options; }
|
||||||
|
@Override
|
||||||
|
public Date getCreated() { return created; }
|
||||||
|
@Override
|
||||||
|
public void setCreated(Date created) { this.created = created; }
|
||||||
|
public BigInteger getCreatedBy() { return createdBy; }
|
||||||
|
public void setCreatedBy(BigInteger createdBy) { this.createdBy = createdBy; }
|
||||||
|
@Override
|
||||||
|
public Date getModified() { return modified; }
|
||||||
|
@Override
|
||||||
|
public void setModified(Date modified) { this.modified = modified; }
|
||||||
|
public BigInteger getModifiedBy() { return modifiedBy; }
|
||||||
|
public void setModifiedBy(BigInteger modifiedBy) { this.modifiedBy = modifiedBy; }
|
||||||
|
}
|
||||||
@@ -0,0 +1,6 @@
|
|||||||
|
package tech.easyflow.datacenter.meta.enums;
|
||||||
|
|
||||||
|
public enum DatacenterAccessMode {
|
||||||
|
READ_ONLY,
|
||||||
|
READ_WRITE
|
||||||
|
}
|
||||||
@@ -0,0 +1,10 @@
|
|||||||
|
package tech.easyflow.datacenter.meta.enums;
|
||||||
|
|
||||||
|
public enum DatacenterCapability {
|
||||||
|
TEST_CONNECTION,
|
||||||
|
BROWSE_METADATA,
|
||||||
|
READ_QUERY,
|
||||||
|
WRITE_MUTATION,
|
||||||
|
EXPORT,
|
||||||
|
MATERIALIZE
|
||||||
|
}
|
||||||
@@ -0,0 +1,12 @@
|
|||||||
|
package tech.easyflow.datacenter.meta.enums;
|
||||||
|
|
||||||
|
public enum DatacenterConnectionErrorCode {
|
||||||
|
INVALID_ARGUMENT,
|
||||||
|
DRIVER_NOT_FOUND,
|
||||||
|
NETWORK_UNREACHABLE,
|
||||||
|
AUTH_FAILED,
|
||||||
|
DATABASE_NOT_FOUND,
|
||||||
|
SCHEMA_NOT_FOUND,
|
||||||
|
PERMISSION_DENIED,
|
||||||
|
UNKNOWN_ERROR
|
||||||
|
}
|
||||||
@@ -0,0 +1,9 @@
|
|||||||
|
package tech.easyflow.datacenter.meta.enums;
|
||||||
|
|
||||||
|
public enum DatacenterImportStatus {
|
||||||
|
PENDING,
|
||||||
|
RUNNING,
|
||||||
|
SUCCESS,
|
||||||
|
FAILED,
|
||||||
|
NOT_IMPLEMENTED
|
||||||
|
}
|
||||||
@@ -0,0 +1,13 @@
|
|||||||
|
package tech.easyflow.datacenter.meta.enums;
|
||||||
|
|
||||||
|
public enum DatacenterSourceType {
|
||||||
|
PROJECT_MYSQL,
|
||||||
|
EXCEL,
|
||||||
|
EXCEL_MATERIALIZED,
|
||||||
|
MYSQL,
|
||||||
|
POSTGRESQL,
|
||||||
|
ORACLE,
|
||||||
|
GAUSSDB_NATIVE,
|
||||||
|
GBASE_8A,
|
||||||
|
GBASE_8S
|
||||||
|
}
|
||||||
@@ -0,0 +1,10 @@
|
|||||||
|
package tech.easyflow.datacenter.meta.enums;
|
||||||
|
|
||||||
|
public enum DatacenterTableKind {
|
||||||
|
PROJECT_MANAGED,
|
||||||
|
EXTERNAL_TABLE,
|
||||||
|
EXTERNAL_VIEW,
|
||||||
|
EXCEL_SHEET,
|
||||||
|
EXCEL_MATERIALIZED,
|
||||||
|
DERIVED_TABLE
|
||||||
|
}
|
||||||
@@ -0,0 +1,36 @@
|
|||||||
|
package tech.easyflow.datacenter.meta.model;
|
||||||
|
|
||||||
|
import java.math.BigInteger;
|
||||||
|
import java.util.ArrayList;
|
||||||
|
import java.util.List;
|
||||||
|
|
||||||
|
public class DatacenterBatchRegisterRequest {
|
||||||
|
|
||||||
|
private BigInteger sourceId;
|
||||||
|
private String catalogName;
|
||||||
|
private List<String> tableNames = new ArrayList<>();
|
||||||
|
|
||||||
|
public BigInteger getSourceId() {
|
||||||
|
return sourceId;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setSourceId(BigInteger sourceId) {
|
||||||
|
this.sourceId = sourceId;
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getCatalogName() {
|
||||||
|
return catalogName;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setCatalogName(String catalogName) {
|
||||||
|
this.catalogName = catalogName;
|
||||||
|
}
|
||||||
|
|
||||||
|
public List<String> getTableNames() {
|
||||||
|
return tableNames;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setTableNames(List<String> tableNames) {
|
||||||
|
this.tableNames = tableNames;
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,18 @@
|
|||||||
|
package tech.easyflow.datacenter.meta.model;
|
||||||
|
|
||||||
|
import java.math.BigInteger;
|
||||||
|
import java.util.ArrayList;
|
||||||
|
import java.util.List;
|
||||||
|
|
||||||
|
public class DatacenterBatchRemoveRequest {
|
||||||
|
|
||||||
|
private List<BigInteger> tableIds = new ArrayList<>();
|
||||||
|
|
||||||
|
public List<BigInteger> getTableIds() {
|
||||||
|
return tableIds;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setTableIds(List<BigInteger> tableIds) {
|
||||||
|
this.tableIds = tableIds;
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,22 @@
|
|||||||
|
package tech.easyflow.datacenter.meta.model;
|
||||||
|
|
||||||
|
import java.math.BigInteger;
|
||||||
|
|
||||||
|
public class DatacenterCatalogMeta {
|
||||||
|
private BigInteger id;
|
||||||
|
private BigInteger sourceId;
|
||||||
|
private String catalogName;
|
||||||
|
private String catalogType;
|
||||||
|
private String catalogDesc;
|
||||||
|
|
||||||
|
public BigInteger getId() { return id; }
|
||||||
|
public void setId(BigInteger id) { this.id = id; }
|
||||||
|
public BigInteger getSourceId() { return sourceId; }
|
||||||
|
public void setSourceId(BigInteger sourceId) { this.sourceId = sourceId; }
|
||||||
|
public String getCatalogName() { return catalogName; }
|
||||||
|
public void setCatalogName(String catalogName) { this.catalogName = catalogName; }
|
||||||
|
public String getCatalogType() { return catalogType; }
|
||||||
|
public void setCatalogType(String catalogType) { this.catalogType = catalogType; }
|
||||||
|
public String getCatalogDesc() { return catalogDesc; }
|
||||||
|
public void setCatalogDesc(String catalogDesc) { this.catalogDesc = catalogDesc; }
|
||||||
|
}
|
||||||
@@ -0,0 +1,25 @@
|
|||||||
|
package tech.easyflow.datacenter.meta.model;
|
||||||
|
|
||||||
|
import java.math.BigInteger;
|
||||||
|
|
||||||
|
public class DatacenterFieldDescriptionUpdate {
|
||||||
|
|
||||||
|
private BigInteger fieldId;
|
||||||
|
private String fieldDesc;
|
||||||
|
|
||||||
|
public BigInteger getFieldId() {
|
||||||
|
return fieldId;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setFieldId(BigInteger fieldId) {
|
||||||
|
this.fieldId = fieldId;
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getFieldDesc() {
|
||||||
|
return fieldDesc;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setFieldDesc(String fieldDesc) {
|
||||||
|
this.fieldDesc = fieldDesc;
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,16 @@
|
|||||||
|
package tech.easyflow.datacenter.meta.model;
|
||||||
|
|
||||||
|
import java.math.BigInteger;
|
||||||
|
|
||||||
|
public class DatacenterRemoveSourceRequest {
|
||||||
|
|
||||||
|
private BigInteger sourceId;
|
||||||
|
|
||||||
|
public BigInteger getSourceId() {
|
||||||
|
return sourceId;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setSourceId(BigInteger sourceId) {
|
||||||
|
this.sourceId = sourceId;
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,36 @@
|
|||||||
|
package tech.easyflow.datacenter.meta.model;
|
||||||
|
|
||||||
|
import java.math.BigInteger;
|
||||||
|
import java.util.ArrayList;
|
||||||
|
import java.util.List;
|
||||||
|
|
||||||
|
public class DatacenterSaveDescriptionsRequest {
|
||||||
|
|
||||||
|
private BigInteger tableId;
|
||||||
|
private String tableDesc;
|
||||||
|
private List<DatacenterFieldDescriptionUpdate> fields = new ArrayList<>();
|
||||||
|
|
||||||
|
public BigInteger getTableId() {
|
||||||
|
return tableId;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setTableId(BigInteger tableId) {
|
||||||
|
this.tableId = tableId;
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getTableDesc() {
|
||||||
|
return tableDesc;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setTableDesc(String tableDesc) {
|
||||||
|
this.tableDesc = tableDesc;
|
||||||
|
}
|
||||||
|
|
||||||
|
public List<DatacenterFieldDescriptionUpdate> getFields() {
|
||||||
|
return fields;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setFields(List<DatacenterFieldDescriptionUpdate> fields) {
|
||||||
|
this.fields = fields;
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,17 @@
|
|||||||
|
package tech.easyflow.datacenter.meta.model;
|
||||||
|
|
||||||
|
import tech.easyflow.datacenter.entity.DatacenterTable;
|
||||||
|
import tech.easyflow.datacenter.entity.DatacenterTableField;
|
||||||
|
|
||||||
|
import java.util.ArrayList;
|
||||||
|
import java.util.List;
|
||||||
|
|
||||||
|
public class DatacenterTableDetailMeta {
|
||||||
|
private DatacenterTable table;
|
||||||
|
private List<DatacenterTableField> fields = new ArrayList<>();
|
||||||
|
|
||||||
|
public DatacenterTable getTable() { return table; }
|
||||||
|
public void setTable(DatacenterTable table) { this.table = table; }
|
||||||
|
public List<DatacenterTableField> getFields() { return fields; }
|
||||||
|
public void setFields(List<DatacenterTableField> fields) { this.fields = fields; }
|
||||||
|
}
|
||||||
@@ -0,0 +1,38 @@
|
|||||||
|
package tech.easyflow.datacenter.meta.service;
|
||||||
|
|
||||||
|
import tech.easyflow.common.entity.LoginAccount;
|
||||||
|
import tech.easyflow.datacenter.entity.DatacenterTable;
|
||||||
|
import tech.easyflow.datacenter.entity.DatacenterTableField;
|
||||||
|
import tech.easyflow.datacenter.execution.model.DatasetRef;
|
||||||
|
import tech.easyflow.datacenter.meta.entity.DatacenterCatalog;
|
||||||
|
import tech.easyflow.datacenter.meta.entity.DatacenterSource;
|
||||||
|
import tech.easyflow.datacenter.meta.model.DatacenterFieldDescriptionUpdate;
|
||||||
|
import tech.easyflow.datacenter.meta.enums.DatacenterSourceType;
|
||||||
|
import tech.easyflow.datacenter.meta.model.DatacenterTableDetailMeta;
|
||||||
|
|
||||||
|
import java.math.BigInteger;
|
||||||
|
import java.util.List;
|
||||||
|
|
||||||
|
public interface DatacenterDatasetRegistryService {
|
||||||
|
DatacenterSource ensureBuiltinSource(DatacenterSourceType sourceType, LoginAccount account);
|
||||||
|
|
||||||
|
DatacenterCatalog ensureCatalog(DatacenterSource source, String catalogName, LoginAccount account);
|
||||||
|
|
||||||
|
DatacenterTable registerTable(DatacenterSource source, DatacenterCatalog catalog, DatacenterTableDetailMeta detail, LoginAccount account);
|
||||||
|
|
||||||
|
DatacenterTable getTableWithFields(BigInteger tableId);
|
||||||
|
|
||||||
|
List<DatacenterTableField> getFields(BigInteger tableId);
|
||||||
|
|
||||||
|
DatasetRef resolveDatasetRef(BigInteger tableId);
|
||||||
|
|
||||||
|
DatacenterSource getSourceRequired(BigInteger sourceId);
|
||||||
|
|
||||||
|
DatacenterCatalog getCatalogById(BigInteger catalogId);
|
||||||
|
|
||||||
|
List<DatacenterTable> listManagedTables(BigInteger sourceId, BigInteger catalogId);
|
||||||
|
|
||||||
|
int removeTables(List<BigInteger> tableIds);
|
||||||
|
|
||||||
|
DatacenterTable saveDescriptions(BigInteger tableId, String tableDesc, List<DatacenterFieldDescriptionUpdate> fields, LoginAccount account);
|
||||||
|
}
|
||||||
@@ -0,0 +1,12 @@
|
|||||||
|
package tech.easyflow.datacenter.meta.service;
|
||||||
|
|
||||||
|
import java.math.BigInteger;
|
||||||
|
|
||||||
|
public final class DatacenterMetaConstants {
|
||||||
|
|
||||||
|
private DatacenterMetaConstants() {
|
||||||
|
}
|
||||||
|
|
||||||
|
public static final BigInteger PROJECT_SOURCE_BASE = new BigInteger("9000000000002000000");
|
||||||
|
public static final BigInteger PROJECT_CATALOG_BASE = new BigInteger("9000000000003000000");
|
||||||
|
}
|
||||||
@@ -0,0 +1,32 @@
|
|||||||
|
package tech.easyflow.datacenter.meta.service;
|
||||||
|
|
||||||
|
import com.mybatisflex.core.paginate.Page;
|
||||||
|
import com.mybatisflex.core.service.IService;
|
||||||
|
import tech.easyflow.common.entity.LoginAccount;
|
||||||
|
import tech.easyflow.datacenter.entity.DatacenterTable;
|
||||||
|
import tech.easyflow.datacenter.execution.model.DatacenterConnectionTestResult;
|
||||||
|
import tech.easyflow.datacenter.meta.entity.DatacenterSource;
|
||||||
|
import tech.easyflow.datacenter.meta.model.DatacenterBatchRegisterRequest;
|
||||||
|
import tech.easyflow.datacenter.meta.model.DatacenterCatalogMeta;
|
||||||
|
import tech.easyflow.datacenter.meta.model.DatacenterTableDetailMeta;
|
||||||
|
|
||||||
|
import java.math.BigInteger;
|
||||||
|
import java.util.List;
|
||||||
|
|
||||||
|
public interface DatacenterSourceService extends IService<DatacenterSource> {
|
||||||
|
DatacenterSource saveSource(DatacenterSource source, LoginAccount account);
|
||||||
|
|
||||||
|
Page<DatacenterSource> pageSources(Long pageNumber, Long pageSize, LoginAccount account);
|
||||||
|
|
||||||
|
DatacenterConnectionTestResult testConnection(DatacenterSource source, LoginAccount account);
|
||||||
|
|
||||||
|
List<DatacenterCatalogMeta> listCatalogs(BigInteger sourceId, LoginAccount account);
|
||||||
|
|
||||||
|
List<DatacenterTable> listTables(BigInteger sourceId, String catalogName, LoginAccount account);
|
||||||
|
|
||||||
|
DatacenterTableDetailMeta getTableDetail(BigInteger sourceId, String catalogName, String tableName, boolean register, LoginAccount account);
|
||||||
|
|
||||||
|
List<DatacenterTable> batchRegisterTables(DatacenterBatchRegisterRequest request, LoginAccount account);
|
||||||
|
|
||||||
|
void removeSource(BigInteger sourceId, LoginAccount account);
|
||||||
|
}
|
||||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user