Compare commits
2 Commits
b6213d0933
...
1ecc28e498
| Author | SHA1 | Date | |
|---|---|---|---|
| 1ecc28e498 | |||
| 798effbd5b |
@@ -13,6 +13,7 @@ import com.easyagents.flow.core.parser.ChainParser;
|
||||
import org.springframework.web.bind.annotation.GetMapping;
|
||||
import org.springframework.web.bind.annotation.RequestMapping;
|
||||
import org.springframework.web.bind.annotation.RestController;
|
||||
import tech.easyflow.ai.easyagentsflow.service.WorkflowDatacenterContentService;
|
||||
import tech.easyflow.ai.entity.Workflow;
|
||||
import tech.easyflow.ai.service.WorkflowService;
|
||||
import tech.easyflow.common.domain.Result;
|
||||
@@ -29,6 +30,8 @@ public class WorkFlowNodeController {
|
||||
private WorkflowService workflowService;
|
||||
@Resource
|
||||
private ChainParser chainParser;
|
||||
@Resource
|
||||
private WorkflowDatacenterContentService workflowDatacenterContentService;
|
||||
|
||||
@GetMapping("/getChainParams")
|
||||
public Result<?> getChainParams(String currentId, String workflowId) {
|
||||
@@ -43,7 +46,7 @@ public class WorkFlowNodeController {
|
||||
nodeData.put("workflowId", workflow.getId());
|
||||
nodeData.put("workflowName", workflow.getTitle());
|
||||
|
||||
ChainDefinition definition = chainParser.parse(workflow.getContent());
|
||||
ChainDefinition definition = chainParser.parse(workflowDatacenterContentService.prepareContent(workflow.getContent()));
|
||||
List<Node> nodes = definition.getNodes();
|
||||
JSONArray inputs = new JSONArray();
|
||||
JSONArray outputs = new JSONArray();
|
||||
|
||||
@@ -21,6 +21,7 @@ import tech.easyflow.ai.easyagentsflow.entity.WorkflowCheckStage;
|
||||
import tech.easyflow.ai.easyagentsflow.service.CodeEngineCapabilityService;
|
||||
import tech.easyflow.ai.easyagentsflow.service.TinyFlowService;
|
||||
import tech.easyflow.ai.easyagentsflow.service.WorkflowCheckService;
|
||||
import tech.easyflow.ai.easyagentsflow.service.WorkflowDatacenterContentService;
|
||||
import tech.easyflow.ai.entity.Workflow;
|
||||
import tech.easyflow.ai.service.BotWorkflowService;
|
||||
import tech.easyflow.ai.service.ModelService;
|
||||
@@ -76,6 +77,8 @@ public class WorkflowController extends BaseCurdController<WorkflowService, Work
|
||||
@Resource
|
||||
private WorkflowCheckService workflowCheckService;
|
||||
@Resource
|
||||
private WorkflowDatacenterContentService workflowDatacenterContentService;
|
||||
@Resource
|
||||
private ResourceAccessService resourceAccessService;
|
||||
@Resource
|
||||
private WorkflowVisibilityQueryHelper workflowVisibilityQueryHelper;
|
||||
@@ -223,7 +226,7 @@ public class WorkflowController extends BaseCurdController<WorkflowService, Work
|
||||
}
|
||||
workflowCheckService.checkOrThrow(workflow.getContent(), WorkflowCheckStage.PRE_EXECUTE, workflow.getId());
|
||||
|
||||
ChainDefinition definition = chainParser.parse(workflow.getContent());
|
||||
ChainDefinition definition = chainParser.parse(workflowDatacenterContentService.prepareContent(workflow.getContent()));
|
||||
if (definition == null) {
|
||||
return Result.fail(2, "节点配置错误,请检查! ");
|
||||
}
|
||||
|
||||
@@ -0,0 +1,68 @@
|
||||
package tech.easyflow.admin.controller.datacenter;
|
||||
|
||||
import cn.dev33.satoken.annotation.SaCheckPermission;
|
||||
import com.mybatisflex.core.paginate.Page;
|
||||
import com.mybatisflex.core.row.Row;
|
||||
import org.springframework.web.bind.annotation.*;
|
||||
import tech.easyflow.common.domain.Result;
|
||||
import tech.easyflow.common.entity.LoginAccount;
|
||||
import tech.easyflow.common.satoken.util.SaTokenUtil;
|
||||
import tech.easyflow.datacenter.entity.DatacenterTable;
|
||||
import tech.easyflow.datacenter.execution.model.DatacenterQueryRequest;
|
||||
import tech.easyflow.datacenter.execution.model.DatacenterSchemaResponse;
|
||||
import tech.easyflow.datacenter.execution.model.DatasetRef;
|
||||
import tech.easyflow.datacenter.meta.model.DatacenterBatchRemoveRequest;
|
||||
import tech.easyflow.datacenter.meta.model.DatacenterSaveDescriptionsRequest;
|
||||
import tech.easyflow.datacenter.execution.service.DatacenterDatasetQueryService;
|
||||
import tech.easyflow.datacenter.meta.service.DatacenterDatasetRegistryService;
|
||||
|
||||
import javax.annotation.Resource;
|
||||
import java.math.BigInteger;
|
||||
import java.util.List;
|
||||
|
||||
@RestController
|
||||
@RequestMapping("/api/v1/datacenterDataset")
|
||||
public class DatacenterDatasetController {
|
||||
|
||||
@Resource
|
||||
private DatacenterDatasetQueryService queryService;
|
||||
@Resource
|
||||
private DatacenterDatasetRegistryService registryService;
|
||||
|
||||
@PostMapping("/queryPage")
|
||||
@SaCheckPermission("/api/v1/datacenterSource/query")
|
||||
public Result<Page<Row>> queryPage(@RequestBody DatacenterQueryRequest request) {
|
||||
return Result.ok(queryService.queryPage(request));
|
||||
}
|
||||
|
||||
@GetMapping("/schema")
|
||||
@SaCheckPermission("/api/v1/datacenterSource/query")
|
||||
public Result<DatacenterSchemaResponse> schema(DatasetRef datasetRef) {
|
||||
return Result.ok(queryService.getSchema(datasetRef));
|
||||
}
|
||||
|
||||
@GetMapping("/managedTables")
|
||||
@SaCheckPermission("/api/v1/datacenterSource/query")
|
||||
public Result<List<DatacenterTable>> managedTables(BigInteger sourceId, BigInteger catalogId) {
|
||||
return Result.ok(registryService.listManagedTables(sourceId, catalogId));
|
||||
}
|
||||
|
||||
@PostMapping("/removeBatch")
|
||||
@SaCheckPermission("/api/v1/datacenterSource/save")
|
||||
public Result<Integer> removeBatch(@RequestBody DatacenterBatchRemoveRequest request) {
|
||||
return Result.ok(registryService.removeTables(request == null ? List.of() : request.getTableIds()));
|
||||
}
|
||||
|
||||
@PostMapping("/saveDescriptions")
|
||||
@SaCheckPermission("/api/v1/datacenterSource/save")
|
||||
public Result<DatacenterSchemaResponse> saveDescriptions(@RequestBody DatacenterSaveDescriptionsRequest request) {
|
||||
LoginAccount account = SaTokenUtil.getLoginAccount();
|
||||
DatacenterTable table = registryService.saveDescriptions(
|
||||
request == null ? null : request.getTableId(),
|
||||
request == null ? null : request.getTableDesc(),
|
||||
request == null ? List.of() : request.getFields(),
|
||||
account
|
||||
);
|
||||
return Result.ok(queryService.getSchema(registryService.resolveDatasetRef(table.getId())));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,108 @@
|
||||
package tech.easyflow.admin.controller.datacenter;
|
||||
|
||||
import cn.dev33.satoken.annotation.SaCheckPermission;
|
||||
import jakarta.servlet.http.HttpServletResponse;
|
||||
import org.springframework.web.bind.annotation.GetMapping;
|
||||
import org.springframework.web.bind.annotation.PostMapping;
|
||||
import org.springframework.web.bind.annotation.RequestBody;
|
||||
import org.springframework.web.bind.annotation.RequestMapping;
|
||||
import org.springframework.web.bind.annotation.RestController;
|
||||
import org.springframework.web.multipart.MultipartFile;
|
||||
import tech.easyflow.common.domain.Result;
|
||||
import tech.easyflow.common.entity.LoginAccount;
|
||||
import tech.easyflow.common.satoken.util.SaTokenUtil;
|
||||
import tech.easyflow.datacenter.excel.model.DatacenterExcelDeriveRequest;
|
||||
import tech.easyflow.datacenter.excel.model.DatacenterExcelExportRequest;
|
||||
import tech.easyflow.datacenter.excel.model.DatacenterExcelMergeRequest;
|
||||
import tech.easyflow.datacenter.excel.model.DatacenterExcelSplitRequest;
|
||||
import tech.easyflow.datacenter.excel.service.DatacenterExcelImportService;
|
||||
import tech.easyflow.datacenter.meta.entity.DatacenterImportJob;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.FileInputStream;
|
||||
import javax.annotation.Resource;
|
||||
import java.math.BigInteger;
|
||||
import java.net.URLEncoder;
|
||||
import java.util.List;
|
||||
|
||||
@RestController
|
||||
@RequestMapping("/api/v1/datacenterExcel")
|
||||
public class DatacenterExcelController {
|
||||
|
||||
@Resource
|
||||
private DatacenterExcelImportService excelImportService;
|
||||
|
||||
@PostMapping("/import")
|
||||
@SaCheckPermission("/api/v1/datacenterSource/save")
|
||||
public Result<DatacenterImportJob> importWorkbook(MultipartFile file) throws Exception {
|
||||
LoginAccount account = SaTokenUtil.getLoginAccount();
|
||||
return Result.ok(excelImportService.importWorkbook(file, account));
|
||||
}
|
||||
|
||||
@PostMapping("/split")
|
||||
@SaCheckPermission("/api/v1/datacenterSource/save")
|
||||
public Result<DatacenterImportJob> split(@RequestBody DatacenterExcelSplitRequest request) {
|
||||
LoginAccount account = SaTokenUtil.getLoginAccount();
|
||||
return Result.ok(excelImportService.splitWorkbook(request, account));
|
||||
}
|
||||
|
||||
@PostMapping("/merge")
|
||||
@SaCheckPermission("/api/v1/datacenterSource/save")
|
||||
public Result<DatacenterImportJob> merge(@RequestBody DatacenterExcelMergeRequest request) {
|
||||
LoginAccount account = SaTokenUtil.getLoginAccount();
|
||||
return Result.ok(excelImportService.mergeWorkbook(request, account));
|
||||
}
|
||||
|
||||
@PostMapping("/derive")
|
||||
@SaCheckPermission("/api/v1/datacenterSource/save")
|
||||
public Result<DatacenterImportJob> derive(@RequestBody DatacenterExcelDeriveRequest request) {
|
||||
LoginAccount account = SaTokenUtil.getLoginAccount();
|
||||
return Result.ok(excelImportService.deriveWorkbook(request, account));
|
||||
}
|
||||
|
||||
@PostMapping("/export")
|
||||
@SaCheckPermission("/api/v1/datacenterSource/query")
|
||||
public Result<DatacenterImportJob> export(@RequestBody DatacenterExcelExportRequest request) throws Exception {
|
||||
LoginAccount account = SaTokenUtil.getLoginAccount();
|
||||
return Result.ok(excelImportService.exportWorkbook(request, account));
|
||||
}
|
||||
|
||||
@GetMapping("/importJob/detail")
|
||||
@SaCheckPermission("/api/v1/datacenterSource/query")
|
||||
public Result<DatacenterImportJob> jobDetail(BigInteger jobId) {
|
||||
return Result.ok(excelImportService.getImportJobDetail(jobId));
|
||||
}
|
||||
|
||||
@GetMapping("/job/detail")
|
||||
@SaCheckPermission("/api/v1/datacenterSource/query")
|
||||
public Result<DatacenterImportJob> newJobDetail(BigInteger jobId) {
|
||||
return Result.ok(excelImportService.getImportJobDetail(jobId));
|
||||
}
|
||||
|
||||
@GetMapping("/job/list")
|
||||
@SaCheckPermission("/api/v1/datacenterSource/query")
|
||||
public Result<List<DatacenterImportJob>> jobList(BigInteger sourceId, BigInteger tableId) {
|
||||
return Result.ok(excelImportService.listJobs(sourceId, tableId));
|
||||
}
|
||||
|
||||
@GetMapping("/download")
|
||||
@SaCheckPermission("/api/v1/datacenterSource/query")
|
||||
public void download(BigInteger jobId, HttpServletResponse response) throws Exception {
|
||||
DatacenterImportJob job = excelImportService.getImportJobDetail(jobId);
|
||||
if (job.getStoragePath() == null || job.getStoragePath().isBlank()) {
|
||||
throw new IllegalStateException("导出文件不存在");
|
||||
}
|
||||
File file = new File(job.getStoragePath());
|
||||
if (!file.exists()) {
|
||||
throw new IllegalStateException("导出文件不存在");
|
||||
}
|
||||
response.setContentType("application/vnd.openxmlformats-officedocument.spreadsheetml.sheet");
|
||||
response.setCharacterEncoding("utf-8");
|
||||
String fileName = URLEncoder.encode(job.getFileName(), "UTF-8").replaceAll("\\+", "%20");
|
||||
response.setHeader("Content-disposition", "attachment;filename*=utf-8''" + fileName);
|
||||
try (FileInputStream inputStream = new FileInputStream(file)) {
|
||||
inputStream.transferTo(response.getOutputStream());
|
||||
response.flushBuffer();
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,86 @@
|
||||
package tech.easyflow.admin.controller.datacenter;
|
||||
|
||||
import cn.dev33.satoken.annotation.SaCheckPermission;
|
||||
import com.mybatisflex.core.paginate.Page;
|
||||
import org.springframework.web.bind.annotation.*;
|
||||
import tech.easyflow.common.domain.Result;
|
||||
import tech.easyflow.common.entity.LoginAccount;
|
||||
import tech.easyflow.common.satoken.util.SaTokenUtil;
|
||||
import tech.easyflow.datacenter.entity.DatacenterTable;
|
||||
import tech.easyflow.datacenter.execution.model.DatacenterConnectionTestResult;
|
||||
import tech.easyflow.datacenter.meta.entity.DatacenterSource;
|
||||
import tech.easyflow.datacenter.meta.model.DatacenterBatchRegisterRequest;
|
||||
import tech.easyflow.datacenter.meta.model.DatacenterCatalogMeta;
|
||||
import tech.easyflow.datacenter.meta.model.DatacenterRemoveSourceRequest;
|
||||
import tech.easyflow.datacenter.meta.model.DatacenterTableDetailMeta;
|
||||
import tech.easyflow.datacenter.meta.service.DatacenterSourceService;
|
||||
|
||||
import javax.annotation.Resource;
|
||||
import java.math.BigInteger;
|
||||
import java.util.List;
|
||||
|
||||
@RestController
|
||||
@RequestMapping("/api/v1/datacenterSource")
|
||||
public class DatacenterSourceController {
|
||||
|
||||
@Resource
|
||||
private DatacenterSourceService sourceService;
|
||||
|
||||
@PostMapping("/testConnection")
|
||||
@SaCheckPermission("/api/v1/datacenterSource/query")
|
||||
public Result<DatacenterConnectionTestResult> testConnection(@RequestBody DatacenterSource source) {
|
||||
LoginAccount account = SaTokenUtil.getLoginAccount();
|
||||
return Result.ok(sourceService.testConnection(source, account));
|
||||
}
|
||||
|
||||
@PostMapping("/save")
|
||||
@SaCheckPermission("/api/v1/datacenterSource/save")
|
||||
public Result<DatacenterSource> save(@RequestBody DatacenterSource source) {
|
||||
LoginAccount account = SaTokenUtil.getLoginAccount();
|
||||
return Result.ok(sourceService.saveSource(source, account));
|
||||
}
|
||||
|
||||
@GetMapping("/page")
|
||||
@SaCheckPermission("/api/v1/datacenterSource/query")
|
||||
public Result<Page<DatacenterSource>> page(Long pageNumber, Long pageSize) {
|
||||
LoginAccount account = SaTokenUtil.getLoginAccount();
|
||||
return Result.ok(sourceService.pageSources(pageNumber, pageSize, account));
|
||||
}
|
||||
|
||||
@GetMapping("/catalogs")
|
||||
@SaCheckPermission("/api/v1/datacenterSource/query")
|
||||
public Result<List<DatacenterCatalogMeta>> catalogs(BigInteger sourceId) {
|
||||
LoginAccount account = SaTokenUtil.getLoginAccount();
|
||||
return Result.ok(sourceService.listCatalogs(sourceId, account));
|
||||
}
|
||||
|
||||
@GetMapping("/tables")
|
||||
@SaCheckPermission("/api/v1/datacenterSource/query")
|
||||
public Result<List<DatacenterTable>> tables(BigInteger sourceId, String catalogName) {
|
||||
LoginAccount account = SaTokenUtil.getLoginAccount();
|
||||
return Result.ok(sourceService.listTables(sourceId, catalogName, account));
|
||||
}
|
||||
|
||||
@GetMapping("/tableDetail")
|
||||
@SaCheckPermission("/api/v1/datacenterSource/query")
|
||||
public Result<DatacenterTableDetailMeta> tableDetail(BigInteger sourceId, String catalogName, String tableName,
|
||||
@RequestParam(defaultValue = "false") boolean register) {
|
||||
LoginAccount account = SaTokenUtil.getLoginAccount();
|
||||
return Result.ok(sourceService.getTableDetail(sourceId, catalogName, tableName, register, account));
|
||||
}
|
||||
|
||||
@PostMapping("/registerBatch")
|
||||
@SaCheckPermission("/api/v1/datacenterSource/save")
|
||||
public Result<List<DatacenterTable>> registerBatch(@RequestBody DatacenterBatchRegisterRequest request) {
|
||||
LoginAccount account = SaTokenUtil.getLoginAccount();
|
||||
return Result.ok(sourceService.batchRegisterTables(request, account));
|
||||
}
|
||||
|
||||
@PostMapping("/remove")
|
||||
@SaCheckPermission("/api/v1/datacenterSource/save")
|
||||
public Result<Void> remove(@RequestBody DatacenterRemoveSourceRequest request) {
|
||||
LoginAccount account = SaTokenUtil.getLoginAccount();
|
||||
sourceService.removeSource(request == null ? null : request.getSourceId(), account);
|
||||
return Result.ok();
|
||||
}
|
||||
}
|
||||
@@ -1,195 +0,0 @@
|
||||
package tech.easyflow.admin.controller.datacenter;
|
||||
|
||||
import cn.dev33.satoken.annotation.SaCheckPermission;
|
||||
import cn.hutool.core.collection.CollectionUtil;
|
||||
import cn.idev.excel.EasyExcel;
|
||||
import cn.idev.excel.FastExcel;
|
||||
import com.alibaba.fastjson2.JSONObject;
|
||||
import com.mybatisflex.core.paginate.Page;
|
||||
import com.mybatisflex.core.query.QueryWrapper;
|
||||
import com.mybatisflex.core.row.Row;
|
||||
import jakarta.servlet.http.HttpServletResponse;
|
||||
import org.springframework.web.bind.annotation.*;
|
||||
import org.springframework.web.multipart.MultipartFile;
|
||||
import tech.easyflow.common.domain.Result;
|
||||
import tech.easyflow.common.entity.DatacenterQuery;
|
||||
import tech.easyflow.common.entity.LoginAccount;
|
||||
import tech.easyflow.common.satoken.util.SaTokenUtil;
|
||||
import tech.easyflow.common.web.controller.BaseCurdController;
|
||||
import tech.easyflow.datacenter.entity.DatacenterTable;
|
||||
import tech.easyflow.datacenter.entity.DatacenterTableField;
|
||||
import tech.easyflow.datacenter.entity.vo.HeaderVo;
|
||||
import tech.easyflow.datacenter.excel.ReadDataListener;
|
||||
import tech.easyflow.datacenter.excel.ReadResVo;
|
||||
import tech.easyflow.datacenter.service.DatacenterTableFieldService;
|
||||
import tech.easyflow.datacenter.service.DatacenterTableService;
|
||||
|
||||
import javax.annotation.Resource;
|
||||
import java.io.InputStream;
|
||||
import java.math.BigInteger;
|
||||
import java.net.URLEncoder;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Date;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* 数据中枢表 控制层。
|
||||
*
|
||||
* @author ArkLight
|
||||
* @since 2025-07-10
|
||||
*/
|
||||
@RestController
|
||||
@RequestMapping("/api/v1/datacenterTable")
|
||||
public class DatacenterTableController extends BaseCurdController<DatacenterTableService, DatacenterTable> {
|
||||
|
||||
@Resource
|
||||
private DatacenterTableFieldService fieldsService;
|
||||
|
||||
public DatacenterTableController(DatacenterTableService service) {
|
||||
super(service);
|
||||
}
|
||||
|
||||
@Override
|
||||
protected Result onSaveOrUpdateBefore(DatacenterTable entity, boolean isSave) {
|
||||
LoginAccount loginUser = SaTokenUtil.getLoginAccount();
|
||||
if (isSave) {
|
||||
commonFiled(entity, loginUser.getId(), loginUser.getTenantId(), loginUser.getDeptId());
|
||||
} else {
|
||||
entity.setModifiedBy(loginUser.getId());
|
||||
}
|
||||
return super.onSaveOrUpdateBefore(entity, isSave);
|
||||
}
|
||||
|
||||
@PostMapping("/saveTable")
|
||||
@SaCheckPermission("/api/v1/datacenterTable/save")
|
||||
public Result<Void> saveTable(@RequestBody DatacenterTable entity) {
|
||||
LoginAccount loginUser = SaTokenUtil.getLoginAccount();
|
||||
List<DatacenterTableField> fields = entity.getFields();
|
||||
if (CollectionUtil.isEmpty(fields)) {
|
||||
return Result.fail(99, "字段不能为空");
|
||||
}
|
||||
BigInteger id = entity.getId();
|
||||
if (id == null) {
|
||||
commonFiled(entity, loginUser.getId(), loginUser.getTenantId(), loginUser.getDeptId());
|
||||
} else {
|
||||
entity.setModified(new Date());
|
||||
entity.setModifiedBy(loginUser.getId());
|
||||
}
|
||||
service.saveTable(entity, loginUser);
|
||||
return Result.ok();
|
||||
}
|
||||
|
||||
@GetMapping("/detailInfo")
|
||||
@SaCheckPermission("/api/v1/datacenterTable/query")
|
||||
public Result<DatacenterTable> detailInfo(BigInteger tableId) {
|
||||
DatacenterTable table = service.getById(tableId);
|
||||
QueryWrapper wrapper = QueryWrapper.create();
|
||||
wrapper.eq(DatacenterTableField::getTableId, tableId);
|
||||
wrapper.orderBy("id");
|
||||
List<DatacenterTableField> fields = fieldsService.list(wrapper);
|
||||
table.setFields(fields);
|
||||
return Result.ok(table);
|
||||
}
|
||||
|
||||
@GetMapping("/removeTable")
|
||||
@SaCheckPermission("/api/v1/datacenterTable/remove")
|
||||
public Result<Void> removeTable(BigInteger tableId) {
|
||||
service.removeTable(tableId);
|
||||
return Result.ok();
|
||||
}
|
||||
|
||||
@GetMapping("/getHeaders")
|
||||
@SaCheckPermission("/api/v1/datacenterTable/query")
|
||||
public Result<List<HeaderVo>> getHeaders(BigInteger tableId) {
|
||||
List<HeaderVo> res = service.getHeaders(tableId);
|
||||
return Result.ok(res);
|
||||
}
|
||||
|
||||
@GetMapping("/getPageData")
|
||||
@SaCheckPermission("/api/v1/datacenterTable/query")
|
||||
public Result<Page<Row>> getPageData(DatacenterQuery where) {
|
||||
Page<Row> res = service.getPageData(where);
|
||||
return Result.ok(res);
|
||||
}
|
||||
|
||||
@PostMapping("/saveValue")
|
||||
@SaCheckPermission("/api/v1/datacenterTable/save")
|
||||
public Result<Void> saveValue(@RequestParam Map<String, Object> map) {
|
||||
JSONObject object = new JSONObject(map);
|
||||
BigInteger tableId = object.getBigInteger("tableId");
|
||||
LoginAccount account = SaTokenUtil.getLoginAccount();
|
||||
if (tableId == null) {
|
||||
return Result.fail(99, "参数错误");
|
||||
}
|
||||
service.saveValue(tableId, object, account);
|
||||
return Result.ok();
|
||||
}
|
||||
|
||||
@PostMapping("/removeValue")
|
||||
@SaCheckPermission("/api/v1/datacenterTable/remove")
|
||||
public Result<Void> removeValue(@RequestParam Map<String, Object> map) {
|
||||
JSONObject object = new JSONObject(map);
|
||||
BigInteger tableId = object.getBigInteger("tableId");
|
||||
BigInteger id = object.getBigInteger("id");
|
||||
if (tableId == null || id == null) {
|
||||
return Result.fail(99, "参数错误");
|
||||
}
|
||||
LoginAccount account = SaTokenUtil.getLoginAccount();
|
||||
service.removeValue(tableId, id, account);
|
||||
return Result.ok();
|
||||
}
|
||||
|
||||
/**
|
||||
* 导入数据
|
||||
*/
|
||||
@PostMapping("/importData")
|
||||
@SaCheckPermission("/api/v1/datacenterTable/save")
|
||||
public Result<ReadResVo> importData(MultipartFile file, @RequestParam Map<String, Object> map) throws Exception {
|
||||
Object tableId = map.get("tableId");
|
||||
DatacenterTable record = service.getById(tableId.toString());
|
||||
if (record == null) {
|
||||
throw new RuntimeException("数据表不存在");
|
||||
}
|
||||
InputStream is = file.getInputStream();
|
||||
List<DatacenterTableField> fields = service.getFields(record.getId());
|
||||
|
||||
LoginAccount account = SaTokenUtil.getLoginAccount();
|
||||
ReadDataListener listener = new ReadDataListener(record.getId(), fields, account);
|
||||
FastExcel.read(is, listener)
|
||||
.sheet()
|
||||
.doRead();
|
||||
int totalCount = listener.getTotalCount();
|
||||
int errorCount = listener.getErrorCount();
|
||||
int successCount = listener.getSuccessCount();
|
||||
List<JSONObject> errorRows = listener.getErrorRows();
|
||||
return Result.ok(new ReadResVo(successCount, errorCount, totalCount, errorRows));
|
||||
}
|
||||
|
||||
@GetMapping("/getTemplate")
|
||||
public void getTemplate(BigInteger tableId, HttpServletResponse response) throws Exception {
|
||||
List<DatacenterTableField> fields = service.getFields(tableId);
|
||||
// 设置响应内容类型
|
||||
response.setContentType("application/vnd.openxmlformats-officedocument.spreadsheetml.sheet");
|
||||
response.setCharacterEncoding("utf-8");
|
||||
|
||||
// 设置文件名
|
||||
String fileName = URLEncoder.encode("导入模板", "UTF-8").replaceAll("\\+", "%20");
|
||||
response.setHeader("Content-disposition", "attachment;filename*=utf-8''" + fileName + ".xlsx");
|
||||
|
||||
// 动态表头数据
|
||||
List<List<String>> headList = new ArrayList<>();
|
||||
|
||||
for (DatacenterTableField field : fields) {
|
||||
List<String> head = new ArrayList<>();
|
||||
head.add(field.getFieldName());
|
||||
headList.add(head);
|
||||
}
|
||||
|
||||
// 写入Excel
|
||||
EasyExcel.write(response.getOutputStream())
|
||||
.head(headList)
|
||||
.sheet("模板")
|
||||
.doWrite(new ArrayList<>()); // 写入空数据,只生成模板
|
||||
}
|
||||
}
|
||||
@@ -1,41 +0,0 @@
|
||||
package tech.easyflow.admin.controller.datacenter;
|
||||
|
||||
import org.springframework.web.bind.annotation.RequestMapping;
|
||||
import org.springframework.web.bind.annotation.RestController;
|
||||
import tech.easyflow.common.annotation.UsePermission;
|
||||
import tech.easyflow.common.domain.Result;
|
||||
import tech.easyflow.common.entity.LoginAccount;
|
||||
import tech.easyflow.common.satoken.util.SaTokenUtil;
|
||||
import tech.easyflow.common.web.controller.BaseCurdController;
|
||||
import tech.easyflow.datacenter.entity.DatacenterTableField;
|
||||
import tech.easyflow.datacenter.service.DatacenterTableFieldService;
|
||||
|
||||
import java.util.Date;
|
||||
|
||||
/**
|
||||
* 控制层。
|
||||
*
|
||||
* @author ArkLight
|
||||
* @since 2025-07-10
|
||||
*/
|
||||
@RestController
|
||||
@RequestMapping("/api/v1/datacenterTableFields")
|
||||
@UsePermission(moduleName = "/api/v1/datacenterTable")
|
||||
public class DatacenterTableFieldsController extends BaseCurdController<DatacenterTableFieldService, DatacenterTableField> {
|
||||
|
||||
public DatacenterTableFieldsController(DatacenterTableFieldService service) {
|
||||
super(service);
|
||||
}
|
||||
|
||||
@Override
|
||||
protected Result onSaveOrUpdateBefore(DatacenterTableField entity, boolean isSave) {
|
||||
LoginAccount loginUser = SaTokenUtil.getLoginAccount();
|
||||
if (isSave) {
|
||||
commonFiled(entity, loginUser.getId(), loginUser.getTenantId(), loginUser.getDeptId());
|
||||
} else {
|
||||
entity.setModified(new Date());
|
||||
entity.setModifiedBy(loginUser.getId());
|
||||
}
|
||||
return null;
|
||||
}
|
||||
}
|
||||
@@ -13,6 +13,7 @@ import tech.easyflow.ai.easyagentsflow.entity.NodeInfo;
|
||||
import tech.easyflow.ai.easyagentsflow.entity.WorkflowCheckStage;
|
||||
import tech.easyflow.ai.easyagentsflow.service.TinyFlowService;
|
||||
import tech.easyflow.ai.easyagentsflow.service.WorkflowCheckService;
|
||||
import tech.easyflow.ai.easyagentsflow.service.WorkflowDatacenterContentService;
|
||||
import tech.easyflow.ai.entity.Workflow;
|
||||
import tech.easyflow.ai.service.WorkflowService;
|
||||
import tech.easyflow.common.domain.Result;
|
||||
@@ -40,6 +41,8 @@ public class PublicWorkflowController {
|
||||
private TinyFlowService tinyFlowService;
|
||||
@Resource
|
||||
private WorkflowCheckService workflowCheckService;
|
||||
@Resource
|
||||
private WorkflowDatacenterContentService workflowDatacenterContentService;
|
||||
|
||||
/**
|
||||
* 通过id或别名获取工作流详情
|
||||
@@ -123,7 +126,7 @@ public class PublicWorkflowController {
|
||||
}
|
||||
workflowCheckService.checkOrThrow(workflow.getContent(), WorkflowCheckStage.PRE_EXECUTE, workflow.getId());
|
||||
|
||||
ChainDefinition definition = chainParser.parse(workflow.getContent());
|
||||
ChainDefinition definition = chainParser.parse(workflowDatacenterContentService.prepareContent(workflow.getContent()));
|
||||
if (definition == null) {
|
||||
return Result.fail(2, "节点配置错误,请检查! ");
|
||||
}
|
||||
|
||||
@@ -13,6 +13,7 @@ import tech.easyflow.ai.permission.WorkflowVisibilityQueryHelper;
|
||||
import tech.easyflow.ai.easyagentsflow.entity.ChainInfo;
|
||||
import tech.easyflow.ai.easyagentsflow.entity.NodeInfo;
|
||||
import tech.easyflow.ai.easyagentsflow.entity.WorkflowCheckStage;
|
||||
import tech.easyflow.ai.easyagentsflow.service.WorkflowDatacenterContentService;
|
||||
import tech.easyflow.ai.easyagentsflow.service.TinyFlowService;
|
||||
import tech.easyflow.ai.easyagentsflow.service.WorkflowCheckService;
|
||||
import tech.easyflow.ai.entity.Workflow;
|
||||
@@ -53,6 +54,8 @@ public class UcWorkflowController extends BaseCurdController<WorkflowService, Wo
|
||||
@Resource
|
||||
private WorkflowCheckService workflowCheckService;
|
||||
@Resource
|
||||
private WorkflowDatacenterContentService workflowDatacenterContentService;
|
||||
@Resource
|
||||
private WorkflowVisibilityQueryHelper workflowVisibilityQueryHelper;
|
||||
|
||||
public UcWorkflowController(WorkflowService service) {
|
||||
@@ -168,7 +171,7 @@ public class UcWorkflowController extends BaseCurdController<WorkflowService, Wo
|
||||
}
|
||||
workflowCheckService.checkOrThrow(workflow.getContent(), WorkflowCheckStage.PRE_EXECUTE, workflow.getId());
|
||||
|
||||
ChainDefinition definition = chainParser.parse(workflow.getContent());
|
||||
ChainDefinition definition = chainParser.parse(workflowDatacenterContentService.prepareContent(workflow.getContent()));
|
||||
if (definition == null) {
|
||||
return Result.fail(2, "节点配置错误,请检查! ");
|
||||
}
|
||||
|
||||
@@ -4,6 +4,7 @@ import com.easyagents.flow.core.chain.ChainDefinition;
|
||||
import com.easyagents.flow.core.chain.repository.ChainDefinitionRepository;
|
||||
import com.easyagents.flow.core.parser.ChainParser;
|
||||
import org.springframework.stereotype.Component;
|
||||
import tech.easyflow.ai.easyagentsflow.service.WorkflowDatacenterContentService;
|
||||
import tech.easyflow.ai.entity.Workflow;
|
||||
import tech.easyflow.ai.service.WorkflowService;
|
||||
|
||||
@@ -16,11 +17,13 @@ public class ChainDefinitionRepositoryImpl implements ChainDefinitionRepository
|
||||
private WorkflowService workflowService;
|
||||
@Resource
|
||||
private ChainParser chainParser;
|
||||
@Resource
|
||||
private WorkflowDatacenterContentService workflowDatacenterContentService;
|
||||
|
||||
@Override
|
||||
public ChainDefinition getChainDefinitionById(String id) {
|
||||
Workflow workflow = workflowService.getById(id);
|
||||
String json = workflow.getContent();
|
||||
String json = workflowDatacenterContentService.prepareContent(workflow.getContent());
|
||||
ChainDefinition chainDefinition = chainParser.parse(json);
|
||||
chainDefinition.setId(workflow.getId().toString());
|
||||
chainDefinition.setName(workflow.getEnglishName());
|
||||
|
||||
@@ -65,14 +65,12 @@ public class TinyFlowConfigService {
|
||||
MakeFileNodeParser makeFileNodeParser = new MakeFileNodeParser();
|
||||
// 插件
|
||||
PluginToolNodeParser pluginToolNodeParser = new PluginToolNodeParser();
|
||||
// SQL查询
|
||||
SqlNodeParser sqlNodeParser = new SqlNodeParser();
|
||||
// 下载文件节点
|
||||
DownloadNodeParser downloadNodeParser = new DownloadNodeParser();
|
||||
// 保存数据节点
|
||||
SaveToDatacenterNodeParser saveDaveParser = new SaveToDatacenterNodeParser();
|
||||
SaveDatasetNodeParser saveDatasetNodeParser = new SaveDatasetNodeParser();
|
||||
// 查询数据节点
|
||||
SearchDatacenterNodeParser searchDatacenterNodeParser = new SearchDatacenterNodeParser();
|
||||
SearchDatasetNodeParser searchDatasetNodeParser = new SearchDatasetNodeParser();
|
||||
// 工作流节点
|
||||
WorkflowNodeParser workflowNodeParser = new WorkflowNodeParser();
|
||||
// 条件判断节点
|
||||
@@ -81,10 +79,9 @@ public class TinyFlowConfigService {
|
||||
chainParser.addNodeParser(docNodeParser.getNodeName(), docNodeParser);
|
||||
chainParser.addNodeParser(makeFileNodeParser.getNodeName(), makeFileNodeParser);
|
||||
chainParser.addNodeParser(pluginToolNodeParser.getNodeName(), pluginToolNodeParser);
|
||||
chainParser.addNodeParser(sqlNodeParser.getNodeName(), sqlNodeParser);
|
||||
chainParser.addNodeParser(downloadNodeParser.getNodeName(), downloadNodeParser);
|
||||
chainParser.addNodeParser(saveDaveParser.getNodeName(), saveDaveParser);
|
||||
chainParser.addNodeParser(searchDatacenterNodeParser.getNodeName(), searchDatacenterNodeParser);
|
||||
chainParser.addNodeParser(saveDatasetNodeParser.getNodeName(), saveDatasetNodeParser);
|
||||
chainParser.addNodeParser(searchDatasetNodeParser.getNodeName(), searchDatasetNodeParser);
|
||||
chainParser.addNodeParser(workflowNodeParser.getNodeName(), workflowNodeParser);
|
||||
chainParser.addNodeParser(conditionNodeParser.getNodeName(), conditionNodeParser);
|
||||
}
|
||||
|
||||
@@ -12,15 +12,30 @@ import tech.easyflow.ai.easyagentsflow.entity.WorkflowCheckStage;
|
||||
import tech.easyflow.ai.entity.Workflow;
|
||||
import tech.easyflow.ai.service.WorkflowService;
|
||||
import tech.easyflow.common.web.exceptions.BusinessException;
|
||||
import tech.easyflow.datacenter.execution.model.DatacenterSchemaResponse;
|
||||
|
||||
import javax.annotation.Resource;
|
||||
import java.math.BigInteger;
|
||||
import java.util.*;
|
||||
import java.util.ArrayDeque;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.Collection;
|
||||
import java.util.Collections;
|
||||
import java.util.Deque;
|
||||
import java.util.HashMap;
|
||||
import java.util.HashSet;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.LinkedHashSet;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Objects;
|
||||
import java.util.Set;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
@Service
|
||||
public class WorkflowCheckService {
|
||||
private static final String LEVEL_ERROR = "ERROR";
|
||||
private static final String LEVEL_WARNING = "WARNING";
|
||||
private static final String TYPE_START = "startNode";
|
||||
private static final String TYPE_END = "endNode";
|
||||
private static final String TYPE_LOOP = "loopNode";
|
||||
@@ -30,6 +45,8 @@ public class WorkflowCheckService {
|
||||
private WorkflowService workflowService;
|
||||
@Resource
|
||||
private ChainParser chainParser;
|
||||
@Resource
|
||||
private WorkflowDatacenterContentService workflowDatacenterContentService;
|
||||
|
||||
public WorkflowCheckResult checkWorkflow(BigInteger workflowId, WorkflowCheckStage stage) {
|
||||
if (workflowId == null) {
|
||||
@@ -177,9 +194,91 @@ public class WorkflowCheckService {
|
||||
parsedWorkflow.nodes = nodes;
|
||||
parsedWorkflow.edges = edges;
|
||||
parsedWorkflow.nodeMap = nodeMap;
|
||||
checkDatacenterNodes(parsedWorkflow, issues, issueKeys);
|
||||
return parsedWorkflow;
|
||||
}
|
||||
|
||||
private void checkDatacenterNodes(ParsedWorkflow parsed, List<WorkflowCheckIssue> issues, Set<String> issueKeys) {
|
||||
for (NodeView node : parsed.nodes) {
|
||||
if (node == null) {
|
||||
continue;
|
||||
}
|
||||
if (workflowDatacenterContentService.isSearchDatasetNode(node.type)) {
|
||||
checkSearchDatasetNode(node, issues, issueKeys);
|
||||
continue;
|
||||
}
|
||||
if (workflowDatacenterContentService.isSaveDatasetNode(node.type)) {
|
||||
checkSaveDatasetNode(node, issues, issueKeys);
|
||||
continue;
|
||||
}
|
||||
if (workflowDatacenterContentService.isLlmNode(node.type)) {
|
||||
checkLlmQueryContext(node, parsed, issues, issueKeys);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void checkSearchDatasetNode(NodeView node,
|
||||
List<WorkflowCheckIssue> issues,
|
||||
Set<String> issueKeys) {
|
||||
try {
|
||||
workflowDatacenterContentService.requireSearchDatasetRef(node.data);
|
||||
} catch (BusinessException e) {
|
||||
addIssue(issues, issueKeys, "SEARCH_DATASET_INVALID", e.getMessage(), node.id, null, node.name);
|
||||
} catch (Exception e) {
|
||||
addIssue(issues, issueKeys, "SEARCH_DATASET_INVALID",
|
||||
"查询数据节点校验失败: " + shortError(e), node.id, null, node.name);
|
||||
}
|
||||
}
|
||||
|
||||
private void checkSaveDatasetNode(NodeView node,
|
||||
List<WorkflowCheckIssue> issues,
|
||||
Set<String> issueKeys) {
|
||||
try {
|
||||
workflowDatacenterContentService.requireSaveDatasetRef(node.data);
|
||||
} catch (BusinessException e) {
|
||||
addIssue(issues, issueKeys, "SAVE_DATASET_INVALID", e.getMessage(), node.id, null, node.name);
|
||||
} catch (Exception e) {
|
||||
addIssue(issues, issueKeys, "SAVE_DATASET_INVALID",
|
||||
"写入数据节点校验失败: " + shortError(e), node.id, null, node.name);
|
||||
}
|
||||
}
|
||||
|
||||
private void checkLlmQueryContext(NodeView node,
|
||||
ParsedWorkflow parsed,
|
||||
List<WorkflowCheckIssue> issues,
|
||||
Set<String> issueKeys) {
|
||||
if (node.data == null) {
|
||||
return;
|
||||
}
|
||||
Object rawNodeIds = node.data.get("queryContextNodeIds");
|
||||
if (rawNodeIds == null) {
|
||||
return;
|
||||
}
|
||||
if (!(rawNodeIds instanceof JSONArray nodeIds)) {
|
||||
addIssue(issues, issueKeys, "LLM_QUERY_CONTEXT_INVALID",
|
||||
"查询上下文配置无效,请重新选择查询数据节点", node.id, null, node.name);
|
||||
return;
|
||||
}
|
||||
for (int i = 0; i < nodeIds.size(); i++) {
|
||||
String refNodeId = trimToNull(nodeIds.getString(i));
|
||||
if (!StringUtils.hasText(refNodeId)) {
|
||||
addIssue(issues, issueKeys, "LLM_QUERY_CONTEXT_EMPTY",
|
||||
"查询上下文配置无效,请重新选择查询数据节点", node.id, null, node.name);
|
||||
continue;
|
||||
}
|
||||
NodeView refNode = parsed.nodeMap.get(refNodeId);
|
||||
if (refNode == null) {
|
||||
addIssue(issues, issueKeys, "LLM_QUERY_CONTEXT_NOT_FOUND",
|
||||
"查询上下文节点不存在: " + refNodeId, node.id, null, node.name);
|
||||
continue;
|
||||
}
|
||||
if (!workflowDatacenterContentService.isSearchDatasetNode(refNode.type)) {
|
||||
addIssue(issues, issueKeys, "LLM_QUERY_CONTEXT_TYPE_INVALID",
|
||||
"查询上下文只能选择查询数据节点", node.id, null, node.name);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void runStrictChecks(String content, ParsedWorkflow parsed, BigInteger currentWorkflowId,
|
||||
List<WorkflowCheckIssue> issues, Set<String> issueKeys) {
|
||||
if (parsed.nodes.isEmpty()) {
|
||||
@@ -190,7 +289,8 @@ public class WorkflowCheckService {
|
||||
}
|
||||
|
||||
try {
|
||||
Object definition = chainParser.parse(content);
|
||||
String preparedContent = workflowDatacenterContentService.prepareContent(content);
|
||||
Object definition = chainParser.parse(preparedContent);
|
||||
if (definition == null) {
|
||||
addIssue(issues, issueKeys, "PARSE_NULL", "预执行校验失败:节点配置错误,请检查", null, null, null);
|
||||
}
|
||||
@@ -606,31 +706,47 @@ public class WorkflowCheckService {
|
||||
result.setStage(stage);
|
||||
result.setIssues(issues);
|
||||
result.setIssueCount(issues.size());
|
||||
result.setPassed(issues.isEmpty());
|
||||
result.setPassed(issues.stream().noneMatch(issue -> LEVEL_ERROR.equalsIgnoreCase(issue.getLevel())));
|
||||
return result;
|
||||
}
|
||||
|
||||
private void throwIfFailed(WorkflowCheckResult result) {
|
||||
if (result == null || result.isPassed()) {
|
||||
if (result == null) {
|
||||
return;
|
||||
}
|
||||
String summary = result.getIssues().stream()
|
||||
List<WorkflowCheckIssue> errorIssues = result.getIssues().stream()
|
||||
.filter(issue -> LEVEL_ERROR.equalsIgnoreCase(issue.getLevel()))
|
||||
.collect(Collectors.toList());
|
||||
if (errorIssues.isEmpty()) {
|
||||
return;
|
||||
}
|
||||
String summary = errorIssues.stream()
|
||||
.limit(5)
|
||||
.map(WorkflowCheckIssue::getMessage)
|
||||
.collect(Collectors.joining(";"));
|
||||
if (result.getIssueCount() > 5) {
|
||||
if (errorIssues.size() > 5) {
|
||||
summary = summary + ";等";
|
||||
}
|
||||
throw new BusinessException("工作流校验未通过(" + result.getStage() + "),共 " + result.getIssueCount() + " 项:" + summary);
|
||||
throw new BusinessException("工作流校验未通过(" + result.getStage() + "),共 " + errorIssues.size() + " 项:" + summary);
|
||||
}
|
||||
|
||||
private void addIssue(List<WorkflowCheckIssue> issues, Set<String> issueKeys, String code,
|
||||
String message, String nodeId, String edgeId, String nodeName) {
|
||||
addIssue(issues, issueKeys, code, LEVEL_ERROR, message, nodeId, edgeId, nodeName);
|
||||
}
|
||||
|
||||
private void addWarning(List<WorkflowCheckIssue> issues, Set<String> issueKeys, String code,
|
||||
String message, String nodeId, String edgeId, String nodeName) {
|
||||
addIssue(issues, issueKeys, code, LEVEL_WARNING, message, nodeId, edgeId, nodeName);
|
||||
}
|
||||
|
||||
private void addIssue(List<WorkflowCheckIssue> issues, Set<String> issueKeys, String code, String level,
|
||||
String message, String nodeId, String edgeId, String nodeName) {
|
||||
String key = code + "|" + safe(nodeId) + "|" + safe(edgeId) + "|" + message;
|
||||
if (!issueKeys.add(key)) {
|
||||
return;
|
||||
}
|
||||
issues.add(new WorkflowCheckIssue(code, LEVEL_ERROR, message, nodeId, edgeId, nodeName));
|
||||
issues.add(new WorkflowCheckIssue(code, level, message, nodeId, edgeId, nodeName));
|
||||
}
|
||||
|
||||
private String extractNodeName(JSONObject nodeJson, JSONObject data, String fallback) {
|
||||
|
||||
@@ -0,0 +1,349 @@
|
||||
package tech.easyflow.ai.easyagentsflow.service;
|
||||
|
||||
import com.alibaba.fastjson2.JSON;
|
||||
import com.alibaba.fastjson2.JSONArray;
|
||||
import com.alibaba.fastjson2.JSONObject;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.springframework.util.StringUtils;
|
||||
import tech.easyflow.common.constant.enums.EnumFieldType;
|
||||
import tech.easyflow.common.web.exceptions.BusinessException;
|
||||
import tech.easyflow.datacenter.entity.DatacenterTable;
|
||||
import tech.easyflow.datacenter.entity.DatacenterTableField;
|
||||
import tech.easyflow.datacenter.execution.model.DatacenterSchemaResponse;
|
||||
import tech.easyflow.datacenter.execution.model.DatasetRef;
|
||||
import tech.easyflow.datacenter.execution.service.DatacenterDatasetQueryService;
|
||||
import tech.easyflow.datacenter.meta.entity.DatacenterCatalog;
|
||||
import tech.easyflow.datacenter.meta.entity.DatacenterSource;
|
||||
import tech.easyflow.datacenter.meta.service.DatacenterDatasetRegistryService;
|
||||
|
||||
import javax.annotation.Resource;
|
||||
import java.math.BigInteger;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Comparator;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.LinkedHashSet;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
|
||||
@Service
|
||||
public class WorkflowDatacenterContentService {
|
||||
|
||||
public static final String SEARCH_NODE_TYPE = "search-dataset-node";
|
||||
public static final String SAVE_NODE_TYPE = "save-dataset-node";
|
||||
public static final String LLM_NODE_TYPE = "llmNode";
|
||||
public static final String QUERY_DATA_CONTEXT = "queryDataContext";
|
||||
public static final String SEARCH_SOURCE_MISSING_MESSAGE = "查询数据节点未选择连接服务";
|
||||
public static final String SEARCH_SQL_MISSING_MESSAGE = "查询数据节点未设置 SQL";
|
||||
public static final String SAVE_EXPIRED_MESSAGE = "写入数据节点配置已过期,请重新选择已接入表";
|
||||
public static final String INVALID_QUERY_CONTEXT_MESSAGE = "查询上下文配置无效,请重新选择查询数据节点";
|
||||
private static final String QUERY_CONTEXT_PROMPT = """
|
||||
你是为工作流中的查询数据节点生成只读 SQL 的生成器,你的职责是返回可直接执行的 SQL,并且你只能输出 SQL。
|
||||
|
||||
必须严格遵守以下规则:
|
||||
1. 只能从下面给出的连接摘要中选择最合适的表。
|
||||
2. 只能使用摘要中给出的字段名,不要虚构表名或字段名。
|
||||
3. 只输出 SQL,不要输出解释、注释、Markdown、JSON 或多余文本。
|
||||
4. 只能生成只读 SELECT SQL,允许 WITH、JOIN、子查询、聚合、分组、排序。
|
||||
5. 不要生成 INSERT、UPDATE、DELETE、DDL、多语句、存储过程调用。
|
||||
6. 优先使用逻辑表名和逻辑字段名,不要输出物理表名、JDBC、驱动信息。
|
||||
7. 如果存在重名表,请使用 catalog.table 形式消除歧义。
|
||||
|
||||
以下是可用的连接摘要:
|
||||
""";
|
||||
|
||||
@Resource
|
||||
private DatacenterDatasetQueryService queryService;
|
||||
@Resource
|
||||
private DatacenterDatasetRegistryService registryService;
|
||||
|
||||
public boolean isSearchDatasetNode(String nodeType) {
|
||||
return SEARCH_NODE_TYPE.equals(nodeType);
|
||||
}
|
||||
|
||||
public boolean isSaveDatasetNode(String nodeType) {
|
||||
return SAVE_NODE_TYPE.equals(nodeType);
|
||||
}
|
||||
|
||||
public boolean isLlmNode(String nodeType) {
|
||||
return LLM_NODE_TYPE.equals(nodeType);
|
||||
}
|
||||
|
||||
public String prepareContent(String content) {
|
||||
if (!StringUtils.hasText(content)) {
|
||||
throw new BusinessException("工作流内容不能为空");
|
||||
}
|
||||
Object parsed;
|
||||
try {
|
||||
parsed = JSON.parse(content);
|
||||
} catch (Exception e) {
|
||||
throw new BusinessException("工作流内容不是合法JSON: " + shortError(e));
|
||||
}
|
||||
if (!(parsed instanceof JSONObject root)) {
|
||||
throw new BusinessException("工作流内容必须是JSON对象");
|
||||
}
|
||||
JSONObject copy = JSON.parseObject(root.toJSONString());
|
||||
prepareRoot(copy);
|
||||
return copy.toJSONString();
|
||||
}
|
||||
|
||||
public JSONObject prepareRoot(JSONObject root) {
|
||||
if (root == null) {
|
||||
throw new BusinessException("工作流内容不能为空");
|
||||
}
|
||||
JSONArray nodes = root.getJSONArray("nodes");
|
||||
if (nodes == null || nodes.isEmpty()) {
|
||||
return root;
|
||||
}
|
||||
Map<String, JSONObject> nodeMap = buildNodeMap(nodes);
|
||||
for (int i = 0; i < nodes.size(); i++) {
|
||||
JSONObject node = nodes.getJSONObject(i);
|
||||
if (node == null) {
|
||||
continue;
|
||||
}
|
||||
String nodeType = node.getString("type");
|
||||
JSONObject data = node.getJSONObject("data");
|
||||
if (isSearchDatasetNode(nodeType)) {
|
||||
requireSearchDatasetRef(data);
|
||||
} else if (isSaveDatasetNode(nodeType)) {
|
||||
requireSaveDatasetRef(data);
|
||||
}
|
||||
}
|
||||
for (int i = 0; i < nodes.size(); i++) {
|
||||
JSONObject node = nodes.getJSONObject(i);
|
||||
if (node == null || !isLlmNode(node.getString("type"))) {
|
||||
continue;
|
||||
}
|
||||
injectQueryDataContext(node.getJSONObject("data"), nodeMap);
|
||||
}
|
||||
return root;
|
||||
}
|
||||
|
||||
public boolean hasSaveLegacyFields(JSONObject data) {
|
||||
return hasAnyField(data, "tableId", "sourceId", "catalogId", "versionId");
|
||||
}
|
||||
|
||||
public DatasetRef readDatasetRef(JSONObject data) {
|
||||
return data == null ? null : data.getObject("datasetRef", DatasetRef.class);
|
||||
}
|
||||
|
||||
public DatasetRef requireSearchDatasetRef(JSONObject data) {
|
||||
DatasetRef datasetRef = readDatasetRef(data);
|
||||
if (datasetRef == null || datasetRef.getSourceId() == null) {
|
||||
throw new BusinessException(SEARCH_SOURCE_MISSING_MESSAGE);
|
||||
}
|
||||
String querySql = data == null ? null : trimToNull(data.getString("querySql"));
|
||||
if (!StringUtils.hasText(querySql)) {
|
||||
throw new BusinessException(SEARCH_SQL_MISSING_MESSAGE);
|
||||
}
|
||||
return datasetRef;
|
||||
}
|
||||
|
||||
public DatasetRef requireSaveDatasetRef(JSONObject data) {
|
||||
if (hasSaveLegacyFields(data)) {
|
||||
throw new BusinessException(SAVE_EXPIRED_MESSAGE);
|
||||
}
|
||||
DatasetRef datasetRef = readDatasetRef(data);
|
||||
if (datasetRef == null || datasetRef.getTableId() == null) {
|
||||
throw new BusinessException(SAVE_EXPIRED_MESSAGE);
|
||||
}
|
||||
return datasetRef;
|
||||
}
|
||||
|
||||
public DatacenterSchemaResponse loadSchema(DatasetRef datasetRef, Map<BigInteger, DatacenterSchemaResponse> schemaCache) {
|
||||
if (datasetRef == null || datasetRef.getTableId() == null) {
|
||||
throw new BusinessException("缺少已接入表配置");
|
||||
}
|
||||
BigInteger tableId = datasetRef.getTableId();
|
||||
if (schemaCache != null && schemaCache.containsKey(tableId)) {
|
||||
return schemaCache.get(tableId);
|
||||
}
|
||||
DatacenterSchemaResponse schema = queryService.getSchema(copyDatasetRef(datasetRef));
|
||||
if (schemaCache != null) {
|
||||
schemaCache.put(tableId, schema);
|
||||
}
|
||||
return schema;
|
||||
}
|
||||
|
||||
public JSONObject buildSourceSummary(DatasetRef datasetRef) {
|
||||
if (datasetRef == null || datasetRef.getSourceId() == null) {
|
||||
throw new BusinessException(SEARCH_SOURCE_MISSING_MESSAGE);
|
||||
}
|
||||
DatacenterSource source = registryService.getSourceRequired(datasetRef.getSourceId());
|
||||
List<DatacenterTable> managedTables = registryService.listManagedTables(datasetRef.getSourceId(), datasetRef.getCatalogId());
|
||||
managedTables.sort(Comparator.comparing(table -> table.getTableName() == null ? "" : table.getTableName()));
|
||||
JSONObject sourceSummary = new JSONObject();
|
||||
sourceSummary.put("sourceName", source.getSourceName());
|
||||
sourceSummary.put("sourceType", source.getSourceType());
|
||||
JSONArray tables = new JSONArray();
|
||||
for (DatacenterTable table : managedTables) {
|
||||
DatacenterTable fullTable = registryService.getTableWithFields(table.getId());
|
||||
DatacenterCatalog catalog = registryService.getCatalogById(fullTable.getCatalogId());
|
||||
if (StringUtils.hasText(datasetRef.getCatalogName())
|
||||
&& (catalog == null || !datasetRef.getCatalogName().equals(catalog.getCatalogName()))) {
|
||||
continue;
|
||||
}
|
||||
JSONObject tableSummary = new JSONObject();
|
||||
tableSummary.put("catalogName", catalog == null ? null : catalog.getCatalogName());
|
||||
tableSummary.put("tableName", fullTable.getTableName());
|
||||
tableSummary.put("tableDesc", fullTable.getTableDesc());
|
||||
JSONArray fields = new JSONArray();
|
||||
if (fullTable.getFields() != null) {
|
||||
for (DatacenterTableField field : fullTable.getFields()) {
|
||||
JSONObject fieldSummary = new JSONObject();
|
||||
fieldSummary.put("fieldName", field.getFieldName());
|
||||
fieldSummary.put("fieldDesc", field.getFieldDesc());
|
||||
fieldSummary.put("fieldType", resolveFieldType(field));
|
||||
fields.add(fieldSummary);
|
||||
}
|
||||
}
|
||||
tableSummary.put("fields", fields);
|
||||
tables.add(tableSummary);
|
||||
}
|
||||
sourceSummary.put("tables", tables);
|
||||
return sourceSummary;
|
||||
}
|
||||
|
||||
private void injectQueryDataContext(JSONObject data, Map<String, JSONObject> nodeMap) {
|
||||
if (data == null) {
|
||||
return;
|
||||
}
|
||||
JSONArray nodeIds = data.getJSONArray("queryContextNodeIds");
|
||||
if (nodeIds == null || nodeIds.isEmpty()) {
|
||||
removeQueryDataContextParameter(data);
|
||||
return;
|
||||
}
|
||||
Map<BigInteger, JSONObject> sourceSummaries = new LinkedHashMap<>();
|
||||
Set<String> visitedNodeIds = new LinkedHashSet<>();
|
||||
for (int i = 0; i < nodeIds.size(); i++) {
|
||||
String nodeId = trimToNull(nodeIds.getString(i));
|
||||
if (!StringUtils.hasText(nodeId) || !visitedNodeIds.add(nodeId)) {
|
||||
continue;
|
||||
}
|
||||
JSONObject targetNode = nodeMap.get(nodeId);
|
||||
if (targetNode == null || !isSearchDatasetNode(targetNode.getString("type"))) {
|
||||
throw new BusinessException(INVALID_QUERY_CONTEXT_MESSAGE);
|
||||
}
|
||||
DatasetRef datasetRef = requireSearchDatasetRef(targetNode.getJSONObject("data"));
|
||||
sourceSummaries.putIfAbsent(datasetRef.getSourceId(), buildSourceSummary(datasetRef));
|
||||
}
|
||||
String contextValue = QUERY_CONTEXT_PROMPT + "\n" + JSON.toJSONString(new ArrayList<>(sourceSummaries.values()));
|
||||
upsertQueryDataContextParameter(data, contextValue);
|
||||
}
|
||||
|
||||
private String resolveFieldType(DatacenterTableField field) {
|
||||
if (field == null) {
|
||||
return null;
|
||||
}
|
||||
if (StringUtils.hasText(field.getJdbcType())) {
|
||||
return field.getJdbcType();
|
||||
}
|
||||
try {
|
||||
EnumFieldType enumFieldType = EnumFieldType.getByCode(field.getFieldType());
|
||||
if (enumFieldType != null) {
|
||||
return enumFieldType.getText();
|
||||
}
|
||||
} catch (Exception ignored) {
|
||||
}
|
||||
return field.getFieldType() == null ? null : String.valueOf(field.getFieldType());
|
||||
}
|
||||
|
||||
private void upsertQueryDataContextParameter(JSONObject data, String contextValue) {
|
||||
JSONArray parameters = data.getJSONArray("parameters");
|
||||
if (parameters == null) {
|
||||
parameters = new JSONArray();
|
||||
data.put("parameters", parameters);
|
||||
}
|
||||
for (int i = parameters.size() - 1; i >= 0; i--) {
|
||||
JSONObject parameter = parameters.getJSONObject(i);
|
||||
if (parameter != null && QUERY_DATA_CONTEXT.equals(parameter.getString("name"))) {
|
||||
parameters.remove(i);
|
||||
}
|
||||
}
|
||||
JSONObject parameter = new JSONObject();
|
||||
parameter.put("id", QUERY_DATA_CONTEXT);
|
||||
parameter.put("name", QUERY_DATA_CONTEXT);
|
||||
parameter.put("title", "查询上下文");
|
||||
parameter.put("description", "数据查询规则与连接表摘要");
|
||||
parameter.put("dataType", "String");
|
||||
parameter.put("refType", "fixed");
|
||||
parameter.put("value", contextValue);
|
||||
parameter.put("required", false);
|
||||
parameter.put("nameDisabled", true);
|
||||
parameter.put("dataTypeDisabled", true);
|
||||
parameter.put("deleteDisabled", true);
|
||||
parameters.add(parameter);
|
||||
}
|
||||
|
||||
private void removeQueryDataContextParameter(JSONObject data) {
|
||||
if (data == null) {
|
||||
return;
|
||||
}
|
||||
JSONArray parameters = data.getJSONArray("parameters");
|
||||
if (parameters == null || parameters.isEmpty()) {
|
||||
return;
|
||||
}
|
||||
for (int i = parameters.size() - 1; i >= 0; i--) {
|
||||
JSONObject parameter = parameters.getJSONObject(i);
|
||||
if (parameter != null && QUERY_DATA_CONTEXT.equals(parameter.getString("name"))) {
|
||||
parameters.remove(i);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private Map<String, JSONObject> buildNodeMap(JSONArray nodes) {
|
||||
Map<String, JSONObject> nodeMap = new LinkedHashMap<>();
|
||||
for (int i = 0; i < nodes.size(); i++) {
|
||||
JSONObject node = nodes.getJSONObject(i);
|
||||
if (node == null) {
|
||||
continue;
|
||||
}
|
||||
String nodeId = trimToNull(node.getString("id"));
|
||||
if (StringUtils.hasText(nodeId)) {
|
||||
nodeMap.put(nodeId, node);
|
||||
}
|
||||
}
|
||||
return nodeMap;
|
||||
}
|
||||
|
||||
private DatasetRef copyDatasetRef(DatasetRef datasetRef) {
|
||||
DatasetRef copy = new DatasetRef();
|
||||
copy.setSourceId(datasetRef.getSourceId());
|
||||
copy.setCatalogId(datasetRef.getCatalogId());
|
||||
copy.setCatalogName(datasetRef.getCatalogName());
|
||||
copy.setTableId(datasetRef.getTableId());
|
||||
copy.setTableName(datasetRef.getTableName());
|
||||
copy.setVersionId(datasetRef.getVersionId());
|
||||
return copy;
|
||||
}
|
||||
|
||||
private boolean hasAnyField(JSONObject data, String... fieldNames) {
|
||||
if (data == null || fieldNames == null) {
|
||||
return false;
|
||||
}
|
||||
for (String fieldName : fieldNames) {
|
||||
if (data.containsKey(fieldName)) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
private String trimToNull(String value) {
|
||||
if (!StringUtils.hasText(value)) {
|
||||
return null;
|
||||
}
|
||||
return value.trim();
|
||||
}
|
||||
|
||||
private String shortError(Throwable throwable) {
|
||||
if (throwable == null) {
|
||||
return "unknown";
|
||||
}
|
||||
String message = throwable.getMessage();
|
||||
if (!StringUtils.hasText(message)) {
|
||||
return throwable.getClass().getSimpleName();
|
||||
}
|
||||
return message.length() > 120 ? message.substring(0, 120) + "..." : message;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,75 @@
|
||||
package tech.easyflow.ai.node;
|
||||
|
||||
import com.alibaba.fastjson2.JSONArray;
|
||||
import com.alibaba.fastjson2.JSONObject;
|
||||
import com.easyagents.flow.core.chain.Chain;
|
||||
import com.easyagents.flow.core.node.BaseNode;
|
||||
import com.mybatisflex.core.tenant.TenantManager;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import tech.easyflow.ai.utils.WorkFlowUtil;
|
||||
import tech.easyflow.common.entity.LoginAccount;
|
||||
import tech.easyflow.common.util.SpringContextUtil;
|
||||
import tech.easyflow.datacenter.execution.model.DatasetRef;
|
||||
import tech.easyflow.datacenter.execution.service.DatacenterDatasetQueryService;
|
||||
import tech.easyflow.datacenter.execution.service.DatacenterDatasetWriteService;
|
||||
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
|
||||
public class SaveDatasetNode extends BaseNode {
|
||||
|
||||
private static final Logger log = LoggerFactory.getLogger(SaveDatasetNode.class);
|
||||
|
||||
private DatasetRef datasetRef;
|
||||
|
||||
public SaveDatasetNode() {
|
||||
}
|
||||
|
||||
public SaveDatasetNode(DatasetRef datasetRef) {
|
||||
this.datasetRef = datasetRef;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Map<String, Object> execute(Chain chain) {
|
||||
Map<String, Object> state = chain.getState().resolveParameters(this);
|
||||
JSONObject payload = new JSONObject(state);
|
||||
JSONArray saveList = payload.getJSONArray("saveList");
|
||||
if (saveList == null || saveList.isEmpty()) {
|
||||
throw new RuntimeException("saveList 不能为空");
|
||||
}
|
||||
LoginAccount account = WorkFlowUtil.getOperator(chain);
|
||||
DatacenterDatasetWriteService writeService = SpringContextUtil.getBean(DatacenterDatasetWriteService.class);
|
||||
DatacenterDatasetQueryService queryService = SpringContextUtil.getBean(DatacenterDatasetQueryService.class);
|
||||
int successRows = 0;
|
||||
try {
|
||||
TenantManager.ignoreTenantCondition();
|
||||
for (Object item : saveList) {
|
||||
JSONObject row = item instanceof JSONObject json ? json : JSONObject.from(item);
|
||||
writeService.saveRow(datasetRef, row, account);
|
||||
successRows++;
|
||||
}
|
||||
var schema = queryService.getSchema(datasetRef);
|
||||
Map<String, Object> result = new HashMap<>();
|
||||
result.put("successRows", successRows);
|
||||
result.put("source", schema.getSource());
|
||||
result.put("catalog", schema.getCatalog());
|
||||
result.put("table", schema.getTable());
|
||||
result.put("version", datasetRef.getVersionId());
|
||||
return result;
|
||||
} catch (Exception ex) {
|
||||
log.error("工作流保存数据到统一数据集失败,datasetRef={}", datasetRef, ex);
|
||||
throw ex;
|
||||
} finally {
|
||||
TenantManager.restoreTenantCondition();
|
||||
}
|
||||
}
|
||||
|
||||
public DatasetRef getDatasetRef() {
|
||||
return datasetRef;
|
||||
}
|
||||
|
||||
public void setDatasetRef(DatasetRef datasetRef) {
|
||||
this.datasetRef = datasetRef;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,41 @@
|
||||
package tech.easyflow.ai.node;
|
||||
|
||||
import com.alibaba.fastjson.JSONObject;
|
||||
import com.easyagents.flow.core.node.BaseNode;
|
||||
import com.easyagents.flow.core.parser.BaseNodeParser;
|
||||
import tech.easyflow.common.web.exceptions.BusinessException;
|
||||
import tech.easyflow.datacenter.execution.model.DatasetRef;
|
||||
|
||||
public class SaveDatasetNodeParser extends BaseNodeParser {
|
||||
|
||||
private static final String EXPIRED_MESSAGE = "写入数据节点配置已过期,请重新选择已接入表";
|
||||
|
||||
@Override
|
||||
protected BaseNode doParse(JSONObject root, JSONObject data, JSONObject tinyflow) {
|
||||
if (data == null) {
|
||||
throw new BusinessException(EXPIRED_MESSAGE);
|
||||
}
|
||||
if (hasLegacyFields(data)) {
|
||||
throw new BusinessException(EXPIRED_MESSAGE);
|
||||
}
|
||||
DatasetRef datasetRef = data.getObject("datasetRef", DatasetRef.class);
|
||||
if (datasetRef == null || datasetRef.getTableId() == null) {
|
||||
throw new BusinessException(EXPIRED_MESSAGE);
|
||||
}
|
||||
return new SaveDatasetNode(datasetRef);
|
||||
}
|
||||
|
||||
public String getNodeName() {
|
||||
return "save-dataset-node";
|
||||
}
|
||||
|
||||
private boolean hasLegacyFields(JSONObject data) {
|
||||
if (data == null) {
|
||||
return false;
|
||||
}
|
||||
return data.containsKey("tableId")
|
||||
|| data.containsKey("sourceId")
|
||||
|| data.containsKey("catalogId")
|
||||
|| data.containsKey("versionId");
|
||||
}
|
||||
}
|
||||
@@ -1,75 +0,0 @@
|
||||
package tech.easyflow.ai.node;
|
||||
|
||||
import com.alibaba.fastjson2.JSONArray;
|
||||
import com.alibaba.fastjson2.JSONObject;
|
||||
import com.mybatisflex.core.tenant.TenantManager;
|
||||
import com.easyagents.flow.core.chain.Chain;
|
||||
import com.easyagents.flow.core.node.BaseNode;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import tech.easyflow.ai.utils.WorkFlowUtil;
|
||||
import tech.easyflow.common.entity.LoginAccount;
|
||||
import tech.easyflow.common.util.SpringContextUtil;
|
||||
import tech.easyflow.datacenter.service.DatacenterTableService;
|
||||
|
||||
import java.math.BigInteger;
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
|
||||
public class SaveToDatacenterNode extends BaseNode {
|
||||
|
||||
private static final Logger log = LoggerFactory.getLogger(SaveToDatacenterNode.class);
|
||||
|
||||
private BigInteger tableId;
|
||||
|
||||
public SaveToDatacenterNode() {
|
||||
}
|
||||
|
||||
public SaveToDatacenterNode(BigInteger tableId) {
|
||||
this.tableId = tableId;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Map<String, Object> execute(Chain chain) {
|
||||
|
||||
Map<String, Object> map = chain.getState().resolveParameters(this);
|
||||
JSONObject json = new JSONObject(map);
|
||||
|
||||
Map<String, Object> res = new HashMap<>();
|
||||
|
||||
// 默认为未知来源
|
||||
LoginAccount account = WorkFlowUtil.getOperator(chain);
|
||||
|
||||
DatacenterTableService service = SpringContextUtil.getBean(DatacenterTableService.class);
|
||||
|
||||
JSONArray saveList = json.getJSONArray("saveList");
|
||||
|
||||
int successRows = 0;
|
||||
for (Object object : saveList) {
|
||||
JSONObject obj = new JSONObject((com.alibaba.fastjson.JSONObject) object);
|
||||
obj.put("table_id", tableId);
|
||||
try {
|
||||
TenantManager.ignoreTenantCondition();
|
||||
service.saveValue(tableId, obj, account);
|
||||
} catch (Exception e) {
|
||||
log.error("工作流保存数据到数据中枢失败,表ID:{},具体值:{}", tableId, obj, e);
|
||||
throw e;
|
||||
} finally {
|
||||
TenantManager.restoreTenantCondition();
|
||||
}
|
||||
successRows++;
|
||||
}
|
||||
|
||||
res.put("successRows", successRows);
|
||||
return res;
|
||||
}
|
||||
|
||||
public BigInteger getTableId() {
|
||||
return tableId;
|
||||
}
|
||||
|
||||
public void setTableId(BigInteger tableId) {
|
||||
this.tableId = tableId;
|
||||
}
|
||||
|
||||
}
|
||||
@@ -1,23 +0,0 @@
|
||||
package tech.easyflow.ai.node;
|
||||
|
||||
import com.alibaba.fastjson.JSONObject;
|
||||
import com.easyagents.flow.core.node.BaseNode;
|
||||
import com.easyagents.flow.core.parser.BaseNodeParser;
|
||||
|
||||
import java.math.BigInteger;
|
||||
|
||||
public class SaveToDatacenterNodeParser extends BaseNodeParser {
|
||||
|
||||
@Override
|
||||
protected BaseNode doParse(JSONObject root, JSONObject data, JSONObject tinyflow) {
|
||||
BigInteger tableId = data.getBigInteger("tableId");
|
||||
if (tableId == null) {
|
||||
throw new RuntimeException("请选择数据表");
|
||||
}
|
||||
return new SaveToDatacenterNode(tableId);
|
||||
}
|
||||
|
||||
public String getNodeName() {
|
||||
return "save-to-datacenter-node";
|
||||
}
|
||||
}
|
||||
@@ -1,143 +0,0 @@
|
||||
package tech.easyflow.ai.node;
|
||||
|
||||
import com.easyagents.core.util.StringUtil;
|
||||
import com.mybatisflex.core.paginate.Page;
|
||||
import com.mybatisflex.core.row.Row;
|
||||
import com.mybatisflex.core.tenant.TenantManager;
|
||||
import com.easyagents.flow.core.chain.Chain;
|
||||
import com.easyagents.flow.core.chain.Parameter;
|
||||
import com.easyagents.flow.core.node.BaseNode;
|
||||
import net.sf.jsqlparser.expression.Expression;
|
||||
import net.sf.jsqlparser.parser.CCJSqlParserUtil;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import tech.easyflow.common.entity.DatacenterQuery;
|
||||
import tech.easyflow.common.util.SpringContextUtil;
|
||||
import tech.easyflow.datacenter.entity.DatacenterTableField;
|
||||
import tech.easyflow.datacenter.service.DatacenterTableService;
|
||||
import tech.easyflow.datacenter.utils.WhereConditionSecurityChecker;
|
||||
|
||||
import java.math.BigInteger;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.regex.Matcher;
|
||||
import java.util.regex.Pattern;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
public class SearchDatacenterNode extends BaseNode {
|
||||
|
||||
private static final Logger log = LoggerFactory.getLogger(SearchDatacenterNode.class);
|
||||
private BigInteger tableId;
|
||||
private String where;
|
||||
private Long limit;
|
||||
|
||||
public SearchDatacenterNode() {
|
||||
}
|
||||
|
||||
public SearchDatacenterNode(BigInteger tableId, String where, Long limit) {
|
||||
this.tableId = tableId;
|
||||
this.where = where;
|
||||
this.limit = limit;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Map<String, Object> execute(Chain chain) {
|
||||
|
||||
Map<String, Object> map = chain.getState().resolveParameters(this);
|
||||
Map<String, Object> res = new HashMap<>();
|
||||
long limitNum = 10;
|
||||
if (limit != null) {
|
||||
limitNum = Long.parseLong(limit.toString());
|
||||
}
|
||||
|
||||
DatacenterTableService service = SpringContextUtil.getBean(DatacenterTableService.class);
|
||||
|
||||
DatacenterQuery condition = new DatacenterQuery();
|
||||
condition.setTableId(tableId);
|
||||
condition.setPageNumber(1L);
|
||||
condition.setPageSize(limitNum);
|
||||
// 组合查询条件
|
||||
if (where != null) {
|
||||
setCondition(where, condition, map);
|
||||
}
|
||||
try {
|
||||
TenantManager.ignoreTenantCondition();
|
||||
Page<Row> pageData = service.getPageData(condition);
|
||||
|
||||
String key = "rows";
|
||||
List<Parameter> outputDefs = getOutputDefs();
|
||||
if (outputDefs != null && !outputDefs.isEmpty()) {
|
||||
String defName = outputDefs.get(0).getName();
|
||||
if (StringUtil.hasText(defName)) key = defName;
|
||||
}
|
||||
res.put(key, pageData.getRecords());
|
||||
} finally {
|
||||
TenantManager.restoreTenantCondition();
|
||||
}
|
||||
return res;
|
||||
}
|
||||
|
||||
public BigInteger getTableId() {
|
||||
return tableId;
|
||||
}
|
||||
|
||||
public void setTableId(BigInteger tableId) {
|
||||
this.tableId = tableId;
|
||||
}
|
||||
|
||||
public String getWhere() {
|
||||
return where;
|
||||
}
|
||||
|
||||
public void setWhere(String where) {
|
||||
this.where = where;
|
||||
}
|
||||
|
||||
public Long getLimit() {
|
||||
return limit;
|
||||
}
|
||||
|
||||
public void setLimit(Long limit) {
|
||||
this.limit = limit;
|
||||
}
|
||||
|
||||
private void setCondition(String where, DatacenterQuery condition, Map<String, Object> params) {
|
||||
// 条件封装
|
||||
Pattern pattern = Pattern.compile("\\{\\{(.+?)\\}\\}");
|
||||
Matcher matcher = pattern.matcher(where);
|
||||
|
||||
StringBuffer result = new StringBuffer();
|
||||
while (matcher.find()) {
|
||||
String key = matcher.group(1);
|
||||
Object value = params.get(key);
|
||||
if (value == null) {
|
||||
throw new RuntimeException("参数" + key + "不存在");
|
||||
}
|
||||
String replacement = value.toString();
|
||||
matcher.appendReplacement(result, "'" + replacement + "'");
|
||||
}
|
||||
matcher.appendTail(result);
|
||||
|
||||
try {
|
||||
Expression expression = CCJSqlParserUtil.parseCondExpression(result.toString());
|
||||
if (expression != null) {
|
||||
WhereConditionSecurityChecker checker = new WhereConditionSecurityChecker();
|
||||
DatacenterTableService service = SpringContextUtil.getBean(DatacenterTableService.class);
|
||||
List<DatacenterTableField> fields = service.getFields(tableId);
|
||||
Set<String> columns = fields.stream().map(DatacenterTableField::getFieldName).collect(Collectors.toSet());
|
||||
columns.add("id");
|
||||
columns.add("created");
|
||||
columns.add("modified");
|
||||
columns.add("created_by");
|
||||
columns.add("modified_by");
|
||||
checker.checkConditionSafety(expression, columns);
|
||||
condition.setWhere(expression.toString());
|
||||
}
|
||||
} catch (Exception e) {
|
||||
log.error("WHERE SQL解析错误:", e);
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,25 +0,0 @@
|
||||
package tech.easyflow.ai.node;
|
||||
|
||||
import com.alibaba.fastjson.JSONObject;
|
||||
import com.easyagents.flow.core.node.BaseNode;
|
||||
import com.easyagents.flow.core.parser.BaseNodeParser;
|
||||
|
||||
import java.math.BigInteger;
|
||||
|
||||
public class SearchDatacenterNodeParser extends BaseNodeParser {
|
||||
|
||||
@Override
|
||||
protected BaseNode doParse(JSONObject root, JSONObject data, JSONObject tinyflow) {
|
||||
BigInteger tableId = data.getBigInteger("tableId");
|
||||
String where = data.getString("where");
|
||||
Long limit = data.getLong("limit");
|
||||
if (tableId == null) {
|
||||
throw new RuntimeException("请选择数据表");
|
||||
}
|
||||
return new SearchDatacenterNode(tableId,where,limit);
|
||||
}
|
||||
|
||||
public String getNodeName() {
|
||||
return "search-datacenter-node";
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,120 @@
|
||||
package tech.easyflow.ai.node;
|
||||
|
||||
import com.easyagents.core.util.StringUtil;
|
||||
import com.easyagents.flow.core.chain.Chain;
|
||||
import com.easyagents.flow.core.chain.Parameter;
|
||||
import com.easyagents.flow.core.node.BaseNode;
|
||||
import com.mybatisflex.core.row.Row;
|
||||
import com.mybatisflex.core.tenant.TenantManager;
|
||||
import tech.easyflow.common.util.SpringContextUtil;
|
||||
import tech.easyflow.common.web.exceptions.BusinessException;
|
||||
import tech.easyflow.datacenter.execution.model.DatacenterSqlQueryRequest;
|
||||
import tech.easyflow.datacenter.execution.model.DatasetRef;
|
||||
import tech.easyflow.datacenter.execution.service.DatacenterDatasetQueryService;
|
||||
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.regex.Matcher;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
public class SearchDatasetNode extends BaseNode {
|
||||
|
||||
private static final Pattern PARAM_PATTERN = Pattern.compile("\\{\\{(.+?)\\}\\}");
|
||||
|
||||
private DatasetRef datasetRef;
|
||||
private String querySql;
|
||||
|
||||
public SearchDatasetNode() {
|
||||
}
|
||||
|
||||
public SearchDatasetNode(DatasetRef datasetRef) {
|
||||
this.datasetRef = datasetRef;
|
||||
}
|
||||
|
||||
public SearchDatasetNode(DatasetRef datasetRef, String querySql) {
|
||||
this.datasetRef = datasetRef;
|
||||
this.querySql = querySql;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Map<String, Object> execute(Chain chain) {
|
||||
Map<String, Object> params = chain.getState().resolveParameters(this);
|
||||
DatacenterDatasetQueryService queryService = SpringContextUtil.getBean(DatacenterDatasetQueryService.class);
|
||||
DatacenterSqlQueryRequest request = buildRuntimeRequest(params);
|
||||
Map<String, Object> result = new HashMap<>();
|
||||
try {
|
||||
TenantManager.ignoreTenantCondition();
|
||||
List<Row> rows = queryService.queryBySql(request);
|
||||
result.put(resolveOutputKey("data"), rows);
|
||||
return result;
|
||||
} finally {
|
||||
TenantManager.restoreTenantCondition();
|
||||
}
|
||||
}
|
||||
|
||||
private DatacenterSqlQueryRequest buildRuntimeRequest(Map<String, Object> params) {
|
||||
DatacenterSqlQueryRequest request = new DatacenterSqlQueryRequest();
|
||||
request.setDatasetRef(copyDatasetRef());
|
||||
request.setSql(resolveQuerySql(params));
|
||||
return request;
|
||||
}
|
||||
|
||||
private String resolveQuerySql(Map<String, Object> params) {
|
||||
String sql = resolveTemplateString(querySql, params);
|
||||
if (!StringUtil.hasText(sql)) {
|
||||
throw new BusinessException("查询数据节点未设置 SQL");
|
||||
}
|
||||
return sql.trim();
|
||||
}
|
||||
|
||||
private DatasetRef copyDatasetRef() {
|
||||
DatasetRef copy = new DatasetRef();
|
||||
copy.setSourceId(datasetRef == null ? null : datasetRef.getSourceId());
|
||||
copy.setCatalogId(datasetRef == null ? null : datasetRef.getCatalogId());
|
||||
copy.setCatalogName(datasetRef == null ? null : datasetRef.getCatalogName());
|
||||
copy.setTableId(null);
|
||||
copy.setTableName(null);
|
||||
copy.setVersionId(null);
|
||||
return copy;
|
||||
}
|
||||
|
||||
private String resolveTemplateString(String text, Map<String, Object> params) {
|
||||
if (!StringUtil.hasText(text) || params == null || params.isEmpty()) {
|
||||
return text;
|
||||
}
|
||||
Matcher matcher = PARAM_PATTERN.matcher(text);
|
||||
StringBuffer buffer = new StringBuffer();
|
||||
while (matcher.find()) {
|
||||
Object replacement = params.get(matcher.group(1));
|
||||
matcher.appendReplacement(buffer, replacement == null ? "" : Matcher.quoteReplacement(String.valueOf(replacement)));
|
||||
}
|
||||
matcher.appendTail(buffer);
|
||||
return buffer.toString();
|
||||
}
|
||||
|
||||
private String resolveOutputKey(String defaultName) {
|
||||
List<Parameter> outputDefs = getOutputDefs();
|
||||
if (outputDefs == null || outputDefs.isEmpty()) {
|
||||
return defaultName;
|
||||
}
|
||||
String name = outputDefs.get(0).getName();
|
||||
return StringUtil.hasText(name) ? name : defaultName;
|
||||
}
|
||||
|
||||
public DatasetRef getDatasetRef() {
|
||||
return datasetRef;
|
||||
}
|
||||
|
||||
public void setDatasetRef(DatasetRef datasetRef) {
|
||||
this.datasetRef = datasetRef;
|
||||
}
|
||||
|
||||
public String getQuerySql() {
|
||||
return querySql;
|
||||
}
|
||||
|
||||
public void setQuerySql(String querySql) {
|
||||
this.querySql = querySql;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,33 @@
|
||||
package tech.easyflow.ai.node;
|
||||
|
||||
import com.alibaba.fastjson.JSONObject;
|
||||
import com.easyagents.flow.core.node.BaseNode;
|
||||
import com.easyagents.flow.core.parser.BaseNodeParser;
|
||||
import tech.easyflow.common.web.exceptions.BusinessException;
|
||||
import tech.easyflow.datacenter.execution.model.DatasetRef;
|
||||
|
||||
public class SearchDatasetNodeParser extends BaseNodeParser {
|
||||
|
||||
private static final String SOURCE_MISSING_MESSAGE = "查询数据节点未选择连接服务";
|
||||
private static final String SQL_MISSING_MESSAGE = "查询数据节点未设置 SQL";
|
||||
|
||||
@Override
|
||||
protected BaseNode doParse(JSONObject root, JSONObject data, JSONObject tinyflow) {
|
||||
if (data == null) {
|
||||
throw new BusinessException(SOURCE_MISSING_MESSAGE);
|
||||
}
|
||||
DatasetRef datasetRef = data.getObject("datasetRef", DatasetRef.class);
|
||||
if (datasetRef == null || datasetRef.getSourceId() == null) {
|
||||
throw new BusinessException(SOURCE_MISSING_MESSAGE);
|
||||
}
|
||||
String querySql = data.getString("querySql");
|
||||
if (querySql == null || querySql.isBlank()) {
|
||||
throw new BusinessException(SQL_MISSING_MESSAGE);
|
||||
}
|
||||
return new SearchDatasetNode(datasetRef, querySql);
|
||||
}
|
||||
|
||||
public String getNodeName() {
|
||||
return "search-dataset-node";
|
||||
}
|
||||
}
|
||||
@@ -1,149 +0,0 @@
|
||||
package tech.easyflow.ai.node;
|
||||
|
||||
import com.mybatisflex.core.row.Db;
|
||||
import com.mybatisflex.core.row.Row;
|
||||
import com.easyagents.flow.core.chain.Chain;
|
||||
import com.easyagents.flow.core.node.BaseNode;
|
||||
import net.sf.jsqlparser.JSQLParserException;
|
||||
import net.sf.jsqlparser.parser.CCJSqlParserUtil;
|
||||
import net.sf.jsqlparser.statement.Statement;
|
||||
import net.sf.jsqlparser.statement.select.Select;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.util.StringUtils;
|
||||
import tech.easyflow.common.web.exceptions.BusinessException;
|
||||
|
||||
import java.util.*;
|
||||
import java.util.regex.Matcher;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
|
||||
/**
|
||||
* SQL查询节点
|
||||
*
|
||||
* @author tao
|
||||
* @date 2025-05-21
|
||||
*/
|
||||
public class SqlNode extends BaseNode {
|
||||
|
||||
private String sql;
|
||||
|
||||
private static final Logger logger = LoggerFactory.getLogger(SqlNode.class);
|
||||
|
||||
public SqlNode() {
|
||||
}
|
||||
|
||||
public SqlNode(String sql) {
|
||||
this.sql = sql;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Map<String, Object> execute(Chain chain) {
|
||||
|
||||
Map<String, Object> map = chain.getState().resolveParameters(this);
|
||||
Map<String, Object> res = new HashMap<>();
|
||||
|
||||
|
||||
Map<String, Object> formatSqlMap = formatSql(sql, map);
|
||||
String formatSql = (String) formatSqlMap.get("replacedSql");
|
||||
|
||||
Statement statement = null;
|
||||
try {
|
||||
statement = CCJSqlParserUtil.parse(formatSql);
|
||||
|
||||
} catch (JSQLParserException e) {
|
||||
logger.error("sql 解析报错:", e);
|
||||
throw new BusinessException("SQL解析失败,请确认SQL语法无误");
|
||||
}
|
||||
|
||||
if (!(statement instanceof Select)) {
|
||||
logger.error("sql 解析报错:statement instanceof Select 结果为false");
|
||||
throw new BusinessException("仅支持查询语句!");
|
||||
}
|
||||
|
||||
List<String> paramNames = (List<String>) formatSqlMap.get("paramNames");
|
||||
|
||||
List<Object> paramValues = new ArrayList<>();
|
||||
paramNames.forEach(paramName -> {
|
||||
Object o = map.get(paramName);
|
||||
paramValues.add(o);
|
||||
});
|
||||
|
||||
List<Row> rows = Db.selectListBySql(formatSql, paramValues.toArray());
|
||||
|
||||
if (rows == null || rows.isEmpty()) {
|
||||
return Collections.emptyMap();
|
||||
}
|
||||
|
||||
res.put("queryData", rows);
|
||||
return res;
|
||||
}
|
||||
|
||||
private Map<String, Object> formatSql(String rawSql, Map<String, Object> paramMap) {
|
||||
|
||||
if (!StringUtils.hasLength(rawSql)) {
|
||||
logger.error("sql解析报错:sql为空");
|
||||
throw new BusinessException("sql 不能为空!");
|
||||
}
|
||||
|
||||
// 匹配 {{?...}} 表示可用占位符的参数
|
||||
Pattern paramPattern = Pattern.compile("\\{\\{\\?([^}]+)}}");
|
||||
|
||||
// 匹配 {{...}} 表示直接替换的参数(非占位符)
|
||||
Pattern directPattern = Pattern.compile("\\{\\{([^}?][^}]*)}}");
|
||||
|
||||
List<String> paramNames = new ArrayList<>();
|
||||
StringBuffer sqlBuffer = new StringBuffer();
|
||||
|
||||
// 替换 {{?...}} -> ?
|
||||
Matcher paramMatcher = paramPattern.matcher(rawSql);
|
||||
while (paramMatcher.find()) {
|
||||
String paramName = paramMatcher.group(1).trim();
|
||||
paramNames.add(paramName);
|
||||
paramMatcher.appendReplacement(sqlBuffer, "?");
|
||||
}
|
||||
paramMatcher.appendTail(sqlBuffer);
|
||||
String intermediateSql = sqlBuffer.toString();
|
||||
|
||||
// 替换 {{...}} -> 实际值(用于表名/列名等)
|
||||
sqlBuffer = new StringBuffer(); // 清空 buffer 重新处理
|
||||
Matcher directMatcher = directPattern.matcher(intermediateSql);
|
||||
while (directMatcher.find()) {
|
||||
String key = directMatcher.group(1).trim();
|
||||
Object value = paramMap.get(key);
|
||||
if (value == null) {
|
||||
logger.error("未找到参数:" + key);
|
||||
throw new BusinessException("sql解析失败,请确保sql语法正确!");
|
||||
}
|
||||
|
||||
String safeValue = value.toString();
|
||||
|
||||
directMatcher.appendReplacement(sqlBuffer, Matcher.quoteReplacement(safeValue));
|
||||
}
|
||||
directMatcher.appendTail(sqlBuffer);
|
||||
|
||||
String finalSql = sqlBuffer.toString().trim();
|
||||
|
||||
// 清理末尾分号与中文引号
|
||||
if (finalSql.endsWith(";") || finalSql.endsWith(";")) {
|
||||
finalSql = finalSql.substring(0, finalSql.length() - 1);
|
||||
}
|
||||
finalSql = finalSql.replace("“", "\"").replace("”", "\"");
|
||||
|
||||
logger.info("Final SQL: {}", finalSql);
|
||||
logger.info("Param names: {}", paramNames);
|
||||
|
||||
Map<String, Object> result = new HashMap<>();
|
||||
result.put("replacedSql", finalSql);
|
||||
result.put("paramNames", paramNames);
|
||||
return result;
|
||||
}
|
||||
|
||||
public String getSql() {
|
||||
return sql;
|
||||
}
|
||||
|
||||
public void setSql(String sql) {
|
||||
this.sql = sql;
|
||||
}
|
||||
}
|
||||
@@ -1,25 +0,0 @@
|
||||
package tech.easyflow.ai.node;
|
||||
|
||||
import com.alibaba.fastjson.JSONObject;
|
||||
import com.easyagents.flow.core.node.BaseNode;
|
||||
import com.easyagents.flow.core.parser.BaseNodeParser;
|
||||
|
||||
/**
|
||||
* Sql查询节点解析
|
||||
*
|
||||
* @author tao
|
||||
* @date 2025-05-21
|
||||
*/
|
||||
public class SqlNodeParser extends BaseNodeParser {
|
||||
|
||||
|
||||
@Override
|
||||
public BaseNode doParse(JSONObject root, JSONObject data, JSONObject tinyflow) {
|
||||
String sql = data.getString("sql");
|
||||
return new SqlNode(sql);
|
||||
}
|
||||
|
||||
public String getNodeName() {
|
||||
return "sql-node";
|
||||
}
|
||||
}
|
||||
@@ -8,6 +8,7 @@ import org.junit.Test;
|
||||
import tech.easyflow.ai.easyagentsflow.entity.WorkflowCheckResult;
|
||||
import tech.easyflow.ai.easyagentsflow.entity.WorkflowCheckStage;
|
||||
import tech.easyflow.ai.entity.Workflow;
|
||||
import tech.easyflow.ai.node.SearchDatasetNodeParser;
|
||||
import tech.easyflow.ai.node.WorkflowNodeParser;
|
||||
import tech.easyflow.ai.service.WorkflowService;
|
||||
|
||||
@@ -69,6 +70,24 @@ public class WorkflowCheckServiceTest {
|
||||
assertHasCode(result, "EDGE_TARGET_NOT_FOUND");
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testSaveShouldBlockSearchDatasetWithoutSql() throws Exception {
|
||||
WorkflowCheckService service = newService(new HashMap<>());
|
||||
JSONObject searchData = data("查询数据");
|
||||
JSONObject datasetRef = new JSONObject();
|
||||
datasetRef.put("sourceId", "1001");
|
||||
datasetRef.put("tableId", "2001");
|
||||
searchData.put("datasetRef", datasetRef);
|
||||
String content = workflowJson(
|
||||
array(node("search-1", "search-dataset-node", null, searchData)),
|
||||
new JSONArray()
|
||||
);
|
||||
|
||||
WorkflowCheckResult result = service.checkContent(content, WorkflowCheckStage.SAVE, null);
|
||||
Assert.assertFalse(result.isPassed());
|
||||
assertHasCode(result, "SEARCH_DATASET_INVALID");
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testPreExecuteShouldBlockMissingStartOrEnd() throws Exception {
|
||||
WorkflowCheckService service = newService(new HashMap<>());
|
||||
@@ -86,6 +105,26 @@ public class WorkflowCheckServiceTest {
|
||||
assertHasCode(result, "END_NODE_MISSING");
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testPreExecuteShouldPassForSourceOnlySearchDatasetNode() throws Exception {
|
||||
WorkflowCheckService service = newService(new HashMap<>());
|
||||
String content = workflowJson(
|
||||
array(
|
||||
node("s1", "startNode", null, data("开始")),
|
||||
searchDatasetNode("q1", null, "1001"),
|
||||
node("e1", "endNode", null, data("结束"))
|
||||
),
|
||||
array(
|
||||
edge("e1", "s1", "q1"),
|
||||
edge("e2", "q1", "e1")
|
||||
)
|
||||
);
|
||||
|
||||
WorkflowCheckResult result = service.checkContent(content, WorkflowCheckStage.PRE_EXECUTE, BigInteger.ONE);
|
||||
Assert.assertTrue(result.isPassed());
|
||||
Assert.assertEquals(0, result.getIssueCount());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testPreExecuteShouldBlockRootEntryNotStart() throws Exception {
|
||||
WorkflowCheckService service = newService(new HashMap<>());
|
||||
@@ -203,8 +242,10 @@ public class WorkflowCheckServiceTest {
|
||||
.withDefaultParsers(true)
|
||||
.build();
|
||||
parser.addNodeParser("workflow-node", new WorkflowNodeParser());
|
||||
parser.addNodeParser("search-dataset-node", new SearchDatasetNodeParser());
|
||||
setField(service, "chainParser", parser);
|
||||
setField(service, "workflowService", mockWorkflowService(workflowStore));
|
||||
setField(service, "workflowDatacenterContentService", new WorkflowDatacenterContentService());
|
||||
return service;
|
||||
}
|
||||
|
||||
@@ -294,6 +335,15 @@ public class WorkflowCheckServiceTest {
|
||||
return node(id, "workflow-node", parentId, data);
|
||||
}
|
||||
|
||||
private static JSONObject searchDatasetNode(String id, String parentId, String sourceId) {
|
||||
JSONObject data = data("查询数据");
|
||||
JSONObject datasetRef = new JSONObject();
|
||||
datasetRef.put("sourceId", sourceId);
|
||||
data.put("datasetRef", datasetRef);
|
||||
data.put("querySql", "SELECT 1");
|
||||
return node(id, "search-dataset-node", parentId, data);
|
||||
}
|
||||
|
||||
private static JSONObject data(String title) {
|
||||
JSONObject data = new JSONObject();
|
||||
data.put("title", title);
|
||||
|
||||
@@ -0,0 +1,73 @@
|
||||
package tech.easyflow.ai.node;
|
||||
|
||||
import org.junit.Assert;
|
||||
import org.junit.Test;
|
||||
import tech.easyflow.common.web.exceptions.BusinessException;
|
||||
import tech.easyflow.datacenter.execution.model.DatasetRef;
|
||||
|
||||
import java.lang.reflect.Method;
|
||||
import java.math.BigInteger;
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
|
||||
public class SearchDatasetNodeTest {
|
||||
|
||||
@Test
|
||||
public void testResolveQuerySqlShouldUseNodeQuerySqlAndResolveTemplate() throws Exception {
|
||||
DatasetRef datasetRef = new DatasetRef();
|
||||
datasetRef.setSourceId(BigInteger.valueOf(1001L));
|
||||
SearchDatasetNode node = new SearchDatasetNode(datasetRef, """
|
||||
SELECT id, name
|
||||
FROM orders_{{biz}}
|
||||
WHERE name LIKE '%{{keyword}}%'
|
||||
ORDER BY created_at {{direction}}
|
||||
""");
|
||||
|
||||
Map<String, Object> params = new HashMap<>();
|
||||
params.put("biz", "prod");
|
||||
params.put("keyword", "vip");
|
||||
params.put("direction", "DESC");
|
||||
|
||||
String sql = invokeResolveQuerySql(node, params);
|
||||
Assert.assertEquals("""
|
||||
SELECT id, name
|
||||
FROM orders_prod
|
||||
WHERE name LIKE '%vip%'
|
||||
ORDER BY created_at DESC
|
||||
""".trim(), sql);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testResolveQuerySqlShouldUseNodeQuerySqlWhenParamsDoNotContainSql() throws Exception {
|
||||
DatasetRef datasetRef = new DatasetRef();
|
||||
datasetRef.setSourceId(BigInteger.valueOf(2002L));
|
||||
SearchDatasetNode node = new SearchDatasetNode(datasetRef, "SELECT * FROM fallback_table");
|
||||
|
||||
Map<String, Object> params = new HashMap<>();
|
||||
|
||||
String sql = invokeResolveQuerySql(node, params);
|
||||
Assert.assertEquals("SELECT * FROM fallback_table", sql);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testResolveQuerySqlShouldRejectBlankSql() throws Exception {
|
||||
DatasetRef datasetRef = new DatasetRef();
|
||||
datasetRef.setSourceId(BigInteger.valueOf(3003L));
|
||||
SearchDatasetNode node = new SearchDatasetNode(datasetRef, " ");
|
||||
|
||||
try {
|
||||
invokeResolveQuerySql(node, new HashMap<>());
|
||||
Assert.fail("expected BusinessException");
|
||||
} catch (Exception e) {
|
||||
Throwable cause = e.getCause() == null ? e : e.getCause();
|
||||
Assert.assertTrue(cause instanceof BusinessException);
|
||||
Assert.assertEquals("查询数据节点未设置 SQL", cause.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
private String invokeResolveQuerySql(SearchDatasetNode node, Map<String, Object> params) throws Exception {
|
||||
Method method = SearchDatasetNode.class.getDeclaredMethod("resolveQuerySql", Map.class);
|
||||
method.setAccessible(true);
|
||||
return (String) method.invoke(node, params);
|
||||
}
|
||||
}
|
||||
@@ -16,6 +16,19 @@
|
||||
<groupId>com.mybatis-flex</groupId>
|
||||
<artifactId>mybatis-flex-spring-boot3-starter</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>com.zaxxer</groupId>
|
||||
<artifactId>HikariCP</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.postgresql</groupId>
|
||||
<artifactId>postgresql</artifactId>
|
||||
<version>42.7.3</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>cn.hutool</groupId>
|
||||
<artifactId>hutool-crypto</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>tech.easyflow</groupId>
|
||||
<artifactId>easyflow-common-base</artifactId>
|
||||
|
||||
@@ -0,0 +1,12 @@
|
||||
package tech.easyflow.datacenter.connector;
|
||||
|
||||
import tech.easyflow.datacenter.meta.enums.DatacenterCapability;
|
||||
import tech.easyflow.datacenter.meta.enums.DatacenterSourceType;
|
||||
|
||||
import java.util.Set;
|
||||
|
||||
public interface DatacenterConnector extends SourceHealthChecker, MetadataExplorer, QueryExecutor, WriteExecutor {
|
||||
DatacenterSourceType getSourceType();
|
||||
|
||||
Set<DatacenterCapability> getCapabilities();
|
||||
}
|
||||
@@ -0,0 +1,29 @@
|
||||
package tech.easyflow.datacenter.connector;
|
||||
|
||||
import org.springframework.stereotype.Component;
|
||||
import tech.easyflow.common.web.exceptions.BusinessException;
|
||||
import tech.easyflow.datacenter.meta.enums.DatacenterSourceType;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.function.Function;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
@Component
|
||||
public class DatacenterConnectorRegistry {
|
||||
|
||||
private final Map<DatacenterSourceType, DatacenterConnector> connectorMap;
|
||||
|
||||
public DatacenterConnectorRegistry(List<DatacenterConnector> connectors) {
|
||||
this.connectorMap = connectors.stream().collect(Collectors.toMap(DatacenterConnector::getSourceType, Function.identity()));
|
||||
}
|
||||
|
||||
public DatacenterConnector getConnector(String sourceType) {
|
||||
DatacenterSourceType type = DatacenterSourceType.valueOf(sourceType);
|
||||
DatacenterConnector connector = connectorMap.get(type);
|
||||
if (connector == null) {
|
||||
throw new BusinessException("未找到连接器: " + sourceType);
|
||||
}
|
||||
return connector;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,16 @@
|
||||
package tech.easyflow.datacenter.connector;
|
||||
|
||||
import tech.easyflow.datacenter.entity.DatacenterTable;
|
||||
import tech.easyflow.datacenter.meta.entity.DatacenterSource;
|
||||
import tech.easyflow.datacenter.meta.model.DatacenterCatalogMeta;
|
||||
import tech.easyflow.datacenter.meta.model.DatacenterTableDetailMeta;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
public interface MetadataExplorer {
|
||||
List<DatacenterCatalogMeta> listCatalogs(DatacenterSource source);
|
||||
|
||||
List<DatacenterTable> listTables(DatacenterSource source, String catalogName);
|
||||
|
||||
DatacenterTableDetailMeta getTableDetail(DatacenterSource source, String catalogName, String tableName);
|
||||
}
|
||||
@@ -0,0 +1,15 @@
|
||||
package tech.easyflow.datacenter.connector;
|
||||
|
||||
import com.mybatisflex.core.paginate.Page;
|
||||
import com.mybatisflex.core.row.Row;
|
||||
import tech.easyflow.datacenter.entity.DatacenterTable;
|
||||
import tech.easyflow.datacenter.execution.model.DatacenterQueryRequest;
|
||||
import tech.easyflow.datacenter.meta.entity.DatacenterSource;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
public interface QueryExecutor {
|
||||
Page<Row> queryPage(DatacenterSource source, DatacenterTable table, DatacenterQueryRequest request);
|
||||
|
||||
List<Row> queryBySql(DatacenterSource source, String sql);
|
||||
}
|
||||
@@ -0,0 +1,8 @@
|
||||
package tech.easyflow.datacenter.connector;
|
||||
|
||||
import tech.easyflow.datacenter.execution.model.DatacenterConnectionTestResult;
|
||||
import tech.easyflow.datacenter.meta.entity.DatacenterSource;
|
||||
|
||||
public interface SourceHealthChecker {
|
||||
DatacenterConnectionTestResult testConnection(DatacenterSource source);
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
package tech.easyflow.datacenter.connector;
|
||||
|
||||
public interface SqlDialect {
|
||||
String quoteIdentifier(String identifier);
|
||||
|
||||
String qualifyTable(String namespace, String tableName);
|
||||
|
||||
String buildPageSql(String baseSql, long pageNumber, long pageSize);
|
||||
}
|
||||
@@ -0,0 +1,14 @@
|
||||
package tech.easyflow.datacenter.connector;
|
||||
|
||||
import com.alibaba.fastjson2.JSONObject;
|
||||
import tech.easyflow.common.entity.LoginAccount;
|
||||
import tech.easyflow.datacenter.entity.DatacenterTable;
|
||||
import tech.easyflow.datacenter.meta.entity.DatacenterSource;
|
||||
|
||||
import java.math.BigInteger;
|
||||
|
||||
public interface WriteExecutor {
|
||||
void saveRow(DatacenterSource source, DatacenterTable table, JSONObject data, LoginAccount account);
|
||||
|
||||
void deleteRow(DatacenterSource source, DatacenterTable table, BigInteger id, LoginAccount account);
|
||||
}
|
||||
@@ -0,0 +1,24 @@
|
||||
package tech.easyflow.datacenter.connector.dialect;
|
||||
|
||||
import cn.hutool.core.util.StrUtil;
|
||||
import tech.easyflow.datacenter.connector.SqlDialect;
|
||||
|
||||
public class GaussdbSqlDialect implements SqlDialect {
|
||||
@Override
|
||||
public String quoteIdentifier(String identifier) {
|
||||
return '"' + identifier + '"';
|
||||
}
|
||||
|
||||
@Override
|
||||
public String qualifyTable(String namespace, String tableName) {
|
||||
if (StrUtil.isBlank(namespace)) {
|
||||
return quoteIdentifier(tableName);
|
||||
}
|
||||
return quoteIdentifier(namespace) + "." + quoteIdentifier(tableName);
|
||||
}
|
||||
|
||||
@Override
|
||||
public String buildPageSql(String baseSql, long pageNumber, long pageSize) {
|
||||
return baseSql + " OFFSET ? LIMIT ?";
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,24 @@
|
||||
package tech.easyflow.datacenter.connector.dialect;
|
||||
|
||||
import cn.hutool.core.util.StrUtil;
|
||||
import tech.easyflow.datacenter.connector.SqlDialect;
|
||||
|
||||
public class MysqlSqlDialect implements SqlDialect {
|
||||
@Override
|
||||
public String quoteIdentifier(String identifier) {
|
||||
return "`" + identifier + "`";
|
||||
}
|
||||
|
||||
@Override
|
||||
public String qualifyTable(String namespace, String tableName) {
|
||||
if (StrUtil.isBlank(namespace)) {
|
||||
return quoteIdentifier(tableName);
|
||||
}
|
||||
return quoteIdentifier(namespace) + "." + quoteIdentifier(tableName);
|
||||
}
|
||||
|
||||
@Override
|
||||
public String buildPageSql(String baseSql, long pageNumber, long pageSize) {
|
||||
return baseSql + " LIMIT ?, ?";
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,24 @@
|
||||
package tech.easyflow.datacenter.connector.dialect;
|
||||
|
||||
import cn.hutool.core.util.StrUtil;
|
||||
import tech.easyflow.datacenter.connector.SqlDialect;
|
||||
|
||||
public class OracleSqlDialect implements SqlDialect {
|
||||
@Override
|
||||
public String quoteIdentifier(String identifier) {
|
||||
return '"' + identifier + '"';
|
||||
}
|
||||
|
||||
@Override
|
||||
public String qualifyTable(String namespace, String tableName) {
|
||||
if (StrUtil.isBlank(namespace)) {
|
||||
return quoteIdentifier(tableName);
|
||||
}
|
||||
return quoteIdentifier(namespace) + "." + quoteIdentifier(tableName);
|
||||
}
|
||||
|
||||
@Override
|
||||
public String buildPageSql(String baseSql, long pageNumber, long pageSize) {
|
||||
return baseSql + " OFFSET ? ROWS FETCH NEXT ? ROWS ONLY";
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,24 @@
|
||||
package tech.easyflow.datacenter.connector.dialect;
|
||||
|
||||
import cn.hutool.core.util.StrUtil;
|
||||
import tech.easyflow.datacenter.connector.SqlDialect;
|
||||
|
||||
public class PostgresqlSqlDialect implements SqlDialect {
|
||||
@Override
|
||||
public String quoteIdentifier(String identifier) {
|
||||
return '"' + identifier + '"';
|
||||
}
|
||||
|
||||
@Override
|
||||
public String qualifyTable(String namespace, String tableName) {
|
||||
if (StrUtil.isBlank(namespace)) {
|
||||
return quoteIdentifier(tableName);
|
||||
}
|
||||
return quoteIdentifier(namespace) + "." + quoteIdentifier(tableName);
|
||||
}
|
||||
|
||||
@Override
|
||||
public String buildPageSql(String baseSql, long pageNumber, long pageSize) {
|
||||
return baseSql + " OFFSET ? LIMIT ?";
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,22 @@
|
||||
package tech.easyflow.datacenter.connector.impl;
|
||||
|
||||
import org.springframework.stereotype.Component;
|
||||
import tech.easyflow.datacenter.connector.support.AbstractInternalTableConnector;
|
||||
import tech.easyflow.datacenter.meta.enums.DatacenterCapability;
|
||||
import tech.easyflow.datacenter.meta.enums.DatacenterSourceType;
|
||||
|
||||
import java.util.EnumSet;
|
||||
|
||||
@Component
|
||||
public class ExcelConnector extends AbstractInternalTableConnector {
|
||||
public ExcelConnector() {
|
||||
super(DatacenterSourceType.EXCEL, EnumSet.of(
|
||||
DatacenterCapability.TEST_CONNECTION,
|
||||
DatacenterCapability.BROWSE_METADATA,
|
||||
DatacenterCapability.READ_QUERY,
|
||||
DatacenterCapability.WRITE_MUTATION,
|
||||
DatacenterCapability.MATERIALIZE,
|
||||
DatacenterCapability.EXPORT
|
||||
));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,22 @@
|
||||
package tech.easyflow.datacenter.connector.impl;
|
||||
|
||||
import org.springframework.stereotype.Component;
|
||||
import tech.easyflow.datacenter.connector.support.AbstractInternalTableConnector;
|
||||
import tech.easyflow.datacenter.meta.enums.DatacenterCapability;
|
||||
import tech.easyflow.datacenter.meta.enums.DatacenterSourceType;
|
||||
|
||||
import java.util.EnumSet;
|
||||
|
||||
@Component
|
||||
public class ExcelMaterializedConnector extends AbstractInternalTableConnector {
|
||||
public ExcelMaterializedConnector() {
|
||||
super(DatacenterSourceType.EXCEL_MATERIALIZED, EnumSet.of(
|
||||
DatacenterCapability.TEST_CONNECTION,
|
||||
DatacenterCapability.BROWSE_METADATA,
|
||||
DatacenterCapability.READ_QUERY,
|
||||
DatacenterCapability.WRITE_MUTATION,
|
||||
DatacenterCapability.MATERIALIZE,
|
||||
DatacenterCapability.EXPORT
|
||||
));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,40 @@
|
||||
package tech.easyflow.datacenter.connector.impl;
|
||||
|
||||
import com.zaxxer.hikari.HikariDataSource;
|
||||
import org.springframework.stereotype.Component;
|
||||
import tech.easyflow.datacenter.connector.dialect.GaussdbSqlDialect;
|
||||
import tech.easyflow.datacenter.connector.support.AbstractJdbcConnector;
|
||||
import tech.easyflow.datacenter.connector.support.DatacenterDatasourceManager;
|
||||
import tech.easyflow.datacenter.meta.entity.DatacenterSource;
|
||||
import tech.easyflow.datacenter.meta.enums.DatacenterCapability;
|
||||
import tech.easyflow.datacenter.meta.enums.DatacenterSourceType;
|
||||
|
||||
import java.sql.Connection;
|
||||
import java.util.EnumSet;
|
||||
|
||||
@Component
|
||||
public class GaussdbNativeConnector extends AbstractJdbcConnector {
|
||||
|
||||
private final DatacenterDatasourceManager datasourceManager;
|
||||
|
||||
public GaussdbNativeConnector(DatacenterDatasourceManager datasourceManager) {
|
||||
super(DatacenterSourceType.GAUSSDB_NATIVE, new GaussdbSqlDialect(), EnumSet.of(
|
||||
DatacenterCapability.TEST_CONNECTION,
|
||||
DatacenterCapability.BROWSE_METADATA,
|
||||
DatacenterCapability.READ_QUERY
|
||||
));
|
||||
this.datasourceManager = datasourceManager;
|
||||
}
|
||||
|
||||
@Override
|
||||
protected <T> T withConnection(DatacenterSource source, boolean cacheable, JdbcCallback<T> callback) throws Exception {
|
||||
HikariDataSource dataSource = cacheable ? datasourceManager.getOrCreateExternalDatasource(source) : datasourceManager.createExternalDatasource(source);
|
||||
try (Connection connection = dataSource.getConnection()) {
|
||||
return callback.apply(connection);
|
||||
} finally {
|
||||
if (!cacheable || source.getId() == null) {
|
||||
dataSource.close();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,40 @@
|
||||
package tech.easyflow.datacenter.connector.impl;
|
||||
|
||||
import com.zaxxer.hikari.HikariDataSource;
|
||||
import org.springframework.stereotype.Component;
|
||||
import tech.easyflow.datacenter.connector.dialect.MysqlSqlDialect;
|
||||
import tech.easyflow.datacenter.connector.support.AbstractJdbcConnector;
|
||||
import tech.easyflow.datacenter.connector.support.DatacenterDatasourceManager;
|
||||
import tech.easyflow.datacenter.meta.entity.DatacenterSource;
|
||||
import tech.easyflow.datacenter.meta.enums.DatacenterCapability;
|
||||
import tech.easyflow.datacenter.meta.enums.DatacenterSourceType;
|
||||
|
||||
import java.sql.Connection;
|
||||
import java.util.EnumSet;
|
||||
|
||||
@Component
|
||||
public class Gbase8aConnector extends AbstractJdbcConnector {
|
||||
|
||||
private final DatacenterDatasourceManager datasourceManager;
|
||||
|
||||
public Gbase8aConnector(DatacenterDatasourceManager datasourceManager) {
|
||||
super(DatacenterSourceType.GBASE_8A, new MysqlSqlDialect(), EnumSet.of(
|
||||
DatacenterCapability.TEST_CONNECTION,
|
||||
DatacenterCapability.BROWSE_METADATA,
|
||||
DatacenterCapability.READ_QUERY
|
||||
));
|
||||
this.datasourceManager = datasourceManager;
|
||||
}
|
||||
|
||||
@Override
|
||||
protected <T> T withConnection(DatacenterSource source, boolean cacheable, JdbcCallback<T> callback) throws Exception {
|
||||
HikariDataSource dataSource = cacheable ? datasourceManager.getOrCreateExternalDatasource(source) : datasourceManager.createExternalDatasource(source);
|
||||
try (Connection connection = dataSource.getConnection()) {
|
||||
return callback.apply(connection);
|
||||
} finally {
|
||||
if (!cacheable || source.getId() == null) {
|
||||
dataSource.close();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,40 @@
|
||||
package tech.easyflow.datacenter.connector.impl;
|
||||
|
||||
import com.zaxxer.hikari.HikariDataSource;
|
||||
import org.springframework.stereotype.Component;
|
||||
import tech.easyflow.datacenter.connector.dialect.MysqlSqlDialect;
|
||||
import tech.easyflow.datacenter.connector.support.AbstractJdbcConnector;
|
||||
import tech.easyflow.datacenter.connector.support.DatacenterDatasourceManager;
|
||||
import tech.easyflow.datacenter.meta.entity.DatacenterSource;
|
||||
import tech.easyflow.datacenter.meta.enums.DatacenterCapability;
|
||||
import tech.easyflow.datacenter.meta.enums.DatacenterSourceType;
|
||||
|
||||
import java.sql.Connection;
|
||||
import java.util.EnumSet;
|
||||
|
||||
@Component
|
||||
public class Gbase8sConnector extends AbstractJdbcConnector {
|
||||
|
||||
private final DatacenterDatasourceManager datasourceManager;
|
||||
|
||||
public Gbase8sConnector(DatacenterDatasourceManager datasourceManager) {
|
||||
super(DatacenterSourceType.GBASE_8S, new MysqlSqlDialect(), EnumSet.of(
|
||||
DatacenterCapability.TEST_CONNECTION,
|
||||
DatacenterCapability.BROWSE_METADATA,
|
||||
DatacenterCapability.READ_QUERY
|
||||
));
|
||||
this.datasourceManager = datasourceManager;
|
||||
}
|
||||
|
||||
@Override
|
||||
protected <T> T withConnection(DatacenterSource source, boolean cacheable, JdbcCallback<T> callback) throws Exception {
|
||||
HikariDataSource dataSource = cacheable ? datasourceManager.getOrCreateExternalDatasource(source) : datasourceManager.createExternalDatasource(source);
|
||||
try (Connection connection = dataSource.getConnection()) {
|
||||
return callback.apply(connection);
|
||||
} finally {
|
||||
if (!cacheable || source.getId() == null) {
|
||||
dataSource.close();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,40 @@
|
||||
package tech.easyflow.datacenter.connector.impl;
|
||||
|
||||
import com.zaxxer.hikari.HikariDataSource;
|
||||
import org.springframework.stereotype.Component;
|
||||
import tech.easyflow.datacenter.connector.dialect.MysqlSqlDialect;
|
||||
import tech.easyflow.datacenter.connector.support.AbstractJdbcConnector;
|
||||
import tech.easyflow.datacenter.connector.support.DatacenterDatasourceManager;
|
||||
import tech.easyflow.datacenter.meta.entity.DatacenterSource;
|
||||
import tech.easyflow.datacenter.meta.enums.DatacenterCapability;
|
||||
import tech.easyflow.datacenter.meta.enums.DatacenterSourceType;
|
||||
|
||||
import java.sql.Connection;
|
||||
import java.util.EnumSet;
|
||||
|
||||
@Component
|
||||
public class MysqlConnector extends AbstractJdbcConnector {
|
||||
|
||||
private final DatacenterDatasourceManager datasourceManager;
|
||||
|
||||
public MysqlConnector(DatacenterDatasourceManager datasourceManager) {
|
||||
super(DatacenterSourceType.MYSQL, new MysqlSqlDialect(), EnumSet.of(
|
||||
DatacenterCapability.TEST_CONNECTION,
|
||||
DatacenterCapability.BROWSE_METADATA,
|
||||
DatacenterCapability.READ_QUERY
|
||||
));
|
||||
this.datasourceManager = datasourceManager;
|
||||
}
|
||||
|
||||
@Override
|
||||
protected <T> T withConnection(DatacenterSource source, boolean cacheable, JdbcCallback<T> callback) throws Exception {
|
||||
HikariDataSource dataSource = cacheable ? datasourceManager.getOrCreateExternalDatasource(source) : datasourceManager.createExternalDatasource(source);
|
||||
try (Connection connection = dataSource.getConnection()) {
|
||||
return callback.apply(connection);
|
||||
} finally {
|
||||
if (!cacheable || source.getId() == null) {
|
||||
dataSource.close();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,40 @@
|
||||
package tech.easyflow.datacenter.connector.impl;
|
||||
|
||||
import com.zaxxer.hikari.HikariDataSource;
|
||||
import org.springframework.stereotype.Component;
|
||||
import tech.easyflow.datacenter.connector.dialect.OracleSqlDialect;
|
||||
import tech.easyflow.datacenter.connector.support.AbstractJdbcConnector;
|
||||
import tech.easyflow.datacenter.connector.support.DatacenterDatasourceManager;
|
||||
import tech.easyflow.datacenter.meta.entity.DatacenterSource;
|
||||
import tech.easyflow.datacenter.meta.enums.DatacenterCapability;
|
||||
import tech.easyflow.datacenter.meta.enums.DatacenterSourceType;
|
||||
|
||||
import java.sql.Connection;
|
||||
import java.util.EnumSet;
|
||||
|
||||
@Component
|
||||
public class OracleConnector extends AbstractJdbcConnector {
|
||||
|
||||
private final DatacenterDatasourceManager datasourceManager;
|
||||
|
||||
public OracleConnector(DatacenterDatasourceManager datasourceManager) {
|
||||
super(DatacenterSourceType.ORACLE, new OracleSqlDialect(), EnumSet.of(
|
||||
DatacenterCapability.TEST_CONNECTION,
|
||||
DatacenterCapability.BROWSE_METADATA,
|
||||
DatacenterCapability.READ_QUERY
|
||||
));
|
||||
this.datasourceManager = datasourceManager;
|
||||
}
|
||||
|
||||
@Override
|
||||
protected <T> T withConnection(DatacenterSource source, boolean cacheable, JdbcCallback<T> callback) throws Exception {
|
||||
HikariDataSource dataSource = cacheable ? datasourceManager.getOrCreateExternalDatasource(source) : datasourceManager.createExternalDatasource(source);
|
||||
try (Connection connection = dataSource.getConnection()) {
|
||||
return callback.apply(connection);
|
||||
} finally {
|
||||
if (!cacheable || source.getId() == null) {
|
||||
dataSource.close();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,40 @@
|
||||
package tech.easyflow.datacenter.connector.impl;
|
||||
|
||||
import com.zaxxer.hikari.HikariDataSource;
|
||||
import org.springframework.stereotype.Component;
|
||||
import tech.easyflow.datacenter.connector.dialect.PostgresqlSqlDialect;
|
||||
import tech.easyflow.datacenter.connector.support.AbstractJdbcConnector;
|
||||
import tech.easyflow.datacenter.connector.support.DatacenterDatasourceManager;
|
||||
import tech.easyflow.datacenter.meta.entity.DatacenterSource;
|
||||
import tech.easyflow.datacenter.meta.enums.DatacenterCapability;
|
||||
import tech.easyflow.datacenter.meta.enums.DatacenterSourceType;
|
||||
|
||||
import java.sql.Connection;
|
||||
import java.util.EnumSet;
|
||||
|
||||
@Component
|
||||
public class PostgresqlConnector extends AbstractJdbcConnector {
|
||||
|
||||
private final DatacenterDatasourceManager datasourceManager;
|
||||
|
||||
public PostgresqlConnector(DatacenterDatasourceManager datasourceManager) {
|
||||
super(DatacenterSourceType.POSTGRESQL, new PostgresqlSqlDialect(), EnumSet.of(
|
||||
DatacenterCapability.TEST_CONNECTION,
|
||||
DatacenterCapability.BROWSE_METADATA,
|
||||
DatacenterCapability.READ_QUERY
|
||||
));
|
||||
this.datasourceManager = datasourceManager;
|
||||
}
|
||||
|
||||
@Override
|
||||
protected <T> T withConnection(DatacenterSource source, boolean cacheable, JdbcCallback<T> callback) throws Exception {
|
||||
HikariDataSource dataSource = cacheable ? datasourceManager.getOrCreateExternalDatasource(source) : datasourceManager.createExternalDatasource(source);
|
||||
try (Connection connection = dataSource.getConnection()) {
|
||||
return callback.apply(connection);
|
||||
} finally {
|
||||
if (!cacheable || source.getId() == null) {
|
||||
dataSource.close();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,135 @@
|
||||
package tech.easyflow.datacenter.connector.impl;
|
||||
|
||||
import cn.hutool.core.util.StrUtil;
|
||||
import com.alibaba.fastjson2.JSONObject;
|
||||
import com.mybatisflex.core.paginate.Page;
|
||||
import com.mybatisflex.core.row.Row;
|
||||
import org.springframework.stereotype.Component;
|
||||
import tech.easyflow.common.entity.LoginAccount;
|
||||
import tech.easyflow.common.web.exceptions.BusinessException;
|
||||
import tech.easyflow.datacenter.connector.dialect.MysqlSqlDialect;
|
||||
import tech.easyflow.datacenter.connector.support.AbstractJdbcConnector;
|
||||
import tech.easyflow.datacenter.entity.DatacenterTable;
|
||||
import tech.easyflow.datacenter.entity.DatacenterTableField;
|
||||
import tech.easyflow.datacenter.execution.model.DatacenterQueryRequest;
|
||||
import tech.easyflow.datacenter.meta.entity.DatacenterSource;
|
||||
import tech.easyflow.datacenter.meta.enums.DatacenterCapability;
|
||||
import tech.easyflow.datacenter.meta.enums.DatacenterSourceType;
|
||||
|
||||
import javax.sql.DataSource;
|
||||
import java.math.BigInteger;
|
||||
import java.sql.Connection;
|
||||
import java.sql.PreparedStatement;
|
||||
import java.util.ArrayList;
|
||||
import java.util.EnumSet;
|
||||
import java.util.List;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
@Component
|
||||
public class ProjectMysqlConnector extends AbstractJdbcConnector {
|
||||
|
||||
private final DataSource dataSource;
|
||||
private final MysqlSqlDialect dialect = new MysqlSqlDialect();
|
||||
|
||||
public ProjectMysqlConnector(DataSource dataSource) {
|
||||
super(DatacenterSourceType.PROJECT_MYSQL, new MysqlSqlDialect(), EnumSet.of(
|
||||
DatacenterCapability.TEST_CONNECTION,
|
||||
DatacenterCapability.BROWSE_METADATA,
|
||||
DatacenterCapability.READ_QUERY,
|
||||
DatacenterCapability.WRITE_MUTATION
|
||||
));
|
||||
this.dataSource = dataSource;
|
||||
}
|
||||
|
||||
@Override
|
||||
protected <T> T withConnection(DatacenterSource source, boolean cacheable, JdbcCallback<T> callback) throws Exception {
|
||||
try (Connection connection = dataSource.getConnection()) {
|
||||
return callback.apply(connection);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
protected boolean requiresJdbcUrl() {
|
||||
return false;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Page<Row> queryPage(DatacenterSource source, DatacenterTable table, DatacenterQueryRequest request) {
|
||||
return super.queryPage(source, table, request);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void saveRow(DatacenterSource source, DatacenterTable table, JSONObject data, LoginAccount account) {
|
||||
List<DatacenterTableField> writableFields = table.getFields().stream()
|
||||
.filter(field -> field.getWritable() == null || field.getWritable() == 1)
|
||||
.collect(Collectors.toList());
|
||||
Object id = data.get("id");
|
||||
try (Connection connection = dataSource.getConnection()) {
|
||||
if (id == null) {
|
||||
List<String> columns = new ArrayList<>();
|
||||
List<Object> values = new ArrayList<>();
|
||||
for (DatacenterTableField field : writableFields) {
|
||||
Object value = data.get(field.getFieldName());
|
||||
if (value != null) {
|
||||
columns.add(field.getFieldName());
|
||||
values.add(value);
|
||||
}
|
||||
}
|
||||
if (columns.isEmpty()) {
|
||||
throw new BusinessException("没有可写字段");
|
||||
}
|
||||
String sql = "INSERT INTO " + dialect.qualifyTable(source.getDatabaseName(), resolvePhysicalTableName(table))
|
||||
+ " (" + columns.stream().map(dialect::quoteIdentifier).collect(Collectors.joining(",")) + ") VALUES ("
|
||||
+ columns.stream().map(item -> "?").collect(Collectors.joining(",")) + ")";
|
||||
try (PreparedStatement statement = connection.prepareStatement(sql)) {
|
||||
for (int i = 0; i < values.size(); i++) {
|
||||
statement.setObject(i + 1, values.get(i));
|
||||
}
|
||||
statement.executeUpdate();
|
||||
}
|
||||
return;
|
||||
}
|
||||
List<String> setClauses = new ArrayList<>();
|
||||
List<Object> values = new ArrayList<>();
|
||||
for (DatacenterTableField field : writableFields) {
|
||||
if (!data.containsKey(field.getFieldName())) {
|
||||
continue;
|
||||
}
|
||||
setClauses.add(dialect.quoteIdentifier(field.getFieldName()) + " = ?");
|
||||
values.add(data.get(field.getFieldName()));
|
||||
}
|
||||
if (setClauses.isEmpty()) {
|
||||
return;
|
||||
}
|
||||
String sql = "UPDATE " + dialect.qualifyTable(source.getDatabaseName(), resolvePhysicalTableName(table))
|
||||
+ " SET " + String.join(",", setClauses)
|
||||
+ " WHERE " + dialect.quoteIdentifier("id") + " = ?";
|
||||
values.add(id);
|
||||
try (PreparedStatement statement = connection.prepareStatement(sql)) {
|
||||
for (int i = 0; i < values.size(); i++) {
|
||||
statement.setObject(i + 1, values.get(i));
|
||||
}
|
||||
statement.executeUpdate();
|
||||
}
|
||||
} catch (Exception ex) {
|
||||
throw new BusinessException("项目 MySQL 写入失败: " + ex.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void deleteRow(DatacenterSource source, DatacenterTable table, BigInteger id, LoginAccount account) {
|
||||
String sql = "DELETE FROM " + dialect.qualifyTable(source.getDatabaseName(), resolvePhysicalTableName(table))
|
||||
+ " WHERE " + dialect.quoteIdentifier("id") + " = ?";
|
||||
try (Connection connection = dataSource.getConnection(); PreparedStatement statement = connection.prepareStatement(sql)) {
|
||||
statement.setObject(1, id);
|
||||
statement.executeUpdate();
|
||||
} catch (Exception ex) {
|
||||
throw new BusinessException("项目 MySQL 删除失败: " + ex.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
protected String resolveCatalogName(DatacenterSource source, String requestedCatalogName) {
|
||||
return StrUtil.blankToDefault(requestedCatalogName, source.getDatabaseName());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,152 @@
|
||||
package tech.easyflow.datacenter.connector.support;
|
||||
|
||||
import cn.hutool.core.util.StrUtil;
|
||||
import com.alibaba.fastjson2.JSONObject;
|
||||
import com.mybatisflex.core.paginate.Page;
|
||||
import com.mybatisflex.core.query.QueryWrapper;
|
||||
import com.mybatisflex.core.row.Db;
|
||||
import com.mybatisflex.core.row.Row;
|
||||
import com.mybatisflex.core.row.RowKey;
|
||||
import org.springframework.util.CollectionUtils;
|
||||
import tech.easyflow.common.entity.LoginAccount;
|
||||
import tech.easyflow.common.web.exceptions.BusinessException;
|
||||
import tech.easyflow.datacenter.connector.DatacenterConnector;
|
||||
import tech.easyflow.datacenter.entity.DatacenterTable;
|
||||
import tech.easyflow.datacenter.entity.DatacenterTableField;
|
||||
import tech.easyflow.datacenter.execution.model.DatacenterConnectionTestResult;
|
||||
import tech.easyflow.datacenter.execution.model.DatacenterQueryRequest;
|
||||
import tech.easyflow.datacenter.meta.entity.DatacenterSource;
|
||||
import tech.easyflow.datacenter.meta.enums.DatacenterCapability;
|
||||
import tech.easyflow.datacenter.meta.enums.DatacenterSourceType;
|
||||
import tech.easyflow.datacenter.meta.model.DatacenterCatalogMeta;
|
||||
import tech.easyflow.datacenter.meta.model.DatacenterTableDetailMeta;
|
||||
|
||||
import java.math.BigDecimal;
|
||||
import java.math.BigInteger;
|
||||
import java.util.*;
|
||||
|
||||
public abstract class AbstractInternalTableConnector implements DatacenterConnector {
|
||||
|
||||
private final DatacenterSourceType sourceType;
|
||||
private final Set<DatacenterCapability> capabilities;
|
||||
|
||||
protected AbstractInternalTableConnector(DatacenterSourceType sourceType, Set<DatacenterCapability> capabilities) {
|
||||
this.sourceType = sourceType;
|
||||
this.capabilities = capabilities;
|
||||
}
|
||||
|
||||
@Override
|
||||
public DatacenterSourceType getSourceType() {
|
||||
return sourceType;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Set<DatacenterCapability> getCapabilities() {
|
||||
return capabilities;
|
||||
}
|
||||
|
||||
@Override
|
||||
public DatacenterConnectionTestResult testConnection(DatacenterSource source) {
|
||||
DatacenterConnectionTestResult result = new DatacenterConnectionTestResult();
|
||||
result.setSuccess(true);
|
||||
result.setMessage("连接成功");
|
||||
result.setCapabilities(capabilities.stream().map(Enum::name).toList());
|
||||
result.setDetails(Map.of("sourceType", sourceType.name()));
|
||||
return result;
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<DatacenterCatalogMeta> listCatalogs(DatacenterSource source) {
|
||||
return Collections.emptyList();
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<DatacenterTable> listTables(DatacenterSource source, String catalogName) {
|
||||
return Collections.emptyList();
|
||||
}
|
||||
|
||||
@Override
|
||||
public DatacenterTableDetailMeta getTableDetail(DatacenterSource source, String catalogName, String tableName) {
|
||||
throw new BusinessException("内部数据源请从元数据注册表读取表结构");
|
||||
}
|
||||
|
||||
@Override
|
||||
public Page<Row> queryPage(DatacenterSource source, DatacenterTable table, DatacenterQueryRequest request) {
|
||||
String actualTable = resolveTableName(table);
|
||||
QueryWrapper wrapper = QueryWrapper.create();
|
||||
if (StrUtil.isNotBlank(request.getWhere())) {
|
||||
wrapper.where(request.getWhere());
|
||||
}
|
||||
long count = Db.selectCountByQuery(actualTable, wrapper);
|
||||
if (count == 0) {
|
||||
return new Page<>(new ArrayList<>(), request.getPageNumber(), request.getPageSize(), count);
|
||||
}
|
||||
Page<Row> page = Db.paginate(actualTable, new Page<>(request.getPageNumber(), request.getPageSize(), count), wrapper);
|
||||
normalizeRows(page.getRecords());
|
||||
return page;
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<Row> queryBySql(DatacenterSource source, String sql) {
|
||||
List<Row> rows = Db.selectListBySql(sql);
|
||||
normalizeRows(rows);
|
||||
return rows;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void saveRow(DatacenterSource source, DatacenterTable table, JSONObject data, LoginAccount account) {
|
||||
List<DatacenterTableField> fields = table.getFields();
|
||||
if (CollectionUtils.isEmpty(fields)) {
|
||||
throw new BusinessException("数据集字段为空,无法写入");
|
||||
}
|
||||
String actualTable = resolveTableName(table);
|
||||
Object id = data.get("id");
|
||||
if (id == null) {
|
||||
Row row = Row.ofKey(RowKey.SNOW_FLAKE_ID);
|
||||
row.put("dept_id", account.getDeptId());
|
||||
row.put("tenant_id", account.getTenantId());
|
||||
row.put("created", new Date());
|
||||
row.put("created_by", account.getId());
|
||||
row.put("modified", new Date());
|
||||
row.put("modified_by", account.getId());
|
||||
row.put("remark", data.get("remark"));
|
||||
for (DatacenterTableField field : fields) {
|
||||
row.put(field.getFieldName(), data.get(field.getFieldName()));
|
||||
}
|
||||
Db.insert(actualTable, row);
|
||||
return;
|
||||
}
|
||||
Row row = Row.ofKey("id", id);
|
||||
row.put("modified", new Date());
|
||||
row.put("modified_by", account.getId());
|
||||
for (DatacenterTableField field : fields) {
|
||||
row.put(field.getFieldName(), data.get(field.getFieldName()));
|
||||
}
|
||||
Db.updateById(actualTable, row);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void deleteRow(DatacenterSource source, DatacenterTable table, BigInteger id, LoginAccount account) {
|
||||
Db.deleteById(resolveTableName(table), Row.ofKey("id", id));
|
||||
}
|
||||
|
||||
protected String resolveTableName(DatacenterTable table) {
|
||||
return StrUtil.blankToDefault(table.getMaterializedTable(), table.getActualTable());
|
||||
}
|
||||
|
||||
private void normalizeRows(List<Row> records) {
|
||||
for (Row record : records) {
|
||||
Map<String, Object> converted = new LinkedHashMap<>();
|
||||
for (Map.Entry<String, Object> entry : record.entrySet()) {
|
||||
Object value = entry.getValue();
|
||||
if (value instanceof BigInteger || value instanceof BigDecimal || value instanceof Long) {
|
||||
converted.put(entry.getKey(), value.toString());
|
||||
} else {
|
||||
converted.put(entry.getKey(), value);
|
||||
}
|
||||
}
|
||||
record.clear();
|
||||
record.putAll(converted);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,523 @@
|
||||
package tech.easyflow.datacenter.connector.support;
|
||||
|
||||
import cn.hutool.core.util.StrUtil;
|
||||
import com.alibaba.fastjson2.JSONObject;
|
||||
import com.mybatisflex.core.paginate.Page;
|
||||
import com.mybatisflex.core.row.Row;
|
||||
import org.springframework.util.CollectionUtils;
|
||||
import tech.easyflow.common.constant.enums.EnumFieldType;
|
||||
import tech.easyflow.common.entity.LoginAccount;
|
||||
import tech.easyflow.common.web.exceptions.BusinessException;
|
||||
import tech.easyflow.datacenter.connector.DatacenterConnector;
|
||||
import tech.easyflow.datacenter.connector.SqlDialect;
|
||||
import tech.easyflow.datacenter.entity.DatacenterTable;
|
||||
import tech.easyflow.datacenter.entity.DatacenterTableField;
|
||||
import tech.easyflow.datacenter.execution.model.DatacenterConnectionTestResult;
|
||||
import tech.easyflow.datacenter.execution.model.DatacenterQueryFilter;
|
||||
import tech.easyflow.datacenter.execution.model.DatacenterQueryRequest;
|
||||
import tech.easyflow.datacenter.execution.model.DatacenterQuerySort;
|
||||
import tech.easyflow.datacenter.meta.entity.DatacenterSource;
|
||||
import tech.easyflow.datacenter.meta.enums.DatacenterCapability;
|
||||
import tech.easyflow.datacenter.meta.enums.DatacenterConnectionErrorCode;
|
||||
import tech.easyflow.datacenter.meta.enums.DatacenterSourceType;
|
||||
import tech.easyflow.datacenter.meta.enums.DatacenterTableKind;
|
||||
import tech.easyflow.datacenter.meta.model.DatacenterCatalogMeta;
|
||||
import tech.easyflow.datacenter.meta.model.DatacenterTableDetailMeta;
|
||||
|
||||
import java.math.BigDecimal;
|
||||
import java.math.BigInteger;
|
||||
import java.sql.*;
|
||||
import java.util.*;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
public abstract class AbstractJdbcConnector implements DatacenterConnector {
|
||||
|
||||
private final DatacenterSourceType sourceType;
|
||||
private final SqlDialect sqlDialect;
|
||||
private final Set<DatacenterCapability> capabilities;
|
||||
|
||||
protected AbstractJdbcConnector(DatacenterSourceType sourceType, SqlDialect sqlDialect, Set<DatacenterCapability> capabilities) {
|
||||
this.sourceType = sourceType;
|
||||
this.sqlDialect = sqlDialect;
|
||||
this.capabilities = capabilities;
|
||||
}
|
||||
|
||||
@Override
|
||||
public DatacenterSourceType getSourceType() {
|
||||
return sourceType;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Set<DatacenterCapability> getCapabilities() {
|
||||
return capabilities;
|
||||
}
|
||||
|
||||
protected abstract <T> T withConnection(DatacenterSource source, boolean cacheable, JdbcCallback<T> callback) throws Exception;
|
||||
|
||||
@Override
|
||||
public DatacenterConnectionTestResult testConnection(DatacenterSource source) {
|
||||
DatacenterConnectionTestResult result = new DatacenterConnectionTestResult();
|
||||
result.setCapabilities(capabilities.stream().map(Enum::name).toList());
|
||||
if (StrUtil.isBlank(source.getJdbcUrl()) && requiresJdbcUrl()) {
|
||||
result.setSuccess(false);
|
||||
result.setErrorCode(DatacenterConnectionErrorCode.INVALID_ARGUMENT.name());
|
||||
result.setMessage("缺少 JDBC URL");
|
||||
return result;
|
||||
}
|
||||
try {
|
||||
if (StrUtil.isNotBlank(source.getDriverClassName())) {
|
||||
Class.forName(source.getDriverClassName());
|
||||
}
|
||||
Map<String, Object> details = withConnection(source, false, connection -> {
|
||||
DatabaseMetaData metaData = connection.getMetaData();
|
||||
Map<String, Object> map = new LinkedHashMap<>();
|
||||
map.put("databaseProductName", metaData.getDatabaseProductName());
|
||||
map.put("databaseProductVersion", metaData.getDatabaseProductVersion());
|
||||
map.put("url", metaData.getURL());
|
||||
return map;
|
||||
});
|
||||
result.setSuccess(true);
|
||||
result.setMessage("连接成功");
|
||||
result.setDetails(details);
|
||||
return result;
|
||||
} catch (ClassNotFoundException ex) {
|
||||
result.setSuccess(false);
|
||||
result.setErrorCode(DatacenterConnectionErrorCode.DRIVER_NOT_FOUND.name());
|
||||
result.setMessage(ex.getMessage());
|
||||
return result;
|
||||
} catch (SQLException ex) {
|
||||
result.setSuccess(false);
|
||||
result.setErrorCode(mapSqlError(ex));
|
||||
result.setMessage(ex.getMessage());
|
||||
return result;
|
||||
} catch (Exception ex) {
|
||||
result.setSuccess(false);
|
||||
result.setErrorCode(DatacenterConnectionErrorCode.UNKNOWN_ERROR.name());
|
||||
result.setMessage(ex.getMessage());
|
||||
return result;
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<DatacenterCatalogMeta> listCatalogs(DatacenterSource source) {
|
||||
try {
|
||||
return withConnection(source, true, connection -> {
|
||||
List<DatacenterCatalogMeta> result = new ArrayList<>();
|
||||
DatabaseMetaData metaData = connection.getMetaData();
|
||||
try (ResultSet catalogs = metaData.getCatalogs()) {
|
||||
while (catalogs.next()) {
|
||||
String name = catalogs.getString("TABLE_CAT");
|
||||
if (StrUtil.isBlank(name)) {
|
||||
continue;
|
||||
}
|
||||
DatacenterCatalogMeta meta = new DatacenterCatalogMeta();
|
||||
meta.setSourceId(source.getId());
|
||||
meta.setCatalogName(name);
|
||||
meta.setCatalogType("CATALOG");
|
||||
result.add(meta);
|
||||
}
|
||||
}
|
||||
try (ResultSet schemas = metaData.getSchemas()) {
|
||||
while (schemas.next()) {
|
||||
String name = schemas.getString("TABLE_SCHEM");
|
||||
if (StrUtil.isBlank(name) || containsCatalog(result, name)) {
|
||||
continue;
|
||||
}
|
||||
DatacenterCatalogMeta meta = new DatacenterCatalogMeta();
|
||||
meta.setSourceId(source.getId());
|
||||
meta.setCatalogName(name);
|
||||
meta.setCatalogType("SCHEMA");
|
||||
result.add(meta);
|
||||
}
|
||||
}
|
||||
result = filterConfiguredCatalogs(source, result);
|
||||
if (result.isEmpty()) {
|
||||
String fallback = resolveCatalogName(source, null);
|
||||
if (StrUtil.isNotBlank(fallback)) {
|
||||
DatacenterCatalogMeta meta = new DatacenterCatalogMeta();
|
||||
meta.setSourceId(source.getId());
|
||||
meta.setCatalogName(fallback);
|
||||
meta.setCatalogType("DEFAULT");
|
||||
result.add(meta);
|
||||
}
|
||||
}
|
||||
return result;
|
||||
});
|
||||
} catch (Exception ex) {
|
||||
throw DatacenterConnectorExceptionSupport.wrapAccessException("读取目录失败", ex);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<DatacenterTable> listTables(DatacenterSource source, String catalogName) {
|
||||
try {
|
||||
return withConnection(source, true, connection -> {
|
||||
DatabaseMetaData metaData = connection.getMetaData();
|
||||
List<DatacenterTable> tables = new ArrayList<>();
|
||||
try (ResultSet resultSet = metaData.getTables(resolveCatalogArgument(source, catalogName), resolveSchemaArgument(source, catalogName), "%", new String[]{"TABLE", "VIEW"})) {
|
||||
while (resultSet.next()) {
|
||||
DatacenterTable table = new DatacenterTable();
|
||||
table.setSourceId(source.getId());
|
||||
table.setTableName(resultSet.getString("TABLE_NAME"));
|
||||
table.setTableDesc(resultSet.getString("REMARKS"));
|
||||
table.setActualTable(resultSet.getString("TABLE_NAME"));
|
||||
table.setMaterializedTable(resultSet.getString("TABLE_NAME"));
|
||||
table.setAccessMode(capabilities.contains(DatacenterCapability.WRITE_MUTATION) ? "READ_WRITE" : "READ_ONLY");
|
||||
table.setTableKind(resolveTableKind(resultSet.getString("TABLE_TYPE")).name());
|
||||
table.setCapabilitiesJson(Map.of("capabilities", capabilities.stream().map(Enum::name).toList()));
|
||||
tables.add(table);
|
||||
}
|
||||
}
|
||||
return tables;
|
||||
});
|
||||
} catch (Exception ex) {
|
||||
throw DatacenterConnectorExceptionSupport.wrapAccessException("读取表列表失败", ex);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public DatacenterTableDetailMeta getTableDetail(DatacenterSource source, String catalogName, String tableName) {
|
||||
try {
|
||||
return withConnection(source, true, connection -> {
|
||||
DatabaseMetaData metaData = connection.getMetaData();
|
||||
DatacenterTableDetailMeta detail = new DatacenterTableDetailMeta();
|
||||
DatacenterTable table = new DatacenterTable();
|
||||
table.setSourceId(source.getId());
|
||||
table.setTableName(tableName);
|
||||
table.setActualTable(tableName);
|
||||
table.setMaterializedTable(tableName);
|
||||
table.setAccessMode(capabilities.contains(DatacenterCapability.WRITE_MUTATION) ? "READ_WRITE" : "READ_ONLY");
|
||||
table.setTableKind(DatacenterTableKind.EXTERNAL_TABLE.name());
|
||||
table.setCapabilitiesJson(Map.of("capabilities", capabilities.stream().map(Enum::name).toList()));
|
||||
detail.setTable(table);
|
||||
|
||||
try (ResultSet tableSet = metaData.getTables(
|
||||
resolveCatalogArgument(source, catalogName),
|
||||
resolveSchemaArgument(source, catalogName),
|
||||
tableName,
|
||||
new String[]{"TABLE", "VIEW"})) {
|
||||
while (tableSet.next()) {
|
||||
String currentTableName = tableSet.getString("TABLE_NAME");
|
||||
if (!matchesTableName(currentTableName, tableName)) {
|
||||
continue;
|
||||
}
|
||||
table.setTableDesc(tableSet.getString("REMARKS"));
|
||||
table.setTableKind(resolveTableKind(tableSet.getString("TABLE_TYPE")).name());
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
Set<String> primaryKeys = new HashSet<>();
|
||||
try (ResultSet pkSet = metaData.getPrimaryKeys(resolveCatalogArgument(source, catalogName), resolveSchemaArgument(source, catalogName), tableName)) {
|
||||
while (pkSet.next()) {
|
||||
primaryKeys.add(pkSet.getString("COLUMN_NAME"));
|
||||
}
|
||||
}
|
||||
|
||||
List<DatacenterTableField> fields = new ArrayList<>();
|
||||
try (ResultSet columns = metaData.getColumns(resolveCatalogArgument(source, catalogName), resolveSchemaArgument(source, catalogName), tableName, "%")) {
|
||||
while (columns.next()) {
|
||||
DatacenterTableField field = new DatacenterTableField();
|
||||
field.setFieldName(columns.getString("COLUMN_NAME"));
|
||||
field.setSourceColumnName(columns.getString("COLUMN_NAME"));
|
||||
field.setFieldDesc(columns.getString("REMARKS"));
|
||||
field.setJdbcType(columns.getString("TYPE_NAME"));
|
||||
field.setPrecision(columns.getInt("COLUMN_SIZE"));
|
||||
field.setScale(columns.getInt("DECIMAL_DIGITS"));
|
||||
field.setRequired(columns.getInt("NULLABLE") == DatabaseMetaData.columnNoNulls ? 1 : 0);
|
||||
field.setQueryable(1);
|
||||
field.setSortable(1);
|
||||
field.setWritable(capabilities.contains(DatacenterCapability.WRITE_MUTATION) ? 1 : 0);
|
||||
field.setIndexed(primaryKeys.contains(field.getFieldName()) ? 1 : 0);
|
||||
field.setFieldType(mapFieldType(columns.getInt("DATA_TYPE")));
|
||||
fields.add(field);
|
||||
}
|
||||
}
|
||||
detail.setFields(fields);
|
||||
return detail;
|
||||
});
|
||||
} catch (Exception ex) {
|
||||
throw DatacenterConnectorExceptionSupport.wrapAccessException("读取表详情失败", ex);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public Page<Row> queryPage(DatacenterSource source, DatacenterTable table, DatacenterQueryRequest request) {
|
||||
if (!capabilities.contains(DatacenterCapability.READ_QUERY)) {
|
||||
throw new BusinessException("当前数据源暂不支持查询");
|
||||
}
|
||||
try {
|
||||
return withConnection(source, true, connection -> doQueryPage(connection, source, table, request));
|
||||
} catch (Exception ex) {
|
||||
throw DatacenterConnectorExceptionSupport.wrapAccessException("查询失败", ex);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<Row> queryBySql(DatacenterSource source, String sql) {
|
||||
if (!capabilities.contains(DatacenterCapability.READ_QUERY)) {
|
||||
throw new BusinessException("当前数据源暂不支持查询");
|
||||
}
|
||||
try {
|
||||
return withConnection(source, true, connection -> doQueryBySql(connection, sql));
|
||||
} catch (Exception ex) {
|
||||
throw DatacenterConnectorExceptionSupport.wrapAccessException("SQL 查询失败", ex);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void saveRow(DatacenterSource source, DatacenterTable table, JSONObject data, LoginAccount account) {
|
||||
throw new BusinessException("当前数据源不支持写入");
|
||||
}
|
||||
|
||||
@Override
|
||||
public void deleteRow(DatacenterSource source, DatacenterTable table, BigInteger id, LoginAccount account) {
|
||||
throw new BusinessException("当前数据源不支持删除");
|
||||
}
|
||||
|
||||
protected boolean requiresJdbcUrl() {
|
||||
return true;
|
||||
}
|
||||
|
||||
protected Page<Row> doQueryPage(Connection connection, DatacenterSource source, DatacenterTable table, DatacenterQueryRequest request) throws SQLException {
|
||||
List<Object> params = new ArrayList<>();
|
||||
String selectColumns = CollectionUtils.isEmpty(request.getSelectedColumns())
|
||||
? "*"
|
||||
: request.getSelectedColumns().stream().map(sqlDialect::quoteIdentifier).collect(Collectors.joining(", "));
|
||||
String qualifiedTable = sqlDialect.qualifyTable(resolveCatalogName(source, request.getDatasetRef() == null ? null : request.getDatasetRef().getCatalogName()), resolvePhysicalTableName(table));
|
||||
StringBuilder whereClause = new StringBuilder();
|
||||
if (StrUtil.isNotBlank(request.getWhere())) {
|
||||
whereClause.append(" WHERE ").append(request.getWhere());
|
||||
} else if (!CollectionUtils.isEmpty(request.getFilters())) {
|
||||
whereClause.append(" WHERE 1=1 ");
|
||||
for (DatacenterQueryFilter filter : request.getFilters()) {
|
||||
appendFilter(whereClause, params, filter);
|
||||
}
|
||||
}
|
||||
String orderClause = buildOrderClause(request.getSorts());
|
||||
String baseSql = "SELECT " + selectColumns + " FROM " + qualifiedTable + whereClause + orderClause;
|
||||
String countSql = "SELECT COUNT(1) FROM " + qualifiedTable + whereClause;
|
||||
long total = queryCount(connection, countSql, params);
|
||||
if (total == 0L) {
|
||||
return new Page<>(new ArrayList<>(), request.getPageNumber(), request.getPageSize(), total);
|
||||
}
|
||||
String pageSql = sqlDialect.buildPageSql(baseSql, request.getPageNumber(), request.getPageSize());
|
||||
List<Object> pageParams = new ArrayList<>(params);
|
||||
if (pageSql.contains("FETCH NEXT") || pageSql.toLowerCase(Locale.ROOT).contains("limit")) {
|
||||
pageParams.add((request.getPageNumber() - 1) * request.getPageSize());
|
||||
pageParams.add(request.getPageSize());
|
||||
}
|
||||
List<Row> records = new ArrayList<>();
|
||||
try (PreparedStatement statement = connection.prepareStatement(pageSql)) {
|
||||
bindParameters(statement, pageParams);
|
||||
try (ResultSet resultSet = statement.executeQuery()) {
|
||||
ResultSetMetaData metaData = resultSet.getMetaData();
|
||||
while (resultSet.next()) {
|
||||
Row row = new Row();
|
||||
for (int i = 1; i <= metaData.getColumnCount(); i++) {
|
||||
String columnLabel = metaData.getColumnLabel(i);
|
||||
row.put(columnLabel, normalizeValue(resultSet.getObject(i)));
|
||||
}
|
||||
records.add(row);
|
||||
}
|
||||
}
|
||||
}
|
||||
return new Page<>(records, request.getPageNumber(), request.getPageSize(), total);
|
||||
}
|
||||
|
||||
protected String resolvePhysicalTableName(DatacenterTable table) {
|
||||
return StrUtil.blankToDefault(table.getActualTable(), table.getTableName());
|
||||
}
|
||||
|
||||
protected List<Row> doQueryBySql(Connection connection, String sql) throws SQLException {
|
||||
try (PreparedStatement statement = connection.prepareStatement(sql);
|
||||
ResultSet resultSet = statement.executeQuery()) {
|
||||
return readRows(resultSet);
|
||||
}
|
||||
}
|
||||
|
||||
protected String resolveCatalogName(DatacenterSource source, String requestedCatalogName) {
|
||||
if (StrUtil.isNotBlank(requestedCatalogName)) {
|
||||
return requestedCatalogName;
|
||||
}
|
||||
if (usesCatalogNamespace()) {
|
||||
return source.getDatabaseName();
|
||||
}
|
||||
return source.getSchemaName();
|
||||
}
|
||||
|
||||
protected List<Row> readRows(ResultSet resultSet) throws SQLException {
|
||||
List<Row> records = new ArrayList<>();
|
||||
ResultSetMetaData metaData = resultSet.getMetaData();
|
||||
while (resultSet.next()) {
|
||||
Row row = new Row();
|
||||
for (int i = 1; i <= metaData.getColumnCount(); i++) {
|
||||
String columnLabel = metaData.getColumnLabel(i);
|
||||
row.put(columnLabel, normalizeValue(resultSet.getObject(i)));
|
||||
}
|
||||
records.add(row);
|
||||
}
|
||||
return records;
|
||||
}
|
||||
|
||||
protected String resolveCatalogArgument(DatacenterSource source, String catalogName) {
|
||||
return usesCatalogNamespace() ? resolveCatalogName(source, catalogName) : source.getDatabaseName();
|
||||
}
|
||||
|
||||
protected String resolveSchemaArgument(DatacenterSource source, String catalogName) {
|
||||
return usesCatalogNamespace() ? source.getSchemaName() : resolveCatalogName(source, catalogName);
|
||||
}
|
||||
|
||||
protected boolean usesCatalogNamespace() {
|
||||
return sourceType == DatacenterSourceType.MYSQL
|
||||
|| sourceType == DatacenterSourceType.PROJECT_MYSQL
|
||||
|| sourceType == DatacenterSourceType.GBASE_8A
|
||||
|| sourceType == DatacenterSourceType.GBASE_8S;
|
||||
}
|
||||
|
||||
private List<DatacenterCatalogMeta> filterConfiguredCatalogs(DatacenterSource source, List<DatacenterCatalogMeta> items) {
|
||||
if (items == null || items.isEmpty() || source == null) {
|
||||
return items;
|
||||
}
|
||||
String configuredName = usesCatalogNamespace()
|
||||
? StrUtil.trimToNull(source.getDatabaseName())
|
||||
: StrUtil.trimToNull(source.getSchemaName());
|
||||
if (StrUtil.isBlank(configuredName)) {
|
||||
return items;
|
||||
}
|
||||
List<DatacenterCatalogMeta> matched = items.stream()
|
||||
.filter(item -> configuredName.equalsIgnoreCase(item.getCatalogName()))
|
||||
.collect(Collectors.toList());
|
||||
return matched.isEmpty() ? items : matched;
|
||||
}
|
||||
|
||||
private boolean containsCatalog(List<DatacenterCatalogMeta> items, String catalogName) {
|
||||
return items.stream().anyMatch(item -> catalogName.equalsIgnoreCase(item.getCatalogName()));
|
||||
}
|
||||
|
||||
private DatacenterTableKind resolveTableKind(String tableType) {
|
||||
return "VIEW".equalsIgnoreCase(tableType) ? DatacenterTableKind.EXTERNAL_VIEW : DatacenterTableKind.EXTERNAL_TABLE;
|
||||
}
|
||||
|
||||
private boolean matchesTableName(String currentTableName, String targetTableName) {
|
||||
if (currentTableName == null || targetTableName == null) {
|
||||
return false;
|
||||
}
|
||||
return currentTableName.equals(targetTableName)
|
||||
|| currentTableName.equalsIgnoreCase(targetTableName);
|
||||
}
|
||||
|
||||
private Integer mapFieldType(int jdbcType) {
|
||||
return switch (jdbcType) {
|
||||
case Types.INTEGER, Types.TINYINT, Types.SMALLINT, Types.BIGINT -> EnumFieldType.INTEGER.getCode();
|
||||
case Types.FLOAT, Types.DOUBLE, Types.REAL, Types.NUMERIC, Types.DECIMAL -> EnumFieldType.NUMBER.getCode();
|
||||
case Types.TIMESTAMP, Types.DATE, Types.TIME -> EnumFieldType.TIME.getCode();
|
||||
case Types.BOOLEAN, Types.BIT -> EnumFieldType.BOOLEAN.getCode();
|
||||
default -> EnumFieldType.STRING.getCode();
|
||||
};
|
||||
}
|
||||
|
||||
private void appendFilter(StringBuilder sql, List<Object> params, DatacenterQueryFilter filter) {
|
||||
String operator = StrUtil.blankToDefault(filter.getOperator(), "EQ").toUpperCase(Locale.ROOT);
|
||||
String column = sqlDialect.quoteIdentifier(filter.getColumn());
|
||||
switch (operator) {
|
||||
case "EQ" -> {
|
||||
sql.append(" AND ").append(column).append(" = ?");
|
||||
params.add(filter.getValue());
|
||||
}
|
||||
case "LIKE" -> {
|
||||
sql.append(" AND ").append(column).append(" LIKE ?");
|
||||
params.add("%" + filter.getValue() + "%");
|
||||
}
|
||||
case "GT" -> {
|
||||
sql.append(" AND ").append(column).append(" > ?");
|
||||
params.add(filter.getValue());
|
||||
}
|
||||
case "GTE" -> {
|
||||
sql.append(" AND ").append(column).append(" >= ?");
|
||||
params.add(filter.getValue());
|
||||
}
|
||||
case "LT" -> {
|
||||
sql.append(" AND ").append(column).append(" < ?");
|
||||
params.add(filter.getValue());
|
||||
}
|
||||
case "LTE" -> {
|
||||
sql.append(" AND ").append(column).append(" <= ?");
|
||||
params.add(filter.getValue());
|
||||
}
|
||||
case "IN" -> {
|
||||
List<Object> values = filter.getValues() == null ? List.of() : filter.getValues();
|
||||
if (values.isEmpty()) {
|
||||
sql.append(" AND 1 = 0");
|
||||
} else {
|
||||
sql.append(" AND ").append(column).append(" IN (");
|
||||
sql.append(values.stream().map(item -> "?").collect(Collectors.joining(",")));
|
||||
sql.append(")");
|
||||
params.addAll(values);
|
||||
}
|
||||
}
|
||||
case "IS_NULL" -> sql.append(" AND ").append(column).append(" IS NULL");
|
||||
default -> throw new BusinessException("不支持的过滤操作: " + operator);
|
||||
}
|
||||
}
|
||||
|
||||
private String buildOrderClause(List<DatacenterQuerySort> sorts) {
|
||||
if (CollectionUtils.isEmpty(sorts)) {
|
||||
return "";
|
||||
}
|
||||
return " ORDER BY " + sorts.stream()
|
||||
.map(sort -> sqlDialect.quoteIdentifier(sort.getColumn()) + " " + ("DESC".equalsIgnoreCase(sort.getDirection()) ? "DESC" : "ASC"))
|
||||
.collect(Collectors.joining(", "));
|
||||
}
|
||||
|
||||
private long queryCount(Connection connection, String sql, List<Object> params) throws SQLException {
|
||||
try (PreparedStatement statement = connection.prepareStatement(sql)) {
|
||||
bindParameters(statement, params);
|
||||
try (ResultSet resultSet = statement.executeQuery()) {
|
||||
if (resultSet.next()) {
|
||||
return resultSet.getLong(1);
|
||||
}
|
||||
return 0L;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void bindParameters(PreparedStatement statement, List<Object> params) throws SQLException {
|
||||
for (int i = 0; i < params.size(); i++) {
|
||||
statement.setObject(i + 1, params.get(i));
|
||||
}
|
||||
}
|
||||
|
||||
private Object normalizeValue(Object value) {
|
||||
if (value instanceof BigDecimal || value instanceof BigInteger || value instanceof Long) {
|
||||
return value.toString();
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
private String mapSqlError(SQLException ex) {
|
||||
String state = ex.getSQLState();
|
||||
String message = ex.getMessage() == null ? "" : ex.getMessage().toLowerCase(Locale.ROOT);
|
||||
if (message.contains("access denied") || message.contains("password") || message.contains("authentication")) {
|
||||
return DatacenterConnectionErrorCode.AUTH_FAILED.name();
|
||||
}
|
||||
if (message.contains("unknown database") || message.contains("database does not exist")) {
|
||||
return DatacenterConnectionErrorCode.DATABASE_NOT_FOUND.name();
|
||||
}
|
||||
if (message.contains("schema") && message.contains("does not exist")) {
|
||||
return DatacenterConnectionErrorCode.SCHEMA_NOT_FOUND.name();
|
||||
}
|
||||
if (message.contains("permission denied") || message.contains("insufficient privilege")) {
|
||||
return DatacenterConnectionErrorCode.PERMISSION_DENIED.name();
|
||||
}
|
||||
if (state != null && state.startsWith("08")) {
|
||||
return DatacenterConnectionErrorCode.NETWORK_UNREACHABLE.name();
|
||||
}
|
||||
return DatacenterConnectionErrorCode.UNKNOWN_ERROR.name();
|
||||
}
|
||||
|
||||
@FunctionalInterface
|
||||
protected interface JdbcCallback<T> {
|
||||
T apply(Connection connection) throws Exception;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,80 @@
|
||||
package tech.easyflow.datacenter.connector.support;
|
||||
|
||||
import tech.easyflow.common.web.exceptions.BusinessException;
|
||||
|
||||
import java.net.ConnectException;
|
||||
import java.net.NoRouteToHostException;
|
||||
import java.net.SocketException;
|
||||
import java.net.SocketTimeoutException;
|
||||
import java.net.UnknownHostException;
|
||||
import java.sql.SQLException;
|
||||
import java.util.Locale;
|
||||
|
||||
public final class DatacenterConnectorExceptionSupport {
|
||||
|
||||
public static final String SOURCE_UNAVAILABLE_MESSAGE = "当前连接不可用,请检查连接配置后重试";
|
||||
|
||||
private DatacenterConnectorExceptionSupport() {
|
||||
}
|
||||
|
||||
public static BusinessException wrapAccessException(String fallbackMessage, Exception ex) {
|
||||
if (ex instanceof BusinessException businessException && !isConnectionUnavailable(ex)) {
|
||||
return businessException;
|
||||
}
|
||||
if (isConnectionUnavailable(ex)) {
|
||||
return new BusinessException(SOURCE_UNAVAILABLE_MESSAGE);
|
||||
}
|
||||
return new BusinessException(fallbackMessage);
|
||||
}
|
||||
|
||||
public static boolean isConnectionUnavailable(Throwable throwable) {
|
||||
Throwable current = throwable;
|
||||
while (current != null) {
|
||||
if (current instanceof ClassNotFoundException
|
||||
|| current instanceof ConnectException
|
||||
|| current instanceof NoRouteToHostException
|
||||
|| current instanceof SocketTimeoutException
|
||||
|| current instanceof UnknownHostException) {
|
||||
return true;
|
||||
}
|
||||
if (current instanceof SocketException socketException) {
|
||||
String socketMessage = lowerCase(socketException.getMessage());
|
||||
if (socketMessage.contains("broken pipe")
|
||||
|| socketMessage.contains("connection reset")
|
||||
|| socketMessage.contains("network is unreachable")) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
if (current instanceof SQLException sqlException) {
|
||||
String sqlState = sqlException.getSQLState();
|
||||
String message = lowerCase(sqlException.getMessage());
|
||||
if ((sqlState != null && sqlState.startsWith("08"))
|
||||
|| message.contains("communications link failure")
|
||||
|| message.contains("connection refused")
|
||||
|| message.contains("connection attempt failed")
|
||||
|| message.contains("connect timed out")
|
||||
|| message.contains("i/o error")
|
||||
|| message.contains("io error")
|
||||
|| message.contains("the network adapter could not establish the connection")
|
||||
|| message.contains("unknown host")
|
||||
|| message.contains("access denied")
|
||||
|| message.contains("authentication failed")
|
||||
|| message.contains("login failed")
|
||||
|| message.contains("password authentication failed")
|
||||
|| message.contains("unknown database")
|
||||
|| message.contains("database does not exist")
|
||||
|| (message.contains("schema") && message.contains("does not exist"))
|
||||
|| message.contains("insufficient privilege")
|
||||
|| message.contains("permission denied")) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
current = current.getCause();
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
private static String lowerCase(String message) {
|
||||
return message == null ? "" : message.toLowerCase(Locale.ROOT);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,44 @@
|
||||
package tech.easyflow.datacenter.connector.support;
|
||||
|
||||
import com.zaxxer.hikari.HikariConfig;
|
||||
import com.zaxxer.hikari.HikariDataSource;
|
||||
import org.springframework.stereotype.Component;
|
||||
import tech.easyflow.datacenter.meta.entity.DatacenterSource;
|
||||
import tech.easyflow.datacenter.security.DatacenterCredentialCipher;
|
||||
|
||||
import javax.annotation.Resource;
|
||||
import java.math.BigInteger;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
|
||||
@Component
|
||||
public class DatacenterDatasourceManager {
|
||||
|
||||
private final Map<BigInteger, HikariDataSource> externalDatasourceCache = new ConcurrentHashMap<>();
|
||||
|
||||
@Resource
|
||||
private DatacenterCredentialCipher credentialCipher;
|
||||
|
||||
public HikariDataSource createExternalDatasource(DatacenterSource source) {
|
||||
HikariConfig config = new HikariConfig();
|
||||
config.setPoolName("dc-ext-" + (source.getId() == null ? "temp" : source.getId()));
|
||||
config.setJdbcUrl(source.getJdbcUrl());
|
||||
config.setUsername(source.getUsername());
|
||||
config.setPassword(credentialCipher.decrypt(source.getCredentialCipher()));
|
||||
config.setMaximumPoolSize(3);
|
||||
config.setMinimumIdle(0);
|
||||
config.setConnectionTimeout(5000);
|
||||
config.setValidationTimeout(3000);
|
||||
if (source.getDriverClassName() != null && !source.getDriverClassName().isBlank()) {
|
||||
config.setDriverClassName(source.getDriverClassName());
|
||||
}
|
||||
return new HikariDataSource(config);
|
||||
}
|
||||
|
||||
public HikariDataSource getOrCreateExternalDatasource(DatacenterSource source) {
|
||||
if (source.getId() == null) {
|
||||
return createExternalDatasource(source);
|
||||
}
|
||||
return externalDatasourceCache.computeIfAbsent(source.getId(), key -> createExternalDatasource(source));
|
||||
}
|
||||
}
|
||||
@@ -33,6 +33,18 @@ public class DatacenterTableBase extends DateEntity implements Serializable {
|
||||
@Column(tenantId = true, comment = "租户ID")
|
||||
private BigInteger tenantId;
|
||||
|
||||
/**
|
||||
* 数据源ID
|
||||
*/
|
||||
@Column(comment = "数据源ID")
|
||||
private BigInteger sourceId;
|
||||
|
||||
/**
|
||||
* 目录ID
|
||||
*/
|
||||
@Column(comment = "目录ID")
|
||||
private BigInteger catalogId;
|
||||
|
||||
/**
|
||||
* 数据表名
|
||||
*/
|
||||
@@ -51,6 +63,30 @@ public class DatacenterTableBase extends DateEntity implements Serializable {
|
||||
@Column(comment = "物理表名")
|
||||
private String actualTable;
|
||||
|
||||
/**
|
||||
* 表类型
|
||||
*/
|
||||
@Column(comment = "表类型")
|
||||
private String tableKind;
|
||||
|
||||
/**
|
||||
* 访问模式
|
||||
*/
|
||||
@Column(comment = "访问模式")
|
||||
private String accessMode;
|
||||
|
||||
/**
|
||||
* 物化表名
|
||||
*/
|
||||
@Column(comment = "物化表名")
|
||||
private String materializedTable;
|
||||
|
||||
/**
|
||||
* 是否开启版本
|
||||
*/
|
||||
@Column(comment = "是否开启版本")
|
||||
private Integer versioningEnabled;
|
||||
|
||||
/**
|
||||
* 数据状态
|
||||
*/
|
||||
@@ -87,6 +123,12 @@ public class DatacenterTableBase extends DateEntity implements Serializable {
|
||||
@Column(typeHandler = FastjsonTypeHandler.class, comment = "扩展项")
|
||||
private Map<String, Object> options;
|
||||
|
||||
/**
|
||||
* 能力声明
|
||||
*/
|
||||
@Column(typeHandler = FastjsonTypeHandler.class, comment = "能力声明")
|
||||
private Map<String, Object> capabilitiesJson;
|
||||
|
||||
public BigInteger getId() {
|
||||
return id;
|
||||
}
|
||||
@@ -111,6 +153,22 @@ public class DatacenterTableBase extends DateEntity implements Serializable {
|
||||
this.tenantId = tenantId;
|
||||
}
|
||||
|
||||
public BigInteger getSourceId() {
|
||||
return sourceId;
|
||||
}
|
||||
|
||||
public void setSourceId(BigInteger sourceId) {
|
||||
this.sourceId = sourceId;
|
||||
}
|
||||
|
||||
public BigInteger getCatalogId() {
|
||||
return catalogId;
|
||||
}
|
||||
|
||||
public void setCatalogId(BigInteger catalogId) {
|
||||
this.catalogId = catalogId;
|
||||
}
|
||||
|
||||
public String getTableName() {
|
||||
return tableName;
|
||||
}
|
||||
@@ -135,6 +193,38 @@ public class DatacenterTableBase extends DateEntity implements Serializable {
|
||||
this.actualTable = actualTable;
|
||||
}
|
||||
|
||||
public String getTableKind() {
|
||||
return tableKind;
|
||||
}
|
||||
|
||||
public void setTableKind(String tableKind) {
|
||||
this.tableKind = tableKind;
|
||||
}
|
||||
|
||||
public String getAccessMode() {
|
||||
return accessMode;
|
||||
}
|
||||
|
||||
public void setAccessMode(String accessMode) {
|
||||
this.accessMode = accessMode;
|
||||
}
|
||||
|
||||
public String getMaterializedTable() {
|
||||
return materializedTable;
|
||||
}
|
||||
|
||||
public void setMaterializedTable(String materializedTable) {
|
||||
this.materializedTable = materializedTable;
|
||||
}
|
||||
|
||||
public Integer getVersioningEnabled() {
|
||||
return versioningEnabled;
|
||||
}
|
||||
|
||||
public void setVersioningEnabled(Integer versioningEnabled) {
|
||||
this.versioningEnabled = versioningEnabled;
|
||||
}
|
||||
|
||||
public Integer getStatus() {
|
||||
return status;
|
||||
}
|
||||
@@ -183,4 +273,12 @@ public class DatacenterTableBase extends DateEntity implements Serializable {
|
||||
this.options = options;
|
||||
}
|
||||
|
||||
public Map<String, Object> getCapabilitiesJson() {
|
||||
return capabilitiesJson;
|
||||
}
|
||||
|
||||
public void setCapabilitiesJson(Map<String, Object> capabilitiesJson) {
|
||||
this.capabilitiesJson = capabilitiesJson;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -33,6 +33,12 @@ public class DatacenterTableFieldBase extends DateEntity implements Serializable
|
||||
@Column(comment = "字段名称")
|
||||
private String fieldName;
|
||||
|
||||
/**
|
||||
* 源字段名
|
||||
*/
|
||||
@Column(comment = "源字段名")
|
||||
private String sourceColumnName;
|
||||
|
||||
/**
|
||||
* 字段描述
|
||||
*/
|
||||
@@ -45,12 +51,54 @@ public class DatacenterTableFieldBase extends DateEntity implements Serializable
|
||||
@Column(comment = "字段类型")
|
||||
private Integer fieldType;
|
||||
|
||||
/**
|
||||
* JDBC 类型
|
||||
*/
|
||||
@Column(comment = "JDBC类型")
|
||||
private String jdbcType;
|
||||
|
||||
/**
|
||||
* 精度
|
||||
*/
|
||||
@Column(comment = "精度")
|
||||
private Integer precision;
|
||||
|
||||
/**
|
||||
* 小数位
|
||||
*/
|
||||
@Column(comment = "小数位")
|
||||
private Integer scale;
|
||||
|
||||
/**
|
||||
* 是否必填
|
||||
*/
|
||||
@Column(comment = "是否必填")
|
||||
private Integer required;
|
||||
|
||||
/**
|
||||
* 可查询
|
||||
*/
|
||||
@Column(comment = "可查询")
|
||||
private Integer queryable;
|
||||
|
||||
/**
|
||||
* 可排序
|
||||
*/
|
||||
@Column(comment = "可排序")
|
||||
private Integer sortable;
|
||||
|
||||
/**
|
||||
* 可写入
|
||||
*/
|
||||
@Column(comment = "可写入")
|
||||
private Integer writable;
|
||||
|
||||
/**
|
||||
* 是否索引
|
||||
*/
|
||||
@Column(comment = "是否索引")
|
||||
private Integer indexed;
|
||||
|
||||
/**
|
||||
* 扩展项
|
||||
*/
|
||||
@@ -105,6 +153,14 @@ public class DatacenterTableFieldBase extends DateEntity implements Serializable
|
||||
this.fieldName = fieldName;
|
||||
}
|
||||
|
||||
public String getSourceColumnName() {
|
||||
return sourceColumnName;
|
||||
}
|
||||
|
||||
public void setSourceColumnName(String sourceColumnName) {
|
||||
this.sourceColumnName = sourceColumnName;
|
||||
}
|
||||
|
||||
public String getFieldDesc() {
|
||||
return fieldDesc;
|
||||
}
|
||||
@@ -121,6 +177,30 @@ public class DatacenterTableFieldBase extends DateEntity implements Serializable
|
||||
this.fieldType = fieldType;
|
||||
}
|
||||
|
||||
public String getJdbcType() {
|
||||
return jdbcType;
|
||||
}
|
||||
|
||||
public void setJdbcType(String jdbcType) {
|
||||
this.jdbcType = jdbcType;
|
||||
}
|
||||
|
||||
public Integer getPrecision() {
|
||||
return precision;
|
||||
}
|
||||
|
||||
public void setPrecision(Integer precision) {
|
||||
this.precision = precision;
|
||||
}
|
||||
|
||||
public Integer getScale() {
|
||||
return scale;
|
||||
}
|
||||
|
||||
public void setScale(Integer scale) {
|
||||
this.scale = scale;
|
||||
}
|
||||
|
||||
public Integer getRequired() {
|
||||
return required;
|
||||
}
|
||||
@@ -129,6 +209,38 @@ public class DatacenterTableFieldBase extends DateEntity implements Serializable
|
||||
this.required = required;
|
||||
}
|
||||
|
||||
public Integer getQueryable() {
|
||||
return queryable;
|
||||
}
|
||||
|
||||
public void setQueryable(Integer queryable) {
|
||||
this.queryable = queryable;
|
||||
}
|
||||
|
||||
public Integer getSortable() {
|
||||
return sortable;
|
||||
}
|
||||
|
||||
public void setSortable(Integer sortable) {
|
||||
this.sortable = sortable;
|
||||
}
|
||||
|
||||
public Integer getWritable() {
|
||||
return writable;
|
||||
}
|
||||
|
||||
public void setWritable(Integer writable) {
|
||||
this.writable = writable;
|
||||
}
|
||||
|
||||
public Integer getIndexed() {
|
||||
return indexed;
|
||||
}
|
||||
|
||||
public void setIndexed(Integer indexed) {
|
||||
this.indexed = indexed;
|
||||
}
|
||||
|
||||
public Map<String, Object> getOptions() {
|
||||
return options;
|
||||
}
|
||||
|
||||
@@ -1,107 +0,0 @@
|
||||
package tech.easyflow.datacenter.excel;
|
||||
|
||||
import cn.idev.excel.context.AnalysisContext;
|
||||
import cn.idev.excel.metadata.data.ReadCellData;
|
||||
import cn.idev.excel.read.listener.ReadListener;
|
||||
import com.alibaba.fastjson2.JSONObject;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import tech.easyflow.common.entity.LoginAccount;
|
||||
import tech.easyflow.common.util.SpringContextUtil;
|
||||
import tech.easyflow.datacenter.entity.DatacenterTableField;
|
||||
import tech.easyflow.datacenter.service.DatacenterTableService;
|
||||
|
||||
import java.math.BigInteger;
|
||||
import java.util.*;
|
||||
|
||||
public class ReadDataListener implements ReadListener<LinkedHashMap<Integer, Object>> {
|
||||
|
||||
private static final Logger log = LoggerFactory.getLogger(ReadDataListener.class);
|
||||
|
||||
private BigInteger tableId;
|
||||
|
||||
private List<DatacenterTableField> fields;
|
||||
|
||||
private LoginAccount loginAccount;
|
||||
|
||||
private final Map<String, Integer> headFieldIndex = new HashMap<>();
|
||||
|
||||
private int successCount = 0;
|
||||
private int errorCount = 0;
|
||||
private int totalCount = 0;
|
||||
|
||||
private final List<JSONObject> errorRows = new ArrayList<>();
|
||||
|
||||
public ReadDataListener() {
|
||||
}
|
||||
|
||||
public ReadDataListener(BigInteger tableId, List<DatacenterTableField> fields, LoginAccount loginAccount) {
|
||||
this.tableId = tableId;
|
||||
this.fields = fields;
|
||||
this.loginAccount = loginAccount;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void invoke(LinkedHashMap<Integer, Object> o, AnalysisContext analysisContext) {
|
||||
DatacenterTableService service = SpringContextUtil.getBean(DatacenterTableService.class);
|
||||
JSONObject obj = new JSONObject();
|
||||
for (DatacenterTableField field : fields) {
|
||||
String fieldName = field.getFieldName();
|
||||
Integer i = headFieldIndex.get(fieldName);
|
||||
if (i != null) {
|
||||
obj.put(fieldName, o.get(i));
|
||||
}
|
||||
}
|
||||
try {
|
||||
service.saveValue(tableId, obj, loginAccount);
|
||||
successCount++;
|
||||
} catch (Exception e) {
|
||||
errorCount++;
|
||||
log.error("导入数据到数据中枢失败,具体值:{}", obj, e);
|
||||
errorRows.add(obj);
|
||||
}
|
||||
totalCount++;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void invokeHead(Map<Integer, ReadCellData<?>> headMap, AnalysisContext context) {
|
||||
Set<Map.Entry<Integer, ReadCellData<?>>> entries = headMap.entrySet();
|
||||
for (Map.Entry<Integer, ReadCellData<?>> entry : entries) {
|
||||
Integer key = entry.getKey();
|
||||
String field = entry.getValue().getStringValue();
|
||||
headFieldIndex.put(field, key);
|
||||
}
|
||||
if (headFieldIndex.size() != fields.size()) {
|
||||
throw new RuntimeException("表头字段数量与表结构对应不上!");
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void doAfterAllAnalysed(AnalysisContext analysisContext) {
|
||||
|
||||
}
|
||||
|
||||
public List<DatacenterTableField> getFields() {
|
||||
return fields;
|
||||
}
|
||||
|
||||
public void setFields(List<DatacenterTableField> fields) {
|
||||
this.fields = fields;
|
||||
}
|
||||
|
||||
public int getSuccessCount() {
|
||||
return successCount;
|
||||
}
|
||||
|
||||
public int getErrorCount() {
|
||||
return errorCount;
|
||||
}
|
||||
|
||||
public int getTotalCount() {
|
||||
return totalCount;
|
||||
}
|
||||
|
||||
public List<JSONObject> getErrorRows() {
|
||||
return errorRows;
|
||||
}
|
||||
}
|
||||
@@ -1,68 +0,0 @@
|
||||
package tech.easyflow.datacenter.excel;
|
||||
|
||||
import com.alibaba.fastjson2.JSONObject;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
public class ReadResVo {
|
||||
|
||||
/**
|
||||
* 成功数
|
||||
*/
|
||||
private int successCount = 0;
|
||||
/**
|
||||
* 失败数
|
||||
*/
|
||||
private int errorCount = 0;
|
||||
/**
|
||||
* 总数
|
||||
*/
|
||||
private int totalCount = 0;
|
||||
|
||||
/**
|
||||
* 错误行
|
||||
*/
|
||||
private List<JSONObject> errorRows;
|
||||
|
||||
public ReadResVo() {
|
||||
}
|
||||
|
||||
public ReadResVo(int successCount, int errorCount, int totalCount, List<JSONObject> errorRows) {
|
||||
this.successCount = successCount;
|
||||
this.errorCount = errorCount;
|
||||
this.totalCount = totalCount;
|
||||
this.errorRows = errorRows;
|
||||
}
|
||||
|
||||
public int getSuccessCount() {
|
||||
return successCount;
|
||||
}
|
||||
|
||||
public void setSuccessCount(int successCount) {
|
||||
this.successCount = successCount;
|
||||
}
|
||||
|
||||
public int getErrorCount() {
|
||||
return errorCount;
|
||||
}
|
||||
|
||||
public void setErrorCount(int errorCount) {
|
||||
this.errorCount = errorCount;
|
||||
}
|
||||
|
||||
public int getTotalCount() {
|
||||
return totalCount;
|
||||
}
|
||||
|
||||
public void setTotalCount(int totalCount) {
|
||||
this.totalCount = totalCount;
|
||||
}
|
||||
|
||||
public List<JSONObject> getErrorRows() {
|
||||
return errorRows;
|
||||
}
|
||||
|
||||
public void setErrorRows(List<JSONObject> errorRows) {
|
||||
this.errorRows = errorRows;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,28 @@
|
||||
package tech.easyflow.datacenter.excel.model;
|
||||
|
||||
import tech.easyflow.datacenter.execution.model.DatacenterQueryFilter;
|
||||
import tech.easyflow.datacenter.execution.model.DatasetRef;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
public class DatacenterExcelDeriveRequest {
|
||||
private DatasetRef datasetRef;
|
||||
private String targetTableName;
|
||||
private List<String> selectedColumns = new ArrayList<>();
|
||||
private Map<String, String> renameMappings = new LinkedHashMap<>();
|
||||
private List<DatacenterQueryFilter> filters = new ArrayList<>();
|
||||
|
||||
public DatasetRef getDatasetRef() { return datasetRef; }
|
||||
public void setDatasetRef(DatasetRef datasetRef) { this.datasetRef = datasetRef; }
|
||||
public String getTargetTableName() { return targetTableName; }
|
||||
public void setTargetTableName(String targetTableName) { this.targetTableName = targetTableName; }
|
||||
public List<String> getSelectedColumns() { return selectedColumns; }
|
||||
public void setSelectedColumns(List<String> selectedColumns) { this.selectedColumns = selectedColumns; }
|
||||
public Map<String, String> getRenameMappings() { return renameMappings; }
|
||||
public void setRenameMappings(Map<String, String> renameMappings) { this.renameMappings = renameMappings; }
|
||||
public List<DatacenterQueryFilter> getFilters() { return filters; }
|
||||
public void setFilters(List<DatacenterQueryFilter> filters) { this.filters = filters; }
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
package tech.easyflow.datacenter.excel.model;
|
||||
|
||||
import tech.easyflow.datacenter.execution.model.DatasetRef;
|
||||
|
||||
import java.math.BigInteger;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
public class DatacenterExcelExportRequest {
|
||||
private BigInteger sourceId;
|
||||
private BigInteger catalogId;
|
||||
private List<DatasetRef> datasetRefs = new ArrayList<>();
|
||||
private String fileName;
|
||||
|
||||
public BigInteger getSourceId() { return sourceId; }
|
||||
public void setSourceId(BigInteger sourceId) { this.sourceId = sourceId; }
|
||||
public BigInteger getCatalogId() { return catalogId; }
|
||||
public void setCatalogId(BigInteger catalogId) { this.catalogId = catalogId; }
|
||||
public List<DatasetRef> getDatasetRefs() { return datasetRefs; }
|
||||
public void setDatasetRefs(List<DatasetRef> datasetRefs) { this.datasetRefs = datasetRefs; }
|
||||
public String getFileName() { return fileName; }
|
||||
public void setFileName(String fileName) { this.fileName = fileName; }
|
||||
}
|
||||
@@ -0,0 +1,22 @@
|
||||
package tech.easyflow.datacenter.excel.model;
|
||||
|
||||
import tech.easyflow.datacenter.execution.model.DatasetRef;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
public class DatacenterExcelMergeRequest {
|
||||
private List<DatasetRef> datasetRefs = new ArrayList<>();
|
||||
private String mergeMode;
|
||||
private String targetTableName;
|
||||
private String joinKey;
|
||||
|
||||
public List<DatasetRef> getDatasetRefs() { return datasetRefs; }
|
||||
public void setDatasetRefs(List<DatasetRef> datasetRefs) { this.datasetRefs = datasetRefs; }
|
||||
public String getMergeMode() { return mergeMode; }
|
||||
public void setMergeMode(String mergeMode) { this.mergeMode = mergeMode; }
|
||||
public String getTargetTableName() { return targetTableName; }
|
||||
public void setTargetTableName(String targetTableName) { this.targetTableName = targetTableName; }
|
||||
public String getJoinKey() { return joinKey; }
|
||||
public void setJoinKey(String joinKey) { this.joinKey = joinKey; }
|
||||
}
|
||||
@@ -0,0 +1,30 @@
|
||||
package tech.easyflow.datacenter.excel.model;
|
||||
|
||||
import tech.easyflow.datacenter.execution.model.DatasetRef;
|
||||
|
||||
import java.math.BigInteger;
|
||||
|
||||
public class DatacenterExcelSplitRequest {
|
||||
private BigInteger sourceId;
|
||||
private BigInteger catalogId;
|
||||
private DatasetRef datasetRef;
|
||||
private String splitMode;
|
||||
private Integer rowBatchSize;
|
||||
private String fieldName;
|
||||
private String targetNamePrefix;
|
||||
|
||||
public BigInteger getSourceId() { return sourceId; }
|
||||
public void setSourceId(BigInteger sourceId) { this.sourceId = sourceId; }
|
||||
public BigInteger getCatalogId() { return catalogId; }
|
||||
public void setCatalogId(BigInteger catalogId) { this.catalogId = catalogId; }
|
||||
public DatasetRef getDatasetRef() { return datasetRef; }
|
||||
public void setDatasetRef(DatasetRef datasetRef) { this.datasetRef = datasetRef; }
|
||||
public String getSplitMode() { return splitMode; }
|
||||
public void setSplitMode(String splitMode) { this.splitMode = splitMode; }
|
||||
public Integer getRowBatchSize() { return rowBatchSize; }
|
||||
public void setRowBatchSize(Integer rowBatchSize) { this.rowBatchSize = rowBatchSize; }
|
||||
public String getFieldName() { return fieldName; }
|
||||
public void setFieldName(String fieldName) { this.fieldName = fieldName; }
|
||||
public String getTargetNamePrefix() { return targetNamePrefix; }
|
||||
public void setTargetNamePrefix(String targetNamePrefix) { this.targetNamePrefix = targetNamePrefix; }
|
||||
}
|
||||
@@ -0,0 +1,28 @@
|
||||
package tech.easyflow.datacenter.excel.service;
|
||||
|
||||
import org.springframework.web.multipart.MultipartFile;
|
||||
import tech.easyflow.common.entity.LoginAccount;
|
||||
import tech.easyflow.datacenter.excel.model.DatacenterExcelDeriveRequest;
|
||||
import tech.easyflow.datacenter.excel.model.DatacenterExcelExportRequest;
|
||||
import tech.easyflow.datacenter.excel.model.DatacenterExcelMergeRequest;
|
||||
import tech.easyflow.datacenter.excel.model.DatacenterExcelSplitRequest;
|
||||
import tech.easyflow.datacenter.meta.entity.DatacenterImportJob;
|
||||
|
||||
import java.math.BigInteger;
|
||||
import java.util.List;
|
||||
|
||||
public interface DatacenterExcelImportService {
|
||||
DatacenterImportJob importWorkbook(MultipartFile file, LoginAccount account) throws Exception;
|
||||
|
||||
DatacenterImportJob splitWorkbook(DatacenterExcelSplitRequest request, LoginAccount account);
|
||||
|
||||
DatacenterImportJob mergeWorkbook(DatacenterExcelMergeRequest request, LoginAccount account);
|
||||
|
||||
DatacenterImportJob deriveWorkbook(DatacenterExcelDeriveRequest request, LoginAccount account);
|
||||
|
||||
DatacenterImportJob exportWorkbook(DatacenterExcelExportRequest request, LoginAccount account) throws Exception;
|
||||
|
||||
DatacenterImportJob getImportJobDetail(BigInteger jobId);
|
||||
|
||||
List<DatacenterImportJob> listJobs(BigInteger sourceId, BigInteger tableId);
|
||||
}
|
||||
@@ -0,0 +1,950 @@
|
||||
package tech.easyflow.datacenter.excel.service.impl;
|
||||
|
||||
import com.alibaba.fastjson2.JSONObject;
|
||||
import com.mybatisflex.core.keygen.impl.SnowFlakeIDKeyGenerator;
|
||||
import com.mybatisflex.core.paginate.Page;
|
||||
import com.mybatisflex.core.row.Row;
|
||||
import org.apache.poi.ss.usermodel.Cell;
|
||||
import org.apache.poi.ss.usermodel.DataFormatter;
|
||||
import org.apache.poi.ss.usermodel.Sheet;
|
||||
import org.apache.poi.ss.usermodel.Workbook;
|
||||
import org.apache.poi.ss.usermodel.WorkbookFactory;
|
||||
import org.apache.poi.xssf.streaming.SXSSFWorkbook;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.springframework.transaction.annotation.Transactional;
|
||||
import org.springframework.util.CollectionUtils;
|
||||
import org.springframework.web.multipart.MultipartFile;
|
||||
import tech.easyflow.common.constant.enums.EnumFieldType;
|
||||
import tech.easyflow.common.entity.LoginAccount;
|
||||
import tech.easyflow.common.web.exceptions.BusinessException;
|
||||
import tech.easyflow.datacenter.adapter.DbHandleManager;
|
||||
import tech.easyflow.datacenter.entity.DatacenterTable;
|
||||
import tech.easyflow.datacenter.entity.DatacenterTableField;
|
||||
import tech.easyflow.datacenter.excel.model.DatacenterExcelDeriveRequest;
|
||||
import tech.easyflow.datacenter.excel.model.DatacenterExcelExportRequest;
|
||||
import tech.easyflow.datacenter.excel.model.DatacenterExcelMergeRequest;
|
||||
import tech.easyflow.datacenter.excel.model.DatacenterExcelSplitRequest;
|
||||
import tech.easyflow.datacenter.excel.service.DatacenterExcelImportService;
|
||||
import tech.easyflow.datacenter.execution.model.DatacenterQueryFilter;
|
||||
import tech.easyflow.datacenter.execution.model.DatacenterQueryRequest;
|
||||
import tech.easyflow.datacenter.execution.model.DatasetRef;
|
||||
import tech.easyflow.datacenter.execution.service.DatacenterDatasetQueryService;
|
||||
import tech.easyflow.datacenter.mapper.DatacenterDatasetVersionMapper;
|
||||
import tech.easyflow.datacenter.mapper.DatacenterDerivedTableMapper;
|
||||
import tech.easyflow.datacenter.mapper.DatacenterImportJobMapper;
|
||||
import tech.easyflow.datacenter.meta.entity.DatacenterCatalog;
|
||||
import tech.easyflow.datacenter.meta.entity.DatacenterDatasetVersion;
|
||||
import tech.easyflow.datacenter.meta.entity.DatacenterDerivedTable;
|
||||
import tech.easyflow.datacenter.meta.entity.DatacenterImportJob;
|
||||
import tech.easyflow.datacenter.meta.entity.DatacenterSource;
|
||||
import tech.easyflow.datacenter.meta.enums.DatacenterImportStatus;
|
||||
import tech.easyflow.datacenter.meta.enums.DatacenterSourceType;
|
||||
import tech.easyflow.datacenter.meta.enums.DatacenterTableKind;
|
||||
import tech.easyflow.datacenter.meta.model.DatacenterTableDetailMeta;
|
||||
import tech.easyflow.datacenter.meta.service.DatacenterDatasetRegistryService;
|
||||
import tech.easyflow.datacenter.meta.service.DatacenterSourceService;
|
||||
|
||||
import javax.annotation.Resource;
|
||||
import java.io.File;
|
||||
import java.io.FileOutputStream;
|
||||
import java.io.InputStream;
|
||||
import java.math.BigInteger;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.time.LocalDateTime;
|
||||
import java.time.format.DateTimeFormatter;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Date;
|
||||
import java.util.HashMap;
|
||||
import java.util.HashSet;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.LinkedHashSet;
|
||||
import java.util.List;
|
||||
import java.util.Locale;
|
||||
import java.util.Map;
|
||||
import java.util.Objects;
|
||||
import java.util.Set;
|
||||
import java.util.UUID;
|
||||
|
||||
@Service
|
||||
public class DatacenterExcelImportServiceImpl implements DatacenterExcelImportService {
|
||||
|
||||
private static final long QUERY_BATCH_SIZE = 500L;
|
||||
private static final DateTimeFormatter EXPORT_TIME_FORMAT = DateTimeFormatter.ofPattern("yyyyMMddHHmmss");
|
||||
|
||||
@Resource
|
||||
private DatacenterSourceService sourceService;
|
||||
@Resource
|
||||
private DatacenterDatasetRegistryService registryService;
|
||||
@Resource
|
||||
private DatacenterImportJobMapper importJobMapper;
|
||||
@Resource
|
||||
private DatacenterDatasetVersionMapper datasetVersionMapper;
|
||||
@Resource
|
||||
private DatacenterDerivedTableMapper derivedTableMapper;
|
||||
@Resource
|
||||
private DatacenterDatasetQueryService queryService;
|
||||
@Resource
|
||||
private DbHandleManager dbHandleManager;
|
||||
|
||||
@Override
|
||||
@Transactional(rollbackFor = Exception.class)
|
||||
public DatacenterImportJob importWorkbook(MultipartFile file, LoginAccount account) throws Exception {
|
||||
if (file == null || file.isEmpty()) {
|
||||
throw new BusinessException("Excel 文件不能为空");
|
||||
}
|
||||
String workbookName = extractWorkbookName(file.getOriginalFilename());
|
||||
DatacenterSource source = new DatacenterSource();
|
||||
source.setSourceName(workbookName);
|
||||
source.setSourceCode("EXCEL_" + UUID.randomUUID());
|
||||
source.setSourceType(DatacenterSourceType.EXCEL.name());
|
||||
source.setAccessMode("READ_WRITE");
|
||||
source.setBuiltinFlag(0);
|
||||
source.setConfigJson(Map.of("originFileName", file.getOriginalFilename()));
|
||||
source = sourceService.saveSource(source, account);
|
||||
DatacenterCatalog catalog = registryService.ensureCatalog(source, workbookName, account);
|
||||
|
||||
DatacenterImportJob job = createJob("EXCEL_IMPORT", source.getId(), catalog.getId(), null,
|
||||
file.getOriginalFilename(), Map.of("operation", "import"), account);
|
||||
|
||||
long totalRows = 0L;
|
||||
long successRows = 0L;
|
||||
List<BigInteger> createdTableIds = new ArrayList<>();
|
||||
try (InputStream inputStream = file.getInputStream(); Workbook workbook = WorkbookFactory.create(inputStream)) {
|
||||
DataFormatter formatter = new DataFormatter();
|
||||
for (int sheetIndex = 0; sheetIndex < workbook.getNumberOfSheets(); sheetIndex++) {
|
||||
Sheet sheet = workbook.getSheetAt(sheetIndex);
|
||||
org.apache.poi.ss.usermodel.Row headerRow = sheet.getRow(sheet.getFirstRowNum());
|
||||
if (headerRow == null) {
|
||||
continue;
|
||||
}
|
||||
List<DatacenterTableField> fields = buildFields(headerRow, formatter);
|
||||
if (fields.isEmpty()) {
|
||||
continue;
|
||||
}
|
||||
DatacenterTable table = new DatacenterTable();
|
||||
table.setTableName(uniqueTableName(source.getId(), catalog.getId(), sheet.getSheetName()));
|
||||
table.setTableDesc(sheet.getSheetName());
|
||||
table.setActualTable(buildMaterializedTableName(source.getId(), sheetIndex));
|
||||
table.setMaterializedTable(table.getActualTable());
|
||||
table.setTableKind(DatacenterTableKind.EXCEL_MATERIALIZED.name());
|
||||
table.setAccessMode("READ_WRITE");
|
||||
table.setVersioningEnabled(1);
|
||||
table.setCapabilitiesJson(defaultExcelCapabilities());
|
||||
table.setFields(fields);
|
||||
|
||||
DatacenterTableDetailMeta detail = new DatacenterTableDetailMeta();
|
||||
detail.setTable(table);
|
||||
detail.setFields(fields);
|
||||
|
||||
dbHandleManager.getDbHandler().createTable(table);
|
||||
DatacenterTable savedTable = registryService.registerTable(source, catalog, detail, account);
|
||||
savedTable.setFields(registryService.getFields(savedTable.getId()));
|
||||
createdTableIds.add(savedTable.getId());
|
||||
|
||||
for (int rowIndex = sheet.getFirstRowNum() + 1; rowIndex <= sheet.getLastRowNum(); rowIndex++) {
|
||||
org.apache.poi.ss.usermodel.Row row = sheet.getRow(rowIndex);
|
||||
if (row == null) {
|
||||
continue;
|
||||
}
|
||||
JSONObject payload = new JSONObject();
|
||||
boolean hasValue = false;
|
||||
for (int cellIndex = 0; cellIndex < savedTable.getFields().size(); cellIndex++) {
|
||||
String value = formatter.formatCellValue(row.getCell(cellIndex));
|
||||
if (value != null && !value.isBlank()) {
|
||||
hasValue = true;
|
||||
}
|
||||
payload.put(savedTable.getFields().get(cellIndex).getFieldName(), value);
|
||||
}
|
||||
if (!hasValue) {
|
||||
continue;
|
||||
}
|
||||
dbHandleManager.getDbHandler().saveValue(savedTable, payload, account);
|
||||
totalRows++;
|
||||
successRows++;
|
||||
}
|
||||
|
||||
createVersion(savedTable, "initial-import", Map.of(
|
||||
"sheetName", sheet.getSheetName(),
|
||||
"sourceId", source.getId(),
|
||||
"originFileName", file.getOriginalFilename()
|
||||
), account);
|
||||
}
|
||||
job.setTableId(createdTableIds.isEmpty() ? null : createdTableIds.get(0));
|
||||
finishJobSuccess(job, totalRows, successRows, Map.of("tableIds", createdTableIds));
|
||||
return job;
|
||||
} catch (Exception ex) {
|
||||
finishJobFailure(job, ex);
|
||||
throw ex;
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
@Transactional(rollbackFor = Exception.class)
|
||||
public DatacenterImportJob splitWorkbook(DatacenterExcelSplitRequest request, LoginAccount account) {
|
||||
DatasetRef datasetRef = request == null ? null : request.getDatasetRef();
|
||||
DatacenterImportJob job = createJob("EXCEL_SPLIT",
|
||||
request == null ? null : request.getSourceId(),
|
||||
request == null ? null : request.getCatalogId(),
|
||||
datasetRef == null ? null : datasetRef.getTableId(),
|
||||
null,
|
||||
buildPayload("request", request),
|
||||
account);
|
||||
try {
|
||||
String splitMode = normalizeMode(request == null ? null : request.getSplitMode(), "BY_ROW_COUNT");
|
||||
return switch (splitMode) {
|
||||
case "BY_SHEET" -> doSplitBySheet(request, account, job);
|
||||
case "BY_FIELD_VALUE" -> doSplitByFieldValue(request, account, job);
|
||||
default -> doSplitByRowCount(request, account, job);
|
||||
};
|
||||
} catch (Exception ex) {
|
||||
finishJobFailure(job, ex);
|
||||
throw ex;
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
@Transactional(rollbackFor = Exception.class)
|
||||
public DatacenterImportJob mergeWorkbook(DatacenterExcelMergeRequest request, LoginAccount account) {
|
||||
if (request == null || CollectionUtils.isEmpty(request.getDatasetRefs())) {
|
||||
throw new BusinessException("合并数据集不能为空");
|
||||
}
|
||||
DatacenterImportJob job = createJob("EXCEL_MERGE", null, null, null, null, buildPayload("request", request), account);
|
||||
try {
|
||||
String mergeMode = normalizeMode(request.getMergeMode(), "VERTICAL");
|
||||
return switch (mergeMode) {
|
||||
case "HORIZONTAL" -> doHorizontalMerge(request, account, job);
|
||||
default -> doVerticalMerge(request, account, job);
|
||||
};
|
||||
} catch (Exception ex) {
|
||||
finishJobFailure(job, ex);
|
||||
throw ex;
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
@Transactional(rollbackFor = Exception.class)
|
||||
public DatacenterImportJob deriveWorkbook(DatacenterExcelDeriveRequest request, LoginAccount account) {
|
||||
if (request == null || request.getDatasetRef() == null) {
|
||||
throw new BusinessException("派生数据集不能为空");
|
||||
}
|
||||
DatacenterImportJob job = createJob("EXCEL_DERIVE",
|
||||
request.getDatasetRef().getSourceId(),
|
||||
request.getDatasetRef().getCatalogId(),
|
||||
request.getDatasetRef().getTableId(),
|
||||
null,
|
||||
buildPayload("request", request),
|
||||
account);
|
||||
try {
|
||||
DatacenterTable sourceTable = resolveTable(request.getDatasetRef());
|
||||
DatacenterSource source = registryService.getSourceRequired(sourceTable.getSourceId());
|
||||
DatacenterCatalog catalog = requireCatalog(sourceTable.getCatalogId());
|
||||
List<String> selectedColumns = resolveSelectedColumns(sourceTable, request.getSelectedColumns());
|
||||
List<DatacenterTableField> targetFields = buildDerivedFields(sourceTable, request);
|
||||
DatacenterTable targetTable = createDerivedTable(source, catalog, targetFields,
|
||||
request.getTargetTableName(), "DERIVE", Map.of("sourceTableId", sourceTable.getId()), account);
|
||||
|
||||
DatacenterQueryRequest queryRequest = new DatacenterQueryRequest();
|
||||
queryRequest.setDatasetRef(registryService.resolveDatasetRef(sourceTable.getId()));
|
||||
queryRequest.setSelectedColumns(selectedColumns);
|
||||
queryRequest.setFilters(request.getFilters());
|
||||
|
||||
long successRows = copyRows(queryRequest, rows -> mapDerivedRow(rows, selectedColumns, request), targetTable, account);
|
||||
createLineage(sourceTable.getId(), targetTable.getId(), "DERIVE", Map.of("request", request), account);
|
||||
finishJobSuccess(job, successRows, successRows, Map.of("derivedTableId", targetTable.getId()));
|
||||
job.setTableId(targetTable.getId());
|
||||
return job;
|
||||
} catch (Exception ex) {
|
||||
finishJobFailure(job, ex);
|
||||
throw ex;
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
@Transactional(rollbackFor = Exception.class)
|
||||
public DatacenterImportJob exportWorkbook(DatacenterExcelExportRequest request, LoginAccount account) throws Exception {
|
||||
List<DatacenterTable> tables = resolveExportTables(request);
|
||||
if (tables.isEmpty()) {
|
||||
throw new BusinessException("没有可导出的 Excel 数据集");
|
||||
}
|
||||
String fileName = buildExportFileName(request == null ? null : request.getFileName());
|
||||
DatacenterImportJob job = createJob("EXCEL_EXPORT",
|
||||
request == null ? null : request.getSourceId(),
|
||||
request == null ? null : request.getCatalogId(),
|
||||
null,
|
||||
fileName,
|
||||
buildPayload("request", request),
|
||||
account);
|
||||
Path exportDir = ensureExportDir();
|
||||
Path exportFile = exportDir.resolve(fileName);
|
||||
long totalRows = 0L;
|
||||
try (SXSSFWorkbook workbook = new SXSSFWorkbook(200); FileOutputStream outputStream = new FileOutputStream(exportFile.toFile())) {
|
||||
workbook.setCompressTempFiles(true);
|
||||
Set<String> usedSheetNames = new HashSet<>();
|
||||
for (DatacenterTable table : tables) {
|
||||
String sheetName = uniqueSheetName(table.getTableName(), usedSheetNames);
|
||||
org.apache.poi.ss.usermodel.Sheet sheet = workbook.createSheet(sheetName);
|
||||
writeHeaderRow(sheet, table.getFields());
|
||||
DatacenterQueryRequest queryRequest = new DatacenterQueryRequest();
|
||||
queryRequest.setDatasetRef(registryService.resolveDatasetRef(table.getId()));
|
||||
queryRequest.setSelectedColumns(table.getFields().stream().map(DatacenterTableField::getFieldName).toList());
|
||||
final int[] rowIndex = {1};
|
||||
totalRows += iterateRows(queryRequest, row -> {
|
||||
org.apache.poi.ss.usermodel.Row excelRow = sheet.createRow(rowIndex[0]++);
|
||||
for (int i = 0; i < table.getFields().size(); i++) {
|
||||
Cell cell = excelRow.createCell(i);
|
||||
Object value = row.get(table.getFields().get(i).getFieldName());
|
||||
cell.setCellValue(value == null ? "" : String.valueOf(value));
|
||||
}
|
||||
});
|
||||
}
|
||||
workbook.write(outputStream);
|
||||
workbook.dispose();
|
||||
job.setStoragePath(exportFile.toAbsolutePath().toString());
|
||||
finishJobSuccess(job, totalRows, totalRows, Map.of("storagePath", job.getStoragePath(), "fileName", fileName));
|
||||
return job;
|
||||
} catch (Exception ex) {
|
||||
finishJobFailure(job, ex);
|
||||
throw ex;
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public DatacenterImportJob getImportJobDetail(BigInteger jobId) {
|
||||
DatacenterImportJob job = importJobMapper.selectOneById(jobId);
|
||||
if (job == null) {
|
||||
throw new BusinessException("导入任务不存在: " + jobId);
|
||||
}
|
||||
return job;
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<DatacenterImportJob> listJobs(BigInteger sourceId, BigInteger tableId) {
|
||||
var wrapper = com.mybatisflex.core.query.QueryWrapper.create();
|
||||
if (sourceId != null) {
|
||||
wrapper.eq(DatacenterImportJob::getSourceId, sourceId);
|
||||
}
|
||||
if (tableId != null) {
|
||||
wrapper.eq(DatacenterImportJob::getTableId, tableId);
|
||||
}
|
||||
wrapper.orderBy("created desc");
|
||||
wrapper.limit(20L);
|
||||
return importJobMapper.selectListByQuery(wrapper);
|
||||
}
|
||||
|
||||
private DatacenterImportJob doSplitBySheet(DatacenterExcelSplitRequest request, LoginAccount account, DatacenterImportJob job) {
|
||||
BigInteger sourceId = request.getSourceId();
|
||||
BigInteger catalogId = request.getCatalogId();
|
||||
if (sourceId == null && request.getDatasetRef() != null) {
|
||||
sourceId = request.getDatasetRef().getSourceId();
|
||||
catalogId = request.getDatasetRef().getCatalogId();
|
||||
}
|
||||
if (sourceId == null) {
|
||||
throw new BusinessException("按 sheet 拆分需要 sourceId");
|
||||
}
|
||||
DatacenterSource source = registryService.getSourceRequired(sourceId);
|
||||
DatacenterCatalog catalog = requireCatalog(catalogId);
|
||||
List<DatacenterTable> sourceTables = registryService.listManagedTables(sourceId, catalogId);
|
||||
if (sourceTables.isEmpty()) {
|
||||
throw new BusinessException("当前 workbook 下没有可拆分的 sheet 表");
|
||||
}
|
||||
List<BigInteger> derivedIds = new ArrayList<>();
|
||||
long successRows = 0L;
|
||||
for (DatacenterTable sourceTable : sourceTables) {
|
||||
DatacenterTable fullTable = registryService.getTableWithFields(sourceTable.getId());
|
||||
DatacenterTable targetTable = createDerivedTable(source, catalog, cloneFields(fullTable.getFields()),
|
||||
resolveSplitPrefix(request, fullTable.getTableName()) + "_copy", "SPLIT_BY_SHEET",
|
||||
Map.of("sourceTableId", fullTable.getId()), account);
|
||||
DatacenterQueryRequest queryRequest = new DatacenterQueryRequest();
|
||||
queryRequest.setDatasetRef(registryService.resolveDatasetRef(fullTable.getId()));
|
||||
queryRequest.setSelectedColumns(fullTable.getFields().stream().map(DatacenterTableField::getFieldName).toList());
|
||||
successRows += copyRows(queryRequest, this::mapRow, targetTable, account);
|
||||
createLineage(fullTable.getId(), targetTable.getId(), "SPLIT_BY_SHEET", Map.of("sourceTableId", fullTable.getId()), account);
|
||||
derivedIds.add(targetTable.getId());
|
||||
}
|
||||
finishJobSuccess(job, successRows, successRows, Map.of("derivedTableIds", derivedIds));
|
||||
return job;
|
||||
}
|
||||
|
||||
private DatacenterImportJob doSplitByRowCount(DatacenterExcelSplitRequest request, LoginAccount account, DatacenterImportJob job) {
|
||||
if (request == null || request.getDatasetRef() == null) {
|
||||
throw new BusinessException("按行数拆分需要数据集");
|
||||
}
|
||||
int rowBatchSize = request.getRowBatchSize() == null || request.getRowBatchSize() < 1 ? 1000 : request.getRowBatchSize();
|
||||
DatacenterTable sourceTable = resolveTable(request.getDatasetRef());
|
||||
DatacenterSource source = registryService.getSourceRequired(sourceTable.getSourceId());
|
||||
DatacenterCatalog catalog = requireCatalog(sourceTable.getCatalogId());
|
||||
String baseName = resolveSplitPrefix(request, sourceTable.getTableName());
|
||||
List<BigInteger> derivedIds = new ArrayList<>();
|
||||
final Holder holder = new Holder();
|
||||
long totalRows = iterateRows(buildFullQuery(sourceTable), row -> {
|
||||
if (holder.targetTable == null || holder.currentSize >= rowBatchSize) {
|
||||
holder.batchNo++;
|
||||
holder.targetTable = createDerivedTable(source, catalog, cloneFields(sourceTable.getFields()),
|
||||
baseName + "_part_" + holder.batchNo, "SPLIT_BY_ROW_COUNT",
|
||||
Map.of("sourceTableId", sourceTable.getId(), "batchNo", holder.batchNo, "rowBatchSize", rowBatchSize), account);
|
||||
createLineage(sourceTable.getId(), holder.targetTable.getId(), "SPLIT_BY_ROW_COUNT",
|
||||
Map.of("sourceTableId", sourceTable.getId(), "batchNo", holder.batchNo), account);
|
||||
derivedIds.add(holder.targetTable.getId());
|
||||
holder.currentSize = 0;
|
||||
}
|
||||
saveToTable(holder.targetTable, mapRow(row), account);
|
||||
holder.currentSize++;
|
||||
});
|
||||
finishJobSuccess(job, totalRows, totalRows, Map.of("derivedTableIds", derivedIds));
|
||||
return job;
|
||||
}
|
||||
|
||||
private DatacenterImportJob doSplitByFieldValue(DatacenterExcelSplitRequest request, LoginAccount account, DatacenterImportJob job) {
|
||||
if (request == null || request.getDatasetRef() == null || request.getFieldName() == null || request.getFieldName().isBlank()) {
|
||||
throw new BusinessException("按字段值拆分需要数据集和字段名");
|
||||
}
|
||||
DatacenterTable sourceTable = resolveTable(request.getDatasetRef());
|
||||
DatacenterSource source = registryService.getSourceRequired(sourceTable.getSourceId());
|
||||
DatacenterCatalog catalog = requireCatalog(sourceTable.getCatalogId());
|
||||
DatacenterTableField splitField = sourceTable.getFields().stream()
|
||||
.filter(field -> request.getFieldName().equals(field.getFieldName()))
|
||||
.findFirst()
|
||||
.orElseThrow(() -> new BusinessException("拆分字段不存在: " + request.getFieldName()));
|
||||
String prefix = resolveSplitPrefix(request, sourceTable.getTableName());
|
||||
Map<String, DatacenterTable> targets = new LinkedHashMap<>();
|
||||
List<BigInteger> derivedIds = new ArrayList<>();
|
||||
long totalRows = iterateRows(buildFullQuery(sourceTable), row -> {
|
||||
String fieldValue = stringify(row.get(splitField.getFieldName()));
|
||||
String bucket = fieldValue == null || fieldValue.isBlank() ? "empty" : fieldValue;
|
||||
DatacenterTable targetTable = targets.get(bucket);
|
||||
if (targetTable == null) {
|
||||
targetTable = createDerivedTable(source, catalog, cloneFields(sourceTable.getFields()),
|
||||
prefix + "_" + normalizeIdentifier(bucket), "SPLIT_BY_FIELD_VALUE",
|
||||
Map.of("sourceTableId", sourceTable.getId(), "fieldName", splitField.getFieldName(), "fieldValue", bucket), account);
|
||||
createLineage(sourceTable.getId(), targetTable.getId(), "SPLIT_BY_FIELD_VALUE",
|
||||
Map.of("sourceTableId", sourceTable.getId(), "fieldName", splitField.getFieldName(), "fieldValue", bucket), account);
|
||||
targets.put(bucket, targetTable);
|
||||
derivedIds.add(targetTable.getId());
|
||||
}
|
||||
saveToTable(targetTable, mapRow(row), account);
|
||||
});
|
||||
finishJobSuccess(job, totalRows, totalRows, Map.of("derivedTableIds", derivedIds));
|
||||
return job;
|
||||
}
|
||||
|
||||
private DatacenterImportJob doVerticalMerge(DatacenterExcelMergeRequest request, LoginAccount account, DatacenterImportJob job) {
|
||||
List<DatacenterTable> tables = request.getDatasetRefs().stream().map(this::resolveTable).toList();
|
||||
DatacenterTable firstTable = tables.get(0);
|
||||
DatacenterSource source = registryService.getSourceRequired(firstTable.getSourceId());
|
||||
DatacenterCatalog catalog = requireCatalog(firstTable.getCatalogId());
|
||||
assertSameCatalog(tables);
|
||||
assertSameFields(tables);
|
||||
DatacenterTable targetTable = createDerivedTable(source, catalog, cloneFields(firstTable.getFields()),
|
||||
request.getTargetTableName(), "MERGE_VERTICAL", Map.of("sourceTableIds", tables.stream().map(DatacenterTable::getId).toList()), account);
|
||||
long successRows = 0L;
|
||||
for (DatacenterTable table : tables) {
|
||||
successRows += copyRows(buildFullQuery(table), this::mapRow, targetTable, account);
|
||||
createLineage(table.getId(), targetTable.getId(), "MERGE_VERTICAL", Map.of("sourceTableId", table.getId()), account);
|
||||
}
|
||||
job.setTableId(targetTable.getId());
|
||||
finishJobSuccess(job, successRows, successRows, Map.of("derivedTableId", targetTable.getId()));
|
||||
return job;
|
||||
}
|
||||
|
||||
private DatacenterImportJob doHorizontalMerge(DatacenterExcelMergeRequest request, LoginAccount account, DatacenterImportJob job) {
|
||||
if (request.getJoinKey() == null || request.getJoinKey().isBlank()) {
|
||||
throw new BusinessException("横向合并必须指定 joinKey");
|
||||
}
|
||||
List<DatacenterTable> tables = request.getDatasetRefs().stream().map(this::resolveTable).toList();
|
||||
DatacenterTable firstTable = tables.get(0);
|
||||
DatacenterSource source = registryService.getSourceRequired(firstTable.getSourceId());
|
||||
DatacenterCatalog catalog = requireCatalog(firstTable.getCatalogId());
|
||||
assertSameCatalog(tables);
|
||||
|
||||
List<DatacenterTableField> mergedFields = new ArrayList<>();
|
||||
Set<String> usedFieldNames = new LinkedHashSet<>();
|
||||
Map<BigInteger, Map<String, String>> fieldMappings = new LinkedHashMap<>();
|
||||
for (DatacenterTable table : tables) {
|
||||
Map<String, String> mapping = new LinkedHashMap<>();
|
||||
for (DatacenterTableField field : table.getFields()) {
|
||||
String targetFieldName;
|
||||
if (field.getFieldName().equals(request.getJoinKey())) {
|
||||
targetFieldName = field.getFieldName();
|
||||
} else {
|
||||
targetFieldName = field.getFieldName();
|
||||
if (usedFieldNames.contains(targetFieldName)) {
|
||||
targetFieldName = normalizeIdentifier(table.getTableName()) + "_" + targetFieldName;
|
||||
}
|
||||
}
|
||||
if (!usedFieldNames.contains(targetFieldName)) {
|
||||
usedFieldNames.add(targetFieldName);
|
||||
mergedFields.add(cloneField(field, targetFieldName, field.getFieldDesc()));
|
||||
}
|
||||
mapping.put(field.getFieldName(), targetFieldName);
|
||||
}
|
||||
fieldMappings.put(table.getId(), mapping);
|
||||
}
|
||||
|
||||
DatacenterTable targetTable = createDerivedTable(source, catalog, mergedFields,
|
||||
request.getTargetTableName(), "MERGE_HORIZONTAL", Map.of("sourceTableIds", tables.stream().map(DatacenterTable::getId).toList(), "joinKey", request.getJoinKey()), account);
|
||||
Map<String, JSONObject> mergedRows = new LinkedHashMap<>();
|
||||
for (DatacenterTable table : tables) {
|
||||
Map<String, String> mapping = fieldMappings.get(table.getId());
|
||||
iterateRows(buildFullQuery(table), row -> {
|
||||
String joinValue = stringify(row.get(request.getJoinKey()));
|
||||
if (joinValue == null || joinValue.isBlank()) {
|
||||
return;
|
||||
}
|
||||
JSONObject target = mergedRows.computeIfAbsent(joinValue, key -> new JSONObject());
|
||||
mapping.forEach((sourceField, targetField) -> target.put(targetField, row.get(sourceField)));
|
||||
});
|
||||
createLineage(table.getId(), targetTable.getId(), "MERGE_HORIZONTAL", Map.of("sourceTableId", table.getId(), "joinKey", request.getJoinKey()), account);
|
||||
}
|
||||
for (JSONObject row : mergedRows.values()) {
|
||||
saveToTable(targetTable, row, account);
|
||||
}
|
||||
job.setTableId(targetTable.getId());
|
||||
finishJobSuccess(job, (long) mergedRows.size(), (long) mergedRows.size(), Map.of("derivedTableId", targetTable.getId()));
|
||||
return job;
|
||||
}
|
||||
|
||||
private DatacenterImportJob createJob(String jobType, BigInteger sourceId, BigInteger catalogId, BigInteger tableId,
|
||||
String fileName, Map<String, Object> payload, LoginAccount account) {
|
||||
DatacenterImportJob job = new DatacenterImportJob();
|
||||
job.setSourceId(sourceId);
|
||||
job.setCatalogId(catalogId);
|
||||
job.setTableId(tableId);
|
||||
job.setTenantId(account == null || account.getTenantId() == null ? BigInteger.ZERO : account.getTenantId());
|
||||
job.setDeptId(account == null || account.getDeptId() == null ? BigInteger.ZERO : account.getDeptId());
|
||||
job.setJobType(jobType);
|
||||
job.setFileName(fileName);
|
||||
job.setStatus(DatacenterImportStatus.RUNNING.name());
|
||||
job.setPayloadJson(payload == null ? new LinkedHashMap<>() : new LinkedHashMap<>(payload));
|
||||
job.setStartedAt(new Date());
|
||||
job.setCreated(new Date());
|
||||
job.setModified(new Date());
|
||||
job.setCreatedBy(account == null ? BigInteger.ZERO : account.getId());
|
||||
job.setModifiedBy(account == null ? BigInteger.ZERO : account.getId());
|
||||
importJobMapper.insert(job);
|
||||
return job;
|
||||
}
|
||||
|
||||
private Map<String, Object> buildPayload(String key, Object value) {
|
||||
Map<String, Object> payload = new LinkedHashMap<>();
|
||||
payload.put(key, value);
|
||||
return payload;
|
||||
}
|
||||
|
||||
private void finishJobSuccess(DatacenterImportJob job, long totalRows, long successRows, Map<String, Object> payload) {
|
||||
job.setStatus(DatacenterImportStatus.SUCCESS.name());
|
||||
job.setTotalRows(totalRows);
|
||||
job.setSuccessRows(successRows);
|
||||
job.setErrorRows(Math.max(0L, totalRows - successRows));
|
||||
if (payload != null) {
|
||||
job.setPayloadJson(new LinkedHashMap<>(payload));
|
||||
}
|
||||
job.setFinishedAt(new Date());
|
||||
job.setModified(new Date());
|
||||
importJobMapper.update(job);
|
||||
}
|
||||
|
||||
private void finishJobFailure(DatacenterImportJob job, Exception ex) {
|
||||
job.setStatus(DatacenterImportStatus.FAILED.name());
|
||||
job.setErrorSummary(ex.getMessage());
|
||||
job.setFinishedAt(new Date());
|
||||
job.setModified(new Date());
|
||||
importJobMapper.update(job);
|
||||
}
|
||||
|
||||
private DatacenterTable resolveTable(DatasetRef datasetRef) {
|
||||
if (datasetRef == null || datasetRef.getTableId() == null) {
|
||||
throw new BusinessException("缺少数据集 tableId");
|
||||
}
|
||||
return registryService.getTableWithFields(datasetRef.getTableId());
|
||||
}
|
||||
|
||||
private DatacenterCatalog requireCatalog(BigInteger catalogId) {
|
||||
DatacenterCatalog catalog = registryService.getCatalogById(catalogId);
|
||||
if (catalog == null) {
|
||||
throw new BusinessException("目录不存在: " + catalogId);
|
||||
}
|
||||
return catalog;
|
||||
}
|
||||
|
||||
private DatacenterTable createDerivedTable(DatacenterSource source, DatacenterCatalog catalog, List<DatacenterTableField> fields,
|
||||
String tableName, String deriveType, Map<String, Object> config, LoginAccount account) {
|
||||
DatacenterTable table = new DatacenterTable();
|
||||
String resolvedName = uniqueTableName(source.getId(), catalog.getId(), normalizeLogicalName(tableName, deriveType));
|
||||
table.setTableName(resolvedName);
|
||||
table.setTableDesc(resolvedName);
|
||||
table.setActualTable(buildMaterializedTableName(source.getId(), Math.abs(Objects.hash(resolvedName, deriveType))));
|
||||
table.setMaterializedTable(table.getActualTable());
|
||||
table.setTableKind(DatacenterTableKind.DERIVED_TABLE.name());
|
||||
table.setAccessMode("READ_WRITE");
|
||||
table.setVersioningEnabled(1);
|
||||
table.setCapabilitiesJson(defaultExcelCapabilities());
|
||||
table.setFields(fields);
|
||||
|
||||
DatacenterTableDetailMeta detail = new DatacenterTableDetailMeta();
|
||||
detail.setTable(table);
|
||||
detail.setFields(fields);
|
||||
dbHandleManager.getDbHandler().createTable(table);
|
||||
DatacenterTable savedTable = registryService.registerTable(source, catalog, detail, account);
|
||||
savedTable.setFields(registryService.getFields(savedTable.getId()));
|
||||
createVersion(savedTable, deriveType.toLowerCase(Locale.ROOT), config, account);
|
||||
return savedTable;
|
||||
}
|
||||
|
||||
private DatacenterDatasetVersion createVersion(DatacenterTable table, String versionLabel, Map<String, Object> snapshot, LoginAccount account) {
|
||||
QueryWrapperWrapper wrapper = new QueryWrapperWrapper(table.getId());
|
||||
DatacenterDatasetVersion version = new DatacenterDatasetVersion();
|
||||
version.setTableId(table.getId());
|
||||
version.setTenantId(table.getTenantId());
|
||||
version.setDeptId(table.getDeptId());
|
||||
version.setVersionNo(wrapper.nextVersionNo(datasetVersionMapper));
|
||||
version.setVersionLabel(versionLabel);
|
||||
version.setMaterializedTable(table.getMaterializedTable());
|
||||
version.setSnapshotJson(snapshot == null ? new LinkedHashMap<>() : new LinkedHashMap<>(snapshot));
|
||||
version.setStatus(0);
|
||||
version.setCreated(new Date());
|
||||
version.setModified(new Date());
|
||||
version.setCreatedBy(account == null ? BigInteger.ZERO : account.getId());
|
||||
version.setModifiedBy(account == null ? BigInteger.ZERO : account.getId());
|
||||
datasetVersionMapper.insert(version);
|
||||
return version;
|
||||
}
|
||||
|
||||
private void createLineage(BigInteger sourceTableId, BigInteger derivedTableId, String deriveType, Map<String, Object> config, LoginAccount account) {
|
||||
DatacenterDerivedTable relation = new DatacenterDerivedTable();
|
||||
relation.setSourceTableId(sourceTableId);
|
||||
relation.setDerivedTableId(derivedTableId);
|
||||
relation.setDeriveType(deriveType);
|
||||
relation.setDeriveConfigJson(config == null ? new LinkedHashMap<>() : new LinkedHashMap<>(config));
|
||||
relation.setStatus(0);
|
||||
relation.setTenantId(account == null || account.getTenantId() == null ? BigInteger.ZERO : account.getTenantId());
|
||||
relation.setDeptId(account == null || account.getDeptId() == null ? BigInteger.ZERO : account.getDeptId());
|
||||
relation.setCreated(new Date());
|
||||
relation.setModified(new Date());
|
||||
relation.setCreatedBy(account == null ? BigInteger.ZERO : account.getId());
|
||||
relation.setModifiedBy(account == null ? BigInteger.ZERO : account.getId());
|
||||
derivedTableMapper.insert(relation);
|
||||
}
|
||||
|
||||
private long copyRows(DatacenterQueryRequest queryRequest, RowMapper mapper, DatacenterTable targetTable, LoginAccount account) {
|
||||
return iterateRows(queryRequest, row -> saveToTable(targetTable, mapper.map(row), account));
|
||||
}
|
||||
|
||||
private long iterateRows(DatacenterQueryRequest queryRequest, RowConsumer consumer) {
|
||||
long total = 0L;
|
||||
long pageNumber = 1L;
|
||||
while (true) {
|
||||
queryRequest.setPageNumber(pageNumber);
|
||||
queryRequest.setPageSize(QUERY_BATCH_SIZE);
|
||||
Page<Row> page = queryService.queryPage(queryRequest);
|
||||
if (page.getRecords() == null || page.getRecords().isEmpty()) {
|
||||
break;
|
||||
}
|
||||
for (Row row : page.getRecords()) {
|
||||
consumer.accept(row);
|
||||
total++;
|
||||
}
|
||||
if (page.getRecords().size() < QUERY_BATCH_SIZE) {
|
||||
break;
|
||||
}
|
||||
pageNumber++;
|
||||
}
|
||||
return total;
|
||||
}
|
||||
|
||||
private void saveToTable(DatacenterTable targetTable, JSONObject data, LoginAccount account) {
|
||||
dbHandleManager.getDbHandler().saveValue(targetTable, data, account);
|
||||
}
|
||||
|
||||
private DatacenterQueryRequest buildFullQuery(DatacenterTable table) {
|
||||
DatacenterQueryRequest queryRequest = new DatacenterQueryRequest();
|
||||
queryRequest.setDatasetRef(registryService.resolveDatasetRef(table.getId()));
|
||||
queryRequest.setSelectedColumns(table.getFields().stream().map(DatacenterTableField::getFieldName).toList());
|
||||
return queryRequest;
|
||||
}
|
||||
|
||||
private JSONObject mapRow(Row row) {
|
||||
JSONObject payload = new JSONObject();
|
||||
row.forEach(payload::put);
|
||||
payload.remove("id");
|
||||
payload.remove("dept_id");
|
||||
payload.remove("tenant_id");
|
||||
payload.remove("created");
|
||||
payload.remove("created_by");
|
||||
payload.remove("modified");
|
||||
payload.remove("modified_by");
|
||||
payload.remove("remark");
|
||||
return payload;
|
||||
}
|
||||
|
||||
private JSONObject mapDerivedRow(Row row, List<String> selectedColumns, DatacenterExcelDeriveRequest request) {
|
||||
JSONObject payload = new JSONObject();
|
||||
for (String column : selectedColumns) {
|
||||
String targetName = request.getRenameMappings().getOrDefault(column, column);
|
||||
payload.put(targetName, row.get(column));
|
||||
}
|
||||
return payload;
|
||||
}
|
||||
|
||||
private List<String> resolveSelectedColumns(DatacenterTable sourceTable, List<String> selectedColumns) {
|
||||
if (CollectionUtils.isEmpty(selectedColumns)) {
|
||||
return sourceTable.getFields().stream().map(DatacenterTableField::getFieldName).toList();
|
||||
}
|
||||
return selectedColumns;
|
||||
}
|
||||
|
||||
private List<DatacenterTableField> buildDerivedFields(DatacenterTable sourceTable, DatacenterExcelDeriveRequest request) {
|
||||
List<String> selectedColumns = resolveSelectedColumns(sourceTable, request.getSelectedColumns());
|
||||
Map<String, DatacenterTableField> fieldMap = new LinkedHashMap<>();
|
||||
for (DatacenterTableField field : sourceTable.getFields()) {
|
||||
fieldMap.put(field.getFieldName(), field);
|
||||
}
|
||||
List<DatacenterTableField> fields = new ArrayList<>();
|
||||
for (String column : selectedColumns) {
|
||||
DatacenterTableField sourceField = fieldMap.get(column);
|
||||
if (sourceField == null) {
|
||||
throw new BusinessException("派生字段不存在: " + column);
|
||||
}
|
||||
String targetName = request.getRenameMappings().getOrDefault(column, column);
|
||||
fields.add(cloneField(sourceField, targetName, targetName));
|
||||
}
|
||||
return fields;
|
||||
}
|
||||
|
||||
private List<DatacenterTableField> cloneFields(List<DatacenterTableField> sourceFields) {
|
||||
List<DatacenterTableField> fields = new ArrayList<>();
|
||||
for (DatacenterTableField field : sourceFields) {
|
||||
fields.add(cloneField(field, field.getFieldName(), field.getFieldDesc()));
|
||||
}
|
||||
return fields;
|
||||
}
|
||||
|
||||
private DatacenterTableField cloneField(DatacenterTableField source, String fieldName, String fieldDesc) {
|
||||
DatacenterTableField field = new DatacenterTableField();
|
||||
field.setFieldName(fieldName);
|
||||
field.setSourceColumnName(source.getSourceColumnName());
|
||||
field.setFieldDesc(fieldDesc);
|
||||
field.setFieldType(source.getFieldType());
|
||||
field.setJdbcType(source.getJdbcType());
|
||||
field.setPrecision(source.getPrecision());
|
||||
field.setScale(source.getScale());
|
||||
field.setRequired(source.getRequired());
|
||||
field.setQueryable(source.getQueryable());
|
||||
field.setSortable(source.getSortable());
|
||||
field.setWritable(source.getWritable());
|
||||
field.setIndexed(source.getIndexed());
|
||||
field.setOptions(source.getOptions());
|
||||
return field;
|
||||
}
|
||||
|
||||
private Map<String, Object> defaultExcelCapabilities() {
|
||||
return Map.of("capabilities", List.of("READ_QUERY", "WRITE_MUTATION", "MATERIALIZE", "EXPORT"));
|
||||
}
|
||||
|
||||
private void assertSameCatalog(List<DatacenterTable> tables) {
|
||||
Set<BigInteger> catalogIds = new HashSet<>();
|
||||
Set<BigInteger> sourceIds = new HashSet<>();
|
||||
for (DatacenterTable table : tables) {
|
||||
catalogIds.add(table.getCatalogId());
|
||||
sourceIds.add(table.getSourceId());
|
||||
}
|
||||
if (catalogIds.size() > 1 || sourceIds.size() > 1) {
|
||||
throw new BusinessException("Excel 操作暂只支持同一 workbook/catalog 下的数据集");
|
||||
}
|
||||
}
|
||||
|
||||
private void assertSameFields(List<DatacenterTable> tables) {
|
||||
List<String> first = tables.get(0).getFields().stream().map(DatacenterTableField::getFieldName).toList();
|
||||
for (int i = 1; i < tables.size(); i++) {
|
||||
List<String> current = tables.get(i).getFields().stream().map(DatacenterTableField::getFieldName).toList();
|
||||
if (!first.equals(current)) {
|
||||
throw new BusinessException("纵向合并仅支持同结构表");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private List<DatacenterTable> resolveExportTables(DatacenterExcelExportRequest request) {
|
||||
List<DatacenterTable> tables = new ArrayList<>();
|
||||
if (request != null && !CollectionUtils.isEmpty(request.getDatasetRefs())) {
|
||||
for (DatasetRef datasetRef : request.getDatasetRefs()) {
|
||||
tables.add(resolveTable(datasetRef));
|
||||
}
|
||||
return tables;
|
||||
}
|
||||
if (request == null || request.getSourceId() == null) {
|
||||
throw new BusinessException("导出需要 sourceId 或 datasetRefs");
|
||||
}
|
||||
tables.addAll(registryService.listManagedTables(request.getSourceId(), request.getCatalogId()));
|
||||
return tables.stream().map(table -> registryService.getTableWithFields(table.getId())).toList();
|
||||
}
|
||||
|
||||
private void writeHeaderRow(org.apache.poi.ss.usermodel.Sheet sheet, List<DatacenterTableField> fields) {
|
||||
org.apache.poi.ss.usermodel.Row headerRow = sheet.createRow(0);
|
||||
for (int i = 0; i < fields.size(); i++) {
|
||||
Cell cell = headerRow.createCell(i);
|
||||
cell.setCellValue(fields.get(i).getFieldDesc());
|
||||
}
|
||||
}
|
||||
|
||||
private Path ensureExportDir() throws Exception {
|
||||
Path dir = Path.of(System.getProperty("java.io.tmpdir"), "easyflow-datacenter", "exports");
|
||||
Files.createDirectories(dir);
|
||||
return dir;
|
||||
}
|
||||
|
||||
private String buildExportFileName(String rawFileName) {
|
||||
String baseName = rawFileName == null || rawFileName.isBlank() ? "excel_export" : extractWorkbookName(rawFileName);
|
||||
return normalizeIdentifier(baseName) + "_" + EXPORT_TIME_FORMAT.format(LocalDateTime.now()) + ".xlsx";
|
||||
}
|
||||
|
||||
private String uniqueSheetName(String rawName, Set<String> usedSheetNames) {
|
||||
String base = rawName == null || rawName.isBlank() ? "Sheet" : rawName;
|
||||
base = base.length() > 31 ? base.substring(0, 31) : base;
|
||||
String result = base;
|
||||
int suffix = 1;
|
||||
while (usedSheetNames.contains(result)) {
|
||||
String suffixText = "_" + suffix++;
|
||||
int limit = Math.max(1, 31 - suffixText.length());
|
||||
result = base.substring(0, Math.min(base.length(), limit)) + suffixText;
|
||||
}
|
||||
usedSheetNames.add(result);
|
||||
return result;
|
||||
}
|
||||
|
||||
private List<DatacenterTableField> buildFields(org.apache.poi.ss.usermodel.Row headerRow, DataFormatter formatter) {
|
||||
List<DatacenterTableField> fields = new ArrayList<>();
|
||||
Set<String> usedNames = new HashSet<>();
|
||||
short lastCellNum = headerRow.getLastCellNum();
|
||||
for (int cellIndex = 0; cellIndex < lastCellNum; cellIndex++) {
|
||||
String header = formatter.formatCellValue(headerRow.getCell(cellIndex));
|
||||
String fieldName = normalizeIdentifier(header, cellIndex, usedNames);
|
||||
DatacenterTableField field = new DatacenterTableField();
|
||||
field.setFieldName(fieldName);
|
||||
field.setSourceColumnName(header);
|
||||
field.setFieldDesc(header == null || header.isBlank() ? fieldName : header);
|
||||
field.setFieldType(EnumFieldType.STRING.getCode());
|
||||
field.setJdbcType("VARCHAR");
|
||||
field.setPrecision(255);
|
||||
field.setScale(0);
|
||||
field.setRequired(0);
|
||||
field.setQueryable(1);
|
||||
field.setSortable(1);
|
||||
field.setWritable(1);
|
||||
field.setIndexed(0);
|
||||
fields.add(field);
|
||||
}
|
||||
return fields;
|
||||
}
|
||||
|
||||
private String normalizeMode(String value, String defaultValue) {
|
||||
return value == null || value.isBlank() ? defaultValue : value.trim().toUpperCase(Locale.ROOT);
|
||||
}
|
||||
|
||||
private String resolveSplitPrefix(DatacenterExcelSplitRequest request, String fallback) {
|
||||
if (request != null && request.getTargetNamePrefix() != null && !request.getTargetNamePrefix().isBlank()) {
|
||||
return request.getTargetNamePrefix();
|
||||
}
|
||||
return fallback + "_split";
|
||||
}
|
||||
|
||||
private String normalizeLogicalName(String tableName, String deriveType) {
|
||||
if (tableName != null && !tableName.isBlank()) {
|
||||
return tableName;
|
||||
}
|
||||
return deriveType.toLowerCase(Locale.ROOT) + "_" + System.currentTimeMillis();
|
||||
}
|
||||
|
||||
private String uniqueTableName(BigInteger sourceId, BigInteger catalogId, String rawName) {
|
||||
String baseName = rawName == null || rawName.isBlank() ? "dataset" : rawName;
|
||||
baseName = baseName.trim();
|
||||
String result = baseName;
|
||||
int suffix = 1;
|
||||
while (tableNameExists(sourceId, catalogId, result)) {
|
||||
result = baseName + "_" + suffix++;
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
private boolean tableNameExists(BigInteger sourceId, BigInteger catalogId, String tableName) {
|
||||
List<DatacenterTable> tables = registryService.listManagedTables(sourceId, catalogId);
|
||||
return tables.stream().anyMatch(table -> tableName.equals(table.getTableName()));
|
||||
}
|
||||
|
||||
private String extractWorkbookName(String originalFileName) {
|
||||
if (originalFileName == null || originalFileName.isBlank()) {
|
||||
return "excel_workbook";
|
||||
}
|
||||
int index = originalFileName.lastIndexOf('.');
|
||||
return index > 0 ? originalFileName.substring(0, index) : originalFileName;
|
||||
}
|
||||
|
||||
private String buildMaterializedTableName(BigInteger sourceId, int sheetIndex) {
|
||||
long snowId = new SnowFlakeIDKeyGenerator().nextId();
|
||||
return "tb_excel_" + sourceId + "_" + sheetIndex + "_" + snowId;
|
||||
}
|
||||
|
||||
private String normalizeIdentifier(String raw) {
|
||||
if (raw == null || raw.isBlank()) {
|
||||
return "value";
|
||||
}
|
||||
String normalized = raw.trim().toLowerCase(Locale.ROOT).replaceAll("[^a-z0-9_\\u4e00-\\u9fa5]+", "_");
|
||||
normalized = normalized.replaceAll("_+", "_");
|
||||
if (normalized.isBlank()) {
|
||||
return "value";
|
||||
}
|
||||
return normalized;
|
||||
}
|
||||
|
||||
private String normalizeIdentifier(String raw, int index, Set<String> usedNames) {
|
||||
String value = normalizeIdentifier(raw);
|
||||
if (value.isBlank() || "value".equals(value)) {
|
||||
value = "col_" + (index + 1);
|
||||
}
|
||||
if (Character.isDigit(value.charAt(0))) {
|
||||
value = "col_" + value;
|
||||
}
|
||||
String result = value;
|
||||
int suffix = 1;
|
||||
while (usedNames.contains(result)) {
|
||||
result = value + "_" + suffix++;
|
||||
}
|
||||
usedNames.add(result);
|
||||
return result;
|
||||
}
|
||||
|
||||
private String stringify(Object value) {
|
||||
return value == null ? null : String.valueOf(value);
|
||||
}
|
||||
|
||||
private static final class Holder {
|
||||
private DatacenterTable targetTable;
|
||||
private int batchNo;
|
||||
private int currentSize;
|
||||
}
|
||||
|
||||
private interface RowConsumer {
|
||||
void accept(Row row);
|
||||
}
|
||||
|
||||
private interface RowMapper {
|
||||
JSONObject map(Row row);
|
||||
}
|
||||
|
||||
private static final class QueryWrapperWrapper {
|
||||
private final BigInteger tableId;
|
||||
|
||||
private QueryWrapperWrapper(BigInteger tableId) {
|
||||
this.tableId = tableId;
|
||||
}
|
||||
|
||||
private int nextVersionNo(DatacenterDatasetVersionMapper mapper) {
|
||||
return mapper.selectListByQuery(com.mybatisflex.core.query.QueryWrapper.create()
|
||||
.eq(DatacenterDatasetVersion::getTableId, tableId)
|
||||
.orderBy("version_no desc"))
|
||||
.stream()
|
||||
.findFirst()
|
||||
.map(version -> version.getVersionNo() + 1)
|
||||
.orElse(1);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,24 @@
|
||||
package tech.easyflow.datacenter.execution.model;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
public class DatacenterConnectionTestResult {
|
||||
private boolean success;
|
||||
private String errorCode;
|
||||
private String message;
|
||||
private List<String> capabilities = new ArrayList<>();
|
||||
private Map<String, Object> details;
|
||||
|
||||
public boolean isSuccess() { return success; }
|
||||
public void setSuccess(boolean success) { this.success = success; }
|
||||
public String getErrorCode() { return errorCode; }
|
||||
public void setErrorCode(String errorCode) { this.errorCode = errorCode; }
|
||||
public String getMessage() { return message; }
|
||||
public void setMessage(String message) { this.message = message; }
|
||||
public List<String> getCapabilities() { return capabilities; }
|
||||
public void setCapabilities(List<String> capabilities) { this.capabilities = capabilities; }
|
||||
public Map<String, Object> getDetails() { return details; }
|
||||
public void setDetails(Map<String, Object> details) { this.details = details; }
|
||||
}
|
||||
@@ -0,0 +1,19 @@
|
||||
package tech.easyflow.datacenter.execution.model;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
public class DatacenterQueryFilter {
|
||||
private String column;
|
||||
private String operator;
|
||||
private Object value;
|
||||
private List<Object> values;
|
||||
|
||||
public String getColumn() { return column; }
|
||||
public void setColumn(String column) { this.column = column; }
|
||||
public String getOperator() { return operator; }
|
||||
public void setOperator(String operator) { this.operator = operator; }
|
||||
public Object getValue() { return value; }
|
||||
public void setValue(Object value) { this.value = value; }
|
||||
public List<Object> getValues() { return values; }
|
||||
public void setValues(List<Object> values) { this.values = values; }
|
||||
}
|
||||
@@ -0,0 +1,29 @@
|
||||
package tech.easyflow.datacenter.execution.model;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
public class DatacenterQueryRequest {
|
||||
private DatasetRef datasetRef;
|
||||
private Long pageNumber = 1L;
|
||||
private Long pageSize = 10L;
|
||||
private List<DatacenterQueryFilter> filters = new ArrayList<>();
|
||||
private List<DatacenterQuerySort> sorts = new ArrayList<>();
|
||||
private List<String> selectedColumns = new ArrayList<>();
|
||||
private String where;
|
||||
|
||||
public DatasetRef getDatasetRef() { return datasetRef; }
|
||||
public void setDatasetRef(DatasetRef datasetRef) { this.datasetRef = datasetRef; }
|
||||
public Long getPageNumber() { return pageNumber; }
|
||||
public void setPageNumber(Long pageNumber) { this.pageNumber = pageNumber; }
|
||||
public Long getPageSize() { return pageSize; }
|
||||
public void setPageSize(Long pageSize) { this.pageSize = pageSize; }
|
||||
public List<DatacenterQueryFilter> getFilters() { return filters; }
|
||||
public void setFilters(List<DatacenterQueryFilter> filters) { this.filters = filters; }
|
||||
public List<DatacenterQuerySort> getSorts() { return sorts; }
|
||||
public void setSorts(List<DatacenterQuerySort> sorts) { this.sorts = sorts; }
|
||||
public List<String> getSelectedColumns() { return selectedColumns; }
|
||||
public void setSelectedColumns(List<String> selectedColumns) { this.selectedColumns = selectedColumns; }
|
||||
public String getWhere() { return where; }
|
||||
public void setWhere(String where) { this.where = where; }
|
||||
}
|
||||
@@ -0,0 +1,11 @@
|
||||
package tech.easyflow.datacenter.execution.model;
|
||||
|
||||
public class DatacenterQuerySort {
|
||||
private String column;
|
||||
private String direction;
|
||||
|
||||
public String getColumn() { return column; }
|
||||
public void setColumn(String column) { this.column = column; }
|
||||
public String getDirection() { return direction; }
|
||||
public void setDirection(String direction) { this.direction = direction; }
|
||||
}
|
||||
@@ -0,0 +1,39 @@
|
||||
package tech.easyflow.datacenter.execution.model;
|
||||
|
||||
import tech.easyflow.datacenter.entity.DatacenterTable;
|
||||
import tech.easyflow.datacenter.entity.DatacenterTableField;
|
||||
import tech.easyflow.datacenter.meta.entity.DatacenterCatalog;
|
||||
import tech.easyflow.datacenter.meta.entity.DatacenterDatasetVersion;
|
||||
import tech.easyflow.datacenter.meta.entity.DatacenterDerivedTable;
|
||||
import tech.easyflow.datacenter.meta.entity.DatacenterSource;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
public class DatacenterSchemaResponse {
|
||||
private DatasetRef datasetRef;
|
||||
private DatacenterSource source;
|
||||
private DatacenterCatalog catalog;
|
||||
private DatacenterTable table;
|
||||
private List<DatacenterTableField> fields = new ArrayList<>();
|
||||
private List<DatacenterDatasetVersion> versions = new ArrayList<>();
|
||||
private List<DatacenterDerivedTable> upstreamLineage = new ArrayList<>();
|
||||
private List<DatacenterDerivedTable> downstreamLineage = new ArrayList<>();
|
||||
|
||||
public DatasetRef getDatasetRef() { return datasetRef; }
|
||||
public void setDatasetRef(DatasetRef datasetRef) { this.datasetRef = datasetRef; }
|
||||
public DatacenterSource getSource() { return source; }
|
||||
public void setSource(DatacenterSource source) { this.source = source; }
|
||||
public DatacenterCatalog getCatalog() { return catalog; }
|
||||
public void setCatalog(DatacenterCatalog catalog) { this.catalog = catalog; }
|
||||
public DatacenterTable getTable() { return table; }
|
||||
public void setTable(DatacenterTable table) { this.table = table; }
|
||||
public List<DatacenterTableField> getFields() { return fields; }
|
||||
public void setFields(List<DatacenterTableField> fields) { this.fields = fields; }
|
||||
public List<DatacenterDatasetVersion> getVersions() { return versions; }
|
||||
public void setVersions(List<DatacenterDatasetVersion> versions) { this.versions = versions; }
|
||||
public List<DatacenterDerivedTable> getUpstreamLineage() { return upstreamLineage; }
|
||||
public void setUpstreamLineage(List<DatacenterDerivedTable> upstreamLineage) { this.upstreamLineage = upstreamLineage; }
|
||||
public List<DatacenterDerivedTable> getDownstreamLineage() { return downstreamLineage; }
|
||||
public void setDownstreamLineage(List<DatacenterDerivedTable> downstreamLineage) { this.downstreamLineage = downstreamLineage; }
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
package tech.easyflow.datacenter.execution.model;
|
||||
|
||||
public class DatacenterSqlQueryRequest {
|
||||
|
||||
private DatasetRef datasetRef;
|
||||
private String sql;
|
||||
|
||||
public DatasetRef getDatasetRef() {
|
||||
return datasetRef;
|
||||
}
|
||||
|
||||
public void setDatasetRef(DatasetRef datasetRef) {
|
||||
this.datasetRef = datasetRef;
|
||||
}
|
||||
|
||||
public String getSql() {
|
||||
return sql;
|
||||
}
|
||||
|
||||
public void setSql(String sql) {
|
||||
this.sql = sql;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,25 @@
|
||||
package tech.easyflow.datacenter.execution.model;
|
||||
|
||||
import java.math.BigInteger;
|
||||
|
||||
public class DatasetRef {
|
||||
private BigInteger sourceId;
|
||||
private BigInteger catalogId;
|
||||
private String catalogName;
|
||||
private BigInteger tableId;
|
||||
private String tableName;
|
||||
private BigInteger versionId;
|
||||
|
||||
public BigInteger getSourceId() { return sourceId; }
|
||||
public void setSourceId(BigInteger sourceId) { this.sourceId = sourceId; }
|
||||
public BigInteger getCatalogId() { return catalogId; }
|
||||
public void setCatalogId(BigInteger catalogId) { this.catalogId = catalogId; }
|
||||
public String getCatalogName() { return catalogName; }
|
||||
public void setCatalogName(String catalogName) { this.catalogName = catalogName; }
|
||||
public BigInteger getTableId() { return tableId; }
|
||||
public void setTableId(BigInteger tableId) { this.tableId = tableId; }
|
||||
public String getTableName() { return tableName; }
|
||||
public void setTableName(String tableName) { this.tableName = tableName; }
|
||||
public BigInteger getVersionId() { return versionId; }
|
||||
public void setVersionId(BigInteger versionId) { this.versionId = versionId; }
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
package tech.easyflow.datacenter.execution.service;
|
||||
|
||||
import com.mybatisflex.core.paginate.Page;
|
||||
import com.mybatisflex.core.row.Row;
|
||||
import tech.easyflow.datacenter.execution.model.DatacenterQueryRequest;
|
||||
import tech.easyflow.datacenter.execution.model.DatacenterSchemaResponse;
|
||||
import tech.easyflow.datacenter.execution.model.DatacenterSqlQueryRequest;
|
||||
import tech.easyflow.datacenter.execution.model.DatasetRef;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
public interface DatacenterDatasetQueryService {
|
||||
Page<Row> queryPage(DatacenterQueryRequest request);
|
||||
|
||||
List<Row> queryBySql(DatacenterSqlQueryRequest request);
|
||||
|
||||
DatacenterSchemaResponse getSchema(DatasetRef datasetRef);
|
||||
}
|
||||
@@ -0,0 +1,13 @@
|
||||
package tech.easyflow.datacenter.execution.service;
|
||||
|
||||
import com.alibaba.fastjson2.JSONObject;
|
||||
import tech.easyflow.common.entity.LoginAccount;
|
||||
import tech.easyflow.datacenter.execution.model.DatasetRef;
|
||||
|
||||
import java.math.BigInteger;
|
||||
|
||||
public interface DatacenterDatasetWriteService {
|
||||
void saveRow(DatasetRef datasetRef, JSONObject data, LoginAccount account);
|
||||
|
||||
void deleteRow(DatasetRef datasetRef, BigInteger id, LoginAccount account);
|
||||
}
|
||||
@@ -0,0 +1,311 @@
|
||||
package tech.easyflow.datacenter.execution.service.impl;
|
||||
|
||||
import com.mybatisflex.core.paginate.Page;
|
||||
import com.mybatisflex.core.query.QueryWrapper;
|
||||
import com.mybatisflex.core.row.Row;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.springframework.util.CollectionUtils;
|
||||
import org.springframework.util.StringUtils;
|
||||
import tech.easyflow.common.web.exceptions.BusinessException;
|
||||
import tech.easyflow.datacenter.connector.DatacenterConnector;
|
||||
import tech.easyflow.datacenter.connector.DatacenterConnectorRegistry;
|
||||
import tech.easyflow.datacenter.entity.DatacenterTable;
|
||||
import tech.easyflow.datacenter.entity.DatacenterTableField;
|
||||
import tech.easyflow.datacenter.execution.model.DatacenterQueryRequest;
|
||||
import tech.easyflow.datacenter.execution.model.DatacenterSchemaResponse;
|
||||
import tech.easyflow.datacenter.execution.model.DatacenterSqlQueryRequest;
|
||||
import tech.easyflow.datacenter.execution.model.DatasetRef;
|
||||
import tech.easyflow.datacenter.execution.service.DatacenterDatasetQueryService;
|
||||
import tech.easyflow.datacenter.mapper.DatacenterCatalogMapper;
|
||||
import tech.easyflow.datacenter.mapper.DatacenterDatasetVersionMapper;
|
||||
import tech.easyflow.datacenter.mapper.DatacenterDerivedTableMapper;
|
||||
import tech.easyflow.datacenter.mapper.DatacenterTableMapper;
|
||||
import tech.easyflow.datacenter.meta.entity.DatacenterCatalog;
|
||||
import tech.easyflow.datacenter.meta.entity.DatacenterDatasetVersion;
|
||||
import tech.easyflow.datacenter.meta.entity.DatacenterDerivedTable;
|
||||
import tech.easyflow.datacenter.meta.entity.DatacenterSource;
|
||||
import tech.easyflow.datacenter.meta.service.DatacenterDatasetRegistryService;
|
||||
import tech.easyflow.datacenter.utils.SqlSupportUtils;
|
||||
|
||||
import javax.annotation.Resource;
|
||||
import java.math.BigInteger;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
@Service
|
||||
public class DatacenterDatasetQueryServiceImpl implements DatacenterDatasetQueryService {
|
||||
|
||||
@Resource
|
||||
private DatacenterDatasetRegistryService registryService;
|
||||
@Resource
|
||||
private DatacenterConnectorRegistry connectorRegistry;
|
||||
@Resource
|
||||
private DatacenterTableMapper tableMapper;
|
||||
@Resource
|
||||
private DatacenterCatalogMapper catalogMapper;
|
||||
@Resource
|
||||
private DatacenterDatasetVersionMapper datasetVersionMapper;
|
||||
@Resource
|
||||
private DatacenterDerivedTableMapper derivedTableMapper;
|
||||
|
||||
@Override
|
||||
public Page<Row> queryPage(DatacenterQueryRequest request) {
|
||||
if (request == null || request.getDatasetRef() == null) {
|
||||
throw new BusinessException("datasetRef 不能为空");
|
||||
}
|
||||
normalizePage(request);
|
||||
DatacenterTable table = resolveTable(request.getDatasetRef());
|
||||
DatacenterSource source = registryService.getSourceRequired(table.getSourceId());
|
||||
DatacenterCatalog catalog = registryService.getCatalogById(table.getCatalogId());
|
||||
DatacenterTable queryTable = resolveQueryTable(table, request.getDatasetRef());
|
||||
validateRequest(queryTable, request, source);
|
||||
request.getDatasetRef().setSourceId(table.getSourceId());
|
||||
request.getDatasetRef().setCatalogId(table.getCatalogId());
|
||||
request.getDatasetRef().setTableId(table.getId());
|
||||
request.getDatasetRef().setTableName(table.getTableName());
|
||||
if (catalog != null) {
|
||||
request.getDatasetRef().setCatalogName(catalog.getCatalogName());
|
||||
}
|
||||
DatacenterConnector connector = connectorRegistry.getConnector(source.getSourceType());
|
||||
return connector.queryPage(source, queryTable, request);
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<Row> queryBySql(DatacenterSqlQueryRequest request) {
|
||||
if (request == null || request.getDatasetRef() == null) {
|
||||
throw new BusinessException("datasetRef 不能为空");
|
||||
}
|
||||
String sql = trimToNull(request.getSql());
|
||||
if (!StringUtils.hasText(sql)) {
|
||||
throw new BusinessException("SQL 不能为空");
|
||||
}
|
||||
DatasetRef datasetRef = request.getDatasetRef();
|
||||
if (datasetRef.getSourceId() == null) {
|
||||
throw new BusinessException("缺少连接服务配置");
|
||||
}
|
||||
DatacenterSource source = registryService.getSourceRequired(datasetRef.getSourceId());
|
||||
BigInteger catalogId = resolveRequestedCatalogId(datasetRef);
|
||||
List<DatacenterTable> managedTables = registryService.listManagedTables(datasetRef.getSourceId(), catalogId);
|
||||
if (CollectionUtils.isEmpty(managedTables)) {
|
||||
throw new BusinessException("当前连接下没有已接入表");
|
||||
}
|
||||
SqlSupportUtils.ResolvedSql resolvedSql = SqlSupportUtils.resolve(
|
||||
sql,
|
||||
managedTables.stream().map(this::toManagedSqlTable).toList()
|
||||
);
|
||||
DatacenterConnector connector = connectorRegistry.getConnector(source.getSourceType());
|
||||
return connector.queryBySql(source, resolvedSql.getExecutableSql());
|
||||
}
|
||||
|
||||
@Override
|
||||
public DatacenterSchemaResponse getSchema(DatasetRef datasetRef) {
|
||||
DatacenterTable table = resolveTable(datasetRef);
|
||||
DatacenterSource source = registryService.getSourceRequired(table.getSourceId());
|
||||
DatacenterCatalog catalog = registryService.getCatalogById(table.getCatalogId());
|
||||
DatacenterSchemaResponse response = new DatacenterSchemaResponse();
|
||||
response.setDatasetRef(registryService.resolveDatasetRef(table.getId()));
|
||||
response.setSource(source);
|
||||
response.setCatalog(catalog);
|
||||
response.setTable(table);
|
||||
response.setFields(table.getFields());
|
||||
response.setVersions(listVersions(table.getId()));
|
||||
response.setUpstreamLineage(listUpstream(table.getId()));
|
||||
response.setDownstreamLineage(listDownstream(table.getId()));
|
||||
return response;
|
||||
}
|
||||
|
||||
private DatacenterTable resolveTable(DatasetRef datasetRef) {
|
||||
if (datasetRef.getTableId() != null) {
|
||||
return registryService.getTableWithFields(datasetRef.getTableId());
|
||||
}
|
||||
if (datasetRef.getSourceId() == null || !StringUtils.hasText(datasetRef.getTableName())) {
|
||||
throw new BusinessException("缺少数据集定位信息");
|
||||
}
|
||||
QueryWrapper wrapper = QueryWrapper.create();
|
||||
wrapper.eq(DatacenterTable::getSourceId, datasetRef.getSourceId());
|
||||
wrapper.eq(DatacenterTable::getTableName, datasetRef.getTableName().trim());
|
||||
|
||||
boolean hasCatalogCondition = false;
|
||||
if (datasetRef.getCatalogId() != null) {
|
||||
wrapper.eq(DatacenterTable::getCatalogId, datasetRef.getCatalogId());
|
||||
hasCatalogCondition = true;
|
||||
} else if (StringUtils.hasText(datasetRef.getCatalogName())) {
|
||||
DatacenterCatalog catalog = resolveCatalog(datasetRef.getSourceId(), datasetRef.getCatalogName().trim());
|
||||
wrapper.eq(DatacenterTable::getCatalogId, catalog.getId());
|
||||
hasCatalogCondition = true;
|
||||
}
|
||||
|
||||
List<DatacenterTable> tables = tableMapper.selectListByQuery(wrapper);
|
||||
if (CollectionUtils.isEmpty(tables)) {
|
||||
throw new BusinessException("数据集不存在: " + datasetRef.getTableName());
|
||||
}
|
||||
if (!hasCatalogCondition && tables.size() > 1) {
|
||||
throw new BusinessException("数据集存在重名表,请指定库名: " + datasetRef.getTableName());
|
||||
}
|
||||
return registryService.getTableWithFields(tables.get(0).getId());
|
||||
}
|
||||
|
||||
private DatacenterCatalog resolveCatalog(java.math.BigInteger sourceId, String catalogName) {
|
||||
QueryWrapper wrapper = QueryWrapper.create();
|
||||
wrapper.eq(DatacenterCatalog::getSourceId, sourceId);
|
||||
wrapper.eq(DatacenterCatalog::getCatalogName, catalogName);
|
||||
List<DatacenterCatalog> catalogs = catalogMapper.selectListByQuery(wrapper);
|
||||
if (CollectionUtils.isEmpty(catalogs)) {
|
||||
throw new BusinessException("库不存在: " + catalogName);
|
||||
}
|
||||
if (catalogs.size() > 1) {
|
||||
throw new BusinessException("库存在重复配置,请检查: " + catalogName);
|
||||
}
|
||||
return catalogs.get(0);
|
||||
}
|
||||
|
||||
private BigInteger resolveRequestedCatalogId(DatasetRef datasetRef) {
|
||||
if (datasetRef == null) {
|
||||
return null;
|
||||
}
|
||||
if (datasetRef.getCatalogId() != null) {
|
||||
return datasetRef.getCatalogId();
|
||||
}
|
||||
if (StringUtils.hasText(datasetRef.getCatalogName())) {
|
||||
DatacenterCatalog catalog = resolveCatalog(datasetRef.getSourceId(), datasetRef.getCatalogName().trim());
|
||||
return catalog.getId();
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
private SqlSupportUtils.ManagedTable toManagedSqlTable(DatacenterTable table) {
|
||||
DatacenterCatalog catalog = registryService.getCatalogById(table.getCatalogId());
|
||||
return new SqlSupportUtils.ManagedTable(
|
||||
catalog == null ? null : catalog.getCatalogName(),
|
||||
table.getTableName(),
|
||||
resolvePhysicalTableName(table)
|
||||
);
|
||||
}
|
||||
|
||||
private String resolvePhysicalTableName(DatacenterTable table) {
|
||||
if (table == null) {
|
||||
return null;
|
||||
}
|
||||
if ("EXTERNAL_TABLE".equals(table.getTableKind()) || "EXTERNAL_VIEW".equals(table.getTableKind())) {
|
||||
return trimToNull(table.getActualTable()) == null ? table.getTableName() : table.getActualTable().trim();
|
||||
}
|
||||
String materializedTable = trimToNull(table.getMaterializedTable());
|
||||
if (materializedTable != null) {
|
||||
return materializedTable;
|
||||
}
|
||||
String actualTable = trimToNull(table.getActualTable());
|
||||
return actualTable != null ? actualTable : table.getTableName();
|
||||
}
|
||||
|
||||
private void normalizePage(DatacenterQueryRequest request) {
|
||||
if (request.getPageNumber() == null || request.getPageNumber() < 1L) {
|
||||
request.setPageNumber(1L);
|
||||
}
|
||||
if (request.getPageSize() == null || request.getPageSize() < 1L) {
|
||||
throw new BusinessException("pageSize 必须大于 0");
|
||||
}
|
||||
}
|
||||
|
||||
private DatacenterTable resolveQueryTable(DatacenterTable table, DatasetRef datasetRef) {
|
||||
if (datasetRef == null || datasetRef.getVersionId() == null) {
|
||||
return table;
|
||||
}
|
||||
DatacenterDatasetVersion version = datasetVersionMapper.selectOneById(datasetRef.getVersionId());
|
||||
if (version == null || !table.getId().equals(version.getTableId())) {
|
||||
throw new BusinessException("数据集版本不存在: " + datasetRef.getVersionId());
|
||||
}
|
||||
DatacenterTable queryTable = new DatacenterTable();
|
||||
queryTable.setId(table.getId());
|
||||
queryTable.setSourceId(table.getSourceId());
|
||||
queryTable.setCatalogId(table.getCatalogId());
|
||||
queryTable.setTableName(table.getTableName());
|
||||
queryTable.setTableDesc(table.getTableDesc());
|
||||
queryTable.setActualTable(table.getActualTable());
|
||||
queryTable.setMaterializedTable(version.getMaterializedTable());
|
||||
queryTable.setAccessMode(table.getAccessMode());
|
||||
queryTable.setTableKind(table.getTableKind());
|
||||
queryTable.setVersioningEnabled(table.getVersioningEnabled());
|
||||
queryTable.setCapabilitiesJson(table.getCapabilitiesJson());
|
||||
queryTable.setFields(table.getFields());
|
||||
return queryTable;
|
||||
}
|
||||
|
||||
private void validateRequest(DatacenterTable table, DatacenterQueryRequest request, DatacenterSource source) {
|
||||
Map<String, DatacenterTableField> fieldMap = new LinkedHashMap<>();
|
||||
for (DatacenterTableField field : table.getFields()) {
|
||||
fieldMap.put(field.getFieldName(), field);
|
||||
}
|
||||
if (!CollectionUtils.isEmpty(request.getSelectedColumns())) {
|
||||
for (String column : request.getSelectedColumns()) {
|
||||
DatacenterTableField field = fieldMap.get(column);
|
||||
if (field == null || !isEnabled(field.getQueryable())) {
|
||||
throw new BusinessException("字段不可查询: " + column);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
request.setSelectedColumns(
|
||||
table.getFields().stream()
|
||||
.filter(field -> isEnabled(field.getQueryable()))
|
||||
.map(DatacenterTableField::getFieldName)
|
||||
.toList()
|
||||
);
|
||||
}
|
||||
if (!CollectionUtils.isEmpty(request.getFilters())) {
|
||||
request.getFilters().forEach(filter -> {
|
||||
DatacenterTableField field = fieldMap.get(filter.getColumn());
|
||||
if (field == null || !isEnabled(field.getQueryable())) {
|
||||
throw new BusinessException("字段不可过滤: " + filter.getColumn());
|
||||
}
|
||||
});
|
||||
}
|
||||
if (!CollectionUtils.isEmpty(request.getSorts())) {
|
||||
request.getSorts().forEach(sort -> {
|
||||
DatacenterTableField field = fieldMap.get(sort.getColumn());
|
||||
if (field == null || !isEnabled(field.getSortable())) {
|
||||
throw new BusinessException("字段不可排序: " + sort.getColumn());
|
||||
}
|
||||
});
|
||||
}
|
||||
if (request.getWhere() != null && !request.getWhere().isBlank()) {
|
||||
boolean allowLegacyWhere = "PROJECT_MYSQL".equals(source.getSourceType())
|
||||
|| "MYSQL".equals(source.getSourceType())
|
||||
|| "POSTGRESQL".equals(source.getSourceType());
|
||||
if (!allowLegacyWhere) {
|
||||
throw new BusinessException("当前数据源仅支持结构化 DSL 查询");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private boolean isEnabled(Integer value) {
|
||||
return value == null || value == 1;
|
||||
}
|
||||
|
||||
private String trimToNull(String value) {
|
||||
if (!StringUtils.hasText(value)) {
|
||||
return null;
|
||||
}
|
||||
return value.trim();
|
||||
}
|
||||
|
||||
private List<DatacenterDatasetVersion> listVersions(java.math.BigInteger tableId) {
|
||||
QueryWrapper wrapper = QueryWrapper.create();
|
||||
wrapper.eq(DatacenterDatasetVersion::getTableId, tableId);
|
||||
wrapper.orderBy("version_no desc");
|
||||
return datasetVersionMapper.selectListByQuery(wrapper);
|
||||
}
|
||||
|
||||
private List<DatacenterDerivedTable> listUpstream(java.math.BigInteger tableId) {
|
||||
QueryWrapper wrapper = QueryWrapper.create();
|
||||
wrapper.eq(DatacenterDerivedTable::getDerivedTableId, tableId);
|
||||
wrapper.orderBy("created desc");
|
||||
return derivedTableMapper.selectListByQuery(wrapper);
|
||||
}
|
||||
|
||||
private List<DatacenterDerivedTable> listDownstream(java.math.BigInteger tableId) {
|
||||
QueryWrapper wrapper = QueryWrapper.create();
|
||||
wrapper.eq(DatacenterDerivedTable::getSourceTableId, tableId);
|
||||
wrapper.orderBy("created desc");
|
||||
return derivedTableMapper.selectListByQuery(wrapper);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,48 @@
|
||||
package tech.easyflow.datacenter.execution.service.impl;
|
||||
|
||||
import com.alibaba.fastjson2.JSONObject;
|
||||
import org.springframework.stereotype.Service;
|
||||
import tech.easyflow.common.entity.LoginAccount;
|
||||
import tech.easyflow.common.web.exceptions.BusinessException;
|
||||
import tech.easyflow.datacenter.connector.DatacenterConnector;
|
||||
import tech.easyflow.datacenter.connector.DatacenterConnectorRegistry;
|
||||
import tech.easyflow.datacenter.entity.DatacenterTable;
|
||||
import tech.easyflow.datacenter.execution.model.DatasetRef;
|
||||
import tech.easyflow.datacenter.meta.entity.DatacenterSource;
|
||||
import tech.easyflow.datacenter.meta.service.DatacenterDatasetRegistryService;
|
||||
import tech.easyflow.datacenter.execution.service.DatacenterDatasetWriteService;
|
||||
|
||||
import javax.annotation.Resource;
|
||||
import java.math.BigInteger;
|
||||
|
||||
@Service
|
||||
public class DatacenterDatasetWriteServiceImpl implements DatacenterDatasetWriteService {
|
||||
|
||||
@Resource
|
||||
private DatacenterDatasetRegistryService registryService;
|
||||
@Resource
|
||||
private DatacenterConnectorRegistry connectorRegistry;
|
||||
|
||||
@Override
|
||||
public void saveRow(DatasetRef datasetRef, JSONObject data, LoginAccount account) {
|
||||
DatacenterTable table = resolveTable(datasetRef);
|
||||
DatacenterSource source = registryService.getSourceRequired(table.getSourceId());
|
||||
DatacenterConnector connector = connectorRegistry.getConnector(source.getSourceType());
|
||||
connector.saveRow(source, table, data, account);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void deleteRow(DatasetRef datasetRef, BigInteger id, LoginAccount account) {
|
||||
DatacenterTable table = resolveTable(datasetRef);
|
||||
DatacenterSource source = registryService.getSourceRequired(table.getSourceId());
|
||||
DatacenterConnector connector = connectorRegistry.getConnector(source.getSourceType());
|
||||
connector.deleteRow(source, table, id, account);
|
||||
}
|
||||
|
||||
private DatacenterTable resolveTable(DatasetRef datasetRef) {
|
||||
if (datasetRef == null || datasetRef.getTableId() == null) {
|
||||
throw new BusinessException("缺少 tableId");
|
||||
}
|
||||
return registryService.getTableWithFields(datasetRef.getTableId());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,7 @@
|
||||
package tech.easyflow.datacenter.integration;
|
||||
|
||||
import tech.easyflow.datacenter.execution.model.DatacenterQueryRequest;
|
||||
|
||||
public interface AssistantDatacenterBridge {
|
||||
AssistantDatacenterResult queryPage(DatacenterQueryRequest request);
|
||||
}
|
||||
@@ -0,0 +1,34 @@
|
||||
package tech.easyflow.datacenter.integration;
|
||||
|
||||
import com.mybatisflex.core.row.Row;
|
||||
import tech.easyflow.datacenter.entity.DatacenterTable;
|
||||
import tech.easyflow.datacenter.meta.entity.DatacenterCatalog;
|
||||
import tech.easyflow.datacenter.meta.entity.DatacenterDatasetVersion;
|
||||
import tech.easyflow.datacenter.meta.entity.DatacenterSource;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
public class AssistantDatacenterResult {
|
||||
private List<Row> rows = new ArrayList<>();
|
||||
private DatacenterSource source;
|
||||
private DatacenterCatalog catalog;
|
||||
private DatacenterTable table;
|
||||
private DatacenterDatasetVersion version;
|
||||
private Map<String, Object> querySummary = new LinkedHashMap<>();
|
||||
|
||||
public List<Row> getRows() { return rows; }
|
||||
public void setRows(List<Row> rows) { this.rows = rows; }
|
||||
public DatacenterSource getSource() { return source; }
|
||||
public void setSource(DatacenterSource source) { this.source = source; }
|
||||
public DatacenterCatalog getCatalog() { return catalog; }
|
||||
public void setCatalog(DatacenterCatalog catalog) { this.catalog = catalog; }
|
||||
public DatacenterTable getTable() { return table; }
|
||||
public void setTable(DatacenterTable table) { this.table = table; }
|
||||
public DatacenterDatasetVersion getVersion() { return version; }
|
||||
public void setVersion(DatacenterDatasetVersion version) { this.version = version; }
|
||||
public Map<String, Object> getQuerySummary() { return querySummary; }
|
||||
public void setQuerySummary(Map<String, Object> querySummary) { this.querySummary = querySummary; }
|
||||
}
|
||||
@@ -0,0 +1,49 @@
|
||||
package tech.easyflow.datacenter.integration;
|
||||
|
||||
import org.springframework.stereotype.Component;
|
||||
import tech.easyflow.datacenter.execution.model.DatacenterQueryRequest;
|
||||
import tech.easyflow.datacenter.execution.model.DatacenterSchemaResponse;
|
||||
import tech.easyflow.datacenter.execution.service.DatacenterDatasetQueryService;
|
||||
import tech.easyflow.datacenter.meta.entity.DatacenterDatasetVersion;
|
||||
|
||||
import javax.annotation.Resource;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
|
||||
@Component
|
||||
public class DefaultAssistantDatacenterBridge implements AssistantDatacenterBridge {
|
||||
|
||||
@Resource
|
||||
private DatacenterDatasetQueryService queryService;
|
||||
|
||||
@Override
|
||||
public AssistantDatacenterResult queryPage(DatacenterQueryRequest request) {
|
||||
var page = queryService.queryPage(request);
|
||||
DatacenterSchemaResponse schema = queryService.getSchema(request.getDatasetRef());
|
||||
AssistantDatacenterResult result = new AssistantDatacenterResult();
|
||||
result.setRows(page.getRecords());
|
||||
result.setSource(schema.getSource());
|
||||
result.setCatalog(schema.getCatalog());
|
||||
result.setTable(schema.getTable());
|
||||
result.setVersion(resolveVersion(schema.getVersions(), request == null || request.getDatasetRef() == null ? null : request.getDatasetRef().getVersionId()));
|
||||
result.setQuerySummary(new LinkedHashMap<>() {{
|
||||
put("pageNumber", page.getPageNumber());
|
||||
put("pageSize", page.getPageSize());
|
||||
put("totalRows", page.getTotalRow());
|
||||
put("selectedColumns", request == null ? List.of() : request.getSelectedColumns());
|
||||
put("filterCount", request == null || request.getFilters() == null ? 0 : request.getFilters().size());
|
||||
put("sortCount", request == null || request.getSorts() == null ? 0 : request.getSorts().size());
|
||||
}});
|
||||
return result;
|
||||
}
|
||||
|
||||
private DatacenterDatasetVersion resolveVersion(List<DatacenterDatasetVersion> versions, java.math.BigInteger versionId) {
|
||||
if (versions == null || versions.isEmpty()) {
|
||||
return null;
|
||||
}
|
||||
if (versionId == null) {
|
||||
return versions.get(0);
|
||||
}
|
||||
return versions.stream().filter(version -> versionId.equals(version.getId())).findFirst().orElse(null);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,7 @@
|
||||
package tech.easyflow.datacenter.mapper;
|
||||
|
||||
import com.mybatisflex.core.BaseMapper;
|
||||
import tech.easyflow.datacenter.meta.entity.DatacenterCatalog;
|
||||
|
||||
public interface DatacenterCatalogMapper extends BaseMapper<DatacenterCatalog> {
|
||||
}
|
||||
@@ -0,0 +1,7 @@
|
||||
package tech.easyflow.datacenter.mapper;
|
||||
|
||||
import com.mybatisflex.core.BaseMapper;
|
||||
import tech.easyflow.datacenter.meta.entity.DatacenterDatasetVersion;
|
||||
|
||||
public interface DatacenterDatasetVersionMapper extends BaseMapper<DatacenterDatasetVersion> {
|
||||
}
|
||||
@@ -0,0 +1,7 @@
|
||||
package tech.easyflow.datacenter.mapper;
|
||||
|
||||
import com.mybatisflex.core.BaseMapper;
|
||||
import tech.easyflow.datacenter.meta.entity.DatacenterDerivedTable;
|
||||
|
||||
public interface DatacenterDerivedTableMapper extends BaseMapper<DatacenterDerivedTable> {
|
||||
}
|
||||
@@ -0,0 +1,7 @@
|
||||
package tech.easyflow.datacenter.mapper;
|
||||
|
||||
import com.mybatisflex.core.BaseMapper;
|
||||
import tech.easyflow.datacenter.meta.entity.DatacenterImportJob;
|
||||
|
||||
public interface DatacenterImportJobMapper extends BaseMapper<DatacenterImportJob> {
|
||||
}
|
||||
@@ -0,0 +1,7 @@
|
||||
package tech.easyflow.datacenter.mapper;
|
||||
|
||||
import com.mybatisflex.core.BaseMapper;
|
||||
import tech.easyflow.datacenter.meta.entity.DatacenterSource;
|
||||
|
||||
public interface DatacenterSourceMapper extends BaseMapper<DatacenterSource> {
|
||||
}
|
||||
@@ -0,0 +1,75 @@
|
||||
package tech.easyflow.datacenter.meta.entity;
|
||||
|
||||
import com.mybatisflex.annotation.Column;
|
||||
import com.mybatisflex.annotation.Id;
|
||||
import com.mybatisflex.annotation.KeyType;
|
||||
import com.mybatisflex.annotation.Table;
|
||||
import com.mybatisflex.core.handler.FastjsonTypeHandler;
|
||||
import tech.easyflow.common.entity.DateEntity;
|
||||
|
||||
import java.io.Serializable;
|
||||
import java.math.BigInteger;
|
||||
import java.util.Date;
|
||||
import java.util.Map;
|
||||
|
||||
@Table(value = "tb_datacenter_catalog", comment = "数据中心逻辑库/命名空间")
|
||||
public class DatacenterCatalog extends DateEntity implements Serializable {
|
||||
|
||||
@Id(keyType = KeyType.Generator, value = "snowFlakeId", comment = "主键")
|
||||
private BigInteger id;
|
||||
@Column(comment = "部门ID")
|
||||
private BigInteger deptId;
|
||||
@Column(tenantId = true, comment = "租户ID")
|
||||
private BigInteger tenantId;
|
||||
@Column(comment = "数据源ID")
|
||||
private BigInteger sourceId;
|
||||
@Column(comment = "目录名")
|
||||
private String catalogName;
|
||||
@Column(comment = "目录描述")
|
||||
private String catalogDesc;
|
||||
@Column(comment = "目录类型")
|
||||
private String catalogType;
|
||||
@Column(comment = "状态")
|
||||
private Integer status;
|
||||
@Column(typeHandler = FastjsonTypeHandler.class, comment = "扩展项")
|
||||
private Map<String, Object> options;
|
||||
@Column(comment = "创建时间")
|
||||
private Date created;
|
||||
@Column(comment = "创建人")
|
||||
private BigInteger createdBy;
|
||||
@Column(comment = "修改时间")
|
||||
private Date modified;
|
||||
@Column(comment = "修改人")
|
||||
private BigInteger modifiedBy;
|
||||
|
||||
public BigInteger getId() { return id; }
|
||||
public void setId(BigInteger id) { this.id = id; }
|
||||
public BigInteger getDeptId() { return deptId; }
|
||||
public void setDeptId(BigInteger deptId) { this.deptId = deptId; }
|
||||
public BigInteger getTenantId() { return tenantId; }
|
||||
public void setTenantId(BigInteger tenantId) { this.tenantId = tenantId; }
|
||||
public BigInteger getSourceId() { return sourceId; }
|
||||
public void setSourceId(BigInteger sourceId) { this.sourceId = sourceId; }
|
||||
public String getCatalogName() { return catalogName; }
|
||||
public void setCatalogName(String catalogName) { this.catalogName = catalogName; }
|
||||
public String getCatalogDesc() { return catalogDesc; }
|
||||
public void setCatalogDesc(String catalogDesc) { this.catalogDesc = catalogDesc; }
|
||||
public String getCatalogType() { return catalogType; }
|
||||
public void setCatalogType(String catalogType) { this.catalogType = catalogType; }
|
||||
public Integer getStatus() { return status; }
|
||||
public void setStatus(Integer status) { this.status = status; }
|
||||
public Map<String, Object> getOptions() { return options; }
|
||||
public void setOptions(Map<String, Object> options) { this.options = options; }
|
||||
@Override
|
||||
public Date getCreated() { return created; }
|
||||
@Override
|
||||
public void setCreated(Date created) { this.created = created; }
|
||||
public BigInteger getCreatedBy() { return createdBy; }
|
||||
public void setCreatedBy(BigInteger createdBy) { this.createdBy = createdBy; }
|
||||
@Override
|
||||
public Date getModified() { return modified; }
|
||||
@Override
|
||||
public void setModified(Date modified) { this.modified = modified; }
|
||||
public BigInteger getModifiedBy() { return modifiedBy; }
|
||||
public void setModifiedBy(BigInteger modifiedBy) { this.modifiedBy = modifiedBy; }
|
||||
}
|
||||
@@ -0,0 +1,70 @@
|
||||
package tech.easyflow.datacenter.meta.entity;
|
||||
|
||||
import com.mybatisflex.annotation.Column;
|
||||
import com.mybatisflex.annotation.Id;
|
||||
import com.mybatisflex.annotation.KeyType;
|
||||
import com.mybatisflex.annotation.Table;
|
||||
import com.mybatisflex.core.handler.FastjsonTypeHandler;
|
||||
import tech.easyflow.common.entity.DateEntity;
|
||||
|
||||
import java.io.Serializable;
|
||||
import java.math.BigInteger;
|
||||
import java.util.Date;
|
||||
import java.util.Map;
|
||||
|
||||
@Table(value = "tb_datacenter_dataset_version", comment = "数据集版本")
|
||||
public class DatacenterDatasetVersion extends DateEntity implements Serializable {
|
||||
@Id(keyType = KeyType.Generator, value = "snowFlakeId")
|
||||
private BigInteger id;
|
||||
@Column(comment = "部门ID")
|
||||
private BigInteger deptId;
|
||||
@Column(tenantId = true, comment = "租户ID")
|
||||
private BigInteger tenantId;
|
||||
@Column(comment = "表ID")
|
||||
private BigInteger tableId;
|
||||
@Column(comment = "版本号")
|
||||
private Integer versionNo;
|
||||
@Column(comment = "版本标签")
|
||||
private String versionLabel;
|
||||
@Column(comment = "物化表名")
|
||||
private String materializedTable;
|
||||
@Column(typeHandler = FastjsonTypeHandler.class, comment = "版本快照")
|
||||
private Map<String, Object> snapshotJson;
|
||||
@Column(comment = "状态")
|
||||
private Integer status;
|
||||
@Column(comment = "创建时间")
|
||||
private Date created;
|
||||
@Column(comment = "创建人")
|
||||
private BigInteger createdBy;
|
||||
@Column(comment = "修改时间")
|
||||
private Date modified;
|
||||
@Column(comment = "修改人")
|
||||
private BigInteger modifiedBy;
|
||||
|
||||
public BigInteger getId() { return id; }
|
||||
public void setId(BigInteger id) { this.id = id; }
|
||||
public BigInteger getDeptId() { return deptId; }
|
||||
public void setDeptId(BigInteger deptId) { this.deptId = deptId; }
|
||||
public BigInteger getTenantId() { return tenantId; }
|
||||
public void setTenantId(BigInteger tenantId) { this.tenantId = tenantId; }
|
||||
public BigInteger getTableId() { return tableId; }
|
||||
public void setTableId(BigInteger tableId) { this.tableId = tableId; }
|
||||
public Integer getVersionNo() { return versionNo; }
|
||||
public void setVersionNo(Integer versionNo) { this.versionNo = versionNo; }
|
||||
public String getVersionLabel() { return versionLabel; }
|
||||
public void setVersionLabel(String versionLabel) { this.versionLabel = versionLabel; }
|
||||
public String getMaterializedTable() { return materializedTable; }
|
||||
public void setMaterializedTable(String materializedTable) { this.materializedTable = materializedTable; }
|
||||
public Map<String, Object> getSnapshotJson() { return snapshotJson; }
|
||||
public void setSnapshotJson(Map<String, Object> snapshotJson) { this.snapshotJson = snapshotJson; }
|
||||
public Integer getStatus() { return status; }
|
||||
public void setStatus(Integer status) { this.status = status; }
|
||||
@Override public Date getCreated() { return created; }
|
||||
@Override public void setCreated(Date created) { this.created = created; }
|
||||
public BigInteger getCreatedBy() { return createdBy; }
|
||||
public void setCreatedBy(BigInteger createdBy) { this.createdBy = createdBy; }
|
||||
@Override public Date getModified() { return modified; }
|
||||
@Override public void setModified(Date modified) { this.modified = modified; }
|
||||
public BigInteger getModifiedBy() { return modifiedBy; }
|
||||
public void setModifiedBy(BigInteger modifiedBy) { this.modifiedBy = modifiedBy; }
|
||||
}
|
||||
@@ -0,0 +1,66 @@
|
||||
package tech.easyflow.datacenter.meta.entity;
|
||||
|
||||
import com.mybatisflex.annotation.Column;
|
||||
import com.mybatisflex.annotation.Id;
|
||||
import com.mybatisflex.annotation.KeyType;
|
||||
import com.mybatisflex.annotation.Table;
|
||||
import com.mybatisflex.core.handler.FastjsonTypeHandler;
|
||||
import tech.easyflow.common.entity.DateEntity;
|
||||
|
||||
import java.io.Serializable;
|
||||
import java.math.BigInteger;
|
||||
import java.util.Date;
|
||||
import java.util.Map;
|
||||
|
||||
@Table(value = "tb_datacenter_derived_table", comment = "数据中心派生表关系")
|
||||
public class DatacenterDerivedTable extends DateEntity implements Serializable {
|
||||
@Id(keyType = KeyType.Generator, value = "snowFlakeId")
|
||||
private BigInteger id;
|
||||
@Column(comment = "部门ID")
|
||||
private BigInteger deptId;
|
||||
@Column(tenantId = true, comment = "租户ID")
|
||||
private BigInteger tenantId;
|
||||
@Column(comment = "源表ID")
|
||||
private BigInteger sourceTableId;
|
||||
@Column(comment = "派生表ID")
|
||||
private BigInteger derivedTableId;
|
||||
@Column(comment = "派生类型")
|
||||
private String deriveType;
|
||||
@Column(typeHandler = FastjsonTypeHandler.class, comment = "派生配置")
|
||||
private Map<String, Object> deriveConfigJson;
|
||||
@Column(comment = "状态")
|
||||
private Integer status;
|
||||
@Column(comment = "创建时间")
|
||||
private Date created;
|
||||
@Column(comment = "创建人")
|
||||
private BigInteger createdBy;
|
||||
@Column(comment = "修改时间")
|
||||
private Date modified;
|
||||
@Column(comment = "修改人")
|
||||
private BigInteger modifiedBy;
|
||||
|
||||
public BigInteger getId() { return id; }
|
||||
public void setId(BigInteger id) { this.id = id; }
|
||||
public BigInteger getDeptId() { return deptId; }
|
||||
public void setDeptId(BigInteger deptId) { this.deptId = deptId; }
|
||||
public BigInteger getTenantId() { return tenantId; }
|
||||
public void setTenantId(BigInteger tenantId) { this.tenantId = tenantId; }
|
||||
public BigInteger getSourceTableId() { return sourceTableId; }
|
||||
public void setSourceTableId(BigInteger sourceTableId) { this.sourceTableId = sourceTableId; }
|
||||
public BigInteger getDerivedTableId() { return derivedTableId; }
|
||||
public void setDerivedTableId(BigInteger derivedTableId) { this.derivedTableId = derivedTableId; }
|
||||
public String getDeriveType() { return deriveType; }
|
||||
public void setDeriveType(String deriveType) { this.deriveType = deriveType; }
|
||||
public Map<String, Object> getDeriveConfigJson() { return deriveConfigJson; }
|
||||
public void setDeriveConfigJson(Map<String, Object> deriveConfigJson) { this.deriveConfigJson = deriveConfigJson; }
|
||||
public Integer getStatus() { return status; }
|
||||
public void setStatus(Integer status) { this.status = status; }
|
||||
@Override public Date getCreated() { return created; }
|
||||
@Override public void setCreated(Date created) { this.created = created; }
|
||||
public BigInteger getCreatedBy() { return createdBy; }
|
||||
public void setCreatedBy(BigInteger createdBy) { this.createdBy = createdBy; }
|
||||
@Override public Date getModified() { return modified; }
|
||||
@Override public void setModified(Date modified) { this.modified = modified; }
|
||||
public BigInteger getModifiedBy() { return modifiedBy; }
|
||||
public void setModifiedBy(BigInteger modifiedBy) { this.modifiedBy = modifiedBy; }
|
||||
}
|
||||
@@ -0,0 +1,102 @@
|
||||
package tech.easyflow.datacenter.meta.entity;
|
||||
|
||||
import com.mybatisflex.annotation.Column;
|
||||
import com.mybatisflex.annotation.Id;
|
||||
import com.mybatisflex.annotation.KeyType;
|
||||
import com.mybatisflex.annotation.Table;
|
||||
import com.mybatisflex.core.handler.FastjsonTypeHandler;
|
||||
import tech.easyflow.common.entity.DateEntity;
|
||||
|
||||
import java.io.Serializable;
|
||||
import java.math.BigInteger;
|
||||
import java.util.Date;
|
||||
import java.util.Map;
|
||||
|
||||
@Table(value = "tb_datacenter_import_job", comment = "数据中心导入任务")
|
||||
public class DatacenterImportJob extends DateEntity implements Serializable {
|
||||
@Id(keyType = KeyType.Generator, value = "snowFlakeId")
|
||||
private BigInteger id;
|
||||
@Column(comment = "部门ID")
|
||||
private BigInteger deptId;
|
||||
@Column(tenantId = true, comment = "租户ID")
|
||||
private BigInteger tenantId;
|
||||
@Column(comment = "数据源ID")
|
||||
private BigInteger sourceId;
|
||||
@Column(comment = "目录ID")
|
||||
private BigInteger catalogId;
|
||||
@Column(comment = "表ID")
|
||||
private BigInteger tableId;
|
||||
@Column(comment = "任务类型")
|
||||
private String jobType;
|
||||
@Column(comment = "文件名")
|
||||
private String fileName;
|
||||
@Column(comment = "文件存储路径")
|
||||
private String storagePath;
|
||||
@Column(comment = "任务状态")
|
||||
private String status;
|
||||
@Column(comment = "总行数")
|
||||
private Long totalRows;
|
||||
@Column(comment = "成功行数")
|
||||
private Long successRows;
|
||||
@Column(comment = "失败行数")
|
||||
private Long errorRows;
|
||||
@Column(comment = "错误摘要")
|
||||
private String errorSummary;
|
||||
@Column(typeHandler = FastjsonTypeHandler.class, comment = "任务载荷")
|
||||
private Map<String, Object> payloadJson;
|
||||
@Column(comment = "开始时间")
|
||||
private Date startedAt;
|
||||
@Column(comment = "结束时间")
|
||||
private Date finishedAt;
|
||||
@Column(comment = "创建时间")
|
||||
private Date created;
|
||||
@Column(comment = "创建人")
|
||||
private BigInteger createdBy;
|
||||
@Column(comment = "修改时间")
|
||||
private Date modified;
|
||||
@Column(comment = "修改人")
|
||||
private BigInteger modifiedBy;
|
||||
|
||||
public BigInteger getId() { return id; }
|
||||
public void setId(BigInteger id) { this.id = id; }
|
||||
public BigInteger getDeptId() { return deptId; }
|
||||
public void setDeptId(BigInteger deptId) { this.deptId = deptId; }
|
||||
public BigInteger getTenantId() { return tenantId; }
|
||||
public void setTenantId(BigInteger tenantId) { this.tenantId = tenantId; }
|
||||
public BigInteger getSourceId() { return sourceId; }
|
||||
public void setSourceId(BigInteger sourceId) { this.sourceId = sourceId; }
|
||||
public BigInteger getCatalogId() { return catalogId; }
|
||||
public void setCatalogId(BigInteger catalogId) { this.catalogId = catalogId; }
|
||||
public BigInteger getTableId() { return tableId; }
|
||||
public void setTableId(BigInteger tableId) { this.tableId = tableId; }
|
||||
public String getJobType() { return jobType; }
|
||||
public void setJobType(String jobType) { this.jobType = jobType; }
|
||||
public String getFileName() { return fileName; }
|
||||
public void setFileName(String fileName) { this.fileName = fileName; }
|
||||
public String getStoragePath() { return storagePath; }
|
||||
public void setStoragePath(String storagePath) { this.storagePath = storagePath; }
|
||||
public String getStatus() { return status; }
|
||||
public void setStatus(String status) { this.status = status; }
|
||||
public Long getTotalRows() { return totalRows; }
|
||||
public void setTotalRows(Long totalRows) { this.totalRows = totalRows; }
|
||||
public Long getSuccessRows() { return successRows; }
|
||||
public void setSuccessRows(Long successRows) { this.successRows = successRows; }
|
||||
public Long getErrorRows() { return errorRows; }
|
||||
public void setErrorRows(Long errorRows) { this.errorRows = errorRows; }
|
||||
public String getErrorSummary() { return errorSummary; }
|
||||
public void setErrorSummary(String errorSummary) { this.errorSummary = errorSummary; }
|
||||
public Map<String, Object> getPayloadJson() { return payloadJson; }
|
||||
public void setPayloadJson(Map<String, Object> payloadJson) { this.payloadJson = payloadJson; }
|
||||
public Date getStartedAt() { return startedAt; }
|
||||
public void setStartedAt(Date startedAt) { this.startedAt = startedAt; }
|
||||
public Date getFinishedAt() { return finishedAt; }
|
||||
public void setFinishedAt(Date finishedAt) { this.finishedAt = finishedAt; }
|
||||
@Override public Date getCreated() { return created; }
|
||||
@Override public void setCreated(Date created) { this.created = created; }
|
||||
public BigInteger getCreatedBy() { return createdBy; }
|
||||
public void setCreatedBy(BigInteger createdBy) { this.createdBy = createdBy; }
|
||||
@Override public Date getModified() { return modified; }
|
||||
@Override public void setModified(Date modified) { this.modified = modified; }
|
||||
public BigInteger getModifiedBy() { return modifiedBy; }
|
||||
public void setModifiedBy(BigInteger modifiedBy) { this.modifiedBy = modifiedBy; }
|
||||
}
|
||||
@@ -0,0 +1,131 @@
|
||||
package tech.easyflow.datacenter.meta.entity;
|
||||
|
||||
import com.mybatisflex.annotation.Column;
|
||||
import com.mybatisflex.annotation.Id;
|
||||
import com.mybatisflex.annotation.KeyType;
|
||||
import com.mybatisflex.annotation.Table;
|
||||
import com.mybatisflex.core.handler.FastjsonTypeHandler;
|
||||
import tech.easyflow.common.entity.DateEntity;
|
||||
|
||||
import java.io.Serializable;
|
||||
import java.math.BigInteger;
|
||||
import java.util.Date;
|
||||
import java.util.Map;
|
||||
|
||||
@Table(value = "tb_datacenter_source", comment = "数据中心数据源")
|
||||
public class DatacenterSource extends DateEntity implements Serializable {
|
||||
|
||||
@Id(keyType = KeyType.Generator, value = "snowFlakeId", comment = "主键")
|
||||
private BigInteger id;
|
||||
@Column(comment = "部门ID")
|
||||
private BigInteger deptId;
|
||||
@Column(tenantId = true, comment = "租户ID")
|
||||
private BigInteger tenantId;
|
||||
@Column(comment = "数据源名称")
|
||||
private String sourceName;
|
||||
@Column(comment = "数据源编码")
|
||||
private String sourceCode;
|
||||
@Column(comment = "数据源类型")
|
||||
private String sourceType;
|
||||
@Column(comment = "访问模式")
|
||||
private String accessMode;
|
||||
@Column(comment = "是否内置")
|
||||
private Integer builtinFlag;
|
||||
@Column(comment = "驱动类名")
|
||||
private String driverClassName;
|
||||
@Column(comment = "JDBC URL")
|
||||
private String jdbcUrl;
|
||||
@Column(comment = "主机")
|
||||
private String host;
|
||||
@Column(comment = "端口")
|
||||
private Integer port;
|
||||
@Column(comment = "数据库名")
|
||||
private String databaseName;
|
||||
@Column(comment = "Schema名")
|
||||
private String schemaName;
|
||||
@Column(comment = "用户名")
|
||||
private String username;
|
||||
@Column(comment = "凭据密文")
|
||||
private String credentialCipher;
|
||||
@Column(typeHandler = FastjsonTypeHandler.class, comment = "连接配置")
|
||||
private Map<String, Object> configJson;
|
||||
@Column(typeHandler = FastjsonTypeHandler.class, comment = "能力声明")
|
||||
private Map<String, Object> capabilitiesJson;
|
||||
@Column(comment = "最近测试状态")
|
||||
private String lastTestStatus;
|
||||
@Column(comment = "最近测试信息")
|
||||
private String lastTestMessage;
|
||||
@Column(comment = "最近测试时间")
|
||||
private Date lastTestedAt;
|
||||
@Column(comment = "状态")
|
||||
private Integer status;
|
||||
@Column(typeHandler = FastjsonTypeHandler.class, comment = "扩展项")
|
||||
private Map<String, Object> options;
|
||||
@Column(comment = "创建时间")
|
||||
private Date created;
|
||||
@Column(comment = "创建人")
|
||||
private BigInteger createdBy;
|
||||
@Column(comment = "修改时间")
|
||||
private Date modified;
|
||||
@Column(comment = "修改人")
|
||||
private BigInteger modifiedBy;
|
||||
|
||||
public BigInteger getId() { return id; }
|
||||
public void setId(BigInteger id) { this.id = id; }
|
||||
public BigInteger getDeptId() { return deptId; }
|
||||
public void setDeptId(BigInteger deptId) { this.deptId = deptId; }
|
||||
public BigInteger getTenantId() { return tenantId; }
|
||||
public void setTenantId(BigInteger tenantId) { this.tenantId = tenantId; }
|
||||
public String getSourceName() { return sourceName; }
|
||||
public void setSourceName(String sourceName) { this.sourceName = sourceName; }
|
||||
public String getSourceCode() { return sourceCode; }
|
||||
public void setSourceCode(String sourceCode) { this.sourceCode = sourceCode; }
|
||||
public String getSourceType() { return sourceType; }
|
||||
public void setSourceType(String sourceType) { this.sourceType = sourceType; }
|
||||
public String getAccessMode() { return accessMode; }
|
||||
public void setAccessMode(String accessMode) { this.accessMode = accessMode; }
|
||||
public Integer getBuiltinFlag() { return builtinFlag; }
|
||||
public void setBuiltinFlag(Integer builtinFlag) { this.builtinFlag = builtinFlag; }
|
||||
public String getDriverClassName() { return driverClassName; }
|
||||
public void setDriverClassName(String driverClassName) { this.driverClassName = driverClassName; }
|
||||
public String getJdbcUrl() { return jdbcUrl; }
|
||||
public void setJdbcUrl(String jdbcUrl) { this.jdbcUrl = jdbcUrl; }
|
||||
public String getHost() { return host; }
|
||||
public void setHost(String host) { this.host = host; }
|
||||
public Integer getPort() { return port; }
|
||||
public void setPort(Integer port) { this.port = port; }
|
||||
public String getDatabaseName() { return databaseName; }
|
||||
public void setDatabaseName(String databaseName) { this.databaseName = databaseName; }
|
||||
public String getSchemaName() { return schemaName; }
|
||||
public void setSchemaName(String schemaName) { this.schemaName = schemaName; }
|
||||
public String getUsername() { return username; }
|
||||
public void setUsername(String username) { this.username = username; }
|
||||
public String getCredentialCipher() { return credentialCipher; }
|
||||
public void setCredentialCipher(String credentialCipher) { this.credentialCipher = credentialCipher; }
|
||||
public Map<String, Object> getConfigJson() { return configJson; }
|
||||
public void setConfigJson(Map<String, Object> configJson) { this.configJson = configJson; }
|
||||
public Map<String, Object> getCapabilitiesJson() { return capabilitiesJson; }
|
||||
public void setCapabilitiesJson(Map<String, Object> capabilitiesJson) { this.capabilitiesJson = capabilitiesJson; }
|
||||
public String getLastTestStatus() { return lastTestStatus; }
|
||||
public void setLastTestStatus(String lastTestStatus) { this.lastTestStatus = lastTestStatus; }
|
||||
public String getLastTestMessage() { return lastTestMessage; }
|
||||
public void setLastTestMessage(String lastTestMessage) { this.lastTestMessage = lastTestMessage; }
|
||||
public Date getLastTestedAt() { return lastTestedAt; }
|
||||
public void setLastTestedAt(Date lastTestedAt) { this.lastTestedAt = lastTestedAt; }
|
||||
public Integer getStatus() { return status; }
|
||||
public void setStatus(Integer status) { this.status = status; }
|
||||
public Map<String, Object> getOptions() { return options; }
|
||||
public void setOptions(Map<String, Object> options) { this.options = options; }
|
||||
@Override
|
||||
public Date getCreated() { return created; }
|
||||
@Override
|
||||
public void setCreated(Date created) { this.created = created; }
|
||||
public BigInteger getCreatedBy() { return createdBy; }
|
||||
public void setCreatedBy(BigInteger createdBy) { this.createdBy = createdBy; }
|
||||
@Override
|
||||
public Date getModified() { return modified; }
|
||||
@Override
|
||||
public void setModified(Date modified) { this.modified = modified; }
|
||||
public BigInteger getModifiedBy() { return modifiedBy; }
|
||||
public void setModifiedBy(BigInteger modifiedBy) { this.modifiedBy = modifiedBy; }
|
||||
}
|
||||
@@ -0,0 +1,6 @@
|
||||
package tech.easyflow.datacenter.meta.enums;
|
||||
|
||||
public enum DatacenterAccessMode {
|
||||
READ_ONLY,
|
||||
READ_WRITE
|
||||
}
|
||||
@@ -0,0 +1,10 @@
|
||||
package tech.easyflow.datacenter.meta.enums;
|
||||
|
||||
public enum DatacenterCapability {
|
||||
TEST_CONNECTION,
|
||||
BROWSE_METADATA,
|
||||
READ_QUERY,
|
||||
WRITE_MUTATION,
|
||||
EXPORT,
|
||||
MATERIALIZE
|
||||
}
|
||||
@@ -0,0 +1,12 @@
|
||||
package tech.easyflow.datacenter.meta.enums;
|
||||
|
||||
public enum DatacenterConnectionErrorCode {
|
||||
INVALID_ARGUMENT,
|
||||
DRIVER_NOT_FOUND,
|
||||
NETWORK_UNREACHABLE,
|
||||
AUTH_FAILED,
|
||||
DATABASE_NOT_FOUND,
|
||||
SCHEMA_NOT_FOUND,
|
||||
PERMISSION_DENIED,
|
||||
UNKNOWN_ERROR
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
package tech.easyflow.datacenter.meta.enums;
|
||||
|
||||
public enum DatacenterImportStatus {
|
||||
PENDING,
|
||||
RUNNING,
|
||||
SUCCESS,
|
||||
FAILED,
|
||||
NOT_IMPLEMENTED
|
||||
}
|
||||
@@ -0,0 +1,13 @@
|
||||
package tech.easyflow.datacenter.meta.enums;
|
||||
|
||||
public enum DatacenterSourceType {
|
||||
PROJECT_MYSQL,
|
||||
EXCEL,
|
||||
EXCEL_MATERIALIZED,
|
||||
MYSQL,
|
||||
POSTGRESQL,
|
||||
ORACLE,
|
||||
GAUSSDB_NATIVE,
|
||||
GBASE_8A,
|
||||
GBASE_8S
|
||||
}
|
||||
@@ -0,0 +1,10 @@
|
||||
package tech.easyflow.datacenter.meta.enums;
|
||||
|
||||
public enum DatacenterTableKind {
|
||||
PROJECT_MANAGED,
|
||||
EXTERNAL_TABLE,
|
||||
EXTERNAL_VIEW,
|
||||
EXCEL_SHEET,
|
||||
EXCEL_MATERIALIZED,
|
||||
DERIVED_TABLE
|
||||
}
|
||||
@@ -0,0 +1,36 @@
|
||||
package tech.easyflow.datacenter.meta.model;
|
||||
|
||||
import java.math.BigInteger;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
public class DatacenterBatchRegisterRequest {
|
||||
|
||||
private BigInteger sourceId;
|
||||
private String catalogName;
|
||||
private List<String> tableNames = new ArrayList<>();
|
||||
|
||||
public BigInteger getSourceId() {
|
||||
return sourceId;
|
||||
}
|
||||
|
||||
public void setSourceId(BigInteger sourceId) {
|
||||
this.sourceId = sourceId;
|
||||
}
|
||||
|
||||
public String getCatalogName() {
|
||||
return catalogName;
|
||||
}
|
||||
|
||||
public void setCatalogName(String catalogName) {
|
||||
this.catalogName = catalogName;
|
||||
}
|
||||
|
||||
public List<String> getTableNames() {
|
||||
return tableNames;
|
||||
}
|
||||
|
||||
public void setTableNames(List<String> tableNames) {
|
||||
this.tableNames = tableNames;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
package tech.easyflow.datacenter.meta.model;
|
||||
|
||||
import java.math.BigInteger;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
public class DatacenterBatchRemoveRequest {
|
||||
|
||||
private List<BigInteger> tableIds = new ArrayList<>();
|
||||
|
||||
public List<BigInteger> getTableIds() {
|
||||
return tableIds;
|
||||
}
|
||||
|
||||
public void setTableIds(List<BigInteger> tableIds) {
|
||||
this.tableIds = tableIds;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,22 @@
|
||||
package tech.easyflow.datacenter.meta.model;
|
||||
|
||||
import java.math.BigInteger;
|
||||
|
||||
public class DatacenterCatalogMeta {
|
||||
private BigInteger id;
|
||||
private BigInteger sourceId;
|
||||
private String catalogName;
|
||||
private String catalogType;
|
||||
private String catalogDesc;
|
||||
|
||||
public BigInteger getId() { return id; }
|
||||
public void setId(BigInteger id) { this.id = id; }
|
||||
public BigInteger getSourceId() { return sourceId; }
|
||||
public void setSourceId(BigInteger sourceId) { this.sourceId = sourceId; }
|
||||
public String getCatalogName() { return catalogName; }
|
||||
public void setCatalogName(String catalogName) { this.catalogName = catalogName; }
|
||||
public String getCatalogType() { return catalogType; }
|
||||
public void setCatalogType(String catalogType) { this.catalogType = catalogType; }
|
||||
public String getCatalogDesc() { return catalogDesc; }
|
||||
public void setCatalogDesc(String catalogDesc) { this.catalogDesc = catalogDesc; }
|
||||
}
|
||||
@@ -0,0 +1,25 @@
|
||||
package tech.easyflow.datacenter.meta.model;
|
||||
|
||||
import java.math.BigInteger;
|
||||
|
||||
public class DatacenterFieldDescriptionUpdate {
|
||||
|
||||
private BigInteger fieldId;
|
||||
private String fieldDesc;
|
||||
|
||||
public BigInteger getFieldId() {
|
||||
return fieldId;
|
||||
}
|
||||
|
||||
public void setFieldId(BigInteger fieldId) {
|
||||
this.fieldId = fieldId;
|
||||
}
|
||||
|
||||
public String getFieldDesc() {
|
||||
return fieldDesc;
|
||||
}
|
||||
|
||||
public void setFieldDesc(String fieldDesc) {
|
||||
this.fieldDesc = fieldDesc;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,16 @@
|
||||
package tech.easyflow.datacenter.meta.model;
|
||||
|
||||
import java.math.BigInteger;
|
||||
|
||||
public class DatacenterRemoveSourceRequest {
|
||||
|
||||
private BigInteger sourceId;
|
||||
|
||||
public BigInteger getSourceId() {
|
||||
return sourceId;
|
||||
}
|
||||
|
||||
public void setSourceId(BigInteger sourceId) {
|
||||
this.sourceId = sourceId;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,36 @@
|
||||
package tech.easyflow.datacenter.meta.model;
|
||||
|
||||
import java.math.BigInteger;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
public class DatacenterSaveDescriptionsRequest {
|
||||
|
||||
private BigInteger tableId;
|
||||
private String tableDesc;
|
||||
private List<DatacenterFieldDescriptionUpdate> fields = new ArrayList<>();
|
||||
|
||||
public BigInteger getTableId() {
|
||||
return tableId;
|
||||
}
|
||||
|
||||
public void setTableId(BigInteger tableId) {
|
||||
this.tableId = tableId;
|
||||
}
|
||||
|
||||
public String getTableDesc() {
|
||||
return tableDesc;
|
||||
}
|
||||
|
||||
public void setTableDesc(String tableDesc) {
|
||||
this.tableDesc = tableDesc;
|
||||
}
|
||||
|
||||
public List<DatacenterFieldDescriptionUpdate> getFields() {
|
||||
return fields;
|
||||
}
|
||||
|
||||
public void setFields(List<DatacenterFieldDescriptionUpdate> fields) {
|
||||
this.fields = fields;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
package tech.easyflow.datacenter.meta.model;
|
||||
|
||||
import tech.easyflow.datacenter.entity.DatacenterTable;
|
||||
import tech.easyflow.datacenter.entity.DatacenterTableField;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
public class DatacenterTableDetailMeta {
|
||||
private DatacenterTable table;
|
||||
private List<DatacenterTableField> fields = new ArrayList<>();
|
||||
|
||||
public DatacenterTable getTable() { return table; }
|
||||
public void setTable(DatacenterTable table) { this.table = table; }
|
||||
public List<DatacenterTableField> getFields() { return fields; }
|
||||
public void setFields(List<DatacenterTableField> fields) { this.fields = fields; }
|
||||
}
|
||||
@@ -0,0 +1,38 @@
|
||||
package tech.easyflow.datacenter.meta.service;
|
||||
|
||||
import tech.easyflow.common.entity.LoginAccount;
|
||||
import tech.easyflow.datacenter.entity.DatacenterTable;
|
||||
import tech.easyflow.datacenter.entity.DatacenterTableField;
|
||||
import tech.easyflow.datacenter.execution.model.DatasetRef;
|
||||
import tech.easyflow.datacenter.meta.entity.DatacenterCatalog;
|
||||
import tech.easyflow.datacenter.meta.entity.DatacenterSource;
|
||||
import tech.easyflow.datacenter.meta.model.DatacenterFieldDescriptionUpdate;
|
||||
import tech.easyflow.datacenter.meta.enums.DatacenterSourceType;
|
||||
import tech.easyflow.datacenter.meta.model.DatacenterTableDetailMeta;
|
||||
|
||||
import java.math.BigInteger;
|
||||
import java.util.List;
|
||||
|
||||
public interface DatacenterDatasetRegistryService {
|
||||
DatacenterSource ensureBuiltinSource(DatacenterSourceType sourceType, LoginAccount account);
|
||||
|
||||
DatacenterCatalog ensureCatalog(DatacenterSource source, String catalogName, LoginAccount account);
|
||||
|
||||
DatacenterTable registerTable(DatacenterSource source, DatacenterCatalog catalog, DatacenterTableDetailMeta detail, LoginAccount account);
|
||||
|
||||
DatacenterTable getTableWithFields(BigInteger tableId);
|
||||
|
||||
List<DatacenterTableField> getFields(BigInteger tableId);
|
||||
|
||||
DatasetRef resolveDatasetRef(BigInteger tableId);
|
||||
|
||||
DatacenterSource getSourceRequired(BigInteger sourceId);
|
||||
|
||||
DatacenterCatalog getCatalogById(BigInteger catalogId);
|
||||
|
||||
List<DatacenterTable> listManagedTables(BigInteger sourceId, BigInteger catalogId);
|
||||
|
||||
int removeTables(List<BigInteger> tableIds);
|
||||
|
||||
DatacenterTable saveDescriptions(BigInteger tableId, String tableDesc, List<DatacenterFieldDescriptionUpdate> fields, LoginAccount account);
|
||||
}
|
||||
@@ -0,0 +1,12 @@
|
||||
package tech.easyflow.datacenter.meta.service;
|
||||
|
||||
import java.math.BigInteger;
|
||||
|
||||
public final class DatacenterMetaConstants {
|
||||
|
||||
private DatacenterMetaConstants() {
|
||||
}
|
||||
|
||||
public static final BigInteger PROJECT_SOURCE_BASE = new BigInteger("9000000000002000000");
|
||||
public static final BigInteger PROJECT_CATALOG_BASE = new BigInteger("9000000000003000000");
|
||||
}
|
||||
@@ -0,0 +1,32 @@
|
||||
package tech.easyflow.datacenter.meta.service;
|
||||
|
||||
import com.mybatisflex.core.paginate.Page;
|
||||
import com.mybatisflex.core.service.IService;
|
||||
import tech.easyflow.common.entity.LoginAccount;
|
||||
import tech.easyflow.datacenter.entity.DatacenterTable;
|
||||
import tech.easyflow.datacenter.execution.model.DatacenterConnectionTestResult;
|
||||
import tech.easyflow.datacenter.meta.entity.DatacenterSource;
|
||||
import tech.easyflow.datacenter.meta.model.DatacenterBatchRegisterRequest;
|
||||
import tech.easyflow.datacenter.meta.model.DatacenterCatalogMeta;
|
||||
import tech.easyflow.datacenter.meta.model.DatacenterTableDetailMeta;
|
||||
|
||||
import java.math.BigInteger;
|
||||
import java.util.List;
|
||||
|
||||
public interface DatacenterSourceService extends IService<DatacenterSource> {
|
||||
DatacenterSource saveSource(DatacenterSource source, LoginAccount account);
|
||||
|
||||
Page<DatacenterSource> pageSources(Long pageNumber, Long pageSize, LoginAccount account);
|
||||
|
||||
DatacenterConnectionTestResult testConnection(DatacenterSource source, LoginAccount account);
|
||||
|
||||
List<DatacenterCatalogMeta> listCatalogs(BigInteger sourceId, LoginAccount account);
|
||||
|
||||
List<DatacenterTable> listTables(BigInteger sourceId, String catalogName, LoginAccount account);
|
||||
|
||||
DatacenterTableDetailMeta getTableDetail(BigInteger sourceId, String catalogName, String tableName, boolean register, LoginAccount account);
|
||||
|
||||
List<DatacenterTable> batchRegisterTables(DatacenterBatchRegisterRequest request, LoginAccount account);
|
||||
|
||||
void removeSource(BigInteger sourceId, LoginAccount account);
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user