add assebmle

kimi42345

2020-03-02 2411eadd4ad4bef5c08c6ba2ea5898ea72136e29

add assebmle

 pom.xml

@@ -10,16 +10,24 @@
    </parent>
    <groupId>com.highdatas</groupId>
    <artifactId>mdm</artifactId>
    <version>0.9.0-SNAPSHOT</version>
    <version>1.0.0</version>
    <name>mdm</name>
    <description>main datas Governance system</description>
    <packaging>jar</packaging>

    <repositories>
        <repository>
            <id>pentaho</id>
            <url>http://nexus.pentaho.org/content/groups/omni/</url>
        </repository>
    </repositories>

    <properties>
        <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
        <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
        <java.version>1.8</java.version>
        <activiti.version>5.22.0</activiti.version>
        <kettle-version>7.1.0.0-12</kettle-version>
    </properties>

    <dependencies>
@@ -84,11 +92,15 @@
            <version>1.2.62</version>
        </dependency>


        <dependency>
            <groupId>com.xuxueli</groupId>
            <artifactId>xxl-job-core</artifactId>
            <version>2.1.0</version>
        </dependency>
        <dependency>
            <groupId>mysql</groupId>
            <artifactId>mysql-connector-java</artifactId>
            <scope>runtime</scope>
            <version>5.1.46</version>
        </dependency>
        <dependency>
            <groupId>org.projectlombok</groupId>
@@ -132,7 +144,11 @@
                </exclusion>
            </exclusions>
        </dependency>

        <dependency>
            <groupId>com.xuxueli</groupId>
            <artifactId>xxl-job-core</artifactId>
            <version>2.1.2</version>
        </dependency>
        <dependency>
            <groupId>org.activiti</groupId>
            <artifactId>activiti-spring-boot-starter-actuator</artifactId>
@@ -242,6 +258,156 @@
            <artifactId>pinyin4j</artifactId>
            <version>2.5.0</version>
        </dependency>


        <!-- settle -->
        <!-- with pentaho-kettle -->
        <dependency>
            <groupId>pentaho-kettle</groupId>
            <artifactId>kettle-core</artifactId>
            <version>${kettle-version}</version>
        </dependency>
        <dependency>
            <groupId>pentaho-kettle</groupId>
            <artifactId>kettle-engine</artifactId>
            <version>${kettle-version}</version>
        </dependency>
        <dependency>
            <groupId>pentaho</groupId>
            <artifactId>metastore</artifactId>
            <version>${kettle-version}</version>
        </dependency>

        <dependency>
            <groupId>pentaho-kettle</groupId>
            <artifactId>kettle-dbdialog</artifactId>
            <version>${kettle-version}</version>
        </dependency>

        <!--<dependency>-->
            <!--<groupId>pentaho</groupId>-->
            <!--<artifactId>pentaho-big-data-kettle-plugins-hbase-meta</artifactId>-->
            <!--<version>${kettle-version}</version>-->
            <!--<scope>compile</scope>-->
        <!--</dependency>-->

        <dependency>
            <groupId>com.alibaba</groupId>
            <artifactId>druid</artifactId>
            <version>1.1.10</version>
        </dependency>

        <dependency>
            <groupId>com.alibaba</groupId>
            <artifactId>fastjson</artifactId>
            <version>1.2.58</version>
        </dependency>

        <dependency>
            <groupId>org.apache.commons</groupId>
            <artifactId>commons-vfs2</artifactId>
            <version>2.1</version>
        </dependency>

        <dependency>
            <groupId>com.google.guava</groupId>
            <artifactId>guava</artifactId>
            <version>17.0</version>
        </dependency>

        <dependency>
            <groupId>net.sourceforge.jtds</groupId>
            <artifactId>jtds</artifactId>
            <version>1.3.1</version>
        </dependency>




        <dependency>
            <groupId>javax.servlet</groupId>
            <artifactId>jstl</artifactId>
        </dependency>
        <!--jdbc-->

        <dependency>
            <groupId>monetdb</groupId>
            <artifactId>monetdb-jdbc</artifactId>
            <version>2.8</version>
        </dependency>

        <!--<dependency>-->
            <!--<groupId>org.mongodb</groupId>-->
            <!--<artifactId>mongo-java-driver</artifactId>-->
            <!--<version>3.2.2</version>-->
        <!--</dependency>-->
        <dependency>
            <groupId>org.zenframework.z8.dependencies.commons</groupId>
            <artifactId>ojdbc6</artifactId>
            <version>2.0</version>
            <classifier>sources</classifier>
            <type>java-source</type>
        </dependency>

        <dependency>
            <groupId>org.apache.phoenix</groupId>
            <artifactId>phoenix-core</artifactId>
            <version>4.7.0-HBase-1.1</version>
        </dependency>
        <dependency>
            <groupId>org.apache.hbase</groupId>
            <artifactId>hbase-client</artifactId>
            <version>1.1.2</version>
        </dependency>
        <!--<dependency>-->
            <!--<groupId>pentaho</groupId>-->
            <!--<artifactId>pentaho-mongo-utils</artifactId>-->
            <!--<version>${kettle-version}</version>-->
            <!--<scope>compile</scope>-->
        <!--</dependency>-->
        <!--<dependency>-->
            <!--<groupId>pentaho</groupId>-->
            <!--<artifactId>pentaho-mongodb-plugin</artifactId>-->
            <!--<version>${kettle-version}</version>-->
            <!--<scope>compile</scope>-->
        <!--</dependency>-->
        <!--<dependency>-->
            <!--<groupId>pentaho</groupId>-->
            <!--<artifactId>pentaho-big-data-kettle-plugins-hbase-meta</artifactId>-->
            <!--<version>${kettle-version}</version>-->
            <!--<scope>compile</scope>-->
        <!--</dependency>-->
        <!--<dependency>-->
            <!--<groupId>pentaho</groupId>-->
            <!--<artifactId>pentaho-big-data-kettle-plugins-hadoop-cluster</artifactId>-->
            <!--<version>${kettle-version}</version>-->
            <!--<scope>compile</scope>-->
        <!--</dependency>-->
        <dependency>
            <groupId>pentaho</groupId>
            <artifactId>pentaho-database-model</artifactId>
            <version>${kettle-version}</version>
            <scope>compile</scope>
        </dependency>
        <dependency>
        <groupId>pentaho</groupId>
        <artifactId>pentaho-metadata</artifactId>
        <version>${kettle-version}</version>
        <scope>compile</scope>
        </dependency>
        <dependency>
            <groupId>com.microsoft.sqlserver</groupId>
            <artifactId>sqljdbc4</artifactId>
            <version>4.0</version>
        </dependency>

        <dependency>
            <groupId>org.xerial</groupId>
            <artifactId>sqlite-jdbc</artifactId>
            <version>3.7.2</version>
        </dependency>


    </dependencies>

    <build>

 src/main/java/com/highdatas/mdm/controller/ActivitiController.java

@@ -68,6 +68,7 @@
    IMaintainService maintainService;
    @Autowired
    ITUserService userService;

    @Value("${img.url}")
    String basePath;

@@ -299,7 +300,7 @@
        ActivitiStatus status = flows.getStatus();

        String workflowId = flows.getWorkflowId();
//        Task task = taskService.geTask(workflowId);
//        Task task = TimeTaskService.geTask(workflowId);
//        String id = task.getId();
        String taskId = null;
        if (taskService.checkClaim(workflowId)) {

 src/main/java/com/highdatas/mdm/controller/AssembleController.java

New file
@@ -0,0 +1,362 @@
package com.highdatas.mdm.controller;



import com.alibaba.fastjson.JSON;

import com.alibaba.fastjson.JSONArray;

import com.alibaba.fastjson.JSONObject;

import com.baomidou.mybatisplus.mapper.EntityWrapper;

import com.baomidou.mybatisplus.mapper.Wrapper;

import com.baomidou.mybatisplus.plugins.Page;

import com.highdatas.mdm.entity.*;

import com.highdatas.mdm.pojo.CodeMsg;

import com.highdatas.mdm.pojo.Result;

import com.highdatas.mdm.pojo.SysAssembleStatus;

import com.highdatas.mdm.pojo.SysAssembleUpdateType;

import com.highdatas.mdm.pojo.kettle.BigDataDataSourceInfo;

import com.highdatas.mdm.pojo.kettle.DataSourceInfo;

import com.highdatas.mdm.pojo.kettle.UnBigDataDataSourceInfo;

import com.highdatas.mdm.service.*;

import com.highdatas.mdm.util.Constant;

import com.highdatas.mdm.util.DbUtils;

import lombok.extern.slf4j.Slf4j;

import org.apache.commons.lang3.StringUtils;

import org.springframework.beans.factory.annotation.Autowired;

import org.springframework.scheduling.support.CronSequenceGenerator;

import org.springframework.transaction.annotation.Transactional;

import org.springframework.web.bind.annotation.*;



import javax.servlet.http.HttpServletRequest;

import java.util.*;

import java.util.stream.Collectors;



/**

 * @author kimi

 * @description

 * @date 2020-02-17 10:58

 */



@Slf4j

@RestController

@RequestMapping("/assemble")

public class AssembleController {

    @Autowired

    ISysAssembleService assembleService;

    @Autowired

    ISysAssembleDbService dbService;

    @Autowired

    ISysAssembleDbFieldService fieldService;

    @Autowired

    ISysAssembleDbTableService tableService;

    @Autowired

    ISysAssembleApiService apiService;

    @Autowired

    ISysAssembleCommonparamsService commonparamsService;

    @Autowired

    ISysAssembleParamsService paramsService;

    @Autowired

    ISysMenuService menuService;

    @Autowired

    IMenuMappingService menuMappingService;

    @Autowired

    BigDataDataSourceInfo bigDataDataSourceInfo;

    @Autowired

    UnBigDataDataSourceInfo unBigDataDataSourceInfo;



    @RequestMapping(value = "run/{id}", method = RequestMethod.GET)

    public Result delete(@PathVariable String id, HttpServletRequest request) {

        Result run = assembleService.run(id, request.getSession());

        return run;

    }







    @RequestMapping(value = "new", method = RequestMethod.GET)

    public Result delete(HttpServletRequest request) {

        SysAssemble assemble = new SysAssemble();

        TUser user = (TUser) request.getSession().getAttribute(Constant.USER);

        assemble.setStatus(SysAssembleStatus.edit).setCreateTime(new Date()).setUserId(user.getUserId()).setUpdateType(SysAssembleUpdateType.All);

        boolean insert = assemble.setId(DbUtils.getUUID()).insert();

        if (insert) {

            String assembleId = assemble.getId();

            // 创建job的时候将所有 通用参数搬一份过去

            List<SysAssembleCommonparams> sysAssembleCommonparams = commonparamsService.selectList(null);

            for (SysAssembleCommonparams commonParam : sysAssembleCommonparams) {

                SysAssembleParams sysAssembleParams = new SysAssembleParams();

                sysAssembleParams

                        .setName(commonParam.getName())

                        .setCode(commonParam.getCode())

                        .setInitSql(commonParam.getInitSql()).setUpdateSql(commonParam.getUpdateSql())

                        .setCreateTime(new Date())

                        .setParentId(assembleId)

                        .setId(DbUtils.getUUID()).insert();

            }

            return Result.success(assemble);

        }else {

            return Result.error(CodeMsg.INSERT_ERROR);

        }

    }







    @RequestMapping(value = "/saveCommon/{id}", method = RequestMethod.GET)

    public Result add(@PathVariable String id, @RequestParam String menuId, @RequestParam SysAssembleUpdateType updateType, @RequestParam Boolean bigData, @RequestParam String updateFields, HttpServletRequest request) {

        SysAssemble assemble = assembleService.selectById(id);

        if (assemble == null) {

            return Result.error(CodeMsg.ERROR_PARAMS_NOT_MATHED);

        }

        assemble.setUpdateType(updateType).setUpdateFields(updateFields);

        assemble.setBigdata(Boolean.valueOf(bigData));

        TUser user = (TUser) request.getSession().getAttribute(Constant.USER);

        boolean insert = assemble.setUserId(user.getUserId()).setMenuId(menuId).setUpdateTime(new Date()).updateById();



        if (insert) {

            return Result.success(assemble);

        }else {

            return Result.error(CodeMsg.INSERT_ERROR);

        }

    }



    @Transactional(rollbackFor=Exception.class)

    @RequestMapping(value = "delete/{id}", method = RequestMethod.GET)

    public Result add(@PathVariable String id) {

        SysAssemble assemble = assembleService.selectById(id);

        if (assemble == null) {

            return Result.error(CodeMsg.ERROR_PARAMS_NOT_MATHED);

        }

        Boolean bigdata = assemble.getBigdata();

        DataSourceInfo dataSourceInfo;

        if (bigdata) {

            dataSourceInfo = bigDataDataSourceInfo;

        }else {

            dataSourceInfo = unBigDataDataSourceInfo;

        }

        //delete db

        List<SysAssembleDb> dbList = dbService.selectList(new EntityWrapper<SysAssembleDb>().eq(Constant.PARENT_ID, id));

        for (SysAssembleDb sysAssembleDb : dbList) {

            String dbId = sysAssembleDb.getId();

            //delete table



            Wrapper<SysAssembleDbTable> eq = new EntityWrapper<SysAssembleDbTable>().eq(Constant.PARENT_ID, dbId);

            List<SysAssembleDbTable> tableList = tableService.selectList(eq);

            for (SysAssembleDbTable dbTable : tableList) {

                String tableId = dbTable.getId();

                //delete field

                String tempTableName = dbTable.getTempTableName();

                dataSourceInfo.dropData(tempTableName);



                Wrapper<SysAssembleDbField> fieldWrapper = new EntityWrapper<SysAssembleDbField>().eq(Constant.PARENT_ID, tableId);

                boolean delete = fieldService.delete(fieldWrapper);

                if (!delete) {

                    return Result.error(CodeMsg.DELETE_ERROR);

                }

                dbTable.deleteById();

            }

            sysAssembleDb.deleteById();

        }

        //TODO delete api

        apiService.delete(new EntityWrapper<SysAssembleApi>().eq(Constant.PARENT_ID, id));

        //delete param

        paramsService.delete(new EntityWrapper<SysAssembleParams>().eq(Constant.PARENT_ID, id));



        boolean deleted = assemble.deleteById();

        if (deleted) {

            return Result.success(assemble);

        }else {

            return Result.error(CodeMsg.DELETE_ERROR);

        }

    }







    @RequestMapping(value = "updateJob/{id}", method = RequestMethod.GET)

    public Result updateStatus(@RequestParam String jobId,@RequestParam String id) {

        SysAssemble assemble = assembleService.selectById(id);

        if (assemble == null) {

            return Result.error(CodeMsg.ERROR_PARAMS_NOT_MATHED);

        }

        assemble.setJobId(jobId);

        boolean updated = assemble.updateById();

        if (updated) {

            return Result.success(assemble);

        }else {

            return Result.error(CodeMsg.UPDATE_ERROR);

        }

    }





    @RequestMapping(value = "updateStatus/{id}", method = RequestMethod.GET)

    public Result updateStatus(@RequestParam SysAssembleStatus status,@RequestParam String id) {

        SysAssemble assemble = assembleService.selectById(id);

        if (assemble == null) {

            return Result.error(CodeMsg.ERROR_PARAMS_NOT_MATHED);

        }

        assemble.setStatus(status);

        boolean updated = assemble.updateById();

        if (updated) {

            return Result.success(assemble);

        }else {

            return Result.error(CodeMsg.UPDATE_ERROR);

        }

    }



    @RequestMapping(value = "loadPurgeSql/{id}", method = RequestMethod.POST)

    public Result loadPurgeSql(@RequestBody JSONObject object,@PathVariable String id) {

        String sql = object.getString("sql");

        SysAssemble assemble = assembleService.selectById(id);

        if (assemble == null) {

            return Result.error(CodeMsg.ERROR_PARAMS_NOT_MATHED);

        }

        assemble.setPurgeSql(sql);

        boolean updated = assemble.updateById();

        if (updated) {

            return Result.success(assemble);

        }else {

            return Result.error(CodeMsg.UPDATE_ERROR);

        }

    }



    @RequestMapping(value = "getTables/{id}", method = RequestMethod.GET)

    public Result getTables(@PathVariable String id) {

        SysAssemble assemble = assembleService.selectById(id);

        if (assemble == null) {

            return Result.error(CodeMsg.ERROR_PARAMS_NOT_MATHED);

        }

        //TODO  ONLY db 缺少 api的

        List<SysAssembleDb> dbList = dbService.selectList(new EntityWrapper<SysAssembleDb>().eq(Constant.PARENT_ID, id));

        JSONArray sourceTables = new JSONArray();



        for (SysAssembleDb sysAssembleDb : dbList) {

            List<SysAssembleDbTable> tableList = tableService.selectList(new EntityWrapper<SysAssembleDbTable>().eq(Constant.PARENT_ID, sysAssembleDb.getId()));

            for (SysAssembleDbTable dbTable : tableList) {

                JSONObject object = new JSONObject();

                object.fluentPut(Constant.ID, dbTable.getId());

                object.fluentPut(Constant.tableName, dbTable.getTempTableName());

                sourceTables.add(object);

            }

        }

        MenuMapping menuMapping = menuMappingService.selectOne(new EntityWrapper<MenuMapping>().eq("menu_id", assemble.getMenuId()));

        String tableName = menuMapping.getTableName();

        String tempTableName = Constant.Temp + tableName;

        JSONObject object = new JSONObject();

        object.fluentPut(Constant.ID, null);

        object.fluentPut(Constant.tableName, tempTableName);

        JSONObject result = new JSONObject();

        result.fluentPut("source", sourceTables);

        result.fluentPut("target", object);

        return Result.success(result);

    }



    @RequestMapping(value = "saveCorn/{id}", method = RequestMethod.GET)

    public Result add(@RequestParam String cron,@PathVariable String id) {

        SysAssemble assemble = assembleService.selectById(id);

        if (assemble == null) {

            return Result.error(CodeMsg.ERROR_PARAMS_NOT_MATHED);

        }

        boolean validExpression = CronSequenceGenerator.isValidExpression(cron);

        if (!validExpression) {

            return Result.error(new CodeMsg(6006, "corn 表达式不规范"));

        }

        assemble.setCron(cron);

        boolean b = paramsService.updateCornVal(assemble.getId(), cron);

        if (!b) {

            return Result.error(new CodeMsg(6007, "corn 表达式更新值错误"));

        }

        assemble.setStatus(SysAssembleStatus.working);



        boolean updated = assemble.updateById();

        if (updated) {

            return Result.success(assemble);

        }else {

            return Result.error(CodeMsg.UPDATE_ERROR);

        }

    }



    @RequestMapping(value = "get/{id}", method = RequestMethod.GET)

    public Result get(@PathVariable String id) {

        SysAssemble assemble = assembleService.selectById(id);

        String menuId = assemble.getMenuId();

        JSONObject o = (JSONObject) JSON.toJSON(assemble);

        if (!StringUtils.isEmpty(menuId)) {

            LinkedHashSet<String> ids = new LinkedHashSet<>();

            ids.add(menuId);

            Set<String> byParentId = menuService.getByParentId(ids);

            ArrayList<String> strings = new ArrayList<>(byParentId);

            Collections.reverse(strings);



            o.fluentPut("menupath", strings);

        }



        if (assemble == null) {

            return Result.error(CodeMsg.ERROR_PARAMS_NOT_MATHED);

        }

        return Result.success(o);

    }



    @RequestMapping(value = "/{pageNo}", method = RequestMethod.GET)

    public Result page(@PathVariable Integer pageNo, HttpServletRequest request)  {

        EntityWrapper<SysAssemble> wrapper = new EntityWrapper<>();

        String dbName = request.getParameter("dbName");

        if (!StringUtils.isEmpty(dbName)) {

            List<SysAssembleDb> dbList = dbService.selectList(new EntityWrapper<SysAssembleDb>().eq("datasource_name", dbName));

            if (dbList != null && dbList.size() != 0) {

                List<String> collect = dbList.stream().map(sysAssembleDb -> sysAssembleDb.getParentId()).collect(Collectors.toList());

                wrapper.in(Constant.ID, collect);

            }

        }



        String menuId = request.getParameter("menuId");

        if (!StringUtils.isEmpty(menuId)) {

            wrapper.eq("menu_id", menuId);

        }

        String bigdata = request.getParameter("bigdata");

        if (!StringUtils.isEmpty(bigdata)) {

            wrapper.eq("bigdata", Boolean.valueOf(bigdata));

        }

        String status = request.getParameter("status");

        if (!StringUtils.isEmpty(status)) {

            wrapper.eq("status", status);

        }



        Page<SysAssemble> page;

        String pageSize = request.getParameter("pageSize");

        if (!StringUtils.isEmpty(pageSize)) {

            page = new Page(pageNo, Integer.valueOf(pageSize));

        }else {

            page = new Page(pageNo, 15);

        }



        Page<SysAssemble> resultPage = assembleService.selectPage(page,wrapper);

        List<SysAssemble> records = resultPage.getRecords();



        List<JSONObject> array = new ArrayList<>();

        for (SysAssemble record : records) {

            String id = record.getId();

            Wrapper<SysAssembleParams> eq = new EntityWrapper<SysAssembleParams>().eq(Constant.PARENT_ID, id).eq(Constant.Code, Constant.Cron);

            JSONObject o = (JSONObject) JSON.toJSON(record);



            SysAssembleParams sysAssembleParams = paramsService.selectOne(eq);

            String val = sysAssembleParams.getVal();

            o.fluentPut("nextTime", val);



            String realMenuId = record.getMenuId();

            if (!StringUtils.isEmpty(realMenuId)) {

                LinkedHashSet<String> menuSet = new LinkedHashSet<>();

                menuSet.add(realMenuId);

                LinkedHashSet<String> byParentId = menuService.getByParentId(menuSet);

                List<SysMenu> sysMenus = menuService.selectBatchIds(byParentId);

                o.fluentPut("menuList", sysMenus);

            }





            List<SysAssembleDb> dbList = dbService.selectList(new EntityWrapper<SysAssembleDb>().eq(Constant.PARENT_ID, record.getId()));

            o.fluentPut("dbList", dbList);



            array.add(o);

        }

        Page<JSONObject> jsonObjectPage = new Page<>();

        jsonObjectPage.setCurrent(resultPage.getCurrent());

        jsonObjectPage.setSize(resultPage.getSize());

        jsonObjectPage.setTotal(resultPage.getTotal());

        jsonObjectPage.setRecords(array);

        return Result.success(jsonObjectPage);

    }

}


 src/main/java/com/highdatas/mdm/controller/FileController.java

@@ -1,7 +1,26 @@
package com.highdatas.mdm.controller;

import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
import com.alibaba.fastjson.JSONObject;
import com.highdatas.mdm.entity.SysAssemble;
import com.highdatas.mdm.entity.SysAssembleParams;
import com.highdatas.mdm.pojo.CodeMsg;
import com.highdatas.mdm.pojo.Result;
import com.highdatas.mdm.service.ISysAssembleDbService;
import com.highdatas.mdm.service.ISysAssembleParamsService;
import com.highdatas.mdm.service.ISysAssembleService;
import com.highdatas.mdm.util.Constant;
import com.highdatas.mdm.util.DbUtils;
import com.highdatas.mdm.util.RedisClient;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.web.bind.annotation.*;
import org.springframework.web.multipart.MultipartFile;

import javax.servlet.http.HttpServletRequest;
import java.io.BufferedReader;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.util.Date;
import java.util.List;

/**
 * @author kimi
@@ -12,4 +31,96 @@
@RestController
@RequestMapping("/file")
public class FileController {
    @Autowired
    ISysAssembleDbService dbService;
    @Autowired
    ISysAssembleService assembleService;
    @Autowired
    ISysAssembleParamsService paramsService;
    @Autowired
    RedisClient redisClient;

    @RequestMapping(value = "/loadSqlTable", method = RequestMethod.POST)
    @ResponseBody
    public Result loadSqlTable(@RequestParam("file")MultipartFile file,@RequestParam String dbId, HttpServletRequest request) {
        try {
            byte[] bytes = file.getBytes();

            String sql = new String (bytes);
            sql = DbUtils.replaceEscape(sql);
            String[] split = sql.split(Constant.SEMICOLON);
            if (split.length > 1) {
                return Result.error(new CodeMsg(6001 , "sql脚本上传错误,仅能支持一条语句"));
            }
            String sqlId = DbUtils.getUUID();
            boolean b = redisClient.putRedisVal(sqlId, sql);
            if (!b){
                return Result.error(new CodeMsg(6002, "sql保存失败"));
            }
            // 暂不校验是不是select语句了  后续需要再校验
            List<String> fieldsBySql = dbService.getFieldsBySql(dbId, sql);
            JSONObject object = new JSONObject();
            object.fluentPut("fields", fieldsBySql);
            object.fluentPut("sqlId",sqlId);

            return Result.success(object);
        }catch (Exception e) {
            e.printStackTrace();
            return Result.error(new CodeMsg(6002, e.getMessage()));
        }

    }

    @RequestMapping(value = "/loadPurgeSql", method = RequestMethod.POST)
    @ResponseBody
    public Result loadPurgeSql(@RequestParam("file")MultipartFile file,@RequestParam String id, HttpServletRequest request) {
        // 暂不校验是不是select语句了  后续需要再校验
        SysAssemble assemble = assembleService.selectById(id);
        if (assemble == null){
            return Result.error(CodeMsg.ERROR_PARAMS_NOT_MATHED);
        }
        try {
            byte[] bytes = file.getBytes();

            String sql = new String (bytes);
            sql = DbUtils.replaceEscape(sql);
            //TODO 校验sql 暂不进行
            assemble.setPurgeSql(sql);
            boolean update = assemble.updateById();
            if (update) {
                return Result.success(assemble);
            }else {
                return Result.error(CodeMsg.UPDATE_ERROR);
            }

        }catch (Exception e) {
            e.printStackTrace();
            return Result.error(new CodeMsg(6002, e.getMessage()));
        }

    }

    @RequestMapping(value = "/loadParamsUpdateSql", method = RequestMethod.POST)
    @ResponseBody
    public Result loadParamsUpdateSql(@RequestParam("file")MultipartFile file,@RequestParam String id, HttpServletRequest request) {
        // 暂不校验是不是select语句了  后续需要再校验
        SysAssembleParams sysAssembleParams = paramsService.selectById(id);
        if (sysAssembleParams == null){
            return Result.error(CodeMsg.ERROR_PARAMS_NOT_MATHED);
        }
        try {
            byte[] bytes = file.getBytes();

            String sql = new String (bytes);
            sql = DbUtils.replaceEscape(sql);
            //TODO 校验sql 暂不进行
            sysAssembleParams.setUpdateSql(sql).setUpdateTime(new Date()).updateById();

            return Result.success(sysAssembleParams);
        }catch (Exception e) {
            e.printStackTrace();
            return Result.error(new CodeMsg(6002, e.getMessage()));
        }

    }
}

 src/main/java/com/highdatas/mdm/controller/MaintainController.java

@@ -142,7 +142,7 @@
            result = maintainService.selectPage(maintainPage);
        } else {
            Wrapper<Maintain> tableNameWrapper = new EntityWrapper<Maintain>().eq("table_name", tableName);
            tableNameWrapper.orderBy("version desc");
            tableNameWrapper.orderBy("order_by desc");
            result = maintainService.selectPage(maintainPage, tableNameWrapper);
        }
        List<Maintain> records = result.getRecords();

 src/main/java/com/highdatas/mdm/controller/MasterDataController.java

@@ -8,6 +8,7 @@
import com.highdatas.mdm.pojo.CodeMsg;
import com.highdatas.mdm.pojo.Operate;
import com.highdatas.mdm.pojo.Result;
import com.highdatas.mdm.pojo.SysAssembleUpdateType;
import com.highdatas.mdm.service.IMaintainDetailService;
import com.highdatas.mdm.service.IMaintainService;
import com.highdatas.mdm.service.IMenuMappingService;
@@ -52,67 +53,14 @@

    @RequestMapping(value = "{tableName}/uploaded",  method = RequestMethod.GET)
    public Result get(@PathVariable String tableName, HttpServletRequest request) {
        String uploadType = request.getParameter("uploadType");
        if (StringUtils.isEmpty(uploadType)) {
            uploadType = Constant.All;
        String uploadTypeStr = request.getParameter("uploadType");
        SysAssembleUpdateType uploadType;
        if (StringUtils.isEmpty(uploadTypeStr)) {
            uploadType = SysAssembleUpdateType.Increment;
        }
        MenuMapping menuMapping = menuMappingService.selectOne(new EntityWrapper<MenuMapping>().eq("table_name",tableName));
        SysOperateLog operateLog = new SysOperateLog();

        Maintain maxVersion = maintainService.getMaxVersion(tableName);
        Maintain maintain = new Maintain();
        String biggerVersion = null;
        if (maxVersion == null) {
            biggerVersion = Constant.VERSION_Default;
            maintain.setOrderNo(0);
            operateLog.setOperate("初始化主题数据");
        } else {
            operateLog.setOperate("更新主题");
            String flowId = maxVersion.getFlowId();
            String mVersion = maxVersion.getVersion();
            if (StringUtils.isEmpty(flowId)) {
                biggerVersion = mVersion;
                maintain.setOrderNo(maxVersion.getOrderNo());
                String tempName = maxVersion.getTableName() + Constant.RECORD;

                tableInfoMapper.deleteTempByMaintainId(tempName, DbUtils.quotedStr(maxVersion.getId()));
                maintainDetailService.delete(new EntityWrapper<MaintainDetail>().eq("parent_id", maxVersion.getId()));

                maxVersion.deleteById();
            }else {
                biggerVersion = DbUtils.versionAddBig(mVersion);
                maintain.setOrderNo(DbUtils.getOrderNoAdd(maxVersion.getOrderNo()));
            }
        }

        String uuid = DbUtils.getUUID();
        maintain.setId(uuid);
        uploadType = SysAssembleUpdateType.valueOf(uploadTypeStr);
        TUser user = (TUser) request.getSession().getAttribute(Constant.USER);
        if (user == null) {
            return Result.error(CodeMsg.ERROR_PARAMS_NOT_MATHED);
        }
        maintain.setVersion(biggerVersion);
        maintain.setChargeId(user.getUserId());
        maintain.setTableName(tableName);
        maintain.setCreateTime(new Date());


        tableInfoMapper.insertMatintainDetailFromTemp(DbUtils.quotedStr(maintain.getId()), maintain.getTableName() + Constant.RECORD);

        tableInfoMapper.updateStdId( maintain.getTableName() + Constant.RECORD);

        Long aLong = tableInfoMapper.tempDeal(maintain.getTableName() + Constant.RECORD, DbUtils.quotedStr(maintain.getId()));

        maintain.setDesp(MessageFormat.format("上传数据{0}条", aLong));

        operateLog.setCreateTime(new Date())
                .setMaintainId(maintain.getId())
                .setId(DbUtils.getUUID())
                .setMaintainId(menuMapping.getId())
                .setOperate("open")
                .setDesp(MessageFormat.format("上传数据{0}条", aLong));

        maintain.insert();
        Maintain maintain = masterDataService.uploadedData(tableName, uploadType, user.getUserId());
        JSONObject object = new JSONObject();
        object.fluentPut("maintainId",maintain.getId());
        object.fluentPut("version",maintain.getVersion());
@@ -190,8 +138,9 @@
            case delete:
                //datas is id;
                Integer deleteCount = tableInfoMapper.deleteRecordByMaintainId(maintain.getTableName() + Constant.RECORD, DbUtils.quotedStr(datas), DbUtils.quotedStr(maintainId));

                result = Result.success(deleteCount);


                break;
            case update:
                JSONObject jsonObject = JSONObject.parseObject(datas);
@@ -282,8 +231,9 @@
            String nowMaintainId = nowMaintain.getId();
            String tempId = tableInfoMapper.getTempIdByStdId(tableName + Constant.RECORD,DbUtils.quotedStr(datas), DbUtils.quotedStr(nowMaintainId));
            if (!StringUtils.isEmpty(tempId)){
                maintainDetailService.delete(new EntityWrapper<MaintainDetail>().eq("parent_id", maintain.getId()).eq("operate",Operate.delete.name()).eq("pre_merge_id",tempId));
               return masterDataService.deleteById(tableName + Constant.RECORD, tempId);
                maintainDetailService.delete(new EntityWrapper<MaintainDetail>().eq("parent_id", maintain.getId()).eq("pre_merge_id",tempId));
                result = masterDataService.deleteById(tableName + Constant.RECORD, tempId);
                return result;
            }else {
                result = masterDataService.selectById(tableName, datas);
                Object data = result.getData();

 src/main/java/com/highdatas/mdm/controller/MenuMappingController.java

@@ -38,10 +38,12 @@
    public Result<MenuMapping> get(@PathVariable String id, HttpServletRequest request) {
        return menuMappingService.getMapping(request.getSession(),id);
    }


    @RequestMapping(value = "/add", method = RequestMethod.POST)
    public Result<Object> insert(@RequestParam String data, HttpServletRequest request) {
        HttpSession session = request.getSession();
        MenuMapping inserted =menuMappingService.create(data, session);
        MenuMapping inserted = menuMappingService.create(data, session);
        if (inserted != null) {
            return  Result.success(inserted);
        } else {

 src/main/java/com/highdatas/mdm/controller/SettleController.java

New file
@@ -0,0 +1,772 @@
package com.highdatas.mdm.controller;



import com.alibaba.druid.pool.DruidPooledConnection;

import com.alibaba.fastjson.JSON;

import com.alibaba.fastjson.JSONArray;

import com.alibaba.fastjson.JSONObject;



import com.highdatas.mdm.entity.TUser;

import com.highdatas.mdm.pojo.kettle.UnBigDataDataSourceInfo;

import com.highdatas.mdm.pojo.kettle.KettleDBTrans;

import com.highdatas.mdm.pojo.kettle.TableColumn;

import org.pentaho.di.core.KettleClientEnvironment;

import org.pentaho.di.core.database.Database;

import org.pentaho.di.core.database.DatabaseMeta;

import org.pentaho.di.core.logging.LoggingObjectInterface;

import org.pentaho.di.core.logging.LoggingObjectType;

import org.pentaho.di.core.logging.SimpleLoggingObject;

import org.pentaho.di.core.row.RowMetaInterface;

import org.pentaho.di.trans.Trans;

import org.pentaho.di.trans.TransHopMeta;

import org.pentaho.di.trans.TransMeta;

import org.pentaho.di.trans.step.StepMeta;

import org.pentaho.di.trans.steps.dummytrans.DummyTransMeta;



import org.pentaho.di.trans.steps.tableinput.TableInputMeta;

import org.pentaho.di.trans.steps.tableoutput.TableOutputMeta;



import org.springframework.beans.factory.annotation.Autowired;

import org.springframework.stereotype.Controller;

import org.springframework.web.bind.annotation.RequestBody;

import org.springframework.web.bind.annotation.RequestMapping;

import org.springframework.web.bind.annotation.ResponseBody;



import javax.servlet.http.HttpServletResponse;

import java.sql.Connection;

import java.sql.PreparedStatement;

import java.sql.ResultSet;

import java.sql.ResultSetMetaData;

import java.text.SimpleDateFormat;

import java.util.ArrayList;

import java.util.Date;

import java.util.List;



@Controller

@RequestMapping("/database")

//http://localhost:8080/users/addUser

public class SettleController {



	
    @Autowired

    private UnBigDataDataSourceInfo dataSourceConn;

	
    private SimpleDateFormat sdf = new SimpleDateFormat("yyyyMMddHHmmss");

    private String STEP_READ_FROM_TABLE = "Read data from table";

    private String STEP_INSERT_UPDATE = "Insert or update";

    private String STEP_DUMMY = "Dummy";

	


    /**

     * 提取mongodb数据

     * @param response

     * @param transData

     * @return

     * @throws

     */

//    @RequestMapping("/drawMongoDB")

//    @ResponseBody

//    public String drawMongoDB(HttpServletResponse response, @RequestBody JSONObject transData) {

//        String returnStr = "";

//        try {

//             KettleClientEnvironment.init();

//

//             TransMeta transMeta = new TransMeta();

//             transMeta.setName("抽取mongodb数据");

//             //导出数据的数据库连接

//             DatabaseMeta dataBaseOutput = new DatabaseMeta("Output", dataSourceConn.getDbType(), "Native", dataSourceConn.getDbHostName(), dataSourceConn.getDbName(), dataSourceConn.getDbPort(),

//                                                            dataSourceConn.getUsername(), dataSourceConn.getPassword());

//             transMeta.addDatabase(dataBaseOutput);

//

//                 MongoDbInputMeta mongoDbInputMeta = new MongoDbInputMeta();

//                 mongoDbInputMeta.setHostnames(transData.getString("hostName"));

//                 mongoDbInputMeta.setAuthenticationDatabaseName(transData.getString("authDBName"));

//                 mongoDbInputMeta.setPort(transData.getString("portNum"));

//                 mongoDbInputMeta.setAuthenticationUser(transData.getString("userName"));

//                 mongoDbInputMeta.setAuthenticationPassword(transData.getString("pwd"));

//                 mongoDbInputMeta.setDbName(transData.getString("dbName"));

//                 mongoDbInputMeta.setCollection(transData.getString("collection"));

//                 //得到抽取字段

//                 ArrayList<String> fieldList = new ArrayList<String>(transData.getString("drawfield").split(",").length);

//                 Collections.addAll(fieldList, transData.getString("drawfield").split(","));

//                 //设置不取JSON字段

//                 mongoDbInputMeta.setOutputJson(false);

//             //设置提取字段名称、path、type

//             List<MongoField> normalList = new ArrayList<MongoField>();

//             for (String fieldStr : fieldList) {

//                  MongoField newField = new MongoField();

//                  newField.m_fieldName = fieldStr;

//                  newField.m_fieldPath = "$." + fieldStr;

//                  newField.m_kettleType = "String";

//                  normalList.add(newField);

//             }

//             //设置mongodb提取字段

//             mongoDbInputMeta.setMongoFields(normalList);

//             //设置mogodb步骤元

//             StepMeta inputMongoDBStep = new StepMeta(STEP_READ_FROM_TABLE, mongoDbInputMeta);

//             inputMongoDBStep.setLocation(50, 50);

//             inputMongoDBStep.setDraw(true);

//             //将mogodb步骤元加入转化中

//             transMeta.addStep(inputMongoDBStep);

//

//             //设置mysql元

//             TableOutputMeta tableOutputMeta = new TableOutputMeta();

//             //设置数据库元

//             tableOutputMeta.setDatabaseMeta(transMeta.findDatabase("Output"));

//             //mongodb中数据库表/集合,就是表名

//             tableOutputMeta.setTableName(transData.getString("collection"));

//             //将mysql元加入步骤元

//             StepMeta insertUpdateStep = new StepMeta(STEP_INSERT_UPDATE, tableOutputMeta);

//             insertUpdateStep.setLocation(150, 50);

//             insertUpdateStep.setDraw(true);

//

//             transMeta.addStep(insertUpdateStep);

//

//             //增加个空元，

//             DummyTransMeta dummyMeta = new DummyTransMeta();

//

//             //将空元加入步骤元

//             StepMeta dummyStep = new StepMeta(STEP_DUMMY,dummyMeta);

//             dummyStep.setLocation(200, 50);

//             dummyStep.setDraw(true);

//

//             transMeta.addStep(dummyStep);

//

//             //设置步骤直接的关系

//             TransHopMeta hop = new TransHopMeta(inputMongoDBStep, insertUpdateStep);

//             transMeta.addTransHop(hop);

//             TransHopMeta hop2 = new TransHopMeta(insertUpdateStep, dummyStep);

//             transMeta.addTransHop(hop2);

//

//             //开始执行数据抽取

//             //将转化元实例化转换

//             Trans trans = new Trans(transMeta);

//             trans.prepareExecution(null);

//

//             trans.startThreads();

//             trans.waitUntilFinished();

//

//             if (trans.getErrors() > 0) {

//                 System.out.println(">>>>>>>>>> ERROR");

//                 returnStr = "{result:\"fail\"}";

//             }

//             else {

//                 System.out.println(">>>>>>>>>> SUCCESS ");

//                 returnStr = "{result:\"success\"}";

//             }

//        } catch(Exception e) {

//             return "{result:\"fail\"}";

//        }

//        return returnStr;

//    }

//

//    /**

//     * 提取mongodb数据

//     * @param response

//     * @param transData

//     * @return

//     * @throws

//     */

//    @RequestMapping("/drawHbase")

//    @ResponseBody

//    public String drawHbase(HttpServletResponse response, @RequestBody JSONObject transData) {

//        String returnStr = "";

//        try {

//            KettleClientEnvironment.init();

//

//            TransMeta transMeta = new TransMeta();

//            transMeta.setName("抽取Hbase数据");

//            //导出数据的数据库连接

//            DatabaseMeta dataBaseOutput = new DatabaseMeta("Output", dataSourceConn.getDbType(), "Native", dataSourceConn.getDbHostName(), dataSourceConn.getDbName(), dataSourceConn.getDbPort(),

//                    dataSourceConn.getUsername(), dataSourceConn.getPassword());

//            transMeta.addDatabase(dataBaseOutput);

//            //HBaseInputMeta hBaseInputMeta= new HBaseInputMeta()

//

//            MongoDbInputMeta mongoDbInputMeta = new MongoDbInputMeta();

//            mongoDbInputMeta.setHostnames(transData.getString("hostName"));

//            mongoDbInputMeta.setAuthenticationDatabaseName(transData.getString("authDBName"));

//            mongoDbInputMeta.setPort(transData.getString("portNum"));

//            mongoDbInputMeta.setAuthenticationUser(transData.getString("userName"));

//            mongoDbInputMeta.setAuthenticationPassword(transData.getString("pwd"));

//            mongoDbInputMeta.setDbName(transData.getString("dbName"));

//            mongoDbInputMeta.setCollection(transData.getString("collection"));

//            //得到抽取字段

//            ArrayList<String> fieldList = new ArrayList<String>(transData.getString("drawfield").split(",").length);

//            Collections.addAll(fieldList, transData.getString("drawfield").split(","));

//            //设置不取JSON字段

//            mongoDbInputMeta.setOutputJson(false);

//            //设置提取字段名称、path、type

//            List<MongoField> normalList = new ArrayList<MongoField>();

//            for (String fieldStr : fieldList) {

//                MongoField newField = new MongoField();

//                newField.m_fieldName = fieldStr;

//                newField.m_fieldPath = "$." + fieldStr;

//                newField.m_kettleType = "String";

//                normalList.add(newField);

//            }

//            //设置mongodb提取字段

//            mongoDbInputMeta.setMongoFields(normalList);

//            //设置mogodb步骤元

//            StepMeta inputMongoDBStep = new StepMeta(STEP_READ_FROM_TABLE, mongoDbInputMeta);

//            inputMongoDBStep.setLocation(50, 50);

//            inputMongoDBStep.setDraw(true);

//            //将mogodb步骤元加入转化中

//            transMeta.addStep(inputMongoDBStep);

//

//            //设置mysql元

//            TableOutputMeta tableOutputMeta = new TableOutputMeta();

//            //设置数据库元

//            tableOutputMeta.setDatabaseMeta(transMeta.findDatabase("Output"));

//            //mongodb中数据库表/集合,就是表名

//            tableOutputMeta.setTableName(transData.getString("collection"));

//            //将mysql元加入步骤元

//            StepMeta insertUpdateStep = new StepMeta(STEP_INSERT_UPDATE, tableOutputMeta);

//            insertUpdateStep.setLocation(150, 50);

//            insertUpdateStep.setDraw(true);

//

//            transMeta.addStep(insertUpdateStep);

//

//            //增加个空元，

//            DummyTransMeta dummyMeta = new DummyTransMeta();

//

//            //将空元加入步骤元

//            StepMeta dummyStep = new StepMeta(STEP_DUMMY,dummyMeta);

//            dummyStep.setLocation(200, 50);

//            dummyStep.setDraw(true);

//

//            transMeta.addStep(dummyStep);

//

//            //设置步骤直接的关系

//            TransHopMeta hop = new TransHopMeta(inputMongoDBStep, insertUpdateStep);

//            transMeta.addTransHop(hop);

//            TransHopMeta hop2 = new TransHopMeta(insertUpdateStep, dummyStep);

//            transMeta.addTransHop(hop2);

//

//            //开始执行数据抽取

//            //将转化元实例化转换

//            Trans trans = new Trans(transMeta);

//            trans.prepareExecution(null);

//

//            trans.startThreads();

//            trans.waitUntilFinished();

//

//            if (trans.getErrors() > 0) {

//                System.out.println(">>>>>>>>>> ERROR");

//                returnStr = "{result:\"fail\"}";

//            }

//            else {

//                System.out.println(">>>>>>>>>> SUCCESS ");

//                returnStr = "{result:\"success\"}";

//            }

//        } catch(Exception e) {

//            return "{result:\"fail\"}";

//        }

//        return returnStr;

//    }



    /**

     * 测试数据库连接

     * @param response

     * @param transData

     * @return

     * @throws Exception

     */

    @RequestMapping("/connectionDB")

    @ResponseBody

    public String connectionDB(HttpServletResponse response, @RequestBody JSONObject transData, TUser users) throws Exception {

        //userService.addUser(users);

        KettleClientEnvironment.init();

        DatabaseMeta dataMeta = new DatabaseMeta("KettleDBRep", transData.getString("type"), "Native", transData.getString("hostName"), transData.getString("dbName"), transData.getString("portNum"),

                                                 transData.getString("userName"), transData.getString("pwd"));

        LoggingObjectInterface loggingObject = new SimpleLoggingObject("Database factory", LoggingObjectType.GENERAL, null );

        Database ds = new Database(loggingObject, dataMeta);

        ds.normalConnect(null);

        Connection conn = ds.getConnection(); 
/*        PreparedStatement stmt = conn.prepareStatement("select * from usr");

        ResultSet rslt = stmt.executeQuery();

        while (rslt.next()) {

            System.out.println(rslt.getString("username"));

        }*/

        ds.disconnect();

        return "{result:\"success\"}";

    }

	
    /**

     * 创建SQL中tablename

     * 返回表名

     */

    public String createSQLTable(DruidPooledConnection connSys, ResultSet rslt) throws Exception {

        List<TableColumn> columnNameAndType = new ArrayList<TableColumn>();

        ResultSetMetaData md = rslt.getMetaData();

        for (int i = 1; i <= md.getColumnCount(); i++) {

             TableColumn tableColumn = new TableColumn();

             tableColumn.setColumnName(md.getColumnName(i));

             tableColumn.setColumnType(md.getColumnTypeName(i));

             tableColumn.setColumnLength(String.valueOf(md.getColumnDisplaySize(i)));

             columnNameAndType.add(tableColumn);			 
        }

        //创建表

        String SQLStr = "", tableName = "";

        if (!columnNameAndType.isEmpty()) {

            tableName = "SQL_" + sdf.format(new Date());

            SQLStr = splicingSQL(tableName, columnNameAndType);

            PreparedStatement stmt = connSys.prepareStatement(SQLStr);

            stmt.execute();	
        }

        return tableName;	    
    }

	
    /**

     * 通过SQL提取数据

     */

    @RequestMapping("/sqlPick")

    @ResponseBody

    public String sqlPick(@RequestBody JSONObject dbData) {

        try {

             String tableName = "", sqlStr = "";



             Database dsInput = null;

             Database dsOutput = null;

             TableInputMeta tableInputMeta = new TableInputMeta();

             TableOutputMeta talbeOutputMeta = new TableOutputMeta();

             TransMeta transMeta = new TransMeta();

             KettleDBTrans kettleDBTrans = new KettleDBTrans();

             kettleDBTrans.setDsInput(dsInput);

             kettleDBTrans.setDsOutput(dsOutput);

             kettleDBTrans.setTableInputMeta(tableInputMeta);

             kettleDBTrans.setTalbeOutputMeta(talbeOutputMeta);

             kettleDBTrans.setTransMeta(transMeta);

			 
             kettleDBConn(kettleDBTrans);

		     
             //SQL提取数据

             tableName = "SQL_" + sdf.format(new Date());

             sqlStr = dbData.getString("sqlStr");

             dataExtraction(tableName, sqlStr, kettleDBTrans);

             kettleDBTrans.getDsInput().disconnect();

             kettleDBTrans.getDsOutput().disconnect();

        } catch(Exception e) {

             return "{result:\"fail\"}";	
        }

        return "{result:\"success\"}";

    }

    /**

     * 抽取SQL数据

     */

    public void  extractSQLData(DruidPooledConnection connSys, ResultSet rslt, String tableName) throws Exception {

        PreparedStatement stmtDes;

        int columnCount;

        String SQLStr = "", valueStr = "";

        columnCount = rslt.getMetaData().getColumnCount();

        SQLStr = " insert into "+tableName+" ( ";

        valueStr = " values (";

        for(int i = 0; i < columnCount; i++) {

            SQLStr = SQLStr + rslt.getMetaData().getColumnName(i+1) + ", ";

            valueStr = valueStr + "?, ";

        }

        SQLStr = SQLStr.substring(0, SQLStr.length() - 2) + ")";

        valueStr = valueStr.substring(0, valueStr.length() - 2) + ")";

        SQLStr = SQLStr + valueStr;

        stmtDes = connSys.prepareStatement(SQLStr);

        while (rslt.next()) {

               for(int i = 0; i < columnCount; i++) {

                   stmtDes.setString(i + 1, rslt.getString(i+1));

               }

               stmtDes.addBatch();

        }

        stmtDes.executeBatch();		
    }

    /**

     * 测试SQL语句正确性

     */

    @RequestMapping("/sqlTest")

    @ResponseBody

    public String sqlTest(@RequestBody JSONObject dbData) {

		
        try {

             //连接数据库

             String dbConn = dbData.getString("DBConn");

             JSONObject DBParam = JSONArray.parseObject(dbConn);

			 
             KettleClientEnvironment.init();

             DatabaseMeta dataMeta = new DatabaseMeta("KettleDBRep", DBParam.getString("type"), "Native", DBParam.getString("hostName"), DBParam.getString("dbName"), DBParam.getString("portNum"),

                                                       DBParam.getString("userName"), DBParam.getString("pwd"));

             LoggingObjectInterface loggingObject = new SimpleLoggingObject("Database factory", LoggingObjectType.GENERAL, null );

             Database ds = new Database(loggingObject, dataMeta);

             ds.connect();

             Connection conn = ds.getConnection();

             String sqlStr = dbData.getString("sqlStr");

             PreparedStatement stmt = conn.prepareStatement(sqlStr);

             stmt.executeQuery();	
             ds.disconnect();

        } catch(Exception e) {

             return "{result:\"fail\"}";	
        }

        return "{result:\"success\"}";		
    }

    /**

     * 当选中数据表选项卡，显示数据表及视图名称

     * @param response

     * @param transData

     * @return

     * @throws Exception

     */

    @RequestMapping("/showTableAndView")

    @ResponseBody

    public String showTableAndView(HttpServletResponse response, @RequestBody JSONObject transData) throws Exception {

        KettleClientEnvironment.init();

        DatabaseMeta dataMeta = new DatabaseMeta("KettleDBRep", transData.getString("type"), "Native", transData.getString("hostName"), transData.getString("dbName"), transData.getString("portNum"),

                                                 transData.getString("userName"), transData.getString("pwd"));

		
        LoggingObjectInterface loggingObject = new SimpleLoggingObject("Database factory", LoggingObjectType.GENERAL, null );

        Database ds = new Database(loggingObject, dataMeta);

        //ds.normalConnect(null);

        ds.connect();

        Connection conn = ds.getConnection();

        List<String> UTable = new ArrayList<String>();

        List<String> VTable = new ArrayList<String>();

        //得到数据库中的表

//        PreparedStatement stmt = conn.prepareStatement("select name from sysobjects where xtype='U'");

//        ResultSet rslt = stmt.executeQuery();

//        while (rslt.next()) {

//               UTable.add(rslt.getString("name"));

//        }

//        String[] UTableArray = ds.getTablenames("dbo", true);

//        for (int i = 0; i < UTableArray.length; i++) {

//             UTable.add(UTableArray[i].replace("dbo.", ""));

//        }

        String[] UTableArray = ds.getTablenames(true);

        for (int i = 0; i < UTableArray.length; i++) {

             int lastHao = UTableArray[i].lastIndexOf(".");

             String uTable = UTableArray[i];

             if (lastHao > 0) {

                 UTable.add(uTable.substring(lastHao + 1, uTable.length()));

             }

             else {

                 UTable.add(UTableArray[i]);

             }

		     
        }		
        String tableJson = JSON.toJSONString(UTable);

        //得到数据库中的视图

//        stmt = conn.prepareStatement("select name from sysobjects where xtype='V'");

//        rslt = stmt.executeQuery();

//        while (rslt.next()) {

//               VTable.add(rslt.getString("name"));

//        }	
        String[] VTableArray = ds.getViews("dbo", true);

        for (int i = 0; i < VTableArray.length; i++) {

             VTable.add(VTableArray[i].replace("dbo.", ""));

        }			
        String viewJson = JSON.toJSONString(VTable);

        ds.disconnect();

        return "{result:\"success\",table:["+tableJson+"],view:["+viewJson+"]}";

    }

    /**

     * 使用kettle创建表 
     */

    public void createTable(Database dsOutput, RowMetaInterface rm, String tableName) {

       try {

            System.out.println("开始创建表:" + tableName);

            String dbProName = dsOutput.getConnection().getMetaData().getDatabaseProductName();

            System.out.println("数据库类型:" + dbProName);

            String sql = dsOutput.getDDL(tableName, rm);

            dsOutput.execStatement(sql.replace(";", ""));

            System.out.println("创建表成功");		   
       } catch(Exception e) {

            System.out.println("创建表失败");

       }



    }

    /**

     * 新版抽取数据

     * dataExtraction(String tableName, String sqlStr, TransMeta transMeta, Database dsInput, Database dsOutput, TableInputMeta tableInputMeta, TableOutputMeta tableOutputMeta) throws Exception {

       tableInputMeta.setSQL(sqlStr);

     */

    public void dataExtraction(String tableName, String sqlStr, KettleDBTrans kettleDBTrans) throws Exception {

       kettleDBTrans.getTableInputMeta().setSQL(sqlStr);

	   
       StepMeta inputStep = new StepMeta(STEP_READ_FROM_TABLE, kettleDBTrans.getTableInputMeta());

       inputStep.setLocation(50, 50);

       inputStep.setDraw(true);

       kettleDBTrans.getTransMeta().addStep(inputStep);

	   
       //判断目标数据库中，是否存在表，如:存在重新命名表名，用新表名创建表，如:不存在,直接创建表

       boolean hasTable = kettleDBTrans.getDsOutput().checkTableExists(tableName);

       if (hasTable) {

           tableName = tableName + "_" + sdf.format(new Date());

       }

       //得到原始表结构

       PreparedStatement stmt = null;

       RowMetaInterface rm = null;

       stmt = kettleDBTrans.getDsInput().getConnection().prepareStatement(sqlStr);

       stmt.executeQuery();

       rm = kettleDBTrans.getDsInput().getQueryFields(sqlStr, false);

       //创建数据表

       createTable(kettleDBTrans.getDsOutput(), rm, tableName);

	   
       kettleDBTrans.getTalbeOutputMeta().setTableName(tableName);

	   
       StepMeta insertUpdateStep = new StepMeta(STEP_INSERT_UPDATE, kettleDBTrans.getTalbeOutputMeta());

       insertUpdateStep.setLocation(150, 50);

       insertUpdateStep.setDraw(true);

       kettleDBTrans.getTransMeta().addStep(insertUpdateStep);

	   
       DummyTransMeta dummyMeta = new DummyTransMeta();

	   
       StepMeta dummyStep = new StepMeta(STEP_DUMMY, dummyMeta);

       dummyStep.setLocation(200, 50);

       dummyStep.setDraw(true);

	   
       kettleDBTrans.getTransMeta().addStep(dummyStep);

	   
       //设置步骤直接的关系

       TransHopMeta hop = new TransHopMeta(inputStep, insertUpdateStep);

       kettleDBTrans.getTransMeta().addTransHop(hop);

       TransHopMeta hop2 = new TransHopMeta(insertUpdateStep, dummyStep);

       kettleDBTrans.getTransMeta().addTransHop(hop2);

	   
       //开始执行

       Trans trans = new Trans(kettleDBTrans.getTransMeta());

       trans.prepareExecution(null);

	   
       trans.startThreads();

	   
       trans.waitUntilFinished();

	   
       if (trans.getErrors() > 0) {

           System.out.println(">>>>>>>>>> ERROR");

       }

       else {

           System.out.println(">>>>>>>>>> SUCCESS ");

       }

    }

    /**

     * 创建集成方法

     */

    public void kettleDBConn(KettleDBTrans kettleDBTrans) throws Exception {

        //连接数据库

//        String dbConn = kettleDBTrans.getAssemble().getString("DBConn");\

        String dbConn= "";

        JSONObject DBParam = JSONArray.parseObject(dbConn);

        //初始化kettle环境

        KettleClientEnvironment.init();

        KettleClientEnvironment.init();

        //导入数据的数据库连接

        DatabaseMeta dataMetaInput = new DatabaseMeta("Input", DBParam.getString("type"), "Native", DBParam.getString("hostName"), DBParam.getString("dbName"), DBParam.getString("portNum"),

                                                      DBParam.getString("userName"), DBParam.getString("pwd"));

        //导出数据的数据库连接

        DatabaseMeta dataMetaOutput = new DatabaseMeta("Output", dataSourceConn.getDbType(), "Native", dataSourceConn.getDbHostName(), dataSourceConn.getDbName(), dataSourceConn.getDbPort(),

                                                       dataSourceConn.getUsername(), dataSourceConn.getPassword());

        //导出数据库连接

        LoggingObjectInterface loggingObject = new SimpleLoggingObject("Database factory", LoggingObjectType.GENERAL, null );

        kettleDBTrans.setDsInput(new Database(loggingObject, dataMetaInput));

        kettleDBTrans.getDsInput().connect();

        Connection connInput = kettleDBTrans.getDsInput().getConnection();

        //导入数据库连接

        kettleDBTrans.setDsOutput(new Database(loggingObject, dataMetaOutput));

        kettleDBTrans.getDsOutput().connect();

        Connection connOutput = kettleDBTrans.getDsOutput().getConnection();

		
        kettleDBTrans.getTransMeta().setName("数据抽取");

		
        kettleDBTrans.getTransMeta().addDatabase(dataMetaInput);

        kettleDBTrans.getTransMeta().addDatabase(dataMetaOutput);

		
        //导出数据

        kettleDBTrans.getTableInputMeta().setDatabaseMeta(kettleDBTrans.getTransMeta().findDatabase("Input"));

        //导入数据

        kettleDBTrans.getTalbeOutputMeta().setDatabaseMeta(kettleDBTrans.getTransMeta().findDatabase("Output"));		
    }

    /**

     * 1.得到要提取数据的表与视图名称

     * 2.在程序数据库中创建要提取数据的表与视图的表的名称

     * 3.将要提取表与视图中的数据存储到系统的实体表中

     * 4.最后，返回结果

     */

    //提取表、视图中的数据

    @RequestMapping("/extractDBData")

    @ResponseBody

    public String extractDBData(HttpServletResponse response, @RequestBody JSONObject dbData) throws Exception {

        //表

        int k = 0;

        String sqlStr = "", tableName = "";

		
        Database dsInput = null;

        Database dsOutput = null;

        TableInputMeta tableInputMeta = new TableInputMeta();

        TableOutputMeta talbeOutputMeta = new TableOutputMeta();

        TransMeta transMeta = new TransMeta();

        KettleDBTrans kettleDBTrans    = new KettleDBTrans();

//        kettleDBTrans.setDbData(dbData);

        kettleDBTrans.setDsInput(dsInput);

        kettleDBTrans.setDsOutput(dsOutput);

        kettleDBTrans.setTableInputMeta(tableInputMeta);

        kettleDBTrans.setTalbeOutputMeta(talbeOutputMeta);

        kettleDBTrans.setTransMeta(transMeta);

		
        kettleDBConn(kettleDBTrans);

		
        JSONArray uTable = dbData.getJSONArray("UTable");

        //循环实体表

        if (!uTable.isEmpty()) {

            for(k = 0; k < uTable.size(); k++) {

                tableName = uTable.getString(k);

                sqlStr = " select * from " + tableName;

                dataExtraction(tableName, sqlStr, kettleDBTrans);

            }

        }

		
        JSONArray vTable = dbData.getJSONArray("VTable");

        //循环视图

        if (!vTable.isEmpty()) {

            for(k = 0; k < vTable.size(); k++) {

                tableName = vTable.getString(k);

                sqlStr = " select * from " + tableName;

                dataExtraction(tableName, sqlStr, kettleDBTrans);

            }

        }		
        //关闭数据库连接

        kettleDBTrans.getDsInput().disconnect();

        kettleDBTrans.getDsOutput().disconnect();

        return "{result: \"success\"}";

    }

	
    /**

     *     抽取数据

     * extractData(connSys, realTableName, conn, tableName);

     * connSys 系统数据库

     * realTableName 系统数据库表名

     * conn 提取数据库

     * tableName 提取数据库表名 
     */

    public void extractData(DruidPooledConnection connSys, String tableNameDes, Connection conn, String tableNameSrc) throws Exception {

        PreparedStatement stmt, stmtDes;

        ResultSet rslt;

        int columnCount;

        String SQLStr = "", valueStr = "";

		
        SQLStr = " select * from "+tableNameSrc+" ";

        stmt = conn.prepareStatement(SQLStr);

        rslt = stmt.executeQuery();

        columnCount = rslt.getMetaData().getColumnCount();

        SQLStr = " insert into "+tableNameDes+" ( ";

        valueStr = " values (";

        for(int i = 0; i < columnCount; i++) {

            SQLStr = SQLStr + rslt.getMetaData().getColumnName(i+1) + ", ";

            valueStr = valueStr + "?, ";

        }

        SQLStr = SQLStr.substring(0, SQLStr.length() - 2) + ")";

        valueStr = valueStr.substring(0, valueStr.length() - 2) + ")";

        SQLStr = SQLStr + valueStr;

        stmtDes = connSys.prepareStatement(SQLStr);

        while (rslt.next()) {

               for(int i = 0; i < columnCount; i++) {

                   stmtDes.setString(i + 1, rslt.getString(i+1));

               }

               stmtDes.addBatch();

        }

        stmtDes.executeBatch();

    }

	
    /**

     * 根据数据库类型转换对应类型

     * 
     */

    public String getColumnStr(TableColumn tableColumn) throws Exception {

        StringBuffer sqlStr = new StringBuffer();

        String returnStr = "";

        if (tableColumn.getColumnType().contains("varchar")) {

            sqlStr.append(tableColumn.getColumnName() + " varchar(" + tableColumn.getColumnLength() + "),");

        }

        else if (tableColumn.getColumnType().contains("int")) {

            sqlStr.append(tableColumn.getColumnName() + " int(" + tableColumn.getColumnLength() + "),");

        }

        else if (tableColumn.getColumnType().contains("date")) {

            sqlStr.append(tableColumn.getColumnName() + " date,");

        }

        else if (tableColumn.getColumnType().contains("nvarchar")) {

            sqlStr.append(tableColumn.getColumnName() + " varchar(" + tableColumn.getColumnLength() + "),");

        }

        else if (tableColumn.getColumnType().contains("nchar")) {

            sqlStr.append(tableColumn.getColumnName() + " varchar(" + tableColumn.getColumnLength() + "),");

        }	
        return sqlStr.toString();

    }

	
    /**

     * 拼接创建新表的语句

     * 1.系统数据库是mysql,所以，需要将其它数据库的类型，转换成mysql可以识别的类型

     */

    public String splicingSQL(String tableName,  List<TableColumn> columnNameAndType) throws Exception {

        String sqlStr = "";

        //第一步,类型 识别类型

        sqlStr = "create table " + tableName + " (";

        StringBuffer columnStr = new StringBuffer();

        columnStr.append(sqlStr);

        for (TableColumn tableColumn : columnNameAndType) {

             columnStr.append(getColumnStr(tableColumn));

        }

        sqlStr = columnStr.toString();

        return sqlStr.substring(0, sqlStr.length() - 1) + ")";

    }

    /**

     * 创建表

     * 1.需要判段,系统数据库中是否已存在表名(1.有的话重新命名,tableName + 年月日时分秒

     */

    public String createTable(DruidPooledConnection connSys, String tableName, List<TableColumn> columnNameAndType) throws Exception {

        String SQLStr = "";

        if (!columnNameAndType.isEmpty()) {

            PreparedStatement stmt;

            ResultSet rslt;

			
            SQLStr = " select count(table_name) as tableNum "

                   + " from information_schema.tables "

                   + " where table_schema='ssm' and table_name='"+tableName+"'";

            stmt = connSys.prepareStatement(SQLStr);

            rslt = stmt.executeQuery();

            rslt.next();

            if (rslt.getInt("tableNum") > 0) {

                tableName = tableName + "_" + sdf.format(new Date());

            }

            SQLStr = splicingSQL(tableName, columnNameAndType);

            stmt = connSys.prepareStatement(SQLStr);

            stmt.execute();	
        }

        return tableName;

    }

	
	
    //得到抽取数据库表的字段名称及字段类型

    public List<TableColumn> getColumnNameAndType(Connection conn, String tableName, String type) throws Exception {

        List<TableColumn> columnNameAndType = new ArrayList<TableColumn>();

        //得到数据库中的表

        String SQLStr = " select c.name As ColumnsName , t.name as ColumnsType, c.length as ColumnsLength "

                       + " from SysObjects As o "

                       + " left join SysColumns As c on o.id=c.id "

                       + " left join SysTypes As t on c.xtype=t.xusertype "

                       + " where o.type = '"+type+"' and o.name='"+tableName+"' ";

	    
        PreparedStatement stmt = conn.prepareStatement(SQLStr);

        ResultSet rslt = stmt.executeQuery();

        while (rslt.next()) {

               TableColumn tableColumn = new TableColumn();

               tableColumn.setColumnName(rslt.getString("ColumnsName"));

               tableColumn.setColumnType(rslt.getString("ColumnsType"));

               tableColumn.setColumnLength(rslt.getString("ColumnsLength"));

               columnNameAndType.add(tableColumn);

        }

	    
        return columnNameAndType;

    }

	
	


/*    protected void sendJsonData(HttpServletResponse response) throws Exception{

        response.setContentType("text/javascript;charset=UTF-8");

        PrintWriter out = response.getWriter();

        out.println("alert(\"数据库连接成功!!!\")");

        out.flush();

        out.close();

    }*/	
	
}


 src/main/java/com/highdatas/mdm/controller/SysAssembleApiController.java

File was renamed from src/main/java/com/highdatas/mdm/controller/UserController.java
@@ -1,8 +1,9 @@
package com.highdatas.mdm.controller;


import org.springframework.stereotype.Controller;
import org.springframework.web.bind.annotation.RequestMapping;

import org.springframework.stereotype.Controller;

/**
 * <p>
@@ -10,10 +11,10 @@
 * </p>
 *
 * @author kimi
 * @since 2019-12-16
 * @since 2020-02-19
 */
@Controller
@RequestMapping("/user")
public class UserController {
@RequestMapping("/assemble/api")
public class SysAssembleApiController {

}

 src/main/java/com/highdatas/mdm/controller/SysAssembleDbController.java

New file
@@ -0,0 +1,195 @@
package com.highdatas.mdm.controller;


import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONObject;
import com.baomidou.mybatisplus.mapper.EntityWrapper;
import com.baomidou.mybatisplus.mapper.Wrapper;
import com.highdatas.mdm.entity.SysAssemble;
import com.highdatas.mdm.entity.SysAssembleDb;
import com.highdatas.mdm.entity.SysAssembleDbField;
import com.highdatas.mdm.entity.SysAssembleDbTable;
import com.highdatas.mdm.pojo.CodeMsg;
import com.highdatas.mdm.pojo.Result;
import com.highdatas.mdm.service.ISysAssembleDbFieldService;
import com.highdatas.mdm.service.ISysAssembleDbService;
import com.highdatas.mdm.service.ISysAssembleDbTableService;
import com.highdatas.mdm.service.ISysAssembleService;
import com.highdatas.mdm.util.Constant;
import com.highdatas.mdm.util.DbUtils;
import org.apache.commons.lang3.StringUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.transaction.annotation.Transactional;
import org.springframework.web.bind.annotation.*;

import javax.servlet.http.HttpServletRequest;
import java.sql.Connection;
import java.sql.DatabaseMetaData;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.*;
import java.util.stream.Collectors;

/**
 * <p>
 *  前端控制器
 * </p>
 *
 * @author kimi
 * @since 2020-02-19
 */
@RestController
@RequestMapping("/assemble/db")
public class SysAssembleDbController {
    @Autowired
    ISysAssembleDbService dbService;
    @Autowired
    ISysAssembleDbTableService dbTableService;
    @Autowired
    ISysAssembleDbFieldService dbFieldService;

    @Autowired
    ISysAssembleService assembleService;


    @RequestMapping(value = "/get/parent/{id}", method = RequestMethod.GET)
    public Result get(@PathVariable String id, HttpServletRequest request) {
        SysAssemble assemble = assembleService.selectById(id);
        if (assemble == null) {
            return Result.error(CodeMsg.ERROR_PARAMS_NOT_MATHED);
        }
        //未分页 看需求  cnt

        Wrapper<SysAssembleDb> parentWrapper = new EntityWrapper<SysAssembleDb>().eq(Constant.PARENT_ID, id);
        List<SysAssembleDb> sysAssembleDbs = dbService.selectList(parentWrapper);
        ArrayList<JSONObject> result = new ArrayList<>();
        for (SysAssembleDb sysAssembleDb : sysAssembleDbs) {
            JSONObject o = (JSONObject) JSON.toJSON(sysAssembleDb);
            String dbId = sysAssembleDb.getId();
            Wrapper<SysAssembleDbTable> eq = new EntityWrapper<SysAssembleDbTable>().eq(Constant.PARENT_ID, dbId);
            int i = dbTableService.selectCount(eq);
            o.fluentPut("cnt", i);
            o.fluentPut("type", "db");
            result.add(o);
        }
        return Result.success(result);

    }

    @RequestMapping(value = "/testConnection/{id}", method = RequestMethod.GET)
    public Result testConnection(@PathVariable String id, HttpServletRequest request) {
        boolean b = dbService.testConnection(id);
        return Result.success(b);
    }

    @RequestMapping(value = "/getSet", method = RequestMethod.GET)
    public Result getSet(HttpServletRequest request) {
        List<SysAssembleDb> dbList = dbService.selectList(null);
        HashSet<String> result = new HashSet<>();
        for (SysAssembleDb sysAssembleDb : dbList) {
            result.add(sysAssembleDb.getDatasourceName());
        }
        return Result.success(result);
    }



    @RequestMapping(value = "/getTableList/{id}", method = RequestMethod.GET)
    public Result getTable(@PathVariable String id, HttpServletRequest request) {
        Connection connection = null;
        try {
            SysAssembleDb sysAssembleDb = dbService.selectById(id);
            String username = sysAssembleDb.getDatasourceUser();
            connection = dbService.getConnection(id);
            DatabaseMetaData metaData = connection.getMetaData();
            ResultSet tables = metaData.getTables( null, username.toUpperCase(), null, new String[]{ "TABLE" } );
            ArrayList<String> tableNameList = new ArrayList<>();
            while (tables.next()) {
                String tableName = tables.getString(Constant.TABLE_NAME);
                tableNameList.add(tableName);
            }
            List<SysAssembleDbTable> tableList = dbTableService.selectList(new EntityWrapper<SysAssembleDbTable>().eq(Constant.PARENT_ID, id));
            List<String> selectedTableNameList = tableList.stream().map(sysAssembleDbTable -> sysAssembleDbTable.getTableName()).collect(Collectors.toList());

            List<String> result = tableNameList.stream().filter(s -> !selectedTableNameList.contains(s)).collect(Collectors.toList());


            return Result.success(result);
        }
        catch (Exception e) {
            e.printStackTrace();
            return Result.error(new CodeMsg(6003, e.getMessage()));
        }
        finally {
            if (connection != null) {
                try {
                    connection.close();
                } catch (SQLException e) {
                    e.printStackTrace();
                    return Result.error(new CodeMsg(6003, e.getMessage()));
                }
            }
        }
    }


    @RequestMapping(value = "/getFields/{id}", method = RequestMethod.GET)
    public Result getFields(@PathVariable String id, @RequestParam String tableName, HttpServletRequest request) {
        List<String> fieldsByTableName = dbService.getFieldsByTableName(id, tableName);
        if (fieldsByTableName == null){
            return Result.error(new CodeMsg(6008, "获取字段失败"));
        }
        return Result.success(fieldsByTableName);
    }


    @Transactional(rollbackFor=Exception.class)
    @RequestMapping(value = "/delete/{id}", method = RequestMethod.GET)
    public Result delete(@PathVariable String id) {
        SysAssembleDb sysAssembleDb = dbService.selectById(id);
        if (sysAssembleDb == null) {
            return Result.error(CodeMsg.ERROR_PARAMS_NOT_MATHED);
        }
        // delete tables;
        Wrapper<SysAssembleDbTable> tablesWrapper = new EntityWrapper<SysAssembleDbTable>().eq("parent_id", id);
        List<SysAssembleDbTable> sysAssembleDbTables = dbTableService.selectList(tablesWrapper);
        for (SysAssembleDbTable dbTable : sysAssembleDbTables) {
            String tableId = dbTable.getId();
            boolean delete = dbFieldService.delete(new EntityWrapper<SysAssembleDbField>().eq("parent_id", tableId));
            if (delete) {
                delete = dbTable.deleteById();
                if (!delete) {
                    return Result.error(CodeMsg.DELETE_ERROR);
                }
            }else {
                return Result.error(CodeMsg.DELETE_ERROR);
            }
        }
        boolean delete = sysAssembleDb.deleteById();

        if (delete) {
            return Result.success(sysAssembleDb);
        }else {
            return Result.error(CodeMsg.UPDATE_ERROR);
        }
    }


    @ResponseBody
    @RequestMapping(value = "/addOrUpdate", method = RequestMethod.POST)
    public Result addOne(@RequestBody SysAssembleDb sysAssembleDb) {
        if (sysAssembleDb == null) {
            return Result.error(CodeMsg.ERROR_PARAMS_NOT_MATHED);
        }
        if (StringUtils.isEmpty(sysAssembleDb.getId())) {
            sysAssembleDb.setId(DbUtils.getUUID()).setCreateTime(new Date());
        }
        sysAssembleDb.setUpdateTime(new Date());
        boolean update = sysAssembleDb.insertOrUpdate();
        if (update) {
            return Result.success(sysAssembleDb);
        }else {
            return Result.error(CodeMsg.UPDATE_ERROR);
        }
    }
}

 src/main/java/com/highdatas/mdm/controller/SysAssembleDbDetailController.java

copy from src/main/java/com/highdatas/mdm/controller/UserController.java
copy to src/main/java/com/highdatas/mdm/controller/SysAssembleDbDetailController.java
File was copied from src/main/java/com/highdatas/mdm/controller/UserController.java
@@ -1,8 +1,9 @@
package com.highdatas.mdm.controller;


import org.springframework.stereotype.Controller;
import org.springframework.web.bind.annotation.RequestMapping;

import org.springframework.stereotype.Controller;

/**
 * <p>
@@ -10,10 +11,10 @@
 * </p>
 *
 * @author kimi
 * @since 2019-12-16
 * @since 2020-02-19
 */
@Controller
@RequestMapping("/user")
public class UserController {
@RequestMapping("/sysAssembleDbDetail")
public class SysAssembleDbDetailController {

}

 src/main/java/com/highdatas/mdm/controller/SysAssembleDbFieldController.java

copy from src/main/java/com/highdatas/mdm/controller/UserController.java
copy to src/main/java/com/highdatas/mdm/controller/SysAssembleDbFieldController.java
File was copied from src/main/java/com/highdatas/mdm/controller/UserController.java
@@ -10,10 +10,10 @@
 * </p>
 *
 * @author kimi
 * @since 2019-12-16
 * @since 2020-02-20
 */
@Controller
@RequestMapping("/user")
public class UserController {
@RequestMapping("/assemble/db/field")
public class SysAssembleDbFieldController {

}

 src/main/java/com/highdatas/mdm/controller/SysAssembleDbTableController.java

New file
@@ -0,0 +1,260 @@
package com.highdatas.mdm.controller;


import com.baomidou.mybatisplus.mapper.EntityWrapper;
import com.baomidou.mybatisplus.mapper.Wrapper;
import com.highdatas.mdm.entity.*;
import com.highdatas.mdm.pojo.CodeMsg;
import com.highdatas.mdm.pojo.Result;
import com.highdatas.mdm.pojo.SysAssembleTableType;
import com.highdatas.mdm.pojo.kettle.BigDataDataSourceInfo;
import com.highdatas.mdm.pojo.kettle.DataSourceInfo;
import com.highdatas.mdm.pojo.kettle.UnBigDataDataSourceInfo;
import com.highdatas.mdm.service.*;
import com.highdatas.mdm.util.Constant;
import com.highdatas.mdm.util.DbUtils;
import com.highdatas.mdm.util.RedisClient;
import org.apache.commons.lang3.StringUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.transaction.annotation.Transactional;
import org.springframework.web.bind.annotation.*;

import javax.servlet.http.HttpServletRequest;
import java.sql.Connection;
import java.text.MessageFormat;
import java.util.Date;
import java.util.List;
import java.util.Set;
import java.util.stream.Collectors;

/**
 * <p>
 *  前端控制器
 * </p>
 *
 * @author kimi
 * @since 2020-02-20
 */
@RestController
@RequestMapping("/assemble/db/table")
public class SysAssembleDbTableController {
    @Autowired
    ISysAssembleService assembleService;
    @Autowired
    ISysAssembleDbService dbService;
    @Autowired
    ISysAssembleDbTableService dbTableService;
    @Autowired
    ISysAssembleDbFieldService dbFieldService;
    @Autowired
    ISysAssembleParamsService paramsService;
    @Autowired
    UnBigDataDataSourceInfo unBigDataDataSourceInfo;
    @Autowired
    BigDataDataSourceInfo bigDataDataSourceInfo;
    @Autowired
    RedisClient redisClient;

    @RequestMapping(value = "/get/parent/{id}", method = RequestMethod.GET)
    public Result get(@PathVariable String id, HttpServletRequest request) {
        SysAssembleDb sysAssembleDb = dbService.selectById(id);
        if (sysAssembleDb == null) {
            return Result.error(CodeMsg.ERROR_PARAMS_NOT_MATHED);
        }
        //check 是否需要额外字段
        Wrapper<SysAssembleDbTable> parentWrapper = new EntityWrapper<SysAssembleDbTable>().eq(Constant.PARENT_ID, id);
        List<SysAssembleDbTable> sysAssembleDbs = dbTableService.selectList(parentWrapper);
        return Result.success(sysAssembleDbs);

    }

    @Transactional(rollbackFor=Exception.class)
    @RequestMapping(value = "/delete/{id}", method = RequestMethod.GET)
    public Result delete(@PathVariable String id) {
        SysAssembleDbTable sysAssembleDb = dbTableService.selectById(id);
        if (sysAssembleDb == null) {
            return Result.error(CodeMsg.ERROR_PARAMS_NOT_MATHED);
        }
        boolean delete = dbFieldService.delete(new EntityWrapper<SysAssembleDbField>().eq("parent_id", id));
        if (!delete) {
            return Result.error(CodeMsg.DELETE_ERROR);
        }
        delete = sysAssembleDb.deleteById();

        if (delete) {
            return Result.success(sysAssembleDb);
        } else {
            return Result.error(CodeMsg.DELETE_ERROR);
        }
    }

    @RequestMapping(value = "/get/{id}", method = RequestMethod.GET)
    public Result get(@PathVariable String id) {
        SysAssembleDbTable sysAssembleDb = dbTableService.selectById(id);
        if (sysAssembleDb == null) {
            return Result.error(CodeMsg.ERROR_PARAMS_NOT_MATHED);
        }
        Wrapper<SysAssembleDbField> fieldWrapper = new EntityWrapper<SysAssembleDbField>().eq(Constant.PARENT_ID, id);
        List<SysAssembleDbField> sysAssembleDbFields = dbFieldService.selectList(fieldWrapper);
        sysAssembleDb.setFieldList(sysAssembleDbFields);
        return Result.success(sysAssembleDb);
    }

    @RequestMapping(value = "/getField/{id}", method = RequestMethod.GET)
    public Result check(@PathVariable String id) {
        SysAssembleDbTable sysAssembleDb = dbTableService.selectById(id);
        if (sysAssembleDb == null) {
            return Result.error(CodeMsg.ERROR_PARAMS_NOT_MATHED);
        }
        Wrapper<SysAssembleDbField> fieldWrapper = new EntityWrapper<SysAssembleDbField>().eq(Constant.PARENT_ID, id);
        List<SysAssembleDbField> sysAssembleDbFields = dbFieldService.selectList(fieldWrapper);
        sysAssembleDb.setFieldList(sysAssembleDbFields);
        return Result.success(sysAssembleDb);
    }

    @RequestMapping(value = "/getSqlField/{id}", method = RequestMethod.GET)
    public Result getSqlField(@PathVariable String id) {
        SysAssembleDbTable sysAssembleDb = dbTableService.selectById(id);
        if (sysAssembleDb == null || !sysAssembleDb.getType().equals(SysAssembleTableType.sql)) {
            return Result.error(CodeMsg.ERROR_PARAMS_NOT_MATHED);
        }
        String sql = sysAssembleDb.getSql();
        List<String> fieldsBySql = dbService.getFieldsBySql(sysAssembleDb.getParentId(), sql);

        return Result.success(fieldsBySql);
    }

    @RequestMapping(value = "/setFilterSql/{id}", method = RequestMethod.GET)
    public Result setFilterSql(@PathVariable String id,@RequestParam String filter, HttpServletRequest request) {
        //TODO 对filter 做检验?

        SysAssembleDbTable sysAssembleDbTable = dbTableService.selectById(id);
        if (sysAssembleDbTable == null) {
            return Result.error(CodeMsg.ERROR_PARAMS_NOT_MATHED);
        }
        sysAssembleDbTable.setFilter(filter).updateById();
        return Result.success(sysAssembleDbTable);
    }

    @RequestMapping(value = "/testFilter/{id}", method = RequestMethod.GET)
    public Result testFilter(@PathVariable String id, @RequestParam String filter) {
        SysAssembleDbTable sysAssembleDbTable = dbTableService.selectById(id);
        if (sysAssembleDbTable == null) {
            return Result.error(CodeMsg.ERROR_PARAMS_NOT_MATHED);
        }
        String parentId = sysAssembleDbTable.getParentId();
        if (StringUtils.isEmpty(parentId)) {
            return Result.error(CodeMsg.ERROR_PARAMS_NOT_MATHED);
        }
        SysAssembleDb sysAssembleDb = dbService.selectById(parentId);
        if (sysAssembleDb == null) {
            return Result.error(CodeMsg.ERROR_PARAMS_NOT_MATHED);
        }
        String assembleId = sysAssembleDb.getParentId();

        List<SysAssembleDbField> fieldList = dbFieldService.selectList(new EntityWrapper<SysAssembleDbField>().eq(Constant.PARENT_ID, id));
        String fields = fieldList.stream().map(SysAssembleDbField::getField).collect(Collectors.joining(Constant.COMMA));
        SysAssembleTableType type = sysAssembleDbTable.getType();
        String tableName = null;
        if (type.equals(SysAssembleTableType.sql)) {
            tableName = sysAssembleDbTable.getSql();
        } else if (type.equals(SysAssembleTableType.table)) {
            tableName = sysAssembleDbTable.getTableName();
        }

        Set<String> matcher = DbUtils.matcher(filter);
        for (String code : matcher) {
            SysAssembleParams sysAssembleParams = paramsService.selectOne(new EntityWrapper<SysAssembleParams>().eq(Constant.PARENT_ID, assembleId).eq(Constant.Code, code));
            if (sysAssembleParams == null){
                return Result.error(new CodeMsg(6009, assembleId + "有变量未匹配到:"+ code));
            }
            String val = sysAssembleParams.getVal();
            if (StringUtils.isEmpty(val)) {
                return Result.error(new CodeMsg(6009, assembleId + "有变量未获取到值:"+ code));
            }
            val = DbUtils.quotedStr(val);
            filter = filter.replace(DbUtils.assemblParam(code), val);
        }
        String runSqlTemplate = null;
        if (type.equals(SysAssembleTableType.table)){
            runSqlTemplate = Constant.selectFieldTableTemplate;
        }else if(type.equals(SysAssembleTableType.sql)){
            runSqlTemplate = Constant.selectFieldSqlTemplate;
        }
        String sql = MessageFormat.format(runSqlTemplate, fields, tableName, filter);
        boolean b = dbService.testSql(parentId, sql);
        return Result.success(b);
    }

    @ResponseBody
    @RequestMapping(value = "/addOrUpdate", method = RequestMethod.POST)
    public Result addOne(@RequestBody SysAssembleDbTable sysAssembleDb) {
        if (sysAssembleDb == null) {
            return Result.error(CodeMsg.ERROR_PARAMS_NOT_MATHED);
        }
        if (StringUtils.isEmpty(sysAssembleDb.getId())) {
            sysAssembleDb.setId(DbUtils.getUUID());
            if (StringUtils.isEmpty(sysAssembleDb.getSqlId()) && sysAssembleDb.getType().equals(SysAssembleTableType.sql)) {
                return Result.error(CodeMsg.ERROR_PARAMS_NOT_MATHED);
            }
        }

        if (StringUtils.isEmpty(sysAssembleDb.getTableName()) && sysAssembleDb.getType().equals(SysAssembleTableType.table)){
            return Result.error(CodeMsg.ERROR_PARAMS_NOT_MATHED);
        }

        SysAssemble assemble = assembleService.selectById(sysAssembleDb.getAssembleId());
        if (assemble == null) {
            return Result.error(CodeMsg.ERROR_PARAMS_NOT_MATHED);
        }
        Boolean bigdata = assemble.getBigdata();
        DataSourceInfo dataSourceInfo = null;
        if (bigdata) {
            dataSourceInfo =  bigDataDataSourceInfo;
        }else {
            dataSourceInfo = unBigDataDataSourceInfo;
        }
        List<String> fieldStrList = sysAssembleDb.getFieldList().stream().map(SysAssembleDbField::getField).collect(Collectors.toList());
        String suffix = null;
        try {
            suffix = DbUtils.Bit16(assemble.getId());
        } catch (Exception e) {
            e.printStackTrace();
            suffix = DbUtils.getUUID(16);
        }
        String tempTableName = null;
        if (sysAssembleDb.getType().equals(SysAssembleTableType.table)) {
            tempTableName = (Constant.AssembleTempTable  +  sysAssembleDb.getTableName() + Constant.UnderLine + suffix);
        }else {
            tempTableName = (Constant.AssembleTempSql +  DbUtils.getUUID(5) + Constant.UnderLine + suffix);
            String redisVal = redisClient.getRedisVal(sysAssembleDb.getSqlId());
            if (StringUtils.isEmpty(redisVal)) {
                return Result.error(new CodeMsg(6008,"获取sql失败"));
            }
            sysAssembleDb.setSql(redisVal);
            redisClient.deleteRedisVal(sysAssembleDb.getSqlId());
        }

        sysAssembleDb.setTempTableName(tempTableName);

        boolean created = dataSourceInfo.createTable(tempTableName, fieldStrList);
        if (!created) {
            return Result.error(new CodeMsg(6008,"创建表失败:" + tempTableName));
        }
        boolean update = sysAssembleDb.insertOrUpdate();
        if (update) {
            // update field
            boolean delete = dbFieldService.delete(new EntityWrapper<SysAssembleDbField>().eq(Constant.PARENT_ID, sysAssembleDb.getId()));
            if (!delete) {
                return Result.error(CodeMsg.DELETE_ERROR);
            }
            List<SysAssembleDbField> fieldList = sysAssembleDb.getFieldList();
            for (SysAssembleDbField sysAssembleDbField : fieldList) {
                sysAssembleDbField.setId(DbUtils.getUUID()).setCreateTime(new Date()).setParentId(sysAssembleDb.getId()).insert();
            }
            return Result.success(sysAssembleDb);
        }else {
            return Result.error(CodeMsg.UPDATE_ERROR);
        }
    }
}

 src/main/java/com/highdatas/mdm/controller/SysAssembleDetailController.java

New file
@@ -0,0 +1,21 @@
package com.highdatas.mdm.controller;


import org.springframework.web.bind.annotation.RequestMapping;

import org.springframework.stereotype.Controller;
import org.springframework.web.bind.annotation.RestController;

/**
 * <p>
 *  前端控制器
 * </p>
 *
 * @author kimi
 * @since 2020-02-15
 */
@RestController
@RequestMapping("/sysAssembleDetail")
public class SysAssembleDetailController {

}

 src/main/java/com/highdatas/mdm/controller/SysAssembleLogController.java

New file
@@ -0,0 +1,21 @@
package com.highdatas.mdm.controller;


import org.springframework.web.bind.annotation.RequestMapping;

import org.springframework.stereotype.Controller;
import org.springframework.web.bind.annotation.RestController;

/**
 * <p>
 *  前端控制器
 * </p>
 *
 * @author kimi
 * @since 2020-02-15
 */
@RestController
@RequestMapping("/sysAssembleLog")
public class SysAssembleLogController {

}

 src/main/java/com/highdatas/mdm/controller/SysAssembleParamsController.java

New file
@@ -0,0 +1,86 @@
package com.highdatas.mdm.controller;


import com.baomidou.mybatisplus.mapper.EntityWrapper;
import com.baomidou.mybatisplus.mapper.Wrapper;
import com.highdatas.mdm.entity.SysAssemble;
import com.highdatas.mdm.entity.SysAssembleDb;
import com.highdatas.mdm.entity.SysAssembleParams;
import com.highdatas.mdm.pojo.CodeMsg;
import com.highdatas.mdm.pojo.Result;
import com.highdatas.mdm.service.ISysAssembleParamsService;
import com.highdatas.mdm.service.ISysAssembleService;
import com.highdatas.mdm.util.Constant;
import com.highdatas.mdm.util.DbUtils;
import org.apache.commons.lang3.StringUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Controller;
import org.springframework.web.bind.annotation.*;

import javax.servlet.http.HttpServletRequest;
import java.util.Date;
import java.util.List;

/**
 * <p>
 *  前端控制器
 * </p>
 *
 * @author kimi
 * @since 2020-02-20
 */
@RestController
@RequestMapping("/assemble/param")
public class SysAssembleParamsController {
    @Autowired
    ISysAssembleService assembleService;
    @Autowired
    ISysAssembleParamsService paramsService;

    @RequestMapping(value = "/addOrUpdate/{id}", method = RequestMethod.POST)
    public Result get(@PathVariable String id, @RequestBody SysAssembleParams params) {
        SysAssemble assemble = assembleService.selectById(id);
        if (assemble == null) {
            return Result.error(CodeMsg.ERROR_PARAMS_NOT_MATHED);
        }
        params.setParentId(id);
        if (StringUtils.isEmpty(params.getId())) {
            params.setCreateTime(new Date()).setId(DbUtils.getUUID());
        }else {
            params.setUpdateTime(new Date());
        }
        boolean update = params.insertOrUpdate();

      if (update) {
          return Result.success(params);
      }else {
          return Result.error(CodeMsg.INSERT_ERROR);
      }

    }

    @RequestMapping(value = "/list/{id}", method = RequestMethod.GET)
    public Result list(@PathVariable String id) {
        SysAssemble assemble = assembleService.selectById(id);
        if (assemble == null) {
            return Result.error(CodeMsg.ERROR_PARAMS_NOT_MATHED);
        }
        List<SysAssembleParams> paramsList = paramsService.selectList(new EntityWrapper<SysAssembleParams>().eq(Constant.PARENT_ID, id));
        return Result.success(paramsList);
    }

    @RequestMapping(value = "/delete/{id}", method = RequestMethod.GET)
    public Result delete(@PathVariable String id) {
        SysAssembleParams sysAssembleParams = paramsService.selectById(id);
        if (sysAssembleParams == null){
            return Result.error(CodeMsg.ERROR_PARAMS_NOT_MATHED);
        }
        boolean delete = sysAssembleParams.deleteById();
        if (delete) {
            return Result.success(CodeMsg.DELETE_SUCCESS);
        }else {
            return Result.error(CodeMsg.DELETE_ERROR);
        }

    }
}

 src/main/java/com/highdatas/mdm/controller/SysAssembleRuleController.java

New file
@@ -0,0 +1,55 @@
package com.highdatas.mdm.controller;


import com.highdatas.mdm.entity.SysAssembleDbTable;
import com.highdatas.mdm.entity.SysAssembleRule;
import com.highdatas.mdm.pojo.CodeMsg;
import com.highdatas.mdm.pojo.Result;
import com.highdatas.mdm.service.ISysAssembleDbService;
import com.highdatas.mdm.service.ISysAssembleRuleService;
import com.highdatas.mdm.util.DbUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Controller;
import org.springframework.web.bind.annotation.*;

import javax.servlet.http.HttpServletRequest;
import java.util.Date;

/**
 * <p>
 *  前端控制器
 * </p>
 *
 * @author kimi
 * @since 2020-02-21
 */
@RestController
@RequestMapping("/assemble/rule")
public class SysAssembleRuleController {
    @Autowired
    ISysAssembleRuleService ruleService;

    @RequestMapping(value = "/add/{id}", method = RequestMethod.GET)
    public Result add(@PathVariable String id, @RequestParam String ruleId) {
        SysAssembleRule sysAssembleRule = new SysAssembleRule();
        boolean insert = sysAssembleRule.setCreateTime(new Date()).setId(DbUtils.getUUID()).setParentId(id).setRuleId(ruleId).insert();
        if (insert) {
           return Result.success(sysAssembleRule);
        }else {
            return Result.error(CodeMsg.INSERT_ERROR);
        }
    }
    @RequestMapping(value = "/update/{id}", method = RequestMethod.GET)
    public Result update(@PathVariable String id, @RequestParam String ruleId) {
        SysAssembleRule sysAssembleRule = ruleService.selectById(id);
        if (sysAssembleRule == null) {
            return Result.error(CodeMsg.ERROR_PARAMS_NOT_MATHED);
        }
        boolean insert = sysAssembleRule.setUpdateTime(new Date()).setRuleId(ruleId).updateById();
        if (insert) {
            return Result.success(sysAssembleRule);
        }else {
            return Result.error(CodeMsg.UPDATE_ERROR);
        }
    }
}

 src/main/java/com/highdatas/mdm/controller/SysDbtypeController.java

New file
@@ -0,0 +1,48 @@
package com.highdatas.mdm.controller;


import com.baomidou.mybatisplus.mapper.EntityWrapper;
import com.fasterxml.jackson.databind.DeserializationFeature;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.highdatas.mdm.entity.SysDbtype;
import com.highdatas.mdm.entity.SysField;
import com.highdatas.mdm.pojo.CodeMsg;
import com.highdatas.mdm.pojo.Result;
import com.highdatas.mdm.service.IFlowsService;
import com.highdatas.mdm.service.ISysDbtypeService;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.web.bind.annotation.RequestMapping;

import org.springframework.stereotype.Controller;
import org.springframework.web.bind.annotation.RequestMethod;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.RestController;

import java.util.Date;
import java.util.List;

/**
 * <p>
 *  前端控制器
 * </p>
 *
 * @author kimi
 * @since 2020-02-15
 */
@RestController
@RequestMapping("/sysDbtype")
public class SysDbtypeController {
    @Autowired
    ISysDbtypeService dbtypeService;


    @RequestMapping(value = "/all", method = RequestMethod.GET)
    public Result all()  {
        List<SysDbtype> sysDbtypes = dbtypeService.selectList(new EntityWrapper<SysDbtype>().eq("active", true));
        if (sysDbtypes != null) {
            return Result.success(sysDbtypes);
        }else {
            return Result.error(CodeMsg.SELECT_ERROR);
        }
    }
}

 src/main/java/com/highdatas/mdm/controller/SysFieldController.java

@@ -4,14 +4,19 @@
import com.baomidou.mybatisplus.mapper.EntityWrapper;
import com.fasterxml.jackson.databind.DeserializationFeature;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.highdatas.mdm.entity.MenuMapping;
import com.highdatas.mdm.entity.SysField;
import com.highdatas.mdm.entity.SysMenu;
import com.highdatas.mdm.pojo.CodeMsg;
import com.highdatas.mdm.pojo.Result;
import com.highdatas.mdm.service.IMenuMappingService;
import com.highdatas.mdm.service.ISysFieldService;
import com.highdatas.mdm.service.ISysMenuService;
import org.apache.commons.lang3.StringUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.web.bind.annotation.*;

import java.awt.*;
import java.util.Date;
import java.util.List;

@@ -28,6 +33,11 @@
public class SysFieldController {
    @Autowired
    ISysFieldService fieldService;
    @Autowired
    ISysMenuService menuService;
    @Autowired
    IMenuMappingService menuMappingService;


    @RequestMapping(value = "/{tableName}", method = RequestMethod.GET)
    public Result todoTask(@PathVariable String tableName){
@@ -38,6 +48,20 @@

        return Result.success(fieldList);
    }
    @RequestMapping(value = "/menu/{id}", method = RequestMethod.GET)
    public Result getFieldByMenu(@PathVariable String id){
        MenuMapping menuMapping = menuMappingService.selectOne(new EntityWrapper<MenuMapping>().eq("menu_id", id));
        if (menuMapping == null) {
            return Result.error(CodeMsg.ERROR_PARAMS_NOT_MATHED);
        }
        String tableName = menuMapping.getTableName();
        if (StringUtils.isEmpty(tableName)) {
            return Result.error(CodeMsg.ERROR_PARAMS_NOT_MATHED);
        }
        List<SysField> fieldList = fieldService.selectList(new EntityWrapper<SysField>().eq("table_name", tableName).orderBy("order_no"));

        return Result.success(fieldList);
    }

    @RequestMapping(value = "/update", method = RequestMethod.GET)
    public Result update(@RequestParam String json)  {

 src/main/java/com/highdatas/mdm/controller/SysMenuController.java

@@ -19,9 +19,7 @@
import org.springframework.web.bind.annotation.*;

import javax.servlet.http.HttpServletRequest;
import java.util.Date;
import java.util.HashSet;
import java.util.List;
import java.util.*;

/**
 * <p>
@@ -51,6 +49,25 @@
        return Result.success(menuService.selectList(sysMenuEntityWrapper)) ;
    }

    @RequestMapping(value = "/list", method = RequestMethod.GET)
    public Result<List<SysMenu>> getMenuList() {
        EntityWrapper<SysMenu> sysMenuEntityWrapper = new EntityWrapper<>();
        sysMenuEntityWrapper.eq("data_type","currentData");
        sysMenuEntityWrapper.orderBy(" parent_id, order_no");
        List<SysMenu> sysMenus = menuService.selectList(sysMenuEntityWrapper);
        LinkedHashSet<String> parentIdSet = new LinkedHashSet<>();
        for (SysMenu sysMenu : sysMenus) {
            String parentId = sysMenu.getParentId();
            parentIdSet.add(parentId);
        }
        Set<String> byParentId = menuService.getByParentId(parentIdSet);
        List<SysMenu> parentList = menuService.selectBatchIds(byParentId);

        return Result.success(parentList) ;
    }



    @RequestMapping(value = "/allTheme", method = RequestMethod.GET)
    public Result<List<SysMenu>> getAllTheme() {
        EntityWrapper<SysMenu> sysMenuEntityWrapper = new EntityWrapper<>();
@@ -63,7 +80,7 @@
    public Result<Page<SysMenu>> getAll(@PathVariable int pageno, HttpServletRequest request) {
        String pageSize = request.getParameter("pageSize");
        if (StringUtils.isEmpty(pageSize)) {
            pageSize = "20";
            pageSize = "15";
        }
        Integer size = Integer.valueOf(pageSize);
        Page page = new Page(pageno, size);

 src/main/java/com/highdatas/mdm/entity/MenuMapping.java

@@ -24,6 +24,8 @@

    private String id;

    private Boolean audit;

    /**
     * 资产编号
     */

 src/main/java/com/highdatas/mdm/entity/SysAssemble.java

New file
@@ -0,0 +1,271 @@
package com.highdatas.mdm.entity;

import java.util.Date;
import com.baomidou.mybatisplus.annotations.TableField;
import com.baomidou.mybatisplus.activerecord.Model;
import com.baomidou.mybatisplus.annotations.TableName;
import com.highdatas.mdm.pojo.SysAssembleCheckType;
import com.highdatas.mdm.pojo.SysAssembleRunStatus;
import com.highdatas.mdm.pojo.SysAssembleStatus;
import com.highdatas.mdm.pojo.SysAssembleUpdateType;

import java.io.Serializable;

/**
 * <p>
 * 
 * </p>
 *
 * @author kimi
 * @since 2020-02-20
 */
@TableName("sys_assemble")
public class SysAssemble extends Model<SysAssemble> {

    private static final long serialVersionUID = 1L;

    private String id;

    private String name;
    @TableField("menu_id")
    private String menuId;

    @TableField("pre_time")
    private Date preTime;

    @TableField("pre_msg")
    private String preMsg;

    @TableField("pre_cnt")
    private Integer preCnt;

    @TableField("pre_status")
    private SysAssembleRunStatus preStatus;

    @TableField("user_id")
    private String userId;

    @TableField("purge_sql")
    private String purgeSql;

    @TableField("flow_id")
    private String flowId;

    @TableField("job_id")
    private String jobId;

    @TableField("update_type")
    private SysAssembleUpdateType updateType;

    @TableField("check_type")
    private SysAssembleCheckType checkType;
    @TableField("check_fields")
    private String checkFields;

     @TableField("update_fields")
    private String updateFields;

    private String cron;

    private Boolean bigdata;

    private SysAssembleStatus status;

    @TableField("create_time")
    private Date createTime;

    @TableField("update_time")
    private Date updateTime;

    public String getPreMsg() {
        return preMsg;
    }

    public SysAssemble setPreMsg(String preMsg) {
        this.preMsg = preMsg;
        return this;
    }

    public String getFlowId() {
        return flowId;
    }

    public SysAssemble setFlowId(String flowId) {
        this.flowId = flowId;
        return this;
    }

    public Date getPreTime() {
        return preTime;
    }

    public SysAssemble setPreTime(Date preTime) {
        this.preTime = preTime;
        return this;
    }

    public Integer getPreCnt() {
        return preCnt;
    }

    public SysAssemble setPreCnt(Integer preCnt) {
        this.preCnt = preCnt;
        return this;
    }

    public SysAssembleRunStatus getPreStatus() {
        return preStatus;
    }

    public SysAssemble setPreStatus(SysAssembleRunStatus preStatus) {
        this.preStatus = preStatus;
        return this;
    }

    public String getCheckFields() {
        return checkFields;
    }

    public SysAssemble setCheckFields(String checkFields) {
        this.checkFields = checkFields;
        return this;
    }

    public SysAssembleCheckType getCheckType() {
        return checkType;
    }

    public SysAssemble setCheckType(SysAssembleCheckType checkType) {
        this.checkType = checkType;
        return this;
    }

    public String getUpdateFields() {
        return updateFields;
    }

    public SysAssemble setUpdateFields(String updateFields) {
        this.updateFields = updateFields;
        return this;
    }

    public String getPurgeSql() {
        return purgeSql;
    }

    public SysAssemble setPurgeSql(String purgeSql) {
        this.purgeSql = purgeSql;
        return this;
    }

    public String getName() {
        return name;
    }

    public SysAssemble setName(String name) {
        this.name = name;
        return this;
    }

    public SysAssembleUpdateType getUpdateType() {
        return updateType;
    }

    public SysAssemble setUpdateType(SysAssembleUpdateType updateType) {
        this.updateType = updateType;
        return this;
    }

    public String getId() {
        return id;
    }

    public SysAssemble setId(String id) {
        this.id = id;
        return this;
    }
    public String getMenuId() {
        return menuId;
    }

    public SysAssemble setMenuId(String menuId) {
        this.menuId = menuId;
        return this;
    }
    public String getUserId() {
        return userId;
    }

    public SysAssemble setUserId(String userId) {
        this.userId = userId;
        return this;
    }
    public String getJobId() {
        return jobId;
    }

    public SysAssemble setJobId(String jobId) {
        this.jobId = jobId;
        return this;
    }
    public String getCron() {
        return cron;
    }

    public SysAssemble setCron(String cron) {
        this.cron = cron;
        return this;
    }
    public Boolean getBigdata() {
        return bigdata;
    }

    public SysAssemble setBigdata(Boolean bigdata) {
        this.bigdata = bigdata;
        return this;
    }
    public SysAssembleStatus getStatus() {
        return status;
    }

    public SysAssemble setStatus(SysAssembleStatus status) {
        this.status = status;
        return this;
    }
    public Date getCreateTime() {
        return createTime;
    }

    public SysAssemble setCreateTime(Date createTime) {
        this.createTime = createTime;
        return this;
    }
    public Date getUpdateTime() {
        return updateTime;
    }

    public SysAssemble setUpdateTime(Date updateTime) {
        this.updateTime = updateTime;
        return this;
    }

    @Override
    protected Serializable pkVal() {
        return this.id;
    }

    @Override
    public String toString() {
        return "SysAssemble{" +
        "id=" + id +
        ", menuId=" + menuId +
        ", userId=" + userId +
        ", jobId=" + jobId +
        ", cron=" + cron +
        ", bigdata=" + bigdata +
        ", status=" + status +
        ", createTime=" + createTime +
        ", updateTime=" + updateTime +
        "}";
    }
}

 src/main/java/com/highdatas/mdm/entity/SysAssembleApi.java

New file
@@ -0,0 +1,121 @@
package com.highdatas.mdm.entity;

import com.baomidou.mybatisplus.annotations.TableField;
import com.baomidou.mybatisplus.activerecord.Model;
import com.baomidou.mybatisplus.annotations.TableName;
import java.io.Serializable;

/**
 * <p>
 * 
 * </p>
 *
 * @author kimi
 * @since 2020-02-19
 */
@TableName("sys_assemble_api")
public class SysAssembleApi extends Model<SysAssembleApi> {

    private static final long serialVersionUID = 1L;

    private String id;

    @TableField("parent_id")
    private String parentId;

    private String url;

    private String headers;

    private Boolean pagination;

    private Integer size;

    private String pageurl;

    private String resultfield;

    public String getId() {
        return id;
    }

    public SysAssembleApi setId(String id) {
        this.id = id;
        return this;
    }
    public String getParentId() {
        return parentId;
    }

    public SysAssembleApi setParentId(String parentId) {
        this.parentId = parentId;
        return this;
    }
    public String getUrl() {
        return url;
    }

    public SysAssembleApi setUrl(String url) {
        this.url = url;
        return this;
    }
    public String getHeaders() {
        return headers;
    }

    public SysAssembleApi setHeaders(String headers) {
        this.headers = headers;
        return this;
    }
    public Boolean getPagination() {
        return pagination;
    }

    public SysAssembleApi setPagination(Boolean pagination) {
        this.pagination = pagination;
        return this;
    }
    public Integer getSize() {
        return size;
    }

    public SysAssembleApi setSize(Integer size) {
        this.size = size;
        return this;
    }
    public String getPageurl() {
        return pageurl;
    }

    public SysAssembleApi setPageurl(String pageurl) {
        this.pageurl = pageurl;
        return this;
    }
    public String getResultfield() {
        return resultfield;
    }

    public SysAssembleApi setResultfield(String resultfield) {
        this.resultfield = resultfield;
        return this;
    }

    @Override
    protected Serializable pkVal() {
        return this.id;
    }

    @Override
    public String toString() {
        return "SysAssembleApi{" +
        "id=" + id +
        ", parentId=" + parentId +
        ", url=" + url +
        ", headers=" + headers +
        ", pagination=" + pagination +
        ", size=" + size +
        ", pageurl=" + pageurl +
        ", resultfield=" + resultfield +
        "}";
    }
}

 src/main/java/com/highdatas/mdm/entity/SysAssembleCommonparams.java

New file
@@ -0,0 +1,125 @@
package com.highdatas.mdm.entity;

import java.util.Date;
import com.baomidou.mybatisplus.annotations.TableField;
import com.baomidou.mybatisplus.activerecord.Model;
import com.baomidou.mybatisplus.annotations.TableName;
import java.io.Serializable;

/**
 * <p>
 * 
 * </p>
 *
 * @author kimi
 * @since 2020-02-20
 */
@TableName("sys_assemble_commonparams")
public class SysAssembleCommonparams extends Model<SysAssembleCommonparams> {

    private static final long serialVersionUID = 1L;

    private String id;

    private String code;

    private String name;

    private String val;

    @TableField("init_sql")
    private String initSql;

    @TableField("update_sql")
    private String updateSql;

    @TableField("create_time")
    private Date createTime;

    @TableField("update_time")
    private Date updateTime;

    public String getId() {
        return id;
    }

    public SysAssembleCommonparams setId(String id) {
        this.id = id;
        return this;
    }
    public String getCode() {
        return code;
    }

    public SysAssembleCommonparams setCode(String code) {
        this.code = code;
        return this;
    }
    public String getName() {
        return name;
    }

    public SysAssembleCommonparams setName(String name) {
        this.name = name;
        return this;
    }
    public String getVal() {
        return val;
    }

    public SysAssembleCommonparams setVal(String val) {
        this.val = val;
        return this;
    }
    public String getInitSql() {
        return initSql;
    }

    public SysAssembleCommonparams setInitSql(String initSql) {
        this.initSql = initSql;
        return this;
    }
    public String getUpdateSql() {
        return updateSql;
    }

    public SysAssembleCommonparams setUpdateSql(String updateSql) {
        this.updateSql = updateSql;
        return this;
    }
    public Date getCreateTime() {
        return createTime;
    }

    public SysAssembleCommonparams setCreateTime(Date createTime) {
        this.createTime = createTime;
        return this;
    }
    public Date getUpdateTime() {
        return updateTime;
    }

    public SysAssembleCommonparams setUpdateTime(Date updateTime) {
        this.updateTime = updateTime;
        return this;
    }

    @Override
    protected Serializable pkVal() {
        return this.id;
    }

    @Override
    public String toString() {
        return "SysAssembleCommonparams{" +
        "id=" + id +
        ", code=" + code +
        ", name=" + name +
        ", val=" + val +
        ", initSql=" + initSql +
        ", updateSql=" + updateSql +
        ", createTime=" + createTime +
        ", updateTime=" + updateTime +
        "}";
    }
}

 src/main/java/com/highdatas/mdm/entity/SysAssembleDb.java

New file
@@ -0,0 +1,186 @@
package com.highdatas.mdm.entity;

import java.util.Date;
import com.baomidou.mybatisplus.annotations.TableField;
import com.baomidou.mybatisplus.activerecord.Model;
import com.baomidou.mybatisplus.annotations.TableName;
import java.io.Serializable;

/**
 * <p>
 * 
 * </p>
 *
 * @author kimi
 * @since 2020-02-20
 */
@TableName("sys_assemble_db")
public class SysAssembleDb extends Model<SysAssembleDb> {

    private static final long serialVersionUID = 1L;

    private String id;

    @TableField("parent_id")
    private String parentId;

    @TableField("datasource_type")
    private String datasourceType;

    @TableField("datasource_name")
    private String datasourceName;

    @TableField("ipaddress_port")
    private String ipaddressPort;

    @TableField("database_name")
    private String databaseName;

    @TableField("datasource_url")
    private String datasourceUrl;

    @TableField("datasource_user")
    private String datasourceUser;

    @TableField("datasource_pass")
    private String datasourcePass;

    private Integer orderby;

    private Boolean active;

    @TableField("create_time")
    private Date createTime;

    @TableField("update_time")
    private Date updateTime;

    public String getId() {
        return id;
    }

    public SysAssembleDb setId(String id) {
        this.id = id;
        return this;
    }
    public String getParentId() {
        return parentId;
    }

    public SysAssembleDb setParentId(String parentId) {
        this.parentId = parentId;
        return this;
    }
    public String getDatasourceType() {
        return datasourceType;
    }

    public SysAssembleDb setDatasourceType(String datasourceType) {
        this.datasourceType = datasourceType;
        return this;
    }
    public String getDatasourceName() {
        return datasourceName;
    }

    public SysAssembleDb setDatasourceName(String datasourceName) {
        this.datasourceName = datasourceName;
        return this;
    }
    public String getIpaddressPort() {
        return ipaddressPort;
    }

    public SysAssembleDb setIpaddressPort(String ipaddressPort) {
        this.ipaddressPort = ipaddressPort;
        return this;
    }
    public String getDatabaseName() {
        return databaseName;
    }

    public SysAssembleDb setDatabaseName(String databaseName) {
        this.databaseName = databaseName;
        return this;
    }
    public String getDatasourceUrl() {
        return datasourceUrl;
    }

    public SysAssembleDb setDatasourceUrl(String datasourceUrl) {
        this.datasourceUrl = datasourceUrl;
        return this;
    }
    public String getDatasourceUser() {
        return datasourceUser;
    }

    public SysAssembleDb setDatasourceUser(String datasourceUser) {
        this.datasourceUser = datasourceUser;
        return this;
    }
    public String getDatasourcePass() {
        return datasourcePass;
    }

    public SysAssembleDb setDatasourcePass(String datasourcePass) {
        this.datasourcePass = datasourcePass;
        return this;
    }
    public Integer getOrderby() {
        return orderby;
    }

    public SysAssembleDb setOrderby(Integer orderby) {
        this.orderby = orderby;
        return this;
    }
    public Boolean getActive() {
        return active;
    }

    public SysAssembleDb setActive(Boolean active) {
        this.active = active;
        return this;
    }
    public Date getCreateTime() {
        return createTime;
    }

    public SysAssembleDb setCreateTime(Date createTime) {
        this.createTime = createTime;
        return this;
    }
    public Date getUpdateTime() {
        return updateTime;
    }

    public SysAssembleDb setUpdateTime(Date updateTime) {
        this.updateTime = updateTime;
        return this;
    }

    @Override
    protected Serializable pkVal() {
        return this.id;
    }

    @Override
    public String toString() {
        return "SysAssembleDb{" +
        "id=" + id +
        ", parentId=" + parentId +
        ", datasourceType=" + datasourceType +
        ", datasourceName=" + datasourceName +
        ", ipaddressPort=" + ipaddressPort +
        ", databaseName=" + databaseName +
        ", datasourceUrl=" + datasourceUrl +
        ", datasourceUser=" + datasourceUser +
        ", datasourcePass=" + datasourcePass +
        ", orderby=" + orderby +
        ", active=" + active +
        ", createTime=" + createTime +
        ", updateTime=" + updateTime +
        "}";
    }
}

 src/main/java/com/highdatas/mdm/entity/SysAssembleDbDetail.java

New file
@@ -0,0 +1,136 @@
package com.highdatas.mdm.entity;

import java.util.Date;
import com.baomidou.mybatisplus.annotations.TableField;
import com.baomidou.mybatisplus.activerecord.Model;
import com.baomidou.mybatisplus.annotations.TableName;
import java.io.Serializable;

/**
 * <p>
 * 
 * </p>
 *
 * @author kimi
 * @since 2020-02-19
 */
@TableName("sys_assemble_db_detail")
public class SysAssembleDbDetail extends Model<SysAssembleDbDetail> {

    private static final long serialVersionUID = 1L;

    private String id;

    @TableField("assemble_db_id")
    private String assembleDbId;

    private String name;

    private String field;

    private Boolean extra;

    private String val;

    @TableField("update_sql")
    private String updateSql;

    @TableField("create_time")
    private Date createTime;

    @TableField("update_time")
    private Date updateTime;

    public String getId() {
        return id;
    }

    public SysAssembleDbDetail setId(String id) {
        this.id = id;
        return this;
    }
    public String getAssembleDbId() {
        return assembleDbId;
    }

    public SysAssembleDbDetail setAssembleDbId(String assembleDbId) {
        this.assembleDbId = assembleDbId;
        return this;
    }
    public String getName() {
        return name;
    }

    public SysAssembleDbDetail setName(String name) {
        this.name = name;
        return this;
    }
    public String getField() {
        return field;
    }

    public SysAssembleDbDetail setField(String field) {
        this.field = field;
        return this;
    }
    public Boolean getExtra() {
        return extra;
    }

    public SysAssembleDbDetail setExtra(Boolean extra) {
        this.extra = extra;
        return this;
    }
    public String getVal() {
        return val;
    }

    public SysAssembleDbDetail setVal(String val) {
        this.val = val;
        return this;
    }
    public String getUpdateSql() {
        return updateSql;
    }

    public SysAssembleDbDetail setUpdateSql(String updateSql) {
        this.updateSql = updateSql;
        return this;
    }
    public Date getCreateTime() {
        return createTime;
    }

    public SysAssembleDbDetail setCreateTime(Date createTime) {
        this.createTime = createTime;
        return this;
    }
    public Date getUpdateTime() {
        return updateTime;
    }

    public SysAssembleDbDetail setUpdateTime(Date updateTime) {
        this.updateTime = updateTime;
        return this;
    }

    @Override
    protected Serializable pkVal() {
        return this.id;
    }

    @Override
    public String toString() {
        return "SysAssembleDbDetail{" +
        "id=" + id +
        ", assembleDbId=" + assembleDbId +
        ", name=" + name +
        ", field=" + field +
        ", extra=" + extra +
        ", val=" + val +
        ", updateSql=" + updateSql +
        ", createTime=" + createTime +
        ", updateTime=" + updateTime +
        "}";
    }
}

 src/main/java/com/highdatas/mdm/entity/SysAssembleDbField.java

New file
@@ -0,0 +1,102 @@
package com.highdatas.mdm.entity;

import java.util.Date;
import com.baomidou.mybatisplus.annotations.TableField;
import com.baomidou.mybatisplus.activerecord.Model;
import com.baomidou.mybatisplus.annotations.TableName;
import java.io.Serializable;

/**
 * <p>
 * 
 * </p>
 *
 * @author kimi
 * @since 2020-02-20
 */
@TableName("sys_assemble_db_field")
public class SysAssembleDbField extends Model<SysAssembleDbField> {

    private static final long serialVersionUID = 1L;

    private String id;

    @TableField("parent_id")
    private String parentId;

    private String name;

    private String field;

    @TableField("create_time")
    private Date createTime;

    @TableField("update_time")
    private Date updateTime;

    public String getId() {
        return id;
    }

    public SysAssembleDbField setId(String id) {
        this.id = id;
        return this;
    }
    public String getParentId() {
        return parentId;
    }

    public SysAssembleDbField setParentId(String parentId) {
        this.parentId = parentId;
        return this;
    }
    public String getName() {
        return name;
    }

    public SysAssembleDbField setName(String name) {
        this.name = name;
        return this;
    }
    public String getField() {
        return field;
    }

    public SysAssembleDbField setField(String field) {
        this.field = field;
        return this;
    }
    public Date getCreateTime() {
        return createTime;
    }

    public SysAssembleDbField setCreateTime(Date createTime) {
        this.createTime = createTime;
        return this;
    }
    public Date getUpdateTime() {
        return updateTime;
    }

    public SysAssembleDbField setUpdateTime(Date updateTime) {
        this.updateTime = updateTime;
        return this;
    }

    @Override
    protected Serializable pkVal() {
        return this.id;
    }

    @Override
    public String toString() {
        return "SysAssembleDbField{" +
        "id=" + id +
        ", parentId=" + parentId +
        ", name=" + name +
        ", field=" + field +
        ", createTime=" + createTime +
        ", updateTime=" + updateTime +
        "}";
    }
}

 src/main/java/com/highdatas/mdm/entity/SysAssembleDbTable.java

New file
@@ -0,0 +1,192 @@
package com.highdatas.mdm.entity;

import java.util.Date;
import com.baomidou.mybatisplus.annotations.TableField;
import com.baomidou.mybatisplus.activerecord.Model;
import com.baomidou.mybatisplus.annotations.TableName;
import com.highdatas.mdm.pojo.SysAssembleTableType;

import java.io.Serializable;
import java.util.List;

/**
 * <p>
 * 
 * </p>
 *
 * @author kimi
 * @since 2020-02-20
 */
@TableName("sys_assemble_db_table")
public class SysAssembleDbTable extends Model<SysAssembleDbTable> {

    private static final long serialVersionUID = 1L;

    private String id;

    @TableField("parent_id")
    private String parentId;

    private SysAssembleTableType type;

    @TableField("table_name")
    private String tableName;

    @TableField("temp_table_name")
    private String tempTableName;

    private String sql;

    private String filter;

    private Integer orderby;

    private Boolean active;

    @TableField("create_time")
    private Date createTime;

    @TableField("update_time")
    private Date updateTime;

    private transient List<SysAssembleDbField> fieldList;
    private transient String assembleId;
    private transient String sqlId;

    public String getSqlId() {
        return sqlId;
    }

    public void setSqlId(String sqlId) {
        this.sqlId = sqlId;
    }

    public String getAssembleId() {
        return assembleId;
    }

    public SysAssembleDbTable setAssembleId(String assembleId) {
        this.assembleId = assembleId;
        return this;
    }

    public List<SysAssembleDbField> getFieldList() {
        return fieldList;
    }

    public void setFieldList(List<SysAssembleDbField> fieldList) {
        this.fieldList = fieldList;
    }


    public String getTempTableName() {
        return tempTableName;
    }

    public SysAssembleDbTable setTempTableName(String tempTableName) {
        this.tempTableName = tempTableName;
        return this;
    }

    public String getId() {
        return id;
    }

    public SysAssembleDbTable setId(String id) {
        this.id = id;
        return this;
    }
    public String getParentId() {
        return parentId;
    }

    public SysAssembleDbTable setParentId(String parentId) {
        this.parentId = parentId;
        return this;
    }
    public SysAssembleTableType getType() {
        return type;
    }

    public SysAssembleDbTable setType(SysAssembleTableType type) {
        this.type = type;
        return this;
    }
    public String getTableName() {
        return tableName;
    }

    public SysAssembleDbTable setTableName(String tableName) {
        this.tableName = tableName;
        return this;
    }
    public String getSql() {
        return sql;
    }

    public SysAssembleDbTable setSql(String sql) {
        this.sql = sql;
        return this;
    }
    public String getFilter() {
        return filter;
    }

    public SysAssembleDbTable setFilter(String filter) {
        this.filter = filter;
        return this;
    }
    public Integer getOrderby() {
        return orderby;
    }

    public SysAssembleDbTable setOrderby(Integer orderby) {
        this.orderby = orderby;
        return this;
    }
    public Boolean getActive() {
        return active;
    }

    public SysAssembleDbTable setActive(Boolean active) {
        this.active = active;
        return this;
    }
    public Date getCreateTime() {
        return createTime;
    }

    public SysAssembleDbTable setCreateTime(Date createTime) {
        this.createTime = createTime;
        return this;
    }
    public Date getUpdateTime() {
        return updateTime;
    }

    public SysAssembleDbTable setUpdateTime(Date updateTime) {
        this.updateTime = updateTime;
        return this;
    }

    @Override
    protected Serializable pkVal() {
        return this.id;
    }

    @Override
    public String toString() {
        return "SysAssembleDbTable{" +
        "id=" + id +
        ", parentId=" + parentId +
        ", type=" + type +
        ", tableName=" + tableName +
        ", sql=" + sql +
        ", filter=" + filter +
        ", orderby=" + orderby +
        ", active=" + active +
        ", createTime=" + createTime +
        ", updateTime=" + updateTime +
        "}";
    }
}

 src/main/java/com/highdatas/mdm/entity/SysAssembleDbdriver.java

New file
@@ -0,0 +1,86 @@
package com.highdatas.mdm.entity;

import com.baomidou.mybatisplus.activerecord.Model;
import com.baomidou.mybatisplus.annotations.TableField;
import com.baomidou.mybatisplus.annotations.TableName;
import java.io.Serializable;

/**
 * <p>
 * 
 * </p>
 *
 * @author kimi
 * @since 2020-02-21
 */
@TableName("sys_assemble_dbdriver")
public class SysAssembleDbdriver extends Model<SysAssembleDbdriver> {

    private static final long serialVersionUID = 1L;

    private String id;

    private String type;

    private String driver;
    @TableField("first_template")
    private String firstTemplate;

    private Boolean active;

    public String getFirstTemplate() {
        return firstTemplate;
    }

    public void setFirstTemplate(String firstTemplate) {
        this.firstTemplate = firstTemplate;
    }

    public String getId() {
        return id;
    }

    public SysAssembleDbdriver setId(String id) {
        this.id = id;
        return this;
    }
    public String getType() {
        return type;
    }

    public SysAssembleDbdriver setType(String type) {
        this.type = type;
        return this;
    }
    public String getDriver() {
        return driver;
    }

    public SysAssembleDbdriver setDriver(String driver) {
        this.driver = driver;
        return this;
    }
    public Boolean getActive() {
        return active;
    }

    public SysAssembleDbdriver setActive(Boolean active) {
        this.active = active;
        return this;
    }

    @Override
    protected Serializable pkVal() {
        return this.id;
    }

    @Override
    public String toString() {
        return "SysAssembleDbdriver{" +
        "id=" + id +
        ", type=" + type +
        ", driver=" + driver +
        ", active=" + active +
        "}";
    }
}

 src/main/java/com/highdatas/mdm/entity/SysAssembleDetail.java

New file
@@ -0,0 +1,79 @@
package com.highdatas.mdm.entity;

import java.util.Date;
import com.baomidou.mybatisplus.annotations.TableField;
import com.baomidou.mybatisplus.activerecord.Model;
import com.baomidou.mybatisplus.annotations.TableName;
import java.io.Serializable;

/**
 * <p>
 * 
 * </p>
 *
 * @author kimi
 * @since 2020-02-15
 */
@TableName("sys_assemble_detail")
public class SysAssembleDetail extends Model<SysAssembleDetail> {

    private static final long serialVersionUID = 1L;

    private String id;

    @TableField("assemble_id")
    private String assembleId;

    private String field;

    @TableField("create_time")
    private Date createTime;

    public String getId() {
        return id;
    }

    public SysAssembleDetail setId(String id) {
        this.id = id;
        return this;
    }
    public String getAssembleId() {
        return assembleId;
    }

    public SysAssembleDetail setAssembleId(String assembleId) {
        this.assembleId = assembleId;
        return this;
    }
    public String getField() {
        return field;
    }

    public SysAssembleDetail setField(String field) {
        this.field = field;
        return this;
    }
    public Date getCreateTime() {
        return createTime;
    }

    public SysAssembleDetail setCreateTime(Date createTime) {
        this.createTime = createTime;
        return this;
    }

    @Override
    protected Serializable pkVal() {
        return this.id;
    }

    @Override
    public String toString() {
        return "SysAssembleDetail{" +
        "id=" + id +
        ", assembleId=" + assembleId +
        ", field=" + field +
        ", createTime=" + createTime +
        "}";
    }
}

 src/main/java/com/highdatas/mdm/entity/SysAssembleLog.java

New file
@@ -0,0 +1,115 @@
package com.highdatas.mdm.entity;

import java.util.Date;
import com.baomidou.mybatisplus.annotations.TableField;
import com.baomidou.mybatisplus.activerecord.Model;
import com.baomidou.mybatisplus.annotations.TableName;
import com.highdatas.mdm.pojo.SysAssembleLogStatus;

import java.io.Serializable;

/**
 * <p>
 * 
 * </p>
 *
 * @author kimi
 * @since 2020-02-15
 */
@TableName("sys_assemble_log")
public class SysAssembleLog extends Model<SysAssembleLog> {

    private static final long serialVersionUID = 1L;

    private String id;

    @TableField("assemble_id")
    private String assembleId;

    private Integer cnt;

    private SysAssembleLogStatus status;

    private String message;

    @TableField("create_time")
    private Date createTime;

    @TableField("update_time")
    private Date updateTime;

    public Date getUpdateTime() {
        return updateTime;
    }

    public SysAssembleLog setUpdateTime(Date updateTime) {
        this.updateTime = updateTime;
        return this;
    }

    public String getMessage() {
        return message;
    }

    public SysAssembleLog setMessage(String message) {
        this.message = message;
        return this;
    }

    public String getId() {
        return id;
    }

    public SysAssembleLog setId(String id) {
        this.id = id;
        return this;
    }
    public String getAssembleId() {
        return assembleId;
    }

    public SysAssembleLog setAssembleId(String assembleId) {
        this.assembleId = assembleId;
        return this;
    }
    public Integer getCnt() {
        return cnt;
    }

    public SysAssembleLog setCnt(Integer cnt) {
        this.cnt = cnt;
        return this;
    }
    public SysAssembleLogStatus getStatus() {
        return status;
    }

    public SysAssembleLog setStatus(SysAssembleLogStatus status) {
        this.status = status;
        return this;
    }
    public Date getCreateTime() {
        return createTime;
    }

    public SysAssembleLog setCreateTime(Date createTime) {
        this.createTime = createTime;
        return this;
    }

    @Override
    protected Serializable pkVal() {
        return this.id;
    }

    @Override
    public String toString() {
        return "SysAssembleLog{" +
        "id=" + id +
        ", assembleId=" + assembleId +
        ", cnt=" + cnt +
        ", status=" + status +
        ", createTime=" + createTime +
        "}";
    }
}

 src/main/java/com/highdatas/mdm/entity/SysAssembleParams.java

New file
@@ -0,0 +1,137 @@
package com.highdatas.mdm.entity;

import java.util.Date;
import com.baomidou.mybatisplus.annotations.TableField;
import com.baomidou.mybatisplus.activerecord.Model;
import com.baomidou.mybatisplus.annotations.TableName;
import java.io.Serializable;

/**
 * <p>
 * 
 * </p>
 *
 * @author kimi
 * @since 2020-02-20
 */
@TableName("sys_assemble_params")
public class SysAssembleParams extends Model<SysAssembleParams> {

    private static final long serialVersionUID = 1L;

    private String id;

    @TableField("parent_id")
    private String parentId;

    private String code;

    private String name;

    private String val;

    @TableField("init_sql")
    private String initSql;

    @TableField("update_sql")
    private String updateSql;

    @TableField("create_time")
    private Date createTime;

    @TableField("update_time")
    private Date updateTime;

    public String getId() {
        return id;
    }

    public SysAssembleParams setId(String id) {
        this.id = id;
        return this;
    }
    public String getParentId() {
        return parentId;
    }

    public SysAssembleParams setParentId(String parentId) {
        this.parentId = parentId;
        return this;
    }
    public String getCode() {
        return code;
    }

    public SysAssembleParams setCode(String code) {
        this.code = code;
        return this;
    }
    public String getName() {
        return name;
    }

    public SysAssembleParams setName(String name) {
        this.name = name;
        return this;
    }
    public String getVal() {
        return val;
    }

    public SysAssembleParams setVal(String val) {
        this.val = val;
        return this;
    }
    public String getInitSql() {
        return initSql;
    }

    public SysAssembleParams setInitSql(String initSql) {
        this.initSql = initSql;
        return this;
    }
    public String getUpdateSql() {
        return updateSql;
    }

    public SysAssembleParams setUpdateSql(String updateSql) {
        this.updateSql = updateSql;
        return this;
    }
    public Date getCreateTime() {
        return createTime;
    }

    public SysAssembleParams setCreateTime(Date createTime) {
        this.createTime = createTime;
        return this;
    }
    public Date getUpdateTime() {
        return updateTime;
    }

    public SysAssembleParams setUpdateTime(Date updateTime) {
        this.updateTime = updateTime;
        return this;
    }

    @Override
    protected Serializable pkVal() {
        return this.id;
    }

    @Override
    public String toString() {
        return "SysAssembleParams{" +
        "id=" + id +
        ", parentId=" + parentId +
        ", code=" + code +
        ", name=" + name +
        ", val=" + val +
        ", initSql=" + initSql +
        ", updateSql=" + updateSql +
        ", createTime=" + createTime +
        ", updateTime=" + updateTime +
        "}";
    }
}

 src/main/java/com/highdatas/mdm/entity/SysAssembleRule.java

New file
@@ -0,0 +1,92 @@
package com.highdatas.mdm.entity;

import java.util.Date;
import com.baomidou.mybatisplus.annotations.TableField;
import com.baomidou.mybatisplus.activerecord.Model;
import com.baomidou.mybatisplus.annotations.TableName;
import java.io.Serializable;

/**
 * <p>
 * 
 * </p>
 *
 * @author kimi
 * @since 2020-02-21
 */
@TableName("sys_assemble_rule")
public class SysAssembleRule extends Model<SysAssembleRule> {

    private static final long serialVersionUID = 1L;

    private String id;

    @TableField("parent_id")
    private String parentId;

    @TableField("rule_id")
    private String ruleId;

    @TableField("create_time")
    private Date createTime;

    @TableField("update_time")
    private Date updateTime;

    public String getId() {
        return id;
    }

    public SysAssembleRule setId(String id) {
        this.id = id;
        return this;
    }
    public String getParentId() {
        return parentId;
    }

    public SysAssembleRule setParentId(String parentId) {
        this.parentId = parentId;
        return this;
    }
    public String getRuleId() {
        return ruleId;
    }

    public SysAssembleRule setRuleId(String ruleId) {
        this.ruleId = ruleId;
        return this;
    }
    public Date getCreateTime() {
        return createTime;
    }

    public SysAssembleRule setCreateTime(Date createTime) {
        this.createTime = createTime;
        return this;
    }
    public Date getUpdateTime() {
        return updateTime;
    }

    public SysAssembleRule setUpdateTime(Date updateTime) {
        this.updateTime = updateTime;
        return this;
    }

    @Override
    protected Serializable pkVal() {
        return this.id;
    }

    @Override
    public String toString() {
        return "SysAssembleRule{" +
        "id=" + id +
        ", parentId=" + parentId +
        ", ruleId=" + ruleId +
        ", createTime=" + createTime +
        ", updateTime=" + updateTime +
        "}";
    }
}

 src/main/java/com/highdatas/mdm/entity/SysDbtype.java

New file
@@ -0,0 +1,77 @@
package com.highdatas.mdm.entity;

import com.baomidou.mybatisplus.activerecord.Model;
import com.baomidou.mybatisplus.annotations.TableName;
import com.highdatas.mdm.pojo.DbAccessType;

import java.io.Serializable;

/**
 * <p>
 * 
 * </p>
 *
 * @author kimi
 * @since 2020-02-15
 */
@TableName("sys_dbtype")
public class SysDbtype extends Model<SysDbtype> {

    private static final long serialVersionUID = 1L;

    private String id;

    private DbAccessType type;

    private String name;

    private Boolean active;

    public String getId() {
        return id;
    }

    public SysDbtype setId(String id) {
        this.id = id;
        return this;
    }
    public DbAccessType getType() {
        return type;
    }

    public SysDbtype setType(DbAccessType type) {
        this.type = type;
        return this;
    }
    public String getName() {
        return name;
    }

    public SysDbtype setName(String name) {
        this.name = name;
        return this;
    }
    public Boolean getActive() {
        return active;
    }

    public SysDbtype setActive(Boolean active) {
        this.active = active;
        return this;
    }

    @Override
    protected Serializable pkVal() {
        return this.id;
    }

    @Override
    public String toString() {
        return "SysDbtype{" +
        "id=" + id +
        ", type=" + type +
        ", name=" + name +
        ", active=" + active +
        "}";
    }
}

 src/main/java/com/highdatas/mdm/entity/SysMenu.java

@@ -42,10 +42,10 @@
    private Integer orderNo;

    @TableField("create_time")
    private transient Date createTime;
    private  Date createTime;

    @TableField("update_time")
    private transient Date updateTime;
    private  Date updateTime;

    @TableField("data_type")
    private MaintainDataType dataType;

 src/main/java/com/highdatas/mdm/job/XxlJobHandler.java

New file
@@ -0,0 +1,47 @@
package com.highdatas.mdm.job;



import com.highdatas.mdm.entity.SysAssemble;

import com.highdatas.mdm.service.ISysAssembleService;

import com.xxl.job.core.biz.model.ReturnT;

import com.xxl.job.core.log.XxlJobLogger;

import lombok.extern.slf4j.Slf4j;

import org.apache.catalina.session.StandardSession;

import org.apache.commons.lang3.StringUtils;

import org.springframework.beans.factory.annotation.Autowired;

import org.springframework.stereotype.Component;

import com.xxl.job.core.handler.annotation.XxlJob;

import org.springframework.web.context.request.RequestContextHolder;



import javax.servlet.ServletContext;

import javax.servlet.http.HttpSession;

import javax.servlet.http.HttpSessionContext;

import java.util.Enumeration;



/**

 * @author kimi

 * @description

 * @date 2020-03-02 9:35

 */

@Slf4j

@Component

public class XxlJobHandler {

    @Autowired

    ISysAssembleService assembleService;



    @XxlJob("assembleJob")

    public ReturnT<String> demoJobHandler(String assembleId) throws Exception {

        XxlJobLogger.log("assemble Job start");

        if (StringUtils.isEmpty(assembleId)) {

            XxlJobLogger.log("params: assembleId is null");

        }

        SysAssemble assemble = assembleService.selectById(assembleId);

        if (assemble == null) {

            XxlJobLogger.log("params: assemble is not found");

        }



//        assembleService.run()



        return ReturnT.SUCCESS;

    }



}


 src/main/java/com/highdatas/mdm/mapper/SysAssembleApiMapper.java

New file
@@ -0,0 +1,16 @@
package com.highdatas.mdm.mapper;

import com.highdatas.mdm.entity.SysAssembleApi;
import com.baomidou.mybatisplus.mapper.BaseMapper;

/**
 * <p>
 *  Mapper 接口
 * </p>
 *
 * @author kimi
 * @since 2020-02-19
 */
public interface SysAssembleApiMapper extends BaseMapper<SysAssembleApi> {

}

 src/main/java/com/highdatas/mdm/mapper/SysAssembleCommonparamsMapper.java

New file
@@ -0,0 +1,16 @@
package com.highdatas.mdm.mapper;

import com.highdatas.mdm.entity.SysAssembleCommonparams;
import com.baomidou.mybatisplus.mapper.BaseMapper;

/**
 * <p>
 *  Mapper 接口
 * </p>
 *
 * @author kimi
 * @since 2020-02-20
 */
public interface SysAssembleCommonparamsMapper extends BaseMapper<SysAssembleCommonparams> {

}

 src/main/java/com/highdatas/mdm/mapper/SysAssembleDbDetailMapper.java

New file
@@ -0,0 +1,16 @@
package com.highdatas.mdm.mapper;

import com.highdatas.mdm.entity.SysAssembleDbDetail;
import com.baomidou.mybatisplus.mapper.BaseMapper;

/**
 * <p>
 *  Mapper 接口
 * </p>
 *
 * @author kimi
 * @since 2020-02-19
 */
public interface SysAssembleDbDetailMapper extends BaseMapper<SysAssembleDbDetail> {

}

 src/main/java/com/highdatas/mdm/mapper/SysAssembleDbFieldMapper.java

New file
@@ -0,0 +1,16 @@
package com.highdatas.mdm.mapper;

import com.highdatas.mdm.entity.SysAssembleDbField;
import com.baomidou.mybatisplus.mapper.BaseMapper;

/**
 * <p>
 *  Mapper 接口
 * </p>
 *
 * @author kimi
 * @since 2020-02-20
 */
public interface SysAssembleDbFieldMapper extends BaseMapper<SysAssembleDbField> {

}

 src/main/java/com/highdatas/mdm/mapper/SysAssembleDbMapper.java

New file
@@ -0,0 +1,16 @@
package com.highdatas.mdm.mapper;

import com.highdatas.mdm.entity.SysAssembleDb;
import com.baomidou.mybatisplus.mapper.BaseMapper;

/**
 * <p>
 *  Mapper 接口
 * </p>
 *
 * @author kimi
 * @since 2020-02-20
 */
public interface SysAssembleDbMapper extends BaseMapper<SysAssembleDb> {

}

 src/main/java/com/highdatas/mdm/mapper/SysAssembleDbTableMapper.java

New file
@@ -0,0 +1,16 @@
package com.highdatas.mdm.mapper;

import com.highdatas.mdm.entity.SysAssembleDbTable;
import com.baomidou.mybatisplus.mapper.BaseMapper;

/**
 * <p>
 *  Mapper 接口
 * </p>
 *
 * @author kimi
 * @since 2020-02-20
 */
public interface SysAssembleDbTableMapper extends BaseMapper<SysAssembleDbTable> {

}

 src/main/java/com/highdatas/mdm/mapper/SysAssembleDbdriverMapper.java

New file
@@ -0,0 +1,16 @@
package com.highdatas.mdm.mapper;

import com.highdatas.mdm.entity.SysAssembleDbdriver;
import com.baomidou.mybatisplus.mapper.BaseMapper;

/**
 * <p>
 *  Mapper 接口
 * </p>
 *
 * @author kimi
 * @since 2020-02-21
 */
public interface SysAssembleDbdriverMapper extends BaseMapper<SysAssembleDbdriver> {

}

 src/main/java/com/highdatas/mdm/mapper/SysAssembleDetailMapper.java

New file
@@ -0,0 +1,16 @@
package com.highdatas.mdm.mapper;

import com.highdatas.mdm.entity.SysAssembleDetail;
import com.baomidou.mybatisplus.mapper.BaseMapper;

/**
 * <p>
 *  Mapper 接口
 * </p>
 *
 * @author kimi
 * @since 2020-02-15
 */
public interface SysAssembleDetailMapper extends BaseMapper<SysAssembleDetail> {

}

 src/main/java/com/highdatas/mdm/mapper/SysAssembleLogMapper.java

New file
@@ -0,0 +1,16 @@
package com.highdatas.mdm.mapper;

import com.highdatas.mdm.entity.SysAssembleLog;
import com.baomidou.mybatisplus.mapper.BaseMapper;

/**
 * <p>
 *  Mapper 接口
 * </p>
 *
 * @author kimi
 * @since 2020-02-15
 */
public interface SysAssembleLogMapper extends BaseMapper<SysAssembleLog> {

}

 src/main/java/com/highdatas/mdm/mapper/SysAssembleMapper.java

New file
@@ -0,0 +1,16 @@
package com.highdatas.mdm.mapper;

import com.highdatas.mdm.entity.SysAssemble;
import com.baomidou.mybatisplus.mapper.BaseMapper;

/**
 * <p>
 *  Mapper 接口
 * </p>
 *
 * @author kimi
 * @since 2020-02-20
 */
public interface SysAssembleMapper extends BaseMapper<SysAssemble> {

}

 src/main/java/com/highdatas/mdm/mapper/SysAssembleParamsMapper.java

New file
@@ -0,0 +1,16 @@
package com.highdatas.mdm.mapper;

import com.highdatas.mdm.entity.SysAssembleParams;
import com.baomidou.mybatisplus.mapper.BaseMapper;

/**
 * <p>
 *  Mapper 接口
 * </p>
 *
 * @author kimi
 * @since 2020-02-20
 */
public interface SysAssembleParamsMapper extends BaseMapper<SysAssembleParams> {

}

 src/main/java/com/highdatas/mdm/mapper/SysAssembleRuleMapper.java

New file
@@ -0,0 +1,16 @@
package com.highdatas.mdm.mapper;

import com.highdatas.mdm.entity.SysAssembleRule;
import com.baomidou.mybatisplus.mapper.BaseMapper;

/**
 * <p>
 *  Mapper 接口
 * </p>
 *
 * @author kimi
 * @since 2020-02-21
 */
public interface SysAssembleRuleMapper extends BaseMapper<SysAssembleRule> {

}

 src/main/java/com/highdatas/mdm/mapper/SysDbtypeMapper.java

New file
@@ -0,0 +1,16 @@
package com.highdatas.mdm.mapper;

import com.highdatas.mdm.entity.SysDbtype;
import com.baomidou.mybatisplus.mapper.BaseMapper;

/**
 * <p>
 *  Mapper 接口
 * </p>
 *
 * @author kimi
 * @since 2020-02-15
 */
public interface SysDbtypeMapper extends BaseMapper<SysDbtype> {

}

 src/main/java/com/highdatas/mdm/mapper/TableInfoMapper.java

@@ -1,6 +1,7 @@
package com.highdatas.mdm.mapper;

import com.highdatas.mdm.entity.TableSchemaResult;
import io.swagger.models.auth.In;
import org.apache.ibatis.annotations.Mapper;
import org.apache.ibatis.annotations.Param;

@@ -50,9 +51,11 @@
    void tempCreate2std(@Param("tableName") String tableName,@Param("tempFields") String tempFields, @Param("tableTempName") String tableTempName, @Param("maintainId") String maintainId);
    List<Map<String,Object>> tempByOperate(@Param("operate") String operate, @Param("fields") String fields, @Param("tableTempName") String tableTempName, @Param("maintainId") String maintainId);

    Long tempDeal(@Param("tableTempName") String tableTempName, @Param("maintainId") String maintainId);
    Integer tempDeal(@Param("tableTempName") String tableTempName, @Param("maintainId") String maintainId);

    Integer insertMatintainDetailFromTemp(@Param("maintainId")String maintainId, @Param("tableTempName")String tableTempName);
    Integer insertMatintainDetailFromTemp(@Param("maintainId")String maintainId, @Param("tableTempName")String tableTempName, @Param("operate") String operate);

    Integer insertMatintainDetailFromStandrad(@Param("maintainId")String maintainId, @Param("tableTempName")String tableTempName);

    Integer updateStdId(@Param("tableTempName")String tableTempName);

@@ -60,5 +63,6 @@

    Integer deleteRecordByMaintainId(@Param("tableTempName")String tableTempName, @Param("stdId") String stdId, @Param("maintainId")String maintainId);

    Integer insertRecordFromStandrad(@Param("tableName")String tableName, @Param("recordTableName")String recordTableName, @Param("tempFieldList")String tempFieldList);
}


 src/main/java/com/highdatas/mdm/mapper/xml/SysAssembleApiMapper.xml

New file
@@ -0,0 +1,17 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd">
<mapper namespace="com.highdatas.mdm.mapper.SysAssembleApiMapper">

    <!-- 通用查询映射结果 -->
    <resultMap id="BaseResultMap" type="com.highdatas.mdm.entity.SysAssembleApi">
        <id column="id" property="id" />
        <result column="parent_id" property="parentId" />
        <result column="url" property="url" />
        <result column="headers" property="headers" />
        <result column="pagination" property="pagination" />
        <result column="size" property="size" />
        <result column="pageurl" property="pageurl" />
        <result column="resultfield" property="resultfield" />
    </resultMap>

</mapper>

 src/main/java/com/highdatas/mdm/mapper/xml/SysAssembleCommonparamsMapper.xml

New file
@@ -0,0 +1,17 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd">
<mapper namespace="com.highdatas.mdm.mapper.SysAssembleCommonparamsMapper">

    <!-- 通用查询映射结果 -->
    <resultMap id="BaseResultMap" type="com.highdatas.mdm.entity.SysAssembleCommonparams">
        <id column="id" property="id" />
        <result column="code" property="code" />
        <result column="name" property="name" />
        <result column="val" property="val" />
        <result column="init_sql" property="initSql" />
        <result column="update_sql" property="updateSql" />
        <result column="create_time" property="createTime" />
        <result column="update_time" property="updateTime" />
    </resultMap>

</mapper>

 src/main/java/com/highdatas/mdm/mapper/xml/SysAssembleDbDetailMapper.xml

New file
@@ -0,0 +1,18 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd">
<mapper namespace="com.highdatas.mdm.mapper.SysAssembleDbDetailMapper">

    <!-- 通用查询映射结果 -->
    <resultMap id="BaseResultMap" type="com.highdatas.mdm.entity.SysAssembleDbDetail">
        <id column="id" property="id" />
        <result column="assemble_db_id" property="assembleDbId" />
        <result column="name" property="name" />
        <result column="field" property="field" />
        <result column="extra" property="extra" />
        <result column="val" property="val" />
        <result column="update_sql" property="updateSql" />
        <result column="create_time" property="createTime" />
        <result column="update_time" property="updateTime" />
    </resultMap>

</mapper>

 src/main/java/com/highdatas/mdm/mapper/xml/SysAssembleDbFieldMapper.xml

New file
@@ -0,0 +1,15 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd">
<mapper namespace="com.highdatas.mdm.mapper.SysAssembleDbFieldMapper">

    <!-- 通用查询映射结果 -->
    <resultMap id="BaseResultMap" type="com.highdatas.mdm.entity.SysAssembleDbField">
        <id column="id" property="id" />
        <result column="parent_id" property="parentId" />
        <result column="name" property="name" />
        <result column="field" property="field" />
        <result column="create_time" property="createTime" />
        <result column="update_time" property="updateTime" />
    </resultMap>

</mapper>

 src/main/java/com/highdatas/mdm/mapper/xml/SysAssembleDbMapper.xml

New file
@@ -0,0 +1,22 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd">
<mapper namespace="com.highdatas.mdm.mapper.SysAssembleDbMapper">

    <!-- 通用查询映射结果 -->
    <resultMap id="BaseResultMap" type="com.highdatas.mdm.entity.SysAssembleDb">
        <id column="id" property="id" />
        <result column="parent_id" property="parentId" />
        <result column="datasource_type" property="datasourceType" />
        <result column="datasource_name" property="datasourceName" />
        <result column="ipaddress_port" property="ipaddressPort" />
        <result column="database_name" property="databaseName" />
        <result column="datasource_url" property="datasourceUrl" />
        <result column="datasource_user" property="datasourceUser" />
        <result column="datasource_pass" property="datasourcePass" />
        <result column="orderby" property="orderby" />
        <result column="active" property="active" />
        <result column="create_time" property="createTime" />
        <result column="update_time" property="updateTime" />
    </resultMap>

</mapper>

 src/main/java/com/highdatas/mdm/mapper/xml/SysAssembleDbTableMapper.xml

New file
@@ -0,0 +1,19 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd">
<mapper namespace="com.highdatas.mdm.mapper.SysAssembleDbTableMapper">

    <!-- 通用查询映射结果 -->
    <resultMap id="BaseResultMap" type="com.highdatas.mdm.entity.SysAssembleDbTable">
        <id column="id" property="id" />
        <result column="parent_id" property="parentId" />
        <result column="type" property="type" />
        <result column="table_name" property="tableName" />
        <result column="sql" property="sql" />
        <result column="filter" property="filter" />
        <result column="orderby" property="orderby" />
        <result column="active" property="active" />
        <result column="create_time" property="createTime" />
        <result column="update_time" property="updateTime" />
    </resultMap>

</mapper>

 src/main/java/com/highdatas/mdm/mapper/xml/SysAssembleDbdriverMapper.xml

New file
@@ -0,0 +1,13 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd">
<mapper namespace="com.highdatas.mdm.mapper.SysAssembleDbdriverMapper">

    <!-- 通用查询映射结果 -->
    <resultMap id="BaseResultMap" type="com.highdatas.mdm.entity.SysAssembleDbdriver">
        <id column="id" property="id" />
        <result column="type" property="type" />
        <result column="driver" property="driver" />
        <result column="active" property="active" />
    </resultMap>

</mapper>

 src/main/java/com/highdatas/mdm/mapper/xml/SysAssembleDetailMapper.xml

New file
@@ -0,0 +1,13 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd">
<mapper namespace="com.highdatas.mdm.mapper.SysAssembleDetailMapper">

    <!-- 通用查询映射结果 -->
    <resultMap id="BaseResultMap" type="com.highdatas.mdm.entity.SysAssembleDetail">
        <id column="id" property="id" />
        <result column="assemble_id" property="assembleId" />
        <result column="field" property="field" />
        <result column="create_time" property="createTime" />
    </resultMap>

</mapper>

 src/main/java/com/highdatas/mdm/mapper/xml/SysAssembleLogMapper.xml

New file
@@ -0,0 +1,14 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd">
<mapper namespace="com.highdatas.mdm.mapper.SysAssembleLogMapper">

    <!-- 通用查询映射结果 -->
    <resultMap id="BaseResultMap" type="com.highdatas.mdm.entity.SysAssembleLog">
        <id column="id" property="id" />
        <result column="assemble_id" property="assembleId" />
        <result column="cnt" property="cnt" />
        <result column="status" property="status" />
        <result column="create_time" property="createTime" />
    </resultMap>

</mapper>

 src/main/java/com/highdatas/mdm/mapper/xml/SysAssembleMapper.xml

New file
@@ -0,0 +1,18 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd">
<mapper namespace="com.highdatas.mdm.mapper.SysAssembleMapper">

    <!-- 通用查询映射结果 -->
    <resultMap id="BaseResultMap" type="com.highdatas.mdm.entity.SysAssemble">
        <id column="id" property="id" />
        <result column="menu_id" property="menuId" />
        <result column="user_id" property="userId" />
        <result column="job_id" property="jobId" />
        <result column="cron" property="cron" />
        <result column="bigdata" property="bigdata" />
        <result column="status" property="status" />
        <result column="create_time" property="createTime" />
        <result column="update_time" property="updateTime" />
    </resultMap>

</mapper>

 src/main/java/com/highdatas/mdm/mapper/xml/SysAssembleParamsMapper.xml

New file
@@ -0,0 +1,18 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd">
<mapper namespace="com.highdatas.mdm.mapper.SysAssembleParamsMapper">

    <!-- 通用查询映射结果 -->
    <resultMap id="BaseResultMap" type="com.highdatas.mdm.entity.SysAssembleParams">
        <id column="id" property="id" />
        <result column="parent_id" property="parentId" />
        <result column="code" property="code" />
        <result column="name" property="name" />
        <result column="val" property="val" />
        <result column="init_sql" property="initSql" />
        <result column="update_sql" property="updateSql" />
        <result column="create_time" property="createTime" />
        <result column="update_time" property="updateTime" />
    </resultMap>

</mapper>

 src/main/java/com/highdatas/mdm/mapper/xml/SysAssembleRuleMapper.xml

New file
@@ -0,0 +1,14 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd">
<mapper namespace="com.highdatas.mdm.mapper.SysAssembleRuleMapper">

    <!-- 通用查询映射结果 -->
    <resultMap id="BaseResultMap" type="com.highdatas.mdm.entity.SysAssembleRule">
        <id column="id" property="id" />
        <result column="parent_id" property="parentId" />
        <result column="rule_id" property="ruleId" />
        <result column="create_time" property="createTime" />
        <result column="update_time" property="updateTime" />
    </resultMap>

</mapper>

 src/main/java/com/highdatas/mdm/mapper/xml/SysDbtypeMapper.xml

New file
@@ -0,0 +1,13 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd">
<mapper namespace="com.highdatas.mdm.mapper.SysDbtypeMapper">

    <!-- 通用查询映射结果 -->
    <resultMap id="BaseResultMap" type="com.highdatas.mdm.entity.SysDbtype">
        <result column="id" property="id" />
        <result column="type" property="type" />
        <result column="name" property="name" />
        <result column="active" property="active" />
    </resultMap>

</mapper>

 src/main/java/com/highdatas/mdm/pojo/DbAccessType.java

New file
@@ -0,0 +1,12 @@
package com.highdatas.mdm.pojo;



/**

 * @author kimi

 * @description

 * @date 2020-02-15 22:57

 */





public enum  DbAccessType {

    relational,unRelational

}


 src/main/java/com/highdatas/mdm/pojo/SysAssembleCheckType.java

New file
@@ -0,0 +1,12 @@
package com.highdatas.mdm.pojo;



/**

 * @author kimi

 * @description

 * @date 2020-02-17 15:30

 */





public enum SysAssembleCheckType {

    successAdd,partSuccessAdd,total

}


 src/main/java/com/highdatas/mdm/pojo/SysAssembleLogStatus.java

New file
@@ -0,0 +1,12 @@
package com.highdatas.mdm.pojo;



/**

 * @author kimi

 * @description

 * @date 2020-02-17 15:30

 */





public enum SysAssembleLogStatus {

    working,success,defeat

}


 src/main/java/com/highdatas/mdm/pojo/SysAssembleRunStatus.java

New file
@@ -0,0 +1,12 @@
package com.highdatas.mdm.pojo;



/**

 * @author kimi

 * @description

 * @date 2020-02-17 15:30

 */





public enum SysAssembleRunStatus {

    working,success,fail

}


 src/main/java/com/highdatas/mdm/pojo/SysAssembleStatus.java

New file
@@ -0,0 +1,12 @@
package com.highdatas.mdm.pojo;



/**

 * @author kimi

 * @description

 * @date 2020-02-17 15:30

 */





public enum SysAssembleStatus {

    working,edit,freeze,

}


 src/main/java/com/highdatas/mdm/pojo/SysAssembleTableType.java

New file
@@ -0,0 +1,12 @@
package com.highdatas.mdm.pojo;



/**

 * @author kimi

 * @description

 * @date 2020-02-17 15:30

 */





public enum SysAssembleTableType {

    table,sql

}


 src/main/java/com/highdatas/mdm/pojo/SysAssembleUpdateType.java

New file
@@ -0,0 +1,12 @@
package com.highdatas.mdm.pojo;



/**

 * @author kimi

 * @description

 * @date 2020-02-17 15:30

 */





public enum SysAssembleUpdateType {

    All,Increment

}


 src/main/java/com/highdatas/mdm/pojo/kettle/BigDataDataSourceInfo.java

New file
@@ -0,0 +1,165 @@
package com.highdatas.mdm.pojo.kettle;



import com.alibaba.druid.pool.DruidDataSource;

import com.alibaba.druid.pool.DruidPooledConnection;

import org.springframework.beans.factory.annotation.Value;

import org.springframework.boot.context.properties.ConfigurationProperties;

import org.springframework.context.annotation.Bean;

import org.springframework.context.annotation.Primary;

import org.springframework.stereotype.Component;



import javax.sql.DataSource;

import java.sql.SQLException;

import java.util.List;



/**

 * Created by LF on 2017/4/18.

 */

@ConfigurationProperties(prefix = "master.unbigdata.db")

@Component

public class BigDataDataSourceInfo implements DataSourceInfo{

  
  @Value("${master.unbigdata.db.port}")

  private String dbPort;

  
  @Value("${master.unbigdata.db.schema}")

  private String dbName;

	
  @Value("${master.unbigdata.db.type}")

  private String dbType;

  
  @Value("${master.unbigdata.db.host}")

  private String dbHostName;



  @Value("${master.unbigdata.db.url}")

  private String dbUrl;



  @Value("${spring.datasource.username}")

  private String username;

 
  @Value("${spring.datasource.password}")

  private String password;

 
  @Value("${spring.datasource.driver-class-name}")

  private String driverClassName;

  
  private DruidDataSource datasource;

  
  public String getDbPort() {

    return dbPort;

  }



  public void setDbPort(String dbPort) {

    this.dbPort = dbPort;

  }



  public String getDbName() {

    return dbName;

  }



  public void setDbName(String dbName) {

    this.dbName = dbName;

  }



  public String getDbHostName() {

    return dbHostName;

  }



  public void setDbHostName(String dbHostName) {

    this.dbHostName = dbHostName;

  }



  public String getDbType() {

    return dbType;

  }



  public void setDbType(String dbType) {

    this.dbType = dbType;

  }



  public String getDbUrl() {

    return dbUrl;

  }



  public void setDbUrl(String dbUrl) {

    this.dbUrl = dbUrl;

  }

 
  public String getUsername() {

    return username;

  }

 
  public void setUsername(String username) {

    this.username = username;

  }

 
  public String getPassword() {

    return password;

  }

 
  public void setPassword(String password) {

    this.password = password;

  }

 
  public String getDriverClassName() {

    return driverClassName;

  }

 
  public void setDriverClassName(String driverClassName) {

    this.driverClassName = driverClassName;

  }

 
  @Bean   //声明其为Bean实例

  @Primary //在同样的DataSource中，首先使用被标注的DataSource

  public DataSource dataSource()  {

    datasource = new DruidDataSource();



    //datasource.setUrl("jdbc:mysql://localhost:3306/ssm?useUnicode=true&useJDBCCompliantTimezoneShift=true&useLegacyDatetimeCode=false&serverTimezone=UTC&characterEncoding=UTF8");

    //datasource.setUsername("root");

    //datasource.setPassword("123456");

    //datasource.setDriverClassName("com.mysql.cj.jdbc.Driver");

    datasource.setUrl(this.dbUrl);

    datasource.setUsername(this.username);

    datasource.setPassword(this.password);

    datasource.setDriverClassName(this.driverClassName);



    return this.datasource;

  }



  @Override

  public boolean createTable(String assembleTempTableName, List<String> fieldList) {

    return false;

  }



  @Override

  public boolean checkFieldList(String tableName, List<String> fieldList) {

    return false;

  }



  @Override

  public boolean checkTableExists(String assembleTempTableName) {

    return false;

  }



  @Override

  public List<String> getFieldsFromTable(String tableName) {

    return null;

  }



  @Bean

  @Primary

  public DruidPooledConnection conn() throws SQLException {

      return this.datasource.getConnection();

  }



  @Override

  public boolean truncateData(String tempTableName) {

    return true;

  }



  @Override

  public boolean dropData(String tempTableName) {

    return false;

  }



}

 src/main/java/com/highdatas/mdm/pojo/kettle/DataSourceInfo.java

New file
@@ -0,0 +1,27 @@
package com.highdatas.mdm.pojo.kettle;



import java.sql.Connection;

import java.sql.SQLException;

import java.util.List;



/**

 * @author kimi

 * @description

 * @date 2020-02-21 21:23

 */





public interface DataSourceInfo {



    boolean createTable(String assembleTempTableName, List<String> fieldList);



    boolean checkFieldList(String tableName, List<String> fieldList);



    boolean checkTableExists(String assembleTempTableName);



    List<String> getFieldsFromTable(String tableName);

    Connection conn() throws SQLException;



    boolean truncateData(String tempTableName);

    boolean dropData(String tempTableName);

}


 src/main/java/com/highdatas/mdm/pojo/kettle/KettleDBTrans.java

New file
@@ -0,0 +1,72 @@
package com.highdatas.mdm.pojo.kettle;



import com.alibaba.fastjson.JSONObject;

import com.highdatas.mdm.entity.SysAssemble;

import org.pentaho.di.core.database.Database;

import org.pentaho.di.trans.TransMeta;

import org.pentaho.di.trans.steps.tableinput.TableInputMeta;

import org.pentaho.di.trans.steps.tableoutput.TableOutputMeta;



public class KettleDBTrans {



    private SysAssemble assemble;

    private TransMeta transMeta;

    private Database dsInput;

    private Database dsOutput;

    private TableInputMeta tableInputMeta;

    private TableOutputMeta talbeOutputMeta;



    public SysAssemble getAssemble() {

        return assemble;

    }



    public void setAssemble(SysAssemble assemble) {

        this.assemble = assemble;

    }



    public TransMeta getTransMeta() {

        return transMeta;

    }

	
    public void setTransMeta(TransMeta transMeta) {

        this.transMeta = transMeta;

    }

	
    public Database getDsInput() {

        return dsInput;

    }

	
    public void setDsInput(Database dsInput) {

        this.dsInput = dsInput;

    }

	
    public Database getDsOutput() {

        return dsOutput;

    }

	
    public void setDsOutput(Database dsOutput) {

        this.dsOutput = dsOutput;

    }

	
    public TableInputMeta getTableInputMeta() {

        return tableInputMeta;

    }

	
    public void setTableInputMeta(TableInputMeta tableInputMeta) {

        this.tableInputMeta = tableInputMeta;

    }

	
    public TableOutputMeta getTalbeOutputMeta() {

        return talbeOutputMeta;

    }

	
    public void setTalbeOutputMeta(TableOutputMeta talbeOutputMeta) {

        this.talbeOutputMeta = talbeOutputMeta;

    }

	
}












 src/main/java/com/highdatas/mdm/pojo/kettle/TableColumn.java

New file
@@ -0,0 +1,33 @@
package com.highdatas.mdm.pojo.kettle;



public class TableColumn {



    private String columnName;

    private String columnType;

    private String columnLength;

	
    public String getColumnName() {

        return columnName;

    }

	
    public void setColumnName(String columnName) {

        this.columnName = columnName;

    }

	
    public String getColumnType() {

        return columnType;

    }

	
    public void setColumnType(String columnType) {

        this.columnType = columnType;

    }

	
    public String getColumnLength() {

        return columnLength;

    }

	
    public void setColumnLength(String columnLength) {

        this.columnLength = columnLength;

    }

	
}


 src/main/java/com/highdatas/mdm/pojo/kettle/UnBigDataDataSourceInfo.java

New file
@@ -0,0 +1,301 @@
package com.highdatas.mdm.pojo.kettle;



import com.alibaba.druid.pool.DruidDataSource;

import com.highdatas.mdm.util.Constant;

import com.highdatas.mdm.util.DbUtils;

import org.springframework.beans.factory.annotation.Autowired;

import org.springframework.beans.factory.annotation.Value;

import org.springframework.boot.context.properties.ConfigurationProperties;

import org.springframework.context.annotation.Bean;

import org.springframework.context.annotation.Primary;

import org.springframework.stereotype.Component;



import javax.sql.DataSource;

import java.sql.*;

import java.text.MessageFormat;

import java.util.ArrayList;

import java.util.List;

import java.util.stream.Collectors;



/**

 * Created by LF on 2017/4/18.

 */

@ConfigurationProperties(prefix = "master.unbigdata.db")

@Component

public class UnBigDataDataSourceInfo implements DataSourceInfo{

  @Autowired

  private DataSource dataSource;



  @Value("${master.unbigdata.db.port}")

  private String dbPort;

  
  @Value("${master.unbigdata.db.schema}")

  private String dbName;

	
  @Value("${master.unbigdata.db.type}")

  private String dbType;

  
  @Value("${master.unbigdata.db.host}")

  private String dbHostName;



  @Value("${master.unbigdata.db.url}")

  private String dbUrl;



  @Value("${spring.datasource.username}")

  private String username;

 
  @Value("${spring.datasource.password}")

  private String password;

 
  @Value("${spring.datasource.driver-class-name}")

  private String driverClassName;

  
  private DruidDataSource datasource;

  
  public String getDbPort() {

    return dbPort;

  }



  public void setDbPort(String dbPort) {

    this.dbPort = dbPort;

  }



  public String getDbName() {

    return dbName;

  }



  public void setDbName(String dbName) {

    this.dbName = dbName;

  }



  public String getDbHostName() {

    return dbHostName;

  }



  public void setDbHostName(String dbHostName) {

    this.dbHostName = dbHostName;

  }



  public String getDbType() {

    return dbType;

  }



  public void setDbType(String dbType) {

    this.dbType = dbType;

  }



  public String getDbUrl() {

    return dbUrl;

  }



  public void setDbUrl(String dbUrl) {

    this.dbUrl = dbUrl;

  }

 
  public String getUsername() {

    return username;

  }

 
  public void setUsername(String username) {

    this.username = username;

  }

 
  public String getPassword() {

    return password;

  }

 
  public void setPassword(String password) {

    this.password = password;

  }

 
  public String getDriverClassName() {

    return driverClassName;

  }

 
  public void setDriverClassName(String driverClassName) {

    this.driverClassName = driverClassName;

  }





  @Override

  public boolean createTable(String assembleTempTableName, List<String> fieldList) {

    String fieldStr = fieldList.stream().map(s -> MessageFormat.format(Constant.CreateFieldMysql,s)).collect(Collectors.joining(Constant.COMMA));

    String create_sql = MessageFormat.format(Constant.CreateTableMysql, assembleTempTableName, fieldStr);

    Connection conn = null;

    try {

      conn = conn();

      PreparedStatement preparedStatement = conn.prepareStatement(create_sql);

      preparedStatement.execute();

      return true;

    } catch (SQLException e) {

      e.printStackTrace();

      return false;

    }

    finally {

      if (conn != null) {

        try {

          conn.close();

        } catch (SQLException e) {

          e.printStackTrace();

          return false;

        }

      }

    }

  }



  @Override

  public boolean checkFieldList(String tableName, List<String> fieldList) {

      Connection conn = null;

    try {



        conn = conn();

        PreparedStatement preparedStatement = conn.prepareStatement(MessageFormat.format(Constant.selectFieldSqlMysqlTemplate, tableName));

        ResultSet resultSet = preparedStatement.executeQuery();

        ResultSetMetaData metaData = resultSet.getMetaData();

        int columnCount = metaData.getColumnCount();

        List<String> tableFieldList = new ArrayList<>();

        for (int i = 1; i <= columnCount; i++) {

            tableFieldList.add(metaData.getColumnLabel(i)) ;

        }

        return  tableFieldList.containsAll(fieldList);

    } catch (SQLException e) {

      e.printStackTrace();

        return false;

    }finally {

        if (conn != null) {

            try {

                conn.close();

            } catch (SQLException e) {

                e.printStackTrace();

                return false;

            }

        }

    }





  }



  @Override

  public boolean checkTableExists(String assembleTempTableName) {

      Connection conn = null;

      try {

          conn = conn();

          PreparedStatement preparedStatement = conn.prepareStatement(MessageFormat.format(Constant.checkTableByNameBySchema, DbUtils.quotedStr(this.dbName), DbUtils.quotedStr(assembleTempTableName)));

          ResultSet resultSet = preparedStatement.executeQuery();

          resultSet.last();

          int row = resultSet.getRow();

          if (row > 0) {

              return true;

          }else {

              return false;

          }

      } catch (SQLException e) {

          e.printStackTrace();

          return false;

      }finally {

          if (conn != null) {

              try {

                  conn.close();

              } catch (SQLException e) {

                  e.printStackTrace();

                  return false;

              }

          }

      }



  }



    @Override

    public List<String> getFieldsFromTable(String tableName) {

        Connection conn = null;

        try {

            conn = conn();

            PreparedStatement preparedStatement = conn.prepareStatement(MessageFormat.format(Constant.selectFieldSqlMysqlTemplate, tableName));

            ResultSet resultSet = preparedStatement.executeQuery();

            ResultSetMetaData metaData = resultSet.getMetaData();

            int columnCount = metaData.getColumnCount();

            List<String> tableFieldList = new ArrayList<>();

            for (int i = 1; i <= columnCount; i++) {

                tableFieldList.add(metaData.getColumnLabel(i)) ;

            }

            return tableFieldList;

        } catch (SQLException e) {

            e.printStackTrace();

            return null;

        }finally {

            if (conn != null) {

                try {

                    conn.close();

                } catch (SQLException e) {

                    e.printStackTrace();

                    return null;

                }

            }

        }

    }



    @Bean

  @Primary

  public Connection conn() throws SQLException {

      return dataSource.getConnection();

  }



    @Override

    public boolean truncateData(String tempTableName) {

        boolean exists = checkTableExists(tempTableName);

        if (!exists) {

            return false;

        }

        String truncateData = MessageFormat.format(Constant.truncateTableMysql, tempTableName);

        Connection conn = null;

        try {

            conn = conn();

            PreparedStatement preparedStatement = conn.prepareStatement(truncateData);

            preparedStatement.execute();

            return true;

        } catch (SQLException e) {

            e.printStackTrace();

            return false;

        }

        finally {

            if (conn != null) {

                try {

                    conn.close();

                } catch (SQLException e) {

                    e.printStackTrace();

                    return false;

                }

            }

        }

    }



    @Override

    public boolean dropData(String tempTableName) {

        boolean exists = checkTableExists(tempTableName);

        if (!exists) {

            return true;

        }

        String truncateData = MessageFormat.format(Constant.DropTableMysql, tempTableName);

        Connection conn = null;

        try {

            conn = conn();

            PreparedStatement preparedStatement = conn.prepareStatement(truncateData);

            preparedStatement.execute();

            return true;

        } catch (SQLException e) {

            e.printStackTrace();

            return false;

        }

        finally {

            if (conn != null) {

                try {

                    conn.close();

                } catch (SQLException e) {

                    e.printStackTrace();

                    return false;

                }

            }

        }

    }



}

 src/main/java/com/highdatas/mdm/service/ISysAssembleApiService.java

New file
@@ -0,0 +1,16 @@
package com.highdatas.mdm.service;

import com.highdatas.mdm.entity.SysAssembleApi;
import com.baomidou.mybatisplus.service.IService;

/**
 * <p>
 *  服务类
 * </p>
 *
 * @author kimi
 * @since 2020-02-19
 */
public interface ISysAssembleApiService extends IService<SysAssembleApi> {

}

 src/main/java/com/highdatas/mdm/service/ISysAssembleCommonparamsService.java

New file
@@ -0,0 +1,16 @@
package com.highdatas.mdm.service;

import com.highdatas.mdm.entity.SysAssembleCommonparams;
import com.baomidou.mybatisplus.service.IService;

/**
 * <p>
 *  服务类
 * </p>
 *
 * @author kimi
 * @since 2020-02-20
 */
public interface ISysAssembleCommonparamsService extends IService<SysAssembleCommonparams> {

}

 src/main/java/com/highdatas/mdm/service/ISysAssembleDbDetailService.java

New file
@@ -0,0 +1,16 @@
package com.highdatas.mdm.service;

import com.highdatas.mdm.entity.SysAssembleDbDetail;
import com.baomidou.mybatisplus.service.IService;

/**
 * <p>
 *  服务类
 * </p>
 *
 * @author kimi
 * @since 2020-02-19
 */
public interface ISysAssembleDbDetailService extends IService<SysAssembleDbDetail> {

}

 src/main/java/com/highdatas/mdm/service/ISysAssembleDbFieldService.java

New file
@@ -0,0 +1,16 @@
package com.highdatas.mdm.service;

import com.highdatas.mdm.entity.SysAssembleDbField;
import com.baomidou.mybatisplus.service.IService;

/**
 * <p>
 *  服务类
 * </p>
 *
 * @author kimi
 * @since 2020-02-20
 */
public interface ISysAssembleDbFieldService extends IService<SysAssembleDbField> {

}

 src/main/java/com/highdatas/mdm/service/ISysAssembleDbService.java

New file
@@ -0,0 +1,27 @@
package com.highdatas.mdm.service;

import com.highdatas.mdm.entity.SysAssembleDb;
import com.baomidou.mybatisplus.service.IService;

import java.sql.Connection;
import java.util.List;

/**
 * <p>
 *  服务类
 * </p>
 *
 * @author kimi
 * @since 2020-02-20
 */
public interface ISysAssembleDbService extends IService<SysAssembleDb> {

    boolean testConnection(String parentId);
    boolean testSql(String parentId, String sql);

    List<String> getFieldsBySql(String parentId, String sql);

    Connection getConnection(String dbId);

    List<String> getFieldsByTableName(String parentId, String tableName);
}

 src/main/java/com/highdatas/mdm/service/ISysAssembleDbTableService.java

New file
@@ -0,0 +1,16 @@
package com.highdatas.mdm.service;

import com.highdatas.mdm.entity.SysAssembleDbTable;
import com.baomidou.mybatisplus.service.IService;

/**
 * <p>
 *  服务类
 * </p>
 *
 * @author kimi
 * @since 2020-02-20
 */
public interface ISysAssembleDbTableService extends IService<SysAssembleDbTable> {

}

 src/main/java/com/highdatas/mdm/service/ISysAssembleDbdriverService.java

New file
@@ -0,0 +1,16 @@
package com.highdatas.mdm.service;

import com.highdatas.mdm.entity.SysAssembleDbdriver;
import com.baomidou.mybatisplus.service.IService;

/**
 * <p>
 *  服务类
 * </p>
 *
 * @author kimi
 * @since 2020-02-21
 */
public interface ISysAssembleDbdriverService extends IService<SysAssembleDbdriver> {

}

 src/main/java/com/highdatas/mdm/service/ISysAssembleDetailService.java

New file
@@ -0,0 +1,16 @@
package com.highdatas.mdm.service;

import com.highdatas.mdm.entity.SysAssembleDetail;
import com.baomidou.mybatisplus.service.IService;

/**
 * <p>
 *  服务类
 * </p>
 *
 * @author kimi
 * @since 2020-02-15
 */
public interface ISysAssembleDetailService extends IService<SysAssembleDetail> {

}

 src/main/java/com/highdatas/mdm/service/ISysAssembleLogService.java

New file
@@ -0,0 +1,16 @@
package com.highdatas.mdm.service;

import com.highdatas.mdm.entity.SysAssembleLog;
import com.baomidou.mybatisplus.service.IService;

/**
 * <p>
 *  服务类
 * </p>
 *
 * @author kimi
 * @since 2020-02-15
 */
public interface ISysAssembleLogService extends IService<SysAssembleLog> {

}

 src/main/java/com/highdatas/mdm/service/ISysAssembleParamsService.java

New file
@@ -0,0 +1,17 @@
package com.highdatas.mdm.service;

import com.highdatas.mdm.entity.SysAssembleParams;
import com.baomidou.mybatisplus.service.IService;

/**
 * <p>
 *  服务类
 * </p>
 *
 * @author kimi
 * @since 2020-02-20
 */
public interface ISysAssembleParamsService extends IService<SysAssembleParams> {

    boolean updateCornVal(String assembleId, String cron);
}

 src/main/java/com/highdatas/mdm/service/ISysAssembleRuleService.java

New file
@@ -0,0 +1,16 @@
package com.highdatas.mdm.service;

import com.highdatas.mdm.entity.SysAssembleRule;
import com.baomidou.mybatisplus.service.IService;

/**
 * <p>
 *  服务类
 * </p>
 *
 * @author kimi
 * @since 2020-02-21
 */
public interface ISysAssembleRuleService extends IService<SysAssembleRule> {

}

 src/main/java/com/highdatas/mdm/service/ISysAssembleService.java

New file
@@ -0,0 +1,23 @@
package com.highdatas.mdm.service;

import com.highdatas.mdm.entity.SysAssemble;
import com.baomidou.mybatisplus.service.IService;
import com.highdatas.mdm.pojo.Result;
import org.springframework.transaction.annotation.Transactional;

import javax.servlet.http.HttpSession;

/**
 * <p>
 *  服务类
 * </p>
 *
 * @author kimi
 * @since 2020-02-20
 */
public interface ISysAssembleService extends IService<SysAssemble> {


    @Transactional(rollbackFor = {Exception.class, Error.class})
    Result run(String id, HttpSession session);
}

 src/main/java/com/highdatas/mdm/service/ISysDbtypeService.java

New file
@@ -0,0 +1,16 @@
package com.highdatas.mdm.service;

import com.highdatas.mdm.entity.SysDbtype;
import com.baomidou.mybatisplus.service.IService;

/**
 * <p>
 *  服务类
 * </p>
 *
 * @author kimi
 * @since 2020-02-15
 */
public interface ISysDbtypeService extends IService<SysDbtype> {

}

 src/main/java/com/highdatas/mdm/service/ISysMenuService.java

@@ -3,6 +3,9 @@
import com.highdatas.mdm.entity.SysMenu;
import com.baomidou.mybatisplus.service.IService;

import java.util.LinkedHashSet;
import java.util.Set;

/**
 * <p>
 *  服务类
@@ -14,4 +17,5 @@
public interface ISysMenuService extends IService<SysMenu> {


    LinkedHashSet<String> getByParentId(LinkedHashSet<String> parentIdSet);
}

 src/main/java/com/highdatas/mdm/service/MasterDataService.java

@@ -1,6 +1,8 @@
package com.highdatas.mdm.service;

import com.highdatas.mdm.entity.Maintain;
import com.highdatas.mdm.pojo.Result;
import com.highdatas.mdm.pojo.SysAssembleUpdateType;

import java.util.List;

@@ -76,4 +78,6 @@
    Result selectListByPageByVersion(String tableName, String whereSegment, Integer pageNo, String version, boolean findMax);

    Result selectListByPageByVersion(String tableName, List<String> fieldList, String whereSegment, Integer pageNo, Integer pageSize, String version, boolean findMax);

    Maintain uploadedData(String tableName, SysAssembleUpdateType uploadType, String userId);
}

 src/main/java/com/highdatas/mdm/service/act/impl/TaskServiceImpl.java

@@ -90,7 +90,7 @@
    public List<String> getTodoTask()  {
        TUser onlineUser = getOnlineUser();
        Set<String> result = new HashSet<>();
        //List<Task> list1 = taskService.createTaskQuery().active().list();
        //List<Task> list1 = TimeTaskService.createTaskQuery().active().list();
        List<Task> list = taskService.createTaskQuery().taskAssignee(onlineUser.getUserId()).active().list();

        result.addAll(list.stream().map(task -> task.getProcessInstanceId()).collect(Collectors.toList()));

 src/main/java/com/highdatas/mdm/service/impl/MaintainServiceImpl.java

@@ -98,7 +98,10 @@
        if (status.equals(ActivitiStatus.close)) {
             String collect = maintainDetailList.stream().map(detail -> DbUtils.quotedStr(detail.getPreMergeId())).collect(Collectors.joining(Constant.COMMA));
            //delete temp
            masterDataService.delete(tableName + Constant.RECORD, "id in (" +collect+ ")");
            if (!StringUtils.isEmpty(collect)) {
                masterDataService.delete(tableName + Constant.RECORD, "id in (" +collect+ ")");
            }

            //delete detail
            for (MaintainDetail maintainDetail : maintainDetailList) {
                maintainDetail.deleteById();
@@ -173,7 +176,7 @@

    @Override
    public Result getUnSubmitData(String tableName, Integer pageNo, String pageSize, String whereSegment) {
        Wrapper<Maintain> versionWrapper = new EntityWrapper<Maintain>().eq("table_name", tableName).isNull("flow_id").orderBy("version desc");
        Wrapper<Maintain> versionWrapper = new EntityWrapper<Maintain>().eq("table_name", tableName).isNull("flow_id").orderBy("order_no desc");
        Maintain maintain = selectOne(versionWrapper);
        if (maintain == null) {
            return Result.error(CodeMsg.SELECT_ERROR_NOTFOUND);
@@ -212,7 +215,7 @@

    @Override
    public Maintain getMaxVersion(String tableName) {
        Wrapper<Maintain> versionWrapper = new EntityWrapper<Maintain>().eq("table_name", tableName).orderBy("version desc");
        Wrapper<Maintain> versionWrapper = new EntityWrapper<Maintain>().eq("table_name", tableName).orderBy("order_no desc");

        Maintain maxVersion = selectOne(versionWrapper);
        return maxVersion;
@@ -220,7 +223,7 @@

    @Override
    public Maintain getUnFlowMaxVersion(String tableName) {
        Wrapper<Maintain> versionWrapper = new EntityWrapper<Maintain>().isNull("flow_id").eq("table_name", tableName).orderBy("version desc");
        Wrapper<Maintain> versionWrapper = new EntityWrapper<Maintain>().isNull("flow_id").eq("table_name", tableName).orderBy("order_no desc");

        Maintain maxVersion = selectOne(versionWrapper);
        return maxVersion;
@@ -229,7 +232,7 @@
    @Override
    public Maintain getInvalidMaxVersion(String tableName) {
        Maintain result = null;
        Wrapper<Maintain> versionWrapper = new EntityWrapper<Maintain>().eq("table_name", tableName).orderBy("version desc");
        Wrapper<Maintain> versionWrapper = new EntityWrapper<Maintain>().eq("table_name", tableName).orderBy("order_no desc");

        List<Maintain> maintainList = selectList(versionWrapper);

@@ -279,7 +282,7 @@

    @Override
    public Maintain getNowVersion(String tableName) {
        Wrapper<Maintain> versionWrapper = new EntityWrapper<Maintain>().eq("table_name", tableName).isNotNull("flow_id").orderBy("version desc");
        Wrapper<Maintain> versionWrapper = new EntityWrapper<Maintain>().eq("table_name", tableName).isNotNull("flow_id").orderBy("order_no desc");

        List<Maintain> maintains = selectList(versionWrapper);
        for (Maintain maintain : maintains) {
@@ -299,7 +302,7 @@
    @Override
    public Maintain getRecentHistoryVersion(String tableName) {
        Maintain nowMaintain = null;
        Wrapper<Maintain> versionWrapper = new EntityWrapper<Maintain>().eq("table_name", tableName).isNotNull("flow_id").orderBy("version desc");
        Wrapper<Maintain> versionWrapper = new EntityWrapper<Maintain>().eq("table_name", tableName).isNotNull("flow_id").orderBy("order_no desc");
        List<Maintain> maintains = selectList(versionWrapper);
        for (Maintain maintain : maintains) {
            Flows flows = flowsService.selectById(maintain.getFlowId());
@@ -316,7 +319,7 @@
            }
        }
        if (nowMaintain == null) {
            nowMaintain = selectOne(new EntityWrapper<Maintain>().eq("table_name", tableName).orderBy("version desc"));
            nowMaintain = selectOne(new EntityWrapper<Maintain>().eq("table_name", tableName).orderBy("order_no desc"));
        }
        int orderNo = nowMaintain.getOrderNo();
        Maintain nowVersion = getNowVersion(tableName);

 src/main/java/com/highdatas/mdm/service/impl/MasterDataServiceImpl.java

@@ -4,23 +4,21 @@
import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import com.baomidou.mybatisplus.mapper.EntityWrapper;
import com.highdatas.mdm.entity.Flows;
import com.highdatas.mdm.entity.Maintain;
import com.highdatas.mdm.entity.SysField;
import com.highdatas.mdm.entity.TableSchemaResult;
import com.highdatas.mdm.entity.*;
import com.highdatas.mdm.mapper.TableInfoMapper;
import com.highdatas.mdm.pojo.*;
import com.highdatas.mdm.service.IFlowsService;
import com.highdatas.mdm.service.IMaintainService;
import com.highdatas.mdm.service.ISysFieldService;
import com.highdatas.mdm.service.MasterDataService;
import com.highdatas.mdm.service.*;
import com.highdatas.mdm.util.Constant;
import com.highdatas.mdm.util.ContentBuilder;
import com.highdatas.mdm.util.DbUtils;
import org.apache.commons.lang3.StringUtils;
import org.mybatis.spring.SqlSessionTemplate;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;

import javax.annotation.Resource;
import java.sql.Connection;
import java.sql.SQLException;
import java.text.MessageFormat;
import java.util.*;
import java.util.stream.Collectors;
@@ -34,14 +32,19 @@
@Service
public class MasterDataServiceImpl implements MasterDataService {
    @Autowired
    IMenuMappingService menuMappingService;
    @Autowired
    TableInfoMapper mapper;
    @Autowired
    ISysFieldService fieldService;
    @Autowired
    IMaintainService maintainService;
    @Autowired
    IMaintainDetailService maintainDetailService;
    @Autowired
    IFlowsService flowsService;

    @Resource
    protected SqlSessionTemplate sqlSessionTemplate;
    @Override
    public Result selectById(String tableName, String id) {
        try{
@@ -391,7 +394,107 @@
        return Result.success(resultobject);
    }

   //bak
    @Override
    public Maintain uploadedData(String tableName, SysAssembleUpdateType uploadType, String userId) {
        MenuMapping menuMapping = menuMappingService.selectOne(new EntityWrapper<MenuMapping>().eq("table_name",tableName));
        SysOperateLog operateLog = new SysOperateLog();

        Maintain maxVersion = maintainService.getMaxVersion(tableName);
        Maintain maintain = new Maintain();
        String biggerVersion = null;
        if (maxVersion == null) {
            biggerVersion = Constant.VERSION_Default;
            maintain.setOrderNo(0);
            operateLog.setOperate("初始化主题数据");
        } else {
            operateLog.setOperate("更新主题");
            String flowId = maxVersion.getFlowId();
            String mVersion = maxVersion.getVersion();
            if (StringUtils.isEmpty(flowId)) {
                biggerVersion = mVersion;
                maintain.setOrderNo(maxVersion.getOrderNo());
                String tempName = maxVersion.getTableName() + Constant.RECORD;

                mapper.deleteTempByMaintainId(tempName, DbUtils.quotedStr(maxVersion.getId()));
                maintainDetailService.delete(new EntityWrapper<MaintainDetail>().eq("parent_id", maxVersion.getId()));

                maxVersion.deleteById();
            }else {
                biggerVersion = DbUtils.versionAddBig(mVersion);
                maintain.setOrderNo(DbUtils.getOrderNoAdd(maxVersion.getOrderNo()));
            }
        }

        String uuid = DbUtils.getUUID();
        maintain.setId(uuid);

        maintain.setVersion(biggerVersion);
        maintain.setChargeId(userId);
        maintain.setTableName(tableName);
        maintain.setCreateTime(new Date());

        Connection connection = sqlSessionTemplate.getConnection();
        try {
            mapper.insertMatintainDetailFromTemp(DbUtils.quotedStr(maintain.getId()), maintain.getTableName() + Constant.RECORD , DbUtils.quotedStr(Operate.create.toString()));
            connection.commit();
            mapper.updateStdId( maintain.getTableName() + Constant.RECORD);
            connection.commit();
            Integer aLong = mapper.tempDeal(maintain.getTableName() + Constant.RECORD, DbUtils.quotedStr(maintain.getId()));
            connection.commit();
            if (aLong == null) {
                maintain.setDesp("上传数据");
            }
            maintain.setDesp(MessageFormat.format("上传数据{0}条", aLong));

            if (uploadType.equals(SysAssembleUpdateType.All)) {
                List<TableSchemaResult> tableField = mapper.getTableField(tableName);
                List<String> tableFieldList = tableField.stream().map(tableSchemaResult -> tableSchemaResult.getFieldName()).collect(Collectors.toList());
                tableFieldList.add(Constant.STD_ID);
                String collect = tableFieldList.stream().map(s -> s.equalsIgnoreCase(Constant.ID) ? MessageFormat.format(Constant.FieldAsAlias, Constant.MYSQL_UUID, Constant.ID) : s)
                        .map(s -> s.equalsIgnoreCase(Constant.STD_ID) ? MessageFormat.format(Constant.FieldAsAlias, Constant.ID, Constant.STD_ID) : s).collect(Collectors.joining(Constant.COMMA));

                mapper.insertRecordFromStandrad(maintain.getTableName(), maintain.getTableName() + Constant.RECORD, collect);
                connection.commit();
                mapper.insertMatintainDetailFromStandrad(DbUtils.quotedStr(maintain.getId()), maintain.getTableName() + Constant.RECORD);
                connection.commit();
                Integer standradCnt = mapper.tempDeal(maintain.getTableName() + Constant.RECORD, DbUtils.quotedStr(maintain.getId()));
                connection.commit();
                if (standradCnt != null) {
                    maintain.setDesp(maintain.getDesp() + MessageFormat.format("删除原数据{0}条", standradCnt));
                }else {
                    maintain.setDesp(maintain.getDesp() + "删除原数据");
                }
                maintain.setDesp(maintain.getDesp() + MessageFormat.format("删除原数据{0}条", standradCnt));
            }

            operateLog.setCreateTime(new Date())
                    .setMaintainId(maintain.getId())
                    .setId(DbUtils.getUUID())
                    .setMaintainId(menuMapping.getId())
                    .setOperate("open")
                    .setDesp(MessageFormat.format("上传数据{0}条", aLong));

            maintain.insert();
            connection.commit();
        }
        catch (Exception e) {
            e.printStackTrace();


        }
        finally {
            if (connection != null) {
                try {
                    connection.commit();
                } catch (SQLException e) {
                    e.printStackTrace();
                }
            }
        }
        return maintain;
    }

    //bak
    public Result selectListByPage(String tableName, List<String> fieldList, String whereSegment, Integer pageNo, Integer pageSize, String version) {
        Maintain maintainFromVersion = maintainService.getMaintainFromVersion(tableName, version);
        long changeCount = maintainService.countVersionOperatorByType(tableName + Constant.RECORD, whereSegment, maintainFromVersion.getId(), Operate.create);

 src/main/java/com/highdatas/mdm/service/impl/SysAssembleApiServiceImpl.java

New file
@@ -0,0 +1,20 @@
package com.highdatas.mdm.service.impl;

import com.highdatas.mdm.entity.SysAssembleApi;
import com.highdatas.mdm.mapper.SysAssembleApiMapper;
import com.highdatas.mdm.service.ISysAssembleApiService;
import com.baomidou.mybatisplus.service.impl.ServiceImpl;
import org.springframework.stereotype.Service;

/**
 * <p>
 *  服务实现类
 * </p>
 *
 * @author kimi
 * @since 2020-02-19
 */
@Service
public class SysAssembleApiServiceImpl extends ServiceImpl<SysAssembleApiMapper, SysAssembleApi> implements ISysAssembleApiService {

}

 src/main/java/com/highdatas/mdm/service/impl/SysAssembleCommonparamsServiceImpl.java

New file
@@ -0,0 +1,20 @@
package com.highdatas.mdm.service.impl;

import com.highdatas.mdm.entity.SysAssembleCommonparams;
import com.highdatas.mdm.mapper.SysAssembleCommonparamsMapper;
import com.highdatas.mdm.service.ISysAssembleCommonparamsService;
import com.baomidou.mybatisplus.service.impl.ServiceImpl;
import org.springframework.stereotype.Service;

/**
 * <p>
 *  服务实现类
 * </p>
 *
 * @author kimi
 * @since 2020-02-20
 */
@Service
public class SysAssembleCommonparamsServiceImpl extends ServiceImpl<SysAssembleCommonparamsMapper, SysAssembleCommonparams> implements ISysAssembleCommonparamsService {

}

 src/main/java/com/highdatas/mdm/service/impl/SysAssembleDbDetailServiceImpl.java

New file
@@ -0,0 +1,20 @@
package com.highdatas.mdm.service.impl;

import com.highdatas.mdm.entity.SysAssembleDbDetail;
import com.highdatas.mdm.mapper.SysAssembleDbDetailMapper;
import com.highdatas.mdm.service.ISysAssembleDbDetailService;
import com.baomidou.mybatisplus.service.impl.ServiceImpl;
import org.springframework.stereotype.Service;

/**
 * <p>
 *  服务实现类
 * </p>
 *
 * @author kimi
 * @since 2020-02-19
 */
@Service
public class SysAssembleDbDetailServiceImpl extends ServiceImpl<SysAssembleDbDetailMapper, SysAssembleDbDetail> implements ISysAssembleDbDetailService {

}

 src/main/java/com/highdatas/mdm/service/impl/SysAssembleDbFieldServiceImpl.java

New file
@@ -0,0 +1,20 @@
package com.highdatas.mdm.service.impl;

import com.highdatas.mdm.entity.SysAssembleDbField;
import com.highdatas.mdm.mapper.SysAssembleDbFieldMapper;
import com.highdatas.mdm.service.ISysAssembleDbFieldService;
import com.baomidou.mybatisplus.service.impl.ServiceImpl;
import org.springframework.stereotype.Service;

/**
 * <p>
 *  服务实现类
 * </p>
 *
 * @author kimi
 * @since 2020-02-20
 */
@Service
public class SysAssembleDbFieldServiceImpl extends ServiceImpl<SysAssembleDbFieldMapper, SysAssembleDbField> implements ISysAssembleDbFieldService {

}

 src/main/java/com/highdatas/mdm/service/impl/SysAssembleDbServiceImpl.java

New file
@@ -0,0 +1,220 @@
package com.highdatas.mdm.service.impl;

import com.alibaba.druid.pool.DruidDataSource;
import com.baomidou.mybatisplus.mapper.EntityWrapper;
import com.highdatas.mdm.entity.SysAssemble;
import com.highdatas.mdm.entity.SysAssembleDb;
import com.highdatas.mdm.entity.SysAssembleDbdriver;
import com.highdatas.mdm.mapper.SysAssembleDbMapper;
import com.highdatas.mdm.pojo.kettle.UnBigDataDataSourceInfo;
import com.highdatas.mdm.service.ISysAssembleDbService;
import com.baomidou.mybatisplus.service.impl.ServiceImpl;
import com.highdatas.mdm.service.ISysAssembleDbdriverService;
import com.highdatas.mdm.service.ISysAssembleService;
import com.highdatas.mdm.util.Constant;
import org.apache.commons.lang3.StringUtils;
import org.pentaho.di.core.KettleClientEnvironment;
import org.pentaho.di.core.database.DatabaseMeta;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;

import java.sql.*;
import java.text.MessageFormat;
import java.util.ArrayList;
import java.util.List;

/**
 * <p>
 *  服务实现类
 * </p>
 *
 * @author kimi
 * @since 2020-02-20
 */
@Service
public class SysAssembleDbServiceImpl extends ServiceImpl<SysAssembleDbMapper, SysAssembleDb> implements ISysAssembleDbService {

    @Autowired
    ISysAssembleDbdriverService dbdriverService;

    public Connection getConnection(String parentId) {
        SysAssembleDb assemble = this.selectById(parentId);
        if(assemble == null){
            return null;
        }

        Connection connection  = null;
        try {
            SysAssembleDbdriver sysAssembleDbdriver = dbdriverService.selectOne(new EntityWrapper<SysAssembleDbdriver>().eq(Constant.TYPE, assemble.getDatasourceType()));
            if (sysAssembleDbdriver != null) {
                String driver = sysAssembleDbdriver.getDriver();
                Class.forName(driver).newInstance();
            }

//            Boolean bigdata = assemble.getBigdata();
//            if (bigdata) {
//                //HBASE
//            }else  {
//                dataMetaOutput = new DatabaseMeta("Output", unBigDataDataSourceInfo.getDbType(), "Native", unBigDataDataSourceInfo.getDbHostName(), unBigDataDataSourceInfo.getDbName(), unBigDataDataSourceInfo.getDbPort(),
//                        unBigDataDataSourceInfo.getUsername(), unBigDataDataSourceInfo.getPassword());
//            }

            connection = DriverManager.getConnection(assemble.getDatasourceUrl(), assemble.getDatasourceUser(), assemble.getDatasourcePass());
            return connection;
        }  catch (Exception e) {
                e.printStackTrace();
                return null;
            }

    }

    @Override
    public List<String> getFieldsByTableName(String parentId, String tableName) {
        SysAssembleDb assemble = this.selectById(parentId);
        if(assemble == null){
            return null;
        }

        Connection connection = null;
        try {
            connection = getConnection(parentId);
            SysAssembleDbdriver sysAssembleDbdriver = dbdriverService.selectOne(new EntityWrapper<SysAssembleDbdriver>().eq(Constant.TYPE, assemble.getDatasourceType()));
            String fieldSql = "";
            if(sysAssembleDbdriver != null) {
                String firstTemplate = sysAssembleDbdriver.getFirstTemplate();
                fieldSql = MessageFormat.format(firstTemplate, MessageFormat.format(Constant.selectFieldSqlMysqlTemplate, tableName));
            }else {
                fieldSql =  MessageFormat.format(Constant.selectFieldSqlMysqlTemplate, tableName);
            }
            PreparedStatement preparedStatement = connection.prepareStatement(fieldSql);
            ResultSet resultSet = preparedStatement.executeQuery();
            ResultSetMetaData metaData = resultSet.getMetaData();
            int columnCount = metaData.getColumnCount();
            ArrayList<String> columList = new ArrayList<>();
            for (int i = 1; i <= columnCount; i++) {
                String columnLabel = metaData.getColumnLabel(i);
                columList.add(columnLabel);
            }
            return columList;
        }catch (Exception e) {
            e.printStackTrace();
            return null;
        }finally {
            if (connection != null) {
                try {
                    connection.close();
                } catch (SQLException e) {
                    e.printStackTrace();
                }
            }
        }

    }

    @Override
    public boolean testConnection(String parentId) {
        Connection connection = null;
        try{
            connection = getConnection(parentId);
            if (connection != null) {
                    return true;
            }
            return false;
        }
        catch (Exception e) {
            e.printStackTrace();
            return false;
        }
        finally {
            if (connection != null) {
                try {
                    connection.close();
                } catch (SQLException e) {
                    e.printStackTrace();
                }
            }
        }

    }

    @Override
    public boolean testSql(String parentId, String sql) {
        Connection connection = null;
        if (StringUtils.isEmpty(sql)) {
            return false;
        }
        try {

            connection = getConnection(parentId);
            if (connection == null) {
                return false;
            }

            PreparedStatement preparedStatement = connection.prepareStatement(sql);
            preparedStatement.executeQuery();

            return true;

        }catch (Exception e) {
            e.printStackTrace();
            return  false;
        }
        finally {
            if (connection != null){
                try {
                    connection.close();
                } catch (SQLException e) {
                    e.printStackTrace();
                    return false;
                }
            }
        }
    }

    @Override
    public List<String> getFieldsBySql(String parentId, String sql) {
        Connection connection = null;
        if (StringUtils.isEmpty(sql)) {
            return null;
        }
        try {

            connection = getConnection(parentId);
            if (connection == null) {
                return null;
            }
            SysAssembleDb assemble = this.selectById(parentId);
            SysAssembleDbdriver dbdriver = dbdriverService.selectOne(new EntityWrapper<SysAssembleDbdriver>().eq(Constant.TYPE, assemble.getDatasourceType()));
            String firstTemplate = dbdriver.getFirstTemplate();
            if (!StringUtils.isEmpty(firstTemplate)) {
                //有sql模板的 可以只取第一条
                sql = MessageFormat.format(firstTemplate, sql);
            }
            PreparedStatement preparedStatement = connection.prepareStatement(sql);
            ResultSet resultSet = preparedStatement.executeQuery();
            ResultSetMetaData metaData = resultSet.getMetaData();
            int columnCount = metaData.getColumnCount();
            ArrayList<String> columList = new ArrayList<>();
            for (int i = 1; i <= columnCount; i++) {
                String columnLabel = metaData.getColumnLabel(i);
                columList.add(columnLabel);
            }
            return columList;

        }catch (Exception e) {
            e.printStackTrace();
            return  null;
        }
        finally {
            if (connection != null){
                try {
                    connection.close();
                } catch (SQLException e) {
                    e.printStackTrace();
                }
            }
        }


    }
}

 src/main/java/com/highdatas/mdm/service/impl/SysAssembleDbTableServiceImpl.java

New file
@@ -0,0 +1,20 @@
package com.highdatas.mdm.service.impl;

import com.highdatas.mdm.entity.SysAssembleDbTable;
import com.highdatas.mdm.mapper.SysAssembleDbTableMapper;
import com.highdatas.mdm.service.ISysAssembleDbTableService;
import com.baomidou.mybatisplus.service.impl.ServiceImpl;
import org.springframework.stereotype.Service;

/**
 * <p>
 *  服务实现类
 * </p>
 *
 * @author kimi
 * @since 2020-02-20
 */
@Service
public class SysAssembleDbTableServiceImpl extends ServiceImpl<SysAssembleDbTableMapper, SysAssembleDbTable> implements ISysAssembleDbTableService {

}

 src/main/java/com/highdatas/mdm/service/impl/SysAssembleDbdriverServiceImpl.java

New file
@@ -0,0 +1,20 @@
package com.highdatas.mdm.service.impl;

import com.highdatas.mdm.entity.SysAssembleDbdriver;
import com.highdatas.mdm.mapper.SysAssembleDbdriverMapper;
import com.highdatas.mdm.service.ISysAssembleDbdriverService;
import com.baomidou.mybatisplus.service.impl.ServiceImpl;
import org.springframework.stereotype.Service;

/**
 * <p>
 *  服务实现类
 * </p>
 *
 * @author kimi
 * @since 2020-02-21
 */
@Service
public class SysAssembleDbdriverServiceImpl extends ServiceImpl<SysAssembleDbdriverMapper, SysAssembleDbdriver> implements ISysAssembleDbdriverService {

}

 src/main/java/com/highdatas/mdm/service/impl/SysAssembleDetailServiceImpl.java

New file
@@ -0,0 +1,20 @@
package com.highdatas.mdm.service.impl;

import com.highdatas.mdm.entity.SysAssembleDetail;
import com.highdatas.mdm.mapper.SysAssembleDetailMapper;
import com.highdatas.mdm.service.ISysAssembleDetailService;
import com.baomidou.mybatisplus.service.impl.ServiceImpl;
import org.springframework.stereotype.Service;

/**
 * <p>
 *  服务实现类
 * </p>
 *
 * @author kimi
 * @since 2020-02-15
 */
@Service
public class SysAssembleDetailServiceImpl extends ServiceImpl<SysAssembleDetailMapper, SysAssembleDetail> implements ISysAssembleDetailService {

}

 src/main/java/com/highdatas/mdm/service/impl/SysAssembleLogServiceImpl.java

New file
@@ -0,0 +1,20 @@
package com.highdatas.mdm.service.impl;

import com.highdatas.mdm.entity.SysAssembleLog;
import com.highdatas.mdm.mapper.SysAssembleLogMapper;
import com.highdatas.mdm.service.ISysAssembleLogService;
import com.baomidou.mybatisplus.service.impl.ServiceImpl;
import org.springframework.stereotype.Service;

/**
 * <p>
 *  服务实现类
 * </p>
 *
 * @author kimi
 * @since 2020-02-15
 */
@Service
public class SysAssembleLogServiceImpl extends ServiceImpl<SysAssembleLogMapper, SysAssembleLog> implements ISysAssembleLogService {

}

 src/main/java/com/highdatas/mdm/service/impl/SysAssembleParamsServiceImpl.java

New file
@@ -0,0 +1,62 @@
package com.highdatas.mdm.service.impl;

import com.baomidou.mybatisplus.mapper.EntityWrapper;
import com.highdatas.mdm.entity.SysAssemble;
import com.highdatas.mdm.entity.SysAssembleParams;
import com.highdatas.mdm.mapper.SysAssembleParamsMapper;
import com.highdatas.mdm.service.ISysAssembleParamsService;
import com.baomidou.mybatisplus.service.impl.ServiceImpl;
import com.highdatas.mdm.service.ISysAssembleService;
import com.highdatas.mdm.util.Constant;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.scheduling.support.CronSequenceGenerator;
import org.springframework.stereotype.Service;

import java.text.SimpleDateFormat;
import java.util.Date;

/**
 * <p>
 *  服务实现类
 * </p>
 *
 * @author kimi
 * @since 2020-02-20
 */
@Slf4j
@Service
public class SysAssembleParamsServiceImpl extends ServiceImpl<SysAssembleParamsMapper, SysAssembleParams> implements ISysAssembleParamsService {
    @Autowired
    ISysAssembleParamsService paramsService;
    @Autowired
    ISysAssembleService assembleService;


    @Override
    public boolean updateCornVal(String assembleId, String cron) {
        SysAssemble assemble = assembleService.selectById(assembleId);
        if (assemble == null ) {
            return false;
        }
        SysAssembleParams sysAssembleParams = paramsService.selectOne(new EntityWrapper<SysAssembleParams>().eq("parent_id", assembleId).eq("code", Constant.Cron));
        if (sysAssembleParams == null) {
            return false;
        }
        if (StringUtils.isEmpty(cron)) {
            return false;
        }
        if (!CronSequenceGenerator.isValidExpression(cron)) {
            return false;
        }
        CronSequenceGenerator cronSequenceGenerator = new CronSequenceGenerator(cron);
        Date currentTime = new Date();
        Date nextTimePoint = cronSequenceGenerator.next(currentTime);
        SimpleDateFormat simpleDateFormat = new SimpleDateFormat("yyyy-MM-dd");

        sysAssembleParams.setVal(simpleDateFormat.format(nextTimePoint));
        boolean update = sysAssembleParams.updateById();
        return update;
    }
}

 src/main/java/com/highdatas/mdm/service/impl/SysAssembleRuleServiceImpl.java

New file
@@ -0,0 +1,20 @@
package com.highdatas.mdm.service.impl;

import com.highdatas.mdm.entity.SysAssembleRule;
import com.highdatas.mdm.mapper.SysAssembleRuleMapper;
import com.highdatas.mdm.service.ISysAssembleRuleService;
import com.baomidou.mybatisplus.service.impl.ServiceImpl;
import org.springframework.stereotype.Service;

/**
 * <p>
 *  服务实现类
 * </p>
 *
 * @author kimi
 * @since 2020-02-21
 */
@Service
public class SysAssembleRuleServiceImpl extends ServiceImpl<SysAssembleRuleMapper, SysAssembleRule> implements ISysAssembleRuleService {

}

 src/main/java/com/highdatas/mdm/service/impl/SysAssembleServiceImpl.java

New file
@@ -0,0 +1,780 @@
package com.highdatas.mdm.service.impl;

import com.baomidou.mybatisplus.mapper.EntityWrapper;
import com.highdatas.mdm.entity.*;
import com.highdatas.mdm.mapper.SysAssembleMapper;
import com.highdatas.mdm.mapper.TableInfoMapper;
import com.highdatas.mdm.pojo.*;
import com.highdatas.mdm.pojo.kettle.BigDataDataSourceInfo;
import com.highdatas.mdm.pojo.kettle.DataSourceInfo;
import com.highdatas.mdm.pojo.kettle.UnBigDataDataSourceInfo;
import com.highdatas.mdm.service.*;
import com.baomidou.mybatisplus.service.impl.ServiceImpl;
import com.highdatas.mdm.util.Constant;
import com.highdatas.mdm.util.ContentBuilder;
import com.highdatas.mdm.util.DbUtils;
import lombok.Data;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.apache.ibatis.session.SqlSession;
import org.apache.regexp.RE;
import org.mybatis.spring.SqlSessionTemplate;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;

import javax.annotation.Resource;
import javax.servlet.http.HttpSession;
import java.io.ByteArrayInputStream;
import java.io.InputStream;
import java.io.UnsupportedEncodingException;
import java.sql.*;
import java.text.MessageFormat;
import java.util.*;
import java.util.Date;
import java.util.stream.Collectors;

/**
 * <p>
 *  服务实现类
 * </p>
 *
 * @author kimi
 * @since 2020-02-20
 */
@Slf4j
@Service
public class SysAssembleServiceImpl extends ServiceImpl<SysAssembleMapper, SysAssemble> implements ISysAssembleService {
    @Autowired
    ISysAssembleRuleService ruleService;
    @Autowired
    ISysAssembleDbService dbService;
    @Autowired
    ISysAssembleApiService apiService;
    @Autowired
    ISysAssembleDbTableService tableService;
    @Autowired
    ISysAssembleDbFieldService fieldService;
    @Autowired
    ISysAssembleParamsService paramsService;
    @Autowired
    ISysAssembleLogService logService;
    @Autowired
    IMenuMappingService menuMappingService;
    @Autowired
    ActivitiService activitiService;
    @Autowired
    IMaintainService maintainService;
    @Autowired
    IFlowsService flowsService;
    @Resource
    SqlSessionTemplate sqlSessionTemplate;
    @Autowired
    TableInfoMapper tableInfoMapper;
    @Autowired
    UnBigDataDataSourceInfo unBigDataDataSourceInfo;
    @Autowired
    BigDataDataSourceInfo bigDataDataSourceInfo;
    @Autowired
    MasterDataService masterDataService;


    @Override
    @Transactional(rollbackFor=Exception.class)
    public Result run(String id, HttpSession session) {
        if (StringUtils.isEmpty(id)) {
            return Result.error(CodeMsg.ERROR_PARAMS_NOT_MATHED);
        }

        SysAssemble assemble = selectById(id);
        if (assemble == null) {
            return Result.error(CodeMsg.ERROR_PARAMS_NOT_MATHED);
        }

        if (!assemble.getStatus().equals(SysAssembleStatus.working)) {
            return  Result.error(new CodeMsg(6009,"当前任务不在工作中,状态:" + assemble.getStatus()));
        }

        String flowId = assemble.getFlowId();
        if (!StringUtils.isEmpty(flowId)){
            Flows flows = flowsService.selectById(flowId);
            if (flows != null) {
                ActivitiStatus status = flows.getStatus();
                if (!status.equals(ActivitiStatus.close) && !status.equals(ActivitiStatus.open)) {
                    return  Result.error(new CodeMsg(6009,"当前有流程正在运行,暂时无法汇集下次数据:" + status.toString()));
                }
            }
        }
        assemble.setPreTime(new Date());
        assemble.setPreStatus(SysAssembleRunStatus.working);
        assemble.updateById();

        Boolean bigData = assemble.getBigdata();

        //1 load  from db
        try{
            Result result = runByDb(assemble);
            if (!result.getSuccess()) {
               assemble.setPreStatus(SysAssembleRunStatus.fail);
               assemble.updateById();
                return result;
            }
            //2 load  from api
            result = runByApi(id);
            if (!result.getSuccess()) {
                assemble.setPreStatus(SysAssembleRunStatus.fail);
                assemble.updateById();
                return result;
            }
            //3check temp table;
            result = checkTempTable(assemble);
            if (!result.getSuccess()) {
                assemble.setPreStatus(SysAssembleRunStatus.fail);
                assemble.setPreMsg("检测主数据临时表出现错误");
                assemble.updateById();
                return result;
            }

            //4 purge data
            String purgeSql = assemble.getPurgeSql();
            purgeSql = DbUtils.replaceEscape(purgeSql);
            result = purgeData(purgeSql, assemble.getId(), bigData);
            if (!result.getSuccess()) {
                assemble.setPreStatus(SysAssembleRunStatus.fail);
                assemble.setPreMsg("运行清洗sql出现错误: "+purgeSql );
                assemble.updateById();
                return result;
            }

            //5 check temp data
            result = checkTempData(id, assemble.getCheckType(), assemble.getCheckFields());

            if (!result.getSuccess()) {
                assemble.setPreStatus(SysAssembleRunStatus.fail);
                assemble.setPreMsg("质量检验环节出现错误" );
                assemble.updateById();
                return result;
            }
            //6 temp 2 record

            result = temp2record(assemble);
            if (!result.getSuccess()) {
                assemble.setPreStatus(SysAssembleRunStatus.fail);
                assemble.setPreMsg("数据搬运至主数据记录环节出现错误" );
                assemble.updateById();
                return result;
            }

            //7 record 链接 版本
            result = linkMaintain(assemble, session);
            if (!result.getSuccess()) {
                assemble.setPreStatus(SysAssembleRunStatus.fail);
                assemble.setPreMsg("联接版本出现错误" );
                assemble.updateById();
                return result;
            }

            //todo 添加事务  log记录
            //8 更新 参数
            result = updateParams(assemble);
            if (!result.getSuccess()) {
                assemble.setPreStatus(SysAssembleRunStatus.fail);
                assemble.setPreMsg("更新变量环节出现错误" );
            }else {
                assemble.setPreMsg("汇集成功");
                assemble.setPreStatus(SysAssembleRunStatus.success);
            }
            assemble.updateById();
            return result;
        }
        catch (Exception e) {
            assemble.setPreMsg("汇集任务出现错误");
            assemble.setPreStatus(SysAssembleRunStatus.fail).updateById();
            e.printStackTrace();

            return Result.error(new CodeMsg(6009, e.getMessage()));
        }
    }

    private Result linkMaintain(SysAssemble assemble, HttpSession session) {
        try{
            String menuId = assemble.getMenuId();
            MenuMapping menuMapping = menuMappingService.selectOne(new EntityWrapper<MenuMapping>().eq("menu_id", menuId));
            String tableName = menuMapping.getTableName();
            Maintain maintain = masterDataService.uploadedData(tableName, assemble.getUpdateType(), assemble.getUserId());

            Result result = temp2recordUpdate(assemble, maintain);
            if (!result.getSuccess()) {
                assemble.setPreStatus(SysAssembleRunStatus.fail);
                assemble.setPreMsg("搬运更新字段出现错误" );
                assemble.updateById();
                return result;
            }
            Boolean audit = menuMapping.getAudit();
            if (audit == null){
                //默认需要审批
                audit = true;
            }
            if (audit) {
                Flows flows = activitiService.start("process", session, maintain.getId(),  ActivitiBusinessType.maintain);
                assemble.setFlowId(flows.getId()).updateById();
                maintain.setFlowId(flows.getId());
                maintain.setDesp("启动汇集流程");
                maintain.updateById();
            }else  {
                //直接运行
                Flows flows = new Flows().setStatus(ActivitiStatus.open).setBusinessId(maintain.getId()).setId(DbUtils.getUUID()).setCreateTime(new Date());
                flows.insert();
                maintain.setFlowId(flows.getId());
                maintain.updateById();
                maintainService.dealFlow(maintain.getId(), ActivitiStatus.open);
            }
            return Result.success(null);
        }catch (Exception e) {
            e.printStackTrace();
            return Result.error(new CodeMsg(6009, e.getMessage()));
        }

    }

    private Result updateParams(SysAssemble assemble) {
        Connection conn = null;
        try{
            String id = assemble.getId();
            Boolean bigdata = assemble.getBigdata();

            DataSourceInfo dataSourceInfo;
            if (bigdata) {
                dataSourceInfo = bigDataDataSourceInfo;
            }else {
                dataSourceInfo = unBigDataDataSourceInfo;
            }
            conn = dataSourceInfo.conn();
            List<SysAssembleParams> paramsList = paramsService.selectList(new EntityWrapper<SysAssembleParams>().eq("parent_id", id));

            for (SysAssembleParams sysAssembleParams : paramsList) {
                String updateSql = sysAssembleParams.getUpdateSql();
                if (StringUtils.isEmpty(updateSql)){
                    continue;
                }

                PreparedStatement preparedStatement = conn.prepareStatement(updateSql, ResultSet.TYPE_SCROLL_INSENSITIVE,ResultSet.CONCUR_READ_ONLY);
                ResultSet resultSet = preparedStatement.executeQuery();
                //默认只有一个结果
                if (resultSet.first()) {
                    String val = resultSet.getString(1);
                    sysAssembleParams.setVal(val);
                    sysAssembleParams.updateById();
                }

            }
            return Result.success(null) ;
        }
        catch (Exception e) {

            return Result.error(new CodeMsg(6006, e.getMessage())) ;
        }
        finally {
            if (conn != null) {
                try {
                    conn.close();
                } catch (SQLException e) {
                    e.printStackTrace();
                    return Result.error(new CodeMsg(6006, e.getMessage())) ;
                }
            }
        }

    }

    private Result checkTempTable(SysAssemble assemble) {
        try {
            String menuId = assemble.getMenuId();
            MenuMapping menuMapping = menuMappingService.selectOne(new EntityWrapper<MenuMapping>().eq("menu_id", menuId));
            String tableName = menuMapping.getTableName();
            String tempTableName = Constant.Temp + tableName;
            Boolean bigdata = assemble.getBigdata();
            DataSourceInfo dataSourceInfo;
            if (bigdata) {
                dataSourceInfo = bigDataDataSourceInfo;
            }else {
                dataSourceInfo = unBigDataDataSourceInfo;
            }
            boolean exists = dataSourceInfo.checkTableExists(tempTableName);
            if (!exists){
                List<String> fieldsFromTable = dataSourceInfo.getFieldsFromTable(tableName);
                if (fieldsFromTable == null) {
                    return Result.error(new CodeMsg(6003, "查询正式表的字段:" + tableName));
                }
                dataSourceInfo.createTable(tempTableName, fieldsFromTable);
            }
            //drop temp data
            dataSourceInfo.truncateData(tempTableName);
            return Result.success(null);
        }catch (Exception e) {
            e.printStackTrace();
            return Result.error(new CodeMsg(6004, e.getMessage()));
        }
    }
    private Result temp2recordUpdate(SysAssemble assemble, Maintain maintain) {
        Connection conn = null;
        try {
            String menuId = assemble.getMenuId();
            Boolean bigdata = assemble.getBigdata();
            MenuMapping menuMapping = menuMappingService.selectOne(new EntityWrapper<MenuMapping>().eq("menu_id", menuId));
            String tableName = menuMapping.getTableName();
            String tempTableName = Constant.Temp + tableName;
            String recordTableName =  tableName + Constant.RECORD;
            DataSourceInfo dataSourceInfo;
            String updateSql = null;
            if (bigdata) {
                dataSourceInfo = bigDataDataSourceInfo;
            }else {
                dataSourceInfo = unBigDataDataSourceInfo;
                updateSql = Constant.MYSQLJoinUpdateSql;
            }
            conn = dataSourceInfo.conn();
            SysAssembleUpdateType updateType = assemble.getUpdateType();
            List<String> fieldsFromTable = dataSourceInfo.getFieldsFromTable(tableName);
            if (fieldsFromTable.contains(Constant.ID)) {
                fieldsFromTable.remove(Constant.ID);
            }

            List<String> fieldFromRecordTable = new ArrayList<>(fieldsFromTable);
            fieldFromRecordTable.add(Constant.ID);
            fieldFromRecordTable.add(Constant.STD_ID);
            fieldFromRecordTable.add(Constant.DEAL);

            String recordFieldStr = StringUtils.join(fieldFromRecordTable, Constant.COMMA);
            int cnt = 0;
            if(updateType.equals(SysAssembleUpdateType.Increment)) {
                String updateFields = assemble.getUpdateFields();
                String[] split = updateFields.split(Constant.SEMICOLON);
                ArrayList<String> unionCodeFields = new ArrayList<String>(Arrays.asList(split));
                String joinStr = getJoinFieldParse(unionCodeFields);
                List<String> otherFields = fieldsFromTable.stream().filter(s -> !unionCodeFields.contains(s)).filter(s -> !s.equalsIgnoreCase(Constant.ID)).collect(Collectors.toList());
                if (otherFields.size() != 0) {
                    String insertFieldStr = fieldsFromTable.stream()
                            .map(s -> MessageFormat.format(Constant.Alias,Constant.T1,s))
                            .collect(Collectors.joining(Constant.COMMA));

                    String updatedSql = MessageFormat.format(updateSql, recordTableName, recordFieldStr,insertFieldStr, tempTableName, tableName, joinStr);
                    PreparedStatement updatedPreparedStatement = conn.prepareStatement(updatedSql);
                    int updateCnt = updatedPreparedStatement.executeUpdate();
                    cnt = updateCnt;
                }
            }

            tableInfoMapper.insertMatintainDetailFromTemp(DbUtils.quotedStr(maintain.getId()), maintain.getTableName() + Constant.RECORD , DbUtils.quotedStr(Operate.update.toString()));

            //tableInfoMapper.updateStdId( maintain.getTableName() + Constant.RECORD);

            tableInfoMapper.tempDeal(maintain.getTableName() + Constant.RECORD, DbUtils.quotedStr(maintain.getId()));

            assemble.setPreCnt(assemble.getPreCnt() + cnt);
            assemble.updateById();
            return Result.success(null);
        }
        catch (Exception e) {
            e.printStackTrace();
            return Result.error(new CodeMsg(6005, e.getMessage()));
        }
        finally {
            if (conn != null) {
                try {
                    conn.close();
                } catch (SQLException e) {
                    e.printStackTrace();
                    return Result.error(new CodeMsg(6005, e.getMessage()));
                }
            }
        }
    }
    private Result temp2record(SysAssemble assemble) {
        Connection conn = null;
        try {
            String menuId = assemble.getMenuId();
            Boolean bigdata = assemble.getBigdata();
            MenuMapping menuMapping = menuMappingService.selectOne(new EntityWrapper<MenuMapping>().eq("menu_id", menuId));
            String tableName = menuMapping.getTableName();
            String tempTableName = Constant.Temp + tableName;
            String recordTableName =  tableName + Constant.RECORD;
            DataSourceInfo dataSourceInfo;
            String transSql;
            String updateSql = null;
            String insertSql = null;
            if (bigdata) {
                dataSourceInfo = bigDataDataSourceInfo;
                transSql = Constant.Temp2RecordHbaseTemplate;
            }else {
                dataSourceInfo = unBigDataDataSourceInfo;
                transSql = Constant.Temp2RecordMySQLTemplate;
                updateSql = Constant.MYSQLJoinUpdateSql;
                insertSql = Constant.MYSQLJoinAddSql;
            }
            conn = dataSourceInfo.conn();
            SysAssembleUpdateType updateType = assemble.getUpdateType();
            List<String> fieldsFromTable = dataSourceInfo.getFieldsFromTable(tableName);
            if (fieldsFromTable.contains(Constant.ID)) {
                fieldsFromTable.remove(Constant.ID);
            }
            List<String> fieldFromRecordTable = new ArrayList<>(fieldsFromTable);
            fieldFromRecordTable.add(Constant.ID);
            fieldFromRecordTable.add(Constant.STD_ID);
            fieldFromRecordTable.add(Constant.DEAL);

            String recordFieldStr = StringUtils.join(fieldFromRecordTable, Constant.COMMA);
            String fieldStr = fieldsFromTable.stream()
                    .collect(Collectors.joining(Constant.COMMA));

            int cnt = 0;
            if (updateType.equals(SysAssembleUpdateType.All)) {
                String sql = MessageFormat.format(transSql, recordTableName, recordFieldStr, fieldStr, tempTableName);
                PreparedStatement preparedStatement = conn.prepareStatement(sql);
                cnt = preparedStatement.executeUpdate();
            }else if(updateType.equals(SysAssembleUpdateType.Increment)) {
                String updateFields = assemble.getUpdateFields();
                String[] split = updateFields.split(Constant.SEMICOLON);
                ArrayList<String> unionCodeFields = new ArrayList<String>(Arrays.asList(split));
                String joinStr = getJoinFieldParse(unionCodeFields);

                // insert

                String insertFieldStr = fieldsFromTable.stream()
                        .map(s -> MessageFormat.format(Constant.Alias,Constant.T1,s))
                        .collect(Collectors.joining(Constant.COMMA));

                String insetSql = MessageFormat.format(insertSql, recordTableName, recordFieldStr,insertFieldStr, tempTableName, tableName, joinStr);
                PreparedStatement preparedStatement = conn.prepareStatement(insetSql);
                int insertCnt = preparedStatement.executeUpdate();
                cnt += insertCnt;
            }
            assemble.setPreCnt(cnt);
            assemble.updateById();
            return Result.success(null);
        }
        catch (Exception e) {
            e.printStackTrace();
            return Result.error(new CodeMsg(6005, e.getMessage()));
        }
        finally {
            if (conn != null) {
                try {
                    conn.close();
                } catch (SQLException e) {
                    e.printStackTrace();
                    return Result.error(new CodeMsg(6005, e.getMessage()));
                }
            }
        }
    }

    private String getOthetFieldParse(List<String> unionCodeFields) {
        ContentBuilder builder = new ContentBuilder(Constant.AND);
        for (String s : unionCodeFields) {
            builder.append(MessageFormat.format(Constant.MYSQLJoinParse, s));
        }
        return builder.toString();
    }

    private String getJoinFieldParse(ArrayList<String> updateFields) {

        ContentBuilder builder = new ContentBuilder(Constant.AND);
        for (String s : updateFields) {
            builder.append(MessageFormat.format(Constant.MYSQLJoinParse, s));
        }
        return builder.toString();
    }

    private Result checkTempData(String s,SysAssembleCheckType checkType, String id) {
        //TODO
        return Result.success(null);
    }

    private Result purgeData(String purgeSql, String assembleId, boolean bigData) {
        Connection connection = null;
        try {
            DataSourceInfo dataSourceInfo = null;
            if (bigData) {
                dataSourceInfo = bigDataDataSourceInfo;
            }else {
                dataSourceInfo = unBigDataDataSourceInfo;
            }
            connection = dataSourceInfo.conn();

            List<SysAssembleParams> paramsList = paramsService.selectList(new EntityWrapper<SysAssembleParams>().eq("parent_id", assembleId));
            HashMap<String, String> paramsMap = new HashMap<>();
            Set<String> matcher = DbUtils.matcher(purgeSql);
            for (String s : matcher) {
                List<SysAssembleParams> collect = paramsList.stream().filter(sysAssembleParams -> sysAssembleParams.getCode().equalsIgnoreCase(s)).collect(Collectors.toList());
                if (collect.size() == 0) {
                    return Result.error(new CodeMsg(6006, "未匹配到参数:" + s));
                }else if (collect.size() > 1) {
                    return Result.error(new CodeMsg(6006, "匹配到多个参数:" + s));
                }else {
                    SysAssembleParams sysAssembleParams = collect.get(0);
                    String val = sysAssembleParams.getVal();
                    if (StringUtils.isEmpty(val)) {
                        String initSql = sysAssembleParams.getInitSql();
                        if (StringUtils.isEmpty(initSql)) {
                            return Result.error(new CodeMsg(6006, "初始化参数:" + s + "失败"));
                        }
                        PreparedStatement preparedStatement = connection.prepareStatement(initSql);
                        ResultSet resultSet = preparedStatement.executeQuery();
                        //默认只有一个结果
                        val = resultSet.getString(1);
                    }
                    paramsMap.put(s, val);
                }
            }
            for (String key : paramsMap.keySet()) {
                String val = paramsMap.get(key);
                String preParams = MessageFormat.format(Constant.ParamsShell, key);
                purgeSql = purgeSql.replace(preParams, val);
            }

            PreparedStatement preparedStatement = connection.prepareStatement(purgeSql);
            preparedStatement.execute();
            return Result.success(null);

        } catch (SQLException e) {
            e.printStackTrace();
            return Result.error(new CodeMsg(6004, "运行清洗sql出错," + e.getSQLState()));
        } finally {
            if (connection != null) {
                try {
                    connection.close();
                } catch (SQLException e) {
                    e.printStackTrace();
                    return Result.error(new CodeMsg(6003, "本地connection 无法关闭"));
                }
            }
        }
    }

    private Result runByApi(String id) {
        //TODO
        return Result.success(null);
    }

    private Result runByDb(SysAssemble assemble) {
        String id = assemble.getId();
        Boolean bigdata = assemble.getBigdata();

        List<SysAssembleDb> dbList = dbService.selectList(new EntityWrapper<SysAssembleDb>().eq(Constant.PARENT_ID, id).eq(Constant.Active, true));
        for (SysAssembleDb sysAssembleDb : dbList) {
            String dbId = sysAssembleDb.getId();
            Result result = loadOneDb(sysAssembleDb, bigdata, id);
            if (!result.getSuccess()) {
                assemble.setPreMsg("导入源数据错误:" + sysAssembleDb.getDatabaseName()).updateById();
                return result;
            }
        }
        return Result.success(null);
    }

    private Result loadOneDb(SysAssembleDb sysAssembleDb, boolean bigData, String assembleId) {
        String dbId = sysAssembleDb.getId();
        Connection conn = dbService.getConnection(dbId);
        if (conn == null) {
            return Result.error(new CodeMsg(6002, MessageFormat.format("未能连接到源 id: {0}", dbId)));
        }
        try {
            List<SysAssembleDbTable> tableList = tableService.selectList(new EntityWrapper<SysAssembleDbTable>().eq(Constant.PARENT_ID, dbId).eq(Constant.Active, true));
            for (SysAssembleDbTable dbTable : tableList) {
                String tableId = dbTable.getId();
                String tempTableName = dbTable.getTempTableName();

                List<SysAssembleDbField> fieldList = fieldService.selectList(new EntityWrapper<SysAssembleDbField>().eq(Constant.PARENT_ID, tableId));
                String fields = fieldList.stream().map(SysAssembleDbField::getField).collect(Collectors.joining(Constant.COMMA));
                SysAssembleTableType type = dbTable.getType();
                String tableName = null;
                String assembleTempTableName = tempTableName;
                if (type.equals(SysAssembleTableType.sql)) {
                    tableName = dbTable.getSql();
                } else if (type.equals(SysAssembleTableType.table)) {
                    tableName = dbTable.getTableName();
                }
                //TODO assembleTempTableName 可能会超长 64字节 后续修改
                DataSourceInfo dataSourceInfo = null;
                if (!bigData) {
                    dataSourceInfo = unBigDataDataSourceInfo;
                }else {
                    dataSourceInfo = bigDataDataSourceInfo;
                }

                // drop temp data
                dataSourceInfo.truncateData(tempTableName);

                fixAssembleTempTable(dataSourceInfo, assembleTempTableName, fieldList, dbId);

                //checkAssembleTempTableExists(assembleTempTableName);
                //TODO 未分页
                String filter = dbTable.getFilter();
                Set<String> matcher = DbUtils.matcher(filter);
                for (String code : matcher) {
                    SysAssembleParams sysAssembleParams = paramsService.selectOne(new EntityWrapper<SysAssembleParams>().eq(Constant.PARENT_ID, assembleId).eq(Constant.Code, code));
                    if (sysAssembleParams == null){
                        return Result.error(new CodeMsg(6009, assembleId + "有变量未匹配到:"+ code));
                    }
                    String val = sysAssembleParams.getVal();
                    if (StringUtils.isEmpty(val)) {
                        return Result.error(new CodeMsg(6009, assembleId + "有变量未获取到值:"+ code));
                    }
                    val = DbUtils.quotedStr(val);
                    filter = filter.replace(DbUtils.assemblParam(code), val);
                }
                if (StringUtils.isEmpty(filter)) {
                    filter = Constant.WHERE_DEFAULT;
                }
                String runSqlTemplate = null;
                if (type.equals(SysAssembleTableType.table)){
                    runSqlTemplate = Constant.selectFieldTableTemplate;
                }else if(type.equals(SysAssembleTableType.sql)){
                    runSqlTemplate = Constant.selectFieldSqlTemplate;
                }
                String sql = MessageFormat.format(runSqlTemplate, fields, tableName, filter);
                PreparedStatement ps = conn.prepareStatement(sql, ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY);
                ps.setFetchSize(Integer.MIN_VALUE);
                ps.setFetchDirection(ResultSet.FETCH_REVERSE);
                ResultSet resultSet = ps.executeQuery();
                ResultSetMetaData metaData = resultSet.getMetaData();
                int columnCount = metaData.getColumnCount();

                String insertMySqlSql = assembleSql(unBigDataDataSourceInfo.getDbName(), assembleTempTableName, fields);
                int cnt = 0;
                StringBuilder builder = new StringBuilder();
                while (resultSet.next()) {

                    for (int i = 1; i <= columnCount; i++) {
                        if (i == columnCount) {
                            builderEnd(builder, resultSet.getObject(i));
                        }else {
                            builderAppend(builder, resultSet.getObject(i));
                        }

                    }
                    cnt++;
                    if (cnt == 5000) {
                        runOneBatch(bigData, insertMySqlSql, builder);
                    }
                }
                if (builder.length() > 0) {
                    runOneBatch(bigData, insertMySqlSql, builder);
                }
            }
            return Result.success(null);
        }
        catch (Exception e) {
            e.printStackTrace();
            return Result.error(new CodeMsg(6004, e.getMessage()));
        }finally {
            if (conn != null) {
                try {
                    conn.close();
                } catch (SQLException e) {
                    e.printStackTrace();
                }
            }
        }

    }

    private void runOneBatch(boolean bigData, String insertMySqlSql, StringBuilder builder) throws UnsupportedEncodingException, SQLException {
        byte[] bytes = builder.toString().getBytes("UTF-8");
        if (bytes.length > 0) {
            InputStream stream = new ByteArrayInputStream(bytes);
            bulkLoadFromInputStream(insertMySqlSql, stream, bigData);
        }
        //清空bulider
        builder.delete(0, builder.length());
    }

    private void fixAssembleTempTable(DataSourceInfo dataSourceInfo, String assembleTempTableName, List<SysAssembleDbField> dbFieldList, String dbId) {
        boolean exists = dataSourceInfo.checkTableExists(assembleTempTableName);
        List<String> fieldList = dbFieldList.stream().map(SysAssembleDbField::getField).collect(Collectors.toList());
        if (!exists) {
            dataSourceInfo.createTable(assembleTempTableName, fieldList);
        }else  {
            //check 字段是否相同
            boolean checked = dataSourceInfo.checkFieldList(assembleTempTableName, fieldList);
            if (!checked){
                assembleTempTableName = reSetTableName(assembleTempTableName, dbId);
                dataSourceInfo.createTable(assembleTempTableName, fieldList);

            }
        }
    }

    private String reSetTableName(String assembleTempTableName, String tableId) {
        //TODO  Db
        SysAssembleDbTable dbTable = tableService.selectById(tableId);
        String suffix = null;
        suffix = DbUtils.getUUID(16);
        String tempTableName = null;
        if (dbTable.getType().equals(SysAssembleTableType.table)) {
            tempTableName = Constant.AssembleTempTable + dbTable.getTableName() + suffix;
        }else {
            tempTableName = Constant.AssembleTempSql+ DbUtils.getUUID(5)+ suffix;
        }

        dbTable.setTempTableName(tempTableName).updateById();
        return tempTableName;
    }

    public void builderEnd(StringBuilder builder, Object object) {
        builder.append(object);
        builder.append("\n");
    }


    public void builderAppend(StringBuilder builder, Object object) {
        builder.append(object);
        builder.append("\t");
    }


    public String assembleSql(String dataBaseName, String tableName, String fields) {
        String sql = "LOAD DATA LOCAL INFILE 'sql.csv' IGNORE INTO TABLE " + dataBaseName + "." + tableName + "(" + fields + ")";
        return sql;
    }

    private int bulkLoadFromInputStream(String sql, InputStream dataStream, boolean bigData) {
        if (null == dataStream) {
            log.error("输入流为NULL，没有数据导入。");
            return 0;
        }
        Connection conn = null;
        try {
            DataSourceInfo dataSourceInfo = null;
            if (bigData) {
                dataSourceInfo = bigDataDataSourceInfo;
            }else {
                dataSourceInfo = unBigDataDataSourceInfo;
            }

            conn = dataSourceInfo.conn();
            PreparedStatement statement = conn.prepareStatement(sql);
            int result = 0;
            if (statement.isWrapperFor(com.mysql.jdbc.Statement.class)) {
                com.mysql.jdbc.PreparedStatement mysqlStatement = statement.unwrap(com.mysql.jdbc.PreparedStatement.class);
                mysqlStatement.setLocalInfileInputStream(dataStream);
                result = mysqlStatement.executeUpdate();
            }
            return result;
        }
        catch (Exception e) {
            e.printStackTrace();
        }
        finally {
            if (conn != null) {
                try {
                    conn.close();
                } catch (SQLException e) {
                    e.printStackTrace();
                }
            }
        }
        return 0;
    }
}

 src/main/java/com/highdatas/mdm/service/impl/SysDbtypeServiceImpl.java

New file
@@ -0,0 +1,20 @@
package com.highdatas.mdm.service.impl;

import com.highdatas.mdm.entity.SysDbtype;
import com.highdatas.mdm.mapper.SysDbtypeMapper;
import com.highdatas.mdm.service.ISysDbtypeService;
import com.baomidou.mybatisplus.service.impl.ServiceImpl;
import org.springframework.stereotype.Service;

/**
 * <p>
 *  服务实现类
 * </p>
 *
 * @author kimi
 * @since 2020-02-15
 */
@Service
public class SysDbtypeServiceImpl extends ServiceImpl<SysDbtypeMapper, SysDbtype> implements ISysDbtypeService {

}

 src/main/java/com/highdatas/mdm/service/impl/SysMenuServiceImpl.java

@@ -4,7 +4,13 @@
import com.highdatas.mdm.mapper.SysMenuMapper;
import com.highdatas.mdm.service.ISysMenuService;
import com.baomidou.mybatisplus.service.impl.ServiceImpl;
import org.apache.commons.lang3.StringUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;

import java.util.HashSet;
import java.util.LinkedHashSet;
import java.util.Set;

/**
 * <p>
@@ -16,5 +22,28 @@
 */
@Service
public class SysMenuServiceImpl extends ServiceImpl<SysMenuMapper, SysMenu> implements ISysMenuService {
    @Autowired
    ISysMenuService menuService;

    @Override
    public LinkedHashSet<String> getByParentId(LinkedHashSet<String> parentIdSet) {
        if (parentIdSet.size() == 0) {
            return null;
        }
        LinkedHashSet<String> subSet = new LinkedHashSet<>();
        for (String s : parentIdSet) {
            SysMenu sysMenu = menuService.selectById(s);
            String parentId = sysMenu.getParentId();
            if (StringUtils.isEmpty(parentId)) {
                continue;
            }
            subSet.add(parentId);
        }
        LinkedHashSet<String> byParentId = getByParentId(subSet);
        if (byParentId != null) {
            parentIdSet.addAll(byParentId);
        }

        return parentIdSet;
    }
}

 src/main/java/com/highdatas/mdm/util/Constant.java

@@ -15,6 +15,8 @@
    public static final String LIKE = " like ";
    public static final String EMPTY = " ";
    public static final String SEMICOLON = ";";
    public static final String colon = ":";
    public static final String UnderLine = "_";
    public static final String WHERE_SEGMENT = "whereSegment";
    public static final String VERSION = "version";
    public static final String VERSION_Default = "V1.0";
@@ -22,11 +24,13 @@
    public static final String TOTAL = "total";
    public static final String FIELDS = "fields";
    public static final String DEAL = "deal";
    public static final String PARENT_ID = "parent_id";

    public static final String RECORD = "_record";
    public static final String MAX = "max";

    String Dot = "\\.";
    String Default_Patter = "(?<=@\\{)(.+?)(?=\\})";
    String addDataType = "addDataType";
    String addAll = "addAll";

@@ -40,6 +44,7 @@
    String StructureMenu = "StructureMenu";
    String tableName = "tableName";
    String Alias = "{0}.{1}";
    String FieldAsAlias = "{0} AS {1}";
    String caseWhen = " case when a2.{0} != null then a2.{0} else a1.{0} end  as {0}";
    String H = "h";
    String A1 = "a1";
@@ -48,4 +53,42 @@
    String DataMenu = "DataMenu";
    String AND = " and ";
    String MD = "md_";


    String CNT = "cnt";
    String checkFieldSqlTemplate = "select count(1) as cnt from (select {0} from {1}) a1";
    String selectRecordFieldSqlTemplate = "select {0}, 0 as deal from {1}";
    String selectFieldSqlTemplate = "select {0} from ({1}) a  where {2}";
    String selectFieldTableTemplate = "select {0} from {1}  where {2}";
    String selectFieldSqlMysqlTemplate = "select * from {0} limit 1";
    String STEP_READ_FROM_TABLE = "Read data from table";
    String STEP_INSERT_UPDATE = "Insert or update";
    String STEP_DUMMY = "Dummy";
    String MYSQL_UUID = "replace(UUID(),'-', '')";
    String TYPE =  "type";
    String Active =  "active";
    String AssembleTempTable = "hj_table_";
    String AssembleTempSql = "hj_sql_";
    String CreateFieldMysql = "`{0}` VARCHAR(255)  NULL";
    String CreateTableMysql = "CREATE TABLE IF NOT EXISTS `{0}` ({1}) ENGINE=InnoDB DEFAULT CHARSET=utf8;";
    String checkTableByNameBySchema = "select table_name from information_schema.tables where table_schema={0} and table_name = {1};";
    String Temp =  "temp_";
    String Temp2RecordMySQLTemplate = "INSERT INTO {0} ({1}) SELECT {2}, replace(uuid(),''-'', '''') as std_id, replace(uuid(),''-'', '''') as id, 0 as deal from  {3}";
    String MYSQLJoinUpdateSql = "INSERT INTO {0} ({1})  SELECT {2} , replace(uuid(),''-'', '''') as id, s1.id as std_id, 0 as deal  FROM {3} t1 inner join {4} s1  on  {5};";
    String MYSQLJoinAddSql = "INSERT INTO {0} ({1})  SELECT {2} , replace(uuid(),''-'', '''') as std_id, replace(uuid(),''-'', '''') as id, 0 as deal  FROM {3} t1 WHERE NOT EXISTS(SELECT * FROM  {4} s1 WHERE {5});";
    String Temp2RecordHbaseTemplate = "INSERT INTO {0} ({1}) SELECT ({2}, ) from {3}";
    String ParamsShell = "@\\{{0}\\}";
    String Cron = "cron";
    String TABLE_NAME = "TABLE_NAME";
    String Success = "success";
    String Data = "data";
    String Mysql_UUID = "replace(uuid(),''-'', '''')";
    String MainData = "MainData";
    String MYSQLJoinParse =  " t1.{0} = s1.{0} ";
    String T1 = "t1";
    String S1 = "s1";
    String Code = "code";
    String truncateTableMysql = "truncate table {0};";
    String DropTableMysql = "DROP TABLE IF EXISTS {0};";
    String DeleteDealMysqlTemplate = "delete from {0} where deal = 0";
}

 src/main/java/com/highdatas/mdm/util/DbUtils.java

@@ -10,9 +10,11 @@

import java.lang.reflect.Field;
import java.lang.reflect.Method;
import java.security.MessageDigest;
import java.text.MessageFormat;
import java.util.Random;
import java.util.UUID;
import java.util.*;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

/**
 * @author kimi
@@ -22,6 +24,7 @@

@Slf4j
public class DbUtils {
    private static final char HEX_DIGITS[] = { '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'A', 'B', 'C', 'D', 'E', 'F' };

    public static String versionAddSub (String baseVersion) {
        if (StringUtils.isEmpty(baseVersion)) {
@@ -191,6 +194,15 @@
        }
        return builder.toString();
    }

    public static String assemblParam(String objects){
        if (StringUtils.isEmpty(objects)) {
            return null;
        }else  {
            return  "@{" + objects + "}";
        }

    }
    public static String getRandomString(int length) {
        Random random = new Random();
        StringBuffer sb = new StringBuffer();
@@ -234,4 +246,55 @@
        }
        return sbf.toString();
    }


    public static Set<String> matcher(String testStr) {
        return matcher(Constant.Default_Patter, testStr);
    }

    public static String replaceEscape(String testStr) {
        String new_str = Pattern.compile("\t|\r|\n| ").matcher(testStr).replaceAll(Constant.EMPTY);
        return new_str;
    }

    public static Set<String> matcher(String patter, String testStr) {
//        String test = "@{databaseIp}:{databasePort}{instanceName};database";
//        String initCompile = "(?<=@\\{)(.+?)(?=\\})";

        Set<String> ls=new HashSet<>();

        Pattern pattern = Pattern.compile(patter);
        Matcher matcher = pattern.matcher(testStr);

        while(matcher.find()){

            ls.add(matcher.group());

        }
        return ls;
    }




    private static String toHexString(byte[] b) {
        StringBuilder sb = new StringBuilder(b.length * 2);
        for (int i = 0; i < b.length; i++) {
            sb.append(HEX_DIGITS[(b[i] & 0xf0) >>> 4]);
            sb.append(HEX_DIGITS[b[i] & 0x0f]);
        }
        return sb.toString();
    }

    public static String Bit32(String SourceString) throws Exception {
        MessageDigest digest = java.security.MessageDigest.getInstance("MD5");
        digest.update(SourceString.getBytes());
        byte messageDigest[] = digest.digest();
        return toHexString(messageDigest);
    }

    public static String Bit16(String SourceString) throws Exception {
        return Bit32(SourceString).substring(8, 24);
    }

}

 src/main/java/com/highdatas/mdm/util/HttpUtils.java

@@ -1,5 +1,7 @@
package com.highdatas.mdm.util;

import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.http.*;
import org.springframework.http.client.SimpleClientHttpRequestFactory;
import org.springframework.util.MultiValueMap;
@@ -14,6 +16,7 @@

public class HttpUtils {


    public static String HttpRestClient(String url, HttpMethod method, MultiValueMap<String, String> params) {
        SimpleClientHttpRequestFactory requestFactory = new SimpleClientHttpRequestFactory();
        requestFactory.setConnectTimeout(15*1000);
@@ -27,4 +30,6 @@
        ResponseEntity<String> response = client.exchange(url, method, requestEntity, String.class);
        return response.getBody();
    }


}

 src/main/java/com/highdatas/mdm/util/MpGenerator.java

@@ -28,8 +28,8 @@
        // 全局配置
        GlobalConfig gc = new GlobalConfig();
        gc.setAuthor("kimi");
        gc.setOutputDir("D://workspace/workspace_idea/mdm/src/main/java");
        gc.setFileOverride(false);// 是否覆盖同名文件，默认是false
        gc.setOutputDir("H://workspace/workspace_idea/mdm/src/main/java");
        gc.setFileOverride(true);// 是否覆盖同名文件，默认是false
        gc.setActiveRecord(true);// 不需要ActiveRecord特性的请改为false
        gc.setEnableCache(false);// XML 二级缓存
        gc.setBaseResultMap(true);// XML ResultMap
@@ -57,7 +57,7 @@
        dsc.setDriverName("com.mysql.jdbc.Driver");
        dsc.setUsername("root");
        dsc.setPassword("123456");
        dsc.setUrl("jdbc:mysql://localhost:3306/data_admin?useUnicode=true&characterEncoding=utf8&serverTimezone=UTC");
        dsc.setUrl("jdbc:mysql://localhost:3306/mdm?useUnicode=true&characterEncoding=utf8&serverTimezone=UTC");
        mpg.setDataSource(dsc);

        // 策略配置
@@ -65,7 +65,7 @@
        // strategy.setCapitalMode(true);// 全局大写命名 ORACLE 注意
        strategy.setTablePrefix(new String[]{""});// 此处可以修改为您的表前缀
        strategy.setNaming(NamingStrategy.underline_to_camel);// 表名生成策略
        strategy.setInclude(new String[]{"deleted_table_log"}); // 需要生成的表
        strategy.setInclude(new String[]{"sys_assemble_rule",}); // 需要生成的表
        // strategy.setExclude(new String[]{"test"}); // 排除生成的表
        // 自定义实体父类
        // strategy.setSuperEntityClass("com.baomidou.demo.TestEntity");

 src/main/java/com/highdatas/mdm/util/RedisClient.java

New file
@@ -0,0 +1,78 @@
package com.highdatas.mdm.util;



import com.alibaba.fastjson.JSON;

import com.alibaba.fastjson.JSONObject;

import org.springframework.beans.factory.annotation.Value;

import org.springframework.boot.context.properties.ConfigurationProperties;

import org.springframework.http.HttpMethod;

import org.springframework.stereotype.Component;

import org.springframework.util.LinkedMultiValueMap;

import org.springframework.util.MultiValueMap;

import org.springframework.util.StringUtils;



@ConfigurationProperties(prefix = "redis")

@Component

public class RedisClient {



    @Value("${redis.host}")

    String host;

    @Value("${redis.port}")

    String port;



    String prefix ;



    private RedisClient() {

        this.prefix = "/api/datacvg/redis/";

    }

    public  boolean putRedisVal(String key, String value) {

        String url = host+ Constant.colon +  port + prefix + "put";

        MultiValueMap<String, String> params = new LinkedMultiValueMap<>();

        params.set("key",key);

        params.set("value",value);

        params.set("type",Constant.MainData);

        String s = HttpUtils.HttpRestClient(url, HttpMethod.POST, params);

        JSONObject result = (JSONObject) JSON.parse(s);

        String sucess = result.getString(Constant.Success);

        if (StringUtils.isEmpty(sucess) || !Boolean.valueOf(sucess)) {

            return false;

        }else {

            return true;

        }

    }



    public String getRedisVal(String key) {

        try {

            String url =  host+ Constant.colon +  port + prefix + "get";

            MultiValueMap<String, String> params = new LinkedMultiValueMap<>();

            params.set("key",key);

            params.set("type",Constant.MainData);

            String s = HttpUtils.HttpRestClient(url, HttpMethod.POST, params);

            JSONObject result = (JSONObject) JSON.parse(s);

            String sucess = result.getString(Constant.Success);

            if (StringUtils.isEmpty(sucess) || !Boolean.valueOf(sucess)) {

                return null;

            }else {

                return result.getString(Constant.Data);

            }

        }catch (Exception e){

            e.printStackTrace();

            return null;

        }



    }



    public boolean deleteRedisVal(String key) {

        String url =  host+ Constant.colon +  port + prefix + "delete";

        MultiValueMap<String, String> params = new LinkedMultiValueMap<>();

        params.set("key",key);

        params.set("type",Constant.MainData);

        String s = HttpUtils.HttpRestClient(url, HttpMethod.POST, params);

        JSONObject result = (JSONObject) JSON.parse(s);

        String sucess = result.getString(Constant.Success);

        if (StringUtils.isEmpty(sucess) || !Boolean.valueOf(sucess)) {

            return false;

        }else {

            return true;

        }

    }

}


 src/main/resources/application-dev.yml

@@ -4,10 +4,11 @@

spring:
  datasource:
    driver-class-name: com.mysql.cj.jdbc.Driver
    driver-class-name: com.mysql.jdbc.Driver
    url: jdbc:mysql://localhost:3306/mdm?useUnicode=true&characterEncoding=utf-8&useSSL=false&serverTimezone=UTC&nullCatalogMeansCurrent=true
    username: root
    password: 123456
    type: com.alibaba.druid.pool.DruidDataSource

  activiti:
    database-schema-update: true
@@ -40,4 +41,20 @@
  callback:
    url: http://192.168.4.225:18080/activiti/getBusinessId
#activiti:
#  check-process-definitions: false
#  check-process-definitions: false
redis:
  host: http://180.169.94.250
  port: 9107

master:
  unbigdata:
    db:
      host: localhost
      url:  jdbc:mysql://localhost:3306/mdm?useUnicode=true&characterEncoding=utf-8&useSSL=false&serverTimezone=UTC&nullCatalogMeansCurrent=true
      type: MYSQL
      port: 3306
      schema: mdm
      username: root
      password: 123456
  bigdata:
    db:hbase?

 src/main/resources/application-prod.yml

@@ -11,10 +11,10 @@

spring:
  datasource:
    driver-class-name: com.mysql.cj.jdbc.Driver
    url: jdbc:mysql://127.0.0.1:3306/data_admin?useUnicode=true&characterEncoding=utf-8&useSSL=false&serverTimezone=UTC&nullCatalogMeansCurrent=true
    username: root
    password: datacvg123!
    driver-class-name: com.mysql.jdbc.Driver
    url:  jdbc:mysql://180.169.94.250:8306/data_admin?useUnicode=true&characterEncoding=utf-8&useSSL=false&serverTimezone=UTC&nullCatalogMeansCurrent=true
    username: masterdata
    password: md123!
    hikari:
      minimum-idle: 3
      max-lifetime: 30000
@@ -43,3 +43,16 @@
  global-config:
    db-config:
      column-underline: true

master:
  unbigdata:
    db:
      host: 180.169.94.250
      url:  jdbc:mysql://180.169.94.250:8306/data_admin?useUnicode=true&characterEncoding=utf-8&useSSL=false&serverTimezone=UTC&nullCatalogMeansCurrent=true
      type: MYSQL
      port: 8306
      schema: data_admin
      username: masterdata
      password: md123!
  bigdata:
    db:hbase?

 src/main/resources/application.yml

@@ -1,4 +1,6 @@
spring:
  profiles:
    active:
      prod
      dev
  main:
    allow-bean-definition-overriding: true

 src/main/resources/bak/TaskServixceImpl.java

New file
@@ -0,0 +1,465 @@
package com.highdatas.mdm.service.impl;



import com.alibaba.fastjson.JSONArray;

import com.alibaba.fastjson.JSONObject;

import com.baomidou.mybatisplus.mapper.EntityWrapper;

import com.highdatas.mdm.entity.*;

import com.highdatas.mdm.mapper.TableInfoMapper;

import com.highdatas.mdm.pojo.CodeMsg;

import com.highdatas.mdm.pojo.DbAccessType;

import com.highdatas.mdm.pojo.Result;

import com.highdatas.mdm.pojo.SysAssembleLogStatus;

import com.highdatas.mdm.pojo.kettle.KettleDBTrans;

import com.highdatas.mdm.pojo.kettle.UnBigDataDataSourceInfo;

import com.highdatas.mdm.service.*;

import com.highdatas.mdm.util.Constant;

import com.highdatas.mdm.util.DbUtils;

import org.apache.commons.lang3.StringUtils;

import org.pentaho.di.core.KettleClientEnvironment;

import org.pentaho.di.core.database.Database;

import org.pentaho.di.core.database.DatabaseMeta;

import org.pentaho.di.core.exception.KettleException;

import org.pentaho.di.core.logging.LoggingObjectInterface;

import org.pentaho.di.core.logging.LoggingObjectType;

import org.pentaho.di.core.logging.SimpleLoggingObject;

import org.pentaho.di.trans.Trans;

import org.pentaho.di.trans.TransHopMeta;

import org.pentaho.di.trans.TransMeta;

import org.pentaho.di.trans.step.StepMeta;

import org.pentaho.di.trans.steps.dummytrans.DummyTransMeta;

import org.pentaho.di.trans.steps.tableinput.TableInputMeta;

import org.pentaho.di.trans.steps.tableoutput.TableOutputMeta;

import org.springframework.beans.factory.annotation.Autowired;

import org.springframework.beans.factory.annotation.Value;

import org.springframework.stereotype.Service;



import java.awt.*;

import java.sql.Connection;

import java.sql.PreparedStatement;

import java.sql.ResultSet;

import java.sql.SQLException;

import java.text.MessageFormat;

import java.util.Date;

import java.util.List;

import java.util.stream.Collectors;



/**

 * @author kimi

 * @description

 * @date 2020-02-15 14:26

 */



@Service

public class TaskServixceImpl  {

    @Autowired

    ISysAssembleService assembleService;

    @Autowired

    ISysAssembleDetailService assembleDetailService;

    @Autowired

    ISysAssembleLogService assembleLogService;

    @Autowired

    ISysDbtypeService dbtypeService;

    @Autowired

    IMenuMappingService menuMappingService;

    @Autowired

    UnBigDataDataSourceInfo unBigDataDataSourceInfo;

    @Autowired

    MasterDataService masterDataService;

    @Autowired

    TableInfoMapper infoMapper;



    @Override

    public Result run(String assembleId) {

        SysAssembleLog sysAssembleLog = new SysAssembleLog();

        sysAssembleLog.setId(DbUtils.getUUID());

        sysAssembleLog.setCreateTime(new Date());

        sysAssembleLog.setAssembleId(assembleId);

        sysAssembleLog.setStatus(SysAssembleLogStatus.working);

        sysAssembleLog.insert();





        if (StringUtils.isEmpty(assembleId)) {

            sysAssembleLog.setMessage("assembleId is null").setUpdateTime(new Date()).setStatus(SysAssembleLogStatus.defeat).updateById();

            return Result.error(CodeMsg.ERROR_PARAMS_NOT_MATHED);

        }



        SysAssembleLog runtimeLog = assembleLogService.selectOne(new EntityWrapper<SysAssembleLog>().eq("assemble_id", assembleId).orderBy("create_time desc"));

        if (runtimeLog != null) {

            if (runtimeLog.getStatus().equals(SysAssembleLogStatus.working)) {

                sysAssembleLog.setMessage("当前正在有同一任务在运行").setUpdateTime(new Date()).setStatus(SysAssembleLogStatus.defeat).updateById();

                return Result.error(new CodeMsg(3001, "当前正在有同一任务在运行"));

            }

        }



        KettleDBTrans kettleDBTrans = null;

        try {

            SysAssemble assemble = assembleService.selectById(assembleId);

            if (assemble == null) {

                //false

                sysAssembleLog.setMessage("assemble is not found").setUpdateTime(new Date()).setStatus(SysAssembleLogStatus.defeat).updateById();

                return Result.error(CodeMsg.ERROR_PARAMS_NOT_MATHED);

            }

            String sqlStr = "", tableName = "";



            Database dsInput = null;

            Database dsOutput = null;

            TableInputMeta tableInputMeta = new TableInputMeta();

            TableOutputMeta talbeOutputMeta = new TableOutputMeta();

            TransMeta transMeta = new TransMeta();



            kettleDBTrans    = new KettleDBTrans();

            kettleDBTrans.setAssemble(assemble);

            kettleDBTrans.setDsInput(dsInput);

            kettleDBTrans.setDsOutput(dsOutput);

            kettleDBTrans.setTableInputMeta(tableInputMeta);

            kettleDBTrans.setTalbeOutputMeta(talbeOutputMeta);

            kettleDBTrans.setTransMeta(transMeta);



            boolean connected = kettleDBConn(kettleDBTrans);

            if (!connected){

                //false

                sysAssembleLog.setMessage("connection failed").setUpdateTime(new Date()).setStatus(SysAssembleLogStatus.defeat).updateById();

            }

            String dbType = "";

//            assemble.getDbType();

            SysDbtype sysDbtype = dbtypeService.selectOne(new EntityWrapper<SysDbtype>().eq("name", dbType));

            DbAccessType type = sysDbtype.getType();

            switch (type) {

                case unRelational:

                    break;

                case relational:

                    extractDBData(kettleDBTrans, sysAssembleLog);



            }

        } catch (KettleException e) {

            e.printStackTrace();

            sysAssembleLog.setMessage(e.getMessage()).setUpdateTime(new Date()).setStatus(SysAssembleLogStatus.defeat).updateById();

        }finally {

            Database dsInput = kettleDBTrans.getDsInput();

            if (dsInput != null) {

                dsInput.disconnect();

            }

            Database dsOutput = kettleDBTrans.getDsOutput();

            if (dsOutput != null) {

                dsOutput.disconnect();

            }



        }

        return null;

    }



    private boolean kettleDBConn(KettleDBTrans kettleDBTrans) throws KettleException {

        SysAssemble assemble = kettleDBTrans.getAssemble();



        //初始化kettle环境

        KettleClientEnvironment.init();

        //导入数据的数据库连接

        DatabaseMeta dataMetaInput = null;

//        DatabaseMeta dataMetaInput = new DatabaseMeta("Input", assemble.getDbType(), "Native",assemble.getHost(), assemble.getSchema(),assemble.getPort().toString(),

//                assemble.getUserName(), assemble.getPassword());



        Boolean bigdata = true;

//        /assemble.getBigdata();

        DatabaseMeta dataMetaOutput = null;

        if (bigdata) {

            // 导入hbase数据库

        }else {

            // 导入mysql数据库

            //导出数据的数据库连接

            dataMetaOutput = new DatabaseMeta("Output", unBigDataDataSourceInfo.getDbType(), "Native", unBigDataDataSourceInfo.getDbHostName(), unBigDataDataSourceInfo.getDbName(), unBigDataDataSourceInfo.getDbPort(),

                    unBigDataDataSourceInfo.getUsername(), unBigDataDataSourceInfo.getPassword());

        }



        if (dataMetaOutput == null) {

            return false;



        }



        //导出数据库连接

        LoggingObjectInterface loggingObject = new SimpleLoggingObject("Database factory", LoggingObjectType.GENERAL, null );

        kettleDBTrans.setDsInput(new Database(loggingObject, dataMetaInput));

        kettleDBTrans.getDsInput().connect();

        Connection connInput = kettleDBTrans.getDsInput().getConnection();

        //导入数据库连接

        kettleDBTrans.setDsOutput(new Database(loggingObject, dataMetaOutput));

        kettleDBTrans.getDsOutput().connect();

        Connection connOutput = kettleDBTrans.getDsOutput().getConnection();



        kettleDBTrans.getTransMeta().setName("数据抽取");



        kettleDBTrans.getTransMeta().addDatabase(dataMetaInput);

        kettleDBTrans.getTransMeta().addDatabase(dataMetaOutput);



        //导出数据

        kettleDBTrans.getTableInputMeta().setDatabaseMeta(kettleDBTrans.getTransMeta().findDatabase("Input"));

        //导入数据

        kettleDBTrans.getTalbeOutputMeta().setDatabaseMeta(kettleDBTrans.getTransMeta().findDatabase("Output"));

        return true;

    }



    private void extractDBData(KettleDBTrans dbTrans, SysAssembleLog sysAssembleLog) {

        try {

            SysAssemble assemble = dbTrans.getAssemble();

            boolean checked = checkField(dbTrans);

            if (!checked){

                //false log

                sysAssembleLog.setMessage("source target field is not matched").setUpdateTime(new Date()).setStatus(SysAssembleLogStatus.defeat).updateById();

                return;

            }



            List<SysAssembleDetail> detailList = assembleDetailService.selectList(new EntityWrapper<SysAssembleDetail>().eq("assemble_id", assemble.getId()));

            List<String> fieldList = detailList.stream().map(sysAssembleDetail -> sysAssembleDetail.getField()).collect(Collectors.toList());

            if (fieldList.contains(Constant.ID)) {

                 fieldList.remove(Constant.ID);

                 fieldList.add(MessageFormat.format(Constant.FieldAsAlias, Constant.ID, Constant.STD_ID));

                 if (!assemble.getBigdata()){

                     //MYSQL

                     fieldList.add(MessageFormat.format(Constant.MYSQL_UUID, Constant.ID));

                 }

                //fieldList = fieldList.stream().map(s -> ( s.equalsIgnoreCase(Constant.ID) ? MessageFormat.format(Constant.FieldAsAlias,Constant.ID, Constant.STD_ID) : s))

            }else {

                if (!assemble.getBigdata()) {

                    fieldList.add(MessageFormat.format(Constant.MYSQL_UUID, Constant.STD_ID));

                    fieldList.add(MessageFormat.format(Constant.MYSQL_UUID, Constant.ID));

                }

            }



            String fieldStr = fieldList.stream().collect(Collectors.joining(Constant.COMMA));

            String countSqlStr = MessageFormat.format(Constant.selectFieldSqlTemplate, fieldStr, assemble.getTableName());

            String selectSqlStr = MessageFormat.format(Constant.selectFieldSqlTemplate, fieldStr, assemble.getTableName());

            dbTrans.getTableInputMeta().setSQL(selectSqlStr);

            Database dsInput = dbTrans.getDsInput();

            Connection inputConnection = dsInput.getConnection();

            PreparedStatement cntStmt = inputConnection.prepareStatement(countSqlStr);

            cntStmt.executeQuery();

            ResultSet cntResultSet = cntStmt.executeQuery();

            Integer sorceCnt = cntResultSet.getInt(Constant.CNT);



            // StepMeta

            StepMeta inputStep = new StepMeta(Constant.STEP_READ_FROM_TABLE, dbTrans.getTableInputMeta());

            inputStep.setLocation(50, 50);

            inputStep.setDraw(true);

            dbTrans.getTransMeta().addStep(inputStep);



            String trmpTableName = getTempTableNameFromMenu(assemble.getMenuId());



            dbTrans.getTalbeOutputMeta().setTableName(trmpTableName);

            StepMeta insertUpdateStep = new StepMeta(Constant.STEP_INSERT_UPDATE, dbTrans.getTalbeOutputMeta());

            insertUpdateStep.setLocation(150, 50);

            insertUpdateStep.setDraw(true);

            dbTrans.getTransMeta().addStep(insertUpdateStep);



            DummyTransMeta dummyMeta = new DummyTransMeta();



            StepMeta dummyStep = new StepMeta(Constant.STEP_DUMMY, dummyMeta);

            dummyStep.setLocation(200, 50);

            dummyStep.setDraw(true);



            dbTrans.getTransMeta().addStep(dummyStep);



            //设置步骤直接的关系

            TransHopMeta hop = new TransHopMeta(inputStep, insertUpdateStep);

            dbTrans.getTransMeta().addTransHop(hop);

            TransHopMeta hop2 = new TransHopMeta(insertUpdateStep, dummyStep);

            dbTrans.getTransMeta().addTransHop(hop2);



            //开始执行

            Trans trans = new Trans(dbTrans.getTransMeta());

            trans.prepareExecution(null);

            trans.startThreads();

            trans.waitUntilFinished();



            int errors = trans.getErrors();

            if (errors == 0) {

                //搬运相关数据

                if (!assemble.getBigdata()){

                    masterDataService.uploadedData(assemble.getTableName(), "All", assemble.getUserId());

                }

               else {

                    // TODO: 2020/2/16 hbase 搬运数据

                }

                sysAssembleLog.setCnt(sorceCnt).setMessage("汇集成功").setUpdateTime(new Date()).setStatus(SysAssembleLogStatus.success).updateById();

            }else {

                sysAssembleLog.setMessage("汇集搬运过程中发生问题").setUpdateTime(new Date()).setStatus(SysAssembleLogStatus.defeat).updateById();

            }

            boolean hasTable = dbTrans.getDsOutput().checkTableExists(assemble.getTableName());

            if (hasTable) {

                //TODO 全量还是增量

            }

            else {

                //创建表相关 目前禁用   log

                sysAssembleLog.setMessage("当前系统中不存在相关表数据,请先初始化表结构等基础主数据结构信息").setUpdateTime(new Date()).setStatus(SysAssembleLogStatus.defeat).updateById();

            }

        }

        catch (Exception e) {

            e.printStackTrace();

        }

        finally {

            Database dsOutput = dbTrans.getDsOutput();

            if (dsOutput!= null) {

                dsOutput.disconnect();

            }

            Database dsInput = dbTrans.getDsInput();

            if (dsInput != null) {

                dsInput.disconnect();

            }

        }

    }



    private String getTempTableNameFromMenu(String menuId) {

        MenuMapping menuMapping = menuMappingService.selectOne(new EntityWrapper<MenuMapping>().eq("menu_id", menuId));

        String tableName = menuMapping.getTableName();

        return tableName + Constant.RECORD;

    }



    private boolean checkField(KettleDBTrans dbTrans) {

        SysAssemble assemble = dbTrans.getAssemble();

        boolean checked = true;

        Connection conn = null;

        Database ds = null;

        try {

            //1 check 源库中是否有相关字段

            List<SysAssembleDetail> detailList = assembleDetailService.selectList(new EntityWrapper<SysAssembleDetail>().eq("assemble_id", assemble.getId()));

            String fieldStr = detailList.stream().map(sysAssembleDetail -> sysAssembleDetail.getField()).collect(Collectors.joining(Constant.COMMA));

            List<String> fieldList = detailList.stream().map(sysAssembleDetail -> sysAssembleDetail.getField()).collect(Collectors.toList());

            String checkSqlStr = MessageFormat.format(Constant.checkFieldSqlTemplate, fieldStr, assemble.getTableName());

            //连接数据库

            KettleClientEnvironment.init();

            DatabaseMeta dataMeta = new DatabaseMeta("KettleDBRep",assemble.getDbType(), "Native", assemble.getHost(),assemble.getSchema(),assemble.getPort().toString(),assemble.getUserName(),assemble.getPassword());

            LoggingObjectInterface loggingObject = new SimpleLoggingObject("Database factory", LoggingObjectType.GENERAL, null );

            ds = new Database(loggingObject, dataMeta);

            ds.connect();

            conn = ds.getConnection();

            PreparedStatement stmt = conn.prepareStatement(checkSqlStr);

            stmt.executeQuery();

            ds.disconnect();

            ds = null;

            //2 验证 目标库中是否有相关字段

            Boolean bigdata = assemble.getBigdata();

            if (bigdata) {

                //TODO hbase

            }

            else {

                boolean exists = dbTrans.getDsOutput().checkTableExists(assemble.getTableName());

                if (exists) {

                    List<TableSchemaResult> tableFieldFromDb = infoMapper.getTableFieldFromDb(assemble.getTableName(), assemble.getSchema());

                    List<String> targetFieldList = tableFieldFromDb.stream().map(tableSchemaResult -> tableSchemaResult.getFieldName()).collect(Collectors.toList());

                    long count = fieldList.stream().filter(sourceField -> !targetFieldList.contains(sourceField)).count();

                    if (count > 0) {

                        //TODO 有部分字段未能匹配上 且是mysql数据库 暂处理为不导入   log

                        checked = false;

                    }

                }



            }

            checked = true;

        } catch(Exception e) {

            e.printStackTrace();

            checked = false;

        }finally {

            if (ds != null) {

                ds.disconnect();

            }

            if (conn != null) {

                try {

                    conn.close();

                } catch (SQLException e) {

                    e.printStackTrace();

                    checked = false;

                }

            }

        }

        return checked;

    }





    @Override

    public Result updateTask(String assembleId) {

        return null;

    }



    private Database getDataBaseByKettle(SysAssemble assemble) throws KettleException {

        Database ds = null;

        KettleClientEnvironment.init();

        DatabaseMeta dataMeta = new DatabaseMeta("KettleDBRep", assemble.getDbType(), "Native",assemble.getHost(), assemble.getSchema(), assemble.getPort().toString(),

                assemble.getUserName(), assemble.getPassword());

        LoggingObjectInterface loggingObject = new SimpleLoggingObject("Database factory", LoggingObjectType.GENERAL, null );

        ds = new Database(loggingObject, dataMeta);



        return ds;

    }

    @Override

    public boolean connectionDb(SysAssemble assemble) {

        Connection conn = null;

        Database ds = null;

        boolean connected = false;

        try {

            ds = getDataBaseByKettle(assemble);

            ds.normalConnect(null);

            conn = ds.getConnection();



        if (conn == null) {

            connected = false;

        }else {

            connected = true;

        }



        ds.disconnect();

        ds = null;

        } catch (KettleException e) {

            e.printStackTrace();

            connected = false;

        }

        finally {

            if (ds != null) {

                ds.disconnect();

            }



            try {

                if (conn != null) {

                    conn.close();

                }



            } catch (SQLException e) {

                e.printStackTrace();

            }

        }

        return connected;

    }



    @Override

    public boolean sqlTest(SysAssemble assemble, String sqlStr) {

        Connection conn = null;

        Database ds = null;

        boolean connected = false;

        try {

            ds = getDataBaseByKettle(assemble);

            ds.normalConnect(null);

            conn = ds.getConnection();

            PreparedStatement stmt = conn.prepareStatement(sqlStr);

            stmt.executeQuery();



            ds.disconnect();

            ds = null;

            connected = true;

        } catch (Exception e) {

            e.printStackTrace();

            connected = false;

        }

        finally {

            if (ds != null) {

                ds.disconnect();

            }



            try {

                if (conn != null) {

                    conn.close();

                }



            } catch (SQLException e) {

                e.printStackTrace();

            }

        }

        return connected;

    }

}


 src/main/resources/lib/mysql-connector-java-5.1.46-bin.jar

Binary files differ

 src/main/resources/mapping/MenuMappingMapper.xml

@@ -20,7 +20,7 @@

    <select id="getMapping" resultType="java.util.Map">
        SELECT
        m.id,m.code, m.name, u.user_name as userName,system_id as systemId,
        m.id,m.code, m.name, u.user_name as userName,system_id as systemId, m.audit as audit,
        sm.data_type as dataType,
        c2.name as cname1, c2.id as cid1, c1.name as cname2, c1.id as cid2, c.name as cname3, c.id as cid3,
        start_time as startTime, end_time as endTime, m.create_time as createTime, m.update_time as updateTime,

 src/main/resources/mapping/TableInfoMapper.xml

@@ -195,7 +195,7 @@
        select ${fields} from ${tableName} where ${where}
    </select>

    <select id="tempDeal"  statementType="STATEMENT"  resultType="java.lang.Long">
    <select id="tempDeal"  statementType="STATEMENT" resultType="java.lang.Integer">

        UPDATE ${tableTempName} t
        SET deal = 1
@@ -435,7 +435,15 @@
    <select id="insertMatintainDetailFromTemp" statementType="STATEMENT" resultType="java.lang.Integer">

        INSERT INTO maintain_detail
        SELECT md5(uuid()) as id, ${maintainId} as parent_id, 'create' as operate, id as pre_merge_id, NOW() as createdate
        SELECT md5(uuid()) as id, ${maintainId} as parent_id, ${operate} as operate, id as pre_merge_id, NOW() as create_time
        from ${tableTempName}  m WHERE m.deal = 0

    </select>

    <select id="insertMatintainDetailFromStandrad" statementType="STATEMENT" resultType="java.lang.Integer">

        INSERT INTO maintain_detail
        SELECT md5(uuid()) as id, ${maintainId} as parent_id, 'delete' as operate, id as pre_merge_id, NOW() as create_time
        from ${tableTempName}  m WHERE m.deal = 0

    </select>
@@ -444,6 +452,13 @@

        update ${tableTempName}
        set std_id = md5(uuid()) WHERE deal = 0

    </select>

    <select id="insertRecordFromStandrad" statementType="STATEMENT" resultType="java.lang.Integer">
         INSERT INTO ${recordTableName}
        SELECT ${tempFieldList}, 0 as deal
        from  ${tableName}

    </select>

@@ -476,7 +491,7 @@
        DELETE  ${tableTempName} from  ${tableTempName}
        INNER JOIN maintain_detail on maintain_detail.pre_merge_id =  ${tableTempName}.id
        WHERE maintain_detail.parent_id = ${maintainId} and  ${tableTempName}.std_id = ${stdId}
\


    </select>
</mapper>