diff --git a/build-docker/build_and_push_image.sh b/build-docker/build_and_push_image.sh index 88b55ebe94809718d73156a8956c4b6a4940946f..87084bb78b972fc41856149703e51e9a1e7e9453 100644 --- a/build-docker/build_and_push_image.sh +++ b/build-docker/build_and_push_image.sh @@ -2,7 +2,7 @@ set -e -DBSWITCH_VERSION=1.6.8 +DBSWITCH_VERSION=1.6.9 BUILD_DOCKER_DIR="$( cd "$( dirname "$0" )" && pwd )" PROJECT_ROOT_DIR=$( dirname "$BUILD_DOCKER_DIR") DOCKER_DBSWITCH_DIR=$BUILD_DOCKER_DIR/dbswitch diff --git a/build-docker/install/docker-compose.yml b/build-docker/install/docker-compose.yml index 41f3193cd5eb437d3b12fa0e53c378330d1060ec..65b05d89d9261c58a1729fc38cd6997c750badb2 100644 --- a/build-docker/install/docker-compose.yml +++ b/build-docker/install/docker-compose.yml @@ -13,7 +13,7 @@ services: MYSQL_ROOT_HOST: '%' dbswitch: container_name: dbswitch_webui - image: inrgihc/dbswitch:1.6.8 + image: inrgihc/dbswitch:1.6.9 environment: MYSQLDB_HOST: dbswitch_mysqldb MYSQLDB_PORT: 3306 diff --git a/dbswitch-admin/pom.xml b/dbswitch-admin/pom.xml index a8e72a150e4afaaad877f2e23d3c2f04bf2e5c98..1be8e5d6dbadf85d2558e78c00f70599ff950810 100644 --- a/dbswitch-admin/pom.xml +++ b/dbswitch-admin/pom.xml @@ -5,7 +5,7 @@ com.gitee.dbswitch dbswitch-parent - 1.6.8 + 1.6.9 dbswitch-admin diff --git a/dbswitch-admin/src/main/java/com/gitee/dbswitch/admin/model/request/AssigmentCreateRequest.java b/dbswitch-admin/src/main/java/com/gitee/dbswitch/admin/model/request/AssigmentCreateRequest.java index 69e98f3d7595574f363f1d498a195bb01c7c5765..65ca170e26d086e68542ab5083ec92dd0ae654bc 100644 --- a/dbswitch-admin/src/main/java/com/gitee/dbswitch/admin/model/request/AssigmentCreateRequest.java +++ b/dbswitch-admin/src/main/java/com/gitee/dbswitch/admin/model/request/AssigmentCreateRequest.java @@ -13,14 +13,17 @@ import com.gitee.dbswitch.admin.common.excption.DbswitchException; import com.gitee.dbswitch.admin.common.response.ResultCode; import com.gitee.dbswitch.admin.entity.AssignmentConfigEntity; import com.gitee.dbswitch.admin.entity.AssignmentTaskEntity; -import com.gitee.dbswitch.admin.service.ScheduleService; import com.gitee.dbswitch.admin.type.IncludeExcludeEnum; import com.gitee.dbswitch.admin.type.ScheduleModeEnum; +import com.gitee.dbswitch.admin.util.CronExprUtils; import com.gitee.dbswitch.common.entity.PatternMapper; +import com.gitee.dbswitch.common.util.PatterNameUtils; import java.util.List; import java.util.Objects; import lombok.Data; import lombok.NoArgsConstructor; +import org.springframework.util.CollectionUtils; +import org.springframework.util.StringUtils; @NoArgsConstructor @Data @@ -55,11 +58,7 @@ public class AssigmentCreateRequest { assignment.setDescription(description); assignment.setScheduleMode(scheduleMode); if (ScheduleModeEnum.SYSTEM_SCHEDULED == this.getScheduleMode()) { - if (!ScheduleService.checkCronExpressionValid(this.getCronExpression())) { - throw new DbswitchException(ResultCode.ERROR_INVALID_ARGUMENT, - "CRON表达式[" + this.getCronExpression() + "]"); - } - + CronExprUtils.checkCronExpressionValid(this.getCronExpression(), 120); assignment.setCronExpression(this.getCronExpression()); } @@ -91,6 +90,19 @@ public class AssigmentCreateRequest { ); assignmentConfigEntity.setFirstFlag(Boolean.TRUE); + if (!assignmentConfigEntity.getExcluded() + && !CollectionUtils.isEmpty(assignmentConfigEntity.getSourceTables())) { + for (String tableName : assignmentConfigEntity.getSourceTables()) { + String targetTableName = PatterNameUtils.getFinalName(tableName, + assignmentConfigEntity.getTableNameMap()); + if (StringUtils.isEmpty(targetTableName)) { + throw new DbswitchException( + ResultCode.ERROR_INVALID_ASSIGNMENT_CONFIG, + "表名的映射关系配置有误，不允许将表[" + tableName + "]映射为空"); + } + } + } + return assignmentConfigEntity; } diff --git a/dbswitch-admin/src/main/java/com/gitee/dbswitch/admin/model/request/AssigmentUpdateRequest.java b/dbswitch-admin/src/main/java/com/gitee/dbswitch/admin/model/request/AssigmentUpdateRequest.java index 1b566daa2d5ce416844a1ab405e0f4a186ebd73c..7af39019570a12f7c2e7f239522b3c0a2586dc0a 100644 --- a/dbswitch-admin/src/main/java/com/gitee/dbswitch/admin/model/request/AssigmentUpdateRequest.java +++ b/dbswitch-admin/src/main/java/com/gitee/dbswitch/admin/model/request/AssigmentUpdateRequest.java @@ -13,9 +13,9 @@ import com.gitee.dbswitch.admin.common.excption.DbswitchException; import com.gitee.dbswitch.admin.common.response.ResultCode; import com.gitee.dbswitch.admin.entity.AssignmentConfigEntity; import com.gitee.dbswitch.admin.entity.AssignmentTaskEntity; -import com.gitee.dbswitch.admin.service.ScheduleService; import com.gitee.dbswitch.admin.type.IncludeExcludeEnum; import com.gitee.dbswitch.admin.type.ScheduleModeEnum; +import com.gitee.dbswitch.admin.util.CronExprUtils; import com.gitee.dbswitch.common.entity.PatternMapper; import java.util.List; import java.util.Objects; @@ -56,11 +56,7 @@ public class AssigmentUpdateRequest { assignment.setDescription(description); assignment.setScheduleMode(scheduleMode); if (ScheduleModeEnum.SYSTEM_SCHEDULED == this.getScheduleMode()) { - if (!ScheduleService.checkCronExpressionValid(this.getCronExpression())) { - throw new DbswitchException(ResultCode.ERROR_INVALID_ARGUMENT, - "CRON表达式[" + this.getCronExpression() + "]"); - } - + CronExprUtils.checkCronExpressionValid(this.getCronExpression(), 120); assignment.setCronExpression(this.getCronExpression()); } diff --git a/dbswitch-admin/src/main/java/com/gitee/dbswitch/admin/service/AssignmentService.java b/dbswitch-admin/src/main/java/com/gitee/dbswitch/admin/service/AssignmentService.java index 5e1399f8f113729f24c7b0f81048406c864c560a..8bd652580c688fde1d9687c2ecf4eb146d7a59bb 100644 --- a/dbswitch-admin/src/main/java/com/gitee/dbswitch/admin/service/AssignmentService.java +++ b/dbswitch-admin/src/main/java/com/gitee/dbswitch/admin/service/AssignmentService.java @@ -41,8 +41,11 @@ import java.util.function.Supplier; import java.util.stream.Collectors; import javax.annotation.Resource; import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.lang.StringUtils; +import org.quartz.CronExpression; import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; +import org.quartz.CronExpression; @Service public class AssignmentService { diff --git a/dbswitch-admin/src/main/java/com/gitee/dbswitch/admin/service/JobExecutorService.java b/dbswitch-admin/src/main/java/com/gitee/dbswitch/admin/service/JobExecutorService.java index 4e76ec0ff96c763f71d0f44e4e611513a02e99a1..906be8fa0a84d9a7e7b53acc1d1b17b63d0f45e5 100644 --- a/dbswitch-admin/src/main/java/com/gitee/dbswitch/admin/service/JobExecutorService.java +++ b/dbswitch-admin/src/main/java/com/gitee/dbswitch/admin/service/JobExecutorService.java @@ -118,7 +118,10 @@ public class JobExecutorService extends QuartzJobBean implements InterruptableJo try { ReentrantLock lock = mutexes.get(taskId.toString(), ReentrantLock::new); - lock.lock(); + while (!lock.tryLock(1, TimeUnit.SECONDS)) { + TimeUnit.SECONDS.sleep(1); + } + try { log.info("Execute Quartz Job, and task id is : {} , job id is: {}", taskId, assignmentJobEntity.getId()); @@ -141,6 +144,12 @@ public class JobExecutorService extends QuartzJobBean implements InterruptableJo } MigrationService mainService = new MigrationService(properties); + if (interrupted) { + log.info("Quartz task id:{} interrupted", jobDataMap.getLong(TASK_ID)); + return; + } + + // 实际执行JOB mainService.run(); if (assignmentConfigEntity.getFirstFlag()) { @@ -166,7 +175,7 @@ public class JobExecutorService extends QuartzJobBean implements InterruptableJo } finally { lock.unlock(); } - } catch (ExecutionException e) { + } catch (ExecutionException | InterruptedException e) { throw new RuntimeException(e); } diff --git a/dbswitch-admin/src/main/java/com/gitee/dbswitch/admin/service/PatternMapperService.java b/dbswitch-admin/src/main/java/com/gitee/dbswitch/admin/service/PatternMapperService.java index 026d6de4ef71916ced1f4e9349d301a84f322b39..93e134d5e2a5941a569723e176c6a78d3bb19cdc 100644 --- a/dbswitch-admin/src/main/java/com/gitee/dbswitch/admin/service/PatternMapperService.java +++ b/dbswitch-admin/src/main/java/com/gitee/dbswitch/admin/service/PatternMapperService.java @@ -22,6 +22,9 @@ import org.springframework.stereotype.Service; @Service public class PatternMapperService { + private final String STRING_EMPTY = ""; + private final String STRING_DELETE = ""; + @Resource private DbConnectionService connectionService; @@ -34,18 +37,22 @@ public class PatternMapperService { List result = new ArrayList<>(); if (CollectionUtils.isEmpty(request.getTableNames())) { for (TableDescription td : getAllTableNames(request)) { + String targetName = PatterNameUtils.getFinalName( + td.getTableName(), request.getNameMapper()); result.add(PreviewNameMapperResponse.builder() .originalName(td.getTableName()) - .targetName(PatterNameUtils.getFinalName(td.getTableName(), request.getNameMapper())) + .targetName(StringUtils.isNotBlank(targetName) ? targetName : STRING_EMPTY) .build()); } } else { if (include) { for (String name : request.getTableNames()) { if (StringUtils.isNotBlank(name)) { + String targetName = PatterNameUtils.getFinalName( + name, request.getNameMapper()); result.add(PreviewNameMapperResponse.builder() .originalName(name) - .targetName(PatterNameUtils.getFinalName(name, request.getNameMapper())) + .targetName(StringUtils.isNotBlank(targetName) ? targetName : STRING_EMPTY) .build()); } } @@ -83,10 +90,18 @@ public class PatternMapperService { dbConn.getUrl(), dbConn.getUsername(), dbConn.getPassword(), request.getSchemaName(), request.getTableName()); for (ColumnDescription cd : tables) { - result.add(PreviewNameMapperResponse.builder() - .originalName(cd.getFieldName()) - .targetName(PatterNameUtils.getFinalName(cd.getFieldName(), request.getNameMapper())) - .build()); + String targetName = PatterNameUtils.getFinalName(cd.getFieldName(), request.getNameMapper()); + if (StringUtils.isNotBlank(targetName)) { + result.add(PreviewNameMapperResponse.builder() + .originalName(cd.getFieldName()) + .targetName(targetName) + .build()); + } else { + result.add(PreviewNameMapperResponse.builder() + .originalName(cd.getFieldName()) + .targetName(STRING_DELETE) + .build()); + } } return Result.success(result); @@ -102,14 +117,10 @@ public class PatternMapperService { if (null == dbConn) { throw new DbswitchException(ResultCode.ERROR_RESOURCE_NOT_EXISTS, "id=" + request.getId()); } + IMetaDataByJdbcService service = connectionService.getMetaDataCoreService(dbConn); - return service.queryTableList( - dbConn.getUrl(), - dbConn.getUsername(), - dbConn.getPassword(), - request.getSchemaName() - ).stream() - .filter(td -> !td.isViewTable()) + return service.queryTableList(dbConn.getUrl(), dbConn.getUsername(), dbConn.getPassword(), + request.getSchemaName()).stream().filter(td -> !td.isViewTable()) .collect(Collectors.toList()); } diff --git a/dbswitch-admin/src/main/java/com/gitee/dbswitch/admin/service/ScheduleService.java b/dbswitch-admin/src/main/java/com/gitee/dbswitch/admin/service/ScheduleService.java index 8e0b138a1ffb7c19b090e00ec44daf7859be35a7..e38cf3dc176e0c357070fd31a3fe44c2b1d73461 100644 --- a/dbswitch-admin/src/main/java/com/gitee/dbswitch/admin/service/ScheduleService.java +++ b/dbswitch-admin/src/main/java/com/gitee/dbswitch/admin/service/ScheduleService.java @@ -55,16 +55,6 @@ public class ScheduleService { @Resource private AssignmentJobDAO assignmentJobDAO; - public static boolean checkCronExpressionValid(String cronExpression) { - try { - CronScheduleBuilder.cronSchedule(cronExpression); - return true; - } catch (Exception e) { - return false; - } - - } - // public Trigger getQuartzJobDetail(String jobKey) { // Scheduler scheduler = schedulerFactoryBean.getScheduler(); // diff --git a/dbswitch-admin/src/main/java/com/gitee/dbswitch/admin/util/CronExprUtils.java b/dbswitch-admin/src/main/java/com/gitee/dbswitch/admin/util/CronExprUtils.java new file mode 100644 index 0000000000000000000000000000000000000000..8301d9a2cc3d3b7856c651abb1ea97fd12344300 --- /dev/null +++ b/dbswitch-admin/src/main/java/com/gitee/dbswitch/admin/util/CronExprUtils.java @@ -0,0 +1,54 @@ +// Copyright tang. All rights reserved. +// https://gitee.com/inrgihc/dbswitch +// +// Use of this source code is governed by a BSD-style license +// +// Author: tang (inrgihc@126.com) +// Date : 2020/1/2 +// Location: beijing , china +///////////////////////////////////////////////////////////// +package com.gitee.dbswitch.admin.util; + +import com.gitee.dbswitch.admin.common.excption.DbswitchException; +import com.gitee.dbswitch.admin.common.response.ResultCode; +import java.text.ParseException; +import java.util.Date; +import org.apache.commons.lang.StringUtils; +import org.quartz.CronExpression; + +/** + * CRON表达式工具类 + */ +public final class CronExprUtils { + + /** + * 检查CRON表达式的有效性 + * + * @param cronExpression CRON表达式 + * @param minIntervalSeconds 最小间隔时间（单位：秒） + */ + public static void checkCronExpressionValid(String cronExpression, int minIntervalSeconds) { + if (StringUtils.isNotBlank(cronExpression)) { + CronExpression expression; + try { + expression = new CronExpression(cronExpression); + } catch (ParseException e) { + throw new DbswitchException(ResultCode.ERROR_INVALID_ARGUMENT, String.format("正则表达式%s无效")); + } + Date nextDate = expression.getNextValidTimeAfter(new Date(System.currentTimeMillis())); + if (null == nextDate) { + throw new DbswitchException(ResultCode.ERROR_INVALID_ARGUMENT, + String.format("cron表达式[%s]不可以在历史时间运行", cronExpression)); + } + Date calculateDate = expression.getNextValidTimeAfter(new Date(nextDate.getTime() + 1)); + if (null != calculateDate) { + long intervalSeconds = (calculateDate.getTime() - nextDate.getTime()) / 1000; + if (intervalSeconds < minIntervalSeconds) { + throw new DbswitchException(ResultCode.ERROR_INVALID_ARGUMENT, + String.format("cron表达式[%s]运行间隔时间为%d秒, 小于设定的阈值 [%s秒]", + cronExpression, intervalSeconds, minIntervalSeconds)); + } + } + } + } +} diff --git a/dbswitch-admin/src/main/java/com/gitee/dbswitch/admin/util/JsonUtils.java b/dbswitch-admin/src/main/java/com/gitee/dbswitch/admin/util/JsonUtils.java index f8b14de0b1b8f74b2e0f2f8051e78d6fb6137a45..1d96b8869a9eca739f2bee356b9111784c18cf4e 100644 --- a/dbswitch-admin/src/main/java/com/gitee/dbswitch/admin/util/JsonUtils.java +++ b/dbswitch-admin/src/main/java/com/gitee/dbswitch/admin/util/JsonUtils.java @@ -17,6 +17,9 @@ import java.util.List; import java.util.Objects; import lombok.extern.slf4j.Slf4j; +/** + * JSON序列化与反序列化工具类 + */ @Slf4j public final class JsonUtils { diff --git a/dbswitch-admin/src/main/java/com/gitee/dbswitch/admin/util/PageUtils.java b/dbswitch-admin/src/main/java/com/gitee/dbswitch/admin/util/PageUtils.java index b7cdc4293802e94bc99cf9a312a07433daff7b50..069d58e7e7c87fed8a90fd831760d70a2958da39 100644 --- a/dbswitch-admin/src/main/java/com/gitee/dbswitch/admin/util/PageUtils.java +++ b/dbswitch-admin/src/main/java/com/gitee/dbswitch/admin/util/PageUtils.java @@ -16,6 +16,9 @@ import java.util.ArrayList; import java.util.List; import java.util.function.Supplier; +/** + * 分页工具类 + */ public class PageUtils { public static PageResult getPage(Supplier> method, Integer pageNum, diff --git a/dbswitch-admin/src/main/java/com/gitee/dbswitch/admin/util/PasswordUtils.java b/dbswitch-admin/src/main/java/com/gitee/dbswitch/admin/util/PasswordUtils.java index 6cdb62442e42e300cd09a3ed34b5e5501f22d749..364eafedb0c834ef5fb6ccbf28638eb6ab4eef38 100644 --- a/dbswitch-admin/src/main/java/com/gitee/dbswitch/admin/util/PasswordUtils.java +++ b/dbswitch-admin/src/main/java/com/gitee/dbswitch/admin/util/PasswordUtils.java @@ -11,6 +11,9 @@ package com.gitee.dbswitch.admin.util; import cn.hutool.crypto.digest.BCrypt; +/** + * 密码工具类 + */ public final class PasswordUtils { public static String encryptPassword(String password, String credentialsSalt) { diff --git a/dbswitch-admin/src/main/java/com/gitee/dbswitch/admin/util/ServletUtils.java b/dbswitch-admin/src/main/java/com/gitee/dbswitch/admin/util/ServletUtils.java index 539a6aa7f7d8669e84e0690857633499b481b5ca..a9c5135c3f7620411e3c93b5b4acac5f0c96e1ee 100644 --- a/dbswitch-admin/src/main/java/com/gitee/dbswitch/admin/util/ServletUtils.java +++ b/dbswitch-admin/src/main/java/com/gitee/dbswitch/admin/util/ServletUtils.java @@ -15,6 +15,9 @@ import org.springframework.util.StringUtils; import org.springframework.web.context.request.RequestContextHolder; import org.springframework.web.context.request.ServletRequestAttributes; +/** + * 获取Servlet服务器的HTTP参数相关工具类 + */ @Slf4j public class ServletUtils { @@ -74,7 +77,7 @@ public class ServletUtils { ip = request.getRemoteAddr(); } } catch (Exception e) { - log.error("IPUtils ERROR ", e); + log.error("get client IP address error: ", e); } return ip; diff --git a/dbswitch-admin/src/main/java/com/gitee/dbswitch/admin/util/SpringUtils.java b/dbswitch-admin/src/main/java/com/gitee/dbswitch/admin/util/SpringUtils.java index 88dec89e7864a77e810aae1806a3e1d619b1ed32..30475d31b4f5b6c37657fc7b9bd43cd5c16e4fd3 100644 --- a/dbswitch-admin/src/main/java/com/gitee/dbswitch/admin/util/SpringUtils.java +++ b/dbswitch-admin/src/main/java/com/gitee/dbswitch/admin/util/SpringUtils.java @@ -14,6 +14,9 @@ import org.springframework.context.ApplicationContext; import org.springframework.context.ApplicationContextAware; import org.springframework.stereotype.Component; +/** + * Spring容器获取BEAN工具类 + */ @Component public class SpringUtils implements ApplicationContextAware { diff --git a/dbswitch-admin/src/main/java/com/gitee/dbswitch/admin/util/UuidUtils.java b/dbswitch-admin/src/main/java/com/gitee/dbswitch/admin/util/UuidUtils.java index 84a6fb9f5d10fef253248a0618a760ad972496bb..cdb10392c28e53ac1b95a7885a60fc1280857ac9 100644 --- a/dbswitch-admin/src/main/java/com/gitee/dbswitch/admin/util/UuidUtils.java +++ b/dbswitch-admin/src/main/java/com/gitee/dbswitch/admin/util/UuidUtils.java @@ -11,6 +11,9 @@ package com.gitee.dbswitch.admin.util; import java.util.UUID; +/** + * UUID工具类 + */ public class UuidUtils { public static String generateUuid() { diff --git a/dbswitch-common/pom.xml b/dbswitch-common/pom.xml index 9144ce9467a24d2d5b2e4637da198692c697a197..e17e60901d4c9495e62429a1a0103872bbcd4484 100644 --- a/dbswitch-common/pom.xml +++ b/dbswitch-common/pom.xml @@ -5,7 +5,7 @@ com.gitee.dbswitch dbswitch-parent - 1.6.8 + 1.6.9 dbswitch-common diff --git a/dbswitch-core/pom.xml b/dbswitch-core/pom.xml index a12f6b4b3855e3b7189bcac19b36308f2128fdb4..7f5b9b7b70b29fcbe290fed4b07e91142f27307d 100644 --- a/dbswitch-core/pom.xml +++ b/dbswitch-core/pom.xml @@ -5,7 +5,7 @@ com.gitee.dbswitch dbswitch-parent - 1.6.8 + 1.6.9 dbswitch-core diff --git a/dbswitch-core/src/main/java/com/gitee/dbswitch/core/database/impl/DatabaseDB2Impl.java b/dbswitch-core/src/main/java/com/gitee/dbswitch/core/database/impl/DatabaseDB2Impl.java index e62aef5c952c1676a4bdf408e80da272fb0f39dc..1024aecbd41f1f8655387786835fbb2a506e688a 100644 --- a/dbswitch-core/src/main/java/com/gitee/dbswitch/core/database/impl/DatabaseDB2Impl.java +++ b/dbswitch-core/src/main/java/com/gitee/dbswitch/core/database/impl/DatabaseDB2Impl.java @@ -141,7 +141,7 @@ public class DatabaseDB2Impl extends AbstractDatabase implements IDatabaseInterf retval += "DATE"; break; case ColumnMetaData.TYPE_BOOLEAN: - retval += "CHARACTER(32)"; + retval += "BOOLEAN"; break; case ColumnMetaData.TYPE_NUMBER: case ColumnMetaData.TYPE_BIGNUMBER: diff --git a/dbswitch-core/src/main/java/com/gitee/dbswitch/core/database/impl/DatabaseDmImpl.java b/dbswitch-core/src/main/java/com/gitee/dbswitch/core/database/impl/DatabaseDmImpl.java index 30bf40e94ee329d838101e14cde7b70a3dd2f1c2..e3dd16ad0e9412f5c736cce9f72d102eb3777552 100644 --- a/dbswitch-core/src/main/java/com/gitee/dbswitch/core/database/impl/DatabaseDmImpl.java +++ b/dbswitch-core/src/main/java/com/gitee/dbswitch/core/database/impl/DatabaseDmImpl.java @@ -116,7 +116,7 @@ public class DatabaseDmImpl extends AbstractDatabase implements IDatabaseInterfa retval.append("DATE"); break; case ColumnMetaData.TYPE_BOOLEAN: - retval.append("VARCHAR(32)"); + retval.append("BIT"); break; case ColumnMetaData.TYPE_NUMBER: case ColumnMetaData.TYPE_BIGNUMBER: diff --git a/dbswitch-core/src/main/java/com/gitee/dbswitch/core/database/impl/DatabaseGreenplumImpl.java b/dbswitch-core/src/main/java/com/gitee/dbswitch/core/database/impl/DatabaseGreenplumImpl.java index e68e0ed0762ef99ec2ac14f363f572782de7cee5..39476862ee34ce02c9217c8ca12b3d538723baf8 100644 --- a/dbswitch-core/src/main/java/com/gitee/dbswitch/core/database/impl/DatabaseGreenplumImpl.java +++ b/dbswitch-core/src/main/java/com/gitee/dbswitch/core/database/impl/DatabaseGreenplumImpl.java @@ -139,7 +139,7 @@ public class DatabaseGreenplumImpl extends AbstractDatabase implements IDatabase retval += "DATE"; break; case ColumnMetaData.TYPE_BOOLEAN: - retval += "VARCHAR(32)"; + retval += "BOOLEAN"; break; case ColumnMetaData.TYPE_NUMBER: case ColumnMetaData.TYPE_INTEGER: diff --git a/dbswitch-core/src/main/java/com/gitee/dbswitch/core/database/impl/DatabaseKingbaseImpl.java b/dbswitch-core/src/main/java/com/gitee/dbswitch/core/database/impl/DatabaseKingbaseImpl.java index cc88648055926ed9e91606e0708cf971e0389091..e7dc34bb1bcf86c542ec5189c2a6b76c797faa43 100644 --- a/dbswitch-core/src/main/java/com/gitee/dbswitch/core/database/impl/DatabaseKingbaseImpl.java +++ b/dbswitch-core/src/main/java/com/gitee/dbswitch/core/database/impl/DatabaseKingbaseImpl.java @@ -118,7 +118,7 @@ public class DatabaseKingbaseImpl extends AbstractDatabase implements IDatabaseI retval += "DATE"; break; case ColumnMetaData.TYPE_BOOLEAN: - retval += "VARCHAR(32)"; + retval += "BOOLEAN"; break; case ColumnMetaData.TYPE_NUMBER: case ColumnMetaData.TYPE_INTEGER: diff --git a/dbswitch-core/src/main/java/com/gitee/dbswitch/core/database/impl/DatabaseMysqlImpl.java b/dbswitch-core/src/main/java/com/gitee/dbswitch/core/database/impl/DatabaseMysqlImpl.java index abcabfe59dcd1e2287b2bf7ac2c6840ef280f6c2..304a2bdafeb23385a316576486c4d3fb71723d6f 100644 --- a/dbswitch-core/src/main/java/com/gitee/dbswitch/core/database/impl/DatabaseMysqlImpl.java +++ b/dbswitch-core/src/main/java/com/gitee/dbswitch/core/database/impl/DatabaseMysqlImpl.java @@ -180,7 +180,7 @@ public class DatabaseMysqlImpl extends AbstractDatabase implements IDatabaseInte retval += "DATE"; break; case ColumnMetaData.TYPE_BOOLEAN: - retval += "VARCHAR(32)"; + retval += "TINYINT"; break; case ColumnMetaData.TYPE_NUMBER: case ColumnMetaData.TYPE_INTEGER: diff --git a/dbswitch-core/src/main/java/com/gitee/dbswitch/core/database/impl/DatabaseOracleImpl.java b/dbswitch-core/src/main/java/com/gitee/dbswitch/core/database/impl/DatabaseOracleImpl.java index 597d11a67119c32e8d1fcd010b129f25c87b2ec3..62447d873f62518e9aee9419b67d2d8ee6cb920e 100644 --- a/dbswitch-core/src/main/java/com/gitee/dbswitch/core/database/impl/DatabaseOracleImpl.java +++ b/dbswitch-core/src/main/java/com/gitee/dbswitch/core/database/impl/DatabaseOracleImpl.java @@ -185,7 +185,7 @@ public class DatabaseOracleImpl extends AbstractDatabase implements IDatabaseInt retval.append("DATE"); break; case ColumnMetaData.TYPE_BOOLEAN: - retval.append("VARCHAR(32)"); + retval.append("NUMBER(1)"); break; case ColumnMetaData.TYPE_NUMBER: case ColumnMetaData.TYPE_BIGNUMBER: diff --git a/dbswitch-core/src/main/java/com/gitee/dbswitch/core/database/impl/DatabasePostgresImpl.java b/dbswitch-core/src/main/java/com/gitee/dbswitch/core/database/impl/DatabasePostgresImpl.java index 5f8882811e3d31d6b403f9801551c08ed0336b3f..faa2f6639cb00160fa9524d844619fd5c2ca23be 100644 --- a/dbswitch-core/src/main/java/com/gitee/dbswitch/core/database/impl/DatabasePostgresImpl.java +++ b/dbswitch-core/src/main/java/com/gitee/dbswitch/core/database/impl/DatabasePostgresImpl.java @@ -154,7 +154,7 @@ public class DatabasePostgresImpl extends AbstractDatabase implements IDatabaseI retval += "DATE"; break; case ColumnMetaData.TYPE_BOOLEAN: - retval += "VARCHAR(32)"; + retval += "BOOLEAN"; break; case ColumnMetaData.TYPE_NUMBER: case ColumnMetaData.TYPE_INTEGER: diff --git a/dbswitch-core/src/main/java/com/gitee/dbswitch/core/database/impl/DatabaseSqlserverImpl.java b/dbswitch-core/src/main/java/com/gitee/dbswitch/core/database/impl/DatabaseSqlserverImpl.java index 5f90f0fe3016a1e8682ee33dfdf60c16fec10768..b192bc5e94cb16471805accb86341c5f48ee49b5 100644 --- a/dbswitch-core/src/main/java/com/gitee/dbswitch/core/database/impl/DatabaseSqlserverImpl.java +++ b/dbswitch-core/src/main/java/com/gitee/dbswitch/core/database/impl/DatabaseSqlserverImpl.java @@ -238,7 +238,7 @@ public class DatabaseSqlserverImpl extends AbstractDatabase implements IDatabase retval += "DATE"; break; case ColumnMetaData.TYPE_BOOLEAN: - retval += "VARCHAR(32)"; + retval += "BIT"; break; case ColumnMetaData.TYPE_NUMBER: case ColumnMetaData.TYPE_INTEGER: diff --git a/dbswitch-data/pom.xml b/dbswitch-data/pom.xml index c0d2577fc77bfe3d08fa7028d3347159975a5440..7f26be541a46b2e34aadd90eefa95c07099599fc 100644 --- a/dbswitch-data/pom.xml +++ b/dbswitch-data/pom.xml @@ -5,7 +5,7 @@ com.gitee.dbswitch dbswitch-parent - 1.6.8 + 1.6.9 dbswitch-data diff --git a/dbswitch-data/src/main/java/com/gitee/dbswitch/data/handler/MigrationHandler.java b/dbswitch-data/src/main/java/com/gitee/dbswitch/data/handler/MigrationHandler.java index d33ee0939d1970c38f591473240e51674985de66..039f38795c845d9edb3592659cc1e54b32f50c8f 100644 --- a/dbswitch-data/src/main/java/com/gitee/dbswitch/data/handler/MigrationHandler.java +++ b/dbswitch-data/src/main/java/com/gitee/dbswitch/data/handler/MigrationHandler.java @@ -46,6 +46,7 @@ import java.util.stream.Collectors; import lombok.extern.slf4j.Slf4j; import org.ehcache.sizeof.SizeOf; import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.util.StringUtils; /** * 在一个线程内的单表迁移处理逻辑 @@ -111,6 +112,10 @@ public class MigrationHandler implements Supplier { this.targetTableName = PatterNameUtils.getFinalName(td.getTableName(), sourceProperties.getRegexTableMapper()); + if (StringUtils.isEmpty(this.targetTableName)) { + throw new RuntimeException("表名的映射规则配置有误，不能将[" + this.sourceTableName + "]映射为空"); + } + this.tableNameMapString = String.format("%s.%s --> %s.%s", td.getSchemaName(), td.getTableName(), targetSchemaName, targetTableName); @@ -153,14 +158,28 @@ public class MigrationHandler implements Supplier { for (int i = 0; i < sourceColumnDescriptions.size(); ++i) { String sourceColumnName = sourceColumnDescriptions.get(i).getFieldName(); String targetColumnName = targetColumnDescriptions.get(i).getFieldName(); - columnMapperPairs.add(String.format("%s --> %s", sourceColumnName, targetColumnName)); - mapChecker.put(sourceColumnName, targetColumnName); + if (StringUtils.hasLength(targetColumnName)) { + columnMapperPairs.add(String.format("%s --> %s", sourceColumnName, targetColumnName)); + mapChecker.put(sourceColumnName, targetColumnName); + } else { + columnMapperPairs.add(String.format( + "%s --> %s", + sourceColumnName, + String.format("", (i + 1)) + )); + } } log.info("Mapping relation : \ntable mapper :\n\t{} \ncolumn mapper :\n\t{} ", - tableNameMapString, columnMapperPairs.stream().collect(Collectors.joining("\n\t"))); + tableNameMapString, String.join("\n\t", columnMapperPairs)); Set valueSet = new HashSet<>(mapChecker.values()); + if (valueSet.size() <= 0) { + throw new RuntimeException("字段映射配置有误，禁止通过映射将表所有的字段都删除!"); + } + if (!valueSet.containsAll(this.targetPrimaryKeys)) { + throw new RuntimeException("字段映射配置有误，禁止通过映射将表的主键字段删除!"); + } if (mapChecker.keySet().size() != valueSet.size()) { - throw new RuntimeException("字段映射配置有误，多个字段映射到一个同名字段!"); + throw new RuntimeException("字段映射配置有误，禁止将多个字段映射到一个同名字段!"); } IDatabaseWriter writer = DatabaseWriterFactory.createDatabaseWriter( @@ -183,8 +202,15 @@ public class MigrationHandler implements Supplier { // 生成建表语句并创建 String sqlCreateTable = sourceMetaDataService.getDDLCreateTableSQL( - targetProductType, targetColumnDescriptions, targetPrimaryKeys, - targetSchemaName, targetTableName, properties.getTarget().getCreateTableAutoIncrement()); + targetProductType, + targetColumnDescriptions.stream() + .filter(column -> StringUtils.hasLength(column.getFieldName())) + .collect(Collectors.toList()), + targetPrimaryKeys, + targetSchemaName, + targetTableName, + properties.getTarget().getCreateTableAutoIncrement() + ); JdbcTemplate targetJdbcTemplate = new JdbcTemplate(targetDataSource); targetJdbcTemplate.execute(sqlCreateTable); @@ -227,8 +253,18 @@ public class MigrationHandler implements Supplier { private Long doFullCoverSynchronize(IDatabaseWriter writer) { final int BATCH_SIZE = fetchSize; + List sourceFields = new ArrayList<>(); + List targetFields = new ArrayList<>(); + for (int i = 0; i < targetColumnDescriptions.size(); ++i) { + ColumnDescription scd = sourceColumnDescriptions.get(i); + ColumnDescription tcd = targetColumnDescriptions.get(i); + if (!StringUtils.isEmpty(tcd.getFieldName())) { + sourceFields.add(scd.getFieldName()); + targetFields.add(tcd.getFieldName()); + } + } // 准备目的端的数据写入操作 - writer.prepareWrite(targetSchemaName, targetTableName); + writer.prepareWrite(targetSchemaName, targetTableName, targetFields); // 清空目的端表的数据 IDatabaseOperator targetOperator = DatabaseOperatorFactory @@ -240,14 +276,9 @@ public class MigrationHandler implements Supplier { .createDatabaseOperator(sourceDataSource); sourceOperator.setFetchSize(BATCH_SIZE); - List sourceFields = sourceColumnDescriptions.stream() - .map(ColumnDescription::getFieldName) - .collect(Collectors.toList()); - List targetFields = targetColumnDescriptions.stream() - .map(ColumnDescription::getFieldName) - .collect(Collectors.toList()); StatementResultSet srs = sourceOperator.queryTableData( - sourceSchemaName, sourceTableName, sourceFields); + sourceSchemaName, sourceTableName, sourceFields + ); List cache = new LinkedList<>(); long cacheBytes = 0; @@ -306,16 +337,18 @@ public class MigrationHandler implements Supplier { */ private Long doIncreaseSynchronize(IDatabaseWriter writer) { final int BATCH_SIZE = fetchSize; - List sourceFields = sourceColumnDescriptions.stream() - .map(ColumnDescription::getFieldName) - .collect(Collectors.toList()); - List targetFields = targetColumnDescriptions.stream() - .map(ColumnDescription::getFieldName) - .collect(Collectors.toList()); + List sourceFields = new ArrayList<>(); + List targetFields = new ArrayList<>(); Map columnNameMaps = new HashMap<>(); - for (int i = 0; i < sourceFields.size(); ++i) { - columnNameMaps.put(sourceFields.get(i), targetFields.get(i)); + for (int i = 0; i < targetColumnDescriptions.size(); ++i) { + ColumnDescription scd = sourceColumnDescriptions.get(i); + ColumnDescription tcd = targetColumnDescriptions.get(i); + if (!StringUtils.isEmpty(tcd.getFieldName())) { + sourceFields.add(scd.getFieldName()); + targetFields.add(tcd.getFieldName()); + columnNameMaps.put(scd.getFieldName(), tcd.getFieldName()); + } } TaskParamEntity.TaskParamEntityBuilder taskBuilder = TaskParamEntity.builder(); diff --git a/dbswitch-dbchange/pom.xml b/dbswitch-dbchange/pom.xml index 81b06970af4390854e58990944dee313a09ba9a4..5e32307b9ab12128617da4b41507bc20137c6c01 100644 --- a/dbswitch-dbchange/pom.xml +++ b/dbswitch-dbchange/pom.xml @@ -5,7 +5,7 @@ com.gitee.dbswitch dbswitch-parent - 1.6.8 + 1.6.9 dbswitch-dbchange diff --git a/dbswitch-dbcommon/pom.xml b/dbswitch-dbcommon/pom.xml index b3500df363037eec2d243431d53542b1be79ad12..4abe143fd520e3baea21b15630b288665618f7e6 100644 --- a/dbswitch-dbcommon/pom.xml +++ b/dbswitch-dbcommon/pom.xml @@ -5,7 +5,7 @@ com.gitee.dbswitch dbswitch-parent - 1.6.8 + 1.6.9 dbswitch-dbcommon diff --git a/dbswitch-dbsynch/pom.xml b/dbswitch-dbsynch/pom.xml index 8f29acc69e50884edbbbf4a02803df162ac954c8..8da778dc416d9b0a5f5936b0056be7c63146679e 100644 --- a/dbswitch-dbsynch/pom.xml +++ b/dbswitch-dbsynch/pom.xml @@ -5,7 +5,7 @@ com.gitee.dbswitch dbswitch-parent - 1.6.8 + 1.6.9 dbswitch-dbsynch diff --git a/dbswitch-dbwriter/pom.xml b/dbswitch-dbwriter/pom.xml index e05a409c286d5ae92704949d10425e2bcd687a18..698e5ef1f088aa22bdebf08c33f52d250e6f98fb 100644 --- a/dbswitch-dbwriter/pom.xml +++ b/dbswitch-dbwriter/pom.xml @@ -5,7 +5,7 @@ com.gitee.dbswitch dbswitch-parent - 1.6.8 + 1.6.9 dbswitch-dbwriter diff --git a/dbswitch-dbwriter/src/main/java/com/gitee/dbswitch/dbwriter/AbstractDatabaseWriter.java b/dbswitch-dbwriter/src/main/java/com/gitee/dbswitch/dbwriter/AbstractDatabaseWriter.java index 332b5d06555ffeefd3e6bdc25a781109b673025b..f867f33278ef3d4e3f2601b25bf756ce5542ca5a 100644 --- a/dbswitch-dbwriter/src/main/java/com/gitee/dbswitch/dbwriter/AbstractDatabaseWriter.java +++ b/dbswitch-dbwriter/src/main/java/com/gitee/dbswitch/dbwriter/AbstractDatabaseWriter.java @@ -13,7 +13,6 @@ import java.sql.Connection; import java.sql.ResultSet; import java.sql.ResultSetMetaData; import java.sql.Statement; -import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.List; @@ -29,6 +28,7 @@ import org.springframework.transaction.TransactionDefinition; import org.springframework.transaction.TransactionException; import org.springframework.transaction.TransactionStatus; import org.springframework.transaction.support.DefaultTransactionDefinition; +import org.springframework.util.CollectionUtils; /** * 数据库写入抽象基类 @@ -58,8 +58,8 @@ public abstract class AbstractDatabaseWriter implements IDatabaseWriter { } @Override - public void prepareWrite(String schemaName, String tableName) { - String sql = this.selectTableMetaDataSqlString(schemaName, tableName); + public void prepareWrite(String schemaName, String tableName, List fieldNames) { + String sql = this.selectTableMetaDataSqlString(schemaName, tableName, fieldNames); Map columnMetaData = new HashMap<>(); jdbcTemplate.execute((Connection conn) -> { try (Statement stmt = conn.createStatement(); @@ -86,8 +86,14 @@ public abstract class AbstractDatabaseWriter implements IDatabaseWriter { } - protected String selectTableMetaDataSqlString(String schemaName, String tableName) { - return String.format("SELECT * FROM \"%s\".\"%s\" WHERE 1=2", schemaName, tableName); + protected String selectTableMetaDataSqlString(String schemaName, String tableName, + List fieldNames) { + if (CollectionUtils.isEmpty(fieldNames)) { + return String.format("SELECT * FROM \"%s\".\"%s\" WHERE 1=2", schemaName, tableName); + } else { + return String.format("SELECT \"%s\" FROM \"%s\".\"%s\" WHERE 1=2", + StringUtils.join(fieldNames, "\",\""), schemaName, tableName); + } } protected abstract String getDatabaseProductName(); diff --git a/dbswitch-dbwriter/src/main/java/com/gitee/dbswitch/dbwriter/IDatabaseWriter.java b/dbswitch-dbwriter/src/main/java/com/gitee/dbswitch/dbwriter/IDatabaseWriter.java index f20a47dd33dfe2e27d1adfbd04ea3dcdd5dbff59..b375cb6ad18cca715bff31c253135dda78108bc8 100644 --- a/dbswitch-dbwriter/src/main/java/com/gitee/dbswitch/dbwriter/IDatabaseWriter.java +++ b/dbswitch-dbwriter/src/main/java/com/gitee/dbswitch/dbwriter/IDatabaseWriter.java @@ -32,7 +32,7 @@ public interface IDatabaseWriter { * @param schemaName schema名称 * @param tableName table名称 */ - void prepareWrite(String schemaName, String tableName); + void prepareWrite(String schemaName, String tableName,List fieldNames); /** * 批量数据写入 diff --git a/dbswitch-dbwriter/src/main/java/com/gitee/dbswitch/dbwriter/mssql/SqlServerWriterImpl.java b/dbswitch-dbwriter/src/main/java/com/gitee/dbswitch/dbwriter/mssql/SqlServerWriterImpl.java index 45f1e42b93d48585e2b91ca35f1c897c4ea6e9d6..d0c4bb7274644539b6fbdb49bef008b0ddd93ed6 100644 --- a/dbswitch-dbwriter/src/main/java/com/gitee/dbswitch/dbwriter/mssql/SqlServerWriterImpl.java +++ b/dbswitch-dbwriter/src/main/java/com/gitee/dbswitch/dbwriter/mssql/SqlServerWriterImpl.java @@ -24,6 +24,7 @@ import org.springframework.transaction.TransactionDefinition; import org.springframework.transaction.TransactionException; import org.springframework.transaction.TransactionStatus; import org.springframework.transaction.support.DefaultTransactionDefinition; +import org.springframework.util.CollectionUtils; /** * SQLServer批量写入实现类 @@ -43,8 +44,14 @@ public class SqlServerWriterImpl extends AbstractDatabaseWriter implements IData } @Override - protected String selectTableMetaDataSqlString(String schemaName, String tableName) { - return String.format("SELECT * FROM [%s].[%s] WHERE 1=2", schemaName, tableName); + protected String selectTableMetaDataSqlString(String schemaName, String tableName, + List fieldNames) { + if (CollectionUtils.isEmpty(fieldNames)) { + return String.format("SELECT * FROM [%s].[%s] WHERE 1=2", schemaName, tableName); + } else { + return String.format("SELECT [%s] FROM [%s].[%s] WHERE 1=2", + StringUtils.join(fieldNames, "],["), schemaName, tableName); + } } @Override diff --git a/dbswitch-dbwriter/src/main/java/com/gitee/dbswitch/dbwriter/mysql/MySqlWriterImpl.java b/dbswitch-dbwriter/src/main/java/com/gitee/dbswitch/dbwriter/mysql/MySqlWriterImpl.java index d823d7b55fe263212898173911841bbf6bf42f12..e26aa763e7301de33ddfe17cedc7fd297431c341 100644 --- a/dbswitch-dbwriter/src/main/java/com/gitee/dbswitch/dbwriter/mysql/MySqlWriterImpl.java +++ b/dbswitch-dbwriter/src/main/java/com/gitee/dbswitch/dbwriter/mysql/MySqlWriterImpl.java @@ -32,6 +32,7 @@ import org.springframework.transaction.TransactionDefinition; import org.springframework.transaction.TransactionStatus; import org.springframework.transaction.support.DefaultTransactionDefinition; import org.springframework.transaction.support.TransactionTemplate; +import org.springframework.util.CollectionUtils; /** * MySQL数据库写入实现类 @@ -58,8 +59,14 @@ public class MySqlWriterImpl extends AbstractDatabaseWriter implements IDatabase } @Override - protected String selectTableMetaDataSqlString(String schemaName, String tableName) { - return String.format("SELECT * FROM `%s`.`%s` WHERE 1=2", schemaName, tableName); + protected String selectTableMetaDataSqlString(String schemaName, String tableName, + List fieldNames) { + if (CollectionUtils.isEmpty(fieldNames)) { + return String.format("SELECT * FROM `%s`.`%s` WHERE 1=2", schemaName, tableName); + } else { + return String.format("SELECT `%s` FROM `%s`.`%s` WHERE 1=2", + StringUtils.join(fieldNames, "`,`"), schemaName, tableName); + } } @Override diff --git a/dbswitch-pgwriter/pom.xml b/dbswitch-pgwriter/pom.xml index 34ee867f1f8298dfa4e34ff30129c6f9e555f766..ed757fbbbec39ae61f797b9bf82391e76b4b6d09 100644 --- a/dbswitch-pgwriter/pom.xml +++ b/dbswitch-pgwriter/pom.xml @@ -5,7 +5,7 @@ com.gitee.dbswitch dbswitch-parent - 1.6.8 + 1.6.9 dbswitch-pgwriter diff --git a/dbswitch-sql/pom.xml b/dbswitch-sql/pom.xml index 39fced8d4d8a59af7e722d3e4133f3ee2886e5ce..8bcb01feae78d550c50eb0cac87c0c8fc4a38f64 100644 --- a/dbswitch-sql/pom.xml +++ b/dbswitch-sql/pom.xml @@ -5,7 +5,7 @@ com.gitee.dbswitch dbswitch-parent - 1.6.8 + 1.6.9 dbswitch-sql diff --git a/package-tool/pom.xml b/package-tool/pom.xml index 9394977f6a101b124d1b92ee9de71b46961b245f..8cffc3b522d80ab08a3f4b998c3f5242ca7ac29b 100644 --- a/package-tool/pom.xml +++ b/package-tool/pom.xml @@ -5,7 +5,7 @@ com.gitee.dbswitch dbswitch-parent - 1.6.8 + 1.6.9 package-tool diff --git a/pom.xml b/pom.xml index 6668ac80712beafb7831545fae28ec94c67aba01..49cd994258ee7895ce48fe786e4f52a74bff68be 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ 4.0.0 com.gitee.dbswitch dbswitch-parent - 1.6.8 + 1.6.9 pom dbswitch database switch project diff --git a/version.cmd b/version.cmd index 5f3caeba0b3d3f08390bb381926d2714b063b19a..b3c276e867b7bde4f9cd96c3c280fda65b15457a 100644 --- a/version.cmd +++ b/version.cmd @@ -1,6 +1,6 @@ @echo off -set APP_VERSION=1.6.8 +set APP_VERSION=1.6.9 echo "Clean Project ..." call mvn clean -f pom.xml