Skip to content

Commit f1d1997

Browse files
dstreevdstreev
committed
Address threading issue when converting from Future<> to CompletableFuture<>.
1 parent 20d9f1b commit f1d1997

File tree

8 files changed

+261
-44
lines changed

8 files changed

+261
-44
lines changed

‎Writerside/topics/Release-Notes.md‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,22 @@ The latest set of enhancement requests can be found [here](https://github.com/cl
1010

1111
If there is something you'd like to see, add a new issue [here](https://github.com/cloudera-labs/hms-mirror/issues)
1212

13+
## 3.0.0.2
14+
15+
**Bug Fixes**
16+
[--sync not dropping table on right when left is missing. #189](https://github.com/cloudera-labs/hms-mirror/issues/189)
17+
18+
[When doing a Schema Sync for 'VIEWS', there aren't being dropped. #190](https://github.com/cloudera-labs/hms-mirror/issues/190)
19+
20+
[SHADOW table schema for partitioned tables are creating ALTER TABLE ... PARTITION location details with LEFT locations. #191](https://github.com/cloudera-labs/hms-mirror/issues/191)
21+
22+
[Add ability to save a comment for a run and record it in the report. #192](https://github.com/cloudera-labs/hms-mirror/issues/192)
23+
24+
[Themeleaf syntax causing excessive logging. #193](https://github.com/cloudera-labs/hms-mirror/issues/193)
25+
26+
- Removed Spring Autowiring
27+
- Function Documentation
28+
1329
## 3.0.0.1
1430

1531
This release is based on the 2.3.1.5 release and includes all the features and bug fixes from that release.

‎pom.xml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,7 @@
2828

2929
<groupId>com.cloudera.utils.hadoop</groupId>
3030
<artifactId>hms-mirror</artifactId>
31-
<version>3.0.0.2</version>
31+
<version>3.0.0.3</version>
3232
<packaging>jar</packaging>
3333

3434
<name>hms-mirror</name>

‎src/main/java/com/cloudera/utils/hms/mirror/cli/run/HmsMirrorAppCfg.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@
2828
import org.springframework.context.annotation.Configuration;
2929
import org.springframework.core.annotation.Order;
3030

31+
import java.util.concurrent.CompletableFuture;
3132
import java.util.concurrent.Future;
3233

3334
/*
@@ -82,7 +83,7 @@ public HmsMirrorAppCfg(
8283
public CommandLineRunner start() {
8384
return args -> {
8485
// NOTE: The transitionToActive process happens in another bean....
85-
Future<Boolean> result = hmsMirrorAppService.run();
86+
CompletableFuture<Boolean> result = hmsMirrorAppService.run();
8687
while (!result.isDone()) {
8788
try {
8889
Thread.sleep(1000);

‎src/main/java/com/cloudera/utils/hms/mirror/domain/support/RunStatus.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,7 @@
3636
import java.net.URL;
3737
import java.nio.charset.StandardCharsets;
3838
import java.util.*;
39+
import java.util.concurrent.CompletableFuture;
3940
import java.util.concurrent.Future;
4041

4142
import static java.util.Objects.isNull;
@@ -58,7 +59,7 @@ public class RunStatus implements Comparable<RunStatus>, Cloneable {
5859
// @JsonIgnore
5960
Integer concurrency;
6061
@JsonIgnore
61-
Future<Boolean> runningTask = null;
62+
CompletableFuture<Boolean> runningTask = null;
6263

6364
List<String> errorMessages = new ArrayList<>();
6465
List<String> warningMessages = new ArrayList<>();

‎src/main/java/com/cloudera/utils/hms/mirror/service/DatabaseService.java‎

Lines changed: 32 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -340,7 +340,6 @@ public boolean loadEnvironmentVars() {
340340
public List<String> listAvailableDatabases(Environment environment) {
341341
List<String> dbs = new ArrayList<>();
342342
Connection conn = null;
343-
HmsMirrorConfig config = executeSessionService.getSession().getConfig();
344343
try {
345344
conn = connectionPoolService.getHS2EnvironmentConnection(environment);
346345
if (conn != null) {
@@ -414,8 +413,8 @@ public boolean buildDBStatements(DBMirror dbMirror) {
414413
boolean skipManagedLocation = Boolean.FALSE;
415414
boolean forceLocations = Boolean.FALSE;
416415

417-
Map<String, String> dbDefLeft = dbMirror.getProperty(Environment.LEFT);
418-
Map<String, String> dbDefRight = dbMirror.getProperty(Environment.RIGHT);
416+
Map<String, String> dbPropsLeft = dbMirror.getProperty(Environment.LEFT);
417+
Map<String, String> dbPropsRight = dbMirror.getProperty(Environment.RIGHT);
419418

420419
switch (config.getDataStrategy()) {
421420
case DUMP:
@@ -438,9 +437,9 @@ public boolean buildDBStatements(DBMirror dbMirror) {
438437
config.getClusters().put(Environment.RIGHT, cluster);
439438
}
440439
// Build the Right Def as a Clone of the Left to Seed it.
441-
if (isNull(dbDefRight)) {
442-
dbDefRight = new TreeMap<String, String>();
443-
dbMirror.setProperty(Environment.RIGHT, dbDefRight);
440+
if (isNull(dbPropsRight)) {
441+
dbPropsRight = new TreeMap<String, String>();
442+
dbMirror.setProperty(Environment.RIGHT, dbPropsRight);
444443
}
445444

446445
break;
@@ -452,11 +451,11 @@ public boolean buildDBStatements(DBMirror dbMirror) {
452451
buildRight = Boolean.FALSE;
453452
}
454453
// Build the Right Def as a Clone of the Left to Seed it.
455-
if (isNull(dbDefRight)) {
454+
if (isNull(dbPropsRight)) {
456455
// No Right DB Definition. So we're going to create it.
457456
createRight = Boolean.TRUE;
458-
dbDefRight = new TreeMap<String, String>();
459-
dbMirror.setProperty(Environment.RIGHT, dbDefRight);
457+
dbPropsRight = new TreeMap<String, String>();
458+
dbMirror.setProperty(Environment.RIGHT, dbPropsRight);
460459
}
461460
// Force Locations to ensure DB and new tables are created in the right locations.
462461
forceLocations = Boolean.TRUE;
@@ -471,13 +470,13 @@ public boolean buildDBStatements(DBMirror dbMirror) {
471470
buildLeft = Boolean.FALSE;
472471
buildRight = Boolean.TRUE;
473472
// Build the Right Def as a Clone of the Left to Seed it.
474-
if (isNull(dbDefRight)) {
473+
if (isNull(dbPropsRight)) {
475474
// No Right DB Definition. So we're going to create it.
476475
createRight = Boolean.TRUE;
477476
// dbDefRight = new TreeMap<String, String>(dbDefLeft);
478-
dbDefRight = new TreeMap<String, String>();
479-
dbDefRight.put(DB_NAME, HmsMirrorConfigUtil.getResolvedDB(dbMirror.getName(), config));
480-
dbMirror.setProperty(Environment.RIGHT, dbDefRight);
477+
dbPropsRight = new TreeMap<String, String>();
478+
dbPropsRight.put(DB_NAME, HmsMirrorConfigUtil.getResolvedDB(dbMirror.getName(), config));
479+
dbMirror.setProperty(Environment.RIGHT, dbPropsRight);
481480
}
482481
// Force Locations to ensure DB and new tables are created in the right locations.
483482
if (config.getTransfer().getStorageMigration().getTranslationType() == TranslationTypeEnum.ALIGNED) {
@@ -592,7 +591,7 @@ public boolean buildDBStatements(DBMirror dbMirror) {
592591
"So we're not going to set it and let it default to the ENV warehouse location: {}.", targetLocation, envWarehouse.getExternalDirectory());
593592
// The new target location is the same as the ENV warehouse location. So we're not
594593
// going to set it and let it default to the ENV warehouse location.
595-
dbDefRight.put(DB_LOCATION, targetNamespace + targetLocation);
594+
dbPropsRight.put(DB_LOCATION, targetNamespace + targetLocation);
596595
dbMirror.addIssue(Environment.RIGHT, "The database location is the same as the ENV warehouse location. The database location will NOT be set and will depend on the ENV warehouse location.");
597596
targetLocation = null;
598597
}
@@ -651,7 +650,7 @@ public boolean buildDBStatements(DBMirror dbMirror) {
651650
log.debug("The new target managed location: {} is the same as the ENV warehouse managed location. " +
652651
"So we're not going to set it and let it default to the ENV warehouse managed location: {}.",
653652
targetManagedLocation, envWarehouse.getManagedDirectory());
654-
dbDefRight.put(DB_MANAGED_LOCATION, targetNamespace + targetManagedLocation);
653+
dbPropsRight.put(DB_MANAGED_LOCATION, targetNamespace + targetManagedLocation);
655654
dbMirror.addIssue(Environment.RIGHT, "The database 'Managed' location is the same as the ENV warehouse Managed location. The database location will NOT be set and will depend on the ENV warehouse location.");
656655
targetManagedLocation = null;
657656
}
@@ -675,7 +674,7 @@ public boolean buildDBStatements(DBMirror dbMirror) {
675674
}
676675

677676
// Upsert Source DB Parameters to Target DB Parameters.
678-
DatabaseUtils.upsertParameters(dbDefLeft, dbDefRight, skipList);
677+
DatabaseUtils.upsertParameters(dbPropsLeft, dbPropsRight, skipList);
679678

680679
// Deal with ReadOnly.
681680
if (config.isReadOnly()) {
@@ -749,16 +748,16 @@ public boolean buildDBStatements(DBMirror dbMirror) {
749748
String createDb = MessageFormat.format(CREATE_DB, targetDatabase);
750749
StringBuilder sb = new StringBuilder();
751750
sb.append(createDb).append("\n");
752-
if (dbDefLeft.get(COMMENT) != null && !dbDefLeft.get(COMMENT).trim().isEmpty()) {
753-
sb.append(COMMENT).append(" \"").append(dbDefLeft.get(COMMENT)).append("\"\n");
751+
if (dbPropsLeft.get(COMMENT) != null && !dbPropsLeft.get(COMMENT).trim().isEmpty()) {
752+
sb.append(COMMENT).append(" \"").append(dbPropsLeft.get(COMMENT)).append("\"\n");
754753
}
755754
if (nonNull(originalLocation)) {
756755
sb.append(DB_LOCATION).append(" \"").append(originalLocation).append("\"\n");
757-
dbDefLeft.put(DB_LOCATION, originalLocation);
756+
dbPropsLeft.put(DB_LOCATION, originalLocation);
758757
}
759758
if (nonNull(originalManagedLocation)) {
760759
sb.append(DB_MANAGED_LOCATION).append(" \"").append(originalManagedLocation).append("\"\n");
761-
dbDefLeft.put(DB_MANAGED_LOCATION, originalManagedLocation);
760+
dbPropsLeft.put(DB_MANAGED_LOCATION, originalManagedLocation);
762761
}
763762
dbMirror.getSql(Environment.LEFT).add(new Pair(CREATE_DB_DESC, sb.toString()));
764763

@@ -784,21 +783,21 @@ public boolean buildDBStatements(DBMirror dbMirror) {
784783
if (!isBlank(targetLocation)) {
785784
String alterDbLoc = MessageFormat.format(ALTER_DB_LOCATION, targetDatabase, targetLocation);
786785
dbMirror.getSql(Environment.LEFT).add(new Pair(ALTER_DB_LOCATION_DESC, alterDbLoc));
787-
dbDefRight.put(DB_LOCATION, targetLocation);
786+
dbPropsRight.put(DB_LOCATION, targetLocation);
788787
}
789788
}
790789
if (!config.getCluster(Environment.LEFT).isHdpHive3()) {
791790
if (!isBlank(targetManagedLocation)) {
792791
String alterDbMngdLoc = MessageFormat.format(ALTER_DB_MNGD_LOCATION, targetDatabase, targetManagedLocation);
793792
dbMirror.getSql(Environment.LEFT).add(new Pair(ALTER_DB_MNGD_LOCATION_DESC, alterDbMngdLoc));
794-
dbDefRight.put(DB_MANAGED_LOCATION, targetManagedLocation);
793+
dbPropsRight.put(DB_MANAGED_LOCATION, targetManagedLocation);
795794
}
796795
} else {
797796
if (!isBlank(targetManagedLocation)) {
798797
String alterDbMngdLoc = MessageFormat.format(ALTER_DB_LOCATION, targetDatabase, targetManagedLocation);
799798
dbMirror.getSql(Environment.LEFT).add(new Pair(ALTER_DB_LOCATION_DESC, alterDbMngdLoc));
800799
dbMirror.addIssue(Environment.LEFT, HDPHIVE3_DB_LOCATION.getDesc());
801-
dbDefRight.put(DB_LOCATION, targetManagedLocation);
800+
dbPropsRight.put(DB_LOCATION, targetManagedLocation);
802801
}
803802
}
804803

@@ -835,36 +834,36 @@ public boolean buildDBStatements(DBMirror dbMirror) {
835834
String createDbL = MessageFormat.format(CREATE_DB, targetDatabase);
836835
StringBuilder sbL = new StringBuilder();
837836
sbL.append(createDbL).append("\n");
838-
if (dbDefLeft.get(COMMENT) != null && !dbDefLeft.get(COMMENT).trim().isEmpty()) {
839-
sbL.append(COMMENT).append(" \"").append(dbDefLeft.get(COMMENT)).append("\"\n");
837+
if (dbPropsLeft.get(COMMENT) != null && !dbPropsLeft.get(COMMENT).trim().isEmpty()) {
838+
sbL.append(COMMENT).append(" \"").append(dbPropsLeft.get(COMMENT)).append("\"\n");
840839
}
841840
dbMirror.getSql(Environment.RIGHT).add(new Pair(CREATE_DB_DESC, sbL.toString()));
842841
log.trace("RIGHT DB Create SQL: {}", sbL);
843842
}
844843

845844
if (nonNull(targetLocation) && !config.getCluster(Environment.RIGHT).isHdpHive3()) {
846-
String origRightLocation = dbDefRight.get(DB_LOCATION);
845+
String origRightLocation = dbPropsRight.get(DB_LOCATION);
847846
// If the original location is null or doesn't equal the target location, set it.
848847
if (isNull(origRightLocation) || !origRightLocation.equals(targetLocation)) {
849848
String alterDbLoc = MessageFormat.format(ALTER_DB_LOCATION, targetDatabase, targetLocation);
850849
dbMirror.getSql(Environment.RIGHT).add(new Pair(ALTER_DB_LOCATION_DESC, alterDbLoc));
851-
dbDefRight.put(DB_LOCATION, targetLocation);
850+
dbPropsRight.put(DB_LOCATION, targetLocation);
852851
log.trace("RIGHT DB Location SQL: {}", alterDbLoc);
853852
}
854853
}
855854
if (nonNull(targetManagedLocation)) {
856-
String origRightManagedLocation = config.getCluster(Environment.RIGHT).isHdpHive3() ? dbDefRight.get(DB_LOCATION) : dbDefRight.get(DB_MANAGED_LOCATION);
855+
String origRightManagedLocation = config.getCluster(Environment.RIGHT).isHdpHive3() ? dbPropsRight.get(DB_LOCATION) : dbPropsRight.get(DB_MANAGED_LOCATION);
857856
if (isNull(origRightManagedLocation) || !origRightManagedLocation.equals(targetManagedLocation)) {
858857
if (!config.getCluster(Environment.RIGHT).isHdpHive3()) {
859858
String alterDbMngdLoc = MessageFormat.format(ALTER_DB_MNGD_LOCATION, targetDatabase, targetManagedLocation);
860859
dbMirror.getSql(Environment.RIGHT).add(new Pair(ALTER_DB_MNGD_LOCATION_DESC, alterDbMngdLoc));
861-
dbDefRight.put(DB_MANAGED_LOCATION, targetManagedLocation);
860+
dbPropsRight.put(DB_MANAGED_LOCATION, targetManagedLocation);
862861
log.trace("RIGHT DB Managed Location SQL: {}", alterDbMngdLoc);
863862
} else {
864863
String alterDbMngdLoc = MessageFormat.format(ALTER_DB_LOCATION, targetDatabase, targetManagedLocation);
865864
dbMirror.getSql(Environment.RIGHT).add(new Pair(ALTER_DB_LOCATION_DESC, alterDbMngdLoc));
866865
dbMirror.addIssue(Environment.RIGHT, HDPHIVE3_DB_LOCATION.getDesc());
867-
dbDefRight.put(DB_LOCATION, targetManagedLocation);
866+
dbPropsRight.put(DB_LOCATION, targetManagedLocation);
868867
log.trace("RIGHT DB Managed Location SQL: {}", alterDbMngdLoc);
869868
}
870869
}
@@ -873,7 +872,7 @@ public boolean buildDBStatements(DBMirror dbMirror) {
873872
// Build the DBPROPERITES
874873
// Check if the user has specified any DB Properties to skip.
875874
Set<String> lclSkipList = new HashSet<>(skipList);
876-
Map<String, String> dbProperties = DatabaseUtils.getParameters(dbDefRight, lclSkipList, config.getFilter().getDbPropertySkipListPattern());
875+
Map<String, String> dbProperties = DatabaseUtils.getParameters(dbPropsRight, lclSkipList, config.getFilter().getDbPropertySkipListPattern());
877876
if (!dbProperties.isEmpty()) {
878877
for (Map.Entry<String, String> entry : dbProperties.entrySet()) {
879878
String alterDbProps = MessageFormat.format(ALTER_DB_PROPERTIES, targetDatabase, entry.getKey(), entry.getValue());
@@ -883,8 +882,8 @@ public boolean buildDBStatements(DBMirror dbMirror) {
883882
}
884883

885884
if (config.getOwnershipTransfer().isDatabase()) {
886-
String ownerFromLeft = dbDefLeft.get(OWNER_NAME);
887-
String ownerTypeFromLeft = dbDefLeft.get(OWNER_TYPE);
885+
String ownerFromLeft = dbPropsLeft.get(OWNER_NAME);
886+
String ownerTypeFromLeft = dbPropsLeft.get(OWNER_TYPE);
888887
if (nonNull(ownerFromLeft) && nonNull(ownerTypeFromLeft)) {
889888
log.info("Setting Owner: {} of type: {} on Database: {}", ownerFromLeft, ownerTypeFromLeft, targetDatabase);
890889
if (ownerTypeFromLeft.equals("USER")) {

0 commit comments

Comments
 (0)