https://github.com/cloudera-labs/hms-mirror/issues/194

Added Commandline Option: -cdcs --consolidate-db-create-statements that allows user to consolidate the db create statements.

Author: dstreev

pom.xml

@@ -28,7 +28,7 @@
 
     <groupId>com.cloudera.utils.hadoop</groupId>
     <artifactId>hms-mirror</artifactId>
-    <version>3.1.0.1</version>
+    <version>3.2.0.0</version>
     <packaging>jar</packaging>
 
     <name>hms-mirror</name>

src/main/java/com/cloudera/utils/hms/mirror/cli/HmsMirrorCommandLineOptions.java

@@ -1076,6 +1076,19 @@ CommandLineRunner configSkipLinkCheckTrue(HmsMirrorConfig hmsMirrorConfig) {
         };
     }
 
+    @Bean
+    @Order(1)
+    @ConditionalOnProperty(
+            name = "hms-mirror.config.database.consolidate-db-create-statements",
+            havingValue = "true")
+    CommandLineRunner configconsolidateDBCreate(HmsMirrorConfig hmsMirrorConfig) {
+        return args -> {
+            log.info("consolidate-db-create-statements: {}", Boolean.TRUE);
+            hmsMirrorConfig.setConsolidateDBCreateStatements(Boolean.TRUE);
+        };
+    }
+
+
     @Bean
     @Order(1)
     @ConditionalOnProperty(
@@ -1498,6 +1511,12 @@ private static Options getOptions() {
         flipOption.setRequired(Boolean.FALSE);
         options.addOption(flipOption);
 
+        Option cdcs = new Option("cdcs", "consolidate-db-create-statements", false,
+                "Database Create with Locations will be consolidated into a single CREATE DATABASE statement.");
+        cdcs.setOptionalArg(Boolean.FALSE);
+        cdcs.setRequired(Boolean.FALSE);
+        options.addOption(cdcs);
+
         Option smDistCpOption = new Option("dc", "distcp", false,
                 "Build the 'distcp' workplans.  Optional argument (PULL, PUSH) to define which cluster is running " +
                         "the distcp commands.  Default is PULL.");

src/main/java/com/cloudera/utils/hms/mirror/cli/HmsMirrorCommandLineOptionsEnum.java

@@ -29,6 +29,7 @@ public enum HmsMirrorCommandLineOptionsEnum {
     COMMENT("com", "comment", null, "Comments to add to report output."),
     CONCURRENCY("c", "concurrency", "threads", ""),
     CONSOLIDATE_TABLES_FOR_DISTCP("ctfd", "consolidate-tables-for-distcp", null, ""),
+    CONSOLIDATE_DB_CREATE_STATEMENTS("cdcs", "consolidate-db-create-statements", null, ""),
     CREATE_IF_NOT_EXIST("cine", "create-if-not-exist", null, ""),
     TARGET_NAMESPACE("tns", "target-namespace", "target", ""),
     COMPRESS_TEXT_OUTPUT("cto", "compress-text-output", null, ""),

src/main/java/com/cloudera/utils/hms/mirror/domain/HmsMirrorConfig.java

@@ -101,6 +101,9 @@ public class HmsMirrorConfig implements Cloneable {
     */
     private String dbPrefix = null;
     private String dbRename = null;
+
+    private boolean consolidateDBCreateStatements = Boolean.FALSE;
+
     private Environment dumpSource = Environment.LEFT;
     private boolean execute = Boolean.FALSE;
     /*

src/main/java/com/cloudera/utils/hms/mirror/service/DatabaseService.java

@@ -761,7 +761,6 @@ public boolean buildDBStatements(DBMirror dbMirror) {
                     }
                     dbMirror.getSql(Environment.LEFT).add(new Pair(CREATE_DB_DESC, sb.toString()));
 
-
                     // TODO: DB Properties.
 
                     break;
@@ -827,7 +826,7 @@ public boolean buildDBStatements(DBMirror dbMirror) {
                             altRightDB = Boolean.TRUE;
                         }
                     }
-
+                    StringBuilder sbDef = new StringBuilder();
                     // Create the DB on the RIGHT is it doesn't exist.
                     if (createRight) {
                         log.debug("Building RIGHT DB SQL for {} -> CREATE", dbMirror.getName());
@@ -837,7 +836,10 @@ public boolean buildDBStatements(DBMirror dbMirror) {
                         if (dbPropsLeft.get(COMMENT) != null && !dbPropsLeft.get(COMMENT).trim().isEmpty()) {
                             sbL.append(COMMENT).append(" \"").append(dbPropsLeft.get(COMMENT)).append("\"\n");
                         }
-                        dbMirror.getSql(Environment.RIGHT).add(new Pair(CREATE_DB_DESC, sbL.toString()));
+                        if (!config.isConsolidateDBCreateStatements()) {
+                            dbMirror.getSql(Environment.RIGHT).add(new Pair(CREATE_DB_DESC, sbL.toString()));
+                        }
+                        sbDef.append(sbL.toString());
                         log.trace("RIGHT DB Create SQL: {}", sbL);
                     }
 
@@ -846,7 +848,11 @@ public boolean buildDBStatements(DBMirror dbMirror) {
                         // If the original location is null or doesn't equal the target location, set it.
                         if (isNull(origRightLocation) || !origRightLocation.equals(targetLocation)) {
                             String alterDbLoc = MessageFormat.format(ALTER_DB_LOCATION, targetDatabase, targetLocation);
-                            dbMirror.getSql(Environment.RIGHT).add(new Pair(ALTER_DB_LOCATION_DESC, alterDbLoc));
+                            if (!config.isConsolidateDBCreateStatements()) {
+                                dbMirror.getSql(Environment.RIGHT).add(new Pair(ALTER_DB_LOCATION_DESC, alterDbLoc));
+                            } else {
+                                sbDef.append(DB_LOCATION).append(" \"").append(targetLocation).append("\"\n");
+                            }
                             dbPropsRight.put(DB_LOCATION, targetLocation);
                             log.trace("RIGHT DB Location SQL: {}", alterDbLoc);
                         }
@@ -856,18 +862,29 @@ public boolean buildDBStatements(DBMirror dbMirror) {
                         if (isNull(origRightManagedLocation) || !origRightManagedLocation.equals(targetManagedLocation)) {
                             if (!config.getCluster(Environment.RIGHT).isHdpHive3()) {
                                 String alterDbMngdLoc = MessageFormat.format(ALTER_DB_MNGD_LOCATION, targetDatabase, targetManagedLocation);
-                                dbMirror.getSql(Environment.RIGHT).add(new Pair(ALTER_DB_MNGD_LOCATION_DESC, alterDbMngdLoc));
+                                if (!config.isConsolidateDBCreateStatements()) {
+                                    dbMirror.getSql(Environment.RIGHT).add(new Pair(ALTER_DB_MNGD_LOCATION_DESC, alterDbMngdLoc));
+                                } else {
+                                    sbDef.append(DB_MANAGED_LOCATION).append(" \"").append(targetManagedLocation).append("\"\n");
+                                }
                                 dbPropsRight.put(DB_MANAGED_LOCATION, targetManagedLocation);
                                 log.trace("RIGHT DB Managed Location SQL: {}", alterDbMngdLoc);
                             } else {
                                 String alterDbMngdLoc = MessageFormat.format(ALTER_DB_LOCATION, targetDatabase, targetManagedLocation);
-                                dbMirror.getSql(Environment.RIGHT).add(new Pair(ALTER_DB_LOCATION_DESC, alterDbMngdLoc));
+                                if (!config.isConsolidateDBCreateStatements()) {
+                                    dbMirror.getSql(Environment.RIGHT).add(new Pair(ALTER_DB_LOCATION_DESC, alterDbMngdLoc));
+                                } else {
+                                    sbDef.append(DB_LOCATION).append(" \"").append(targetManagedLocation).append("\"\n");
+                                }
                                 dbMirror.addIssue(Environment.RIGHT, HDPHIVE3_DB_LOCATION.getDesc());
                                 dbPropsRight.put(DB_LOCATION, targetManagedLocation);
                                 log.trace("RIGHT DB Managed Location SQL: {}", alterDbMngdLoc);
                             }
                         }
                     }
+                    if (config.isConsolidateDBCreateStatements()) {
+                        dbMirror.getSql(Environment.RIGHT).add(new Pair(CREATE_DB_DESC, sbDef.toString()));
+                    }
 
                     // Build the DBPROPERITES
                     // Check if the user has specified any DB Properties to skip.

src/main/resources/messages.properties

@@ -325,6 +325,7 @@ config.misc=Miscellaneous
 config.save=Save Configuration
 config.copyAvroSchemaUrls=Copy Avro Schema URLs
 config.databaseOnly=Database Only
+config.consolidateDBCreateStatements=Consolidate DB Create Statements
 config.skipLinkCheck=Skip Link Check
 config.skipLinkCheck.tooltip=Skip the link check from the Right cluster to the Left Cluster
 config.dbPrefix=Database Prefix

src/main/resources/templates/fragments/config/misc.html

@@ -60,6 +60,13 @@
                        th:checked="${CONFIG.databaseOnly}"/>
             </div>
 
+            <div class="pure-control-group">
+                <label for="consolidateDBCreateStatements" th:text="#{config.consolidateDBCreateStatements}"></label>
+                <input type="checkbox" name="consolidateDBCreateStatements"
+                       th:field="*{consolidateDBCreateStatements}"
+                       th:checked="${CONFIG.consolidateDBCreateStatements}"/>
+            </div>
+
             <div class="pure-control-group">
                 <label for="readOnly" th:text="#{config.readOnly}"></label>
                 <input type="checkbox" name="readOnly"

src/test/java/com/cloudera/utils/hms/mirror/integration/end_to_end/cdp/Test_sm_smn_wd_cds.java

@@ -0,0 +1,80 @@
+/*
+ * Copyright (c) 2024. Cloudera, Inc. All Rights Reserved
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *        http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ *
+ */
+
+package com.cloudera.utils.hms.mirror.integration.end_to_end.cdp;
+
+import com.cloudera.utils.hms.mirror.cli.Mirror;
+import com.cloudera.utils.hms.mirror.integration.end_to_end.E2EBaseTest;
+import lombok.extern.slf4j.Slf4j;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.ExtendWith;
+import org.springframework.boot.test.context.SpringBootTest;
+import org.springframework.test.context.junit.jupiter.SpringExtension;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+@ExtendWith(SpringExtension.class)
+@SpringBootTest(classes = Mirror.class,
+        args = {
+                "--hms-mirror.config.data-strategy=STORAGE_MIGRATION",
+                "--hms-mirror.config.storage-migration-namespace=s3a://my_cs_bucket",
+                "--hms-mirror.config.warehouse-directory=/warehouse/managed",
+                "--hms-mirror.config.external-warehouse-directory=/warehouse/external",
+                "--hms-mirror.conversion.test-filename=/test_data/assorted_tbls_02.yaml",
+                "--hms-mirror.config.database.consolidate-db-create-statements=true",
+                "--hms-mirror.config.filename=/config/default.yaml.cdp",
+                "--hms-mirror.config.output-dir=${user.home}/.hms-mirror/test-output/e2e/cdp/sm_smn_wd_cds"
+        })
+@Slf4j
+public class Test_sm_smn_wd_cds extends E2EBaseTest {
+    //        String[] args = new String[]{
+//                "-d", "STORAGE_MIGRATION",
+//                "-smn", TARGET_NAMESPACE,
+//                "-wd", "/warehouse/managed_tables", "-ewd", "/warehouse/external_tables",
+//                "-ltd", ASSORTED_TBLS_04,
+//                "-cfg", CDP_CDP,
+//                "-o", outputDir
+//        };
+//
+//        long rtn = 0;
+//        MirrorLegacy mirror = new MirrorLegacy();
+//        rtn = mirror.go(args);
+//        int check = 0;
+//        assertEquals("Return Code Failure: " + rtn + " doesn't match: " + check * -1, check * -1, rtn);
+
+
+    @Test
+    public void returnCodeTest() {
+        // Get Runtime Return Code.
+        long rtn = getReturnCode();
+        // Verify the return code.
+        long check = 0L;
+        assertEquals(check, rtn, "Return Code Failure: " + rtn);
+    }
+
+//    @Test
+//    public void phaseTest() {
+//        validatePhase("ext_purge_odd_parts", "web_sales", PhaseState.CALCULATED_SQL);
+//    }
+//
+//    @Test
+//    public void issueTest() {
+//        validateTableIssueCount("ext_purge_odd_parts", "web_sales",
+//                Environment.LEFT, 17);
+//    }
+
+}

src/test/java/com/cloudera/utils/hms/mirror/integration/end_to_end/cdp_to_cdp/Test_sql_mao_cds.java

@@ -0,0 +1,64 @@
+/*
+ * Copyright (c) 2024. Cloudera, Inc. All Rights Reserved
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *        http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ *
+ */
+
+package com.cloudera.utils.hms.mirror.integration.end_to_end.cdp_to_cdp;
+
+import com.cloudera.utils.hms.mirror.cli.Mirror;
+import com.cloudera.utils.hms.mirror.integration.end_to_end.E2EBaseTest;
+import lombok.extern.slf4j.Slf4j;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.ExtendWith;
+import org.springframework.boot.test.context.SpringBootTest;
+import org.springframework.test.context.junit.jupiter.SpringExtension;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+@ExtendWith(SpringExtension.class)
+@SpringBootTest(classes = Mirror.class,
+        args = {
+                "--hms-mirror.config.data-strategy=SQL",
+                "--hms-mirror.config.migrate-acid-only=true",
+                "--hms-mirror.conversion.test-filename=/test_data/assorted_tbls_01.yaml",
+                "--hms-mirror.config.filename=/config/default.yaml.cdp-cdp",
+                "--hms-mirror.config.database.consolidate-db-create-statements=true",
+                "--hms-mirror.config.output-dir=${user.home}/.hms-mirror/test-output/e2e/cdp_cdp/sql_mao_cds"
+        })
+@Slf4j
+public class Test_sql_mao_cds extends E2EBaseTest {
+    //        String[] args = new String[]{"-d", "SQL",
+//                "-mao",
+//                "-ltd", ASSORTED_TBLS_04,
+//                "-cfg", CDP_CDP,
+//                "-o", outputDir
+//        };
+//
+//
+//        long rtn = 0;
+//        MirrorLegacy mirror = new MirrorLegacy();
+//        rtn = mirror.go(args);
+//        int check = 0;
+//        assertEquals("Return Code Failure: " + rtn + " doesn't match: " + check, rtn, check);
+
+    @Test
+    public void returnCodeTest() {
+        // Get Runtime Return Code.
+        long rtn = getReturnCode();
+        // Verify the return code.
+        long check = 0L;
+        assertEquals(check * -1, rtn, "Return Code Failure: " + rtn);
+    }
+}