[--sync not dropping table on right when left is missing. #189](#189)

[When doing a Schema Sync for 'VIEWS', there aren't being dropped. #190](#190)

[SHADOW table schema for partitioned tables are creating ALTER TABLE ... PARTITION location details with LEFT locations. #191](#191)

[Add ability to save a comment for a run and record it in the report. #192](#192)

[Themeleaf syntax causing excessive logging. #193](#193)

- Removed Spring Autowiring
- Function Documentation

Author: dstreev

.gitignore

@@ -9,6 +9,7 @@
 | -c, --concurrency                          | <threads>                                      | Set application concurrency, default is 10 |
 | -cfg, --config                             | <filename>                                     | Config with details for the HMS-Mirror. Default: $HOME/.hms-mirror/cfg/default.yaml |
 | -cine, --create-if-not-exist               |                                                | CREATE table/partition statements will be adjusted to include 'IF NOT EXISTS'. This will ensure all remaining sql statements will be run. This can be used to sync partition definitions for existing tables. |
+| -com, --comment                            | <comment>                                      | Comment to add to report output |
 | -cs, --common-storage                      | <storage-path>                                 | Common Storage used with Data Strategy HYBRID, SQL, EXPORT_IMPORT. This will change the way these methods are implemented by using the specified storage location as an 'common' storage point between two clusters. In this case, the cluster do NOT need to be 'linked'. Each cluster DOES need to have access to the location and authorization to interact with the location. This may mean additional configuration requirements for 'hdfs' to ensure this seamless access. |
 | -cto, --compress-text-output               |                                                | Data movement (SQL/STORAGE_MIGRATION) of TEXT based file formats will be compressed in the new table. |
 | -d, --data-strategy                        | <strategy>                                     | Specify how the data will follow the schema. [DUMP, SCHEMA_ONLY, LINKED, SQL, EXPORT_IMPORT, HYBRID, CONVERT_LINKED, STORAGE_MIGRATION, COMMON, ICEBERG_CONVERSION] |

Writerside/topics/cli-options.md

@@ -4,5 +4,5 @@
 <ihp version="2.0">
     <topics dir="topics" web-path="topics"/>
     <images dir="images" web-path="images"/>
-    <instance src="hms-mirror.tree" web-path="/hms-mirror" version="v2.3.1.x"/>
+    <instance src="hms-mirror.tree" web-path="/hms-mirror" version="v3.x"/>
 </ihp>

Writerside/writerside.cfg

@@ -22,7 +22,7 @@
     <parent>
         <groupId>com.cloudera.utils</groupId>
         <artifactId>hadoop.parent</artifactId>
-        <version>1.3.0.1</version>
+        <version>1.4.0.0</version>
         <relativePath>../cloudera-utils</relativePath>
     </parent>
 

pom.xml

@@ -30,7 +30,6 @@
 import lombok.Getter;
 import lombok.Setter;
 import lombok.extern.slf4j.Slf4j;
-import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.boot.CommandLineRunner;
 import org.springframework.context.annotation.Bean;
 import org.springframework.core.annotation.Order;
@@ -59,27 +58,24 @@ public class CliReporter {
     private final List<String> reportTemplateOutput = new ArrayList<>();
     private final Map<String, String> varMap = new TreeMap<>();
     private final List<TableMirror> startedTables = new ArrayList<>();
+    private final ConfigService configService;
+    private final ExecuteSessionService executeSessionService;
+
     private Thread worker;
     private Boolean retry = Boolean.FALSE;
     private Boolean quiet = Boolean.FALSE;
-
-    private ConfigService configService;
-    private ExecuteSessionService executeSessionService;
-
     private boolean tiktok = false;
 
-    @Autowired
-    public void setConfigService(ConfigService configService) {
+    // Constructor injection for dependencies
+    public CliReporter(ConfigService configService, ExecuteSessionService executeSessionService) {
         this.configService = configService;
+        this.executeSessionService = executeSessionService;
     }
 
-
     @Bean
     @Order(20)
     CommandLineRunner configQuiet(HmsMirrorConfig hmsMirrorConfig) {
-        return args -> {
-            setQuiet(hmsMirrorConfig.isQuiet());
-        };
+        return args -> setQuiet(hmsMirrorConfig.isQuiet());
     }
 
     protected void displayReport(Boolean showAll) {
@@ -303,13 +299,8 @@ public void run() {
         }
     }
 
-    @Autowired
-    public void setExecuteSessionService(ExecuteSessionService executeSessionService) {
-        this.executeSessionService = executeSessionService;
-    }
-
     public void setVariable(String key, String value) {
         varMap.put(key, value);
     }
 
-}
+}

src/main/java/com/cloudera/utils/hms/mirror/cli/CliReporter.java

@@ -20,7 +20,6 @@
 import lombok.Getter;
 import lombok.Setter;
 import lombok.extern.slf4j.Slf4j;
-import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.boot.CommandLineRunner;
 import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
 import org.springframework.context.annotation.Bean;
@@ -32,9 +31,8 @@
 @Setter
 @Slf4j
 public class CliReporterConfig {
-    private CliReporter cliReporter;
+    private final CliReporter cliReporter;
 
-    @Autowired
     public CliReporterConfig(CliReporter cliReporter) {
         this.cliReporter = cliReporter;
     }
@@ -48,9 +46,7 @@ public CliReporterConfig(CliReporter cliReporter) {
     public CommandLineRunner launchCliReporting() {
         return args -> {
             log.info("Launching CLI Reporting");
-            getCliReporter().run();
+            cliReporter.run();
         };
     }
-
-
-}
+}

src/main/java/com/cloudera/utils/hms/mirror/cli/CliReporterConfig.java

@@ -28,7 +28,6 @@
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.cli.*;
 import org.apache.logging.log4j.util.Strings;
-import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.beans.factory.annotation.Value;
 import org.springframework.boot.CommandLineRunner;
 import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
@@ -51,19 +50,25 @@
 public class HmsMirrorCommandLineOptions {
     public static String SPRING_CONFIG_PREFIX = "hms-mirror.config";
 
-    private ConfigService configService;
+    private final ConfigService configService;
 
-    @Autowired
-    public void setConfigService(ConfigService configService) {
+    /**
+     * Constructor injection for HmsMirrorCommandLineOptions.
+     *
+     * @param configService the injected config service
+     * @param log injected logger
+     */
+    public HmsMirrorCommandLineOptions(ConfigService configService) {
         this.configService = configService;
     }
 
-    public static void main(String[] args) {
-        HmsMirrorCommandLineOptions pcli = new HmsMirrorCommandLineOptions();
-        String[] convertedArgs = pcli.toSpringBootOption(Boolean.TRUE, args);
-        String newCmdLn = String.join(" ", convertedArgs);
-        System.out.println(newCmdLn);
-    }
+
+//    public static void main(String[] args) {
+//        HmsMirrorCommandLineOptions pcli = new HmsMirrorCommandLineOptions();
+//        String[] convertedArgs = pcli.toSpringBootOption(Boolean.TRUE, args);
+//        String newCmdLn = String.join(" ", convertedArgs);
+//        System.out.println(newCmdLn);
+//    }
 
     @Bean
     @Order(1)
@@ -244,6 +249,18 @@ CommandLineRunner configCompressTextOutputFalse(HmsMirrorConfig hmsMirrorConfig)
         };
     }
 
+    @Bean
+    @Order(1)
+    @ConditionalOnProperty(
+            name = "hms-mirror.config.comment"
+    )
+    CommandLineRunner configComment(HmsMirrorConfig hmsMirrorConfig, @Value("${hms-mirror.config.comment}") String value){
+        return args -> {
+            log.info("comment: {}", value);
+            hmsMirrorConfig.setComment(value);
+        };
+    }
+
     @Bean
     @Order(1)
     @ConditionalOnProperty(
@@ -1506,7 +1523,7 @@ CommandLineRunner configWarehousePlans(HmsMirrorConfig config, @Value("${hms-mir
     }
 
 
-    public CommandLine getCommandLine(String[] args) {
+    public static CommandLine getCommandLine(String[] args) {
         Options options = getOptions();
 
         CommandLineParser parser = new PosixParser();
@@ -1534,7 +1551,7 @@ public CommandLine getCommandLine(String[] args) {
         return cmd;
     }
 
-    private Options getOptions() {
+    private static Options getOptions() {
         // create Options object
         Options options = new Options();
 
@@ -1544,6 +1561,11 @@ private Options getOptions() {
         quietOutput.setRequired(Boolean.FALSE);
         options.addOption(quietOutput);
 
+        Option commentOption = new Option("com", "comment", true, "Comment to add to report output");
+        commentOption.setRequired(Boolean.FALSE);
+        commentOption.setType(String.class);
+        options.addOption(commentOption);
+
         Option concurrencyOption = new Option("c", "concurrency", true,
                 "Set application concurrency, default is 10");
         concurrencyOption.setOptionalArg(Boolean.TRUE);
@@ -2142,7 +2164,7 @@ CommandLineRunner initializeConnectionsAfterConfigSetup(ConnectionPoolService co
         };
     }
 
-    public String[] toSpringBootOption(Boolean withoutWeb, String[] args) {
+    public static String[] toSpringBootOption(Boolean withoutWeb, String[] args) {
         CommandLine cmd = getCommandLine(args);
         List<String> springOptions = new ArrayList<>();
         // Turn off web ui

src/main/java/com/cloudera/utils/hms/mirror/cli/HmsMirrorCommandLineOptions.java

@@ -26,6 +26,7 @@ public enum HmsMirrorCommandLineOptionsEnum {
     AUTO_TUNE("at", "auto-tune", null, ""),
     BETA("b", "beta", null, ""),
     CFG("cfg", "config", "filename", ""),
+    COMMENT("com", "comment", null, "Comments to add to report output."),
     CONCURRENCY("c", "concurrency", "threads", ""),
     CONSOLIDATE_TABLES_FOR_DISTCP("ctfd", "consolidate-tables-for-distcp", null, ""),
     CREATE_IF_NOT_EXIST("cine", "create-if-not-exist", null, ""),

src/main/java/com/cloudera/utils/hms/mirror/cli/HmsMirrorCommandLineOptionsEnum.java

@@ -47,8 +47,8 @@ public static void main(String[] args) {
         // Translate the legacy command line arguments to Spring Boot arguments
         //    before starting the application.
         log.info("Translating command line arguments to Spring Boot arguments");
-        HmsMirrorCommandLineOptions hmsMirrorCommandLineOptions = new HmsMirrorCommandLineOptions();
-        String[] springArgs = hmsMirrorCommandLineOptions.toSpringBootOption(Boolean.TRUE, args);
+//        HmsMirrorCommandLineOptions hmsMirrorCommandLineOptions = new HmsMirrorCommandLineOptions();
+        String[] springArgs = HmsMirrorCommandLineOptions.toSpringBootOption(Boolean.TRUE, args);
         log.info("Translated Spring Boot arguments: {}", String.join(" ", springArgs));
         log.info("STARTING THE APPLICATION");
 

src/main/java/com/cloudera/utils/hms/mirror/cli/Mirror.java

@@ -32,13 +32,13 @@
 import com.fasterxml.jackson.dataformat.yaml.YAMLFactory;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.io.IOUtils;
-import org.springframework.beans.factory.annotation.Autowired;
+import org.slf4j.Logger;
 import org.springframework.beans.factory.annotation.Value;
 import org.springframework.boot.CommandLineRunner;
 import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
 import org.springframework.context.annotation.Bean;
-import org.springframework.context.annotation.Configuration;
 import org.springframework.core.annotation.Order;
+import org.springframework.stereotype.Component;
 
 import java.io.File;
 import java.io.IOException;
@@ -53,37 +53,45 @@
 import static java.util.Objects.isNull;
 import static java.util.Objects.nonNull;
 
-@Configuration
+@Component
 @Slf4j
 public class CliInit {
 
-    private ConfigService configService;
-    private DomainService domainService;
-    private ExecuteSessionService executeSessionService;
-    private ObjectMapper yamlMapper;
+    private final DomainService domainService;
+    private final ExecuteSessionService executeSessionService;
+    private final ObjectMapper yamlMapper;
 
-    @Autowired
-    public void setConfigService(ConfigService configService) {
-        this.configService = configService;
-    }
-
-    @Autowired
-    public void setDomainService(DomainService domainService) {
+    /**
+     * Initializes the CliInit class with required services and utilities.
+     *
+     * @param domainService the service responsible for managing domain-specific logic.
+     * @param executeSessionService the service responsible for managing execution sessions.
+     * @param yamlMapper the object mapper for parsing and generating YAML files.
+     */
+    public CliInit(
+            DomainService domainService,
+            ExecuteSessionService executeSessionService,
+            ObjectMapper yamlMapper
+    ) {
         this.domainService = domainService;
-    }
-
-    @Autowired
-    public void setExecuteSessionService(ExecuteSessionService executeSessionService) {
         this.executeSessionService = executeSessionService;
-    }
-
-    @Autowired
-    public void setYamlMapper(ObjectMapper yamlMapper) {
         this.yamlMapper = yamlMapper;
     }
 
+    /**
+     * Initializes the HmsMirrorConfig object by loading the configuration information
+     * from the specified YAML file. This method attempts to locate the configuration file
+     * first from the filesystem and then from the classpath, providing compatibility for
+     * various runtime environments.
+     *
+     * @param configFilename the name of the configuration file to load; it can be a file path
+     *                       or a resource name on the classpath.
+     * @return the initialized HmsMirrorConfig object created from the provided configuration file.
+     * @throws RuntimeException if the configuration file cannot be found or if an I/O error
+     *                          occurs while reading or parsing the file.
+     */
     private HmsMirrorConfig initializeConfig(String configFilename) {
-        HmsMirrorConfig hmsMirrorConfig;
+        HmsMirrorConfig config;
         log.info("Initializing Config.");
         ObjectMapper mapper = new ObjectMapper(new YAMLFactory());
         mapper.enable(SerializationFeature.INDENT_OUTPUT);
@@ -111,14 +119,14 @@ private HmsMirrorConfig initializeConfig(String configFilename) {
             }
 
             String yamlCfgFile = IOUtils.toString(cfgUrl, StandardCharsets.UTF_8);
-            hmsMirrorConfig = mapper.readerFor(HmsMirrorConfig.class).readValue(yamlCfgFile);
-            hmsMirrorConfig.setConfigFilename(configFilename);
+            config = mapper.readerFor(HmsMirrorConfig.class).readValue(yamlCfgFile);
+            config.setConfigFilename(configFilename);
         } catch (IOException e) {
             log.error("IO Exception", e);
             throw new RuntimeException(e);
         }
         log.info("Config loaded.");
-        return hmsMirrorConfig;
+        return config;
     }
 
     @Bean
@@ -157,9 +165,6 @@ public HmsMirrorConfig loadHmsMirrorConfigWithSetup() {
         return new HmsMirrorConfig();
     }
 
-    /*
-
-     */
     @Bean
     @Order(5)
     @ConditionalOnProperty(
@@ -403,15 +408,6 @@ CommandLineRunner configOutputDirInternal(HmsMirrorConfig hmsMirrorConfig, boole
             // Identify it as being set by the user.
             hmsMirrorConfig.setUserSetOutputDirectory(userSetOutputDir);
             executeSessionService.setReportOutputDirectory(value, false);
-//            File reportPathDir = new File(value);
-//            if (!reportPathDir.exists()) {
-//                reportPathDir.mkdirs();
-//            }
-//            reporter.setReportOutputFile(value + FileSystems.getDefault().getSeparator() + "<db>_hms-mirror.md|html|yaml");
-//            reporter.setLeftExecuteFile(value + FileSystems.getDefault().getSeparator() + "<db>_LEFT_execute.sql");
-//            reporter.setLeftCleanUpFile(value + FileSystems.getDefault().getSeparator() + "<db>_LEFT_CleanUp_execute.sql");
-//            reporter.setRightExecuteFile(value + FileSystems.getDefault().getSeparator() + "<db>_RIGHT_execute.sql");
-//            reporter.setRightCleanUpFile(value + FileSystems.getDefault().getSeparator() + "<db>_RIGHT_CleanUp_execute.sql");
 
             File testFile = new File(value + FileSystems.getDefault().getSeparator() + ".dir-check");
 
@@ -422,14 +418,6 @@ CommandLineRunner configOutputDirInternal(HmsMirrorConfig hmsMirrorConfig, boole
                 retryPath.mkdirs();
             }
 
-            // Test file to ensure we can write to it for the report.
-//            try {
-//                new FileOutputStream(testFile).close();
-//            } catch (IOException e) {
-//                throw new RuntimeException("Can't write to output directory: " + value, e);
-//            }
-
         };
     }
-
-}
+}