https://github.com/cloudera-labs/hms-mirror/issues/197

#196

Author: dstreev

pom.xml

@@ -28,7 +28,7 @@
 
     <groupId>com.cloudera.utils.hadoop</groupId>
     <artifactId>hms-mirror</artifactId>
-    <version>2.3.1.6</version>
+    <version>2.3.1.7</version>
     <packaging>jar</packaging>
 
     <name>hms-mirror</name>

src/main/java/com/cloudera/utils/hms/mirror/datastrategy/DataStrategyBase.java

@@ -249,7 +249,7 @@ public Boolean buildTableSchema(CopySpec copySpec) throws RequiredConfigurationE
                     if (!TableUtils.isACID(source)) {
                         // Non ACID tables.
                         if (copySpec.isUpgrade() && TableUtils.isManaged(source)) {
-                            converted = TableUtils.makeExternal(target);
+                            converted = TableUtils.makeExternal(target, config);
                             if (converted) {
                                 target.addIssue("Schema 'converted' from LEGACY managed to EXTERNAL");
                                 target.addProperty(HMS_MIRROR_LEGACY_MANAGED_FLAG, converted.toString());
@@ -264,7 +264,7 @@ public Boolean buildTableSchema(CopySpec copySpec) throws RequiredConfigurationE
                             }
                         } else {
                             if (copySpec.isMakeExternal()) {
-                                converted = TableUtils.makeExternal(target);
+                                converted = TableUtils.makeExternal(target, config);
                             }
                             if (copySpec.isTakeOwnership()) {
                                 if (TableUtils.isACID(source)) {
@@ -281,12 +281,14 @@ public Boolean buildTableSchema(CopySpec copySpec) throws RequiredConfigurationE
                         if (copySpec.isMakeNonTransactional()) {
                             TableUtils.removeTblProperty(TRANSACTIONAL, target);
                             TableUtils.removeTblProperty(TRANSACTIONAL_PROPERTIES, target);
-                            TableUtils.removeTblProperty(BUCKETING_VERSION, target);
+                            if (!config.getTransfer().getStorageMigration().isDistcp()) {
+                                TableUtils.removeTblProperty(BUCKETING_VERSION, target);
+                            }
                         }
 
                         // We should also convert to external if we've enabled the conversion to Iceberg.
                         if (copySpec.isMakeExternal() || config.getIcebergConversion().isEnable()) {
-                            converted = TableUtils.makeExternal(target);
+                            converted = TableUtils.makeExternal(target, config);
                         }
 
                         if (copySpec.isTakeOwnership()) {
@@ -313,7 +315,7 @@ public Boolean buildTableSchema(CopySpec copySpec) throws RequiredConfigurationE
                         }
 
                         if (config.getMigrateACID().isDowngrade() && copySpec.isMakeExternal()) {
-                            converted = TableUtils.makeExternal(target);
+                            converted = TableUtils.makeExternal(target, config);
                             if (!config.isNoPurge()) {
                                 target.addProperty(EXTERNAL_TABLE_PURGE, Boolean.TRUE.toString());
                             }
@@ -501,7 +503,9 @@ public Boolean buildTableSchema(CopySpec copySpec) throws RequiredConfigurationE
                         // remove newer flags;
                         TableUtils.removeTblProperty(EXTERNAL_TABLE_PURGE, target);
                         TableUtils.removeTblProperty(DISCOVER_PARTITIONS, target);
-                        TableUtils.removeTblProperty(BUCKETING_VERSION, target);
+                        if (!config.getTransfer().getStorageMigration().isDistcp()) {
+                            TableUtils.removeTblProperty(BUCKETING_VERSION, target);
+                        }
                     }
 
                 } else if (TableUtils.isView(target)) {

src/main/java/com/cloudera/utils/hms/mirror/service/DistCpService.java

@@ -191,7 +191,7 @@ public void buildAllDistCpReports(ExecuteSession session, String outputDir) {
                                 String targetProtocol = NamespaceUtils.getProtocol(target);
 
                                 if (nonNull(sourceProtocol) && nonNull(targetProtocol) && !sourceProtocol.equals(targetProtocol)) {
-                                        distcpScriptSb.append("#  Source and target protocols are different. This may cause issues with 'distcp' is -skipcrccheck isn't set.");
+                                        distcpScriptSb.append("#  Source and target protocols are different. This may cause issues with 'distcp' if -skipcrccheck isn't set.").append("\n");
                                         // Add -skipcrccheck to the distcp command
                                         distcpScriptSb.append("hadoop distcp ${DISTCP_OPTS} -skipcrccheck ").append(source).append(" ").append(target).append("\n").append("\n");
                                 } else {

src/main/java/com/cloudera/utils/hms/util/TableUtils.java

@@ -21,6 +21,7 @@
 import com.cloudera.utils.hms.mirror.MirrorConf;
 import com.cloudera.utils.hms.mirror.TablePropertyVars;
 import com.cloudera.utils.hms.mirror.domain.Cluster;
+import com.cloudera.utils.hms.mirror.domain.HmsMirrorConfig;
 import com.cloudera.utils.hms.mirror.domain.support.SerdeType;
 import com.cloudera.utils.hms.mirror.feature.IcebergState;
 import lombok.extern.slf4j.Slf4j;
@@ -551,7 +552,7 @@ public static Boolean prefixTableName(String tableName, String prefix, List<Stri
         return rtn;
     }
 
-    public static Boolean makeExternal(EnvironmentTable envTable) {
+    public static Boolean makeExternal(EnvironmentTable envTable, HmsMirrorConfig config) {
         Boolean rtn = Boolean.FALSE;
         if (isManaged(envTable)) {
             log.debug("Converting table: {} to EXTERNAL", envTable.getName());
@@ -566,7 +567,9 @@ public static Boolean makeExternal(EnvironmentTable envTable) {
             // If ACID, remove transactional property to complete conversion to external.
             removeTblProperty(TablePropertyVars.TRANSACTIONAL, envTable.getDefinition());
             removeTblProperty(TablePropertyVars.TRANSACTIONAL_PROPERTIES, envTable.getDefinition());
-            removeTblProperty(TablePropertyVars.BUCKETING_VERSION, envTable.getDefinition());
+            if (!config.getTransfer().getStorageMigration().isDistcp()) {
+                removeTblProperty(TablePropertyVars.BUCKETING_VERSION, envTable.getDefinition());
+            }
         }
         return rtn;
     }