Support using EXPAND.EXE as well as EXTRACT.EXE for testing

Author: kyz

libmspack/ChangeLog

@@ -7,6 +7,21 @@
 	drvs/2019/11/016c7f3e-809d-4720-893b-
 	e0d74f10c39d_35e12507628e8dc8ae5fb3332835f4253d2dab23.cab)
 
+        * cabd_compare: use EXPAND.EXE instead of EXTRACT.EXE when
+	testing files in a directory called 'expand'. The example
+	cab file above is extracted wrongly by EXTRACT.EXE, but
+	correctly by EXPAND.EXE because they take different approaches
+	to E8 transformations:
+
+	- EXTRACT.EXE writes "E8E8E8E8E8E8' to the last 6 bytes of
+	  frame, looks for E8 bytes up to the last 6 bytes, then restores
+	  the last 6 bytes, leaving partial transforms of 1-3 bytes if
+	  E8 byte is found near the end of the frame
+
+	- EXPAND.EXE looks for E8 bytes up to the last 10 bytes of a
+	  frame, therefore the last 6 bytes are never altered and all
+	  transforms are 4 bytes
+
 2019-02-18  Stuart Caie <[email protected]>
 
 	* chmd_read_headers(): a CHM file name beginning "::" but shorter

libmspack/README

@@ -90,14 +90,15 @@ examples/msexpand.c     - expands an SZDD or KWAJ file
 examples/oabextract.c   - extracts an Exchange Offline Address Book (.LZX) file
 
 test/cabd_c10          - tests the CAB decompressor on the C10 collection
-test/cabd_compare      - compares libmspack with Microsoft's EXTRACT.EXE
+test/cabd_compare      - compares libmspack with Microsoft's EXTRACT/EXPAND.EXE
 test/cabd_md5          - shows MD5 checksums of all files in a CAB file/set
 test/chmd_compare      - compares libmspack with Microsoft's HH.EXE
 test/chmd_find.c       - checks all files in a CHM file can be fast-found
 test/chmd_md5.c        - shows MD5 checksums of all files within a CHM file
 test/chmd_order.c      - extracts files in a CHM file in four different ways
 test/chminfo.c         - prints verbose information about CHM file structures
 test/msdecompile_md5   - runs Microsoft's HH.EXE -DECOMPILE via WINE
+test/msexpand_md5      - runs Microsoft's EXPAND.EXE via WINE
 test/msextract_md5     - runs Microsoft's EXTRACT.EXE via WINE
 
 Here is a simple example of usage, which will create a CAB decompressor,

libmspack/test/cabd_compare

@@ -1,34 +1,41 @@
 #!/bin/sh
 # Test if cabd_md5 expands cab files identically to Microsoft's EXTRACT.EXE
+# (or EXPAND.EXE if cab file is in a directory called 'expand')
 
 [ -d .cache ] || mkdir .cache
 BASEDIR=`dirname "$0"`
 
 cnt=1
+orig=.$$.orig
+test=.$$.test
 for cab in "$@"; do
     name=`printf '%d/%d %s' $cnt $# $cab`
     cnt=`expr $cnt + 1`
 
     echo "test $name"
     cached=`echo $cab | sed -e 's/\//-/g' -e 's/^/.cache\//'`
     if [ ! -s $cached ]; then
-        $BASEDIR/msextract_md5 $cab >.orig.out 2>.orig.err
-        if [ -s .orig.err ]; then
+        case $cab in
+        */expand/*) $BASEDIR/msexpand_md5 $cab >$orig 2>$orig.err;;
+        *)          $BASEDIR/msextract_md5 $cab >$orig 2>$orig.err;;
+        esac
+
+        if [ -s $orig.err ]; then
             echo "FAIL $name: MS errors" >&2
-            cat .orig.err >&2
-        else
-            mv .orig.out $cached
+            cat $orig.err >&2
+            continue
         fi
+        mv $orig $cached
     fi
 
-    $BASEDIR/cabd_md5 $cab >.test.out 2>.test.err
-    perl -pi -e 'if($.>1){s{\\}{/}g;s{  /}{  }}' .test.out
+    $BASEDIR/cabd_md5 $cab >$test 2>$test.err
+    perl -pi -e 'if($.>1){s{\\}{/}g;s{  /}{  }}' $test
 
     # suppress warning. PRECOPY2.CAB does not extend to CATALOG3.CAB, but
     # CATALOG3.CAB extends backwards to PRECOPY2.CAB. cabd_md5 supports this
     # but msextract_md5 does not, so differences appear. As a workaround, test
     # PRECOPYn.CAB separately and suppress the warning when testing CATALOG3.CAB
-    sed -i "/can't find \"PRECOPY2.CAB\" to prepend/d" .test.err
+    sed -i "/can't find \"PRECOPY2.CAB\" to prepend/d" $test.err
 
     # suppress warning. One cabinet set has this structure:
     # * cab1: file1 FROM_PREV, file2 TO_NEXT
@@ -38,18 +45,19 @@ for cab in "$@"; do
     # This is wrong. file3 and file4 are in the same folder, so both should
     # be FROM_PREV_AND_TO_NEXT in cab3, and both should be listed in cab4.
     # However, the set unpacks despite the warning, so suppress it.
-    sed -i '/rainloop.xa not listed in both cabinets/d' .test.err
+    sed -i '/rainloop.xa not listed in both cabinets/d' $test.err
 
-    if [ -s .test.err ]; then
+    if [ -s $test.err ]; then
         echo "FAIL $name: errors" >&2
-        cat .test.err >&2
+        cat $test.err >&2
+        continue
     fi
 
-    if cmp $cached .test.out >/dev/null; then
+    if cmp $cached $test >/dev/null; then
        echo "OK   $name"
     else
        echo "FAIL $name: differences" >&2
-       diff -u $cached .test.out >&2
+       diff -u $cached $test >&2
     fi
 done
-rm -f .orig.out .orig.err .test.out .test.err
+rm -f $orig $orig.err $test $test.err

libmspack/test/chmd_compare

@@ -5,38 +5,41 @@
 BASEDIR=`dirname $0`
 
 cnt=1
+orig=.$$.orig
+test=.$$.test
 for chm in "$@"; do
     name=`printf '%d/%d %s' $cnt $# $chm`; cnt=`expr $cnt + 1`
 
     echo "test $name"
     cached=`echo $chm | sed -e 's/\//-/g' -e 's/^/.cache\//'`
     if [ ! -s $cached ]; then
-        $BASEDIR/msdecompile_md5 $chm >.orig.out 2>.orig.err
-        if [ -s .orig.err ]; then
+        $BASEDIR/msdecompile_md5 $chm >$orig 2>$orig.err
+        if [ -s $orig.err ]; then
             echo "FAIL $name: MS errors" >&2
-            cat .orig.err >&2
-        else
-            LANG=C sort -k2 .orig.out >$cached
+            cat $orig.err >&2
+            continue
         fi
+        LANG=C sort -k2 $orig >$cached
     fi
 
-    $BASEDIR/chmd_md5 $chm >.test.out 2>.test.errwarn
-    perl -pe 'if($.>1){$_=""if/^[0-9a-f]{32} \/[#\$]/;s{ /}{  }}' .test.out | LANG=C sort -k2 >.test.sorted
-    sed '/^WARNING; /d' .test.errwarn > .test.err
-    if [ -s .test.err ]; then
+    $BASEDIR/chmd_md5 $chm >$test.unsorted 2>$test.errwarn
+    perl -pe 'if($.>1){$_=""if/^[0-9a-f]{32} \/[#\$]/;s{ /}{  }}' $test.unsorted | LANG=C sort -k2 >$test
+    sed '/^WARNING; /d' $test.errwarn > $test.err
+    if [ -s $test.err ]; then
         echo "FAIL $name: errors" >&2
-        cat .test.errwarn >&2
+        cat $test.errwarn >&2
+        continue
     fi
 
-    if cmp $cached .test.sorted >/dev/null; then
+    if cmp $cached $test >/dev/null; then
         echo "OK   $name"
     else
-       if [ `diff $cached .test.sorted | grep -c '^<'` -gt 0 ]; then
+       if [ `diff $cached $test | grep -c '^<'` -gt 0 ]; then
            echo "FAIL $name: differences" >&2
-           diff -u $cached .test.sorted >&2
+           diff -u $cached $test >&2
        else
            echo "OK   $name (better than hh.exe)"
        fi
     fi
 done
-rm -f .orig.out .orig.err .test.out .test.err .test.errwarn .test.sorted
+rm -f $orig $orig.err $test $test.err $test.errwarn $test.unsorted

libmspack/test/msexpand_md5

@@ -0,0 +1,44 @@
+#!/usr/bin/perl -w
+# put expand.exe and dpx.dll (or a link to them) into this directory
+# for this script to work.
+#
+# NOTE: when a cabinet only contains one file, expand.exe fails to
+# preserve its filename, it renames the output after the source cabinet
+
+use strict;
+use File::Temp qw(tempdir);
+use Cwd qw(cwd);
+
+my $expand = $0; $expand =~ s{[^/]+$}{expand.exe};
+my $HDR1 = 'Microsoft (R) File Expansion Utility';
+my $HDR2 = 'Copyright (c) Microsoft Corp';
+
+my $dir = tempdir("./.tempXXXX", CLEANUP => 1) . '/extradir';
+mkdir $dir;
+$ENV{LANG} = 'C';
+$ENV{WINEPREFIX} = "$ENV{HOME}/.wine64";
+$ENV{WINEARCH} = 'win64';
+
+for my $cab (@ARGV) {
+    my @files;
+    print "*** $cab\n";
+    for (`wine $expand $cab -F:* $dir 2>&1`) {
+        s/\015?\012$//s; # remove line endings
+        next if /^(\Q$HDR1\E|\Q$HDR2\E|\s*$|Expanding Files |Progress: |\d+ files total)/;
+        if (/^Adding \Q$dir\E\\(.+) to Extraction Queue$/) {
+            my $file = $1;
+            $file =~ s{\\}{/}g;
+            $file =~ s{^/+}{};
+            push @files, $file;
+        }
+        else {
+            print STDERR "$_\n";
+        }
+    }
+
+    next unless @files;
+    my $olddir = cwd();
+    chdir $dir;
+    system 'md5sum', @files;
+    chdir $olddir;
+}