empty install dir but don't remove it

easybuild/base/testing.py

@@ -114,13 +114,13 @@ def assertEqual(self, a, b, msg=None):
             raise AssertionError("%s:\nDIFF%s:\n%s" % (msg, limit, ''.join(diff[:self.ASSERT_MAX_DIFF]))) from None
 
     def assertExists(self, path, msg=None):
-        """Assert the given path exists"""
+        """Assert that the given path exists"""
         if msg is None:
             msg = "'%s' should exist" % path
         self.assertTrue(os.path.exists(path), msg)
 
     def assertNotExists(self, path, msg=None):
-        """Assert the given path exists"""
+        """Assert that the given path does not exist"""
         if msg is None:
             msg = "'%s' should not exist" % path
         self.assertFalse(os.path.exists(path), msg)

easybuild/framework/easyblock.py

@@ -93,10 +93,11 @@
 from easybuild.tools.filetools import adjust_permissions, apply_patch, back_up_file, change_dir, check_lock
 from easybuild.tools.filetools import compute_checksum, convert_name, copy_dir, copy_file, create_lock
 from easybuild.tools.filetools import create_non_existing_paths, create_patch_info, derive_alt_pypi_url, diff_files
-from easybuild.tools.filetools import download_file, encode_class_name, extract_file, find_backup_name_candidate
-from easybuild.tools.filetools import get_cwd, get_source_tarball_from_git, is_alt_pypi_url, is_binary, is_parent_path
-from easybuild.tools.filetools import is_sha256_checksum, mkdir, move_file, move_logs, read_file, remove_dir
-from easybuild.tools.filetools import remove_file, remove_lock, symlink, verify_checksum, weld_paths, write_file
+from easybuild.tools.filetools import download_file, empty_dir, encode_class_name, extract_file
+from easybuild.tools.filetools import find_backup_name_candidate, get_cwd, get_source_tarball_from_git, is_alt_pypi_url
+from easybuild.tools.filetools import is_binary, is_parent_path, is_sha256_checksum, mkdir, move_file, move_logs
+from easybuild.tools.filetools import read_file, remove_dir, remove_file, remove_lock, symlink, verify_checksum
+from easybuild.tools.filetools import weld_paths, write_file
 from easybuild.tools.hooks import (
     BUILD_STEP, CLEANUP_STEP, CONFIGURE_STEP, EASYBLOCK, EXTENSIONS_STEP, EXTRACT_STEP, FETCH_STEP, INSTALL_STEP,
     MODULE_STEP, MODULE_WRITE, PACKAGE_STEP, PATCH_STEP, PERMISSIONS_STEP, POSTITER_STEP, POSTPROC_STEP, PREPARE_STEP,
@@ -1171,8 +1172,13 @@ def make_builddir(self):
         # unless we're building in installation directory and we iterating over a list of (pre)config/build/installopts,
         # otherwise we wipe the already partially populated installation directory,
         # see https://github.com/easybuilders/easybuild-framework/issues/2556
-        if not (self.build_in_installdir and self.iter_idx > 0):
-            # make sure we no longer sit in the build directory before cleaning it.
+        if self.build_in_installdir and self.iter_idx > 0:
+            pass
+        elif self.build_in_installdir:
+            # building in installation directory, so empty it but don't remove it
+            self.make_dir(self.builddir, self.cfg['cleanupoldbuild'], isinstalldir=True)
+        else:
+            # make sure we no longer sit in the build directory before removing it.
             change_dir(self.orig_workdir)
             self.make_dir(self.builddir, self.cfg['cleanupoldbuild'])
 
@@ -1220,9 +1226,10 @@ def make_installdir(self, dontcreate=None):
         if self.build_in_installdir:
             self.cfg['keeppreviousinstall'] = True
         dontcreate = (dontcreate is None and self.cfg['dontcreateinstalldir']) or dontcreate
-        self.make_dir(self.installdir, self.cfg['cleanupoldinstall'], dontcreateinstalldir=dontcreate)
+        self.make_dir(self.installdir, self.cfg['cleanupoldinstall'], dontcreateinstalldir=dontcreate,
+                      isinstalldir=True)
 
-    def make_dir(self, dir_name, clean, dontcreateinstalldir=False):
+    def make_dir(self, dir_name, clean, dontcreateinstalldir=False, isinstalldir=False):
         """
         Create the directory.
         """
@@ -1233,15 +1240,20 @@ def make_dir(self, dir_name, clean, dontcreateinstalldir=False):
                 return
             elif build_option('module_only') or self.cfg['module_only']:
                 self.log.info("Not touching existing directory %s in module-only mode...", dir_name)
-            elif clean:
-                remove_dir(dir_name)
-                self.log.info("Removed old directory %s", dir_name)
             else:
-                self.log.info("Moving existing directory %s out of the way...", dir_name)
-                timestamp = time.strftime("%Y%m%d-%H%M%S")
-                backupdir = "%s.%s" % (dir_name, timestamp)
-                move_file(dir_name, backupdir)
-                self.log.info("Moved old directory %s to %s", dir_name, backupdir)
+                if not clean:
+                    self.log.info("Creating backup of directory %s...", dir_name)
+                    timestamp = time.strftime("%Y%m%d-%H%M%S")
+                    backupdir = "%s.%s" % (dir_name, timestamp)
+                    copy_dir(dir_name, backupdir)
+                    self.log.info("Copied old directory %s to %s", dir_name, backupdir)
+                if isinstalldir:
+                    # empty the installation directory, but never remove it
+                    empty_dir(dir_name)
+                    self.log.info("Emptied old directory %s", dir_name)
+                else:
+                    remove_dir(dir_name)
+                    self.log.info("Removed old directory %s", dir_name)
 
         if dontcreateinstalldir:
             olddir = dir_name

easybuild/tools/filetools.py

@@ -376,6 +376,40 @@ def remove_file(path):
         raise EasyBuildError("Failed to remove file %s: %s", path, err)
 
 
+def empty_dir(path):
+    """Empty directory at specified path, keeping directory itself intact."""
+    # early exit in 'dry run' mode
+    if build_option('extended_dry_run'):
+        dry_run_msg("directory %s emptied" % path, silent=build_option('silent'))
+        return
+
+    if os.path.exists(path):
+        ok = False
+        errors = []
+        # Try multiple times to cater for temporary failures on e.g. NFS mounted paths
+        max_attempts = 3
+        for i in range(0, max_attempts):
+            try:
+                for item in os.listdir(path):
+                    subpath = os.path.join(path, item)
+                    if os.path.isfile(subpath) or os.path.islink(subpath):
+                        os.remove(subpath)
+                    elif os.path.isdir(subpath):
+                        shutil.rmtree(subpath)
+                ok = True
+            except OSError as err:
+                _log.debug("Failed to empty path %s at attempt %d: %s" % (path, i, err))
+                errors.append(err)
+                time.sleep(2)
+                # make sure write permissions are enabled on entire directory
+                adjust_permissions(path, stat.S_IWUSR, add=True, recursive=True)
+        if ok:
+            _log.info("Path %s successfully emptied." % path)
+        else:
+            raise EasyBuildError("Failed to empty directory %s even after %d attempts.\nReasons: %s",
+                                 path, max_attempts, errors)
+
+
 def remove_dir(path):
     """Remove directory at specified path."""
     # early exit in 'dry run' mode
@@ -1952,7 +1986,7 @@ def adjust_permissions(provided_path, permission_bits, add=True, onlyfiles=False
     if failed_paths:
         raise EasyBuildError("Failed to chmod/chown several paths: %s (last error: %s)", failed_paths, err_msg)
 
-    # we ignore some errors, but if there are to many, something is definitely wrong
+    # we ignore some errors, but if there are too many, something is definitely wrong
     fail_ratio = fail_cnt / float(len(allpaths))
     max_fail_ratio = float(build_option('max_fail_ratio_adjust_permissions'))
     if fail_ratio > max_fail_ratio:

test/framework/filetools.py

@@ -2546,6 +2546,40 @@ def test_extract_file(self):
         self.assertFalse(stderr)
         self.assertFalse(stdout)
 
+    def test_empty_dir(self):
+        """Test empty_dir function"""
+        test_dir = os.path.join(self.test_prefix, 'test123')
+        testfile = os.path.join(test_dir, 'foo')
+        testfile_hidden = os.path.join(test_dir, '.foo')
+        test_link = os.path.join(test_dir, 'foolink')
+        test_subdir = os.path.join(test_dir, 'foodir')
+
+        ft.mkdir(test_subdir, parents=True)
+        ft.write_file(testfile, 'bar')
+        ft.write_file(testfile_hidden, 'bar')
+        ft.symlink(testfile, test_link)
+        ft.empty_dir(test_dir)
+        self.assertExists(test_dir)
+        self.assertNotExists(testfile)
+        self.assertNotExists(testfile_hidden)
+        self.assertNotExists(test_link)
+
+        # also test behaviour under --dry-run
+        build_options = {
+            'extended_dry_run': True,
+            'silent': False,
+        }
+        init_config(build_options=build_options)
+
+        self.mock_stdout(True)
+        ft.mkdir(test_dir)
+        ft.empty_dir(test_dir)
+        txt = self.get_stdout()
+        self.mock_stdout(False)
+
+        regex = re.compile("^directory [^ ]* emptied$")
+        self.assertTrue(regex.match(txt), f"Pattern '{regex.pattern}' found in: {txt}")
+
     def test_remove(self):
         """Test remove_file, remove_dir and join remove functions."""
         testfile = os.path.join(self.test_prefix, 'foo')

test/framework/toy_build.py

@@ -798,6 +798,9 @@ def test_toy_gid_sticky_bits(self):
             self.assertFalse(perms & stat.S_ISVTX, "no sticky bit on %s" % fullpath)
 
         # git/sticky bits are set, but only on (re)created directories
+        # first remove install dir because it is not recreated
+        toy_installdir = os.path.join(self.test_installpath, *subdirs[3][0])
+        remove_dir(toy_installdir)
         self.run_test_toy_build_with_output(extra_args=['--set-gid-bit', '--sticky-bit'])
         for subdir, bits_set in subdirs:
             fullpath = os.path.join(self.test_installpath, *subdir)