diff --git a/easybuild/base/testing.py b/easybuild/base/testing.py index 92789fd4ce..71700794d3 100644 --- a/easybuild/base/testing.py +++ b/easybuild/base/testing.py @@ -114,13 +114,13 @@ def assertEqual(self, a, b, msg=None): raise AssertionError("%s:\nDIFF%s:\n%s" % (msg, limit, ''.join(diff[:self.ASSERT_MAX_DIFF]))) from None def assertExists(self, path, msg=None): - """Assert the given path exists""" + """Assert that the given path exists""" if msg is None: msg = "'%s' should exist" % path self.assertTrue(os.path.exists(path), msg) def assertNotExists(self, path, msg=None): - """Assert the given path exists""" + """Assert that the given path does not exist""" if msg is None: msg = "'%s' should not exist" % path self.assertFalse(os.path.exists(path), msg) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 021d524012..efec180e4d 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -93,10 +93,11 @@ from easybuild.tools.filetools import adjust_permissions, apply_patch, back_up_file, change_dir, check_lock from easybuild.tools.filetools import compute_checksum, convert_name, copy_dir, copy_file, create_lock from easybuild.tools.filetools import create_non_existing_paths, create_patch_info, derive_alt_pypi_url, diff_files -from easybuild.tools.filetools import download_file, encode_class_name, extract_file, find_backup_name_candidate -from easybuild.tools.filetools import get_cwd, get_source_tarball_from_git, is_alt_pypi_url, is_binary, is_parent_path -from easybuild.tools.filetools import is_sha256_checksum, mkdir, move_file, move_logs, read_file, remove_dir -from easybuild.tools.filetools import remove_file, remove_lock, symlink, verify_checksum, weld_paths, write_file +from easybuild.tools.filetools import download_file, empty_dir, encode_class_name, extract_file +from easybuild.tools.filetools import find_backup_name_candidate, get_cwd, get_source_tarball_from_git, is_alt_pypi_url +from easybuild.tools.filetools import is_binary, is_parent_path, is_sha256_checksum, mkdir, move_file, move_logs +from easybuild.tools.filetools import read_file, remove_dir, remove_file, remove_lock, symlink, verify_checksum +from easybuild.tools.filetools import weld_paths, write_file from easybuild.tools.hooks import ( BUILD_STEP, CLEANUP_STEP, CONFIGURE_STEP, EASYBLOCK, EXTENSIONS_STEP, EXTRACT_STEP, FETCH_STEP, INSTALL_STEP, MODULE_STEP, MODULE_WRITE, PACKAGE_STEP, PATCH_STEP, PERMISSIONS_STEP, POSTITER_STEP, POSTPROC_STEP, PREPARE_STEP, @@ -1171,8 +1172,13 @@ def make_builddir(self): # unless we're building in installation directory and we iterating over a list of (pre)config/build/installopts, # otherwise we wipe the already partially populated installation directory, # see https://github.com/easybuilders/easybuild-framework/issues/2556 - if not (self.build_in_installdir and self.iter_idx > 0): - # make sure we no longer sit in the build directory before cleaning it. + if self.build_in_installdir and self.iter_idx > 0: + pass + elif self.build_in_installdir: + # building in installation directory, so empty it but don't remove it + self.make_dir(self.builddir, self.cfg['cleanupoldbuild'], isinstalldir=True) + else: + # make sure we no longer sit in the build directory before removing it. change_dir(self.orig_workdir) self.make_dir(self.builddir, self.cfg['cleanupoldbuild']) @@ -1220,9 +1226,10 @@ def make_installdir(self, dontcreate=None): if self.build_in_installdir: self.cfg['keeppreviousinstall'] = True dontcreate = (dontcreate is None and self.cfg['dontcreateinstalldir']) or dontcreate - self.make_dir(self.installdir, self.cfg['cleanupoldinstall'], dontcreateinstalldir=dontcreate) + self.make_dir(self.installdir, self.cfg['cleanupoldinstall'], dontcreateinstalldir=dontcreate, + isinstalldir=True) - def make_dir(self, dir_name, clean, dontcreateinstalldir=False): + def make_dir(self, dir_name, clean, dontcreateinstalldir=False, isinstalldir=False): """ Create the directory. """ @@ -1233,15 +1240,20 @@ def make_dir(self, dir_name, clean, dontcreateinstalldir=False): return elif build_option('module_only') or self.cfg['module_only']: self.log.info("Not touching existing directory %s in module-only mode...", dir_name) - elif clean: - remove_dir(dir_name) - self.log.info("Removed old directory %s", dir_name) else: - self.log.info("Moving existing directory %s out of the way...", dir_name) - timestamp = time.strftime("%Y%m%d-%H%M%S") - backupdir = "%s.%s" % (dir_name, timestamp) - move_file(dir_name, backupdir) - self.log.info("Moved old directory %s to %s", dir_name, backupdir) + if not clean: + self.log.info("Creating backup of directory %s...", dir_name) + timestamp = time.strftime("%Y%m%d-%H%M%S") + backupdir = "%s.%s" % (dir_name, timestamp) + copy_dir(dir_name, backupdir) + self.log.info("Copied old directory %s to %s", dir_name, backupdir) + if isinstalldir: + # empty the installation directory, but never remove it + empty_dir(dir_name) + self.log.info("Emptied old directory %s", dir_name) + else: + remove_dir(dir_name) + self.log.info("Removed old directory %s", dir_name) if dontcreateinstalldir: olddir = dir_name diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index 3126c1c020..db13534d2a 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -376,6 +376,40 @@ def remove_file(path): raise EasyBuildError("Failed to remove file %s: %s", path, err) +def empty_dir(path): + """Empty directory at specified path, keeping directory itself intact.""" + # early exit in 'dry run' mode + if build_option('extended_dry_run'): + dry_run_msg("directory %s emptied" % path, silent=build_option('silent')) + return + + if os.path.exists(path): + ok = False + errors = [] + # Try multiple times to cater for temporary failures on e.g. NFS mounted paths + max_attempts = 3 + for i in range(0, max_attempts): + try: + for item in os.listdir(path): + subpath = os.path.join(path, item) + if os.path.isfile(subpath) or os.path.islink(subpath): + os.remove(subpath) + elif os.path.isdir(subpath): + shutil.rmtree(subpath) + ok = True + except OSError as err: + _log.debug("Failed to empty path %s at attempt %d: %s" % (path, i, err)) + errors.append(err) + time.sleep(2) + # make sure write permissions are enabled on entire directory + adjust_permissions(path, stat.S_IWUSR, add=True, recursive=True) + if ok: + _log.info("Path %s successfully emptied." % path) + else: + raise EasyBuildError("Failed to empty directory %s even after %d attempts.\nReasons: %s", + path, max_attempts, errors) + + def remove_dir(path): """Remove directory at specified path.""" # early exit in 'dry run' mode @@ -1952,7 +1986,7 @@ def adjust_permissions(provided_path, permission_bits, add=True, onlyfiles=False if failed_paths: raise EasyBuildError("Failed to chmod/chown several paths: %s (last error: %s)", failed_paths, err_msg) - # we ignore some errors, but if there are to many, something is definitely wrong + # we ignore some errors, but if there are too many, something is definitely wrong fail_ratio = fail_cnt / float(len(allpaths)) max_fail_ratio = float(build_option('max_fail_ratio_adjust_permissions')) if fail_ratio > max_fail_ratio: diff --git a/test/framework/filetools.py b/test/framework/filetools.py index ca2b9bcb47..78baa9b8ea 100644 --- a/test/framework/filetools.py +++ b/test/framework/filetools.py @@ -2546,6 +2546,40 @@ def test_extract_file(self): self.assertFalse(stderr) self.assertFalse(stdout) + def test_empty_dir(self): + """Test empty_dir function""" + test_dir = os.path.join(self.test_prefix, 'test123') + testfile = os.path.join(test_dir, 'foo') + testfile_hidden = os.path.join(test_dir, '.foo') + test_link = os.path.join(test_dir, 'foolink') + test_subdir = os.path.join(test_dir, 'foodir') + + ft.mkdir(test_subdir, parents=True) + ft.write_file(testfile, 'bar') + ft.write_file(testfile_hidden, 'bar') + ft.symlink(testfile, test_link) + ft.empty_dir(test_dir) + self.assertExists(test_dir) + self.assertNotExists(testfile) + self.assertNotExists(testfile_hidden) + self.assertNotExists(test_link) + + # also test behaviour under --dry-run + build_options = { + 'extended_dry_run': True, + 'silent': False, + } + init_config(build_options=build_options) + + self.mock_stdout(True) + ft.mkdir(test_dir) + ft.empty_dir(test_dir) + txt = self.get_stdout() + self.mock_stdout(False) + + regex = re.compile("^directory [^ ]* emptied$") + self.assertTrue(regex.match(txt), f"Pattern '{regex.pattern}' found in: {txt}") + def test_remove(self): """Test remove_file, remove_dir and join remove functions.""" testfile = os.path.join(self.test_prefix, 'foo') diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py index 914daa306e..d0c9dfea5b 100644 --- a/test/framework/toy_build.py +++ b/test/framework/toy_build.py @@ -798,6 +798,9 @@ def test_toy_gid_sticky_bits(self): self.assertFalse(perms & stat.S_ISVTX, "no sticky bit on %s" % fullpath) # git/sticky bits are set, but only on (re)created directories + # first remove install dir because it is not recreated + toy_installdir = os.path.join(self.test_installpath, *subdirs[3][0]) + remove_dir(toy_installdir) self.run_test_toy_build_with_output(extra_args=['--set-gid-bit', '--sticky-bit']) for subdir, bits_set in subdirs: fullpath = os.path.join(self.test_installpath, *subdir)