From 6cff626975a2defbbd8bd067b73e01e44a113527 Mon Sep 17 00:00:00 2001 From: dkazanc Date: Tue, 8 Jul 2025 14:30:09 +0100 Subject: [PATCH 01/19] trying parallel tests1 --- tests/conftest.py | 10 ++++++ tests/test_pipeline_big.py | 66 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 76 insertions(+) diff --git a/tests/conftest.py b/tests/conftest.py index 84f273942..8e1d24f21 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -112,6 +112,16 @@ def cmd(): "--save-all", ] +@pytest.fixture +def cmd2(): + return [ + 'mpirun -v -np 2 ' + sys.executable, + "-m", + "httomo", + "run", + "--save-all", + ] + @pytest.fixture def standard_data(): diff --git a/tests/test_pipeline_big.py b/tests/test_pipeline_big.py index 8e2558cf4..25f573cd6 100644 --- a/tests/test_pipeline_big.py +++ b/tests/test_pipeline_big.py @@ -508,6 +508,72 @@ def test_pipe_FBP2d_astra_i12_119647_preview( assert res_norm < 1e-6 +# ######################################################################## + +@pytest.mark.full_data +def test_parallel_pipe_FBP2d_astra_i12_119647_preview( + get_files: Callable, + cmd2, + i12_119647, + FBP2d_astra, + FBP2d_astra_i12_119647_npz, + output_folder, +): + + change_value_parameters_method_pipeline( + FBP2d_astra, + method=[ + "standard_tomo", + ], + key=[ + "preview", + ], + value=[ + {"detector_y": {"start": 900, "stop": 1200}}, + ], + ) + + cmd2.pop(4) #: don't save all + cmd2.insert(5, i12_119647) + cmd2.insert(7, FBP2d_astra) + cmd2.insert(8, output_folder) + + subprocess.check_output(cmd2) + + files = get_files(output_folder) + + #: check the generated reconstruction (hdf5 file) + h5_files = list(filter(lambda x: ".h5" in x, files)) + assert len(h5_files) == 1 + + # load the pre-saved numpy array for comparison bellow + data_gt = FBP2d_astra_i12_119647_npz["data"] + axis_slice = FBP2d_astra_i12_119647_npz["axis_slice"] + (slices, sizeX, sizeY) = np.shape(data_gt) + + step = axis_slice // (slices + 2) + # store for the result + data_result = np.zeros((slices, sizeX, sizeY), dtype=np.float32) + + path_to_data = "data/" + h5_file_name = "FBP2d_astra" + for file_to_open in h5_files: + if h5_file_name in file_to_open: + h5f = h5py.File(file_to_open, "r") + index_prog = step + for i in range(slices): + data_result[i, :, :] = h5f[path_to_data][:, index_prog, :] + index_prog += step + h5f.close() + else: + message_str = f"File name with {h5_file_name} string cannot be found." + raise FileNotFoundError(message_str) + + residual_im = data_gt - data_result + res_norm = np.linalg.norm(residual_im.flatten()).astype("float32") + assert res_norm < 1e-6 + + # ######################################################################## From 6d21eafb6f2b2b5630eed9d564642b30562cf0c3 Mon Sep 17 00:00:00 2001 From: dkazanc Date: Tue, 8 Jul 2025 14:48:25 +0100 Subject: [PATCH 02/19] trying parallel tests2 --- tests/conftest.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/conftest.py b/tests/conftest.py index 8e1d24f21..5473ef81b 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -114,8 +114,9 @@ def cmd(): @pytest.fixture def cmd2(): + syspath = os.path.dirname(os.path.realpath(__file__)) return [ - 'mpirun -v -np 2 ' + sys.executable, + 'mpirun -v -N 2 ' + str(sys.executable) + " -m mpi4py " + syspath, "-m", "httomo", "run", From a111a68bec0bdcd65764aa52210f7172f8260ee2 Mon Sep 17 00:00:00 2001 From: dkazanc Date: Tue, 8 Jul 2025 14:55:01 +0100 Subject: [PATCH 03/19] trying parallel tests3 --- tests/test_pipeline_big.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/test_pipeline_big.py b/tests/test_pipeline_big.py index 25f573cd6..1fa35ba2b 100644 --- a/tests/test_pipeline_big.py +++ b/tests/test_pipeline_big.py @@ -513,7 +513,7 @@ def test_pipe_FBP2d_astra_i12_119647_preview( @pytest.mark.full_data def test_parallel_pipe_FBP2d_astra_i12_119647_preview( get_files: Callable, - cmd2, + cmd, i12_119647, FBP2d_astra, FBP2d_astra_i12_119647_npz, @@ -533,12 +533,12 @@ def test_parallel_pipe_FBP2d_astra_i12_119647_preview( ], ) - cmd2.pop(4) #: don't save all - cmd2.insert(5, i12_119647) - cmd2.insert(7, FBP2d_astra) - cmd2.insert(8, output_folder) + cmd.pop(4) #: don't save all + cmd.insert(5, i12_119647) + cmd.insert(7, FBP2d_astra) + cmd.insert(8, output_folder) - subprocess.check_output(cmd2) + subprocess.check_output(cmd) files = get_files(output_folder) From 45ffef5d9c5976f5f62504dacbaf5877b4295273 Mon Sep 17 00:00:00 2001 From: dkazanc Date: Tue, 8 Jul 2025 15:02:03 +0100 Subject: [PATCH 04/19] trying parallel tests4 --- tests/test_pipeline_big.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_pipeline_big.py b/tests/test_pipeline_big.py index 1fa35ba2b..54e59465e 100644 --- a/tests/test_pipeline_big.py +++ b/tests/test_pipeline_big.py @@ -511,6 +511,7 @@ def test_pipe_FBP2d_astra_i12_119647_preview( # ######################################################################## @pytest.mark.full_data +@pytest.mark.mpi def test_parallel_pipe_FBP2d_astra_i12_119647_preview( get_files: Callable, cmd, From 73c8bf597aade2134d2fbf47a925beea73353e0c Mon Sep 17 00:00:00 2001 From: dkazanc Date: Tue, 8 Jul 2025 15:23:21 +0100 Subject: [PATCH 05/19] trying parallel tests5 --- tests/conftest.py | 6 +++--- tests/test_pipeline_big.py | 13 ++++++------- 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 5473ef81b..c16fc3920 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -113,10 +113,10 @@ def cmd(): ] @pytest.fixture -def cmd2(): - syspath = os.path.dirname(os.path.realpath(__file__)) +def cmd2(): return [ - 'mpirun -v -N 2 ' + str(sys.executable) + " -m mpi4py " + syspath, + "mpirun -v -N 2", + str(sys.executable), "-m", "httomo", "run", diff --git a/tests/test_pipeline_big.py b/tests/test_pipeline_big.py index 54e59465e..3e8340ec5 100644 --- a/tests/test_pipeline_big.py +++ b/tests/test_pipeline_big.py @@ -511,10 +511,9 @@ def test_pipe_FBP2d_astra_i12_119647_preview( # ######################################################################## @pytest.mark.full_data -@pytest.mark.mpi def test_parallel_pipe_FBP2d_astra_i12_119647_preview( get_files: Callable, - cmd, + cmd2, i12_119647, FBP2d_astra, FBP2d_astra_i12_119647_npz, @@ -534,12 +533,12 @@ def test_parallel_pipe_FBP2d_astra_i12_119647_preview( ], ) - cmd.pop(4) #: don't save all - cmd.insert(5, i12_119647) - cmd.insert(7, FBP2d_astra) - cmd.insert(8, output_folder) + cmd2.pop(5) #: don't save all + cmd2.insert(6, i12_119647) + cmd2.insert(8, FBP2d_astra) + cmd2.insert(9, output_folder) - subprocess.check_output(cmd) + subprocess.check_output(cmd2) files = get_files(output_folder) From c692ed2f652837b9348c60b982f4ce89328281de Mon Sep 17 00:00:00 2001 From: dkazanc Date: Tue, 8 Jul 2025 15:31:49 +0100 Subject: [PATCH 06/19] trying parallel tests6 --- tests/test_pipeline_big.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/tests/test_pipeline_big.py b/tests/test_pipeline_big.py index 3e8340ec5..f3447d984 100644 --- a/tests/test_pipeline_big.py +++ b/tests/test_pipeline_big.py @@ -1,6 +1,8 @@ import subprocess from typing import Callable, List, Tuple, Union +import pytest_parallel + import h5py import numpy as np import pytest @@ -511,9 +513,10 @@ def test_pipe_FBP2d_astra_i12_119647_preview( # ######################################################################## @pytest.mark.full_data +@pytest_parallel.mark.parallel(2) def test_parallel_pipe_FBP2d_astra_i12_119647_preview( get_files: Callable, - cmd2, + cmd, i12_119647, FBP2d_astra, FBP2d_astra_i12_119647_npz, @@ -533,12 +536,13 @@ def test_parallel_pipe_FBP2d_astra_i12_119647_preview( ], ) - cmd2.pop(5) #: don't save all - cmd2.insert(6, i12_119647) - cmd2.insert(8, FBP2d_astra) - cmd2.insert(9, output_folder) + cmd.pop(4) #: don't save all + cmd.insert(5, i12_119647) + cmd.insert(7, FBP2d_astra) + cmd.insert(8, output_folder) + - subprocess.check_output(cmd2) + subprocess.check_output(cmd) files = get_files(output_folder) @@ -577,6 +581,7 @@ def test_parallel_pipe_FBP2d_astra_i12_119647_preview( # ######################################################################## + @pytest.mark.full_data def test_pipe_FBP3d_tomobar_denoising_i13_177906_preview( get_files: Callable, From dfa41deba498f8b5bfb36924c75f723df6fcddc1 Mon Sep 17 00:00:00 2001 From: dkazanc Date: Tue, 8 Jul 2025 15:36:59 +0100 Subject: [PATCH 07/19] trying parallel tests7 --- tests/test_pipeline_big.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_pipeline_big.py b/tests/test_pipeline_big.py index f3447d984..c1aee6bbb 100644 --- a/tests/test_pipeline_big.py +++ b/tests/test_pipeline_big.py @@ -521,6 +521,7 @@ def test_parallel_pipe_FBP2d_astra_i12_119647_preview( FBP2d_astra, FBP2d_astra_i12_119647_npz, output_folder, + comm, ): change_value_parameters_method_pipeline( From 62a23887d59b842a247a70142382345789847429 Mon Sep 17 00:00:00 2001 From: dkazanc Date: Tue, 8 Jul 2025 15:42:52 +0100 Subject: [PATCH 08/19] trying parallel tests8 --- tests/test_pipeline_big.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/test_pipeline_big.py b/tests/test_pipeline_big.py index c1aee6bbb..ed16d0ed7 100644 --- a/tests/test_pipeline_big.py +++ b/tests/test_pipeline_big.py @@ -513,7 +513,7 @@ def test_pipe_FBP2d_astra_i12_119647_preview( # ######################################################################## @pytest.mark.full_data -@pytest_parallel.mark.parallel(2) +@pytest.mark.mpi def test_parallel_pipe_FBP2d_astra_i12_119647_preview( get_files: Callable, cmd, @@ -521,7 +521,6 @@ def test_parallel_pipe_FBP2d_astra_i12_119647_preview( FBP2d_astra, FBP2d_astra_i12_119647_npz, output_folder, - comm, ): change_value_parameters_method_pipeline( From 87a6f6575e7dea3cc2763eb95446c4ccd40ab491 Mon Sep 17 00:00:00 2001 From: dkazanc Date: Tue, 8 Jul 2025 15:44:11 +0100 Subject: [PATCH 09/19] trying parallel tests9 --- tests/test_pipeline_big.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/test_pipeline_big.py b/tests/test_pipeline_big.py index ed16d0ed7..8ddf78e69 100644 --- a/tests/test_pipeline_big.py +++ b/tests/test_pipeline_big.py @@ -1,8 +1,6 @@ import subprocess from typing import Callable, List, Tuple, Union -import pytest_parallel - import h5py import numpy as np import pytest From 152c092ed1b5f5657d770bdb65a6820ee8783ac1 Mon Sep 17 00:00:00 2001 From: dkazanc Date: Tue, 8 Jul 2025 15:49:08 +0100 Subject: [PATCH 10/19] trying parallel tests10 --- tests/test_pipeline_big.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/test_pipeline_big.py b/tests/test_pipeline_big.py index 8ddf78e69..ddb4741d8 100644 --- a/tests/test_pipeline_big.py +++ b/tests/test_pipeline_big.py @@ -509,6 +509,12 @@ def test_pipe_FBP2d_astra_i12_119647_preview( # ######################################################################## +@pytest.mark.full_data +@pytest.mark.mpi(min_size=2) +def test_size(): + from mpi4py import MPI + comm = MPI.COMM_WORLD + assert comm.size >= 2 @pytest.mark.full_data @pytest.mark.mpi From 81f6ed96c7804106b3d821a4faa1397cb99880a7 Mon Sep 17 00:00:00 2001 From: dkazanc Date: Tue, 15 Jul 2025 10:09:31 +0100 Subject: [PATCH 11/19] trying parallel tests11 --- tests/conftest.py | 1 - tests/test_pipeline_big.py | 18 +++++------------- 2 files changed, 5 insertions(+), 14 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index c16fc3920..b1b17988c 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -120,7 +120,6 @@ def cmd2(): "-m", "httomo", "run", - "--save-all", ] diff --git a/tests/test_pipeline_big.py b/tests/test_pipeline_big.py index ddb4741d8..25542c8ab 100644 --- a/tests/test_pipeline_big.py +++ b/tests/test_pipeline_big.py @@ -509,18 +509,11 @@ def test_pipe_FBP2d_astra_i12_119647_preview( # ######################################################################## -@pytest.mark.full_data -@pytest.mark.mpi(min_size=2) -def test_size(): - from mpi4py import MPI - comm = MPI.COMM_WORLD - assert comm.size >= 2 @pytest.mark.full_data -@pytest.mark.mpi def test_parallel_pipe_FBP2d_astra_i12_119647_preview( get_files: Callable, - cmd, + cmd2, i12_119647, FBP2d_astra, FBP2d_astra_i12_119647_npz, @@ -540,13 +533,12 @@ def test_parallel_pipe_FBP2d_astra_i12_119647_preview( ], ) - cmd.pop(4) #: don't save all - cmd.insert(5, i12_119647) - cmd.insert(7, FBP2d_astra) - cmd.insert(8, output_folder) + cmd2.insert(6, i12_119647) + cmd2.insert(7, FBP2d_astra) + cmd2.insert(8, output_folder) - subprocess.check_output(cmd) + subprocess.check_output(cmd2) files = get_files(output_folder) From 3758b713a62258fa21d2e6e3508fe29dcda6e986 Mon Sep 17 00:00:00 2001 From: dkazanc Date: Tue, 15 Jul 2025 16:21:51 +0100 Subject: [PATCH 12/19] working parallel mpi tests --- tests/conftest.py | 28 +- tests/test_parallel_pipeline_big.py | 381 ++++++++++++++++++++++++++++ tests/test_pipeline_big.py | 221 ---------------- 3 files changed, 406 insertions(+), 224 deletions(-) create mode 100644 tests/test_parallel_pipeline_big.py diff --git a/tests/conftest.py b/tests/conftest.py index b1b17988c..017d4212f 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -30,6 +30,9 @@ def pytest_configure(config): config.addinivalue_line( "markers", "full_data: mark tests to run full pipelines on raw big data" ) + config.addinivalue_line( + "markers", "full_data_parallel: mark tests to run full pipelines on raw big data in parallel" + ) config.addinivalue_line( "markers", "preview: mark test to run with `httomo preview`" ) @@ -54,6 +57,12 @@ def pytest_addoption(parser): default=False, help="run full pipelines on raw (big) data", ) + parser.addoption( + "--full_data_parallel", + action="store_true", + default=False, + help="run full pipelines on raw (big) data in parallel on two processes", + ) def pytest_collection_modifyitems(config, items): @@ -93,7 +102,18 @@ def pytest_collection_modifyitems(config, items): for item in items: if "full_data" in item.keywords: item.add_marker(skip_perf) - + if config.getoption("--full_data_parallel"): + skip_other = pytest.mark.skip(reason="not a pipeline raw big data test in parallel") + for item in items: + if "full_data_parallel" not in item.keywords: + item.add_marker(skip_other) + else: + skip_perf = pytest.mark.skip( + reason="pipeline raw big data test in parallel - use '--full_data_parallel' to run" + ) + for item in items: + if "full_data_parallel" in item.keywords: + item.add_marker(skip_perf) @pytest.fixture def output_folder(tmp_path): @@ -113,9 +133,11 @@ def cmd(): ] @pytest.fixture -def cmd2(): +def cmd_mpirun(): return [ - "mpirun -v -N 2", + "mpirun", + "-n", + "2", str(sys.executable), "-m", "httomo", diff --git a/tests/test_parallel_pipeline_big.py b/tests/test_parallel_pipeline_big.py new file mode 100644 index 000000000..76fea3534 --- /dev/null +++ b/tests/test_parallel_pipeline_big.py @@ -0,0 +1,381 @@ +import subprocess +from typing import Callable, List, Tuple, Union + +from subprocess import Popen, PIPE +import os + + +import h5py +import numpy as np +import pytest +from plumbum import local +from .conftest import change_value_parameters_method_pipeline, check_tif, compare_tif + +# NOTE: those tests have path integrated that are compatible with running jobs in Jenkins at DLS infrastructure. + +######################################################################## + + +@pytest.mark.full_data_parallel +def test_pipe_parallel_FBP3d_tomobar_k11_38730_in_disk( + get_files: Callable, + cmd_mpirun, + diad_k11_38730, + FBP3d_tomobar_noimagesaving, + FBP3d_tomobar_k11_38730_npz, + output_folder, +): + + change_value_parameters_method_pipeline( + FBP3d_tomobar_noimagesaving, + method=[ + "standard_tomo", + "standard_tomo", + "standard_tomo", + ], + key=[ + "data_path", + "image_key_path", + "rotation_angles", + ], + value=[ + "/entry/imaging/data", + "/entry/instrument/imaging/image_key", + {"data_path": "/entry/imaging_sum/gts_cs_theta"}, + ], + ) + + cmd_mpirun.insert(9, diad_k11_38730) + cmd_mpirun.insert(10, FBP3d_tomobar_noimagesaving) + cmd_mpirun.insert(11, output_folder) + cmd_mpirun.insert(12, "--max-memory") + cmd_mpirun.insert(13, "40G") + cmd_mpirun.insert(14, "--reslice-dir") + cmd_mpirun.insert(15, "/scratch/jenkins_agent/workspace/") + + + process = Popen(cmd_mpirun, env=os.environ, shell=False, stdin=PIPE, stdout=PIPE, stderr=PIPE) + output, error = process.communicate() + print (output) + + files = get_files(output_folder) + + #: check the generated reconstruction (hdf5 file) + h5_files = list(filter(lambda x: ".h5" in x, files)) + assert len(h5_files) == 1 + + # load the pre-saved numpy array for comparison bellow + data_gt = FBP3d_tomobar_k11_38730_npz["data"] + axis_slice = FBP3d_tomobar_k11_38730_npz["axis_slice"] + (slices, sizeX, sizeY) = np.shape(data_gt) + + step = axis_slice // (slices + 2) + # store for the result + data_result = np.zeros((slices, sizeX, sizeY), dtype=np.float32) + + path_to_data = "data/" + h5_file_name = "FBP3d_tomobar" + for file_to_open in h5_files: + if h5_file_name in file_to_open: + h5f = h5py.File(file_to_open, "r") + index_prog = step + for i in range(slices): + data_result[i, :, :] = h5f[path_to_data][:, index_prog, :] + index_prog += step + h5f.close() + else: + message_str = f"File name with {h5_file_name} string cannot be found." + raise FileNotFoundError(message_str) + + residual_im = data_gt - data_result + res_norm = np.linalg.norm(residual_im.flatten()).astype("float32") + assert res_norm < 1e-6 + + +# ######################################################################## + +@pytest.mark.full_data_parallel +def test_pipe_parallel_FBP3d_tomobar_k11_38730_in_memory( + get_files: Callable, + cmd_mpirun, + diad_k11_38730, + FBP3d_tomobar_noimagesaving, + FBP3d_tomobar_k11_38730_npz, + output_folder, +): + + change_value_parameters_method_pipeline( + FBP3d_tomobar_noimagesaving, + method=[ + "standard_tomo", + "standard_tomo", + "standard_tomo", + ], + key=[ + "data_path", + "image_key_path", + "rotation_angles", + ], + value=[ + "/entry/imaging/data", + "/entry/instrument/imaging/image_key", + {"data_path": "/entry/imaging_sum/gts_cs_theta"}, + ], + ) + + cmd_mpirun.insert(9, diad_k11_38730) + cmd_mpirun.insert(10, FBP3d_tomobar_noimagesaving) + cmd_mpirun.insert(11, output_folder) + + process = Popen(cmd_mpirun, env=os.environ, shell=False, stdin=PIPE, stdout=PIPE, stderr=PIPE) + output, error = process.communicate() # print (output) + print (output) + + files = get_files(output_folder) + + #: check the generated reconstruction (hdf5 file) + h5_files = list(filter(lambda x: ".h5" in x, files)) + assert len(h5_files) == 1 + + # load the pre-saved numpy array for comparison bellow + data_gt = FBP3d_tomobar_k11_38730_npz["data"] + axis_slice = FBP3d_tomobar_k11_38730_npz["axis_slice"] + (slices, sizeX, sizeY) = np.shape(data_gt) + + step = axis_slice // (slices + 2) + # store for the result + data_result = np.zeros((slices, sizeX, sizeY), dtype=np.float32) + + path_to_data = "data/" + h5_file_name = "FBP3d_tomobar" + for file_to_open in h5_files: + if h5_file_name in file_to_open: + h5f = h5py.File(file_to_open, "r") + index_prog = step + for i in range(slices): + data_result[i, :, :] = h5f[path_to_data][:, index_prog, :] + index_prog += step + h5f.close() + else: + message_str = f"File name with {h5_file_name} string cannot be found." + raise FileNotFoundError(message_str) + + residual_im = data_gt - data_result + res_norm = np.linalg.norm(residual_im.flatten()).astype("float32") + assert res_norm < 1e-6 + + +# ######################################################################## + +@pytest.mark.full_data_parallel +def test_parallel_pipe_FBP3d_tomobar_denoising_i13_177906_preview( + get_files: Callable, + cmd_mpirun, + i13_177906, + FBP3d_tomobar_denoising, + FBP3d_tomobar_TVdenoising_i13_177906_npz, + output_folder, +): + + change_value_parameters_method_pipeline( + FBP3d_tomobar_denoising, + method=[ + "standard_tomo", + ], + key=[ + "preview", + ], + value=[ + {"detector_y": {"start": 900, "stop": 1200}}, + ], + ) + + # do not save the result of FBP3d_tomobar + change_value_parameters_method_pipeline( + FBP3d_tomobar_denoising, + method=[ + "FBP3d_tomobar", + ], + key=[ + "recon_size", + ], + value=[ + None, + ], + save_result=False, + ) + + # save the result of denoising instead + change_value_parameters_method_pipeline( + FBP3d_tomobar_denoising, + method=[ + "total_variation_PD", + "total_variation_PD", + ], + key=[ + "regularisation_parameter", + "iterations", + ], + value=[ + 1.0e-04, + 25, + ], + save_result=True, + ) + + cmd_mpirun.insert(9, i13_177906) + cmd_mpirun.insert(10, FBP3d_tomobar_denoising) + cmd_mpirun.insert(11, output_folder) + + process = Popen(cmd_mpirun, env=os.environ, shell=False, stdin=PIPE, stdout=PIPE, stderr=PIPE) + output, error = process.communicate() + print (output) + + files = get_files(output_folder) + + #: check the generated reconstruction (hdf5 file) + h5_files = list(filter(lambda x: ".h5" in x, files)) + assert len(h5_files) == 1 + + # load the pre-saved numpy array for comparison bellow + data_gt_tv = FBP3d_tomobar_TVdenoising_i13_177906_npz["data"] + axis_slice = FBP3d_tomobar_TVdenoising_i13_177906_npz["axis_slice"] + (slices, sizeX, sizeY) = np.shape(data_gt_tv) + + step = axis_slice // (slices + 2) + # store for the result + data_result = np.zeros((slices, sizeX, sizeY), dtype=np.float32) + + path_to_data = "data/" + h5_file_name = "total_variation_PD" + for file_to_open in h5_files: + if h5_file_name in file_to_open: + h5f = h5py.File(file_to_open, "r") + index_prog = step + for i in range(slices): + data_result[i, :, :] = h5f[path_to_data][:, index_prog, :] + index_prog += step + h5f.close() + else: + message_str = f"File name with {h5_file_name} string cannot be found." + raise FileNotFoundError(message_str) + + residual_im = data_gt_tv - data_result + res_norm_tv_res = np.linalg.norm(residual_im.flatten()).astype("float32") + assert res_norm_tv_res < 1e-2 + +# ######################################################################## + +@pytest.mark.full_data_parallel +def test_parallel_pipe_360deg_distortion_FBP3d_tomobar_i13_179623_preview( + get_files: Callable, + cmd_mpirun, + i13_179623, + deg360_distortion_FBP3d_tomobar, + FBP3d_tomobar_distortion_i13_179623_npz, + output_folder, +): + change_value_parameters_method_pipeline( + deg360_distortion_FBP3d_tomobar, + method=[ + "standard_tomo", + "normalize", + "find_center_360", + "find_center_360", + "find_center_360", + "distortion_correction_proj_discorpy", + "FBP", + ], + key=[ + "preview", + "minus_log", + "ind", + "use_overlap", + "norm", + "metadata_path", + "neglog", + ], + value=[ + {"detector_y": {"start": 900, "stop": 1200}}, + False, + "mid", + True, + True, + "/data/tomography/raw_data/i13/360/179623_coeff.txt", + True, + ], + ) + + cmd_mpirun.insert(9, i13_179623) + cmd_mpirun.insert(10, deg360_distortion_FBP3d_tomobar) + cmd_mpirun.insert(11, output_folder) + + process = Popen(cmd_mpirun, env=os.environ, shell=False, stdin=PIPE, stdout=PIPE, stderr=PIPE) + output, error = process.communicate() + print (output) + + files = get_files(output_folder) + + #: check the generated reconstruction (hdf5 file) + h5_files = list(filter(lambda x: ".h5" in x, files)) + assert len(h5_files) == 1 + + # load the pre-saved numpy array for comparison bellow + data_gt = FBP3d_tomobar_distortion_i13_179623_npz["data"] + axis_slice = FBP3d_tomobar_distortion_i13_179623_npz["axis_slice"] + (slices, sizeX, sizeY) = np.shape(data_gt) + + step = axis_slice // (slices + 2) + # store for the result + data_result = np.zeros((slices, sizeX, sizeY), dtype=np.float32) + + path_to_data = "data/" + h5_file_name = "FBP3d_tomobar" + for file_to_open in h5_files: + if h5_file_name in file_to_open: + h5f = h5py.File(file_to_open, "r") + index_prog = step + for i in range(slices): + data_result[i, :, :] = h5f[path_to_data][:, index_prog, :] + index_prog += step + h5f.close() + else: + message_str = f"File name with {h5_file_name} string cannot be found." + raise FileNotFoundError(message_str) + + residual_im = data_gt - data_result + res_norm = np.linalg.norm(residual_im.flatten()).astype("float32") + assert res_norm < 1e-4 + + +# ######################################################################## +@pytest.mark.full_data_parallel +def test_parallel_pipe_sweep_FBP3d_tomobar_i13_177906( + get_files: Callable, + cmd_mpirun, + i13_177906, + sweep_center_FBP3d_tomobar, + pipeline_sweep_FBP3d_tomobar_i13_177906_tiffs, + output_folder, +): + cmd_mpirun.insert(9, i13_177906) + cmd_mpirun.insert(10, sweep_center_FBP3d_tomobar) + cmd_mpirun.insert(11, output_folder) + + process = Popen(cmd_mpirun, env=os.environ, shell=False, stdin=PIPE, stdout=PIPE, stderr=PIPE) + output, error = process.communicate() + print (output) + + files = get_files(output_folder) + files_references = get_files(pipeline_sweep_FBP3d_tomobar_i13_177906_tiffs) + + # recurse through output_dir and check that all files are there + files = get_files(output_folder) + assert len(files) == 12 + + #: check the number of the resulting tif files + check_tif(files, 8, (2560, 2560)) + compare_tif(files, files_references) + + +# ######################################################################## diff --git a/tests/test_pipeline_big.py b/tests/test_pipeline_big.py index 25542c8ab..ff0d6fa6b 100644 --- a/tests/test_pipeline_big.py +++ b/tests/test_pipeline_big.py @@ -1,6 +1,5 @@ import subprocess from typing import Callable, List, Tuple, Union - import h5py import numpy as np import pytest @@ -157,158 +156,6 @@ def test_pipe_FBP3d_tomobar_k11_38731_in_memory( res_norm = np.linalg.norm(residual_im.flatten()).astype("float32") assert res_norm < 1e-6 - -######################################################################## - - -@pytest.mark.full_data -def test_pipe_FBP3d_tomobar_k11_38730_in_disk( - get_files: Callable, - cmd, - diad_k11_38730, - FBP3d_tomobar_noimagesaving, - FBP3d_tomobar_k11_38730_npz, - output_folder, -): - - change_value_parameters_method_pipeline( - FBP3d_tomobar_noimagesaving, - method=[ - "standard_tomo", - "standard_tomo", - "standard_tomo", - ], - key=[ - "data_path", - "image_key_path", - "rotation_angles", - ], - value=[ - "/entry/imaging/data", - "/entry/instrument/imaging/image_key", - {"data_path": "/entry/imaging_sum/gts_cs_theta"}, - ], - ) - - # NOTE that the intermediate file with file-based processing will be saved to /tmp - cmd.pop(4) #: don't save all - cmd.insert(5, diad_k11_38730) - cmd.insert(7, FBP3d_tomobar_noimagesaving) - cmd.insert(8, output_folder) - cmd.insert(9, "--max-memory") - cmd.insert(10, "40G") - cmd.insert(11, "--reslice-dir") - cmd.insert(12, "/scratch/jenkins_agent/workspace/") - - subprocess.check_output(cmd) - - files = get_files(output_folder) - - #: check the generated reconstruction (hdf5 file) - h5_files = list(filter(lambda x: ".h5" in x, files)) - assert len(h5_files) == 1 - - # load the pre-saved numpy array for comparison bellow - data_gt = FBP3d_tomobar_k11_38730_npz["data"] - axis_slice = FBP3d_tomobar_k11_38730_npz["axis_slice"] - (slices, sizeX, sizeY) = np.shape(data_gt) - - step = axis_slice // (slices + 2) - # store for the result - data_result = np.zeros((slices, sizeX, sizeY), dtype=np.float32) - - path_to_data = "data/" - h5_file_name = "FBP3d_tomobar" - for file_to_open in h5_files: - if h5_file_name in file_to_open: - h5f = h5py.File(file_to_open, "r") - index_prog = step - for i in range(slices): - data_result[i, :, :] = h5f[path_to_data][:, index_prog, :] - index_prog += step - h5f.close() - else: - message_str = f"File name with {h5_file_name} string cannot be found." - raise FileNotFoundError(message_str) - - residual_im = data_gt - data_result - res_norm = np.linalg.norm(residual_im.flatten()).astype("float32") - assert res_norm < 1e-6 - - -# ######################################################################## - - -@pytest.mark.full_data -def test_pipe_FBP3d_tomobar_k11_38730_in_memory( - get_files: Callable, - cmd, - diad_k11_38730, - FBP3d_tomobar_noimagesaving, - FBP3d_tomobar_k11_38730_npz, - output_folder, -): - - change_value_parameters_method_pipeline( - FBP3d_tomobar_noimagesaving, - method=[ - "standard_tomo", - "standard_tomo", - "standard_tomo", - ], - key=[ - "data_path", - "image_key_path", - "rotation_angles", - ], - value=[ - "/entry/imaging/data", - "/entry/instrument/imaging/image_key", - {"data_path": "/entry/imaging_sum/gts_cs_theta"}, - ], - ) - - cmd.pop(4) #: don't save all - cmd.insert(5, diad_k11_38730) - cmd.insert(7, FBP3d_tomobar_noimagesaving) - cmd.insert(8, output_folder) - - subprocess.check_output(cmd) - - files = get_files(output_folder) - - #: check the generated reconstruction (hdf5 file) - h5_files = list(filter(lambda x: ".h5" in x, files)) - assert len(h5_files) == 1 - - # load the pre-saved numpy array for comparison bellow - data_gt = FBP3d_tomobar_k11_38730_npz["data"] - axis_slice = FBP3d_tomobar_k11_38730_npz["axis_slice"] - (slices, sizeX, sizeY) = np.shape(data_gt) - - step = axis_slice // (slices + 2) - # store for the result - data_result = np.zeros((slices, sizeX, sizeY), dtype=np.float32) - - path_to_data = "data/" - h5_file_name = "FBP3d_tomobar" - for file_to_open in h5_files: - if h5_file_name in file_to_open: - h5f = h5py.File(file_to_open, "r") - index_prog = step - for i in range(slices): - data_result[i, :, :] = h5f[path_to_data][:, index_prog, :] - index_prog += step - h5f.close() - else: - message_str = f"File name with {h5_file_name} string cannot be found." - raise FileNotFoundError(message_str) - - residual_im = data_gt - data_result - res_norm = np.linalg.norm(residual_im.flatten()).astype("float32") - assert res_norm < 1e-6 - - # ######################################################################## @pytest.mark.full_data def test_pipe_FBP3d_tomobar_i12_119647_preview( @@ -510,74 +357,6 @@ def test_pipe_FBP2d_astra_i12_119647_preview( # ######################################################################## -@pytest.mark.full_data -def test_parallel_pipe_FBP2d_astra_i12_119647_preview( - get_files: Callable, - cmd2, - i12_119647, - FBP2d_astra, - FBP2d_astra_i12_119647_npz, - output_folder, -): - - change_value_parameters_method_pipeline( - FBP2d_astra, - method=[ - "standard_tomo", - ], - key=[ - "preview", - ], - value=[ - {"detector_y": {"start": 900, "stop": 1200}}, - ], - ) - - cmd2.insert(6, i12_119647) - cmd2.insert(7, FBP2d_astra) - cmd2.insert(8, output_folder) - - - subprocess.check_output(cmd2) - - files = get_files(output_folder) - - #: check the generated reconstruction (hdf5 file) - h5_files = list(filter(lambda x: ".h5" in x, files)) - assert len(h5_files) == 1 - - # load the pre-saved numpy array for comparison bellow - data_gt = FBP2d_astra_i12_119647_npz["data"] - axis_slice = FBP2d_astra_i12_119647_npz["axis_slice"] - (slices, sizeX, sizeY) = np.shape(data_gt) - - step = axis_slice // (slices + 2) - # store for the result - data_result = np.zeros((slices, sizeX, sizeY), dtype=np.float32) - - path_to_data = "data/" - h5_file_name = "FBP2d_astra" - for file_to_open in h5_files: - if h5_file_name in file_to_open: - h5f = h5py.File(file_to_open, "r") - index_prog = step - for i in range(slices): - data_result[i, :, :] = h5f[path_to_data][:, index_prog, :] - index_prog += step - h5f.close() - else: - message_str = f"File name with {h5_file_name} string cannot be found." - raise FileNotFoundError(message_str) - - residual_im = data_gt - data_result - res_norm = np.linalg.norm(residual_im.flatten()).astype("float32") - assert res_norm < 1e-6 - - -# ######################################################################## - - - @pytest.mark.full_data def test_pipe_FBP3d_tomobar_denoising_i13_177906_preview( get_files: Callable, From 0c5d578a90508b37bfbdbffbe096bc0f320f38bf Mon Sep 17 00:00:00 2001 From: dkazanc Date: Wed, 16 Jul 2025 10:56:57 +0100 Subject: [PATCH 13/19] adding lprec parallel test --- tests/test_parallel_pipeline_big.py | 101 +++++++++++++++++++--------- tests/test_pipeline_big.py | 85 ----------------------- 2 files changed, 69 insertions(+), 117 deletions(-) diff --git a/tests/test_parallel_pipeline_big.py b/tests/test_parallel_pipeline_big.py index 76fea3534..9c670eb0d 100644 --- a/tests/test_parallel_pipeline_big.py +++ b/tests/test_parallel_pipeline_big.py @@ -262,7 +262,75 @@ def test_parallel_pipe_FBP3d_tomobar_denoising_i13_177906_preview( residual_im = data_gt_tv - data_result res_norm_tv_res = np.linalg.norm(residual_im.flatten()).astype("float32") - assert res_norm_tv_res < 1e-2 + assert res_norm_tv_res < 1e-1 + +# ######################################################################## + + +@pytest.mark.full_data +def test_parallel_pipe_LPRec3d_tomobar_i12_119647_preview( + get_files: Callable, + cmd_mpirun, + i12_119647, + LPRec3d_tomobar, + LPRec3d_tomobar_i12_119647_npz, + output_folder, +): + + change_value_parameters_method_pipeline( + LPRec3d_tomobar, + method=[ + "standard_tomo", + ], + key=[ + "preview", + ], + value=[ + {"detector_y": {"start": 900, "stop": 1200}}, + ], + ) + + + cmd_mpirun.insert(9, i12_119647) + cmd_mpirun.insert(10, LPRec3d_tomobar) + cmd_mpirun.insert(11, output_folder) + + process = Popen(cmd_mpirun, env=os.environ, shell=False, stdin=PIPE, stdout=PIPE, stderr=PIPE) + output, error = process.communicate() + print (output) + + files = get_files(output_folder) + + #: check the generated reconstruction (hdf5 file) + h5_files = list(filter(lambda x: ".h5" in x, files)) + assert len(h5_files) == 1 + + # load the pre-saved numpy array for comparison bellow + data_gt = LPRec3d_tomobar_i12_119647_npz["data"] + axis_slice = LPRec3d_tomobar_i12_119647_npz["axis_slice"] + (slices, sizeX, sizeY) = np.shape(data_gt) + + step = axis_slice // (slices + 2) + # store for the result + data_result = np.zeros((slices, sizeX, sizeY), dtype=np.float32) + + path_to_data = "data/" + h5_file_name = "LPRec3d_tomobar" + for file_to_open in h5_files: + if h5_file_name in file_to_open: + h5f = h5py.File(file_to_open, "r") + index_prog = step + for i in range(slices): + data_result[i, :, :] = h5f[path_to_data][:, index_prog, :] + index_prog += step + h5f.close() + else: + message_str = f"File name with {h5_file_name} string cannot be found." + raise FileNotFoundError(message_str) + + residual_im = data_gt - data_result + res_norm = np.linalg.norm(residual_im.flatten()).astype("float32") + assert res_norm < 0.02 # ######################################################################## @@ -347,35 +415,4 @@ def test_parallel_pipe_360deg_distortion_FBP3d_tomobar_i13_179623_preview( res_norm = np.linalg.norm(residual_im.flatten()).astype("float32") assert res_norm < 1e-4 - -# ######################################################################## -@pytest.mark.full_data_parallel -def test_parallel_pipe_sweep_FBP3d_tomobar_i13_177906( - get_files: Callable, - cmd_mpirun, - i13_177906, - sweep_center_FBP3d_tomobar, - pipeline_sweep_FBP3d_tomobar_i13_177906_tiffs, - output_folder, -): - cmd_mpirun.insert(9, i13_177906) - cmd_mpirun.insert(10, sweep_center_FBP3d_tomobar) - cmd_mpirun.insert(11, output_folder) - - process = Popen(cmd_mpirun, env=os.environ, shell=False, stdin=PIPE, stdout=PIPE, stderr=PIPE) - output, error = process.communicate() - print (output) - - files = get_files(output_folder) - files_references = get_files(pipeline_sweep_FBP3d_tomobar_i13_177906_tiffs) - - # recurse through output_dir and check that all files are there - files = get_files(output_folder) - assert len(files) == 12 - - #: check the number of the resulting tif files - check_tif(files, 8, (2560, 2560)) - compare_tif(files, files_references) - - # ######################################################################## diff --git a/tests/test_pipeline_big.py b/tests/test_pipeline_big.py index ff0d6fa6b..f05cb3147 100644 --- a/tests/test_pipeline_big.py +++ b/tests/test_pipeline_big.py @@ -531,91 +531,6 @@ def test_pipe_360deg_paganin_FBP3d_tomobar_i13_179623_preview( res_norm = np.linalg.norm(residual_im.flatten()).astype("float32") assert res_norm < 1e-4 - -# ######################################################################## - - -@pytest.mark.full_data -def test_pipe_360deg_distortion_FBP3d_tomobar_i13_179623_preview( - get_files: Callable, - cmd, - i13_179623, - deg360_distortion_FBP3d_tomobar, - FBP3d_tomobar_distortion_i13_179623_npz, - output_folder, -): - change_value_parameters_method_pipeline( - deg360_distortion_FBP3d_tomobar, - method=[ - "standard_tomo", - "normalize", - "find_center_360", - "find_center_360", - "find_center_360", - "distortion_correction_proj_discorpy", - "FBP", - ], - key=[ - "preview", - "minus_log", - "ind", - "use_overlap", - "norm", - "metadata_path", - "neglog", - ], - value=[ - {"detector_y": {"start": 900, "stop": 1200}}, - False, - "mid", - True, - True, - "/data/tomography/raw_data/i13/360/179623_coeff.txt", - True, - ], - ) - - cmd.pop(4) #: don't save all - cmd.insert(5, i13_179623) - cmd.insert(7, deg360_distortion_FBP3d_tomobar) - cmd.insert(8, output_folder) - - subprocess.check_output(cmd) - - files = get_files(output_folder) - - #: check the generated reconstruction (hdf5 file) - h5_files = list(filter(lambda x: ".h5" in x, files)) - assert len(h5_files) == 1 - - # load the pre-saved numpy array for comparison bellow - data_gt = FBP3d_tomobar_distortion_i13_179623_npz["data"] - axis_slice = FBP3d_tomobar_distortion_i13_179623_npz["axis_slice"] - (slices, sizeX, sizeY) = np.shape(data_gt) - - step = axis_slice // (slices + 2) - # store for the result - data_result = np.zeros((slices, sizeX, sizeY), dtype=np.float32) - - path_to_data = "data/" - h5_file_name = "FBP3d_tomobar" - for file_to_open in h5_files: - if h5_file_name in file_to_open: - h5f = h5py.File(file_to_open, "r") - index_prog = step - for i in range(slices): - data_result[i, :, :] = h5f[path_to_data][:, index_prog, :] - index_prog += step - h5f.close() - else: - message_str = f"File name with {h5_file_name} string cannot be found." - raise FileNotFoundError(message_str) - - residual_im = data_gt - data_result - res_norm = np.linalg.norm(residual_im.flatten()).astype("float32") - assert res_norm < 1e-4 - - # ######################################################################## @pytest.mark.full_data def test_pipe_sweep_FBP3d_tomobar_i13_177906( From eee0c11afb37fd8a7c01642929d7a5fde699f8cd Mon Sep 17 00:00:00 2001 From: dkazanc Date: Wed, 16 Jul 2025 13:47:40 +0100 Subject: [PATCH 14/19] indisk test modification --- tests/test_parallel_pipeline_big.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_parallel_pipeline_big.py b/tests/test_parallel_pipeline_big.py index 9c670eb0d..f31e693ea 100644 --- a/tests/test_parallel_pipeline_big.py +++ b/tests/test_parallel_pipeline_big.py @@ -49,9 +49,9 @@ def test_pipe_parallel_FBP3d_tomobar_k11_38730_in_disk( cmd_mpirun.insert(10, FBP3d_tomobar_noimagesaving) cmd_mpirun.insert(11, output_folder) cmd_mpirun.insert(12, "--max-memory") - cmd_mpirun.insert(13, "40G") + cmd_mpirun.insert(13, "10G") cmd_mpirun.insert(14, "--reslice-dir") - cmd_mpirun.insert(15, "/scratch/jenkins_agent/workspace/") + cmd_mpirun.insert(15, "/data/tomography/temp") process = Popen(cmd_mpirun, env=os.environ, shell=False, stdin=PIPE, stdout=PIPE, stderr=PIPE) From 319d2df61fa125b19fdea3d101a3356c76c21f1a Mon Sep 17 00:00:00 2001 From: dkazanc Date: Wed, 16 Jul 2025 14:05:39 +0100 Subject: [PATCH 15/19] revert the change --- tests/test_parallel_pipeline_big.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_parallel_pipeline_big.py b/tests/test_parallel_pipeline_big.py index f31e693ea..31e09792d 100644 --- a/tests/test_parallel_pipeline_big.py +++ b/tests/test_parallel_pipeline_big.py @@ -51,7 +51,7 @@ def test_pipe_parallel_FBP3d_tomobar_k11_38730_in_disk( cmd_mpirun.insert(12, "--max-memory") cmd_mpirun.insert(13, "10G") cmd_mpirun.insert(14, "--reslice-dir") - cmd_mpirun.insert(15, "/data/tomography/temp") + cmd_mpirun.insert(15, "/scratch/jenkins_agent/workspace/") process = Popen(cmd_mpirun, env=os.environ, shell=False, stdin=PIPE, stdout=PIPE, stderr=PIPE) From b76b0622bd1f2290d7888124648f8d547c2edc4e Mon Sep 17 00:00:00 2001 From: dkazanc Date: Wed, 16 Jul 2025 14:25:36 +0100 Subject: [PATCH 16/19] tests correction --- tests/test_parallel_pipeline_big.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_parallel_pipeline_big.py b/tests/test_parallel_pipeline_big.py index 31e09792d..3a211a79b 100644 --- a/tests/test_parallel_pipeline_big.py +++ b/tests/test_parallel_pipeline_big.py @@ -267,7 +267,7 @@ def test_parallel_pipe_FBP3d_tomobar_denoising_i13_177906_preview( # ######################################################################## -@pytest.mark.full_data +@pytest.mark.full_data_parallel def test_parallel_pipe_LPRec3d_tomobar_i12_119647_preview( get_files: Callable, cmd_mpirun, From 3bf5f155d00a7ecf8567e546c1a9fdf34aa83bb5 Mon Sep 17 00:00:00 2001 From: dkazanc Date: Thu, 17 Jul 2025 10:43:20 +0100 Subject: [PATCH 17/19] adding new parallel tests --- tests/test_parallel_pipeline_big.py | 127 +++++++++++++++++++++++++++- tests/test_pipeline_big.py | 73 ---------------- 2 files changed, 124 insertions(+), 76 deletions(-) diff --git a/tests/test_parallel_pipeline_big.py b/tests/test_parallel_pipeline_big.py index 3a211a79b..9c821aff8 100644 --- a/tests/test_parallel_pipeline_big.py +++ b/tests/test_parallel_pipeline_big.py @@ -56,7 +56,7 @@ def test_pipe_parallel_FBP3d_tomobar_k11_38730_in_disk( process = Popen(cmd_mpirun, env=os.environ, shell=False, stdin=PIPE, stdout=PIPE, stderr=PIPE) output, error = process.communicate() - print (output) + print(output) files = get_files(output_folder) @@ -128,8 +128,8 @@ def test_pipe_parallel_FBP3d_tomobar_k11_38730_in_memory( cmd_mpirun.insert(11, output_folder) process = Popen(cmd_mpirun, env=os.environ, shell=False, stdin=PIPE, stdout=PIPE, stderr=PIPE) - output, error = process.communicate() # print (output) - print (output) + output, error = process.communicate() + print(output) files = get_files(output_folder) @@ -164,6 +164,127 @@ def test_pipe_parallel_FBP3d_tomobar_k11_38730_in_memory( res_norm = np.linalg.norm(residual_im.flatten()).astype("float32") assert res_norm < 1e-6 +# ######################################################################## + +@pytest.mark.full_data_parallel +def test_parallel_pipe_FBP3d_tomobar_k11_38731_in_memory( + get_files: Callable, + cmd_mpirun, + diad_k11_38731, + FBP3d_tomobar, + FBP3d_tomobar_k11_38731_npz, + output_folder, +): + + change_value_parameters_method_pipeline( + FBP3d_tomobar, + method=[ + "standard_tomo", + "standard_tomo", + "standard_tomo", + ], + key=[ + "data_path", + "image_key_path", + "rotation_angles", + ], + value=[ + "/entry/imaging/data", + "/entry/instrument/imaging/image_key", + {"data_path": "/entry/imaging_sum/gts_cs_theta"}, + ], + ) + + cmd_mpirun.insert(9, diad_k11_38731) + cmd_mpirun.insert(10, FBP3d_tomobar) + cmd_mpirun.insert(11, output_folder) + + process = Popen(cmd_mpirun, env=os.environ, shell=False, stdin=PIPE, stdout=PIPE, stderr=PIPE) + output, error = process.communicate() + print(output) + + files = get_files(output_folder) + + #: check the generated reconstruction (hdf5 file) + h5_files = list(filter(lambda x: ".h5" in x, files)) + assert len(h5_files) == 1 + + # load the pre-saved numpy array for comparison bellow + data_gt = FBP3d_tomobar_k11_38731_npz["data"] + axis_slice = FBP3d_tomobar_k11_38731_npz["axis_slice"] + (slices, sizeX, sizeY) = np.shape(data_gt) + + check_tif(files, 2160, (2560, 2560)) + + step = axis_slice // (slices + 2) + # store for the result + data_result = np.zeros((slices, sizeX, sizeY), dtype=np.float32) + + path_to_data = "data/" + h5_file_name = "FBP3d_tomobar" + for file_to_open in h5_files: + print(file_to_open) + if h5_file_name in file_to_open: + h5f = h5py.File(file_to_open, "r") + index_prog = step + for i in range(slices): + data_result[i, :, :] = h5f[path_to_data][:, index_prog, :] + index_prog += step + h5f.close() + else: + message_str = f"File name with {h5_file_name} string cannot be found." + raise FileNotFoundError(message_str) + + residual_im = data_gt - data_result + res_norm = np.linalg.norm(residual_im.flatten()).astype("float32") + assert res_norm < 1e-6 + +# ######################################################################## + +@pytest.mark.full_data_parallel +def test_parallel_pipe_FBP3d_tomobar_k11_38729_in_memory( + get_files: Callable, + cmd_mpirun, + diad_k11_38729, + FBP3d_tomobar, + FBP3d_tomobar_k11_38731_npz, + output_folder, +): + + change_value_parameters_method_pipeline( + FBP3d_tomobar, + method=[ + "standard_tomo", + "standard_tomo", + "standard_tomo", + ], + key=[ + "data_path", + "image_key_path", + "rotation_angles", + ], + value=[ + "/entry/imaging/data", + "/entry/instrument/imaging/image_key", + {"data_path": "/entry/imaging_sum/gts_cs_theta"}, + ], + ) + + cmd_mpirun.insert(9, diad_k11_38729) + cmd_mpirun.insert(10, FBP3d_tomobar) + cmd_mpirun.insert(11, output_folder) + + process = Popen(cmd_mpirun, env=os.environ, shell=False, stdin=PIPE, stdout=PIPE, stderr=PIPE) + output, error = process.communicate() + print(output) + + files = get_files(output_folder) + + #: check the generated reconstruction (hdf5 file) + h5_files = list(filter(lambda x: ".h5" in x, files)) + assert len(h5_files) == 1 + + check_tif(files, 2160, (2560, 2560)) # ######################################################################## diff --git a/tests/test_pipeline_big.py b/tests/test_pipeline_big.py index f05cb3147..7acaaea38 100644 --- a/tests/test_pipeline_big.py +++ b/tests/test_pipeline_big.py @@ -83,79 +83,6 @@ def test_pipe_FBP3d_tomobar_k11_38731_in_disk( assert res_norm < 1e-6 -####################################################################### - - -@pytest.mark.full_data -def test_pipe_FBP3d_tomobar_k11_38731_in_memory( - get_files: Callable, - cmd, - diad_k11_38731, - FBP3d_tomobar_noimagesaving, - FBP3d_tomobar_k11_38731_npz, - output_folder, -): - - change_value_parameters_method_pipeline( - FBP3d_tomobar_noimagesaving, - method=[ - "standard_tomo", - "standard_tomo", - "standard_tomo", - ], - key=[ - "data_path", - "image_key_path", - "rotation_angles", - ], - value=[ - "/entry/imaging/data", - "/entry/instrument/imaging/image_key", - {"data_path": "/entry/imaging_sum/gts_cs_theta"}, - ], - ) - - cmd.pop(4) #: don't save all - cmd.insert(5, diad_k11_38731) - cmd.insert(7, FBP3d_tomobar_noimagesaving) - cmd.insert(8, output_folder) - - subprocess.check_output(cmd) - - files = get_files(output_folder) - - #: check the generated reconstruction (hdf5 file) - h5_files = list(filter(lambda x: ".h5" in x, files)) - assert len(h5_files) == 1 - - # load the pre-saved numpy array for comparison bellow - data_gt = FBP3d_tomobar_k11_38731_npz["data"] - axis_slice = FBP3d_tomobar_k11_38731_npz["axis_slice"] - (slices, sizeX, sizeY) = np.shape(data_gt) - - step = axis_slice // (slices + 2) - # store for the result - data_result = np.zeros((slices, sizeX, sizeY), dtype=np.float32) - - path_to_data = "data/" - h5_file_name = "FBP3d_tomobar" - for file_to_open in h5_files: - print(file_to_open) - if h5_file_name in file_to_open: - h5f = h5py.File(file_to_open, "r") - index_prog = step - for i in range(slices): - data_result[i, :, :] = h5f[path_to_data][:, index_prog, :] - index_prog += step - h5f.close() - else: - message_str = f"File name with {h5_file_name} string cannot be found." - raise FileNotFoundError(message_str) - - residual_im = data_gt - data_result - res_norm = np.linalg.norm(residual_im.flatten()).astype("float32") - assert res_norm < 1e-6 - # ######################################################################## @pytest.mark.full_data def test_pipe_FBP3d_tomobar_i12_119647_preview( From 8eccb4b897e76d39e01bce7ee05962edf2df2299 Mon Sep 17 00:00:00 2001 From: dkazanc Date: Fri, 25 Jul 2025 10:06:02 +0100 Subject: [PATCH 18/19] reformatting and removing the stalling test --- tests/conftest.py | 17 ++++-- tests/test_parallel_pipeline_big.py | 92 ++++++++++------------------- tests/test_pipeline_big.py | 2 + 3 files changed, 45 insertions(+), 66 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 017d4212f..38cc38cb9 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -31,8 +31,9 @@ def pytest_configure(config): "markers", "full_data: mark tests to run full pipelines on raw big data" ) config.addinivalue_line( - "markers", "full_data_parallel: mark tests to run full pipelines on raw big data in parallel" - ) + "markers", + "full_data_parallel: mark tests to run full pipelines on raw big data in parallel", + ) config.addinivalue_line( "markers", "preview: mark test to run with `httomo preview`" ) @@ -62,7 +63,7 @@ def pytest_addoption(parser): action="store_true", default=False, help="run full pipelines on raw (big) data in parallel on two processes", - ) + ) def pytest_collection_modifyitems(config, items): @@ -103,7 +104,9 @@ def pytest_collection_modifyitems(config, items): if "full_data" in item.keywords: item.add_marker(skip_perf) if config.getoption("--full_data_parallel"): - skip_other = pytest.mark.skip(reason="not a pipeline raw big data test in parallel") + skip_other = pytest.mark.skip( + reason="not a pipeline raw big data test in parallel" + ) for item in items: if "full_data_parallel" not in item.keywords: item.add_marker(skip_other) @@ -115,6 +118,7 @@ def pytest_collection_modifyitems(config, items): if "full_data_parallel" in item.keywords: item.add_marker(skip_perf) + @pytest.fixture def output_folder(tmp_path): tmp_dir = tmp_path / "output_dir" @@ -132,12 +136,13 @@ def cmd(): "--save-all", ] + @pytest.fixture -def cmd_mpirun(): +def cmd_mpirun(): return [ "mpirun", "-n", - "2", + "2", str(sys.executable), "-m", "httomo", diff --git a/tests/test_parallel_pipeline_big.py b/tests/test_parallel_pipeline_big.py index 9c821aff8..1e54cfeb3 100644 --- a/tests/test_parallel_pipeline_big.py +++ b/tests/test_parallel_pipeline_big.py @@ -53,8 +53,9 @@ def test_pipe_parallel_FBP3d_tomobar_k11_38730_in_disk( cmd_mpirun.insert(14, "--reslice-dir") cmd_mpirun.insert(15, "/scratch/jenkins_agent/workspace/") - - process = Popen(cmd_mpirun, env=os.environ, shell=False, stdin=PIPE, stdout=PIPE, stderr=PIPE) + process = Popen( + cmd_mpirun, env=os.environ, shell=False, stdin=PIPE, stdout=PIPE, stderr=PIPE + ) output, error = process.communicate() print(output) @@ -94,6 +95,7 @@ def test_pipe_parallel_FBP3d_tomobar_k11_38730_in_disk( # ######################################################################## + @pytest.mark.full_data_parallel def test_pipe_parallel_FBP3d_tomobar_k11_38730_in_memory( get_files: Callable, @@ -127,7 +129,9 @@ def test_pipe_parallel_FBP3d_tomobar_k11_38730_in_memory( cmd_mpirun.insert(10, FBP3d_tomobar_noimagesaving) cmd_mpirun.insert(11, output_folder) - process = Popen(cmd_mpirun, env=os.environ, shell=False, stdin=PIPE, stdout=PIPE, stderr=PIPE) + process = Popen( + cmd_mpirun, env=os.environ, shell=False, stdin=PIPE, stdout=PIPE, stderr=PIPE + ) output, error = process.communicate() print(output) @@ -164,8 +168,10 @@ def test_pipe_parallel_FBP3d_tomobar_k11_38730_in_memory( res_norm = np.linalg.norm(residual_im.flatten()).astype("float32") assert res_norm < 1e-6 + # ######################################################################## + @pytest.mark.full_data_parallel def test_parallel_pipe_FBP3d_tomobar_k11_38731_in_memory( get_files: Callable, @@ -199,7 +205,9 @@ def test_parallel_pipe_FBP3d_tomobar_k11_38731_in_memory( cmd_mpirun.insert(10, FBP3d_tomobar) cmd_mpirun.insert(11, output_folder) - process = Popen(cmd_mpirun, env=os.environ, shell=False, stdin=PIPE, stdout=PIPE, stderr=PIPE) + process = Popen( + cmd_mpirun, env=os.environ, shell=False, stdin=PIPE, stdout=PIPE, stderr=PIPE + ) output, error = process.communicate() print(output) @@ -239,55 +247,10 @@ def test_parallel_pipe_FBP3d_tomobar_k11_38731_in_memory( res_norm = np.linalg.norm(residual_im.flatten()).astype("float32") assert res_norm < 1e-6 -# ######################################################################## - -@pytest.mark.full_data_parallel -def test_parallel_pipe_FBP3d_tomobar_k11_38729_in_memory( - get_files: Callable, - cmd_mpirun, - diad_k11_38729, - FBP3d_tomobar, - FBP3d_tomobar_k11_38731_npz, - output_folder, -): - - change_value_parameters_method_pipeline( - FBP3d_tomobar, - method=[ - "standard_tomo", - "standard_tomo", - "standard_tomo", - ], - key=[ - "data_path", - "image_key_path", - "rotation_angles", - ], - value=[ - "/entry/imaging/data", - "/entry/instrument/imaging/image_key", - {"data_path": "/entry/imaging_sum/gts_cs_theta"}, - ], - ) - - cmd_mpirun.insert(9, diad_k11_38729) - cmd_mpirun.insert(10, FBP3d_tomobar) - cmd_mpirun.insert(11, output_folder) - - process = Popen(cmd_mpirun, env=os.environ, shell=False, stdin=PIPE, stdout=PIPE, stderr=PIPE) - output, error = process.communicate() - print(output) - - files = get_files(output_folder) - - #: check the generated reconstruction (hdf5 file) - h5_files = list(filter(lambda x: ".h5" in x, files)) - assert len(h5_files) == 1 - - check_tif(files, 2160, (2560, 2560)) # ######################################################################## + @pytest.mark.full_data_parallel def test_parallel_pipe_FBP3d_tomobar_denoising_i13_177906_preview( get_files: Callable, @@ -348,9 +311,11 @@ def test_parallel_pipe_FBP3d_tomobar_denoising_i13_177906_preview( cmd_mpirun.insert(10, FBP3d_tomobar_denoising) cmd_mpirun.insert(11, output_folder) - process = Popen(cmd_mpirun, env=os.environ, shell=False, stdin=PIPE, stdout=PIPE, stderr=PIPE) - output, error = process.communicate() - print (output) + process = Popen( + cmd_mpirun, env=os.environ, shell=False, stdin=PIPE, stdout=PIPE, stderr=PIPE + ) + output, error = process.communicate() + print(output) files = get_files(output_folder) @@ -385,6 +350,7 @@ def test_parallel_pipe_FBP3d_tomobar_denoising_i13_177906_preview( res_norm_tv_res = np.linalg.norm(residual_im.flatten()).astype("float32") assert res_norm_tv_res < 1e-1 + # ######################################################################## @@ -411,14 +377,15 @@ def test_parallel_pipe_LPRec3d_tomobar_i12_119647_preview( ], ) - cmd_mpirun.insert(9, i12_119647) cmd_mpirun.insert(10, LPRec3d_tomobar) cmd_mpirun.insert(11, output_folder) - process = Popen(cmd_mpirun, env=os.environ, shell=False, stdin=PIPE, stdout=PIPE, stderr=PIPE) - output, error = process.communicate() - print (output) + process = Popen( + cmd_mpirun, env=os.environ, shell=False, stdin=PIPE, stdout=PIPE, stderr=PIPE + ) + output, error = process.communicate() + print(output) files = get_files(output_folder) @@ -453,8 +420,10 @@ def test_parallel_pipe_LPRec3d_tomobar_i12_119647_preview( res_norm = np.linalg.norm(residual_im.flatten()).astype("float32") assert res_norm < 0.02 + # ######################################################################## + @pytest.mark.full_data_parallel def test_parallel_pipe_360deg_distortion_FBP3d_tomobar_i13_179623_preview( get_files: Callable, @@ -499,9 +468,11 @@ def test_parallel_pipe_360deg_distortion_FBP3d_tomobar_i13_179623_preview( cmd_mpirun.insert(10, deg360_distortion_FBP3d_tomobar) cmd_mpirun.insert(11, output_folder) - process = Popen(cmd_mpirun, env=os.environ, shell=False, stdin=PIPE, stdout=PIPE, stderr=PIPE) - output, error = process.communicate() - print (output) + process = Popen( + cmd_mpirun, env=os.environ, shell=False, stdin=PIPE, stdout=PIPE, stderr=PIPE + ) + output, error = process.communicate() + print(output) files = get_files(output_folder) @@ -536,4 +507,5 @@ def test_parallel_pipe_360deg_distortion_FBP3d_tomobar_i13_179623_preview( res_norm = np.linalg.norm(residual_im.flatten()).astype("float32") assert res_norm < 1e-4 + # ######################################################################## diff --git a/tests/test_pipeline_big.py b/tests/test_pipeline_big.py index 7acaaea38..7477d688f 100644 --- a/tests/test_pipeline_big.py +++ b/tests/test_pipeline_big.py @@ -284,6 +284,7 @@ def test_pipe_FBP2d_astra_i12_119647_preview( # ######################################################################## + @pytest.mark.full_data def test_pipe_FBP3d_tomobar_denoising_i13_177906_preview( get_files: Callable, @@ -458,6 +459,7 @@ def test_pipe_360deg_paganin_FBP3d_tomobar_i13_179623_preview( res_norm = np.linalg.norm(residual_im.flatten()).astype("float32") assert res_norm < 1e-4 + # ######################################################################## @pytest.mark.full_data def test_pipe_sweep_FBP3d_tomobar_i13_177906( From 05d68e2971cdd648e8d0b85b95eebd15f34d321e Mon Sep 17 00:00:00 2001 From: dkazanc Date: Fri, 25 Jul 2025 10:11:52 +0100 Subject: [PATCH 19/19] adding stalling test test_parallel_pipe_FBP3d_tomobar_k11_38729_in_memory --- tests/test_parallel_pipeline_big.py | 48 +++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/tests/test_parallel_pipeline_big.py b/tests/test_parallel_pipeline_big.py index 1e54cfeb3..71b4a9cf8 100644 --- a/tests/test_parallel_pipeline_big.py +++ b/tests/test_parallel_pipeline_big.py @@ -509,3 +509,51 @@ def test_parallel_pipe_360deg_distortion_FBP3d_tomobar_i13_179623_preview( # ######################################################################## + + +@pytest.mark.full_data_parallel +def test_parallel_pipe_FBP3d_tomobar_k11_38729_in_memory( + get_files: Callable, + cmd_mpirun, + diad_k11_38729, + FBP3d_tomobar, + FBP3d_tomobar_k11_38731_npz, + output_folder, +): + + change_value_parameters_method_pipeline( + FBP3d_tomobar, + method=[ + "standard_tomo", + "standard_tomo", + "standard_tomo", + ], + key=[ + "data_path", + "image_key_path", + "rotation_angles", + ], + value=[ + "/entry/imaging/data", + "/entry/instrument/imaging/image_key", + {"data_path": "/entry/imaging_sum/gts_cs_theta"}, + ], + ) + + cmd_mpirun.insert(9, diad_k11_38729) + cmd_mpirun.insert(10, FBP3d_tomobar) + cmd_mpirun.insert(11, output_folder) + + process = Popen( + cmd_mpirun, env=os.environ, shell=False, stdin=PIPE, stdout=PIPE, stderr=PIPE + ) + output, error = process.communicate() + print(output) + + files = get_files(output_folder) + + #: check the generated reconstruction (hdf5 file) + h5_files = list(filter(lambda x: ".h5" in x, files)) + assert len(h5_files) == 1 + + check_tif(files, 2160, (2560, 2560))