Skip to content

Commit d702555

Browse files
AgnieszkaZabaslayoo
AgnieszkaZaba
and
slayoo
authored
CI: new tests-suite strategy (#1706)
Co-authored-by: Sylwester Arabas <[email protected]>
1 parent 3e828db commit d702555

File tree

4 files changed

+85
-35
lines changed

4 files changed

+85
-35
lines changed

‎.github/workflows/tests.yml‎

Lines changed: 62 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -105,7 +105,6 @@ jobs:
105105
with:
106106
python-version: ${{ matrix.python-version }}
107107
setup-name: unit-tests-${{ matrix.platform }}-${{ matrix.python-version }}
108-
109108
- name: Generate coverage report
110109
run: |
111110
python -m pip install pytest-cov
@@ -139,8 +138,6 @@ jobs:
139138
with:
140139
python-version: ${{ matrix.python-version }}
141140
setup-name: unit-tests-${{ matrix.platform }}-${{ matrix.python-version }}
142-
143-
144141
- if: startsWith(matrix.platform, 'ubuntu-')
145142
run: echo NUMBA_THREADING_LAYER=omp >> $GITHUB_ENV
146143
- name: Run pytest
@@ -158,20 +155,41 @@ jobs:
158155
matrix:
159156
platform: [ ubuntu-24.04, macos-14, windows-latest ]
160157
python-version: [ "3.12" ]
161-
test-suite: [ "smoke_tests/no_env", "smoke_tests/box", "smoke_tests/parcel_a", "smoke_tests/parcel_b", "smoke_tests/parcel_c", "smoke_tests/parcel_d", "smoke_tests/kinematic_1d", "smoke_tests/kinematic_2d", "tutorials_tests" ]
158+
test-suite:
159+
- "tests/smoke_tests/no_env tests/smoke_tests/box tests/tutorials_tests"
160+
- "tests/smoke_tests/parcel_a"
161+
- "tests/smoke_tests/parcel_b"
162+
- "tests/smoke_tests/parcel_c"
163+
- "tests/smoke_tests/parcel_d"
164+
- "tests/smoke_tests/kinematic_1d"
165+
- "tests/smoke_tests/kinematic_2d"
166+
include:
167+
- platform: "macos-14"
168+
python-version: "3.12"
169+
test-suite: "tests/smoke_tests/parcel_b tests/smoke_tests/parcel_c tests/smoke_tests/parcel_d" # combined for macos
170+
- platform: "macos-14"
171+
python-version: "3.12"
172+
test-suite: "tests/smoke_tests/no_env tests/tutorials_tests tests/smoke_tests/kinematic_2d tests/smoke_tests/kinematic_1d" # combined for macos
162173
exclude:
163174
# TODO #1406
164175
- platform: "macos-14"
165-
test-suite: "smoke_tests/box"
176+
test-suite: "tests/smoke_tests/no_env tests/smoke_tests/box tests/tutorials_tests"
177+
- platform: "macos-14"
178+
test-suite: "tests/smoke_tests/parcel_a"
179+
# test-suites already included in combined run
166180
- platform: "macos-14"
167-
test-suite: "smoke_tests/parcel_a"
181+
test-suite: "tests/smoke_tests/kinematic_1d"
168182
- platform: "macos-14"
169-
test-suite: "smoke_tests/parcel_d"
183+
test-suite: "tests/smoke_tests/kinematic_2d"
170184
- platform: "macos-14"
171-
test-suite: "smoke_tests/kinematic_2d"
185+
test-suite: "tests/smoke_tests/parcel_b"
186+
- platform: "macos-14"
187+
test-suite: "tests/smoke_tests/parcel_c"
188+
- platform: "macos-14"
189+
test-suite: "tests/smoke_tests/parcel_d"
172190
fail-fast: true
173191
runs-on: ${{ matrix.platform }}
174-
timeout-minutes: ${{ startsWith(matrix.platform, 'ubuntu-') && 25 || 40 }}
192+
timeout-minutes: ${{ startsWith(matrix.platform, 'windows-') && 35 || 20 }}
175193
steps:
176194
- uses: actions/[email protected]
177195
with:
@@ -191,19 +209,41 @@ jobs:
191209
run: |
192210
shopt -s extglob
193211
python -m pip install pytest-timeout
194-
python -m pytest ${{ env.pytest_options }} --timeout=900 --timeout_method=thread tests/${{ matrix.test-suite }}
195-
212+
python -m pytest ${{ env.pytest_options }} --timeout=900 --timeout_method=thread ${{ matrix.test-suite }}
196213
197214
examples:
198215
needs: [pylint, env-nonunit-tests-setup]
199216
strategy:
200217
matrix:
201218
platform: [ubuntu-24.04, macos-14, windows-latest]
202219
python-version: ["3.12"]
203-
test-suite: [ "isotopes_chemistry_extraterrestrial", "coagulation_freezing", "condensation_a", "condensation_b", "condensation_c", "multi-process_a", "multi-process_b", "multi-process_c_breakup", "multi-process_d", "multi-process_e"]
220+
test-suite:
221+
- "isotopes_chemistry_extraterrestrial"
222+
- "coagulation_freezing"
223+
- "condensation_a"
224+
- "condensation_b"
225+
- "condensation_c"
226+
- "multi-process_a"
227+
- "multi-process_b"
228+
- "multi-process_c_breakup"
229+
- "multi-process_d"
230+
- "multi-process_e"
231+
include:
232+
- platform: "macos-14"
233+
python-version: "3.12"
234+
test-suite: "multi-process_a,multi-process_b,multi-process_d,multi-process_e"
235+
exclude:
236+
- platform: "macos-14"
237+
test-suite: "multi-process_a"
238+
- platform: "macos-14"
239+
test-suite: "multi-process_b"
240+
- platform: "macos-14"
241+
test-suite: "multi-process_d"
242+
- platform: "macos-14"
243+
test-suite: "multi-process_e"
204244
fail-fast: true
205245
runs-on: ${{ matrix.platform }}
206-
timeout-minutes: ${{ startsWith(matrix.platform, 'windows-') && 28 || 17 }}
246+
timeout-minutes: ${{ startsWith(matrix.platform, 'windows-') && 28 || 30 }}
207247
steps:
208248
- name: Debug contexts
209249
run: |
@@ -224,40 +264,36 @@ jobs:
224264
- if: startsWith(matrix.platform, 'ubuntu-')
225265
run: echo NUMBA_THREADING_LAYER=omp >> $GITHUB_ENV
226266

227-
- if: startsWith(matrix.platform, 'ubuntu-') && (matrix.test-suite == 'multi-process_a' || matrix.test-suite == 'multi-process_b')
267+
- if: startsWith(matrix.platform, 'ubuntu-') && (contains(matrix.test-suite, 'multi-process_a') || contains(matrix.test-suite, 'multi-process_b'))
228268
run: |
229269
sudo apt-get update
230270
sudo apt-get install libegl1 libopengl0 ghostscript ffmpeg
231271
wget -nv "https://paraview.org/paraview-downloads/download.php?submit=Download&version=v5.13&type=binary&os=Linux&downloadFile=ParaView-5.13.1-egl-MPI-Linux-Python3.10-x86_64.tar.gz" -O paraview.tar.gz
232272
tar xzf paraview.tar.gz
233273
echo `pwd`/`dirname ParaView*/bin/pvpython` >> $GITHUB_PATH
234-
235-
- if: startsWith(matrix.platform, 'macos-') && (matrix.test-suite == 'multi-process_a' || matrix.test-suite == 'multi-process_b')
274+
- if: startsWith(matrix.platform, 'macos-') && (contains(matrix.test-suite, 'multi-process_a') || contains(matrix.test-suite, 'multi-process_b'))
236275
run: |
237276
brew update
238277
brew install --cask paraview
239278
echo `dirname /Applications/ParaView-*.app/Contents/bin/pvpython | head -1` >> $GITHUB_PATH
240279
brew install ghostscript ffmpeg
241-
242-
- if: startsWith(matrix.platform, 'windows-') && (matrix.test-suite == 'multi-process_a' || matrix.test-suite == 'multi-process_b')
280+
- if: startsWith(matrix.platform, 'windows-') && (contains(matrix.test-suite, 'multi-process_a') || contains(matrix.test-suite, 'multi-process_b'))
243281
run: |
244282
curl "https://www.paraview.org/paraview-downloads/download.php?submit=Download&version=v5.13&type=binary&os=Windows&downloadFile=ParaView-5.13.1-Windows-Python3.10-msvc2017-AMD64.msi" -o paraview.msi
245-
msiexec //i paraview.msi //quiet //qn //norestart
283+
msiexec //i paraview.msi //quiet //qn //norestart
246284
echo "C:\\Program Files\\ParaView 5.13.1\\bin\\" >> $GITHUB_PATH
247-
248-
- if: matrix.test-suite == 'multi-process_a' || matrix.test-suite == 'multi-process_b'
285+
- if: contains(matrix.test-suite, 'multi-process_a') || contains(matrix.test-suite, 'multi-process_b')
249286
run: |
250287
pvpython --version
251288
pvpython -c "from paraview import simple"
252289
253290
- run: |
254291
python -m pytest ${{ env.pytest_options }} tests/examples_tests/test_tests_completeness.py
255-
256292
- run: |
257293
python -m pip install pytest-timeout
258294
python -m pytest ${{ env.pytest_options }} --basetemp=/tmp/pytest --timeout_method=thread --timeout=${{ startsWith(matrix.platform, 'windows-') && 1000 || 900 }} tests/examples_tests/test_run* --suite ${{ matrix.test-suite }}
259295
260-
- if: ( ! startsWith(matrix.platform, 'windows-') ) && matrix.test-suite == env.anim_test-suite && matrix.python-version == env.anim_python-version
296+
- if: ( ! startsWith(matrix.platform, 'windows-') ) && contains(matrix.test-suite, env.anim_test-suite) && matrix.python-version == env.anim_python-version
261297
run: |
262298
set -x
263299
mkdir -p $HOME/work/_temp/_github_home/figures
@@ -269,23 +305,21 @@ jobs:
269305
ls $HOME/work/_temp/_github_home/figures
270306
271307
- name: animation movie upload
272-
if: ( ! startsWith(matrix.platform, 'windows-') ) && matrix.test-suite == env.anim_test-suite && matrix.python-version == env.anim_python-version
308+
if: ( ! startsWith(matrix.platform, 'windows-') ) && contains(matrix.test-suite, env.anim_test-suite) && matrix.python-version == env.anim_python-version
273309
uses: actions/upload-artifact@v4
274310
with:
275311
name: animation-movie-${{ matrix.platform }}
276312
if-no-files-found: error
277313
path: ~/work/_temp/_github_home/figures/*.gif
278-
279314
- name: animation frame upload
280-
if: ( ! startsWith(matrix.platform, 'windows-') ) && matrix.test-suite == env.anim_test-suite && matrix.python-version == env.anim_python-version
315+
if: ( ! startsWith(matrix.platform, 'windows-') ) && contains(matrix.test-suite, env.anim_test-suite) && matrix.python-version == env.anim_python-version
281316
uses: actions/upload-artifact@v4
282317
with:
283318
name: animation-frame-${{ matrix.platform }}
284319
if-no-files-found: error
285320
path: ~/work/_temp/_github_home/figures/*.pdf
286-
287321
- name: tip release upload
288-
if: github.ref == 'refs/heads/main' && ( startsWith(matrix.platform, 'ubuntu-') ) && matrix.test-suite == env.anim_test-suite && matrix.python-version == env.anim_python-version
322+
if: github.ref == 'refs/heads/main' && ( startsWith(matrix.platform, 'ubuntu-') ) && contains(matrix.test-suite, env.anim_test-suite) && matrix.python-version == env.anim_python-version
289323
uses: eine/tip@master
290324
with:
291325
token: ${{ secrets.GITHUB_TOKEN }}

‎tests/examples_tests/conftest.py‎

Lines changed: 20 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -181,11 +181,16 @@ def get_selected_test_paths(suite_name, paths):
181181

182182

183183
def pytest_addoption(parser):
184-
parser.addoption("--suite", action="store")
184+
parser.addoption("--suite", action="append")
185185

186186

187187
def pytest_generate_tests(metafunc):
188-
suite_name = metafunc.config.option.suite
188+
suite_args = metafunc.config.option.suite or []
189+
190+
# Support both --suite name1 --suite name2 and --suite name1,name2
191+
suite_names = []
192+
for arg in suite_args:
193+
suite_names.extend([s.strip() for s in arg.split(",") if s.strip()])
189194

190195
pysdm_examples_abs_path = (
191196
pathlib.Path(__file__)
@@ -199,9 +204,14 @@ def pytest_generate_tests(metafunc):
199204
for path in findfiles(pysdm_examples_abs_path, r".*\.ipynb$")
200205
if ".ipynb_checkpoints" not in str(path)
201206
]
202-
selected_paths = set(get_selected_test_paths(suite_name, notebook_paths)) - set(
203-
SMOKE_TEST_COVERED_PATHS
204-
)
207+
selected_paths = set()
208+
for suite_name in suite_names:
209+
selected_paths.update(
210+
set(get_selected_test_paths(suite_name, notebook_paths))
211+
)
212+
213+
# Remove duplicates and any smoke-tested paths
214+
selected_paths = selected_paths - set(SMOKE_TEST_COVERED_PATHS)
205215
metafunc.parametrize(
206216
"notebook_filename",
207217
selected_paths,
@@ -213,7 +223,11 @@ def pytest_generate_tests(metafunc):
213223
pysdm_examples_abs_path,
214224
r".*\.(py)$",
215225
)
216-
selected_paths = get_selected_test_paths(suite_name, examples_paths)
226+
selected_paths = set()
227+
for suite_name in suite_names:
228+
selected_paths.update(
229+
set(get_selected_test_paths(suite_name, examples_paths))
230+
)
217231
metafunc.parametrize(
218232
"example_filename",
219233
selected_paths,

‎tests/smoke_tests/box/srivastava_1982/test_eq_13.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -63,6 +63,8 @@ def test_pysdm_coalescence_and_breakup_is_close_to_analytic_coalescence_and_brea
6363

6464
if plot:
6565
pyplot.show()
66+
else:
67+
pyplot.close("all") # TODO #1717
6668

6769
# assert
6870
assert_prod = SimProducts.Computed.mean_drop_volume_total_volume_ratio.name

‎tests/smoke_tests/kinematic_2d/arabas_et_al_2015/test_initialisation.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -94,7 +94,7 @@ def test_initialisation(backend_class, plot=False):
9494
mass_conc_wet = np.sum(histogram_wet[:, level]) / rhod[level]
9595
mass_conc_STP = settings.spectrum_per_mass_of_dry_air.norm_factor
9696
assert 0.5 * mass_conc_STP < mass_conc_dry < 1.5 * mass_conc_STP
97-
np.testing.assert_approx_equal(mass_conc_dry, mass_conc_wet, significant=5)
97+
np.testing.assert_approx_equal(mass_conc_dry, mass_conc_wet, significant=3)
9898

9999
# Assert - decreasing number density
100100
assert np.sum(histogram_dry[:, 0]) > np.sum(histogram_dry[:, -1])

0 commit comments

Comments
 (0)