Skip to content

Add derecho-gpu machine config #1

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 7 commits into
base: current-0.082
Choose a base branch
from
Open

Add derecho-gpu machine config #1

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
d0ce24a
Add derecho-gpu machine config
ma595 Jul 23, 2025
83bb1a3
Remove comment
ma595 Jul 23, 2025
74ec5dc
Remove config_compilers.xml and add derecho-gpu.cmake
ma595 Jul 29, 2025
b649827
Set some sensible memory defaults for derecho-gpu
ma595 Jul 29, 2025
5d54f6f
Fix formatting and add sensible defaults
ma595 Aug 19, 2025
41b32c3
Change back to 64 tasks
ma595 Aug 19, 2025
d6d079a
Hardcode ngpus to 4
ma595 Aug 19, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 11 additions & 0 deletions machines/cmake_macros/derecho-gpu.cmake
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
if (COMP_NAME STREQUAL gptl)
string(APPEND CPPDEFS " -DHAVE_NANOTIME -DBIT64 -DHAVE_VPRINTF -DHAVE_BACKTRACE -DHAVE_SLASHPROC -DHAVE_COMM_F2C -DHAVE_TIMES -DHAVE_GETTIMEOFDAY")
endif()
set(NETCDF_PATH "$ENV{NETCDF}")
set(PIO_FILESYSTEM_HINTS "lustre")
set(PNETCDF_PATH "$ENV{PNETCDF}")
# If we want to use cray-libsci instead of mkl uncomment this line as well as the module in config_machines.xml
string(REPLACE "-mkl=cluster" "" SLIBS "${SLIBS}")
#string(REPLACE "-mkl=cluster" "-qmkl=cluster" SLIBS "${SLIBS}")
#string(APPEND CPPDEFS " -DNO_SHR_VMATH ")
string(APPEND CPPDEFS " -DHAVE_GETTID")
15 changes: 15 additions & 0 deletions machines/config_batch.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -374,6 +374,21 @@
</queues>
</batch_system>

<batch_system MACH="derecho-gpu" type="pbs" >
<batch_submit>qsub</batch_submit>
<submit_args>
<argument> -l job_priority="$JOB_PRIORITY" </argument>
</submit_args>
<directives>
<directive default="/bin/bash" > -S {{ shell }} </directive>
<directive> -l select={{ num_nodes }}:ncpus={{ max_tasks_per_node }}:mpiprocs={{ tasks_per_node }}:ompthreads={{ thread_count }}:mem=480GB:ngpus=4</directive>
</directives>
<queues>
<queue walltimemax="12:00:00" nodemin="1" nodemax="82" default="true">main</queue>
<queue walltimemax="1:00:00" nodemin="1" nodemax="2">develop</queue>
</queues>
</batch_system>

<batch_system MACH="eastwind" type="slurm" >
<batch_submit>sbatch</batch_submit>
<submit_args>
Expand Down
102 changes: 102 additions & 0 deletions machines/config_machines.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1255,6 +1255,108 @@ This allows using a different mpirun command to launch unit tests
<env name="OMP_STACKSIZE">64M</env>
</environment_variables>
</machine>

<machine MACH="derecho-gpu">
<DESC>NCAR AMD EPYC </DESC>
<NODENAME_REGEX>de.*.hpc.ucar.edu</NODENAME_REGEX>
<OS>CNL</OS>
<COMPILERS>intel</COMPILERS>
<!-- <COMPILERS>intel,gnu,cray,nvhpc,intel-oneapi,intel-classic</COMPILERS> -->
<MPILIBS>mpich</MPILIBS>
<CIME_OUTPUT_ROOT>$ENV{SCRATCH}</CIME_OUTPUT_ROOT>
<DIN_LOC_ROOT>$ENV{CESMDATAROOT}/inputdata</DIN_LOC_ROOT>
<DIN_LOC_ROOT_CLMFORC>$ENV{CESMDATAROOT}/inputdata/atm/datm7</DIN_LOC_ROOT_CLMFORC>
<DOUT_S_ROOT>$CIME_OUTPUT_ROOT/archive/$CASE</DOUT_S_ROOT>
<BASELINE_ROOT>$ENV{CESMDATAROOT}/cesm_baselines</BASELINE_ROOT>
<CCSM_CPRNC>$ENV{CESMDATAROOT}/cprnc/cprnc</CCSM_CPRNC>
<GMAKE_J>16</GMAKE_J>
<BATCH_SYSTEM>pbs</BATCH_SYSTEM>
<SUPPORTED_BY>cseg</SUPPORTED_BY>
<MAX_TASKS_PER_NODE>64</MAX_TASKS_PER_NODE>
<MAX_MPITASKS_PER_NODE>64</MAX_MPITASKS_PER_NODE>
<PROJECT_REQUIRED>TRUE</PROJECT_REQUIRED>
<mpirun mpilib="default">
<executable>mpiexec</executable>
<arguments>
<arg name="label"> --label</arg>
<arg name="buffer"> --line-buffer</arg>
<arg name="num_tasks" > -n {{ total_tasks }}</arg>
</arguments>
</mpirun>
<module_system type="module" allow_error="true">
<init_path lang="perl">$LMOD_ROOT/lmod/init/perl</init_path>
<init_path lang="python">$LMOD_ROOT/lmod/init/env_modules_python.py</init_path>
<init_path lang="sh">$LMOD_ROOT/lmod/init/sh</init_path>
<init_path lang="csh">$LMOD_ROOT/lmod/init/csh</init_path>
<cmd_path lang="perl">$LMOD_ROOT/lmod/libexec/lmod perl</cmd_path>
<cmd_path lang="python">$LMOD_ROOT/lmod/libexec/lmod python</cmd_path>
<cmd_path lang="sh">module</cmd_path>
<cmd_path lang="csh">module</cmd_path>
<modules>
<command name="load">cesmdev/1.0</command>
<command name="load">ncarenv/23.09</command>
<command name="purge"/>
<command name="load">craype</command>
<command name="load">conda/latest</command>
</modules>
<modules compiler="intel">
<command name="load">intel/2023.2.1</command>
<command name="load">mkl</command>
<command name="load">spherepack/3.2</command>
</modules>
<modules compiler="intel-oneapi">
<command name="load">intel-oneapi/2023.2.1</command>
<command name="load">mkl</command>
</modules>
<modules compiler="intel-classic">
<command name="load">intel-classic/2023.2.1</command>
<command name="load">mkl</command>
</modules>
<modules compiler="cray">
<command name="load">cce/15.0.1</command>
<command name="load">cray-libsci/23.02.1.1</command>
</modules>
<modules compiler="gnu">
<command name="load">gcc/12.2.0</command>
<command name="load">cray-libsci/23.02.1.1</command>
</modules>
<modules compiler="nvhpc">
<command name="load">nvhpc/23.7</command>
</modules>
<modules>
<command name="load">ncarcompilers/1.0.0</command>
<command name="load">cmake</command>
</modules>
<modules mpilib="mpich">
<command name="load">cray-mpich/8.1.27</command>
</modules>
<modules mpilib="mpi-serial">
<command name="load">mpi-serial/2.3.0</command>
</modules>

<modules mpilib="mpi-serial">
<command name="load">netcdf/4.9.2</command>
</modules>

<modules mpilib="!mpi-serial">
<command name="load">netcdf-mpi/4.9.2</command>
<command name="load">parallel-netcdf/1.12.3</command>
</modules>

<modules>
<command name="load">esmf/8.6.0</command>
</modules>

</module_system>

<environment_variables>
<env name="OMP_STACKSIZE">64M</env>
<env name="FI_CXI_RX_MATCH_MODE">hybrid</env>
<env name="FI_MR_CACHE_MONITOR">memhooks</env>
<env name="SPHEREPACK_LIBDIR">$ENV{NCAR_ROOT_SPHEREPACK}/lib</env>
</environment_variables>

</machine>

<machine MACH="derecho">
<DESC>NCAR AMD EPYC system</DESC>
Expand Down