From d0ce24ac9783c2675d0370198cf12558dd6b7c65 Mon Sep 17 00:00:00 2001
From: Matt Archer <ma595@cam.ac.uk>
Date: Wed, 23 Jul 2025 04:20:55 -0600
Subject: [PATCH 1/7] Add derecho-gpu machine config

---
 machines/config_batch.xml     |  15 ++
 machines/config_compilers.xml | 423 ++++++++++++++++++++++++++++++++++
 machines/config_machines.xml  | 102 ++++++++
 3 files changed, 540 insertions(+)
 create mode 100644 machines/config_compilers.xml

diff --git a/machines/config_batch.xml b/machines/config_batch.xml
index cb56c334..f35d2838 100644
--- a/machines/config_batch.xml
+++ b/machines/config_batch.xml
@@ -374,6 +374,21 @@
     </queues>
   </batch_system>
 
+  <batch_system MACH="derecho-gpu" type="pbs" >
+    <batch_submit>qsub</batch_submit>
+    <submit_args>
+      <argument> -l job_priority="$JOB_PRIORITY" </argument>
+    </submit_args>
+    <directives>
+      <directive default="/bin/bash" > -S {{ shell }}  </directive>
+      <directive> -l select={{ num_nodes }}:ncpus={{ max_tasks_per_node }}:mpiprocs={{ tasks_per_node }}:ompthreads={{ thread_count }}:ngpus=4</directive>
+    </directives>
+    <queues>
+      <queue walltimemax="12:00:00" nodemin="1" nodemax="82" default="true">main</queue>
+      <queue walltimemax="1:00:00" nodemin="1" nodemax="2">develop</queue>
+    </queues>
+  </batch_system>
+
   <batch_system MACH="eastwind" type="slurm" >
     <batch_submit>sbatch</batch_submit>
     <submit_args>
diff --git a/machines/config_compilers.xml b/machines/config_compilers.xml
new file mode 100644
index 00000000..6f302100
--- /dev/null
+++ b/machines/config_compilers.xml
@@ -0,0 +1,423 @@
+<compiler>
+  <CPPDEFS>
+    <append MODEL="pop"> -D_USE_FLOW_CONTROL </append>
+  </CPPDEFS>
+  <SUPPORTS_CXX>FALSE</SUPPORTS_CXX>
+</compiler>
+
+<compiler COMPILER="cray">
+  <CFLAGS>
+    <append compile_threaded="false"> -h noomp </append>
+    <append DEBUG="TRUE"> -g -O0 </append>
+    <append DEBUG="FALSE"> -O2 </append>
+  </CFLAGS>
+  <CPPDEFS>
+    <!--http://docs.cray.com/cgi-bin/craydoc.cgi?mode=View;id=S-3901-83;idx=books_search;this_sort=;q=;type=books;title=Cray%20Fortran%20Reference%20Manual -->
+    <append> -DFORTRANUNDERSCORE -DNO_R16 -DCPRCRAY</append>
+    <append MODEL="pop"> -DDIR=NOOP </append>
+    <append MODEL="moby"> -DDIR=NOOP </append>
+  </CPPDEFS>
+  <FC_AUTO_R8>
+    <base> -s real64 </base>
+  </FC_AUTO_R8>
+  <FFLAGS>
+    <base> -f free -N 255  -h byteswapio -x dir </base>
+    <append compile_threaded="false"> -h noomp </append>
+    <append DEBUG="TRUE"> -g -O0 -K trap=fp  -m1 </append>
+    <append DEBUG="FALSE"> -O2,ipa2 -em </append>
+  </FFLAGS>
+  <FFLAGS_NOOPT>
+    <base> -O1,fp2,ipa0,scalar0,vector0 </base>
+  </FFLAGS_NOOPT>
+  <HAS_F2008_CONTIGUOUS>TRUE</HAS_F2008_CONTIGUOUS>
+  <LDFLAGS>
+    <base> -Wl,--allow-multiple-definition -h byteswapio </base>
+  </LDFLAGS>
+</compiler>
+
+<compiler COMPILER="gnu">
+  <CFLAGS>
+    <base> -std=gnu99 </base>
+    <append compile_threaded="true"> -fopenmp </append>
+    <append DEBUG="TRUE"> -g -Wall -Og -fbacktrace -ffpe-trap=invalid,zero,overflow -fcheck=bounds </append>
+    <append DEBUG="FALSE"> -O </append>
+  </CFLAGS>
+  <CPPDEFS>
+    <!-- http://gcc.gnu.org/onlinedocs/gfortran/ -->
+    <append> -DFORTRANUNDERSCORE -DNO_R16 -DCPRGNU</append>
+  </CPPDEFS>
+  <CXX_LINKER>FORTRAN</CXX_LINKER>
+  <FC_AUTO_R8>
+    <base> -fdefault-real-8 </base>
+  </FC_AUTO_R8>
+  <FFLAGS>
+    <!-- -ffree-line-length-none and -ffixed-line-length-none need to be in FFLAGS rather than in FIXEDFLAGS/FREEFLAGS
+       so that these are passed to cmake builds (cmake builds don't use FIXEDFLAGS and FREEFLAGS). -->
+    <base>  -fconvert=big-endian -ffree-line-length-none -ffixed-line-length-none </base>
+    <append compile_threaded="true"> -fopenmp </append>
+    <!-- Ideally, we would also have 'invalid' in the ffpe-trap list. But at
+         least with some versions of gfortran (confirmed with 5.4.0, 6.3.0 and
+         7.1.0), gfortran's isnan (which is called in cime via the
+         CPRGNU-specific shr_infnan_isnan) causes a floating point exception
+         when called on a signaling NaN. -->
+    <append DEBUG="TRUE"> -g -Wall -Og -fbacktrace -ffpe-trap=zero,overflow -fcheck=bounds </append>
+    <append DEBUG="FALSE"> -O </append>
+  </FFLAGS>
+  <FFLAGS_NOOPT>
+    <base> -O0 </base>
+  </FFLAGS_NOOPT>
+  <FIXEDFLAGS>
+    <base>  -ffixed-form </base>
+  </FIXEDFLAGS>
+  <FREEFLAGS>
+    <base> -ffree-form </base>
+  </FREEFLAGS>
+  <HAS_F2008_CONTIGUOUS>FALSE</HAS_F2008_CONTIGUOUS>
+  <LDFLAGS>
+    <append compile_threaded="true"> -fopenmp </append>
+  </LDFLAGS>
+  <MPICC> mpicc  </MPICC>
+  <MPICXX> mpicxx </MPICXX>
+  <MPIFC> mpif90 </MPIFC>
+  <SCC> gcc </SCC>
+  <SCXX> g++ </SCXX>
+  <SFC> gfortran </SFC>
+  <SUPPORTS_CXX>TRUE</SUPPORTS_CXX>
+</compiler>
+
+<compiler COMPILER="ibm">
+  <CFLAGS>
+    <base> -g -qfullpath -qmaxmem=-1 </base>
+    <append DEBUG="FALSE"> -O3  </append>
+    <append compile_threaded="true"> -qsmp=omp </append>
+    <append DEBUG="TRUE" compile_threaded="true"> -qsmp=omp:noopt </append>
+  </CFLAGS>
+  <CPPDEFS>
+    <!-- http://publib.boulder.ibm.com/infocenter/comphelp/v7v91/index.jsp
+ Notes:  (see xlf user's guide for the details)
+  -lmass          => IBM-tuned intrinsic lib
+  -qsmp=noauto    => enable SMP directives, but don't add any
+  -qsmp=omp       => enable SMP directives, strict omp
+  -qstrict        => don't turn divides into multiplies, etc
+  -qhot           => higher-order-transformations (eg. loop padding)
+  -qalias=noaryovrlp => assume no array overlap wrt equivalance, etc
+  -qmaxmem=-1     => memory available to compiler during optimization
+  -qipa=level=2   => InterProcedure Analysis (eg. inlining) => slow compiles
+  -p -pg          => enable profiling (use in both FFLAGS and LDFLAGS)
+  -qreport        => for smp/omp only
+  -g              => always leave it on because overhead is minimal
+  -qflttrap=...   => enable default sigtrap (core dump)
+  -C              => runtime array bounds checking (runs slow)
+  -qinitauto=...  => initializes automatic variables
+  -->
+    <append> -DFORTRAN_SAME -DCPRIBM </append>
+  </CPPDEFS>
+  <CPRE>-WF,-D</CPRE>
+  <FC_AUTO_R8>
+    <base> -qrealsize=8 </base>
+  </FC_AUTO_R8>
+  <FFLAGS>
+    <base> -g -qfullpath -qmaxmem=-1 </base>
+    <append DEBUG="FALSE"> -O2 -qstrict -qinline=auto </append>
+    <append compile_threaded="true"> -qsmp=omp </append>
+    <append DEBUG="TRUE"> -qinitauto=7FF7FFFF -qflttrap=ov:zero:inv:en </append>
+    <append DEBUG="TRUE" compile_threaded="true"> -qsmp=omp:noopt </append>
+    <append DEBUG="TRUE" MODEL="pop"> -C </append>
+  </FFLAGS>
+  <FIXEDFLAGS>
+    <base>  -qsuffix=f=f -qfixed=132 </base>
+  </FIXEDFLAGS>
+  <FREEFLAGS>
+    <base> -qsuffix=f=f90:cpp=F90  </base>
+  </FREEFLAGS>
+  <HAS_F2008_CONTIGUOUS>TRUE</HAS_F2008_CONTIGUOUS>
+  <LDFLAGS>
+    <append compile_threaded="true"> -qsmp=omp </append>
+    <append DEBUG="TRUE" compile_threaded="true"> -qsmp=omp:noopt </append>
+  </LDFLAGS>
+</compiler>
+
+<compiler COMPILER="intel">
+  <CFLAGS>
+    <base>  -qno-opt-dynamic-align -fp-model precise -std=gnu99 </base>
+    <append compile_threaded="true"> -qopenmp </append>
+    <append DEBUG="FALSE"> -O2 -debug minimal </append>
+    <append DEBUG="TRUE"> -O0 -g </append>
+  </CFLAGS>
+  <CPPDEFS>
+    <!-- http://software.intel.com/en-us/articles/intel-composer-xe/ -->
+    <append> -DFORTRANUNDERSCORE -DCPRINTEL</append>
+  </CPPDEFS>
+  <CXX_LDFLAGS>
+    <base> -cxxlib </base>
+  </CXX_LDFLAGS>
+  <CXX_LINKER>FORTRAN</CXX_LINKER>
+  <FC_AUTO_R8>
+    <base> -r8 </base>
+  </FC_AUTO_R8>
+  <FFLAGS>
+    <base> -qno-opt-dynamic-align  -convert big_endian -assume byterecl -ftz -traceback -assume realloc_lhs -fp-model source  </base>
+    <append compile_threaded="true"> -qopenmp </append>
+    <append DEBUG="TRUE"> -O0 -g -check uninit -check bounds -check pointers -fpe0 -check noarg_temp_created </append>
+    <append DEBUG="FALSE"> -O2 -debug minimal </append>
+  </FFLAGS>
+  <FFLAGS_NOOPT>
+    <base> -O0 </base>
+    <append compile_threaded="true"> -qopenmp </append>
+  </FFLAGS_NOOPT>
+  <FIXEDFLAGS>
+    <base> -fixed -132 </base>
+  </FIXEDFLAGS>
+  <FREEFLAGS>
+    <base> -free </base>
+  </FREEFLAGS>
+  <LDFLAGS>
+    <append compile_threaded="true"> -qopenmp </append>
+  </LDFLAGS>
+  <MPICC> mpicc  </MPICC>
+  <MPICXX> mpicxx </MPICXX>
+  <MPIFC> mpif90 </MPIFC>
+  <SCC> icc </SCC>
+  <SCXX> icpc </SCXX>
+  <SFC> ifort </SFC>
+  <SLIBS>
+    <append MPILIB="mpich"> -mkl=cluster </append>
+    <append MPILIB="mpich2"> -mkl=cluster </append>
+    <append MPILIB="mvapich"> -mkl=cluster </append>
+    <append MPILIB="mvapich2"> -mkl=cluster </append>
+    <append MPILIB="mpt"> -mkl=cluster </append>
+    <append MPILIB="openmpi"> -mkl=cluster </append>
+    <append MPILIB="impi"> -mkl=cluster </append>
+    <append MPILIB="mpi-serial"> -mkl </append>
+  </SLIBS>
+  <SUPPORTS_CXX>TRUE</SUPPORTS_CXX>
+</compiler>
+
+<compiler COMPILER="nag">
+  <CFLAGS>
+    <base> -std=gnu99 </base>
+    <append DEBUG="TRUE"> -g </append>
+  </CFLAGS>
+  <CPPDEFS>
+    <append> -DFORTRANUNDERSCORE -DNO_CRAY_POINTERS -DNO_SHR_VMATH -DCPRNAG </append>
+  </CPPDEFS>
+  <FC_AUTO_R8>
+    <base> -r8 </base>
+  </FC_AUTO_R8>
+  <FFLAGS>
+    <!-- The indirect flag below is to deal with MPI functions that violate    -->
+    <!-- the Fortran standard, by adding a large set of arguments from a file. -->
+    <base>-Wp,-macro=no_com -convert=BIG_ENDIAN -indirect $ENV{CIMEROOT}/config/cesm/machines/nag_mpi_argument.txt</base>
+    <!-- DEBUG vs. non-DEBUG runs.                                             -->
+    <append DEBUG="FALSE"> -ieee=full -O2 </append>
+    <!-- The "-gline" option is nice, but it doesn't work with OpenMP.         -->
+    <!-- Runtime checks with OpenMP (in fact, all OpenMP cases) are WIP.       -->
+    <append DEBUG="TRUE"> -C=all -g -time -f2003 -ieee=stop </append>
+    <append DEBUG="TRUE" compile_threaded="false"> -gline </append>
+    <!-- The SLAP library (which is part of the CISM build) has many instances of
+       arguments being passed to different types. So disable argument type
+       checking when building CISM. This can be removed once we remove SLAP from
+       CISM. -->
+    <append MODEL="cism"> -mismatch_all </append>
+  </FFLAGS>
+  <FFLAGS_NOOPT>
+    <base>-Wp,-macro=no_com -convert=BIG_ENDIAN -indirect $ENV{CIMEROOT}/config/cesm/machines/nag_mpi_argument.txt</base>
+    <append DEBUG="FALSE"> -ieee=full     </append>
+    <!-- Hack! If DEBUG="TRUE", put runtime checks in FFLAGS, but not into     -->
+    <!-- FFLAGS_NOOPT, allowing strict checks to be removed from files by      -->
+    <!-- having them use FFLAGS_NOOPT in Depends.nag                           -->
+    <append DEBUG="TRUE"> -g -time -f2003 -ieee=stop </append>
+    <append DEBUG="TRUE" compile_threaded="false"> -gline        </append>
+  </FFLAGS_NOOPT>
+  <FIXEDFLAGS>
+    <base> -fixed </base>
+  </FIXEDFLAGS>
+  <FREEFLAGS>
+    <base> -free </base>
+  </FREEFLAGS>
+  <HAS_F2008_CONTIGUOUS>FALSE</HAS_F2008_CONTIGUOUS>
+  <MPICC> mpicc </MPICC>
+  <MPIFC> mpif90 </MPIFC>
+  <SCC> gcc </SCC>
+  <SFC> nagfor </SFC>
+</compiler>
+
+<compiler COMPILER="pgi">
+  <CFLAGS>
+    <base> -gopt  -time </base>
+    <append compile_threaded="true"> -mp </append>
+  </CFLAGS>
+  <CPPDEFS>
+    <!-- http://www.pgroup.com/resources/docs.htm                                              -->
+    <!-- Notes:  (see pgi man page & user's guide for the details) -->
+    <!--  -Mextend        => Allow 132-column source lines -->
+    <!--  -Mfixed         => Assume fixed-format source -->
+    <!--  -Mfree          => Assume free-format source -->
+    <!--  -byteswapio     => Swap byte-order for unformatted i/o (big/little-endian) -->
+    <!--  -target=linux   => Specifies the target architecture to Compute Node Linux (CNL only) -->
+    <!--  -fast           => Chooses generally optimal flags for the target platform -->
+    <!--  -Mnovect        => Disables automatic vector pipelining -->
+    <!--  -Mvect=nosse    => Don't generate SSE, SSE2, 3Dnow, and prefetch instructions in loops    -->
+    <!--  -Mflushz        => Set SSE to flush-to-zero mode (underflow) loops where possible  -->
+    <!--  -Kieee          => Perform fp ops in strict conformance with the IEEE 754 standard.  -->
+    <!--                     Some optimizations disabled, slightly slower, more accurate math.  -->
+    <!--  -mp=nonuma      => Don't use thread/processors affinity (for NUMA architectures)  -->
+    <!-- -->
+    <!--  -g              => Generate symbolic debug information. Turns off optimization.   -->
+    <!--  -gopt           => Generate information for debugger without disabling optimizations  -->
+    <!--  -Mbounds        => Add array bounds checking  -->
+    <!--  -Ktrap=fp       => Determine IEEE Trap conditions fp => inv,divz,ovf   -->
+    <!--                     * inv: invalid operands         -->
+    <!--                     * divz divide by zero           -->
+    <!--                     * ovf: floating point overflow   -->
+    <!--  -Mlist          => Create a listing file             -->
+    <!--  -F              => leaves file.f for each preprocessed file.F file  -->
+    <!--  -time           => Print execution time for each compiler step  -->
+    <append> -DFORTRANUNDERSCORE -DNO_SHR_VMATH -DNO_R16  -DCPRPGI </append>
+  </CPPDEFS>
+  <CXX_LINKER>CXX</CXX_LINKER>
+  <FC_AUTO_R8>
+    <base> -r8 </base>
+  </FC_AUTO_R8>
+  <FFLAGS>
+    <base>  -i4 -gopt  -time -Mextend -byteswapio -Mflushz -Kieee  </base>
+    <append compile_threaded="true"> -mp </append>
+    <append DEBUG="TRUE"> -O0 -g -Ktrap=fp -Mbounds -Kieee </append>
+    <append MODEL="datm"> -Mnovect </append>
+    <append MODEL="dlnd"> -Mnovect </append>
+    <append MODEL="drof"> -Mnovect </append>
+    <append MODEL="dwav"> -Mnovect </append>
+    <append MODEL="dice"> -Mnovect </append>
+    <append MODEL="docn"> -Mnovect </append>
+  </FFLAGS>
+  <FFLAGS_NOOPT>
+    <base> -O0 -g -Ktrap=fp -Mbounds -Kieee  </base>
+    <append compile_threaded="true"> -mp </append>
+  </FFLAGS_NOOPT>
+  <FIXEDFLAGS>
+    <base> -Mfixed </base>
+  </FIXEDFLAGS>
+  <FREEFLAGS>
+    <base> -Mfree </base>
+  </FREEFLAGS>
+  <!-- Note that SUPPORTS_CXX is false for pgi in general, because we
+       need some machine-specific libraries - see hopper pgi for an
+       example -->
+  <!-- Technically, PGI does recognize this keyword during parsing,
+       but support is either buggy or incomplete, notably in that
+       the "contiguous" attribute is incompatible with "intent".-->
+  <HAS_F2008_CONTIGUOUS>FALSE</HAS_F2008_CONTIGUOUS>
+  <LDFLAGS>
+    <base> -time -Wl,--allow-multiple-definition </base>
+    <append compile_threaded="true"> -mp </append>
+  </LDFLAGS>
+  <MPICC> mpicc </MPICC>
+  <MPICXX> mpicxx </MPICXX>
+  <MPIFC> mpif90 </MPIFC>
+  <SCC> pgcc </SCC>
+  <SCXX> pgc++ </SCXX>
+  <SFC> pgf95 </SFC>
+</compiler>
+
+<compiler OS="AIX" COMPILER="ibm">
+  <CFLAGS>
+    <append> -qarch=auto -qtune=auto -qcache=auto </append>
+  </CFLAGS>
+  <CONFIG_SHELL> /usr/bin/bash </CONFIG_SHELL>
+  <FFLAGS>
+    <append> -qarch=auto -qtune=auto -qcache=auto -qsclk=micro </append>
+    <append MODEL="cam"> -qspill=6000 </append>
+  </FFLAGS>
+  <LDFLAGS>
+    <append DEBUG="TRUE"> -qsigtrap=xl__trcedump </append>
+    <append> -bdatapsize:64K -bstackpsize:64K -btextpsize:32K </append>
+  </LDFLAGS>
+  <MPICC> mpcc_r </MPICC>
+  <MPIFC> mpxlf2003_r </MPIFC>
+  <SCC> cc_r </SCC>
+  <SFC> xlf2003_r </SFC>
+  <SLIBS>
+    <append> -lmassv -lessl </append>
+    <append DEBUG="FALSE"> -lmass </append>
+  </SLIBS>
+</compiler>
+
+<compiler OS="BGQ" COMPILER="ibm">
+  <CONFIG_ARGS>
+    <base> --build=powerpc-bgp-linux --host=powerpc64-suse-linux </base>
+  </CONFIG_ARGS>
+  <CPPDEFS>
+    <append> -DLINUX  </append>
+  </CPPDEFS>
+  <FFLAGS>
+    <base> -g -qfullpath -qmaxmem=-1 -qspillsize=2500 -qextname=flush </base>
+    <append DEBUG="FALSE"> -O3 -qstrict -qinline=auto </append>
+    <append DEBUG="FALSE" compile_threaded="true"> -qsmp=omp </append>
+    <append DEBUG="TRUE" compile_threaded="true"> -qsmp=omp:noopt </append>
+  </FFLAGS>
+  <LDFLAGS>
+    <base>  -Wl,--relax -Wl,--allow-multiple-definition </base>
+  </LDFLAGS>
+</compiler>
+
+<compiler OS="CNL">
+  <CMAKE_OPTS>
+    <base> -DCMAKE_SYSTEM_NAME=Catamount</base>
+  </CMAKE_OPTS>
+  <CPPDEFS>
+    <append> -DLINUX </append>
+    <append MODEL="gptl"> -DHAVE_NANOTIME -DBIT64 -DHAVE_VPRINTF -DHAVE_BACKTRACE -DHAVE_SLASHPROC -DHAVE_COMM_F2C -DHAVE_TIMES -DHAVE_GETTIMEOFDAY  </append>
+  </CPPDEFS>
+  <MPICC> cc </MPICC>
+  <MPICXX> CC </MPICXX>
+  <MPIFC> ftn </MPIFC>
+  <NETCDF_PATH>$ENV{NETCDF_DIR}</NETCDF_PATH>
+  <PIO_FILESYSTEM_HINTS>lustre</PIO_FILESYSTEM_HINTS>
+  <PNETCDF_PATH>$ENV{PARALLEL_NETCDF_DIR}</PNETCDF_PATH>
+  <SCC> cc </SCC>
+  <SCXX> CC </SCXX>
+  <SFC> ftn </SFC>
+</compiler>
+
+<?xml version="1.0"?>
+
+<config_compilers>
+  <compiler MACH="derecho-gpu">
+    <NETCDF_PATH>/glade/u/apps/derecho/23.06/spack/opt/spack/netcdf/4.9.2/cray-mpich/8.1.25/oneapi/2023.0.0/wzol</NETCDF_PATH>
+    <PNETCDF_PATH>/glade/u/apps/derecho/23.06/spack/opt/spack/parallel-netcdf/1.12.3/cray-mpich/8.1.25/oneapi/2023.0.0/blyr</PNETCDF_PATH>
+    <SLIBS>
+      <append> -lnetcdff -lnetcdf </append>
+    </SLIBS>
+  </compiler>
+
+  <compiler MACH="derecho-gpu" COMPILER="intel">
+    <CFLAGS>
+      <base>  -qno-opt-dynamic-align -fp-model precise -std=gnu99 </base>
+      <append MODEL="mpi-serial"> -std=gnu89 </append>
+      <append> -march=core-avx2 -no-fma</append>
+    </CFLAGS>
+    <FFLAGS>
+      <append> -march=core-avx2 -no-fma</append>
+    </FFLAGS>
+  </compiler>
+</config_compilers>
+
+
+<!-- <compiler MACH="derecho-gpu">
+  <NETCDF_PATH>/glade/u/apps/derecho/23.06/spack/opt/spack/netcdf/4.9.2/cray-mpich/8.1.25/oneapi/2023.0.0/wzol</NETCDF_PATH>
+  <PNETCDF_PATH>/glade/u/apps/derecho/23.06/spack/opt/spack/parallel-netcdf/1.12.3/cray-mpich/8.1.25/oneapi/2023.0.0/blyr</PNETCDF_PATH>
+  <SLIBS>
+    <append> -Wl,--copy-dt-needed-entries -lnetcdff -lnetcdf </append>
+  </SLIBS>
+</compiler>
+
+<compiler MACH="derecho-gpu" COMPILER="intel">
+  <CFLAGS>
+    <base>  -qno-opt-dynamic-align -fp-model precise -std=gnu99 </base>
+    <append MODEL="mpi-serial"> -std=gnu89 </append>
+    <append> -march=core-avx2 -no-fma</append>
+  </CFLAGS>
+  <FFLAGS>
+    <append> -march=core-avx2 -no-fma</append>
+  </FFLAGS>
+</compiler> -->
diff --git a/machines/config_machines.xml b/machines/config_machines.xml
index fa3a4492..d4904157 100644
--- a/machines/config_machines.xml
+++ b/machines/config_machines.xml
@@ -1255,6 +1255,108 @@ This allows using a different mpirun command to launch unit tests
       <env name="OMP_STACKSIZE">64M</env>
     </environment_variables>
   </machine>
+  
+  <machine MACH="derecho-gpu">
+    <DESC>NCAR AMD EPYC </DESC>
+    <NODENAME_REGEX>de.*.hpc.ucar.edu</NODENAME_REGEX>
+    <OS>CNL</OS>
+    <COMPILERS>intel</COMPILERS>
+<!--    <COMPILERS>intel,gnu,cray,nvhpc,intel-oneapi,intel-classic</COMPILERS> -->
+    <MPILIBS>mpich</MPILIBS>
+    <CIME_OUTPUT_ROOT>$ENV{SCRATCH}</CIME_OUTPUT_ROOT>
+    <DIN_LOC_ROOT>$ENV{CESMDATAROOT}/inputdata</DIN_LOC_ROOT>
+    <DIN_LOC_ROOT_CLMFORC>$ENV{CESMDATAROOT}/inputdata/atm/datm7</DIN_LOC_ROOT_CLMFORC>
+    <DOUT_S_ROOT>$CIME_OUTPUT_ROOT/archive/$CASE</DOUT_S_ROOT>
+    <BASELINE_ROOT>$ENV{CESMDATAROOT}/cesm_baselines</BASELINE_ROOT>
+    <CCSM_CPRNC>$ENV{CESMDATAROOT}/cprnc/cprnc</CCSM_CPRNC>
+    <GMAKE_J>16</GMAKE_J>
+    <BATCH_SYSTEM>pbs</BATCH_SYSTEM>
+    <SUPPORTED_BY>cseg</SUPPORTED_BY>
+    <MAX_TASKS_PER_NODE>64</MAX_TASKS_PER_NODE>
+    <MAX_MPITASKS_PER_NODE>64</MAX_MPITASKS_PER_NODE>
+    <PROJECT_REQUIRED>TRUE</PROJECT_REQUIRED>
+    <mpirun mpilib="default">
+      <executable>mpiexec</executable>
+      <arguments>
+	<arg name="label"> --label</arg>
+	<arg name="buffer"> --line-buffer</arg>
+	<arg name="num_tasks" > -n {{ total_tasks }}</arg>
+      </arguments>
+    </mpirun>
+    <module_system type="module" allow_error="true">
+      <init_path lang="perl">$LMOD_ROOT/lmod/init/perl</init_path>
+      <init_path lang="python">$LMOD_ROOT/lmod/init/env_modules_python.py</init_path>
+      <init_path lang="sh">$LMOD_ROOT/lmod/init/sh</init_path>
+      <init_path lang="csh">$LMOD_ROOT/lmod/init/csh</init_path>
+      <cmd_path lang="perl">$LMOD_ROOT/lmod/libexec/lmod perl</cmd_path>
+      <cmd_path lang="python">$LMOD_ROOT/lmod/libexec/lmod python</cmd_path>
+      <cmd_path lang="sh">module</cmd_path>
+      <cmd_path lang="csh">module</cmd_path>
+      <modules>
+	<command name="load">cesmdev/1.0</command>
+	<command name="load">ncarenv/23.09</command>
+	<command name="purge"/>
+	<command name="load">craype</command>
+        <command name="load">conda/latest</command>
+      </modules>
+      <modules compiler="intel">
+	<command name="load">intel/2023.2.1</command>
+	<command name="load">mkl</command>
+	<command name="load">spherepack/3.2</command>
+      </modules>
+      <modules compiler="intel-oneapi">
+	<command name="load">intel-oneapi/2023.2.1</command>
+	<command name="load">mkl</command>
+      </modules>
+      <modules compiler="intel-classic">
+	<command name="load">intel-classic/2023.2.1</command>
+	<command name="load">mkl</command>
+      </modules>
+      <modules compiler="cray">
+	<command name="load">cce/15.0.1</command>
+	<command name="load">cray-libsci/23.02.1.1</command>
+      </modules>
+      <modules compiler="gnu">
+	<command name="load">gcc/12.2.0</command>
+	<command name="load">cray-libsci/23.02.1.1</command>
+      </modules>
+      <modules compiler="nvhpc">
+	<command name="load">nvhpc/23.7</command>
+      </modules>
+      <modules>
+	<command name="load">ncarcompilers/1.0.0</command>
+	<command name="load">cmake</command>
+      </modules>
+      <modules mpilib="mpich">
+	<command name="load">cray-mpich/8.1.27</command>
+      </modules>
+      <modules mpilib="mpi-serial">
+	<command name="load">mpi-serial/2.3.0</command>
+      </modules>
+
+      <modules mpilib="mpi-serial">
+	<command name="load">netcdf/4.9.2</command>
+      </modules>
+
+      <modules mpilib="!mpi-serial">
+	<command name="load">netcdf-mpi/4.9.2</command>
+	<command name="load">parallel-netcdf/1.12.3</command>
+      </modules>
+
+      <modules>
+        <command name="load">esmf/8.6.0</command>
+      </modules>
+
+    </module_system>
+
+    <environment_variables>
+      <env name="OMP_STACKSIZE">64M</env>
+      <env name="FI_CXI_RX_MATCH_MODE">hybrid</env>
+      <env name="FI_MR_CACHE_MONITOR">memhooks</env>
+      <env name="SPHEREPACK_LIBDIR">$ENV{NCAR_ROOT_SPHEREPACK}/lib</env>
+    </environment_variables>
+
+  </machine>
 
   <machine MACH="derecho">
     <DESC>NCAR AMD EPYC system</DESC>

From 83bb1a353b02c8da35c3923ddbac404e7162413b Mon Sep 17 00:00:00 2001
From: Matt Archer <ma595@cam.ac.uk>
Date: Wed, 23 Jul 2025 05:03:33 -0600
Subject: [PATCH 2/7] Remove comment

---
 machines/config_compilers.xml | 20 --------------------
 1 file changed, 20 deletions(-)

diff --git a/machines/config_compilers.xml b/machines/config_compilers.xml
index 6f302100..a7612f31 100644
--- a/machines/config_compilers.xml
+++ b/machines/config_compilers.xml
@@ -401,23 +401,3 @@
     </FFLAGS>
   </compiler>
 </config_compilers>
-
-
-<!-- <compiler MACH="derecho-gpu">
-  <NETCDF_PATH>/glade/u/apps/derecho/23.06/spack/opt/spack/netcdf/4.9.2/cray-mpich/8.1.25/oneapi/2023.0.0/wzol</NETCDF_PATH>
-  <PNETCDF_PATH>/glade/u/apps/derecho/23.06/spack/opt/spack/parallel-netcdf/1.12.3/cray-mpich/8.1.25/oneapi/2023.0.0/blyr</PNETCDF_PATH>
-  <SLIBS>
-    <append> -Wl,--copy-dt-needed-entries -lnetcdff -lnetcdf </append>
-  </SLIBS>
-</compiler>
-
-<compiler MACH="derecho-gpu" COMPILER="intel">
-  <CFLAGS>
-    <base>  -qno-opt-dynamic-align -fp-model precise -std=gnu99 </base>
-    <append MODEL="mpi-serial"> -std=gnu89 </append>
-    <append> -march=core-avx2 -no-fma</append>
-  </CFLAGS>
-  <FFLAGS>
-    <append> -march=core-avx2 -no-fma</append>
-  </FFLAGS>
-</compiler> -->

From 74ec5dc008cea51eb48dd1d7f67edcc09bc438f8 Mon Sep 17 00:00:00 2001
From: Matt Archer <ma595@cam.ac.uk>
Date: Tue, 29 Jul 2025 09:31:53 -0600
Subject: [PATCH 3/7] Remove config_compilers.xml and add derecho-gpu.cmake

---
 machines/cmake_macros/derecho-gpu.cmake |  11 +
 machines/config_compilers.xml           | 403 ------------------------
 2 files changed, 11 insertions(+), 403 deletions(-)
 create mode 100644 machines/cmake_macros/derecho-gpu.cmake
 delete mode 100644 machines/config_compilers.xml

diff --git a/machines/cmake_macros/derecho-gpu.cmake b/machines/cmake_macros/derecho-gpu.cmake
new file mode 100644
index 00000000..2772a168
--- /dev/null
+++ b/machines/cmake_macros/derecho-gpu.cmake
@@ -0,0 +1,11 @@
+if (COMP_NAME STREQUAL gptl)
+  string(APPEND CPPDEFS " -DHAVE_NANOTIME -DBIT64 -DHAVE_VPRINTF -DHAVE_BACKTRACE -DHAVE_SLASHPROC -DHAVE_COMM_F2C -DHAVE_TIMES -DHAVE_GETTIMEOFDAY")
+endif()
+set(NETCDF_PATH "$ENV{NETCDF}")
+set(PIO_FILESYSTEM_HINTS "lustre")
+set(PNETCDF_PATH "$ENV{PNETCDF}")
+# If we want to use cray-libsci instead of mkl uncomment this line as well as the module in config_machines.xml
+string(REPLACE "-mkl=cluster" "" SLIBS "${SLIBS}")
+#string(REPLACE "-mkl=cluster" "-qmkl=cluster" SLIBS "${SLIBS}")
+#string(APPEND CPPDEFS " -DNO_SHR_VMATH ")
+string(APPEND CPPDEFS " -DHAVE_GETTID")
diff --git a/machines/config_compilers.xml b/machines/config_compilers.xml
deleted file mode 100644
index a7612f31..00000000
--- a/machines/config_compilers.xml
+++ /dev/null
@@ -1,403 +0,0 @@
-<compiler>
-  <CPPDEFS>
-    <append MODEL="pop"> -D_USE_FLOW_CONTROL </append>
-  </CPPDEFS>
-  <SUPPORTS_CXX>FALSE</SUPPORTS_CXX>
-</compiler>
-
-<compiler COMPILER="cray">
-  <CFLAGS>
-    <append compile_threaded="false"> -h noomp </append>
-    <append DEBUG="TRUE"> -g -O0 </append>
-    <append DEBUG="FALSE"> -O2 </append>
-  </CFLAGS>
-  <CPPDEFS>
-    <!--http://docs.cray.com/cgi-bin/craydoc.cgi?mode=View;id=S-3901-83;idx=books_search;this_sort=;q=;type=books;title=Cray%20Fortran%20Reference%20Manual -->
-    <append> -DFORTRANUNDERSCORE -DNO_R16 -DCPRCRAY</append>
-    <append MODEL="pop"> -DDIR=NOOP </append>
-    <append MODEL="moby"> -DDIR=NOOP </append>
-  </CPPDEFS>
-  <FC_AUTO_R8>
-    <base> -s real64 </base>
-  </FC_AUTO_R8>
-  <FFLAGS>
-    <base> -f free -N 255  -h byteswapio -x dir </base>
-    <append compile_threaded="false"> -h noomp </append>
-    <append DEBUG="TRUE"> -g -O0 -K trap=fp  -m1 </append>
-    <append DEBUG="FALSE"> -O2,ipa2 -em </append>
-  </FFLAGS>
-  <FFLAGS_NOOPT>
-    <base> -O1,fp2,ipa0,scalar0,vector0 </base>
-  </FFLAGS_NOOPT>
-  <HAS_F2008_CONTIGUOUS>TRUE</HAS_F2008_CONTIGUOUS>
-  <LDFLAGS>
-    <base> -Wl,--allow-multiple-definition -h byteswapio </base>
-  </LDFLAGS>
-</compiler>
-
-<compiler COMPILER="gnu">
-  <CFLAGS>
-    <base> -std=gnu99 </base>
-    <append compile_threaded="true"> -fopenmp </append>
-    <append DEBUG="TRUE"> -g -Wall -Og -fbacktrace -ffpe-trap=invalid,zero,overflow -fcheck=bounds </append>
-    <append DEBUG="FALSE"> -O </append>
-  </CFLAGS>
-  <CPPDEFS>
-    <!-- http://gcc.gnu.org/onlinedocs/gfortran/ -->
-    <append> -DFORTRANUNDERSCORE -DNO_R16 -DCPRGNU</append>
-  </CPPDEFS>
-  <CXX_LINKER>FORTRAN</CXX_LINKER>
-  <FC_AUTO_R8>
-    <base> -fdefault-real-8 </base>
-  </FC_AUTO_R8>
-  <FFLAGS>
-    <!-- -ffree-line-length-none and -ffixed-line-length-none need to be in FFLAGS rather than in FIXEDFLAGS/FREEFLAGS
-       so that these are passed to cmake builds (cmake builds don't use FIXEDFLAGS and FREEFLAGS). -->
-    <base>  -fconvert=big-endian -ffree-line-length-none -ffixed-line-length-none </base>
-    <append compile_threaded="true"> -fopenmp </append>
-    <!-- Ideally, we would also have 'invalid' in the ffpe-trap list. But at
-         least with some versions of gfortran (confirmed with 5.4.0, 6.3.0 and
-         7.1.0), gfortran's isnan (which is called in cime via the
-         CPRGNU-specific shr_infnan_isnan) causes a floating point exception
-         when called on a signaling NaN. -->
-    <append DEBUG="TRUE"> -g -Wall -Og -fbacktrace -ffpe-trap=zero,overflow -fcheck=bounds </append>
-    <append DEBUG="FALSE"> -O </append>
-  </FFLAGS>
-  <FFLAGS_NOOPT>
-    <base> -O0 </base>
-  </FFLAGS_NOOPT>
-  <FIXEDFLAGS>
-    <base>  -ffixed-form </base>
-  </FIXEDFLAGS>
-  <FREEFLAGS>
-    <base> -ffree-form </base>
-  </FREEFLAGS>
-  <HAS_F2008_CONTIGUOUS>FALSE</HAS_F2008_CONTIGUOUS>
-  <LDFLAGS>
-    <append compile_threaded="true"> -fopenmp </append>
-  </LDFLAGS>
-  <MPICC> mpicc  </MPICC>
-  <MPICXX> mpicxx </MPICXX>
-  <MPIFC> mpif90 </MPIFC>
-  <SCC> gcc </SCC>
-  <SCXX> g++ </SCXX>
-  <SFC> gfortran </SFC>
-  <SUPPORTS_CXX>TRUE</SUPPORTS_CXX>
-</compiler>
-
-<compiler COMPILER="ibm">
-  <CFLAGS>
-    <base> -g -qfullpath -qmaxmem=-1 </base>
-    <append DEBUG="FALSE"> -O3  </append>
-    <append compile_threaded="true"> -qsmp=omp </append>
-    <append DEBUG="TRUE" compile_threaded="true"> -qsmp=omp:noopt </append>
-  </CFLAGS>
-  <CPPDEFS>
-    <!-- http://publib.boulder.ibm.com/infocenter/comphelp/v7v91/index.jsp
- Notes:  (see xlf user's guide for the details)
-  -lmass          => IBM-tuned intrinsic lib
-  -qsmp=noauto    => enable SMP directives, but don't add any
-  -qsmp=omp       => enable SMP directives, strict omp
-  -qstrict        => don't turn divides into multiplies, etc
-  -qhot           => higher-order-transformations (eg. loop padding)
-  -qalias=noaryovrlp => assume no array overlap wrt equivalance, etc
-  -qmaxmem=-1     => memory available to compiler during optimization
-  -qipa=level=2   => InterProcedure Analysis (eg. inlining) => slow compiles
-  -p -pg          => enable profiling (use in both FFLAGS and LDFLAGS)
-  -qreport        => for smp/omp only
-  -g              => always leave it on because overhead is minimal
-  -qflttrap=...   => enable default sigtrap (core dump)
-  -C              => runtime array bounds checking (runs slow)
-  -qinitauto=...  => initializes automatic variables
-  -->
-    <append> -DFORTRAN_SAME -DCPRIBM </append>
-  </CPPDEFS>
-  <CPRE>-WF,-D</CPRE>
-  <FC_AUTO_R8>
-    <base> -qrealsize=8 </base>
-  </FC_AUTO_R8>
-  <FFLAGS>
-    <base> -g -qfullpath -qmaxmem=-1 </base>
-    <append DEBUG="FALSE"> -O2 -qstrict -qinline=auto </append>
-    <append compile_threaded="true"> -qsmp=omp </append>
-    <append DEBUG="TRUE"> -qinitauto=7FF7FFFF -qflttrap=ov:zero:inv:en </append>
-    <append DEBUG="TRUE" compile_threaded="true"> -qsmp=omp:noopt </append>
-    <append DEBUG="TRUE" MODEL="pop"> -C </append>
-  </FFLAGS>
-  <FIXEDFLAGS>
-    <base>  -qsuffix=f=f -qfixed=132 </base>
-  </FIXEDFLAGS>
-  <FREEFLAGS>
-    <base> -qsuffix=f=f90:cpp=F90  </base>
-  </FREEFLAGS>
-  <HAS_F2008_CONTIGUOUS>TRUE</HAS_F2008_CONTIGUOUS>
-  <LDFLAGS>
-    <append compile_threaded="true"> -qsmp=omp </append>
-    <append DEBUG="TRUE" compile_threaded="true"> -qsmp=omp:noopt </append>
-  </LDFLAGS>
-</compiler>
-
-<compiler COMPILER="intel">
-  <CFLAGS>
-    <base>  -qno-opt-dynamic-align -fp-model precise -std=gnu99 </base>
-    <append compile_threaded="true"> -qopenmp </append>
-    <append DEBUG="FALSE"> -O2 -debug minimal </append>
-    <append DEBUG="TRUE"> -O0 -g </append>
-  </CFLAGS>
-  <CPPDEFS>
-    <!-- http://software.intel.com/en-us/articles/intel-composer-xe/ -->
-    <append> -DFORTRANUNDERSCORE -DCPRINTEL</append>
-  </CPPDEFS>
-  <CXX_LDFLAGS>
-    <base> -cxxlib </base>
-  </CXX_LDFLAGS>
-  <CXX_LINKER>FORTRAN</CXX_LINKER>
-  <FC_AUTO_R8>
-    <base> -r8 </base>
-  </FC_AUTO_R8>
-  <FFLAGS>
-    <base> -qno-opt-dynamic-align  -convert big_endian -assume byterecl -ftz -traceback -assume realloc_lhs -fp-model source  </base>
-    <append compile_threaded="true"> -qopenmp </append>
-    <append DEBUG="TRUE"> -O0 -g -check uninit -check bounds -check pointers -fpe0 -check noarg_temp_created </append>
-    <append DEBUG="FALSE"> -O2 -debug minimal </append>
-  </FFLAGS>
-  <FFLAGS_NOOPT>
-    <base> -O0 </base>
-    <append compile_threaded="true"> -qopenmp </append>
-  </FFLAGS_NOOPT>
-  <FIXEDFLAGS>
-    <base> -fixed -132 </base>
-  </FIXEDFLAGS>
-  <FREEFLAGS>
-    <base> -free </base>
-  </FREEFLAGS>
-  <LDFLAGS>
-    <append compile_threaded="true"> -qopenmp </append>
-  </LDFLAGS>
-  <MPICC> mpicc  </MPICC>
-  <MPICXX> mpicxx </MPICXX>
-  <MPIFC> mpif90 </MPIFC>
-  <SCC> icc </SCC>
-  <SCXX> icpc </SCXX>
-  <SFC> ifort </SFC>
-  <SLIBS>
-    <append MPILIB="mpich"> -mkl=cluster </append>
-    <append MPILIB="mpich2"> -mkl=cluster </append>
-    <append MPILIB="mvapich"> -mkl=cluster </append>
-    <append MPILIB="mvapich2"> -mkl=cluster </append>
-    <append MPILIB="mpt"> -mkl=cluster </append>
-    <append MPILIB="openmpi"> -mkl=cluster </append>
-    <append MPILIB="impi"> -mkl=cluster </append>
-    <append MPILIB="mpi-serial"> -mkl </append>
-  </SLIBS>
-  <SUPPORTS_CXX>TRUE</SUPPORTS_CXX>
-</compiler>
-
-<compiler COMPILER="nag">
-  <CFLAGS>
-    <base> -std=gnu99 </base>
-    <append DEBUG="TRUE"> -g </append>
-  </CFLAGS>
-  <CPPDEFS>
-    <append> -DFORTRANUNDERSCORE -DNO_CRAY_POINTERS -DNO_SHR_VMATH -DCPRNAG </append>
-  </CPPDEFS>
-  <FC_AUTO_R8>
-    <base> -r8 </base>
-  </FC_AUTO_R8>
-  <FFLAGS>
-    <!-- The indirect flag below is to deal with MPI functions that violate    -->
-    <!-- the Fortran standard, by adding a large set of arguments from a file. -->
-    <base>-Wp,-macro=no_com -convert=BIG_ENDIAN -indirect $ENV{CIMEROOT}/config/cesm/machines/nag_mpi_argument.txt</base>
-    <!-- DEBUG vs. non-DEBUG runs.                                             -->
-    <append DEBUG="FALSE"> -ieee=full -O2 </append>
-    <!-- The "-gline" option is nice, but it doesn't work with OpenMP.         -->
-    <!-- Runtime checks with OpenMP (in fact, all OpenMP cases) are WIP.       -->
-    <append DEBUG="TRUE"> -C=all -g -time -f2003 -ieee=stop </append>
-    <append DEBUG="TRUE" compile_threaded="false"> -gline </append>
-    <!-- The SLAP library (which is part of the CISM build) has many instances of
-       arguments being passed to different types. So disable argument type
-       checking when building CISM. This can be removed once we remove SLAP from
-       CISM. -->
-    <append MODEL="cism"> -mismatch_all </append>
-  </FFLAGS>
-  <FFLAGS_NOOPT>
-    <base>-Wp,-macro=no_com -convert=BIG_ENDIAN -indirect $ENV{CIMEROOT}/config/cesm/machines/nag_mpi_argument.txt</base>
-    <append DEBUG="FALSE"> -ieee=full     </append>
-    <!-- Hack! If DEBUG="TRUE", put runtime checks in FFLAGS, but not into     -->
-    <!-- FFLAGS_NOOPT, allowing strict checks to be removed from files by      -->
-    <!-- having them use FFLAGS_NOOPT in Depends.nag                           -->
-    <append DEBUG="TRUE"> -g -time -f2003 -ieee=stop </append>
-    <append DEBUG="TRUE" compile_threaded="false"> -gline        </append>
-  </FFLAGS_NOOPT>
-  <FIXEDFLAGS>
-    <base> -fixed </base>
-  </FIXEDFLAGS>
-  <FREEFLAGS>
-    <base> -free </base>
-  </FREEFLAGS>
-  <HAS_F2008_CONTIGUOUS>FALSE</HAS_F2008_CONTIGUOUS>
-  <MPICC> mpicc </MPICC>
-  <MPIFC> mpif90 </MPIFC>
-  <SCC> gcc </SCC>
-  <SFC> nagfor </SFC>
-</compiler>
-
-<compiler COMPILER="pgi">
-  <CFLAGS>
-    <base> -gopt  -time </base>
-    <append compile_threaded="true"> -mp </append>
-  </CFLAGS>
-  <CPPDEFS>
-    <!-- http://www.pgroup.com/resources/docs.htm                                              -->
-    <!-- Notes:  (see pgi man page & user's guide for the details) -->
-    <!--  -Mextend        => Allow 132-column source lines -->
-    <!--  -Mfixed         => Assume fixed-format source -->
-    <!--  -Mfree          => Assume free-format source -->
-    <!--  -byteswapio     => Swap byte-order for unformatted i/o (big/little-endian) -->
-    <!--  -target=linux   => Specifies the target architecture to Compute Node Linux (CNL only) -->
-    <!--  -fast           => Chooses generally optimal flags for the target platform -->
-    <!--  -Mnovect        => Disables automatic vector pipelining -->
-    <!--  -Mvect=nosse    => Don't generate SSE, SSE2, 3Dnow, and prefetch instructions in loops    -->
-    <!--  -Mflushz        => Set SSE to flush-to-zero mode (underflow) loops where possible  -->
-    <!--  -Kieee          => Perform fp ops in strict conformance with the IEEE 754 standard.  -->
-    <!--                     Some optimizations disabled, slightly slower, more accurate math.  -->
-    <!--  -mp=nonuma      => Don't use thread/processors affinity (for NUMA architectures)  -->
-    <!-- -->
-    <!--  -g              => Generate symbolic debug information. Turns off optimization.   -->
-    <!--  -gopt           => Generate information for debugger without disabling optimizations  -->
-    <!--  -Mbounds        => Add array bounds checking  -->
-    <!--  -Ktrap=fp       => Determine IEEE Trap conditions fp => inv,divz,ovf   -->
-    <!--                     * inv: invalid operands         -->
-    <!--                     * divz divide by zero           -->
-    <!--                     * ovf: floating point overflow   -->
-    <!--  -Mlist          => Create a listing file             -->
-    <!--  -F              => leaves file.f for each preprocessed file.F file  -->
-    <!--  -time           => Print execution time for each compiler step  -->
-    <append> -DFORTRANUNDERSCORE -DNO_SHR_VMATH -DNO_R16  -DCPRPGI </append>
-  </CPPDEFS>
-  <CXX_LINKER>CXX</CXX_LINKER>
-  <FC_AUTO_R8>
-    <base> -r8 </base>
-  </FC_AUTO_R8>
-  <FFLAGS>
-    <base>  -i4 -gopt  -time -Mextend -byteswapio -Mflushz -Kieee  </base>
-    <append compile_threaded="true"> -mp </append>
-    <append DEBUG="TRUE"> -O0 -g -Ktrap=fp -Mbounds -Kieee </append>
-    <append MODEL="datm"> -Mnovect </append>
-    <append MODEL="dlnd"> -Mnovect </append>
-    <append MODEL="drof"> -Mnovect </append>
-    <append MODEL="dwav"> -Mnovect </append>
-    <append MODEL="dice"> -Mnovect </append>
-    <append MODEL="docn"> -Mnovect </append>
-  </FFLAGS>
-  <FFLAGS_NOOPT>
-    <base> -O0 -g -Ktrap=fp -Mbounds -Kieee  </base>
-    <append compile_threaded="true"> -mp </append>
-  </FFLAGS_NOOPT>
-  <FIXEDFLAGS>
-    <base> -Mfixed </base>
-  </FIXEDFLAGS>
-  <FREEFLAGS>
-    <base> -Mfree </base>
-  </FREEFLAGS>
-  <!-- Note that SUPPORTS_CXX is false for pgi in general, because we
-       need some machine-specific libraries - see hopper pgi for an
-       example -->
-  <!-- Technically, PGI does recognize this keyword during parsing,
-       but support is either buggy or incomplete, notably in that
-       the "contiguous" attribute is incompatible with "intent".-->
-  <HAS_F2008_CONTIGUOUS>FALSE</HAS_F2008_CONTIGUOUS>
-  <LDFLAGS>
-    <base> -time -Wl,--allow-multiple-definition </base>
-    <append compile_threaded="true"> -mp </append>
-  </LDFLAGS>
-  <MPICC> mpicc </MPICC>
-  <MPICXX> mpicxx </MPICXX>
-  <MPIFC> mpif90 </MPIFC>
-  <SCC> pgcc </SCC>
-  <SCXX> pgc++ </SCXX>
-  <SFC> pgf95 </SFC>
-</compiler>
-
-<compiler OS="AIX" COMPILER="ibm">
-  <CFLAGS>
-    <append> -qarch=auto -qtune=auto -qcache=auto </append>
-  </CFLAGS>
-  <CONFIG_SHELL> /usr/bin/bash </CONFIG_SHELL>
-  <FFLAGS>
-    <append> -qarch=auto -qtune=auto -qcache=auto -qsclk=micro </append>
-    <append MODEL="cam"> -qspill=6000 </append>
-  </FFLAGS>
-  <LDFLAGS>
-    <append DEBUG="TRUE"> -qsigtrap=xl__trcedump </append>
-    <append> -bdatapsize:64K -bstackpsize:64K -btextpsize:32K </append>
-  </LDFLAGS>
-  <MPICC> mpcc_r </MPICC>
-  <MPIFC> mpxlf2003_r </MPIFC>
-  <SCC> cc_r </SCC>
-  <SFC> xlf2003_r </SFC>
-  <SLIBS>
-    <append> -lmassv -lessl </append>
-    <append DEBUG="FALSE"> -lmass </append>
-  </SLIBS>
-</compiler>
-
-<compiler OS="BGQ" COMPILER="ibm">
-  <CONFIG_ARGS>
-    <base> --build=powerpc-bgp-linux --host=powerpc64-suse-linux </base>
-  </CONFIG_ARGS>
-  <CPPDEFS>
-    <append> -DLINUX  </append>
-  </CPPDEFS>
-  <FFLAGS>
-    <base> -g -qfullpath -qmaxmem=-1 -qspillsize=2500 -qextname=flush </base>
-    <append DEBUG="FALSE"> -O3 -qstrict -qinline=auto </append>
-    <append DEBUG="FALSE" compile_threaded="true"> -qsmp=omp </append>
-    <append DEBUG="TRUE" compile_threaded="true"> -qsmp=omp:noopt </append>
-  </FFLAGS>
-  <LDFLAGS>
-    <base>  -Wl,--relax -Wl,--allow-multiple-definition </base>
-  </LDFLAGS>
-</compiler>
-
-<compiler OS="CNL">
-  <CMAKE_OPTS>
-    <base> -DCMAKE_SYSTEM_NAME=Catamount</base>
-  </CMAKE_OPTS>
-  <CPPDEFS>
-    <append> -DLINUX </append>
-    <append MODEL="gptl"> -DHAVE_NANOTIME -DBIT64 -DHAVE_VPRINTF -DHAVE_BACKTRACE -DHAVE_SLASHPROC -DHAVE_COMM_F2C -DHAVE_TIMES -DHAVE_GETTIMEOFDAY  </append>
-  </CPPDEFS>
-  <MPICC> cc </MPICC>
-  <MPICXX> CC </MPICXX>
-  <MPIFC> ftn </MPIFC>
-  <NETCDF_PATH>$ENV{NETCDF_DIR}</NETCDF_PATH>
-  <PIO_FILESYSTEM_HINTS>lustre</PIO_FILESYSTEM_HINTS>
-  <PNETCDF_PATH>$ENV{PARALLEL_NETCDF_DIR}</PNETCDF_PATH>
-  <SCC> cc </SCC>
-  <SCXX> CC </SCXX>
-  <SFC> ftn </SFC>
-</compiler>
-
-<?xml version="1.0"?>
-
-<config_compilers>
-  <compiler MACH="derecho-gpu">
-    <NETCDF_PATH>/glade/u/apps/derecho/23.06/spack/opt/spack/netcdf/4.9.2/cray-mpich/8.1.25/oneapi/2023.0.0/wzol</NETCDF_PATH>
-    <PNETCDF_PATH>/glade/u/apps/derecho/23.06/spack/opt/spack/parallel-netcdf/1.12.3/cray-mpich/8.1.25/oneapi/2023.0.0/blyr</PNETCDF_PATH>
-    <SLIBS>
-      <append> -lnetcdff -lnetcdf </append>
-    </SLIBS>
-  </compiler>
-
-  <compiler MACH="derecho-gpu" COMPILER="intel">
-    <CFLAGS>
-      <base>  -qno-opt-dynamic-align -fp-model precise -std=gnu99 </base>
-      <append MODEL="mpi-serial"> -std=gnu89 </append>
-      <append> -march=core-avx2 -no-fma</append>
-    </CFLAGS>
-    <FFLAGS>
-      <append> -march=core-avx2 -no-fma</append>
-    </FFLAGS>
-  </compiler>
-</config_compilers>

From b6498274d2c08d72eaaa7674c352e01db631781a Mon Sep 17 00:00:00 2001
From: Matt Archer <ma595@cam.ac.uk>
Date: Tue, 29 Jul 2025 10:38:46 -0600
Subject: [PATCH 4/7] Set some sensible memory defaults for derecho-gpu

---
 machines/config_batch.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/machines/config_batch.xml b/machines/config_batch.xml
index f35d2838..70b5ec8f 100644
--- a/machines/config_batch.xml
+++ b/machines/config_batch.xml
@@ -381,7 +381,7 @@
     </submit_args>
     <directives>
       <directive default="/bin/bash" > -S {{ shell }}  </directive>
-      <directive> -l select={{ num_nodes }}:ncpus={{ max_tasks_per_node }}:mpiprocs={{ tasks_per_node }}:ompthreads={{ thread_count }}:ngpus=4</directive>
+      <directive> -l select={{ num_nodes }}:ncpus={{ max_tasks_per_node }}:mpiprocs={{ tasks_per_node }}:ompthreads={{ thread_count }}:mem=480GB:ngpus={{ ngpus_per_node }}</directive>
     </directives>
     <queues>
       <queue walltimemax="12:00:00" nodemin="1" nodemax="82" default="true">main</queue>

From 5d54f6f1356100eaa0e7b73c28b31f682c52161c Mon Sep 17 00:00:00 2001
From: Matt Archer <ma595@cam.ac.uk>
Date: Tue, 19 Aug 2025 03:55:44 -0600
Subject: [PATCH 5/7] Fix formatting and add sensible defaults

---
 machines/config_machines.xml | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/machines/config_machines.xml b/machines/config_machines.xml
index d4904157..cfcd4ab2 100644
--- a/machines/config_machines.xml
+++ b/machines/config_machines.xml
@@ -1272,15 +1272,15 @@ This allows using a different mpirun command to launch unit tests
     <GMAKE_J>16</GMAKE_J>
     <BATCH_SYSTEM>pbs</BATCH_SYSTEM>
     <SUPPORTED_BY>cseg</SUPPORTED_BY>
-    <MAX_TASKS_PER_NODE>64</MAX_TASKS_PER_NODE>
-    <MAX_MPITASKS_PER_NODE>64</MAX_MPITASKS_PER_NODE>
+    <MAX_TASKS_PER_NODE>128</MAX_TASKS_PER_NODE>
+    <MAX_MPITASKS_PER_NODE>128</MAX_MPITASKS_PER_NODE>
     <PROJECT_REQUIRED>TRUE</PROJECT_REQUIRED>
     <mpirun mpilib="default">
       <executable>mpiexec</executable>
       <arguments>
-	<arg name="label"> --label</arg>
-	<arg name="buffer"> --line-buffer</arg>
-	<arg name="num_tasks" > -n {{ total_tasks }}</arg>
+        <arg name="label"> --label</arg>
+        <arg name="buffer"> --line-buffer</arg>
+        <arg name="num_tasks" > -n {{ total_tasks }}</arg>
       </arguments>
     </mpirun>
     <module_system type="module" allow_error="true">

From 41b32c325c61cb077e55a9d7e5e944a64a729d12 Mon Sep 17 00:00:00 2001
From: Matt Archer <ma595@cam.ac.uk>
Date: Tue, 19 Aug 2025 04:01:22 -0600
Subject: [PATCH 6/7] Change back to 64 tasks

---
 machines/config_machines.xml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/machines/config_machines.xml b/machines/config_machines.xml
index cfcd4ab2..c7bd1337 100644
--- a/machines/config_machines.xml
+++ b/machines/config_machines.xml
@@ -1272,8 +1272,8 @@ This allows using a different mpirun command to launch unit tests
     <GMAKE_J>16</GMAKE_J>
     <BATCH_SYSTEM>pbs</BATCH_SYSTEM>
     <SUPPORTED_BY>cseg</SUPPORTED_BY>
-    <MAX_TASKS_PER_NODE>128</MAX_TASKS_PER_NODE>
-    <MAX_MPITASKS_PER_NODE>128</MAX_MPITASKS_PER_NODE>
+    <MAX_TASKS_PER_NODE>64</MAX_TASKS_PER_NODE>
+    <MAX_MPITASKS_PER_NODE>64</MAX_MPITASKS_PER_NODE>
     <PROJECT_REQUIRED>TRUE</PROJECT_REQUIRED>
     <mpirun mpilib="default">
       <executable>mpiexec</executable>

From d6d079af1f05cf9943f79f7e788944b082daf86a Mon Sep 17 00:00:00 2001
From: Matt Archer <ma595@cam.ac.uk>
Date: Tue, 19 Aug 2025 04:02:10 -0600
Subject: [PATCH 7/7] Hardcode ngpus to 4

---
 machines/config_batch.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/machines/config_batch.xml b/machines/config_batch.xml
index 70b5ec8f..d6789f10 100644
--- a/machines/config_batch.xml
+++ b/machines/config_batch.xml
@@ -381,7 +381,7 @@
     </submit_args>
     <directives>
       <directive default="/bin/bash" > -S {{ shell }}  </directive>
-      <directive> -l select={{ num_nodes }}:ncpus={{ max_tasks_per_node }}:mpiprocs={{ tasks_per_node }}:ompthreads={{ thread_count }}:mem=480GB:ngpus={{ ngpus_per_node }}</directive>
+      <directive> -l select={{ num_nodes }}:ncpus={{ max_tasks_per_node }}:mpiprocs={{ tasks_per_node }}:ompthreads={{ thread_count }}:mem=480GB:ngpus=4</directive>
     </directives>
     <queues>
       <queue walltimemax="12:00:00" nodemin="1" nodemax="82" default="true">main</queue>