Add support for Pawsey Setonix

.gitignore

@@ -162,3 +162,6 @@ cython_debug/
 #.idea/
 
 .*.sw?
+
+# singulatity images
+*.sif

README.md

@@ -58,6 +58,12 @@ network access and a build stage without network access
 
 See [site/nci/README.md](site/nci/README.md)
 
+### Pawsey
+
+Build the environment at Pawsey Setonix
+
+See [site/pawsey/README.md](site/pawsey/README.md)
+
 ## Environments
 
 ### LFRic

environments/lfric/etc/env.activate.sh

@@ -3,9 +3,14 @@
 SPACK_ENV_VIEW=$SPACK_ENV/.spack-env/view
 
 export FCM_KEYWORDS=${FCM_KEYWORDS:-$NGMOENVS_ENVDIR/etc/fcm-keyword.cfg}
-export LDMPI=mpifort
 export FPP="cpp -traditional"
 export FFLAGS="${FFLAGS:-} -I$SPACK_ENV_VIEW/include -I$SPACK_ENV_VIEW/lib"
+export LDFLAGS="-L$SPACK_ENV_VIEW/lib"
 export PFUNIT=$SPACK_ENV_VIEW
-export LIBRARY_PATH=$SPACK_ENV_VIEW/lib:$LIBRARY_PATH
 
+# Handle Cray linking with ftn rather than mpifort
+if [[ "$(basename "$FC")" == "ftn" ]]; then
+    export LDMPI=$FC
+else
+    export LDMPI=mpifort
+fi

environments/lfric/spack.yaml

@@ -3,7 +3,7 @@ spack:
     - [email protected] +lfric
     - [email protected] idxtype=long
     - pfunit@3 max_array_rank=6 +mpi +openmp
-    - tau +mpi +openmp +opari ~pthreads +lfric
+    # - tau +mpi +openmp +opari ~pthreads +lfric
     - fcm
     - rsync
     - [email protected]

site/pawsey/README.md

@@ -0,0 +1,48 @@
+# Pawsey Setonix Containers
+
+Containers are built similar to the apptainer site, but in two stages. The
+first stage runs in 'copyq' which has internet access, the second stage runs on
+the compute nodes for faster builds.
+
+Pawsey containers make use of Pawsey's pre-installed compilers and MPI. Images are not
+portable to other sites, for portable images use the `apptainer` site directory.
+
+## Building the environment
+
+A container can be built with
+
+```bash
+# Install an environment
+./site/pawsey/install.sh lfric
+```
+
+## Using environments
+
+By default the environments will be installed into
+`/scratch/$PAWSEY_PROJECT/$USER/ngmo-envs/envs/$ENVIRONMENT/$BRANCH`.
+
+Make sure the environment's `bin/` directory is on your `PATH`, e.g.
+by loading the module:
+
+```bash
+module use /scratch/$PAWSEY_PROJECT/$USER/ngmo-envs/modules
+module load lfric
+```
+
+Run commands inside the container using the `envrun` script. If you build an
+executable inside the container you must also run it inside the container.
+
+```
+envrun make
+envrun mpirun -n 6 lfric
+```
+
+A script `envrun-wrapped` is also provided, this can be symlinked to other
+names to run the named command inside the environment:
+
+```
+ln -s /scratch/$PAWSEY_PROJECT/$USER/ngmo-envs/envs/lfric/master/bin/{envrun-wrapped,python}
+
+# Now `python` will get run inside the environment
+python
+```

site/pawsey/baseimage/Dockerfile

@@ -0,0 +1,38 @@
+FROM opensuse/leap:15
+
+# Setup default linker search paths
+COPY pawsey-ld.conf /etc/ld.so.conf.d/pawsey-ld.conf
+
+RUN zypper --non-interactive install \
+             bzip2 \
+             curl \
+             diffutils \
+             file \
+             findutils \
+             gcc \
+             gcc-c++ \
+             gcc-fortran \
+             git \
+             gzip \
+             libatomic1 \
+             libtool \
+             libgfortran5 \
+             lua53 \
+             lua53-luaposix \
+             lua53-luasocket \
+             make \
+             openssl \
+             patch \
+             subversion \
+             tar \
+             tcl \
+             time \
+             unzip \
+             wget \
+             which \
+             xz \
+             xz-devel \
+             zstd \
+    && zypper clean
+
+ENTRYPOINT ["/ngmo/bin/entrypoint.sh"]

site/pawsey/baseimage/README.md

@@ -0,0 +1,10 @@
+NCI Apptainer base image
+========================
+
+Apptainer containers can't be built directly at NCI, this requires root
+privileges. Instead we create a base image using Docker, then convert that base
+image to apptainer format.
+
+Environment containers are created as squashfs overlays on top of this base
+image, these are inserted into the base image as part of the install process so
+the end result is a single container file.

site/pawsey/baseimage/build.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+docker build --tag ngmoenvs-baseimage:latest .
+
+apptainer build ngmoenvs-baseimage.sif docker-daemon:ngmoenvs-baseimage:latest

site/pawsey/baseimage/pawsey-ld.conf

@@ -0,0 +1,9 @@
+/opt/rocm-5.7.1/lib
+/opt/amdgpu/lib64
+/opt/amdgpu/lib
+/opt/rocm/lib
+/opt/cray/pe/lib64
+/opt/cray/pe/lib64/cce
+/opt/cray/xpmem/default/lib64
+/opt/esmi/e_smi/lib/
+/host/lib64

site/pawsey/bootstrap.sh

@@ -0,0 +1,25 @@
+#!/bin/bash
+# Set up Pawsey Setonix to build environments
+
+set -eu
+set -o pipefail
+SCRIPT_DIR=$( cd -- "$( dirname -- "$(readlink -f "${BASH_SOURCE[0]}")" )" &> /dev/null && pwd )
+
+: "${NGMOENVS_BASEDIR:="/scratch/$PAWSEY_PROJECT/$USER/ngmo-envs"}"
+export NGMOENVS_BASEDIR
+
+# Cache paths
+: "${NGMOENVS_SPACK_MIRROR:="file://$NGMOENVS_BASEDIR/spack-mirror"}"
+: "${CONDA_BLD_PATH:="$NGMOENVS_BASEDIR/conda-bld"}"
+export NGMOENVS_SPACK_MIRROR
+export CONDA_BLD_PATH
+
+# Run the common bootstrap to install conda and spack
+"$SCRIPT_DIR/../../utils/bootstrap.sh"
+
+# Load up spack
+source "$NGMOENVS_BASEDIR/bin/activate"
+
+# Configure spack
+spack config --scope=site add -f "$SCRIPT_DIR/spack-packages.yaml"
+spack config --scope=site add -f "$SCRIPT_DIR/spack-compilers.yaml"

site/pawsey/entrypoint.sh

@@ -0,0 +1,25 @@
+#!/bin/bash
+
+# Isolate Spack
+export SPACK_DISABLE_LOCAL_CONFIG=true
+
+# These variables are set during the install process
+NGMOENVS_CONTAINER_BASEDIR=''
+NGMOENVS_ENVIRONMENT=''
+
+# Add host and cce libraries as a fallback
+#export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:/opt/cray/pe/lib64:/opt/cray/pe/lib64/cce:/host/lib64"
+export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:/usr/lib64:/host/lib64"
+
+# Activate the base environment
+if [[ -f "$NGMOENVS_CONTAINER_BASEDIR/bin/activate" ]]; then
+    source "$NGMOENVS_CONTAINER_BASEDIR/bin/activate"
+fi
+
+# Activate the actual environment
+if [[ -f "$NGMOENVS_CONTAINER_BASEDIR/env/bin/activate" ]]; then
+    source "$NGMOENVS_CONTAINER_BASEDIR/env/bin/activate" 
+fi
+
+# Run the given command
+exec "$@"

site/pawsey/envrun

@@ -0,0 +1,23 @@
+#!/bin/bash
+
+ENV_DIR=$( cd -- "$( dirname -- "$(readlink -f "${BASH_SOURCE[0]}")" )" &> /dev/null && pwd )/..
+
+# Arguments to apptainer
+export MOUNT_ARGS=(
+    "--bind" "/opt/AMD" # AOCC compiler
+    "--bind" "/usr/lib64:/host/lib64" # System libraries
+)
+
+# GPG-agent socket
+if [[ -d "/run/user/$UID" ]]; then
+    MOUNT_ARGS+=(--bind "/run/user/$UID")
+fi
+
+# Used by srun on the queue
+if [[ -d /var/spool/slurmd ]]; then
+    MOUNT_ARGS+=(--bind /var/spool/slurmd)
+fi
+
+module load "singularity/4.1.0-slurm"
+
+singularity run "${MOUNT_ARGS[@]}" "$ENV_DIR/etc/image.sif" "$@"

site/pawsey/install-container.sh

@@ -0,0 +1,83 @@
+#!/bin/bash
+
+set -eu
+set -o pipefail
+
+# Build the container
+source "$NGMOENVS_DEFS/utils/common.sh"
+
+# Where we'll set up squashfs within the container
+export CONTAINER_BASEDIR="/ngmo"
+
+# Set variables inside the container
+export NGMOENVS_BASEDIR="$CONTAINER_BASEDIR"
+export NGMOENVS_ENVDIR="$CONTAINER_BASEDIR/env"
+export SPACK_BOOTSTRAP_ROOT="$CONTAINER_BASEDIR/spack-bootstrap"
+
+# Arguments to apptainer
+export MOUNT_ARGS=(
+    "--bind" "$LOCAL_SQUASHFS$CONTAINER_BASEDIR:$CONTAINER_BASEDIR:rw"
+    "--bind" "/opt/AMD" # AOCC compiler
+    "--bind" "/opt/rocm-5.7.1" # ROCM/clang compiler
+    "--bind" "/usr/lib64:/host/lib64" # System libraries
+    "--bind" "/run/user/$UID" # gpgagent sockets
+)
+
+# Set up squashfs directory
+mkdir -p "$LOCAL_SQUASHFS$CONTAINER_BASEDIR/bin"
+
+# Set up entrypoint
+sed "$SITE_DIR/entrypoint.sh" \
+    -e "s|^\(NGMOENVS_CONTAINER_BASEDIR\)=.*|\1='$CONTAINER_BASEDIR'|" \
+    -e "s|^\(NGMOENVS_ENVIRONMENT\)=.*|\1='$ENVIRONMENT'|" \
+    > "$LOCAL_SQUASHFS$CONTAINER_BASEDIR/bin/entrypoint.sh"
+chmod +x "$LOCAL_SQUASHFS$CONTAINER_BASEDIR/bin/entrypoint.sh"
+
+# Bootstrap container
+e singularity run "${MOUNT_ARGS[@]}" "$IMAGE" /bin/bash "$NGMOENVS_DEFS/utils/bootstrap.sh"
+
+# Configure site
+e singularity run "${MOUNT_ARGS[@]}" "$IMAGE" spack config --scope=site add -f "$SITE_DIR/spack-packages.yaml"
+e singularity run "${MOUNT_ARGS[@]}" "$IMAGE" spack config --scope=site add -f "$SITE_DIR/spack-compilers.yaml"
+
+# Install container environment
+e singularity run "${MOUNT_ARGS[@]}" "$IMAGE" /bin/bash "$NGMOENVS_DEFS/utils/install-stage-one.sh"
+
+if [[ "${NGMOENVS_COMPILER%@*}" == cce ]]; then
+    # Set compiler variables 
+    cat >> "$LOCAL_SQUASHFS$NGMOENVS_ENVDIR/bin/activate" <<EOF
+
+# Load Cray PE
+module load PrgEnv-cray
+module load craype-x86-milan
+module load craype-network-ofi
+
+module load cce/18.0.1
+module load cray-mpich/8.1.28
+module load cray-hdf5-parallel/1.12.2.9
+module load cray-netcdf-hdf5parallel/4.9.0.9
+
+export PKG_CONFIG_PATH=/opt/cray/xpmem/2.8.4-1.0_7.23__ga37cbd9.shasta/lib64/pkgconfig:\$PKG_CONFIG_PATH
+export LD_LIBRARY_PATH=\$LD_LIBRARY_PATH:/opt/cray/pe/lib64:/opt/cray/pe/lib64/cce
+EOF
+fi
+
+if [[ "${NGMOENVS_COMPILER%@*}" == gcc ]]; then
+    # Set compiler variables 
+    cat >> "$LOCAL_SQUASHFS$NGMOENVS_ENVDIR/bin/activate" <<EOF
+
+# Load Cray PE
+module load PrgEnv-gnu
+module load craype-x86-milan
+module load craype-network-ofi
+
+module load gcc/12.2.0
+module load cray-mpich/8.1.27
+module load cray-hdf5-parallel/1.12.2.7
+module load cray-netcdf-hdf5parallel/4.9.0.7
+
+export PKG_CONFIG_PATH=/opt/cray/xpmem/2.8.4-1.0_7.23__ga37cbd9.shasta/lib64/pkgconfig:\$PKG_CONFIG_PATH
+export LD_LIBRARY_PATH=\$LD_LIBRARY_PATH:/opt/cray/pe/lib64:/opt/cray/pe/lib64/cce
+EOF
+fi
+