diff --git a/.gitignore b/.gitignore index eacba71..1a0d635 100644 --- a/.gitignore +++ b/.gitignore @@ -162,3 +162,6 @@ cython_debug/ #.idea/ .*.sw? + +# singulatity images +*.sif diff --git a/README.md b/README.md index cf168c7..f4d11aa 100644 --- a/README.md +++ b/README.md @@ -58,6 +58,12 @@ network access and a build stage without network access See [site/nci/README.md](site/nci/README.md) +### Pawsey + +Build the environment at Pawsey Setonix + +See [site/pawsey/README.md](site/pawsey/README.md) + ## Environments ### LFRic diff --git a/environments/lfric/etc/env.activate.sh b/environments/lfric/etc/env.activate.sh index 0173c24..59feca0 100644 --- a/environments/lfric/etc/env.activate.sh +++ b/environments/lfric/etc/env.activate.sh @@ -3,9 +3,14 @@ SPACK_ENV_VIEW=$SPACK_ENV/.spack-env/view export FCM_KEYWORDS=${FCM_KEYWORDS:-$NGMOENVS_ENVDIR/etc/fcm-keyword.cfg} -export LDMPI=mpifort export FPP="cpp -traditional" export FFLAGS="${FFLAGS:-} -I$SPACK_ENV_VIEW/include -I$SPACK_ENV_VIEW/lib" +export LDFLAGS="-L$SPACK_ENV_VIEW/lib" export PFUNIT=$SPACK_ENV_VIEW -export LIBRARY_PATH=$SPACK_ENV_VIEW/lib:$LIBRARY_PATH +# Handle Cray linking with ftn rather than mpifort +if [[ "$(basename "$FC")" == "ftn" ]]; then + export LDMPI=$FC +else + export LDMPI=mpifort +fi diff --git a/environments/lfric/spack.yaml b/environments/lfric/spack.yaml index ed67c7b..8b64968 100644 --- a/environments/lfric/spack.yaml +++ b/environments/lfric/spack.yaml @@ -3,7 +3,7 @@ spack: - xios@2.5.2629 +lfric - yaxt@0.11.3 idxtype=long - pfunit@3 max_array_rank=6 +mpi +openmp - - tau +mpi +openmp +opari ~pthreads +lfric + # - tau +mpi +openmp +opari ~pthreads +lfric - fcm - rsync - python@3.10 diff --git a/site/pawsey/README.md b/site/pawsey/README.md new file mode 100644 index 0000000..7f1e5ec --- /dev/null +++ b/site/pawsey/README.md @@ -0,0 +1,48 @@ +# Pawsey Setonix Containers + +Containers are built similar to the apptainer site, but in two stages. The +first stage runs in 'copyq' which has internet access, the second stage runs on +the compute nodes for faster builds. + +Pawsey containers make use of Pawsey's pre-installed compilers and MPI. Images are not +portable to other sites, for portable images use the `apptainer` site directory. + +## Building the environment + +A container can be built with + +```bash +# Install an environment +./site/pawsey/install.sh lfric +``` + +## Using environments + +By default the environments will be installed into +`/scratch/$PAWSEY_PROJECT/$USER/ngmo-envs/envs/$ENVIRONMENT/$BRANCH`. + +Make sure the environment's `bin/` directory is on your `PATH`, e.g. +by loading the module: + +```bash +module use /scratch/$PAWSEY_PROJECT/$USER/ngmo-envs/modules +module load lfric +``` + +Run commands inside the container using the `envrun` script. If you build an +executable inside the container you must also run it inside the container. + +``` +envrun make +envrun mpirun -n 6 lfric +``` + +A script `envrun-wrapped` is also provided, this can be symlinked to other +names to run the named command inside the environment: + +``` +ln -s /scratch/$PAWSEY_PROJECT/$USER/ngmo-envs/envs/lfric/master/bin/{envrun-wrapped,python} + +# Now `python` will get run inside the environment +python +``` diff --git a/site/pawsey/baseimage/Dockerfile b/site/pawsey/baseimage/Dockerfile new file mode 100644 index 0000000..947741e --- /dev/null +++ b/site/pawsey/baseimage/Dockerfile @@ -0,0 +1,38 @@ +FROM opensuse/leap:15 + +# Setup default linker search paths +COPY pawsey-ld.conf /etc/ld.so.conf.d/pawsey-ld.conf + +RUN zypper --non-interactive install \ + bzip2 \ + curl \ + diffutils \ + file \ + findutils \ + gcc \ + gcc-c++ \ + gcc-fortran \ + git \ + gzip \ + libatomic1 \ + libtool \ + libgfortran5 \ + lua53 \ + lua53-luaposix \ + lua53-luasocket \ + make \ + openssl \ + patch \ + subversion \ + tar \ + tcl \ + time \ + unzip \ + wget \ + which \ + xz \ + xz-devel \ + zstd \ + && zypper clean + +ENTRYPOINT ["/ngmo/bin/entrypoint.sh"] diff --git a/site/pawsey/baseimage/README.md b/site/pawsey/baseimage/README.md new file mode 100644 index 0000000..ca34ea0 --- /dev/null +++ b/site/pawsey/baseimage/README.md @@ -0,0 +1,10 @@ +NCI Apptainer base image +======================== + +Apptainer containers can't be built directly at NCI, this requires root +privileges. Instead we create a base image using Docker, then convert that base +image to apptainer format. + +Environment containers are created as squashfs overlays on top of this base +image, these are inserted into the base image as part of the install process so +the end result is a single container file. diff --git a/site/pawsey/baseimage/build.sh b/site/pawsey/baseimage/build.sh new file mode 100755 index 0000000..54ce7ad --- /dev/null +++ b/site/pawsey/baseimage/build.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +docker build --tag ngmoenvs-baseimage:latest . + +apptainer build ngmoenvs-baseimage.sif docker-daemon:ngmoenvs-baseimage:latest diff --git a/site/pawsey/baseimage/pawsey-ld.conf b/site/pawsey/baseimage/pawsey-ld.conf new file mode 100644 index 0000000..b99df82 --- /dev/null +++ b/site/pawsey/baseimage/pawsey-ld.conf @@ -0,0 +1,9 @@ +/opt/rocm-5.7.1/lib +/opt/amdgpu/lib64 +/opt/amdgpu/lib +/opt/rocm/lib +/opt/cray/pe/lib64 +/opt/cray/pe/lib64/cce +/opt/cray/xpmem/default/lib64 +/opt/esmi/e_smi/lib/ +/host/lib64 diff --git a/site/pawsey/bootstrap.sh b/site/pawsey/bootstrap.sh new file mode 100755 index 0000000..e7080ee --- /dev/null +++ b/site/pawsey/bootstrap.sh @@ -0,0 +1,25 @@ +#!/bin/bash +# Set up Pawsey Setonix to build environments + +set -eu +set -o pipefail +SCRIPT_DIR=$( cd -- "$( dirname -- "$(readlink -f "${BASH_SOURCE[0]}")" )" &> /dev/null && pwd ) + +: "${NGMOENVS_BASEDIR:="/scratch/$PAWSEY_PROJECT/$USER/ngmo-envs"}" +export NGMOENVS_BASEDIR + +# Cache paths +: "${NGMOENVS_SPACK_MIRROR:="file://$NGMOENVS_BASEDIR/spack-mirror"}" +: "${CONDA_BLD_PATH:="$NGMOENVS_BASEDIR/conda-bld"}" +export NGMOENVS_SPACK_MIRROR +export CONDA_BLD_PATH + +# Run the common bootstrap to install conda and spack +"$SCRIPT_DIR/../../utils/bootstrap.sh" + +# Load up spack +source "$NGMOENVS_BASEDIR/bin/activate" + +# Configure spack +spack config --scope=site add -f "$SCRIPT_DIR/spack-packages.yaml" +spack config --scope=site add -f "$SCRIPT_DIR/spack-compilers.yaml" diff --git a/site/pawsey/entrypoint.sh b/site/pawsey/entrypoint.sh new file mode 100644 index 0000000..bf98a20 --- /dev/null +++ b/site/pawsey/entrypoint.sh @@ -0,0 +1,25 @@ +#!/bin/bash + +# Isolate Spack +export SPACK_DISABLE_LOCAL_CONFIG=true + +# These variables are set during the install process +NGMOENVS_CONTAINER_BASEDIR='' +NGMOENVS_ENVIRONMENT='' + +# Add host and cce libraries as a fallback +#export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:/opt/cray/pe/lib64:/opt/cray/pe/lib64/cce:/host/lib64" +export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:/usr/lib64:/host/lib64" + +# Activate the base environment +if [[ -f "$NGMOENVS_CONTAINER_BASEDIR/bin/activate" ]]; then + source "$NGMOENVS_CONTAINER_BASEDIR/bin/activate" +fi + +# Activate the actual environment +if [[ -f "$NGMOENVS_CONTAINER_BASEDIR/env/bin/activate" ]]; then + source "$NGMOENVS_CONTAINER_BASEDIR/env/bin/activate" +fi + +# Run the given command +exec "$@" diff --git a/site/pawsey/envrun b/site/pawsey/envrun new file mode 100644 index 0000000..6f9cf27 --- /dev/null +++ b/site/pawsey/envrun @@ -0,0 +1,23 @@ +#!/bin/bash + +ENV_DIR=$( cd -- "$( dirname -- "$(readlink -f "${BASH_SOURCE[0]}")" )" &> /dev/null && pwd )/.. + +# Arguments to apptainer +export MOUNT_ARGS=( + "--bind" "/opt/AMD" # AOCC compiler + "--bind" "/usr/lib64:/host/lib64" # System libraries +) + +# GPG-agent socket +if [[ -d "/run/user/$UID" ]]; then + MOUNT_ARGS+=(--bind "/run/user/$UID") +fi + +# Used by srun on the queue +if [[ -d /var/spool/slurmd ]]; then + MOUNT_ARGS+=(--bind /var/spool/slurmd) +fi + +module load "singularity/4.1.0-slurm" + +singularity run "${MOUNT_ARGS[@]}" "$ENV_DIR/etc/image.sif" "$@" diff --git a/site/pawsey/install-container.sh b/site/pawsey/install-container.sh new file mode 100644 index 0000000..1c82758 --- /dev/null +++ b/site/pawsey/install-container.sh @@ -0,0 +1,83 @@ +#!/bin/bash + +set -eu +set -o pipefail + +# Build the container +source "$NGMOENVS_DEFS/utils/common.sh" + +# Where we'll set up squashfs within the container +export CONTAINER_BASEDIR="/ngmo" + +# Set variables inside the container +export NGMOENVS_BASEDIR="$CONTAINER_BASEDIR" +export NGMOENVS_ENVDIR="$CONTAINER_BASEDIR/env" +export SPACK_BOOTSTRAP_ROOT="$CONTAINER_BASEDIR/spack-bootstrap" + +# Arguments to apptainer +export MOUNT_ARGS=( + "--bind" "$LOCAL_SQUASHFS$CONTAINER_BASEDIR:$CONTAINER_BASEDIR:rw" + "--bind" "/opt/AMD" # AOCC compiler + "--bind" "/opt/rocm-5.7.1" # ROCM/clang compiler + "--bind" "/usr/lib64:/host/lib64" # System libraries + "--bind" "/run/user/$UID" # gpgagent sockets +) + +# Set up squashfs directory +mkdir -p "$LOCAL_SQUASHFS$CONTAINER_BASEDIR/bin" + +# Set up entrypoint +sed "$SITE_DIR/entrypoint.sh" \ + -e "s|^\(NGMOENVS_CONTAINER_BASEDIR\)=.*|\1='$CONTAINER_BASEDIR'|" \ + -e "s|^\(NGMOENVS_ENVIRONMENT\)=.*|\1='$ENVIRONMENT'|" \ + > "$LOCAL_SQUASHFS$CONTAINER_BASEDIR/bin/entrypoint.sh" +chmod +x "$LOCAL_SQUASHFS$CONTAINER_BASEDIR/bin/entrypoint.sh" + +# Bootstrap container +e singularity run "${MOUNT_ARGS[@]}" "$IMAGE" /bin/bash "$NGMOENVS_DEFS/utils/bootstrap.sh" + +# Configure site +e singularity run "${MOUNT_ARGS[@]}" "$IMAGE" spack config --scope=site add -f "$SITE_DIR/spack-packages.yaml" +e singularity run "${MOUNT_ARGS[@]}" "$IMAGE" spack config --scope=site add -f "$SITE_DIR/spack-compilers.yaml" + +# Install container environment +e singularity run "${MOUNT_ARGS[@]}" "$IMAGE" /bin/bash "$NGMOENVS_DEFS/utils/install-stage-one.sh" + +if [[ "${NGMOENVS_COMPILER%@*}" == cce ]]; then + # Set compiler variables + cat >> "$LOCAL_SQUASHFS$NGMOENVS_ENVDIR/bin/activate" <> "$LOCAL_SQUASHFS$NGMOENVS_ENVDIR/bin/activate" < /dev/null && pwd ) +export SITE_DIR + +# Base environments directory +export NGMOENVS_DEFS="$SITE_DIR/../.." + +module purge +module load "pawsey" +module load "pawseyenv/2024.05" +module load "singularity/4.1.0-slurm" +source "$NGMOENVS_DEFS/utils/common.sh" + +export ENVIRONMENT="$1" + +: "${NGMOENVS_BASEDIR:="/scratch/$PAWSEY_PROJECT/$USER/ngmo-envs"}" +export NGMOENVS_BASEDIR + +# Cache paths +: "${NGMOENVS_SPACK_MIRROR:="file://$NGMOENVS_BASEDIR/spack-mirror"}" +: "${CONDA_BLD_PATH:="$NGMOENVS_BASEDIR/conda-bld"}" +export NGMOENVS_SPACK_MIRROR +export CONDA_BLD_PATH + +# Default compiler and MPI +: "${NGMOENVS_COMPILER:="cce"}" +: "${NGMOENVS_MPI:="cray-mpich"}" +export NGMOENVS_COMPILER NGMOENVS_MPI + +# Base image +: "${NGMOENVS_BASEIMAGE:="$HOME/ngmoenvs-baseimage.sif"}" + +# Directory we'll install the container to +: "${NGMOENVS_ENVDIR:="$NGMOENVS_BASEDIR/envs/$ENVIRONMENT/$VERSION"}" + +# Path for modulefiles +: "${NGMOENVS_MODULE_ROOT:="$NGMOENVS_BASEDIR/modules"}" + +# Where we'll set up squashfs on the host machine +export LOCAL_SQUASHFS="$NGMOENVS_ENVDIR/squashfs" + +# Start clean +[[ -f "$NGMOENVS_ENVDIR" ]] && rm -r "$NGMOENVS_ENVDIR" + +mkdir -p "$NGMOENVS_ENVDIR"/{bin,etc} + +# Set up the image +export IMAGE="$NGMOENVS_ENVDIR/etc/image.sif" +cp "$NGMOENVS_BASEIMAGE" "$IMAGE" + +# Set temp dir +export NGMOENVS_TMPDIR=/scratch/$PAWSEY_PROJECT/$USER/tmp + +# Build the container +bash "$SITE_DIR/install-container.sh" + +# Convert to squashfs +SQUASHFS="$NGMOENVS_ENVDIR/etc/env.squashfs" +e mksquashfs "$LOCAL_SQUASHFS" "$SQUASHFS" -all-root -noappend -processors 2 + +# Install into the container +e singularity sif add \ + --datatype 4 \ + --partfs 1 \ + --parttype 4 \ + --partarch 2 \ + --groupid 1 \ + "$IMAGE" \ + "$SQUASHFS" + +# Clean up temp squashfs +e rm "$SQUASHFS" +e rm -r "$LOCAL_SQUASHFS" + +# Set up scripts +for SCRIPT in envrun; do + cp "$SITE_DIR/$SCRIPT" "$NGMOENVS_ENVDIR/bin/$SCRIPT" + chmod +x "$NGMOENVS_ENVDIR/bin/$SCRIPT" +done + +NGMOENVS_MODULE="$NGMOENVS_MODULE_ROOT/$ENVIRONMENT/$VERSION" +mkdir -p "$(dirname "$NGMOENVS_MODULE")" +cat > "$NGMOENVS_MODULE" << EOF +#%Module1.0 + +set name "$ENVIRONMENT" +set version "$VERSION" +set origin "$(git remote get-url origin) $(git rev-parse HEAD)" +set install_date "$(date --iso=minute)" +set installed_by "$USER - $(getent passwd "$USER" | cut -d ':' -f 5)" +set prefix "$NGMOENVS_ENVDIR" + +proc ModulesHelp {} { + global name version origin install_date installed_by + + puts stderr "NGMO Environment \$name/\$version" + puts stderr " Install info:" + puts stderr " repo: \$origin" + puts stderr " ver: \$version" + puts stderr " date: \$install_date" + puts stderr " by: \$installed_by" +} + +set name_upcase [string toupper [string map {- _} \$name]] + +setenv \${name_upcase}_ROOT "\$prefix" +setenv \${name_upcase}_VERSION "\$version" + +prepend-path PATH "\$prefix/bin" +EOF + +cat < /dev/null && pwd ) + +ENVIRONMENT="$1" + +# Grab $VERSION +source "$SCRIPT_DIR/../../utils/common.sh" + +# Load the environment +module purge +module load pawsey pawseyenv +module use "/scratch/$PAWSEY_PROJECT/$USER/ngmo-envs/modules" +module load "$ENVIRONMENT/$VERSION" +module list + +# Set up run directory +export BASEDIR="/scratch/$PAWSEY_PROJECT/$USER/tmp/ngmo-envs-test/$ENVIRONMENT" +mkdir -p "$BASEDIR" + +# Run the test +/bin/bash "$SCRIPT_DIR/../envs/${ENVIRONMENT}.sh" diff --git a/utils/bootstrap.sh b/utils/bootstrap.sh index f32bd9c..72fe1f0 100755 --- a/utils/bootstrap.sh +++ b/utils/bootstrap.sh @@ -15,7 +15,6 @@ SCRIPT_DIR=$( cd -- "$( dirname -- "$(readlink -f "${BASH_SOURCE[0]}")" )" &> /d source "$SCRIPT_DIR/common.sh" SPACK_VERSION=0.22.2 -export SPACK_DISABLE_LOCAL_CONFIG=1 mkdir -p "$NGMOENVS_BASEDIR/bin" pushd "$NGMOENVS_BASEDIR" > /dev/null @@ -44,6 +43,11 @@ e spack compiler find --scope site /usr/bin e spack external find --scope site --path /usr/bin gcc e spack config add --file "$SCRIPT_DIR/spack-packages.yaml" +if [[ -v SPACK_BOOTSTRAP_ROOT ]]; then + e spack bootstrap root --scope site "$SPACK_BOOTSTRAP_ROOT" +fi +e spack bootstrap now + echo "Default Compiler and MPI:" e spack spec --format '{name}@{version}%{compiler.name}@{compiler.version}' mpi diff --git a/utils/common.sh b/utils/common.sh index 4cae14d..5240d42 100644 --- a/utils/common.sh +++ b/utils/common.sh @@ -24,3 +24,5 @@ error() { : "${VERSION="$(git symbolic-ref --short HEAD)"}" export VERSION +# Don't consider ~/.spack +export SPACK_DISABLE_LOCAL_CONFIG=1 diff --git a/utils/install-compiler.sh b/utils/install-compiler.sh index 59a79a5..9786e88 100755 --- a/utils/install-compiler.sh +++ b/utils/install-compiler.sh @@ -42,9 +42,9 @@ fi mkdir -p "$ENVDIR/etc" cat > "$ENVDIR/etc/compiler.sh" <> "$ENVDIR/bin/activate" +cat >> "$ENVDIR/bin/activate" <> "$ENVDIR/bin/activate" +e cat "$ENVDIR/bin/activate" # Run script cat > "$ENVDIR/bin/envrun" <