Slurm ci

.github/workflows/ci.yml

@@ -104,3 +104,61 @@ jobs:
           cd containers/spindle-flux-ubuntu
           docker compose down
 
+  spindle-slurm-ubuntu:
+    name: Testsuite (Slurm, Ubuntu)
+    environment: Spindle CI
+    runs-on: ubuntu-latest
+    timeout-minutes: 20
+    steps:
+      - name: Check out Spindle
+        uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8
+
+      - name: Setup Docker Compose
+        uses: docker/setup-compose-action@364cc21a5de5b1ee4a7f5f9d3fa374ce0ccde746
+        with:
+          version: latest
+
+      - name: Login to GitHub Container Registry
+        if: ${{ !env.ACT }}
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Generate MariaDB configuration
+        id: slurm-ubuntu-mariadb
+        run: |
+          cd containers/spindle-slurm-ubuntu/testing
+          ./generate_config.sh
+
+      - name: Build spindle-slurm-ubuntu image
+        id: slurm-ubuntu-build
+        run: |
+          cd containers/spindle-slurm-ubuntu/testing
+          docker compose --progress=plain build
+
+      - name: Bring spindle-slurm-ubuntu up
+        id: slurm-ubuntu-up
+        run: |
+          cd containers/spindle-slurm-ubuntu/testing
+          docker compose up -d --wait --wait-timeout 60
+
+      - name: Verify munge works in spindle-slurm-ubuntu
+        id: slurm-ubuntu-munge
+        run: |
+          docker exec slurm-head bash -c 'munge -n | unmunge'
+
+      - name: Run spindle-slurm-ubuntu testsuite
+        id: slurm-ubuntu-testsuite
+        run: |
+          docker exec slurm-head bash -c 'cd Spindle-build/testsuite && salloc -n${workers} -N${workers} ./runTests ${workers}'
+
+      - name: Bring spindle-slurm-ubuntu down
+        id: slurm-ubuntu-down
+        if: ${{ always() }}
+        continue-on-error: true
+        run: |
+          cd containers/spindle-slurm-ubuntu/testing
+          docker compose down
+

.github/workflows/container.yml

@@ -0,0 +1,30 @@
+name: Build & Push Slurm Base Container
+on:
+  workflow_dispatch:
+  workflow_call:
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@b5ca514318bd6ebac0fb2aedd5d36ec1b5c232a2
+
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Build & Push Slurm Base Image
+        uses: docker/build-push-action@1dc73863535b631f98b2378be8619f83b136f4a0
+        with:
+          context: ./containers/spindle-slurm-ubuntu/base
+          platforms: linux/amd64
+          push: true
+          tags: ghcr.io/paratoolsinc/spindle-slurm-base:latest
+          cache-from: type=registry,ref=ghcr.io/paratoolsinc/spindle-slurm-base:buildcache
+          cache-to: type=registry,ref=ghcr.io/paratoolsinc/spindle-slurm-base:buildcache,mode=max

containers/spindle-slurm-ubuntu/base/Dockerfile

@@ -0,0 +1,81 @@
+ARG UBUNTU_VERSION=noble
+FROM ubuntu:${UBUNTU_VERSION}
+USER root
+
+RUN apt-get update \
+ && DEBIAN_FRONTEND="noninteractive" apt-get -qq install -y --no-install-recommends \
+        apt-utils
+
+RUN apt-get update \
+ && DEBIAN_FRONTEND="noninteractive" apt-get -qq install -y --no-install-recommends \
+        locales \
+        ca-certificates \
+        wget \
+        git \
+        ssh \
+        sudo \
+        build-essential \
+        pkg-config \
+        autotools-dev \
+        libtool \
+        autoconf \
+        automake \
+        make \
+        gfortran-13 \
+        gcc-13 \
+        g++-13 \
+        munge \
+        libmunge-dev \
+        libhwloc-dev \
+        python3-dev \
+        python3-pip \
+        python3-setuptools \
+        python3-wheel \
+        python-is-python3 \
+        openssh-server \
+        openssh-client \
+        mariadb-client \
+        libmariadb-dev \
+        libhttp-parser-dev \
+        psmisc \
+        libjson-c-dev \
+        fd-find \
+        silversearcher-ag \
+        vim
+
+
+# Prevent hwloc from trying to use graphics cards
+# as this fails when X is not running.
+ENV HWLOC_COMPONENTS=-gl
+
+# Set up munge
+RUN mkdir -p /run/munge && \
+    chown munge:munge /run/munge && \
+    chmod 0755 /run/munge
+
+ARG BUILD_ROOT=.
+COPY ${BUILD_ROOT}/scripts/add_docker_user.sh /add_docker_user.sh
+
+# Slurm daemons run as $SLURM_USER
+ARG SLURM_USER=slurm
+ARG USER=${SLURM_USER}
+ARG UID=1002
+RUN /add_docker_user.sh
+
+# Applications run as $USER
+ARG USER=slurmuser
+ARG UID=1001
+RUN /add_docker_user.sh
+
+ARG SLURM_VERSION=slurm-25-05-3-1
+COPY ${BUILD_ROOT}/scripts/build_slurm.sh /build_slurm.sh
+RUN /build_slurm.sh
+
+ARG MPICH_VERSION=4.2.2
+COPY ${BUILD_ROOT}/scripts/build_mpich.sh /build_mpich.sh
+RUN /build_mpich.sh
+
+COPY ${BUILD_ROOT}/conf/mpich.conf /etc/ld.so.conf.d/mpich.conf
+RUN ldconfig
+
+ENV SLURM_MPI_TYPE pmi2

containers/spindle-slurm-ubuntu/base/conf/mpich.conf

@@ -0,0 +1 @@
+/usr/lib

containers/spindle-slurm-ubuntu/base/scripts/add_docker_user.sh

@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+set -euxo pipefail
+
+sudo groupadd -g ${UID} ${USER}
+sudo useradd -g ${USER} -u ${UID} -d /home/${USER} -m ${USER}
+# Allow user to run as other users so that munge can be started as the munge user
+sudo sh -c "printf \"${USER} ALL=(ALL) NOPASSWD: ALL\\n\" >> /etc/sudoers"
+sudo adduser ${USER} sudo
+sudo usermod -s /bin/bash ${USER}
+

containers/spindle-slurm-ubuntu/base/scripts/build_mpich.sh

@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+set -euxo pipefail
+
+mkdir mpich
+pushd mpich
+wget -O - https://www.mpich.org/static/downloads/${MPICH_VERSION}/mpich-${MPICH_VERSION}.tar.gz | tar xvz --strip-components 1
+mkdir -p build
+pushd build
+../configure --prefix=/usr --disable-fortran --with-slurm=/usr/include/slurm 
+make -j$(nproc) install
+popd
+popd
+rm -rf mpich
+

containers/spindle-slurm-ubuntu/base/scripts/build_slurm.sh

@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+set -euxo pipefail
+
+export SLURM_SRC="/home/${SLURM_USER}/slurm"
+git clone -b ${SLURM_VERSION} --single-branch --depth=1 https://github.com/SchedMD/slurm.git ${SLURM_SRC}
+cd ${SLURM_SRC}
+./configure --enable-debug --prefix=/usr --sysconfdir=/etc/slurm --with-mysql_config=/usr/bin  --libdir=/usr/lib
+make -j$(nproc)
+make install
+

containers/spindle-slurm-ubuntu/testing/Dockerfile

@@ -0,0 +1,39 @@
+ARG BASE_VERSION=latest
+FROM ghcr.io/paratoolsinc/spindle-slurm-base:${BASE_VERSION}
+ARG replicas=4
+ENV workers=${replicas}
+
+ARG BUILD_ROOT=containers/spindle-slurm-ubuntu/testing
+
+# Slurm daemons run as $SLURM_USER
+ARG SLURM_USER=slurm
+
+# Applications run as $USER
+ARG USER=slurmuser
+ARG UID=1001
+
+# Set up the Slurm install already present in the base image
+COPY ${BUILD_ROOT}/scripts/setup_slurm.sh /setup_slurm.sh
+COPY ${BUILD_ROOT}/conf/slurm.conf /home/${SLURM_USER}/slurm.conf
+COPY ${BUILD_ROOT}/conf/slurmdbd.conf /home/${SLURM_USER}/slurmdbd.conf
+COPY ${BUILD_ROOT}/conf/cgroup.conf /home/${SLURM_USER}/cgroup.conf
+RUN /setup_slurm.sh
+
+# Slurm without Spank plugin needs passwordless ssh
+USER ${USER}
+WORKDIR /home/${USER}
+COPY ${BUILD_ROOT}/conf/ssh_config /home/${USER}/
+COPY ${BUILD_ROOT}/scripts/setup_ssh.sh /home/${USER}/
+RUN /home/${USER}/setup_ssh.sh
+
+# Copy the Spindle repo into the container and build it
+RUN mkdir -p /home/${USER}/Spindle
+COPY . /home/${USER}/Spindle
+COPY ${BUILD_ROOT}/scripts/build_spindle.sh /home/${USER}/build_spindle.sh
+RUN ./build_spindle.sh
+
+COPY ${BUILD_ROOT}/scripts/entrypoint.sh /home/${USER}/entrypoint.sh
+ENV PATH /home/${USER}/Spindle-inst/bin:$PATH
+
+ENTRYPOINT /bin/bash ./entrypoint.sh
+

containers/spindle-slurm-ubuntu/testing/conf/cgroup.conf

@@ -0,0 +1 @@
+CgroupPlugin=cgroup/v1

containers/spindle-slurm-ubuntu/testing/conf/slurm.conf

@@ -0,0 +1,42 @@
+ClusterName=linux
+ControlMachine=slurm-head
+ControlAddr=slurm-head
+SlurmUser=slurm
+SlurmctldPort=6817
+SlurmdPort=6818
+AuthType=auth/munge
+StateSaveLocation=/var/lib/slurmd
+SlurmdSpoolDir=/var/spool/slurmd
+SwitchType=switch/none
+MpiDefault=none
+SlurmctldPidFile=/var/run/slurmd/slurmctld.pid
+SlurmdPidFile=/var/run/slurmd/slurmd.pid
+ProctrackType=proctrack/linuxproc
+TaskPlugin=task/affinity
+ReturnToService=2
+SlurmctldTimeout=300
+SlurmdTimeout=300
+InactiveLimit=0
+MinJobAge=300
+KillWait=30
+Waittime=0
+SchedulerType=sched/backfill
+SelectType=select/cons_tres
+SelectTypeParameters=CR_Core_Memory
+SlurmctldDebug=3
+SlurmctldLogFile=/var/log/slurm/slurmctld.log
+SlurmdDebug=3
+SlurmdLogFile=/var/log/slurm/slurmd.log
+JobCompType=jobcomp/filetxt
+JobCompLoc=/var/log/slurm/jobcomp.log
+JobAcctGatherType=jobacct_gather/linux
+JobAcctGatherFrequency=30
+AccountingStorageType=accounting_storage/slurmdbd
+AccountingStorageHost=slurm-db
+AccountingStoragePort=6819
+NodeName=slurm-node-1 NodeAddr=slurm-node-1 CPUs=3 RealMemory=1000 State=UNKNOWN
+NodeName=slurm-node-2 NodeAddr=slurm-node-2 CPUs=3 RealMemory=1000 State=UNKNOWN
+NodeName=slurm-node-3 NodeAddr=slurm-node-3 CPUs=3 RealMemory=1000 State=UNKNOWN
+NodeName=slurm-node-4 NodeAddr=slurm-node-4 CPUs=3 RealMemory=1000 State=UNKNOWN
+PartitionName=debug Nodes=ALL Default=YES MaxTime=INFINITE State=UP
+

containers/spindle-slurm-ubuntu/testing/conf/slurmdbd.conf.template

@@ -0,0 +1,10 @@
+AuthType=auth/munge
+DbdAddr=slurm-db
+DbdHost=slurm-db
+SlurmUser=slurm
+DebugLevel=4
+LogFile=/var/log/slurm/slurmdbd.log
+PidFile=/var/run/slurmdbd/slurmdbd.pid
+StorageType=accounting_storage/mysql
+StorageHost=slurm-mariadb
+StorageUser=slurm

containers/spindle-slurm-ubuntu/testing/conf/ssh_config

@@ -0,0 +1,3 @@
+Host slurm-*
+    StrictHostKeyChecking no
+