DigitalSlideArchive · BryonLewis · Apr 9, 2025 · Apr 18, 2025 · Apr 18, 2025 · Apr 22, 2025
diff --git a/.github/workflows/publish-pypi.yml b/.github/workflows/publish-pypi.yml
@@ -0,0 +1,59 @@
+name: Build and Publish Python Package
+
+on:
+  push:
+    tags:
+      - 'v*'  # Only run on version tags like v0.1.0, v1.2.3, etc.
+
+permissions:
+  id-token: write  # <-- ADD THIS FOR OIDC
+  contents: read
+
+jobs:
+  build-and-publish:
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Checkout source code
+      uses: actions/checkout@v4
+
+    - name: Set up Node.js (for web clients)
+      uses: actions/setup-node@v4
+      with:
+        node-version: '20'
+
+    - name: Set up Python
+      uses: actions/setup-python@v5
+      with:
+        python-version: '3.11'
+
+    - name: Install Python build tools
+      run: |
+        python -m pip install --upgrade pip
+        pip install build twine
+
+    - name: Build web clients
+      run: |
+        cd server/dive_server/web_client
+        npm install
+        npm run build
+        cd ../../../../
+
+        cd client
+        yarn install
+        yarn build:web
+        rm -rf ../server/dive_server/dive_client/*
+        cp -r dist/* ../server/dive_server/dive_client/
+        cd ..
+
+    - name: Build Python package
+      working-directory: server
+      run: |
+        python -m build
+
+    - name: Publish to PyPI using OIDC
+      working-directory: server
+      env:
+        TWINE_USERNAME: __token__
+      run: |
+        twine upload dist/*
diff --git a/client/.env.production b/client/.env.production
@@ -1 +1 @@
-VUE_APP_STATIC_PATH=./static/dive/
+VUE_APP_STATIC_PATH=/dive/static
diff --git a/client/vue.config.js b/client/vue.config.js
@@ -7,7 +7,7 @@ const keepAliveAgent = new http.Agent({ keepAlive: true });
 
 process.env.VUE_APP_GIT_HASH = gitDescribeSync().hash;
 process.env.VUE_APP_VERSION = packagejson.version;
-const staticPath = process.env.NODE_ENV === 'production' ? process.env.VUE_APP_STATIC_PATH || './static/dive' : './';
+const staticPath = process.env.NODE_ENV === 'production' ? process.env.VUE_APP_STATIC_PATH || './static/dive' : '/dive/';
 function chainWebpack(config) {
   config.output.strictModuleExceptionHandling(true);
   config.resolve.symlinks(false);

diff --git a/client/yarn.lock b/client/yarn.lock
@@ -11588,4 +11588,4 @@ yorkie@^2.0.0:
     execa "^0.8.0"
     is-ci "^1.0.10"
     normalize-path "^1.0.0"
-    strip-indent "^2.0.0"
+    strip-indent "^2.0.0"
diff --git a/dive-dsa-slicer/example-docker-containers/DIVEUINotifications/DIVEUINotifications.py b/dive-dsa-slicer/example-docker-containers/DIVEUINotifications/DIVEUINotifications.py
@@ -9,7 +9,6 @@
 
 from ctk_cli import CLIArgumentParser  # noqa I004
 # imported for side effects
-from slicer_cli_web import ctk_cli_adjustment  # noqa
 
 
 logging.basicConfig(level=logging.CRITICAL)

diff --git a/docker-compose.override.yml b/docker-compose.override.yml
@@ -4,7 +4,9 @@ services:
     volumes:
       - ./server:/opt/dive/src
       - ./docker/server_setup.py:/server_setup.py
-    command: ["--dev"]
+    command: ["--mode", "development"]
+    environment:
+      - DEVELOPMENT_MODE=True
 
   girder_worker_default:
     volumes:

diff --git a/docker-compose.yml b/docker-compose.yml
@@ -7,7 +7,7 @@ x-worker: &base-worker
   build:
     context: .
     dockerfile: docker/girder_worker.Dockerfile
-  image:  ghcr.io/digitalslidearchive/dive-dsa/dive-dsa-worker:${TAG:-latest}
+  image:  ghcr.io/digitalslidearchive/dive-dsa/dive-dsa-worker:${TAG:-girder-5}
   volumes:
     - addons:/tmp/addons:ro # readonly
     - /var/run/docker.sock:/var/run/docker.sock
@@ -54,7 +54,7 @@ services:
     build:
       context: .
       dockerfile: docker/girder.Dockerfile
-    image: ghcr.io/digitalslidearchive/dive-dsa/dive-dsa-web:${TAG:-latest}
+    image: ghcr.io/digitalslidearchive/dive-dsa/dive-dsa-web:${TAG:-girder-5}
     init: true
     command: ["--mode", "production"]
     depends_on:
@@ -73,8 +73,9 @@ services:
       - "GIRDER_MONGO_URI=mongodb://mongo:27017/girder"
       - "GIRDER_ADMIN_USER=${GIRDER_ADMIN_USER:-admin}"
       - "GIRDER_ADMIN_PASS=${GIRDER_ADMIN_PASS:-letmein}"
-      - "CELERY_BROKER_URL=${CELERY_BROKER_URL:-amqp://guest:guest@rabbit/default}"
-      - "WORKER_API_URL=${WORKER_API_URL:-http://girder:8080/api/v1}"
+      - "GIRDER_WORKER_BROKER=${GIRDER_WORKER_BROKER:-amqp://guest:guest@rabbit/default}"
+      - "GIRDER_WORKER_BACKEND=${GIRDER_WORKER_BACKEND:-rpc://guest:guest@localhost/}"
+      - "GIRDER_SETTING_WORKER_API_URL=${GIRDER_SETTING_WORKER_API_URL}:-http://localhost:8080/api/v1"
       # Rabbitmq management variables
       - "RABBITMQ_MANAGEMENT_USERNAME=${RABBITMQ_MANAGEMENT_USERNAME:-guest}"
       - "RABBITMQ_MANAGEMENT_PASSWORD=${RABBITMQ_MANAGEMENT_PASSWORD:-guest}"
@@ -101,10 +102,13 @@ services:
       - ${TMPDIR:-/tmp}:${TMPDIR:-/tmp}
     environment:
       - "DSA_USER=${DSA_USER:-}"
-      - "WORKER_WATCHING_QUEUES=celery"
+      - "WORKER_WATCHING_QUEUES=celery, local"
       - "WORKER_CONCURRENCY=${DEFAULT_WORKER_CONCURRENCY:-3}"
-      - "CELERY_BROKER_URL=${CELERY_BROKER_URL:-amqp://guest:guest@rabbit/default}"
+      - "GIRDER_WORKER_BROKER=${GIRDER_WORKER_BROKER:-amqp://guest:guest@rabbit/default}"
+      - "GIRDER_WORKER_BACKEND=${GIRDER_WORKER_BACKEND:-rpc://guest:guest@localhost/}"
+      - "GIRDER_SETTING_WORKER_API_URL=${GIRDER_SETTING_WORKER_API_URL}:-http://localhost:8080/api/v1"
       - "TMPDIR="
+
 volumes:
   mongo_db:
   girder_assetstore:
diff --git a/docker/entrypoint_server.sh b/docker/entrypoint_server.sh
@@ -19,6 +19,14 @@ adduser $(id -nu ${DSA_USER%%:*}) $(getent group $(stat -c "%g" /var/run/docker.
 # OSX where the users don't translate
 chmod 777 /var/run/docker.sock 2>/dev/null || true
 
+if [[ "$DEVELOPMENT_MODE" == "True" ]]; then
+  echo "Development mode is enabled, syncing client builds..."
+  echo "Syncing Dive client..."
+  cp -r /opt/dive/clients/dive/ /opt/dive/src/dive_server/dive_client/
+  echo "Syncing Girder client..."
+  cp -r /opt/dive/clients/girder/ /opt/dive/src/dive_server/web_client/dist/
+fi
+
 
 set -e
 python /server_setup.py

diff --git a/docker/entrypoint_worker.sh b/docker/entrypoint_worker.sh
@@ -5,7 +5,7 @@
 # WORKER_WATCHING_QUEUES: The comma separated list of queues to take tasks from. If not supplied, the celery default of 'celery' is used.
 
 QUEUE_ARGUMENT=
-CONCURRENCY_ARGUMENT=
+CONCURRENCY_ARGUMENT=()
 
 if [[ -z "$DSA_USER" ]]; then
   echo "Set the DSA_USER before starting (e.g, DSA_USER=\$$(id -u):\$$(id -g) <up command>"
@@ -28,14 +28,21 @@ chmod 777 /var/run/docker.sock 2>/dev/null || true
 chmod 777 ${TMPDIR:-/tmp} || true
 
 if [ -n "$WORKER_WATCHING_QUEUES" ]; then
-    QUEUE_ARGUMENT="--queues $WORKER_WATCHING_QUEUES"
+    QUEUE_ARGUMENT="-Q '$WORKER_WATCHING_QUEUES'"
 fi
 
 if [ -n "$WORKER_CONCURRENCY" ]; then
     CONCURRENCY_ARGUMENT="--concurrency $WORKER_CONCURRENCY"
 fi
 
-su $(id -nu ${DSA_USER%%:*}) -c "GW_DIRECT_PATHS=true python -m dive_tasks -l info --without-gossip --without-mingle $QUEUE_ARGUMENT $CONCURRENCY_ARGUMENT -Ofair --prefetch-multiplier=1"
+su $(id -nu ${DSA_USER%%:*}) -c "GW_DIRECT_PATHS=true python -m dive_tasks \
+  -l info \
+  --without-gossip \
+  --without-mingle \
+  $QUEUE_ARGUMENT \
+  ${CONCURRENCY_ARGUMENT[@]} \
+  -Ofair \
+  --prefetch-multiplier=1"
 
 # su $(id -nu ${DSA_USER%%:*}) -c "GW_DIRECT_PATHS=true python -m girder_worker $CONCURRENCY_ARGUMENT -Ofair --prefetch-multiplier=1"
 # exec python \

diff --git a/docker/girder.Dockerfile b/docker/girder.Dockerfile
@@ -2,73 +2,86 @@
 # == CLIENT BUILD STAGE ==
 # ========================
 FROM node:20 as client-builder
+
 WORKDIR /app
 
 # Install dependencies
 COPY client/package.json client/yarn.lock /app/
 RUN yarn install --frozen-lockfile --network-timeout 300000
-# Build
+
+# Copy client source and git
 COPY .git/ /app/.git/
 COPY client/ /app/
+
+# Build client
 RUN yarn build:web
 
 # ========================
 # == SERVER BUILD STAGE ==
 # ========================
-# Note: server-builder stage will be the same in both dockerfiles
+FROM node:20 as server-node
+
+WORKDIR /opt/dive/src/dive_server/web_client
+
+# Install dependencies and build the girder client
+COPY server/dive_server/web_client/package.json server/dive_server/web_client/package-lock.json ./
+RUN npm install
+COPY server/dive_server/web_client/ ./
+RUN npm run build
+
+# ========================
+# == PYTHON SERVER BUILD ==
+# ========================
 FROM python:3.11-buster as server-builder
 
 WORKDIR /opt/dive/src
 
-# https://cryptography.io/en/latest/installation/#debian-ubuntu
-RUN apt-get update
-RUN apt-get install -y build-essential libssl-dev libffi-dev python3-dev cargo npm
-# Recommended poetry install https://python-poetry.org/docs/master/#installation
+# Install system dependencies
+RUN apt-get update && apt-get install -y \
+    build-essential libssl-dev libffi-dev python3-dev cargo
+
+# Install Poetry
 RUN curl -sSL https://install.python-poetry.org | POETRY_VERSION=1.7.0 POETRY_HOME=/opt/dive/poetry python -
 ENV PATH="/opt/dive/poetry/bin:$PATH"
-# Create a virtual environment for the installation
+
+# Setup Python virtual environment
 RUN python -m venv /opt/dive/local/venv
-# Poetry needs this set to recognize it as ane existing environment
 ENV VIRTUAL_ENV="/opt/dive/local/venv"
 ENV PATH="/opt/dive/local/venv/bin:$PATH"
 
-# Copy only the lock and project files to optimize cache
+# Copy project files
 COPY server/pyproject.toml /opt/dive/src/
 COPY .git/ /opt/dive/src/.git/
 RUN poetry env use system
 RUN poetry config virtualenvs.create false
-# Install dependencies only
 RUN poetry install --no-root
-# Build girder client, including plugins like worker/jobs
-# Copy full source code and install
+
+# Copy server source
 COPY server/ /opt/dive/src/
 RUN poetry install --only main
-RUN curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.39.5/install.sh | bash
-RUN . ~/.bashrc && \
-    nvm install 14 && \
-    nvm alias default 14 && \
-    nvm use default && \
-    ln -s $(dirname `which npm`) /usr/local/node
-
-ENV PATH="/usr/local/node:$PATH"
 
-RUN girder build
+# Copy girder client (from node build stage)
+COPY --from=server-node /opt/dive/src/dive_server/web_client/dist /opt/dive/clients/girder
 
 # =================
-# == DIST SERVER ==
+# == FINAL SERVER ==
 # =================
 FROM python:3.11-slim-buster as server
 
-# Hack: Tell GitPython to be quiet, we aren't using git
 ENV GIT_PYTHON_REFRESH="quiet"
 ENV PATH="/opt/dive/local/venv/bin:$PATH"
 
-# Copy site packages and executables
+# Copy Python venv and server code
 COPY --from=server-builder /opt/dive/local/venv /opt/dive/local/venv
-# Copy the source code of the editable module
 COPY --from=server-builder /opt/dive/src /opt/dive/src
-# Copy the client code into the static source location
-COPY --from=client-builder /app/dist/ /opt/dive/local/venv/share/girder/static/dive/
+
+# Copy built girder client
+COPY --from=server-builder /opt/dive/clients/girder /opt/dive/clients/girder
+
+# Copy DIVE VUE client
+COPY --from=client-builder /app/dist/ /opt/dive/clients/dive
+COPY --from=client-builder /app/dist/ /opt/dive/src/dive_server/dive_client
+
 # Install startup scripts
 COPY docker/entrypoint_server.sh docker/server_setup.py /
 

diff --git a/docker/server_setup.py b/docker/server_setup.py
@@ -10,8 +10,6 @@
 from girder.models.user import User
 from pymongo.errors import OperationFailure
 
-cherrypy.config["database"]["uri"] = os.getenv("GIRDER_MONGO_URI")
-
 ADMIN_USER = os.getenv("GIRDER_ADMIN_USER", "admin")
 ADMIN_PASS = os.getenv("GIRDER_ADMIN_PASS", "letmein")
 

diff --git a/docs/Deployment-Docker-Compose.md b/docs/Deployment-Docker-Compose.md
@@ -87,7 +87,7 @@ It's possible to split your web server and task runner between multiple nodes.
 
 * Make two cloud VM instances, one with NVIDIA drivers and container toolkit, and one without.
 * Clone the dive repository on both, and set up `.env` on both with the same configuration.
-* Be sure that `WORKER_API_URL` and `CELERY_BROKER_URL` in particular are uncommented and set to the IP or domain name of your web server.  This is how the worker will talk to the web server, so the web server must be network accessible from the worker.
+* Be sure that `GIRDER_WORKER_BACKEND` and `GIRDER_WORKER_BROKER` in particular are uncommented and set to the IP or domain name of your web server.  This is how the worker will talk to the web server, so the web server must be network accessible from the worker.
 
 ``` bash
 ## On the web server
@@ -141,16 +141,16 @@ This image contains both the backend and client.
 | GIRDER_MONGO_URI | `mongodb://mongo:27017/girder` | a mongodb connection string |
 | GIRDER_ADMIN_USER | `admin` | admin username |
 | GIRDER_ADMIN_PASS | `letmein` | admin password |
-| CELERY_BROKER_URL | `amqp://guest:guest@default/` | rabbitmq connection string |
-| WORKER_API_URL | `http://girder:8080/api/v1` | Address for workers to reach web server |
+| GIRDER_WORKER_BROKER | `amqp://guest:guest@default/` | rabbitmq connection string |
+| GIRDER_WORKER_BACKEND | `http://girder:8080/api/v1` | Address for workers to reach web server |
 
 There is additional configuration for the RabbitMQ Management plugin. It only matters if you intend to allow individual users to configure private job runners in standalone mode, and can otherwise be ignored.
 
 | Variable | Default | Description |
 |----------|---------|-------------|
 | RABBITMQ_MANAGEMENT_USERNAME | `guest` | Management API username |
 | RABBITMQ_MANAGEMENT_PASSWORD | `guest` | Management API password |
-| RABBITMQ_MANAGEMENT_VHOST | `default` | Virtual host should match `CELERY_BROKER_URL` |
+| RABBITMQ_MANAGEMENT_VHOST | `default` | Virtual host should match `GIRDER_WORKER_BROKER` |
 | RABBITMQ_MANAGEMENT_URL | `http://rabbit:15672/` | Management API Url |
 
 You can also pass [girder configuration](https://girder.readthedocs.io/en/latest/) and [celery configuration](https://docs.celeryproject.org/en/stable/userguide/configuration.html#std-setting-broker_connection_timeout).
@@ -166,7 +166,7 @@ This image contains a celery worker to run transcoding jobs.
 | WORKER_WATCHING_QUEUES | null | one of `celery`.  Ignored in standalone mode. |
 | WORKER_CONCURRENCY | `# of CPU cores` | max concurrnet jobs. **Lower this if you run training** |
 | WORKER_GPU_UUID | null | leave empty to use all GPUs.  Specify UUID to use specific device |
-| CELERY_BROKER_URL | `amqp://guest:guest@default/` | rabbitmq connection string. Ignored in standalone mode. |
+| GIRDER_WORKER_BROKER | `amqp://guest:guest@default/` | rabbitmq connection string. Ignored in standalone mode. |
 | KWIVER_DEFAULT_LOG_LEVEL | `warn` | kwiver log level |
 | DIVE_USERNAME | null | Username to start private queue processor. Providing this enables standalone mode. |
 | DIVE_PASSWORD | null | Password for private queue processor. Providing this enables standalone mode. |

diff --git a/server/dive_server/.gitignore b/server/dive_server/.gitignore
@@ -0,0 +1 @@
+/dive_server
Original file line number	Diff line number	Diff line change
		@@ -1 +1 @@
		VUE_APP_STATIC_PATH=./static/dive/
		VUE_APP_STATIC_PATH=/dive/static