improve local dev for server side development

Procfile

@@ -0,0 +1,4 @@
+minikube-mount: minikube start; minikube mount $LOCAL_DIR/servicex_app:/mnt/servicex & sleep 5 && cd $CHART_DIR && helm install -f $VALUES_FILE servicex . && sleep infinity
+port-forward-app: cd $LOCAL_DIR && bash local/port-forward.sh app
+port-forward-minio: cd $LOCAL_DIR && bash local/port-forward.sh minio
+port-forward-db: cd $LOCAL_DIR && bash local/port-forward.sh db

helm/servicex/templates/app/deployment.yaml

@@ -37,6 +37,11 @@ spec:
       containers:
       - name: {{ .Release.Name }}-servicex-app
         image: {{ .Values.app.image }}:{{ .Values.app.tag }}
+        {{- if eq .Values.app.environment "dev" }}
+        {{- if .Values.app.reload }}
+        command: [ "./boot.sh", "--reload" ]
+        {{- end }}
+        {{- end }}
         env:
         - name: APP_CONFIG_FILE
           value: "/opt/servicex/app.conf"
@@ -135,6 +140,12 @@ spec:
         {{- end }}
 
         volumeMounts:
+          {{- if eq .Values.app.environment "dev" }}
+          {{- if .Values.app.mount_local }}
+          - name: host-volume
+            mountPath: /home/servicex
+          {{- end }}
+          {{- end }}
           - name: app-cfg
             mountPath: /opt/servicex
           - name: sqlite
@@ -149,6 +160,14 @@ spec:
           - containerPort: 5000
 
       volumes:
+        {{- if eq .Values.app.environment "dev" }}
+        {{- if .Values.app.mount_local }}
+        - name: host-volume
+          hostPath:
+          - path: /mnt/servicex
+          - type: DirectoryOrCreate
+        {{- end }}
+        {{- end }}
         - name: app-cfg
           configMap:
             name:  {{ .Release.Name }}-flask-config

helm/servicex/values.yaml

@@ -1,4 +1,5 @@
 app:
+  environment: production
   adminEmail: [email protected]
   auth: false
   authExpires: 21600

local/port-forward.sh

@@ -0,0 +1,254 @@
+#!/bin/bash
+
+# Constants
+NAMESPACE="default"
+PING_INTERVAL=2  # seconds
+MAX_HELM_RETRIES=12  # Maximum number of times to check for helm installation
+
+# Configuration based on service type
+if [ "$1" = "app" ]; then
+    APP_LABEL="servicex-servicex-app"
+    LOCAL_PORT=5000
+    CONTAINER_PORT=5000
+    PING_URL="http://localhost:${LOCAL_PORT}/servicex"
+elif [ "$1" = "minio" ]; then
+    APP_LABEL="minio"
+    LOCAL_PORT=9000
+    CONTAINER_PORT=9000
+    # Don't use direct HTTP check for Minio as it might require auth
+    PING_URL=""
+elif [ "$1" = "db" ]; then
+    APP_LABEL="postgresql"
+    LOCAL_PORT=5432
+    CONTAINER_PORT=5432
+    PING_URL=""  # No HTTP endpoint to check for DB
+    POD_NAME="servicex-postgresql-0"  # Fixed pod name for DB
+else
+    echo "Usage: $0 [app|minio|db]"
+    echo "  app   - Port forward to ServiceX app (5000:5000)"
+    echo "  minio - Port forward to Minio (9000:9000)"
+    echo "  db    - Port forward to PostgreSQL (5432:5432)"
+    exit 1
+fi
+
+# Function to log with timestamp
+log() {
+    echo "[$(date '+%Y-%m-%d %H:%M:%S')] $1"
+}
+
+# Function to check if servicex helm installation exists
+check_helm_installation() {
+    log "Checking if 'servicex' helm installation exists"
+    if helm list | grep -q servicex; then
+        log "Found 'servicex' helm installation"
+        return 0
+    else
+        log "No 'servicex' helm installation found"
+        return 1
+    fi
+}
+
+# Function to wait for helm installation
+wait_for_helm_installation() {
+    log "Waiting for 'servicex' helm installation"
+    local retries=0
+
+    while ! check_helm_installation && [ $retries -lt $MAX_HELM_RETRIES ]; do
+        retries=$((retries+1))
+        log "Waiting for 'servicex' helm installation... ($retries/$MAX_HELM_RETRIES)"
+        sleep 5
+    done
+
+    if [ $retries -ge $MAX_HELM_RETRIES ]; then
+        log "Timed out waiting for 'servicex' helm installation"
+        return 1
+    fi
+
+    return 0
+}
+
+# Function to get pod name
+get_pod_name() {
+    # Skip for DB since we already know the pod name
+    if [ "$1" = "db" ]; then
+        return 0
+    fi
+
+    log "Getting pod name for $APP_LABEL"
+
+    if [ "$1" = "app" ]; then
+        POD_NAME=$(kubectl get pods --namespace $NAMESPACE -l "app=servicex-servicex-app" -o jsonpath="{.items[0].metadata.name}" 2>/dev/null)
+    elif [ "$1" = "minio" ]; then
+        # Try multiple approaches to find the minio pod without detailed logging
+        POD_NAME=$(kubectl get pods --namespace $NAMESPACE -l "app=minio,release=servicex" -o jsonpath="{.items[0].metadata.name}" 2>/dev/null)
+
+        if [ -z "$POD_NAME" ]; then
+            POD_NAME=$(kubectl get pods --namespace $NAMESPACE -l "app=minio" -o jsonpath="{.items[0].metadata.name}" 2>/dev/null)
+        fi
+
+        if [ -z "$POD_NAME" ]; then
+            POD_NAME=$(kubectl get pods --namespace $NAMESPACE | grep -i "servicex-minio" | awk '{print $1}' | head -1)
+        fi
+
+        if [ -z "$POD_NAME" ]; then
+            POD_NAME=$(kubectl get pods --namespace $NAMESPACE | grep -i "minio" | awk '{print $1}' | head -1)
+        fi
+
+        if [ -z "$POD_NAME" ]; then
+            log "Error: Could not find any minio pods."
+            return 1
+        fi
+    fi
+
+    if [ -z "$POD_NAME" ]; then
+        log "Error: Could not find pod for $APP_LABEL"
+        return 1
+    fi
+
+    log "Found pod: ${POD_NAME}"
+    return 0
+}
+
+# Function to start port forwarding
+start_port_forward() {
+    if [ "$1" = "db" ]; then
+        log "Starting port forwarding from localhost:${LOCAL_PORT} to servicex-postgresql-0:${CONTAINER_PORT}"
+        kubectl port-forward --namespace $NAMESPACE servicex-postgresql-0 ${LOCAL_PORT}:${CONTAINER_PORT} &
+    else
+        log "Starting port forwarding from localhost:${LOCAL_PORT} to ${POD_NAME}:${CONTAINER_PORT}"
+        kubectl port-forward --namespace $NAMESPACE $POD_NAME ${LOCAL_PORT}:${CONTAINER_PORT} &
+    fi
+
+    PORT_FORWARD_PID=$!
+
+    # Give it a moment to establish connection
+    sleep 2
+
+    # Check if port forward is successful
+    if ! ps -p $PORT_FORWARD_PID > /dev/null; then
+        log "Error: Port forwarding failed to start"
+        return 1
+    fi
+
+    log "Port forwarding established with PID: ${PORT_FORWARD_PID}"
+    return 0
+}
+
+# Function to check if the port forwarding is working
+check_port_forward() {
+    # For services without HTTP endpoints or with auth requirements like Minio
+    if [ -z "$PING_URL" ]; then
+        # Check if the port is listening
+        if command -v nc >/dev/null 2>&1; then
+            if nc -z localhost ${LOCAL_PORT} >/dev/null 2>&1; then
+                log "Port forward is working - port ${LOCAL_PORT} is open"
+                return 0
+            else
+                log "Connection failed - port ${LOCAL_PORT} is not open"
+                return 1
+            fi
+        else
+            # If nc is not available, try lsof
+            if command -v lsof >/dev/null 2>&1; then
+                if lsof -i:${LOCAL_PORT} >/dev/null 2>&1; then
+                    log "Port forward is working - port ${LOCAL_PORT} is open"
+                    return 0
+                else
+                    log "Connection failed - port ${LOCAL_PORT} is not open"
+                    return 1
+                fi
+            else
+                # If neither nc nor lsof is available, just check if the process is running
+                if ps -p $PORT_FORWARD_PID > /dev/null; then
+                    log "Port forward process is still running"
+                    return 0
+                else
+                    log "Port forward process is not running"
+                    return 1
+                fi
+            fi
+        fi
+    else
+        # For services with HTTP endpoints
+        local response_code
+        # Use curl with a short timeout to prevent long waits
+        response_code=$(curl -s -o /dev/null -w "%{http_code}" --max-time 3 "${PING_URL}" || echo "failed")
+
+        if [[ "$response_code" =~ ^(200|302|301|303|307|308|403)$ ]]; then
+            # Any of these HTTP response codes indicates the port forward is working
+            log "Port forward is working - received HTTP ${response_code}"
+            return 0
+        elif [[ "$response_code" == "failed" || "$response_code" == "0" || "$response_code" == "000" ]]; then
+            log "Connection failed - no response from service. Will retry."
+            return 1
+        else
+            log "Received unexpected status code: ${response_code}"
+            # Check if we should consider this a success
+            if [[ "$response_code" =~ ^[1-5][0-9][0-9]$ ]]; then
+                log "But got a valid HTTP response, so port forward is working"
+                return 0
+            else
+                log "Invalid response - port forward may not be working correctly"
+                return 1
+            fi
+        fi
+    fi
+}
+
+# Function to clean up resources
+cleanup() {
+    log "Cleaning up resources..."
+    if [ ! -z "$PORT_FORWARD_PID" ]; then
+        log "Killing port forwarding process (PID: ${PORT_FORWARD_PID})"
+        kill $PORT_FORWARD_PID 2>/dev/null || true
+    fi
+    log "Cleanup complete, exiting"
+    exit 0
+}
+
+# Set trap for cleanup
+trap cleanup SIGINT SIGTERM
+
+# Main loop
+log "Starting port forwarding monitor script for $1"
+
+# First, wait for the helm installation
+if ! wait_for_helm_installation; then
+    log "Failed to find servicex helm installation after multiple attempts"
+    log "Please check if the servicex helm chart is properly installed"
+    exit 1
+fi
+
+while true; do
+    # Get pod name
+    if ! get_pod_name "$1"; then
+        log "Retrying in 5 seconds..."
+        sleep 5
+        continue
+    fi
+
+    # Start port forwarding
+    if ! start_port_forward "$1"; then
+        log "Retrying in 5 seconds..."
+        sleep 5
+        continue
+    fi
+
+    # Monitor port forwarding
+    while true; do
+        if ! check_port_forward "$1"; then
+            log "Port forward appears to be down, restarting..."
+
+            # Kill existing port forward process if it exists
+            if [ ! -z "$PORT_FORWARD_PID" ]; then
+                kill $PORT_FORWARD_PID 2>/dev/null || true
+                unset PORT_FORWARD_PID
+            fi
+
+            break
+        fi
+
+        # Wait before checking again
+        sleep $PING_INTERVAL
+    done
+done

servicex_app/boot.sh

@@ -1,4 +1,17 @@
 #!/bin/sh
+
+# Initialize reload flag
+RELOAD=""
+
+# Parse command line arguments
+for arg in "$@"
+do
+    if [ "$arg" = "--reload" ]; then
+        RELOAD="--reload"
+        break
+    fi
+done
+
 mkdir instance
 # SQLite doesn't handle migrations, so rely on SQLAlchmy table creation
 if grep "sqlite://" $APP_CONFIG_FILE; then
@@ -7,5 +20,5 @@ else
   FLASK_APP=servicex_app/app.py flask db upgrade;
 fi
 [ -d "/default_users" ] && python3 servicex/cli/create_default_users.py
-exec gunicorn -b [::]:5000 --workers=5 --threads=1 --timeout 120 --log-level=warning --access-logfile /tmp/gunicorn.log --error-logfile - "servicex_app:create_app()"
+exec gunicorn -b [::]:5000 $RELOAD --workers=5 --threads=1 --timeout 120 --log-level=warning --access-logfile /tmp/gunicorn.log --error-logfile - "servicex_app:create_app()"
 # to log requests to stdout  --access-logfile -