Fixing upload issues

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 5.0.1
+current_version = 5.11.0
 commit = True
 tag = True
 

.dockerignore

@@ -6,4 +6,5 @@ data
 src/
 # ignore sentinel conf outputs
 *running.conf
-*.git
+*.git
+.venv/

.github/workflows/materialization_ci.yml

@@ -10,7 +10,8 @@ on:
     paths:
       - "materializationengine/**"
       - "tests/**"
-      - "requirements.txt"
+      - "uv.lock"
+      - "pyproject.toml"
       - ".github/workflows/**"
   pull_request:
     branches: master
@@ -37,35 +38,17 @@ jobs:
         options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5
     steps:
       - uses: actions/checkout@v4
-      - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v5
-        with:
-          python-version: ${{ matrix.python-version }}
-      - uses: actions/cache@v4
-        with:
-          path: ~/.cache/pip
-          key: ${{ runner.os }}-pip-${{ hashFiles('**/test_requirements.txt') }}
-          restore-keys: |
-            ${{ runner.os }}-pip-
-      - uses: conda-incubator/setup-miniconda@v3
+      - name: Install uv
+        uses: astral-sh/setup-uv@v4
         with:
-          auto-update-conda: true
-          auto-activate-base: true
-          python-version: ${{ matrix.python-version }}
+          version: "latest"
+      - name: Set up Python ${{ matrix.python-version }}
+        run: uv python install ${{ matrix.python-version }}
       - name: Install dependencies
-        shell: bash -l {0}
-        run: |
-          pip install flake8 pytest
-          pip install -r requirements.txt
-          if [ -f test_requirements.txt ]; then pip install -r test_requirements.txt; fi
-      - name: Lint with flake8
-        shell: bash -l {0}
+        run: uv sync --all-extras --dev
+      - name: Lint with ruff
         run: |
-          # stop the build if there are Python syntax errors or undefined names
-          # flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
-          # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
-          flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
+          uv run ruff check . --output-format=github
       - name: Test with pytest
-        shell: bash -l {0}
         run: |
-          pytest
+          uv run pytest

.github/workflows/release.yml

@@ -2,7 +2,6 @@
 name: publish release
 
 on:
-  # run this workflow when manually triggered
   workflow_dispatch:
     inputs:
       part:
@@ -21,6 +20,18 @@ on:
         type: boolean
         required: true
         default: false
+  workflow_call:
+    inputs:
+      part:
+        description: "Semver part to bump (major, minor, patch)"
+        type: string
+        required: false
+        default: "patch"
+      dry-run:
+        description: "Dry run"
+        type: boolean
+        required: false
+        default: false
 
 jobs:
   bump:
@@ -49,16 +60,27 @@ jobs:
           python-version: "3.11"
       - name: Install bumpversion
         run: pip install bumpversion
+      - name: Validate version consistency
+        run: |
+          pyproject_version=$(grep '^version = ' pyproject.toml | cut -d'"' -f2)
+          init_version=$(grep '^__version__ = ' materializationengine/__init__.py | cut -d'"' -f2)
+          bumpversion_version=$(grep '^current_version = ' pyproject.toml | cut -d'"' -f2)
+
+          echo "pyproject.toml version: $pyproject_version"
+          echo "__init__.py version: $init_version"  
+          echo "bumpversion current_version: $bumpversion_version"
+
+          if [ "$pyproject_version" != "$init_version" ] || [ "$pyproject_version" != "$bumpversion_version" ]; then
+            echo "❌ Version mismatch detected! All version files must be in sync before release."
+            exit 1
+          fi
+          echo "✅ All versions are in sync"
       - name: Bump version with bumpversion
         run: |
-          bumpversion ${{ github.event.inputs.part }}
-      - name: Commit and push with tags
-        if: ${{ github.event.inputs.dry-run == 'false' }}
-        run: git push --follow-tags
+          bumpversion ${{ inputs.part || github.event.inputs.part }}
       - name: Get version
         id: get-version
         run: |
-
           version="$(git describe --tags)"
           # remove the leading v from version
           version="${version:1}"
@@ -71,3 +93,74 @@ jobs:
           echo "SHORT_VERSION=$short_version" >> $GITHUB_OUTPUT
       - name: Show short version
         run: echo ${{ steps.get-version.outputs.SHORT_VERSION }}
+
+      - name: Commit and push with tags
+        if: ${{ (inputs.dry-run == false) || (github.event.inputs.dry-run == 'false') }}
+        run: |
+          git pull --rebase origin master
+          git push --follow-tags
+
+      - name: Create GitHub Release
+        if: ${{ (inputs.dry-run == false) || (github.event.inputs.dry-run == 'false') }}
+        uses: softprops/action-gh-release@v2
+        with:
+          tag_name: v${{ steps.get-version.outputs.VERSION }}
+          name: Release v${{ steps.get-version.outputs.VERSION }}
+          body: |
+            ## MaterializationEngine v${{ steps.get-version.outputs.VERSION }}
+
+            This release was automatically generated by the release workflow.
+
+            ### Changes
+            - Version bumped from previous release
+
+            See commit history for detailed changes.
+          draft: false
+          prerelease: false
+
+  update-chart:
+    name: Update materializationengine Helm chart
+    runs-on: ubuntu-latest
+    needs: bump
+    if: ${{ (inputs.dry-run == false) || (github.event.inputs.dry-run == 'false') }}
+    permissions:
+      contents: read
+      id-token: write
+    steps:
+      - name: Checkout MaterializationEngine repo
+        uses: actions/checkout@v4
+
+      - name: Checkout cave-helm-charts repo
+        uses: actions/checkout@v4
+        with:
+          repository: CAVEconnectome/cave-helm-charts
+          token: ${{ secrets.HELM_CHART_UPDATE_TOKEN }}
+          path: helm-charts
+
+      - name: Update Chart.yaml appVersion
+        id: update-chart
+        run: |
+          cd helm-charts
+          chart_file="charts/materializationengine/Chart.yaml"
+
+          # Update both appVersion and chart version to match application version
+          sed -i "s/^appVersion: .*/appVersion: \"${{ needs.bump.outputs.VERSION }}\"/" $chart_file
+          sed -i "s/^version: .*/version: ${{ needs.bump.outputs.VERSION }}/" $chart_file
+
+          echo "Updated chart version to: ${{ needs.bump.outputs.VERSION }}"
+          echo "Updated app version to: ${{ needs.bump.outputs.VERSION }}"
+          echo "CHART_VERSION=${{ needs.bump.outputs.VERSION }}" >> $GITHUB_OUTPUT
+
+      - name: Commit and push changes
+        run: |
+          cd helm-charts
+          git config --local user.email "github-actions[bot]@users.noreply.github.com"
+          git config --local user.name "github-actions[bot]"
+          git add charts/materializationengine/Chart.yaml
+          git commit -m "Update materializationengine to v${{ needs.bump.outputs.VERSION }}
+
+          - Update appVersion to ${{ needs.bump.outputs.VERSION }}
+          - Update chart version to ${{ needs.bump.outputs.VERSION }}
+
+          Auto-generated by MaterializationEngine release workflow"
+          git push

.github/workflows/update-dependency.yml

@@ -0,0 +1,126 @@
+name: Update Dependency
+
+on:
+  workflow_dispatch:
+    inputs:
+      dependency:
+        description: "Name of the dependency to update"
+        required: true
+        type: string
+      version:
+        description: "New version of the dependency"
+        required: true
+        type: string
+      bump_type:
+        description: "Type of version bump to perform after updating dependency"
+        type: choice
+        required: true
+        default: "minor"
+        options: ["major", "minor", "patch"]
+
+jobs:
+  update-dependency:
+    name: Update ${{ github.event.inputs.dependency }} dependency
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+      id-token: write
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v4
+
+      - name: Update dependency in pyproject.toml
+        run: |
+          echo "Updating ${{ github.event.inputs.dependency }} to version ${{ github.event.inputs.version }}"
+
+          # Update in pyproject.toml dependencies section
+          if [ -f "pyproject.toml" ]; then
+            # Handle both quoted and unquoted dependency specifications
+            sed -i "s/\"${{ github.event.inputs.dependency }}[^\"]*\"/\"${{ github.event.inputs.dependency }}>=${{ github.event.inputs.version }}\"/g" pyproject.toml
+            sed -i "s/'${{ github.event.inputs.dependency }}[^']*'/'${{ github.event.inputs.dependency }}>=${{ github.event.inputs.version }}'/g" pyproject.toml
+            echo "Updated pyproject.toml"
+          fi
+
+          # Update in requirements.txt if it exists
+          if [ -f "requirements.txt" ]; then
+            sed -i "s/^${{ github.event.inputs.dependency }}==.*/${{ github.event.inputs.dependency }}==${{ github.event.inputs.version }}/" requirements.txt
+            echo "Updated requirements.txt"
+          fi
+
+          # Update in requirements.in if it exists  
+          if [ -f "requirements.in" ]; then
+            sed -i "s/^${{ github.event.inputs.dependency }}.*/${{ github.event.inputs.dependency }}>=${{ github.event.inputs.version }}/" requirements.in
+            echo "Updated requirements.in"
+          fi
+
+      - name: Update uv.lock file
+        run: |
+          echo "Updating uv.lock file for ${{ github.event.inputs.dependency }}"
+
+          # Try targeted update first (faster, less disruptive)
+          for i in {1..3}; do
+            echo "Attempt $i/3: Running uv sync --no-upgrade (targeted update)..."
+            if uv sync --no-upgrade; then
+              echo "✅ Targeted uv sync succeeded on attempt $i"
+              echo "Only ${{ github.event.inputs.dependency }} was updated, other dependencies unchanged"
+              echo "UPDATE_TYPE=targeted" >> $GITHUB_ENV
+              break
+            else
+              echo "❌ Targeted uv sync failed on attempt $i (likely dependency conflicts)"
+              if [ $i -eq 3 ]; then
+                echo "Targeted updates failed, falling back to full dependency resolution..."
+
+                # Fallback to full sync (resolves conflicts but updates more deps)
+                for j in {1..3}; do
+                  echo "Fallback attempt $j/3: Running full uv sync..."
+                  if uv sync; then
+                    echo "✅ Full uv sync succeeded on attempt $j"
+                    echo "Dependency conflicts resolved, multiple packages may have been updated"
+                    echo "UPDATE_TYPE=full" >> $GITHUB_ENV
+                    break
+                  else
+                    echo "❌ Full uv sync failed on attempt $j"
+                    if [ $j -lt 3 ]; then
+                      echo "Waiting 30 seconds before retry..."
+                      sleep 30
+                    else
+                      echo "All sync attempts failed - dependency resolution impossible"
+                      exit 1
+                    fi
+                  fi
+                done
+                break
+              else
+                echo "Waiting 30 seconds before retry..."
+                sleep 30
+              fi
+            fi
+          done
+          echo "Updated uv.lock"
+
+      - name: Commit dependency update
+        run: |
+          git config --local user.email "github-actions[bot]@users.noreply.github.com"
+          git config --local user.name "github-actions[bot]"
+          git add -A
+          git diff --staged --quiet || git commit -m "Update ${{ github.event.inputs.dependency }} to v${{ github.event.inputs.version }}
+
+          Auto-generated dependency update triggered by ${{ github.event.inputs.dependency }} release"
+          git push
+
+  release:
+    name: Release with updated dependency
+    needs: update-dependency
+    uses: ./.github/workflows/release.yml
+    with:
+      part: ${{ github.event.inputs.bump_type }}
+      dry-run: false
+    secrets: inherit