benchmark.yml

name: Tests

on:
  workflow_dispatch:
    inputs:
      ref:
        description: Commit ref to checkout
        required: true
        default: main
        type: string

jobs:
  Unit-Tests:
    name: unit-tests
    runs-on: depot-ubuntu-24.04
    steps:
      - uses: actions/checkout@v4
        with:
          repository: weaviate/weaviate
          ref: ${{ inputs.ref }}
      - name: Set up Go
        uses: actions/setup-go@v5
        with:
          go-version: "1.22"
          cache: true
      - name: Unit test
        run: ./test/run.sh --unit-only
      - name: Archive code coverage results
        uses: actions/upload-artifact@v4
        with:
          name: coverage-report-unit
          path: coverage-unit.txt

  Integration-Tests:
    name: integration-tests
    runs-on: depot-ubuntu-24.04
    steps:
      - uses: actions/checkout@v4
        with:
          repository: weaviate/weaviate
          ref: ${{ inputs.ref }}
      - name: Set up Go
        uses: actions/setup-go@v5
        with:
          go-version: "1.22"
          cache: true
      - name: Integration test
        run: ./test/run.sh --integration-only
      - name: Archive code coverage results
        uses: actions/upload-artifact@v4
        with:
          name: coverage-report-integration
          path: coverage-integration.txt

  Modules-Acceptance-Tests:
    name: modules-acceptance-tests
    runs-on: depot-ubuntu-24.04
    strategy:
      fail-fast: false
      matrix:
        test:
          [
            "--only-module-backup-azure",
            "--only-module-backup-filesystem",
            "--only-module-backup-gcs",
            "--only-module-backup-s3",
            "--only-module-offload-s3",
            "--only-module-img2vec-neural",
            "--only-module-many-modules",
            "--only-module-many-generative",
            "--only-module-ref2vec-centroid",
            "--only-module-text2vec-contextionary",
            "--only-module-text2vec-transformers",
            "--only-module-text2vec-ollama",
            "--only-module-generative-ollama",
            "--only-module-many-generative",
          ]
    steps:
      - uses: actions/checkout@v4
        with:
          repository: weaviate/weaviate
          ref: ${{ inputs.ref }}
      - name: Set up Go
        uses: actions/setup-go@v5
        with:
          go-version: "1.22"
          cache: true
      - name: Acceptance tests (modules)
        uses: nick-fields/retry@v3
        with:
          # 15 Minute is a large enough timeout for most of our tests
          timeout_minutes: 15
          max_attempts: 3
          command: ./test/run.sh ${{ matrix.test }}
          on_retry_command: ./test/run.sh --cleanup

  # Modules-Acceptance-Tests-large:
  #   name: modules-acceptance-tests-large
  #   runs-on: depot-ubuntu-24.04-4
  #   strategy:
  #     fail-fast: false
  #     matrix:
  #       test:
  #         [
  #           "--only-module-qna-transformers",
  #           "--only-module-sum-transformers",
  #           "--only-module-multi2vec-clip",
  #           "--only-module-reranker-transformers",
  #         ]
  #   steps:
  #     - uses: actions/checkout@v4
  #     - name: Set up Go
  #       uses: actions/setup-go@v5
  #       with:
  #         go-version: "1.22"
  #         cache: true
  #     - name: Acceptance tests Large (modules)
  #       uses: nick-fields/retry@v3
  #       with:
  #         # 15 Minute is a large enough timeout for most of our tests
  #         timeout_minutes: 15
  #         max_attempts: 3
  #         command: ./test/run.sh ${{ matrix.test }}
  #         on_retry_command: ./test/run.sh --cleanup

  # Modules-Acceptance-Tests-api:
  #   name: modules-acceptance-tests-api
  #   runs-on: depot-ubuntu-24.04
  #   strategy:
  #     fail-fast: false
  #     matrix:
  #       test:
  #         [
  #           "--only-module-multi2vec-cohere",
  #           "--only-module-multi2vec-google",
  #           "--only-module-generative-aws",
  #           "--only-module-generative-cohere",
  #           "--only-module-generative-google",
  #           "--only-module-generative-openai",
  #           "--only-module-text2vec-google",
  #           "--only-module-text2vec-aws",
  #           "--only-module-text2vec-jinaai",
  #           "--only-module-multi2vec-jinaai",
  #           "--only-module-text2colbert-jinaai",
  #         ]
  #   steps:
  #     - uses: actions/checkout@v4
  #       with:
  #         fetch-depth: 2
  #     - name: Set up Go
  #       uses: actions/setup-go@v5
  #       with:
  #         go-version: "1.22"
  #         cache: true
  #     - name: check
  #       env:
  #         COMMIT_SHA: ${{ github.event.pull_request.head.sha }}
  #       run: |
  #         commit_message=$(git log -1 --format=%B $COMMIT_SHA)
  #         if [[ "$commit_message" == *"[api]"* || "$commit_message" == "prepare release"* ]]; then
  #           echo "Run pipeline"
  #           echo "run_pipeline=true" >> $GITHUB_ENV
  #         else
  #           echo "Skip pipeline. In order to run the pipeline commit title must contain: [api]"
  #           echo "run_pipeline=false" >> $GITHUB_ENV
  #         fi
  #     - name: configure gcp
  #       if: ${{ env.run_pipeline == 'true' || startsWith(github.ref, 'refs/tags') }}
  #       id: creds-gcp
  #       env:
  #         GPG_PASSPHRASE: ${{ secrets.GPG_PASSPHRASE }}
  #         GCP_PROJECT: ${{ secrets.GCP_PROJECT }}
  #       run: |
  #         ./tools/ci/gcloud.sh
  #         google_apikey=$(gcloud auth print-access-token)
  #         echo "::add-mask::$google_apikey"
  #         echo "google_apikey=$google_apikey" >> "$GITHUB_OUTPUT"
  #     - name: configure aws
  #       if: ${{ env.run_pipeline == 'true' || startsWith(github.ref, 'refs/tags') }}
  #       id: creds-aws
  #       uses: aws-actions/configure-aws-credentials@v4
  #       with:
  #         aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
  #         aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
  #         aws-region: ${{ secrets.AWS_REGION }}
  #         role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }}
  #         role-external-id: ${{ secrets.AWS_ROLE_EXTERNAL_ID }}
  #         role-skip-session-tagging: true
  #         output-credentials: true
  #     - name: Acceptance tests (modules)
  #       if: ${{ env.run_pipeline == 'true' || startsWith(github.ref, 'refs/tags') }}
  #       env:
  #         GCP_PROJECT: ${{ secrets.GCP_PROJECT }}
  #         GOOGLE_APIKEY: ${{ steps.creds-gcp.outputs.google_apikey }}
  #         AWS_REGION: ${{ secrets.AWS_REGION }}
  #         AWS_ACCESS_KEY_ID: ${{ steps.creds-aws.outputs.aws-access-key-id }}
  #         AWS_SECRET_ACCESS_KEY: ${{ steps.creds-aws.outputs.aws-secret-access-key }}
  #         AWS_SESSION_TOKEN: ${{ steps.creds-aws.outputs.aws-session-token }}
  #         OPENAI_APIKEY: ${{ secrets.OPENAI_APIKEY }}
  #         OPENAI_ORGANIZATION: ${{ secrets.OPENAI_ORGANIZATION }}
  #         COHERE_APIKEY: ${{ secrets.COHERE_APIKEY }}
  #         ANTHROPIC_APIKEY: ${{ secrets.ANTHROPIC_APIKEY }}
  #         JINAAI_APIKEY: ${{ secrets.JINAAI_APIKEY }}
  #       run: ./test/run.sh ${{ matrix.test }}

  Acceptance-Tests:
    name: acceptance-tests
    runs-on: depot-ubuntu-24.04
    strategy:
      fail-fast: false
      matrix:
        test:
          [
            "--acceptance-only-fast",
            "--acceptance-only-graphql",
            "--acceptance-only-authz",
            "--acceptance-only-replication",
            "--acceptance-only-async-replication",
            "--acceptance-go-client-only-fast",
            "--acceptance-only-python",
          ]
    steps:
      - uses: actions/checkout@v4
        with:
          repository: weaviate/weaviate
          ref: ${{ inputs.ref }}
      - name: Set up Go
        uses: actions/setup-go@v5
        with:
          go-version: "1.22"
          cache: true
      - name: Acceptance tests
        env:
          WCS_DUMMY_CI_PW: ${{ secrets.WCS_DUMMY_CI_PW || 'pw1' }}
          WCS_DUMMY_CI_PW_2: ${{ secrets.WCS_DUMMY_CI_PW_2 || 'pw2' }}
        run: ./test/run.sh ${{ matrix.test }}

  Acceptance-Tests-large:
    name: acceptance-tests-large
    runs-on: depot-ubuntu-24.04-4
    strategy:
      fail-fast: false
      matrix:
        test:
          [
            "--acceptance-go-client-named-vectors-single-node",
            "--acceptance-go-client-named-vectors-cluster",
            "--acceptance-lsmkv",
          ]
    steps:
      - uses: actions/checkout@v4
        with:
          repository: weaviate/weaviate
          ref: ${{ inputs.ref }}
      - name: Set up Go
        uses: actions/setup-go@v5
        with:
          go-version: "1.22"
          cache: true
      - name: Determine retry max_attempts value
        env:
          MATRIX_TEST: ${{ matrix.test }}
        run: |
          if [[ "$MATRIX_TEST" == *"lsmkv"* ]]; then
            echo "retry_max_attempts=2" >> $GITHUB_ENV
          else
            echo "retry_max_attempts=1" >> $GITHUB_ENV
          fi
      - name: Acceptance tests Large
        uses: nick-fields/retry@v3
        with:
          # 15 Minute is a large enough timeout for most of our tests
          timeout_minutes: 15
          max_attempts: ${{ env.retry_max_attempts }}
          command: ./test/run.sh ${{ matrix.test }}
          on_retry_command: ./test/run.sh --cleanup

  Compile-and-upload-binaries:
    name: compile-and-upload-binaries
    runs-on: depot-ubuntu-24.04-4
    steps:
      - uses: actions/checkout@v4
        with:
          repository: weaviate/weaviate
          ref: ${{ inputs.ref }}
      - name: Set up Go
        uses: actions/setup-go@v5
        with:
          go-version: "1.22"
          cache: true
      - name: Install GoReleaser
        uses: goreleaser/goreleaser-action@v6
        with:
          install-only: true
      - name: goreleaser
        run: |
          GIT_REVISION=$(git rev-parse --short HEAD) GIT_BRANCH=$(git rev-parse --abbrev-ref HEAD) BUILD_USER=ci BUILD_DATE=$(date -u +"%Y-%m-%dT%H:%M:%SZ") goreleaser build  --clean --snapshot
      - name: Upload macos
        uses: actions/upload-artifact@v4
        with:
          name: binaries-macos-unsigned
          path: dist/weaviate_darwin_all
      - name: Upload windows
        uses: actions/upload-artifact@v4
        with:
          name: binaries-windows-amd64
          path: dist/weaviate_windows_amd64_v1
      - name: Upload windows
        uses: actions/upload-artifact@v4
        with:
          name: binaries-windows-arm64
          path: dist/weaviate_windows_arm64
      - name: Upload linux amd64
        uses: actions/upload-artifact@v4
        with:
          name: binaries-linux-amd64
          path: dist/weaviate_linux_amd64_v1
      - name: Upload linux arm64
        uses: actions/upload-artifact@v4
        with:
          name: binaries-linux-arm64
          path: dist/weaviate_linux_arm64

  # Acceptance-Tests-windows:
  #   name: acceptance-tests-windows
  #   needs: Compile-and-upload-binaries
  #   runs-on: windows-latest
  #   env:
  #     AUTHENTICATION_ANONYMOUS_ACCESS_ENABLED: true
  #     PERSISTENCE_DATA_PATH: /tmp
  #     QUERY_DEFAULTS_LIMIT: 20
  #     CLUSTER_HOSTNAME: node1
  #     RAFT_BOOTSTRAP_EXPECT: 1
  #     RAFT_JOIN: node1
  #     GRPC_PORT: 50052
  #   steps:
  #     - uses: actions/checkout@v4
  #     - name: Download binaries
  #       uses: actions/download-artifact@v4
  #       with:
  #         name: binaries-windows-amd64
  #     - name: Set up Go
  #       uses: actions/setup-go@v5
  #       with:
  #         go-version: "1.22"
  #         cache: true
  #     - name: start weaviate
  #       shell: bash
  #       # Weaviate is started without a Vectorizer as running text2vec-contextionary on GH actions is difficult:
  #       # - docker on GHA only supports windows container - which we currently are not build
  #       # - building those containers without a windows machine is difficult to figure out
  #       run: ./weaviate.exe --scheme http --port 8080 &
  #     - name: run acceptance tests
  #       shell: bash
  #       run: go test -count 1 -race test/acceptance/actions/*.go # tests that don't need a Vectorizer

  # Push-Docker:
  #   if: ${{ !github.event.pull_request.head.repo.fork && !startsWith(github.head_ref, 'build') && github.triggering_actor != 'dependabot[bot]' }} # no PRs from fork
  #   needs: [Unit-Tests, Integration-Tests]
  #   name: push-docker
  #   timeout-minutes: 30
  #   runs-on: depot-ubuntu-24.04-8-cores
  #   steps:
  #     - uses: actions/checkout@v4
  #     - name: Login to Docker Hub
  #       uses: docker/login-action@v3
  #       with:
  #         username: ${{secrets.DOCKER_USERNAME}}
  #         password: ${{secrets.DOCKER_PASSWORD}}
  #     - name: Push container
  #       id: push-container
  #       run: ./ci/push_docker.sh
  #       env:
  #         PR_TITLE: "${{ github.event.pull_request.title }}"
  #     - name: Generate Report
  #       env:
  #         PREVIEW_TAG: "${{ steps.push-container.outputs.PREVIEW_TAG }}"
  #         PREVIEW_SEMVER_TAG: "${{ steps.push-container.outputs.PREVIEW_SEMVER_TAG }}"
  #       run: ./ci/generate_docker_report.sh
  # Push-Docker-Fast:
  #   if: ${{ !github.event.pull_request.head.repo.fork && startsWith(github.head_ref, 'build') && github.triggering_actor != 'dependabot[bot]' }} # no PRs from fork
  #   name: push-docker-fast
  #   timeout-minutes: 30
  #   runs-on: depot-ubuntu-24.04-8-cores
  #   steps:
  #     - uses: actions/checkout@v4
  #     - name: Login to Docker Hub
  #       uses: docker/login-action@v3
  #       with:
  #         username: ${{secrets.DOCKER_USERNAME}}
  #         password: ${{secrets.DOCKER_PASSWORD}}
  #     - name: Push container
  #       id: push-container
  #       run: ./ci/push_docker.sh
  #       env:
  #         PR_TITLE: "${{ github.event.pull_request.title }}"
  #     - name: Generate Report
  #       env:
  #         PREVIEW_TAG: "${{ steps.push-container.outputs.PREVIEW_TAG }}"
  #         PREVIEW_SEMVER_TAG: "${{ steps.push-container.outputs.PREVIEW_SEMVER_TAG }}"
  #       run: ./ci/generate_docker_report.sh