ci: Extend for integration tests

.github/actions/test-template/action.yml

@@ -26,13 +26,45 @@ inputs:
     description: "Failure will cancel all other tests if set to true"
     required: false
     default: "false"
-  is_unit_test:
-    description: "Upload coverage as unit test"
+  type_of_test:
+    description: "Type of test to run"
     required: false
-    default: "false"
+    default: "unit_tests"
   package:
-    description: "Package to test"
-    required: true
+    description: "Package to test. Only required for unit and functional tests."
+    required: false
+  use-credentials:
+    description: "Use credentials for test"
+    required: false
+    default: "false"
+  ngc-api-user:
+    description: "NGC API user"
+    required: false
+    default: "false"
+  ngc-api-key:
+    description: "NGC API key"
+    required: false
+    default: "false"
+  cpu-only:
+    description: "CPU only"
+    required: false
+    default: "true"
+  has-azure-credentials:
+    description: "Has Azure credentials"
+    required: false
+    default: "false"
+  azure-client-id:
+    description: "Azure client ID"
+    required: false
+    default: "false"
+  azure-tenant-id:
+    description: "Azure tenant ID"
+    required: false
+    default: "false"
+  azure-subscription-id:
+    description: "Azure subscription ID"
+    required: false
+    default: "false"
 
 runs:
   using: "composite"
@@ -42,30 +74,139 @@ runs:
       with:
         path: Eval
 
-    - name: Build container
+    - name: Validate type of test
+      shell: bash
+      run: |
+        echo ::group::Validate type of test
+        if [[ "${{ inputs.type_of_test }}" != "unit_tests" && "${{ inputs.type_of_test }}" != "functional_tests" && "${{ inputs.type_of_test }}" != "integration_tests" ]]; then
+          echo "Invalid type of test: ${{ inputs.type_of_test }}"
+          exit 1
+        fi
+        echo ::endgroup::
+
+    - name: Use credentials
+      shell: bash
+      if: ${{ inputs.use-credentials == 'true' }}
+      run: |
+        echo ::group::Use credentials
+        echo ${{ inputs.ngc-api-key }} | docker login -u '${{ inputs.ngc-api-user }}' --password-stdin nvcr.io
+        echo ::endgroup::
+
+    - name: Build container for unit and functional tests
       shell: bash
+      if: ${{ inputs.type_of_test != 'integration_tests' }}
       run: |
         echo ::group::Build test container
-        docker build -f docker/Dockerfile.ci --build-arg PACKAGE=${{ inputs.package }} -t eval .
+        docker build -f docker/Dockerfile.ci \
+          --build-arg PACKAGE=${{ inputs.package }} \
+          -t eval .
         echo ::endgroup::
 
+    - name: Build container for integration tests
+      shell: bash
+      if: ${{ inputs.type_of_test == 'integration_tests' }}
+      run: |
+        echo ::group::Build test container
+        docker build -f docker/Dockerfile.ci-integration \
+          -t eval .
+        echo ::endgroup::
+
+    - name: Azure Login
+      if: ${{ inputs.has-azure-credentials == 'true' }}
+      uses: azure/login@v2
+      with:
+        client-id: ${{ inputs.azure-client-id }}
+        tenant-id: ${{ inputs.azure-tenant-id }}
+        subscription-id: ${{ inputs.azure-subscription-id }}
+
+    - name: Azure Fileshare
+      if: ${{ inputs.has-azure-credentials == 'true' && inputs.type_of_test != 'unit-tests' }}
+      shell: bash
+      id: azure-fileshare
+      run: |
+        echo "::group::Mount SMB drive"
+        sudo apt update
+        sudo apt install -y cifs-utils
+
+        RESOURCE_GROUP_NAME="azure-gpu-vm-runner_group"
+        STORAGE_ACCOUNT_NAME="nemocistorageaccount2"
+        FILE_SHARE_NAME="fileshare"
+
+        MNT_PATH="/mnt/datadrive"
+
+        sudo mkdir -p $MNT_PATH
+
+        # Create a folder to store the credentials for this storage account and
+        # any other that you might set up.
+        CREDENTIAL_ROOT="/etc/smbcredentials"
+        sudo mkdir -p "/etc/smbcredentials"
+
+        # Get the storage account key for the indicated storage account.
+        # You must be logged in with az login and your user identity must have
+        # permissions to list the storage account keys for this command to work.
+        STORAGE_ACCOUNT_KEY=$(az storage account keys list \
+            --resource-group $RESOURCE_GROUP_NAME \
+            --account-name $STORAGE_ACCOUNT_NAME \
+            --query "[0].value" --output tsv | tr -d '"')
+
+        # Create the credential file for this individual storage account
+        SMB_CREDENTIAL_FILE="$CREDENTIAL_ROOT/$STORAGE_ACCOUNT_NAME.cred"
+        if [ ! -f $SMB_CREDENTIAL_FILE ]; then
+            echo "username=$STORAGE_ACCOUNT_NAME" | sudo tee $SMB_CREDENTIAL_FILE > /dev/null
+            echo "password=$STORAGE_ACCOUNT_KEY" | sudo tee -a $SMB_CREDENTIAL_FILE > /dev/null
+        else
+            echo "The credential file $SMB_CREDENTIAL_FILE already exists, and was not modified."
+        fi
+
+        # Change permissions on the credential file so only root can read or modify the password file.
+        sudo chmod 600 $SMB_CREDENTIAL_FILE
+
+        # This command assumes you have logged in with az login
+        HTTP_ENDPOINT=$(az storage account show --resource-group $RESOURCE_GROUP_NAME --name $STORAGE_ACCOUNT_NAME --query "primaryEndpoints.file" --output tsv | tr -d '"')
+        SMB_PATH=$(echo $HTTP_ENDPOINT | cut -c7-${#HTTP_ENDPOINT})$FILE_SHARE_NAME
+
+        STORAGE_ACCOUNT_KEY=$(az storage account keys list --resource-group $RESOURCE_GROUP_NAME --account-name $STORAGE_ACCOUNT_NAME --query "[0].value" --output tsv | tr -d '"')
+
+        sudo mount -t cifs $SMB_PATH $MNT_PATH -o credentials=$SMB_CREDENTIAL_FILE,serverino,nosharesock,actimeo=30,mfsymlinks,fsc,cache=strict
+
+        ls -al $MNT_PATH/TestData
+        echo "::endgroup::"
+
     - name: Start container
       shell: bash
+      env:
+        MOUNT_FS: ${{ inputs.type_of_test == 'integration_tests' || inputs.type_of_test == 'functional_tests' }}
       run: |
-        echo ::group::Start test container
+        echo "::group::Start test container"
+        if [[ "$MOUNT_FS" == "true" ]]; then
+          ls -al /mnt/datadrive/TestData
+          VOLUME_ARGS="--volume /mnt/datadrive/TestData:/home/TestData"
+        else
+          VOLUME_ARGS=""
+        fi
+
+        ARG=("")
+        if [[ "${{ inputs.type_of_test }}" == "integration_tests" ]]; then
+          ARG=("--runtime=nvidia --gpus all")
+        fi
+
         cmd=$(cat <<RUN_TEST_EOF
         #!/bin/bash
         docker container rm -f nemo_container_${{ github.run_id }} || true
+
         docker run \
           --rm \
           -d \
           --name nemo_container_${{ github.run_id }} \
+          ${ARG[@]} \
           --shm-size=64g \
-          --env TRANSFORMERS_OFFLINE=0 \
+          --env TRANSFORMERS_OFFLINE=1 \
           --env HYDRA_FULL_ERROR=1 \
+          --env HF_HUB_CACHE_PATH=/home/TestData/HF_HOME/hub \
           --env HF_HOME=/home/TestData/HF_HOME \
           --env RUN_ID=${{ github.run_id }} \
           --volume $(pwd)/Eval:/workspace \
+          $VOLUME_ARGS \
           eval \
           bash -c "sleep $(( ${{ inputs.timeout }} * 60 + 60 ))"
         RUN_TEST_EOF
@@ -80,16 +221,20 @@ runs:
       shell: bash
       run: |
         echo ::group::Create run-script
-        COVERAGE_PREFIX=$([[ "${{ inputs.is_unit_test }}" == "true" ]] && echo "unit-test" || echo "e2e")
+        COVERAGE_PREFIX=$([[ "${{ inputs.type_of_test }}" == "unit_tests" ]] && echo "unit-test" || echo "e2e")
         echo "coverage-prefix=$COVERAGE_PREFIX" | tee -a "$GITHUB_OUTPUT"
 
         cmd=$(cat <<'RUN_TEST_EOF'
         #!/bin/bash
 
         docker exec -t nemo_container_${{ github.run_id }} bash -c '
           set -e
-          cd packages/${{ inputs.package }}
-          bash tests/${{ inputs.is_unit_test == 'true' && 'unit_tests' || 'functional_tests' }}/${{ inputs.script }}.sh
+          if [[ "${{ inputs.type_of_test }}" != "integration_tests" ]]; then
+            cd packages/${{ inputs.package }}
+            bash tests/${{ inputs.type_of_test == 'unit_tests' && 'unit_tests' || 'functional_tests' }}/${{ inputs.script }}.sh
+          else
+            bash tests/integration_tests/Launch_Integration_Tests.sh
+          fi
         '
 
         RUN_TEST_EOF

.github/workflows/cicd-main.yml

@@ -30,8 +30,19 @@ permissions:
   contents: read
 
 jobs:
+  # pre-flight:
+  #   uses: NVIDIA-NeMo/FW-CI-templates/.github/workflows/[email protected]
+
   pre-flight:
-    uses: NVIDIA-NeMo/FW-CI-templates/.github/workflows/[email protected]
+    runs-on: ubuntu-latest
+    outputs:
+      is_ci_workload: "true"
+      force_run_all: "false"
+      docs_only: "false"
+      is_deployment_workflow: "false"
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
 
   linting:
     runs-on: ubuntu-latest
@@ -95,7 +106,7 @@ jobs:
         with:
           script: Launch_Unit_Tests
           timeout: 10
-          is_unit_test: "true"
+          type_of_test: "unit_tests"
           cpu-only: true
           package: nemo-evaluator
 
@@ -119,7 +130,7 @@ jobs:
         with:
           script: Launch_Unit_Tests
           timeout: 10
-          is_unit_test: "true"
+          type_of_test: "unit_tests"
           cpu-only: true
           package: nemo-evaluator-launcher
 
@@ -143,16 +154,44 @@ jobs:
         with:
           script: Launch_Functional_Tests
           timeout: ${{ matrix.timeout || 10 }}
-          is_unit_test: "false"
+          type_of_test: "functional_tests"
           cpu-only: true
           package: nemo-evaluator
+          has-azure-credentials: true
+          azure-client-id: ${{ secrets.AZURE_CLIENT_ID }}
+          azure-tenant-id: ${{ secrets.AZURE_TENANT_ID }}
+          azure-subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
+
+  cicd-integration-tests:
+    needs:
+      - cicd-unit-tests-nemo-evaluator
+    environment: main
+    runs-on: self-hosted-azure
+    if: |
+      needs.pre-flight.outputs.is_ci_workload == 'true'
+      && !cancelled()
+    name: integration-tests
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      - name: main
+        uses: ./.github/actions/test-template
+        with:
+          script: Launch_Integration_Tests
+          timeout: 60
+          type_of_test: "integration_tests"
+          cpu-only: false
+          use-credentials: true
+          ngc-api-user: ${{ secrets.NGC_API_USER }}
+          ngc-api-key: ${{ secrets.NGC_API_KEY }}
 
   Nemo_CICD_Test:
     needs:
       - pre-flight
       - cicd-unit-tests-nemo-evaluator
       - cicd-unit-tests-nemo-evaluator-launcher
       - cicd-e2e-tests-nemo-evaluator
+      - cicd-integration-tests
     if: |
       (
         needs.pre-flight.outputs.docs_only == 'true'
@@ -173,10 +212,14 @@ jobs:
         env:
           GH_TOKEN: ${{ github.token }}
           RUN_ID: ${{ github.run_id }}
+          IS_CI_WORKLOAD: ${{ needs.pre-flight.outputs.is_ci_workload == 'true' }}
           SKIPPING_IS_ALLOWED: ${{ needs.pre-flight.outputs.docs_only == 'true' || needs.pre-flight.outputs.is_deployment_workflow == 'true' || needs.pre-flight.outputs.is_ci_workload == 'true' }}
         run: |
-          FAILED_JOBS=$(gh run view $GITHUB_RUN_ID --json jobs --jq '[.jobs[] | select(.status == "completed" and .conclusion != "success")] | length') || echo 0
-
+          if [ "$IS_CI_WORKLOAD" == "true" ]; then
+            FAILED_JOBS=$(gh run view $GITHUB_RUN_ID --json jobs --jq '[.jobs[] | select(.status == "completed" and .conclusion != "success")] | length') || echo 0
+          else
+            FAILED_JOBS=$(gh run view $GITHUB_RUN_ID --json jobs --jq '[.jobs[] | select(.status == "completed" and .conclusion != "success" and .name != "integration-tests")] | length') || echo 0
+          fi
           if [ "${FAILED_JOBS:-0}" -eq 0 ] || [ "$SKIPPING_IS_ALLOWED" == "true" ]; then
               echo "✅ All previous jobs completed successfully"
               exit 0

docker/Dockerfile.ci-integration

@@ -0,0 +1,17 @@
+# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+FROM nvcr.io/nvidian/nemo:nightly
+COPY . /opt/Evaluator/
+RUN cd /opt/Evaluator/packages/nemo-evaluator && uv pip install -e .

packages/nemo-evaluator/pyproject.toml

@@ -65,7 +65,7 @@ repository = "https://github.com/NVIDIA-NeMo/Evaluator/packages/nemo-evaluator"
 # END(if-changed)
 
 [dependency-groups]
-test = ["pytest", "pytest-cov", "pytest-subtests", "pytest-httpserver"]
+test = ["pytest", "pytest-cov", "pytest-subtests", "pytest-httpserver", "nvidia-simple-evals"]
 
 docs = [
     "sphinx",

packages/nemo-evaluator/tests/functional_tests/Launch_Functional_Tests.sh

@@ -14,6 +14,10 @@
 set -xeuo pipefail # Exit immediately if a command exits with a non-zero status
 
 export CUDA_VISIBLE_DEVICES=""
+export HF_HOME="/home/TestData/HF_HOME"
+export HF_DATASETS_OFFLINE="1"
+export TRANSFORMERS_OFFLINE="1"
+export HF_DATASETS_CACHE="${HF_HOME}/datasets"
 
 SCRIPT_DIR=$(dirname "$0")
 PROJECT_DIR=$SCRIPT_DIR/../../

tests/integration_tests/eval_factory/conftest.py → packages/nemo-evaluator/tests/functional_tests/e2e/conftest.py

@@ -23,6 +23,7 @@
 import pytest
 import requests
 from flask import Flask, jsonify, request
+
 from nemo_evaluator.logging.utils import logger
 
 

tests/integration_tests/eval_factory/test_eval_factory_fake_endpoint.py → packages/nemo-evaluator/tests/functional_tests/e2e/test_eval_factory_fake_endpoint.py

@@ -15,6 +15,7 @@
 """End-to-end tests for eval-factory using fake endpoint."""
 
 import pytest
+
 from nemo_evaluator.logging.utils import logger