Push agent v2 logs to QE grafana

.github/actions/start-promtail/action.yml

@@ -0,0 +1,30 @@
+name: Start Promtail
+description: Start promtail in a Docker container to send test results to loki
+inputs:
+    loki-dashboard-url:
+        description: "URL of the Loki dashboard to send the logs"
+        required: true
+runs:
+    using: "composite"
+    steps:
+        - name: Start Promtail container
+          shell: bash
+          run: |
+              docker run -d \
+               --name=promtail \
+               -v "${{ github.workspace }}/test/dashboard/prep/promtail.yaml:/etc/promtail/config.yaml" \
+               -v "${{ github.workspace }}/test/dashboard/logs:/var/log" \
+               -e TEST_OUTDIR=test/dashboard/logs \
+               -e GITHUB_RUN_ID=${{ github.run_id }} \
+               -e GITHUB_JOB=${{ github.job }} \
+               -e GITHUB_WORKFLOW=${{ github.workflow }} \
+               -e GITHUB_EVENT_NAME=${{ github.event_name }} \
+               -e GITHUB_SERVER_URL=${{ github.server_url }} \
+               -e GITHUB_REPOSITORY=${{ github.repository }} \
+               -e GITHUB_HEAD_REF=${{ github.head_ref }} \
+               -e GITHUB_SHA=${{ github.sha }} \
+               -e GITHUB_ACTOR=${{ github.actor }} \
+               -e LOKI_DASHBOARD_URL=${{ inputs.loki-dashboard-url }} \
+               grafana/promtail:3.4.4 \
+               -config.file=/etc/promtail/config.yaml \
+               -config.expand-env=true

.github/workflows/ci.yml

@@ -57,8 +57,19 @@ jobs:
       - uses: actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491 # v5.0.0
         with:
           go-version-file: 'go.mod'
+      - name: Set Start Time
+        run: echo "START_TIME=$(date +"%Y-%m-%dT%H:%M:%S.%NZ")" >> ${GITHUB_ENV}
+      - name: Create Directory
+        run: mkdir -p ${{github.workspace}}/test/dashboard/logs/${{github.job}}/
+      - name: Start Promtail
+        uses: ./.github/actions/start-promtail
+        with:
+            loki-dashboard-url: ${{ secrets.LOKI_DASHBOARD_URL }}
       - name: Run Unit Tests
-        run: make unit-test
+        run: make unit-test | tee ${{github.workspace}}/test/dashboard/logs/${{github.job}}/raw_logs.log && exit "${PIPESTATUS[0]}"
+      - name: Generate Test Results
+        if: always()
+        run: bash ./scripts/workflow/generate_results.sh ${{job.status}} ${{env.START_TIME}} ${{github.job}} ${{github.workspace}}
       - name: Upload Test Coverage
         uses: codecov/codecov-action@84508663e988701840491b86de86b666e8a86bed # v4.3.0
         with:
@@ -161,18 +172,29 @@ jobs:
             with:
                 name: nginx-agent-unsigned-snapshots
                 path: build
+          - name: Set Start Time
+            run: echo "START_TIME=$(date +"%Y-%m-%dT%H:%M:%S.%NZ")" >> ${GITHUB_ENV}
+          - name: Create Directory
+            run: mkdir -p ${{github.workspace}}/test/dashboard/logs/${{github.job}}/${{matrix.container.image}}${{matrix.container.version}}/
+          - name: Start Promtail
+            uses: ./.github/actions/start-promtail
+            with:
+                loki-dashboard-url: ${{ secrets.LOKI_DASHBOARD_URL }}
           - name: Run Integration Tests
             run: |
                 go install github.com/goreleaser/nfpm/v2/cmd/nfpm@${{ env.NFPM_VERSION }}
                 OS_RELEASE="${{ matrix.container.image }}" OS_VERSION="${{ matrix.container.version }}" \
-                  make integration-test
+                  make integration-test | tee ${{github.workspace}}/test/dashboard/logs/${{github.job}}/${{matrix.container.image}}${{matrix.container.version}}/raw_logs.log && exit "${PIPESTATUS[0]}"
+          - name: Generate Test Results
+            if: always()
+            run: bash ./scripts/workflow/generate_results.sh ${{job.status}} ${{env.START_TIME}} ${{github.job}}/${{matrix.container.image}}${{matrix.container.version}} ${{github.workspace}}
           - name: Container Output Logs
             if: failure()
             run: |
                 docker ps -a
                 dockerid=$(docker ps -a --format "{{.ID}}")
                 docker logs "$dockerid"
-
+                
           - name: Archive integration test logs
             if: success() || failure()
             uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3
@@ -214,11 +236,22 @@ jobs:
         with:
           name: nginx-agent-unsigned-snapshots
           path: build
+      - name: Set Start Time
+        run: echo "START_TIME=$(date +"%Y-%m-%dT%H:%M:%S.%NZ")" >> ${GITHUB_ENV}
+      - name: Create Directory
+        run: mkdir -p ${{github.workspace}}/test/dashboard/logs/${{github.job}}/${{matrix.container.image}}${{matrix.container.version}}/
+      - name: Start Promtail
+        uses: ./.github/actions/start-promtail
+        with:
+            loki-dashboard-url: ${{ secrets.LOKI_DASHBOARD_URL }}
       - name: Run Integration Tests
         run: |
           go install github.com/goreleaser/nfpm/v2/cmd/nfpm@${{ env.NFPM_VERSION }}
           CONTAINER_NGINX_IMAGE_REGISTRY="${{ env.NGINX_OSS_REGISTRY }}" TAG="${{ matrix.container.version }}-${{ matrix.container.image }}" OS_RELEASE="${{ matrix.container.release }}"\
-            make official-image-integration-test
+            make official-image-integration-test | tee ${{github.workspace}}/test/dashboard/logs/${{github.job}}/${{matrix.container.image}}${{matrix.container.version}}/raw_logs.log && exit "${PIPESTATUS[0]}"
+      - name: Generate Test Results
+        if: always()
+        run: bash ./scripts/workflow/generate_results.sh ${{job.status}} ${{env.START_TIME}} ${{github.job}}/${{matrix.container.image}}${{matrix.container.version}} ${{github.workspace}}
       - name: Container Output Logs 
         if: failure()
         run: |
@@ -281,12 +314,23 @@ jobs:
                 registry: ${{ secrets.REGISTRY_URL }}
                 username: ${{ secrets.REGISTRY_USERNAME }}
                 password: ${{ secrets.REGISTRY_PASSWORD }}
+          - name: Set Start Time
+            run: echo "START_TIME=$(date +"%Y-%m-%dT%H:%M:%S.%NZ")" >> ${GITHUB_ENV}
+          - name: Create Directory
+            run: mkdir -p ${{github.workspace}}/test/dashboard/logs/${{github.job}}/${{matrix.container.image}}${{matrix.container.version}}/
+          - name: Start Promtail
+            uses: ./.github/actions/start-promtail
+            with:
+                loki-dashboard-url: ${{ secrets.LOKI_DASHBOARD_URL }}
           - name: Run Integration Tests
             run: |
                 go install github.com/goreleaser/nfpm/v2/cmd/nfpm@${{ env.NFPM_VERSION }}
                 CONTAINER_NGINX_IMAGE_REGISTRY="${{ secrets.REGISTRY_URL }}" TAG="${{ matrix.container.plus }}-${{ matrix.container.image }}-${{ matrix.container.version }}" \
                 OS_RELEASE="${{ matrix.container.release }}" IMAGE_PATH="${{ matrix.container.path }}" \
-                  make official-image-integration-test
+                  make official-image-integration-test | tee ${{github.workspace}}/test/dashboard/logs/${{github.job}}/${{matrix.container.image}}${{matrix.container.version}}/raw_logs.log && exit "${PIPESTATUS[0]}"
+          - name: Generate Test Results
+            if: always()
+            run: bash ./scripts/workflow/generate_results.sh ${{job.status}} ${{env.START_TIME}} ${{github.job}}/${{matrix.container.image}}${{matrix.container.version}} ${{github.workspace}}
           - name: Container Output Logs
             if: failure()
             run: |

Makefile

@@ -193,16 +193,16 @@ unit-test: $(TEST_BUILD_DIR) test-core test-plugins test-sdk test-extensions ##
 	@printf "\nTotal code coverage: " && $(GOTOOL) cover -func=$(TEST_BUILD_DIR)/coverage.out | grep 'total:' | awk '{print $$3}'
 
 test-core: $(TEST_BUILD_DIR) ## Run core unit tests
-	GOWORK=off CGO_ENABLED=0 go test -count=1 -coverprofile=$(TEST_BUILD_DIR)/core_coverage.out -covermode count ./src/core/...
+	GOWORK=off CGO_ENABLED=0 go test -v -count=1 -coverprofile=$(TEST_BUILD_DIR)/core_coverage.out -covermode count ./src/core/...
 
 test-plugins: $(TEST_BUILD_DIR) ## Run plugins unit tests
-	GOWORK=off CGO_ENABLED=0 go test -count=1 -coverprofile=$(TEST_BUILD_DIR)/plugins_coverage.out -covermode count ./src/plugins/...
+	GOWORK=off CGO_ENABLED=0 go test -v -count=1 -coverprofile=$(TEST_BUILD_DIR)/plugins_coverage.out -covermode count ./src/plugins/...
 
 test-extensions: $(TEST_BUILD_DIR) ## Run extensions unit tests
-	GOWORK=off CGO_ENABLED=0 go test -count=1 -coverprofile=$(TEST_BUILD_DIR)/extensions_coverage.out -covermode count ./src/extensions/...
+	GOWORK=off CGO_ENABLED=0 go test -v -count=1 -coverprofile=$(TEST_BUILD_DIR)/extensions_coverage.out -covermode count ./src/extensions/...
 
 test-sdk: $(TEST_BUILD_DIR) ## Run sdk unit tests from root directory
-	cd sdk && GOWORK=off CGO_ENABLED=0 go test -count=1 -coverprofile=../$(TEST_BUILD_DIR)/sdk_coverage.out -covermode count ./...
+	cd sdk && GOWORK=off CGO_ENABLED=0 go test -v -count=1 -coverprofile=../$(TEST_BUILD_DIR)/sdk_coverage.out -covermode count ./...
 
 # Component tests
 component-test: test-component-build test-component-run ## Run component tests

scripts/workflow/generate_results.sh

@@ -0,0 +1,102 @@
+#!/bin/bash
+
+JOB_RESULT="$1"
+START_TIME="$2"
+TEST_TYPE="$3"
+WORKSPACE="$4"
+
+INPUT_FILE="$WORKSPACE/test/dashboard/logs/$TEST_TYPE/raw_logs.log"
+OUTPUT_PATH="$WORKSPACE/test/dashboard/logs/$TEST_TYPE"
+JOB_OUTPUT_FILE="$WORKSPACE/test/dashboard/logs/$TEST_TYPE/result.json"
+
+END_TIME="`date "+%Y-%m-%dT%H:%M:%S.%NZ"`"
+START_SECONDS=$(date -d "$START_TIME" +%s.%N)
+END_SECONDS=$(date -d "$END_TIME" +%s.%N)
+DURATION=$(echo "$END_SECONDS - $START_SECONDS" | bc)
+
+MSG=""        # individual test msg
+FAIL_MSG=""   # msg for entire job run
+RESULT=""
+HAS_FAILED=false
+IS_RUNNING=false
+
+load_job_status(){
+    if [ "$JOB_RESULT" == "success" ]; then
+        RESULT="pass"
+    elif [ "$JOB_RESULT" == "failure" ]; then
+        RESULT="fail"
+    else
+        RESULT="skip"
+    fi
+}
+
+format_logs_to_json(){
+    line="$1"
+    json="{"
+
+    while [[ "$line" =~ ([a-zA-Z0-9_]+)=((\"([^\"\\]|\\.)*\")|[^[:space:]]+) ]]; do
+        key="${BASH_REMATCH[1]}"
+        value="${BASH_REMATCH[2]}"
+        line="${line#*"${key}=${value}"}"
+
+        if [[ "$value" == \"*\" ]]; then
+            value="${value:1:${#value}-2}"
+            value="${value//\"/\\\"}"
+        fi
+        json+="\"$key\":\"$value\","
+    done
+
+    json="${json%,}}"    
+    echo "$json"
+}
+
+format_results(){
+    while IFS= read -r line; do
+
+        if [[ "$line" =~ ^===\ RUN[[:space:]]+(.+) ]]; then
+            TEST_NAME="${BASH_REMATCH[1]}"
+            IS_RUNNING=true
+            MSG=""
+            TEST_START=""
+            TEST_END=""
+            mkdir -p "$OUTPUT_PATH/$TEST_NAME/"
+            RESULT_FILE="$OUTPUT_PATH/$TEST_NAME/result.json"
+            LOG_FILE="$OUTPUT_PATH/$TEST_NAME/test.log"
+        elif [[ "$line" =~ ([0-9T:\.\-Z]+)[[:space:]]+testing ]]; then
+            TEST_START="${BASH_REMATCH[1]}"
+        elif [[ "$line" =~ ([0-9T:\.\-Z]+)[[:space:]]+finished[[:space:]]testing ]]; then
+            TEST_END="${BASH_REMATCH[1]}"
+        elif [[ "$line" == "FAIL" ]]; then
+            HAS_FAILED=false
+            MSG="$MSG_STR"
+            FAIL_MSG+="$MSG"
+            HAS_FAILED=false
+            echo "{\"start_at\": \"$START_TIME\", \"end_at\": \"$END_TIME\", \"duration_seconds\": \"$DURATION\", \"result\": \"$TEST_RES\", \"msg\": \"$MSG\"}" > $RESULT_FILE
+        elif [[ "$line" == "--- PASS"* ]]; then
+            TEST_RES="pass"
+            IS_RUNNING=false
+            echo "{\"start_at\": \"$START_TIME\", \"end_at\": \"$END_TIME\", \"duration_seconds\": \"$DURATION\", \"result\": \"$TEST_RES\", \"msg\": \"$MSG\"}" > $RESULT_FILE
+        elif [[ "$line" == "--- FAIL"* ]]; then
+            TEST_RES="fail"
+            HAS_FAILED=true
+            IS_RUNNING=false
+        elif [[ "$line" == time=* && "$line" == *level=* ]]; then
+            LOG_LINE=$(format_logs_to_json "$line")
+            echo "$LOG_LINE" >> "$LOG_FILE"
+        fi
+
+        if [ $HAS_FAILED == true ]; then
+            MSG_STR+="$line"
+        fi
+
+    done < "$INPUT_FILE"
+
+    # Store the result of the whole job
+    echo "{\"start_at\": \"$START_TIME\", \"end_at\": \"$END_TIME\", \"duration_seconds\": \"$DURATION\", \"result\": \"$RESULT\", \"msg\": \"$FAIL_MSG\"}" > $JOB_OUTPUT_FILE
+}
+
+# Main body of the script
+{
+    load_job_status
+    format_results
+}

test/dashboard/prep/promtail.yaml

@@ -0,0 +1,134 @@
+server:
+    http_listen_port: 9080
+    grpc_listen_port: 0
+    log_level: info
+
+positions:
+    filename: /tmp/positions.yaml
+    sync_period: "10s"
+
+clients:
+    - url: "${LOKI_DASHBOARD_URL}"
+      external_labels:
+          systest_project: agent_v2
+          systest_type: "${GITHUB_JOB}"
+      timeout: 30s
+      backoff_config:
+          min_period: 500ms
+          max_period: 5s
+          max_retries: 5
+
+scrape_configs:
+    - job_name: test-results
+      static_configs:
+          - targets:
+                - localhost
+            labels:
+                systest_job: test-results
+                __path__: /var/log/**/result.json
+      pipeline_stages:
+          - json:
+                expressions:
+                    time:
+          - timestamp:
+                source: time
+                format: RFC3339Nano
+          - template:
+                source: ci_pipeline_id
+                template: "${GITHUB_RUN_ID}"
+          - template:
+                source: ci_pipeline_url
+                template: "${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}"
+          - template:
+                source: ci_pipeline_name
+                template: "${WORKFLOW}"
+          - template:
+                source: ci_pipeline_source
+                template: "${GITHUB_EVENT_NAME}"
+          - template:
+                source: ci_job_name
+                template: "${GITHUB_JOB}"
+          - template:
+                source: ci_commit_ref
+                template: "${GITHUB_HEAD_REF}"
+          - template:
+                source: ci_commit_sha
+                template: "${GITHUB_SHA}"
+          - template:
+                source: ci_commit_author
+                template: "${GITHUB_ACTOR}"
+          - template:
+                source: systest_path
+                template: '{{ trimPrefix "${TEST_OUTDIR}/" .filename | dir | replace "." "/" }}'
+
+          - structured_metadata:
+                ci_pipeline_id:
+                ci_pipeline_url:
+                ci_pipeline_name:
+                ci_pipeline_source:
+                ci_job_name:
+                ci_commit_ref:
+                ci_commit_sha:
+                ci_commit_author:
+                filename:
+                systest_path:
+
+          - labeldrop:
+                - filename
+
+    - job_name: test-logs
+      static_configs:
+        - targets:
+            - localhost
+          labels:
+            systest_job: test-logs
+            __path__: /var/log/**/test.log
+      pipeline_stages:
+        - json:
+            expressions:
+              time:
+        - timestamp:
+            source: time
+            format: RFC3339Nano
+        - template:
+            source: ci_pipeline_id
+            template: "${GITHUB_RUN_ID}"
+        - template:
+            source: ci_pipeline_url
+            template: "${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}"
+        - template:
+            source: ci_pipeline_name
+            template: "${WORKFLOW}"
+        - template:
+            source: ci_pipeline_source
+            template: "${GITHUB_EVENT_NAME}"
+        - template:
+            source: ci_job_name
+            template: "${GITHUB_JOB}"
+        - template:
+            source: ci_commit_ref
+            template: "${GITHUB_HEAD_REF}"
+        - template:
+            source: ci_commit_sha
+            template: "${GITHUB_SHA}"
+        - template:
+            source: ci_commit_author
+            template: "${GITHUB_ACTOR}"
+        - template:
+            source: systest_path
+            template: '{{ trimPrefix "${TEST_OUTDIR}/" .filename | dir | replace "." "/" }}'
+
+        - structured_metadata:
+            ci_pipeline_id:
+            ci_pipeline_url:
+            ci_pipeline_name:
+            ci_pipeline_source:
+            ci_job_name:
+            ci_commit_ref:
+            ci_commit_sha:
+            ci_commit_author:
+            filename:
+            systest_path:
+
+        - labeldrop:
+            - filename

test/docker/nginx-official-image/apk/Dockerfile

@@ -11,7 +11,6 @@ ARG CONTAINER_OS_TYPE
 
 WORKDIR /agent
 COPY ./build/${PACKAGE_NAME}.${CONTAINER_OS_TYPE} /agent/build/${PACKAGE_NAME}.${CONTAINER_OS_TYPE}
-COPY ./ /agent
 
 RUN apk add --allow-untrusted /agent/build/${PACKAGE_NAME}.${CONTAINER_OS_TYPE}