From 6d4fe19a606b50b9f9f2a6feffc0918b7507a289 Mon Sep 17 00:00:00 2001
From: John David White <joh.white@f5.com>
Date: Fri, 25 Jul 2025 15:39:27 +0100
Subject: [PATCH 01/33] Push agent v2 logs to QE grafana

---
 .github/actions/start-promtail/action.yml     |  25 ++++
 .github/workflows/ci.yml                      |  33 ++++-
 scripts/workflow/generate_results.sh          | 100 +++++++++++++
 test/dashboard/prep/promtail.yaml             | 135 ++++++++++++++++++
 test/integration/api/api_test.go              |   4 +
 test/integration/features/features_test.go    |   6 +
 .../grpc/grpc_config_apply_test.go            |   2 +
 .../install/install_uninstall_test.go         |   2 +
 8 files changed, 304 insertions(+), 3 deletions(-)
 create mode 100644 .github/actions/start-promtail/action.yml
 create mode 100644 scripts/workflow/generate_results.sh
 create mode 100644 test/dashboard/prep/promtail.yaml

diff --git a/.github/actions/start-promtail/action.yml b/.github/actions/start-promtail/action.yml
new file mode 100644
index 000000000..3415319be
--- /dev/null
+++ b/.github/actions/start-promtail/action.yml
@@ -0,0 +1,25 @@
+name: Start Promtail
+description: Start promtail in a Docker container to send test results to loki
+runs:
+    using: "composite"
+    steps:
+        - name: Start Promtail container
+          shell: bash
+          run: |
+              docker run -d \
+               --name=promtail \
+               -v "${{ github.workspace }}/test/dashboard/prep/promtail.yaml:/etc/promtail/config.yaml" \
+               -v "${{ github.workspace }}/test/dashboard/logs:/var/log" \
+               -e TEST_OUTDIR=test/dashboard/logs \
+               -e GITHUB_RUN_ID=${{ github.run_id }} \
+               -e GITHUB_JOB=${{ github.job }} \
+               -e GITHUB_WORKFLOW=${{ github.workflow }} \
+               -e GITHUB_EVENT_NAME=${{ github.event_name }} \
+               -e GITHUB_SERVER_URL=${{ github.server_url }} \
+               -e GITHUB_REPOSITORY=${{ github.repository }} \
+               -e GITHUB_HEAD_REF=${{ github.head_ref }} \
+               -e GITHUB_SHA=${{ github.sha }} \
+               -e GITHUB_ACTOR=${{ github.actor }} \
+               grafana/promtail:3.4.4 \
+               -config.file=/etc/promtail/config.yaml \
+               -config.expand-env=true
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 3c78516c8..191bb758d 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -161,11 +161,20 @@ jobs:
             with:
                 name: nginx-agent-unsigned-snapshots
                 path: build
+          - name: Set Start Time
+            run: echo "START_TIME=$(date +"%Y-%m-%dT%H:%M:%S.%NZ")" >> ${GITHUB_ENV}
+          - name: Create Directory for Results
+            run: mkdir -p ${{github.workspace}}/test/dashboard/logs/${{github.job}}/
+          - name: Start Promtail
+            uses: ./.github/actions/start-promtail
           - name: Run Integration Tests
             run: |
                 go install github.com/goreleaser/nfpm/v2/cmd/nfpm@${{ env.NFPM_VERSION }}
                 OS_RELEASE="${{ matrix.container.image }}" OS_VERSION="${{ matrix.container.version }}" \
-                  make integration-test
+                  make integration-test | tee ${{github.workspace}}/test/dashboard/logs/${{github.job}}-${{matrix.container.image}}-${{matrix-container-version}}/raw_logs.log && exit "${PIPESTATUS[0]}"
+          - name: Generate Test Results
+            if: always()
+            run: bash ./scripts/workflow/generate_results.sh ${{job.status}} ${{env.START_TIME}} ${{github.job}}
           - name: Container Output Logs
             if: failure()
             run: |
@@ -209,6 +218,12 @@ jobs:
       - uses: actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491 # v5.0.0
         with:
           go-version-file: 'go.mod'
+      -   name: Set Start Time
+          run: echo "START_TIME=$(date +"%Y-%m-%dT%H:%M:%S.%NZ")" >> ${GITHUB_ENV}
+      -   name: Create Directory for Results
+          run: mkdir -p ${{github.workspace}}/test/dashboard/logs/${{github.job}}/
+      -   name: Start Promtail
+          uses: ./.github/actions/start-promtail
       - name: Download Packages
         uses: actions/download-artifact@65a9edc5881444af0b9093a5e628f2fe47ea3b2e # v4.1.7
         with:
@@ -218,7 +233,10 @@ jobs:
         run: |
           go install github.com/goreleaser/nfpm/v2/cmd/nfpm@${{ env.NFPM_VERSION }}
           CONTAINER_NGINX_IMAGE_REGISTRY="${{ env.NGINX_OSS_REGISTRY }}" TAG="${{ matrix.container.version }}-${{ matrix.container.image }}" OS_RELEASE="${{ matrix.container.release }}"\
-            make official-image-integration-test
+            make official-image-integration-test | tee ${{github.workspace}}/test/dashboard/logs/${{github.job}}-${{matrix.container.image}}-${{matrix-container-version}}/raw_logs.log && exit "${PIPESTATUS[0]}"
+      - name: Generate Test Results
+        if: always()
+        run: bash ./scripts/workflow/generate_results.sh ${{job.status}} ${{env.START_TIME}} ${{github.job}}
       - name: Container Output Logs 
         if: failure()
         run: |
@@ -275,6 +293,12 @@ jobs:
             with:
                 name: nginx-agent-unsigned-snapshots
                 path: build
+          - name: Set Start Time
+            run: echo "START_TIME=$(date +"%Y-%m-%dT%H:%M:%S.%NZ")" >> ${GITHUB_ENV}
+          - name: Create Directory for Results
+            run: mkdir -p ${{github.workspace}}/test/dashboard/logs/${{github.job}}/
+          - name: Start Promtail
+            uses: ./.github/actions/start-promtail
           - name: Login to Docker Registry
             uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v3.3.0
             with:
@@ -286,7 +310,10 @@ jobs:
                 go install github.com/goreleaser/nfpm/v2/cmd/nfpm@${{ env.NFPM_VERSION }}
                 CONTAINER_NGINX_IMAGE_REGISTRY="${{ secrets.REGISTRY_URL }}" TAG="${{ matrix.container.plus }}-${{ matrix.container.image }}-${{ matrix.container.version }}" \
                 OS_RELEASE="${{ matrix.container.release }}" IMAGE_PATH="${{ matrix.container.path }}" \
-                  make official-image-integration-test
+                  make official-image-integration-test | tee ${{github.workspace}}/test/dashboard/logs/${{github.job}}-${{matrix.container.image}}-${{matrix-container-version}}/raw_logs.log && exit "${PIPESTATUS[0]}"
+          - name: Generate Test Results
+            if: always()
+            run: bash ./scripts/workflow/generate_results.sh ${{job.status}} ${{env.START_TIME}} ${{github.job}}
           - name: Container Output Logs
             if: failure()
             run: |
diff --git a/scripts/workflow/generate_results.sh b/scripts/workflow/generate_results.sh
new file mode 100644
index 000000000..fe998da0a
--- /dev/null
+++ b/scripts/workflow/generate_results.sh
@@ -0,0 +1,100 @@
+#!/bin/bash
+
+# TODO: `start_time` and `end_time`, separate test name from `$line`
+
+JOB_RESULT="$1"
+START_TIME="$2"
+TEST_TYPE="$3"
+
+INPUT_FILE="./test/dashboard/logs/$TEST_TYPE/raw_logs.log"
+JOB_OUTPUT_FILE="./test/dashboard/logs/$TEST_TYPE/result.json"
+LOG_OUTPUT_FILE="./test/dashboard/logs/$TEST_TYPE/test.log"
+OUTPUT_PATH="./test/dashboard/logs/$TEST_TYPE/"
+
+END_TIME="`date "+%Y-%m-%dT%H:%M:%S.%NZ"`"
+START_SECONDS=$(date -d "$START_TIME" +%s.%N)
+END_SECONDS=$(date -d "$END_TIME" +%s.%N)
+DURATION=$(echo "$END_SECONDS - $START_SECONDS" | bc)
+
+MSG=""        # individual test msg
+FAIL_MSG=""   # msg for entire job run
+RESULT=""
+HAS_FAILED=false
+IS_RUNNING=false
+
+load_job_status(){
+    if [ $JOB_RESULT == "success" ]; then
+        RESULT="pass"
+    elif [ $JOB_RESULT == "failure" ]; then
+        RESULT="fail"
+    else
+        RESULT="skip"
+    fi
+}
+
+format_logs_to_json(){
+    local LINE="$1"
+    local JSON="{"
+    local key value
+    
+    while [[ "$line" =~ ([a-zA-Z0-9_]+)=((\"[^\"]*\")|([:space:]]+)) ]]; do
+        key="${BASH_REMATCH[1]}"
+        value="${BASH_REMATCH[2]}"
+        LINE="${line#*${BASH_REMATCH[0]}}"
+        json+="\"$key\": \"$value\", "
+    done
+    
+    json="${json%, }"
+    json+="}"
+    
+    echo "$json"
+}
+
+format_results(){
+    while IFS= read -r line; do
+        
+        if [[ "$line" =~ ^===\ RUN[[:space:]]+(.+) ]]; then
+            TEST_NAME="${BASH_REMATCH[1]}"
+            IS_RUNNING=true
+            MSG=""
+            TEST_START=""
+            TEST_END=""
+            OUTPUT_FILE="${OUTPUT_PATH}${TEST_NAME}/result.json"
+        elif [[ "$line" =~ ([0-9T:\.\-Z]+)[[:space:]]+testing ]]; then
+            TEST_START="${BASH_REMATCH[1]}"
+        elif [[ "$line" =~ ([0-9T:\.\-Z]+)[[:space:]]+finished testing ]]; then
+            TEST_END="${BASH_REMATCH[1]}"
+        elif [[ "$line" == "FAIL" ]]; then
+            HAS_FAILED=false
+            MSG="$MSG_STR"
+            FAIL_MSG+="$MSG"
+            HAS_FAILED=false
+            echo "{\"start_at\": \"$START_TIME\", \"end_at\": \"$END_TIME\", \"duration_seconds\": \"$DURATION\", \"result\": \"$TEST_RES\", \"msg\": \"$MSG\"}" > $OUTPUT_FILE
+        elif [[ "$line" == "--- PASS"* ]]; then
+            TEST_RES="pass"
+            IS_RUNNING=false
+            echo "{\"start_at\": \"$START_TIME\", \"end_at\": \"$END_TIME\", \"duration_seconds\": \"$DURATION\", \"result\": \"$TEST_RES\", \"msg\": \"$MSG\"}" > $OUTPUT_FILE
+        elif [[ "$line" == "--- FAIL"* ]]; then
+            TEST_RES="fail"
+            HAS_FAILED=true
+            IS_RUNNING=false
+        elif [[ "$line" == time=* && "$line" == *level=* ]]; then
+            LOG_LINE=$(format_logs_to_json "$line")
+            echo "$LOG_LINE" >> "$LOG_OUTPUT_FILE"
+        fi
+        
+        if [ $HAS_FAILED == true ]; then
+            MSG_STR+="$line"
+        fi
+
+    done < "$INPUT_FILE"
+            
+    # Store the result of the whole job
+    echo "{\"start_at\": \"$START_TIME\", \"end_at\": \"$END_TIME\", \"duration_seconds\": \"$DURATION\", \"result\": \"$RESULT\", \"msg\": \"$FAIL_MSG\"}" > $JOB_OUTPUT_FILE
+}
+
+# Main body of the script
+{
+    load_job_status
+    format_results
+}
diff --git a/test/dashboard/prep/promtail.yaml b/test/dashboard/prep/promtail.yaml
new file mode 100644
index 000000000..1ed46f47d
--- /dev/null
+++ b/test/dashboard/prep/promtail.yaml
@@ -0,0 +1,135 @@
+server:
+    http_listen_port: 9080
+    grpc_listen_port: 0
+    log_level: info
+    
+positions:
+    filename: /tmp/positions.yaml
+    sync_period: "10s"
+
+clients:
+    - url: ${{ secrets.LOKI_DASHBOARD_URL }}
+      external_labels:
+          systest_project: agent_v2
+          systest_type: ${GITHUB_JOB}
+      timeout: 30s
+      backoff_config:
+          min_period: 500ms
+          max_period: 5s
+          max_retries: 5
+
+scrape_configs:
+    - job_name: test-results
+      static_configs:
+          - targets:
+                - localhost
+            labels:
+                systest_job: test-results
+                __path__: /var/log/**/result.json
+      pipeline_stages:
+          - json:
+                expressions:
+                    time:
+          - timestamp:
+                source: time
+                format: RFC3339Nano
+          - template:
+                source: ci_pipeline_id
+                template: "${GITHUB_RUN_ID}"
+          - template:
+                source: ci_pipeline_url
+                template: "${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}"
+          - template:
+                source: ci_pipeline_name
+                template: "${WORKFLOW}"
+          - template:
+                source: ci_pipeline_source
+                template: "${GITHUB_EVENT_NAME}"
+          - template:
+                source: ci_job_name
+                template: "${GITHUB_JOB}"
+          - template:
+                source: ci_commit_ref
+                template: "${GITHUB_HEAD_REF}"
+          - template:
+                source: ci_commit_sha
+                template: "${GITHUB_SHA}"
+          - template:
+                source: ci_commit_author
+                template: "${GITHUB_ACTOR}"
+          - template:
+                source: systest_path
+                template: '{{ trimPrefix "${TEST_OUTDIR}/" .filename | dir | replace "." "/" }}'
+
+          - structured_metadata:
+                ci_pipeline_id:
+                ci_pipeline_url:
+                ci_pipeline_name:
+                ci_pipeline_source:
+                ci_job_name:
+                ci_commit_ref:
+                ci_commit_sha:
+                ci_commit_author:
+                filename:
+                systest_path:
+
+          - labeldrop:
+                - filename
+
+
+    - job_name: test-logs
+      static_configs:
+        - targets:
+            - localhost
+          labels:
+            systest_job: test-logs
+            __path__: /var/log/**/test.log
+      pipeline_stages:
+        - json:
+            expressions:
+              time:
+        - timestamp:
+            source: time
+            format: RFC3339Nano
+        - template:
+            source: ci_pipeline_id
+            template: "${GITHUB_RUN_ID}"
+        - template:
+            source: ci_pipeline_url
+            template: "${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}"
+        - template:
+            source: ci_pipeline_name
+            template: "${WORKFLOW}"
+        - template:
+            source: ci_pipeline_source
+            template: "${GITHUB_EVENT_NAME}"
+        - template:
+            source: ci_job_name
+            template: "${GITHUB_JOB}"
+        - template:
+            source: ci_commit_ref
+            template: "${GITHUB_HEAD_REF}"
+        - template:
+            source: ci_commit_sha
+            template: "${GITHUB_SHA}"
+        - template:
+            source: ci_commit_author
+            template: "${GITHUB_ACTOR}"
+        - template:
+            source: systest_path
+            template: '{{ trimPrefix "${TEST_OUTDIR}/" .filename | dir | replace "." "/" }}'
+
+        - structured_metadata:
+            ci_pipeline_id:
+            ci_pipeline_url:
+            ci_pipeline_name:
+            ci_pipeline_source:
+            ci_job_name:
+            ci_commit_ref:
+            ci_commit_sha:
+            ci_commit_author:
+            filename:
+            systest_path:
+
+        - labeldrop:
+            - filename
diff --git a/test/integration/api/api_test.go b/test/integration/api/api_test.go
index 0f336b5b7..13d7f6ba2 100644
--- a/test/integration/api/api_test.go
+++ b/test/integration/api/api_test.go
@@ -25,6 +25,7 @@ import (
 var delay = time.Duration(5 * time.Second)
 
 func TestAPI_Nginx(t *testing.T) {
+	t.Log("testing nginx api")
 	ctx := context.Background()
 	containerNetwork := utils.CreateContainerNetwork(ctx, t)
 
@@ -80,9 +81,11 @@ func TestAPI_Nginx(t *testing.T) {
 	assert.Equal(t, "/etc/nginx/nginx.conf", nginxDetailsResponse[0].ConfPath)
 
 	utils.TestAgentHasNoErrorLogs(t, testContainer)
+	t.Log("finished testing nginx api")
 }
 
 func TestAPI_Metrics(t *testing.T) {
+	t.Log("testing metrics api")
 	ctx := context.Background()
 	containerNetwork := utils.CreateContainerNetwork(ctx, t)
 
@@ -157,4 +160,5 @@ func TestAPI_Metrics(t *testing.T) {
 	}
 
 	utils.TestAgentHasNoErrorLogs(t, testContainer)
+	t.Log("finished testing metrics api")
 }
diff --git a/test/integration/features/features_test.go b/test/integration/features/features_test.go
index c5c6e5aa1..233b69f2b 100644
--- a/test/integration/features/features_test.go
+++ b/test/integration/features/features_test.go
@@ -11,6 +11,7 @@ import (
 )
 
 func TestFeatures_NginxCountingEnabled(t *testing.T) {
+	t.Log("testing nginx counting enabled")
 	enabledFeatureLogs := []string{
 		"level=info msg=\"NGINX Counter initializing", "level=info msg=\"MetricsThrottle initializing\"", "level=info msg=\"DataPlaneStatus initializing\"",
 		"level=info msg=\"OneTimeRegistration initializing\"", "level=info msg=\"Metrics initializing\"",
@@ -56,9 +57,11 @@ func TestFeatures_NginxCountingEnabled(t *testing.T) {
 	for _, logLine := range disabledFeatureLogs {
 		assert.NotContains(t, string(agentLogContent), logLine, "agent log file contains disabled feature log")
 	}
+	t.Log("finished testing nginx counting enabled")
 }
 
 func TestFeatures_MetricsEnabled(t *testing.T) {
+	t.Log("testing metrics enabled")
 	enabledFeatureLogs := []string{"level=info msg=\"Metrics initializing\"", "level=info msg=\"MetricsThrottle initializing\"", "level=info msg=\"DataPlaneStatus initializing\""}
 	disabledFeatureLogs := []string{"level=info msg=\"OneTimeRegistration initializing\"", "level=info msg=\"Events initializing\"", "level=info msg=\"Agent API initializing\""}
 
@@ -101,9 +104,11 @@ func TestFeatures_MetricsEnabled(t *testing.T) {
 	for _, logLine := range disabledFeatureLogs {
 		assert.NotContains(t, string(agentLogContent), logLine, "agent log file contains disabled feature log")
 	}
+	t.Log("finished testing metrics enabled")
 }
 
 func TestFeatures_ConfigEnabled(t *testing.T) {
+	t.Log("testing config enabled")
 	enabledFeatureLogs := []string{"level=info msg=\"DataPlaneStatus initializing\""}
 	disabledFeatureLogs := []string{"level=info msg=\"Events initializing\"", "level=info msg=\"Agent API initializing\"", "level=info msg=\"Metrics initializing\"", "level=info msg=\"MetricsThrottle initializing\""}
 
@@ -146,4 +151,5 @@ func TestFeatures_ConfigEnabled(t *testing.T) {
 	for _, logLine := range disabledFeatureLogs {
 		assert.NotContains(t, string(agentLogContent), logLine, "agent log file contains disabled feature log")
 	}
+	t.Log("finished testing config enabled")
 }
diff --git a/test/integration/grpc/grpc_config_apply_test.go b/test/integration/grpc/grpc_config_apply_test.go
index 25001d122..9f033bc83 100644
--- a/test/integration/grpc/grpc_config_apply_test.go
+++ b/test/integration/grpc/grpc_config_apply_test.go
@@ -67,6 +67,7 @@ config_dirs: "/etc/nginx:/usr/local/etc/nginx:/usr/share/nginx/modules:/etc/nms:
 )
 
 func TestRegistrationAndConfigApply(t *testing.T) {
+	t.Log("testing registration and config apply")
 	ctx := context.Background()
 	grpcListener, grpcClose := createListener()
 	defer grpcClose()
@@ -210,6 +211,7 @@ messageLoop:
 			}
 		})
 	}
+	t.Log("finished testing registration and config apply")
 }
 
 func createListener() (listener net.Listener, close func() error) {
diff --git a/test/integration/install/install_uninstall_test.go b/test/integration/install/install_uninstall_test.go
index ee2ae7e9f..f51cfb20f 100644
--- a/test/integration/install/install_uninstall_test.go
+++ b/test/integration/install/install_uninstall_test.go
@@ -64,6 +64,7 @@ func installUninstallSetup(tb testing.TB, expectNoErrorsInLogs bool) (testcontai
 // Verifies that agent installs with correct output and files.
 // Verifies that agent uninstalls and removes all the files.
 func TestAgentManualInstallUninstall(t *testing.T) {
+	t.Log("testing agent install uninstall")
 	expectedInstallLogMsgs := map[string]string{
 		"InstallFoundNginxAgent": "Found nginx-agent /usr/bin/nginx-agent",
 		"InstallAgentSuccess":    "NGINX Agent package has been successfully installed.",
@@ -148,6 +149,7 @@ func TestAgentManualInstallUninstall(t *testing.T) {
 		_, err = testContainer.CopyFileFromContainer(ctx, agentPath)
 		assert.Error(t, err)
 	}
+	t.Log("finished testing agent install uninstall")
 }
 
 // installAgent installs the agent returning total install time and install output

From 7e969e878c78549798978eff0bb7d2a1836b1509 Mon Sep 17 00:00:00 2001
From: John David White <joh.white@f5.com>
Date: Fri, 25 Jul 2025 15:50:33 +0100
Subject: [PATCH 02/33] fix ci env var

---
 .github/workflows/ci.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 191bb758d..d47870b9d 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -171,7 +171,7 @@ jobs:
             run: |
                 go install github.com/goreleaser/nfpm/v2/cmd/nfpm@${{ env.NFPM_VERSION }}
                 OS_RELEASE="${{ matrix.container.image }}" OS_VERSION="${{ matrix.container.version }}" \
-                  make integration-test | tee ${{github.workspace}}/test/dashboard/logs/${{github.job}}-${{matrix.container.image}}-${{matrix-container-version}}/raw_logs.log && exit "${PIPESTATUS[0]}"
+                  make integration-test | tee ${{github.workspace}}/test/dashboard/logs/${{github.job}}-${{matrix.container.image}}-${{matrix.container.version}}/raw_logs.log && exit "${PIPESTATUS[0]}"
           - name: Generate Test Results
             if: always()
             run: bash ./scripts/workflow/generate_results.sh ${{job.status}} ${{env.START_TIME}} ${{github.job}}

From 8c89e17760141eb5d5af23016f5962cfe6b666da Mon Sep 17 00:00:00 2001
From: John David White <joh.white@f5.com>
Date: Fri, 25 Jul 2025 15:55:27 +0100
Subject: [PATCH 03/33] Update env vars

---
 .github/workflows/ci.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index d47870b9d..b05cb0d1e 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -233,7 +233,7 @@ jobs:
         run: |
           go install github.com/goreleaser/nfpm/v2/cmd/nfpm@${{ env.NFPM_VERSION }}
           CONTAINER_NGINX_IMAGE_REGISTRY="${{ env.NGINX_OSS_REGISTRY }}" TAG="${{ matrix.container.version }}-${{ matrix.container.image }}" OS_RELEASE="${{ matrix.container.release }}"\
-            make official-image-integration-test | tee ${{github.workspace}}/test/dashboard/logs/${{github.job}}-${{matrix.container.image}}-${{matrix-container-version}}/raw_logs.log && exit "${PIPESTATUS[0]}"
+            make official-image-integration-test | tee ${{github.workspace}}/test/dashboard/logs/${{github.job}}-${{matrix.container.image}}-${{matrix.container.version}}/raw_logs.log && exit "${PIPESTATUS[0]}"
       - name: Generate Test Results
         if: always()
         run: bash ./scripts/workflow/generate_results.sh ${{job.status}} ${{env.START_TIME}} ${{github.job}}
@@ -310,7 +310,7 @@ jobs:
                 go install github.com/goreleaser/nfpm/v2/cmd/nfpm@${{ env.NFPM_VERSION }}
                 CONTAINER_NGINX_IMAGE_REGISTRY="${{ secrets.REGISTRY_URL }}" TAG="${{ matrix.container.plus }}-${{ matrix.container.image }}-${{ matrix.container.version }}" \
                 OS_RELEASE="${{ matrix.container.release }}" IMAGE_PATH="${{ matrix.container.path }}" \
-                  make official-image-integration-test | tee ${{github.workspace}}/test/dashboard/logs/${{github.job}}-${{matrix.container.image}}-${{matrix-container-version}}/raw_logs.log && exit "${PIPESTATUS[0]}"
+                  make official-image-integration-test | tee ${{github.workspace}}/test/dashboard/logs/${{github.job}}-${{matrix.container.image}}-${{matrix.container.version}}/raw_logs.log && exit "${PIPESTATUS[0]}"
           - name: Generate Test Results
             if: always()
             run: bash ./scripts/workflow/generate_results.sh ${{job.status}} ${{env.START_TIME}} ${{github.job}}

From ee360093fc6e7fb8e99e4d433275b78579eb60ad Mon Sep 17 00:00:00 2001
From: John David White <joh.white@f5.com>
Date: Mon, 28 Jul 2025 11:53:48 +0100
Subject: [PATCH 04/33] use bash for pipestatus

---
 .github/workflows/ci.yml                           |  9 ++++++---
 scripts/workflow/generate_results.sh               |  6 +++---
 test/integration/api/api_test.go                   |  9 +++++----
 test/integration/features/features_test.go         | 13 +++++++------
 test/integration/grpc/grpc_config_apply_test.go    |  4 ++--
 test/integration/install/install_uninstall_test.go |  5 +++--
 6 files changed, 26 insertions(+), 20 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index b05cb0d1e..e69e74467 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -168,10 +168,11 @@ jobs:
           - name: Start Promtail
             uses: ./.github/actions/start-promtail
           - name: Run Integration Tests
+            shell: bash
             run: |
                 go install github.com/goreleaser/nfpm/v2/cmd/nfpm@${{ env.NFPM_VERSION }}
                 OS_RELEASE="${{ matrix.container.image }}" OS_VERSION="${{ matrix.container.version }}" \
-                  make integration-test | tee ${{github.workspace}}/test/dashboard/logs/${{github.job}}-${{matrix.container.image}}-${{matrix.container.version}}/raw_logs.log && exit "${PIPESTATUS[0]}"
+                  make integration-test | tee ${{github.workspace}}/test/dashboard/logs/${{github.job}}.${{matrix.container.image}}.${{matrix.container.version}}/raw_logs.log && exit "${PIPESTATUS[0]}"
           - name: Generate Test Results
             if: always()
             run: bash ./scripts/workflow/generate_results.sh ${{job.status}} ${{env.START_TIME}} ${{github.job}}
@@ -230,10 +231,11 @@ jobs:
           name: nginx-agent-unsigned-snapshots
           path: build
       - name: Run Integration Tests
+        shell: bash
         run: |
           go install github.com/goreleaser/nfpm/v2/cmd/nfpm@${{ env.NFPM_VERSION }}
           CONTAINER_NGINX_IMAGE_REGISTRY="${{ env.NGINX_OSS_REGISTRY }}" TAG="${{ matrix.container.version }}-${{ matrix.container.image }}" OS_RELEASE="${{ matrix.container.release }}"\
-            make official-image-integration-test | tee ${{github.workspace}}/test/dashboard/logs/${{github.job}}-${{matrix.container.image}}-${{matrix.container.version}}/raw_logs.log && exit "${PIPESTATUS[0]}"
+            make official-image-integration-test | tee ${{github.workspace}}/test/dashboard/logs/${{github.job}}.${{matrix.container.image}}.${{matrix.container.version}}/raw_logs.log && exit "${PIPESTATUS[0]}"
       - name: Generate Test Results
         if: always()
         run: bash ./scripts/workflow/generate_results.sh ${{job.status}} ${{env.START_TIME}} ${{github.job}}
@@ -306,11 +308,12 @@ jobs:
                 username: ${{ secrets.REGISTRY_USERNAME }}
                 password: ${{ secrets.REGISTRY_PASSWORD }}
           - name: Run Integration Tests
+            shell: bash
             run: |
                 go install github.com/goreleaser/nfpm/v2/cmd/nfpm@${{ env.NFPM_VERSION }}
                 CONTAINER_NGINX_IMAGE_REGISTRY="${{ secrets.REGISTRY_URL }}" TAG="${{ matrix.container.plus }}-${{ matrix.container.image }}-${{ matrix.container.version }}" \
                 OS_RELEASE="${{ matrix.container.release }}" IMAGE_PATH="${{ matrix.container.path }}" \
-                  make official-image-integration-test | tee ${{github.workspace}}/test/dashboard/logs/${{github.job}}-${{matrix.container.image}}-${{matrix.container.version}}/raw_logs.log && exit "${PIPESTATUS[0]}"
+                  make official-image-integration-test | tee ${{github.workspace}}/test/dashboard/logs/${{github.job}}.${{matrix.container.image}}.${{matrix.container.version}}/raw_logs.log && exit "${PIPESTATUS[0]}"
           - name: Generate Test Results
             if: always()
             run: bash ./scripts/workflow/generate_results.sh ${{job.status}} ${{env.START_TIME}} ${{github.job}}
diff --git a/scripts/workflow/generate_results.sh b/scripts/workflow/generate_results.sh
index fe998da0a..fa963a48a 100644
--- a/scripts/workflow/generate_results.sh
+++ b/scripts/workflow/generate_results.sh
@@ -23,9 +23,9 @@ HAS_FAILED=false
 IS_RUNNING=false
 
 load_job_status(){
-    if [ $JOB_RESULT == "success" ]; then
+    if [ "$JOB_RESULT" == "success" ]; then
         RESULT="pass"
-    elif [ $JOB_RESULT == "failure" ]; then
+    elif [ "$JOB_RESULT" == "failure" ]; then
         RESULT="fail"
     else
         RESULT="skip"
@@ -62,7 +62,7 @@ format_results(){
             OUTPUT_FILE="${OUTPUT_PATH}${TEST_NAME}/result.json"
         elif [[ "$line" =~ ([0-9T:\.\-Z]+)[[:space:]]+testing ]]; then
             TEST_START="${BASH_REMATCH[1]}"
-        elif [[ "$line" =~ ([0-9T:\.\-Z]+)[[:space:]]+finished testing ]]; then
+        elif [[ "$line" =~ ([0-9T:\.\-Z]+)[[:space:]]+finished[[:space]]testing ]]; then
             TEST_END="${BASH_REMATCH[1]}"
         elif [[ "$line" == "FAIL" ]]; then
             HAS_FAILED=false
diff --git a/test/integration/api/api_test.go b/test/integration/api/api_test.go
index 13d7f6ba2..eddc6d865 100644
--- a/test/integration/api/api_test.go
+++ b/test/integration/api/api_test.go
@@ -4,6 +4,7 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
+	log "github.com/sirupsen/logrus"
 	"net"
 	"net/http"
 	"os"
@@ -25,7 +26,7 @@ import (
 var delay = time.Duration(5 * time.Second)
 
 func TestAPI_Nginx(t *testing.T) {
-	t.Log("testing nginx api")
+	log.Info("testing nginx api")
 	ctx := context.Background()
 	containerNetwork := utils.CreateContainerNetwork(ctx, t)
 
@@ -81,11 +82,11 @@ func TestAPI_Nginx(t *testing.T) {
 	assert.Equal(t, "/etc/nginx/nginx.conf", nginxDetailsResponse[0].ConfPath)
 
 	utils.TestAgentHasNoErrorLogs(t, testContainer)
-	t.Log("finished testing nginx api")
+	log.Info("finished testing nginx api")
 }
 
 func TestAPI_Metrics(t *testing.T) {
-	t.Log("testing metrics api")
+	log.Info("testing metrics api")
 	ctx := context.Background()
 	containerNetwork := utils.CreateContainerNetwork(ctx, t)
 
@@ -160,5 +161,5 @@ func TestAPI_Metrics(t *testing.T) {
 	}
 
 	utils.TestAgentHasNoErrorLogs(t, testContainer)
-	t.Log("finished testing metrics api")
+	log.Info("finished testing metrics api")
 }
diff --git a/test/integration/features/features_test.go b/test/integration/features/features_test.go
index 233b69f2b..d65c173d7 100644
--- a/test/integration/features/features_test.go
+++ b/test/integration/features/features_test.go
@@ -2,6 +2,7 @@ package features
 
 import (
 	"context"
+	log "github.com/sirupsen/logrus"
 	"io"
 	"os"
 	"testing"
@@ -11,7 +12,7 @@ import (
 )
 
 func TestFeatures_NginxCountingEnabled(t *testing.T) {
-	t.Log("testing nginx counting enabled")
+	log.Info("testing nginx counting enabled")
 	enabledFeatureLogs := []string{
 		"level=info msg=\"NGINX Counter initializing", "level=info msg=\"MetricsThrottle initializing\"", "level=info msg=\"DataPlaneStatus initializing\"",
 		"level=info msg=\"OneTimeRegistration initializing\"", "level=info msg=\"Metrics initializing\"",
@@ -57,11 +58,11 @@ func TestFeatures_NginxCountingEnabled(t *testing.T) {
 	for _, logLine := range disabledFeatureLogs {
 		assert.NotContains(t, string(agentLogContent), logLine, "agent log file contains disabled feature log")
 	}
-	t.Log("finished testing nginx counting enabled")
+	log.Info("finished testing nginx counting enabled")
 }
 
 func TestFeatures_MetricsEnabled(t *testing.T) {
-	t.Log("testing metrics enabled")
+	log.Info("testing metrics enabled")
 	enabledFeatureLogs := []string{"level=info msg=\"Metrics initializing\"", "level=info msg=\"MetricsThrottle initializing\"", "level=info msg=\"DataPlaneStatus initializing\""}
 	disabledFeatureLogs := []string{"level=info msg=\"OneTimeRegistration initializing\"", "level=info msg=\"Events initializing\"", "level=info msg=\"Agent API initializing\""}
 
@@ -104,11 +105,11 @@ func TestFeatures_MetricsEnabled(t *testing.T) {
 	for _, logLine := range disabledFeatureLogs {
 		assert.NotContains(t, string(agentLogContent), logLine, "agent log file contains disabled feature log")
 	}
-	t.Log("finished testing metrics enabled")
+	log.Info("finished testing metrics enabled")
 }
 
 func TestFeatures_ConfigEnabled(t *testing.T) {
-	t.Log("testing config enabled")
+	log.Info("testing config enabled")
 	enabledFeatureLogs := []string{"level=info msg=\"DataPlaneStatus initializing\""}
 	disabledFeatureLogs := []string{"level=info msg=\"Events initializing\"", "level=info msg=\"Agent API initializing\"", "level=info msg=\"Metrics initializing\"", "level=info msg=\"MetricsThrottle initializing\""}
 
@@ -151,5 +152,5 @@ func TestFeatures_ConfigEnabled(t *testing.T) {
 	for _, logLine := range disabledFeatureLogs {
 		assert.NotContains(t, string(agentLogContent), logLine, "agent log file contains disabled feature log")
 	}
-	t.Log("finished testing config enabled")
+	log.Info("finished testing config enabled")
 }
diff --git a/test/integration/grpc/grpc_config_apply_test.go b/test/integration/grpc/grpc_config_apply_test.go
index 9f033bc83..98b9ab858 100644
--- a/test/integration/grpc/grpc_config_apply_test.go
+++ b/test/integration/grpc/grpc_config_apply_test.go
@@ -67,7 +67,7 @@ config_dirs: "/etc/nginx:/usr/local/etc/nginx:/usr/share/nginx/modules:/etc/nms:
 )
 
 func TestRegistrationAndConfigApply(t *testing.T) {
-	t.Log("testing registration and config apply")
+	log.Info("testing registration and config apply")
 	ctx := context.Background()
 	grpcListener, grpcClose := createListener()
 	defer grpcClose()
@@ -211,7 +211,7 @@ messageLoop:
 			}
 		})
 	}
-	t.Log("finished testing registration and config apply")
+	log.Info("finished testing registration and config apply")
 }
 
 func createListener() (listener net.Listener, close func() error) {
diff --git a/test/integration/install/install_uninstall_test.go b/test/integration/install/install_uninstall_test.go
index f51cfb20f..0c9ec9f61 100644
--- a/test/integration/install/install_uninstall_test.go
+++ b/test/integration/install/install_uninstall_test.go
@@ -3,6 +3,7 @@ package install
 import (
 	"context"
 	"fmt"
+	log "github.com/sirupsen/logrus"
 	"io"
 	"os"
 	"path"
@@ -64,7 +65,7 @@ func installUninstallSetup(tb testing.TB, expectNoErrorsInLogs bool) (testcontai
 // Verifies that agent installs with correct output and files.
 // Verifies that agent uninstalls and removes all the files.
 func TestAgentManualInstallUninstall(t *testing.T) {
-	t.Log("testing agent install uninstall")
+	log.Info("testing agent install uninstall")
 	expectedInstallLogMsgs := map[string]string{
 		"InstallFoundNginxAgent": "Found nginx-agent /usr/bin/nginx-agent",
 		"InstallAgentSuccess":    "NGINX Agent package has been successfully installed.",
@@ -149,7 +150,7 @@ func TestAgentManualInstallUninstall(t *testing.T) {
 		_, err = testContainer.CopyFileFromContainer(ctx, agentPath)
 		assert.Error(t, err)
 	}
-	t.Log("finished testing agent install uninstall")
+	log.Info("finished testing agent install uninstall")
 }
 
 // installAgent installs the agent returning total install time and install output

From b04704e024840e4cbaa02d1ecd7d8b386760d46a Mon Sep 17 00:00:00 2001
From: John David White <joh.white@f5.com>
Date: Mon, 28 Jul 2025 11:55:16 +0100
Subject: [PATCH 05/33] remove comment

---
 scripts/workflow/generate_results.sh | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/scripts/workflow/generate_results.sh b/scripts/workflow/generate_results.sh
index fa963a48a..e50a833ce 100644
--- a/scripts/workflow/generate_results.sh
+++ b/scripts/workflow/generate_results.sh
@@ -1,13 +1,11 @@
 #!/bin/bash
 
-# TODO: `start_time` and `end_time`, separate test name from `$line`
-
 JOB_RESULT="$1"
 START_TIME="$2"
 TEST_TYPE="$3"
 
 INPUT_FILE="./test/dashboard/logs/$TEST_TYPE/raw_logs.log"
-JOB_OUTPUT_FILE="./test/dashboard/logs/$TEST_TYPE/result.json"
+RESULT_OUTPUT_FILE="./test/dashboard/logs/$TEST_TYPE/result.json"
 LOG_OUTPUT_FILE="./test/dashboard/logs/$TEST_TYPE/test.log"
 OUTPUT_PATH="./test/dashboard/logs/$TEST_TYPE/"
 
@@ -90,7 +88,7 @@ format_results(){
     done < "$INPUT_FILE"
             
     # Store the result of the whole job
-    echo "{\"start_at\": \"$START_TIME\", \"end_at\": \"$END_TIME\", \"duration_seconds\": \"$DURATION\", \"result\": \"$RESULT\", \"msg\": \"$FAIL_MSG\"}" > $JOB_OUTPUT_FILE
+    echo "{\"start_at\": \"$START_TIME\", \"end_at\": \"$END_TIME\", \"duration_seconds\": \"$DURATION\", \"result\": \"$RESULT\", \"msg\": \"$FAIL_MSG\"}" > $RESULT_OUTPUT_FILE
 }
 
 # Main body of the script

From 5be6f3968dcb9183b03bbb64a32fb66df424af7e Mon Sep 17 00:00:00 2001
From: John David White <joh.white@f5.com>
Date: Mon, 28 Jul 2025 12:17:01 +0100
Subject: [PATCH 06/33] add temp status check

---
 .github/workflows/ci.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index e69e74467..ef2a889d9 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -172,7 +172,7 @@ jobs:
             run: |
                 go install github.com/goreleaser/nfpm/v2/cmd/nfpm@${{ env.NFPM_VERSION }}
                 OS_RELEASE="${{ matrix.container.image }}" OS_VERSION="${{ matrix.container.version }}" \
-                  make integration-test | tee ${{github.workspace}}/test/dashboard/logs/${{github.job}}.${{matrix.container.image}}.${{matrix.container.version}}/raw_logs.log && exit "${PIPESTATUS[0]}"
+                  make integration-test | tee ${{github.workspace}}/test/dashboard/logs/${{github.job}}.${{matrix.container.image}}.${{matrix.container.version}}/raw_logs.log && echo "${PIPESTATUS}" && exit "${PIPESTATUS[0]}"
           - name: Generate Test Results
             if: always()
             run: bash ./scripts/workflow/generate_results.sh ${{job.status}} ${{env.START_TIME}} ${{github.job}}

From 47e2b87f8e2ce6b62e073d5c74d96172576537f6 Mon Sep 17 00:00:00 2001
From: John David White <joh.white@f5.com>
Date: Mon, 28 Jul 2025 13:42:58 +0100
Subject: [PATCH 07/33] add logs

---
 test/integration/api/api_test.go                   | 3 ++-
 test/integration/features/features_test.go         | 3 ++-
 test/integration/install/install_uninstall_test.go | 3 ++-
 3 files changed, 6 insertions(+), 3 deletions(-)

diff --git a/test/integration/api/api_test.go b/test/integration/api/api_test.go
index eddc6d865..1ecd67171 100644
--- a/test/integration/api/api_test.go
+++ b/test/integration/api/api_test.go
@@ -4,7 +4,6 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
-	log "github.com/sirupsen/logrus"
 	"net"
 	"net/http"
 	"os"
@@ -13,6 +12,8 @@ import (
 	"testing"
 	"time"
 
+	log "github.com/sirupsen/logrus"
+
 	"github.com/stretchr/testify/require"
 
 	"github.com/nginx/agent/sdk/v2/proto"
diff --git a/test/integration/features/features_test.go b/test/integration/features/features_test.go
index d65c173d7..0f83a8b2f 100644
--- a/test/integration/features/features_test.go
+++ b/test/integration/features/features_test.go
@@ -2,11 +2,12 @@ package features
 
 import (
 	"context"
-	log "github.com/sirupsen/logrus"
 	"io"
 	"os"
 	"testing"
 
+	log "github.com/sirupsen/logrus"
+
 	"github.com/nginx/agent/test/integration/utils"
 	"github.com/stretchr/testify/assert"
 )
diff --git a/test/integration/install/install_uninstall_test.go b/test/integration/install/install_uninstall_test.go
index 0c9ec9f61..e838491f3 100644
--- a/test/integration/install/install_uninstall_test.go
+++ b/test/integration/install/install_uninstall_test.go
@@ -3,7 +3,6 @@ package install
 import (
 	"context"
 	"fmt"
-	log "github.com/sirupsen/logrus"
 	"io"
 	"os"
 	"path"
@@ -12,6 +11,8 @@ import (
 	"testing"
 	"time"
 
+	log "github.com/sirupsen/logrus"
+
 	"github.com/nginx/agent/test/integration/utils"
 
 	"github.com/stretchr/testify/assert"

From 0f825392cfc807d61c5ed01b4faa64d60b82d243 Mon Sep 17 00:00:00 2001
From: John David White <joh.white@f5.com>
Date: Mon, 28 Jul 2025 14:15:39 +0100
Subject: [PATCH 08/33] add mkdir for sub-directories

---
 .github/workflows/ci.yml             | 14 +++++++-------
 scripts/workflow/generate_results.sh |  9 +++++----
 2 files changed, 12 insertions(+), 11 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index ef2a889d9..7db57e1ee 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -168,14 +168,14 @@ jobs:
           - name: Start Promtail
             uses: ./.github/actions/start-promtail
           - name: Run Integration Tests
-            shell: bash
             run: |
+                mkdir -p ${{github.workspace}}/test/dashboard/logs/${{github.job}}/${{matrix.container.image}}.${{matrix.container.version}}/
                 go install github.com/goreleaser/nfpm/v2/cmd/nfpm@${{ env.NFPM_VERSION }}
                 OS_RELEASE="${{ matrix.container.image }}" OS_VERSION="${{ matrix.container.version }}" \
-                  make integration-test | tee ${{github.workspace}}/test/dashboard/logs/${{github.job}}.${{matrix.container.image}}.${{matrix.container.version}}/raw_logs.log && echo "${PIPESTATUS}" && exit "${PIPESTATUS[0]}"
+                  make integration-test | tee ${{github.workspace}}/test/dashboard/logs/${{github.job}}/${{matrix.container.image}}.${{matrix.container.version}}/raw_logs.log && exit "${PIPESTATUS[0]}"
           - name: Generate Test Results
             if: always()
-            run: bash ./scripts/workflow/generate_results.sh ${{job.status}} ${{env.START_TIME}} ${{github.job}}
+            run: bash ./scripts/workflow/generate_results.sh ${{job.status}} ${{env.START_TIME}} ${{github.job}} ${{ github.workspace }}
           - name: Container Output Logs
             if: failure()
             run: |
@@ -231,11 +231,11 @@ jobs:
           name: nginx-agent-unsigned-snapshots
           path: build
       - name: Run Integration Tests
-        shell: bash
         run: |
+          mkdir -p ${{github.workspace}}/test/dashboard/logs/${{github.job}}/${{matrix.container.image}}.${{matrix.container.version}}/
           go install github.com/goreleaser/nfpm/v2/cmd/nfpm@${{ env.NFPM_VERSION }}
           CONTAINER_NGINX_IMAGE_REGISTRY="${{ env.NGINX_OSS_REGISTRY }}" TAG="${{ matrix.container.version }}-${{ matrix.container.image }}" OS_RELEASE="${{ matrix.container.release }}"\
-            make official-image-integration-test | tee ${{github.workspace}}/test/dashboard/logs/${{github.job}}.${{matrix.container.image}}.${{matrix.container.version}}/raw_logs.log && exit "${PIPESTATUS[0]}"
+            make official-image-integration-test | tee ${{github.workspace}}/test/dashboard/logs/${{github.job}}/${{matrix.container.image}}.${{matrix.container.version}}/raw_logs.log && exit "${PIPESTATUS[0]}"
       - name: Generate Test Results
         if: always()
         run: bash ./scripts/workflow/generate_results.sh ${{job.status}} ${{env.START_TIME}} ${{github.job}}
@@ -308,12 +308,12 @@ jobs:
                 username: ${{ secrets.REGISTRY_USERNAME }}
                 password: ${{ secrets.REGISTRY_PASSWORD }}
           - name: Run Integration Tests
-            shell: bash
             run: |
+                mkdir -p ${{github.workspace}}/test/dashboard/logs/${{github.job}}/${{matrix.container.image}}.${{matrix.container.version}}/
                 go install github.com/goreleaser/nfpm/v2/cmd/nfpm@${{ env.NFPM_VERSION }}
                 CONTAINER_NGINX_IMAGE_REGISTRY="${{ secrets.REGISTRY_URL }}" TAG="${{ matrix.container.plus }}-${{ matrix.container.image }}-${{ matrix.container.version }}" \
                 OS_RELEASE="${{ matrix.container.release }}" IMAGE_PATH="${{ matrix.container.path }}" \
-                  make official-image-integration-test | tee ${{github.workspace}}/test/dashboard/logs/${{github.job}}.${{matrix.container.image}}.${{matrix.container.version}}/raw_logs.log && exit "${PIPESTATUS[0]}"
+                  make official-image-integration-test | tee ${{github.workspace}}/test/dashboard/logs/${{github.job}}/${{matrix.container.image}}.${{matrix.container.version}}/raw_logs.log && exit "${PIPESTATUS[0]}"
           - name: Generate Test Results
             if: always()
             run: bash ./scripts/workflow/generate_results.sh ${{job.status}} ${{env.START_TIME}} ${{github.job}}
diff --git a/scripts/workflow/generate_results.sh b/scripts/workflow/generate_results.sh
index e50a833ce..d3de7b859 100644
--- a/scripts/workflow/generate_results.sh
+++ b/scripts/workflow/generate_results.sh
@@ -3,11 +3,12 @@
 JOB_RESULT="$1"
 START_TIME="$2"
 TEST_TYPE="$3"
+WORKSPACE="$4"
 
-INPUT_FILE="./test/dashboard/logs/$TEST_TYPE/raw_logs.log"
-RESULT_OUTPUT_FILE="./test/dashboard/logs/$TEST_TYPE/result.json"
-LOG_OUTPUT_FILE="./test/dashboard/logs/$TEST_TYPE/test.log"
-OUTPUT_PATH="./test/dashboard/logs/$TEST_TYPE/"
+INPUT_FILE="$WORKSPACE/test/dashboard/logs/$TEST_TYPE/raw_logs.log"
+RESULT_OUTPUT_FILE="$WORKSPACE/test/dashboard/logs/$TEST_TYPE/result.json"
+LOG_OUTPUT_FILE="$WORKSPACE/test/dashboard/logs/$TEST_TYPE/test.log"
+OUTPUT_PATH="$WORKSPACE/test/dashboard/logs/$TEST_TYPE/"
 
 END_TIME="`date "+%Y-%m-%dT%H:%M:%S.%NZ"`"
 START_SECONDS=$(date -d "$START_TIME" +%s.%N)

From 8d2e214876ddc8d40c7482680ea267226495180e Mon Sep 17 00:00:00 2001
From: John David White <joh.white@f5.com>
Date: Mon, 28 Jul 2025 15:42:58 +0100
Subject: [PATCH 09/33] update integration tests

---
 .github/workflows/ci.yml | 32 +++++++-------------------------
 1 file changed, 7 insertions(+), 25 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 7db57e1ee..4cf5e7e1e 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -164,15 +164,17 @@ jobs:
           - name: Set Start Time
             run: echo "START_TIME=$(date +"%Y-%m-%dT%H:%M:%S.%NZ")" >> ${GITHUB_ENV}
           - name: Create Directory for Results
-            run: mkdir -p ${{github.workspace}}/test/dashboard/logs/${{github.job}}/
+            run: mkdir -p ${{github.workspace}}/test/dashboard/logs/${{github.job}}
           - name: Start Promtail
             uses: ./.github/actions/start-promtail
           - name: Run Integration Tests
             run: |
-                mkdir -p ${{github.workspace}}/test/dashboard/logs/${{github.job}}/${{matrix.container.image}}.${{matrix.container.version}}/
+                mkdir -p ${{github.workspace}}/test/dashboard/logs/${{github.job}}/${{matrix.container.image}}.${{matrix.container.version}}
                 go install github.com/goreleaser/nfpm/v2/cmd/nfpm@${{ env.NFPM_VERSION }}
                 OS_RELEASE="${{ matrix.container.image }}" OS_VERSION="${{ matrix.container.version }}" \
-                  make integration-test | tee ${{github.workspace}}/test/dashboard/logs/${{github.job}}/${{matrix.container.image}}.${{matrix.container.version}}/raw_logs.log && exit "${PIPESTATUS[0]}"
+                  make integration-test | tee ${{github.workspace}}/test/dashboard/logs/${{github.job}}/${{matrix.container.image}}.${{matrix.container.version}}/raw_logs.log && exit "${PIPESTATUS[0]}" \
+                echo "THIS IS A DEBUG LINE" \
+                cat ${{github.workspace}}/test/dashboard/logs/${{github.job}}/${{matrix.container.image}}.${{matrix.container.version}}/raw_logs.log
           - name: Generate Test Results
             if: always()
             run: bash ./scripts/workflow/generate_results.sh ${{job.status}} ${{env.START_TIME}} ${{github.job}} ${{ github.workspace }}
@@ -219,12 +221,6 @@ jobs:
       - uses: actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491 # v5.0.0
         with:
           go-version-file: 'go.mod'
-      -   name: Set Start Time
-          run: echo "START_TIME=$(date +"%Y-%m-%dT%H:%M:%S.%NZ")" >> ${GITHUB_ENV}
-      -   name: Create Directory for Results
-          run: mkdir -p ${{github.workspace}}/test/dashboard/logs/${{github.job}}/
-      -   name: Start Promtail
-          uses: ./.github/actions/start-promtail
       - name: Download Packages
         uses: actions/download-artifact@65a9edc5881444af0b9093a5e628f2fe47ea3b2e # v4.1.7
         with:
@@ -232,13 +228,9 @@ jobs:
           path: build
       - name: Run Integration Tests
         run: |
-          mkdir -p ${{github.workspace}}/test/dashboard/logs/${{github.job}}/${{matrix.container.image}}.${{matrix.container.version}}/
           go install github.com/goreleaser/nfpm/v2/cmd/nfpm@${{ env.NFPM_VERSION }}
           CONTAINER_NGINX_IMAGE_REGISTRY="${{ env.NGINX_OSS_REGISTRY }}" TAG="${{ matrix.container.version }}-${{ matrix.container.image }}" OS_RELEASE="${{ matrix.container.release }}"\
-            make official-image-integration-test | tee ${{github.workspace}}/test/dashboard/logs/${{github.job}}/${{matrix.container.image}}.${{matrix.container.version}}/raw_logs.log && exit "${PIPESTATUS[0]}"
-      - name: Generate Test Results
-        if: always()
-        run: bash ./scripts/workflow/generate_results.sh ${{job.status}} ${{env.START_TIME}} ${{github.job}}
+            make official-image-integration-test
       - name: Container Output Logs 
         if: failure()
         run: |
@@ -295,12 +287,6 @@ jobs:
             with:
                 name: nginx-agent-unsigned-snapshots
                 path: build
-          - name: Set Start Time
-            run: echo "START_TIME=$(date +"%Y-%m-%dT%H:%M:%S.%NZ")" >> ${GITHUB_ENV}
-          - name: Create Directory for Results
-            run: mkdir -p ${{github.workspace}}/test/dashboard/logs/${{github.job}}/
-          - name: Start Promtail
-            uses: ./.github/actions/start-promtail
           - name: Login to Docker Registry
             uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v3.3.0
             with:
@@ -309,14 +295,10 @@ jobs:
                 password: ${{ secrets.REGISTRY_PASSWORD }}
           - name: Run Integration Tests
             run: |
-                mkdir -p ${{github.workspace}}/test/dashboard/logs/${{github.job}}/${{matrix.container.image}}.${{matrix.container.version}}/
                 go install github.com/goreleaser/nfpm/v2/cmd/nfpm@${{ env.NFPM_VERSION }}
                 CONTAINER_NGINX_IMAGE_REGISTRY="${{ secrets.REGISTRY_URL }}" TAG="${{ matrix.container.plus }}-${{ matrix.container.image }}-${{ matrix.container.version }}" \
                 OS_RELEASE="${{ matrix.container.release }}" IMAGE_PATH="${{ matrix.container.path }}" \
-                  make official-image-integration-test | tee ${{github.workspace}}/test/dashboard/logs/${{github.job}}/${{matrix.container.image}}.${{matrix.container.version}}/raw_logs.log && exit "${PIPESTATUS[0]}"
-          - name: Generate Test Results
-            if: always()
-            run: bash ./scripts/workflow/generate_results.sh ${{job.status}} ${{env.START_TIME}} ${{github.job}}
+                  make official-image-integration-test
           - name: Container Output Logs
             if: failure()
             run: |

From a6ad44ecd3bc440425e72f5c42768cfc6e72e1dc Mon Sep 17 00:00:00 2001
From: John David White <joh.white@f5.com>
Date: Mon, 28 Jul 2025 16:33:17 +0100
Subject: [PATCH 10/33] fix filepaths

---
 .github/workflows/ci.yml             | 11 ++++-------
 scripts/workflow/generate_results.sh | 16 ++++++++--------
 2 files changed, 12 insertions(+), 15 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 4cf5e7e1e..e3e2cc442 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -163,21 +163,18 @@ jobs:
                 path: build
           - name: Set Start Time
             run: echo "START_TIME=$(date +"%Y-%m-%dT%H:%M:%S.%NZ")" >> ${GITHUB_ENV}
-          - name: Create Directory for Results
-            run: mkdir -p ${{github.workspace}}/test/dashboard/logs/${{github.job}}
           - name: Start Promtail
             uses: ./.github/actions/start-promtail
           - name: Run Integration Tests
             run: |
-                mkdir -p ${{github.workspace}}/test/dashboard/logs/${{github.job}}/${{matrix.container.image}}.${{matrix.container.version}}
+                mkdir -p ${{github.workspace}}/test/dashboard/logs/${{github.job}}/${{matrix.container.image}}${{matrix.container.version}}/
                 go install github.com/goreleaser/nfpm/v2/cmd/nfpm@${{ env.NFPM_VERSION }}
                 OS_RELEASE="${{ matrix.container.image }}" OS_VERSION="${{ matrix.container.version }}" \
-                  make integration-test | tee ${{github.workspace}}/test/dashboard/logs/${{github.job}}/${{matrix.container.image}}.${{matrix.container.version}}/raw_logs.log && exit "${PIPESTATUS[0]}" \
-                echo "THIS IS A DEBUG LINE" \
-                cat ${{github.workspace}}/test/dashboard/logs/${{github.job}}/${{matrix.container.image}}.${{matrix.container.version}}/raw_logs.log
+                  make integration-test | tee ${{github.workspace}}/test/dashboard/logs/${{github.job}}/${{matrix.container.image}}${{matrix.container.version}}/raw_logs.log && exit "${PIPESTATUS[0]}"
+          - run: cat ${{github.workspace}}/test/dashboard/logs/${{github.job}}/${{matrix.container.image}}${{matrix.container.version}}/raw_logs.log
           - name: Generate Test Results
             if: always()
-            run: bash ./scripts/workflow/generate_results.sh ${{job.status}} ${{env.START_TIME}} ${{github.job}} ${{ github.workspace }}
+            run: bash ./scripts/workflow/generate_results.sh ${{job.status}} ${{env.START_TIME}} ${{github.job}}/${{matrix.container.image}}${{matrix.container.version}} ${{github.workspace}}
           - name: Container Output Logs
             if: failure()
             run: |
diff --git a/scripts/workflow/generate_results.sh b/scripts/workflow/generate_results.sh
index d3de7b859..ec2476d52 100644
--- a/scripts/workflow/generate_results.sh
+++ b/scripts/workflow/generate_results.sh
@@ -6,9 +6,8 @@ TEST_TYPE="$3"
 WORKSPACE="$4"
 
 INPUT_FILE="$WORKSPACE/test/dashboard/logs/$TEST_TYPE/raw_logs.log"
-RESULT_OUTPUT_FILE="$WORKSPACE/test/dashboard/logs/$TEST_TYPE/result.json"
-LOG_OUTPUT_FILE="$WORKSPACE/test/dashboard/logs/$TEST_TYPE/test.log"
-OUTPUT_PATH="$WORKSPACE/test/dashboard/logs/$TEST_TYPE/"
+OUTPUT_PATH="$WORKSPACE/test/dashboard/logs/$TEST_TYPE"
+JOB_OUTPUT_FILE="$WORKSPACE/test/dashboard/logs/$TEST_TYPE/result.json"
 
 END_TIME="`date "+%Y-%m-%dT%H:%M:%S.%NZ"`"
 START_SECONDS=$(date -d "$START_TIME" +%s.%N)
@@ -58,7 +57,8 @@ format_results(){
             MSG=""
             TEST_START=""
             TEST_END=""
-            OUTPUT_FILE="${OUTPUT_PATH}${TEST_NAME}/result.json"
+            RESULT_FILE="$OUTPUT_PATH/$TEST_NAME/result.json"
+            LOG_FILE="$OUTPUT_PATH/$TEST_NAME/test.log"
         elif [[ "$line" =~ ([0-9T:\.\-Z]+)[[:space:]]+testing ]]; then
             TEST_START="${BASH_REMATCH[1]}"
         elif [[ "$line" =~ ([0-9T:\.\-Z]+)[[:space:]]+finished[[:space]]testing ]]; then
@@ -68,18 +68,18 @@ format_results(){
             MSG="$MSG_STR"
             FAIL_MSG+="$MSG"
             HAS_FAILED=false
-            echo "{\"start_at\": \"$START_TIME\", \"end_at\": \"$END_TIME\", \"duration_seconds\": \"$DURATION\", \"result\": \"$TEST_RES\", \"msg\": \"$MSG\"}" > $OUTPUT_FILE
+            echo "{\"start_at\": \"$START_TIME\", \"end_at\": \"$END_TIME\", \"duration_seconds\": \"$DURATION\", \"result\": \"$TEST_RES\", \"msg\": \"$MSG\"}" > $RESULT_FILE
         elif [[ "$line" == "--- PASS"* ]]; then
             TEST_RES="pass"
             IS_RUNNING=false
-            echo "{\"start_at\": \"$START_TIME\", \"end_at\": \"$END_TIME\", \"duration_seconds\": \"$DURATION\", \"result\": \"$TEST_RES\", \"msg\": \"$MSG\"}" > $OUTPUT_FILE
+            echo "{\"start_at\": \"$START_TIME\", \"end_at\": \"$END_TIME\", \"duration_seconds\": \"$DURATION\", \"result\": \"$TEST_RES\", \"msg\": \"$MSG\"}" > $RESULT_FILE
         elif [[ "$line" == "--- FAIL"* ]]; then
             TEST_RES="fail"
             HAS_FAILED=true
             IS_RUNNING=false
         elif [[ "$line" == time=* && "$line" == *level=* ]]; then
             LOG_LINE=$(format_logs_to_json "$line")
-            echo "$LOG_LINE" >> "$LOG_OUTPUT_FILE"
+            echo "$LOG_LINE" >> "$LOG_FILE"
         fi
         
         if [ $HAS_FAILED == true ]; then
@@ -89,7 +89,7 @@ format_results(){
     done < "$INPUT_FILE"
             
     # Store the result of the whole job
-    echo "{\"start_at\": \"$START_TIME\", \"end_at\": \"$END_TIME\", \"duration_seconds\": \"$DURATION\", \"result\": \"$RESULT\", \"msg\": \"$FAIL_MSG\"}" > $RESULT_OUTPUT_FILE
+    echo "{\"start_at\": \"$START_TIME\", \"end_at\": \"$END_TIME\", \"duration_seconds\": \"$DURATION\", \"result\": \"$RESULT\", \"msg\": \"$FAIL_MSG\"}" > $JOB_OUTPUT_FILE
 }
 
 # Main body of the script

From 88acdeb706186796adb34cbba6e46c6de7e61772 Mon Sep 17 00:00:00 2001
From: John David White <joh.white@f5.com>
Date: Mon, 28 Jul 2025 16:46:39 +0100
Subject: [PATCH 11/33] add mkdir step

---
 .github/workflows/ci.yml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index e3e2cc442..df936518a 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -163,11 +163,12 @@ jobs:
                 path: build
           - name: Set Start Time
             run: echo "START_TIME=$(date +"%Y-%m-%dT%H:%M:%S.%NZ")" >> ${GITHUB_ENV}
+          - name: Create Directory
+            run: mkdir -p ${{github.workspace}}/test/dashboard/logs/${{github.job}}/${{matrix.container.image}}${{matrix.container.version}}/
           - name: Start Promtail
             uses: ./.github/actions/start-promtail
           - name: Run Integration Tests
             run: |
-                mkdir -p ${{github.workspace}}/test/dashboard/logs/${{github.job}}/${{matrix.container.image}}${{matrix.container.version}}/
                 go install github.com/goreleaser/nfpm/v2/cmd/nfpm@${{ env.NFPM_VERSION }}
                 OS_RELEASE="${{ matrix.container.image }}" OS_VERSION="${{ matrix.container.version }}" \
                   make integration-test | tee ${{github.workspace}}/test/dashboard/logs/${{github.job}}/${{matrix.container.image}}${{matrix.container.version}}/raw_logs.log && exit "${PIPESTATUS[0]}"

From e6ef34198d3f094c193fcd697f0e19e7706bfbb0 Mon Sep 17 00:00:00 2001
From: John David White <joh.white@f5.com>
Date: Wed, 30 Jul 2025 10:12:36 +0100
Subject: [PATCH 12/33] fix regex and move promtail setup

---
 .github/workflows/ci.yml             |  4 ++--
 scripts/workflow/generate_results.sh | 20 +++++++++++---------
 2 files changed, 13 insertions(+), 11 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index df936518a..fdb59ef00 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -165,13 +165,13 @@ jobs:
             run: echo "START_TIME=$(date +"%Y-%m-%dT%H:%M:%S.%NZ")" >> ${GITHUB_ENV}
           - name: Create Directory
             run: mkdir -p ${{github.workspace}}/test/dashboard/logs/${{github.job}}/${{matrix.container.image}}${{matrix.container.version}}/
-          - name: Start Promtail
-            uses: ./.github/actions/start-promtail
           - name: Run Integration Tests
             run: |
                 go install github.com/goreleaser/nfpm/v2/cmd/nfpm@${{ env.NFPM_VERSION }}
                 OS_RELEASE="${{ matrix.container.image }}" OS_VERSION="${{ matrix.container.version }}" \
                   make integration-test | tee ${{github.workspace}}/test/dashboard/logs/${{github.job}}/${{matrix.container.image}}${{matrix.container.version}}/raw_logs.log && exit "${PIPESTATUS[0]}"
+          - name: Start Promtail
+            uses: ./.github/actions/start-promtail
           - run: cat ${{github.workspace}}/test/dashboard/logs/${{github.job}}/${{matrix.container.image}}${{matrix.container.version}}/raw_logs.log
           - name: Generate Test Results
             if: always()
diff --git a/scripts/workflow/generate_results.sh b/scripts/workflow/generate_results.sh
index ec2476d52..4778db39d 100644
--- a/scripts/workflow/generate_results.sh
+++ b/scripts/workflow/generate_results.sh
@@ -31,20 +31,22 @@ load_job_status(){
 }
 
 format_logs_to_json(){
-    local LINE="$1"
-    local JSON="{"
-    local key value
+    line="$1"
+    json="{"
     
-    while [[ "$line" =~ ([a-zA-Z0-9_]+)=((\"[^\"]*\")|([:space:]]+)) ]]; do
+    while [[ "$line" =~ ([a-zA-Z0-9_]+)=((\"([^\"\\]|\\.)*\")|[^[:space:]]+) ]]; do
         key="${BASH_REMATCH[1]}"
         value="${BASH_REMATCH[2]}"
-        LINE="${line#*${BASH_REMATCH[0]}}"
-        json+="\"$key\": \"$value\", "
+        line="${line#*"${key}=${value}"}"
+        
+        if [[ "$value" == \"*\" ]]; then
+            value="${value:1:${#value}-2}"
+            value="${value//\"/\\\"}"
+        fi
+        json+="\"$key\":\"$value\","
     done
     
-    json="${json%, }"
-    json+="}"
-    
+    json="${json%,}}"    
     echo "$json"
 }
 

From 8bfc1415896af2659bdb76cca1f370d27fb1e677 Mon Sep 17 00:00:00 2001
From: John David White <joh.white@f5.com>
Date: Wed, 30 Jul 2025 10:28:44 +0100
Subject: [PATCH 13/33] cleanup

---
 .github/workflows/ci.yml          | 1 -
 test/dashboard/prep/promtail.yaml | 1 -
 2 files changed, 2 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index fdb59ef00..5ac1bce56 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -172,7 +172,6 @@ jobs:
                   make integration-test | tee ${{github.workspace}}/test/dashboard/logs/${{github.job}}/${{matrix.container.image}}${{matrix.container.version}}/raw_logs.log && exit "${PIPESTATUS[0]}"
           - name: Start Promtail
             uses: ./.github/actions/start-promtail
-          - run: cat ${{github.workspace}}/test/dashboard/logs/${{github.job}}/${{matrix.container.image}}${{matrix.container.version}}/raw_logs.log
           - name: Generate Test Results
             if: always()
             run: bash ./scripts/workflow/generate_results.sh ${{job.status}} ${{env.START_TIME}} ${{github.job}}/${{matrix.container.image}}${{matrix.container.version}} ${{github.workspace}}
diff --git a/test/dashboard/prep/promtail.yaml b/test/dashboard/prep/promtail.yaml
index 1ed46f47d..49125be2e 100644
--- a/test/dashboard/prep/promtail.yaml
+++ b/test/dashboard/prep/promtail.yaml
@@ -76,7 +76,6 @@ scrape_configs:
           - labeldrop:
                 - filename
 
-
     - job_name: test-logs
       static_configs:
         - targets:

From 6d9dbbe3083202877d553f085748be30e61d509b Mon Sep 17 00:00:00 2001
From: John David White <joh.white@f5.com>
Date: Wed, 30 Jul 2025 10:36:17 +0100
Subject: [PATCH 14/33] Add mkdir to script

---
 scripts/workflow/generate_results.sh | 1 +
 1 file changed, 1 insertion(+)

diff --git a/scripts/workflow/generate_results.sh b/scripts/workflow/generate_results.sh
index 4778db39d..10ca77e24 100644
--- a/scripts/workflow/generate_results.sh
+++ b/scripts/workflow/generate_results.sh
@@ -59,6 +59,7 @@ format_results(){
             MSG=""
             TEST_START=""
             TEST_END=""
+            mkdir -p "$OUTPUT_PATH/$TEST_NAME/"
             RESULT_FILE="$OUTPUT_PATH/$TEST_NAME/result.json"
             LOG_FILE="$OUTPUT_PATH/$TEST_NAME/test.log"
         elif [[ "$line" =~ ([0-9T:\.\-Z]+)[[:space:]]+testing ]]; then

From 85058aa4e3dac479475388a6b782675b1f3de136 Mon Sep 17 00:00:00 2001
From: John David White <joh.white@f5.com>
Date: Wed, 30 Jul 2025 10:44:04 +0100
Subject: [PATCH 15/33] fix typo

---
 scripts/workflow/generate_results.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/workflow/generate_results.sh b/scripts/workflow/generate_results.sh
index 10ca77e24..0e272866f 100644
--- a/scripts/workflow/generate_results.sh
+++ b/scripts/workflow/generate_results.sh
@@ -64,7 +64,7 @@ format_results(){
             LOG_FILE="$OUTPUT_PATH/$TEST_NAME/test.log"
         elif [[ "$line" =~ ([0-9T:\.\-Z]+)[[:space:]]+testing ]]; then
             TEST_START="${BASH_REMATCH[1]}"
-        elif [[ "$line" =~ ([0-9T:\.\-Z]+)[[:space:]]+finished[[:space]]testing ]]; then
+        elif [[ "$line" =~ ([0-9T:\.\-Z]+)[[:space:]]+finished[[:space:]]testing ]]; then
             TEST_END="${BASH_REMATCH[1]}"
         elif [[ "$line" == "FAIL" ]]; then
             HAS_FAILED=false

From 142b3fea1f3b059767ab137853fbb2f9d3049005 Mon Sep 17 00:00:00 2001
From: John David White <joh.white@f5.com>
Date: Wed, 30 Jul 2025 14:54:27 +0100
Subject: [PATCH 16/33] Move promtail setup

---
 .github/workflows/ci.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 5ac1bce56..200aaf42a 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -165,13 +165,13 @@ jobs:
             run: echo "START_TIME=$(date +"%Y-%m-%dT%H:%M:%S.%NZ")" >> ${GITHUB_ENV}
           - name: Create Directory
             run: mkdir -p ${{github.workspace}}/test/dashboard/logs/${{github.job}}/${{matrix.container.image}}${{matrix.container.version}}/
+          - name: Start Promtail
+            uses: ./.github/actions/start-promtail
           - name: Run Integration Tests
             run: |
                 go install github.com/goreleaser/nfpm/v2/cmd/nfpm@${{ env.NFPM_VERSION }}
                 OS_RELEASE="${{ matrix.container.image }}" OS_VERSION="${{ matrix.container.version }}" \
                   make integration-test | tee ${{github.workspace}}/test/dashboard/logs/${{github.job}}/${{matrix.container.image}}${{matrix.container.version}}/raw_logs.log && exit "${PIPESTATUS[0]}"
-          - name: Start Promtail
-            uses: ./.github/actions/start-promtail
           - name: Generate Test Results
             if: always()
             run: bash ./scripts/workflow/generate_results.sh ${{job.status}} ${{env.START_TIME}} ${{github.job}}/${{matrix.container.image}}${{matrix.container.version}} ${{github.workspace}}

From f2da76fa67c7a3b23ccf97d685aa23982f9b4b81 Mon Sep 17 00:00:00 2001
From: John David White <joh.white@f5.com>
Date: Wed, 30 Jul 2025 15:20:10 +0100
Subject: [PATCH 17/33] prune container before starting promtail

---
 .github/workflows/ci.yml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 200aaf42a..6a6e347e0 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -165,6 +165,8 @@ jobs:
             run: echo "START_TIME=$(date +"%Y-%m-%dT%H:%M:%S.%NZ")" >> ${GITHUB_ENV}
           - name: Create Directory
             run: mkdir -p ${{github.workspace}}/test/dashboard/logs/${{github.job}}/${{matrix.container.image}}${{matrix.container.version}}/
+          - name: Clean Docker
+            run: docker system prune -f
           - name: Start Promtail
             uses: ./.github/actions/start-promtail
           - name: Run Integration Tests

From 0cb955430bba32200e0a0e168e00e26501b0a87e Mon Sep 17 00:00:00 2001
From: John David White <joh.white@f5.com>
Date: Wed, 30 Jul 2025 15:45:56 +0100
Subject: [PATCH 18/33] move cleanup

---
 .github/workflows/ci.yml | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 6a6e347e0..23c7c5481 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -156,6 +156,10 @@ jobs:
           - uses: actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491 # v5.0.0
             with:
                 go-version-file: 'go.mod'
+          - name: Clean Docker Environment
+            run: |
+                docker system prune -af --volumes
+                docker builder prune -af
           - name: Download Packages
             uses: actions/download-artifact@65a9edc5881444af0b9093a5e628f2fe47ea3b2e # v4.1.7
             with:
@@ -165,8 +169,6 @@ jobs:
             run: echo "START_TIME=$(date +"%Y-%m-%dT%H:%M:%S.%NZ")" >> ${GITHUB_ENV}
           - name: Create Directory
             run: mkdir -p ${{github.workspace}}/test/dashboard/logs/${{github.job}}/${{matrix.container.image}}${{matrix.container.version}}/
-          - name: Clean Docker
-            run: docker system prune -f
           - name: Start Promtail
             uses: ./.github/actions/start-promtail
           - name: Run Integration Tests
@@ -183,7 +185,7 @@ jobs:
                 docker ps -a
                 dockerid=$(docker ps -a --format "{{.ID}}")
                 docker logs "$dockerid"
-
+                
           - name: Archive integration test logs
             if: success() || failure()
             uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3

From 891038ca294cca337802d597868aae17084258ea Mon Sep 17 00:00:00 2001
From: John David White <joh.white@f5.com>
Date: Wed, 30 Jul 2025 16:19:00 +0100
Subject: [PATCH 19/33] update promtail

---
 .github/actions/start-promtail/action.yml | 3 ++-
 .github/workflows/ci.yml                  | 8 ++++----
 2 files changed, 6 insertions(+), 5 deletions(-)

diff --git a/.github/actions/start-promtail/action.yml b/.github/actions/start-promtail/action.yml
index 3415319be..5f0a26d75 100644
--- a/.github/actions/start-promtail/action.yml
+++ b/.github/actions/start-promtail/action.yml
@@ -6,8 +6,9 @@ runs:
         - name: Start Promtail container
           shell: bash
           run: |
+              CONTAINER_NAME="promtail=${{github.run_id}}"
               docker run -d \
-               --name=promtail \
+               --name="$CONTAINER_NAME" \
                -v "${{ github.workspace }}/test/dashboard/prep/promtail.yaml:/etc/promtail/config.yaml" \
                -v "${{ github.workspace }}/test/dashboard/logs:/var/log" \
                -e TEST_OUTDIR=test/dashboard/logs \
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 23c7c5481..a737c8061 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -156,10 +156,10 @@ jobs:
           - uses: actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491 # v5.0.0
             with:
                 go-version-file: 'go.mod'
-          - name: Clean Docker Environment
-            run: |
-                docker system prune -af --volumes
-                docker builder prune -af
+#          - name: Clean Docker Environment
+#            run: |
+#                docker system prune -af --volumes
+#                docker builder prune -af
           - name: Download Packages
             uses: actions/download-artifact@65a9edc5881444af0b9093a5e628f2fe47ea3b2e # v4.1.7
             with:

From ab1bac266abbe0407013ae5c50552cc31f1a7aaa Mon Sep 17 00:00:00 2001
From: John David White <joh.white@f5.com>
Date: Wed, 30 Jul 2025 16:30:52 +0100
Subject: [PATCH 20/33] fix typo

---
 .github/actions/start-promtail/action.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/actions/start-promtail/action.yml b/.github/actions/start-promtail/action.yml
index 5f0a26d75..4dbddae6c 100644
--- a/.github/actions/start-promtail/action.yml
+++ b/.github/actions/start-promtail/action.yml
@@ -6,7 +6,7 @@ runs:
         - name: Start Promtail container
           shell: bash
           run: |
-              CONTAINER_NAME="promtail=${{github.run_id}}"
+              CONTAINER_NAME="promtail-${{github.run_id}}"
               docker run -d \
                --name="$CONTAINER_NAME" \
                -v "${{ github.workspace }}/test/dashboard/prep/promtail.yaml:/etc/promtail/config.yaml" \

From 729e6a1a9f64e1c1f862279dcebb1af2fe870117 Mon Sep 17 00:00:00 2001
From: John David White <joh.white@f5.com>
Date: Wed, 30 Jul 2025 16:32:43 +0100
Subject: [PATCH 21/33] update promtail

---
 .github/actions/start-promtail/action.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/actions/start-promtail/action.yml b/.github/actions/start-promtail/action.yml
index 4dbddae6c..19719a880 100644
--- a/.github/actions/start-promtail/action.yml
+++ b/.github/actions/start-promtail/action.yml
@@ -6,7 +6,7 @@ runs:
         - name: Start Promtail container
           shell: bash
           run: |
-              CONTAINER_NAME="promtail-${{github.run_id}}"
+              CONTAINER_NAME="promtail-${{ github.run_id }}-${{ matrix.container.image }}-${{ matrix.container.version }}"
               docker run -d \
                --name="$CONTAINER_NAME" \
                -v "${{ github.workspace }}/test/dashboard/prep/promtail.yaml:/etc/promtail/config.yaml" \

From eb66f0734a3528041015b2cf53c36dc81ea00fd7 Mon Sep 17 00:00:00 2001
From: John David White <joh.white@f5.com>
Date: Wed, 30 Jul 2025 16:51:24 +0100
Subject: [PATCH 22/33] comment promtail

---
 .github/workflows/ci.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index a737c8061..0fc6e3240 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -169,8 +169,8 @@ jobs:
             run: echo "START_TIME=$(date +"%Y-%m-%dT%H:%M:%S.%NZ")" >> ${GITHUB_ENV}
           - name: Create Directory
             run: mkdir -p ${{github.workspace}}/test/dashboard/logs/${{github.job}}/${{matrix.container.image}}${{matrix.container.version}}/
-          - name: Start Promtail
-            uses: ./.github/actions/start-promtail
+#          - name: Start Promtail
+#            uses: ./.github/actions/start-promtail
           - name: Run Integration Tests
             run: |
                 go install github.com/goreleaser/nfpm/v2/cmd/nfpm@${{ env.NFPM_VERSION }}

From f612e57a6bd5943eb3b93b11789d88238e08baf2 Mon Sep 17 00:00:00 2001
From: John David White <joh.white@f5.com>
Date: Thu, 31 Jul 2025 09:00:57 +0100
Subject: [PATCH 23/33] change build dir

---
 test/docker/nginx-oss/rpm/Dockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/docker/nginx-oss/rpm/Dockerfile b/test/docker/nginx-oss/rpm/Dockerfile
index d14dff242..fa4ec6b9b 100644
--- a/test/docker/nginx-oss/rpm/Dockerfile
+++ b/test/docker/nginx-oss/rpm/Dockerfile
@@ -8,7 +8,7 @@ ARG OS_VERSION
 ARG OS_RELEASE
 
 WORKDIR /agent
-COPY ./ /agent
+COPY ./build /agent/build
 COPY $ENTRY_POINT /agent/entrypoint.sh
 
 RUN if [ "$OS_VERSION" = "7" ] && [ "$OS_RELEASE" = "oraclelinux" ]; \

From 6530e2810e620fdda78ebbc1519722eb8212e364 Mon Sep 17 00:00:00 2001
From: John David White <joh.white@f5.com>
Date: Thu, 31 Jul 2025 09:22:43 +0100
Subject: [PATCH 24/33] Add OSS tests

---
 .github/actions/start-promtail/action.yml |  3 +--
 .github/workflows/ci.yml                  | 19 ++++++++++++-------
 2 files changed, 13 insertions(+), 9 deletions(-)

diff --git a/.github/actions/start-promtail/action.yml b/.github/actions/start-promtail/action.yml
index 19719a880..3cd81b172 100644
--- a/.github/actions/start-promtail/action.yml
+++ b/.github/actions/start-promtail/action.yml
@@ -6,9 +6,8 @@ runs:
         - name: Start Promtail container
           shell: bash
           run: |
-              CONTAINER_NAME="promtail-${{ github.run_id }}-${{ matrix.container.image }}-${{ matrix.container.version }}"
               docker run -d \
-               --name="$CONTAINER_NAME" \
+               --name="promtail" \
                -v "${{ github.workspace }}/test/dashboard/prep/promtail.yaml:/etc/promtail/config.yaml" \
                -v "${{ github.workspace }}/test/dashboard/logs:/var/log" \
                -e TEST_OUTDIR=test/dashboard/logs \
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 0fc6e3240..9f109727a 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -156,10 +156,6 @@ jobs:
           - uses: actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491 # v5.0.0
             with:
                 go-version-file: 'go.mod'
-#          - name: Clean Docker Environment
-#            run: |
-#                docker system prune -af --volumes
-#                docker builder prune -af
           - name: Download Packages
             uses: actions/download-artifact@65a9edc5881444af0b9093a5e628f2fe47ea3b2e # v4.1.7
             with:
@@ -169,8 +165,8 @@ jobs:
             run: echo "START_TIME=$(date +"%Y-%m-%dT%H:%M:%S.%NZ")" >> ${GITHUB_ENV}
           - name: Create Directory
             run: mkdir -p ${{github.workspace}}/test/dashboard/logs/${{github.job}}/${{matrix.container.image}}${{matrix.container.version}}/
-#          - name: Start Promtail
-#            uses: ./.github/actions/start-promtail
+          - name: Start Promtail
+            uses: ./.github/actions/start-promtail
           - name: Run Integration Tests
             run: |
                 go install github.com/goreleaser/nfpm/v2/cmd/nfpm@${{ env.NFPM_VERSION }}
@@ -227,11 +223,20 @@ jobs:
         with:
           name: nginx-agent-unsigned-snapshots
           path: build
+      - name: Set Start Time
+        run: echo "START_TIME=$(date +"%Y-%m-%dT%H:%M:%S.%NZ")" >> ${GITHUB_ENV}
+      - name: Create Directory
+        run: mkdir -p ${{github.workspace}}/test/dashboard/logs/${{github.job}}/${{matrix.container.image}}${{matrix.container.version}}/
+      - name: Start Promtail
+        uses: ./.github/actions/start-promtail
       - name: Run Integration Tests
         run: |
           go install github.com/goreleaser/nfpm/v2/cmd/nfpm@${{ env.NFPM_VERSION }}
           CONTAINER_NGINX_IMAGE_REGISTRY="${{ env.NGINX_OSS_REGISTRY }}" TAG="${{ matrix.container.version }}-${{ matrix.container.image }}" OS_RELEASE="${{ matrix.container.release }}"\
-            make official-image-integration-test
+            make official-image-integration-test | tee ${{github.workspace}}/test/dashboard/logs/${{github.job}}/${{matrix.container.image}}${{matrix.container.version}}/raw_logs.log && exit "${PIPESTATUS[0]}"
+      - name: Generate Test Results
+        if: always()
+        run: bash ./scripts/workflow/generate_results.sh ${{job.status}} ${{env.START_TIME}} ${{github.job}}/${{matrix.container.image}}${{matrix.container.version}} ${{github.workspace}}
       - name: Container Output Logs 
         if: failure()
         run: |

From 70f1326f1276a67e61d6a09df9839fee476bfac9 Mon Sep 17 00:00:00 2001
From: John David White <joh.white@f5.com>
Date: Thu, 31 Jul 2025 09:52:18 +0100
Subject: [PATCH 25/33] change dockerfile build

---
 test/docker/nginx-official-image/apk/Dockerfile | 1 -
 test/docker/nginx-official-image/deb/Dockerfile | 1 -
 2 files changed, 2 deletions(-)

diff --git a/test/docker/nginx-official-image/apk/Dockerfile b/test/docker/nginx-official-image/apk/Dockerfile
index dbca0f18c..34d3c1008 100644
--- a/test/docker/nginx-official-image/apk/Dockerfile
+++ b/test/docker/nginx-official-image/apk/Dockerfile
@@ -11,7 +11,6 @@ ARG CONTAINER_OS_TYPE
 
 WORKDIR /agent
 COPY ./build/${PACKAGE_NAME}.${CONTAINER_OS_TYPE} /agent/build/${PACKAGE_NAME}.${CONTAINER_OS_TYPE}
-COPY ./ /agent
 
 RUN apk add --allow-untrusted /agent/build/${PACKAGE_NAME}.${CONTAINER_OS_TYPE}
 
diff --git a/test/docker/nginx-official-image/deb/Dockerfile b/test/docker/nginx-official-image/deb/Dockerfile
index 960e9cceb..ad68acc26 100644
--- a/test/docker/nginx-official-image/deb/Dockerfile
+++ b/test/docker/nginx-official-image/deb/Dockerfile
@@ -12,7 +12,6 @@ ARG CONTAINER_OS_TYPE
 
 WORKDIR /agent
 COPY ./build/${PACKAGE_NAME}.${CONTAINER_OS_TYPE} /agent/build/${PACKAGE_NAME}.${CONTAINER_OS_TYPE}
-COPY ./ /agent
 
 RUN  apt-get update \ 
     && apt install --no-install-recommends --no-install-suggests --allow-downgrades -y /agent/build/${PACKAGE_NAME}.${CONTAINER_OS_TYPE} \ 

From e4c3cba6649b96bd1f740f6a7ac456773fcb9375 Mon Sep 17 00:00:00 2001
From: John David White <joh.white@f5.com>
Date: Thu, 31 Jul 2025 10:13:48 +0100
Subject: [PATCH 26/33] debug

---
 .github/workflows/ci.yml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 9f109727a..218d39480 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -234,9 +234,11 @@ jobs:
           go install github.com/goreleaser/nfpm/v2/cmd/nfpm@${{ env.NFPM_VERSION }}
           CONTAINER_NGINX_IMAGE_REGISTRY="${{ env.NGINX_OSS_REGISTRY }}" TAG="${{ matrix.container.version }}-${{ matrix.container.image }}" OS_RELEASE="${{ matrix.container.release }}"\
             make official-image-integration-test | tee ${{github.workspace}}/test/dashboard/logs/${{github.job}}/${{matrix.container.image}}${{matrix.container.version}}/raw_logs.log && exit "${PIPESTATUS[0]}"
+      - run: cat ${{github.workspace}}/test/dashboard/logs/${{github.job}}/${{matrix.container.image}}${{matrix.container.version}}/raw_logs.log
       - name: Generate Test Results
         if: always()
         run: bash ./scripts/workflow/generate_results.sh ${{job.status}} ${{env.START_TIME}} ${{github.job}}/${{matrix.container.image}}${{matrix.container.version}} ${{github.workspace}}
+      - run: find ${{github.workspace}}/test/dashboard/logs/${{github.job}}/${{matrix.container.image}}${{matrix.container.version}}/ -type f -name "test.log" -o -name "result.json"
       - name: Container Output Logs 
         if: failure()
         run: |

From 5388dc60647cfcba406e107e51c477214b117314 Mon Sep 17 00:00:00 2001
From: John David White <joh.white@f5.com>
Date: Thu, 31 Jul 2025 10:55:58 +0100
Subject: [PATCH 27/33] update path

---
 test/dashboard/prep/promtail.yaml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/test/dashboard/prep/promtail.yaml b/test/dashboard/prep/promtail.yaml
index 49125be2e..f23b75b74 100644
--- a/test/dashboard/prep/promtail.yaml
+++ b/test/dashboard/prep/promtail.yaml
@@ -25,7 +25,7 @@ scrape_configs:
                 - localhost
             labels:
                 systest_job: test-results
-                __path__: /var/log/**/result.json
+                __path__: /var/log/test-results/**/result.json
       pipeline_stages:
           - json:
                 expressions:
@@ -82,7 +82,7 @@ scrape_configs:
             - localhost
           labels:
             systest_job: test-logs
-            __path__: /var/log/**/test.log
+            __path__: /var/log/test-logs/**/test.log
       pipeline_stages:
         - json:
             expressions:

From 95d71f9d9e6ab21e6362360ec273567e78123752 Mon Sep 17 00:00:00 2001
From: John David White <joh.white@f5.com>
Date: Thu, 31 Jul 2025 11:13:20 +0100
Subject: [PATCH 28/33] debug

---
 .github/workflows/ci.yml          | 6 ++++--
 test/dashboard/prep/promtail.yaml | 4 ++--
 2 files changed, 6 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 218d39480..8fb502414 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -234,11 +234,13 @@ jobs:
           go install github.com/goreleaser/nfpm/v2/cmd/nfpm@${{ env.NFPM_VERSION }}
           CONTAINER_NGINX_IMAGE_REGISTRY="${{ env.NGINX_OSS_REGISTRY }}" TAG="${{ matrix.container.version }}-${{ matrix.container.image }}" OS_RELEASE="${{ matrix.container.release }}"\
             make official-image-integration-test | tee ${{github.workspace}}/test/dashboard/logs/${{github.job}}/${{matrix.container.image}}${{matrix.container.version}}/raw_logs.log && exit "${PIPESTATUS[0]}"
-      - run: cat ${{github.workspace}}/test/dashboard/logs/${{github.job}}/${{matrix.container.image}}${{matrix.container.version}}/raw_logs.log
       - name: Generate Test Results
         if: always()
         run: bash ./scripts/workflow/generate_results.sh ${{job.status}} ${{env.START_TIME}} ${{github.job}}/${{matrix.container.image}}${{matrix.container.version}} ${{github.workspace}}
-      - run: find ${{github.workspace}}/test/dashboard/logs/${{github.job}}/${{matrix.container.image}}${{matrix.container.version}}/ -type f -name "test.log" -o -name "result.json"
+      - run: |
+            find ${{github.workspace}}/test/dashboard/logs/${{github.job}}/${{matrix.container.image}}${{matrix.container.version}}/ -type f -name "test.log" -o -name "result.json"
+            cat ${{github.workspace}}/test/dashboard/logs/${{github.job}}/${{matrix.container.image}}${{matrix.container.version}}/test.log
+            cat ${{github.workspace}}/test/dashboard/logs/${{github.job}}/${{matrix.container.image}}${{matrix.container.version}}/result.json
       - name: Container Output Logs 
         if: failure()
         run: |
diff --git a/test/dashboard/prep/promtail.yaml b/test/dashboard/prep/promtail.yaml
index f23b75b74..49125be2e 100644
--- a/test/dashboard/prep/promtail.yaml
+++ b/test/dashboard/prep/promtail.yaml
@@ -25,7 +25,7 @@ scrape_configs:
                 - localhost
             labels:
                 systest_job: test-results
-                __path__: /var/log/test-results/**/result.json
+                __path__: /var/log/**/result.json
       pipeline_stages:
           - json:
                 expressions:
@@ -82,7 +82,7 @@ scrape_configs:
             - localhost
           labels:
             systest_job: test-logs
-            __path__: /var/log/test-logs/**/test.log
+            __path__: /var/log/**/test.log
       pipeline_stages:
         - json:
             expressions:

From 56a495fff3b2b3a8b30a8e6834e23de2556ee319 Mon Sep 17 00:00:00 2001
From: John David White <joh.white@f5.com>
Date: Thu, 31 Jul 2025 11:23:25 +0100
Subject: [PATCH 29/33] fix filename

---
 .github/workflows/ci.yml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 8fb502414..23340ab53 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -239,7 +239,6 @@ jobs:
         run: bash ./scripts/workflow/generate_results.sh ${{job.status}} ${{env.START_TIME}} ${{github.job}}/${{matrix.container.image}}${{matrix.container.version}} ${{github.workspace}}
       - run: |
             find ${{github.workspace}}/test/dashboard/logs/${{github.job}}/${{matrix.container.image}}${{matrix.container.version}}/ -type f -name "test.log" -o -name "result.json"
-            cat ${{github.workspace}}/test/dashboard/logs/${{github.job}}/${{matrix.container.image}}${{matrix.container.version}}/test.log
             cat ${{github.workspace}}/test/dashboard/logs/${{github.job}}/${{matrix.container.image}}${{matrix.container.version}}/result.json
       - name: Container Output Logs 
         if: failure()

From 7664bf87998ff8b1a94cd6bda90220f8234ef10e Mon Sep 17 00:00:00 2001
From: John David White <joh.white@f5.com>
Date: Thu, 31 Jul 2025 11:44:09 +0100
Subject: [PATCH 30/33] Move promtail

---
 .github/actions/start-promtail/action.yml | 2 +-
 .github/workflows/ci.yml                  | 7 ++-----
 2 files changed, 3 insertions(+), 6 deletions(-)

diff --git a/.github/actions/start-promtail/action.yml b/.github/actions/start-promtail/action.yml
index 3cd81b172..3415319be 100644
--- a/.github/actions/start-promtail/action.yml
+++ b/.github/actions/start-promtail/action.yml
@@ -7,7 +7,7 @@ runs:
           shell: bash
           run: |
               docker run -d \
-               --name="promtail" \
+               --name=promtail \
                -v "${{ github.workspace }}/test/dashboard/prep/promtail.yaml:/etc/promtail/config.yaml" \
                -v "${{ github.workspace }}/test/dashboard/logs:/var/log" \
                -e TEST_OUTDIR=test/dashboard/logs \
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 23340ab53..62444c916 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -165,8 +165,6 @@ jobs:
             run: echo "START_TIME=$(date +"%Y-%m-%dT%H:%M:%S.%NZ")" >> ${GITHUB_ENV}
           - name: Create Directory
             run: mkdir -p ${{github.workspace}}/test/dashboard/logs/${{github.job}}/${{matrix.container.image}}${{matrix.container.version}}/
-          - name: Start Promtail
-            uses: ./.github/actions/start-promtail
           - name: Run Integration Tests
             run: |
                 go install github.com/goreleaser/nfpm/v2/cmd/nfpm@${{ env.NFPM_VERSION }}
@@ -175,6 +173,8 @@ jobs:
           - name: Generate Test Results
             if: always()
             run: bash ./scripts/workflow/generate_results.sh ${{job.status}} ${{env.START_TIME}} ${{github.job}}/${{matrix.container.image}}${{matrix.container.version}} ${{github.workspace}}
+          - name: Start Promtail
+            uses: ./.github/actions/start-promtail
           - name: Container Output Logs
             if: failure()
             run: |
@@ -237,9 +237,6 @@ jobs:
       - name: Generate Test Results
         if: always()
         run: bash ./scripts/workflow/generate_results.sh ${{job.status}} ${{env.START_TIME}} ${{github.job}}/${{matrix.container.image}}${{matrix.container.version}} ${{github.workspace}}
-      - run: |
-            find ${{github.workspace}}/test/dashboard/logs/${{github.job}}/${{matrix.container.image}}${{matrix.container.version}}/ -type f -name "test.log" -o -name "result.json"
-            cat ${{github.workspace}}/test/dashboard/logs/${{github.job}}/${{matrix.container.image}}${{matrix.container.version}}/result.json
       - name: Container Output Logs 
         if: failure()
         run: |

From 44908051d80d0677b3f00681f6372e5645d8e550 Mon Sep 17 00:00:00 2001
From: John David White <joh.white@f5.com>
Date: Thu, 31 Jul 2025 12:05:13 +0100
Subject: [PATCH 31/33] pass secret to config

---
 .github/actions/start-promtail/action.yml |  1 +
 .github/workflows/ci.yml                  | 15 ++++++++++++---
 test/dashboard/prep/promtail.yaml         |  4 ++--
 3 files changed, 15 insertions(+), 5 deletions(-)

diff --git a/.github/actions/start-promtail/action.yml b/.github/actions/start-promtail/action.yml
index 3415319be..21fe03e26 100644
--- a/.github/actions/start-promtail/action.yml
+++ b/.github/actions/start-promtail/action.yml
@@ -20,6 +20,7 @@ runs:
                -e GITHUB_HEAD_REF=${{ github.head_ref }} \
                -e GITHUB_SHA=${{ github.sha }} \
                -e GITHUB_ACTOR=${{ github.actor }} \
+               -e LOKI_DASHBOARD_URL=${{ secrets.LOKI_DASHBOARD_URL }} \
                grafana/promtail:3.4.4 \
                -config.file=/etc/promtail/config.yaml \
                -config.expand-env=true
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 62444c916..cfa4f1968 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -165,6 +165,8 @@ jobs:
             run: echo "START_TIME=$(date +"%Y-%m-%dT%H:%M:%S.%NZ")" >> ${GITHUB_ENV}
           - name: Create Directory
             run: mkdir -p ${{github.workspace}}/test/dashboard/logs/${{github.job}}/${{matrix.container.image}}${{matrix.container.version}}/
+          - name: Start Promtail
+            uses: ./.github/actions/start-promtail
           - name: Run Integration Tests
             run: |
                 go install github.com/goreleaser/nfpm/v2/cmd/nfpm@${{ env.NFPM_VERSION }}
@@ -173,8 +175,6 @@ jobs:
           - name: Generate Test Results
             if: always()
             run: bash ./scripts/workflow/generate_results.sh ${{job.status}} ${{env.START_TIME}} ${{github.job}}/${{matrix.container.image}}${{matrix.container.version}} ${{github.workspace}}
-          - name: Start Promtail
-            uses: ./.github/actions/start-promtail
           - name: Container Output Logs
             if: failure()
             run: |
@@ -299,12 +299,21 @@ jobs:
                 registry: ${{ secrets.REGISTRY_URL }}
                 username: ${{ secrets.REGISTRY_USERNAME }}
                 password: ${{ secrets.REGISTRY_PASSWORD }}
+          - name: Set Start Time
+            run: echo "START_TIME=$(date +"%Y-%m-%dT%H:%M:%S.%NZ")" >> ${GITHUB_ENV}
+          - name: Create Directory
+            run: mkdir -p ${{github.workspace}}/test/dashboard/logs/${{github.job}}/${{matrix.container.image}}${{matrix.container.version}}/
+          - name: Start Promtail
+            uses: ./.github/actions/start-promtail
           - name: Run Integration Tests
             run: |
                 go install github.com/goreleaser/nfpm/v2/cmd/nfpm@${{ env.NFPM_VERSION }}
                 CONTAINER_NGINX_IMAGE_REGISTRY="${{ secrets.REGISTRY_URL }}" TAG="${{ matrix.container.plus }}-${{ matrix.container.image }}-${{ matrix.container.version }}" \
                 OS_RELEASE="${{ matrix.container.release }}" IMAGE_PATH="${{ matrix.container.path }}" \
-                  make official-image-integration-test
+                  make official-image-integration-test | tee ${{github.workspace}}/test/dashboard/logs/${{github.job}}/${{matrix.container.image}}${{matrix.container.version}}/raw_logs.log && exit "${PIPESTATUS[0]}"
+          - name: Generate Test Results
+            if: always()
+            run: bash ./scripts/workflow/generate_results.sh ${{job.status}} ${{env.START_TIME}} ${{github.job}}/${{matrix.container.image}}${{matrix.container.version}} ${{github.workspace}}
           - name: Container Output Logs
             if: failure()
             run: |
diff --git a/test/dashboard/prep/promtail.yaml b/test/dashboard/prep/promtail.yaml
index 49125be2e..c327d47b5 100644
--- a/test/dashboard/prep/promtail.yaml
+++ b/test/dashboard/prep/promtail.yaml
@@ -8,10 +8,10 @@ positions:
     sync_period: "10s"
 
 clients:
-    - url: ${{ secrets.LOKI_DASHBOARD_URL }}
+    - url: "${LOKI_DASHBOARD_URL}"
       external_labels:
           systest_project: agent_v2
-          systest_type: ${GITHUB_JOB}
+          systest_type: "${GITHUB_JOB}"
       timeout: 30s
       backoff_config:
           min_period: 500ms

From a95e94864ed9f6a29a5359e8ddf21a3a68798155 Mon Sep 17 00:00:00 2001
From: John David White <joh.white@f5.com>
Date: Thu, 31 Jul 2025 13:36:46 +0100
Subject: [PATCH 32/33] fix secret

---
 .github/actions/start-promtail/action.yml | 6 +++++-
 .github/workflows/ci.yml                  | 6 ++++++
 2 files changed, 11 insertions(+), 1 deletion(-)

diff --git a/.github/actions/start-promtail/action.yml b/.github/actions/start-promtail/action.yml
index 21fe03e26..c9c38227c 100644
--- a/.github/actions/start-promtail/action.yml
+++ b/.github/actions/start-promtail/action.yml
@@ -1,5 +1,9 @@
 name: Start Promtail
 description: Start promtail in a Docker container to send test results to loki
+inputs:
+    loki-dashboard-url:
+        description: "URL of the Loki dashboard to send the logs"
+        required: true
 runs:
     using: "composite"
     steps:
@@ -20,7 +24,7 @@ runs:
                -e GITHUB_HEAD_REF=${{ github.head_ref }} \
                -e GITHUB_SHA=${{ github.sha }} \
                -e GITHUB_ACTOR=${{ github.actor }} \
-               -e LOKI_DASHBOARD_URL=${{ secrets.LOKI_DASHBOARD_URL }} \
+               -e LOKI_DASHBOARD_URL=${{ inputs.loki-dashboard-url }} \
                grafana/promtail:3.4.4 \
                -config.file=/etc/promtail/config.yaml \
                -config.expand-env=true
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index cfa4f1968..5bdef4663 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -167,6 +167,8 @@ jobs:
             run: mkdir -p ${{github.workspace}}/test/dashboard/logs/${{github.job}}/${{matrix.container.image}}${{matrix.container.version}}/
           - name: Start Promtail
             uses: ./.github/actions/start-promtail
+            with:
+                loki-dashboard-url: ${{ secrets.LOKI_DASHBOARD_URL }}
           - name: Run Integration Tests
             run: |
                 go install github.com/goreleaser/nfpm/v2/cmd/nfpm@${{ env.NFPM_VERSION }}
@@ -229,6 +231,8 @@ jobs:
         run: mkdir -p ${{github.workspace}}/test/dashboard/logs/${{github.job}}/${{matrix.container.image}}${{matrix.container.version}}/
       - name: Start Promtail
         uses: ./.github/actions/start-promtail
+        with:
+            loki-dashboard-url: ${{ secrets.LOKI_DASHBOARD_URL }}
       - name: Run Integration Tests
         run: |
           go install github.com/goreleaser/nfpm/v2/cmd/nfpm@${{ env.NFPM_VERSION }}
@@ -305,6 +309,8 @@ jobs:
             run: mkdir -p ${{github.workspace}}/test/dashboard/logs/${{github.job}}/${{matrix.container.image}}${{matrix.container.version}}/
           - name: Start Promtail
             uses: ./.github/actions/start-promtail
+            with:
+                loki-dashboard-url: ${{ secrets.LOKI_DASHBOARD_URL }}
           - name: Run Integration Tests
             run: |
                 go install github.com/goreleaser/nfpm/v2/cmd/nfpm@${{ env.NFPM_VERSION }}

From 1fd361bf0844885eebf3467dafafd65030b9ecde Mon Sep 17 00:00:00 2001
From: John David White <joh.white@f5.com>
Date: Thu, 31 Jul 2025 13:53:41 +0100
Subject: [PATCH 33/33] add unit tests

---
 .github/workflows/ci.yml | 13 ++++++++++++-
 Makefile                 |  8 ++++----
 2 files changed, 16 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 5bdef4663..24fe9c449 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -57,8 +57,19 @@ jobs:
       - uses: actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491 # v5.0.0
         with:
           go-version-file: 'go.mod'
+      - name: Set Start Time
+        run: echo "START_TIME=$(date +"%Y-%m-%dT%H:%M:%S.%NZ")" >> ${GITHUB_ENV}
+      - name: Create Directory
+        run: mkdir -p ${{github.workspace}}/test/dashboard/logs/${{github.job}}/
+      - name: Start Promtail
+        uses: ./.github/actions/start-promtail
+        with:
+            loki-dashboard-url: ${{ secrets.LOKI_DASHBOARD_URL }}
       - name: Run Unit Tests
-        run: make unit-test
+        run: make unit-test | tee ${{github.workspace}}/test/dashboard/logs/${{github.job}}/raw_logs.log && exit "${PIPESTATUS[0]}"
+      - name: Generate Test Results
+        if: always()
+        run: bash ./scripts/workflow/generate_results.sh ${{job.status}} ${{env.START_TIME}} ${{github.job}} ${{github.workspace}}
       - name: Upload Test Coverage
         uses: codecov/codecov-action@84508663e988701840491b86de86b666e8a86bed # v4.3.0
         with:
diff --git a/Makefile b/Makefile
index e541ac1b4..6b17823e0 100644
--- a/Makefile
+++ b/Makefile
@@ -193,16 +193,16 @@ unit-test: $(TEST_BUILD_DIR) test-core test-plugins test-sdk test-extensions ##
 	@printf "\nTotal code coverage: " && $(GOTOOL) cover -func=$(TEST_BUILD_DIR)/coverage.out | grep 'total:' | awk '{print $$3}'
 
 test-core: $(TEST_BUILD_DIR) ## Run core unit tests
-	GOWORK=off CGO_ENABLED=0 go test -count=1 -coverprofile=$(TEST_BUILD_DIR)/core_coverage.out -covermode count ./src/core/...
+	GOWORK=off CGO_ENABLED=0 go test -v -count=1 -coverprofile=$(TEST_BUILD_DIR)/core_coverage.out -covermode count ./src/core/...
 
 test-plugins: $(TEST_BUILD_DIR) ## Run plugins unit tests
-	GOWORK=off CGO_ENABLED=0 go test -count=1 -coverprofile=$(TEST_BUILD_DIR)/plugins_coverage.out -covermode count ./src/plugins/...
+	GOWORK=off CGO_ENABLED=0 go test -v -count=1 -coverprofile=$(TEST_BUILD_DIR)/plugins_coverage.out -covermode count ./src/plugins/...
 
 test-extensions: $(TEST_BUILD_DIR) ## Run extensions unit tests
-	GOWORK=off CGO_ENABLED=0 go test -count=1 -coverprofile=$(TEST_BUILD_DIR)/extensions_coverage.out -covermode count ./src/extensions/...
+	GOWORK=off CGO_ENABLED=0 go test -v -count=1 -coverprofile=$(TEST_BUILD_DIR)/extensions_coverage.out -covermode count ./src/extensions/...
 
 test-sdk: $(TEST_BUILD_DIR) ## Run sdk unit tests from root directory
-	cd sdk && GOWORK=off CGO_ENABLED=0 go test -count=1 -coverprofile=../$(TEST_BUILD_DIR)/sdk_coverage.out -covermode count ./...
+	cd sdk && GOWORK=off CGO_ENABLED=0 go test -v -count=1 -coverprofile=../$(TEST_BUILD_DIR)/sdk_coverage.out -covermode count ./...
 
 # Component tests
 component-test: test-component-build test-component-run ## Run component tests