From 6e194778992dc4d0f900237f89059bd17cde8b29 Mon Sep 17 00:00:00 2001 From: doomwastaken Date: Wed, 2 Apr 2025 14:59:51 +0300 Subject: [PATCH 01/11] added retries to QA workflows for science, I know what DRY means, didnt want to make extra yaml files and flood workflows --- .github/workflows/unit_tests.yml | 147 +++++++++++++++++++++++++++-- .github/workflows/updater_test.yml | 99 +++++++++++++++++-- 2 files changed, 233 insertions(+), 13 deletions(-) diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml index d8d83abb8e3..76d815487fd 100644 --- a/.github/workflows/unit_tests.yml +++ b/.github/workflows/unit_tests.yml @@ -9,8 +9,10 @@ env: FBT_GIT_SUBMODULE_SHALLOW: 1 jobs: - run_units_on_bench: + unit_attempt_1: runs-on: [ self-hosted, FlipperZeroTest ] + outputs: + timed_out: ${{ steps.check_timeout.outputs.timed_out }} steps: - name: Checkout code uses: actions/checkout@v4 @@ -18,19 +20,149 @@ jobs: fetch-depth: 1 ref: ${{ github.event.pull_request.head.sha }} - - name: 'Flash unit tests firmware' + - name: Flash unit tests firmware id: flashing - if: success() timeout-minutes: 5 + continue-on-error: true run: | source scripts/toolchain/fbtenv.sh ./fbt resources firmware_latest flash LIB_DEBUG=1 FIRMWARE_APP_SET=unit_tests FORCE=1 + - name: Copy assets and unit data, reboot and wait for flipper + id: copy + if: steps.flashing.outcome == 'success' + timeout-minutes: 5 + continue-on-error: true + run: | + source scripts/toolchain/fbtenv.sh + python3 scripts/testops.py -t=15 await_flipper + python3 scripts/storage.py -f send build/latest/resources /ext + python3 scripts/storage.py -f send /region_data /ext/.int/.region_data + python3 scripts/power.py reboot + python3 scripts/testops.py -t=30 await_flipper + + - name: Run units and validate results + id: run_units + if: steps.copy.outcome == 'success' + timeout-minutes: 5 + continue-on-error: true + run: | + source scripts/toolchain/fbtenv.sh + python3 scripts/testops.py run_units + + - name: Upload test results + if: failure() && steps.flashing.outcome == 'success' && steps.run_units.outcome != 'skipped' + uses: actions/upload-artifact@v4 + with: + name: unit-tests_output + path: unit_tests*.txt + + - name: Check GDB output + if: failure() && steps.flashing.outcome == 'success' + run: | + ./fbt gdb_trace_all LIB_DEBUG=1 FIRMWARE_APP_SET=unit_tests FORCE=1 + + - name: Check for timeout + id: check_timeout + run: | + if [ "${{ steps.flashing.conclusion }}" == "timed_out" ] || \ + [ "${{ steps.copy.conclusion }}" == "timed_out" ] || \ + [ "${{ steps.run_units.conclusion }}" == "timed_out" ]; then + echo "timed_out=true" >> $GITHUB_OUTPUT + else + echo "timed_out=false" >> $GITHUB_OUTPUT + fi + + unit_attempt_2: + needs: unit_attempt_1 + if: needs.unit_attempt_1.outputs.timed_out == 'true' + runs-on: [ self-hosted, FlipperZeroTest ] + outputs: + timed_out: ${{ steps.check_timeout.outputs.timed_out }} + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + fetch-depth: 1 + ref: ${{ github.event.pull_request.head.sha }} + + - name: Flash unit tests firmware + id: flashing + timeout-minutes: 5 + continue-on-error: true + run: | + source scripts/toolchain/fbtenv.sh + ./fbt resources firmware_latest flash LIB_DEBUG=1 FIRMWARE_APP_SET=unit_tests FORCE=1 + + - name: Copy assets and unit data, reboot and wait for flipper + id: copy + if: steps.flashing.outcome == 'success' + timeout-minutes: 5 + continue-on-error: true + run: | + source scripts/toolchain/fbtenv.sh + python3 scripts/testops.py -t=15 await_flipper + python3 scripts/storage.py -f send build/latest/resources /ext + python3 scripts/storage.py -f send /region_data /ext/.int/.region_data + python3 scripts/power.py reboot + python3 scripts/testops.py -t=30 await_flipper + + - name: Run units and validate results + id: run_units + if: steps.copy.outcome == 'success' + timeout-minutes: 5 + continue-on-error: true + run: | + source scripts/toolchain/fbtenv.sh + python3 scripts/testops.py run_units + + - name: Upload test results + if: failure() && steps.flashing.outcome == 'success' && steps.run_units.outcome != 'skipped' + uses: actions/upload-artifact@v4 + with: + name: unit-tests_output + path: unit_tests*.txt + + - name: Check GDB output + if: failure() && steps.flashing.outcome == 'success' + run: | + ./fbt gdb_trace_all LIB_DEBUG=1 FIRMWARE_APP_SET=unit_tests FORCE=1 + + - name: Check for timeout + id: check_timeout + run: | + if [ "${{ steps.flashing.conclusion }}" == "timed_out" ] || \ + [ "${{ steps.copy.conclusion }}" == "timed_out" ] || \ + [ "${{ steps.run_units.conclusion }}" == "timed_out" ]; then + echo "timed_out=true" >> $GITHUB_OUTPUT + else + echo "timed_out=false" >> $GITHUB_OUTPUT + fi + + unit_attempt_3: + needs: unit_attempt_2 + if: needs.unit_attempt_2.outputs.timed_out == 'true' + runs-on: [ self-hosted, FlipperZeroTest ] + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + fetch-depth: 1 + ref: ${{ github.event.pull_request.head.sha }} + + - name: Flash unit tests firmware + id: flashing + timeout-minutes: 5 + continue-on-error: true + run: | + source scripts/toolchain/fbtenv.sh + ./fbt resources firmware_latest flash LIB_DEBUG=1 FIRMWARE_APP_SET=unit_tests FORCE=1 - - name: 'Copy assets and unit data, reboot and wait for flipper' + - name: Copy assets and unit data, reboot and wait for flipper id: copy if: steps.flashing.outcome == 'success' timeout-minutes: 5 + continue-on-error: true run: | source scripts/toolchain/fbtenv.sh python3 scripts/testops.py -t=15 await_flipper @@ -39,22 +171,23 @@ jobs: python3 scripts/power.py reboot python3 scripts/testops.py -t=30 await_flipper - - name: 'Run units and validate results' + - name: Run units and validate results id: run_units if: steps.copy.outcome == 'success' timeout-minutes: 5 + continue-on-error: true run: | source scripts/toolchain/fbtenv.sh python3 scripts/testops.py run_units - - name: 'Upload test results' + - name: Upload test results if: failure() && steps.flashing.outcome == 'success' && steps.run_units.outcome != 'skipped' uses: actions/upload-artifact@v4 with: name: unit-tests_output path: unit_tests*.txt - - name: 'Check GDB output' + - name: Check GDB output if: failure() && steps.flashing.outcome == 'success' run: | ./fbt gdb_trace_all LIB_DEBUG=1 FIRMWARE_APP_SET=unit_tests FORCE=1 diff --git a/.github/workflows/updater_test.yml b/.github/workflows/updater_test.yml index b5265df9c91..2c7851cef3b 100644 --- a/.github/workflows/updater_test.yml +++ b/.github/workflows/updater_test.yml @@ -9,8 +9,10 @@ env: FBT_GIT_SUBMODULE_SHALLOW: 1 jobs: - test_updater_on_bench: - runs-on: [self-hosted, FlipperZeroTest ] + attempt1: + runs-on: [self-hosted, FlipperZeroTest] + outputs: + timed_out: ${{ steps.check_timeout.outputs.timed_out }} steps: - name: Checkout code uses: actions/checkout@v4 @@ -18,22 +20,107 @@ jobs: fetch-depth: 1 ref: ${{ github.event.pull_request.head.sha }} - - name: 'Flashing target firmware' + - name: Flashing target firmware id: first_full_flash timeout-minutes: 5 + continue-on-error: true run: | source scripts/toolchain/fbtenv.sh python3 scripts/testops.py -t=180 await_flipper ./fbt flash_usb_full FORCE=1 - - - name: 'Validating updater' + - name: Validating updater id: second_full_flash + if: steps.first_full_flash.outcome == 'success' timeout-minutes: 5 - if: success() + continue-on-error: true run: | source scripts/toolchain/fbtenv.sh python3 scripts/testops.py -t=180 await_flipper ./fbt flash_usb FORCE=1 python3 scripts/testops.py -t=180 await_flipper + - name: Check for timeout + id: check_timeout + run: | + if grep -q "Timeout" "${{ github.workspace }}/logs.txt" 2>/dev/null; then + echo "timed_out=true" >> $GITHUB_OUTPUT + elif [ "${{ steps.first_full_flash.outcome }}" == "failure" ] && [ "${{ steps.first_full_flash.conclusion }}" == "timed_out" ]; then + echo "timed_out=true" >> $GITHUB_OUTPUT + elif [ "${{ steps.second_full_flash.outcome }}" == "failure" ] && [ "${{ steps.second_full_flash.conclusion }}" == "timed_out" ]; then + echo "timed_out=true" >> $GITHUB_OUTPUT + else + echo "timed_out=false" >> $GITHUB_OUTPUT + fi + + attempt2: + needs: attempt1 + if: needs.attempt1.outputs.timed_out == 'true' + runs-on: [self-hosted, FlipperZeroTest] + outputs: + timed_out: ${{ steps.check_timeout.outputs.timed_out }} + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + fetch-depth: 1 + ref: ${{ github.event.pull_request.head.sha }} + + - name: Flashing target firmware + id: first_full_flash + timeout-minutes: 5 + continue-on-error: true + run: | + source scripts/toolchain/fbtenv.sh + python3 scripts/testops.py -t=180 await_flipper + ./fbt flash_usb_full FORCE=1 + + - name: Validating updater + id: second_full_flash + if: steps.first_full_flash.outcome == 'success' + timeout-minutes: 5 + continue-on-error: true + run: | + source scripts/toolchain/fbtenv.sh + python3 scripts/testops.py -t=180 await_flipper + ./fbt flash_usb FORCE=1 + python3 scripts/testops.py -t=180 await_flipper + + - name: Check for timeout + id: check_timeout + run: | + if grep -q "Timeout" "${{ github.workspace }}/logs.txt" 2>/dev/null; then + echo "timed_out=true" >> $GITHUB_OUTPUT + elif [ "${{ steps.first_full_flash.outcome }}" == "failure" ] && [ "${{ steps.first_full_flash.conclusion }}" == "timed_out" ]; then + echo "timed_out=true" >> $GITHUB_OUTPUT + elif [ "${{ steps.second_full_flash.outcome }}" == "failure" ] && [ "${{ steps.second_full_flash.conclusion }}" == "timed_out" ]; then + echo "timed_out=true" >> $GITHUB_OUTPUT + else + echo "timed_out=false" >> $GITHUB_OUTPUT + fi + + attempt3: + needs: attempt2 + if: needs.attempt2.outputs.timed_out == 'true' + runs-on: [self-hosted, FlipperZeroTest] + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + fetch-depth: 1 + ref: ${{ github.event.pull_request.head.sha }} + + - name: Flashing target firmware + timeout-minutes: 5 + run: | + source scripts/toolchain/fbtenv.sh + python3 scripts/testops.py -t=180 await_flipper + ./fbt flash_usb_full FORCE=1 + + - name: Validating updater + timeout-minutes: 5 + run: | + source scripts/toolchain/fbtenv.sh + python3 scripts/testops.py -t=180 await_flipper + ./fbt flash_usb FORCE=1 + python3 scripts/testops.py -t=180 await_flipper From 3080a433e8b89eee3497ec864ad33b874af7c805 Mon Sep 17 00:00:00 2001 From: doomwastaken Date: Wed, 2 Apr 2025 16:56:52 +0300 Subject: [PATCH 02/11] reverted back to workflows from actions, branch reset, added faulty timeout to units for tests, retries set to 2 --- .github/workflows/unit_tests.yml | 70 +----------------------------- .github/workflows/updater_test.yml | 62 ++++---------------------- 2 files changed, 10 insertions(+), 122 deletions(-) diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml index 76d815487fd..46a16371e1d 100644 --- a/.github/workflows/unit_tests.yml +++ b/.github/workflows/unit_tests.yml @@ -22,7 +22,7 @@ jobs: - name: Flash unit tests firmware id: flashing - timeout-minutes: 5 + timeout-minutes: 1 continue-on-error: true run: | source scripts/toolchain/fbtenv.sh @@ -31,7 +31,7 @@ jobs: - name: Copy assets and unit data, reboot and wait for flipper id: copy if: steps.flashing.outcome == 'success' - timeout-minutes: 5 + timeout-minutes: 2 continue-on-error: true run: | source scripts/toolchain/fbtenv.sh @@ -77,72 +77,6 @@ jobs: needs: unit_attempt_1 if: needs.unit_attempt_1.outputs.timed_out == 'true' runs-on: [ self-hosted, FlipperZeroTest ] - outputs: - timed_out: ${{ steps.check_timeout.outputs.timed_out }} - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - fetch-depth: 1 - ref: ${{ github.event.pull_request.head.sha }} - - - name: Flash unit tests firmware - id: flashing - timeout-minutes: 5 - continue-on-error: true - run: | - source scripts/toolchain/fbtenv.sh - ./fbt resources firmware_latest flash LIB_DEBUG=1 FIRMWARE_APP_SET=unit_tests FORCE=1 - - - name: Copy assets and unit data, reboot and wait for flipper - id: copy - if: steps.flashing.outcome == 'success' - timeout-minutes: 5 - continue-on-error: true - run: | - source scripts/toolchain/fbtenv.sh - python3 scripts/testops.py -t=15 await_flipper - python3 scripts/storage.py -f send build/latest/resources /ext - python3 scripts/storage.py -f send /region_data /ext/.int/.region_data - python3 scripts/power.py reboot - python3 scripts/testops.py -t=30 await_flipper - - - name: Run units and validate results - id: run_units - if: steps.copy.outcome == 'success' - timeout-minutes: 5 - continue-on-error: true - run: | - source scripts/toolchain/fbtenv.sh - python3 scripts/testops.py run_units - - - name: Upload test results - if: failure() && steps.flashing.outcome == 'success' && steps.run_units.outcome != 'skipped' - uses: actions/upload-artifact@v4 - with: - name: unit-tests_output - path: unit_tests*.txt - - - name: Check GDB output - if: failure() && steps.flashing.outcome == 'success' - run: | - ./fbt gdb_trace_all LIB_DEBUG=1 FIRMWARE_APP_SET=unit_tests FORCE=1 - - - name: Check for timeout - id: check_timeout - run: | - if [ "${{ steps.flashing.conclusion }}" == "timed_out" ] || \ - [ "${{ steps.copy.conclusion }}" == "timed_out" ] || \ - [ "${{ steps.run_units.conclusion }}" == "timed_out" ]; then - echo "timed_out=true" >> $GITHUB_OUTPUT - else - echo "timed_out=false" >> $GITHUB_OUTPUT - fi - - unit_attempt_3: - needs: unit_attempt_2 - if: needs.unit_attempt_2.outputs.timed_out == 'true' - runs-on: [ self-hosted, FlipperZeroTest ] steps: - name: Checkout code uses: actions/checkout@v4 diff --git a/.github/workflows/updater_test.yml b/.github/workflows/updater_test.yml index 2c7851cef3b..6e5f8d51e37 100644 --- a/.github/workflows/updater_test.yml +++ b/.github/workflows/updater_test.yml @@ -9,7 +9,7 @@ env: FBT_GIT_SUBMODULE_SHALLOW: 1 jobs: - attempt1: + updater_attempt1: runs-on: [self-hosted, FlipperZeroTest] outputs: timed_out: ${{ steps.check_timeout.outputs.timed_out }} @@ -22,7 +22,7 @@ jobs: - name: Flashing target firmware id: first_full_flash - timeout-minutes: 5 + timeout-minutes: 4 continue-on-error: true run: | source scripts/toolchain/fbtenv.sh @@ -32,7 +32,7 @@ jobs: - name: Validating updater id: second_full_flash if: steps.first_full_flash.outcome == 'success' - timeout-minutes: 5 + timeout-minutes: 4 continue-on-error: true run: | source scripts/toolchain/fbtenv.sh @@ -53,55 +53,9 @@ jobs: echo "timed_out=false" >> $GITHUB_OUTPUT fi - attempt2: - needs: attempt1 - if: needs.attempt1.outputs.timed_out == 'true' - runs-on: [self-hosted, FlipperZeroTest] - outputs: - timed_out: ${{ steps.check_timeout.outputs.timed_out }} - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - fetch-depth: 1 - ref: ${{ github.event.pull_request.head.sha }} - - - name: Flashing target firmware - id: first_full_flash - timeout-minutes: 5 - continue-on-error: true - run: | - source scripts/toolchain/fbtenv.sh - python3 scripts/testops.py -t=180 await_flipper - ./fbt flash_usb_full FORCE=1 - - - name: Validating updater - id: second_full_flash - if: steps.first_full_flash.outcome == 'success' - timeout-minutes: 5 - continue-on-error: true - run: | - source scripts/toolchain/fbtenv.sh - python3 scripts/testops.py -t=180 await_flipper - ./fbt flash_usb FORCE=1 - python3 scripts/testops.py -t=180 await_flipper - - - name: Check for timeout - id: check_timeout - run: | - if grep -q "Timeout" "${{ github.workspace }}/logs.txt" 2>/dev/null; then - echo "timed_out=true" >> $GITHUB_OUTPUT - elif [ "${{ steps.first_full_flash.outcome }}" == "failure" ] && [ "${{ steps.first_full_flash.conclusion }}" == "timed_out" ]; then - echo "timed_out=true" >> $GITHUB_OUTPUT - elif [ "${{ steps.second_full_flash.outcome }}" == "failure" ] && [ "${{ steps.second_full_flash.conclusion }}" == "timed_out" ]; then - echo "timed_out=true" >> $GITHUB_OUTPUT - else - echo "timed_out=false" >> $GITHUB_OUTPUT - fi - - attempt3: - needs: attempt2 - if: needs.attempt2.outputs.timed_out == 'true' + updater_attempt2: + needs: updater_attempt1 + if: needs.updater_attempt1.outputs.timed_out == 'true' runs-on: [self-hosted, FlipperZeroTest] steps: - name: Checkout code @@ -111,14 +65,14 @@ jobs: ref: ${{ github.event.pull_request.head.sha }} - name: Flashing target firmware - timeout-minutes: 5 + timeout-minutes: 4 run: | source scripts/toolchain/fbtenv.sh python3 scripts/testops.py -t=180 await_flipper ./fbt flash_usb_full FORCE=1 - name: Validating updater - timeout-minutes: 5 + timeout-minutes: 4 run: | source scripts/toolchain/fbtenv.sh python3 scripts/testops.py -t=180 await_flipper From 093081aa458bd46b16298ca89b92bbe7726ea06e Mon Sep 17 00:00:00 2001 From: doomwastaken Date: Wed, 2 Apr 2025 17:08:07 +0300 Subject: [PATCH 03/11] removed continue on error, still has failure set timeout, added check for failure on output stage --- .github/workflows/unit_tests.yml | 7 +------ .github/workflows/updater_test.yml | 3 +-- 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml index 46a16371e1d..ee9f8da29a6 100644 --- a/.github/workflows/unit_tests.yml +++ b/.github/workflows/unit_tests.yml @@ -23,7 +23,6 @@ jobs: - name: Flash unit tests firmware id: flashing timeout-minutes: 1 - continue-on-error: true run: | source scripts/toolchain/fbtenv.sh ./fbt resources firmware_latest flash LIB_DEBUG=1 FIRMWARE_APP_SET=unit_tests FORCE=1 @@ -32,7 +31,6 @@ jobs: id: copy if: steps.flashing.outcome == 'success' timeout-minutes: 2 - continue-on-error: true run: | source scripts/toolchain/fbtenv.sh python3 scripts/testops.py -t=15 await_flipper @@ -45,7 +43,6 @@ jobs: id: run_units if: steps.copy.outcome == 'success' timeout-minutes: 5 - continue-on-error: true run: | source scripts/toolchain/fbtenv.sh python3 scripts/testops.py run_units @@ -64,6 +61,7 @@ jobs: - name: Check for timeout id: check_timeout + if: failure() run: | if [ "${{ steps.flashing.conclusion }}" == "timed_out" ] || \ [ "${{ steps.copy.conclusion }}" == "timed_out" ] || \ @@ -87,7 +85,6 @@ jobs: - name: Flash unit tests firmware id: flashing timeout-minutes: 5 - continue-on-error: true run: | source scripts/toolchain/fbtenv.sh ./fbt resources firmware_latest flash LIB_DEBUG=1 FIRMWARE_APP_SET=unit_tests FORCE=1 @@ -96,7 +93,6 @@ jobs: id: copy if: steps.flashing.outcome == 'success' timeout-minutes: 5 - continue-on-error: true run: | source scripts/toolchain/fbtenv.sh python3 scripts/testops.py -t=15 await_flipper @@ -109,7 +105,6 @@ jobs: id: run_units if: steps.copy.outcome == 'success' timeout-minutes: 5 - continue-on-error: true run: | source scripts/toolchain/fbtenv.sh python3 scripts/testops.py run_units diff --git a/.github/workflows/updater_test.yml b/.github/workflows/updater_test.yml index 6e5f8d51e37..45c705f413c 100644 --- a/.github/workflows/updater_test.yml +++ b/.github/workflows/updater_test.yml @@ -23,7 +23,6 @@ jobs: - name: Flashing target firmware id: first_full_flash timeout-minutes: 4 - continue-on-error: true run: | source scripts/toolchain/fbtenv.sh python3 scripts/testops.py -t=180 await_flipper @@ -33,7 +32,6 @@ jobs: id: second_full_flash if: steps.first_full_flash.outcome == 'success' timeout-minutes: 4 - continue-on-error: true run: | source scripts/toolchain/fbtenv.sh python3 scripts/testops.py -t=180 await_flipper @@ -42,6 +40,7 @@ jobs: - name: Check for timeout id: check_timeout + if: failure() run: | if grep -q "Timeout" "${{ github.workspace }}/logs.txt" 2>/dev/null; then echo "timed_out=true" >> $GITHUB_OUTPUT From 5916d2b49e355b8e6abfd27fb356b8c009f69aae Mon Sep 17 00:00:00 2001 From: doomwastaken Date: Wed, 2 Apr 2025 17:16:49 +0300 Subject: [PATCH 04/11] added debug checks --- .github/workflows/updater_test.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/updater_test.yml b/.github/workflows/updater_test.yml index 45c705f413c..1a781326242 100644 --- a/.github/workflows/updater_test.yml +++ b/.github/workflows/updater_test.yml @@ -42,6 +42,7 @@ jobs: id: check_timeout if: failure() run: | + echo "${{ steps.flashing.outcome }}" "${{ steps.flashing.conclusion }}" if grep -q "Timeout" "${{ github.workspace }}/logs.txt" 2>/dev/null; then echo "timed_out=true" >> $GITHUB_OUTPUT elif [ "${{ steps.first_full_flash.outcome }}" == "failure" ] && [ "${{ steps.first_full_flash.conclusion }}" == "timed_out" ]; then From 93e2e1246db2bc1df19055582fd959802aaf331d Mon Sep 17 00:00:00 2001 From: doomwastaken Date: Wed, 2 Apr 2025 17:25:19 +0300 Subject: [PATCH 05/11] added dirty hacks for timeout, debugging conclussion vs outcome --- .github/workflows/unit_tests.yml | 5 ++++- .github/workflows/updater_test.yml | 3 +-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml index ee9f8da29a6..8a3cb467479 100644 --- a/.github/workflows/unit_tests.yml +++ b/.github/workflows/unit_tests.yml @@ -63,7 +63,10 @@ jobs: id: check_timeout if: failure() run: | - if [ "${{ steps.flashing.conclusion }}" == "timed_out" ] || \ + echo "${{ steps.flashing.outcome }}" "${{ steps.flashing.conclusion }}" + if grep -q "has timed out" "${{ github.workspace }}/logs.txt" 2>/dev/null; then + echo "timed_out=true" >> $GITHUB_OUTPUT + elif [ "${{ steps.flashing.conclusion }}" == "timed_out" ] || \ [ "${{ steps.copy.conclusion }}" == "timed_out" ] || \ [ "${{ steps.run_units.conclusion }}" == "timed_out" ]; then echo "timed_out=true" >> $GITHUB_OUTPUT diff --git a/.github/workflows/updater_test.yml b/.github/workflows/updater_test.yml index 1a781326242..a1a7218f293 100644 --- a/.github/workflows/updater_test.yml +++ b/.github/workflows/updater_test.yml @@ -42,8 +42,7 @@ jobs: id: check_timeout if: failure() run: | - echo "${{ steps.flashing.outcome }}" "${{ steps.flashing.conclusion }}" - if grep -q "Timeout" "${{ github.workspace }}/logs.txt" 2>/dev/null; then + if grep -q "has timed out" "${{ github.workspace }}/logs.txt" 2>/dev/null; then echo "timed_out=true" >> $GITHUB_OUTPUT elif [ "${{ steps.first_full_flash.outcome }}" == "failure" ] && [ "${{ steps.first_full_flash.conclusion }}" == "timed_out" ]; then echo "timed_out=true" >> $GITHUB_OUTPUT From 06a1df69bef7324b70db272446fb47cdff211658 Mon Sep 17 00:00:00 2001 From: doomwastaken Date: Wed, 2 Apr 2025 17:32:10 +0300 Subject: [PATCH 06/11] added exitcode echo --- .github/workflows/unit_tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml index 8a3cb467479..b44227de765 100644 --- a/.github/workflows/unit_tests.yml +++ b/.github/workflows/unit_tests.yml @@ -63,7 +63,7 @@ jobs: id: check_timeout if: failure() run: | - echo "${{ steps.flashing.outcome }}" "${{ steps.flashing.conclusion }}" + echo "outcome" "${{ steps.flashing.outcome }}" "conclusion" "${{ steps.flashing.conclusion }}" "exitCode" "${{ steps.flashing.exitCode }}" if grep -q "has timed out" "${{ github.workspace }}/logs.txt" 2>/dev/null; then echo "timed_out=true" >> $GITHUB_OUTPUT elif [ "${{ steps.flashing.conclusion }}" == "timed_out" ] || \ From f577492af8ad8fdc6991036fe537b3c1c42e56c4 Mon Sep 17 00:00:00 2001 From: doomwastaken Date: Wed, 2 Apr 2025 18:01:09 +0300 Subject: [PATCH 07/11] searching repo --- .github/workflows/unit_tests.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml index b44227de765..0d68f7730d6 100644 --- a/.github/workflows/unit_tests.yml +++ b/.github/workflows/unit_tests.yml @@ -64,7 +64,9 @@ jobs: if: failure() run: | echo "outcome" "${{ steps.flashing.outcome }}" "conclusion" "${{ steps.flashing.conclusion }}" "exitCode" "${{ steps.flashing.exitCode }}" - if grep -q "has timed out" "${{ github.workspace }}/logs.txt" 2>/dev/null; then + ls "${{ github.workspace }}" + ls "${{ github.workspace }}/logs" + if grep -q "has timed out" "${{ github.workspace }}/logs/job/job-logs.txt" 2>/dev/null; then echo "timed_out=true" >> $GITHUB_OUTPUT elif [ "${{ steps.flashing.conclusion }}" == "timed_out" ] || \ [ "${{ steps.copy.conclusion }}" == "timed_out" ] || \ From 949e890fbf7f6987354582c20904215fd619b18a Mon Sep 17 00:00:00 2001 From: doomwastaken Date: Mon, 7 Apr 2025 11:27:06 +0300 Subject: [PATCH 08/11] dropped timeout retries, now do debug on failure with gdb, increased unit await timeout --- .github/workflows/unit_tests.yml | 21 ++------------------- .github/workflows/updater_test.yml | 25 ++++++++----------------- 2 files changed, 10 insertions(+), 36 deletions(-) diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml index 0d68f7730d6..b1451ae1dc6 100644 --- a/.github/workflows/unit_tests.yml +++ b/.github/workflows/unit_tests.yml @@ -33,7 +33,7 @@ jobs: timeout-minutes: 2 run: | source scripts/toolchain/fbtenv.sh - python3 scripts/testops.py -t=15 await_flipper + python3 scripts/testops.py -t=90 await_flipper python3 scripts/storage.py -f send build/latest/resources /ext python3 scripts/storage.py -f send /region_data /ext/.int/.region_data python3 scripts/power.py reboot @@ -59,26 +59,9 @@ jobs: run: | ./fbt gdb_trace_all LIB_DEBUG=1 FIRMWARE_APP_SET=unit_tests FORCE=1 - - name: Check for timeout - id: check_timeout - if: failure() - run: | - echo "outcome" "${{ steps.flashing.outcome }}" "conclusion" "${{ steps.flashing.conclusion }}" "exitCode" "${{ steps.flashing.exitCode }}" - ls "${{ github.workspace }}" - ls "${{ github.workspace }}/logs" - if grep -q "has timed out" "${{ github.workspace }}/logs/job/job-logs.txt" 2>/dev/null; then - echo "timed_out=true" >> $GITHUB_OUTPUT - elif [ "${{ steps.flashing.conclusion }}" == "timed_out" ] || \ - [ "${{ steps.copy.conclusion }}" == "timed_out" ] || \ - [ "${{ steps.run_units.conclusion }}" == "timed_out" ]; then - echo "timed_out=true" >> $GITHUB_OUTPUT - else - echo "timed_out=false" >> $GITHUB_OUTPUT - fi - unit_attempt_2: needs: unit_attempt_1 - if: needs.unit_attempt_1.outputs.timed_out == 'true' + if: failure() runs-on: [ self-hosted, FlipperZeroTest ] steps: - name: Checkout code diff --git a/.github/workflows/updater_test.yml b/.github/workflows/updater_test.yml index a1a7218f293..12616240dbb 100644 --- a/.github/workflows/updater_test.yml +++ b/.github/workflows/updater_test.yml @@ -38,23 +38,9 @@ jobs: ./fbt flash_usb FORCE=1 python3 scripts/testops.py -t=180 await_flipper - - name: Check for timeout - id: check_timeout - if: failure() - run: | - if grep -q "has timed out" "${{ github.workspace }}/logs.txt" 2>/dev/null; then - echo "timed_out=true" >> $GITHUB_OUTPUT - elif [ "${{ steps.first_full_flash.outcome }}" == "failure" ] && [ "${{ steps.first_full_flash.conclusion }}" == "timed_out" ]; then - echo "timed_out=true" >> $GITHUB_OUTPUT - elif [ "${{ steps.second_full_flash.outcome }}" == "failure" ] && [ "${{ steps.second_full_flash.conclusion }}" == "timed_out" ]; then - echo "timed_out=true" >> $GITHUB_OUTPUT - else - echo "timed_out=false" >> $GITHUB_OUTPUT - fi - updater_attempt2: needs: updater_attempt1 - if: needs.updater_attempt1.outputs.timed_out == 'true' + if: failure() runs-on: [self-hosted, FlipperZeroTest] steps: - name: Checkout code @@ -68,12 +54,17 @@ jobs: run: | source scripts/toolchain/fbtenv.sh python3 scripts/testops.py -t=180 await_flipper - ./fbt flash_usb_full FORCE=1 + ./fbt flash_usb_full FORCE=1 DEBUG=1 - name: Validating updater timeout-minutes: 4 run: | source scripts/toolchain/fbtenv.sh python3 scripts/testops.py -t=180 await_flipper - ./fbt flash_usb FORCE=1 + ./fbt flash_usb FORCE=1 DEBUG=1 python3 scripts/testops.py -t=180 await_flipper + + - name: Check GDB output + if: failure() + run: | + ./fbt gdb_trace_all LIB_DEBUG=1 FIRMWARE_APP_SET=unit_tests FORCE=1 From d6ebf70d56a7df42494eda440bdc189fbc04e77c Mon Sep 17 00:00:00 2001 From: doomwastaken Date: Mon, 7 Apr 2025 11:45:06 +0300 Subject: [PATCH 09/11] made sure both runs fail, fixed gdb --- .github/workflows/unit_tests.yml | 2 +- .github/workflows/updater_test.yml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml index b1451ae1dc6..2cfe2a41ffd 100644 --- a/.github/workflows/unit_tests.yml +++ b/.github/workflows/unit_tests.yml @@ -105,6 +105,6 @@ jobs: path: unit_tests*.txt - name: Check GDB output - if: failure() && steps.flashing.outcome == 'success' + if: failure() run: | ./fbt gdb_trace_all LIB_DEBUG=1 FIRMWARE_APP_SET=unit_tests FORCE=1 diff --git a/.github/workflows/updater_test.yml b/.github/workflows/updater_test.yml index 12616240dbb..fa6933549a9 100644 --- a/.github/workflows/updater_test.yml +++ b/.github/workflows/updater_test.yml @@ -22,7 +22,7 @@ jobs: - name: Flashing target firmware id: first_full_flash - timeout-minutes: 4 + timeout-minutes: 1 run: | source scripts/toolchain/fbtenv.sh python3 scripts/testops.py -t=180 await_flipper @@ -67,4 +67,4 @@ jobs: - name: Check GDB output if: failure() run: | - ./fbt gdb_trace_all LIB_DEBUG=1 FIRMWARE_APP_SET=unit_tests FORCE=1 + ./fbt gdb_trace_all FORCE=1 From 7772cf93cc26a01d7a07cd660aa41f7306ae761b Mon Sep 17 00:00:00 2001 From: doomwastaken Date: Mon, 7 Apr 2025 11:54:15 +0300 Subject: [PATCH 10/11] fixed timeouts, ready to undraft, fixed gdb launch conditions --- .github/workflows/unit_tests.yml | 4 ++-- .github/workflows/updater_test.yml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml index 2cfe2a41ffd..e78daf28260 100644 --- a/.github/workflows/unit_tests.yml +++ b/.github/workflows/unit_tests.yml @@ -22,7 +22,7 @@ jobs: - name: Flash unit tests firmware id: flashing - timeout-minutes: 1 + timeout-minutes: 5 run: | source scripts/toolchain/fbtenv.sh ./fbt resources firmware_latest flash LIB_DEBUG=1 FIRMWARE_APP_SET=unit_tests FORCE=1 @@ -33,7 +33,7 @@ jobs: timeout-minutes: 2 run: | source scripts/toolchain/fbtenv.sh - python3 scripts/testops.py -t=90 await_flipper + python3 scripts/testops.py -t=15 await_flipper python3 scripts/storage.py -f send build/latest/resources /ext python3 scripts/storage.py -f send /region_data /ext/.int/.region_data python3 scripts/power.py reboot diff --git a/.github/workflows/updater_test.yml b/.github/workflows/updater_test.yml index fa6933549a9..a575cac50d8 100644 --- a/.github/workflows/updater_test.yml +++ b/.github/workflows/updater_test.yml @@ -22,7 +22,7 @@ jobs: - name: Flashing target firmware id: first_full_flash - timeout-minutes: 1 + timeout-minutes: 5 run: | source scripts/toolchain/fbtenv.sh python3 scripts/testops.py -t=180 await_flipper From 4f5cee325dc4452ecb3ba169c8be664d9429bf15 Mon Sep 17 00:00:00 2001 From: doomwastaken Date: Mon, 7 Apr 2025 20:02:52 +0300 Subject: [PATCH 11/11] renamed jobs --- .github/workflows/unit_tests.yml | 6 +++--- .github/workflows/updater_test.yml | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml index e78daf28260..5b65ac39f6e 100644 --- a/.github/workflows/unit_tests.yml +++ b/.github/workflows/unit_tests.yml @@ -9,7 +9,7 @@ env: FBT_GIT_SUBMODULE_SHALLOW: 1 jobs: - unit_attempt_1: + Run_unit_tests: runs-on: [ self-hosted, FlipperZeroTest ] outputs: timed_out: ${{ steps.check_timeout.outputs.timed_out }} @@ -59,8 +59,8 @@ jobs: run: | ./fbt gdb_trace_all LIB_DEBUG=1 FIRMWARE_APP_SET=unit_tests FORCE=1 - unit_attempt_2: - needs: unit_attempt_1 + Retry_unit_tests: + needs: Run_unit_tests if: failure() runs-on: [ self-hosted, FlipperZeroTest ] steps: diff --git a/.github/workflows/updater_test.yml b/.github/workflows/updater_test.yml index a575cac50d8..f2a609c4441 100644 --- a/.github/workflows/updater_test.yml +++ b/.github/workflows/updater_test.yml @@ -9,7 +9,7 @@ env: FBT_GIT_SUBMODULE_SHALLOW: 1 jobs: - updater_attempt1: + Run_updater_test: runs-on: [self-hosted, FlipperZeroTest] outputs: timed_out: ${{ steps.check_timeout.outputs.timed_out }} @@ -38,8 +38,8 @@ jobs: ./fbt flash_usb FORCE=1 python3 scripts/testops.py -t=180 await_flipper - updater_attempt2: - needs: updater_attempt1 + Retry_updater_test: + needs: Run_updater_test if: failure() runs-on: [self-hosted, FlipperZeroTest] steps: