Retries on QA jobs

.github/workflows/unit_tests.yml

@@ -9,25 +9,75 @@ env:
   FBT_GIT_SUBMODULE_SHALLOW: 1
 
 jobs:
-  run_units_on_bench:
+  Run_unit_tests:
     runs-on: [ self-hosted, FlipperZeroTest ]
+    outputs:
+      timed_out: ${{ steps.check_timeout.outputs.timed_out }}
     steps:
       - name: Checkout code
         uses: actions/checkout@v4
         with:
           fetch-depth: 1
           ref: ${{ github.event.pull_request.head.sha }}
 
-      - name: 'Flash unit tests firmware'
+      - name: Flash unit tests firmware
         id: flashing
-        if: success()
         timeout-minutes: 5
         run: |
           source scripts/toolchain/fbtenv.sh
           ./fbt resources firmware_latest flash LIB_DEBUG=1 FIRMWARE_APP_SET=unit_tests FORCE=1
 
+      - name: Copy assets and unit data, reboot and wait for flipper
+        id: copy
+        if: steps.flashing.outcome == 'success'
+        timeout-minutes: 2
+        run: |
+          source scripts/toolchain/fbtenv.sh
+          python3 scripts/testops.py -t=15 await_flipper
+          python3 scripts/storage.py -f send build/latest/resources /ext
+          python3 scripts/storage.py -f send /region_data /ext/.int/.region_data
+          python3 scripts/power.py reboot
+          python3 scripts/testops.py -t=30 await_flipper
 
-      - name: 'Copy assets and unit data, reboot and wait for flipper'
+      - name: Run units and validate results
+        id: run_units
+        if: steps.copy.outcome == 'success'
+        timeout-minutes: 5
+        run: |
+          source scripts/toolchain/fbtenv.sh
+          python3 scripts/testops.py run_units
+
+      - name: Upload test results
+        if: failure() && steps.flashing.outcome == 'success' && steps.run_units.outcome != 'skipped'
+        uses: actions/upload-artifact@v4
+        with:
+          name: unit-tests_output
+          path: unit_tests*.txt
+
+      - name: Check GDB output
+        if: failure() && steps.flashing.outcome == 'success'
+        run: |
+          ./fbt gdb_trace_all LIB_DEBUG=1 FIRMWARE_APP_SET=unit_tests FORCE=1
+
+  Retry_unit_tests:
+    needs: Run_unit_tests
+    if: failure()
+    runs-on: [ self-hosted, FlipperZeroTest ]
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 1
+          ref: ${{ github.event.pull_request.head.sha }}
+
+      - name: Flash unit tests firmware
+        id: flashing
+        timeout-minutes: 5
+        run: |
+          source scripts/toolchain/fbtenv.sh
+          ./fbt resources firmware_latest flash LIB_DEBUG=1 FIRMWARE_APP_SET=unit_tests FORCE=1
+
+      - name: Copy assets and unit data, reboot and wait for flipper
         id: copy
         if: steps.flashing.outcome == 'success'
         timeout-minutes: 5
@@ -39,22 +89,22 @@ jobs:
           python3 scripts/power.py reboot
           python3 scripts/testops.py -t=30 await_flipper
 
-      - name: 'Run units and validate results'
+      - name: Run units and validate results
         id: run_units
         if: steps.copy.outcome == 'success'
         timeout-minutes: 5
         run: |
           source scripts/toolchain/fbtenv.sh
           python3 scripts/testops.py run_units
 
-      - name: 'Upload test results'
+      - name: Upload test results
         if: failure() && steps.flashing.outcome == 'success' && steps.run_units.outcome != 'skipped'
         uses: actions/upload-artifact@v4
         with:
           name: unit-tests_output
           path: unit_tests*.txt
 
-      - name: 'Check GDB output'
-        if: failure() && steps.flashing.outcome == 'success'
+      - name: Check GDB output
+        if: failure()
         run: |
           ./fbt gdb_trace_all LIB_DEBUG=1 FIRMWARE_APP_SET=unit_tests FORCE=1

.github/workflows/updater_test.yml

@@ -9,31 +9,62 @@ env:
   FBT_GIT_SUBMODULE_SHALLOW: 1
 
 jobs:
-  test_updater_on_bench:
-    runs-on: [self-hosted, FlipperZeroTest ]
+  Run_updater_test:
+    runs-on: [self-hosted, FlipperZeroTest]
+    outputs:
+      timed_out: ${{ steps.check_timeout.outputs.timed_out }}
     steps:
       - name: Checkout code
         uses: actions/checkout@v4
         with:
           fetch-depth: 1
           ref: ${{ github.event.pull_request.head.sha }}
 
-      - name: 'Flashing target firmware'
+      - name: Flashing target firmware
         id: first_full_flash
         timeout-minutes: 5
         run: |
           source scripts/toolchain/fbtenv.sh
           python3 scripts/testops.py -t=180 await_flipper
           ./fbt flash_usb_full FORCE=1
-
 
-      - name: 'Validating updater'
+      - name: Validating updater
         id: second_full_flash
-        timeout-minutes: 5
-        if: success()
+        if: steps.first_full_flash.outcome == 'success'
+        timeout-minutes: 4
         run: |
           source scripts/toolchain/fbtenv.sh
           python3 scripts/testops.py -t=180 await_flipper
           ./fbt flash_usb FORCE=1
           python3 scripts/testops.py -t=180 await_flipper
 
+  Retry_updater_test:
+    needs: Run_updater_test
+    if: failure()
+    runs-on: [self-hosted, FlipperZeroTest]
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 1
+          ref: ${{ github.event.pull_request.head.sha }}
+
+      - name: Flashing target firmware
+        timeout-minutes: 4
+        run: |
+          source scripts/toolchain/fbtenv.sh
+          python3 scripts/testops.py -t=180 await_flipper
+          ./fbt flash_usb_full FORCE=1 DEBUG=1
+
+      - name: Validating updater
+        timeout-minutes: 4
+        run: |
+          source scripts/toolchain/fbtenv.sh
+          python3 scripts/testops.py -t=180 await_flipper
+          ./fbt flash_usb FORCE=1  DEBUG=1
+          python3 scripts/testops.py -t=180 await_flipper
+
+      - name: Check GDB output
+        if: failure()
+        run: |
+          ./fbt gdb_trace_all FORCE=1