ci: Add a simple plugin to report test results to our flakiness tracker

.github/workflows/ci.yaml

@@ -16,6 +16,7 @@ env:
   RUST_PROFILE: release
   SLOW_MACHINE: 1
   CI_SERVER_URL: "http://35.239.136.52:3170"
+  GLOBAL_PYTEST_OPTS: "--reruns=10 -vvv"
 
 jobs:
   prebuild:
@@ -108,7 +109,8 @@ jobs:
       matrix:
         include:
           - CFG: compile-gcc
-            VALGRIND: 1
+            # TODO re-enable valgrind when sped up
+            VALGRIND: 0
             COMPILER: gcc
           - CFG: compile-gcc-O3
             VALGRIND: 1
@@ -313,7 +315,7 @@ jobs:
         run: |
           env
           cat config.vars
-          uv run eatmydata pytest tests/test_downgrade.py -vvv -n ${PYTEST_PAR} ${PYTEST_OPTS}
+          uv run eatmydata pytest tests/test_downgrade.py ${GLOBAL_PYTEST_OPTS} -n ${PYTEST_PAR} ${PYTEST_OPTS}
 
   integration:
     name: Test CLN ${{ matrix.name }}
@@ -421,7 +423,7 @@ jobs:
         run: |
           env
           cat config.vars
-          VALGRIND=0 uv run eatmydata pytest tests/ -vvv -n ${PYTEST_PAR} ${PYTEST_OPTS}
+          VALGRIND=0 uv run eatmydata pytest tests/ -vvv ${GLOBAL_PYTEST_OPTS} -n ${PYTEST_PAR} ${PYTEST_OPTS}
 
   integration-valgrind:
     name: Valgrind Test CLN ${{ matrix.name }}
@@ -430,7 +432,7 @@ jobs:
     env:
       RUST_PROFILE: release # Has to match the one in the compile step
       CFG: compile-gcc
-      PYTEST_OPTS: --test-group-random-seed=42 --timeout=1800 --durations=10
+      PYTEST_OPTS: --test-group-random-seed=42 --timeout=1800 --durations=10 --reruns=10
     needs:
       - compile
     strategy:
@@ -491,7 +493,7 @@ jobs:
           SLOW_MACHINE: 1
           TEST_DEBUG: 1
         run: |
-          VALGRIND=1 uv run eatmydata pytest tests/ -vvv -n 3 ${PYTEST_OPTS}  ${{ matrix.PYTEST_OPTS }}
+          VALGRIND=1 uv run eatmydata pytest tests/ ${GLOBAL_PYTEST_OPTS} -n 3 ${PYTEST_OPTS}  ${{ matrix.PYTEST_OPTS }}
 
   integration-sanitizers:
     name: Sanitizers Test CLN
@@ -501,7 +503,7 @@ jobs:
       RUST_PROFILE: release
       SLOW_MACHINE: 1
       TEST_DEBUG: 1
-      PYTEST_OPTS: --test-group-random-seed=42 --timeout=1800 --durations=10
+      PYTEST_OPTS: --test-group-random-seed=42 --timeout=1800 --durations=10 --reruns=10
     needs:
       - compile
     strategy:
@@ -557,7 +559,7 @@ jobs:
 
       - name: Test
         run: |
-          uv run eatmydata pytest tests/ -vvv -n 2 ${PYTEST_OPTS}  ${{ matrix.PYTEST_OPTS }}
+          uv run eatmydata pytest tests/ ${GLOBAL_PYTEST_OPTS} -n 2 ${PYTEST_OPTS}  ${{ matrix.PYTEST_OPTS }}
 
   update-docs-examples:
     name: Update examples in doc schemas (disabled temporarily!)
@@ -674,7 +676,7 @@ jobs:
         run: |
           env
           cat config.vars
-          VALGRIND=0 uv run eatmydata pytest tests/ -vvv -n ${PYTEST_PAR} ${PYTEST_OPTS}
+          VALGRIND=0 uv run eatmydata pytest tests/ ${GLOBAL_PYTEST_OPTS} -n ${PYTEST_PAR} ${PYTEST_OPTS}
 
   check-flake:
     name: Check Nix Flake

.gitignore

@@ -36,6 +36,8 @@ monkeytype.sqlite3
 !*/test/exp-run-*.c
 */*/test/run-*
 !*/*/test/run-*.c
+devtools/check-bolt
+tools/lightning-downgrade
 test/test_protocol
 test/test_sphinx
 tests/.pytest.restart

contrib/pytest-trackflaky/README.md

@@ -0,0 +1,63 @@
+# pytest-trackflaky
+
+A pytest plugin to track and report test flakiness to a central server.
+
+## Features
+
+- Automatically tracks test execution times and outcomes
+- Collects GitHub Actions metadata (commit SHA, branch, run ID, etc.)
+- Reports test results to a configurable server endpoint
+- Zero configuration needed when running in CI environments
+
+## Installation
+
+Install the plugin using pip or uv:
+
+```bash
+pip install -e contrib/pytest-trackflaky
+```
+
+Or with uv:
+
+```bash
+uv pip install -e contrib/pytest-trackflaky
+```
+
+## Usage
+
+Once installed, the plugin is automatically activated when running pytest. No additional configuration is needed.
+
+### Configuration
+
+The plugin is controlled via environment variables:
+
+- `CI_SERVER_URL`: The base URL of the server to report results to (required for reporting)
+  - Test results will be POSTed to `{CI_SERVER_URL}/hook/test`
+- `GITHUB_*`: Standard GitHub Actions environment variables are automatically collected
+
+### Example
+
+```bash
+export CI_SERVER_URL="https://your-flaky-tracker.example.com"
+pytest
+```
+
+## Data Collected
+
+For each test, the plugin collects:
+
+- Test name
+- Outcome (success/skip/fail)
+- Start and end times
+- GitHub repository information
+- Git commit SHA and branch
+- GitHub Actions run metadata
+
+## Development
+
+To work on the plugin locally:
+
+```bash
+cd contrib/pytest-trackflaky
+pip install -e .
+```

contrib/pytest-trackflaky/pyproject.toml

@@ -0,0 +1,19 @@
+[project]
+name = "pytest-trackflaky"
+version = "0.1.0"
+description = "A pytest plugin to track and report test flakiness"
+authors = [
+    {name = "Lightning Development Team"}
+]
+readme = "README.md"
+requires-python = ">=3.8"
+dependencies = [
+    "pytest>=7.0.0",
+]
+
+[project.entry-points.pytest11]
+trackflaky = "pytest_trackflaky.plugin"
+
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"

contrib/pytest-trackflaky/pytest_trackflaky/__init__.py

@@ -0,0 +1,3 @@
+"""pytest-trackflaky: A pytest plugin to track and report test flakiness."""
+
+__version__ = "0.1.0"