From 9b5aa68a7b0519d7a38fd13cd517eb2b8a80146b Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Fri, 20 Feb 2026 12:01:49 +1030 Subject: [PATCH] CI: use nproc + 1 for pytest parallelism.. More future proof, better for self-hosted runners. Example run times before: 4s CI completion 2m 36s Check we can downgrade the node (compile-gcc, sqlite3, liquid-regtest) 2m 52s Check we can downgrade the node (compile-gcc, postgres, regtest) 3m 20s Check we can downgrade the node (compile-gcc, sqlite3, regtest, 1) 3m 20s Check we can downgrade the node (compile-gcc, sqlite3, regtest, 1) 8m 1s Run fuzz regression tests 11m 46s Run unit tests (compile-clang-sanitizers, 0) 12m 59s Compile CLN compile-gcc 13m 42s Compile CLN compile-clang 14m 10s Compile CLN compile-gcc-O3 14m 15s Compile CLN compile-clang-sanitizers 15m 23s Pre-build checks 19m 37s Test CLN dual-fund 26m 49s Run unit tests (compile-gcc, 1) 43m 19s Valgrind Test CLN Valgrind (06/10) 43m 21s Valgrind Test CLN Valgrind (08/10) 43m 40s Valgrind Test CLN Valgrind (05/10) 43m 59s Valgrind Test CLN Valgrind (03/10) 44m 4s Valgrind Test CLN Valgrind (07/10) 47m 28s Valgrind Test CLN Valgrind (09/10) 47m 37s Valgrind Test CLN Valgrind (02/10) 47m 45s Valgrind Test CLN Valgrind (01/10) 52m 51s Valgrind Test CLN Valgrind (04/10) 54m 6s Valgrind Test CLN Valgrind (10/10) 1h 14m 13s Sanitizers Test CLN (ASan/UBSan (03/12), 3, --test-group=3 --test-group-count=12) 1h 14m 6s Sanitizers Test CLN (ASan/UBSan (06/12), 6, --test-group=6 --test-group-count=12) 1h 15m 23s Sanitizers Test CLN (ASan/UBSan (01/12), 1, --test-group=1 --test-group-count=12) 1h 20m 30s Sanitizers Test CLN (ASan/UBSan (08/12), 8, --test-group=8 --test-group-count=12) 1h 22m 35s Sanitizers Test CLN (ASan/UBSan (12/12), 12, --test-group=12 --test-group-count=12) 1h 23m 49s Sanitizers Test CLN (ASan/UBSan (04/12), 4, --test-group=4 --test-group-count=12) 1h 26m 11s Sanitizers Test CLN (ASan/UBSan (07/12), 7, --test-group=7 --test-group-count=12) 1h 28m 44s Sanitizers Test CLN (ASan/UBSan (05/12), 5, --test-group=5 --test-group-count=12) 1h 29m 1s Sanitizers Test CLN (ASan/UBSan (02/12), 2, --test-group=2 --test-group-count=12) 1h 29m 4s Sanitizers Test CLN (ASan/UBSan (11/12), 11, --test-group=11 --test-group-count=12) 1h 31m 36s Sanitizers Test CLN (ASan/UBSan (09/12), 9, --test-group=9 --test-group-count=12) 1h 38m 54s Sanitizers Test CLN (ASan/UBSan (10/12), 10, --test-group=10 --test-group-count=12) 1h 39m 22s Test CLN liquid 1h 52m 23s Test minimum supported BTC v25.0 with clang 1h 52m 28s Test CLN gcc 1h 53m 57s Test CLN splicing 1h 57m 25s Test CLN postgres 1h 59m 25s Test CLN clang 2d 17h 56m 16s TOTAL Signed-off-by: Rusty Russell --- .github/workflows/ci.yaml | 20 +++++++------------- 1 file changed, 7 insertions(+), 13 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index b9a93a23e727..f566806ab4b6 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -93,7 +93,7 @@ jobs: VALGRIND: 0 PYTEST_OPTS: ${{ env.PYTEST_OPTS_BASE }} run: | - uv run make check-source BASE_REF="origin/${{ github.base_ref }}" + uv run make -j $(nproc) check-source BASE_REF="origin/${{ github.base_ref }}" - name: Upload test results if: always() uses: actions/upload-artifact@v4 @@ -312,7 +312,6 @@ jobs: - name: Test env: SLOW_MACHINE: 1 - PYTEST_PAR: 4 TEST_DEBUG: 1 TEST_DB_PROVIDER: ${{ matrix.TEST_DB_PROVIDER }} TEST_NETWORK: ${{ matrix.TEST_NETWORK }} @@ -323,7 +322,7 @@ jobs: run: | env cat config.vars - sg wireshark "uv run eatmydata pytest tests/test_downgrade.py -n ${PYTEST_PAR} ${PYTEST_OPTS}" + sg wireshark "uv run eatmydata pytest tests/test_downgrade.py -n $(($(nproc) + 1)) ${PYTEST_OPTS}" - name: Upload test results if: always() uses: actions/upload-artifact@v4 @@ -430,7 +429,6 @@ jobs: COMPAT: 1 CFG: ${{ matrix.CFG }} SLOW_MACHINE: 1 - PYTEST_PAR: 4 TEST_DEBUG: 1 TEST_DB_PROVIDER: ${{ matrix.TEST_DB_PROVIDER }} TEST_NETWORK: ${{ matrix.TEST_NETWORK }} @@ -438,7 +436,7 @@ jobs: run: | env cat config.vars - VALGRIND=0 sg wireshark "uv run eatmydata pytest tests/ -n ${PYTEST_PAR} ${PYTEST_OPTS}" + VALGRIND=0 sg wireshark "uv run eatmydata pytest tests/ -n $(($(nproc) + 1)) ${PYTEST_OPTS}" - name: Upload test results if: always() uses: actions/upload-artifact@v4 @@ -524,9 +522,8 @@ jobs: env: SLOW_MACHINE: 1 TEST_DEBUG: 1 - PYTEST_PAR: 2 run: | - VALGRIND=1 sg wireshark "uv run eatmydata pytest tests/ -n ${PYTEST_PAR} ${PYTEST_OPTS} ${{ matrix.PYTEST_OPTS }}" + VALGRIND=1 sg wireshark "uv run eatmydata pytest tests/ -n $(($(nproc) + 1)) ${PYTEST_OPTS} ${{ matrix.PYTEST_OPTS }}" - name: Upload test results if: always() uses: actions/upload-artifact@v4 @@ -614,10 +611,8 @@ jobs: run: tar -xvjf cln-compile-clang-sanitizers.tar.bz2 - name: Test - env: - PYTEST_PAR: 2 run: | - sg wireshark "uv run eatmydata pytest tests/ -n ${PYTEST_PAR} ${PYTEST_OPTS} ${{ matrix.PYTEST_OPTS }}" + sg wireshark "uv run eatmydata pytest tests/ -n $(($(nproc) + 1)) ${PYTEST_OPTS} ${{ matrix.PYTEST_OPTS }}" - name: Upload test results if: always() uses: actions/upload-artifact@v4 @@ -667,7 +662,7 @@ jobs: tar -xaf cln-compile-gcc.tar.bz2 - name: Test run: | - uv run eatmydata make -j $(nproc) check-doc-examples + uv run eatmydata make -j $(($(nproc) + 1)) check-doc-examples - name: Upload test results if: always() uses: actions/upload-artifact@v4 @@ -740,7 +735,6 @@ jobs: COMPAT: 1 CFG: ${{ matrix.CFG }} SLOW_MACHINE: 1 - PYTEST_PAR: 4 TEST_DEBUG: 1 TEST_DB_PROVIDER: ${{ matrix.TEST_DB_PROVIDER }} TEST_NETWORK: ${{ matrix.TEST_NETWORK }} @@ -748,7 +742,7 @@ jobs: run: | env cat config.vars - VALGRIND=0 sg wireshark "uv run eatmydata pytest tests/ -n ${PYTEST_PAR} ${PYTEST_OPTS}" + VALGRIND=0 sg wireshark "uv run eatmydata pytest tests/ -n $(($(nproc) + 1)) ${PYTEST_OPTS}" - name: Upload test results if: always() uses: actions/upload-artifact@v4