build: auto-set GitHub runner make job count

Dynamically determine an appropriate number of make jobs for the GitHub
runner container, based upon the number of available CPU cores and RAM,
using the rule that each job requires a dedicated core and 2.25GiB of RAM.

Author: iamamyth

.github/actions/set-make-job-count/action.yml

@@ -0,0 +1,22 @@
+name: 'set-make-job-count'
+description: 'Set the MAKE_JOB_COUNT environment variable to a value suitable for the host runner'
+runs:
+  using: "composite"
+  steps:
+    # Each job runner requires 2.25 GiB (i.e. 1024 * 9/4 MiB) memory and
+    # a dedicated logical CPU core
+    - name: set-jobs-macOS
+      if: runner.os == 'macOS'
+      run: |
+        echo MAKE_JOB_COUNT=$(printf '%s\n%s' $(( $(sysctl -n hw.memsize) * 4 / (1073741824 * 9) )) $(sysctl -n hw.logicalcpu) | sort -n | head -n1 | sed -e 's/^0$/1/') >> $GITHUB_ENV
+      shell: bash
+    - name: set-jobs-windows
+      if: runner.os == 'Windows'
+      run: |
+        echo MAKE_JOB_COUNT=$(printf '%s\n%s' $(( $(grep MemTotal: /proc/meminfo | cut -d: -f2 | cut -dk -f1) * 4 / (1048576 * 9) )) $(nproc) | sort -n | head -n1 | sed -e 's/^0$/1/') >> $GITHUB_ENV
+      shell: msys2 {0}
+    - name: set-jobs-linux
+      if: runner.os == 'Linux'
+      run: |
+        echo MAKE_JOB_COUNT=$(printf '%s\n%s' $(( $(vmstat -s --unit M | grep 'total memory' | cut -dM -f1) * 4 / (1024 * 9) )) $(nproc) | sort -n | head -n1 | sed -e 's/^0$/1/') >> $GITHUB_ENV
+      shell: bash

.github/workflows/build.yml

@@ -13,8 +13,7 @@ on:
 # The below variables reduce repetitions across similar targets
 env:
   REMOVE_BUNDLED_PACKAGES : sudo rm -rf /usr/local
-  BUILD_DEFAULT_LINUX: |
-        cmake -S . -B build -D ARCH="default" -D BUILD_TESTS=ON -D CMAKE_BUILD_TYPE=Release && cmake --build build --parallel 4
+  BUILD_DEFAULT_LINUX: 'cmake -S . -B build -D ARCH="default" -D BUILD_TESTS=ON -D CMAKE_BUILD_TYPE=Release && cmake --build build'
   APT_INSTALL_LINUX: 'apt -y install build-essential cmake libboost-all-dev miniupnpc libunbound-dev graphviz doxygen libunwind8-dev pkg-config libssl-dev libzmq3-dev libsodium-dev libhidapi-dev libusb-1.0-0-dev libprotobuf-dev protobuf-compiler ccache git'
   APT_SET_CONF: |
         tee -a /etc/apt/apt.conf.d/80-custom << EOF
@@ -41,14 +40,15 @@ jobs:
         path: /Users/runner/Library/Caches/ccache
         key: ccache-${{ runner.os }}-build-${{ github.sha }}
         restore-keys: ccache-${{ runner.os }}-build-
+    - uses: ./.github/actions/set-make-job-count
     - name: install dependencies
       run: |
         HOMEBREW_NO_AUTO_UPDATE=1 brew install boost@1.85 hidapi openssl zmq miniupnpc expat libunwind-headers protobuf@21 ccache
         brew link protobuf@21 boost@1.85
     - name: build
       run: |
         ${{env.CCACHE_SETTINGS}}
-        make -j3
+        make -j${{env.MAKE_JOB_COUNT}}
 
   build-windows:
     runs-on: windows-latest
@@ -78,10 +78,11 @@ jobs:
         curl -O https://repo.msys2.org/mingw/mingw64/mingw-w64-x86_64-boost-libs-1.86.0-7-any.pkg.tar.zst
         echo "4cb1d1066fffa6a5788b212ccb920c6d8cc93a8ecbbc633565bfc9b2ebc6feb5 mingw-w64-x86_64-boost-libs-1.86.0-7-any.pkg.tar.zst" | sha256sum -c
         pacman --noconfirm -U mingw-w64-x86_64-boost-1.86.0-7-any.pkg.tar.zst mingw-w64-x86_64-boost-libs-1.86.0-7-any.pkg.tar.zst
+    - uses: ./.github/actions/set-make-job-count
     - name: build
       run: |
         ${{env.CCACHE_SETTINGS}}
-        make release-static-win64 -j4
+        make release-static-win64 -j${{env.MAKE_JOB_COUNT}}
 
   build-debian:
     name: 'build-debian (debian-10)'
@@ -102,8 +103,9 @@ jobs:
       - uses: actions/checkout@v4
         with:
           submodules: recursive
+      - uses: ./.github/actions/set-make-job-count
       - name: build
-        run: ${{env.BUILD_DEFAULT_LINUX}}
+        run: ${{env.BUILD_DEFAULT_LINUX}} --parallel ${{env.MAKE_JOB_COUNT}}
 
 # See the OS labels and monitor deprecations here:
 # https://docs.github.com/en/actions/using-github-hosted-runners/about-github-hosted-runners#supported-runners-and-hardware-resources
@@ -125,6 +127,7 @@ jobs:
         path: ~/.ccache
         key: ccache-${{ runner.os }}-build-${{ matrix.os }}-${{ github.sha }}
         restore-keys: ccache-${{ runner.os }}-build-${{ matrix.os }}
+    - uses: ./.github/actions/set-make-job-count
     - name: remove bundled packages
       run: ${{env.REMOVE_BUNDLED_PACKAGES}}
     - name: set apt conf
@@ -136,7 +139,7 @@ jobs:
     - name: build
       run: |
         ${{env.CCACHE_SETTINGS}}
-        ${{env.BUILD_DEFAULT_LINUX}}
+        ${{env.BUILD_DEFAULT_LINUX}} --parallel ${{env.MAKE_JOB_COUNT}}
 
   libwallet-ubuntu:
     runs-on: ubuntu-20.04
@@ -151,6 +154,7 @@ jobs:
         path: ~/.ccache
         key: ccache-${{ runner.os }}-libwallet-${{ github.sha }}
         restore-keys: ccache-${{ runner.os }}-libwallet-
+    - uses: ./.github/actions/set-make-job-count
     - name: remove bundled packages
       run: ${{env.REMOVE_BUNDLED_PACKAGES}}
     - name: set apt conf
@@ -163,7 +167,7 @@ jobs:
       run: |
         ${{env.CCACHE_SETTINGS}}
         cmake .
-        make wallet_api -j4
+        make wallet_api -j${{env.MAKE_JOB_COUNT}}
 
   test-ubuntu:
     needs: build-ubuntu
@@ -180,6 +184,7 @@ jobs:
         path: ~/.ccache
         key: ccache-${{ runner.os }}-build-ubuntu-latest-${{ github.sha }}
         restore-keys: ccache-${{ runner.os }}-build-ubuntu-latest
+    - uses: ./.github/actions/set-make-job-count
     - name: remove bundled packages
       run: ${{env.REMOVE_BUNDLED_PACKAGES}}
     - name: set apt conf
@@ -196,7 +201,7 @@ jobs:
         DNS_PUBLIC: tcp://9.9.9.9
       run: |
         ${{env.CCACHE_SETTINGS}}
-        ${{env.BUILD_DEFAULT_LINUX}}
+        ${{env.BUILD_DEFAULT_LINUX}} --parallel ${{env.MAKE_JOB_COUNT}}
         cmake --build build --target test
 
 # ARCH="default" (not "native") ensures, that a different execution host can execute binaries compiled elsewhere.

.github/workflows/depends.yml

@@ -80,6 +80,7 @@ jobs:
         restore-keys: |
           depends-${{ matrix.toolchain.host }}-${{ hashFiles('contrib/depends/packages/*') }}
           depends-${{ matrix.toolchain.host }}-
+    - uses: ./.github/actions/set-make-job-count
     - name: set apt conf
       run: ${{env.APT_SET_CONF}}
     - name: install dependencies
@@ -92,7 +93,7 @@ jobs:
     - name: build
       run: |
         ${{env.CCACHE_SETTINGS}}
-        make depends target=${{ matrix.toolchain.host }} -j4
+        make depends target=${{ matrix.toolchain.host }} -j${{env.MAKE_JOB_COUNT}}
     - uses: actions/upload-artifact@v4
       if: ${{ matrix.toolchain.host == 'x86_64-w64-mingw32' || matrix.toolchain.host == 'x86_64-apple-darwin' || matrix.toolchain.host == 'aarch64-apple-darwin' || matrix.toolchain.host == 'x86_64-unknown-linux-gnu' }}
       with: