Merge f6d49d0a09 into merged_master (Bitcoin PR bitcoin/bitcoin#33403)

.github/actions/configure-docker/action.yml

@@ -0,0 +1,52 @@
+name: 'Configure Docker'
+description: 'Set up Docker build driver and configure build cache args'
+inputs:
+  use-cirrus:
+    description: 'Use cirrus cache'
+    required: true
+runs:
+  using: 'composite'
+  steps:
+    - name: Set up Docker Buildx
+      uses: docker/setup-buildx-action@v3
+      with:
+        # Use host network to allow access to cirrus gha cache running on the host
+        driver-opts: |
+          network=host
+
+    # This is required to allow buildkit to access the actions cache
+    - name: Expose actions cache variables
+      uses: actions/github-script@v6
+      with:
+        script: |
+          core.exportVariable('ACTIONS_CACHE_URL', process.env['ACTIONS_CACHE_URL'])
+          core.exportVariable('ACTIONS_RUNTIME_TOKEN', process.env['ACTIONS_RUNTIME_TOKEN'])
+
+    - name: Construct docker build cache args
+      shell: bash
+      run: |
+        # Configure docker build cache backend
+        #
+        # On forks the gha cache will work but will use Github's cache backend.
+        # Docker will check for variables $ACTIONS_CACHE_URL, $ACTIONS_RESULTS_URL and $ACTIONS_RUNTIME_TOKEN
+        # which are set automatically when running on GitHub infra: https://docs.docker.com/build/cache/backends/gha/#synopsis
+
+        # Use cirrus cache host
+        if [[ ${{ inputs.use-cirrus }} == 'true' ]]; then
+          url_args="url=${CIRRUS_CACHE_HOST},url_v2=${CIRRUS_CACHE_HOST}"
+        else
+          url_args=""
+        fi
+
+        # Always optimistically --cache‑from in case a cache blob exists
+        args=(--cache-from "type=gha${url_args:+,${url_args}},scope=${CONTAINER_NAME}")
+
+        # If this is a push to the default branch, also add --cache‑to to save the cache
+        if [[ ${{ github.event_name }} == "push" && ${{ github.ref_name }} == ${{ github.event.repository.default_branch }} ]]; then
+          args+=(--cache-to "type=gha${url_args:+,${url_args}},mode=max,ignore-error=true,scope=${CONTAINER_NAME}")
+        fi
+
+        # Always `--load` into docker images (needed when using the `docker-container` build driver).
+        args+=(--load)
+
+        echo "DOCKER_BUILD_CACHE_ARG=${args[*]}" >> $GITHUB_ENV

.github/actions/configure-environment/action.yml

@@ -0,0 +1,27 @@
+name: 'Configure environment'
+description: 'Configure CI, cache and container name environment variables'
+runs:
+  using: 'composite'
+  steps:
+    - name: Set CI and cache directories
+      shell: bash
+      run: |
+        echo "BASE_ROOT_DIR=${{ runner.temp }}" >> "$GITHUB_ENV"
+        echo "BASE_BUILD_DIR=${{ runner.temp }}/build" >> "$GITHUB_ENV"
+        echo "CCACHE_DIR=${{ runner.temp }}/ccache_dir" >> $GITHUB_ENV
+        echo "DEPENDS_DIR=${{ runner.temp }}/depends" >> "$GITHUB_ENV"
+        echo "BASE_CACHE=${{ runner.temp }}/depends/built" >> $GITHUB_ENV
+        echo "SOURCES_PATH=${{ runner.temp }}/depends/sources" >> $GITHUB_ENV
+        echo "PREVIOUS_RELEASES_DIR=${{ runner.temp }}/previous_releases" >> $GITHUB_ENV
+
+    - name: Set cache hashes
+      shell: bash
+      run: |
+        echo "DEPENDS_HASH=$(git ls-tree HEAD depends "ci/test/$FILE_ENV" | sha256sum | cut -d' ' -f1)" >> $GITHUB_ENV
+        echo "PREVIOUS_RELEASES_HASH=$(git ls-tree HEAD test/get_previous_releases.py | sha256sum | cut -d' ' -f1)" >> $GITHUB_ENV
+
+    - name: Get container name
+      shell: bash
+      run: |
+        source $FILE_ENV
+        echo "CONTAINER_NAME=$CONTAINER_NAME" >> "$GITHUB_ENV"

.github/actions/restore-caches/action.yml

@@ -0,0 +1,47 @@
+name: 'Restore Caches'
+description: 'Restore ccache, depends sources, and built depends caches'
+runs:
+  using: 'composite'
+  steps:
+    - name: Restore Ccache cache
+      id: ccache-cache
+      uses: cirruslabs/cache/restore@v4
+      with:
+        path: ${{ env.CCACHE_DIR }}
+        key: ccache-${{ env.CONTAINER_NAME }}-${{ github.run_id }}
+        restore-keys: |
+          ccache-${{ env.CONTAINER_NAME }}-
+
+    - name: Restore depends sources cache
+      id: depends-sources
+      uses: cirruslabs/cache/restore@v4
+      with:
+        path: ${{ env.SOURCES_PATH }}
+        key: depends-sources-${{ env.CONTAINER_NAME }}-${{ env.DEPENDS_HASH }}
+        restore-keys: |
+          depends-sources-${{ env.CONTAINER_NAME }}-
+
+    - name: Restore built depends cache
+      id: depends-built
+      uses: cirruslabs/cache/restore@v4
+      with:
+        path: ${{ env.BASE_CACHE }}
+        key: depends-built-${{ env.CONTAINER_NAME }}-${{ env.DEPENDS_HASH }}
+        restore-keys: |
+          depends-built-${{ env.CONTAINER_NAME }}-
+
+    - name: Restore previous releases cache
+      id: previous-releases
+      uses: cirruslabs/cache/restore@v4
+      with:
+        path: ${{ env.PREVIOUS_RELEASES_DIR }}
+        key: previous-releases-${{ env.CONTAINER_NAME }}-${{ env.PREVIOUS_RELEASES_HASH }}
+        restore-keys: |
+          previous-releases-${{ env.CONTAINER_NAME }}-
+
+    - name: export cache hits
+      shell: bash
+      run: |
+        echo "depends-sources-cache-hit=${{ steps.depends-sources.outputs.cache-hit }}" >> $GITHUB_ENV
+        echo "depends-built-cache-hit=${{ steps.depends-built.outputs.cache-hit }}" >> $GITHUB_ENV
+        echo "previous-releases-cache-hit=${{ steps.previous-releases.outputs.cache-hit }}" >> $GITHUB_ENV

.github/actions/save-caches/action.yml

@@ -0,0 +1,39 @@
+name: 'Save Caches'
+description: 'Save ccache, depends sources, and built depends caches'
+runs:
+  using: 'composite'
+  steps:
+    - name: debug cache hit inputs
+      shell: bash
+      run: |
+        echo "depends sources direct cache hit to primary key: ${{ env.depends-sources-cache-hit }}"
+        echo "depends built direct cache hit to primary key: ${{ env.depends-built-cache-hit }}"
+        echo "previous releases direct cache hit to primary key: ${{ env.previous-releases-cache-hit }}"
+
+    - name: Save Ccache cache
+      uses: cirruslabs/cache/save@v4
+      if: ${{ (github.event_name == 'push') && (github.ref_name == github.event.repository.default_branch) }}
+      with:
+        path: ${{ env.CCACHE_DIR }}
+        key: ccache-${{ env.CONTAINER_NAME }}-${{ github.run_id }}
+
+    - name: Save depends sources cache
+      uses: cirruslabs/cache/save@v4
+      if: ${{ (github.event_name == 'push') && (github.ref_name == github.event.repository.default_branch) && (env.depends-sources-cache-hit != 'true') }}
+      with:
+        path: ${{ env.SOURCES_PATH }}
+        key: depends-sources-${{ env.CONTAINER_NAME }}-${{ env.DEPENDS_HASH }}
+
+    - name: Save built depends cache
+      uses: cirruslabs/cache/save@v4
+      if: ${{ (github.event_name == 'push') && (github.ref_name == github.event.repository.default_branch) && (env.depends-built-cache-hit != 'true' )}}
+      with:
+        path: ${{ env.BASE_CACHE }}
+        key: depends-built-${{ env.CONTAINER_NAME }}-${{ env.DEPENDS_HASH }}
+
+    - name: Save previous releases cache
+      uses: cirruslabs/cache/save@v4
+      if: ${{ (github.event_name == 'push') && (github.ref_name == github.event.repository.default_branch) && (env.previous-releases-cache-hit != 'true' )}}
+      with:
+        path: ${{ env.PREVIOUS_RELEASES_DIR }}
+        key: previous-releases-${{ env.CONTAINER_NAME }}-${{ env.PREVIOUS_RELEASES_HASH }}

.github/workflows/ci.yml

@@ -19,9 +19,26 @@ concurrency:
 
 env:
   CI_FAILFAST_TEST_LEAVE_DANGLING: 1  # GHA does not care about dangling processes and setting this variable avoids killing the CI script itself on error
-  MAKEJOBS: '-j10'
+  CIRRUS_CACHE_HOST: http://127.0.0.1:12321/ # When using Cirrus Runners this host can be used by the docker `gha` build cache type.
+  REPO_USE_CIRRUS_RUNNERS: 'bitcoin/bitcoin' # Use cirrus runners and cache for this repo, instead of falling back to the slow GHA runners
 
 jobs:
+  runners:
+    name: 'determine runners'
+    runs-on: ubuntu-latest
+    outputs:
+      use-cirrus-runners: ${{ steps.runners.outputs.use-cirrus-runners }}
+    steps:
+      - id: runners
+        run: |
+          if [[ "${REPO_USE_CIRRUS_RUNNERS}" == "${{ github.repository }}" ]]; then
+            echo "use-cirrus-runners=true" >> "$GITHUB_OUTPUT"
+            echo "::notice title=Runner Selection::Using Cirrus Runners"
+          else
+            echo "use-cirrus-runners=false" >> "$GITHUB_OUTPUT"
+            echo "::notice title=Runner Selection::Using GitHub-hosted runners"
+          fi
+
   test-each-commit:
     name: 'test each commit'
     runs-on: ubuntu-24.04
@@ -106,8 +123,12 @@ jobs:
       BASE_ROOT_DIR: ${{ github.workspace }}
 
     steps:
-      - name: Checkout
-        uses: actions/checkout@v4
+      - &CHECKOUT
+        name: Checkout
+        uses: actions/checkout@v5
+        with:
+          # Ensure the latest merged pull request state is used, even on re-runs.
+          ref: &CHECKOUT_REF_TMPL ${{ github.event_name == 'pull_request' && github.ref || '' }}
 
       - name: Clang version
         run: |
@@ -183,8 +204,7 @@ jobs:
             job-name: 'Win64 native fuzz, VS 2022'
 
     steps:
-      - name: Checkout
-        uses: actions/checkout@v4
+      - *CHECKOUT
 
       - name: Configure Developer Command Prompt for Microsoft Visual C++
         # Using microsoft/setup-msbuild is not enough.
@@ -273,43 +293,136 @@ jobs:
         run: |
           py -3 test\fuzz\test_runner.py --par %NUMBER_OF_PROCESSORS% --loglevel DEBUG %RUNNER_TEMP%\qa-assets\fuzz_corpora
 
-  asan-lsan-ubsan-integer-no-depends-usdt:
-    name: 'ASan + LSan + UBSan + integer, no depends, USDT'
-    runs-on: ubuntu-24.04 # has to match container in ci/test/00_setup_env_native_asan.sh for tracing tools
+  ci-matrix:
+    name: ${{ matrix.name }}
+    needs: runners
+    runs-on: ${{ needs.runners.outputs.use-cirrus-runners == 'true' && matrix.cirrus-runner || matrix.fallback-runner }}
     if: ${{ vars.SKIP_BRANCH_PUSH != 'true' || github.event_name == 'pull_request' }}
-    timeout-minutes: 120
+    timeout-minutes: ${{ matrix.timeout-minutes }}
+
     env:
-      FILE_ENV: "./ci/test/00_setup_env_native_asan.sh"
       DANGER_CI_ON_HOST_FOLDERS: 1
+      FILE_ENV: ${{ matrix.file-env }}
+
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - name: '32 bit ARM, unit tests, no functional tests'
+            cirrus-runner: 'ubuntu-24.04-arm' # Cirrus' Arm runners are Apple (with virtual Linux aarch64), which doesn't support 32-bit mode
+            fallback-runner: 'ubuntu-24.04-arm'
+            timeout-minutes: 120
+            file-env: './ci/test/00_setup_env_arm.sh'
+
+          - name: 'win64 Cross'
+            cirrus-runner: 'ghcr.io/cirruslabs/ubuntu-runner-amd64:24.04-sm'
+            fallback-runner: 'ubuntu-24.04'
+            timeout-minutes: 120
+            file-env: './ci/test/00_setup_env_win64.sh'
+
+          - name: 'ASan + LSan + UBSan + integer, no depends, USDT'
+            cirrus-runner: 'ghcr.io/cirruslabs/ubuntu-runner-amd64:24.04-md' # has to match container in ci/test/00_setup_env_native_asan.sh for tracing tools
+            fallback-runner: 'ubuntu-24.04'
+            timeout-minutes: 120
+            file-env: './ci/test/00_setup_env_native_asan.sh'
+
+          - name: 'macOS-cross, gui, no tests'
+            cirrus-runner: 'ghcr.io/cirruslabs/ubuntu-runner-amd64:24.04-sm'
+            fallback-runner: 'ubuntu-24.04'
+            timeout-minutes: 120
+            file-env: './ci/test/00_setup_env_mac_cross.sh'
+
+          - name: 'No wallet, libbitcoinkernel'
+            cirrus-runner: 'ghcr.io/cirruslabs/ubuntu-runner-amd64:24.04-sm'
+            fallback-runner: 'ubuntu-24.04'
+            timeout-minutes: 120
+            file-env: './ci/test/00_setup_env_native_nowallet_libbitcoinkernel.sh'
+
+          - name: 'i686, multiprocess, DEBUG'
+            cirrus-runner: 'ghcr.io/cirruslabs/ubuntu-runner-amd64:24.04-md'
+            fallback-runner: 'ubuntu-24.04'
+            timeout-minutes: 120
+            file-env: './ci/test/00_setup_env_i686_multiprocess.sh'
+
+          - name: 'fuzzer,address,undefined,integer, no depends'
+            cirrus-runner: 'ghcr.io/cirruslabs/ubuntu-runner-amd64:24.04-lg'
+            fallback-runner: 'ubuntu-24.04'
+            timeout-minutes: 240
+            file-env: './ci/test/00_setup_env_native_fuzz.sh'
+
+          - name: 'previous releases, depends DEBUG'
+            cirrus-runner: 'ghcr.io/cirruslabs/ubuntu-runner-amd64:24.04-md'
+            fallback-runner: 'ubuntu-24.04'
+            timeout-minutes: 120
+            file-env: './ci/test/00_setup_env_native_previous_releases.sh'
+
+          - name: 'CentOS, depends, gui'
+            cirrus-runner: 'ghcr.io/cirruslabs/ubuntu-runner-amd64:24.04-lg'
+            fallback-runner: 'ubuntu-24.04'
+            timeout-minutes: 120
+            file-env: './ci/test/00_setup_env_native_centos.sh'
+
+          - name: 'tidy'
+            cirrus-runner: 'ghcr.io/cirruslabs/ubuntu-runner-amd64:24.04-md'
+            fallback-runner: 'ubuntu-24.04'
+            timeout-minutes: 120
+            file-env: './ci/test/00_setup_env_native_tidy.sh'
+
+          - name: 'TSan, depends, no gui'
+            cirrus-runner: 'ghcr.io/cirruslabs/ubuntu-runner-amd64:24.04-md'
+            fallback-runner: 'ubuntu-24.04'
+            timeout-minutes: 120
+            file-env: './ci/test/00_setup_env_native_tsan.sh'
+
+          - name: 'MSan, depends'
+            cirrus-runner: 'ghcr.io/cirruslabs/ubuntu-runner-amd64:24.04-lg'
+            fallback-runner: 'ubuntu-24.04'
+            timeout-minutes: 120
+            file-env: './ci/test/00_setup_env_native_msan.sh'
+
     steps:
-      - name: Checkout
-        uses: actions/checkout@v4
+      - *CHECKOUT
 
-      - name: Set CI directories
-        run: |
-          echo "CCACHE_DIR=${{ runner.temp }}/ccache_dir" >> "$GITHUB_ENV"
-          echo "BASE_ROOT_DIR=${{ runner.temp }}" >> "$GITHUB_ENV"
-          echo "BASE_BUILD_DIR=${{ runner.temp }}/build-asan" >> "$GITHUB_ENV"
+      - name: Configure environment
+        uses: ./.github/actions/configure-environment
 
-      - name: Restore Ccache cache
-        id: ccache-cache
-        uses: actions/cache/restore@v4
+      - name: Restore caches
+        id: restore-cache
+        uses: ./.github/actions/restore-caches
+
+      - name: Configure Docker
+        uses: ./.github/actions/configure-docker
         with:
-          path: ${{ env.CCACHE_DIR }}
-          key: ${{ github.job }}-ccache-${{ github.run_id }}
-          restore-keys: ${{ github.job }}-ccache-
+          use-cirrus: ${{ needs.runners.outputs.use-cirrus-runners }}
 
       - name: Enable bpfcc script
+        if: ${{ env.CONTAINER_NAME == 'ci_native_asan' }}
         # In the image build step, no external environment variables are available,
         # so any settings will need to be written to the settings env file:
         run: sed -i "s|\${INSTALL_BCC_TRACING_TOOLS}|true|g" ./ci/test/00_setup_env_native_asan.sh
 
+      - name: Set mmap_rnd_bits
+        if: ${{ env.CONTAINER_NAME == 'ci_native_tsan' || env.CONTAINER_NAME == 'ci_native_msan' }}
+        # Prevents crashes due to high ASLR entropy
+        run: sudo sysctl -w vm.mmap_rnd_bits=28
+
       - name: CI script
         run: ./ci/test_run_all.sh
 
-      - name: Save Ccache cache
-        uses: actions/cache/save@v4
-        if: github.event_name != 'pull_request' && steps.ccache-cache.outputs.cache-hit != 'true'
+      - name: Save caches
+        uses: ./.github/actions/save-caches
+
+  lint:
+    name: 'lint'
+    needs: runners
+    runs-on: ${{ needs.runners.outputs.use-cirrus-runners == 'true' && 'ghcr.io/cirruslabs/ubuntu-runner-amd64:24.04-xs' || 'ubuntu-24.04' }}
+    if: ${{ vars.SKIP_BRANCH_PUSH != 'true' || github.event_name == 'pull_request' }}
+    timeout-minutes: 20
+    env:
+      CONTAINER_NAME: "bitcoin-linter"
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v5
         with:
           path: ${{ env.CCACHE_DIR }}
           # https://github.com/actions/cache/blob/main/tips-and-workarounds.md#update-a-cache
@@ -325,4 +438,4 @@ jobs:
       - name: Build lint Docker image
         run: DOCKER_BUILDKIT=1 docker build -t bitcoin-linter --file "./ci/lint_imagefile" ./
       - name: Run linter
-        run: docker run --rm -v $(pwd):/bitcoin bitcoin-linter
+        run: docker run --rm -v $(pwd):/bitcoin bitcoin-linter