From d5a71e97853ea9e1b879e8c76bfb01d4bef33172 Mon Sep 17 00:00:00 2001 From: Carl Dong Date: Mon, 22 Feb 2021 17:04:19 -0500 Subject: [PATCH 01/15] guix: Use --cores instead of --max-jobs In Guix, there are two flags for controlling parallelism: Note: When I say "derivation," think "package" --cores=n - controls the number of CPU cores to build each derivation. This is the value passed to `make`'s `--jobs=` flag. - defaults to 0: as many cores as is available --max-jobs=n - controls how many derivations can be built in parallel - defaults to 1 Therefore, if set --max-jobs=$MAX_JOBS and don't set --cores, Guix could theoretically spin up $MAX_JOBS * $(nproc) number of threads, and that's no good. So we could either default to --cores=1, --max-jobs=$MAX_JOBS - Pro: --cores=1 means that `make` will be invoked with `-j1`, avoiding problems with package whose build systems and test suites break when running multi-threaded. - Con: There will be times when only 1 or 2 derivations can be built at a time, because the rest of the dependency graph all depend on those 1 or 2 derivations. During these times, the machine will be severely under-utilized. or --cores=$MAX_JOBS, --max-jobs=1 - Pro: We don't encounter prolonged periods of severe under-utilization mentioned above. - Con: Many packages' build systems and test suites break when running multi-threaded. or --cores=1, --max-jobs=1 and let the user override with $ADDITIONAL_GUIX_COMMON_FLAGS --- contrib/guix/README.md | 58 ++++++++++++++++++++++++++++++++--- contrib/guix/guix-build.sh | 12 ++++---- contrib/guix/libexec/build.sh | 8 ++--- 3 files changed, 63 insertions(+), 15 deletions(-) diff --git a/contrib/guix/README.md b/contrib/guix/README.md index 1122ec9ba5..133f250ad2 100644 --- a/contrib/guix/README.md +++ b/contrib/guix/README.md @@ -80,6 +80,50 @@ at the end of the `guix pull`) export PATH="${HOME}/.config/guix/current/bin${PATH:+:}$PATH" ``` +### Controlling the number of threads used by `guix` build commands + +By default, the scripts under `./contrib/guix` will invoke all `guix` build +commands with `--cores="$JOBS"`. Note that `$JOBS` defaults to `$(nproc)` if not +specified. However, astute manual readers will also notice that there is a +`--max-jobs=` flag (which defaults to 1 if unspecified). + +Here is the difference between `--cores=` and `--max-jobs=`: + +> Note: When I say "derivation," think "package" + +`--cores=` + + - controls the number of CPU cores to build each derivation. This is the value + passed to `make`'s `--jobs=` flag. + +`--max-jobs=` + + - controls how many derivations can be built in parallel + - defaults to 1 + +Therefore, the default is for `guix` build commands to build one derivation at a +time, utilizing `$JOBS` threads. + +Specifying the `$JOBS` environment variable will only modify `--cores=`, but you +can also modify the value for `--max-jobs=` by specifying +`$ADDITIONAL_GUIX_COMMON_FLAGS`. For example, if you have a LOT of memory, you +may want to set: + +```sh +export ADDITIONAL_GUIX_COMMON_FLAGS='--max-jobs=8' +``` + +Which allows for a maximum of 8 derivations to be built at the same time, each +utilizing `$JOBS` threads. + +Or, if you'd like to avoid spurious build failures caused by issues with +parallelism within a single package, but would still like to build multiple +packages when the dependency graph allows for it, you may want to try: + +```sh +export JOBS=1 ADDITIONAL_GUIX_COMMON_FLAGS='--max-jobs=8' +``` + ## Usage ### As a Tool for Deterministic Builds @@ -125,12 +169,16 @@ find output/ -type f -print0 | sort -z | xargs -r0 sha256sum the actual SDK (e.g. SDK_PATH=$HOME/Downloads/macOS-SDKs instead of $HOME/Downloads/macOS-SDKs/Xcode-11.3.1-11C505-extracted-SDK-with-libcxx-headers). -* _**MAX_JOBS**_ +* _**JOBS**_ - Override the maximum number of jobs to run simultaneously, you might want to - do so on a memory-limited machine. This may be passed to `make` as in `make - --jobs="$MAX_JOBS"` or `xargs` as in `xargs -P"$MAX_JOBS"`. _(defaults to the - value of `nproc` outside the container)_ + Override the number of jobs to run simultaneously, you might want to do so on + a memory-limited machine. This may be passed to: + + - `guix` build commands as in `guix environment --cores="$JOBS"` + - `make` as in `make --jobs="$JOBS"` + - `xargs` as in `xargs -P"$JOBS"` + + _(defaults to the value of `nproc` outside the container)_ * _**SOURCE_DATE_EPOCH**_ diff --git a/contrib/guix/guix-build.sh b/contrib/guix/guix-build.sh index 7c32fb5726..430b7c3209 100755 --- a/contrib/guix/guix-build.sh +++ b/contrib/guix/guix-build.sh @@ -134,7 +134,7 @@ done # Determine the maximum number of jobs to run simultaneously (overridable by # environment) -MAX_JOBS="${MAX_JOBS:-$(nproc)}" +JOBS="${JOBS:-$(nproc)}" # Usage: host_to_commonname HOST # @@ -152,7 +152,7 @@ host_to_commonname() { # Download the depends sources now as we won't have internet access in the build # container for host in $HOSTS; do - make -C "${PWD}/depends" -j"$MAX_JOBS" download-"$(host_to_commonname "$host")" ${V:+V=1} ${SOURCES_PATH:+SOURCES_PATH="$SOURCES_PATH"} + make -C "${PWD}/depends" -j"$JOBS" download-"$(host_to_commonname "$host")" ${V:+V=1} ${SOURCES_PATH:+SOURCES_PATH="$SOURCES_PATH"} done # Determine the reference time used for determinism (overridable by environment) @@ -164,7 +164,7 @@ time-machine() { # shellcheck disable=SC2086 guix time-machine --url=https://github.com/dongcarl/guix.git \ --commit=490e39ff303f4f6873a04bfb8253755bdae1b29c \ - --max-jobs="$MAX_JOBS" \ + --cores="$JOBS" \ --keep-failed \ ${SUBSTITUTE_URLS:+--substitute-urls="$SUBSTITUTE_URLS"} \ ${ADDITIONAL_GUIX_COMMON_FLAGS} ${ADDITIONAL_GUIX_TIMEMACHINE_FLAGS} \ @@ -218,7 +218,7 @@ for host in $HOSTS; do cat << EOF INFO: Building commit ${GIT_COMMIT:?not set} for platform triple ${HOST:?not set}: ...using reference timestamp: ${SOURCE_DATE_EPOCH:?not set} - ...running at most ${MAX_JOBS:?not set} jobs + ...running at most ${JOBS:?not set} jobs ...from worktree directory: '${PWD}' ...bind-mounted in container to: '/bitcoin' ...in build directory: '$(distsrc_for_host "$HOST")' @@ -304,12 +304,12 @@ EOF ${SOURCES_PATH:+--share="$SOURCES_PATH"} \ ${BASE_CACHE:+--share="$BASE_CACHE"} \ ${SDK_PATH:+--share="$SDK_PATH"} \ - --max-jobs="$MAX_JOBS" \ + --cores="$JOBS" \ --keep-failed \ ${SUBSTITUTE_URLS:+--substitute-urls="$SUBSTITUTE_URLS"} \ ${ADDITIONAL_GUIX_COMMON_FLAGS} ${ADDITIONAL_GUIX_ENVIRONMENT_FLAGS} \ -- env HOST="$host" \ - MAX_JOBS="$MAX_JOBS" \ + JOBS="$JOBS" \ SOURCE_DATE_EPOCH="${SOURCE_DATE_EPOCH:?unable to determine value}" \ ${V:+V=1} \ ${SOURCES_PATH:+SOURCES_PATH="$SOURCES_PATH"} \ diff --git a/contrib/guix/libexec/build.sh b/contrib/guix/libexec/build.sh index 051066a6a0..0301d10522 100644 --- a/contrib/guix/libexec/build.sh +++ b/contrib/guix/libexec/build.sh @@ -26,7 +26,7 @@ cat << EOF Required environment variables as seen inside the container: HOST: ${HOST:?not set} SOURCE_DATE_EPOCH: ${SOURCE_DATE_EPOCH:?not set} - MAX_JOBS: ${MAX_JOBS:?not set} + JOBS: ${JOBS:?not set} DISTSRC: ${DISTSRC:?not set} OUTDIR: ${OUTDIR:?not set} EOF @@ -173,7 +173,7 @@ esac #################### # Build the depends tree, overriding variables that assume multilib gcc -make -C depends --jobs="$MAX_JOBS" HOST="$HOST" \ +make -C depends --jobs="$JOBS" HOST="$HOST" \ ${V:+V=1} \ ${SOURCES_PATH+SOURCES_PATH="$SOURCES_PATH"} \ ${BASE_CACHE+BASE_CACHE="$BASE_CACHE"} \ @@ -267,7 +267,7 @@ mkdir -p "$DISTSRC" sed -i.old 's/-lstdc++ //g' config.status libtool src/univalue/config.status src/univalue/libtool # Build Bitcoin Core - make --jobs="$MAX_JOBS" ${V:+V=1} + make --jobs="$JOBS" ${V:+V=1} # Perform basic ELF security checks on a series of executables. make -C src --jobs=1 check-security ${V:+V=1} @@ -344,7 +344,7 @@ mkdir -p "$DISTSRC" { find "${DISTNAME}/bin" -type f -executable -print0 find "${DISTNAME}/lib" -type f -print0 - } | xargs -0 -n1 -P"$MAX_JOBS" -I{} "${DISTSRC}/contrib/devtools/split-debug.sh" {} {} {}.dbg + } | xargs -0 -n1 -P"$JOBS" -I{} "${DISTSRC}/contrib/devtools/split-debug.sh" {} {} {}.dbg ;; esac From 3e9982ab3877eb8fe0a8c0cb3d847ac0913c7336 Mon Sep 17 00:00:00 2001 From: Carl Dong Date: Fri, 26 Feb 2021 17:38:35 -0500 Subject: [PATCH 02/15] contrib: Silence git-describe when looking for tag Otherwise, it prints a rather disturbing message to stderr: fatal: no tag exactly matches '' --- contrib/gitian-descriptors/assign_DISTNAME | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/gitian-descriptors/assign_DISTNAME b/contrib/gitian-descriptors/assign_DISTNAME index a2ca768aaa..330fbc041b 100755 --- a/contrib/gitian-descriptors/assign_DISTNAME +++ b/contrib/gitian-descriptors/assign_DISTNAME @@ -4,7 +4,7 @@ # # A helper script to be sourced into the gitian descriptors -if RECENT_TAG="$(git describe --exact-match HEAD)"; then +if RECENT_TAG="$(git describe --exact-match HEAD 2> /dev/null)"; then VERSION="${RECENT_TAG#v}" else VERSION="$(git rev-parse --short=12 HEAD)" From e5b49a01f5d0f631e7f08f86ca8a2c2b8213319f Mon Sep 17 00:00:00 2001 From: Carl Dong Date: Fri, 26 Feb 2021 17:39:44 -0500 Subject: [PATCH 03/15] guix: Create windeploy inside distsrc-* ./windeploy is a "working directory", and therefore belongs inside distsrc-*. Many people have noticed their Guix builds failing after hours simply because they did not remove windeploy (but did remove the distsrc-* directories). --- contrib/guix/libexec/build.sh | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/contrib/guix/libexec/build.sh b/contrib/guix/libexec/build.sh index 0301d10522..9888a84c9d 100644 --- a/contrib/guix/libexec/build.sh +++ b/contrib/guix/libexec/build.sh @@ -394,21 +394,21 @@ mkdir -p "$DISTSRC" || ( rm -f "${OUTDIR}/${DISTNAME}-${HOST//x86_64-apple-darwin18/osx64}.tar.gz" && exit 1 ) ;; esac - ) -) + ) # $DISTSRC/installed -case "$HOST" in - *mingw*) - cp -rf --target-directory=. contrib/windeploy - ( - cd ./windeploy - mkdir unsigned - cp --target-directory=unsigned/ "${OUTDIR}/${DISTNAME}-win64-setup-unsigned.exe" - find . -print0 \ - | sort --zero-terminated \ - | tar --create --no-recursion --mode='u+rw,go+r-w,a+X' --null --files-from=- \ - | gzip -9n > "${OUTDIR}/${DISTNAME}-win-unsigned.tar.gz" \ - || ( rm -f "${OUTDIR}/${DISTNAME}-win-unsigned.tar.gz" && exit 1 ) - ) - ;; -esac + case "$HOST" in + *mingw*) + cp -rf --target-directory=. contrib/windeploy + ( + cd ./windeploy + mkdir -p unsigned + cp --target-directory=unsigned/ "${OUTDIR}/${DISTNAME}-win64-setup-unsigned.exe" + find . -print0 \ + | sort --zero-terminated \ + | tar --create --no-recursion --mode='u+rw,go+r-w,a+X' --null --files-from=- \ + | gzip -9n > "${OUTDIR}/${DISTNAME}-win-unsigned.tar.gz" \ + || ( rm -f "${OUTDIR}/${DISTNAME}-win-unsigned.tar.gz" && exit 1 ) + ) + ;; + esac +) # $DISTSRC From 7753357a7bae98ec775c707b9dec4cea1e945802 Mon Sep 17 00:00:00 2001 From: Carl Dong Date: Wed, 17 Mar 2021 12:36:55 -0400 Subject: [PATCH 04/15] guix: Add source-able bash prelude and utils --- contrib/guix/libexec/prelude.bash | 60 ++++++++++++++++++++++++++ contrib/shell/git-utils.bash | 14 ++++++ contrib/shell/realpath.bash | 71 +++++++++++++++++++++++++++++++ 3 files changed, 145 insertions(+) create mode 100644 contrib/guix/libexec/prelude.bash create mode 100644 contrib/shell/git-utils.bash create mode 100644 contrib/shell/realpath.bash diff --git a/contrib/guix/libexec/prelude.bash b/contrib/guix/libexec/prelude.bash new file mode 100644 index 0000000000..33a319012c --- /dev/null +++ b/contrib/guix/libexec/prelude.bash @@ -0,0 +1,60 @@ +#!/usr/bin/env bash +export LC_ALL=C +set -e -o pipefail + +# shellcheck source=../../shell/realpath.bash +source contrib/shell/realpath.bash + +# shellcheck source=../../shell/git-utils.bash +source contrib/shell/git-utils.bash + +################ +# Required non-builtin commands should be invokable +################ + +check_tools() { + for cmd in "$@"; do + if ! command -v "$cmd" > /dev/null 2>&1; then + echo "ERR: This script requires that '$cmd' is installed and available in your \$PATH" + exit 1 + fi + done +} + +check_tools cat env readlink dirname basename git + +################ +# We should be at the top directory of the repository +################ + +same_dir() { + local resolved1 resolved2 + resolved1="$(bash_realpath "${1}")" + resolved2="$(bash_realpath "${2}")" + [ "$resolved1" = "$resolved2" ] +} + +if ! same_dir "${PWD}" "$(git_root)"; then +cat << EOF +ERR: This script must be invoked from the top level of the git repository + +Hint: This may look something like: + env FOO=BAR ./contrib/guix/guix- + +EOF +exit 1 +fi + +################ +# Set common variables +################ + +VERSION="${VERSION:-$(git_head_version)}" +DISTNAME="${DISTNAME:-bitcoin-${VERSION}}" + +version_base_prefix="${PWD}/guix-build-" +VERSION_BASE="${version_base_prefix}${VERSION}" # TOP + +DISTSRC_BASE="${DISTSRC_BASE:-${VERSION_BASE}}" + +OUTDIR_BASE="${OUTDIR_BASE:-${VERSION_BASE}/output}" diff --git a/contrib/shell/git-utils.bash b/contrib/shell/git-utils.bash new file mode 100644 index 0000000000..37bac1f38d --- /dev/null +++ b/contrib/shell/git-utils.bash @@ -0,0 +1,14 @@ +#!/usr/bin/env bash + +git_root() { + git rev-parse --show-toplevel 2> /dev/null +} + +git_head_version() { + local recent_tag + if recent_tag="$(git describe --exact-match HEAD 2> /dev/null)"; then + echo "${recent_tag#v}" + else + git rev-parse --short=12 HEAD + fi +} diff --git a/contrib/shell/realpath.bash b/contrib/shell/realpath.bash new file mode 100644 index 0000000000..389b77b562 --- /dev/null +++ b/contrib/shell/realpath.bash @@ -0,0 +1,71 @@ +#!/usr/bin/env bash + +# Based on realpath.sh written by Michael Kropat +# Found at: https://github.com/mkropat/sh-realpath/blob/65512368b8155b176b67122aa395ac580d9acc5b/realpath.sh + +bash_realpath() { + canonicalize_path "$(resolve_symlinks "$1")" +} + +resolve_symlinks() { + _resolve_symlinks "$1" +} + +_resolve_symlinks() { + _assert_no_path_cycles "$@" || return + + local dir_context path + if path=$(readlink -- "$1"); then + dir_context=$(dirname -- "$1") + _resolve_symlinks "$(_prepend_dir_context_if_necessary "$dir_context" "$path")" "$@" + else + printf '%s\n' "$1" + fi +} + +_prepend_dir_context_if_necessary() { + if [ "$1" = . ]; then + printf '%s\n' "$2" + else + _prepend_path_if_relative "$1" "$2" + fi +} + +_prepend_path_if_relative() { + case "$2" in + /* ) printf '%s\n' "$2" ;; + * ) printf '%s\n' "$1/$2" ;; + esac +} + +_assert_no_path_cycles() { + local target path + + target=$1 + shift + + for path in "$@"; do + if [ "$path" = "$target" ]; then + return 1 + fi + done +} + +canonicalize_path() { + if [ -d "$1" ]; then + _canonicalize_dir_path "$1" + else + _canonicalize_file_path "$1" + fi +} + +_canonicalize_dir_path() { + (cd "$1" 2>/dev/null && pwd -P) +} + +_canonicalize_file_path() { + local dir file + dir=$(dirname -- "$1") + file=$(basename -- "$1") + (cd "$dir" 2>/dev/null && printf '%s/%s\n' "$(pwd -P)" "$file") +} From 4eccf063b252bfe256cf72d363a24cf0183e926e Mon Sep 17 00:00:00 2001 From: Carl Dong Date: Wed, 17 Mar 2021 14:33:10 -0400 Subject: [PATCH 05/15] guix: Remove guix-build.sh filename extension --- contrib/guix/{guix-build.sh => guix-build} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename contrib/guix/{guix-build.sh => guix-build} (100%) diff --git a/contrib/guix/guix-build.sh b/contrib/guix/guix-build similarity index 100% rename from contrib/guix/guix-build.sh rename to contrib/guix/guix-build From 7f401c953f8bb3574cec48561e13ef3b47dedc6e Mon Sep 17 00:00:00 2001 From: Carl Dong Date: Wed, 17 Mar 2021 12:59:18 -0400 Subject: [PATCH 06/15] guix: Adapt guix-build to prelude, restructure hier --- contrib/guix/guix-build | 67 +++++++++++++++++++++-------------- contrib/guix/libexec/build.sh | 9 +++-- 2 files changed, 44 insertions(+), 32 deletions(-) diff --git a/contrib/guix/guix-build b/contrib/guix/guix-build index 430b7c3209..a1c120f412 100755 --- a/contrib/guix/guix-build +++ b/contrib/guix/guix-build @@ -2,22 +2,26 @@ export LC_ALL=C set -e -o pipefail +# Source the common prelude, which: +# 1. Checks if we're at the top directory of the Bitcoin Core repository +# 2. Defines a few common functions and variables +# +# shellcheck source=libexec/prelude.bash +source "$(dirname "${BASH_SOURCE[0]}")/libexec/prelude.bash" + + ################### -## Sanity Checks ## +## SANITY CHECKS ## ################### ################ -# Check 1: Make sure that we can invoke required tools +# Required non-builtin commands should be invokable ################ -for cmd in git make guix cat mkdir curl; do - if ! command -v "$cmd" > /dev/null 2>&1; then - echo "ERR: This script requires that '$cmd' is installed and available in your \$PATH" - exit 1 - fi -done + +check_tools cat mkdir make git guix ################ -# Check 2: Make sure GUIX_BUILD_OPTIONS is empty +# GUIX_BUILD_OPTIONS should be empty ################ # # GUIX_BUILD_OPTIONS is an environment variable recognized by guix commands that @@ -45,8 +49,9 @@ exit 1 fi ################ -# Check 3: Make sure that we're not in a dirty worktree +# The git worktree should not be dirty ################ + if ! git diff-index --quiet HEAD -- && [ -z "$FORCE_DIRTY_WORKTREE" ]; then cat << EOF ERR: The current git worktree is dirty, which may lead to broken builds. @@ -60,12 +65,12 @@ Hint: To make your git worktree clean, You may want to: using a dirty worktree EOF exit 1 -else - GIT_COMMIT=$(git rev-parse --short=12 HEAD) fi +mkdir -p "$VERSION_BASE" + ################ -# Check 4: Make sure that build directories do not exist +# Build directories should not exist ################ # Default to building for all supported HOSTs (overridable by environment) @@ -73,14 +78,12 @@ export HOSTS="${HOSTS:-x86_64-linux-gnu arm-linux-gnueabihf aarch64-linux-gnu ri x86_64-w64-mingw32 x86_64-apple-darwin18}" -DISTSRC_BASE="${DISTSRC_BASE:-${PWD}}" - # Usage: distsrc_for_host HOST # # HOST: The current platform triple we're building for # distsrc_for_host() { - echo "${DISTSRC_BASE}/distsrc-${GIT_COMMIT}-${1}" + echo "${DISTSRC_BASE}/distsrc-${VERSION}-${1}" } # Accumulate a list of build directories that already exist... @@ -106,12 +109,11 @@ for host in $hosts_distsrc_exists; do done exit 1 else - mkdir -p "$DISTSRC_BASE" fi ################ -# Check 5: When building for darwin, make sure that the macOS SDK exists +# When building for darwin, the macOS SDK should exists ################ for host in $HOSTS; do @@ -129,7 +131,7 @@ for host in $HOSTS; do done ######### -# Setup # +# SETUP # ######### # Determine the maximum number of jobs to run simultaneously (overridable by @@ -172,11 +174,20 @@ time-machine() { } # Make sure an output directory exists for our builds -OUTDIR="${OUTDIR:-${PWD}/output}" -[ -e "$OUTDIR" ] || mkdir -p "$OUTDIR" +OUTDIR_BASE="${OUTDIR_BASE:-${VERSION_BASE}/output}" +mkdir -p "$OUTDIR_BASE" + +# Usage: outdir_for_host HOST +# +# HOST: The current platform triple we're building for +# +outdir_for_host() { + echo "${OUTDIR_BASE}/${1}" +} + ######### -# Build # +# BUILD # ######### # Function to be called when building for host ${1} and the user interrupts the @@ -216,15 +227,15 @@ for host in $HOSTS; do # shellcheck disable=SC2030 cat << EOF -INFO: Building commit ${GIT_COMMIT:?not set} for platform triple ${HOST:?not set}: +INFO: Building ${VERSION:?not set} for platform triple ${HOST:?not set}: ...using reference timestamp: ${SOURCE_DATE_EPOCH:?not set} ...running at most ${JOBS:?not set} jobs ...from worktree directory: '${PWD}' ...bind-mounted in container to: '/bitcoin' ...in build directory: '$(distsrc_for_host "$HOST")' ...bind-mounted in container to: '$(DISTSRC_BASE=/distsrc-base && distsrc_for_host "$HOST")' - ...outputting in: '${OUTDIR:?not set}' - ...bind-mounted in container to: '/outdir' + ...outdirting in: '$(outdir_for_host "$HOST")' + ...bind-mounted in container to: '$(OUTDIR_BASE=/outdir-base && outdir_for_host "$HOST")' EOF # Run the build script 'contrib/guix/libexec/build.sh' in the build @@ -299,7 +310,7 @@ EOF --no-cwd \ --share="$PWD"=/bitcoin \ --share="$DISTSRC_BASE"=/distsrc-base \ - --share="$OUTDIR"=/outdir \ + --share="$OUTDIR_BASE"=/outdir-base \ --expose="$(git rev-parse --git-common-dir)" \ ${SOURCES_PATH:+--share="$SOURCES_PATH"} \ ${BASE_CACHE:+--share="$BASE_CACHE"} \ @@ -309,6 +320,7 @@ EOF ${SUBSTITUTE_URLS:+--substitute-urls="$SUBSTITUTE_URLS"} \ ${ADDITIONAL_GUIX_COMMON_FLAGS} ${ADDITIONAL_GUIX_ENVIRONMENT_FLAGS} \ -- env HOST="$host" \ + DISTNAME="$DISTNAME" \ JOBS="$JOBS" \ SOURCE_DATE_EPOCH="${SOURCE_DATE_EPOCH:?unable to determine value}" \ ${V:+V=1} \ @@ -316,7 +328,8 @@ EOF ${BASE_CACHE:+BASE_CACHE="$BASE_CACHE"} \ ${SDK_PATH:+SDK_PATH="$SDK_PATH"} \ DISTSRC="$(DISTSRC_BASE=/distsrc-base && distsrc_for_host "$HOST")" \ - OUTDIR=/outdir \ + OUTDIR="$(OUTDIR_BASE=/outdir-base && outdir_for_host "$HOST")" \ + DIST_ARCHIVE_BASE=/outdir-base/dist-archive \ bash -c "cd /bitcoin && bash contrib/guix/libexec/build.sh" ) diff --git a/contrib/guix/libexec/build.sh b/contrib/guix/libexec/build.sh index 9888a84c9d..bb83516f89 100644 --- a/contrib/guix/libexec/build.sh +++ b/contrib/guix/libexec/build.sh @@ -24,6 +24,8 @@ fi # Check that required environment variables are set cat << EOF Required environment variables as seen inside the container: + DIST_ARCHIVE_BASE: ${DIST_ARCHIVE_BASE:?not set} + DISTNAME: ${DISTNAME:?not set} HOST: ${HOST:?not set} SOURCE_DATE_EPOCH: ${SOURCE_DATE_EPOCH:?not set} JOBS: ${JOBS:?not set} @@ -198,11 +200,7 @@ make -C depends --jobs="$JOBS" HOST="$HOST" \ # Source Tarball Building # ########################### -# Define DISTNAME variable. -# shellcheck source=contrib/gitian-descriptors/assign_DISTNAME -source contrib/gitian-descriptors/assign_DISTNAME - -GIT_ARCHIVE="${OUTDIR}/src/${DISTNAME}.tar.gz" +GIT_ARCHIVE="${DIST_ARCHIVE_BASE}/${DISTNAME}.tar.gz" # Create the source tarball if not already there if [ ! -e "$GIT_ARCHIVE" ]; then @@ -275,6 +273,7 @@ mkdir -p "$DISTSRC" # version symbols for Linux distro back-compatibility. make -C src --jobs=1 check-symbols ${V:+V=1} + mkdir -p ${OUTDIR} # Make the os-specific installers case "$HOST" in *mingw*) From d55a1056ee565afed64e42d6f6efb6b0adc5599b Mon Sep 17 00:00:00 2001 From: Carl Dong Date: Wed, 17 Mar 2021 13:01:07 -0400 Subject: [PATCH 07/15] guix: Add troubleshooting documentation entries --- contrib/guix/README.md | 51 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/contrib/guix/README.md b/contrib/guix/README.md index 133f250ad2..2d128c7ed6 100644 --- a/contrib/guix/README.md +++ b/contrib/guix/README.md @@ -265,6 +265,57 @@ To use dongcarl's substitute server for Bitcoin Core builds after having export SUBSTITUTE_URLS='https://guix.carldong.io https://ci.guix.gnu.org' ``` +## Troubleshooting + +### Derivation failed to build + +When you see a build failure like below: + +``` +building /gnu/store/...-foo-3.6.12.drv... +/ 'check' phasenote: keeping build directory `/tmp/guix-build-foo-3.6.12.drv-0' +builder for `/gnu/store/...-foo-3.6.12.drv' failed with exit code 1 +build of /gnu/store/...-foo-3.6.12.drv failed +View build log at '/var/log/guix/drvs/../...-foo-3.6.12.drv.bz2'. +cannot build derivation `/gnu/store/...-qux-7.69.1.drv': 1 dependencies couldn't be built +cannot build derivation `/gnu/store/...-bar-3.16.5.drv': 1 dependencies couldn't be built +cannot build derivation `/gnu/store/...-baz-2.0.5.drv': 1 dependencies couldn't be built +guix time-machine: error: build of `/gnu/store/...-baz-2.0.5.drv' failed +``` + +It means that `guix` failed to build a package named `foo`, which was a +dependency of `qux`, `bar`, and `baz`. Importantly, note that the last "failed" +line is not necessarily the root cause, the first "failed" line is. + +Most of the time, the build failure is due to a spurious test failure or the +package's build system/test suite breaking when running multi-threaded. To +rebuild _just_ this derivation in a single-threaded fashion: + +```sh +$ guix build --cores=1 /gnu/store/...-foo-3.6.12.drv +``` + +If the single-threaded rebuild stil did not succeed, you may need to dig deeper. +You may view `foo`'s build logs in `less` like so (please replace paths with the +path you see in the build failure output): + +```sh +$ bzcat /var/log/guix/drvs/../...-foo-3.6.12.drv.bz2 | less +``` + +`foo`'s build directory is also preserved and available at +`/tmp/guix-build-foo-3.6.12.drv-0`. However, if you fail to build `foo` multiple +times, it may be `/tmp/...drv-1` or `/tmp/...drv-2`. Always consult the build +failure output for the most accurate, up-to-date information. + +#### python(-minimal): [Errno 84] Invalid or incomplete multibyte or wide character + +This error occurs when your `$TMPDIR` (default: /tmp) exists on a filesystem +which rejects characters not present in the UTF-8 character code set. An example +is ZFS with the utf8only=on option set. + +More information: https://bugs.python.org/issue37584 + ## FAQ ### How can I trust the binary installation? From 39741128d3775d198dbee34dc827353bfd18acd8 Mon Sep 17 00:00:00 2001 From: Carl Dong Date: Wed, 17 Mar 2021 13:03:46 -0400 Subject: [PATCH 08/15] guix: Supply --link-profile --- contrib/guix/guix-build | 1 + depends/Makefile | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/contrib/guix/guix-build b/contrib/guix/guix-build index a1c120f412..6c0930df18 100755 --- a/contrib/guix/guix-build +++ b/contrib/guix/guix-build @@ -317,6 +317,7 @@ EOF ${SDK_PATH:+--share="$SDK_PATH"} \ --cores="$JOBS" \ --keep-failed \ + --link-profile \ ${SUBSTITUTE_URLS:+--substitute-urls="$SUBSTITUTE_URLS"} \ ${ADDITIONAL_GUIX_COMMON_FLAGS} ${ADDITIONAL_GUIX_ENVIRONMENT_FLAGS} \ -- env HOST="$host" \ diff --git a/depends/Makefile b/depends/Makefile index 4cd4d72fc2..dadb21515a 100644 --- a/depends/Makefile +++ b/depends/Makefile @@ -141,8 +141,8 @@ build_id_string+=system_clang $(host_arch)_$(host_os)_id_string+=system_clang endif -build_id_string+=GUIX_ENVIRONMENT=$(GUIX_ENVIRONMENT) -$(host_arch)_$(host_os)_id_string+=GUIX_ENVIRONMENT=$(GUIX_ENVIRONMENT) +build_id_string+=GUIX_ENVIRONMENT=$(realpath $(GUIX_ENVIRONMENT)) +$(host_arch)_$(host_os)_id_string+=GUIX_ENVIRONMENT=$(realpath $(GUIX_ENVIRONMENT)) qrencode_packages_$(NO_QR) = $(qrencode_packages) From c1ae726a13ecfa5e7e9fdc3030a8110b8bb263f8 Mon Sep 17 00:00:00 2001 From: Carl Dong Date: Wed, 17 Mar 2021 13:07:40 -0400 Subject: [PATCH 09/15] guix: More thoroughly control native toolchain --- contrib/guix/libexec/build.sh | 26 ++++++++++++++++--- depends/packages/qt.mk | 3 ++- .../qt/qtbase-moc-ignore-gcc-macro.patch | 17 ++++++++++++ 3 files changed, 41 insertions(+), 5 deletions(-) create mode 100644 depends/patches/qt/qtbase-moc-ignore-gcc-macro.patch diff --git a/contrib/guix/libexec/build.sh b/contrib/guix/libexec/build.sh index bb83516f89..7d4609c17a 100644 --- a/contrib/guix/libexec/build.sh +++ b/contrib/guix/libexec/build.sh @@ -54,16 +54,34 @@ store_path() { # Set environment variables to point the NATIVE toolchain to the right # includes/libs NATIVE_GCC="$(store_path gcc-toolchain)" -export LIBRARY_PATH="${NATIVE_GCC}/lib:${NATIVE_GCC}/lib64" -export CPATH="${NATIVE_GCC}/include" + +unset LIBRARY_PATH +unset CPATH unset C_INCLUDE_PATH unset CPLUS_INCLUDE_PATH +unset OBJC_INCLUDE_PATH +unset OBJCPLUS_INCLUDE_PATH + +export LIBRARY_PATH="${NATIVE_GCC}/lib:${NATIVE_GCC}/lib64" +export C_INCLUDE_PATH="${NATIVE_GCC}/include" +export CPLUS_INCLUDE_PATH="${NATIVE_GCC}/include/c++:${NATIVE_GCC}/include" +export OBJC_INCLUDE_PATH="${NATIVE_GCC}/include" +export OBJCPLUS_INCLUDE_PATH="${NATIVE_GCC}/include/c++:${NATIVE_GCC}/include" + +prepend_to_search_env_var() { + export "${1}=${2}${!1:+:}${!1}" +} + case "$HOST" in *darwin*) # When targeting darwin, zlib is required by native_libdmg-hfsplus. zlib_store_path=$(store_path "zlib") - export LIBRARY_PATH="${zlib_store_path}/lib:${LIBRARY_PATH}" - export CPATH="${zlib_store_path}/include:${CPATH}" + + prepend_to_search_env_var LIBRARY_PATH "${zlib_store_path}/lib" + prepend_to_search_env_var C_INCLUDE_PATH "${zlib_store_path}/include" + prepend_to_search_env_var CPLUS_INCLUDE_PATH "${zlib_store_path}/include" + prepend_to_search_env_var OBJC_INCLUDE_PATH "${zlib_store_path}/include" + prepend_to_search_env_var OBJCPLUS_INCLUDE_PATH "${zlib_store_path}/include" esac # Set environment variables to point the CROSS toolchain to the right diff --git a/depends/packages/qt.mk b/depends/packages/qt.mk index ee6796e2ad..ae83eae911 100644 --- a/depends/packages/qt.mk +++ b/depends/packages/qt.mk @@ -10,7 +10,7 @@ $(package)_patches=fix_qt_pkgconfig.patch mac-qmake.conf fix_no_printer.patch no $(package)_patches+= fix_android_qmake_conf.patch fix_android_jni_static.patch dont_hardcode_pwd.patch $(package)_patches+= drop_lrelease_dependency.patch no_sdk_version_check.patch $(package)_patches+= fix_qpainter_non_determinism.patch fix_lib_paths.patch fix_android_pch.patch -$(package)_patches+= fix_bigsur_drawing.patch +$(package)_patches+= fix_bigsur_drawing.patch qtbase-moc-ignore-gcc-macro.patch $(package)_qttranslations_file_name=qttranslations-$($(package)_suffix) $(package)_qttranslations_sha256_hash=e1de58ed108b7e0a138815ea60fd46a2c4e1fc31396a707e5630e92de79c53de @@ -232,6 +232,7 @@ define $(package)_preprocess_cmds patch -p1 -i $($(package)_patch_dir)/no_sdk_version_check.patch && \ patch -p1 -i $($(package)_patch_dir)/fix_lib_paths.patch && \ patch -p1 -i $($(package)_patch_dir)/fix_bigsur_drawing.patch && \ + patch -p1 -i $($(package)_patch_dir)/qtbase-moc-ignore-gcc-macro.patch && \ sed -i.old "s|updateqm.commands = \$$$$\$$$$LRELEASE|updateqm.commands = $($(package)_extract_dir)/qttools/bin/lrelease|" qttranslations/translations/translations.pro && \ mkdir -p qtbase/mkspecs/macx-clang-linux &&\ cp -f qtbase/mkspecs/macx-clang/qplatformdefs.h qtbase/mkspecs/macx-clang-linux/ &&\ diff --git a/depends/patches/qt/qtbase-moc-ignore-gcc-macro.patch b/depends/patches/qt/qtbase-moc-ignore-gcc-macro.patch new file mode 100644 index 0000000000..0358bea6e9 --- /dev/null +++ b/depends/patches/qt/qtbase-moc-ignore-gcc-macro.patch @@ -0,0 +1,17 @@ +The moc executable loops through headers on CPLUS_INCLUDE_PATH and stumbles +on the GCC internal _GLIBCXX_VISIBILITY macro. Tell it to ignore it as it is +not supposed to be looking there to begin with. + +Upstream report: https://bugreports.qt.io/browse/QTBUG-83160 + +diff --git a/qtbase/src/tools/moc/main.cpp b/qtbase/src/tools/moc/main.cpp +--- a/qtbase/src/tools/moc/main.cpp ++++ b/qtbase/src/tools/moc/main.cpp +@@ -188,6 +188,7 @@ int runMoc(int argc, char **argv) + dummyVariadicFunctionMacro.arguments += Symbol(0, PP_IDENTIFIER, "__VA_ARGS__"); + pp.macros["__attribute__"] = dummyVariadicFunctionMacro; + pp.macros["__declspec"] = dummyVariadicFunctionMacro; ++ pp.macros["_GLIBCXX_VISIBILITY"] = dummyVariadicFunctionMacro; + + QString filename; + QString output; From 1742f8e12d163852df09575e03edcd3db73198ee Mon Sep 17 00:00:00 2001 From: Carl Dong Date: Wed, 17 Mar 2021 13:14:00 -0400 Subject: [PATCH 10/15] guix: Add early health check for guix-daemon --- contrib/guix/guix-build | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/contrib/guix/guix-build b/contrib/guix/guix-build index 6c0930df18..7f74524ddc 100755 --- a/contrib/guix/guix-build +++ b/contrib/guix/guix-build @@ -130,6 +130,33 @@ for host in $HOSTS; do esac done +################ +# Check that we can connect to the guix-daemon +################ + +cat << EOF +Checking that we can connect to the guix-daemon... + +Hint: If this hangs, you may want to try turning your guix-daemon off and on + again. + +EOF +if ! guix gc --list-failures > /dev/null; then +cat << EOF + +ERR: Failed to connect to the guix-daemon, please ensure that one is running and + reachable. +EOF +exit 1 +fi + +# Developer note: we could use `guix repl` for this check and run: +# +# (import (guix store)) (close-connection (open-connection)) +# +# However, the internal API is likely to change more than the CLI invocation + + ######### # SETUP # ######### From 1aec0eda8fd31a57b0621eea616398017c2ead98 Mon Sep 17 00:00:00 2001 From: Carl Dong Date: Wed, 17 Mar 2021 13:14:21 -0400 Subject: [PATCH 11/15] guix: Fallback to local build for substitute-enabled Guix users --- contrib/guix/guix-build | 2 ++ 1 file changed, 2 insertions(+) diff --git a/contrib/guix/guix-build b/contrib/guix/guix-build index 7f74524ddc..d817fb74ba 100755 --- a/contrib/guix/guix-build +++ b/contrib/guix/guix-build @@ -195,6 +195,7 @@ time-machine() { --commit=490e39ff303f4f6873a04bfb8253755bdae1b29c \ --cores="$JOBS" \ --keep-failed \ + --fallback \ ${SUBSTITUTE_URLS:+--substitute-urls="$SUBSTITUTE_URLS"} \ ${ADDITIONAL_GUIX_COMMON_FLAGS} ${ADDITIONAL_GUIX_TIMEMACHINE_FLAGS} \ -- "$@" @@ -344,6 +345,7 @@ EOF ${SDK_PATH:+--share="$SDK_PATH"} \ --cores="$JOBS" \ --keep-failed \ + --fallback \ --link-profile \ ${SUBSTITUTE_URLS:+--substitute-urls="$SUBSTITUTE_URLS"} \ ${ADDITIONAL_GUIX_COMMON_FLAGS} ${ADDITIONAL_GUIX_ENVIRONMENT_FLAGS} \ From ca85679eb43b8375a95d82101977829d08fb1e1b Mon Sep 17 00:00:00 2001 From: Carl Dong Date: Wed, 17 Mar 2021 13:15:08 -0400 Subject: [PATCH 12/15] guix: Use clang-toolchain instead of clang --- contrib/guix/manifest.scm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/guix/manifest.scm b/contrib/guix/manifest.scm index 4228532cb1..3fc3f4d404 100644 --- a/contrib/guix/manifest.scm +++ b/contrib/guix/manifest.scm @@ -237,5 +237,5 @@ chain for " target " development.")) ((string-contains target "-linux-") (list (make-bitcoin-cross-toolchain target))) ((string-contains target "darwin") - (list clang-8 binutils imagemagick libtiff librsvg font-tuffy cmake xorriso)) + (list clang-toolchain-8 binutils imagemagick libtiff librsvg font-tuffy cmake xorriso)) (else '()))))) From 65176ab5730dff34466caaecdd292625ef8294fc Mon Sep 17 00:00:00 2001 From: Carl Dong Date: Wed, 17 Mar 2021 15:33:18 -0400 Subject: [PATCH 13/15] guix: Remove codesign_allocate+pagestuff from unsigned tarball --- contrib/guix/libexec/build.sh | 3 --- 1 file changed, 3 deletions(-) diff --git a/contrib/guix/libexec/build.sh b/contrib/guix/libexec/build.sh index 7d4609c17a..4a6b7792a9 100644 --- a/contrib/guix/libexec/build.sh +++ b/contrib/guix/libexec/build.sh @@ -323,9 +323,6 @@ mkdir -p "$DISTSRC" osx_volname \ contrib/macdeploy/detached-sig-{apply,create}.sh \ "${BASEPREFIX}/${HOST}"/native/bin/dmg - for util in codesign_allocate pagestuff; do - cp --no-target-directory {"${BASEPREFIX}/${HOST}/native/bin/${HOST}-","unsigned-app-${HOST}/"}"$util" - done mv --target-directory="unsigned-app-${HOST}" dist ( cd "unsigned-app-${HOST}" From 06d6cf6784421290e6235fe8684d5e08ed6f1b62 Mon Sep 17 00:00:00 2001 From: Carl Dong Date: Thu, 18 Mar 2021 16:20:18 -0400 Subject: [PATCH 14/15] depends: libdmg-hfsplus: Skip CMake RPATH patching CMake's RPATH patching apparently causes non-reproducibility in the executables which are produced, manifesting in a difference in padding in the .dynstr section (we found this while investigating non-reproducibility in the "dmg" tool). This RPATH patching can be safely skipped for executables which don't depend on internal shared libraries. Documentation sources: 1. https://gitlab.kitware.com/cmake/community/-/wikis/doc/cmake/RPATH-handling 2. https://reproducible-builds.org/docs/deterministic-build-systems/#cmake-notes Prior debugging art: 1. https://stackoverflow.com/questions/63438206/cmake-g-reproducible-build-issue-with-changing-build-path 2. https://github.com/NXPmicro/mfgtools/pull/229/files --- depends/packages/native_libdmg-hfsplus.mk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/depends/packages/native_libdmg-hfsplus.mk b/depends/packages/native_libdmg-hfsplus.mk index 035b767188..f9d4b0797f 100644 --- a/depends/packages/native_libdmg-hfsplus.mk +++ b/depends/packages/native_libdmg-hfsplus.mk @@ -12,7 +12,7 @@ define $(package)_preprocess_cmds endef define $(package)_config_cmds - $($(package)_cmake) -DCMAKE_C_FLAGS="$$($(1)_cflags) -Wl,--build-id=none" .. + $($(package)_cmake) -DCMAKE_C_FLAGS="$$($(1)_cflags) -Wl,--build-id=none" -DCMAKE_SKIP_RPATH="ON" .. endef define $(package)_build_cmds From 7476b46f1893a4858616d2a8456a7c43238851ed Mon Sep 17 00:00:00 2001 From: Carl Dong Date: Tue, 23 Mar 2021 12:59:59 -0400 Subject: [PATCH 15/15] guix: Build dmg as a static binary This relatively easy change eliminates all runtime dependencies (except for the kernel) for dmg, which is the only native build tool that gets put in our output tarballs. This allows much more flexibility when constructing the codesigning environment, and is much more robust. --- contrib/guix/libexec/build.sh | 6 ++++-- contrib/guix/manifest.scm | 4 +++- depends/packages/native_libdmg-hfsplus.mk | 2 +- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/contrib/guix/libexec/build.sh b/contrib/guix/libexec/build.sh index 4a6b7792a9..4239c3d475 100644 --- a/contrib/guix/libexec/build.sh +++ b/contrib/guix/libexec/build.sh @@ -54,6 +54,7 @@ store_path() { # Set environment variables to point the NATIVE toolchain to the right # includes/libs NATIVE_GCC="$(store_path gcc-toolchain)" +NATIVE_GCC_STATIC="$(store_path gcc-toolchain static)" unset LIBRARY_PATH unset CPATH @@ -62,7 +63,7 @@ unset CPLUS_INCLUDE_PATH unset OBJC_INCLUDE_PATH unset OBJCPLUS_INCLUDE_PATH -export LIBRARY_PATH="${NATIVE_GCC}/lib:${NATIVE_GCC}/lib64" +export LIBRARY_PATH="${NATIVE_GCC}/lib:${NATIVE_GCC}/lib64:${NATIVE_GCC_STATIC}/lib:${NATIVE_GCC_STATIC}/lib64" export C_INCLUDE_PATH="${NATIVE_GCC}/include" export CPLUS_INCLUDE_PATH="${NATIVE_GCC}/include/c++:${NATIVE_GCC}/include" export OBJC_INCLUDE_PATH="${NATIVE_GCC}/include" @@ -76,8 +77,9 @@ case "$HOST" in *darwin*) # When targeting darwin, zlib is required by native_libdmg-hfsplus. zlib_store_path=$(store_path "zlib") + zlib_static_store_path=$(store_path "zlib" static) - prepend_to_search_env_var LIBRARY_PATH "${zlib_store_path}/lib" + prepend_to_search_env_var LIBRARY_PATH "${zlib_static_store_path}/lib:${zlib_store_path}/lib" prepend_to_search_env_var C_INCLUDE_PATH "${zlib_store_path}/include" prepend_to_search_env_var CPLUS_INCLUDE_PATH "${zlib_store_path}/include" prepend_to_search_env_var OBJC_INCLUDE_PATH "${zlib_store_path}/include" diff --git a/contrib/guix/manifest.scm b/contrib/guix/manifest.scm index 3fc3f4d404..910a9dd6f6 100644 --- a/contrib/guix/manifest.scm +++ b/contrib/guix/manifest.scm @@ -214,6 +214,7 @@ chain for " target " development.")) gzip xz zlib + (list zlib "static") ;; Build tools gnu-make libtool @@ -227,7 +228,8 @@ chain for " target " development.")) ;; Git git ;; Native gcc 7 toolchain - gcc-toolchain-7) + gcc-toolchain-7 + (list gcc-toolchain-7 "static")) (let ((target (getenv "HOST"))) (cond ((string-suffix? "-mingw32" target) ;; Windows diff --git a/depends/packages/native_libdmg-hfsplus.mk b/depends/packages/native_libdmg-hfsplus.mk index f9d4b0797f..c7c8adef41 100644 --- a/depends/packages/native_libdmg-hfsplus.mk +++ b/depends/packages/native_libdmg-hfsplus.mk @@ -12,7 +12,7 @@ define $(package)_preprocess_cmds endef define $(package)_config_cmds - $($(package)_cmake) -DCMAKE_C_FLAGS="$$($(1)_cflags) -Wl,--build-id=none" -DCMAKE_SKIP_RPATH="ON" .. + $($(package)_cmake) -DCMAKE_C_FLAGS="$$($(1)_cflags) -Wl,--build-id=none" -DCMAKE_SKIP_RPATH="ON" -DCMAKE_EXE_LINKER_FLAGS="-static" -DCMAKE_FIND_LIBRARY_SUFFIXES=".a" .. endef define $(package)_build_cmds