diff --git a/.appveyor.yml b/.appveyor.yml deleted file mode 100644 index 3ca7818eca..0000000000 --- a/.appveyor.yml +++ /dev/null @@ -1,63 +0,0 @@ -version: '{branch}.{build}' -skip_tags: true -image: Visual Studio 2019 -configuration: Release -platform: x64 -clone_depth: 5 -environment: - PATH: 'C:\Python37-x64;C:\Python37-x64\Scripts;%PATH%' - PYTHONUTF8: 1 - QT_DOWNLOAD_URL: 'https://github.com/sipsorcery/qt_win_binary/releases/download/qt51211x64_static_vs2019_16101/Qt5.12.11_x64_static_vs2019_16101.zip' - QT_DOWNLOAD_HASH: 'cf1b58107fadbf0d9a957d14dab16cde6b6eb6936a1908472da1f967dda34a3a' - QT_LOCAL_PATH: 'C:\Qt5.12.11_x64_static_vs2019_16101' - VCPKG_TAG: '75522bb1f2e7d863078bcd06322348f053a9e33f' -install: -# Disable zmq test for now since python zmq library on Windows would cause Access violation sometimes. -# - cmd: pip install zmq -# The powershell block below is to set up vcpkg to install the c++ dependencies. The pseudo code is: -# a. Checkout the vcpkg source (including port files) for the specific checkout and build the vcpkg binary, -# b. Append a setting to the vcpkg cmake config file to only do release builds of dependencies (skipping deubg builds saves ~5 mins). -# Note originally this block also installed the dependencies using 'vcpkg install'. Dependencies are now installed -# as part of the msbuild command using vcpkg mainfests. -- ps: | - cd c:\tools\vcpkg - $env:GIT_REDIRECT_STDERR = '2>&1' # git is writing non-errors to STDERR when doing git pull. Send to STDOUT instead. - git -c advice.detachedHead=false checkout $env:VCPKG_TAG - .\bootstrap-vcpkg.bat > $null - Add-Content "C:\tools\vcpkg\triplets\$env:PLATFORM-windows-static.cmake" "set(VCPKG_BUILD_TYPE release)" - cd "$env:APPVEYOR_BUILD_FOLDER" -before_build: -# Powershell block below is to download and extract the Qt static libraries. The pseudo code is: -# a. Download the zip file with the prebuilt Qt static libraries. -# b. Check that the downloaded file matches the expected hash. -# c. Extract the zip file to the specific destination path expected by the msbuild projects. -- ps: | - Write-Host "Downloading Qt binaries."; - Invoke-WebRequest -Uri $env:QT_DOWNLOAD_URL -Out qtdownload.zip; - Write-Host "Qt binaries successfully downloaded, checking hash against $env:QT_DOWNLOAD_HASH..."; - if((Get-FileHash qtdownload.zip).Hash -eq $env:QT_DOWNLOAD_HASH) { - Expand-Archive qtdownload.zip -DestinationPath $env:QT_LOCAL_PATH; - Write-Host "Qt binary download matched the expected hash."; - } - else { - Write-Host "ERROR: Qt binary download did not match the expected hash."; - Exit-AppveyorBuild; - } -- cmd: python build_msvc\msvc-autogen.py -build_script: -- cmd: msbuild /p:TrackFileAccess=false build_msvc\bitcoin.sln /m /v:q /nologo -after_build: -#- 7z a bitcoin-%APPVEYOR_BUILD_VERSION%.zip %APPVEYOR_BUILD_FOLDER%\build_msvc\%platform%\%configuration%\*.exe -test_script: -- cmd: src\test_bitcoin.exe -l test_suite -- cmd: src\bench_bitcoin.exe > NUL -- ps: python test\util\bitcoin-util-test.py -- cmd: python test\util\rpcauth-test.py -# Fee estimation test failing on appveyor with: WinError 10048] Only one usage of each socket address (protocol/network address/port) is normally permitted. -# functional tests disabled for now. See -# https://github.com/bitcoin/bitcoin/pull/18626#issuecomment-613396202 -# https://github.com/bitcoin/bitcoin/issues/18623 -# - cmd: python test\functional\test_runner.py --ci --quiet --combinedlogslen=4000 --failfast --exclude feature_fee_estimation -artifacts: -#- path: bitcoin-%APPVEYOR_BUILD_VERSION%.zip -deploy: off diff --git a/.cirrus.yml b/.cirrus.yml index dd94f725fb..65f9fa07db 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -1,6 +1,4 @@ -### Global defaults - -env: +env: # Global defaults SECP256K1_TEST_ITERS: 16 # ELEMENTS: avoid test timeouts on arm PACKAGE_MANAGER_INSTALL: "apt-get update && apt-get install -y" MAKEJOBS: "-j3" # ELEMENTS: reduced from j4 @@ -19,57 +17,47 @@ persistent_worker_template: &PERSISTENT_WORKER_TEMPLATE persistent_worker: {} # https://cirrus-ci.org/guide/persistent-workers/ # https://cirrus-ci.org/guide/tips-and-tricks/#sharing-configuration-between-tasks -base_template: &BASE_TEMPLATE +filter_template: &FILTER_TEMPLATE skip: $CIRRUS_REPO_FULL_NAME == "bitcoin-core/gui" && $CIRRUS_PR == "" # No need to run on the read-only mirror, unless it is a PR. https://cirrus-ci.org/guide/writing-tasks/#conditional-task-execution - merge_base_script: - - if [ "$CIRRUS_PR" = "" ]; then exit 0; fi - - bash -c "$PACKAGE_MANAGER_INSTALL git" - - git fetch $CIRRUS_REPO_CLONE_URL $CIRRUS_BASE_BRANCH - - git config --global user.email "ci@ci.ci" - - git config --global user.name "ci" - - git merge FETCH_HEAD # Merge base to detect silent merge conflicts stateful: false # https://cirrus-ci.org/guide/writing-tasks/#stateful-tasks -global_task_template: &GLOBAL_TASK_TEMPLATE - << : *BASE_TEMPLATE +base_template: &BASE_TEMPLATE + << : *FILTER_TEMPLATE + merge_base_script: + # Unconditionally install git (used in fingerprint_script) and set the + # default git author name (used in verify-commits.py) + - bash -c "$PACKAGE_MANAGER_INSTALL git" + - git config --global user.email "ci@ci.ci" + - git config --global user.name "ci" + - if [ "$CIRRUS_PR" = "" ]; then exit 0; fi + - git fetch $CIRRUS_REPO_CLONE_URL $CIRRUS_BASE_BRANCH + - git merge FETCH_HEAD # Merge base to detect silent merge conflicts + +main_template: &MAIN_TEMPLATE timeout_in: 120m # https://cirrus-ci.org/faq/#instance-timed-out container: # https://cirrus-ci.org/faq/#are-there-any-limits # Each project has 16 CPU in total, assign 2 to each container, so that 8 tasks run in parallel cpu: 2 + greedy: true memory: 8G # Set to 8GB to avoid OOM. https://cirrus-ci.org/guide/linux/#linux-containers ccache_cache: folder: "/tmp/ccache_dir" depends_built_cache: folder: "depends/built" + fingerprint_script: echo $CIRRUS_TASK_NAME $(git rev-list -1 HEAD ./depends) ci_script: - ./ci/test_run_all.sh -depends_sdk_cache_template: &DEPENDS_SDK_CACHE_TEMPLATE - depends_sdk_cache: - folder: "depends/sdk-sources" +global_task_template: &GLOBAL_TASK_TEMPLATE + << : *BASE_TEMPLATE + << : *MAIN_TEMPLATE compute_credits_template: &CREDITS_TEMPLATE # https://cirrus-ci.org/pricing/#compute-credits # Only use credits for pull requests to the main repo use_compute_credits: $CIRRUS_REPO_FULL_NAME == 'ElementsProject/elements' && $CIRRUS_PR != "" -#task: -# name: "Windows" -# windows_container: -# image: cirrusci/windowsservercore:2019 -# env: -# CIRRUS_SHELL: powershell -# PATH: 'C:\Python37;C:\Python37\Scripts;%PATH%' -# PYTHONUTF8: 1 -# QT_DOWNLOAD_URL: 'https://github.com/sipsorcery/qt_win_binary/releases/download/v1.6/Qt5.9.8_x64_static_vs2019.zip' -# QT_DOWNLOAD_HASH: '9a8c6eb20967873785057fdcd329a657c7f922b0af08c5fde105cc597dd37e21' -# QT_LOCAL_PATH: 'C:\Qt5.9.8_x64_static_vs2019' -# VCPKG_INSTALL_PATH: 'C:\tools\vcpkg\installed' -# VCPKG_COMMIT_ID: 'ed0df8ecc4ed7e755ea03e18aaf285fd9b4b4a74' -# install_script: -# - choco install python --version=3.7.7 -y - task: name: 'lint [bionic]' << : *BASE_TEMPLATE @@ -85,19 +73,114 @@ task: << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV task: - name: 'ARM [unit tests, no functional tests] [buster]' + name: "Win64 native [msvc]" + << : *FILTER_TEMPLATE + windows_container: + cpu: 4 + memory: 8G + image: cirrusci/windowsservercore:visualstudio2019 + timeout_in: 120m + env: + PATH: 'C:\jom;C:\Python39;C:\Python39\Scripts;C:\Program Files (x86)\Microsoft Visual Studio\2019\BuildTools\MSBuild\Current\Bin;%PATH%' + PYTHONUTF8: 1 + CI_VCPKG_TAG: '2021.05.12' + VCPKG_DOWNLOADS: 'C:\Users\ContainerAdministrator\AppData\Local\vcpkg\downloads' + VCPKG_DEFAULT_BINARY_CACHE: 'C:\Users\ContainerAdministrator\AppData\Local\vcpkg\archives' + QT_DOWNLOAD_URL: 'https://download.qt.io/official_releases/qt/5.15/5.15.2/single/qt-everywhere-src-5.15.2.zip' + QT_LOCAL_PATH: 'C:\qt-everywhere-src-5.15.2.zip' + QT_SOURCE_DIR: 'C:\qt-everywhere-src-5.15.2' + QTBASEDIR: 'C:\Qt_static' + x64_NATIVE_TOOLS: '"C:\Program Files (x86)\Microsoft Visual Studio\2019\BuildTools\VC\Auxiliary\Build\vcvars64.bat"' + IgnoreWarnIntDirInTempDetected: 'true' + merge_script: + - git config --global user.email "ci@ci.ci" + - git config --global user.name "ci" + # Windows filesystem loses the executable bit, and all of the executable + # files are considered "modified" now. It will break the following `git merge` + # command. The next two commands make git ignore this issue. + - git config core.filemode false + - git reset --hard + - PowerShell -NoLogo -Command if ($env:CIRRUS_PR -ne $null) { git fetch $env:CIRRUS_REPO_CLONE_URL $env:CIRRUS_BASE_BRANCH; git merge FETCH_HEAD; } + msvc_qt_built_cache: + folder: "%QTBASEDIR%" + reupload_on_changes: false + fingerprint_script: + - echo %QT_DOWNLOAD_URL% + - msbuild -version + populate_script: + - curl -L -o C:\jom.zip http://download.qt.io/official_releases/jom/jom.zip + - mkdir C:\jom + - tar -xf C:\jom.zip -C C:\jom + - curl -L -o %QT_LOCAL_PATH% %QT_DOWNLOAD_URL% + - tar -xf %QT_LOCAL_PATH% -C C:\ + - '%x64_NATIVE_TOOLS%' + - cd %QT_SOURCE_DIR% + - mkdir build + - cd build + - ..\configure -release -silent -opensource -confirm-license -opengl desktop -static -static-runtime -mp -qt-zlib -qt-pcre -qt-libpng -nomake examples -nomake tests -nomake tools -no-angle -no-dbus -no-gif -no-gtk -no-ico -no-icu -no-libjpeg -no-libudev -no-sql-sqlite -no-sql-odbc -no-sqlite -no-vulkan -skip qt3d -skip qtactiveqt -skip qtandroidextras -skip qtcharts -skip qtconnectivity -skip qtdatavis3d -skip qtdeclarative -skip doc -skip qtdoc -skip qtgamepad -skip qtgraphicaleffects -skip qtimageformats -skip qtlocation -skip qtlottie -skip qtmacextras -skip qtmultimedia -skip qtnetworkauth -skip qtpurchasing -skip qtquick3d -skip qtquickcontrols -skip qtquickcontrols2 -skip qtquicktimeline -skip qtremoteobjects -skip qtscript -skip qtscxml -skip qtsensors -skip qtserialbus -skip qtserialport -skip qtspeech -skip qtsvg -skip qtvirtualkeyboard -skip qtwayland -skip qtwebchannel -skip qtwebengine -skip qtwebglplugin -skip qtwebsockets -skip qtwebview -skip qtx11extras -skip qtxmlpatterns -no-openssl -no-feature-bearermanagement -no-feature-printdialog -no-feature-printer -no-feature-printpreviewdialog -no-feature-printpreviewwidget -no-feature-sql -no-feature-sqlmodel -no-feature-textbrowser -no-feature-textmarkdownwriter -no-feature-textodfwriter -no-feature-xml -prefix %QTBASEDIR% + - jom + - jom install + vcpkg_tools_cache: + folder: '%VCPKG_DOWNLOADS%\tools' + reupload_on_changes: false + fingerprint_script: + - echo %CI_VCPKG_TAG% + - msbuild -version + vcpkg_binary_cache: + folder: '%VCPKG_DEFAULT_BINARY_CACHE%' + reupload_on_changes: true + fingerprint_script: + - echo %CI_VCPKG_TAG% + - msbuild -version + populate_script: + - mkdir %VCPKG_DEFAULT_BINARY_CACHE% + install_python_script: + - choco install --yes --no-progress python3 --version=3.9.6 + - pip install zmq + - python -VV + install_vcpkg_script: + - cd .. + - git clone --quiet https://github.com/microsoft/vcpkg.git + - cd vcpkg + - git -c advice.detachedHead=false checkout %CI_VCPKG_TAG% + - .\bootstrap-vcpkg -disableMetrics + - echo set(VCPKG_BUILD_TYPE release) >> triplets\x64-windows-static.cmake + - .\vcpkg integrate install + - .\vcpkg version + build_script: + - cd %CIRRUS_WORKING_DIR% + - python build_msvc\msvc-autogen.py + - msbuild build_msvc\bitcoin.sln -property:Configuration=Release -maxCpuCount -verbosity:minimal -noLogo + unit_tests_script: + - src\test_bitcoin.exe -l test_suite + - src\bench_bitcoin.exe > NUL + - python test\util\test_runner.py + - python test\util\rpcauth-test.py + functional_tests_script: + # Increase the dynamic port range to the maximum allowed value to mitigate "OSError: [WinError 10048] Only one usage of each socket address (protocol/network address/port) is normally permitted". + # See: https://docs.microsoft.com/en-us/biztalk/technical-guides/settings-that-can-be-modified-to-improve-network-performance + - netsh int ipv4 set dynamicport tcp start=1025 num=64511 + - netsh int ipv6 set dynamicport tcp start=1025 num=64511 + # Exclude feature_dbcrash for now due to timeout + - python test\functional\test_runner.py --nocleanup --ci --quiet --combinedlogslen=4000 --jobs=4 --timeout-factor=8 --extended --exclude feature_dbcrash + +task: + name: 'ARM [unit tests, no functional tests] [bullseye]' << : *GLOBAL_TASK_TEMPLATE - container: - image: debian:buster + arm_container: + image: debian:bullseye + cpu: 2 + memory: 8G env: << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV FILE_ENV: "./ci/test/00_setup_env_arm.sh" + QEMU_USER_CMD: "" # Disable qemu and run the test natively task: - name: 'Win64 [unit tests, no gui tests, no boost::process, no functional tests] [focal]' + name: 'Win64 [unit tests, no gui tests, no boost::process, no functional tests] [jammy]' << : *GLOBAL_TASK_TEMPLATE container: - image: ubuntu:focal + image: ubuntu:jammy env: << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV FILE_ENV: "./ci/test/00_setup_env_win64.sh" @@ -123,51 +206,51 @@ task: FILE_ENV: "./ci/test/00_setup_env_native_qt5.sh" task: - name: '[depends, sanitizers: thread (TSan), no gui] [hirsute]' + name: '[TSan, depends, gui] [jammy]' << : *GLOBAL_TASK_TEMPLATE container: - image: ubuntu:hirsute + image: ubuntu:jammy cpu: 6 # Increase CPU and Memory to avoid timeout memory: 24G env: << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV - MAKEJOBS: "-j6" # ELEMENTS: reduced from -j8 FILE_ENV: "./ci/test/00_setup_env_native_tsan.sh" task: - name: '[depends, sanitizers: memory (MSan)] [focal]' + name: '[MSan, depends] [focal]' << : *GLOBAL_TASK_TEMPLATE container: image: ubuntu:focal env: << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV FILE_ENV: "./ci/test/00_setup_env_native_msan.sh" + MAKEJOBS: "-j4" # Avoid excessive memory use due to MSan task: - name: '[no depends, sanitizers: address/leak (ASan + LSan) + undefined (UBSan) + integer] [hirsute]' + name: '[ASan + LSan + UBSan + integer, no depends] [jammy]' << : *GLOBAL_TASK_TEMPLATE container: - image: ubuntu:hirsute + image: ubuntu:jammy memory: 16G # ELEMENTS: need more memory cpu: 4 # ELEMENTS: cirrus wants more CPUs if you want more memory env: << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV FILE_ENV: "./ci/test/00_setup_env_native_asan.sh" + MAKEJOBS: "-j4" # Avoid excessive memory use task: - name: '[no depends, sanitizers: fuzzer,address,undefined,integer] [focal]' + name: '[fuzzer,address,undefined,integer, no depends] [jammy]' << : *GLOBAL_TASK_TEMPLATE container: - image: ubuntu:focal + image: ubuntu:jammy cpu: 8 # Increase CPU and memory to avoid timeout memory: 16G env: << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV - MAKEJOBS: "-j8" FILE_ENV: "./ci/test/00_setup_env_native_fuzz.sh" task: - name: '[multiprocess, DEBUG] [focal]' + name: '[multiprocess, i686, DEBUG] [focal]' # Disable for Elements for now; Multiprocess build is not supported or tested and fails CI. only_if: false << : *GLOBAL_TASK_TEMPLATE @@ -177,36 +260,39 @@ task: memory: 16G # The default memory is sometimes just a bit too small, so double everything env: << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV - MAKEJOBS: "-j8" - FILE_ENV: "./ci/test/00_setup_env_native_multiprocess.sh" + FILE_ENV: "./ci/test/00_setup_env_i686_multiprocess.sh" task: - name: '[no wallet] [bionic]' + name: '[no wallet, libbitcoinkernel] [bionic]' << : *GLOBAL_TASK_TEMPLATE container: image: ubuntu:bionic env: << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV - FILE_ENV: "./ci/test/00_setup_env_native_nowallet.sh" + FILE_ENV: "./ci/test/00_setup_env_native_nowallet_libbitcoinkernel.sh" task: - name: 'macOS 10.14 [gui, no tests] [focal]' - << : *DEPENDS_SDK_CACHE_TEMPLATE - << : *GLOBAL_TASK_TEMPLATE + name: 'macOS 10.15 [gui, no tests] [focal]' + << : *BASE_TEMPLATE + macos_sdk_cache: + folder: "depends/SDKs/$MACOS_SDK" + fingerprint_key: "$MACOS_SDK" + << : *MAIN_TEMPLATE container: image: ubuntu:focal env: + MACOS_SDK: "Xcode-12.2-12B45b-extracted-SDK-with-libcxx-headers" << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV FILE_ENV: "./ci/test/00_setup_env_mac.sh" task: - name: 'macOS 11 native [gui] [no depends]' + name: 'macOS 12 native [gui, system sqlite only] [no depends]' brew_install_script: - - brew install boost libevent berkeley-db4 qt@5 miniupnpc libnatpmp ccache zeromq qrencode sqlite libtool automake pkg-config gnu-getopt + - brew install boost libevent qt@5 miniupnpc libnatpmp ccache zeromq qrencode libtool automake gnu-getopt << : *GLOBAL_TASK_TEMPLATE - osx_instance: + macos_instance: # Use latest image, but hardcode version to avoid silent upgrades (and breaks) - image: big-sur-xcode-12.5 # https://cirrus-ci.org/guide/macOS + image: monterey-xcode-13.2 # https://cirrus-ci.org/guide/macOS env: << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV CI_USE_APT_INSTALL: "no" @@ -217,10 +303,13 @@ task: name: 'ARM64 Android APK [focal]' # Disable for Elements; Android build is broken and unsupported. only_if: false - << : *DEPENDS_SDK_CACHE_TEMPLATE + android_sdk_cache: + folder: "depends/SDKs/android" + fingerprint_key: "ANDROID_API_LEVEL=28 ANDROID_BUILD_TOOLS_VERSION=28.0.3 ANDROID_NDK_VERSION=23.1.7779620" depends_sources_cache: folder: "depends/sources" - << : *GLOBAL_TASK_TEMPLATE + fingerprint_script: git rev-list -1 HEAD ./depends + << : *MAIN_TEMPLATE container: image: ubuntu:focal env: diff --git a/.editorconfig b/.editorconfig index 4967e675f6..ae7e92d1c8 100644 --- a/.editorconfig +++ b/.editorconfig @@ -13,7 +13,7 @@ trim_trailing_whitespace = true [*.{h,cpp,py,sh}] indent_size = 4 -# .cirrus.yml, .appveyor.yml, .fuzzbuzz.yml, etc. +# .cirrus.yml, .fuzzbuzz.yml, etc. [*.yml] indent_size = 2 diff --git a/.gitignore b/.gitignore index ff1c765972..fcb8fddf63 100644 --- a/.gitignore +++ b/.gitignore @@ -9,6 +9,7 @@ src/elements-gui src/elements-node src/elements-tx src/elements-util +src/elements-chainstate src/elements-wallet src/test/fuzz/fuzz src/test/test_bitcoin @@ -83,6 +84,7 @@ src/qt/bitcoin-qt.includes # Only ignore unexpected patches *.patch +!contrib/guix/patches/*.patch !depends/patches/**/*.patch #libtool object files @@ -95,7 +97,6 @@ Makefile !depends/Makefile src/qt/elements-qt Elements-Qt.app -background.tiff* # Qt Creator Makefile.am.user @@ -148,6 +149,5 @@ db4/ osx_volname dist/ -*.background.tiff /guix-build-* diff --git a/.tx/config b/.tx/config index 232d481e18..c4fe7cc324 100644 --- a/.tx/config +++ b/.tx/config @@ -1,7 +1,7 @@ [main] host = https://www.transifex.com -[bitcoin.qt-translation-022x] +[bitcoin.qt-translation-023x] file_filter = src/qt/locale/bitcoin_.xlf source_file = src/qt/locale/bitcoin_en.xlf source_lang = en diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index b2aac20a66..e81b477714 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -12,8 +12,8 @@ revolves around a meritocracy where contributors earn trust from the developer community over time. Nevertheless, some hierarchy is necessary for practical purposes. As such, there are repository "maintainers" who are responsible for merging pull requests, as well as a "lead maintainer" who is responsible for the -release cycle as well as overall merging, moderation and appointment of -maintainers. +[release cycle](/doc/release-process.md) as well as overall merging, moderation +and appointment of maintainers. Getting Started --------------- @@ -60,8 +60,8 @@ Most communication about Bitcoin Core development happens on IRC, in the `#bitcoin-core-dev` channel on Libera Chat. The easiest way to participate on IRC is with the web client, [web.libera.chat](https://web.libera.chat/#bitcoin-core-dev). Chat history logs can be found -on [http://www.erisian.com.au/bitcoin-core-dev/](http://www.erisian.com.au/bitcoin-core-dev/) -and [http://gnusha.org/bitcoin-core-dev/](http://gnusha.org/bitcoin-core-dev/). +on [https://www.erisian.com.au/bitcoin-core-dev/](https://www.erisian.com.au/bitcoin-core-dev/) +and [https://gnusha.org/bitcoin-core-dev/](https://gnusha.org/bitcoin-core-dev/). Discussion about codebase improvements happens in GitHub issues and pull requests. @@ -81,7 +81,7 @@ facilitates social contribution, easy testing and peer review. To contribute a patch, the workflow is as follows: - 1. Fork repository ([only for the first time](https://help.github.com/en/articles/fork-a-repo)) + 1. Fork repository ([only for the first time](https://docs.github.com/en/get-started/quickstart/fork-a-repo)) 1. Create topic branch 1. Commit patches @@ -119,7 +119,7 @@ own without warnings, errors, regressions, or test failures. Commit messages should be verbose by default consisting of a short subject line (50 chars max), a blank line and detailed explanatory text as separate -paragraph(s), unless the title alone is self-explanatory (like "Corrected typo +paragraph(s), unless the title alone is self-explanatory (like "Correct typo in init.cpp") in which case a single title line is sufficient. Commit messages should be helpful to people reading your code in the future, so explain the reasoning for your decisions. Further explanation [here](https://chris.beams.io/posts/git-commit/). @@ -182,16 +182,21 @@ for more information on helping with translations. ### Work in Progress Changes and Requests for Comments If a pull request is not to be considered for merging (yet), please -prefix the title with [WIP] or use [Tasks Lists](https://help.github.com/articles/basic-writing-and-formatting-syntax/#task-lists) +prefix the title with [WIP] or use [Tasks Lists](https://docs.github.com/en/github/writing-on-github/getting-started-with-writing-and-formatting-on-github/basic-writing-and-formatting-syntax#task-lists) in the body of the pull request to indicate tasks are pending. ### Address Feedback At this stage, one should expect comments and review from other contributors. You can add more commits to your pull request by committing them locally and pushing -to your fork until you have satisfied all feedback. +to your fork. -Note: Code review is a burdensome but important part of the development process, and as such, certain types of pull requests are rejected. In general, if the **improvements** do not warrant the **review effort** required, the PR has a high chance of being rejected. It is up to the PR author to convince the reviewers that the changes warrant the review effort, and if reviewers are "Concept NACK'ing" the PR, the author may need to present arguments and/or do research backing their suggested changes. +You are expected to reply to any review comments before your pull request is +merged. You may update the code or reject the feedback if you do not agree with +it, but you should express so in a reply. If there is outstanding feedback and +you are not actively working on it, your pull request may be closed. + +Please refer to the [peer review](#peer-review) section below for more details. ### Squashing Commits @@ -322,6 +327,14 @@ maintainers take into account the peer review when determining if there is consensus to merge a pull request (remember that discussions may have been spread out over GitHub, mailing list and IRC discussions). +Code review is a burdensome but important part of the development process, and +as such, certain types of pull requests are rejected. In general, if the +**improvements** do not warrant the **review effort** required, the PR has a +high chance of being rejected. It is up to the PR author to convince the +reviewers that the changes warrant the review effort, and if reviewers are +"Concept NACK'ing" the PR, the author may need to present arguments and/or do +research backing their suggested changes. + #### Conceptual Review A review can be a conceptual review, where the reviewer leaves a comment @@ -386,7 +399,7 @@ about: - It may be because your code is too complex for all but a few people, and those people may not have realized your pull request even exists. A great way to find people who are qualified and care about the code you are touching is the - [Git Blame feature](https://help.github.com/articles/tracing-changes-in-a-file/). Simply + [Git Blame feature](https://docs.github.com/en/github/managing-files-in-a-repository/managing-files-on-github/tracking-changes-in-a-file). Simply look up who last modified the code you are changing and see if you can find them and give them a nudge. Don't be incessant about the nudging, though. - Finally, if all else fails, ask on IRC or elsewhere for someone to give your pull request @@ -422,11 +435,6 @@ https://github.com/bitcoin/bitcoin/pull/16189). Also see the [backport.py script]( https://github.com/bitcoin-core/bitcoin-maintainer-tools#backport). -Release Policy --------------- - -The project leader is the release manager for each Bitcoin Core release. - Copyright --------- diff --git a/COPYING b/COPYING index c34f575b8c..b157c5fe49 100644 --- a/COPYING +++ b/COPYING @@ -1,7 +1,7 @@ The MIT License (MIT) -Copyright (c) 2009-2021 The Bitcoin Core developers -Copyright (c) 2009-2021 Bitcoin Developers +Copyright (c) 2009-2022 The Bitcoin Core developers +Copyright (c) 2009-2022 Bitcoin Developers Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/INSTALL.md b/INSTALL.md index 520a47d960..4cead03036 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -1,5 +1 @@ -Building Bitcoin -================ - -See doc/build-*.md for instructions on building the various -elements of the Bitcoin Core reference implementation of Bitcoin. +See [doc/build-\*.md](/doc) \ No newline at end of file diff --git a/Makefile.am b/Makefile.am index 2ec8d5c642..59fb949699 100644 --- a/Makefile.am +++ b/Makefile.am @@ -38,9 +38,7 @@ OSX_APP=Elements-Qt.app OSX_VOLNAME = $(subst $(space),-,$(PACKAGE_NAME)) OSX_DMG = $(OSX_VOLNAME).dmg OSX_TEMP_ISO = $(OSX_DMG:.dmg=).temp.iso -OSX_BACKGROUND_SVG=background.svg -OSX_BACKGROUND_IMAGE=background.tiff -OSX_BACKGROUND_IMAGE_DPIS=36 72 +OSX_BACKGROUND_IMAGE=$(top_srcdir)/contrib/macdeploy/background.tiff OSX_DEPLOY_SCRIPT=$(top_srcdir)/contrib/macdeploy/macdeployqtplus OSX_INSTALLER_ICONS=$(top_srcdir)/src/qt/res/icons/bitcoin.icns OSX_PLIST=$(top_builddir)/share/qt/Info.plist #not installed @@ -58,8 +56,7 @@ DIST_SHARE = \ BIN_CHECKS=$(top_srcdir)/contrib/devtools/symbol-check.py \ $(top_srcdir)/contrib/devtools/security-check.py \ - $(top_srcdir)/contrib/devtools/utils.py \ - $(top_srcdir)/contrib/devtools/pixie.py + $(top_srcdir)/contrib/devtools/utils.py WINDOWS_PACKAGING = $(top_srcdir)/share/pixmaps/bitcoin.ico \ $(top_srcdir)/share/pixmaps/nsis-header.bmp \ @@ -67,7 +64,6 @@ WINDOWS_PACKAGING = $(top_srcdir)/share/pixmaps/bitcoin.ico \ $(top_srcdir)/doc/README_windows.txt OSX_PACKAGING = $(OSX_DEPLOY_SCRIPT) $(OSX_INSTALLER_ICONS) \ - $(top_srcdir)/contrib/macdeploy/$(OSX_BACKGROUND_SVG) \ $(top_srcdir)/contrib/macdeploy/detached-sig-apply.sh \ $(top_srcdir)/contrib/macdeploy/detached-sig-create.sh @@ -128,20 +124,13 @@ osx_volname: echo $(OSX_VOLNAME) >$@ if BUILD_DARWIN -$(OSX_DMG): $(OSX_APP_BUILT) $(OSX_PACKAGING) $(OSX_BACKGROUND_IMAGE) +$(OSX_DMG): $(OSX_APP_BUILT) $(OSX_PACKAGING) $(PYTHON) $(OSX_DEPLOY_SCRIPT) $(OSX_APP) $(OSX_VOLNAME) -translations-dir=$(QT_TRANSLATION_DIR) -dmg -$(OSX_BACKGROUND_IMAGE).png: contrib/macdeploy/$(OSX_BACKGROUND_SVG) - sed 's/PACKAGE_NAME/$(PACKAGE_NAME)/' < "$<" | $(RSVG_CONVERT) -f png -d 36 -p 36 -o $@ -$(OSX_BACKGROUND_IMAGE)@2x.png: contrib/macdeploy/$(OSX_BACKGROUND_SVG) - sed 's/PACKAGE_NAME/$(PACKAGE_NAME)/' < "$<" | $(RSVG_CONVERT) -f png -d 72 -p 72 -o $@ -$(OSX_BACKGROUND_IMAGE): $(OSX_BACKGROUND_IMAGE).png $(OSX_BACKGROUND_IMAGE)@2x.png - tiffutil -cathidpicheck $^ -out $@ - deploydir: $(OSX_DMG) else !BUILD_DARWIN APP_DIST_DIR=$(top_builddir)/dist -APP_DIST_EXTRAS=$(APP_DIST_DIR)/.background/$(OSX_BACKGROUND_IMAGE) $(APP_DIST_DIR)/.DS_Store $(APP_DIST_DIR)/Applications +APP_DIST_EXTRAS=$(APP_DIST_DIR)/.background/background.tiff $(APP_DIST_DIR)/.DS_Store $(APP_DIST_DIR)/Applications $(APP_DIST_DIR)/Applications: @rm -f $@ @@ -155,12 +144,9 @@ $(OSX_TEMP_ISO): $(APP_DIST_EXTRAS) $(OSX_DMG): $(OSX_TEMP_ISO) $(DMG) dmg "$<" "$@" -dpi%.$(OSX_BACKGROUND_IMAGE): contrib/macdeploy/$(OSX_BACKGROUND_SVG) - sed 's/PACKAGE_NAME/$(PACKAGE_NAME)/' < "$<" | $(RSVG_CONVERT) -f png -d $* -p $* | $(IMAGEMAGICK_CONVERT) - $@ -OSX_BACKGROUND_IMAGE_DPIFILES := $(foreach dpi,$(OSX_BACKGROUND_IMAGE_DPIS),dpi$(dpi).$(OSX_BACKGROUND_IMAGE)) -$(APP_DIST_DIR)/.background/$(OSX_BACKGROUND_IMAGE): $(OSX_BACKGROUND_IMAGE_DPIFILES) +$(APP_DIST_DIR)/.background/background.tiff: $(MKDIR_P) $(@D) - $(TIFFCP) -c none $(OSX_BACKGROUND_IMAGE_DPIFILES) $@ + cp $(OSX_BACKGROUND_IMAGE) $@ $(APP_DIST_DIR)/$(OSX_APP)/Contents/MacOS/Elements-Qt: $(OSX_APP_BUILT) $(OSX_PACKAGING) INSTALLNAMETOOL=$(INSTALLNAMETOOL) OTOOL=$(OTOOL) STRIP=$(STRIP) $(PYTHON) $(OSX_DEPLOY_SCRIPT) $(OSX_APP) $(OSX_VOLNAME) -translations-dir=$(QT_TRANSLATION_DIR) @@ -207,6 +193,7 @@ LCOV_FILTER_PATTERN = \ -p "src/bench/" \ -p "src/univalue" \ -p "src/crypto/ctaes" \ + -p "src/minisketch" \ -p "src/secp256k1" \ -p "depends" @@ -282,7 +269,7 @@ dist_noinst_SCRIPTS = autogen.sh EXTRA_DIST = $(DIST_SHARE) test/functional/test_runner.py test/functional test/bitcoin_functional/functional test/bitcoin_functional/functional/test_runner.py test/fuzz $(DIST_CONTRIB) $(WINDOWS_PACKAGING) $(OSX_PACKAGING) $(BIN_CHECKS) EXTRA_DIST += \ - test/util/bitcoin-util-test.py \ + test/util/test_runner.py \ test/util/data/bitcoin-util-test.json \ test/util/data/blanktxv1.hex \ test/util/data/blanktxv1.json \ @@ -332,6 +319,7 @@ EXTRA_DIST += \ test/util/data/txcreatescript4.json \ test/util/data/txcreatescript5.hex \ test/util/data/txcreatescript6.hex \ + test/util/data/txcreatesignsegwit1.hex \ test/util/data/txcreatesignv1.hex \ test/util/data/txcreatesignv1.json \ test/util/data/txcreatesignv2.hex \ @@ -361,18 +349,18 @@ clean-docs: clean-local: clean-docs rm -rf coverage_percent.txt test_bitcoin.coverage/ total.coverage/ fuzz.coverage/ test/tmp/ cache/ $(OSX_APP) rm -rf test/functional/__pycache__ test/functional/test_framework/__pycache__ test/cache share/rpcauth/__pycache__ - rm -rf osx_volname dist/ dpi36.background.tiff dpi72.background.tiff + rm -rf osx_volname dist/ test-security-check: if TARGET_DARWIN - $(AM_V_at) CC='$(CC)' $(PYTHON) $(top_srcdir)/contrib/devtools/test-security-check.py TestSecurityChecks.test_MACHO - $(AM_V_at) CC='$(CC)' $(PYTHON) $(top_srcdir)/contrib/devtools/test-symbol-check.py TestSymbolChecks.test_MACHO + $(AM_V_at) CC='$(CC)' CFLAGS='$(CFLAGS)' CPPFLAGS='$(CPPFLAGS)' LDFLAGS='$(LDFLAGS)' $(PYTHON) $(top_srcdir)/contrib/devtools/test-security-check.py TestSecurityChecks.test_MACHO + $(AM_V_at) CC='$(CC)' CFLAGS='$(CFLAGS)' CPPFLAGS='$(CPPFLAGS)' LDFLAGS='$(LDFLAGS)' $(PYTHON) $(top_srcdir)/contrib/devtools/test-symbol-check.py TestSymbolChecks.test_MACHO endif if TARGET_WINDOWS - $(AM_V_at) CC='$(CC)' $(PYTHON) $(top_srcdir)/contrib/devtools/test-security-check.py TestSecurityChecks.test_PE - $(AM_V_at) CC='$(CC)' $(PYTHON) $(top_srcdir)/contrib/devtools/test-symbol-check.py TestSymbolChecks.test_PE + $(AM_V_at) CC='$(CC)' CFLAGS='$(CFLAGS)' CPPFLAGS='$(CPPFLAGS)' LDFLAGS='$(LDFLAGS)' $(PYTHON) $(top_srcdir)/contrib/devtools/test-security-check.py TestSecurityChecks.test_PE + $(AM_V_at) CC='$(CC)' CFLAGS='$(CFLAGS)' CPPFLAGS='$(CPPFLAGS)' LDFLAGS='$(LDFLAGS)' $(PYTHON) $(top_srcdir)/contrib/devtools/test-symbol-check.py TestSymbolChecks.test_PE endif if TARGET_LINUX - $(AM_V_at) CC='$(CC)' $(PYTHON) $(top_srcdir)/contrib/devtools/test-security-check.py TestSecurityChecks.test_ELF - $(AM_V_at) CC='$(CC)' CPPFILT='$(CPPFILT)' $(PYTHON) $(top_srcdir)/contrib/devtools/test-symbol-check.py TestSymbolChecks.test_ELF + $(AM_V_at) CC='$(CC)' CFLAGS='$(CFLAGS)' CPPFLAGS='$(CPPFLAGS)' LDFLAGS='$(LDFLAGS)' $(PYTHON) $(top_srcdir)/contrib/devtools/test-security-check.py TestSecurityChecks.test_ELF + $(AM_V_at) CC='$(CC)' CFLAGS='$(CFLAGS)' CPPFLAGS='$(CPPFLAGS)' LDFLAGS='$(LDFLAGS)' $(PYTHON) $(top_srcdir)/contrib/devtools/test-symbol-check.py TestSymbolChecks.test_ELF endif diff --git a/REVIEWERS b/REVIEWERS index e048036e42..a4b05f86c0 100644 --- a/REVIEWERS +++ b/REVIEWERS @@ -18,12 +18,11 @@ # Maintainers +# @achow101 # @fanquake # @hebasto -# @jonasschnelli # @laanwj # @marcofalke -# @meshcollider # @sipa # Docs @@ -41,7 +40,6 @@ /doc/dependencies.md @fanquake /doc/developer-notes.md @laanwj /doc/files.md @hebasto -/doc/gitian-building.md @laanwj /doc/reduce-memory.md @fanquake /doc/reduce-traffic.md @jonasschnelli /doc/release-process.md @laanwj @@ -77,8 +75,7 @@ /contrib/devtools/test-security-check.py @fanquake /contrib/devtools/symbol-check.py @fanquake -# Gitian/Guix -/contrib/gitian-build.py @hebasto +# Guix /contrib/guix/ @dongcarl # Compatibility @@ -136,3 +133,8 @@ /src/script/interpreter.* @sipa /src/validation.* @sipa /src/consensus/ @sipa + +# Tracing +/doc/tracing.md @jb55 @0xB10C +/src/util/trace.h @jb55 @0xB10C +/contrib/tracing/ @jb55 @0xB10C diff --git a/SECURITY.md b/SECURITY.md index d25a930f11..8403daf58c 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -305,7 +305,7 @@ The following keys may be used to communicate sensitive information to developer | Name | Fingerprint | |------|-------------| | Wladimir van der Laan | 71A3 B167 3540 5025 D447 E8F2 7481 0B01 2346 C9A6 | -| Jonas Schnelli | 32EE 5C4C 3FA1 5CCA DB46 ABE5 29D4 BCB6 416F 53EC | | Pieter Wuille | 133E AC17 9436 F14A 5CF1 B794 860F EB80 4E66 9320 | +| Michael Ford | E777 299F C265 DD04 7930 70EB 944D 35F9 AC3D B76A | -You can import a key by running the following command with that individual’s fingerprint: `gpg --recv-keys ""` Ensure that you put quotes around fingerprints containing spaces. +You can import a key by running the following command with that individual’s fingerprint: `gpg --keyserver hkps://keys.openpgp.org --recv-keys ""` Ensure that you put quotes around fingerprints containing spaces. diff --git a/build-aux/m4/ax_boost_filesystem.m4 b/build-aux/m4/ax_boost_filesystem.m4 deleted file mode 100644 index 12f7bc5e2e..0000000000 --- a/build-aux/m4/ax_boost_filesystem.m4 +++ /dev/null @@ -1,118 +0,0 @@ -# =========================================================================== -# https://www.gnu.org/software/autoconf-archive/ax_boost_filesystem.html -# =========================================================================== -# -# SYNOPSIS -# -# AX_BOOST_FILESYSTEM -# -# DESCRIPTION -# -# Test for Filesystem library from the Boost C++ libraries. The macro -# requires a preceding call to AX_BOOST_BASE. Further documentation is -# available at . -# -# This macro calls: -# -# AC_SUBST(BOOST_FILESYSTEM_LIB) -# -# And sets: -# -# HAVE_BOOST_FILESYSTEM -# -# LICENSE -# -# Copyright (c) 2009 Thomas Porschberg -# Copyright (c) 2009 Michael Tindal -# Copyright (c) 2009 Roman Rybalko -# -# Copying and distribution of this file, with or without modification, are -# permitted in any medium without royalty provided the copyright notice -# and this notice are preserved. This file is offered as-is, without any -# warranty. - -#serial 28 - -AC_DEFUN([AX_BOOST_FILESYSTEM], -[ - AC_ARG_WITH([boost-filesystem], - AS_HELP_STRING([--with-boost-filesystem@<:@=special-lib@:>@], - [use the Filesystem library from boost - it is possible to specify a certain library for the linker - e.g. --with-boost-filesystem=boost_filesystem-gcc-mt ]), - [ - if test "$withval" = "no"; then - want_boost="no" - elif test "$withval" = "yes"; then - want_boost="yes" - ax_boost_user_filesystem_lib="" - else - want_boost="yes" - ax_boost_user_filesystem_lib="$withval" - fi - ], - [want_boost="yes"] - ) - - if test "x$want_boost" = "xyes"; then - AC_REQUIRE([AC_PROG_CC]) - CPPFLAGS_SAVED="$CPPFLAGS" - CPPFLAGS="$CPPFLAGS $BOOST_CPPFLAGS" - export CPPFLAGS - - LDFLAGS_SAVED="$LDFLAGS" - LDFLAGS="$LDFLAGS $BOOST_LDFLAGS" - export LDFLAGS - - LIBS_SAVED=$LIBS - LIBS="$LIBS $BOOST_SYSTEM_LIB" - export LIBS - - AC_CACHE_CHECK(whether the Boost::Filesystem library is available, - ax_cv_boost_filesystem, - [AC_LANG_PUSH([C++]) - AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[@%:@include ]], - [[using namespace boost::filesystem; - path my_path( "foo/bar/data.txt" ); - return 0;]])], - ax_cv_boost_filesystem=yes, ax_cv_boost_filesystem=no) - AC_LANG_POP([C++]) - ]) - if test "x$ax_cv_boost_filesystem" = "xyes"; then - AC_DEFINE(HAVE_BOOST_FILESYSTEM,,[define if the Boost::Filesystem library is available]) - BOOSTLIBDIR=`echo $BOOST_LDFLAGS | sed -e 's/@<:@^\/@:>@*//'` - if test "x$ax_boost_user_filesystem_lib" = "x"; then - for libextension in `ls -r $BOOSTLIBDIR/libboost_filesystem* 2>/dev/null | sed 's,.*/lib,,' | sed 's,\..*,,'` ; do - ax_lib=${libextension} - AC_CHECK_LIB($ax_lib, exit, - [BOOST_FILESYSTEM_LIB="-l$ax_lib"; AC_SUBST(BOOST_FILESYSTEM_LIB) link_filesystem="yes"; break], - [link_filesystem="no"]) - done - if test "x$link_filesystem" != "xyes"; then - for libextension in `ls -r $BOOSTLIBDIR/boost_filesystem* 2>/dev/null | sed 's,.*/,,' | sed -e 's,\..*,,'` ; do - ax_lib=${libextension} - AC_CHECK_LIB($ax_lib, exit, - [BOOST_FILESYSTEM_LIB="-l$ax_lib"; AC_SUBST(BOOST_FILESYSTEM_LIB) link_filesystem="yes"; break], - [link_filesystem="no"]) - done - fi - else - for ax_lib in $ax_boost_user_filesystem_lib boost_filesystem-$ax_boost_user_filesystem_lib; do - AC_CHECK_LIB($ax_lib, exit, - [BOOST_FILESYSTEM_LIB="-l$ax_lib"; AC_SUBST(BOOST_FILESYSTEM_LIB) link_filesystem="yes"; break], - [link_filesystem="no"]) - done - - fi - if test "x$ax_lib" = "x"; then - AC_MSG_ERROR(Could not find a version of the Boost::Filesystem library!) - fi - if test "x$link_filesystem" != "xyes"; then - AC_MSG_ERROR(Could not link against $ax_lib !) - fi - fi - - CPPFLAGS="$CPPFLAGS_SAVED" - LDFLAGS="$LDFLAGS_SAVED" - LIBS="$LIBS_SAVED" - fi -]) diff --git a/build-aux/m4/ax_boost_system.m4 b/build-aux/m4/ax_boost_system.m4 deleted file mode 100644 index 323e2a676a..0000000000 --- a/build-aux/m4/ax_boost_system.m4 +++ /dev/null @@ -1,121 +0,0 @@ -# =========================================================================== -# https://www.gnu.org/software/autoconf-archive/ax_boost_system.html -# =========================================================================== -# -# SYNOPSIS -# -# AX_BOOST_SYSTEM -# -# DESCRIPTION -# -# Test for System library from the Boost C++ libraries. The macro requires -# a preceding call to AX_BOOST_BASE. Further documentation is available at -# . -# -# This macro calls: -# -# AC_SUBST(BOOST_SYSTEM_LIB) -# -# And sets: -# -# HAVE_BOOST_SYSTEM -# -# LICENSE -# -# Copyright (c) 2008 Thomas Porschberg -# Copyright (c) 2008 Michael Tindal -# Copyright (c) 2008 Daniel Casimiro -# -# Copying and distribution of this file, with or without modification, are -# permitted in any medium without royalty provided the copyright notice -# and this notice are preserved. This file is offered as-is, without any -# warranty. - -#serial 20 - -AC_DEFUN([AX_BOOST_SYSTEM], -[ - AC_ARG_WITH([boost-system], - AS_HELP_STRING([--with-boost-system@<:@=special-lib@:>@], - [use the System library from boost - it is possible to specify a certain library for the linker - e.g. --with-boost-system=boost_system-gcc-mt ]), - [ - if test "$withval" = "no"; then - want_boost="no" - elif test "$withval" = "yes"; then - want_boost="yes" - ax_boost_user_system_lib="" - else - want_boost="yes" - ax_boost_user_system_lib="$withval" - fi - ], - [want_boost="yes"] - ) - - if test "x$want_boost" = "xyes"; then - AC_REQUIRE([AC_PROG_CC]) - AC_REQUIRE([AC_CANONICAL_BUILD]) - CPPFLAGS_SAVED="$CPPFLAGS" - CPPFLAGS="$CPPFLAGS $BOOST_CPPFLAGS" - export CPPFLAGS - - LDFLAGS_SAVED="$LDFLAGS" - LDFLAGS="$LDFLAGS $BOOST_LDFLAGS" - export LDFLAGS - - AC_CACHE_CHECK(whether the Boost::System library is available, - ax_cv_boost_system, - [AC_LANG_PUSH([C++]) - CXXFLAGS_SAVE=$CXXFLAGS - CXXFLAGS= - - AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[@%:@include ]], - [[boost::system::error_category *a = 0;]])], - ax_cv_boost_system=yes, ax_cv_boost_system=no) - CXXFLAGS=$CXXFLAGS_SAVE - AC_LANG_POP([C++]) - ]) - if test "x$ax_cv_boost_system" = "xyes"; then - AC_SUBST(BOOST_CPPFLAGS) - - AC_DEFINE(HAVE_BOOST_SYSTEM,,[define if the Boost::System library is available]) - BOOSTLIBDIR=`echo $BOOST_LDFLAGS | sed -e 's/@<:@^\/@:>@*//'` - - LDFLAGS_SAVE=$LDFLAGS - if test "x$ax_boost_user_system_lib" = "x"; then - for libextension in `ls -r $BOOSTLIBDIR/libboost_system* 2>/dev/null | sed 's,.*/lib,,' | sed 's,\..*,,'` ; do - ax_lib=${libextension} - AC_CHECK_LIB($ax_lib, exit, - [BOOST_SYSTEM_LIB="-l$ax_lib"; AC_SUBST(BOOST_SYSTEM_LIB) link_system="yes"; break], - [link_system="no"]) - done - if test "x$link_system" != "xyes"; then - for libextension in `ls -r $BOOSTLIBDIR/boost_system* 2>/dev/null | sed 's,.*/,,' | sed -e 's,\..*,,'` ; do - ax_lib=${libextension} - AC_CHECK_LIB($ax_lib, exit, - [BOOST_SYSTEM_LIB="-l$ax_lib"; AC_SUBST(BOOST_SYSTEM_LIB) link_system="yes"; break], - [link_system="no"]) - done - fi - - else - for ax_lib in $ax_boost_user_system_lib boost_system-$ax_boost_user_system_lib; do - AC_CHECK_LIB($ax_lib, exit, - [BOOST_SYSTEM_LIB="-l$ax_lib"; AC_SUBST(BOOST_SYSTEM_LIB) link_system="yes"; break], - [link_system="no"]) - done - - fi - if test "x$ax_lib" = "x"; then - AC_MSG_ERROR(Could not find a version of the Boost::System library!) - fi - if test "x$link_system" = "xno"; then - AC_MSG_ERROR(Could not link against $ax_lib !) - fi - fi - - CPPFLAGS="$CPPFLAGS_SAVED" - LDFLAGS="$LDFLAGS_SAVED" - fi -]) diff --git a/build-aux/m4/ax_boost_unit_test_framework.m4 b/build-aux/m4/ax_boost_unit_test_framework.m4 deleted file mode 100644 index 4cca32fcfd..0000000000 --- a/build-aux/m4/ax_boost_unit_test_framework.m4 +++ /dev/null @@ -1,137 +0,0 @@ -# ================================================================================= -# https://www.gnu.org/software/autoconf-archive/ax_boost_unit_test_framework.html -# ================================================================================= -# -# SYNOPSIS -# -# AX_BOOST_UNIT_TEST_FRAMEWORK -# -# DESCRIPTION -# -# Test for Unit_Test_Framework library from the Boost C++ libraries. The -# macro requires a preceding call to AX_BOOST_BASE. Further documentation -# is available at . -# -# This macro calls: -# -# AC_SUBST(BOOST_UNIT_TEST_FRAMEWORK_LIB) -# -# And sets: -# -# HAVE_BOOST_UNIT_TEST_FRAMEWORK -# -# LICENSE -# -# Copyright (c) 2008 Thomas Porschberg -# -# Copying and distribution of this file, with or without modification, are -# permitted in any medium without royalty provided the copyright notice -# and this notice are preserved. This file is offered as-is, without any -# warranty. - -#serial 22 - -AC_DEFUN([AX_BOOST_UNIT_TEST_FRAMEWORK], -[ - AC_ARG_WITH([boost-unit-test-framework], - AS_HELP_STRING([--with-boost-unit-test-framework@<:@=special-lib@:>@], - [use the Unit_Test_Framework library from boost - it is possible to specify a certain library for the linker - e.g. --with-boost-unit-test-framework=boost_unit_test_framework-gcc ]), - [ - if test "$withval" = "no"; then - want_boost="no" - elif test "$withval" = "yes"; then - want_boost="yes" - ax_boost_user_unit_test_framework_lib="" - else - want_boost="yes" - ax_boost_user_unit_test_framework_lib="$withval" - fi - ], - [want_boost="yes"] - ) - - if test "x$want_boost" = "xyes"; then - AC_REQUIRE([AC_PROG_CC]) - CPPFLAGS_SAVED="$CPPFLAGS" - CPPFLAGS="$CPPFLAGS $BOOST_CPPFLAGS" - export CPPFLAGS - - LDFLAGS_SAVED="$LDFLAGS" - LDFLAGS="$LDFLAGS $BOOST_LDFLAGS" - export LDFLAGS - - AC_CACHE_CHECK(whether the Boost::Unit_Test_Framework library is available, - ax_cv_boost_unit_test_framework, - [AC_LANG_PUSH([C++]) - AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[@%:@include ]], - [[using boost::unit_test::test_suite; - test_suite* test= BOOST_TEST_SUITE( "Unit test example 1" ); if (test == NULL) { return 1; } else { return 0; }]])], - ax_cv_boost_unit_test_framework=yes, ax_cv_boost_unit_test_framework=no) - AC_LANG_POP([C++]) - ]) - if test "x$ax_cv_boost_unit_test_framework" = "xyes"; then - AC_DEFINE(HAVE_BOOST_UNIT_TEST_FRAMEWORK,,[define if the Boost::Unit_Test_Framework library is available]) - BOOSTLIBDIR=`echo $BOOST_LDFLAGS | sed -e 's/@<:@^\/@:>@*//'` - - if test "x$ax_boost_user_unit_test_framework_lib" = "x"; then - saved_ldflags="${LDFLAGS}" - for monitor_library in `ls $BOOSTLIBDIR/libboost_unit_test_framework*.so* $BOOSTLIBDIR/libboost_unit_test_framework*.dylib* $BOOSTLIBDIR/libboost_unit_test_framework*.a* 2>/dev/null` ; do - if test -r $monitor_library ; then - libextension=`echo $monitor_library | sed 's,.*/,,' | sed -e 's;^lib\(boost_unit_test_framework.*\)\.so.*$;\1;' -e 's;^lib\(boost_unit_test_framework.*\)\.dylib.*$;\1;' -e 's;^lib\(boost_unit_test_framework.*\)\.a.*$;\1;'` - ax_lib=${libextension} - link_unit_test_framework="yes" - else - link_unit_test_framework="no" - fi - - if test "x$link_unit_test_framework" = "xyes"; then - BOOST_UNIT_TEST_FRAMEWORK_LIB="-l$ax_lib" - AC_SUBST(BOOST_UNIT_TEST_FRAMEWORK_LIB) - break - fi - done - if test "x$link_unit_test_framework" != "xyes"; then - for libextension in `ls $BOOSTLIBDIR/boost_unit_test_framework*.dll* $BOOSTLIBDIR/boost_unit_test_framework*.a* 2>/dev/null | sed 's,.*/,,' | sed -e 's;^\(boost_unit_test_framework.*\)\.dll.*$;\1;' -e 's;^\(boost_unit_test_framework.*\)\.a.*$;\1;'` ; do - ax_lib=${libextension} - AC_CHECK_LIB($ax_lib, exit, - [BOOST_UNIT_TEST_FRAMEWORK_LIB="-l$ax_lib"; AC_SUBST(BOOST_UNIT_TEST_FRAMEWORK_LIB) link_unit_test_framework="yes"; break], - [link_unit_test_framework="no"]) - done - fi - else - link_unit_test_framework="no" - saved_ldflags="${LDFLAGS}" - for ax_lib in boost_unit_test_framework-$ax_boost_user_unit_test_framework_lib $ax_boost_user_unit_test_framework_lib ; do - if test "x$link_unit_test_framework" = "xyes"; then - break; - fi - for unittest_library in `ls $BOOSTLIBDIR/lib${ax_lib}.so* $BOOSTLIBDIR/lib${ax_lib}.a* 2>/dev/null` ; do - if test -r $unittest_library ; then - libextension=`echo $unittest_library | sed 's,.*/,,' | sed -e 's;^lib\(boost_unit_test_framework.*\)\.so.*$;\1;' -e 's;^lib\(boost_unit_test_framework.*\)\.a*$;\1;'` - ax_lib=${libextension} - link_unit_test_framework="yes" - else - link_unit_test_framework="no" - fi - - if test "x$link_unit_test_framework" = "xyes"; then - BOOST_UNIT_TEST_FRAMEWORK_LIB="-l$ax_lib" - AC_SUBST(BOOST_UNIT_TEST_FRAMEWORK_LIB) - break - fi - done - done - fi - if test "x$ax_lib" = "x"; then - AC_MSG_ERROR(Could not find a version of the Boost::Unit_Test_Framework library!) - fi - if test "x$link_unit_test_framework" != "xyes"; then - AC_MSG_ERROR(Could not link against $ax_lib !) - fi - fi - - CPPFLAGS="$CPPFLAGS_SAVED" - LDFLAGS="$LDFLAGS_SAVED" - fi -]) diff --git a/build-aux/m4/ax_check_compile_flag.m4 b/build-aux/m4/ax_check_compile_flag.m4 index ca3639715e..bd753b34d7 100644 --- a/build-aux/m4/ax_check_compile_flag.m4 +++ b/build-aux/m4/ax_check_compile_flag.m4 @@ -1,5 +1,5 @@ # =========================================================================== -# http://www.gnu.org/software/autoconf-archive/ax_check_compile_flag.html +# https://www.gnu.org/software/autoconf-archive/ax_check_compile_flag.html # =========================================================================== # # SYNOPSIS @@ -29,33 +29,12 @@ # Copyright (c) 2008 Guido U. Draheim # Copyright (c) 2011 Maarten Bosmans # -# This program is free software: you can redistribute it and/or modify it -# under the terms of the GNU General Public License as published by the -# Free Software Foundation, either version 3 of the License, or (at your -# option) any later version. -# -# This program is distributed in the hope that it will be useful, but -# WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General -# Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program. If not, see . -# -# As a special exception, the respective Autoconf Macro's copyright owner -# gives unlimited permission to copy, distribute and modify the configure -# scripts that are the output of Autoconf when processing the Macro. You -# need not follow the terms of the GNU General Public License when using -# or distributing such scripts, even though portions of the text of the -# Macro appear in them. The GNU General Public License (GPL) does govern -# all other use of the material that constitutes the Autoconf Macro. -# -# This special exception to the GPL applies to versions of the Autoconf -# Macro released by the Autoconf Archive. When you make and distribute a -# modified version of the Autoconf Macro, you may extend this special -# exception to the GPL to apply to your modified version as well. +# Copying and distribution of this file, with or without modification, are +# permitted in any medium without royalty provided the copyright notice +# and this notice are preserved. This file is offered as-is, without any +# warranty. -#serial 4 +#serial 6 AC_DEFUN([AX_CHECK_COMPILE_FLAG], [AC_PREREQ(2.64)dnl for _AC_LANG_PREFIX and AS_VAR_IF diff --git a/build-aux/m4/ax_check_link_flag.m4 b/build-aux/m4/ax_check_link_flag.m4 index eb01a6ce13..03a30ce4c7 100644 --- a/build-aux/m4/ax_check_link_flag.m4 +++ b/build-aux/m4/ax_check_link_flag.m4 @@ -1,5 +1,5 @@ # =========================================================================== -# http://www.gnu.org/software/autoconf-archive/ax_check_link_flag.html +# https://www.gnu.org/software/autoconf-archive/ax_check_link_flag.html # =========================================================================== # # SYNOPSIS @@ -29,33 +29,12 @@ # Copyright (c) 2008 Guido U. Draheim # Copyright (c) 2011 Maarten Bosmans # -# This program is free software: you can redistribute it and/or modify it -# under the terms of the GNU General Public License as published by the -# Free Software Foundation, either version 3 of the License, or (at your -# option) any later version. -# -# This program is distributed in the hope that it will be useful, but -# WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General -# Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program. If not, see . -# -# As a special exception, the respective Autoconf Macro's copyright owner -# gives unlimited permission to copy, distribute and modify the configure -# scripts that are the output of Autoconf when processing the Macro. You -# need not follow the terms of the GNU General Public License when using -# or distributing such scripts, even though portions of the text of the -# Macro appear in them. The GNU General Public License (GPL) does govern -# all other use of the material that constitutes the Autoconf Macro. -# -# This special exception to the GPL applies to versions of the Autoconf -# Macro released by the Autoconf Archive. When you make and distribute a -# modified version of the Autoconf Macro, you may extend this special -# exception to the GPL to apply to your modified version as well. +# Copying and distribution of this file, with or without modification, are +# permitted in any medium without royalty provided the copyright notice +# and this notice are preserved. This file is offered as-is, without any +# warranty. -#serial 4 +#serial 6 AC_DEFUN([AX_CHECK_LINK_FLAG], [AC_PREREQ(2.64)dnl for _AC_LANG_PREFIX and AS_VAR_IF diff --git a/build-aux/m4/ax_check_preproc_flag.m4 b/build-aux/m4/ax_check_preproc_flag.m4 index ca1d5ee2b6..e43560fbd3 100644 --- a/build-aux/m4/ax_check_preproc_flag.m4 +++ b/build-aux/m4/ax_check_preproc_flag.m4 @@ -1,5 +1,5 @@ # =========================================================================== -# http://www.gnu.org/software/autoconf-archive/ax_check_preproc_flag.html +# https://www.gnu.org/software/autoconf-archive/ax_check_preproc_flag.html # =========================================================================== # # SYNOPSIS @@ -29,33 +29,12 @@ # Copyright (c) 2008 Guido U. Draheim # Copyright (c) 2011 Maarten Bosmans # -# This program is free software: you can redistribute it and/or modify it -# under the terms of the GNU General Public License as published by the -# Free Software Foundation, either version 3 of the License, or (at your -# option) any later version. -# -# This program is distributed in the hope that it will be useful, but -# WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General -# Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program. If not, see . -# -# As a special exception, the respective Autoconf Macro's copyright owner -# gives unlimited permission to copy, distribute and modify the configure -# scripts that are the output of Autoconf when processing the Macro. You -# need not follow the terms of the GNU General Public License when using -# or distributing such scripts, even though portions of the text of the -# Macro appear in them. The GNU General Public License (GPL) does govern -# all other use of the material that constitutes the Autoconf Macro. -# -# This special exception to the GPL applies to versions of the Autoconf -# Macro released by the Autoconf Archive. When you make and distribute a -# modified version of the Autoconf Macro, you may extend this special -# exception to the GPL to apply to your modified version as well. +# Copying and distribution of this file, with or without modification, are +# permitted in any medium without royalty provided the copyright notice +# and this notice are preserved. This file is offered as-is, without any +# warranty. -#serial 4 +#serial 6 AC_DEFUN([AX_CHECK_PREPROC_FLAG], [AC_PREREQ(2.64)dnl for _AC_LANG_PREFIX and AS_VAR_IF diff --git a/build-aux/m4/ax_pthread.m4 b/build-aux/m4/ax_pthread.m4 index 1598d077ff..9f35d13914 100644 --- a/build-aux/m4/ax_pthread.m4 +++ b/build-aux/m4/ax_pthread.m4 @@ -14,20 +14,24 @@ # flags that are needed. (The user can also force certain compiler # flags/libs to be tested by setting these environment variables.) # -# Also sets PTHREAD_CC to any special C compiler that is needed for -# multi-threaded programs (defaults to the value of CC otherwise). (This -# is necessary on AIX to use the special cc_r compiler alias.) +# Also sets PTHREAD_CC and PTHREAD_CXX to any special C compiler that is +# needed for multi-threaded programs (defaults to the value of CC +# respectively CXX otherwise). (This is necessary on e.g. AIX to use the +# special cc_r/CC_r compiler alias.) # # NOTE: You are assumed to not only compile your program with these flags, # but also to link with them as well. For example, you might link with # $PTHREAD_CC $CFLAGS $PTHREAD_CFLAGS $LDFLAGS ... $PTHREAD_LIBS $LIBS +# $PTHREAD_CXX $CXXFLAGS $PTHREAD_CFLAGS $LDFLAGS ... $PTHREAD_LIBS $LIBS # # If you are only building threaded programs, you may wish to use these # variables in your default LIBS, CFLAGS, and CC: # # LIBS="$PTHREAD_LIBS $LIBS" # CFLAGS="$CFLAGS $PTHREAD_CFLAGS" +# CXXFLAGS="$CXXFLAGS $PTHREAD_CFLAGS" # CC="$PTHREAD_CC" +# CXX="$PTHREAD_CXX" # # In addition, if the PTHREAD_CREATE_JOINABLE thread-attribute constant # has a nonstandard name, this macro defines PTHREAD_CREATE_JOINABLE to @@ -83,7 +87,7 @@ # modified version of the Autoconf Macro, you may extend this special # exception to the GPL to apply to your modified version as well. -#serial 27 +#serial 31 AU_ALIAS([ACX_PTHREAD], [AX_PTHREAD]) AC_DEFUN([AX_PTHREAD], [ @@ -105,6 +109,7 @@ if test "x$PTHREAD_CFLAGS$PTHREAD_LIBS" != "x"; then ax_pthread_save_CFLAGS="$CFLAGS" ax_pthread_save_LIBS="$LIBS" AS_IF([test "x$PTHREAD_CC" != "x"], [CC="$PTHREAD_CC"]) + AS_IF([test "x$PTHREAD_CXX" != "x"], [CXX="$PTHREAD_CXX"]) CFLAGS="$CFLAGS $PTHREAD_CFLAGS" LIBS="$PTHREAD_LIBS $LIBS" AC_MSG_CHECKING([for pthread_join using $CC $PTHREAD_CFLAGS $PTHREAD_LIBS]) @@ -386,7 +391,7 @@ if test "x$ax_pthread_clang" = "xyes"; then # step ax_pthread_save_ac_link="$ac_link" ax_pthread_sed='s/conftest\.\$ac_ext/conftest.$ac_objext/g' - ax_pthread_link_step=`$as_echo "$ac_link" | sed "$ax_pthread_sed"` + ax_pthread_link_step=`AS_ECHO(["$ac_link"]) | sed "$ax_pthread_sed"` ax_pthread_2step_ac_link="($ac_compile) && (echo ==== >&5) && ($ax_pthread_link_step)" ax_pthread_save_CFLAGS="$CFLAGS" for ax_pthread_try in '' -Qunused-arguments -Wno-unused-command-line-argument unknown; do @@ -482,18 +487,28 @@ if test "x$ax_pthread_ok" = "xyes"; then [#handle absolute path differently from PATH based program lookup AS_CASE(["x$CC"], [x/*], - [AS_IF([AS_EXECUTABLE_P([${CC}_r])],[PTHREAD_CC="${CC}_r"])], - [AC_CHECK_PROGS([PTHREAD_CC],[${CC}_r],[$CC])])]) + [ + AS_IF([AS_EXECUTABLE_P([${CC}_r])],[PTHREAD_CC="${CC}_r"]) + AS_IF([test "x${CXX}" != "x"], [AS_IF([AS_EXECUTABLE_P([${CXX}_r])],[PTHREAD_CXX="${CXX}_r"])]) + ], + [ + AC_CHECK_PROGS([PTHREAD_CC],[${CC}_r],[$CC]) + AS_IF([test "x${CXX}" != "x"], [AC_CHECK_PROGS([PTHREAD_CXX],[${CXX}_r],[$CXX])]) + ] + ) + ]) ;; esac fi fi test -n "$PTHREAD_CC" || PTHREAD_CC="$CC" +test -n "$PTHREAD_CXX" || PTHREAD_CXX="$CXX" AC_SUBST([PTHREAD_LIBS]) AC_SUBST([PTHREAD_CFLAGS]) AC_SUBST([PTHREAD_CC]) +AC_SUBST([PTHREAD_CXX]) # Finally, execute ACTION-IF-FOUND/ACTION-IF-NOT-FOUND: if test "x$ax_pthread_ok" = "xyes"; then diff --git a/build-aux/m4/bitcoin_find_bdb48.m4 b/build-aux/m4/bitcoin_find_bdb48.m4 index 5fc5b493d3..3ef7fab5b5 100644 --- a/build-aux/m4/bitcoin_find_bdb48.m4 +++ b/build-aux/m4/bitcoin_find_bdb48.m4 @@ -3,12 +3,12 @@ dnl Distributed under the MIT software license, see the accompanying dnl file COPYING or http://www.opensource.org/licenses/mit-license.php. AC_DEFUN([BITCOIN_FIND_BDB48],[ - AC_ARG_VAR(BDB_CFLAGS, [C compiler flags for BerkeleyDB, bypasses autodetection]) - AC_ARG_VAR(BDB_LIBS, [Linker flags for BerkeleyDB, bypasses autodetection]) + AC_ARG_VAR([BDB_CFLAGS], [C compiler flags for BerkeleyDB, bypasses autodetection]) + AC_ARG_VAR([BDB_LIBS], [Linker flags for BerkeleyDB, bypasses autodetection]) - if test "x$use_bdb" = "xno"; then + if test "$use_bdb" = "no"; then use_bdb=no - elif test "x$BDB_CFLAGS" = "x"; then + elif test "$BDB_CFLAGS" = ""; then AC_MSG_CHECKING([for Berkeley DB C++ headers]) BDB_CPPFLAGS= bdbpath=X @@ -28,7 +28,7 @@ AC_DEFUN([BITCOIN_FIND_BDB48],[ #error "failed to find bdb 4.8+" #endif ]])],[ - if test "x$bdbpath" = "xX"; then + if test "$bdbpath" = "X"; then bdbpath="${searchpath}" fi ],[ @@ -45,18 +45,25 @@ AC_DEFUN([BITCOIN_FIND_BDB48],[ break ],[]) done - if test "x$bdbpath" = "xX"; then + if test "$bdbpath" = "X"; then use_bdb=no AC_MSG_RESULT([no]) - AC_MSG_ERROR([libdb_cxx headers missing, ]AC_PACKAGE_NAME[ requires this library for BDB wallet support (--without-bdb to disable BDB wallet support)]) - elif test "x$bdb48path" = "xX"; then + AC_MSG_WARN([libdb_cxx headers missing]) + AC_MSG_WARN(AC_PACKAGE_NAME[ requires this library for BDB (legacy) wallet support]) + AC_MSG_WARN([Passing --without-bdb will suppress this warning]) + elif test "$bdb48path" = "X"; then BITCOIN_SUBDIR_TO_INCLUDE(BDB_CPPFLAGS,[${bdbpath}],db_cxx) AC_ARG_WITH([incompatible-bdb],[AS_HELP_STRING([--with-incompatible-bdb], [allow using a bdb version other than 4.8])],[ - AC_MSG_WARN([Found Berkeley DB other than 4.8; BDB wallets opened by this build will not be portable!]) + AC_MSG_WARN([Found Berkeley DB other than 4.8]) + AC_MSG_WARN([BDB (legacy) wallets opened by this build will not be portable!]) + use_bdb=yes ],[ - AC_MSG_ERROR([Found Berkeley DB other than 4.8, required for portable BDB wallets (--with-incompatible-bdb to ignore or --without-bdb to disable BDB wallet support)]) + AC_MSG_WARN([Found Berkeley DB other than 4.8]) + AC_MSG_WARN([BDB (legacy) wallets opened by this build would not be portable!]) + AC_MSG_WARN([If this is intended, pass --with-incompatible-bdb]) + AC_MSG_WARN([Passing --without-bdb will suppress this warning]) + use_bdb=no ]) - use_bdb=yes else BITCOIN_SUBDIR_TO_INCLUDE(BDB_CPPFLAGS,[${bdb48path}],db_cxx) bdbpath="${bdb48path}" @@ -67,9 +74,9 @@ AC_DEFUN([BITCOIN_FIND_BDB48],[ fi AC_SUBST(BDB_CPPFLAGS) - if test "x$use_bdb" = "xno"; then + if test "$use_bdb" = "no"; then use_bdb=no - elif test "x$BDB_LIBS" = "x"; then + elif test "$BDB_LIBS" = ""; then # TODO: Ideally this could find the library version and make sure it matches the headers being used for searchlib in db_cxx-4.8 db_cxx db4_cxx; do AC_CHECK_LIB([$searchlib],[main],[ @@ -77,12 +84,13 @@ AC_DEFUN([BITCOIN_FIND_BDB48],[ break ]) done - if test "x$BDB_LIBS" = "x"; then - AC_MSG_ERROR([libdb_cxx missing, ]AC_PACKAGE_NAME[ requires this library for BDB wallet support (--without-bdb to disable BDB wallet support)]) + if test "$BDB_LIBS" = ""; then + AC_MSG_WARN([libdb_cxx headers missing]) + AC_MSG_WARN(AC_PACKAGE_NAME[ requires this library for BDB (legacy) wallet support]) + AC_MSG_WARN([Passing --without-bdb will suppress this warning]) fi fi - if test "x$use_bdb" != "xno"; then - AC_SUBST(BDB_LIBS) + if test "$use_bdb" != "no"; then AC_DEFINE([USE_BDB], [1], [Define if BDB support should be compiled in]) use_bdb=yes fi diff --git a/build-aux/m4/bitcoin_qt.m4 b/build-aux/m4/bitcoin_qt.m4 index 5b5a8ed16e..1454e33f6e 100644 --- a/build-aux/m4/bitcoin_qt.m4 +++ b/build-aux/m4/bitcoin_qt.m4 @@ -5,8 +5,8 @@ dnl file COPYING or http://www.opensource.org/licenses/mit-license.php. dnl Helper for cases where a qt dependency is not met. dnl Output: If qt version is auto, set bitcoin_enable_qt to false. Else, exit. AC_DEFUN([BITCOIN_QT_FAIL],[ - if test "x$bitcoin_qt_want_version" = xauto && test "x$bitcoin_qt_force" != xyes; then - if test "x$bitcoin_enable_qt" != xno; then + if test "$bitcoin_qt_want_version" = "auto" && test "$bitcoin_qt_force" != "yes"; then + if test "$bitcoin_enable_qt" != "no"; then AC_MSG_WARN([$1; bitcoin-qt frontend will not be built]) fi bitcoin_enable_qt=no @@ -17,7 +17,7 @@ AC_DEFUN([BITCOIN_QT_FAIL],[ ]) AC_DEFUN([BITCOIN_QT_CHECK],[ - if test "x$bitcoin_enable_qt" != xno && test "x$bitcoin_qt_want_version" != xno; then + if test "$bitcoin_enable_qt" != "no" && test "$bitcoin_qt_want_version" != "no"; then true $1 else @@ -35,12 +35,12 @@ dnl Inputs: $4: If "yes", don't fail if $2 is not found. dnl Output: $1 is set to the path of $2 if found. $2 are searched in order. AC_DEFUN([BITCOIN_QT_PATH_PROGS],[ BITCOIN_QT_CHECK([ - if test "x$3" != x; then - AC_PATH_PROGS($1,$2,,$3) + if test "$3" != ""; then + AC_PATH_PROGS([$1], [$2], [], [$3]) else - AC_PATH_PROGS($1,$2) + AC_PATH_PROGS([$1], [$2]) fi - if test "x$$1" = x && test "x$4" != xyes; then + if test "$$1" = "" && test "$4" != "yes"; then BITCOIN_QT_FAIL([$1 not found]) fi ]) @@ -57,20 +57,21 @@ AC_DEFUN([BITCOIN_QT_INIT],[ [build bitcoin-qt GUI (default=auto)])], [ bitcoin_qt_want_version=$withval - if test "x$bitcoin_qt_want_version" = xyes; then + if test "$bitcoin_qt_want_version" = "yes"; then bitcoin_qt_force=yes bitcoin_qt_want_version=auto fi ], [bitcoin_qt_want_version=auto]) - AS_IF([test "x$with_gui" = xqt5_debug], + AS_IF([test "$with_gui" = "qt5_debug"], [AS_CASE([$host], [*darwin*], [qt_lib_suffix=_debug], - [*mingw*], [qt_lib_suffix=d], [qt_lib_suffix= ]); bitcoin_qt_want_version=qt5], [qt_lib_suffix= ]) + AS_CASE([$host], [*android*], [qt_lib_suffix=_$ANDROID_ARCH]) + AC_ARG_WITH([qt-incdir],[AS_HELP_STRING([--with-qt-incdir=INC_DIR],[specify qt include path (overridden by pkgconfig)])], [qt_include_path=$withval], []) AC_ARG_WITH([qt-libdir],[AS_HELP_STRING([--with-qt-libdir=LIB_DIR],[specify qt lib path (overridden by pkgconfig)])], [qt_lib_path=$withval], []) AC_ARG_WITH([qt-plugindir],[AS_HELP_STRING([--with-qt-plugindir=PLUGIN_DIR],[specify qt plugin path (overridden by pkgconfig)])], [qt_plugin_path=$withval], []) @@ -86,7 +87,7 @@ AC_DEFUN([BITCOIN_QT_INIT],[ dnl Android doesn't support D-Bus and certainly doesn't use it for notifications case $host in *android*) - if test "x$use_dbus" != xyes; then + if test "$use_dbus" != "yes"; then use_dbus=no fi ;; @@ -118,10 +119,10 @@ AC_DEFUN([BITCOIN_QT_CONFIGURE],[ CPPFLAGS="$QT_INCLUDES $CPPFLAGS" CXXFLAGS="$PIC_FLAGS $CXXFLAGS" _BITCOIN_QT_IS_STATIC - if test "x$bitcoin_cv_static_qt" = xyes; then + if test "$bitcoin_cv_static_qt" = "yes"; then _BITCOIN_QT_CHECK_STATIC_LIBS - if test "x$qt_plugin_path" != x; then + if test "$qt_plugin_path" != ""; then if test -d "$qt_plugin_path/platforms"; then QT_LIBS="$QT_LIBS -L$qt_plugin_path/platforms" fi @@ -136,47 +137,45 @@ AC_DEFUN([BITCOIN_QT_CONFIGURE],[ fi fi - AC_DEFINE(QT_STATICPLUGIN, 1, [Define this symbol if qt plugins are static]) - if test "x$TARGET_OS" != xandroid; then + AC_DEFINE([QT_STATICPLUGIN], [1], [Define this symbol if qt plugins are static]) + if test "$TARGET_OS" != "android"; then _BITCOIN_QT_CHECK_STATIC_PLUGIN([QMinimalIntegrationPlugin], [-lqminimal]) - AC_DEFINE(QT_QPA_PLATFORM_MINIMAL, 1, [Define this symbol if the minimal qt platform exists]) + AC_DEFINE([QT_QPA_PLATFORM_MINIMAL], [1], [Define this symbol if the minimal qt platform exists]) fi - if test "x$TARGET_OS" = xwindows; then + if test "$TARGET_OS" = "windows"; then dnl Linking against wtsapi32 is required. See #17749 and dnl https://bugreports.qt.io/browse/QTBUG-27097. AX_CHECK_LINK_FLAG([-lwtsapi32], [QT_LIBS="$QT_LIBS -lwtsapi32"], [AC_MSG_ERROR([could not link against -lwtsapi32])]) _BITCOIN_QT_CHECK_STATIC_PLUGIN([QWindowsIntegrationPlugin], [-lqwindows]) _BITCOIN_QT_CHECK_STATIC_PLUGIN([QWindowsVistaStylePlugin], [-lqwindowsvistastyle]) - AC_DEFINE(QT_QPA_PLATFORM_WINDOWS, 1, [Define this symbol if the qt platform is windows]) - elif test "x$TARGET_OS" = xlinux; then - dnl workaround for https://bugreports.qt.io/browse/QTBUG-74874 - AX_CHECK_LINK_FLAG([-lxcb-shm], [QT_LIBS="$QT_LIBS -lxcb-shm"], [AC_MSG_ERROR([could not link against -lxcb-shm])]) + AC_DEFINE([QT_QPA_PLATFORM_WINDOWS], [1], [Define this symbol if the qt platform is windows]) + elif test "$TARGET_OS" = "linux"; then _BITCOIN_QT_CHECK_STATIC_PLUGIN([QXcbIntegrationPlugin], [-lqxcb]) - AC_DEFINE(QT_QPA_PLATFORM_XCB, 1, [Define this symbol if the qt platform is xcb]) - elif test "x$TARGET_OS" = xdarwin; then - AX_CHECK_LINK_FLAG([[-framework Carbon]],[QT_LIBS="$QT_LIBS -framework Carbon"],[AC_MSG_ERROR(could not link against Carbon framework)]) - AX_CHECK_LINK_FLAG([[-framework IOSurface]],[QT_LIBS="$QT_LIBS -framework IOSurface"],[AC_MSG_ERROR(could not link against IOSurface framework)]) - AX_CHECK_LINK_FLAG([[-framework Metal]],[QT_LIBS="$QT_LIBS -framework Metal"],[AC_MSG_ERROR(could not link against Metal framework)]) - AX_CHECK_LINK_FLAG([[-framework QuartzCore]],[QT_LIBS="$QT_LIBS -framework QuartzCore"],[AC_MSG_ERROR(could not link against QuartzCore framework)]) + AC_DEFINE([QT_QPA_PLATFORM_XCB], [1], [Define this symbol if the qt platform is xcb]) + elif test "$TARGET_OS" = "darwin"; then + AX_CHECK_LINK_FLAG([-framework Carbon], [QT_LIBS="$QT_LIBS -framework Carbon"], [AC_MSG_ERROR(could not link against Carbon framework)]) + AX_CHECK_LINK_FLAG([-framework IOSurface], [QT_LIBS="$QT_LIBS -framework IOSurface"], [AC_MSG_ERROR(could not link against IOSurface framework)]) + AX_CHECK_LINK_FLAG([-framework Metal], [QT_LIBS="$QT_LIBS -framework Metal"], [AC_MSG_ERROR(could not link against Metal framework)]) + AX_CHECK_LINK_FLAG([-framework QuartzCore], [QT_LIBS="$QT_LIBS -framework QuartzCore"], [AC_MSG_ERROR(could not link against QuartzCore framework)]) _BITCOIN_QT_CHECK_STATIC_PLUGIN([QCocoaIntegrationPlugin], [-lqcocoa]) _BITCOIN_QT_CHECK_STATIC_PLUGIN([QMacStylePlugin], [-lqmacstyle]) - AC_DEFINE(QT_QPA_PLATFORM_COCOA, 1, [Define this symbol if the qt platform is cocoa]) - elif test "x$TARGET_OS" = xandroid; then - QT_LIBS="-Wl,--export-dynamic,--undefined=JNI_OnLoad -lqtforandroid -ljnigraphics -landroid -lqtfreetype $QT_LIBS" - AC_DEFINE(QT_QPA_PLATFORM_ANDROID, 1, [Define this symbol if the qt platform is android]) + AC_DEFINE([QT_QPA_PLATFORM_COCOA], [1], [Define this symbol if the qt platform is cocoa]) + elif test "$TARGET_OS" = "android"; then + QT_LIBS="-Wl,--export-dynamic,--undefined=JNI_OnLoad -lplugins_platforms_qtforandroid${qt_lib_suffix} -ljnigraphics -landroid -lqtfreetype${qt_lib_suffix} $QT_LIBS" + AC_DEFINE([QT_QPA_PLATFORM_ANDROID], [1], [Define this symbol if the qt platform is android]) fi fi CPPFLAGS=$TEMP_CPPFLAGS CXXFLAGS=$TEMP_CXXFLAGS ]) - if test "x$qt_bin_path" = x; then + if test "$qt_bin_path" = ""; then qt_bin_path="`$PKG_CONFIG --variable=host_bins ${qt_lib_prefix}Core 2>/dev/null`" fi - if test "x$use_hardening" != xno; then + if test "$use_hardening" != "no"; then BITCOIN_QT_CHECK([ - AC_MSG_CHECKING(whether -fPIE can be used with this Qt config) + AC_MSG_CHECKING([whether -fPIE can be used with this Qt config]) TEMP_CPPFLAGS=$CPPFLAGS TEMP_CXXFLAGS=$CXXFLAGS CPPFLAGS="$QT_INCLUDES $CPPFLAGS" @@ -192,15 +191,15 @@ AC_DEFUN([BITCOIN_QT_CONFIGURE],[ choke #endif ]])], - [ AC_MSG_RESULT(yes); QT_PIE_FLAGS=$PIE_FLAGS ], - [ AC_MSG_RESULT(no); QT_PIE_FLAGS=$PIC_FLAGS] + [ AC_MSG_RESULT([yes]); QT_PIE_FLAGS=$PIE_FLAGS ], + [ AC_MSG_RESULT([no]); QT_PIE_FLAGS=$PIC_FLAGS] ) CPPFLAGS=$TEMP_CPPFLAGS CXXFLAGS=$TEMP_CXXFLAGS ]) else BITCOIN_QT_CHECK([ - AC_MSG_CHECKING(whether -fPIC is needed with this Qt config) + AC_MSG_CHECKING([whether -fPIC is needed with this Qt config]) TEMP_CPPFLAGS=$CPPFLAGS CPPFLAGS="$QT_INCLUDES $CPPFLAGS" AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ @@ -214,8 +213,8 @@ AC_DEFUN([BITCOIN_QT_CONFIGURE],[ choke #endif ]])], - [ AC_MSG_RESULT(no)], - [ AC_MSG_RESULT(yes); QT_PIE_FLAGS=$PIC_FLAGS] + [ AC_MSG_RESULT([no])], + [ AC_MSG_RESULT([yes]); QT_PIE_FLAGS=$PIC_FLAGS] ) CPPFLAGS=$TEMP_CPPFLAGS ]) @@ -234,12 +233,12 @@ AC_DEFUN([BITCOIN_QT_CONFIGURE],[ BITCOIN_QT_CHECK([ MOC_DEFS="${MOC_DEFS} -DQ_OS_MAC" base_frameworks="-framework Foundation -framework AppKit" - AX_CHECK_LINK_FLAG([[$base_frameworks]],[QT_LIBS="$QT_LIBS $base_frameworks"],[AC_MSG_ERROR(could not find base frameworks)]) + AX_CHECK_LINK_FLAG([$base_frameworks], [QT_LIBS="$QT_LIBS $base_frameworks"], [AC_MSG_ERROR(could not find base frameworks)]) ]) ;; *mingw*) BITCOIN_QT_CHECK([ - AX_CHECK_LINK_FLAG([[-mwindows]],[QT_LDFLAGS="$QT_LDFLAGS -mwindows"],[AC_MSG_WARN(-mwindows linker support not detected)]) + AX_CHECK_LINK_FLAG([-mwindows], [QT_LDFLAGS="$QT_LDFLAGS -mwindows"], [AC_MSG_WARN([-mwindows linker support not detected])]) ]) esac @@ -249,26 +248,26 @@ AC_DEFUN([BITCOIN_QT_CONFIGURE],[ BITCOIN_QT_CHECK([ bitcoin_enable_qt=yes bitcoin_enable_qt_test=yes - if test "x$have_qt_test" = xno; then + if test "$have_qt_test" = "no"; then bitcoin_enable_qt_test=no fi bitcoin_enable_qt_dbus=no - if test "x$use_dbus" != xno && test "x$have_qt_dbus" = xyes; then + if test "$use_dbus" != "no" && test "$have_qt_dbus" = "yes"; then bitcoin_enable_qt_dbus=yes fi - if test "x$use_dbus" = xyes && test "x$have_qt_dbus" = xno; then + if test "$use_dbus" = "yes" && test "$have_qt_dbus" = "no"; then AC_MSG_ERROR([libQtDBus not found. Install libQtDBus or remove --with-qtdbus.]) fi - if test "x$LUPDATE" = x; then + if test "$LUPDATE" = ""; then AC_MSG_WARN([lupdate tool is required to update Qt translations.]) fi - if test "x$LCONVERT" = x; then + if test "$LCONVERT" = ""; then AC_MSG_WARN([lconvert tool is required to update Qt translations.]) fi ],[ bitcoin_enable_qt=no ]) - if test x$bitcoin_enable_qt = xyes; then + if test $bitcoin_enable_qt = "yes"; then AC_MSG_RESULT([$bitcoin_enable_qt ($qt_lib_prefix)]) else AC_MSG_RESULT([$bitcoin_enable_qt]) @@ -279,9 +278,7 @@ AC_DEFUN([BITCOIN_QT_CONFIGURE],[ AC_SUBST(QT_LIBS) AC_SUBST(QT_LDFLAGS) AC_SUBST(QT_DBUS_INCLUDES) - AC_SUBST(QT_DBUS_LIBS) AC_SUBST(QT_TEST_INCLUDES) - AC_SUBST(QT_TEST_LIBS) AC_SUBST(QT_SELECT, qt5) AC_SUBST(MOC_DEFS) ]) @@ -349,18 +346,20 @@ AC_DEFUN([_BITCOIN_QT_CHECK_STATIC_LIBS], [ PKG_CHECK_MODULES([QT_FB], [${qt_lib_prefix}FbSupport${qt_lib_suffix}], [QT_LIBS="$QT_FB_LIBS $QT_LIBS"]) PKG_CHECK_MODULES([QT_FONTDATABASE], [${qt_lib_prefix}FontDatabaseSupport${qt_lib_suffix}], [QT_LIBS="$QT_FONTDATABASE_LIBS $QT_LIBS"]) PKG_CHECK_MODULES([QT_THEME], [${qt_lib_prefix}ThemeSupport${qt_lib_suffix}], [QT_LIBS="$QT_THEME_LIBS $QT_LIBS"]) - if test "x$TARGET_OS" = xlinux; then - PKG_CHECK_MODULES([QT_INPUT], [${qt_lib_prefix}XcbQpa], [QT_LIBS="$QT_INPUT_LIBS $QT_LIBS"]) + if test "$TARGET_OS" = "linux"; then + PKG_CHECK_MODULES([QT_INPUT], [${qt_lib_prefix}InputSupport], [QT_LIBS="$QT_INPUT_LIBS $QT_LIBS"]) PKG_CHECK_MODULES([QT_SERVICE], [${qt_lib_prefix}ServiceSupport], [QT_LIBS="$QT_SERVICE_LIBS $QT_LIBS"]) PKG_CHECK_MODULES([QT_XCBQPA], [${qt_lib_prefix}XcbQpa], [QT_LIBS="$QT_XCBQPA_LIBS $QT_LIBS"]) - elif test "x$TARGET_OS" = xdarwin; then + PKG_CHECK_MODULES([QT_XKBCOMMON], [${qt_lib_prefix}XkbCommonSupport], [QT_LIBS="$QT_XKBCOMMON_LIBS $QT_LIBS"]) + elif test "$TARGET_OS" = "darwin"; then PKG_CHECK_MODULES([QT_CLIPBOARD], [${qt_lib_prefix}ClipboardSupport${qt_lib_suffix}], [QT_LIBS="$QT_CLIPBOARD_LIBS $QT_LIBS"]) PKG_CHECK_MODULES([QT_GRAPHICS], [${qt_lib_prefix}GraphicsSupport${qt_lib_suffix}], [QT_LIBS="$QT_GRAPHICS_LIBS $QT_LIBS"]) PKG_CHECK_MODULES([QT_SERVICE], [${qt_lib_prefix}ServiceSupport${qt_lib_suffix}], [QT_LIBS="$QT_SERVICE_LIBS $QT_LIBS"]) - elif test "x$TARGET_OS" = xwindows; then + elif test "$TARGET_OS" = "windows"; then PKG_CHECK_MODULES([QT_WINDOWSUIAUTOMATION], [${qt_lib_prefix}WindowsUIAutomationSupport${qt_lib_suffix}], [QT_LIBS="$QT_WINDOWSUIAUTOMATION_LIBS $QT_LIBS"]) - elif test "x$TARGET_OS" = xandroid; then - PKG_CHECK_MODULES([QT_EGL], [${qt_lib_prefix}EglSupport], [QT_LIBS="$QT_EGL_LIBS $QT_LIBS"]) + elif test "$TARGET_OS" = "android"; then + PKG_CHECK_MODULES([QT_EGL], [${qt_lib_prefix}EglSupport${qt_lib_suffix}], [QT_LIBS="$QT_EGL_LIBS $QT_LIBS"]) + PKG_CHECK_MODULES([QT_SERVICE], [${qt_lib_prefix}ServiceSupport${qt_lib_suffix}], [QT_LIBS="$QT_SERVICE_LIBS $QT_LIBS"]) fi ]) @@ -391,7 +390,7 @@ AC_DEFUN([_BITCOIN_QT_FIND_LIBS],[ BITCOIN_QT_CHECK([ PKG_CHECK_MODULES([QT_TEST], [${qt_lib_prefix}Test${qt_lib_suffix} $qt_version], [QT_TEST_INCLUDES="$QT_TEST_CFLAGS"; have_qt_test=yes], [have_qt_test=no]) - if test "x$use_dbus" != xno; then + if test "$use_dbus" != "no"; then PKG_CHECK_MODULES([QT_DBUS], [${qt_lib_prefix}DBus $qt_version], [QT_DBUS_INCLUDES="$QT_DBUS_CFLAGS"; have_qt_dbus=yes], [have_qt_dbus=no]) fi ]) diff --git a/build-aux/m4/bitcoin_runtime_lib.m4 b/build-aux/m4/bitcoin_runtime_lib.m4 new file mode 100644 index 0000000000..1a6922deca --- /dev/null +++ b/build-aux/m4/bitcoin_runtime_lib.m4 @@ -0,0 +1,42 @@ +# On some platforms clang builtin implementations +# require compiler-rt as a runtime library to use. +# +# See: +# - https://bugs.llvm.org/show_bug.cgi?id=28629 + +m4_define([_CHECK_RUNTIME_testbody], [[ + bool f(long long x, long long y, long long* p) + { + return __builtin_mul_overflow(x, y, p); + } + int main() { return 0; } +]]) + +AC_DEFUN([CHECK_RUNTIME_LIB], [ + + AC_LANG_PUSH([C++]) + + AC_MSG_CHECKING([for __builtin_mul_overflow]) + AC_LINK_IFELSE( + [AC_LANG_SOURCE([_CHECK_RUNTIME_testbody])], + [ + AC_MSG_RESULT([yes]) + AC_DEFINE([HAVE_BUILTIN_MUL_OVERFLOW], [1], [Define if you have a working __builtin_mul_overflow]) + ], + [ + ax_check_save_flags="$LDFLAGS" + LDFLAGS="$LDFLAGS --rtlib=compiler-rt -lgcc_s" + AC_LINK_IFELSE( + [AC_LANG_SOURCE([_CHECK_RUNTIME_testbody])], + [ + AC_MSG_RESULT([yes, with additional linker flags]) + RUNTIME_LDFLAGS="--rtlib=compiler-rt -lgcc_s" + AC_DEFINE([HAVE_BUILTIN_MUL_OVERFLOW], [1], [Define if you have a working __builtin_mul_overflow]) + ], + [AC_MSG_RESULT([no])]) + LDFLAGS="$ax_check_save_flags" + ]) + + AC_LANG_POP + AC_SUBST([RUNTIME_LDFLAGS]) +]) diff --git a/build-aux/m4/bitcoin_subdir_to_include.m4 b/build-aux/m4/bitcoin_subdir_to_include.m4 index 7841042ac8..736270afea 100644 --- a/build-aux/m4/bitcoin_subdir_to_include.m4 +++ b/build-aux/m4/bitcoin_subdir_to_include.m4 @@ -5,13 +5,13 @@ dnl file COPYING or http://www.opensource.org/licenses/mit-license.php. dnl BITCOIN_SUBDIR_TO_INCLUDE([CPPFLAGS-VARIABLE-NAME],[SUBDIRECTORY-NAME],[HEADER-FILE]) dnl SUBDIRECTORY-NAME must end with a path separator AC_DEFUN([BITCOIN_SUBDIR_TO_INCLUDE],[ - if test "x$2" = "x"; then + if test "$2" = ""; then AC_MSG_RESULT([default]) else echo "#include <$2$3.h>" >conftest.cpp newinclpath=`${CXXCPP} ${CPPFLAGS} -M conftest.cpp 2>/dev/null | [ tr -d '\\n\\r\\\\' | sed -e 's/^.*[[:space:]:]\(\/[^[:space:]]*\)]$3[\.h[[:space:]].*$/\1/' -e t -e d`] AC_MSG_RESULT([${newinclpath}]) - if test "x${newinclpath}" != "x"; then + if test "${newinclpath}" != ""; then eval "$1=\"\$$1\"' -I${newinclpath}'" fi fi diff --git a/build-aux/m4/l_filesystem.m4 b/build-aux/m4/l_filesystem.m4 new file mode 100644 index 0000000000..ca3a0cd41c --- /dev/null +++ b/build-aux/m4/l_filesystem.m4 @@ -0,0 +1,47 @@ +dnl Copyright (c) 2022 The Bitcoin Core developers +dnl Distributed under the MIT software license, see the accompanying +dnl file COPYING or http://www.opensource.org/licenses/mit-license.php. + +# GCC 8.1 and earlier requires -lstdc++fs +# Clang 8.0.0 (libc++) and earlier requires -lc++fs + +m4_define([_CHECK_FILESYSTEM_testbody], [[ + #include + + namespace fs = std::filesystem; + + int main() { + (void)fs::current_path().root_name(); + return 0; + } +]]) + +AC_DEFUN([CHECK_FILESYSTEM], [ + + AC_LANG_PUSH(C++) + + AC_MSG_CHECKING([whether std::filesystem can be used without link library]) + + AC_LINK_IFELSE([AC_LANG_SOURCE([_CHECK_FILESYSTEM_testbody])],[ + AC_MSG_RESULT([yes]) + ],[ + AC_MSG_RESULT([no]) + SAVED_LIBS="$LIBS" + LIBS="$SAVED_LIBS -lstdc++fs" + AC_MSG_CHECKING([whether std::filesystem needs -lstdc++fs]) + AC_LINK_IFELSE([AC_LANG_SOURCE([_CHECK_FILESYSTEM_testbody])],[ + AC_MSG_RESULT([yes]) + ],[ + AC_MSG_RESULT([no]) + AC_MSG_CHECKING([whether std::filesystem needs -lc++fs]) + LIBS="$SAVED_LIBS -lc++fs" + AC_LINK_IFELSE([AC_LANG_SOURCE([_CHECK_FILESYSTEM_testbody])],[ + AC_MSG_RESULT([yes]) + ],[ + AC_MSG_FAILURE([cannot figure out how to use std::filesystem]) + ]) + ]) + ]) + + AC_LANG_POP +]) diff --git a/build_msvc/.gitignore b/build_msvc/.gitignore index ae8120fdf3..b0e557bc0c 100644 --- a/build_msvc/.gitignore +++ b/build_msvc/.gitignore @@ -13,7 +13,7 @@ packages/* libbitcoin_cli/libbitcoin_cli.vcxproj libbitcoin_common/libbitcoin_common.vcxproj libbitcoin_crypto/libbitcoin_crypto.vcxproj -libbitcoin_server/libbitcoin_server.vcxproj +libbitcoin_node/libbitcoin_node.vcxproj libbitcoin_util/libbitcoin_util.vcxproj libbitcoin_wallet_tool/libbitcoin_wallet_tool.vcxproj libbitcoin_wallet/libbitcoin_wallet.vcxproj @@ -21,6 +21,8 @@ libbitcoin_zmq/libbitcoin_zmq.vcxproj bench_bitcoin/bench_bitcoin.vcxproj libtest_util/libtest_util.vcxproj +/bitcoin_config.h + */Win32 libbitcoin_qt/QtGeneratedFiles/* test_bitcoin-qt/QtGeneratedFiles/* diff --git a/build_msvc/README.md b/build_msvc/README.md index 88a05644a7..cabe4d55ec 100644 --- a/build_msvc/README.md +++ b/build_msvc/README.md @@ -3,78 +3,65 @@ Building Bitcoin Core with Visual Studio Introduction --------------------- -Solution and project files to build the Bitcoin Core applications `msbuild` or Visual Studio can be found in the `build_msvc` directory. The build has been tested with Visual Studio 2019 (building with earlier versions of Visual Studio should not be expected to work). +Solution and project files to build Bitcoin Core with `msbuild` or Visual Studio can be found in the `build_msvc` directory. The build has been tested with Visual Studio 2019 (building with earlier versions of Visual Studio should not be expected to work). -Building with Visual Studio is an alternative to the Linux based [cross-compiler build](https://github.com/bitcoin/bitcoin/blob/master/doc/build-windows.md). +To build Bitcoin Core from the command-line, it is sufficient to only install the Visual Studio Build Tools component. -Quick Start +Building with Visual Studio is an alternative to the Linux based [cross-compiler build](../doc/build-windows.md). + + +Prerequisites --------------------- -The minimal steps required to build Bitcoin Core with the msbuild toolchain are below. More detailed instructions are contained in the following sections. +To build [dependencies](../doc/dependencies.md) (except for [Qt](#qt)), +the default approach is to use the [vcpkg](https://docs.microsoft.com/en-us/cpp/vcpkg) package manager from Microsoft: -``` -cd build_msvc -py -3 msvc-autogen.py -msbuild /m bitcoin.sln /p:Platform=x64 /p:Configuration=Release /t:build -``` +1. [Install](https://vcpkg.io/en/getting-started.html) vcpkg. -Dependencies ---------------------- -A number of [open source libraries](https://github.com/bitcoin/bitcoin/blob/master/doc/dependencies.md) are required in order to be able to build Bitcoin Core. +2. By default, vcpkg makes both `release` and `debug` builds for each package. +To save build time and disk space, one could skip `debug` builds (example uses PowerShell): +```powershell -Options for installing the dependencies in a Visual Studio compatible manner are: - -- Use Microsoft's [vcpkg](https://docs.microsoft.com/en-us/cpp/vcpkg) to download the source packages and build locally. This is the recommended approach. -- Download the source code, build each dependency, add the required include paths, link libraries and binary tools to the Visual Studio project files. -- Use [nuget](https://www.nuget.org/) packages with the understanding that any binary files have been compiled by an untrusted third party. - -The [external dependencies](https://github.com/bitcoin/bitcoin/blob/master/doc/dependencies.md) required for building are listed in the `build_msvc/vcpkg.json` file. To ensure `msbuild` project files automatically install the `vcpkg` dependencies use: - -``` -vcpkg integrate install +Add-Content -Path "vcpkg\triplets\x64-windows-static.cmake" -Value "set(VCPKG_BUILD_TYPE release)" ``` Qt --------------------- -In order to build Bitcoin Core a static build of Qt is required. The runtime library version (e.g. v142) and platform type (x86 or x64) must also match. +To build Bitcoin Core with the GUI, a static build of Qt is required. -Some prebuilt x64 versions of Qt can be downloaded from [here](https://github.com/sipsorcery/qt_win_binary/releases). Please be aware these downloads are NOT officially sanctioned by Bitcoin Core and are provided for developer convenience only. They should NOT be used for builds that will be used in a production environment or with real funds. +1. Download a single ZIP archive of Qt source code from https://download.qt.io/official_releases/qt/ (e.g., [`qt-everywhere-src-5.15.2.zip`](https://download.qt.io/official_releases/qt/5.15/5.15.2/single/qt-everywhere-src-5.15.2.zip)), and expand it into a dedicated folder. The following instructions assume that this folder is `C:\dev\qt-source`. -To determine which Qt prebuilt version to download open the `.appveyor.yml` file and note the `QT_DOWNLOAD_URL`. When extracting the zip file the destination path must be set to `C:\`. This is due to the way that Qt includes, libraries and tools use internal paths. +2. Open "x64 Native Tools Command Prompt for VS 2019", and input the following commands: +```cmd +cd C:\dev\qt-source +mkdir build +cd build +..\configure -release -silent -opensource -confirm-license -opengl desktop -static -static-runtime -mp -qt-zlib -qt-pcre -qt-libpng -nomake examples -nomake tests -nomake tools -no-angle -no-dbus -no-gif -no-gtk -no-ico -no-icu -no-libjpeg -no-libudev -no-sql-sqlite -no-sql-odbc -no-sqlite -no-vulkan -skip qt3d -skip qtactiveqt -skip qtandroidextras -skip qtcharts -skip qtconnectivity -skip qtdatavis3d -skip qtdeclarative -skip doc -skip qtdoc -skip qtgamepad -skip qtgraphicaleffects -skip qtimageformats -skip qtlocation -skip qtlottie -skip qtmacextras -skip qtmultimedia -skip qtnetworkauth -skip qtpurchasing -skip qtquick3d -skip qtquickcontrols -skip qtquickcontrols2 -skip qtquicktimeline -skip qtremoteobjects -skip qtscript -skip qtscxml -skip qtsensors -skip qtserialbus -skip qtserialport -skip qtspeech -skip qtsvg -skip qtvirtualkeyboard -skip qtwayland -skip qtwebchannel -skip qtwebengine -skip qtwebglplugin -skip qtwebsockets -skip qtwebview -skip qtx11extras -skip qtxmlpatterns -no-openssl -no-feature-bearermanagement -no-feature-printdialog -no-feature-printer -no-feature-printpreviewdialog -no-feature-printpreviewwidget -no-feature-sql -no-feature-sqlmodel -no-feature-textbrowser -no-feature-textmarkdownwriter -no-feature-textodfwriter -no-feature-xml -prefix C:\Qt_static +nmake +nmake install +``` + +One could speed up building with [`jom`](https://wiki.qt.io/Jom), a replacement for `nmake` which makes use of all CPU cores. + +To build Bitcoin Core without Qt, unload or disable the `bitcoin-qt`, `libbitcoin_qt` and `test_bitcoin-qt` projects. -To build Bitcoin Core without Qt unload or disable the `bitcoin-qt`, `libbitcoin_qt` and `test_bitcoin-qt` projects. Building --------------------- -The instructions below use `vcpkg` to install the dependencies. - -- Install [`vcpkg`](https://github.com/Microsoft/vcpkg). - -- Use Python to generate `*.vcxproj` from Makefile +1. Use Python to generate `*.vcxproj` from Makefile: ``` PS >py -3 msvc-autogen.py ``` -- An optional step is to adjust the settings in the `build_msvc` directory and the `common.init.vcxproj` file. This project file contains settings that are common to all projects such as the runtime library version and target Windows SDK version. The Qt directories can also be set. +2. An optional step is to adjust the settings in the `build_msvc` directory and the `common.init.vcxproj` file. This project file contains settings that are common to all projects such as the runtime library version and target Windows SDK version. The Qt directories can also be set. To specify a non-default path to a static Qt package directory, use the `QTBASEDIR` environment variable. -- To build from the command line with the Visual Studio 2019 toolchain use: +3. To build from the command-line with the Visual Studio 2019 toolchain use: -``` -msbuild /m bitcoin.sln /p:Platform=x64 /p:Configuration=Release /t:build +```cmd +msbuild -property:Configuration=Release -maxCpuCount -verbosity:minimal bitcoin.sln ``` -- Alternatively, open the `build_msvc/bitcoin.sln` file in Visual Studio 2019. - -AppVeyor ---------------------- -The .appveyor.yml in the root directory is suitable to perform builds on [AppVeyor](https://www.appveyor.com/) Continuous Integration servers. The simplest way to perform an AppVeyor build is to fork Bitcoin Core and then configure a new AppVeyor Project pointing to the forked repository. - -For safety reasons the Bitcoin Core .appveyor.yml file has the artifact options disabled. The build will be performed but no executable files will be available. To enable artifacts on a forked repository uncomment the lines shown below: - -``` - #- 7z a bitcoin-%APPVEYOR_BUILD_VERSION%.zip %APPVEYOR_BUILD_FOLDER%\build_msvc\%platform%\%configuration%\*.exe - #- path: bitcoin-%APPVEYOR_BUILD_VERSION%.zip -``` +Alternatively, open the `build_msvc/bitcoin.sln` file in Visual Studio 2019. Security --------------------- diff --git a/build_msvc/bench_bitcoin/bench_bitcoin.vcxproj.in b/build_msvc/bench_bitcoin/bench_bitcoin.vcxproj.in index 128c1bd8e7..fc9d7cbed6 100644 --- a/build_msvc/bench_bitcoin/bench_bitcoin.vcxproj.in +++ b/build_msvc/bench_bitcoin/bench_bitcoin.vcxproj.in @@ -21,7 +21,7 @@ {6190199c-6cf4-4dad-bfbd-93fa72a760c1} - + {460fee33-1fe1-483f-b3bf-931ff8e969a5} diff --git a/build_msvc/bitcoin-qt/bitcoin-qt.vcxproj b/build_msvc/bitcoin-qt/bitcoin-qt.vcxproj index a697c1dfb6..0d6358e0d0 100644 --- a/build_msvc/bitcoin-qt/bitcoin-qt.vcxproj +++ b/build_msvc/bitcoin-qt/bitcoin-qt.vcxproj @@ -9,6 +9,7 @@ + @@ -27,7 +28,7 @@ {2b4abff8-d1fd-4845-88c9-1f3c0a6512bf} - + {460fee33-1fe1-483f-b3bf-931ff8e969a5} diff --git a/build_msvc/bitcoin-util/bitcoin-util.vcxproj b/build_msvc/bitcoin-util/bitcoin-util.vcxproj index 3a6aa4a837..8a0964824b 100644 --- a/build_msvc/bitcoin-util/bitcoin-util.vcxproj +++ b/build_msvc/bitcoin-util/bitcoin-util.vcxproj @@ -2,7 +2,7 @@ - {D3022AF6-AD33-4CE3-B358-87CB6A1B29CF} + {57A04EC9-542A-4E40-83D0-AC3BE1F36805} Application diff --git a/build_msvc/bitcoin-wallet/bitcoin-wallet.vcxproj b/build_msvc/bitcoin-wallet/bitcoin-wallet.vcxproj index 40c5db5522..2ac0be9814 100644 --- a/build_msvc/bitcoin-wallet/bitcoin-wallet.vcxproj +++ b/build_msvc/bitcoin-wallet/bitcoin-wallet.vcxproj @@ -10,32 +10,26 @@ + + $(IntDir)init_bitcoin-wallet.obj + {2b384fa8-9ee1-4544-93cb-0d733c25e8ce} - - {0667528c-d734-4009-adf9-c0d6c4a5a5a6} - {7c87e378-df58-482e-aa2f-1bc129bc19ce} {6190199c-6cf4-4dad-bfbd-93fa72a760c1} - - {460fee33-1fe1-483f-b3bf-931ff8e969a5} - {b53a5535-ee9d-4c6f-9a26-f79ee3bc3754} {93b86837-b543-48a5-a89b-7c87abb77df2} - - {792d487f-f14c-49fc-a9de-3fc150f31c3f} - {5724ba7d-a09a-4ba8-800b-c4c1561b3d69} @@ -45,9 +39,6 @@ {bb493552-3b8c-4a8c-bf69-a6e7a51d2ea6} - - {18430fef-6b61-4c53-b396-718e02850f1b} - diff --git a/build_msvc/bitcoin.sln b/build_msvc/bitcoin.sln index 7d8591c10b..2a1ccf58fe 100644 --- a/build_msvc/bitcoin.sln +++ b/build_msvc/bitcoin.sln @@ -4,8 +4,6 @@ VisualStudioVersion = 16.0.28803.452 MinimumVisualStudioVersion = 10.0.40219.1 Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "libbitcoinconsensus", "libbitcoinconsensus\libbitcoinconsensus.vcxproj", "{2B384FA8-9EE1-4544-93CB-0D733C25E8CE}" EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "testconsensus", "testconsensus\testconsensus.vcxproj", "{E78473E9-B850-456C-9120-276301E04C06}" -EndProject Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "bitcoind", "bitcoind\bitcoind.vcxproj", "{D4513DDF-6013-44DC-ADCC-12EAF6D1F038}" EndProject Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "libbitcoin_util", "libbitcoin_util\libbitcoin_util.vcxproj", "{B53A5535-EE9D-4C6F-9A26-F79EE3BC3754}" @@ -14,7 +12,7 @@ Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "libbitcoin_common", "libbit EndProject Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "libbitcoin_crypto", "libbitcoin_crypto\libbitcoin_crypto.vcxproj", "{6190199C-6CF4-4DAD-BFBD-93FA72A760C1}" EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "libbitcoin_server", "libbitcoin_server\libbitcoin_server.vcxproj", "{460FEE33-1FE1-483F-B3BF-931FF8E969A5}" +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "libbitcoin_node", "libbitcoin_node\libbitcoin_node.vcxproj", "{460FEE33-1FE1-483F-B3BF-931FF8E969A5}" EndProject Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "libunivalue", "libunivalue\libunivalue.vcxproj", "{5724BA7D-A09A-4BA8-800B-C4C1561B3D69}" EndProject @@ -32,7 +30,7 @@ Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "bench_bitcoin", "bench_bitc EndProject Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "bitcoin-tx", "bitcoin-tx\bitcoin-tx.vcxproj", "{D3022AF6-AD33-4CE3-B358-87CB6A1B29CF}" EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "bitcoin-util", "bitcoin-util\bitcoin-util.vcxproj", "{D3022AF6-AD33-4CE3-B358-87CB6A1B29CF}" +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "bitcoin-util", "bitcoin-util\bitcoin-util.vcxproj", "{57A04EC9-542A-4E40-83D0-AC3BE1F36805}" EndProject Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "bitcoin-wallet", "bitcoin-wallet\bitcoin-wallet.vcxproj", "{84DE8790-EDE3-4483-81AC-C32F15E861F4}" EndProject @@ -50,205 +48,115 @@ Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "libtest_util", "libtest_uti EndProject Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "test_bitcoin-qt", "test_bitcoin-qt\test_bitcoin-qt.vcxproj", "{51201D5E-D939-4854-AE9D-008F03FF518E}" EndProject +Project("{542007E3-BE0D-4B0D-A6B0-AA8813E2558D}") = "libminisketch", "libminisketch\libminisketch.vcxproj", "{542007E3-BE0D-4B0D-A6B0-AA8813E2558D}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|x64 = Debug|x64 - Debug|x86 = Debug|x86 Release|x64 = Release|x64 - Release|x86 = Release|x86 EndGlobalSection GlobalSection(ProjectConfigurationPlatforms) = postSolution {2B384FA8-9EE1-4544-93CB-0D733C25E8CE}.Debug|x64.ActiveCfg = Debug|x64 {2B384FA8-9EE1-4544-93CB-0D733C25E8CE}.Debug|x64.Build.0 = Debug|x64 - {2B384FA8-9EE1-4544-93CB-0D733C25E8CE}.Debug|x86.ActiveCfg = Debug|Win32 - {2B384FA8-9EE1-4544-93CB-0D733C25E8CE}.Debug|x86.Build.0 = Debug|Win32 {2B384FA8-9EE1-4544-93CB-0D733C25E8CE}.Release|x64.ActiveCfg = Release|x64 {2B384FA8-9EE1-4544-93CB-0D733C25E8CE}.Release|x64.Build.0 = Release|x64 - {2B384FA8-9EE1-4544-93CB-0D733C25E8CE}.Release|x86.ActiveCfg = Release|Win32 - {2B384FA8-9EE1-4544-93CB-0D733C25E8CE}.Release|x86.Build.0 = Release|Win32 - {E78473E9-B850-456C-9120-276301E04C06}.Debug|x64.ActiveCfg = Debug|x64 - {E78473E9-B850-456C-9120-276301E04C06}.Debug|x64.Build.0 = Debug|x64 - {E78473E9-B850-456C-9120-276301E04C06}.Debug|x86.ActiveCfg = Debug|Win32 - {E78473E9-B850-456C-9120-276301E04C06}.Debug|x86.Build.0 = Debug|Win32 - {E78473E9-B850-456C-9120-276301E04C06}.Release|x64.ActiveCfg = Release|x64 - {E78473E9-B850-456C-9120-276301E04C06}.Release|x64.Build.0 = Release|x64 - {E78473E9-B850-456C-9120-276301E04C06}.Release|x86.ActiveCfg = Release|Win32 - {E78473E9-B850-456C-9120-276301E04C06}.Release|x86.Build.0 = Release|Win32 {D4513DDF-6013-44DC-ADCC-12EAF6D1F038}.Debug|x64.ActiveCfg = Debug|x64 {D4513DDF-6013-44DC-ADCC-12EAF6D1F038}.Debug|x64.Build.0 = Debug|x64 - {D4513DDF-6013-44DC-ADCC-12EAF6D1F038}.Debug|x86.ActiveCfg = Debug|Win32 - {D4513DDF-6013-44DC-ADCC-12EAF6D1F038}.Debug|x86.Build.0 = Debug|Win32 {D4513DDF-6013-44DC-ADCC-12EAF6D1F038}.Release|x64.ActiveCfg = Release|x64 {D4513DDF-6013-44DC-ADCC-12EAF6D1F038}.Release|x64.Build.0 = Release|x64 - {D4513DDF-6013-44DC-ADCC-12EAF6D1F038}.Release|x86.ActiveCfg = Release|Win32 - {D4513DDF-6013-44DC-ADCC-12EAF6D1F038}.Release|x86.Build.0 = Release|Win32 {B53A5535-EE9D-4C6F-9A26-F79EE3BC3754}.Debug|x64.ActiveCfg = Debug|x64 {B53A5535-EE9D-4C6F-9A26-F79EE3BC3754}.Debug|x64.Build.0 = Debug|x64 - {B53A5535-EE9D-4C6F-9A26-F79EE3BC3754}.Debug|x86.ActiveCfg = Debug|Win32 - {B53A5535-EE9D-4C6F-9A26-F79EE3BC3754}.Debug|x86.Build.0 = Debug|Win32 {B53A5535-EE9D-4C6F-9A26-F79EE3BC3754}.Release|x64.ActiveCfg = Release|x64 {B53A5535-EE9D-4C6F-9A26-F79EE3BC3754}.Release|x64.Build.0 = Release|x64 - {B53A5535-EE9D-4C6F-9A26-F79EE3BC3754}.Release|x86.ActiveCfg = Release|Win32 - {B53A5535-EE9D-4C6F-9A26-F79EE3BC3754}.Release|x86.Build.0 = Release|Win32 {7C87E378-DF58-482E-AA2F-1BC129BC19CE}.Debug|x64.ActiveCfg = Debug|x64 {7C87E378-DF58-482E-AA2F-1BC129BC19CE}.Debug|x64.Build.0 = Debug|x64 - {7C87E378-DF58-482E-AA2F-1BC129BC19CE}.Debug|x86.ActiveCfg = Debug|Win32 - {7C87E378-DF58-482E-AA2F-1BC129BC19CE}.Debug|x86.Build.0 = Debug|Win32 {7C87E378-DF58-482E-AA2F-1BC129BC19CE}.Release|x64.ActiveCfg = Release|x64 {7C87E378-DF58-482E-AA2F-1BC129BC19CE}.Release|x64.Build.0 = Release|x64 - {7C87E378-DF58-482E-AA2F-1BC129BC19CE}.Release|x86.ActiveCfg = Release|Win32 - {7C87E378-DF58-482E-AA2F-1BC129BC19CE}.Release|x86.Build.0 = Release|Win32 {6190199C-6CF4-4DAD-BFBD-93FA72A760C1}.Debug|x64.ActiveCfg = Debug|x64 {6190199C-6CF4-4DAD-BFBD-93FA72A760C1}.Debug|x64.Build.0 = Debug|x64 - {6190199C-6CF4-4DAD-BFBD-93FA72A760C1}.Debug|x86.ActiveCfg = Debug|Win32 - {6190199C-6CF4-4DAD-BFBD-93FA72A760C1}.Debug|x86.Build.0 = Debug|Win32 {6190199C-6CF4-4DAD-BFBD-93FA72A760C1}.Release|x64.ActiveCfg = Release|x64 {6190199C-6CF4-4DAD-BFBD-93FA72A760C1}.Release|x64.Build.0 = Release|x64 - {6190199C-6CF4-4DAD-BFBD-93FA72A760C1}.Release|x86.ActiveCfg = Release|Win32 - {6190199C-6CF4-4DAD-BFBD-93FA72A760C1}.Release|x86.Build.0 = Release|Win32 {460FEE33-1FE1-483F-B3BF-931FF8E969A5}.Debug|x64.ActiveCfg = Debug|x64 {460FEE33-1FE1-483F-B3BF-931FF8E969A5}.Debug|x64.Build.0 = Debug|x64 - {460FEE33-1FE1-483F-B3BF-931FF8E969A5}.Debug|x86.ActiveCfg = Debug|Win32 - {460FEE33-1FE1-483F-B3BF-931FF8E969A5}.Debug|x86.Build.0 = Debug|Win32 {460FEE33-1FE1-483F-B3BF-931FF8E969A5}.Release|x64.ActiveCfg = Release|x64 {460FEE33-1FE1-483F-B3BF-931FF8E969A5}.Release|x64.Build.0 = Release|x64 - {460FEE33-1FE1-483F-B3BF-931FF8E969A5}.Release|x86.ActiveCfg = Release|Win32 - {460FEE33-1FE1-483F-B3BF-931FF8E969A5}.Release|x86.Build.0 = Release|Win32 {5724BA7D-A09A-4BA8-800B-C4C1561B3D69}.Debug|x64.ActiveCfg = Debug|x64 {5724BA7D-A09A-4BA8-800B-C4C1561B3D69}.Debug|x64.Build.0 = Debug|x64 - {5724BA7D-A09A-4BA8-800B-C4C1561B3D69}.Debug|x86.ActiveCfg = Debug|Win32 - {5724BA7D-A09A-4BA8-800B-C4C1561B3D69}.Debug|x86.Build.0 = Debug|Win32 {5724BA7D-A09A-4BA8-800B-C4C1561B3D69}.Release|x64.ActiveCfg = Release|x64 {5724BA7D-A09A-4BA8-800B-C4C1561B3D69}.Release|x64.Build.0 = Release|x64 - {5724BA7D-A09A-4BA8-800B-C4C1561B3D69}.Release|x86.ActiveCfg = Release|Win32 - {5724BA7D-A09A-4BA8-800B-C4C1561B3D69}.Release|x86.Build.0 = Release|Win32 {93B86837-B543-48A5-A89B-7C87ABB77DF2}.Debug|x64.ActiveCfg = Debug|x64 {93B86837-B543-48A5-A89B-7C87ABB77DF2}.Debug|x64.Build.0 = Debug|x64 - {93B86837-B543-48A5-A89B-7C87ABB77DF2}.Debug|x86.ActiveCfg = Debug|Win32 - {93B86837-B543-48A5-A89B-7C87ABB77DF2}.Debug|x86.Build.0 = Debug|Win32 {93B86837-B543-48A5-A89B-7C87ABB77DF2}.Release|x64.ActiveCfg = Release|x64 {93B86837-B543-48A5-A89B-7C87ABB77DF2}.Release|x64.Build.0 = Release|x64 - {93B86837-B543-48A5-A89B-7C87ABB77DF2}.Release|x86.ActiveCfg = Release|Win32 - {93B86837-B543-48A5-A89B-7C87ABB77DF2}.Release|x86.Build.0 = Release|Win32 {792D487F-F14C-49FC-A9DE-3FC150F31C3F}.Debug|x64.ActiveCfg = Debug|x64 {792D487F-F14C-49FC-A9DE-3FC150F31C3F}.Debug|x64.Build.0 = Debug|x64 - {792D487F-F14C-49FC-A9DE-3FC150F31C3F}.Debug|x86.ActiveCfg = Debug|Win32 - {792D487F-F14C-49FC-A9DE-3FC150F31C3F}.Debug|x86.Build.0 = Debug|Win32 {792D487F-F14C-49FC-A9DE-3FC150F31C3F}.Release|x64.ActiveCfg = Release|x64 {792D487F-F14C-49FC-A9DE-3FC150F31C3F}.Release|x64.Build.0 = Release|x64 - {792D487F-F14C-49FC-A9DE-3FC150F31C3F}.Release|x86.ActiveCfg = Release|Win32 - {792D487F-F14C-49FC-A9DE-3FC150F31C3F}.Release|x86.Build.0 = Release|Win32 {A56B73DB-D46D-4882-8374-1FE3FFA08F07}.Debug|x64.ActiveCfg = Debug|x64 {A56B73DB-D46D-4882-8374-1FE3FFA08F07}.Debug|x64.Build.0 = Debug|x64 - {A56B73DB-D46D-4882-8374-1FE3FFA08F07}.Debug|x86.ActiveCfg = Debug|Win32 - {A56B73DB-D46D-4882-8374-1FE3FFA08F07}.Debug|x86.Build.0 = Debug|Win32 {A56B73DB-D46D-4882-8374-1FE3FFA08F07}.Release|x64.ActiveCfg = Release|x64 {A56B73DB-D46D-4882-8374-1FE3FFA08F07}.Release|x64.Build.0 = Release|x64 - {A56B73DB-D46D-4882-8374-1FE3FFA08F07}.Release|x86.ActiveCfg = Release|Win32 - {A56B73DB-D46D-4882-8374-1FE3FFA08F07}.Release|x86.Build.0 = Release|Win32 {0667528C-D734-4009-ADF9-C0D6C4A5A5A6}.Debug|x64.ActiveCfg = Debug|x64 {0667528C-D734-4009-ADF9-C0D6C4A5A5A6}.Debug|x64.Build.0 = Debug|x64 - {0667528C-D734-4009-ADF9-C0D6C4A5A5A6}.Debug|x86.ActiveCfg = Debug|Win32 - {0667528C-D734-4009-ADF9-C0D6C4A5A5A6}.Debug|x86.Build.0 = Debug|Win32 {0667528C-D734-4009-ADF9-C0D6C4A5A5A6}.Release|x64.ActiveCfg = Release|x64 {0667528C-D734-4009-ADF9-C0D6C4A5A5A6}.Release|x64.Build.0 = Release|x64 - {0667528C-D734-4009-ADF9-C0D6C4A5A5A6}.Release|x86.ActiveCfg = Release|Win32 - {0667528C-D734-4009-ADF9-C0D6C4A5A5A6}.Release|x86.Build.0 = Release|Win32 {0B2D7431-F876-4A58-87BF-F748338CD3BF}.Debug|x64.ActiveCfg = Debug|x64 {0B2D7431-F876-4A58-87BF-F748338CD3BF}.Debug|x64.Build.0 = Debug|x64 - {0B2D7431-F876-4A58-87BF-F748338CD3BF}.Debug|x86.ActiveCfg = Debug|Win32 - {0B2D7431-F876-4A58-87BF-F748338CD3BF}.Debug|x86.Build.0 = Debug|Win32 {0B2D7431-F876-4A58-87BF-F748338CD3BF}.Release|x64.ActiveCfg = Release|x64 {0B2D7431-F876-4A58-87BF-F748338CD3BF}.Release|x64.Build.0 = Release|x64 - {0B2D7431-F876-4A58-87BF-F748338CD3BF}.Release|x86.ActiveCfg = Release|Win32 - {0B2D7431-F876-4A58-87BF-F748338CD3BF}.Release|x86.Build.0 = Release|Win32 {1125654E-E1B2-4431-8B5C-62EA9A2FEECB}.Debug|x64.ActiveCfg = Debug|x64 {1125654E-E1B2-4431-8B5C-62EA9A2FEECB}.Debug|x64.Build.0 = Debug|x64 - {1125654E-E1B2-4431-8B5C-62EA9A2FEECB}.Debug|x86.ActiveCfg = Debug|Win32 - {1125654E-E1B2-4431-8B5C-62EA9A2FEECB}.Debug|x86.Build.0 = Debug|Win32 {1125654E-E1B2-4431-8B5C-62EA9A2FEECB}.Release|x64.ActiveCfg = Release|x64 {1125654E-E1B2-4431-8B5C-62EA9A2FEECB}.Release|x64.Build.0 = Release|x64 - {1125654E-E1B2-4431-8B5C-62EA9A2FEECB}.Release|x86.ActiveCfg = Release|Win32 - {1125654E-E1B2-4431-8B5C-62EA9A2FEECB}.Release|x86.Build.0 = Release|Win32 {D3022AF6-AD33-4CE3-B358-87CB6A1B29CF}.Debug|x64.ActiveCfg = Debug|x64 {D3022AF6-AD33-4CE3-B358-87CB6A1B29CF}.Debug|x64.Build.0 = Debug|x64 - {D3022AF6-AD33-4CE3-B358-87CB6A1B29CF}.Debug|x86.ActiveCfg = Debug|Win32 - {D3022AF6-AD33-4CE3-B358-87CB6A1B29CF}.Debug|x86.Build.0 = Debug|Win32 {D3022AF6-AD33-4CE3-B358-87CB6A1B29CF}.Release|x64.ActiveCfg = Release|x64 {D3022AF6-AD33-4CE3-B358-87CB6A1B29CF}.Release|x64.Build.0 = Release|x64 - {D3022AF6-AD33-4CE3-B358-87CB6A1B29CF}.Release|x86.ActiveCfg = Release|Win32 - {D3022AF6-AD33-4CE3-B358-87CB6A1B29CF}.Release|x86.Build.0 = Release|Win32 + {57A04EC9-542A-4E40-83D0-AC3BE1F36805}.Debug|x64.ActiveCfg = Debug|x64 + {57A04EC9-542A-4E40-83D0-AC3BE1F36805}.Debug|x64.Build.0 = Debug|x64 + {57A04EC9-542A-4E40-83D0-AC3BE1F36805}.Release|x64.ActiveCfg = Release|x64 + {57A04EC9-542A-4E40-83D0-AC3BE1F36805}.Release|x64.Build.0 = Release|x64 {84DE8790-EDE3-4483-81AC-C32F15E861F4}.Debug|x64.ActiveCfg = Debug|x64 {84DE8790-EDE3-4483-81AC-C32F15E861F4}.Debug|x64.Build.0 = Debug|x64 - {84DE8790-EDE3-4483-81AC-C32F15E861F4}.Debug|x86.ActiveCfg = Debug|Win32 - {84DE8790-EDE3-4483-81AC-C32F15E861F4}.Debug|x86.Build.0 = Debug|Win32 {84DE8790-EDE3-4483-81AC-C32F15E861F4}.Release|x64.ActiveCfg = Release|x64 {84DE8790-EDE3-4483-81AC-C32F15E861F4}.Release|x64.Build.0 = Release|x64 - {84DE8790-EDE3-4483-81AC-C32F15E861F4}.Release|x86.ActiveCfg = Release|Win32 - {84DE8790-EDE3-4483-81AC-C32F15E861F4}.Release|x86.Build.0 = Release|Win32 {F91AC55E-6F5E-4C58-9AC5-B40DB7DEEF93}.Debug|x64.ActiveCfg = Debug|x64 {F91AC55E-6F5E-4C58-9AC5-B40DB7DEEF93}.Debug|x64.Build.0 = Debug|x64 - {F91AC55E-6F5E-4C58-9AC5-B40DB7DEEF93}.Debug|x86.ActiveCfg = Debug|Win32 - {F91AC55E-6F5E-4C58-9AC5-B40DB7DEEF93}.Debug|x86.Build.0 = Debug|Win32 {F91AC55E-6F5E-4C58-9AC5-B40DB7DEEF93}.Release|x64.ActiveCfg = Release|x64 {F91AC55E-6F5E-4C58-9AC5-B40DB7DEEF93}.Release|x64.Build.0 = Release|x64 - {F91AC55E-6F5E-4C58-9AC5-B40DB7DEEF93}.Release|x86.ActiveCfg = Release|Win32 - {F91AC55E-6F5E-4C58-9AC5-B40DB7DEEF93}.Release|x86.Build.0 = Release|Win32 {BB493552-3B8C-4A8C-BF69-A6E7A51D2EA6}.Debug|x64.ActiveCfg = Debug|x64 {BB493552-3B8C-4A8C-BF69-A6E7A51D2EA6}.Debug|x64.Build.0 = Debug|x64 - {BB493552-3B8C-4A8C-BF69-A6E7A51D2EA6}.Debug|x86.ActiveCfg = Debug|Win32 - {BB493552-3B8C-4A8C-BF69-A6E7A51D2EA6}.Debug|x86.Build.0 = Debug|Win32 {BB493552-3B8C-4A8C-BF69-A6E7A51D2EA6}.Release|x64.ActiveCfg = Release|x64 {BB493552-3B8C-4A8C-BF69-A6E7A51D2EA6}.Release|x64.Build.0 = Release|x64 - {BB493552-3B8C-4A8C-BF69-A6E7A51D2EA6}.Release|x86.ActiveCfg = Release|Win32 - {BB493552-3B8C-4A8C-BF69-A6E7A51D2EA6}.Release|x86.Build.0 = Release|Win32 {18430FEF-6B61-4C53-B396-718E02850F1B}.Debug|x64.ActiveCfg = Debug|x64 {18430FEF-6B61-4C53-B396-718E02850F1B}.Debug|x64.Build.0 = Debug|x64 - {18430FEF-6B61-4C53-B396-718E02850F1B}.Debug|x86.ActiveCfg = Debug|Win32 - {18430FEF-6B61-4C53-B396-718E02850F1B}.Debug|x86.Build.0 = Debug|Win32 {18430FEF-6B61-4C53-B396-718E02850F1B}.Release|x64.ActiveCfg = Release|x64 {18430FEF-6B61-4C53-B396-718E02850F1B}.Release|x64.Build.0 = Release|x64 - {18430FEF-6B61-4C53-B396-718E02850F1B}.Release|x86.ActiveCfg = Release|Win32 - {18430FEF-6B61-4C53-B396-718E02850F1B}.Release|x86.Build.0 = Release|Win32 {2B4ABFF8-D1FD-4845-88C9-1F3C0A6512BF}.Debug|x64.ActiveCfg = Debug|x64 {2B4ABFF8-D1FD-4845-88C9-1F3C0A6512BF}.Debug|x64.Build.0 = Debug|x64 - {2B4ABFF8-D1FD-4845-88C9-1F3C0A6512BF}.Debug|x86.ActiveCfg = Debug|Win32 - {2B4ABFF8-D1FD-4845-88C9-1F3C0A6512BF}.Debug|x86.Build.0 = Debug|Win32 {2B4ABFF8-D1FD-4845-88C9-1F3C0A6512BF}.Release|x64.ActiveCfg = Release|x64 {2B4ABFF8-D1FD-4845-88C9-1F3C0A6512BF}.Release|x64.Build.0 = Release|x64 - {2B4ABFF8-D1FD-4845-88C9-1F3C0A6512BF}.Release|x86.ActiveCfg = Release|Win32 - {2B4ABFF8-D1FD-4845-88C9-1F3C0A6512BF}.Release|x86.Build.0 = Release|Win32 {7E99172D-7FF2-4CB6-B736-AC9B76ED412A}.Debug|x64.ActiveCfg = Debug|x64 {7E99172D-7FF2-4CB6-B736-AC9B76ED412A}.Debug|x64.Build.0 = Debug|x64 - {7E99172D-7FF2-4CB6-B736-AC9B76ED412A}.Debug|x86.ActiveCfg = Debug|Win32 - {7E99172D-7FF2-4CB6-B736-AC9B76ED412A}.Debug|x86.Build.0 = Debug|Win32 {7E99172D-7FF2-4CB6-B736-AC9B76ED412A}.Release|x64.ActiveCfg = Release|x64 {7E99172D-7FF2-4CB6-B736-AC9B76ED412A}.Release|x64.Build.0 = Release|x64 - {7E99172D-7FF2-4CB6-B736-AC9B76ED412A}.Release|x86.ActiveCfg = Release|Win32 - {7E99172D-7FF2-4CB6-B736-AC9B76ED412A}.Release|x86.Build.0 = Release|Win32 {868474FD-35F6-4400-8EED-30A33E7521D4}.Debug|x64.ActiveCfg = Debug|x64 {868474FD-35F6-4400-8EED-30A33E7521D4}.Debug|x64.Build.0 = Debug|x64 - {868474FD-35F6-4400-8EED-30A33E7521D4}.Debug|x86.ActiveCfg = Debug|Win32 - {868474FD-35F6-4400-8EED-30A33E7521D4}.Debug|x86.Build.0 = Debug|Win32 {868474FD-35F6-4400-8EED-30A33E7521D4}.Release|x64.ActiveCfg = Release|x64 {868474FD-35F6-4400-8EED-30A33E7521D4}.Release|x64.Build.0 = Release|x64 - {868474FD-35F6-4400-8EED-30A33E7521D4}.Release|x86.ActiveCfg = Release|Win32 - {868474FD-35F6-4400-8EED-30A33E7521D4}.Release|x86.Build.0 = Release|Win32 {51201D5E-D939-4854-AE9D-008F03FF518E}.Debug|x64.ActiveCfg = Debug|x64 {51201D5E-D939-4854-AE9D-008F03FF518E}.Debug|x64.Build.0 = Debug|x64 - {51201D5E-D939-4854-AE9D-008F03FF518E}.Debug|x86.ActiveCfg = Debug|Win32 - {51201D5E-D939-4854-AE9D-008F03FF518E}.Debug|x86.Build.0 = Debug|Win32 {51201D5E-D939-4854-AE9D-008F03FF518E}.Release|x64.ActiveCfg = Release|x64 {51201D5E-D939-4854-AE9D-008F03FF518E}.Release|x64.Build.0 = Release|x64 - {51201D5E-D939-4854-AE9D-008F03FF518E}.Release|x86.ActiveCfg = Release|Win32 - {51201D5E-D939-4854-AE9D-008F03FF518E}.Release|x86.Build.0 = Release|Win32 + {542007E3-BE0D-4B0D-A6B0-AA8813E2558D}.Debug|x64.ActiveCfg = Debug|x64 + {542007E3-BE0D-4B0D-A6B0-AA8813E2558D}.Debug|x64.Build.0 = Debug|x64 + {542007E3-BE0D-4B0D-A6B0-AA8813E2558D}.Release|x64.ActiveCfg = Release|x64 + {542007E3-BE0D-4B0D-A6B0-AA8813E2558D}.Release|x64.Build.0 = Release|x64 EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution - SolutionGuid = {8AA72EDA-2CD4-4564-B1E4-688B760EEEE9} - SolutionGuid = {8607C0F4-F33D-41B8-8D51-18E366A0F8DF} SolutionGuid = {58AAB032-7274-49BD-845E-5EF4DBB69B70} EndGlobalSection EndGlobal diff --git a/build_msvc/bitcoin_config.h b/build_msvc/bitcoin_config.h.in similarity index 54% rename from build_msvc/bitcoin_config.h rename to build_msvc/bitcoin_config.h.in index aba5607eb6..91b9c694f2 100644 --- a/build_msvc/bitcoin_config.h +++ b/build_msvc/bitcoin_config.h.in @@ -5,20 +5,17 @@ #ifndef BITCOIN_BITCOIN_CONFIG_H #define BITCOIN_BITCOIN_CONFIG_H -/* Define if building universal (internal helper macro) */ -/* #undef AC_APPLE_UNIVERSAL_BUILD */ - /* Version Build */ -#define CLIENT_VERSION_BUILD 0 +#define CLIENT_VERSION_BUILD $ /* Version is release */ -#define CLIENT_VERSION_IS_RELEASE false +#define CLIENT_VERSION_IS_RELEASE $ /* Major version */ -#define CLIENT_VERSION_MAJOR 21 +#define CLIENT_VERSION_MAJOR $ /* Minor version */ -#define CLIENT_VERSION_MINOR 99 +#define CLIENT_VERSION_MINOR $ /* Copyright holder(s) before %s replacement */ #define COPYRIGHT_HOLDERS "The %s developers" @@ -30,7 +27,7 @@ #define COPYRIGHT_HOLDERS_SUBSTITUTION "Bitcoin Core" /* Copyright year */ -#define COPYRIGHT_YEAR 2019 +#define COPYRIGHT_YEAR $ /* Define to 1 to enable wallet functions */ #define ENABLE_WALLET 1 @@ -47,26 +44,14 @@ /* define if the Boost library is available */ #define HAVE_BOOST /**/ -/* define if the Boost::Filesystem library is available */ -#define HAVE_BOOST_FILESYSTEM /**/ - /* define if external signer support is enabled (requires Boost::Process) */ #define ENABLE_EXTERNAL_SIGNER /**/ -/* define if the Boost::System library is available */ -#define HAVE_BOOST_SYSTEM /**/ - -/* define if the Boost::Unit_Test_Framework library is available */ -#define HAVE_BOOST_UNIT_TEST_FRAMEWORK /**/ - -/* Define to 1 if you have the header file. */ -/* #undef HAVE_BYTESWAP_H */ - /* Define this symbol if the consensus lib has been built */ #define HAVE_CONSENSUS_LIB 1 -/* define if the compiler supports basic C++11 syntax */ -#define HAVE_CXX11 1 +/* define if the compiler supports basic C++17 syntax */ +#define HAVE_CXX17 1 /* Define to 1 if you have the declaration of `be16toh', and to 0 if you don't. */ @@ -144,37 +129,12 @@ don't. */ #define HAVE_DECL_STRNLEN 1 -/* Define to 1 if you have the header file. */ -/* #undef HAVE_DLFCN_H */ - -/* Define to 1 if you have the header file. */ -/* #undef HAVE_ENDIAN_H */ - -/* Define to 1 if the system has the `dllexport' function attribute */ -#define HAVE_FUNC_ATTRIBUTE_DLLEXPORT 1 - -/* Define to 1 if the system has the `dllimport' function attribute */ -#define HAVE_FUNC_ATTRIBUTE_DLLIMPORT 1 - -/* Define to 1 if the system has the `visibility' function attribute */ -#define HAVE_FUNC_ATTRIBUTE_VISIBILITY 1 - -/* Define this symbol if the BSD getentropy system call is available */ -/* #undef HAVE_GETENTROPY */ - -/* Define this symbol if the BSD getentropy system call is available with - sys/random.h */ -/* #undef HAVE_GETENTROPY_RAND */ +/* Define if the dllexport attribute is supported. */ +#define HAVE_DLLEXPORT_ATTRIBUTE 1 /* Define to 1 if you have the header file. */ #define HAVE_INTTYPES_H 1 -/* Define this symbol if you have malloc_info */ -/* #undef HAVE_MALLOC_INFO */ - -/* Define this symbol if you have mallopt with M_ARENA_MAX */ -/* #undef HAVE_MALLOPT_ARENA_MAX */ - /* Define to 1 if you have the header file. */ #define HAVE_MEMORY_H 1 @@ -187,18 +147,6 @@ /* Define to 1 if you have the header file. */ #define HAVE_MINIUPNPC_UPNPERRORS_H 1 -/* Define this symbol if you have MSG_DONTWAIT */ -/* #undef HAVE_MSG_DONTWAIT */ - -/* Define this symbol if you have MSG_NOSIGNAL */ -/* #undef HAVE_MSG_NOSIGNAL */ - -/* Define if you have POSIX threads libraries and header files. */ -//#define HAVE_PTHREAD 1 - -/* Have PTHREAD_PRIO_INHERIT. */ -//#define HAVE_PTHREAD_PRIO_INHERIT 1 - /* Define to 1 if you have the header file. */ #define HAVE_STDINT_H 1 @@ -208,69 +156,32 @@ /* Define to 1 if you have the header file. */ #define HAVE_STDLIB_H 1 -/* Define to 1 if you have the `strerror_r' function. */ -/* #undef HAVE_STRERROR_R */ - /* Define to 1 if you have the header file. */ #define HAVE_STRINGS_H 1 /* Define to 1 if you have the header file. */ #define HAVE_STRING_H 1 -/* Define this symbol if the BSD sysctl(KERN_ARND) is available */ -/* #undef HAVE_SYSCTL_ARND */ - -/* Define to 1 if you have the header file. */ -/* #undef HAVE_SYS_ENDIAN_H */ - -/* Define this symbol if the Linux getrandom system call is available */ -/* #undef HAVE_SYS_GETRANDOM */ - -/* Define to 1 if you have the header file. */ -/* #undef HAVE_SYS_PRCTL_H */ - -/* Define to 1 if you have the header file. */ -/* #undef HAVE_SYS_SELECT_H */ - /* Define to 1 if you have the header file. */ #define HAVE_SYS_STAT_H 1 /* Define to 1 if you have the header file. */ #define HAVE_SYS_TYPES_H 1 -/* Define to 1 if you have the header file. */ -//#define HAVE_UNISTD_H 1 - -/* Define if the visibility attribute is supported. */ -#define HAVE_VISIBILITY_ATTRIBUTE 1 - -/* Define to the sub-directory where libtool stores uninstalled libraries. */ -#define LT_OBJDIR ".libs/" - /* Define to the address where bug reports for this package should be sent. */ -#define PACKAGE_BUGREPORT "https://github.com/bitcoin/bitcoin/issues" +#define PACKAGE_BUGREPORT "https://github.com/ElementsProject/elements/issues" /* Define to the full name of this package. */ -#define PACKAGE_NAME "Bitcoin Core" +#define PACKAGE_NAME "Elements Core" /* Define to the full name and version of this package. */ -#define PACKAGE_STRING "Bitcoin Core 21.99.0" - -/* Define to the one symbol short name of this package. */ -#define PACKAGE_TARNAME "bitcoin" +#define PACKAGE_STRING $ /* Define to the home page for this package. */ -#define PACKAGE_URL "https://bitcoincore.org/" +#define PACKAGE_URL "https://elementsproject.org/" /* Define to the version of this package. */ -#define PACKAGE_VERSION "21.99.0" - -/* Define to necessary symbol if this constant uses a non-standard name on - your system. */ -/* #undef PTHREAD_CREATE_JOINABLE */ - -/* Define this symbol if the qt platform is cocoa */ -/* #undef QT_QPA_PLATFORM_COCOA */ +#define PACKAGE_VERSION $ /* Define this symbol if the minimal qt platform exists */ #define QT_QPA_PLATFORM_MINIMAL 1 @@ -278,54 +189,9 @@ /* Define this symbol if the qt platform is windows */ #define QT_QPA_PLATFORM_WINDOWS 1 -/* Define this symbol if the qt platform is xcb */ -/* #undef QT_QPA_PLATFORM_XCB */ - /* Define this symbol if qt plugins are static */ #define QT_STATICPLUGIN 1 -/* Define to 1 if you have the ANSI C header files. */ -#define STDC_HEADERS 1 - -/* Define to 1 if strerror_r returns char *. */ -/* #undef STRERROR_R_CHAR_P */ - -/* Define this symbol to build in assembly routines */ -//#define USE_ASM 1 - -/* Define if dbus support should be compiled in */ -/* #undef USE_DBUS */ - -/* Define if QR support should be compiled in */ -//#define USE_QRCODE 1 - -/* UPnP support not compiled if undefined, otherwise value (0 or 1) determines - default state */ -//#define USE_UPNP 0 - -/* Define WORDS_BIGENDIAN to 1 if your processor stores words with the most - significant byte first (like Motorola and SPARC, unlike Intel). */ -#if defined AC_APPLE_UNIVERSAL_BUILD -# if defined __BIG_ENDIAN__ -# define WORDS_BIGENDIAN 1 -# endif -#else -# ifndef WORDS_BIGENDIAN -/* # undef WORDS_BIGENDIAN */ -# endif -#endif - -/* Enable large inode numbers on Mac OS X 10.5. */ -#ifndef _DARWIN_USE_64_BIT_INODE -# define _DARWIN_USE_64_BIT_INODE 1 -#endif - -/* Number of bits in a file offset, on hosts where this is settable. */ -#define _FILE_OFFSET_BITS 64 - -/* Define for large files, on AIX-style hosts. */ -/* #undef _LARGE_FILES */ - /* Windows Universal Platform constraints */ #if !defined(WINAPI_FAMILY) || (WINAPI_FAMILY == WINAPI_FAMILY_DESKTOP_APP) /* Either a desktop application without API restrictions, or and older system diff --git a/build_msvc/bitcoind/bitcoind.vcxproj b/build_msvc/bitcoind/bitcoind.vcxproj index c2c32af838..b1204d0d5d 100644 --- a/build_msvc/bitcoind/bitcoind.vcxproj +++ b/build_msvc/bitcoind/bitcoind.vcxproj @@ -24,7 +24,7 @@ {6190199c-6cf4-4dad-bfbd-93fa72a760c1} - + {460fee33-1fe1-483f-b3bf-931ff8e969a5} @@ -57,6 +57,8 @@ + + + diff --git a/build_msvc/common.init.vcxproj b/build_msvc/common.init.vcxproj index 6ea018d846..0cbe2effd5 100644 --- a/build_msvc/common.init.vcxproj +++ b/build_msvc/common.init.vcxproj @@ -14,7 +14,6 @@ true true $(Configuration) - x86-windows-static x64-windows-static @@ -35,14 +34,6 @@ Debug x64 - - Release - Win32 - - - Debug - Win32 - @@ -100,7 +91,7 @@ 4018;4244;4267;4334;4715;4805;4834 true _SILENCE_CXX17_CODECVT_HEADER_DEPRECATION_WARNING;_SILENCE_CXX17_OLD_ALLOCATOR_MEMBERS_DEPRECATION_WARNING;ZMQ_STATIC;NOMINMAX;WIN32;HAVE_CONFIG_H;_CRT_SECURE_NO_WARNINGS;_SCL_SECURE_NO_WARNINGS;_CONSOLE;_WIN32_WINNT=0x0601;_WIN32_IE=0x0501;WIN32_LEAN_AND_MEAN;%(PreprocessorDefinitions) - ..\..\src;..\..\src\univalue\include;..\..\src\secp256k1\include;..\..\src\leveldb\include;..\..\src\leveldb\helpers\memenv;%(AdditionalIncludeDirectories) + ..\..\src;..\..\src\minisketch\include;..\..\src\univalue\include;..\..\src\secp256k1\include;..\..\src\leveldb\include;..\..\src\leveldb\helpers\memenv;%(AdditionalIncludeDirectories) Console diff --git a/build_msvc/common.qt.init.vcxproj b/build_msvc/common.qt.init.vcxproj index ce66a7ab34..cc8063e545 100644 --- a/build_msvc/common.qt.init.vcxproj +++ b/build_msvc/common.qt.init.vcxproj @@ -2,7 +2,7 @@ - C:\Qt5.12.11_x64_static_vs2019_16101 + C:\Qt_static $(QtBaseDir)\plugins $(QtBaseDir)\lib $(QtBaseDir)\include diff --git a/build_msvc/libbitcoin_server/libbitcoin_server.vcxproj.in b/build_msvc/libbitcoin_node/libbitcoin_node.vcxproj.in similarity index 100% rename from build_msvc/libbitcoin_server/libbitcoin_server.vcxproj.in rename to build_msvc/libbitcoin_node/libbitcoin_node.vcxproj.in diff --git a/build_msvc/libbitcoin_qt/libbitcoin_qt.vcxproj b/build_msvc/libbitcoin_qt/libbitcoin_qt.vcxproj index 96bb584375..a64ae881f2 100644 --- a/build_msvc/libbitcoin_qt/libbitcoin_qt.vcxproj +++ b/build_msvc/libbitcoin_qt/libbitcoin_qt.vcxproj @@ -24,6 +24,7 @@ + @@ -78,6 +79,7 @@ + @@ -138,20 +140,6 @@ - - - _X86_;%(PreprocessorDefinitions) - $(QtIncludes);$(GeneratedFilesOutDir)\..;%(AdditionalIncludeDirectories) - - - - - - _X86_;%(PreprocessorDefinitions) - $(QtIncludes);$(GeneratedFilesOutDir)\..;%(AdditionalIncludeDirectories) - - - diff --git a/build_msvc/libminisketch/libminisketch.vcxproj b/build_msvc/libminisketch/libminisketch.vcxproj new file mode 100644 index 0000000000..b34593fe5c --- /dev/null +++ b/build_msvc/libminisketch/libminisketch.vcxproj @@ -0,0 +1,38 @@ + + + + + {542007E3-BE0D-4B0D-A6B0-AA8813E2558D} + + + StaticLibrary + + + + + + + + + + + + + + + + + + + + + + + 4060;4065;4146;4244;4267;4554 + HAVE_CLMUL;DISABLE_DEFAULT_FIELDS;ENABLE_FIELD_32;%(PreprocessorDefinitions) + + + + + + diff --git a/build_msvc/libsecp256k1_config.h b/build_msvc/libsecp256k1_config.h index 5978b9a0d9..57f2f144ff 100644 --- a/build_msvc/libsecp256k1_config.h +++ b/build_msvc/libsecp256k1_config.h @@ -29,4 +29,4 @@ #define ECMULT_GEN_PREC_BITS 4 #define ECMULT_WINDOW_SIZE 15 -#endif /* BITCOIN_LIBSECP256K1_CONFIG_H */ +#endif // BITCOIN_LIBSECP256K1_CONFIG_H diff --git a/build_msvc/msvc-autogen.py b/build_msvc/msvc-autogen.py index d99b17d381..2a70cd9332 100755 --- a/build_msvc/msvc-autogen.py +++ b/build_msvc/msvc-autogen.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2016-2019 The Bitcoin Core developers +# Copyright (c) 2016-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -9,13 +9,13 @@ import argparse from shutil import copyfile SOURCE_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', 'src')) -DEFAULT_PLATFORM_TOOLSET = R'v141' +DEFAULT_PLATFORM_TOOLSET = R'v142' libs = [ 'libbitcoin_cli', 'libbitcoin_common', 'libbitcoin_crypto', - 'libbitcoin_server', + 'libbitcoin_node', 'libbitcoin_util', 'libbitcoin_wallet_tool', 'libbitcoin_wallet', @@ -57,6 +57,41 @@ def set_common_properties(toolset): with open(os.path.join(SOURCE_DIR, '../build_msvc/common.init.vcxproj'), 'w', encoding='utf-8',newline='\n') as wfile: wfile.write(s) +def parse_config_into_btc_config(): + def find_between( s, first, last ): + try: + start = s.index( first ) + len( first ) + end = s.index( last, start ) + return s[start:end] + except ValueError: + return "" + + config_info = [] + with open(os.path.join(SOURCE_DIR,'../configure.ac'), encoding="utf8") as f: + for line in f: + if line.startswith("define"): + config_info.append(find_between(line, "(_", ")")) + + config_info = [c for c in config_info if not c.startswith("COPYRIGHT_HOLDERS")] + + config_dict = dict(item.split(", ") for item in config_info) + config_dict["PACKAGE_VERSION"] = f"\"{config_dict['CLIENT_VERSION_MAJOR']}.{config_dict['CLIENT_VERSION_MINOR']}.{config_dict['CLIENT_VERSION_BUILD']}\"" + version = config_dict["PACKAGE_VERSION"].strip('"') + config_dict["PACKAGE_STRING"] = f"\"Bitcoin Core {version}\"" + + with open(os.path.join(SOURCE_DIR,'../build_msvc/bitcoin_config.h.in'), "r", encoding="utf8") as template_file: + template = template_file.readlines() + + for index, line in enumerate(template): + header = "" + if line.startswith("#define"): + header = line.split(" ")[1] + if header in config_dict: + template[index] = line.replace("$", f"{config_dict[header]}") + + with open(os.path.join(SOURCE_DIR,'../build_msvc/bitcoin_config.h'), "w", encoding="utf8") as btc_config: + btc_config.writelines(template) + def main(): parser = argparse.ArgumentParser(description='Bitcoin-core msbuild configuration initialiser.') parser.add_argument('-toolset', nargs='?',help='Optionally sets the msbuild platform toolset, e.g. v142 for Visual Studio 2019.' @@ -79,6 +114,7 @@ def main(): with open(vcxproj_filename, 'w', encoding='utf-8') as vcxproj_file: vcxproj_file.write(vcxproj_in_file.read().replace( '@SOURCE_FILES@\n', content)) + parse_config_into_btc_config() copyfile(os.path.join(SOURCE_DIR,'../build_msvc/bitcoin_config.h'), os.path.join(SOURCE_DIR, 'config/bitcoin-config.h')) copyfile(os.path.join(SOURCE_DIR,'../build_msvc/libsecp256k1_config.h'), os.path.join(SOURCE_DIR, 'secp256k1/src/libsecp256k1-config.h')) diff --git a/build_msvc/test_bitcoin-qt/test_bitcoin-qt.vcxproj b/build_msvc/test_bitcoin-qt/test_bitcoin-qt.vcxproj index 1d2c86b7ac..d3be693e99 100644 --- a/build_msvc/test_bitcoin-qt/test_bitcoin-qt.vcxproj +++ b/build_msvc/test_bitcoin-qt/test_bitcoin-qt.vcxproj @@ -8,6 +8,7 @@ $(SolutionDir)$(Platform)\$(Configuration)\ + @@ -39,7 +40,7 @@ {2b4abff8-d1fd-4845-88c9-1f3c0a6512bf} - + {460fee33-1fe1-483f-b3bf-931ff8e969a5} diff --git a/build_msvc/test_bitcoin/test_bitcoin.vcxproj b/build_msvc/test_bitcoin/test_bitcoin.vcxproj index 5c4b777d51..4182448ec3 100644 --- a/build_msvc/test_bitcoin/test_bitcoin.vcxproj +++ b/build_msvc/test_bitcoin/test_bitcoin.vcxproj @@ -16,8 +16,12 @@ + + + {542007e3-be0d-4b0d-a6b0-aa8813e2558d} + {2b384fa8-9ee1-4544-93cb-0d733c25e8ce} @@ -30,7 +34,7 @@ {6190199c-6cf4-4dad-bfbd-93fa72a760c1} - + {460fee33-1fe1-483f-b3bf-931ff8e969a5} diff --git a/build_msvc/testconsensus/testconsensus.cpp b/build_msvc/testconsensus/testconsensus.cpp deleted file mode 100644 index f3c8517130..0000000000 --- a/build_msvc/testconsensus/testconsensus.cpp +++ /dev/null @@ -1,54 +0,0 @@ -// Copyright (c) 2018-2020 The Bitcoin Core developers -// Distributed under the MIT software license, see the accompanying -// file COPYING or http://www.opensource.org/licenses/mit-license.php. - -#include - -// bitcoin includes. -#include <..\src\script\bitcoinconsensus.h> -#include <..\src\primitives\transaction.h> -#include <..\src\script\script.h> -#include <..\src\streams.h> -#include <..\src\version.h> - -CMutableTransaction BuildSpendingTransaction(const CScript& scriptSig, const CScriptWitness& scriptWitness, int nValue = 0) -{ - CMutableTransaction txSpend; - txSpend.nVersion = 1; - txSpend.nLockTime = 0; - txSpend.vin.resize(1); - txSpend.vout.resize(1); - txSpend.vin[0].scriptWitness = scriptWitness; - txSpend.vin[0].prevout.hash = uint256(); - txSpend.vin[0].prevout.n = 0; - txSpend.vin[0].scriptSig = scriptSig; - txSpend.vin[0].nSequence = CTxIn::SEQUENCE_FINAL; - txSpend.vout[0].scriptPubKey = CScript(); - txSpend.vout[0].nValue = nValue; - - return txSpend; -} - -int main() -{ - std::cout << "bitcoinconsensus version: " << bitcoinconsensus_version() << std::endl; - - CScript pubKeyScript; - pubKeyScript << OP_1 << OP_0 << OP_1; - - int amount = 0; // 600000000; - - CScript scriptSig; - CScriptWitness scriptWitness; - CTransaction vanillaSpendTx = BuildSpendingTransaction(scriptSig, scriptWitness, amount); - CDataStream stream(SER_NETWORK, PROTOCOL_VERSION); - stream << vanillaSpendTx; - - bitcoinconsensus_error err; - auto op0Result = bitcoinconsensus_verify_script_with_amount(pubKeyScript.data(), pubKeyScript.size(), amount, stream.data(), stream.size(), 0, bitcoinconsensus_SCRIPT_FLAGS_VERIFY_ALL, &err); - std::cout << "Op0 result: " << op0Result << ", error code " << err << std::endl; - - getchar(); - - return 0; -} diff --git a/build_msvc/testconsensus/testconsensus.vcxproj b/build_msvc/testconsensus/testconsensus.vcxproj deleted file mode 100644 index 776c40920a..0000000000 --- a/build_msvc/testconsensus/testconsensus.vcxproj +++ /dev/null @@ -1,28 +0,0 @@ - - - - - {E78473E9-B850-456C-9120-276301E04C06} - - - Application - $(SolutionDir)$(Platform)\$(Configuration)\ - - - - - - - {2B384FA8-9EE1-4544-93CB-0D733C25E8CE} - - - {B53A5535-EE9D-4C6F-9A26-F79EE3BC3754} - - - {BB493552-3B8C-4A8C-BF69-A6E7A51D2EA6} - - - - - - diff --git a/build_msvc/vcpkg.json b/build_msvc/vcpkg.json index 42b9a5d16f..d8753ec21f 100644 --- a/build_msvc/vcpkg.json +++ b/build_msvc/vcpkg.json @@ -3,7 +3,6 @@ "version-string": "1", "dependencies": [ "berkeleydb", - "boost-filesystem", "boost-multi-index", "boost-process", "boost-signals2", diff --git a/ci/lint/04_install.sh b/ci/lint/04_install.sh index 2c63a9efac..1518c033f4 100755 --- a/ci/lint/04_install.sh +++ b/ci/lint/04_install.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # -# Copyright (c) 2018-2020 The Bitcoin Core developers +# Copyright (c) 2018-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -8,15 +8,15 @@ export LC_ALL=C ${CI_RETRY_EXE} apt-get update ${CI_RETRY_EXE} apt-get install -y clang-format-9 python3-pip curl git gawk jq -update-alternatives --install /usr/bin/clang-format clang-format $(which clang-format-9 ) 100 -update-alternatives --install /usr/bin/clang-format-diff clang-format-diff $(which clang-format-diff-9) 100 +update-alternatives --install /usr/bin/clang-format clang-format "$(which clang-format-9 )" 100 +update-alternatives --install /usr/bin/clang-format-diff clang-format-diff "$(which clang-format-diff-9)" 100 ${CI_RETRY_EXE} pip3 install codespell==2.0.0 ${CI_RETRY_EXE} pip3 install flake8==3.8.3 -${CI_RETRY_EXE} pip3 install yq -${CI_RETRY_EXE} pip3 install mypy==0.781 +${CI_RETRY_EXE} pip3 install mypy==0.910 +${CI_RETRY_EXE} pip3 install pyzmq==22.3.0 ${CI_RETRY_EXE} pip3 install vulture==2.3 -SHELLCHECK_VERSION=v0.7.2 +SHELLCHECK_VERSION=v0.8.0 curl -sL "https://github.com/koalaman/shellcheck/releases/download/${SHELLCHECK_VERSION}/shellcheck-${SHELLCHECK_VERSION}.linux.x86_64.tar.xz" | tar --xz -xf - --directory /tmp/ export PATH="/tmp/shellcheck-${SHELLCHECK_VERSION}:${PATH}" diff --git a/ci/lint/06_script.sh b/ci/lint/06_script.sh index e38cfe8eef..6761c0c86f 100755 --- a/ci/lint/06_script.sh +++ b/ci/lint/06_script.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # -# Copyright (c) 2018-2020 The Bitcoin Core developers +# Copyright (c) 2018-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -8,8 +8,8 @@ export LC_ALL=C GIT_HEAD=$(git rev-parse HEAD) if [ -n "$CIRRUS_PR" ]; then - COMMIT_RANGE="$CIRRUS_BASE_SHA..$GIT_HEAD" - test/lint/commit-script-check.sh $COMMIT_RANGE + COMMIT_RANGE="${CIRRUS_BASE_SHA}..$GIT_HEAD" + test/lint/commit-script-check.sh "$COMMIT_RANGE" fi export COMMIT_RANGE @@ -17,17 +17,26 @@ export COMMIT_RANGE # check with -r to not have to fetch all the remotes. test/lint/git-subtree-check.sh src/crypto/ctaes test/lint/git-subtree-check.sh src/secp256k1 +test/lint/git-subtree-check.sh src/minisketch test/lint/git-subtree-check.sh src/univalue test/lint/git-subtree-check.sh src/leveldb test/lint/git-subtree-check.sh src/crc32c test/lint/check-doc.py test/lint/lint-all.sh -if [ "$CIRRUS_REPO_FULL_NAME" = "bitcoin/bitcoin" ] && [ -n "$CIRRUS_CRON" ]; then - git log --merges --before="2 days ago" -1 --format='%H' > ./contrib/verify-commits/trusted-sha512-root-commit - ${CI_RETRY_EXE} gpg --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys $( ./contrib/verify-commits/trusted-sha512-root-commit + git log HEAD~10 -1 --format='%H' > ./contrib/verify-commits/trusted-git-root + mapfile -t KEYS < contrib/verify-commits/trusted-keys + ${CI_RETRY_EXE} gpg --keyserver hkps://keys.openpgp.org --recv-keys "${KEYS[@]}" && + ./contrib/verify-commits/verify-commits.py; fi -echo -git log --no-merges --oneline $COMMIT_RANGE +if [ -n "$COMMIT_RANGE" ]; then + echo + git --no-pager log --no-merges --oneline "$COMMIT_RANGE" +fi diff --git a/ci/test/00_setup_env.sh b/ci/test/00_setup_env.sh index 8a9d808f5d..e806683128 100755 --- a/ci/test/00_setup_env.sh +++ b/ci/test/00_setup_env.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # -# Copyright (c) 2019-2020 The Bitcoin Core developers +# Copyright (c) 2019-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/ci/test/00_setup_env_android.sh b/ci/test/00_setup_env_android.sh index f78a84eeac..522a5497fa 100755 --- a/ci/test/00_setup_env_android.sh +++ b/ci/test/00_setup_env_android.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # -# Copyright (c) 2019-2020 The Bitcoin Core developers +# Copyright (c) 2019-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -16,10 +16,10 @@ export RUN_FUNCTIONAL_TESTS=false export ANDROID_API_LEVEL=28 export ANDROID_BUILD_TOOLS_VERSION=28.0.3 -export ANDROID_NDK_VERSION=21.1.6352462 +export ANDROID_NDK_VERSION=23.1.7779620 export ANDROID_TOOLS_URL=https://dl.google.com/android/repository/commandlinetools-linux-6609375_latest.zip export ANDROID_HOME="${DEPENDS_DIR}/SDKs/android" export ANDROID_NDK_HOME="${ANDROID_HOME}/ndk/${ANDROID_NDK_VERSION}" export DEP_OPTS="ANDROID_SDK=${ANDROID_HOME} ANDROID_NDK=${ANDROID_NDK_HOME} ANDROID_API_LEVEL=${ANDROID_API_LEVEL} ANDROID_TOOLCHAIN_BIN=${ANDROID_NDK_HOME}/toolchains/llvm/prebuilt/linux-x86_64/bin/" -export BITCOIN_CONFIG="--disable-ccache" +export BITCOIN_CONFIG="--disable-tests --enable-gui-tests --disable-bench --disable-fuzz-binary --without-utils --without-libs --without-daemon" diff --git a/ci/test/00_setup_env_arm.sh b/ci/test/00_setup_env_arm.sh index 8d2b70e549..932be4b43d 100755 --- a/ci/test/00_setup_env_arm.sh +++ b/ci/test/00_setup_env_arm.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # -# Copyright (c) 2019-2020 The Bitcoin Core developers +# Copyright (c) 2019-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -18,11 +18,11 @@ if [ -n "$QEMU_USER_CMD" ]; then fi export CONTAINER_NAME=ci_arm_linux # Use debian to avoid 404 apt errors when cross compiling -export DOCKER_NAME_TAG="debian:buster" +export DOCKER_NAME_TAG="debian:bullseye" export USE_BUSY_BOX=true export RUN_UNIT_TESTS=true export RUN_FUNCTIONAL_TESTS=false export GOAL="install" # -Wno-psabi is to disable ABI warnings: "note: parameter passing for argument of type ... changed in GCC 7.1" # This could be removed once the ABI change warning does not show up by default -export BITCOIN_CONFIG="--enable-glibc-back-compat --enable-reduce-exports CXXFLAGS=-Wno-psabi" +export BITCOIN_CONFIG="--enable-reduce-exports CXXFLAGS=-Wno-psabi" diff --git a/ci/test/00_setup_env_i686_centos.sh b/ci/test/00_setup_env_i686_centos.sh index 4d0c7f123b..8f1cc8af29 100755 --- a/ci/test/00_setup_env_i686_centos.sh +++ b/ci/test/00_setup_env_i686_centos.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # -# Copyright (c) 2020 The Bitcoin Core developers +# Copyright (c) 2020-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/ci/test/00_setup_env_native_multiprocess.sh b/ci/test/00_setup_env_i686_multiprocess.sh similarity index 50% rename from ci/test/00_setup_env_native_multiprocess.sh rename to ci/test/00_setup_env_i686_multiprocess.sh index 549c51fb74..ec3b14fe83 100755 --- a/ci/test/00_setup_env_native_multiprocess.sh +++ b/ci/test/00_setup_env_i686_multiprocess.sh @@ -1,16 +1,18 @@ #!/usr/bin/env bash # -# Copyright (c) 2020 The Bitcoin Core developers +# Copyright (c) 2020-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. export LC_ALL=C.UTF-8 -export CONTAINER_NAME=ci_native_multiprocess +export HOST=i686-pc-linux-gnu +export CONTAINER_NAME=ci_i686_multiprocess export DOCKER_NAME_TAG=ubuntu:20.04 -export PACKAGES="cmake python3 python3-pip llvm clang" +export PACKAGES="cmake python3 python3-pip llvm clang g++-multilib" export DEP_OPTS="DEBUG=1 MULTIPROCESS=1" export GOAL="install" -export BITCOIN_CONFIG="--enable-debug CC=clang CXX=clang++" # Use clang to avoid OOM +export BITCOIN_CONFIG="--enable-debug CC='clang -m32' CXX='clang++ -m32' LDFLAGS='--rtlib=compiler-rt -lgcc_s'" export TEST_RUNNER_ENV="BITCOIND=elements-node" +export TEST_RUNNER_EXTRA="--nosandbox" export PIP_PACKAGES="lief" diff --git a/ci/test/00_setup_env_mac.sh b/ci/test/00_setup_env_mac.sh index 73ac09c1de..c4f22c8f9e 100755 --- a/ci/test/00_setup_env_mac.sh +++ b/ci/test/00_setup_env_mac.sh @@ -1,17 +1,17 @@ #!/usr/bin/env bash # -# Copyright (c) 2019-2020 The Bitcoin Core developers +# Copyright (c) 2019-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. export LC_ALL=C.UTF-8 export CONTAINER_NAME=ci_macos_cross -export DOCKER_NAME_TAG=ubuntu:20.04 # Check that Focal can cross-compile to macos (Focal is used in the gitian build as well) -export HOST=x86_64-apple-darwin18 -export PACKAGES="cmake imagemagick librsvg2-bin libz-dev libtiff-tools libtinfo5 python3-setuptools xorriso" -export XCODE_VERSION=12.1 -export XCODE_BUILD_ID=12A7403 +export DOCKER_NAME_TAG=ubuntu:20.04 # Check that Focal can cross-compile to macos +export HOST=x86_64-apple-darwin +export PACKAGES="cmake libz-dev libtinfo5 python3-setuptools xorriso" +export XCODE_VERSION=12.2 +export XCODE_BUILD_ID=12B45b export RUN_UNIT_TESTS=false export RUN_FUNCTIONAL_TESTS=false export GOAL="deploy" diff --git a/ci/test/00_setup_env_mac_host.sh b/ci/test/00_setup_env_mac_host.sh index ad94b5b6a0..012ab011a6 100755 --- a/ci/test/00_setup_env_mac_host.sh +++ b/ci/test/00_setup_env_mac_host.sh @@ -1,12 +1,12 @@ #!/usr/bin/env bash # -# Copyright (c) 2019-2020 The Bitcoin Core developers +# Copyright (c) 2019-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. export LC_ALL=C.UTF-8 -export HOST=x86_64-apple-darwin18 +export HOST=x86_64-apple-darwin export PIP_PACKAGES="zmq lief" export GOAL="install" # ELEMENTS: add -fno-stack-check to work around clang bug on macos diff --git a/ci/test/00_setup_env_native_asan.sh b/ci/test/00_setup_env_native_asan.sh index ab185b6e71..b03a7edb54 100755 --- a/ci/test/00_setup_env_native_asan.sh +++ b/ci/test/00_setup_env_native_asan.sh @@ -1,14 +1,14 @@ #!/usr/bin/env bash # -# Copyright (c) 2019-2020 The Bitcoin Core developers +# Copyright (c) 2019-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. export LC_ALL=C.UTF-8 export CONTAINER_NAME=ci_native_asan -export PACKAGES="clang llvm python3-zmq qtbase5-dev qttools5-dev-tools libevent-dev bsdmainutils libboost-dev libboost-system-dev libboost-filesystem-dev libboost-test-dev libdb5.3++-dev libminiupnpc-dev libnatpmp-dev libzmq3-dev libqrencode-dev libsqlite3-dev" -export DOCKER_NAME_TAG=ubuntu:hirsute +export PACKAGES="clang llvm python3-zmq qtbase5-dev qttools5-dev-tools libevent-dev bsdmainutils libboost-dev libdb5.3++-dev libminiupnpc-dev libnatpmp-dev libzmq3-dev libqrencode-dev libsqlite3-dev" +export DOCKER_NAME_TAG=ubuntu:22.04 export NO_DEPENDS=1 export GOAL="install" export BITCOIN_CONFIG="--enable-zmq --with-incompatible-bdb --with-gui=qt5 CPPFLAGS='-DARENA_DEBUG -DDEBUG_LOCKORDER' --with-sanitizers=address,integer,undefined CC=clang CXX=clang++" diff --git a/ci/test/00_setup_env_native_fuzz.sh b/ci/test/00_setup_env_native_fuzz.sh index ac871b17c4..13edd3ba13 100755 --- a/ci/test/00_setup_env_native_fuzz.sh +++ b/ci/test/00_setup_env_native_fuzz.sh @@ -1,19 +1,19 @@ #!/usr/bin/env bash # -# Copyright (c) 2019-2020 The Bitcoin Core developers +# Copyright (c) 2019-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. export LC_ALL=C.UTF-8 -export DOCKER_NAME_TAG="ubuntu:20.04" +export DOCKER_NAME_TAG="ubuntu:22.04" export CONTAINER_NAME=ci_native_fuzz -export PACKAGES="clang llvm python3 libevent-dev bsdmainutils libboost-dev libboost-system-dev libboost-filesystem-dev libboost-test-dev" +export PACKAGES="clang llvm python3 libevent-dev bsdmainutils libboost-dev libsqlite3-dev" export NO_DEPENDS=1 export RUN_UNIT_TESTS=false export RUN_FUNCTIONAL_TESTS=false export RUN_FUZZ_TESTS=true export FUZZ_TESTS_CONFIG="--exclude=coins_view" # work around https://github.com/bitcoin/bitcoin/issues/22233 export GOAL="install" -export BITCOIN_CONFIG="--enable-fuzz --with-sanitizers=fuzzer,address,undefined,integer CC=clang CXX=clang++" +export BITCOIN_CONFIG="--enable-fuzz --with-sanitizers=fuzzer,address,undefined,integer CC='clang -ftrivial-auto-var-init=pattern' CXX='clang++ -ftrivial-auto-var-init=pattern'" export CCACHE_SIZE=200M diff --git a/ci/test/00_setup_env_native_fuzz_with_msan.sh b/ci/test/00_setup_env_native_fuzz_with_msan.sh index d5b28ca5cf..619acc4677 100755 --- a/ci/test/00_setup_env_native_fuzz_with_msan.sh +++ b/ci/test/00_setup_env_native_fuzz_with_msan.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # -# Copyright (c) 2020 The Bitcoin Core developers +# Copyright (c) 2020-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -14,7 +14,7 @@ export MSAN_AND_LIBCXX_FLAGS="${MSAN_FLAGS} ${LIBCXX_FLAGS}" export CONTAINER_NAME="ci_native_msan" export PACKAGES="clang-9 llvm-9 cmake" -export DEP_OPTS="NO_BDB=1 NO_QT=1 CC='clang' CXX='clang++' CFLAGS='${MSAN_FLAGS}' CXXFLAGS='${MSAN_AND_LIBCXX_FLAGS}' boost_cxxflags='-std=c++17 -fvisibility=hidden -fPIC ${MSAN_AND_LIBCXX_FLAGS}' libevent_cflags='${MSAN_FLAGS}' zeromq_cxxflags='-std=c++17 ${MSAN_AND_LIBCXX_FLAGS}'" +export DEP_OPTS="NO_BDB=1 NO_QT=1 CC='clang' CXX='clang++' CFLAGS='${MSAN_FLAGS}' CXXFLAGS='${MSAN_AND_LIBCXX_FLAGS}' libevent_cflags='${MSAN_FLAGS}' sqlite_cflags='${MSAN_FLAGS}' zeromq_cxxflags='-std=c++17 ${MSAN_AND_LIBCXX_FLAGS}'" export GOAL="install" export BITCOIN_CONFIG="--enable-fuzz --with-sanitizers=fuzzer,memory --with-asm=no --prefix=${DEPENDS_DIR}/x86_64-pc-linux-gnu/ CC=clang CXX=clang++ CFLAGS='${MSAN_FLAGS}' CXXFLAGS='${MSAN_AND_LIBCXX_FLAGS}'" export USE_MEMORY_SANITIZER="true" diff --git a/ci/test/00_setup_env_native_fuzz_with_valgrind.sh b/ci/test/00_setup_env_native_fuzz_with_valgrind.sh index 2cf672b91e..a7715a6ded 100755 --- a/ci/test/00_setup_env_native_fuzz_with_valgrind.sh +++ b/ci/test/00_setup_env_native_fuzz_with_valgrind.sh @@ -1,14 +1,14 @@ #!/usr/bin/env bash # -# Copyright (c) 2019-2020 The Bitcoin Core developers +# Copyright (c) 2019-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. export LC_ALL=C.UTF-8 -export DOCKER_NAME_TAG="ubuntu:20.04" +export DOCKER_NAME_TAG="ubuntu:22.04" export CONTAINER_NAME=ci_native_fuzz_valgrind -export PACKAGES="clang llvm python3 libevent-dev bsdmainutils libboost-dev libboost-system-dev libboost-filesystem-dev libboost-test-dev valgrind" +export PACKAGES="clang llvm python3 libevent-dev bsdmainutils libboost-dev libsqlite3-dev valgrind" export NO_DEPENDS=1 export RUN_UNIT_TESTS=false export RUN_FUNCTIONAL_TESTS=false diff --git a/ci/test/00_setup_env_native_msan.sh b/ci/test/00_setup_env_native_msan.sh index 7bcf9f23a2..b8418106c6 100755 --- a/ci/test/00_setup_env_native_msan.sh +++ b/ci/test/00_setup_env_native_msan.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # -# Copyright (c) 2020 The Bitcoin Core developers +# Copyright (c) 2020-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -15,7 +15,7 @@ export BDB_PREFIX="${BASE_ROOT_DIR}/db4" export CONTAINER_NAME="ci_native_msan" export PACKAGES="clang-9 llvm-9 cmake" -export DEP_OPTS="NO_BDB=1 NO_QT=1 CC='clang' CXX='clang++' CFLAGS='${MSAN_FLAGS}' CXXFLAGS='${MSAN_AND_LIBCXX_FLAGS}' boost_cxxflags='-std=c++17 -fvisibility=hidden -fPIC ${MSAN_AND_LIBCXX_FLAGS}' libevent_cflags='${MSAN_FLAGS}' sqlite_cflags='${MSAN_FLAGS}' zeromq_cxxflags='-std=c++17 ${MSAN_AND_LIBCXX_FLAGS}'" +export DEP_OPTS="NO_BDB=1 NO_QT=1 CC='clang' CXX='clang++' CFLAGS='${MSAN_FLAGS}' CXXFLAGS='${MSAN_AND_LIBCXX_FLAGS}' libevent_cflags='${MSAN_FLAGS}' sqlite_cflags='${MSAN_FLAGS}' zeromq_cxxflags='-std=c++17 ${MSAN_AND_LIBCXX_FLAGS}'" export GOAL="install" export BITCOIN_CONFIG="--enable-wallet --with-sanitizers=memory --with-asm=no --prefix=${DEPENDS_DIR}/x86_64-pc-linux-gnu/ CC=clang CXX=clang++ CFLAGS='${MSAN_FLAGS}' CXXFLAGS='${MSAN_AND_LIBCXX_FLAGS}' BDB_LIBS='-L${BDB_PREFIX}/lib -ldb_cxx-4.8' BDB_CFLAGS='-I${BDB_PREFIX}/include'" export USE_MEMORY_SANITIZER="true" diff --git a/ci/test/00_setup_env_native_nowallet.sh b/ci/test/00_setup_env_native_nowallet.sh deleted file mode 100755 index d167c9198a..0000000000 --- a/ci/test/00_setup_env_native_nowallet.sh +++ /dev/null @@ -1,14 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright (c) 2019-2020 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. - -export LC_ALL=C.UTF-8 - -export CONTAINER_NAME=ci_native_nowallet -export DOCKER_NAME_TAG=ubuntu:18.04 # Use bionic to have one config run the tests in python3.6, see doc/dependencies.md -export PACKAGES="python3-zmq clang-5.0 llvm-5.0" # Use clang-5 to test C++17 compatibility, see doc/dependencies.md -export DEP_OPTS="NO_WALLET=1" -export GOAL="install" -export BITCOIN_CONFIG="--enable-glibc-back-compat --enable-reduce-exports CC=clang-5.0 CXX=clang++-5.0" diff --git a/ci/test/00_setup_env_native_nowallet_libbitcoinkernel.sh b/ci/test/00_setup_env_native_nowallet_libbitcoinkernel.sh new file mode 100755 index 0000000000..89d6256298 --- /dev/null +++ b/ci/test/00_setup_env_native_nowallet_libbitcoinkernel.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash +# +# Copyright (c) 2019-2021 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +export LC_ALL=C.UTF-8 + +export CONTAINER_NAME=ci_native_nowallet_libbitcoinkernel +export DOCKER_NAME_TAG=ubuntu:18.04 # Use bionic to have one config run the tests in python3.6, see doc/dependencies.md +export PACKAGES="python3-zmq clang-7 llvm-7 libc++abi-7-dev libc++-7-dev" # Use clang-7 to test C++17 compatibility, see doc/dependencies.md +export DEP_OPTS="NO_WALLET=1 CC=clang-7 CXX='clang++-7 -stdlib=libc++'" +export GOAL="install" +export BITCOIN_CONFIG="--enable-reduce-exports CC=clang-7 CXX='clang++-7 -stdlib=libc++' --enable-experimental-util-chainstate" diff --git a/ci/test/00_setup_env_native_qt5.sh b/ci/test/00_setup_env_native_qt5.sh index b3e967c898..f8b366e77f 100755 --- a/ci/test/00_setup_env_native_qt5.sh +++ b/ci/test/00_setup_env_native_qt5.sh @@ -1,19 +1,19 @@ #!/usr/bin/env bash # -# Copyright (c) 2019-2020 The Bitcoin Core developers +# Copyright (c) 2019-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. export LC_ALL=C.UTF-8 export CONTAINER_NAME=ci_native_qt5 -export DOCKER_NAME_TAG=ubuntu:18.04 # Check that bionic gcc-7 can compile our c++17 and run our functional tests in python3, see doc/dependencies.md -export PACKAGES="python3-zmq qtbase5-dev qttools5-dev-tools libdbus-1-dev libharfbuzz-dev" -export DEP_OPTS="NO_QT=1 NO_UPNP=1 NO_NATPMP=1 DEBUG=1 ALLOW_HOST_PACKAGES=1" +export DOCKER_NAME_TAG=ubuntu:18.04 # Check that bionic gcc-8 can compile our C++17 and run our functional tests in python3, see doc/dependencies.md +export PACKAGES="gcc-8 g++-8 python3-zmq qtbase5-dev qttools5-dev-tools libdbus-1-dev libharfbuzz-dev" +export DEP_OPTS="NO_QT=1 NO_UPNP=1 NO_NATPMP=1 DEBUG=1 ALLOW_HOST_PACKAGES=1 CC=gcc-8 CXX=g++-8" export TEST_RUNNER_EXTRA="--previous-releases --coverage --extended --exclude feature_dbcrash" # Run extended tests so that coverage does not fail, but exclude the very slow dbcrash export RUN_UNIT_TESTS_SEQUENTIAL="true" export RUN_UNIT_TESTS="false" export GOAL="install" -export PREVIOUS_RELEASES_TO_DOWNLOAD="v0.15.2 v0.16.3 v0.17.2 v0.18.1 v0.19.1 v0.20.1" -export BITCOIN_CONFIG="--enable-zmq --with-libs=no --with-gui=qt5 --enable-glibc-back-compat --enable-reduce-exports ---enable-debug --disable-fuzz-binary CFLAGS=\"-g0 -O2 -funsigned-char\" CXXFLAGS=\"-g0 -O2 -funsigned-char\"" +export PREVIOUS_RELEASES_TO_DOWNLOAD="v0.15.2 v0.16.3 v0.17.2 v0.18.1 v0.19.1 v0.20.1 v0.21.0 v22.0" +export BITCOIN_CONFIG="--enable-zmq --with-libs=no --with-gui=qt5 --enable-reduce-exports \ +--enable-debug --disable-fuzz-binary CFLAGS=\"-g0 -O2 -funsigned-char\" CXXFLAGS=\"-g0 -O2 -funsigned-char\" CC=gcc-8 CXX=g++-8" diff --git a/ci/test/00_setup_env_native_tsan.sh b/ci/test/00_setup_env_native_tsan.sh index a5082bdaab..0036255caf 100755 --- a/ci/test/00_setup_env_native_tsan.sh +++ b/ci/test/00_setup_env_native_tsan.sh @@ -1,14 +1,14 @@ #!/usr/bin/env bash # -# Copyright (c) 2019-2020 The Bitcoin Core developers +# Copyright (c) 2019-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. export LC_ALL=C.UTF-8 export CONTAINER_NAME=ci_native_tsan -export DOCKER_NAME_TAG=ubuntu:hirsute +export DOCKER_NAME_TAG=ubuntu:22.04 export PACKAGES="clang llvm libc++abi-dev libc++-dev python3-zmq" export DEP_OPTS="CC=clang CXX='clang++ -stdlib=libc++'" export GOAL="install" -export BITCOIN_CONFIG="--enable-zmq --with-gui=no CPPFLAGS='-DARENA_DEBUG -DDEBUG_LOCKORDER' CXXFLAGS='-g' --with-sanitizers=thread CC=clang CXX='clang++ -stdlib=libc++'" +export BITCOIN_CONFIG="--enable-zmq CPPFLAGS='-DARENA_DEBUG -DDEBUG_LOCKORDER' CXXFLAGS='-g' --with-sanitizers=thread CC=clang CXX='clang++ -stdlib=libc++'" diff --git a/ci/test/00_setup_env_native_valgrind.sh b/ci/test/00_setup_env_native_valgrind.sh index e079a7057c..646070a84e 100755 --- a/ci/test/00_setup_env_native_valgrind.sh +++ b/ci/test/00_setup_env_native_valgrind.sh @@ -1,15 +1,16 @@ #!/usr/bin/env bash # -# Copyright (c) 2019-2020 The Bitcoin Core developers +# Copyright (c) 2019-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. export LC_ALL=C.UTF-8 +export DOCKER_NAME_TAG="ubuntu:22.04" export CONTAINER_NAME=ci_native_valgrind -export PACKAGES="valgrind clang llvm python3-zmq libevent-dev bsdmainutils libboost-dev libboost-system-dev libboost-filesystem-dev libboost-test-dev libdb5.3++-dev libminiupnpc-dev libnatpmp-dev libzmq3-dev libsqlite3-dev" +export PACKAGES="valgrind clang llvm python3-zmq libevent-dev bsdmainutils libboost-dev libdb5.3++-dev libminiupnpc-dev libnatpmp-dev libzmq3-dev libsqlite3-dev" export USE_VALGRIND=1 export NO_DEPENDS=1 -export TEST_RUNNER_EXTRA="--exclude rpc_bind" # Excluded for now, see https://github.com/bitcoin/bitcoin/issues/17765#issuecomment-602068547 +export TEST_RUNNER_EXTRA="--nosandbox --exclude feature_init,rpc_bind,feature_bind_extra" # Excluded for now, see https://github.com/bitcoin/bitcoin/issues/17765#issuecomment-602068547 export GOAL="install" export BITCOIN_CONFIG="--enable-zmq --with-incompatible-bdb --with-gui=no CC=clang CXX=clang++" # TODO enable GUI diff --git a/ci/test/00_setup_env_s390x.sh b/ci/test/00_setup_env_s390x.sh index 51a0fd9117..136edb6662 100755 --- a/ci/test/00_setup_env_s390x.sh +++ b/ci/test/00_setup_env_s390x.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # -# Copyright (c) 2019-2020 The Bitcoin Core developers +# Copyright (c) 2019-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -14,13 +14,13 @@ export PACKAGES="python3-zmq" if [ -n "$QEMU_USER_CMD" ]; then # Likely cross-compiling, so install the needed gcc and qemu-user export DPKG_ADD_ARCH="s390x" - export PACKAGES="$PACKAGES g++-s390x-linux-gnu qemu-user libc6:s390x libstdc++6:s390x libfontconfig1:s390x libxcb1:s390x" + export PACKAGES="$PACKAGES g++-s390x-linux-gnu qemu-user libc6:s390x libstdc++6:s390x" fi # Use debian to avoid 404 apt errors export CONTAINER_NAME=ci_s390x -export DOCKER_NAME_TAG="debian:buster" -export RUN_UNIT_TESTS=true +export DOCKER_NAME_TAG="debian:bookworm" export TEST_RUNNER_ENV="LC_ALL=C" +export TEST_RUNNER_EXTRA="--exclude feature_init,rpc_bind,feature_bind_extra" # Excluded for now, see https://github.com/bitcoin/bitcoin/issues/17765#issuecomment-602068547 export RUN_FUNCTIONAL_TESTS=true export GOAL="install" -export BITCOIN_CONFIG="--enable-reduce-exports --with-incompatible-bdb" +export BITCOIN_CONFIG="--enable-reduce-exports --disable-gui-tests" # GUI tests disabled for now, see https://github.com/bitcoin/bitcoin/issues/23730 diff --git a/ci/test/00_setup_env_win64.sh b/ci/test/00_setup_env_win64.sh index 4d5bde13fd..3f43bf227b 100755 --- a/ci/test/00_setup_env_win64.sh +++ b/ci/test/00_setup_env_win64.sh @@ -1,20 +1,16 @@ #!/usr/bin/env bash # -# Copyright (c) 2019-2020 The Bitcoin Core developers +# Copyright (c) 2019-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. export LC_ALL=C.UTF-8 export CONTAINER_NAME=ci_win64 -export DOCKER_NAME_TAG=ubuntu:20.04 # Check that Focal can cross-compile to win64 (Focal is used in the gitian build as well) +export DOCKER_NAME_TAG=ubuntu:22.04 # Check that Jammy can cross-compile to win64 export HOST=x86_64-w64-mingw32 export DPKG_ADD_ARCH="i386" export PACKAGES="python3 nsis g++-mingw-w64-x86-64 wine-binfmt wine64 wine32 file" export RUN_FUNCTIONAL_TESTS=false export GOAL="deploy" -export BITCOIN_CONFIG="--enable-reduce-exports --disable-gui-tests --disable-external-signer" - -# Compiler for MinGW-w64 causes false -Wreturn-type warning. -# See https://sourceforge.net/p/mingw-w64/bugs/306/ -export NO_WERROR=1 +export BITCOIN_CONFIG="--enable-reduce-exports --disable-external-signer --disable-gui-tests" diff --git a/ci/test/04_install.sh b/ci/test/04_install.sh index c46f6fdb36..01cb9d1be8 100755 --- a/ci/test/04_install.sh +++ b/ci/test/04_install.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # -# Copyright (c) 2018-2020 The Bitcoin Core developers +# Copyright (c) 2018-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -12,6 +12,7 @@ fi if [ "$CI_OS_NAME" == "macos" ]; then sudo -H pip3 install --upgrade pip + # shellcheck disable=SC2086 IN_GETOPT_BIN="/usr/local/opt/gnu-getopt/bin/getopt" ${CI_RETRY_EXE} pip3 install --user $PIP_PACKAGES fi @@ -39,6 +40,7 @@ if [ -z "$DANGER_RUN_CI_ON_HOST" ]; then systemctl restart docker fi + # shellcheck disable=SC2086 DOCKER_ID=$(docker run $DOCKER_ADMIN --rm --interactive --detach --tty \ --mount type=bind,src=$BASE_ROOT_DIR,dst=/ro_base,readonly \ --mount type=bind,src=$CCACHE_DIR,dst=$CCACHE_DIR \ @@ -53,22 +55,23 @@ else echo "Running on host system without docker wrapper" fi -DOCKER_EXEC () { - $DOCKER_CI_CMD_PREFIX bash -c "export PATH=$BASE_SCRATCH_DIR/bins/:\$PATH && cd $P_CI_DIR && $*" +CI_EXEC () { + $DOCKER_CI_CMD_PREFIX bash -c "export PATH=$BASE_SCRATCH_DIR/bins/:\$PATH && cd \"$P_CI_DIR\" && $*" } -export -f DOCKER_EXEC +export -f CI_EXEC if [ -n "$DPKG_ADD_ARCH" ]; then - DOCKER_EXEC dpkg --add-architecture "$DPKG_ADD_ARCH" + CI_EXEC dpkg --add-architecture "$DPKG_ADD_ARCH" fi if [[ $DOCKER_NAME_TAG == *centos* ]]; then - ${CI_RETRY_EXE} DOCKER_EXEC dnf -y install epel-release - ${CI_RETRY_EXE} DOCKER_EXEC dnf -y --allowerasing install $DOCKER_PACKAGES $PACKAGES + ${CI_RETRY_EXE} CI_EXEC dnf -y install epel-release + ${CI_RETRY_EXE} CI_EXEC dnf -y --allowerasing install "$DOCKER_PACKAGES" "$PACKAGES" elif [ "$CI_USE_APT_INSTALL" != "no" ]; then - ${CI_RETRY_EXE} DOCKER_EXEC apt-get update - ${CI_RETRY_EXE} DOCKER_EXEC apt-get install --no-install-recommends --no-upgrade -y $PACKAGES $DOCKER_PACKAGES + ${CI_RETRY_EXE} CI_EXEC apt-get update + ${CI_RETRY_EXE} CI_EXEC apt-get install --no-install-recommends --no-upgrade -y "$PACKAGES" "$DOCKER_PACKAGES" if [ -n "$PIP_PACKAGES" ]; then + # shellcheck disable=SC2086 ${CI_RETRY_EXE} pip3 install --user $PIP_PACKAGES fi fi @@ -77,46 +80,50 @@ if [ "$CI_OS_NAME" == "macos" ]; then top -l 1 -s 0 | awk ' /PhysMem/ {print}' echo "Number of CPUs: $(sysctl -n hw.logicalcpu)" else - DOCKER_EXEC free -m -h - DOCKER_EXEC echo "Number of CPUs \(nproc\):" \$\(nproc\) - DOCKER_EXEC echo $(lscpu | grep Endian) + CI_EXEC free -m -h + CI_EXEC echo "Number of CPUs \(nproc\):" \$\(nproc\) + CI_EXEC echo "$(lscpu | grep Endian)" fi -DOCKER_EXEC echo "Free disk space:" -DOCKER_EXEC df -h - -if [ "$RUN_FUZZ_TESTS" = "true" ] || [ "$RUN_UNIT_TESTS" = "true" ] || [ "$RUN_UNIT_TESTS_SEQUENTIAL" = "true" ]; then - if [ ! -d ${DIR_QA_ASSETS} ]; then - DOCKER_EXEC git clone --depth=1 https://github.com/ElementsProject/qa-assets ${DIR_QA_ASSETS} - fi +CI_EXEC echo "Free disk space:" +CI_EXEC df -h +if [ "$RUN_FUZZ_TESTS" = "true" ]; then export DIR_FUZZ_IN=${DIR_QA_ASSETS}/fuzz_seed_corpus/ + if [ ! -d "$DIR_FUZZ_IN" ]; then + CI_EXEC git clone --depth=1 https://github.com/ElementsProject/qa-assets "${DIR_QA_ASSETS}" + fi +elif [ "$RUN_UNIT_TESTS" = "true" ] || [ "$RUN_UNIT_TESTS_SEQUENTIAL" = "true" ]; then export DIR_UNIT_TEST_DATA=${DIR_QA_ASSETS}/unit_test_data/ + if [ ! -d "$DIR_UNIT_TEST_DATA" ]; then + CI_EXEC mkdir -p "$DIR_UNIT_TEST_DATA" + CI_EXEC curl --location --fail https://github.com/ElementsProject/qa-assets/raw/master/unit_test_data/script_assets_test.json -o "${DIR_UNIT_TEST_DATA}/script_assets_test.json" + fi fi -DOCKER_EXEC mkdir -p "${BASE_SCRATCH_DIR}/sanitizer-output/" +CI_EXEC mkdir -p "${BASE_SCRATCH_DIR}/sanitizer-output/" if [[ ${USE_MEMORY_SANITIZER} == "true" ]]; then - DOCKER_EXEC "update-alternatives --install /usr/bin/clang++ clang++ \$(which clang++-9) 100" - DOCKER_EXEC "update-alternatives --install /usr/bin/clang clang \$(which clang-9) 100" - DOCKER_EXEC "mkdir -p ${BASE_SCRATCH_DIR}/msan/build/" - DOCKER_EXEC "git clone --depth=1 https://github.com/llvm/llvm-project -b llvmorg-12.0.0 ${BASE_SCRATCH_DIR}/msan/llvm-project" - DOCKER_EXEC "cd ${BASE_SCRATCH_DIR}/msan/build/ && cmake -DLLVM_ENABLE_PROJECTS='libcxx;libcxxabi' -DCMAKE_BUILD_TYPE=Release -DLLVM_USE_SANITIZER=Memory -DCMAKE_C_COMPILER=clang -DCMAKE_CXX_COMPILER=clang++ -DLLVM_TARGETS_TO_BUILD=X86 ../llvm-project/llvm/" - DOCKER_EXEC "cd ${BASE_SCRATCH_DIR}/msan/build/ && make $MAKEJOBS cxx" + CI_EXEC "update-alternatives --install /usr/bin/clang++ clang++ \$(which clang++-9) 100" + CI_EXEC "update-alternatives --install /usr/bin/clang clang \$(which clang-9) 100" + CI_EXEC "mkdir -p ${BASE_SCRATCH_DIR}/msan/build/" + CI_EXEC "git clone --depth=1 https://github.com/llvm/llvm-project -b llvmorg-12.0.0 ${BASE_SCRATCH_DIR}/msan/llvm-project" + CI_EXEC "cd ${BASE_SCRATCH_DIR}/msan/build/ && cmake -DLLVM_ENABLE_PROJECTS='libcxx;libcxxabi' -DCMAKE_BUILD_TYPE=Release -DLLVM_USE_SANITIZER=Memory -DCMAKE_C_COMPILER=clang -DCMAKE_CXX_COMPILER=clang++ -DLLVM_TARGETS_TO_BUILD=X86 ../llvm-project/llvm/" + CI_EXEC "cd ${BASE_SCRATCH_DIR}/msan/build/ && make $MAKEJOBS cxx" fi if [ -z "$DANGER_RUN_CI_ON_HOST" ]; then echo "Create $BASE_ROOT_DIR" - DOCKER_EXEC rsync -a /ro_base/ $BASE_ROOT_DIR + CI_EXEC rsync -a /ro_base/ "$BASE_ROOT_DIR" fi if [ "$USE_BUSY_BOX" = "true" ]; then echo "Setup to use BusyBox utils" - DOCKER_EXEC mkdir -p $BASE_SCRATCH_DIR/bins/ + CI_EXEC mkdir -p "${BASE_SCRATCH_DIR}/bins/" # tar excluded for now because it requires passing in the exact archive type in ./depends (fixed in later BusyBox version) # find excluded for now because it does not recognize the -delete option in ./depends (fixed in later BusyBox version) # ar excluded for now because it does not recognize the -q option in ./depends (unknown if fixed) # shellcheck disable=SC1010 - DOCKER_EXEC for util in \$\(busybox --list \| grep -v "^ar$" \| grep -v "^tar$" \| grep -v "^find$"\)\; do ln -s \$\(command -v busybox\) $BASE_SCRATCH_DIR/bins/\$util\; done + CI_EXEC for util in \$\(busybox --list \| grep -v "^ar$" \| grep -v "^tar$" \| grep -v "^find$"\)\; do ln -s \$\(command -v busybox\) "${BASE_SCRATCH_DIR}/bins/\$util"\; done # Print BusyBox version - DOCKER_EXEC patch --help + CI_EXEC patch --help fi diff --git a/ci/test/05_before_script.sh b/ci/test/05_before_script.sh index b5301dbf03..8f75fbd1fa 100755 --- a/ci/test/05_before_script.sh +++ b/ci/test/05_before_script.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # -# Copyright (c) 2018-2020 The Bitcoin Core developers +# Copyright (c) 2018-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -8,53 +8,52 @@ export LC_ALL=C.UTF-8 # Make sure default datadir does not exist and is never read by creating a dummy file if [ "$CI_OS_NAME" == "macos" ]; then - echo > $HOME/Library/Application\ Support/Bitcoin + echo > "${HOME}/Library/Application Support/Bitcoin" else - DOCKER_EXEC echo \> \$HOME/.bitcoin + CI_EXEC echo \> \$HOME/.bitcoin fi -DOCKER_EXEC mkdir -p ${DEPENDS_DIR}/SDKs ${DEPENDS_DIR}/sdk-sources +CI_EXEC mkdir -p "${DEPENDS_DIR}/SDKs" "${DEPENDS_DIR}/sdk-sources" -OSX_SDK_BASENAME="Xcode-${XCODE_VERSION}-${XCODE_BUILD_ID}-extracted-SDK-with-libcxx-headers.tar.gz" -OSX_SDK_PATH="${DEPENDS_DIR}/sdk-sources/${OSX_SDK_BASENAME}" +OSX_SDK_BASENAME="Xcode-${XCODE_VERSION}-${XCODE_BUILD_ID}-extracted-SDK-with-libcxx-headers" -if [ -n "$XCODE_VERSION" ] && [ ! -f "$OSX_SDK_PATH" ]; then - DOCKER_EXEC curl --location --fail "${SDK_URL}/${OSX_SDK_BASENAME}" -o "$OSX_SDK_PATH" +if [ -n "$XCODE_VERSION" ] && [ ! -d "${DEPENDS_DIR}/SDKs/${OSX_SDK_BASENAME}" ]; then + OSX_SDK_FILENAME="${OSX_SDK_BASENAME}.tar.gz" + OSX_SDK_PATH="${DEPENDS_DIR}/sdk-sources/${OSX_SDK_FILENAME}" + if [ ! -f "$OSX_SDK_PATH" ]; then + CI_EXEC curl --location --fail "${SDK_URL}/${OSX_SDK_FILENAME}" -o "$OSX_SDK_PATH" + fi + CI_EXEC tar -C "${DEPENDS_DIR}/SDKs" -xf "$OSX_SDK_PATH" fi -if [ -n "$ANDROID_TOOLS_URL" ]; then - ANDROID_TOOLS_PATH=$DEPENDS_DIR/sdk-sources/android-tools.zip - - DOCKER_EXEC curl --location --fail "${ANDROID_TOOLS_URL}" -o "$ANDROID_TOOLS_PATH" - DOCKER_EXEC mkdir -p "${ANDROID_HOME}/cmdline-tools" - DOCKER_EXEC unzip -o "$ANDROID_TOOLS_PATH" -d "${ANDROID_HOME}/cmdline-tools" - DOCKER_EXEC "yes | ${ANDROID_HOME}/cmdline-tools/tools/bin/sdkmanager --install \"build-tools;${ANDROID_BUILD_TOOLS_VERSION}\" \"platform-tools\" \"platforms;android-${ANDROID_API_LEVEL}\" \"ndk;${ANDROID_NDK_VERSION}\"" +if [ -n "$ANDROID_HOME" ] && [ ! -d "$ANDROID_HOME" ]; then + ANDROID_TOOLS_PATH=${DEPENDS_DIR}/sdk-sources/android-tools.zip + if [ ! -f "$ANDROID_TOOLS_PATH" ]; then + CI_EXEC curl --location --fail "${ANDROID_TOOLS_URL}" -o "$ANDROID_TOOLS_PATH" + fi + CI_EXEC mkdir -p "${ANDROID_HOME}/cmdline-tools" + CI_EXEC unzip -o "$ANDROID_TOOLS_PATH" -d "${ANDROID_HOME}/cmdline-tools" + CI_EXEC "yes | ${ANDROID_HOME}/cmdline-tools/tools/bin/sdkmanager --install \"build-tools;${ANDROID_BUILD_TOOLS_VERSION}\" \"platform-tools\" \"platforms;android-${ANDROID_API_LEVEL}\" \"ndk;${ANDROID_NDK_VERSION}\"" fi if [[ ${USE_MEMORY_SANITIZER} == "true" ]]; then # Use BDB compiled using install_db4.sh script to work around linking issue when using BDB # from depends. See https://github.com/bitcoin/bitcoin/pull/18288#discussion_r433189350 for # details. - DOCKER_EXEC "contrib/install_db4.sh \$(pwd) --enable-umrw CC=clang CXX=clang++ CFLAGS='${MSAN_FLAGS}' CXXFLAGS='${MSAN_AND_LIBCXX_FLAGS}'" + CI_EXEC "contrib/install_db4.sh \$(pwd) --enable-umrw CC=clang CXX=clang++ CFLAGS='${MSAN_FLAGS}' CXXFLAGS='${MSAN_AND_LIBCXX_FLAGS}'" fi -if [ -n "$XCODE_VERSION" ] && [ -f "$OSX_SDK_PATH" ]; then - DOCKER_EXEC tar -C "${DEPENDS_DIR}/SDKs" -xf "$OSX_SDK_PATH" -fi -if [[ $HOST = *-mingw32 ]]; then - DOCKER_EXEC update-alternatives --set $HOST-g++ \$\(which $HOST-g++-posix\) -fi if [ -z "$NO_DEPENDS" ]; then if [[ $DOCKER_NAME_TAG == *centos* ]]; then # CentOS has problems building the depends if the config shell is not explicitly set # (i.e. for libevent a Makefile with an empty SHELL variable is generated, leading to # an error as the first command is executed) - SHELL_OPTS="LC_ALL=en_US.UTF-8 CONFIG_SHELL=/bin/bash" + SHELL_OPTS="LC_ALL=en_US.UTF-8 CONFIG_SHELL=/bin/dash" else SHELL_OPTS="CONFIG_SHELL=" fi - DOCKER_EXEC $SHELL_OPTS make $MAKEJOBS -C depends HOST=$HOST $DEP_OPTS + CI_EXEC "$SHELL_OPTS" make "$MAKEJOBS" -C depends HOST="$HOST" "$DEP_OPTS" fi if [ -n "$PREVIOUS_RELEASES_TO_DOWNLOAD" ]; then - DOCKER_EXEC test/get_previous_releases.py -b -t "$PREVIOUS_RELEASES_DIR" "${PREVIOUS_RELEASES_TO_DOWNLOAD}" + CI_EXEC test/get_previous_releases.py -b -t "$PREVIOUS_RELEASES_DIR" "${PREVIOUS_RELEASES_TO_DOWNLOAD}" fi diff --git a/ci/test/06_script_a.sh b/ci/test/06_script_a.sh index 564460cc4a..4d63042a1e 100755 --- a/ci/test/06_script_a.sh +++ b/ci/test/06_script_a.sh @@ -1,55 +1,55 @@ #!/usr/bin/env bash # -# Copyright (c) 2018-2020 The Bitcoin Core developers +# Copyright (c) 2018-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. export LC_ALL=C.UTF-8 if [ -n "$ANDROID_TOOLS_URL" ]; then - DOCKER_EXEC make distclean || true - DOCKER_EXEC ./autogen.sh - DOCKER_EXEC ./configure $BITCOIN_CONFIG --prefix=$DEPENDS_DIR/aarch64-linux-android || ( (DOCKER_EXEC cat config.log) && false) - DOCKER_EXEC "cd src/qt && make $MAKEJOBS && ANDROID_HOME=${ANDROID_HOME} ANDROID_NDK_HOME=${ANDROID_NDK_HOME} make apk" + CI_EXEC make distclean || true + CI_EXEC ./autogen.sh + CI_EXEC ./configure "$BITCOIN_CONFIG" --prefix="${DEPENDS_DIR}/aarch64-linux-android" || ( (CI_EXEC cat config.log) && false) + CI_EXEC "make $MAKEJOBS && cd src/qt && ANDROID_HOME=${ANDROID_HOME} ANDROID_NDK_HOME=${ANDROID_NDK_HOME} make apk" exit 0 fi -BITCOIN_CONFIG_ALL="--enable-suppress-external-warnings --disable-dependency-tracking --prefix=$DEPENDS_DIR/$HOST --bindir=$BASE_OUTDIR/bin --libdir=$BASE_OUTDIR/lib" +BITCOIN_CONFIG_ALL="--enable-external-signer --enable-suppress-external-warnings --disable-dependency-tracking --prefix=$DEPENDS_DIR/$HOST --bindir=$BASE_OUTDIR/bin --libdir=$BASE_OUTDIR/lib" if [ -z "$NO_WERROR" ]; then BITCOIN_CONFIG_ALL="${BITCOIN_CONFIG_ALL} --enable-werror" fi -DOCKER_EXEC "ccache --zero-stats --max-size=$CCACHE_SIZE" +CI_EXEC "ccache --zero-stats --max-size=$CCACHE_SIZE" if [ -n "$CONFIG_SHELL" ]; then - DOCKER_EXEC "$CONFIG_SHELL" -c "./autogen.sh" + CI_EXEC "$CONFIG_SHELL" -c "./autogen.sh" else - DOCKER_EXEC ./autogen.sh + CI_EXEC ./autogen.sh fi -DOCKER_EXEC mkdir -p "${BASE_BUILD_DIR}" +CI_EXEC mkdir -p "${BASE_BUILD_DIR}" export P_CI_DIR="${BASE_BUILD_DIR}" -DOCKER_EXEC "${BASE_ROOT_DIR}/configure" --cache-file=config.cache $BITCOIN_CONFIG_ALL $BITCOIN_CONFIG || ( (DOCKER_EXEC cat config.log) && false) +CI_EXEC "${BASE_ROOT_DIR}/configure" --cache-file=config.cache "$BITCOIN_CONFIG_ALL" "$BITCOIN_CONFIG" || ( (CI_EXEC cat config.log) && false) -DOCKER_EXEC make distdir VERSION=$HOST +CI_EXEC make distdir VERSION="$HOST" export P_CI_DIR="${BASE_BUILD_DIR}/elements-$HOST" -DOCKER_EXEC ./configure --cache-file=../config.cache $BITCOIN_CONFIG_ALL $BITCOIN_CONFIG || ( (DOCKER_EXEC cat config.log) && false) +CI_EXEC ./configure --cache-file=../config.cache "$BITCOIN_CONFIG_ALL" "$BITCOIN_CONFIG" || ( (CI_EXEC cat config.log) && false) set -o errtrace -trap 'DOCKER_EXEC "cat ${BASE_SCRATCH_DIR}/sanitizer-output/* 2> /dev/null"' ERR +trap 'CI_EXEC "cat ${BASE_SCRATCH_DIR}/sanitizer-output/* 2> /dev/null"' ERR if [[ ${USE_MEMORY_SANITIZER} == "true" ]]; then # MemorySanitizer (MSAN) does not support tracking memory initialization done by # using the Linux getrandom syscall. Avoid using getrandom by undefining # HAVE_SYS_GETRANDOM. See https://github.com/google/sanitizers/issues/852 for # details. - DOCKER_EXEC 'grep -v HAVE_SYS_GETRANDOM src/config/bitcoin-config.h > src/config/bitcoin-config.h.tmp && mv src/config/bitcoin-config.h.tmp src/config/bitcoin-config.h' + CI_EXEC 'grep -v HAVE_SYS_GETRANDOM src/config/bitcoin-config.h > src/config/bitcoin-config.h.tmp && mv src/config/bitcoin-config.h.tmp src/config/bitcoin-config.h' fi -DOCKER_EXEC make $MAKEJOBS $GOAL || ( echo "Build failure. Verbose build follows." && DOCKER_EXEC make $GOAL V=1 ; false ) +CI_EXEC make "$MAKEJOBS" "$GOAL" || ( echo "Build failure. Verbose build follows." && CI_EXEC make "$GOAL" V=1 ; false ) -DOCKER_EXEC "ccache --version | head -n 1 && ccache --show-stats" -DOCKER_EXEC du -sh "${DEPENDS_DIR}"/*/ -DOCKER_EXEC du -sh "${PREVIOUS_RELEASES_DIR}" +CI_EXEC "ccache --version | head -n 1 && ccache --show-stats" +CI_EXEC du -sh "${DEPENDS_DIR}"/*/ +CI_EXEC du -sh "${PREVIOUS_RELEASES_DIR}" diff --git a/ci/test/06_script_b.sh b/ci/test/06_script_b.sh index 9ade87f766..d459704f65 100755 --- a/ci/test/06_script_b.sh +++ b/ci/test/06_script_b.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # -# Copyright (c) 2018-2020 The Bitcoin Core developers +# Copyright (c) 2018-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -8,38 +8,36 @@ export LC_ALL=C.UTF-8 if [[ $HOST = *-mingw32 ]]; then # Generate all binaries, so that they can be wrapped - DOCKER_EXEC make $MAKEJOBS -C src/secp256k1 VERBOSE=1 - DOCKER_EXEC make $MAKEJOBS -C src/univalue VERBOSE=1 - DOCKER_EXEC "${BASE_ROOT_DIR}/ci/test/wrap-wine.sh" + CI_EXEC make "$MAKEJOBS" -C src/secp256k1 VERBOSE=1 + CI_EXEC "${BASE_ROOT_DIR}/ci/test/wrap-wine.sh" fi if [ -n "$QEMU_USER_CMD" ]; then # Generate all binaries, so that they can be wrapped - DOCKER_EXEC make $MAKEJOBS -C src/secp256k1 VERBOSE=1 - DOCKER_EXEC make $MAKEJOBS -C src/univalue VERBOSE=1 - DOCKER_EXEC "${BASE_ROOT_DIR}/ci/test/wrap-qemu.sh" + CI_EXEC make "$MAKEJOBS" -C src/secp256k1 VERBOSE=1 + CI_EXEC "${BASE_ROOT_DIR}/ci/test/wrap-qemu.sh" fi if [ -n "$USE_VALGRIND" ]; then - DOCKER_EXEC "${BASE_ROOT_DIR}/ci/test/wrap-valgrind.sh" + CI_EXEC "${BASE_ROOT_DIR}/ci/test/wrap-valgrind.sh" fi if [ "$RUN_UNIT_TESTS" = "true" ]; then - DOCKER_EXEC ${TEST_RUNNER_ENV} DIR_UNIT_TEST_DATA=${DIR_UNIT_TEST_DATA} LD_LIBRARY_PATH=$DEPENDS_DIR/$HOST/lib make $MAKEJOBS check VERBOSE=1 + CI_EXEC "${TEST_RUNNER_ENV}" DIR_UNIT_TEST_DATA="${DIR_UNIT_TEST_DATA}" LD_LIBRARY_PATH="${DEPENDS_DIR}/${HOST}/lib" make "$MAKEJOBS" check VERBOSE=1 fi if [ "$RUN_UNIT_TESTS_SEQUENTIAL" = "true" ]; then - DOCKER_EXEC ${TEST_RUNNER_ENV} DIR_UNIT_TEST_DATA=${DIR_UNIT_TEST_DATA} LD_LIBRARY_PATH=$DEPENDS_DIR/$HOST/lib "${BASE_BUILD_DIR}/elements-*/src/test/test_bitcoin*" --catch_system_errors=no -l test_suite + CI_EXEC "${TEST_RUNNER_ENV}" DIR_UNIT_TEST_DATA="${DIR_UNIT_TEST_DATA}" LD_LIBRARY_PATH="${DEPENDS_DIR}/${HOST}/lib" "${BASE_BUILD_DIR}/elements-*/src/test/test_bitcoin*" --catch_system_errors=no -l test_suite fi if [ "$RUN_FUNCTIONAL_TESTS" = "true" ]; then - DOCKER_EXEC LD_LIBRARY_PATH=$DEPENDS_DIR/$HOST/lib ${TEST_RUNNER_ENV} test/functional/test_runner.py --ci $MAKEJOBS --tmpdirprefix "${BASE_SCRATCH_DIR}/test_runner/" --ansi --combinedlogslen=4000 --timeout-factor=${TEST_RUNNER_TIMEOUT_FACTOR} ${TEST_RUNNER_EXTRA} --quiet --failfast + CI_EXEC LD_LIBRARY_PATH="${DEPENDS_DIR}/${HOST}/lib" "${TEST_RUNNER_ENV}" test/functional/test_runner.py --ci "$MAKEJOBS" --tmpdirprefix "${BASE_SCRATCH_DIR}/test_runner/" --ansi --combinedlogslen=4000 --timeout-factor="${TEST_RUNNER_TIMEOUT_FACTOR}" "${TEST_RUNNER_EXTRA}" --quiet --failfast fi if [ "$RUN_SECURITY_TESTS" = "true" ]; then - DOCKER_EXEC make test-security-check + CI_EXEC make test-security-check fi if [ "$RUN_FUZZ_TESTS" = "true" ]; then - DOCKER_EXEC LD_LIBRARY_PATH=$DEPENDS_DIR/$HOST/lib test/fuzz/test_runner.py ${FUZZ_TESTS_CONFIG} $MAKEJOBS -l DEBUG ${DIR_FUZZ_IN} + CI_EXEC LD_LIBRARY_PATH="${DEPENDS_DIR}/${HOST}/lib" test/fuzz/test_runner.py "${FUZZ_TESTS_CONFIG}" "$MAKEJOBS" -l DEBUG "${DIR_FUZZ_IN}" fi diff --git a/ci/test/wrap-qemu.sh b/ci/test/wrap-qemu.sh index 2cd7c8cec2..fcd56f533e 100755 --- a/ci/test/wrap-qemu.sh +++ b/ci/test/wrap-qemu.sh @@ -1,14 +1,14 @@ #!/usr/bin/env bash # -# Copyright (c) 2018-2020 The Bitcoin Core developers +# Copyright (c) 2018-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. export LC_ALL=C.UTF-8 -for b_name in {"${BASE_OUTDIR}/bin"/*,src/secp256k1/*tests,src/univalue/{no_nul,test_json,unitester,object}}; do +for b_name in {"${BASE_OUTDIR}/bin"/*,src/secp256k1/*tests,src/minisketch/test{,-verify},src/univalue/{no_nul,test_json,unitester,object}}; do # shellcheck disable=SC2044 - for b in $(find "${BASE_ROOT_DIR}" -executable -type f -name $(basename $b_name)); do + for b in $(find "${BASE_ROOT_DIR}" -executable -type f -name "$(basename "$b_name")"); do echo "Wrap $b ..." mv "$b" "${b}_orig" echo '#!/usr/bin/env bash' > "$b" diff --git a/ci/test/wrap-valgrind.sh b/ci/test/wrap-valgrind.sh index 6b3e6eb7e7..2775483184 100755 --- a/ci/test/wrap-valgrind.sh +++ b/ci/test/wrap-valgrind.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # -# Copyright (c) 2018-2019 The Bitcoin Core developers +# Copyright (c) 2018-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -8,7 +8,7 @@ export LC_ALL=C.UTF-8 for b_name in "${BASE_OUTDIR}/bin"/*; do # shellcheck disable=SC2044 - for b in $(find "${BASE_ROOT_DIR}" -executable -type f -name $(basename $b_name)); do + for b in $(find "${BASE_ROOT_DIR}" -executable -type f -name "$(basename "$b_name")"); do echo "Wrap $b ..." mv "$b" "${b}_orig" echo '#!/usr/bin/env bash' > "$b" diff --git a/ci/test/wrap-wine.sh b/ci/test/wrap-wine.sh index 82964897e1..525db9eded 100755 --- a/ci/test/wrap-wine.sh +++ b/ci/test/wrap-wine.sh @@ -1,14 +1,14 @@ #!/usr/bin/env bash # -# Copyright (c) 2020 The Bitcoin Core developers +# Copyright (c) 2020-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. export LC_ALL=C.UTF-8 -for b_name in {"${BASE_OUTDIR}/bin"/*,src/secp256k1/*tests,src/univalue/{no_nul,test_json,unitester,object}}.exe; do +for b_name in {"${BASE_OUTDIR}/bin"/*,src/secp256k1/*tests,src/minisketch/test{,-verify},src/univalue/{no_nul,test_json,unitester,object}}.exe; do # shellcheck disable=SC2044 - for b in $(find "${BASE_ROOT_DIR}" -executable -type f -name "$(basename $b_name)"); do + for b in $(find "${BASE_ROOT_DIR}" -executable -type f -name "$(basename "$b_name")"); do if (file "$b" | grep "Windows"); then echo "Wrap $b ..." mv "$b" "${b}_orig" diff --git a/configure.ac b/configure.ac index 6bbb436877..689d81df3a 100644 --- a/configure.ac +++ b/configure.ac @@ -1,10 +1,10 @@ AC_PREREQ([2.69]) -define(_CLIENT_VERSION_MAJOR, 21) +define(_CLIENT_VERSION_MAJOR, 23) define(_CLIENT_VERSION_MINOR, 99) define(_CLIENT_VERSION_BUILD, 0) define(_CLIENT_VERSION_RC, 0) define(_CLIENT_VERSION_IS_RELEASE, false) -define(_COPYRIGHT_YEAR, 2021) +define(_COPYRIGHT_YEAR, 2022) define(_COPYRIGHT_HOLDERS,[The %s developers]) define(_COPYRIGHT_HOLDERS_SUBSTITUTION,[[Elements Project]]) AC_INIT([Elements Core],m4_join([.], _CLIENT_VERSION_MAJOR, _CLIENT_VERSION_MINOR, _CLIENT_VERSION_BUILD)m4_if(_CLIENT_VERSION_RC, [0], [], [rc]_CLIENT_VERSION_RC),[https://github.com/ElementsProject/elements/issues],[elements],[https://elementsproject.org/]) @@ -13,9 +13,9 @@ AC_CONFIG_HEADERS([src/config/bitcoin-config.h]) AC_CONFIG_AUX_DIR([build-aux]) AC_CONFIG_MACRO_DIR([build-aux/m4]) -m4_ifndef([PKG_PROG_PKG_CONFIG], [AC_MSG_ERROR([PKG_PROG_PKG_CONFIG macro not found. Please install pkg-config and re-run autogen.sh])]) +m4_ifndef([PKG_PROG_PKG_CONFIG], [m4_fatal([PKG_PROG_PKG_CONFIG macro not found. Please install pkg-config and re-run autogen.sh])]) PKG_PROG_PKG_CONFIG -if test "x$PKG_CONFIG" = x; then +if test "$PKG_CONFIG" = ""; then AC_MSG_ERROR([pkg-config not found]) fi @@ -24,14 +24,15 @@ BITCOIN_GUI_NAME=elements-qt BITCOIN_CLI_NAME=elements-cli BITCOIN_TX_NAME=elements-tx BITCOIN_UTIL_NAME=elements-util +BITCOIN_CHAINSTATE_NAME=elements-chainstate BITCOIN_WALLET_TOOL_NAME=elements-wallet dnl Multi Process BITCOIN_MP_NODE_NAME=elements-node BITCOIN_MP_GUI_NAME=elements-gui dnl Unless the user specified ARFLAGS, force it to be cr -AC_ARG_VAR(ARFLAGS, [Flags for the archiver, defaults to if not set]) -if test "x${ARFLAGS+set}" != "xset"; then +AC_ARG_VAR([ARFLAGS], [Flags for the archiver, defaults to if not set]) +if test "${ARFLAGS+set}" != "set"; then ARFLAGS="cr" fi @@ -55,7 +56,7 @@ dnl make the compilation flags quiet unless V=1 is used AM_SILENT_RULES([yes]) dnl Compiler checks (here before libtool). -if test "x${CXXFLAGS+set}" = "xset"; then +if test "${CXXFLAGS+set}" = "set"; then CXXFLAGS_overridden=yes else CXXFLAGS_overridden=no @@ -71,16 +72,25 @@ case $host in ;; esac +AC_ARG_WITH([seccomp], + [AS_HELP_STRING([--with-seccomp], + [enable experimental syscall sandbox feature (-sandbox), default is yes if seccomp-bpf is detected under Linux x86_64])], + [seccomp_found=$withval], + [seccomp_found=auto]) + dnl Require C++17 compiler (no GNU extensions) AX_CXX_COMPILE_STDCXX([17], [noext], [mandatory]) dnl Check if -latomic is required for CHECK_ATOMIC +dnl check if additional link flags are required for std::filesystem +CHECK_FILESYSTEM + dnl Unless the user specified OBJCXX, force it to be the same as CXX. This ensures dnl that we get the same -std flags for both. m4_ifdef([AC_PROG_OBJCXX],[ -if test "x${OBJCXX+set}" = "x"; then +if test "${OBJCXX+set}" = ""; then OBJCXX="${CXX}" fi AC_PROG_OBJCXX @@ -91,28 +101,27 @@ dnl sets RPATH for any directories in the dynamic linker search path. dnl See more: https://wiki.debian.org/RpathIssue LT_PREREQ([1.5.2]) dnl Libtool init checks. -LT_INIT([pic-only]) +LT_INIT([pic-only win32-dll]) dnl Check/return PATH for base programs. -AC_PATH_TOOL(AR, ar) -AC_PATH_TOOL(RANLIB, ranlib) -AC_PATH_TOOL(STRIP, strip) -AC_PATH_TOOL(GCOV, gcov) -AC_PATH_TOOL(LLVM_COV, llvm-cov) -AC_PATH_PROG(LCOV, lcov) +AC_PATH_TOOL([AR], [ar]) +AC_PATH_TOOL([RANLIB], [ranlib]) +AC_PATH_TOOL([STRIP], [strip]) +AC_PATH_TOOL([GCOV], [gcov]) +AC_PATH_TOOL([LLVM_COV], [llvm-cov]) +AC_PATH_PROG([LCOV], [lcov]) dnl Python 3.6 is specified in .python-version and should be used if available, see doc/dependencies.md -AC_PATH_PROGS([PYTHON], [python3.6 python3.7 python3.8 python3.9 python3 python]) -AC_PATH_PROG(GENHTML, genhtml) +AC_PATH_PROGS([PYTHON], [python3.6 python3.7 python3.8 python3.9 python3.10 python3.11 python3 python]) +AC_PATH_PROG([GENHTML], [genhtml]) AC_PATH_PROG([GIT], [git]) -AC_PATH_PROG(CCACHE,ccache) -AC_PATH_PROG(XGETTEXT,xgettext) -AC_PATH_PROG(HEXDUMP,hexdump) -AC_PATH_TOOL(CPPFILT, c++filt) -AC_PATH_TOOL(OBJCOPY, objcopy) -AC_PATH_PROG(DOXYGEN, doxygen) +AC_PATH_PROG([CCACHE], [ccache]) +AC_PATH_PROG([XGETTEXT], [xgettext]) +AC_PATH_PROG([HEXDUMP], [hexdump]) +AC_PATH_TOOL([OBJCOPY], [objcopy]) +AC_PATH_PROG([DOXYGEN], [doxygen]) AM_CONDITIONAL([HAVE_DOXYGEN], [test -n "$DOXYGEN"]) -AC_ARG_VAR(PYTHONPATH, Augments the default search path for python module files) +AC_ARG_VAR([PYTHONPATH], [Augments the default search path for python module files]) AC_ARG_ENABLE([wallet], [AS_HELP_STRING([--disable-wallet], @@ -132,11 +141,11 @@ AC_ARG_WITH([bdb], [use_bdb=$withval], [use_bdb=auto]) -AC_ARG_ENABLE([ebpf], - [AS_HELP_STRING([--enable-ebpf], - [enable eBPF tracing (default is yes if sys/sdt.h is found)])], - [use_ebpf=$enableval], - [use_ebpf=yes]) +AC_ARG_ENABLE([usdt], + [AS_HELP_STRING([--enable-usdt], + [enable tracepoints for Userspace, Statically Defined Tracing (default is yes if sys/sdt.h is found)])], + [use_usdt=$enableval], + [use_usdt=yes]) AC_ARG_WITH([miniupnpc], [AS_HELP_STRING([--with-miniupnpc], @@ -240,15 +249,9 @@ AC_ARG_ENABLE([lcov-branch-coverage], [use_lcov_branch=yes], [use_lcov_branch=no]) -AC_ARG_ENABLE([glibc-back-compat], - [AS_HELP_STRING([--enable-glibc-back-compat], - [enable backwards compatibility with glibc])], - [use_glibc_compat=$enableval], - [use_glibc_compat=no]) - AC_ARG_ENABLE([threadlocal], [AS_HELP_STRING([--enable-threadlocal], - [enable features that depend on the c++ thread_local keyword (currently just thread names in debug logs). (default is to enabled if there is platform support and glibc-back-compat is not enabled)])], + [enable features that depend on the c++ thread_local keyword (currently just thread names in debug logs). (default is to enable if there is platform support)])], [use_thread_local=$enableval], [use_thread_local=auto]) @@ -264,20 +267,13 @@ AC_ARG_ENABLE([liquid], [liquid_build=yes], [liquid_build=no]) -if test "x$use_asm" = xyes; then - AC_DEFINE(USE_ASM, 1, [Define this symbol to build in assembly routines]) +if test "$use_asm" = "yes"; then + AC_DEFINE([USE_ASM], [1], [Define this symbol to build in assembly routines]) fi -if test "x$liquid_build" = xyes; then +if test "$liquid_build" = "yes"; then AC_DEFINE(LIQUID, 1, [Define this symbol for Liquid builds]) fi - -AC_ARG_WITH([system-univalue], - [AS_HELP_STRING([--with-system-univalue], - [Build with system UniValue (default is no)])], - [system_univalue=$withval], - [system_univalue=no] -) AC_ARG_ENABLE([zmq], [AS_HELP_STRING([--disable-zmq], [disable ZMQ notifications])], @@ -306,7 +302,7 @@ AC_ARG_ENABLE(man, [AS_HELP_STRING([--disable-man], [do not install man pages (default is to install)])],, enable_man=yes) -AM_CONDITIONAL(ENABLE_MAN, test "$enable_man" != no) +AM_CONDITIONAL([ENABLE_MAN], [test "$enable_man" != "no"]) dnl Enable debug AC_ARG_ENABLE([debug], @@ -331,14 +327,19 @@ AC_ARG_ENABLE([gprof], dnl Turn warnings into errors AC_ARG_ENABLE([werror], [AS_HELP_STRING([--enable-werror], - [Treat certain compiler warnings as errors (default is no)])], + [Treat compiler warnings as errors (default is no)])], [enable_werror=$enableval], [enable_werror=no]) AC_ARG_ENABLE([external-signer], [AS_HELP_STRING([--enable-external-signer],[compile external signer support (default is yes, requires Boost::Process)])], [use_external_signer=$enableval], - [use_external_signer=yes]) + [use_external_signer=auto]) + +AC_ARG_ENABLE([lto], + [AS_HELP_STRING([--enable-lto],[build using LTO (default is no)])], + [enable_lto=$enableval], + [enable_lto=no]) AC_LANG_PUSH([C++]) @@ -349,7 +350,7 @@ dnl Note that this is not necessarily a check to see if -Werror is supported, bu dnl a compile with -Werror can succeed. This is important because the compiler may already be dnl warning about something unrelated, for example about some path issue. If that is the case, dnl -Werror cannot be used because all of those warnings would be turned into errors. -AX_CHECK_COMPILE_FLAG([-Werror],[CXXFLAG_WERROR="-Werror"],[CXXFLAG_WERROR=""]) +AX_CHECK_COMPILE_FLAG([-Werror], [CXXFLAG_WERROR="-Werror"], [CXXFLAG_WERROR=""]) dnl Check for a flag to turn linker warnings into errors. When flags are passed to linkers via the dnl compiler driver using a -Wl,-foo flag, linker warnings may be swallowed rather than bubbling up. @@ -358,42 +359,47 @@ dnl dnl LDFLAG_WERROR Should only be used when testing -Wl,* case $host in *darwin*) - AX_CHECK_LINK_FLAG([-Wl,-fatal_warnings],[LDFLAG_WERROR="-Wl,-fatal_warnings"],[LDFLAG_WERROR=""]) + AX_CHECK_LINK_FLAG([-Wl,-fatal_warnings], [LDFLAG_WERROR="-Wl,-fatal_warnings"], [LDFLAG_WERROR=""]) ;; *) - AX_CHECK_LINK_FLAG([-Wl,--fatal-warnings],[LDFLAG_WERROR="-Wl,--fatal-warnings"],[LDFLAG_WERROR=""]) + AX_CHECK_LINK_FLAG([-Wl,--fatal-warnings], [LDFLAG_WERROR="-Wl,--fatal-warnings"], [LDFLAG_WERROR=""]) ;; esac -if test "x$enable_debug" = xyes; then +if test "$enable_debug" = "yes"; then dnl Clear default -g -O2 flags - if test "x$CXXFLAGS_overridden" = xno; then - CXXFLAGS="" + if test "$CXXFLAGS_overridden" = "no"; then + CXXFLAGS="" fi dnl Disable all optimizations - AX_CHECK_COMPILE_FLAG([-O0], [[DEBUG_CXXFLAGS="$DEBUG_CXXFLAGS -O0"]],,[[$CXXFLAG_WERROR]]) + AX_CHECK_COMPILE_FLAG([-O0], [DEBUG_CXXFLAGS="$DEBUG_CXXFLAGS -O0"], [], [$CXXFLAG_WERROR]) dnl Prefer -g3, fall back to -g if that is unavailable. AX_CHECK_COMPILE_FLAG( [-g3], - [[DEBUG_CXXFLAGS="$DEBUG_CXXFLAGS -g3"]], - [AX_CHECK_COMPILE_FLAG([-g],[[DEBUG_CXXFLAGS="$DEBUG_CXXFLAGS -g"]],,[[$CXXFLAG_WERROR]])], - [[$CXXFLAG_WERROR]]) + [DEBUG_CXXFLAGS="$DEBUG_CXXFLAGS -g3"], + [AX_CHECK_COMPILE_FLAG([-g], [DEBUG_CXXFLAGS="$DEBUG_CXXFLAGS -g"], [], [$CXXFLAG_WERROR])], + [$CXXFLAG_WERROR]) - AX_CHECK_PREPROC_FLAG([-DDEBUG],[[DEBUG_CPPFLAGS="$DEBUG_CPPFLAGS -DDEBUG"]],,[[$CXXFLAG_WERROR]]) - AX_CHECK_PREPROC_FLAG([-DDEBUG_LOCKORDER],[[DEBUG_CPPFLAGS="$DEBUG_CPPFLAGS -DDEBUG_LOCKORDER"]],,[[$CXXFLAG_WERROR]]) - AX_CHECK_PREPROC_FLAG([-DABORT_ON_FAILED_ASSUME],[[DEBUG_CPPFLAGS="$DEBUG_CPPFLAGS -DABORT_ON_FAILED_ASSUME"]],,[[$CXXFLAG_WERROR]]) - AX_CHECK_COMPILE_FLAG([-ftrapv],[DEBUG_CXXFLAGS="$DEBUG_CXXFLAGS -ftrapv"],,[[$CXXFLAG_WERROR]]) + AX_CHECK_PREPROC_FLAG([-DDEBUG], [DEBUG_CPPFLAGS="$DEBUG_CPPFLAGS -DDEBUG"], [], [$CXXFLAG_WERROR]) + AX_CHECK_PREPROC_FLAG([-DDEBUG_LOCKORDER], [DEBUG_CPPFLAGS="$DEBUG_CPPFLAGS -DDEBUG_LOCKORDER"], [], [$CXXFLAG_WERROR]) + AX_CHECK_PREPROC_FLAG([-DABORT_ON_FAILED_ASSUME], [DEBUG_CPPFLAGS="$DEBUG_CPPFLAGS -DABORT_ON_FAILED_ASSUME"], [], [$CXXFLAG_WERROR]) + AX_CHECK_COMPILE_FLAG([-ftrapv], [DEBUG_CXXFLAGS="$DEBUG_CXXFLAGS -ftrapv"], [], [$CXXFLAG_WERROR]) fi -if test x$use_sanitizers != x; then +if test "$enable_lto" = "yes"; then + AX_CHECK_COMPILE_FLAG([-flto], [LTO_CXXFLAGS="$LTO_CXXFLAGS -flto"], [AC_MSG_ERROR([compile failed with -flto])], [$CXXFLAG_WERROR]) + AX_CHECK_LINK_FLAG([-flto], [LTO_LDFLAGS="$LTO_LDFLAGS -flto"], [AC_MSG_ERROR([link failed with -flto])], [$CXXFLAG_WERROR]) +fi + +if test "$use_sanitizers" != ""; then dnl First check if the compiler accepts flags. If an incompatible pair like dnl -fsanitize=address,thread is used here, this check will fail. This will also dnl fail if a bad argument is passed, e.g. -fsanitize=undfeined AX_CHECK_COMPILE_FLAG( - [[-fsanitize=$use_sanitizers]], - [[SANITIZER_CXXFLAGS=-fsanitize=$use_sanitizers]], + [-fsanitize=$use_sanitizers], + [SANITIZER_CXXFLAGS="-fsanitize=$use_sanitizers"], [AC_MSG_ERROR([compiler did not accept requested flags])]) dnl Some compilers (e.g. GCC) require additional libraries like libasan, @@ -402,8 +408,8 @@ if test x$use_sanitizers != x; then dnl the sanitize flags are supported by the compiler but the actual sanitizer dnl libs are missing. AX_CHECK_LINK_FLAG( - [[-fsanitize=$use_sanitizers]], - [[SANITIZER_LDFLAGS=-fsanitize=$use_sanitizers]], + [-fsanitize=$use_sanitizers], + [SANITIZER_LDFLAGS="-fsanitize=$use_sanitizers"], [AC_MSG_ERROR([linker did not accept requested flags, you are missing required libraries])], [], [AC_LANG_PROGRAM([[ @@ -415,98 +421,99 @@ if test x$use_sanitizers != x; then fi ERROR_CXXFLAGS= -if test "x$enable_werror" = "xyes"; then - if test "x$CXXFLAG_WERROR" = "x"; then - AC_MSG_ERROR("enable-werror set but -Werror is not usable") +if test "$enable_werror" = "yes"; then + if test "$CXXFLAG_WERROR" = ""; then + AC_MSG_ERROR([enable-werror set but -Werror is not usable]) fi - AX_CHECK_COMPILE_FLAG([-Werror=gnu],[ERROR_CXXFLAGS="$ERROR_CXXFLAGS -Werror=gnu"],,[[$CXXFLAG_WERROR]]) - AX_CHECK_COMPILE_FLAG([-Werror=vla],[ERROR_CXXFLAGS="$ERROR_CXXFLAGS -Werror=vla"],,[[$CXXFLAG_WERROR]]) - AX_CHECK_COMPILE_FLAG([-Werror=shadow-field],[ERROR_CXXFLAGS="$ERROR_CXXFLAGS -Werror=shadow-field"],,[[$CXXFLAG_WERROR]]) - AX_CHECK_COMPILE_FLAG([-Werror=switch],[ERROR_CXXFLAGS="$ERROR_CXXFLAGS -Werror=switch"],,[[$CXXFLAG_WERROR]]) - AX_CHECK_COMPILE_FLAG([-Werror=thread-safety],[ERROR_CXXFLAGS="$ERROR_CXXFLAGS -Werror=thread-safety"],,[[$CXXFLAG_WERROR]]) - AX_CHECK_COMPILE_FLAG([-Werror=range-loop-analysis],[ERROR_CXXFLAGS="$ERROR_CXXFLAGS -Werror=range-loop-analysis"],,[[$CXXFLAG_WERROR]]) - AX_CHECK_COMPILE_FLAG([-Werror=unused-variable],[ERROR_CXXFLAGS="$ERROR_CXXFLAGS -Werror=unused-variable"],,[[$CXXFLAG_WERROR]]) - AX_CHECK_COMPILE_FLAG([-Werror=date-time],[ERROR_CXXFLAGS="$ERROR_CXXFLAGS -Werror=date-time"],,[[$CXXFLAG_WERROR]]) - AX_CHECK_COMPILE_FLAG([-Werror=return-type],[ERROR_CXXFLAGS="$ERROR_CXXFLAGS -Werror=return-type"],,[[$CXXFLAG_WERROR]]) - AX_CHECK_COMPILE_FLAG([-Werror=conditional-uninitialized],[ERROR_CXXFLAGS="$ERROR_CXXFLAGS -Werror=conditional-uninitialized"],,[[$CXXFLAG_WERROR]]) - AX_CHECK_COMPILE_FLAG([-Werror=sign-compare],[ERROR_CXXFLAGS="$ERROR_CXXFLAGS -Werror=sign-compare"],,[[$CXXFLAG_WERROR]]) - dnl -Wsuggest-override is broken with GCC before 9.2 - dnl https://gcc.gnu.org/bugzilla/show_bug.cgi?id=78010 - AX_CHECK_COMPILE_FLAG([-Werror=suggest-override],[ERROR_CXXFLAGS="$ERROR_CXXFLAGS -Werror=suggest-override"],,[[$CXXFLAG_WERROR]], - [AC_LANG_SOURCE([[struct A { virtual void f(); }; struct B : A { void f() final; };]])]) - AX_CHECK_COMPILE_FLAG([-Werror=unreachable-code-loop-increment],[ERROR_CXXFLAGS="$ERROR_CXXFLAGS -Werror=unreachable-code-loop-increment"],,[[$CXXFLAG_WERROR]]) - AX_CHECK_COMPILE_FLAG([-Werror=mismatched-tags], [ERROR_CXXFLAGS="$ERROR_CXXFLAGS -Werror=mismatched-tags"], [], [$CXXFLAG_WERROR]) - AX_CHECK_COMPILE_FLAG([-Werror=implicit-fallthrough], [ERROR_CXXFLAGS="$ERROR_CXXFLAGS -Werror=implicit-fallthrough"], [], [$CXXFLAG_WERROR]) + ERROR_CXXFLAGS=$CXXFLAG_WERROR - if test x$suppress_external_warnings != xno ; then - AX_CHECK_COMPILE_FLAG([-Werror=documentation],[ERROR_CXXFLAGS="$ERROR_CXXFLAGS -Werror=documentation"],,[[$CXXFLAG_WERROR]]) - fi + dnl -Wreturn-type is broken in GCC for MinGW-w64. + dnl https://sourceforge.net/p/mingw-w64/bugs/306/ + AX_CHECK_COMPILE_FLAG([-Werror=return-type], [], [ERROR_CXXFLAGS="$ERROR_CXXFLAGS -Wno-error=return-type"], [$CXXFLAG_WERROR], + [AC_LANG_SOURCE([[#include + int f(){ assert(false); }]])]) fi -if test "x$CXXFLAGS_overridden" = "xno"; then - AX_CHECK_COMPILE_FLAG([-Wall],[WARN_CXXFLAGS="$WARN_CXXFLAGS -Wall"],,[[$CXXFLAG_WERROR]]) - AX_CHECK_COMPILE_FLAG([-Wextra],[WARN_CXXFLAGS="$WARN_CXXFLAGS -Wextra"],,[[$CXXFLAG_WERROR]]) - AX_CHECK_COMPILE_FLAG([-Wgnu],[WARN_CXXFLAGS="$WARN_CXXFLAGS -Wgnu"],,[[$CXXFLAG_WERROR]]) +if test "$CXXFLAGS_overridden" = "no"; then + AX_CHECK_COMPILE_FLAG([-Wall], [WARN_CXXFLAGS="$WARN_CXXFLAGS -Wall"], [], [$CXXFLAG_WERROR]) + AX_CHECK_COMPILE_FLAG([-Wextra], [WARN_CXXFLAGS="$WARN_CXXFLAGS -Wextra"], [], [$CXXFLAG_WERROR]) + AX_CHECK_COMPILE_FLAG([-Wgnu], [WARN_CXXFLAGS="$WARN_CXXFLAGS -Wgnu"], [], [$CXXFLAG_WERROR]) dnl some compilers will ignore -Wformat-security without -Wformat, so just combine the two here. - AX_CHECK_COMPILE_FLAG([-Wformat -Wformat-security],[WARN_CXXFLAGS="$WARN_CXXFLAGS -Wformat -Wformat-security"],,[[$CXXFLAG_WERROR]]) - AX_CHECK_COMPILE_FLAG([-Wvla],[WARN_CXXFLAGS="$WARN_CXXFLAGS -Wvla"],,[[$CXXFLAG_WERROR]]) - AX_CHECK_COMPILE_FLAG([-Wshadow-field],[WARN_CXXFLAGS="$WARN_CXXFLAGS -Wshadow-field"],,[[$CXXFLAG_WERROR]]) - AX_CHECK_COMPILE_FLAG([-Wswitch],[WARN_CXXFLAGS="$WARN_CXXFLAGS -Wswitch"],,[[$CXXFLAG_WERROR]]) - AX_CHECK_COMPILE_FLAG([-Wthread-safety],[WARN_CXXFLAGS="$WARN_CXXFLAGS -Wthread-safety"],,[[$CXXFLAG_WERROR]]) - AX_CHECK_COMPILE_FLAG([-Wrange-loop-analysis],[WARN_CXXFLAGS="$WARN_CXXFLAGS -Wrange-loop-analysis"],,[[$CXXFLAG_WERROR]]) - AX_CHECK_COMPILE_FLAG([-Wredundant-decls],[WARN_CXXFLAGS="$WARN_CXXFLAGS -Wredundant-decls"],,[[$CXXFLAG_WERROR]]) - AX_CHECK_COMPILE_FLAG([-Wunused-variable],[WARN_CXXFLAGS="$WARN_CXXFLAGS -Wunused-variable"],,[[$CXXFLAG_WERROR]]) - AX_CHECK_COMPILE_FLAG([-Wunused-member-function],[WARN_CXXFLAGS="$WARN_CXXFLAGS -Wunused-member-function"],,[[$CXXFLAG_WERROR]]) - AX_CHECK_COMPILE_FLAG([-Wdate-time],[WARN_CXXFLAGS="$WARN_CXXFLAGS -Wdate-time"],,[[$CXXFLAG_WERROR]]) - AX_CHECK_COMPILE_FLAG([-Wconditional-uninitialized],[WARN_CXXFLAGS="$WARN_CXXFLAGS -Wconditional-uninitialized"],,[[$CXXFLAG_WERROR]]) - AX_CHECK_COMPILE_FLAG([-Wsign-compare],[WARN_CXXFLAGS="$WARN_CXXFLAGS -Wsign-compare"],,[[$CXXFLAG_WERROR]]) - AX_CHECK_COMPILE_FLAG([-Wduplicated-branches],[WARN_CXXFLAGS="$WARN_CXXFLAGS -Wduplicated-branches"],,[[$CXXFLAG_WERROR]]) - AX_CHECK_COMPILE_FLAG([-Wduplicated-cond],[WARN_CXXFLAGS="$WARN_CXXFLAGS -Wduplicated-cond"],,[[$CXXFLAG_WERROR]]) - AX_CHECK_COMPILE_FLAG([-Wlogical-op],[WARN_CXXFLAGS="$WARN_CXXFLAGS -Wlogical-op"],,[[$CXXFLAG_WERROR]]) - AX_CHECK_COMPILE_FLAG([-Woverloaded-virtual],[WARN_CXXFLAGS="$WARN_CXXFLAGS -Woverloaded-virtual"],,[[$CXXFLAG_WERROR]]) - AX_CHECK_COMPILE_FLAG([-Wsuggest-override],[WARN_CXXFLAGS="$WARN_CXXFLAGS -Wsuggest-override"],,[[$CXXFLAG_WERROR]], + AX_CHECK_COMPILE_FLAG([-Wformat -Wformat-security], [WARN_CXXFLAGS="$WARN_CXXFLAGS -Wformat -Wformat-security"], [], [$CXXFLAG_WERROR]) + AX_CHECK_COMPILE_FLAG([-Wvla], [WARN_CXXFLAGS="$WARN_CXXFLAGS -Wvla"], [], [$CXXFLAG_WERROR]) + AX_CHECK_COMPILE_FLAG([-Wshadow-field], [WARN_CXXFLAGS="$WARN_CXXFLAGS -Wshadow-field"], [], [$CXXFLAG_WERROR]) + AX_CHECK_COMPILE_FLAG([-Wthread-safety], [WARN_CXXFLAGS="$WARN_CXXFLAGS -Wthread-safety"], [], [$CXXFLAG_WERROR]) + AX_CHECK_COMPILE_FLAG([-Wloop-analysis], [WARN_CXXFLAGS="$WARN_CXXFLAGS -Wrange-loop-analysis"], [], [$CXXFLAG_WERROR]) + AX_CHECK_COMPILE_FLAG([-Wredundant-decls], [WARN_CXXFLAGS="$WARN_CXXFLAGS -Wredundant-decls"], [], [$CXXFLAG_WERROR]) + AX_CHECK_COMPILE_FLAG([-Wunused-member-function], [WARN_CXXFLAGS="$WARN_CXXFLAGS -Wunused-member-function"], [], [$CXXFLAG_WERROR]) + AX_CHECK_COMPILE_FLAG([-Wdate-time], [WARN_CXXFLAGS="$WARN_CXXFLAGS -Wdate-time"], [], [$CXXFLAG_WERROR]) + AX_CHECK_COMPILE_FLAG([-Wconditional-uninitialized], [WARN_CXXFLAGS="$WARN_CXXFLAGS -Wconditional-uninitialized"], [], [$CXXFLAG_WERROR]) + AX_CHECK_COMPILE_FLAG([-Wduplicated-branches], [WARN_CXXFLAGS="$WARN_CXXFLAGS -Wduplicated-branches"], [], [$CXXFLAG_WERROR]) + AX_CHECK_COMPILE_FLAG([-Wduplicated-cond], [WARN_CXXFLAGS="$WARN_CXXFLAGS -Wduplicated-cond"], [], [$CXXFLAG_WERROR]) + AX_CHECK_COMPILE_FLAG([-Wlogical-op], [WARN_CXXFLAGS="$WARN_CXXFLAGS -Wlogical-op"], [], [$CXXFLAG_WERROR]) + AX_CHECK_COMPILE_FLAG([-Woverloaded-virtual], [WARN_CXXFLAGS="$WARN_CXXFLAGS -Woverloaded-virtual"], [], [$CXXFLAG_WERROR]) + dnl -Wsuggest-override is broken with GCC before 9.2 + dnl https://gcc.gnu.org/bugzilla/show_bug.cgi?id=78010 + AX_CHECK_COMPILE_FLAG([-Wsuggest-override], [WARN_CXXFLAGS="$WARN_CXXFLAGS -Wsuggest-override"], [], [$CXXFLAG_WERROR], [AC_LANG_SOURCE([[struct A { virtual void f(); }; struct B : A { void f() final; };]])]) - AX_CHECK_COMPILE_FLAG([-Wunreachable-code-loop-increment],[WARN_CXXFLAGS="$WARN_CXXFLAGS -Wunreachable-code-loop-increment"],,[[$CXXFLAG_WERROR]]) + AX_CHECK_COMPILE_FLAG([-Wunreachable-code-loop-increment], [WARN_CXXFLAGS="$WARN_CXXFLAGS -Wunreachable-code-loop-increment"], [], [$CXXFLAG_WERROR]) AX_CHECK_COMPILE_FLAG([-Wimplicit-fallthrough], [WARN_CXXFLAGS="$WARN_CXXFLAGS -Wimplicit-fallthrough"], [], [$CXXFLAG_WERROR]) - if test x$suppress_external_warnings != xno ; then - AX_CHECK_COMPILE_FLAG([-Wdocumentation],[WARN_CXXFLAGS="$WARN_CXXFLAGS -Wdocumentation"],,[[$CXXFLAG_WERROR]]) + if test "$suppress_external_warnings" != "no" ; then + AX_CHECK_COMPILE_FLAG([-Wdocumentation], [WARN_CXXFLAGS="$WARN_CXXFLAGS -Wdocumentation"], [], [$CXXFLAG_WERROR]) fi dnl Some compilers (gcc) ignore unknown -Wno-* options, but warn about all dnl unknown options if any other warning is produced. Test the -Wfoo case, and dnl set the -Wno-foo case if it works. - AX_CHECK_COMPILE_FLAG([-Wunused-parameter],[NOWARN_CXXFLAGS="$NOWARN_CXXFLAGS -Wno-unused-parameter"],,[[$CXXFLAG_WERROR]]) - AX_CHECK_COMPILE_FLAG([-Wself-assign],[NOWARN_CXXFLAGS="$NOWARN_CXXFLAGS -Wno-self-assign"],,[[$CXXFLAG_WERROR]]) - AX_CHECK_COMPILE_FLAG([-Wunused-local-typedef],[NOWARN_CXXFLAGS="$NOWARN_CXXFLAGS -Wno-unused-local-typedef"],,[[$CXXFLAG_WERROR]]) - if test x$suppress_external_warnings != xyes ; then - AX_CHECK_COMPILE_FLAG([-Wdeprecated-copy],[NOWARN_CXXFLAGS="$NOWARN_CXXFLAGS -Wno-deprecated-copy"],,[[$CXXFLAG_WERROR]]) + AX_CHECK_COMPILE_FLAG([-Wunused-parameter], [NOWARN_CXXFLAGS="$NOWARN_CXXFLAGS -Wno-unused-parameter"], [], [$CXXFLAG_WERROR]) + AX_CHECK_COMPILE_FLAG([-Wself-assign], [NOWARN_CXXFLAGS="$NOWARN_CXXFLAGS -Wno-self-assign"], [], [$CXXFLAG_WERROR]) + if test "$suppress_external_warnings" != "yes" ; then + AX_CHECK_COMPILE_FLAG([-Wdeprecated-copy], [NOWARN_CXXFLAGS="$NOWARN_CXXFLAGS -Wno-deprecated-copy"], [], [$CXXFLAG_WERROR]) fi fi dnl Don't allow extended (non-ASCII) symbols in identifiers. This is easier for code review. -AX_CHECK_COMPILE_FLAG([-fno-extended-identifiers],[[CXXFLAGS="$CXXFLAGS -fno-extended-identifiers"]],,[[$CXXFLAG_WERROR]]) +AX_CHECK_COMPILE_FLAG([-fno-extended-identifiers], [CXXFLAGS="$CXXFLAGS -fno-extended-identifiers"], [], [$CXXFLAG_WERROR]) enable_sse42=no enable_sse41=no enable_avx2=no -enable_shani=no +enable_x86_shani=no -if test "x$use_asm" = "xyes"; then +if test "$use_asm" = "yes"; then dnl Check for optional instruction set support. Enabling these does _not_ imply that all code will dnl be compiled with them, rather that specific objects/libs may use them after checking for runtime dnl compatibility. dnl x86 -AX_CHECK_COMPILE_FLAG([-msse4.2],[[SSE42_CXXFLAGS="-msse4.2"]],,[[$CXXFLAG_WERROR]]) -AX_CHECK_COMPILE_FLAG([-msse4.1],[[SSE41_CXXFLAGS="-msse4.1"]],,[[$CXXFLAG_WERROR]]) -AX_CHECK_COMPILE_FLAG([-mavx -mavx2],[[AVX2_CXXFLAGS="-mavx -mavx2"]],,[[$CXXFLAG_WERROR]]) -AX_CHECK_COMPILE_FLAG([-msse4 -msha],[[SHANI_CXXFLAGS="-msse4 -msha"]],,[[$CXXFLAG_WERROR]]) +AX_CHECK_COMPILE_FLAG([-msse4.2], [SSE42_CXXFLAGS="-msse4.2"], [], [$CXXFLAG_WERROR]) +AX_CHECK_COMPILE_FLAG([-msse4.1], [SSE41_CXXFLAGS="-msse4.1"], [], [$CXXFLAG_WERROR]) +AX_CHECK_COMPILE_FLAG([-mavx -mavx2], [AVX2_CXXFLAGS="-mavx -mavx2"], [], [$CXXFLAG_WERROR]) +AX_CHECK_COMPILE_FLAG([-msse4 -msha], [X86_SHANI_CXXFLAGS="-msse4 -msha"], [], [$CXXFLAG_WERROR]) + +enable_clmul= +AX_CHECK_COMPILE_FLAG([-mpclmul], [enable_clmul=yes], [], [$CXXFLAG_WERROR], [AC_LANG_PROGRAM([ + #include + #include +], [ + __m128i a = _mm_cvtsi64_si128((uint64_t)7); + __m128i b = _mm_clmulepi64_si128(a, a, 37); + __m128i c = _mm_srli_epi64(b, 41); + __m128i d = _mm_xor_si128(b, c); + uint64_t e = _mm_cvtsi128_si64(d); + return e == 0; +])]) + +if test "$enable_clmul" = "yes"; then + CLMUL_CXXFLAGS="-mpclmul" + AC_DEFINE([HAVE_CLMUL], [1], [Define this symbol if clmul instructions can be used]) +fi TEMP_CXXFLAGS="$CXXFLAGS" CXXFLAGS="$CXXFLAGS $SSE42_CXXFLAGS" -AC_MSG_CHECKING(for SSE4.2 intrinsics) +AC_MSG_CHECKING([for SSE4.2 intrinsics]) AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ #include #if defined(_MSC_VER) @@ -521,14 +528,14 @@ AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ l = _mm_crc32_u64(l, 0); return l; ]])], - [ AC_MSG_RESULT(yes); enable_sse42=yes], - [ AC_MSG_RESULT(no)] + [ AC_MSG_RESULT([yes]); enable_sse42=yes], + [ AC_MSG_RESULT([no])] ) CXXFLAGS="$TEMP_CXXFLAGS" TEMP_CXXFLAGS="$CXXFLAGS" CXXFLAGS="$CXXFLAGS $SSE41_CXXFLAGS" -AC_MSG_CHECKING(for SSE4.1 intrinsics) +AC_MSG_CHECKING([for SSE4.1 intrinsics]) AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ #include #include @@ -536,14 +543,14 @@ AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ __m128i l = _mm_set1_epi32(0); return _mm_extract_epi32(l, 3); ]])], - [ AC_MSG_RESULT(yes); enable_sse41=yes; AC_DEFINE(ENABLE_SSE41, 1, [Define this symbol to build code that uses SSE4.1 intrinsics]) ], - [ AC_MSG_RESULT(no)] + [ AC_MSG_RESULT([yes]); enable_sse41=yes; AC_DEFINE([ENABLE_SSE41], [1], [Define this symbol to build code that uses SSE4.1 intrinsics]) ], + [ AC_MSG_RESULT([no])] ) CXXFLAGS="$TEMP_CXXFLAGS" TEMP_CXXFLAGS="$CXXFLAGS" CXXFLAGS="$CXXFLAGS $AVX2_CXXFLAGS" -AC_MSG_CHECKING(for AVX2 intrinsics) +AC_MSG_CHECKING([for AVX2 intrinsics]) AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ #include #include @@ -551,14 +558,14 @@ AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ __m256i l = _mm256_set1_epi32(0); return _mm256_extract_epi32(l, 7); ]])], - [ AC_MSG_RESULT(yes); enable_avx2=yes; AC_DEFINE(ENABLE_AVX2, 1, [Define this symbol to build code that uses AVX2 intrinsics]) ], - [ AC_MSG_RESULT(no)] + [ AC_MSG_RESULT([yes]); enable_avx2=yes; AC_DEFINE([ENABLE_AVX2], [1], [Define this symbol to build code that uses AVX2 intrinsics]) ], + [ AC_MSG_RESULT([no])] ) CXXFLAGS="$TEMP_CXXFLAGS" TEMP_CXXFLAGS="$CXXFLAGS" -CXXFLAGS="$CXXFLAGS $SHANI_CXXFLAGS" -AC_MSG_CHECKING(for SHA-NI intrinsics) +CXXFLAGS="$CXXFLAGS $X86_SHANI_CXXFLAGS" +AC_MSG_CHECKING([for x86 SHA-NI intrinsics]) AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ #include #include @@ -568,32 +575,55 @@ AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ __m128i k = _mm_set1_epi32(2); return _mm_extract_epi32(_mm_sha256rnds2_epu32(i, i, k), 0); ]])], - [ AC_MSG_RESULT(yes); enable_shani=yes; AC_DEFINE(ENABLE_SHANI, 1, [Define this symbol to build code that uses SHA-NI intrinsics]) ], - [ AC_MSG_RESULT(no)] + [ AC_MSG_RESULT([yes]); enable_x86_shani=yes; AC_DEFINE([ENABLE_X86_SHANI], [1], [Define this symbol to build code that uses x86 SHA-NI intrinsics]) ], + [ AC_MSG_RESULT([no])] ) CXXFLAGS="$TEMP_CXXFLAGS" # ARM -AX_CHECK_COMPILE_FLAG([-march=armv8-a+crc+crypto],[[ARM_CRC_CXXFLAGS="-march=armv8-a+crc+crypto"]],,[[$CXXFLAG_WERROR]]) +AX_CHECK_COMPILE_FLAG([-march=armv8-a+crc+crypto], [ARM_CRC_CXXFLAGS="-march=armv8-a+crc+crypto"], [], [$CXXFLAG_WERROR]) +AX_CHECK_COMPILE_FLAG([-march=armv8-a+crc+crypto], [ARM_SHANI_CXXFLAGS="-march=armv8-a+crc+crypto"], [], [$CXXFLAG_WERROR]) TEMP_CXXFLAGS="$CXXFLAGS" CXXFLAGS="$CXXFLAGS $ARM_CRC_CXXFLAGS" -AC_MSG_CHECKING(for ARM CRC32 intrinsics) +AC_MSG_CHECKING([for ARMv8 CRC32 intrinsics]) AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ #include #include ]],[[ +#ifdef __aarch64__ __crc32cb(0, 0); __crc32ch(0, 0); __crc32cw(0, 0); __crc32cd(0, 0); vmull_p64(0, 0); +#else +#error "crc32c library does not support hardware acceleration on 32-bit ARM" +#endif ]])], - [ AC_MSG_RESULT(yes); enable_arm_crc=yes; ], - [ AC_MSG_RESULT(no)] + [ AC_MSG_RESULT([yes]); enable_arm_crc=yes; ], + [ AC_MSG_RESULT([no])] +) +CXXFLAGS="$TEMP_CXXFLAGS" + +TEMP_CXXFLAGS="$CXXFLAGS" +CXXFLAGS="$CXXFLAGS $ARM_SHANI_CXXFLAGS" +AC_MSG_CHECKING([for ARMv8 SHA-NI intrinsics]) +AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ + #include + #include + ]],[[ + uint32x4_t a, b, c; + vsha256h2q_u32(a, b, c); + vsha256hq_u32(a, b, c); + vsha256su0q_u32(a, b); + vsha256su1q_u32(a, b, c); + ]])], + [ AC_MSG_RESULT([yes]); enable_arm_shani=yes; AC_DEFINE([ENABLE_ARM_SHANI], [1], [Define this symbol to build code that uses ARMv8 SHA-NI intrinsics]) ], + [ AC_MSG_RESULT([no])] ) CXXFLAGS="$TEMP_CXXFLAGS" fi -CPPFLAGS="$CPPFLAGS -DHAVE_BUILD_INFO -D__STDC_FORMAT_MACROS" +CPPFLAGS="$CPPFLAGS -DHAVE_BUILD_INFO" AC_ARG_WITH([utils], [AS_HELP_STRING([--with-utils], @@ -625,6 +655,12 @@ AC_ARG_ENABLE([util-util], [build_bitcoin_util=$enableval], [build_bitcoin_util=$build_bitcoin_utils]) +AC_ARG_ENABLE([experimental-util-chainstate], + [AS_HELP_STRING([--enable-experimental-util-chainstate], + [build experimental bitcoin-chainstate executable (default=no)])], + [build_bitcoin_chainstate=$enableval], + [build_bitcoin_chainstate=no]) + AC_ARG_WITH([libs], [AS_HELP_STRING([--with-libs], [build libraries (default=yes)])], @@ -640,33 +676,33 @@ AC_ARG_WITH([daemon], case $host in *mingw*) TARGET_OS=windows - AC_CHECK_LIB([kernel32], [GetModuleFileNameA],, AC_MSG_ERROR(libkernel32 missing)) - AC_CHECK_LIB([user32], [main],, AC_MSG_ERROR(libuser32 missing)) - AC_CHECK_LIB([gdi32], [main],, AC_MSG_ERROR(libgdi32 missing)) - AC_CHECK_LIB([comdlg32], [main],, AC_MSG_ERROR(libcomdlg32 missing)) - AC_CHECK_LIB([winmm], [main],, AC_MSG_ERROR(libwinmm missing)) - AC_CHECK_LIB([shell32], [SHGetSpecialFolderPathW],, AC_MSG_ERROR(libshell32 missing)) - AC_CHECK_LIB([comctl32], [main],, AC_MSG_ERROR(libcomctl32 missing)) - AC_CHECK_LIB([ole32], [CoCreateInstance],, AC_MSG_ERROR(libole32 missing)) - AC_CHECK_LIB([oleaut32], [main],, AC_MSG_ERROR(liboleaut32 missing)) - AC_CHECK_LIB([uuid], [main],, AC_MSG_ERROR(libuuid missing)) - AC_CHECK_LIB([advapi32], [CryptAcquireContextW],, AC_MSG_ERROR(libadvapi32 missing)) - AC_CHECK_LIB([ws2_32], [WSAStartup],, AC_MSG_ERROR(libws2_32 missing)) - AC_CHECK_LIB([shlwapi], [PathRemoveFileSpecW],, AC_MSG_ERROR(libshlwapi missing)) - AC_CHECK_LIB([iphlpapi], [GetAdaptersAddresses],, AC_MSG_ERROR(libiphlpapi missing)) + AC_CHECK_LIB([kernel32], [GetModuleFileNameA], [], [AC_MSG_ERROR([libkernel32 missing])]) + AC_CHECK_LIB([user32], [main], [], [AC_MSG_ERROR([libuser32 missing])]) + AC_CHECK_LIB([gdi32], [main], [], [AC_MSG_ERROR([libgdi32 missing])]) + AC_CHECK_LIB([comdlg32], [main], [], [AC_MSG_ERROR([libcomdlg32 missing])]) + AC_CHECK_LIB([winmm], [main], [], [AC_MSG_ERROR([libwinmm missing])]) + AC_CHECK_LIB([shell32], [SHGetSpecialFolderPathW], [], [AC_MSG_ERROR([libshell32 missing])]) + AC_CHECK_LIB([comctl32], [main], [], [AC_MSG_ERROR([libcomctl32 missing])]) + AC_CHECK_LIB([ole32], [CoCreateInstance], [], [AC_MSG_ERROR([libole32 missing])]) + AC_CHECK_LIB([oleaut32], [main], [], [AC_MSG_ERROR([liboleaut32 missing])]) + AC_CHECK_LIB([uuid], [main], [], [AC_MSG_ERROR([libuuid missing])]) + AC_CHECK_LIB([advapi32], [CryptAcquireContextW], [], [AC_MSG_ERROR([libadvapi32 missing])]) + AC_CHECK_LIB([ws2_32], [WSAStartup], [], [AC_MSG_ERROR([libws2_32 missing])]) + AC_CHECK_LIB([shlwapi], [PathRemoveFileSpecW], [], [AC_MSG_ERROR([libshlwapi missing])]) + AC_CHECK_LIB([iphlpapi], [GetAdaptersAddresses], [], [AC_MSG_ERROR([libiphlpapi missing])]) dnl -static is interpreted by libtool, where it has a different meaning. dnl In libtool-speak, it's -all-static. - AX_CHECK_LINK_FLAG([[-static]],[LIBTOOL_APP_LDFLAGS="$LIBTOOL_APP_LDFLAGS -all-static"]) + AX_CHECK_LINK_FLAG([-static], [LIBTOOL_APP_LDFLAGS="$LIBTOOL_APP_LDFLAGS -all-static"]) - AC_PATH_PROG([MAKENSIS], [makensis], none) - if test x$MAKENSIS = xnone; then - AC_MSG_WARN("makensis not found. Cannot create installer.") + AC_PATH_PROG([MAKENSIS], [makensis], [none]) + if test "$MAKENSIS" = "none"; then + AC_MSG_WARN([makensis not found. Cannot create installer.]) fi - AC_PATH_TOOL(WINDRES, windres, none) - if test x$WINDRES = xnone; then - AC_MSG_ERROR("windres not found") + AC_PATH_TOOL([WINDRES], [windres], [none]) + if test "$WINDRES" = "none"; then + AC_MSG_ERROR([windres not found]) fi CPPFLAGS="$CPPFLAGS -DSECP256K1_STATIC" @@ -681,37 +717,63 @@ case $host in postdeps_CXX= dnl We require Windows 7 (NT 6.1) or later - AX_CHECK_LINK_FLAG([[-Wl,--major-subsystem-version -Wl,6 -Wl,--minor-subsystem-version -Wl,1]],[LDFLAGS="$LDFLAGS -Wl,--major-subsystem-version -Wl,6 -Wl,--minor-subsystem-version -Wl,1"],,[[$LDFLAG_WERROR]]) + AX_CHECK_LINK_FLAG([-Wl,--major-subsystem-version -Wl,6 -Wl,--minor-subsystem-version -Wl,1], [LDFLAGS="$LDFLAGS -Wl,--major-subsystem-version -Wl,6 -Wl,--minor-subsystem-version -Wl,1"], [], [$LDFLAG_WERROR]) ;; *darwin*) TARGET_OS=darwin - if test x$cross_compiling != xyes; then + if test $cross_compiling != "yes"; then BUILD_OS=darwin - AC_PATH_PROGS([RSVG_CONVERT], [rsvg-convert rsvg],rsvg-convert) - AC_CHECK_PROG([BREW],brew, brew) - if test x$BREW = xbrew; then + AC_CHECK_PROG([BREW], [brew], [brew]) + if test "$BREW" = "brew"; then dnl These Homebrew packages may be keg-only, meaning that they won't be found dnl in expected paths because they may conflict with system files. Ask dnl Homebrew where each one is located, then adjust paths accordingly. dnl It's safe to add these paths even if the functionality is disabled by dnl the user (--without-wallet or --without-gui for example). - if test "x$use_bdb" != xno && $BREW list --versions berkeley-db4 >/dev/null && test "x$BDB_CFLAGS" = "x" && test "x$BDB_LIBS" = "x"; then - bdb_prefix=$($BREW --prefix berkeley-db4 2>/dev/null) + if test "$use_bdb" != "no" && $BREW list --versions berkeley-db@4 >/dev/null && test "$BDB_CFLAGS" = "" && test "$BDB_LIBS" = ""; then + bdb_prefix=$($BREW --prefix berkeley-db@4 2>/dev/null) dnl This must precede the call to BITCOIN_FIND_BDB48 below. BDB_CFLAGS="-I$bdb_prefix/include" BDB_LIBS="-L$bdb_prefix/lib -ldb_cxx-4.8" fi - if test "x$use_sqlite" != xno && $BREW list --versions sqlite3 >/dev/null; then + if test "$use_sqlite" != "no" && $BREW list --versions sqlite3 >/dev/null; then export PKG_CONFIG_PATH="$($BREW --prefix sqlite3 2>/dev/null)/lib/pkgconfig:$PKG_CONFIG_PATH" fi - if $BREW list --versions qt5 >/dev/null; then - export PKG_CONFIG_PATH="$($BREW --prefix qt5 2>/dev/null)/lib/pkgconfig:$PKG_CONFIG_PATH" + if $BREW list --versions qt@5 >/dev/null; then + export PKG_CONFIG_PATH="$($BREW --prefix qt@5 2>/dev/null)/lib/pkgconfig:$PKG_CONFIG_PATH" fi dnl On some versions of osx stack check is turned on by default and is broken AX_CHECK_COMPILE_FLAG([-fno-stack-check],[HARDENED_CXXFLAGS="$HARDENED_CXXFLAGS -fno-stack-check"]) + + case $host in + *aarch64*) + dnl The preferred Homebrew prefix for Apple Silicon is /opt/homebrew. + dnl Therefore, as we do not use pkg-config to detect miniupnpc and libnatpmp + dnl packages, we should set the CPPFLAGS and LDFLAGS variables for them + dnl explicitly. + if test "$use_upnp" != "no" && $BREW list --versions miniupnpc >/dev/null; then + miniupnpc_prefix=$($BREW --prefix miniupnpc 2>/dev/null) + if test "$suppress_external_warnings" != "no"; then + CPPFLAGS="$CPPFLAGS -isystem $miniupnpc_prefix/include" + else + CPPFLAGS="$CPPFLAGS -I$miniupnpc_prefix/include" + fi + LDFLAGS="$LDFLAGS -L$miniupnpc_prefix/lib" + fi + if test "$use_natpmp" != "no" && $BREW list --versions libnatpmp >/dev/null; then + libnatpmp_prefix=$($BREW --prefix libnatpmp 2>/dev/null) + if test "$suppress_external_warnings" != "no"; then + CPPFLAGS="$CPPFLAGS -isystem $libnatpmp_prefix/include" + else + CPPFLAGS="$CPPFLAGS -I$libnatpmp_prefix/include" + fi + LDFLAGS="$LDFLAGS -L$libnatpmp_prefix/lib" + fi + ;; + esac fi else case $build_os in @@ -719,14 +781,11 @@ case $host in BUILD_OS=darwin ;; *) - AC_PATH_TOOL([DSYMUTIL], [dsymutil], dsymutil) - AC_PATH_TOOL([INSTALLNAMETOOL], [install_name_tool], install_name_tool) - AC_PATH_TOOL([OTOOL], [otool], otool) - AC_PATH_PROGS([XORRISOFS], [xorrisofs], xorrisofs) - AC_PATH_PROGS([DMG], [dmg], dmg) - AC_PATH_PROGS([RSVG_CONVERT], [rsvg-convert rsvg],rsvg-convert) - AC_PATH_PROGS([IMAGEMAGICK_CONVERT], [convert],convert) - AC_PATH_PROGS([TIFFCP], [tiffcp],tiffcp) + AC_PATH_TOOL([DSYMUTIL], [dsymutil], [dsymutil]) + AC_PATH_TOOL([INSTALLNAMETOOL], [install_name_tool], [install_name_tool]) + AC_PATH_TOOL([OTOOL], [otool], [otool]) + AC_PATH_PROGS([XORRISOFS], [xorrisofs], [xorrisofs]) + AC_PATH_PROGS([DMG], [dmg], [dmg]) dnl libtool will try to strip the static lib, which is a problem for dnl cross-builds because strip attempts to call a hard-coded ld, @@ -737,7 +796,7 @@ case $host in esac fi - AX_CHECK_LINK_FLAG([[-Wl,-headerpad_max_install_names]], [LDFLAGS="$LDFLAGS -Wl,-headerpad_max_install_names"],, [[$LDFLAG_WERROR]]) + AX_CHECK_LINK_FLAG([-Wl,-headerpad_max_install_names], [LDFLAGS="$LDFLAGS -Wl,-headerpad_max_install_names"], [], [$LDFLAG_WERROR]) CPPFLAGS="$CPPFLAGS -DMAC_OSX -DOBJC_OLD_DISPATCH_PROTOTYPES=0" OBJCXXFLAGS="$CXXFLAGS" ;; @@ -754,10 +813,7 @@ case $host in *armv7a*) ANDROID_ARCH=armeabi-v7a ;; - *i686*) - ANDROID_ARCH=i686 - ;; - *) AC_MSG_ERROR("Could not determine Android arch") ;; + *) AC_MSG_ERROR([Could not determine Android arch, or it is unsupported]) ;; esac ;; *linux*) @@ -765,19 +821,19 @@ case $host in ;; esac -if test x$use_extended_functional_tests != xno; then +if test "$use_extended_functional_tests" != "no"; then AC_SUBST(EXTENDED_FUNCTIONAL_TESTS, --extended) fi -if test x$use_lcov = xyes; then - if test x$LCOV = x; then - AC_MSG_ERROR("lcov testing requested but lcov not found") +if test "$use_lcov" = "yes"; then + if test "$LCOV" = ""; then + AC_MSG_ERROR([lcov testing requested but lcov not found]) fi - if test x$PYTHON = x; then - AC_MSG_ERROR("lcov testing requested but python not found") + if test "$PYTHON" = ""; then + AC_MSG_ERROR([lcov testing requested but python not found]) fi - if test x$GENHTML = x; then - AC_MSG_ERROR("lcov testing requested but genhtml not found") + if test "$GENHTML" = ""; then + AC_MSG_ERROR([lcov testing requested but genhtml not found]) fi AC_MSG_CHECKING([whether compiler is Clang]) @@ -789,13 +845,13 @@ if test x$use_lcov = xyes; then #endif ]])],[ AC_MSG_RESULT([yes]) - if test x$LLVM_COV = x; then + if test "$LLVM_COV" = ""; then AC_MSG_ERROR([lcov testing requested but llvm-cov not found]) fi COV_TOOL="$LLVM_COV gcov" ],[ AC_MSG_RESULT([no]) - if test x$GCOV = x; then + if test "$GCOV" = "x"; then AC_MSG_ERROR([lcov testing requested but gcov not found]) fi COV_TOOL="$GCOV" @@ -804,20 +860,17 @@ if test x$use_lcov = xyes; then AC_SUBST(COV_TOOL_WRAPPER, "cov_tool_wrapper.sh") LCOV="$LCOV --gcov-tool $(pwd)/$COV_TOOL_WRAPPER" - AX_CHECK_LINK_FLAG([[--coverage]], [LDFLAGS="$LDFLAGS --coverage"], - [AC_MSG_ERROR("lcov testing requested but --coverage linker flag does not work")]) + AX_CHECK_LINK_FLAG([--coverage], [LDFLAGS="$LDFLAGS --coverage"], + [AC_MSG_ERROR([lcov testing requested but --coverage linker flag does not work])]) AX_CHECK_COMPILE_FLAG([--coverage],[CXXFLAGS="$CXXFLAGS --coverage"], - [AC_MSG_ERROR("lcov testing requested but --coverage flag does not work")]) + [AC_MSG_ERROR([lcov testing requested but --coverage flag does not work])]) CXXFLAGS="$CXXFLAGS -Og" fi -if test x$use_lcov_branch != xno; then +if test "$use_lcov_branch" != "no"; then AC_SUBST(LCOV_OPTS, "$LCOV_OPTS --rc lcov_branch_coverage=1") fi -dnl Check for __int128 -AC_CHECK_TYPES([__int128]) - dnl Check for endianness AC_C_BIGENDIAN @@ -831,65 +884,55 @@ AC_SYS_LARGEFILE dnl detect POSIX or GNU variant of strerror_r AC_FUNC_STRERROR_R -if test x$ac_cv_sys_file_offset_bits != x && - test x$ac_cv_sys_file_offset_bits != xno && - test x$ac_cv_sys_file_offset_bits != xunknown; then +if test "$ac_cv_sys_file_offset_bits" != "" && + test "$ac_cv_sys_file_offset_bits" != "no" && + test "$ac_cv_sys_file_offset_bits" != "unknown"; then CPPFLAGS="$CPPFLAGS -D_FILE_OFFSET_BITS=$ac_cv_sys_file_offset_bits" fi -if test x$ac_cv_sys_large_files != x && - test x$ac_cv_sys_large_files != xno && - test x$ac_cv_sys_large_files != xunknown; then +if test "$ac_cv_sys_large_files" != "" && + test "$ac_cv_sys_large_files" != "no" && + test "$ac_cv_sys_large_files" != "unknown"; then CPPFLAGS="$CPPFLAGS -D_LARGE_FILES=$ac_cv_sys_large_files" fi -if test x$use_glibc_compat != xno; then - AX_CHECK_LINK_FLAG([[-Wl,--wrap=__divmoddi4]], [COMPAT_LDFLAGS="$COMPAT_LDFLAGS -Wl,--wrap=__divmoddi4"]) - AX_CHECK_LINK_FLAG([[-Wl,--wrap=log2f]], [COMPAT_LDFLAGS="$COMPAT_LDFLAGS -Wl,--wrap=log2f"]) - case $host in - powerpc64* | ppc64*) - AX_CHECK_LINK_FLAG([[-Wl,--no-tls-get-addr-optimize]], [COMPAT_LDFLAGS="$COMPAT_LDFLAGS -Wl,--no-tls-get-addr-optimize"]) - ;; - esac -else - AC_SEARCH_LIBS([clock_gettime],[rt]) -fi +AC_SEARCH_LIBS([clock_gettime],[rt]) -if test "x$enable_gprof" = xyes; then +if test "$enable_gprof" = "yes"; then dnl -pg is incompatible with -pie. Since hardening and profiling together doesn't make sense, dnl we simply make them mutually exclusive here. Additionally, hardened toolchains may force dnl -pie by default, in which case it needs to be turned off with -no-pie. - if test x$use_hardening = xyes; then - AC_MSG_ERROR(gprof profiling is not compatible with hardening. Reconfigure with --disable-hardening or --disable-gprof) + if test "$use_hardening" = "yes"; then + AC_MSG_ERROR([gprof profiling is not compatible with hardening. Reconfigure with --disable-hardening or --disable-gprof]) fi use_hardening=no AX_CHECK_COMPILE_FLAG([-pg],[GPROF_CXXFLAGS="-pg"], - [AC_MSG_ERROR(gprof profiling requested but not available)], [[$CXXFLAG_WERROR]]) + [AC_MSG_ERROR([gprof profiling requested but not available])], [$CXXFLAG_WERROR]) - AX_CHECK_LINK_FLAG([[-no-pie]], [GPROF_LDFLAGS="-no-pie"]) - AX_CHECK_LINK_FLAG([[-pg]],[GPROF_LDFLAGS="$GPROF_LDFLAGS -pg"], - [AC_MSG_ERROR(gprof profiling requested but not available)], [[$GPROF_LDFLAGS]]) + AX_CHECK_LINK_FLAG([-no-pie], [GPROF_LDFLAGS="-no-pie"]) + AX_CHECK_LINK_FLAG([-pg], [GPROF_LDFLAGS="$GPROF_LDFLAGS -pg"], + [AC_MSG_ERROR([gprof profiling requested but not available])], [$GPROF_LDFLAGS]) fi -if test x$TARGET_OS != xwindows; then +if test "$TARGET_OS" != "windows"; then dnl All windows code is PIC, forcing it on just adds useless compile warnings - AX_CHECK_COMPILE_FLAG([-fPIC],[PIC_FLAGS="-fPIC"]) + AX_CHECK_COMPILE_FLAG([-fPIC], [PIC_FLAGS="-fPIC"]) fi dnl All versions of gcc that we commonly use for building are subject to bug dnl https://gcc.gnu.org/bugzilla/show_bug.cgi?id=90348. To work around that, set dnl -fstack-reuse=none for all gcc builds. (Only gcc understands this flag) -AX_CHECK_COMPILE_FLAG([-fstack-reuse=none],[HARDENED_CXXFLAGS="$HARDENED_CXXFLAGS -fstack-reuse=none"]) -if test x$use_hardening != xno; then +AX_CHECK_COMPILE_FLAG([-fstack-reuse=none], [HARDENED_CXXFLAGS="$HARDENED_CXXFLAGS -fstack-reuse=none"]) +if test "$use_hardening" != "no"; then use_hardening=yes - AX_CHECK_COMPILE_FLAG([-Wstack-protector],[HARDENED_CXXFLAGS="$HARDENED_CXXFLAGS -Wstack-protector"]) - AX_CHECK_COMPILE_FLAG([-fstack-protector-all],[HARDENED_CXXFLAGS="$HARDENED_CXXFLAGS -fstack-protector-all"]) + AX_CHECK_COMPILE_FLAG([-Wstack-protector], [HARDENED_CXXFLAGS="$HARDENED_CXXFLAGS -Wstack-protector"]) + AX_CHECK_COMPILE_FLAG([-fstack-protector-all], [HARDENED_CXXFLAGS="$HARDENED_CXXFLAGS -fstack-protector-all"]) dnl -fcf-protection used with Clang 7 causes ld to emit warnings: dnl ld: error: ... dnl Use CHECK_LINK_FLAG & --fatal-warnings to ensure we won't use the flag in this case. - AX_CHECK_LINK_FLAG([-fcf-protection=full],[HARDENED_CXXFLAGS="$HARDENED_CXXFLAGS -fcf-protection=full"],, [[$LDFLAG_WERROR]]) + AX_CHECK_LINK_FLAG([-fcf-protection=full], [HARDENED_CXXFLAGS="$HARDENED_CXXFLAGS -fcf-protection=full"], [], [$LDFLAG_WERROR]) case $host in *mingw*) @@ -905,7 +948,7 @@ if test x$use_hardening != xno; then dnl When enable_debug is yes, all optimizations are disabled. dnl However, FORTIFY_SOURCE requires that there is some level of optimization, otherwise it does nothing and just creates a compiler warning. dnl Since FORTIFY_SOURCE is a no-op without optimizations, do not enable it when enable_debug is yes. - if test x$enable_debug != xyes; then + if test "$enable_debug" != "yes"; then AX_CHECK_PREPROC_FLAG([-D_FORTIFY_SOURCE=2],[ AX_CHECK_PREPROC_FLAG([-U_FORTIFY_SOURCE],[ HARDENED_CPPFLAGS="$HARDENED_CPPFLAGS -U_FORTIFY_SOURCE" @@ -914,18 +957,18 @@ if test x$use_hardening != xno; then ]) fi - AX_CHECK_LINK_FLAG([[-Wl,--enable-reloc-section]], [HARDENED_LDFLAGS="$HARDENED_LDFLAGS -Wl,--enable-reloc-section"],, [[$LDFLAG_WERROR]]) - AX_CHECK_LINK_FLAG([[-Wl,--dynamicbase]], [HARDENED_LDFLAGS="$HARDENED_LDFLAGS -Wl,--dynamicbase"],, [[$LDFLAG_WERROR]]) - AX_CHECK_LINK_FLAG([[-Wl,--nxcompat]], [HARDENED_LDFLAGS="$HARDENED_LDFLAGS -Wl,--nxcompat"],, [[$LDFLAG_WERROR]]) - AX_CHECK_LINK_FLAG([[-Wl,--high-entropy-va]], [HARDENED_LDFLAGS="$HARDENED_LDFLAGS -Wl,--high-entropy-va"],, [[$LDFLAG_WERROR]]) - AX_CHECK_LINK_FLAG([[-Wl,-z,relro]], [HARDENED_LDFLAGS="$HARDENED_LDFLAGS -Wl,-z,relro"],, [[$LDFLAG_WERROR]]) - AX_CHECK_LINK_FLAG([[-Wl,-z,now]], [HARDENED_LDFLAGS="$HARDENED_LDFLAGS -Wl,-z,now"],, [[$LDFLAG_WERROR]]) - AX_CHECK_LINK_FLAG([[-Wl,-z,separate-code]], [HARDENED_LDFLAGS="$HARDENED_LDFLAGS -Wl,-z,separate-code"],, [[$LDFLAG_WERROR]]) - AX_CHECK_LINK_FLAG([[-fPIE -pie]], [PIE_FLAGS="-fPIE"; HARDENED_LDFLAGS="$HARDENED_LDFLAGS -pie"],, [[$CXXFLAG_WERROR]]) + AX_CHECK_LINK_FLAG([-Wl,--enable-reloc-section], [HARDENED_LDFLAGS="$HARDENED_LDFLAGS -Wl,--enable-reloc-section"], [], [$LDFLAG_WERROR]) + AX_CHECK_LINK_FLAG([-Wl,--dynamicbase], [HARDENED_LDFLAGS="$HARDENED_LDFLAGS -Wl,--dynamicbase"], [], [$LDFLAG_WERROR]) + AX_CHECK_LINK_FLAG([-Wl,--nxcompat], [HARDENED_LDFLAGS="$HARDENED_LDFLAGS -Wl,--nxcompat"], [], [$LDFLAG_WERROR]) + AX_CHECK_LINK_FLAG([-Wl,--high-entropy-va], [HARDENED_LDFLAGS="$HARDENED_LDFLAGS -Wl,--high-entropy-va"], [], [$LDFLAG_WERROR]) + AX_CHECK_LINK_FLAG([-Wl,-z,relro], [HARDENED_LDFLAGS="$HARDENED_LDFLAGS -Wl,-z,relro"], [], [$LDFLAG_WERROR]) + AX_CHECK_LINK_FLAG([-Wl,-z,now], [HARDENED_LDFLAGS="$HARDENED_LDFLAGS -Wl,-z,now"], [], [$LDFLAG_WERROR]) + AX_CHECK_LINK_FLAG([-Wl,-z,separate-code], [HARDENED_LDFLAGS="$HARDENED_LDFLAGS -Wl,-z,separate-code"], [], [$LDFLAG_WERROR]) + AX_CHECK_LINK_FLAG([-fPIE -pie], [PIE_FLAGS="-fPIE"; HARDENED_LDFLAGS="$HARDENED_LDFLAGS -pie"], [], [$CXXFLAG_WERROR]) case $host in *mingw*) - AC_CHECK_LIB([ssp], [main],, AC_MSG_ERROR(libssp missing)) + AC_CHECK_LIB([ssp], [main], [], [AC_MSG_ERROR([libssp missing])]) ;; esac fi @@ -933,10 +976,10 @@ fi dnl These flags are specific to ld64, and may cause issues with other linkers. dnl For example: GNU ld will interpret -dead_strip as -de and then try and use dnl "ad_strip" as the symbol for the entry point. -if test x$TARGET_OS = xdarwin; then - AX_CHECK_LINK_FLAG([[-Wl,-dead_strip]], [LDFLAGS="$LDFLAGS -Wl,-dead_strip"],, [[$LDFLAG_WERROR]]) - AX_CHECK_LINK_FLAG([[-Wl,-dead_strip_dylibs]], [LDFLAGS="$LDFLAGS -Wl,-dead_strip_dylibs"],, [[$LDFLAG_WERROR]]) - AX_CHECK_LINK_FLAG([[-Wl,-bind_at_load]], [HARDENED_LDFLAGS="$HARDENED_LDFLAGS -Wl,-bind_at_load"],, [[$LDFLAG_WERROR]]) +if test "$TARGET_OS" = "darwin"; then + AX_CHECK_LINK_FLAG([-Wl,-dead_strip], [LDFLAGS="$LDFLAGS -Wl,-dead_strip"], [], [$LDFLAG_WERROR]) + AX_CHECK_LINK_FLAG([-Wl,-dead_strip_dylibs], [LDFLAGS="$LDFLAGS -Wl,-dead_strip_dylibs"], [], [$LDFLAG_WERROR]) + AX_CHECK_LINK_FLAG([-Wl,-bind_at_load], [HARDENED_LDFLAGS="$HARDENED_LDFLAGS -Wl,-bind_at_load"], [], [$LDFLAG_WERROR]) fi AC_CHECK_HEADERS([endian.h sys/endian.h byteswap.h stdio.h stdlib.h unistd.h strings.h sys/types.h sys/stat.h sys/select.h sys/prctl.h sys/sysctl.h vm/vm_param.h sys/vmmeter.h sys/resources.h]) @@ -954,51 +997,51 @@ AC_CHECK_DECLS([setsid]) AC_CHECK_DECLS([pipe2]) AC_CHECK_DECLS([le16toh, le32toh, le64toh, htole16, htole32, htole64, be16toh, be32toh, be64toh, htobe16, htobe32, htobe64],,, - [#if HAVE_ENDIAN_H + [#if HAVE_ENDIAN_H #include #elif HAVE_SYS_ENDIAN_H #include #endif]) AC_CHECK_DECLS([bswap_16, bswap_32, bswap_64],,, - [#if HAVE_BYTESWAP_H + [#if HAVE_BYTESWAP_H #include #endif]) -AC_MSG_CHECKING(for __builtin_clzl) +AC_MSG_CHECKING([for __builtin_clzl]) AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ ]], [[ (void) __builtin_clzl(0); ]])], - [ AC_MSG_RESULT(yes); AC_DEFINE(HAVE_BUILTIN_CLZL, 1, [Define this symbol if you have __builtin_clzl])], - [ AC_MSG_RESULT(no)] + [ AC_MSG_RESULT([yes]); have_clzl=yes; AC_DEFINE([HAVE_BUILTIN_CLZL], [1], [Define this symbol if you have __builtin_clzl])], + [ AC_MSG_RESULT([no]); have_clzl=no;] ) -AC_MSG_CHECKING(for __builtin_clzll) +AC_MSG_CHECKING([for __builtin_clzll]) AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ ]], [[ (void) __builtin_clzll(0); ]])], - [ AC_MSG_RESULT(yes); AC_DEFINE(HAVE_BUILTIN_CLZLL, 1, [Define this symbol if you have __builtin_clzll])], - [ AC_MSG_RESULT(no)] + [ AC_MSG_RESULT([yes]); have_clzll=yes; AC_DEFINE([HAVE_BUILTIN_CLZLL], [1], [Define this symbol if you have __builtin_clzll])], + [ AC_MSG_RESULT([no]); have_clzll=no;] ) dnl Check for malloc_info (for memory statistics information in getmemoryinfo) -AC_MSG_CHECKING(for getmemoryinfo) +AC_MSG_CHECKING([for getmemoryinfo]) AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include ]], [[ int f = malloc_info(0, NULL); ]])], - [ AC_MSG_RESULT(yes); AC_DEFINE(HAVE_MALLOC_INFO, 1,[Define this symbol if you have malloc_info]) ], - [ AC_MSG_RESULT(no)] + [ AC_MSG_RESULT([yes]); AC_DEFINE([HAVE_MALLOC_INFO], [1], [Define this symbol if you have malloc_info]) ], + [ AC_MSG_RESULT([no])] ) dnl Check for mallopt(M_ARENA_MAX) (to set glibc arenas) -AC_MSG_CHECKING(for mallopt M_ARENA_MAX) +AC_MSG_CHECKING([for mallopt M_ARENA_MAX]) AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include ]], [[ mallopt(M_ARENA_MAX, 1); ]])], - [ AC_MSG_RESULT(yes); AC_DEFINE(HAVE_MALLOPT_ARENA_MAX, 1,[Define this symbol if you have mallopt with M_ARENA_MAX]) ], - [ AC_MSG_RESULT(no)] + [ AC_MSG_RESULT([yes]); AC_DEFINE([HAVE_MALLOPT_ARENA_MAX], [1], [Define this symbol if you have mallopt with M_ARENA_MAX]) ], + [ AC_MSG_RESULT([no])] ) dnl Check for posix_fallocate -AC_MSG_CHECKING(for posix_fallocate) +AC_MSG_CHECKING([for posix_fallocate]) AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ // same as in src/util/system.cpp #ifdef __linux__ @@ -1009,8 +1052,8 @@ AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ #endif // __linux__ #include ]], [[ int f = posix_fallocate(0, 0, 0); ]])], - [ AC_MSG_RESULT(yes); AC_DEFINE(HAVE_POSIX_FALLOCATE, 1,[Define this symbol if you have posix_fallocate]) ], - [ AC_MSG_RESULT(no)] + [ AC_MSG_RESULT([yes]); AC_DEFINE([HAVE_POSIX_FALLOCATE], [1], [Define this symbol if you have posix_fallocate]) ], + [ AC_MSG_RESULT([no])] ) AC_MSG_CHECKING([for default visibility attribute]) @@ -1019,12 +1062,12 @@ AC_COMPILE_IFELSE([AC_LANG_SOURCE([ int main(){} ])], [ - AC_DEFINE(HAVE_DEFAULT_VISIBILITY_ATTRIBUTE,1,[Define if the visibility attribute is supported.]) - AC_MSG_RESULT(yes) + AC_DEFINE([HAVE_DEFAULT_VISIBILITY_ATTRIBUTE], [1], [Define if the visibility attribute is supported.]) + AC_MSG_RESULT([yes]) ], [ - AC_MSG_RESULT(no) - if test x$use_reduce_exports = xyes; then + AC_MSG_RESULT([no]) + if test "$use_reduce_exports" = "yes"; then AC_MSG_ERROR([Cannot find a working visibility attribute. Use --disable-reduce-exports.]) fi ] @@ -1036,16 +1079,13 @@ AC_COMPILE_IFELSE([AC_LANG_SOURCE([ int main(){} ])], [ - AC_DEFINE(HAVE_DLLEXPORT_ATTRIBUTE,1,[Define if the dllexport attribute is supported.]) - AC_MSG_RESULT(yes) + AC_DEFINE([HAVE_DLLEXPORT_ATTRIBUTE], [1], [Define if the dllexport attribute is supported.]) + AC_MSG_RESULT([yes]) ], - [AC_MSG_RESULT(no)] + [AC_MSG_RESULT([no])] ) -dnl thread_local is currently disabled when building with glibc back compat. -dnl Our minimum supported glibc is 2.17, however support for thread_local -dnl did not arrive in glibc until 2.18. -if test "x$use_thread_local" = xyes || { test "x$use_thread_local" = xauto && test "x$use_glibc_compat" = xno; }; then +if test "$use_thread_local" = "yes" || test "$use_thread_local" = "auto"; then TEMP_LDFLAGS="$LDFLAGS" LDFLAGS="$TEMP_LDFLAGS $PTHREAD_CFLAGS" AC_MSG_CHECKING([for thread_local support]) @@ -1064,21 +1104,21 @@ if test "x$use_thread_local" = xyes || { test "x$use_thread_local" = xauto && te dnl mingw32's implementation of thread_local has also been shown to behave dnl erroneously under concurrent usage; see: dnl https://gist.github.com/jamesob/fe9a872051a88b2025b1aa37bfa98605 - AC_MSG_RESULT(no) + AC_MSG_RESULT([no]) ;; *freebsd*) dnl FreeBSD's implementation of thread_local is also buggy (per dnl https://groups.google.com/d/msg/bsdmailinglist/22ncTZAbDp4/Dii_pII5AwAJ) - AC_MSG_RESULT(no) + AC_MSG_RESULT([no]) ;; *) - AC_DEFINE(HAVE_THREAD_LOCAL,1,[Define if thread_local is supported.]) - AC_MSG_RESULT(yes) + AC_DEFINE([HAVE_THREAD_LOCAL], [1], [Define if thread_local is supported.]) + AC_MSG_RESULT([yes]) ;; esac ], [ - AC_MSG_RESULT(no) + AC_MSG_RESULT([no]) ] ) LDFLAGS="$TEMP_LDFLAGS" @@ -1086,57 +1126,50 @@ fi dnl check for gmtime_r(), fallback to gmtime_s() if that is unavailable dnl fail if neither are available. -AC_MSG_CHECKING(for gmtime_r) +AC_MSG_CHECKING([for gmtime_r]) AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include ]], [[ gmtime_r((const time_t *) nullptr, (struct tm *) nullptr); ]])], - [ AC_MSG_RESULT(yes); AC_DEFINE(HAVE_GMTIME_R, 1, [Define this symbol if gmtime_r is available]) ], - [ AC_MSG_RESULT(no); - AC_MSG_CHECKING(for gmtime_s); + [ AC_MSG_RESULT([yes]); AC_DEFINE([HAVE_GMTIME_R], [1], [Define this symbol if gmtime_r is available]) ], + [ AC_MSG_RESULT([no]); + AC_MSG_CHECKING([for gmtime_s]); AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include ]], [[ gmtime_s((struct tm *) nullptr, (const time_t *) nullptr); ]])], - [ AC_MSG_RESULT(yes)], - [ AC_MSG_RESULT(no); AC_MSG_ERROR(Both gmtime_r and gmtime_s are unavailable) ] + [ AC_MSG_RESULT([yes])], + [ AC_MSG_RESULT([no]); AC_MSG_ERROR([Both gmtime_r and gmtime_s are unavailable]) ] ) ] ) dnl Check for different ways of gathering OS randomness -AC_MSG_CHECKING(for Linux getrandom syscall) +AC_MSG_CHECKING([for Linux getrandom syscall]) AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include #include #include ]], [[ syscall(SYS_getrandom, nullptr, 32, 0); ]])], - [ AC_MSG_RESULT(yes); AC_DEFINE(HAVE_SYS_GETRANDOM, 1,[Define this symbol if the Linux getrandom system call is available]) ], - [ AC_MSG_RESULT(no)] + [ AC_MSG_RESULT([yes]); AC_DEFINE([HAVE_SYS_GETRANDOM], [1], [Define this symbol if the Linux getrandom system call is available]) ], + [ AC_MSG_RESULT([no])] ) -AC_MSG_CHECKING(for getentropy) -AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include ]], - [[ getentropy(nullptr, 32) ]])], - [ AC_MSG_RESULT(yes); AC_DEFINE(HAVE_GETENTROPY, 1,[Define this symbol if the BSD getentropy system call is available]) ], - [ AC_MSG_RESULT(no)] -) - -AC_MSG_CHECKING(for getentropy via random.h) +AC_MSG_CHECKING([for getentropy via random.h]) AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include #include ]], [[ getentropy(nullptr, 32) ]])], - [ AC_MSG_RESULT(yes); AC_DEFINE(HAVE_GETENTROPY_RAND, 1,[Define this symbol if the BSD getentropy system call is available with sys/random.h]) ], - [ AC_MSG_RESULT(no)] + [ AC_MSG_RESULT([yes]); AC_DEFINE([HAVE_GETENTROPY_RAND], [1], [Define this symbol if the BSD getentropy system call is available with sys/random.h]) ], + [ AC_MSG_RESULT([no])] ) -AC_MSG_CHECKING(for sysctl) +AC_MSG_CHECKING([for sysctl]) AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include #include ]], [[ #ifdef __linux__ #error "Don't use sysctl on Linux, it's deprecated even when it works" #endif sysctl(nullptr, 2, nullptr, nullptr, nullptr, 0); ]])], - [ AC_MSG_RESULT(yes); AC_DEFINE(HAVE_SYSCTL, 1,[Define this symbol if the BSD sysctl() is available]) ], - [ AC_MSG_RESULT(no)] + [ AC_MSG_RESULT([yes]); AC_DEFINE([HAVE_SYSCTL], [1], [Define this symbol if the BSD sysctl() is available]) ], + [ AC_MSG_RESULT([no])] ) -AC_MSG_CHECKING(for sysctl KERN_ARND) +AC_MSG_CHECKING([for sysctl KERN_ARND]) AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include #include ]], [[ #ifdef __linux__ @@ -1144,95 +1177,81 @@ AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include #endif static int name[2] = {CTL_KERN, KERN_ARND}; sysctl(name, 2, nullptr, nullptr, nullptr, 0); ]])], - [ AC_MSG_RESULT(yes); AC_DEFINE(HAVE_SYSCTL_ARND, 1,[Define this symbol if the BSD sysctl(KERN_ARND) is available]) ], - [ AC_MSG_RESULT(no)] + [ AC_MSG_RESULT([yes]); AC_DEFINE([HAVE_SYSCTL_ARND], [1], [Define this symbol if the BSD sysctl(KERN_ARND) is available]) ], + [ AC_MSG_RESULT([no])] ) -AC_MSG_CHECKING(for if type char equals int8_t) +AC_MSG_CHECKING([for if type char equals int8_t]) AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include #include ]], [[ static_assert(std::is_same::value, ""); ]])], - [ AC_MSG_RESULT(yes); AC_DEFINE(CHAR_EQUALS_INT8, 1,[Define this symbol if type char equals int8_t]) ], - [ AC_MSG_RESULT(no)] + [ AC_MSG_RESULT([yes]); AC_DEFINE([CHAR_EQUALS_INT8], [1], [Define this symbol if type char equals int8_t]) ], + [ AC_MSG_RESULT([no])] ) -AC_MSG_CHECKING(for fdatasync) +AC_MSG_CHECKING([for fdatasync]) AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include ]], [[ fdatasync(0); ]])], - [ AC_MSG_RESULT(yes); HAVE_FDATASYNC=1 ], - [ AC_MSG_RESULT(no); HAVE_FDATASYNC=0 ] + [ AC_MSG_RESULT([yes]); HAVE_FDATASYNC=1 ], + [ AC_MSG_RESULT([no]); HAVE_FDATASYNC=0 ] ) AC_DEFINE_UNQUOTED([HAVE_FDATASYNC], [$HAVE_FDATASYNC], [Define to 1 if fdatasync is available.]) -AC_MSG_CHECKING(for F_FULLFSYNC) +AC_MSG_CHECKING([for F_FULLFSYNC]) AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include ]], [[ fcntl(0, F_FULLFSYNC, 0); ]])], - [ AC_MSG_RESULT(yes); HAVE_FULLFSYNC=1 ], - [ AC_MSG_RESULT(no); HAVE_FULLFSYNC=0 ] + [ AC_MSG_RESULT([yes]); HAVE_FULLFSYNC=1 ], + [ AC_MSG_RESULT([no]); HAVE_FULLFSYNC=0 ] ) -AC_MSG_CHECKING(for O_CLOEXEC) +AC_MSG_CHECKING([for O_CLOEXEC]) AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include ]], [[ open("", O_CLOEXEC); ]])], - [ AC_MSG_RESULT(yes); HAVE_O_CLOEXEC=1 ], - [ AC_MSG_RESULT(no); HAVE_O_CLOEXEC=0 ] + [ AC_MSG_RESULT([yes]); HAVE_O_CLOEXEC=1 ], + [ AC_MSG_RESULT([no]); HAVE_O_CLOEXEC=0 ] ) AC_DEFINE_UNQUOTED([HAVE_O_CLOEXEC], [$HAVE_O_CLOEXEC], [Define to 1 if O_CLOEXEC flag is available.]) dnl crc32c platform checks -AC_MSG_CHECKING(for __builtin_prefetch) +AC_MSG_CHECKING([for __builtin_prefetch]) AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ ]], [[ char data = 0; const char* address = &data; __builtin_prefetch(address, 0, 0); ]])], - [ AC_MSG_RESULT(yes); HAVE_BUILTIN_PREFETCH=1 ], - [ AC_MSG_RESULT(no); HAVE_BUILTIN_PREFETCH=0 ] + [ AC_MSG_RESULT([yes]); HAVE_BUILTIN_PREFETCH=1 ], + [ AC_MSG_RESULT([no]); HAVE_BUILTIN_PREFETCH=0 ] ) -AC_MSG_CHECKING(for _mm_prefetch) +AC_MSG_CHECKING([for _mm_prefetch]) AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include ]], [[ char data = 0; const char* address = &data; _mm_prefetch(address, _MM_HINT_NTA); ]])], - [ AC_MSG_RESULT(yes); HAVE_MM_PREFETCH=1 ], - [ AC_MSG_RESULT(no); HAVE_MM_PREFETCH=0 ] + [ AC_MSG_RESULT([yes]); HAVE_MM_PREFETCH=1 ], + [ AC_MSG_RESULT([no]); HAVE_MM_PREFETCH=0 ] ) -AC_MSG_CHECKING(for strong getauxval support in the system headers) +AC_MSG_CHECKING([for strong getauxval support in the system headers]) AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ - #include - #include #include ]], [[ getauxval(AT_HWCAP); ]])], - [ AC_MSG_RESULT(yes); HAVE_STRONG_GETAUXVAL=1; AC_DEFINE(HAVE_STRONG_GETAUXVAL, 1, [Define this symbol to build code that uses getauxval)]) ], - [ AC_MSG_RESULT(no); HAVE_STRONG_GETAUXVAL=0 ] -) - -AC_MSG_CHECKING(for weak getauxval support in the compiler) -AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ - #ifdef __linux__ - unsigned long getauxval(unsigned long type) __attribute__((weak)); - #define AT_HWCAP 16 - #endif - ]], [[ - getauxval(AT_HWCAP); - ]])], - [ AC_MSG_RESULT(yes); HAVE_WEAK_GETAUXVAL=1; AC_DEFINE(HAVE_WEAK_GETAUXVAL, 1, [Define this symbol to build code that uses getauxval (weak linking)]) ], - [ AC_MSG_RESULT(no); HAVE_WEAK_GETAUXVAL=0 ] + [ AC_MSG_RESULT([yes]); HAVE_STRONG_GETAUXVAL=1; AC_DEFINE([HAVE_STRONG_GETAUXVAL], [1], [Define this symbol to build code that uses getauxval)]) ], + [ AC_MSG_RESULT([no]); HAVE_STRONG_GETAUXVAL=0 ] ) +have_any_system=no AC_MSG_CHECKING([for std::system]) AC_LINK_IFELSE( [ AC_LANG_PROGRAM( [[ #include ]], [[ int nErr = std::system(""); ]] )], - [ AC_MSG_RESULT(yes); AC_DEFINE(HAVE_STD__SYSTEM, 1, Define to 1 if std::system is available.)], - [ AC_MSG_RESULT(no) ] + [ AC_MSG_RESULT([yes]); have_any_system=yes], + [ AC_MSG_RESULT([no]) ] ) AC_MSG_CHECKING([for ::_wsystem]) @@ -1241,19 +1260,13 @@ AC_LINK_IFELSE( [[ ]], [[ int nErr = ::_wsystem(""); ]] )], - [ AC_MSG_RESULT(yes); AC_DEFINE(HAVE_WSYSTEM, 1, Define to 1 if ::wsystem is available.)], - [ AC_MSG_RESULT(no) ] + [ AC_MSG_RESULT([yes]); have_any_system=yes], + [ AC_MSG_RESULT([no]) ] ) -AC_DEFINE([HAVE_SYSTEM], [HAVE_STD__SYSTEM || HAVE_WSYSTEM], [std::system or ::wsystem]) - -LEVELDB_CPPFLAGS= -LIBLEVELDB= -LIBMEMENV= -AM_CONDITIONAL([EMBEDDED_LEVELDB],[true]) -AC_SUBST(LEVELDB_CPPFLAGS) -AC_SUBST(LIBLEVELDB) -AC_SUBST(LIBMEMENV) +if test "$have_any_system" != "no"; then + AC_DEFINE([HAVE_SYSTEM], [1], [Define to 1 if std::system or ::wsystem is available.]) +fi dnl SUPPRESSED_CPPFLAGS=SUPPRESS_WARNINGS([$SOME_CPPFLAGS]) dnl Replace -I with -isystem in $SOME_CPPFLAGS to suppress warnings from @@ -1268,19 +1281,19 @@ AC_DEFUN([SUPPRESS_WARNINGS], [$(echo $1 |${SED} -E -e 's/(^| )-I/\1-isystem /g' -e 's;-isystem /usr/include([/ ]|$);-I/usr/include\1;g')]) dnl enable-fuzz should disable all other targets -if test "x$enable_fuzz" = "xyes"; then - AC_MSG_WARN(enable-fuzz will disable all other targets and force --enable-fuzz-binary=yes) +if test "$enable_fuzz" = "yes"; then + AC_MSG_WARN([enable-fuzz will disable all other targets and force --enable-fuzz-binary=yes]) build_bitcoin_utils=no build_bitcoin_cli=no build_bitcoin_tx=no build_bitcoin_util=no + build_bitcoin_chainstate=no build_bitcoin_wallet=no build_bitcoind=no build_bitcoin_libs=no bitcoin_enable_qt=no bitcoin_enable_qt_test=no bitcoin_enable_qt_dbus=no - enable_wallet=no use_bench=no use_external_signer=no use_upnp=no @@ -1288,14 +1301,13 @@ if test "x$enable_fuzz" = "xyes"; then use_zmq=no enable_fuzz_binary=yes - AX_CHECK_PREPROC_FLAG([-DABORT_ON_FAILED_ASSUME],[[DEBUG_CPPFLAGS="$DEBUG_CPPFLAGS -DABORT_ON_FAILED_ASSUME"]],,[[$CXXFLAG_WERROR]]) + AX_CHECK_PREPROC_FLAG([-DABORT_ON_FAILED_ASSUME], [DEBUG_CPPFLAGS="$DEBUG_CPPFLAGS -DABORT_ON_FAILED_ASSUME"], [], [$CXXFLAG_WERROR]) AC_MSG_CHECKING([whether main function is needed for fuzz binary]) AX_CHECK_LINK_FLAG( - [[-fsanitize=$use_sanitizers]], + [-fsanitize=$use_sanitizers], [AC_MSG_RESULT([no])], - [AC_MSG_RESULT([yes]) - CPPFLAGS="$CPPFLAGS -DPROVIDE_FUZZ_MAIN_FUNCTION"], + [AC_MSG_RESULT([yes]); CPPFLAGS="$CPPFLAGS -DPROVIDE_FUZZ_MAIN_FUNCTION"], [], [AC_LANG_PROGRAM([[ #include @@ -1313,7 +1325,7 @@ else dnl Keep a copy of the original $QT_INCLUDES and use it when invoking qt's moc QT_INCLUDES_UNSUPPRESSED=$QT_INCLUDES - if test x$suppress_external_warnings != xno ; then + if test "$suppress_external_warnings" != "no" ; then QT_INCLUDES=SUPPRESS_WARNINGS($QT_INCLUDES) QT_DBUS_INCLUDES=SUPPRESS_WARNINGS($QT_DBUS_INCLUDES) QT_TEST_INCLUDES=SUPPRESS_WARNINGS($QT_TEST_INCLUDES) @@ -1322,29 +1334,29 @@ else CPPFLAGS="$CPPFLAGS -DPROVIDE_FUZZ_MAIN_FUNCTION" fi -if test x$enable_wallet != xno; then +if test "$enable_wallet" != "no"; then dnl Check for libdb_cxx only if wallet enabled - if test "x$use_bdb" != "xno"; then + if test "$use_bdb" != "no"; then BITCOIN_FIND_BDB48 - if test x$suppress_external_warnings != xno ; then + if test "$suppress_external_warnings" != "no" ; then BDB_CPPFLAGS=SUPPRESS_WARNINGS($BDB_CPPFLAGS) fi fi dnl Check for sqlite3 - if test "x$use_sqlite" != "xno"; then + if test "$use_sqlite" != "no"; then PKG_CHECK_MODULES([SQLITE], [sqlite3 >= 3.7.17], [have_sqlite=yes], [have_sqlite=no]) fi AC_MSG_CHECKING([whether to build wallet with support for sqlite]) - if test "x$use_sqlite" = "xno"; then + if test "$use_sqlite" = "no"; then use_sqlite=no - elif test "x$have_sqlite" = "xno"; then - if test "x$use_sqlite" = "xyes"; then + elif test "$have_sqlite" = "no"; then + if test "$use_sqlite" = "yes"; then AC_MSG_ERROR([sqlite support requested but cannot be built. Use --without-sqlite]) fi use_sqlite=no else - if test x$use_sqlite != xno; then + if test "$use_sqlite" != "no"; then AC_DEFINE([USE_SQLITE],[1],[Define if sqlite support should be compiled in]) use_sqlite=yes fi @@ -1352,28 +1364,34 @@ if test x$enable_wallet != xno; then AC_MSG_RESULT([$use_sqlite]) dnl Disable wallet if both --without-bdb and --without-sqlite - if test "x$use_bdb$use_sqlite" = "xnono"; then - if test "x$enable_wallet" = "xyes"; then + if test "$use_bdb$use_sqlite" = "nono"; then + if test "$enable_wallet" = "yes"; then AC_MSG_ERROR([wallet functionality requested but no BDB or SQLite support available.]) fi enable_wallet=no fi fi -if test x$use_ebpf != xno; then - AC_MSG_CHECKING([whether eBPF tracepoints are supported]) +if test "$use_usdt" != "no"; then + AC_MSG_CHECKING([whether Userspace, Statically Defined Tracing tracepoints are supported]) AC_COMPILE_IFELSE([ AC_LANG_PROGRAM( [#include ], [DTRACE_PROBE("context", "event");] )], - [AC_MSG_RESULT(yes); have_sdt=yes; AC_DEFINE([ENABLE_TRACING], [1], [Define to 1 to enable eBPF user static defined tracepoints])], - [AC_MSG_RESULT(no); have_sdt=no;] + [AC_MSG_RESULT([yes]); AC_DEFINE([ENABLE_TRACING], [1], [Define to 1 to enable tracepoints for Userspace, Statically Defined Tracing])], + [AC_MSG_RESULT([no]); use_usdt=no;] ) fi +if test "$build_bitcoin_cli$build_bitcoin_tx$build_bitcoin_util$build_bitcoind$bitcoin_enable_qt$use_bench$use_tests" = "nonononononono"; then + use_upnp=no + use_natpmp=no + use_zmq=no +fi + dnl Check for libminiupnpc (optional) -if test x$use_upnp != xno; then +if test "$use_upnp" != "no"; then AC_CHECK_HEADERS( [miniupnpc/miniupnpc.h miniupnpc/upnpcommands.h miniupnpc/upnperrors.h], [AC_CHECK_LIB([miniupnpc], [upnpDiscover], [MINIUPNPC_LIBS=-lminiupnpc], [have_miniupnpc=no])], @@ -1381,7 +1399,7 @@ if test x$use_upnp != xno; then ) dnl The minimum supported miniUPnPc API version is set to 10. This keeps compatibility dnl with Ubuntu 16.04 LTS and Debian 8 libminiupnpc-dev packages. -if test x$have_miniupnpc != xno; then +if test "$have_miniupnpc" != "no"; then AC_MSG_CHECKING([whether miniUPnPc API version is supported]) AC_PREPROC_IFELSE([AC_LANG_PROGRAM([[ @%:@include @@ -1392,9 +1410,9 @@ if test x$have_miniupnpc != xno; then # error miniUPnPc API version is too old #endif ]])],[ - AC_MSG_RESULT(yes) + AC_MSG_RESULT([yes]) ],[ - AC_MSG_RESULT(no) + AC_MSG_RESULT([no]) AC_MSG_WARN([miniUPnPc API version < 10 is unsupported, disabling UPnP support.]) have_miniupnpc=no ]) @@ -1402,100 +1420,152 @@ fi fi dnl Check for libnatpmp (optional). -if test "x$use_natpmp" != xno; then +if test "$use_natpmp" != "no"; then AC_CHECK_HEADERS([natpmp.h], [AC_CHECK_LIB([natpmp], [initnatpmp], [NATPMP_LIBS=-lnatpmp], [have_natpmp=no])], [have_natpmp=no]) fi -if test x$build_bitcoin_wallet$build_bitcoin_cli$build_bitcoin_tx$build_bitcoind$bitcoin_enable_qt$use_tests$use_bench = xnonononononono; then +if test "$build_bitcoin_wallet$build_bitcoin_cli$build_bitcoin_tx$build_bitcoind$bitcoin_enable_qt$use_tests$use_bench" = "nonononononono"; then use_boost=no else use_boost=yes fi -if test x$use_boost = xyes; then +if test "$use_boost" = "yes"; then dnl Check for Boost headers AX_BOOST_BASE([1.64.0],[],[AC_MSG_ERROR([Boost is not available!])]) - if test x$want_boost = xno; then - AC_MSG_ERROR([[only libbitcoinconsensus can be built without boost]]) + if test "$want_boost" = "no"; then + AC_MSG_ERROR([only libbitcoinconsensus can be built without Boost]) fi - AX_BOOST_SYSTEM - AX_BOOST_FILESYSTEM - if test x$suppress_external_warnings != xno; then + if test "$suppress_external_warnings" != "no"; then BOOST_CPPFLAGS=SUPPRESS_WARNINGS($BOOST_CPPFLAGS) fi - - BOOST_LIBS="$BOOST_LDFLAGS $BOOST_SYSTEM_LIB $BOOST_FILESYSTEM_LIB" fi -if test "x$use_external_signer" != xno; then - AC_DEFINE([ENABLE_EXTERNAL_SIGNER],,[define if external signer support is enabled]) +if test "$use_external_signer" != "no"; then + case $host in + *mingw*) + dnl Boost Process uses Boost Filesystem when targeting Windows. Also, + dnl since Boost 1.71.0, Process does not work with mingw-w64 without + dnl workarounds. See 67669ab425b52a2b6be3d2f3b3b7e3939b676a2c. + if test "$use_external_signer" = "yes"; then + AC_MSG_ERROR([External signing is not supported on Windows]) + fi + use_external_signer="no"; + ;; + *) + AC_MSG_CHECKING([whether Boost.Process can be used]) + TEMP_LDFLAGS="$LDFLAGS" + dnl Boost 1.73 and older require the following workaround. + LDFLAGS="$LDFLAGS $PTHREAD_CFLAGS" + AC_LINK_IFELSE([AC_LANG_PROGRAM([[#include ]])], + [have_boost_process="yes"], + [have_boost_process="no"]) + LDFLAGS="$TEMP_LDFLAGS" + AC_MSG_RESULT([$have_boost_process]) + if test "$have_boost_process" = "yes"; then + use_external_signer="yes" + AC_DEFINE([ENABLE_EXTERNAL_SIGNER], [1], [Define if external signer support is enabled]) + else + if test "$use_external_signer" = "yes"; then + AC_MSG_ERROR([External signing is not supported for this Boost version]) + fi + use_external_signer="no"; + fi + ;; + esac fi -AM_CONDITIONAL([ENABLE_EXTERNAL_SIGNER], [test "x$use_external_signer" = "xyes"]) +AM_CONDITIONAL([ENABLE_EXTERNAL_SIGNER], [test "$use_external_signer" = "yes"]) + +dnl Do not compile with syscall sandbox support when compiling under the sanitizers. +dnl The sanitizers introduce use of syscalls that are not typically used in bitcoind +dnl (such as execve when the sanitizers execute llvm-symbolizer). +if test "$use_sanitizers" != ""; then + AC_MSG_WARN([Specifying --with-sanitizers forces --without-seccomp since the sanitizers introduce use of syscalls not allowed by the bitcoind syscall sandbox (-sandbox=).]) + seccomp_found=no +fi +if test "$seccomp_found" != "no"; then + AC_MSG_CHECKING([for seccomp-bpf (Linux x86-64)]) + AC_PREPROC_IFELSE([AC_LANG_PROGRAM([[ + @%:@include + ]], [[ + #if !defined(__x86_64__) + # error Syscall sandbox is an experimental feature currently available only under Linux x86-64. + #endif + ]])],[ + AC_MSG_RESULT([yes]) + seccomp_found="yes" + AC_DEFINE([USE_SYSCALL_SANDBOX], [1], [Define this symbol to build with syscall sandbox support.]) + ],[ + AC_MSG_RESULT([no]) + seccomp_found="no" + ]) +fi +dnl Currently only enable -sandbox= feature if seccomp is found. +dnl In the future, sandboxing could be also be supported with other +dnl sandboxing mechanisms besides seccomp. +use_syscall_sandbox=$seccomp_found +AM_CONDITIONAL([ENABLE_SYSCALL_SANDBOX], [test "$use_syscall_sandbox" != "no"]) dnl Check for reduced exports -if test x$use_reduce_exports = xyes; then - AX_CHECK_COMPILE_FLAG([-fvisibility=hidden],[CXXFLAGS="$CXXFLAGS -fvisibility=hidden"], - [AC_MSG_ERROR([Cannot set hidden symbol visibility. Use --disable-reduce-exports.])],[[$CXXFLAG_WERROR]]) - AX_CHECK_LINK_FLAG([[-Wl,--exclude-libs,ALL]],[RELDFLAGS="-Wl,--exclude-libs,ALL"],,[[$LDFLAG_WERROR]]) +if test "$use_reduce_exports" = "yes"; then + AX_CHECK_COMPILE_FLAG([-fvisibility=hidden], [CXXFLAGS="$CXXFLAGS -fvisibility=hidden"], + [AC_MSG_ERROR([Cannot set hidden symbol visibility. Use --disable-reduce-exports.])], [$CXXFLAG_WERROR]) + AX_CHECK_LINK_FLAG([-Wl,--exclude-libs,ALL], [RELDFLAGS="-Wl,--exclude-libs,ALL"], [], [$LDFLAG_WERROR]) fi -if test x$use_tests = xyes; then - - if test x$HEXDUMP = x; then - AC_MSG_ERROR(hexdump is required for tests) - fi - - if test x$use_boost = xyes; then - - AX_BOOST_UNIT_TEST_FRAMEWORK - - dnl Determine if -DBOOST_TEST_DYN_LINK is needed - AC_MSG_CHECKING([for dynamic linked boost test]) - TEMP_LIBS="$LIBS" - LIBS="$LIBS $BOOST_LDFLAGS $BOOST_UNIT_TEST_FRAMEWORK_LIB" - TEMP_CPPFLAGS="$CPPFLAGS" - CPPFLAGS="$CPPFLAGS $BOOST_CPPFLAGS" - AC_LINK_IFELSE([AC_LANG_SOURCE([ - #define BOOST_TEST_DYN_LINK - #define BOOST_TEST_MAIN - #include - - ])], - [AC_MSG_RESULT(yes)] - [TESTDEFS="$TESTDEFS -DBOOST_TEST_DYN_LINK"], - [AC_MSG_RESULT(no)]) - LIBS="$TEMP_LIBS" - CPPFLAGS="$TEMP_CPPFLAGS" +if test "$use_tests" = "yes"; then + if test "$HEXDUMP" = ""; then + AC_MSG_ERROR([hexdump is required for tests]) fi fi dnl libevent check -if test x$build_bitcoin_cli$build_bitcoind$bitcoin_enable_qt$use_tests$use_bench != xnonononono; then +if test "$build_bitcoin_cli$build_bitcoind$bitcoin_enable_qt$use_tests$use_bench" != "nonononono"; then PKG_CHECK_MODULES([EVENT], [libevent >= 2.0.21], [use_libevent=yes], [AC_MSG_ERROR([libevent version 2.0.21 or greater not found.])]) - if test x$TARGET_OS != xwindows; then + if test "$TARGET_OS" != "windows"; then PKG_CHECK_MODULES([EVENT_PTHREADS], [libevent_pthreads >= 2.0.21],, [AC_MSG_ERROR([libevent_pthreads version 2.0.21 or greater not found.])]) fi - if test x$suppress_external_warnings != xno; then + if test "$suppress_external_warnings" != "no"; then EVENT_CFLAGS=SUPPRESS_WARNINGS($EVENT_CFLAGS) fi fi +if test x$use_libevent = xyes; then + TEMP_CXXFLAGS="$CXXFLAGS" + CXXFLAGS="$CXXFLAGS $EVENT_CFLAGS" + AC_MSG_CHECKING([if evhttp_connection_get_peer expects const char**]) + AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ + #include + #include + ]], [[ + evhttp_connection *conn = (evhttp_connection *)1; + const char *host; + uint16_t port; + + evhttp_connection_get_peer(conn, &host, &port); + ]])], + [ AC_MSG_RESULT([yes]); AC_DEFINE([HAVE_EVHTTP_CONNECTION_GET_PEER_CONST_CHAR], [1], [Define this symbol if evhttp_connection_get_peer expects const char**]) ], + [ AC_MSG_RESULT([no]) ] + ) + CXXFLAGS="$TEMP_CXXFLAGS" +fi + dnl QR Code encoding library check -if test "x$use_qr" != xno; then +if test "$use_qr" != "no"; then BITCOIN_QT_CHECK([PKG_CHECK_MODULES([QR], [libqrencode], [have_qrencode=yes], [have_qrencode=no])]) fi dnl ZMQ check -if test "x$use_zmq" = xyes; then +if test "$use_zmq" = "yes"; then PKG_CHECK_MODULES([ZMQ], [libzmq >= 4], AC_DEFINE([ENABLE_ZMQ], [1], [Define to 1 to enable ZMQ functions]), [AC_DEFINE([ENABLE_ZMQ], [0], [Define to 1 to enable ZMQ functions]) @@ -1505,7 +1575,7 @@ else AC_DEFINE_UNQUOTED([ENABLE_ZMQ], [0], [Define to 1 to enable ZMQ functions]) fi -if test "x$use_zmq" = xyes; then +if test "$use_zmq" = "yes"; then dnl Assume libzmq was built for static linking case $host in *mingw*) @@ -1514,110 +1584,84 @@ if test "x$use_zmq" = xyes; then esac fi -dnl univalue check - -need_bundled_univalue=yes -if test x$build_bitcoin_wallet$build_bitcoin_cli$build_bitcoin_tx$build_bitcoin_util$build_bitcoind$bitcoin_enable_qt$use_tests$use_bench = xnononononononono; then - need_bundled_univalue=no -else - if test x$system_univalue != xno; then - PKG_CHECK_MODULES([UNIVALUE], [libunivalue >= 1.0.4], [found_univalue=yes], [found_univalue=no]) - if test x$found_univalue = xyes; then - system_univalue=yes - need_bundled_univalue=no - elif test x$system_univalue = xyes; then - AC_MSG_ERROR([univalue not found]) - else - system_univalue=no - fi - fi - - if test x$need_bundled_univalue = xyes; then - UNIVALUE_CFLAGS='-I$(srcdir)/univalue/include' - UNIVALUE_LIBS='univalue/libunivalue.la' - fi -fi - -AM_CONDITIONAL([EMBEDDED_UNIVALUE],[test x$need_bundled_univalue = xyes]) -AC_SUBST(UNIVALUE_CFLAGS) -AC_SUBST(UNIVALUE_LIBS) - dnl libmultiprocess library check libmultiprocess_found=no -if test "x$with_libmultiprocess" = xyes || test "x$with_libmultiprocess" = xauto; then - m4_ifdef([PKG_CHECK_MODULES], [PKG_CHECK_MODULES([LIBMULTIPROCESS], [libmultiprocess], [ +if test "$with_libmultiprocess" = "yes" || test "$with_libmultiprocess" = "auto"; then + PKG_CHECK_MODULES([LIBMULTIPROCESS], [libmultiprocess], [ libmultiprocess_found=yes; libmultiprocess_prefix=`$PKG_CONFIG --variable=prefix libmultiprocess`; - ], [true])]) -elif test "x$with_libmultiprocess" != xno; then + ], [true]) +elif test "$with_libmultiprocess" != "no"; then AC_MSG_ERROR([--with-libmultiprocess=$with_libmultiprocess value is not yes, auto, or no]) fi -AC_SUBST(LIBMULTIPROCESS_CFLAGS) -AC_SUBST(LIBMULTIPROCESS_LIBS) dnl Enable multiprocess check -if test "x$enable_multiprocess" = xyes; then - if test "x$libmultiprocess_found" != xyes; then +if test "$enable_multiprocess" = "yes"; then + if test "$libmultiprocess_found" != "yes"; then AC_MSG_ERROR([--enable-multiprocess=yes option specified but libmultiprocess library was not found. May need to install libmultiprocess library, or specify install path with PKG_CONFIG_PATH environment variable. Running 'pkg-config --debug libmultiprocess' may be helpful for debugging.]) fi build_multiprocess=yes -elif test "x$enable_multiprocess" = xauto; then +elif test "$enable_multiprocess" = "auto"; then build_multiprocess=$libmultiprocess_found else build_multiprocess=no fi -AM_CONDITIONAL([BUILD_MULTIPROCESS],[test "x$build_multiprocess" = xyes]) -AM_CONDITIONAL([BUILD_BITCOIN_NODE], [test "x$build_multiprocess" = xyes]) -AM_CONDITIONAL([BUILD_BITCOIN_GUI], [test "x$build_multiprocess" = xyes]) +AM_CONDITIONAL([BUILD_MULTIPROCESS], [test "$build_multiprocess" = "yes"]) +AM_CONDITIONAL([BUILD_BITCOIN_NODE], [test "$build_multiprocess" = "yes"]) +AM_CONDITIONAL([BUILD_BITCOIN_GUI], [test "$build_multiprocess" = "yes"]) dnl codegen tools check -if test x$build_multiprocess != xno; then - if test "x$with_mpgen" = xyes || test "x$with_mpgen" = xauto; then +if test "$build_multiprocess" != "no"; then + if test "$with_mpgen" = "yes" || test "$with_mpgen" = "auto"; then MPGEN_PREFIX="$libmultiprocess_prefix" - elif test "x$with_mpgen" != xno; then + elif test "$with_mpgen" != "no"; then MPGEN_PREFIX="$with_mpgen"; fi AC_SUBST(MPGEN_PREFIX) fi AC_MSG_CHECKING([whether to build bitcoind]) -AM_CONDITIONAL([BUILD_BITCOIND], [test x$build_bitcoind = xyes]) +AM_CONDITIONAL([BUILD_BITCOIND], [test $build_bitcoind = "yes"]) AC_MSG_RESULT($build_bitcoind) AC_MSG_CHECKING([whether to build bitcoin-cli]) -AM_CONDITIONAL([BUILD_BITCOIN_CLI], [test x$build_bitcoin_cli = xyes]) +AM_CONDITIONAL([BUILD_BITCOIN_CLI], [test $build_bitcoin_cli = "yes"]) AC_MSG_RESULT($build_bitcoin_cli) AC_MSG_CHECKING([whether to build bitcoin-tx]) -AM_CONDITIONAL([BUILD_BITCOIN_TX], [test x$build_bitcoin_tx = xyes]) +AM_CONDITIONAL([BUILD_BITCOIN_TX], [test $build_bitcoin_tx = "yes"]) AC_MSG_RESULT($build_bitcoin_tx) AC_MSG_CHECKING([whether to build bitcoin-wallet]) -AM_CONDITIONAL([BUILD_BITCOIN_WALLET], [test x$build_bitcoin_wallet = xyes]) +AM_CONDITIONAL([BUILD_BITCOIN_WALLET], [test $build_bitcoin_wallet = "yes"]) AC_MSG_RESULT($build_bitcoin_wallet) AC_MSG_CHECKING([whether to build bitcoin-util]) -AM_CONDITIONAL([BUILD_BITCOIN_UTIL], [test x$build_bitcoin_util = xyes]) +AM_CONDITIONAL([BUILD_BITCOIN_UTIL], [test $build_bitcoin_util = "yes"]) AC_MSG_RESULT($build_bitcoin_util) +AC_MSG_CHECKING([whether to build experimental bitcoin-chainstate]) +AM_CONDITIONAL([BUILD_BITCOIN_CHAINSTATE], [test $build_bitcoin_chainstate = "yes"]) +AC_MSG_RESULT($build_bitcoin_chainstate) + AC_MSG_CHECKING([whether to build libraries]) -AM_CONDITIONAL([BUILD_BITCOIN_LIBS], [test x$build_bitcoin_libs = xyes]) -if test x$build_bitcoin_libs = xyes; then - AC_DEFINE(HAVE_CONSENSUS_LIB, 1, [Define this symbol if the consensus lib has been built]) +AM_CONDITIONAL([BUILD_BITCOIN_LIBS], [test $build_bitcoin_libs = "yes"]) +if test "$build_bitcoin_libs" = "yes"; then + AC_DEFINE([HAVE_CONSENSUS_LIB], [1], [Define this symbol if the consensus lib has been built]) AC_CONFIG_FILES([libbitcoinconsensus.pc:libbitcoinconsensus.pc.in]) fi AC_MSG_RESULT($build_bitcoin_libs) AC_LANG_POP -if test "x$use_ccache" != "xno"; then - AC_MSG_CHECKING(if ccache should be used) - if test x$CCACHE = x; then - if test "x$use_ccache" = "xyes"; then +if test "$use_ccache" != "no"; then + AC_MSG_CHECKING([if ccache should be used]) + if test "$CCACHE" = ""; then + if test "$use_ccache" = "yes"; then AC_MSG_ERROR([ccache not found.]); else use_ccache=no @@ -1628,72 +1672,72 @@ if test "x$use_ccache" != "xno"; then CXX="$ac_cv_path_CCACHE $CXX" fi AC_MSG_RESULT($use_ccache) - if test "x$use_ccache" = "xyes"; then - AX_CHECK_COMPILE_FLAG([-fdebug-prefix-map=A=B],[DEBUG_CXXFLAGS="$DEBUG_CXXFLAGS -fdebug-prefix-map=\$(abs_srcdir)=."],,[[$CXXFLAG_WERROR]]) - AX_CHECK_PREPROC_FLAG([-fmacro-prefix-map=A=B],[DEBUG_CPPFLAGS="$DEBUG_CPPFLAGS -fmacro-prefix-map=\$(abs_srcdir)=."],,[[$CXXFLAG_WERROR]]) + if test "$use_ccache" = "yes"; then + AX_CHECK_COMPILE_FLAG([-fdebug-prefix-map=A=B], [DEBUG_CXXFLAGS="$DEBUG_CXXFLAGS -fdebug-prefix-map=\$(abs_top_srcdir)=."], [], [$CXXFLAG_WERROR]) + AX_CHECK_PREPROC_FLAG([-fmacro-prefix-map=A=B], [DEBUG_CPPFLAGS="$DEBUG_CPPFLAGS -fmacro-prefix-map=\$(abs_top_srcdir)=."], [], [$CXXFLAG_WERROR]) fi fi dnl enable wallet AC_MSG_CHECKING([if wallet should be enabled]) -if test x$enable_wallet != xno; then - AC_MSG_RESULT(yes) +if test "$enable_wallet" != "no"; then + AC_MSG_RESULT([yes]) AC_DEFINE_UNQUOTED([ENABLE_WALLET],[1],[Define to 1 to enable wallet functions]) enable_wallet=yes else - AC_MSG_RESULT(no) + AC_MSG_RESULT([no]) fi dnl enable upnp support AC_MSG_CHECKING([whether to build with support for UPnP]) -if test x$have_miniupnpc = xno; then - if test x$use_upnp = xyes; then - AC_MSG_ERROR("UPnP requested but cannot be built. Use --without-miniupnpc.") +if test "$have_miniupnpc" = "no"; then + if test "$use_upnp" = "yes"; then + AC_MSG_ERROR([UPnP requested but cannot be built. Use --without-miniupnpc]) fi - AC_MSG_RESULT(no) + AC_MSG_RESULT([no]) use_upnp=no else - if test x$use_upnp != xno; then - AC_MSG_RESULT(yes) + if test "$use_upnp" != "no"; then + AC_MSG_RESULT([yes]) AC_MSG_CHECKING([whether to build with UPnP enabled by default]) use_upnp=yes upnp_setting=0 - if test x$use_upnp_default != xno; then + if test "$use_upnp_default" != "no"; then use_upnp_default=yes upnp_setting=1 fi - AC_MSG_RESULT($use_upnp_default) + AC_MSG_RESULT([$use_upnp_default]) AC_DEFINE_UNQUOTED([USE_UPNP],[$upnp_setting],[UPnP support not compiled if undefined, otherwise value (0 or 1) determines default state]) - if test x$TARGET_OS = xwindows; then + if test "$TARGET_OS" = "windows"; then MINIUPNPC_CPPFLAGS="-DSTATICLIB -DMINIUPNP_STATICLIB" fi else - AC_MSG_RESULT(no) + AC_MSG_RESULT([no]) fi fi dnl Enable NAT-PMP support. AC_MSG_CHECKING([whether to build with support for NAT-PMP]) -if test "x$have_natpmp" = xno; then - if test "x$use_natpmp" = xyes; then +if test "$have_natpmp" = "no"; then + if test "$use_natpmp" = "yes"; then AC_MSG_ERROR([NAT-PMP requested but cannot be built. Use --without-natpmp]) fi AC_MSG_RESULT([no]) use_natpmp=no else - if test "x$use_natpmp" != xno; then + if test "$use_natpmp" != "no"; then AC_MSG_RESULT([yes]) AC_MSG_CHECKING([whether to build with NAT-PMP enabled by default]) use_natpmp=yes natpmp_setting=0 - if test "x$use_natpmp_default" != xno; then + if test "$use_natpmp_default" != "no"; then use_natpmp_default=yes natpmp_setting=1 fi AC_MSG_RESULT($use_natpmp_default) AC_DEFINE_UNQUOTED([USE_NATPMP], [$natpmp_setting], [NAT-PMP support not compiled if undefined, otherwise value (0 or 1) determines default state]) - if test x$TARGET_OS = xwindows; then + if test "$TARGET_OS" = "windows"; then NATPMP_CPPFLAGS="-DSTATICLIB -DNATPMP_STATICLIB" fi else @@ -1703,35 +1747,35 @@ fi dnl these are only used when qt is enabled BUILD_TEST_QT="" -if test x$bitcoin_enable_qt != xno; then +if test "$bitcoin_enable_qt" != "no"; then dnl enable dbus support AC_MSG_CHECKING([whether to build GUI with support for D-Bus]) - if test x$bitcoin_enable_qt_dbus != xno; then - AC_DEFINE([USE_DBUS],[1],[Define if dbus support should be compiled in]) + if test "$bitcoin_enable_qt_dbus" != "no"; then + AC_DEFINE([USE_DBUS], [1], [Define if dbus support should be compiled in]) fi - AC_MSG_RESULT($bitcoin_enable_qt_dbus) + AC_MSG_RESULT([$bitcoin_enable_qt_dbus]) dnl enable qr support AC_MSG_CHECKING([whether to build GUI with support for QR codes]) - if test x$have_qrencode = xno; then - if test x$use_qr = xyes; then + if test "$have_qrencode" = "no"; then + if test "$use_qr" = "yes"; then AC_MSG_ERROR([QR support requested but cannot be built. Use --without-qrencode]) fi use_qr=no else - if test x$use_qr != xno; then - AC_DEFINE([USE_QRCODE],[1],[Define if QR support should be compiled in]) + if test "$use_qr" != "no"; then + AC_DEFINE([USE_QRCODE], [1], [Define if QR support should be compiled in]) use_qr=yes fi fi AC_MSG_RESULT([$use_qr]) - if test x$XGETTEXT = x; then - AC_MSG_WARN("xgettext is required to update qt translations") + if test "$XGETTEXT" = ""; then + AC_MSG_WARN([xgettext is required to update qt translations]) fi AC_MSG_CHECKING([whether to build test_bitcoin-qt]) - if test x$use_gui_tests$bitcoin_enable_qt_test = xyesyes; then + if test "$use_gui_tests$bitcoin_enable_qt_test" = "yesyes"; then AC_MSG_RESULT([yes]) BUILD_TEST_QT="yes" else @@ -1739,11 +1783,11 @@ if test x$bitcoin_enable_qt != xno; then fi fi -AM_CONDITIONAL([ENABLE_ZMQ], [test "x$use_zmq" = "xyes"]) +AM_CONDITIONAL([ENABLE_ZMQ], [test "$use_zmq" = "yes"]) AC_MSG_CHECKING([whether to build test_bitcoin]) -if test x$use_tests = xyes; then - if test "x$enable_fuzz" = "xyes"; then +if test "$use_tests" = "yes"; then + if test "$enable_fuzz" = "yes"; then AC_MSG_RESULT([no, because fuzzing is enabled]) else AC_MSG_RESULT([yes]) @@ -1755,55 +1799,62 @@ else fi AC_MSG_CHECKING([whether to reduce exports]) -if test x$use_reduce_exports = xyes; then +if test "$use_reduce_exports" = "yes"; then AC_MSG_RESULT([yes]) else AC_MSG_RESULT([no]) fi -if test x$build_bitcoin_wallet$build_bitcoin_cli$build_bitcoin_tx$build_bitcoin_libs$build_bitcoind$bitcoin_enable_qt$use_bench$use_tests = xnononononononono; then +if test "$build_bitcoin_wallet$build_bitcoin_cli$build_bitcoin_tx$build_bitcoin_libs$build_bitcoind$bitcoin_enable_qt$use_bench$use_tests" = "nononononononono"; then AC_MSG_ERROR([No targets! Please specify at least one of: --with-utils --with-libs --with-daemon --with-gui --enable-bench or --enable-tests]) fi -AM_CONDITIONAL([TARGET_DARWIN], [test x$TARGET_OS = xdarwin]) -AM_CONDITIONAL([BUILD_DARWIN], [test x$BUILD_OS = xdarwin]) -AM_CONDITIONAL([TARGET_LINUX], [test x$TARGET_OS = xlinux]) -AM_CONDITIONAL([TARGET_WINDOWS], [test x$TARGET_OS = xwindows]) -AM_CONDITIONAL([ENABLE_WALLET],[test x$enable_wallet = xyes]) -AM_CONDITIONAL([USE_SQLITE], [test "x$use_sqlite" = "xyes"]) -AM_CONDITIONAL([USE_BDB], [test "x$use_bdb" = "xyes"]) -AM_CONDITIONAL([ENABLE_TRACING],[test x$have_sdt = xyes]) -AM_CONDITIONAL([ENABLE_TESTS],[test x$BUILD_TEST = xyes]) -AM_CONDITIONAL([ENABLE_FUZZ],[test x$enable_fuzz = xyes]) -AM_CONDITIONAL([ENABLE_FUZZ_BINARY],[test x$enable_fuzz_binary = xyes]) -AM_CONDITIONAL([ENABLE_QT],[test x$bitcoin_enable_qt = xyes]) -AM_CONDITIONAL([ENABLE_QT_TESTS],[test x$BUILD_TEST_QT = xyes]) -AM_CONDITIONAL([ENABLE_BENCH],[test x$use_bench = xyes]) -AM_CONDITIONAL([USE_QRCODE], [test x$use_qr = xyes]) -AM_CONDITIONAL([USE_LCOV],[test x$use_lcov = xyes]) -AM_CONDITIONAL([USE_LIBEVENT],[test x$use_libevent = xyes]) -AM_CONDITIONAL([GLIBC_BACK_COMPAT],[test x$use_glibc_compat = xyes]) -AM_CONDITIONAL([HARDEN],[test x$use_hardening = xyes]) -AM_CONDITIONAL([ENABLE_SSE42],[test x$enable_sse42 = xyes]) -AM_CONDITIONAL([ENABLE_SSE41],[test x$enable_sse41 = xyes]) -AM_CONDITIONAL([ENABLE_AVX2],[test x$enable_avx2 = xyes]) -AM_CONDITIONAL([ENABLE_SHANI],[test x$enable_shani = xyes]) -AM_CONDITIONAL([ENABLE_ARM_CRC],[test x$enable_arm_crc = xyes]) -AM_CONDITIONAL([USE_ASM],[test x$use_asm = xyes]) -AM_CONDITIONAL([WORDS_BIGENDIAN],[test x$ac_cv_c_bigendian = xyes]) -AM_CONDITIONAL([USE_NATPMP],[test x$use_natpmp = xyes]) -AM_CONDITIONAL([USE_UPNP],[test x$use_upnp = xyes]) -AM_CONDITIONAL([LIQUID],[test x$liquid_build = xyes]) +if test "$enable_fuzz_binary" = "yes"; then + CHECK_RUNTIME_LIB +fi -AC_DEFINE(CLIENT_VERSION_MAJOR, _CLIENT_VERSION_MAJOR, [Major version]) -AC_DEFINE(CLIENT_VERSION_MINOR, _CLIENT_VERSION_MINOR, [Minor version]) -AC_DEFINE(CLIENT_VERSION_BUILD, _CLIENT_VERSION_BUILD, [Version Build]) -AC_DEFINE(CLIENT_VERSION_IS_RELEASE, _CLIENT_VERSION_IS_RELEASE, [Version is release]) -AC_DEFINE(COPYRIGHT_YEAR, _COPYRIGHT_YEAR, [Copyright year]) -AC_DEFINE(COPYRIGHT_HOLDERS, "_COPYRIGHT_HOLDERS", [Copyright holder(s) before %s replacement]) -AC_DEFINE(COPYRIGHT_HOLDERS_SUBSTITUTION, "_COPYRIGHT_HOLDERS_SUBSTITUTION", [Replacement for %s in copyright holders string]) +AM_CONDITIONAL([TARGET_DARWIN], [test "$TARGET_OS" = "darwin"]) +AM_CONDITIONAL([BUILD_DARWIN], [test "$BUILD_OS" = "darwin"]) +AM_CONDITIONAL([TARGET_LINUX], [test "$TARGET_OS" = "linux"]) +AM_CONDITIONAL([TARGET_WINDOWS], [test "$TARGET_OS" = "windows"]) +AM_CONDITIONAL([ENABLE_WALLET], [test "$enable_wallet" = "yes"]) +AM_CONDITIONAL([USE_SQLITE], [test "$use_sqlite" = "yes"]) +AM_CONDITIONAL([USE_BDB], [test "$use_bdb" = "yes"]) +AM_CONDITIONAL([ENABLE_TESTS], [test "$BUILD_TEST" = "yes"]) +AM_CONDITIONAL([ENABLE_FUZZ], [test "$enable_fuzz" = "yes"]) +AM_CONDITIONAL([ENABLE_FUZZ_BINARY], [test "$enable_fuzz_binary" = "yes"]) +AM_CONDITIONAL([ENABLE_QT], [test "$bitcoin_enable_qt" = "yes"]) +AM_CONDITIONAL([ENABLE_QT_TESTS], [test "$BUILD_TEST_QT" = "yes"]) +AM_CONDITIONAL([ENABLE_BENCH], [test "$use_bench" = "yes"]) +AM_CONDITIONAL([USE_QRCODE], [test "$use_qr" = "yes"]) +AM_CONDITIONAL([USE_LCOV], [test "$use_lcov" = "yes"]) +AM_CONDITIONAL([USE_LIBEVENT], [test "$use_libevent" = "yes"]) +AM_CONDITIONAL([HARDEN], [test "$use_hardening" = "yes"]) +AM_CONDITIONAL([ENABLE_SSE42], [test "$enable_sse42" = "yes"]) +AM_CONDITIONAL([ENABLE_SSE41], [test "$enable_sse41" = "yes"]) +AM_CONDITIONAL([ENABLE_AVX2], [test "$enable_avx2" = "yes"]) +AM_CONDITIONAL([ENABLE_X86_SHANI], [test "$enable_x86_shani" = "yes"]) +AM_CONDITIONAL([ENABLE_ARM_CRC], [test "$enable_arm_crc" = "yes"]) +AM_CONDITIONAL([ENABLE_ARM_SHANI], [test "$enable_arm_shani" = "yes"]) +AM_CONDITIONAL([USE_ASM], [test "$use_asm" = "yes"]) +AM_CONDITIONAL([WORDS_BIGENDIAN], [test "$ac_cv_c_bigendian" = "yes"]) +AM_CONDITIONAL([USE_NATPMP], [test "$use_natpmp" = "yes"]) +AM_CONDITIONAL([USE_UPNP], [test "$use_upnp" = "yes"]) +AM_CONDITIONAL([LIQUID], [test "$liquid_build" = "yes"]) + +dnl for minisketch +AM_CONDITIONAL([ENABLE_CLMUL], [test "$enable_clmul" = "yes"]) +AM_CONDITIONAL([HAVE_CLZ], [test "$have_clzl$have_clzll" = "yesyes"]) + +AC_DEFINE([CLIENT_VERSION_MAJOR], [_CLIENT_VERSION_MAJOR], [Major version]) +AC_DEFINE([CLIENT_VERSION_MINOR], [_CLIENT_VERSION_MINOR], [Minor version]) +AC_DEFINE([CLIENT_VERSION_BUILD], [_CLIENT_VERSION_BUILD], [Version Build]) +AC_DEFINE([CLIENT_VERSION_IS_RELEASE], [_CLIENT_VERSION_IS_RELEASE], [Version is release]) +AC_DEFINE([COPYRIGHT_YEAR], [_COPYRIGHT_YEAR], [Copyright year]) +AC_DEFINE([COPYRIGHT_HOLDERS], ["_COPYRIGHT_HOLDERS"], [Copyright holder(s) before %s replacement]) +AC_DEFINE([COPYRIGHT_HOLDERS_SUBSTITUTION], ["_COPYRIGHT_HOLDERS_SUBSTITUTION"], [Replacement for %s in copyright holders string]) define(_COPYRIGHT_HOLDERS_FINAL, [patsubst(_COPYRIGHT_HOLDERS, [%s], [_COPYRIGHT_HOLDERS_SUBSTITUTION])]) -AC_DEFINE(COPYRIGHT_HOLDERS_FINAL, "_COPYRIGHT_HOLDERS_FINAL", [Copyright holder(s)]) +AC_DEFINE([COPYRIGHT_HOLDERS_FINAL], ["_COPYRIGHT_HOLDERS_FINAL"], [Copyright holder(s)]) AC_SUBST(CLIENT_VERSION_MAJOR, _CLIENT_VERSION_MAJOR) AC_SUBST(CLIENT_VERSION_MINOR, _CLIENT_VERSION_MINOR) AC_SUBST(CLIENT_VERSION_BUILD, _CLIENT_VERSION_BUILD) @@ -1817,6 +1868,7 @@ AC_SUBST(BITCOIN_GUI_NAME) AC_SUBST(BITCOIN_CLI_NAME) AC_SUBST(BITCOIN_TX_NAME) AC_SUBST(BITCOIN_UTIL_NAME) +AC_SUBST(BITCOIN_CHAINSTATE_NAME) AC_SUBST(BITCOIN_WALLET_TOOL_NAME) AC_SUBST(BITCOIN_MP_NODE_NAME) AC_SUBST(BITCOIN_MP_GUI_NAME) @@ -1826,39 +1878,36 @@ AC_SUBST(DEBUG_CPPFLAGS) AC_SUBST(WARN_CXXFLAGS) AC_SUBST(NOWARN_CXXFLAGS) AC_SUBST(DEBUG_CXXFLAGS) -AC_SUBST(COMPAT_LDFLAGS) AC_SUBST(ERROR_CXXFLAGS) AC_SUBST(GPROF_CXXFLAGS) AC_SUBST(GPROF_LDFLAGS) AC_SUBST(HARDENED_CXXFLAGS) AC_SUBST(HARDENED_CPPFLAGS) AC_SUBST(HARDENED_LDFLAGS) +AC_SUBST(LTO_CXXFLAGS) +AC_SUBST(LTO_LDFLAGS) AC_SUBST(PIC_FLAGS) AC_SUBST(PIE_FLAGS) AC_SUBST(SANITIZER_CXXFLAGS) AC_SUBST(SANITIZER_LDFLAGS) AC_SUBST(SSE42_CXXFLAGS) AC_SUBST(SSE41_CXXFLAGS) +AC_SUBST(CLMUL_CXXFLAGS) AC_SUBST(AVX2_CXXFLAGS) -AC_SUBST(SHANI_CXXFLAGS) +AC_SUBST(X86_SHANI_CXXFLAGS) AC_SUBST(ARM_CRC_CXXFLAGS) +AC_SUBST(ARM_SHANI_CXXFLAGS) AC_SUBST(LIBTOOL_APP_LDFLAGS) AC_SUBST(USE_SQLITE) AC_SUBST(USE_BDB) AC_SUBST(ENABLE_EXTERNAL_SIGNER) AC_SUBST(USE_UPNP) AC_SUBST(USE_QRCODE) -AC_SUBST(BOOST_LIBS) -AC_SUBST(SQLITE_LIBS) AC_SUBST(TESTDEFS) AC_SUBST(MINIUPNPC_CPPFLAGS) AC_SUBST(MINIUPNPC_LIBS) AC_SUBST(NATPMP_CPPFLAGS) AC_SUBST(NATPMP_LIBS) -AC_SUBST(EVENT_LIBS) -AC_SUBST(EVENT_PTHREADS_LIBS) -AC_SUBST(ZMQ_LIBS) -AC_SUBST(QR_LIBS) AC_SUBST(HAVE_GMTIME_R) AC_SUBST(HAVE_FDATASYNC) AC_SUBST(HAVE_FULLFSYNC) @@ -1866,8 +1915,8 @@ AC_SUBST(HAVE_O_CLOEXEC) AC_SUBST(HAVE_BUILTIN_PREFETCH) AC_SUBST(HAVE_MM_PREFETCH) AC_SUBST(HAVE_STRONG_GETAUXVAL) -AC_SUBST(HAVE_WEAK_GETAUXVAL) AC_SUBST(ANDROID_ARCH) +AC_SUBST(HAVE_EVHTTP_CONNECTION_GET_PEER_CONST_CHAR) AC_CONFIG_FILES([Makefile src/Makefile doc/man/Makefile share/setup.nsi share/qt/Info.plist test/config.ini]) AC_CONFIG_FILES([contrib/devtools/split-debug.sh],[chmod +x contrib/devtools/split-debug.sh]) AM_COND_IF([HAVE_DOXYGEN], [AC_CONFIG_FILES([doc/Doxyfile])]) @@ -1876,10 +1925,11 @@ AC_CONFIG_LINKS([contrib/devtools/symbol-check.py:contrib/devtools/symbol-check. AC_CONFIG_LINKS([contrib/devtools/test-security-check.py:contrib/devtools/test-security-check.py]) AC_CONFIG_LINKS([contrib/devtools/test-symbol-check.py:contrib/devtools/test-symbol-check.py]) AC_CONFIG_LINKS([contrib/filter-lcov.py:contrib/filter-lcov.py]) +AC_CONFIG_LINKS([contrib/macdeploy/background.tiff:contrib/macdeploy/background.tiff]) AC_CONFIG_LINKS([test/functional/test_runner.py:test/functional/test_runner.py]) AC_CONFIG_LINKS([test/bitcoin_functional/functional/test_runner.py:test/bitcoin_functional/functional/test_runner.py]) AC_CONFIG_LINKS([test/fuzz/test_runner.py:test/fuzz/test_runner.py]) -AC_CONFIG_LINKS([test/util/bitcoin-util-test.py:test/util/bitcoin-util-test.py]) +AC_CONFIG_LINKS([test/util/test_runner.py:test/util/test_runner.py]) AC_CONFIG_LINKS([test/util/rpcauth-test.py:test/util/rpcauth-test.py]) dnl boost's m4 checks do something really nasty: they export these vars. As a @@ -1905,10 +1955,6 @@ PKGCONFIG_LIBDIR_TEMP="$PKG_CONFIG_LIBDIR" unset PKG_CONFIG_LIBDIR PKG_CONFIG_LIBDIR="$PKGCONFIG_LIBDIR_TEMP" -if test x$need_bundled_univalue = xyes; then - AC_CONFIG_SUBDIRS([src/univalue]) -fi - ac_configure_args="${ac_configure_args} --disable-shared --with-pic --enable-benchmark=no --with-bignum=no --enable-module-recovery --enable-module-schnorrsig --enable-experimental --enable-module-whitelist --enable-module-rangeproof --enable-module-generator --enable-module-surjectionproof --enable-module-ecdh" AC_CONFIG_SUBDIRS([src/secp256k1]) @@ -1926,42 +1972,44 @@ echo echo "Options used to compile and link:" echo " external signer = $use_external_signer" echo " multiprocess = $build_multiprocess" +echo " with experimental syscall sandbox support = $use_syscall_sandbox" echo " with libs = $build_bitcoin_libs" echo " with wallet = $enable_wallet" -if test "x$enable_wallet" != "xno"; then +if test "$enable_wallet" != "no"; then echo " with sqlite = $use_sqlite" echo " with bdb = $use_bdb" fi echo " with gui / qt = $bitcoin_enable_qt" -if test x$bitcoin_enable_qt != xno; then +if test $bitcoin_enable_qt != "no"; then echo " with qr = $use_qr" fi echo " with zmq = $use_zmq" -if test x$enable_fuzz == xno; then +if test $enable_fuzz = "no"; then echo " with test = $use_tests" else echo " with test = not building test_bitcoin because fuzzing is enabled" - echo " with fuzz = $enable_fuzz" fi +echo " with fuzz binary = $enable_fuzz_binary" echo " with bench = $use_bench" echo " with upnp = $use_upnp" echo " with natpmp = $use_natpmp" echo " use asm = $use_asm" -echo " ebpf tracing = $have_sdt" echo " liquid_build = $liquid_build" +echo " USDT tracing = $use_usdt" echo " sanitizers = $use_sanitizers" echo " debug enabled = $enable_debug" echo " gprof enabled = $enable_gprof" echo " werror = $enable_werror" +echo " LTO = $enable_lto" echo -echo " target os = $TARGET_OS" +echo " target os = $host_os" echo " build os = $build_os" echo echo " CC = $CC" echo " CFLAGS = $PTHREAD_CFLAGS $CFLAGS" echo " CPPFLAGS = $DEBUG_CPPFLAGS $HARDENED_CPPFLAGS $CPPFLAGS" echo " CXX = $CXX" -echo " CXXFLAGS = $DEBUG_CXXFLAGS $HARDENED_CXXFLAGS $WARN_CXXFLAGS $NOWARN_CXXFLAGS $ERROR_CXXFLAGS $GPROF_CXXFLAGS $CXXFLAGS" -echo " LDFLAGS = $PTHREAD_LIBS $HARDENED_LDFLAGS $GPROF_LDFLAGS $LDFLAGS" +echo " CXXFLAGS = $LTO_CXXFLAGS $DEBUG_CXXFLAGS $HARDENED_CXXFLAGS $WARN_CXXFLAGS $NOWARN_CXXFLAGS $ERROR_CXXFLAGS $GPROF_CXXFLAGS $CXXFLAGS" +echo " LDFLAGS = $LTO_LDFLAGS $PTHREAD_LIBS $HARDENED_LDFLAGS $GPROF_LDFLAGS $LDFLAGS" echo " ARFLAGS = $ARFLAGS" echo diff --git a/contrib/README.md b/contrib/README.md index a2612ab958..ae1372e95d 100644 --- a/contrib/README.md +++ b/contrib/README.md @@ -26,18 +26,12 @@ The [Debian](/contrib/debian) subfolder contains the copyright file. All other packaging related files can be found in the [bitcoin-core/packaging](https://github.com/bitcoin-core/packaging) repository. -### [Gitian-descriptors](/contrib/gitian-descriptors) ### -Files used during the gitian build process. For more information about gitian, see the [the Bitcoin Core documentation repository](https://github.com/bitcoin-core/docs). - ### [Builder keys](/contrib/builder-keys) PGP keys used for signing Bitcoin Core [release](/doc/release-process.md) results. ### [MacDeploy](/contrib/macdeploy) ### Scripts and notes for Mac builds. -### [Gitian-build](/contrib/gitian-build.py) ### -Script for running full Gitian builds. - Test and Verify Tools --------------------- diff --git a/contrib/builder-keys/README.md b/contrib/builder-keys/README.md index a7c1d5ae0a..56bd87d0af 100644 --- a/contrib/builder-keys/README.md +++ b/contrib/builder-keys/README.md @@ -20,7 +20,7 @@ To fetch keys of builders and active developers, feed the list of fingerprints of the primary keys into gpg: ```sh -while read fingerprint keyholder_name; do gpg --keyserver hkp://subset.pool.sks-keyservers.net --recv-keys ${fingerprint}; done < ./keys.txt +while read fingerprint keyholder_name; do gpg --keyserver hkps://keys.openpgp.org --recv-keys ${fingerprint}; done < ./keys.txt ``` Add your key to the list if you provided Guix attestations for two major or diff --git a/contrib/builder-keys/keys.txt b/contrib/builder-keys/keys.txt index db28cd07a0..e8032f66ee 100644 --- a/contrib/builder-keys/keys.txt +++ b/contrib/builder-keys/keys.txt @@ -5,6 +5,7 @@ E944AE667CF960B1004BC32FCA662BE18B877A60 Andreas Schildbach (aschildbach) 590B7292695AFFA5B672CBB2E13FC145CD3F4304 Antoine Poinsot (darosior) 0AD83877C1F0CD1EE9BD660AD7CC770B81FD22A8 Ben Carman (benthecarman) 912FD3228387123DC97E0E57D5566241A0295FA9 BtcDrak (btcdrak) +04017A2A6D9A0CCDC81D8EC296AB007F1A7ED999 Carl Dong (dongcarl) C519EBCF3B926298946783EFF6430754120EC2F4 Christian Decker (cdecker) 18AE2F798E0D239755DA4FD24B79F986CBDF8736 Chun Kuan Le (ken2812221) 101598DC823C1B5F9A6624ABA5E0907A0380E6C3 CoinForensics (CoinForensics) @@ -19,6 +20,7 @@ D35176BE9264832E4ACA8986BF0792FBE95DC863 fivepiece (fivepiece) 01CDF4627A3B88AAE4A571C87588242FBE38D3A8 Gavin Andresen (gavinandresen) D1DBF2C4B96F2DEBF4C16654410108112E7EA81F Hennadii Stepanov (hebasto) A2FD494D0021AA9B4FA58F759102B7AE654A4A5A Ilyas Ridhuan (IlyasRidhuan) +2688F5A9A4BE0F295E921E8A25F27A38A47AD566 James O'Beirne (jamesob) D3F22A3A4C366C2DCB66D3722DA9C5A7FA81EA35 Jarol Rodriguez (jarolrod) 7480909378D544EA6B6DCEB7535B12980BB8A4D3 Jeffri H Frontz (jhfrontz) D3CC177286005BB8FF673294C5242A1AB3936517 jl2012 (jl2012) @@ -26,6 +28,7 @@ D3CC177286005BB8FF673294C5242A1AB3936517 jl2012 (jl2012) 32EE5C4C3FA15CCADB46ABE529D4BCB6416F53EC Jonas Schnelli (jonasschnelli) 4B4E840451149DD7FB0D633477DFAB5C3108B9A8 Jorge Timon (jtimon) C42AFF7C61B3E44A1454CD3557AF762DB3353322 Karl-Johan Alm (kallewoof) +70A1D47DD44F59DF8B22244333E472FE870C7E5D Kristaps Kaupe (kristapsk) 30DE693AE0DE9E37B3E7EB6BBFF0F67810C1EED1 Lisa Neigut (niftynei) E463A93F5F3117EEDE6C7316BD02942421F4889F Luke Dashjr (luke-jr) B8B3F1C0E58C15DB6A81D30C3648A882F4316B9B Marco Falke (marco) diff --git a/contrib/debian/copyright b/contrib/debian/copyright index 6d23f600c3..95a281ce05 100644 --- a/contrib/debian/copyright +++ b/contrib/debian/copyright @@ -5,7 +5,7 @@ Upstream-Contact: Satoshi Nakamoto Source: https://github.com/bitcoin/bitcoin Files: * -Copyright: 2009-2021, Bitcoin Core Developers +Copyright: 2009-2022, Bitcoin Core Developers License: Expat Comment: The Bitcoin Core Developers encompasses the current developers listed on bitcoin.org, as well as the numerous contributors to the project. diff --git a/contrib/devtools/README.md b/contrib/devtools/README.md index 1fa850af1a..79b0134adc 100644 --- a/contrib/devtools/README.md +++ b/contrib/devtools/README.md @@ -76,7 +76,7 @@ year rather than two hyphenated years. If the file already has a copyright for `The Bitcoin Core developers`, the script will exit. -gen-manpages.sh +gen-manpages.py =============== A small script to automatically create manpages in ../../doc/man by running the release binaries with the -help option. @@ -87,7 +87,7 @@ repostitory. To use this tool with out-of-tree builds set `BUILDDIR`. For example: ```bash -BUILDDIR=$PWD/build contrib/devtools/gen-manpages.sh +BUILDDIR=$PWD/build contrib/devtools/gen-manpages.py ``` security-check.py and test-security-check.py @@ -98,7 +98,7 @@ Perform basic security checks on a series of executables. symbol-check.py =============== -A script to check that the executables produced by gitian only contain +A script to check that release executables only contain certain symbols and are only linked against allowed libraries. For Linux this means checking for allowed gcc, glibc and libstdc++ version symbols. @@ -106,9 +106,9 @@ This makes sure they are still compatible with the minimum supported distributio For macOS and Windows we check that the executables are only linked against libraries we allow. -Example usage after a gitian build: +Example usage: - find ../gitian-builder/build -type f -executable | xargs python3 contrib/devtools/symbol-check.py + find ../path/to/executables -type f -executable | xargs python3 contrib/devtools/symbol-check.py If no errors occur the return value will be 0 and the output will be empty. diff --git a/contrib/devtools/copyright_header.py b/contrib/devtools/copyright_header.py index 9a555c70bb..38f3df77c9 100755 --- a/contrib/devtools/copyright_header.py +++ b/contrib/devtools/copyright_header.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2016-2020 The Bitcoin Core developers +# Copyright (c) 2016-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -33,6 +33,7 @@ EXCLUDE_DIRS = [ # git subtrees "src/crypto/ctaes/", "src/leveldb/", + "src/minisketch", "src/secp256k1/", "src/univalue/", "src/crc32c/", @@ -370,7 +371,7 @@ def create_updated_copyright_line(line, last_git_change_year): space_split = after_copyright.split(' ') year_range = space_split[0] start_year, end_year = parse_year_range(year_range) - if end_year == last_git_change_year: + if end_year >= last_git_change_year: return line return (before_copyright + copyright_splitter + year_range_to_str(start_year, last_git_change_year) + ' ' + diff --git a/contrib/devtools/gen-manpages.py b/contrib/devtools/gen-manpages.py new file mode 100755 index 0000000000..03cd3ddab4 --- /dev/null +++ b/contrib/devtools/gen-manpages.py @@ -0,0 +1,71 @@ +#!/usr/bin/env python3 +# Copyright (c) 2022 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +import os +import subprocess +import sys +import tempfile + +BINARIES = [ +'src/elementsd', +'src/elements-cli', +'src/elements-tx', +'src/elements-wallet', +'src/elements-util', +'src/qt/elements-qt', +] + +# Paths to external utilities. +git = os.getenv('GIT', 'git') +help2man = os.getenv('HELP2MAN', 'help2man') + +# If not otherwise specified, get top directory from git. +topdir = os.getenv('TOPDIR') +if not topdir: + r = subprocess.run([git, 'rev-parse', '--show-toplevel'], stdout=subprocess.PIPE, check=True, universal_newlines=True) + topdir = r.stdout.rstrip() + +# Get input and output directories. +builddir = os.getenv('BUILDDIR', topdir) +mandir = os.getenv('MANDIR', os.path.join(topdir, 'doc/man')) + +# Verify that all the required binaries are usable, and extract copyright +# message in a first pass. +versions = [] +for relpath in BINARIES: + abspath = os.path.join(builddir, relpath) + try: + r = subprocess.run([abspath, '--version'], stdout=subprocess.PIPE, universal_newlines=True) + except IOError: + print(f'{abspath} not found or not an executable', file=sys.stderr) + sys.exit(1) + # take first line (which must contain version) + verstr = r.stdout.splitlines()[0] + # last word of line is the actual version e.g. v22.99.0-5c6b3d5b3508 + verstr = verstr.split()[-1] + assert verstr.startswith('v') + # remaining lines are copyright + copyright = r.stdout.split('\n')[1:] + assert copyright[0].startswith('Copyright (C)') + + versions.append((abspath, verstr, copyright)) + +if any(verstr.endswith('-dirty') for (_, verstr, _) in versions): + print("WARNING: Binaries were built from a dirty tree.") + print('man pages generated from dirty binaries should NOT be committed.') + print('To properly generate man pages, please commit your changes (or discard them), rebuild, then run this script again.') + print() + +with tempfile.NamedTemporaryFile('w', suffix='.h2m') as footer: + # Create copyright footer, and write it to a temporary include file. + # Copyright is the same for all binaries, so just use the first. + footer.write('[COPYRIGHT]\n') + footer.write('\n'.join(versions[0][2]).strip()) + footer.flush() + + # Call the binaries through help2man to produce a manual page for each of them. + for (abspath, verstr, _) in versions: + outname = os.path.join(mandir, os.path.basename(abspath) + '.1') + print(f'Generating {outname}…') + subprocess.run([help2man, '-N', '--version-string=' + verstr, '--include=' + footer.name, '-o', outname, abspath], check=True) diff --git a/contrib/devtools/gen-manpages.sh b/contrib/devtools/gen-manpages.sh deleted file mode 100755 index 887550ae7b..0000000000 --- a/contrib/devtools/gen-manpages.sh +++ /dev/null @@ -1,53 +0,0 @@ -#!/usr/bin/env bash -# Copyright (c) 2016-2020 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. - -export LC_ALL=C -TOPDIR=${TOPDIR:-$(git rev-parse --show-toplevel)} -BUILDDIR=${BUILDDIR:-$TOPDIR} - -BINDIR=${BINDIR:-$BUILDDIR/src} -MANDIR=${MANDIR:-$TOPDIR/doc/man} - -BITCOIND=${BITCOIND:-$BINDIR/elementsd} -BITCOINCLI=${BITCOINCLI:-$BINDIR/elements-cli} -BITCOINTX=${BITCOINTX:-$BINDIR/elements-tx} -WALLET_TOOL=${WALLET_TOOL:-$BINDIR/elements-wallet} -BITCOINUTIL=${BITCOINQT:-$BINDIR/elements-util} -BITCOINQT=${BITCOINQT:-$BINDIR/qt/elements-qt} - -[ ! -x $BITCOIND ] && echo "$BITCOIND not found or not executable." && exit 1 - -# Don't allow man pages to be generated for binaries built from a dirty tree -DIRTY="" -for cmd in $BITCOIND $BITCOINCLI $BITCOINTX $WALLET_TOOL $BITCOINUTIL $BITCOINQT; do - VERSION_OUTPUT=$($cmd --version) - if [[ $VERSION_OUTPUT == *"dirty"* ]]; then - DIRTY="${DIRTY}${cmd}\n" - fi -done -if [ -n "$DIRTY" ] -then - echo -e "WARNING: the following binaries were built from a dirty tree:\n" - echo -e $DIRTY - echo "man pages generated from dirty binaries should NOT be committed." - echo "To properly generate man pages, please commit your changes to the above binaries, rebuild them, then run this script again." -fi - -# The autodetected version git tag can screw up manpage output a little bit -read -r -a BTCVER <<< "$($BITCOINCLI --version | head -n1 | awk -F'[ -]' '{ print $6, $7 }')" - -# Create a footer file with copyright content. -# This gets autodetected fine for bitcoind if --version-string is not set, -# but has different outcomes for bitcoin-qt and bitcoin-cli. -echo "[COPYRIGHT]" > footer.h2m -$BITCOIND --version | sed -n '1!p' >> footer.h2m - -for cmd in $BITCOIND $BITCOINCLI $BITCOINTX $WALLET_TOOL $BITCOINUTIL $BITCOINQT; do - cmdname="${cmd##*/}" - help2man -N --version-string=${BTCVER[0]} --include=footer.h2m -o ${MANDIR}/${cmdname}.1 ${cmd} - sed -i "s/\\\-${BTCVER[1]}//g" ${MANDIR}/${cmdname}.1 -done - -rm -f footer.h2m diff --git a/contrib/devtools/pixie.py b/contrib/devtools/pixie.py deleted file mode 100644 index 64660968ad..0000000000 --- a/contrib/devtools/pixie.py +++ /dev/null @@ -1,323 +0,0 @@ -#!/usr/bin/env python3 -# Copyright (c) 2020 Wladimir J. van der Laan -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. -''' -Compact, self-contained ELF implementation for bitcoin-core security checks. -''' -import struct -import types -from typing import Dict, List, Optional, Union, Tuple - -# you can find all these values in elf.h -EI_NIDENT = 16 - -# Byte indices in e_ident -EI_CLASS = 4 # ELFCLASSxx -EI_DATA = 5 # ELFDATAxxxx - -ELFCLASS32 = 1 # 32-bit -ELFCLASS64 = 2 # 64-bit - -ELFDATA2LSB = 1 # little endian -ELFDATA2MSB = 2 # big endian - -# relevant values for e_machine -EM_386 = 3 -EM_PPC64 = 21 -EM_ARM = 40 -EM_AARCH64 = 183 -EM_X86_64 = 62 -EM_RISCV = 243 - -# relevant values for e_type -ET_DYN = 3 - -# relevant values for sh_type -SHT_PROGBITS = 1 -SHT_STRTAB = 3 -SHT_DYNAMIC = 6 -SHT_DYNSYM = 11 -SHT_GNU_verneed = 0x6ffffffe -SHT_GNU_versym = 0x6fffffff - -# relevant values for p_type -PT_LOAD = 1 -PT_GNU_STACK = 0x6474e551 -PT_GNU_RELRO = 0x6474e552 - -# relevant values for p_flags -PF_X = (1 << 0) -PF_W = (1 << 1) -PF_R = (1 << 2) - -# relevant values for d_tag -DT_NEEDED = 1 -DT_FLAGS = 30 - -# relevant values of `d_un.d_val' in the DT_FLAGS entry -DF_BIND_NOW = 0x00000008 - -# relevant d_tags with string payload -STRING_TAGS = {DT_NEEDED} - -# rrlevant values for ST_BIND subfield of st_info (symbol binding) -STB_LOCAL = 0 - -class ELFRecord(types.SimpleNamespace): - '''Unified parsing for ELF records.''' - def __init__(self, data: bytes, offset: int, eh: 'ELFHeader', total_size: Optional[int]) -> None: - hdr_struct = self.STRUCT[eh.ei_class][0][eh.ei_data] - if total_size is not None and hdr_struct.size > total_size: - raise ValueError(f'{self.__class__.__name__} header size too small ({total_size} < {hdr_struct.size})') - for field, value in zip(self.STRUCT[eh.ei_class][1], hdr_struct.unpack(data[offset:offset + hdr_struct.size])): - setattr(self, field, value) - -def BiStruct(chars: str) -> Dict[int, struct.Struct]: - '''Compile a struct parser for both endians.''' - return { - ELFDATA2LSB: struct.Struct('<' + chars), - ELFDATA2MSB: struct.Struct('>' + chars), - } - -class ELFHeader(ELFRecord): - FIELDS = ['e_type', 'e_machine', 'e_version', 'e_entry', 'e_phoff', 'e_shoff', 'e_flags', 'e_ehsize', 'e_phentsize', 'e_phnum', 'e_shentsize', 'e_shnum', 'e_shstrndx'] - STRUCT = { - ELFCLASS32: (BiStruct('HHIIIIIHHHHHH'), FIELDS), - ELFCLASS64: (BiStruct('HHIQQQIHHHHHH'), FIELDS), - } - - def __init__(self, data: bytes, offset: int) -> None: - self.e_ident = data[offset:offset + EI_NIDENT] - if self.e_ident[0:4] != b'\x7fELF': - raise ValueError('invalid ELF magic') - self.ei_class = self.e_ident[EI_CLASS] - self.ei_data = self.e_ident[EI_DATA] - - super().__init__(data, offset + EI_NIDENT, self, None) - - def __repr__(self) -> str: - return f'Header(e_ident={self.e_ident!r}, e_type={self.e_type}, e_machine={self.e_machine}, e_version={self.e_version}, e_entry={self.e_entry}, e_phoff={self.e_phoff}, e_shoff={self.e_shoff}, e_flags={self.e_flags}, e_ehsize={self.e_ehsize}, e_phentsize={self.e_phentsize}, e_phnum={self.e_phnum}, e_shentsize={self.e_shentsize}, e_shnum={self.e_shnum}, e_shstrndx={self.e_shstrndx})' - -class Section(ELFRecord): - name: Optional[bytes] = None - FIELDS = ['sh_name', 'sh_type', 'sh_flags', 'sh_addr', 'sh_offset', 'sh_size', 'sh_link', 'sh_info', 'sh_addralign', 'sh_entsize'] - STRUCT = { - ELFCLASS32: (BiStruct('IIIIIIIIII'), FIELDS), - ELFCLASS64: (BiStruct('IIQQQQIIQQ'), FIELDS), - } - - def __init__(self, data: bytes, offset: int, eh: ELFHeader) -> None: - super().__init__(data, offset, eh, eh.e_shentsize) - self._data = data - - def __repr__(self) -> str: - return f'Section(sh_name={self.sh_name}({self.name!r}), sh_type=0x{self.sh_type:x}, sh_flags={self.sh_flags}, sh_addr=0x{self.sh_addr:x}, sh_offset=0x{self.sh_offset:x}, sh_size={self.sh_size}, sh_link={self.sh_link}, sh_info={self.sh_info}, sh_addralign={self.sh_addralign}, sh_entsize={self.sh_entsize})' - - def contents(self) -> bytes: - '''Return section contents.''' - return self._data[self.sh_offset:self.sh_offset + self.sh_size] - -class ProgramHeader(ELFRecord): - STRUCT = { - # different ELF classes have the same fields, but in a different order to optimize space versus alignment - ELFCLASS32: (BiStruct('IIIIIIII'), ['p_type', 'p_offset', 'p_vaddr', 'p_paddr', 'p_filesz', 'p_memsz', 'p_flags', 'p_align']), - ELFCLASS64: (BiStruct('IIQQQQQQ'), ['p_type', 'p_flags', 'p_offset', 'p_vaddr', 'p_paddr', 'p_filesz', 'p_memsz', 'p_align']), - } - - def __init__(self, data: bytes, offset: int, eh: ELFHeader) -> None: - super().__init__(data, offset, eh, eh.e_phentsize) - - def __repr__(self) -> str: - return f'ProgramHeader(p_type={self.p_type}, p_offset={self.p_offset}, p_vaddr={self.p_vaddr}, p_paddr={self.p_paddr}, p_filesz={self.p_filesz}, p_memsz={self.p_memsz}, p_flags={self.p_flags}, p_align={self.p_align})' - -class Symbol(ELFRecord): - STRUCT = { - # different ELF classes have the same fields, but in a different order to optimize space versus alignment - ELFCLASS32: (BiStruct('IIIBBH'), ['st_name', 'st_value', 'st_size', 'st_info', 'st_other', 'st_shndx']), - ELFCLASS64: (BiStruct('IBBHQQ'), ['st_name', 'st_info', 'st_other', 'st_shndx', 'st_value', 'st_size']), - } - - def __init__(self, data: bytes, offset: int, eh: ELFHeader, symtab: Section, strings: bytes, version: Optional[bytes]) -> None: - super().__init__(data, offset, eh, symtab.sh_entsize) - self.name = _lookup_string(strings, self.st_name) - self.version = version - - def __repr__(self) -> str: - return f'Symbol(st_name={self.st_name}({self.name!r}), st_value={self.st_value}, st_size={self.st_size}, st_info={self.st_info}, st_other={self.st_other}, st_shndx={self.st_shndx}, version={self.version!r})' - - @property - def is_import(self) -> bool: - '''Returns whether the symbol is an imported symbol.''' - return self.st_bind != STB_LOCAL and self.st_shndx == 0 - - @property - def is_export(self) -> bool: - '''Returns whether the symbol is an exported symbol.''' - return self.st_bind != STB_LOCAL and self.st_shndx != 0 - - @property - def st_bind(self) -> int: - '''Returns STB_*.''' - return self.st_info >> 4 - -class Verneed(ELFRecord): - DEF = (BiStruct('HHIII'), ['vn_version', 'vn_cnt', 'vn_file', 'vn_aux', 'vn_next']) - STRUCT = { ELFCLASS32: DEF, ELFCLASS64: DEF } - - def __init__(self, data: bytes, offset: int, eh: ELFHeader) -> None: - super().__init__(data, offset, eh, None) - - def __repr__(self) -> str: - return f'Verneed(vn_version={self.vn_version}, vn_cnt={self.vn_cnt}, vn_file={self.vn_file}, vn_aux={self.vn_aux}, vn_next={self.vn_next})' - -class Vernaux(ELFRecord): - DEF = (BiStruct('IHHII'), ['vna_hash', 'vna_flags', 'vna_other', 'vna_name', 'vna_next']) - STRUCT = { ELFCLASS32: DEF, ELFCLASS64: DEF } - - def __init__(self, data: bytes, offset: int, eh: ELFHeader, strings: bytes) -> None: - super().__init__(data, offset, eh, None) - self.name = _lookup_string(strings, self.vna_name) - - def __repr__(self) -> str: - return f'Veraux(vna_hash={self.vna_hash}, vna_flags={self.vna_flags}, vna_other={self.vna_other}, vna_name={self.vna_name}({self.name!r}), vna_next={self.vna_next})' - -class DynTag(ELFRecord): - STRUCT = { - ELFCLASS32: (BiStruct('II'), ['d_tag', 'd_val']), - ELFCLASS64: (BiStruct('QQ'), ['d_tag', 'd_val']), - } - - def __init__(self, data: bytes, offset: int, eh: ELFHeader, section: Section) -> None: - super().__init__(data, offset, eh, section.sh_entsize) - - def __repr__(self) -> str: - return f'DynTag(d_tag={self.d_tag}, d_val={self.d_val})' - -def _lookup_string(data: bytes, index: int) -> bytes: - '''Look up string by offset in ELF string table.''' - endx = data.find(b'\x00', index) - assert endx != -1 - return data[index:endx] - -VERSYM_S = BiStruct('H') # .gnu_version section has a single 16-bit integer per symbol in the linked section -def _parse_symbol_table(section: Section, strings: bytes, eh: ELFHeader, versym: bytes, verneed: Dict[int, bytes]) -> List[Symbol]: - '''Parse symbol table, return a list of symbols.''' - data = section.contents() - symbols = [] - versym_iter = (verneed.get(v[0]) for v in VERSYM_S[eh.ei_data].iter_unpack(versym)) - for ofs, version in zip(range(0, len(data), section.sh_entsize), versym_iter): - symbols.append(Symbol(data, ofs, eh, section, strings, version)) - return symbols - -def _parse_verneed(section: Section, strings: bytes, eh: ELFHeader) -> Dict[int, bytes]: - '''Parse .gnu.version_r section, return a dictionary of {versym: 'GLIBC_...'}.''' - data = section.contents() - ofs = 0 - result = {} - while True: - verneed = Verneed(data, ofs, eh) - aofs = ofs + verneed.vn_aux - while True: - vernaux = Vernaux(data, aofs, eh, strings) - result[vernaux.vna_other] = vernaux.name - if not vernaux.vna_next: - break - aofs += vernaux.vna_next - - if not verneed.vn_next: - break - ofs += verneed.vn_next - - return result - -def _parse_dyn_tags(section: Section, strings: bytes, eh: ELFHeader) -> List[Tuple[int, Union[bytes, int]]]: - '''Parse dynamic tags. Return array of tuples.''' - data = section.contents() - ofs = 0 - result = [] - for ofs in range(0, len(data), section.sh_entsize): - tag = DynTag(data, ofs, eh, section) - val = _lookup_string(strings, tag.d_val) if tag.d_tag in STRING_TAGS else tag.d_val - result.append((tag.d_tag, val)) - - return result - -class ELFFile: - sections: List[Section] - program_headers: List[ProgramHeader] - dyn_symbols: List[Symbol] - dyn_tags: List[Tuple[int, Union[bytes, int]]] - - def __init__(self, data: bytes) -> None: - self.data = data - self.hdr = ELFHeader(self.data, 0) - self._load_sections() - self._load_program_headers() - self._load_dyn_symbols() - self._load_dyn_tags() - self._section_to_segment_mapping() - - def _load_sections(self) -> None: - self.sections = [] - for idx in range(self.hdr.e_shnum): - offset = self.hdr.e_shoff + idx * self.hdr.e_shentsize - self.sections.append(Section(self.data, offset, self.hdr)) - - shstr = self.sections[self.hdr.e_shstrndx].contents() - for section in self.sections: - section.name = _lookup_string(shstr, section.sh_name) - - def _load_program_headers(self) -> None: - self.program_headers = [] - for idx in range(self.hdr.e_phnum): - offset = self.hdr.e_phoff + idx * self.hdr.e_phentsize - self.program_headers.append(ProgramHeader(self.data, offset, self.hdr)) - - def _load_dyn_symbols(self) -> None: - # first, load 'verneed' section - verneed = None - for section in self.sections: - if section.sh_type == SHT_GNU_verneed: - strtab = self.sections[section.sh_link].contents() # associated string table - assert verneed is None # only one section of this kind please - verneed = _parse_verneed(section, strtab, self.hdr) - assert verneed is not None - - # then, correlate GNU versym sections with dynamic symbol sections - versym = {} - for section in self.sections: - if section.sh_type == SHT_GNU_versym: - versym[section.sh_link] = section - - # finally, load dynsym sections - self.dyn_symbols = [] - for idx, section in enumerate(self.sections): - if section.sh_type == SHT_DYNSYM: # find dynamic symbol tables - strtab_data = self.sections[section.sh_link].contents() # associated string table - versym_data = versym[idx].contents() # associated symbol version table - self.dyn_symbols += _parse_symbol_table(section, strtab_data, self.hdr, versym_data, verneed) - - def _load_dyn_tags(self) -> None: - self.dyn_tags = [] - for idx, section in enumerate(self.sections): - if section.sh_type == SHT_DYNAMIC: # find dynamic tag tables - strtab = self.sections[section.sh_link].contents() # associated string table - self.dyn_tags += _parse_dyn_tags(section, strtab, self.hdr) - - def _section_to_segment_mapping(self) -> None: - for ph in self.program_headers: - ph.sections = [] - for section in self.sections: - if ph.p_vaddr <= section.sh_addr < (ph.p_vaddr + ph.p_memsz): - ph.sections.append(section) - - def query_dyn_tags(self, tag_in: int) -> List[Union[int, bytes]]: - '''Return the values of all dyn tags with the specified tag.''' - return [val for (tag, val) in self.dyn_tags if tag == tag_in] - - -def load(filename: str) -> ELFFile: - with open(filename, 'rb') as f: - data = f.read() - return ELFFile(data) diff --git a/contrib/devtools/security-check.py b/contrib/devtools/security-check.py index 0b59d8eada..e6a29b73b9 100755 --- a/contrib/devtools/security-check.py +++ b/contrib/devtools/security-check.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2015-2020 The Bitcoin Core developers +# Copyright (c) 2015-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. ''' @@ -8,248 +8,258 @@ Exit status will be 0 if successful, and the program will be silent. Otherwise the exit status will be 1 and it will log which executables failed which checks. ''' import sys -from typing import List, Optional +from typing import List -import lief -import pixie +import lief #type:ignore -def check_ELF_PIE(executable) -> bool: - ''' - Check for position independent executable (PIE), allowing for address space randomization. - ''' - elf = pixie.load(executable) - return elf.hdr.e_type == pixie.ET_DYN +# temporary constant, to be replaced with lief.ELF.ARCH.RISCV +# https://github.com/lief-project/LIEF/pull/562 +LIEF_ELF_ARCH_RISCV = lief.ELF.ARCH(243) -def check_ELF_NX(executable) -> bool: - ''' - Check that no sections are writable and executable (including the stack) - ''' - elf = pixie.load(executable) - have_wx = False - have_gnu_stack = False - for ph in elf.program_headers: - if ph.p_type == pixie.PT_GNU_STACK: - have_gnu_stack = True - if (ph.p_flags & pixie.PF_W) != 0 and (ph.p_flags & pixie.PF_X) != 0: # section is both writable and executable - have_wx = True - return have_gnu_stack and not have_wx - -def check_ELF_RELRO(executable) -> bool: +def check_ELF_RELRO(binary) -> bool: ''' Check for read-only relocations. GNU_RELRO program header must exist Dynamic section must have BIND_NOW flag ''' - elf = pixie.load(executable) have_gnu_relro = False - for ph in elf.program_headers: + for segment in binary.segments: # Note: not checking p_flags == PF_R: here as linkers set the permission differently # This does not affect security: the permission flags of the GNU_RELRO program # header are ignored, the PT_LOAD header determines the effective permissions. # However, the dynamic linker need to write to this area so these are RW. # Glibc itself takes care of mprotecting this area R after relocations are finished. # See also https://marc.info/?l=binutils&m=1498883354122353 - if ph.p_type == pixie.PT_GNU_RELRO: + if segment.type == lief.ELF.SEGMENT_TYPES.GNU_RELRO: have_gnu_relro = True have_bindnow = False - for flags in elf.query_dyn_tags(pixie.DT_FLAGS): - assert isinstance(flags, int) - if flags & pixie.DF_BIND_NOW: + try: + flags = binary.get(lief.ELF.DYNAMIC_TAGS.FLAGS) + if flags.value & lief.ELF.DYNAMIC_FLAGS.BIND_NOW: have_bindnow = True + except: + have_bindnow = False return have_gnu_relro and have_bindnow -def check_ELF_Canary(executable) -> bool: +def check_ELF_Canary(binary) -> bool: ''' Check for use of stack canary ''' - elf = pixie.load(executable) - ok = False - for symbol in elf.dyn_symbols: - if symbol.name == b'__stack_chk_fail': - ok = True - return ok + return binary.has_symbol('__stack_chk_fail') -def check_ELF_separate_code(executable): +def check_ELF_separate_code(binary): ''' Check that sections are appropriately separated in virtual memory, based on their permissions. This checks for missing -Wl,-z,separate-code and potentially other problems. ''' - elf = pixie.load(executable) - R = pixie.PF_R - W = pixie.PF_W - E = pixie.PF_X + R = lief.ELF.SEGMENT_FLAGS.R + W = lief.ELF.SEGMENT_FLAGS.W + E = lief.ELF.SEGMENT_FLAGS.X EXPECTED_FLAGS = { # Read + execute - b'.init': R | E, - b'.plt': R | E, - b'.plt.got': R | E, - b'.plt.sec': R | E, - b'.text': R | E, - b'.fini': R | E, + '.init': R | E, + '.plt': R | E, + '.plt.got': R | E, + '.plt.sec': R | E, + '.text': R | E, + '.fini': R | E, # Read-only data - b'.interp': R, - b'.note.gnu.property': R, - b'.note.gnu.build-id': R, - b'.note.ABI-tag': R, - b'.gnu.hash': R, - b'.dynsym': R, - b'.dynstr': R, - b'.gnu.version': R, - b'.gnu.version_r': R, - b'.rela.dyn': R, - b'.rela.plt': R, - b'.rodata': R, - b'.eh_frame_hdr': R, - b'.eh_frame': R, - b'.qtmetadata': R, - b'.gcc_except_table': R, - b'.stapsdt.base': R, + '.interp': R, + '.note.gnu.property': R, + '.note.gnu.build-id': R, + '.note.ABI-tag': R, + '.gnu.hash': R, + '.dynsym': R, + '.dynstr': R, + '.gnu.version': R, + '.gnu.version_r': R, + '.rela.dyn': R, + '.rela.plt': R, + '.rodata': R, + '.eh_frame_hdr': R, + '.eh_frame': R, + '.qtmetadata': R, + '.gcc_except_table': R, + '.stapsdt.base': R, # Writable data - b'.init_array': R | W, - b'.fini_array': R | W, - b'.dynamic': R | W, - b'.got': R | W, - b'.data': R | W, - b'.bss': R | W, + '.init_array': R | W, + '.fini_array': R | W, + '.dynamic': R | W, + '.got': R | W, + '.data': R | W, + '.bss': R | W, } - if elf.hdr.e_machine == pixie.EM_PPC64: + if binary.header.machine_type == lief.ELF.ARCH.PPC64: # .plt is RW on ppc64 even with separate-code - EXPECTED_FLAGS[b'.plt'] = R | W + EXPECTED_FLAGS['.plt'] = R | W # For all LOAD program headers get mapping to the list of sections, # and for each section, remember the flags of the associated program header. flags_per_section = {} - for ph in elf.program_headers: - if ph.p_type == pixie.PT_LOAD: - for section in ph.sections: + for segment in binary.segments: + if segment.type == lief.ELF.SEGMENT_TYPES.LOAD: + for section in segment.sections: assert(section.name not in flags_per_section) - flags_per_section[section.name] = ph.p_flags + flags_per_section[section.name] = segment.flags # Spot-check ELF LOAD program header flags per section # If these sections exist, check them against the expected R/W/E flags for (section, flags) in flags_per_section.items(): if section in EXPECTED_FLAGS: - if EXPECTED_FLAGS[section] != flags: + if int(EXPECTED_FLAGS[section]) != int(flags): return False return True -def check_PE_DYNAMIC_BASE(executable) -> bool: +def check_ELF_control_flow(binary) -> bool: + ''' + Check for control flow instrumentation + ''' + main = binary.get_function_address('main') + content = binary.get_content_from_virtual_address(main, 4, lief.Binary.VA_TYPES.AUTO) + + if content == [243, 15, 30, 250]: # endbr64 + return True + return False + +def check_PE_DYNAMIC_BASE(binary) -> bool: '''PIE: DllCharacteristics bit 0x40 signifies dynamicbase (ASLR)''' - binary = lief.parse(executable) return lief.PE.DLL_CHARACTERISTICS.DYNAMIC_BASE in binary.optional_header.dll_characteristics_lists # Must support high-entropy 64-bit address space layout randomization # in addition to DYNAMIC_BASE to have secure ASLR. -def check_PE_HIGH_ENTROPY_VA(executable) -> bool: +def check_PE_HIGH_ENTROPY_VA(binary) -> bool: '''PIE: DllCharacteristics bit 0x20 signifies high-entropy ASLR''' - binary = lief.parse(executable) return lief.PE.DLL_CHARACTERISTICS.HIGH_ENTROPY_VA in binary.optional_header.dll_characteristics_lists -def check_PE_RELOC_SECTION(executable) -> bool: +def check_PE_RELOC_SECTION(binary) -> bool: '''Check for a reloc section. This is required for functional ASLR.''' - binary = lief.parse(executable) return binary.has_relocations -def check_MACHO_NOUNDEFS(executable) -> bool: +def check_PE_control_flow(binary) -> bool: + ''' + Check for control flow instrumentation + ''' + main = binary.get_symbol('main').value + + section_addr = binary.section_from_rva(main).virtual_address + virtual_address = binary.optional_header.imagebase + section_addr + main + + content = binary.get_content_from_virtual_address(virtual_address, 4, lief.Binary.VA_TYPES.VA) + + if content == [243, 15, 30, 250]: # endbr64 + return True + return False + +def check_MACHO_NOUNDEFS(binary) -> bool: ''' Check for no undefined references. ''' - binary = lief.parse(executable) return binary.header.has(lief.MachO.HEADER_FLAGS.NOUNDEFS) -def check_MACHO_LAZY_BINDINGS(executable) -> bool: +def check_MACHO_LAZY_BINDINGS(binary) -> bool: ''' Check for no lazy bindings. We don't use or check for MH_BINDATLOAD. See #18295. ''' - binary = lief.parse(executable) return binary.dyld_info.lazy_bind == (0,0) -def check_MACHO_Canary(executable) -> bool: +def check_MACHO_Canary(binary) -> bool: ''' Check for use of stack canary ''' - binary = lief.parse(executable) return binary.has_symbol('___stack_chk_fail') -def check_PIE(executable) -> bool: +def check_PIE(binary) -> bool: ''' Check for position independent executable (PIE), allowing for address space randomization. ''' - binary = lief.parse(executable) return binary.is_pie -def check_NX(executable) -> bool: +def check_NX(binary) -> bool: ''' Check for no stack execution ''' - binary = lief.parse(executable) return binary.has_nx -def check_control_flow(executable) -> bool: +def check_MACHO_control_flow(binary) -> bool: ''' Check for control flow instrumentation ''' - binary = lief.parse(executable) - content = binary.get_content_from_virtual_address(binary.entrypoint, 4, lief.Binary.VA_TYPES.AUTO) if content == [243, 15, 30, 250]: # endbr64 return True return False - -CHECKS = { -'ELF': [ - ('PIE', check_ELF_PIE), - ('NX', check_ELF_NX), +BASE_ELF = [ + ('PIE', check_PIE), + ('NX', check_NX), ('RELRO', check_ELF_RELRO), ('Canary', check_ELF_Canary), ('separate_code', check_ELF_separate_code), -], -'PE': [ +] + +BASE_PE = [ ('PIE', check_PIE), ('DYNAMIC_BASE', check_PE_DYNAMIC_BASE), ('HIGH_ENTROPY_VA', check_PE_HIGH_ENTROPY_VA), ('NX', check_NX), - ('RELOC_SECTION', check_PE_RELOC_SECTION) -], -'MACHO': [ - ('PIE', check_PIE), + ('RELOC_SECTION', check_PE_RELOC_SECTION), + ('CONTROL_FLOW', check_PE_control_flow), +] + +BASE_MACHO = [ ('NOUNDEFS', check_MACHO_NOUNDEFS), - ('NX', check_NX), ('LAZY_BINDINGS', check_MACHO_LAZY_BINDINGS), ('Canary', check_MACHO_Canary), - ('CONTROL_FLOW', check_control_flow), ] -} -def identify_executable(executable) -> Optional[str]: - with open(filename, 'rb') as f: - magic = f.read(4) - if magic.startswith(b'MZ'): - return 'PE' - elif magic.startswith(b'\x7fELF'): - return 'ELF' - elif magic.startswith(b'\xcf\xfa'): - return 'MACHO' - return None +CHECKS = { + lief.EXE_FORMATS.ELF: { + lief.ARCHITECTURES.X86: BASE_ELF + [('CONTROL_FLOW', check_ELF_control_flow)], + lief.ARCHITECTURES.ARM: BASE_ELF, + lief.ARCHITECTURES.ARM64: BASE_ELF, + lief.ARCHITECTURES.PPC: BASE_ELF, + LIEF_ELF_ARCH_RISCV: BASE_ELF, + }, + lief.EXE_FORMATS.PE: { + lief.ARCHITECTURES.X86: BASE_PE, + }, + lief.EXE_FORMATS.MACHO: { + lief.ARCHITECTURES.X86: BASE_MACHO + [('PIE', check_PIE), + ('NX', check_NX), + ('CONTROL_FLOW', check_MACHO_control_flow)], + lief.ARCHITECTURES.ARM64: BASE_MACHO, + } +} if __name__ == '__main__': retval: int = 0 for filename in sys.argv[1:]: try: - etype = identify_executable(filename) - if etype is None: - print(f'{filename}: unknown format') + binary = lief.parse(filename) + etype = binary.format + arch = binary.abstract.header.architecture + binary.concrete + + if etype == lief.EXE_FORMATS.UNKNOWN: + print(f'{filename}: unknown executable format') retval = 1 continue + if arch == lief.ARCHITECTURES.NONE: + if binary.header.machine_type == LIEF_ELF_ARCH_RISCV: + arch = LIEF_ELF_ARCH_RISCV + else: + print(f'{filename}: unknown architecture') + retval = 1 + continue + failed: List[str] = [] - for (name, func) in CHECKS[etype]: - if not func(filename): + for (name, func) in CHECKS[etype][arch]: + if not func(binary): failed.append(name) if failed: print(f'{filename}: failed {" ".join(failed)}') diff --git a/contrib/devtools/symbol-check.py b/contrib/devtools/symbol-check.py index 61f727fa63..461132ae63 100755 --- a/contrib/devtools/symbol-check.py +++ b/contrib/devtools/symbol-check.py @@ -10,45 +10,41 @@ Example usage: find ../path/to/binaries -type f -executable | xargs python3 contrib/devtools/symbol-check.py ''' -import subprocess import sys -from typing import List, Optional +from typing import List, Dict -import lief -import pixie +import lief #type:ignore -from utils import determine_wellknown_cmd +# temporary constant, to be replaced with lief.ELF.ARCH.RISCV +# https://github.com/lief-project/LIEF/pull/562 +LIEF_ELF_ARCH_RISCV = lief.ELF.ARCH(243) -# Debian 8 (Jessie) EOL: 2020. https://wiki.debian.org/DebianReleases#Production_Releases +# Debian 9 (Stretch) EOL: 2022. https://wiki.debian.org/DebianReleases#Production_Releases # -# - g++ version 4.9.2 (https://packages.debian.org/search?suite=jessie&arch=any&searchon=names&keywords=g%2B%2B) -# - libc version 2.19 (https://packages.debian.org/search?suite=jessie&arch=any&searchon=names&keywords=libc6) +# - g++ version 6.3.0 (https://packages.debian.org/search?suite=stretch&arch=any&searchon=names&keywords=g%2B%2B) +# - libc version 2.24 (https://packages.debian.org/search?suite=stretch&arch=any&searchon=names&keywords=libc6) # -# Ubuntu 16.04 (Xenial) EOL: 2024. https://wiki.ubuntu.com/Releases +# Ubuntu 16.04 (Xenial) EOL: 2026. https://wiki.ubuntu.com/Releases # -# - g++ version 5.3.1 (https://packages.ubuntu.com/search?keywords=g%2B%2B&searchon=names&suite=xenial§ion=all) -# - libc version 2.23.0 (https://packages.ubuntu.com/search?keywords=libc6&searchon=names&suite=xenial§ion=all) +# - g++ version 5.3.1 +# - libc version 2.23 # -# CentOS 7 EOL: 2024. https://wiki.centos.org/FAQ/General +# CentOS Stream 8 EOL: 2024. https://wiki.centos.org/About/Product # -# - g++ version 4.8.5 (http://mirror.centos.org/centos/7/os/x86_64/Packages/) -# - libc version 2.17 (http://mirror.centos.org/centos/7/os/x86_64/Packages/) -# -# Taking the minimum of these as our target. -# -# According to GNU ABI document (https://gcc.gnu.org/onlinedocs/libstdc++/manual/abi.html) this corresponds to: -# GCC 4.8.5: GCC_4.8.0 -# (glibc) GLIBC_2_17 +# - g++ version 8.5.0 (http://mirror.centos.org/centos/8-stream/AppStream/x86_64/os/Packages/) +# - libc version 2.28 (http://mirror.centos.org/centos/8-stream/AppStream/x86_64/os/Packages/) # +# See https://gcc.gnu.org/onlinedocs/libstdc++/manual/abi.html for more info. + MAX_VERSIONS = { 'GCC': (4,8,0), 'GLIBC': { - pixie.EM_386: (2,17), - pixie.EM_X86_64: (2,17), - pixie.EM_ARM: (2,17), - pixie.EM_AARCH64:(2,17), - pixie.EM_PPC64: (2,17), - pixie.EM_RISCV: (2,27), + lief.ELF.ARCH.i386: (2,18), + lief.ELF.ARCH.x86_64: (2,18), + lief.ELF.ARCH.ARM: (2,18), + lief.ELF.ARCH.AARCH64:(2,18), + lief.ELF.ARCH.PPC64: (2,18), + LIEF_ELF_ARCH_RISCV: (2,27), }, 'LIBATOMIC': (1,0), 'V': (0,5,0), # xkb (bitcoin-qt only) @@ -58,10 +54,35 @@ MAX_VERSIONS = { # Ignore symbols that are exported as part of every executable IGNORE_EXPORTS = { -'_edata', '_end', '__end__', '_init', '__bss_start', '__bss_start__', '_bss_end__', '__bss_end__', '_fini', '_IO_stdin_used', 'stdin', 'stdout', 'stderr', +'_edata', '_end', '__end__', '_init', '__bss_start', '__bss_start__', '_bss_end__', +'__bss_end__', '_fini', '_IO_stdin_used', 'stdin', 'stdout', 'stderr', 'environ', '_environ', '__environ', } +# Expected linker-loader names can be found here: +# https://sourceware.org/glibc/wiki/ABIList?action=recall&rev=16 +ELF_INTERPRETER_NAMES: Dict[lief.ELF.ARCH, Dict[lief.ENDIANNESS, str]] = { + lief.ELF.ARCH.i386: { + lief.ENDIANNESS.LITTLE: "/lib/ld-linux.so.2", + }, + lief.ELF.ARCH.x86_64: { + lief.ENDIANNESS.LITTLE: "/lib64/ld-linux-x86-64.so.2", + }, + lief.ELF.ARCH.ARM: { + lief.ENDIANNESS.LITTLE: "/lib/ld-linux-armhf.so.3", + }, + lief.ELF.ARCH.AARCH64: { + lief.ENDIANNESS.LITTLE: "/lib/ld-linux-aarch64.so.1", + }, + lief.ELF.ARCH.PPC64: { + lief.ENDIANNESS.BIG: "/lib64/ld64.so.1", + lief.ENDIANNESS.LITTLE: "/lib64/ld64.so.2", + }, + LIEF_ELF_ARCH_RISCV: { + lief.ENDIANNESS.LITTLE: "/lib/ld-linux-riscv64-lp64d.so.1", + }, +} + # Allowed NEEDED libraries ELF_ALLOWED_LIBRARIES = { # bitcoind and bitcoin-qt @@ -84,7 +105,19 @@ ELF_ALLOWED_LIBRARIES = { 'libxkbcommon-x11.so.0', # keyboard keymapping 'libfontconfig.so.1', # font support 'libfreetype.so.6', # font parsing -'libdl.so.2' # programming interface to dynamic linker +'libdl.so.2', # programming interface to dynamic linker +'libxcb-icccm.so.4', +'libxcb-image.so.0', +'libxcb-shm.so.0', +'libxcb-keysyms.so.1', +'libxcb-randr.so.0', +'libxcb-render-util.so.0', +'libxcb-render.so.0', +'libxcb-shape.so.0', +'libxcb-sync.so.1', +'libxcb-xfixes.so.0', +'libxcb-xinerama.so.0', +'libxcb-xkb.so.1', } MACHO_ALLOWED_LIBRARIES = { @@ -95,6 +128,7 @@ MACHO_ALLOWED_LIBRARIES = { 'AppKit', # user interface 'ApplicationServices', # common application tasks. 'Carbon', # deprecated c back-compat API +'ColorSync', 'CoreFoundation', # low level func, data types 'CoreGraphics', # 2D rendering 'CoreServices', # operating system services @@ -133,31 +167,8 @@ PE_ALLOWED_LIBRARIES = { 'WTSAPI32.dll', } -class CPPFilt(object): - ''' - Demangle C++ symbol names. - - Use a pipe to the 'c++filt' command. - ''' - def __init__(self): - self.proc = subprocess.Popen(determine_wellknown_cmd('CPPFILT', 'c++filt'), stdin=subprocess.PIPE, stdout=subprocess.PIPE, universal_newlines=True) - - def __call__(self, mangled): - self.proc.stdin.write(mangled + '\n') - self.proc.stdin.flush() - return self.proc.stdout.readline().rstrip() - - def close(self): - self.proc.stdin.close() - self.proc.stdout.close() - self.proc.wait() - def check_version(max_versions, version, arch) -> bool: - if '_' in version: - (lib, _, ver) = version.rpartition('_') - else: - lib = version - ver = '0' + (lib, _, ver) = version.rpartition('_') ver = tuple([int(x) for x in ver.split('.')]) if not lib in max_versions: return False @@ -166,48 +177,45 @@ def check_version(max_versions, version, arch) -> bool: else: return ver <= max_versions[lib][arch] -def check_imported_symbols(filename) -> bool: - elf = pixie.load(filename) - cppfilt = CPPFilt() +def check_imported_symbols(binary) -> bool: ok: bool = True - for symbol in elf.dyn_symbols: - if not symbol.is_import: + for symbol in binary.imported_symbols: + if not symbol.imported: continue - sym = symbol.name.decode() - version = symbol.version.decode() if symbol.version is not None else None - if version and not check_version(MAX_VERSIONS, version, elf.hdr.e_machine): - print('{}: symbol {} from unsupported version {}'.format(filename, cppfilt(sym), version)) - ok = False + + version = symbol.symbol_version if symbol.has_version else None + + if version: + aux_version = version.symbol_version_auxiliary.name if version.has_auxiliary_version else None + if aux_version and not check_version(MAX_VERSIONS, aux_version, binary.header.machine_type): + print(f'{filename}: symbol {symbol.name} from unsupported version {version}') + ok = False return ok -def check_exported_symbols(filename) -> bool: - elf = pixie.load(filename) - cppfilt = CPPFilt() +def check_exported_symbols(binary) -> bool: ok: bool = True - for symbol in elf.dyn_symbols: - if not symbol.is_export: + + for symbol in binary.dynamic_symbols: + if not symbol.exported: continue - sym = symbol.name.decode() - if elf.hdr.e_machine == pixie.EM_RISCV or sym in IGNORE_EXPORTS: + name = symbol.name + if binary.header.machine_type == LIEF_ELF_ARCH_RISCV or name in IGNORE_EXPORTS: continue - print('{}: export of symbol {} not allowed'.format(filename, cppfilt(sym))) + print(f'{binary.name}: export of symbol {name} not allowed!') ok = False return ok -def check_ELF_libraries(filename) -> bool: +def check_ELF_libraries(binary) -> bool: ok: bool = True - elf = pixie.load(filename) - for library_name in elf.query_dyn_tags(pixie.DT_NEEDED): - assert(isinstance(library_name, bytes)) - if library_name.decode() not in ELF_ALLOWED_LIBRARIES: - print('{}: NEEDED library {} is not allowed'.format(filename, library_name.decode())) + for library in binary.libraries: + if library not in ELF_ALLOWED_LIBRARIES: + print(f'{filename}: {library} is not in ALLOWED_LIBRARIES!') ok = False return ok -def check_MACHO_libraries(filename) -> bool: +def check_MACHO_libraries(binary) -> bool: ok: bool = True - binary = lief.parse(filename) for dylib in binary.libraries: split = dylib.name.split('/') if split[-1] not in MACHO_ALLOWED_LIBRARIES: @@ -215,76 +223,68 @@ def check_MACHO_libraries(filename) -> bool: ok = False return ok -def check_MACHO_min_os(filename) -> bool: - binary = lief.parse(filename) - if binary.build_version.minos == [10,14,0]: +def check_MACHO_min_os(binary) -> bool: + if binary.build_version.minos == [10,15,0]: return True return False -def check_MACHO_sdk(filename) -> bool: - binary = lief.parse(filename) - if binary.build_version.sdk == [10, 15, 6]: +def check_MACHO_sdk(binary) -> bool: + if binary.build_version.sdk == [11, 0, 0]: return True return False -def check_PE_libraries(filename) -> bool: +def check_PE_libraries(binary) -> bool: ok: bool = True - binary = lief.parse(filename) for dylib in binary.libraries: if dylib not in PE_ALLOWED_LIBRARIES: print(f'{dylib} is not in ALLOWED_LIBRARIES!') ok = False return ok -def check_PE_subsystem_version(filename) -> bool: - binary = lief.parse(filename) +def check_PE_subsystem_version(binary) -> bool: major: int = binary.optional_header.major_subsystem_version minor: int = binary.optional_header.minor_subsystem_version if major == 6 and minor == 1: return True return False +def check_ELF_interpreter(binary) -> bool: + expected_interpreter = ELF_INTERPRETER_NAMES[binary.header.machine_type][binary.abstract.header.endianness] + + return binary.concrete.interpreter == expected_interpreter + CHECKS = { -'ELF': [ +lief.EXE_FORMATS.ELF: [ ('IMPORTED_SYMBOLS', check_imported_symbols), ('EXPORTED_SYMBOLS', check_exported_symbols), - ('LIBRARY_DEPENDENCIES', check_ELF_libraries) + ('LIBRARY_DEPENDENCIES', check_ELF_libraries), + ('INTERPRETER_NAME', check_ELF_interpreter), ], -'MACHO': [ +lief.EXE_FORMATS.MACHO: [ ('DYNAMIC_LIBRARIES', check_MACHO_libraries), ('MIN_OS', check_MACHO_min_os), ('SDK', check_MACHO_sdk), ], -'PE' : [ +lief.EXE_FORMATS.PE: [ ('DYNAMIC_LIBRARIES', check_PE_libraries), ('SUBSYSTEM_VERSION', check_PE_subsystem_version), ] } -def identify_executable(executable) -> Optional[str]: - with open(filename, 'rb') as f: - magic = f.read(4) - if magic.startswith(b'MZ'): - return 'PE' - elif magic.startswith(b'\x7fELF'): - return 'ELF' - elif magic.startswith(b'\xcf\xfa'): - return 'MACHO' - return None - if __name__ == '__main__': retval: int = 0 for filename in sys.argv[1:]: try: - etype = identify_executable(filename) - if etype is None: - print(f'{filename}: unknown format') + binary = lief.parse(filename) + etype = binary.format + if etype == lief.EXE_FORMATS.UNKNOWN: + print(f'{filename}: unknown executable format') retval = 1 continue failed: List[str] = [] for (name, func) in CHECKS[etype]: - if not func(filename): + if not func(binary): failed.append(name) if failed: print(f'{filename}: failed {" ".join(failed)}') diff --git a/contrib/devtools/test-security-check.py b/contrib/devtools/test-security-check.py index 14058e2cc8..d3d225f3ab 100755 --- a/contrib/devtools/test-security-check.py +++ b/contrib/devtools/test-security-check.py @@ -1,12 +1,14 @@ #!/usr/bin/env python3 -# Copyright (c) 2015-2020 The Bitcoin Core developers +# Copyright (c) 2015-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. ''' Test script for security-check.py ''' +import lief #type:ignore import os import subprocess +from typing import List import unittest from utils import determine_wellknown_cmd @@ -27,29 +29,62 @@ def clean_files(source, executable): os.remove(executable) def call_security_check(cc, source, executable, options): - subprocess.run([*cc,source,'-o',executable] + options, check=True) + # This should behave the same as AC_TRY_LINK, so arrange well-known flags + # in the same order as autoconf would. + # + # See the definitions for ac_link in autoconf's lib/autoconf/c.m4 file for + # reference. + env_flags: List[str] = [] + for var in ['CFLAGS', 'CPPFLAGS', 'LDFLAGS']: + env_flags += filter(None, os.environ.get(var, '').split(' ')) + + subprocess.run([*cc,source,'-o',executable] + env_flags + options, check=True) p = subprocess.run(['./contrib/devtools/security-check.py',executable], stdout=subprocess.PIPE, universal_newlines=True) return (p.returncode, p.stdout.rstrip()) +def get_arch(cc, source, executable): + subprocess.run([*cc, source, '-o', executable], check=True) + binary = lief.parse(executable) + arch = binary.abstract.header.architecture + os.remove(executable) + return arch + class TestSecurityChecks(unittest.TestCase): def test_ELF(self): source = 'test1.c' executable = 'test1' cc = determine_wellknown_cmd('CC', 'gcc') write_testcode(source) + arch = get_arch(cc, source, executable) - self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-zexecstack','-fno-stack-protector','-Wl,-znorelro','-no-pie','-fno-PIE', '-Wl,-z,separate-code']), - (1, executable+': failed PIE NX RELRO Canary')) - self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-znoexecstack','-fno-stack-protector','-Wl,-znorelro','-no-pie','-fno-PIE', '-Wl,-z,separate-code']), - (1, executable+': failed PIE RELRO Canary')) - self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-znoexecstack','-fstack-protector-all','-Wl,-znorelro','-no-pie','-fno-PIE', '-Wl,-z,separate-code']), - (1, executable+': failed PIE RELRO')) - self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-znoexecstack','-fstack-protector-all','-Wl,-znorelro','-pie','-fPIE', '-Wl,-z,separate-code']), - (1, executable+': failed RELRO')) - self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-znoexecstack','-fstack-protector-all','-Wl,-zrelro','-Wl,-z,now','-pie','-fPIE', '-Wl,-z,noseparate-code']), - (1, executable+': failed separate_code')) - self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-znoexecstack','-fstack-protector-all','-Wl,-zrelro','-Wl,-z,now','-pie','-fPIE', '-Wl,-z,separate-code']), - (0, '')) + if arch == lief.ARCHITECTURES.X86: + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-zexecstack','-fno-stack-protector','-Wl,-znorelro','-no-pie','-fno-PIE', '-Wl,-z,separate-code']), + (1, executable+': failed PIE NX RELRO Canary CONTROL_FLOW')) + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-znoexecstack','-fno-stack-protector','-Wl,-znorelro','-no-pie','-fno-PIE', '-Wl,-z,separate-code']), + (1, executable+': failed PIE RELRO Canary CONTROL_FLOW')) + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-znoexecstack','-fstack-protector-all','-Wl,-znorelro','-no-pie','-fno-PIE', '-Wl,-z,separate-code']), + (1, executable+': failed PIE RELRO CONTROL_FLOW')) + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-znoexecstack','-fstack-protector-all','-Wl,-znorelro','-pie','-fPIE', '-Wl,-z,separate-code']), + (1, executable+': failed RELRO CONTROL_FLOW')) + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-znoexecstack','-fstack-protector-all','-Wl,-zrelro','-Wl,-z,now','-pie','-fPIE', '-Wl,-z,noseparate-code']), + (1, executable+': failed separate_code CONTROL_FLOW')) + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-znoexecstack','-fstack-protector-all','-Wl,-zrelro','-Wl,-z,now','-pie','-fPIE', '-Wl,-z,separate-code']), + (1, executable+': failed CONTROL_FLOW')) + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-znoexecstack','-fstack-protector-all','-Wl,-zrelro','-Wl,-z,now','-pie','-fPIE', '-Wl,-z,separate-code', '-fcf-protection=full']), + (0, '')) + else: + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-zexecstack','-fno-stack-protector','-Wl,-znorelro','-no-pie','-fno-PIE', '-Wl,-z,separate-code']), + (1, executable+': failed PIE NX RELRO Canary')) + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-znoexecstack','-fno-stack-protector','-Wl,-znorelro','-no-pie','-fno-PIE', '-Wl,-z,separate-code']), + (1, executable+': failed PIE RELRO Canary')) + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-znoexecstack','-fstack-protector-all','-Wl,-znorelro','-no-pie','-fno-PIE', '-Wl,-z,separate-code']), + (1, executable+': failed PIE RELRO')) + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-znoexecstack','-fstack-protector-all','-Wl,-znorelro','-pie','-fPIE', '-Wl,-z,separate-code']), + (1, executable+': failed RELRO')) + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-znoexecstack','-fstack-protector-all','-Wl,-zrelro','-Wl,-z,now','-pie','-fPIE', '-Wl,-z,noseparate-code']), + (1, executable+': failed separate_code')) + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-znoexecstack','-fstack-protector-all','-Wl,-zrelro','-Wl,-z,now','-pie','-fPIE', '-Wl,-z,separate-code']), + (0, '')) clean_files(source, executable) @@ -59,17 +94,19 @@ class TestSecurityChecks(unittest.TestCase): cc = determine_wellknown_cmd('CC', 'x86_64-w64-mingw32-gcc') write_testcode(source) - self.assertEqual(call_security_check(cc, source, executable, ['-Wl,--no-nxcompat','-Wl,--disable-reloc-section','-Wl,--no-dynamicbase','-Wl,--no-high-entropy-va','-no-pie','-fno-PIE']), - (1, executable+': failed PIE DYNAMIC_BASE HIGH_ENTROPY_VA NX RELOC_SECTION')) - self.assertEqual(call_security_check(cc, source, executable, ['-Wl,--nxcompat','-Wl,--disable-reloc-section','-Wl,--no-dynamicbase','-Wl,--no-high-entropy-va','-no-pie','-fno-PIE']), - (1, executable+': failed PIE DYNAMIC_BASE HIGH_ENTROPY_VA RELOC_SECTION')) - self.assertEqual(call_security_check(cc, source, executable, ['-Wl,--nxcompat','-Wl,--enable-reloc-section','-Wl,--no-dynamicbase','-Wl,--no-high-entropy-va','-no-pie','-fno-PIE']), - (1, executable+': failed PIE DYNAMIC_BASE HIGH_ENTROPY_VA')) - self.assertEqual(call_security_check(cc, source, executable, ['-Wl,--nxcompat','-Wl,--enable-reloc-section','-Wl,--no-dynamicbase','-Wl,--no-high-entropy-va','-pie','-fPIE']), - (1, executable+': failed PIE DYNAMIC_BASE HIGH_ENTROPY_VA')) # -pie -fPIE does nothing unless --dynamicbase is also supplied - self.assertEqual(call_security_check(cc, source, executable, ['-Wl,--nxcompat','-Wl,--enable-reloc-section','-Wl,--dynamicbase','-Wl,--no-high-entropy-va','-pie','-fPIE']), - (1, executable+': failed HIGH_ENTROPY_VA')) + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,--disable-nxcompat','-Wl,--disable-reloc-section','-Wl,--disable-dynamicbase','-Wl,--disable-high-entropy-va','-no-pie','-fno-PIE']), + (1, executable+': failed PIE DYNAMIC_BASE HIGH_ENTROPY_VA NX RELOC_SECTION CONTROL_FLOW')) + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,--nxcompat','-Wl,--disable-reloc-section','-Wl,--disable-dynamicbase','-Wl,--disable-high-entropy-va','-no-pie','-fno-PIE']), + (1, executable+': failed PIE DYNAMIC_BASE HIGH_ENTROPY_VA RELOC_SECTION CONTROL_FLOW')) + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,--nxcompat','-Wl,--enable-reloc-section','-Wl,--disable-dynamicbase','-Wl,--disable-high-entropy-va','-no-pie','-fno-PIE']), + (1, executable+': failed PIE DYNAMIC_BASE HIGH_ENTROPY_VA CONTROL_FLOW')) + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,--nxcompat','-Wl,--enable-reloc-section','-Wl,--disable-dynamicbase','-Wl,--disable-high-entropy-va','-pie','-fPIE']), + (1, executable+': failed PIE DYNAMIC_BASE HIGH_ENTROPY_VA CONTROL_FLOW')) # -pie -fPIE does nothing unless --dynamicbase is also supplied + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,--nxcompat','-Wl,--enable-reloc-section','-Wl,--dynamicbase','-Wl,--disable-high-entropy-va','-pie','-fPIE']), + (1, executable+': failed HIGH_ENTROPY_VA CONTROL_FLOW')) self.assertEqual(call_security_check(cc, source, executable, ['-Wl,--nxcompat','-Wl,--enable-reloc-section','-Wl,--dynamicbase','-Wl,--high-entropy-va','-pie','-fPIE']), + (1, executable+': failed CONTROL_FLOW')) + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,--nxcompat','-Wl,--enable-reloc-section','-Wl,--dynamicbase','-Wl,--high-entropy-va','-pie','-fPIE', '-fcf-protection=full']), (0, '')) clean_files(source, executable) @@ -79,21 +116,34 @@ class TestSecurityChecks(unittest.TestCase): executable = 'test1' cc = determine_wellknown_cmd('CC', 'clang') write_testcode(source) + arch = get_arch(cc, source, executable) + + if arch == lief.ARCHITECTURES.X86: + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-no_pie','-Wl,-flat_namespace','-Wl,-allow_stack_execute','-fno-stack-protector']), + (1, executable+': failed NOUNDEFS LAZY_BINDINGS Canary PIE NX CONTROL_FLOW')) + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-no_pie','-Wl,-flat_namespace','-Wl,-allow_stack_execute','-fstack-protector-all']), + (1, executable+': failed NOUNDEFS LAZY_BINDINGS PIE NX CONTROL_FLOW')) + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-no_pie','-Wl,-flat_namespace','-fstack-protector-all']), + (1, executable+': failed NOUNDEFS LAZY_BINDINGS PIE CONTROL_FLOW')) + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-no_pie','-fstack-protector-all']), + (1, executable+': failed LAZY_BINDINGS PIE CONTROL_FLOW')) + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-no_pie','-Wl,-bind_at_load','-fstack-protector-all']), + (1, executable+': failed PIE CONTROL_FLOW')) + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-no_pie','-Wl,-bind_at_load','-fstack-protector-all', '-fcf-protection=full']), + (1, executable+': failed PIE')) + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-pie','-Wl,-bind_at_load','-fstack-protector-all', '-fcf-protection=full']), + (0, '')) + else: + # arm64 darwin doesn't support non-PIE binaries, control flow or executable stacks + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-flat_namespace','-fno-stack-protector']), + (1, executable+': failed NOUNDEFS LAZY_BINDINGS Canary')) + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-flat_namespace','-fstack-protector-all']), + (1, executable+': failed NOUNDEFS LAZY_BINDINGS')) + self.assertEqual(call_security_check(cc, source, executable, ['-fstack-protector-all']), + (1, executable+': failed LAZY_BINDINGS')) + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-bind_at_load','-fstack-protector-all']), + (0, '')) - self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-no_pie','-Wl,-flat_namespace','-Wl,-allow_stack_execute','-fno-stack-protector']), - (1, executable+': failed PIE NOUNDEFS NX LAZY_BINDINGS Canary CONTROL_FLOW')) - self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-no_pie','-Wl,-flat_namespace','-Wl,-allow_stack_execute','-fstack-protector-all']), - (1, executable+': failed PIE NOUNDEFS NX LAZY_BINDINGS CONTROL_FLOW')) - self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-no_pie','-Wl,-flat_namespace','-fstack-protector-all']), - (1, executable+': failed PIE NOUNDEFS LAZY_BINDINGS CONTROL_FLOW')) - self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-no_pie','-fstack-protector-all']), - (1, executable+': failed PIE LAZY_BINDINGS CONTROL_FLOW')) - self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-no_pie','-Wl,-bind_at_load','-fstack-protector-all']), - (1, executable+': failed PIE CONTROL_FLOW')) - self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-no_pie','-Wl,-bind_at_load','-fstack-protector-all', '-fcf-protection=full']), - (1, executable+': failed PIE')) - self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-pie','-Wl,-bind_at_load','-fstack-protector-all', '-fcf-protection=full']), - (0, '')) clean_files(source, executable) diff --git a/contrib/devtools/test-symbol-check.py b/contrib/devtools/test-symbol-check.py index 7d83c5f751..e1a2ebc491 100755 --- a/contrib/devtools/test-symbol-check.py +++ b/contrib/devtools/test-symbol-check.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2020 The Bitcoin Core developers +# Copyright (c) 2020-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. ''' @@ -13,7 +13,16 @@ import unittest from utils import determine_wellknown_cmd def call_symbol_check(cc: List[str], source, executable, options): - subprocess.run([*cc,source,'-o',executable] + options, check=True) + # This should behave the same as AC_TRY_LINK, so arrange well-known flags + # in the same order as autoconf would. + # + # See the definitions for ac_link in autoconf's lib/autoconf/c.m4 file for + # reference. + env_flags: List[str] = [] + for var in ['CFLAGS', 'CPPFLAGS', 'LDFLAGS']: + env_flags += filter(None, os.environ.get(var, '').split(' ')) + + subprocess.run([*cc,source,'-o',executable] + env_flags + options, check=True) p = subprocess.run(['./contrib/devtools/symbol-check.py',executable], stdout=subprocess.PIPE, universal_newlines=True) os.remove(source) os.remove(executable) @@ -35,7 +44,7 @@ class TestSymbolChecks(unittest.TestCase): self.skipTest("test not available for RISC-V") # nextup was introduced in GLIBC 2.24, so is newer than our supported - # glibc (2.17), and available in our release build environment (2.24). + # glibc (2.18), and available in our release build environment (2.24). with open(source, 'w', encoding="utf8") as f: f.write(''' #define _GNU_SOURCE @@ -51,7 +60,7 @@ class TestSymbolChecks(unittest.TestCase): ''') self.assertEqual(call_symbol_check(cc, source, executable, ['-lm']), - (1, executable + ': symbol nextup from unsupported version GLIBC_2.24\n' + + (1, executable + ': symbol nextup from unsupported version GLIBC_2.24(3)\n' + executable + ': failed IMPORTED_SYMBOLS')) # -lutil is part of the libc6 package so a safe bet that it's installed @@ -70,23 +79,24 @@ class TestSymbolChecks(unittest.TestCase): ''') self.assertEqual(call_symbol_check(cc, source, executable, ['-lutil']), - (1, executable + ': NEEDED library libutil.so.1 is not allowed\n' + + (1, executable + ': libutil.so.1 is not in ALLOWED_LIBRARIES!\n' + executable + ': failed LIBRARY_DEPENDENCIES')) - # finally, check a conforming file that simply uses a math function + # finally, check a simple conforming binary source = 'test3.c' executable = 'test3' with open(source, 'w', encoding="utf8") as f: f.write(''' - #include + #include int main() { - return (int)pow(2.0, 4.0); + printf("42"); + return 0; } ''') - self.assertEqual(call_symbol_check(cc, source, executable, ['-lm']), + self.assertEqual(call_symbol_check(cc, source, executable, []), (0, '')) def test_MACHO(self): @@ -136,7 +146,7 @@ class TestSymbolChecks(unittest.TestCase): } ''') - self.assertEqual(call_symbol_check(cc, source, executable, ['-Wl,-platform_version','-Wl,macos', '-Wl,10.14', '-Wl,11.4']), + self.assertEqual(call_symbol_check(cc, source, executable, ['-Wl,-platform_version','-Wl,macos', '-Wl,10.15', '-Wl,11.4']), (1, f'{executable}: failed SDK')) def test_PE(self): diff --git a/contrib/gitian-build.py b/contrib/gitian-build.py deleted file mode 100755 index 5df87d9e70..0000000000 --- a/contrib/gitian-build.py +++ /dev/null @@ -1,263 +0,0 @@ -#!/usr/bin/env python3 -# Copyright (c) 2018-2020 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. - -import argparse -import os -import subprocess -import sys - -def setup(): - global args, workdir - programs = ['ruby', 'git', 'make', 'wget', 'curl'] - if args.kvm: - programs += ['apt-cacher-ng', 'python-vm-builder', 'qemu-kvm', 'qemu-utils'] - elif args.docker: - if not os.path.isfile('/lib/systemd/system/docker.service'): - dockers = ['docker.io', 'docker-ce'] - for i in dockers: - return_code = subprocess.call(['sudo', 'apt-get', 'install', '-qq', i]) - if return_code == 0: - break - if return_code != 0: - print('Cannot find any way to install Docker.', file=sys.stderr) - sys.exit(1) - else: - programs += ['apt-cacher-ng', 'lxc', 'debootstrap'] - subprocess.check_call(['sudo', 'apt-get', 'install', '-qq'] + programs) - if not os.path.isdir('gitian.sigs'): - subprocess.check_call(['git', 'clone', 'https://github.com/bitcoin-core/gitian.sigs.git']) - if not os.path.isdir('bitcoin-detached-sigs'): - subprocess.check_call(['git', 'clone', 'https://github.com/bitcoin-core/bitcoin-detached-sigs.git']) - if not os.path.isdir('gitian-builder'): - subprocess.check_call(['git', 'clone', 'https://github.com/devrandom/gitian-builder.git']) - if not os.path.isdir('bitcoin'): - subprocess.check_call(['git', 'clone', 'https://github.com/bitcoin/bitcoin.git']) - os.chdir('gitian-builder') - make_image_prog = ['bin/make-base-vm', '--suite', 'focal', '--arch', 'amd64'] - if args.docker: - make_image_prog += ['--docker'] - elif not args.kvm: - make_image_prog += ['--lxc', '--disksize', '13000'] - subprocess.check_call(make_image_prog) - os.chdir(workdir) - if args.is_focal and not args.kvm and not args.docker: - subprocess.check_call(['sudo', 'sed', '-i', 's/lxcbr0/br0/', '/etc/default/lxc-net']) - print('Reboot is required') - sys.exit(0) - -def build(): - global args, workdir - - os.makedirs('bitcoin-binaries/' + args.version, exist_ok=True) - print('\nBuilding Dependencies\n') - os.chdir('gitian-builder') - os.makedirs('inputs', exist_ok=True) - - subprocess.check_call(['wget', '-O', 'inputs/osslsigncode-2.0.tar.gz', 'https://github.com/mtrojnar/osslsigncode/archive/2.0.tar.gz']) - subprocess.check_call(["echo '5a60e0a4b3e0b4d655317b2f12a810211c50242138322b16e7e01c6fbb89d92f inputs/osslsigncode-2.0.tar.gz' | sha256sum -c"], shell=True) - subprocess.check_call(['make', '-C', '../bitcoin/depends', 'download', 'SOURCES_PATH=' + os.getcwd() + '/cache/common']) - - if args.linux: - print('\nCompiling ' + args.version + ' Linux') - subprocess.check_call(['bin/gbuild', '-j', args.jobs, '-m', args.memory, '--commit', 'bitcoin='+args.commit, '--url', 'bitcoin='+args.url, '../bitcoin/contrib/gitian-descriptors/gitian-linux.yml']) - subprocess.check_call(['bin/gsign', '-p', args.sign_prog, '--signer', args.signer, '--release', args.version+'-linux', '--destination', '../gitian.sigs/', '../bitcoin/contrib/gitian-descriptors/gitian-linux.yml']) - subprocess.check_call('mv build/out/bitcoin-*.tar.gz build/out/src/bitcoin-*.tar.gz ../bitcoin-binaries/'+args.version, shell=True) - - if args.windows: - print('\nCompiling ' + args.version + ' Windows') - subprocess.check_call(['bin/gbuild', '-j', args.jobs, '-m', args.memory, '--commit', 'bitcoin='+args.commit, '--url', 'bitcoin='+args.url, '../bitcoin/contrib/gitian-descriptors/gitian-win.yml']) - subprocess.check_call(['bin/gsign', '-p', args.sign_prog, '--signer', args.signer, '--release', args.version+'-win-unsigned', '--destination', '../gitian.sigs/', '../bitcoin/contrib/gitian-descriptors/gitian-win.yml']) - subprocess.check_call('mv build/out/bitcoin-*-win-unsigned.tar.gz inputs/', shell=True) - subprocess.check_call('mv build/out/bitcoin-*.zip build/out/bitcoin-*.exe build/out/src/bitcoin-*.tar.gz ../bitcoin-binaries/'+args.version, shell=True) - - if args.macos: - print('\nCompiling ' + args.version + ' MacOS') - subprocess.check_call(['bin/gbuild', '-j', args.jobs, '-m', args.memory, '--commit', 'bitcoin='+args.commit, '--url', 'bitcoin='+args.url, '../bitcoin/contrib/gitian-descriptors/gitian-osx.yml']) - subprocess.check_call(['bin/gsign', '-p', args.sign_prog, '--signer', args.signer, '--release', args.version+'-osx-unsigned', '--destination', '../gitian.sigs/', '../bitcoin/contrib/gitian-descriptors/gitian-osx.yml']) - subprocess.check_call('mv build/out/bitcoin-*-osx-unsigned.tar.gz inputs/', shell=True) - subprocess.check_call('mv build/out/bitcoin-*.tar.gz build/out/bitcoin-*.dmg build/out/src/bitcoin-*.tar.gz ../bitcoin-binaries/'+args.version, shell=True) - - os.chdir(workdir) - - if args.commit_files: - print('\nCommitting '+args.version+' Unsigned Sigs\n') - os.chdir('gitian.sigs') - subprocess.check_call(['git', 'add', args.version+'-linux/'+args.signer]) - subprocess.check_call(['git', 'add', args.version+'-win-unsigned/'+args.signer]) - subprocess.check_call(['git', 'add', args.version+'-osx-unsigned/'+args.signer]) - subprocess.check_call(['git', 'commit', '-m', 'Add '+args.version+' unsigned sigs for '+args.signer]) - os.chdir(workdir) - -def sign(): - global args, workdir - os.chdir('gitian-builder') - - if args.windows: - print('\nSigning ' + args.version + ' Windows') - subprocess.check_call('cp inputs/bitcoin-' + args.version + '-win-unsigned.tar.gz inputs/bitcoin-win-unsigned.tar.gz', shell=True) - subprocess.check_call(['bin/gbuild', '--skip-image', '--upgrade', '--commit', 'signature='+args.commit, '../bitcoin/contrib/gitian-descriptors/gitian-win-signer.yml']) - subprocess.check_call(['bin/gsign', '-p', args.sign_prog, '--signer', args.signer, '--release', args.version+'-win-signed', '--destination', '../gitian.sigs/', '../bitcoin/contrib/gitian-descriptors/gitian-win-signer.yml']) - subprocess.check_call('mv build/out/bitcoin-*win64-setup.exe ../bitcoin-binaries/'+args.version, shell=True) - - if args.macos: - print('\nSigning ' + args.version + ' MacOS') - subprocess.check_call('cp inputs/bitcoin-' + args.version + '-osx-unsigned.tar.gz inputs/bitcoin-osx-unsigned.tar.gz', shell=True) - subprocess.check_call(['bin/gbuild', '--skip-image', '--upgrade', '--commit', 'signature='+args.commit, '../bitcoin/contrib/gitian-descriptors/gitian-osx-signer.yml']) - subprocess.check_call(['bin/gsign', '-p', args.sign_prog, '--signer', args.signer, '--release', args.version+'-osx-signed', '--destination', '../gitian.sigs/', '../bitcoin/contrib/gitian-descriptors/gitian-osx-signer.yml']) - subprocess.check_call('mv build/out/bitcoin-osx-signed.dmg ../bitcoin-binaries/'+args.version+'/bitcoin-'+args.version+'-osx.dmg', shell=True) - - os.chdir(workdir) - - if args.commit_files: - print('\nCommitting '+args.version+' Signed Sigs\n') - os.chdir('gitian.sigs') - subprocess.check_call(['git', 'add', args.version+'-win-signed/'+args.signer]) - subprocess.check_call(['git', 'add', args.version+'-osx-signed/'+args.signer]) - subprocess.check_call(['git', 'commit', '-a', '-m', 'Add '+args.version+' signed binary sigs for '+args.signer]) - os.chdir(workdir) - -def verify(): - global args, workdir - rc = 0 - os.chdir('gitian-builder') - - print('\nVerifying v'+args.version+' Linux\n') - if subprocess.call(['bin/gverify', '-v', '-d', '../gitian.sigs/', '-r', args.version+'-linux', '../bitcoin/contrib/gitian-descriptors/gitian-linux.yml']): - print('Verifying v'+args.version+' Linux FAILED\n') - rc = 1 - - print('\nVerifying v'+args.version+' Windows\n') - if subprocess.call(['bin/gverify', '-v', '-d', '../gitian.sigs/', '-r', args.version+'-win-unsigned', '../bitcoin/contrib/gitian-descriptors/gitian-win.yml']): - print('Verifying v'+args.version+' Windows FAILED\n') - rc = 1 - - print('\nVerifying v'+args.version+' MacOS\n') - if subprocess.call(['bin/gverify', '-v', '-d', '../gitian.sigs/', '-r', args.version+'-osx-unsigned', '../bitcoin/contrib/gitian-descriptors/gitian-osx.yml']): - print('Verifying v'+args.version+' MacOS FAILED\n') - rc = 1 - - print('\nVerifying v'+args.version+' Signed Windows\n') - if subprocess.call(['bin/gverify', '-v', '-d', '../gitian.sigs/', '-r', args.version+'-win-signed', '../bitcoin/contrib/gitian-descriptors/gitian-win-signer.yml']): - print('Verifying v'+args.version+' Signed Windows FAILED\n') - rc = 1 - - print('\nVerifying v'+args.version+' Signed MacOS\n') - if subprocess.call(['bin/gverify', '-v', '-d', '../gitian.sigs/', '-r', args.version+'-osx-signed', '../bitcoin/contrib/gitian-descriptors/gitian-osx-signer.yml']): - print('Verifying v'+args.version+' Signed MacOS FAILED\n') - rc = 1 - - os.chdir(workdir) - return rc - -def main(): - global args, workdir - - parser = argparse.ArgumentParser(description='Script for running full Gitian builds.') - parser.add_argument('-c', '--commit', action='store_true', dest='commit', help='Indicate that the version argument is for a commit or branch') - parser.add_argument('-p', '--pull', action='store_true', dest='pull', help='Indicate that the version argument is the number of a github repository pull request') - parser.add_argument('-u', '--url', dest='url', default='https://github.com/bitcoin/bitcoin', help='Specify the URL of the repository. Default is %(default)s') - parser.add_argument('-v', '--verify', action='store_true', dest='verify', help='Verify the Gitian build') - parser.add_argument('-b', '--build', action='store_true', dest='build', help='Do a Gitian build') - parser.add_argument('-s', '--sign', action='store_true', dest='sign', help='Make signed binaries for Windows and MacOS') - parser.add_argument('-B', '--buildsign', action='store_true', dest='buildsign', help='Build both signed and unsigned binaries') - parser.add_argument('-o', '--os', dest='os', default='lwm', help='Specify which Operating Systems the build is for. Default is %(default)s. l for Linux, w for Windows, m for MacOS') - parser.add_argument('-j', '--jobs', dest='jobs', default='2', help='Number of processes to use. Default %(default)s') - parser.add_argument('-m', '--memory', dest='memory', default='2000', help='Memory to allocate in MiB. Default %(default)s') - parser.add_argument('-k', '--kvm', action='store_true', dest='kvm', help='Use KVM instead of LXC') - parser.add_argument('-d', '--docker', action='store_true', dest='docker', help='Use Docker instead of LXC') - parser.add_argument('-S', '--setup', action='store_true', dest='setup', help='Set up the Gitian building environment. Only works on Debian-based systems (Ubuntu, Debian)') - parser.add_argument('-D', '--detach-sign', action='store_true', dest='detach_sign', help='Create the assert file for detached signing. Will not commit anything.') - parser.add_argument('-n', '--no-commit', action='store_false', dest='commit_files', help='Do not commit anything to git') - parser.add_argument('signer', nargs='?', help='GPG signer to sign each build assert file') - parser.add_argument('version', nargs='?', help='Version number, commit, or branch to build. If building a commit or branch, the -c option must be specified') - - args = parser.parse_args() - workdir = os.getcwd() - - args.is_focal = b'focal' in subprocess.check_output(['lsb_release', '-cs']) - - if args.kvm and args.docker: - raise Exception('Error: cannot have both kvm and docker') - - # Ensure no more than one environment variable for gitian-builder (USE_LXC, USE_VBOX, USE_DOCKER) is set as they - # can interfere (e.g., USE_LXC being set shadows USE_DOCKER; for details see gitian-builder/libexec/make-clean-vm). - os.environ['USE_LXC'] = '' - os.environ['USE_VBOX'] = '' - os.environ['USE_DOCKER'] = '' - if args.docker: - os.environ['USE_DOCKER'] = '1' - elif not args.kvm: - os.environ['USE_LXC'] = '1' - if 'GITIAN_HOST_IP' not in os.environ.keys(): - os.environ['GITIAN_HOST_IP'] = '10.0.3.1' - if 'LXC_GUEST_IP' not in os.environ.keys(): - os.environ['LXC_GUEST_IP'] = '10.0.3.5' - - if args.setup: - setup() - - if args.buildsign: - args.build = True - args.sign = True - - if not args.build and not args.sign and not args.verify: - sys.exit(0) - - args.linux = 'l' in args.os - args.windows = 'w' in args.os - args.macos = 'm' in args.os - - # Disable for MacOS if no SDK found - if args.macos and not os.path.isfile('gitian-builder/inputs/Xcode-12.1-12A7403-extracted-SDK-with-libcxx-headers.tar.gz'): - print('Cannot build for MacOS, SDK does not exist. Will build for other OSes') - args.macos = False - - args.sign_prog = 'true' if args.detach_sign else 'gpg --detach-sign' - - script_name = os.path.basename(sys.argv[0]) - if not args.signer: - print(script_name+': Missing signer') - print('Try '+script_name+' --help for more information') - sys.exit(1) - if not args.version: - print(script_name+': Missing version') - print('Try '+script_name+' --help for more information') - sys.exit(1) - - # Add leading 'v' for tags - if args.commit and args.pull: - raise Exception('Cannot have both commit and pull') - args.commit = ('' if args.commit else 'v') + args.version - - os.chdir('bitcoin') - if args.pull: - subprocess.check_call(['git', 'fetch', args.url, 'refs/pull/'+args.version+'/merge']) - os.chdir('../gitian-builder/inputs/bitcoin') - subprocess.check_call(['git', 'fetch', args.url, 'refs/pull/'+args.version+'/merge']) - args.commit = subprocess.check_output(['git', 'show', '-s', '--format=%H', 'FETCH_HEAD'], universal_newlines=True, encoding='utf8').strip() - args.version = 'pull-' + args.version - print(args.commit) - subprocess.check_call(['git', 'fetch']) - subprocess.check_call(['git', 'checkout', args.commit]) - os.chdir(workdir) - - os.chdir('gitian-builder') - subprocess.check_call(['git', 'pull']) - os.chdir(workdir) - - if args.build: - build() - - if args.sign: - sign() - - if args.verify: - os.chdir('gitian.sigs') - subprocess.check_call(['git', 'pull']) - os.chdir(workdir) - sys.exit(verify()) - -if __name__ == '__main__': - main() diff --git a/contrib/gitian-descriptors/assign_DISTNAME b/contrib/gitian-descriptors/assign_DISTNAME deleted file mode 100644 index 13469d6b75..0000000000 --- a/contrib/gitian-descriptors/assign_DISTNAME +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright (c) 2020 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. -# -# A helper script to be sourced into the gitian descriptors - -if RECENT_TAG="$(git describe --exact-match HEAD 2> /dev/null)"; then - RECENT_TAG="${RECENT_TAG#elements-}" - VERSION="${RECENT_TAG#v}" -else - VERSION="$(git rev-parse --short=12 HEAD)" -fi -DISTNAME="elements-${VERSION}" diff --git a/contrib/gitian-descriptors/gitian-linux.yml b/contrib/gitian-descriptors/gitian-linux.yml deleted file mode 100644 index 99410ca553..0000000000 --- a/contrib/gitian-descriptors/gitian-linux.yml +++ /dev/null @@ -1,173 +0,0 @@ ---- -name: "elements-linux-22" -enable_cache: true -distro: "ubuntu" -suites: -- "focal" -architectures: -- "amd64" -packages: -# Common dependencies. -- "autoconf" -- "automake" -- "binutils" -- "bison" -- "bsdmainutils" -- "ca-certificates" -- "curl" -- "faketime" -- "g++-8" -- "gcc-8" -- "git" -- "libtool" -- "patch" -- "pkg-config" -- "python3" -- "python3-pip" -# Cross compilation HOSTS: -# - arm-linux-gnueabihf -- "binutils-arm-linux-gnueabihf" -- "g++-8-arm-linux-gnueabihf" -# - aarch64-linux-gnu -- "binutils-aarch64-linux-gnu" -- "g++-8-aarch64-linux-gnu" -# - powerpc64-linux-gnu -- "binutils-powerpc64-linux-gnu" -- "g++-8-powerpc64-linux-gnu" -# - powerpc64le-linux-gnu -- "binutils-powerpc64le-linux-gnu" -- "g++-8-powerpc64le-linux-gnu" -# - riscv64-linux-gnu -- "binutils-riscv64-linux-gnu" -- "g++-8-riscv64-linux-gnu" -# Elements -- "icnsutils" -- "imagemagick" -- "librsvg2-2" -- "librsvg2-bin" -remotes: -- "url": "https://github.com/ElementsProject/elements.git" - "dir": "elements" -files: [] -script: | - set -e -o pipefail - - WRAP_DIR=$HOME/wrapped - HOSTS="x86_64-linux-gnu arm-linux-gnueabihf aarch64-linux-gnu powerpc64-linux-gnu powerpc64le-linux-gnu riscv64-linux-gnu" - CONFIGFLAGS="--enable-glibc-back-compat --enable-reduce-exports --disable-bench --disable-gui-tests --disable-fuzz-binary" - FAKETIME_HOST_PROGS="gcc g++" - FAKETIME_PROGS="date ar ranlib nm" - HOST_CFLAGS="-O2 -g" - HOST_CXXFLAGS="-O2 -g" - HOST_LDFLAGS_BASE="-static-libstdc++ -Wl,-O2" - - export TZ="UTC" - export BUILD_DIR="$PWD" - mkdir -p ${WRAP_DIR} - if test -n "$GBUILD_CACHE_ENABLED"; then - export SOURCES_PATH=${GBUILD_COMMON_CACHE} - export BASE_CACHE=${GBUILD_PACKAGE_CACHE} - mkdir -p ${BASE_CACHE} ${SOURCES_PATH} - fi - - # Use $LIB in LD_PRELOAD to avoid hardcoding the dir (See `man ld.so`) - function create_global_faketime_wrappers { - for prog in ${FAKETIME_PROGS}; do - echo '#!/usr/bin/env bash' > ${WRAP_DIR}/${prog} - echo "REAL=\`which -a ${prog} | grep -v ${WRAP_DIR}/${prog} | head -1\`" >> ${WRAP_DIR}/${prog} - echo "export LD_PRELOAD='/usr/\$LIB/faketime/libfaketime.so.1'" >> ${WRAP_DIR}/${prog} - echo "export FAKETIME=\"$1\"" >> ${WRAP_DIR}/${prog} - echo "exec \"\$REAL\" \"\$@\"" >> $WRAP_DIR/${prog} - chmod +x ${WRAP_DIR}/${prog} - done - } - - function create_per-host_faketime_wrappers { - for i in $HOSTS; do - for prog in ${FAKETIME_HOST_PROGS}; do - if which ${i}-${prog}-8 - then - echo '#!/usr/bin/env bash' > ${WRAP_DIR}/${i}-${prog} - echo "REAL=\`which -a ${i}-${prog}-8 | grep -v ${WRAP_DIR}/${i}-${prog} | head -1\`" >> ${WRAP_DIR}/${i}-${prog} - echo "export LD_PRELOAD='/usr/\$LIB/faketime/libfaketime.so.1'" >> ${WRAP_DIR}/${i}-${prog} - echo "export FAKETIME=\"$1\"" >> ${WRAP_DIR}/${i}-${prog} - if [ "${i:0:11}" = "powerpc64le" ]; then - echo "exec \"\$REAL\" -mcpu=power8 -mtune=power9 \"\$@\"" >> $WRAP_DIR/${i}-${prog} - elif [ "${i:0:9}" = "powerpc64" ]; then - echo "exec \"\$REAL\" -mcpu=970 -mtune=power9 \"\$@\"" >> $WRAP_DIR/${i}-${prog} - else - echo "exec \"\$REAL\" \"\$@\"" >> $WRAP_DIR/${i}-${prog} - fi - chmod +x ${WRAP_DIR}/${i}-${prog} - fi - done - done - } - - pip3 install lief==0.11.5 - - # Faketime for depends so intermediate results are comparable - export PATH_orig=${PATH} - create_global_faketime_wrappers "2000-01-01 12:00:00" - create_per-host_faketime_wrappers "2000-01-01 12:00:00" - export PATH=${WRAP_DIR}:${PATH} - - cd elements - BASEPREFIX="${PWD}/depends" - # Build dependencies for each host - for i in $HOSTS; do - make ${MAKEOPTS} -C ${BASEPREFIX} HOST="${i}" CC=${i}-gcc-8 CXX=${i}-g++-8 - done - - # Faketime for binaries - export PATH=${PATH_orig} - create_global_faketime_wrappers "${REFERENCE_DATETIME}" - create_per-host_faketime_wrappers "${REFERENCE_DATETIME}" - export PATH=${WRAP_DIR}:${PATH} - - # Define DISTNAME variable. - # shellcheck source=contrib/gitian-descriptors/assign_DISTNAME - source contrib/gitian-descriptors/assign_DISTNAME - - GIT_ARCHIVE="${OUTDIR}/src/${DISTNAME}.tar.gz" - - # Create the source tarball - mkdir -p "$(dirname "$GIT_ARCHIVE")" - git archive --prefix="${DISTNAME}/" --output="$GIT_ARCHIVE" HEAD - - ORIGPATH="$PATH" - # Extract the git archive into a dir for each host and build - for i in ${HOSTS}; do - export PATH=${BASEPREFIX}/${i}/native/bin:${ORIGPATH} - if [ "${i}" = "powerpc64-linux-gnu" ]; then - # Workaround for https://bugs.launchpad.net/ubuntu/+source/gcc-8-cross-ports/+bug/1853740 - # TODO: remove this when no longer needed - HOST_LDFLAGS="${HOST_LDFLAGS_BASE} -Wl,-z,noexecstack" - else - HOST_LDFLAGS="${HOST_LDFLAGS_BASE}" - fi - mkdir -p distsrc-${i} - cd distsrc-${i} - INSTALLPATH="${PWD}/installed/${DISTNAME}" - mkdir -p ${INSTALLPATH} - tar --strip-components=1 -xf "${GIT_ARCHIVE}" - - ./autogen.sh - CONFIG_SITE=${BASEPREFIX}/${i}/share/config.site ./configure --prefix=/ --disable-ccache --disable-maintainer-mode --disable-dependency-tracking ${CONFIGFLAGS} CFLAGS="${HOST_CFLAGS}" CXXFLAGS="${HOST_CXXFLAGS}" LDFLAGS="${HOST_LDFLAGS}" CC=${i}-gcc-8 CXX=${i}-g++-8 - make ${MAKEOPTS} - make ${MAKEOPTS} -C src check-security - make ${MAKEOPTS} -C src check-symbols - make install DESTDIR=${INSTALLPATH} - cd installed - find . -name "lib*.la" -delete - find . -name "lib*.a" -delete - rm -rf ${DISTNAME}/lib/pkgconfig - - find ${DISTNAME}/bin -type f -executable -print0 | xargs -0 -n1 -I{} ../contrib/devtools/split-debug.sh {} {} {}.dbg - find ${DISTNAME}/lib -type f -print0 | xargs -0 -n1 -I{} ../contrib/devtools/split-debug.sh {} {} {}.dbg - cp ../README.md ${DISTNAME}/ - find ${DISTNAME} -not -name "*.dbg" | sort | tar --mtime="$REFERENCE_DATETIME" --no-recursion --mode='u+rw,go+r-w,a+X' --owner=0 --group=0 -c -T - | gzip -9n > ${OUTDIR}/${DISTNAME}-${i}.tar.gz - find ${DISTNAME} -name "*.dbg" | sort | tar --mtime="$REFERENCE_DATETIME" --no-recursion --mode='u+rw,go+r-w,a+X' --owner=0 --group=0 -c -T - | gzip -9n > ${OUTDIR}/${DISTNAME}-${i}-debug.tar.gz - cd ../../ - rm -rf distsrc-${i} - done diff --git a/contrib/gitian-descriptors/gitian-osx-signer.yml b/contrib/gitian-descriptors/gitian-osx-signer.yml deleted file mode 100644 index addad0a5d2..0000000000 --- a/contrib/gitian-descriptors/gitian-osx-signer.yml +++ /dev/null @@ -1,53 +0,0 @@ ---- -name: "bitcoin-dmg-signer" -distro: "ubuntu" -suites: -- "focal" -architectures: -- "amd64" -packages: -- "faketime" -- "xorriso" -- "python3-pip" -remotes: -- "url": "https://github.com/bitcoin-core/bitcoin-detached-sigs.git" - "dir": "signature" -- "url": "https://github.com/achow101/signapple.git" - "dir": "signapple" - "commit": "b084cbbf44d5330448ffce0c7d118f75781b64bd" -files: -- "bitcoin-osx-unsigned.tar.gz" -script: | - set -e -o pipefail - - WRAP_DIR=$HOME/wrapped - mkdir -p ${WRAP_DIR} - export PATH="$PWD":$PATH - FAKETIME_PROGS="dmg xorrisofs" - - # Create global faketime wrappers - for prog in ${FAKETIME_PROGS}; do - echo '#!/usr/bin/env bash' > ${WRAP_DIR}/${prog} - echo "REAL=\`which -a ${prog} | grep -v ${WRAP_DIR}/${prog} | head -1\`" >> ${WRAP_DIR}/${prog} - echo "export LD_PRELOAD='/usr/\$LIB/faketime/libfaketime.so.1'" >> ${WRAP_DIR}/${prog} - echo "export FAKETIME=\"${REFERENCE_DATETIME}\"" >> ${WRAP_DIR}/${prog} - echo "exec \"\$REAL\" \"\$@\"" >> $WRAP_DIR/${prog} - chmod +x ${WRAP_DIR}/${prog} - done - - # Install signapple - cd signapple - python3 -m pip install -U pip setuptools - python3 -m pip install . - export PATH="$HOME/.local/bin":$PATH - cd .. - - UNSIGNED_TARBALL=bitcoin-osx-unsigned.tar.gz - UNSIGNED_APP=dist/Bitcoin-Qt.app - SIGNED=bitcoin-osx-signed.dmg - - tar -xf ${UNSIGNED_TARBALL} - OSX_VOLNAME="$(cat osx_volname)" - ./detached-sig-apply.sh ${UNSIGNED_APP} signature/osx/dist - ${WRAP_DIR}/xorrisofs -D -l -V "${OSX_VOLNAME}" -no-pad -r -dir-mode 0755 -o uncompressed.dmg signed-app - ${WRAP_DIR}/dmg dmg uncompressed.dmg ${OUTDIR}/${SIGNED} diff --git a/contrib/gitian-descriptors/gitian-osx.yml b/contrib/gitian-descriptors/gitian-osx.yml deleted file mode 100644 index 556edef264..0000000000 --- a/contrib/gitian-descriptors/gitian-osx.yml +++ /dev/null @@ -1,155 +0,0 @@ ---- -name: "elements-osx-22" -enable_cache: true -distro: "ubuntu" -suites: -- "focal" -architectures: -- "amd64" -packages: -- "ca-certificates" -- "curl" -- "g++" -- "git" -- "icnsutils" -- "pkg-config" -- "autoconf" -- "librsvg2-2" -- "librsvg2-bin" -- "libtiff-tools" -- "libtool" -- "automake" -- "faketime" -- "bsdmainutils" -- "cmake" -- "imagemagick" -- "libz-dev" -- "python3" -- "python3-pip" -- "python3-setuptools" -- "fonts-tuffy" -- "xorriso" -- "libtinfo5" -remotes: -- "url": "https://github.com/ElementsProject/elements.git" - "dir": "elements" -files: -- "Xcode-12.1-12A7403-extracted-SDK-with-libcxx-headers.tar.gz" -script: | - set -e -o pipefail - - WRAP_DIR=$HOME/wrapped - HOSTS="x86_64-apple-darwin18" - CONFIGFLAGS="--enable-reduce-exports --disable-bench --disable-gui-tests --disable-fuzz-binary XORRISOFS=${WRAP_DIR}/xorrisofs DMG=${WRAP_DIR}/dmg" - FAKETIME_HOST_PROGS="" - FAKETIME_PROGS="ar ranlib date dmg xorrisofs" - - export TZ="UTC" - export BUILD_DIR="$PWD" - mkdir -p ${WRAP_DIR} - if test -n "$GBUILD_CACHE_ENABLED"; then - export SOURCES_PATH=${GBUILD_COMMON_CACHE} - export BASE_CACHE=${GBUILD_PACKAGE_CACHE} - mkdir -p ${BASE_CACHE} ${SOURCES_PATH} - fi - - export ZERO_AR_DATE=1 - - # Use $LIB in LD_PRELOAD to avoid hardcoding the dir (See `man ld.so`) - function create_global_faketime_wrappers { - for prog in ${FAKETIME_PROGS}; do - echo '#!/usr/bin/env bash' > ${WRAP_DIR}/${prog} - echo "REAL=\`which -a ${prog} | grep -v ${WRAP_DIR}/${prog} | head -1\`" >> ${WRAP_DIR}/${prog} - echo "export LD_PRELOAD='/usr/\$LIB/faketime/libfaketime.so.1'" >> ${WRAP_DIR}/${prog} - echo "export FAKETIME=\"$1\"" >> ${WRAP_DIR}/${prog} - echo "exec \"\$REAL\" \"\$@\"" >> $WRAP_DIR/${prog} - chmod +x ${WRAP_DIR}/${prog} - done - } - - function create_per-host_faketime_wrappers { - for i in $HOSTS; do - for prog in ${FAKETIME_HOST_PROGS}; do - echo '#!/usr/bin/env bash' > ${WRAP_DIR}/${i}-${prog} - echo "REAL=\`which -a ${i}-${prog} | grep -v ${WRAP_DIR}/${i}-${prog} | head -1\`" >> ${WRAP_DIR}/${i}-${prog} - echo "export LD_PRELOAD='/usr/\$LIB/faketime/libfaketime.so.1'" >> ${WRAP_DIR}/${i}-${prog} - echo "export FAKETIME=\"$1\"" >> ${WRAP_DIR}/${i}-${prog} - echo "exec \"\$REAL\" \"\$@\"" >> $WRAP_DIR/${i}-${prog} - chmod +x ${WRAP_DIR}/${i}-${prog} - done - done - } - - pip3 install lief==0.11.5 - - # Faketime for depends so intermediate results are comparable - export PATH_orig=${PATH} - create_global_faketime_wrappers "2000-01-01 12:00:00" - create_per-host_faketime_wrappers "2000-01-01 12:00:00" - export PATH=${WRAP_DIR}:${PATH} - - cd elements - BASEPREFIX="${PWD}/depends" - - mkdir -p ${BASEPREFIX}/SDKs - tar -C ${BASEPREFIX}/SDKs -xf ${BUILD_DIR}/Xcode-12.1-12A7403-extracted-SDK-with-libcxx-headers.tar.gz - - # Build dependencies for each host - for i in $HOSTS; do - make ${MAKEOPTS} -C ${BASEPREFIX} HOST="${i}" - done - - # Faketime for binaries - export PATH=${PATH_orig} - create_global_faketime_wrappers "${REFERENCE_DATETIME}" - create_per-host_faketime_wrappers "${REFERENCE_DATETIME}" - export PATH=${WRAP_DIR}:${PATH} - - # Define DISTNAME variable. - # shellcheck source=contrib/gitian-descriptors/assign_DISTNAME - source contrib/gitian-descriptors/assign_DISTNAME - - GIT_ARCHIVE="${OUTDIR}/src/${DISTNAME}.tar.gz" - - # Create the source tarball - mkdir -p "$(dirname "$GIT_ARCHIVE")" - git archive --prefix="${DISTNAME}/" --output="$GIT_ARCHIVE" HEAD - - ORIGPATH="$PATH" - # Extract the git archive into a dir for each host and build - for i in ${HOSTS}; do - export PATH=${BASEPREFIX}/${i}/native/bin:${ORIGPATH} - mkdir -p distsrc-${i} - cd distsrc-${i} - INSTALLPATH="${PWD}/installed/${DISTNAME}" - mkdir -p ${INSTALLPATH} - tar --strip-components=1 -xf "${GIT_ARCHIVE}" - - ./autogen.sh - CONFIG_SITE=${BASEPREFIX}/${i}/share/config.site ./configure --prefix=/ --disable-ccache --disable-maintainer-mode --disable-dependency-tracking ${CONFIGFLAGS} - make ${MAKEOPTS} - make ${MAKEOPTS} -C src check-security - make ${MAKEOPTS} -C src check-symbols - make install-strip DESTDIR=${INSTALLPATH} - - make osx_volname - make deploydir - mkdir -p unsigned-app-${i} - cp osx_volname unsigned-app-${i}/ - cp contrib/macdeploy/detached-sig-apply.sh unsigned-app-${i} - cp contrib/macdeploy/detached-sig-create.sh unsigned-app-${i} - cp ${BASEPREFIX}/${i}/native/bin/dmg unsigned-app-${i} - mv dist unsigned-app-${i} - pushd unsigned-app-${i} - find . | sort | tar --mtime="$REFERENCE_DATETIME" --no-recursion --mode='u+rw,go+r-w,a+X' --owner=0 --group=0 -c -T - | gzip -9n > ${OUTDIR}/${DISTNAME}-osx-unsigned.tar.gz - popd - - make deploy OSX_DMG="${OUTDIR}/${DISTNAME}-osx-unsigned.dmg" - - cd installed - - find ${DISTNAME} | sort | tar --mtime="$REFERENCE_DATETIME" --no-recursion --mode='u+rw,go+r-w,a+X' --owner=0 --group=0 -c -T - | gzip -9n > ${OUTDIR}/${DISTNAME}-${i}.tar.gz - cd ../../ - done - - mv ${OUTDIR}/${DISTNAME}-x86_64-*.tar.gz ${OUTDIR}/${DISTNAME}-osx64.tar.gz diff --git a/contrib/gitian-descriptors/gitian-win-signer.yml b/contrib/gitian-descriptors/gitian-win-signer.yml deleted file mode 100644 index c13c24c3cc..0000000000 --- a/contrib/gitian-descriptors/gitian-win-signer.yml +++ /dev/null @@ -1,42 +0,0 @@ ---- -name: "bitcoin-win-signer" -distro: "ubuntu" -suites: -- "focal" -architectures: -- "amd64" -packages: -- "libssl-dev" -- "autoconf" -- "automake" -- "libtool" -- "pkg-config" -remotes: -- "url": "https://github.com/bitcoin-core/bitcoin-detached-sigs.git" - "dir": "signature" -files: -- "osslsigncode-2.0.tar.gz" -- "bitcoin-win-unsigned.tar.gz" -script: | - set -e -o pipefail - - BUILD_DIR="$PWD" - SIGDIR=${BUILD_DIR}/signature/win - UNSIGNED_DIR=${BUILD_DIR}/unsigned - - echo "5a60e0a4b3e0b4d655317b2f12a810211c50242138322b16e7e01c6fbb89d92f osslsigncode-2.0.tar.gz" | sha256sum -c - - mkdir -p ${UNSIGNED_DIR} - tar -C ${UNSIGNED_DIR} -xf bitcoin-win-unsigned.tar.gz - - tar xf osslsigncode-2.0.tar.gz - cd osslsigncode-2.0 - - ./autogen.sh - ./configure --without-gsf --without-curl --disable-dependency-tracking - make - find ${UNSIGNED_DIR} -name "*-unsigned.exe" | while read i; do - INFILE="$(basename "${i}")" - OUTFILE="${INFILE/-unsigned}" - ./osslsigncode attach-signature -in "${i}" -out "${OUTDIR}/${OUTFILE}" -sigin "${SIGDIR}/${INFILE}.pem" - done diff --git a/contrib/gitian-descriptors/gitian-win.yml b/contrib/gitian-descriptors/gitian-win.yml deleted file mode 100644 index b4b940715a..0000000000 --- a/contrib/gitian-descriptors/gitian-win.yml +++ /dev/null @@ -1,156 +0,0 @@ ---- -name: "elements-win-22" -enable_cache: true -distro: "ubuntu" -suites: -- "focal" -architectures: -- "amd64" -packages: -- "curl" -- "g++" -- "git" -- "icnsutils" -- "imagemagick" -- "librsvg2-2" -- "librsvg2-bin" -- "pkg-config" -- "autoconf" -- "libtool" -- "automake" -- "faketime" -- "bsdmainutils" -- "mingw-w64" -- "g++-mingw-w64" -- "nsis" -- "zip" -- "ca-certificates" -- "python3" -- "python3-pip" -remotes: -- "url": "https://github.com/ElementsProject/elements.git" - "dir": "elements" -files: [] -script: | - set -e -o pipefail - - WRAP_DIR=$HOME/wrapped - HOSTS="x86_64-w64-mingw32" - CONFIGFLAGS="--enable-reduce-exports --disable-bench --disable-gui-tests --disable-fuzz-binary" - FAKETIME_HOST_PROGS="ar ranlib nm windres strip objcopy" - FAKETIME_PROGS="date makensis zip" - HOST_CFLAGS="-O2 -g -fno-ident" - HOST_CXXFLAGS="-O2 -g -fno-ident" - - export TZ="UTC" - export BUILD_DIR="$PWD" - mkdir -p ${WRAP_DIR} - if test -n "$GBUILD_CACHE_ENABLED"; then - export SOURCES_PATH=${GBUILD_COMMON_CACHE} - export BASE_CACHE=${GBUILD_PACKAGE_CACHE} - mkdir -p ${BASE_CACHE} ${SOURCES_PATH} - fi - - # Use $LIB in LD_PRELOAD to avoid hardcoding the dir (See `man ld.so`) - function create_global_faketime_wrappers { - for prog in ${FAKETIME_PROGS}; do - echo '#!/usr/bin/env bash' > ${WRAP_DIR}/${prog} - echo "REAL=\`which -a ${prog} | grep -v ${WRAP_DIR}/${prog} | head -1\`" >> ${WRAP_DIR}/${prog} - echo "export LD_PRELOAD='/usr/\$LIB/faketime/libfaketime.so.1'" >> ${WRAP_DIR}/${prog} - echo "export FAKETIME=\"$1\"" >> ${WRAP_DIR}/${prog} - echo "exec \"\$REAL\" \"\$@\"" >> $WRAP_DIR/${prog} - chmod +x ${WRAP_DIR}/${prog} - done - } - - function create_per-host_faketime_wrappers { - for i in $HOSTS; do - for prog in ${FAKETIME_HOST_PROGS}; do - echo '#!/usr/bin/env bash' > ${WRAP_DIR}/${i}-${prog} - echo "REAL=\`which -a ${i}-${prog} | grep -v ${WRAP_DIR}/${i}-${prog} | head -1\`" >> ${WRAP_DIR}/${i}-${prog} - echo "export LD_PRELOAD='/usr/\$LIB/faketime/libfaketime.so.1'" >> ${WRAP_DIR}/${i}-${prog} - echo "export FAKETIME=\"$1\"" >> ${WRAP_DIR}/${i}-${prog} - echo "exec \"\$REAL\" \"\$@\"" >> $WRAP_DIR/${i}-${prog} - chmod +x ${WRAP_DIR}/${i}-${prog} - done - done - } - - function create_per-host_compiler_wrapper { - # -posix variant is required for c++11 threading. - for i in $HOSTS; do - for prog in gcc g++; do - echo '#!/usr/bin/env bash' > ${WRAP_DIR}/${i}-${prog} - echo "REAL=\`which -a ${i}-${prog}-posix | grep -v ${WRAP_DIR}/${i}-${prog} | head -1\`" >> ${WRAP_DIR}/${i}-${prog} - echo "export LD_PRELOAD='/usr/\$LIB/faketime/libfaketime.so.1'" >> ${WRAP_DIR}/${i}-${prog} - echo "export FAKETIME=\"$1\"" >> ${WRAP_DIR}/${i}-${prog} - echo "exec \"\$REAL\" \"\$@\"" >> $WRAP_DIR/${i}-${prog} - chmod +x ${WRAP_DIR}/${i}-${prog} - done - done - } - - pip3 install lief==0.11.5 - - # Faketime for depends so intermediate results are comparable - export PATH_orig=${PATH} - create_global_faketime_wrappers "2000-01-01 12:00:00" - create_per-host_faketime_wrappers "2000-01-01 12:00:00" - create_per-host_compiler_wrapper "2000-01-01 12:00:00" - export PATH=${WRAP_DIR}:${PATH} - - cd elements - BASEPREFIX="${PWD}/depends" - # Build dependencies for each host - for i in $HOSTS; do - make ${MAKEOPTS} -C ${BASEPREFIX} HOST="${i}" - done - - # Faketime for binaries - export PATH=${PATH_orig} - create_global_faketime_wrappers "${REFERENCE_DATETIME}" - create_per-host_faketime_wrappers "${REFERENCE_DATETIME}" - create_per-host_compiler_wrapper "${REFERENCE_DATETIME}" - export PATH=${WRAP_DIR}:${PATH} - - # Define DISTNAME variable. - # shellcheck source=contrib/gitian-descriptors/assign_DISTNAME - source contrib/gitian-descriptors/assign_DISTNAME - - GIT_ARCHIVE="${OUTDIR}/src/${DISTNAME}.tar.gz" - - # Create the source tarball - mkdir -p "$(dirname "$GIT_ARCHIVE")" - git archive --prefix="${DISTNAME}/" --output="$GIT_ARCHIVE" HEAD - - ORIGPATH="$PATH" - # Extract the git archive into a dir for each host and build - for i in ${HOSTS}; do - export PATH=${BASEPREFIX}/${i}/native/bin:${ORIGPATH} - mkdir -p distsrc-${i} - cd distsrc-${i} - INSTALLPATH="${PWD}/installed/${DISTNAME}" - mkdir -p ${INSTALLPATH} - tar --strip-components=1 -xf "${GIT_ARCHIVE}" - - ./autogen.sh - CONFIG_SITE=${BASEPREFIX}/${i}/share/config.site ./configure --prefix=/ --disable-ccache --disable-maintainer-mode --disable-dependency-tracking ${CONFIGFLAGS} CFLAGS="${HOST_CFLAGS}" CXXFLAGS="${HOST_CXXFLAGS}" - make ${MAKEOPTS} - make ${MAKEOPTS} -C src check-security - make ${MAKEOPTS} -C src check-symbols - make deploy BITCOIN_WIN_INSTALLER="${OUTDIR}/${DISTNAME}-win64-setup-unsigned.exe" - make install DESTDIR=${INSTALLPATH} - cd installed - find ${DISTNAME}/bin -type f -executable -print0 | xargs -0 -n1 -I{} ../contrib/devtools/split-debug.sh {} {} {}.dbg - cp ../doc/README_windows.txt ${DISTNAME}/readme.txt - find ${DISTNAME} -not -name "*.dbg" -type f | sort | zip -X@ ${OUTDIR}/${DISTNAME}-${i//x86_64-w64-mingw32/win64}.zip - find ${DISTNAME} -name "*.dbg" -type f | sort | zip -X@ ${OUTDIR}/${DISTNAME}-${i//x86_64-w64-mingw32/win64}-debug.zip - cd ../../ - rm -rf distsrc-${i} - done - - cp -rf contrib/windeploy $BUILD_DIR - cd $BUILD_DIR/windeploy - mkdir unsigned - cp ${OUTDIR}/${DISTNAME}-win64-setup-unsigned.exe unsigned/ - find . | sort | tar --mtime="$REFERENCE_DATETIME" --no-recursion --mode='u+rw,go+r-w,a+X' --owner=0 --group=0 -c -T - | gzip -9n > ${OUTDIR}/${DISTNAME}-win-unsigned.tar.gz diff --git a/contrib/guix/INSTALL.md b/contrib/guix/INSTALL.md index 75f2e1272b..68aae18731 100644 --- a/contrib/guix/INSTALL.md +++ b/contrib/guix/INSTALL.md @@ -79,8 +79,8 @@ Guix v1.2.0 is available as a distribution package starting in [Debian 21.04](https://packages.ubuntu.com/hirsute/guix). Note that if you intend on using Guix without using any substitutes (more -details [here][security-model]), v1.2.0 has a known problems when building -GnuTLS from source. Solutions and workarounds are documented +details [here][security-model]), v1.2.0 has a known problem when building GnuTLS +from source. Solutions and workarounds are documented [here](#gnutls-test-suite-fail-status-request-revoked). @@ -270,23 +270,11 @@ Note that these environment variables are used to check for packages during `./configure`, so they should be set as soon as possible should you want to use a prefix other than `/usr`. - - - - - - - - - - - - #### Building and installing source-built packages -***IMPORTANT**: A few dependencies have non-obvious quirks/erratas which are documented in the -sub-sections immediately below. Please read these sections before proceeding to -build and install these packages.* +***IMPORTANT**: A few dependencies have non-obvious quirks/errata which are +documented in the sub-sections immediately below. Please read these sections +before proceeding to build and install these packages.* Although you should always refer to the README or INSTALL files for the most accurate information, most of these dependencies use autoconf-style build @@ -370,7 +358,7 @@ This is especially notable because Ubuntu Focal packages `libgit2 v0.28.4`, and Should you be in this situation, you need to build both `libgit2 v1.1.x` and `guile-git` from source. -Source: http://logs.guix.gnu.org/guix/2020-11-12.log#232527 +Source: https://logs.guix.gnu.org/guix/2020-11-12.log#232527 ##### `{scheme,guile}-bytestructures` v1.0.8 and v1.0.9 are broken for Guile v2.2 diff --git a/contrib/guix/README.md b/contrib/guix/README.md index 4680368a6f..90289f9d40 100644 --- a/contrib/guix/README.md +++ b/contrib/guix/README.md @@ -11,7 +11,7 @@ We achieve bootstrappability by using Guix as a functional package manager. # Requirements -Conservatively, a x86_64 machine with: +Conservatively, you will need an x86_64 machine with: - 16GB of free disk space on the partition that /gnu/store will reside in - 8GB of free disk space **per platform triple** you're planning on building @@ -75,7 +75,7 @@ crucial differences: 1. Since only Windows and macOS build outputs require codesigning, the `HOSTS` environment variable will have a sane default value of `x86_64-w64-mingw32 - x86_64-apple-darwin18` instead of all the platforms. + x86_64-apple-darwin` instead of all the platforms. 2. The `guix-codesign` command ***requires*** a `DETACHED_SIGS_REPO` flag. * _**DETACHED_SIGS_REPO**_ @@ -87,7 +87,7 @@ crucial differences: An invocation with all default options would look like: ``` -env DETACHED_SIGS_REPO= ./contrib/guix-codesign +env DETACHED_SIGS_REPO= ./contrib/guix/guix-codesign ``` ## Cleaning intermediate work directories @@ -159,7 +159,7 @@ which case you can override the default list by setting the space-separated `HOSTS` environment variable: ```sh -env HOSTS='x86_64-w64-mingw32 x86_64-apple-darwin18' ./contrib/guix/guix-build +env HOSTS='x86_64-w64-mingw32 x86_64-apple-darwin' ./contrib/guix/guix-build ``` See the [recognized environment variables][env-vars-list] section for more @@ -224,7 +224,7 @@ details. _(defaults to "x86\_64-linux-gnu arm-linux-gnueabihf aarch64-linux-gnu riscv64-linux-gnu powerpc64-linux-gnu powerpc64le-linux-gnu - x86\_64-w64-mingw32 x86\_64-apple-darwin18")_ + x86\_64-w64-mingw32 x86\_64-apple-darwin arm64-apple-darwin")_ * _**SOURCES_PATH**_ @@ -249,7 +249,7 @@ details. Set the path where _extracted_ SDKs can be found. This is passed through to the depends tree. Note that this is should be set to the _parent_ directory of the actual SDK (e.g. `SDK_PATH=$HOME/Downloads/macOS-SDKs` instead of - `$HOME/Downloads/macOS-SDKs/Xcode-12.1-12A7403-extracted-SDK-with-libcxx-headers`). + `$HOME/Downloads/macOS-SDKs/Xcode-12.2-12B45b-extracted-SDK-with-libcxx-headers`). The path that this environment variable points to **must be a directory**, and **NOT a symlink to a directory**. @@ -437,9 +437,8 @@ In the extraordinarily rare case where you messed up your Guix installation in an irreversible way, you may want to completely purge Guix from your system and start over. -1. Uninstall Guix itself according to the way you installed it. (e.g. `sudo apt - purge guix` for Ubuntu packaging, `sudo make uninstall` for - built-from-source). +1. Uninstall Guix itself according to the way you installed it (e.g. `sudo apt + purge guix` for Ubuntu packaging, `sudo make uninstall` for a build from source). 2. Remove all build users and groups You may check for relevant users and groups using: @@ -468,7 +467,7 @@ start over. - `/root/.cache/guix/` - `/root/.guix-profile/` -[b17e]: http://bootstrappable.org/ +[b17e]: https://bootstrappable.org/ [r12e/source-date-epoch]: https://reproducible-builds.org/docs/source-date-epoch/ [guix/install.sh]: https://git.savannah.gnu.org/cgit/guix.git/plain/etc/guix-install.sh diff --git a/contrib/guix/guix-attest b/contrib/guix/guix-attest index 51d589c1de..6e12cbead7 100755 --- a/contrib/guix/guix-attest +++ b/contrib/guix/guix-attest @@ -159,23 +159,21 @@ Hint: You may wish to remove the existing attestations and their signatures by EOF } -# Given a document with unix line endings (just ) in stdin, make all lines -# end in and make sure there's no trailing at the end of the file. -# -# This is necessary as cleartext signatures are calculated on text after their -# line endings are canonicalized. -# -# For more information: -# 1. https://security.stackexchange.com/a/104261 -# 2. https://datatracker.ietf.org/doc/html/rfc4880#section-7.1 -# -rfc4880_normalize_document() { - sed 's/$/\r/' | head -c -2 -} - echo "Attesting to build outputs for version: '${VERSION}'" echo "" +# Given a SHA256SUMS file as stdin that has lines like: +# 0ba536819b221a91d3d42e978be016aac918f40984754d74058aa0c921cd3ea6 a/b/d/c/d/s/bitcoin-22.0rc2-riscv64-linux-gnu.tar.gz +# ... +# +# Replace each line's file name with its basename: +# 0ba536819b221a91d3d42e978be016aac918f40984754d74058aa0c921cd3ea6 bitcoin-22.0rc2-riscv64-linux-gnu.tar.gz +# ... +# +basenameify_SHA256SUMS() { + sed -E 's@(^[[:xdigit:]]{64}[[:space:]]+).+/([^/]+$)@\1\2@' +} + outsigdir="$GUIX_SIGS_REPO/$VERSION/$signer_name" mkdir -p "$outsigdir" ( @@ -188,7 +186,7 @@ mkdir -p "$outsigdir" cat "${noncodesigned_fragments[@]}" \ | sort -u \ | sort -k2 \ - | rfc4880_normalize_document \ + | basenameify_SHA256SUMS \ > "$temp_noncodesigned" if [ -e noncodesigned.SHA256SUMS ]; then # The SHA256SUMS already exists, make sure it's exactly what we @@ -207,8 +205,8 @@ mkdir -p "$outsigdir" exit 1 fi - temp_codesigned="$(mktemp)" - trap 'rm -rf -- "$temp_codesigned"' EXIT + temp_all="$(mktemp)" + trap 'rm -rf -- "$temp_all"' EXIT if (( ${#codesigned_fragments[@]} )); then # Note: all.SHA256SUMS attests to all of $sha256sum_fragments, but is @@ -216,20 +214,19 @@ mkdir -p "$outsigdir" cat "${sha256sum_fragments[@]}" \ | sort -u \ | sort -k2 \ - | sed 's/$/\r/' \ - | rfc4880_normalize_document \ - > "$temp_codesigned" - if [ -e codesigned.SHA256SUMS ]; then + | basenameify_SHA256SUMS \ + > "$temp_all" + if [ -e all.SHA256SUMS ]; then # The SHA256SUMS already exists, make sure it's exactly what we # expect, error out if not - if diff -u all.SHA256SUMS "$temp_codesigned"; then + if diff -u all.SHA256SUMS "$temp_all"; then echo "An all.SHA256SUMS file already exists for '${VERSION}' and is up-to-date." else shasum_already_exists all.SHA256SUMS exit 1 fi else - mv "$temp_codesigned" codesigned.SHA256SUMS + mv "$temp_all" all.SHA256SUMS fi else # It is fine to have the codesigned outputs be missing (perhaps the diff --git a/contrib/guix/guix-build b/contrib/guix/guix-build index f6da8435e9..b38f91f3c7 100755 --- a/contrib/guix/guix-build +++ b/contrib/guix/guix-build @@ -76,7 +76,7 @@ mkdir -p "$VERSION_BASE" # Default to building for all supported HOSTs (overridable by environment) export HOSTS="${HOSTS:-x86_64-linux-gnu arm-linux-gnueabihf aarch64-linux-gnu riscv64-linux-gnu powerpc64-linux-gnu powerpc64le-linux-gnu x86_64-w64-mingw32 - x86_64-apple-darwin18}" + x86_64-apple-darwin arm64-apple-darwin}" # Usage: distsrc_for_host HOST # @@ -190,9 +190,9 @@ fi # Services database must have basic entries ################ -if ! getent services http https ftp; then +if ! getent services http https ftp > /dev/null 2>&1; then cat << EOF -ERR: Your system's C library can not find service database entries for at least +ERR: Your system's C library cannot find service database entries for at least one of the following services: http, https, ftp. Hint: Most likely, /etc/services does not exist yet (common for docker images @@ -232,14 +232,14 @@ host_to_commonname() { } # Determine the reference time used for determinism (overridable by environment) -SOURCE_DATE_EPOCH="${SOURCE_DATE_EPOCH:-$(git log --format=%at -1)}" +SOURCE_DATE_EPOCH="${SOURCE_DATE_EPOCH:-$(git -c log.showSignature=false log --format=%at -1)}" # Execute "$@" in a pinned, possibly older version of Guix, for reproducibility # across time. time-machine() { # shellcheck disable=SC2086 guix time-machine --url=https://git.savannah.gnu.org/git/guix.git \ - --commit=aa34d4d28dfe25ba47d5800d05000fb7221788c0 \ + --commit=ae03f401381e956c4c41b4cf495cbde964fa43d0 \ --cores="$JOBS" \ --keep-failed \ --fallback \ diff --git a/contrib/guix/guix-clean b/contrib/guix/guix-clean index 9fa17191e8..9af0a793cf 100755 --- a/contrib/guix/guix-clean +++ b/contrib/guix/guix-clean @@ -34,7 +34,7 @@ check_tools cat mkdir make git guix # under_dir() { local path_residue - path_residue="${2##${1}}" + path_residue="${2##"${1}"}" if [ -z "$path_residue" ] || [ "$path_residue" = "$2" ]; then return 1 else diff --git a/contrib/guix/guix-codesign b/contrib/guix/guix-codesign index 11610a92e1..2dd30bfa64 100755 --- a/contrib/guix/guix-codesign +++ b/contrib/guix/guix-codesign @@ -91,7 +91,7 @@ fi ################ # Default to building for all supported HOSTs (overridable by environment) -export HOSTS="${HOSTS:-x86_64-w64-mingw32 x86_64-apple-darwin18}" +export HOSTS="${HOSTS:-x86_64-w64-mingw32 x86_64-apple-darwin arm64-apple-darwin}" # Usage: distsrc_for_host HOST # @@ -220,7 +220,7 @@ fi JOBS="${JOBS:-$(nproc)}" # Determine the reference time used for determinism (overridable by environment) -SOURCE_DATE_EPOCH="${SOURCE_DATE_EPOCH:-$(git log --format=%at -1)}" +SOURCE_DATE_EPOCH="${SOURCE_DATE_EPOCH:-$(git -c log.showSignature=false log --format=%at -1)}" # Execute "$@" in a pinned, possibly older version of Guix, for reproducibility # across time. diff --git a/contrib/guix/guix-verify b/contrib/guix/guix-verify index a6e2c4065e..02ae022741 100755 --- a/contrib/guix/guix-verify +++ b/contrib/guix/guix-verify @@ -28,7 +28,11 @@ cmd_usage() { cat < ./contrib/guix/guix-verify + env GUIX_SIGS_REPO= [ SIGNER= ] ./contrib/guix/guix-verify + +Example overriding signer's manifest to use as base + + env GUIX_SIGS_REPO=/home/dongcarl/guix.sigs SIGNER=achow101 ./contrib/guix/guix-verify EOF } @@ -73,11 +77,13 @@ verify() { echo "" echo "Hint: Either the signature is invalid or the public key is missing" echo "" + failure=1 elif ! diff --report-identical "$compare_manifest" "$current_manifest" 1>&2; then echo "ERR: The SHA256SUMS attestation in these two directories differ:" echo " '${compare_manifest}'" echo " '${current_manifest}'" echo "" + failure=1 else echo "Verified: '${current_manifest}'" echo "" @@ -92,6 +98,17 @@ echo "--------------------" echo "" if (( ${#all_noncodesigned[@]} )); then compare_noncodesigned="${all_noncodesigned[0]}" + if [[ -n "$SIGNER" ]]; then + signer_noncodesigned="$OUTSIGDIR_BASE/$SIGNER/noncodesigned.SHA256SUMS" + if [[ -f "$signer_noncodesigned" ]]; then + echo "Using $SIGNER's manifest as the base to compare against" + compare_noncodesigned="$signer_noncodesigned" + else + echo "Unable to find $SIGNER's manifest, using the first one found" + fi + else + echo "No SIGNER provided, using the first manifest found" + fi for current_manifest in "${all_noncodesigned[@]}"; do verify "$compare_noncodesigned" "$current_manifest" @@ -112,6 +129,17 @@ echo "--------------------" echo "" if (( ${#all_all[@]} )); then compare_all="${all_all[0]}" + if [[ -n "$SIGNER" ]]; then + signer_all="$OUTSIGDIR_BASE/$SIGNER/all.SHA256SUMS" + if [[ -f "$signer_all" ]]; then + echo "Using $SIGNER's manifest as the base to compare against" + compare_all="$signer_all" + else + echo "Unable to find $SIGNER's manifest, using the first one found" + fi + else + echo "No SIGNER provided, using the first manifest found" + fi for current_manifest in "${all_all[@]}"; do verify "$compare_all" "$current_manifest" @@ -140,3 +168,7 @@ if (( ${#all_noncodesigned[@]} + ${#all_all[@]} == 0 )); then echo "" exit 1 fi + +if [ -n "$failure" ]; then + exit 1 +fi diff --git a/contrib/guix/libexec/build.sh b/contrib/guix/libexec/build.sh index bc3391e089..e06a469338 100755 --- a/contrib/guix/libexec/build.sh +++ b/contrib/guix/libexec/build.sh @@ -1,4 +1,7 @@ #!/usr/bin/env bash +# Copyright (c) 2019-2021 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. export LC_ALL=C set -e -o pipefail export TZ=UTC @@ -106,7 +109,7 @@ case "$HOST" in # 2. kernel-header-related search paths (not applicable to mingw-w64 hosts) export CROSS_C_INCLUDE_PATH="${CROSS_GCC_LIB}/include:${CROSS_GCC_LIB}/include-fixed:${CROSS_GLIBC}/include" export CROSS_CPLUS_INCLUDE_PATH="${CROSS_GCC}/include/c++:${CROSS_GCC}/include/c++/${HOST}:${CROSS_GCC}/include/c++/backward:${CROSS_C_INCLUDE_PATH}" - export CROSS_LIBRARY_PATH="${CROSS_GCC_LIB_STORE}/lib:${CROSS_GCC}/${HOST}/lib:${CROSS_GCC_LIB}:${CROSS_GLIBC}/lib" + export CROSS_LIBRARY_PATH="${CROSS_GCC_LIB_STORE}/lib:${CROSS_GCC_LIB}:${CROSS_GLIBC}/lib" ;; *darwin*) # The CROSS toolchain for darwin uses the SDK and ignores environment variables. @@ -123,7 +126,7 @@ case "$HOST" in export CROSS_C_INCLUDE_PATH="${CROSS_GCC_LIB}/include:${CROSS_GCC_LIB}/include-fixed:${CROSS_GLIBC}/include:${CROSS_KERNEL}/include" export CROSS_CPLUS_INCLUDE_PATH="${CROSS_GCC}/include/c++:${CROSS_GCC}/include/c++/${HOST}:${CROSS_GCC}/include/c++/backward:${CROSS_C_INCLUDE_PATH}" - export CROSS_LIBRARY_PATH="${CROSS_GCC_LIB_STORE}/lib:${CROSS_GCC}/${HOST}/lib:${CROSS_GCC_LIB}:${CROSS_GLIBC}/lib:${CROSS_GLIBC_STATIC}/lib" + export CROSS_LIBRARY_PATH="${CROSS_GCC_LIB_STORE}/lib:${CROSS_GCC_LIB}:${CROSS_GLIBC}/lib:${CROSS_GLIBC_STATIC}/lib" ;; *) exit 1 ;; @@ -147,7 +150,7 @@ case "$HOST" in # # After the native packages in depends are built, the ld wrapper should # no longer affect our build, as clang would instead reach for - # x86_64-apple-darwin18-ld from cctools + # x86_64-apple-darwin-ld from cctools ;; *) export GUIX_LD_WRAPPER_DISABLE_RPATH=yes ;; esac @@ -169,8 +172,8 @@ case "$HOST" in arm-linux-gnueabihf) echo /lib/ld-linux-armhf.so.3 ;; aarch64-linux-gnu) echo /lib/ld-linux-aarch64.so.1 ;; riscv64-linux-gnu) echo /lib/ld-linux-riscv64-lp64d.so.1 ;; - powerpc64-linux-gnu) echo /lib/ld64.so.1;; - powerpc64le-linux-gnu) echo /lib/ld64.so.2;; + powerpc64-linux-gnu) echo /lib64/ld64.so.1;; + powerpc64le-linux-gnu) echo /lib64/ld64.so.2;; *) exit 1 ;; esac ) @@ -227,7 +230,6 @@ GIT_ARCHIVE="${DIST_ARCHIVE_BASE}/${DISTNAME}.tar.gz" # Create the source tarball if not already there if [ ! -e "$GIT_ARCHIVE" ]; then mkdir -p "$(dirname "$GIT_ARCHIVE")" - touch "${DIST_ARCHIVE_BASE}"/SKIPATTEST.TAG git archive --prefix="${DISTNAME}/" --output="$GIT_ARCHIVE" HEAD fi @@ -239,9 +241,6 @@ mkdir -p "$OUTDIR" # CONFIGFLAGS CONFIGFLAGS="--enable-reduce-exports --disable-bench --disable-gui-tests --disable-fuzz-binary" -case "$HOST" in - *linux*) CONFIGFLAGS+=" --disable-threadlocal" ;; -esac # CFLAGS HOST_CFLAGS="-O2 -g" @@ -264,7 +263,7 @@ case "$HOST" in *mingw*) HOST_LDFLAGS="-Wl,--no-insert-timestamp" ;; esac -# Using --no-tls-get-addr-optimize retains compatibility with glibc 2.17, by +# Using --no-tls-get-addr-optimize retains compatibility with glibc 2.18, by # avoiding a PowerPC64 optimisation available in glibc 2.22 and later. # https://sourceware.org/binutils/docs-2.35/ld/PowerPC64-ELF64.html case "$HOST" in @@ -298,7 +297,7 @@ mkdir -p "$DISTSRC" ${HOST_CXXFLAGS:+CXXFLAGS="${HOST_CXXFLAGS}"} \ ${HOST_LDFLAGS:+LDFLAGS="${HOST_LDFLAGS}"} - sed -i.old 's/-lstdc++ //g' config.status libtool src/univalue/config.status src/univalue/libtool + sed -i.old 's/-lstdc++ //g' config.status libtool # Build Bitcoin Core make --jobs="$JOBS" ${V:+V=1} @@ -378,7 +377,7 @@ mkdir -p "$DISTSRC" { find "${DISTNAME}/bin" -type f -executable -print0 find "${DISTNAME}/lib" -type f -print0 - } | xargs -0 -n1 -P"$JOBS" -I{} "${DISTSRC}/contrib/devtools/split-debug.sh" {} {} {}.dbg + } | xargs -0 -P"$JOBS" -I{} "${DISTSRC}/contrib/devtools/split-debug.sh" {} {} {}.dbg ;; esac @@ -424,8 +423,8 @@ mkdir -p "$DISTSRC" find "${DISTNAME}" -print0 \ | sort --zero-terminated \ | tar --create --no-recursion --mode='u+rw,go+r-w,a+X' --null --files-from=- \ - | gzip -9n > "${OUTDIR}/${DISTNAME}-${HOST//x86_64-apple-darwin18/osx64}.tar.gz" \ - || ( rm -f "${OUTDIR}/${DISTNAME}-${HOST//x86_64-apple-darwin18/osx64}.tar.gz" && exit 1 ) + | gzip -9n > "${OUTDIR}/${DISTNAME}-${HOST//x86_64-apple-darwin/osx64}.tar.gz" \ + || ( rm -f "${OUTDIR}/${DISTNAME}-${HOST//x86_64-apple-darwin/osx64}.tar.gz" && exit 1 ) ;; esac ) # $DISTSRC/installed diff --git a/contrib/guix/libexec/codesign.sh b/contrib/guix/libexec/codesign.sh index f484ac5774..a8867ca351 100755 --- a/contrib/guix/libexec/codesign.sh +++ b/contrib/guix/libexec/codesign.sh @@ -1,4 +1,7 @@ #!/usr/bin/env bash +# Copyright (c) 2021 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. export LC_ALL=C set -e -o pipefail export TZ=UTC diff --git a/contrib/guix/libexec/prelude.bash b/contrib/guix/libexec/prelude.bash index dcc7885cd1..882d497a1d 100644 --- a/contrib/guix/libexec/prelude.bash +++ b/contrib/guix/libexec/prelude.bash @@ -2,10 +2,10 @@ export LC_ALL=C set -e -o pipefail -# shellcheck source=../../shell/realpath.bash +# shellcheck source=contrib/shell/realpath.bash source contrib/shell/realpath.bash -# shellcheck source=../../shell/git-utils.bash +# shellcheck source=contrib/shell/git-utils.bash source contrib/shell/git-utils.bash ################ @@ -49,7 +49,7 @@ fi # Set common variables ################ -VERSION="${VERSION:-$(git_head_version)}" +VERSION="${FORCE_VERSION:-$(git_head_version)}" DISTNAME="${DISTNAME:-elements-${VERSION}}" version_base_prefix="${PWD}/guix-build-" diff --git a/contrib/guix/manifest.scm b/contrib/guix/manifest.scm index 5805006053..d296eb9543 100644 --- a/contrib/guix/manifest.scm +++ b/contrib/guix/manifest.scm @@ -16,8 +16,6 @@ (gnu packages gawk) (gnu packages gcc) (gnu packages gnome) - (gnu packages image) - (gnu packages imagemagick) (gnu packages installers) (gnu packages linux) (gnu packages llvm) @@ -26,11 +24,11 @@ (gnu packages perl) (gnu packages pkg-config) (gnu packages python) + (gnu packages python-crypto) (gnu packages python-web) (gnu packages shells) (gnu packages tls) (gnu packages version-control) - (guix build-system font) (guix build-system gnu) (guix build-system python) (guix build-system trivial) @@ -80,10 +78,6 @@ http://www.linuxfromscratch.org/hlfs/view/development/chapter05/gcc-pass1.html" (("-rpath=") "-rpath-link=")) #t)))))))) -(define (make-binutils-with-mingw-w64-disable-flags xbinutils) - (package-with-extra-patches xbinutils - (search-our-patches "binutils-mingw-w64-disable-flags.patch"))) - (define (make-cross-toolchain target base-gcc-for-libc base-kernel-headers @@ -135,9 +129,7 @@ chain for " target " development.")) (home-page (package-home-page xgcc)) (license (package-license xgcc))))) -(define base-gcc - (package-with-extra-patches gcc-8 - (search-our-patches "gcc-8-sort-libtool-find-output.patch"))) +(define base-gcc gcc-10) ;; Building glibc with stack smashing protector first landed in glibc 2.25, use ;; this function to disable for older glibcs @@ -170,13 +162,17 @@ desirable for building Bitcoin Core release binaries." (define (make-gcc-with-pthreads gcc) (package-with-extra-configure-variable gcc "--enable-threads" "posix")) +;; Required to support std::filesystem for mingw-w64 target. +(define (make-gcc-without-newlib gcc) + (package-with-extra-configure-variable gcc "--with-newlib" "no")) + (define (make-mingw-pthreads-cross-toolchain target) "Create a cross-compilation toolchain package for TARGET" - (let* ((xbinutils (make-binutils-with-mingw-w64-disable-flags (cross-binutils target))) + (let* ((xbinutils (cross-binutils target)) (pthreads-xlibc mingw-w64-x86_64-winpthreads) (pthreads-xgcc (make-gcc-with-pthreads (cross-gcc target - #:xgcc (make-ssp-fixed-gcc base-gcc) + #:xgcc (make-gcc-without-newlib (make-ssp-fixed-gcc base-gcc)) #:xbinutils xbinutils #:libc pthreads-xlibc)))) ;; Define a meta-package that propagates the resulting XBINUTILS, XLIBC, and @@ -198,28 +194,9 @@ chain for " target " development.")) (home-page (package-home-page pthreads-xgcc)) (license (package-license pthreads-xgcc))))) -(define (make-nsis-with-sde-support base-nsis) +(define (make-nsis-for-gcc-10 base-nsis) (package-with-extra-patches base-nsis - (search-our-patches "nsis-SConstruct-sde-support.patch"))) - -(define-public font-tuffy - (package - (name "font-tuffy") - (version "20120614") - (source - (origin - (method url-fetch) - (uri (string-append "http://tulrich.com/fonts/tuffy-" version ".tar.gz")) - (file-name (string-append name "-" version ".tar.gz")) - (sha256 - (base32 - "02vf72bgrp30vrbfhxjw82s115z27dwfgnmmzfb0n9wfhxxfpyf6")))) - (build-system font-build-system) - (home-page "http://tulrich.com/fonts/") - (synopsis "The Tuffy Truetype Font Family") - (description - "Thatcher Ulrich's first outline font design. He started with the goal of producing a neutral, readable sans-serif text font. There are lots of \"expressive\" fonts out there, but he wanted to start with something very plain and clean, something he might want to actually use. ") - (license license:public-domain))) + (search-our-patches "nsis-gcc-10-memmove.patch"))) (define-public lief (package @@ -276,34 +253,6 @@ signing and timestamping. But osslsigncode is based on OpenSSL and cURL, and thus should be able to compile on most platforms where these exist.") (license license:gpl3+))) ; license is with openssl exception -(define-public python-asn1crypto - (package - (name "python-asn1crypto") - (version "1.4.0") - (source - (origin - (method git-fetch) - (uri (git-reference - (url "https://github.com/wbond/asn1crypto") - (commit version))) - (file-name (git-file-name name version)) - (sha256 - (base32 - "19abibn6jw20mzi1ln4n9jjvpdka8ygm4m439hplyrdfqbvgm01r")))) - (build-system python-build-system) - (arguments - '(#:phases - (modify-phases %standard-phases - (replace 'check - (lambda _ - (invoke "python" "run.py" "tests")))))) - (home-page "https://github.com/wbond/asn1crypto") - (synopsis "ASN.1 parser and serializer in Python") - (description "asn1crypto is an ASN.1 parser and serializer with definitions -for private keys, public keys, certificates, CRL, OCSP, CMS, PKCS#3, PKCS#7, -PKCS#8, PKCS#12, PKCS#5, X.509 and TSP.") - (license license:expat))) - (define-public python-elfesteem (let ((commit "87bbd79ab7e361004c98cc8601d4e5f029fd8bd5")) (package @@ -318,7 +267,8 @@ PKCS#8, PKCS#12, PKCS#5, X.509 and TSP.") (file-name (git-file-name name commit)) (sha256 (base32 - "1nyvjisvyxyxnd0023xjf5846xd03lwawp5pfzr8vrky7wwm5maz")))) + "1nyvjisvyxyxnd0023xjf5846xd03lwawp5pfzr8vrky7wwm5maz")) + (patches (search-our-patches "elfsteem-value-error-python-39.patch")))) (build-system python-build-system) ;; There are no tests, but attempting to run python setup.py test leads to ;; PYTHONPATH problems, just disable the test @@ -390,6 +340,8 @@ PKCS#8, PKCS#12, PKCS#5, X.509 and TSP.") (define-public python-oscryptotests (package (inherit python-oscrypto) (name "python-oscryptotests") + (propagated-inputs + `(("python-oscrypto" ,python-oscrypto))) (arguments `(#:tests? #f #:phases @@ -449,6 +401,11 @@ PKCS#8, PKCS#12, PKCS#5, X.509 and TSP.") (string-append indent "@unittest.skip(\"Disabled by Guix\")\n" line))) + (substitute* "tests/test_validate.py" + (("^(.*)def test_revocation_mode_soft" line indent) + (string-append indent + "@unittest.skip(\"Disabled by Guix\")\n" + line))) #t)) (replace 'check (lambda _ @@ -461,16 +418,6 @@ certificates or paths. Supports various options, including: validation at a specific moment in time, whitelisting and revocation checks.") (license license:expat)))) -(define-public python-requests-2.25.1 - (package (inherit python-requests) - (version "2.25.1") - (source (origin - (method url-fetch) - (uri (pypi-uri "requests" version)) - (sha256 - (base32 - "015qflyqsgsz09gnar69s6ga74ivq5kch69s4qxz3904m7a3v5r7")))))) - (define-public python-altgraph (package (name "python-altgraph") @@ -563,7 +510,7 @@ and endian independent.") ("python-oscrypto" ,python-oscrypto) ("python-certvalidator" ,python-certvalidator) ("python-elfesteem" ,python-elfesteem) - ("python-requests" ,python-requests-2.25.1) + ("python-requests" ,python-requests) ("python-macholib" ,python-macholib) ("libcrypto" ,openssl))) ;; There are no tests, but attempting to run python setup.py test leads to @@ -577,7 +524,7 @@ inspecting signatures in Mach-O binaries.") (define-public glibc-2.24 (package - (inherit glibc) + (inherit glibc-2.31) (version "2.24") (source (origin (method git-fetch) @@ -593,9 +540,21 @@ inspecting signatures in Mach-O binaries.") "glibc-2.24-elfm-loadaddr-dynamic-rewrite.patch" "glibc-2.24-no-build-time-cxx-header-run.patch")))))) -(define glibc-2.27/bitcoin-patched - (package-with-extra-patches glibc-2.27 - (search-our-patches "glibc-2.27-riscv64-Use-__has_include__-to-include-asm-syscalls.h.patch"))) +(define-public glibc-2.27/bitcoin-patched + (package + (inherit glibc-2.31) + (version "2.27") + (source (origin + (method git-fetch) + (uri (git-reference + (url "https://sourceware.org/git/glibc.git") + (commit "23158b08a0908f381459f273a984c6fd328363cb"))) + (file-name (git-file-name "glibc" "23158b08a0908f381459f273a984c6fd328363cb")) + (sha256 + (base32 + "1b2n1gxv9f4fd5yy68qjbnarhf8mf4vmlxk10i3328c1w5pmp0ca")) + (patches (search-our-patches "glibc-ldd-x86_64.patch" + "glibc-2.27-riscv64-Use-__has_include__-to-include-asm-syscalls.h.patch")))))) (packages->manifest (append @@ -624,7 +583,7 @@ inspecting signatures in Mach-O binaries.") ;; Build tools gnu-make libtool - autoconf + autoconf-2.71 automake pkg-config bison @@ -643,7 +602,7 @@ inspecting signatures in Mach-O binaries.") ;; Windows (list zip (make-mingw-pthreads-cross-toolchain "x86_64-w64-mingw32") - (make-nsis-with-sde-support nsis-x86_64) + (make-nsis-for-gcc-10 nsis-x86_64) osslsigncode)) ((string-contains target "-linux-") (list (cond ((string-contains target "riscv64-") @@ -653,5 +612,5 @@ inspecting signatures in Mach-O binaries.") (else (make-bitcoin-cross-toolchain target))))) ((string-contains target "darwin") - (list clang-toolchain-10 binutils imagemagick libtiff librsvg font-tuffy cmake xorriso python-signapple)) + (list clang-toolchain-10 binutils cmake xorriso python-signapple)) (else '()))))) diff --git a/contrib/guix/patches/binutils-mingw-w64-disable-flags.patch b/contrib/guix/patches/binutils-mingw-w64-disable-flags.patch deleted file mode 100644 index 8f88eb9dfd..0000000000 --- a/contrib/guix/patches/binutils-mingw-w64-disable-flags.patch +++ /dev/null @@ -1,171 +0,0 @@ -Description: Add disable opposites to the security-related flags -Author: Stephen Kitt - -This patch adds "no-" variants to disable the various security flags: -"no-dynamicbase", "no-nxcompat", "no-high-entropy-va", "disable-reloc-section". - ---- a/ld/emultempl/pe.em -+++ b/ld/emultempl/pe.em -@@ -259,9 +261,11 @@ - (OPTION_ENABLE_LONG_SECTION_NAMES + 1) - /* DLLCharacteristics flags. */ - #define OPTION_DYNAMIC_BASE (OPTION_DISABLE_LONG_SECTION_NAMES + 1) --#define OPTION_FORCE_INTEGRITY (OPTION_DYNAMIC_BASE + 1) -+#define OPTION_NO_DYNAMIC_BASE (OPTION_DYNAMIC_BASE + 1) -+#define OPTION_FORCE_INTEGRITY (OPTION_NO_DYNAMIC_BASE + 1) - #define OPTION_NX_COMPAT (OPTION_FORCE_INTEGRITY + 1) --#define OPTION_NO_ISOLATION (OPTION_NX_COMPAT + 1) -+#define OPTION_NO_NX_COMPAT (OPTION_NX_COMPAT + 1) -+#define OPTION_NO_ISOLATION (OPTION_NO_NX_COMPAT + 1) - #define OPTION_NO_SEH (OPTION_NO_ISOLATION + 1) - #define OPTION_NO_BIND (OPTION_NO_SEH + 1) - #define OPTION_WDM_DRIVER (OPTION_NO_BIND + 1) -@@ -271,6 +275,7 @@ - #define OPTION_NO_INSERT_TIMESTAMP (OPTION_INSERT_TIMESTAMP + 1) - #define OPTION_BUILD_ID (OPTION_NO_INSERT_TIMESTAMP + 1) - #define OPTION_ENABLE_RELOC_SECTION (OPTION_BUILD_ID + 1) -+#define OPTION_DISABLE_RELOC_SECTION (OPTION_ENABLE_RELOC_SECTION + 1) - - static void - gld${EMULATION_NAME}_add_options -@@ -342,8 +347,10 @@ - {"enable-long-section-names", no_argument, NULL, OPTION_ENABLE_LONG_SECTION_NAMES}, - {"disable-long-section-names", no_argument, NULL, OPTION_DISABLE_LONG_SECTION_NAMES}, - {"dynamicbase",no_argument, NULL, OPTION_DYNAMIC_BASE}, -+ {"no-dynamicbase", no_argument, NULL, OPTION_NO_DYNAMIC_BASE}, - {"forceinteg", no_argument, NULL, OPTION_FORCE_INTEGRITY}, - {"nxcompat", no_argument, NULL, OPTION_NX_COMPAT}, -+ {"no-nxcompat", no_argument, NULL, OPTION_NO_NX_COMPAT}, - {"no-isolation", no_argument, NULL, OPTION_NO_ISOLATION}, - {"no-seh", no_argument, NULL, OPTION_NO_SEH}, - {"no-bind", no_argument, NULL, OPTION_NO_BIND}, -@@ -351,6 +358,7 @@ - {"tsaware", no_argument, NULL, OPTION_TERMINAL_SERVER_AWARE}, - {"build-id", optional_argument, NULL, OPTION_BUILD_ID}, - {"enable-reloc-section", no_argument, NULL, OPTION_ENABLE_RELOC_SECTION}, -+ {"disable-reloc-section", no_argument, NULL, OPTION_DISABLE_RELOC_SECTION}, - {NULL, no_argument, NULL, 0} - }; - -@@ -485,9 +494,12 @@ - in object files\n")); - fprintf (file, _(" --dynamicbase Image base address may be relocated using\n\ - address space layout randomization (ASLR)\n")); -+ fprintf (file, _(" --no-dynamicbase Image base address may not be relocated\n")); - fprintf (file, _(" --enable-reloc-section Create the base relocation table\n")); -+ fprintf (file, _(" --disable-reloc-section Disable the base relocation table\n")); - fprintf (file, _(" --forceinteg Code integrity checks are enforced\n")); - fprintf (file, _(" --nxcompat Image is compatible with data execution prevention\n")); -+ fprintf (file, _(" --no-nxcompat Image is not compatible with data execution prevention\n")); - fprintf (file, _(" --no-isolation Image understands isolation but do not isolate the image\n")); - fprintf (file, _(" --no-seh Image does not use SEH. No SE handler may\n\ - be called in this image\n")); -@@ -862,12 +874,21 @@ - case OPTION_ENABLE_RELOC_SECTION: - pe_dll_enable_reloc_section = 1; - break; -+ case OPTION_DISABLE_RELOC_SECTION: -+ pe_dll_enable_reloc_section = 0; -+ /* fall through */ -+ case OPTION_NO_DYNAMIC_BASE: -+ pe_dll_characteristics &= ~IMAGE_DLL_CHARACTERISTICS_DYNAMIC_BASE; -+ break; - case OPTION_FORCE_INTEGRITY: - pe_dll_characteristics |= IMAGE_DLL_CHARACTERISTICS_FORCE_INTEGRITY; - break; - case OPTION_NX_COMPAT: - pe_dll_characteristics |= IMAGE_DLL_CHARACTERISTICS_NX_COMPAT; - break; -+ case OPTION_NO_NX_COMPAT: -+ pe_dll_characteristics &= ~IMAGE_DLL_CHARACTERISTICS_NX_COMPAT; -+ break; - case OPTION_NO_ISOLATION: - pe_dll_characteristics |= IMAGE_DLLCHARACTERISTICS_NO_ISOLATION; - break; ---- a/ld/emultempl/pep.em -+++ b/ld/emultempl/pep.em -@@ -237,9 +240,12 @@ - OPTION_ENABLE_LONG_SECTION_NAMES, - OPTION_DISABLE_LONG_SECTION_NAMES, - OPTION_HIGH_ENTROPY_VA, -+ OPTION_NO_HIGH_ENTROPY_VA, - OPTION_DYNAMIC_BASE, -+ OPTION_NO_DYNAMIC_BASE, - OPTION_FORCE_INTEGRITY, - OPTION_NX_COMPAT, -+ OPTION_NO_NX_COMPAT, - OPTION_NO_ISOLATION, - OPTION_NO_SEH, - OPTION_NO_BIND, -@@ -248,7 +254,8 @@ - OPTION_NO_INSERT_TIMESTAMP, - OPTION_TERMINAL_SERVER_AWARE, - OPTION_BUILD_ID, -- OPTION_ENABLE_RELOC_SECTION -+ OPTION_ENABLE_RELOC_SECTION, -+ OPTION_DISABLE_RELOC_SECTION - }; - - static void -@@ -315,9 +322,12 @@ - {"enable-long-section-names", no_argument, NULL, OPTION_ENABLE_LONG_SECTION_NAMES}, - {"disable-long-section-names", no_argument, NULL, OPTION_DISABLE_LONG_SECTION_NAMES}, - {"high-entropy-va", no_argument, NULL, OPTION_HIGH_ENTROPY_VA}, -+ {"no-high-entropy-va", no_argument, NULL, OPTION_NO_HIGH_ENTROPY_VA}, - {"dynamicbase",no_argument, NULL, OPTION_DYNAMIC_BASE}, -+ {"no-dynamicbase", no_argument, NULL, OPTION_NO_DYNAMIC_BASE}, - {"forceinteg", no_argument, NULL, OPTION_FORCE_INTEGRITY}, - {"nxcompat", no_argument, NULL, OPTION_NX_COMPAT}, -+ {"no-nxcompat", no_argument, NULL, OPTION_NO_NX_COMPAT}, - {"no-isolation", no_argument, NULL, OPTION_NO_ISOLATION}, - {"no-seh", no_argument, NULL, OPTION_NO_SEH}, - {"no-bind", no_argument, NULL, OPTION_NO_BIND}, -@@ -327,6 +337,7 @@ - {"no-insert-timestamp", no_argument, NULL, OPTION_NO_INSERT_TIMESTAMP}, - {"build-id", optional_argument, NULL, OPTION_BUILD_ID}, - {"enable-reloc-section", no_argument, NULL, OPTION_ENABLE_RELOC_SECTION}, -+ {"disable-reloc-section", no_argument, NULL, OPTION_DISABLE_RELOC_SECTION}, - {NULL, no_argument, NULL, 0} - }; - -@@ -448,11 +461,15 @@ - in object files\n")); - fprintf (file, _(" --high-entropy-va Image is compatible with 64-bit address space\n\ - layout randomization (ASLR)\n")); -+ fprintf (file, _(" --no-high-entropy-va Image is not compatible with 64-bit ASLR\n")); - fprintf (file, _(" --dynamicbase Image base address may be relocated using\n\ - address space layout randomization (ASLR)\n")); -+ fprintf (file, _(" --no-dynamicbase Image base address may not be relocated\n")); - fprintf (file, _(" --enable-reloc-section Create the base relocation table\n")); -+ fprintf (file, _(" --disable-reloc-section Disable the base relocation table\n")); - fprintf (file, _(" --forceinteg Code integrity checks are enforced\n")); - fprintf (file, _(" --nxcompat Image is compatible with data execution prevention\n")); -+ fprintf (file, _(" --no-nxcompat Image is not compatible with data execution prevention\n")); - fprintf (file, _(" --no-isolation Image understands isolation but do not isolate the image\n")); - fprintf (file, _(" --no-seh Image does not use SEH; no SE handler may\n\ - be called in this image\n")); -@@ -809,12 +826,24 @@ - case OPTION_ENABLE_RELOC_SECTION: - pep_dll_enable_reloc_section = 1; - break; -+ case OPTION_DISABLE_RELOC_SECTION: -+ pep_dll_enable_reloc_section = 0; -+ /* fall through */ -+ case OPTION_NO_DYNAMIC_BASE: -+ pe_dll_characteristics &= ~IMAGE_DLL_CHARACTERISTICS_DYNAMIC_BASE; -+ /* fall through */ -+ case OPTION_NO_HIGH_ENTROPY_VA: -+ pe_dll_characteristics &= ~IMAGE_DLL_CHARACTERISTICS_HIGH_ENTROPY_VA; -+ break; - case OPTION_FORCE_INTEGRITY: - pe_dll_characteristics |= IMAGE_DLL_CHARACTERISTICS_FORCE_INTEGRITY; - break; - case OPTION_NX_COMPAT: - pe_dll_characteristics |= IMAGE_DLL_CHARACTERISTICS_NX_COMPAT; - break; -+ case OPTION_NO_NX_COMPAT: -+ pe_dll_characteristics &= ~IMAGE_DLL_CHARACTERISTICS_NX_COMPAT; -+ break; - case OPTION_NO_ISOLATION: - pe_dll_characteristics |= IMAGE_DLLCHARACTERISTICS_NO_ISOLATION; - break; diff --git a/contrib/guix/patches/elfsteem-value-error-python-39.patch b/contrib/guix/patches/elfsteem-value-error-python-39.patch new file mode 100644 index 0000000000..21e1228afd --- /dev/null +++ b/contrib/guix/patches/elfsteem-value-error-python-39.patch @@ -0,0 +1,13 @@ +diff --git a/examples/otool.py b/examples/otool.py +index 2b8efc0..d797b2e 100755 +--- a/examples/otool.py ++++ b/examples/otool.py +@@ -342,7 +342,7 @@ if __name__ == '__main__': + try: + e = macho_init.MACHO(raw, + parseSymbols = False) +- except ValueError, err: ++ except ValueError as err: + print("%s:" %file) + print(" %s" % err) + continue diff --git a/contrib/guix/patches/gcc-8-sort-libtool-find-output.patch b/contrib/guix/patches/gcc-8-sort-libtool-find-output.patch deleted file mode 100644 index f327c464f3..0000000000 --- a/contrib/guix/patches/gcc-8-sort-libtool-find-output.patch +++ /dev/null @@ -1,400 +0,0 @@ -guix: repro: Sort find output in libtool for gcc-8 - -Otherwise the resulting .a static libraries (e.g. libstdc++.a) will not -be reproducible and end up making the Bitcoin binaries non-reproducible -as well. - -See: https://reproducible-builds.org/docs/archives/#gnu-libtool - -diff --git a/gcc/configure b/gcc/configure -index 97ba7d7d69c..e37a96f0c0c 100755 ---- a/gcc/configure -+++ b/gcc/configure -@@ -19720,20 +19720,20 @@ if test -z "$aix_libpath"; then aix_libpath="/usr/lib:/lib"; fi - prelink_cmds_CXX='tpldir=Template.dir~ - rm -rf $tpldir~ - $CC --prelink_objects --instantiation_dir $tpldir $objs $libobjs $compile_deplibs~ -- compile_command="$compile_command `find $tpldir -name \*.o | $NL2SP`"' -+ compile_command="$compile_command `find $tpldir -name \*.o | sort | $NL2SP`"' - old_archive_cmds_CXX='tpldir=Template.dir~ - rm -rf $tpldir~ - $CC --prelink_objects --instantiation_dir $tpldir $oldobjs$old_deplibs~ -- $AR $AR_FLAGS $oldlib$oldobjs$old_deplibs `find $tpldir -name \*.o | $NL2SP`~ -+ $AR $AR_FLAGS $oldlib$oldobjs$old_deplibs `find $tpldir -name \*.o | sort | $NL2SP`~ - $RANLIB $oldlib' - archive_cmds_CXX='tpldir=Template.dir~ - rm -rf $tpldir~ - $CC --prelink_objects --instantiation_dir $tpldir $predep_objects $libobjs $deplibs $convenience $postdep_objects~ -- $CC -shared $pic_flag $predep_objects $libobjs $deplibs `find $tpldir -name \*.o | $NL2SP` $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname -o $lib' -+ $CC -shared $pic_flag $predep_objects $libobjs $deplibs `find $tpldir -name \*.o | sort | $NL2SP` $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname -o $lib' - archive_expsym_cmds_CXX='tpldir=Template.dir~ - rm -rf $tpldir~ - $CC --prelink_objects --instantiation_dir $tpldir $predep_objects $libobjs $deplibs $convenience $postdep_objects~ -- $CC -shared $pic_flag $predep_objects $libobjs $deplibs `find $tpldir -name \*.o | $NL2SP` $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname ${wl}-retain-symbols-file ${wl}$export_symbols -o $lib' -+ $CC -shared $pic_flag $predep_objects $libobjs $deplibs `find $tpldir -name \*.o | sort | $NL2SP` $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname ${wl}-retain-symbols-file ${wl}$export_symbols -o $lib' - ;; - *) # Version 6 and above use weak symbols - archive_cmds_CXX='$CC -shared $pic_flag $predep_objects $libobjs $deplibs $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname -o $lib' -diff --git a/libcc1/configure b/libcc1/configure -index f53a121611c..5740ca90cab 100755 ---- a/libcc1/configure -+++ b/libcc1/configure -@@ -12221,20 +12221,20 @@ if test -z "$aix_libpath"; then aix_libpath="/usr/lib:/lib"; fi - prelink_cmds_CXX='tpldir=Template.dir~ - rm -rf $tpldir~ - $CC --prelink_objects --instantiation_dir $tpldir $objs $libobjs $compile_deplibs~ -- compile_command="$compile_command `find $tpldir -name \*.o | $NL2SP`"' -+ compile_command="$compile_command `find $tpldir -name \*.o | sort | $NL2SP`"' - old_archive_cmds_CXX='tpldir=Template.dir~ - rm -rf $tpldir~ - $CC --prelink_objects --instantiation_dir $tpldir $oldobjs$old_deplibs~ -- $AR $AR_FLAGS $oldlib$oldobjs$old_deplibs `find $tpldir -name \*.o | $NL2SP`~ -+ $AR $AR_FLAGS $oldlib$oldobjs$old_deplibs `find $tpldir -name \*.o | sort | $NL2SP`~ - $RANLIB $oldlib' - archive_cmds_CXX='tpldir=Template.dir~ - rm -rf $tpldir~ - $CC --prelink_objects --instantiation_dir $tpldir $predep_objects $libobjs $deplibs $convenience $postdep_objects~ -- $CC -shared $pic_flag $predep_objects $libobjs $deplibs `find $tpldir -name \*.o | $NL2SP` $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname -o $lib' -+ $CC -shared $pic_flag $predep_objects $libobjs $deplibs `find $tpldir -name \*.o | sort | $NL2SP` $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname -o $lib' - archive_expsym_cmds_CXX='tpldir=Template.dir~ - rm -rf $tpldir~ - $CC --prelink_objects --instantiation_dir $tpldir $predep_objects $libobjs $deplibs $convenience $postdep_objects~ -- $CC -shared $pic_flag $predep_objects $libobjs $deplibs `find $tpldir -name \*.o | $NL2SP` $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname ${wl}-retain-symbols-file ${wl}$export_symbols -o $lib' -+ $CC -shared $pic_flag $predep_objects $libobjs $deplibs `find $tpldir -name \*.o | sort | $NL2SP` $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname ${wl}-retain-symbols-file ${wl}$export_symbols -o $lib' - ;; - *) # Version 6 and above use weak symbols - archive_cmds_CXX='$CC -shared $pic_flag $predep_objects $libobjs $deplibs $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname -o $lib' -diff --git a/libffi/configure b/libffi/configure -index 790a291011f..54b1ac18306 100755 ---- a/libffi/configure -+++ b/libffi/configure -@@ -12661,20 +12661,20 @@ if test -z "$aix_libpath"; then aix_libpath="/usr/lib:/lib"; fi - prelink_cmds_CXX='tpldir=Template.dir~ - rm -rf $tpldir~ - $CC --prelink_objects --instantiation_dir $tpldir $objs $libobjs $compile_deplibs~ -- compile_command="$compile_command `find $tpldir -name \*.o | $NL2SP`"' -+ compile_command="$compile_command `find $tpldir -name \*.o | sort | $NL2SP`"' - old_archive_cmds_CXX='tpldir=Template.dir~ - rm -rf $tpldir~ - $CC --prelink_objects --instantiation_dir $tpldir $oldobjs$old_deplibs~ -- $AR $AR_FLAGS $oldlib$oldobjs$old_deplibs `find $tpldir -name \*.o | $NL2SP`~ -+ $AR $AR_FLAGS $oldlib$oldobjs$old_deplibs `find $tpldir -name \*.o | sort | $NL2SP`~ - $RANLIB $oldlib' - archive_cmds_CXX='tpldir=Template.dir~ - rm -rf $tpldir~ - $CC --prelink_objects --instantiation_dir $tpldir $predep_objects $libobjs $deplibs $convenience $postdep_objects~ -- $CC -shared $pic_flag $predep_objects $libobjs $deplibs `find $tpldir -name \*.o | $NL2SP` $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname -o $lib' -+ $CC -shared $pic_flag $predep_objects $libobjs $deplibs `find $tpldir -name \*.o | sort | $NL2SP` $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname -o $lib' - archive_expsym_cmds_CXX='tpldir=Template.dir~ - rm -rf $tpldir~ - $CC --prelink_objects --instantiation_dir $tpldir $predep_objects $libobjs $deplibs $convenience $postdep_objects~ -- $CC -shared $pic_flag $predep_objects $libobjs $deplibs `find $tpldir -name \*.o | $NL2SP` $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname ${wl}-retain-symbols-file ${wl}$export_symbols -o $lib' -+ $CC -shared $pic_flag $predep_objects $libobjs $deplibs `find $tpldir -name \*.o | sort | $NL2SP` $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname ${wl}-retain-symbols-file ${wl}$export_symbols -o $lib' - ;; - *) # Version 6 and above use weak symbols - archive_cmds_CXX='$CC -shared $pic_flag $predep_objects $libobjs $deplibs $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname -o $lib' -diff --git a/libgo/config/libtool.m4 b/libgo/config/libtool.m4 -index f7005947454..8a84417b828 100644 ---- a/libgo/config/libtool.m4 -+++ b/libgo/config/libtool.m4 -@@ -6010,20 +6010,20 @@ if test "$_lt_caught_CXX_error" != yes; then - _LT_TAGVAR(prelink_cmds, $1)='tpldir=Template.dir~ - rm -rf $tpldir~ - $CC --prelink_objects --instantiation_dir $tpldir $objs $libobjs $compile_deplibs~ -- compile_command="$compile_command `find $tpldir -name \*.o | $NL2SP`"' -+ compile_command="$compile_command `find $tpldir -name \*.o | sort | $NL2SP`"' - _LT_TAGVAR(old_archive_cmds, $1)='tpldir=Template.dir~ - rm -rf $tpldir~ - $CC --prelink_objects --instantiation_dir $tpldir $oldobjs$old_deplibs~ -- $AR $AR_FLAGS $oldlib$oldobjs$old_deplibs `find $tpldir -name \*.o | $NL2SP`~ -+ $AR $AR_FLAGS $oldlib$oldobjs$old_deplibs `find $tpldir -name \*.o | sort | $NL2SP`~ - $RANLIB $oldlib' - _LT_TAGVAR(archive_cmds, $1)='tpldir=Template.dir~ - rm -rf $tpldir~ - $CC --prelink_objects --instantiation_dir $tpldir $predep_objects $libobjs $deplibs $convenience $postdep_objects~ -- $CC -shared $pic_flag $predep_objects $libobjs $deplibs `find $tpldir -name \*.o | $NL2SP` $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname -o $lib' -+ $CC -shared $pic_flag $predep_objects $libobjs $deplibs `find $tpldir -name \*.o | sort | $NL2SP` $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname -o $lib' - _LT_TAGVAR(archive_expsym_cmds, $1)='tpldir=Template.dir~ - rm -rf $tpldir~ - $CC --prelink_objects --instantiation_dir $tpldir $predep_objects $libobjs $deplibs $convenience $postdep_objects~ -- $CC -shared $pic_flag $predep_objects $libobjs $deplibs `find $tpldir -name \*.o | $NL2SP` $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname ${wl}-retain-symbols-file ${wl}$export_symbols -o $lib' -+ $CC -shared $pic_flag $predep_objects $libobjs $deplibs `find $tpldir -name \*.o | sort | $NL2SP` $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname ${wl}-retain-symbols-file ${wl}$export_symbols -o $lib' - ;; - *) # Version 6 and above use weak symbols - _LT_TAGVAR(archive_cmds, $1)='$CC -shared $pic_flag $predep_objects $libobjs $deplibs $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname -o $lib' -diff --git a/libgo/config/ltmain.sh b/libgo/config/ltmain.sh -index ce66b44906a..0f81c401407 100644 ---- a/libgo/config/ltmain.sh -+++ b/libgo/config/ltmain.sh -@@ -2917,7 +2917,7 @@ func_extract_archives () - darwin_file= - darwin_files= - for darwin_file in $darwin_filelist; do -- darwin_files=`find unfat-$$ -name $darwin_file -print | $NL2SP` -+ darwin_files=`find unfat-$$ -name $darwin_file -print | sort | $NL2SP` - $LIPO -create -output "$darwin_file" $darwin_files - done # $darwin_filelist - $RM -rf unfat-$$ -@@ -2932,7 +2932,7 @@ func_extract_archives () - func_extract_an_archive "$my_xdir" "$my_xabs" - ;; - esac -- my_oldobjs="$my_oldobjs "`find $my_xdir -name \*.$objext -print -o -name \*.lo -print | $NL2SP` -+ my_oldobjs="$my_oldobjs "`find $my_xdir -name \*.$objext -print -o -name \*.lo -print | sort | $NL2SP` - done - - func_extract_archives_result="$my_oldobjs" -diff --git a/libhsail-rt/configure b/libhsail-rt/configure -index a4fcc10c1f9..8e671229fcd 100755 ---- a/libhsail-rt/configure -+++ b/libhsail-rt/configure -@@ -12244,20 +12244,20 @@ if test -z "$aix_libpath"; then aix_libpath="/usr/lib:/lib"; fi - prelink_cmds_CXX='tpldir=Template.dir~ - rm -rf $tpldir~ - $CC --prelink_objects --instantiation_dir $tpldir $objs $libobjs $compile_deplibs~ -- compile_command="$compile_command `find $tpldir -name \*.o | $NL2SP`"' -+ compile_command="$compile_command `find $tpldir -name \*.o | sort | $NL2SP`"' - old_archive_cmds_CXX='tpldir=Template.dir~ - rm -rf $tpldir~ - $CC --prelink_objects --instantiation_dir $tpldir $oldobjs$old_deplibs~ -- $AR $AR_FLAGS $oldlib$oldobjs$old_deplibs `find $tpldir -name \*.o | $NL2SP`~ -+ $AR $AR_FLAGS $oldlib$oldobjs$old_deplibs `find $tpldir -name \*.o | sort | $NL2SP`~ - $RANLIB $oldlib' - archive_cmds_CXX='tpldir=Template.dir~ - rm -rf $tpldir~ - $CC --prelink_objects --instantiation_dir $tpldir $predep_objects $libobjs $deplibs $convenience $postdep_objects~ -- $CC -shared $pic_flag $predep_objects $libobjs $deplibs `find $tpldir -name \*.o | $NL2SP` $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname -o $lib' -+ $CC -shared $pic_flag $predep_objects $libobjs $deplibs `find $tpldir -name \*.o | sort | $NL2SP` $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname -o $lib' - archive_expsym_cmds_CXX='tpldir=Template.dir~ - rm -rf $tpldir~ - $CC --prelink_objects --instantiation_dir $tpldir $predep_objects $libobjs $deplibs $convenience $postdep_objects~ -- $CC -shared $pic_flag $predep_objects $libobjs $deplibs `find $tpldir -name \*.o | $NL2SP` $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname ${wl}-retain-symbols-file ${wl}$export_symbols -o $lib' -+ $CC -shared $pic_flag $predep_objects $libobjs $deplibs `find $tpldir -name \*.o | sort | $NL2SP` $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname ${wl}-retain-symbols-file ${wl}$export_symbols -o $lib' - ;; - *) # Version 6 and above use weak symbols - archive_cmds_CXX='$CC -shared $pic_flag $predep_objects $libobjs $deplibs $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname -o $lib' -diff --git a/libitm/configure b/libitm/configure -index dbf386db434..29d4f10611f 100644 ---- a/libitm/configure -+++ b/libitm/configure -@@ -13067,20 +13067,20 @@ if test -z "$aix_libpath"; then aix_libpath="/usr/lib:/lib"; fi - prelink_cmds_CXX='tpldir=Template.dir~ - rm -rf $tpldir~ - $CC --prelink_objects --instantiation_dir $tpldir $objs $libobjs $compile_deplibs~ -- compile_command="$compile_command `find $tpldir -name \*.o | $NL2SP`"' -+ compile_command="$compile_command `find $tpldir -name \*.o | sort | $NL2SP`"' - old_archive_cmds_CXX='tpldir=Template.dir~ - rm -rf $tpldir~ - $CC --prelink_objects --instantiation_dir $tpldir $oldobjs$old_deplibs~ -- $AR $AR_FLAGS $oldlib$oldobjs$old_deplibs `find $tpldir -name \*.o | $NL2SP`~ -+ $AR $AR_FLAGS $oldlib$oldobjs$old_deplibs `find $tpldir -name \*.o | sort | $NL2SP`~ - $RANLIB $oldlib' - archive_cmds_CXX='tpldir=Template.dir~ - rm -rf $tpldir~ - $CC --prelink_objects --instantiation_dir $tpldir $predep_objects $libobjs $deplibs $convenience $postdep_objects~ -- $CC -shared $pic_flag $predep_objects $libobjs $deplibs `find $tpldir -name \*.o | $NL2SP` $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname -o $lib' -+ $CC -shared $pic_flag $predep_objects $libobjs $deplibs `find $tpldir -name \*.o | sort | $NL2SP` $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname -o $lib' - archive_expsym_cmds_CXX='tpldir=Template.dir~ - rm -rf $tpldir~ - $CC --prelink_objects --instantiation_dir $tpldir $predep_objects $libobjs $deplibs $convenience $postdep_objects~ -- $CC -shared $pic_flag $predep_objects $libobjs $deplibs `find $tpldir -name \*.o | $NL2SP` $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname ${wl}-retain-symbols-file ${wl}$export_symbols -o $lib' -+ $CC -shared $pic_flag $predep_objects $libobjs $deplibs `find $tpldir -name \*.o | sort | $NL2SP` $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname ${wl}-retain-symbols-file ${wl}$export_symbols -o $lib' - ;; - *) # Version 6 and above use weak symbols - archive_cmds_CXX='$CC -shared $pic_flag $predep_objects $libobjs $deplibs $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname -o $lib' -diff --git a/liboffloadmic/configure b/liboffloadmic/configure -index f873716991b..7aa9186b10e 100644 ---- a/liboffloadmic/configure -+++ b/liboffloadmic/configure -@@ -12379,20 +12379,20 @@ if test -z "$aix_libpath"; then aix_libpath="/usr/lib:/lib"; fi - prelink_cmds_CXX='tpldir=Template.dir~ - rm -rf $tpldir~ - $CC --prelink_objects --instantiation_dir $tpldir $objs $libobjs $compile_deplibs~ -- compile_command="$compile_command `find $tpldir -name \*.o | $NL2SP`"' -+ compile_command="$compile_command `find $tpldir -name \*.o | sort | $NL2SP`"' - old_archive_cmds_CXX='tpldir=Template.dir~ - rm -rf $tpldir~ - $CC --prelink_objects --instantiation_dir $tpldir $oldobjs$old_deplibs~ -- $AR $AR_FLAGS $oldlib$oldobjs$old_deplibs `find $tpldir -name \*.o | $NL2SP`~ -+ $AR $AR_FLAGS $oldlib$oldobjs$old_deplibs `find $tpldir -name \*.o | sort | $NL2SP`~ - $RANLIB $oldlib' - archive_cmds_CXX='tpldir=Template.dir~ - rm -rf $tpldir~ - $CC --prelink_objects --instantiation_dir $tpldir $predep_objects $libobjs $deplibs $convenience $postdep_objects~ -- $CC -shared $pic_flag $predep_objects $libobjs $deplibs `find $tpldir -name \*.o | $NL2SP` $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname -o $lib' -+ $CC -shared $pic_flag $predep_objects $libobjs $deplibs `find $tpldir -name \*.o | sort | $NL2SP` $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname -o $lib' - archive_expsym_cmds_CXX='tpldir=Template.dir~ - rm -rf $tpldir~ - $CC --prelink_objects --instantiation_dir $tpldir $predep_objects $libobjs $deplibs $convenience $postdep_objects~ -- $CC -shared $pic_flag $predep_objects $libobjs $deplibs `find $tpldir -name \*.o | $NL2SP` $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname ${wl}-retain-symbols-file ${wl}$export_symbols -o $lib' -+ $CC -shared $pic_flag $predep_objects $libobjs $deplibs `find $tpldir -name \*.o | sort | $NL2SP` $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname ${wl}-retain-symbols-file ${wl}$export_symbols -o $lib' - ;; - *) # Version 6 and above use weak symbols - archive_cmds_CXX='$CC -shared $pic_flag $predep_objects $libobjs $deplibs $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname -o $lib' -diff --git a/liboffloadmic/plugin/configure b/liboffloadmic/plugin/configure -index c031eb3e7fa..67fc7368f21 100644 ---- a/liboffloadmic/plugin/configure -+++ b/liboffloadmic/plugin/configure -@@ -12086,20 +12086,20 @@ if test -z "$aix_libpath"; then aix_libpath="/usr/lib:/lib"; fi - prelink_cmds_CXX='tpldir=Template.dir~ - rm -rf $tpldir~ - $CC --prelink_objects --instantiation_dir $tpldir $objs $libobjs $compile_deplibs~ -- compile_command="$compile_command `find $tpldir -name \*.o | $NL2SP`"' -+ compile_command="$compile_command `find $tpldir -name \*.o | sort | $NL2SP`"' - old_archive_cmds_CXX='tpldir=Template.dir~ - rm -rf $tpldir~ - $CC --prelink_objects --instantiation_dir $tpldir $oldobjs$old_deplibs~ -- $AR $AR_FLAGS $oldlib$oldobjs$old_deplibs `find $tpldir -name \*.o | $NL2SP`~ -+ $AR $AR_FLAGS $oldlib$oldobjs$old_deplibs `find $tpldir -name \*.o | sort | $NL2SP`~ - $RANLIB $oldlib' - archive_cmds_CXX='tpldir=Template.dir~ - rm -rf $tpldir~ - $CC --prelink_objects --instantiation_dir $tpldir $predep_objects $libobjs $deplibs $convenience $postdep_objects~ -- $CC -shared $pic_flag $predep_objects $libobjs $deplibs `find $tpldir -name \*.o | $NL2SP` $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname -o $lib' -+ $CC -shared $pic_flag $predep_objects $libobjs $deplibs `find $tpldir -name \*.o | sort | $NL2SP` $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname -o $lib' - archive_expsym_cmds_CXX='tpldir=Template.dir~ - rm -rf $tpldir~ - $CC --prelink_objects --instantiation_dir $tpldir $predep_objects $libobjs $deplibs $convenience $postdep_objects~ -- $CC -shared $pic_flag $predep_objects $libobjs $deplibs `find $tpldir -name \*.o | $NL2SP` $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname ${wl}-retain-symbols-file ${wl}$export_symbols -o $lib' -+ $CC -shared $pic_flag $predep_objects $libobjs $deplibs `find $tpldir -name \*.o | sort | $NL2SP` $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname ${wl}-retain-symbols-file ${wl}$export_symbols -o $lib' - ;; - *) # Version 6 and above use weak symbols - archive_cmds_CXX='$CC -shared $pic_flag $predep_objects $libobjs $deplibs $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname -o $lib' -diff --git a/libsanitizer/configure b/libsanitizer/configure -index 4695bc7d4f7..cb7d25c07e6 100755 ---- a/libsanitizer/configure -+++ b/libsanitizer/configure -@@ -13308,20 +13308,20 @@ if test -z "$aix_libpath"; then aix_libpath="/usr/lib:/lib"; fi - prelink_cmds_CXX='tpldir=Template.dir~ - rm -rf $tpldir~ - $CC --prelink_objects --instantiation_dir $tpldir $objs $libobjs $compile_deplibs~ -- compile_command="$compile_command `find $tpldir -name \*.o | $NL2SP`"' -+ compile_command="$compile_command `find $tpldir -name \*.o | sort | $NL2SP`"' - old_archive_cmds_CXX='tpldir=Template.dir~ - rm -rf $tpldir~ - $CC --prelink_objects --instantiation_dir $tpldir $oldobjs$old_deplibs~ -- $AR $AR_FLAGS $oldlib$oldobjs$old_deplibs `find $tpldir -name \*.o | $NL2SP`~ -+ $AR $AR_FLAGS $oldlib$oldobjs$old_deplibs `find $tpldir -name \*.o | sort | $NL2SP`~ - $RANLIB $oldlib' - archive_cmds_CXX='tpldir=Template.dir~ - rm -rf $tpldir~ - $CC --prelink_objects --instantiation_dir $tpldir $predep_objects $libobjs $deplibs $convenience $postdep_objects~ -- $CC -shared $pic_flag $predep_objects $libobjs $deplibs `find $tpldir -name \*.o | $NL2SP` $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname -o $lib' -+ $CC -shared $pic_flag $predep_objects $libobjs $deplibs `find $tpldir -name \*.o | sort | $NL2SP` $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname -o $lib' - archive_expsym_cmds_CXX='tpldir=Template.dir~ - rm -rf $tpldir~ - $CC --prelink_objects --instantiation_dir $tpldir $predep_objects $libobjs $deplibs $convenience $postdep_objects~ -- $CC -shared $pic_flag $predep_objects $libobjs $deplibs `find $tpldir -name \*.o | $NL2SP` $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname ${wl}-retain-symbols-file ${wl}$export_symbols -o $lib' -+ $CC -shared $pic_flag $predep_objects $libobjs $deplibs `find $tpldir -name \*.o | sort | $NL2SP` $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname ${wl}-retain-symbols-file ${wl}$export_symbols -o $lib' - ;; - *) # Version 6 and above use weak symbols - archive_cmds_CXX='$CC -shared $pic_flag $predep_objects $libobjs $deplibs $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname -o $lib' -diff --git a/libstdc++-v3/configure b/libstdc++-v3/configure -index 61457e940ec..21ef1f61e41 100755 ---- a/libstdc++-v3/configure -+++ b/libstdc++-v3/configure -@@ -13087,20 +13087,20 @@ if test -z "$aix_libpath"; then aix_libpath="/usr/lib:/lib"; fi - prelink_cmds_CXX='tpldir=Template.dir~ - rm -rf $tpldir~ - $CC --prelink_objects --instantiation_dir $tpldir $objs $libobjs $compile_deplibs~ -- compile_command="$compile_command `find $tpldir -name \*.o | $NL2SP`"' -+ compile_command="$compile_command `find $tpldir -name \*.o | sort | $NL2SP`"' - old_archive_cmds_CXX='tpldir=Template.dir~ - rm -rf $tpldir~ - $CC --prelink_objects --instantiation_dir $tpldir $oldobjs$old_deplibs~ -- $AR $AR_FLAGS $oldlib$oldobjs$old_deplibs `find $tpldir -name \*.o | $NL2SP`~ -+ $AR $AR_FLAGS $oldlib$oldobjs$old_deplibs `find $tpldir -name \*.o | sort | $NL2SP`~ - $RANLIB $oldlib' - archive_cmds_CXX='tpldir=Template.dir~ - rm -rf $tpldir~ - $CC --prelink_objects --instantiation_dir $tpldir $predep_objects $libobjs $deplibs $convenience $postdep_objects~ -- $CC -shared $pic_flag $predep_objects $libobjs $deplibs `find $tpldir -name \*.o | $NL2SP` $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname -o $lib' -+ $CC -shared $pic_flag $predep_objects $libobjs $deplibs `find $tpldir -name \*.o | sort | $NL2SP` $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname -o $lib' - archive_expsym_cmds_CXX='tpldir=Template.dir~ - rm -rf $tpldir~ - $CC --prelink_objects --instantiation_dir $tpldir $predep_objects $libobjs $deplibs $convenience $postdep_objects~ -- $CC -shared $pic_flag $predep_objects $libobjs $deplibs `find $tpldir -name \*.o | $NL2SP` $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname ${wl}-retain-symbols-file ${wl}$export_symbols -o $lib' -+ $CC -shared $pic_flag $predep_objects $libobjs $deplibs `find $tpldir -name \*.o | sort | $NL2SP` $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname ${wl}-retain-symbols-file ${wl}$export_symbols -o $lib' - ;; - *) # Version 6 and above use weak symbols - archive_cmds_CXX='$CC -shared $pic_flag $predep_objects $libobjs $deplibs $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname -o $lib' -diff --git a/libtool.m4 b/libtool.m4 -index 24d13f34409..940faaa161d 100644 ---- a/libtool.m4 -+++ b/libtool.m4 -@@ -6005,20 +6005,20 @@ if test "$_lt_caught_CXX_error" != yes; then - _LT_TAGVAR(prelink_cmds, $1)='tpldir=Template.dir~ - rm -rf $tpldir~ - $CC --prelink_objects --instantiation_dir $tpldir $objs $libobjs $compile_deplibs~ -- compile_command="$compile_command `find $tpldir -name \*.o | $NL2SP`"' -+ compile_command="$compile_command `find $tpldir -name \*.o | sort | $NL2SP`"' - _LT_TAGVAR(old_archive_cmds, $1)='tpldir=Template.dir~ - rm -rf $tpldir~ - $CC --prelink_objects --instantiation_dir $tpldir $oldobjs$old_deplibs~ -- $AR $AR_FLAGS $oldlib$oldobjs$old_deplibs `find $tpldir -name \*.o | $NL2SP`~ -+ $AR $AR_FLAGS $oldlib$oldobjs$old_deplibs `find $tpldir -name \*.o | sort | $NL2SP`~ - $RANLIB $oldlib' - _LT_TAGVAR(archive_cmds, $1)='tpldir=Template.dir~ - rm -rf $tpldir~ - $CC --prelink_objects --instantiation_dir $tpldir $predep_objects $libobjs $deplibs $convenience $postdep_objects~ -- $CC -shared $pic_flag $predep_objects $libobjs $deplibs `find $tpldir -name \*.o | $NL2SP` $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname -o $lib' -+ $CC -shared $pic_flag $predep_objects $libobjs $deplibs `find $tpldir -name \*.o | sort | $NL2SP` $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname -o $lib' - _LT_TAGVAR(archive_expsym_cmds, $1)='tpldir=Template.dir~ - rm -rf $tpldir~ - $CC --prelink_objects --instantiation_dir $tpldir $predep_objects $libobjs $deplibs $convenience $postdep_objects~ -- $CC -shared $pic_flag $predep_objects $libobjs $deplibs `find $tpldir -name \*.o | $NL2SP` $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname ${wl}-retain-symbols-file ${wl}$export_symbols -o $lib' -+ $CC -shared $pic_flag $predep_objects $libobjs $deplibs `find $tpldir -name \*.o | sort | $NL2SP` $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname ${wl}-retain-symbols-file ${wl}$export_symbols -o $lib' - ;; - *) # Version 6 and above use weak symbols - _LT_TAGVAR(archive_cmds, $1)='$CC -shared $pic_flag $predep_objects $libobjs $deplibs $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname -o $lib' -diff --git a/libvtv/configure b/libvtv/configure -index a197f750453..31ab3a0637b 100755 ---- a/libvtv/configure -+++ b/libvtv/configure -@@ -13339,20 +13339,20 @@ if test -z "$aix_libpath"; then aix_libpath="/usr/lib:/lib"; fi - prelink_cmds_CXX='tpldir=Template.dir~ - rm -rf $tpldir~ - $CC --prelink_objects --instantiation_dir $tpldir $objs $libobjs $compile_deplibs~ -- compile_command="$compile_command `find $tpldir -name \*.o | $NL2SP`"' -+ compile_command="$compile_command `find $tpldir -name \*.o | sort | $NL2SP`"' - old_archive_cmds_CXX='tpldir=Template.dir~ - rm -rf $tpldir~ - $CC --prelink_objects --instantiation_dir $tpldir $oldobjs$old_deplibs~ -- $AR $AR_FLAGS $oldlib$oldobjs$old_deplibs `find $tpldir -name \*.o | $NL2SP`~ -+ $AR $AR_FLAGS $oldlib$oldobjs$old_deplibs `find $tpldir -name \*.o | sort | $NL2SP`~ - $RANLIB $oldlib' - archive_cmds_CXX='tpldir=Template.dir~ - rm -rf $tpldir~ - $CC --prelink_objects --instantiation_dir $tpldir $predep_objects $libobjs $deplibs $convenience $postdep_objects~ -- $CC -shared $pic_flag $predep_objects $libobjs $deplibs `find $tpldir -name \*.o | $NL2SP` $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname -o $lib' -+ $CC -shared $pic_flag $predep_objects $libobjs $deplibs `find $tpldir -name \*.o | sort | $NL2SP` $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname -o $lib' - archive_expsym_cmds_CXX='tpldir=Template.dir~ - rm -rf $tpldir~ - $CC --prelink_objects --instantiation_dir $tpldir $predep_objects $libobjs $deplibs $convenience $postdep_objects~ -- $CC -shared $pic_flag $predep_objects $libobjs $deplibs `find $tpldir -name \*.o | $NL2SP` $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname ${wl}-retain-symbols-file ${wl}$export_symbols -o $lib' -+ $CC -shared $pic_flag $predep_objects $libobjs $deplibs `find $tpldir -name \*.o | sort | $NL2SP` $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname ${wl}-retain-symbols-file ${wl}$export_symbols -o $lib' - ;; - *) # Version 6 and above use weak symbols - archive_cmds_CXX='$CC -shared $pic_flag $predep_objects $libobjs $deplibs $postdep_objects $compiler_flags ${wl}-soname ${wl}$soname -o $lib' -diff --git a/ltmain.sh b/ltmain.sh -index 9503ec85d70..79f9ba89af5 100644 ---- a/ltmain.sh -+++ b/ltmain.sh -@@ -2917,7 +2917,7 @@ func_extract_archives () - darwin_file= - darwin_files= - for darwin_file in $darwin_filelist; do -- darwin_files=`find unfat-$$ -name $darwin_file -print | $NL2SP` -+ darwin_files=`find unfat-$$ -name $darwin_file -print | sort | $NL2SP` - $LIPO -create -output "$darwin_file" $darwin_files - done # $darwin_filelist - $RM -rf unfat-$$ -@@ -2932,7 +2932,7 @@ func_extract_archives () - func_extract_an_archive "$my_xdir" "$my_xabs" - ;; - esac -- my_oldobjs="$my_oldobjs "`find $my_xdir -name \*.$objext -print -o -name \*.lo -print | $NL2SP` -+ my_oldobjs="$my_oldobjs "`find $my_xdir -name \*.$objext -print -o -name \*.lo -print | sort | $NL2SP` - done - - func_extract_archives_result="$my_oldobjs" diff --git a/contrib/guix/patches/nsis-SConstruct-sde-support.patch b/contrib/guix/patches/nsis-SConstruct-sde-support.patch deleted file mode 100644 index f58406a7a0..0000000000 --- a/contrib/guix/patches/nsis-SConstruct-sde-support.patch +++ /dev/null @@ -1,18 +0,0 @@ -https://github.com/kichik/nsis/pull/13 -https://sourceforge.net/p/nsis/code/7248/ - -diff --git a/SConstruct b/SConstruct -index e8252c9..41786f2 100755 ---- a/SConstruct -+++ b/SConstruct -@@ -95,8 +95,8 @@ default_doctype = 'html' - if defenv.WhereIs('hhc', os.environ['PATH']): - default_doctype = 'chm' - --from time import strftime, gmtime --cvs_version = strftime('%d-%b-%Y.cvs', gmtime()) -+import time -+cvs_version = time.strftime('%d-%b-%Y.cvs', time.gmtime(int(os.environ.get('SOURCE_DATE_EPOCH', time.time())))) - - opts = Variables() - diff --git a/contrib/guix/patches/nsis-gcc-10-memmove.patch b/contrib/guix/patches/nsis-gcc-10-memmove.patch new file mode 100644 index 0000000000..a1aadfd4f3 --- /dev/null +++ b/contrib/guix/patches/nsis-gcc-10-memmove.patch @@ -0,0 +1,23 @@ +commit f6df41524e703dc471e283e566a48e05a735b7f2 +Author: Anders +Date: Sat Jun 27 23:18:45 2020 +0000 + + Don't let GCC 10 generate memmove calls (bug #1248) + + git-svn-id: https://svn.code.sf.net/p/nsis/code/NSIS/trunk@7189 212acab6-be3b-0410-9dea-997c60f758d6 + +diff --git a/SCons/Config/gnu b/SCons/Config/gnu +index bfcb362d..21fa446b 100644 +--- a/SCons/Config/gnu ++++ b/SCons/Config/gnu +@@ -103,6 +103,10 @@ stub_env.Append(LINKFLAGS = ['$NODEFLIBS_FLAG']) # no standard libraries + stub_env.Append(LINKFLAGS = ['$ALIGN_FLAG']) # 512 bytes align + stub_env.Append(LINKFLAGS = ['$MAP_FLAG']) # generate map file + ++conf = FlagsConfigure(stub_env) ++conf.CheckCompileFlag('-fno-tree-loop-distribute-patterns') # GCC 10: Don't generate msvcrt!memmove calls (bug #1248) ++conf.Finish() ++ + stub_uenv = stub_env.Clone() + stub_uenv.Append(CPPDEFINES = ['_UNICODE', 'UNICODE']) + diff --git a/contrib/install_db4.sh b/contrib/install_db4.sh index 4037936404..2850c4b993 100755 --- a/contrib/install_db4.sh +++ b/contrib/install_db4.sh @@ -1,5 +1,5 @@ #!/bin/sh -# Copyright (c) 2017-2019 The Bitcoin Core developers +# Copyright (c) 2017-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -20,7 +20,7 @@ expand_path() { cd "${1}" && pwd -P } -BDB_PREFIX="$(expand_path ${1})/db4"; shift; +BDB_PREFIX="$(expand_path "${1}")/db4"; shift; BDB_VERSION='db-4.8.30.NC' BDB_HASH='12edc0df75bf9abd7f82f821795bcee50f42cb2e5f76a6a281b85732798364ef' BDB_URL="https://download.oracle.com/berkeley-db/${BDB_VERSION}.tar.gz" @@ -55,13 +55,22 @@ http_get() { echo "File ${2} already exists; not downloading again" elif check_exists curl; then curl --insecure --retry 5 "${1}" -o "${2}" - else + elif check_exists wget; then wget --no-check-certificate "${1}" -O "${2}" + else + echo "Simple transfer utilities 'curl' and 'wget' not found. Please install one of them and try again." + exit 1 fi sha256_check "${3}" "${2}" } +# Ensure the commands we use exist on the system +if ! check_exists patch; then + echo "Command-line tool 'patch' not found. Install patch and try again." + exit 1 +fi + mkdir -p "${BDB_PREFIX}" http_get "${BDB_URL}" "${BDB_VERSION}.tar.gz" "${BDB_HASH}" tar -xzvf ${BDB_VERSION}.tar.gz -C "$BDB_PREFIX" @@ -221,10 +230,10 @@ EOF # The packaged config.guess and config.sub are ancient (2009) and can cause build issues. # Replace them with modern versions. # See https://github.com/bitcoin/bitcoin/issues/16064 -CONFIG_GUESS_URL='https://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.guess;hb=55eaf3e779455c4e5cc9f82efb5278be8f8f900b' -CONFIG_GUESS_HASH='2d1ff7bca773d2ec3c6217118129220fa72d8adda67c7d2bf79994b3129232c1' -CONFIG_SUB_URL='https://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.sub;hb=55eaf3e779455c4e5cc9f82efb5278be8f8f900b' -CONFIG_SUB_HASH='3a4befde9bcdf0fdb2763fc1bfa74e8696df94e1ad7aac8042d133c8ff1d2e32' +CONFIG_GUESS_URL='https://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.guess;hb=4550d2f15b3a7ce2451c1f29500b9339430c877f' +CONFIG_GUESS_HASH='c8f530e01840719871748a8071113435bdfdf75b74c57e78e47898edea8754ae' +CONFIG_SUB_URL='https://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.sub;hb=4550d2f15b3a7ce2451c1f29500b9339430c877f' +CONFIG_SUB_HASH='3969f7d5f6967ccc6f792401b8ef3916a1d1b1d0f0de5a4e354c95addb8b800e' rm -f "dist/config.guess" rm -f "dist/config.sub" diff --git a/contrib/linearize/linearize-data.py b/contrib/linearize/linearize-data.py index 73f54cd488..441b5da764 100755 --- a/contrib/linearize/linearize-data.py +++ b/contrib/linearize/linearize-data.py @@ -2,7 +2,7 @@ # # linearize-data.py: Construct a linear, no-fork version of the chain. # -# Copyright (c) 2013-2020 The Bitcoin Core developers +# Copyright (c) 2013-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. # @@ -17,7 +17,6 @@ import datetime import time import glob from collections import namedtuple -from binascii import unhexlify settings = {} @@ -332,7 +331,7 @@ if __name__ == '__main__': settings['max_out_sz'] = int(settings['max_out_sz']) settings['split_timestamp'] = int(settings['split_timestamp']) settings['file_timestamp'] = int(settings['file_timestamp']) - settings['netmagic'] = unhexlify(settings['netmagic'].encode('utf-8')) + settings['netmagic'] = bytes.fromhex(settings['netmagic']) settings['out_of_order_cache_sz'] = int(settings['out_of_order_cache_sz']) settings['debug_output'] = settings['debug_output'].lower() diff --git a/contrib/macdeploy/README.md b/contrib/macdeploy/README.md index 21f6ba2eb3..ce69079e29 100644 --- a/contrib/macdeploy/README.md +++ b/contrib/macdeploy/README.md @@ -12,11 +12,16 @@ When complete, it will have produced `Bitcoin-Core.dmg`. ### Step 1: Obtaining `Xcode.app` +A free Apple Developer Account is required to proceed. + Our current macOS SDK -(`Xcode-12.1-12A7403-extracted-SDK-with-libcxx-headers.tar.gz`) can be +(`Xcode-12.2-12B45b-extracted-SDK-with-libcxx-headers.tar.gz`) can be extracted from -[Xcode_12.1.xip](https://download.developer.apple.com/Developer_Tools/Xcode_12.1/Xcode_12.1.xip). -An Apple ID is needed to download this. +[Xcode_12.2.xip](https://download.developer.apple.com/Developer_Tools/Xcode_12.2/Xcode_12.2.xip). +Alternatively, after logging in to your account go to 'Downloads', then 'More' +and search for [`Xcode_12.2`](https://developer.apple.com/download/all/?q=Xcode%2012.2). +An Apple ID and cookies enabled for the hostname are needed to download this. +The `sha256sum` of the archive should be `28d352f8c14a43d9b8a082ac6338dc173cb153f964c6e8fb6ba389e5be528bd0`. After Xcode version 7.x, Apple started shipping the `Xcode.app` in a `.xip` archive. This makes the SDK less-trivial to extract on non-macOS machines. One @@ -27,25 +32,25 @@ approach (tested on Debian Buster) is outlined below: apt install cpio git clone https://github.com/bitcoin-core/apple-sdk-tools.git -# Unpack Xcode_12.1.xip and place the resulting Xcode.app in your current +# Unpack Xcode_12.2.xip and place the resulting Xcode.app in your current # working directory -python3 apple-sdk-tools/extract_xcode.py -f Xcode_12.1.xip | cpio -d -i +python3 apple-sdk-tools/extract_xcode.py -f Xcode_12.2.xip | cpio -d -i ``` On macOS the process is more straightforward: ```bash -xip -x Xcode_12.1.xip +xip -x Xcode_12.2.xip ``` -### Step 2: Generating `Xcode-12.1-12A7403-extracted-SDK-with-libcxx-headers.tar.gz` from `Xcode.app` +### Step 2: Generating `Xcode-12.2-12B45b-extracted-SDK-with-libcxx-headers.tar.gz` from `Xcode.app` -To generate `Xcode-12.1-12A7403-extracted-SDK-with-libcxx-headers.tar.gz`, run +To generate `Xcode-12.2-12B45b-extracted-SDK-with-libcxx-headers.tar.gz`, run the script [`gen-sdk`](./gen-sdk) with the path to `Xcode.app` (extracted in the previous stage) as the first argument. ```bash -# Generate a Xcode-12.1-12A7403-extracted-SDK-with-libcxx-headers.tar.gz from +# Generate a Xcode-12.2-12B45b-extracted-SDK-with-libcxx-headers.tar.gz from # the supplied Xcode.app ./contrib/macdeploy/gen-sdk '/path/to/Xcode.app' ``` @@ -75,21 +80,11 @@ This version of `cctools` has been patched to use the current version of `clang` and its `libLTO.so` rather than those from `llvmgcc`, as it was originally done in `toolchain4`. To complicate things further, all builds must target an Apple SDK. These SDKs are free to -download, but not redistributable. To obtain it, register for an Apple Developer Account, -then download [Xcode_11.3.1](https://download.developer.apple.com/Developer_Tools/Xcode_11.3.1/Xcode_11.3.1.xip). +download, but not redistributable. See the SDK Extraction notes above for how to obtain it. -This file is many gigabytes in size, but most (but not all) of what we need is -contained only in a single directory: - -```bash -Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX.sdk -``` - -See the SDK Extraction notes above for how to obtain it. - -The Gitian descriptors build 2 sets of files: Linux tools, then Apple binaries which are +The Guix process builds 2 sets of files: Linux tools, then Apple binaries which are created using these tools. The build process has been designed to avoid including the -SDK's files in Gitian's outputs. All interim tarballs are fully deterministic and may be freely +SDK's files in Guix's outputs. All interim tarballs are fully deterministic and may be freely redistributed. [`xorrisofs`](https://www.gnu.org/software/xorriso/) is used to create the DMG. @@ -110,11 +105,11 @@ order to satisfy the new Gatekeeper requirements. Because this private key canno shared, we'll have to be a bit creative in order for the build process to remain somewhat deterministic. Here's how it works: -- Builders use Gitian to create an unsigned release. This outputs an unsigned DMG which +- Builders use Guix to create an unsigned release. This outputs an unsigned DMG which users may choose to bless and run. It also outputs an unsigned app structure in the form of a tarball, which also contains all of the tools that have been previously (deterministically) built in order to create a final DMG. - The Apple keyholder uses this unsigned app to create a detached signature, using the script that is also included there. Detached signatures are available from this [repository](https://github.com/bitcoin-core/bitcoin-detached-sigs). -- Builders feed the unsigned app + detached signature back into Gitian. It uses the +- Builders feed the unsigned app + detached signature back into Guix. It uses the pre-built tools to recombine the pieces into a deterministic DMG. diff --git a/contrib/macdeploy/background.svg b/contrib/macdeploy/background.svg deleted file mode 100644 index 9c330af451..0000000000 --- a/contrib/macdeploy/background.svg +++ /dev/null @@ -1,34 +0,0 @@ - - - - - - - - - - - - - - - - PACKAGE_NAME - - - - - diff --git a/contrib/macdeploy/background.tiff b/contrib/macdeploy/background.tiff new file mode 100644 index 0000000000..1fb088c837 Binary files /dev/null and b/contrib/macdeploy/background.tiff differ diff --git a/contrib/macdeploy/detached-sig-apply.sh b/contrib/macdeploy/detached-sig-apply.sh index d481413cc3..c7296387eb 100755 --- a/contrib/macdeploy/detached-sig-apply.sh +++ b/contrib/macdeploy/detached-sig-apply.sh @@ -1,5 +1,5 @@ #!/bin/sh -# Copyright (c) 2014-2019 The Bitcoin Core developers +# Copyright (c) 2014-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -22,6 +22,6 @@ if [ -z "$SIGNATURE" ]; then exit 1 fi -${SIGNAPPLE} apply ${UNSIGNED} ${SIGNATURE} +${SIGNAPPLE} apply "${UNSIGNED}" "${SIGNATURE}" mv ${ROOTDIR} ${OUTDIR} echo "Signed: ${OUTDIR}" diff --git a/contrib/macdeploy/detached-sig-create.sh b/contrib/macdeploy/detached-sig-create.sh index dde18376fb..7fff046931 100755 --- a/contrib/macdeploy/detached-sig-create.sh +++ b/contrib/macdeploy/detached-sig-create.sh @@ -1,5 +1,5 @@ #!/bin/sh -# Copyright (c) 2014-2019 The Bitcoin Core developers +# Copyright (c) 2014-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/contrib/macdeploy/gen-sdk b/contrib/macdeploy/gen-sdk index 457d8f5e64..ebef1d2db0 100755 --- a/contrib/macdeploy/gen-sdk +++ b/contrib/macdeploy/gen-sdk @@ -81,7 +81,7 @@ def run(): print("Creating output .tar.gz file...") with out_sdktgz_path.open("wb") as fp: - with gzip.GzipFile(fileobj=fp, compresslevel=9, mtime=0) as gzf: + with gzip.GzipFile(fileobj=fp, mode='wb', compresslevel=9, mtime=0) as gzf: with tarfile.open(mode="w", fileobj=gzf) as tarfp: print("Adding MacOSX SDK {} files...".format(sdk_version)) tarfp_add_with_base_change(tarfp, sdk_dir, out_name) diff --git a/contrib/macdeploy/macdeployqtplus b/contrib/macdeploy/macdeployqtplus index 9542f37468..2ad717f529 100755 --- a/contrib/macdeploy/macdeployqtplus +++ b/contrib/macdeploy/macdeployqtplus @@ -16,7 +16,7 @@ # along with this program. If not, see . # -import sys, re, os, shutil, stat, os.path +import sys, re, os, platform, shutil, stat, subprocess, os.path from argparse import ArgumentParser from ds_store import DSStore from mac_alias import Alias @@ -244,49 +244,47 @@ def copyFramework(framework: FrameworkInfo, path: str, verbose: int) -> Optional fromPath = framework.sourceFilePath toDir = os.path.join(path, framework.destinationDirectory) toPath = os.path.join(toDir, framework.binaryName) - - if not os.path.exists(fromPath): - raise RuntimeError(f"No file at {fromPath}") - - if os.path.exists(toPath): - return None # Already there - - if not os.path.exists(toDir): - os.makedirs(toDir) - - shutil.copy2(fromPath, toPath) - if verbose: - print("Copied:", fromPath) - print(" to:", toPath) + + if framework.isDylib(): + if not os.path.exists(fromPath): + raise RuntimeError(f"No file at {fromPath}") + + if os.path.exists(toPath): + return None # Already there + + if not os.path.exists(toDir): + os.makedirs(toDir) + + shutil.copy2(fromPath, toPath) + if verbose: + print("Copied:", fromPath) + print(" to:", toPath) + else: + to_dir = os.path.join(path, "Contents", "Frameworks", framework.frameworkName) + if os.path.exists(to_dir): + return None # Already there + + from_dir = framework.frameworkPath + if not os.path.exists(from_dir): + raise RuntimeError(f"No directory at {from_dir}") + + shutil.copytree(from_dir, to_dir, symlinks=True) + if verbose: + print("Copied:", from_dir) + print(" to:", to_dir) + + headers_link = os.path.join(to_dir, "Headers") + if os.path.exists(headers_link): + os.unlink(headers_link) + + headers_dir = os.path.join(to_dir, framework.binaryDirectory, "Headers") + if os.path.exists(headers_dir): + shutil.rmtree(headers_dir) permissions = os.stat(toPath) if not permissions.st_mode & stat.S_IWRITE: os.chmod(toPath, permissions.st_mode | stat.S_IWRITE) - if not framework.isDylib(): # Copy resources for real frameworks - - linkfrom = os.path.join(path, "Contents","Frameworks", framework.frameworkName, "Versions", "Current") - linkto = framework.version - if not os.path.exists(linkfrom): - os.symlink(linkto, linkfrom) - print("Linked:", linkfrom, "->", linkto) - fromResourcesDir = framework.sourceResourcesDirectory - if os.path.exists(fromResourcesDir): - toResourcesDir = os.path.join(path, framework.destinationResourcesDirectory) - shutil.copytree(fromResourcesDir, toResourcesDir, symlinks=True) - if verbose: - print("Copied resources:", fromResourcesDir) - print(" to:", toResourcesDir) - fromContentsDir = framework.sourceVersionContentsDirectory - if not os.path.exists(fromContentsDir): - fromContentsDir = framework.sourceContentsDirectory - if os.path.exists(fromContentsDir): - toContentsDir = os.path.join(path, framework.destinationVersionContentsDirectory) - shutil.copytree(fromContentsDir, toContentsDir, symlinks=True) - if verbose: - print("Copied Contents:", fromContentsDir) - print(" to:", toContentsDir) - return toPath def deployFrameworks(frameworks: List[FrameworkInfo], bundlePath: str, binaryPath: str, strip: bool, verbose: int, deploymentInfo: Optional[DeploymentInfo] = None) -> DeploymentInfo: @@ -543,6 +541,9 @@ ds.close() # ------------------------------------------------ +if platform.system() == "Darwin": + subprocess.check_call(f"codesign --deep --force --sign - {target}", shell=True) + if config.dmg is not None: print("+ Preparing .dmg disk image +") @@ -575,7 +576,7 @@ if config.dmg is not None: os.mkdir(os.path.dirname(bg_path)) if verbose: print('background.tiff', "->", bg_path) - shutil.copy2('background.tiff', bg_path) + shutil.copy2('contrib/macdeploy/background.tiff', bg_path) os.symlink("/Applications", os.path.join(disk_root, "Applications")) diff --git a/contrib/qos/tc.sh b/contrib/qos/tc.sh index 1cde19efd1..7ebcbf2251 100755 --- a/contrib/qos/tc.sh +++ b/contrib/qos/tc.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # -# Copyright (c) 2017-2019 The Bitcoin Core developers +# Copyright (c) 2017-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/contrib/seeds/README.md b/contrib/seeds/README.md index 3bca094d3b..c53446bfb0 100644 --- a/contrib/seeds/README.md +++ b/contrib/seeds/README.md @@ -10,8 +10,9 @@ to addrman with). The seeds compiled into the release are created from sipa's DNS seed data, like this: - curl -s http://bitcoin.sipa.be/seeds.txt.gz | gzip -dc > seeds_main.txt + curl https://bitcoin.sipa.be/seeds.txt.gz | gzip -dc > seeds_main.txt python3 makeseeds.py < seeds_main.txt > nodes_main.txt + cat nodes_main_manual.txt >> nodes_main.txt python3 generate-seeds.py . > ../../src/chainparamsseeds.h ## Dependencies diff --git a/contrib/seeds/generate-seeds.py b/contrib/seeds/generate-seeds.py index dbecba7d1d..44345e3987 100755 --- a/contrib/seeds/generate-seeds.py +++ b/contrib/seeds/generate-seeds.py @@ -61,7 +61,7 @@ def name_to_bip155(addr): raise ValueError(f'Invalid I2P {vchAddr}') elif '.' in addr: # IPv4 return (BIP155Network.IPV4, bytes((int(x) for x in addr.split('.')))) - elif ':' in addr: # IPv6 + elif ':' in addr: # IPv6 or CJDNS sub = [[], []] # prefix, suffix x = 0 addr = addr.split(':') @@ -77,7 +77,14 @@ def name_to_bip155(addr): sub[x].append(val & 0xff) nullbytes = 16 - len(sub[0]) - len(sub[1]) assert((x == 0 and nullbytes == 0) or (x == 1 and nullbytes > 0)) - return (BIP155Network.IPV6, bytes(sub[0] + ([0] * nullbytes) + sub[1])) + addr_bytes = bytes(sub[0] + ([0] * nullbytes) + sub[1]) + if addr_bytes[0] == 0xfc: + # Assume that seeds with fc00::/8 addresses belong to CJDNS, + # not to the publicly unroutable "Unique Local Unicast" network, see + # RFC4193: https://datatracker.ietf.org/doc/html/rfc4193#section-8 + return (BIP155Network.CJDNS, addr_bytes) + else: + return (BIP155Network.IPV6, addr_bytes) else: raise ValueError('Could not parse address %s' % addr) diff --git a/contrib/seeds/makeseeds.py b/contrib/seeds/makeseeds.py index 9be6a690a6..2b377f6c01 100755 --- a/contrib/seeds/makeseeds.py +++ b/contrib/seeds/makeseeds.py @@ -25,7 +25,7 @@ with open("suspicious_hosts.txt", mode="r", encoding="utf-8") as f: PATTERN_IPV4 = re.compile(r"^((\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})):(\d+)$") PATTERN_IPV6 = re.compile(r"^\[([0-9a-z:]+)\]:(\d+)$") -PATTERN_ONION = re.compile(r"^([abcdefghijklmnopqrstuvwxyz234567]{16}\.onion):(\d+)$") +PATTERN_ONION = re.compile(r"^([a-z2-7]{56}\.onion):(\d+)$") PATTERN_AGENT = re.compile( r"^/Satoshi:(" r"0.14.(0|1|2|3|99)|" @@ -33,9 +33,11 @@ PATTERN_AGENT = re.compile( r"0.16.(0|1|2|3|99)|" r"0.17.(0|0.1|1|2|99)|" r"0.18.(0|1|99)|" - r"0.19.(0|1|99)|" - r"0.20.(0|1|99)|" - r"0.21.99" + r"0.19.(0|1|2|99)|" + r"0.20.(0|1|2|99)|" + r"0.21.(0|1|2|99)|" + r"22.(0|99)|" + r"23.99" r")") def parseline(line): @@ -140,8 +142,8 @@ def lookup_asn(net, ip): reversed(ipaddr.split('.'))) + prefix + '.asn.cymru.com', 'TXT').response.answer][0].split('\"')[1].split(' ')[0]) return asn - except Exception: - sys.stderr.write('ERR: Could not resolve ASN for "' + ip + '"\n') + except Exception as e: + sys.stderr.write(f'ERR: Could not resolve ASN for "{ip}": {e}\n') return None # Based on Greg Maxwell's seed_filter.py diff --git a/contrib/seeds/nodes_main.txt b/contrib/seeds/nodes_main.txt index f7bfb6eb0a..d8e34bdb60 100644 --- a/contrib/seeds/nodes_main.txt +++ b/contrib/seeds/nodes_main.txt @@ -1,655 +1,649 @@ -2.39.173.126:8333 -3.14.168.201:48333 -4.36.112.44:8333 -5.8.18.31:8333 -5.14.200.167:8333 -5.56.20.2:8333 -5.102.146.99:8333 +2.37.30.144:8777 +2.138.174.158:8333 +2.152.78.124:8333 +5.8.18.154:8333 +5.45.74.50:8333 +5.79.123.3:8333 +5.102.168.217:22220 5.103.137.146:9333 5.128.87.126:8333 -5.133.65.82:8333 -5.187.55.242:8333 -5.188.62.24:8333 -5.188.62.33:8333 -5.199.133.193:8333 -8.38.89.152:8333 -13.231.20.249:8333 -18.27.79.17:8333 +5.172.132.200:8333 +5.188.62.18:8333 +5.254.101.226:8334 +8.210.18.56:8333 +8.210.92.32:8333 +14.13.34.225:16181 +14.39.151.167:8333 +18.196.79.108:8333 +18.218.139.58:48333 20.184.15.116:8433 -23.28.205.97:8333 -23.106.252.230:8333 -23.175.0.202:8333 -23.175.0.212:8333 -23.241.250.252:8333 -23.245.24.154:8333 -24.86.184.66:8333 -24.116.246.9:8333 -24.141.34.166:8333 -24.155.196.246:8333 -24.157.130.222:8333 -24.188.176.255:8333 -24.237.70.53:8333 -27.124.4.67:8333 -31.17.70.80:8333 -31.21.8.32:8333 -31.45.118.10:8333 -31.132.17.56:8333 -31.134.121.223:8333 -32.214.183.114:8333 -35.137.236.32:8333 -35.185.145.105:8333 -35.209.51.212:8333 -35.245.175.76:8333 -37.116.95.41:8333 -37.143.9.107:8333 -37.143.116.43:8333 -37.191.244.149:8333 -37.211.78.253:8333 -37.221.209.222:24333 -37.228.92.110:8333 +23.175.0.220:8333 +23.233.107.21:8333 +24.35.68.229:8333 +24.37.3.26:8333 +24.102.91.203:8333 +24.116.153.115:8333 +24.134.6.165:8333 +24.155.218.13:8333 +24.160.137.173:8333 +24.177.106.85:8333 +24.184.0.146:8333 +24.194.222.116:8333 +24.205.215.192:8333 +27.124.108.19:8333 +31.14.40.64:8333 +31.47.202.112:8333 +31.165.115.7:8333 +34.65.45.157:8333 +34.78.48.104:8333 +34.80.134.68:8333 +34.101.132.198:8333 +34.227.68.216:8333 +35.137.212.22:8333 +35.231.190.134:8333 +37.1.217.35:8333 +37.15.62.32:8333 +37.143.118.174:8333 +37.200.59.67:8333 +37.205.9.165:8333 +38.23.180.228:8333 +38.65.119.26:8333 +38.141.134.140:8333 +39.109.122.127:8444 +41.79.70.146:8333 +41.193.122.191:8333 43.225.62.107:8333 -43.225.157.152:8333 -45.36.184.6:8333 -45.48.168.16:8333 -45.85.85.8:8333 -45.85.85.9:8333 +45.35.73.152:8333 +45.43.97.103:8333 +45.63.10.52:20008 +45.84.153.40:8333 +45.95.64.225:8333 45.129.180.214:8333 -45.149.78.128:8333 -45.151.125.218:8333 -45.154.255.46:8333 -45.155.157.239:8333 -46.28.132.34:8333 -46.28.204.21:8333 +45.154.255.162:8333 +45.226.80.102:8333 +46.6.10.230:8333 +46.23.87.218:8333 46.32.50.98:8333 -46.59.13.35:8333 -46.128.40.173:8333 -46.128.140.193:8333 +46.47.84.85:8333 +46.48.126.58:8333 46.146.248.89:8333 +46.165.221.209:9333 +46.166.142.2:8333 46.166.162.45:20001 -46.188.15.6:8333 -46.229.165.142:8333 +46.173.50.58:8333 +46.175.178.3:8333 +46.188.30.118:8333 +46.219.120.59:3673 46.229.238.187:8333 -46.249.83.82:8333 -46.254.217.169:8333 -47.74.191.34:8333 -47.115.53.163:8333 -47.187.26.135:8333 -47.222.103.234:8333 -47.253.5.99:8333 -49.232.82.76:8333 -49.247.215.43:8333 -50.2.13.166:8333 -50.34.39.72:8333 -50.45.232.189:8333 -50.68.104.92:8333 -51.68.36.57:8333 -51.154.60.34:8333 -52.169.238.66:8333 -54.197.30.223:8333 -54.227.66.57:8333 +47.93.230.171:8333 +47.100.162.210:18332 +47.144.106.249:8333 +47.188.70.205:8333 +47.227.226.242:8333 +50.2.13.164:8333 +50.5.46.195:8333 +50.45.128.28:8333 +51.148.153.60:8333 +51.154.62.103:8333 +51.154.131.18:8333 +51.158.150.155:8333 +51.159.2.218:8333 +54.198.19.34:8333 +58.105.168.41:8333 58.158.0.86:8333 -58.171.135.242:8333 -58.229.208.158:8333 -60.244.109.19:8333 -62.38.75.208:8333 -62.74.143.11:8333 -62.80.227.49:8333 +60.251.129.61:8336 +61.239.91.250:8333 +62.28.190.194:8333 62.152.58.16:9421 -62.210.167.199:8333 -62.234.188.160:8333 +62.171.129.32:8333 62.251.54.163:8333 -63.227.116.162:8333 -65.19.155.82:8333 -65.95.49.102:8333 -66.18.172.21:8333 -66.240.237.155:8333 +63.247.147.166:8333 +64.33.68.176:8333 +64.156.192.61:8333 +64.187.175.226:8333 +64.233.245.39:8333 +64.237.82.149:8333 +65.101.247.26:8333 +66.29.129.218:8333 +66.49.204.11:8333 +66.58.243.215:8333 +66.85.234.129:8333 +66.130.120.52:8333 +67.10.121.145:8333 67.210.228.203:8333 -69.30.215.42:8333 -69.59.18.206:8333 -69.64.33.71:8333 +67.213.87.21:8333 +68.181.4.12:8333 +69.7.124.146:8333 +69.8.175.201:8333 +69.59.18.22:8333 69.119.193.9:8333 -69.209.23.72:8333 -70.123.125.237:8333 -70.185.56.136:8333 -71.38.90.235:8333 -72.12.73.70:8333 -72.53.134.182:8333 -72.225.7.80:8333 -72.234.182.39:8333 -72.250.184.57:8333 -73.83.103.79:8333 +69.130.201.27:8333 +69.131.101.176:8333 +70.15.194.32:8333 +70.64.27.12:8333 +72.29.170.151:8333 +72.74.34.99:8333 +72.133.177.119:8333 +73.166.84.222:8333 +74.67.240.204:8333 +74.91.115.229:8333 74.118.137.119:8333 -74.133.100.74:8333 -74.215.219.214:8333 +74.213.251.203:8333 74.220.255.190:8333 -75.158.39.231:8333 -77.53.53.196:8333 +76.11.60.155:8333 +76.66.144.127:8333 77.70.16.245:8333 +77.85.204.149:8333 77.105.87.97:8333 77.120.113.69:8433 -77.120.122.22:8433 -77.166.83.167:8333 -77.247.178.130:8333 +77.120.113.71:8433 +77.120.122.116:8433 +77.120.122.118:8433 +77.162.190.90:8333 +77.167.245.239:55544 +77.232.41.189:8333 +78.20.227.249:8333 +78.21.167.8:8333 78.27.139.13:8333 +78.43.208.25:8333 78.63.28.146:8333 -78.83.103.4:8333 -78.141.123.99:8333 -79.77.33.131:8333 -79.77.133.30:8333 -79.101.1.25:8333 -79.117.192.229:8333 -79.133.228.55:8333 -79.146.21.163:8333 +78.72.228.239:8333 +78.108.102.8:8333 +78.129.0.39:8333 +78.129.169.69:8333 +79.77.182.180:8333 +79.77.182.183:8333 +79.107.178.59:8333 +80.55.225.158:8333 +80.64.211.102:8333 +80.64.211.103:8333 +80.71.57.50:8333 +80.81.3.27:8333 +80.82.55.43:8333 +80.88.172.227:64264 80.89.203.172:8001 80.93.213.246:8333 -80.192.98.110:8334 +80.147.82.165:8333 80.229.28.60:8333 -80.232.247.210:8333 -80.242.39.76:8333 -80.253.94.252:8333 -81.0.198.25:8333 -81.7.13.84:8333 -81.117.225.245:8333 -81.135.137.225:8333 +80.247.233.40:8333 +80.255.8.93:8333 +81.7.17.202:8333 +81.10.241.165:8333 +81.21.86.157:8333 81.171.22.143:8333 -81.191.233.134:8333 -81.232.78.75:8333 -81.242.91.23:8333 -82.29.58.109:8333 -82.136.99.22:8333 +81.237.206.224:8343 +82.69.23.195:8333 +82.96.96.40:8333 +82.116.50.101:8333 +82.136.99.122:8333 82.149.97.25:17567 -82.165.19.48:8333 -82.194.153.233:8333 -82.197.215.125:8333 -82.199.102.10:8333 -82.200.205.30:8333 +82.154.24.209:8333 +82.165.241.50:8333 +82.197.218.253:8333 82.202.68.231:8333 -82.221.128.31:8333 -82.228.6.131:8333 -83.85.139.94:8333 -83.99.245.20:8333 83.137.41.10:8333 -83.174.209.87:8333 +83.208.6.211:8333 83.217.8.31:44420 -84.38.3.249:8333 -84.38.185.122:8333 -84.92.92.247:8333 -84.192.16.234:8333 -84.194.158.124:8333 -84.212.145.24:8333 -84.212.244.95:8333 -84.216.51.36:8333 -84.255.249.163:8333 -85.25.255.147:8333 -85.70.156.209:8333 -85.145.142.46:8333 -85.170.233.95:8333 +83.220.110.48:8333 +83.222.138.85:8333 +83.243.191.199:8333 +84.22.139.57:8333 +84.27.155.17:8333 +84.75.28.247:8333 +84.112.60.16:8333 +84.211.7.56:8333 +84.237.7.249:8333 +85.23.51.177:8333 +85.24.145.198:8333 85.184.138.108:8333 -85.190.0.5:8333 -85.191.200.51:8333 -85.192.191.6:18500 -85.194.238.131:8333 +85.194.238.134:8333 85.195.54.110:8333 +85.208.71.36:8333 +85.208.71.39:8333 +85.214.136.45:8333 85.214.161.252:8333 -85.214.185.51:8333 -85.241.106.203:8333 -85.246.168.252:8333 -86.56.238.247:8333 -87.61.90.230:8333 -87.79.68.86:8333 -87.79.94.221:8333 +85.227.245.128:8333 +86.18.34.243:8333 +86.20.50.170:8333 +86.49.105.90:8333 +86.76.7.132:8333 +86.100.26.188:8333 +86.106.143.143:55373 +86.120.58.66:8333 +86.133.251.239:8901 +86.149.8.23:8901 +87.78.197.234:8333 87.120.8.5:20008 -87.246.46.132:8333 -87.247.111.222:8333 -88.84.222.252:8333 -88.86.243.241:8333 +87.121.37.156:8333 +88.82.181.44:8333 88.87.93.52:1691 -88.119.197.200:8333 -88.129.253.94:8333 +88.98.235.134:8333 +88.136.187.214:8333 88.147.244.250:8333 -88.208.3.195:8333 -88.212.44.33:8333 -88.214.57.95:8333 -89.106.199.38:8333 -89.108.126.228:8333 -89.115.120.43:8333 -89.133.68.65:8333 -89.190.19.162:8333 -89.248.172.10:8333 -90.146.153.21:8333 -90.182.165.18:8333 +88.148.153.148:8333 +88.212.45.166:8333 +88.212.55.138:8333 +89.38.96.153:9273 +89.47.161.135:8333 +89.88.62.190:8333 +89.158.32.44:8333 +89.163.145.240:8333 +89.163.249.234:3673 +89.176.196.80:8333 +89.216.21.96:8333 +90.84.227.255:8333 +90.146.130.214:8333 +90.250.9.1:8333 +91.93.194.154:8333 91.106.188.229:8333 -91.193.237.116:8333 +91.126.40.109:8333 +91.137.127.123:8333 +91.147.232.98:8333 +91.152.123.18:8333 +91.178.17.120:8333 91.204.99.178:8333 -91.204.149.5:8333 -91.214.70.63:8333 -91.228.152.236:8333 -92.12.154.115:8333 -92.249.143.44:8333 -93.12.66.98:8333 -93.46.54.4:8333 -93.115.20.130:8333 +91.223.175.14:8333 +92.42.110.242:8333 +92.53.90.84:8333 +92.221.155.228:8333 +93.57.81.162:8333 +93.95.88.13:8333 +93.103.13.1:8333 93.123.180.164:8333 -93.189.145.169:8333 -93.241.228.102:8333 -94.19.7.55:8333 -94.19.128.204:8333 -94.52.112.227:8333 -94.154.96.130:8333 -94.156.174.201:8333 -94.158.246.183:8333 -94.177.171.73:8333 -94.199.178.233:8100 -94.237.125.30:8333 -94.247.134.77:8333 -95.48.228.45:8333 -95.69.249.63:8333 -95.82.146.70:8333 +93.190.117.26:8333 +94.105.125.240:8333 +94.110.23.215:8333 +94.154.159.99:8333 +94.189.161.119:8333 +94.203.255.70:8333 +94.232.173.93:8333 +95.79.122.99:8333 +95.80.1.110:8333 95.83.73.31:8333 -95.84.164.43:8333 -95.87.226.56:8333 +95.110.133.223:8333 95.110.234.93:8333 -95.163.71.126:8333 95.164.65.194:8333 -95.174.66.211:8333 -95.211.174.137:8333 -95.216.11.156:8433 -96.47.114.108:8333 -97.84.232.105:8333 -97.99.205.241:8333 -98.25.193.114:8333 -99.115.25.13:8333 -101.32.19.184:8333 -101.100.174.240:8333 +95.165.8.182:8333 +95.174.219.101:8333 +95.191.130.100:8333 +95.214.53.154:8333 +95.215.205.180:8333 +96.43.130.234:8333 +98.25.201.31:8333 +98.128.247.182:8333 +98.171.21.129:8333 +99.147.135.161:8333 +101.100.163.118:8327 102.132.245.16:8333 -103.14.244.190:8333 -103.76.48.5:8333 +102.182.204.96:8333 +102.182.235.245:8333 +103.14.245.250:8333 +103.47.192.15:8333 103.84.84.250:8335 -103.99.168.150:8333 -103.109.101.216:8333 -103.122.247.102:8333 -103.129.13.45:8333 +103.99.168.130:8333 +103.99.168.140:8333 103.198.192.14:20008 -103.224.119.99:8333 -103.231.191.7:8333 -103.235.230.196:8333 -104.171.242.155:8333 +103.232.104.227:8333 +104.143.2.195:8333 +104.172.235.227:8333 104.238.220.199:8333 -106.163.158.127:8333 -107.150.41.179:8333 -107.159.93.103:8333 -108.183.77.12:8333 -109.9.175.65:8333 -109.99.63.159:8333 -109.110.81.90:8333 -109.123.213.130:8333 -109.134.232.81:8333 -109.169.20.168:8333 -109.199.241.148:8333 -109.229.210.6:8333 -109.236.105.40:8333 +107.11.115.68:8333 +107.173.166.43:8333 +108.4.212.83:8333 +109.136.73.97:8333 +109.173.98.23:8333 +109.190.68.116:8333 +109.235.246.60:8333 109.248.206.13:8333 -111.42.74.65:8333 -111.90.140.179:8333 -112.215.205.236:8333 -113.52.135.125:8333 -114.23.246.137:8333 +110.12.64.96:8333 +111.90.140.46:8333 +111.90.159.184:50001 +113.107.201.131:8333 115.47.141.250:8885 -115.70.110.4:8333 -116.34.189.55:8333 +116.58.171.67:8333 +116.87.57.218:8333 +116.202.161.56:8333 +117.51.159.130:8333 118.103.126.140:28333 -118.189.187.219:8333 -119.3.208.236:8333 -119.8.47.225:8333 -119.17.151.61:8333 -120.25.24.30:8333 -120.241.34.10:8333 -121.98.205.100:8333 +121.45.190.210:8333 +121.99.193.25:8333 122.112.148.153:8339 -122.116.42.140:8333 -124.217.235.180:8333 -125.236.215.133:8333 -129.13.189.212:8333 -130.185.77.105:8333 +122.148.135.234:8333 +128.0.190.26:8333 +128.65.194.136:8333 +129.126.172.115:8333 +129.226.125.10:8333 131.188.40.191:8333 -131.193.220.15:8333 -135.23.124.239:8333 -136.33.185.32:8333 +134.195.185.52:8333 +135.180.44.61:8333 +136.52.114.123:8333 136.56.170.96:8333 +137.116.213.143:8333 137.226.34.46:8333 -138.229.26.42:8333 -139.9.249.234:8333 -141.101.8.36:8333 -143.176.224.104:8333 -144.2.69.224:8333 +138.43.233.57:8333 +139.130.41.82:8333 +140.190.12.129:8333 +142.4.105.77:8333 +142.54.181.218:8333 +143.177.231.247:8333 +143.178.64.10:8333 144.34.161.65:18333 -144.91.116.44:8333 -144.137.29.181:8333 +146.4.124.134:8333 +146.83.56.69:8333 +146.90.193.68:8333 +146.196.55.156:28833 148.66.50.50:8335 -148.72.150.231:8333 -148.170.212.44:8333 -149.167.99.190:8333 -154.92.16.191:8333 -154.221.27.21:8333 -156.19.19.90:8333 -156.241.5.190:8333 -157.13.61.76:8333 -157.13.61.80:8333 -157.230.166.98:14391 -158.75.203.2:8333 -158.181.125.150:8333 -158.181.226.33:8333 -159.100.242.254:8333 -159.100.248.234:8333 -159.138.87.18:8333 -160.16.0.30:8333 -162.0.227.54:8333 -162.0.227.56:8333 -162.62.18.226:8333 -162.209.1.233:8333 -162.243.175.86:8333 -162.244.80.208:8333 -162.250.188.87:8333 -162.250.189.53:8333 -163.158.202.112:8333 -163.158.243.230:8333 -165.73.62.31:8333 +148.251.1.20:8343 +151.48.95.212:8333 +151.252.193.245:8333 +152.44.137.83:8333 +152.115.191.196:8333 +154.221.31.86:8333 +156.17.103.2:8088 +157.138.20.22:8333 +158.58.188.37:8333 +158.140.209.79:8333 +159.89.230.128:8333 +159.246.25.52:8333 +160.20.59.250:8433 +162.0.234.190:8333 +162.62.26.218:8333 +162.250.188.194:8333 +162.251.70.82:8333 +163.158.206.255:8333 +164.68.105.105:8333 +165.228.174.117:8333 166.62.82.103:32771 -166.70.94.106:8333 -167.86.90.239:8333 -169.44.34.203:8333 -172.93.101.73:8333 -172.105.7.47:8333 +166.70.49.26:8333 +166.78.241.9:8333 +166.78.241.25:8333 +167.71.73.244:8333 +167.179.147.155:8333 +168.91.238.8:8333 +172.105.21.216:8333 +172.117.105.95:8333 173.23.103.30:8000 -173.53.79.6:8333 -173.70.12.86:8333 -173.89.28.137:8333 -173.176.184.54:8333 -173.208.128.10:8333 -173.254.204.69:8333 -173.255.204.124:8333 -174.94.155.224:8333 +173.205.92.151:54805 +173.205.92.154:54805 +173.205.92.157:54805 +173.208.152.218:8333 +173.241.227.243:8333 +174.3.4.232:8333 +174.17.11.22:8333 +174.88.241.167:8333 174.114.102.41:8333 -174.114.124.12:8333 -176.10.227.59:8333 -176.31.224.214:8333 +174.114.250.86:8333 +174.142.191.136:8333 +175.39.72.87:8333 +176.12.16.135:8333 +176.37.23.30:8333 +176.62.179.221:8333 176.74.136.237:8333 -176.99.2.207:8333 -176.106.191.2:8333 -176.160.228.9:8333 -176.191.182.3:8333 +176.99.6.226:8333 176.212.185.153:8333 -176.241.137.183:8333 -177.38.215.73:8333 -178.16.222.146:8333 +177.81.236.117:8333 +178.19.106.26:8333 +178.21.118.178:8333 +178.33.232.69:8333 +178.79.84.139:8333 +178.124.162.209:8333 178.132.2.246:8333 -178.143.191.171:8333 -178.148.172.209:8333 -178.148.226.180:8333 178.150.96.46:8333 -178.182.227.50:8333 +178.162.212.44:8333 +178.193.226.120:8333 178.236.137.63:8333 -178.255.42.126:8333 -180.150.52.37:8333 -181.39.32.99:8333 -181.48.77.26:8333 -181.52.223.52:8333 -181.238.51.152:8333 -183.88.223.208:8333 +180.150.46.187:8333 +181.164.210.228:8530 183.110.220.210:30301 184.95.58.166:8336 184.164.147.82:41333 184.171.208.109:8333 -185.25.48.39:8333 +185.17.143.220:8333 +185.21.217.49:8333 185.25.48.184:8333 +185.28.96.16:8333 +185.31.136.246:8333 185.64.116.15:8333 -185.80.219.132:8333 -185.85.3.140:8333 -185.95.219.53:8333 -185.108.244.41:8333 -185.134.233.121:8333 -185.145.128.21:8333 +185.68.249.91:8333 +185.108.247.190:8333 +185.141.60.36:8333 185.148.3.227:8333 -185.153.196.240:8333 -185.158.114.184:8333 -185.165.168.196:8333 -185.181.230.74:8333 +185.148.145.74:8333 +185.159.20.143:8333 +185.167.113.59:8333 185.185.26.141:8111 -185.186.208.162:8333 185.189.132.178:57780 -185.211.59.50:8333 -185.233.148.146:8333 +185.204.197.112:8333 +185.209.70.17:8333 +185.220.156.193:8333 185.238.129.113:8333 -185.249.199.106:8333 -185.251.161.54:8333 -187.189.153.136:8333 -188.37.24.190:8333 +185.239.221.5:8333 +185.244.217.39:8333 +185.254.97.164:8333 +186.33.167.11:8333 +188.32.14.31:8334 188.42.40.234:18333 -188.61.46.36:8333 -188.68.45.143:8333 -188.127.229.105:8333 -188.134.6.84:8333 188.134.8.36:8333 +188.138.88.14:8333 +188.156.110.239:8333 +188.165.244.143:8333 +188.213.68.38:8333 188.214.129.65:20012 -188.230.168.114:8333 -189.34.14.93:8333 +188.242.15.74:8333 +188.244.4.78:8333 +189.39.6.82:8333 189.207.46.32:8333 -190.211.204.68:8333 -191.209.21.188:8333 +189.212.121.74:8333 192.3.11.20:8333 -192.3.185.210:8333 192.65.170.15:8333 -192.65.170.50:8333 -192.146.137.18:8333 -192.157.202.178:8333 +192.146.137.44:8333 +192.182.157.119:8333 +192.187.109.141:8333 192.227.80.83:8333 193.10.203.23:8334 -193.25.6.206:8333 -193.42.110.30:8333 +193.32.127.160:58477 +193.32.127.162:58477 193.58.196.212:8333 -193.106.28.8:8333 -193.189.190.123:8333 -193.194.163.35:8333 -193.194.163.53:8333 +193.106.29.106:8333 +193.138.154.43:8333 +193.178.170.232:8333 +193.196.37.62:8333 +193.222.130.14:8333 +193.234.50.227:8333 194.14.246.205:8333 -194.36.91.253:8333 -194.126.113.135:8333 194.135.135.69:8333 +194.147.113.201:8333 +194.165.30.20:8333 +194.219.62.23:8333 195.56.63.4:8333 -195.56.63.5:8333 -195.67.139.54:8333 -195.135.194.8:8333 -195.202.169.149:8333 -195.206.105.42:8333 -195.209.249.164:8333 +195.134.183.188:8333 +195.208.103.30:8444 +195.208.103.31:8444 198.1.231.6:8333 -198.200.43.215:8333 +198.12.14.136:8333 +198.84.237.70:8333 +198.178.120.5:8112 +199.48.92.184:8333 +199.68.199.19:8333 199.182.184.204:8333 +199.189.242.141:8333 199.247.7.208:8333 -199.247.249.188:8333 -200.7.252.118:8333 -200.20.186.254:8333 -200.83.166.136:8333 -202.55.87.45:8333 -202.79.167.65:8333 +200.122.181.37:8333 +201.191.6.103:8333 +202.107.219.130:8333 202.108.211.135:8333 -202.169.102.73:8333 +203.94.33.112:8333 203.130.48.117:8885 -203.132.95.10:8333 -203.151.166.123:8333 -204.93.113.108:8333 -204.111.241.195:8333 -206.124.149.66:8333 -207.115.102.98:8333 -207.229.46.150:8333 -208.76.252.198:8333 -208.100.13.56:8333 -208.100.178.175:8333 -208.110.99.105:8333 -209.6.210.179:8333 -209.133.220.74:8333 -209.141.57.57:8333 -211.27.147.67:8333 -212.34.225.118:8333 -212.89.173.216:8333 +203.132.94.196:8333 +203.162.13.181:8332 +204.191.201.43:8333 +204.229.10.90:8333 +205.178.41.124:8333 +206.55.178.157:8333 +206.126.203.8:8333 +206.174.115.96:8333 +206.223.153.52:8333 +207.188.159.25:8333 +207.229.46.80:8333 +209.58.145.157:8333 +209.126.81.147:8333 +209.145.63.150:8333 +209.209.10.30:8333 +209.237.127.227:8333 212.99.226.36:9020 -212.237.96.98:8333 -213.89.131.53:8333 -216.38.129.164:8333 -216.134.165.55:8333 +212.185.86.84:8333 +212.227.211.87:8333 +213.5.36.58:8333 +213.89.236.219:8333 +213.93.145.183:8333 +213.214.66.182:8333 +216.41.249.178:8333 216.146.251.8:8333 -216.189.190.95:8333 -216.226.128.189:8333 -216.236.164.82:8333 -217.19.216.210:8333 -217.26.32.10:8333 -217.64.47.138:8333 -217.64.133.220:8333 -217.92.55.246:8333 -218.31.113.245:8333 -218.255.242.114:8333 -220.133.39.61:8333 -223.16.30.175:8333 -[2001:19f0:6001:306f:ec4:7aff:fe8f:66ec]:8333 -[2001:1bc0:cc::a001]:8333 -[2001:1c02:2f18:d00:b62e:99ff:fe49:d492]:8333 -[2001:4100:0:64::93]:8333 -[2001:4100:0:64:dcaf:afff:fe00:6707]:8333 -[2001:470:a:c13::2]:8333 -[2001:4801:7819:74:b745:b9d5:ff10:a61a]:8333 -[2001:4ba0:fffa:5d::93]:8333 -[2001:610:1908:ff01:f816:3eff:fe33:2e32]:8333 +216.249.70.22:8333 +217.11.240.4:8333 +217.15.178.7:8333 +217.24.233.116:8333 +217.64.148.98:51401 +217.113.121.169:8333 +217.170.124.170:8333 +220.132.135.54:8333 +220.221.58.25:8333 +220.233.178.199:8333 +221.219.97.105:2001 +[2001:1608:1b:f9::1]:26491 +[2001:1620:510::2]:8333 +[2001:1bc0:c1::2000]:8333 +[2001:470:1f0a:89a::2]:8333 +[2001:470:de5a::ec]:9333 +[2001:4b98:dc0:45:216:3eff:fea2:95cd]:8333 +[2001:4dd0:3564:0:fd76:c1d3:1854:5bd9]:8333 +[2001:4de8:b1b2:1:0:dead:beef:7]:8333 [2001:638:a000:4140::ffff:191]:8333 [2001:648:2800:131:4b1f:f6fc:20f7:f99f]:8333 -[2001:678:7dc:8::2]:8333 [2001:678:cc8::1:10:88]:20008 -[2001:67c:1220:80c::93e5:dd2]:8333 -[2001:67c:1220:80c:e5dc:ad0c:9289:c28f]:8333 -[2001:67c:16dc:1201:5054:ff:fe17:4dac]:8333 -[2001:67c:2354:2::22]:8333 -[2001:67c:26b4:12:7ae3:b5ff:fe04:6f9c]:8333 -[2001:67c:2f0::20:fa]:8333 -[2001:718:801:311:5054:ff:fe19:c483]:8333 -[2001:8d8:87c:7c00::99:3c1]:8333 +[2001:67c:26b4:ff00::44]:8333 +[2001:67c:2db8:13::92]:8333 +[2001:7c0:2310:0:f816:3eff:fe6c:4f58]:8333 +[2001:818:ea1b:7600:f053:aade:f47b:b701]:8333 [2001:8f1:1404:3700:8e49:715a:2e09:b634]:9444 -[2001:b07:5d29:99a5:194b:3874:d65e:a90d]:8333 -[2001:ba8:1f1:f0fe::2]:8333 -[2001:bc8:1200:0:dac4:97ff:fe2a:3554]:20008 -[2001:da8:100d:22:10fa:d85f:10f2:21fd]:8333 -[2001:da8:8001:7a39:f035:7d:b99f:eb79]:8333 -[2001:e42:103:100::30]:8333 -[2400:2412:103:c900:825:8f20:eaff:65c2]:8333 -[2400:4052:e20:4f00:69fe:bb33:7b1c:a1ca]:8333 -[2401:1800:7800:105:be76:4eff:fe1c:b35]:8333 -[2401:3900:2:1::2]:8333 -[2401:b140::44:150]:8333 -[2401:d002:4402:0:8f28:591a:6ea0:c683]:8333 -[2403:6200:8821:3d68:195b:87e9:6819:d5c8]:8333 -[2405:6580:2140:3a00:c28c:983:364b:5d70]:8333 -[2405:9800:b911:a18a:58eb:cd3c:9d82:ea4a]:8333 +[2001:985:55a0:1::2]:8333 +[2001:999:270:2c2c:c8b:3a20:3f2f:318f]:8333 +[2001:b07:ac9:442b:79d6:bbbe:b37c:a783]:8333 +[2002:2f5b:a5f9::2f5b:a5f9]:8885 +[2002:b6ff:3dca::b6ff:3dca]:28364 +[2400:2410:cea2:d00:41bc:c9ea:861b:51ee]:8333 +[2400:3b00:20:c:bacb:29ff:feab:8886]:8333 +[2401:d002:3902:700:d72c:5e22:4e95:389d]:8333 +[2403:6200:88a0:fb17:f5f2:d8b5:b7ba:f4d3]:8333 +[2405:9800:b910:5f8e:1830:f630:2cc6:88fb]:8333 +[2405:9800:b970:c64c:109f:74e7:ae5f:87c7]:8333 [2405:aa00:2::40]:8333 +[2407:8800:bc61:2202:d63d:7eff:fe6c:dc36]:8333 +[2408:8248:7004:f831::83c]:8333 [2409:10:ca20:1df0:224:e8ff:fe1f:60d9]:8333 -[2409:8a1e:a9af:3660:1c5a:5b6b:8a2d:9848]:8333 -[2409:8a1e:a9af:3660:404:39ba:88f2:e8df]:8333 -[240b:10:9141:400:49b4:3a2e:1e5:84c]:8333 -[240d:1a:759:6000:a7b1:451a:8874:e1ac]:8333 -[240d:1a:759:6000:ddab:3141:4da0:8878]:8333 +[240b:11:43a1:bd00:e589:f8a7:49b:3b86]:8333 +[240d:1a:791:3400:d65d:64ff:fe28:927e]:8333 +[240d:1a:791:3400:d681:d7ff:fef6:a21e]:10050 +[2600:1700:5b2b:5f::8040]:8333 +[2600:2104:1003:c5ab:dc5e:90ff:fe18:1d08]:8333 +[2600:3c00:e002:2e32::1:14]:8333 [2600:8805:2400:14e:12dd:b1ff:fef2:3013]:8333 -[2601:602:8d80:b63:dc3e:24ff:fe92:5eb]:8333 -[2602:ffb6:4:2798:f816:3eff:fe2f:5441]:8333 -[2602:ffb6:4:739e:f816:3eff:fe00:c2b3]:8333 [2602:ffb8::208:72:57:200]:8333 -[2604:1380:4111:9300::1]:8333 -[2604:4300:a:2e:21b:21ff:fe11:392]:8333 +[2603:301f:1ebf:e000:e23f:49ff:fee7:7431]:8333 +[2603:6081:1800:6600:16dd:a9ff:feee:b2f3]:8333 +[2604:1380:1000:7400::1]:8333 [2604:4500::2e06]:8112 -[2604:5500:706a:4000:fc79:b9bb:1d7:c325]:8333 +[2604:5500:c134:4000:7285:c2ff:fe4a:e143]:32797 [2604:5500:c134:4000::3fc]:32797 -[2604:6800:5e11:162:5c8f:d2ff:fe26:146f]:8333 -[2605:4d00::50]:8333 -[2605:6400:20:13bf:df1d:181c:83bb:22e8]:8333 +[2604:7c00:120:4b::eb24]:8333 +[2605:6400:30:f220::]:8333 +[2605:6f80:0:7:fc1b:ccff:fe8a:d822]:8333 [2605:ae00:203::203]:8333 [2605:c000:2a0a:1::102]:8333 -[2607:f2c0:f00e:300::54]:8333 +[2605:f700:c0:827:225:90ff:fee3:34a6]:8333 +[2607:9280:b:73b:250:56ff:fe14:25b5]:8333 [2607:f2f8:ad40:bc1::1]:8333 -[2607:f470:8:1048:ae1f:6bff:fe70:7240]:8333 -[2607:ff28:800f:97:225:90ff:fe75:1110]:8333 +[2607:fa18:3a01::20]:8333 [2620:11c:5001:1118:d267:e5ff:fee9:e673]:8333 -[2620:6e:a000:2001::6]:8333 -[2804:14d:4c93:9809:9769:da80:1832:3480]:8333 +[2620:11c:5001:2199:d267:e5ff:fee9:e673]:8333 +[2620:6:2003:105:2d8:61ff:fe0f:853]:8333 +[2620:6e:a000:1:42:42:42:42]:8333 +[2803:cf00:af8:f200:b89e:cf34:92c7:2d26]:8333 +[2804:14c:65d1:402c:bc53:bf5d:68a:2136]:8333 +[2804:7f1:e783:d401:661c:67ff:feba:5547]:8333 +[2804:d57:5537:4800:21e:67ff:fea8:d798]:8333 +[2804:d57:5537:4800:3615:9eff:fe23:d610]:8333 +[2806:2f0:2080:62a:86f:1a01:c44f:1794]:8333 +[2a00:1028:8382:bf22:5f7f:b78f:2737:7739]:8333 +[2a00:12e0:101:99:20c:29ff:fe29:d03f]:8333 [2a00:1328:e101:c00::163]:8333 -[2a00:1398:4:2a03:215:5dff:fed6:1033]:8333 -[2a00:13a0:3015:1:85:14:79:26]:8333 -[2a00:1630:14::101]:8333 +[2a00:1630:10:1003:0:b19:b00b:babe]:8333 [2a00:1768:2001:27::ef6a]:8333 [2a00:1828:a004:2::666]:8333 -[2a00:1838:36:17::38cb]:8333 -[2a00:1838:36:7d::d3c6]:8333 -[2a00:1c10:2:709:58f7:e0ff:fe24:a0ba]:22220 +[2a00:1838:2a:1400:92e2:baff:fe4a:c416]:8333 [2a00:1c10:2:709::217]:22220 -[2a00:1f40:5001:100::31]:8333 -[2a00:6020:1395:1400:baf7:2d43:60b3:198b]:8333 -[2a00:7c80:0:10b::3faf]:8333 +[2a00:1f40:5001:108:5d17:7703:b0f5:4133]:8333 +[2a00:6020:15dd:ee00:c8c2:2c77:1749:35db]:8333 +[2a00:6020:b482:9200:491a:358c:d8f7:1da]:8333 +[2a00:7145:c1:1:ae29:727:2b87:f64]:5141 [2a00:8a60:e012:a00::21]:8333 -[2a00:ab00:603:84::3]:8333 -[2a00:bbe0:cc:0:62a4:4cff:fe23:7510]:8333 -[2a00:ca8:a1f:3025:f949:e442:c940:13e8]:8333 -[2a00:d2a0:a:3d00:1cdf:38bb:a7d6:c251]:8333 -[2a00:d880:11::20e]:8333 -[2a00:ec0:7207:9100:5f8f:25dd:2574:3982]:8333 -[2a00:f820:433::36]:8333 -[2a01:138:a017:b018::42]:8333 -[2a01:430:17:1::ffff:1153]:8333 +[2a00:a040:100:f3:45a5:ac0:fea3:71e1]:8333 +[2a01:488:2000:9801::d]:8333 [2a01:490:16:301::2]:8333 -[2a01:4b00:807c:1b00:cda1:c6a:2bad:2418]:8333 -[2a01:4b00:80e7:5405::1]:8333 -[2a01:4f8:192:4212::2]:8433 +[2a01:5200:6c:6162:7a61:746b:6f2e:736b]:8333 +[2a01:6380:fffe:73:4e3:b3cc:a871:36d1]:8333 [2a01:7a0:2:137c::3]:8333 -[2a01:7a7:2:1467:ec4:7aff:fee2:5690]:8333 -[2a01:7c8:d002:10f:5054:ff:fe5c:dac7]:8333 -[2a01:7c8:d002:318:5054:ff:febe:cbb1]:8333 +[2a01:7c8:aac9:c9:5054:ff:fedf:ff95]:8333 +[2a01:7c8:d001:1c1:5054:ff:feee:3e1a]:8333 [2a01:8740:1:ffc5::8c6a]:8333 -[2a01:cb00:f98:ca00:5054:ff:fed4:763d]:8333 -[2a01:cb14:cf6:bc00:21e5:f12e:32c8:145]:8333 -[2a01:d0:0:1c::245]:8333 +[2a01:cb00:d3d:7700:227:eff:fe28:c565]:8333 +[2a01:d0:0:1c::253]:8333 [2a01:d0:bef2::12]:8333 -[2a01:e35:2e40:6830:211:32ff:fea6:de3d]:8333 -[2a02:1205:c6aa:60c0:70d8:aaee:a82d:993c]:8333 -[2a02:169:502::614]:8333 -[2a02:180:1:1::5b8f:538c]:8333 -[2a02:348:62:5ef7::1]:8333 -[2a02:390:9000:0:218:7dff:fe10:be33]:8333 +[2a01:e0a:9fb:b0e0:54f8:1901:6e83:62c1]:8333 +[2a01:e0a:aa7:c8c0:9679:affa:b6e5:efc7]:8333 +[2a02:13b8:f000:101::a]:8333 +[2a02:168:6328:0:2a8:2cff:fe68:e32c]:8333 +[2a02:2780:9000:70::7]:8333 +[2a02:2e02:3900:5400:a099:e1ff:feb6:d0e]:8333 +[2a02:390:9000:0:aaa1:59ff:fe43:b57b]:8333 +[2a02:58:97:7d20::60]:8333 +[2a02:6d40:305e:601:dea6:32ff:fe44:4b25]:8333 +[2a02:7a01::91:228:45:130]:8333 [2a02:7aa0:1619::adc:8de0]:8333 -[2a02:7b40:b0df:8925::1]:8333 -[2a02:7b40:b905:37db::1]:8333 -[2a02:810d:8cbf:f3a8:96c6:91ff:fe17:ae1d]:8333 -[2a02:8389:1c0:9680:201:2eff:fe82:b3cc]:8333 -[2a02:a454:a516:1:517:928:7e0d:957c]:8333 -[2a02:af8:fab0:804:151:236:34:161]:8333 +[2a02:7b40:3e4d:998d::1]:8333 +[2a02:7b40:592f:a187::1]:8333 +[2a02:8388:e5c6:d380:201:2eff:fe82:b3cc]:8333 +[2a02:9a0:102::110]:8333 +[2a02:a311:8143:8c00::4]:8353 [2a02:af8:fab0:808:85:234:145:132]:8333 -[2a02:e00:fff0:1e2::a]:8333 -[2a03:2260:3006:d:d307:5d1d:32ca:1fe8]:8333 +[2a02:e00:fff0:506::1]:8444 +[2a02:e00:fff0:506::a]:8444 +[2a02:e98:20:1504::1]:8333 +[2a03:4000:47:f1::1]:8333 [2a03:6000:870:0:46:23:87:218]:8333 -[2a03:9da0:f6:1::2]:8333 -[2a03:c980:db:47::]:8333 -[2a03:e2c0:1ce::2]:8333 -[2a04:3544:1000:1510:706c:abff:fe6c:501c]:8333 -[2a04:52c0:101:383::2a87]:8333 -[2a04:52c0:101:3fb::4c27]:8333 -[2a04:ee41:83:50df:d908:f71d:2a86:b337]:8333 -[2a05:6d40:b94e:d100:225:90ff:fe0d:cfc2]:8333 -[2a05:e5c0:0:100:250:56ff:feb9:d6cb]:8333 -[2a05:fc87:1:6::2]:8333 -[2a05:fc87:4::8]:8333 -[2a07:5741:0:115d::1]:8333 +[2a03:7380:3015:524:afc5:d3bc:7c66:8f94]:8333 +[2a03:ec0:0:928:8c00:93ff:fe84:a007]:8333 +[2a03:ec0:0:928::701]:8333 +[2a04:2180:0:2::aa]:8333 +[2a04:52c0:101:29e::]:8333 +[2a04:52c0:103:c455::1]:8333 +[2a04:bc40:1dc3:8d::2:1001]:8333 +[2a05:1500:702:0:1c00:40ff:fe00:c]:8333 +[2a06:dd00:10:3:225:90ff:fe32:64cc]:8333 +[2a06:dd00:1:22:225:90ff:fe0e:bd48]:8333 +[2a07:6b47:100:464::9357:ffda]:8333 [2a07:a880:4601:1062:b4b4:bd2a:39d4:7acf]:51401 [2a07:abc4::1:946]:8333 -[2a07:b400:1:34c::2:1002]:8333 -[2a0a:8c41::b4]:8333 +[2a07:abc4::89:234:180:194]:8333 +[2a09:2681:102::210]:8333 [2a0a:c801:1:7::183]:8333 -[2a0b:ae40:3:4a0a::15]:8333 -[2a0f:df00:0:254::46]:8333 -[2c0f:f598:5:1:1001::1]:8333 -[2c0f:fce8:0:400:b7c::1]:8333 +[2a0b:f300:2:6::2]:8333 +[2a0c:59c0:18::a20e]:57658 +[2a0d:5600:24:a8e::a91e]:55373 +[2a0d:eb00:8005:1::13]:8333 +[2a10:4740:45:1:a013:d1ff:fe85:36e3]:8333 +[2a10:8b40:1::103]:8335 +[2c0f:f8f0:da51:0:70c3:eea9:9717:9579]:8333 # manually added 2021-03 for minimal torv3 bootstrap support 2g5qfdkn2vvcbqhzcyvyiitg4ceukybxklraxjnu7atlhd22gdwywaid.onion:8333 @@ -677,12 +671,19 @@ tddeij4qigtjr6jfnrmq6btnirmq5msgwcsdpcdjr7atftm7cxlqztid.onion:8333 vi5bnbxkleeqi6hfccjochnn65lcxlfqs4uwgmhudph554zibiusqnad.onion:8333 xqt25cobm5zqucac3634zfght72he6u3eagfyej5ellbhcdgos7t2had.onion:8333 -# manually added 2021-05 for minimal i2p bootstrap support -72l3ucjkuscrbiiepoehuwqgknyzgo7zuix5ty4puwrkyhtmnsga.b32.i2p:0 +# manually added 2021-08 for minimal i2p bootstrap support +a5qsnv3maw77mlmmzlcglu6twje6ttctd3fhpbfwcbpmewx6fczq.b32.i2p:0 +bitcornrd36coazsbzsz4pdebyzvaplmsalq4kpoljmn6cg6x5zq.b32.i2p:0 c4gfnttsuwqomiygupdqqqyy5y5emnk5c73hrfvatri67prd7vyq.b32.i2p:0 -gehtac45oaghz54ypyopim64mql7oad2bqclla74l6tfeolzmodq.b32.i2p:0 +dhtq2p76tyhi442aidb3vd2bv7yxxjuddpb2jydnnrl2ons5bhha.b32.i2p:0 h3r6bkn46qxftwja53pxiykntegfyfjqtnzbm6iv6r5mungmqgmq.b32.i2p:0 hnbbyjpxx54623l555sta7pocy3se4sdgmuebi5k6reesz5rjp6q.b32.i2p:0 -pjs7or2ctvteeo5tu4bwyrtydeuhqhvdprtujn4daxr75jpebjxa.b32.i2p:0 +jz3s4eurm5vzjresf4mwo7oni4bk36daolwxh4iqtewakylgkxmq.b32.i2p:0 +kokkmpquqlkptu5hkmzqlttsmtwxicldr4so7wqsufk6bwf32nma.b32.i2p:0 +sedndhv5vpcgdmykyi5st4yqhdxl3hpdtglta4do435wupahhx6q.b32.i2p:0 wwbw7nqr3ahkqv62cuqfwgtneekvvpnuc4i4f6yo7tpoqjswvcwa.b32.i2p:0 zsxwyo6qcn3chqzwxnseusqgsnuw3maqnztkiypyfxtya4snkoka.b32.i2p:0 + +# manually added 2022-01 for minimal cjdns bootstrap support +[fc32:17ea:e415:c3bf:9808:149d:b5a2:c9aa]:8333 +[fcc7:be49:ccd1:dc91:3125:f0da:457d:8ce]:8333 diff --git a/contrib/seeds/nodes_main_manual.txt b/contrib/seeds/nodes_main_manual.txt new file mode 100644 index 0000000000..a6e0b8763a --- /dev/null +++ b/contrib/seeds/nodes_main_manual.txt @@ -0,0 +1,43 @@ + +# manually added 2021-03 for minimal torv3 bootstrap support +2g5qfdkn2vvcbqhzcyvyiitg4ceukybxklraxjnu7atlhd22gdwywaid.onion:8333 +2jmtxvyup3ijr7u6uvu7ijtnojx4g5wodvaedivbv74w4vzntxbrhvad.onion:8333 +37m62wn7dz3uqpathpc4qfmgrbupachj52nt3jbtbjugpbu54kbud7yd.onion:8333 +5g72ppm3krkorsfopcm2bi7wlv4ohhs4u4mlseymasn7g7zhdcyjpfid.onion:8333 +7cgwjuwi5ehvcay4tazy7ya6463bndjk6xzrttw5t3xbpq4p22q6fyid.onion:8333 +7pyrpvqdhmayxggpcyqn5l3m5vqkw3qubnmgwlpya2mdo6x7pih7r7id.onion:8333 +b64xcbleqmwgq2u46bh4hegnlrzzvxntyzbmucn3zt7cssm7y4ubv3id.onion:8333 +ejxefzf5fpst4mg2rib7grksvscl7p6fvjp6agzgfc2yglxnjtxc3aid.onion:8333 +fjdyxicpm4o42xmedlwl3uvk5gmqdfs5j37wir52327vncjzvtpfv7yd.onion:8333 +fpz6r5ppsakkwypjcglz6gcnwt7ytfhxskkfhzu62tnylcknh3eq6pad.onion:8333 +fzhn4uoxfbfss7h7d6ffbn266ca432ekbbzvqtsdd55ylgxn4jucm5qd.onion:8333 +gxo5anvfnffnftfy5frkgvplq3rpga2ie3tcblo2vl754fvnhgorn5yd.onion:8333 +ifdu5qvbofrt4ekui2iyb3kbcyzcsglazhx2hn4wfskkrx2v24qxriid.onion:8333 +itz3oxsihs62muvknc237xabl5f6w6rfznfhbpayrslv2j2ubels47yd.onion:8333 +lrjh6fywjqttmlifuemq3puhvmshxzzyhoqx7uoufali57eypuenzzid.onion:8333 +m7cbpjolo662uel7rpaid46as2otcj44vvwg3gccodnvaeuwbm3anbyd.onion:8333 +opnyfyeiibe5qo5a3wbxzbb4xdiagc32bbce46owmertdknta5mi7uyd.onion:8333 +owjsdxmzla6d7lrwkbmetywqym5cyswpihciesfl5qdv2vrmwsgy4uqd.onion:8333 +q7kgmd7n7h27ds4fg7wocgniuqb3oe2zxp4nfe4skd5da6wyipibqzqd.onion:8333 +rp7k2go3s5lyj3fnj6zn62ktarlrsft2ohlsxkyd7v3e3idqyptvread.onion:8333 +sys54sv4xv3hn3sdiv3oadmzqpgyhd4u4xphv4xqk64ckvaxzm57a7yd.onion:8333 +tddeij4qigtjr6jfnrmq6btnirmq5msgwcsdpcdjr7atftm7cxlqztid.onion:8333 +vi5bnbxkleeqi6hfccjochnn65lcxlfqs4uwgmhudph554zibiusqnad.onion:8333 +xqt25cobm5zqucac3634zfght72he6u3eagfyej5ellbhcdgos7t2had.onion:8333 + +# manually added 2021-08 for minimal i2p bootstrap support +a5qsnv3maw77mlmmzlcglu6twje6ttctd3fhpbfwcbpmewx6fczq.b32.i2p:0 +bitcornrd36coazsbzsz4pdebyzvaplmsalq4kpoljmn6cg6x5zq.b32.i2p:0 +c4gfnttsuwqomiygupdqqqyy5y5emnk5c73hrfvatri67prd7vyq.b32.i2p:0 +dhtq2p76tyhi442aidb3vd2bv7yxxjuddpb2jydnnrl2ons5bhha.b32.i2p:0 +h3r6bkn46qxftwja53pxiykntegfyfjqtnzbm6iv6r5mungmqgmq.b32.i2p:0 +hnbbyjpxx54623l555sta7pocy3se4sdgmuebi5k6reesz5rjp6q.b32.i2p:0 +jz3s4eurm5vzjresf4mwo7oni4bk36daolwxh4iqtewakylgkxmq.b32.i2p:0 +kokkmpquqlkptu5hkmzqlttsmtwxicldr4so7wqsufk6bwf32nma.b32.i2p:0 +sedndhv5vpcgdmykyi5st4yqhdxl3hpdtglta4do435wupahhx6q.b32.i2p:0 +wwbw7nqr3ahkqv62cuqfwgtneekvvpnuc4i4f6yo7tpoqjswvcwa.b32.i2p:0 +zsxwyo6qcn3chqzwxnseusqgsnuw3maqnztkiypyfxtya4snkoka.b32.i2p:0 + +# manually added 2022-01 for minimal cjdns bootstrap support +[fc32:17ea:e415:c3bf:9808:149d:b5a2:c9aa]:8333 +[fcc7:be49:ccd1:dc91:3125:f0da:457d:8ce]:8333 diff --git a/contrib/signet/getcoins.py b/contrib/signet/getcoins.py index 691f0bb1b6..147d12600d 100755 --- a/contrib/signet/getcoins.py +++ b/contrib/signet/getcoins.py @@ -1,36 +1,158 @@ #!/usr/bin/env python3 -# Copyright (c) 2020 The Bitcoin Core developers +# Copyright (c) 2020-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. import argparse -import subprocess +import io import requests +import subprocess import sys +import xml.etree.ElementTree + +DEFAULT_GLOBAL_FAUCET = 'https://signetfaucet.com/claim' +DEFAULT_GLOBAL_CAPTCHA = 'https://signetfaucet.com/captcha' +GLOBAL_FIRST_BLOCK_HASH = '00000086d6b2636cb2a392d45edc4ec544a10024d30141c9adf4bfd9de533b53' + +# braille unicode block +BASE = 0x2800 +BIT_PER_PIXEL = [ + [0x01, 0x08], + [0x02, 0x10], + [0x04, 0x20], + [0x40, 0x80], +] +BW = 2 +BH = 4 + +# imagemagick or compatible fork (used for converting SVG) +CONVERT = 'convert' + +class PPMImage: + ''' + Load a PPM image (Pillow-ish API). + ''' + def __init__(self, f): + if f.readline() != b'P6\n': + raise ValueError('Invalid ppm format: header') + line = f.readline() + (width, height) = (int(x) for x in line.rstrip().split(b' ')) + if f.readline() != b'255\n': + raise ValueError('Invalid ppm format: color depth') + data = f.read(width * height * 3) + stride = width * 3 + self.size = (width, height) + self._grid = [[tuple(data[stride * y + 3 * x:stride * y + 3 * (x + 1)]) for x in range(width)] for y in range(height)] + + def getpixel(self, pos): + return self._grid[pos[1]][pos[0]] + +def print_image(img, threshold=128): + '''Print black-and-white image to terminal in braille unicode characters.''' + x_blocks = (img.size[0] + BW - 1) // BW + y_blocks = (img.size[1] + BH - 1) // BH + + for yb in range(y_blocks): + line = [] + for xb in range(x_blocks): + ch = BASE + for y in range(BH): + for x in range(BW): + try: + val = img.getpixel((xb * BW + x, yb * BH + y)) + except IndexError: + pass + else: + if val[0] < threshold: + ch |= BIT_PER_PIXEL[y][x] + line.append(chr(ch)) + print(''.join(line)) parser = argparse.ArgumentParser(description='Script to get coins from a faucet.', epilog='You may need to start with double-dash (--) when providing bitcoin-cli arguments.') parser.add_argument('-c', '--cmd', dest='cmd', default='bitcoin-cli', help='bitcoin-cli command to use') -parser.add_argument('-f', '--faucet', dest='faucet', default='https://signetfaucet.com/claim', help='URL of the faucet') +parser.add_argument('-f', '--faucet', dest='faucet', default=DEFAULT_GLOBAL_FAUCET, help='URL of the faucet') +parser.add_argument('-g', '--captcha', dest='captcha', default=DEFAULT_GLOBAL_CAPTCHA, help='URL of the faucet captcha, or empty if no captcha is needed') parser.add_argument('-a', '--addr', dest='addr', default='', help='Bitcoin address to which the faucet should send') parser.add_argument('-p', '--password', dest='password', default='', help='Faucet password, if any') +parser.add_argument('-n', '--amount', dest='amount', default='0.001', help='Amount to request (0.001-0.1, default is 0.001)') +parser.add_argument('-i', '--imagemagick', dest='imagemagick', default=CONVERT, help='Path to imagemagick convert utility') parser.add_argument('bitcoin_cli_args', nargs='*', help='Arguments to pass on to bitcoin-cli (default: -signet)') args = parser.parse_args() -if args.addr == '': - if args.bitcoin_cli_args == []: - args.bitcoin_cli_args = ['-signet'] - # get address for receiving coins - try: - args.addr = subprocess.check_output([args.cmd] + args.bitcoin_cli_args + ['getnewaddress', 'faucet', 'bech32']).strip() - except FileNotFoundError: - print('The binary', args.cmd, 'could not be found.') - exit() +if args.bitcoin_cli_args == []: + args.bitcoin_cli_args = ['-signet'] + + +def bitcoin_cli(rpc_command_and_params): + argv = [args.cmd] + args.bitcoin_cli_args + rpc_command_and_params + try: + return subprocess.check_output(argv).strip().decode() + except FileNotFoundError: + raise SystemExit(f"The binary {args.cmd} could not be found") + except subprocess.CalledProcessError: + cmdline = ' '.join(argv) + raise SystemExit(f"-----\nError while calling {cmdline} (see output above).") + + +if args.faucet.lower() == DEFAULT_GLOBAL_FAUCET: + # Get the hash of the block at height 1 of the currently active signet chain + curr_signet_hash = bitcoin_cli(['getblockhash', '1']) + if curr_signet_hash != GLOBAL_FIRST_BLOCK_HASH: + raise SystemExit('The global faucet cannot be used with a custom Signet network. Please use the global signet or setup your custom faucet to use this functionality.\n') +else: + # For custom faucets, don't request captcha by default. + if args.captcha == DEFAULT_GLOBAL_CAPTCHA: + args.captcha = '' + +if args.addr == '': + # get address for receiving coins + args.addr = bitcoin_cli(['getnewaddress', 'faucet', 'bech32']) + +data = {'address': args.addr, 'password': args.password, 'amount': args.amount} + +# Store cookies +# for debugging: print(session.cookies.get_dict()) +session = requests.Session() + +if args.captcha != '': # Retrieve a captcha + try: + res = session.get(args.captcha) + res.raise_for_status() + except requests.exceptions.RequestException as e: + raise SystemExit(f"Unexpected error when contacting faucet: {e}") + + # Size limitation + svg = xml.etree.ElementTree.fromstring(res.content) + if svg.attrib.get('width') != '150' or svg.attrib.get('height') != '50': + raise SystemExit("Captcha size doesn't match expected dimensions 150x50") + + # Convert SVG image to PPM, and load it + try: + rv = subprocess.run([args.imagemagick, 'svg:-', '-depth', '8', 'ppm:-'], input=res.content, check=True, capture_output=True) + except FileNotFoundError: + raise SystemExit(f"The binary {args.imagemagick} could not be found. Please make sure ImageMagick (or a compatible fork) is installed and that the correct path is specified.") + + img = PPMImage(io.BytesIO(rv.stdout)) + + # Terminal interaction + print_image(img) + print(f"Captcha from URL {args.captcha}") + data['captcha'] = input('Enter captcha: ') -data = {'address': args.addr, 'password': args.password} try: - res = requests.post(args.faucet, data=data) + res = session.post(args.faucet, data=data) except: - print('Unexpected error when contacting faucet:', sys.exc_info()[0]) - exit() -print(res.text) + raise SystemExit(f"Unexpected error when contacting faucet: {sys.exc_info()[0]}") + +# Display the output as per the returned status code +if res: + # When the return code is in between 200 and 400 i.e. successful + print(res.text) +elif res.status_code == 404: + print('The specified faucet URL does not exist. Please check for any server issues/typo.') +elif res.status_code == 429: + print('The script does not allow for repeated transactions as the global faucet is rate-limitied to 1 request/IP/day. You can access the faucet website to get more coins manually') +else: + print(f'Returned Error Code {res.status_code}\n{res.text}\n') + print('Please check the provided arguments for their validity and/or any possible typo.') diff --git a/contrib/signet/miner b/contrib/signet/miner index 78e1fa5ecd..012bd6cc31 100755 --- a/contrib/signet/miner +++ b/contrib/signet/miner @@ -15,7 +15,6 @@ import sys import time import subprocess -from binascii import unhexlify from io import BytesIO PATH_BASE_CONTRIB_SIGNET = os.path.abspath(os.path.dirname(os.path.realpath(__file__))) @@ -202,7 +201,7 @@ def finish_block(block, signet_solution, grind_cmd): def generate_psbt(tmpl, reward_spk, *, blocktime=None): signet_spk = tmpl["signet_challenge"] - signet_spk_bin = unhexlify(signet_spk) + signet_spk_bin = bytes.fromhex(signet_spk) cbtx = create_coinbase(height=tmpl["height"], value=tmpl["coinbasevalue"], spk=reward_spk) cbtx.vin[0].nSequence = 2**32-2 @@ -258,7 +257,7 @@ def get_reward_addr_spk(args, height): return args.address, args.reward_spk reward_addr = get_reward_address(args, height) - reward_spk = unhexlify(json.loads(args.bcli("getaddressinfo", reward_addr))["scriptPubKey"]) + reward_spk = bytes.fromhex(json.loads(args.bcli("getaddressinfo", reward_addr))["scriptPubKey"]) if args.address is not None: # will always be the same, so cache args.reward_spk = reward_spk diff --git a/contrib/testgen/gen_key_io_test_vectors.py b/contrib/testgen/gen_key_io_test_vectors.py index 74918cfb04..eafaaaeceb 100755 --- a/contrib/testgen/gen_key_io_test_vectors.py +++ b/contrib/testgen/gen_key_io_test_vectors.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2012-2020 The Bitcoin Core developers +# Copyright (c) 2012-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. ''' diff --git a/contrib/tracing/README.md b/contrib/tracing/README.md new file mode 100644 index 0000000000..a409a23ef8 --- /dev/null +++ b/contrib/tracing/README.md @@ -0,0 +1,288 @@ +Example scripts for User-space, Statically Defined Tracing (USDT) +================================================================= + +This directory contains scripts showcasing User-space, Statically Defined +Tracing (USDT) support for Bitcoin Core on Linux using. For more information on +USDT support in Bitcoin Core see the [USDT documentation]. + +[USDT documentation]: ../../doc/tracing.md + + +Examples for the two main eBPF front-ends, [bpftrace] and +[BPF Compiler Collection (BCC)], with support for USDT, are listed. BCC is used +for complex tools and daemons and `bpftrace` is preferred for one-liners and +shorter scripts. + +[bpftrace]: https://github.com/iovisor/bpftrace +[BPF Compiler Collection (BCC)]: https://github.com/iovisor/bcc + + +To develop and run bpftrace and BCC scripts you need to install the +corresponding packages. See [installing bpftrace] and [installing BCC] for more +information. For development there exist a [bpftrace Reference Guide], a +[BCC Reference Guide], and a [bcc Python Developer Tutorial]. + +[installing bpftrace]: https://github.com/iovisor/bpftrace/blob/master/INSTALL.md +[installing BCC]: https://github.com/iovisor/bcc/blob/master/INSTALL.md +[bpftrace Reference Guide]: https://github.com/iovisor/bpftrace/blob/master/docs/reference_guide.md +[BCC Reference Guide]: https://github.com/iovisor/bcc/blob/master/docs/reference_guide.md +[bcc Python Developer Tutorial]: https://github.com/iovisor/bcc/blob/master/docs/tutorial_bcc_python_developer.md + +## Examples + +The bpftrace examples contain a relative path to the `bitcoind` binary. By +default, the scripts should be run from the repository-root and assume a +self-compiled `bitcoind` binary. The paths in the examples can be changed, for +example, to point to release builds if needed. See the +[Bitcoin Core USDT documentation] on how to list available tracepoints in your +`bitcoind` binary. + +[Bitcoin Core USDT documentation]: ../../doc/tracing.md#listing-available-tracepoints + +**WARNING: eBPF programs require root privileges to be loaded into a Linux +kernel VM. This means the bpftrace and BCC examples must be executed with root +privileges. Make sure to carefully review any scripts that you run with root +privileges first!** + +### log_p2p_traffic.bt + +A bpftrace script logging information about inbound and outbound P2P network +messages. Based on the `net:inbound_message` and `net:outbound_message` +tracepoints. + +By default, `bpftrace` limits strings to 64 bytes due to the limited stack size +in the eBPF VM. For example, Tor v3 addresses exceed the string size limit which +results in the port being cut off during logging. The string size limit can be +increased with the `BPFTRACE_STRLEN` environment variable (`BPFTRACE_STRLEN=70` +works fine). + +``` +$ bpftrace contrib/tracing/log_p2p_traffic.bt +``` + +Output +``` +outbound 'ping' msg to peer 11 (outbound-full-relay, [2a02:b10c:f747:1:ef:fake:ipv6:addr]:8333) with 8 bytes +inbound 'pong' msg from peer 11 (outbound-full-relay, [2a02:b10c:f747:1:ef:fake:ipv6:addr]:8333) with 8 bytes +inbound 'inv' msg from peer 16 (outbound-full-relay, XX.XX.XXX.121:8333) with 37 bytes +outbound 'getdata' msg to peer 16 (outbound-full-relay, XX.XX.XXX.121:8333) with 37 bytes +inbound 'tx' msg from peer 16 (outbound-full-relay, XX.XX.XXX.121:8333) with 222 bytes +outbound 'inv' msg to peer 9 (outbound-full-relay, faketorv3addressa2ufa6odvoi3s77j4uegey0xb10csyfyve2t33curbyd.onion:8333) with 37 bytes +outbound 'inv' msg to peer 7 (outbound-full-relay, XX.XX.XXX.242:8333) with 37 bytes +… +``` + +### p2p_monitor.py + +A BCC Python script using curses for an interactive P2P message monitor. Based +on the `net:inbound_message` and `net:outbound_message` tracepoints. + +Inbound and outbound traffic is listed for each peer together with information +about the connection. Peers can be selected individually to view recent P2P +messages. + +``` +$ python3 contrib/tracing/p2p_monitor.py ./src/bitcoind +``` + +Lists selectable peers and traffic and connection information. +``` + P2P Message Monitor + Navigate with UP/DOWN or J/K and select a peer with ENTER or SPACE to see individual P2P messages + + PEER OUTBOUND INBOUND TYPE ADDR + 0 46 398 byte 61 1407590 byte block-relay-only XX.XX.XXX.196:8333 + 11 1156 253570 byte 3431 2394924 byte outbound-full-relay XXX.X.XX.179:8333 + 13 3425 1809620 byte 1236 305458 byte inbound XXX.X.X.X:60380 + 16 1046 241633 byte 1589 1199220 byte outbound-full-relay 4faketorv2pbfu7x.onion:8333 + 19 577 181679 byte 390 148951 byte outbound-full-relay kfake4vctorjv2o2.onion:8333 + 20 11 1248 byte 13 1283 byte block-relay-only [2600:fake:64d9:b10c:4436:aaaa:fe:bb]:8333 + 21 11 1248 byte 13 1299 byte block-relay-only XX.XXX.X.155:8333 + 22 5 103 byte 1 102 byte feeler XX.XX.XXX.173:8333 + 23 11 1248 byte 12 1255 byte block-relay-only XX.XXX.XXX.220:8333 + 24 3 103 byte 1 102 byte feeler XXX.XXX.XXX.64:8333 +… +``` + +Showing recent P2P messages between our node and a selected peer. + +``` + ---------------------------------------------------------------------- + | PEER 16 (4faketorv2pbfu7x.onion:8333) | + | OUR NODE outbound-full-relay PEER | + | <--- sendcmpct (9 bytes) | + | inv (37 byte) ---> | + | <--- ping (8 bytes) | + | pong (8 byte) ---> | + | inv (37 byte) ---> | + | <--- addr (31 bytes) | + | inv (37 byte) ---> | + | <--- getheaders (1029 bytes) | + | headers (1 byte) ---> | + | <--- feefilter (8 bytes) | + | <--- pong (8 bytes) | + | <--- headers (82 bytes) | + | <--- addr (30003 bytes) | + | inv (1261 byte) ---> | + | … | + +``` + +### log_raw_p2p_msgs.py + +A BCC Python script showcasing eBPF and USDT limitations when passing data +larger than about 32kb. Based on the `net:inbound_message` and +`net:outbound_message` tracepoints. + +Bitcoin P2P messages can be larger than 32kb (e.g. `tx`, `block`, ...). The +eBPF VM's stack is limited to 512 bytes, and we can't allocate more than about +32kb for a P2P message in the eBPF VM. The **message data is cut off** when the +message is larger than MAX_MSG_DATA_LENGTH (see script). This can be detected +in user-space by comparing the data length to the message length variable. The +message is cut off when the data length is smaller than the message length. +A warning is included with the printed message data. + +Data is submitted to user-space (i.e. to this script) via a ring buffer. The +throughput of the ring buffer is limited. Each p2p_message is about 32kb in +size. In- or outbound messages submitted to the ring buffer in rapid +succession fill the ring buffer faster than it can be read. Some messages are +lost. BCC prints: `Possibly lost 2 samples` on lost messages. + + +``` +$ python3 contrib/tracing/log_raw_p2p_msgs.py ./src/bitcoind +``` + +``` +Logging raw P2P messages. +Messages larger that about 32kb will be cut off! +Some messages might be lost! + outbound msg 'inv' from peer 4 (outbound-full-relay, XX.XXX.XX.4:8333) with 253 bytes: 0705000000be2245c8f844c9f763748e1a7… +… +Warning: incomplete message (only 32568 out of 53552 bytes)! inbound msg 'tx' from peer 32 (outbound-full-relay, XX.XXX.XXX.43:8333) with 53552 bytes: 020000000001fd3c01939c85ad6756ed9fc… +… +Possibly lost 2 samples +``` + +### connectblock_benchmark.bt + +A `bpftrace` script to benchmark the `ConnectBlock()` function during, for +example, a blockchain re-index. Based on the `validation:block_connected` USDT +tracepoint. + +The script takes three positional arguments. The first two arguments, the start, +and end height indicate between which blocks the benchmark should be run. The +third acts as a duration threshold in milliseconds. When the `ConnectBlock()` +function takes longer than the threshold, information about the block, is +printed. For more details, see the header comment in the script. + +The following command can be used to benchmark, for example, `ConnectBlock()` +between height 20000 and 38000 on SigNet while logging all blocks that take +longer than 25ms to connect. + +``` +$ bpftrace contrib/tracing/connectblock_benchmark.bt 20000 38000 25 +``` + +In a different terminal, starting Bitcoin Core in SigNet mode and with +re-indexing enabled. + +``` +$ ./src/bitcoind -signet -reindex +``` + +This produces the following output. +``` +Attaching 5 probes... +ConnectBlock Benchmark between height 20000 and 38000 inclusive +Logging blocks taking longer than 25 ms to connect. +Starting Connect Block Benchmark between height 20000 and 38000. +BENCH 39 blk/s 59 tx/s 59 inputs/s 20 sigops/s (height 20038) +Block 20492 (000000f555653bb05e2f3c6e79925e01a20dd57033f4dc7c354b46e34735d32b) 20 tx 2319 ins 2318 sigops took 38 ms +BENCH 1840 blk/s 2117 tx/s 4478 inputs/s 2471 sigops/s (height 21879) +BENCH 1816 blk/s 4972 tx/s 4982 inputs/s 125 sigops/s (height 23695) +BENCH 2095 blk/s 2890 tx/s 2910 inputs/s 152 sigops/s (height 25790) +BENCH 1684 blk/s 3979 tx/s 4053 inputs/s 288 sigops/s (height 27474) +BENCH 1155 blk/s 3216 tx/s 3252 inputs/s 115 sigops/s (height 28629) +BENCH 1797 blk/s 2488 tx/s 2503 inputs/s 111 sigops/s (height 30426) +BENCH 1849 blk/s 6318 tx/s 6569 inputs/s 12189 sigops/s (height 32275) +BENCH 946 blk/s 20209 tx/s 20775 inputs/s 83809 sigops/s (height 33221) +Block 33406 (0000002adfe4a15cfcd53bd890a89bbae836e5bb7f38bac566f61ad4548c87f6) 25 tx 2045 ins 2090 sigops took 29 ms +Block 33687 (00000073231307a9828e5607ceb8156b402efe56747271a4442e75eb5b77cd36) 52 tx 1797 ins 1826 sigops took 26 ms +BENCH 582 blk/s 21581 tx/s 27673 inputs/s 60345 sigops/s (height 33803) +BENCH 1035 blk/s 19735 tx/s 19776 inputs/s 51355 sigops/s (height 34838) +Block 35625 (0000006b00b347390c4768ea9df2655e9ff4b120f29d78594a2a702f8a02c997) 20 tx 3374 ins 3371 sigops took 49 ms +BENCH 887 blk/s 17857 tx/s 22191 inputs/s 24404 sigops/s (height 35725) +Block 35937 (000000d816d13d6e39b471cd4368db60463a764ba1f29168606b04a22b81ea57) 75 tx 3943 ins 3940 sigops took 61 ms +BENCH 823 blk/s 16298 tx/s 21031 inputs/s 18440 sigops/s (height 36548) +Block 36583 (000000c3e260556dbf42968aae3f904dba8b8c1ff96a6f6e3aa5365d2e3ad317) 24 tx 2198 ins 2194 sigops took 34 ms +Block 36700 (000000b3b173de9e65a3cfa738d976af6347aaf83fa17ab3f2a4d2ede3ddfac4) 73 tx 1615 ins 1611 sigops took 31 ms +Block 36832 (0000007859578c02c1ac37dabd1b9ec19b98f350b56935f5dd3a41e9f79f836e) 34 tx 1440 ins 1436 sigops took 26 ms +BENCH 613 blk/s 16718 tx/s 25074 inputs/s 23022 sigops/s (height 37161) +Block 37870 (000000f5c1086291ba2d943fb0c3bc82e71c5ee341ee117681d1456fbf6c6c38) 25 tx 1517 ins 1514 sigops took 29 ms +BENCH 811 blk/s 16031 tx/s 20921 inputs/s 18696 sigops/s (height 37972) + +Took 14055 ms to connect the blocks between height 20000 and 38000. + +Histogram of block connection times in milliseconds (ms). +@durations: +[0] 16838 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@| +[1] 882 |@@ | +[2, 4) 236 | | +[4, 8) 23 | | +[8, 16) 9 | | +[16, 32) 9 | | +[32, 64) 4 | | +``` + +### log_utxocache_flush.py + +A BCC Python script to log the UTXO cache flushes. Based on the +`utxocache:flush` tracepoint. + +```bash +$ python3 contrib/tracing/log_utxocache_flush.py ./src/bitcoind +``` + +``` +Logging utxocache flushes. Ctrl-C to end... +Duration (µs) Mode Coins Count Memory Usage Prune +730451 IF_NEEDED 22990 3323.54 kB True +637657 ALWAYS 122320 17124.80 kB False +81349 ALWAYS 0 1383.49 kB False +``` + +### log_utxos.bt + +A `bpftrace` script to log information about the coins that are added, spent, or +uncached from the UTXO set. Based on the `utxocache:add`, `utxocache:spend` and +`utxocache:uncache` tracepoints. + +```bash +$ bpftrace contrib/tracing/log_utxos.bt +``` + +This should produce an output similar to the following. If you see bpftrace +warnings like `Lost 24 events`, the eBPF perf ring-buffer is filled faster +than it is being read. You can increase the ring-buffer size by setting the +ENV variable `BPFTRACE_PERF_RB_PAGES` (default 64) at a cost of higher +memory usage. See the [bpftrace reference guide] for more information. + +[bpftrace reference guide]: https://github.com/iovisor/bpftrace/blob/master/docs/reference_guide.md#98-bpftrace_perf_rb_pages + +```bash +Attaching 4 probes... +OP Outpoint Value Height Coinbase +Added 6ba9ad857e1ef2eb2a2c94f06813c414c7ab273e3d6bd7ad64e000315a887e7c:1 10000 2094512 No +Spent fa7dc4db56637a151f6649d8f26732956d1c5424c82aae400a83d02b2cc2c87b:0 182264897 2094512 No +Added eeb2f099b1af6a2a12e6ddd2eeb16fc5968582241d7f08ba202d28b60ac264c7:0 10000 2094512 No +Added eeb2f099b1af6a2a12e6ddd2eeb16fc5968582241d7f08ba202d28b60ac264c7:1 182254756 2094512 No +Added a0c7f4ec9cccef2d89672a624a4e6c8237a17572efdd4679eea9e9ee70d2db04:0 10072679 2094513 Yes +Spent 25e0df5cc1aeb1b78e6056bf403e5e8b7e41f138060ca0a50a50134df0549a5e:2 540 2094508 No +Spent 42f383c04e09c26a2378272ec33aa0c1bf4883ca5ab739e8b7e06be5a5787d61:1 3848399 2007724 No +Added f85e3b4b89270863a389395cc9a4123e417ab19384cef96533c6649abd6b0561:0 3788399 2094513 No +Added f85e3b4b89270863a389395cc9a4123e417ab19384cef96533c6649abd6b0561:2 540 2094513 No +Spent a05880b8c77971ed0b9f73062c7c4cdb0ff3856ab14cbf8bc481ed571cd34b83:1 5591281046 2094511 No +Added eb689865f7d957938978d6207918748f74e6aa074f47874724327089445b0960:0 5589696005 2094513 No +Added eb689865f7d957938978d6207918748f74e6aa074f47874724327089445b0960:1 1565556 2094513 No +``` diff --git a/contrib/tracing/connectblock_benchmark.bt b/contrib/tracing/connectblock_benchmark.bt new file mode 100755 index 0000000000..6e7a98ef07 --- /dev/null +++ b/contrib/tracing/connectblock_benchmark.bt @@ -0,0 +1,156 @@ +#!/usr/bin/env bpftrace + +/* + + USAGE: + + bpftrace contrib/tracing/connectblock_benchmark.bt + + - sets the height at which the benchmark should start. Setting + the start height to 0 starts the benchmark immediately, even before the + first block is connected. + - sets the height after which the benchmark should end. Setting + the end height to 0 disables the benchmark. The script only logs blocks + over . + - Threshold + + This script requires a 'bitcoind' binary compiled with eBPF support and the + 'validation:block_connected' USDT. By default, it's assumed that 'bitcoind' is + located in './src/bitcoind'. This can be modified in the script below. + + EXAMPLES: + + bpftrace contrib/tracing/connectblock_benchmark.bt 300000 680000 1000 + + When run together 'bitcoind -reindex', this benchmarks the time it takes to + connect the blocks between height 300.000 and 680.000 (inclusive) and prints + details about all blocks that take longer than 1000ms to connect. Prints a + histogram with block connection times when the benchmark is finished. + + + bpftrace contrib/tracing/connectblock_benchmark.bt 0 0 500 + + When running together 'bitcoind', all newly connected blocks that + take longer than 500ms to connect are logged. A histogram with block + connection times is shown when the script is terminated. + +*/ + +BEGIN +{ + $start_height = $1; + $end_height = $2; + $logging_threshold_ms = $3; + + if ($end_height < $start_height) { + printf("Error: start height (%d) larger than end height (%d)!\n", $start_height, $end_height); + exit(); + } + + if ($end_height > 0) { + printf("ConnectBlock benchmark between height %d and %d inclusive\n", $start_height, $end_height); + } else { + printf("ConnectBlock logging starting at height %d\n", $start_height); + } + + if ($logging_threshold_ms > 0) { + printf("Logging blocks taking longer than %d ms to connect.\n", $3); + } + + if ($start_height == 0) { + @start = nsecs; + } +} + +/* + Attaches to the 'validation:block_connected' USDT and collects stats when the + connected block is between the start and end height (or the end height is + unset). +*/ +usdt:./src/bitcoind:validation:block_connected /arg1 >= $1 && (arg1 <= $2 || $2 == 0 )/ +{ + $height = arg1; + $transactions = arg2; + $inputs = arg3; + $sigops = arg4; + $duration = (uint64) arg5; + + @height = $height; + + @blocks = @blocks + 1; + @transactions = @transactions + $transactions; + @inputs = @inputs + $inputs; + @sigops = @sigops + $sigops; + + @durations = hist($duration / 1000); + + if ($height == $1 && $height != 0) { + @start = nsecs; + printf("Starting Connect Block Benchmark between height %d and %d.\n", $1, $2); + } + + if ($2 > 0 && $height >= $2) { + @end = nsecs; + $duration = @end - @start; + printf("\nTook %d ms to connect the blocks between height %d and %d.\n", $duration / 1000000, $1, $2); + exit(); + } +} + +/* + Attaches to the 'validation:block_connected' USDT and logs information about + blocks where the time it took to connect the block is above the + . +*/ +usdt:./src/bitcoind:validation:block_connected / (uint64) arg5 / 1000> $3 / +{ + $hash = arg0; + $height = (int32) arg1; + $transactions = (uint64) arg2; + $inputs = (int32) arg3; + $sigops = (int64) arg4; + $duration = (int64) arg5; + + + printf("Block %d (", $height); + /* Prints each byte of the block hash as hex in big-endian (the block-explorer format) */ + $p = $hash + 31; + unroll(32) { + $b = *(uint8*)$p; + printf("%02x", $b); + $p -= 1; + } + printf(") %4d tx %5d ins %5d sigops took %4d ms\n", $transactions, $inputs, $sigops, (uint64) $duration / 1000); +} + + +/* + Prints stats about the blocks, transactions, inputs, and sigops processed in + the last second (if any). +*/ +interval:s:1 { + if (@blocks > 0) { + printf("BENCH %4d blk/s %6d tx/s %7d inputs/s %8d sigops/s (height %d)\n", @blocks, @transactions, @inputs, @sigops, @height); + + zero(@blocks); + zero(@transactions); + zero(@inputs); + zero(@sigops); + } +} + +END +{ + printf("\nHistogram of block connection times in milliseconds (ms).\n"); + print(@durations); + + clear(@durations); + clear(@blocks); + clear(@transactions); + clear(@inputs); + clear(@sigops); + clear(@height); + clear(@start); + clear(@end); +} + diff --git a/contrib/tracing/log_p2p_traffic.bt b/contrib/tracing/log_p2p_traffic.bt new file mode 100755 index 0000000000..f62956aa5e --- /dev/null +++ b/contrib/tracing/log_p2p_traffic.bt @@ -0,0 +1,28 @@ +#!/usr/bin/env bpftrace + +BEGIN +{ + printf("Logging P2P traffic\n") +} + +usdt:./src/bitcoind:net:inbound_message +{ + $peer_id = (int64) arg0; + $peer_addr = str(arg1); + $peer_type = str(arg2); + $msg_type = str(arg3); + $msg_len = arg4; + printf("inbound '%s' msg from peer %d (%s, %s) with %d bytes\n", $msg_type, $peer_id, $peer_type, $peer_addr, $msg_len); +} + +usdt:./src/bitcoind:net:outbound_message +{ + $peer_id = (int64) arg0; + $peer_addr = str(arg1); + $peer_type = str(arg2); + $msg_type = str(arg3); + $msg_len = arg4; + + printf("outbound '%s' msg to peer %d (%s, %s) with %d bytes\n", $msg_type, $peer_id, $peer_type, $peer_addr, $msg_len); +} + diff --git a/contrib/tracing/log_raw_p2p_msgs.py b/contrib/tracing/log_raw_p2p_msgs.py new file mode 100755 index 0000000000..c0ab704106 --- /dev/null +++ b/contrib/tracing/log_raw_p2p_msgs.py @@ -0,0 +1,183 @@ +#!/usr/bin/env python3 +# Copyright (c) 2021 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +""" Demonstration of eBPF limitations and the effect on USDT with the + net:inbound_message and net:outbound_message tracepoints. """ + +# This script shows a limitation of eBPF when data larger than 32kb is passed to +# user-space. It uses BCC (https://github.com/iovisor/bcc) to load a sandboxed +# eBPF program into the Linux kernel (root privileges are required). The eBPF +# program attaches to two statically defined tracepoints. The tracepoint +# 'net:inbound_message' is called when a new P2P message is received, and +# 'net:outbound_message' is called on outbound P2P messages. The eBPF program +# submits the P2P messages to this script via a BPF ring buffer. The submitted +# messages are printed. + +# eBPF Limitations: +# +# Bitcoin P2P messages can be larger than 32kb (e.g. tx, block, ...). The eBPF +# VM's stack is limited to 512 bytes, and we can't allocate more than about 32kb +# for a P2P message in the eBPF VM. The message data is cut off when the message +# is larger than MAX_MSG_DATA_LENGTH (see definition below). This can be detected +# in user-space by comparing the data length to the message length variable. The +# message is cut off when the data length is smaller than the message length. +# A warning is included with the printed message data. +# +# Data is submitted to user-space (i.e. to this script) via a ring buffer. The +# throughput of the ring buffer is limited. Each p2p_message is about 32kb in +# size. In- or outbound messages submitted to the ring buffer in rapid +# succession fill the ring buffer faster than it can be read. Some messages are +# lost. +# +# BCC prints: "Possibly lost 2 samples" on lost messages. + +import sys +from bcc import BPF, USDT + +# BCC: The C program to be compiled to an eBPF program (by BCC) and loaded into +# a sandboxed Linux kernel VM. +program = """ +#include + +#define MIN(a,b) ({ __typeof__ (a) _a = (a); __typeof__ (b) _b = (b); _a < _b ? _a : _b; }) + +// Maximum possible allocation size +// from include/linux/percpu.h in the Linux kernel +#define PCPU_MIN_UNIT_SIZE (32 << 10) + +// Tor v3 addresses are 62 chars + 6 chars for the port (':12345'). +#define MAX_PEER_ADDR_LENGTH 62 + 6 +#define MAX_PEER_CONN_TYPE_LENGTH 20 +#define MAX_MSG_TYPE_LENGTH 20 +#define MAX_MSG_DATA_LENGTH PCPU_MIN_UNIT_SIZE - 200 + +struct p2p_message +{ + u64 peer_id; + char peer_addr[MAX_PEER_ADDR_LENGTH]; + char peer_conn_type[MAX_PEER_CONN_TYPE_LENGTH]; + char msg_type[MAX_MSG_TYPE_LENGTH]; + u64 msg_size; + u8 msg[MAX_MSG_DATA_LENGTH]; +}; + +// We can't store the p2p_message struct on the eBPF stack as it is limited to +// 512 bytes and P2P message can be bigger than 512 bytes. However, we can use +// an BPF-array with a length of 1 to allocate up to 32768 bytes (this is +// defined by PCPU_MIN_UNIT_SIZE in include/linux/percpu.h in the Linux kernel). +// Also see https://github.com/iovisor/bcc/issues/2306 +BPF_ARRAY(msg_arr, struct p2p_message, 1); + +// Two BPF perf buffers for pushing data (here P2P messages) to user-space. +BPF_PERF_OUTPUT(inbound_messages); +BPF_PERF_OUTPUT(outbound_messages); + +int trace_inbound_message(struct pt_regs *ctx) { + int idx = 0; + struct p2p_message *msg = msg_arr.lookup(&idx); + + // lookup() does not return a NULL pointer. However, the BPF verifier + // requires an explicit check that that the `msg` pointer isn't a NULL + // pointer. See https://github.com/iovisor/bcc/issues/2595 + if (msg == NULL) return 1; + + bpf_usdt_readarg(1, ctx, &msg->peer_id); + bpf_usdt_readarg_p(2, ctx, &msg->peer_addr, MAX_PEER_ADDR_LENGTH); + bpf_usdt_readarg_p(3, ctx, &msg->peer_conn_type, MAX_PEER_CONN_TYPE_LENGTH); + bpf_usdt_readarg_p(4, ctx, &msg->msg_type, MAX_MSG_TYPE_LENGTH); + bpf_usdt_readarg(5, ctx, &msg->msg_size); + bpf_usdt_readarg_p(6, ctx, &msg->msg, MIN(msg->msg_size, MAX_MSG_DATA_LENGTH)); + + inbound_messages.perf_submit(ctx, msg, sizeof(*msg)); + return 0; +}; + +int trace_outbound_message(struct pt_regs *ctx) { + int idx = 0; + struct p2p_message *msg = msg_arr.lookup(&idx); + + // lookup() does not return a NULL pointer. However, the BPF verifier + // requires an explicit check that that the `msg` pointer isn't a NULL + // pointer. See https://github.com/iovisor/bcc/issues/2595 + if (msg == NULL) return 1; + + bpf_usdt_readarg(1, ctx, &msg->peer_id); + bpf_usdt_readarg_p(2, ctx, &msg->peer_addr, MAX_PEER_ADDR_LENGTH); + bpf_usdt_readarg_p(3, ctx, &msg->peer_conn_type, MAX_PEER_CONN_TYPE_LENGTH); + bpf_usdt_readarg_p(4, ctx, &msg->msg_type, MAX_MSG_TYPE_LENGTH); + bpf_usdt_readarg(5, ctx, &msg->msg_size); + bpf_usdt_readarg_p(6, ctx, &msg->msg, MIN(msg->msg_size, MAX_MSG_DATA_LENGTH)); + + outbound_messages.perf_submit(ctx, msg, sizeof(*msg)); + return 0; +}; +""" + + +def print_message(event, inbound): + print(f"%s %s msg '%s' from peer %d (%s, %s) with %d bytes: %s" % + ( + f"Warning: incomplete message (only %d out of %d bytes)!" % ( + len(event.msg), event.msg_size) if len(event.msg) < event.msg_size else "", + "inbound" if inbound else "outbound", + event.msg_type.decode("utf-8"), + event.peer_id, + event.peer_conn_type.decode("utf-8"), + event.peer_addr.decode("utf-8"), + event.msg_size, + bytes(event.msg[:event.msg_size]).hex(), + ) + ) + + +def main(bitcoind_path): + bitcoind_with_usdts = USDT(path=str(bitcoind_path)) + + # attaching the trace functions defined in the BPF program to the tracepoints + bitcoind_with_usdts.enable_probe( + probe="inbound_message", fn_name="trace_inbound_message") + bitcoind_with_usdts.enable_probe( + probe="outbound_message", fn_name="trace_outbound_message") + bpf = BPF(text=program, usdt_contexts=[bitcoind_with_usdts]) + + # BCC: perf buffer handle function for inbound_messages + def handle_inbound(_, data, size): + """ Inbound message handler. + + Called each time a message is submitted to the inbound_messages BPF table.""" + + event = bpf["inbound_messages"].event(data) + print_message(event, True) + + # BCC: perf buffer handle function for outbound_messages + + def handle_outbound(_, data, size): + """ Outbound message handler. + + Called each time a message is submitted to the outbound_messages BPF table.""" + + event = bpf["outbound_messages"].event(data) + print_message(event, False) + + # BCC: add handlers to the inbound and outbound perf buffers + bpf["inbound_messages"].open_perf_buffer(handle_inbound) + bpf["outbound_messages"].open_perf_buffer(handle_outbound) + + print("Logging raw P2P messages.") + print("Messages larger that about 32kb will be cut off!") + print("Some messages might be lost!") + while True: + try: + bpf.perf_buffer_poll() + except KeyboardInterrupt: + exit() + + +if __name__ == "__main__": + if len(sys.argv) < 2: + print("USAGE:", sys.argv[0], "path/to/bitcoind") + exit() + path = sys.argv[1] + main(path) diff --git a/contrib/tracing/log_utxocache_flush.py b/contrib/tracing/log_utxocache_flush.py new file mode 100755 index 0000000000..8c073bea0d --- /dev/null +++ b/contrib/tracing/log_utxocache_flush.py @@ -0,0 +1,107 @@ +#!/usr/bin/env python3 +# Copyright (c) 2021 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +import sys +import ctypes +from bcc import BPF, USDT + +"""Example logging Bitcoin Core utxo set cache flushes utilizing + the utxocache:flush tracepoint.""" + +# USAGE: ./contrib/tracing/log_utxocache_flush.py path/to/bitcoind + +# BCC: The C program to be compiled to an eBPF program (by BCC) and loaded into +# a sandboxed Linux kernel VM. +program = """ +# include + +struct data_t +{ + u64 duration; + u32 mode; + u64 coins_count; + u64 coins_mem_usage; + bool is_flush_for_prune; +}; + +// BPF perf buffer to push the data to user space. +BPF_PERF_OUTPUT(flush); + +int trace_flush(struct pt_regs *ctx) { + struct data_t data = {}; + bpf_usdt_readarg(1, ctx, &data.duration); + bpf_usdt_readarg(2, ctx, &data.mode); + bpf_usdt_readarg(3, ctx, &data.coins_count); + bpf_usdt_readarg(4, ctx, &data.coins_mem_usage); + bpf_usdt_readarg(5, ctx, &data.is_flush_for_prune); + flush.perf_submit(ctx, &data, sizeof(data)); + return 0; +} +""" + +FLUSH_MODES = [ + 'NONE', + 'IF_NEEDED', + 'PERIODIC', + 'ALWAYS' +] + + +class Data(ctypes.Structure): + # define output data structure corresponding to struct data_t + _fields_ = [ + ("duration", ctypes.c_uint64), + ("mode", ctypes.c_uint32), + ("coins_count", ctypes.c_uint64), + ("coins_mem_usage", ctypes.c_uint64), + ("is_flush_for_prune", ctypes.c_bool) + ] + + +def print_event(event): + print("%-15d %-10s %-15d %-15s %-8s" % ( + event.duration, + FLUSH_MODES[event.mode], + event.coins_count, + "%.2f kB" % (event.coins_mem_usage/1000), + event.is_flush_for_prune + )) + + +def main(bitcoind_path): + bitcoind_with_usdts = USDT(path=str(bitcoind_path)) + + # attaching the trace functions defined in the BPF program + # to the tracepoints + bitcoind_with_usdts.enable_probe( + probe="flush", fn_name="trace_flush") + b = BPF(text=program, usdt_contexts=[bitcoind_with_usdts]) + + def handle_flush(_, data, size): + """ Coins Flush handler. + Called each time coin caches and indexes are flushed.""" + event = ctypes.cast(data, ctypes.POINTER(Data)).contents + print_event(event) + + b["flush"].open_perf_buffer(handle_flush) + print("Logging utxocache flushes. Ctrl-C to end...") + print("%-15s %-10s %-15s %-15s %-8s" % ("Duration (µs)", "Mode", + "Coins Count", "Memory Usage", + "Flush for Prune")) + + while True: + try: + b.perf_buffer_poll() + except KeyboardInterrupt: + exit(0) + + +if __name__ == "__main__": + if len(sys.argv) < 2: + print("USAGE: ", sys.argv[0], "path/to/bitcoind") + exit(1) + + path = sys.argv[1] + main(path) diff --git a/contrib/tracing/log_utxos.bt b/contrib/tracing/log_utxos.bt new file mode 100755 index 0000000000..54d5010f82 --- /dev/null +++ b/contrib/tracing/log_utxos.bt @@ -0,0 +1,86 @@ +#!/usr/bin/env bpftrace + +/* + + USAGE: + + bpftrace contrib/tracing/log_utxos.bt + + This script requires a 'bitcoind' binary compiled with eBPF support and the + 'utxocache' tracepoints. By default, it's assumed that 'bitcoind' is + located in './src/bitcoind'. This can be modified in the script below. + + NOTE: requires bpftrace v0.12.0 or above. +*/ + +BEGIN +{ + printf("%-7s %-71s %16s %7s %8s\n", + "OP", "Outpoint", "Value", "Height", "Coinbase"); +} + +/* + Attaches to the 'utxocache:add' tracepoint and prints additions to the UTXO set cache. +*/ +usdt:./src/bitcoind:utxocache:add +{ + $txid = arg0; + $index = (uint32)arg1; + $height = (uint32)arg2; + $value = (int64)arg3; + $isCoinbase = arg4; + + printf("Added "); + $p = $txid + 31; + unroll(32) { + $b = *(uint8*)$p; + printf("%02x", $b); + $p-=1; + } + + printf(":%-6d %16ld %7d %s\n", $index, $value, $height, ($isCoinbase ? "Yes" : "No" )); +} + +/* + Attaches to the 'utxocache:spent' tracepoint and prints spents from the UTXO set cache. +*/ +usdt:./src/bitcoind:utxocache:spent +{ + $txid = arg0; + $index = (uint32)arg1; + $height = (uint32)arg2; + $value = (int64)arg3; + $isCoinbase = arg4; + + printf("Spent "); + $p = $txid + 31; + unroll(32) { + $b = *(uint8*)$p; + printf("%02x", $b); + $p-=1; + } + + printf(":%-6d %16ld %7d %s\n", $index, $value, $height, ($isCoinbase ? "Yes" : "No" )); +} + +/* + Attaches to the 'utxocache:uncache' tracepoint and uncache UTXOs from the UTXO set cache. +*/ +usdt:./src/bitcoind:utxocache:uncache +{ + $txid = arg0; + $index = (uint32)arg1; + $height = (uint32)arg2; + $value = (int64)arg3; + $isCoinbase = arg4; + + printf("Uncache "); + $p = $txid + 31; + unroll(32) { + $b = *(uint8*)$p; + printf("%02x", $b); + $p-=1; + } + + printf(":%-6d %16ld %7d %s\n", $index, $value, $height, ($isCoinbase ? "Yes" : "No" )); +} diff --git a/contrib/tracing/p2p_monitor.py b/contrib/tracing/p2p_monitor.py new file mode 100755 index 0000000000..4ff701cac3 --- /dev/null +++ b/contrib/tracing/p2p_monitor.py @@ -0,0 +1,253 @@ +#!/usr/bin/env python3 +# Copyright (c) 2021 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +""" Interactive bitcoind P2P network traffic monitor utilizing USDT and the + net:inbound_message and net:outbound_message tracepoints. """ + +# This script demonstrates what USDT for Bitcoin Core can enable. It uses BCC +# (https://github.com/iovisor/bcc) to load a sandboxed eBPF program into the +# Linux kernel (root privileges are required). The eBPF program attaches to two +# statically defined tracepoints. The tracepoint 'net:inbound_message' is called +# when a new P2P message is received, and 'net:outbound_message' is called on +# outbound P2P messages. The eBPF program submits the P2P messages to +# this script via a BPF ring buffer. + +import sys +import curses +from curses import wrapper, panel +from bcc import BPF, USDT + +# BCC: The C program to be compiled to an eBPF program (by BCC) and loaded into +# a sandboxed Linux kernel VM. +program = """ +#include + +// Tor v3 addresses are 62 chars + 6 chars for the port (':12345'). +// I2P addresses are 60 chars + 6 chars for the port (':12345'). +#define MAX_PEER_ADDR_LENGTH 62 + 6 +#define MAX_PEER_CONN_TYPE_LENGTH 20 +#define MAX_MSG_TYPE_LENGTH 20 + +struct p2p_message +{ + u64 peer_id; + char peer_addr[MAX_PEER_ADDR_LENGTH]; + char peer_conn_type[MAX_PEER_CONN_TYPE_LENGTH]; + char msg_type[MAX_MSG_TYPE_LENGTH]; + u64 msg_size; +}; + + +// Two BPF perf buffers for pushing data (here P2P messages) to user space. +BPF_PERF_OUTPUT(inbound_messages); +BPF_PERF_OUTPUT(outbound_messages); + +int trace_inbound_message(struct pt_regs *ctx) { + struct p2p_message msg = {}; + + bpf_usdt_readarg(1, ctx, &msg.peer_id); + bpf_usdt_readarg_p(2, ctx, &msg.peer_addr, MAX_PEER_ADDR_LENGTH); + bpf_usdt_readarg_p(3, ctx, &msg.peer_conn_type, MAX_PEER_CONN_TYPE_LENGTH); + bpf_usdt_readarg_p(4, ctx, &msg.msg_type, MAX_MSG_TYPE_LENGTH); + bpf_usdt_readarg(5, ctx, &msg.msg_size); + + inbound_messages.perf_submit(ctx, &msg, sizeof(msg)); + return 0; +}; + +int trace_outbound_message(struct pt_regs *ctx) { + struct p2p_message msg = {}; + + bpf_usdt_readarg(1, ctx, &msg.peer_id); + bpf_usdt_readarg_p(2, ctx, &msg.peer_addr, MAX_PEER_ADDR_LENGTH); + bpf_usdt_readarg_p(3, ctx, &msg.peer_conn_type, MAX_PEER_CONN_TYPE_LENGTH); + bpf_usdt_readarg_p(4, ctx, &msg.msg_type, MAX_MSG_TYPE_LENGTH); + bpf_usdt_readarg(5, ctx, &msg.msg_size); + + outbound_messages.perf_submit(ctx, &msg, sizeof(msg)); + return 0; +}; +""" + + +class Message: + """ A P2P network message. """ + msg_type = "" + size = 0 + data = bytes() + inbound = False + + def __init__(self, msg_type, size, inbound): + self.msg_type = msg_type + self.size = size + self.inbound = inbound + + +class Peer: + """ A P2P network peer. """ + id = 0 + address = "" + connection_type = "" + last_messages = list() + + total_inbound_msgs = 0 + total_inbound_bytes = 0 + total_outbound_msgs = 0 + total_outbound_bytes = 0 + + def __init__(self, id, address, connection_type): + self.id = id + self.address = address + self.connection_type = connection_type + self.last_messages = list() + + def add_message(self, message): + self.last_messages.append(message) + if len(self.last_messages) > 25: + self.last_messages.pop(0) + if message.inbound: + self.total_inbound_bytes += message.size + self.total_inbound_msgs += 1 + else: + self.total_outbound_bytes += message.size + self.total_outbound_msgs += 1 + + +def main(bitcoind_path): + peers = dict() + + bitcoind_with_usdts = USDT(path=str(bitcoind_path)) + + # attaching the trace functions defined in the BPF program to the tracepoints + bitcoind_with_usdts.enable_probe( + probe="inbound_message", fn_name="trace_inbound_message") + bitcoind_with_usdts.enable_probe( + probe="outbound_message", fn_name="trace_outbound_message") + bpf = BPF(text=program, usdt_contexts=[bitcoind_with_usdts]) + + # BCC: perf buffer handle function for inbound_messages + def handle_inbound(_, data, size): + """ Inbound message handler. + + Called each time a message is submitted to the inbound_messages BPF table.""" + event = bpf["inbound_messages"].event(data) + if event.peer_id not in peers: + peer = Peer(event.peer_id, event.peer_addr.decode( + "utf-8"), event.peer_conn_type.decode("utf-8")) + peers[peer.id] = peer + peers[event.peer_id].add_message( + Message(event.msg_type.decode("utf-8"), event.msg_size, True)) + + # BCC: perf buffer handle function for outbound_messages + def handle_outbound(_, data, size): + """ Outbound message handler. + + Called each time a message is submitted to the outbound_messages BPF table.""" + event = bpf["outbound_messages"].event(data) + if event.peer_id not in peers: + peer = Peer(event.peer_id, event.peer_addr.decode( + "utf-8"), event.peer_conn_type.decode("utf-8")) + peers[peer.id] = peer + peers[event.peer_id].add_message( + Message(event.msg_type.decode("utf-8"), event.msg_size, False)) + + # BCC: add handlers to the inbound and outbound perf buffers + bpf["inbound_messages"].open_perf_buffer(handle_inbound) + bpf["outbound_messages"].open_perf_buffer(handle_outbound) + + wrapper(loop, bpf, peers) + + +def loop(screen, bpf, peers): + screen.nodelay(1) + cur_list_pos = 0 + win = curses.newwin(30, 70, 2, 7) + win.erase() + win.border(ord("|"), ord("|"), ord("-"), ord("-"), + ord("-"), ord("-"), ord("-"), ord("-")) + info_panel = panel.new_panel(win) + info_panel.hide() + + ROWS_AVALIABLE_FOR_LIST = curses.LINES - 5 + scroll = 0 + + while True: + try: + # BCC: poll the perf buffers for new events or timeout after 50ms + bpf.perf_buffer_poll(timeout=50) + + ch = screen.getch() + if (ch == curses.KEY_DOWN or ch == ord("j")) and cur_list_pos < len( + peers.keys()) -1 and info_panel.hidden(): + cur_list_pos += 1 + if cur_list_pos >= ROWS_AVALIABLE_FOR_LIST: + scroll += 1 + if (ch == curses.KEY_UP or ch == ord("k")) and cur_list_pos > 0 and info_panel.hidden(): + cur_list_pos -= 1 + if scroll > 0: + scroll -= 1 + if ch == ord('\n') or ch == ord(' '): + if info_panel.hidden(): + info_panel.show() + else: + info_panel.hide() + screen.erase() + render(screen, peers, cur_list_pos, scroll, ROWS_AVALIABLE_FOR_LIST, info_panel) + curses.panel.update_panels() + screen.refresh() + except KeyboardInterrupt: + exit() + + +def render(screen, peers, cur_list_pos, scroll, ROWS_AVALIABLE_FOR_LIST, info_panel): + """ renders the list of peers and details panel + + This code is unrelated to USDT, BCC and BPF. + """ + header_format = "%6s %-20s %-20s %-22s %-67s" + row_format = "%6s %-5d %9d byte %-5d %9d byte %-22s %-67s" + + screen.addstr(0, 1, (" P2P Message Monitor "), curses.A_REVERSE) + screen.addstr( + 1, 0, (" Navigate with UP/DOWN or J/K and select a peer with ENTER or SPACE to see individual P2P messages"), curses.A_NORMAL) + screen.addstr(3, 0, + header_format % ("PEER", "OUTBOUND", "INBOUND", "TYPE", "ADDR"), curses.A_BOLD | curses.A_UNDERLINE) + peer_list = sorted(peers.keys())[scroll:ROWS_AVALIABLE_FOR_LIST+scroll] + for i, peer_id in enumerate(peer_list): + peer = peers[peer_id] + screen.addstr(i + 4, 0, + row_format % (peer.id, peer.total_outbound_msgs, peer.total_outbound_bytes, + peer.total_inbound_msgs, peer.total_inbound_bytes, + peer.connection_type, peer.address), + curses.A_REVERSE if i + scroll == cur_list_pos else curses.A_NORMAL) + if i + scroll == cur_list_pos: + info_window = info_panel.window() + info_window.erase() + info_window.border( + ord("|"), ord("|"), ord("-"), ord("-"), + ord("-"), ord("-"), ord("-"), ord("-")) + + info_window.addstr( + 1, 1, f"PEER {peer.id} ({peer.address})".center(68), curses.A_REVERSE | curses.A_BOLD) + info_window.addstr( + 2, 1, f" OUR NODE{peer.connection_type:^54}PEER ", + curses.A_BOLD) + for i, msg in enumerate(peer.last_messages): + if msg.inbound: + info_window.addstr( + i + 3, 1, "%68s" % + (f"<--- {msg.msg_type} ({msg.size} bytes) "), curses.A_NORMAL) + else: + info_window.addstr( + i + 3, 1, " %s (%d byte) --->" % + (msg.msg_type, msg.size), curses.A_NORMAL) + + +if __name__ == "__main__": + if len(sys.argv) < 2: + print("USAGE:", sys.argv[0], "path/to/bitcoind") + exit() + path = sys.argv[1] + main(path) diff --git a/contrib/valgrind.supp b/contrib/valgrind.supp index ece02dc24e..99ca305fe7 100644 --- a/contrib/valgrind.supp +++ b/contrib/valgrind.supp @@ -13,8 +13,8 @@ # # Note that suppressions may depend on OS and/or library versions. # Tested on: -# * aarch64 (Ubuntu 20.04 system libs, without gui) -# * x86_64 (Ubuntu 18.04 system libs, without gui) +# * aarch64 (Ubuntu 22.04 system libs, clang, without gui) +# * x86_64 (Ubuntu 22.04 system libs, clang, without gui) { Suppress libstdc++ warning - https://gcc.gnu.org/bugzilla/show_bug.cgi?id=65434 Memcheck:Leak @@ -112,18 +112,6 @@ ... fun:GetCoin } -{ - Suppress wcsnrtombs glibc SSE4 warning (could be related: https://stroika.atlassian.net/browse/STK-626) - Memcheck:Addr16 - fun:__wcsnlen_sse4_1 - fun:wcsnrtombs -} -{ - Suppress wcsnrtombs warning (remove after removing boost::fs) - Memcheck:Cond - ... - fun:_ZN5boost10filesystem6detail11unique_pathERKNS0_4pathEPNS_6system10error_codeE -} { Suppress boost warning Memcheck:Leak @@ -134,32 +122,6 @@ fun:_ZN5boost9unit_test14unit_test_mainEPFbvEiPPc fun:main } -{ - Suppress boost::filesystem warning (fixed in boost 1.70: https://github.com/boostorg/filesystem/commit/bbe9d1771e5d679b3f10c42a58fc81f7e8c024a9) - Memcheck:Cond - fun:_ZN5boost10filesystem6detail28directory_iterator_incrementERNS0_18directory_iteratorEPNS_6system10error_codeE - ... - obj:*/libboost_filesystem.so.* -} -{ - Suppress boost::filesystem warning (could be related: https://stackoverflow.com/questions/9830182/function-boostfilesystemcomplete-being-reported-as-possible-memory-leak-by-v) - Memcheck:Leak - match-leak-kinds: reachable - fun:_Znwm - ... - fun:_ZN5boost10filesystem8absoluteERKNS0_4pathES3_ -} -{ - Suppress boost still reachable memory warning - Memcheck:Leak - match-leak-kinds: reachable - fun:_Znwm - ... - fun:_M_construct_aux - fun:_M_construct - fun:basic_string - fun:path -} { Suppress LogInstance still reachable memory warning Memcheck:Leak diff --git a/contrib/verify-commits/README.md b/contrib/verify-commits/README.md index e95a57586f..b8b15280ba 100644 --- a/contrib/verify-commits/README.md +++ b/contrib/verify-commits/README.md @@ -40,7 +40,7 @@ Import trusted keys In order to check the commit signatures, you must add the trusted PGP keys to your machine. [GnuPG](https://gnupg.org/) may be used to import the trusted keys by running the following command: ```sh -gpg --keyserver hkp://keyserver.ubuntu.com --recv-keys $( /dev/null 2>&1; then + if ! ./contrib/verify-commits/verify-commits.py "$3" > /dev/null 2>&1; then echo "ERROR: A commit is not signed, can't push" ./contrib/verify-commits/verify-commits.py exit 1 diff --git a/contrib/verifybinaries/verify.py b/contrib/verifybinaries/verify.py index 6cbaf2dc0c..b5e4f1318b 100755 --- a/contrib/verifybinaries/verify.py +++ b/contrib/verifybinaries/verify.py @@ -1,8 +1,8 @@ #!/usr/bin/env python3 -# Copyright (c) 2020 The Bitcoin Core developers +# Copyright (c) 2020-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. -"""Script for verifying Bitoin Core release binaries +"""Script for verifying Bitcoin Core release binaries This script attempts to download the signature file SHA256SUMS.asc from bitcoincore.org and bitcoin.org and compares them. diff --git a/contrib/windeploy/detached-sig-create.sh b/contrib/windeploy/detached-sig-create.sh index 29802e622e..82fcf2d406 100755 --- a/contrib/windeploy/detached-sig-create.sh +++ b/contrib/windeploy/detached-sig-create.sh @@ -1,5 +1,5 @@ #!/bin/sh -# Copyright (c) 2014-2019 The Bitcoin Core developers +# Copyright (c) 2014-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -23,6 +23,7 @@ TIMESERVER=http://timestamp.comodoca.com CERTFILE="win-codesign.cert" mkdir -p "${OUTSUBDIR}" +# shellcheck disable=SC2046 basename -a $(ls -1 "${SRCDIR}"/*-unsigned.exe) | while read UNSIGNED; do echo Signing "${UNSIGNED}" "${OSSLSIGNCODE}" sign -certs "${CERTFILE}" -t "${TIMESERVER}" -h sha256 -in "${SRCDIR}/${UNSIGNED}" -out "${WORKDIR}/${UNSIGNED}" "$@" diff --git a/contrib/zmq/zmq_sub.py b/contrib/zmq/zmq_sub.py index 9cb887e2dc..d8087a4db3 100755 --- a/contrib/zmq/zmq_sub.py +++ b/contrib/zmq/zmq_sub.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2020 The Bitcoin Core developers +# Copyright (c) 2014-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -23,7 +23,6 @@ https://github.com/bitcoin/bitcoin/blob/37a7fe9e440b83e2364d5498931253937abe9294/contrib/zmq/zmq_sub.py """ -import binascii import asyncio import zmq import zmq.asyncio @@ -58,18 +57,18 @@ class ZMQHandler(): sequence = str(struct.unpack(' $@ - $(AT)touch $@ + touch $@ define check_or_remove_cached @@ -285,3 +290,4 @@ $(foreach package,$(all_packages),$(eval $(call ext_add_stages,$(package)))) .PHONY: install cached clean clean-all download-one download-osx download-linux download-win download check-packages check-sources .PHONY: FORCE +$(V).SILENT: diff --git a/depends/README.md b/depends/README.md index 4f3b6df487..f7647198c6 100644 --- a/depends/README.md +++ b/depends/README.md @@ -28,7 +28,8 @@ Common `host-platform-triplet`s for cross compilation are: - `i686-pc-linux-gnu` for Linux 32 bit - `x86_64-pc-linux-gnu` for x86 Linux - `x86_64-w64-mingw32` for Win64 -- `x86_64-apple-darwin18` for macOS +- `x86_64-apple-darwin` for macOS +- `arm64-apple-darwin` for ARM macOS - `arm-linux-gnueabihf` for Linux ARM 32 bit - `aarch64-linux-gnu` for Linux ARM 64 bit - `powerpc64-linux-gnu` for Linux POWER 64-bit (big endian) @@ -38,16 +39,15 @@ Common `host-platform-triplet`s for cross compilation are: - `s390x-linux-gnu` for Linux S390X - `armv7a-linux-android` for Android ARM 32 bit - `aarch64-linux-android` for Android ARM 64 bit -- `i686-linux-android` for Android x86 32 bit - `x86_64-linux-android` for Android x86 64 bit -The paths are automatically configured and no other options are needed unless targeting [Android](#Android). +The paths are automatically configured and no other options are needed unless targeting [Android](../doc/build-android.md). ### Install the required dependencies: Ubuntu & Debian #### For macOS cross compilation - sudo apt-get install curl librsvg2-bin libtiff-tools bsdmainutils cmake imagemagick libz-dev python3-setuptools libtinfo5 xorriso + sudo apt-get install curl bsdmainutils cmake libz-dev python3-setuptools libtinfo5 xorriso Note: You must obtain the macOS SDK before proceeding with a cross-compile. Under the depends directory, create a subdirectory named `SDKs`. @@ -80,20 +80,13 @@ For linux RISC-V 64-bit cross compilation (there are no packages for 32-bit): sudo apt-get install g++-riscv64-linux-gnu binutils-riscv64-linux-gnu -RISC-V known issue: gcc-7.3.0 and gcc-7.3.1 result in a broken `test_bitcoin` executable (see https://github.com/bitcoin/bitcoin/pull/13543), -this is apparently fixed in gcc-8.1.0. - For linux S390X cross compilation: sudo apt-get install g++-s390x-linux-gnu binutils-s390x-linux-gnu -### Install the required dependencies: M1-based macOS +### Install the required dependencies: OpenBSD -To be able to build the `qt` package, ensure that Rosetta 2 is installed: - -``` -softwareupdate --install-rosetta -``` + pkg_add bash gtar ### Dependency Options @@ -133,18 +126,6 @@ options will be passed to bitcoin's configure. In this case, `--disable-wallet`. download-linux: run 'make download-linux' to fetch all sources needed for linux builds -### Android - -Before proceeding with an Android build one needs to get the [Android SDK](https://developer.android.com/studio) and use the "SDK Manager" tool to download the NDK and one or more "Platform packages" (these are Android versions and have a corresponding API level). -In order to build `ANDROID_API_LEVEL` (API level corresponding to the Android version targeted, e.g. Android 9.0 Pie is 28 and its "Platform package" needs to be available) and `ANDROID_TOOLCHAIN_BIN` (path to toolchain binaries depending on the platform the build is being performed on) need to be set. - -API levels from 24 to 29 have been tested to work. - -If the build includes Qt, environment variables `ANDROID_SDK` and `ANDROID_NDK` need to be set as well but can otherwise be omitted. -This is an example command for a default build with no disabled dependencies: - - ANDROID_SDK=/home/user/Android/Sdk ANDROID_NDK=/home/user/Android/Sdk/ndk-bundle make HOST=aarch64-linux-android ANDROID_API_LEVEL=28 ANDROID_TOOLCHAIN_BIN=/home/user/Android/Sdk/ndk-bundle/toolchains/llvm/prebuilt/linux-x86_64/bin - ### Other documentation - [description.md](description.md): General description of the depends system diff --git a/depends/builders/darwin.mk b/depends/builders/darwin.mk index 001c928424..ced01229dd 100644 --- a/depends/builders/darwin.mk +++ b/depends/builders/darwin.mk @@ -21,3 +21,8 @@ darwin_NM:=$(shell xcrun -f nm) darwin_INSTALL_NAME_TOOL:=$(shell xcrun -f install_name_tool) darwin_native_binutils= darwin_native_toolchain= + +x86_64_darwin_CFLAGS += -arch x86_64 +x86_64_darwin_CXXFLAGS += -arch x86_64 +aarch64_darwin_CFLAGS += -arch arm64 +aarch64_darwin_CXXFLAGS += -arch arm64 diff --git a/depends/builders/default.mk b/depends/builders/default.mk index f097db65d6..0370fb9acb 100644 --- a/depends/builders/default.mk +++ b/depends/builders/default.mk @@ -1,6 +1,7 @@ default_build_CC = gcc default_build_CXX = g++ default_build_AR = ar +default_build_TAR = tar default_build_RANLIB = ranlib default_build_STRIP = strip default_build_NM = nm @@ -12,7 +13,7 @@ build_$(build_os)_$1 ?= $$(default_build_$1) build_$(build_arch)_$(build_os)_$1 ?= $$(build_$(build_os)_$1) build_$1=$$(build_$(build_arch)_$(build_os)_$1) endef -$(foreach var,CC CXX AR RANLIB NM STRIP SHA256SUM DOWNLOAD OTOOL INSTALL_NAME_TOOL,$(eval $(call add_build_tool_func,$(var)))) +$(foreach var,CC CXX AR TAR RANLIB NM STRIP SHA256SUM DOWNLOAD OTOOL INSTALL_NAME_TOOL,$(eval $(call add_build_tool_func,$(var)))) define add_build_flags_func build_$(build_arch)_$(build_os)_$1 += $(build_$(build_os)_$1) build_$1=$$(build_$(build_arch)_$(build_os)_$1) diff --git a/depends/builders/freebsd.mk b/depends/builders/freebsd.mk new file mode 100644 index 0000000000..465f58e04d --- /dev/null +++ b/depends/builders/freebsd.mk @@ -0,0 +1,5 @@ +build_freebsd_CC=clang +build_freebsd_CXX=clang++ + +build_freebsd_SHA256SUM = shasum -a 256 +build_freebsd_DOWNLOAD = curl --location --fail --connect-timeout $(DOWNLOAD_CONNECT_TIMEOUT) --retry $(DOWNLOAD_RETRIES) -o diff --git a/depends/builders/netbsd.mk b/depends/builders/netbsd.mk new file mode 100644 index 0000000000..b7cf1f7514 --- /dev/null +++ b/depends/builders/netbsd.mk @@ -0,0 +1,2 @@ +build_netbsd_SHA256SUM = shasum -a 256 +build_netbsd_DOWNLOAD = curl --location --fail --connect-timeout $(DOWNLOAD_CONNECT_TIMEOUT) --retry $(DOWNLOAD_RETRIES) -o diff --git a/depends/builders/openbsd.mk b/depends/builders/openbsd.mk new file mode 100644 index 0000000000..44825d106a --- /dev/null +++ b/depends/builders/openbsd.mk @@ -0,0 +1,7 @@ +build_openbsd_CC = clang +build_openbsd_CXX = clang++ + +build_openbsd_SHA256SUM = sha256 +build_openbsd_DOWNLOAD = curl --location --fail --connect-timeout $(DOWNLOAD_CONNECT_TIMEOUT) --retry $(DOWNLOAD_RETRIES) -o + +build_openbsd_TAR = gtar diff --git a/depends/config.site.in b/depends/config.site.in index 5cf107f19b..95e6ae85cf 100644 --- a/depends/config.site.in +++ b/depends/config.site.in @@ -62,7 +62,7 @@ if test -z "$with_gui" && test -n "@no_qt@"; then with_gui=no fi -if test -n "@debug@" && test -z "@no_qt@" && test "x$with_gui" != xno; then +if test -n "@debug@" && test -z "@no_qt@" && test "$with_gui" != "no"; then with_gui=qt5_debug fi @@ -70,9 +70,12 @@ if test -z "$enable_zmq" && test -n "@no_zmq@"; then enable_zmq=no fi +if test -z "$enable_usdt" && test -n "@no_usdt@"; then + enable_usdt=no +fi + if test "@host_os@" = darwin; then BREW=no - PORT=no fi PATH="${depends_prefix}/native/bin:${PATH}" diff --git a/depends/funcs.mk b/depends/funcs.mk index 34a030fab7..c4ad109c69 100644 --- a/depends/funcs.mk +++ b/depends/funcs.mk @@ -84,7 +84,7 @@ $(1)_download_path_fixed=$(subst :,\:,$$($(1)_download_path)) #default commands # The default behavior for tar will try to set ownership when running as uid 0 and may not succeed, --no-same-owner disables this behavior $(1)_fetch_cmds ?= $(call fetch_file,$(1),$(subst \:,:,$$($(1)_download_path_fixed)),$$($(1)_download_file),$($(1)_file_name),$($(1)_sha256_hash)) -$(1)_extract_cmds ?= mkdir -p $$($(1)_extract_dir) && echo "$$($(1)_sha256_hash) $$($(1)_source)" > $$($(1)_extract_dir)/.$$($(1)_file_name).hash && $(build_SHA256SUM) -c $$($(1)_extract_dir)/.$$($(1)_file_name).hash && tar --no-same-owner --strip-components=1 -xf $$($(1)_source) +$(1)_extract_cmds ?= mkdir -p $$($(1)_extract_dir) && echo "$$($(1)_sha256_hash) $$($(1)_source)" > $$($(1)_extract_dir)/.$$($(1)_file_name).hash && $(build_SHA256SUM) -c $$($(1)_extract_dir)/.$$($(1)_file_name).hash && $(build_TAR) --no-same-owner --strip-components=1 -xf $$($(1)_source) $(1)_preprocess_cmds ?= $(1)_build_cmds ?= $(1)_config_cmds ?= @@ -188,53 +188,53 @@ endef define int_add_cmds $($(1)_fetched): - $(AT)mkdir -p $$(@D) $(SOURCES_PATH) - $(AT)rm -f $$@ - $(AT)touch $$@ - $(AT)cd $$(@D); $(call $(1)_fetch_cmds,$(1)) - $(AT)cd $($(1)_source_dir); $(foreach source,$($(1)_all_sources),$(build_SHA256SUM) $(source) >> $$(@);) - $(AT)touch $$@ + mkdir -p $$(@D) $(SOURCES_PATH) + rm -f $$@ + touch $$@ + cd $$(@D); $(call $(1)_fetch_cmds,$(1)) + cd $($(1)_source_dir); $(foreach source,$($(1)_all_sources),$(build_SHA256SUM) $(source) >> $$(@);) + touch $$@ $($(1)_extracted): | $($(1)_fetched) - $(AT)echo Extracting $(1)... - $(AT)mkdir -p $$(@D) - $(AT)cd $$(@D); $(call $(1)_extract_cmds,$(1)) - $(AT)touch $$@ + echo Extracting $(1)... + mkdir -p $$(@D) + cd $$(@D); $(call $(1)_extract_cmds,$(1)) + touch $$@ $($(1)_preprocessed): | $($(1)_extracted) - $(AT)echo Preprocessing $(1)... - $(AT)mkdir -p $$(@D) $($(1)_patch_dir) - $(AT)$(foreach patch,$($(1)_patches),cd $(PATCHES_PATH)/$(1); cp $(patch) $($(1)_patch_dir) ;) - $(AT)cd $$(@D); $(call $(1)_preprocess_cmds, $(1)) - $(AT)touch $$@ + echo Preprocessing $(1)... + mkdir -p $$(@D) $($(1)_patch_dir) + $(foreach patch,$($(1)_patches),cd $(PATCHES_PATH)/$(1); cp $(patch) $($(1)_patch_dir) ;) + cd $$(@D); $(call $(1)_preprocess_cmds, $(1)) + touch $$@ $($(1)_configured): | $($(1)_dependencies) $($(1)_preprocessed) - $(AT)echo Configuring $(1)... - $(AT)rm -rf $(host_prefix); mkdir -p $(host_prefix)/lib; cd $(host_prefix); $(foreach package,$($(1)_all_dependencies), tar --no-same-owner -xf $($(package)_cached); ) - $(AT)mkdir -p $$(@D) - $(AT)+cd $$(@D); $($(1)_config_env) $(call $(1)_config_cmds, $(1)) - $(AT)touch $$@ + echo Configuring $(1)... + rm -rf $(host_prefix); mkdir -p $(host_prefix)/lib; cd $(host_prefix); $(foreach package,$($(1)_all_dependencies), $(build_TAR) --no-same-owner -xf $($(package)_cached); ) + mkdir -p $$(@D) + +cd $$(@D); $($(1)_config_env) $(call $(1)_config_cmds, $(1)) + touch $$@ $($(1)_built): | $($(1)_configured) - $(AT)echo Building $(1)... - $(AT)mkdir -p $$(@D) - $(AT)+cd $$(@D); $($(1)_build_env) $(call $(1)_build_cmds, $(1)) - $(AT)touch $$@ + echo Building $(1)... + mkdir -p $$(@D) + +cd $$(@D); $($(1)_build_env) $(call $(1)_build_cmds, $(1)) + touch $$@ $($(1)_staged): | $($(1)_built) - $(AT)echo Staging $(1)... - $(AT)mkdir -p $($(1)_staging_dir)/$(host_prefix) - $(AT)cd $($(1)_build_dir); $($(1)_stage_env) $(call $(1)_stage_cmds, $(1)) - $(AT)rm -rf $($(1)_extract_dir) - $(AT)touch $$@ + echo Staging $(1)... + mkdir -p $($(1)_staging_dir)/$(host_prefix) + cd $($(1)_build_dir); $($(1)_stage_env) $(call $(1)_stage_cmds, $(1)) + rm -rf $($(1)_extract_dir) + touch $$@ $($(1)_postprocessed): | $($(1)_staged) - $(AT)echo Postprocessing $(1)... - $(AT)cd $($(1)_staging_prefix_dir); $(call $(1)_postprocess_cmds) - $(AT)touch $$@ + echo Postprocessing $(1)... + cd $($(1)_staging_prefix_dir); $(call $(1)_postprocess_cmds) + touch $$@ $($(1)_cached): | $($(1)_dependencies) $($(1)_postprocessed) - $(AT)echo Caching $(1)... - $(AT)cd $$($(1)_staging_dir)/$(host_prefix); find . | sort | tar --no-recursion -czf $$($(1)_staging_dir)/$$(@F) -T - - $(AT)mkdir -p $$(@D) - $(AT)rm -rf $$(@D) && mkdir -p $$(@D) - $(AT)mv $$($(1)_staging_dir)/$$(@F) $$(@) - $(AT)rm -rf $($(1)_staging_dir) + echo Caching $(1)... + cd $$($(1)_staging_dir)/$(host_prefix); find . | sort | $(build_TAR) --no-recursion -czf $$($(1)_staging_dir)/$$(@F) -T - + mkdir -p $$(@D) + rm -rf $$(@D) && mkdir -p $$(@D) + mv $$($(1)_staging_dir)/$$(@F) $$(@) + rm -rf $($(1)_staging_dir) $($(1)_cached_checksum): $($(1)_cached) - $(AT)cd $$(@D); $(build_SHA256SUM) $$( $$(@) + cd $$(@D); $(build_SHA256SUM) $$( $$(@) .PHONY: $(1) $(1): | $($(1)_cached_checksum) @@ -264,7 +264,8 @@ $(foreach package,$(packages),$(eval $(package)_type=$(host_arch)_$(host_os))) $(foreach package,$(all_packages),$(eval $(call int_vars,$(package)))) #include package files -$(foreach package,$(all_packages),$(eval include packages/$(package).mk)) +$(foreach native_package,$(native_packages),$(eval include packages/$(native_package).mk)) +$(foreach package,$(packages),$(eval include packages/$(package).mk)) #compute a hash of all files that comprise this package's build recipe $(foreach package,$(all_packages),$(eval $(call int_get_build_recipe_hash,$(package)))) diff --git a/depends/hosts/android.mk b/depends/hosts/android.mk index eabd84bbbe..fcc1c4f5c3 100644 --- a/depends/hosts/android.mk +++ b/depends/hosts/android.mk @@ -1,12 +1,11 @@ ifeq ($(HOST),armv7a-linux-android) -android_AR=$(ANDROID_TOOLCHAIN_BIN)/arm-linux-androideabi-ar android_CXX=$(ANDROID_TOOLCHAIN_BIN)/$(HOST)eabi$(ANDROID_API_LEVEL)-clang++ android_CC=$(ANDROID_TOOLCHAIN_BIN)/$(HOST)eabi$(ANDROID_API_LEVEL)-clang -android_RANLIB=$(ANDROID_TOOLCHAIN_BIN)/arm-linux-androideabi-ranlib else -android_AR=$(ANDROID_TOOLCHAIN_BIN)/$(HOST)-ar android_CXX=$(ANDROID_TOOLCHAIN_BIN)/$(HOST)$(ANDROID_API_LEVEL)-clang++ android_CC=$(ANDROID_TOOLCHAIN_BIN)/$(HOST)$(ANDROID_API_LEVEL)-clang -android_RANLIB=$(ANDROID_TOOLCHAIN_BIN)/$(HOST)-ranlib endif +android_AR=$(ANDROID_TOOLCHAIN_BIN)/llvm-ar +android_RANLIB=$(ANDROID_TOOLCHAIN_BIN)/llvm-ranlib + android_cmake_system=Android diff --git a/depends/hosts/darwin.mk b/depends/hosts/darwin.mk index 5a7ae2df9a..6bf30b499a 100644 --- a/depends/hosts/darwin.mk +++ b/depends/hosts/darwin.mk @@ -1,7 +1,7 @@ -OSX_MIN_VERSION=10.14 -OSX_SDK_VERSION=10.15.6 -XCODE_VERSION=12.1 -XCODE_BUILD_ID=12A7403 +OSX_MIN_VERSION=10.15 +OSX_SDK_VERSION=11.0 +XCODE_VERSION=12.2 +XCODE_BUILD_ID=12B45b LD64_VERSION=609 OSX_SDK=$(SDK_PATH)/Xcode-$(XCODE_VERSION)-$(XCODE_BUILD_ID)-extracted-SDK-with-libcxx-headers diff --git a/depends/hosts/freebsd.mk b/depends/hosts/freebsd.mk new file mode 100644 index 0000000000..0a62347b57 --- /dev/null +++ b/depends/hosts/freebsd.mk @@ -0,0 +1,31 @@ +freebsd_CFLAGS=-pipe +freebsd_CXXFLAGS=$(freebsd_CFLAGS) + +freebsd_release_CFLAGS=-O2 +freebsd_release_CXXFLAGS=$(freebsd_release_CFLAGS) + +freebsd_debug_CFLAGS=-O1 +freebsd_debug_CXXFLAGS=$(freebsd_debug_CFLAGS) + +ifeq (86,$(findstring 86,$(build_arch))) +i686_freebsd_CC=clang -m32 +i686_freebsd_CXX=clang++ -m32 +i686_freebsd_AR=ar +i686_freebsd_RANLIB=ranlib +i686_freebsd_NM=nm +i686_freebsd_STRIP=strip + +x86_64_freebsd_CC=clang -m64 +x86_64_freebsd_CXX=clang++ -m64 +x86_64_freebsd_AR=ar +x86_64_freebsd_RANLIB=ranlib +x86_64_freebsd_NM=nm +x86_64_freebsd_STRIP=strip +else +i686_freebsd_CC=$(default_host_CC) -m32 +i686_freebsd_CXX=$(default_host_CXX) -m32 +x86_64_freebsd_CC=$(default_host_CC) -m64 +x86_64_freebsd_CXX=$(default_host_CXX) -m64 +endif + +freebsd_cmake_system=FreeBSD diff --git a/depends/hosts/mingw32.mk b/depends/hosts/mingw32.mk index be5fec570c..2f370d2b87 100644 --- a/depends/hosts/mingw32.mk +++ b/depends/hosts/mingw32.mk @@ -1,3 +1,7 @@ +ifneq ($(shell $(SHELL) $(.SHELLFLAGS) "command -v $(host)-g++-posix"),) +mingw32_CXX := $(host)-g++-posix +endif + mingw32_CFLAGS=-pipe mingw32_CXXFLAGS=$(mingw32_CFLAGS) diff --git a/depends/hosts/netbsd.mk b/depends/hosts/netbsd.mk new file mode 100644 index 0000000000..b3e4545a64 --- /dev/null +++ b/depends/hosts/netbsd.mk @@ -0,0 +1,31 @@ +netbsd_CFLAGS=-pipe +netbsd_CXXFLAGS=$(netbsd_CFLAGS) + +netbsd_release_CFLAGS=-O2 +netbsd_release_CXXFLAGS=$(netbsd_release_CFLAGS) + +netbsd_debug_CFLAGS=-O1 +netbsd_debug_CXXFLAGS=$(netbsd_debug_CFLAGS) + +ifeq (86,$(findstring 86,$(build_arch))) +i686_netbsd_CC=gcc -m32 +i686_netbsd_CXX=g++ -m32 +i686_netbsd_AR=ar +i686_netbsd_RANLIB=ranlib +i686_netbsd_NM=nm +i686_netbsd_STRIP=strip + +x86_64_netbsd_CC=gcc -m64 +x86_64_netbsd_CXX=g++ -m64 +x86_64_netbsd_AR=ar +x86_64_netbsd_RANLIB=ranlib +x86_64_netbsd_NM=nm +x86_64_netbsd_STRIP=strip +else +i686_netbsd_CC=$(default_host_CC) -m32 +i686_netbsd_CXX=$(default_host_CXX) -m32 +x86_64_netbsd_CC=$(default_host_CC) -m64 +x86_64_netbsd_CXX=$(default_host_CXX) -m64 +endif + +netbsd_cmake_system=NetBSD diff --git a/depends/hosts/openbsd.mk b/depends/hosts/openbsd.mk new file mode 100644 index 0000000000..dc8393e04c --- /dev/null +++ b/depends/hosts/openbsd.mk @@ -0,0 +1,31 @@ +openbsd_CFLAGS=-pipe +openbsd_CFLAGS_CXXFLAGS=$(openbsd_CFLAGS) + +openbsd_CFLAGS_release_CFLAGS=-O2 +openbsd_CFLAGS_release_CXXFLAGS=$(openbsd_release_CFLAGS) + +openbsd_CFLAGS_debug_CFLAGS=-O1 +openbsd_CFLAGS_debug_CXXFLAGS=$(openbsd_debug_CFLAGS) + +ifeq (86,$(findstring 86,$(build_arch))) +i686_openbsd_CC=clang -m32 +i686_openbsd_CXX=clang++ -m32 +i686_openbsd_AR=ar +i686_openbsd_RANLIB=ranlib +i686_openbsd_NM=nm +i686_openbsd_STRIP=strip + +x86_64_openbsd_CC=clang -m64 +x86_64_openbsd_CXX=clang++ -m64 +x86_64_openbsd_AR=ar +x86_64_openbsd_RANLIB=ranlib +x86_64_openbsd_NM=nm +x86_64_openbsd_STRIP=strip +else +i686_openbsd_CC=$(default_host_CC) -m32 +i686_openbsd_CXX=$(default_host_CXX) -m32 +x86_64_openbsd_CC=$(default_host_CC) -m64 +x86_64_openbsd_CXX=$(default_host_CXX) -m64 +endif + +openbsd_cmake_system=OpenBSD diff --git a/depends/packages.md b/depends/packages.md index 7ed20ea129..4158b46d28 100644 --- a/depends/packages.md +++ b/depends/packages.md @@ -178,8 +178,8 @@ not sufficient to just say `libprimary`. For us, it's much easier to just link a static `libsecondary` into a shared `libprimary`. Especially because in our case, we are linking against a dummy `libprimary` anyway that we'll throw away. We don't care if the end-user has a -static or dynamic `libseconday`, that's not our concern. With a static -`libseconday`, when we need to link `libprimary` into our executable, there's no +static or dynamic `libsecondary`, that's not our concern. With a static +`libsecondary`, when we need to link `libprimary` into our executable, there's no dependency chain to worry about as `libprimary` has all the symbols. ## Build targets: diff --git a/depends/packages/bdb.mk b/depends/packages/bdb.mk index d45ac3d03f..dc536fd399 100644 --- a/depends/packages/bdb.mk +++ b/depends/packages/bdb.mk @@ -10,9 +10,12 @@ define $(package)_set_vars $(package)_config_opts=--disable-shared --enable-cxx --disable-replication --enable-option-checking $(package)_config_opts_mingw32=--enable-mingw $(package)_config_opts_linux=--with-pic +$(package)_config_opts_freebsd=--with-pic +$(package)_config_opts_netbsd=--with-pic +$(package)_config_opts_openbsd=--with-pic $(package)_config_opts_android=--with-pic $(package)_cflags+=-Wno-error=implicit-function-declaration -$(package)_cxxflags=-std=c++17 +$(package)_cxxflags+=-std=c++17 $(package)_cppflags_mingw32=-DUNICODE -D_UNICODE endef diff --git a/depends/packages/boost.mk b/depends/packages/boost.mk index f879d176f5..563848c398 100644 --- a/depends/packages/boost.mk +++ b/depends/packages/boost.mk @@ -1,46 +1,10 @@ package=boost -$(package)_version=1_71_0 -$(package)_download_path=https://boostorg.jfrog.io/artifactory/main/release/$(subst _,.,$($(package)_version))/source/ -$(package)_file_name=boost_$($(package)_version).tar.bz2 -$(package)_sha256_hash=d73a8da01e8bf8c7eda40b4c84915071a8c8a0df4a6734537ddde4a8580524ee -$(package)_dependencies=native_b2 - -define $(package)_set_vars -$(package)_config_opts_release=variant=release -$(package)_config_opts_debug=variant=debug -$(package)_config_opts=--layout=tagged --build-type=complete --user-config=user-config.jam -$(package)_config_opts+=threading=multi link=static -sNO_COMPRESSION=1 -$(package)_config_opts_linux=target-os=linux threadapi=pthread runtime-link=shared -$(package)_config_opts_darwin=target-os=darwin runtime-link=shared -$(package)_config_opts_mingw32=target-os=windows binary-format=pe threadapi=win32 runtime-link=static -$(package)_config_opts_x86_64=architecture=x86 address-model=64 -$(package)_config_opts_i686=architecture=x86 address-model=32 -$(package)_config_opts_aarch64=address-model=64 -$(package)_config_opts_armv7a=address-model=32 -ifneq (,$(findstring clang,$($(package)_cxx))) -$(package)_toolset_$(host_os)=clang -else -$(package)_toolset_$(host_os)=gcc -endif -$(package)_config_libraries=filesystem,system,test -$(package)_cxxflags=-std=c++17 -fvisibility=hidden -$(package)_cxxflags_linux=-fPIC -$(package)_cxxflags_android=-fPIC -$(package)_cxxflags_x86_64_darwin=-fcf-protection=full -endef - -define $(package)_preprocess_cmds - echo "using $($(package)_toolset_$(host_os)) : : $($(package)_cxx) : \"$($(package)_cflags)\" \"$($(package)_cxxflags)\" \"$($(package)_cppflags)\" \"$($(package)_ldflags)\" \"$($(package)_ar)\" \"$(host_STRIP)\" \"$(host_RANLIB)\" \"$(host_WINDRES)\" : ;" > user-config.jam -endef - -define $(package)_config_cmds - ./bootstrap.sh --without-icu --with-libraries=$($(package)_config_libraries) --with-toolset=$($(package)_toolset_$(host_os)) --with-bjam=b2 -endef - -define $(package)_build_cmds - b2 -d2 -j2 -d1 --prefix=$($(package)_staging_prefix_dir) $($(package)_config_opts) toolset=$($(package)_toolset_$(host_os)) stage -endef +$(package)_version=1.77.0 +$(package)_download_path=https://boostorg.jfrog.io/artifactory/main/release/$($(package)_version)/source/ +$(package)_file_name=boost_$(subst .,_,$($(package)_version)).tar.bz2 +$(package)_sha256_hash=fc9f85fc030e233142908241af7a846e60630aa7388de9a5fafb1f3a26840854 define $(package)_stage_cmds - b2 -d0 -j4 --prefix=$($(package)_staging_prefix_dir) $($(package)_config_opts) toolset=$($(package)_toolset_$(host_os)) install + mkdir -p $($(package)_staging_prefix_dir)/include && \ + cp -r boost $($(package)_staging_prefix_dir)/include endef diff --git a/depends/packages/capnp.mk b/depends/packages/capnp.mk index abeb26545f..8a3a14810d 100644 --- a/depends/packages/capnp.mk +++ b/depends/packages/capnp.mk @@ -1,6 +1,7 @@ package=capnp $(package)_version=$(native_$(package)_version) $(package)_download_path=$(native_$(package)_download_path) +$(package)_download_file=$(native_$(package)_download_file) $(package)_file_name=$(native_$(package)_file_name) $(package)_sha256_hash=$(native_$(package)_sha256_hash) $(package)_dependencies=native_$(package) diff --git a/depends/packages/expat.mk b/depends/packages/expat.mk index 902fe43be2..50791ebc6e 100644 --- a/depends/packages/expat.mk +++ b/depends/packages/expat.mk @@ -1,12 +1,13 @@ package=expat -$(package)_version=2.2.7 -$(package)_download_path=https://github.com/libexpat/libexpat/releases/download/R_2_2_7/ -$(package)_file_name=$(package)-$($(package)_version).tar.bz2 -$(package)_sha256_hash=cbc9102f4a31a8dafd42d642e9a3aa31e79a0aedaa1f6efd2795ebc83174ec18 +$(package)_version=2.4.1 +$(package)_download_path=https://github.com/libexpat/libexpat/releases/download/R_$(subst .,_,$($(package)_version))/ +$(package)_file_name=$(package)-$($(package)_version).tar.xz +$(package)_sha256_hash=cf032d0dba9b928636548e32b327a2d66b1aab63c4f4a13dd132c2d1d2f2fb6a define $(package)_set_vars $(package)_config_opts=--disable-shared --without-docbook --without-tests --without-examples $(package)_config_opts += --disable-dependency-tracking --enable-option-checking + $(package)_config_opts += --without-xmlwf $(package)_config_opts_linux=--with-pic endef @@ -23,5 +24,5 @@ define $(package)_stage_cmds endef define $(package)_postprocess_cmds - rm lib/*.la + rm -rf share lib/cmake lib/*.la endef diff --git a/depends/packages/fontconfig.mk b/depends/packages/fontconfig.mk index 0d5f94f380..c8b2fc33d5 100644 --- a/depends/packages/fontconfig.mk +++ b/depends/packages/fontconfig.mk @@ -1,10 +1,10 @@ package=fontconfig -$(package)_version=2.12.1 +$(package)_version=2.12.6 $(package)_download_path=https://www.freedesktop.org/software/fontconfig/release/ $(package)_file_name=$(package)-$($(package)_version).tar.bz2 -$(package)_sha256_hash=b449a3e10c47e1d1c7a6ec6e2016cca73d3bd68fbbd4f0ae5cc6b573f7d6c7f3 +$(package)_sha256_hash=cf0c30807d08f6a28ab46c61b8dbd55c97d2f292cf88f3a07d3384687f31f017 $(package)_dependencies=freetype expat -$(package)_patches=remove_char_width_usage.patch gperf_header_regen.patch +$(package)_patches=gperf_header_regen.patch define $(package)_set_vars $(package)_config_opts=--disable-docs --disable-static --disable-libxml2 --disable-iconv @@ -12,7 +12,6 @@ define $(package)_set_vars endef define $(package)_preprocess_cmds - patch -p1 < $($(package)_patch_dir)/remove_char_width_usage.patch && \ patch -p1 < $($(package)_patch_dir)/gperf_header_regen.patch endef @@ -29,5 +28,5 @@ define $(package)_stage_cmds endef define $(package)_postprocess_cmds - rm lib/*.la + rm -rf var lib/*.la endef diff --git a/depends/packages/freetype.mk b/depends/packages/freetype.mk index a1584608e1..6f5dbe0f01 100644 --- a/depends/packages/freetype.mk +++ b/depends/packages/freetype.mk @@ -1,12 +1,12 @@ package=freetype -$(package)_version=2.7.1 +$(package)_version=2.11.0 $(package)_download_path=https://download.savannah.gnu.org/releases/$(package) -$(package)_file_name=$(package)-$($(package)_version).tar.bz2 -$(package)_sha256_hash=3a3bb2c4e15ffb433f2032f50a5b5a92558206822e22bfe8cbe339af4aa82f88 +$(package)_file_name=$(package)-$($(package)_version).tar.xz +$(package)_sha256_hash=8bee39bd3968c4804b70614a0a3ad597299ad0e824bc8aad5ce8aaf48067bde7 define $(package)_set_vars $(package)_config_opts=--without-zlib --without-png --without-harfbuzz --without-bzip2 --disable-static - $(package)_config_opts += --enable-option-checking + $(package)_config_opts += --enable-option-checking --without-brotli $(package)_config_opts_linux=--with-pic endef @@ -23,5 +23,5 @@ define $(package)_stage_cmds endef define $(package)_postprocess_cmds - rm lib/*.la + rm -rf share/man lib/*.la endef diff --git a/depends/packages/libXau.mk b/depends/packages/libXau.mk index 4c55c2df04..b7e032c0b2 100644 --- a/depends/packages/libXau.mk +++ b/depends/packages/libXau.mk @@ -1,8 +1,8 @@ package=libXau -$(package)_version=1.0.8 +$(package)_version=1.0.9 $(package)_download_path=https://xorg.freedesktop.org/releases/individual/lib/ $(package)_file_name=$(package)-$($(package)_version).tar.bz2 -$(package)_sha256_hash=fdd477320aeb5cdd67272838722d6b7d544887dfe7de46e1e7cc0c27c2bea4f2 +$(package)_sha256_hash=ccf8cbf0dbf676faa2ea0a6d64bcc3b6746064722b606c8c52917ed00dcb73ec $(package)_dependencies=xproto # When updating this package, check the default value of @@ -30,5 +30,5 @@ define $(package)_stage_cmds endef define $(package)_postprocess_cmds - rm lib/*.la + rm -rf share lib/*.la endef diff --git a/depends/packages/libevent.mk b/depends/packages/libevent.mk index dad317193c..748ed510c1 100644 --- a/depends/packages/libevent.mk +++ b/depends/packages/libevent.mk @@ -12,10 +12,17 @@ define $(package)_set_vars $(package)_config_opts += --disable-dependency-tracking --enable-option-checking $(package)_config_opts_release=--disable-debug-mode $(package)_config_opts_linux=--with-pic + $(package)_config_opts_freebsd=--with-pic + $(package)_config_opts_netbsd=--with-pic + $(package)_config_opts_openbsd=--with-pic $(package)_config_opts_android=--with-pic $(package)_cppflags_mingw32=-D_WIN32_WINNT=0x0601 endef +define $(package)_preprocess_cmds + cp -f $(BASEDIR)/config.guess $(BASEDIR)/config.sub build-aux +endef + define $(package)_config_cmds $($(package)_autoconf) endef diff --git a/depends/packages/libmultiprocess.mk b/depends/packages/libmultiprocess.mk index 3e5cf5f160..40ab3c68ea 100644 --- a/depends/packages/libmultiprocess.mk +++ b/depends/packages/libmultiprocess.mk @@ -3,7 +3,7 @@ $(package)_version=$(native_$(package)_version) $(package)_download_path=$(native_$(package)_download_path) $(package)_file_name=$(native_$(package)_file_name) $(package)_sha256_hash=$(native_$(package)_sha256_hash) -$(package)_dependencies=native_$(package) boost capnp +$(package)_dependencies=native_$(package) capnp define $(package)_config_cmds $($(package)_cmake) diff --git a/depends/packages/libxcb.mk b/depends/packages/libxcb.mk index 710f43959c..fa30e80f5c 100644 --- a/depends/packages/libxcb.mk +++ b/depends/packages/libxcb.mk @@ -1,25 +1,21 @@ package=libxcb -$(package)_version=1.10 +$(package)_version=1.14 $(package)_download_path=https://xcb.freedesktop.org/dist -$(package)_file_name=$(package)-$($(package)_version).tar.bz2 -$(package)_sha256_hash=98d9ab05b636dd088603b64229dd1ab2d2cc02ab807892e107d674f9c3f2d5b5 +$(package)_file_name=$(package)-$($(package)_version).tar.xz +$(package)_sha256_hash=a55ed6db98d43469801262d81dc2572ed124edc3db31059d4e9916eb9f844c34 $(package)_dependencies=xcb_proto libXau define $(package)_set_vars -$(package)_config_opts=--disable-static --disable-build-docs --without-doxygen --without-launchd +$(package)_config_opts=--disable-static --disable-devel-docs --without-doxygen --without-launchd $(package)_config_opts += --disable-dependency-tracking --enable-option-checking -# Because we pass -qt-xcb to Qt, it will compile in a set of xcb helper libraries and extensions, -# so we skip building all of the extensions here. -# More info is available from: https://doc.qt.io/qt-5.9/linux-requirements.html +# Disable unneeded extensions. +# More info is available from: https://doc.qt.io/qt-5.15/linux-requirements.html $(package)_config_opts += --disable-composite --disable-damage --disable-dpms $(package)_config_opts += --disable-dri2 --disable-dri3 --disable-glx -$(package)_config_opts += --disable-present --disable-randr --disable-record -$(package)_config_opts += --disable-render --disable-resource --disable-screensaver -$(package)_config_opts += --disable-shape --disable-sync -$(package)_config_opts += --disable-xevie --disable-xfixes --disable-xfree86-dri -$(package)_config_opts += --disable-xinerama --disable-xinput -$(package)_config_opts += --disable-xprint --disable-selinux --disable-xtest -$(package)_config_opts += --disable-xv --disable-xvmc +$(package)_config_opts += --disable-present --disable-record --disable-resource +$(package)_config_opts += --disable-screensaver --disable-xevie --disable-xfree86-dri +$(package)_config_opts += --disable-xinput --disable-xprint --disable-selinux +$(package)_config_opts += --disable-xtest --disable-xv --disable-xvmc endef define $(package)_preprocess_cmds diff --git a/depends/packages/libxcb_util.mk b/depends/packages/libxcb_util.mk new file mode 100644 index 0000000000..6f1b9cd7c6 --- /dev/null +++ b/depends/packages/libxcb_util.mk @@ -0,0 +1,32 @@ +package=libxcb_util +$(package)_version=0.4.0 +$(package)_download_path=https://xcb.freedesktop.org/dist +$(package)_file_name=xcb-util-$($(package)_version).tar.bz2 +$(package)_sha256_hash=46e49469cb3b594af1d33176cd7565def2be3fa8be4371d62271fabb5eae50e9 +$(package)_dependencies=libxcb + +define $(package)_set_vars +$(package)_config_opts = --disable-shared --disable-devel-docs --without-doxygen +$(package)_config_opts += --disable-dependency-tracking --enable-option-checking +$(package)_config_opts += --with-pic +endef + +define $(package)_preprocess_cmds + cp -f $(BASEDIR)/config.guess $(BASEDIR)/config.sub . +endef + +define $(package)_config_cmds + $($(package)_autoconf) +endef + +define $(package)_build_cmds + $(MAKE) +endef + +define $(package)_stage_cmds + $(MAKE) DESTDIR=$($(package)_staging_dir) install +endef + +define $(package)_postprocess_cmds + rm -rf share/man share/doc lib/*.la +endef diff --git a/depends/packages/libxcb_util_image.mk b/depends/packages/libxcb_util_image.mk new file mode 100644 index 0000000000..d12d67e8e8 --- /dev/null +++ b/depends/packages/libxcb_util_image.mk @@ -0,0 +1,31 @@ +package=libxcb_util_image +$(package)_version=0.4.0 +$(package)_download_path=https://xcb.freedesktop.org/dist +$(package)_file_name=xcb-util-image-$($(package)_version).tar.bz2 +$(package)_sha256_hash=2db96a37d78831d643538dd1b595d7d712e04bdccf8896a5e18ce0f398ea2ffc +$(package)_dependencies=libxcb libxcb_util + +define $(package)_set_vars +$(package)_config_opts=--disable-static --disable-devel-docs --without-doxygen +$(package)_config_opts+= --disable-dependency-tracking --enable-option-checking +endef + +define $(package)_preprocess_cmds + cp -f $(BASEDIR)/config.guess $(BASEDIR)/config.sub . +endef + +define $(package)_config_cmds + $($(package)_autoconf) +endef + +define $(package)_build_cmds + $(MAKE) +endef + +define $(package)_stage_cmds + $(MAKE) DESTDIR=$($(package)_staging_dir) install +endef + +define $(package)_postprocess_cmds + rm -rf share/man share/doc lib/*.la +endef diff --git a/depends/packages/libxcb_util_keysyms.mk b/depends/packages/libxcb_util_keysyms.mk new file mode 100644 index 0000000000..d4f72dedbe --- /dev/null +++ b/depends/packages/libxcb_util_keysyms.mk @@ -0,0 +1,31 @@ +package=libxcb_util_keysyms +$(package)_version=0.4.0 +$(package)_download_path=https://xcb.freedesktop.org/dist +$(package)_file_name=xcb-util-keysyms-$($(package)_version).tar.bz2 +$(package)_sha256_hash=0ef8490ff1dede52b7de533158547f8b454b241aa3e4dcca369507f66f216dd9 +$(package)_dependencies=libxcb xproto + +define $(package)_set_vars +$(package)_config_opts=--disable-static --disable-devel-docs --without-doxygen +$(package)_config_opts += --disable-dependency-tracking --enable-option-checking +endef + +define $(package)_preprocess_cmds + cp -f $(BASEDIR)/config.guess $(BASEDIR)/config.sub . +endef + +define $(package)_config_cmds + $($(package)_autoconf) +endef + +define $(package)_build_cmds + $(MAKE) +endef + +define $(package)_stage_cmds + $(MAKE) DESTDIR=$($(package)_staging_dir) install +endef + +define $(package)_postprocess_cmds + rm -rf share/man share/doc lib/*.la +endef diff --git a/depends/packages/libxcb_util_render.mk b/depends/packages/libxcb_util_render.mk new file mode 100644 index 0000000000..28f1fb073c --- /dev/null +++ b/depends/packages/libxcb_util_render.mk @@ -0,0 +1,31 @@ +package=libxcb_util_render +$(package)_version=0.3.9 +$(package)_download_path=https://xcb.freedesktop.org/dist +$(package)_file_name=xcb-util-renderutil-$($(package)_version).tar.bz2 +$(package)_sha256_hash=c6e97e48fb1286d6394dddb1c1732f00227c70bd1bedb7d1acabefdd340bea5b +$(package)_dependencies=libxcb + +define $(package)_set_vars +$(package)_config_opts=--disable-static --disable-devel-docs --without-doxygen +$(package)_config_opts += --disable-dependency-tracking --enable-option-checking +endef + +define $(package)_preprocess_cmds + cp -f $(BASEDIR)/config.guess $(BASEDIR)/config.sub . +endef + +define $(package)_config_cmds + $($(package)_autoconf) +endef + +define $(package)_build_cmds + $(MAKE) +endef + +define $(package)_stage_cmds + $(MAKE) DESTDIR=$($(package)_staging_dir) install +endef + +define $(package)_postprocess_cmds + rm -rf share/man share/doc lib/*.la +endef diff --git a/depends/packages/libxcb_util_wm.mk b/depends/packages/libxcb_util_wm.mk new file mode 100644 index 0000000000..3b905ba4ec --- /dev/null +++ b/depends/packages/libxcb_util_wm.mk @@ -0,0 +1,31 @@ +package=libxcb_util_wm +$(package)_version=0.4.1 +$(package)_download_path=https://xcb.freedesktop.org/dist +$(package)_file_name=xcb-util-wm-$($(package)_version).tar.bz2 +$(package)_sha256_hash=28bf8179640eaa89276d2b0f1ce4285103d136be6c98262b6151aaee1d3c2a3f +$(package)_dependencies=libxcb + +define $(package)_set_vars +$(package)_config_opts=--disable-static --disable-devel-docs --without-doxygen +$(package)_config_opts += --disable-dependency-tracking --enable-option-checking +endef + +define $(package)_preprocess_cmds + cp -f $(BASEDIR)/config.guess $(BASEDIR)/config.sub . +endef + +define $(package)_config_cmds + $($(package)_autoconf) +endef + +define $(package)_build_cmds + $(MAKE) +endef + +define $(package)_stage_cmds + $(MAKE) DESTDIR=$($(package)_staging_dir) install +endef + +define $(package)_postprocess_cmds + rm -rf share/man share/doc lib/*.la +endef diff --git a/depends/packages/native_b2.mk b/depends/packages/native_b2.mk deleted file mode 100644 index aaa37cdcfa..0000000000 --- a/depends/packages/native_b2.mk +++ /dev/null @@ -1,20 +0,0 @@ -package=native_b2 -$(package)_version=$(boost_version) -$(package)_download_path=$(boost_download_path) -$(package)_file_name=$(boost_file_name) -$(package)_sha256_hash=$(boost_sha256_hash) -$(package)_build_subdir=tools/build/src/engine -ifneq (,$(findstring clang,$($(package)_cxx))) -$(package)_toolset_$(host_os)=clang -else -$(package)_toolset_$(host_os)=gcc -endif - -define $(package)_build_cmds - CXX="$($(package)_cxx)" CXXFLAGS="$($(package)_cxxflags)" ./build.sh "$($(package)_toolset_$(host_os))" -endef - -define $(package)_stage_cmds - mkdir -p "$($(package)_staging_prefix_dir)"/bin/ && \ - cp b2 "$($(package)_staging_prefix_dir)"/bin/ -endef diff --git a/depends/packages/native_cctools.mk b/depends/packages/native_cctools.mk index 885207fce9..d169eb6723 100644 --- a/depends/packages/native_cctools.mk +++ b/depends/packages/native_cctools.mk @@ -16,6 +16,10 @@ define $(package)_set_vars $(package)_cxx=$(clangxx_prog) endef +define $(package)_preprocess_cmds + cp -f $(BASEDIR)/config.guess $(BASEDIR)/config.sub cctools +endef + define $(package)_config_cmds $($(package)_autoconf) endef diff --git a/depends/packages/native_clang.mk b/depends/packages/native_clang.mk index 25ac77c1a3..245269a9d3 100644 --- a/depends/packages/native_clang.mk +++ b/depends/packages/native_clang.mk @@ -2,11 +2,9 @@ package=native_clang $(package)_version=10.0.1 $(package)_download_path=https://github.com/llvm/llvm-project/releases/download/llvmorg-$($(package)_version) ifneq (,$(findstring aarch64,$(BUILD))) -$(package)_download_file=clang+llvm-$($(package)_version)-aarch64-linux-gnu.tar.xz $(package)_file_name=clang+llvm-$($(package)_version)-aarch64-linux-gnu.tar.xz $(package)_sha256_hash=90dc69a4758ca15cd0ffa45d07fbf5bf4309d47d2c7745a9f0735ecffde9c31f else -$(package)_download_file=clang+llvm-$($(package)_version)-x86_64-linux-gnu-ubuntu-16.04.tar.xz $(package)_file_name=clang+llvm-$($(package)_version)-x86_64-linux-gnu-ubuntu-16.04.tar.xz $(package)_sha256_hash=48b83ef827ac2c213d5b64f5ad7ed082c8bcb712b46644e0dc5045c6f462c231 endif @@ -18,15 +16,10 @@ endef define $(package)_stage_cmds mkdir -p $($(package)_staging_prefix_dir)/lib/clang/$($(package)_version)/include && \ mkdir -p $($(package)_staging_prefix_dir)/bin && \ - mkdir -p $($(package)_staging_prefix_dir)/include && \ cp bin/clang $($(package)_staging_prefix_dir)/bin/ && \ cp -P bin/clang++ $($(package)_staging_prefix_dir)/bin/ && \ cp bin/dsymutil $($(package)_staging_prefix_dir)/bin/$(host)-dsymutil && \ cp bin/llvm-config $($(package)_staging_prefix_dir)/bin/ && \ cp lib/libLTO.so $($(package)_staging_prefix_dir)/lib/ && \ - cp -rf lib/clang/$($(package)_version)/include/* $($(package)_staging_prefix_dir)/lib/clang/$($(package)_version)/include/ -endef - -define $(package)_postprocess_cmds - rmdir include + cp -r lib/clang/$($(package)_version)/include/* $($(package)_staging_prefix_dir)/lib/clang/$($(package)_version)/include/ endef diff --git a/depends/packages/native_libtapi.mk b/depends/packages/native_libtapi.mk index 60b898da5f..1633213a42 100644 --- a/depends/packages/native_libtapi.mk +++ b/depends/packages/native_libtapi.mk @@ -1,7 +1,6 @@ package=native_libtapi $(package)_version=664b8414f89612f2dfd35a9b679c345aa5389026 $(package)_download_path=https://github.com/tpoechtrager/apple-libtapi/archive -$(package)_download_file=$($(package)_version).tar.gz $(package)_file_name=$($(package)_version).tar.gz $(package)_sha256_hash=62e419c12d1c9fad67cc1cd523132bc00db050998337c734c15bc8d73cc02b61 diff --git a/depends/packages/packages.mk b/depends/packages/packages.mk index 9094e327ef..991db7f46e 100644 --- a/depends/packages/packages.mk +++ b/depends/packages/packages.mk @@ -1,10 +1,12 @@ packages:=boost libevent -qrencode_packages = qrencode +qrencode_linux_packages = qrencode +qrencode_android_packages = qrencode +qrencode_darwin_packages = qrencode +qrencode_mingw32_packages = qrencode -qt_linux_packages:=qt expat libxcb xcb_proto libXau xproto freetype fontconfig libxkbcommon +qt_linux_packages:=qt expat libxcb xcb_proto libXau xproto freetype fontconfig libxkbcommon libxcb_util libxcb_util_render libxcb_util_keysyms libxcb_util_image libxcb_util_wm qt_android_packages=qt - qt_darwin_packages=qt qt_mingw32_packages=qt @@ -19,9 +21,9 @@ natpmp_packages=libnatpmp multiprocess_packages = libmultiprocess capnp multiprocess_native_packages = native_libmultiprocess native_capnp -darwin_native_packages = native_ds_store native_mac_alias +usdt_linux_packages=systemtap -$(host_arch)_$(host_os)_native_packages += native_b2 +darwin_native_packages = native_ds_store native_mac_alias ifneq ($(build_os),darwin) darwin_native_packages += native_cctools native_libtapi native_libdmg-hfsplus diff --git a/depends/packages/qt.mk b/depends/packages/qt.mk index 0c3ef8c82f..9cdfd21d2c 100644 --- a/depends/packages/qt.mk +++ b/depends/packages/qt.mk @@ -1,24 +1,25 @@ -PACKAGE=qt -$(package)_version=5.12.11 -$(package)_download_path=https://download.qt.io/official_releases/qt/5.12/$($(package)_version)/submodules +package=qt +$(package)_version=5.15.2 +$(package)_download_path=https://download.qt.io/official_releases/qt/5.15/$($(package)_version)/submodules $(package)_suffix=everywhere-src-$($(package)_version).tar.xz $(package)_file_name=qtbase-$($(package)_suffix) -$(package)_sha256_hash=1c1b4e33137ca77881074c140d54c3c9747e845a31338cfe8680f171f0bc3a39 -$(package)_linux_dependencies=freetype fontconfig libxcb libxkbcommon +$(package)_sha256_hash=909fad2591ee367993a75d7e2ea50ad4db332f05e1c38dd7a5a274e156a4e0f8 +$(package)_linux_dependencies=freetype fontconfig libxcb libxkbcommon libxcb_util libxcb_util_render libxcb_util_keysyms libxcb_util_image libxcb_util_wm $(package)_qt_libs=corelib network widgets gui plugins testlib $(package)_linguist_tools = lrelease lupdate lconvert $(package)_patches = qt.pro qttools_src.pro $(package)_patches += fix_qt_pkgconfig.patch mac-qmake.conf fix_no_printer.patch no-xlib.patch -$(package)_patches+= fix_android_qmake_conf.patch fix_android_jni_static.patch dont_hardcode_pwd.patch -$(package)_patches+= no_sdk_version_check.patch -$(package)_patches+= fix_lib_paths.patch fix_android_pch.patch -$(package)_patches+= qtbase-moc-ignore-gcc-macro.patch fix_limits_header.patch +$(package)_patches += dont_use_avx_android_x86_64.patch dont_hardcode_x86_64.patch fix_montery_include.patch +$(package)_patches += fix_android_jni_static.patch dont_hardcode_pwd.patch +$(package)_patches += qtbase-moc-ignore-gcc-macro.patch fix_limits_header.patch +$(package)_patches += fix_bigsur_style.patch use_android_ndk23.patch +$(package)_patches += rcc_hardcode_timestamp.patch $(package)_qttranslations_file_name=qttranslations-$($(package)_suffix) -$(package)_qttranslations_sha256_hash=577b0668a777eb2b451c61e8d026d79285371597ce9df06b6dee6c814164b7c3 +$(package)_qttranslations_sha256_hash=d5788e86257b21d5323f1efd94376a213e091d1e5e03b45a95dd052b5f570db8 $(package)_qttools_file_name=qttools-$($(package)_suffix) -$(package)_qttools_sha256_hash=98b2aaca230458f65996f3534fd471d2ffd038dd58ac997c0589c06dc2385b4f +$(package)_qttools_sha256_hash=c189d0ce1ff7c739db9a3ace52ac3e24cb8fd6dbf234e49f075249b38f43c1cc $(package)_extra_sources = $($(package)_qttranslations_file_name) $(package)_extra_sources += $($(package)_qttools_file_name) @@ -29,14 +30,14 @@ $(package)_config_opts_release += -silent $(package)_config_opts_debug = -debug $(package)_config_opts_debug += -optimized-tools $(package)_config_opts += -bindir $(build_prefix)/bin -$(package)_config_opts += -c++std c++1z +$(package)_config_opts += -c++std c++17 $(package)_config_opts += -confirm-license $(package)_config_opts += -hostprefix $(build_prefix) $(package)_config_opts += -no-compile-examples $(package)_config_opts += -no-cups $(package)_config_opts += -no-egl $(package)_config_opts += -no-eglfs -$(package)_config_opts += -no-freetype +$(package)_config_opts += -no-evdev $(package)_config_opts += -no-gif $(package)_config_opts += -no-glib $(package)_config_opts += -no-icu @@ -51,6 +52,7 @@ $(package)_config_opts += -no-mtdev $(package)_config_opts += -no-openssl $(package)_config_opts += -no-openvg $(package)_config_opts += -no-reduce-relocations +$(package)_config_opts += -no-schannel $(package)_config_opts += -no-sctp $(package)_config_opts += -no-securetransport $(package)_config_opts += -no-sql-db2 @@ -64,6 +66,7 @@ $(package)_config_opts += -no-sql-sqlite $(package)_config_opts += -no-sql-sqlite2 $(package)_config_opts += -no-system-proxies $(package)_config_opts += -no-use-gold-linker +$(package)_config_opts += -no-zstd $(package)_config_opts += -nomake examples $(package)_config_opts += -nomake tests $(package)_config_opts += -nomake tools @@ -101,6 +104,7 @@ $(package)_config_opts += -no-feature-sqlmodel $(package)_config_opts += -no-feature-statemachine $(package)_config_opts += -no-feature-syntaxhighlighter $(package)_config_opts += -no-feature-textbrowser +$(package)_config_opts += -no-feature-textmarkdownwriter $(package)_config_opts += -no-feature-textodfwriter $(package)_config_opts += -no-feature-topleveldomain $(package)_config_opts += -no-feature-udpsocket @@ -116,6 +120,7 @@ $(package)_config_opts_darwin = -no-dbus $(package)_config_opts_darwin += -no-opengl $(package)_config_opts_darwin += -pch $(package)_config_opts_darwin += -no-feature-corewlan +$(package)_config_opts_darwin += -no-freetype $(package)_config_opts_darwin += QMAKE_MACOSX_DEPLOYMENT_TARGET=$(OSX_MIN_VERSION) ifneq ($(build_os),darwin) @@ -127,10 +132,12 @@ $(package)_config_opts_darwin += -device-option MAC_TARGET=$(host) $(package)_config_opts_darwin += -device-option XCODE_VERSION=$(XCODE_VERSION) endif -# for macOS on Apple Silicon (ARM) see https://bugreports.qt.io/browse/QTBUG-85279 +ifneq ($(build_arch),$(host_arch)) $(package)_config_opts_aarch64_darwin += -device-option QMAKE_APPLE_DEVICE_ARCHS=arm64 +$(package)_config_opts_x86_64_darwin += -device-option QMAKE_APPLE_DEVICE_ARCHS=x86_64 +endif -$(package)_config_opts_linux = -qt-xcb +$(package)_config_opts_linux = -xcb $(package)_config_opts_linux += -no-xcb-xlib $(package)_config_opts_linux += -no-feature-xlib $(package)_config_opts_linux += -system-freetype @@ -140,7 +147,11 @@ $(package)_config_opts_linux += -no-feature-vulkan $(package)_config_opts_linux += -dbus-runtime $(package)_config_opts_arm_linux += -platform linux-g++ -xplatform bitcoin-linux-g++ $(package)_config_opts_i686_linux = -xplatform linux-g++-32 +ifneq (,$(findstring -stdlib=libc++,$($(1)_cxx))) +$(package)_config_opts_x86_64_linux = -xplatform linux-clang-libc++ +else $(package)_config_opts_x86_64_linux = -xplatform linux-g++-64 +endif $(package)_config_opts_aarch64_linux = -xplatform linux-aarch64-gnu-g++ $(package)_config_opts_powerpc64_linux = -platform linux-g++ -xplatform bitcoin-linux-g++ $(package)_config_opts_powerpc64le_linux = -platform linux-g++ -xplatform bitcoin-linux-g++ @@ -149,8 +160,10 @@ $(package)_config_opts_s390x_linux = -platform linux-g++ -xplatform bitcoin-linu $(package)_config_opts_mingw32 = -no-opengl $(package)_config_opts_mingw32 += -no-dbus +$(package)_config_opts_mingw32 += -no-freetype $(package)_config_opts_mingw32 += -xplatform win32-g++ $(package)_config_opts_mingw32 += "QMAKE_CFLAGS = '$($(package)_cflags) $($(package)_cppflags)'" +$(package)_config_opts_mingw32 += "QMAKE_CXX = '$($(package)_cxx)'" $(package)_config_opts_mingw32 += "QMAKE_CXXFLAGS = '$($(package)_cflags) $($(package)_cppflags)'" $(package)_config_opts_mingw32 += "QMAKE_LFLAGS = '$($(package)_ldflags)'" $(package)_config_opts_mingw32 += -device-option CROSS_COMPILE="$(host)-" @@ -160,10 +173,7 @@ $(package)_config_opts_android = -xplatform android-clang $(package)_config_opts_android += -android-sdk $(ANDROID_SDK) $(package)_config_opts_android += -android-ndk $(ANDROID_NDK) $(package)_config_opts_android += -android-ndk-platform android-$(ANDROID_API_LEVEL) -$(package)_config_opts_android += -device-option CROSS_COMPILE="$(host)-" $(package)_config_opts_android += -egl -$(package)_config_opts_android += -qpa xcb -$(package)_config_opts_android += -no-eglfs $(package)_config_opts_android += -no-dbus $(package)_config_opts_android += -opengl es2 $(package)_config_opts_android += -qt-freetype @@ -176,7 +186,6 @@ $(package)_config_opts_android += -no-feature-vulkan $(package)_config_opts_aarch64_android += -android-arch arm64-v8a $(package)_config_opts_armv7a_android += -android-arch armeabi-v7a $(package)_config_opts_x86_64_android += -android-arch x86_64 -$(package)_config_opts_i686_android += -android-arch i686 endef define $(package)_fetch_cmds @@ -192,11 +201,11 @@ define $(package)_extract_cmds echo "$($(package)_qttools_sha256_hash) $($(package)_source_dir)/$($(package)_qttools_file_name)" >> $($(package)_extract_dir)/.$($(package)_file_name).hash && \ $(build_SHA256SUM) -c $($(package)_extract_dir)/.$($(package)_file_name).hash && \ mkdir qtbase && \ - tar --no-same-owner --strip-components=1 -xf $($(package)_source) -C qtbase && \ + $(build_TAR) --no-same-owner --strip-components=1 -xf $($(package)_source) -C qtbase && \ mkdir qttranslations && \ - tar --no-same-owner --strip-components=1 -xf $($(package)_source_dir)/$($(package)_qttranslations_file_name) -C qttranslations && \ + $(build_TAR) --no-same-owner --strip-components=1 -xf $($(package)_source_dir)/$($(package)_qttranslations_file_name) -C qttranslations && \ mkdir qttools && \ - tar --no-same-owner --strip-components=1 -xf $($(package)_source_dir)/$($(package)_qttools_file_name) -C qttools + $(build_TAR) --no-same-owner --strip-components=1 -xf $($(package)_source_dir)/$($(package)_qttools_file_name) -C qttools endef # Preprocessing steps work as follows: @@ -224,14 +233,16 @@ define $(package)_preprocess_cmds patch -p1 -i $($(package)_patch_dir)/dont_hardcode_pwd.patch && \ patch -p1 -i $($(package)_patch_dir)/fix_qt_pkgconfig.patch && \ patch -p1 -i $($(package)_patch_dir)/fix_no_printer.patch && \ - patch -p1 -i $($(package)_patch_dir)/fix_android_qmake_conf.patch && \ patch -p1 -i $($(package)_patch_dir)/fix_android_jni_static.patch && \ - patch -p1 -i $($(package)_patch_dir)/fix_android_pch.patch && \ patch -p1 -i $($(package)_patch_dir)/no-xlib.patch && \ - patch -p1 -i $($(package)_patch_dir)/no_sdk_version_check.patch && \ - patch -p1 -i $($(package)_patch_dir)/fix_lib_paths.patch && \ + patch -p1 -i $($(package)_patch_dir)/dont_use_avx_android_x86_64.patch && \ + patch -p1 -i $($(package)_patch_dir)/dont_hardcode_x86_64.patch && \ patch -p1 -i $($(package)_patch_dir)/qtbase-moc-ignore-gcc-macro.patch && \ patch -p1 -i $($(package)_patch_dir)/fix_limits_header.patch && \ + patch -p1 -i $($(package)_patch_dir)/fix_montery_include.patch && \ + patch -p1 -i $($(package)_patch_dir)/fix_bigsur_style.patch && \ + patch -p1 -i $($(package)_patch_dir)/use_android_ndk23.patch && \ + patch -p1 -i $($(package)_patch_dir)/rcc_hardcode_timestamp.patch && \ mkdir -p qtbase/mkspecs/macx-clang-linux &&\ cp -f qtbase/mkspecs/macx-clang/qplatformdefs.h qtbase/mkspecs/macx-clang-linux/ &&\ cp -f $($(package)_patch_dir)/mac-qmake.conf qtbase/mkspecs/macx-clang-linux/qmake.conf && \ @@ -248,7 +259,7 @@ endef define $(package)_config_cmds export PKG_CONFIG_SYSROOT_DIR=/ && \ export PKG_CONFIG_LIBDIR=$(host_prefix)/lib/pkgconfig && \ - export PKG_CONFIG_PATH=$(host_prefix)/share/pkgconfig && \ + export QT_MAC_SDK_NO_VERSION_CHECK=1 && \ cd qtbase && \ ./configure -top-level $($(package)_config_opts) endef diff --git a/depends/packages/sqlite.mk b/depends/packages/sqlite.mk index 5b3a61b239..126781ceeb 100644 --- a/depends/packages/sqlite.mk +++ b/depends/packages/sqlite.mk @@ -7,6 +7,13 @@ $(package)_sha256_hash=486748abfb16abd8af664e3a5f03b228e5f124682b0c942e157644bf6 define $(package)_set_vars $(package)_config_opts=--disable-shared --disable-readline --disable-dynamic-extensions --enable-option-checking $(package)_config_opts_linux=--with-pic +$(package)_config_opts_freebsd=--with-pic +$(package)_config_opts_netbsd=--with-pic +$(package)_config_opts_openbsd=--with-pic +endef + +define $(package)_preprocess_cmds + cp -f $(BASEDIR)/config.guess $(BASEDIR)/config.sub . endef define $(package)_config_cmds diff --git a/depends/packages/systemtap.mk b/depends/packages/systemtap.mk new file mode 100644 index 0000000000..833e75b978 --- /dev/null +++ b/depends/packages/systemtap.mk @@ -0,0 +1,12 @@ +package=systemtap +$(package)_version=4.5 +$(package)_download_path=https://sourceware.org/systemtap/ftp/releases/ +$(package)_file_name=$(package)-$($(package)_version).tar.gz +$(package)_sha256_hash=75078ed37e0dd2a769c9d1f9394170b2d9f4d7daa425f43ca80c13bad6cfc925 +$(package)_patches=remove_SDT_ASM_SECTION_AUTOGROUP_SUPPORT_check.patch + +define $(package)_preprocess_cmds + patch -p1 < $($(package)_patch_dir)/remove_SDT_ASM_SECTION_AUTOGROUP_SUPPORT_check.patch && \ + mkdir -p $($(package)_staging_prefix_dir)/include/sys && \ + cp includes/sys/sdt.h $($(package)_staging_prefix_dir)/include/sys/sdt.h +endef diff --git a/depends/packages/xcb_proto.mk b/depends/packages/xcb_proto.mk index 01203a0718..9be822506d 100644 --- a/depends/packages/xcb_proto.mk +++ b/depends/packages/xcb_proto.mk @@ -1,8 +1,8 @@ package=xcb_proto -$(package)_version=1.10 -$(package)_download_path=https://xcb.freedesktop.org/dist -$(package)_file_name=xcb-proto-$($(package)_version).tar.bz2 -$(package)_sha256_hash=7ef40ddd855b750bc597d2a435da21e55e502a0fefa85b274f2c922800baaf05 +$(package)_version=1.14.1 +$(package)_download_path=https://xorg.freedesktop.org/archive/individual/proto +$(package)_file_name=xcb-proto-$($(package)_version).tar.xz +$(package)_sha256_hash=f04add9a972ac334ea11d9d7eb4fc7f8883835da3e4859c9afa971efdf57fcc3 define $(package)_config_cmds $($(package)_autoconf) @@ -17,6 +17,5 @@ define $(package)_stage_cmds endef define $(package)_postprocess_cmds - find -name "*.pyc" -delete && \ - find -name "*.pyo" -delete + rm -rf lib/python*/site-packages/xcbgen/__pycache__ endef diff --git a/depends/packages/xproto.mk b/depends/packages/xproto.mk index 6bd867d02b..7a43c52faf 100644 --- a/depends/packages/xproto.mk +++ b/depends/packages/xproto.mk @@ -1,8 +1,8 @@ package=xproto -$(package)_version=7.0.26 +$(package)_version=7.0.31 $(package)_download_path=https://xorg.freedesktop.org/releases/individual/proto $(package)_file_name=$(package)-$($(package)_version).tar.bz2 -$(package)_sha256_hash=636162c1759805a5a0114a369dffdeccb8af8c859ef6e1445f26a4e6e046514f +$(package)_sha256_hash=c6f9747da0bd3a95f86b17fb8dd5e717c8f3ab7f0ece3ba1b247899ec1ef7747 define $(package)_set_vars $(package)_config_opts=--without-fop --without-xmlto --without-xsltproc --disable-specs diff --git a/depends/packages/zeromq.mk b/depends/packages/zeromq.mk index 3b7f3690a4..c74ae15b31 100644 --- a/depends/packages/zeromq.mk +++ b/depends/packages/zeromq.mk @@ -1,26 +1,32 @@ package=zeromq -$(package)_version=4.3.1 +$(package)_version=4.3.4 $(package)_download_path=https://github.com/zeromq/libzmq/releases/download/v$($(package)_version)/ $(package)_file_name=$(package)-$($(package)_version).tar.gz -$(package)_sha256_hash=bcbabe1e2c7d0eec4ed612e10b94b112dd5f06fcefa994a0c79a45d835cd21eb -$(package)_patches=remove_libstd_link.patch +$(package)_sha256_hash=c593001a89f5a85dd2ddf564805deb860e02471171b3f204944857336295c3e5 +$(package)_patches=remove_libstd_link.patch netbsd_kevent_void.patch define $(package)_set_vars - $(package)_config_opts=--without-docs --disable-shared --disable-curve --disable-curve-keygen --disable-perf + $(package)_config_opts = --without-docs --disable-shared --disable-valgrind + $(package)_config_opts += --disable-perf --disable-curve-keygen --disable-curve --disable-libbsd $(package)_config_opts += --without-libsodium --without-libgssapi_krb5 --without-pgm --without-norm --without-vmci $(package)_config_opts += --disable-libunwind --disable-radix-tree --without-gcov --disable-dependency-tracking $(package)_config_opts += --disable-Werror --disable-drafts --enable-option-checking $(package)_config_opts_linux=--with-pic + $(package)_config_opts_freebsd=--with-pic + $(package)_config_opts_netbsd=--with-pic + $(package)_config_opts_openbsd=--with-pic $(package)_config_opts_android=--with-pic - $(package)_cxxflags=-std=c++17 + $(package)_cxxflags+=-std=c++17 endef define $(package)_preprocess_cmds patch -p1 < $($(package)_patch_dir)/remove_libstd_link.patch && \ + patch -p1 < $($(package)_patch_dir)/netbsd_kevent_void.patch && \ cp -f $(BASEDIR)/config.guess $(BASEDIR)/config.sub config endef define $(package)_config_cmds + ./autogen.sh && \ $($(package)_autoconf) endef diff --git a/depends/patches/fontconfig/gperf_header_regen.patch b/depends/patches/fontconfig/gperf_header_regen.patch index 3ffd1674e0..b1a70d5fb1 100644 --- a/depends/patches/fontconfig/gperf_header_regen.patch +++ b/depends/patches/fontconfig/gperf_header_regen.patch @@ -13,12 +13,12 @@ diff --git a/src/Makefile.in b/src/Makefile.in index f4626ad..4ae1b00 100644 --- a/src/Makefile.in +++ b/src/Makefile.in -@@ -903,7 +903,7 @@ fcobjshash.gperf: fcobjshash.gperf.h fcobjs.h +@@ -912,7 +912,7 @@ ' - > $@.tmp && \ - mv -f $@.tmp $@ || ( $(RM) $@.tmp && false ) + mv -f $@.tmp fcobjshash.gperf && touch $@ || ( $(RM) $@.tmp && false ) --fcobjshash.h: fcobjshash.gperf +-fcobjshash.h: Makefile fcobjshash.gperf +fcobjshash.h: - $(AM_V_GEN) $(GPERF) -m 100 $< > $@.tmp && \ + $(AM_V_GEN) $(GPERF) --pic -m 100 fcobjshash.gperf > $@.tmp && \ mv -f $@.tmp $@ || ( $(RM) $@.tmp && false ) diff --git a/depends/patches/fontconfig/remove_char_width_usage.patch b/depends/patches/fontconfig/remove_char_width_usage.patch deleted file mode 100644 index 9f69081890..0000000000 --- a/depends/patches/fontconfig/remove_char_width_usage.patch +++ /dev/null @@ -1,62 +0,0 @@ -commit 28165a9b078583dc8e9e5c344510e37582284cef -Author: fanquake -Date: Mon Aug 17 20:35:42 2020 +0800 - - Remove usage of CHAR_WIDTH - - CHAR_WIDTH which is reserved and clashes with glibc 2.25+ - - See #10851. - -diff --git a/fontconfig/fontconfig.h b/fontconfig/fontconfig.h -index 5c72b22..843c532 100644 ---- a/fontconfig/fontconfig.h -+++ b/fontconfig/fontconfig.h -@@ -128,7 +128,7 @@ typedef int FcBool; - #define FC_USER_CACHE_FILE ".fonts.cache-" FC_CACHE_VERSION - - /* Adjust outline rasterizer */ --#define FC_CHAR_WIDTH "charwidth" /* Int */ -+#define FC_CHARWIDTH "charwidth" /* Int */ - #define FC_CHAR_HEIGHT "charheight"/* Int */ - #define FC_MATRIX "matrix" /* FcMatrix */ - -diff --git a/src/fcobjs.h b/src/fcobjs.h -index 1fc4f65..d27864b 100644 ---- a/src/fcobjs.h -+++ b/src/fcobjs.h -@@ -51,7 +51,7 @@ FC_OBJECT (DPI, FcTypeDouble, NULL) - FC_OBJECT (RGBA, FcTypeInteger, NULL) - FC_OBJECT (SCALE, FcTypeDouble, NULL) - FC_OBJECT (MINSPACE, FcTypeBool, NULL) --FC_OBJECT (CHAR_WIDTH, FcTypeInteger, NULL) -+FC_OBJECT (CHARWIDTH, FcTypeInteger, NULL) - FC_OBJECT (CHAR_HEIGHT, FcTypeInteger, NULL) - FC_OBJECT (MATRIX, FcTypeMatrix, NULL) - FC_OBJECT (CHARSET, FcTypeCharSet, FcCompareCharSet) -diff --git a/src/fcobjshash.gperf b/src/fcobjshash.gperf -index 80a0237..eb4ad84 100644 ---- a/src/fcobjshash.gperf -+++ b/src/fcobjshash.gperf -@@ -44,7 +44,7 @@ int id; - "rgba",FC_RGBA_OBJECT - "scale",FC_SCALE_OBJECT - "minspace",FC_MINSPACE_OBJECT --"charwidth",FC_CHAR_WIDTH_OBJECT -+"charwidth",FC_CHARWIDTH_OBJECT - "charheight",FC_CHAR_HEIGHT_OBJECT - "matrix",FC_MATRIX_OBJECT - "charset",FC_CHARSET_OBJECT -diff --git a/src/fcobjshash.h b/src/fcobjshash.h -index 5a4d1ea..4e66bb0 100644 ---- a/src/fcobjshash.h -+++ b/src/fcobjshash.h -@@ -284,7 +284,7 @@ FcObjectTypeLookup (register const char *str, register unsigned int len) - {(int)(long)&((struct FcObjectTypeNamePool_t *)0)->FcObjectTypeNamePool_str43,FC_CHARSET_OBJECT}, - {-1}, - #line 47 "fcobjshash.gperf" -- {(int)(long)&((struct FcObjectTypeNamePool_t *)0)->FcObjectTypeNamePool_str45,FC_CHAR_WIDTH_OBJECT}, -+ {(int)(long)&((struct FcObjectTypeNamePool_t *)0)->FcObjectTypeNamePool_str45,FC_CHARWIDTH_OBJECT}, - #line 48 "fcobjshash.gperf" - {(int)(long)&((struct FcObjectTypeNamePool_t *)0)->FcObjectTypeNamePool_str46,FC_CHAR_HEIGHT_OBJECT}, - #line 55 "fcobjshash.gperf" diff --git a/depends/patches/qt/dont_hardcode_x86_64.patch b/depends/patches/qt/dont_hardcode_x86_64.patch new file mode 100644 index 0000000000..5c1e030fa4 --- /dev/null +++ b/depends/patches/qt/dont_hardcode_x86_64.patch @@ -0,0 +1,119 @@ +macOS: Don't hard-code x86_64 as the architecture when using qmake + +Upstream commit: + - Qt 6.1: 9082cc8e8d5a6441dabe5e7a95bc0cd9085b95fe + +For other Qt branches see +https://codereview.qt-project.org/q/I70db7e4c27f0d3da5d0af33cb491d72c312d3fa8 + + +--- old/qtbase/configure.json ++++ new/qtbase/configure.json +@@ -244,11 +244,18 @@ + + "testTypeDependencies": { + "linkerSupportsFlag": [ "use_bfd_linker", "use_gold_linker", "use_lld_linker" ], +- "verifySpec": [ "shared", "use_bfd_linker", "use_gold_linker", "use_lld_linker", "compiler-flags", "qmakeargs", "commit" ], ++ "verifySpec": [ ++ "shared", ++ "use_bfd_linker", "use_gold_linker", "use_lld_linker", ++ "compiler-flags", "qmakeargs", ++ "simulator_and_device", ++ "thread", ++ "commit" ], + "compile": [ "verifyspec" ], + "detectPkgConfig": [ "cross_compile", "machineTuple" ], + "library": [ "pkg-config", "compiler-flags" ], +- "getPkgConfigVariable": [ "pkg-config" ] ++ "getPkgConfigVariable": [ "pkg-config" ], ++ "architecture" : [ "verifyspec" ] + }, + + "testTypeAliases": { +@@ -762,7 +769,7 @@ + }, + "architecture": { + "label": "Architecture", +- "output": [ "architecture" ] ++ "output": [ "architecture", "commitConfig" ] + }, + "pkg-config": { + "label": "Using pkg-config", +diff --git a/configure.pri b/configure.pri +index 49755f7abfd..8be9b10d7d4 100644 +--- old/qtbase/configure.pri ++++ new/qtbase/configure.pri +@@ -662,6 +662,13 @@ defineTest(qtConfOutput_commitOptions) { + write_file($$QT_BUILD_TREE/mkspecs/qdevice.pri, $${currentConfig}.output.devicePro)|error() + } + ++# Output is written after configuring each Qt module, ++# but some tests within a module might depend on the ++# configuration output of previous tests. ++defineTest(qtConfOutput_commitConfig) { ++ qtConfProcessOutput() ++} ++ + # type (empty or 'host'), option name, default value + defineTest(processQtPath) { + out_var = config.rel_input.$${2} +diff --git a/mkspecs/common/macx.conf b/mkspecs/common/macx.conf +index d16b77acb8e..4ba0a8eaa36 100644 +--- old/qtbase/mkspecs/common/macx.conf ++++ new/qtbase/mkspecs/common/macx.conf +@@ -6,7 +6,6 @@ QMAKE_PLATFORM += macos osx macx + QMAKE_MAC_SDK = macosx + + QMAKE_MACOSX_DEPLOYMENT_TARGET = 10.13 +-QMAKE_APPLE_DEVICE_ARCHS = x86_64 + + # Should be 10.15, but as long as the CI builds with + # older SDKs we have to keep this. +diff --git a/mkspecs/features/mac/default_post.prf b/mkspecs/features/mac/default_post.prf +index 92a9112bca6..d888731ec8d 100644 +--- old/qtbase/mkspecs/features/mac/default_post.prf ++++ new/qtbase/mkspecs/features/mac/default_post.prf +@@ -90,6 +90,11 @@ app_extension_api_only { + QMAKE_LFLAGS += $$QMAKE_CFLAGS_APPLICATION_EXTENSION + } + ++# Non-universal builds do not set QMAKE_APPLE_DEVICE_ARCHS, ++# so we pick it up from what the arch test resolved instead. ++isEmpty(QMAKE_APPLE_DEVICE_ARCHS): \ ++ QMAKE_APPLE_DEVICE_ARCHS = $$QT_ARCH ++ + macx-xcode { + qmake_pkginfo_typeinfo.name = QMAKE_PKGINFO_TYPEINFO + !isEmpty(QMAKE_PKGINFO_TYPEINFO): \ +@@ -145,9 +150,6 @@ macx-xcode { + simulator: VALID_SIMULATOR_ARCHS = $$QMAKE_APPLE_SIMULATOR_ARCHS + VALID_ARCHS = $$VALID_DEVICE_ARCHS $$VALID_SIMULATOR_ARCHS + +- isEmpty(VALID_ARCHS): \ +- error("QMAKE_APPLE_DEVICE_ARCHS or QMAKE_APPLE_SIMULATOR_ARCHS must contain at least one architecture") +- + single_arch: VALID_ARCHS = $$first(VALID_ARCHS) + + ACTIVE_ARCHS = $(filter $(EXPORT_VALID_ARCHS), $(ARCHS)) +diff --git a/mkspecs/features/toolchain.prf b/mkspecs/features/toolchain.prf +index efbe7c1e55b..8add6dc8043 100644 +--- old/qtbase/mkspecs/features/toolchain.prf ++++ new/qtbase/mkspecs/features/toolchain.prf +@@ -182,9 +182,14 @@ isEmpty($${target_prefix}.INCDIRS) { + # UIKit simulator platforms will see the device SDK's sysroot in + # QMAKE_DEFAULT_*DIRS, because they're handled in a single build pass. + darwin { +- # Clang doesn't pick up the architecture from the sysroot, and will +- # default to the host architecture, so we need to manually set it. +- cxx_flags += -arch $$QMAKE_APPLE_DEVICE_ARCHS ++ uikit { ++ # Clang doesn't automatically pick up the architecture, just because ++ # we're passing the iOS sysroot below, and we will end up building the ++ # test for the host architecture, resulting in linker errors when ++ # linking against the iOS libraries. We work around this by passing ++ # the architecture explicitly. ++ cxx_flags += -arch $$first(QMAKE_APPLE_DEVICE_ARCHS) ++ } + + uikit:macx-xcode: \ + cxx_flags += -isysroot $$sdk_path_device.value diff --git a/depends/patches/qt/dont_use_avx_android_x86_64.patch b/depends/patches/qt/dont_use_avx_android_x86_64.patch new file mode 100644 index 0000000000..b12ac8f4d0 --- /dev/null +++ b/depends/patches/qt/dont_use_avx_android_x86_64.patch @@ -0,0 +1,32 @@ +Android: don't use avx and avx2 when building for Android x86_64 + +AVX and AVX2 are not supported on x86_64 ABI. +See: + - https://developer.android.com/ndk/guides/abis#86-64 + - https://bugreports.qt.io/browse/QTBUG-86785 + +Upstream commits: + - Qt 6.0: ff1a44be33f4bc05d502a2ca49171e0408992f61 + - Qt 5.15: 8b2cc0f6deb038a4c9d4f0d9b690c7726bd809a9 + + +--- old/qtbase/configure.json ++++ new/qtbase/configure.json +@@ -1098,7 +1098,7 @@ + }, + "avx": { + "label": "AVX", +- "condition": "features.sse4_2 && tests.avx", ++ "condition": "features.sse4_2 && tests.avx && (!config.android || !arch.x86_64)", + "output": [ + "privateConfig", + { "type": "define", "name": "QT_COMPILER_SUPPORTS_AVX", "value": 1 } +@@ -1114,7 +1114,7 @@ + }, + "avx2": { + "label": "AVX2", +- "condition": "features.avx && tests.avx2", ++ "condition": "features.avx && tests.avx2 && (!config.android || !arch.x86_64)", + "output": [ + "privateConfig", + "privateFeature", diff --git a/depends/patches/qt/fix_android_jni_static.patch b/depends/patches/qt/fix_android_jni_static.patch index a186aeb8f6..bb64661761 100644 --- a/depends/patches/qt/fix_android_jni_static.patch +++ b/depends/patches/qt/fix_android_jni_static.patch @@ -1,6 +1,6 @@ --- old/qtbase/src/plugins/platforms/android/androidjnimain.cpp +++ new/qtbase/src/plugins/platforms/android/androidjnimain.cpp -@@ -898,6 +898,14 @@ +@@ -914,6 +914,14 @@ Q_DECL_EXPORT jint JNICALL JNI_OnLoad(JavaVM *vm, void */*reserved*/) __android_log_print(ANDROID_LOG_FATAL, "Qt", "registerNatives failed"); return -1; } diff --git a/depends/patches/qt/fix_android_pch.patch b/depends/patches/qt/fix_android_pch.patch deleted file mode 100644 index bed6e4bb63..0000000000 --- a/depends/patches/qt/fix_android_pch.patch +++ /dev/null @@ -1,10 +0,0 @@ ---- old/qtbase/mkspecs/common/android-base-head.conf -+++ new/qtbase/mkspecs/common/android-base-head.conf -@@ -73,6 +73,6 @@ CROSS_COMPILE = $$NDK_TOOLCHAIN_PATH/bin/$$NDK_TOOLS_PREFIX- - QMAKE_PCH_OUTPUT_EXT = .gch - - QMAKE_CFLAGS_PRECOMPILE = -x c-header -c ${QMAKE_PCH_INPUT} -o ${QMAKE_PCH_OUTPUT} --QMAKE_CFLAGS_USE_PRECOMPILE = -include ${QMAKE_PCH_OUTPUT_BASE} -+QMAKE_CFLAGS_USE_PRECOMPILE = -include-pch ${QMAKE_PCH_OUTPUT} - QMAKE_CXXFLAGS_PRECOMPILE = -x c++-header -c ${QMAKE_PCH_INPUT} -o ${QMAKE_PCH_OUTPUT} - QMAKE_CXXFLAGS_USE_PRECOMPILE = $$QMAKE_CFLAGS_USE_PRECOMPILE diff --git a/depends/patches/qt/fix_android_qmake_conf.patch b/depends/patches/qt/fix_android_qmake_conf.patch deleted file mode 100644 index 3a8753fd1d..0000000000 --- a/depends/patches/qt/fix_android_qmake_conf.patch +++ /dev/null @@ -1,10 +0,0 @@ ---- old/qtbase/mkspecs/android-clang/qmake.conf -+++ new/qtbase/mkspecs/android-clang/qmake.conf -@@ -47,7 +47,7 @@ ANDROID_STDCPP_PATH = $$ANDROID_SOURCES_CXX_STL_LIBDIR/libc++_shared.so - ANDROID_USE_LLVM = true - - exists($$ANDROID_SOURCES_CXX_STL_LIBDIR/libc++.so): \ -- ANDROID_CXX_STL_LIBS = -lc++ -+ ANDROID_CXX_STL_LIBS = -lc++_shared - else: \ - ANDROID_CXX_STL_LIBS = $$ANDROID_SOURCES_CXX_STL_LIBDIR/libc++.so.$$replace(ANDROID_PLATFORM, "android-", "") diff --git a/depends/patches/qt/fix_bigsur_style.patch b/depends/patches/qt/fix_bigsur_style.patch new file mode 100644 index 0000000000..0f6c848bb0 --- /dev/null +++ b/depends/patches/qt/fix_bigsur_style.patch @@ -0,0 +1,90 @@ +Fix rendering in macOS BigSur + +See: https://bugreports.qt.io/browse/QTBUG-86513. + +Upstream commits (combined in this patch): + - Qt 6.0: 40fb97e97f550b8afd13ecc3a038d9d0c2d82bbb + - Qt 6.0: 3857f104cac127f62e64e55a20613f0ac2e6b843 + - Qt 6.1: abee4cdd5925a8513f51784754fca8fa35031732 + +--- old/qtbase/src/plugins/styles/mac/qmacstyle_mac.mm ++++ new/qtbase/src/plugins/styles/mac/qmacstyle_mac.mm +@@ -3870,6 +3870,7 @@ void QMacStyle::drawControl(ControlElement ce, const QStyleOption *opt, QPainter + const auto cs = d->effectiveAquaSizeConstrain(opt, w); + // Extra hacks to get the proper pressed appreance when not selected or selected and inactive + const bool needsInactiveHack = (!isActive && isSelected); ++ const bool isBigSurOrAbove = QOperatingSystemVersion::current() >= QOperatingSystemVersion::MacOSBigSur; + const auto ct = !needsInactiveHack && (isSelected || tp == QStyleOptionTab::OnlyOneTab) ? + QMacStylePrivate::Button_PushButton : + QMacStylePrivate::Button_PopupButton; +@@ -3878,6 +3879,12 @@ void QMacStyle::drawControl(ControlElement ce, const QStyleOption *opt, QPainter + auto *pb = static_cast(d->cocoaControl(cw)); + + auto vOffset = isPopupButton ? 1 : 2; ++ if (isBigSurOrAbove) { ++ // Make it 1, otherwise, offset is very visible compared ++ // to selected tab (which is not a popup button). ++ vOffset = 1; ++ } ++ + if (tabDirection == QMacStylePrivate::East) + vOffset -= 1; + const auto outerAdjust = isPopupButton ? 1 : 4; +@@ -3894,9 +3901,22 @@ void QMacStyle::drawControl(ControlElement ce, const QStyleOption *opt, QPainter + frameRect = frameRect.adjusted(-innerAdjust, 0, outerAdjust, 0); + else + frameRect = frameRect.adjusted(-outerAdjust, 0, innerAdjust, 0); ++ ++ if (isSelected && isBigSurOrAbove) { ++ // 1 pixed of 'roundness' is still visible on the right ++ // (the left is OK, it's rounded). ++ frameRect = frameRect.adjusted(0, 0, 1, 0); ++ } ++ + break; + case QStyleOptionTab::Middle: + frameRect = frameRect.adjusted(-innerAdjust, 0, innerAdjust, 0); ++ ++ if (isSelected && isBigSurOrAbove) { ++ // 1 pixel of 'roundness' is still visible on both ++ // sides - left and right. ++ frameRect = frameRect.adjusted(-1, 0, 1, 0); ++ } + break; + case QStyleOptionTab::End: + // Pressed state hack: tweak adjustments in preparation for flip below +@@ -3904,6 +3924,11 @@ void QMacStyle::drawControl(ControlElement ce, const QStyleOption *opt, QPainter + frameRect = frameRect.adjusted(-innerAdjust, 0, outerAdjust, 0); + else + frameRect = frameRect.adjusted(-outerAdjust, 0, innerAdjust, 0); ++ ++ if (isSelected && isBigSurOrAbove) { ++ // 1 pixel of 'roundness' is still visible on the left. ++ frameRect = frameRect.adjusted(-1, 0, 0, 0); ++ } + break; + case QStyleOptionTab::OnlyOneTab: + frameRect = frameRect.adjusted(-outerAdjust, 0, outerAdjust, 0); +@@ -3951,7 +3976,10 @@ void QMacStyle::drawControl(ControlElement ce, const QStyleOption *opt, QPainter + NSPopUpArrowPosition oldPosition = NSPopUpArrowAtCenter; + NSPopUpButtonCell *pbCell = nil; + auto rAdjusted = r; +- if (isPopupButton && tp == QStyleOptionTab::OnlyOneTab) { ++ if (isPopupButton && (tp == QStyleOptionTab::OnlyOneTab || isBigSurOrAbove)) { ++ // Note: starting from macOS BigSur NSPopupButton has this ++ // arrow 'button' in a different place and it became ++ // quite visible 'in between' inactive tabs. + pbCell = static_cast(pb.cell); + oldPosition = pbCell.arrowPosition; + pbCell.arrowPosition = NSPopUpNoArrow; +@@ -3959,6 +3987,10 @@ void QMacStyle::drawControl(ControlElement ce, const QStyleOption *opt, QPainter + // NSPopUpButton in this state is smaller. + rAdjusted.origin.x -= 3; + rAdjusted.size.width += 6; ++ if (isBigSurOrAbove) { ++ if (tp == QStyleOptionTab::End) ++ rAdjusted.origin.x -= 2; ++ } + } + } + diff --git a/depends/patches/qt/fix_lib_paths.patch b/depends/patches/qt/fix_lib_paths.patch deleted file mode 100644 index d1a15373f4..0000000000 --- a/depends/patches/qt/fix_lib_paths.patch +++ /dev/null @@ -1,193 +0,0 @@ ---- old/qtbase/mkspecs/common/mac.conf -+++ new/qtbase/mkspecs/common/mac.conf -@@ -14,7 +14,6 @@ - - QMAKE_RESOURCE = /Developer/Tools/Rez - QMAKE_EXTENSION_SHLIB = dylib --QMAKE_EXTENSIONS_AUX_SHLIB = tbd - QMAKE_LIBDIR = - - # sdk.prf will prefix the proper SDK sysroot - ---- old/qtbase/mkspecs/features/qmake_use.prf -+++ new/qtbase/mkspecs/features/qmake_use.prf -@@ -22,6 +22,8 @@ - !defined(QMAKE_LIBS_$$nu, var): \ - error("Library '$$lower($$replace(nu, _, -))' is not defined.") - -+ QMAKE_LIBDIR += $$eval(QMAKE_LIBDIR_$$nu) -+ - debug: \ - LIBS$${suffix} += $$eval(QMAKE_LIBS_$${nu}_DEBUG) $$eval(QMAKE_LIBS_$$nu) - else: \ - ---- old/qtbase/mkspecs/features/qt_configure.prf -+++ new/qtbase/mkspecs/features/qt_configure.prf -@@ -526,98 +526,23 @@ - return($$sysrootified) - } - --# libs-var, libs, in-paths, out-paths-var -+# libs-var, libs, in-paths - defineTest(qtConfResolveLibs) { -- ret = true -- paths = $$3 -- out = -- copy = false -- for (l, 2) { -- $$copy { -- copy = false -- out += $$l -- } else: equals(l, "-s") { -- # em++ flag to link libraries from emscripten-ports; passed on literally. -- copy = true -- out += $$l -- } else: contains(l, "^-L.*") { -- lp = $$replace(l, "^-L", ) -- gcc: lp = $$qtGccSysrootifiedPath($$lp) -- !exists($$lp/.) { -- qtLog("Library path $$val_escape(lp) is invalid.") -- ret = false -- } else { -- paths += $$lp -- } -- } else: contains(l, "^-l.*") { -- lib = $$replace(l, "^-l", ) -- lcan = -- integrity:contains(lib, "^.*\\.a") { -- # INTEGRITY compiler searches for exact filename -- # if -l argument has .a suffix -- lcan += $${lib} -- } else: contains(lib, "^:.*") { -- # Use exact filename when -l:filename syntax is used. -- lib ~= s/^:// -- lcan += $${lib} -- } else: unix { -- # Under UNIX, we look for actual shared libraries, in addition -- # to static ones. -- shexts = $$QMAKE_EXTENSION_SHLIB $$QMAKE_EXTENSIONS_AUX_SHLIB -- for (ext, shexts) { -- lcan += $${QMAKE_PREFIX_SHLIB}$${lib}.$${ext} -- } -- lcan += \ -- $${QMAKE_PREFIX_STATICLIB}$${lib}.$${QMAKE_EXTENSION_STATICLIB} -- } else { -- # Under Windows, we look only for static libraries, as even for DLLs -- # one actually links against a static import library. -- mingw { -- lcan += \ -- # MinGW supports UNIX-style library naming in addition to -- # the MSVC style. -- lib$${lib}.dll.a lib$${lib}.a \ -- # Fun fact: prefix-less libraries are also supported. -- $${lib}.dll.a $${lib}.a -- } -- lcan += $${lib}.lib -- } -- l = $$qtConfFindInPathList($$lcan, $$paths $$EXTRA_LIBDIR $$QMAKE_DEFAULT_LIBDIRS) -- isEmpty(l) { -- qtLog("None of [$$val_escape(lcan)] found in [$$val_escape(paths)] and global paths.") -- ret = false -- } else { -- out += $$l -- } -- } else { -- out += $$l -- } -- } -- $$1 = $$out -+ for (path, 3): \ -+ pre_lflags += -L$$path -+ $$1 = $$pre_lflags $$2 - export($$1) -- !isEmpty(4) { -- $$4 = $$paths -- export($$4) -- } -- return($$ret) --} -- --# source-var --defineTest(qtConfResolveAllLibs) { -- ret = true -- !qtConfResolveLibs($${1}.libs, $$eval($${1}.libs), , $${1}.libdirs): \ -- ret = false -- for (b, $${1}.builds._KEYS_): \ -- !qtConfResolveLibs($${1}.builds.$${b}, $$eval($${1}.builds.$${b}), $$eval($${1}.libdirs), ): \ -- ret = false -- return($$ret) -+ return(true) - } - - # libs-var, in-paths, libs - defineTest(qtConfResolvePathLibs) { - ret = true -- gcc: 2 = $$qtGccSysrootifiedPaths($$2) -- for (libdir, 2) { -+ gcc: \ -+ local_paths = $$qtGccSysrootifiedPaths($$2) -+ else: \ -+ local_paths = $$2 -+ for (libdir, local_paths) { - !exists($$libdir/.) { - qtLog("Library path $$val_escape(libdir) is invalid.") - ret = false -@@ -667,8 +592,11 @@ - # includes-var, in-paths, test-object-var - defineTest(qtConfResolvePathIncs) { - ret = true -- gcc: 2 = $$qtGccSysrootifiedPaths($$2) -- for (incdir, 2) { -+ gcc: \ -+ local_paths = $$qtGccSysrootifiedPaths($$2) -+ else: \ -+ local_paths = $$2 -+ for (incdir, local_paths) { - !exists($$incdir/.) { - qtLog("Include path $$val_escape(incdir) is invalid.") - ret = false -@@ -727,6 +655,7 @@ - vars += $$eval(config.commandline.rev_assignments.$${iv}) - defined(config.input.$${iv}, var) { - eval($${1}.builds.$${b} = $$eval(config.input.$${iv})) -+ export($${1}.builds.$${b}) - $${1}.builds._KEYS_ *= $${b} - any = true - } else { -@@ -741,11 +670,14 @@ - export($${1}.builds._KEYS_) - # we also reset the generic libs, to avoid surprises. - $${1}.libs = -+ export($${1}.libs) - } - - # direct libs. overwrites inline libs. -- defined(config.input.$${input}.libs, var): \ -+ defined(config.input.$${input}.libs, var) { - eval($${1}.libs = $$eval(config.input.$${input}.libs)) -+ export($${1}.libs) -+ } - - includes = $$eval(config.input.$${input}.incdir) - -@@ -754,6 +686,7 @@ - !isEmpty(prefix) { - includes += $$prefix/include - $${1}.libs = -L$$prefix/lib $$eval($${1}.libs) -+ export($${1}.libs) - } - - libdir = $$eval(config.input.$${input}.libdir) -@@ -762,11 +695,9 @@ - for (ld, libdir): \ - libs += -L$$ld - $${1}.libs = $$libs $$eval($${1}.libs) -+ export($${1}.libs) - } - -- !qtConfResolveAllLibs($$1): \ -- return(false) -- - !qtConfResolvePathIncs($${1}.includedir, $$includes, $$2): \ - return(false) - diff --git a/depends/patches/qt/fix_limits_header.patch b/depends/patches/qt/fix_limits_header.patch index e4313770e5..cb5a8cd1b5 100644 --- a/depends/patches/qt/fix_limits_header.patch +++ b/depends/patches/qt/fix_limits_header.patch @@ -19,17 +19,16 @@ Upstream commits: #include #include ---- old/qtbase/src/corelib/tools/qbytearraymatcher.h -+++ new/qtbase/src/corelib/tools/qbytearraymatcher.h -@@ -42,6 +42,8 @@ - - #include +--- old/qtbase/src/corelib/global/qfloat16.h ++++ new/qtbase/src/corelib/global/qfloat16.h +@@ -43,6 +43,7 @@ + #include + #include +#include -+ - QT_BEGIN_NAMESPACE - + #include + #if defined(QT_COMPILER_SUPPORTS_F16C) && defined(__AVX2__) && !defined(__F16C__) --- old/qtbase/src/tools/moc/generator.cpp +++ new/qtbase/src/tools/moc/generator.cpp @@ -42,3 +41,24 @@ Upstream commits: #include #include +--- old/qtbase/src/corelib/text/qbytearraymatcher.h ++++ new/qtbase/src/corelib/text/qbytearraymatcher.h +@@ -42,6 +42,8 @@ + + #include + ++#include ++ + QT_BEGIN_NAMESPACE + + +--- old/qtbase/src/corelib/tools/qoffsetstringarray_p.h ++++ new/qtbase/src/corelib/tools/qoffsetstringarray_p.h +@@ -55,6 +55,7 @@ + + #include + #include ++#include + + QT_BEGIN_NAMESPACE + diff --git a/depends/patches/qt/fix_montery_include.patch b/depends/patches/qt/fix_montery_include.patch new file mode 100644 index 0000000000..38b700addf --- /dev/null +++ b/depends/patches/qt/fix_montery_include.patch @@ -0,0 +1,21 @@ +From dece6f5840463ae2ddf927d65eb1b3680e34a547 +From: Øystein Heskestad +Date: Wed, 27 Oct 2021 13:07:46 +0200 +Subject: [PATCH] Add missing macOS header file that was indirectly included before + +See: https://bugreports.qt.io/browse/QTBUG-97855 + +Upstream Commits: + - Qt 6.2: c884bf138a21dd7320e35cef34d24e22e74d7ce0 + +diff --git a/qtbase/src/plugins/platforms/cocoa/qiosurfacegraphicsbuffer.h b/qtbase/src/plugins/platforms/cocoa/qiosurfacegraphicsbuffer.h +index e070ba97..07c75b04 100644 +--- a/qtbase/src/plugins/platforms/cocoa/qiosurfacegraphicsbuffer.h ++++ b/qtbase/src/plugins/platforms/cocoa/qiosurfacegraphicsbuffer.h +@@ -40,6 +40,7 @@ + #ifndef QIOSURFACEGRAPHICSBUFFER_H + #define QIOSURFACEGRAPHICSBUFFER_H + ++#include + #include + #include diff --git a/depends/patches/qt/fix_qt_pkgconfig.patch b/depends/patches/qt/fix_qt_pkgconfig.patch index a5de2b4b9e..73f4d89f73 100644 --- a/depends/patches/qt/fix_qt_pkgconfig.patch +++ b/depends/patches/qt/fix_qt_pkgconfig.patch @@ -4,20 +4,8 @@ load(qt_targets) # this builds on top of qt_common --!internal_module:if(unix|mingw) { +-!internal_module:if(unix|mingw):!if(darwin:debug_and_release:CONFIG(debug, debug|release)) { +if(unix|mingw):!if(darwin:debug_and_release:CONFIG(debug, debug|release)) { CONFIG += create_pc QMAKE_PKGCONFIG_DESTDIR = pkgconfig host_build: \ -@@ -284,9 +284,9 @@ load(qt_targets) - QMAKE_PKGCONFIG_CFLAGS = -D$$MODULE_DEFINE -I${includedir}/$$MODULE_INCNAME - } - QMAKE_PKGCONFIG_NAME = $$replace(TARGET, ^Qt, "Qt$$QT_MAJOR_VERSION ") -- QMAKE_PKGCONFIG_FILE = $$replace(TARGET, ^Qt, Qt$$QT_MAJOR_VERSION) -+ QMAKE_PKGCONFIG_FILE = $$replace(TARGET, ^Qt, Qt$$QT_MAJOR_VERSION)$$qtPlatformTargetSuffix() - for(i, MODULE_DEPENDS): \ -- QMAKE_PKGCONFIG_REQUIRES += $$replace(QT.$${i}.name, ^Qt, Qt$$section(QT.$${i}.VERSION, ., 0, 0)) -+ QMAKE_PKGCONFIG_REQUIRES += $$replace(QT.$${i}.name, ^Qt, Qt$$section(QT.$${i}.VERSION, ., 0, 0))$$qtPlatformTargetSuffix() - isEmpty(QMAKE_PKGCONFIG_DESCRIPTION): \ - QMAKE_PKGCONFIG_DESCRIPTION = $$replace(TARGET, ^Qt, "Qt ") module - !isEmpty(lib_replace0.match) { diff --git a/depends/patches/qt/mac-qmake.conf b/depends/patches/qt/mac-qmake.conf index 190ab7a160..e4bfaa1463 100644 --- a/depends/patches/qt/mac-qmake.conf +++ b/depends/patches/qt/mac-qmake.conf @@ -13,7 +13,6 @@ QMAKE_MAC_SDK.macosx.Path = $${MAC_SDK_PATH} QMAKE_MAC_SDK.macosx.platform_name = macosx QMAKE_MAC_SDK.macosx.SDKVersion = $${MAC_SDK_VERSION} QMAKE_MAC_SDK.macosx.PlatformPath = /phony -QMAKE_APPLE_DEVICE_ARCHS=x86_64 !host_build: QMAKE_CFLAGS += -target $${MAC_TARGET} !host_build: QMAKE_OBJECTIVE_CFLAGS += $$QMAKE_CFLAGS !host_build: QMAKE_CXXFLAGS += $$QMAKE_CFLAGS diff --git a/depends/patches/qt/no-xlib.patch b/depends/patches/qt/no-xlib.patch index f4a6f09ee4..d6846aaca2 100644 --- a/depends/patches/qt/no-xlib.patch +++ b/depends/patches/qt/no-xlib.patch @@ -22,7 +22,7 @@ index 7c62c2e2b3..c05c6c0a07 100644 #include #include -@@ -391,6 +393,7 @@ void QXcbCursor::changeCursor(QCursor *cursor, QWindow *window) +@@ -391,6 +391,7 @@ void QXcbCursor::changeCursor(QCursor *cursor, QWindow *window) xcb_flush(xcb_connection()); } @@ -30,7 +30,7 @@ index 7c62c2e2b3..c05c6c0a07 100644 static int cursorIdForShape(int cshape) { int cursorId = 0; -@@ -444,6 +447,7 @@ static int cursorIdForShape(int cshape) +@@ -444,6 +445,7 @@ static int cursorIdForShape(int cshape) } return cursorId; } @@ -38,7 +38,7 @@ index 7c62c2e2b3..c05c6c0a07 100644 xcb_cursor_t QXcbCursor::createNonStandardCursor(int cshape) { -@@ -556,7 +560,9 @@ static xcb_cursor_t loadCursor(void *dpy, int cshape) +@@ -556,7 +558,9 @@ static xcb_cursor_t loadCursor(void *dpy, int cshape) xcb_cursor_t QXcbCursor::createFontCursor(int cshape) { xcb_connection_t *conn = xcb_connection(); @@ -48,16 +48,15 @@ index 7c62c2e2b3..c05c6c0a07 100644 xcb_cursor_t cursor = XCB_NONE; // Try Xcursor first -@@ -585,7 +591,7 @@ xcb_cursor_t QXcbCursor::createFontCursor(int cshape) - +@@ -586,6 +590,7 @@ xcb_cursor_t QXcbCursor::createFontCursor(int cshape) // Non-standard X11 cursors are created from bitmaps cursor = createNonStandardCursor(cshape); -- + +#if QT_CONFIG(xcb_xlib) && QT_CONFIG(library) // Create a glpyh cursor if everything else failed if (!cursor && cursorId) { cursor = xcb_generate_id(conn); -@@ -593,6 +599,7 @@ xcb_cursor_t QXcbCursor::createFontCursor(int cshape) +@@ -593,6 +598,7 @@ xcb_cursor_t QXcbCursor::createFontCursor(int cshape) cursorId, cursorId + 1, 0xFFFF, 0xFFFF, 0xFFFF, 0, 0, 0); } diff --git a/depends/patches/qt/no_sdk_version_check.patch b/depends/patches/qt/no_sdk_version_check.patch deleted file mode 100644 index b16635b572..0000000000 --- a/depends/patches/qt/no_sdk_version_check.patch +++ /dev/null @@ -1,20 +0,0 @@ -commit f5eb142cd04be2bc4ca610ed3b5b7e8ce3520ee3 -Author: fanquake -Date: Tue Jan 5 16:08:49 2021 +0800 - - Don't invoke macOS SDK version checking - - This tries to use xcrun which is not available when cross-compiling. - -diff --git a/qtbase/mkspecs/features/mac/default_post.prf b/qtbase/mkspecs/features/mac/default_post.prf -index 92a9112bca6..447e186eb26 100644 ---- a/qtbase/mkspecs/features/mac/default_post.prf -+++ b/qtbase/mkspecs/features/mac/default_post.prf -@@ -8,7 +8,6 @@ contains(TEMPLATE, .*app) { - !macx-xcode:if(isEmpty(BUILDS)|build_pass) { - # Detect changes to the platform SDK - QMAKE_EXTRA_VARIABLES += QMAKE_MAC_SDK QMAKE_MAC_SDK_VERSION QMAKE_XCODE_DEVELOPER_PATH -- QMAKE_EXTRA_INCLUDES += $$shell_quote($$PWD/sdk.mk) - } - - # Detect incompatible SDK versions diff --git a/depends/patches/qt/qtbase-moc-ignore-gcc-macro.patch b/depends/patches/qt/qtbase-moc-ignore-gcc-macro.patch index 0358bea6e9..f0c14a9400 100644 --- a/depends/patches/qt/qtbase-moc-ignore-gcc-macro.patch +++ b/depends/patches/qt/qtbase-moc-ignore-gcc-macro.patch @@ -7,7 +7,7 @@ Upstream report: https://bugreports.qt.io/browse/QTBUG-83160 diff --git a/qtbase/src/tools/moc/main.cpp b/qtbase/src/tools/moc/main.cpp --- a/qtbase/src/tools/moc/main.cpp +++ b/qtbase/src/tools/moc/main.cpp -@@ -188,6 +188,7 @@ int runMoc(int argc, char **argv) +@@ -238,6 +238,7 @@ int runMoc(int argc, char **argv) dummyVariadicFunctionMacro.arguments += Symbol(0, PP_IDENTIFIER, "__VA_ARGS__"); pp.macros["__attribute__"] = dummyVariadicFunctionMacro; pp.macros["__declspec"] = dummyVariadicFunctionMacro; diff --git a/depends/patches/qt/rcc_hardcode_timestamp.patch b/depends/patches/qt/rcc_hardcode_timestamp.patch new file mode 100644 index 0000000000..03f3897975 --- /dev/null +++ b/depends/patches/qt/rcc_hardcode_timestamp.patch @@ -0,0 +1,24 @@ +Hardcode last modified timestamp in Qt RCC + +This change allows the already built qt package to be reused even with +the SOURCE_DATE_EPOCH variable set, e.g., for Guix builds. + + +--- old/qtbase/src/tools/rcc/rcc.cpp ++++ new/qtbase/src/tools/rcc/rcc.cpp +@@ -227,14 +227,7 @@ void RCCFileInfo::writeDataInfo(RCCResourceLibrary &lib) + + if (lib.formatVersion() >= 2) { + // last modified time stamp +- const QDateTime lastModified = m_fileInfo.lastModified(); +- quint64 lastmod = quint64(lastModified.isValid() ? lastModified.toMSecsSinceEpoch() : 0); +- static const quint64 sourceDate = 1000 * qgetenv("QT_RCC_SOURCE_DATE_OVERRIDE").toULongLong(); +- if (sourceDate != 0) +- lastmod = sourceDate; +- static const quint64 sourceDate2 = 1000 * qgetenv("SOURCE_DATE_EPOCH").toULongLong(); +- if (sourceDate2 != 0) +- lastmod = sourceDate2; ++ quint64 lastmod = quint64(1); + lib.writeNumber8(lastmod); + if (text || pass1) + lib.writeChar('\n'); diff --git a/depends/patches/qt/use_android_ndk23.patch b/depends/patches/qt/use_android_ndk23.patch new file mode 100644 index 0000000000..85b8690218 --- /dev/null +++ b/depends/patches/qt/use_android_ndk23.patch @@ -0,0 +1,13 @@ +Use Android NDK r23 LTS + +--- old/qtbase/mkspecs/features/android/default_pre.prf ++++ new/qtbase/mkspecs/features/android/default_pre.prf +@@ -73,7 +73,7 @@ else: equals(QT_ARCH, x86_64): CROSS_COMPILE = $$NDK_LLVM_PATH/bin/x86_64-linux- + else: equals(QT_ARCH, arm64-v8a): CROSS_COMPILE = $$NDK_LLVM_PATH/bin/aarch64-linux-android- + else: CROSS_COMPILE = $$NDK_LLVM_PATH/bin/arm-linux-androideabi- + +-QMAKE_RANLIB = $${CROSS_COMPILE}ranlib ++QMAKE_RANLIB = $$NDK_LLVM_PATH/bin/llvm-ranlib + QMAKE_LINK_SHLIB = $$QMAKE_LINK + QMAKE_LFLAGS = + diff --git a/depends/patches/systemtap/remove_SDT_ASM_SECTION_AUTOGROUP_SUPPORT_check.patch b/depends/patches/systemtap/remove_SDT_ASM_SECTION_AUTOGROUP_SUPPORT_check.patch new file mode 100644 index 0000000000..eae0cf72d6 --- /dev/null +++ b/depends/patches/systemtap/remove_SDT_ASM_SECTION_AUTOGROUP_SUPPORT_check.patch @@ -0,0 +1,31 @@ +commit b92d4c121486f3c6e8a2cea537c53eb09894479a +Author: 0xb10c <0xb10c@gmail.com> +Date: Tue Dec 7 11:02:07 2021 +0100 + + Remove _SDT_ASM_SECTION_AUTOGROUP_SUPPORT check + + We assume that the assembler supports "?" in .pushsection directives. + This enables us to skip configure and make. + + See https://github.com/bitcoin/bitcoin/issues/23297. + +diff --git a/includes/sys/sdt.h b/includes/sys/sdt.h +index 97766e710..352b4ee25 100644 +--- a/includes/sys/sdt.h ++++ b/includes/sys/sdt.h +@@ -230,12 +230,10 @@ __extension__ extern unsigned long long __sdt_unsp; + nice with code in COMDAT sections, which comes up in C++ code. + Without that assembler support, some combinations of probe placements + in certain kinds of C++ code may produce link-time errors. */ +-#include "sdt-config.h" +-#if _SDT_ASM_SECTION_AUTOGROUP_SUPPORT ++/* PATCH: We assume that the assembler supports the feature. This ++ enables us to skip configure and make. In turn, this means we ++ require fewer dependencies and have shorter depend build times. */ + # define _SDT_ASM_AUTOGROUP "?" +-#else +-# define _SDT_ASM_AUTOGROUP "" +-#endif + + #define _SDT_ASM_BODY(provider, name, pack_args, args) \ + _SDT_ASM_1(990: _SDT_NOP) \ diff --git a/depends/patches/zeromq/netbsd_kevent_void.patch b/depends/patches/zeromq/netbsd_kevent_void.patch new file mode 100644 index 0000000000..845c6bdda6 --- /dev/null +++ b/depends/patches/zeromq/netbsd_kevent_void.patch @@ -0,0 +1,57 @@ +commit 129137d5182967dbfcfec66bad843df2a992a78f +Author: fanquake +Date: Mon Jan 3 20:13:33 2022 +0800 + + problem: kevent udata is now void* on NetBSD Current (10) + + solution: check for the intptr_t variant in configure. + +diff --git a/configure.ac b/configure.ac +index 1a571291..402f8b86 100644 +--- a/configure.ac ++++ b/configure.ac +@@ -307,6 +307,27 @@ case "${host_os}" in + if test "x$libzmq_netbsd_has_atomic" = "xno"; then + AC_DEFINE(ZMQ_FORCE_MUTEXES, 1, [Force to use mutexes]) + fi ++ # NetBSD Current (to become 10) has changed the type of udata in it's ++ # kevent struct from intptr_t to void * to align with darwin and other ++ # BSDs, see upstream commit: ++ # https://github.com/NetBSD/src/commit/e5ead823eb916b56589d2c6c560dbcfe4a2d0afc ++ AC_MSG_CHECKING([whether kevent udata type is intptr_t]) ++ AC_LANG_PUSH([C++]) ++ AC_LINK_IFELSE([AC_LANG_PROGRAM( ++ [[#include ++ #include ++ #include ]], ++ [[struct kevent ev; ++ intptr_t udata; ++ EV_SET(&ev, 0, 0, EV_ADD, 0, 0, udata); ++ return 0;]])], ++ [libzmq_netbsd_kevent_udata_intptr_t=yes], ++ [libzmq_netbsd_kevent_udata_intptr_t=no]) ++ AC_LANG_POP([C++]) ++ AC_MSG_RESULT([$libzmq_netbsd_kevent_udata_intptr_t]) ++ if test "x$libzmq_netbsd_kevent_udata_intptr_t" = "xyes"; then ++ AC_DEFINE(ZMQ_NETBSD_KEVENT_UDATA_INTPTR_T, 1, [kevent udata type is intptr_t]) ++ fi + ;; + *openbsd*|*bitrig*) + # Define on OpenBSD to enable all library features +diff --git a/src/kqueue.cpp b/src/kqueue.cpp +index 53d82ac4..a6a7a7f2 100644 +--- a/src/kqueue.cpp ++++ b/src/kqueue.cpp +@@ -46,9 +46,9 @@ + #include "i_poll_events.hpp" + #include "likely.hpp" + +-// NetBSD defines (struct kevent).udata as intptr_t, everyone else +-// as void *. +-#if defined ZMQ_HAVE_NETBSD ++// NetBSD up to version 9 defines (struct kevent).udata as intptr_t, ++// everyone else as void *. ++#if defined ZMQ_HAVE_NETBSD && defined(ZMQ_NETBSD_KEVENT_UDATA_INTPTR_T) + #define kevent_udata_t intptr_t + #else + #define kevent_udata_t void * diff --git a/doc/Doxyfile.in b/doc/Doxyfile.in index 21bf587eaf..d8fd46d1c7 100644 --- a/doc/Doxyfile.in +++ b/doc/Doxyfile.in @@ -863,9 +863,7 @@ RECURSIVE = YES EXCLUDE = src/crc32c \ src/leveldb \ - src/json \ - src/test \ - src/qt/test + src/json # The EXCLUDE_SYMLINKS tag can be used to select whether or not files or # directories that are symbolic links (a Unix file system feature) are excluded diff --git a/doc/README.md b/doc/README.md index 9a5fe70e04..897fdb59cf 100644 --- a/doc/README.md +++ b/doc/README.md @@ -46,7 +46,6 @@ The following are developer notes on how to build Bitcoin Core on your native pl - [OpenBSD Build Notes](build-openbsd.md) - [NetBSD Build Notes](build-netbsd.md) - [Android Build Notes](build-android.md) -- [Gitian Building Guide (External Link)](https://github.com/bitcoin-core/docs/blob/master/gitian-building.md) Development --------------------- @@ -72,15 +71,20 @@ The Bitcoin repo's [root README](/README.md) contains relevant information on th ### Miscellaneous - [Assets Attribution](assets-attribution.md) +- [Assumeutxo design](assumeutxo.md) - [bitcoin.conf Configuration File](bitcoin-conf.md) - [Files](files.md) - [Fuzz-testing](fuzzing.md) - [I2P Support](i2p.md) - [Init Scripts (systemd/upstart/openrc)](init.md) +- [Managing Wallets](managing-wallets.md) +- [Multisig Tutorial](multisig-tutorial.md) +- [P2P bad ports definition and list](p2p-bad-ports.md) - [PSBT support](psbt.md) - [Reduce Memory](reduce-memory.md) - [Reduce Traffic](reduce-traffic.md) - [Tor Support](tor.md) +- [Transaction Relay Policy](policy/README.md) - [ZMQ](zmq.md) License diff --git a/doc/REST-interface.md b/doc/REST-interface.md index 8b281acca7..1f0a07a284 100644 --- a/doc/REST-interface.md +++ b/doc/REST-interface.md @@ -52,6 +52,20 @@ With the /notxdetails/ option JSON response will only contain the transaction ha Given a block hash: returns amount of blockheaders in upward direction. Returns empty if the block doesn't exist or it isn't in the active chain. +#### Blockfilter Headers +`GET /rest/blockfilterheaders///.` + +Given a block hash: returns amount of blockfilter headers in upward +direction for the filter type . +Returns empty if the block doesn't exist or it isn't in the active chain. + +#### Blockfilters +`GET /rest/blockfilter//.` + +Given a block hash: returns the block filter of the given block of type +. +Responds with 404 if the block doesn't exist. + #### Blockhash by height `GET /rest/blockhashbyheight/.` @@ -94,6 +108,7 @@ $ curl localhost:18332/rest/getutxos/checkmempool/b2cdfd7b89def827ff8af7cd9bff76 "value" : 8.8687, "scriptPubKey" : { "asm" : "OP_DUP OP_HASH160 1c7cebb529b86a04c683dfa87be49de35bcf589e OP_EQUALVERIFY OP_CHECKSIG", + "desc" : "addr(mi7as51dvLJsizWnTMurtRmrP8hG2m1XvD)#gj9tznmy" "hex" : "76a9141c7cebb529b86a04c683dfa87be49de35bcf589e88ac", "type" : "pubkeyhash", "address" : "mi7as51dvLJsizWnTMurtRmrP8hG2m1XvD" diff --git a/doc/assumeutxo.md b/doc/assumeutxo.md new file mode 100644 index 0000000000..2726cf779b --- /dev/null +++ b/doc/assumeutxo.md @@ -0,0 +1,138 @@ +# assumeutxo + +Assumeutxo is a feature that allows fast bootstrapping of a validating bitcoind +instance with a very similar security model to assumevalid. + +The RPC commands `dumptxoutset` and `loadtxoutset` are used to respectively generate +and load UTXO snapshots. The utility script `./contrib/devtools/utxo_snapshot.sh` may +be of use. + +## General background + +- [assumeutxo proposal](https://github.com/jamesob/assumeutxo-docs/tree/2019-04-proposal/proposal) +- [Github issue](https://github.com/bitcoin/bitcoin/issues/15605) +- [draft PR](https://github.com/bitcoin/bitcoin/pull/15606) + +## Design notes + +- A new block index `nStatus` flag is introduced, `BLOCK_ASSUMED_VALID`, to mark block + index entries that are required to be assumed-valid by a chainstate created + from a UTXO snapshot. This flag is mostly used as a way to modify certain + CheckBlockIndex() logic to account for index entries that are pending validation by a + chainstate running asynchronously in the background. We also use this flag to control + which index entries are added to setBlockIndexCandidates during LoadBlockIndex(). + +- Indexing implementations via BaseIndex can no longer assume that indexation happens + sequentially, since background validation chainstates can submit BlockConnected + events out of order with the active chain. + +- The concept of UTXO snapshots is treated as an implementation detail that lives + behind the ChainstateManager interface. The external presentation of the changes + required to facilitate the use of UTXO snapshots is the understanding that there are + now certain regions of the chain that can be temporarily assumed to be valid (using + the nStatus flag mentioned above). In certain cases, e.g. wallet rescanning, this is + very similar to dealing with a pruned chain. + + Logic outside ChainstateManager should try not to know about snapshots, instead + preferring to work in terms of more general states like assumed-valid. + + +## Chainstate phases + +Chainstate within the system goes through a number of phases when UTXO snapshots are +used, as managed by `ChainstateManager`. At various points there can be multiple +`CChainState` objects in existence to facilitate both maintaining the network tip and +performing historical validation of the assumed-valid chain. + +It is worth noting that though there are multiple separate chainstates, those +chainstates share use of a common block index (i.e. they hold the same `BlockManager` +reference). + +The subheadings below outline the phases and the corresponding changes to chainstate +data. + +### "Normal" operation via initial block download + +`ChainstateManager` manages a single CChainState object, for which +`m_snapshot_blockhash` is null. This chainstate is (maybe obviously) +considered active. This is the "traditional" mode of operation for bitcoind. + +| | | +| ---------- | ----------- | +| number of chainstates | 1 | +| active chainstate | ibd | + +### User loads a UTXO snapshot via `loadtxoutset` RPC + +`ChainstateManager` initializes a new chainstate (see `ActivateSnapshot()`) to load the +snapshot contents into. During snapshot load and validation (see +`PopulateAndValidateSnapshot()`), the new chainstate is not considered active and the +original chainstate remains in use as active. + +| | | +| ---------- | ----------- | +| number of chainstates | 2 | +| active chainstate | ibd | + +Once the snapshot chainstate is loaded and validated, it is promoted to active +chainstate and a sync to tip begins. A new chainstate directory is created in the +datadir for the snapshot chainstate called +`chainstate_[SHA256 blockhash of snapshot base block]`. + +| | | +| ---------- | ----------- | +| number of chainstates | 2 | +| active chainstate | snapshot | + +The snapshot begins to sync to tip from its base block, technically in parallel with +the original chainstate, but it is given priority during block download and is +allocated most of the cache (see `MaybeRebalanceCaches()` and usages) as our chief +consideration is getting to network tip. + +**Failure consideration:** if shutdown happens at any point during this phase, both +chainstates will be detected during the next init and the process will resume. + +### Snapshot chainstate hits network tip + +Once the snapshot chainstate leaves IBD, caches are rebalanced +(via `MaybeRebalanceCaches()` in `ActivateBestChain()`) and more cache is given +to the background chainstate, which is responsible for doing full validation of the +assumed-valid parts of the chain. + +**Note:** at this point, ValidationInterface callbacks will be coming in from both +chainstates. Considerations here must be made for indexing, which may no longer be happening +sequentially. + +### Background chainstate hits snapshot base block + +Once the tip of the background chainstate hits the base block of the snapshot +chainstate, we stop use of the background chainstate by setting `m_stop_use` (not yet +committed - see #15606), in `CompleteSnapshotValidation()`, which is checked in +`ActivateBestChain()`). We hash the background chainstate's UTXO set contents and +ensure it matches the compiled value in `CMainParams::m_assumeutxo_data`. + +The background chainstate data lingers on disk until shutdown, when in +`ChainstateManager::Reset()`, the background chainstate is cleaned up with +`ValidatedSnapshotShutdownCleanup()`, which renames the `chainstate_[hash]` datadir as +`chainstate`. + +| | | +| ---------- | ----------- | +| number of chainstates | 2 (ibd has `m_stop_use=true`) | +| active chainstate | snapshot | + +**Failure consideration:** if bitcoind unexpectedly halts after `m_stop_use` is set on +the background chainstate but before `CompleteSnapshotValidation()` can finish, the +need to complete snapshot validation will be detected on subsequent init by +`ChainstateManager::CheckForUncleanShutdown()`. + +### Bitcoind restarts sometime after snapshot validation has completed + +When bitcoind initializes again, what began as the snapshot chainstate is now +indistinguishable from a chainstate that has been built from the traditional IBD +process, and will be initialized as such. + +| | | +| ---------- | ----------- | +| number of chainstates | 1 | +| active chainstate | ibd | diff --git a/doc/bips.md b/doc/bips.md index 45954bcfd8..0f3f61daf1 100644 --- a/doc/bips.md +++ b/doc/bips.md @@ -1,4 +1,4 @@ -BIPs that are implemented by Bitcoin Core (up-to-date up to **v22.0**): +BIPs that are implemented by Bitcoin Core (up-to-date up to **v23.0**): * [`BIP 9`](https://github.com/bitcoin/bips/blob/master/bip-0009.mediawiki): The changes allowing multiple soft-forks to be deployed in parallel have been implemented since **v0.12.1** ([PR #7575](https://github.com/bitcoin/bitcoin/pull/7575)) * [`BIP 11`](https://github.com/bitcoin/bips/blob/master/bip-0011.mediawiki): Multisig outputs are standard since **v0.6.0** ([PR #669](https://github.com/bitcoin/bitcoin/pull/669)). @@ -32,7 +32,7 @@ BIPs that are implemented by Bitcoin Core (up-to-date up to **v22.0**): * [`BIP 111`](https://github.com/bitcoin/bips/blob/master/bip-0111.mediawiki): `NODE_BLOOM` service bit added, and enforced for all peer versions as of **v0.13.0** ([PR #6579](https://github.com/bitcoin/bitcoin/pull/6579) and [PR #6641](https://github.com/bitcoin/bitcoin/pull/6641)). * [`BIP 112`](https://github.com/bitcoin/bips/blob/master/bip-0112.mediawiki): The CHECKSEQUENCEVERIFY opcode has been implemented since **v0.12.1** ([PR #7524](https://github.com/bitcoin/bitcoin/pull/7524)), and has been *buried* since **v0.19.0** ([PR #16060](https://github.com/bitcoin/bitcoin/pull/16060)). * [`BIP 113`](https://github.com/bitcoin/bips/blob/master/bip-0113.mediawiki): Median time past lock-time calculations have been implemented since **v0.12.1** ([PR #6566](https://github.com/bitcoin/bitcoin/pull/6566)), and has been *buried* since **v0.19.0** ([PR #16060](https://github.com/bitcoin/bitcoin/pull/16060)). -* [`BIP 125`](https://github.com/bitcoin/bips/blob/master/bip-0125.mediawiki): Opt-in full replace-by-fee signaling honoured in mempool and mining as of **v0.12.0** ([PR 6871](https://github.com/bitcoin/bitcoin/pull/6871)). Enabled by default in the wallet GUI as of **v0.18.1** ([PR #11605](https://github.com/bitcoin/bitcoin/pull/11605)) +* [`BIP 125`](https://github.com/bitcoin/bips/blob/master/bip-0125.mediawiki): Opt-in full replace-by-fee signaling partially implemented. See doc/policy/mempool-replacements.md. * [`BIP 130`](https://github.com/bitcoin/bips/blob/master/bip-0130.mediawiki): direct headers announcement is negotiated with peer versions `>=70012` as of **v0.12.0** ([PR 6494](https://github.com/bitcoin/bitcoin/pull/6494)). * [`BIP 133`](https://github.com/bitcoin/bips/blob/master/bip-0133.mediawiki): feefilter messages are respected and sent for peer versions `>=70013` as of **v0.13.0** ([PR 7542](https://github.com/bitcoin/bitcoin/pull/7542)). * [`BIP 141`](https://github.com/bitcoin/bips/blob/master/bip-0141.mediawiki): Segregated Witness (Consensus Layer) as of **v0.13.0** ([PR 8149](https://github.com/bitcoin/bitcoin/pull/8149)), defined for mainnet as of **v0.13.1** ([PR 8937](https://github.com/bitcoin/bitcoin/pull/8937)), and *buried* since **v0.19.0** ([PR #16060](https://github.com/bitcoin/bitcoin/pull/16060)). @@ -42,7 +42,8 @@ BIPs that are implemented by Bitcoin Core (up-to-date up to **v22.0**): * [`BIP 147`](https://github.com/bitcoin/bips/blob/master/bip-0147.mediawiki): NULLDUMMY softfork as of **v0.13.1** ([PR 8636](https://github.com/bitcoin/bitcoin/pull/8636) and [PR 8937](https://github.com/bitcoin/bitcoin/pull/8937)), *buried* since **v0.19.0** ([PR #16060](https://github.com/bitcoin/bitcoin/pull/16060)). * [`BIP 152`](https://github.com/bitcoin/bips/blob/master/bip-0152.mediawiki): Compact block transfer and related optimizations are used as of **v0.13.0** ([PR 8068](https://github.com/bitcoin/bitcoin/pull/8068)). * [`BIP 155`](https://github.com/bitcoin/bips/blob/master/bip-0155.mediawiki): The 'addrv2' and 'sendaddrv2' messages which enable relay of Tor V3 addresses (and other networks) are supported as of **v0.21.0** ([PR 19954](https://github.com/bitcoin/bitcoin/pull/19954)). -* [`BIP 158`](https://github.com/bitcoin/bips/blob/master/bip-0158.mediawiki): Compact Block Filters for Light Clients can be indexed as of **v0.19.0** ([PR #14121](https://github.com/bitcoin/bitcoin/pull/14121)). +* [`BIP 157`](https://github.com/bitcoin/bips/blob/master/bip-0157.mediawiki) + [`158`](https://github.com/bitcoin/bips/blob/master/bip-0158.mediawiki): Compact Block Filters for Light Clients can be indexed as of **v0.19.0** ([PR #14121](https://github.com/bitcoin/bitcoin/pull/14121)) and served to peers on the P2P network as of **v0.21.0** ([PR #16442](https://github.com/bitcoin/bitcoin/pull/16442)). * [`BIP 159`](https://github.com/bitcoin/bips/blob/master/bip-0159.mediawiki): The `NODE_NETWORK_LIMITED` service bit is signalled as of **v0.16.0** ([PR 11740](https://github.com/bitcoin/bitcoin/pull/11740)), and such nodes are connected to as of **v0.17.0** ([PR 10387](https://github.com/bitcoin/bitcoin/pull/10387)). * [`BIP 173`](https://github.com/bitcoin/bips/blob/master/bip-0173.mediawiki): Bech32 addresses for native Segregated Witness outputs are supported as of **v0.16.0** ([PR 11167](https://github.com/bitcoin/bitcoin/pull/11167)). Bech32 addresses are generated by default as of **v0.20.0** ([PR 16884](https://github.com/bitcoin/bitcoin/pull/16884)). * [`BIP 174`](https://github.com/bitcoin/bips/blob/master/bip-0174.mediawiki): RPCs to operate on Partially Signed Bitcoin Transactions (PSBT) are present as of **v0.17.0** ([PR 13557](https://github.com/bitcoin/bitcoin/pull/13557)). @@ -57,3 +58,11 @@ BIPs that are implemented by Bitcoin Core (up-to-date up to **v22.0**): with mainnet activation as of **v0.21.1** ([PR 21377](https://github.com/bitcoin/bitcoin/pull/21377), [PR 21686](https://github.com/bitcoin/bitcoin/pull/21686)). * [`BIP 350`](https://github.com/bitcoin/bips/blob/master/bip-0350.mediawiki): Addresses for native v1+ segregated Witness outputs use Bech32m instead of Bech32 as of **v22.0** ([PR 20861](https://github.com/bitcoin/bitcoin/pull/20861)). +* [`BIP 380`](https://github.com/bitcoin/bips/blob/master/bip-0380.mediawiki) + [`381`](https://github.com/bitcoin/bips/blob/master/bip-0381.mediawiki) + [`382`](https://github.com/bitcoin/bips/blob/master/bip-0382.mediawiki) + [`383`](https://github.com/bitcoin/bips/blob/master/bip-0383.mediawiki) + [`384`](https://github.com/bitcoin/bips/blob/master/bip-0384.mediawiki) + [`385`](https://github.com/bitcoin/bips/blob/master/bip-0385.mediawiki): + Output Script Descriptors, and most of Script Expressions are implemented as of **v0.17.0** ([PR 13697](https://github.com/bitcoin/bitcoin/pull/13697)). +* [`BIP 386`](https://github.com/bitcoin/bips/blob/master/bip-0386.mediawiki): tr() Output Script Descriptors are implemented as of **v22.0** ([PR 22051](https://github.com/bitcoin/bitcoin/pull/22051)). diff --git a/doc/build-android.md b/doc/build-android.md index 7a8a9e6a65..2f2e01c441 100644 --- a/doc/build-android.md +++ b/doc/build-android.md @@ -3,9 +3,22 @@ ANDROID BUILD NOTES This guide describes how to build and package the `bitcoin-qt` GUI for Android on Linux and macOS. -## Preparation -You will need to get the Android NDK and build dependencies for Android as described in [depends/README.md](../depends/README.md). +## Dependencies + +Before proceeding with an Android build one needs to get the [Android SDK](https://developer.android.com/studio) and use the "SDK Manager" tool to download the NDK and one or more "Platform packages" (these are Android versions and have a corresponding API level). + +The minimum supported Android NDK version is [r23](https://github.com/android/ndk/wiki/Changelog-r23). + +In order to build `ANDROID_API_LEVEL` (API level corresponding to the Android version targeted, e.g. Android 9.0 Pie is 28 and its "Platform package" needs to be available) and `ANDROID_TOOLCHAIN_BIN` (path to toolchain binaries depending on the platform the build is being performed on) need to be set. + +API levels from 24 to 29 have been tested to work. + +If the build includes Qt, environment variables `ANDROID_SDK` and `ANDROID_NDK` need to be set as well but can otherwise be omitted. +This is an example command for a default build with no disabled dependencies: + + ANDROID_SDK=/home/user/Android/Sdk ANDROID_NDK=/home/user/Android/Sdk/ndk-bundle make HOST=aarch64-linux-android ANDROID_API_LEVEL=28 ANDROID_TOOLCHAIN_BIN=/home/user/Android/Sdk/ndk-bundle/toolchains/llvm/prebuilt/linux-x86_64/bin + ## Building and packaging diff --git a/doc/build-openbsd.md b/doc/build-openbsd.md index 6e54f67edc..275b7ce124 100644 --- a/doc/build-openbsd.md +++ b/doc/build-openbsd.md @@ -25,8 +25,8 @@ See [dependencies.md](dependencies.md) for a complete overview. **Important**: From OpenBSD 6.2 onwards a C++11-supporting clang compiler is part of the base image, and while building it is necessary to make sure that this compiler is used and not ancient g++ 4.2.1. This is done by appending -`CC=cc CC_FOR_BUILD=cc CXX=c++` to configuration commands. Mixing different -compilers within the same executable will result in errors. +`CC=cc CXX=c++` to configuration commands. Mixing different compilers within +the same executable will result in errors. ### Building BerkeleyDB @@ -84,7 +84,7 @@ To configure with wallet: To configure without wallet: ```bash -./configure --disable-wallet --with-gui=no --disable-external-signer CC=cc CC_FOR_BUILD=cc CXX=c++ MAKE=gmake +./configure --disable-wallet --with-gui=no --disable-external-signer CC=cc CXX=c++ MAKE=gmake ``` To configure with GUI: diff --git a/doc/build-osx.md b/doc/build-osx.md index 8f9f1132aa..bf20a0dd52 100644 --- a/doc/build-osx.md +++ b/doc/build-osx.md @@ -35,7 +35,6 @@ The following dependencies are **optional** packages required for deploying: Library | Purpose | Description ----------------------------------------------------|------------------|---------------------- -[librsvg](https://formulae.brew.sh/formula/librsvg) | Deploy Dependency| Library to render SVG files [ds_store](https://pypi.org/project/ds-store/) | Deploy Dependency| Examine and modify .DS_Store files [mac_alias](https://pypi.org/project/mac-alias/) | Deploy Dependency| Generate/Read binary alias and bookmark records @@ -218,10 +217,6 @@ This command depends on a couple of python packages, so it is required that you Ensuring that `python` is installed, you can install the deploy dependencies by running the following commands in your terminal: -``` bash -brew install librsvg libicns imagemagick -``` - ``` bash pip3 install ds_store mac_alias ``` diff --git a/doc/build-unix.md b/doc/build-unix.md index 3da8af397c..fe239cc372 100644 --- a/doc/build-unix.md +++ b/doc/build-unix.md @@ -42,12 +42,12 @@ Optional dependencies: ------------|------------------|---------------------- miniupnpc | UPnP Support | Firewall-jumping support libnatpmp | NAT-PMP Support | Firewall-jumping support - libdb4.8 | Berkeley DB | Optional, wallet storage (only needed when wallet enabled) + libdb4.8 | Berkeley DB | Wallet storage (only needed when legacy wallet enabled) qt | GUI | GUI toolkit (only needed when GUI enabled) - libqrencode | QR codes in GUI | Optional for generating QR codes (only needed when GUI enabled) - univalue | Utility | JSON parsing and encoding (bundled version will be used unless --with-system-univalue passed to configure) - libzmq3 | ZMQ notification | Optional, allows generating ZMQ notifications (requires ZMQ version >= 4.0.0) - sqlite3 | SQLite DB | Optional, wallet storage (only needed when wallet enabled) + libqrencode | QR codes in GUI | QR code generation (only needed when GUI enabled) + libzmq3 | ZMQ notification | ZMQ notifications (requires ZMQ version >= 4.0.0) + sqlite3 | SQLite DB | Wallet storage (only needed when descriptor wallet enabled) + systemtap | Tracing (USDT) | Statically defined tracepoints For the versions used, see [dependencies.md](dependencies.md) @@ -82,21 +82,17 @@ Build requirements: Now, you can either build from self-compiled [depends](/depends/README.md) or install the required dependencies: - sudo apt-get install libevent-dev libboost-dev libboost-system-dev libboost-filesystem-dev libboost-test-dev - -Berkeley DB is required for the wallet. - -Ubuntu and Debian have their own `libdb-dev` and `libdb++-dev` packages, but these will install -Berkeley DB 5.1 or later. This will break binary wallet compatibility with the distributed executables, which -are based on BerkeleyDB 4.8. If you do not care about wallet compatibility, -pass `--with-incompatible-bdb` to configure. - -Otherwise, you can build Berkeley DB [yourself](#berkeley-db). + sudo apt-get install libevent-dev libboost-dev SQLite is required for the descriptor wallet: sudo apt install libsqlite3-dev +Berkeley DB is required for the legacy wallet. Ubuntu and Debian have their own `libdb-dev` and `libdb++-dev` packages, +but these will install Berkeley DB 5.1 or later. This will break binary wallet compatibility with the distributed +executables, which are based on BerkeleyDB 4.8. If you do not care about wallet compatibility, pass +`--with-incompatible-bdb` to configure. Otherwise, you can build Berkeley DB [yourself](#berkeley-db). + To build Bitcoin Core without wallet, see [*Disable-wallet mode*](#disable-wallet-mode) Optional port mapping libraries (see: `--with-miniupnpc`, `--enable-upnp-default`, and `--with-natpmp`, `--enable-natpmp-default`): @@ -107,6 +103,10 @@ ZMQ dependencies (provides ZMQ API): sudo apt-get install libzmq3-dev +User-Space, Statically Defined Tracing (USDT) dependencies: + + sudo apt install systemtap-sdt-dev + GUI dependencies: If you want to build Elements-Qt, make sure that the required packages for Qt development @@ -119,6 +119,10 @@ To build with Qt 5 you need the following: The last two dependencies are required to generate Elements images that don't exist in Bitcoin Core. +Additionally, to support Wayland protocol for modern desktop environments: + + sudo apt install qtwayland5 + libqrencode (optional) can be installed with: sudo apt-get install libqrencode-dev @@ -139,20 +143,18 @@ Now, you can either build from self-compiled [depends](/depends/README.md) or in sudo dnf install libevent-devel boost-devel -Berkeley DB is required for the wallet: +SQLite is required for the descriptor wallet: + + sudo dnf install sqlite-devel + +Berkeley DB is required for the legacy wallet: sudo dnf install libdb4-devel libdb4-cxx-devel Newer Fedora releases, since Fedora 33, have only `libdb-devel` and `libdb-cxx-devel` packages, but these will install Berkeley DB 5.3 or later. This will break binary wallet compatibility with the distributed executables, which are based on Berkeley DB 4.8. If you do not care about wallet compatibility, -pass `--with-incompatible-bdb` to configure. - -Otherwise, you can build Berkeley DB [yourself](#berkeley-db). - -SQLite is required for the descriptor wallet: - - sudo dnf install sqlite-devel +pass `--with-incompatible-bdb` to configure. Otherwise, you can build Berkeley DB [yourself](#berkeley-db). To build Bitcoin Core without wallet, see [*Disable-wallet mode*](#disable-wallet-mode) @@ -164,6 +166,10 @@ ZMQ dependencies (provides ZMQ API): sudo dnf install zeromq-devel +User-Space, Statically Defined Tracing (USDT) dependencies: + + sudo dnf install systemtap + GUI dependencies: If you want to build bitcoin-qt, make sure that the required packages for Qt development @@ -174,6 +180,10 @@ To build with Qt 5 you need the following: sudo dnf install qt5-qttools-devel qt5-qtbase-devel +Additionally, to support Wayland protocol for modern desktop environments: + + sudo dnf install qt5-qtwayland + libqrencode (optional) can be installed with: sudo dnf install qrencode-devel @@ -210,8 +220,10 @@ turned off by default. See the configure options for NAT-PMP behavior desired: Berkeley DB ----------- -It is recommended to use Berkeley DB 4.8. If you have to build it yourself, -you can use [the installation script included in contrib/](/contrib/install_db4.sh) + +The legacy wallet uses Berkeley DB. To ensure backwards compatibility it is +recommended to use Berkeley DB 4.8. If you have to build it yourself, you can +use [the installation script included in contrib/](/contrib/install_db4.sh) like so: ```shell @@ -224,15 +236,6 @@ Otherwise, you can build Bitcoin Core from self-compiled [depends](/depends/READ **Note**: You only need Berkeley DB if the wallet is enabled (see [*Disable-wallet mode*](#disable-wallet-mode)). -Boost ------ -If you need to build Boost yourself: - - sudo su - ./bootstrap.sh - ./bjam install - - Security -------- To help make your Bitcoin Core installation more secure by making certain attacks impossible to @@ -311,7 +314,7 @@ This example lists the steps necessary to setup and build a command line only, n Note: Enabling wallet support requires either compiling against a Berkeley DB newer than 4.8 (package `db`) using `--with-incompatible-bdb`, or building and depending on a local version of Berkeley DB 4.8. The readily available Arch Linux packages are currently built using -`--with-incompatible-bdb` according to the [PKGBUILD](https://projects.archlinux.org/svntogit/community.git/tree/bitcoin/trunk/PKGBUILD). +`--with-incompatible-bdb` according to the [PKGBUILD](https://github.com/archlinux/svntogit-community/blob/packages/bitcoin/trunk/PKGBUILD). As mentioned above, when maintaining portability of the wallet between the standard Bitcoin Core distributions and independently built node software is desired, Berkeley DB 4.8 must be used. @@ -333,7 +336,7 @@ To build executables for ARM: make HOST=arm-linux-gnueabihf NO_QT=1 cd .. ./autogen.sh - CONFIG_SITE=$PWD/depends/arm-linux-gnueabihf/share/config.site ./configure --enable-glibc-back-compat --enable-reduce-exports LDFLAGS=-static-libstdc++ + CONFIG_SITE=$PWD/depends/arm-linux-gnueabihf/share/config.site ./configure --enable-reduce-exports LDFLAGS=-static-libstdc++ make diff --git a/doc/build-windows.md b/doc/build-windows.md index f88b9739de..e35d3bcbd0 100644 --- a/doc/build-windows.md +++ b/doc/build-windows.md @@ -5,11 +5,9 @@ Below are some notes on how to build Bitcoin Core for Windows. The options known to work for building Bitcoin Core on Windows are: -* On Linux, using the [Mingw-w64](https://mingw-w64.org/doku.php) cross compiler tool chain. Ubuntu Bionic 18.04 is required -and is the platform used to build the Bitcoin Core Windows release binaries. -* On Windows, using [Windows -Subsystem for Linux (WSL)](https://docs.microsoft.com/windows/wsl/about) and the Mingw-w64 cross compiler tool chain. -* On Windows, using a native compiler tool chain such as [Visual Studio](https://www.visualstudio.com). See [README.md](/build_msvc/README.md). +* On Linux, using the [Mingw-w64](https://www.mingw-w64.org/) cross compiler tool chain. +* On Windows, using [Windows Subsystem for Linux (WSL)](https://docs.microsoft.com/windows/wsl/about) and Mingw-w64. +* On Windows, using [Microsoft Visual Studio](https://www.visualstudio.com). See [README.md](/build_msvc/README.md). Other options which may work, but which have not been extensively tested are (please contribute instructions): @@ -18,40 +16,12 @@ Other options which may work, but which have not been extensively tested are (pl Installing Windows Subsystem for Linux --------------------------------------- -With Windows 10, Microsoft has released a new feature named the [Windows -Subsystem for Linux (WSL)](https://docs.microsoft.com/windows/wsl/about). This -feature allows you to run a bash shell directly on Windows in an Ubuntu-based -environment. Within this environment you can cross compile for Windows without -the need for a separate Linux VM or server. Note that while WSL can be installed with -other Linux variants, such as OpenSUSE, the following instructions have only been -tested with Ubuntu. - -This feature is not supported in versions of Windows prior to Windows 10 or on -Windows Server SKUs. In addition, it is available [only for 64-bit versions of -Windows](https://docs.microsoft.com/windows/wsl/install-win10). - -Full instructions to install WSL are available on the above link. -To install WSL on Windows 10 with Fall Creators Update installed (version >= 16215.0) do the following: - -1. Enable the Windows Subsystem for Linux feature - * Open the Windows Features dialog (`OptionalFeatures.exe`) - * Enable 'Windows Subsystem for Linux' - * Click 'OK' and restart if necessary -2. Install Ubuntu - * Open Microsoft Store and search for "Ubuntu 18.04" or use [this link](https://www.microsoft.com/store/productId/9N9TNGVNDL3Q) - * Click Install -3. Complete Installation - * Open a cmd prompt and type "Ubuntu1804" - * Create a new UNIX user account (this is a separate account from your Windows account) - -After the bash shell is active, you can follow the instructions below, starting -with the "Cross-compilation" section. Compiling the 64-bit version is -recommended, but it is possible to compile the 32-bit version. +Follow the upstream installation instructions, available [here](https://docs.microsoft.com/windows/wsl/install-win10). Cross-compilation for Ubuntu and Windows Subsystem for Linux ------------------------------------------------------------ -The steps below can be performed on Ubuntu (including in a VM) or WSL. The depends system +The steps below can be performed on Ubuntu or WSL. The depends system will also work on other Linux distributions, however the commands for installing the toolchain will be different. @@ -78,28 +48,16 @@ Acquire the source in the usual way: ## Building for 64-bit Windows The first step is to install the mingw-w64 cross-compilation tool chain: + - on modern systems (Ubuntu 21.04 Hirsute Hippo or newer, Debian 11 Bullseye or newer): - sudo apt install g++-mingw-w64-x86-64 - -Next, set the default `mingw32 g++` compiler option to POSIX[1](#footnote1): - -``` -sudo update-alternatives --config x86_64-w64-mingw32-g++ +```sh +sudo apt install g++-mingw-w64-x86-64-posix ``` -After running the above command, you should see output similar to that below. -Choose the option that ends with `posix`. + - on older systems: -``` -There are 2 choices for the alternative x86_64-w64-mingw32-g++ (providing /usr/bin/x86_64-w64-mingw32-g++). - - Selection Path Priority Status ------------------------------------------------------------- - 0 /usr/bin/x86_64-w64-mingw32-g++-win32 60 auto mode -* 1 /usr/bin/x86_64-w64-mingw32-g++-posix 30 manual mode - 2 /usr/bin/x86_64-w64-mingw32-g++-win32 60 manual mode - -Press to keep the current choice[*], or type selection number: +```sh +sudo apt install g++-mingw-w64-x86-64 ``` Once the toolchain is installed the build steps are common: @@ -142,13 +100,3 @@ way. This will install to `c:\workspace\bitcoin`, for example: You can also create an installer using: make deploy - -Footnotes ---------- - -1: Starting from Ubuntu Xenial 16.04, both the 32 and 64 bit Mingw-w64 packages install two different -compiler options to allow a choice between either posix or win32 threads. The default option is win32 threads which is the more -efficient since it will result in binary code that links directly with the Windows kernel32.lib. Unfortunately, the headers -required to support win32 threads conflict with some of the classes in the C++11 standard library, in particular std::mutex. -It's not possible to build the Bitcoin Core code using the win32 version of the Mingw-w64 cross compilers (at least not without -modifying headers in the Bitcoin Core source code). diff --git a/doc/dependencies.md b/doc/dependencies.md index 66c5a76b3b..99ff26f457 100644 --- a/doc/dependencies.md +++ b/doc/dependencies.md @@ -6,24 +6,24 @@ These are the dependencies currently used by Bitcoin Core. You can find instruct | Dependency | Version used | Minimum required | CVEs | Shared | [Bundled Qt library](https://doc.qt.io/qt-5/configure-options.html#third-party-libraries) | | --- | --- | --- | --- | --- | --- | | Berkeley DB | [4.8.30](https://www.oracle.com/technetwork/database/database-technologies/berkeleydb/downloads/index.html) | 4.8.x | No | | | -| Boost | [1.71.0](https://www.boost.org/users/download/) | [1.64.0](https://github.com/bitcoin/bitcoin/pull/22320) | No | | | -| Clang[ \* ](#note1) | | [5.0+](https://releases.llvm.org/download.html) (C++17 support) | | | | -| Expat | [2.2.7](https://libexpat.github.io/) | | No | Yes | | -| fontconfig | [2.12.1](https://www.freedesktop.org/software/fontconfig/release/) | | No | Yes | | -| FreeType | [2.7.1](https://download.savannah.gnu.org/releases/freetype) | | No | | [Yes](https://github.com/bitcoin/bitcoin/blob/master/depends/packages/qt.mk) (Android only) | -| GCC | | [7+](https://gcc.gnu.org/) (C++17 support) | | | | +| Boost | [1.77.0](https://www.boost.org/users/download/) | [1.64.0](https://github.com/bitcoin/bitcoin/pull/22320) | No | | | +| Clang[ \* ](#note1) | | [7.0](https://releases.llvm.org/download.html) (C++17 & std::filesystem support) | | | | +| Fontconfig | [2.12.6](https://www.freedesktop.org/software/fontconfig/release/) | | No | Yes | | +| FreeType | [2.11.0](https://download.savannah.gnu.org/releases/freetype) | | No | | [Yes](https://github.com/bitcoin/bitcoin/blob/master/depends/packages/qt.mk) (Android only) | +| GCC | | [8.1](https://gcc.gnu.org/) (C++17 & std::filesystem support) | | | | +| glibc | | [2.18](https://www.gnu.org/software/libc/) | | | | | | HarfBuzz-NG | | | | | [Yes](https://github.com/bitcoin/bitcoin/blob/master/depends/packages/qt.mk) | | libevent | [2.1.12-stable](https://github.com/libevent/libevent/releases) | [2.0.21](https://github.com/bitcoin/bitcoin/pull/18676) | No | | | | libnatpmp | git commit [4536032...](https://github.com/miniupnp/libnatpmp/tree/4536032ae32268a45c073a4d5e91bbab4534773a) | | No | | | | libpng | | | | | [Yes](https://github.com/bitcoin/bitcoin/blob/master/depends/packages/qt.mk) | -| librsvg | | | | | | | MiniUPnPc | [2.2.2](https://miniupnp.tuxfamily.org/files) | | No | | | | PCRE | | | | | [Yes](https://github.com/bitcoin/bitcoin/blob/master/depends/packages/qt.mk) | | Python (tests) | | [3.6](https://www.python.org/downloads) | | | | | qrencode | [3.4.4](https://fukuchi.org/works/qrencode) | | No | | | -| Qt | [5.12.11](https://download.qt.io/official_releases/qt/) | [5.9.5](https://github.com/bitcoin/bitcoin/issues/20104) | No | | | +| Qt | [5.15.2](https://download.qt.io/official_releases/qt/) | [5.9.5](https://github.com/bitcoin/bitcoin/issues/20104) | No | | | | SQLite | [3.32.1](https://sqlite.org/download.html) | [3.7.17](https://github.com/bitcoin/bitcoin/pull/19077) | | | | | XCB | | | | | [Yes](https://github.com/bitcoin/bitcoin/blob/master/depends/packages/qt.mk) (Linux only) | +| systemtap ([tracing](tracing.md))| [4.5](https://sourceware.org/systemtap/ftp/releases/) | | | | | | xkbcommon | | | | | [Yes](https://github.com/bitcoin/bitcoin/blob/master/depends/packages/qt.mk) (Linux only) | | ZeroMQ | [4.3.1](https://github.com/zeromq/libzmq/releases) | 4.0.0 | No | | | | zlib | | | | | [Yes](https://github.com/bitcoin/bitcoin/blob/master/depends/packages/qt.mk) | @@ -41,8 +41,8 @@ Some dependencies are not needed in all configurations. The following are some f * SQLite is not needed with `--disable-wallet` or `--without-sqlite`. * Qt is not needed with `--without-gui`. * If the qrencode dependency is absent, QR support won't be added. To force an error when that happens, pass `--with-qrencode`. +* If the systemtap dependency is absent, USDT support won't compiled in. * ZeroMQ is needed only with the `--with-zmq` option. #### Other -* librsvg is only needed if you need to run `make deploy` on (cross-compilation to) macOS. * Not-Qt-bundled zlib is required to build the [DMG tool](../contrib/macdeploy/README.md#deterministic-macos-dmg-notes) from the libdmg-hfsplus project. diff --git a/doc/descriptors.md b/doc/descriptors.md index e27ff87546..318d065fdb 100644 --- a/doc/descriptors.md +++ b/doc/descriptors.md @@ -90,16 +90,16 @@ Descriptors consist of several types of expressions. The top level expression is - Optionally followed by a single `/*` or `/*'` final step to denote all (direct) unhardened or hardened children. - The usage of hardened derivation steps requires providing the private key. +(Anywhere a `'` suffix is permitted to denote hardened derivation, the suffix `h` can be used instead.) + `TREE` expressions: - any `SCRIPT` expression - An open brace `{`, a `TREE` expression, a comma `,`, a `TREE` expression, and a closing brace `}` -(Anywhere a `'` suffix is permitted to denote hardened derivation, the suffix `h` can be used instead.) - `ADDR` expressions are any type of supported address: - P2PKH addresses (base58, of the form `1...` for mainnet or `[nm]...` for testnet). Note that P2PKH addresses in descriptors cannot be used for P2PK outputs (use the `pk` function instead). - P2SH addresses (base58, of the form `3...` for mainnet or `2...` for testnet, defined in [BIP 13](https://github.com/bitcoin/bips/blob/master/bip-0013.mediawiki)). -- Segwit addresses (bech32, of the form `bc1...` for mainnet or `tb1...` for testnet, defined in [BIP 173](https://github.com/bitcoin/bips/blob/master/bip-0173.mediawiki)). +- Segwit addresses (bech32 and bech32m, of the form `bc1...` for mainnet or `tb1...` for testnet, defined in [BIP 173](https://github.com/bitcoin/bips/blob/master/bip-0173.mediawiki) and [BIP 350](https://github.com/bitcoin/bips/blob/master/bip-0350.mediawiki)). ## Explanation @@ -139,6 +139,47 @@ Key order does not matter for `sortedmulti()`. `sortedmulti()` behaves in the sa as `multi()` does but the keys are reordered in the resulting script such that they are lexicographically ordered as described in BIP67. +#### Basic multisig example + +For a good example of a basic M-of-N multisig between multiple participants using descriptor +wallets and PSBTs, as well as a signing flow, see [this functional test](/test/functional/wallet_multisig_descriptor_psbt.py). + +Disclaimers: It is important to note that this example serves as a quick-start and is kept basic for readability. A downside of the approach +outlined here is that each participant must maintain (and backup) two separate wallets: a signer and the corresponding multisig. +It should also be noted that privacy best-practices are not "by default" here - participants should take care to only use the signer to sign +transactions related to the multisig. Lastly, it is not recommended to use anything other than a Bitcoin Core descriptor wallet to serve as your +signer(s). Other wallets, whether hardware or software, likely impose additional checks and safeguards to prevent users from signing transactions that +could lead to loss of funds, or are deemed security hazards. Conforming to various 3rd-party checks and verifications is not in the scope of this example. + +The basic steps are: + + 1. Every participant generates an xpub. The most straightforward way is to create a new descriptor wallet which we will refer to as + the participant's signer wallet. Avoid reusing this wallet for any purpose other than signing transactions from the + corresponding multisig we are about to create. Hint: extract the wallet's xpubs using `listdescriptors` and pick the one from the + `pkh` descriptor since it's least likely to be accidentally reused (legacy addresses) + 2. Create a watch-only descriptor wallet (blank, private keys disabled). Now the multisig is created by importing the two descriptors: + `wsh(sortedmulti(,XPUB1/0/*,XPUB2/0/*,…,XPUBN/0/*))` and `wsh(sortedmulti(,XPUB1/1/*,XPUB2/1/*,…,XPUBN/1/*))` + (one descriptor w/ `0` for receiving addresses and another w/ `1` for change). Every participant does this + 3. A receiving address is generated for the multisig. As a check to ensure step 2 was done correctly, every participant + should verify they get the same addresses + 4. Funds are sent to the resulting address + 5. A sending transaction from the multisig is created using `walletcreatefundedpsbt` (anyone can initiate this). It is simple to do + this in the GUI by going to the `Send` tab in the multisig wallet and creating an unsigned transaction (PSBT) + 6. At least `M` participants check the PSBT with their multisig using `decodepsbt` to verify the transaction is OK before signing it. + 7. (If OK) the participant signs the PSBT with their signer wallet using `walletprocesspsbt`. It is simple to do this in the GUI by + loading the PSBT from file and signing it + 8. The signed PSBTs are collected with `combinepsbt`, finalized w/ `finalizepsbt`, and then the resulting transaction is broadcasted + to the network. Note that any wallet (eg one of the signers or multisig) is capable of doing this. + 9. Checks that balances are correct after the transaction has been included in a block + +You may prefer a daisy chained signing flow where each participant signs the PSBT one after another until +the PSBT has been signed `M` times and is "complete." For the most part, the steps above remain the same, except (6, 7) +change slightly from signing the original PSBT in parallel to signing it in series. `combinepsbt` is not necessary with +this signing flow and the last (`m`th) signer can just broadcast the PSBT after signing. Note that a parallel signing flow may be +preferable in cases where there are more signers. This signing flow is also included in the test / Python example. +[The test](/test/functional/wallet_multisig_descriptor_psbt.py) is meant to be documentation as much as it is a functional test, so +it is kept as simple and readable as possible. + ### BIP32 derived keys and chains Most modern wallet software and hardware uses keys that are derived using diff --git a/doc/developer-notes.md b/doc/developer-notes.md index 583c50a763..bfb64093e1 100644 --- a/doc/developer-notes.md +++ b/doc/developer-notes.md @@ -12,6 +12,7 @@ Developer Notes - [Generating Documentation](#generating-documentation) - [Development tips and tricks](#development-tips-and-tricks) - [Compiling for debugging](#compiling-for-debugging) + - [Show sources in debugging](#show-sources-in-debugging) - [Compiling for gprof profiling](#compiling-for-gprof-profiling) - [`debug.log`](#debuglog) - [Signet, testnet, and regtest modes](#signet-testnet-and-regtest-modes) @@ -89,6 +90,10 @@ code. - Class member variables have a `m_` prefix. - Global variables have a `g_` prefix. - Constant names are all uppercase, and use `_` to separate words. + - Enumerator constants may be `snake_case`, `PascalCase` or `ALL_CAPS`. + This is a more tolerant policy than the [C++ Core + Guidelines](https://isocpp.github.io/CppCoreGuidelines/CppCoreGuidelines#Renum-caps), + which recommend using `snake_case`. Please use what seems appropriate. - Class names, function names, and method names are UpperCamelCase (PascalCase). Do not prefix class names with `C`. - Test suite naming convention: The Boost test suite in file @@ -249,6 +254,35 @@ Development tips and tricks Run configure with `--enable-debug` to add additional compiler flags that produce better debugging builds. +### Show sources in debugging + +If you have ccache enabled, absolute paths are stripped from debug information +with the -fdebug-prefix-map and -fmacro-prefix-map options (if supported by the +compiler). This might break source file detection in case you move binaries +after compilation, debug from the directory other than the project root or use +an IDE that only supports absolute paths for debugging. + +There are a few possible fixes: + +1. Configure source file mapping. + +For `gdb` create or append to `.gdbinit` file: +``` +set substitute-path ./src /path/to/project/root/src +``` + +For `lldb` create or append to `.lldbinit` file: +``` +settings set target.source-map ./src /path/to/project/root/src +``` + +2. Add a symlink to the `./src` directory: +``` +ln -s /path/to/project/root/src src +``` + +3. Use `debugedit` to modify debug information in the binary. + ### Compiling for gprof profiling Run configure with the `--enable-gprof` option, then make. @@ -297,7 +331,7 @@ other input. failure, it will throw an exception, which can be caught to recover from the error. - For example, a nullptr dereference or any other logic bug in RPC code - means that the RPC code is faulty and can not be executed. However, the + means that the RPC code is faulty and cannot be executed. However, the logic bug can be shown to the user and the program can continue to run. * `Assume` should be used to document assumptions when program execution can safely continue even if the assumption is violated. In debug builds it @@ -345,7 +379,7 @@ make cov Profiling is a good way to get a precise idea of where time is being spent in code. One tool for doing profiling on Linux platforms is called -[`perf`](http://www.brendangregg.com/perf.html), and has been integrated into +[`perf`](https://www.brendangregg.com/perf.html), and has been integrated into the functional test framework. Perf can observe a running process and sample (at some frequency) where its execution is. @@ -669,19 +703,19 @@ Foo(vec); ```cpp enum class Tabs { - INFO, - CONSOLE, - GRAPH, - PEERS + info, + console, + network_graph, + peers }; int GetInt(Tabs tab) { switch (tab) { - case Tabs::INFO: return 0; - case Tabs::CONSOLE: return 1; - case Tabs::GRAPH: return 2; - case Tabs::PEERS: return 3; + case Tabs::info: return 0; + case Tabs::console: return 1; + case Tabs::network_graph: return 2; + case Tabs::peers: return 3; } // no default case, so the compiler can warn about missing cases assert(false); } @@ -959,37 +993,44 @@ Subtrees Several parts of the repository are subtrees of software maintained elsewhere. -Some of these are maintained by active developers of Bitcoin Core, in which case changes should probably go -directly upstream without being PRed directly against the project. They will be merged back in the next -subtree merge. +Some of these are maintained by active developers of Bitcoin Core, in which case +changes should go directly upstream without being PRed directly against the project. +They will be merged back in the next subtree merge. -Others are external projects without a tight relationship with our project. Changes to these should also -be sent upstream, but bugfixes may also be prudent to PR against Bitcoin Core so that they can be integrated -quickly. Cosmetic changes should be purely taken upstream. +Others are external projects without a tight relationship with our project. Changes +to these should also be sent upstream, but bugfixes may also be prudent to PR against +a Bitcoin Core subtree, so that they can be integrated quickly. Cosmetic changes +should be taken upstream. -There is a tool in `test/lint/git-subtree-check.sh` ([instructions](../test/lint#git-subtree-checksh)) to check a subtree directory for consistency with -its upstream repository. +There is a tool in `test/lint/git-subtree-check.sh` ([instructions](../test/lint#git-subtree-checksh)) +to check a subtree directory for consistency with its upstream repository. Current subtrees include: - src/leveldb - - Upstream at https://github.com/google/leveldb ; Maintained by Google, but - open important PRs to Core to avoid delay. + - Subtree at https://github.com/bitcoin-core/leveldb-subtree ; maintained by Core contributors. + - Upstream at https://github.com/google/leveldb ; maintained by Google. Open + important PRs to the subtree to avoid delay. - **Note**: Follow the instructions in [Upgrading LevelDB](#upgrading-leveldb) when merging upstream changes to the LevelDB subtree. - src/crc32c - Used by leveldb for hardware acceleration of CRC32C checksums for data integrity. - - Upstream at https://github.com/google/crc32c ; Maintained by Google. + - Subtree at https://github.com/bitcoin-core/crc32c-subtree ; maintained by Core contributors. + - Upstream at https://github.com/google/crc32c ; maintained by Google. - src/secp256k1 - - Upstream at https://github.com/bitcoin-core/secp256k1/ ; actively maintained by Core contributors. + - Upstream at https://github.com/bitcoin-core/secp256k1/ ; maintained by Core contributors. - src/crypto/ctaes - - Upstream at https://github.com/bitcoin-core/ctaes ; actively maintained by Core contributors. + - Upstream at https://github.com/bitcoin-core/ctaes ; maintained by Core contributors. - src/univalue - - Upstream at https://github.com/bitcoin-core/univalue ; actively maintained by Core contributors, deviates from upstream https://github.com/jgarzik/univalue + - Subtree at https://github.com/bitcoin-core/univalue-subtree ; maintained by Core contributors. + - Deviates from upstream https://github.com/jgarzik/univalue. + +- src/minisketch + - Upstream at https://github.com/sipa/minisketch ; maintained by Core contributors. Upgrading LevelDB --------------------- @@ -1158,7 +1199,7 @@ A few guidelines for introducing and reviewing new RPC interfaces: - Don't forget to fill in the argument names correctly in the RPC command table. - - *Rationale*: If not, the call can not be used with name-based arguments. + - *Rationale*: If not, the call cannot be used with name-based arguments. - Add every non-string RPC argument `(method, idx, name)` to the table `vRPCConvertParams` in `rpc/client.cpp`. @@ -1213,6 +1254,12 @@ A few guidelines for introducing and reviewing new RPC interfaces: - *Rationale*: User-facing consistency. +- Use `fs::path::u8string()` and `fs::u8path()` functions when converting path + to JSON strings, not `fs::PathToString` and `fs::PathFromString` + + - *Rationale*: JSON strings are Unicode strings, not byte strings, and + RFC8259 requires JSON to be encoded as UTF-8. + Internal interface guidelines ----------------------------- diff --git a/doc/files.md b/doc/files.md index e670d77ae5..f88d3f91a1 100644 --- a/doc/files.md +++ b/doc/files.md @@ -56,7 +56,6 @@ Subdirectory | File(s) | Description `indexes/coinstats/db/` | LevelDB database | Coinstats index; *optional*, used if `-coinstatsindex=1` `wallets/` | | [Contains wallets](#multi-wallet-environment); can be specified by `-walletdir` option; if `wallets/` subdirectory does not exist, wallets reside in the [data directory](#data-directory-location) `./` | `anchors.dat` | Anchor IP address database, created on shutdown and deleted at startup. Anchors are last known outgoing block-relay-only peers that are tried to re-connect to on startup -`./` | `banlist.dat` | Stores the addresses/subnets of banned nodes (deprecated). `bitcoind` or `bitcoin-qt` no longer save the banlist to this file, but read it on startup if `banlist.json` is not present. `./` | `banlist.json` | Stores the addresses/subnets of banned nodes. `./` | `bitcoin.conf` | User-defined [configuration settings](bitcoin-conf.md) for `bitcoind` or `bitcoin-qt`. File is not written to by the software and must be created manually. Path can be specified by `-conf` option `./` | `bitcoind.pid` | Stores the process ID (PID) of `bitcoind` or `bitcoin-qt` while running; created at start and deleted on shutdown; can be specified by `-pid` option @@ -114,6 +113,7 @@ These subdirectories and files are no longer used by Bitcoin Core: Path | Description | Repository notes ---------------|-------------|----------------- +`banlist.dat` | Stores the addresses/subnets of banned nodes; superseded by `banlist.json` in 22.0 and completely ignored in 23.0 | [PR #20966](https://github.com/bitcoin/bitcoin/pull/20966), [PR #22570](https://github.com/bitcoin/bitcoin/pull/22570) `blktree/` | Blockchain index; replaced by `blocks/index/` in [0.8.0](https://github.com/bitcoin/bitcoin/blob/master/doc/release-notes/release-notes-0.8.0.md#improvements) | [PR #2231](https://github.com/bitcoin/bitcoin/pull/2231), [`8fdc94cc`](https://github.com/bitcoin/bitcoin/commit/8fdc94cc8f0341e96b1edb3a5b56811c0b20bd15) `coins/` | Unspent transaction output database; replaced by `chainstate/` in 0.8.0 | [PR #2231](https://github.com/bitcoin/bitcoin/pull/2231), [`8fdc94cc`](https://github.com/bitcoin/bitcoin/commit/8fdc94cc8f0341e96b1edb3a5b56811c0b20bd15) `blkindex.dat` | Blockchain index BDB database; replaced by {`chainstate/`, `blocks/index/`, `blocks/revNNNNN.dat`[\[2\]](#note2)} in 0.8.0 | [PR #1677](https://github.com/bitcoin/bitcoin/pull/1677) diff --git a/doc/fuzzing.md b/doc/fuzzing.md index 6fc9077e4c..9abfbc9213 100644 --- a/doc/fuzzing.md +++ b/doc/fuzzing.md @@ -16,6 +16,13 @@ $ FUZZ=process_message src/test/fuzz/fuzz # abort fuzzing using ctrl-c ``` +There is also a runner script to execute all fuzz targets. Refer to +`./test/fuzz/test_runner.py --help` for more details. + +## Overview of Bitcoin Core fuzzing + +[Google](https://github.com/google/fuzzing/) has a good overview of fuzzing in general, with contributions from key architects of some of the most-used fuzzers. [This paper](https://agroce.github.io/bitcoin_report.pdf) includes an external overview of the status of Bitcoin Core fuzzing, as of summer 2021. [John Regehr](https://blog.regehr.org/archives/1687) provides good advice on writing code that assists fuzzers in finding bugs, which is useful for developers to keep in mind. + ## Fuzzing harnesses and output [`process_message`](https://github.com/bitcoin/bitcoin/blob/master/src/test/fuzz/process_message.cpp) is a fuzzing harness for the [`ProcessMessage(...)` function (`net_processing`)](https://github.com/bitcoin/bitcoin/blob/master/src/net_processing.cpp). The available fuzzing harnesses are found in [`src/test/fuzz/`](https://github.com/bitcoin/bitcoin/tree/master/src/test/fuzz). @@ -64,6 +71,15 @@ block^@M-^?M-^?M-^?M-^?M-^?nM-^?M-^? In this case the fuzzer managed to create a `block` message which when passed to `ProcessMessage(...)` increased coverage. +It is possible to specify `bitcoind` arguments to the `fuzz` executable. +Depending on the test, they may be ignored or consumed and alter the behavior +of the test. Just make sure to use double-dash to distinguish them from the +fuzzer's own arguments: + +```sh +$ FUZZ=address_deserialize_v2 src/test/fuzz/fuzz -runs=1 fuzz_seed_corpus/address_deserialize_v2 --checkaddrman=5 --printtoconsole=1 +``` + ## Fuzzing corpora The project's collection of seed corpora is found in the [`bitcoin-core/qa-assets`](https://github.com/bitcoin-core/qa-assets) repo. @@ -83,6 +99,10 @@ INFO: seed corpus: files: 991 min: 1b max: 1858b total: 288291b rss: 150Mb … ``` +## Run without sanitizers for increased throughput + +Fuzzing on a harness compiled with `--with-sanitizers=address,fuzzer,undefined` is good for finding bugs. However, the very slow execution even under libFuzzer will limit the ability to find new coverage. A good approach is to perform occasional long runs without the additional bug-detectors (configure `--with-sanitizers=fuzzer`) and then merge new inputs into a corpus as described in the qa-assets repo (https://github.com/bitcoin-core/qa-assets/blob/main/.github/PULL_REQUEST_TEMPLATE.md). Patience is useful; even with improved throughput, libFuzzer may need days and 10s of millions of executions to reach deep/hard targets. + ## Reproduce a fuzzer crash reported by the CI - `cd` into the `qa-assets` directory and update it with `git pull qa-assets` @@ -247,6 +267,73 @@ $ honggfuzz/honggfuzz --exit_upon_crash --quiet --timeout 4 -n 1 -Q \ -debug ``` +# Fuzzing Bitcoin Core using Eclipser (v1.x) + +## Quickstart guide + +To quickly get started fuzzing Bitcoin Core using [Eclipser v1.x](https://github.com/SoftSec-KAIST/Eclipser/tree/v1.x): + +```sh +$ git clone https://github.com/bitcoin/bitcoin +$ cd bitcoin/ +$ sudo vim /etc/apt/sources.list # Uncomment the lines starting with 'deb-src'. +$ sudo apt-get update +$ sudo apt-get build-dep qemu +$ sudo apt-get install libtool libtool-bin wget automake autoconf bison gdb +``` + +At this point, you must install the .NET core. The process differs, depending on your Linux distribution. +See [this link](https://docs.microsoft.com/en-us/dotnet/core/install/linux) for details. +On ubuntu 20.04, the following should work: + +```sh +$ wget -q https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb +$ sudo dpkg -i packages-microsoft-prod.deb +$ rm packages-microsoft-prod.deb +$ sudo apt-get update +$ sudo apt-get install -y dotnet-sdk-2.1 +``` + +You will also want to make sure Python is installed as `python` for the Eclipser install to succeed. + +```sh +$ git clone https://github.com/SoftSec-KAIST/Eclipser.git +$ cd Eclipser +$ git checkout v1.x +$ make +$ cd .. +$ ./autogen.sh +$ ./configure --enable-fuzz +$ make +$ mkdir -p outputs/ +$ FUZZ=bech32 dotnet Eclipser/build/Eclipser.dll fuzz -p src/test/fuzz/fuzz -t 36000 -o outputs --src stdin +``` + +This will perform 10 hours of fuzzing. + +To make further use of the inputs generated by Eclipser, you +must first decode them: + +```sh +$ dotnet Eclipser/build/Eclipser.dll decode -i outputs/testcase -o decoded_outputs +``` +This will place raw inputs in the directory `decoded_outputs/decoded_stdins`. Crashes are in the `outputs/crashes` directory, and must +be decoded in the same way. + +Fuzzing with Eclipser will likely be much more effective if using an existing corpus: + +```sh +$ git clone https://github.com/bitcoin-core/qa-assets +$ FUZZ=bech32 dotnet Eclipser/build/Eclipser.dll fuzz -p src/test/fuzz/fuzz -t 36000 -i qa-assets/fuzz_seed_corpus/bech32 outputs --src stdin +``` + +Note that fuzzing with Eclipser on certain targets (those that create 'full nodes', e.g. `process_message*`) will, +for now, slowly fill `/tmp/` with improperly cleaned-up files, which will cause spurious crashes. +See [this proposed patch](https://github.com/bitcoin/bitcoin/pull/22472) for more information. + +Read the [Eclipser documentation for v1.x](https://github.com/SoftSec-KAIST/Eclipser/tree/v1.x) for more details on using Eclipser. + + # OSS-Fuzz Bitcoin Core participates in Google's [OSS-Fuzz](https://github.com/google/oss-fuzz/tree/master/projects/bitcoin-core) diff --git a/doc/gitian-building.md b/doc/gitian-building.md deleted file mode 100644 index 3a48f4a0b3..0000000000 --- a/doc/gitian-building.md +++ /dev/null @@ -1,4 +0,0 @@ -Gitian building -================ - -This file was moved to [the Bitcoin Core documentation repository](https://github.com/bitcoin-core/docs/blob/master/gitian-building.md) at [https://github.com/bitcoin-core/docs](https://github.com/bitcoin-core/docs). diff --git a/doc/i2p.md b/doc/i2p.md index 27ef4d9d9f..ee650f3999 100644 --- a/doc/i2p.md +++ b/doc/i2p.md @@ -10,11 +10,22 @@ started with I2P terminology. ## Run Bitcoin Core with an I2P router (proxy) A running I2P router (proxy) with [SAM](https://geti2p.net/en/docs/api/samv3) -enabled is required (there is an [official one](https://geti2p.net) and -[a few alternatives](https://en.wikipedia.org/wiki/I2P#Routers)). Notice the IP -address and port the SAM proxy is listening to; usually, it is -`127.0.0.1:7656`. Once it is up and running with SAM enabled, use the following -Bitcoin Core options: +enabled is required. Options include: + +- [i2prouter (I2P Router)](https://geti2p.net), the official implementation in + Java +- [i2pd (I2P Daemon)](https://github.com/PurpleI2P/i2pd) + ([documentation](https://i2pd.readthedocs.io/en/latest)), a lighter + alternative in C++ (successfully tested with version 2.23 and up; version 2.36 + or later recommended) +- [i2p-zero](https://github.com/i2p-zero/i2p-zero) +- [other alternatives](https://en.wikipedia.org/wiki/I2P#Routers) + +Note the IP address and port the SAM proxy is listening to; usually, it is +`127.0.0.1:7656`. + +Once an I2P router with SAM enabled is up and running, use the following Bitcoin +Core configuration options: ``` -i2psam= @@ -42,18 +53,37 @@ named `i2p_private_key` in the Bitcoin Core data directory. ## Additional configuration options related to I2P -You may set the `debug=i2p` config logging option to have additional -information in the debug log about your I2P configuration and connections. Run -`bitcoin-cli help logging` for more information. +``` +-debug=i2p +``` -It is possible to restrict outgoing connections in the usual way with -`onlynet=i2p`. I2P support was added to Bitcoin Core in version 22.0 (mid 2021) -and there may be fewer I2P peers than Tor or IP ones. Therefore, using -`onlynet=i2p` alone (without other `onlynet=`) may make a node more susceptible -to [Sybil attacks](https://en.bitcoin.it/wiki/Weaknesses#Sybil_attack). Use +Set the `debug=i2p` config logging option to see additional information in the +debug log about your I2P configuration and connections. Run `bitcoin-cli help +logging` for more information. + +``` +-onlynet=i2p +``` + +Make outgoing connections only to I2P addresses. Incoming connections are not +affected by this option. It can be specified multiple times to allow multiple +network types, e.g. onlynet=onion, onlynet=i2p. + +I2P support was added to Bitcoin Core in version 22.0 and there may be fewer I2P +peers than Tor or IP ones. Therefore, using I2P alone without other networks may +make a node more susceptible to [Sybil +attacks](https://en.bitcoin.it/wiki/Weaknesses#Sybil_attack). You can use `bitcoin-cli -addrinfo` to see the number of I2P addresses known to your node. -## I2P related information in Bitcoin Core +Another consideration with `onlynet=i2p` is that the initial blocks download +phase when syncing up a new node can be very slow. This phase can be sped up by +using other networks, for instance `onlynet=onion`, at the same time. + +In general, a node can be run with both onion and I2P hidden services (or +any/all of IPv4/IPv6/onion/I2P), which can provide a potential fallback if one +of the networks has issues. + +## I2P-related information in Bitcoin Core There are several ways to see your I2P address in Bitcoin Core: - in the debug log (grep for `AddLocal`, the I2P address ends in `.b32.i2p`) diff --git a/doc/managing-wallets.md b/doc/managing-wallets.md new file mode 100644 index 0000000000..6c1e13c503 --- /dev/null +++ b/doc/managing-wallets.md @@ -0,0 +1,123 @@ +# Managing the Wallet + +## 1. Backing Up and Restoring The Wallet + +### 1.1 Creating the Wallet + +Since version 0.21, Bitcoin Core no longer has a default wallet. +Wallets can be created with the `createwallet` RPC or with the `Create wallet` GUI menu item. + +In the GUI, the `Create a new wallet` button is displayed on the main screen when there is no wallet loaded. Alternatively, there is the option `File` ->`Create wallet`. + +The following command, for example, creates a descriptor wallet. More information about this command may be found by running `bitcoin-cli help createwallet`. + +``` +$ bitcoin-cli createwallet "wallet-01" +``` + +By default, wallets are created in the `wallets` folder of the data directory, which varies by operating system, as shown below. The user can change the default by using the `-datadir` or `-walletdir` initialization parameters. + +| Operating System | Default wallet directory | +| -----------------|:------------------------------------------------------------| +| Linux | `/home//.bitcoin/wallets` | +| Windows | `C:\Users\\AppData\Roaming\Bitcoin\wallets` | +| macOS | `/Users//Library/Application Support/Bitcoin/wallets` | + +### 1.2 Encrypting the Wallet + +The `wallet.dat` file is not encrypted by default and is, therefore, vulnerable if an attacker gains access to the device where the wallet or the backups are stored. + +Wallet encryption may prevent unauthorized access. However, this significantly increases the risk of losing coins due to forgotten passphrases. There is no way to recover a passphrase. This tradeoff should be well thought out by the user. + +Wallet encryption may also not protect against more sophisticated attacks. An attacker can, for example, obtain the password by installing a keylogger on the user's machine. + +After encrypting the wallet or changing the passphrase, a new backup needs to be created immediately. The reason is that the keypool is flushed and a new HD seed is generated after encryption. Any bitcoins received by the new seed cannot be recovered from the previous backups. + +The wallet's private key may be encrypted with the following command: + +``` +$ bitcoin-cli -rpcwallet="wallet-01" encryptwallet "passphrase" +``` + +Once encrypted, the passphrase can be changed with the `walletpassphrasechange` command. + +``` +$ bitcoin-cli -rpcwallet="wallet-01" walletpassphrasechange "oldpassphrase" "newpassphrase" +``` + +The argument passed to `-rpcwallet` is the name of the wallet to be encrypted. + +Only the wallet's private key is encrypted. All other wallet information, such as transactions, is still visible. + +The wallet's private key can also be encrypted in the `createwallet` command via the `passphrase` argument: + +``` +$ bitcoin-cli -named createwallet wallet_name="wallet-01" passphrase="passphrase" +``` + +Note that if the passphrase is lost, all the coins in the wallet will also be lost forever. + +### 1.3 Unlocking the Wallet + +If the wallet is encrypted and the user tries any operation related to private keys, such as sending bitcoins, an error message will be displayed. + +``` +$ bitcoin-cli -rpcwallet="wallet-01" sendtoaddress "tb1qw508d6qejxtdg4y5r3zarvary0c5xw7kxpjzsx" 0.01 +error code: -13 +error message: +Error: Please enter the wallet passphrase with walletpassphrase first. +``` + +To unlock the wallet and allow it to run these operations, the `walletpassphrase` RPC is required. + +This command takes the passphrase and an argument called `timeout`, which specifies the time in seconds that the wallet decryption key is stored in memory. After this period expires, the user needs to execute this RPC again. + +``` +$ bitcoin-cli -rpcwallet="wallet-01" walletpassphrase "passphrase" 120 +``` + +In the GUI, there is no specific menu item to unlock the wallet. When the user sends bitcoins, the passphrase will be prompted automatically. + +### 1.4 Backing Up the Wallet + +To backup the wallet, the `backupwallet` RPC or the `Backup Wallet` GUI menu item must be used to ensure the file is in a safe state when the copy is made. + +In the RPC, the destination parameter must include the name of the file. Otherwise, the command will return an error message like "Error: Wallet backup failed!" for descriptor wallets. If it is a legacy wallet, it will be copied and a file will be created with the default file name `wallet.dat`. + +``` +$ bitcoin-cli -rpcwallet="wallet-01" backupwallet /home/node01/Backups/backup-01.dat +``` + +In the GUI, the wallet is selected in the `Wallet` drop-down list in the upper right corner. If this list is not present, the wallet can be loaded in `File` ->`Open wallet` if necessary. Then, the backup can be done in `File` -> `Backup Wallet...`. + +This backup file can be stored on one or multiple offline devices, which must be reliable enough to work in an emergency and be malware free. Backup files can be regularly tested to avoid problems in the future. + +If the computer has malware, it can compromise the wallet when recovering the backup file. One way to minimize this is to not connect the backup to an online device. + +If both the wallet and all backups are lost for any reason, the bitcoins related to this wallet will become permanently inaccessible. + +### 1.5 Backup Frequency + +The original Bitcoin Core wallet was a collection of unrelated private keys. If a non-HD wallet had received funds to an address and then was restored from a backup made before the address was generated, then any funds sent to that address would have been lost because there was no deterministic mechanism to derive the address again. + +Bitcoin Core [version 0.13](https://github.com/bitcoin/bitcoin/blob/master/doc/release-notes/release-notes-0.13.0.md) introduced HD wallets with deterministic key derivation. With HD wallets, users no longer lose funds when restoring old backups because all addresses are derived from the HD wallet seed. + +This means that a single backup is enough to recover the coins at any time. It is still recommended to make regular backups (once a week) or after a significant number of new transactions to maintain the metadata, such as labels. Metadata cannot be retrieved from a blockchain rescan, so if the backup is too old, the metadata will be lost forever. + +Wallets created before version 0.13 are not HD and must be backed up every 100 keys used since the previous backup, or even more often to maintain the metadata. + +### 1.6 Restoring the Wallet From a Backup + +To restore a wallet, the `restorewallet` RPC must be used. + +``` +$ bitcoin-cli restorewallet "restored-wallet" /home/node01/Backups/backup-01.dat +``` + +After that, `getwalletinfo` can be used to check if the wallet has been fully restored. + +``` +$ bitcoin-cli -rpcwallet="restored-wallet" getwalletinfo +``` + +The restored wallet can also be loaded in the GUI via `File` ->`Open wallet`. \ No newline at end of file diff --git a/doc/multisig-tutorial.md b/doc/multisig-tutorial.md new file mode 100644 index 0000000000..0793040418 --- /dev/null +++ b/doc/multisig-tutorial.md @@ -0,0 +1,241 @@ +# 1. Multisig Tutorial + +Currently, it is possible to create a multisig wallet using Bitcoin Core only. + +Although there is already a brief explanation about the multisig in the [Descriptors documentation](https://github.com/bitcoin/bitcoin/blob/master/doc/descriptors.md#multisig), this tutorial proposes to use the signet (instead of regtest), bringing the reader closer to a real environment and explaining some functions in more detail. + +This tutorial uses [jq](https://github.com/stedolan/jq) JSON processor to process the results from RPC and stores the relevant values in bash variables. This makes the tutorial reproducible and easier to follow step by step. + +Before starting this tutorial, start the bitcoin node on the signet network. + +```bash +./src/bitcoind -signet -daemon +``` + +This tutorial also uses the default WPKH derivation path to get the xpubs and does not conform to [BIP 45](https://github.com/bitcoin/bips/blob/master/bip-0045.mediawiki) or [BIP 87](https://github.com/bitcoin/bips/blob/master/bip-0087.mediawiki). + +At the time of writing, there is no way to extract a specific path from wallets in Bitcoin Core. For this, an external signer/xpub can be used. + +[PR #22341](https://github.com/bitcoin/bitcoin/pull/22341), which is still under development, introduces a new wallet RPC `getxpub`. It takes a BIP32 path as an argument and returns the xpub, along with the master key fingerprint. + +## 1.1 Basic Multisig Workflow + +### 1.1 Create the Descriptor Wallets + +For a 2-of-3 multisig, create 3 descriptor wallets. It is important that they are of the descriptor type in order to retrieve the wallet descriptors. These wallets contain HD seed and private keys, which will be used to sign the PSBTs and derive the xpub. + +These three wallets should not be used directly for privacy reasons (public key reuse). They should only be used to sign transactions for the (watch-only) multisig wallet. + +```bash +for ((n=1;n<=3;n++)) +do + ./src/bitcoin-cli -signet -named createwallet wallet_name="participant_${n}" descriptors=true +done +``` + +Extract the xpub of each wallet. To do this, the `listdescriptors` RPC is used. By default, Bitcoin Core single-sig wallets are created using path `m/44'/1'/0'` for PKH, `m/84'/1'/0'` for WPKH, `m/49'/1'/0'` for P2WPKH-nested-in-P2SH and `m/86'/1'/0'` for P2TR based accounts. Each of them uses the chain 0 for external addresses and chain 1 for internal ones, as shown in the example below. + +``` +wpkh([1004658e/84'/1'/0']tpubDCBEcmVKbfC9KfdydyLbJ2gfNL88grZu1XcWSW9ytTM6fitvaRmVyr8Ddf7SjZ2ZfMx9RicjYAXhuh3fmLiVLPodPEqnQQURUfrBKiiVZc8/0/*)#g8l47ngv + +wpkh([1004658e/84'/1'/0']tpubDCBEcmVKbfC9KfdydyLbJ2gfNL88grZu1XcWSW9ytTM6fitvaRmVyr8Ddf7SjZ2ZfMx9RicjYAXhuh3fmLiVLPodPEqnQQURUfrBKiiVZc8/1/*)#en65rxc5 +``` + +The suffix (after #) is the checksum. Descriptors can optionally be suffixed with a checksum to protect against typos or copy-paste errors. +All RPCs in Bitcoin Core will include the checksum in their output. + +```bash +declare -A xpubs + +for ((n=1;n<=3;n++)) +do + xpubs["internal_xpub_${n}"]=$(./src/bitcoin-cli -signet -rpcwallet="participant_${n}" listdescriptors | jq '.descriptors | [.[] | select(.desc | startswith("wpkh") and contains("/1/*"))][0] | .desc' | grep -Po '(?<=\().*(?=\))') + + xpubs["external_xpub_${n}"]=$(./src/bitcoin-cli -signet -rpcwallet="participant_${n}" listdescriptors | jq '.descriptors | [.[] | select(.desc | startswith("wpkh") and contains("/0/*") )][0] | .desc' | grep -Po '(?<=\().*(?=\))') +done +``` + +`jq` is used to extract the xpub from the `wpkh` descriptor. + +The following command can be used to verify if the xpub was generated correctly. + +```bash +for x in "${!xpubs[@]}"; do printf "[%s]=%s\n" "$x" "${xpubs[$x]}" ; done +``` + +As previously mentioned, this step extracts the `m/84'/1'/0'` account instead of the path defined in [BIP 45](https://github.com/bitcoin/bips/blob/master/bip-0045.mediawiki) or [BIP 87](https://github.com/bitcoin/bips/blob/master/bip-0087.mediawiki), since there is no way to extract a specific path in Bitcoin Core at the time of writing. + +### 1.2 Define the Multisig Descriptors + +Define the external and internal multisig descriptors, add the checksum and then, join both in a JSON array. + +```bash +external_desc="wsh(sortedmulti(2,${xpubs["external_xpub_1"]},${xpubs["external_xpub_2"]},${xpubs["external_xpub_3"]}))" +internal_desc="wsh(sortedmulti(2,${xpubs["internal_xpub_1"]},${xpubs["internal_xpub_2"]},${xpubs["internal_xpub_3"]}))" + +external_desc_sum=$(./src/bitcoin-cli -signet getdescriptorinfo $external_desc | jq '.descriptor') +internal_desc_sum=$(./src/bitcoin-cli -signet getdescriptorinfo $internal_desc | jq '.descriptor') + +multisig_ext_desc="{\"desc\": $external_desc_sum, \"active\": true, \"internal\": false, \"timestamp\": \"now\"}" +multisig_int_desc="{\"desc\": $internal_desc_sum, \"active\": true, \"internal\": true, \"timestamp\": \"now\"}" + +multisig_desc="[$multisig_ext_desc, $multisig_int_desc]" +``` + +`external_desc` and `internal_desc` specify the output type (`wsh`, in this case) and the xpubs involved. They also use BIP 67 (`sortedmulti`), so the wallet can be recreated without worrying about the order of xpubs. Conceptually, descriptors describe a list of scriptPubKey (along with information for spending from it) [[source](https://github.com/bitcoin/bitcoin/issues/21199#issuecomment-780772418)]. + +Note that at least two descriptors are usually used, one for internal derivation paths and external ones. There are discussions about eliminating this redundancy, as can been seen in the issue [#17190](https://github.com/bitcoin/bitcoin/issues/17190). + +After creating the descriptors, it is necessary to add the checksum, which is required by the `importdescriptors` RPC. + +The checksum for a descriptor without one can be computed using the `getdescriptorinfo` RPC. The response has the `descriptor` field, which is the descriptor with the checksum added. + +There are other fields that can be added to the descriptors: + +* `active`: Sets the descriptor to be the active one for the corresponding output type (`wsh`, in this case). +* `internal`: Indicates whether matching outputs should be treated as something other than incoming payments (e.g. change). +* `timestamp`: Sets the time from which to start rescanning the blockchain for the descriptor, in UNIX epoch time. + +Documentation for these and other parameters can be found by typing `./src/bitcoin-cli help importdescriptors`. + +`multisig_desc` concatenates external and internal descriptors in a JSON array and then it will be used to create the multisig wallet. + +### 1.3 Create the Multisig Wallet + +To create the multisig wallet, first create an empty one (no keys, HD seed and private keys disabled). + +Then import the descriptors created in the previous step using the `importdescriptors` RPC. + +After that, `getwalletinfo` can be used to check if the wallet was created successfully. + +```bash +./src/bitcoin-cli -signet -named createwallet wallet_name="multisig_wallet_01" disable_private_keys=true blank=true descriptors=true + +./src/bitcoin-cli -signet -rpcwallet="multisig_wallet_01" importdescriptors "$multisig_desc" + +./src/bitcoin-cli -signet -rpcwallet="multisig_wallet_01" getwalletinfo +``` + +Once the wallets have already been created and this tutorial needs to be repeated or resumed, it is not necessary to recreate them, just load them with the command below: + +```bash +for ((n=1;n<=3;n++)); do ./src/bitcoin-cli -signet loadwallet "participant_${n}"; done +``` + +### 1.4 Fund the wallet + +The wallet can receive signet coins by generating a new address and passing it as parameters to `getcoins.py` script. + +This script will print a captcha in dot-matrix to the terminal, using unicode Braille characters. After solving the captcha, the coins will be sent directly to the address or wallet (according to the parameters). + +The url used by the script can also be accessed directly. At time of writing, the url is [`https://signetfaucet.com`](https://signetfaucet.com). + +Coins received by the wallet must have at least 1 confirmation before they can be spent. It is necessary to wait for a new block to be mined before continuing. + +```bash +receiving_address=$(./src/bitcoin-cli -signet -rpcwallet="multisig_wallet_01" getnewaddress) + +./contrib/signet/getcoins.py -c ./src/bitcoin-cli -a $receiving_address +``` + +To copy the receiving address onto the clipboard, use the following command. This can be useful when getting coins via the signet faucet mentioned above. + +```bash +echo -n "$receiving_address" | xclip -sel clip +``` + +The `getbalances` RPC may be used to check the balance. Coins with `trusted` status can be spent. + +```bash +./src/bitcoin-cli -signet -rpcwallet="multisig_wallet_01" getbalances +``` + +### 1.5 Create a PSBT + +Unlike singlesig wallets, multisig wallets cannot create and sign transactions directly because they require the signatures of the co-signers. Instead they create a Partially Signed Bitcoin Transaction (PSBT). + +PSBT is a data format that allows wallets and other tools to exchange information about a Bitcoin transaction and the signatures necessary to complete it. [[source](https://bitcoinops.org/en/topics/psbt/)] + +The current PSBT version (v0) is defined in [BIP 174](https://github.com/bitcoin/bips/blob/master/bip-0174.mediawiki). + +For simplicity, the destination address is taken from the `participant_1` wallet in the code above, but it can be any valid bitcoin address. + +The `walletcreatefundedpsbt` RPC is used to create and fund a transaction in the PSBT format. It is the first step in creating the PSBT. + +```bash +balance=$(./src/bitcoin-cli -signet -rpcwallet="multisig_wallet_01" getbalance) + +amount=$(echo "$balance * 0.8" | bc -l | sed -e 's/^\./0./' -e 's/^-\./-0./') + +destination_addr=$(./src/bitcoin-cli -signet -rpcwallet="participant_1" getnewaddress) + +funded_psbt=$(./src/bitcoin-cli -signet -named -rpcwallet="multisig_wallet_01" walletcreatefundedpsbt outputs="{\"$destination_addr\": $amount}" | jq -r '.psbt') +``` + +There is also the `createpsbt` RPC, which serves the same purpose, but it has no access to the wallet or to the UTXO set. It is functionally the same as `createrawtransaction` and just drops the raw transaction into an otherwise blank PSBT. [[source](https://bitcointalk.org/index.php?topic=5131043.msg50573609#msg50573609)] In most cases, `walletcreatefundedpsbt` solves the problem. + +The `send` RPC can also return a PSBT if more signatures are needed to sign the transaction. + +### 1.6 Decode or Analyze the PSBT + +Optionally, the PSBT can be decoded to a JSON format using `decodepsbt` RPC. + +The `analyzepsbt` RPC analyzes and provides information about the current status of a PSBT and its inputs, e.g. missing signatures. + +```bash +./src/bitcoin-cli -signet decodepsbt $funded_psbt + +./src/bitcoin-cli -signet analyzepsbt $funded_psbt +``` + +### 1.7 Update the PSBT + +In the code above, two PSBTs are created. One signed by `participant_1` wallet and other, by the `participant_2` wallet. + +The `walletprocesspsbt` is used by the wallet to sign a PSBT. + +```bash +psbt_1=$(./src/bitcoin-cli -signet -rpcwallet="participant_1" walletprocesspsbt $funded_psbt | jq '.psbt') + +psbt_2=$(./src/bitcoin-cli -signet -rpcwallet="participant_2" walletprocesspsbt $funded_psbt | jq '.psbt') +``` + +### 1.8 Combine the PSBT + +The PSBT, if signed separately by the co-signers, must be combined into one transaction before being finalized. This is done by `combinepsbt` RPC. + +```bash +combined_psbt=$(./src/bitcoin-cli -signet combinepsbt "[$psbt_1, $psbt_2]") +``` + +There is an RPC called `joinpsbts`, but it has a different purpose than `combinepsbt`. `joinpsbts` joins the inputs from multiple distinct PSBTs into one PSBT. + +In the example above, the PSBTs are the same, but signed by different participants. If the user tries to merge them using `joinpsbts`, the error `Input txid:pos exists in multiple PSBTs` is returned. To be able to merge different PSBTs into one, they must have different inputs and outputs. + +### 1.9 Finalize and Broadcast the PSBT + +The `finalizepsbt` RPC is used to produce a network serialized transaction which can be broadcast with `sendrawtransaction`. + +It checks that all inputs have complete scriptSigs and scriptWitnesses and, if so, encodes them into network serialized transactions. + +```bash +finalized_psbt_hex=$(./src/bitcoin-cli -signet finalizepsbt $combined_psbt | jq -r '.hex') + +./src/bitcoin-cli -signet sendrawtransaction $finalized_psbt_hex +``` + +### 1.10 Alternative Workflow (PSBT sequential signatures) + +Instead of each wallet signing the original PSBT and combining them later, the wallets can also sign the PSBTs sequentially. This is less scalable than the previously presented parallel workflow, but it works. + +After that, the rest of the process is the same: the PSBT is finalized and transmitted to the network. + +```bash +psbt_1=$(./src/bitcoin-cli -signet -rpcwallet="participant_1" walletprocesspsbt $funded_psbt | jq -r '.psbt') + +psbt_2=$(./src/bitcoin-cli -signet -rpcwallet="participant_2" walletprocesspsbt $psbt_1 | jq -r '.psbt') + +finalized_psbt_hex=$(./src/bitcoin-cli -signet finalizepsbt $psbt_2 | jq -r '.hex') + +./src/bitcoin-cli -signet sendrawtransaction $finalized_psbt_hex +``` \ No newline at end of file diff --git a/doc/p2p-bad-ports.md b/doc/p2p-bad-ports.md new file mode 100644 index 0000000000..0dd7d36cf4 --- /dev/null +++ b/doc/p2p-bad-ports.md @@ -0,0 +1,114 @@ +When Bitcoin Core automatically opens outgoing P2P connections it chooses +a peer (address and port) from its list of potential peers. This list is +populated with unchecked data, gossiped over the P2P network by other peers. + +A malicious actor may gossip an address:port where no Bitcoin node is listening, +or one where a service is listening that is not related to the Bitcoin network. +As a result, this service may occasionally get connection attempts from Bitcoin +nodes. + +"Bad" ports are ones used by services which are usually not open to the public +and usually require authentication. A connection attempt (by Bitcoin Core, +trying to connect because it thinks there is a Bitcoin node on that +address:port) to such service may be considered a malicious action by an +ultra-paranoid administrator. An example for such a port is 22 (ssh). On the +other hand, connection attempts to public services that usually do not require +authentication are unlikely to be considered a malicious action, +e.g. port 80 (http). + +Below is a list of "bad" ports which Bitcoin Core avoids when choosing a peer to +connect to. If a node is listening on such a port, it will likely receive less +incoming connections. + + 1: tcpmux + 7: echo + 9: discard + 11: systat + 13: daytime + 15: netstat + 17: qotd + 19: chargen + 20: ftp data + 21: ftp access + 22: ssh + 23: telnet + 25: smtp + 37: time + 42: name + 43: nicname + 53: domain + 69: tftp + 77: priv-rjs + 79: finger + 87: ttylink + 95: supdup + 101: hostname + 102: iso-tsap + 103: gppitnp + 104: acr-nema + 109: pop2 + 110: pop3 + 111: sunrpc + 113: auth + 115: sftp + 117: uucp-path + 119: nntp + 123: NTP + 135: loc-srv /epmap + 137: netbios + 139: netbios + 143: imap2 + 161: snmp + 179: BGP + 389: ldap + 427: SLP (Also used by Apple Filing Protocol) + 465: smtp+ssl + 512: print / exec + 513: login + 514: shell + 515: printer + 526: tempo + 530: courier + 531: chat + 532: netnews + 540: uucp + 548: AFP (Apple Filing Protocol) + 554: rtsp + 556: remotefs + 563: nntp+ssl + 587: smtp (rfc6409) + 601: syslog-conn (rfc3195) + 636: ldap+ssl + 989: ftps-data + 990: ftps + 993: ldap+ssl + 995: pop3+ssl + 1719: h323gatestat + 1720: h323hostcall + 1723: pptp + 2049: nfs + 3659: apple-sasl / PasswordServer + 4045: lockd + 5060: sip + 5061: sips + 6000: X11 + 6566: sane-port + 6665: Alternate IRC + 6666: Alternate IRC + 6667: Standard IRC + 6668: Alternate IRC + 6669: Alternate IRC + 6697: IRC + TLS + 10080: Amanda + +For further information see: + +[pull/23306](https://github.com/bitcoin/bitcoin/pull/23306#issuecomment-947516736) + +[pull/23542](https://github.com/bitcoin/bitcoin/pull/23542) + +[fetch.spec.whatwg.org](https://fetch.spec.whatwg.org/#port-blocking) + +[chromium.googlesource.com](https://chromium.googlesource.com/chromium/src.git/+/refs/heads/main/net/base/port_util.cc) + +[hg.mozilla.org](https://hg.mozilla.org/mozilla-central/file/tip/netwerk/base/nsIOService.cpp) diff --git a/doc/policy/README.md b/doc/policy/README.md new file mode 100644 index 0000000000..6e8686365d --- /dev/null +++ b/doc/policy/README.md @@ -0,0 +1,15 @@ +# Transaction Relay Policy + +**Policy** (Mempool or Transaction Relay Policy) is the node's set of validation rules, in addition +to consensus, enforced for unconfirmed transactions before submitting them to the mempool. These +rules are local to the node and configurable (e.g. `-minrelaytxfee`, `-limitancestorsize`, +`-incrementalRelayFee`). Policy may include restrictions on the transaction itself, the transaction +in relation to the current chain tip, and the transaction in relation to the node's mempool +contents. Policy is *not* applied to transactions in blocks. + +This documentation is not an exhaustive list of all policy rules. + +- [Mempool Limits](mempool-limits.md) +- [Mempool Replacements](mempool-replacements.md) +- [Packages](packages.md) + diff --git a/doc/policy/mempool-limits.md b/doc/policy/mempool-limits.md new file mode 100644 index 0000000000..73ab017f1b --- /dev/null +++ b/doc/policy/mempool-limits.md @@ -0,0 +1,65 @@ +# Mempool Limits + +## Definitions + +Given any two transactions Tx0 and Tx1 where Tx1 spends an output of Tx0, +Tx0 is a *parent* of Tx1 and Tx1 is a *child* of Tx0. + +A transaction's *ancestors* include, recursively, its parents, the parents of its parents, etc. +A transaction's *descendants* include, recursively, its children, the children of its children, etc. + +A mempool entry's *ancestor count* is the total number of in-mempool (unconfirmed) transactions in +its ancestor set, including itself. +A mempool entry's *descendant count* is the total number of in-mempool (unconfirmed) transactions in +its descendant set, including itself. + +A mempool entry's *ancestor size* is the aggregated virtual size of in-mempool (unconfirmed) +transactions in its ancestor set, including itself. +A mempool entry's *descendant size* is the aggregated virtual size of in-mempool (unconfirmed) +transactions in its descendant set, including itself. + +Transactions submitted to the mempool must not exceed the ancestor and descendant limits (aka +mempool *package limits*) set by the node (see `-limitancestorcount`, `-limitancestorsize`, +`-limitdescendantcount`, `-limitdescendantsize`). + +## Exemptions + +### CPFP Carve Out + +**CPFP Carve Out** if a transaction candidate for submission to the +mempool would cause some mempool entry to exceed its descendant limits, an exemption is made if all +of the following conditions are met: + +1. The candidate transaction is no more than 10,000 virtual bytes. + +2. The candidate transaction has an ancestor count of 2 (itself and exactly 1 ancestor). + +3. The in-mempool transaction's descendant count, including the candidate transaction, would only + exceed the limit by 1. + +*Rationale*: this rule was introduced to prevent pinning by domination of a transaction's descendant +limits in two-party contract protocols such as LN. Also see the [mailing list +post](https://lists.linuxfoundation.org/pipermail/bitcoin-dev/2018-November/016518.html). + +This rule was introduced in [PR #15681](https://github.com/bitcoin/bitcoin/pull/15681). + +### Single-Conflict RBF Carve Out + +When a candidate transaction for submission to the mempool would replace mempool entries, it may +also decrease the descendant count of other mempool entries. Since ancestor/descendant limits are +calculated prior to removing the would-be-replaced transactions, they may be overestimated. + +An exemption is given for a candidate transaction that would replace mempool transactions and meets +all of the following conditions: + +1. The candidate transaction has exactly 1 directly conflicting transaction. + +2. The candidate transaction does not spend any unconfirmed inputs that are not also spent by the + directly conflicting transaction. + +The following discounts are given to account for the would-be-replaced transaction(s): + +1. The descendant count limit is temporarily increased by 1. + +2. The descendant size limit temporarily is increased by the virtual size of the to-be-replaced + directly conflicting transaction. diff --git a/doc/policy/mempool-replacements.md b/doc/policy/mempool-replacements.md new file mode 100644 index 0000000000..3e844f8d7b --- /dev/null +++ b/doc/policy/mempool-replacements.md @@ -0,0 +1,69 @@ +# Mempool Replacements + +## Current Replace-by-Fee Policy + +A transaction conflicts with an in-mempool transaction ("directly conflicting transaction") if they +spend one or more of the same inputs. A transaction may conflict with multiple in-mempool +transactions. + +A transaction ("replacement transaction") may replace its directly conflicting transactions and +their in-mempool descendants (together, "original transactions") if, in addition to passing all +other consensus and policy rules, each of the following conditions are met: + +1. The directly conflicting transactions all signal replaceability explicitly. A transaction is + signaling replaceability if any of its inputs have an nSequence number less than (0xffffffff - 1). + + *Rationale*: See [BIP125 + explanation](https://github.com/bitcoin/bips/blob/master/bip-0125.mediawiki#motivation). + +2. The replacement transaction only include an unconfirmed input if that input was included in + one of the directly conflicting transactions. An unconfirmed input spends an output from a + currently-unconfirmed transaction. + + *Rationale*: When RBF was originally implemented, the mempool did not keep track of + ancestor feerates yet. This rule was suggested as a temporary restriction. + +3. The replacement transaction pays an absolute fee of at least the sum paid by the original + transactions. + + *Rationale*: Only requiring the replacement transaction to have a higher feerate could allow an + attacker to bypass node minimum relay feerate requirements and cause the network to repeatedly + relay slightly smaller replacement transactions without adding any more fees. Additionally, if + any of the original transactions would be included in the next block assembled by an economically + rational miner, a replacement policy allowing the replacement transaction to decrease the absolute + fees in the next block would be incentive-incompatible. + +4. The additional fees (difference between absolute fee paid by the replacement transaction and the + sum paid by the original transactions) pays for the replacement transaction's bandwidth at or + above the rate set by the node's incremental relay feerate. For example, if the incremental relay + feerate is 1 satoshi/vB and the replacement transaction is 500 virtual bytes total, then the + replacement pays a fee at least 500 satoshis higher than the sum of the original transactions. + + *Rationale*: Try to prevent DoS attacks where an attacker causes the network to repeatedly relay + transactions each paying a tiny additional amount in fees, e.g. just 1 satoshi. + +5. The number of original transactions does not exceed 100. More precisely, the sum of all + directly conflicting transactions' descendant counts (number of transactions inclusive of itself + and its descendants) must not exceed 100; it is possible that this overestimates the true number + of original transactions. + + *Rationale*: Try to prevent DoS attacks where an attacker is able to easily occupy and flush out + significant portions of the node's mempool using replacements with multiple directly conflicting + transactions, each with large descendant sets. + +This set of rules is similar but distinct from BIP125. + +## History + +* Opt-in full replace-by-fee (without inherited signaling) honoured in mempool and mining as of + **v0.12.0** ([PR 6871](https://github.com/bitcoin/bitcoin/pull/6871)). + +* [BIP125](https://github.com/bitcoin/bips/blob/master/bip-0125.mediawiki) defined based on + Bitcoin Core implementation. + +* The incremental relay feerate used to calculate the required additional fees is distinct from + `minRelayTxFee` and configurable using `-incrementalrelayfee` + ([PR #9380](https://github.com/bitcoin/bitcoin/pull/9380)). + +* RBF enabled by default in the wallet GUI as of **v0.18.1** ([PR + #11605](https://github.com/bitcoin/bitcoin/pull/11605)). diff --git a/doc/policy/packages.md b/doc/policy/packages.md new file mode 100644 index 0000000000..7f7fbe18cd --- /dev/null +++ b/doc/policy/packages.md @@ -0,0 +1,74 @@ +# Package Mempool Accept + +## Definitions + +A **package** is an ordered list of transactions, representable by a connected Directed Acyclic +Graph (a directed edge exists between a transaction that spends the output of another transaction). + +For every transaction `t` in a **topologically sorted** package, if any of its parents are present +in the package, they appear somewhere in the list before `t`. + +A **child-with-unconfirmed-parents** package is a topologically sorted package that consists of +exactly one child and all of its unconfirmed parents (no other transactions may be present). +The last transaction in the package is the child, and its package can be canonically defined based +on the current state: each of its inputs must be available in the UTXO set as of the current chain +tip or some preceding transaction in the package. + +## Package Mempool Acceptance Rules + +The following rules are enforced for all packages: + +* Packages cannot exceed `MAX_PACKAGE_COUNT=25` count and `MAX_PACKAGE_SIZE=101KvB` total size + (#20833) + + - *Rationale*: This is already enforced as mempool ancestor/descendant limits. If + transactions in a package are all related, exceeding this limit would mean that the package + can either be split up or it wouldn't pass individual mempool policy. + + - Note that, if these mempool limits change, package limits should be reconsidered. Users may + also configure their mempool limits differently. + +* Packages must be topologically sorted. (#20833) + +* Packages cannot have conflicting transactions, i.e. no two transactions in a package can spend + the same inputs. Packages cannot have duplicate transactions. (#20833) + +* No transaction in a package can conflict with a mempool transaction. BIP125 Replace By Fee is + currently disabled for packages. (#20833) + + - Package RBF may be enabled in the future. + +* When packages are evaluated against ancestor/descendant limits, the union of all transactions' + descendants and ancestors is considered. (#21800) + + - *Rationale*: This is essentially a "worst case" heuristic intended for packages that are + heavily connected, i.e. some transaction in the package is the ancestor or descendant of all + the other transactions. + +The following rules are only enforced for packages to be submitted to the mempool (not enforced for +test accepts): + +* Packages must be child-with-unconfirmed-parents packages. This also means packages must contain at + least 2 transactions. (#22674) + + - *Rationale*: This allows for fee-bumping by CPFP. Allowing multiple parents makes it possible + to fee-bump a batch of transactions. Restricting packages to a defined topology is easier to + reason about and simplifies the validation logic greatly. + + - Warning: Batched fee-bumping may be unsafe for some use cases. Users and application developers + should take caution if utilizing multi-parent packages. + +* Transactions in the package that have the same txid as another transaction already in the mempool + will be removed from the package prior to submission ("deduplication"). + + - *Rationale*: Node operators are free to set their mempool policies however they please, nodes + may receive transactions in different orders, and malicious counterparties may try to take + advantage of policy differences to pin or delay propagation of transactions. As such, it's + possible for some package transaction(s) to already be in the mempool, and there is no need to + repeat validation for those transactions or double-count them in fees. + + - *Rationale*: We want to prevent potential censorship vectors. We should not reject entire + packages because we already have one of the transactions. Also, if an attacker first broadcasts + a competing package or transaction with a mutated witness, even though the two + same-txid-different-witness transactions are conflicting and cannot replace each other, the + honest package should still be considered for acceptance. diff --git a/doc/psbt.md b/doc/psbt.md index c411b31d5d..0f31cb8eba 100644 --- a/doc/psbt.md +++ b/doc/psbt.md @@ -92,6 +92,9 @@ hardware implementations will typically implement multiple roles simultaneously. #### Multisig with multiple Bitcoin Core instances +For a quick start see [Basic M-of-N multisig example using descriptor wallets and PSBTs](./descriptors.md#basic-multisig-example). +If you are using legacy wallets feel free to continue with the example provided here. + Alice, Bob, and Carol want to create a 2-of-3 multisig address. They're all using Bitcoin Core. We assume their wallets only contain the multisig funds. In case they also have a personal wallet, this can be accomplished through the diff --git a/doc/release-notes.md b/doc/release-notes.md index dc28ccb9ed..2342342ae2 100644 --- a/doc/release-notes.md +++ b/doc/release-notes.md @@ -1,5 +1,3 @@ -# Release notes now being edited on https://github.com/bitcoin-core/bitcoin-devwiki/wiki/22.0-Release-Notes-draft - *After branching off for a major version release of Bitcoin Core, use this template to create the initial release notes draft.* @@ -8,7 +6,7 @@ template to create the initial release notes draft.* for the process.* *Create the draft, named* "*version* Release Notes Draft" -*(e.g. "0.20.0 Release Notes Draft"), as a collaborative wiki in:* +*(e.g. "23.0 Release Notes Draft"), as a collaborative wiki in:* https://github.com/bitcoin-core/bitcoin-devwiki/wiki/ @@ -48,171 +46,17 @@ Compatibility ============== Bitcoin Core is supported and extensively tested on operating systems -using the Linux kernel, macOS 10.14+, and Windows 7 and newer. Bitcoin +using the Linux kernel, macOS 10.15+, and Windows 7 and newer. Bitcoin Core should also work on most other Unix-like systems but is not as frequently tested on them. It is not recommended to use Bitcoin Core on unsupported systems. -From Bitcoin Core 22.0 onwards, macOS versions earlier than 10.14 are no longer supported. - Notable changes =============== -P2P and network changes ------------------------ - -- This release removes support for Tor version 2 hidden services in favor of Tor - v3 only, as the Tor network [dropped support for Tor - v2](https://blog.torproject.org/v2-deprecation-timeline) with the release of - Tor version 0.4.6. Henceforth, Bitcoin Core ignores Tor v2 addresses; it - neither rumors them over the network to other peers, nor stores them in memory - or to `peers.dat`. (#22050) - -- Added NAT-PMP port mapping support via - [`libnatpmp`](https://miniupnp.tuxfamily.org/libnatpmp.html). (#18077) - -Updated RPCs +Example item ------------ -- Due to [BIP 350](https://github.com/bitcoin/bips/blob/master/bip-0350.mediawiki) - being implemented, behavior for all RPCs that accept addresses is changed when - a native witness version 1 (or higher) is passed. These now require a Bech32m - encoding instead of a Bech32 one, and Bech32m encoding will be used for such - addresses in RPC output as well. No version 1 addresses should be created - for mainnet until consensus rules are adopted that give them meaning - (e.g. through [BIP 341](https://github.com/bitcoin/bips/blob/master/bip-0341.mediawiki)). - Once that happens, Bech32m is expected to be used for them, so this shouldn't - affect any production systems, but may be observed on other networks where such - addresses already have meaning (like signet). (#20861) - -- The `getpeerinfo` RPC returns two new boolean fields, `bip152_hb_to` and - `bip152_hb_from`, that respectively indicate whether we selected a peer to be - in compact blocks high-bandwidth mode or whether a peer selected us as a - compact blocks high-bandwidth peer. High-bandwidth peers send new block - announcements via a `cmpctblock` message rather than the usual inv/headers - announcements. See BIP 152 for more details. (#19776) - -- `getpeerinfo` no longer returns the following fields: `addnode`, `banscore`, - and `whitelisted`, which were previously deprecated in 0.21. Instead of - `addnode`, the `connection_type` field returns manual. Instead of - `whitelisted`, the `permissions` field indicates if the peer has special - privileges. The `banscore` field has simply been removed. (#20755) - -- The following RPCs: `gettxout`, `getrawtransaction`, `decoderawtransaction`, - `decodescript`, `gettransaction`, and REST endpoints: `/rest/tx`, - `/rest/getutxos`, `/rest/block` deprecated the following fields (which are no - longer returned in the responses by default): `addresses`, `reqSigs`. - The `-deprecatedrpc=addresses` flag must be passed for these fields to be - included in the RPC response. This flag/option will be available only for this major release, after which - the deprecation will be removed entirely. Note that these fields are attributes of - the `scriptPubKey` object returned in the RPC response. However, in the response - of `decodescript` these fields are top-level attributes, and included again as attributes - of the `scriptPubKey` object. (#20286) - -- When creating a hex-encoded bitcoin transaction using the `bitcoin-tx` utility - with the `-json` option set, the following fields: `addresses`, `reqSigs` are no longer - returned in the tx output of the response. (#20286) - -- The `listbanned` RPC now returns two new numeric fields: `ban_duration` and `time_remaining`. - Respectively, these new fields indicate the duration of a ban and the time remaining until a ban expires, - both in seconds. Additionally, the `ban_created` field is repositioned to come before `banned_until`. (#21602) - -- The `getnodeaddresses` RPC now returns a "network" field indicating the - network type (ipv4, ipv6, onion, or i2p) for each address. (#21594) - -- `getnodeaddresses` now also accepts a "network" argument (ipv4, ipv6, onion, - or i2p) to return only addresses of the specified network. (#21843) - -- The `testmempoolaccept` RPC now accepts multiple transactions (still experimental at the moment, - API may be unstable). This is intended for testing transaction packages with dependency - relationships; it is not recommended for batch-validating independent transactions. In addition to - mempool policy, package policies apply: the list cannot contain more than 25 transactions or have a - total size exceeding 101K virtual bytes, and cannot conflict with (spend the same inputs as) each other or - the mempool, even if it would be a valid BIP125 replace-by-fee. There are some known limitations to - the accuracy of the test accept: it's possible for `testmempoolaccept` to return "allowed"=True for a - group of transactions, but "too-long-mempool-chain" if they are actually submitted. (#20833) - -- `addmultisigaddress` and `createmultisig` now support up to 20 keys for - Segwit addresses. (#20867) - -Changes to Wallet or GUI related RPCs can be found in the GUI or Wallet section below. - -New RPCs --------- - -Build System ------------- - -New settings ------------- - -- The `-natpmp` option has been added to use NAT-PMP to map the listening port. - If both UPnP and NAT-PMP are enabled, a successful allocation from UPnP - prevails over one from NAT-PMP. (#18077) - -Updated settings ----------------- - -Changes to Wallet or GUI related settings can be found in the GUI or Wallet section below. - -- Passing an invalid `-rpcauth` argument now cause bitcoind to fail to start. (#20461) - -Tools and Utilities -------------------- - -- A new CLI `-addrinfo` command returns the number of addresses known to the - node per network type (including Tor v2 versus v3) and total. This can be - useful to see if the node knows enough addresses in a network to use options - like `-onlynet=` or to upgrade to this release of Bitcoin Core 22.0 - that supports Tor v3 only. (#21595) - -- A new `-rpcwaittimeout` argument to `bitcoin-cli` sets the timeout - in seconds to use with `-rpcwait`. If the timeout expires, - `bitcoin-cli` will report a failure. (#21056) - -Wallet ------- - -- A new `listdescriptors` RPC is available to inspect the contents of descriptor-enabled wallets. - The RPC returns public versions of all imported descriptors, including their timestamp and flags. - For ranged descriptors, it also returns the range boundaries and the next index to generate addresses from. (#20226) - -- The `bumpfee` RPC is not available with wallets that have private keys - disabled. `psbtbumpfee` can be used instead. (#20891) - -- The `fundrawtransaction`, `send` and `walletcreatefundedpsbt` RPCs now support an `include_unsafe` option - that when `true` allows using unsafe inputs to fund the transaction. - Note that the resulting transaction may become invalid if one of the unsafe inputs disappears. - If that happens, the transaction must be funded with different inputs and republished. (#21359) - -- We now support up to 20 keys in `multi()` and `sortedmulti()` descriptors - under `wsh()`. (#20867) - -GUI changes ------------ - -Low-level changes -================= - -RPC ---- - -- The RPC server can process a limited number of simultaneous RPC requests. - Previously, if this limit was exceeded, the RPC server would respond with - [status code 500 (`HTTP_INTERNAL_SERVER_ERROR`)](https://en.wikipedia.org/wiki/List_of_HTTP_status_codes#5xx_server_errors). - Now it returns status code 503 (`HTTP_SERVICE_UNAVAILABLE`). (#18335) - -- Error codes have been updated to be more accurate for the following error cases (#18466): - - `signmessage` now returns RPC_INVALID_ADDRESS_OR_KEY (-5) if the - passed address is invalid. Previously returned RPC_TYPE_ERROR (-3). - - `verifymessage` now returns RPC_INVALID_ADDRESS_OR_KEY (-5) if the - passed address is invalid. Previously returned RPC_TYPE_ERROR (-3). - - `verifymessage` now returns RPC_TYPE_ERROR (-3) if the passed signature - is malformed. Previously returned RPC_INVALID_ADDRESS_OR_KEY (-5). - -Tests ------ - Credits ======= diff --git a/doc/release-notes/release-notes-22.0.md b/doc/release-notes/release-notes-22.0.md new file mode 100644 index 0000000000..972c91aa6f --- /dev/null +++ b/doc/release-notes/release-notes-22.0.md @@ -0,0 +1,1163 @@ +22.0 Release Notes +================== + +Bitcoin Core version 22.0 is now available from: + + + +This release includes new features, various bug fixes and performance +improvements, as well as updated translations. + +Please report bugs using the issue tracker at GitHub: + + + +To receive security and update notifications, please subscribe to: + + + +How to Upgrade +============== + +If you are running an older version, shut it down. Wait until it has completely +shut down (which might take a few minutes in some cases), then run the +installer (on Windows) or just copy over `/Applications/Bitcoin-Qt` (on Mac) +or `bitcoind`/`bitcoin-qt` (on Linux). + +Upgrading directly from a version of Bitcoin Core that has reached its EOL is +possible, but it might take some time if the data directory needs to be migrated. Old +wallet versions of Bitcoin Core are generally supported. + +Compatibility +============== + +Bitcoin Core is supported and extensively tested on operating systems +using the Linux kernel, macOS 10.14+, and Windows 7 and newer. Bitcoin +Core should also work on most other Unix-like systems but is not as +frequently tested on them. It is not recommended to use Bitcoin Core on +unsupported systems. + +From Bitcoin Core 22.0 onwards, macOS versions earlier than 10.14 are no longer supported. + +Notable changes +=============== + +P2P and network changes +----------------------- +- Added support for running Bitcoin Core as an + [I2P (Invisible Internet Project)](https://en.wikipedia.org/wiki/I2P) service + and connect to such services. See [i2p.md](https://github.com/bitcoin/bitcoin/blob/22.x/doc/i2p.md) for details. (#20685) +- This release removes support for Tor version 2 hidden services in favor of Tor + v3 only, as the Tor network [dropped support for Tor + v2](https://blog.torproject.org/v2-deprecation-timeline) with the release of + Tor version 0.4.6. Henceforth, Bitcoin Core ignores Tor v2 addresses; it + neither rumors them over the network to other peers, nor stores them in memory + or to `peers.dat`. (#22050) + +- Added NAT-PMP port mapping support via + [`libnatpmp`](https://miniupnp.tuxfamily.org/libnatpmp.html). (#18077) + +New and Updated RPCs +-------------------- + +- Due to [BIP 350](https://github.com/bitcoin/bips/blob/master/bip-0350.mediawiki) + being implemented, behavior for all RPCs that accept addresses is changed when + a native witness version 1 (or higher) is passed. These now require a Bech32m + encoding instead of a Bech32 one, and Bech32m encoding will be used for such + addresses in RPC output as well. No version 1 addresses should be created + for mainnet until consensus rules are adopted that give them meaning + (as will happen through [BIP 341](https://github.com/bitcoin/bips/blob/master/bip-0341.mediawiki)). + Once that happens, Bech32m is expected to be used for them, so this shouldn't + affect any production systems, but may be observed on other networks where such + addresses already have meaning (like signet). (#20861) + +- The `getpeerinfo` RPC returns two new boolean fields, `bip152_hb_to` and + `bip152_hb_from`, that respectively indicate whether we selected a peer to be + in compact blocks high-bandwidth mode or whether a peer selected us as a + compact blocks high-bandwidth peer. High-bandwidth peers send new block + announcements via a `cmpctblock` message rather than the usual inv/headers + announcements. See BIP 152 for more details. (#19776) + +- `getpeerinfo` no longer returns the following fields: `addnode`, `banscore`, + and `whitelisted`, which were previously deprecated in 0.21. Instead of + `addnode`, the `connection_type` field returns manual. Instead of + `whitelisted`, the `permissions` field indicates if the peer has special + privileges. The `banscore` field has simply been removed. (#20755) + +- The following RPCs: `gettxout`, `getrawtransaction`, `decoderawtransaction`, + `decodescript`, `gettransaction`, and REST endpoints: `/rest/tx`, + `/rest/getutxos`, `/rest/block` deprecated the following fields (which are no + longer returned in the responses by default): `addresses`, `reqSigs`. + The `-deprecatedrpc=addresses` flag must be passed for these fields to be + included in the RPC response. This flag/option will be available only for this major release, after which + the deprecation will be removed entirely. Note that these fields are attributes of + the `scriptPubKey` object returned in the RPC response. However, in the response + of `decodescript` these fields are top-level attributes, and included again as attributes + of the `scriptPubKey` object. (#20286) + +- When creating a hex-encoded bitcoin transaction using the `bitcoin-tx` utility + with the `-json` option set, the following fields: `addresses`, `reqSigs` are no longer + returned in the tx output of the response. (#20286) + +- The `listbanned` RPC now returns two new numeric fields: `ban_duration` and `time_remaining`. + Respectively, these new fields indicate the duration of a ban and the time remaining until a ban expires, + both in seconds. Additionally, the `ban_created` field is repositioned to come before `banned_until`. (#21602) + +- The `setban` RPC can ban onion addresses again. This fixes a regression introduced in version 0.21.0. (#20852) + +- The `getnodeaddresses` RPC now returns a "network" field indicating the + network type (ipv4, ipv6, onion, or i2p) for each address. (#21594) + +- `getnodeaddresses` now also accepts a "network" argument (ipv4, ipv6, onion, + or i2p) to return only addresses of the specified network. (#21843) + +- The `testmempoolaccept` RPC now accepts multiple transactions (still experimental at the moment, + API may be unstable). This is intended for testing transaction packages with dependency + relationships; it is not recommended for batch-validating independent transactions. In addition to + mempool policy, package policies apply: the list cannot contain more than 25 transactions or have a + total size exceeding 101K virtual bytes, and cannot conflict with (spend the same inputs as) each other or + the mempool, even if it would be a valid BIP125 replace-by-fee. There are some known limitations to + the accuracy of the test accept: it's possible for `testmempoolaccept` to return "allowed"=True for a + group of transactions, but "too-long-mempool-chain" if they are actually submitted. (#20833) + +- `addmultisigaddress` and `createmultisig` now support up to 20 keys for + Segwit addresses. (#20867) + +Changes to Wallet or GUI related RPCs can be found in the GUI or Wallet section below. + +Build System +------------ + +- Release binaries are now produced using the new `guix`-based build system. + The [/doc/release-process.md](/doc/release-process.md) document has been updated accordingly. + +Files +----- + +- The list of banned hosts and networks (via `setban` RPC) is now saved on disk + in JSON format in `banlist.json` instead of `banlist.dat`. `banlist.dat` is + only read on startup if `banlist.json` is not present. Changes are only written to the new + `banlist.json`. A future version of Bitcoin Core may completely ignore + `banlist.dat`. (#20966) + +New settings +------------ + +- The `-natpmp` option has been added to use NAT-PMP to map the listening port. + If both UPnP and NAT-PMP are enabled, a successful allocation from UPnP + prevails over one from NAT-PMP. (#18077) + +Updated settings +---------------- + +Changes to Wallet or GUI related settings can be found in the GUI or Wallet section below. + +- Passing an invalid `-rpcauth` argument now cause bitcoind to fail to start. (#20461) + +Tools and Utilities +------------------- + +- A new CLI `-addrinfo` command returns the number of addresses known to the + node per network type (including Tor v2 versus v3) and total. This can be + useful to see if the node knows enough addresses in a network to use options + like `-onlynet=` or to upgrade to this release of Bitcoin Core 22.0 + that supports Tor v3 only. (#21595) + +- A new `-rpcwaittimeout` argument to `bitcoin-cli` sets the timeout + in seconds to use with `-rpcwait`. If the timeout expires, + `bitcoin-cli` will report a failure. (#21056) + +Wallet +------ + +- External signers such as hardware wallets can now be used through the new RPC methods `enumeratesigners` and `displayaddress`. Support is also added to the `send` RPC call. This feature is experimental. See [external-signer.md](https://github.com/bitcoin/bitcoin/blob/22.x/doc/external-signer.md) for details. (#16546) + +- A new `listdescriptors` RPC is available to inspect the contents of descriptor-enabled wallets. + The RPC returns public versions of all imported descriptors, including their timestamp and flags. + For ranged descriptors, it also returns the range boundaries and the next index to generate addresses from. (#20226) + +- The `bumpfee` RPC is not available with wallets that have private keys + disabled. `psbtbumpfee` can be used instead. (#20891) + +- The `fundrawtransaction`, `send` and `walletcreatefundedpsbt` RPCs now support an `include_unsafe` option + that when `true` allows using unsafe inputs to fund the transaction. + Note that the resulting transaction may become invalid if one of the unsafe inputs disappears. + If that happens, the transaction must be funded with different inputs and republished. (#21359) + +- We now support up to 20 keys in `multi()` and `sortedmulti()` descriptors + under `wsh()`. (#20867) + +- Taproot descriptors can be imported into the wallet only after activation has occurred on the network (e.g. mainnet, testnet, signet) in use. See [descriptors.md](https://github.com/bitcoin/bitcoin/blob/22.x/doc/descriptors.md) for supported descriptors. + +GUI changes +----------- + +- External signers such as hardware wallets can now be used. These require an external tool such as [HWI](https://github.com/bitcoin-core/HWI) to be installed and configured under Options -> Wallet. When creating a new wallet a new option "External signer" will appear in the dialog. If the device is detected, its name is suggested as the wallet name. The watch-only keys are then automatically imported. Receive addresses can be verified on the device. The send dialog will automatically use the connected device. This feature is experimental and the UI may freeze for a few seconds when performing these actions. + +Low-level changes +================= + +RPC +--- + +- The RPC server can process a limited number of simultaneous RPC requests. + Previously, if this limit was exceeded, the RPC server would respond with + [status code 500 (`HTTP_INTERNAL_SERVER_ERROR`)](https://en.wikipedia.org/wiki/List_of_HTTP_status_codes#5xx_server_errors). + Now it returns status code 503 (`HTTP_SERVICE_UNAVAILABLE`). (#18335) + +- Error codes have been updated to be more accurate for the following error cases (#18466): + - `signmessage` now returns RPC_INVALID_ADDRESS_OR_KEY (-5) if the + passed address is invalid. Previously returned RPC_TYPE_ERROR (-3). + - `verifymessage` now returns RPC_INVALID_ADDRESS_OR_KEY (-5) if the + passed address is invalid. Previously returned RPC_TYPE_ERROR (-3). + - `verifymessage` now returns RPC_TYPE_ERROR (-3) if the passed signature + is malformed. Previously returned RPC_INVALID_ADDRESS_OR_KEY (-5). + +Tests +----- + +22.0 change log +=============== + +A detailed list of changes in this version follows. To keep the list to a manageable length, small refactors and typo fixes are not included, and similar changes are sometimes condensed into one line. + +### Consensus +- bitcoin/bitcoin#19438 Introduce deploymentstatus (ajtowns) +- bitcoin/bitcoin#20207 Follow-up extra comments on taproot code and tests (sipa) +- bitcoin/bitcoin#21330 Deal with missing data in signature hashes more consistently (sipa) + +### Policy +- bitcoin/bitcoin#18766 Disable fee estimation in blocksonly mode (by removing the fee estimates global) (darosior) +- bitcoin/bitcoin#20497 Add `MAX_STANDARD_SCRIPTSIG_SIZE` to policy (sanket1729) +- bitcoin/bitcoin#20611 Move `TX_MAX_STANDARD_VERSION` to policy (MarcoFalke) + +### Mining +- bitcoin/bitcoin#19937, bitcoin/bitcoin#20923 Signet mining utility (ajtowns) + +### Block and transaction handling +- bitcoin/bitcoin#14501 Fix possible data race when committing block files (luke-jr) +- bitcoin/bitcoin#15946 Allow maintaining the blockfilterindex when using prune (jonasschnelli) +- bitcoin/bitcoin#18710 Add local thread pool to CCheckQueue (hebasto) +- bitcoin/bitcoin#19521 Coinstats Index (fjahr) +- bitcoin/bitcoin#19806 UTXO snapshot activation (jamesob) +- bitcoin/bitcoin#19905 Remove dead CheckForkWarningConditionsOnNewFork (MarcoFalke) +- bitcoin/bitcoin#19935 Move SaltedHashers to separate file and add some new ones (achow101) +- bitcoin/bitcoin#20054 Remove confusing and useless "unexpected version" warning (MarcoFalke) +- bitcoin/bitcoin#20519 Handle rename failure in `DumpMempool(…)` by using the `RenameOver(…)` return value (practicalswift) +- bitcoin/bitcoin#20749, bitcoin/bitcoin#20750, bitcoin/bitcoin#21055, bitcoin/bitcoin#21270, bitcoin/bitcoin#21525, bitcoin/bitcoin#21391, bitcoin/bitcoin#21767, bitcoin/bitcoin#21866 Prune `g_chainman` usage (dongcarl) +- bitcoin/bitcoin#20833 rpc/validation: enable packages through testmempoolaccept (glozow) +- bitcoin/bitcoin#20834 Locks and docs in ATMP and CheckInputsFromMempoolAndCache (glozow) +- bitcoin/bitcoin#20854 Remove unnecessary try-block (amitiuttarwar) +- bitcoin/bitcoin#20868 Remove redundant check on pindex (jarolrod) +- bitcoin/bitcoin#20921 Don't try to invalidate genesis block in CChainState::InvalidateBlock (theStack) +- bitcoin/bitcoin#20972 Locks: Annotate CTxMemPool::check to require `cs_main` (dongcarl) +- bitcoin/bitcoin#21009 Remove RewindBlockIndex logic (dhruv) +- bitcoin/bitcoin#21025 Guard chainman chainstates with `cs_main` (dongcarl) +- bitcoin/bitcoin#21202 Two small clang lock annotation improvements (amitiuttarwar) +- bitcoin/bitcoin#21523 Run VerifyDB on all chainstates (jamesob) +- bitcoin/bitcoin#21573 Update libsecp256k1 subtree to latest master (sipa) +- bitcoin/bitcoin#21582, bitcoin/bitcoin#21584, bitcoin/bitcoin#21585 Fix assumeutxo crashes (MarcoFalke) +- bitcoin/bitcoin#21681 Fix ActivateSnapshot to use hardcoded nChainTx (jamesob) +- bitcoin/bitcoin#21796 index: Avoid async shutdown on init error (MarcoFalke) +- bitcoin/bitcoin#21946 Document and test lack of inherited signaling in RBF policy (ariard) +- bitcoin/bitcoin#22084 Package testmempoolaccept followups (glozow) +- bitcoin/bitcoin#22102 Remove `Warning:` from warning message printed for unknown new rules (prayank23) +- bitcoin/bitcoin#22112 Force port 0 in I2P (vasild) +- bitcoin/bitcoin#22135 CRegTestParams: Use `args` instead of `gArgs` (kiminuo) +- bitcoin/bitcoin#22146 Reject invalid coin height and output index when loading assumeutxo (MarcoFalke) +- bitcoin/bitcoin#22253 Distinguish between same tx and same-nonwitness-data tx in mempool (glozow) +- bitcoin/bitcoin#22261 Two small fixes to node broadcast logic (jnewbery) +- bitcoin/bitcoin#22415 Make `m_mempool` optional in CChainState (jamesob) +- bitcoin/bitcoin#22499 Update assumed chain params (sriramdvt) +- bitcoin/bitcoin#22589 net, doc: update I2P hardcoded seeds and docs for 22.0 (jonatack) + +### P2P protocol and network code +- bitcoin/bitcoin#18077 Add NAT-PMP port forwarding support (hebasto) +- bitcoin/bitcoin#18722 addrman: improve performance by using more suitable containers (vasild) +- bitcoin/bitcoin#18819 Replace `cs_feeFilter` with simple std::atomic (MarcoFalke) +- bitcoin/bitcoin#19203 Add regression fuzz harness for CVE-2017-18350. Add FuzzedSocket (practicalswift) +- bitcoin/bitcoin#19288 fuzz: Add fuzzing harness for TorController (practicalswift) +- bitcoin/bitcoin#19415 Make DNS lookup mockable, add fuzzing harness (practicalswift) +- bitcoin/bitcoin#19509 Per-Peer Message Capture (troygiorshev) +- bitcoin/bitcoin#19763 Don't try to relay to the address' originator (vasild) +- bitcoin/bitcoin#19771 Replace enum CConnMan::NumConnections with enum class ConnectionDirection (luke-jr) +- bitcoin/bitcoin#19776 net, rpc: expose high bandwidth mode state via getpeerinfo (theStack) +- bitcoin/bitcoin#19832 Put disconnecting logs into BCLog::NET category (hebasto) +- bitcoin/bitcoin#19858 Periodically make block-relay connections and sync headers (sdaftuar) +- bitcoin/bitcoin#19884 No delay in adding fixed seeds if -dnsseed=0 and peers.dat is empty (dhruv) +- bitcoin/bitcoin#20079 Treat handshake misbehavior like unknown message (MarcoFalke) +- bitcoin/bitcoin#20138 Assume that SetCommonVersion is called at most once per peer (MarcoFalke) +- bitcoin/bitcoin#20162 p2p: declare Announcement::m_state as uint8_t, add getter/setter (jonatack) +- bitcoin/bitcoin#20197 Protect onions in AttemptToEvictConnection(), add eviction protection test coverage (jonatack) +- bitcoin/bitcoin#20210 assert `CNode::m_inbound_onion` is inbound in ctor, add getter, unit tests (jonatack) +- bitcoin/bitcoin#20228 addrman: Make addrman a top-level component (jnewbery) +- bitcoin/bitcoin#20234 Don't bind on 0.0.0.0 if binds are restricted to Tor (vasild) +- bitcoin/bitcoin#20477 Add unit testing of node eviction logic (practicalswift) +- bitcoin/bitcoin#20516 Well-defined CAddress disk serialization, and addrv2 anchors.dat (sipa) +- bitcoin/bitcoin#20557 addrman: Fix new table bucketing during unserialization (jnewbery) +- bitcoin/bitcoin#20561 Periodically clear `m_addr_known` (sdaftuar) +- bitcoin/bitcoin#20599 net processing: Tolerate sendheaders and sendcmpct messages before verack (jnewbery) +- bitcoin/bitcoin#20616 Check CJDNS address is valid (lontivero) +- bitcoin/bitcoin#20617 Remove `m_is_manual_connection` from CNodeState (ariard) +- bitcoin/bitcoin#20624 net processing: Remove nStartingHeight check from block relay (jnewbery) +- bitcoin/bitcoin#20651 Make p2p recv buffer timeout 20 minutes for all peers (jnewbery) +- bitcoin/bitcoin#20661 Only select from addrv2-capable peers for torv3 address relay (sipa) +- bitcoin/bitcoin#20685 Add I2P support using I2P SAM (vasild) +- bitcoin/bitcoin#20690 Clean up logging of outbound connection type (sdaftuar) +- bitcoin/bitcoin#20721 Move ping data to `net_processing` (jnewbery) +- bitcoin/bitcoin#20724 Cleanup of -debug=net log messages (ajtowns) +- bitcoin/bitcoin#20747 net processing: Remove dropmessagestest (jnewbery) +- bitcoin/bitcoin#20764 cli -netinfo peer connections dashboard updates 🎄 ✨ (jonatack) +- bitcoin/bitcoin#20788 add RAII socket and use it instead of bare SOCKET (vasild) +- bitcoin/bitcoin#20791 remove unused legacyWhitelisted in AcceptConnection() (jonatack) +- bitcoin/bitcoin#20816 Move RecordBytesSent() call out of `cs_vSend` lock (jnewbery) +- bitcoin/bitcoin#20845 Log to net debug in MaybeDiscourageAndDisconnect except for noban and manual peers (MarcoFalke) +- bitcoin/bitcoin#20864 Move SocketSendData lock annotation to header (MarcoFalke) +- bitcoin/bitcoin#20965 net, rpc: return `NET_UNROUTABLE` as `not_publicly_routable`, automate helps (jonatack) +- bitcoin/bitcoin#20966 banman: save the banlist in a JSON format on disk (vasild) +- bitcoin/bitcoin#21015 Make all of `net_processing` (and some of net) use std::chrono types (dhruv) +- bitcoin/bitcoin#21029 bitcoin-cli: Correct docs (no "generatenewaddress" exists) (luke-jr) +- bitcoin/bitcoin#21148 Split orphan handling from `net_processing` into txorphanage (ajtowns) +- bitcoin/bitcoin#21162 Net Processing: Move RelayTransaction() into PeerManager (jnewbery) +- bitcoin/bitcoin#21167 make `CNode::m_inbound_onion` public, initialize explicitly (jonatack) +- bitcoin/bitcoin#21186 net/net processing: Move addr data into `net_processing` (jnewbery) +- bitcoin/bitcoin#21187 Net processing: Only call PushAddress() from `net_processing` (jnewbery) +- bitcoin/bitcoin#21198 Address outstanding review comments from PR20721 (jnewbery) +- bitcoin/bitcoin#21222 log: Clarify log message when file does not exist (MarcoFalke) +- bitcoin/bitcoin#21235 Clarify disconnect log message in ProcessGetBlockData, remove send bool (MarcoFalke) +- bitcoin/bitcoin#21236 Net processing: Extract `addr` send functionality into MaybeSendAddr() (jnewbery) +- bitcoin/bitcoin#21261 update inbound eviction protection for multiple networks, add I2P peers (jonatack) +- bitcoin/bitcoin#21328 net, refactor: pass uint16 CService::port as uint16 (jonatack) +- bitcoin/bitcoin#21387 Refactor sock to add I2P fuzz and unit tests (vasild) +- bitcoin/bitcoin#21395 Net processing: Remove unused CNodeState.address member (jnewbery) +- bitcoin/bitcoin#21407 i2p: limit the size of incoming messages (vasild) +- bitcoin/bitcoin#21506 p2p, refactor: make NetPermissionFlags an enum class (jonatack) +- bitcoin/bitcoin#21509 Don't send FEEFILTER in blocksonly mode (mzumsande) +- bitcoin/bitcoin#21560 Add Tor v3 hardcoded seeds (laanwj) +- bitcoin/bitcoin#21563 Restrict period when `cs_vNodes` mutex is locked (hebasto) +- bitcoin/bitcoin#21564 Avoid calling getnameinfo when formatting IPv4 addresses in CNetAddr::ToStringIP (practicalswift) +- bitcoin/bitcoin#21631 i2p: always check the return value of Sock::Wait() (vasild) +- bitcoin/bitcoin#21644 p2p, bugfix: use NetPermissions::HasFlag() in CConnman::Bind() (jonatack) +- bitcoin/bitcoin#21659 flag relevant Sock methods with [[nodiscard]] (vasild) +- bitcoin/bitcoin#21750 remove unnecessary check of `CNode::cs_vSend` (vasild) +- bitcoin/bitcoin#21756 Avoid calling `getnameinfo` when formatting IPv6 addresses in `CNetAddr::ToStringIP` (practicalswift) +- bitcoin/bitcoin#21775 Limit `m_block_inv_mutex` (MarcoFalke) +- bitcoin/bitcoin#21825 Add I2P hardcoded seeds (jonatack) +- bitcoin/bitcoin#21843 p2p, rpc: enable GetAddr, GetAddresses, and getnodeaddresses by network (jonatack) +- bitcoin/bitcoin#21845 net processing: Don't require locking `cs_main` before calling RelayTransactions() (jnewbery) +- bitcoin/bitcoin#21872 Sanitize message type for logging (laanwj) +- bitcoin/bitcoin#21914 Use stronger AddLocal() for our I2P address (vasild) +- bitcoin/bitcoin#21985 Return IPv6 scope id in `CNetAddr::ToStringIP()` (laanwj) +- bitcoin/bitcoin#21992 Remove -feefilter option (amadeuszpawlik) +- bitcoin/bitcoin#21996 Pass strings to NetPermissions::TryParse functions by const ref (jonatack) +- bitcoin/bitcoin#22013 ignore block-relay-only peers when skipping DNS seed (ajtowns) +- bitcoin/bitcoin#22050 Remove tor v2 support (jonatack) +- bitcoin/bitcoin#22096 AddrFetch - don't disconnect on self-announcements (mzumsande) +- bitcoin/bitcoin#22141 net processing: Remove hash and fValidatedHeaders from QueuedBlock (jnewbery) +- bitcoin/bitcoin#22144 Randomize message processing peer order (sipa) +- bitcoin/bitcoin#22147 Protect last outbound HB compact block peer (sdaftuar) +- bitcoin/bitcoin#22179 Torv2 removal followups (vasild) +- bitcoin/bitcoin#22211 Relay I2P addresses even if not reachable (by us) (vasild) +- bitcoin/bitcoin#22284 Performance improvements to ProtectEvictionCandidatesByRatio() (jonatack) +- bitcoin/bitcoin#22387 Rate limit the processing of rumoured addresses (sipa) +- bitcoin/bitcoin#22455 addrman: detect on-disk corrupted nNew and nTried during unserialization (vasild) + +### Wallet +- bitcoin/bitcoin#15710 Catch `ios_base::failure` specifically (Bushstar) +- bitcoin/bitcoin#16546 External signer support - Wallet Box edition (Sjors) +- bitcoin/bitcoin#17331 Use effective values throughout coin selection (achow101) +- bitcoin/bitcoin#18418 Increase `OUTPUT_GROUP_MAX_ENTRIES` to 100 (fjahr) +- bitcoin/bitcoin#18842 Mark replaced tx to not be in the mempool anymore (MarcoFalke) +- bitcoin/bitcoin#19136 Add `parent_desc` to `getaddressinfo` (achow101) +- bitcoin/bitcoin#19137 wallettool: Add dump and createfromdump commands (achow101) +- bitcoin/bitcoin#19651 `importdescriptor`s update existing (S3RK) +- bitcoin/bitcoin#20040 Refactor OutputGroups to handle fees and spending eligibility on grouping (achow101) +- bitcoin/bitcoin#20202 Make BDB support optional (achow101) +- bitcoin/bitcoin#20226, bitcoin/bitcoin#21277, - bitcoin/bitcoin#21063 Add `listdescriptors` command (S3RK) +- bitcoin/bitcoin#20267 Disable and fix tests for when BDB is not compiled (achow101) +- bitcoin/bitcoin#20275 List all wallets in non-SQLite and non-BDB builds (ryanofsky) +- bitcoin/bitcoin#20365 wallettool: Add parameter to create descriptors wallet (S3RK) +- bitcoin/bitcoin#20403 `upgradewallet` fixes, improvements, test coverage (jonatack) +- bitcoin/bitcoin#20448 `unloadwallet`: Allow specifying `wallet_name` param matching RPC endpoint wallet (luke-jr) +- bitcoin/bitcoin#20536 Error with "Transaction too large" if the funded tx will end up being too large after signing (achow101) +- bitcoin/bitcoin#20687 Add missing check for -descriptors wallet tool option (MarcoFalke) +- bitcoin/bitcoin#20952 Add BerkeleyDB version sanity check at init time (laanwj) +- bitcoin/bitcoin#21127 Load flags before everything else (Sjors) +- bitcoin/bitcoin#21141 Add new format string placeholders for walletnotify (maayank) +- bitcoin/bitcoin#21238 A few descriptor improvements to prepare for Taproot support (sipa) +- bitcoin/bitcoin#21302 `createwallet` examples for descriptor wallets (S3RK) +- bitcoin/bitcoin#21329 descriptor wallet: Cache last hardened xpub and use in normalized descriptors (achow101) +- bitcoin/bitcoin#21365 Basic Taproot signing support for descriptor wallets (sipa) +- bitcoin/bitcoin#21417 Misc external signer improvement and HWI 2 support (Sjors) +- bitcoin/bitcoin#21467 Move external signer out of wallet module (Sjors) +- bitcoin/bitcoin#21572 Fix wrong wallet RPC context set after #21366 (ryanofsky) +- bitcoin/bitcoin#21574 Drop JSONRPCRequest constructors after #21366 (ryanofsky) +- bitcoin/bitcoin#21666 Miscellaneous external signer changes (fanquake) +- bitcoin/bitcoin#21759 Document coin selection code (glozow) +- bitcoin/bitcoin#21786 Ensure sat/vB feerates are in range (mantissa of 3) (jonatack) +- bitcoin/bitcoin#21944 Fix issues when `walletdir` is root directory (prayank23) +- bitcoin/bitcoin#22042 Replace size/weight estimate tuple with struct for named fields (instagibbs) +- bitcoin/bitcoin#22051 Basic Taproot derivation support for descriptors (sipa) +- bitcoin/bitcoin#22154 Add OutputType::BECH32M and related wallet support for fetching bech32m addresses (achow101) +- bitcoin/bitcoin#22156 Allow tr() import only when Taproot is active (achow101) +- bitcoin/bitcoin#22166 Add support for inferring tr() descriptors (sipa) +- bitcoin/bitcoin#22173 Do not load external signers wallets when unsupported (achow101) +- bitcoin/bitcoin#22308 Add missing BlockUntilSyncedToCurrentChain (MarcoFalke) +- bitcoin/bitcoin#22334 Do not spam about non-existent spk managers (S3RK) +- bitcoin/bitcoin#22379 Erase spkmans rather than setting to nullptr (achow101) +- bitcoin/bitcoin#22421 Make IsSegWitOutput return true for taproot outputs (sipa) +- bitcoin/bitcoin#22461 Change ScriptPubKeyMan::Upgrade default to True (achow101) +- bitcoin/bitcoin#22492 Reorder locks in dumpwallet to avoid lock order assertion (achow101) +- bitcoin/bitcoin#22686 Use GetSelectionAmount in ApproximateBestSubset (achow101) + +### RPC and other APIs +- bitcoin/bitcoin#18335, bitcoin/bitcoin#21484 cli: Print useful error if bitcoind rpc work queue exceeded (LarryRuane) +- bitcoin/bitcoin#18466 Fix invalid parameter error codes for `{sign,verify}message` RPCs (theStack) +- bitcoin/bitcoin#18772 Calculate fees in `getblock` using BlockUndo data (robot-visions) +- bitcoin/bitcoin#19033 http: Release work queue after event base finish (promag) +- bitcoin/bitcoin#19055 Add MuHash3072 implementation (fjahr) +- bitcoin/bitcoin#19145 Add `hash_type` MUHASH for gettxoutsetinfo (fjahr) +- bitcoin/bitcoin#19847 Avoid duplicate set lookup in `gettxoutproof` (promag) +- bitcoin/bitcoin#20286 Deprecate `addresses` and `reqSigs` from RPC outputs (mjdietzx) +- bitcoin/bitcoin#20459 Fail to return undocumented return values (MarcoFalke) +- bitcoin/bitcoin#20461 Validate `-rpcauth` arguments (promag) +- bitcoin/bitcoin#20556 Properly document return values (`submitblock`, `gettxout`, `getblocktemplate`, `scantxoutset`) (MarcoFalke) +- bitcoin/bitcoin#20755 Remove deprecated fields from `getpeerinfo` (amitiuttarwar) +- bitcoin/bitcoin#20832 Better error messages for invalid addresses (eilx2) +- bitcoin/bitcoin#20867 Support up to 20 keys for multisig under Segwit context (darosior) +- bitcoin/bitcoin#20877 cli: `-netinfo` user help and argument parsing improvements (jonatack) +- bitcoin/bitcoin#20891 Remove deprecated bumpfee behavior (achow101) +- bitcoin/bitcoin#20916 Return wtxid from `testmempoolaccept` (MarcoFalke) +- bitcoin/bitcoin#20917 Add missing signet mentions in network name lists (theStack) +- bitcoin/bitcoin#20941 Document `RPC_TRANSACTION_ALREADY_IN_CHAIN` exception (jarolrod) +- bitcoin/bitcoin#20944 Return total fee in `getmempoolinfo` (MarcoFalke) +- bitcoin/bitcoin#20964 Add specific error code for "wallet already loaded" (laanwj) +- bitcoin/bitcoin#21053 Document {previous,next}blockhash as optional (theStack) +- bitcoin/bitcoin#21056 Add a `-rpcwaittimeout` parameter to limit time spent waiting (cdecker) +- bitcoin/bitcoin#21192 cli: Treat high detail levels as maximum in `-netinfo` (laanwj) +- bitcoin/bitcoin#21311 Document optional fields for `getchaintxstats` result (theStack) +- bitcoin/bitcoin#21359 `include_unsafe` option for fundrawtransaction (t-bast) +- bitcoin/bitcoin#21426 Remove `scantxoutset` EXPERIMENTAL warning (jonatack) +- bitcoin/bitcoin#21544 Missing doc updates for bumpfee psbt update (MarcoFalke) +- bitcoin/bitcoin#21594 Add `network` field to `getnodeaddresses` (jonatack) +- bitcoin/bitcoin#21595, bitcoin/bitcoin#21753 cli: Create `-addrinfo` (jonatack) +- bitcoin/bitcoin#21602 Add additional ban time fields to `listbanned` (jarolrod) +- bitcoin/bitcoin#21679 Keep default argument value in correct type (promag) +- bitcoin/bitcoin#21718 Improve error message for `getblock` invalid datatype (klementtan) +- bitcoin/bitcoin#21913 RPCHelpMan fixes (kallewoof) +- bitcoin/bitcoin#22021 `bumpfee`/`psbtbumpfee` fixes and updates (jonatack) +- bitcoin/bitcoin#22043 `addpeeraddress` test coverage, code simplify/constness (jonatack) +- bitcoin/bitcoin#22327 cli: Avoid truncating `-rpcwaittimeout` (MarcoFalke) + +### GUI +- bitcoin/bitcoin#18948 Call setParent() in the parent's context (hebasto) +- bitcoin/bitcoin#20482 Add depends qt fix for ARM macs (jonasschnelli) +- bitcoin/bitcoin#21836 scripted-diff: Replace three dots with ellipsis in the ui strings (hebasto) +- bitcoin/bitcoin#21935 Enable external signer support for GUI builds (Sjors) +- bitcoin/bitcoin#22133 Make QWindowsVistaStylePlugin available again (regression) (hebasto) +- bitcoin-core/gui#4 UI external signer support (e.g. hardware wallet) (Sjors) +- bitcoin-core/gui#13 Hide peer detail view if multiple are selected (promag) +- bitcoin-core/gui#18 Add peertablesortproxy module (hebasto) +- bitcoin-core/gui#21 Improve pruning tooltip (fluffypony, BitcoinErrorLog) +- bitcoin-core/gui#72 Log static plugins meta data and used style (hebasto) +- bitcoin-core/gui#79 Embed monospaced font (hebasto) +- bitcoin-core/gui#85 Remove unused "What's This" button in dialogs on Windows OS (hebasto) +- bitcoin-core/gui#115 Replace "Hide tray icon" option with positive "Show tray icon" one (hebasto) +- bitcoin-core/gui#118 Remove BDB version from the Information tab (hebasto) +- bitcoin-core/gui#121 Early subscribe core signals in transaction table model (promag) +- bitcoin-core/gui#123 Do not accept command while executing another one (hebasto) +- bitcoin-core/gui#125 Enable changing the autoprune block space size in intro dialog (luke-jr) +- bitcoin-core/gui#138 Unlock encrypted wallet "OK" button bugfix (mjdietzx) +- bitcoin-core/gui#139 doc: Improve gui/src/qt README.md (jarolrod) +- bitcoin-core/gui#154 Support macOS Dark mode (goums, Uplab) +- bitcoin-core/gui#162 Add network to peers window and peer details (jonatack) +- bitcoin-core/gui#163, bitcoin-core/gui#180 Peer details: replace Direction with Connection Type (jonatack) +- bitcoin-core/gui#164 Handle peer addition/removal in a right way (hebasto) +- bitcoin-core/gui#165 Save QSplitter state in QSettings (hebasto) +- bitcoin-core/gui#173 Follow Qt docs when implementing rowCount and columnCount (hebasto) +- bitcoin-core/gui#179 Add Type column to peers window, update peer details name/tooltip (jonatack) +- bitcoin-core/gui#186 Add information to "Confirm fee bump" window (prayank23) +- bitcoin-core/gui#189 Drop workaround for QTBUG-42503 which was fixed in Qt 5.5.0 (prusnak) +- bitcoin-core/gui#194 Save/restore RPCConsole geometry only for window (hebasto) +- bitcoin-core/gui#202 Fix right panel toggle in peers tab (RandyMcMillan) +- bitcoin-core/gui#203 Display plain "Inbound" in peer details (jonatack) +- bitcoin-core/gui#204 Drop buggy TableViewLastColumnResizingFixer class (hebasto) +- bitcoin-core/gui#205, bitcoin-core/gui#229 Save/restore TransactionView and recentRequestsView tables column sizes (hebasto) +- bitcoin-core/gui#206 Display fRelayTxes and `bip152_highbandwidth_{to, from}` in peer details (jonatack) +- bitcoin-core/gui#213 Add Copy Address Action to Payment Requests (jarolrod) +- bitcoin-core/gui#214 Disable requests context menu actions when appropriate (jarolrod) +- bitcoin-core/gui#217 Make warning label look clickable (jarolrod) +- bitcoin-core/gui#219 Prevent the main window popup menu (hebasto) +- bitcoin-core/gui#220 Do not translate file extensions (hebasto) +- bitcoin-core/gui#221 RPCConsole translatable string fixes and improvements (jonatack) +- bitcoin-core/gui#226 Add "Last Block" and "Last Tx" rows to peer details area (jonatack) +- bitcoin-core/gui#233 qt test: Don't bind to regtest port (achow101) +- bitcoin-core/gui#243 Fix issue when disabling the auto-enabled blank wallet checkbox (jarolrod) +- bitcoin-core/gui#246 Revert "qt: Use "fusion" style on macOS Big Sur with old Qt" (hebasto) +- bitcoin-core/gui#248 For values of "Bytes transferred" and "Bytes/s" with 1000-based prefix names use 1000-based divisor instead of 1024-based (wodry) +- bitcoin-core/gui#251 Improve URI/file handling message (hebasto) +- bitcoin-core/gui#256 Save/restore column sizes of the tables in the Peers tab (hebasto) +- bitcoin-core/gui#260 Handle exceptions isntead of crash (hebasto) +- bitcoin-core/gui#263 Revamp context menus (hebasto) +- bitcoin-core/gui#271 Don't clear console prompt when font resizing (jarolrod) +- bitcoin-core/gui#275 Support runtime appearance adjustment on macOS (hebasto) +- bitcoin-core/gui#276 Elide long strings in their middle in the Peers tab (hebasto) +- bitcoin-core/gui#281 Set shortcuts for console's resize buttons (jarolrod) +- bitcoin-core/gui#293 Enable wordWrap for Services (RandyMcMillan) +- bitcoin-core/gui#296 Do not use QObject::tr plural syntax for numbers with a unit symbol (hebasto) +- bitcoin-core/gui#297 Avoid unnecessary translations (hebasto) +- bitcoin-core/gui#298 Peertableview alternating row colors (RandyMcMillan) +- bitcoin-core/gui#300 Remove progress bar on modal overlay (brunoerg) +- bitcoin-core/gui#309 Add access to the Peers tab from the network icon (hebasto) +- bitcoin-core/gui#311 Peers Window rename 'Peer id' to 'Peer' (jarolrod) +- bitcoin-core/gui#313 Optimize string concatenation by default (hebasto) +- bitcoin-core/gui#325 Align numbers in the "Peer Id" column to the right (hebasto) +- bitcoin-core/gui#329 Make console buttons look clickable (jarolrod) +- bitcoin-core/gui#330 Allow prompt icon to be colorized (jarolrod) +- bitcoin-core/gui#331 Make RPC console welcome message translation-friendly (hebasto) +- bitcoin-core/gui#332 Replace disambiguation strings with translator comments (hebasto) +- bitcoin-core/gui#335 test: Use QSignalSpy instead of QEventLoop (jarolrod) +- bitcoin-core/gui#343 Improve the GUI responsiveness when progress dialogs are used (hebasto) +- bitcoin-core/gui#361 Fix GUI segfault caused by bitcoin/bitcoin#22216 (ryanofsky) +- bitcoin-core/gui#362 Add keyboard shortcuts to context menus (luke-jr) +- bitcoin-core/gui#366 Dark Mode fixes/portability (luke-jr) +- bitcoin-core/gui#375 Emit dataChanged signal to dynamically re-sort Peers table (hebasto) +- bitcoin-core/gui#393 Fix regression in "Encrypt Wallet" menu item (hebasto) +- bitcoin-core/gui#396 Ensure external signer option remains disabled without signers (achow101) +- bitcoin-core/gui#406 Handle new added plurals in `bitcoin_en.ts` (hebasto) + +### Build system +- bitcoin/bitcoin#17227 Add Android packaging support (icota) +- bitcoin/bitcoin#17920 guix: Build support for macOS (dongcarl) +- bitcoin/bitcoin#18298 Fix Qt processing of configure script for depends with DEBUG=1 (hebasto) +- bitcoin/bitcoin#19160 multiprocess: Add basic spawn and IPC support (ryanofsky) +- bitcoin/bitcoin#19504 Bump minimum python version to 3.6 (ajtowns) +- bitcoin/bitcoin#19522 fix building libconsensus with reduced exports for Darwin targets (fanquake) +- bitcoin/bitcoin#19683 Pin clang search paths for darwin host (dongcarl) +- bitcoin/bitcoin#19764 Split boost into build/host packages + bump + cleanup (dongcarl) +- bitcoin/bitcoin#19817 libtapi 1100.0.11 (fanquake) +- bitcoin/bitcoin#19846 enable unused member function diagnostic (Zero-1729) +- bitcoin/bitcoin#19867 Document and cleanup Qt hacks (fanquake) +- bitcoin/bitcoin#20046 Set `CMAKE_INSTALL_RPATH` for native packages (ryanofsky) +- bitcoin/bitcoin#20223 Drop the leading 0 from the version number (achow101) +- bitcoin/bitcoin#20333 Remove `native_biplist` dependency (fanquake) +- bitcoin/bitcoin#20353 configure: Support -fdebug-prefix-map and -fmacro-prefix-map (ajtowns) +- bitcoin/bitcoin#20359 Various config.site.in improvements and linting (dongcarl) +- bitcoin/bitcoin#20413 Require C++17 compiler (MarcoFalke) +- bitcoin/bitcoin#20419 Set minimum supported macOS to 10.14 (fanquake) +- bitcoin/bitcoin#20421 miniupnpc 2.2.2 (fanquake) +- bitcoin/bitcoin#20422 Mac deployment unification (fanquake) +- bitcoin/bitcoin#20424 Update univalue subtree (MarcoFalke) +- bitcoin/bitcoin#20449 Fix Windows installer build (achow101) +- bitcoin/bitcoin#20468 Warn when generating man pages for binaries built from a dirty branch (tylerchambers) +- bitcoin/bitcoin#20469 Avoid secp256k1.h include from system (dergoegge) +- bitcoin/bitcoin#20470 Replace genisoimage with xorriso (dongcarl) +- bitcoin/bitcoin#20471 Use C++17 in depends (fanquake) +- bitcoin/bitcoin#20496 Drop unneeded macOS framework dependencies (hebasto) +- bitcoin/bitcoin#20520 Do not force Precompiled Headers (PCH) for building Qt on Linux (hebasto) +- bitcoin/bitcoin#20549 Support make src/bitcoin-node and src/bitcoin-gui (promag) +- bitcoin/bitcoin#20565 Ensure PIC build for bdb on Android (BlockMechanic) +- bitcoin/bitcoin#20594 Fix getauxval calls in randomenv.cpp (jonasschnelli) +- bitcoin/bitcoin#20603 Update crc32c subtree (MarcoFalke) +- bitcoin/bitcoin#20609 configure: output notice that test binary is disabled by fuzzing (apoelstra) +- bitcoin/bitcoin#20619 guix: Quality of life improvements (dongcarl) +- bitcoin/bitcoin#20629 Improve id string robustness (dongcarl) +- bitcoin/bitcoin#20641 Use Qt top-level build facilities (hebasto) +- bitcoin/bitcoin#20650 Drop workaround for a fixed bug in Qt build system (hebasto) +- bitcoin/bitcoin#20673 Use more legible qmake commands in qt package (hebasto) +- bitcoin/bitcoin#20684 Define .INTERMEDIATE target once only (hebasto) +- bitcoin/bitcoin#20720 more robustly check for fcf-protection support (fanquake) +- bitcoin/bitcoin#20734 Make platform-specific targets available for proper platform builds only (hebasto) +- bitcoin/bitcoin#20936 build fuzz tests by default (danben) +- bitcoin/bitcoin#20937 guix: Make nsis reproducible by respecting SOURCE-DATE-EPOCH (dongcarl) +- bitcoin/bitcoin#20938 fix linking against -latomic when building for riscv (fanquake) +- bitcoin/bitcoin#20939 fix `RELOC_SECTION` security check for bitcoin-util (fanquake) +- bitcoin/bitcoin#20963 gitian-linux: Build binaries for 64-bit POWER (continued) (laanwj) +- bitcoin/bitcoin#21036 gitian: Bump descriptors to focal for 22.0 (fanquake) +- bitcoin/bitcoin#21045 Adds switch to enable/disable randomized base address in MSVC builds (EthanHeilman) +- bitcoin/bitcoin#21065 make macOS HOST in download-osx generic (fanquake) +- bitcoin/bitcoin#21078 guix: only download sources for hosts being built (fanquake) +- bitcoin/bitcoin#21116 Disable --disable-fuzz-binary for gitian/guix builds (hebasto) +- bitcoin/bitcoin#21182 remove mostly pointless `BOOST_PROCESS` macro (fanquake) +- bitcoin/bitcoin#21205 actually fail when Boost is missing (fanquake) +- bitcoin/bitcoin#21209 use newer source for libnatpmp (fanquake) +- bitcoin/bitcoin#21226 Fix fuzz binary compilation under windows (danben) +- bitcoin/bitcoin#21231 Add /opt/homebrew to path to look for boost libraries (fyquah) +- bitcoin/bitcoin#21239 guix: Add codesignature attachment support for osx+win (dongcarl) +- bitcoin/bitcoin#21250 Make `HAVE_O_CLOEXEC` available outside LevelDB (bugfix) (theStack) +- bitcoin/bitcoin#21272 guix: Passthrough `SDK_PATH` into container (dongcarl) +- bitcoin/bitcoin#21274 assumptions: Assume C++17 (fanquake) +- bitcoin/bitcoin#21286 Bump minimum Qt version to 5.9.5 (hebasto) +- bitcoin/bitcoin#21298 guix: Bump time-machine, glibc, and linux-headers (dongcarl) +- bitcoin/bitcoin#21304 guix: Add guix-clean script + establish gc-root for container profiles (dongcarl) +- bitcoin/bitcoin#21320 fix libnatpmp macos cross compile (fanquake) +- bitcoin/bitcoin#21321 guix: Add curl to required tool list (hebasto) +- bitcoin/bitcoin#21333 set Unicode true for NSIS installer (fanquake) +- bitcoin/bitcoin#21339 Make `AM_CONDITIONAL([ENABLE_EXTERNAL_SIGNER])` unconditional (hebasto) +- bitcoin/bitcoin#21349 Fix fuzz-cuckoocache cross-compiling with DEBUG=1 (hebasto) +- bitcoin/bitcoin#21354 build, doc: Drop no longer required packages from macOS cross-compiling dependencies (hebasto) +- bitcoin/bitcoin#21363 build, qt: Improve Qt static plugins/libs check code (hebasto) +- bitcoin/bitcoin#21375 guix: Misc feedback-based fixes + hier restructuring (dongcarl) +- bitcoin/bitcoin#21376 Qt 5.12.10 (fanquake) +- bitcoin/bitcoin#21382 Clean remnants of QTBUG-34748 fix (hebasto) +- bitcoin/bitcoin#21400 Fix regression introduced in #21363 (hebasto) +- bitcoin/bitcoin#21403 set --build when configuring packages in depends (fanquake) +- bitcoin/bitcoin#21421 don't try and use -fstack-clash-protection on Windows (fanquake) +- bitcoin/bitcoin#21423 Cleanups and follow ups after bumping Qt to 5.12.10 (hebasto) +- bitcoin/bitcoin#21427 Fix `id_string` invocations (dongcarl) +- bitcoin/bitcoin#21430 Add -Werror=implicit-fallthrough compile flag (hebasto) +- bitcoin/bitcoin#21457 Split libtapi and clang out of `native_cctools` (fanquake) +- bitcoin/bitcoin#21462 guix: Add guix-{attest,verify} scripts (dongcarl) +- bitcoin/bitcoin#21495 build, qt: Fix static builds on macOS Big Sur (hebasto) +- bitcoin/bitcoin#21497 Do not opt-in unused CoreWLAN stuff in depends for macOS (hebasto) +- bitcoin/bitcoin#21543 Enable safe warnings for msvc builds (hebasto) +- bitcoin/bitcoin#21565 Make `bitcoin_qt.m4` more generic (fanquake) +- bitcoin/bitcoin#21610 remove -Wdeprecated-register from NOWARN flags (fanquake) +- bitcoin/bitcoin#21613 enable -Wdocumentation (fanquake) +- bitcoin/bitcoin#21629 Fix configuring when building depends with `NO_BDB=1` (fanquake) +- bitcoin/bitcoin#21654 build, qt: Make Qt rcc output always deterministic (hebasto) +- bitcoin/bitcoin#21655 build, qt: No longer need to set `QT_RCC_TEST=1` for determinism (hebasto) +- bitcoin/bitcoin#21658 fix make deploy for arm64-darwin (sgulls) +- bitcoin/bitcoin#21694 Use XLIFF file to provide more context to Transifex translators (hebasto) +- bitcoin/bitcoin#21708, bitcoin/bitcoin#21593 Drop pointless sed commands (hebasto) +- bitcoin/bitcoin#21731 Update msvc build to use Qt5.12.10 binaries (sipsorcery) +- bitcoin/bitcoin#21733 Re-add command to install vcpkg (dplusplus1024) +- bitcoin/bitcoin#21793 Use `-isysroot` over `--sysroot` on macOS (fanquake) +- bitcoin/bitcoin#21869 Add missing `-D_LIBCPP_DEBUG=1` to debug flags (MarcoFalke) +- bitcoin/bitcoin#21889 macho: check for control flow instrumentation (fanquake) +- bitcoin/bitcoin#21920 Improve macro for testing -latomic requirement (MarcoFalke) +- bitcoin/bitcoin#21991 libevent 2.1.12-stable (fanquake) +- bitcoin/bitcoin#22054 Bump Qt version to 5.12.11 (hebasto) +- bitcoin/bitcoin#22063 Use Qt archive of the same version as the compiled binaries (hebasto) +- bitcoin/bitcoin#22070 Don't use cf-protection when targeting arm-apple-darwin (fanquake) +- bitcoin/bitcoin#22071 Latest config.guess and config.sub (fanquake) +- bitcoin/bitcoin#22075 guix: Misc leftover usability improvements (dongcarl) +- bitcoin/bitcoin#22123 Fix qt.mk for mac arm64 (promag) +- bitcoin/bitcoin#22174 build, qt: Fix libraries linking order for Linux hosts (hebasto) +- bitcoin/bitcoin#22182 guix: Overhaul how guix-{attest,verify} works and hierarchy (dongcarl) +- bitcoin/bitcoin#22186 build, qt: Fix compiling qt package in depends with GCC 11 (hebasto) +- bitcoin/bitcoin#22199 macdeploy: minor fixups and simplifications (fanquake) +- bitcoin/bitcoin#22230 Fix MSVC linker /SubSystem option for bitcoin-qt.exe (hebasto) +- bitcoin/bitcoin#22234 Mark print-% target as phony (dgoncharov) +- bitcoin/bitcoin#22238 improve detection of eBPF support (fanquake) +- bitcoin/bitcoin#22258 Disable deprecated-copy warning only when external warnings are enabled (MarcoFalke) +- bitcoin/bitcoin#22320 set minimum required Boost to 1.64.0 (fanquake) +- bitcoin/bitcoin#22348 Fix cross build for Windows with Boost Process (hebasto) +- bitcoin/bitcoin#22365 guix: Avoid relying on newer symbols by rebasing our cross toolchains on older glibcs (dongcarl) +- bitcoin/bitcoin#22381 guix: Test security-check sanity before performing them (with macOS) (fanquake) +- bitcoin/bitcoin#22405 Remove --enable-glibc-back-compat from Guix build (fanquake) +- bitcoin/bitcoin#22406 Remove --enable-determinism configure option (fanquake) +- bitcoin/bitcoin#22410 Avoid GCC 7.1 ABI change warning in guix build (sipa) +- bitcoin/bitcoin#22436 use aarch64 Clang if cross-compiling for darwin on aarch64 (fanquake) +- bitcoin/bitcoin#22465 guix: Pin kernel-header version, time-machine to upstream 1.3.0 commit (dongcarl) +- bitcoin/bitcoin#22511 guix: Silence `getent(1)` invocation, doc fixups (dongcarl) +- bitcoin/bitcoin#22531 guix: Fixes to guix-{attest,verify} (achow101) +- bitcoin/bitcoin#22642 release: Release with separate sha256sums and sig files (dongcarl) +- bitcoin/bitcoin#22685 clientversion: No suffix `#if CLIENT_VERSION_IS_RELEASE` (dongcarl) +- bitcoin/bitcoin#22713 Fix build with Boost 1.77.0 (sizeofvoid) + +### Tests and QA +- bitcoin/bitcoin#14604 Add test and refactor `feature_block.py` (sanket1729) +- bitcoin/bitcoin#17556 Change `feature_config_args.py` not to rely on strange regtest=0 behavior (ryanofsky) +- bitcoin/bitcoin#18795 wallet issue with orphaned rewards (domob1812) +- bitcoin/bitcoin#18847 compressor: Use a prevector in CompressScript serialization (jb55) +- bitcoin/bitcoin#19259 fuzz: Add fuzzing harness for LoadMempool(…) and DumpMempool(…) (practicalswift) +- bitcoin/bitcoin#19315 Allow outbound & block-relay-only connections in functional tests. (amitiuttarwar) +- bitcoin/bitcoin#19698 Apply strict verification flags for transaction tests and assert backwards compatibility (glozow) +- bitcoin/bitcoin#19801 Check for all possible `OP_CLTV` fail reasons in `feature_cltv.py` (BIP 65) (theStack) +- bitcoin/bitcoin#19893 Remove or explain syncwithvalidationinterfacequeue (MarcoFalke) +- bitcoin/bitcoin#19972 fuzz: Add fuzzing harness for node eviction logic (practicalswift) +- bitcoin/bitcoin#19982 Fix inconsistent lock order in `wallet_tests/CreateWallet` (hebasto) +- bitcoin/bitcoin#20000 Fix creation of "std::string"s with \0s (vasild) +- bitcoin/bitcoin#20047 Use `wait_for_{block,header}` helpers in `p2p_fingerprint.py` (theStack) +- bitcoin/bitcoin#20171 Add functional test `test_txid_inv_delay` (ariard) +- bitcoin/bitcoin#20189 Switch to BIP341's suggested scheme for outputs without script (sipa) +- bitcoin/bitcoin#20248 Fix length of R check in `key_signature_tests` (dgpv) +- bitcoin/bitcoin#20276, bitcoin/bitcoin#20385, bitcoin/bitcoin#20688, bitcoin/bitcoin#20692 Run various mempool tests even with wallet disabled (mjdietzx) +- bitcoin/bitcoin#20323 Create or use existing properly initialized NodeContexts (dongcarl) +- bitcoin/bitcoin#20354 Add `feature_taproot.py --previous_release` (MarcoFalke) +- bitcoin/bitcoin#20370 fuzz: Version handshake (MarcoFalke) +- bitcoin/bitcoin#20377 fuzz: Fill various small fuzzing gaps (practicalswift) +- bitcoin/bitcoin#20425 fuzz: Make CAddrMan fuzzing harness deterministic (practicalswift) +- bitcoin/bitcoin#20430 Sanitizers: Add suppression for unsigned-integer-overflow in libstdc++ (jonasschnelli) +- bitcoin/bitcoin#20437 fuzz: Avoid time-based "non-determinism" in fuzzing harnesses by using mocked GetTime() (practicalswift) +- bitcoin/bitcoin#20458 Add `is_bdb_compiled` helper (Sjors) +- bitcoin/bitcoin#20466 Fix intermittent `p2p_fingerprint` issue (MarcoFalke) +- bitcoin/bitcoin#20472 Add testing of ParseInt/ParseUInt edge cases with leading +/-/0:s (practicalswift) +- bitcoin/bitcoin#20507 sync: print proper lock order location when double lock is detected (vasild) +- bitcoin/bitcoin#20522 Fix sync issue in `disconnect_p2ps` (amitiuttarwar) +- bitcoin/bitcoin#20524 Move `MIN_VERSION_SUPPORTED` to p2p.py (jnewbery) +- bitcoin/bitcoin#20540 Fix `wallet_multiwallet` issue on windows (MarcoFalke) +- bitcoin/bitcoin#20560 fuzz: Link all targets once (MarcoFalke) +- bitcoin/bitcoin#20567 Add option to git-subtree-check to do full check, add help (laanwj) +- bitcoin/bitcoin#20569 Fix intermittent `wallet_multiwallet` issue with `got_loading_error` (MarcoFalke) +- bitcoin/bitcoin#20613 Use Popen.wait instead of RPC in `assert_start_raises_init_error` (MarcoFalke) +- bitcoin/bitcoin#20663 fuzz: Hide `script_assets_test_minimizer` (MarcoFalke) +- bitcoin/bitcoin#20674 fuzz: Call SendMessages after ProcessMessage to increase coverage (MarcoFalke) +- bitcoin/bitcoin#20683 Fix restart node race (MarcoFalke) +- bitcoin/bitcoin#20686 fuzz: replace CNode code with fuzz/util.h::ConsumeNode() (jonatack) +- bitcoin/bitcoin#20733 Inline non-member functions with body in fuzzing headers (pstratem) +- bitcoin/bitcoin#20737 Add missing assignment in `mempool_resurrect.py` (MarcoFalke) +- bitcoin/bitcoin#20745 Correct `epoll_ctl` data race suppression (hebasto) +- bitcoin/bitcoin#20748 Add race:SendZmqMessage tsan suppression (MarcoFalke) +- bitcoin/bitcoin#20760 Set correct nValue for multi-op-return policy check (MarcoFalke) +- bitcoin/bitcoin#20761 fuzz: Check that `NULL_DATA` is unspendable (MarcoFalke) +- bitcoin/bitcoin#20765 fuzz: Check that certain script TxoutType are nonstandard (mjdietzx) +- bitcoin/bitcoin#20772 fuzz: Bolster ExtractDestination(s) checks (mjdietzx) +- bitcoin/bitcoin#20789 fuzz: Rework strong and weak net enum fuzzing (MarcoFalke) +- bitcoin/bitcoin#20828 fuzz: Introduce CallOneOf helper to replace switch-case (MarcoFalke) +- bitcoin/bitcoin#20839 fuzz: Avoid extraneous copy of input data, using Span<> (MarcoFalke) +- bitcoin/bitcoin#20844 Add sanitizer suppressions for AMD EPYC CPUs (MarcoFalke) +- bitcoin/bitcoin#20857 Update documentation in `feature_csv_activation.py` (PiRK) +- bitcoin/bitcoin#20876 Replace getmempoolentry with testmempoolaccept in MiniWallet (MarcoFalke) +- bitcoin/bitcoin#20881 fuzz: net permission flags in net processing (MarcoFalke) +- bitcoin/bitcoin#20882 fuzz: Add missing muhash registration (MarcoFalke) +- bitcoin/bitcoin#20908 fuzz: Use mocktime in `process_message*` fuzz targets (MarcoFalke) +- bitcoin/bitcoin#20915 fuzz: Fail if message type is not fuzzed (MarcoFalke) +- bitcoin/bitcoin#20946 fuzz: Consolidate fuzzing TestingSetup initialization (dongcarl) +- bitcoin/bitcoin#20954 Declare `nodes` type `in test_framework.py` (kiminuo) +- bitcoin/bitcoin#20955 Fix `get_previous_releases.py` for aarch64 (MarcoFalke) +- bitcoin/bitcoin#20969 check that getblockfilter RPC fails without block filter index (theStack) +- bitcoin/bitcoin#20971 Work around libFuzzer deadlock (MarcoFalke) +- bitcoin/bitcoin#20993 Store subversion (user agent) as string in `msg_version` (theStack) +- bitcoin/bitcoin#20995 fuzz: Avoid initializing version to less than `MIN_PEER_PROTO_VERSION` (MarcoFalke) +- bitcoin/bitcoin#20998 Fix BlockToJsonVerbose benchmark (martinus) +- bitcoin/bitcoin#21003 Move MakeNoLogFileContext to `libtest_util`, and use it in bench (MarcoFalke) +- bitcoin/bitcoin#21008 Fix zmq test flakiness, improve speed (theStack) +- bitcoin/bitcoin#21023 fuzz: Disable shuffle when merge=1 (MarcoFalke) +- bitcoin/bitcoin#21037 fuzz: Avoid designated initialization (C++20) in fuzz tests (practicalswift) +- bitcoin/bitcoin#21042 doc, test: Improve `setup_clean_chain` documentation (fjahr) +- bitcoin/bitcoin#21080 fuzz: Configure check for main function (take 2) (MarcoFalke) +- bitcoin/bitcoin#21084 Fix timeout decrease in `feature_assumevalid` (brunoerg) +- bitcoin/bitcoin#21096 Re-add dead code detection (flack) +- bitcoin/bitcoin#21100 Remove unused function `xor_bytes` (theStack) +- bitcoin/bitcoin#21115 Fix Windows cross build (hebasto) +- bitcoin/bitcoin#21117 Remove `assert_blockchain_height` (MarcoFalke) +- bitcoin/bitcoin#21121 Small unit test improvements, including helper to make mempool transaction (amitiuttarwar) +- bitcoin/bitcoin#21124 Remove unnecessary assignment in bdb (brunoerg) +- bitcoin/bitcoin#21125 Change `BOOST_CHECK` to `BOOST_CHECK_EQUAL` for paths (kiminuo) +- bitcoin/bitcoin#21142, bitcoin/bitcoin#21512 fuzz: Add `tx_pool` fuzz target (MarcoFalke) +- bitcoin/bitcoin#21165 Use mocktime in `test_seed_peers` (dhruv) +- bitcoin/bitcoin#21169 fuzz: Add RPC interface fuzzing. Increase fuzzing coverage from 65% to 70% (practicalswift) +- bitcoin/bitcoin#21170 bench: Add benchmark to write json into a string (martinus) +- bitcoin/bitcoin#21178 Run `mempool_reorg.py` even with wallet disabled (DariusParvin) +- bitcoin/bitcoin#21185 fuzz: Remove expensive and redundant muhash from crypto fuzz target (MarcoFalke) +- bitcoin/bitcoin#21200 Speed up `rpc_blockchain.py` by removing miniwallet.generate() (MarcoFalke) +- bitcoin/bitcoin#21211 Move `P2WSH_OP_TRUE` to shared test library (MarcoFalke) +- bitcoin/bitcoin#21228 Avoid comparision of integers with different signs (jonasschnelli) +- bitcoin/bitcoin#21230 Fix `NODE_NETWORK_LIMITED_MIN_BLOCKS` disconnection (MarcoFalke) +- bitcoin/bitcoin#21252 Add missing wait for sync to `feature_blockfilterindex_prune` (MarcoFalke) +- bitcoin/bitcoin#21254 Avoid connecting to real network when running tests (MarcoFalke) +- bitcoin/bitcoin#21264 fuzz: Two scripted diff renames (MarcoFalke) +- bitcoin/bitcoin#21280 Bug fix in `transaction_tests` (glozow) +- bitcoin/bitcoin#21293 Replace accidentally placed bit-OR with logical-OR (hebasto) +- bitcoin/bitcoin#21297 `feature_blockfilterindex_prune.py` improvements (jonatack) +- bitcoin/bitcoin#21310 zmq test: fix sync-up by matching notification to generated block (theStack) +- bitcoin/bitcoin#21334 Additional BIP9 tests (Sjors) +- bitcoin/bitcoin#21338 Add functional test for anchors.dat (brunoerg) +- bitcoin/bitcoin#21345 Bring `p2p_leak.py` up to date (mzumsande) +- bitcoin/bitcoin#21357 Unconditionally check for fRelay field in test framework (jarolrod) +- bitcoin/bitcoin#21358 fuzz: Add missing include (`test/util/setup_common.h`) (MarcoFalke) +- bitcoin/bitcoin#21371 fuzz: fix gcc Woverloaded-virtual build warnings (jonatack) +- bitcoin/bitcoin#21373 Generate fewer blocks in `feature_nulldummy` to fix timeouts, speed up (jonatack) +- bitcoin/bitcoin#21390 Test improvements for UTXO set hash tests (fjahr) +- bitcoin/bitcoin#21410 increase `rpc_timeout` for fundrawtx `test_transaction_too_large` (jonatack) +- bitcoin/bitcoin#21411 add logging, reduce blocks, move `sync_all` in `wallet_` groups (jonatack) +- bitcoin/bitcoin#21438 Add ParseUInt8() test coverage (jonatack) +- bitcoin/bitcoin#21443 fuzz: Implement `fuzzed_dns_lookup_function` as a lambda (practicalswift) +- bitcoin/bitcoin#21445 cirrus: Use SSD cluster for speedup (MarcoFalke) +- bitcoin/bitcoin#21477 Add test for CNetAddr::ToString IPv6 address formatting (RFC 5952) (practicalswift) +- bitcoin/bitcoin#21487 fuzz: Use ConsumeWeakEnum in addrman for service flags (MarcoFalke) +- bitcoin/bitcoin#21488 Add ParseUInt16() unit test and fuzz coverage (jonatack) +- bitcoin/bitcoin#21491 test: remove duplicate assertions in util_tests (jonatack) +- bitcoin/bitcoin#21522 fuzz: Use PickValue where possible (MarcoFalke) +- bitcoin/bitcoin#21531 remove qt byteswap compattests (fanquake) +- bitcoin/bitcoin#21557 small cleanup in RPCNestedTests tests (fanquake) +- bitcoin/bitcoin#21586 Add missing suppression for signed-integer-overflow:txmempool.cpp (MarcoFalke) +- bitcoin/bitcoin#21592 Remove option to make TestChain100Setup non-deterministic (MarcoFalke) +- bitcoin/bitcoin#21597 Document `race:validation_chainstatemanager_tests` suppression (MarcoFalke) +- bitcoin/bitcoin#21599 Replace file level integer overflow suppression with function level suppression (practicalswift) +- bitcoin/bitcoin#21604 Document why no symbol names can be used for suppressions (MarcoFalke) +- bitcoin/bitcoin#21606 fuzz: Extend psbt fuzz target a bit (MarcoFalke) +- bitcoin/bitcoin#21617 fuzz: Fix uninitialized read in i2p test (MarcoFalke) +- bitcoin/bitcoin#21630 fuzz: split FuzzedSock interface and implementation (vasild) +- bitcoin/bitcoin#21634 Skip SQLite fsyncs while testing (achow101) +- bitcoin/bitcoin#21669 Remove spurious double lock tsan suppressions by bumping to clang-12 (MarcoFalke) +- bitcoin/bitcoin#21676 Use mocktime to avoid intermittent failure in `rpc_tests` (MarcoFalke) +- bitcoin/bitcoin#21677 fuzz: Avoid use of low file descriptor ids (which may be in use) in FuzzedSock (practicalswift) +- bitcoin/bitcoin#21678 Fix TestPotentialDeadLockDetected suppression (hebasto) +- bitcoin/bitcoin#21689 Remove intermittently failing and not very meaningful `BOOST_CHECK` in `cnetaddr_basic` (practicalswift) +- bitcoin/bitcoin#21691 Check that no versionbits are re-used (MarcoFalke) +- bitcoin/bitcoin#21707 Extend functional tests for addr relay (mzumsande) +- bitcoin/bitcoin#21712 Test default `include_mempool` value of gettxout (promag) +- bitcoin/bitcoin#21738 Use clang-12 for ASAN, Add missing suppression (MarcoFalke) +- bitcoin/bitcoin#21740 add new python linter to check file names and permissions (windsok) +- bitcoin/bitcoin#21749 Bump shellcheck version (hebasto) +- bitcoin/bitcoin#21754 Run `feature_cltv` with MiniWallet (MarcoFalke) +- bitcoin/bitcoin#21762 Speed up `mempool_spend_coinbase.py` (MarcoFalke) +- bitcoin/bitcoin#21773 fuzz: Ensure prevout is consensus-valid (MarcoFalke) +- bitcoin/bitcoin#21777 Fix `feature_notifications.py` intermittent issue (MarcoFalke) +- bitcoin/bitcoin#21785 Fix intermittent issue in `p2p_addr_relay.py` (MarcoFalke) +- bitcoin/bitcoin#21787 Fix off-by-ones in `rpc_fundrawtransaction` assertions (jonatack) +- bitcoin/bitcoin#21792 Fix intermittent issue in `p2p_segwit.py` (MarcoFalke) +- bitcoin/bitcoin#21795 fuzz: Terminate immediately if a fuzzing harness tries to perform a DNS lookup (belt and suspenders) (practicalswift) +- bitcoin/bitcoin#21798 fuzz: Create a block template in `tx_pool` targets (MarcoFalke) +- bitcoin/bitcoin#21804 Speed up `p2p_segwit.py` (jnewbery) +- bitcoin/bitcoin#21810 fuzz: Various RPC fuzzer follow-ups (practicalswift) +- bitcoin/bitcoin#21814 Fix `feature_config_args.py` intermittent issue (MarcoFalke) +- bitcoin/bitcoin#21821 Add missing test for empty P2WSH redeem (MarcoFalke) +- bitcoin/bitcoin#21822 Resolve bug in `interface_bitcoin_cli.py` (klementtan) +- bitcoin/bitcoin#21846 fuzz: Add `-fsanitize=integer` suppression needed for RPC fuzzer (`generateblock`) (practicalswift) +- bitcoin/bitcoin#21849 fuzz: Limit toxic test globals to their respective scope (MarcoFalke) +- bitcoin/bitcoin#21867 use MiniWallet for `p2p_blocksonly.py` (theStack) +- bitcoin/bitcoin#21873 minor fixes & improvements for files linter test (windsok) +- bitcoin/bitcoin#21874 fuzz: Add `WRITE_ALL_FUZZ_TARGETS_AND_ABORT` (MarcoFalke) +- bitcoin/bitcoin#21884 fuzz: Remove unused --enable-danger-fuzz-link-all option (MarcoFalke) +- bitcoin/bitcoin#21890 fuzz: Limit ParseISO8601DateTime fuzzing to 32-bit (MarcoFalke) +- bitcoin/bitcoin#21891 fuzz: Remove strprintf test cases that are known to fail (MarcoFalke) +- bitcoin/bitcoin#21892 fuzz: Avoid excessively large min fee rate in `tx_pool` (MarcoFalke) +- bitcoin/bitcoin#21895 Add TSA annotations to the WorkQueue class members (hebasto) +- bitcoin/bitcoin#21900 use MiniWallet for `feature_csv_activation.py` (theStack) +- bitcoin/bitcoin#21909 fuzz: Limit max insertions in timedata fuzz test (MarcoFalke) +- bitcoin/bitcoin#21922 fuzz: Avoid timeout in EncodeBase58 (MarcoFalke) +- bitcoin/bitcoin#21927 fuzz: Run const CScript member functions only once (MarcoFalke) +- bitcoin/bitcoin#21929 fuzz: Remove incorrect float round-trip serialization test (MarcoFalke) +- bitcoin/bitcoin#21936 fuzz: Terminate immediately if a fuzzing harness tries to create a TCP socket (belt and suspenders) (practicalswift) +- bitcoin/bitcoin#21941 fuzz: Call const member functions in addrman fuzz test only once (MarcoFalke) +- bitcoin/bitcoin#21945 add P2PK support to MiniWallet (theStack) +- bitcoin/bitcoin#21948 Fix off-by-one in mockscheduler test RPC (MarcoFalke) +- bitcoin/bitcoin#21953 fuzz: Add `utxo_snapshot` target (MarcoFalke) +- bitcoin/bitcoin#21970 fuzz: Add missing CheckTransaction before CheckTxInputs (MarcoFalke) +- bitcoin/bitcoin#21989 Use `COINBASE_MATURITY` in functional tests (kiminuo) +- bitcoin/bitcoin#22003 Add thread safety annotations (ajtowns) +- bitcoin/bitcoin#22004 fuzz: Speed up transaction fuzz target (MarcoFalke) +- bitcoin/bitcoin#22005 fuzz: Speed up banman fuzz target (MarcoFalke) +- bitcoin/bitcoin#22029 [fuzz] Improve transport deserialization fuzz test coverage (dhruv) +- bitcoin/bitcoin#22048 MiniWallet: introduce enum type for output mode (theStack) +- bitcoin/bitcoin#22057 use MiniWallet (P2PK mode) for `feature_dersig.py` (theStack) +- bitcoin/bitcoin#22065 Mark `CheckTxInputs` `[[nodiscard]]`. Avoid UUM in fuzzing harness `coins_view` (practicalswift) +- bitcoin/bitcoin#22069 fuzz: don't try and use fopencookie() when building for Android (fanquake) +- bitcoin/bitcoin#22082 update nanobench from release 4.0.0 to 4.3.4 (martinus) +- bitcoin/bitcoin#22086 remove BasicTestingSetup from unit tests that don't need it (fanquake) +- bitcoin/bitcoin#22089 MiniWallet: fix fee calculation for P2PK and check tx vsize (theStack) +- bitcoin/bitcoin#21107, bitcoin/bitcoin#22092 Convert documentation into type annotations (fanquake) +- bitcoin/bitcoin#22095 Additional BIP32 test vector for hardened derivation with leading zeros (kristapsk) +- bitcoin/bitcoin#22103 Fix IPv6 check on BSD systems (n-thumann) +- bitcoin/bitcoin#22118 check anchors.dat when node starts for the first time (brunoerg) +- bitcoin/bitcoin#22120 `p2p_invalid_block`: Check that a block rejected due to too-new tim… (willcl-ark) +- bitcoin/bitcoin#22153 Fix `p2p_leak.py` intermittent failure (mzumsande) +- bitcoin/bitcoin#22169 p2p, rpc, fuzz: various tiny follow-ups (jonatack) +- bitcoin/bitcoin#22176 Correct outstanding -Werror=sign-compare errors (Empact) +- bitcoin/bitcoin#22180 fuzz: Increase branch coverage of the float fuzz target (MarcoFalke) +- bitcoin/bitcoin#22187 Add `sync_blocks` in `wallet_orphanedreward.py` (domob1812) +- bitcoin/bitcoin#22201 Fix TestShell to allow running in Jupyter Notebook (josibake) +- bitcoin/bitcoin#22202 Add temporary coinstats suppressions (MarcoFalke) +- bitcoin/bitcoin#22203 Use ConnmanTestMsg from test lib in `denialofservice_tests` (MarcoFalke) +- bitcoin/bitcoin#22210 Use MiniWallet in `test_no_inherited_signaling` RBF test (MarcoFalke) +- bitcoin/bitcoin#22224 Update msvc and appveyor builds to use Qt5.12.11 binaries (sipsorcery) +- bitcoin/bitcoin#22249 Kill process group to avoid dangling processes when using `--failfast` (S3RK) +- bitcoin/bitcoin#22267 fuzz: Speed up crypto fuzz target (MarcoFalke) +- bitcoin/bitcoin#22270 Add bitcoin-util tests (+refactors) (MarcoFalke) +- bitcoin/bitcoin#22271 fuzz: Assert roundtrip equality for `CPubKey` (theStack) +- bitcoin/bitcoin#22279 fuzz: add missing ECCVerifyHandle to `base_encode_decode` (apoelstra) +- bitcoin/bitcoin#22292 bench, doc: benchmarking updates and fixups (jonatack) +- bitcoin/bitcoin#22306 Improvements to `p2p_addr_relay.py` (amitiuttarwar) +- bitcoin/bitcoin#22310 Add functional test for replacement relay fee check (ariard) +- bitcoin/bitcoin#22311 Add missing syncwithvalidationinterfacequeue in `p2p_blockfilters` (MarcoFalke) +- bitcoin/bitcoin#22313 Add missing `sync_all` to `feature_coinstatsindex` (MarcoFalke) +- bitcoin/bitcoin#22322 fuzz: Check banman roundtrip (MarcoFalke) +- bitcoin/bitcoin#22363 Use `script_util` helpers for creating P2{PKH,SH,WPKH,WSH} scripts (theStack) +- bitcoin/bitcoin#22399 fuzz: Rework CTxDestination fuzzing (MarcoFalke) +- bitcoin/bitcoin#22408 add tests for `bad-txns-prevout-null` reject reason (theStack) +- bitcoin/bitcoin#22445 fuzz: Move implementations of non-template fuzz helpers from util.h to util.cpp (sriramdvt) +- bitcoin/bitcoin#22446 Fix `wallet_listdescriptors.py` if bdb is not compiled (hebasto) +- bitcoin/bitcoin#22447 Whitelist `rpc_rawtransaction` peers to speed up tests (jonatack) +- bitcoin/bitcoin#22742 Use proper target in `do_fund_send` (S3RK) + +### Miscellaneous +- bitcoin/bitcoin#19337 sync: Detect double lock from the same thread (vasild) +- bitcoin/bitcoin#19809 log: Prefix log messages with function name and source code location if -logsourcelocations is set (practicalswift) +- bitcoin/bitcoin#19866 eBPF Linux tracepoints (jb55) +- bitcoin/bitcoin#20024 init: Fix incorrect warning "Reducing -maxconnections from N to N-1, because of system limitations" (practicalswift) +- bitcoin/bitcoin#20145 contrib: Add getcoins.py script to get coins from (signet) faucet (kallewoof) +- bitcoin/bitcoin#20255 util: Add assume() identity function (MarcoFalke) +- bitcoin/bitcoin#20288 script, doc: Contrib/seeds updates (jonatack) +- bitcoin/bitcoin#20358 src/randomenv.cpp: Fix build on uclibc (ffontaine) +- bitcoin/bitcoin#20406 util: Avoid invalid integer negation in formatmoney and valuefromamount (practicalswift) +- bitcoin/bitcoin#20434 contrib: Parse elf directly for symbol and security checks (laanwj) +- bitcoin/bitcoin#20451 lint: Run mypy over contrib/devtools (fanquake) +- bitcoin/bitcoin#20476 contrib: Add test for elf symbol-check (laanwj) +- bitcoin/bitcoin#20530 lint: Update cppcheck linter to c++17 and improve explicit usage (fjahr) +- bitcoin/bitcoin#20589 log: Clarify that failure to read/write `fee_estimates.dat` is non-fatal (MarcoFalke) +- bitcoin/bitcoin#20602 util: Allow use of c++14 chrono literals (MarcoFalke) +- bitcoin/bitcoin#20605 init: Signal-safe instant shutdown (laanwj) +- bitcoin/bitcoin#20608 contrib: Add symbol check test for PE binaries (fanquake) +- bitcoin/bitcoin#20689 contrib: Replace binary verification script verify.sh with python rewrite (theStack) +- bitcoin/bitcoin#20715 util: Add argsmanager::getcommand() and use it in bitcoin-wallet (MarcoFalke) +- bitcoin/bitcoin#20735 script: Remove outdated extract-osx-sdk.sh (hebasto) +- bitcoin/bitcoin#20817 lint: Update list of spelling linter false positives, bump to codespell 2.0.0 (theStack) +- bitcoin/bitcoin#20884 script: Improve robustness of bitcoind.service on startup (hebasto) +- bitcoin/bitcoin#20906 contrib: Embed c++11 patch in `install_db4.sh` (gruve-p) +- bitcoin/bitcoin#21004 contrib: Fix docker args conditional in gitian-build (setpill) +- bitcoin/bitcoin#21007 bitcoind: Add -daemonwait option to wait for initialization (laanwj) +- bitcoin/bitcoin#21041 log: Move "Pre-allocating up to position 0x[…] in […].dat" log message to debug category (practicalswift) +- bitcoin/bitcoin#21059 Drop boost/preprocessor dependencies (hebasto) +- bitcoin/bitcoin#21087 guix: Passthrough `BASE_CACHE` into container (dongcarl) +- bitcoin/bitcoin#21088 guix: Jump forwards in time-machine and adapt (dongcarl) +- bitcoin/bitcoin#21089 guix: Add support for powerpc64{,le} (dongcarl) +- bitcoin/bitcoin#21110 util: Remove boost `posix_time` usage from `gettime*` (fanquake) +- bitcoin/bitcoin#21111 Improve OpenRC initscript (parazyd) +- bitcoin/bitcoin#21123 code style: Add EditorConfig file (kiminuo) +- bitcoin/bitcoin#21173 util: Faster hexstr => 13% faster blocktojson (martinus) +- bitcoin/bitcoin#21221 tools: Allow argument/parameter bin packing in clang-format (jnewbery) +- bitcoin/bitcoin#21244 Move GetDataDir to ArgsManager (kiminuo) +- bitcoin/bitcoin#21255 contrib: Run test-symbol-check for risc-v (fanquake) +- bitcoin/bitcoin#21271 guix: Explicitly set umask in build container (dongcarl) +- bitcoin/bitcoin#21300 script: Add explanatory comment to tc.sh (dscotese) +- bitcoin/bitcoin#21317 util: Make assume() usable as unary expression (MarcoFalke) +- bitcoin/bitcoin#21336 Make .gitignore ignore src/test/fuzz/fuzz.exe (hebasto) +- bitcoin/bitcoin#21337 guix: Update darwin native packages dependencies (hebasto) +- bitcoin/bitcoin#21405 compat: remove memcpy -> memmove backwards compatibility alias (fanquake) +- bitcoin/bitcoin#21418 contrib: Make systemd invoke dependencies only when ready (laanwj) +- bitcoin/bitcoin#21447 Always add -daemonwait to known command line arguments (hebasto) +- bitcoin/bitcoin#21471 bugfix: Fix `bech32_encode` calls in `gen_key_io_test_vectors.py` (sipa) +- bitcoin/bitcoin#21615 script: Add trusted key for hebasto (hebasto) +- bitcoin/bitcoin#21664 contrib: Use lief for macos and windows symbol & security checks (fanquake) +- bitcoin/bitcoin#21695 contrib: Remove no longer used contrib/bitcoin-qt.pro (hebasto) +- bitcoin/bitcoin#21711 guix: Add full installation and usage documentation (dongcarl) +- bitcoin/bitcoin#21799 guix: Use `gcc-8` across the board (dongcarl) +- bitcoin/bitcoin#21802 Avoid UB in util/asmap (advance a dereferenceable iterator outside its valid range) (MarcoFalke) +- bitcoin/bitcoin#21823 script: Update reviewers (jonatack) +- bitcoin/bitcoin#21850 Remove `GetDataDir(net_specific)` function (kiminuo) +- bitcoin/bitcoin#21871 scripts: Add checks for minimum required os versions (fanquake) +- bitcoin/bitcoin#21966 Remove double serialization; use software encoder for fee estimation (sipa) +- bitcoin/bitcoin#22060 contrib: Add torv3 seed nodes for testnet, drop v2 ones (laanwj) +- bitcoin/bitcoin#22244 devtools: Correctly extract symbol versions in symbol-check (laanwj) +- bitcoin/bitcoin#22533 guix/build: Remove vestigial SKIPATTEST.TAG (dongcarl) +- bitcoin/bitcoin#22643 guix-verify: Non-zero exit code when anything fails (dongcarl) +- bitcoin/bitcoin#22654 guix: Don't include directory name in SHA256SUMS (achow101) + +### Documentation +- bitcoin/bitcoin#15451 clarify getdata limit after #14897 (HashUnlimited) +- bitcoin/bitcoin#15545 Explain why CheckBlock() is called before AcceptBlock (Sjors) +- bitcoin/bitcoin#17350 Add developer documentation to isminetype (HAOYUatHZ) +- bitcoin/bitcoin#17934 Use `CONFIG_SITE` variable instead of --prefix option (hebasto) +- bitcoin/bitcoin#18030 Coin::IsSpent() can also mean never existed (Sjors) +- bitcoin/bitcoin#18096 IsFinalTx comment about nSequence & `OP_CLTV` (nothingmuch) +- bitcoin/bitcoin#18568 Clarify developer notes about constant naming (ryanofsky) +- bitcoin/bitcoin#19961 doc: tor.md updates (jonatack) +- bitcoin/bitcoin#19968 Clarify CRollingBloomFilter size estimate (robot-dreams) +- bitcoin/bitcoin#20200 Rename CODEOWNERS to REVIEWERS (adamjonas) +- bitcoin/bitcoin#20329 docs/descriptors.md: Remove hardened marker in the path after xpub (dgpv) +- bitcoin/bitcoin#20380 Add instructions on how to fuzz the P2P layer using Honggfuzz NetDriver (practicalswift) +- bitcoin/bitcoin#20414 Remove generated manual pages from master branch (laanwj) +- bitcoin/bitcoin#20473 Document current boost dependency as 1.71.0 (laanwj) +- bitcoin/bitcoin#20512 Add bash as an OpenBSD dependency (emilengler) +- bitcoin/bitcoin#20568 Use FeeModes doc helper in estimatesmartfee (MarcoFalke) +- bitcoin/bitcoin#20577 libconsensus: add missing error code description, fix NBitcoin link (theStack) +- bitcoin/bitcoin#20587 Tidy up Tor doc (more stringent) (wodry) +- bitcoin/bitcoin#20592 Update wtxidrelay documentation per BIP339 (jonatack) +- bitcoin/bitcoin#20601 Update for FreeBSD 12.2, add GUI Build Instructions (jarolrod) +- bitcoin/bitcoin#20635 fix misleading comment about call to non-existing function (pox) +- bitcoin/bitcoin#20646 Refer to BIPs 339/155 in feature negotiation (jonatack) +- bitcoin/bitcoin#20653 Move addr relay comment in net to correct place (MarcoFalke) +- bitcoin/bitcoin#20677 Remove shouty enums in `net_processing` comments (sdaftuar) +- bitcoin/bitcoin#20741 Update 'Secure string handling' (prayank23) +- bitcoin/bitcoin#20757 tor.md and -onlynet help updates (jonatack) +- bitcoin/bitcoin#20829 Add -netinfo help (jonatack) +- bitcoin/bitcoin#20830 Update developer notes with signet (jonatack) +- bitcoin/bitcoin#20890 Add explicit macdeployqtplus dependencies install step (hebasto) +- bitcoin/bitcoin#20913 Add manual page generation for bitcoin-util (laanwj) +- bitcoin/bitcoin#20985 Add xorriso to macOS depends packages (fanquake) +- bitcoin/bitcoin#20986 Update developer notes to discourage very long lines (jnewbery) +- bitcoin/bitcoin#20987 Add instructions for generating RPC docs (ben-kaufman) +- bitcoin/bitcoin#21026 Document use of make-tag script to make tags (laanwj) +- bitcoin/bitcoin#21028 doc/bips: Add BIPs 43, 44, 49, and 84 (luke-jr) +- bitcoin/bitcoin#21049 Add release notes for listdescriptors RPC (S3RK) +- bitcoin/bitcoin#21060 More precise -debug and -debugexclude doc (wodry) +- bitcoin/bitcoin#21077 Clarify -timeout and -peertimeout config options (glozow) +- bitcoin/bitcoin#21105 Correctly identify script type (niftynei) +- bitcoin/bitcoin#21163 Guix is shipped in Debian and Ubuntu (MarcoFalke) +- bitcoin/bitcoin#21210 Rework internal and external links (MarcoFalke) +- bitcoin/bitcoin#21246 Correction for VerifyTaprootCommitment comments (roconnor-blockstream) +- bitcoin/bitcoin#21263 Clarify that squashing should happen before review (MarcoFalke) +- bitcoin/bitcoin#21323 guix, doc: Update default HOSTS value (hebasto) +- bitcoin/bitcoin#21324 Update build instructions for Fedora (hebasto) +- bitcoin/bitcoin#21343 Revamp macOS build doc (jarolrod) +- bitcoin/bitcoin#21346 install qt5 when building on macOS (fanquake) +- bitcoin/bitcoin#21384 doc: add signet to bitcoin.conf documentation (jonatack) +- bitcoin/bitcoin#21394 Improve comment about protected peers (amitiuttarwar) +- bitcoin/bitcoin#21398 Update fuzzing docs for afl-clang-lto (MarcoFalke) +- bitcoin/bitcoin#21444 net, doc: Doxygen updates and fixes in netbase.{h,cpp} (jonatack) +- bitcoin/bitcoin#21481 Tell howto install clang-format on Debian/Ubuntu (wodry) +- bitcoin/bitcoin#21567 Fix various misleading comments (glozow) +- bitcoin/bitcoin#21661 Fix name of script guix-build (Emzy) +- bitcoin/bitcoin#21672 Remove boostrap info from `GUIX_COMMON_FLAGS` doc (fanquake) +- bitcoin/bitcoin#21688 Note on SDK for macOS depends cross-compile (jarolrod) +- bitcoin/bitcoin#21709 Update reduce-memory.md and bitcoin.conf -maxconnections info (jonatack) +- bitcoin/bitcoin#21710 update helps for addnode rpc and -addnode/-maxconnections config options (jonatack) +- bitcoin/bitcoin#21752 Clarify that feerates are per virtual size (MarcoFalke) +- bitcoin/bitcoin#21811 Remove Visual Studio 2017 reference from readme (sipsorcery) +- bitcoin/bitcoin#21818 Fixup -coinstatsindex help, update bitcoin.conf and files.md (jonatack) +- bitcoin/bitcoin#21856 add OSS-Fuzz section to fuzzing.md doc (adamjonas) +- bitcoin/bitcoin#21912 Remove mention of priority estimation (MarcoFalke) +- bitcoin/bitcoin#21925 Update bips.md for 0.21.1 (MarcoFalke) +- bitcoin/bitcoin#21942 improve make with parallel jobs description (klementtan) +- bitcoin/bitcoin#21947 Fix OSS-Fuzz links (MarcoFalke) +- bitcoin/bitcoin#21988 note that brew installed qt is not supported (jarolrod) +- bitcoin/bitcoin#22056 describe in fuzzing.md how to reproduce a CI crash (jonatack) +- bitcoin/bitcoin#22080 add maxuploadtarget to bitcoin.conf example (jarolrod) +- bitcoin/bitcoin#22088 Improve note on choosing posix mingw32 (jarolrod) +- bitcoin/bitcoin#22109 Fix external links (IRC, …) (MarcoFalke) +- bitcoin/bitcoin#22121 Various validation doc fixups (MarcoFalke) +- bitcoin/bitcoin#22172 Update tor.md, release notes with removal of tor v2 support (jonatack) +- bitcoin/bitcoin#22204 Remove obsolete `okSafeMode` RPC guideline from developer notes (theStack) +- bitcoin/bitcoin#22208 Update `REVIEWERS` (practicalswift) +- bitcoin/bitcoin#22250 add basic I2P documentation (vasild) +- bitcoin/bitcoin#22296 Final merge of release notes snippets, mv to wiki (MarcoFalke) +- bitcoin/bitcoin#22335 recommend `--disable-external-signer` in OpenBSD build guide (theStack) +- bitcoin/bitcoin#22339 Document minimum required libc++ version (hebasto) +- bitcoin/bitcoin#22349 Repository IRC updates (jonatack) +- bitcoin/bitcoin#22360 Remove unused section from release process (MarcoFalke) +- bitcoin/bitcoin#22369 Add steps for Transifex to release process (jonatack) +- bitcoin/bitcoin#22393 Added info to bitcoin.conf doc (bliotti) +- bitcoin/bitcoin#22402 Install Rosetta on M1-macOS for qt in depends (hebasto) +- bitcoin/bitcoin#22432 Fix incorrect `testmempoolaccept` doc (glozow) +- bitcoin/bitcoin#22648 doc, test: improve i2p/tor docs and i2p reachable unit tests (jonatack) + +Credits +======= + +Thanks to everyone who directly contributed to this release: + +- Aaron Clauson +- Adam Jonas +- amadeuszpawlik +- Amiti Uttarwar +- Andrew Chow +- Andrew Poelstra +- Anthony Towns +- Antoine Poinsot +- Antoine Riard +- apawlik +- apitko +- Ben Carman +- Ben Woosley +- benk10 +- Bezdrighin +- Block Mechanic +- Brian Liotti +- Bruno Garcia +- Carl Dong +- Christian Decker +- coinforensics +- Cory Fields +- Dan Benjamin +- Daniel Kraft +- Darius Parvin +- Dhruv Mehta +- Dmitry Goncharov +- Dmitry Petukhov +- dplusplus1024 +- dscotese +- Duncan Dean +- Elle Mouton +- Elliott Jin +- Emil Engler +- Ethan Heilman +- eugene +- Evan Klitzke +- Fabian Jahr +- Fabrice Fontaine +- fanquake +- fdov +- flack +- Fotis Koutoupas +- Fu Yong Quah +- fyquah +- glozow +- Gregory Sanders +- Guido Vranken +- Gunar C. Gessner +- h +- HAOYUatHZ +- Hennadii Stepanov +- Igor Cota +- Ikko Ashimine +- Ivan Metlushko +- jackielove4u +- James O'Beirne +- Jarol Rodriguez +- Joel Klabo +- John Newbery +- Jon Atack +- Jonas Schnelli +- João Barbosa +- Josiah Baker +- Karl-Johan Alm +- Kiminuo +- Klement Tan +- Kristaps Kaupe +- Larry Ruane +- lisa neigut +- Lucas Ontivero +- Luke Dashjr +- Maayan Keshet +- MarcoFalke +- Martin Ankerl +- Martin Zumsande +- Michael Dietz +- Michael Polzer +- Michael Tidwell +- Niklas Gögge +- nthumann +- Oliver Gugger +- parazyd +- Patrick Strateman +- Pavol Rusnak +- Peter Bushnell +- Pierre K +- Pieter Wuille +- PiRK +- pox +- practicalswift +- Prayank +- R E Broadley +- Rafael Sadowski +- randymcmillan +- Raul Siles +- Riccardo Spagni +- Russell O'Connor +- Russell Yanofsky +- S3RK +- saibato +- Samuel Dobson +- sanket1729 +- Sawyer Billings +- Sebastian Falbesoner +- setpill +- sgulls +- sinetek +- Sjors Provoost +- Sriram +- Stephan Oeste +- Suhas Daftuar +- Sylvain Goumy +- t-bast +- Troy Giorshev +- Tushar Singla +- Tyler Chambers +- Uplab +- Vasil Dimov +- W. J. van der Laan +- willcl-ark +- William Bright +- William Casarin +- windsok +- wodry +- Yerzhan Mazhkenov +- Yuval Kogman +- Zero + +As well as to everyone that helped with translations on +[Transifex](https://www.transifex.com/bitcoin/bitcoin/). diff --git a/doc/release-process.md b/doc/release-process.md index 0ac67b9146..5a74f72b6e 100644 --- a/doc/release-process.md +++ b/doc/release-process.md @@ -6,21 +6,21 @@ Release Process ### Before every release candidate * Update translations see [translation_process.md](https://github.com/bitcoin/bitcoin/blob/master/doc/translation_process.md#synchronising-translations). -* Update manpages, see [gen-manpages.sh](https://github.com/bitcoin/bitcoin/blob/master/contrib/devtools/README.md#gen-manpagessh). * Update release candidate version in `configure.ac` (`CLIENT_VERSION_RC`). +* Update manpages (after rebuilding the binaries), see [gen-manpages.py](https://github.com/bitcoin/bitcoin/blob/master/contrib/devtools/README.md#gen-manpagespy). ### Before every major and minor release * Update [bips.md](bips.md) to account for changes since the last release (don't forget to bump the version number on the first line). * Update version in `configure.ac` (don't forget to set `CLIENT_VERSION_RC` to `0`). +* Update manpages (see previous section) * Write release notes (see "Write the release notes" below). ### Before every major release * On both the master branch and the new release branch: - update `CLIENT_VERSION_MAJOR` in [`configure.ac`](../configure.ac) - - update `CLIENT_VERSION_MAJOR`, `PACKAGE_VERSION`, and `PACKAGE_STRING` in [`build_msvc/bitcoin_config.h`](/build_msvc/bitcoin_config.h) -* On the new release branch in [`configure.ac`](../configure.ac) and [`build_msvc/bitcoin_config.h`](/build_msvc/bitcoin_config.h) (see [this commit](https://github.com/bitcoin/bitcoin/commit/742f7dd)): +* On the new release branch in [`configure.ac`](../configure.ac)(see [this commit](https://github.com/bitcoin/bitcoin/commit/742f7dd)): - set `CLIENT_VERSION_MINOR` to `0` - set `CLIENT_VERSION_BUILD` to `0` - set `CLIENT_VERSION_IS_RELEASE` to `true` @@ -60,7 +60,7 @@ Release Process To tag the version (or release candidate) in git, use the `make-tag.py` script from [bitcoin-maintainer-tools](https://github.com/bitcoin-core/bitcoin-maintainer-tools). From the root of the repository run: - ../bitcoin-maintainer-tools/make-tag.py v(new version, e.g. 0.20.0) + ../bitcoin-maintainer-tools/make-tag.py v(new version, e.g. 23.0) This will perform a few last-minute consistency checks in the build system files, and if they pass, create a signed tag. @@ -98,7 +98,7 @@ Checkout the Bitcoin Core version you'd like to build: pushd ./bitcoin SIGNER='(your builder key, ie bluematt, sipa, etc)' VERSION='(new version without v-prefix, e.g. 0.20.0)' -git fetch "v${VERSION}" +git fetch origin "v${VERSION}" git checkout "v${VERSION}" popd ``` @@ -119,7 +119,7 @@ details. ### Build and attest to build outputs: Follow the relevant Guix README.md sections: -- [Performing a build](/contrib/guix/README.md#performing-a-build) +- [Building](/contrib/guix/README.md#building) - [Attesting to build outputs](/contrib/guix/README.md#attesting-to-build-outputs) ### Verify other builders' signatures to your own. (Optional) @@ -136,7 +136,7 @@ Commit your signature to guix.sigs: ```sh pushd ./guix.sigs git add "${VERSION}/${SIGNER}"/noncodesigned.SHA256SUMS{,.asc} -git commit -m "Add ${VERSION} unsigned sigs for ${SIGNER}" +git commit -m "Add attestations by ${SIGNER} for ${VERSION} non-codesigned" git push # Assuming you can push to the guix.sigs tree popd ``` @@ -160,6 +160,9 @@ Codesigner only: Sign the windows binaries: Enter the passphrase for the key when prompted signature-win.tar.gz will be created +Code-signer only: It is advised to test that the code signature attaches properly prior to tagging by performing the `guix-codesign` step. +However if this is done, once the release has been tagged in the bitcoin-detached-sigs repo, the `guix-codesign` step must be performed again in order for the guix attestation to be valid when compared against the attestations of non-codesigner builds. + Codesigner only: Commit the detached codesign payloads: ```sh @@ -189,37 +192,29 @@ Commit your signature for the signed macOS/Windows binaries: ```sh pushd ./guix.sigs git add "${VERSION}/${SIGNER}"/all.SHA256SUMS{,.asc} -git commit -m "Add ${SIGNER} ${VERSION} signed binaries signatures" +git commit -m "Add attestations by ${SIGNER} for ${VERSION} codesigned" git push # Assuming you can push to the guix.sigs tree popd ``` ### After 3 or more people have guix-built and their results match: -Combine `all.SHA256SUMS` and `all.SHA256SUMS.asc` into a clear-signed -`SHA256SUMS.asc` message: - -```sh -echo -e "-----BEGIN PGP SIGNED MESSAGE-----\nHash: SHA256\n\n$(cat all.SHA256SUMS)\n$(cat filename.txt.asc)" > SHA256SUMS.asc -``` - -Here's an equivalent, more readable command if you're confident that you won't -mess up whitespaces when copy-pasting: +Combine the `all.SHA256SUMS.asc` file from all signers into `SHA256SUMS.asc`: ```bash -cat << EOF > SHA256SUMS.asc ------BEGIN PGP SIGNED MESSAGE----- -Hash: SHA256 - -$(cat all.SHA256SUMS) -$(cat all.SHA256SUMS.asc) -EOF +cat "$VERSION"/*/all.SHA256SUMS.asc > SHA256SUMS.asc ``` -- Upload to the bitcoincore.org server (`/var/www/bin/bitcoin-core-${VERSION}`): - 1. The contents of `./bitcoin/guix-build-${VERSION}/output`, except for + +- Upload to the bitcoincore.org server (`/var/www/bin/bitcoin-core-${VERSION}/`): + 1. The contents of each `./bitcoin/guix-build-${VERSION}/output/${HOST}/` directory, except for `*-debug*` files. + Guix will output all of the results into host subdirectories, but the SHA256SUMS + file does not include these subdirectories. In order for downloads via torrent + to verify without directory structure modification, all of the uploaded files + need to be in the same directory as the SHA256SUMS file. + The `*-debug*` files generated by the guix build contain debug symbols for troubleshooting by developers. It is assumed that anyone that is interested in debugging can run guix to generate the files for @@ -227,7 +222,13 @@ EOF as save storage space *do not upload these to the bitcoincore.org server, nor put them in the torrent*. - 2. The combined clear-signed message you just created `SHA256SUMS.asc` + ```sh + find guix-build-${VERSION}/output/ -maxdepth 2 -type f -not -name "SHA256SUMS.part" -and -not -name "*debug*" -exec scp {} user@bitcoincore.org:/var/www/bin/bitcoin-core-${VERSION} \; + ``` + + 2. The `SHA256SUMS` file + + 3. The `SHA256SUMS.asc` combined signature file you just created - Create a torrent of the `/var/www/bin/bitcoin-core-${VERSION}` directory such that at the top level there is only one file: the `bitcoin-core-${VERSION}` @@ -252,6 +253,10 @@ EOF - bitcoincore.org maintained versions update: [table](https://github.com/bitcoin-core/bitcoincore.org/commits/master/_includes/posts/maintenance-table.md) + - Delete post-EOL [release branches](https://github.com/bitcoin/bitcoin/branches/all) and create a tag `v${branch_name}-final`. + + - Delete ["Needs backport" labels](https://github.com/bitcoin/bitcoin/labels?q=backport) for non-existing branches. + - bitcoincore.org RPC documentation update - Install [golang](https://golang.org/doc/install) @@ -270,26 +275,7 @@ EOF - Push the flatpak to flathub, e.g. https://github.com/flathub/org.bitcoincore.bitcoin-qt/pull/2 - - Push the latest version to master (if applicable), e.g. https://github.com/bitcoin-core/packaging/pull/32 - - - Create a new branch for the major release "0.xx" from master (used to build the snap package) and request the - track (if applicable), e.g. https://forum.snapcraft.io/t/track-request-for-bitcoin-core-snap/10112/7 - - - Notify MarcoFalke so that he can start building the snap package - - - https://code.launchpad.net/~bitcoin-core/bitcoin-core-snap/+git/packaging (Click "Import Now" to fetch the branch) - - https://code.launchpad.net/~bitcoin-core/bitcoin-core-snap/+git/packaging/+ref/0.xx (Click "Create snap package") - - Name it "bitcoin-core-snap-0.xx" - - Leave owner and series as-is - - Select architectures that are compiled via guix - - Leave "automatically build when branch changes" unticked - - Tick "automatically upload to store" - - Put "bitcoin-core" in the registered store package name field - - Tick the "edge" box - - Put "0.xx" in the track field - - Click "create snap package" - - Click "Request builds" for every new release on this branch (after updating the snapcraft.yml in the branch to reflect the latest guix results) - - Promote release on https://snapcraft.io/bitcoin-core/releases if it passes sanity checks + - Push the snap, see https://github.com/bitcoin-core/packaging/blob/master/snap/build.md - This repo diff --git a/doc/tapscript_opcodes.md b/doc/tapscript_opcodes.md index 1c7ba6fa39..7e673e1af8 100644 --- a/doc/tapscript_opcodes.md +++ b/doc/tapscript_opcodes.md @@ -83,4 +83,4 @@ When defining the opcodes which can fail, we only define the success path, and a - In order to inspect the current input, `OP_PUSHCURRENTINPUTINDEX` can be used in combination with `OP_INSPECTINPUTXX` to obtain information about input being spent on stack - The input nNonce field is not consistently stored in elements UTXO database. Therefore, it is not covered in sighash or `wtxid` and hence introspecting it is not possible. -- Coversion opcodes can be used be used to convert ScriptNums/LE32 nums to LE64 for operations. \ No newline at end of file +- Conversion opcodes can be used be used to convert ScriptNums/LE32 nums to LE64 for operations. diff --git a/doc/tor.md b/doc/tor.md index 7d134b64e0..086e6747bf 100644 --- a/doc/tor.md +++ b/doc/tor.md @@ -23,10 +23,9 @@ There are several ways to see your local onion address in Bitcoin Core: You may set the `-debug=tor` config logging option to have additional information in the debug log about your Tor configuration. -CLI `-addrinfo` returns the number of addresses known to your node per network -type, including Tor v2 and v3. This is useful to see how many onion addresses -are known to your node for `-onlynet=onion` and how many Tor v3 addresses it -knows when upgrading to Bitcoin Core v22.0 and up that supports Tor v3 only. +CLI `-addrinfo` returns the number of addresses known to your node per +network. This can be useful to see how many onion peers your node knows, +e.g. for `-onlynet=onion`. ## 1. Run Bitcoin Core behind a Tor proxy @@ -41,9 +40,11 @@ outgoing connections, but more is possible. -onion=ip:port Set the proxy server to use for Tor onion services. You do not need to set this if it's the same as -proxy. You can use -onion=0 to explicitly disable access to onion services. + ------------------------------------------------------------------ Note: Only the -proxy option sets the proxy for DNS requests; with -onion they will not route over Tor, so use -proxy if you have privacy concerns. + ------------------------------------------------------------------ -listen When using -proxy, listening is disabled by default. If you want to manually configure an onion service (see section 3), you'll @@ -57,11 +58,7 @@ outgoing connections, but more is possible. -onlynet=onion Make outgoing connections only to .onion addresses. Incoming connections are not affected by this option. This option can be specified multiple times to allow multiple network types, e.g. - ipv4, ipv6 or onion. If you use this option with values other - than onion you *cannot* disable onion connections; outgoing onion - connections will be enabled when you use -proxy or -onion. Use - -noonion or -onion=0 if you want to be sure there are no outbound - onion connections over the default proxy or your defined -proxy. + onlynet=onion, onlynet=i2p. In a typical situation, this suffices to run behind a Tor proxy: @@ -134,7 +131,7 @@ You can also check the group of the cookie file. On most Linux systems, the Tor auth cookie will usually be `/run/tor/control.authcookie`: ``` -stat -c '%G' /run/tor/control.authcookie +TORGROUP=$(stat -c '%G' /run/tor/control.authcookie) ``` Once you have determined the `${TORGROUP}` and selected the `${USER}` that will diff --git a/doc/tracing.md b/doc/tracing.md new file mode 100644 index 0000000000..6ec6a6c218 --- /dev/null +++ b/doc/tracing.md @@ -0,0 +1,319 @@ +# User-space, Statically Defined Tracing (USDT) for Bitcoin Core + +Bitcoin Core includes statically defined tracepoints to allow for more +observability during development, debugging, code review, and production usage. +These tracepoints make it possible to keep track of custom statistics and +enable detailed monitoring of otherwise hidden internals. They have +little to no performance impact when unused. + +``` +eBPF and USDT Overview +====================== + + ┌──────────────────┐ ┌──────────────┐ + │ tracing script │ │ bitcoind │ + │==================│ 2. │==============│ + │ eBPF │ tracing │ hooks │ │ + │ code │ logic │ into┌─┤►tracepoint 1─┼───┐ 3. + └────┬───┴──▲──────┘ ├─┤►tracepoint 2 │ │ pass args + 1. │ │ 4. │ │ ... │ │ to eBPF + User compiles │ │ pass data to │ └──────────────┘ │ program + Space & loads │ │ tracing script │ │ + ─────────────────┼──────┼─────────────────┼────────────────────┼─── + Kernel │ │ │ │ + Space ┌──┬─▼──────┴─────────────────┴────────────┐ │ + │ │ eBPF program │◄──────┘ + │ └───────────────────────────────────────┤ + │ eBPF kernel Virtual Machine (sandboxed) │ + └──────────────────────────────────────────┘ + +1. The tracing script compiles the eBPF code and loads the eBPF program into a kernel VM +2. The eBPF program hooks into one or more tracepoints +3. When the tracepoint is called, the arguments are passed to the eBPF program +4. The eBPF program processes the arguments and returns data to the tracing script +``` + +The Linux kernel can hook into the tracepoints during runtime and pass data to +sandboxed [eBPF] programs running in the kernel. These eBPF programs can, for +example, collect statistics or pass data back to user-space scripts for further +processing. + +[eBPF]: https://ebpf.io/ + +The two main eBPF front-ends with support for USDT are [bpftrace] and +[BPF Compiler Collection (BCC)]. BCC is used for complex tools and daemons and +`bpftrace` is preferred for one-liners and shorter scripts. Examples for both can +be found in [contrib/tracing]. + +[bpftrace]: https://github.com/iovisor/bpftrace +[BPF Compiler Collection (BCC)]: https://github.com/iovisor/bcc +[contrib/tracing]: ../contrib/tracing/ + +## Tracepoint documentation + +The currently available tracepoints are listed here. + +### Context `net` + +#### Tracepoint `net:inbound_message` + +Is called when a message is received from a peer over the P2P network. Passes +information about our peer, the connection and the message as arguments. + +Arguments passed: +1. Peer ID as `int64` +2. Peer Address and Port (IPv4, IPv6, Tor v3, I2P, ...) as `pointer to C-style String` (max. length 68 characters) +3. Connection Type (inbound, feeler, outbound-full-relay, ...) as `pointer to C-style String` (max. length 20 characters) +4. Message Type (inv, ping, getdata, addrv2, ...) as `pointer to C-style String` (max. length 20 characters) +5. Message Size in bytes as `uint64` +6. Message Bytes as `pointer to unsigned chars` (i.e. bytes) + +Note: The message is passed to the tracepoint in full, however, due to space +limitations in the eBPF kernel VM it might not be possible to pass the message +to user-space in full. Messages longer than a 32kb might be cut off. This can +be detected in tracing scripts by comparing the message size to the length of +the passed message. + +#### Tracepoint `net:outbound_message` + +Is called when a message is send to a peer over the P2P network. Passes +information about our peer, the connection and the message as arguments. + +Arguments passed: +1. Peer ID as `int64` +2. Peer Address and Port (IPv4, IPv6, Tor v3, I2P, ...) as `pointer to C-style String` (max. length 68 characters) +3. Connection Type (inbound, feeler, outbound-full-relay, ...) as `pointer to C-style String` (max. length 20 characters) +4. Message Type (inv, ping, getdata, addrv2, ...) as `pointer to C-style String` (max. length 20 characters) +5. Message Size in bytes as `uint64` +6. Message Bytes as `pointer to unsigned chars` (i.e. bytes) + +Note: The message is passed to the tracepoint in full, however, due to space +limitations in the eBPF kernel VM it might not be possible to pass the message +to user-space in full. Messages longer than a 32kb might be cut off. This can +be detected in tracing scripts by comparing the message size to the length of +the passed message. + +### Context `validation` + +#### Tracepoint `validation:block_connected` + +Is called *after* a block is connected to the chain. Can, for example, be used +to benchmark block connections together with `-reindex`. + +Arguments passed: +1. Block Header Hash as `pointer to unsigned chars` (i.e. 32 bytes in little-endian) +2. Block Height as `int32` +3. Transactions in the Block as `uint64` +4. Inputs spend in the Block as `int32` +5. SigOps in the Block (excluding coinbase SigOps) `uint64` +6. Time it took to connect the Block in microseconds (µs) as `uint64` + +### Context `utxocache` + +The following tracepoints cover the in-memory UTXO cache. UTXOs are, for example, +added to and removed (spent) from the cache when we connect a new block. +**Note**: Bitcoin Core uses temporary clones of the _main_ UTXO cache +(`chainstate.CoinsTip()`). For example, the RPCs `generateblock` and +`getblocktemplate` call `TestBlockValidity()`, which applies the UTXO set +changes to a temporary cache. Similarly, mempool consistency checks, which are +frequent on regtest, also apply the the UTXO set changes to a temporary cache. +Changes to the _main_ UTXO cache and to temporary caches trigger the tracepoints. +We can't tell if a temporary cache or the _main_ cache was changed. + +#### Tracepoint `utxocache:flush` + +Is called *after* the in-memory UTXO cache is flushed. + +Arguments passed: +1. Time it took to flush the cache microseconds as `int64` +2. Flush state mode as `uint32`. It's an enumerator class with values `0` + (`NONE`), `1` (`IF_NEEDED`), `2` (`PERIODIC`), `3` (`ALWAYS`) +3. Cache size (number of coins) before the flush as `uint64` +4. Cache memory usage in bytes as `uint64` +5. If pruning caused the flush as `bool` + +#### Tracepoint `utxocache:add` + +Is called when a coin is added to a UTXO cache. This can be a temporary UTXO cache too. + +Arguments passed: +1. Transaction ID (hash) as `pointer to unsigned chars` (i.e. 32 bytes in little-endian) +2. Output index as `uint32` +3. Block height the coin was added to the UTXO-set as `uint32` +4. Value of the coin as `int64` +5. If the coin is a coinbase as `bool` + +#### Tracepoint `utxocache:spent` + +Is called when a coin is spent from a UTXO cache. This can be a temporary UTXO cache too. + +Arguments passed: +1. Transaction ID (hash) as `pointer to unsigned chars` (i.e. 32 bytes in little-endian) +2. Output index as `uint32` +3. Block height the coin was spent, as `uint32` +4. Value of the coin as `int64` +5. If the coin is a coinbase as `bool` + +#### Tracepoint `utxocache:uncache` + +Is called when a coin is purposefully unloaded from a UTXO cache. This +happens, for example, when we load an UTXO into a cache when trying to accept +a transaction that turns out to be invalid. The loaded UTXO is uncached to avoid +filling our UTXO cache up with irrelevant UTXOs. + +Arguments passed: +1. Transaction ID (hash) as `pointer to unsigned chars` (i.e. 32 bytes in little-endian) +2. Output index as `uint32` +3. Block height the coin was uncached, as `uint32` +4. Value of the coin as `int64` +5. If the coin is a coinbase as `bool` + +## Adding tracepoints to Bitcoin Core + +To add a new tracepoint, `#include ` in the compilation unit where +the tracepoint is inserted. Use one of the `TRACEx` macros listed below +depending on the number of arguments passed to the tracepoint. Up to 12 +arguments can be provided. The `context` and `event` specify the names by which +the tracepoint is referred to. Please use `snake_case` and try to make sure that +the tracepoint names make sense even without detailed knowledge of the +implementation details. Do not forget to update the tracepoint list in this +document. + +```c +#define TRACE(context, event) +#define TRACE1(context, event, a) +#define TRACE2(context, event, a, b) +#define TRACE3(context, event, a, b, c) +#define TRACE4(context, event, a, b, c, d) +#define TRACE5(context, event, a, b, c, d, e) +#define TRACE6(context, event, a, b, c, d, e, f) +#define TRACE7(context, event, a, b, c, d, e, f, g) +#define TRACE8(context, event, a, b, c, d, e, f, g, h) +#define TRACE9(context, event, a, b, c, d, e, f, g, h, i) +#define TRACE10(context, event, a, b, c, d, e, f, g, h, i, j) +#define TRACE11(context, event, a, b, c, d, e, f, g, h, i, j, k) +#define TRACE12(context, event, a, b, c, d, e, f, g, h, i, j, k, l) +``` + +For example: + +```C++ +TRACE6(net, inbound_message, + pnode->GetId(), + pnode->m_addr_name.c_str(), + pnode->ConnectionTypeAsString().c_str(), + sanitizedType.c_str(), + msg.data.size(), + msg.data.data() +); +``` + +### Guidelines and best practices + +#### Clear motivation and use-case +Tracepoints need a clear motivation and use-case. The motivation should +outweigh the impact on, for example, code readability. There is no point in +adding tracepoints that don't end up being used. + +#### Provide an example +When adding a new tracepoint, provide an example. Examples can show the use case +and help reviewers testing that the tracepoint works as intended. The examples +can be kept simple but should give others a starting point when working with +the tracepoint. See existing examples in [contrib/tracing/]. + +[contrib/tracing/]: ../contrib/tracing/ + +#### No expensive computations for tracepoints +Data passed to the tracepoint should be inexpensive to compute. Although the +tracepoint itself only has overhead when enabled, the code to compute arguments +is always run - even if the tracepoint is not used. For example, avoid +serialization and parsing. + +#### Semi-stable API +Tracepoints should have a semi-stable API. Users should be able to rely on the +tracepoints for scripting. This means tracepoints need to be documented, and the +argument order ideally should not change. If there is an important reason to +change argument order, make sure to document the change and update the examples +using the tracepoint. + +#### eBPF Virtual Machine limits +Keep the eBPF Virtual Machine limits in mind. eBPF programs receiving data from +the tracepoints run in a sandboxed Linux kernel VM. This VM has a limited stack +size of 512 bytes. Check if it makes sense to pass larger amounts of data, for +example, with a tracing script that can handle the passed data. + +#### `bpftrace` argument limit +While tracepoints can have up to 12 arguments, bpftrace scripts currently only +support reading from the first six arguments (`arg0` till `arg5`) on `x86_64`. +bpftrace currently lacks real support for handling and printing binary data, +like block header hashes and txids. When a tracepoint passes more than six +arguments, then string and integer arguments should preferably be placed in the +first six argument fields. Binary data can be placed in later arguments. The BCC +supports reading from all 12 arguments. + +#### Strings as C-style String +Generally, strings should be passed into the `TRACEx` macros as pointers to +C-style strings (a null-terminated sequence of characters). For C++ +`std::strings`, [`c_str()`] can be used. It's recommended to document the +maximum expected string size if known. + + +[`c_str()`]: https://www.cplusplus.com/reference/string/string/c_str/ + + +## Listing available tracepoints + +Multiple tools can list the available tracepoints in a `bitcoind` binary with +USDT support. + +### GDB - GNU Project Debugger + +To list probes in Bitcoin Core, use `info probes` in `gdb`: + +``` +$ gdb ./src/bitcoind +… +(gdb) info probes +Type Provider Name Where Semaphore Object +stap net inbound_message 0x000000000014419e /src/bitcoind +stap net outbound_message 0x0000000000107c05 /src/bitcoind +stap validation block_connected 0x00000000002fb10c /src/bitcoind +… +``` + +### With `readelf` + +The `readelf` tool can be used to display the USDT tracepoints in Bitcoin Core. +Look for the notes with the description `NT_STAPSDT`. + +``` +$ readelf -n ./src/bitcoind | grep NT_STAPSDT -A 4 -B 2 +Displaying notes found in: .note.stapsdt + Owner Data size Description + stapsdt 0x0000005d NT_STAPSDT (SystemTap probe descriptors) + Provider: net + Name: outbound_message + Location: 0x0000000000107c05, Base: 0x0000000000579c90, Semaphore: 0x0000000000000000 + Arguments: -8@%r12 8@%rbx 8@%rdi 8@192(%rsp) 8@%rax 8@%rdx +… +``` + +### With `tplist` + +The `tplist` tool is provided by BCC (see [Installing BCC]). It displays kernel +tracepoints or USDT probes and their formats (for more information, see the +[`tplist` usage demonstration]). There are slight binary naming differences +between distributions. For example, on +[Ubuntu the binary is called `tplist-bpfcc`][ubuntu binary]. + +[Installing BCC]: https://github.com/iovisor/bcc/blob/master/INSTALL.md +[`tplist` usage demonstration]: https://github.com/iovisor/bcc/blob/master/tools/tplist_example.txt +[ubuntu binary]: https://github.com/iovisor/bcc/blob/master/INSTALL.md#ubuntu---binary + +``` +$ tplist -l ./src/bitcoind -v +b'net':b'outbound_message' [sema 0x0] + 1 location(s) + 6 argument(s) +… +``` diff --git a/doc/zmq.md b/doc/zmq.md index 85f3370130..b832e71734 100644 --- a/doc/zmq.md +++ b/doc/zmq.md @@ -84,15 +84,18 @@ For instance: $ bitcoind -zmqpubhashtx=tcp://127.0.0.1:28332 \ -zmqpubhashtx=tcp://192.168.1.2:28332 \ + -zmqpubhashblock="tcp://[::1]:28333" \ -zmqpubrawtx=ipc:///tmp/bitcoind.tx.raw \ -zmqpubhashtxhwm=10000 Each PUB notification has a topic and body, where the header corresponds to the notification type. For instance, for the notification `-zmqpubhashtx` the topic is `hashtx` (no null -terminator) and the body is the transaction hash (32 -bytes) for all but `sequence` topic. For `sequence`, the body -is structured as the following based on the type of message: +terminator). These options can also be provided in bitcoin.conf. + +The topics are: + +`sequence`: the body is structured as the following based on the type of message: <32-byte hash>C : Blockhash connected <32-byte hash>D : Blockhash disconnected @@ -101,7 +104,24 @@ is structured as the following based on the type of message: Where the 8-byte uints correspond to the mempool sequence number. -These options can also be provided in bitcoin.conf. +`rawtx`: Notifies about all transactions, both when they are added to mempool or when a new block arrives. This means a transaction could be published multiple times. First, when it enters the mempool and then again in each block that includes it. The messages are ZMQ multipart messages with three parts. The first part is the topic (`rawtx`), the second part is the serialized transaction, and the last part is a sequence number (representing the message count to detect lost messages). + + | rawtx | | + +`hashtx`: Notifies about all transactions, both when they are added to mempool or when a new block arrives. This means a transaction could be published multiple times. First, when it enters the mempool and then again in each block that includes it. The messages are ZMQ multipart messages with three parts. The first part is the topic (`hashtx`), the second part is the 32-byte transaction hash, and the last part is a sequence number (representing the message count to detect lost messages). + + | hashtx | <32-byte transaction hash in Little Endian> | + + +`rawblock`: Notifies when the chain tip is updated. Messages are ZMQ multipart messages with three parts. The first part is the topic (`rawblock`), the second part is the serialized block, and the last part is a sequence number (representing the message count to detect lost messages). + + | rawblock | | + +`hashblock`: Notifies when the chain tip is updated. Messages are ZMQ multipart messages with three parts. The first part is the topic (`hashblock`), the second part is the 32-byte block hash, and the last part is a sequence number (representing the message count to detect lost messages). + + | hashblock | <32-byte block hash in Little Endian> | + +**_NOTE:_** Note that the 32-byte hashes are in Little Endian and not in the Big Endian format that the RPC interface and block explorers use to display transaction and block hashes. ZeroMQ endpoint specifiers for TCP (and others) are documented in the [ZeroMQ API](http://api.zeromq.org/4-0:_start). @@ -125,6 +145,9 @@ Setting the keepalive values appropriately for your operating environment may improve connectivity in situations where long-lived connections are silently dropped by network middle boxes. +Also, the socket's ZMQ_IPV6 option is enabled to accept connections from IPv6 +hosts as well. If needed, this option has to be set on the client side too. + ## Remarks From the perspective of bitcoind, the ZeroMQ socket is write-only; PUB diff --git a/share/examples/bitcoin.conf b/share/examples/bitcoin.conf index 4a947001fa..c5b79709c7 100644 --- a/share/examples/bitcoin.conf +++ b/share/examples/bitcoin.conf @@ -157,7 +157,7 @@ #coinstatsindex=1 # Enable pruning to reduce storage requirements by deleting old blocks. -# This mode is incompatible with -txindex, -coinstatsindex and -rescan. +# This mode is incompatible with -txindex and -coinstatsindex. # 0 = default (no pruning). # 1 = allows manual pruning via RPC. # >=550 = target to stay under in MiB. diff --git a/share/genbuild.sh b/share/genbuild.sh index fe89ae1fde..ecc96160e6 100755 --- a/share/genbuild.sh +++ b/share/genbuild.sh @@ -1,5 +1,5 @@ #!/bin/sh -# Copyright (c) 2012-2020 The Bitcoin Core developers +# Copyright (c) 2012-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -26,7 +26,7 @@ if [ "${BITCOIN_GENBUILD_NO_GIT}" != "1" ] && [ -e "$(command -v git)" ] && [ "$ # if latest commit is tagged and not dirty, then override using the tag name RAWDESC=$(git describe --abbrev=0 2>/dev/null) - if [ "$(git rev-parse HEAD)" = "$(git rev-list -1 $RAWDESC 2>/dev/null)" ]; then + if [ "$(git rev-parse HEAD)" = "$(git rev-list -1 "$RAWDESC" 2>/dev/null)" ]; then git diff-index --quiet HEAD -- && GIT_TAG=$RAWDESC fi diff --git a/share/qt/Info.plist.in b/share/qt/Info.plist.in index 21b34f93b2..3a371f9f5c 100644 --- a/share/qt/Info.plist.in +++ b/share/qt/Info.plist.in @@ -3,7 +3,7 @@ LSMinimumSystemVersion - 10.14.0 + 10.15.0 LSArchitecturePriority @@ -16,6 +16,11 @@ CFBundlePackageType APPL + CFBundleSupportedPlatforms + + MacOSX + + NSHumanReadableCopyright @CLIENT_VERSION_MAJOR@.@CLIENT_VERSION_MINOR@.@CLIENT_VERSION_BUILD@, Copyright © 2009-@COPYRIGHT_YEAR@ @COPYRIGHT_HOLDERS_FINAL@ diff --git a/share/qt/extract_strings_qt.py b/share/qt/extract_strings_qt.py index e4d438fdb0..39acec8942 100755 --- a/share/qt/extract_strings_qt.py +++ b/share/qt/extract_strings_qt.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2012-2019 The Bitcoin Core developers +# Copyright (c) 2012-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. ''' diff --git a/share/rpcauth/rpcauth.py b/share/rpcauth/rpcauth.py index b14c80171e..d441d5f21d 100755 --- a/share/rpcauth/rpcauth.py +++ b/share/rpcauth/rpcauth.py @@ -1,11 +1,10 @@ #!/usr/bin/env python3 -# Copyright (c) 2015-2018 The Bitcoin Core developers +# Copyright (c) 2015-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. from argparse import ArgumentParser from base64 import urlsafe_b64encode -from binascii import hexlify from getpass import getpass from os import urandom @@ -13,7 +12,7 @@ import hmac def generate_salt(size): """Create size byte hex salt""" - return hexlify(urandom(size)).decode() + return urandom(size).hex() def generate_password(): """Create 32 byte b64 password""" diff --git a/share/setup.nsi.in b/share/setup.nsi.in index 86e364e31c..196bb15140 100644 --- a/share/setup.nsi.in +++ b/share/setup.nsi.in @@ -53,7 +53,7 @@ Var StartMenuGroup # Installer attributes InstallDir $PROGRAMFILES64\Elements -CRCCheck on +CRCCheck force XPStyle on BrandingText " " ShowInstDetails show @@ -72,14 +72,14 @@ ShowUninstDetails show Section -Main SEC0000 SetOutPath $INSTDIR SetOverwrite on - File @abs_top_srcdir@/release/@BITCOIN_GUI_NAME@@EXEEXT@ + File @abs_top_builddir@/release/@BITCOIN_GUI_NAME@@EXEEXT@ File /oname=COPYING.txt @abs_top_srcdir@/COPYING File /oname=readme.txt @abs_top_srcdir@/doc/README_windows.txt SetOutPath $INSTDIR\daemon - File @abs_top_srcdir@/release/@BITCOIN_DAEMON_NAME@@EXEEXT@ - File @abs_top_srcdir@/release/@BITCOIN_CLI_NAME@@EXEEXT@ - File @abs_top_srcdir@/release/@BITCOIN_TX_NAME@@EXEEXT@ - File @abs_top_srcdir@/release/@BITCOIN_WALLET_TOOL_NAME@@EXEEXT@ + File @abs_top_builddir@/release/@BITCOIN_DAEMON_NAME@@EXEEXT@ + File @abs_top_builddir@/release/@BITCOIN_CLI_NAME@@EXEEXT@ + File @abs_top_builddir@/release/@BITCOIN_TX_NAME@@EXEEXT@ + File @abs_top_builddir@/release/@BITCOIN_WALLET_TOOL_NAME@@EXEEXT@ SetOutPath $INSTDIR\doc File /r /x Makefile* @abs_top_srcdir@/doc\*.* SetOutPath $INSTDIR diff --git a/src/.clang-format b/src/.clang-format index a69c57f3e0..791b3b8f9f 100644 --- a/src/.clang-format +++ b/src/.clang-format @@ -34,14 +34,6 @@ IndentWidth: 4 KeepEmptyLinesAtTheStartOfBlocks: false MaxEmptyLinesToKeep: 2 NamespaceIndentation: None -ObjCSpaceAfterProperty: false -ObjCSpaceBeforeProtocolList: false -PenaltyBreakBeforeFirstCallParameter: 1 -PenaltyBreakComment: 300 -PenaltyBreakFirstLessLess: 120 -PenaltyBreakString: 1000 -PenaltyExcessCharacter: 1000000 -PenaltyReturnTypeOnItsOwnLine: 200 PointerAlignment: Left SpaceBeforeAssignmentOperators: true SpaceBeforeParens: ControlStatements @@ -51,6 +43,5 @@ SpacesInAngles: false SpacesInContainerLiterals: true SpacesInCStyleCastParentheses: false SpacesInParentheses: false -Standard: Cpp11 -TabWidth: 8 +Standard: c++17 UseTab: Never diff --git a/src/.clang-tidy b/src/.clang-tidy new file mode 100644 index 0000000000..27616ad072 --- /dev/null +++ b/src/.clang-tidy @@ -0,0 +1,2 @@ +Checks: '-*,bugprone-argument-comment' +WarningsAsErrors: bugprone-argument-comment diff --git a/src/Makefile.am b/src/Makefile.am index 311d50348b..7a8e53cc34 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -6,29 +6,18 @@ print-%: FORCE @echo '$*'='$($*)' -DIST_SUBDIRS = secp256k1 univalue +DIST_SUBDIRS = secp256k1 -AM_LDFLAGS = $(LIBTOOL_LDFLAGS) $(HARDENED_LDFLAGS) $(GPROF_LDFLAGS) $(SANITIZER_LDFLAGS) -AM_CXXFLAGS = $(DEBUG_CXXFLAGS) $(HARDENED_CXXFLAGS) $(WARN_CXXFLAGS) $(NOWARN_CXXFLAGS) $(ERROR_CXXFLAGS) $(GPROF_CXXFLAGS) $(SANITIZER_CXXFLAGS) +AM_LDFLAGS = $(LIBTOOL_LDFLAGS) $(HARDENED_LDFLAGS) $(GPROF_LDFLAGS) $(SANITIZER_LDFLAGS) $(LTO_LDFLAGS) +AM_CXXFLAGS = $(DEBUG_CXXFLAGS) $(HARDENED_CXXFLAGS) $(WARN_CXXFLAGS) $(NOWARN_CXXFLAGS) $(ERROR_CXXFLAGS) $(GPROF_CXXFLAGS) $(SANITIZER_CXXFLAGS) $(LTO_CXXFLAGS) AM_CPPFLAGS = $(DEBUG_CPPFLAGS) $(HARDENED_CPPFLAGS) AM_LIBTOOLFLAGS = --preserve-dup-deps PTHREAD_FLAGS = $(PTHREAD_CFLAGS) $(PTHREAD_LIBS) EXTRA_LIBRARIES = -if EMBEDDED_UNIVALUE -LIBUNIVALUE = univalue/libunivalue.la +BITCOIN_INCLUDES=-I$(builddir) -I$(srcdir)/$(MINISKETCH_INCLUDE_DIR_INT) -I$(srcdir)/secp256k1/include -I$(srcdir)/$(UNIVALUE_INCLUDE_DIR_INT) $(BDB_CPPFLAGS) $(BOOST_CPPFLAGS) $(LEVELDB_CPPFLAGS) -$(LIBUNIVALUE): $(wildcard univalue/lib/*) $(wildcard univalue/include/*) - $(AM_V_at)$(MAKE) $(AM_MAKEFLAGS) -C $(@D) $(@F) -else -LIBUNIVALUE = $(UNIVALUE_LIBS) -endif - -BITCOIN_INCLUDES=-I$(builddir) -I$(srcdir)/secp256k1/include $(BDB_CPPFLAGS) $(BOOST_CPPFLAGS) $(LEVELDB_CPPFLAGS) - -BITCOIN_INCLUDES += $(UNIVALUE_CFLAGS) - -LIBBITCOIN_SERVER=libbitcoin_server.a +LIBBITCOIN_NODE=libbitcoin_node.a LIBBITCOIN_COMMON=libbitcoin_common.a LIBBITCOIN_CONSENSUS=libelements_consensus.a LIBBITCOIN_CLI=libbitcoin_cli.a @@ -57,9 +46,13 @@ if ENABLE_AVX2 LIBBITCOIN_CRYPTO_AVX2 = crypto/libbitcoin_crypto_avx2.a LIBBITCOIN_CRYPTO += $(LIBBITCOIN_CRYPTO_AVX2) endif -if ENABLE_SHANI -LIBBITCOIN_CRYPTO_SHANI = crypto/libbitcoin_crypto_shani.a -LIBBITCOIN_CRYPTO += $(LIBBITCOIN_CRYPTO_SHANI) +if ENABLE_X86_SHANI +LIBBITCOIN_CRYPTO_X86_SHANI = crypto/libbitcoin_crypto_x86_shani.a +LIBBITCOIN_CRYPTO += $(LIBBITCOIN_CRYPTO_X86_SHANI) +endif +if ENABLE_ARM_SHANI +LIBBITCOIN_CRYPTO_ARM_SHANI = crypto/libbitcoin_crypto_arm_shani.a +LIBBITCOIN_CRYPTO += $(LIBBITCOIN_CRYPTO_ARM_SHANI) endif $(LIBSECP256K1): $(wildcard secp256k1/src/*.h) $(wildcard secp256k1/src/*.c) $(wildcard secp256k1/include/*) @@ -72,7 +65,7 @@ EXTRA_LIBRARIES += \ $(LIBBITCOIN_UTIL) \ $(LIBBITCOIN_COMMON) \ $(LIBBITCOIN_CONSENSUS) \ - $(LIBBITCOIN_SERVER) \ + $(LIBBITCOIN_NODE) \ $(LIBBITCOIN_CLI) \ $(LIBBITCOIN_IPC) \ $(LIBBITCOIN_WALLET) \ @@ -80,6 +73,7 @@ EXTRA_LIBRARIES += \ $(LIBBITCOIN_ZMQ) lib_LTLIBRARIES = $(LIBBITCOINCONSENSUS) +noinst_LTLIBRARIES = bin_PROGRAMS = noinst_PROGRAMS = @@ -112,6 +106,10 @@ if BUILD_BITCOIN_UTIL bin_PROGRAMS += elements-util endif +if BUILD_BITCOIN_CHAINSTATE + bin_PROGRAMS += bitcoin-chainstate +endif + .PHONY: FORCE check-symbols check-security # bitcoin core # BITCOIN_CORE_H = \ @@ -119,6 +117,7 @@ BITCOIN_CORE_H = \ addrman.h \ asset.h \ assetsdir.h \ + addrman_impl.h \ attributes.h \ banman.h \ base58.h \ @@ -129,7 +128,6 @@ BITCOIN_CORE_H = \ blockencodings.h \ blockfilter.h \ block_proof.h \ - bloom.h \ chain.h \ chainparams.h \ chainparamsbase.h \ @@ -137,6 +135,7 @@ BITCOIN_CORE_H = \ checkqueue.h \ clientversion.h \ coins.h \ + common/bloom.h \ compat.h \ compat/assumptions.h \ compat/byteswap.h \ @@ -185,7 +184,6 @@ BITCOIN_CORE_H = \ mapport.h \ memusage.h \ merkleblock.h \ - miner.h \ net.h \ net_permissions.h \ net_processing.h \ @@ -194,9 +192,13 @@ BITCOIN_CORE_H = \ netbase.h \ netmessagemaker.h \ node/blockstorage.h \ + node/caches.h \ + node/chainstate.h \ node/coin.h \ node/coinstats.h \ node/context.h \ + node/miner.h \ + node/minisketchwrapper.h \ node/psbt.h \ node/transaction.h \ node/ui_interface.h \ @@ -220,12 +222,12 @@ BITCOIN_CORE_H = \ rpc/blockchain.h \ rpc/client.h \ rpc/mining.h \ - rpc/net.h \ rpc/protocol.h \ rpc/rawtransaction_util.h \ rpc/register.h \ rpc/request.h \ rpc/server.h \ + rpc/server_util.h \ rpc/util.h \ scheduler.h \ script/descriptor.h \ @@ -260,6 +262,7 @@ BITCOIN_CORE_H = \ util/check.h \ util/epochguard.h \ util/error.h \ + util/fastrange.h \ util/fees.h \ util/getuniquepath.h \ util/golombrice.h \ @@ -268,6 +271,8 @@ BITCOIN_CORE_H = \ util/macros.h \ util/message.h \ util/moneystr.h \ + util/overflow.h \ + util/overloaded.h \ util/rbf.h \ util/readwritefile.h \ util/serfloat.h \ @@ -275,6 +280,7 @@ BITCOIN_CORE_H = \ util/sock.h \ util/spanparsing.h \ util/string.h \ + util/syscall_sandbox.h \ util/system.h \ util/thread.h \ util/threadnames.h \ @@ -282,6 +288,7 @@ BITCOIN_CORE_H = \ util/tokenpipe.h \ util/trace.h \ util/translation.h \ + util/types.h \ util/ui_change_type.h \ util/url.h \ util/vector.h \ @@ -301,7 +308,8 @@ BITCOIN_CORE_H = \ wallet/ismine.h \ wallet/load.h \ wallet/receive.h \ - wallet/rpcwallet.h \ + wallet/rpc/util.h \ + wallet/rpc/wallet.h \ wallet/salvage.h \ wallet/scriptpubkeyman.h \ wallet/spend.h \ @@ -332,9 +340,9 @@ ipc/capnp/libbitcoin_ipc_a-ipc.$(OBJEXT): $(libbitcoin_ipc_mpgen_input:=.h) # Contains code accessing mempool and chain state that is meant to be separated # from wallet and gui code (see node/README.md). Shared code should go in # libbitcoin_common or libbitcoin_util libraries, instead. -libbitcoin_server_a_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) $(MINIUPNPC_CPPFLAGS) $(NATPMP_CPPFLAGS) $(EVENT_CFLAGS) $(EVENT_PTHREADS_CFLAGS) -libbitcoin_server_a_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -libbitcoin_server_a_SOURCES = \ +libbitcoin_node_a_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) $(MINIUPNPC_CPPFLAGS) $(NATPMP_CPPFLAGS) $(EVENT_CFLAGS) $(EVENT_PTHREADS_CFLAGS) +libbitcoin_node_a_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) +libbitcoin_node_a_SOURCES = \ addrdb.cpp \ addrman.cpp \ banman.cpp \ @@ -358,14 +366,17 @@ libbitcoin_server_a_SOURCES = \ init.cpp \ mainchainrpc.cpp \ mapport.cpp \ - miner.cpp \ net.cpp \ net_processing.cpp \ node/blockstorage.cpp \ + node/caches.cpp \ + node/chainstate.cpp \ node/coin.cpp \ node/coinstats.cpp \ node/context.cpp \ node/interfaces.cpp \ + node/miner.cpp \ + node/minisketchwrapper.cpp \ node/psbt.cpp \ node/transaction.cpp \ node/ui_interface.cpp \ @@ -384,6 +395,7 @@ libbitcoin_server_a_SOURCES = \ rpc/net.cpp \ rpc/rawtransaction.cpp \ rpc/server.cpp \ + rpc/server_util.cpp \ script/sigcache.cpp \ shutdown.cpp \ signet.cpp \ @@ -399,10 +411,10 @@ libbitcoin_server_a_SOURCES = \ $(BITCOIN_CORE_H) if ENABLE_WALLET -libbitcoin_server_a_SOURCES += wallet/init.cpp +libbitcoin_node_a_SOURCES += wallet/init.cpp endif if !ENABLE_WALLET -libbitcoin_server_a_SOURCES += dummywallet.cpp +libbitcoin_node_a_SOURCES += dummywallet.cpp endif if ENABLE_ZMQ @@ -433,8 +445,16 @@ libbitcoin_wallet_a_SOURCES = \ wallet/interfaces.cpp \ wallet/load.cpp \ wallet/receive.cpp \ - wallet/rpcdump.cpp \ - wallet/rpcwallet.cpp \ + wallet/rpc/addresses.cpp \ + wallet/rpc/backup.cpp \ + wallet/rpc/coins.cpp \ + wallet/rpc/elements.cpp \ + wallet/rpc/encrypt.cpp \ + wallet/rpc/spend.cpp \ + wallet/rpc/signmessage.cpp \ + wallet/rpc/transactions.cpp \ + wallet/rpc/util.cpp \ + wallet/rpc/wallet.cpp \ wallet/scriptpubkeyman.cpp \ wallet/spend.cpp \ wallet/transaction.cpp \ @@ -445,7 +465,7 @@ libbitcoin_wallet_a_SOURCES = \ primitives/bitcoin/merkleblock.cpp \ primitives/bitcoin/block.cpp \ $(libbitcoin_common_a_SOURCES) \ - $(libbitcoin_server_a_SOURCES) \ + $(libbitcoin_node_a_SOURCES) \ $(BITCOIN_CORE_H) if USE_SQLITE @@ -511,20 +531,26 @@ crypto_libbitcoin_crypto_avx2_a_CXXFLAGS += $(AVX2_CXXFLAGS) crypto_libbitcoin_crypto_avx2_a_CPPFLAGS += -DENABLE_AVX2 crypto_libbitcoin_crypto_avx2_a_SOURCES = crypto/sha256_avx2.cpp -crypto_libbitcoin_crypto_shani_a_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -crypto_libbitcoin_crypto_shani_a_CPPFLAGS = $(AM_CPPFLAGS) -crypto_libbitcoin_crypto_shani_a_CXXFLAGS += $(SHANI_CXXFLAGS) -crypto_libbitcoin_crypto_shani_a_CPPFLAGS += -DENABLE_SHANI -crypto_libbitcoin_crypto_shani_a_SOURCES = crypto/sha256_shani.cpp +crypto_libbitcoin_crypto_x86_shani_a_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) +crypto_libbitcoin_crypto_x86_shani_a_CPPFLAGS = $(AM_CPPFLAGS) +crypto_libbitcoin_crypto_x86_shani_a_CXXFLAGS += $(X86_SHANI_CXXFLAGS) +crypto_libbitcoin_crypto_x86_shani_a_CPPFLAGS += -DENABLE_X86_SHANI +crypto_libbitcoin_crypto_x86_shani_a_SOURCES = crypto/sha256_x86_shani.cpp + +crypto_libbitcoin_crypto_arm_shani_a_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) +crypto_libbitcoin_crypto_arm_shani_a_CPPFLAGS = $(AM_CPPFLAGS) +crypto_libbitcoin_crypto_arm_shani_a_CXXFLAGS += $(ARM_SHANI_CXXFLAGS) +crypto_libbitcoin_crypto_arm_shani_a_CPPFLAGS += -DENABLE_ARM_SHANI +crypto_libbitcoin_crypto_arm_shani_a_SOURCES = crypto/sha256_arm_shani.cpp # consensus: shared between all executables that validate any consensus rules. libelements_consensus_a_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) libelements_consensus_a_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) libelements_consensus_a_SOURCES = \ - amount.h \ arith_uint256.cpp \ arith_uint256.h \ asset.cpp \ + consensus/amount.h \ consensus/merkle.cpp \ consensus/merkle.h \ consensus/params.h \ @@ -576,9 +602,9 @@ libbitcoin_common_a_SOURCES = \ blind.cpp \ blindpsbt.cpp \ block_proof.cpp \ - bloom.cpp \ chainparams.cpp \ coins.cpp \ + common/bloom.cpp \ compressor.cpp \ core_read.cpp \ core_write.cpp \ @@ -589,6 +615,7 @@ libbitcoin_common_a_SOURCES = \ key.cpp \ key_io.cpp \ merkleblock.cpp \ + net_types.cpp \ netaddress.cpp \ netbase.cpp \ net_permissions.cpp \ @@ -611,7 +638,7 @@ libbitcoin_common_a_SOURCES = \ # util: shared between all executables. # This library *must* be included to make sure that the glibc -# backward-compatibility objects and their sanity checks are linked. +# sanity checks are linked. libbitcoin_util_a_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) libbitcoin_util_a_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) libbitcoin_util_a_SOURCES = \ @@ -651,6 +678,7 @@ libbitcoin_util_a_SOURCES = \ util/spanparsing.cpp \ util/strencodings.cpp \ util/string.cpp \ + util/syscall_sandbox.cpp \ util/time.cpp \ util/tokenpipe.cpp \ $(BITCOIN_CORE_H) @@ -659,11 +687,6 @@ if USE_LIBEVENT libbitcoin_util_a_SOURCES += util/url.cpp endif -if GLIBC_BACK_COMPAT -libbitcoin_util_a_SOURCES += compat/glibc_compat.cpp -AM_LDFLAGS += $(COMPAT_LDFLAGS) -endif - # cli: shared between bitcoin-cli and bitcoin-qt libbitcoin_cli_a_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) libbitcoin_cli_a_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) @@ -699,19 +722,19 @@ elements_bin_ldadd = \ $(LIBMEMENV) \ $(LIBSECP256K1) -elements_bin_ldadd += $(BOOST_LIBS) $(BDB_LIBS) $(MINIUPNPC_LIBS) $(NATPMP_LIBS) $(EVENT_PTHREADS_LIBS) $(EVENT_LIBS) $(ZMQ_LIBS) $(SQLITE_LIBS) +elements_bin_ldadd += $(BDB_LIBS) $(MINIUPNPC_LIBS) $(NATPMP_LIBS) $(EVENT_PTHREADS_LIBS) $(EVENT_LIBS) $(ZMQ_LIBS) $(SQLITE_LIBS) elementsd_SOURCES = $(elements_daemon_sources) init/bitcoind.cpp elementsd_CPPFLAGS = $(elements_bin_cppflags) elementsd_CXXFLAGS = $(elements_bin_cxxflags) elementsd_LDFLAGS = $(elements_bin_ldflags) -elementsd_LDADD = $(LIBBITCOIN_SERVER) $(elements_bin_ldadd) +elementsd_LDADD = $(LIBBITCOIN_NODE) $(elements_bin_ldadd) elements_node_SOURCES = $(elements_daemon_sources) init/bitcoin-node.cpp elements_node_CPPFLAGS = $(elements_bin_cppflags) elements_node_CXXFLAGS = $(elements_bin_cxxflags) elements_node_LDFLAGS = $(elements_bin_ldflags) -elements_node_LDADD = $(LIBBITCOIN_SERVER) $(elements_bin_ldadd) $(LIBBITCOIN_IPC) $(LIBMULTIPROCESS_LIBS) +elements_node_LDADD = $(LIBBITCOIN_NODE) $(elements_bin_ldadd) $(LIBBITCOIN_IPC) $(LIBMULTIPROCESS_LIBS) # bitcoin-cli binary # elements_cli_SOURCES = bitcoin-cli.cpp @@ -729,7 +752,7 @@ elements_cli_LDADD = \ $(LIBBITCOIN_UTIL) \ $(LIBBITCOIN_CRYPTO) -elements_cli_LDADD += $(BOOST_LIBS) $(EVENT_LIBS) +elements_cli_LDADD += $(EVENT_LIBS) # # bitcoin-tx binary # @@ -749,16 +772,29 @@ elements_tx_LDADD = \ $(LIBBITCOIN_CONSENSUS) \ $(LIBBITCOIN_CRYPTO) \ $(LIBSECP256K1) - -elements_tx_LDADD += $(BOOST_LIBS) # # bitcoin-wallet binary # elements_wallet_SOURCES = bitcoin-wallet.cpp +elements_wallet_SOURCES += init/bitcoin-wallet.cpp elements_wallet_CPPFLAGS = $(elements_bin_cppflags) elements_wallet_CXXFLAGS = $(elements_bin_cxxflags) elements_wallet_LDFLAGS = $(elements_bin_ldflags) -elements_wallet_LDADD = $(LIBBITCOIN_WALLET_TOOL) $(elements_bin_ldadd) +elements_wallet_LDADD = \ + $(LIBBITCOIN_WALLET_TOOL) \ + $(LIBBITCOIN_WALLET) \ + $(LIBBITCOIN_COMMON) \ + $(LIBBITCOIN_UTIL) \ + $(LIBUNIVALUE) \ + $(LIBBITCOIN_CONSENSUS) \ + $(LIBBITCOIN_CRYPTO) \ + $(LIBSECP256K1) \ + $(BDB_LIBS) \ + $(EVENT_LIBS) \ + $(LIBLEVELDB) \ + $(LIBLEVELDB_SSE42) \ + $(LIBMEMENV) \ + $(SQLITE_LIBS) if TARGET_WINDOWS elements_wallet_SOURCES += bitcoin-wallet-res.rc @@ -782,8 +818,132 @@ elements_util_LDADD = \ $(LIBBITCOIN_CONSENSUS) \ $(LIBBITCOIN_CRYPTO) \ $(LIBSECP256K1) +# -elements_util_LDADD += $(BOOST_LIBS) +# bitcoin-chainstate binary # +bitcoin_chainstate_SOURCES = \ + bitcoin-chainstate.cpp \ + arith_uint256.cpp \ + blockfilter.cpp \ + chain.cpp \ + chainparamsbase.cpp \ + chainparams.cpp \ + clientversion.cpp \ + coins.cpp \ + compat/glibcxx_sanity.cpp \ + compressor.cpp \ + consensus/merkle.cpp \ + consensus/tx_check.cpp \ + consensus/tx_verify.cpp \ + core_read.cpp \ + dbwrapper.cpp \ + deploymentinfo.cpp \ + deploymentstatus.cpp \ + flatfile.cpp \ + fs.cpp \ + hash.cpp \ + index/base.cpp \ + index/blockfilterindex.cpp \ + index/coinstatsindex.cpp \ + init/common.cpp \ + key.cpp \ + logging.cpp \ + netaddress.cpp \ + node/blockstorage.cpp \ + node/chainstate.cpp \ + node/coinstats.cpp \ + node/ui_interface.cpp \ + policy/feerate.cpp \ + policy/fees.cpp \ + policy/packages.cpp \ + policy/policy.cpp \ + policy/rbf.cpp \ + policy/settings.cpp \ + pow.cpp \ + primitives/block.cpp \ + primitives/transaction.cpp \ + pubkey.cpp \ + random.cpp \ + randomenv.cpp \ + scheduler.cpp \ + script/interpreter.cpp \ + script/script.cpp \ + script/script_error.cpp \ + script/sigcache.cpp \ + script/standard.cpp \ + shutdown.cpp \ + signet.cpp \ + support/cleanse.cpp \ + support/lockedpool.cpp \ + sync.cpp \ + threadinterrupt.cpp \ + timedata.cpp \ + txdb.cpp \ + txmempool.cpp \ + uint256.cpp \ + util/asmap.cpp \ + util/bytevectorhash.cpp \ + util/getuniquepath.cpp \ + util/hasher.cpp \ + util/moneystr.cpp \ + util/rbf.cpp \ + util/serfloat.cpp \ + util/settings.cpp \ + util/strencodings.cpp \ + util/syscall_sandbox.cpp \ + util/system.cpp \ + util/thread.cpp \ + util/threadnames.cpp \ + util/time.cpp \ + util/tokenpipe.cpp \ + validation.cpp \ + validationinterface.cpp \ + versionbits.cpp \ + warnings.cpp +# ELEMENTS: additional sources for chainstate +bitcoin_chainstate_SOURCES += \ + asset.cpp \ + blind.cpp \ + blindpsbt.cpp \ + block_proof.cpp \ + common/bloom.cpp \ + confidential_validation.cpp \ + dynafed.cpp \ + issuance.cpp \ + mainchainrpc.cpp \ + merkleblock.cpp \ + pegins.cpp \ + primitives/bitcoin/block.cpp \ + primitives/bitcoin/merkleblock.cpp \ + primitives/bitcoin/transaction.cpp \ + primitives/confidential.cpp \ + primitives/pak.cpp \ + primitives/txwitness.cpp \ + psbt.cpp \ + rpc/request.cpp \ + script/pegins.cpp \ + script/sign.cpp \ + script/signingprovider.cpp \ + support/events.h \ + util/url.cpp + +bitcoin_chainstate_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) +bitcoin_chainstate_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) +bitcoin_chainstate_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) $(PTHREAD_FLAGS) +bitcoin_chainstate_LDADD = \ + $(LIBBITCOIN_CRYPTO) \ + $(LIBUNIVALUE) \ + $(LIBSECP256K1) \ + $(LIBLEVELDB) \ + $(LIBLEVELDB_SSE42) \ + $(LIBMEMENV) +# ELEMENTS: +bitcoin_chainstate_LDADD += \ + $(EVENT_LIBS) + +# Required for obj/build.h to be generated first. +# More details: https://www.gnu.org/software/automake/manual/html_node/Built-Sources-Example.html +bitcoin_chainstate-clientversion.$(OBJEXT): obj/build.h # # bitcoinconsensus library # @@ -791,10 +951,6 @@ if BUILD_BITCOIN_LIBS include_HEADERS = script/bitcoinconsensus.h libelementsconsensus_la_SOURCES = support/cleanse.cpp $(crypto_libbitcoin_crypto_base_a_SOURCES) $(libelements_consensus_a_SOURCES) -if GLIBC_BACK_COMPAT - libelementsconsensus_la_SOURCES += compat/glibc_compat.cpp -endif - libelementsconsensus_la_LDFLAGS = $(AM_LDFLAGS) -no-undefined $(RELDFLAGS) libelementsconsensus_la_LIBADD = $(LIBSECP256K1) libelementsconsensus_la_CPPFLAGS = $(AM_CPPFLAGS) -I$(builddir)/obj -I$(srcdir)/secp256k1/include -DBUILD_BITCOIN_INTERNAL @@ -843,7 +999,6 @@ $(top_srcdir)/$(subdir)/config/bitcoin-config.h.in: $(am__configure_deps) clean-local: -$(MAKE) -C secp256k1 clean - -$(MAKE) -C univalue clean -rm -f leveldb/*/*.gcda leveldb/*/*.gcno leveldb/helpers/memenv/*.gcda leveldb/helpers/memenv/*.gcno -rm -f config.h -rm -rf test/__pycache__ @@ -854,20 +1009,8 @@ clean-local: $(AM_V_GEN) $(WINDRES) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(CPPFLAGS) -DWINDRES_PREPROC -i $< -o $@ check-symbols: $(bin_PROGRAMS) -if TARGET_DARWIN - @echo "Checking macOS dynamic libraries..." + @echo "Running symbol and dynamic library checks..." $(AM_V_at) $(PYTHON) $(top_srcdir)/contrib/devtools/symbol-check.py $(bin_PROGRAMS) -endif - -if TARGET_WINDOWS - @echo "Checking Windows dynamic libraries..." - $(AM_V_at) $(PYTHON) $(top_srcdir)/contrib/devtools/symbol-check.py $(bin_PROGRAMS) -endif - -if TARGET_LINUX - @echo "Checking glibc back compat..." - $(AM_V_at) CPPFILT='$(CPPFILT)' $(PYTHON) $(top_srcdir)/contrib/devtools/symbol-check.py $(bin_PROGRAMS) -endif check-security: $(bin_PROGRAMS) if HARDEN @@ -884,9 +1027,11 @@ EXTRA_DIST += $(libbitcoin_ipc_mpgen_input) if BUILD_MULTIPROCESS LIBBITCOIN_IPC=libbitcoin_ipc.a libbitcoin_ipc_a_SOURCES = \ + ipc/capnp/context.h \ ipc/capnp/init-types.h \ ipc/capnp/protocol.cpp \ ipc/capnp/protocol.h \ + ipc/context.h \ ipc/exception.h \ ipc/interfaces.cpp \ ipc/process.cpp \ @@ -908,10 +1053,10 @@ nodist_libbitcoin_ipc_a_SOURCES = $(libbitcoin_ipc_mpgen_output) CLEANFILES += $(libbitcoin_ipc_mpgen_output) endif -if EMBEDDED_LEVELDB +include Makefile.minisketch.include + include Makefile.crc32c.include include Makefile.leveldb.include -endif include Makefile.test_util.include include Makefile.test_fuzz.include @@ -931,3 +1076,5 @@ endif if ENABLE_QT_TESTS include Makefile.qttest.include endif + +include Makefile.univalue.include diff --git a/src/Makefile.bench.include b/src/Makefile.bench.include index 2a8e4a0aac..0bcce6ebe1 100644 --- a/src/Makefile.bench.include +++ b/src/Makefile.bench.include @@ -51,7 +51,7 @@ nodist_bench_bench_bitcoin_SOURCES = $(GENERATED_BENCH_FILES) bench_bench_bitcoin_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) $(EVENT_CFLAGS) $(EVENT_PTHREADS_CFLAGS) -I$(builddir)/bench/ bench_bench_bitcoin_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) bench_bench_bitcoin_LDADD = \ - $(LIBBITCOIN_SERVER) \ + $(LIBBITCOIN_NODE) \ $(LIBBITCOIN_WALLET) \ $(LIBBITCOIN_COMMON) \ $(LIBBITCOIN_UTIL) \ @@ -75,7 +75,7 @@ bench_bench_bitcoin_SOURCES += bench/coin_selection.cpp bench_bench_bitcoin_SOURCES += bench/wallet_balance.cpp endif -bench_bench_bitcoin_LDADD += $(BOOST_LIBS) $(BDB_LIBS) $(EVENT_PTHREADS_LIBS) $(EVENT_LIBS) $(MINIUPNPC_LIBS) $(NATPMP_LIBS) $(SQLITE_LIBS) +bench_bench_bitcoin_LDADD += $(BDB_LIBS) $(EVENT_PTHREADS_LIBS) $(EVENT_LIBS) $(MINIUPNPC_LIBS) $(NATPMP_LIBS) $(SQLITE_LIBS) bench_bench_bitcoin_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) $(PTHREAD_FLAGS) CLEAN_BITCOIN_BENCH = bench/*.gcda bench/*.gcno $(GENERATED_BENCH_FILES) diff --git a/src/Makefile.crc32c.include b/src/Makefile.crc32c.include index 113272e65e..3cbe71792c 100644 --- a/src/Makefile.crc32c.include +++ b/src/Makefile.crc32c.include @@ -14,7 +14,6 @@ CRC32C_CPPFLAGS_INT += -I$(srcdir)/crc32c/include CRC32C_CPPFLAGS_INT += -DHAVE_BUILTIN_PREFETCH=@HAVE_BUILTIN_PREFETCH@ CRC32C_CPPFLAGS_INT += -DHAVE_MM_PREFETCH=@HAVE_MM_PREFETCH@ CRC32C_CPPFLAGS_INT += -DHAVE_STRONG_GETAUXVAL=@HAVE_STRONG_GETAUXVAL@ -CRC32C_CPPFLAGS_INT += -DHAVE_WEAK_GETAUXVAL=@HAVE_WEAK_GETAUXVAL@ CRC32C_CPPFLAGS_INT += -DCRC32C_TESTS_BUILT_WITH_GLOG=0 if ENABLE_SSE42 diff --git a/src/Makefile.leveldb.include b/src/Makefile.leveldb.include index ce1f93f11f..3bec92482a 100644 --- a/src/Makefile.leveldb.include +++ b/src/Makefile.leveldb.include @@ -8,9 +8,10 @@ LIBMEMENV_INT = leveldb/libmemenv.a EXTRA_LIBRARIES += $(LIBLEVELDB_INT) EXTRA_LIBRARIES += $(LIBMEMENV_INT) -LIBLEVELDB += $(LIBLEVELDB_INT) $(LIBCRC32C) -LIBMEMENV += $(LIBMEMENV_INT) +LIBLEVELDB = $(LIBLEVELDB_INT) $(LIBCRC32C) +LIBMEMENV = $(LIBMEMENV_INT) +LEVELDB_CPPFLAGS = LEVELDB_CPPFLAGS += -I$(srcdir)/leveldb/include LEVELDB_CPPFLAGS += -I$(srcdir)/leveldb/helpers/memenv diff --git a/src/Makefile.minisketch.include b/src/Makefile.minisketch.include new file mode 100644 index 0000000000..b337f48349 --- /dev/null +++ b/src/Makefile.minisketch.include @@ -0,0 +1,43 @@ +include minisketch/sources.mk + +LIBMINISKETCH_CPPFLAGS= +LIBMINISKETCH_CPPFLAGS += -DDISABLE_DEFAULT_FIELDS -DENABLE_FIELD_32 + +LIBMINISKETCH = minisketch/libminisketch.a +MINISKETCH_LIBS = $(LIBMINISKETCH) + +if ENABLE_CLMUL +LIBMINISKETCH_CLMUL = minisketch/libminisketch_clmul.a +LIBMINISKETCH_CPPFLAGS += -DHAVE_CLMUL +MINISKETCH_LIBS += $(LIBMINISKETCH_CLMUL) +endif + +if HAVE_CLZ +LIBMINISKETCH_CPPFLAGS += -DHAVE_CLZ +endif + +EXTRA_LIBRARIES += $(MINISKETCH_LIBS) + +minisketch_libminisketch_clmul_a_SOURCES = $(MINISKETCH_FIELD_CLMUL_SOURCES_INT) $(MINISKETCH_FIELD_CLMUL_HEADERS_INT) +minisketch_libminisketch_clmul_a_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) $(CLMUL_CXXFLAGS) +minisketch_libminisketch_clmul_a_CPPFLAGS = $(AM_CPPFLAGS) $(LIBMINISKETCH_CPPFLAGS) + +minisketch_libminisketch_a_SOURCES = $(MINISKETCH_FIELD_GENERIC_SOURCES_INT) $(MINISKETCH_LIB_SOURCES_INT) +minisketch_libminisketch_a_SOURCES += $(MINISKETCH_FIELD_GENERIC_HEADERS_INT) $(MINISKETCH_LIB_HEADERS_INT) $(MINISKETCH_DIST_HEADERS_INT) +minisketch_libminisketch_a_CPPFLAGS = $(AM_CPPFLAGS) $(LIBMINISKETCH_CPPFLAGS) +minisketch_libminisketch_a_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) + +if ENABLE_TESTS +if !ENABLE_FUZZ +MINISKETCH_TEST = minisketch/test +TESTS += $(MINISKETCH_TEST) +noinst_PROGRAMS += $(MINISKETCH_TEST) + +minisketch_test_SOURCES = $(MINISKETCH_TEST_SOURCES_INT) +minisketch_test_CPPFLAGS = $(AM_CPPFLAGS) $(LIBMINISKETCH_CPPFLAGS) +minisketch_test_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) +minisketch_test_LDADD = $(MINISKETCH_LIBS) +minisketch_test_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) $(PTHREAD_FLAGS) + +endif +endif diff --git a/src/Makefile.qt.include b/src/Makefile.qt.include index af36d2f3ca..4487ff13a0 100644 --- a/src/Makefile.qt.include +++ b/src/Makefile.qt.include @@ -40,9 +40,9 @@ QT_MOC_CPP = \ qt/moc_askpassphrasedialog.cpp \ qt/moc_createwalletdialog.cpp \ qt/moc_bantablemodel.cpp \ + qt/moc_bitcoin.cpp \ qt/moc_bitcoinaddressvalidator.cpp \ qt/moc_bitcoinamountfield.cpp \ - qt/moc_bitcoin.cpp \ qt/moc_bitcoingui.cpp \ qt/moc_bitcoinunits.cpp \ qt/moc_clientmodel.cpp \ @@ -51,6 +51,7 @@ QT_MOC_CPP = \ qt/moc_csvmodelwriter.cpp \ qt/moc_editaddressdialog.cpp \ qt/moc_guiutil.cpp \ + qt/moc_initexecutor.cpp \ qt/moc_intro.cpp \ qt/moc_macdockiconhandler.cpp \ qt/moc_macnotificationhandler.cpp \ @@ -109,9 +110,9 @@ BITCOIN_QT_H = \ qt/addresstablemodel.h \ qt/askpassphrasedialog.h \ qt/bantablemodel.h \ + qt/bitcoin.h \ qt/bitcoinaddressvalidator.h \ qt/bitcoinamountfield.h \ - qt/bitcoin.h \ qt/bitcoingui.h \ qt/bitcoinunits.h \ qt/clientmodel.h \ @@ -122,6 +123,7 @@ BITCOIN_QT_H = \ qt/editaddressdialog.h \ qt/guiconstants.h \ qt/guiutil.h \ + qt/initexecutor.h \ qt/intro.h \ qt/macdockiconhandler.h \ qt/macnotificationhandler.h \ @@ -165,10 +167,10 @@ BITCOIN_QT_H = \ qt/walletview.h \ qt/winshutdownmonitor.h -RES_FONTS = \ +QT_RES_FONTS = \ qt/res/fonts/RobotoMono-Bold.ttf -RES_ICONS = \ +QT_RES_ICONS = \ qt/res/icons/add.png \ qt/res/icons/address-book.png \ qt/res/icons/bitcoin.ico \ @@ -226,6 +228,7 @@ BITCOIN_QT_BASE_CPP = \ qt/clientmodel.cpp \ qt/csvmodelwriter.cpp \ qt/guiutil.cpp \ + qt/initexecutor.cpp \ qt/intro.cpp \ qt/modaloverlay.cpp \ qt/networkstyle.cpp \ @@ -282,9 +285,9 @@ if ENABLE_WALLET BITCOIN_QT_CPP += $(BITCOIN_QT_WALLET_CPP) endif # ENABLE_WALLET -RES_ANIMATION = $(wildcard $(srcdir)/qt/res/animation/spinner-*.png) +QT_RES_ANIMATION = $(wildcard $(srcdir)/qt/res/animation/spinner-*.png) -BITCOIN_RC = qt/res/bitcoin-qt-res.rc +BITCOIN_QT_RC = qt/res/bitcoin-qt-res.rc BITCOIN_QT_INCLUDES = -DQT_NO_KEYWORDS -DQT_USE_QSTRINGBUILDER @@ -294,7 +297,7 @@ qt_libbitcoinqt_a_CXXFLAGS = $(AM_CXXFLAGS) $(QT_PIE_FLAGS) qt_libbitcoinqt_a_OBJCXXFLAGS = $(AM_OBJCXXFLAGS) $(QT_PIE_FLAGS) qt_libbitcoinqt_a_SOURCES = $(BITCOIN_QT_CPP) $(BITCOIN_QT_H) $(QT_FORMS_UI) \ - $(QT_QRC) $(QT_QRC_LOCALE) $(QT_TS) $(RES_FONTS) $(RES_ICONS) $(RES_ANIMATION) + $(QT_QRC) $(QT_QRC_LOCALE) $(QT_TS) $(QT_RES_FONTS) $(QT_RES_ICONS) $(QT_RES_ANIMATION) if TARGET_DARWIN qt_libbitcoinqt_a_SOURCES += $(BITCOIN_MM) endif @@ -316,9 +319,9 @@ elements_qt_cxxflags = $(AM_CXXFLAGS) $(QT_PIE_FLAGS) elements_qt_sources = qt/main.cpp if TARGET_WINDOWS - elements_qt_sources += $(BITCOIN_RC) + elements_qt_sources += $(BITCOIN_QT_RC) endif -elements_qt_ldadd = qt/libbitcoinqt.a $(LIBBITCOIN_SERVER) +elements_qt_ldadd = qt/libbitcoinqt.a $(LIBBITCOIN_NODE) if ENABLE_WALLET elements_qt_ldadd += $(LIBBITCOIN_UTIL) $(LIBBITCOIN_WALLET) endif @@ -326,22 +329,22 @@ if ENABLE_ZMQ elements_qt_ldadd += $(LIBBITCOIN_ZMQ) $(ZMQ_LIBS) endif elements_qt_ldadd += $(LIBBITCOIN_CLI) $(LIBBITCOIN_COMMON) $(LIBBITCOIN_UTIL) $(LIBBITCOIN_CONSENSUS) $(LIBBITCOIN_CRYPTO) $(LIBUNIVALUE) $(LIBLEVELDB) $(LIBLEVELDB_SSE42) $(LIBMEMENV) \ - $(BOOST_LIBS) $(QT_LIBS) $(QT_DBUS_LIBS) $(QR_LIBS) $(BDB_LIBS) $(MINIUPNPC_LIBS) $(NATPMP_LIBS) $(LIBSECP256K1) \ + $(QT_LIBS) $(QT_DBUS_LIBS) $(QR_LIBS) $(BDB_LIBS) $(MINIUPNPC_LIBS) $(NATPMP_LIBS) $(LIBSECP256K1) \ $(EVENT_PTHREADS_LIBS) $(EVENT_LIBS) $(SQLITE_LIBS) elements_qt_ldflags = $(RELDFLAGS) $(AM_LDFLAGS) $(QT_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) $(PTHREAD_FLAGS) elements_qt_libtoolflags = $(AM_LIBTOOLFLAGS) --tag CXX qt_elements_qt_CPPFLAGS = $(elements_qt_cppflags) qt_elements_qt_CXXFLAGS = $(elements_qt_cxxflags) -qt_elements_qt_SOURCES = $(elements_qt_sources) +qt_elements_qt_SOURCES = $(elements_qt_sources) init/bitcoin-qt.cpp qt_elements_qt_LDADD = $(elements_qt_ldadd) qt_elements_qt_LDFLAGS = $(elements_qt_ldflags) qt_elements_qt_LIBTOOLFLAGS = $(elements_qt_libtoolflags) elements_gui_CPPFLAGS = $(elements_qt_cppflags) elements_gui_CXXFLAGS = $(elements_qt_cxxflags) -elements_gui_SOURCES = $(elements_qt_sources) -elements_gui_LDADD = $(elements_qt_ldadd) +elements_gui_SOURCES = $(elements_qt_sources) init/bitcoin-gui.cpp +elements_gui_LDADD = $(elements_qt_ldadd) $(LIBBITCOIN_IPC) $(LIBBITCOIN_UTIL) $(LIBMULTIPROCESS_LIBS) elements_gui_LDFLAGS = $(elements_qt_ldflags) elements_gui_LIBTOOLFLAGS = $(elements_qt_libtoolflags) @@ -352,13 +355,17 @@ SECONDARY: $(QT_QM) $(srcdir)/qt/bitcoinstrings.cpp: FORCE @test -n $(XGETTEXT) || echo "xgettext is required for updating translations" - $(AM_V_GEN) cd $(srcdir); XGETTEXT=$(XGETTEXT) COPYRIGHT_HOLDERS="$(COPYRIGHT_HOLDERS)" $(PYTHON) ../share/qt/extract_strings_qt.py $(libbitcoin_server_a_SOURCES) $(libbitcoin_wallet_a_SOURCES) $(libbitcoin_common_a_SOURCES) $(libbitcoin_zmq_a_SOURCES) $(libbitcoin_consensus_a_SOURCES) $(libbitcoin_util_a_SOURCES) + $(AM_V_GEN) cd $(srcdir); XGETTEXT=$(XGETTEXT) COPYRIGHT_HOLDERS="$(COPYRIGHT_HOLDERS)" $(PYTHON) ../share/qt/extract_strings_qt.py $(libbitcoin_node_a_SOURCES) $(libbitcoin_wallet_a_SOURCES) $(libbitcoin_common_a_SOURCES) $(libbitcoin_zmq_a_SOURCES) $(libbitcoin_consensus_a_SOURCES) $(libbitcoin_util_a_SOURCES) +# The resulted bitcoin_en.xlf source file should follow Transifex requirements. +# See: https://docs.transifex.com/formats/xliff#how-to-distinguish-between-a-source-file-and-a-translation-file translate: $(srcdir)/qt/bitcoinstrings.cpp $(QT_FORMS_UI) $(QT_FORMS_UI) $(BITCOIN_QT_BASE_CPP) qt/bitcoin.cpp $(BITCOIN_QT_WINDOWS_CPP) $(BITCOIN_QT_WALLET_CPP) $(BITCOIN_QT_H) $(BITCOIN_MM) @test -n $(LUPDATE) || echo "lupdate is required for updating translations" $(AM_V_GEN) QT_SELECT=$(QT_SELECT) $(LUPDATE) -no-obsolete -I $(srcdir) -locations relative $^ -ts $(srcdir)/qt/locale/bitcoin_en.ts @test -n $(LCONVERT) || echo "lconvert is required for updating translations" - $(AM_V_GEN) QT_SELECT=$(QT_SELECT) $(LCONVERT) -o $(srcdir)/qt/locale/bitcoin_en.xlf -i $(srcdir)/qt/locale/bitcoin_en.ts + $(AM_V_GEN) QT_SELECT=$(QT_SELECT) $(LCONVERT) -drop-translations -o $(srcdir)/qt/locale/bitcoin_en.xlf -i $(srcdir)/qt/locale/bitcoin_en.ts + @$(SED) -i.old -e 's|source-language="en" target-language="en"|source-language="en"|' -e '/<\/target>/d' $(srcdir)/qt/locale/bitcoin_en.xlf + @rm -f $(srcdir)/qt/locale/bitcoin_en.xlf.old $(QT_QRC_LOCALE_CPP): $(QT_QRC_LOCALE) $(QT_QM) @test -f $(RCC) @@ -369,7 +376,7 @@ $(QT_QRC_LOCALE_CPP): $(QT_QRC_LOCALE) $(QT_QM) $(QT_QRC_BUILD): $(QT_QRC) $(SED) -e 's:\(res\/\(icons\|movies\)/\):$(abs_srcdir)/qt/\1:' <"$<" >"$@" -$(QT_QRC_CPP): $(QT_QRC) $(QT_FORMS_H) $(RES_FONTS) $(RES_ICONS) $(RES_ANIMATION) +$(QT_QRC_CPP): $(QT_QRC) $(QT_FORMS_H) $(QT_RES_FONTS) $(QT_RES_ICONS) $(QT_RES_ANIMATION) @test -f $(RCC) $(AM_V_GEN) QT_SELECT=$(QT_SELECT) $(RCC) -name bitcoin --format-version 1 $< > $@ @@ -389,11 +396,10 @@ QT_BASE_TLD = $(shell tar tf $(QT_BASE_PATH) --exclude='*/*') elements_qt_apk: FORCE mkdir -p $(APK_LIB_DIR) - cp $(dir $(CC))../sysroot/usr/lib/$(host_alias)/libc++_shared.so $(APK_LIB_DIR) + cp $(dir $(lastword $(CC)))../sysroot/usr/lib/$(host_alias)/libc++_shared.so $(APK_LIB_DIR) tar xf $(QT_BASE_PATH) -C qt/android/src/ $(QT_BASE_TLD)src/android/jar/src --strip-components=5 tar xf $(QT_BASE_PATH) -C qt/android/src/ $(QT_BASE_TLD)src/android/java/src --strip-components=5 - tar xf $(QT_BASE_PATH) -C qt/android/res/ $(QT_BASE_TLD)src/android/java/res --strip-components=5 - cp qt/bitcoin-qt $(APK_LIB_DIR)/libbitcoin-qt.so + cp qt/bitcoin-qt $(APK_LIB_DIR)/libbitcoin-qt_$(ANDROID_ARCH).so cd qt/android && gradle wrapper --gradle-version=6.6.1 cd qt/android && ./gradlew build diff --git a/src/Makefile.qt_locale.include b/src/Makefile.qt_locale.include index aea42fd902..8638bd024c 100644 --- a/src/Makefile.qt_locale.include +++ b/src/Makefile.qt_locale.include @@ -1,7 +1,7 @@ QT_TS = \ - qt/locale/bitcoin_af.ts \ qt/locale/bitcoin_am.ts \ qt/locale/bitcoin_ar.ts \ + qt/locale/bitcoin_az.ts \ qt/locale/bitcoin_be.ts \ qt/locale/bitcoin_bg.ts \ qt/locale/bitcoin_bn.ts \ @@ -13,7 +13,6 @@ QT_TS = \ qt/locale/bitcoin_de.ts \ qt/locale/bitcoin_el.ts \ qt/locale/bitcoin_en.ts \ - qt/locale/bitcoin_en_GB.ts \ qt/locale/bitcoin_eo.ts \ qt/locale/bitcoin_es.ts \ qt/locale/bitcoin_es_CL.ts \ @@ -27,9 +26,12 @@ QT_TS = \ qt/locale/bitcoin_fi.ts \ qt/locale/bitcoin_fil.ts \ qt/locale/bitcoin_fr.ts \ + qt/locale/bitcoin_ga.ts \ + qt/locale/bitcoin_gd.ts \ + qt/locale/bitcoin_gl.ts \ qt/locale/bitcoin_gl_ES.ts \ + qt/locale/bitcoin_gu.ts \ qt/locale/bitcoin_he.ts \ - qt/locale/bitcoin_hi.ts \ qt/locale/bitcoin_hr.ts \ qt/locale/bitcoin_hu.ts \ qt/locale/bitcoin_id.ts \ @@ -38,6 +40,7 @@ QT_TS = \ qt/locale/bitcoin_ja.ts \ qt/locale/bitcoin_ka.ts \ qt/locale/bitcoin_kk.ts \ + qt/locale/bitcoin_kl.ts \ qt/locale/bitcoin_km.ts \ qt/locale/bitcoin_ko.ts \ qt/locale/bitcoin_ku_IQ.ts \ @@ -54,12 +57,14 @@ QT_TS = \ qt/locale/bitcoin_nb.ts \ qt/locale/bitcoin_ne.ts \ qt/locale/bitcoin_nl.ts \ + qt/locale/bitcoin_no.ts \ qt/locale/bitcoin_pam.ts \ qt/locale/bitcoin_pl.ts \ qt/locale/bitcoin_pt.ts \ qt/locale/bitcoin_pt_BR.ts \ qt/locale/bitcoin_ro.ts \ qt/locale/bitcoin_ru.ts \ + qt/locale/bitcoin_sc.ts \ qt/locale/bitcoin_si.ts \ qt/locale/bitcoin_sk.ts \ qt/locale/bitcoin_sl.ts \ @@ -68,11 +73,13 @@ QT_TS = \ qt/locale/bitcoin_sr.ts \ qt/locale/bitcoin_sr@latin.ts \ qt/locale/bitcoin_sv.ts \ + qt/locale/bitcoin_sw.ts \ qt/locale/bitcoin_szl.ts \ qt/locale/bitcoin_ta.ts \ qt/locale/bitcoin_te.ts \ qt/locale/bitcoin_th.ts \ qt/locale/bitcoin_tr.ts \ + qt/locale/bitcoin_ug.ts \ qt/locale/bitcoin_uk.ts \ qt/locale/bitcoin_ur.ts \ qt/locale/bitcoin_uz@Cyrl.ts \ diff --git a/src/Makefile.qttest.include b/src/Makefile.qttest.include index c51a4d562b..7ddb4f36e4 100644 --- a/src/Makefile.qttest.include +++ b/src/Makefile.qttest.include @@ -28,6 +28,7 @@ qt_test_test_elements_qt_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) $(BITCOIN $(QT_INCLUDES) $(QT_TEST_INCLUDES) qt_test_test_elements_qt_SOURCES = \ + init/bitcoin-qt.cpp \ qt/test/apptests.cpp \ qt/test/rpcnestedtests.cpp \ qt/test/test_main.cpp \ @@ -43,7 +44,7 @@ endif # ENABLE_WALLET nodist_qt_test_test_elements_qt_SOURCES = $(TEST_QT_MOC_CPP) -qt_test_test_elements_qt_LDADD = $(LIBBITCOINQT) $(LIBBITCOIN_SERVER) $(LIBTEST_UTIL) +qt_test_test_elements_qt_LDADD = $(LIBBITCOINQT) $(LIBBITCOIN_NODE) $(LIBTEST_UTIL) if ENABLE_WALLET qt_test_test_elements_qt_LDADD += $(LIBBITCOIN_UTIL) $(LIBBITCOIN_WALLET) endif @@ -51,7 +52,7 @@ if ENABLE_ZMQ qt_test_test_elements_qt_LDADD += $(LIBBITCOIN_ZMQ) $(ZMQ_LIBS) endif qt_test_test_elements_qt_LDADD += $(LIBBITCOIN_CLI) $(LIBBITCOIN_COMMON) $(LIBBITCOIN_UTIL) $(LIBBITCOIN_CONSENSUS) $(LIBBITCOIN_CRYPTO) $(LIBUNIVALUE) $(LIBLEVELDB) \ - $(LIBLEVELDB_SSE42) $(LIBMEMENV) $(BOOST_LIBS) $(QT_DBUS_LIBS) $(QT_TEST_LIBS) $(QT_LIBS) \ + $(LIBLEVELDB_SSE42) $(LIBMEMENV) $(QT_LIBS) $(QT_DBUS_LIBS) $(QT_TEST_LIBS) \ $(QR_LIBS) $(BDB_LIBS) $(MINIUPNPC_LIBS) $(NATPMP_LIBS) $(LIBSECP256K1) \ $(EVENT_PTHREADS_LIBS) $(EVENT_LIBS) $(SQLITE_LIBS) qt_test_test_elements_qt_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(QT_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) $(PTHREAD_FLAGS) diff --git a/src/Makefile.test.include b/src/Makefile.test.include index e8b3efd81a..9dd760525a 100644 --- a/src/Makefile.test.include +++ b/src/Makefile.test.include @@ -16,6 +16,7 @@ FUZZ_BINARY=test/fuzz/fuzz$(EXEEXT) JSON_TEST_FILES = \ test/data/script_tests.json \ + test/data/bip341_wallet_vectors.json \ test/data/base58_encode_decode.json \ test/data/blockfilters.json \ test/data/key_io_valid.json \ @@ -37,7 +38,7 @@ BITCOIN_TEST_SUITE = \ FUZZ_SUITE_LD_COMMON = \ $(LIBTEST_UTIL) \ $(LIBTEST_FUZZ) \ - $(LIBBITCOIN_SERVER) \ + $(LIBBITCOIN_NODE) \ $(LIBBITCOIN_WALLET) \ $(LIBBITCOIN_COMMON) \ $(LIBBITCOIN_UTIL) \ @@ -47,9 +48,9 @@ FUZZ_SUITE_LD_COMMON = \ $(LIBUNIVALUE) \ $(LIBLEVELDB) \ $(LIBLEVELDB_SSE42) \ - $(BOOST_LIBS) \ $(LIBMEMENV) \ $(LIBSECP256K1) \ + $(MINISKETCH_LIBS) \ $(EVENT_LIBS) \ $(EVENT_PTHREADS_LIBS) @@ -64,11 +65,11 @@ endif # test_bitcoin binary # BITCOIN_TESTS =\ test/pegin_witness_tests.cpp \ - test/arith_uint256_tests.cpp \ - test/scriptnum10.h \ test/addrman_tests.cpp \ - test/amount_tests.cpp \ test/allocator_tests.cpp \ + test/amount_tests.cpp \ + test/arith_uint256_tests.cpp \ + test/banman_tests.cpp \ test/base32_tests.cpp \ test/base58_tests.cpp \ test/base64_tests.cpp \ @@ -77,8 +78,8 @@ BITCOIN_TESTS =\ test/bip32_tests.cpp \ test/blockchain_tests.cpp \ test/blockencodings_tests.cpp \ - test/blockfilter_tests.cpp \ test/blockfilter_index_tests.cpp \ + test/blockfilter_tests.cpp \ test/bloom_tests.cpp \ test/bswap_tests.cpp \ test/checkqueue_tests.cpp \ @@ -88,6 +89,7 @@ BITCOIN_TESTS =\ test/compress_tests.cpp \ test/crypto_tests.cpp \ test/cuckoocache_tests.cpp \ + test/dbwrapper_tests.cpp \ test/denialofservice_tests.cpp \ test/descriptor_tests.cpp \ test/flatfile_tests.cpp \ @@ -99,12 +101,11 @@ BITCOIN_TESTS =\ test/key_io_tests.cpp \ test/key_tests.cpp \ test/logging_tests.cpp \ - test/dbwrapper_tests.cpp \ - test/validation_tests.cpp \ test/mempool_tests.cpp \ test/merkle_tests.cpp \ test/merkleblock_tests.cpp \ test/miner_tests.cpp \ + test/minisketch_tests.cpp \ test/multisig_tests.cpp \ test/net_peer_eviction_tests.cpp \ test/net_tests.cpp \ @@ -121,8 +122,10 @@ BITCOIN_TESTS =\ test/sanity_tests.cpp \ test/scheduler_tests.cpp \ test/script_p2sh_tests.cpp \ - test/script_tests.cpp \ + test/script_parse_tests.cpp \ test/script_standard_tests.cpp \ + test/script_tests.cpp \ + test/scriptnum10.h \ test/scriptnum_tests.cpp \ test/serfloat_tests.cpp \ test/serialize_tests.cpp \ @@ -134,20 +137,22 @@ BITCOIN_TESTS =\ test/streams_tests.cpp \ test/sync_tests.cpp \ test/system_tests.cpp \ - test/util_threadnames_tests.cpp \ test/timedata_tests.cpp \ test/torcontrol_tests.cpp \ test/transaction_tests.cpp \ test/txindex_tests.cpp \ + test/txpackage_tests.cpp \ test/txrequest_tests.cpp \ test/txvalidation_tests.cpp \ test/txvalidationcache_tests.cpp \ test/uint256_tests.cpp \ test/util_tests.cpp \ + test/util_threadnames_tests.cpp \ test/validation_block_tests.cpp \ test/validation_chainstate_tests.cpp \ test/validation_chainstatemanager_tests.cpp \ test/validation_flush_tests.cpp \ + test/validation_tests.cpp \ test/validationinterface_tests.cpp \ test/versionbits_tests.cpp \ test/pegin_spent_tests.cpp \ @@ -157,9 +162,11 @@ BITCOIN_TESTS =\ if ENABLE_WALLET BITCOIN_TESTS += \ + wallet/test/spend_tests.cpp \ wallet/test/wallet_tests.cpp \ wallet/test/walletdb_tests.cpp \ wallet/test/wallet_crypto_tests.cpp \ + wallet/test/wallet_transaction_tests.cpp \ wallet/test/coinselector_tests.cpp \ wallet/test/init_tests.cpp \ wallet/test/ismine_tests.cpp \ @@ -175,13 +182,19 @@ if USE_BDB BITCOIN_TESTS += wallet/test/db_tests.cpp endif +if USE_SQLITE +FUZZ_WALLET_SRC = \ + wallet/test/fuzz/notifications.cpp +endif # USE_SQLITE BITCOIN_TEST_SUITE += \ + wallet/test/util.cpp \ + wallet/test/util.h \ wallet/test/wallet_test_fixture.cpp \ wallet/test/wallet_test_fixture.h \ wallet/test/init_test_fixture.cpp \ wallet/test/init_test_fixture.h -endif +endif # ENABLE_WALLET test_test_bitcoin_SOURCES = $(BITCOIN_TEST_SUITE) $(BITCOIN_TESTS) $(JSON_TEST_FILES) $(RAW_TEST_FILES) test_test_bitcoin_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) $(TESTDEFS) $(EVENT_CFLAGS) @@ -190,8 +203,8 @@ if ENABLE_WALLET test_test_bitcoin_LDADD += $(LIBBITCOIN_WALLET) endif -test_test_bitcoin_LDADD += $(LIBBITCOIN_SERVER) $(LIBBITCOIN_CLI) $(LIBBITCOIN_COMMON) $(LIBBITCOIN_UTIL) $(LIBBITCOIN_CONSENSUS) $(LIBBITCOIN_CRYPTO) $(LIBUNIVALUE) \ - $(LIBLEVELDB) $(LIBLEVELDB_SSE42) $(LIBMEMENV) $(BOOST_LIBS) $(BOOST_UNIT_TEST_FRAMEWORK_LIB) $(LIBSECP256K1) $(EVENT_LIBS) $(EVENT_PTHREADS_LIBS) +test_test_bitcoin_LDADD += $(LIBBITCOIN_NODE) $(LIBBITCOIN_CLI) $(LIBBITCOIN_COMMON) $(LIBBITCOIN_UTIL) $(LIBBITCOIN_CONSENSUS) $(LIBBITCOIN_CRYPTO) $(LIBUNIVALUE) \ + $(LIBLEVELDB) $(LIBLEVELDB_SSE42) $(LIBMEMENV) $(LIBSECP256K1) $(EVENT_LIBS) $(EVENT_PTHREADS_LIBS) $(MINISKETCH_LIBS) test_test_bitcoin_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_test_bitcoin_LDADD += $(BDB_LIBS) $(MINIUPNPC_LIBS) $(NATPMP_LIBS) $(SQLITE_LIBS) @@ -202,16 +215,14 @@ test_test_bitcoin_LDADD += $(LIBBITCOIN_ZMQ) $(ZMQ_LIBS) FUZZ_SUITE_LD_COMMON += $(LIBBITCOIN_ZMQ) $(ZMQ_LIBS) endif -FUZZ_SUITE_LDFLAGS_COMMON = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) $(PTHREAD_FLAGS) - if ENABLE_FUZZ_BINARY test_fuzz_fuzz_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) test_fuzz_fuzz_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_fuzz_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_fuzz_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) +test_fuzz_fuzz_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) $(PTHREAD_FLAGS) $(RUNTIME_LDFLAGS) test_fuzz_fuzz_SOURCES = \ + $(FUZZ_WALLET_SRC) \ test/fuzz/addition_overflow.cpp \ - test/fuzz/addrdb.cpp \ test/fuzz/addrman.cpp \ test/fuzz/asmap.cpp \ test/fuzz/asmap_direct.cpp \ @@ -234,10 +245,10 @@ test_fuzz_fuzz_SOURCES = \ test/fuzz/crypto_chacha20.cpp \ test/fuzz/crypto_chacha20_poly1305_aead.cpp \ test/fuzz/crypto_common.cpp \ + test/fuzz/crypto_diff_fuzz_chacha20.cpp \ test/fuzz/crypto_hkdf_hmac_sha256_l32.cpp \ test/fuzz/crypto_poly1305.cpp \ test/fuzz/cuckoocache.cpp \ - test/fuzz/data_stream.cpp \ test/fuzz/decode_tx.cpp \ test/fuzz/descriptor_parse.cpp \ test/fuzz/deserialize.cpp \ @@ -258,6 +269,7 @@ test_fuzz_fuzz_SOURCES = \ test/fuzz/locale.cpp \ test/fuzz/merkleblock.cpp \ test/fuzz/message.cpp \ + test/fuzz/minisketch.cpp \ test/fuzz/muhash.cpp \ test/fuzz/multiplication_overflow.cpp \ test/fuzz/net.cpp \ @@ -289,6 +301,7 @@ test_fuzz_fuzz_SOURCES = \ test/fuzz/script_bitcoin_consensus.cpp \ test/fuzz/script_descriptor_cache.cpp \ test/fuzz/script_flags.cpp \ + test/fuzz/script_format.cpp \ test/fuzz/script_interpreter.cpp \ test/fuzz/script_ops.cpp \ test/fuzz/script_sigcache.cpp \ @@ -343,8 +356,8 @@ bitcoin_test_clean : FORCE check-local: $(BITCOIN_TESTS:.cpp=.cpp.test) if BUILD_BITCOIN_TX - @echo "Running test/util/bitcoin-util-test.py..." - $(PYTHON) $(top_builddir)/test/util/bitcoin-util-test.py + @echo "Running test/util/test_runner.py..." + $(PYTHON) $(top_builddir)/test/util/test_runner.py endif @echo "Running test/util/rpcauth-test.py..." $(PYTHON) $(top_builddir)/test/util/rpcauth-test.py @@ -356,8 +369,26 @@ if ENABLE_BENCH endif endif $(AM_V_at)$(MAKE) $(AM_MAKEFLAGS) -C secp256k1 check -if EMBEDDED_UNIVALUE - $(AM_V_at)$(MAKE) $(AM_MAKEFLAGS) -C univalue check + +if !ENABLE_FUZZ +UNIVALUE_TESTS = univalue/test/object univalue/test/unitester univalue/test/no_nul +noinst_PROGRAMS += $(UNIVALUE_TESTS) +TESTS += $(UNIVALUE_TESTS) + +univalue_test_unitester_SOURCES = $(UNIVALUE_TEST_UNITESTER_INT) +univalue_test_unitester_LDADD = $(LIBUNIVALUE) +univalue_test_unitester_CPPFLAGS = -I$(srcdir)/$(UNIVALUE_INCLUDE_DIR_INT) -DJSON_TEST_SRC=\"$(srcdir)/$(UNIVALUE_TEST_DATA_DIR_INT)\" +univalue_test_unitester_LDFLAGS = -static $(LIBTOOL_APP_LDFLAGS) + +univalue_test_no_nul_SOURCES = $(UNIVALUE_TEST_NO_NUL_INT) +univalue_test_no_nul_LDADD = $(LIBUNIVALUE) +univalue_test_no_nul_CPPFLAGS = -I$(srcdir)/$(UNIVALUE_INCLUDE_DIR_INT) +univalue_test_no_nul_LDFLAGS = -static $(LIBTOOL_APP_LDFLAGS) + +univalue_test_object_SOURCES = $(UNIVALUE_TEST_OBJECT_INT) +univalue_test_object_LDADD = $(LIBUNIVALUE) +univalue_test_object_CPPFLAGS = -I$(srcdir)/$(UNIVALUE_INCLUDE_DIR_INT) +univalue_test_object_LDFLAGS = -static $(LIBTOOL_APP_LDFLAGS) endif %.cpp.test: %.cpp diff --git a/src/Makefile.test_fuzz.include b/src/Makefile.test_fuzz.include index 2d772f2fca..9574454fd2 100644 --- a/src/Makefile.test_fuzz.include +++ b/src/Makefile.test_fuzz.include @@ -19,7 +19,7 @@ libtest_fuzz_a_SOURCES = \ test/fuzz/util.cpp \ $(TEST_FUZZ_H) -LIBTEST_FUZZ += $(LIBBITCOIN_SERVER) +LIBTEST_FUZZ += $(LIBBITCOIN_NODE) LIBTEST_FUZZ += $(LIBBITCOIN_COMMON) LIBTEST_FUZZ += $(LIBBITCOIN_UTIL) LIBTEST_FUZZ += $(LIBBITCOIN_CRYPTO_BASE) diff --git a/src/Makefile.test_util.include b/src/Makefile.test_util.include index 85e50ebf70..92cb8a5ce6 100644 --- a/src/Makefile.test_util.include +++ b/src/Makefile.test_util.include @@ -9,6 +9,7 @@ EXTRA_LIBRARIES += \ TEST_UTIL_H = \ test/util/blockfilter.h \ + test/util/chainstate.h \ test/util/logging.h \ test/util/mining.h \ test/util/net.h \ @@ -34,7 +35,7 @@ libtest_util_a_SOURCES = \ test/util/wallet.cpp \ $(TEST_UTIL_H) -LIBTEST_UTIL += $(LIBBITCOIN_SERVER) +LIBTEST_UTIL += $(LIBBITCOIN_NODE) LIBTEST_UTIL += $(LIBBITCOIN_COMMON) LIBTEST_UTIL += $(LIBBITCOIN_UTIL) LIBTEST_UTIL += $(LIBBITCOIN_CRYPTO_BASE) diff --git a/src/Makefile.univalue.include b/src/Makefile.univalue.include new file mode 100644 index 0000000000..3644e36368 --- /dev/null +++ b/src/Makefile.univalue.include @@ -0,0 +1,6 @@ +include univalue/sources.mk + +LIBUNIVALUE = libunivalue.la +noinst_LTLIBRARIES += $(LIBUNIVALUE) +libunivalue_la_SOURCES = $(UNIVALUE_LIB_SOURCES_INT) $(UNIVALUE_DIST_HEADERS_INT) $(UNIVALUE_LIB_HEADERS_INT) $(UNIVALUE_TEST_FILES_INT) +libunivalue_la_CPPFLAGS = $(AM_CPPFLAGS) -I$(srcdir)/$(UNIVALUE_INCLUDE_DIR_INT) diff --git a/src/addrdb.cpp b/src/addrdb.cpp index b8fd019bab..0fa8f3c3da 100644 --- a/src/addrdb.cpp +++ b/src/addrdb.cpp @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2020 The Bitcoin Core developers +// Copyright (c) 2009-2021 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -9,6 +9,7 @@ #include #include #include +#include #include #include #include @@ -18,64 +19,14 @@ #include #include #include - -CBanEntry::CBanEntry(const UniValue& json) - : nVersion(json["version"].get_int()), nCreateTime(json["ban_created"].get_int64()), - nBanUntil(json["banned_until"].get_int64()) -{ -} - -UniValue CBanEntry::ToJson() const -{ - UniValue json(UniValue::VOBJ); - json.pushKV("version", nVersion); - json.pushKV("ban_created", nCreateTime); - json.pushKV("banned_until", nBanUntil); - return json; -} +#include namespace { -static const char* BANMAN_JSON_ADDR_KEY = "address"; - -/** - * Convert a `banmap_t` object to a JSON array. - * @param[in] bans Bans list to convert. - * @return a JSON array, similar to the one returned by the `listbanned` RPC. Suitable for - * passing to `BanMapFromJson()`. - */ -UniValue BanMapToJson(const banmap_t& bans) +class DbNotFoundError : public std::exception { - UniValue bans_json(UniValue::VARR); - for (const auto& it : bans) { - const auto& address = it.first; - const auto& ban_entry = it.second; - UniValue j = ban_entry.ToJson(); - j.pushKV(BANMAN_JSON_ADDR_KEY, address.ToString()); - bans_json.push_back(j); - } - return bans_json; -} - -/** - * Convert a JSON array to a `banmap_t` object. - * @param[in] bans_json JSON to convert, must be as returned by `BanMapToJson()`. - * @param[out] bans Bans list to create from the JSON. - * @throws std::runtime_error if the JSON does not have the expected fields or they contain - * unparsable values. - */ -void BanMapFromJson(const UniValue& bans_json, banmap_t& bans) -{ - for (const auto& ban_entry_json : bans_json.getValues()) { - CSubNet subnet; - const auto& subnet_str = ban_entry_json[BANMAN_JSON_ADDR_KEY].get_str(); - if (!LookupSubNet(subnet_str, subnet)) { - throw std::runtime_error( - strprintf("Cannot parse banned address or subnet: %s", subnet_str)); - } - bans.insert_or_assign(subnet, CBanEntry{ban_entry_json}); - } -} + using std::exception::exception; +}; template bool SerializeDB(Stream& stream, const Data& data) @@ -108,7 +59,7 @@ bool SerializeFileDB(const std::string& prefix, const fs::path& path, const Data if (fileout.IsNull()) { fileout.fclose(); remove(pathTmp); - return error("%s: Failed to open file %s", __func__, pathTmp.string()); + return error("%s: Failed to open file %s", __func__, fs::PathToString(pathTmp)); } // Serialize @@ -120,7 +71,7 @@ bool SerializeFileDB(const std::string& prefix, const fs::path& path, const Data if (!FileCommit(fileout.Get())) { fileout.fclose(); remove(pathTmp); - return error("%s: Failed to flush file %s", __func__, pathTmp.string()); + return error("%s: Failed to flush file %s", __func__, fs::PathToString(pathTmp)); } fileout.fclose(); @@ -134,53 +85,46 @@ bool SerializeFileDB(const std::string& prefix, const fs::path& path, const Data } template -bool DeserializeDB(Stream& stream, Data& data, bool fCheckSum = true) +void DeserializeDB(Stream& stream, Data& data, bool fCheckSum = true) { - try { - CHashVerifier verifier(&stream); - // de-serialize file header (network specific magic number) and .. - unsigned char pchMsgTmp[4]; - verifier >> pchMsgTmp; - // ... verify the network matches ours - if (memcmp(pchMsgTmp, Params().MessageStart(), sizeof(pchMsgTmp))) - return error("%s: Invalid network magic number", __func__); + CHashVerifier verifier(&stream); + // de-serialize file header (network specific magic number) and .. + unsigned char pchMsgTmp[4]; + verifier >> pchMsgTmp; + // ... verify the network matches ours + if (memcmp(pchMsgTmp, Params().MessageStart(), sizeof(pchMsgTmp))) { + throw std::runtime_error{"Invalid network magic number"}; + } - // de-serialize data - verifier >> data; + // de-serialize data + verifier >> data; - // verify checksum - if (fCheckSum) { - uint256 hashTmp; - stream >> hashTmp; - if (hashTmp != verifier.GetHash()) { - return error("%s: Checksum mismatch, data corrupted", __func__); - } + // verify checksum + if (fCheckSum) { + uint256 hashTmp; + stream >> hashTmp; + if (hashTmp != verifier.GetHash()) { + throw std::runtime_error{"Checksum mismatch, data corrupted"}; } } - catch (const std::exception& e) { - return error("%s: Deserialize or I/O error - %s", __func__, e.what()); - } - - return true; } template -bool DeserializeFileDB(const fs::path& path, Data& data, int version) +void DeserializeFileDB(const fs::path& path, Data& data, int version) { // open input file, and associate with CAutoFile FILE* file = fsbridge::fopen(path, "rb"); CAutoFile filein(file, SER_DISK, version); if (filein.IsNull()) { - LogPrintf("Missing or invalid file %s\n", path.string()); - return false; + throw DbNotFoundError{}; } - return DeserializeDB(filein, data); + DeserializeDB(filein, data); } } // namespace CBanDB::CBanDB(fs::path ban_list_path) - : m_banlist_dat(ban_list_path.string() + ".dat"), - m_banlist_json(ban_list_path.string() + ".json") + : m_banlist_dat(ban_list_path + ".dat"), + m_banlist_json(ban_list_path + ".json") { } @@ -197,23 +141,22 @@ bool CBanDB::Write(const banmap_t& banSet) return false; } -bool CBanDB::Read(banmap_t& banSet, bool& dirty) +bool CBanDB::Read(banmap_t& banSet) { - // If the JSON banlist does not exist, then try to read the non-upgraded banlist.dat. - if (!fs::exists(m_banlist_json)) { - // If this succeeds then we need to flush to disk in order to create the JSON banlist. - dirty = true; - return DeserializeFileDB(m_banlist_dat, banSet, CLIENT_VERSION); + if (fs::exists(m_banlist_dat)) { + LogPrintf("banlist.dat ignored because it can only be read by " PACKAGE_NAME " version 22.x. Remove %s to silence this warning.\n", fs::quoted(fs::PathToString(m_banlist_dat))); + } + // If the JSON banlist does not exist, then recreate it + if (!fs::exists(m_banlist_json)) { + return false; } - - dirty = false; std::map settings; std::vector errors; if (!util::ReadSettings(m_banlist_json, settings, errors)) { for (const auto& err : errors) { - LogPrintf("Cannot load banlist %s: %s\n", m_banlist_json.string(), err); + LogPrintf("Cannot load banlist %s: %s\n", fs::PathToString(m_banlist_json), err); } return false; } @@ -221,36 +164,54 @@ bool CBanDB::Read(banmap_t& banSet, bool& dirty) try { BanMapFromJson(settings[JSON_KEY], banSet); } catch (const std::runtime_error& e) { - LogPrintf("Cannot parse banlist %s: %s\n", m_banlist_json.string(), e.what()); + LogPrintf("Cannot parse banlist %s: %s\n", fs::PathToString(m_banlist_json), e.what()); return false; } return true; } -CAddrDB::CAddrDB() -{ - pathAddr = gArgs.GetDataDirNet() / "peers.dat"; -} - -bool CAddrDB::Write(const CAddrMan& addr) +bool DumpPeerAddresses(const ArgsManager& args, const AddrMan& addr) { + const auto pathAddr = args.GetDataDirNet() / "peers.dat"; return SerializeFileDB("peers", pathAddr, addr, CLIENT_VERSION); } -bool CAddrDB::Read(CAddrMan& addr) +void ReadFromStream(AddrMan& addr, CDataStream& ssPeers) { - return DeserializeFileDB(pathAddr, addr, CLIENT_VERSION); + DeserializeDB(ssPeers, addr, false); } -bool CAddrDB::Read(CAddrMan& addr, CDataStream& ssPeers) +std::optional LoadAddrman(const std::vector& asmap, const ArgsManager& args, std::unique_ptr& addrman) { - bool ret = DeserializeDB(ssPeers, addr, false); - if (!ret) { - // Ensure addrman is left in a clean state - addr.Clear(); + auto check_addrman = std::clamp(args.GetIntArg("-checkaddrman", DEFAULT_ADDRMAN_CONSISTENCY_CHECKS), 0, 1000000); + addrman = std::make_unique(asmap, /* deterministic */ false, /* consistency_check_ratio */ check_addrman); + + int64_t nStart = GetTimeMillis(); + const auto path_addr{args.GetDataDirNet() / "peers.dat"}; + try { + DeserializeFileDB(path_addr, *addrman, CLIENT_VERSION); + LogPrintf("Loaded %i addresses from peers.dat %dms\n", addrman->size(), GetTimeMillis() - nStart); + } catch (const DbNotFoundError&) { + // Addrman can be in an inconsistent state after failure, reset it + addrman = std::make_unique(asmap, /* deterministic */ false, /* consistency_check_ratio */ check_addrman); + LogPrintf("Creating peers.dat because the file was not found (%s)\n", fs::quoted(fs::PathToString(path_addr))); + DumpPeerAddresses(args, *addrman); + } catch (const InvalidAddrManVersionError&) { + if (!RenameOver(path_addr, (fs::path)path_addr + ".bak")) { + addrman = nullptr; + return strprintf(_("Failed to rename invalid peers.dat file. Please move or delete it and try again.")); + } + // Addrman can be in an inconsistent state after failure, reset it + addrman = std::make_unique(asmap, /* deterministic */ false, /* consistency_check_ratio */ check_addrman); + LogPrintf("Creating new peers.dat because the file version was not compatible (%s). Original backed up to peers.dat.bak\n", fs::quoted(fs::PathToString(path_addr))); + DumpPeerAddresses(args, *addrman); + } catch (const std::exception& e) { + addrman = nullptr; + return strprintf(_("Invalid or corrupt peers.dat (%s). If you believe this is a bug, please report it to %s. As a workaround, you can move the file (%s) out of the way (rename, move, or delete) to have a new one created on the next start."), + e.what(), PACKAGE_BUGREPORT, fs::quoted(fs::PathToString(path_addr))); } - return ret; + return std::nullopt; } void DumpAnchors(const fs::path& anchors_db_path, const std::vector& anchors) @@ -262,9 +223,10 @@ void DumpAnchors(const fs::path& anchors_db_path, const std::vector& a std::vector ReadAnchors(const fs::path& anchors_db_path) { std::vector anchors; - if (DeserializeFileDB(anchors_db_path, anchors, CLIENT_VERSION | ADDRV2_FORMAT)) { - LogPrintf("Loaded %i addresses from %s\n", anchors.size(), anchors_db_path.filename()); - } else { + try { + DeserializeFileDB(anchors_db_path, anchors, CLIENT_VERSION | ADDRV2_FORMAT); + LogPrintf("Loaded %i addresses from %s\n", anchors.size(), fs::quoted(fs::PathToString(anchors_db_path.filename()))); + } catch (const std::exception&) { anchors.clear(); } diff --git a/src/addrdb.h b/src/addrdb.h index 399103c991..f568cb05ab 100644 --- a/src/addrdb.h +++ b/src/addrdb.h @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2020 The Bitcoin Core developers +// Copyright (c) 2009-2021 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -8,75 +8,23 @@ #include #include // For banmap_t -#include #include -#include +#include #include +#include +class ArgsManager; +class AddrMan; class CAddress; -class CAddrMan; class CDataStream; +struct bilingual_str; -class CBanEntry -{ -public: - static const int CURRENT_VERSION=1; - int nVersion; - int64_t nCreateTime; - int64_t nBanUntil; +bool DumpPeerAddresses(const ArgsManager& args, const AddrMan& addr); +/** Only used by tests. */ +void ReadFromStream(AddrMan& addr, CDataStream& ssPeers); - CBanEntry() - { - SetNull(); - } - - explicit CBanEntry(int64_t nCreateTimeIn) - { - SetNull(); - nCreateTime = nCreateTimeIn; - } - - /** - * Create a ban entry from JSON. - * @param[in] json A JSON representation of a ban entry, as created by `ToJson()`. - * @throw std::runtime_error if the JSON does not have the expected fields. - */ - explicit CBanEntry(const UniValue& json); - - SERIALIZE_METHODS(CBanEntry, obj) - { - uint8_t ban_reason = 2; //! For backward compatibility - READWRITE(obj.nVersion, obj.nCreateTime, obj.nBanUntil, ban_reason); - } - - void SetNull() - { - nVersion = CBanEntry::CURRENT_VERSION; - nCreateTime = 0; - nBanUntil = 0; - } - - /** - * Generate a JSON representation of this ban entry. - * @return JSON suitable for passing to the `CBanEntry(const UniValue&)` constructor. - */ - UniValue ToJson() const; -}; - -/** Access to the (IP) address database (peers.dat) */ -class CAddrDB -{ -private: - fs::path pathAddr; -public: - CAddrDB(); - bool Write(const CAddrMan& addr); - bool Read(CAddrMan& addr); - static bool Read(CAddrMan& addr, CDataStream& ssPeers); -}; - -/** Access to the banlist databases (banlist.json and banlist.dat) */ +/** Access to the banlist database (banlist.json) */ class CBanDB { private: @@ -95,13 +43,14 @@ public: * Read the banlist from disk. * @param[out] banSet The loaded list. Set if `true` is returned, otherwise it is left * in an undefined state. - * @param[out] dirty Indicates whether the loaded list needs flushing to disk. Set if - * `true` is returned, otherwise it is left in an undefined state. * @return true on success */ - bool Read(banmap_t& banSet, bool& dirty); + bool Read(banmap_t& banSet); }; +/** Returns an error string on failure */ +std::optional LoadAddrman(const std::vector& asmap, const ArgsManager& args, std::unique_ptr& addrman); + /** * Dump the anchor IP address database (anchors.dat) * diff --git a/src/addrman.cpp b/src/addrman.cpp index 8192b4eba6..f91a979934 100644 --- a/src/addrman.cpp +++ b/src/addrman.cpp @@ -1,48 +1,70 @@ // Copyright (c) 2012 Pieter Wuille -// Copyright (c) 2012-2020 The Bitcoin Core developers +// Copyright (c) 2012-2021 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include +#include #include #include +#include #include +#include +#include #include +#include +#include +#include +#include +#include #include #include -#include -#include -int CAddrInfo::GetTriedBucket(const uint256& nKey, const std::vector &asmap) const +/** Over how many buckets entries with tried addresses from a single group (/16 for IPv4) are spread */ +static constexpr uint32_t ADDRMAN_TRIED_BUCKETS_PER_GROUP{8}; +/** Over how many buckets entries with new addresses originating from a single group are spread */ +static constexpr uint32_t ADDRMAN_NEW_BUCKETS_PER_SOURCE_GROUP{64}; +/** Maximum number of times an address can occur in the new table */ +static constexpr int32_t ADDRMAN_NEW_BUCKETS_PER_ADDRESS{8}; +/** How old addresses can maximally be */ +static constexpr int64_t ADDRMAN_HORIZON_DAYS{30}; +/** After how many failed attempts we give up on a new node */ +static constexpr int32_t ADDRMAN_RETRIES{3}; +/** How many successive failures are allowed ... */ +static constexpr int32_t ADDRMAN_MAX_FAILURES{10}; +/** ... in at least this many days */ +static constexpr int64_t ADDRMAN_MIN_FAIL_DAYS{7}; +/** How recent a successful connection should be before we allow an address to be evicted from tried */ +static constexpr int64_t ADDRMAN_REPLACEMENT_HOURS{4}; +/** The maximum number of tried addr collisions to store */ +static constexpr size_t ADDRMAN_SET_TRIED_COLLISION_SIZE{10}; +/** The maximum time we'll spend trying to resolve a tried table collision, in seconds */ +static constexpr int64_t ADDRMAN_TEST_WINDOW{40*60}; // 40 minutes + +int AddrInfo::GetTriedBucket(const uint256& nKey, const std::vector& asmap) const { uint64_t hash1 = (CHashWriter(SER_GETHASH, 0) << nKey << GetKey()).GetCheapHash(); uint64_t hash2 = (CHashWriter(SER_GETHASH, 0) << nKey << GetGroup(asmap) << (hash1 % ADDRMAN_TRIED_BUCKETS_PER_GROUP)).GetCheapHash(); - int tried_bucket = hash2 % ADDRMAN_TRIED_BUCKET_COUNT; - uint32_t mapped_as = GetMappedAS(asmap); - LogPrint(BCLog::NET, "IP %s mapped to AS%i belongs to tried bucket %i\n", ToStringIP(), mapped_as, tried_bucket); - return tried_bucket; + return hash2 % ADDRMAN_TRIED_BUCKET_COUNT; } -int CAddrInfo::GetNewBucket(const uint256& nKey, const CNetAddr& src, const std::vector &asmap) const +int AddrInfo::GetNewBucket(const uint256& nKey, const CNetAddr& src, const std::vector& asmap) const { std::vector vchSourceGroupKey = src.GetGroup(asmap); uint64_t hash1 = (CHashWriter(SER_GETHASH, 0) << nKey << GetGroup(asmap) << vchSourceGroupKey).GetCheapHash(); uint64_t hash2 = (CHashWriter(SER_GETHASH, 0) << nKey << vchSourceGroupKey << (hash1 % ADDRMAN_NEW_BUCKETS_PER_SOURCE_GROUP)).GetCheapHash(); - int new_bucket = hash2 % ADDRMAN_NEW_BUCKET_COUNT; - uint32_t mapped_as = GetMappedAS(asmap); - LogPrint(BCLog::NET, "IP %s mapped to AS%i belongs to new bucket %i\n", ToStringIP(), mapped_as, new_bucket); - return new_bucket; + return hash2 % ADDRMAN_NEW_BUCKET_COUNT; } -int CAddrInfo::GetBucketPosition(const uint256 &nKey, bool fNew, int nBucket) const +int AddrInfo::GetBucketPosition(const uint256& nKey, bool fNew, int nBucket) const { uint64_t hash1 = (CHashWriter(SER_GETHASH, 0) << nKey << (fNew ? uint8_t{'N'} : uint8_t{'K'}) << nBucket << GetKey()).GetCheapHash(); return hash1 % ADDRMAN_BUCKET_SIZE; } -bool CAddrInfo::IsTerrible(int64_t nNow) const +bool AddrInfo::IsTerrible(int64_t nNow) const { if (nLastTry && nLastTry >= nNow - 60) // never remove things tried in the last minute return false; @@ -62,7 +84,7 @@ bool CAddrInfo::IsTerrible(int64_t nNow) const return false; } -double CAddrInfo::GetChance(int64_t nNow) const +double AddrInfo::GetChance(int64_t nNow) const { double fChance = 1.0; int64_t nSinceLastTry = std::max(nNow - nLastTry, 0); @@ -77,39 +99,305 @@ double CAddrInfo::GetChance(int64_t nNow) const return fChance; } -void CAddrMan::RemoveInvalid() +AddrManImpl::AddrManImpl(std::vector&& asmap, bool deterministic, int32_t consistency_check_ratio) + : insecure_rand{deterministic} + , nKey{deterministic ? uint256{1} : insecure_rand.rand256()} + , m_consistency_check_ratio{consistency_check_ratio} + , m_asmap{std::move(asmap)} { - for (size_t bucket = 0; bucket < ADDRMAN_NEW_BUCKET_COUNT; ++bucket) { - for (size_t i = 0; i < ADDRMAN_BUCKET_SIZE; ++i) { - const auto id = vvNew[bucket][i]; - if (id != -1 && !mapInfo[id].IsValid()) { - ClearNew(bucket, i); - } + for (auto& bucket : vvNew) { + for (auto& entry : bucket) { + entry = -1; } } - - for (size_t bucket = 0; bucket < ADDRMAN_TRIED_BUCKET_COUNT; ++bucket) { - for (size_t i = 0; i < ADDRMAN_BUCKET_SIZE; ++i) { - const auto id = vvTried[bucket][i]; - if (id == -1) { - continue; - } - const auto& addr_info = mapInfo[id]; - if (addr_info.IsValid()) { - continue; - } - vvTried[bucket][i] = -1; - --nTried; - SwapRandom(addr_info.nRandomPos, vRandom.size() - 1); - vRandom.pop_back(); - mapAddr.erase(addr_info); - mapInfo.erase(id); - m_tried_collisions.erase(id); + for (auto& bucket : vvTried) { + for (auto& entry : bucket) { + entry = -1; } } } -CAddrInfo* CAddrMan::Find(const CNetAddr& addr, int* pnId) +AddrManImpl::~AddrManImpl() +{ + nKey.SetNull(); +} + +template +void AddrManImpl::Serialize(Stream& s_) const +{ + LOCK(cs); + + /** + * Serialized format. + * * format version byte (@see `Format`) + * * lowest compatible format version byte. This is used to help old software decide + * whether to parse the file. For example: + * * Bitcoin Core version N knows how to parse up to format=3. If a new format=4 is + * introduced in version N+1 that is compatible with format=3 and it is known that + * version N will be able to parse it, then version N+1 will write + * (format=4, lowest_compatible=3) in the first two bytes of the file, and so + * version N will still try to parse it. + * * Bitcoin Core version N+2 introduces a new incompatible format=5. It will write + * (format=5, lowest_compatible=5) and so any versions that do not know how to parse + * format=5 will not try to read the file. + * * nKey + * * nNew + * * nTried + * * number of "new" buckets XOR 2**30 + * * all new addresses (total count: nNew) + * * all tried addresses (total count: nTried) + * * for each new bucket: + * * number of elements + * * for each element: index in the serialized "all new addresses" + * * asmap checksum + * + * 2**30 is xorred with the number of buckets to make addrman deserializer v0 detect it + * as incompatible. This is necessary because it did not check the version number on + * deserialization. + * + * vvNew, vvTried, mapInfo, mapAddr and vRandom are never encoded explicitly; + * they are instead reconstructed from the other information. + * + * This format is more complex, but significantly smaller (at most 1.5 MiB), and supports + * changes to the ADDRMAN_ parameters without breaking the on-disk structure. + * + * We don't use SERIALIZE_METHODS since the serialization and deserialization code has + * very little in common. + */ + + // Always serialize in the latest version (FILE_FORMAT). + + OverrideStream s(&s_, s_.GetType(), s_.GetVersion() | ADDRV2_FORMAT); + + s << static_cast(FILE_FORMAT); + + // Increment `lowest_compatible` iff a newly introduced format is incompatible with + // the previous one. + static constexpr uint8_t lowest_compatible = Format::V4_MULTIPORT; + s << static_cast(INCOMPATIBILITY_BASE + lowest_compatible); + + s << nKey; + s << nNew; + s << nTried; + + int nUBuckets = ADDRMAN_NEW_BUCKET_COUNT ^ (1 << 30); + s << nUBuckets; + std::unordered_map mapUnkIds; + int nIds = 0; + for (const auto& entry : mapInfo) { + mapUnkIds[entry.first] = nIds; + const AddrInfo& info = entry.second; + if (info.nRefCount) { + assert(nIds != nNew); // this means nNew was wrong, oh ow + s << info; + nIds++; + } + } + nIds = 0; + for (const auto& entry : mapInfo) { + const AddrInfo& info = entry.second; + if (info.fInTried) { + assert(nIds != nTried); // this means nTried was wrong, oh ow + s << info; + nIds++; + } + } + for (int bucket = 0; bucket < ADDRMAN_NEW_BUCKET_COUNT; bucket++) { + int nSize = 0; + for (int i = 0; i < ADDRMAN_BUCKET_SIZE; i++) { + if (vvNew[bucket][i] != -1) + nSize++; + } + s << nSize; + for (int i = 0; i < ADDRMAN_BUCKET_SIZE; i++) { + if (vvNew[bucket][i] != -1) { + int nIndex = mapUnkIds[vvNew[bucket][i]]; + s << nIndex; + } + } + } + // Store asmap checksum after bucket entries so that it + // can be ignored by older clients for backward compatibility. + uint256 asmap_checksum; + if (m_asmap.size() != 0) { + asmap_checksum = SerializeHash(m_asmap); + } + s << asmap_checksum; +} + +template +void AddrManImpl::Unserialize(Stream& s_) +{ + LOCK(cs); + + assert(vRandom.empty()); + + Format format; + s_ >> Using>(format); + + int stream_version = s_.GetVersion(); + if (format >= Format::V3_BIP155) { + // Add ADDRV2_FORMAT to the version so that the CNetAddr and CAddress + // unserialize methods know that an address in addrv2 format is coming. + stream_version |= ADDRV2_FORMAT; + } + + OverrideStream s(&s_, s_.GetType(), stream_version); + + uint8_t compat; + s >> compat; + const uint8_t lowest_compatible = compat - INCOMPATIBILITY_BASE; + if (lowest_compatible > FILE_FORMAT) { + throw InvalidAddrManVersionError(strprintf( + "Unsupported format of addrman database: %u. It is compatible with formats >=%u, " + "but the maximum supported by this version of %s is %u.", + uint8_t{format}, uint8_t{lowest_compatible}, PACKAGE_NAME, uint8_t{FILE_FORMAT})); + } + + s >> nKey; + s >> nNew; + s >> nTried; + int nUBuckets = 0; + s >> nUBuckets; + if (format >= Format::V1_DETERMINISTIC) { + nUBuckets ^= (1 << 30); + } + + if (nNew > ADDRMAN_NEW_BUCKET_COUNT * ADDRMAN_BUCKET_SIZE || nNew < 0) { + throw std::ios_base::failure( + strprintf("Corrupt AddrMan serialization: nNew=%d, should be in [0, %d]", + nNew, + ADDRMAN_NEW_BUCKET_COUNT * ADDRMAN_BUCKET_SIZE)); + } + + if (nTried > ADDRMAN_TRIED_BUCKET_COUNT * ADDRMAN_BUCKET_SIZE || nTried < 0) { + throw std::ios_base::failure( + strprintf("Corrupt AddrMan serialization: nTried=%d, should be in [0, %d]", + nTried, + ADDRMAN_TRIED_BUCKET_COUNT * ADDRMAN_BUCKET_SIZE)); + } + + // Deserialize entries from the new table. + for (int n = 0; n < nNew; n++) { + AddrInfo& info = mapInfo[n]; + s >> info; + mapAddr[info] = n; + info.nRandomPos = vRandom.size(); + vRandom.push_back(n); + } + nIdCount = nNew; + + // Deserialize entries from the tried table. + int nLost = 0; + for (int n = 0; n < nTried; n++) { + AddrInfo info; + s >> info; + int nKBucket = info.GetTriedBucket(nKey, m_asmap); + int nKBucketPos = info.GetBucketPosition(nKey, false, nKBucket); + if (info.IsValid() + && vvTried[nKBucket][nKBucketPos] == -1) { + info.nRandomPos = vRandom.size(); + info.fInTried = true; + vRandom.push_back(nIdCount); + mapInfo[nIdCount] = info; + mapAddr[info] = nIdCount; + vvTried[nKBucket][nKBucketPos] = nIdCount; + nIdCount++; + } else { + nLost++; + } + } + nTried -= nLost; + + // Store positions in the new table buckets to apply later (if possible). + // An entry may appear in up to ADDRMAN_NEW_BUCKETS_PER_ADDRESS buckets, + // so we store all bucket-entry_index pairs to iterate through later. + std::vector> bucket_entries; + + for (int bucket = 0; bucket < nUBuckets; ++bucket) { + int num_entries{0}; + s >> num_entries; + for (int n = 0; n < num_entries; ++n) { + int entry_index{0}; + s >> entry_index; + if (entry_index >= 0 && entry_index < nNew) { + bucket_entries.emplace_back(bucket, entry_index); + } + } + } + + // If the bucket count and asmap checksum haven't changed, then attempt + // to restore the entries to the buckets/positions they were in before + // serialization. + uint256 supplied_asmap_checksum; + if (m_asmap.size() != 0) { + supplied_asmap_checksum = SerializeHash(m_asmap); + } + uint256 serialized_asmap_checksum; + if (format >= Format::V2_ASMAP) { + s >> serialized_asmap_checksum; + } + const bool restore_bucketing{nUBuckets == ADDRMAN_NEW_BUCKET_COUNT && + serialized_asmap_checksum == supplied_asmap_checksum}; + + if (!restore_bucketing) { + LogPrint(BCLog::ADDRMAN, "Bucketing method was updated, re-bucketing addrman entries from disk\n"); + } + + for (auto bucket_entry : bucket_entries) { + int bucket{bucket_entry.first}; + const int entry_index{bucket_entry.second}; + AddrInfo& info = mapInfo[entry_index]; + + // Don't store the entry in the new bucket if it's not a valid address for our addrman + if (!info.IsValid()) continue; + + // The entry shouldn't appear in more than + // ADDRMAN_NEW_BUCKETS_PER_ADDRESS. If it has already, just skip + // this bucket_entry. + if (info.nRefCount >= ADDRMAN_NEW_BUCKETS_PER_ADDRESS) continue; + + int bucket_position = info.GetBucketPosition(nKey, true, bucket); + if (restore_bucketing && vvNew[bucket][bucket_position] == -1) { + // Bucketing has not changed, using existing bucket positions for the new table + vvNew[bucket][bucket_position] = entry_index; + ++info.nRefCount; + } else { + // In case the new table data cannot be used (bucket count wrong or new asmap), + // try to give them a reference based on their primary source address. + bucket = info.GetNewBucket(nKey, m_asmap); + bucket_position = info.GetBucketPosition(nKey, true, bucket); + if (vvNew[bucket][bucket_position] == -1) { + vvNew[bucket][bucket_position] = entry_index; + ++info.nRefCount; + } + } + } + + // Prune new entries with refcount 0 (as a result of collisions or invalid address). + int nLostUnk = 0; + for (auto it = mapInfo.cbegin(); it != mapInfo.cend(); ) { + if (it->second.fInTried == false && it->second.nRefCount == 0) { + const auto itCopy = it++; + Delete(itCopy->first); + ++nLostUnk; + } else { + ++it; + } + } + if (nLost + nLostUnk > 0) { + LogPrint(BCLog::ADDRMAN, "addrman lost %i new and %i tried addresses due to collisions or invalid addresses\n", nLostUnk, nLost); + } + + const int check_code{CheckAddrman()}; + if (check_code != 0) { + throw std::ios_base::failure(strprintf( + "Corrupt data. Consistency check failed with code %s", + check_code)); + } +} + +AddrInfo* AddrManImpl::Find(const CService& addr, int* pnId) { AssertLockHeld(cs); @@ -124,12 +412,12 @@ CAddrInfo* CAddrMan::Find(const CNetAddr& addr, int* pnId) return nullptr; } -CAddrInfo* CAddrMan::Create(const CAddress& addr, const CNetAddr& addrSource, int* pnId) +AddrInfo* AddrManImpl::Create(const CAddress& addr, const CNetAddr& addrSource, int* pnId) { AssertLockHeld(cs); int nId = nIdCount++; - mapInfo[nId] = CAddrInfo(addr, addrSource); + mapInfo[nId] = AddrInfo(addr, addrSource); mapAddr[addr] = nId; mapInfo[nId].nRandomPos = vRandom.size(); vRandom.push_back(nId); @@ -138,7 +426,7 @@ CAddrInfo* CAddrMan::Create(const CAddress& addr, const CNetAddr& addrSource, in return &mapInfo[nId]; } -void CAddrMan::SwapRandom(unsigned int nRndPos1, unsigned int nRndPos2) +void AddrManImpl::SwapRandom(unsigned int nRndPos1, unsigned int nRndPos2) const { AssertLockHeld(cs); @@ -150,22 +438,24 @@ void CAddrMan::SwapRandom(unsigned int nRndPos1, unsigned int nRndPos2) int nId1 = vRandom[nRndPos1]; int nId2 = vRandom[nRndPos2]; - assert(mapInfo.count(nId1) == 1); - assert(mapInfo.count(nId2) == 1); + const auto it_1{mapInfo.find(nId1)}; + const auto it_2{mapInfo.find(nId2)}; + assert(it_1 != mapInfo.end()); + assert(it_2 != mapInfo.end()); - mapInfo[nId1].nRandomPos = nRndPos2; - mapInfo[nId2].nRandomPos = nRndPos1; + it_1->second.nRandomPos = nRndPos2; + it_2->second.nRandomPos = nRndPos1; vRandom[nRndPos1] = nId2; vRandom[nRndPos2] = nId1; } -void CAddrMan::Delete(int nId) +void AddrManImpl::Delete(int nId) { AssertLockHeld(cs); assert(mapInfo.count(nId) != 0); - CAddrInfo& info = mapInfo[nId]; + AddrInfo& info = mapInfo[nId]; assert(!info.fInTried); assert(info.nRefCount == 0); @@ -176,33 +466,37 @@ void CAddrMan::Delete(int nId) nNew--; } -void CAddrMan::ClearNew(int nUBucket, int nUBucketPos) +void AddrManImpl::ClearNew(int nUBucket, int nUBucketPos) { AssertLockHeld(cs); // if there is an entry in the specified bucket, delete it. if (vvNew[nUBucket][nUBucketPos] != -1) { int nIdDelete = vvNew[nUBucket][nUBucketPos]; - CAddrInfo& infoDelete = mapInfo[nIdDelete]; + AddrInfo& infoDelete = mapInfo[nIdDelete]; assert(infoDelete.nRefCount > 0); infoDelete.nRefCount--; vvNew[nUBucket][nUBucketPos] = -1; + LogPrint(BCLog::ADDRMAN, "Removed %s from new[%i][%i]\n", infoDelete.ToString(), nUBucket, nUBucketPos); if (infoDelete.nRefCount == 0) { Delete(nIdDelete); } } } -void CAddrMan::MakeTried(CAddrInfo& info, int nId) +void AddrManImpl::MakeTried(AddrInfo& info, int nId) { AssertLockHeld(cs); // remove the entry from all new buckets - for (int bucket = 0; bucket < ADDRMAN_NEW_BUCKET_COUNT; bucket++) { - int pos = info.GetBucketPosition(nKey, true, bucket); + const int start_bucket{info.GetNewBucket(nKey, m_asmap)}; + for (int n = 0; n < ADDRMAN_NEW_BUCKET_COUNT; ++n) { + const int bucket{(start_bucket + n) % ADDRMAN_NEW_BUCKET_COUNT}; + const int pos{info.GetBucketPosition(nKey, true, bucket)}; if (vvNew[bucket][pos] == nId) { vvNew[bucket][pos] = -1; info.nRefCount--; + if (info.nRefCount == 0) break; } } nNew--; @@ -218,7 +512,7 @@ void CAddrMan::MakeTried(CAddrInfo& info, int nId) // find an item to evict int nIdEvict = vvTried[nKBucket][nKBucketPos]; assert(mapInfo.count(nIdEvict) == 1); - CAddrInfo& infoOld = mapInfo[nIdEvict]; + AddrInfo& infoOld = mapInfo[nIdEvict]; // Remove the to-be-evicted item from the tried set. infoOld.fInTried = false; @@ -235,6 +529,8 @@ void CAddrMan::MakeTried(CAddrInfo& info, int nId) infoOld.nRefCount = 1; vvNew[nUBucket][nUBucketPos] = nIdEvict; nNew++; + LogPrint(BCLog::ADDRMAN, "Moved %s from tried[%i][%i] to new[%i][%i] to make space\n", + infoOld.ToString(), nKBucket, nKBucketPos, nUBucket, nUBucketPos); } assert(vvTried[nKBucket][nKBucketPos] == -1); @@ -243,84 +539,15 @@ void CAddrMan::MakeTried(CAddrInfo& info, int nId) info.fInTried = true; } -void CAddrMan::Good_(const CService& addr, bool test_before_evict, int64_t nTime) -{ - AssertLockHeld(cs); - - int nId; - - nLastGood = nTime; - - CAddrInfo* pinfo = Find(addr, &nId); - - // if not found, bail out - if (!pinfo) - return; - - CAddrInfo& info = *pinfo; - - // check whether we are talking about the exact same CService (including same port) - if (info != addr) - return; - - // update info - info.nLastSuccess = nTime; - info.nLastTry = nTime; - info.nAttempts = 0; - // nTime is not updated here, to avoid leaking information about - // currently-connected peers. - - // if it is already in the tried set, don't do anything else - if (info.fInTried) - return; - - // find a bucket it is in now - int nRnd = insecure_rand.randrange(ADDRMAN_NEW_BUCKET_COUNT); - int nUBucket = -1; - for (unsigned int n = 0; n < ADDRMAN_NEW_BUCKET_COUNT; n++) { - int nB = (n + nRnd) % ADDRMAN_NEW_BUCKET_COUNT; - int nBpos = info.GetBucketPosition(nKey, true, nB); - if (vvNew[nB][nBpos] == nId) { - nUBucket = nB; - break; - } - } - - // if no bucket is found, something bad happened; - // TODO: maybe re-add the node, but for now, just bail out - if (nUBucket == -1) - return; - - // which tried bucket to move the entry to - int tried_bucket = info.GetTriedBucket(nKey, m_asmap); - int tried_bucket_pos = info.GetBucketPosition(nKey, false, tried_bucket); - - // Will moving this address into tried evict another entry? - if (test_before_evict && (vvTried[tried_bucket][tried_bucket_pos] != -1)) { - // Output the entry we'd be colliding with, for debugging purposes - auto colliding_entry = mapInfo.find(vvTried[tried_bucket][tried_bucket_pos]); - LogPrint(BCLog::ADDRMAN, "Collision inserting element into tried table (%s), moving %s to m_tried_collisions=%d\n", colliding_entry != mapInfo.end() ? colliding_entry->second.ToString() : "", addr.ToString(), m_tried_collisions.size()); - if (m_tried_collisions.size() < ADDRMAN_SET_TRIED_COLLISION_SIZE) { - m_tried_collisions.insert(nId); - } - } else { - LogPrint(BCLog::ADDRMAN, "Moving %s to tried\n", addr.ToString()); - - // move nId to the tried tables - MakeTried(info, nId); - } -} - -bool CAddrMan::Add_(const CAddress& addr, const CNetAddr& source, int64_t nTimePenalty) +bool AddrManImpl::AddSingle(const CAddress& addr, const CNetAddr& source, int64_t nTimePenalty) { AssertLockHeld(cs); if (!addr.IsRoutable()) return false; - bool fNew = false; int nId; - CAddrInfo* pinfo = Find(addr, &nId); + AddrInfo* pinfo = Find(addr, &nId); // Do not set a penalty for a source's self-announcement if (addr == source) { @@ -359,15 +586,14 @@ bool CAddrMan::Add_(const CAddress& addr, const CNetAddr& source, int64_t nTimeP pinfo = Create(addr, source, &nId); pinfo->nTime = std::max((int64_t)0, (int64_t)pinfo->nTime - nTimePenalty); nNew++; - fNew = true; } int nUBucket = pinfo->GetNewBucket(nKey, source, m_asmap); int nUBucketPos = pinfo->GetBucketPosition(nKey, true, nUBucket); + bool fInsert = vvNew[nUBucket][nUBucketPos] == -1; if (vvNew[nUBucket][nUBucketPos] != nId) { - bool fInsert = vvNew[nUBucket][nUBucketPos] == -1; if (!fInsert) { - CAddrInfo& infoExisting = mapInfo[vvNew[nUBucket][nUBucketPos]]; + AddrInfo& infoExisting = mapInfo[vvNew[nUBucket][nUBucketPos]]; if (infoExisting.IsTerrible() || (infoExisting.nRefCount > 1 && pinfo->nRefCount == 0)) { // Overwrite the existing new table entry. fInsert = true; @@ -377,30 +603,94 @@ bool CAddrMan::Add_(const CAddress& addr, const CNetAddr& source, int64_t nTimeP ClearNew(nUBucket, nUBucketPos); pinfo->nRefCount++; vvNew[nUBucket][nUBucketPos] = nId; + LogPrint(BCLog::ADDRMAN, "Added %s mapped to AS%i to new[%i][%i]\n", + addr.ToString(), addr.GetMappedAS(m_asmap), nUBucket, nUBucketPos); } else { if (pinfo->nRefCount == 0) { Delete(nId); } } } - return fNew; + return fInsert; } -void CAddrMan::Attempt_(const CService& addr, bool fCountFailure, int64_t nTime) +bool AddrManImpl::Good_(const CService& addr, bool test_before_evict, int64_t nTime) { AssertLockHeld(cs); - CAddrInfo* pinfo = Find(addr); + int nId; + + nLastGood = nTime; + + AddrInfo* pinfo = Find(addr, &nId); + + // if not found, bail out + if (!pinfo) return false; + + AddrInfo& info = *pinfo; + + // update info + info.nLastSuccess = nTime; + info.nLastTry = nTime; + info.nAttempts = 0; + // nTime is not updated here, to avoid leaking information about + // currently-connected peers. + + // if it is already in the tried set, don't do anything else + if (info.fInTried) return false; + + // if it is not in new, something bad happened + if (!Assume(info.nRefCount > 0)) return false; + + + // which tried bucket to move the entry to + int tried_bucket = info.GetTriedBucket(nKey, m_asmap); + int tried_bucket_pos = info.GetBucketPosition(nKey, false, tried_bucket); + + // Will moving this address into tried evict another entry? + if (test_before_evict && (vvTried[tried_bucket][tried_bucket_pos] != -1)) { + if (m_tried_collisions.size() < ADDRMAN_SET_TRIED_COLLISION_SIZE) { + m_tried_collisions.insert(nId); + } + // Output the entry we'd be colliding with, for debugging purposes + auto colliding_entry = mapInfo.find(vvTried[tried_bucket][tried_bucket_pos]); + LogPrint(BCLog::ADDRMAN, "Collision with %s while attempting to move %s to tried table. Collisions=%d\n", + colliding_entry != mapInfo.end() ? colliding_entry->second.ToString() : "", + addr.ToString(), + m_tried_collisions.size()); + return false; + } else { + // move nId to the tried tables + MakeTried(info, nId); + LogPrint(BCLog::ADDRMAN, "Moved %s mapped to AS%i to tried[%i][%i]\n", + addr.ToString(), addr.GetMappedAS(m_asmap), tried_bucket, tried_bucket_pos); + return true; + } +} + +bool AddrManImpl::Add_(const std::vector &vAddr, const CNetAddr& source, int64_t nTimePenalty) +{ + int added{0}; + for (std::vector::const_iterator it = vAddr.begin(); it != vAddr.end(); it++) { + added += AddSingle(*it, source, nTimePenalty) ? 1 : 0; + } + if (added > 0) { + LogPrint(BCLog::ADDRMAN, "Added %i addresses (of %i) from %s: %i tried, %i new\n", added, vAddr.size(), source.ToString(), nTried, nNew); + } + return added > 0; +} + +void AddrManImpl::Attempt_(const CService& addr, bool fCountFailure, int64_t nTime) +{ + AssertLockHeld(cs); + + AddrInfo* pinfo = Find(addr); // if not found, bail out if (!pinfo) return; - CAddrInfo& info = *pinfo; - - // check whether we are talking about the exact same CService (including same port) - if (info != addr) - return; + AddrInfo& info = *pinfo; // update info info.nLastTry = nTime; @@ -410,15 +700,13 @@ void CAddrMan::Attempt_(const CService& addr, bool fCountFailure, int64_t nTime) } } -CAddrInfo CAddrMan::Select_(bool newOnly) +std::pair AddrManImpl::Select_(bool newOnly) const { AssertLockHeld(cs); - if (vRandom.empty()) - return CAddrInfo(); + if (vRandom.empty()) return {}; - if (newOnly && nNew == 0) - return CAddrInfo(); + if (newOnly && nNew == 0) return {}; // Use a 50% chance for choosing between tried and new table entries. if (!newOnly && @@ -426,120 +714,62 @@ CAddrInfo CAddrMan::Select_(bool newOnly) // use a tried node double fChanceFactor = 1.0; while (1) { + // Pick a tried bucket, and an initial position in that bucket. int nKBucket = insecure_rand.randrange(ADDRMAN_TRIED_BUCKET_COUNT); int nKBucketPos = insecure_rand.randrange(ADDRMAN_BUCKET_SIZE); - while (vvTried[nKBucket][nKBucketPos] == -1) { - nKBucket = (nKBucket + insecure_rand.randbits(ADDRMAN_TRIED_BUCKET_COUNT_LOG2)) % ADDRMAN_TRIED_BUCKET_COUNT; - nKBucketPos = (nKBucketPos + insecure_rand.randbits(ADDRMAN_BUCKET_SIZE_LOG2)) % ADDRMAN_BUCKET_SIZE; + // Iterate over the positions of that bucket, starting at the initial one, + // and looping around. + int i; + for (i = 0; i < ADDRMAN_BUCKET_SIZE; ++i) { + if (vvTried[nKBucket][(nKBucketPos + i) % ADDRMAN_BUCKET_SIZE] != -1) break; } - int nId = vvTried[nKBucket][nKBucketPos]; - assert(mapInfo.count(nId) == 1); - CAddrInfo& info = mapInfo[nId]; - if (insecure_rand.randbits(30) < fChanceFactor * info.GetChance() * (1 << 30)) - return info; + // If the bucket is entirely empty, start over with a (likely) different one. + if (i == ADDRMAN_BUCKET_SIZE) continue; + // Find the entry to return. + int nId = vvTried[nKBucket][(nKBucketPos + i) % ADDRMAN_BUCKET_SIZE]; + const auto it_found{mapInfo.find(nId)}; + assert(it_found != mapInfo.end()); + const AddrInfo& info{it_found->second}; + // With probability GetChance() * fChanceFactor, return the entry. + if (insecure_rand.randbits(30) < fChanceFactor * info.GetChance() * (1 << 30)) { + LogPrint(BCLog::ADDRMAN, "Selected %s from tried\n", info.ToString()); + return {info, info.nLastTry}; + } + // Otherwise start over with a (likely) different bucket, and increased chance factor. fChanceFactor *= 1.2; } } else { // use a new node double fChanceFactor = 1.0; while (1) { + // Pick a new bucket, and an initial position in that bucket. int nUBucket = insecure_rand.randrange(ADDRMAN_NEW_BUCKET_COUNT); int nUBucketPos = insecure_rand.randrange(ADDRMAN_BUCKET_SIZE); - while (vvNew[nUBucket][nUBucketPos] == -1) { - nUBucket = (nUBucket + insecure_rand.randbits(ADDRMAN_NEW_BUCKET_COUNT_LOG2)) % ADDRMAN_NEW_BUCKET_COUNT; - nUBucketPos = (nUBucketPos + insecure_rand.randbits(ADDRMAN_BUCKET_SIZE_LOG2)) % ADDRMAN_BUCKET_SIZE; + // Iterate over the positions of that bucket, starting at the initial one, + // and looping around. + int i; + for (i = 0; i < ADDRMAN_BUCKET_SIZE; ++i) { + if (vvNew[nUBucket][(nUBucketPos + i) % ADDRMAN_BUCKET_SIZE] != -1) break; } - int nId = vvNew[nUBucket][nUBucketPos]; - assert(mapInfo.count(nId) == 1); - CAddrInfo& info = mapInfo[nId]; - if (insecure_rand.randbits(30) < fChanceFactor * info.GetChance() * (1 << 30)) - return info; + // If the bucket is entirely empty, start over with a (likely) different one. + if (i == ADDRMAN_BUCKET_SIZE) continue; + // Find the entry to return. + int nId = vvNew[nUBucket][(nUBucketPos + i) % ADDRMAN_BUCKET_SIZE]; + const auto it_found{mapInfo.find(nId)}; + assert(it_found != mapInfo.end()); + const AddrInfo& info{it_found->second}; + // With probability GetChance() * fChanceFactor, return the entry. + if (insecure_rand.randbits(30) < fChanceFactor * info.GetChance() * (1 << 30)) { + LogPrint(BCLog::ADDRMAN, "Selected %s from new\n", info.ToString()); + return {info, info.nLastTry}; + } + // Otherwise start over with a (likely) different bucket, and increased chance factor. fChanceFactor *= 1.2; } } } -#ifdef DEBUG_ADDRMAN -int CAddrMan::Check_() -{ - AssertLockHeld(cs); - - std::unordered_set setTried; - std::unordered_map mapNew; - - if (vRandom.size() != (size_t)(nTried + nNew)) - return -7; - - for (const auto& entry : mapInfo) { - int n = entry.first; - const CAddrInfo& info = entry.second; - if (info.fInTried) { - if (!info.nLastSuccess) - return -1; - if (info.nRefCount) - return -2; - setTried.insert(n); - } else { - if (info.nRefCount < 0 || info.nRefCount > ADDRMAN_NEW_BUCKETS_PER_ADDRESS) - return -3; - if (!info.nRefCount) - return -4; - mapNew[n] = info.nRefCount; - } - if (mapAddr[info] != n) - return -5; - if (info.nRandomPos < 0 || (size_t)info.nRandomPos >= vRandom.size() || vRandom[info.nRandomPos] != n) - return -14; - if (info.nLastTry < 0) - return -6; - if (info.nLastSuccess < 0) - return -8; - } - - if (setTried.size() != (size_t)nTried) - return -9; - if (mapNew.size() != (size_t)nNew) - return -10; - - for (int n = 0; n < ADDRMAN_TRIED_BUCKET_COUNT; n++) { - for (int i = 0; i < ADDRMAN_BUCKET_SIZE; i++) { - if (vvTried[n][i] != -1) { - if (!setTried.count(vvTried[n][i])) - return -11; - if (mapInfo[vvTried[n][i]].GetTriedBucket(nKey, m_asmap) != n) - return -17; - if (mapInfo[vvTried[n][i]].GetBucketPosition(nKey, false, n) != i) - return -18; - setTried.erase(vvTried[n][i]); - } - } - } - - for (int n = 0; n < ADDRMAN_NEW_BUCKET_COUNT; n++) { - for (int i = 0; i < ADDRMAN_BUCKET_SIZE; i++) { - if (vvNew[n][i] != -1) { - if (!mapNew.count(vvNew[n][i])) - return -12; - if (mapInfo[vvNew[n][i]].GetBucketPosition(nKey, true, n) != i) - return -19; - if (--mapNew[vvNew[n][i]] == 0) - mapNew.erase(vvNew[n][i]); - } - } - } - - if (setTried.size()) - return -13; - if (mapNew.size()) - return -15; - if (nKey.IsNull()) - return -16; - - return 0; -} -#endif - -void CAddrMan::GetAddr_(std::vector& vAddr, size_t max_addresses, size_t max_pct, std::optional network) +std::vector AddrManImpl::GetAddr_(size_t max_addresses, size_t max_pct, std::optional network) const { AssertLockHeld(cs); @@ -553,15 +783,17 @@ void CAddrMan::GetAddr_(std::vector& vAddr, size_t max_addresses, size // gather a list of random nodes, skipping those of low quality const int64_t now{GetAdjustedTime()}; + std::vector addresses; for (unsigned int n = 0; n < vRandom.size(); n++) { - if (vAddr.size() >= nNodes) + if (addresses.size() >= nNodes) break; int nRndPos = insecure_rand.randrange(vRandom.size() - n) + n; SwapRandom(n, nRndPos); - assert(mapInfo.count(vRandom[n]) == 1); + const auto it{mapInfo.find(vRandom[n])}; + assert(it != mapInfo.end()); - const CAddrInfo& ai = mapInfo[vRandom[n]]; + const AddrInfo& ai{it->second}; // Filter by network (optional) if (network != std::nullopt && ai.GetNetClass() != network) continue; @@ -569,25 +801,23 @@ void CAddrMan::GetAddr_(std::vector& vAddr, size_t max_addresses, size // Filter for quality if (ai.IsTerrible(now)) continue; - vAddr.push_back(ai); + addresses.push_back(ai); } + LogPrint(BCLog::ADDRMAN, "GetAddr returned %d random addresses\n", addresses.size()); + return addresses; } -void CAddrMan::Connected_(const CService& addr, int64_t nTime) +void AddrManImpl::Connected_(const CService& addr, int64_t nTime) { AssertLockHeld(cs); - CAddrInfo* pinfo = Find(addr); + AddrInfo* pinfo = Find(addr); // if not found, bail out if (!pinfo) return; - CAddrInfo& info = *pinfo; - - // check whether we are talking about the exact same CService (including same port) - if (info != addr) - return; + AddrInfo& info = *pinfo; // update info int64_t nUpdateInterval = 20 * 60; @@ -595,27 +825,23 @@ void CAddrMan::Connected_(const CService& addr, int64_t nTime) info.nTime = nTime; } -void CAddrMan::SetServices_(const CService& addr, ServiceFlags nServices) +void AddrManImpl::SetServices_(const CService& addr, ServiceFlags nServices) { AssertLockHeld(cs); - CAddrInfo* pinfo = Find(addr); + AddrInfo* pinfo = Find(addr); // if not found, bail out if (!pinfo) return; - CAddrInfo& info = *pinfo; - - // check whether we are talking about the exact same CService (including same port) - if (info != addr) - return; + AddrInfo& info = *pinfo; // update info info.nServices = nServices; } -void CAddrMan::ResolveCollisions_() +void AddrManImpl::ResolveCollisions_() { AssertLockHeld(cs); @@ -628,7 +854,7 @@ void CAddrMan::ResolveCollisions_() if (mapInfo.count(id_new) != 1) { erase_collision = true; } else { - CAddrInfo& info_new = mapInfo[id_new]; + AddrInfo& info_new = mapInfo[id_new]; // Which tried bucket to move the entry to. int tried_bucket = info_new.GetTriedBucket(nKey, m_asmap); @@ -639,7 +865,7 @@ void CAddrMan::ResolveCollisions_() // Get the to-be-evicted address that is being tested int id_old = vvTried[tried_bucket][tried_bucket_pos]; - CAddrInfo& info_old = mapInfo[id_old]; + AddrInfo& info_old = mapInfo[id_old]; // Has successfully connected in last X hours if (GetAdjustedTime() - info_old.nLastSuccess < ADDRMAN_REPLACEMENT_HOURS*(60*60)) { @@ -676,11 +902,11 @@ void CAddrMan::ResolveCollisions_() } } -CAddrInfo CAddrMan::SelectTriedCollision_() +std::pair AddrManImpl::SelectTriedCollision_() { AssertLockHeld(cs); - if (m_tried_collisions.size() == 0) return CAddrInfo(); + if (m_tried_collisions.size() == 0) return {}; std::set::iterator it = m_tried_collisions.begin(); @@ -691,43 +917,324 @@ CAddrInfo CAddrMan::SelectTriedCollision_() // If id_new not found in mapInfo remove it from m_tried_collisions if (mapInfo.count(id_new) != 1) { m_tried_collisions.erase(it); - return CAddrInfo(); + return {}; } - const CAddrInfo& newInfo = mapInfo[id_new]; + const AddrInfo& newInfo = mapInfo[id_new]; // which tried bucket to move the entry to int tried_bucket = newInfo.GetTriedBucket(nKey, m_asmap); int tried_bucket_pos = newInfo.GetBucketPosition(nKey, false, tried_bucket); - int id_old = vvTried[tried_bucket][tried_bucket_pos]; - - return mapInfo[id_old]; + const AddrInfo& info_old = mapInfo[vvTried[tried_bucket][tried_bucket_pos]]; + return {info_old, info_old.nLastTry}; } -std::vector CAddrMan::DecodeAsmap(fs::path path) +std::optional AddrManImpl::FindAddressEntry_(const CAddress& addr) { - std::vector bits; - FILE *filestr = fsbridge::fopen(path, "rb"); - CAutoFile file(filestr, SER_DISK, CLIENT_VERSION); - if (file.IsNull()) { - LogPrintf("Failed to open asmap file from disk\n"); - return bits; + AssertLockHeld(cs); + + AddrInfo* addr_info = Find(addr); + + if (!addr_info) return std::nullopt; + + if(addr_info->fInTried) { + int bucket{addr_info->GetTriedBucket(nKey, m_asmap)}; + return AddressPosition(/*tried=*/true, + /*multiplicity=*/1, + /*bucket=*/bucket, + /*position=*/addr_info->GetBucketPosition(nKey, false, bucket)); + } else { + int bucket{addr_info->GetNewBucket(nKey, m_asmap)}; + return AddressPosition(/*tried=*/false, + /*multiplicity=*/addr_info->nRefCount, + /*bucket=*/bucket, + /*position=*/addr_info->GetBucketPosition(nKey, true, bucket)); } - fseek(filestr, 0, SEEK_END); - int length = ftell(filestr); - LogPrintf("Opened asmap file %s (%d bytes) from disk\n", path, length); - fseek(filestr, 0, SEEK_SET); - uint8_t cur_byte; - for (int i = 0; i < length; ++i) { - file >> cur_byte; - for (int bit = 0; bit < 8; ++bit) { - bits.push_back((cur_byte >> bit) & 1); +} + +void AddrManImpl::Check() const +{ + AssertLockHeld(cs); + + // Run consistency checks 1 in m_consistency_check_ratio times if enabled + if (m_consistency_check_ratio == 0) return; + if (insecure_rand.randrange(m_consistency_check_ratio) >= 1) return; + + const int err{CheckAddrman()}; + if (err) { + LogPrintf("ADDRMAN CONSISTENCY CHECK FAILED!!! err=%i\n", err); + assert(false); + } +} + +int AddrManImpl::CheckAddrman() const +{ + AssertLockHeld(cs); + + LOG_TIME_MILLIS_WITH_CATEGORY_MSG_ONCE( + strprintf("new %i, tried %i, total %u", nNew, nTried, vRandom.size()), BCLog::ADDRMAN); + + std::unordered_set setTried; + std::unordered_map mapNew; + + if (vRandom.size() != (size_t)(nTried + nNew)) + return -7; + + for (const auto& entry : mapInfo) { + int n = entry.first; + const AddrInfo& info = entry.second; + if (info.fInTried) { + if (!info.nLastSuccess) + return -1; + if (info.nRefCount) + return -2; + setTried.insert(n); + } else { + if (info.nRefCount < 0 || info.nRefCount > ADDRMAN_NEW_BUCKETS_PER_ADDRESS) + return -3; + if (!info.nRefCount) + return -4; + mapNew[n] = info.nRefCount; + } + const auto it{mapAddr.find(info)}; + if (it == mapAddr.end() || it->second != n) { + return -5; + } + if (info.nRandomPos < 0 || (size_t)info.nRandomPos >= vRandom.size() || vRandom[info.nRandomPos] != n) + return -14; + if (info.nLastTry < 0) + return -6; + if (info.nLastSuccess < 0) + return -8; + } + + if (setTried.size() != (size_t)nTried) + return -9; + if (mapNew.size() != (size_t)nNew) + return -10; + + for (int n = 0; n < ADDRMAN_TRIED_BUCKET_COUNT; n++) { + for (int i = 0; i < ADDRMAN_BUCKET_SIZE; i++) { + if (vvTried[n][i] != -1) { + if (!setTried.count(vvTried[n][i])) + return -11; + const auto it{mapInfo.find(vvTried[n][i])}; + if (it == mapInfo.end() || it->second.GetTriedBucket(nKey, m_asmap) != n) { + return -17; + } + if (it->second.GetBucketPosition(nKey, false, n) != i) { + return -18; + } + setTried.erase(vvTried[n][i]); + } } } - if (!SanityCheckASMap(bits)) { - LogPrintf("Sanity check of asmap file %s failed\n", path); - return {}; + + for (int n = 0; n < ADDRMAN_NEW_BUCKET_COUNT; n++) { + for (int i = 0; i < ADDRMAN_BUCKET_SIZE; i++) { + if (vvNew[n][i] != -1) { + if (!mapNew.count(vvNew[n][i])) + return -12; + const auto it{mapInfo.find(vvNew[n][i])}; + if (it == mapInfo.end() || it->second.GetBucketPosition(nKey, true, n) != i) { + return -19; + } + if (--mapNew[vvNew[n][i]] == 0) + mapNew.erase(vvNew[n][i]); + } + } } - return bits; + + if (setTried.size()) + return -13; + if (mapNew.size()) + return -15; + if (nKey.IsNull()) + return -16; + + return 0; +} + +size_t AddrManImpl::size() const +{ + LOCK(cs); // TODO: Cache this in an atomic to avoid this overhead + return vRandom.size(); +} + +bool AddrManImpl::Add(const std::vector& vAddr, const CNetAddr& source, int64_t nTimePenalty) +{ + LOCK(cs); + Check(); + auto ret = Add_(vAddr, source, nTimePenalty); + Check(); + return ret; +} + +bool AddrManImpl::Good(const CService& addr, int64_t nTime) +{ + LOCK(cs); + Check(); + auto ret = Good_(addr, /*test_before_evict=*/true, nTime); + Check(); + return ret; +} + +void AddrManImpl::Attempt(const CService& addr, bool fCountFailure, int64_t nTime) +{ + LOCK(cs); + Check(); + Attempt_(addr, fCountFailure, nTime); + Check(); +} + +void AddrManImpl::ResolveCollisions() +{ + LOCK(cs); + Check(); + ResolveCollisions_(); + Check(); +} + +std::pair AddrManImpl::SelectTriedCollision() +{ + LOCK(cs); + Check(); + const auto ret = SelectTriedCollision_(); + Check(); + return ret; +} + +std::pair AddrManImpl::Select(bool newOnly) const +{ + LOCK(cs); + Check(); + const auto addrRet = Select_(newOnly); + Check(); + return addrRet; +} + +std::vector AddrManImpl::GetAddr(size_t max_addresses, size_t max_pct, std::optional network) const +{ + LOCK(cs); + Check(); + const auto addresses = GetAddr_(max_addresses, max_pct, network); + Check(); + return addresses; +} + +void AddrManImpl::Connected(const CService& addr, int64_t nTime) +{ + LOCK(cs); + Check(); + Connected_(addr, nTime); + Check(); +} + +void AddrManImpl::SetServices(const CService& addr, ServiceFlags nServices) +{ + LOCK(cs); + Check(); + SetServices_(addr, nServices); + Check(); +} + +std::optional AddrManImpl::FindAddressEntry(const CAddress& addr) +{ + LOCK(cs); + Check(); + auto entry = FindAddressEntry_(addr); + Check(); + return entry; +} + +const std::vector& AddrManImpl::GetAsmap() const +{ + return m_asmap; +} + +AddrMan::AddrMan(std::vector asmap, bool deterministic, int32_t consistency_check_ratio) + : m_impl(std::make_unique(std::move(asmap), deterministic, consistency_check_ratio)) {} + +AddrMan::~AddrMan() = default; + +template +void AddrMan::Serialize(Stream& s_) const +{ + m_impl->Serialize(s_); +} + +template +void AddrMan::Unserialize(Stream& s_) +{ + m_impl->Unserialize(s_); +} + +// explicit instantiation +template void AddrMan::Serialize(CHashWriter& s) const; +template void AddrMan::Serialize(CAutoFile& s) const; +template void AddrMan::Serialize(CDataStream& s) const; +template void AddrMan::Unserialize(CAutoFile& s); +template void AddrMan::Unserialize(CHashVerifier& s); +template void AddrMan::Unserialize(CDataStream& s); +template void AddrMan::Unserialize(CHashVerifier& s); + +size_t AddrMan::size() const +{ + return m_impl->size(); +} + +bool AddrMan::Add(const std::vector& vAddr, const CNetAddr& source, int64_t nTimePenalty) +{ + return m_impl->Add(vAddr, source, nTimePenalty); +} + +bool AddrMan::Good(const CService& addr, int64_t nTime) +{ + return m_impl->Good(addr, nTime); +} + +void AddrMan::Attempt(const CService& addr, bool fCountFailure, int64_t nTime) +{ + m_impl->Attempt(addr, fCountFailure, nTime); +} + +void AddrMan::ResolveCollisions() +{ + m_impl->ResolveCollisions(); +} + +std::pair AddrMan::SelectTriedCollision() +{ + return m_impl->SelectTriedCollision(); +} + +std::pair AddrMan::Select(bool newOnly) const +{ + return m_impl->Select(newOnly); +} + +std::vector AddrMan::GetAddr(size_t max_addresses, size_t max_pct, std::optional network) const +{ + return m_impl->GetAddr(max_addresses, max_pct, network); +} + +void AddrMan::Connected(const CService& addr, int64_t nTime) +{ + m_impl->Connected(addr, nTime); +} + +void AddrMan::SetServices(const CService& addr, ServiceFlags nServices) +{ + m_impl->SetServices(addr, nServices); +} + +const std::vector& AddrMan::GetAsmap() const +{ + return m_impl->GetAsmap(); +} + +std::optional AddrMan::FindAddressEntry(const CAddress& addr) +{ + return m_impl->FindAddressEntry(addr); } diff --git a/src/addrman.h b/src/addrman.h index 6f081b8dc1..472282833b 100644 --- a/src/addrman.h +++ b/src/addrman.h @@ -1,100 +1,56 @@ // Copyright (c) 2012 Pieter Wuille -// Copyright (c) 2012-2020 The Bitcoin Core developers +// Copyright (c) 2012-2021 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #ifndef BITCOIN_ADDRMAN_H #define BITCOIN_ADDRMAN_H -#include -#include -#include -#include #include #include -#include #include -#include #include -#include -#include -#include +#include +#include #include -#include -#include -#include +#include #include -/** - * Extended statistics about a CAddress - */ -class CAddrInfo : public CAddress +class InvalidAddrManVersionError : public std::ios_base::failure { public: - //! last try whatsoever by us (memory only) - int64_t nLastTry{0}; + InvalidAddrManVersionError(std::string msg) : std::ios_base::failure(msg) { } +}; - //! last counted attempt (memory only) - int64_t nLastCountAttempt{0}; +class AddrManImpl; -private: - //! where knowledge about this address first came from - CNetAddr source; +/** Default for -checkaddrman */ +static constexpr int32_t DEFAULT_ADDRMAN_CONSISTENCY_CHECKS{0}; - //! last successful connection by us - int64_t nLastSuccess{0}; +/** Test-only struct, capturing info about an address in AddrMan */ +struct AddressPosition { + // Whether the address is in the new or tried table + const bool tried; - //! connection attempts since last successful attempt - int nAttempts{0}; + // Addresses in the tried table should always have a multiplicity of 1. + // Addresses in the new table can have multiplicity between 1 and + // ADDRMAN_NEW_BUCKETS_PER_ADDRESS + const int multiplicity; - //! reference count in new sets (memory only) - int nRefCount{0}; + // If the address is in the new table, the bucket and position are + // populated based on the first source who sent the address. + // In certain edge cases, this may not be where the address is currently + // located. + const int bucket; + const int position; - //! in tried set? (memory only) - bool fInTried{false}; - - //! position in vRandom - int nRandomPos{-1}; - - friend class CAddrMan; - -public: - - SERIALIZE_METHODS(CAddrInfo, obj) - { - READWRITEAS(CAddress, obj); - READWRITE(obj.source, obj.nLastSuccess, obj.nAttempts); + bool operator==(AddressPosition other) { + return std::tie(tried, multiplicity, bucket, position) == + std::tie(other.tried, other.multiplicity, other.bucket, other.position); } - - CAddrInfo(const CAddress &addrIn, const CNetAddr &addrSource) : CAddress(addrIn), source(addrSource) - { - } - - CAddrInfo() : CAddress(), source() - { - } - - //! Calculate in which "tried" bucket this entry belongs - int GetTriedBucket(const uint256 &nKey, const std::vector &asmap) const; - - //! Calculate in which "new" bucket this entry belongs, given a certain source - int GetNewBucket(const uint256 &nKey, const CNetAddr& src, const std::vector &asmap) const; - - //! Calculate in which "new" bucket this entry belongs, using its default source - int GetNewBucket(const uint256 &nKey, const std::vector &asmap) const - { - return GetNewBucket(nKey, source, asmap); - } - - //! Calculate in which position of a bucket to store this entry. - int GetBucketPosition(const uint256 &nKey, bool fNew, int nBucket) const; - - //! Determine whether the statistics about this entry are bad enough so that it can just be deleted - bool IsTerrible(int64_t nNow = GetAdjustedTime()) const; - - //! Calculate the relative chance this entry should be given when selecting nodes to connect to - double GetChance(int64_t nNow = GetAdjustedTime()) const; + explicit AddressPosition(bool tried_in, int multiplicity_in, int bucket_in, int position_in) + : tried{tried_in}, multiplicity{multiplicity_in}, bucket{bucket_in}, position{position_in} {} }; /** Stochastic address manager @@ -104,490 +60,90 @@ public: * * Make sure no (localized) attacker can fill the entire table with his nodes/addresses. * * To that end: - * * Addresses are organized into buckets. - * * Addresses that have not yet been tried go into 1024 "new" buckets. - * * Based on the address range (/16 for IPv4) of the source of information, 64 buckets are selected at random. + * * Addresses are organized into buckets that can each store up to 64 entries. + * * Addresses to which our node has not successfully connected go into 1024 "new" buckets. + * * Based on the address range (/16 for IPv4) of the source of information, or if an asmap is provided, + * the AS it belongs to (for IPv4/IPv6), 64 buckets are selected at random. * * The actual bucket is chosen from one of these, based on the range in which the address itself is located. + * * The position in the bucket is chosen based on the full address. * * One single address can occur in up to 8 different buckets to increase selection chances for addresses that * are seen frequently. The chance for increasing this multiplicity decreases exponentially. - * * When adding a new address to a full bucket, a randomly chosen entry (with a bias favoring less recently seen - * ones) is removed from it first. + * * When adding a new address to an occupied position of a bucket, it will not replace the existing entry + * unless that address is also stored in another bucket or it doesn't meet one of several quality criteria + * (see IsTerrible for exact criteria). * * Addresses of nodes that are known to be accessible go into 256 "tried" buckets. * * Each address range selects at random 8 of these buckets. * * The actual bucket is chosen from one of these, based on the full address. - * * When adding a new good address to a full bucket, a randomly chosen entry (with a bias favoring less recently - * tried ones) is evicted from it, back to the "new" buckets. + * * When adding a new good address to an occupied position of a bucket, a FEELER connection to the + * old address is attempted. The old entry is only replaced and moved back to the "new" buckets if this + * attempt was unsuccessful. * * Bucket selection is based on cryptographic hashing, using a randomly-generated 256-bit key, which should not * be observable by adversaries. - * * Several indexes are kept for high performance. Defining DEBUG_ADDRMAN will introduce frequent (and expensive) - * consistency checks for the entire data structure. + * * Several indexes are kept for high performance. Setting m_consistency_check_ratio with the -checkaddrman + * configuration option will introduce (expensive) consistency checks for the entire data structure. */ - -//! total number of buckets for tried addresses -#define ADDRMAN_TRIED_BUCKET_COUNT_LOG2 8 - -//! total number of buckets for new addresses -#define ADDRMAN_NEW_BUCKET_COUNT_LOG2 10 - -//! maximum allowed number of entries in buckets for new and tried addresses -#define ADDRMAN_BUCKET_SIZE_LOG2 6 - -//! over how many buckets entries with tried addresses from a single group (/16 for IPv4) are spread -#define ADDRMAN_TRIED_BUCKETS_PER_GROUP 8 - -//! over how many buckets entries with new addresses originating from a single group are spread -#define ADDRMAN_NEW_BUCKETS_PER_SOURCE_GROUP 64 - -//! in how many buckets for entries with new addresses a single address may occur -#define ADDRMAN_NEW_BUCKETS_PER_ADDRESS 8 - -//! how old addresses can maximally be -#define ADDRMAN_HORIZON_DAYS 30 - -//! after how many failed attempts we give up on a new node -#define ADDRMAN_RETRIES 3 - -//! how many successive failures are allowed ... -#define ADDRMAN_MAX_FAILURES 10 - -//! ... in at least this many days -#define ADDRMAN_MIN_FAIL_DAYS 7 - -//! how recent a successful connection should be before we allow an address to be evicted from tried -#define ADDRMAN_REPLACEMENT_HOURS 4 - -//! Convenience -#define ADDRMAN_TRIED_BUCKET_COUNT (1 << ADDRMAN_TRIED_BUCKET_COUNT_LOG2) -#define ADDRMAN_NEW_BUCKET_COUNT (1 << ADDRMAN_NEW_BUCKET_COUNT_LOG2) -#define ADDRMAN_BUCKET_SIZE (1 << ADDRMAN_BUCKET_SIZE_LOG2) - -//! the maximum number of tried addr collisions to store -#define ADDRMAN_SET_TRIED_COLLISION_SIZE 10 - -//! the maximum time we'll spend trying to resolve a tried table collision, in seconds -static const int64_t ADDRMAN_TEST_WINDOW = 40*60; // 40 minutes - -/** - * Stochastical (IP) address manager - */ -class CAddrMan +class AddrMan { +protected: + const std::unique_ptr m_impl; + public: - // Compressed IP->ASN mapping, loaded from a file when a node starts. - // Should be always empty if no file was provided. - // This mapping is then used for bucketing nodes in Addrman. - // - // If asmap is provided, nodes will be bucketed by - // AS they belong to, in order to make impossible for a node - // to connect to several nodes hosted in a single AS. - // This is done in response to Erebus attack, but also to generally - // diversify the connections every node creates, - // especially useful when a large fraction of nodes - // operate under a couple of cloud providers. - // - // If a new asmap was provided, the existing records - // would be re-bucketed accordingly. - std::vector m_asmap; + explicit AddrMan(std::vector asmap, bool deterministic, int32_t consistency_check_ratio); - // Read asmap from provided binary file - static std::vector DecodeAsmap(fs::path path); - - /** - * Serialized format. - * * format version byte (@see `Format`) - * * lowest compatible format version byte. This is used to help old software decide - * whether to parse the file. For example: - * * Bitcoin Core version N knows how to parse up to format=3. If a new format=4 is - * introduced in version N+1 that is compatible with format=3 and it is known that - * version N will be able to parse it, then version N+1 will write - * (format=4, lowest_compatible=3) in the first two bytes of the file, and so - * version N will still try to parse it. - * * Bitcoin Core version N+2 introduces a new incompatible format=5. It will write - * (format=5, lowest_compatible=5) and so any versions that do not know how to parse - * format=5 will not try to read the file. - * * nKey - * * nNew - * * nTried - * * number of "new" buckets XOR 2**30 - * * all new addresses (total count: nNew) - * * all tried addresses (total count: nTried) - * * for each new bucket: - * * number of elements - * * for each element: index in the serialized "all new addresses" - * * asmap checksum - * - * 2**30 is xorred with the number of buckets to make addrman deserializer v0 detect it - * as incompatible. This is necessary because it did not check the version number on - * deserialization. - * - * vvNew, vvTried, mapInfo, mapAddr and vRandom are never encoded explicitly; - * they are instead reconstructed from the other information. - * - * This format is more complex, but significantly smaller (at most 1.5 MiB), and supports - * changes to the ADDRMAN_ parameters without breaking the on-disk structure. - * - * We don't use SERIALIZE_METHODS since the serialization and deserialization code has - * very little in common. - */ - template - void Serialize(Stream& s_) const - EXCLUSIVE_LOCKS_REQUIRED(!cs) - { - LOCK(cs); - - // Always serialize in the latest version (FILE_FORMAT). - - OverrideStream s(&s_, s_.GetType(), s_.GetVersion() | ADDRV2_FORMAT); - - s << static_cast(FILE_FORMAT); - - // Increment `lowest_compatible` iff a newly introduced format is incompatible with - // the previous one. - static constexpr uint8_t lowest_compatible = Format::V3_BIP155; - s << static_cast(INCOMPATIBILITY_BASE + lowest_compatible); - - s << nKey; - s << nNew; - s << nTried; - - int nUBuckets = ADDRMAN_NEW_BUCKET_COUNT ^ (1 << 30); - s << nUBuckets; - std::unordered_map mapUnkIds; - int nIds = 0; - for (const auto& entry : mapInfo) { - mapUnkIds[entry.first] = nIds; - const CAddrInfo &info = entry.second; - if (info.nRefCount) { - assert(nIds != nNew); // this means nNew was wrong, oh ow - s << info; - nIds++; - } - } - nIds = 0; - for (const auto& entry : mapInfo) { - const CAddrInfo &info = entry.second; - if (info.fInTried) { - assert(nIds != nTried); // this means nTried was wrong, oh ow - s << info; - nIds++; - } - } - for (int bucket = 0; bucket < ADDRMAN_NEW_BUCKET_COUNT; bucket++) { - int nSize = 0; - for (int i = 0; i < ADDRMAN_BUCKET_SIZE; i++) { - if (vvNew[bucket][i] != -1) - nSize++; - } - s << nSize; - for (int i = 0; i < ADDRMAN_BUCKET_SIZE; i++) { - if (vvNew[bucket][i] != -1) { - int nIndex = mapUnkIds[vvNew[bucket][i]]; - s << nIndex; - } - } - } - // Store asmap checksum after bucket entries so that it - // can be ignored by older clients for backward compatibility. - uint256 asmap_checksum; - if (m_asmap.size() != 0) { - asmap_checksum = SerializeHash(m_asmap); - } - s << asmap_checksum; - } + ~AddrMan(); template - void Unserialize(Stream& s_) - EXCLUSIVE_LOCKS_REQUIRED(!cs) - { - LOCK(cs); + void Serialize(Stream& s_) const; - assert(vRandom.empty()); - - Format format; - s_ >> Using>(format); - - int stream_version = s_.GetVersion(); - if (format >= Format::V3_BIP155) { - // Add ADDRV2_FORMAT to the version so that the CNetAddr and CAddress - // unserialize methods know that an address in addrv2 format is coming. - stream_version |= ADDRV2_FORMAT; - } - - OverrideStream s(&s_, s_.GetType(), stream_version); - - uint8_t compat; - s >> compat; - const uint8_t lowest_compatible = compat - INCOMPATIBILITY_BASE; - if (lowest_compatible > FILE_FORMAT) { - throw std::ios_base::failure(strprintf( - "Unsupported format of addrman database: %u. It is compatible with formats >=%u, " - "but the maximum supported by this version of %s is %u.", - format, lowest_compatible, PACKAGE_NAME, static_cast(FILE_FORMAT))); - } - - s >> nKey; - s >> nNew; - s >> nTried; - int nUBuckets = 0; - s >> nUBuckets; - if (format >= Format::V1_DETERMINISTIC) { - nUBuckets ^= (1 << 30); - } - - if (nNew > ADDRMAN_NEW_BUCKET_COUNT * ADDRMAN_BUCKET_SIZE || nNew < 0) { - throw std::ios_base::failure( - strprintf("Corrupt CAddrMan serialization: nNew=%d, should be in [0, %u]", - nNew, - ADDRMAN_NEW_BUCKET_COUNT * ADDRMAN_BUCKET_SIZE)); - } - - if (nTried > ADDRMAN_TRIED_BUCKET_COUNT * ADDRMAN_BUCKET_SIZE || nTried < 0) { - throw std::ios_base::failure( - strprintf("Corrupt CAddrMan serialization: nTried=%d, should be in [0, %u]", - nTried, - ADDRMAN_TRIED_BUCKET_COUNT * ADDRMAN_BUCKET_SIZE)); - } - - // Deserialize entries from the new table. - for (int n = 0; n < nNew; n++) { - CAddrInfo &info = mapInfo[n]; - s >> info; - mapAddr[info] = n; - info.nRandomPos = vRandom.size(); - vRandom.push_back(n); - } - nIdCount = nNew; - - // Deserialize entries from the tried table. - int nLost = 0; - for (int n = 0; n < nTried; n++) { - CAddrInfo info; - s >> info; - int nKBucket = info.GetTriedBucket(nKey, m_asmap); - int nKBucketPos = info.GetBucketPosition(nKey, false, nKBucket); - if (vvTried[nKBucket][nKBucketPos] == -1) { - info.nRandomPos = vRandom.size(); - info.fInTried = true; - vRandom.push_back(nIdCount); - mapInfo[nIdCount] = info; - mapAddr[info] = nIdCount; - vvTried[nKBucket][nKBucketPos] = nIdCount; - nIdCount++; - } else { - nLost++; - } - } - nTried -= nLost; - - // Store positions in the new table buckets to apply later (if possible). - // An entry may appear in up to ADDRMAN_NEW_BUCKETS_PER_ADDRESS buckets, - // so we store all bucket-entry_index pairs to iterate through later. - std::vector> bucket_entries; - - for (int bucket = 0; bucket < nUBuckets; ++bucket) { - int num_entries{0}; - s >> num_entries; - for (int n = 0; n < num_entries; ++n) { - int entry_index{0}; - s >> entry_index; - if (entry_index >= 0 && entry_index < nNew) { - bucket_entries.emplace_back(bucket, entry_index); - } - } - } - - // If the bucket count and asmap checksum haven't changed, then attempt - // to restore the entries to the buckets/positions they were in before - // serialization. - uint256 supplied_asmap_checksum; - if (m_asmap.size() != 0) { - supplied_asmap_checksum = SerializeHash(m_asmap); - } - uint256 serialized_asmap_checksum; - if (format >= Format::V2_ASMAP) { - s >> serialized_asmap_checksum; - } - const bool restore_bucketing{nUBuckets == ADDRMAN_NEW_BUCKET_COUNT && - serialized_asmap_checksum == supplied_asmap_checksum}; - - if (!restore_bucketing) { - LogPrint(BCLog::ADDRMAN, "Bucketing method was updated, re-bucketing addrman entries from disk\n"); - } - - for (auto bucket_entry : bucket_entries) { - int bucket{bucket_entry.first}; - const int entry_index{bucket_entry.second}; - CAddrInfo& info = mapInfo[entry_index]; - - // The entry shouldn't appear in more than - // ADDRMAN_NEW_BUCKETS_PER_ADDRESS. If it has already, just skip - // this bucket_entry. - if (info.nRefCount >= ADDRMAN_NEW_BUCKETS_PER_ADDRESS) continue; - - int bucket_position = info.GetBucketPosition(nKey, true, bucket); - if (restore_bucketing && vvNew[bucket][bucket_position] == -1) { - // Bucketing has not changed, using existing bucket positions for the new table - vvNew[bucket][bucket_position] = entry_index; - ++info.nRefCount; - } else { - // In case the new table data cannot be used (bucket count wrong or new asmap), - // try to give them a reference based on their primary source address. - bucket = info.GetNewBucket(nKey, m_asmap); - bucket_position = info.GetBucketPosition(nKey, true, bucket); - if (vvNew[bucket][bucket_position] == -1) { - vvNew[bucket][bucket_position] = entry_index; - ++info.nRefCount; - } - } - } - - // Prune new entries with refcount 0 (as a result of collisions). - int nLostUnk = 0; - for (auto it = mapInfo.cbegin(); it != mapInfo.cend(); ) { - if (it->second.fInTried == false && it->second.nRefCount == 0) { - const auto itCopy = it++; - Delete(itCopy->first); - ++nLostUnk; - } else { - ++it; - } - } - if (nLost + nLostUnk > 0) { - LogPrint(BCLog::ADDRMAN, "addrman lost %i new and %i tried addresses due to collisions\n", nLostUnk, nLost); - } - - RemoveInvalid(); - - Check(); - } - - void Clear() - EXCLUSIVE_LOCKS_REQUIRED(!cs) - { - LOCK(cs); - std::vector().swap(vRandom); - nKey = insecure_rand.rand256(); - for (size_t bucket = 0; bucket < ADDRMAN_NEW_BUCKET_COUNT; bucket++) { - for (size_t entry = 0; entry < ADDRMAN_BUCKET_SIZE; entry++) { - vvNew[bucket][entry] = -1; - } - } - for (size_t bucket = 0; bucket < ADDRMAN_TRIED_BUCKET_COUNT; bucket++) { - for (size_t entry = 0; entry < ADDRMAN_BUCKET_SIZE; entry++) { - vvTried[bucket][entry] = -1; - } - } - - nIdCount = 0; - nTried = 0; - nNew = 0; - nLastGood = 1; //Initially at 1 so that "never" is strictly worse. - mapInfo.clear(); - mapAddr.clear(); - } - - CAddrMan() - { - Clear(); - } - - ~CAddrMan() - { - nKey.SetNull(); - } + template + void Unserialize(Stream& s_); //! Return the number of (unique) addresses in all tables. - size_t size() const - EXCLUSIVE_LOCKS_REQUIRED(!cs) - { - LOCK(cs); // TODO: Cache this in an atomic to avoid this overhead - return vRandom.size(); - } + size_t size() const; - //! Add a single address. - bool Add(const CAddress &addr, const CNetAddr& source, int64_t nTimePenalty = 0) - EXCLUSIVE_LOCKS_REQUIRED(!cs) - { - LOCK(cs); - bool fRet = false; - Check(); - fRet |= Add_(addr, source, nTimePenalty); - Check(); - if (fRet) { - LogPrint(BCLog::ADDRMAN, "Added %s from %s: %i tried, %i new\n", addr.ToStringIPPort(), source.ToString(), nTried, nNew); - } - return fRet; - } + /** + * Attempt to add one or more addresses to addrman's new table. + * + * @param[in] vAddr Address records to attempt to add. + * @param[in] source The address of the node that sent us these addr records. + * @param[in] nTimePenalty A "time penalty" to apply to the address record's nTime. If a peer + * sends us an address record with nTime=n, then we'll add it to our + * addrman with nTime=(n - nTimePenalty). + * @return true if at least one address is successfully added. */ + bool Add(const std::vector& vAddr, const CNetAddr& source, int64_t nTimePenalty = 0); - //! Add multiple addresses. - bool Add(const std::vector &vAddr, const CNetAddr& source, int64_t nTimePenalty = 0) - EXCLUSIVE_LOCKS_REQUIRED(!cs) - { - LOCK(cs); - int nAdd = 0; - Check(); - for (std::vector::const_iterator it = vAddr.begin(); it != vAddr.end(); it++) - nAdd += Add_(*it, source, nTimePenalty) ? 1 : 0; - Check(); - if (nAdd) { - LogPrint(BCLog::ADDRMAN, "Added %i addresses from %s: %i tried, %i new\n", nAdd, source.ToString(), nTried, nNew); - } - return nAdd > 0; - } - - //! Mark an entry as accessible. - void Good(const CService &addr, bool test_before_evict = true, int64_t nTime = GetAdjustedTime()) - EXCLUSIVE_LOCKS_REQUIRED(!cs) - { - LOCK(cs); - Check(); - Good_(addr, test_before_evict, nTime); - Check(); - } + /** + * Mark an address record as accessible and attempt to move it to addrman's tried table. + * + * @param[in] addr Address record to attempt to move to tried table. + * @param[in] nTime The time that we were last connected to this peer. + * @return true if the address is successfully moved from the new table to the tried table. + */ + bool Good(const CService& addr, int64_t nTime = GetAdjustedTime()); //! Mark an entry as connection attempted to. - void Attempt(const CService &addr, bool fCountFailure, int64_t nTime = GetAdjustedTime()) - EXCLUSIVE_LOCKS_REQUIRED(!cs) - { - LOCK(cs); - Check(); - Attempt_(addr, fCountFailure, nTime); - Check(); - } + void Attempt(const CService& addr, bool fCountFailure, int64_t nTime = GetAdjustedTime()); //! See if any to-be-evicted tried table entries have been tested and if so resolve the collisions. - void ResolveCollisions() - EXCLUSIVE_LOCKS_REQUIRED(!cs) - { - LOCK(cs); - Check(); - ResolveCollisions_(); - Check(); - } + void ResolveCollisions(); - //! Randomly select an address in tried that another address is attempting to evict. - CAddrInfo SelectTriedCollision() - EXCLUSIVE_LOCKS_REQUIRED(!cs) - { - LOCK(cs); - Check(); - const CAddrInfo ret = SelectTriedCollision_(); - Check(); - return ret; - } + /** + * Randomly select an address in the tried table that another address is + * attempting to evict. + * + * @return CAddress The record for the selected tried peer. + * int64_t The last time we attempted to connect to that peer. + */ + std::pair SelectTriedCollision(); /** * Choose an address to connect to. + * + * @param[in] newOnly Whether to only select addresses from the new table. + * @return CAddress The record for the selected peer. + * int64_t The last time we attempted to connect to that peer. */ - CAddrInfo Select(bool newOnly = false) - EXCLUSIVE_LOCKS_REQUIRED(!cs) - { - LOCK(cs); - Check(); - const CAddrInfo addrRet = Select_(newOnly); - Check(); - return addrRet; - } + std::pair Select(bool newOnly = false) const; /** * Return all or many randomly selected addresses, optionally by network. @@ -595,164 +151,10 @@ public: * @param[in] max_addresses Maximum number of addresses to return (0 = all). * @param[in] max_pct Maximum percentage of addresses to return (0 = all). * @param[in] network Select only addresses of this network (nullopt = all). - */ - std::vector GetAddr(size_t max_addresses, size_t max_pct, std::optional network) - EXCLUSIVE_LOCKS_REQUIRED(!cs) - { - LOCK(cs); - Check(); - std::vector vAddr; - GetAddr_(vAddr, max_addresses, max_pct, network); - Check(); - return vAddr; - } - - //! Outer function for Connected_() - void Connected(const CService &addr, int64_t nTime = GetAdjustedTime()) - EXCLUSIVE_LOCKS_REQUIRED(!cs) - { - LOCK(cs); - Check(); - Connected_(addr, nTime); - Check(); - } - - void SetServices(const CService &addr, ServiceFlags nServices) - EXCLUSIVE_LOCKS_REQUIRED(!cs) - { - LOCK(cs); - Check(); - SetServices_(addr, nServices); - Check(); - } - -protected: - //! secret key to randomize bucket select with - uint256 nKey; - - //! Source of random numbers for randomization in inner loops - FastRandomContext insecure_rand; - -private: - //! A mutex to protect the inner data structures. - mutable Mutex cs; - - //! Serialization versions. - enum Format : uint8_t { - V0_HISTORICAL = 0, //!< historic format, before commit e6b343d88 - V1_DETERMINISTIC = 1, //!< for pre-asmap files - V2_ASMAP = 2, //!< for files including asmap version - V3_BIP155 = 3, //!< same as V2_ASMAP plus addresses are in BIP155 format - }; - - //! The maximum format this software knows it can unserialize. Also, we always serialize - //! in this format. - //! The format (first byte in the serialized stream) can be higher than this and - //! still this software may be able to unserialize the file - if the second byte - //! (see `lowest_compatible` in `Unserialize()`) is less or equal to this. - static constexpr Format FILE_FORMAT = Format::V3_BIP155; - - //! The initial value of a field that is incremented every time an incompatible format - //! change is made (such that old software versions would not be able to parse and - //! understand the new file format). This is 32 because we overtook the "key size" - //! field which was 32 historically. - //! @note Don't increment this. Increment `lowest_compatible` in `Serialize()` instead. - static constexpr uint8_t INCOMPATIBILITY_BASE = 32; - - //! last used nId - int nIdCount GUARDED_BY(cs); - - //! table with information about all nIds - std::unordered_map mapInfo GUARDED_BY(cs); - - //! find an nId based on its network address - std::unordered_map mapAddr GUARDED_BY(cs); - - //! randomly-ordered vector of all nIds - std::vector vRandom GUARDED_BY(cs); - - // number of "tried" entries - int nTried GUARDED_BY(cs); - - //! list of "tried" buckets - int vvTried[ADDRMAN_TRIED_BUCKET_COUNT][ADDRMAN_BUCKET_SIZE] GUARDED_BY(cs); - - //! number of (unique) "new" entries - int nNew GUARDED_BY(cs); - - //! list of "new" buckets - int vvNew[ADDRMAN_NEW_BUCKET_COUNT][ADDRMAN_BUCKET_SIZE] GUARDED_BY(cs); - - //! last time Good was called (memory only) - int64_t nLastGood GUARDED_BY(cs); - - //! Holds addrs inserted into tried table that collide with existing entries. Test-before-evict discipline used to resolve these collisions. - std::set m_tried_collisions; - - //! Find an entry. - CAddrInfo* Find(const CNetAddr& addr, int *pnId = nullptr) EXCLUSIVE_LOCKS_REQUIRED(cs); - - //! find an entry, creating it if necessary. - //! nTime and nServices of the found node are updated, if necessary. - CAddrInfo* Create(const CAddress &addr, const CNetAddr &addrSource, int *pnId = nullptr) EXCLUSIVE_LOCKS_REQUIRED(cs); - - //! Swap two elements in vRandom. - void SwapRandom(unsigned int nRandomPos1, unsigned int nRandomPos2) EXCLUSIVE_LOCKS_REQUIRED(cs); - - //! Move an entry from the "new" table(s) to the "tried" table - void MakeTried(CAddrInfo& info, int nId) EXCLUSIVE_LOCKS_REQUIRED(cs); - - //! Delete an entry. It must not be in tried, and have refcount 0. - void Delete(int nId) EXCLUSIVE_LOCKS_REQUIRED(cs); - - //! Clear a position in a "new" table. This is the only place where entries are actually deleted. - void ClearNew(int nUBucket, int nUBucketPos) EXCLUSIVE_LOCKS_REQUIRED(cs); - - //! Mark an entry "good", possibly moving it from "new" to "tried". - void Good_(const CService &addr, bool test_before_evict, int64_t time) EXCLUSIVE_LOCKS_REQUIRED(cs); - - //! Add an entry to the "new" table. - bool Add_(const CAddress &addr, const CNetAddr& source, int64_t nTimePenalty) EXCLUSIVE_LOCKS_REQUIRED(cs); - - //! Mark an entry as attempted to connect. - void Attempt_(const CService &addr, bool fCountFailure, int64_t nTime) EXCLUSIVE_LOCKS_REQUIRED(cs); - - //! Select an address to connect to, if newOnly is set to true, only the new table is selected from. - CAddrInfo Select_(bool newOnly) EXCLUSIVE_LOCKS_REQUIRED(cs); - - //! See if any to-be-evicted tried table entries have been tested and if so resolve the collisions. - void ResolveCollisions_() EXCLUSIVE_LOCKS_REQUIRED(cs); - - //! Return a random to-be-evicted tried table address. - CAddrInfo SelectTriedCollision_() EXCLUSIVE_LOCKS_REQUIRED(cs); - - //! Consistency check - void Check() - EXCLUSIVE_LOCKS_REQUIRED(cs) - { -#ifdef DEBUG_ADDRMAN - AssertLockHeld(cs); - const int err = Check_(); - if (err) { - LogPrintf("ADDRMAN CONSISTENCY CHECK FAILED!!! err=%i\n", err); - } -#endif - } - -#ifdef DEBUG_ADDRMAN - //! Perform consistency check. Returns an error code or zero. - int Check_() EXCLUSIVE_LOCKS_REQUIRED(cs); -#endif - - /** - * Return all or many randomly selected addresses, optionally by network. * - * @param[out] vAddr Vector of randomly selected addresses from vRandom. - * @param[in] max_addresses Maximum number of addresses to return (0 = all). - * @param[in] max_pct Maximum percentage of addresses to return (0 = all). - * @param[in] network Select only addresses of this network (nullopt = all). + * @return A vector of randomly selected addresses from vRandom. */ - void GetAddr_(std::vector& vAddr, size_t max_addresses, size_t max_pct, std::optional network) EXCLUSIVE_LOCKS_REQUIRED(cs); + std::vector GetAddr(size_t max_addresses, size_t max_pct, std::optional network) const; /** We have successfully connected to this peer. Calling this function * updates the CAddress's nTime, which is used in our IsTerrible() @@ -765,15 +167,21 @@ private: * @param[in] addr The address of the peer we were connected to * @param[in] nTime The time that we were last connected to this peer */ - void Connected_(const CService& addr, int64_t nTime) EXCLUSIVE_LOCKS_REQUIRED(cs); + void Connected(const CService& addr, int64_t nTime = GetAdjustedTime()); //! Update an entry's service bits. - void SetServices_(const CService &addr, ServiceFlags nServices) EXCLUSIVE_LOCKS_REQUIRED(cs); + void SetServices(const CService& addr, ServiceFlags nServices); - //! Remove invalid addresses. - void RemoveInvalid() EXCLUSIVE_LOCKS_REQUIRED(cs); + const std::vector& GetAsmap() const; - friend class CAddrManTest; + /** Test-only function + * Find the address record in AddrMan and return information about its + * position. + * @param[in] addr The address record to look up. + * @return Information about the address record in AddrMan + * or nullopt if address is not found. + */ + std::optional FindAddressEntry(const CAddress& addr); }; #endif // BITCOIN_ADDRMAN_H diff --git a/src/addrman_impl.h b/src/addrman_impl.h new file mode 100644 index 0000000000..5e76f72342 --- /dev/null +++ b/src/addrman_impl.h @@ -0,0 +1,282 @@ +// Copyright (c) 2021 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_ADDRMAN_IMPL_H +#define BITCOIN_ADDRMAN_IMPL_H + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +/** Total number of buckets for tried addresses */ +static constexpr int32_t ADDRMAN_TRIED_BUCKET_COUNT_LOG2{8}; +static constexpr int ADDRMAN_TRIED_BUCKET_COUNT{1 << ADDRMAN_TRIED_BUCKET_COUNT_LOG2}; +/** Total number of buckets for new addresses */ +static constexpr int32_t ADDRMAN_NEW_BUCKET_COUNT_LOG2{10}; +static constexpr int ADDRMAN_NEW_BUCKET_COUNT{1 << ADDRMAN_NEW_BUCKET_COUNT_LOG2}; +/** Maximum allowed number of entries in buckets for new and tried addresses */ +static constexpr int32_t ADDRMAN_BUCKET_SIZE_LOG2{6}; +static constexpr int ADDRMAN_BUCKET_SIZE{1 << ADDRMAN_BUCKET_SIZE_LOG2}; + +/** + * Extended statistics about a CAddress + */ +class AddrInfo : public CAddress +{ +public: + //! last try whatsoever by us (memory only) + int64_t nLastTry{0}; + + //! last counted attempt (memory only) + int64_t nLastCountAttempt{0}; + + //! where knowledge about this address first came from + CNetAddr source; + + //! last successful connection by us + int64_t nLastSuccess{0}; + + //! connection attempts since last successful attempt + int nAttempts{0}; + + //! reference count in new sets (memory only) + int nRefCount{0}; + + //! in tried set? (memory only) + bool fInTried{false}; + + //! position in vRandom + mutable int nRandomPos{-1}; + + SERIALIZE_METHODS(AddrInfo, obj) + { + READWRITEAS(CAddress, obj); + READWRITE(obj.source, obj.nLastSuccess, obj.nAttempts); + } + + AddrInfo(const CAddress &addrIn, const CNetAddr &addrSource) : CAddress(addrIn), source(addrSource) + { + } + + AddrInfo() : CAddress(), source() + { + } + + //! Calculate in which "tried" bucket this entry belongs + int GetTriedBucket(const uint256 &nKey, const std::vector &asmap) const; + + //! Calculate in which "new" bucket this entry belongs, given a certain source + int GetNewBucket(const uint256 &nKey, const CNetAddr& src, const std::vector &asmap) const; + + //! Calculate in which "new" bucket this entry belongs, using its default source + int GetNewBucket(const uint256 &nKey, const std::vector &asmap) const + { + return GetNewBucket(nKey, source, asmap); + } + + //! Calculate in which position of a bucket to store this entry. + int GetBucketPosition(const uint256 &nKey, bool fNew, int nBucket) const; + + //! Determine whether the statistics about this entry are bad enough so that it can just be deleted + bool IsTerrible(int64_t nNow = GetAdjustedTime()) const; + + //! Calculate the relative chance this entry should be given when selecting nodes to connect to + double GetChance(int64_t nNow = GetAdjustedTime()) const; +}; + +class AddrManImpl +{ +public: + AddrManImpl(std::vector&& asmap, bool deterministic, int32_t consistency_check_ratio); + + ~AddrManImpl(); + + template + void Serialize(Stream& s_) const EXCLUSIVE_LOCKS_REQUIRED(!cs); + + template + void Unserialize(Stream& s_) EXCLUSIVE_LOCKS_REQUIRED(!cs); + + size_t size() const EXCLUSIVE_LOCKS_REQUIRED(!cs); + + bool Add(const std::vector& vAddr, const CNetAddr& source, int64_t nTimePenalty) + EXCLUSIVE_LOCKS_REQUIRED(!cs); + + bool Good(const CService& addr, int64_t nTime) + EXCLUSIVE_LOCKS_REQUIRED(!cs); + + void Attempt(const CService& addr, bool fCountFailure, int64_t nTime) + EXCLUSIVE_LOCKS_REQUIRED(!cs); + + void ResolveCollisions() EXCLUSIVE_LOCKS_REQUIRED(!cs); + + std::pair SelectTriedCollision() EXCLUSIVE_LOCKS_REQUIRED(!cs); + + std::pair Select(bool newOnly) const + EXCLUSIVE_LOCKS_REQUIRED(!cs); + + std::vector GetAddr(size_t max_addresses, size_t max_pct, std::optional network) const + EXCLUSIVE_LOCKS_REQUIRED(!cs); + + void Connected(const CService& addr, int64_t nTime) + EXCLUSIVE_LOCKS_REQUIRED(!cs); + + void SetServices(const CService& addr, ServiceFlags nServices) + EXCLUSIVE_LOCKS_REQUIRED(!cs); + + std::optional FindAddressEntry(const CAddress& addr) + EXCLUSIVE_LOCKS_REQUIRED(!cs); + + const std::vector& GetAsmap() const; + + friend class AddrManDeterministic; + +private: + //! A mutex to protect the inner data structures. + mutable Mutex cs; + + //! Source of random numbers for randomization in inner loops + mutable FastRandomContext insecure_rand GUARDED_BY(cs); + + //! secret key to randomize bucket select with + uint256 nKey; + + //! Serialization versions. + enum Format : uint8_t { + V0_HISTORICAL = 0, //!< historic format, before commit e6b343d88 + V1_DETERMINISTIC = 1, //!< for pre-asmap files + V2_ASMAP = 2, //!< for files including asmap version + V3_BIP155 = 3, //!< same as V2_ASMAP plus addresses are in BIP155 format + V4_MULTIPORT = 4, //!< adds support for multiple ports per IP + }; + + //! The maximum format this software knows it can unserialize. Also, we always serialize + //! in this format. + //! The format (first byte in the serialized stream) can be higher than this and + //! still this software may be able to unserialize the file - if the second byte + //! (see `lowest_compatible` in `Unserialize()`) is less or equal to this. + static constexpr Format FILE_FORMAT = Format::V4_MULTIPORT; + + //! The initial value of a field that is incremented every time an incompatible format + //! change is made (such that old software versions would not be able to parse and + //! understand the new file format). This is 32 because we overtook the "key size" + //! field which was 32 historically. + //! @note Don't increment this. Increment `lowest_compatible` in `Serialize()` instead. + static constexpr uint8_t INCOMPATIBILITY_BASE = 32; + + //! last used nId + int nIdCount GUARDED_BY(cs){0}; + + //! table with information about all nIds + std::unordered_map mapInfo GUARDED_BY(cs); + + //! find an nId based on its network address and port. + std::unordered_map mapAddr GUARDED_BY(cs); + + //! randomly-ordered vector of all nIds + //! This is mutable because it is unobservable outside the class, so any + //! changes to it (even in const methods) are also unobservable. + mutable std::vector vRandom GUARDED_BY(cs); + + // number of "tried" entries + int nTried GUARDED_BY(cs){0}; + + //! list of "tried" buckets + int vvTried[ADDRMAN_TRIED_BUCKET_COUNT][ADDRMAN_BUCKET_SIZE] GUARDED_BY(cs); + + //! number of (unique) "new" entries + int nNew GUARDED_BY(cs){0}; + + //! list of "new" buckets + int vvNew[ADDRMAN_NEW_BUCKET_COUNT][ADDRMAN_BUCKET_SIZE] GUARDED_BY(cs); + + //! last time Good was called (memory only). Initially set to 1 so that "never" is strictly worse. + int64_t nLastGood GUARDED_BY(cs){1}; + + //! Holds addrs inserted into tried table that collide with existing entries. Test-before-evict discipline used to resolve these collisions. + std::set m_tried_collisions; + + /** Perform consistency checks every m_consistency_check_ratio operations (if non-zero). */ + const int32_t m_consistency_check_ratio; + + // Compressed IP->ASN mapping, loaded from a file when a node starts. + // Should be always empty if no file was provided. + // This mapping is then used for bucketing nodes in Addrman. + // + // If asmap is provided, nodes will be bucketed by + // AS they belong to, in order to make impossible for a node + // to connect to several nodes hosted in a single AS. + // This is done in response to Erebus attack, but also to generally + // diversify the connections every node creates, + // especially useful when a large fraction of nodes + // operate under a couple of cloud providers. + // + // If a new asmap was provided, the existing records + // would be re-bucketed accordingly. + const std::vector m_asmap; + + //! Find an entry. + AddrInfo* Find(const CService& addr, int* pnId = nullptr) EXCLUSIVE_LOCKS_REQUIRED(cs); + + //! Create a new entry and add it to the internal data structures mapInfo, mapAddr and vRandom. + AddrInfo* Create(const CAddress& addr, const CNetAddr& addrSource, int* pnId = nullptr) EXCLUSIVE_LOCKS_REQUIRED(cs); + + //! Swap two elements in vRandom. + void SwapRandom(unsigned int nRandomPos1, unsigned int nRandomPos2) const EXCLUSIVE_LOCKS_REQUIRED(cs); + + //! Delete an entry. It must not be in tried, and have refcount 0. + void Delete(int nId) EXCLUSIVE_LOCKS_REQUIRED(cs); + + //! Clear a position in a "new" table. This is the only place where entries are actually deleted. + void ClearNew(int nUBucket, int nUBucketPos) EXCLUSIVE_LOCKS_REQUIRED(cs); + + //! Move an entry from the "new" table(s) to the "tried" table + void MakeTried(AddrInfo& info, int nId) EXCLUSIVE_LOCKS_REQUIRED(cs); + + /** Attempt to add a single address to addrman's new table. + * @see AddrMan::Add() for parameters. */ + bool AddSingle(const CAddress& addr, const CNetAddr& source, int64_t nTimePenalty) EXCLUSIVE_LOCKS_REQUIRED(cs); + + bool Good_(const CService& addr, bool test_before_evict, int64_t time) EXCLUSIVE_LOCKS_REQUIRED(cs); + + bool Add_(const std::vector &vAddr, const CNetAddr& source, int64_t nTimePenalty) EXCLUSIVE_LOCKS_REQUIRED(cs); + + void Attempt_(const CService& addr, bool fCountFailure, int64_t nTime) EXCLUSIVE_LOCKS_REQUIRED(cs); + + std::pair Select_(bool newOnly) const EXCLUSIVE_LOCKS_REQUIRED(cs); + + std::vector GetAddr_(size_t max_addresses, size_t max_pct, std::optional network) const EXCLUSIVE_LOCKS_REQUIRED(cs); + + void Connected_(const CService& addr, int64_t nTime) EXCLUSIVE_LOCKS_REQUIRED(cs); + + void SetServices_(const CService& addr, ServiceFlags nServices) EXCLUSIVE_LOCKS_REQUIRED(cs); + + void ResolveCollisions_() EXCLUSIVE_LOCKS_REQUIRED(cs); + + std::pair SelectTriedCollision_() EXCLUSIVE_LOCKS_REQUIRED(cs); + + std::optional FindAddressEntry_(const CAddress& addr) EXCLUSIVE_LOCKS_REQUIRED(cs); + + //! Consistency check, taking into account m_consistency_check_ratio. + //! Will std::abort if an inconsistency is detected. + void Check() const EXCLUSIVE_LOCKS_REQUIRED(cs); + + //! Perform consistency check, regardless of m_consistency_check_ratio. + //! @returns an error code or zero. + int CheckAddrman() const EXCLUSIVE_LOCKS_REQUIRED(cs); +}; + +#endif // BITCOIN_ADDRMAN_IMPL_H diff --git a/src/arith_uint256.cpp b/src/arith_uint256.cpp index 0bebb0cf54..f7f62dfc68 100644 --- a/src/arith_uint256.cpp +++ b/src/arith_uint256.cpp @@ -96,7 +96,7 @@ base_uint& base_uint::operator/=(const base_uint& b) while (shift >= 0) { if (num >= div) { num -= div; - pn[shift / 32] |= (1 << (shift & 31)); // set a bit of the result. + pn[shift / 32] |= (1U << (shift & 31)); // set a bit of the result. } div >>= 1; // shift back. shift--; @@ -236,7 +236,7 @@ uint32_t arith_uint256::GetCompact(bool fNegative) const nCompact >>= 8; nSize++; } - assert((nCompact & ~0x007fffff) == 0); + assert((nCompact & ~0x007fffffU) == 0); assert(nSize < 256); nCompact |= nSize << 24; nCompact |= (fNegative && (nCompact & 0x007fffff) ? 0x00800000 : 0); diff --git a/src/asset.cpp b/src/asset.cpp index 70c5a91154..36be0adcd2 100644 --- a/src/asset.cpp +++ b/src/asset.cpp @@ -155,3 +155,11 @@ CAmount valueFor(const CAmountMap& mapValue, const CAsset& asset) { return CAmount(0); } } + +std::ostream& operator<<(std::ostream& out, const CAmountMap& map) +{ + for(std::map::const_iterator it = map.begin(); it != map.end(); ++it) { + out << it->first.GetHex() << ": " << it->second << ", "; + } + return out; +} diff --git a/src/asset.h b/src/asset.h index 6dc36f0fc3..ceb1621625 100644 --- a/src/asset.h +++ b/src/asset.h @@ -1,11 +1,11 @@ - #ifndef BITCOIN_ASSET_H #define BITCOIN_ASSET_H #include -#include +#include #include +#include /** * Native Asset Issuance @@ -101,4 +101,6 @@ inline bool MoneyRange(const CAmountMap& mapValue) { CAmount valueFor(const CAmountMap& mapValue, const CAsset& asset); -#endif // BITCOIN_AMOUNT_H +std::ostream& operator<<(std::ostream& out, const CAmountMap& map); + +#endif // BITCOIN_ASSET_H diff --git a/src/banman.cpp b/src/banman.cpp index d2437e6733..b28e3f7f7c 100644 --- a/src/banman.cpp +++ b/src/banman.cpp @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2020 The Bitcoin Core developers +// Copyright (c) 2009-2021 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -7,6 +7,7 @@ #include #include +#include #include #include #include @@ -18,7 +19,7 @@ BanMan::BanMan(fs::path ban_file, CClientUIInterface* client_interface, int64_t if (m_client_interface) m_client_interface->InitMessage(_("Loading banlist…").translated); int64_t n_start = GetTimeMillis(); - if (m_ban_db.Read(m_banned, m_is_dirty)) { + if (m_ban_db.Read(m_banned)) { SweepBanned(); // sweep out unused entries LogPrint(BCLog::NET, "Loaded %d banned node addresses/subnets %dms\n", m_banned.size(), @@ -39,18 +40,23 @@ BanMan::~BanMan() void BanMan::DumpBanlist() { - SweepBanned(); // clean unused entries (if bantime has expired) - - if (!BannedSetIsDirty()) return; - - int64_t n_start = GetTimeMillis(); + static Mutex dump_mutex; + LOCK(dump_mutex); banmap_t banmap; - GetBanned(banmap); - if (m_ban_db.Write(banmap)) { + { + LOCK(m_cs_banned); + SweepBanned(); + if (!BannedSetIsDirty()) return; + banmap = m_banned; SetBannedSetDirty(false); } + int64_t n_start = GetTimeMillis(); + if (!m_ban_db.Write(banmap)) { + SetBannedSetDirty(true); + } + LogPrint(BCLog::NET, "Flushed %d banned node addresses/subnets to disk %dms\n", banmap.size(), GetTimeMillis() - n_start); } diff --git a/src/banman.h b/src/banman.h index 8c75d4037e..f268fffa5a 100644 --- a/src/banman.h +++ b/src/banman.h @@ -1,12 +1,12 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2020 The Bitcoin Core developers +// Copyright (c) 2009-2021 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #ifndef BITCOIN_BANMAN_H #define BITCOIN_BANMAN_H #include -#include +#include #include #include // For banmap_t #include @@ -88,11 +88,11 @@ private: RecursiveMutex m_cs_banned; banmap_t m_banned GUARDED_BY(m_cs_banned); - bool m_is_dirty GUARDED_BY(m_cs_banned); + bool m_is_dirty GUARDED_BY(m_cs_banned){false}; CClientUIInterface* m_client_interface = nullptr; CBanDB m_ban_db; const int64_t m_default_ban_time; CRollingBloomFilter m_discouraged GUARDED_BY(m_cs_banned) {50000, 0.000001}; }; -#endif +#endif // BITCOIN_BANMAN_H diff --git a/src/base58.cpp b/src/base58.cpp index fb04673c5c..dfa2e8db55 100644 --- a/src/base58.cpp +++ b/src/base58.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2020 The Bitcoin Core developers +// Copyright (c) 2014-2021 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -149,7 +149,7 @@ std::string EncodeBase58Check(Span input) return false; } // re-calculate the checksum, ensure it matches the included 4-byte checksum - uint256 hash = Hash(MakeSpan(vchRet).first(vchRet.size() - 4)); + uint256 hash = Hash(Span{vchRet}.first(vchRet.size() - 4)); if (memcmp(&hash, &vchRet[vchRet.size() - 4], 4) != 0) { vchRet.clear(); return false; diff --git a/src/bech32.cpp b/src/bech32.cpp index 5b2eb51e48..70b736621b 100644 --- a/src/bech32.cpp +++ b/src/bech32.cpp @@ -1,11 +1,15 @@ // Copyright (c) 2017, 2021 Pieter Wuille +// Copyright (c) 2021 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include #include +#include #include +#include +#include namespace bech32 { @@ -27,6 +31,90 @@ const int8_t CHARSET_REV[128] = { 1, 0, 3, 16, 11, 28, 12, 14, 6, 4, 2, -1, -1, -1, -1, -1 }; +/** We work with the finite field GF(1024) defined as a degree 2 extension of the base field GF(32) + * The defining polynomial of the extension is x^2 + 9x + 23. + * Let (e) be a root of this defining polynomial. Then (e) is a primitive element of GF(1024), + * that is, a generator of the field. Every non-zero element of the field can then be represented + * as (e)^k for some power k. + * The array GF1024_EXP contains all these powers of (e) - GF1024_EXP[k] = (e)^k in GF(1024). + * Conversely, GF1024_LOG contains the discrete logarithms of these powers, so + * GF1024_LOG[GF1024_EXP[k]] == k. + * The following function generates the two tables GF1024_EXP and GF1024_LOG as constexprs. */ +constexpr std::pair, std::array> GenerateGFTables() +{ + // Build table for GF(32). + // We use these tables to perform arithmetic in GF(32) below, when constructing the + // tables for GF(1024). + std::array GF32_EXP{}; + std::array GF32_LOG{}; + + // fmod encodes the defining polynomial of GF(32) over GF(2), x^5 + x^3 + 1. + // Because coefficients in GF(2) are binary digits, the coefficients are packed as 101001. + const int fmod = 41; + + // Elements of GF(32) are encoded as vectors of length 5 over GF(2), that is, + // 5 binary digits. Each element (b_4, b_3, b_2, b_1, b_0) encodes a polynomial + // b_4*x^4 + b_3*x^3 + b_2*x^2 + b_1*x^1 + b_0 (modulo fmod). + // For example, 00001 = 1 is the multiplicative identity. + GF32_EXP[0] = 1; + GF32_LOG[0] = -1; + GF32_LOG[1] = 0; + int v = 1; + for (int i = 1; i < 31; ++i) { + // Multiplication by x is the same as shifting left by 1, as + // every coefficient of the polynomial is moved up one place. + v = v << 1; + // If the polynomial now has an x^5 term, we subtract fmod from it + // to remain working modulo fmod. Subtraction is the same as XOR in characteristic + // 2 fields. + if (v & 32) v ^= fmod; + GF32_EXP[i] = v; + GF32_LOG[v] = i; + } + + // Build table for GF(1024) + std::array GF1024_EXP{}; + std::array GF1024_LOG{}; + + GF1024_EXP[0] = 1; + GF1024_LOG[0] = -1; + GF1024_LOG[1] = 0; + + // Each element v of GF(1024) is encoded as a 10 bit integer in the following way: + // v = v1 || v0 where v0, v1 are 5-bit integers (elements of GF(32)). + // The element (e) is encoded as 1 || 0, to represent 1*(e) + 0. Every other element + // a*(e) + b is represented as a || b (a and b are both GF(32) elements). Given (v), + // we compute (e)*(v) by multiplying in the following way: + // + // v0' = 23*v1 + // v1' = 9*v1 + v0 + // e*v = v1' || v0' + // + // Where 23, 9 are GF(32) elements encoded as described above. Multiplication in GF(32) + // is done using the log/exp tables: + // e^x * e^y = e^(x + y) so a * b = EXP[ LOG[a] + LOG [b] ] + // for non-zero a and b. + + v = 1; + for (int i = 1; i < 1023; ++i) { + int v0 = v & 31; + int v1 = v >> 5; + + int v0n = v1 ? GF32_EXP.at((GF32_LOG.at(v1) + GF32_LOG.at(23)) % 31) : 0; + int v1n = (v1 ? GF32_EXP.at((GF32_LOG.at(v1) + GF32_LOG.at(9)) % 31) : 0) ^ v0; + + v = v1n << 5 | v0n; + GF1024_EXP[i] = v; + GF1024_LOG[v] = i; + } + + return std::make_pair(GF1024_EXP, GF1024_LOG); +} + +constexpr auto tables = GenerateGFTables(); +constexpr const std::array& GF1024_EXP = tables.first; +constexpr const std::array& GF1024_LOG = tables.second; + /* Determine the final constant to use for the specified encoding. */ uint32_t EncodingConstant(Encoding encoding) { assert(encoding == Encoding::BECH32 || encoding == Encoding::BECH32M); @@ -63,6 +151,26 @@ uint32_t PolyMod(const data& v) // the above example, `c` initially corresponds to 1 mod g(x), and after processing 2 inputs of // v, it corresponds to x^2 + v0*x + v1 mod g(x). As 1 mod g(x) = 1, that is the starting value // for `c`. + + // The following Sage code constructs the generator used: + // + // B = GF(2) # Binary field + // BP. = B[] # Polynomials over the binary field + // F_mod = b**5 + b**3 + 1 + // F. = GF(32, modulus=F_mod, repr='int') # GF(32) definition + // FP. = F[] # Polynomials over GF(32) + // E_mod = x**2 + F.fetch_int(9)*x + F.fetch_int(23) + // E. = F.extension(E_mod) # GF(1024) extension field definition + // for p in divisors(E.order() - 1): # Verify e has order 1023. + // assert((e**p == 1) == (p % 1023 == 0)) + // G = lcm([(e**i).minpoly() for i in range(997,1000)]) + // print(G) # Print out the generator + // + // It demonstrates that g(x) is the least common multiple of the minimal polynomials + // of 3 consecutive powers (997,998,999) of a primitive element (e) of GF(1024). + // That guarantees it is, in fact, the generator of a primitive BCH code with cycle + // length 1023 and distance 4. See https://en.wikipedia.org/wiki/BCH_code for more details. + uint32_t c = 1; for (const auto v_i : v) { // We want to update `c` to correspond to a polynomial with one extra term. If the initial @@ -85,22 +193,118 @@ uint32_t PolyMod(const data& v) // Then compute c1*x^5 + c2*x^4 + c3*x^3 + c4*x^2 + c5*x + v_i: c = ((c & 0x1ffffff) << 5) ^ v_i; - // Finally, for each set bit n in c0, conditionally add {2^n}k(x): + // Finally, for each set bit n in c0, conditionally add {2^n}k(x). These constants can be + // computed using the following Sage code (continuing the code above): + // + // for i in [1,2,4,8,16]: # Print out {1,2,4,8,16}*(g(x) mod x^6), packed in hex integers. + // v = 0 + // for coef in reversed((F.fetch_int(i)*(G % x**6)).coefficients(sparse=True)): + // v = v*32 + coef.integer_representation() + // print("0x%x" % v) + // if (c0 & 1) c ^= 0x3b6a57b2; // k(x) = {29}x^5 + {22}x^4 + {20}x^3 + {21}x^2 + {29}x + {18} if (c0 & 2) c ^= 0x26508e6d; // {2}k(x) = {19}x^5 + {5}x^4 + x^3 + {3}x^2 + {19}x + {13} if (c0 & 4) c ^= 0x1ea119fa; // {4}k(x) = {15}x^5 + {10}x^4 + {2}x^3 + {6}x^2 + {15}x + {26} if (c0 & 8) c ^= 0x3d4233dd; // {8}k(x) = {30}x^5 + {20}x^4 + {4}x^3 + {12}x^2 + {30}x + {29} if (c0 & 16) c ^= 0x2a1462b3; // {16}k(x) = {21}x^5 + x^4 + {8}x^3 + {24}x^2 + {21}x + {19} + } return c; } +/** Syndrome computes the values s_j = R(e^j) for j in [997, 998, 999]. As described above, the + * generator polynomial G is the LCM of the minimal polynomials of (e)^997, (e)^998, and (e)^999. + * + * Consider a codeword with errors, of the form R(x) = C(x) + E(x). The residue is the bit-packed + * result of computing R(x) mod G(X), where G is the generator of the code. Because C(x) is a valid + * codeword, it is a multiple of G(X), so the residue is in fact just E(x) mod G(x). Note that all + * of the (e)^j are roots of G(x) by definition, so R((e)^j) = E((e)^j). + * + * Let R(x) = r1*x^5 + r2*x^4 + r3*x^3 + r4*x^2 + r5*x + r6 + * + * To compute R((e)^j), we are really computing: + * r1*(e)^(j*5) + r2*(e)^(j*4) + r3*(e)^(j*3) + r4*(e)^(j*2) + r5*(e)^j + r6 + * + * Now note that all of the (e)^(j*i) for i in [5..0] are constants and can be precomputed. + * But even more than that, we can consider each coefficient as a bit-string. + * For example, take r5 = (b_5, b_4, b_3, b_2, b_1) written out as 5 bits. Then: + * r5*(e)^j = b_1*(e)^j + b_2*(2*(e)^j) + b_3*(4*(e)^j) + b_4*(8*(e)^j) + b_5*(16*(e)^j) + * where all the (2^i*(e)^j) are constants and can be precomputed. + * + * Then we just add each of these corresponding constants to our final value based on the + * bit values b_i. This is exactly what is done in the Syndrome function below. + */ +constexpr std::array GenerateSyndromeConstants() { + std::array SYNDROME_CONSTS{}; + for (int k = 1; k < 6; ++k) { + for (int shift = 0; shift < 5; ++shift) { + int16_t b = GF1024_LOG.at(1 << shift); + int16_t c0 = GF1024_EXP.at((997*k + b) % 1023); + int16_t c1 = GF1024_EXP.at((998*k + b) % 1023); + int16_t c2 = GF1024_EXP.at((999*k + b) % 1023); + uint32_t c = c2 << 20 | c1 << 10 | c0; + int ind = 5*(k-1) + shift; + SYNDROME_CONSTS[ind] = c; + } + } + return SYNDROME_CONSTS; +} +constexpr std::array SYNDROME_CONSTS = GenerateSyndromeConstants(); + +/** + * Syndrome returns the three values s_997, s_998, and s_999 described above, + * packed into a 30-bit integer, where each group of 10 bits encodes one value. + */ +uint32_t Syndrome(const uint32_t residue) { + // low is the first 5 bits, corresponding to the r6 in the residue + // (the constant term of the polynomial). + uint32_t low = residue & 0x1f; + + // We begin by setting s_j = low = r6 for all three values of j, because these are unconditional. + uint32_t result = low ^ (low << 10) ^ (low << 20); + + // Then for each following bit, we add the corresponding precomputed constant if the bit is 1. + // For example, 0x31edd3c4 is 1100011110 1101110100 1111000100 when unpacked in groups of 10 + // bits, corresponding exactly to a^999 || a^998 || a^997 (matching the corresponding values in + // GF1024_EXP above). In this way, we compute all three values of s_j for j in (997, 998, 999) + // simultaneously. Recall that XOR corresponds to addition in a characteristic 2 field. + for (int i = 0; i < 25; ++i) { + result ^= ((residue >> (5+i)) & 1 ? SYNDROME_CONSTS.at(i) : 0); + } + return result; +} + /** Convert to lower case. */ inline unsigned char LowerCase(unsigned char c) { return (c >= 'A' && c <= 'Z') ? (c - 'A') + 'a' : c; } +/** Return indices of invalid characters in a Bech32 string. */ +bool CheckCharacters(const std::string& str, std::vector& errors) +{ + bool lower = false, upper = false; + for (size_t i = 0; i < str.size(); ++i) { + unsigned char c{(unsigned char)(str[i])}; + if (c >= 'a' && c <= 'z') { + if (upper) { + errors.push_back(i); + } else { + lower = true; + } + } else if (c >= 'A' && c <= 'Z') { + if (lower) { + errors.push_back(i); + } else { + upper = true; + } + } else if (c < 33 || c > 126) { + errors.push_back(i); + } + } + return errors.empty(); +} + /** Expand a HRP for use in checksum computation. */ data ExpandHRP(const std::string& hrp) { @@ -122,7 +326,8 @@ Encoding VerifyChecksum(const std::string& hrp, const data& values) // PolyMod computes what value to xor into the final values to make the checksum 0. However, // if we required that the checksum was 0, it would be the case that appending a 0 to a valid // list of values would result in a new valid list. For that reason, Bech32 requires the - // resulting checksum to be 1 instead. In Bech32m, this constant was amended. + // resulting checksum to be 1 instead. In Bech32m, this constant was amended. See + // https://gist.github.com/sipa/14c248c288c3880a3b191f978a34508e for details. const uint32_t check = PolyMod(Cat(ExpandHRP(hrp), values)); if (check == EncodingConstant(Encoding::BECH32)) return Encoding::BECH32; if (check == EncodingConstant(Encoding::BECH32M)) return Encoding::BECH32M; @@ -161,14 +366,8 @@ std::string Encode(Encoding encoding, const std::string& hrp, const data& values /** Decode a Bech32 or Bech32m string. */ DecodeResult Decode(const std::string& str) { - bool lower = false, upper = false; - for (size_t i = 0; i < str.size(); ++i) { - unsigned char c = str[i]; - if (c >= 'a' && c <= 'z') lower = true; - else if (c >= 'A' && c <= 'Z') upper = true; - else if (c < 33 || c > 126) return {}; - } - if (lower && upper) return {}; + std::vector errors; + if (!CheckCharacters(str, errors)) return {}; size_t pos = str.rfind('1'); if (str.size() > 90 || pos == str.npos || pos == 0 || pos + 7 > str.size()) { return {}; @@ -192,4 +391,174 @@ DecodeResult Decode(const std::string& str) { return {result, std::move(hrp), data(values.begin(), values.end() - 6)}; } +/** Find index of an incorrect character in a Bech32 string. */ +std::pair> LocateErrors(const std::string& str) { + std::vector error_locations{}; + + if (str.size() > 90) { + error_locations.resize(str.size() - 90); + std::iota(error_locations.begin(), error_locations.end(), 90); + return std::make_pair("Bech32 string too long", std::move(error_locations)); + } + + if (!CheckCharacters(str, error_locations)){ + return std::make_pair("Invalid character or mixed case", std::move(error_locations)); + } + + size_t pos = str.rfind('1'); + if (pos == str.npos) { + return std::make_pair("Missing separator", std::vector{}); + } + if (pos == 0 || pos + 7 > str.size()) { + error_locations.push_back(pos); + return std::make_pair("Invalid separator position", std::move(error_locations)); + } + + std::string hrp; + for (size_t i = 0; i < pos; ++i) { + hrp += LowerCase(str[i]); + } + + size_t length = str.size() - 1 - pos; // length of data part + data values(length); + for (size_t i = pos + 1; i < str.size(); ++i) { + unsigned char c = str[i]; + int8_t rev = CHARSET_REV[c]; + if (rev == -1) { + error_locations.push_back(i); + return std::make_pair("Invalid Base 32 character", std::move(error_locations)); + } + values[i - pos - 1] = rev; + } + + // We attempt error detection with both bech32 and bech32m, and choose the one with the fewest errors + // We can't simply use the segwit version, because that may be one of the errors + std::optional error_encoding; + for (Encoding encoding : {Encoding::BECH32, Encoding::BECH32M}) { + std::vector possible_errors; + // Recall that (ExpandHRP(hrp) ++ values) is interpreted as a list of coefficients of a polynomial + // over GF(32). PolyMod computes the "remainder" of this polynomial modulo the generator G(x). + uint32_t residue = PolyMod(Cat(ExpandHRP(hrp), values)) ^ EncodingConstant(encoding); + + // All valid codewords should be multiples of G(x), so this remainder (after XORing with the encoding + // constant) should be 0 - hence 0 indicates there are no errors present. + if (residue != 0) { + // If errors are present, our polynomial must be of the form C(x) + E(x) where C is the valid + // codeword (a multiple of G(x)), and E encodes the errors. + uint32_t syn = Syndrome(residue); + + // Unpack the three 10-bit syndrome values + int s0 = syn & 0x3FF; + int s1 = (syn >> 10) & 0x3FF; + int s2 = syn >> 20; + + // Get the discrete logs of these values in GF1024 for more efficient computation + int l_s0 = GF1024_LOG.at(s0); + int l_s1 = GF1024_LOG.at(s1); + int l_s2 = GF1024_LOG.at(s2); + + // First, suppose there is only a single error. Then E(x) = e1*x^p1 for some position p1 + // Then s0 = E((e)^997) = e1*(e)^(997*p1) and s1 = E((e)^998) = e1*(e)^(998*p1) + // Therefore s1/s0 = (e)^p1, and by the same logic, s2/s1 = (e)^p1 too. + // Hence, s1^2 == s0*s2, which is exactly the condition we check first: + if (l_s0 != -1 && l_s1 != -1 && l_s2 != -1 && (2 * l_s1 - l_s2 - l_s0 + 2046) % 1023 == 0) { + // Compute the error position p1 as l_s1 - l_s0 = p1 (mod 1023) + size_t p1 = (l_s1 - l_s0 + 1023) % 1023; // the +1023 ensures it is positive + // Now because s0 = e1*(e)^(997*p1), we get e1 = s0/((e)^(997*p1)). Remember that (e)^1023 = 1, + // so 1/((e)^997) = (e)^(1023-997). + int l_e1 = l_s0 + (1023 - 997) * p1; + // Finally, some sanity checks on the result: + // - The error position should be within the length of the data + // - e1 should be in GF(32), which implies that e1 = (e)^(33k) for some k (the 31 non-zero elements + // of GF(32) form an index 33 subgroup of the 1023 non-zero elements of GF(1024)). + if (p1 < length && !(l_e1 % 33)) { + // Polynomials run from highest power to lowest, so the index p1 is from the right. + // We don't return e1 because it is dangerous to suggest corrections to the user, + // the user should check the address themselves. + possible_errors.push_back(str.size() - p1 - 1); + } + // Otherwise, suppose there are two errors. Then E(x) = e1*x^p1 + e2*x^p2. + } else { + // For all possible first error positions p1 + for (size_t p1 = 0; p1 < length; ++p1) { + // We have guessed p1, and want to solve for p2. Recall that E(x) = e1*x^p1 + e2*x^p2, so + // s0 = E((e)^997) = e1*(e)^(997^p1) + e2*(e)^(997*p2), and similar for s1 and s2. + // + // Consider s2 + s1*(e)^p1 + // = 2e1*(e)^(999^p1) + e2*(e)^(999*p2) + e2*(e)^(998*p2)*(e)^p1 + // = e2*(e)^(999*p2) + e2*(e)^(998*p2)*(e)^p1 + // (Because we are working in characteristic 2.) + // = e2*(e)^(998*p2) ((e)^p2 + (e)^p1) + // + int s2_s1p1 = s2 ^ (s1 == 0 ? 0 : GF1024_EXP.at((l_s1 + p1) % 1023)); + if (s2_s1p1 == 0) continue; + int l_s2_s1p1 = GF1024_LOG.at(s2_s1p1); + + // Similarly, s1 + s0*(e)^p1 + // = e2*(e)^(997*p2) ((e)^p2 + (e)^p1) + int s1_s0p1 = s1 ^ (s0 == 0 ? 0 : GF1024_EXP.at((l_s0 + p1) % 1023)); + if (s1_s0p1 == 0) continue; + int l_s1_s0p1 = GF1024_LOG.at(s1_s0p1); + + // So, putting these together, we can compute the second error position as + // (e)^p2 = (s2 + s1^p1)/(s1 + s0^p1) + // p2 = log((e)^p2) + size_t p2 = (l_s2_s1p1 - l_s1_s0p1 + 1023) % 1023; + + // Sanity checks that p2 is a valid position and not the same as p1 + if (p2 >= length || p1 == p2) continue; + + // Now we want to compute the error values e1 and e2. + // Similar to above, we compute s1 + s0*(e)^p2 + // = e1*(e)^(997*p1) ((e)^p1 + (e)^p2) + int s1_s0p2 = s1 ^ (s0 == 0 ? 0 : GF1024_EXP.at((l_s0 + p2) % 1023)); + if (s1_s0p2 == 0) continue; + int l_s1_s0p2 = GF1024_LOG.at(s1_s0p2); + + // And compute (the log of) 1/((e)^p1 + (e)^p2)) + int inv_p1_p2 = 1023 - GF1024_LOG.at(GF1024_EXP.at(p1) ^ GF1024_EXP.at(p2)); + + // Then (s1 + s0*(e)^p1) * (1/((e)^p1 + (e)^p2))) + // = e2*(e)^(997*p2) + // Then recover e2 by dividing by (e)^(997*p2) + int l_e2 = l_s1_s0p1 + inv_p1_p2 + (1023 - 997) * p2; + // Check that e2 is in GF(32) + if (l_e2 % 33) continue; + + // In the same way, (s1 + s0*(e)^p2) * (1/((e)^p1 + (e)^p2))) + // = e1*(e)^(997*p1) + // So recover e1 by dividing by (e)^(997*p1) + int l_e1 = l_s1_s0p2 + inv_p1_p2 + (1023 - 997) * p1; + // Check that e1 is in GF(32) + if (l_e1 % 33) continue; + + // Again, we do not return e1 or e2 for safety. + // Order the error positions from the left of the string and return them + if (p1 > p2) { + possible_errors.push_back(str.size() - p1 - 1); + possible_errors.push_back(str.size() - p2 - 1); + } else { + possible_errors.push_back(str.size() - p2 - 1); + possible_errors.push_back(str.size() - p1 - 1); + } + break; + } + } + } else { + // No errors + return std::make_pair("", std::vector{}); + } + + if (error_locations.empty() || (!possible_errors.empty() && possible_errors.size() < error_locations.size())) { + error_locations = std::move(possible_errors); + if (!error_locations.empty()) error_encoding = encoding; + } + } + std::string error_message = error_encoding == Encoding::BECH32M ? "Invalid Bech32m checksum" + : error_encoding == Encoding::BECH32 ? "Invalid Bech32 checksum" + : "Invalid checksum"; + + return std::make_pair(error_message, std::move(error_locations)); +} + } // namespace bech32 diff --git a/src/bech32.h b/src/bech32.h index 30130f8bb5..8594055ef5 100644 --- a/src/bech32.h +++ b/src/bech32.h @@ -1,4 +1,5 @@ // Copyright (c) 2017, 2021 Pieter Wuille +// Copyright (c) 2021 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -47,6 +48,9 @@ DecodeResult Decode(const std::string& str); /// Exported for testing. uint32_t PolyMod(const std::vector& v); +/** Return the positions of errors in a Bech32 string. */ +std::pair> LocateErrors(const std::string& str); + } // namespace bech32 #endif // BITCOIN_BECH32_H diff --git a/src/bench/addrman.cpp b/src/bench/addrman.cpp index b7bd8a3261..3ca58b923e 100644 --- a/src/bench/addrman.cpp +++ b/src/bench/addrman.cpp @@ -1,10 +1,11 @@ -// Copyright (c) 2020-2020 The Bitcoin Core developers +// Copyright (c) 2020-2021 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include #include #include +#include #include #include @@ -15,6 +16,9 @@ static constexpr size_t NUM_SOURCES = 64; static constexpr size_t NUM_ADDRESSES_PER_SOURCE = 256; +static const std::vector EMPTY_ASMAP; +static constexpr uint32_t ADDRMAN_CONSISTENCY_CHECK_RATIO{0}; + static std::vector g_sources; static std::vector> g_addresses; @@ -52,14 +56,14 @@ static void CreateAddresses() } } -static void AddAddressesToAddrMan(CAddrMan& addrman) +static void AddAddressesToAddrMan(AddrMan& addrman) { for (size_t source_i = 0; source_i < NUM_SOURCES; ++source_i) { addrman.Add(g_addresses[source_i], g_sources[source_i]); } } -static void FillAddrMan(CAddrMan& addrman) +static void FillAddrMan(AddrMan& addrman) { CreateAddresses(); @@ -72,29 +76,27 @@ static void AddrManAdd(benchmark::Bench& bench) { CreateAddresses(); - CAddrMan addrman; - bench.run([&] { + AddrMan addrman{EMPTY_ASMAP, /*deterministic=*/false, ADDRMAN_CONSISTENCY_CHECK_RATIO}; AddAddressesToAddrMan(addrman); - addrman.Clear(); }); } static void AddrManSelect(benchmark::Bench& bench) { - CAddrMan addrman; + AddrMan addrman{EMPTY_ASMAP, /*deterministic=*/false, ADDRMAN_CONSISTENCY_CHECK_RATIO}; FillAddrMan(addrman); bench.run([&] { const auto& address = addrman.Select(); - assert(address.GetPort() > 0); + assert(address.first.GetPort() > 0); }); } static void AddrManGetAddr(benchmark::Bench& bench) { - CAddrMan addrman; + AddrMan addrman{EMPTY_ASMAP, /*deterministic=*/false, ADDRMAN_CONSISTENCY_CHECK_RATIO}; FillAddrMan(addrman); @@ -104,38 +106,33 @@ static void AddrManGetAddr(benchmark::Bench& bench) }); } -static void AddrManGood(benchmark::Bench& bench) +static void AddrManAddThenGood(benchmark::Bench& bench) { - /* Create many CAddrMan objects - one to be modified at each loop iteration. - * This is necessary because the CAddrMan::Good() method modifies the - * object, affecting the timing of subsequent calls to the same method and - * we want to do the same amount of work in every loop iteration. */ - - bench.epochs(5).epochIterations(1); - - std::vector addrmans(bench.epochs() * bench.epochIterations()); - for (auto& addrman : addrmans) { - FillAddrMan(addrman); - } - - auto markSomeAsGood = [](CAddrMan& addrman) { + auto markSomeAsGood = [](AddrMan& addrman) { for (size_t source_i = 0; source_i < NUM_SOURCES; ++source_i) { for (size_t addr_i = 0; addr_i < NUM_ADDRESSES_PER_SOURCE; ++addr_i) { - if (addr_i % 32 == 0) { - addrman.Good(g_addresses[source_i][addr_i]); - } + addrman.Good(g_addresses[source_i][addr_i]); } } }; - uint64_t i = 0; + CreateAddresses(); + bench.run([&] { - markSomeAsGood(addrmans.at(i)); - ++i; + // To make the benchmark independent of the number of evaluations, we always prepare a new addrman. + // This is necessary because AddrMan::Good() method modifies the object, affecting the timing of subsequent calls + // to the same method and we want to do the same amount of work in every loop iteration. + // + // This has some overhead (exactly the result of AddrManAdd benchmark), but that overhead is constant so improvements in + // AddrMan::Good() will still be noticeable. + AddrMan addrman{EMPTY_ASMAP, /*deterministic=*/false, ADDRMAN_CONSISTENCY_CHECK_RATIO}; + AddAddressesToAddrMan(addrman); + + markSomeAsGood(addrman); }); } BENCHMARK(AddrManAdd); BENCHMARK(AddrManSelect); BENCHMARK(AddrManGetAddr); -BENCHMARK(AddrManGood); +BENCHMARK(AddrManAddThenGood); diff --git a/src/bench/bech32.cpp b/src/bench/bech32.cpp index 8e10862a37..0f89a8e2b5 100644 --- a/src/bench/bech32.cpp +++ b/src/bench/bech32.cpp @@ -1,9 +1,8 @@ -// Copyright (c) 2018-2020 The Bitcoin Core developers +// Copyright (c) 2018-2021 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include -#include #include #include diff --git a/src/bench/bench.cpp b/src/bench/bench.cpp index 012057e792..033d319750 100644 --- a/src/bench/bench.cpp +++ b/src/bench/bench.cpp @@ -1,33 +1,42 @@ -// Copyright (c) 2015-2020 The Bitcoin Core developers +// Copyright (c) 2015-2021 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include -#include +#include #include -#include +#include +#include +#include +#include +#include #include +#include +#include + +using namespace std::chrono_literals; const std::function G_TEST_LOG_FUN{}; +const std::function()> G_TEST_COMMAND_LINE_ARGUMENTS{}; + namespace { -void GenerateTemplateResults(const std::vector& benchmarkResults, const std::string& filename, const char* tpl) +void GenerateTemplateResults(const std::vector& benchmarkResults, const fs::path& file, const char* tpl) { - if (benchmarkResults.empty() || filename.empty()) { + if (benchmarkResults.empty() || file.empty()) { // nothing to write, bail out return; } - std::ofstream fout(filename); + std::ofstream fout{file}; if (fout.is_open()) { ankerl::nanobench::render(tpl, benchmarkResults, fout); + std::cout << "Created " << file << std::endl; } else { - std::cout << "Could write to file '" << filename << "'" << std::endl; + std::cout << "Could not write to file " << file << std::endl; } - - std::cout << "Created '" << filename << "'" << std::endl; } } // namespace @@ -61,6 +70,12 @@ void benchmark::BenchRunner::RunAll(const Args& args) Bench bench; bench.name(p.first); + if (args.min_time > 0ms) { + // convert to nanos before dividing to reduce rounding errors + std::chrono::nanoseconds min_time_ns = args.min_time; + bench.minEpochTime(min_time_ns / bench.epochs()); + } + if (args.asymptote.empty()) { p.second(bench); } else { diff --git a/src/bench/bench.h b/src/bench/bench.h index c4fcd80e33..6634138beb 100644 --- a/src/bench/bench.h +++ b/src/bench/bench.h @@ -1,10 +1,11 @@ -// Copyright (c) 2015-2020 The Bitcoin Core developers +// Copyright (c) 2015-2021 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #ifndef BITCOIN_BENCH_BENCH_H #define BITCOIN_BENCH_BENCH_H +#include #include #include @@ -41,11 +42,12 @@ using ankerl::nanobench::Bench; typedef std::function BenchFunction; struct Args { - std::string regex_filter; bool is_list_only; + std::chrono::milliseconds min_time; std::vector asymptote; - std::string output_csv; - std::string output_json; + fs::path output_csv; + fs::path output_json; + std::string regex_filter; }; class BenchRunner diff --git a/src/bench/bench_bitcoin.cpp b/src/bench/bench_bitcoin.cpp index aab777cac1..3f8bff4bcf 100644 --- a/src/bench/bench_bitcoin.cpp +++ b/src/bench/bench_bitcoin.cpp @@ -1,24 +1,32 @@ -// Copyright (c) 2015-2020 The Bitcoin Core developers +// Copyright (c) 2015-2021 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include +#include #include +#include #include #include -#include +#include +#include +#include +#include +#include static const char* DEFAULT_BENCH_FILTER = ".*"; +static constexpr int64_t DEFAULT_MIN_TIME_MS{10}; static void SetupBenchArgs(ArgsManager& argsman) { SetupHelpOptions(argsman); - argsman.AddArg("-asymptote=n1,n2,n3,...", "Test asymptotic growth of the runtime of an algorithm, if supported by the benchmark", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); + argsman.AddArg("-asymptote=", "Test asymptotic growth of the runtime of an algorithm, if supported by the benchmark", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); argsman.AddArg("-filter=", strprintf("Regular expression filter to select benchmark by name (default: %s)", DEFAULT_BENCH_FILTER), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); argsman.AddArg("-list", "List benchmarks without executing them", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); + argsman.AddArg("-min_time=", strprintf("Minimum runtime per benchmark, in milliseconds (default: %d)", DEFAULT_MIN_TIME_MS), ArgsManager::ALLOW_ANY | ArgsManager::DISALLOW_NEGATION, OptionsCategory::OPTIONS); argsman.AddArg("-output_csv=", "Generate CSV file with the most important benchmark results", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); argsman.AddArg("-output_json=", "Generate JSON file with all benchmark results", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); } @@ -48,17 +56,62 @@ int main(int argc, char** argv) } if (HelpRequested(argsman)) { - std::cout << argsman.GetHelpMessage(); + std::cout << "Usage: bench_bitcoin [options]\n" + "\n" + << argsman.GetHelpMessage() + << "Description:\n" + "\n" + " bench_bitcoin executes microbenchmarks. The quality of the benchmark results\n" + " highly depend on the stability of the machine. It can sometimes be difficult\n" + " to get stable, repeatable results, so here are a few tips:\n" + "\n" + " * Use pyperf [1] to disable frequency scaling, turbo boost etc. For best\n" + " results, use CPU pinning and CPU isolation (see [2]).\n" + "\n" + " * Each call of run() should do exactly the same work. E.g. inserting into\n" + " a std::vector doesn't do that as it will reallocate on certain calls. Make\n" + " sure each run has exactly the same preconditions.\n" + "\n" + " * If results are still not reliable, increase runtime with e.g.\n" + " -min_time=5000 to let a benchmark run for at least 5 seconds.\n" + "\n" + " * bench_bitcoin uses nanobench [3] for which there is extensive\n" + " documentation available online.\n" + "\n" + "Environment Variables:\n" + "\n" + " To attach a profiler you can run a benchmark in endless mode. This can be\n" + " done with the environment variable NANOBENCH_ENDLESS. E.g. like so:\n" + "\n" + " NANOBENCH_ENDLESS=MuHash ./bench_bitcoin -filter=MuHash\n" + "\n" + " In rare cases it can be useful to suppress stability warnings. This can be\n" + " done with the environment variable NANOBENCH_SUPPRESS_WARNINGS, e.g:\n" + "\n" + " NANOBENCH_SUPPRESS_WARNINGS=1 ./bench_bitcoin\n" + "\n" + "Notes:\n" + "\n" + " 1. pyperf\n" + " https://github.com/psf/pyperf\n" + "\n" + " 2. CPU pinning & isolation\n" + " https://pyperf.readthedocs.io/en/latest/system.html\n" + "\n" + " 3. nanobench\n" + " https://github.com/martinus/nanobench\n" + "\n"; return EXIT_SUCCESS; } benchmark::Args args; - args.regex_filter = argsman.GetArg("-filter", DEFAULT_BENCH_FILTER); - args.is_list_only = argsman.GetBoolArg("-list", false); args.asymptote = parseAsymptote(argsman.GetArg("-asymptote", "")); - args.output_csv = argsman.GetArg("-output_csv", ""); - args.output_json = argsman.GetArg("-output_json", ""); + args.is_list_only = argsman.GetBoolArg("-list", false); + args.min_time = std::chrono::milliseconds(argsman.GetIntArg("-min_time", DEFAULT_MIN_TIME_MS)); + args.output_csv = fs::PathFromString(argsman.GetArg("-output_csv", "")); + args.output_json = fs::PathFromString(argsman.GetArg("-output_json", "")); + args.regex_filter = argsman.GetArg("-filter", DEFAULT_BENCH_FILTER); benchmark::BenchRunner::RunAll(args); diff --git a/src/bench/block_assemble.cpp b/src/bench/block_assemble.cpp index 0092426315..4e1565b89f 100644 --- a/src/bench/block_assemble.cpp +++ b/src/bench/block_assemble.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2020 The Bitcoin Core developers +// Copyright (c) 2011-2021 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -35,10 +35,10 @@ static void AssembleBlock(benchmark::Bench& bench) txs.at(b) = MakeTransactionRef(tx); } { - LOCK(::cs_main); // Required for ::AcceptToMemoryPool. + LOCK(::cs_main); for (const auto& txr : txs) { - const MempoolAcceptResult res = ::AcceptToMemoryPool(test_setup->m_node.chainman->ActiveChainstate(), *test_setup->m_node.mempool, txr, false /* bypass_limits */); + const MempoolAcceptResult res = test_setup->m_node.chainman->ProcessTransaction(txr); assert(res.m_result_type == MempoolAcceptResult::ResultType::VALID); } } diff --git a/src/bench/ccoins_caching.cpp b/src/bench/ccoins_caching.cpp index d5275b0b76..22dfb7aa5b 100644 --- a/src/bench/ccoins_caching.cpp +++ b/src/bench/ccoins_caching.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2016-2020 The Bitcoin Core developers +// Copyright (c) 2016-2021 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -45,7 +45,7 @@ static void CCoinsCaching(benchmark::Bench& bench) // Benchmark. const CTransaction tx_1(t1); bench.run([&] { - bool success = AreInputsStandard(tx_1, coins, false); + bool success{AreInputsStandard(tx_1, coins)}; assert(success); }); ECC_Stop(); diff --git a/src/bench/checkblock.cpp b/src/bench/checkblock.cpp index a9f3f5f84d..52e5cb743f 100644 --- a/src/bench/checkblock.cpp +++ b/src/bench/checkblock.cpp @@ -17,8 +17,8 @@ static void DeserializeBlockTest(benchmark::Bench& bench) { CDataStream stream(benchmark::data::block413567, SER_NETWORK, PROTOCOL_VERSION); - char a = '\0'; - stream.write(&a, 1); // Prevent compaction + std::byte a{0}; + stream.write({&a, 1}); // Prevent compaction bench.unit("block").run([&] { CBlock block; @@ -31,8 +31,8 @@ static void DeserializeBlockTest(benchmark::Bench& bench) static void DeserializeAndCheckBlockTest(benchmark::Bench& bench) { CDataStream stream(benchmark::data::block413567, SER_NETWORK, PROTOCOL_VERSION); - char a = '\0'; - stream.write(&a, 1); // Prevent compaction + std::byte a{0}; + stream.write({&a, 1}); // Prevent compaction ArgsManager bench_args; const auto chainParams = CreateChainParams(bench_args, CBaseChainParams::MAIN); diff --git a/src/bench/checkqueue.cpp b/src/bench/checkqueue.cpp index 683c3231cc..c4ba85e545 100644 --- a/src/bench/checkqueue.cpp +++ b/src/bench/checkqueue.cpp @@ -30,8 +30,9 @@ static void CCheckQueueSpeedPrevectorJob(benchmark::Bench& bench) struct PrevectorJob { prevector p; - PrevectorJob(){ - } + // ELEMENTS: fix unused member function warnings + // PrevectorJob(){ + // } explicit PrevectorJob(FastRandomContext& insecure_rand){ p.resize(insecure_rand.randrange(PREVECTOR_SIZE*2)); } @@ -39,7 +40,7 @@ static void CCheckQueueSpeedPrevectorJob(benchmark::Bench& bench) { return true; } - void swap(PrevectorJob& x){p.swap(x.p);}; + // void swap(PrevectorJob& x){p.swap(x.p);}; }; CCheckQueue queue {QUEUE_BATCH_SIZE}; // The main thread should be counted to prevent thread oversubscription, and diff --git a/src/bench/coin_selection.cpp b/src/bench/coin_selection.cpp index 1ad1560a76..9ebee47ab9 100644 --- a/src/bench/coin_selection.cpp +++ b/src/bench/coin_selection.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2012-2020 The Bitcoin Core developers +// Copyright (c) 2012-2021 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -8,10 +8,24 @@ #include #include #include +#include #include #include +using node::NodeContext; +using wallet::AttemptSelection; +using wallet::CInputCoin; +using wallet::COutput; +using wallet::CWallet; +using wallet::CWalletTx; +using wallet::CoinEligibilityFilter; +using wallet::CoinSelectionParams; +using wallet::CreateDummyWalletDatabase; +using wallet::OutputGroup; +using wallet::SelectCoinsBnB; +using wallet::TxStateInactive; + static void addCoin(const CAmount& nValue, const CWallet& wallet, std::vector>& wtxs) { static int nextLockTime = 0; @@ -19,7 +33,7 @@ static void addCoin(const CAmount& nValue, const CWallet& wallet, std::vector(&wallet, MakeTransactionRef(std::move(tx)))); + wtxs.push_back(std::make_unique(MakeTransactionRef(std::move(tx)), TxStateInactive{})); } // Simple benchmark for wallet coin selection. Note that it maybe be necessary @@ -33,8 +47,7 @@ static void CoinSelection(benchmark::Bench& bench) { NodeContext node; auto chain = interfaces::MakeChain(node); - CWallet wallet(chain.get(), "", CreateDummyWalletDatabase()); - wallet.SetupLegacyScriptPubKeyMan(); + CWallet wallet(chain.get(), "", gArgs, CreateDummyWalletDatabase()); std::vector> wtxs; LOCK(wallet.cs_wallet); @@ -47,23 +60,21 @@ static void CoinSelection(benchmark::Bench& bench) // Create coins std::vector coins; for (const auto& wtx : wtxs) { - coins.emplace_back(wtx.get(), 0 /* iIn */, 6 * 24 /* nDepthIn */, true /* spendable */, true /* solvable */, true /* safe */); + coins.emplace_back(wallet, *wtx, 0 /* iIn */, 6 * 24 /* nDepthIn */, true /* spendable */, true /* solvable */, true /* safe */); } const CoinEligibilityFilter filter_standard(1, 6, 0); const CoinSelectionParams coin_selection_params(/* change_output_size= */ 34, /* change_spend_size= */ 148, /* effective_feerate= */ CFeeRate(0), /* long_term_feerate= */ CFeeRate(0), /* discard_feerate= */ CFeeRate(0), - /* tx_no_inputs_size= */ 0, /* avoid_partial= */ false); + /* tx_noinputs_size= */ 0, /* avoid_partial= */ false); bench.run([&] { - std::set setCoinsRet; - CAmountMap mapValueRet; CAmountMap mapValue; mapValue[::policyAsset] = 1003 * COIN; - bool success = wallet.AttemptSelection(mapValue, filter_standard, coins, setCoinsRet, mapValueRet, coin_selection_params); - assert(success); - assert(mapValueRet[::policyAsset] == 1003 * COIN); - assert(setCoinsRet.size() == 2); + auto result = AttemptSelection(wallet, mapValue, filter_standard, coins, coin_selection_params); + assert(result); + assert(result->GetSelectedValue() == mapValue); + assert(result->GetInputSet().size() == 2); /* std::set > setCoinsRet; CAmountMap nValueRet; @@ -79,8 +90,7 @@ static void CoinSelection(benchmark::Bench& bench) typedef std::set CoinSet; static NodeContext testNode; static auto testChain = interfaces::MakeChain(testNode); -static CWallet testWallet(testChain.get(), "", CreateDummyWalletDatabase()); -std::vector> wtxn; +static CWallet testWallet(testChain.get(), "", gArgs, CreateDummyWalletDatabase()); // Copied from src/wallet/test/coinselector_tests.cpp static void add_coin(const CAmount& nValue, int nInput, std::vector& set) @@ -88,10 +98,10 @@ static void add_coin(const CAmount& nValue, int nInput, std::vector CMutableTransaction tx; tx.vout.resize(nInput + 1); tx.vout[nInput].nValue = nValue; - std::unique_ptr wtx = std::make_unique(&testWallet, MakeTransactionRef(std::move(tx))); + CWalletTx wtx(MakeTransactionRef(tx), TxStateInactive{}); + CInputCoin coin(testWallet, &wtx, nInput); set.emplace_back(); - set.back().Insert(COutput(wtx.get(), nInput, 0, true, true, true).GetInputCoin(), 0, true, 0, 0, false); - wtxn.emplace_back(std::move(wtx)); + set.back().Insert(coin, 0, true, 0, 0, false); } // Copied from src/wallet/test/coinselector_tests.cpp static CAmount make_hard_case(int utxos, std::vector& utxo_pool) @@ -109,19 +119,15 @@ static CAmount make_hard_case(int utxos, std::vector& utxo_pool) static void BnBExhaustion(benchmark::Bench& bench) { // Setup - testWallet.SetupLegacyScriptPubKeyMan(); std::vector utxo_pool; - CoinSet selection; - CAmount value_ret = 0; bench.run([&] { // Benchmark CAmount target = make_hard_case(17, utxo_pool); - SelectCoinsBnB(utxo_pool, target, 0, selection, value_ret); // Should exhaust + SelectCoinsBnB(utxo_pool, target, 0); // Should exhaust // Cleanup utxo_pool.clear(); - selection.clear(); }); } diff --git a/src/bench/crypto_hash.cpp b/src/bench/crypto_hash.cpp index 30fe11be6b..d17ec503e7 100644 --- a/src/bench/crypto_hash.cpp +++ b/src/bench/crypto_hash.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2016-2020 The Bitcoin Core developers +// Copyright (c) 2016-2021 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -110,9 +110,9 @@ static void MuHash(benchmark::Bench& bench) { MuHash3072 acc; unsigned char key[32] = {0}; - int i = 0; + uint32_t i = 0; bench.run([&] { - key[0] = ++i; + key[0] = ++i & 0xFF; acc *= MuHash3072(key); }); } @@ -134,10 +134,6 @@ static void MuHashDiv(benchmark::Bench& bench) FastRandomContext rng(true); MuHash3072 muhash{rng.randbytes(32)}; - for (size_t i = 0; i < bench.epochIterations(); ++i) { - acc *= muhash; - } - bench.run([&] { acc /= muhash; }); diff --git a/src/bench/data.cpp b/src/bench/data.cpp index 481e372105..35558b3aa7 100644 --- a/src/bench/data.cpp +++ b/src/bench/data.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2019 The Bitcoin Core developers +// Copyright (c) 2019-2021 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/bench/duplicate_inputs.cpp b/src/bench/duplicate_inputs.cpp index 8703a1cf94..02a2e689b1 100644 --- a/src/bench/duplicate_inputs.cpp +++ b/src/bench/duplicate_inputs.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2020 The Bitcoin Core developers +// Copyright (c) 2011-2021 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/bench/examples.cpp b/src/bench/examples.cpp index dcd615b9da..72a9922e94 100644 --- a/src/bench/examples.cpp +++ b/src/bench/examples.cpp @@ -13,7 +13,7 @@ static void Trig(benchmark::Bench& bench) { double d = 0.01; bench.run([&] { - sum += sin(d); + sum = sum + sin(d); d += 0.000001; }); } diff --git a/src/bench/mempool_eviction.cpp b/src/bench/mempool_eviction.cpp index c9fa0fe66d..689cd54181 100644 --- a/src/bench/mempool_eviction.cpp +++ b/src/bench/mempool_eviction.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2020 The Bitcoin Core developers +// Copyright (c) 2011-2021 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/bench/mempool_stress.cpp b/src/bench/mempool_stress.cpp index e04a04f62a..35f32d0cb1 100644 --- a/src/bench/mempool_stress.cpp +++ b/src/bench/mempool_stress.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2020 The Bitcoin Core developers +// Copyright (c) 2011-2021 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -6,6 +6,7 @@ #include #include #include +#include #include @@ -27,14 +28,8 @@ struct Available { Available(CTransactionRef& ref, size_t tx_count) : ref(ref), tx_count(tx_count){} }; -static void ComplexMemPool(benchmark::Bench& bench) +static std::vector CreateOrderedCoins(FastRandomContext& det_rand, int childTxs, int min_ancestors) { - int childTxs = 800; - if (bench.complexityN() > 1) { - childTxs = static_cast(bench.complexityN()); - } - - FastRandomContext det_rand{true}; std::vector available_coins; std::vector ordered_coins; // Create some base transactions @@ -60,8 +55,10 @@ static void ComplexMemPool(benchmark::Bench& bench) size_t idx = det_rand.randrange(available_coins.size()); Available coin = available_coins[idx]; uint256 hash = coin.ref->GetHash(); - // biased towards taking just one ancestor, but maybe more - size_t n_to_take = det_rand.randrange(2) == 0 ? 1 : 1+det_rand.randrange(coin.ref->vout.size() - coin.vin_left); + // biased towards taking min_ancestors parents, but maybe more + size_t n_to_take = det_rand.randrange(2) == 0 ? + min_ancestors : + min_ancestors + det_rand.randrange(coin.ref->vout.size() - coin.vin_left); for (size_t i = 0; i < n_to_take; ++i) { tx.vin.emplace_back(); tx.vin.back().prevout = COutPoint(hash, coin.vin_left++); @@ -82,6 +79,17 @@ static void ComplexMemPool(benchmark::Bench& bench) ordered_coins.emplace_back(MakeTransactionRef(tx)); available_coins.emplace_back(ordered_coins.back(), tx_counter++); } + return ordered_coins; +} + +static void ComplexMemPool(benchmark::Bench& bench) +{ + FastRandomContext det_rand{true}; + int childTxs = 800; + if (bench.complexityN() > 1) { + childTxs = static_cast(bench.complexityN()); + } + std::vector ordered_coins = CreateOrderedCoins(det_rand, childTxs, /* min_ancestors */ 1); const auto testing_setup = MakeNoLogFileContext(CBaseChainParams::MAIN); CTxMemPool pool; LOCK2(cs_main, pool.cs); @@ -94,4 +102,22 @@ static void ComplexMemPool(benchmark::Bench& bench) }); } +static void MempoolCheck(benchmark::Bench& bench) +{ + FastRandomContext det_rand{true}; + const int childTxs = bench.complexityN() > 1 ? static_cast(bench.complexityN()) : 2000; + const std::vector ordered_coins = CreateOrderedCoins(det_rand, childTxs, /* min_ancestors */ 5); + const auto testing_setup = MakeNoLogFileContext(CBaseChainParams::MAIN, {"-checkmempool=1"}); + CTxMemPool pool; + LOCK2(cs_main, pool.cs); + const CBlockIndex* chain_tip = testing_setup.get()->m_node.chainman->ActiveChainstate().m_chain.Tip(); + const CCoinsViewCache& coins_tip = testing_setup.get()->m_node.chainman->ActiveChainstate().CoinsTip(); + for (auto& tx : ordered_coins) AddTx(tx, pool); + + bench.run([&]() NO_THREAD_SAFETY_ANALYSIS { + pool.check(chain_tip, coins_tip, /* spendheight */ 2); + }); +} + BENCHMARK(ComplexMemPool); +BENCHMARK(MempoolCheck); diff --git a/src/bench/nanobench.h b/src/bench/nanobench.h index 030d6ebf6a..916a7d61ef 100644 --- a/src/bench/nanobench.h +++ b/src/bench/nanobench.h @@ -33,7 +33,7 @@ // see https://semver.org/ #define ANKERL_NANOBENCH_VERSION_MAJOR 4 // incompatible API changes #define ANKERL_NANOBENCH_VERSION_MINOR 3 // backwards-compatible changes -#define ANKERL_NANOBENCH_VERSION_PATCH 4 // backwards-compatible bug fixes +#define ANKERL_NANOBENCH_VERSION_PATCH 6 // backwards-compatible bug fixes /////////////////////////////////////////////////////////////////////////////////////////////////// // public facing api - as minimal as possible @@ -88,13 +88,15 @@ } while (0) #endif -#if defined(__linux__) && defined(PERF_EVENT_IOC_ID) && defined(PERF_COUNT_HW_REF_CPU_CYCLES) && defined(PERF_FLAG_FD_CLOEXEC) && \ - !defined(ANKERL_NANOBENCH_DISABLE_PERF_COUNTERS) -// only enable perf counters on kernel 3.14 which seems to have all the necessary defines. The three PERF_... defines are not in -// kernel 2.6.32 (all others are). -# define ANKERL_NANOBENCH_PRIVATE_PERF_COUNTERS() 1 -#else -# define ANKERL_NANOBENCH_PRIVATE_PERF_COUNTERS() 0 +#define ANKERL_NANOBENCH_PRIVATE_PERF_COUNTERS() 0 +#if defined(__linux__) && !defined(ANKERL_NANOBENCH_DISABLE_PERF_COUNTERS) +# include +# if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 14, 0) +// PERF_COUNT_HW_REF_CPU_CYCLES only available since kernel 3.3 +// PERF_FLAG_FD_CLOEXEC since kernel 3.14 +# undef ANKERL_NANOBENCH_PRIVATE_PERF_COUNTERS +# define ANKERL_NANOBENCH_PRIVATE_PERF_COUNTERS() 1 +# endif #endif #if defined(__clang__) @@ -856,7 +858,7 @@ public: * @brief Retrieves all benchmark results collected by the bench object so far. * * Each call to run() generates a Result that is stored within the Bench instance. This is mostly for advanced users who want to - * see all the nitty gritty detials. + * see all the nitty gritty details. * * @return All results collected so far. */ @@ -2210,20 +2212,20 @@ struct IterationLogic::Impl { columns.emplace_back(10, 1, "err%", "%", rErrorMedian * 100.0); double rInsMedian = -1.0; - if (mResult.has(Result::Measure::instructions)) { + if (mBench.performanceCounters() && mResult.has(Result::Measure::instructions)) { rInsMedian = mResult.median(Result::Measure::instructions); columns.emplace_back(18, 2, "ins/" + mBench.unit(), "", rInsMedian / mBench.batch()); } double rCycMedian = -1.0; - if (mResult.has(Result::Measure::cpucycles)) { + if (mBench.performanceCounters() && mResult.has(Result::Measure::cpucycles)) { rCycMedian = mResult.median(Result::Measure::cpucycles); columns.emplace_back(18, 2, "cyc/" + mBench.unit(), "", rCycMedian / mBench.batch()); } if (rInsMedian > 0.0 && rCycMedian > 0.0) { columns.emplace_back(9, 3, "IPC", "", rCycMedian <= 0.0 ? 0.0 : rInsMedian / rCycMedian); } - if (mResult.has(Result::Measure::branchinstructions)) { + if (mBench.performanceCounters() && mResult.has(Result::Measure::branchinstructions)) { double rBraMedian = mResult.median(Result::Measure::branchinstructions); columns.emplace_back(17, 2, "bra/" + mBench.unit(), "", rBraMedian / mBench.batch()); if (mResult.has(Result::Measure::branchmisses)) { @@ -2402,6 +2404,14 @@ public: return (a + divisor / 2) / divisor; } + ANKERL_NANOBENCH_NO_SANITIZE("integer", "undefined") + static inline uint32_t mix(uint32_t x) noexcept { + x ^= x << 13; + x ^= x >> 17; + x ^= x << 5; + return x; + } + template ANKERL_NANOBENCH_NO_SANITIZE("integer", "undefined") void calibrate(Op&& op) { @@ -2441,15 +2451,10 @@ public: uint64_t const numIters = 100000U + (std::random_device{}() & 3); uint64_t n = numIters; uint32_t x = 1234567; - auto fn = [&]() { - x ^= x << 13; - x ^= x >> 17; - x ^= x << 5; - }; beginMeasure(); while (n-- > 0) { - fn(); + x = mix(x); } endMeasure(); detail::doNotOptimizeAway(x); @@ -2459,8 +2464,8 @@ public: beginMeasure(); while (n-- > 0) { // we now run *twice* so we can easily calculate the overhead - fn(); - fn(); + x = mix(x); + x = mix(x); } endMeasure(); detail::doNotOptimizeAway(x); diff --git a/src/bench/peer_eviction.cpp b/src/bench/peer_eviction.cpp index 46fd9d999e..f05f5e8f64 100644 --- a/src/bench/peer_eviction.cpp +++ b/src/bench/peer_eviction.cpp @@ -20,19 +20,17 @@ static void EvictionProtectionCommon( { using Candidates = std::vector; FastRandomContext random_context{true}; - bench.warmup(100).epochIterations(1100); Candidates candidates{GetRandomNodeEvictionCandidates(num_candidates, random_context)}; for (auto& c : candidates) { candidate_setup_fn(c); } - std::vector copies{ - static_cast(bench.epochs() * bench.epochIterations()), candidates}; - size_t i{0}; + bench.run([&] { - ProtectEvictionCandidatesByRatio(copies.at(i)); - ++i; + // creating a copy has an overhead of about 3%, so it does not influence the benchmark results much. + auto copy = candidates; + ProtectEvictionCandidatesByRatio(copy); }); } @@ -44,7 +42,7 @@ static void EvictionProtection0Networks250Candidates(benchmark::Bench& bench) bench, 250 /* num_candidates */, [](NodeEvictionCandidate& c) { - c.nTimeConnected = c.id; + c.m_connected = std::chrono::seconds{c.id}; c.m_network = NET_IPV4; }); } @@ -55,7 +53,7 @@ static void EvictionProtection1Networks250Candidates(benchmark::Bench& bench) bench, 250 /* num_candidates */, [](NodeEvictionCandidate& c) { - c.nTimeConnected = c.id; + c.m_connected = std::chrono::seconds{c.id}; c.m_is_local = false; if (c.id >= 130 && c.id < 240) { // 110 Tor c.m_network = NET_ONION; @@ -71,7 +69,7 @@ static void EvictionProtection2Networks250Candidates(benchmark::Bench& bench) bench, 250 /* num_candidates */, [](NodeEvictionCandidate& c) { - c.nTimeConnected = c.id; + c.m_connected = std::chrono::seconds{c.id}; c.m_is_local = false; if (c.id >= 90 && c.id < 160) { // 70 Tor c.m_network = NET_ONION; @@ -89,7 +87,7 @@ static void EvictionProtection3Networks050Candidates(benchmark::Bench& bench) bench, 50 /* num_candidates */, [](NodeEvictionCandidate& c) { - c.nTimeConnected = c.id; + c.m_connected = std::chrono::seconds{c.id}; c.m_is_local = (c.id == 28 || c.id == 47); // 2 localhost if (c.id >= 30 && c.id < 47) { // 17 I2P c.m_network = NET_I2P; @@ -107,7 +105,7 @@ static void EvictionProtection3Networks100Candidates(benchmark::Bench& bench) bench, 100 /* num_candidates */, [](NodeEvictionCandidate& c) { - c.nTimeConnected = c.id; + c.m_connected = std::chrono::seconds{c.id}; c.m_is_local = (c.id >= 55 && c.id < 60); // 5 localhost if (c.id >= 70 && c.id < 80) { // 10 I2P c.m_network = NET_I2P; @@ -125,7 +123,7 @@ static void EvictionProtection3Networks250Candidates(benchmark::Bench& bench) bench, 250 /* num_candidates */, [](NodeEvictionCandidate& c) { - c.nTimeConnected = c.id; + c.m_connected = std::chrono::seconds{c.id}; c.m_is_local = (c.id >= 140 && c.id < 160); // 20 localhost if (c.id >= 170 && c.id < 180) { // 10 I2P c.m_network = NET_I2P; diff --git a/src/bench/prevector.cpp b/src/bench/prevector.cpp index f4fabedab6..6343ed7848 100644 --- a/src/bench/prevector.cpp +++ b/src/bench/prevector.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2015-2020 The Bitcoin Core developers +// Copyright (c) 2015-2021 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/bench/rollingbloom.cpp b/src/bench/rollingbloom.cpp index 997ab56549..9f1679aa84 100644 --- a/src/bench/rollingbloom.cpp +++ b/src/bench/rollingbloom.cpp @@ -1,10 +1,10 @@ -// Copyright (c) 2016-2020 The Bitcoin Core developers +// Copyright (c) 2016-2021 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include -#include +#include static void RollingBloom(benchmark::Bench& bench) { @@ -13,16 +13,16 @@ static void RollingBloom(benchmark::Bench& bench) uint32_t count = 0; bench.run([&] { count++; - data[0] = count; - data[1] = count >> 8; - data[2] = count >> 16; - data[3] = count >> 24; + data[0] = count & 0xFF; + data[1] = (count >> 8) & 0xFF; + data[2] = (count >> 16) & 0xFF; + data[3] = (count >> 24) & 0xFF; filter.insert(data); - data[0] = count >> 24; - data[1] = count >> 16; - data[2] = count >> 8; - data[3] = count; + data[0] = (count >> 24) & 0xFF; + data[1] = (count >> 16) & 0xFF; + data[2] = (count >> 8) & 0xFF; + data[3] = count & 0xFF; filter.contains(data); }); } diff --git a/src/bench/rpc_blockchain.cpp b/src/bench/rpc_blockchain.cpp index c8886a4c23..2143bcf950 100644 --- a/src/bench/rpc_blockchain.cpp +++ b/src/bench/rpc_blockchain.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2016-2020 The Bitcoin Core developers +// Copyright (c) 2016-2021 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -23,8 +23,8 @@ struct TestBlockAndIndex { TestBlockAndIndex() { CDataStream stream(benchmark::data::block413567, SER_NETWORK, PROTOCOL_VERSION); - char a = '\0'; - stream.write(&a, 1); // Prevent compaction + std::byte a{0}; + stream.write({&a, 1}); // Prevent compaction stream >> block; @@ -40,7 +40,7 @@ static void BlockToJsonVerbose(benchmark::Bench& bench) { TestBlockAndIndex data; bench.run([&] { - auto univalue = blockToJSON(data.block, &data.blockindex, &data.blockindex, /*verbose*/ true); + auto univalue = blockToJSON(data.block, &data.blockindex, &data.blockindex, TxVerbosity::SHOW_DETAILS_AND_PREVOUT); ankerl::nanobench::doNotOptimizeAway(univalue); }); } @@ -50,7 +50,7 @@ BENCHMARK(BlockToJsonVerbose); static void BlockToJsonVerboseWrite(benchmark::Bench& bench) { TestBlockAndIndex data; - auto univalue = blockToJSON(data.block, &data.blockindex, &data.blockindex, /*verbose*/ true); + auto univalue = blockToJSON(data.block, &data.blockindex, &data.blockindex, TxVerbosity::SHOW_DETAILS_AND_PREVOUT); bench.run([&] { auto str = univalue.write(); ankerl::nanobench::doNotOptimizeAway(str); diff --git a/src/bench/rpc_mempool.cpp b/src/bench/rpc_mempool.cpp index 64997160a8..545d214d2b 100644 --- a/src/bench/rpc_mempool.cpp +++ b/src/bench/rpc_mempool.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2020 The Bitcoin Core developers +// Copyright (c) 2011-2021 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -13,7 +13,7 @@ static void AddTx(const CTransactionRef& tx, const CAmount& fee, CTxMemPool& poo { LockPoints lp; std::set > setPeginsSpent; - pool.addUnchecked(CTxMemPoolEntry(tx, fee, /* time */ 0, /* height */ 1, /* spendsCoinbase */ false, /* sigOpCost */ 4, lp, setPeginsSpent)); + pool.addUnchecked(CTxMemPoolEntry(tx, fee, /*time=*/0, /*entry_height=*/1, /*spends_coinbase=*/false, /*sigops_cost=*/4, lp, setPeginsSpent)); } static void RpcMempool(benchmark::Bench& bench) diff --git a/src/bench/verify_script.cpp b/src/bench/verify_script.cpp index 3ed950da07..e03a6d106f 100644 --- a/src/bench/verify_script.cpp +++ b/src/bench/verify_script.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2016-2020 The Bitcoin Core developers +// Copyright (c) 2016-2021 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -21,7 +21,7 @@ static void VerifyScriptBench(benchmark::Bench& bench) const ECCVerifyHandle verify_handle; ECC_Start(); - const int flags = SCRIPT_VERIFY_WITNESS | SCRIPT_VERIFY_P2SH; + const uint32_t flags{SCRIPT_VERIFY_WITNESS | SCRIPT_VERIFY_P2SH}; const int witnessversion = 0; // Key pair. diff --git a/src/bench/wallet_balance.cpp b/src/bench/wallet_balance.cpp index 24fc93cb6f..29f830a885 100644 --- a/src/bench/wallet_balance.cpp +++ b/src/bench/wallet_balance.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2012-2020 The Bitcoin Core developers +// Copyright (c) 2012-2021 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -10,25 +10,33 @@ #include #include #include +#include #include #include -static void WalletBalance(benchmark::Bench& bench, const bool set_dirty, const bool add_watchonly, const bool add_mine) +using wallet::CWallet; +using wallet::CreateMockWalletDatabase; +using wallet::DBErrors; +using wallet::GetBalance; +using wallet::WALLET_FLAG_DESCRIPTORS; + +static void WalletBalance(benchmark::Bench& bench, const bool set_dirty, const bool add_mine) { const auto test_setup = MakeNoLogFileContext(); const auto& ADDRESS_WATCHONLY = ADDRESS_BCRT1_UNSPENDABLE; - CWallet wallet{test_setup->m_node.chain.get(), "", CreateMockWalletDatabase()}; + CWallet wallet{test_setup->m_node.chain.get(), "", gArgs, CreateMockWalletDatabase()}; { - wallet.SetupLegacyScriptPubKeyMan(); + LOCK(wallet.cs_wallet); + wallet.SetWalletFlag(WALLET_FLAG_DESCRIPTORS); + wallet.SetupDescriptorScriptPubKeyMans(); if (wallet.LoadWallet() != DBErrors::LOAD_OK) assert(false); } auto handler = test_setup->m_node.chain->handleNotifications({&wallet, [](CWallet*) {}}); const std::optional address_mine{add_mine ? std::optional{getnewaddress(wallet)} : std::nullopt}; - if (add_watchonly) importaddress(wallet, ADDRESS_WATCHONLY); for (int i = 0; i < 100; ++i) { generatetoaddress(test_setup->m_node, address_mine.value_or(ADDRESS_WATCHONLY)); @@ -36,20 +44,19 @@ static void WalletBalance(benchmark::Bench& bench, const bool set_dirty, const b } SyncWithValidationInterfaceQueue(); - auto bal = wallet.GetBalance(); // Cache + auto bal = GetBalance(wallet); // Cache bench.run([&] { if (set_dirty) wallet.MarkDirty(); - bal = wallet.GetBalance(); + bal = GetBalance(wallet); if (add_mine) assert(bal.m_mine_trusted[::policyAsset] > 0); - if (add_watchonly) assert(bal.m_watchonly_trusted[::policyAsset] > 0); }); } -static void WalletBalanceDirty(benchmark::Bench& bench) { WalletBalance(bench, /* set_dirty */ true, /* add_watchonly */ true, /* add_mine */ true); } -static void WalletBalanceClean(benchmark::Bench& bench) { WalletBalance(bench, /* set_dirty */ false, /* add_watchonly */ true, /* add_mine */ true); } -static void WalletBalanceMine(benchmark::Bench& bench) { WalletBalance(bench, /* set_dirty */ false, /* add_watchonly */ false, /* add_mine */ true); } -static void WalletBalanceWatch(benchmark::Bench& bench) { WalletBalance(bench, /* set_dirty */ false, /* add_watchonly */ true, /* add_mine */ false); } +static void WalletBalanceDirty(benchmark::Bench& bench) { WalletBalance(bench, /* set_dirty */ true, /* add_mine */ true); } +static void WalletBalanceClean(benchmark::Bench& bench) { WalletBalance(bench, /* set_dirty */ false, /* add_mine */ true); } +static void WalletBalanceMine(benchmark::Bench& bench) { WalletBalance(bench, /* set_dirty */ false, /* add_mine */ true); } +static void WalletBalanceWatch(benchmark::Bench& bench) { WalletBalance(bench, /* set_dirty */ false, /* add_mine */ false); } BENCHMARK(WalletBalanceDirty); BENCHMARK(WalletBalanceClean); diff --git a/src/bitcoin-chainstate.cpp b/src/bitcoin-chainstate.cpp new file mode 100644 index 0000000000..f93197350d --- /dev/null +++ b/src/bitcoin-chainstate.cpp @@ -0,0 +1,262 @@ +// Copyright (c) 2022 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. +// +// The bitcoin-chainstate executable serves to surface the dependencies required +// by a program wishing to use Bitcoin Core's consensus engine as it is right +// now. +// +// DEVELOPER NOTE: Since this is a "demo-only", experimental, etc. executable, +// it may diverge from Bitcoin Core's coding style. +// +// It is part of the libbitcoinkernel project. + +#include +#include +#include +#include +#include +#include +#include +#include