mirror of
https://github.com/ElementsProject/elements.git
synced 2026-08-13 12:33:42 +02:00
contrib: upstream my merge-prs.sh changes
This commit is contained in:
Byron Hambly
parent
7791d31bcf
commit
bd59321098
2 changed files with 147 additions and 67 deletions
|
|
@ -3,38 +3,58 @@
|
|||
export LC_ALL=C
|
||||
set -eo pipefail
|
||||
|
||||
# Setup base branch and Bitcoin/Elements remote names.
|
||||
BASE_ORIG=merged-master
|
||||
BASE="${BASE_ORIG}"
|
||||
BITCOIN_UPSTREAM_REMOTE=bitcoin
|
||||
BITCOIN_UPSTREAM="${BITCOIN_UPSTREAM_REMOTE}/master"
|
||||
# ELEMENTS_UPSTREAM_REMOTE=upstream
|
||||
# ELEMENTS_UPSTREAM="${ELEMENTS_UPSTREAM_REMOTE}/master"
|
||||
export BITCOIN_UPSTREAM="${BITCOIN_UPSTREAM_REMOTE}/master"
|
||||
ELEMENTS_UPSTREAM_REMOTE=upstream
|
||||
export ELEMENTS_UPSTREAM="${ELEMENTS_UPSTREAM_REMOTE}/master"
|
||||
|
||||
# START USER CONFIG:
|
||||
# Set your target upstream here
|
||||
TARGET_UPSTREAM=$BITCOIN_UPSTREAM
|
||||
TARGET_NAME="Bitcoin"
|
||||
PR_PREFIX="bitcoin/bitcoin"
|
||||
# TARGET_UPSTREAM=$ELEMENTS_UPSTREAM
|
||||
# TARGET_NAME="Elements"
|
||||
# PR_PREFIX="ElementsProject/elements"
|
||||
|
||||
# Set your git worktree location here. This is where the merges will be done, and where you should checkout the merged-master branch.
|
||||
WORKTREE="/home/byron/code/elements-worktree"
|
||||
|
||||
# Set your parallellism during build/test. You probably want as many cores as possible.
|
||||
# Parallel functional tests can somewhat exceed your core count, depends on the build machine CPU/RAM.
|
||||
PARALLEL_BUILD=23 # passed to make -j
|
||||
PARALLEL_TEST=46 # passed to test_runner.py --jobs
|
||||
PARALLEL_FUZZ=12 # passed to test_runner.py -j when fuzzing
|
||||
|
||||
# Setup a ccache dir if necessary.
|
||||
#export CCACHE_DIR="/tmp/ccache"
|
||||
#export CCACHE_MAXSIZE="20G"
|
||||
|
||||
# Set and export a WEBHOOK environment variable to a Discord webhook URL outside of this script to get notifications of progress and failures.
|
||||
|
||||
# We don't currently fuzz during merging. Check fuzzing and CI after a merge run.
|
||||
# Replace this with the location where we should put the fuzz test corpus
|
||||
BITCOIN_QA_ASSETS="${HOME}/.tmp/bitcoin/qa-assets"
|
||||
FUZZ_CORPUS="${BITCOIN_QA_ASSETS}/fuzz_seed_corpus/"
|
||||
mkdir -p "$(dirname "${BITCOIN_QA_ASSETS}")"
|
||||
# BITCOIN_QA_ASSETS="${HOME}/code/bitcoin/qa-assets"
|
||||
# FUZZ_CORPUS="${BITCOIN_QA_ASSETS}/fuzz_seed_corpus/"
|
||||
|
||||
# BEWARE: On some systems /tmp/ gets periodically cleaned, which may cause
|
||||
# random files from this directory to disappear based on timestamp, and
|
||||
# make git very confused
|
||||
WORKTREE="${HOME}/.tmp/elements-merge-worktree"
|
||||
mkdir -p "${HOME}/.tmp"
|
||||
|
||||
# These should be tuned to your machine; below values are for an 8-core
|
||||
# 16-thread macbook pro
|
||||
PARALLEL_BUILD=4 # passed to make -j
|
||||
PARALLEL_TEST=12 # passed to test_runner.py --jobs
|
||||
PARALLEL_FUZZ=8 # passed to test_runner.py -j when fuzzing
|
||||
# END USER CONFIG
|
||||
|
||||
# Script
|
||||
SKIP_MERGE=0
|
||||
DO_BUILD=1
|
||||
KEEP_GOING=1
|
||||
DO_TEST=1
|
||||
DO_FUZZ=0
|
||||
NUM=15
|
||||
COUNT=0
|
||||
|
||||
if [[ "$1" == "setup" ]]; then
|
||||
echo "Setting up..."
|
||||
echo
|
||||
git config remote.upstream.url >/dev/null || remote add upstream "https://github.com/ElementsProject/elements.git"
|
||||
git config remote.upstream.url >/dev/null || git remote add upstream "https://github.com/ElementsProject/elements.git"
|
||||
git config remote.bitcoin.url >/dev/null || git remote add bitcoin "https://github.com/bitcoin/bitcoin.git"
|
||||
if git worktree list --porcelain | grep --silent prunable; then
|
||||
echo "You have stale git worktrees, please either fix them or run 'git worktree prune'."
|
||||
|
|
@ -45,14 +65,7 @@ if [[ "$1" == "setup" ]]; then
|
|||
echo "Fetching all remotes..."
|
||||
echo
|
||||
git fetch --all
|
||||
echo
|
||||
#echo "Cloning fuzz test corpus..."
|
||||
#echo
|
||||
#if [[ ! -d "${BITCOIN_QA_ASSETS}" ]]; then
|
||||
# cd "$(dirname ${BITCOIN_QA_ASSETS})" && git clone https://github.com/bitcoin-core/qa-assets.git
|
||||
#fi
|
||||
#echo
|
||||
echo "Done! Remember to also check out merged-master, and push it back up when finished."
|
||||
echo "Done! Remember to also checkout merged-master at ${WORKTREE}"
|
||||
exit 0
|
||||
elif [[ "$1" == "continue" ]]; then
|
||||
SKIP_MERGE=1
|
||||
|
|
@ -62,17 +75,35 @@ elif [[ "$1" == "list-only" ]]; then
|
|||
DO_BUILD=0
|
||||
elif [[ "$1" == "step" ]]; then
|
||||
KEEP_GOING=0
|
||||
elif [[ "$1" == "merge-only" ]]; then
|
||||
SKIP_MERGE=0
|
||||
KEEP_GOING=0
|
||||
DO_BUILD=0
|
||||
DO_TEST=0
|
||||
elif [[ "$1" == "step-continue" ]]; then
|
||||
SKIP_MERGE=1
|
||||
KEEP_GOING=0
|
||||
elif [[ "$1" == "step-test" ]]; then
|
||||
SKIP_MERGE=1
|
||||
KEEP_GOING=0
|
||||
DO_BUILD=0
|
||||
elif [[ "$1" == "step-fuzz" ]]; then
|
||||
SKIP_MERGE=1
|
||||
KEEP_GOING=0
|
||||
DO_BUILD=0
|
||||
DO_TEST=0
|
||||
elif [[ "$1" == "analyze" ]]; then
|
||||
DO_BUILD=0
|
||||
DO_TEST=0
|
||||
else
|
||||
echo "Usage: $0 <setup|list-only|go|continue|step|step-continue>"
|
||||
echo "Usage: $0 <setup|list-only|go|continue|step|step-continue|analyze>"
|
||||
echo " setup will configure your repository for the first run of this script"
|
||||
echo " list-only will simply list all the PRs yet to be done"
|
||||
echo " go will try to merge every PR, building/testing each"
|
||||
echo " continue assumes the first git-merge has already happened, and starts with building"
|
||||
echo " step will try to merge/build/test a single PR"
|
||||
echo " step-continue assumes the first git-merge has already happened, and will try to build/test a single PR"
|
||||
echo " analyze will analyze the next $NUM PRs and count conflicts NB: DO NOT CTRL+C THIS PROCESS"
|
||||
echo
|
||||
echo "Prior to use, please create a git worktree for the elements repo at:"
|
||||
echo " $WORKTREE"
|
||||
|
|
@ -101,54 +132,95 @@ if [[ "$SKIP_MERGE" == "1" ]]; then
|
|||
fi
|
||||
|
||||
## Get full list of merges
|
||||
# for elements
|
||||
# COMMITS=$(git -C "$WORKTREE" log "$ELEMENTS_UPSTREAM" --not $BASE --merges --first-parent --pretty='format:%ct %cI %h Elements %s')
|
||||
# for bitcoin
|
||||
COMMITS=$(git -C "$WORKTREE" log "$BITCOIN_UPSTREAM" --not $BASE --merges --first-parent --pretty='format:%ct %cI %h Bitcoin %s')
|
||||
COMMITS=$(git -C "$WORKTREE" log "$TARGET_UPSTREAM" --not $BASE --merges --first-parent --pretty="format:%ct %cI %h $TARGET_NAME %s")
|
||||
|
||||
cd "$WORKTREE"
|
||||
cd "$WORKTREE" || exit 1
|
||||
|
||||
VERBOSE=1
|
||||
|
||||
echo start > merge.log
|
||||
|
||||
quietly () {
|
||||
if [[ "$VERBOSE" == "1" ]]; then
|
||||
"$@"
|
||||
date | tee --append merge.log
|
||||
time "$@" 2>&1 | tee --append merge.log
|
||||
else
|
||||
chronic "$@"
|
||||
fi
|
||||
}
|
||||
|
||||
notify () {
|
||||
local MESSAGE="$1"
|
||||
local JSON="{\"content\": \"$MESSAGE\"}"
|
||||
if [ -n "$WEBHOOK" ]; then
|
||||
curl -d "$JSON" -H "Content-Type: application/json" "$WEBHOOK"
|
||||
else
|
||||
echo "$MESSAGE"
|
||||
fi
|
||||
if [[ "$2" == "1" ]]; then
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
## Sort by unix timestamp and iterate over them
|
||||
#echo "$ELT_COMMITS" "$BTC_COMMITS" | sort -n -k1 | while read line
|
||||
echo "$COMMITS" | tac | while read -r line
|
||||
do
|
||||
echo
|
||||
echo "=-=-=-=-=-=-=-=-=-=-="
|
||||
echo
|
||||
|
||||
echo -e "$line"
|
||||
## Extract data and output what we're doing
|
||||
DATE=$(echo "$line" | cut -d ' ' -f 2)
|
||||
HASH=$(echo "$line" | cut -d ' ' -f 3)
|
||||
CHAIN=$(echo "$line" | cut -d ' ' -f 4)
|
||||
PR_ID=$(echo "$line" | cut -d ' ' -f 6 | tr -d :)
|
||||
PR_ID_ALT=$(echo "$line" | cut -d ' ' -f 8 | tr -d :)
|
||||
PR_ID=$(echo "$line" | grep -o -P "#\d+")
|
||||
|
||||
if [[ "$PR_ID" == "pull" ]]; then
|
||||
PR_ID="${PR_ID_ALT}"
|
||||
fi
|
||||
echo -e "$CHAIN PR \e[37m$PR_ID \e[33m$HASH\e[0m on \e[32m$DATE\e[0m "
|
||||
GIT_HEAD=$(git rev-parse HEAD)
|
||||
|
||||
## Do it
|
||||
if [[ "$1" == "list-only" ]]; then
|
||||
echo -e "$line"
|
||||
continue
|
||||
fi
|
||||
if [[ "$1" == "analyze" ]]; then
|
||||
COUNT=$((COUNT + 1))
|
||||
# todo: count conflicts in critical files
|
||||
# CRITICAL_FILES=("src/wallet/spend.h", "src/wallet/spend.cpp")
|
||||
MERGE_FILE="/tmp/$HASH.merge"
|
||||
DIFF_FILE="/tmp/$HASH.diff"
|
||||
git -C "$WORKTREE" merge "$HASH" --no-ff -m "Merge $HASH into merged_master ($CHAIN PR $PR_PREFIX$PR_ID)" > "$MERGE_FILE" || true
|
||||
git -C "$WORKTREE" diff > "$DIFF_FILE"
|
||||
git -C "$WORKTREE" reset --hard "$GIT_HEAD" > /dev/null
|
||||
# FILES=$(grep "CONFLICT" "$MERGE_FILE")
|
||||
NUM_FILES=$(grep -c "CONFLICT" "$MERGE_FILE")
|
||||
NUM_CONFLICTS=$(grep -c "<<<<<<<" "$DIFF_FILE")
|
||||
echo "$COUNT. Merge up to $PR_ID ($HASH) has $NUM_CONFLICTS conflicts in $NUM_FILES files."
|
||||
if [[ "$COUNT" == "$NUM" ]]; then
|
||||
exit 0
|
||||
else
|
||||
continue
|
||||
fi
|
||||
fi
|
||||
notify "starting merge of $PR_ID"
|
||||
|
||||
# check for stoppers and halt if found
|
||||
# a stopper is normally the PR after the version has been changed
|
||||
# ie. the branch point we want to stop at for this version
|
||||
STOPPERS=(
|
||||
"#30716" # bitcoin v28
|
||||
"#32041" # bitcoin v29
|
||||
)
|
||||
for STOPPER in "${STOPPERS[@]}"
|
||||
do
|
||||
if [[ "$PR_ID" == *"$STOPPER"* ]]; then
|
||||
echo "Found $STOPPER in $PR_ID! Exiting."
|
||||
notify "hit stopper, exiting"
|
||||
exit 1
|
||||
else
|
||||
echo "Didn't find $STOPPER in $PR_ID. Continuing."
|
||||
fi
|
||||
done
|
||||
|
||||
if [[ "$SKIP_MERGE" == "1" ]]; then
|
||||
echo -e "Continuing build of \e[37m$PR_ID\e[0m at $(date)"
|
||||
else
|
||||
echo -e "Start merge/build of \e[37m$PR_ID\e[0m at $(date)"
|
||||
git -C "$WORKTREE" merge "$HASH" --no-ff -m "Merge $HASH into merged_master ($CHAIN PR $PR_ID)"
|
||||
git -C "$WORKTREE" merge "$HASH" --no-ff -m "Merge $HASH into merged_master ($CHAIN PR $PR_PREFIX$PR_ID)" || notify "fail merge" 1
|
||||
fi
|
||||
|
||||
if [[ "$DO_BUILD" == "1" ]]; then
|
||||
|
|
@ -163,35 +235,43 @@ do
|
|||
# The following is an expansion of `make check` that skips the libsecp
|
||||
# tests and also the benchmarks (though it does build them!)
|
||||
echo "Building"
|
||||
quietly make -j"$PARALLEL_BUILD" -k
|
||||
# quietly make -j1 check
|
||||
echo "Linting"
|
||||
quietly ./ci/lint/06_script.sh
|
||||
quietly make -j"$PARALLEL_BUILD" -k || notify "fail build" 1
|
||||
# todo: fix linting step
|
||||
# echo "Linting"
|
||||
# quietly ./ci/lint/06_script.sh || notify "fail lint"
|
||||
fi
|
||||
|
||||
if [[ "$DO_TEST" == "1" ]]; then
|
||||
echo "Testing"
|
||||
quietly ./src/qt/test/test_elements-qt
|
||||
quietly ./src/test/test_bitcoin
|
||||
quietly ./src/bench/bench_bitcoin
|
||||
quietly ./test/util/bitcoin-util-test.py
|
||||
quietly ./test/util/rpcauth-test.py
|
||||
quietly make -C src/univalue/ check
|
||||
quietly ./src/qt/test/test_elements-qt || notify "fail test qt" 1
|
||||
quietly ./src/test/test_bitcoin || notify "fail test bitcoin" 1
|
||||
quietly ./src/bench/bench_bitcoin || notify "fail test bench" 1
|
||||
quietly ./test/util/test_runner.py || notify "fail test util" 1
|
||||
quietly ./test/util/rpcauth-test.py || notify "fail test rpc" 1
|
||||
echo "Functional testing"
|
||||
quietly ./test/functional/test_runner.py --jobs="$PARALLEL_TEST"
|
||||
quietly ./test/functional/test_runner.py --jobs="$PARALLEL_TEST" || notify "fail test runner" 1
|
||||
fi
|
||||
|
||||
if [[ "$DO_FUZZ" == "1" ]]; then
|
||||
echo "Cleaning for fuzz"
|
||||
quietly make distclean || true
|
||||
quietly git -C "$WORKTREE" clean -xf
|
||||
echo "Building for fuzz"
|
||||
quietly ./autogen.sh
|
||||
# TODO turn on `,integer` after this rebase
|
||||
quietly ./configure --with-incompatible-bdb --enable-fuzz --with-sanitizers=address,fuzzer,undefined CC=clang CXX=clang++
|
||||
quietly ./configure --enable-fuzz --with-sanitizers=address,fuzzer,undefined CC="ccache clang" CXX="ccache clang++"
|
||||
quietly make -j"$PARALLEL_BUILD" -k
|
||||
echo "Fuzzing"
|
||||
quietly ./test/fuzz/test_runner.py -j"$PARALLEL_FUZZ" "${FUZZ_CORPUS}"
|
||||
quietly ./test/fuzz/test_runner.py -j"$PARALLEL_FUZZ" "${FUZZ_CORPUS}" || notify "fail fuzz" 1
|
||||
fi
|
||||
|
||||
if [[ "$KEEP_GOING" == "0" ]]; then
|
||||
exit 1
|
||||
notify "$PR_ID done, exiting"
|
||||
exit 0
|
||||
else
|
||||
echo "$PR_ID done, continuing"
|
||||
fi
|
||||
|
||||
# bummer1.sh
|
||||
SKIP_MERGE=0
|
||||
echo "end" >> merge.log
|
||||
done
|
||||
|
|
|
|||
|
|
@ -24,8 +24,8 @@ github-merge.py <PR_NUMBER>
|
|||
For example for PR 1518 I see this output:
|
||||
|
||||
```
|
||||
ElementsProject/elements#1518 Avoid Simplicity header dependency propogation into master
|
||||
* 6c7788adf373cd8f0dc5c4fe2b7674625631721e Avoid Simplicity header dependency propogation (Russell O'Connor) (upstream/simplicity, pull/1518/head)
|
||||
ElementsProject/elements#1518 Avoid Simplicity header dependency propagation into master
|
||||
* 6c7788adf373cd8f0dc5c4fe2b7674625631721e Avoid Simplicity header dependency propagation (Russell O'Connor) (upstream/simplicity, pull/1518/head)
|
||||
|
||||
Dropping you on a shell so you can try building/testing the merged source.
|
||||
Run 'git diff HEAD~' to show the changes being merged.
|
||||
|
|
@ -47,10 +47,10 @@ exit
|
|||
In our example this results in the following output:
|
||||
|
||||
```
|
||||
[pull/1518/local-merge 5e1a950e52] Merge ElementsProject/elements#1518: Avoid Simplicity header dependency propogation
|
||||
[pull/1518/local-merge 5e1a950e52] Merge ElementsProject/elements#1518: Avoid Simplicity header dependency propagation
|
||||
Date: Thu Jan 22 14:49:26 2026 +0200
|
||||
ElementsProject/elements#1518 Avoid Simplicity header dependency propogation into master
|
||||
* 6c7788adf373cd8f0dc5c4fe2b7674625631721e Avoid Simplicity header dependency propogation (Russell O'Connor) (upstream/simplicity, pull/1518/head)
|
||||
ElementsProject/elements#1518 Avoid Simplicity header dependency propagation into master
|
||||
* 6c7788adf373cd8f0dc5c4fe2b7674625631721e Avoid Simplicity header dependency propagation (Russell O'Connor) (upstream/simplicity, pull/1518/head)
|
||||
ACKs:
|
||||
* ACK 6c7788a; built and tested locally (delta1)
|
||||
* ACK 6c7788adf373cd8f0dc5c4fe2b7674625631721e; successfully ran local tests (apoelstra)
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue