Replace Travis with Cirrus

* initial cirrus config

* cirrus

* f*ck caching

* images and deleting travis

* check

* bugfix weird ci-issue (directory exists)

* kick

* revert

* kick

* fix

Co-authored-by: benk10 <ben.kaufman10@gmail.com>

.cirrus.yml

@@ -0,0 +1,57 @@
+container:
+#  image: python:slim
+#  image: ubuntu:focal
+#  image: python:3.8-buster
+  image: registry.gitlab.com/cryptoadvance/specter-desktop/cirrus-focal:latest
+
+# We assume here that we're having a proper python3 system including virtualenv and pip
+prep_stuff_template: &PREP_STUFF_TEMPLATE
+  verify_script: python3 --version && virtualenv --version && pip3 --version
+  bitcoind_installation_cache:
+    folder: ./tests/bitcoin
+    populate_script: set -o errexit; source ./tests/install_bitcoind.sh binary
+  pip_script:
+    #folder: /tmp/cirrus-ci-build/.env
+    #fingerprint_script: echo muh && cat requirements.txt && cat test_requirements.txt
+    #populate_script: 
+    - virtualenv --python=python .env 
+    - source ./.env/bin/activate
+    - pip3 install -r requirements.txt --require-hashes && pip3 install -r test_requirements.txt
+  install_script: 
+    - pwd
+    - ls -l ./.env/bin
+    - source ./.env/bin/activate
+    - echo $PATH
+    - ls -l /usr/bin 
+    - pip3 install -e .
+
+test_task:
+  pre_prep_script: 
+    - apt-get update && apt-get install -y --no-install-recommends python3-dev python3-pip wget
+  << : *PREP_STUFF_TEMPLATE
+  test_script: 
+    - pwd
+    - ls -l ./.env/bin
+    - source ./.env/bin/activate
+    - echo $PATH
+    #- pip3 install -e .
+    - pytest --cov=cryptoadvance
+
+cypress_test_task:
+  container:
+    image: registry.gitlab.com/cryptoadvance/specter-desktop/cypress-python
+  pre_prep_script: 
+    - apt-get update && apt-get install -y --no-install-recommends python3-dev python3-pip python3-virtualenv bc
+    # The stupid old debian-package is not installing a proper binary but just the python-package
+    - echo -e '#!/bin/bash\npython3 -m virtualenv "$@"' > /usr/local/bin/virtualenv
+    - chmod +x /usr/local/bin/virtualenv
+    - virtualenv --version
+  << : *PREP_STUFF_TEMPLATE
+  npm_cache:
+    folder: ./node_modules
+    fingerprint_script: cat package-lock.json
+    populate_script: npm ci
+  cypress_script:
+    - source ./.env/bin/activate
+    #- pip3 install -e .
+    - ./utils/test-cypress.sh --debug run 

.gitignore

@@ -28,4 +28,6 @@ cypress/screenshots
 node_modules
 btcd-conn.json
 tests/bitcoin
+tests/bitcoin.binary
+tests/bitcoin.compile
 

.travis.yml

@@ -1,73 +0,0 @@
-language: python
-python:
-  - "3.8"
-os: linux
-dist: focal
-cache:
-  pip: true
-  ccache: true
-  directories:
-    - tests/bitcoin
-  # cypress
-    - ~/.npm
-    - ./node_modules
-    - ~/.cache
-  override:
-    - npm ci
-    - npm run cy:verify
-
-addons:
-  apt:
-    packages:
-      - build-essential
-      - curl
-      - git
-      - libsdl2-dev
-      - libsdl2-image-dev
-      - gcc-arm-none-eabi
-      - libnewlib-arm-none-eabi
-      - libudev-dev
-      - libtool
-      - autotools-dev
-      - automake
-      - pkg-config
-      - bsdmainutils
-      - libssl-dev
-      - libevent-dev
-      - libboost-system-dev
-      - libboost-filesystem-dev
-      - libboost-chrono-dev
-      - libboost-test-dev
-      - libboost-thread-dev
-      - libusb-1.0-0-dev
-      - protobuf-compiler
-      - cython3
-      - ccache
-# From https://dev-notes.eu/2020/10/Build-Bitcoin-Core-in-Ubuntu/
-      - automake
-      - cmake
-      - curl
-      - g++-multilib 
-      - binutils-gold
-      - patch
-# For cypress
-      - nodejs
-
-
-before_install:
-  - npm ci
-  - docker pull registry.gitlab.com/cryptoadvance/specter-desktop/python-bitcoind:latest
-  - set -o errexit; source ./tests/install_bitcoind.sh
-install:
-  - npm ci
-script: 
-  - pip install -e .
-  - pip install -r test_requirements.txt
-  - pytest --cov=cryptoadvance  
-  - ./utils/test-cypress.sh --debug run 
-before_cache:
-  - rm -f $HOME/.cache/pip/log/debug.log
-  - rm -f ./tests/bitcoin/config.log
-  - rm -f ./tests/bitcoin/contrib/db4/db-4.8.30.NC/build_unix/config.log
-  - rm -f ./tests/bitcoin/src/secp256k1/config.log
-  - rm -f ./tests/bitcoin/src/univalue/config.log

docker/cirrus-focal/Dockerfile

@@ -0,0 +1,9 @@
+from ubuntu:focal
+
+# 1. python-stuff and HWI dependencies
+# 2. capability to build bitcoind
+# 3. cypress dependencies
+RUN apt update && DEBIAN_FRONTEND="noninteractive" apt-get install --no-install-recommends -y \
+    libusb-1.0-0-dev libudev-dev python3 python3-virtualenv \
+    build-essential libtool autotools-dev automake pkg-config bsdmainutils libevent-dev libboost-dev libboost-system-dev libboost-filesystem-dev libboost-test-dev bc  \
+    nodejs npm libgtk2.0-0 libgtk-3-0 libgbm-dev libnotify-dev libgconf-2-4 libnss3 libxss1 libasound2 libxtst6 xauth xvfb

docker/cirrus-focal/README.md

@@ -0,0 +1,10 @@
+An image used to run the build on cirrus.
+
+Create it like this:
+
+``` 
+docker build . -t registry.gitlab.com/cryptoadvance/specter-desktop/cirrus-focal:latest
+docker push registry.gitlab.com/cryptoadvance/specter-desktop/cirrus-focal:latest
+```
+
+Check the `.cirrus.yml` on how this is used

docker/cypress-python/Dockerfile

@@ -0,0 +1,10 @@
+FROM cypress/base:12
+
+RUN apt-get update && apt-get install -y \
+    python3-pip python3-virtualenv zip unzip file apt libusb-1.0-0-dev libudev-dev \
+    bc
+
+WORKDIR /test
+RUN rm -rf node_modules package-lock.json ~/.cache/Cypress
+RUN npm install --save-dev cypress@7.1.0
+RUN $(npm bin)/cypress verify

docker/cypress-python/Readme.md

@@ -0,0 +1,7 @@
+
+An image, ready to be used with cypress but also provides all the dependencies we need for testing specter-desktop.
+
+```
+docker build . -t registry.gitlab.com/cryptoadvance/specter-desktop/cypress-python
+docker push registry.gitlab.com/cryptoadvance/specter-desktop/cypress-python
+```

docs/continuous-integration.md

@@ -1,16 +1,15 @@
 # introduction
-Specter-Desktop is using GitLab, Travis-CI and GitHub-Actions for continuous integration purposes but GitHub-actions only for Blackify so far. It might be more effort using more than one CI-approach but it makes us also more resilient. 
-GitLab and Travis-CI have both advantages and disadvantages so ... let's use both!
+Specter-Desktop is using GitLab, Cirrus and GitHub-Actions for continuous integration purposes but GitHub-actions only for Blackify so far. It might be more effort using more than one CI-approach but it makes us also more resilient. 
+GitLab and Cirrus have both advantages and disadvantages so ... let's use both!
 GitLab:
 * is completely open Source for server- and clients
 * the gitlab-runner can run docker and is itself running on docker
 * but does not support Pull-Requests
 * needs to have bitcoind in a prepared docker-container which binds the build to that version
 
-Travis-CI:
+Cirrus-CI:
 * supports the PR-model
-* quite easy to setup even without docker
-* enables to test against any specific version of bitcoind we would like to
+* quite easy to setup even though it's using docker
 
 # Gitlab
 
@@ -35,7 +34,11 @@ start_bitcoind-function:
 
 # Travis-CI
 
-Travis-CI setup is very straightforward. As we're using the build-cache, the bitcoind sources, npm-setup (for cypress) and build is cached. Therefore such a build would only take 10 minutes. If the master-branch has new commits, bitcoind gets automatically rebuilt and the tests are running against the new version (tests/install_bitcoind.sh).py
+We're no longer using travis-ci due to the abuse-detection-system going wild on us.
+
+# Cirrus-CI
+
+[Cirrus-CI](https://cirrus-ci.org) is used by Bitcoin-Core and HWI and is a quite good replacement for travis. We're using it only for PRs so far. The [../.cirrus.yml] file defines the build. We have two task, one for pytest and one for the cypress-tests.
 
 # Releasing
 

package.json

@@ -8,7 +8,7 @@
     "test": "tests"
   },
   "dependencies": {
-    "cypress": "^5.6.0",
+    "cypress": "^7.1.0",
     "rimraf": "^3.0.2",
     "wait-on": "^5.2.0"
   },
@@ -18,12 +18,12 @@
   },
   "repository": {
     "type": "git",
-    "url": "git+https://github.com/k9ert/specter-desktop.git"
+    "url": "git+https://github.com/cryptoadvance/specter-desktop.git"
   },
   "author": "",
   "license": "ISC",
   "bugs": {
-    "url": "https://github.com/k9ert/specter-desktop/issues"
+    "url": "https://github.com/cryptoadvance/specter-desktop/issues"
   },
-  "homepage": "https://github.com/k9ert/specter-desktop#readme"
+  "homepage": "https://github.com/cryptoadvance/specter-desktop#readme"
 }

src/cryptoadvance/specter/cli/cli_bitcoind.py

@@ -151,6 +151,10 @@ def bitcoind(
             my_bitcoind = BitcoindPlainController(
                 bitcoind_path="tests/bitcoin/src/bitcoind"
             )  # always prefer the self-compiled bitcoind if existing
+        elif os.path.isfile("tests/bitcoin/bin/bitcoind"):
+            my_bitcoind = BitcoindPlainController(
+                bitcoind_path="tests/bitcoin/bin/bitcoind"
+            )  # next take the self-installed binary if existing
         else:
             my_bitcoind = (
                 BitcoindPlainController()

src/cryptoadvance/specter/wallet_manager.py

@@ -1,13 +1,19 @@
-import os, json, logging, shutil, threading
-from io import BytesIO
-from collections import OrderedDict
-from .util.descriptor import AddChecksum
-from .helpers import alias, load_jsons
-from .rpc import get_default_datadir, RpcError
-from .specter_error import SpecterError
-from .wallet import Wallet
-from .persistence import delete_file, delete_folder
+import json
+import logging
+import os
+import pathlib
+import shutil
+import threading
 import traceback
+from collections import OrderedDict
+from io import BytesIO
+
+from .helpers import alias, load_jsons
+from .persistence import delete_file, delete_folder
+from .rpc import RpcError, get_default_datadir
+from .specter_error import SpecterError
+from .util.descriptor import AddChecksum
+from .wallet import Wallet
 
 logger = logging.getLogger()
 
@@ -74,8 +80,7 @@ class WalletManager:
         self.working_folder = None
         if self.chain is not None and self.data_folder is not None:
             self.working_folder = os.path.join(self.data_folder, self.chain)
-        if self.working_folder is not None and not os.path.isdir(self.working_folder):
-            os.mkdir(self.working_folder)
+            pathlib.Path(self.working_folder).mkdir(parents=True, exist_ok=True)
         if rpc is not None:
             self.rpc = rpc
         self.wallets_update_list = {}

tests/conftest.py

@@ -58,6 +58,10 @@ def instantiate_bitcoind_controller(docker, request, rpcport=18543, extra_args=[
             bitcoind_controller = BitcoindPlainController(
                 bitcoind_path="tests/bitcoin/src/bitcoind", rpcport=rpcport
             )  # always prefer the self-compiled bitcoind if existing
+        elif os.path.isfile("tests/bitcoin/bin/bitcoind"):
+            bitcoind_controller = BitcoindPlainController(
+                bitcoind_path="tests/bitcoin/bin/bitcoind", rpcport=rpcport
+            )  # next take the self-installed binary if existing
         else:
             bitcoind_controller = BitcoindPlainController(
                 rpcport=rpcport
@@ -83,11 +87,8 @@ def bitcoin_regtest(docker, request):
 @pytest.fixture
 def empty_data_folder():
     # Make sure that this folder never ever gets a reasonable non-testing use-case
-    data_folder = "./test_specter_data_2789334"
-    shutil.rmtree(data_folder, ignore_errors=True)
-    os.mkdir(data_folder)
-    yield data_folder
-    shutil.rmtree(data_folder, ignore_errors=True)
+    with tempfile.TemporaryDirectory("_specter_home_tmp") as data_folder:
+        yield data_folder
 
 
 @pytest.fixture

tests/install_bitcoind.sh

@@ -1,68 +1,141 @@
-#!/bin/sh
+#!/bin/bash
 
-echo "    --> install_bitcoind.sh Start $(date)"
-START=$(date +%s.%N)
-# Clone bitcoind if it doesn't exist, or update it if it does
-# (copied from HWI)
-cd tests
-bitcoind_setup_needed=false
-if [ ! -d "./bitcoin/.git" ]; then
-    echo "    --> cloning bitcoin"
-    git clone https://github.com/bitcoin/bitcoin.git
-    bitcoind_setup_needed=true
-fi
 
-cd bitcoin
+function sub_compile {
 
-# Determine if we need to pull. From https://stackoverflow.com/a/3278427
-UPSTREAM=origin/master
-LOCAL=$(git describe --tags)
-if cat ../../pytest.ini | grep "addopts = --bitcoind-version" ; then
-    PINNED=$(cat ../../pytest.ini | grep "addopts = --bitcoind-version" | cut -d' ' -f4)
-fi
-if [ -z $PINNED ]; then
-    REMOTE=$(git rev-parse "$UPSTREAM")
-    BASE=$(git merge-base @ "$UPSTREAM")
-    if [ $LOCAL = $REMOTE ]; then
-        echo "Up-to-date"
-    elif [ $LOCAL = $BASE ]; then
-        git pull
-        git reset --hard origin/master
+
+    echo "    --> install_bitcoind.sh Start $(date) (compiling)"
+    START=$(date +%s.%N)
+    # Clone bitcoind if it doesn't exist, or update it if it does
+    # (copied from HWI)
+    cd tests
+    bitcoind_setup_needed=false
+    if [ ! -d "./bitcoin/.git" ]; then
+        echo "    --> cloning bitcoin"
+        git clone https://github.com/bitcoin/bitcoin.git
         bitcoind_setup_needed=true
     fi
-else
-    if [ $LOCAL = $PINNED ]; then
-        echo "    --> Pinned: $PINNED! Checkout not needed!"
+
+    cd bitcoin
+
+    # Determine if we need to pull. From https://stackoverflow.com/a/3278427
+    UPSTREAM=origin/master
+    LOCAL=$(git describe --tags)
+    if cat ../../pytest.ini | grep "addopts = --bitcoind-version" ; then
+        PINNED=$(cat ../../pytest.ini | grep "addopts = --bitcoind-version" | cut -d' ' -f4)
+    fi
+    if [ -z $PINNED ]; then
+        REMOTE=$(git rev-parse "$UPSTREAM")
+        BASE=$(git merge-base @ "$UPSTREAM")
+        if [ $LOCAL = $REMOTE ]; then
+            echo "Up-to-date"
+        elif [ $LOCAL = $BASE ]; then
+            git pull
+            git reset --hard origin/master
+            bitcoind_setup_needed=true
+        fi
     else
-        echo "    --> Pinned: $PINNED! Checkout needed!"
-        git fetch
-        git checkout $PINNED || exit 1
-        bitcoind_setup_needed=true
+        if [ $LOCAL = $PINNED ]; then
+            echo "    --> Pinned: $PINNED! Checkout not needed!"
+        else
+            echo "    --> Pinned: $PINNED! Checkout needed!"
+            git fetch
+            git checkout $PINNED || exit 1
+            bitcoind_setup_needed=true
+        fi
     fi
-fi
 
 
 
-# Build bitcoind. 
-if [ "$bitcoind_setup_needed" = "true" ] ; then
-    # Build dependencies. This is super slow, but it is cached so it runs fairly quickly.
-    cd contrib
-    # This is hopefully fullfilles (via .travis.yml most relevantly)
-    # sudo apt install make automake cmake curl g++-multilib libtool binutils-gold bsdmainutils pkg-config python3 patch
-    echo "    --> Building db4"
-    ./install_db4.sh $(pwd)
-    echo "    --> Finishing db4"
-    ls -l
-    cd ..
-    echo "    --> Setup needed. Starting autogen"
-    ./autogen.sh
-    echo "    --> Starting configure"
-    export BDB_PREFIX="$(pwd)/contrib/db4"
-    ./configure BDB_LIBS="-L${BDB_PREFIX}/lib -ldb_cxx-4.8" BDB_CFLAGS="-I${BDB_PREFIX}/include" --with-miniupnpc=no --without-gui --disable-zmq --disable-tests --disable-bench --with-libs=no --with-utils=no
-fi
-make -j$(nproc) src/bitcoind
-cd ../.. #travis is sourcing this script
-echo "    --> Finished build bitcoind"
-END=$(date +%s.%N)
-DIFF=$(echo "$END - $START" | bc)
-echo "    --> install_bitcoind.sh End $(date) took $DIFF"
+    # Build bitcoind. 
+    if [ "$bitcoind_setup_needed" = "true" ] ; then
+        # Build dependencies. This is super slow, but it is cached so it runs fairly quickly.
+        cd contrib
+        # This is hopefully fullfilles (via .travis.yml most relevantly)
+        # sudo apt install make automake cmake curl g++-multilib libtool binutils-gold bsdmainutils pkg-config python3 patch
+        echo "    --> Building db4"
+        ./install_db4.sh $(pwd)
+        echo "    --> Finishing db4"
+        ls -l
+        cd ..
+        echo "    --> Setup needed. Starting autogen"
+        ./autogen.sh
+        echo "    --> Starting configure"
+        export BDB_PREFIX="$(pwd)/contrib/db4"
+        # This is for reducing mem-footprint as for some reason cirrus fails even though it has 4GB Mem
+        # CXXFLAGS="--param ggc-min-expand=1 --param ggc-min-heapsize=32768 -O2"
+        ./configure BDB_LIBS="-L${BDB_PREFIX}/lib -ldb_cxx-4.8" BDB_CFLAGS="-I${BDB_PREFIX}/include" CXXFLAGS="--param ggc-min-expand=1 --param ggc-min-heapsize=32768 -O2" --with-miniupnpc=no --without-gui --disable-zmq --disable-tests --disable-bench --with-libs=no --with-utils=no
+    fi
+    make -j$(nproc) src/bitcoind
+    cd ../.. #travis is sourcing this script
+    echo "    --> Finished build bitcoind"
+    END=$(date +%s.%N)
+    DIFF=$(echo "$END - $START" | bc)
+    echo "    --> install_bitcoind.sh End $(date) took $DIFF"
+}
+
+function sub_binary {
+    echo "    --> install_bitcoind.sh Start $(date) (binary)"
+    START=$(date +%s.%N)
+    cd tests
+    # todo: Parametrize this
+    version=0.20.1
+    wget https://bitcoincore.org/bin/bitcoin-core-${version}/bitcoin-${version}-x86_64-linux-gnu.tar.gz
+    tar -xzf bitcoin-${version}-x86_64-linux-gnu.tar.gz
+    if [[ -f ./bitcoin ]]; then
+        echo "bitcoin -directory exists"
+        return
+    fi
+    mv ./bitcoin-${version} bitcoin
+    cd .. #cirrus is sourcing this script
+    echo "    --> Finished installing bitcoind binary"
+    END=$(date +%s.%N)
+    DIFF=$(echo "$END - $START" | bc)
+    echo "    --> install_bitcoind.sh End $(date) took $DIFF"
+}
+
+function parse_and_execute() {
+  if [[ $# = 0 ]]; then
+    sub_default
+    exit 0
+  fi 
+
+  while [[ $# -gt 0 ]]
+  do
+  arg="$1"
+  case $arg in
+      "" | "-h" | "--help")
+        sub_default
+        shift
+        ;;
+      --debug)
+        set -x
+        DEBUG=true
+        shift
+        ;;
+      compile)
+        sub_compile
+        shift
+        ;;
+      binary)
+        sub_binary
+        shift
+        ;;
+      *)
+          shift
+          sub_${arg} $@
+          ret_value=$?
+          if [ $ret_value = 127 ]; then
+              echo "Error: '$arg' is not a known subcommand." >&2
+              echo "       Run '$progname --help' for a list of known subcommands." >&2
+              exit 1
+          elif [ $ret_value = 0 ]; then
+              exit 0
+          else
+              exit $ret_value
+          fi
+          ;;
+  esac
+  done
+}
+parse_and_execute $@

tests/test_bitcoind.py

@@ -25,6 +25,10 @@ def test_bitcoinddocker_running(caplog, docker, request):
             my_bitcoind = BitcoindPlainController(
                 bitcoind_path="tests/bitcoin/src/bitcoind"
             )
+        elif os.path.isfile("tests/bitcoin/bin/bitcoind"):
+            my_bitcoind = BitcoindPlainController(
+                bitcoind_path="tests/bitcoin/bin/bitcoind"
+            )  # next take the self-installed binary if existing
         else:
             try:
                 which("bitcoind")