GlcoinCore/specter-desktop
1
0
Fork 0
mirror of https://github.com/cryptoadvance/specter-desktop.git synced 2026-08-13 12:33:29 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions 8

ci: retire Cirrus + GitLab, consolidate on GitHub Actions (#2610)

Browse source 
Co-authored-by: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
k9ert 2026-04-19 18:32:29 +02:00 • committed by GitHub
parent 62ea02657b
commit fdd1cd8f3c
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
37 changed files with 207 additions and 3415 deletions
Download patch file Download diff file Expand all files Collapse all files

125
.cirrus.yml
View file

@ -1,125 +0,0 @@
container:
# image: python:slim
# image: ubuntu:jammy
# image: python:3.10
image: ghcr.io/cryptoadvance/specter-desktop/cirrus-jammy:20260412
# We assume here that we're having a proper python3 system including virtualenv and pip
prep_stuff_template: &PREP_STUFF_TEMPLATE
bitcoind_installation_cache:
folder: ./tests/bitcoin
fingerprint_script:
- cat pyproject.toml | grep "addopts = " | cut -d'=' -f2 | sed 's/--/+/g' | tr '+' '\n' | grep bitcoin | cut -d' ' -f2
- cat tests/bitcoin_gitrev_pinned 2> /dev/null || true
- cat /etc/os-release | grep VERSION
- cat ./tests/install_noded.sh
- echo "binary" # if the next line is --bitcoin binary, otherwise use echo "compile" - this ensures different caching keys.
populate_script: ./tests/install_noded.sh --debug --bitcoin binary
elementsd_installation_cache:
folder: ./tests/elements
fingerprint_script:
- cat pyproject.toml | grep "addopts = " | cut -d'=' -f2 | sed 's/--/+/g' | tr '+' '\n' | grep elements | cut -d' ' -f2
- cat tests/elements_gitrev_pinned 2> /dev/null || true
- cat /etc/os-release | grep VERSION
- cat ./tests/install_noded.sh
- echo "binary" # if the next line is --elements binary, otherwise use echo "compile" - this ensures different caching keys.
populate_script: ./tests/install_noded.sh --debug --elements binary
verify_script:
- echo " --> Version of python, virtualenv and pip3"
- python3 --version && virtualenv --version && pip3 --version
- echo " --> Executables in tests/elements/src"
- find tests/elements/src -maxdepth 1 -type f -executable -exec ls -ld {} \; || true
- echo " --> Executables in tests/elements/bin"
- find tests/elements/bin -maxdepth 1 -type f -executable -exec ls -ld {} \; || true
- echo " --> Executables in tests/bitcoin/src"
- find tests/bitcoin/src -maxdepth 1 -type f -executable -exec ls -ld {} \; || true
- echo " --> Executables in tests/bitcoin/bin"
- find tests/bitcoin/bin -maxdepth 1 -type f -executable -exec ls -ld {} \; || true
- echo " --> bitcoind version"
- tests/bitcoin/src/bitcoind -version | head -1 || true
- tests/bitcoin/bin/bitcoind -version | head -1 || true
- echo " --> elements version"
- tests/elements/src/elementsd -version | head -1 || true
- tests/elements/bin/elementsd -version | head -1 || true
pip_script:
#folder: /tmp/cirrus-ci-build/.env
#fingerprint_script: echo muh && cat requirements.txt
#populate_script:
- virtualenv --python=python .env
- source ./.env/bin/activate
- pip3 install -r requirements.txt --require-hashes && pip3 install -e ".[test]"
install_script:
- source ./.env/bin/activate
- pip3 install -e .
test_task:
<< : *PREP_STUFF_TEMPLATE
skip: "false"
test_script:
- source ./.env/bin/activate
- echo $PATH
# needed so that setuptools_scm has a t least one tag to guess the version properly
# and the tests/test_util_version.py doesn't fail
- git fetch origin refs/tags/v1.0.0
- pytest --cov=cryptoadvance --junitxml=./testresults.xml
always:
junit_artifacts:
path: "./testresults.xml"
format: junit
cypress_test_task:
use_compute_credits: $CIRRUS_USER_COLLABORATOR == 'true'
skip: "false"
persistent_worker:
labels:
os: linux
isolation:
container:
image: ghcr.io/cryptoadvance/specter-desktop/cypress-python-jammy:20260411
cpu: 6
memory: 6G
pre_prep_script:
# The stupid old debian-package is not installing a proper binary but just the python-package
- echo -e '#!/bin/bash\npython3 -m virtualenv "$@"' > /usr/local/bin/virtualenv
- chmod +x /usr/local/bin/virtualenv
- virtualenv --version
<< : *PREP_STUFF_TEMPLATE
npm_cache:
folder: ./node_modules
fingerprint_script: cat package-lock.json
populate_script: npm ci
cypress_script:
- source ./.env/bin/activate
#- pip3 install -e .
- ./utils/test-cypress.sh --debug run
junit_artifacts:
path: "cypresstest-output.xml"
type: text/xml
format: junit
always:
cypress_screenshots_artifacts:
path: "./cypress/screenshots/**"
cypress_videos_artifacts:
path: "./cypress/videos/**"
extension_smoketest_task:
<< : *PREP_STUFF_TEMPLATE
test_script:
- git config --global user.name "CI CD"
- git config --global user.email "cicd@example.com"
- source ./.env/bin/activate
- echo $PATH
- mkdir tmp && cd tmp
- mkdir testextension && cd testextension
- pwd
- python3 -m cryptoadvance.specter ext gen --ext-id cicdtest --org cryptoadvance --no-isolated-client --devicename cicddevice
- pip3 install -e .
- python3 -m cryptoadvance.specter server --config DevelopmentConfig --debug 2> specter.log &
- sleep 15
- cat specter.log | grep "Found CicdtestService" # Discovery failed
- cat specter.log | grep "Extension CicdtestService activated (alpha)"
- apt-get update && apt-get -y install curl
- curl http://127.0.0.1:25441/svc/cicdtest/ | grep "CicdtestService 4thewin."

281
.gitlab-ci.yml
View file

@ -1,281 +0,0 @@
# this image contains python, bitcoind and docker
# check docker/python-bitcoind on how it's built
image: registry.gitlab.com/cryptoadvance/specter-desktop/python-bitcoind:v22.0
variables:
# Cache documentation: https://docs.gitlab.com/ee/ci/caching/
PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip"
cache:
# enable per-job and per-branch caching
- key:
files:
- ./requirements.txt
prefix: "$CI_JOB_NAME"
paths:
- .cache/pip
- .env
stages:
- testing
- releasing
- post_releasing
before_script:
- docker info || echo "no docker-command found" # Print out docker version for debugging
- echo CI_PROJECT_NAMESPACE = $CI_PROJECT_NAMESPACE
- echo CI_PROJECT_ROOT_NAMESPACE = $CI_PROJECT_ROOT_NAMESPACE
- python -V # Print out python version for debugging
- apt update
- apt install -y libusb-1.0-0-dev libudev-dev # usb-support in hidapi
# https://github.com/python-babel/babel/issues/990#issuecomment-1760326334
- rm -f /etc/localtime
- ln -s /usr/share/zoneinfo/Etc/UTC /etc/localtime
# This doesn't get cached in gitlab but we don't need it anyway for now:
# - ./tests/install_noded.sh --debug --elements compile
- pip3 install --upgrade virtualenv
- virtualenv --python=python3 .env
- source .env/bin/activate
check:
stage: testing
# We simply check here whether all the tests on github are completed and green
script:
- ./utils/release.sh wait_on_master || exit 1 # that command will have a non-0 exit-value if not everything is green
# jobs with a preceding . like .test are hidden jobs and are not executed. I leave them in here as
# we might want to reactivate them in the case that github explodes or something.
# So effectively, gitlab is currently only used for releasing.
.test:
stage: testing
# We assume here that people who want to get code into the master-branch are
# relying on PRs and people who are working on gitlab-forks are working
# on CI which probably want fast feedback on the releasing-jobs
# and therefore skip the test-job
# tem deactivated as it did not work as expected
#only:
# - $CI_PROJECT_ROOT_NAMESPACE =~ "cryptoadvance"
script:
- pip3 install -r requirements.txt
- pip3 install -e .
- pip3 install -e ".[test]"
- python3 setup.py install # compiles babel stuff as well (might make pip install obsolete)
- py.test --cov-report term --cov cryptoadvance
.test-cypress:
image: registry.gitlab.com/cryptoadvance/specter-desktop/cypress-python-jammy:v9.7.0
stage: testing
script:
# start the server in the background
- pip3 install -e .
- pip3 install -e ".[test]"
- python3 setup.py install # compiles babel stuff as well (might make pip install obsolete)
- npm i
- ./utils/test-cypress.sh --docker --debug run
- docker ps || echo "probably no docker available anyway"
artifacts:
when: always
paths:
- cypress/videos/**/*.mp4
- cypress/screenshots/**/*.png
expire_in: 1 day
release_pip:
stage: releasing
only:
- tags
script:
- pip3 install -e .
- pip3 install -e ".[test]"
- pip3 install .
- pip3 install build==0.10.0 twine
- python3 -m build
- ls -l dist
# twine reads the password from the env-var TWINE_PASSWORD
# Either testing it or doing the real thing depending on which gitlab-project we're running:
- if ! [[ ${CI_PROJECT_ROOT_NAMESPACE} = "cryptoadvance" ]]; then python3 -m twine upload --verbose --user __token__ dist/* --repository-url https://test.pypi.org/legacy/ ; fi
- if [[ ${CI_PROJECT_ROOT_NAMESPACE} = "cryptoadvance" ]]; then python3 -m twine upload --verbose --user __token__ dist/* ; fi
- cd dist
- sha256sum cryptoadvance.specter-*.tar.gz > SHA256SUMS-pip
- ../utils/artifact_signer.sh sign --artifact ./SHA256SUMS-pip
- cd ..
- cat ./dist/SHA256SUMS-pip
#- python ./utils/github.py upload ./dist/SHA256SUMS-pip
#- python ./utils/github.py upload ./dist/SHA256SUMS-pip.asc
- python ./utils/github.py upload ./dist/cryptoadvance.specter-*.tar.gz
artifacts:
when: always
paths:
- dist/*
expire_in: 1 day
release_binary_windows:
stage: releasing
only:
- tags
variables:
GIT_DEPTH: 0 # Disable shallow clone to get all Git history
tags:
- windows
before_script:
- whoami
- python -V
- pip3 --version
- pip install virtualenv
- virtualenv --python=python3 .env
- .\.env\Scripts\activate
- pip3 install -e ".[test]"
script:
# This script won't execute if the script before that fails
# No need to check the version-scheme again
- echo "Releasing for ${CI_PROJECT_ROOT_NAMESPACE}"
- .\pyinstaller\build-win-ci.bat $CI_COMMIT_TAG
- python ./utils/github.py upload ./pyinstaller/release/specterd-$CI_COMMIT_TAG-win64.zip
- cd ./pyinstaller/release
- python ..\..\utils\release_helper.py sha256sums specterd-$CI_COMMIT_TAG-win64.zip > SHA256SUMS-windows
- type SHA256SUMS-windows
- echo $GPG_PASSPHRASE | c:\Program` Files` `(x86`)\GnuPg\bin\gpg --detach-sign --armor --no-tty --batch --yes --passphrase-fd 0 --pinentry-mode loopback SHA256SUMS-windows
artifacts:
when: always
paths:
- pyinstaller/release/*
expire_in: 1 day
cache:
key:
files:
- ./pyinstaller/electron/package-lock.json
prefix: $CI_JOB_NAME
paths:
- ./pyinstaller/electron/node_modules
release_electron_linux_windows:
image: registry.gitlab.com/cryptoadvance/specter-desktop/electron-builder:latest
stage: releasing
only:
- tags
needs:
- release_binary_windows
before_script:
- python3 -V # Print out python version for debugging
- apt update
- apt install -y unzip libusb-1.0-0-dev libudev-dev # usb-support in hidapi
- pip3 install virtualenv
# Only difference to default befor_script: (ToDo fix this)
- python3 -m virtualenv --python=python3 .env
- source .env/bin/activate
# https://github.com/python-babel/babel/issues/990#issuecomment-1760326334
- rm -f /etc/localtime
- ln -s /usr/share/zoneinfo/Etc/UTC /etc/localtime
- pip3 install -e ".[test]" # TZ=UTC because https://github.com/nektos/act/issues/1853
script:
- echo "Releasing for ${CI_PROJECT_ROOT_NAMESPACE}"
- export CI_PROJECT_ROOT_NAMESPACE # needed in the build-script to download the right windows-binary
- ./utils/build-unix.sh --version $CI_COMMIT_TAG make-hash specterd electron-linux electron-win
- ls -l release
- cd release
- sha256sum specterd-${CI_COMMIT_TAG}-x86_64-linux-gnu.zip specter_desktop-${CI_COMMIT_TAG}-x86_64-linux-gnu.tar.gz > ./SHA256SUMS-linux
- cat ./SHA256SUMS-linux
- sha256sum Specter-Setup-${CI_COMMIT_TAG}.exe > ./SHA256SUMS-win
- cat ./SHA256SUMS-win
- cd ..
- ./utils/artifact_signer.sh sign --artifact ./release/SHA256SUMS-win
- ./utils/artifact_signer.sh sign --artifact ./release/SHA256SUMS-linux
- python3 ./utils/github.py upload ./release/Specter-Setup-${CI_COMMIT_TAG}.exe
- python3 ./utils/github.py upload ./release/specterd-${CI_COMMIT_TAG}-x86_64-linux-gnu.zip
- python3 ./utils/github.py upload ./release/specter_desktop-${CI_COMMIT_TAG}-x86_64-linux-gnu.tar.gz
#- python3 ../utils/github.py upload ./release/SHA256SUMS-linux
#- python3 ../utils/github.py upload ./release/SHA256SUMS-linux.asc
#- python3 ../utils/github.py upload ./release/SHA256SUMS-win
#- python3 ../utils/github.py upload ./release/SHA256SUMS-win.asc
cache:
key:
files:
- ./pyinstaller/electron/package-lock.json
prefix: $CI_JOB_NAME
paths:
- ./pyinstaller/electron/node_modules
artifacts:
when: always
paths:
- release/Specter-Setup-${CI_COMMIT_TAG}.exe
- release/specterd-${CI_COMMIT_TAG}-x86_64-linux-gnu.zip
- release/specter_desktop-${CI_COMMIT_TAG}-x86_64-linux-gnu.tar.gz
- release/SHA256SUMS-linux
- release/SHA256SUMS-linux.asc
- release/SHA256SUMS-win
- release/SHA256SUMS-win.asc
expire_in: 1 day
release_signatures:
stage: post_releasing
only:
- tags
before_script:
- python -V # Print out python version for debugging
- pip3 install --upgrade virtualenv
- virtualenv --python=python3 .env
- source .env/bin/activate
- pip3 install -e ".[test]"
- ./utils/artifact_signer.sh init # prepare .gnupg
script:
- python3 -m utils.release_helper download # downloads the job-artifacts from gitlab
- python3 -m utils.release_helper downloadgithub # downloads additional artifacts from github (if not there and is they have SHA256SUMS-something)
- python3 -m utils.release_helper checksigs # checks the signatures of all SHA256SUMM*.asc files
- python3 -m utils.release_helper checkhashes # checks all SHA256SUM* files (might modify files on the fly due to windows line endings)
- python3 -m utils.release_helper create # creates a SHA256SUM
- ./utils/artifact_signer.sh sign --artifact ./signing_dir/SHA256SUMS # Signs the SHA256SUM
- python3 -m utils.release_helper upload_shasums # uploads SHA256SUMS to github
- python3 -m utils.release_helper upload_shasumssig # uploads SHA256SUMS.asc to github
release_docker:
stage: post_releasing
only:
- tags
before_script:
- echo "Triggering Docker Release"
script:
- ./utils/trigger_docker_build.sh
# Tagging the current master-branch of https://github.com/cryptoadvance/specterext-dummy
# with the same CI_COMMIT_TAG
tag_specterext_dummy_repo:
stage: post_releasing
only:
- tags
before_script:
# write access to git@github.com:cryptoadvance/specterext-dummy.git
- source ./utils/prepare_for_git_write.sh "$SSH_SPECTEREXT_DEPLOY_KEY"
script:
- echo "Now tagging ... git@github.com:${CI_PROJECT_ROOT_NAMESPACE}/specterext-dummy.git"
- ./utils/tag_specterext_dummy.sh
update_github:
stage: post_releasing
only:
- tags
needs:
- release_signatures
before_script:
# write access to git@github.com:swan-bitcoin/specter-static.git
- source ./utils/prepare_for_git_write.sh "$SSH_SPECTERSTATIC_DEPLOY_KEY"
script:
- echo "Now updating https://github.com:${CI_PROJECT_ROOT_NAMESPACE}/specter-desktop/releases/tag/${CI_COMMIT_TAG:-v2.0.4-pre8}"
- ./utils/generate_downloadpage.sh --org_name ${CI_PROJECT_ROOT_NAMESPACE:-k9ert} --debug --version ${CI_COMMIT_TAG:-v2.0.4-pre8} generate github # default-value for testing
update_webpage:
stage: post_releasing
only:
- tags
needs:
- release_signatures
before_script:
# write access to git@github.com:swan-bitcoin/specter-static.git
- source ./utils/prepare_for_git_write.sh "$SSH_SPECTERSTATIC_DEPLOY_KEY"
script:
- echo "Now updating https://github.com:${CI_PROJECT_ROOT_NAMESPACE}/specter-static.git"
- ./utils/generate_downloadpage.sh --org_name ${CI_PROJECT_ROOT_NAMESPACE:-k9ert} --debug --version ${CI_COMMIT_TAG:-v2.0.4-pre8} generate webpage # default-value for testing

81
AGENTS.md
View file

@ -67,44 +67,32 @@ python3 -m cryptoadvance.specter server --config DevelopmentConfig --debug
## CI/CD ## CI/CD
The project uses **GitHub Actions** for linting, testing, Docker images, and **the full release pipeline**, plus **Cirrus CI** for the heavyweight test suite (pytest + Cypress). GitLab CI is retained in `.gitlab-ci.yml` but effectively dead — the release flow was migrated to GitHub Actions. **GitHub Actions only.** Cirrus CI and GitLab CI were retired in 2026-Q2 — see `docs/ci-migration-evidence.md` for the cutover evidence and `docs/continuous-integration.md` for the active topology.
### Overview ### Workflows
| Provider | Purpose | Config File | Trigger | | Workflow | File | Trigger | Purpose |
|----------|---------|-------------|---------| |----------|------|---------|---------|
| **GitHub Actions** | Lint, smoke-build, Docker images, **releases** (pip + specterd + Electron for Linux/Win/macOS) | `.github/workflows/` | Push, PR, tags | | Tests | `test.yml` | PR, push | pytest + Cypress + extension smoketest (3 jobs) |
| **Cirrus CI** | Full test suite (pytest + Cypress + extension smoketest) | `.cirrus.yml` | PR | | Release | `release.yml` | Tag push `v*` | pip + specterd + Electron for Linux/Win/macOS + GPG-sign `SHA256SUMS` |
| **GitLab CI** | **Dead** — config retained; `check` job only waits on GH master status. Release jobs no longer run. | `.gitlab-ci.yml` | (vestigial) | | Black Linter | `zblack.yml` | PR, push | `psf/black@26.3.0` action pinned to Black 22.3.0, python-3.12 |
| TOC Generator | `toc.yml` | Push | Auto-generates TOCs for `README.md`, `docs/faq.md`, `docs/development.md` |
| Docker Push | `docker-push.yml` | Push to any branch | Multi-arch image → `ghcr.io/cryptoadvance/specter-desktop:<branch>` |
| Docker Tag | `docker-tag.yml` | Tag push `v*` | Multi-arch image → `ghcr.io/cryptoadvance/specter-desktop:<tag>` |
| Extension Compat | `extension-compat.yml` | PR touching `requirements.*`/`pyproject.toml` | Installs full lock, imports every bundled extension, runs `pip check` |
| specterd Build Smoke | `test-specterd-build.yml` | PR touching `pyinstaller/`, `requirements*`, `src/**` | Builds specterd on Linux and runs `--help` |
| Electron Smoke | `electron-smoketest.yml` | PR touching `pyinstaller/electron/**` | Smoke test Electron packaging |
### GitHub Actions (7 workflows) All workflows use public GitHub-hosted runners (`ubuntu-latest` / `ubuntu-22.04` / `windows-latest` / `macos-14`). **No private runners.**
1. **Black Python Linter** (`zblack.yml`) — Runs on every push and PR. Uses `psf/black@26.3.0` action pinned to Black version `22.3.0`, on python-3.12 (pinned to avoid 3.14 incompatibility with Black 22.3.0). Checks `./src`. ### Test workflow — `test.yml`
2. **TOC Generator** (`toc.yml`) — Auto-generates TOCs for `README.md`, `docs/faq.md`, `docs/development.md` on push.
3. **Docker Push** (`docker-push.yml`) — Builds multi-arch (amd64 + arm64) image on every push. Pushes to `ghcr.io/<owner>/<repo>:<branch>` (upstream: `ghcr.io/cryptoadvance/specter-desktop:<branch>`).
4. **Docker Tag** (`docker-tag.yml`) — Builds multi-arch image on version tags. Pushes to `ghcr.io/<owner>/<repo>:<tag>`.
5. **Extension Compatibility Check** (`extension-compat.yml`) — On changes to `requirements.*` or `pyproject.toml`: installs the full lock file, imports every bundled extension (Swan, LiquidIssuer, DevHelp, Notifications, ExFund, Faucet, Electrum, Spectrum, StackTrack, TimelockRecovery), runs `pip check`, and best-effort runs extension test suites. Catches dep conflicts before they break downstream extensions.
6. **Test specterd build** (`test-specterd-build.yml`) — PR smoke test on changes to `pyinstaller/`, `requirements*`, `src/**`, or packaging files. Builds specterd on Linux and runs `--help` smoke test.
7. **Release** (`release.yml`) — **The release pipeline.** See next section.
All GitHub Actions use public runners (`ubuntu-latest` / `ubuntu-24.04` / `windows-latest` / `macos-14`). **No private runners.** Three jobs on `ubuntu-22.04`:
1. **`test`** — pytest with `--cov=cryptoadvance`, 45-min timeout. Installs system deps inline; no custom image. Caches bitcoind/elementsd via `actions/cache@v4` keyed on `runner.os × runner.arch × hash(pyproject.toml, install_noded.sh, bitcoin_SHA256SUMS, elements_SHA256SUMS)` with `save-always: true`.
2. **`cypress`** — `./utils/test-cypress.sh --debug run` inside `ghcr.io/cryptoadvance/specter-desktop/cypress-python-jammy@sha256:<digest>`, 30-min timeout, `--shm-size=2g`.
3. **`extension-smoketest`** — byte-compatible port of the old Cirrus smoketest, 15-min timeout.
### Cirrus CI (Testing) `tests/install_noded.sh` GPG-verifies upstream `SHA256SUMS.asc` and checks tarball SHA256 against the committed trust anchors on every run (cold cache AND cache hit).
Cirrus CI runs the **full test suite** on PRs. Config: `.cirrus.yml`.
**Three tasks:**
1. **`test_task`** — Full pytest suite with bitcoind + elementsd in regtest mode. Uses cached binary downloads. Produces JUnit XML results.
2. **`cypress_test_task`** — Frontend tests with Cypress. Requires 6 CPU, 6GB RAM. Produces screenshots and video artifacts.
3. **`extension_smoketest_task`** — Generates a test extension, starts the server, verifies the extension loads and responds.
**Docker images used:**
- `registry.gitlab.com/cryptoadvance/specter-desktop/cirrus-jammy:20230206` (pytest)
- `registry.gitlab.com/cryptoadvance/specter-desktop/cypress-python-jammy:20230206` (Cypress)
Both images are pre-built and hosted on the GitLab container registry. They include Python, virtualenv, and other dependencies. The `docker/` directory in the repo contains Dockerfiles for building them.
**Caching:** bitcoind and elementsd binaries are cached by Cirrus based on the version pinned in `pyproject.toml`. The `tests/install_noded.sh` script handles downloading or compiling them.
### Release pipeline — `.github/workflows/release.yml` ### Release pipeline — `.github/workflows/release.yml`
@ -140,12 +128,12 @@ Triggers on tags matching `v[0-9]+.[0-9]+.[0-9]+` (and `-*` suffixes for pre-rel
| Secret | Purpose | Required? | | Secret | Purpose | Required? |
|----------|---------|---| |----------|---------|---|
| `APPLE_CERTIFICATE_BASE64` | Base64-encoded `.p12` Apple signing certificate | Optional — unsigned build if missing | | `GPG_PRIVATE_KEY` + `GPG_PASSPHRASE` | Sign `SHA256SUMS` | Required for signed releases |
| `APPLE_CERTIFICATE_PASSWORD` | Password for the `.p12` | With `APPLE_CERTIFICATE_BASE64` | | `APPLE_CERTIFICATE_BASE64` + `APPLE_CERTIFICATE_PASSWORD` | macOS signing cert | Optional — unsigned build if missing |
| `APPLE_PROVISIONING_PROFILE_BASE64` | Base64-encoded provisioning profile | Optional | | `APPLE_ID` + `APPLE_APP_SPECIFIC_PASSWORD` + `APPLE_TEAM_ID` | macOS notarization | Required with signing |
| PyPI trusted publisher | Configured on PyPI side, not a GH secret | Required for `release-pip` in upstream | | `APPLE_PROVISIONING_PROFILE_BASE64` | Provisioning profile | Optional |
| `AARON_TRIGGER` | Trigger `lncm/docker-specter-desktop` build | Optional — skips Docker trigger if missing |
Historical GitLab secrets (`GH_BIN_UPLOAD_PW`, `TWINE_PASSWORD`, `GPG_PASSPHRASE`, `SSH_SPECTEREXT_DEPLOY_KEY`, `SSH_SPECTERSTATIC_DEPLOY_KEY`) are **no longer used**. `.gitlab-ci.yml` still references them but the pipeline is dead. | PyPI trusted publisher | Configured on PyPI side, not a GH secret | Required for `release-pip` upstream |
### Testing a release on a fork ### Testing a release on a fork
@ -154,12 +142,6 @@ Historical GitLab secrets (`GH_BIN_UPLOAD_PW`, `TWINE_PASSWORD`, `GPG_PASSPHRASE
3. The `release-pip` PyPI publish step is gated on `github.repository == 'cryptoadvance/specter-desktop'`, so forks build the pip package but don't publish. 3. The `release-pip` PyPI publish step is gated on `github.repository == 'cryptoadvance/specter-desktop'`, so forks build the pip package but don't publish.
4. Unsigned macOS builds work out of the box; signing requires you to add your own Apple secrets. 4. Unsigned macOS builds work out of the box; signing requires you to add your own Apple secrets.
### Dead config
- `.gitlab-ci.yml` — still in the repo but release jobs no longer run. Safe to remove in a cleanup pass.
- `pyinstaller/build-win-ci.bat` — former GitLab Windows runner entry point; no longer invoked.
- `utils/release.sh` / `utils/release_helper.py` / `utils/github.py` — may contain dead code paths now that GitLab isn't uploading artifacts. Audit before changes.
## Testing ## Testing
Tests require a `bitcoind` binary (regtest mode). No tests run without it. Tests require a `bitcoind` binary (regtest mode). No tests run without it.
@ -217,10 +199,9 @@ Desktop builds use PyInstaller + Electron:
1. **specterd** (daemon binary): `pyinstaller specterd.spec` from `pyinstaller/` dir 1. **specterd** (daemon binary): `pyinstaller specterd.spec` from `pyinstaller/` dir
2. **Electron app**: wraps specterd, downloads it on first launch with SHA256 + GPG verification 2. **Electron app**: wraps specterd, downloads it on first launch with SHA256 + GPG verification
3. Platform scripts: `utils/build-osx.sh`, `utils/build-unix.sh`, `pyinstaller/build-win-ci.bat` 3. pip package: `python3 -m build`
4. pip package: `python3 -m build`
See `docs/build-instructions.md` for step-by-step manual build instructions. Release builds live in `.github/workflows/release.yml` (triggered by tag push). See `docs/release-guide.md` for the release workflow and `docs/build-instructions.md` for step-by-step manual builds.
## Dependencies ## Dependencies
@ -238,7 +219,7 @@ Extensions live in `specterext` namespace packages. Each extension:
- Registers via entry points in setup.cfg - Registers via entry points in setup.cfg
- Can add UI pages, API endpoints, and background services - Can add UI pages, API endpoints, and background services
- Generate a skeleton: `python3 -m cryptoadvance.specter ext gen --ext-id myext --org myorg` - Generate a skeleton: `python3 -m cryptoadvance.specter ext gen --ext-id myext --org myorg`
- CI smoke-tests extension generation in `extension_smoketest_task` (Cirrus) - CI smoke-tests extension generation in the `extension-smoketest` job (`test.yml`)
- See `docs/extensions/` for the extension development guide - See `docs/extensions/` for the extension development guide
## Key Files for Navigation ## Key Files for Navigation
@ -255,14 +236,14 @@ Extensions live in `specterext` namespace packages. Each extension:
| All managers | `src/cryptoadvance/specter/managers/` | | All managers | `src/cryptoadvance/specter/managers/` |
| Templates | `src/cryptoadvance/specter/templates/` | | Templates | `src/cryptoadvance/specter/templates/` |
| Tests | `tests/` | | Tests | `tests/` |
| CI config | `.github/workflows/`, `.cirrus.yml`, `.gitlab-ci.yml` | | CI config | `.github/workflows/` |
| Build scripts | `pyinstaller/`, `utils/`, `electron/` | | Build scripts | `pyinstaller/`, `utils/`, `electron/` |
| CI Docker images | `docker/` | | CI Docker images | `docker/` |
## Current State (as of 2026-04) ## Current State (as of 2026-04)
- **Last release:** v2.1.1 (2025-01-03) — no release on the new GH Actions pipeline yet; first tagged run will exercise it end-to-end. - **Last release:** v2.1.1 (2025-01-03) — no release on the new GH Actions pipeline yet; first tagged run will exercise it end-to-end.
- **CI migration complete:** release pipeline moved from GitLab to `.github/workflows/release.yml`. `.gitlab-ci.yml` retained but dead. - **CI migration complete:** Cirrus CI and GitLab CI retired; all workflows now on GitHub Actions. See `docs/ci-migration-evidence.md`.
- **macOS automation:** now covered on Apple Silicon free tier; x86_64 macOS gated on paid runner. - **macOS automation:** now covered on Apple Silicon free tier; x86_64 macOS gated on paid runner.
- **Black linter:** reconfigured to pin python-3.12 + `psf/black@26.3.0` action + black version 22.3.0 (worked around 3.14 incompatibility). Verify green state in CI before assuming. - **Black linter:** reconfigured to pin python-3.12 + `psf/black@26.3.0` action + black version 22.3.0 (worked around 3.14 incompatibility). Verify green state in CI before assuming.
- **Issue/PR backlog:** refreshed counts not captured here — use `gh issue list` / `gh pr list` for current state. - **Issue/PR backlog:** refreshed counts not captured here — use `gh issue list` / `gh pr list` for current state.

7
docker/bionic-build/Dockerfile
View file

@ -1,7 +0,0 @@
FROM registry.gitlab.com/cryptoadvance/specter-desktop/python:3.8.5-bionic
RUN apt-get update && apt-get install -y --no-install-recommends libusb-1.0-0-dev libudev-dev
RUN apt-get install -y --no-install-recommends libgl1-mesa-dri gvfs gvfs-libs \
libdrm-amdgpu1 libdrm-nouveau2 libdrm-radeon1 libedit2 libelf1 libllvm10 \
libvulkan1 libzstd1 libtdb1 libcanberra-gtk3-0 virtualenv libcanberra-gtk3-module

10
docker/bionic-build/Readme.md
View file

@ -1,10 +0,0 @@
This Dockerimage is manually created and uploaded:
```
docker build -t registry.gitlab.com/cryptoadvance/specter-desktop/bionic-build:latest .
docker push registry.gitlab.com/cryptoadvance/specter-desktop/bionic-build:latest
```
The reason for this image is explained in [#356](https://github.com/cryptoadvance/specter-desktop/issues/356) and introduced in https://github.com/cryptoadvance/specter-desktop/pull/396/files .
It has been replaced with introducing the electron-build with `registry.gitlab.com/cryptoadvance/specter-desktop/electron-builder:latest` in https://github.com/cryptoadvance/specter-desktop/pull/473/files .

13
docker/cirrus-jammy/Dockerfile
View file

@ -1,13 +0,0 @@
from ubuntu:jammy
# 1. python-stuff and HWI dependencies
# 2. capability to build bitcoind
# 3. cypress dependencies
RUN apt update && DEBIAN_FRONTEND="noninteractive" apt-get install --no-install-recommends -y \
libusb-1.0-0-dev libudev-dev python3 python3-virtualenv python3-dev python3-pip \
build-essential libtool autotools-dev automake autoconf pkg-config bsdmainutils libevent-dev libboost-dev libboost-system-dev libboost-filesystem-dev libboost-test-dev bc \
nodejs npm libgtk2.0-0 libgtk-3-0 libgbm-dev libnotify-dev libgconf-2-4 libnss3 libxss1 libasound2 libxtst6 xauth xvfb \
wget libzmq5-dev
# Stuff needed for Elements (compilation)
RUN DEBIAN_FRONTEND="noninteractive" apt-get install --no-install-recommends -y libboost-thread-dev libsqlite3-dev git

10
docker/cirrus-jammy/README.md
View file

@ -1,10 +0,0 @@
An image used to run the build on cirrus (tests only, not cypress-tests).
Create it like this:
```
docker buildx build --platform linux/amd64 -t ghcr.io/cryptoadvance/specter-desktop/cirrus-jammy:20260412 --load .
docker push ghcr.io/cryptoadvance/specter-desktop/cirrus-jammy:20260412
```
Check the `.cirrus.yml` on how this is used and update the $current_date there.

39
docker/electron-builder-master/README.md
View file

@ -1,39 +0,0 @@
# Purpose
Used for building the electron-app. In short it's the /pyinstaller/build-unix.sh script which is running in this image.
By intention, this is using an older OS-version in order to avoid glibc-issues. For details, see:
* https://github.com/cryptoadvance/specter-desktop/pull/1688#issuecomment-1242796681
* https://github.com/cryptoadvance/specter-desktop/issues/373#issuecomment-695068924
# Usage
If you want to run the image manually, do something like this (copied from [here](https://www.electron.build/multi-platform-build#build-electron-app-using-docker-on-a-local-machine)):
```
docker run --rm -ti \
--env-file <(env | grep -iE 'DEBUG|NODE_|ELECTRON_|YARN_|NPM_|CI|CIRCLE|TRAVIS_TAG|TRAVIS|TRAVIS_REPO_|TRAVIS_BUILD_|TRAVIS_BRANCH|TRAVIS_PULL_REQUEST_|APPVEYOR_|CSC_|GH_|GITHUB_|BT_|AWS_|STRIP|BUILD_') \
--env ELECTRON_CACHE="/root/.cache/electron" \
--env ELECTRON_BUILDER_CACHE="/root/.cache/electron-builder" \
-v ${PWD}:/project \
-v ${PWD##*/}-node-modules:/project/node_modules \
-v ~/.cache/electron:/root/.cache/electron \
-v ~/.cache/electron-builder:/root/.cache/electron-builder \
electronuserland/builder:wine
```
# Building
build the image like:
```
docker build -t registry.gitlab.com/cryptoadvance/specter-desktop/electron-builder:latest .
docker push registry.gitlab.com/cryptoadvance/specter-desktop/electron-builder:latest
```
# Details
This image is putting python3.10 on top of electronuserland/builder:wine. As we want to have stable build-targets the actual `FROM` clause is `electronuserland/builder:14-wine-10.22` as the project is adding the node version (14) and a timestamp (10.22) to the tagname.

188
docker/electron-builder/Dockerfile
View file

@ -1,188 +0,0 @@
FROM electronuserland/builder:14-wine-10.22
# 14-wine-10.22 is a stable tag which has been the same than the "wine" tag but more explicit
# It's based on node 14 and focal and therefore has glibc 2.31.
# It has been created on Oct 22
# Let's put python3.10 on top:
# * electronuserland/builder:14-wine-10.22 is same as wine which depends on "node"
# * which depends on "base" which builds on buildpack-deps:focal curl
# https://github.com/docker-library/buildpack-deps/blob/98a5ab81d47a106c458cdf90733df0ee8beea06c/ubuntu/focal/curl/Dockerfile
# * now the coe below to install python is copied from:
# https://github.com/docker-library/python/blob/master/3.10/bullseye/Dockerfile
# which builds ontop of buildpack-deps:bullseye-scm
# * So to make the python installation fully functional, we have to install some stuff ...
# * and then do the python installation
# Copying some stuff from https://github.com/docker-library/python/blob/master/3.10/bullseye/Dockerfile
RUN set -ex; \
apt-get update; \
apt-get install -y --no-install-recommends \
autoconf \
automake \
bzip2 \
dpkg-dev \
file \
g++ \
gcc \
imagemagick \
libbz2-dev \
libc6-dev \
libcurl4-openssl-dev \
libdb-dev \
libevent-dev \
libffi-dev \
libgdbm-dev \
libglib2.0-dev \
libgmp-dev \
libjpeg-dev \
libkrb5-dev \
liblzma-dev \
libmagickcore-dev \
libmagickwand-dev \
libmaxminddb-dev \
libncurses5-dev \
libncursesw5-dev \
libpng-dev \
libpq-dev \
libreadline-dev \
libsqlite3-dev \
libssl-dev \
libtool \
libwebp-dev \
libxml2-dev \
libxslt-dev \
libyaml-dev \
make \
patch \
unzip \
zip \
jq \
xz-utils \
zlib1g-dev
# This has been copied from:
# https://github.com/docker-library/python/blob/master/3.10/bullseye/Dockerfile
# ensure local python is preferred over distribution python
ENV PATH /usr/local/bin:$PATH
# http://bugs.python.org/issue19846
# > At the moment, setting "LANG=C" on a Linux system *fundamentally breaks Python 3*, and that's not OK.
ENV LANG C.UTF-8
# runtime dependencies
RUN set -eux; \
apt-get update; \
apt-get install -y --no-install-recommends \
libbluetooth-dev \
tk-dev \
uuid-dev \
; \
rm -rf /var/lib/apt/lists/*
ENV GPG_KEY A035C8C19219BA821ECEA86B64E628F8D684696D
ENV PYTHON_VERSION 3.10.9
RUN set -eux; \
\
savedAptMark="$(apt-mark showmanual)"; \
apt-get update; \
apt-get install -y --no-install-recommends \
patchelf \
; \
\
wget -O python.tar.xz "https://www.python.org/ftp/python/${PYTHON_VERSION%%[a-z]*}/Python-$PYTHON_VERSION.tar.xz"; \
wget -O python.tar.xz.asc "https://www.python.org/ftp/python/${PYTHON_VERSION%%[a-z]*}/Python-$PYTHON_VERSION.tar.xz.asc"; \
GNUPGHOME="$(mktemp -d)"; export GNUPGHOME; \
gpg --batch --keyserver hkps://keys.openpgp.org --recv-keys "$GPG_KEY"; \
gpg --batch --verify python.tar.xz.asc python.tar.xz; \
command -v gpgconf > /dev/null && gpgconf --kill all || :; \
rm -rf "$GNUPGHOME" python.tar.xz.asc; \
mkdir -p /usr/src/python; \
tar --extract --directory /usr/src/python --strip-components=1 --file python.tar.xz; \
rm python.tar.xz; \
\
cd /usr/src/python; \
gnuArch="$(dpkg-architecture --query DEB_BUILD_GNU_TYPE)"; \
./configure \
--build="$gnuArch" \
--enable-loadable-sqlite-extensions \
--enable-optimizations \
--enable-option-checking=fatal \
--enable-shared \
--with-lto \
--with-system-expat \
--without-ensurepip \
; \
nproc="$(nproc)"; \
make -j "$nproc" \
; \
make install; \
\
# https://github.com/docker-library/python/issues/784
# prevent accidental usage of a system installed libpython of the same version
bin="$(readlink -vf /usr/local/bin/python3)"; \
patchelf --set-rpath '$ORIGIN/../lib' "$bin"; \
\
# enable GDB to load debugging data: https://github.com/docker-library/python/pull/701
dir="$(dirname "$bin")"; \
mkdir -p "/usr/share/gdb/auto-load/$dir"; \
cp -vL Tools/gdb/libpython.py "/usr/share/gdb/auto-load/$bin-gdb.py"; \
\
cd /; \
rm -rf /usr/src/python; \
\
find /usr/local -depth \
\( \
\( -type d -a \( -name test -o -name tests -o -name idle_test \) \) \
-o \( -type f -a \( -name '*.pyc' -o -name '*.pyo' -o -name 'libpython*.a' \) \) \
\) -exec rm -rf '{}' + \
; \
\
ldconfig; \
\
apt-mark auto '.*' > /dev/null; \
apt-mark manual $savedAptMark; \
apt-get purge -y --auto-remove -o APT::AutoRemove::RecommendsImportant=false; \
rm -rf /var/lib/apt/lists/*; \
\
python3 --version
# make some useful symlinks that are expected to exist ("/usr/local/bin/python" and friends)
RUN set -eux; \
for src in idle3 pydoc3 python3 python3-config; do \
dst="$(echo "$src" | tr -d 3)"; \
[ -s "/usr/local/bin/$src" ]; \
[ ! -e "/usr/local/bin/$dst" ]; \
ln -svT "$src" "/usr/local/bin/$dst"; \
done
# if this is called "PIP_VERSION", pip explodes with "ValueError: invalid truth value '<VERSION>'"
ENV PYTHON_PIP_VERSION 22.3.1
# https://github.com/docker-library/python/issues/365
ENV PYTHON_SETUPTOOLS_VERSION 65.5.1
# https://github.com/pypa/get-pip
ENV PYTHON_GET_PIP_URL https://github.com/pypa/get-pip/raw/1a96dc5acd0303c4700e02655aefd3bc68c78958/public/get-pip.py
ENV PYTHON_GET_PIP_SHA256 d1d09b0f9e745610657a528689ba3ea44a73bd19c60f4c954271b790c71c2653
RUN set -eux; \
\
wget -O get-pip.py "$PYTHON_GET_PIP_URL"; \
echo "$PYTHON_GET_PIP_SHA256 *get-pip.py" | sha256sum -c -; \
\
export PYTHONDONTWRITEBYTECODE=1; \
\
python get-pip.py \
--disable-pip-version-check \
--no-cache-dir \
--no-compile \
"pip==$PYTHON_PIP_VERSION" \
"setuptools==$PYTHON_SETUPTOOLS_VERSION" \
; \
rm -f get-pip.py; \
\
pip --version

28
docker/electron-builder/README.md
View file

@ -1,28 +0,0 @@
Used for building the electron-app. In short it's the /pyinstaller/build-unix.sh script which is running in this image.
By intention, this is using an older OS-version in order to avoid glibc-issues. For details, see:
* https://github.com/cryptoadvance/specter-desktop/pull/1688#issuecomment-1242796681
* https://github.com/cryptoadvance/specter-desktop/issues/373#issuecomment-695068924
If you want to run the image manually, do something like this (copied from [here](https://www.electron.build/multi-platform-build#build-electron-app-using-docker-on-a-local-machine)):
```
docker run --rm -ti \
--env-file <(env | grep -iE 'DEBUG|NODE_|ELECTRON_|YARN_|NPM_|CI|CIRCLE|TRAVIS_TAG|TRAVIS|TRAVIS_REPO_|TRAVIS_BUILD_|TRAVIS_BRANCH|TRAVIS_PULL_REQUEST_|APPVEYOR_|CSC_|GH_|GITHUB_|BT_|AWS_|STRIP|BUILD_') \
--env ELECTRON_CACHE="/root/.cache/electron" \
--env ELECTRON_BUILDER_CACHE="/root/.cache/electron-builder" \
-v ${PWD}:/project \
-v ${PWD##*/}-node-modules:/project/node_modules \
-v ~/.cache/electron:/root/.cache/electron \
-v ~/.cache/electron-builder:/root/.cache/electron-builder \
electronuserland/builder:wine
```
build the image like:
```
docker build -t registry.gitlab.com/cryptoadvance/specter-desktop/electron-builder:latest .
docker push registry.gitlab.com/cryptoadvance/specter-desktop/electron-builder:latest
```

18
docker/github-changelog/Dockerfile
View file

@ -1,18 +0,0 @@
FROM python:3.8
ARG REPO=https://github.com/cryptoadvance/github-changelog
RUN apt update && apt install -y git
WORKDIR /
RUN git clone $REPO;
WORKDIR /github-changelog
RUN git checkout master
RUN python3 setup.py install
ENV PYTHONUNBUFFERED="1"
ENTRYPOINT ["changelog"]

25
docker/github-changelog/Readme.md
View file

@ -1,25 +0,0 @@
An image used to create changelogs.
Create it like this:
```
docker build . -t registry.gitlab.com/cryptoadvance/specter-desktop/github-changelog:latest
docker push registry.gitlab.com/cryptoadvance/specter-desktop/github-changelog:latest
```
Use it like this:
```
latest_version=v0.8.1
export GH_TOKEN=YourTokenHere
docker run 4a3dd375832d --github-token $GH_TOKEN --branch master cryptoadvance specter-desktop $latest_version > docs/new_release_notes.md
cp docs/release-notes.md docs/release-notes.md.orig
cat docs/new_release_notes.md docs/release-notes.md.orig > docs/release-notes.md
rm docs/release-notes.md.orig docs/new_release_notes.md
```
# This will print out links to all PRs in order to review better
```
docker run 4a3dd375832d -m --github-token $GH_TOKEN --branch master cryptoadvance specter-desktop $latest_version > docs/new_release_notes.md
```

17
docs/build-instructions.md
View file

@ -6,13 +6,9 @@ This document addresses the build-system part. For the continuous-integration-pa
## pip-packages ## pip-packages
``` ```
# in the case of a release, the version needs to be adapted: python3 -m build
# sed -i "s/version=\".*/version=\"$CI_COMMIT_TAG\",/" setup.py
python3 setup.py sdist bdist_wheel
cryptoadvance.specter-vx.y.z-get-replaced-by-release-script.tar.gz
``` ```
This process is the same for all platforms. The result unfortunately is not stable in terms of identically sh256-hashes, though. Produces an sdist + wheel under `dist/`. The release pipeline sets `SETUPTOOLS_SCM_PRETEND_VERSION` from the git tag; locally, setuptools-scm derives the version from your working tree. The result is not stable in terms of identical sha256-hashes across machines.
## Electron ## Electron
The electron build is assuming a node-installation. So make sure you have `node` and `npm` available. The electron build is assuming a node-installation. So make sure you have `node` and `npm` available.
@ -20,7 +16,7 @@ The electron build is assuming a node-installation. So make sure you have `node`
The electron-app is built in a way that it's running the `specterd` (specter-demon) internally. It's not bundled with the electron-binary but downloaded with the first start (including sha256- and gpg-verification). The electron-app is built in a way that it's running the `specterd` (specter-demon) internally. It's not bundled with the electron-binary but downloaded with the first start (including sha256- and gpg-verification).
If someone does not want the download, he can manually choose a specterd-binary from the `preferences/Advanced` menu. Nevertheless the Electron-App is tied, at buildtime, to a specific specterd-binary via a sha256-version. This probably doesn't make so much sense if you build outside of a release but we need it anyway. If someone does not want the download, he can manually choose a specterd-binary from the `preferences/Advanced` menu. Nevertheless the Electron-App is tied, at buildtime, to a specific specterd-binary via a sha256-version. This probably doesn't make so much sense if you build outside of a release but we need it anyway.
So let's cover the build of the specterd-binary first. Below is a manual description of the build-process. There are acripts which are doing this but they are partially optimized for the CI-system. Check the `pyinstaller/build-*` scripts for details. So let's cover the build of the specterd-binary first. Below is a manual description of the build-process. The canonical CI build lives in `.github/workflows/release.yml` (`build-specterd-*` and `build-electron-*` jobs).
First set the virtualenv: First set the virtualenv:
@ -31,11 +27,6 @@ source .buildenv/bin/activate
### specterd Linux and MacOS ### specterd Linux and MacOS
Below doesn't seem to work properly, at least on MacOS, better use the build script. For MacOS, that would be:
```bash
./utils/build-osx.sh --version 0.0.0-pre1 specterd
```
```bash ```bash
cd pyinstaller cd pyinstaller
# prerequisites # prerequisites
@ -98,7 +89,7 @@ node ./set-version v1.3.1-custom ../dist/specterd
npm i npm i
# We assume here that no Apple-developer-ID is used to sign the binary. # We assume here that no Apple-developer-ID is used to sign the binary.
# Check `build-osx.sh` if you want to sign # For signed+notarized builds, see .github/workflows/release.yml (build-electron-macos).
echo "`jq '.build.mac.identity=null' package.json`" > package.json echo "`jq '.build.mac.identity=null' package.json`" > package.json
# finally build # finally build

84
docs/ci-migration-evidence.md Normal file
View file

@ -0,0 +1,84 @@
# CI Migration Evidence — Cirrus → GitHub Actions
Evidence artifact per `docs/cirrus-replacement-spec.md` §Acceptance. Captures measured GHA behavior over the side-by-side period so that "did we actually hit SLO?" has a grep-able answer after GH Actions logs are GC'd.
## Snapshot
- **Measurement date:** 2026-04-19
- **PR #1 (`test.yml` added):** merged 2026-04-17 as commit `a24df2eb` (PR [#2606](https://github.com/cryptoadvance/specter-desktop/pull/2606))
- **Cirrus sunset deadline:** 2026-06-30 (~10 weeks remaining)
- **Side-by-side window so far:** ~2 days
## Gate status
Spec §Acceptance requires **all four** to hold before merging the cutover PR:
| # | Criterion | Status | Notes |
|---|----------------------------------------------------------|------------|-----------------------------------------------------------------------|
| 1 | 10 consecutive green master runs | **Not met**| 2 master runs observed, both green |
| 2 | ≥ 3 green PR runs incl. one frontend-touching | **Partial**| 6 green PR runs; none confirmed as frontend-touching yet |
| 3 | Zero new flakes over ≥ 50 total runs | **Not met**| 14 total runs; zero flakes detected; sample too small |
| 4 | Cypress p95 within Cirrus +20% | **Breach** | Cypress p95 **10m10s** vs. Cirrus +20% ceiling **7m14s** — see below |
**Merging PR #2 ahead of the nominal gate is a deliberate choice** driven by Cirrus's hard 2026-06-30 shutdown, preservation of revertability (PR #2 is a pure deletion of dead code + docs updates; revert is one click), and the empirical fact that no flakes have surfaced over the available sample. Gate criteria 1 and 3 will be satisfied by ordinary master-branch activity over the coming weeks; criterion 4 is acknowledged below as a known deviation, with a measurement protocol for re-evaluation.
## Measured wall-clock
n = 8 successful runs (2 master + 6 PR) between 2026-04-17 11:13 UTC and 2026-04-17 20:45 UTC.
| Job | n | median | p95 | min | max | Cirrus median | Cirrus +20% ceiling | Result |
|-----------------------|---|---------|---------|---------|---------|---------------|---------------------|--------------------|
| `test` | 8 | 3m53s | 4m14s | 3m43s | 4m14s | 4m47s | 5m44s | **within ceiling** |
| `cypress` | 8 | 9m42s | 10m10s | 9m28s | 10m10s | 6m02s | 7m14s | **BREACH (+47%)** |
| `extension-smoketest` | 8 | 1m57s | 2m09s | 1m44s | 2m09s | 2m10s | 2m36s | **within ceiling** |
Raw data pulled via `gh api repos/cryptoadvance/specter-desktop/actions/runs/<id>/jobs` for runs `24581485817`, `24585472647` (master), and `24580543016`, `24581562475`, `24582301901`, `24582337498`, `24585550944`, `24562219139` (PR).
Cirrus baselines cited from `docs/cirrus-replacement-spec.md` §SLOs (20-sample baseline captured 2026-04-12).
### Cypress breach — acknowledgement
GHA Cypress p95 is **10m10s**, vs. the spec's Cirrus +20% ceiling of **7m14s** (Cirrus p95 6m55s × 1.20). Root cause not yet investigated. Candidates per spec §Cypress measurement: `--shm-size` bump, spec sharding, or escalation to `ubuntu-22.04-large`.
**Decision:** accepted as a known deviation. Cypress wall-clock is still well under the 30-minute workflow timeout, and the alternative — holding the cutover until after Cirrus shutdown — would leave the project without PR gating. The breach is logged here rather than swept under the rug.
**Follow-up:** re-run the measurement protocol (5× on `ubuntu-22.04` free tier) once 10+ master runs accumulate. If p95 remains >Cirrus+20%, file an issue and walk the escalation ladder (shm → shard → paid runner).
## Flake signal
Over 14 total `test.yml` runs (8 success, 5 failure, 1 action_required):
- **Failures on `kn/cirrus-replacement-spec`** (4): iteration during PR #1 development. Each failure was followed by a targeted fix commit. Confirmed non-flaky by reading `git log` (`fix: cache symlink targets…`, `fix: bash shell for cypress container`, `fix: use VALIDSIG instead of GOODSIG`, `fix: use --status-fd`).
- **Failure on `kn/bump-bitcoind-test-v27.2`** (1): bitcoind version bump branch. Likely a real test failure from the version change, not a CI flake.
- **`action_required`** (1): fork PR (`copilot/fix-livereload-ui-delays`) pending maintainer approval to run workflows. Not a flake.
**Flake count: 0** over this window. Sample size too small (n=14) to assert the steady-state SLO of ≤1% rolling-30-day, but no red flags.
## Sample PR runs
| URL | Branch | Conclusion | Frontend-touching? |
|---------------------------------------------------------------------------------------------------------------------|----------------------------------------------------------------|------------|--------------------|
| https://github.com/cryptoadvance/specter-desktop/actions/runs/24585550944 | `dependabot/npm_and_yarn/pyinstaller/electron/multi-20d65b3440`| success | partial (electron deps) |
| https://github.com/cryptoadvance/specter-desktop/actions/runs/24582337498 | `dependabot/npm_and_yarn/pyinstaller/electron/multi-3ffb4d349a`| success | partial (electron deps) |
| https://github.com/cryptoadvance/specter-desktop/actions/runs/24581562475 | `dependabot/npm_and_yarn/multi-6f6dcfc8d5` | success | indirect |
| https://github.com/cryptoadvance/specter-desktop/actions/runs/24562219139 | `kn/cirrus-replacement-spec` | success | no |
Strict frontend-touching coverage (changes under `src/cryptoadvance/specter/static/` or `src/cryptoadvance/specter/templates/`) is **not yet confirmed** in the available sample. Criterion 2 will be rechecked at PR #2 merge time.
## Sign-off
| Field | Value |
|----------------------|------------------------------------------------------------|
| Evidence captured at | 2026-04-19 |
| Latest master commit | `62ea0265` (2026-04-17 20:33 UTC) |
| `test.yml` added at | `a24df2eb` (2026-04-17 18:50 UTC) |
| Author | @k9ert |
## Rollback contract
Per spec §Rollback, if post-cutover breakage emerges:
1. Revert the PR #2 merge commit → restores `.cirrus.yml` and `docker/cirrus-jammy/`.
2. Re-add the Cirrus required-check names to master branch protection.
3. Cirrus assumed operational through 2026-06-30.
After 2026-06-30, no Cirrus fallback exists — forward fix only. Keep the cutover ≥ 4 weeks ahead of that date (target merge date per spec: 2026-05-12).

131
docs/continuous-integration.md
View file

@ -1,111 +1,60 @@
# Introduction # Continuous Integration
Specter-Desktop is using GitLab, Cirrus and GitHub-Actions for continuous integration purposes but GitHub-actions only for Blackify so far. It might be more effort using more than one CI-approach but it makes us also more resilient.
GitLab and Cirrus have both advantages and disadvantages so ... let's use both!
GitLab:
* is completely open Source for server- and clients
* the gitlab-runner can run docker and is itself running on docker
* but does not support Pull-Requests
* needs to have bitcoind in a prepared docker-container which binds the build to that version
Cirrus-CI: Specter-Desktop runs all CI on **GitHub Actions**. Cirrus CI and GitLab CI were retired in 2026-Q2 — see `docs/ci-migration-evidence.md` for the cutover evidence.
* supports the PR-model
* quite easy to setup even though it's using docker
## Gitlab ## Workflows
Gitlab is a great CI/CD-platform and in the meantime it's quite easy to use it for GitHub-repositories. | Workflow | File | Trigger |
https://docs.gitlab.com/ee/ci/ci_cd_for_external_repos/github_integration.html |--------------------------------|-------------------------------------------------|----------------------------------------------|
The main file which specifies the jobs on GitLab is .gitlab-ci.yml | Lint (black) | `.github/workflows/zblack.yml` | PR, push |
We're using a `gitlab-docker-runner` which means that all jobs are running in a container. | Tests (pytest + Cypress + extension smoketest) | `.github/workflows/test.yml` | PR, push |
However at the same time we're using docker to spinup a bitcoind. | Release | `.github/workflows/release.yml` | Tag push (`v*`) |
| Electron smoketest | `.github/workflows/electron-smoketest.yml` | PR and push to master on `pyinstaller/electron/**` |
| Extension compatibility | `.github/workflows/extension-compat.yml` | PR; push on `requirements.*` / `pyproject.toml`; `workflow_dispatch` |
| Specterd build smoke | `.github/workflows/test-specterd-build.yml` | PR |
| Docker image push | `.github/workflows/docker-push.yml` | Push to any branch |
| Docker image tag | `.github/workflows/docker-tag.yml` | Tag push (`v*`) |
| Docs table of contents | `.github/workflows/toc.yml` | Push |
The image is created manually (see /docker) and used for running the tests AND also for ## Test workflow
spinning up bitcoind.
For that reason we need to share the docker-socket from the host into the container and `test.yml` has three jobs, all on `ubuntu-22.04`:
create our own GitLab specific runner as described here:
https://docs.gitlab.com/ee/ci/docker/using_docker_build.html#use-docker-socket-binding
Due to that setup there are some specifics which are mainly addressed in tests/conftest - **`test`** — pytest with `--cov=cryptoadvance`. Runs in 45 min. Installs system deps inline; no custom image. Caches bitcoind/elementsd binaries via `actions/cache@v4` keyed on `runner.os × runner.arch × hash(pyproject.toml, tests/install_noded.sh, tests/bitcoin_SHA256SUMS, tests/elements_SHA256SUMS)`.
start_bitcoind-function: - **`cypress`** — runs `./utils/test-cypress.sh --debug run` inside `ghcr.io/cryptoadvance/specter-desktop/cypress-python-jammy@sha256:<digest>`. 30-minute timeout. `--shm-size=2g` to avoid Cypress OOMs on the default 64 MB `/dev/shm`. Shares the bitcoind/elements cache with `test`.
* adding -rpcallowip= (from a docker network) to bitcoind - **`extension-smoketest`** — byte-compatible port of the former Cirrus smoketest. 15 min. Smoke-tests `ext gen`, server boot, and log-line / curl assertion. Contract must stay stable — downstream extension developers depend on it.
* not use localhost but the docker-network-ip-address when talking to the bitcoind
## Travis-CI All three jobs use `actions/checkout@v4` with `fetch-depth: 0` so `git describe` resolves annotated tags for `tests/test_util_version.py`.
We're no longer using travis-ci due to the abuse-detection-system going wild on us. ## Caching
## Cirrus-CI `actions/cache@v4` with `save-always: true` on a key that includes `runner.arch` (prevents ARM/x86 cache poisoning). The key hashes the committed `tests/bitcoin_SHA256SUMS` and `tests/elements_SHA256SUMS` trust anchors — bumping a version in `pyproject.toml` rotates the cache via those files.
[Cirrus-CI](https://cirrus-ci.org) is used by Bitcoin-Core and HWI and is a quite good replacement for travis. We're using it only for PRs so far. The [../.cirrus.yml] file defines the build. We have two task, one for pytest and one for the [cypress-tests](./cypress-testing.md). ### Binary verification
## Releasing `tests/install_noded.sh` GPG-verifies the upstream `SHA256SUMS.asc` against the Bitcoin Core and Elements release signing keys, and checks the tarball SHA256 against the committed trust anchors on every run (cold cache AND cache hit). A tampered cache entry fails closed on restore. See PR #2606 for the threat model.
### What gets released ## Cypress container
We're mostly releasing automatically. Currently the following artifacts are released: `ghcr.io/cryptoadvance/specter-desktop/cypress-python-jammy` is pinned by digest (not tag) in `test.yml`. This makes Dockerfile edits visibly require a workflow bump. When editing `docker/cypress-python-jammy/Dockerfile`, rebuild and push to GHCR with a fresh tag, then update the digest pin.
* specterd (daemon) is a binary for kicking off the specter-desktop service on the command-line. We have binaries for windows, Linux and macOS
* We have an Electron-App which we're also releasing for Windows, Linux and MacOS. Unfortunately the macOS build is not yet automated
* We release a pip-package
* Usually some time after the release, the lncm is releasing [docker-images](https://hub.docker.com/r/lncm/specter-desktop). Very much appreciated, even though we can't guarantee for them, obviously.
### How we release ## Release pipeline
As we have a strict build-only-on-private-hardware build-policy, we're using GitLab private runners in order to build our releases. In order to test and develop the releasing automation, people can setup GitLab-projects which are syncing from their GitHub-forks. With such a setup it's possible to create test-releases and therefore test the whole procedure end-to-end.
The automation of that kicks in if someone creates a tag which is named like "vX.Y.Z". This is specified in the gitlab-ci.yml. The release-job will only be triggered in cases of tags. One step will also check that the tag follows the convention above. See [`release-guide.md`](./release-guide.md). Pushing a tag matching `v[0-9]+.[0-9]+.[0-9]+[-*]?` triggers `release.yml`, which builds pip/specterd/Electron artifacts for Linux/Windows/macOS, signs `SHA256SUMS`, and creates a draft GitHub release. Docker images are built by `lncm/docker-specter-desktop` (triggered via `AARON_TRIGGER` secret).
The package upload will need a token. How to obtain the token is described in the packaging-tutorial. It's injected via GitLab-variables. ToDo: put the token on a trusted build-node.
### pyinstaller system-dependent binaries ## Flake policy
The [pyinstaller directory](../pyinstaller) contains scripts to create the platform-specific binaries (plus electron) to use specter-desktop as a desktop-software. Some of them are created and uploaded to [GitHub-releases](https://github.com/cryptoadvance/specter-desktop/releases) via more or less special build-agents.
The [windows-build-agent](https://docs.gitlab.com/runner/install/windows.html) needs manual installation
of git, python and docker. Docker is used to build the innosetup-file.
As docker is available in windows only as a "desktop-edition", one need to also
log into the windows-machine to get docker started.
Clearly there is an opportunity to move all of the creation of the windows-binary to wine on docker,
similiar to the way the innosetup is running within docker.
## CI/CD-dev-env setup - Cypress: `retries: { runMode: 1, openMode: 0 }`. Specs retry-to-green emit a warning annotation.
- pytest: `--reruns 0` (fail fast). Flakes are debt, not a coping mechanism.
- Spec flagged flaky twice in 14 days gets `@skip(reason="flaky", issue="#NNNN")` with a 2-week SLA.
Here is a brief description on how to create a setup where the release-procedures can be tested: ## Secrets
* We assume you have a fork of cryptoadvance/specter-desktop. We also assume that your GitLab-user-handle is the exact same as on GitHub.
* Create a GitLab-account and then a mirroring project ([here](https://gitlab.com/projects/new#cicd_for_external_repo)) obviously with the exact same name: "specter-desktop"
* Activate the private runners and deactivate the public runners. Contact @k9ert for that.
* Create an account and an [API token](https://test.pypi.org/manage/account/) on there
* Create a token for GitHub in order to release to your GitHub-fork
* Configure both tokens on the GitLab-variables (GH_BIN_UPLOAD_PW and TWINE_PASSWORD)
* create a tag on your GitHub-fork
* watch the test-release unfolding, ready to hack
### GitLab-runner setup (Windows) | Secret | Used by | Purpose |
|---------------------------|--------------------------|----------------------------------------------|
| `GITHUB_TOKEN` | (auto-provided) | Checkout, artifact upload, ghcr.io push |
| `GPG_PRIVATE_KEY` + `GPG_PASSPHRASE` | `release.yml` | Sign `SHA256SUMS` |
| `APPLE_*` (six) | `release.yml` macOS | Code signing + notarization (optional) |
| `AARON_TRIGGER` | `release.yml` | Trigger `lncm/docker-specter-desktop` build |
For Windows-releasing, we're using a windows GitLab-runner. Here is a short description on how to set one up. No GitLab secrets remain.
#### Prerequisites
You need at least Windows Home 10 which is up-to-date. The most complex dependency is setting up docker.
Docker-Desktop needs a WSL2 which is a good idea to install on windows anyway. [Here](https://www.omgubuntu.co.uk/how-to-install-wsl2-on-windows-10) is a description on how to do that.
While installing, make sure you know the locations of where that stuff is installed. We'll later need to verify/adjust the PATH.
* Install Python, i took the [3.7.9 webinstaller](https://www.python.org/ftp/python/3.7.9/python-3.7.9-amd64-webinstall.exe)
* Install Git, e.g. [this](https://github.com/git-for-windows/git/releases/download/v2.29.2.windows.2/Git-2.29.2.2-64-bit.exe) (i had 2.28.2)
* Install [Docker-Desktop](https://desktop.docker.com/win/stable/Docker%20Desktop%20Installer.exe)
Now open and check the "Environment-variables" and check that the following lines are in there:
![](./images/continuous-integration_runner_windows_envvars.png)
#### Runner
The runner itself is easy to [setup](https://docs.gitlab.com/runner/install/windows.html). Follow the link or this very brief description:
* `mkdir \Gitlab-Runner`
* download [this binary](https://gitlab-runner-downloads.s3.amazonaws.com/latest/binaries/gitlab-runner-windows-amd64.exe) in that folder and rename to gitlab-runner.exe
* Search for "powershell" in windows an open AS ADMINISTRATOR
* `cd \Gitlab-Runner`
* Copy the Registration-token from [here](https://gitlab.com/k9ert/specter-desktop/-/settings/ci_cd) (unfold runners, see specific runners)
* `./gitlab-runner.exe register`and paste the token (the instance-url is the default)
* give a reasonable description. Make sure to tag this runner with "tag". If that's not possible here, you can do it in the page mentioned above
* `.\gitlab-runner.exe install` will install the runner as system-service
* `.\gitlab-runner.exe start` will start it
Done

BIN
docs/images/continuous-integration_runner_windows_envvars.png
View file

Binary file not shown.
Side by side

Before

Width:  |  Height:  |  Size: 32 KiB

BIN
docs/images/release-guide/github-token.png
View file

Binary file not shown.
Side by side

Before

Width:  |  Height:  |  Size: 158 KiB

BIN
docs/images/release-guide/overview-gitlab-pipline.png
View file

Binary file not shown.
Side by side

Before

Width:  |  Height:  |  Size: 79 KiB

BIN
docs/images/release-guide/rerun-release-signatures.png
View file

Binary file not shown.
Side by side

Before

Width:  |  Height:  |  Size: 448 KiB

186
docs/release-guide.md
View file

@ -1,160 +1,60 @@
# Release Guide # Release Guide
## Creating release notes The release pipeline runs on GitHub Actions. Pushing a version tag to `upstream` triggers `.github/workflows/release.yml`, which builds every artifact, creates a draft GitHub release, and signs `SHA256SUMS` with the Specter Signer GPG key.
### Pre-requisites ## Prerequisites
- You need the correct upstream master. You should see - `upstream` points at `git@github.com:cryptoadvance/specter-desktop.git` (`git remote -v` should show both fetch and push).
- You are on `master` with a clean workspace and `git pull upstream master` applied.
- Release-notes PR has already merged (see [Release notes](#release-notes) below).
## Cut a release
```bash ```bash
git remote -v | grep upstream git tag v1.13.1
upstream git@github.com:cryptoadvance/specter-desktop.git (fetch) git push upstream v1.13.1
upstream git@github.com:cryptoadvance/specter-desktop.git (push)
``` ```
- You need a GitHub token: That's it. The `Release` workflow on GitHub Actions takes it from here:
If you don't have one, get one here https://github.com/settings/tokens and make sure to tick the boxes for repo and workflow as below:
![](./images/release-guide/github-token.png) - **`release-pip`** — builds the sdist/wheel and publishes to PyPI via trusted publishing.
- **`build-specterd-{linux,windows,macos}`** — builds the `specterd` binary on each platform (macOS arm64 on the free `macos-14` runner).
- **`build-electron-{linux,windows,macos}`** — builds the Electron apps using each platform's `specterd` artifact. Windows uses the public `electronuserland/builder:wine` image; macOS signs + notarizes if `APPLE_CERTIFICATE_BASE64` et al. are configured.
- **`create-release`** — collects all artifacts, generates `SHA256SUMS`, signs it with the GPG key from the `GPG_PRIVATE_KEY` secret, generates a release body (with auto-generated "What's Changed" via `gh api .../generate-notes`), and creates a **draft** GitHub release.
- **`trigger-docker`** — POSTs a repository-dispatch to `lncm/docker-specter-desktop` so Aaron's Docker build picks up the new tag (needs `AARON_TRIGGER` secret; skipped otherwise).
Using the new token, run The release lands as a draft — review and publish it manually on GitHub.
### Required secrets
| Secret | Purpose |
|------------------------------------|--------------------------------------------------------------|
| `GPG_PRIVATE_KEY` | ASCII-armored private key for signing `SHA256SUMS` |
| `GPG_PASSPHRASE` | Passphrase for the above |
| `APPLE_CERTIFICATE_BASE64` | Developer ID cert for macOS signing (optional — unsigned fallback) |
| `APPLE_CERTIFICATE_PASSWORD` | p12 password |
| `APPLE_ID`, `APPLE_APP_SPECIFIC_PASSWORD`, `APPLE_TEAM_ID` | Apple notarization credentials |
| `APPLE_PROVISIONING_PROFILE_BASE64`| Optional provisioning profile |
| `AARON_TRIGGER` | PAT for triggering `lncm/docker-specter-desktop` |
PyPI publishing uses trusted publishing (OIDC) — no secret required.
## Release notes
Update `docs/release-notes.md` via a normal PR before tagging. Use the GitHub API or `gh` to pull "What's Changed" between the previous tag and `master`, prepend a heading, and open a PR. The `create-release` workflow job also appends auto-generated notes to the release body.
## GitHub pages download page
`./utils/generate_downloadpage.sh` still generates the `specter-static` website's download page off `utils/templates/`. Clone `specter-static` alongside `specter-desktop` and run:
```bash ```bash
export GH_TOKEN=YOURTOKEN ./utils/generate_downloadpage.sh
``` ```
- You need Docker running The script installs the markdown prerequisite, regenerates the GH-page and download page, asks whether to replace/update the GitHub release page for the latest version, and offers to commit/push the static-site changes.
- Checkout the master branch and ensure a clean workspace.
Now, you can run ## Troubleshooting
```bash If something fails mid-pipeline, re-running individual jobs is safe — they `actions/download-artifact` from prior jobs and overwrite existing release assets via `softprops/action-gh-release`. If the draft release already has assets from a stale run, delete the draft and re-run `create-release`.
./utils/release.sh --release-notes
```
Or, if you want to directly set the new version: macOS builds are the most likely to fail due to Apple signing/notarization glitches. The workflow falls back to unsigned builds when `APPLE_CERTIFICATE_BASE64` is empty — useful for smoke-testing the pipeline on forks.
```bash
./utils/release.sh --new-version v1.13.1 --release-notes
```
## Creating a new tag
Update your master branch after the release notes PR ([example](https://github.com/cryptoadvance/specter-desktop/commit/65ff6959d7fd85cba745e4d454b30031839f857f/)) has been merged and then run:
```bash
git tag v1.13.1 && git push upstream v1.13.1
```
if you have a proper setup of github- AND gitlab forks (and the remote `origin` on git pointing to your github fork), you can push that tag to origin and this is useful to test the procedures. See "CI/CD-dev-env setup" in [continuous-integration](./continuous-integration.md).
```bash
git tag v1.13.1 && git push origin v1.13.1
```
## GitLab - releasing stage
Creating a tag triggers the release process of the GitLab runners.
There exists a mirror of the GitHub repo on GitLab, but only when a tag is created on GitHub will the release part of the runners execute. You can check the status here:
https://gitlab.com/cryptoadvance/specter-desktop/-/pipelines
There are three stages:
![](./images/release-guide/overview-gitlab-pipline.png)
The first relevant stage is "releasing". Here, the Windows, Linux and pip release are created and uploaded to the Specter Desktop GitHub releases page. After this stage, the following artificats should be available:
- cryptoadvance.specter-1.13.1.tar.gz
- Specter-Setup-v1.13.1.exe
- specterd-v1.13.1-win64.zip
- specterd-v1.13.1-x86_64-linux-gnu.zip
- specter_desktop-v1.13.1-x86_64-linux-gnu.tar.gz
The three jobs in more detail:
- release_binary_windows: is creating a binary for specterd and for Windows (Windows runner)
- release_electron_linux_windows: Creates a specterd for Linux, an AppImage for Linux and an executable for Windows (Linux runner).
- release_pip: Is releasing a pypi package on [pypi](https://pypi.org/project/cryptoadvance.specter/) and creates a tarball of the pip package for the GitHub release page (Linux runner).
For details look at `.gitlab-ci.yml`
## MacOS
Ideally, directly after the tag is created, start with the MacOS release. As the binaries of x86/arm64 are not compatible with each other, we need to build on two MacOS architectures.This has to be done manually, for now. There is a script for this. Start with the build on x86:
### MacOS x64 build
```bash
./utils/build-osx.sh --version v2.0.5-pre4 specterd package upload
```
You can also test this procedure without messing the original project via changing the `orgName` to your `orgName` in `pyinstaller/electron/downloadloc.js`.
This will create three artifacts on github:
* specterd-v2.0.5-pre4-osx_x64.zip
* SHA256SUMS-macos_x64
* SHA256SUMS-macos_x64.asc
### MacOS arm64 build
The electron application will get built on the arm architecture. As it needs to store the sha256 hash in the electron-app, the make-hash target
will not only hash the specterd but also download the other specterd and hash it.
```bash
./utils/build-osx.sh --version v2.0.5-pre4 --appleid "Satoshi Nakamoto (appleid)" --mail "satoshi@gmx.com" specterd make-hash electron sign package upload
```
This will create four artifacts on github:
* Specter-v2.0.5-pre4.dmg
* specterd-v2.0.5-pre4-osx_arm64.zip
* SHA256SUMS-macos_arm64
* SHA256SUMS-macos_arm64.asc
## GitLab - post releasing
Back to GitLab, the final stage is "post releasing".
### release_signatures
In this job, the individual SHA256-hashes and signatures are combined into two final files:
- SHA256SUMS
- SHA256SUMS.asc
Everything, apart from the MacOS files, are pulled from the GitLab environment, the MacOS files from GitHub.
Don't forget to delete the four MacOS files (`SHA256SUMS-macos_arm64` and `SHA256SUMS-macos_arm64.asc` and the two corresponding `_x64` files) on the GitHub release page in the end.
This is difficult to automate as sometimes the manual steps has not succeeded while generating the SHASUM-files. As a result, those hashes are not included. So you might want to run this again. And you can, just delete the two generated files - `SHA256SUMS` and `SHA256SUMS.asc` and run the job again.
### release_docker
There are docker images created by the awesome [Chiang Mai LN dev](https://github.com/lncm/docker-specter-desktop). So the task of this job is to trigger their build-system which is done via `utils/trigger_docker_build.sh`. A prerequisite of this is a token in order to authenticate. That token is from Aaron, one of the maintainers of that repo, and can be found in the gitlab variables section of the CI/CD configuration.
### tag_specterext_dummy_repo
Sometimes there are changes on the plugin architecture. In order to create a plugin, it's quite important to know which version of the plugin system should be used. Because of that, we simply assume that the master of the [specterext-dummy](https://github.com/cryptoadvance/specterext-dummy) repo is compatible with the current master which was just tagged with the new version.
So this job will tag that repo with the same tag and the creation of a plugin will take the version into account.
## Trouble shooting
If the MacOS signatures are missing, it can happen that the following Exception will be raised:
```bash
File "/builds/cryptoadvance/specter-desktop/utils/github.py", line 295, in download_artifact
raise Exception(
Exception: Status-cod04 for url ... )
```
In any case, if the macOS binaries arrive on GitHub too late, you have to manually delete the already created `SHA256SUMS` and `SHA256SUMS.asc`, otherwise the upload to GitHub will fail if you rerun the release signatures job on GitLab - for details see ([this PR](https://github.com/cryptoadvance/specter-desktop/pull/689)). The green arrow in the screenshot is where you rerun the release signatures job on GitLab:
![](./images/release-guide/rerun-release-signatures.png)
## GitHub release page and download page
This is handled by the script `./utils/generate_downloadpage.sh`. As a prerequisite, you need to clone the `specter-static` repo which contains the specter website. Clone it on the same level than specter-desktop.
Running that script will:
- install the prerequisites (basically markdown, see pyproject.toml)
- generate the GH-page and the download-page based on the `utils/templates`.
- Asks whether it should replace/update/initialize the Github Release page for the latest version
- copies over the new download-pages and asks whether it should commit/push those

23
pyinstaller/README.md
View file

@ -1,14 +1,7 @@
# Build scripts
Run `build-<your-os> <version_number>` file to build everything.
For example, `build-osx.sh 1.2.3` will create `SpecterDesktop-1.2.3.dmg` and `specterd-1.2.3-osx.zip` in the `release` folder.
If you're making a real release, you should append `"make hash"` at the end of your command calling the build script.
This will update the file hash and version name the Specter Desktop app expects to download from GitHub.
# Pyinstaller build # Pyinstaller build
Releases are built by `.github/workflows/release.yml` (triggered by a version tag). The notes below are for local / manual builds.
Install requirements: Install requirements:
```bash ```bash
@ -52,17 +45,9 @@ If this is the first time you go through this process, you'll need to first set
xcrun altool --store-password-in-keychain-item "AC_PASSWORD" -u "<your-apple-id>" -p "<the-generated-password>" xcrun altool --store-password-in-keychain-item "AC_PASSWORD" -u "<your-apple-id>" -p "<the-generated-password>"
``` ```
After having these set up, you can use the automated script to sign by passing it 2 extra parameters: Release builds sign and notarize via `.github/workflows/release.yml` (`build-electron-macos` job) using the `APPLE_CERTIFICATE_BASE64`, `APPLE_CERTIFICATE_PASSWORD`, `APPLE_ID`, `APPLE_APP_SPECIFIC_PASSWORD`, and `APPLE_TEAM_ID` secrets. See `docs/release-guide.md` for the full secret inventory. For manual local signing, use `electron-builder` directly (`npm run dist -- --mac` in `pyinstaller/electron/`, with the identity configured in `package.json`).
- Your certificate name, which you can see on the Keychain app going to the sidebar -> `My Certificates` and copying the name of the certificate you've created in step 1.
- Your Apple ID.
With these two, you can run the command like so: Notarization takes ~10 minutes, during which Apple emails notification of success/failure.
```bash
./build-osx.sh <version_number> "<certificate_name>" "<apple_id>" "make-hash"
```
*Note: "make-hash" is optional and will automatically calculate hash of specterd generated for the macOS app. Should be used only for real release.*
This should take 10 minutes, during which you should receive an email from Apple notifying whatever the notarization was successful.
If for some reason the notarization failed, you'll be able to get the reason by copying the `Request Identifier` (you should be able to find this in the email and in the logs). If for some reason the notarization failed, you'll be able to get the reason by copying the `Request Identifier` (you should be able to find this in the email and in the logs).
Then run the following command: Then run the following command:
```bash ```bash

67
pyinstaller/build-ci.sh
View file

@ -1,67 +0,0 @@
#!/usr/bin/env bash
set -e
# pass version number as an argument
echo " --> This build got triggered for version $1"
echo " --> Assumed gitlab-project: ${CI_PROJECT_ROOT_NAMESPACE:+x}"
[ -z "${CI_PROJECT_ROOT_NAMESPACE:+x}" ] && \
echo "Redefining CI_PROJECT_ROOT_NAMESPACE=cryptoadvance " && \
export CI_PROJECT_ROOT_NAMESPACE=cryptoadvance
echo $1 > version.txt
echo " --> Installing (build)-requirements"
pip3 install -r requirements.txt --require-hashes
cd ..
python3 setup.py install
pip3 install -e .
cd pyinstaller
echo " --> Cleaning up"
rm -rf build/ dist/ release/ electron/release/ electron/dist release-linux/ release-win/
echo " --> Building specterd"
pyinstaller specterd.spec
echo " --> Making us ready for building electron-app for linux"
cd electron
npm ci
node ./set-version $1 ../dist/specterd
echo " --> building electron-app"
npm i
npm run dist -- --linux
cd ..
echo " --> Making the release-zip"
mkdir release-linux
cd dist
cp -r ../../udev ./udev
echo "Don't forget to set up udev rules! Check out udev folder for instructions." > README.md
zip -r ../release-linux/specterd-"$1"-"$(uname -m)"-linux-gnu.zip specterd udev README.md
cp ../electron/dist/Specter-* ./
tar -czvf ../release-linux/specter_desktop-"$1"-"$(uname -m)"-linux-gnu.tar.gz Specter-* udev README.md
echo " --> Cleaning up"
cd ..
rm -rf dist
mkdir dist
cd dist
echo " --> Downloading the windows-version of specterd for version $1"
wget --progress=dot -e dotbytes=10M https://github.com/${CI_PROJECT_ROOT_NAMESPACE}/specter-desktop/releases/download/$1/specterd-$1-win64.zip -O ./specterd.zip
unzip specterd.zip
cd ../electron
rm -rf dist/
echo " --> Making us ready for building electron-app for windows"
npm ci
node ./set-version $1 ../dist/specterd.exe
npm run dist -- --win
cd ..
mkdir release-win
cp electron/dist/Specter\ Setup\ *.exe release-win/Specter-Setup-$1.exe

42
pyinstaller/build-win-ci.bat
View file

@ -1,42 +0,0 @@
@ECHO OFF
python -V
pip3 install virtualenv
echo " --> cleaning up"
rmdir /s /q .\release\
rmdir /s /q .\dist
rmdir /s /q .buildenv
echo " --> Creating virtualenv"
virtualenv --python=python3 .buildenv
echo " --> Activating virtualenv"
call .\.buildenv\Scripts\activate
echo " --> Installing test-requirement"
pip3 install -e ".[test]"
echo " --> Building pypi package"
pip3 install build==0.10.0
python -m build
echo " --> Installing pypi package"
python .\utils\release_helper.py install_wheel %1%
cd pyinstaller
Rem This file gets further packaged up with the pyinstaller and will help specter to figure out which version it's running on
echo %1% > version.txt
echo " --> installing pyinstaller requirements"
pip3 install -r requirements.txt --require-hashes
rmdir /s /q .\dist\
rmdir /s /q .\build\
rmdir /s /q .\release\
rmdir /s /q .\electron\dist\
echo " --> Creating the pyinstaller binary"
pyinstaller.exe specterd.spec
mkdir release
echo " --> Creating the release-package"
powershell Compress-Archive -Path dist\specterd.exe release\specterd-%1%-win64.zip

31
pyinstaller/build-win.bat
View file

@ -1,31 +0,0 @@
@ECHO OFF
echo %1% > version.txt
pip3 install -r requirements.txt --require-hashes
cd ..
Rem Order is relevant here. If you flip the followng lines, the hiddenimports for services won't work anymore
python3 setup.py install
pip3 install -e .
cd pyinstaller
rmdir /s /q .\dist\
rmdir /s /q .\build\
rmdir /s /q .\release\
rmdir /s /q .\electron\dist\
pyinstaller.exe specterd.spec
cd electron
call npm ci
if "%2%"=="make-hash" (
call node ./set-version "%1%" "../dist/specterd.exe"
) else (
node ./set-version "%1%"
)
call npm i
call npm run dist
cd ..
mkdir release
SET EXE_PATH="electron\dist\Specter Setup *.exe"
SET EXE_RELEASE_PATH="release\Specter Setup %1%.exe"
echo f | xcopy /s/y %EXE_PATH% %EXE_RELEASE_PATH%
powershell Compress-Archive -Path dist\specterd.exe release\specterd-%1%-win64.zip

1
pyproject.toml
View file

@ -82,7 +82,6 @@ test = [
"PySocks==1.7.1", "PySocks==1.7.1",
"pytest-cov==2.10.1", "pytest-cov==2.10.1",
"mock==4.0.2", "mock==4.0.2",
"python-gitlab==2.10.1",
# requirements for stuff in ./utils # requirements for stuff in ./utils
"requests==2.31.0", "requests==2.31.0",
] ]

2
src/cryptoadvance/specter/managers/service_manager/service_manager.py
View file

@ -464,7 +464,7 @@ class ExtensionManager:
site_package = Path(virtuelenv_path, *(Path(site_package).parts[-3:-1])) site_package = Path(virtuelenv_path, *(Path(site_package).parts[-3:-1]))
virtualenv_search_path = site_package virtualenv_search_path = site_package
# ... and as the classes are in the .buildenv (see build-unix.sh) let's add .. # ... and as the classes are in the .buildenv let's add ..
arr = [Path(virtualenv_search_path, path) for path in arr] arr = [Path(virtualenv_search_path, path) for path in arr]
# Non internal-repo extensions sitting in org/specterext/... need to be added, too # Non internal-repo extensions sitting in org/specterext/... need to be added, too

81
utils/artifact_signer.sh
View file

@ -1,81 +0,0 @@
#!/bin/bash
function sub_help {
echo "This script is to sign artifacts or to prepare the gpg-system to be able to verify artifacts."
echo "Do one of these:"
echo "$ ./utils/artifact_signer.sh init"
echo "This makes sense only on a gitlab-runner. It'll unpack a gpg-directory to be ready to sign and verify"
echo "$ ./utils/artifact_signer.sh sign --artifact ./release-win/SHA256SUMS-win"
echo "Signs a specific artifact. Will do the init on the fly. So no need to call it extra."
}
while [[ $# -gt 0 ]]
do
key="$1"
command="main"
case $key in
--help)
sub_help
exit
shift
;;
--artifact)
artifact=$2
shift
shift
;;
sign)
action=sign
shift
;;
init)
action=init
shift
;;
--debug)
set -x
shift # past argument
;;
*) # unknown option
POSITIONAL="$1" # save it in an array for later
shift # past argument
;;
esac
done
# We want a detached signature in cleartext. Extension: .asc (as in bitcoin)
output_file=${artifact}.asc
# lazy init: We're initializing Each thime script is called with these two things.
# So that action "init" is just to have a bit more semantics for the one calling this script
function init {
if [[ -f /credentials/gnupg.tar.gz ]]; then
echo "Init: extracting gnupg.tar.gz"
tar -xzf /credentials/gnupg.tar.gz -C /root
chown -R root:root ~/.gnupg
else
echo "Init: Could not find any /credentials/gnupg.tar.gz"
fi
if [[ -f /credentials/private.key ]]; then
echo "Init: Importing single private key"
gpg --import --no-tty --batch --yes /credentials/private.key
else
echo "Init: Could not find any /credentials/private.key"
fi
}
if [ "$action" = "init" ]; then
init
fi
if [ "$action" = "sign" ]; then
init
if [[ -z $artifact ]]; then
echo "no --artifact given "
exit 1
fi
echo "signing ..."
echo $GPG_PASSPHRASE | gpg --detach-sign --armor --no-tty --batch --yes --passphrase-fd 0 --pinentry-mode loopback $artifact
fi

135
utils/build-common.sh
View file

@ -1,135 +0,0 @@
#!/usr/bin/env bash
# All functions in here are responsible to change directory
# from the project root to wherever they want
# They need to change back to project-root when they finish
function create_virtualenv_for_pyinstaller {
echo " --> Creating new virtualsenv"
if [ -d .buildenv ]; then
echo " But first Delete it ..."
rm -rf .buildenv
fi
virtualenv --python=python3.10 .buildenv
source .buildenv/bin/activate
pip3 install -e ".[test]"
}
function build_pypi_pckgs_and_install {
echo " --> Build pip3-package"
rm -rf dist
if ! git diff --quiet setup.py; then
echo "ERROR: setup.py is dirty, can't reasonably build"
exit 1
fi
if [[ "$OSTYPE" == "darwin"* ]]; then
SML_ADD="\"\""
fi
pip3 install build==0.10.0
python3 -m build
pip3 install ./dist/cryptoadvance.specter-*.whl
}
function configure {
echo " --> Configure some variables"
if [ -z "$app_name" ]; then
# activate virtualenv. This is e.g. not needed in CI
app_name=specter
specterd_filename=specterd
specterimg_filename=Specter
pkg_filename=specter_desktop
else
specterd_filename=${app_name}d # usually "specterd"
specterimg_filename=${app_name^} # usually "Specter"
pkg_filename=${app_name}
fi
export ARCH=$(node -e "console.log(process.arch)")
export dist_mac_folder_name=mac-universal
export CI_COMMIT_TAG=$version
export CI_PROJECT_ROOT_NAMESPACE=$(node -e "const downloadloc = require('./pyinstaller/electron/downloadloc');console.log(downloadloc.orgName())")
echo specterd_filename=${specterd_filename}
echo specterimg_filename=${specterimg_filename}
echo pkg_filename=${pkg_filename}
echo ARCH=$ARCH
echo dist_mac_folder_name=$dist_mac_folder_name
echo CI_COMMIT_TAG=$CI_COMMIT_TAG
echo CI_PROJECT_ROOT_NAMESPACE=$CI_PROJECT_ROOT_NAMESPACE
}
function install_build_requirements {
echo " --> Installing pyinstaller build-requirements"
cd pyinstaller
pip3 install -r requirements.txt --require-hashes > /dev/null
cd ..
}
function cleanup {
echo " --> Cleaning up"
cd pyinstaller
rm -rf build/ dist/ release/ electron/release/ electron/dist
rm *.dmg || true
cd ..
}
function building_app {
echo " --> Building ${specterd_filename}"
cd pyinstaller
specterd_filename=${specterd_filename} pyinstaller specterd.spec > /dev/null
cd ..
}
function prepare_npm {
cd pyinstaller/electron
echo " --> Making us ready for building electron-app"
npm ci
cd ../..
}
function make_hash_if_necessary {
cd pyinstaller/electron
echo " --> calculate the hash of the binary for download"
if [[ "$1" = "win" ]]; then
specterd_plt_filename=../dist/${specterd_filename}.exe
else
specterd_plt_filename=../dist/${specterd_filename}
fi
if [[ "$make_hash" == 'True' ]]
then
node ./set-version $version ${specterd_plt_filename}
else
node ./set-version $version
fi
echo " Hash in version -data.json $(cat ./version-data.json | jq -r '.sha256')"
echo " Hash of file $(sha256sum ${specterd_plt_filename} )"
cd ../..
}
function building_electron_app {
# https://www.electron.build/
# Prerequisites:
# * A developer Certificate (in the System keychain)
# * private and public key in the login-keychain
# * The cert needs to be referenced in pyinstaller/electron/package.json -> build.mac.identity
platform="-- --${1}" # either linux or win (maxOS is empty)
cd pyinstaller/electron
echo " --> building electron-app"
echo " --> Copying over resources"
cp -R ../../src/cryptoadvance/specter/static/fonts ../../src/cryptoadvance/specter/static/output.css ../../src/cryptoadvance/specter/static/typography.css .
npm i
npm run dist ${platform}
cd ../..
}
function make_release_zip {
echo " --> Making the release-zip"
}

404
utils/build-osx.sh
View file

@ -1,404 +0,0 @@
#!/usr/bin/env bash
set -e
# We start in the directory where this script is located
cd "$( dirname "${BASH_SOURCE[0]}" )/."
source build-common.sh
cd ..
# Now in project-root
# Overriding this function
function create_virtualenv_for_pyinstaller {
# This currently assumes to be run with: Python 3.10.11
# Important: pyinstaller needs a Python binary with shared library files
# With pyenv, for example, you get this like so: env PYTHON_CONFIGURE_OPTS="--enable-shared" pyenv install 3.10.4
# Use pyenv if set as environment variable
if [ $USE_PYENV_FOR_SPECTER_BUILD = true ]; then
echo "Trying to use pyenv ..."
if ! command -v pyenv >/dev/null 2>&1; then
echo "Error: pyenv is not available. Please make sure pyenv is installed and configured properly." >&2
exit 1
fi
### This is usually in .zshrc, putting it in .bashrc didn't work ###
export PYENV_ROOT="$HOME/.pyenv"
command -v pyenv >/dev/null || export PATH="$PYENV_ROOT/bin:$PATH"
eval "$(pyenv init -)"
### this needs the pyenv-virtualenv plugin. If you don't have it:
### git clone https://github.com/pyenv/pyenv-virtualenv.git $(pyenv root)/plugins/pyenv-virtualenv
eval "$(pyenv virtualenv-init -)"
### ------------------------------------------------------------ ###
PYTHON_VERSION=3.10.11
export PYENV_VERSION=$PYTHON_VERSION
echo "Setting PYENV_VERSION to 3.10.11, using pyenv-virtualenv to create the buildenv..."
echo " --> Deleting .buildenv"
pyenv uninstall -f .buildenv
rm -rf "$HOME/.pyenv/versions/$PYTHON_VERSION/envs/.buildenv"
pyenv virtualenv 3.10.11 .buildenv
pyenv activate .buildenv
else
echo "pyenv is not available. Using system Python version."
if [ -d .buildenv ]; then
echo " --> Deleting .buildenv"
rm -rf .buildenv
fi
virtualenv .buildenv
source .buildenv/bin/activate
fi
pip3 install -e ".[test]"
}
# Overriding this function to deal with the x86 special case
function make_hash_if_necessary {
cd pyinstaller/electron
echo " --> calculate the hash of the binary for download"
specterd_plt_filename=../dist/${specterd_filename}
# early exit
if [[ "$make_hash" != 'True' ]]; then
node ./set-version $version
return 0
fi
# We need to set-versions for two specterd, one arm and one intel.
# arm64 one
node ./set-version $version ${specterd_plt_filename}
# Download and check the intel one
# this needs some env-vars to be set
rm -rf signing_dir/*
PYTHONPATH=../.. python3 -m utils.release_helper downloadgithub
ret_code=$?
if [ $ret_code -ne 0 ]; then
echo "Downloading and verifying x64 specterd failed with exit code $ret_code"
exit $ret_code
fi
if [[ ! -f ./signing_dir/specterd-${version}-osx_x64.zip ]]; then
echo "Downloading and verifying x64 specterd failed as the file does not seem to be there"
exit 1
fi
rm -f /tmp/specterd
unzip ./signing_dir/specterd-${version}-osx_x64.zip -d /tmp
node ./set-version $version /tmp/specterd x64
echo " Hashes in version-data.json $(cat ./version-data.json | jq -r '.sha256')"
echo " Hash of file $(sha256sum ${specterd_plt_filename} )"
echo " Hash of x64 file $(sha256sum /tmp/specterd )"
cd ../..
}
function macos_code_sign {
# prerequisites for this:
# in short:
# * make sure you have a proper app-specific password on https://appleid.apple.com/account/manage
# * collect some information via scrun altool --list-providers -u "<yourAppleID>"
# * create profile via xcrun notarytool store-credentials --apple-id "<YourAppleID>" --password "app-specific-pw" --team-id "seeFromAbove"
# * Call the profile: SpecterProfile
# For details see:
# * https://www.youtube.com/watch?v=2xJcMzoi0EI
# * https://blog.dgunia.de/2022/09/01/switching-from-altool-to-notarytool/
# * https://scriptingosx.com/2021/07/notarize-a-command-line-tool-with-notarytool/
# This creates a ZIP archive from the app package (using the ditto command).
# This ZIP archive is then used to upload the app to the Apple notarization service via xcrun notarytool (formerly xcrun altool)
# After the app has been uploaded to the Apple servers and notarized, the ZIP archive is not used again.
# The function uses the xcrun stapler command to attach the notarization result to the app, and then exits.
# docs:
# https://help.apple.com/itc/apploader/#/apdATD1E53-D1E1A1303-D1E53A1126
# https://keith.github.io/xcode-man-pages/altool.1.html
cd pyinstaller/electron
echo ' --> Attempting to code sign...'
specterimg_filename_fqfn=dist/${dist_mac_folder_name}/${specterimg_filename}.app
echo " executing: ditto -c -k --keepParent "${specterimg_filename_fqfn}" dist/${specterimg_filename}.zip"
ditto -c -k --keepParent "${specterimg_filename_fqfn}" dist/${specterimg_filename}.zip
# upload
echo ' uploading for notarisation ... '
output_json=$(xcrun notarytool submit dist/${specterimg_filename}.zip --apple-id "kneunert@gmail.com" --keychain-profile "SpecterProfile" --output-format json --wait )
# parsing the requestuuid which we'll need to track progress
requestuuid=$(echo $output_json | jq -r '.id')
status=$(echo $output_json | jq -r '.status')
echo "Request ID: $requestuuid"
if [ "$status" = "Invalid" ]; then
mkdir -p signing_logs
echo "issues with notarisation"
xcrun notarytool log ${requestuuid} --keychain-profile SpecterProfile | tee ./signing_logs/${app_name}_${timestamp}_${requestuuid}.log
exit 1
fi
# The stapler somehow "staples" the result of the notarisation in to your app
# see e.g. https://stackoverflow.com/questions/58817903/how-to-download-notarized-files-from-apple
echo " --> Staple the file dist/${dist_mac_folder_name}/${specterimg_filename}.app"
xcrun stapler staple "dist/${dist_mac_folder_name}/${specterimg_filename}.app"
echo ' Successfully Stapled the file'
cd ../..
}
function sub_help {
cat << EOF
### Quick overview
: <<'END_COMMENT'
What do you need to sign the Specter app with Apple's notary service?
- An Apple Developer account
- You must create a signing certificate in your developer account, which will be used to sign your app.
- This certificate must be stored in your keychain on your Mac.
- When you create a signing certificate in your developer account, you will be asked to specify a password for the certificate.
- You can store this password in the keychain, too, so that it - and thus the certificate - can be accessed automatically during the signing process. Like so:
xcrun altool --store-password-in-keychain-item AC_PASSWORD -u '<your apple id>' -p apassword
- As seen above, you need the the xcrun command line tool: This tool is also used to upload your app to the notary service and check the status of the notarization process.
In summary, to sign a macOS app with Apple's notary service, you need an Apple Developer account, a signing certificate, a password for your keychain, the app package to be signed, and the xcrun command line tool.
END_COMMENT
### Prerequisites
# brew install gmp # to prevent module 'embit.util' has no attribute 'ctypes_secp256k1'
# brew install jq
# npm install --global create-dmg
### Trouble shooting
# If you have the common issue "errSecInternalComponent" while signing the code:
# https://medium.com/@ceyhunkeklik/how-to-fix-ios-application-code-signing-error-4818bd331327
# create-dmg issue? Note that there are 2 create-dmg scripts out there. We use:
# https://github.com/sindresorhus/create-dmg
The different "tasks" are now somehow separated from one another.
We have:
* make-hash is rather a flag for the electron-build to incorporate the hash of the specterd
* specterd will trigger the pyinstaller build of the specterd
* electron will build the electron-app
* sign will upload the electron-app to the Apple notary service and get it back notarized
* upload will upload all the binary artifacts to the github-release-page. This includes the creation of the hash-files
and the gnupg signing
### Trouble shooting (Legacy)
# Currently, only MacOS Catalina is supported to build the dmg-file
# Therefore we expect xcode 12.1 (according to google)
# After installation of xcode: sudo xcode-select -s /Applications/Xcode.app/Contents/Developer
# otherwise you get xcrun: error: unable to find utility "altool", not a developer tool or in PATH
# catalina might have a a too old version of bash. You need at least 4.0 or so
# 3.2 is too low definitely
# brew install bash
# Example-call:
./utils/build-osx.sh --debug --version v1.10.0-pre23 --appleid "Kim Neunert (FWV59JHV83)" --mail "kim@specter.solutions" make-hash specterd electron sign upload
EOF
}
appleid=""
while [[ $# -gt 0 ]]
do
arg="$1"
case $arg in
"" | "-h" | "--help")
sub_help
exit 0
shift
;;
--debug)
set -x
DEBUG=true
shift
;;
--version)
version=$2
shift
shift
;;
--appleid)
appleid=$2
shift
shift
;;
--mail)
mail=$2
shift
shift
;;
specterd)
build_specterd=True
shift
;;
make-hash)
make_hash=True
shift
;;
electron)
build_electron=True
shift
;;
sign)
build_sign=True
shift
;;
package)
build_package=True
shift
;;
upload)
upload=True
shift
;;
help)
sub_help
shift
;;
*)
shift
sub_${arg} $@ && ret=0 || ret=$?
if [ "$ret" = 127 ]; then
echo "Error: '$arg' is not a known subcommand." >&2
echo " Run '$progname --help' for a list of known subcommands." >&2
exit 1
else
exit $ret_value
fi
;;
esac
done
echo " --> This build got triggered for version $version"
echo $version > pyinstaller/version.txt
configure
if [[ "$build_specterd" = "True" ]]; then
create_virtualenv_for_pyinstaller
build_pypi_pckgs_and_install
install_build_requirements
cleanup
building_app
fi
if [[ "$make_hash" = "True" ]]; then
# Making the hash only makes sense on a arm arch
if [[ "$ARCH" != "arm64" ]]; then
echo "ERROR: make-hash target should be only called on an arm64 machine on a mac"
exit 1
fi
make_hash_if_necessary
fi
if [[ "$build_electron" = "True" ]]; then
# Making the hash only makes sense on a arm arch
if [[ "$ARCH" != "arm64" ]]; then
echo "ERROR: electron target should be only called on an arm64 machine on a mac"
exit 1
fi
prepare_npm
npm i
if [[ "${appleid}" == '' ]]
then
echo "`jq '.build.mac.identity=null' package.json`" > package.json
else
echo "`jq '.build.mac.identity="'"${appleid}"'"' package.json`" > package.json
fi
building_electron_app
fi
if [[ "$build_sign" = "True" ]]; then
# if [ "$(uname -m)" = "arm64" ]; then
# dist_mac_folder_name=${dist_mac_folder_name}-arm64
# fi
if [[ "$appleid" != '' ]]; then
macos_code_sign
else
echo "WARNING: Forgot to add the appleid ?!"
exit 1
fi
fi
if [[ "$build_package" = "True" ]]; then
echo " --> Preparing the release"
mkdir -p release
rm -rf release/*
# The specterd-zipfile from specterd
if [[ -f pyinstaller/dist/${specterd_filename} ]]; then
echo " --> Making the release-zip for specterd"
pushd pyinstaller/dist # to not preserve folder structure
zip ../../release/${specterd_filename}-${version}-osx_${ARCH}.zip ${specterd_filename}
popd
fi
# The dmg image file from App
if [[ -d pyinstaller/electron/dist/${dist_mac_folder_name}/${specterimg_filename}.app ]]; then
rm -f pyinstaller/electron/dist/*.dmg
echo " --> Creating dmg"
create-dmg pyinstaller/electron/dist/${dist_mac_folder_name}/${specterimg_filename}.app --identity="Developer ID Application: ${appleid}" pyinstaller/electron/dist
# create-dmg doesn't create the prepending "v" to the version
node_comp_version=$(python3 -c "print('$version'[1:])")
mv "pyinstaller/electron/dist/${specterimg_filename} ${node_comp_version}.dmg" dist/${specterimg_filename}-${version}.dmg
echo " --> Copying img file dist/${specterimg_filename}-${version}.dmg"
cp dist/${specterimg_filename}-${version}.dmg release/${specterimg_filename}-${version}.dmg
else
echo "WARNING: Skipping packaging for electron App"
echo "No pyinstaller/electron/dist/${dist_mac_folder_name}/${specterimg_filename}.app has been found."
fi
file=./release/${specterd_filename}-${version}-osx_${ARCH}.zip
if [[ -f $file ]]; then
echo -n " FYI : "
sha256sum $file
fi
file=./release/${specterimg_filename}-${version}.dmg
if [[ -f $file ]]; then
echo -n " FIY : "
sha256sum $file
fi
fi
if [ "$app_name" != "specter" ]; then
# "early" exit
if [[ "$upload" = "True" ]]; then
echo "no upload for app_name $app_name"
exit 1
fi
exit
fi
if [[ "$upload" = "True" ]]; then
echo " --> gpg-signing the hashes and uploading"
. ../../specter_gh_upload.sh # A simple file looks like: export GH_BIN_UPLOAD_PW=...(GH token)
export CI_COMMIT_TAG=$version
if [[ -z "$CI_PROJECT_ROOT_NAMESPACE" ]]; then
echo "WARNING: Why is CI_PROJECT_ROOT_NAMESPACE not set? Setting to cryptoadvance"
export CI_PROJECT_ROOT_NAMESPACE=cryptoadvance
fi
echo " This build: version: $version gh-project: $CI_PROJECT_ROOT_NAMESPACE"
specterd_zip_fqfn=./release/specterd-${version}-osx_${ARCH}.zip
echo " Checking for file $specterd_zip_fqfn"
if [[ -f $specterd_zip_fqfn ]]; then
python3 ./utils/github.py upload $specterd_zip_fqfn
else
echo " WARNING: not uploading as it does not exist: $specterd_zip_fqfn"
fi
specter_dmg_fqfn=./release/Specter-${version}.dmg
echo " Checking for file $specter_dmg_fqfn"
if [[ -f $specter_dmg_fqfn ]]; then
python3 ./utils/github.py upload $specter_dmg_fqfn
else
echo " WARNING: not uploading as it does not exist: $specter_dmg_fqfn"
fi
cd release
# Maybe we have some SHA256SUMS files from other runs lying around. We don't want to shasum them
rm -f SHA256SUMS*
sha256sum * > SHA256SUMS-macos_${ARCH}
python3 ../utils/github.py upload SHA256SUMS-macos_${ARCH}
# The GPG comman below has a timeout. If that's reached, the script will interrupt. So let's make some noise
say "Hello?! Your overlord is speaking! You're now allowed to sign the binary!"
echo "Just in case you missed the timeout, those three last commands are missing:"
echo "cd release"
echo "gpg --detach-sign --armor SHA256SUMS-macos_${ARCH}"
echo "python3 ../utils/github.py upload SHA256SUMS-macos_${ARCH}.asc"
gpg --detach-sign --armor SHA256SUMS-macos_${ARCH}
python3 ../utils/github.py upload SHA256SUMS-macos_${ARCH}.asc
fi

169
utils/build-unix.sh
View file

@ -1,169 +0,0 @@
#!/usr/bin/env bash
set -e
# We start in the directory where this script is located
cd "$( dirname "${BASH_SOURCE[0]}" )/."
source build-common.sh
cd ..
# Now in project-root
function sub_help {
cat << EOF
Building various components of specter-desktop
Usage: $build-unix [options] <subcommand>
Options:
--debug
will set -x
--version v1.2.3-pre4
If you don't set the version, CI_COMMIT_TAG will determine the version
Subcommands:
make-hash
will make the hash for the electron-app. This hash will get checked after download
specterd
will build the pyinstaller's specterd binary (linux only)
electron-linux
will build the linux binary of the electron-app
electron-win
will build the win binary of the electron-app. The specterd.exe will get downloaded from
github.com/\$CI_PROJECT_ROOT_NAMESPACE/specter-desktop...
This need a wine-environment. See the docker-image electron-builder
Example-call:
./build-unix.sh --debug --version v1.7.0-pre1 make-hash specterd electron-linux
EOF
}
function create_release_zip_linux {
echo " --> Making the release-zip"
# consists of specterd and Specter-version.AppImage
mkdir -p release
# first the specterd
cd pyinstaller/dist
cp -r ../../udev ./udev
echo "Don't forget to set up udev rules! Check out udev folder for instructions." > README.md
zip -r ../../release/${specterd_filename}-"$version"-"$(uname -m)"-linux-gnu.zip ${specterd_filename} udev README.md
echo $app_name
# now the AppImage
cd ../electron/dist
cp -r ../../../udev ./udev
echo "Don't forget to set up udev rules! Check out udev folder for instructions." > README.md
tar -czvf ../../../release/${pkg_filename}-"$version"-"$(uname -m)"-linux-gnu.tar.gz ${app_name^}-* udev README.md
cd ../../..
}
function prepare_building_electron_app_win {
cd pyinstaller/dist
echo " --> Downloading the windows-version of specterd for version $version"
wget --progress=dot -e dotbytes=10M https://github.com/${CI_PROJECT_ROOT_NAMESPACE}/specter-desktop/releases/download/${version}/specterd-${version}-win64.zip -O ./specterd.zip
unzip specterd.zip
cd ../electron
rm -rf dist/
cd ../..
}
version=$CI_COMMIT_TAG
echo " --> Assume gitlab-project: ${CI_PROJECT_ROOT_NAMESPACE}"
[ -z "${CI_PROJECT_ROOT_NAMESPACE:+x}" ] && \
echo " Redefining CI_PROJECT_ROOT_NAMESPACE=cryptoadvance " && \
export CI_PROJECT_ROOT_NAMESPACE=cryptoadvance
while [[ $# -gt 0 ]]
do
arg="$1"
case $arg in
"" | "-h" | "--help")
sub_help
exit 0
shift
;;
--debug)
set -x
DEBUG=true
shift
;;
--version)
version=$2
shift
shift
if [ -n "$CI_COMMIT_TAG" ]; then
if [ "$version" != "$CI_COMMIT_TAG" ]; then
echo "ERROR: Cannot set version to something different than CI_COMMIT_TAG env-var if that var is set. "
exit 1
fi
fi
;;
specterd)
build_specterd=True
shift
;;
make-hash)
make_hash=True
shift
;;
electron-linux)
build_electron_linux=True
shift
;;
electron-win)
build_electron_win=True
shift
;;
help)
sub_help
shift
;;
*)
shift
sub_${arg} $@ && ret=0 || ret=$?
if [ "$ret" = 127 ]; then
echo "Error: '$arg' is not a known subcommand." >&2
echo " Run '$progname --help' for a list of known subcommands." >&2
exit 1
else
exit $ret_value
fi
;;
esac
done
if [[ "$version" = "" ]]; then
echo "ERROR: version could not be determined (--version or CI_COMMIT_TAG)"
exit 1
fi
echo " --> This build got triggered for version $version"
# This file gets further packaged up with the pyinstaller and will help specter to figure out which version it's running on
echo $version > pyinstaller/version.txt
configure
if [[ "$build_specterd" = "True" ]]; then
create_virtualenv_for_pyinstaller
build_pypi_pckgs_and_install
install_build_requirements
cleanup
building_app
fi
if [[ "$build_electron_linux" = "True" ]]; then
prepare_npm
make_hash_if_necessary
building_electron_app linux
create_release_zip_linux
fi
if [ "$build_electron_win" = "True" ]; then
prepare_building_electron_app_win
make_hash_if_necessary win
building_electron_app win
cp pyinstaller/electron/dist/Specter\ Setup\ *.exe release/Specter-Setup-$version.exe
fi

16
utils/create-gitlab-cli-cfg.sh
View file

@ -1,16 +0,0 @@
#!/bin/bash
cat > ~/.python-gitlab.cfg << EOF
[global]
default = specterdesktop
ssl_verify = true
timeout = 5
[specterdesktop]
url = https://gitlab.com
#private_token = ${CI_JOB_TOKEN}
job_token =${CI_JOB_TOKEN}
api_version = 4
EOF

532
utils/github.py
View file

@ -1,532 +0,0 @@
""" We assume that this script is running on a gitlab-runner and therefore has some variables set.
Specifically:
CI_PROJECT_ROOT_NAMESPACE=k9ert
CI_COMMIT_TAG=v0.9.6-pre2
"""
import logging
import os
import sys
from pathlib import Path
import requests
import argparse
import collections
import getpass
import json
import logging
import os
import re
import subprocess
import sys
from typing import (
cast,
Any,
Callable,
List,
Optional,
) # noqa: F401 # pylint: disable=unused-import
try:
# Allow an import of this module without `requests` and `yacl` being installed for meta data queries
# (e.g. version information)
import requests
from yacl import setup_colored_stderr_logging
except ImportError:
pass
logging.basicConfig(format="%(levelname)s:%(message)s", level=logging.INFO)
logger = logging.getLogger(__name__)
logger.setLevel(logging.DEBUG)
github_api_root_url = f"https://api.github.com"
github_username = "gitlab_upload_release_binaries"
def main():
if sys.argv[1] != "upload":
# Maybe something more fancy in the future:
logger.error("Command {sys.argv[1]} not found! Only 'upload' right now")
exit(2)
artifact = sys.argv[2]
if not Path(artifact).exists():
logger.error(f"local artifact {artifact} does not exist.")
exit(2)
if not "CI_PROJECT_ROOT_NAMESPACE" in os.environ:
logger.error("CI_PROJECT_ROOT_NAMESPACE not found")
exit(2)
else:
project = f"{os.environ['CI_PROJECT_ROOT_NAMESPACE']}/specter-desktop"
if not "CI_COMMIT_TAG" in os.environ:
logger.error("CI_COMMIT_TAG not found")
exit(2)
else:
tag = os.environ["CI_COMMIT_TAG"]
if len(sys.argv) != 3:
logger.error("argument artifact not found.")
if artifact_exists(project, tag, Path(artifact).name):
logger.info("Github artifact existing. Skipping upload.")
exit(0)
else:
logger.info(f"Github artifact {artifact} does not exist. Let's upload!")
if not "GH_BIN_UPLOAD_PW" in os.environ:
logger.error("GH_BIN_UPLOAD_PW not found.")
else:
password = os.environ["GH_BIN_UPLOAD_PW"]
publish_release_from_tag(
project,
tag,
[artifact],
github_username,
password,
)
def artifact_exists(project, tag, artifact):
artifact_url = f"https://github.com/{project}/releases/download/{tag}/{artifact}"
logger.debug(f"checking for artifact url {artifact_url}")
r = requests.head(artifact_url)
if r.status_code != 302:
return False
else:
return True
__copyright__ = "Copyright © 2019 Forschungszentrum Jülich GmbH. All rights reserved."
__license__ = "MIT"
__version_info__ = (0, 1, 5)
__version__ = ".".join(map(str, __version_info__))
DEFAULT_GITHUB_ROOT = "github.com"
class MissingDependencyError(Exception):
pass
class FileCommandError(Exception):
pass
class InvalidFileCommandOutputError(Exception):
pass
class NoTagsAvailableError(Exception):
pass
class HTTPError(Exception):
pass
class JSONError(Exception):
pass
class InvalidUploadUrlError(Exception):
pass
class InvalidServerNameError(Exception):
pass
class MissingProjectError(Exception):
pass
class MissingTagError(Exception):
pass
class CredentialsReadError(Exception):
pass
class AttributeDict(dict): # type: ignore
def __getattr__(self, attr: str) -> Any:
return self[attr]
def __setattr__(self, attr: str, value: Any) -> None:
self[attr] = value
Release = collections.namedtuple("Release", ["id", "asset_upload_url"])
Asset = collections.namedtuple("Asset", ["id", "name"])
def setup_stderr_logging() -> None:
logging.basicConfig(level=logging.INFO)
setup_colored_stderr_logging(format_string="[%(levelname)s] %(message)s")
def get_mimetype(filepath: str) -> str:
if not os.path.isfile(filepath):
raise FileNotFoundError(
'The file "{}" does not exist or is not a regular file.'.format(filepath)
)
if not os.access(filepath, os.R_OK):
raise PermissionError('The file "{}" is not readable.'.format(filepath))
if os.name == "nt":
try:
import mimetypes
mime_type = mimetypes.types_map[f".{filepath.split('.')[-1]}"]
except ModuleNotFoundError:
raise Exception(
"mimetypes module not found. Do something like pip install mimetypes"
)
else:
try:
file_command_output = subprocess.check_output(
["file", "--mime", filepath], universal_newlines=True
) # type: str
mime_type = file_command_output.split()[1][:-1]
except subprocess.CalledProcessError as e:
raise FileCommandError(
"The `file` command returned with exit code {:d}".format(e.returncode)
)
except IndexError:
raise InvalidFileCommandOutputError(
'The file command output "{}" could not be parsed.'.format(
file_command_output
)
)
return mime_type
def strip_asset_upload_url(asset_upload_url_with_get_params: str) -> str:
match_obj = re.match(r"([^{]+)(?:\{.*\})?", asset_upload_url_with_get_params)
if not match_obj:
raise InvalidUploadUrlError(
'The upload url "{}" is not in the expected format.'.format(
asset_upload_url_with_get_params
)
)
asset_upload_url = match_obj.group(1) # type: str
return asset_upload_url
class GithubConnection:
def __init__(self, project):
self.github_api_root_url = github_api_root_url
self.project = project
self.username = github_username
self.password = os.environ["GH_BIN_UPLOAD_PW"]
def fetch_existing_release(self, tag) -> Optional[Release]:
try:
release_query_url = "{}/repos/{}/releases/tags/{}".format(
self.github_api_root_url, self.project, tag
)
response = requests.get(
release_query_url,
auth=(self.username, self.password),
headers={"Accept": "application/json"},
)
response.raise_for_status()
logger.info(
'Fetched the existing release "%s" in the GitHub repository "%s"',
tag,
self.project,
)
response_json = response.json()
asset_upload_url_with_get_params = response_json["upload_url"]
asset_upload_url = strip_asset_upload_url(asset_upload_url_with_get_params)
release = Release(response_json["id"], asset_upload_url)
return release
except requests.HTTPError as e:
if e.response.status_code == 404:
return None
raise HTTPError(
'Could not fetch the release "{}" due to a severe HTTP error.'.format(
tag
)
)
def list_assets(self, release: Release) -> List[Asset]:
try:
asset_list_url = "{}/repos/{}/releases/{}/assets".format(
self.github_api_root_url, self.project, release.id
)
response = requests.get(asset_list_url, auth=(self.username, self.password))
response.raise_for_status()
assets = [
Asset(asset_dict["id"], asset_dict["name"])
for asset_dict in response.json()
]
return assets
except requests.HTTPError:
raise HTTPError(
'Could not get a list of assets for project "{}".'.format(self.project)
)
except json.decoder.JSONDecodeError:
raise JSONError("Got an invalid json string.")
except KeyError as e:
raise JSONError(
'Got an unexpected json object missing the key "{}".'.format(e.args[0])
)
def download_artifact(self, tag, artifact, target_dir="."):
artifact_url = (
f"https://github.com/{self.project}/releases/download/{tag}/{artifact}"
)
response = requests.get(artifact_url)
# If the HTTP GET request can be served
if response.status_code == 200:
# Write the file contents in the response to a file specified by local_file_path
with open(os.path.join(target_dir, artifact), "wb") as local_file:
for chunk in response.iter_content(chunk_size=128):
local_file.write(chunk)
else:
raise Exception(
f"Status-code {response.status_code} for url {artifact_url}"
)
def publish_release_from_tag(
project: str,
tag: Optional[str],
asset_filepaths: List[str],
username: str,
password: str,
dry_run: bool = False,
) -> None:
if "requests" not in sys.modules:
raise MissingDependencyError(
'The "requests" package is missing. Please install and run again.'
)
def fetch_latest_tag() -> str:
try:
tags_url = "{}/repos/{}/tags".format(github_api_root_url, project)
response = requests.get(
tags_url,
auth=(username, password),
headers={"Accept": "application/json"},
)
response.raise_for_status()
tags = response.json()
if not tags:
raise NoTagsAvailableError(
'The given repository "{}" has no tags yet.'.format(project)
)
latest_tag = tags[0]["name"] # type: str
logger.info(
'Fetched the latest tag "%s" from the GitHub repository "%s"',
latest_tag,
project,
)
return latest_tag
except requests.HTTPError:
raise HTTPError(
'Could not query the latest tag of the repository "{}" due to a http error.'.format(
project
)
)
except (json.decoder.JSONDecodeError, IndexError):
raise JSONError("Got an invalid json string.")
except KeyError as e:
raise JSONError(
'Got an unexpected json object missing the key "{}".'.format(e.args[0])
)
def publish_release(tag: str) -> Release:
def fetch_existing_release() -> Optional[Release]:
try:
release_query_url = "{}/repos/{}/releases/tags/{}".format(
github_api_root_url, project, tag
)
response = requests.get(
release_query_url,
auth=(username, password),
headers={"Accept": "application/json"},
)
response.raise_for_status()
logger.info(
'Fetched the existing release "%s" in the GitHub repository "%s"',
tag,
project,
)
response_json = response.json()
asset_upload_url_with_get_params = response_json["upload_url"]
asset_upload_url = strip_asset_upload_url(
asset_upload_url_with_get_params
)
release = Release(response_json["id"], asset_upload_url)
return release
except requests.HTTPError as e:
if e.response.status_code == 404:
return None
raise HTTPError(
'Could not fetch the release "{}" due to a severe HTTP error.'.format(
tag
)
)
def create_release() -> Release:
try:
release_creation_url = "{}/repos/{}/releases".format(
github_api_root_url, project
)
response = requests.post(
release_creation_url,
auth=(username, password),
json={
"tag_name": tag,
"name": tag,
"body": "",
"draft": False,
"prerelease": False,
},
)
response.raise_for_status()
logger.info(
'Created the release "%s" in the GitHub repository "%s"',
tag,
project,
)
response_json = response.json()
asset_upload_url_with_get_params = response_json["upload_url"]
asset_upload_url = strip_asset_upload_url(
asset_upload_url_with_get_params
)
release = Release(response_json["id"], asset_upload_url)
return release
except requests.HTTPError:
raise HTTPError('Could not create the release "{}".'.format(tag))
except json.decoder.JSONDecodeError:
raise JSONError("Got an invalid json string.")
except KeyError as e:
raise JSONError(
'Got an unexpected json object missing the key "{}".'.format(
e.args[0]
)
)
release = fetch_existing_release()
if release is None:
release = create_release()
return release
def list_assets(release: Release) -> List[Asset]:
try:
asset_list_url = "{}/repos/{}/releases/{}/assets".format(
github_api_root_url, project, release.id
)
response = requests.get(asset_list_url, auth=(username, password))
response.raise_for_status()
assets = [
Asset(asset_dict["id"], asset_dict["name"])
for asset_dict in response.json()
]
return assets
except requests.HTTPError:
raise HTTPError(
'Could not get a list of assets for project "{}".'.format(project)
)
except json.decoder.JSONDecodeError:
raise JSONError("Got an invalid json string.")
except KeyError as e:
raise JSONError(
'Got an unexpected json object missing the key "{}".'.format(e.args[0])
)
def delete_asset(asset: Asset) -> None:
try:
asset_delete_url = "{}/repos/{}/releases/assets/{}".format(
github_api_root_url, project, asset.id
)
response = requests.delete(asset_delete_url, auth=(username, password))
response.raise_for_status()
logger.info(
'Deleted the asset "%s" attached to release "%s" of the GitHub repository "%s"',
asset.name,
tag,
project,
)
except requests.HTTPError:
raise HTTPError(
'Could not get a list of assets for project "{}".'.format(project)
)
except json.decoder.JSONDecodeError:
raise JSONError("Got an invalid json string.")
except KeyError as e:
raise JSONError(
'Got an unexpected json object missing the key "{}".'.format(e.args[0])
)
def upload_asset(release: Release, asset_filepath: str) -> None:
asset_filename = os.path.basename(asset_filepath)
try:
asset_mimetype = get_mimetype(asset_filepath)
with open(asset_filepath, "rb") as f:
response = requests.post(
"{}?name={}".format(release.asset_upload_url, asset_filename),
auth=(username, password),
data=f,
headers={"Content-Type": asset_mimetype},
)
response.raise_for_status()
logger.info(
'Uploaded the asset "%s" attached to release "%s" of the GitHub repository "%s"',
asset_filename,
tag,
project,
)
except requests.HTTPError:
raise HTTPError('Could not upload the asset "{}".'.format(asset_filename))
if tag is None:
logger.info(
'No tag given, fetching the latest tag from the GitHub repository "%s"',
project,
)
tag = fetch_latest_tag()
if dry_run:
logger.info(
'Would create the release "%s" in the GitHub repository "%s"', tag, project
)
assets = [] # type: List[Asset]
else:
release = publish_release(tag)
assets = list_assets(release)
for asset_filepath in asset_filepaths:
asset_matches = [
asset for asset in assets if asset.name == os.path.basename(asset_filepath)
]
if dry_run:
for asset_match in asset_matches:
logger.info(
'Would delete the asset "%s" attached to release "%s" of the GitHub repository "%s"',
asset_match.name,
tag,
project,
)
logger.info(
'Would upload the asset "%s" attached to release "%s" of the GitHub repository "%s"',
os.path.basename(asset_filepath),
tag,
project,
)
else:
for asset_match in asset_matches:
delete_asset(asset_match)
upload_asset(release, asset_filepath)
if __name__ == "__main__":
main()

49
utils/prepare_for_git_write.sh
View file

@ -1,49 +0,0 @@
#!/bin/bash
# This script prepares the shell so that it can do git-pushes
# It's using the first param as the secret key and the env-var
# KNOWN_HOSTS.
## Install ssh-agent if not already installed, it is required by Docker.
## (change apt-get to yum if you use an RPM-based image)
##
which ssh-agent || ( apk update && apk add --no-cache bash git openssh )
docker info
##
## Run ssh-agent (inside the build environment)
##
eval $(ssh-agent -s)
##
## Add the SSH key stored in SSH_PRIVATE_KEY variable to the agent store
## We're using tr to fix line endings which makes ed25519 keys work
## without extra base64 encoding.
## https://gitlab.com/gitlab-examples/ssh-private-key/issues/1#note_48526556
##
echo "$1" | tr -d '\r' | ssh-add - > /dev/null
##
## Create the SSH directory and give it the right permissions
##
mkdir -p ~/.ssh
chmod 700 ~/.ssh
# Check if Git user email is not set
if [ -z "$(git config --global --get user.email)" ]; then
git config --global user.email "specter@secretvalues"
fi
# Check if Git user name is not set
if [ -z "$(git config --global --get user.name)" ]; then
git config --global user.name "specter"
fi
# Check if KNOWN_HOSTS is set and not empty
if [ -n "$KNOWN_HOSTS" ]; then
# Add KNOWN_HOSTS to known_hosts file
echo "$KNOWN_HOSTS" > ~/.ssh/known_hosts
# Ensure the file permissions are correct
chmod 644 ~/.ssh/known_hosts
fi

232
utils/release.sh
View file

@ -1,232 +0,0 @@
#!/bin/bash
# Replacing MacOS utilities with GNU core utilities to make script more robust
# See: https://apple.stackexchange.com/questions/69223/how-to-replace-mac-os-x-utilities-with-gnu-core-utilities
if [[ "$OSTYPE" == "darwin"* ]]; then
brew ls --versions coreutils > /dev/null;
exitCode=$?
if [[ $exitCode == 0 ]]; then
echo "Coreutils are installed via Homebrew, prepending PATH to use GNU core utilities over MacOS utilities."
export PATH="/usr/local/opt/coreutils/libexec/gnubin:$PATH"
else
echo "GNU core utilities not installed. Run brew install coreutils"
fi
fi
ask_yn() {
while true; do
read -p "Is this correct [y/n]" yn
case $yn in
[Yy]* ) return 0 ;;
[Nn]* ) return 1;;
* ) echo "Please answer yes or no.";;
esac
done
}
while [[ $# -gt 0 ]]
do
key="$1"
command="main"
case $key in
wait_on_master)
command=wait_on_master
shift
;;
--help)
help
shift
exit 0
;;
--release-notes)
RELEASE_NOTES="yes"
shift # past value
;;
--dev)
DEV="yes"
shift
;;
--new-version)
new_version=$2
shift
shift
;;
--tag)
TAG="yes"
shift
;;
--debug)
set -x
shift # past argument
;;
*) # unknown option
POSITIONAL="$1" # save it in an array for later
shift # past argument
;;
esac
done
function help() {
# echo HERE_DOC
# ...
echo "not yet implemented"
}
function main() {
# Sed is used as there can be whitespaces
if ! [ "$(git remote -v | grep upstream | grep 'git@github.com:cryptoadvance/specter-desktop.git' | wc -l | sed -e 's/[[:space:]]*//')" = "2" ]; then
echo " --> You don't have the correct upstream-remote. You need this to release. Please do this:"
echo "git remote add upstream git@github.com:cryptoadvance/specter-desktop.git "
exit 2
fi
if ! [ "$(git remote -v | grep origin | grep 'git@github.com:' | wc -l | sed -e 's/[[:space:]]*//')" = "2" ]; then
echo " --> You don't have a reasonable origin-remote. You need this to release (especially with --dev). Please add one!"
exit 2
fi
current_branch=$(git rev-parse --abbrev-ref HEAD)
if [ "$current_branch" != "master" ]; then
echo "You're currently not on the master-branch, exiting"
exit 2
fi
echo " --> Fetching all tags ..."
git fetch upstream --tags
echo " --> git pull upstream master"
git pull upstream master
if [[ -z "$new_version" ]]; then
echo "What should be the new version? Type in please (e.g. v0.9.3 ):"
read new_version
fi
if ! [[ $new_version =~ ^v([0-9]+)\.([0-9]+)\.([0-9]+)(-([0-9A-Za-z-]+))?$ ]]; then
echo "version $new_version Does not match the pattern!"
exit 1;
fi
if [[ -n "$RELEASE_NOTES" ]]; then
if [ -z $GH_TOKEN ]; then
echo "Your github-token is missing. Please export them like:"
echo "export GH_TOKEN="
exit 2
fi
latest_version=$(git tag -l "v*" | grep -v 'pre' | grep -v 'dev' | sort -V | tail -1)
echo " --> The latest version is $latest_version. "
if ! ask_yn ; then
echo "Ok, then you type in the latest_version:"
read latest_version
if ! [[ $new_version =~ ^v([0-9]+)\.([0-9]+)\.([0-9]+)(-([0-9A-Za-z-]+))?$ ]]; then
echo "Does not match the pattern!"
exit 1;
fi
fi
echo "Here are the release-notes:"
echo "--------------------------------------------------"
echo "# Release Notes" > docs/new_release_notes.md
echo "" >> docs/new_release_notes.md
echo "## ${new_version} $(date +'%B %d, %Y')" >> docs/new_release_notes.md
docker run registry.gitlab.com/cryptoadvance/specter-desktop/github-changelog:latest --github-token $GH_TOKEN --branch master cryptoadvance specter-desktop $latest_version | sort >> docs/new_release_notes.md
echo "" >> docs/new_release_notes.md
cat docs/new_release_notes.md
echo "--------------------------------------------------"
cp docs/release-notes.md docs/release-notes.md.orig
sed -i -e '1,2d' docs/release-notes.md.orig # Assuming the release-Notes start with # Release Notes\n
cat docs/new_release_notes.md docs/release-notes.md.orig > docs/release-notes.md
rm docs/release-notes.md.orig docs/new_release_notes.md
echo "Please check your new File and modify as you find approriate!"
echo "We're waiting here ..."
echo " --> Should we create a PR-branch now? "
if ! ask_yn ; then
echo "break"
#git checkout docs/release-notes.md
exit 2
fi
echo " --> Creating branch ${new_version}_release_notes "
git checkout -b ${new_version}_release_notes
git add docs/release-notes.md
git commit -m "adding release_notes for $new_version"
git push --set-upstream origin ${new_version}_release_notes
echo "Now go ahead and make your PR:"
echo "https://github.com/cryptoadvance/specter-desktop/pulls"
exit 0
fi
if [[ -n "$TAG" ]]; then
echo " --> Should i now create the tag and push the version $new_version ?"
if [ -z $DEV ]; then
echo " --> This will push to your origin-remote!"
else
echo " --> THIS WILL PUSH TO THE UPSTREAM-REMOTE!"
if ! ask_yn ; then
echo "break"
exit 2
fi
fi
git tag $new_version
if [ -z $DEV ]; then
git push origin $new_version
else
git push upstream $new_version
fi
fi
}
function wait_on_master() {
echo "# check status of masterbranch ..."
i=0
# First, wait on the check-runs to be completed:
for i in {1..5} ; do
current_state=$(curl -s https://api.github.com/repos/cryptoadvance/specter-desktop/commits/master/check-runs)
different_states=$(echo $current_state | jq -r '.check_runs[] | select(.status == "completed") | .status' | uniq | wc -l)
status=$(echo $current_state | jq -r '.check_runs[] | select(.status == "completed") | .status' | uniq)
if [[ "$different_states" == 1 ]] && [[ "$status" == "completed" ]] ; then
break
fi
echo "# Builds still running. Will check again in 5 seconds."
sleep 5
done
# Now check all the runs and make sure there are all green:
current_state=$(curl -s https://api.github.com/repos/cryptoadvance/specter-desktop/commits/master/check-runs)
different_conclusions=$(echo $current_state | jq -r '.check_runs[] | select(.conclusion == "success") | .conclusion' | uniq | wc -l)
conclusion=$(echo $current_state | jq -r '.check_runs[] | select(.conclusion == "success") | .conclusion' | uniq)
# We only have one conclusion over all runs:
if [ $different_conclusions -gt 1 ] ; then
echo "# different_conclusions: $different_conclusions"
echo "# Seems that master is not green. Exiting 1"
exit 1
fi
# ... and that conclusion is "success"
if [[ "$conclusion" == "success" ]]; then
echo "# Great, conclusion is success! Exiting 0"
exit 0
fi
echo "# ERROR: I'm confused. This should not happened, exiting 99"
echo "# conclusion = $conclusion"
#echo $current_state
exit 99
}
$command

571
utils/release_helper.py
View file

@ -1,571 +0,0 @@
import hashlib
import logging
import os
import shutil
import subprocess
import sys
import zipfile
from pathlib import Path
from glob import glob
logging.basicConfig(format="%(levelname)s:%(message)s", level=logging.INFO)
logger = logging.getLogger(__name__)
logger.setLevel(logging.DEBUG)
import os
import subprocess
import hashlib
import logging
import gitlab
logger = logging.getLogger(__name__)
class Sha256sumFile:
"""
A class that provides functionality to manage SHA256 checksums for files within a
specified directory.
Attributes:
name (str): The name of the file that contains the SHA256 checksums for other files.
target_dir (str): The path to the directory where the checksum file and other related
files are stored or will be downloaded to. Defaults to `./signing_dir`.
hashed_files (dict): A dictionary storing file names as keys and their corresponding
SHA256 hashes as values.
The `hashed_files` dictionary data structure is used to map each file name (a string)
to its SHA256 hash (also a string). The SHA256 hash is computed for each corresponding
file present in the `target_dir`, allowing for verification of file integrity by comparing
computed hashes against stored hashes.
"""
def __init__(self, name, target_dir="./signing_dir"):
"""
Initializes a Sha256sumFile instance with the provided checksum file name and target directory.
Parameters:
name (str): The name of the checksum file.
target_dir (str): The directory path where the checksum file and other files are present or downloaded.
"""
self.name = name
self.target_dir = target_dir
self.hashed_files = {}
def is_in_target_dir(self):
"""
Checks if the checksum file is present in the target directory.
Returns:
bool: True if the checksum file is present; False otherwise.
"""
return os.path.isfile(os.path.join(self.target_dir, self.name))
def download_from_tag(self, tag, gc):
"""
Downloads the checksum file and its signature from a specific tag using a given client (gc).
Parameters:
tag (str): The tag associated with the artifacts to be downloaded.
gc (object): The client object which provides the `download_artifact` method for downloading.
"""
gc.download_artifact(tag, self.name, target_dir=self.target_dir)
gc.download_artifact(tag, self.name + ".asc", target_dir=self.target_dir)
self.read()
def download_hashed_files(self, tag, gc):
"""
Downloads all files listed in the hashed_files dictionary from a specific tag using a given client (gc).
Parameters:
tag (str): The tag associated with the artifacts to be downloaded.
gc (object): The client object which provides the `download_artifact` method for downloading.
"""
for file in self.hashed_files.keys():
logger.info(f"Downloading {file} from {tag}")
gc.download_artifact(tag, file, target_dir=self.target_dir)
def read(self):
"""
Reads the checksum file and populates the hashed_files dictionary with file names and their corresponding hashes.
"""
with open(os.path.join(self.target_dir, self.name), "r") as file:
line = file.readline()
while line:
line = line.split(maxsplit=2)
self.hashed_files[line[1]] = line[0]
line = file.readline()
def print(self):
"""
Prints each file's hash and name from the hashed_files dictionary to the standard output.
"""
for hashed_file, hash in self.hashed_files.items():
print(f"{hash} {hashed_file}")
def write(self):
"""
Writes the hashed_files dictionary entries to the checksum file in the target directory.
"""
with open(os.path.join(self.target_dir, self.name), "w") as file:
for hashed_file, hash in self.hashed_files.items():
file.write(f"{hash} {hashed_file}\n")
def add_file(self, file):
"""
Computes the SHA256 hash for a given file and adds the file and its hash to the hashed_files dictionary.
Parameters:
file (str): The filename for which the SHA256 hash should be computed and added.
"""
self.hashed_files[file] = Sha256sumFile.sha256_checksum(file, self.target_dir)
def check_hashes(self):
"""
Verifies the integrity of the files by checking their SHA256 hashes against the entries in the checksum file.
Raises:
Exception: If the verification of any file fails, an exception is raised with
the subprocess output that caused the failure.
"""
try:
subprocess.run(
["sha256sum", "-c", self.name], cwd=self.target_dir, check=True
)
except subprocess.CalledProcessError as e:
raise Exception(
f"Could not validate hashes for file {self.name}: {e.output}"
)
def check_sig(self):
"""
Verifies the signature of the checksum file using gpg.
Raises:
Exception: If the verification of the file signature fails, an exception is raised.
"""
returncode = subprocess.call(
["gpg", "--verify", self.name + ".asc"], cwd=self.target_dir
)
if returncode != 0:
raise Exception(f"Could not validate signature of file {self.name}")
@classmethod
def sha256_checksum(cls, filename, folder, block_size=65536):
"""
Computes the SHA256 hash of a given file.
Parameters:
filename (str): The name of the file for which to compute the SHA256 hash.
folder (str): The path to the directory containing the file.
block_size (int): The block size used for reading the file. Defaults to 65536.
Returns:
str: The SHA256 hash of the file.
"""
sha256 = hashlib.sha256()
with open(os.path.join(folder, filename), "rb") as f:
for block in iter(lambda: f.read(block_size), b""):
sha256.update(block)
return sha256.hexdigest()
class ReleaseHelper:
"""
A class that manages software build artifacts for a CI/CD pipeline.
This class is designed to perform operations such as downloading artifacts from CI
pipelines, verifying SHA256 checksums, verifying GPG signatures, and uploading
artifacts to GitHub releases.
The class relies on a number of environment variables being present:
CI_COMMIT_TAG: The git tag to work with (format: export CI_COMMIT_TAG=<tag_name>).
CI_PIPELINE_ID: The pipeline ID for which artifacts are managed
(format: export CI_PIPELINE_ID=<pipeline_id>).
CI_PROJECT_ROOT_NAMESPACE: The root namespace of the CI project
(required for uploading to GitHub).
GH_BIN_UPLOAD_PW: gh_token or token for GitHub to authenticate uploads.
Attributes:
target_dir (str): The directory path where artifacts are to be managed.
tag (str): The git tag associated with the artifacts being managed.
pipeline_id (str): The CI pipeline ID for artifact management operations.
pipeline (Pipeline): A pipeline object fetched from the CI server.
github_project (str): The GitHub repository in which the release should be created or updated.
gh_token (str): The password or token used to authenticate with GitHub.
Methods:
download_and_unpack_all_artifacts(): Downloads and unpacks artifacts from a CI pipeline.
download_and_unpack_new_artifacts_from_github():
Downloads and unpacks new artifacts from GitHub.
create_sha256sum_file(): Creates a SHA256SUMS file with checksums of all artifacts.
check_all_hashes(): Verifies checksums for all artifacts.
check_all_sigs(): Verifies GPG signatures for all artifacts.
calculate_publish_params(): Calculates and validates necessary parameters for publishing.
upload_sha256sum_file(): Uploads the SHA256SUMS file to a GitHub release.
upload_sha256sumsig_file(): Uploads the SHA256SUMS.asc signature file to a GitHub release.
Note: The actual implementation of the methods and the use of additional classes
like `github.GithubConnection` or `Sha256sumFile` are assumed to exist and are
not defined in this documentation.
"""
def __init__(self):
self.target_dir = "signing_dir"
Path(self.target_dir).mkdir(parents=True, exist_ok=True)
@property
def gl(self):
"""https://python-gitlab.readthedocs.io/en/stable/api-usage.html"""
if hasattr(self, "_gl"):
return self._gl
if os.environ.get("GITLAB_PRIVATE_TOKEN"):
logger.info("Using GITLAB_PRIVATE_TOKEN")
self._gl = gitlab.Gitlab(
"http://gitlab.com",
private_token=os.environ.get("GITLAB_PRIVATE_TOKEN"),
)
elif os.environ.get("CI_JOB_TOKEN"):
logger.info("Using CI_JOB_TOKEN")
self._gl = gitlab.Gitlab(
"http://gitlab.com", job_token=os.environ["CI_JOB_TOKEN"]
)
else:
raise Exception(
"Can't authenticate against Gitlab ( export GITLAB_PRIVATE_TOKEN )"
)
return self._gl
@property
def gitlab_project(self):
if hasattr(self, "_gitlab_project"):
return self._gitlab_project
try:
from gitlab.v4.objects import Project
self._gitlab_project: Project = self.gl.projects.get(self.ci_project_id)
except gitlab.exceptions.GitlabAuthenticationError as e:
logger.fatal(e)
logger.error("Your token might be expired or wrong. Get a new one here:")
logger.error(" https://gitlab.com/-/profile/personal_access_tokens")
exit(2)
if (
self._gitlab_project.attributes["namespace"]["path"]
!= self.ci_project_root_namespace
):
logger.fatal(
f"project_root_namespace ({ self.ci_project_root_namespace }) does not match namespace of Project ({self._gitlab_project.attributes['namespace']['path']}) "
)
logger.error("You might want to: unset CI_PROJECT_ID")
exit(2)
return self._gitlab_project
@property
def ci_project_id(self):
if hasattr(self, "_ci_project_id"):
return self._ci_project_id
if os.environ.get("CI_PROJECT_ID"):
self._ci_project_id = os.environ.get("CI_PROJECT_ID")
logger.info(f"Using ci_project_id: {self._ci_project_id} ")
else:
logger.error("No Project given. choose one:")
for project in self.gl.projects.list(search="specter-desktop"):
logger.info(
f" export CI_PROJECT_ID={project.id} # {project.name_with_namespace}"
)
if project.name_with_namespace.startswith("cryptoadvance"):
self._ci_project_id = project.id
logger.warning(
f"{self._ci_project_id} has been chosen as self._ci_project_id"
)
return self._ci_project_id
@property
def github_project(self):
if hasattr(self, "_github_project"):
return self._github_project
self._github_project = f"{self.ci_project_root_namespace}/specter-desktop"
logger.info(f"Using github_project: {self._github_project}")
return self._github_project
@property
def gh_token(self):
if hasattr(self, "_gh_token"):
return self._gh_token
if os.environ.get("GH_BIN_UPLOAD_PW"):
self._gh_token = os.environ.get("GH_BIN_UPLOAD_PW")
logger.info(f"Using gh_token: REDACTED")
else:
raise Exception(
"no Github token given ( export GH_BIN_UPLOAD_PW=v0.0.0.0-pre13 )"
)
return self._gh_token
@property
def ci_commit_tag(self):
if hasattr(self, "_ci_commit_tag"):
return self._ci_commit_tag
if os.environ.get("CI_COMMIT_TAG"):
self._ci_commit_tag = os.environ.get("CI_COMMIT_TAG")
else:
raise Exception("no tag given ( export CI_COMMIT_TAG=v0.0.0.0-pre13 )")
logger.info(f"Using tag: {self._ci_commit_tag}")
return self._ci_commit_tag
@property
def ci_project_root_namespace(self):
if hasattr(self, "_ci_project_root_namespace"):
return self._ci_project_root_namespace
if os.environ.get("CI_PROJECT_ROOT_NAMESPACE"):
self._ci_project_root_namespace = os.environ.get(
"CI_PROJECT_ROOT_NAMESPACE"
)
logger.info(
f"Using project_root_namespace: {self._ci_project_root_namespace} ( export CI_PROJECT_ROOT_NAMESPACE={self._ci_project_root_namespace} )"
)
else:
self._ci_project_root_namespace = "cryptoadvance"
logger.warning(
f"Using project_root_namespace: {self._ci_project_root_namespace} ( export CI_PROJECT_ROOT_NAMESPACE={self._ci_project_root_namespace} )"
)
return self._ci_project_root_namespace
@property
def ci_pipeline_id(self):
if hasattr(self, "_ci_pipeline_id"):
return self._ci_pipeline_id
if os.environ.get("CI_PIPELINE_ID"):
self._ci_pipeline_id = os.environ.get("CI_PIPELINE_ID")
else:
logger.info(
"no CI_PIPELINE_ID given, trying to find an appropriate one ..."
)
pipelines = self.gitlab_project.pipelines.list()
for pipeline in pipelines:
if pipeline.ref == self.ci_commit_tag:
self._ci_pipeline_id = pipeline.id
self._ci_pipeline = pipeline
logger.info(f"Found matching pipeline: {pipeline}")
if not hasattr(self, "_ci_pipeline"):
logger.error(
f"Could not find tag {self.ci_commit_tag} in the pipeline-refs:"
)
for pipeline in self.gitlab_project.pipelines.list():
logger.error(pipeline.ref)
raise Exception(
"no CI_PIPELINE_ID given ( export CI_PIPELINE_ID= ) or maybe you're on the wrong project ( export CI_PROJECT_ROOT_NAMESPACE= )"
)
logger.info(f"Using pipeline_id: {self.ci_pipeline.id}")
return self._ci_pipeline_id
@property
def ci_pipeline(self):
if hasattr(self, "_ci_pipeline"):
return self._ci_pipeline
self._ci_pipeline = self.gitlab_project.pipelines.get(self.ci_pipeline_id)
return self._ci_pipeline
def download_and_unpack_all_artifacts(self):
if os.path.isdir(self.target_dir):
logger.info(f"First purging {self.target_dir}")
shutil.rmtree(self.target_dir)
for job in self.ci_pipeline.jobs.list():
if job.name in [
"release_electron_linux_windows",
"release_binary_windows",
"release_pip",
]:
zipfn = f"/tmp/_artifacts_{job.name}.zip"
job_obj = self.gitlab_project.jobs.get(job.id, lazy=True)
if not os.path.isfile(zipfn):
logger.info(f"Downloading artifacts for {job.name}")
with open(zipfn, "wb") as f:
job_obj.artifacts(streamed=True, action=f.write)
else:
logger.info(f"Skipping Download artifacts for {job.name}")
logger.info(f"Unzipping {zipfn} in target-folder")
with zipfile.ZipFile(zipfn, "r") as zip:
for zip_info in zip.infolist():
if zip_info.filename[-1] == "/":
continue
zip_info.filename = os.path.basename(zip_info.filename)
logger.info(f" Extracting {zip_info.filename}")
zip.extract(zip_info, self.target_dir)
def download_and_unpack_new_artifacts_from_github(self):
gc = github.GithubConnection(self.github_project)
release = gc.fetch_existing_release(self.ci_commit_tag)
assets = gc.list_assets(release)
for asset in assets:
if not asset.name.startswith("SHA256"):
continue
if asset.name == "SHA256SUMS" or asset.name == "SHA256SUMS.asc":
continue
if asset.name.endswith(".asc"):
continue
logger.info("iterating file " + asset.name)
shasumfile = Sha256sumFile(asset.name)
if not shasumfile.is_in_target_dir():
shasumfile.download_from_tag(self.ci_commit_tag, gc)
shasumfile.download_hashed_files(self.ci_commit_tag, gc)
shasumfile.check_hashes()
shasumfile.check_sig()
logger.info("All files have valid signatures")
def create_sha256sum_file(self):
with open(f"{self.target_dir}/SHA256SUMS", "w") as shafile:
for file in os.listdir(self.target_dir):
if file.startswith("SHA256SUMS-") and not file.endswith(".asc"):
logger.debug(f"Processing {file}")
sha_src = Sha256sumFile(file)
sha_src.read()
for hashed_file in sha_src.hashed_files.keys():
print(f"{sha_src.hashed_files[hashed_file]} {hashed_file}\n")
shafile.write(
f"{sha_src.hashed_files[hashed_file]} {hashed_file}\n"
)
returncode = subprocess.call(
["sha256sum", "-c", "SHA256SUMS"], cwd=self.target_dir
)
if returncode != 0:
raise Exception(
f"One of the hashes is not matching: {subprocess.run(['sha256sum', '-c', 'SHA256SUMS'], cwd=self.target_dir)}"
)
def check_all_hashes(self):
for file in os.listdir(self.target_dir):
if file.startswith("SHA256SUM") and not file.endswith(".asc"):
logger.info(f"Checking hashes in {file}")
if file.endswith("windows"):
logger.info(f"Converting dos2unix for {file}")
dos2unix(os.path.join("signing_dir", file))
returncode = subprocess.call(
["sha256sum", "-c", file], cwd=self.target_dir
)
if returncode != 0:
raise Exception(f"Could not validate hashes for file {file}")
logger.info("All files SHA256SUM* (not .asc) has valid hashes")
def check_all_sigs(self):
for file in os.listdir(self.target_dir):
if file.endswith(".asc"):
logger.info(f"Checking signature for {file}")
returncode = subprocess.call(
["gpg", "--verify", file], cwd=self.target_dir
)
if returncode != 0:
raise Exception(
f"Could not validate signature of file {file}: {subprocess.run(['gpg', '--verify', file], cwd=self.target_dir)}"
)
logger.info("All files *.asc has valid signatures")
def upload_sha256sum_file(self):
artifact = os.path.join("signing_dir", "SHA256SUMS")
if github.artifact_exists(
self.github_project, self.ci_commit_tag, Path(artifact).name
):
logger.info(f"Github artifact {artifact} existing. Skipping upload.")
exit(0)
else:
logger.info(f"Github artifact {artifact} does not exist. Let's upload!")
github.publish_release_from_tag(
self.github_project,
self.ci_commit_tag,
[artifact],
"gitlab_upload_release_binaries",
self.gh_token,
)
def upload_sha256sumsig_file(self):
artifact = os.path.join("signing_dir", "SHA256SUMS.asc")
if github.artifact_exists(
self.github_project, self.ci_commit_tag, Path(artifact).name
):
logger.info(f"Github artifact {artifact} existing. Skipping upload.")
exit(0)
else:
logger.info(f"Github artifact {artifact} does not exist. Let's upload!")
github.publish_release_from_tag(
self.github_project,
self.ci_commit_tag,
[artifact],
"gitlab_upload_release_binaries",
self.gh_token,
)
def dos2unix(filename):
content = ""
outsize = 0
with open(filename, "rb") as infile:
content = infile.read()
with open(filename, "wb") as output:
for line in content.splitlines():
outsize += len(line) + 1
output.write(line + b"\n")
def sha256sum(filenames):
sha_file = Sha256sumFile("SHA256SUMS", target_dir=".")
for filename in filenames:
logger.info(f"Adding {filename}")
sha_file.add_file(filename)
sha_file.print()
if __name__ == "__main__":
if "sha256sums" in sys.argv:
# Used by build-win.ci.bat
sha256sum(sys.argv[2:])
exit(0)
if "install_wheel" in sys.argv:
# List all .whl files in the 'dist' directory
wheel_files = glob(
str(Path("dist", "cryptoadvance.specter-*-py3-none-any.whl"))
)
print("found those wheel files: " + str(wheel_files))
# Loop through the wheel files and install them
for wheel_file in wheel_files:
cmd = f"pip3 install {wheel_file}"
res = os.system(cmd)
print(f"Result of command: {cmd}")
print(res)
# If the installation fails, exit with the error code
if res != 0:
exit(res)
# Exit with a success code if all installations were successful
exit(0)
rh = ReleaseHelper()
try:
from utils import github
except Exception as e:
logger.fatal(e)
logger.error("You might have called this script wrong. Execute it like:")
logger.error("python3 -m utils.release_helper ...")
if "download" in sys.argv:
rh.download_and_unpack_all_artifacts()
if "downloadgithub" in sys.argv:
rh.download_and_unpack_new_artifacts_from_github()
if "checkhashes" in sys.argv:
rh.check_all_hashes()
if "checksigs" in sys.argv:
rh.check_all_sigs()
if "create" in sys.argv:
rh.create_sha256sum_file()
if "upload_shasums" in sys.argv:
rh.upload_sha256sum_file()
if "upload_shasumssig" in sys.argv:
rh.upload_sha256sumsig_file()

13
utils/tag_specterext_dummy.sh
View file

@ -1,13 +0,0 @@
#!/bin/bash
set -e
echo "This script will checkout github.com:${CI_PROJECT_ROOT_NAMESPACE}/specterext-dummy.git"
echo "and tag it with ${CI_COMMIT_TAG}"
git clone git@github.com:${CI_PROJECT_ROOT_NAMESPACE}/specterext-dummy.git
cd specterext-dummy
git checkout master
git tag ${CI_COMMIT_TAG}
#git push origin ${CI_COMMIT_TAG}

11
utils/trigger_docker_build.sh
View file

@ -1,11 +0,0 @@
#!/bin/bash
payload="{\"ref\":\"master\", \"inputs\": {\"tag\": \"${CI_COMMIT_TAG}\"}}"
echo $payload
# This Token is controlled by https://github.com/AaronDewes
curl -X POST -H "Accept: application/vnd.github.v3+json" -H "Authorization: token ${AARON_TOKEN}" \
https://api.github.com/repos/lncm/docker-specter-desktop/actions/workflows/dispatch.yml/dispatches -d @<(cat <<EOF
$payload
EOF
)