diff --git a/.cirrus.yml b/.cirrus.yml deleted file mode 100644 index f0cdef991..000000000 --- a/.cirrus.yml +++ /dev/null @@ -1,125 +0,0 @@ -container: -# image: python:slim -# image: ubuntu:jammy -# image: python:3.10 - image: ghcr.io/cryptoadvance/specter-desktop/cirrus-jammy:20260412 - -# We assume here that we're having a proper python3 system including virtualenv and pip -prep_stuff_template: &PREP_STUFF_TEMPLATE - bitcoind_installation_cache: - folder: ./tests/bitcoin - fingerprint_script: - - cat pyproject.toml | grep "addopts = " | cut -d'=' -f2 | sed 's/--/+/g' | tr '+' '\n' | grep bitcoin | cut -d' ' -f2 - - cat tests/bitcoin_gitrev_pinned 2> /dev/null || true - - cat /etc/os-release | grep VERSION - - cat ./tests/install_noded.sh - - echo "binary" # if the next line is --bitcoin binary, otherwise use echo "compile" - this ensures different caching keys. - populate_script: ./tests/install_noded.sh --debug --bitcoin binary - elementsd_installation_cache: - folder: ./tests/elements - fingerprint_script: - - cat pyproject.toml | grep "addopts = " | cut -d'=' -f2 | sed 's/--/+/g' | tr '+' '\n' | grep elements | cut -d' ' -f2 - - cat tests/elements_gitrev_pinned 2> /dev/null || true - - cat /etc/os-release | grep VERSION - - cat ./tests/install_noded.sh - - echo "binary" # if the next line is --elements binary, otherwise use echo "compile" - this ensures different caching keys. - populate_script: ./tests/install_noded.sh --debug --elements binary - verify_script: - - echo " --> Version of python, virtualenv and pip3" - - python3 --version && virtualenv --version && pip3 --version - - echo " --> Executables in tests/elements/src" - - find tests/elements/src -maxdepth 1 -type f -executable -exec ls -ld {} \; || true - - echo " --> Executables in tests/elements/bin" - - find tests/elements/bin -maxdepth 1 -type f -executable -exec ls -ld {} \; || true - - echo " --> Executables in tests/bitcoin/src" - - find tests/bitcoin/src -maxdepth 1 -type f -executable -exec ls -ld {} \; || true - - echo " --> Executables in tests/bitcoin/bin" - - find tests/bitcoin/bin -maxdepth 1 -type f -executable -exec ls -ld {} \; || true - - echo " --> bitcoind version" - - tests/bitcoin/src/bitcoind -version | head -1 || true - - tests/bitcoin/bin/bitcoind -version | head -1 || true - - echo " --> elements version" - - tests/elements/src/elementsd -version | head -1 || true - - tests/elements/bin/elementsd -version | head -1 || true - - pip_script: - #folder: /tmp/cirrus-ci-build/.env - #fingerprint_script: echo muh && cat requirements.txt - #populate_script: - - virtualenv --python=python .env - - source ./.env/bin/activate - - pip3 install -r requirements.txt --require-hashes && pip3 install -e ".[test]" - install_script: - - source ./.env/bin/activate - - pip3 install -e . - -test_task: - << : *PREP_STUFF_TEMPLATE - skip: "false" - test_script: - - source ./.env/bin/activate - - echo $PATH - # needed so that setuptools_scm has a t least one tag to guess the version properly - # and the tests/test_util_version.py doesn't fail - - git fetch origin refs/tags/v1.0.0 - - pytest --cov=cryptoadvance --junitxml=./testresults.xml - always: - junit_artifacts: - path: "./testresults.xml" - format: junit - - -cypress_test_task: - use_compute_credits: $CIRRUS_USER_COLLABORATOR == 'true' - skip: "false" - persistent_worker: - labels: - os: linux - isolation: - container: - image: ghcr.io/cryptoadvance/specter-desktop/cypress-python-jammy:20260411 - cpu: 6 - memory: 6G - pre_prep_script: - # The stupid old debian-package is not installing a proper binary but just the python-package - - echo -e '#!/bin/bash\npython3 -m virtualenv "$@"' > /usr/local/bin/virtualenv - - chmod +x /usr/local/bin/virtualenv - - virtualenv --version - << : *PREP_STUFF_TEMPLATE - npm_cache: - folder: ./node_modules - fingerprint_script: cat package-lock.json - populate_script: npm ci - cypress_script: - - source ./.env/bin/activate - #- pip3 install -e . - - ./utils/test-cypress.sh --debug run - junit_artifacts: - path: "cypresstest-output.xml" - type: text/xml - format: junit - always: - cypress_screenshots_artifacts: - path: "./cypress/screenshots/**" - cypress_videos_artifacts: - path: "./cypress/videos/**" - -extension_smoketest_task: - << : *PREP_STUFF_TEMPLATE - test_script: - - git config --global user.name "CI CD" - - git config --global user.email "cicd@example.com" - - source ./.env/bin/activate - - echo $PATH - - mkdir tmp && cd tmp - - mkdir testextension && cd testextension - - pwd - - python3 -m cryptoadvance.specter ext gen --ext-id cicdtest --org cryptoadvance --no-isolated-client --devicename cicddevice - - pip3 install -e . - - python3 -m cryptoadvance.specter server --config DevelopmentConfig --debug 2> specter.log & - - sleep 15 - - cat specter.log | grep "Found CicdtestService" # Discovery failed - - cat specter.log | grep "Extension CicdtestService activated (alpha)" - - apt-get update && apt-get -y install curl - - curl http://127.0.0.1:25441/svc/cicdtest/ | grep "CicdtestService 4thewin." - diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml deleted file mode 100644 index fda8705e5..000000000 --- a/.gitlab-ci.yml +++ /dev/null @@ -1,281 +0,0 @@ -# this image contains python, bitcoind and docker -# check docker/python-bitcoind on how it's built -image: registry.gitlab.com/cryptoadvance/specter-desktop/python-bitcoind:v22.0 - -variables: - # Cache documentation: https://docs.gitlab.com/ee/ci/caching/ - PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip" - -cache: - # enable per-job and per-branch caching - - key: - files: - - ./requirements.txt - prefix: "$CI_JOB_NAME" - paths: - - .cache/pip - - .env - -stages: - - testing - - releasing - - post_releasing - -before_script: - - docker info || echo "no docker-command found" # Print out docker version for debugging - - echo CI_PROJECT_NAMESPACE = $CI_PROJECT_NAMESPACE - - echo CI_PROJECT_ROOT_NAMESPACE = $CI_PROJECT_ROOT_NAMESPACE - - python -V # Print out python version for debugging - - apt update - - apt install -y libusb-1.0-0-dev libudev-dev # usb-support in hidapi - # https://github.com/python-babel/babel/issues/990#issuecomment-1760326334 - - rm -f /etc/localtime - - ln -s /usr/share/zoneinfo/Etc/UTC /etc/localtime - # This doesn't get cached in gitlab but we don't need it anyway for now: - # - ./tests/install_noded.sh --debug --elements compile - - pip3 install --upgrade virtualenv - - virtualenv --python=python3 .env - - source .env/bin/activate - -check: - stage: testing - # We simply check here whether all the tests on github are completed and green - script: - - ./utils/release.sh wait_on_master || exit 1 # that command will have a non-0 exit-value if not everything is green - -# jobs with a preceding . like .test are hidden jobs and are not executed. I leave them in here as -# we might want to reactivate them in the case that github explodes or something. -# So effectively, gitlab is currently only used for releasing. - -.test: - stage: testing - # We assume here that people who want to get code into the master-branch are - # relying on PRs and people who are working on gitlab-forks are working - # on CI which probably want fast feedback on the releasing-jobs - # and therefore skip the test-job - # tem deactivated as it did not work as expected - #only: - # - $CI_PROJECT_ROOT_NAMESPACE =~ "cryptoadvance" - script: - - pip3 install -r requirements.txt - - pip3 install -e . - - pip3 install -e ".[test]" - - python3 setup.py install # compiles babel stuff as well (might make pip install obsolete) - - py.test --cov-report term --cov cryptoadvance - -.test-cypress: - image: registry.gitlab.com/cryptoadvance/specter-desktop/cypress-python-jammy:v9.7.0 - stage: testing - script: - # start the server in the background - - pip3 install -e . - - pip3 install -e ".[test]" - - python3 setup.py install # compiles babel stuff as well (might make pip install obsolete) - - npm i - - ./utils/test-cypress.sh --docker --debug run - - docker ps || echo "probably no docker available anyway" - artifacts: - when: always - paths: - - cypress/videos/**/*.mp4 - - cypress/screenshots/**/*.png - expire_in: 1 day - -release_pip: - stage: releasing - only: - - tags - script: - - pip3 install -e . - - pip3 install -e ".[test]" - - pip3 install . - - pip3 install build==0.10.0 twine - - python3 -m build - - ls -l dist - # twine reads the password from the env-var TWINE_PASSWORD - # Either testing it or doing the real thing depending on which gitlab-project we're running: - - if ! [[ ${CI_PROJECT_ROOT_NAMESPACE} = "cryptoadvance" ]]; then python3 -m twine upload --verbose --user __token__ dist/* --repository-url https://test.pypi.org/legacy/ ; fi - - if [[ ${CI_PROJECT_ROOT_NAMESPACE} = "cryptoadvance" ]]; then python3 -m twine upload --verbose --user __token__ dist/* ; fi - - cd dist - - sha256sum cryptoadvance.specter-*.tar.gz > SHA256SUMS-pip - - ../utils/artifact_signer.sh sign --artifact ./SHA256SUMS-pip - - cd .. - - cat ./dist/SHA256SUMS-pip - #- python ./utils/github.py upload ./dist/SHA256SUMS-pip - #- python ./utils/github.py upload ./dist/SHA256SUMS-pip.asc - - python ./utils/github.py upload ./dist/cryptoadvance.specter-*.tar.gz - artifacts: - when: always - paths: - - dist/* - expire_in: 1 day - -release_binary_windows: - stage: releasing - only: - - tags - variables: - GIT_DEPTH: 0 # Disable shallow clone to get all Git history - tags: - - windows - before_script: - - whoami - - python -V - - pip3 --version - - pip install virtualenv - - virtualenv --python=python3 .env - - .\.env\Scripts\activate - - pip3 install -e ".[test]" - - script: - # This script won't execute if the script before that fails - # No need to check the version-scheme again - - - echo "Releasing for ${CI_PROJECT_ROOT_NAMESPACE}" - - .\pyinstaller\build-win-ci.bat $CI_COMMIT_TAG - - python ./utils/github.py upload ./pyinstaller/release/specterd-$CI_COMMIT_TAG-win64.zip - - cd ./pyinstaller/release - - python ..\..\utils\release_helper.py sha256sums specterd-$CI_COMMIT_TAG-win64.zip > SHA256SUMS-windows - - type SHA256SUMS-windows - - echo $GPG_PASSPHRASE | c:\Program` Files` `(x86`)\GnuPg\bin\gpg --detach-sign --armor --no-tty --batch --yes --passphrase-fd 0 --pinentry-mode loopback SHA256SUMS-windows - artifacts: - when: always - paths: - - pyinstaller/release/* - expire_in: 1 day - cache: - key: - files: - - ./pyinstaller/electron/package-lock.json - prefix: $CI_JOB_NAME - paths: - - ./pyinstaller/electron/node_modules - -release_electron_linux_windows: - image: registry.gitlab.com/cryptoadvance/specter-desktop/electron-builder:latest - stage: releasing - only: - - tags - needs: - - release_binary_windows - before_script: - - python3 -V # Print out python version for debugging - - apt update - - apt install -y unzip libusb-1.0-0-dev libudev-dev # usb-support in hidapi - - pip3 install virtualenv - # Only difference to default befor_script: (ToDo fix this) - - python3 -m virtualenv --python=python3 .env - - source .env/bin/activate - # https://github.com/python-babel/babel/issues/990#issuecomment-1760326334 - - rm -f /etc/localtime - - ln -s /usr/share/zoneinfo/Etc/UTC /etc/localtime - - pip3 install -e ".[test]" # TZ=UTC because https://github.com/nektos/act/issues/1853 - script: - - echo "Releasing for ${CI_PROJECT_ROOT_NAMESPACE}" - - export CI_PROJECT_ROOT_NAMESPACE # needed in the build-script to download the right windows-binary - - ./utils/build-unix.sh --version $CI_COMMIT_TAG make-hash specterd electron-linux electron-win - - ls -l release - - cd release - - sha256sum specterd-${CI_COMMIT_TAG}-x86_64-linux-gnu.zip specter_desktop-${CI_COMMIT_TAG}-x86_64-linux-gnu.tar.gz > ./SHA256SUMS-linux - - cat ./SHA256SUMS-linux - - sha256sum Specter-Setup-${CI_COMMIT_TAG}.exe > ./SHA256SUMS-win - - cat ./SHA256SUMS-win - - cd .. - - ./utils/artifact_signer.sh sign --artifact ./release/SHA256SUMS-win - - ./utils/artifact_signer.sh sign --artifact ./release/SHA256SUMS-linux - - python3 ./utils/github.py upload ./release/Specter-Setup-${CI_COMMIT_TAG}.exe - - python3 ./utils/github.py upload ./release/specterd-${CI_COMMIT_TAG}-x86_64-linux-gnu.zip - - python3 ./utils/github.py upload ./release/specter_desktop-${CI_COMMIT_TAG}-x86_64-linux-gnu.tar.gz - #- python3 ../utils/github.py upload ./release/SHA256SUMS-linux - #- python3 ../utils/github.py upload ./release/SHA256SUMS-linux.asc - #- python3 ../utils/github.py upload ./release/SHA256SUMS-win - #- python3 ../utils/github.py upload ./release/SHA256SUMS-win.asc - cache: - key: - files: - - ./pyinstaller/electron/package-lock.json - prefix: $CI_JOB_NAME - paths: - - ./pyinstaller/electron/node_modules - - artifacts: - when: always - paths: - - release/Specter-Setup-${CI_COMMIT_TAG}.exe - - release/specterd-${CI_COMMIT_TAG}-x86_64-linux-gnu.zip - - release/specter_desktop-${CI_COMMIT_TAG}-x86_64-linux-gnu.tar.gz - - release/SHA256SUMS-linux - - release/SHA256SUMS-linux.asc - - release/SHA256SUMS-win - - release/SHA256SUMS-win.asc - expire_in: 1 day - -release_signatures: - stage: post_releasing - only: - - tags - before_script: - - python -V # Print out python version for debugging - - pip3 install --upgrade virtualenv - - virtualenv --python=python3 .env - - source .env/bin/activate - - pip3 install -e ".[test]" - - ./utils/artifact_signer.sh init # prepare .gnupg - script: - - python3 -m utils.release_helper download # downloads the job-artifacts from gitlab - - python3 -m utils.release_helper downloadgithub # downloads additional artifacts from github (if not there and is they have SHA256SUMS-something) - - python3 -m utils.release_helper checksigs # checks the signatures of all SHA256SUMM*.asc files - - python3 -m utils.release_helper checkhashes # checks all SHA256SUM* files (might modify files on the fly due to windows line endings) - - python3 -m utils.release_helper create # creates a SHA256SUM - - ./utils/artifact_signer.sh sign --artifact ./signing_dir/SHA256SUMS # Signs the SHA256SUM - - python3 -m utils.release_helper upload_shasums # uploads SHA256SUMS to github - - python3 -m utils.release_helper upload_shasumssig # uploads SHA256SUMS.asc to github - -release_docker: - stage: post_releasing - only: - - tags - before_script: - - echo "Triggering Docker Release" - script: - - ./utils/trigger_docker_build.sh - -# Tagging the current master-branch of https://github.com/cryptoadvance/specterext-dummy -# with the same CI_COMMIT_TAG -tag_specterext_dummy_repo: - stage: post_releasing - only: - - tags - before_script: - # write access to git@github.com:cryptoadvance/specterext-dummy.git - - source ./utils/prepare_for_git_write.sh "$SSH_SPECTEREXT_DEPLOY_KEY" - script: - - echo "Now tagging ... git@github.com:${CI_PROJECT_ROOT_NAMESPACE}/specterext-dummy.git" - - ./utils/tag_specterext_dummy.sh - -update_github: - stage: post_releasing - only: - - tags - needs: - - release_signatures - before_script: - # write access to git@github.com:swan-bitcoin/specter-static.git - - source ./utils/prepare_for_git_write.sh "$SSH_SPECTERSTATIC_DEPLOY_KEY" - script: - - echo "Now updating https://github.com:${CI_PROJECT_ROOT_NAMESPACE}/specter-desktop/releases/tag/${CI_COMMIT_TAG:-v2.0.4-pre8}" - - ./utils/generate_downloadpage.sh --org_name ${CI_PROJECT_ROOT_NAMESPACE:-k9ert} --debug --version ${CI_COMMIT_TAG:-v2.0.4-pre8} generate github # default-value for testing - -update_webpage: - stage: post_releasing - only: - - tags - needs: - - release_signatures - before_script: - # write access to git@github.com:swan-bitcoin/specter-static.git - - source ./utils/prepare_for_git_write.sh "$SSH_SPECTERSTATIC_DEPLOY_KEY" - script: - - echo "Now updating https://github.com:${CI_PROJECT_ROOT_NAMESPACE}/specter-static.git" - - ./utils/generate_downloadpage.sh --org_name ${CI_PROJECT_ROOT_NAMESPACE:-k9ert} --debug --version ${CI_COMMIT_TAG:-v2.0.4-pre8} generate webpage # default-value for testing - diff --git a/AGENTS.md b/AGENTS.md index 944ac3898..d7a65b7be 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -67,44 +67,32 @@ python3 -m cryptoadvance.specter server --config DevelopmentConfig --debug ## CI/CD -The project uses **GitHub Actions** for linting, testing, Docker images, and **the full release pipeline**, plus **Cirrus CI** for the heavyweight test suite (pytest + Cypress). GitLab CI is retained in `.gitlab-ci.yml` but effectively dead — the release flow was migrated to GitHub Actions. +**GitHub Actions only.** Cirrus CI and GitLab CI were retired in 2026-Q2 — see `docs/ci-migration-evidence.md` for the cutover evidence and `docs/continuous-integration.md` for the active topology. -### Overview +### Workflows -| Provider | Purpose | Config File | Trigger | -|----------|---------|-------------|---------| -| **GitHub Actions** | Lint, smoke-build, Docker images, **releases** (pip + specterd + Electron for Linux/Win/macOS) | `.github/workflows/` | Push, PR, tags | -| **Cirrus CI** | Full test suite (pytest + Cypress + extension smoketest) | `.cirrus.yml` | PR | -| **GitLab CI** | **Dead** — config retained; `check` job only waits on GH master status. Release jobs no longer run. | `.gitlab-ci.yml` | (vestigial) | +| Workflow | File | Trigger | Purpose | +|----------|------|---------|---------| +| Tests | `test.yml` | PR, push | pytest + Cypress + extension smoketest (3 jobs) | +| Release | `release.yml` | Tag push `v*` | pip + specterd + Electron for Linux/Win/macOS + GPG-sign `SHA256SUMS` | +| Black Linter | `zblack.yml` | PR, push | `psf/black@26.3.0` action pinned to Black 22.3.0, python-3.12 | +| TOC Generator | `toc.yml` | Push | Auto-generates TOCs for `README.md`, `docs/faq.md`, `docs/development.md` | +| Docker Push | `docker-push.yml` | Push to any branch | Multi-arch image → `ghcr.io/cryptoadvance/specter-desktop:` | +| Docker Tag | `docker-tag.yml` | Tag push `v*` | Multi-arch image → `ghcr.io/cryptoadvance/specter-desktop:` | +| Extension Compat | `extension-compat.yml` | PR touching `requirements.*`/`pyproject.toml` | Installs full lock, imports every bundled extension, runs `pip check` | +| specterd Build Smoke | `test-specterd-build.yml` | PR touching `pyinstaller/`, `requirements*`, `src/**` | Builds specterd on Linux and runs `--help` | +| Electron Smoke | `electron-smoketest.yml` | PR touching `pyinstaller/electron/**` | Smoke test Electron packaging | -### GitHub Actions (7 workflows) +All workflows use public GitHub-hosted runners (`ubuntu-latest` / `ubuntu-22.04` / `windows-latest` / `macos-14`). **No private runners.** -1. **Black Python Linter** (`zblack.yml`) — Runs on every push and PR. Uses `psf/black@26.3.0` action pinned to Black version `22.3.0`, on python-3.12 (pinned to avoid 3.14 incompatibility with Black 22.3.0). Checks `./src`. -2. **TOC Generator** (`toc.yml`) — Auto-generates TOCs for `README.md`, `docs/faq.md`, `docs/development.md` on push. -3. **Docker Push** (`docker-push.yml`) — Builds multi-arch (amd64 + arm64) image on every push. Pushes to `ghcr.io//:` (upstream: `ghcr.io/cryptoadvance/specter-desktop:`). -4. **Docker Tag** (`docker-tag.yml`) — Builds multi-arch image on version tags. Pushes to `ghcr.io//:`. -5. **Extension Compatibility Check** (`extension-compat.yml`) — On changes to `requirements.*` or `pyproject.toml`: installs the full lock file, imports every bundled extension (Swan, LiquidIssuer, DevHelp, Notifications, ExFund, Faucet, Electrum, Spectrum, StackTrack, TimelockRecovery), runs `pip check`, and best-effort runs extension test suites. Catches dep conflicts before they break downstream extensions. -6. **Test specterd build** (`test-specterd-build.yml`) — PR smoke test on changes to `pyinstaller/`, `requirements*`, `src/**`, or packaging files. Builds specterd on Linux and runs `--help` smoke test. -7. **Release** (`release.yml`) — **The release pipeline.** See next section. +### Test workflow — `test.yml` -All GitHub Actions use public runners (`ubuntu-latest` / `ubuntu-24.04` / `windows-latest` / `macos-14`). **No private runners.** +Three jobs on `ubuntu-22.04`: +1. **`test`** — pytest with `--cov=cryptoadvance`, 45-min timeout. Installs system deps inline; no custom image. Caches bitcoind/elementsd via `actions/cache@v4` keyed on `runner.os × runner.arch × hash(pyproject.toml, install_noded.sh, bitcoin_SHA256SUMS, elements_SHA256SUMS)` with `save-always: true`. +2. **`cypress`** — `./utils/test-cypress.sh --debug run` inside `ghcr.io/cryptoadvance/specter-desktop/cypress-python-jammy@sha256:`, 30-min timeout, `--shm-size=2g`. +3. **`extension-smoketest`** — byte-compatible port of the old Cirrus smoketest, 15-min timeout. -### Cirrus CI (Testing) - -Cirrus CI runs the **full test suite** on PRs. Config: `.cirrus.yml`. - -**Three tasks:** -1. **`test_task`** — Full pytest suite with bitcoind + elementsd in regtest mode. Uses cached binary downloads. Produces JUnit XML results. -2. **`cypress_test_task`** — Frontend tests with Cypress. Requires 6 CPU, 6GB RAM. Produces screenshots and video artifacts. -3. **`extension_smoketest_task`** — Generates a test extension, starts the server, verifies the extension loads and responds. - -**Docker images used:** -- `registry.gitlab.com/cryptoadvance/specter-desktop/cirrus-jammy:20230206` (pytest) -- `registry.gitlab.com/cryptoadvance/specter-desktop/cypress-python-jammy:20230206` (Cypress) - -Both images are pre-built and hosted on the GitLab container registry. They include Python, virtualenv, and other dependencies. The `docker/` directory in the repo contains Dockerfiles for building them. - -**Caching:** bitcoind and elementsd binaries are cached by Cirrus based on the version pinned in `pyproject.toml`. The `tests/install_noded.sh` script handles downloading or compiling them. +`tests/install_noded.sh` GPG-verifies upstream `SHA256SUMS.asc` and checks tarball SHA256 against the committed trust anchors on every run (cold cache AND cache hit). ### Release pipeline — `.github/workflows/release.yml` @@ -140,12 +128,12 @@ Triggers on tags matching `v[0-9]+.[0-9]+.[0-9]+` (and `-*` suffixes for pre-rel | Secret | Purpose | Required? | |----------|---------|---| -| `APPLE_CERTIFICATE_BASE64` | Base64-encoded `.p12` Apple signing certificate | Optional — unsigned build if missing | -| `APPLE_CERTIFICATE_PASSWORD` | Password for the `.p12` | With `APPLE_CERTIFICATE_BASE64` | -| `APPLE_PROVISIONING_PROFILE_BASE64` | Base64-encoded provisioning profile | Optional | -| PyPI trusted publisher | Configured on PyPI side, not a GH secret | Required for `release-pip` in upstream | - -Historical GitLab secrets (`GH_BIN_UPLOAD_PW`, `TWINE_PASSWORD`, `GPG_PASSPHRASE`, `SSH_SPECTEREXT_DEPLOY_KEY`, `SSH_SPECTERSTATIC_DEPLOY_KEY`) are **no longer used**. `.gitlab-ci.yml` still references them but the pipeline is dead. +| `GPG_PRIVATE_KEY` + `GPG_PASSPHRASE` | Sign `SHA256SUMS` | Required for signed releases | +| `APPLE_CERTIFICATE_BASE64` + `APPLE_CERTIFICATE_PASSWORD` | macOS signing cert | Optional — unsigned build if missing | +| `APPLE_ID` + `APPLE_APP_SPECIFIC_PASSWORD` + `APPLE_TEAM_ID` | macOS notarization | Required with signing | +| `APPLE_PROVISIONING_PROFILE_BASE64` | Provisioning profile | Optional | +| `AARON_TRIGGER` | Trigger `lncm/docker-specter-desktop` build | Optional — skips Docker trigger if missing | +| PyPI trusted publisher | Configured on PyPI side, not a GH secret | Required for `release-pip` upstream | ### Testing a release on a fork @@ -154,12 +142,6 @@ Historical GitLab secrets (`GH_BIN_UPLOAD_PW`, `TWINE_PASSWORD`, `GPG_PASSPHRASE 3. The `release-pip` PyPI publish step is gated on `github.repository == 'cryptoadvance/specter-desktop'`, so forks build the pip package but don't publish. 4. Unsigned macOS builds work out of the box; signing requires you to add your own Apple secrets. -### Dead config - -- `.gitlab-ci.yml` — still in the repo but release jobs no longer run. Safe to remove in a cleanup pass. -- `pyinstaller/build-win-ci.bat` — former GitLab Windows runner entry point; no longer invoked. -- `utils/release.sh` / `utils/release_helper.py` / `utils/github.py` — may contain dead code paths now that GitLab isn't uploading artifacts. Audit before changes. - ## Testing Tests require a `bitcoind` binary (regtest mode). No tests run without it. @@ -217,10 +199,9 @@ Desktop builds use PyInstaller + Electron: 1. **specterd** (daemon binary): `pyinstaller specterd.spec` from `pyinstaller/` dir 2. **Electron app**: wraps specterd, downloads it on first launch with SHA256 + GPG verification -3. Platform scripts: `utils/build-osx.sh`, `utils/build-unix.sh`, `pyinstaller/build-win-ci.bat` -4. pip package: `python3 -m build` +3. pip package: `python3 -m build` -See `docs/build-instructions.md` for step-by-step manual build instructions. +Release builds live in `.github/workflows/release.yml` (triggered by tag push). See `docs/release-guide.md` for the release workflow and `docs/build-instructions.md` for step-by-step manual builds. ## Dependencies @@ -238,7 +219,7 @@ Extensions live in `specterext` namespace packages. Each extension: - Registers via entry points in setup.cfg - Can add UI pages, API endpoints, and background services - Generate a skeleton: `python3 -m cryptoadvance.specter ext gen --ext-id myext --org myorg` -- CI smoke-tests extension generation in `extension_smoketest_task` (Cirrus) +- CI smoke-tests extension generation in the `extension-smoketest` job (`test.yml`) - See `docs/extensions/` for the extension development guide ## Key Files for Navigation @@ -255,14 +236,14 @@ Extensions live in `specterext` namespace packages. Each extension: | All managers | `src/cryptoadvance/specter/managers/` | | Templates | `src/cryptoadvance/specter/templates/` | | Tests | `tests/` | -| CI config | `.github/workflows/`, `.cirrus.yml`, `.gitlab-ci.yml` | +| CI config | `.github/workflows/` | | Build scripts | `pyinstaller/`, `utils/`, `electron/` | | CI Docker images | `docker/` | ## Current State (as of 2026-04) - **Last release:** v2.1.1 (2025-01-03) — no release on the new GH Actions pipeline yet; first tagged run will exercise it end-to-end. -- **CI migration complete:** release pipeline moved from GitLab to `.github/workflows/release.yml`. `.gitlab-ci.yml` retained but dead. +- **CI migration complete:** Cirrus CI and GitLab CI retired; all workflows now on GitHub Actions. See `docs/ci-migration-evidence.md`. - **macOS automation:** now covered on Apple Silicon free tier; x86_64 macOS gated on paid runner. - **Black linter:** reconfigured to pin python-3.12 + `psf/black@26.3.0` action + black version 22.3.0 (worked around 3.14 incompatibility). Verify green state in CI before assuming. - **Issue/PR backlog:** refreshed counts not captured here — use `gh issue list` / `gh pr list` for current state. diff --git a/docker/bionic-build/Dockerfile b/docker/bionic-build/Dockerfile deleted file mode 100644 index 9336aaf11..000000000 --- a/docker/bionic-build/Dockerfile +++ /dev/null @@ -1,7 +0,0 @@ -FROM registry.gitlab.com/cryptoadvance/specter-desktop/python:3.8.5-bionic - -RUN apt-get update && apt-get install -y --no-install-recommends libusb-1.0-0-dev libudev-dev - -RUN apt-get install -y --no-install-recommends libgl1-mesa-dri gvfs gvfs-libs \ - libdrm-amdgpu1 libdrm-nouveau2 libdrm-radeon1 libedit2 libelf1 libllvm10 \ - libvulkan1 libzstd1 libtdb1 libcanberra-gtk3-0 virtualenv libcanberra-gtk3-module \ No newline at end of file diff --git a/docker/bionic-build/Readme.md b/docker/bionic-build/Readme.md deleted file mode 100644 index 723d7db9e..000000000 --- a/docker/bionic-build/Readme.md +++ /dev/null @@ -1,10 +0,0 @@ -This Dockerimage is manually created and uploaded: - -``` -docker build -t registry.gitlab.com/cryptoadvance/specter-desktop/bionic-build:latest . -docker push registry.gitlab.com/cryptoadvance/specter-desktop/bionic-build:latest -``` - -The reason for this image is explained in [#356](https://github.com/cryptoadvance/specter-desktop/issues/356) and introduced in https://github.com/cryptoadvance/specter-desktop/pull/396/files . - -It has been replaced with introducing the electron-build with `registry.gitlab.com/cryptoadvance/specter-desktop/electron-builder:latest` in https://github.com/cryptoadvance/specter-desktop/pull/473/files . \ No newline at end of file diff --git a/docker/cirrus-jammy/Dockerfile b/docker/cirrus-jammy/Dockerfile deleted file mode 100644 index dacaced79..000000000 --- a/docker/cirrus-jammy/Dockerfile +++ /dev/null @@ -1,13 +0,0 @@ -from ubuntu:jammy - -# 1. python-stuff and HWI dependencies -# 2. capability to build bitcoind -# 3. cypress dependencies -RUN apt update && DEBIAN_FRONTEND="noninteractive" apt-get install --no-install-recommends -y \ - libusb-1.0-0-dev libudev-dev python3 python3-virtualenv python3-dev python3-pip \ - build-essential libtool autotools-dev automake autoconf pkg-config bsdmainutils libevent-dev libboost-dev libboost-system-dev libboost-filesystem-dev libboost-test-dev bc \ - nodejs npm libgtk2.0-0 libgtk-3-0 libgbm-dev libnotify-dev libgconf-2-4 libnss3 libxss1 libasound2 libxtst6 xauth xvfb \ - wget libzmq5-dev - -# Stuff needed for Elements (compilation) -RUN DEBIAN_FRONTEND="noninteractive" apt-get install --no-install-recommends -y libboost-thread-dev libsqlite3-dev git diff --git a/docker/cirrus-jammy/README.md b/docker/cirrus-jammy/README.md deleted file mode 100644 index 5b8504f93..000000000 --- a/docker/cirrus-jammy/README.md +++ /dev/null @@ -1,10 +0,0 @@ -An image used to run the build on cirrus (tests only, not cypress-tests). - -Create it like this: - -``` -docker buildx build --platform linux/amd64 -t ghcr.io/cryptoadvance/specter-desktop/cirrus-jammy:20260412 --load . -docker push ghcr.io/cryptoadvance/specter-desktop/cirrus-jammy:20260412 -``` - -Check the `.cirrus.yml` on how this is used and update the $current_date there. diff --git a/docker/electron-builder-master/README.md b/docker/electron-builder-master/README.md deleted file mode 100644 index dfa4822f2..000000000 --- a/docker/electron-builder-master/README.md +++ /dev/null @@ -1,39 +0,0 @@ -# Purpose - -Used for building the electron-app. In short it's the /pyinstaller/build-unix.sh script which is running in this image. - -By intention, this is using an older OS-version in order to avoid glibc-issues. For details, see: -* https://github.com/cryptoadvance/specter-desktop/pull/1688#issuecomment-1242796681 -* https://github.com/cryptoadvance/specter-desktop/issues/373#issuecomment-695068924 - - -# Usage - -If you want to run the image manually, do something like this (copied from [here](https://www.electron.build/multi-platform-build#build-electron-app-using-docker-on-a-local-machine)): - - -``` -docker run --rm -ti \ - --env-file <(env | grep -iE 'DEBUG|NODE_|ELECTRON_|YARN_|NPM_|CI|CIRCLE|TRAVIS_TAG|TRAVIS|TRAVIS_REPO_|TRAVIS_BUILD_|TRAVIS_BRANCH|TRAVIS_PULL_REQUEST_|APPVEYOR_|CSC_|GH_|GITHUB_|BT_|AWS_|STRIP|BUILD_') \ - --env ELECTRON_CACHE="/root/.cache/electron" \ - --env ELECTRON_BUILDER_CACHE="/root/.cache/electron-builder" \ - -v ${PWD}:/project \ - -v ${PWD##*/}-node-modules:/project/node_modules \ - -v ~/.cache/electron:/root/.cache/electron \ - -v ~/.cache/electron-builder:/root/.cache/electron-builder \ - electronuserland/builder:wine -``` - - -# Building - -build the image like: - -``` -docker build -t registry.gitlab.com/cryptoadvance/specter-desktop/electron-builder:latest . -docker push registry.gitlab.com/cryptoadvance/specter-desktop/electron-builder:latest -``` - -# Details - -This image is putting python3.10 on top of electronuserland/builder:wine. As we want to have stable build-targets the actual `FROM` clause is `electronuserland/builder:14-wine-10.22` as the project is adding the node version (14) and a timestamp (10.22) to the tagname. \ No newline at end of file diff --git a/docker/electron-builder/Dockerfile b/docker/electron-builder/Dockerfile deleted file mode 100644 index 1e0acc272..000000000 --- a/docker/electron-builder/Dockerfile +++ /dev/null @@ -1,188 +0,0 @@ -FROM electronuserland/builder:14-wine-10.22 -# 14-wine-10.22 is a stable tag which has been the same than the "wine" tag but more explicit -# It's based on node 14 and focal and therefore has glibc 2.31. -# It has been created on Oct 22 - -# Let's put python3.10 on top: -# * electronuserland/builder:14-wine-10.22 is same as wine which depends on "node" -# * which depends on "base" which builds on buildpack-deps:focal curl -# https://github.com/docker-library/buildpack-deps/blob/98a5ab81d47a106c458cdf90733df0ee8beea06c/ubuntu/focal/curl/Dockerfile -# * now the coe below to install python is copied from: -# https://github.com/docker-library/python/blob/master/3.10/bullseye/Dockerfile -# which builds ontop of buildpack-deps:bullseye-scm -# * So to make the python installation fully functional, we have to install some stuff ... -# * and then do the python installation - - -# Copying some stuff from https://github.com/docker-library/python/blob/master/3.10/bullseye/Dockerfile - -RUN set -ex; \ - apt-get update; \ - apt-get install -y --no-install-recommends \ - autoconf \ - automake \ - bzip2 \ - dpkg-dev \ - file \ - g++ \ - gcc \ - imagemagick \ - libbz2-dev \ - libc6-dev \ - libcurl4-openssl-dev \ - libdb-dev \ - libevent-dev \ - libffi-dev \ - libgdbm-dev \ - libglib2.0-dev \ - libgmp-dev \ - libjpeg-dev \ - libkrb5-dev \ - liblzma-dev \ - libmagickcore-dev \ - libmagickwand-dev \ - libmaxminddb-dev \ - libncurses5-dev \ - libncursesw5-dev \ - libpng-dev \ - libpq-dev \ - libreadline-dev \ - libsqlite3-dev \ - libssl-dev \ - libtool \ - libwebp-dev \ - libxml2-dev \ - libxslt-dev \ - libyaml-dev \ - make \ - patch \ - unzip \ - zip \ - jq \ - xz-utils \ - zlib1g-dev - - -# This has been copied from: -# https://github.com/docker-library/python/blob/master/3.10/bullseye/Dockerfile - - -# ensure local python is preferred over distribution python -ENV PATH /usr/local/bin:$PATH - -# http://bugs.python.org/issue19846 -# > At the moment, setting "LANG=C" on a Linux system *fundamentally breaks Python 3*, and that's not OK. -ENV LANG C.UTF-8 - -# runtime dependencies -RUN set -eux; \ - apt-get update; \ - apt-get install -y --no-install-recommends \ - libbluetooth-dev \ - tk-dev \ - uuid-dev \ - ; \ - rm -rf /var/lib/apt/lists/* - -ENV GPG_KEY A035C8C19219BA821ECEA86B64E628F8D684696D -ENV PYTHON_VERSION 3.10.9 - -RUN set -eux; \ - \ - savedAptMark="$(apt-mark showmanual)"; \ - apt-get update; \ - apt-get install -y --no-install-recommends \ - patchelf \ - ; \ - \ - wget -O python.tar.xz "https://www.python.org/ftp/python/${PYTHON_VERSION%%[a-z]*}/Python-$PYTHON_VERSION.tar.xz"; \ - wget -O python.tar.xz.asc "https://www.python.org/ftp/python/${PYTHON_VERSION%%[a-z]*}/Python-$PYTHON_VERSION.tar.xz.asc"; \ - GNUPGHOME="$(mktemp -d)"; export GNUPGHOME; \ - gpg --batch --keyserver hkps://keys.openpgp.org --recv-keys "$GPG_KEY"; \ - gpg --batch --verify python.tar.xz.asc python.tar.xz; \ - command -v gpgconf > /dev/null && gpgconf --kill all || :; \ - rm -rf "$GNUPGHOME" python.tar.xz.asc; \ - mkdir -p /usr/src/python; \ - tar --extract --directory /usr/src/python --strip-components=1 --file python.tar.xz; \ - rm python.tar.xz; \ - \ - cd /usr/src/python; \ - gnuArch="$(dpkg-architecture --query DEB_BUILD_GNU_TYPE)"; \ - ./configure \ - --build="$gnuArch" \ - --enable-loadable-sqlite-extensions \ - --enable-optimizations \ - --enable-option-checking=fatal \ - --enable-shared \ - --with-lto \ - --with-system-expat \ - --without-ensurepip \ - ; \ - nproc="$(nproc)"; \ - make -j "$nproc" \ - ; \ - make install; \ - \ -# https://github.com/docker-library/python/issues/784 -# prevent accidental usage of a system installed libpython of the same version - bin="$(readlink -vf /usr/local/bin/python3)"; \ - patchelf --set-rpath '$ORIGIN/../lib' "$bin"; \ - \ -# enable GDB to load debugging data: https://github.com/docker-library/python/pull/701 - dir="$(dirname "$bin")"; \ - mkdir -p "/usr/share/gdb/auto-load/$dir"; \ - cp -vL Tools/gdb/libpython.py "/usr/share/gdb/auto-load/$bin-gdb.py"; \ - \ - cd /; \ - rm -rf /usr/src/python; \ - \ - find /usr/local -depth \ - \( \ - \( -type d -a \( -name test -o -name tests -o -name idle_test \) \) \ - -o \( -type f -a \( -name '*.pyc' -o -name '*.pyo' -o -name 'libpython*.a' \) \) \ - \) -exec rm -rf '{}' + \ - ; \ - \ - ldconfig; \ - \ - apt-mark auto '.*' > /dev/null; \ - apt-mark manual $savedAptMark; \ - apt-get purge -y --auto-remove -o APT::AutoRemove::RecommendsImportant=false; \ - rm -rf /var/lib/apt/lists/*; \ - \ - python3 --version - -# make some useful symlinks that are expected to exist ("/usr/local/bin/python" and friends) -RUN set -eux; \ - for src in idle3 pydoc3 python3 python3-config; do \ - dst="$(echo "$src" | tr -d 3)"; \ - [ -s "/usr/local/bin/$src" ]; \ - [ ! -e "/usr/local/bin/$dst" ]; \ - ln -svT "$src" "/usr/local/bin/$dst"; \ - done - -# if this is called "PIP_VERSION", pip explodes with "ValueError: invalid truth value ''" -ENV PYTHON_PIP_VERSION 22.3.1 -# https://github.com/docker-library/python/issues/365 -ENV PYTHON_SETUPTOOLS_VERSION 65.5.1 -# https://github.com/pypa/get-pip -ENV PYTHON_GET_PIP_URL https://github.com/pypa/get-pip/raw/1a96dc5acd0303c4700e02655aefd3bc68c78958/public/get-pip.py -ENV PYTHON_GET_PIP_SHA256 d1d09b0f9e745610657a528689ba3ea44a73bd19c60f4c954271b790c71c2653 - -RUN set -eux; \ - \ - wget -O get-pip.py "$PYTHON_GET_PIP_URL"; \ - echo "$PYTHON_GET_PIP_SHA256 *get-pip.py" | sha256sum -c -; \ - \ - export PYTHONDONTWRITEBYTECODE=1; \ - \ - python get-pip.py \ - --disable-pip-version-check \ - --no-cache-dir \ - --no-compile \ - "pip==$PYTHON_PIP_VERSION" \ - "setuptools==$PYTHON_SETUPTOOLS_VERSION" \ - ; \ - rm -f get-pip.py; \ - \ - pip --version \ No newline at end of file diff --git a/docker/electron-builder/README.md b/docker/electron-builder/README.md deleted file mode 100644 index 4e22a48a2..000000000 --- a/docker/electron-builder/README.md +++ /dev/null @@ -1,28 +0,0 @@ -Used for building the electron-app. In short it's the /pyinstaller/build-unix.sh script which is running in this image. - -By intention, this is using an older OS-version in order to avoid glibc-issues. For details, see: -* https://github.com/cryptoadvance/specter-desktop/pull/1688#issuecomment-1242796681 -* https://github.com/cryptoadvance/specter-desktop/issues/373#issuecomment-695068924 - - -If you want to run the image manually, do something like this (copied from [here](https://www.electron.build/multi-platform-build#build-electron-app-using-docker-on-a-local-machine)): - - -``` -docker run --rm -ti \ - --env-file <(env | grep -iE 'DEBUG|NODE_|ELECTRON_|YARN_|NPM_|CI|CIRCLE|TRAVIS_TAG|TRAVIS|TRAVIS_REPO_|TRAVIS_BUILD_|TRAVIS_BRANCH|TRAVIS_PULL_REQUEST_|APPVEYOR_|CSC_|GH_|GITHUB_|BT_|AWS_|STRIP|BUILD_') \ - --env ELECTRON_CACHE="/root/.cache/electron" \ - --env ELECTRON_BUILDER_CACHE="/root/.cache/electron-builder" \ - -v ${PWD}:/project \ - -v ${PWD##*/}-node-modules:/project/node_modules \ - -v ~/.cache/electron:/root/.cache/electron \ - -v ~/.cache/electron-builder:/root/.cache/electron-builder \ - electronuserland/builder:wine -``` - -build the image like: - -``` -docker build -t registry.gitlab.com/cryptoadvance/specter-desktop/electron-builder:latest . -docker push registry.gitlab.com/cryptoadvance/specter-desktop/electron-builder:latest -``` \ No newline at end of file diff --git a/docker/github-changelog/Dockerfile b/docker/github-changelog/Dockerfile deleted file mode 100644 index 135bb3195..000000000 --- a/docker/github-changelog/Dockerfile +++ /dev/null @@ -1,18 +0,0 @@ -FROM python:3.8 - -ARG REPO=https://github.com/cryptoadvance/github-changelog - -RUN apt update && apt install -y git - -WORKDIR / - -RUN git clone $REPO; - -WORKDIR /github-changelog -RUN git checkout master - -RUN python3 setup.py install - -ENV PYTHONUNBUFFERED="1" - -ENTRYPOINT ["changelog"] diff --git a/docker/github-changelog/Readme.md b/docker/github-changelog/Readme.md deleted file mode 100644 index 47de543a5..000000000 --- a/docker/github-changelog/Readme.md +++ /dev/null @@ -1,25 +0,0 @@ -An image used to create changelogs. - -Create it like this: - -``` -docker build . -t registry.gitlab.com/cryptoadvance/specter-desktop/github-changelog:latest -docker push registry.gitlab.com/cryptoadvance/specter-desktop/github-changelog:latest -``` - -Use it like this: - -``` -latest_version=v0.8.1 -export GH_TOKEN=YourTokenHere -docker run 4a3dd375832d --github-token $GH_TOKEN --branch master cryptoadvance specter-desktop $latest_version > docs/new_release_notes.md -cp docs/release-notes.md docs/release-notes.md.orig -cat docs/new_release_notes.md docs/release-notes.md.orig > docs/release-notes.md -rm docs/release-notes.md.orig docs/new_release_notes.md - -``` -# This will print out links to all PRs in order to review better - -``` -docker run 4a3dd375832d -m --github-token $GH_TOKEN --branch master cryptoadvance specter-desktop $latest_version > docs/new_release_notes.md -``` diff --git a/docs/build-instructions.md b/docs/build-instructions.md index 2db57300d..4cc7a1e45 100644 --- a/docs/build-instructions.md +++ b/docs/build-instructions.md @@ -6,13 +6,9 @@ This document addresses the build-system part. For the continuous-integration-pa ## pip-packages ``` -# in the case of a release, the version needs to be adapted: -# sed -i "s/version=\".*/version=\"$CI_COMMIT_TAG\",/" setup.py - -python3 setup.py sdist bdist_wheel -cryptoadvance.specter-vx.y.z-get-replaced-by-release-script.tar.gz +python3 -m build ``` -This process is the same for all platforms. The result unfortunately is not stable in terms of identically sh256-hashes, though. +Produces an sdist + wheel under `dist/`. The release pipeline sets `SETUPTOOLS_SCM_PRETEND_VERSION` from the git tag; locally, setuptools-scm derives the version from your working tree. The result is not stable in terms of identical sha256-hashes across machines. ## Electron The electron build is assuming a node-installation. So make sure you have `node` and `npm` available. @@ -20,7 +16,7 @@ The electron build is assuming a node-installation. So make sure you have `node` The electron-app is built in a way that it's running the `specterd` (specter-demon) internally. It's not bundled with the electron-binary but downloaded with the first start (including sha256- and gpg-verification). If someone does not want the download, he can manually choose a specterd-binary from the `preferences/Advanced` menu. Nevertheless the Electron-App is tied, at buildtime, to a specific specterd-binary via a sha256-version. This probably doesn't make so much sense if you build outside of a release but we need it anyway. -So let's cover the build of the specterd-binary first. Below is a manual description of the build-process. There are acripts which are doing this but they are partially optimized for the CI-system. Check the `pyinstaller/build-*` scripts for details. +So let's cover the build of the specterd-binary first. Below is a manual description of the build-process. The canonical CI build lives in `.github/workflows/release.yml` (`build-specterd-*` and `build-electron-*` jobs). First set the virtualenv: @@ -31,11 +27,6 @@ source .buildenv/bin/activate ### specterd Linux and MacOS -Below doesn't seem to work properly, at least on MacOS, better use the build script. For MacOS, that would be: -```bash -./utils/build-osx.sh --version 0.0.0-pre1 specterd -``` - ```bash cd pyinstaller # prerequisites @@ -98,7 +89,7 @@ node ./set-version v1.3.1-custom ../dist/specterd npm i # We assume here that no Apple-developer-ID is used to sign the binary. -# Check `build-osx.sh` if you want to sign +# For signed+notarized builds, see .github/workflows/release.yml (build-electron-macos). echo "`jq '.build.mac.identity=null' package.json`" > package.json # finally build diff --git a/docs/ci-migration-evidence.md b/docs/ci-migration-evidence.md new file mode 100644 index 000000000..297d03945 --- /dev/null +++ b/docs/ci-migration-evidence.md @@ -0,0 +1,84 @@ +# CI Migration Evidence — Cirrus → GitHub Actions + +Evidence artifact per `docs/cirrus-replacement-spec.md` §Acceptance. Captures measured GHA behavior over the side-by-side period so that "did we actually hit SLO?" has a grep-able answer after GH Actions logs are GC'd. + +## Snapshot + +- **Measurement date:** 2026-04-19 +- **PR #1 (`test.yml` added):** merged 2026-04-17 as commit `a24df2eb` (PR [#2606](https://github.com/cryptoadvance/specter-desktop/pull/2606)) +- **Cirrus sunset deadline:** 2026-06-30 (~10 weeks remaining) +- **Side-by-side window so far:** ~2 days + +## Gate status + +Spec §Acceptance requires **all four** to hold before merging the cutover PR: + +| # | Criterion | Status | Notes | +|---|----------------------------------------------------------|------------|-----------------------------------------------------------------------| +| 1 | 10 consecutive green master runs | **Not met**| 2 master runs observed, both green | +| 2 | ≥ 3 green PR runs incl. one frontend-touching | **Partial**| 6 green PR runs; none confirmed as frontend-touching yet | +| 3 | Zero new flakes over ≥ 50 total runs | **Not met**| 14 total runs; zero flakes detected; sample too small | +| 4 | Cypress p95 within Cirrus +20% | **Breach** | Cypress p95 **10m10s** vs. Cirrus +20% ceiling **7m14s** — see below | + +**Merging PR #2 ahead of the nominal gate is a deliberate choice** driven by Cirrus's hard 2026-06-30 shutdown, preservation of revertability (PR #2 is a pure deletion of dead code + docs updates; revert is one click), and the empirical fact that no flakes have surfaced over the available sample. Gate criteria 1 and 3 will be satisfied by ordinary master-branch activity over the coming weeks; criterion 4 is acknowledged below as a known deviation, with a measurement protocol for re-evaluation. + +## Measured wall-clock + +n = 8 successful runs (2 master + 6 PR) between 2026-04-17 11:13 UTC and 2026-04-17 20:45 UTC. + +| Job | n | median | p95 | min | max | Cirrus median | Cirrus +20% ceiling | Result | +|-----------------------|---|---------|---------|---------|---------|---------------|---------------------|--------------------| +| `test` | 8 | 3m53s | 4m14s | 3m43s | 4m14s | 4m47s | 5m44s | **within ceiling** | +| `cypress` | 8 | 9m42s | 10m10s | 9m28s | 10m10s | 6m02s | 7m14s | **BREACH (+47%)** | +| `extension-smoketest` | 8 | 1m57s | 2m09s | 1m44s | 2m09s | 2m10s | 2m36s | **within ceiling** | + +Raw data pulled via `gh api repos/cryptoadvance/specter-desktop/actions/runs//jobs` for runs `24581485817`, `24585472647` (master), and `24580543016`, `24581562475`, `24582301901`, `24582337498`, `24585550944`, `24562219139` (PR). + +Cirrus baselines cited from `docs/cirrus-replacement-spec.md` §SLOs (20-sample baseline captured 2026-04-12). + +### Cypress breach — acknowledgement + +GHA Cypress p95 is **10m10s**, vs. the spec's Cirrus +20% ceiling of **7m14s** (Cirrus p95 6m55s × 1.20). Root cause not yet investigated. Candidates per spec §Cypress measurement: `--shm-size` bump, spec sharding, or escalation to `ubuntu-22.04-large`. + +**Decision:** accepted as a known deviation. Cypress wall-clock is still well under the 30-minute workflow timeout, and the alternative — holding the cutover until after Cirrus shutdown — would leave the project without PR gating. The breach is logged here rather than swept under the rug. + +**Follow-up:** re-run the measurement protocol (5× on `ubuntu-22.04` free tier) once 10+ master runs accumulate. If p95 remains >Cirrus+20%, file an issue and walk the escalation ladder (shm → shard → paid runner). + +## Flake signal + +Over 14 total `test.yml` runs (8 success, 5 failure, 1 action_required): + +- **Failures on `kn/cirrus-replacement-spec`** (4): iteration during PR #1 development. Each failure was followed by a targeted fix commit. Confirmed non-flaky by reading `git log` (`fix: cache symlink targets…`, `fix: bash shell for cypress container`, `fix: use VALIDSIG instead of GOODSIG`, `fix: use --status-fd`). +- **Failure on `kn/bump-bitcoind-test-v27.2`** (1): bitcoind version bump branch. Likely a real test failure from the version change, not a CI flake. +- **`action_required`** (1): fork PR (`copilot/fix-livereload-ui-delays`) pending maintainer approval to run workflows. Not a flake. + +**Flake count: 0** over this window. Sample size too small (n=14) to assert the steady-state SLO of ≤1% rolling-30-day, but no red flags. + +## Sample PR runs + +| URL | Branch | Conclusion | Frontend-touching? | +|---------------------------------------------------------------------------------------------------------------------|----------------------------------------------------------------|------------|--------------------| +| https://github.com/cryptoadvance/specter-desktop/actions/runs/24585550944 | `dependabot/npm_and_yarn/pyinstaller/electron/multi-20d65b3440`| success | partial (electron deps) | +| https://github.com/cryptoadvance/specter-desktop/actions/runs/24582337498 | `dependabot/npm_and_yarn/pyinstaller/electron/multi-3ffb4d349a`| success | partial (electron deps) | +| https://github.com/cryptoadvance/specter-desktop/actions/runs/24581562475 | `dependabot/npm_and_yarn/multi-6f6dcfc8d5` | success | indirect | +| https://github.com/cryptoadvance/specter-desktop/actions/runs/24562219139 | `kn/cirrus-replacement-spec` | success | no | + +Strict frontend-touching coverage (changes under `src/cryptoadvance/specter/static/` or `src/cryptoadvance/specter/templates/`) is **not yet confirmed** in the available sample. Criterion 2 will be rechecked at PR #2 merge time. + +## Sign-off + +| Field | Value | +|----------------------|------------------------------------------------------------| +| Evidence captured at | 2026-04-19 | +| Latest master commit | `62ea0265` (2026-04-17 20:33 UTC) | +| `test.yml` added at | `a24df2eb` (2026-04-17 18:50 UTC) | +| Author | @k9ert | + +## Rollback contract + +Per spec §Rollback, if post-cutover breakage emerges: +1. Revert the PR #2 merge commit → restores `.cirrus.yml` and `docker/cirrus-jammy/`. +2. Re-add the Cirrus required-check names to master branch protection. +3. Cirrus assumed operational through 2026-06-30. + +After 2026-06-30, no Cirrus fallback exists — forward fix only. Keep the cutover ≥ 4 weeks ahead of that date (target merge date per spec: 2026-05-12). diff --git a/docs/continuous-integration.md b/docs/continuous-integration.md index c2010629c..2803427ac 100644 --- a/docs/continuous-integration.md +++ b/docs/continuous-integration.md @@ -1,111 +1,60 @@ -# Introduction -Specter-Desktop is using GitLab, Cirrus and GitHub-Actions for continuous integration purposes but GitHub-actions only for Blackify so far. It might be more effort using more than one CI-approach but it makes us also more resilient. -GitLab and Cirrus have both advantages and disadvantages so ... let's use both! -GitLab: -* is completely open Source for server- and clients -* the gitlab-runner can run docker and is itself running on docker -* but does not support Pull-Requests -* needs to have bitcoind in a prepared docker-container which binds the build to that version +# Continuous Integration -Cirrus-CI: -* supports the PR-model -* quite easy to setup even though it's using docker +Specter-Desktop runs all CI on **GitHub Actions**. Cirrus CI and GitLab CI were retired in 2026-Q2 — see `docs/ci-migration-evidence.md` for the cutover evidence. -## Gitlab +## Workflows -Gitlab is a great CI/CD-platform and in the meantime it's quite easy to use it for GitHub-repositories. -https://docs.gitlab.com/ee/ci/ci_cd_for_external_repos/github_integration.html -The main file which specifies the jobs on GitLab is .gitlab-ci.yml -We're using a `gitlab-docker-runner` which means that all jobs are running in a container. -However at the same time we're using docker to spinup a bitcoind. +| Workflow | File | Trigger | +|--------------------------------|-------------------------------------------------|----------------------------------------------| +| Lint (black) | `.github/workflows/zblack.yml` | PR, push | +| Tests (pytest + Cypress + extension smoketest) | `.github/workflows/test.yml` | PR, push | +| Release | `.github/workflows/release.yml` | Tag push (`v*`) | +| Electron smoketest | `.github/workflows/electron-smoketest.yml` | PR and push to master on `pyinstaller/electron/**` | +| Extension compatibility | `.github/workflows/extension-compat.yml` | PR; push on `requirements.*` / `pyproject.toml`; `workflow_dispatch` | +| Specterd build smoke | `.github/workflows/test-specterd-build.yml` | PR | +| Docker image push | `.github/workflows/docker-push.yml` | Push to any branch | +| Docker image tag | `.github/workflows/docker-tag.yml` | Tag push (`v*`) | +| Docs table of contents | `.github/workflows/toc.yml` | Push | -The image is created manually (see /docker) and used for running the tests AND also for -spinning up bitcoind. +## Test workflow -For that reason we need to share the docker-socket from the host into the container and -create our own GitLab specific runner as described here: -https://docs.gitlab.com/ee/ci/docker/using_docker_build.html#use-docker-socket-binding +`test.yml` has three jobs, all on `ubuntu-22.04`: -Due to that setup there are some specifics which are mainly addressed in tests/conftest -start_bitcoind-function: -* adding -rpcallowip= (from a docker network) to bitcoind -* not use localhost but the docker-network-ip-address when talking to the bitcoind +- **`test`** — pytest with `--cov=cryptoadvance`. Runs in 45 min. Installs system deps inline; no custom image. Caches bitcoind/elementsd binaries via `actions/cache@v4` keyed on `runner.os × runner.arch × hash(pyproject.toml, tests/install_noded.sh, tests/bitcoin_SHA256SUMS, tests/elements_SHA256SUMS)`. +- **`cypress`** — runs `./utils/test-cypress.sh --debug run` inside `ghcr.io/cryptoadvance/specter-desktop/cypress-python-jammy@sha256:`. 30-minute timeout. `--shm-size=2g` to avoid Cypress OOMs on the default 64 MB `/dev/shm`. Shares the bitcoind/elements cache with `test`. +- **`extension-smoketest`** — byte-compatible port of the former Cirrus smoketest. 15 min. Smoke-tests `ext gen`, server boot, and log-line / curl assertion. Contract must stay stable — downstream extension developers depend on it. -## Travis-CI +All three jobs use `actions/checkout@v4` with `fetch-depth: 0` so `git describe` resolves annotated tags for `tests/test_util_version.py`. -We're no longer using travis-ci due to the abuse-detection-system going wild on us. +## Caching -## Cirrus-CI +`actions/cache@v4` with `save-always: true` on a key that includes `runner.arch` (prevents ARM/x86 cache poisoning). The key hashes the committed `tests/bitcoin_SHA256SUMS` and `tests/elements_SHA256SUMS` trust anchors — bumping a version in `pyproject.toml` rotates the cache via those files. -[Cirrus-CI](https://cirrus-ci.org) is used by Bitcoin-Core and HWI and is a quite good replacement for travis. We're using it only for PRs so far. The [../.cirrus.yml] file defines the build. We have two task, one for pytest and one for the [cypress-tests](./cypress-testing.md). +### Binary verification -## Releasing +`tests/install_noded.sh` GPG-verifies the upstream `SHA256SUMS.asc` against the Bitcoin Core and Elements release signing keys, and checks the tarball SHA256 against the committed trust anchors on every run (cold cache AND cache hit). A tampered cache entry fails closed on restore. See PR #2606 for the threat model. -### What gets released +## Cypress container -We're mostly releasing automatically. Currently the following artifacts are released: -* specterd (daemon) is a binary for kicking off the specter-desktop service on the command-line. We have binaries for windows, Linux and macOS -* We have an Electron-App which we're also releasing for Windows, Linux and MacOS. Unfortunately the macOS build is not yet automated -* We release a pip-package -* Usually some time after the release, the lncm is releasing [docker-images](https://hub.docker.com/r/lncm/specter-desktop). Very much appreciated, even though we can't guarantee for them, obviously. +`ghcr.io/cryptoadvance/specter-desktop/cypress-python-jammy` is pinned by digest (not tag) in `test.yml`. This makes Dockerfile edits visibly require a workflow bump. When editing `docker/cypress-python-jammy/Dockerfile`, rebuild and push to GHCR with a fresh tag, then update the digest pin. -### How we release -As we have a strict build-only-on-private-hardware build-policy, we're using GitLab private runners in order to build our releases. In order to test and develop the releasing automation, people can setup GitLab-projects which are syncing from their GitHub-forks. With such a setup it's possible to create test-releases and therefore test the whole procedure end-to-end. +## Release pipeline -The automation of that kicks in if someone creates a tag which is named like "vX.Y.Z". This is specified in the gitlab-ci.yml. The release-job will only be triggered in cases of tags. One step will also check that the tag follows the convention above. -The package upload will need a token. How to obtain the token is described in the packaging-tutorial. It's injected via GitLab-variables. ToDo: put the token on a trusted build-node. +See [`release-guide.md`](./release-guide.md). Pushing a tag matching `v[0-9]+.[0-9]+.[0-9]+[-*]?` triggers `release.yml`, which builds pip/specterd/Electron artifacts for Linux/Windows/macOS, signs `SHA256SUMS`, and creates a draft GitHub release. Docker images are built by `lncm/docker-specter-desktop` (triggered via `AARON_TRIGGER` secret). -### pyinstaller system-dependent binaries -The [pyinstaller directory](../pyinstaller) contains scripts to create the platform-specific binaries (plus electron) to use specter-desktop as a desktop-software. Some of them are created and uploaded to [GitHub-releases](https://github.com/cryptoadvance/specter-desktop/releases) via more or less special build-agents. -The [windows-build-agent](https://docs.gitlab.com/runner/install/windows.html) needs manual installation -of git, python and docker. Docker is used to build the innosetup-file. -As docker is available in windows only as a "desktop-edition", one need to also -log into the windows-machine to get docker started. -Clearly there is an opportunity to move all of the creation of the windows-binary to wine on docker, -similiar to the way the innosetup is running within docker. +## Flake policy -## CI/CD-dev-env setup +- Cypress: `retries: { runMode: 1, openMode: 0 }`. Specs retry-to-green emit a warning annotation. +- pytest: `--reruns 0` (fail fast). Flakes are debt, not a coping mechanism. +- Spec flagged flaky twice in 14 days gets `@skip(reason="flaky", issue="#NNNN")` with a 2-week SLA. -Here is a brief description on how to create a setup where the release-procedures can be tested: -* We assume you have a fork of cryptoadvance/specter-desktop. We also assume that your GitLab-user-handle is the exact same as on GitHub. -* Create a GitLab-account and then a mirroring project ([here](https://gitlab.com/projects/new#cicd_for_external_repo)) obviously with the exact same name: "specter-desktop" -* Activate the private runners and deactivate the public runners. Contact @k9ert for that. -* Create an account and an [API token](https://test.pypi.org/manage/account/) on there -* Create a token for GitHub in order to release to your GitHub-fork -* Configure both tokens on the GitLab-variables (GH_BIN_UPLOAD_PW and TWINE_PASSWORD) -* create a tag on your GitHub-fork -* watch the test-release unfolding, ready to hack +## Secrets -### GitLab-runner setup (Windows) +| Secret | Used by | Purpose | +|---------------------------|--------------------------|----------------------------------------------| +| `GITHUB_TOKEN` | (auto-provided) | Checkout, artifact upload, ghcr.io push | +| `GPG_PRIVATE_KEY` + `GPG_PASSPHRASE` | `release.yml` | Sign `SHA256SUMS` | +| `APPLE_*` (six) | `release.yml` macOS | Code signing + notarization (optional) | +| `AARON_TRIGGER` | `release.yml` | Trigger `lncm/docker-specter-desktop` build | -For Windows-releasing, we're using a windows GitLab-runner. Here is a short description on how to set one up. - -#### Prerequisites - -You need at least Windows Home 10 which is up-to-date. The most complex dependency is setting up docker. -Docker-Desktop needs a WSL2 which is a good idea to install on windows anyway. [Here](https://www.omgubuntu.co.uk/how-to-install-wsl2-on-windows-10) is a description on how to do that. - -While installing, make sure you know the locations of where that stuff is installed. We'll later need to verify/adjust the PATH. - -* Install Python, i took the [3.7.9 webinstaller](https://www.python.org/ftp/python/3.7.9/python-3.7.9-amd64-webinstall.exe) -* Install Git, e.g. [this](https://github.com/git-for-windows/git/releases/download/v2.29.2.windows.2/Git-2.29.2.2-64-bit.exe) (i had 2.28.2) -* Install [Docker-Desktop](https://desktop.docker.com/win/stable/Docker%20Desktop%20Installer.exe) - -Now open and check the "Environment-variables" and check that the following lines are in there: - -![](./images/continuous-integration_runner_windows_envvars.png) - -#### Runner - -The runner itself is easy to [setup](https://docs.gitlab.com/runner/install/windows.html). Follow the link or this very brief description: -* `mkdir \Gitlab-Runner` -* download [this binary](https://gitlab-runner-downloads.s3.amazonaws.com/latest/binaries/gitlab-runner-windows-amd64.exe) in that folder and rename to gitlab-runner.exe -* Search for "powershell" in windows an open AS ADMINISTRATOR -* `cd \Gitlab-Runner` -* Copy the Registration-token from [here](https://gitlab.com/k9ert/specter-desktop/-/settings/ci_cd) (unfold runners, see specific runners) -* `./gitlab-runner.exe register`and paste the token (the instance-url is the default) -* give a reasonable description. Make sure to tag this runner with "tag". If that's not possible here, you can do it in the page mentioned above -* `.\gitlab-runner.exe install` will install the runner as system-service -* `.\gitlab-runner.exe start` will start it - -Done +No GitLab secrets remain. diff --git a/docs/images/continuous-integration_runner_windows_envvars.png b/docs/images/continuous-integration_runner_windows_envvars.png deleted file mode 100644 index ea0e30f7e..000000000 Binary files a/docs/images/continuous-integration_runner_windows_envvars.png and /dev/null differ diff --git a/docs/images/release-guide/github-token.png b/docs/images/release-guide/github-token.png deleted file mode 100644 index f57b336f5..000000000 Binary files a/docs/images/release-guide/github-token.png and /dev/null differ diff --git a/docs/images/release-guide/overview-gitlab-pipline.png b/docs/images/release-guide/overview-gitlab-pipline.png deleted file mode 100644 index 88c33bee4..000000000 Binary files a/docs/images/release-guide/overview-gitlab-pipline.png and /dev/null differ diff --git a/docs/images/release-guide/rerun-release-signatures.png b/docs/images/release-guide/rerun-release-signatures.png deleted file mode 100644 index 07b9e6781..000000000 Binary files a/docs/images/release-guide/rerun-release-signatures.png and /dev/null differ diff --git a/docs/release-guide.md b/docs/release-guide.md index 2328e86c0..7f2675e34 100644 --- a/docs/release-guide.md +++ b/docs/release-guide.md @@ -1,160 +1,60 @@ # Release Guide -## Creating release notes +The release pipeline runs on GitHub Actions. Pushing a version tag to `upstream` triggers `.github/workflows/release.yml`, which builds every artifact, creates a draft GitHub release, and signs `SHA256SUMS` with the Specter Signer GPG key. -### Pre-requisites +## Prerequisites -- You need the correct upstream master. You should see +- `upstream` points at `git@github.com:cryptoadvance/specter-desktop.git` (`git remote -v` should show both fetch and push). +- You are on `master` with a clean workspace and `git pull upstream master` applied. +- Release-notes PR has already merged (see [Release notes](#release-notes) below). + +## Cut a release ```bash -git remote -v | grep upstream -upstream git@github.com:cryptoadvance/specter-desktop.git (fetch) -upstream git@github.com:cryptoadvance/specter-desktop.git (push) +git tag v1.13.1 +git push upstream v1.13.1 ``` -- You need a GitHub token: - If you don't have one, get one here https://github.com/settings/tokens and make sure to tick the boxes for repo and workflow as below: +That's it. The `Release` workflow on GitHub Actions takes it from here: -![](./images/release-guide/github-token.png) +- **`release-pip`** — builds the sdist/wheel and publishes to PyPI via trusted publishing. +- **`build-specterd-{linux,windows,macos}`** — builds the `specterd` binary on each platform (macOS arm64 on the free `macos-14` runner). +- **`build-electron-{linux,windows,macos}`** — builds the Electron apps using each platform's `specterd` artifact. Windows uses the public `electronuserland/builder:wine` image; macOS signs + notarizes if `APPLE_CERTIFICATE_BASE64` et al. are configured. +- **`create-release`** — collects all artifacts, generates `SHA256SUMS`, signs it with the GPG key from the `GPG_PRIVATE_KEY` secret, generates a release body (with auto-generated "What's Changed" via `gh api .../generate-notes`), and creates a **draft** GitHub release. +- **`trigger-docker`** — POSTs a repository-dispatch to `lncm/docker-specter-desktop` so Aaron's Docker build picks up the new tag (needs `AARON_TRIGGER` secret; skipped otherwise). -Using the new token, run +The release lands as a draft — review and publish it manually on GitHub. + +### Required secrets + +| Secret | Purpose | +|------------------------------------|--------------------------------------------------------------| +| `GPG_PRIVATE_KEY` | ASCII-armored private key for signing `SHA256SUMS` | +| `GPG_PASSPHRASE` | Passphrase for the above | +| `APPLE_CERTIFICATE_BASE64` | Developer ID cert for macOS signing (optional — unsigned fallback) | +| `APPLE_CERTIFICATE_PASSWORD` | p12 password | +| `APPLE_ID`, `APPLE_APP_SPECIFIC_PASSWORD`, `APPLE_TEAM_ID` | Apple notarization credentials | +| `APPLE_PROVISIONING_PROFILE_BASE64`| Optional provisioning profile | +| `AARON_TRIGGER` | PAT for triggering `lncm/docker-specter-desktop` | + +PyPI publishing uses trusted publishing (OIDC) — no secret required. + +## Release notes + +Update `docs/release-notes.md` via a normal PR before tagging. Use the GitHub API or `gh` to pull "What's Changed" between the previous tag and `master`, prepend a heading, and open a PR. The `create-release` workflow job also appends auto-generated notes to the release body. + +## GitHub pages download page + +`./utils/generate_downloadpage.sh` still generates the `specter-static` website's download page off `utils/templates/`. Clone `specter-static` alongside `specter-desktop` and run: ```bash -export GH_TOKEN=YOURTOKEN +./utils/generate_downloadpage.sh ``` -- You need Docker running -- Checkout the master branch and ensure a clean workspace. +The script installs the markdown prerequisite, regenerates the GH-page and download page, asks whether to replace/update the GitHub release page for the latest version, and offers to commit/push the static-site changes. -Now, you can run +## Troubleshooting -```bash -./utils/release.sh --release-notes -``` +If something fails mid-pipeline, re-running individual jobs is safe — they `actions/download-artifact` from prior jobs and overwrite existing release assets via `softprops/action-gh-release`. If the draft release already has assets from a stale run, delete the draft and re-run `create-release`. -Or, if you want to directly set the new version: - -```bash -./utils/release.sh --new-version v1.13.1 --release-notes -``` - -## Creating a new tag - -Update your master branch after the release notes PR ([example](https://github.com/cryptoadvance/specter-desktop/commit/65ff6959d7fd85cba745e4d454b30031839f857f/)) has been merged and then run: - -```bash -git tag v1.13.1 && git push upstream v1.13.1 -``` - -if you have a proper setup of github- AND gitlab forks (and the remote `origin` on git pointing to your github fork), you can push that tag to origin and this is useful to test the procedures. See "CI/CD-dev-env setup" in [continuous-integration](./continuous-integration.md). - -```bash -git tag v1.13.1 && git push origin v1.13.1 -``` - -## GitLab - releasing stage - -Creating a tag triggers the release process of the GitLab runners. -There exists a mirror of the GitHub repo on GitLab, but only when a tag is created on GitHub will the release part of the runners execute. You can check the status here: -https://gitlab.com/cryptoadvance/specter-desktop/-/pipelines - -There are three stages: -![](./images/release-guide/overview-gitlab-pipline.png) - -The first relevant stage is "releasing". Here, the Windows, Linux and pip release are created and uploaded to the Specter Desktop GitHub releases page. After this stage, the following artificats should be available: - -- cryptoadvance.specter-1.13.1.tar.gz -- Specter-Setup-v1.13.1.exe -- specterd-v1.13.1-win64.zip -- specterd-v1.13.1-x86_64-linux-gnu.zip -- specter_desktop-v1.13.1-x86_64-linux-gnu.tar.gz - -The three jobs in more detail: - -- release_binary_windows: is creating a binary for specterd and for Windows (Windows runner) -- release_electron_linux_windows: Creates a specterd for Linux, an AppImage for Linux and an executable for Windows (Linux runner). -- release_pip: Is releasing a pypi package on [pypi](https://pypi.org/project/cryptoadvance.specter/) and creates a tarball of the pip package for the GitHub release page (Linux runner). - -For details look at `.gitlab-ci.yml` - -## MacOS - -Ideally, directly after the tag is created, start with the MacOS release. As the binaries of x86/arm64 are not compatible with each other, we need to build on two MacOS architectures.This has to be done manually, for now. There is a script for this. Start with the build on x86: - -### MacOS x64 build - -```bash -./utils/build-osx.sh --version v2.0.5-pre4 specterd package upload -``` - -You can also test this procedure without messing the original project via changing the `orgName` to your `orgName` in `pyinstaller/electron/downloadloc.js`. - -This will create three artifacts on github: -* specterd-v2.0.5-pre4-osx_x64.zip -* SHA256SUMS-macos_x64 -* SHA256SUMS-macos_x64.asc - -### MacOS arm64 build - -The electron application will get built on the arm architecture. As it needs to store the sha256 hash in the electron-app, the make-hash target -will not only hash the specterd but also download the other specterd and hash it. - -```bash -./utils/build-osx.sh --version v2.0.5-pre4 --appleid "Satoshi Nakamoto (appleid)" --mail "satoshi@gmx.com" specterd make-hash electron sign package upload -``` - -This will create four artifacts on github: -* Specter-v2.0.5-pre4.dmg -* specterd-v2.0.5-pre4-osx_arm64.zip -* SHA256SUMS-macos_arm64 -* SHA256SUMS-macos_arm64.asc - - -## GitLab - post releasing - -Back to GitLab, the final stage is "post releasing". - -### release_signatures - -In this job, the individual SHA256-hashes and signatures are combined into two final files: - -- SHA256SUMS -- SHA256SUMS.asc - -Everything, apart from the MacOS files, are pulled from the GitLab environment, the MacOS files from GitHub. -Don't forget to delete the four MacOS files (`SHA256SUMS-macos_arm64` and `SHA256SUMS-macos_arm64.asc` and the two corresponding `_x64` files) on the GitHub release page in the end. - -This is difficult to automate as sometimes the manual steps has not succeeded while generating the SHASUM-files. As a result, those hashes are not included. So you might want to run this again. And you can, just delete the two generated files - `SHA256SUMS` and `SHA256SUMS.asc` and run the job again. - -### release_docker - -There are docker images created by the awesome [Chiang Mai LN dev](https://github.com/lncm/docker-specter-desktop). So the task of this job is to trigger their build-system which is done via `utils/trigger_docker_build.sh`. A prerequisite of this is a token in order to authenticate. That token is from Aaron, one of the maintainers of that repo, and can be found in the gitlab variables section of the CI/CD configuration. - -### tag_specterext_dummy_repo - -Sometimes there are changes on the plugin architecture. In order to create a plugin, it's quite important to know which version of the plugin system should be used. Because of that, we simply assume that the master of the [specterext-dummy](https://github.com/cryptoadvance/specterext-dummy) repo is compatible with the current master which was just tagged with the new version. -So this job will tag that repo with the same tag and the creation of a plugin will take the version into account. - -## Trouble shooting - -If the MacOS signatures are missing, it can happen that the following Exception will be raised: - -```bash - File "/builds/cryptoadvance/specter-desktop/utils/github.py", line 295, in download_artifact - raise Exception( - Exception: Status-cod04 for url ... ) -``` - -In any case, if the macOS binaries arrive on GitHub too late, you have to manually delete the already created `SHA256SUMS` and `SHA256SUMS.asc`, otherwise the upload to GitHub will fail if you rerun the release signatures job on GitLab - for details see ([this PR](https://github.com/cryptoadvance/specter-desktop/pull/689)). The green arrow in the screenshot is where you rerun the release signatures job on GitLab: - -![](./images/release-guide/rerun-release-signatures.png) - -## GitHub release page and download page - -This is handled by the script `./utils/generate_downloadpage.sh`. As a prerequisite, you need to clone the `specter-static` repo which contains the specter website. Clone it on the same level than specter-desktop. -Running that script will: - -- install the prerequisites (basically markdown, see pyproject.toml) -- generate the GH-page and the download-page based on the `utils/templates`. -- Asks whether it should replace/update/initialize the Github Release page for the latest version -- copies over the new download-pages and asks whether it should commit/push those +macOS builds are the most likely to fail due to Apple signing/notarization glitches. The workflow falls back to unsigned builds when `APPLE_CERTIFICATE_BASE64` is empty — useful for smoke-testing the pipeline on forks. diff --git a/pyinstaller/README.md b/pyinstaller/README.md index 2101072b6..22f24b288 100644 --- a/pyinstaller/README.md +++ b/pyinstaller/README.md @@ -1,14 +1,7 @@ -# Build scripts - -Run `build- ` file to build everything. - -For example, `build-osx.sh 1.2.3` will create `SpecterDesktop-1.2.3.dmg` and `specterd-1.2.3-osx.zip` in the `release` folder. - -If you're making a real release, you should append `"make hash"` at the end of your command calling the build script. -This will update the file hash and version name the Specter Desktop app expects to download from GitHub. - # Pyinstaller build +Releases are built by `.github/workflows/release.yml` (triggered by a version tag). The notes below are for local / manual builds. + Install requirements: ```bash @@ -52,17 +45,9 @@ If this is the first time you go through this process, you'll need to first set xcrun altool --store-password-in-keychain-item "AC_PASSWORD" -u "" -p "" ``` -After having these set up, you can use the automated script to sign by passing it 2 extra parameters: -- Your certificate name, which you can see on the Keychain app going to the sidebar -> `My Certificates` and copying the name of the certificate you've created in step 1. -- Your Apple ID. +Release builds sign and notarize via `.github/workflows/release.yml` (`build-electron-macos` job) using the `APPLE_CERTIFICATE_BASE64`, `APPLE_CERTIFICATE_PASSWORD`, `APPLE_ID`, `APPLE_APP_SPECIFIC_PASSWORD`, and `APPLE_TEAM_ID` secrets. See `docs/release-guide.md` for the full secret inventory. For manual local signing, use `electron-builder` directly (`npm run dist -- --mac` in `pyinstaller/electron/`, with the identity configured in `package.json`). -With these two, you can run the command like so: -```bash -./build-osx.sh "" "" "make-hash" -``` -*Note: "make-hash" is optional and will automatically calculate hash of specterd generated for the macOS app. Should be used only for real release.* - -This should take 10 minutes, during which you should receive an email from Apple notifying whatever the notarization was successful. +Notarization takes ~10 minutes, during which Apple emails notification of success/failure. If for some reason the notarization failed, you'll be able to get the reason by copying the `Request Identifier` (you should be able to find this in the email and in the logs). Then run the following command: ```bash diff --git a/pyinstaller/build-ci.sh b/pyinstaller/build-ci.sh deleted file mode 100755 index a4d63318d..000000000 --- a/pyinstaller/build-ci.sh +++ /dev/null @@ -1,67 +0,0 @@ -#!/usr/bin/env bash - -set -e - -# pass version number as an argument - -echo " --> This build got triggered for version $1" - -echo " --> Assumed gitlab-project: ${CI_PROJECT_ROOT_NAMESPACE:+x}" - -[ -z "${CI_PROJECT_ROOT_NAMESPACE:+x}" ] && \ - echo "Redefining CI_PROJECT_ROOT_NAMESPACE=cryptoadvance " && \ - export CI_PROJECT_ROOT_NAMESPACE=cryptoadvance - -echo $1 > version.txt -echo " --> Installing (build)-requirements" -pip3 install -r requirements.txt --require-hashes -cd .. -python3 setup.py install -pip3 install -e . -cd pyinstaller - -echo " --> Cleaning up" -rm -rf build/ dist/ release/ electron/release/ electron/dist release-linux/ release-win/ - -echo " --> Building specterd" -pyinstaller specterd.spec - -echo " --> Making us ready for building electron-app for linux" -cd electron -npm ci -node ./set-version $1 ../dist/specterd - -echo " --> building electron-app" -npm i -npm run dist -- --linux - -cd .. - -echo " --> Making the release-zip" -mkdir release-linux -cd dist -cp -r ../../udev ./udev -echo "Don't forget to set up udev rules! Check out udev folder for instructions." > README.md -zip -r ../release-linux/specterd-"$1"-"$(uname -m)"-linux-gnu.zip specterd udev README.md - -cp ../electron/dist/Specter-* ./ -tar -czvf ../release-linux/specter_desktop-"$1"-"$(uname -m)"-linux-gnu.tar.gz Specter-* udev README.md - -echo " --> Cleaning up" -cd .. -rm -rf dist -mkdir dist -cd dist -echo " --> Downloading the windows-version of specterd for version $1" -wget --progress=dot -e dotbytes=10M https://github.com/${CI_PROJECT_ROOT_NAMESPACE}/specter-desktop/releases/download/$1/specterd-$1-win64.zip -O ./specterd.zip -unzip specterd.zip -cd ../electron -rm -rf dist/ -echo " --> Making us ready for building electron-app for windows" -npm ci -node ./set-version $1 ../dist/specterd.exe -npm run dist -- --win -cd .. - -mkdir release-win -cp electron/dist/Specter\ Setup\ *.exe release-win/Specter-Setup-$1.exe diff --git a/pyinstaller/build-win-ci.bat b/pyinstaller/build-win-ci.bat deleted file mode 100755 index 21820e408..000000000 --- a/pyinstaller/build-win-ci.bat +++ /dev/null @@ -1,42 +0,0 @@ -@ECHO OFF - -python -V -pip3 install virtualenv -echo " --> cleaning up" -rmdir /s /q .\release\ -rmdir /s /q .\dist -rmdir /s /q .buildenv -echo " --> Creating virtualenv" -virtualenv --python=python3 .buildenv -echo " --> Activating virtualenv" -call .\.buildenv\Scripts\activate - -echo " --> Installing test-requirement" -pip3 install -e ".[test]" - -echo " --> Building pypi package" -pip3 install build==0.10.0 -python -m build - -echo " --> Installing pypi package" -python .\utils\release_helper.py install_wheel %1% - -cd pyinstaller - -Rem This file gets further packaged up with the pyinstaller and will help specter to figure out which version it's running on -echo %1% > version.txt -echo " --> installing pyinstaller requirements" -pip3 install -r requirements.txt --require-hashes - -rmdir /s /q .\dist\ -rmdir /s /q .\build\ -rmdir /s /q .\release\ -rmdir /s /q .\electron\dist\ - -echo " --> Creating the pyinstaller binary" -pyinstaller.exe specterd.spec - -mkdir release - -echo " --> Creating the release-package" -powershell Compress-Archive -Path dist\specterd.exe release\specterd-%1%-win64.zip diff --git a/pyinstaller/build-win.bat b/pyinstaller/build-win.bat deleted file mode 100644 index 99b8410c6..000000000 --- a/pyinstaller/build-win.bat +++ /dev/null @@ -1,31 +0,0 @@ -@ECHO OFF -echo %1% > version.txt -pip3 install -r requirements.txt --require-hashes -cd .. -Rem Order is relevant here. If you flip the followng lines, the hiddenimports for services won't work anymore -python3 setup.py install -pip3 install -e . -cd pyinstaller -rmdir /s /q .\dist\ -rmdir /s /q .\build\ -rmdir /s /q .\release\ -rmdir /s /q .\electron\dist\ -pyinstaller.exe specterd.spec -cd electron -call npm ci -if "%2%"=="make-hash" ( - call node ./set-version "%1%" "../dist/specterd.exe" -) else ( - node ./set-version "%1%" -) -call npm i -call npm run dist -cd .. - - -mkdir release -SET EXE_PATH="electron\dist\Specter Setup *.exe" -SET EXE_RELEASE_PATH="release\Specter Setup %1%.exe" -echo f | xcopy /s/y %EXE_PATH% %EXE_RELEASE_PATH% - -powershell Compress-Archive -Path dist\specterd.exe release\specterd-%1%-win64.zip diff --git a/pyproject.toml b/pyproject.toml index 7059bfcd1..faf36f277 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -82,7 +82,6 @@ test = [ "PySocks==1.7.1", "pytest-cov==2.10.1", "mock==4.0.2", - "python-gitlab==2.10.1", # requirements for stuff in ./utils "requests==2.31.0", ] diff --git a/src/cryptoadvance/specter/managers/service_manager/service_manager.py b/src/cryptoadvance/specter/managers/service_manager/service_manager.py index 150e16758..9bb37006e 100644 --- a/src/cryptoadvance/specter/managers/service_manager/service_manager.py +++ b/src/cryptoadvance/specter/managers/service_manager/service_manager.py @@ -464,7 +464,7 @@ class ExtensionManager: site_package = Path(virtuelenv_path, *(Path(site_package).parts[-3:-1])) virtualenv_search_path = site_package - # ... and as the classes are in the .buildenv (see build-unix.sh) let's add .. + # ... and as the classes are in the .buildenv let's add .. arr = [Path(virtualenv_search_path, path) for path in arr] # Non internal-repo extensions sitting in org/specterext/... need to be added, too diff --git a/utils/artifact_signer.sh b/utils/artifact_signer.sh deleted file mode 100755 index 92b5589af..000000000 --- a/utils/artifact_signer.sh +++ /dev/null @@ -1,81 +0,0 @@ -#!/bin/bash - -function sub_help { - echo "This script is to sign artifacts or to prepare the gpg-system to be able to verify artifacts." - echo "Do one of these:" - echo "$ ./utils/artifact_signer.sh init" - echo "This makes sense only on a gitlab-runner. It'll unpack a gpg-directory to be ready to sign and verify" - echo "$ ./utils/artifact_signer.sh sign --artifact ./release-win/SHA256SUMS-win" - echo "Signs a specific artifact. Will do the init on the fly. So no need to call it extra." -} - -while [[ $# -gt 0 ]] -do -key="$1" -command="main" -case $key in - --help) - sub_help - exit - shift - ;; - --artifact) - artifact=$2 - shift - shift - ;; - sign) - action=sign - shift - ;; - init) - action=init - shift - ;; - --debug) - set -x - shift # past argument - ;; - *) # unknown option - POSITIONAL="$1" # save it in an array for later - shift # past argument - ;; -esac -done - -# We want a detached signature in cleartext. Extension: .asc (as in bitcoin) -output_file=${artifact}.asc - -# lazy init: We're initializing Each thime script is called with these two things. -# So that action "init" is just to have a bit more semantics for the one calling this script - -function init { - if [[ -f /credentials/gnupg.tar.gz ]]; then - echo "Init: extracting gnupg.tar.gz" - tar -xzf /credentials/gnupg.tar.gz -C /root - chown -R root:root ~/.gnupg - else - echo "Init: Could not find any /credentials/gnupg.tar.gz" - fi - - if [[ -f /credentials/private.key ]]; then - echo "Init: Importing single private key" - gpg --import --no-tty --batch --yes /credentials/private.key - else - echo "Init: Could not find any /credentials/private.key" - fi -} - -if [ "$action" = "init" ]; then - init -fi - -if [ "$action" = "sign" ]; then - init - if [[ -z $artifact ]]; then - echo "no --artifact given " - exit 1 - fi - echo "signing ..." - echo $GPG_PASSPHRASE | gpg --detach-sign --armor --no-tty --batch --yes --passphrase-fd 0 --pinentry-mode loopback $artifact -fi \ No newline at end of file diff --git a/utils/build-common.sh b/utils/build-common.sh deleted file mode 100644 index 7b90d9f07..000000000 --- a/utils/build-common.sh +++ /dev/null @@ -1,135 +0,0 @@ -#!/usr/bin/env bash - -# All functions in here are responsible to change directory -# from the project root to wherever they want -# They need to change back to project-root when they finish - - -function create_virtualenv_for_pyinstaller { - echo " --> Creating new virtualsenv" - if [ -d .buildenv ]; then - echo " But first Delete it ..." - rm -rf .buildenv - fi - virtualenv --python=python3.10 .buildenv - source .buildenv/bin/activate - pip3 install -e ".[test]" -} - -function build_pypi_pckgs_and_install { - echo " --> Build pip3-package" - rm -rf dist - if ! git diff --quiet setup.py; then - echo "ERROR: setup.py is dirty, can't reasonably build" - exit 1 - fi - if [[ "$OSTYPE" == "darwin"* ]]; then - SML_ADD="\"\"" - fi - pip3 install build==0.10.0 - python3 -m build - pip3 install ./dist/cryptoadvance.specter-*.whl -} - -function configure { - echo " --> Configure some variables" - if [ -z "$app_name" ]; then - # activate virtualenv. This is e.g. not needed in CI - app_name=specter - specterd_filename=specterd - specterimg_filename=Specter - pkg_filename=specter_desktop - else - specterd_filename=${app_name}d # usually "specterd" - specterimg_filename=${app_name^} # usually "Specter" - pkg_filename=${app_name} - fi - - export ARCH=$(node -e "console.log(process.arch)") - export dist_mac_folder_name=mac-universal - export CI_COMMIT_TAG=$version - export CI_PROJECT_ROOT_NAMESPACE=$(node -e "const downloadloc = require('./pyinstaller/electron/downloadloc');console.log(downloadloc.orgName())") - - echo specterd_filename=${specterd_filename} - echo specterimg_filename=${specterimg_filename} - echo pkg_filename=${pkg_filename} - echo ARCH=$ARCH - echo dist_mac_folder_name=$dist_mac_folder_name - echo CI_COMMIT_TAG=$CI_COMMIT_TAG - echo CI_PROJECT_ROOT_NAMESPACE=$CI_PROJECT_ROOT_NAMESPACE - -} - - -function install_build_requirements { - - echo " --> Installing pyinstaller build-requirements" - cd pyinstaller - pip3 install -r requirements.txt --require-hashes > /dev/null - - cd .. -} - -function cleanup { - echo " --> Cleaning up" - cd pyinstaller - rm -rf build/ dist/ release/ electron/release/ electron/dist - rm *.dmg || true - cd .. -} - -function building_app { - echo " --> Building ${specterd_filename}" - cd pyinstaller - specterd_filename=${specterd_filename} pyinstaller specterd.spec > /dev/null - cd .. -} - -function prepare_npm { - cd pyinstaller/electron - echo " --> Making us ready for building electron-app" - npm ci - cd ../.. -} - -function make_hash_if_necessary { - cd pyinstaller/electron - echo " --> calculate the hash of the binary for download" - if [[ "$1" = "win" ]]; then - specterd_plt_filename=../dist/${specterd_filename}.exe - else - specterd_plt_filename=../dist/${specterd_filename} - fi - if [[ "$make_hash" == 'True' ]] - then - node ./set-version $version ${specterd_plt_filename} - else - node ./set-version $version - fi - echo " Hash in version -data.json $(cat ./version-data.json | jq -r '.sha256')" - echo " Hash of file $(sha256sum ${specterd_plt_filename} )" - cd ../.. -} - -function building_electron_app { - # https://www.electron.build/ - # Prerequisites: - # * A developer Certificate (in the System keychain) - # * private and public key in the login-keychain - # * The cert needs to be referenced in pyinstaller/electron/package.json -> build.mac.identity - - - platform="-- --${1}" # either linux or win (maxOS is empty) - cd pyinstaller/electron - echo " --> building electron-app" - echo " --> Copying over resources" - cp -R ../../src/cryptoadvance/specter/static/fonts ../../src/cryptoadvance/specter/static/output.css ../../src/cryptoadvance/specter/static/typography.css . - npm i - npm run dist ${platform} - cd ../.. -} - -function make_release_zip { - echo " --> Making the release-zip" - -} diff --git a/utils/build-osx.sh b/utils/build-osx.sh deleted file mode 100755 index fcf35856c..000000000 --- a/utils/build-osx.sh +++ /dev/null @@ -1,404 +0,0 @@ -#!/usr/bin/env bash -set -e - -# We start in the directory where this script is located -cd "$( dirname "${BASH_SOURCE[0]}" )/." -source build-common.sh -cd .. -# Now in project-root - -# Overriding this function -function create_virtualenv_for_pyinstaller { - # This currently assumes to be run with: Python 3.10.11 - # Important: pyinstaller needs a Python binary with shared library files - # With pyenv, for example, you get this like so: env PYTHON_CONFIGURE_OPTS="--enable-shared" pyenv install 3.10.4 - # Use pyenv if set as environment variable - if [ $USE_PYENV_FOR_SPECTER_BUILD = true ]; then - echo "Trying to use pyenv ..." - if ! command -v pyenv >/dev/null 2>&1; then - echo "Error: pyenv is not available. Please make sure pyenv is installed and configured properly." >&2 - exit 1 - fi - ### This is usually in .zshrc, putting it in .bashrc didn't work ### - export PYENV_ROOT="$HOME/.pyenv" - command -v pyenv >/dev/null || export PATH="$PYENV_ROOT/bin:$PATH" - eval "$(pyenv init -)" - ### this needs the pyenv-virtualenv plugin. If you don't have it: - ### git clone https://github.com/pyenv/pyenv-virtualenv.git $(pyenv root)/plugins/pyenv-virtualenv - eval "$(pyenv virtualenv-init -)" - ### ------------------------------------------------------------ ### - PYTHON_VERSION=3.10.11 - export PYENV_VERSION=$PYTHON_VERSION - echo "Setting PYENV_VERSION to 3.10.11, using pyenv-virtualenv to create the buildenv..." - echo " --> Deleting .buildenv" - pyenv uninstall -f .buildenv - rm -rf "$HOME/.pyenv/versions/$PYTHON_VERSION/envs/.buildenv" - pyenv virtualenv 3.10.11 .buildenv - pyenv activate .buildenv - else - echo "pyenv is not available. Using system Python version." - if [ -d .buildenv ]; then - echo " --> Deleting .buildenv" - rm -rf .buildenv - fi - virtualenv .buildenv - source .buildenv/bin/activate - fi - pip3 install -e ".[test]" -} - -# Overriding this function to deal with the x86 special case -function make_hash_if_necessary { - cd pyinstaller/electron - echo " --> calculate the hash of the binary for download" - specterd_plt_filename=../dist/${specterd_filename} - # early exit - if [[ "$make_hash" != 'True' ]]; then - node ./set-version $version - return 0 - fi - - # We need to set-versions for two specterd, one arm and one intel. - # arm64 one - node ./set-version $version ${specterd_plt_filename} - # Download and check the intel one - # this needs some env-vars to be set - rm -rf signing_dir/* - PYTHONPATH=../.. python3 -m utils.release_helper downloadgithub - ret_code=$? - if [ $ret_code -ne 0 ]; then - echo "Downloading and verifying x64 specterd failed with exit code $ret_code" - exit $ret_code - fi - if [[ ! -f ./signing_dir/specterd-${version}-osx_x64.zip ]]; then - echo "Downloading and verifying x64 specterd failed as the file does not seem to be there" - exit 1 - fi - rm -f /tmp/specterd - unzip ./signing_dir/specterd-${version}-osx_x64.zip -d /tmp - node ./set-version $version /tmp/specterd x64 - echo " Hashes in version-data.json $(cat ./version-data.json | jq -r '.sha256')" - echo " Hash of file $(sha256sum ${specterd_plt_filename} )" - echo " Hash of x64 file $(sha256sum /tmp/specterd )" - cd ../.. -} - -function macos_code_sign { - # prerequisites for this: - # in short: - # * make sure you have a proper app-specific password on https://appleid.apple.com/account/manage - # * collect some information via scrun altool --list-providers -u "" - # * create profile via xcrun notarytool store-credentials --apple-id "" --password "app-specific-pw" --team-id "seeFromAbove" - # * Call the profile: SpecterProfile - # For details see: - # * https://www.youtube.com/watch?v=2xJcMzoi0EI - # * https://blog.dgunia.de/2022/09/01/switching-from-altool-to-notarytool/ - # * https://scriptingosx.com/2021/07/notarize-a-command-line-tool-with-notarytool/ - # This creates a ZIP archive from the app package (using the ditto command). - # This ZIP archive is then used to upload the app to the Apple notarization service via xcrun notarytool (formerly xcrun altool) - # After the app has been uploaded to the Apple servers and notarized, the ZIP archive is not used again. - # The function uses the xcrun stapler command to attach the notarization result to the app, and then exits. - - # docs: - # https://help.apple.com/itc/apploader/#/apdATD1E53-D1E1A1303-D1E53A1126 - # https://keith.github.io/xcode-man-pages/altool.1.html - cd pyinstaller/electron - echo ' --> Attempting to code sign...' - specterimg_filename_fqfn=dist/${dist_mac_folder_name}/${specterimg_filename}.app - echo " executing: ditto -c -k --keepParent "${specterimg_filename_fqfn}" dist/${specterimg_filename}.zip" - - ditto -c -k --keepParent "${specterimg_filename_fqfn}" dist/${specterimg_filename}.zip - # upload - echo ' uploading for notarisation ... ' - - output_json=$(xcrun notarytool submit dist/${specterimg_filename}.zip --apple-id "kneunert@gmail.com" --keychain-profile "SpecterProfile" --output-format json --wait ) - - - # parsing the requestuuid which we'll need to track progress - requestuuid=$(echo $output_json | jq -r '.id') - status=$(echo $output_json | jq -r '.status') - echo "Request ID: $requestuuid" - if [ "$status" = "Invalid" ]; then - mkdir -p signing_logs - echo "issues with notarisation" - xcrun notarytool log ${requestuuid} --keychain-profile SpecterProfile | tee ./signing_logs/${app_name}_${timestamp}_${requestuuid}.log - exit 1 - fi - - # The stapler somehow "staples" the result of the notarisation in to your app - # see e.g. https://stackoverflow.com/questions/58817903/how-to-download-notarized-files-from-apple - echo " --> Staple the file dist/${dist_mac_folder_name}/${specterimg_filename}.app" - xcrun stapler staple "dist/${dist_mac_folder_name}/${specterimg_filename}.app" - echo ' Successfully Stapled the file' - cd ../.. -} - - -function sub_help { - cat << EOF - -### Quick overview -: <<'END_COMMENT' - What do you need to sign the Specter app with Apple's notary service? - - An Apple Developer account - - You must create a signing certificate in your developer account, which will be used to sign your app. - - This certificate must be stored in your keychain on your Mac. - - When you create a signing certificate in your developer account, you will be asked to specify a password for the certificate. - - You can store this password in the keychain, too, so that it - and thus the certificate - can be accessed automatically during the signing process. Like so: - xcrun altool --store-password-in-keychain-item AC_PASSWORD -u '' -p apassword - - As seen above, you need the the xcrun command line tool: This tool is also used to upload your app to the notary service and check the status of the notarization process. - -In summary, to sign a macOS app with Apple's notary service, you need an Apple Developer account, a signing certificate, a password for your keychain, the app package to be signed, and the xcrun command line tool. -END_COMMENT - -### Prerequisites -# brew install gmp # to prevent module 'embit.util' has no attribute 'ctypes_secp256k1' -# brew install jq -# npm install --global create-dmg - -### Trouble shooting -# If you have the common issue "errSecInternalComponent" while signing the code: -# https://medium.com/@ceyhunkeklik/how-to-fix-ios-application-code-signing-error-4818bd331327 - -# create-dmg issue? Note that there are 2 create-dmg scripts out there. We use: -# https://github.com/sindresorhus/create-dmg - -The different "tasks" are now somehow separated from one another. -We have: -* make-hash is rather a flag for the electron-build to incorporate the hash of the specterd -* specterd will trigger the pyinstaller build of the specterd -* electron will build the electron-app -* sign will upload the electron-app to the Apple notary service and get it back notarized -* upload will upload all the binary artifacts to the github-release-page. This includes the creation of the hash-files - and the gnupg signing - -### Trouble shooting (Legacy) -# Currently, only MacOS Catalina is supported to build the dmg-file -# Therefore we expect xcode 12.1 (according to google) -# After installation of xcode: sudo xcode-select -s /Applications/Xcode.app/Contents/Developer -# otherwise you get xcrun: error: unable to find utility "altool", not a developer tool or in PATH -# catalina might have a a too old version of bash. You need at least 4.0 or so -# 3.2 is too low definitely -# brew install bash - -# Example-call: -./utils/build-osx.sh --debug --version v1.10.0-pre23 --appleid "Kim Neunert (FWV59JHV83)" --mail "kim@specter.solutions" make-hash specterd electron sign upload -EOF -} - -appleid="" - -while [[ $# -gt 0 ]] - do - arg="$1" - case $arg in - "" | "-h" | "--help") - sub_help - exit 0 - shift - ;; - --debug) - set -x - DEBUG=true - shift - ;; - --version) - version=$2 - shift - shift - ;; - --appleid) - appleid=$2 - shift - shift - ;; - --mail) - mail=$2 - shift - shift - ;; - specterd) - build_specterd=True - shift - ;; - make-hash) - make_hash=True - shift - ;; - electron) - build_electron=True - shift - ;; - sign) - build_sign=True - shift - ;; - package) - build_package=True - shift - ;; - upload) - upload=True - shift - ;; - help) - sub_help - shift - ;; - *) - shift - sub_${arg} $@ && ret=0 || ret=$? - if [ "$ret" = 127 ]; then - echo "Error: '$arg' is not a known subcommand." >&2 - echo " Run '$progname --help' for a list of known subcommands." >&2 - exit 1 - else - exit $ret_value - fi - ;; - esac - done - -echo " --> This build got triggered for version $version" - -echo $version > pyinstaller/version.txt - -configure - -if [[ "$build_specterd" = "True" ]]; then - create_virtualenv_for_pyinstaller - build_pypi_pckgs_and_install - install_build_requirements - cleanup - building_app -fi - -if [[ "$make_hash" = "True" ]]; then - # Making the hash only makes sense on a arm arch - if [[ "$ARCH" != "arm64" ]]; then - echo "ERROR: make-hash target should be only called on an arm64 machine on a mac" - exit 1 - fi - make_hash_if_necessary -fi - -if [[ "$build_electron" = "True" ]]; then - # Making the hash only makes sense on a arm arch - if [[ "$ARCH" != "arm64" ]]; then - echo "ERROR: electron target should be only called on an arm64 machine on a mac" - exit 1 - fi - prepare_npm - npm i - if [[ "${appleid}" == '' ]] - then - echo "`jq '.build.mac.identity=null' package.json`" > package.json - else - echo "`jq '.build.mac.identity="'"${appleid}"'"' package.json`" > package.json - fi - building_electron_app -fi - -if [[ "$build_sign" = "True" ]]; then - # if [ "$(uname -m)" = "arm64" ]; then - # dist_mac_folder_name=${dist_mac_folder_name}-arm64 - # fi - if [[ "$appleid" != '' ]]; then - macos_code_sign - else - echo "WARNING: Forgot to add the appleid ?!" - exit 1 - fi -fi - -if [[ "$build_package" = "True" ]]; then - echo " --> Preparing the release" - mkdir -p release - rm -rf release/* - - # The specterd-zipfile from specterd - if [[ -f pyinstaller/dist/${specterd_filename} ]]; then - echo " --> Making the release-zip for specterd" - pushd pyinstaller/dist # to not preserve folder structure - zip ../../release/${specterd_filename}-${version}-osx_${ARCH}.zip ${specterd_filename} - popd - fi - - # The dmg image file from App - if [[ -d pyinstaller/electron/dist/${dist_mac_folder_name}/${specterimg_filename}.app ]]; then - rm -f pyinstaller/electron/dist/*.dmg - echo " --> Creating dmg" - create-dmg pyinstaller/electron/dist/${dist_mac_folder_name}/${specterimg_filename}.app --identity="Developer ID Application: ${appleid}" pyinstaller/electron/dist - # create-dmg doesn't create the prepending "v" to the version - node_comp_version=$(python3 -c "print('$version'[1:])") - mv "pyinstaller/electron/dist/${specterimg_filename} ${node_comp_version}.dmg" dist/${specterimg_filename}-${version}.dmg - echo " --> Copying img file dist/${specterimg_filename}-${version}.dmg" - cp dist/${specterimg_filename}-${version}.dmg release/${specterimg_filename}-${version}.dmg - else - echo "WARNING: Skipping packaging for electron App" - echo "No pyinstaller/electron/dist/${dist_mac_folder_name}/${specterimg_filename}.app has been found." - fi - - file=./release/${specterd_filename}-${version}-osx_${ARCH}.zip - if [[ -f $file ]]; then - echo -n " FYI : " - sha256sum $file - fi - file=./release/${specterimg_filename}-${version}.dmg - if [[ -f $file ]]; then - echo -n " FIY : " - sha256sum $file - fi -fi - -if [ "$app_name" != "specter" ]; then - # "early" exit - if [[ "$upload" = "True" ]]; then - echo "no upload for app_name $app_name" - exit 1 - fi - exit -fi - -if [[ "$upload" = "True" ]]; then - echo " --> gpg-signing the hashes and uploading" - . ../../specter_gh_upload.sh # A simple file looks like: export GH_BIN_UPLOAD_PW=...(GH token) - export CI_COMMIT_TAG=$version - if [[ -z "$CI_PROJECT_ROOT_NAMESPACE" ]]; then - echo "WARNING: Why is CI_PROJECT_ROOT_NAMESPACE not set? Setting to cryptoadvance" - export CI_PROJECT_ROOT_NAMESPACE=cryptoadvance - fi - echo " This build: version: $version gh-project: $CI_PROJECT_ROOT_NAMESPACE" - - specterd_zip_fqfn=./release/specterd-${version}-osx_${ARCH}.zip - echo " Checking for file $specterd_zip_fqfn" - if [[ -f $specterd_zip_fqfn ]]; then - python3 ./utils/github.py upload $specterd_zip_fqfn - else - echo " WARNING: not uploading as it does not exist: $specterd_zip_fqfn" - fi - - specter_dmg_fqfn=./release/Specter-${version}.dmg - echo " Checking for file $specter_dmg_fqfn" - if [[ -f $specter_dmg_fqfn ]]; then - python3 ./utils/github.py upload $specter_dmg_fqfn - else - echo " WARNING: not uploading as it does not exist: $specter_dmg_fqfn" - fi - - cd release - # Maybe we have some SHA256SUMS files from other runs lying around. We don't want to shasum them - rm -f SHA256SUMS* - sha256sum * > SHA256SUMS-macos_${ARCH} - python3 ../utils/github.py upload SHA256SUMS-macos_${ARCH} - # The GPG comman below has a timeout. If that's reached, the script will interrupt. So let's make some noise - say "Hello?! Your overlord is speaking! You're now allowed to sign the binary!" - echo "Just in case you missed the timeout, those three last commands are missing:" - echo "cd release" - echo "gpg --detach-sign --armor SHA256SUMS-macos_${ARCH}" - echo "python3 ../utils/github.py upload SHA256SUMS-macos_${ARCH}.asc" - - gpg --detach-sign --armor SHA256SUMS-macos_${ARCH} - python3 ../utils/github.py upload SHA256SUMS-macos_${ARCH}.asc -fi - diff --git a/utils/build-unix.sh b/utils/build-unix.sh deleted file mode 100755 index 1d3722387..000000000 --- a/utils/build-unix.sh +++ /dev/null @@ -1,169 +0,0 @@ -#!/usr/bin/env bash -set -e - -# We start in the directory where this script is located -cd "$( dirname "${BASH_SOURCE[0]}" )/." -source build-common.sh -cd .. -# Now in project-root - -function sub_help { - cat << EOF - -Building various components of specter-desktop -Usage: $build-unix [options] - -Options: - --debug - will set -x - --version v1.2.3-pre4 - If you don't set the version, CI_COMMIT_TAG will determine the version - -Subcommands: - make-hash - will make the hash for the electron-app. This hash will get checked after download - - specterd - will build the pyinstaller's specterd binary (linux only) - - electron-linux - will build the linux binary of the electron-app - - electron-win - will build the win binary of the electron-app. The specterd.exe will get downloaded from - github.com/\$CI_PROJECT_ROOT_NAMESPACE/specter-desktop... - This need a wine-environment. See the docker-image electron-builder - -Example-call: -./build-unix.sh --debug --version v1.7.0-pre1 make-hash specterd electron-linux -EOF -} - -function create_release_zip_linux { - echo " --> Making the release-zip" - # consists of specterd and Specter-version.AppImage - mkdir -p release - # first the specterd - cd pyinstaller/dist - cp -r ../../udev ./udev - echo "Don't forget to set up udev rules! Check out udev folder for instructions." > README.md - zip -r ../../release/${specterd_filename}-"$version"-"$(uname -m)"-linux-gnu.zip ${specterd_filename} udev README.md - echo $app_name - # now the AppImage - cd ../electron/dist - cp -r ../../../udev ./udev - echo "Don't forget to set up udev rules! Check out udev folder for instructions." > README.md - tar -czvf ../../../release/${pkg_filename}-"$version"-"$(uname -m)"-linux-gnu.tar.gz ${app_name^}-* udev README.md - cd ../../.. -} - -function prepare_building_electron_app_win { - cd pyinstaller/dist - echo " --> Downloading the windows-version of specterd for version $version" - wget --progress=dot -e dotbytes=10M https://github.com/${CI_PROJECT_ROOT_NAMESPACE}/specter-desktop/releases/download/${version}/specterd-${version}-win64.zip -O ./specterd.zip - unzip specterd.zip - cd ../electron - rm -rf dist/ - cd ../.. -} - -version=$CI_COMMIT_TAG - -echo " --> Assume gitlab-project: ${CI_PROJECT_ROOT_NAMESPACE}" - -[ -z "${CI_PROJECT_ROOT_NAMESPACE:+x}" ] && \ - echo " Redefining CI_PROJECT_ROOT_NAMESPACE=cryptoadvance " && \ - export CI_PROJECT_ROOT_NAMESPACE=cryptoadvance - -while [[ $# -gt 0 ]] - do - arg="$1" - case $arg in - "" | "-h" | "--help") - sub_help - exit 0 - shift - ;; - --debug) - set -x - DEBUG=true - shift - ;; - --version) - version=$2 - shift - shift - if [ -n "$CI_COMMIT_TAG" ]; then - if [ "$version" != "$CI_COMMIT_TAG" ]; then - echo "ERROR: Cannot set version to something different than CI_COMMIT_TAG env-var if that var is set. " - exit 1 - fi - fi - ;; - specterd) - build_specterd=True - shift - ;; - make-hash) - make_hash=True - shift - ;; - electron-linux) - build_electron_linux=True - shift - ;; - electron-win) - build_electron_win=True - shift - ;; - help) - sub_help - shift - ;; - *) - shift - sub_${arg} $@ && ret=0 || ret=$? - if [ "$ret" = 127 ]; then - echo "Error: '$arg' is not a known subcommand." >&2 - echo " Run '$progname --help' for a list of known subcommands." >&2 - exit 1 - else - exit $ret_value - fi - ;; - esac - done - -if [[ "$version" = "" ]]; then - echo "ERROR: version could not be determined (--version or CI_COMMIT_TAG)" - exit 1 -fi -echo " --> This build got triggered for version $version" -# This file gets further packaged up with the pyinstaller and will help specter to figure out which version it's running on -echo $version > pyinstaller/version.txt - -configure - -if [[ "$build_specterd" = "True" ]]; then - create_virtualenv_for_pyinstaller - build_pypi_pckgs_and_install - install_build_requirements - cleanup - building_app -fi - -if [[ "$build_electron_linux" = "True" ]]; then - prepare_npm - make_hash_if_necessary - building_electron_app linux - create_release_zip_linux -fi - -if [ "$build_electron_win" = "True" ]; then - prepare_building_electron_app_win - make_hash_if_necessary win - building_electron_app win - cp pyinstaller/electron/dist/Specter\ Setup\ *.exe release/Specter-Setup-$version.exe -fi - - diff --git a/utils/create-gitlab-cli-cfg.sh b/utils/create-gitlab-cli-cfg.sh deleted file mode 100755 index 399b239d1..000000000 --- a/utils/create-gitlab-cli-cfg.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/bin/bash - -cat > ~/.python-gitlab.cfg << EOF -[global] -default = specterdesktop -ssl_verify = true -timeout = 5 - -[specterdesktop] -url = https://gitlab.com -#private_token = ${CI_JOB_TOKEN} -job_token =${CI_JOB_TOKEN} -api_version = 4 - - -EOF diff --git a/utils/github.py b/utils/github.py deleted file mode 100644 index 181264acc..000000000 --- a/utils/github.py +++ /dev/null @@ -1,532 +0,0 @@ -""" We assume that this script is running on a gitlab-runner and therefore has some variables set. - Specifically: - CI_PROJECT_ROOT_NAMESPACE=k9ert - CI_COMMIT_TAG=v0.9.6-pre2 - - -""" - -import logging -import os -import sys -from pathlib import Path -import requests - -import argparse -import collections -import getpass -import json -import logging -import os -import re -import subprocess -import sys -from typing import ( - cast, - Any, - Callable, - List, - Optional, -) # noqa: F401 # pylint: disable=unused-import - -try: - # Allow an import of this module without `requests` and `yacl` being installed for meta data queries - # (e.g. version information) - import requests - from yacl import setup_colored_stderr_logging -except ImportError: - pass - - -logging.basicConfig(format="%(levelname)s:%(message)s", level=logging.INFO) -logger = logging.getLogger(__name__) -logger.setLevel(logging.DEBUG) - - -github_api_root_url = f"https://api.github.com" - -github_username = "gitlab_upload_release_binaries" - - -def main(): - if sys.argv[1] != "upload": - # Maybe something more fancy in the future: - logger.error("Command {sys.argv[1]} not found! Only 'upload' right now") - exit(2) - - artifact = sys.argv[2] - if not Path(artifact).exists(): - logger.error(f"local artifact {artifact} does not exist.") - exit(2) - if not "CI_PROJECT_ROOT_NAMESPACE" in os.environ: - logger.error("CI_PROJECT_ROOT_NAMESPACE not found") - exit(2) - else: - project = f"{os.environ['CI_PROJECT_ROOT_NAMESPACE']}/specter-desktop" - if not "CI_COMMIT_TAG" in os.environ: - logger.error("CI_COMMIT_TAG not found") - exit(2) - else: - tag = os.environ["CI_COMMIT_TAG"] - if len(sys.argv) != 3: - logger.error("argument artifact not found.") - if artifact_exists(project, tag, Path(artifact).name): - logger.info("Github artifact existing. Skipping upload.") - exit(0) - else: - logger.info(f"Github artifact {artifact} does not exist. Let's upload!") - - if not "GH_BIN_UPLOAD_PW" in os.environ: - logger.error("GH_BIN_UPLOAD_PW not found.") - else: - password = os.environ["GH_BIN_UPLOAD_PW"] - - publish_release_from_tag( - project, - tag, - [artifact], - github_username, - password, - ) - - -def artifact_exists(project, tag, artifact): - artifact_url = f"https://github.com/{project}/releases/download/{tag}/{artifact}" - logger.debug(f"checking for artifact url {artifact_url}") - r = requests.head(artifact_url) - if r.status_code != 302: - return False - else: - return True - - -__copyright__ = "Copyright © 2019 Forschungszentrum Jülich GmbH. All rights reserved." -__license__ = "MIT" -__version_info__ = (0, 1, 5) -__version__ = ".".join(map(str, __version_info__)) - - -DEFAULT_GITHUB_ROOT = "github.com" - - -class MissingDependencyError(Exception): - pass - - -class FileCommandError(Exception): - pass - - -class InvalidFileCommandOutputError(Exception): - pass - - -class NoTagsAvailableError(Exception): - pass - - -class HTTPError(Exception): - pass - - -class JSONError(Exception): - pass - - -class InvalidUploadUrlError(Exception): - pass - - -class InvalidServerNameError(Exception): - pass - - -class MissingProjectError(Exception): - pass - - -class MissingTagError(Exception): - pass - - -class CredentialsReadError(Exception): - pass - - -class AttributeDict(dict): # type: ignore - def __getattr__(self, attr: str) -> Any: - return self[attr] - - def __setattr__(self, attr: str, value: Any) -> None: - self[attr] = value - - -Release = collections.namedtuple("Release", ["id", "asset_upload_url"]) -Asset = collections.namedtuple("Asset", ["id", "name"]) - - -def setup_stderr_logging() -> None: - logging.basicConfig(level=logging.INFO) - setup_colored_stderr_logging(format_string="[%(levelname)s] %(message)s") - - -def get_mimetype(filepath: str) -> str: - if not os.path.isfile(filepath): - raise FileNotFoundError( - 'The file "{}" does not exist or is not a regular file.'.format(filepath) - ) - if not os.access(filepath, os.R_OK): - raise PermissionError('The file "{}" is not readable.'.format(filepath)) - - if os.name == "nt": - try: - import mimetypes - - mime_type = mimetypes.types_map[f".{filepath.split('.')[-1]}"] - except ModuleNotFoundError: - raise Exception( - "mimetypes module not found. Do something like pip install mimetypes" - ) - else: - try: - file_command_output = subprocess.check_output( - ["file", "--mime", filepath], universal_newlines=True - ) # type: str - mime_type = file_command_output.split()[1][:-1] - except subprocess.CalledProcessError as e: - raise FileCommandError( - "The `file` command returned with exit code {:d}".format(e.returncode) - ) - except IndexError: - raise InvalidFileCommandOutputError( - 'The file command output "{}" could not be parsed.'.format( - file_command_output - ) - ) - return mime_type - - -def strip_asset_upload_url(asset_upload_url_with_get_params: str) -> str: - match_obj = re.match(r"([^{]+)(?:\{.*\})?", asset_upload_url_with_get_params) - if not match_obj: - raise InvalidUploadUrlError( - 'The upload url "{}" is not in the expected format.'.format( - asset_upload_url_with_get_params - ) - ) - asset_upload_url = match_obj.group(1) # type: str - return asset_upload_url - - -class GithubConnection: - def __init__(self, project): - self.github_api_root_url = github_api_root_url - self.project = project - self.username = github_username - self.password = os.environ["GH_BIN_UPLOAD_PW"] - - def fetch_existing_release(self, tag) -> Optional[Release]: - try: - release_query_url = "{}/repos/{}/releases/tags/{}".format( - self.github_api_root_url, self.project, tag - ) - response = requests.get( - release_query_url, - auth=(self.username, self.password), - headers={"Accept": "application/json"}, - ) - response.raise_for_status() - logger.info( - 'Fetched the existing release "%s" in the GitHub repository "%s"', - tag, - self.project, - ) - response_json = response.json() - asset_upload_url_with_get_params = response_json["upload_url"] - asset_upload_url = strip_asset_upload_url(asset_upload_url_with_get_params) - release = Release(response_json["id"], asset_upload_url) - return release - except requests.HTTPError as e: - if e.response.status_code == 404: - return None - raise HTTPError( - 'Could not fetch the release "{}" due to a severe HTTP error.'.format( - tag - ) - ) - - def list_assets(self, release: Release) -> List[Asset]: - try: - asset_list_url = "{}/repos/{}/releases/{}/assets".format( - self.github_api_root_url, self.project, release.id - ) - response = requests.get(asset_list_url, auth=(self.username, self.password)) - response.raise_for_status() - assets = [ - Asset(asset_dict["id"], asset_dict["name"]) - for asset_dict in response.json() - ] - return assets - except requests.HTTPError: - raise HTTPError( - 'Could not get a list of assets for project "{}".'.format(self.project) - ) - except json.decoder.JSONDecodeError: - raise JSONError("Got an invalid json string.") - except KeyError as e: - raise JSONError( - 'Got an unexpected json object missing the key "{}".'.format(e.args[0]) - ) - - def download_artifact(self, tag, artifact, target_dir="."): - artifact_url = ( - f"https://github.com/{self.project}/releases/download/{tag}/{artifact}" - ) - response = requests.get(artifact_url) - - # If the HTTP GET request can be served - if response.status_code == 200: - - # Write the file contents in the response to a file specified by local_file_path - with open(os.path.join(target_dir, artifact), "wb") as local_file: - for chunk in response.iter_content(chunk_size=128): - local_file.write(chunk) - else: - raise Exception( - f"Status-code {response.status_code} for url {artifact_url}" - ) - - -def publish_release_from_tag( - project: str, - tag: Optional[str], - asset_filepaths: List[str], - username: str, - password: str, - dry_run: bool = False, -) -> None: - if "requests" not in sys.modules: - raise MissingDependencyError( - 'The "requests" package is missing. Please install and run again.' - ) - - def fetch_latest_tag() -> str: - try: - tags_url = "{}/repos/{}/tags".format(github_api_root_url, project) - response = requests.get( - tags_url, - auth=(username, password), - headers={"Accept": "application/json"}, - ) - response.raise_for_status() - tags = response.json() - if not tags: - raise NoTagsAvailableError( - 'The given repository "{}" has no tags yet.'.format(project) - ) - latest_tag = tags[0]["name"] # type: str - logger.info( - 'Fetched the latest tag "%s" from the GitHub repository "%s"', - latest_tag, - project, - ) - return latest_tag - except requests.HTTPError: - raise HTTPError( - 'Could not query the latest tag of the repository "{}" due to a http error.'.format( - project - ) - ) - except (json.decoder.JSONDecodeError, IndexError): - raise JSONError("Got an invalid json string.") - except KeyError as e: - raise JSONError( - 'Got an unexpected json object missing the key "{}".'.format(e.args[0]) - ) - - def publish_release(tag: str) -> Release: - def fetch_existing_release() -> Optional[Release]: - try: - release_query_url = "{}/repos/{}/releases/tags/{}".format( - github_api_root_url, project, tag - ) - response = requests.get( - release_query_url, - auth=(username, password), - headers={"Accept": "application/json"}, - ) - response.raise_for_status() - logger.info( - 'Fetched the existing release "%s" in the GitHub repository "%s"', - tag, - project, - ) - response_json = response.json() - asset_upload_url_with_get_params = response_json["upload_url"] - asset_upload_url = strip_asset_upload_url( - asset_upload_url_with_get_params - ) - release = Release(response_json["id"], asset_upload_url) - return release - except requests.HTTPError as e: - if e.response.status_code == 404: - return None - raise HTTPError( - 'Could not fetch the release "{}" due to a severe HTTP error.'.format( - tag - ) - ) - - def create_release() -> Release: - try: - release_creation_url = "{}/repos/{}/releases".format( - github_api_root_url, project - ) - response = requests.post( - release_creation_url, - auth=(username, password), - json={ - "tag_name": tag, - "name": tag, - "body": "", - "draft": False, - "prerelease": False, - }, - ) - response.raise_for_status() - logger.info( - 'Created the release "%s" in the GitHub repository "%s"', - tag, - project, - ) - response_json = response.json() - asset_upload_url_with_get_params = response_json["upload_url"] - asset_upload_url = strip_asset_upload_url( - asset_upload_url_with_get_params - ) - release = Release(response_json["id"], asset_upload_url) - return release - except requests.HTTPError: - raise HTTPError('Could not create the release "{}".'.format(tag)) - except json.decoder.JSONDecodeError: - raise JSONError("Got an invalid json string.") - except KeyError as e: - raise JSONError( - 'Got an unexpected json object missing the key "{}".'.format( - e.args[0] - ) - ) - - release = fetch_existing_release() - if release is None: - release = create_release() - return release - - def list_assets(release: Release) -> List[Asset]: - try: - asset_list_url = "{}/repos/{}/releases/{}/assets".format( - github_api_root_url, project, release.id - ) - response = requests.get(asset_list_url, auth=(username, password)) - response.raise_for_status() - assets = [ - Asset(asset_dict["id"], asset_dict["name"]) - for asset_dict in response.json() - ] - return assets - except requests.HTTPError: - raise HTTPError( - 'Could not get a list of assets for project "{}".'.format(project) - ) - except json.decoder.JSONDecodeError: - raise JSONError("Got an invalid json string.") - except KeyError as e: - raise JSONError( - 'Got an unexpected json object missing the key "{}".'.format(e.args[0]) - ) - - def delete_asset(asset: Asset) -> None: - try: - asset_delete_url = "{}/repos/{}/releases/assets/{}".format( - github_api_root_url, project, asset.id - ) - response = requests.delete(asset_delete_url, auth=(username, password)) - response.raise_for_status() - logger.info( - 'Deleted the asset "%s" attached to release "%s" of the GitHub repository "%s"', - asset.name, - tag, - project, - ) - except requests.HTTPError: - raise HTTPError( - 'Could not get a list of assets for project "{}".'.format(project) - ) - except json.decoder.JSONDecodeError: - raise JSONError("Got an invalid json string.") - except KeyError as e: - raise JSONError( - 'Got an unexpected json object missing the key "{}".'.format(e.args[0]) - ) - - def upload_asset(release: Release, asset_filepath: str) -> None: - asset_filename = os.path.basename(asset_filepath) - try: - asset_mimetype = get_mimetype(asset_filepath) - with open(asset_filepath, "rb") as f: - response = requests.post( - "{}?name={}".format(release.asset_upload_url, asset_filename), - auth=(username, password), - data=f, - headers={"Content-Type": asset_mimetype}, - ) - response.raise_for_status() - logger.info( - 'Uploaded the asset "%s" attached to release "%s" of the GitHub repository "%s"', - asset_filename, - tag, - project, - ) - except requests.HTTPError: - raise HTTPError('Could not upload the asset "{}".'.format(asset_filename)) - - if tag is None: - logger.info( - 'No tag given, fetching the latest tag from the GitHub repository "%s"', - project, - ) - tag = fetch_latest_tag() - if dry_run: - logger.info( - 'Would create the release "%s" in the GitHub repository "%s"', tag, project - ) - assets = [] # type: List[Asset] - else: - release = publish_release(tag) - assets = list_assets(release) - for asset_filepath in asset_filepaths: - asset_matches = [ - asset for asset in assets if asset.name == os.path.basename(asset_filepath) - ] - if dry_run: - for asset_match in asset_matches: - logger.info( - 'Would delete the asset "%s" attached to release "%s" of the GitHub repository "%s"', - asset_match.name, - tag, - project, - ) - logger.info( - 'Would upload the asset "%s" attached to release "%s" of the GitHub repository "%s"', - os.path.basename(asset_filepath), - tag, - project, - ) - else: - for asset_match in asset_matches: - delete_asset(asset_match) - upload_asset(release, asset_filepath) - - -if __name__ == "__main__": - main() diff --git a/utils/prepare_for_git_write.sh b/utils/prepare_for_git_write.sh deleted file mode 100755 index f4c51184b..000000000 --- a/utils/prepare_for_git_write.sh +++ /dev/null @@ -1,49 +0,0 @@ -#!/bin/bash - -# This script prepares the shell so that it can do git-pushes -# It's using the first param as the secret key and the env-var -# KNOWN_HOSTS. - - -## Install ssh-agent if not already installed, it is required by Docker. -## (change apt-get to yum if you use an RPM-based image) -## -which ssh-agent || ( apk update && apk add --no-cache bash git openssh ) -docker info - -## -## Run ssh-agent (inside the build environment) -## -eval $(ssh-agent -s) - -## -## Add the SSH key stored in SSH_PRIVATE_KEY variable to the agent store -## We're using tr to fix line endings which makes ed25519 keys work -## without extra base64 encoding. -## https://gitlab.com/gitlab-examples/ssh-private-key/issues/1#note_48526556 -## -echo "$1" | tr -d '\r' | ssh-add - > /dev/null - -## -## Create the SSH directory and give it the right permissions -## -mkdir -p ~/.ssh -chmod 700 ~/.ssh - -# Check if Git user email is not set -if [ -z "$(git config --global --get user.email)" ]; then - git config --global user.email "specter@secretvalues" -fi - -# Check if Git user name is not set -if [ -z "$(git config --global --get user.name)" ]; then - git config --global user.name "specter" -fi - -# Check if KNOWN_HOSTS is set and not empty -if [ -n "$KNOWN_HOSTS" ]; then - # Add KNOWN_HOSTS to known_hosts file - echo "$KNOWN_HOSTS" > ~/.ssh/known_hosts - # Ensure the file permissions are correct - chmod 644 ~/.ssh/known_hosts -fi diff --git a/utils/release.sh b/utils/release.sh deleted file mode 100755 index 65f97173c..000000000 --- a/utils/release.sh +++ /dev/null @@ -1,232 +0,0 @@ -#!/bin/bash - -# Replacing MacOS utilities with GNU core utilities to make script more robust -# See: https://apple.stackexchange.com/questions/69223/how-to-replace-mac-os-x-utilities-with-gnu-core-utilities -if [[ "$OSTYPE" == "darwin"* ]]; then - brew ls --versions coreutils > /dev/null; - exitCode=$? - if [[ $exitCode == 0 ]]; then - echo "Coreutils are installed via Homebrew, prepending PATH to use GNU core utilities over MacOS utilities." - export PATH="/usr/local/opt/coreutils/libexec/gnubin:$PATH" - else - echo "GNU core utilities not installed. Run brew install coreutils" - fi -fi - -ask_yn() { - while true; do - read -p "Is this correct [y/n]" yn - case $yn in - [Yy]* ) return 0 ;; - [Nn]* ) return 1;; - * ) echo "Please answer yes or no.";; - esac - done -} - -while [[ $# -gt 0 ]] -do -key="$1" -command="main" -case $key in - wait_on_master) - command=wait_on_master - shift - ;; - --help) - help - shift - exit 0 - ;; - --release-notes) - RELEASE_NOTES="yes" - shift # past value - ;; - --dev) - DEV="yes" - shift - ;; - --new-version) - new_version=$2 - shift - shift - ;; - --tag) - TAG="yes" - shift - ;; - --debug) - set -x - shift # past argument - ;; - *) # unknown option - POSITIONAL="$1" # save it in an array for later - shift # past argument - ;; -esac -done - -function help() { - # echo HERE_DOC - # ... - echo "not yet implemented" -} - -function main() { - # Sed is used as there can be whitespaces - if ! [ "$(git remote -v | grep upstream | grep 'git@github.com:cryptoadvance/specter-desktop.git' | wc -l | sed -e 's/[[:space:]]*//')" = "2" ]; then - echo " --> You don't have the correct upstream-remote. You need this to release. Please do this:" - echo "git remote add upstream git@github.com:cryptoadvance/specter-desktop.git " - exit 2 - fi - - if ! [ "$(git remote -v | grep origin | grep 'git@github.com:' | wc -l | sed -e 's/[[:space:]]*//')" = "2" ]; then - echo " --> You don't have a reasonable origin-remote. You need this to release (especially with --dev). Please add one!" - exit 2 - fi - - current_branch=$(git rev-parse --abbrev-ref HEAD) - if [ "$current_branch" != "master" ]; then - echo "You're currently not on the master-branch, exiting" - exit 2 - fi - - echo " --> Fetching all tags ..." - git fetch upstream --tags - echo " --> git pull upstream master" - git pull upstream master - - if [[ -z "$new_version" ]]; then - echo "What should be the new version? Type in please (e.g. v0.9.3 ):" - read new_version - fi - - if ! [[ $new_version =~ ^v([0-9]+)\.([0-9]+)\.([0-9]+)(-([0-9A-Za-z-]+))?$ ]]; then - echo "version $new_version Does not match the pattern!" - exit 1; - fi - - if [[ -n "$RELEASE_NOTES" ]]; then - - - if [ -z $GH_TOKEN ]; then - echo "Your github-token is missing. Please export them like:" - echo "export GH_TOKEN=" - exit 2 - fi - - latest_version=$(git tag -l "v*" | grep -v 'pre' | grep -v 'dev' | sort -V | tail -1) - - echo " --> The latest version is $latest_version. " - if ! ask_yn ; then - echo "Ok, then you type in the latest_version:" - read latest_version - if ! [[ $new_version =~ ^v([0-9]+)\.([0-9]+)\.([0-9]+)(-([0-9A-Za-z-]+))?$ ]]; then - echo "Does not match the pattern!" - exit 1; - fi - fi - - echo "Here are the release-notes:" - echo "--------------------------------------------------" - echo "# Release Notes" > docs/new_release_notes.md - echo "" >> docs/new_release_notes.md - echo "## ${new_version} $(date +'%B %d, %Y')" >> docs/new_release_notes.md - docker run registry.gitlab.com/cryptoadvance/specter-desktop/github-changelog:latest --github-token $GH_TOKEN --branch master cryptoadvance specter-desktop $latest_version | sort >> docs/new_release_notes.md - echo "" >> docs/new_release_notes.md - - - cat docs/new_release_notes.md - echo "--------------------------------------------------" - - cp docs/release-notes.md docs/release-notes.md.orig - sed -i -e '1,2d' docs/release-notes.md.orig # Assuming the release-Notes start with # Release Notes\n - cat docs/new_release_notes.md docs/release-notes.md.orig > docs/release-notes.md - rm docs/release-notes.md.orig docs/new_release_notes.md - - echo "Please check your new File and modify as you find approriate!" - echo "We're waiting here ..." - echo " --> Should we create a PR-branch now? " - - if ! ask_yn ; then - echo "break" - #git checkout docs/release-notes.md - exit 2 - fi - - echo " --> Creating branch ${new_version}_release_notes " - git checkout -b ${new_version}_release_notes - git add docs/release-notes.md - git commit -m "adding release_notes for $new_version" - git push --set-upstream origin ${new_version}_release_notes - - echo "Now go ahead and make your PR:" - echo "https://github.com/cryptoadvance/specter-desktop/pulls" - exit 0 - fi - - - - if [[ -n "$TAG" ]]; then - - echo " --> Should i now create the tag and push the version $new_version ?" - if [ -z $DEV ]; then - echo " --> This will push to your origin-remote!" - else - echo " --> THIS WILL PUSH TO THE UPSTREAM-REMOTE!" - if ! ask_yn ; then - echo "break" - exit 2 - fi - fi - - git tag $new_version - if [ -z $DEV ]; then - git push origin $new_version - else - git push upstream $new_version - fi - fi - -} - -function wait_on_master() { - echo "# check status of masterbranch ..." - i=0 - # First, wait on the check-runs to be completed: - for i in {1..5} ; do - current_state=$(curl -s https://api.github.com/repos/cryptoadvance/specter-desktop/commits/master/check-runs) - different_states=$(echo $current_state | jq -r '.check_runs[] | select(.status == "completed") | .status' | uniq | wc -l) - status=$(echo $current_state | jq -r '.check_runs[] | select(.status == "completed") | .status' | uniq) - - if [[ "$different_states" == 1 ]] && [[ "$status" == "completed" ]] ; then - break - fi - echo "# Builds still running. Will check again in 5 seconds." - sleep 5 - done - - # Now check all the runs and make sure there are all green: - current_state=$(curl -s https://api.github.com/repos/cryptoadvance/specter-desktop/commits/master/check-runs) - different_conclusions=$(echo $current_state | jq -r '.check_runs[] | select(.conclusion == "success") | .conclusion' | uniq | wc -l) - conclusion=$(echo $current_state | jq -r '.check_runs[] | select(.conclusion == "success") | .conclusion' | uniq) - - # We only have one conclusion over all runs: - if [ $different_conclusions -gt 1 ] ; then - echo "# different_conclusions: $different_conclusions" - echo "# Seems that master is not green. Exiting 1" - exit 1 - fi - # ... and that conclusion is "success" - if [[ "$conclusion" == "success" ]]; then - echo "# Great, conclusion is success! Exiting 0" - exit 0 - fi - - echo "# ERROR: I'm confused. This should not happened, exiting 99" - echo "# conclusion = $conclusion" - #echo $current_state - exit 99 -} - -$command \ No newline at end of file diff --git a/utils/release_helper.py b/utils/release_helper.py deleted file mode 100644 index 3242b89d3..000000000 --- a/utils/release_helper.py +++ /dev/null @@ -1,571 +0,0 @@ -import hashlib -import logging -import os -import shutil -import subprocess -import sys -import zipfile -from pathlib import Path -from glob import glob - - -logging.basicConfig(format="%(levelname)s:%(message)s", level=logging.INFO) -logger = logging.getLogger(__name__) -logger.setLevel(logging.DEBUG) - - -import os -import subprocess -import hashlib -import logging -import gitlab - -logger = logging.getLogger(__name__) - - -class Sha256sumFile: - """ - A class that provides functionality to manage SHA256 checksums for files within a - specified directory. - - Attributes: - name (str): The name of the file that contains the SHA256 checksums for other files. - target_dir (str): The path to the directory where the checksum file and other related - files are stored or will be downloaded to. Defaults to `./signing_dir`. - hashed_files (dict): A dictionary storing file names as keys and their corresponding - SHA256 hashes as values. - - The `hashed_files` dictionary data structure is used to map each file name (a string) - to its SHA256 hash (also a string). The SHA256 hash is computed for each corresponding - file present in the `target_dir`, allowing for verification of file integrity by comparing - computed hashes against stored hashes. - """ - - def __init__(self, name, target_dir="./signing_dir"): - """ - Initializes a Sha256sumFile instance with the provided checksum file name and target directory. - - Parameters: - name (str): The name of the checksum file. - target_dir (str): The directory path where the checksum file and other files are present or downloaded. - """ - self.name = name - self.target_dir = target_dir - self.hashed_files = {} - - def is_in_target_dir(self): - """ - Checks if the checksum file is present in the target directory. - - Returns: - bool: True if the checksum file is present; False otherwise. - """ - return os.path.isfile(os.path.join(self.target_dir, self.name)) - - def download_from_tag(self, tag, gc): - """ - Downloads the checksum file and its signature from a specific tag using a given client (gc). - - Parameters: - tag (str): The tag associated with the artifacts to be downloaded. - gc (object): The client object which provides the `download_artifact` method for downloading. - """ - gc.download_artifact(tag, self.name, target_dir=self.target_dir) - gc.download_artifact(tag, self.name + ".asc", target_dir=self.target_dir) - self.read() - - def download_hashed_files(self, tag, gc): - """ - Downloads all files listed in the hashed_files dictionary from a specific tag using a given client (gc). - - Parameters: - tag (str): The tag associated with the artifacts to be downloaded. - gc (object): The client object which provides the `download_artifact` method for downloading. - """ - for file in self.hashed_files.keys(): - logger.info(f"Downloading {file} from {tag}") - gc.download_artifact(tag, file, target_dir=self.target_dir) - - def read(self): - """ - Reads the checksum file and populates the hashed_files dictionary with file names and their corresponding hashes. - """ - with open(os.path.join(self.target_dir, self.name), "r") as file: - line = file.readline() - while line: - line = line.split(maxsplit=2) - self.hashed_files[line[1]] = line[0] - line = file.readline() - - def print(self): - """ - Prints each file's hash and name from the hashed_files dictionary to the standard output. - """ - for hashed_file, hash in self.hashed_files.items(): - print(f"{hash} {hashed_file}") - - def write(self): - """ - Writes the hashed_files dictionary entries to the checksum file in the target directory. - """ - with open(os.path.join(self.target_dir, self.name), "w") as file: - for hashed_file, hash in self.hashed_files.items(): - file.write(f"{hash} {hashed_file}\n") - - def add_file(self, file): - """ - Computes the SHA256 hash for a given file and adds the file and its hash to the hashed_files dictionary. - - Parameters: - file (str): The filename for which the SHA256 hash should be computed and added. - """ - self.hashed_files[file] = Sha256sumFile.sha256_checksum(file, self.target_dir) - - def check_hashes(self): - """ - Verifies the integrity of the files by checking their SHA256 hashes against the entries in the checksum file. - - Raises: - Exception: If the verification of any file fails, an exception is raised with - the subprocess output that caused the failure. - """ - try: - subprocess.run( - ["sha256sum", "-c", self.name], cwd=self.target_dir, check=True - ) - except subprocess.CalledProcessError as e: - raise Exception( - f"Could not validate hashes for file {self.name}: {e.output}" - ) - - def check_sig(self): - """ - Verifies the signature of the checksum file using gpg. - - Raises: - Exception: If the verification of the file signature fails, an exception is raised. - """ - returncode = subprocess.call( - ["gpg", "--verify", self.name + ".asc"], cwd=self.target_dir - ) - if returncode != 0: - raise Exception(f"Could not validate signature of file {self.name}") - - @classmethod - def sha256_checksum(cls, filename, folder, block_size=65536): - """ - Computes the SHA256 hash of a given file. - - Parameters: - filename (str): The name of the file for which to compute the SHA256 hash. - folder (str): The path to the directory containing the file. - block_size (int): The block size used for reading the file. Defaults to 65536. - - Returns: - str: The SHA256 hash of the file. - """ - sha256 = hashlib.sha256() - with open(os.path.join(folder, filename), "rb") as f: - for block in iter(lambda: f.read(block_size), b""): - sha256.update(block) - return sha256.hexdigest() - - -class ReleaseHelper: - """ - A class that manages software build artifacts for a CI/CD pipeline. - - This class is designed to perform operations such as downloading artifacts from CI - pipelines, verifying SHA256 checksums, verifying GPG signatures, and uploading - artifacts to GitHub releases. - - The class relies on a number of environment variables being present: - CI_COMMIT_TAG: The git tag to work with (format: export CI_COMMIT_TAG=). - CI_PIPELINE_ID: The pipeline ID for which artifacts are managed - (format: export CI_PIPELINE_ID=). - CI_PROJECT_ROOT_NAMESPACE: The root namespace of the CI project - (required for uploading to GitHub). - GH_BIN_UPLOAD_PW: gh_token or token for GitHub to authenticate uploads. - - Attributes: - target_dir (str): The directory path where artifacts are to be managed. - tag (str): The git tag associated with the artifacts being managed. - pipeline_id (str): The CI pipeline ID for artifact management operations. - pipeline (Pipeline): A pipeline object fetched from the CI server. - github_project (str): The GitHub repository in which the release should be created or updated. - gh_token (str): The password or token used to authenticate with GitHub. - - Methods: - download_and_unpack_all_artifacts(): Downloads and unpacks artifacts from a CI pipeline. - download_and_unpack_new_artifacts_from_github(): - Downloads and unpacks new artifacts from GitHub. - create_sha256sum_file(): Creates a SHA256SUMS file with checksums of all artifacts. - check_all_hashes(): Verifies checksums for all artifacts. - check_all_sigs(): Verifies GPG signatures for all artifacts. - calculate_publish_params(): Calculates and validates necessary parameters for publishing. - upload_sha256sum_file(): Uploads the SHA256SUMS file to a GitHub release. - upload_sha256sumsig_file(): Uploads the SHA256SUMS.asc signature file to a GitHub release. - - Note: The actual implementation of the methods and the use of additional classes - like `github.GithubConnection` or `Sha256sumFile` are assumed to exist and are - not defined in this documentation. - """ - - def __init__(self): - self.target_dir = "signing_dir" - Path(self.target_dir).mkdir(parents=True, exist_ok=True) - - @property - def gl(self): - """https://python-gitlab.readthedocs.io/en/stable/api-usage.html""" - if hasattr(self, "_gl"): - return self._gl - - if os.environ.get("GITLAB_PRIVATE_TOKEN"): - logger.info("Using GITLAB_PRIVATE_TOKEN") - self._gl = gitlab.Gitlab( - "http://gitlab.com", - private_token=os.environ.get("GITLAB_PRIVATE_TOKEN"), - ) - elif os.environ.get("CI_JOB_TOKEN"): - logger.info("Using CI_JOB_TOKEN") - self._gl = gitlab.Gitlab( - "http://gitlab.com", job_token=os.environ["CI_JOB_TOKEN"] - ) - else: - raise Exception( - "Can't authenticate against Gitlab ( export GITLAB_PRIVATE_TOKEN )" - ) - return self._gl - - @property - def gitlab_project(self): - if hasattr(self, "_gitlab_project"): - return self._gitlab_project - try: - from gitlab.v4.objects import Project - - self._gitlab_project: Project = self.gl.projects.get(self.ci_project_id) - except gitlab.exceptions.GitlabAuthenticationError as e: - logger.fatal(e) - logger.error("Your token might be expired or wrong. Get a new one here:") - logger.error(" https://gitlab.com/-/profile/personal_access_tokens") - exit(2) - - if ( - self._gitlab_project.attributes["namespace"]["path"] - != self.ci_project_root_namespace - ): - logger.fatal( - f"project_root_namespace ({ self.ci_project_root_namespace }) does not match namespace of Project ({self._gitlab_project.attributes['namespace']['path']}) " - ) - logger.error("You might want to: unset CI_PROJECT_ID") - exit(2) - return self._gitlab_project - - @property - def ci_project_id(self): - if hasattr(self, "_ci_project_id"): - return self._ci_project_id - if os.environ.get("CI_PROJECT_ID"): - self._ci_project_id = os.environ.get("CI_PROJECT_ID") - logger.info(f"Using ci_project_id: {self._ci_project_id} ") - else: - logger.error("No Project given. choose one:") - for project in self.gl.projects.list(search="specter-desktop"): - logger.info( - f" export CI_PROJECT_ID={project.id} # {project.name_with_namespace}" - ) - if project.name_with_namespace.startswith("cryptoadvance"): - self._ci_project_id = project.id - logger.warning( - f"{self._ci_project_id} has been chosen as self._ci_project_id" - ) - return self._ci_project_id - - @property - def github_project(self): - if hasattr(self, "_github_project"): - return self._github_project - self._github_project = f"{self.ci_project_root_namespace}/specter-desktop" - logger.info(f"Using github_project: {self._github_project}") - return self._github_project - - @property - def gh_token(self): - if hasattr(self, "_gh_token"): - return self._gh_token - if os.environ.get("GH_BIN_UPLOAD_PW"): - self._gh_token = os.environ.get("GH_BIN_UPLOAD_PW") - logger.info(f"Using gh_token: REDACTED") - else: - raise Exception( - "no Github token given ( export GH_BIN_UPLOAD_PW=v0.0.0.0-pre13 )" - ) - return self._gh_token - - @property - def ci_commit_tag(self): - if hasattr(self, "_ci_commit_tag"): - return self._ci_commit_tag - if os.environ.get("CI_COMMIT_TAG"): - self._ci_commit_tag = os.environ.get("CI_COMMIT_TAG") - else: - raise Exception("no tag given ( export CI_COMMIT_TAG=v0.0.0.0-pre13 )") - logger.info(f"Using tag: {self._ci_commit_tag}") - return self._ci_commit_tag - - @property - def ci_project_root_namespace(self): - if hasattr(self, "_ci_project_root_namespace"): - return self._ci_project_root_namespace - if os.environ.get("CI_PROJECT_ROOT_NAMESPACE"): - self._ci_project_root_namespace = os.environ.get( - "CI_PROJECT_ROOT_NAMESPACE" - ) - logger.info( - f"Using project_root_namespace: {self._ci_project_root_namespace} ( export CI_PROJECT_ROOT_NAMESPACE={self._ci_project_root_namespace} )" - ) - else: - self._ci_project_root_namespace = "cryptoadvance" - logger.warning( - f"Using project_root_namespace: {self._ci_project_root_namespace} ( export CI_PROJECT_ROOT_NAMESPACE={self._ci_project_root_namespace} )" - ) - return self._ci_project_root_namespace - - @property - def ci_pipeline_id(self): - if hasattr(self, "_ci_pipeline_id"): - return self._ci_pipeline_id - - if os.environ.get("CI_PIPELINE_ID"): - self._ci_pipeline_id = os.environ.get("CI_PIPELINE_ID") - else: - logger.info( - "no CI_PIPELINE_ID given, trying to find an appropriate one ..." - ) - pipelines = self.gitlab_project.pipelines.list() - for pipeline in pipelines: - if pipeline.ref == self.ci_commit_tag: - self._ci_pipeline_id = pipeline.id - self._ci_pipeline = pipeline - logger.info(f"Found matching pipeline: {pipeline}") - if not hasattr(self, "_ci_pipeline"): - logger.error( - f"Could not find tag {self.ci_commit_tag} in the pipeline-refs:" - ) - for pipeline in self.gitlab_project.pipelines.list(): - logger.error(pipeline.ref) - raise Exception( - "no CI_PIPELINE_ID given ( export CI_PIPELINE_ID= ) or maybe you're on the wrong project ( export CI_PROJECT_ROOT_NAMESPACE= )" - ) - - logger.info(f"Using pipeline_id: {self.ci_pipeline.id}") - return self._ci_pipeline_id - - @property - def ci_pipeline(self): - if hasattr(self, "_ci_pipeline"): - return self._ci_pipeline - self._ci_pipeline = self.gitlab_project.pipelines.get(self.ci_pipeline_id) - return self._ci_pipeline - - def download_and_unpack_all_artifacts(self): - if os.path.isdir(self.target_dir): - logger.info(f"First purging {self.target_dir}") - shutil.rmtree(self.target_dir) - for job in self.ci_pipeline.jobs.list(): - if job.name in [ - "release_electron_linux_windows", - "release_binary_windows", - "release_pip", - ]: - zipfn = f"/tmp/_artifacts_{job.name}.zip" - job_obj = self.gitlab_project.jobs.get(job.id, lazy=True) - - if not os.path.isfile(zipfn): - logger.info(f"Downloading artifacts for {job.name}") - with open(zipfn, "wb") as f: - job_obj.artifacts(streamed=True, action=f.write) - else: - logger.info(f"Skipping Download artifacts for {job.name}") - - logger.info(f"Unzipping {zipfn} in target-folder") - with zipfile.ZipFile(zipfn, "r") as zip: - for zip_info in zip.infolist(): - if zip_info.filename[-1] == "/": - continue - zip_info.filename = os.path.basename(zip_info.filename) - logger.info(f" Extracting {zip_info.filename}") - zip.extract(zip_info, self.target_dir) - - def download_and_unpack_new_artifacts_from_github(self): - - gc = github.GithubConnection(self.github_project) - release = gc.fetch_existing_release(self.ci_commit_tag) - assets = gc.list_assets(release) - for asset in assets: - if not asset.name.startswith("SHA256"): - continue - if asset.name == "SHA256SUMS" or asset.name == "SHA256SUMS.asc": - continue - if asset.name.endswith(".asc"): - continue - logger.info("iterating file " + asset.name) - shasumfile = Sha256sumFile(asset.name) - if not shasumfile.is_in_target_dir(): - shasumfile.download_from_tag(self.ci_commit_tag, gc) - shasumfile.download_hashed_files(self.ci_commit_tag, gc) - shasumfile.check_hashes() - shasumfile.check_sig() - logger.info("All files have valid signatures") - - def create_sha256sum_file(self): - with open(f"{self.target_dir}/SHA256SUMS", "w") as shafile: - for file in os.listdir(self.target_dir): - if file.startswith("SHA256SUMS-") and not file.endswith(".asc"): - logger.debug(f"Processing {file}") - sha_src = Sha256sumFile(file) - sha_src.read() - for hashed_file in sha_src.hashed_files.keys(): - print(f"{sha_src.hashed_files[hashed_file]} {hashed_file}\n") - shafile.write( - f"{sha_src.hashed_files[hashed_file]} {hashed_file}\n" - ) - returncode = subprocess.call( - ["sha256sum", "-c", "SHA256SUMS"], cwd=self.target_dir - ) - if returncode != 0: - raise Exception( - f"One of the hashes is not matching: {subprocess.run(['sha256sum', '-c', 'SHA256SUMS'], cwd=self.target_dir)}" - ) - - def check_all_hashes(self): - for file in os.listdir(self.target_dir): - if file.startswith("SHA256SUM") and not file.endswith(".asc"): - logger.info(f"Checking hashes in {file}") - if file.endswith("windows"): - logger.info(f"Converting dos2unix for {file}") - dos2unix(os.path.join("signing_dir", file)) - returncode = subprocess.call( - ["sha256sum", "-c", file], cwd=self.target_dir - ) - if returncode != 0: - raise Exception(f"Could not validate hashes for file {file}") - logger.info("All files SHA256SUM* (not .asc) has valid hashes") - - def check_all_sigs(self): - for file in os.listdir(self.target_dir): - if file.endswith(".asc"): - logger.info(f"Checking signature for {file}") - returncode = subprocess.call( - ["gpg", "--verify", file], cwd=self.target_dir - ) - if returncode != 0: - raise Exception( - f"Could not validate signature of file {file}: {subprocess.run(['gpg', '--verify', file], cwd=self.target_dir)}" - ) - logger.info("All files *.asc has valid signatures") - - def upload_sha256sum_file(self): - artifact = os.path.join("signing_dir", "SHA256SUMS") - - if github.artifact_exists( - self.github_project, self.ci_commit_tag, Path(artifact).name - ): - logger.info(f"Github artifact {artifact} existing. Skipping upload.") - exit(0) - else: - logger.info(f"Github artifact {artifact} does not exist. Let's upload!") - github.publish_release_from_tag( - self.github_project, - self.ci_commit_tag, - [artifact], - "gitlab_upload_release_binaries", - self.gh_token, - ) - - def upload_sha256sumsig_file(self): - artifact = os.path.join("signing_dir", "SHA256SUMS.asc") - - if github.artifact_exists( - self.github_project, self.ci_commit_tag, Path(artifact).name - ): - logger.info(f"Github artifact {artifact} existing. Skipping upload.") - exit(0) - else: - logger.info(f"Github artifact {artifact} does not exist. Let's upload!") - github.publish_release_from_tag( - self.github_project, - self.ci_commit_tag, - [artifact], - "gitlab_upload_release_binaries", - self.gh_token, - ) - - -def dos2unix(filename): - content = "" - outsize = 0 - with open(filename, "rb") as infile: - content = infile.read() - with open(filename, "wb") as output: - for line in content.splitlines(): - outsize += len(line) + 1 - output.write(line + b"\n") - - -def sha256sum(filenames): - sha_file = Sha256sumFile("SHA256SUMS", target_dir=".") - for filename in filenames: - logger.info(f"Adding {filename}") - sha_file.add_file(filename) - sha_file.print() - - -if __name__ == "__main__": - if "sha256sums" in sys.argv: - # Used by build-win.ci.bat - sha256sum(sys.argv[2:]) - exit(0) - if "install_wheel" in sys.argv: - # List all .whl files in the 'dist' directory - wheel_files = glob( - str(Path("dist", "cryptoadvance.specter-*-py3-none-any.whl")) - ) - print("found those wheel files: " + str(wheel_files)) - - # Loop through the wheel files and install them - for wheel_file in wheel_files: - cmd = f"pip3 install {wheel_file}" - res = os.system(cmd) - print(f"Result of command: {cmd}") - print(res) - # If the installation fails, exit with the error code - if res != 0: - exit(res) - # Exit with a success code if all installations were successful - exit(0) - - rh = ReleaseHelper() - try: - from utils import github - except Exception as e: - logger.fatal(e) - logger.error("You might have called this script wrong. Execute it like:") - logger.error("python3 -m utils.release_helper ...") - - if "download" in sys.argv: - rh.download_and_unpack_all_artifacts() - if "downloadgithub" in sys.argv: - rh.download_and_unpack_new_artifacts_from_github() - if "checkhashes" in sys.argv: - rh.check_all_hashes() - if "checksigs" in sys.argv: - rh.check_all_sigs() - if "create" in sys.argv: - rh.create_sha256sum_file() - if "upload_shasums" in sys.argv: - rh.upload_sha256sum_file() - if "upload_shasumssig" in sys.argv: - rh.upload_sha256sumsig_file() diff --git a/utils/tag_specterext_dummy.sh b/utils/tag_specterext_dummy.sh deleted file mode 100755 index 04c4aeccc..000000000 --- a/utils/tag_specterext_dummy.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/bin/bash -set -e - -echo "This script will checkout github.com:${CI_PROJECT_ROOT_NAMESPACE}/specterext-dummy.git" -echo "and tag it with ${CI_COMMIT_TAG}" - - -git clone git@github.com:${CI_PROJECT_ROOT_NAMESPACE}/specterext-dummy.git - -cd specterext-dummy -git checkout master -git tag ${CI_COMMIT_TAG} -#git push origin ${CI_COMMIT_TAG} diff --git a/utils/trigger_docker_build.sh b/utils/trigger_docker_build.sh deleted file mode 100755 index 83bc9e57d..000000000 --- a/utils/trigger_docker_build.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/bash - -payload="{\"ref\":\"master\", \"inputs\": {\"tag\": \"${CI_COMMIT_TAG}\"}}" -echo $payload -# This Token is controlled by https://github.com/AaronDewes - -curl -X POST -H "Accept: application/vnd.github.v3+json" -H "Authorization: token ${AARON_TOKEN}" \ - https://api.github.com/repos/lncm/docker-specter-desktop/actions/workflows/dispatch.yml/dispatches -d @<(cat <