GlcoinCore/specter-desktop
1
0
Fork 0
mirror of https://github.com/cryptoadvance/specter-desktop.git synced 2026-08-13 12:33:29 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions 8

Macos intel build (#2446)

Browse source 
* remove conflicting dependencies

* intel preparation (untested)

* make orgName more flexible to test

* adjust set-version logic

* build-osx package target

* fix syntax

* fix syntax

* fix syntax

* append arch for upload

* syntax

* syntax

* fixes

* fix

* complete feature

* fix

* set-version now deleting the file instead of complaining if version-mismatch

* further improvements

* parametrize CI_PROJECT_ROOT_NAMESPACE with --gh-project

* more fixes

* tinyfix

* small fix

* fix packaging

* fix

* fix

* documentation and further polishing

* polishing

* bugfixes and feedback

* docs

* more polishing

* testrun

* kick

* kick

* kick again

* kick yet again

* kick

* kick again

* kick

* kick yet again

* fix

* kick

* kick again and again

* kick again

* kick

* kick again

* kick yet again

* remove comments for restriction
This commit is contained in:
k9ert 2024-05-29 11:45:52 +02:00 • committed by GitHub
parent 7fd2b9158d
commit 92f10e01b0
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
17 changed files with 898 additions and 314 deletions
Download patch file Download diff file Expand all files Collapse all files

79
.gitlab-ci.yml
View file

@ -135,7 +135,7 @@ release_binary_windows:
- .\pyinstaller\build-win-ci.bat $CI_COMMIT_TAG
- python ./utils/github.py upload ./pyinstaller/release/specterd-$CI_COMMIT_TAG-win64.zip
- cd ./pyinstaller/release
- python ..\..\utils\release-helper.py sha256sums specterd-$CI_COMMIT_TAG-win64.zip > SHA256SUMS-windows
- python ..\..\utils\release_helper.py sha256sums specterd-$CI_COMMIT_TAG-win64.zip > SHA256SUMS-windows
- type SHA256SUMS-windows
- echo $GPG_PASSPHRASE | c:\Program` Files` `(x86`)\GnuPg\bin\gpg --detach-sign --armor --no-tty --batch --yes --passphrase-fd 0 --pinentry-mode loopback SHA256SUMS-windows
artifacts:
@ -222,14 +222,14 @@ release_signatures:
- pip3 install -e ".[test]"
- ./utils/artifact_signer.sh init # prepare .gnupg
script:
- python3 -m utils.release-helper download # downloads the job-artifacts from gitlab
- python3 -m utils.release-helper downloadgithub # downloads additional artifacts from github (if not there and is they have SHA256SUMS-something)
- python3 -m utils.release-helper checksigs # checks the signatures of all SHA256SUMM*.asc files
- python3 -m utils.release-helper checkhashes # checks all SHA256SUM* files (might modify files on the fly due to windows line endings)
- python3 -m utils.release-helper create # creates a SHA256SUM
- python3 -m utils.release_helper download # downloads the job-artifacts from gitlab
- python3 -m utils.release_helper downloadgithub # downloads additional artifacts from github (if not there and is they have SHA256SUMS-something)
- python3 -m utils.release_helper checksigs # checks the signatures of all SHA256SUMM*.asc files
- python3 -m utils.release_helper checkhashes # checks all SHA256SUM* files (might modify files on the fly due to windows line endings)
- python3 -m utils.release_helper create # creates a SHA256SUM
- ./utils/artifact_signer.sh sign --artifact ./signing_dir/SHA256SUMS # Signs the SHA256SUM
- python3 -m utils.release-helper upload_shasums # uploads SHA256SUMS to github
- python3 -m utils.release-helper upload_shasumssig # uploads SHA256SUMS.asc to github
- python3 -m utils.release_helper upload_shasums # uploads SHA256SUMS to github
- python3 -m utils.release_helper upload_shasumssig # uploads SHA256SUMS.asc to github
release_docker:
stage: post_releasing
@ -247,46 +247,31 @@ tag_specterext_dummy_repo:
only:
- tags
before_script:
## Install ssh-agent if not already installed, it is required by Docker.
## (change apt-get to yum if you use an RPM-based image)
##
- 'which ssh-agent || ( apk update && apk add --no-cache bash git openssh )'
- docker info
##
## Run ssh-agent (inside the build environment)
##
- eval $(ssh-agent -s)
##
## Add the SSH key stored in SSH_PRIVATE_KEY variable to the agent store
## We're using tr to fix line endings which makes ed25519 keys work
## without extra base64 encoding.
## https://gitlab.com/gitlab-examples/ssh-private-key/issues/1#note_48526556
##
- echo "$SSH_SPECTEREXT_DEPLOY_KEY" | tr -d '\r' | ssh-add - > /dev/null
##
## Create the SSH directory and give it the right permissions
##
- mkdir -p ~/.ssh
- chmod 700 ~/.ssh
##
## Optionally, if you will be using any Git commands, set the user name and
## and email.
##
- git config --global user.email "specter@secretvalues"
- git config --global user.name "specter"
##
## Assuming you created the SSH_KNOWN_HOSTS variable, uncomment the
## following two lines.
##
- echo "$KNOWN_HOSTS" > ~/.ssh/known_hosts
- chmod 644 ~/.ssh/known_hosts
# write access to git@github.com:cryptoadvance/specterext-dummy.git
- source ./utils/prepare_for_git_write.sh "$SSH_SPECTEREXT_DEPLOY_KEY"
script:
- echo "Now tagging ... git@github.com:${CI_PROJECT_ROOT_NAMESPACE}/specterext-dummy.git"
- ./utils/tag_specterext_dummy.sh
update_github:
stage: post_releasing
only:
- tags
before_script:
# write access to git@github.com:swan-bitcoin/specter-static.git
- source ./utils/prepare_for_git_write.sh "$SSH_SPECTERSTATIC_DEPLOY_KEY"
script:
- echo "Now updating https://github.com:${CI_PROJECT_ROOT_NAMESPACE}/specter-desktop/releases/tag/${CI_COMMIT_TAG:-v2.0.4-pre8}"
- ./utils/generate_downloadpage.sh --org_name ${CI_PROJECT_ROOT_NAMESPACE:-k9ert} --debug --version ${CI_COMMIT_TAG:-v2.0.4-pre8} generate github # default-value for testing
update_webpage:
stage: post_releasing
only:
- tags
before_script:
# write access to git@github.com:swan-bitcoin/specter-static.git
- source ./utils/prepare_for_git_write.sh "$SSH_SPECTERSTATIC_DEPLOY_KEY"
script:
- echo "Now updating https://github.com:${CI_PROJECT_ROOT_NAMESPACE}/specter-static.git"
- ./utils/generate_downloadpage.sh --org_name ${CI_PROJECT_ROOT_NAMESPACE:-k9ert} --debug --version ${CI_COMMIT_TAG:-v2.0.4-pre8} generate webpage # default-value for testing

55
docs/release-guide.md
View file

@ -46,6 +46,12 @@ Update your master branch after the release notes PR ([example](https://github.c
git tag v1.13.1 && git push upstream v1.13.1
```
if you have a proper setup of github- AND gitlab forks (and the remote `origin` on git pointing to your github fork), you can push that tag to origin and this is useful to test the procedures. See "CI/CD-dev-env setup" in [continuous-integration](./continuous-integration.md).
```bash
git tag v1.13.1 && git push origin v1.13.1
```
## GitLab - releasing stage
Creating a tag triggers the release process of the GitLab runners.
@ -73,30 +79,61 @@ For details look at `.gitlab-ci.yml`
## MacOS
Ideally, directly after the tag is created, start with the MacOS release. This has to be done manually, for now. There is a script for this:
Ideally, directly after the tag is created, start with the MacOS release. As the binaries of x86/arm64 are not compatible with each other, we need to build on two MacOS architectures.This has to be done manually, for now. There is a script for this. Start with the build on x86:
### MacOS x64 build
```bash
./utils/build-osx.sh --version v1.13.1 --appleid "Satoshi Nakamoto (appleid)" --mail "satoshi@gmx.com" make-hash specterd electron sign upload
./utils/build-osx.sh --version v2.0.5-pre4 specterd package upload
```
This script also runs `github.py upload `, so two more binares and the hash and signature files are uploaded to GitHub:
You can also test this procedure without messing the original project via changing the `orgName` to your `orgName` in `pyinstaller/electron/downloadloc.js`.
This will create three artifacts on github:
* specterd-v2.0.5-pre4-osx_x64.zip
* SHA256SUMS-macos_x64
* SHA256SUMS-macos_x64.asc
### MacOS arm64 build
The electron application will get built on the arm architecture. As it needs to store the sha256 hash in the electron-app, the make-hash target
will not only hash the specterd but also download the other specterd and hash it.
```bash
./utils/build-osx.sh --version v2.0.5-pre4 --appleid "Satoshi Nakamoto (appleid)" --mail "satoshi@gmx.com" specterd make-hash electron sign package upload
```
This will create four artifacts on github:
* Specter-v2.0.5-pre4.dmg
* specterd-v2.0.5-pre4-osx_arm64.zip
* SHA256SUMS-macos_arm64
* SHA256SUMS-macos_arm64.asc
- Specter-v1.13.1.dmg
- specterd-v1.13.1-osx.zip
- SHA256SUMS-macos
- SHA256SUMS-macos.asc
## GitLab - post releasing
Back to GitLab, the final stage is "post releasing".
In this stage, the invididual SHA256-hashes and signatures are combined into two final files:
### release_signatures
In this job, the individual SHA256-hashes and signatures are combined into two final files:
- SHA256SUMS
- SHA256SUMS.asc
Everything, apart from the MacOS files, are pulled from the GitLab environment, the MacOS files from GitHub.
Don't forget to delete the two MacOS files (`SHA256SUMS-macos` and `SHA256SUMS-macos.asc`) on the GitHub release page in the end.
Don't forget to delete the four MacOS files (`SHA256SUMS-macos_arm64` and `SHA256SUMS-macos_arm64.asc` and the two corresponding `_x64` files) on the GitHub release page in the end.
This is difficult to automate as sometimes the manual steps has not succeeded while generating the SHASUM-files. As a result, those hashes are not included. So you might want to run this again. And you can, just delete the two generated files - `SHA256SUMS` and `SHA256SUMS.asc` and run the job again.
### release_docker
There are docker images created by the awesome [Chiang Mai LN dev](https://github.com/lncm/docker-specter-desktop). So the task of this job is to trigger their build-system which is done via `utils/trigger_docker_build.sh`. A prerequisite of this is a token in order to authenticate. That token is from Aaron, one of the maintainers of that repo, and can be found in the gitlab variables section of the CI/CD configuration.
### tag_specterext_dummy_repo
Sometimes there are changes on the plugin architecture. In order to create a plugin, it's quite important to know which version of the plugin system should be used. Because of that, we simply assume that the master of the [specterext-dummy](https://github.com/cryptoadvance/specterext-dummy) repo is compatible with the current master which was just tagged with the new version.
So this job will tag that repo with the same tag and the creation of a plugin will take the version into account.
## Trouble shooting

2
pyinstaller/build-win-ci.bat
View file

@ -19,7 +19,7 @@ pip3 install build==0.10.0
python -m build
echo " --> Installing pypi package"
python .\utils\release-helper.py install_wheel %1%
python .\utils\release_helper.py install_wheel %1%
cd pyinstaller

15
pyinstaller/electron/downloadloc.js
View file

@ -1,6 +1,14 @@
function orgName() {
// This can be changed in order to make download possible from other github orgs
return "cryptoadvance"
}
function getDownloadLocation(version, platformname) {
return `https://github.com/cryptoadvance/specter-desktop/releases/download/${version}/specterd-${version}-${platformname}.zip`
if (platformname != "osx") {
return `https://github.com/${orgName()}/specter-desktop/releases/download/${version}/specterd-${version}-${platformname}.zip`
}
return `https://github.com/${orgName()}/specter-desktop/releases/download/${version}/specterd-${version}-${platformname}_${process.arch}.zip`
}
function appName() {
@ -8,7 +16,8 @@ function appName() {
}
module.exports = {
getDownloadLocation: getDownloadLocation,
appName: appName
getDownloadLocation,
appName,
orgName
}

2
pyinstaller/electron/main.js
View file

@ -146,7 +146,7 @@ app.whenReady().then(() => {
if (!appSettings.versionInitialized || appSettings.versionInitialized != versionData.version) {
logger.info(`Updating ${appSettingsPath} : ${JSON.stringify(appSettings)}`)
appSettings.specterdVersion = versionData.version
appSettings.specterdHash = versionData.sha256
appSettings.specterdHash = versionData.sha256[process.arch]
appSettings.versionInitialized = versionData.version
fs.writeFileSync(appSettingsPath, JSON.stringify(appSettings))
}

73
pyinstaller/electron/set-version.js
View file

@ -1,25 +1,60 @@
const fs = require('fs')
const crypto = require('crypto')
const version = process.argv[2]
const fs = require('fs');
const crypto = require('crypto');
const versionDataFile = './version-data.json';
async function setVersion() {
let package = require('./package.json')
package.version = version
fs.writeFileSync('./package.json', JSON.stringify(package, undefined, 2))
if (process.argv[3]) {
let versionData = {
version,
sha256: (await createHashFromFile(process.argv[3]))
}
fs.writeFileSync('./version-data.json', JSON.stringify(versionData, undefined, 2))
}
const version = process.argv[2];
const file = process.argv[3];
const arch = process.argv[4] || process.arch;
// Set version in package.json
let packageJson = require('./package.json');
packageJson.version = version;
fs.writeFileSync('./package.json', JSON.stringify(packageJson, undefined, 2));
// Set version in version-data.json
if (version && file) {
let versionData;
try {
versionData = require(versionDataFile);
if (versionData.version != version) {
console.log(`Version mismatch. Deleting ${versionDataFile} and creating anew.`);
fs.unlinkSync(versionDataFile); // Delete the existing version-data.json file
versionData = createNewVersionData(version); // Create new version data object
}
} catch (error) {
console.log(`No ${versionDataFile} found. Creating anew.`);
versionData = createNewVersionData(version);
}
// Compute SHA256 hash of the provided file
versionData.sha256[arch] = await createHashFromFile(file);
// Write new version data to file
fs.writeFileSync(versionDataFile, JSON.stringify(versionData, undefined, 2));
console.log("version-data.js: ")
console.log("----------------------------------------------------------")
console.log(versionData);
console.log("----------------------------------------------------------")
} else if (arch || file) {
throw new Error("Declare both arch and file or none.");
}
}
const createHashFromFile = filePath => new Promise(resolve => {
const hash = crypto.createHash('sha256');
fs.createReadStream(filePath).on('data', data => hash.update(data)).on('end', () => resolve(hash.digest('hex')));
function createNewVersionData(version) {
// Return a new version data object
return {
version,
sha256: {}
};
}
const createHashFromFile = filePath => new Promise((resolve, reject) => {
const hash = crypto.createHash('sha256');
fs.createReadStream(filePath)
.on('data', data => hash.update(data))
.on('end', () => resolve(hash.digest('hex')))
.on('error', reject);
});
setVersion()
setVersion().catch(console.error);

2
pyinstaller/electron/src/config.js
View file

@ -44,7 +44,7 @@ function getAppSettings() {
tor: false,
proxyURL: "socks5://127.0.0.1:9050",
specterdVersion: (versionData && versionData.version !== undefined) ? versionData.version : 'unknown',
specterdHash: (versionData && versionData.sha256 !== undefined) ? versionData.sha256 : 'unknown',
specterdHash: (versionData && versionData.sha256 !== undefined) ? versionData.sha256[process.arch] : 'unknown',
specterdCLIArgs: "",
versionInitialized: false
}

2
pyinstaller/electron/src/download.js
View file

@ -3,7 +3,7 @@ const fs = require('fs')
const { app, Menu } = require('electron')
const extract = require('extract-zip')
const { getDownloadLocation } = require('../downloadloc.js')
const { appName, appSettings, platformName, appNameLower, versionData, versionDataPath } = require('./config.js')
const { appName, appSettings, platformName, appNameLower, versionDataPath } = require('./config.js')
const { isMac, getFileHash } = require('./helpers.js')
const { logger } = require('./logging.js')
const ProgressBar = require('electron-progressbar')

2
pyinstaller/requirements.in
View file

@ -2,5 +2,3 @@ pyinstaller==5.2
pefile==2022.5.30
macholib
pywin32-ctypes
babel==2.12.1
pytz==2022.1

8
pyinstaller/requirements.txt
View file

@ -10,10 +10,6 @@ altgraph==0.17 \
# via
# macholib
# pyinstaller
babel==2.12.1 \
--hash=sha256:b4246fb7677d3b98f501a39d43396d3cafdc8eadb045f4a31be01863f655c610 \
--hash=sha256:cc2d99999cd01d44420ae725a21c9e3711b3aadc7976d6147f622d8581963455
# via -r requirements.in
future==0.18.2 \
--hash=sha256:b1bead90b70cf6ec3f0710ae53a525360fa360d306a86583adc6bf83a4db537d
# via pefile
@ -43,10 +39,6 @@ pyinstaller-hooks-contrib==2022.8 \
--hash=sha256:c4210fc50282c9c6a918e485e0bfae9405592390508e3be9fde19acc2213da56 \
--hash=sha256:e46f099934dd4577fb1ddcf37a99fa04027c92f8f5291c8802f326345988d001
# via pyinstaller
pytz==2022.1 \
--hash=sha256:1e760e2fe6a8163bc0b3d9a19c4f84342afa0a2affebfaa84b01b978a02ecaa7 \
--hash=sha256:e68985985296d9a66a881eb3193b0906246245294a881e7c8afe623866ac6a5c
# via -r requirements.in
pywin32-ctypes==0.2.0 \
--hash=sha256:24ffc3b341d457d48e8922352130cf2644024a4ff09762a2261fd34c36ee5942 \
--hash=sha256:9dc2d991b3479cc2df15930958b674a48a227d5361d413827a4cfd0b5876fc98

60
utils/build-common.sh
View file

@ -31,8 +31,8 @@ function build_pypi_pckgs_and_install {
pip3 install ./dist/cryptoadvance.specter-*.whl
}
function specify_app_name {
echo " --> Specify app_name"
function configure {
echo " --> Configure some variables"
if [ -z "$app_name" ]; then
# activate virtualenv. This is e.g. not needed in CI
app_name=specter
@ -44,10 +44,20 @@ function specify_app_name {
specterimg_filename=${app_name^} # usually "Specter"
pkg_filename=${app_name}
fi
export ARCH=$(node -e "console.log(process.arch)")
export dist_mac_folder_name=mac-universal
export CI_COMMIT_TAG=$version
export CI_PROJECT_ROOT_NAMESPACE=$(node -e "const downloadloc = require('./pyinstaller/electron/downloadloc');console.log(downloadloc.orgName())")
echo specterd_filename=${specterd_filename}
echo specterimg_filename=${specterimg_filename}
echo pkg_filename=${pkg_filename}
echo ARCH=$ARCH
echo dist_mac_folder_name=$dist_mac_folder_name
echo CI_COMMIT_TAG=$CI_COMMIT_TAG
echo CI_PROJECT_ROOT_NAMESPACE=$CI_PROJECT_ROOT_NAMESPACE
}
@ -119,52 +129,6 @@ function building_electron_app {
cd ../..
}
function macos_code_sign {
# prerequisites for this:
# in short:
# * make sure you have a proper app-specific password on https://appleid.apple.com/account/manage
# * collect some information via scrun altool --list-providers -u "<yourAppleID>"
# * create profile via xcrun notarytool store-credentials --apple-id "<YourAppleID>" --password "app-specific-pw" --team-id "seeFromAbove"
# * Call the profile: SpecterProfile
# For details see:
# * https://www.youtube.com/watch?v=2xJcMzoi0EI
# * https://blog.dgunia.de/2022/09/01/switching-from-altool-to-notarytool/
# * https://scriptingosx.com/2021/07/notarize-a-command-line-tool-with-notarytool/
# This creates a ZIP archive from the app package (using the ditto command).
# This ZIP archive is then used to upload the app to the Apple notarization service via xcrun notarytool (formerly xcrun altool)
# After the app has been uploaded to the Apple servers and notarized, the ZIP archive is not used again.
# The function uses the xcrun stapler command to attach the notarization result to the app, and then exits.
# docs:
# https://help.apple.com/itc/apploader/#/apdATD1E53-D1E1A1303-D1E53A1126
# https://keith.github.io/xcode-man-pages/altool.1.html
cd pyinstaller/electron
echo ' --> Attempting to code sign...'
echo ' executing: ditto -c -k --keepParent "dist/mac/${specterimg_filename}.app" dist/${specterimg_filename}.zip'
ditto -c -k --keepParent "dist/${dist_mac_folder_name}/${specterimg_filename}.app" dist/${specterimg_filename}.zip
# upload
echo ' uploading ... '
output_json=$(xcrun notarytool submit dist/${specterimg_filename}.zip --apple-id "kneunert@gmail.com" --keychain-profile "SpecterProfile" --output-format json --wait )
echo "Request ID: "
# parsing the requestuuid which we'll need to track progress
requestuuid=$(echo $output_json | jq -r '.id')
status=$(echo $output_json | jq -r '.status')
if [ "$status" = "Invalid" ]; then
mkdir -p signing_logs
echo "issues with notarisation"
xcrun notarytool log ${requestuuid} --keychain-profile SpecterProfile | tee ./signing_logs/${app_name}_${timestamp}_${requestuuid}.log
exit 1
fi
# The stapler somehow "staples" the result of the notarisation in to your app
# see e.g. https://stackoverflow.com/questions/58817903/how-to-download-notarized-files-from-apple
xcrun stapler staple "dist/${dist_mac_folder_name}/${specterimg_filename}.app"
cd ../..
}
function make_release_zip {
echo " --> Making the release-zip"

271
utils/build-osx.sh
View file

@ -13,7 +13,8 @@ function create_virtualenv_for_pyinstaller {
# Important: pyinstaller needs a Python binary with shared library files
# With pyenv, for example, you get this like so: env PYTHON_CONFIGURE_OPTS="--enable-shared" pyenv install 3.10.4
# Use pyenv if available
if command -v pyenv >/dev/null 2>&1; then
#if command -v pyenv >/dev/null 2>&1; then
if /bin/false ; then
### This is usually in .zshrc, putting it in .bashrc didn't work ###
###
export PYENV_ROOT="$HOME/.pyenv"
@ -26,10 +27,10 @@ function create_virtualenv_for_pyinstaller {
PYTHON_VERSION=3.10.11
export PYENV_VERSION=$PYTHON_VERSION
echo "pyenv is available. Setting PYENV_VERSION to 3.10.4, using pyenv-virtualenv to create the buildenv..."
# echo " --> Deleting .buildenv"
# pyenv uninstall -f .buildenv
# rm -rf "$HOME/.pyenv/versions/$PYTHON_VERSION/envs/.buildenv"
# pyenv virtualenv 3.10.4 .buildenv
echo " --> Deleting .buildenv"
pyenv uninstall -f .buildenv
rm -rf "$HOME/.pyenv/versions/$PYTHON_VERSION/envs/.buildenv"
pyenv virtualenv 3.10.4 .buildenv
pyenv activate .buildenv
else
echo "pyenv is not available. Using system Python version."
@ -43,6 +44,92 @@ function create_virtualenv_for_pyinstaller {
pip3 install -e ".[test]"
}
# Overriding this function to deal with the x86 special case
function make_hash_if_necessary {
cd pyinstaller/electron
echo " --> calculate the hash of the binary for download"
specterd_plt_filename=../dist/${specterd_filename}
# early exit
if [[ "$make_hash" != 'True' ]]; then
node ./set-version $version
return 0
fi
# We need to set-versions for two specterd, one arm and one intel.
# arm64 one
node ./set-version $version ${specterd_plt_filename}
# Download and check the intel one
# this needs some env-vars to be set
rm -rf signing_dir/*
PYTHONPATH=../.. python3 -m utils.release_helper downloadgithub
ret_code=$?
if [ $ret_code -ne 0 ]; then
echo "Downloading and verifying x64 specterd failed with exit code $ret_code"
exit $ret_code
fi
if [[ ! -f ./signing_dir/specterd-${version}-osx_x64.zip ]]; then
echo "Downloading and verifying x64 specterd failed as the file does not seem to be there"
exit 1
fi
rm -f /tmp/specterd
unzip ./signing_dir/specterd-${version}-osx_x64.zip -d /tmp
node ./set-version $version /tmp/specterd x64
echo " Hashes in version-data.json $(cat ./version-data.json | jq -r '.sha256')"
echo " Hash of file $(sha256sum ${specterd_plt_filename} )"
echo " Hash of x64 file $(sha256sum /tmp/specterd )"
cd ../..
}
function macos_code_sign {
# prerequisites for this:
# in short:
# * make sure you have a proper app-specific password on https://appleid.apple.com/account/manage
# * collect some information via scrun altool --list-providers -u "<yourAppleID>"
# * create profile via xcrun notarytool store-credentials --apple-id "<YourAppleID>" --password "app-specific-pw" --team-id "seeFromAbove"
# * Call the profile: SpecterProfile
# For details see:
# * https://www.youtube.com/watch?v=2xJcMzoi0EI
# * https://blog.dgunia.de/2022/09/01/switching-from-altool-to-notarytool/
# * https://scriptingosx.com/2021/07/notarize-a-command-line-tool-with-notarytool/
# This creates a ZIP archive from the app package (using the ditto command).
# This ZIP archive is then used to upload the app to the Apple notarization service via xcrun notarytool (formerly xcrun altool)
# After the app has been uploaded to the Apple servers and notarized, the ZIP archive is not used again.
# The function uses the xcrun stapler command to attach the notarization result to the app, and then exits.
# docs:
# https://help.apple.com/itc/apploader/#/apdATD1E53-D1E1A1303-D1E53A1126
# https://keith.github.io/xcode-man-pages/altool.1.html
cd pyinstaller/electron
echo ' --> Attempting to code sign...'
specterimg_filename_fqfn=dist/${dist_mac_folder_name}/${specterimg_filename}.app
echo " executing: ditto -c -k --keepParent "${specterimg_filename_fqfn}" dist/${specterimg_filename}.zip"
ditto -c -k --keepParent "${specterimg_filename_fqfn}" dist/${specterimg_filename}.zip
# upload
echo ' uploading for notarisation ... '
output_json=$(xcrun notarytool submit dist/${specterimg_filename}.zip --apple-id "kneunert@gmail.com" --keychain-profile "SpecterProfile" --output-format json --wait )
# parsing the requestuuid which we'll need to track progress
requestuuid=$(echo $output_json | jq -r '.id')
status=$(echo $output_json | jq -r '.status')
echo "Request ID: $requestuuid"
if [ "$status" = "Invalid" ]; then
mkdir -p signing_logs
echo "issues with notarisation"
xcrun notarytool log ${requestuuid} --keychain-profile SpecterProfile | tee ./signing_logs/${app_name}_${timestamp}_${requestuuid}.log
exit 1
fi
# The stapler somehow "staples" the result of the notarisation in to your app
# see e.g. https://stackoverflow.com/questions/58817903/how-to-download-notarized-files-from-apple
echo " --> Staple the file dist/${dist_mac_folder_name}/${specterimg_filename}.app"
xcrun stapler staple "dist/${dist_mac_folder_name}/${specterimg_filename}.app"
echo ' Successfully Stapled the file'
cd ../..
}
function sub_help {
cat << EOF
@ -143,6 +230,10 @@ while [[ $# -gt 0 ]]
build_sign=True
shift
;;
package)
build_package=True
shift
;;
upload)
upload=True
shift
@ -169,7 +260,7 @@ echo " --> This build got triggered for version $version"
echo $version > pyinstaller/version.txt
specify_app_name
configure
if [[ "$build_specterd" = "True" ]]; then
create_virtualenv_for_pyinstaller
@ -179,9 +270,22 @@ if [[ "$build_specterd" = "True" ]]; then
building_app
fi
if [[ "$build_electron" = "True" ]]; then
prepare_npm
if [[ "$make_hash" = "True" ]]; then
# Making the hash only makes sense on a arm arch
if [[ "$ARCH" != "arm64" ]]; then
echo "ERROR: make-hash target should be only called on an arm64 machine on a mac"
exit 1
fi
make_hash_if_necessary
fi
if [[ "$build_electron" = "True" ]]; then
# Making the hash only makes sense on a arm arch
if [[ "$ARCH" != "arm64" ]]; then
echo "ERROR: electron target should be only called on an arm64 machine on a mac"
exit 1
fi
prepare_npm
npm i
if [[ "${appleid}" == '' ]]
then
@ -189,76 +293,109 @@ if [[ "$build_electron" = "True" ]]; then
else
echo "`jq '.build.mac.identity="'"${appleid}"'"' package.json`" > package.json
fi
building_electron_app
fi
if [[ "$build_sign" = "True" ]]; then
dist_mac_folder_name=mac
if [ "$(uname -m)" = "arm64" ]; then
dist_mac_folder_name=${dist_mac_folder_name}-arm64
fi
if [[ "$appleid" != '' ]]
then
# if [ "$(uname -m)" = "arm64" ]; then
# dist_mac_folder_name=${dist_mac_folder_name}-arm64
# fi
if [[ "$appleid" != '' ]]; then
macos_code_sign
else
echo "WARNING: Forgot to add the appleid ?!"
exit 1
fi
fi
echo " --> Making the release-zip"
if [[ "$build_package" = "True" ]]; then
echo " --> Preparing the release"
mkdir -p release
rm -rf release/*
create-dmg pyinstaller/electron/dist/${dist_mac_folder_name}/${specterimg_filename}.app --identity="Developer ID Application: ${appleid}" dist
# create-dmg doesn't create the prepending "v" to the version
node_comp_version=$(python3 -c "print('$version'[1:])")
mv "dist/${specterimg_filename} ${node_comp_version}.dmg" release/${specterimg_filename}-${version}.dmg
cd pyinstaller/dist # ./pyinstaller/dist
zip ../../release/${specterd_filename}-${version}-osx.zip ${specterd_filename}
cd ../..
sha256sum ./release/${specterd_filename}-${version}-osx.zip
sha256sum ./release/${specterimg_filename}-${version}.dmg
fi
if [ "$app_name" == "specter" ]; then
echo " --> gpg-signing the hashes and uploading"
echo "--------------------------------------------------------------------------"
# The specterd-zipfile from specterd
if [[ -f pyinstaller/dist/${specterd_filename} ]]; then
echo " --> Making the release-zip for specterd"
pushd pyinstaller/dist # to not preserve folder structure
zip ../../release/${specterd_filename}-${version}-osx_${ARCH}.zip ${specterd_filename}
popd
fi
echo "In order to upload these artifacts to github, we now do:"
echo "We're keeping that here in case something fails on the last mile"
echo "export CI_PROJECT_ROOT_NAMESPACE=cryptoadvance"
echo "export CI_COMMIT_TAG=$version"
echo "export GH_BIN_UPLOAD_PW=YourSecretHere"
echo "python3 ../utils/github.py upload ./release/specterd-${version}-osx.zip"
echo "python3 ../utils/github.py upload ./release/Specter-${version}.dmg"
echo "cd release"
echo "sha256sum * > SHA256SUMS-macos"
echo "python3 ../../utils/github.py upload SHA256SUMS-macos"
echo "gpg --detach-sign --armor SHA256SUMS-macos"
echo "python3 ../utils/github.py upload SHA256SUMS-macos.asc"
if [[ "$upload" = "True" ]]; then
echo " --> This build got triggered for version $version"
. ../../specter_gh_upload.sh # A simple file looks like: export GH_BIN_UPLOAD_PW=...(GH token)
export CI_COMMIT_TAG=$version
if [[ -z "$CI_PROJECT_ROOT_NAMESPACE" ]]; then
export CI_PROJECT_ROOT_NAMESPACE=cryptoadvance
fi
python3 ./utils/github.py upload ./release/specterd-${version}-osx.zip
python3 ./utils/github.py upload ./release/Specter-${version}.dmg
cd release
sha256sum * > SHA256SUMS-macos
python3 ../utils/github.py upload SHA256SUMS-macos
# The GPG comman below has a timeout. If that's reached, the script will interrupt. So let's make some noise
say "Hello?! Your overlord is speaking! You're now allowed to sign the binary!"
echo "Just in case you missed the timeout, those three last commands are missing:"
echo "cd release"
echo "gpg --detach-sign --armor SHA256SUMS-macos"
echo "python3 ../utils/github.py upload SHA256SUMS-macos.asc"
gpg --detach-sign --armor SHA256SUMS-macos
python3 ../utils/github.py upload SHA256SUMS-macos.asc
# The dmg image file from App
if [[ -d pyinstaller/electron/dist/${dist_mac_folder_name}/${specterimg_filename}.app ]]; then
rm -f pyinstaller/electron/dist/*.dmg
echo " --> Creating dmg"
create-dmg pyinstaller/electron/dist/${dist_mac_folder_name}/${specterimg_filename}.app --identity="Developer ID Application: ${appleid}" pyinstaller/electron/dist
# create-dmg doesn't create the prepending "v" to the version
node_comp_version=$(python3 -c "print('$version'[1:])")
mv "pyinstaller/electron/dist/${specterimg_filename} ${node_comp_version}.dmg" dist/${specterimg_filename}-${version}.dmg
echo " --> Copying img file dist/${specterimg_filename}-${version}.dmg"
cp dist/${specterimg_filename}-${version}.dmg release/${specterimg_filename}-${version}.dmg
else
echo "WARNING: Skipping packaging for electron App"
echo "No pyinstaller/electron/dist/${dist_mac_folder_name}/${specterimg_filename}.app has been found."
fi
file=./release/${specterd_filename}-${version}-osx_${ARCH}.zip
if [[ -f $file ]]; then
echo -n " FYI : "
sha256sum $file
fi
file=./release/${specterimg_filename}-${version}.dmg
if [[ -f $file ]]; then
echo -n " FIY : "
sha256sum $file
fi
fi
if [ "$app_name" != "specter" ]; then
# "early" exit
if [[ "$upload" = "True" ]]; then
echo "no upload for app_name $app_name"
exit 1
fi
exit
fi
if [[ "$upload" = "True" ]]; then
echo " --> gpg-signing the hashes and uploading"
. ../../specter_gh_upload.sh # A simple file looks like: export GH_BIN_UPLOAD_PW=...(GH token)
export CI_COMMIT_TAG=$version
if [[ -z "$CI_PROJECT_ROOT_NAMESPACE" ]]; then
echo "WARNING: Why is CI_PROJECT_ROOT_NAMESPACE not set? Setting to cryptoadvance"
export CI_PROJECT_ROOT_NAMESPACE=cryptoadvance
fi
echo " This build: version: $version gh-project: $CI_PROJECT_ROOT_NAMESPACE"
specterd_zip_fqfn=./release/specterd-${version}-osx_${ARCH}.zip
echo " Checking for file $specterd_zip_fqfn"
if [[ -f $specterd_zip_fqfn ]]; then
python3 ./utils/github.py upload $specterd_zip_fqfn
else
echo " WARNING: not uploading as it does not exist: $specterd_zip_fqfn"
fi
specter_dmg_fqfn=./release/Specter-${version}.dmg
echo " Checking for file $specter_dmg_fqfn"
if [[ -f $specter_dmg_fqfn ]]; then
python3 ./utils/github.py upload $specter_dmg_fqfn
else
echo " WARNING: not uploading as it does not exist: $specter_dmg_fqfn"
fi
cd release
# Maybe we have some SHA256SUMS files from other runs lying around. We don't want to shasum them
rm -f SHA256SUMS*
sha256sum * > SHA256SUMS-macos_${ARCH}
python3 ../utils/github.py upload SHA256SUMS-macos_${ARCH}
# The GPG comman below has a timeout. If that's reached, the script will interrupt. So let's make some noise
say "Hello?! Your overlord is speaking! You're now allowed to sign the binary!"
echo "Just in case you missed the timeout, those three last commands are missing:"
echo "cd release"
echo "gpg --detach-sign --armor SHA256SUMS-macos_${ARCH}"
echo "python3 ../utils/github.py upload SHA256SUMS-macos_${ARCH}.asc"
gpg --detach-sign --armor SHA256SUMS-macos_${ARCH}
python3 ../utils/github.py upload SHA256SUMS-macos_${ARCH}.asc
fi

2
utils/build-unix.sh
View file

@ -142,7 +142,7 @@ echo " --> This build got triggered for version $version"
# This file gets further packaged up with the pyinstaller and will help specter to figure out which version it's running on
echo $version > pyinstaller/version.txt
specify_app_name
configure
if [[ "$build_specterd" = "True" ]]; then
create_virtualenv_for_pyinstaller

283
utils/generate_downloadpage.sh
View file

@ -3,64 +3,251 @@ set -e
# We start in the directory where this script is located
cd "$(dirname "${BASH_SOURCE[0]}")/."
source ../.env/bin/activate
cd .. && pip3 install -e ".[gendownloadpage]" && cd utils
rm -rf build
mkdir -p build
python3 ./generate_downloadpage.py
# Update Github Release Page
latest_release=$(python3 -c 'import json; import requests; ver = json.loads(requests.get("https://api.github.com/repos/cryptoadvance/specter-desktop/releases/latest", headers={"Accept": "application/vnd.github.v3+json"}).text)["name"]; print(ver)')
# Function to install GitHub CLI on macOS
install_gh_macos() {
if ! command -v brew &> /dev/null; then
echo "Homebrew not found. Installing Homebrew..."
/bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)"
fi
echo "Installing GitHub CLI using Homebrew..."
brew install gh
}
if ! command -v gh &> /dev/null; then
echo "WARNING: 'gh' binary not found or not executable. Please ensure it is installed and in your PATH."
echo "You need now to manually replace the release-Notes. I'll list them here:"
echo "------(snip)-->-8---(snap)--8-<-------------"
cat build/gh_page.md
echo "------(snip)-->-8---(snap)--8-<-------------"
read -p "Press a button to continue when you've done it ..."
else
read -p "Update the Github Release page for version $latest_release ? (y/n): " confirm
if [[ $confirm =~ ^[Yy]$ ]]; then
gh release edit $latest_release --repo cryptoadvance/specter-desktop --notes-file build/gh_page.md
# Function to install GitHub CLI on Linux
install_gh_linux() {
echo "Installing GitHub CLI using official script..."
curl -fsSL https://cli.github.com/packages/githubcli-archive-keyring.gpg | dd of=/usr/share/keyrings/githubcli-archive-keyring.gpg
chmod go+r /usr/share/keyrings/githubcli-archive-keyring.gpg
echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/githubcli-archive-keyring.gpg] https://cli.github.com/packages stable main" | tee /etc/apt/sources.list.d/github-cli.list > /dev/null
apt update
apt install gh
}
function prereq {
if [[ ! -f ../.env/bin/activate ]]; then
pip3 install --upgrade virtualenv
virtualenv --python=python3.10 ../.env
fi
source ../.env/bin/activate
# Check if the 'markdown' package is installed in the Python environment
# 'set -e' does not cause the script to exit if a command in an 'if' statement fails
if ! python3 -c "import markdown" &> /dev/null; then
echo " --> The 'markdown' package is not installed. Installing prerequisites..."
# Change to parent directory, perform installation, and change back
pushd ..
pip3 install -e ".[gendownloadpage]"
popd
else
echo " --> The 'markdown' package is already installed."
fi
}
function clean {
rm -rf build
mkdir -p build
}
check_version_exists() {
org_name="$1"
version="$2"
# Fetch all tags from the GitHub repository
tags_response=$(curl -s "https://api.github.com/repos/${org_name}/specter-desktop/tags" \
-H "Accept: application/vnd.github.v3+json")
# Check the response for the specific version
if ! echo "$tags_response" | jq -e --arg version "$version" '.[] | select(.name == $version)' > /dev/null; then
echo "ERROR: The version $version does not exist in the org $org_name."
exit 1
fi
}
function generate {
mkdir -p build
python3 ./generate_downloadpage.py
}
function update_github {
# Determine the platform and execute the appropriate function
platform="$(uname -s)"
case "${platform}" in
Linux*) install_gh_linux;;
Darwin*) install_gh_macos;;
*) echo "Unsupported platform: ${platform}"; exit 1;;
esac
if ! command -v gh &> /dev/null; then
echo "WARNING: 'gh' binary not found or not executable. Please ensure it is installed and in your PATH."
echo "You need now to manually replace the release-Notes. I'll list them here:"
echo "------(snip)-->-8---(snap)--8-<-------------"
cat build/gh_page.md
echo "------(snip)-->-8---(snap)--8-<-------------"
read -p "Press a button to continue when you've done it ..."
else
gh release edit $version --repo ${org_name}/specter-desktop --notes-file build/gh_page.md
fi
fi
}
function update_webpage {
if [[ $version =~ -pre[0-9]+$ ]]; then
if [[ "$org_name" = "swan-bitcoin" ]]; then
echo "The version has a pre-release suffix. Exiting..."
return 0
fi
echo "we have a pre-release but continuing for testing purposes anyway"
fi
if ! [[ -d ../../specter-static ]]; then
echo "You don't have cloned the specter-static repo."
echo "doing that now"
git clone git@github.com:swan-bitcoin/specter-static.git
specter_static_folder=./specter-static
else
specter_static_folder=../../specter-static
fi
target_file=${specter_static_folder}/specter-httrack-src/specter.solutions/downloads/index.html
if ! [[ -d ../../specter-static ]]; then
echo "You don't have cloned the specter-static repo."
echo "Go and clone it and then run this script again!"
exit 1
fi
cp build/download-page.html $target_file
cp build/download-page.html ../../specter-static/specter-httrack-src/specter.solutions/downloads/index.html
# Change to the Git working directory
cd $specter_static_folder
# Change to the Git working directory
cd ../../specter-static
# Check if working directory is clean
if git diff --quiet && git diff --staged --quiet; then
echo "Git working directory is clean. Exiting..."
exit 0
fi
# Check if working directory is clean
if git diff --quiet && git diff --staged --quiet; then
echo "Git working directory is clean. Exiting..."
exit 0
fi
# Add the file to the staging area
git add specter-httrack-src/specter.solutions/downloads/index.html
# Add the file to the staging area
git add specter-httrack-src/specter.solutions/downloads/index.html
# Commit the changes
read -p "Commit changes? (y/n): " confirm
if [[ $confirm =~ ^[Yy]$ ]]; then
# Commit the changes
git commit -m "Update specter.solutions/downloads/index.html"
else
echo "Commit aborted."
exit 1
# Push the commit
#read -p "Push commit to remote? (y/n): " confirm
#if [[ $confirm =~ ^[Yy]$ ]]; then
git push
#else
# echo "Push aborted."
# exit 1
#fi
}
function sub_help {
cat << EOF
# Example-call:
./utils/generate_downloadpage.sh --debug --version v1.10.0-pre23 generate
EOF
}
while [[ $# -gt 0 ]]
do
arg="$1"
case $arg in
"" | "-h" | "--help")
sub_help
exit 0
shift
;;
--debug)
set -x
DEBUG=true
shift
;;
--version)
version=$2
shift
shift
;;
--org_name)
org_name=$2
shift
shift
;;
generate)
generate=True
shift
;;
github)
change_github=True
shift
;;
webpage)
change_webpage=True
shift
;;
help)
sub_help
shift
;;
*)
shift
sub_${arg} $@ && ret=0 || ret=$?
if [ "$ret" = 127 ]; then
echo "Error: '$arg' is not a known subcommand." >&2
echo " Run '$progname --help' for a list of known subcommands." >&2
exit 1
else
exit $ret_value
fi
;;
esac
done
if [[ "$org_name" = "cryptoadvance" ]]; then
org_name=swan-bitcoin # that's where the original specter-static is hosted
fi
# Push the commit
read -p "Push commit to remote? (y/n): " confirm
if [[ $confirm =~ ^[Yy]$ ]]; then
git push
else
echo "Push aborted."
exit 1
if [[ -z "$version" ]]; then
version=$(python3 << EOF
import json
import requests
org_name = "${org_name}"
url = f"https://api.github.com/repos/{org_name}/specter-desktop/releases/latest"
headers = {"Accept": "application/vnd.github.v3+json"}
response = requests.get(url, headers=headers)
release_info = json.loads(response.text)
print(release_info['name'])
EOF
)
fi
check_version_exists "$org_name" "$version"
echo " --> using version $version"
prereq
if [[ "$generate" = "True" ]]; then
generate
fi
if [[ "$change_github" = "True" ]]; then
# Check if 'gh' is already installed
if command -v gh &> /dev/null; then
echo "GitHub CLI is already installed."
else
echo "GitHub CLI is not installed."
# Detect the platform (Linux or Darwin/macOS)
platform=$(uname -s)
case "$platform" in
Linux) install_gh_linux ;;
Darwin) install_gh_macos ;;
*)
echo "Unsupported platform: $platform"
exit 1
;;
esac
fi
export GH_TOKEN=$GH_BIN_UPLOAD_PW
update_github
fi
if [[ "$change_webpage" = "True" ]]; then
update_webpage
fi

49
utils/prepare_for_git_write.sh Executable file
View file

@ -0,0 +1,49 @@
#!/bin/bash
# This script prepares the shell so that it can do git-pushes
# It's using the first param as the secret key and the env-var
# KNOWN_HOSTS.
## Install ssh-agent if not already installed, it is required by Docker.
## (change apt-get to yum if you use an RPM-based image)
##
which ssh-agent || ( apk update && apk add --no-cache bash git openssh )
docker info
##
## Run ssh-agent (inside the build environment)
##
eval $(ssh-agent -s)
##
## Add the SSH key stored in SSH_PRIVATE_KEY variable to the agent store
## We're using tr to fix line endings which makes ed25519 keys work
## without extra base64 encoding.
## https://gitlab.com/gitlab-examples/ssh-private-key/issues/1#note_48526556
##
echo "$1" | tr -d '\r' | ssh-add - > /dev/null
##
## Create the SSH directory and give it the right permissions
##
mkdir -p ~/.ssh
chmod 700 ~/.ssh
# Check if Git user email is not set
if [ -z "$(git config --global --get user.email)" ]; then
git config --global user.email "specter@secretvalues"
fi
# Check if Git user name is not set
if [ -z "$(git config --global --get user.name)" ]; then
git config --global user.name "specter"
fi
# Check if KNOWN_HOSTS is set and not empty
if [ -n "$KNOWN_HOSTS" ]; then
# Add KNOWN_HOSTS to known_hosts file
echo "$KNOWN_HOSTS" > ~/.ssh/known_hosts
# Ensure the file permissions are correct
chmod 644 ~/.ssh/known_hosts
fi

305
utils/release-helper.py → utils/release_helper.py
View file

@ -14,26 +14,82 @@ logger = logging.getLogger(__name__)
logger.setLevel(logging.DEBUG)
import os
import subprocess
import hashlib
import logging
import gitlab
logger = logging.getLogger(__name__)
class Sha256sumFile:
"""
A class that provides functionality to manage SHA256 checksums for files within a
specified directory.
Attributes:
name (str): The name of the file that contains the SHA256 checksums for other files.
target_dir (str): The path to the directory where the checksum file and other related
files are stored or will be downloaded to. Defaults to `./signing_dir`.
hashed_files (dict): A dictionary storing file names as keys and their corresponding
SHA256 hashes as values.
The `hashed_files` dictionary data structure is used to map each file name (a string)
to its SHA256 hash (also a string). The SHA256 hash is computed for each corresponding
file present in the `target_dir`, allowing for verification of file integrity by comparing
computed hashes against stored hashes.
"""
def __init__(self, name, target_dir="./signing_dir"):
"""
Initializes a Sha256sumFile instance with the provided checksum file name and target directory.
Parameters:
name (str): The name of the checksum file.
target_dir (str): The directory path where the checksum file and other files are present or downloaded.
"""
self.name = name
self.target_dir = target_dir
self.hashed_files = {}
def is_in_target_dir(self):
"""
Checks if the checksum file is present in the target directory.
Returns:
bool: True if the checksum file is present; False otherwise.
"""
return os.path.isfile(os.path.join(self.target_dir, self.name))
def download_from_tag(self, tag, gc):
"""
Downloads the checksum file and its signature from a specific tag using a given client (gc).
Parameters:
tag (str): The tag associated with the artifacts to be downloaded.
gc (object): The client object which provides the `download_artifact` method for downloading.
"""
gc.download_artifact(tag, self.name, target_dir=self.target_dir)
gc.download_artifact(tag, self.name + ".asc", target_dir=self.target_dir)
self.read()
def download_hashed_files(self, tag, gc):
"""
Downloads all files listed in the hashed_files dictionary from a specific tag using a given client (gc).
Parameters:
tag (str): The tag associated with the artifacts to be downloaded.
gc (object): The client object which provides the `download_artifact` method for downloading.
"""
for file in self.hashed_files.keys():
logger.info(f"Downloading {file} from {tag}")
gc.download_artifact(tag, file, target_dir=self.target_dir)
def read(self):
"""
Reads the checksum file and populates the hashed_files dictionary with file names and their corresponding hashes.
"""
with open(os.path.join(self.target_dir, self.name), "r") as file:
line = file.readline()
while line:
@ -42,27 +98,53 @@ class Sha256sumFile:
line = file.readline()
def print(self):
"""
Prints each file's hash and name from the hashed_files dictionary to the standard output.
"""
for hashed_file, hash in self.hashed_files.items():
print(f"{hash} {hashed_file}")
def write(self):
"""
Writes the hashed_files dictionary entries to the checksum file in the target directory.
"""
with open(os.path.join(self.target_dir, self.name), "w") as file:
for hashed_file, hash in self.hashed_files.items():
file.write(f"{hash} {hashed_file}")
file.write(f"{hash} {hashed_file}\n")
def add_file(self, file):
"""
Computes the SHA256 hash for a given file and adds the file and its hash to the hashed_files dictionary.
Parameters:
file (str): The filename for which the SHA256 hash should be computed and added.
"""
self.hashed_files[file] = Sha256sumFile.sha256_checksum(file, self.target_dir)
def check_hashes(self):
returncode = subprocess.call(
["sha256sum", "-c", self.name], cwd=self.target_dir
)
if returncode != 0:
"""
Verifies the integrity of the files by checking their SHA256 hashes against the entries in the checksum file.
Raises:
Exception: If the verification of any file fails, an exception is raised with
the subprocess output that caused the failure.
"""
try:
subprocess.run(
["sha256sum", "-c", self.name], cwd=self.target_dir, check=True
)
except subprocess.CalledProcessError as e:
raise Exception(
f"Could not validate hashes for file {self.name}: {subprocess.run(['sha256sum', '-c', self.name], cwd=self.target_dir)}"
f"Could not validate hashes for file {self.name}: {e.output}"
)
def check_sig(self):
"""
Verifies the signature of the checksum file using gpg.
Raises:
Exception: If the verification of the file signature fails, an exception is raised.
"""
returncode = subprocess.call(
["gpg", "--verify", self.name + ".asc"], cwd=self.target_dir
)
@ -71,6 +153,17 @@ class Sha256sumFile:
@classmethod
def sha256_checksum(cls, filename, folder, block_size=65536):
"""
Computes the SHA256 hash of a given file.
Parameters:
filename (str): The name of the file for which to compute the SHA256 hash.
folder (str): The path to the directory containing the file.
block_size (int): The block size used for reading the file. Defaults to 65536.
Returns:
str: The SHA256 hash of the file.
"""
sha256 = hashlib.sha256()
with open(os.path.join(folder, filename), "rb") as f:
for block in iter(lambda: f.read(block_size), b""):
@ -79,16 +172,57 @@ class Sha256sumFile:
class ReleaseHelper:
"""
A class that manages software build artifacts for a CI/CD pipeline.
This class is designed to perform operations such as downloading artifacts from CI
pipelines, verifying SHA256 checksums, verifying GPG signatures, and uploading
artifacts to GitHub releases.
The class relies on a number of environment variables being present:
CI_COMMIT_TAG: The git tag to work with (format: export CI_COMMIT_TAG=<tag_name>).
CI_PIPELINE_ID: The pipeline ID for which artifacts are managed
(format: export CI_PIPELINE_ID=<pipeline_id>).
CI_PROJECT_ROOT_NAMESPACE: The root namespace of the CI project
(required for uploading to GitHub).
GH_BIN_UPLOAD_PW: Password or token for GitHub to authenticate uploads.
Attributes:
target_dir (str): The directory path where artifacts are to be managed.
tag (str): The git tag associated with the artifacts being managed.
pipeline_id (str): The CI pipeline ID for artifact management operations.
pipeline (Pipeline): A pipeline object fetched from the CI server.
github_project (str): The GitHub repository in which the release should be created or updated.
password (str): The password or token used to authenticate with GitHub.
Methods:
download_and_unpack_all_artifacts(): Downloads and unpacks artifacts from a CI pipeline.
download_and_unpack_new_artifacts_from_github():
Downloads and unpacks new artifacts from GitHub.
create_sha256sum_file(): Creates a SHA256SUMS file with checksums of all artifacts.
check_all_hashes(): Verifies checksums for all artifacts.
check_all_sigs(): Verifies GPG signatures for all artifacts.
calculate_publish_params(): Calculates and validates necessary parameters for publishing.
upload_sha256sum_file(): Uploads the SHA256SUMS file to a GitHub release.
upload_sha256sumsig_file(): Uploads the SHA256SUMS.asc signature file to a GitHub release.
Note: The actual implementation of the methods and the use of additional classes
like `github.GithubConnection` or `Sha256sumFile` are assumed to exist and are
not defined in this documentation.
"""
def __init__(self):
self.target_dir = "signing_dir"
Path(self.target_dir).mkdir(parents=True, exist_ok=True)
def init_gitlab(self):
@property
def gl(self):
# https://python-gitlab.readthedocs.io/en/stable/api-usage.html
import gitlab
if os.environ.get("GITLAB_PRIVATE_TOKEN"):
logger.info("Using GITLAB_PRIVATE_TOKEN")
self.gl = gitlab.Gitlab(
gl = gitlab.Gitlab(
"http://gitlab.com",
private_token=os.environ.get("GITLAB_PRIVATE_TOKEN"),
)
@ -101,88 +235,137 @@ class ReleaseHelper:
raise Exception(
"Can't authenticate against Gitlab ( export GITLAB_PRIVATE_TOKEN )"
)
return gl
if os.environ.get("CI_PROJECT_ROOT_NAMESPACE"):
project_root_namespace = os.environ.get("CI_PROJECT_ROOT_NAMESPACE")
logger.info(
f"Using project_root_namespace: {project_root_namespace} ( export CI_PROJECT_ROOT_NAMESPACE={project_root_namespace} )"
)
else:
raise Exception(
"no CI_PROJECT_ROOT_NAMESPACE given ( export CI_PROJECT_ROOT_NAMESPACE=k9ert )"
)
if os.environ.get("CI_PROJECT_ID"):
self.project_id = os.environ.get("CI_PROJECT_ID")
self.github_project = f"{project_root_namespace}/specter-desktop"
else:
logger.error("No Project given. choose one:")
for project in self.gl.projects.list(search="specter-desktop"):
logger.info(
f" export CI_PROJECT_ID={project.id} # {project.name_with_namespace}"
)
exit(1)
logger.info(f"Using project_id: {self.project_id} ")
logger.info(f"Using github_project: {self.github_project}")
@property
def gitlab_project(self):
if hasattr(self, "_gitlab_project"):
return self._gitlab_project
try:
from gitlab.v4.objects import Project
self.project: Project = self.gl.projects.get(self.project_id)
self._gitlab_project: Project = self.gl.projects.get(self.ci_project_id)
except gitlab.exceptions.GitlabAuthenticationError as e:
logger.fatal(e)
logger.error("Your token might be expired or wrong. Get a new one here:")
logger.error(" https://gitlab.com/-/profile/personal_access_tokens")
exit(2)
if self.project.attributes["namespace"]["path"] != project_root_namespace:
if (
self._gitlab_project.attributes["namespace"]["path"]
!= self.ci_project_root_namespace
):
logger.fatal(
f"project_root_namespace ({ project_root_namespace }) does not match namespace of Project ({self.project.attributes['namespace']['path']}) "
f"project_root_namespace ({ self.ci_project_root_namespace }) does not match namespace of Project ({self._gitlab_project.attributes['namespace']['path']}) "
)
logger.error("You might want to: unset CI_PROJECT_ID")
exit(2)
return self._gitlab_project
@property
def ci_project_id(self):
if hasattr(self, "_ci_project_id"):
return self._ci_project_id
if os.environ.get("CI_PROJECT_ID"):
self._ci_project_id = os.environ.get("CI_PROJECT_ID")
logger.info(f"Using ci_project_id: {self.ci_project_id} ")
else:
logger.error("No Project given. choose one:")
for project in self.gl.projects.list(search="specter-desktop"):
logger.info(
f" export CI_PROJECT_ID={project.id} # {project.name_with_namespace}"
)
if project.name_with_namespace.startswith("cryptoadvance"):
self._ci_project_id = project.id
logger.warn("{self._ci_project_id} has been chosen as self._ci_project_id")
return self._ci_project_id
@property
def github_project(self):
if hasattr(self, "_github_project"):
return self._github_project
self._github_project = f"{self.ci_project_root_namespace}/specter-desktop"
logger.info(f"Using github_project: {self._github_project}")
return self._github_project
@property
def ci_commit_tag(self):
if hasattr(self, "_ci_commit_tag"):
return self._ci_commit_tag
if os.environ.get("CI_COMMIT_TAG"):
self.tag = os.environ.get("CI_COMMIT_TAG")
self._ci_commit_tag = os.environ.get("CI_COMMIT_TAG")
else:
raise Exception("no tag given ( export CI_COMMIT_TAG=v0.0.0.0-pre13 )")
logger.info(f"Using tag: {self.tag}")
logger.info(f"Using tag: {self._ci_commit_tag}")
return self._ci_commit_tag
@property
def ci_project_root_namespace(self):
if hasattr(self, "_ci_project_root_namespace"):
return self._ci_project_root_namespace
if os.environ.get("CI_PROJECT_ROOT_NAMESPACE"):
self._ci_project_root_namespace = os.environ.get(
"CI_PROJECT_ROOT_NAMESPACE"
)
logger.info(
f"Using project_root_namespace: {self._ci_project_root_namespace} ( export CI_PROJECT_ROOT_NAMESPACE={self._ci_project_root_namespace} )"
)
else:
self._ci_project_root_namespace = "cryptoadvance"
logger.warn(
f"Using project_root_namespace: {self._ci_project_root_namespace} ( export CI_PROJECT_ROOT_NAMESPACE={self._ci_project_root_namespace} )"
)
return self._ci_project_root_namespace
@property
def ci_pipeline_id(self):
if hasattr(self, "_ci_pipeline_id"):
return self._ci_pipeline_id
if os.environ.get("CI_PIPELINE_ID"):
self.pipeline_id = os.environ.get("CI_PIPELINE_ID")
self.pipeline = self.project.pipelines.get(self.pipeline_id)
self._ci_pipeline_id = os.environ.get("CI_PIPELINE_ID")
else:
logger.info(
"no CI_PIPELINE_ID given, trying to find an appropriate one ..."
)
pipelines = self.project.pipelines.list()
pipelines = self.gitlab_project.pipelines.list()
for pipeline in pipelines:
if pipeline.ref == self.tag:
self.pipeline = pipeline
if pipeline.ref == self.ci_commit_tag:
self._ci_pipeline_id = pipeline.id
self._ci_pipeline = pipeline
logger.info(f"Found matching pipeline: {pipeline}")
if not hasattr(self, "pipeline"):
logger.error(f"Could not find tag {self.tag} in the pipeline-refs:")
for pipeline in self.project.pipelines.list():
if not hasattr(self, "_ci_pipeline"):
logger.error(
f"Could not find tag {self.ci_commit_tag} in the pipeline-refs:"
)
for pipeline in self.gitlab_project.pipelines.list():
logger.error(pipeline.ref)
raise Exception(
"no CI_PIPELINE_ID given ( export CI_PIPELINE_ID= ) or maybe you're on the wrong project ( export CI_PROJECT_ROOT_NAMESPACE= )"
)
logger.info(f"Using pipeline_id: {self.pipeline.id}")
Path(self.target_dir).mkdir(parents=True, exist_ok=True)
logger.info(f"Using pipeline_id: {self.ci_pipeline.id}")
return self._ci_pipeline_id
@property
def ci_pipeline(self):
if hasattr(self, "_ci_pipeline"):
return self._ci_pipeline
self._ci_pipeline = self.gitlab_project.pipelines.get(self.ci_pipeline_id)
return self._ci_pipeline
def download_and_unpack_all_artifacts(self):
if os.path.isdir(self.target_dir):
logger.info(f"First purging {self.target_dir}")
shutil.rmtree(self.target_dir)
for job in self.pipeline.jobs.list():
for job in self.ci_pipeline.jobs.list():
if job.name in [
"release_electron_linux_windows",
"release_binary_windows",
"release_pip",
]:
zipfn = f"/tmp/_artifacts_{job.name}.zip"
job_obj = self.project.jobs.get(job.id, lazy=True)
job_obj = self.gitlab_project.jobs.get(job.id, lazy=True)
if not os.path.isfile(zipfn):
logger.info(f"Downloading artifacts for {job.name}")
@ -203,7 +386,7 @@ class ReleaseHelper:
def download_and_unpack_new_artifacts_from_github(self):
gc = github.GithubConnection(self.github_project)
release = gc.fetch_existing_release(self.tag)
release = gc.fetch_existing_release(self.ci_commit_tag)
assets = gc.list_assets(release)
for asset in assets:
if not asset.name.startswith("SHA256"):
@ -212,12 +395,14 @@ class ReleaseHelper:
continue
if asset.name.endswith(".asc"):
continue
logger.info("iterating file " + asset.name)
shasumfile = Sha256sumFile(asset.name)
if not shasumfile.is_in_target_dir():
shasumfile.download_from_tag(self.tag, gc)
shasumfile.download_hashed_files(self.tag, gc)
shasumfile.download_from_tag(self.ci_commit_tag, gc)
shasumfile.download_hashed_files(self.ci_commit_tag, gc)
shasumfile.check_hashes()
shasumfile.check_sig()
logger.info("All files have valid signatures")
def create_sha256sum_file(self):
with open(f"{self.target_dir}/SHA256SUMS", "w") as shafile:
@ -251,10 +436,12 @@ class ReleaseHelper:
)
if returncode != 0:
raise Exception(f"Could not validate hashes for file {file}")
logger.info("All files SHA256SUM* (not .asc) has valid hashes")
def check_all_sigs(self):
for file in os.listdir(self.target_dir):
if file.endswith(".asc"):
logger.info(f"Checking signature for {file}")
returncode = subprocess.call(
["gpg", "--verify", file], cwd=self.target_dir
)
@ -262,6 +449,7 @@ class ReleaseHelper:
raise Exception(
f"Could not validate signature of file {file}: {subprocess.run(['gpg', '--verify', file], cwd=self.target_dir)}"
)
logger.info("All files *.asc has valid signatures")
def calculate_publish_params(self):
if not "CI_PROJECT_ROOT_NAMESPACE" in os.environ:
@ -286,14 +474,16 @@ class ReleaseHelper:
artifact = os.path.join("signing_dir", "SHA256SUMS")
self.calculate_publish_params()
if github.artifact_exists(self.github_project, self.tag, Path(artifact).name):
if github.artifact_exists(
self.github_project, self.ci_commit_tag, Path(artifact).name
):
logger.info(f"Github artifact {artifact} existing. Skipping upload.")
exit(0)
else:
logger.info(f"Github artifact {artifact} does not exist. Let's upload!")
github.publish_release_from_tag(
self.github_project,
self.tag,
self.ci_commit_tag,
[artifact],
"gitlab_upload_release_binaries",
self.password,
@ -303,14 +493,16 @@ class ReleaseHelper:
artifact = os.path.join("signing_dir", "SHA256SUMS.asc")
self.calculate_publish_params()
if github.artifact_exists(self.github_project, self.tag, Path(artifact).name):
if github.artifact_exists(
self.github_project, self.ci_commit_tag, Path(artifact).name
):
logger.info(f"Github artifact {artifact} existing. Skipping upload.")
exit(0)
else:
logger.info(f"Github artifact {artifact} does not exist. Let's upload!")
github.publish_release_from_tag(
self.github_project,
self.tag,
self.ci_commit_tag,
[artifact],
"gitlab_upload_release_binaries",
self.password,
@ -361,13 +553,12 @@ if __name__ == "__main__":
exit(0)
rh = ReleaseHelper()
rh.init_gitlab()
try:
from utils import github
except Exception as e:
logger.fatal(e)
logger.error("You might have called this script wrong. Execute it like:")
logger.error("python3 -m utils.release-helper ...")
logger.error("python3 -m utils.release_helper ...")
if "download" in sys.argv:
rh.download_and_unpack_all_artifacts()

2
utils/tag_specterext_dummy.sh
View file

@ -10,4 +10,4 @@ git clone git@github.com:${CI_PROJECT_ROOT_NAMESPACE}/specterext-dummy.git
cd specterext-dummy
git checkout master
git tag ${CI_COMMIT_TAG}
git push origin ${CI_COMMIT_TAG}
#git push origin ${CI_COMMIT_TAG}