diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 3c7eda987..6031ee296 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -103,7 +103,7 @@ release_pip: - if [[ ${CI_PROJECT_ROOT_NAMESPACE} = "cryptoadvance" ]]; then python3 -m twine upload --verbose --user __token__ dist/* ; fi - cd dist - sha256sum cryptoadvance.specter-*.tar.gz > SHA256SUMS-pip - - ../utils/sign_artifact.sh --artifact ./SHA256SUMS-pip + - ../utils/artifact_signer.sh sign --artifact ./SHA256SUMS-pip - cd .. - cat ./dist/SHA256SUMS-pip #- python ./utils/github.py upload ./dist/SHA256SUMS-pip @@ -181,8 +181,8 @@ release_electron_linux_windows: - sha256sum Specter-Setup-${CI_COMMIT_TAG}.exe > ./SHA256SUMS-win - cat ./SHA256SUMS-win - cd .. - - ../utils/sign_artifact.sh --artifact ./release-win/SHA256SUMS-win - - ../utils/sign_artifact.sh --artifact ./release-linux/SHA256SUMS-linux + - ../utils/artifact_signer.sh sign --artifact ./release-win/SHA256SUMS-win + - ../utils/artifact_signer.sh sign --artifact ./release-linux/SHA256SUMS-linux - python3 ../utils/github.py upload ./release-win/Specter-Setup-${CI_COMMIT_TAG}.exe - python3 ../utils/github.py upload ./release-linux/specterd-${CI_COMMIT_TAG}-x86_64-linux-gnu.zip - python3 ../utils/github.py upload ./release-linux/specter_desktop-${CI_COMMIT_TAG}-x86_64-linux-gnu.tar.gz @@ -221,15 +221,15 @@ release_signatures: - source .env/bin/activate - pip3 install -r test_requirements.txt - ./utils/create-gitlab-cli-cfg.sh - - gpg --import --no-tty --batch --yes /credentials/private.key + - ./utils/artifact_signer.sh init # prepare .gnupg script: - python3 -m utils.release-helper download # downloads the job-artifacts from gitlab - python3 -m utils.release-helper downloadgithub # downloads additional artifacts from github (if not there and is they have SHA256SUMS-something) - - python3 -m utils.release-helper checkhashes # checks all SHA256SUM* files - - python3 -m utils.release-helper checksig # checks the signatures of all SHA256SUMM*.asc files + - python3 -m utils.release-helper checksigs # checks the signatures of all SHA256SUMM*.asc files + - python3 -m utils.release-helper checkhashes # checks all SHA256SUM* files (might modify files on the fly due to windows line endings) - python3 -m utils.release-helper create # creates a SHA256SUM.txt - python3 -m utils.release-helper upload # uploads it to github - - ./utils/sign_artifact.sh --artifact ./signing_dir/SHA256SUMS + - ./utils/artifact_signer.sh sign --artifact ./signing_dir/SHA256SUMS - python3 ./utils/github.py upload ./signing_dir/SHA256SUMS.asc release_docker: diff --git a/src/cryptoadvance/specter/managers/wallet_manager.py b/src/cryptoadvance/specter/managers/wallet_manager.py index 41586fdc1..0e5878d7d 100644 --- a/src/cryptoadvance/specter/managers/wallet_manager.py +++ b/src/cryptoadvance/specter/managers/wallet_manager.py @@ -62,7 +62,7 @@ class WalletManager: self.WalletClass = LWallet if is_liquid(chain) else Wallet self.update(data_folder, rpc, chain) - def update(self, data_folder=None, rpc=None, chain=None): + def update(self, data_folder=None, rpc=None, chain=None, allow_threading=True): if self.is_loading: return self.is_loading = True @@ -102,7 +102,7 @@ class WalletManager: for k in list(self.wallets.keys()): if k not in self.wallets_update_list: self.wallets.pop(k) - if self.allow_threading: + if allow_threading and self.allow_threading: t = threading.Thread( target=self._update, args=( diff --git a/src/cryptoadvance/specter/server_endpoints/settings.py b/src/cryptoadvance/specter/server_endpoints/settings.py index 7a7973b3b..d45b7e45c 100644 --- a/src/cryptoadvance/specter/server_endpoints/settings.py +++ b/src/cryptoadvance/specter/server_endpoints/settings.py @@ -147,9 +147,13 @@ def general(): ), "error", ) + handle_exception(e) continue + logger.debug( + f"Wallet {wallet['alias']} already exists, skipping creation" + ) write_wallet(wallet) - app.specter.wallet_manager.update() + app.specter.wallet_manager.update(allow_threading=False) try: wallet_obj = app.specter.wallet_manager.get_by_alias( wallet["alias"] @@ -180,10 +184,11 @@ def general(): "error", ) wallet_obj.getdata() - except Exception: + except Exception as e: flash( _("Failed to import wallet {}").format(wallet["name"]), "error" ) + handle_exception(e) flash(_("Specter data was successfully loaded from backup"), "info") if rescanning: flash( diff --git a/utils/artifact_signer.sh b/utils/artifact_signer.sh new file mode 100755 index 000000000..92b5589af --- /dev/null +++ b/utils/artifact_signer.sh @@ -0,0 +1,81 @@ +#!/bin/bash + +function sub_help { + echo "This script is to sign artifacts or to prepare the gpg-system to be able to verify artifacts." + echo "Do one of these:" + echo "$ ./utils/artifact_signer.sh init" + echo "This makes sense only on a gitlab-runner. It'll unpack a gpg-directory to be ready to sign and verify" + echo "$ ./utils/artifact_signer.sh sign --artifact ./release-win/SHA256SUMS-win" + echo "Signs a specific artifact. Will do the init on the fly. So no need to call it extra." +} + +while [[ $# -gt 0 ]] +do +key="$1" +command="main" +case $key in + --help) + sub_help + exit + shift + ;; + --artifact) + artifact=$2 + shift + shift + ;; + sign) + action=sign + shift + ;; + init) + action=init + shift + ;; + --debug) + set -x + shift # past argument + ;; + *) # unknown option + POSITIONAL="$1" # save it in an array for later + shift # past argument + ;; +esac +done + +# We want a detached signature in cleartext. Extension: .asc (as in bitcoin) +output_file=${artifact}.asc + +# lazy init: We're initializing Each thime script is called with these two things. +# So that action "init" is just to have a bit more semantics for the one calling this script + +function init { + if [[ -f /credentials/gnupg.tar.gz ]]; then + echo "Init: extracting gnupg.tar.gz" + tar -xzf /credentials/gnupg.tar.gz -C /root + chown -R root:root ~/.gnupg + else + echo "Init: Could not find any /credentials/gnupg.tar.gz" + fi + + if [[ -f /credentials/private.key ]]; then + echo "Init: Importing single private key" + gpg --import --no-tty --batch --yes /credentials/private.key + else + echo "Init: Could not find any /credentials/private.key" + fi +} + +if [ "$action" = "init" ]; then + init +fi + +if [ "$action" = "sign" ]; then + init + if [[ -z $artifact ]]; then + echo "no --artifact given " + exit 1 + fi + echo "signing ..." + echo $GPG_PASSPHRASE | gpg --detach-sign --armor --no-tty --batch --yes --passphrase-fd 0 --pinentry-mode loopback $artifact +fi \ No newline at end of file diff --git a/utils/release-helper.py b/utils/release-helper.py index cff26cdf1..211e80e9b 100644 --- a/utils/release-helper.py +++ b/utils/release-helper.py @@ -82,6 +82,7 @@ class ReleaseHelper: pass def init_gitlab(self): + # https://python-gitlab.readthedocs.io/en/stable/api-usage.html import gitlab if os.environ.get("GITLAB_PRIVATE_TOKEN"): @@ -112,9 +113,12 @@ class ReleaseHelper: self.project_id = os.environ.get("CI_PROJECT_ID") self.github_project = f"{project_root_namespace}/specter-desktop" else: - self.project_id = 15721074 # cryptoadvance/specter-desktop - # self.project_id = - self.github_project = f"{project_root_namespace}/specter-desktop" + logger.error("No Project given. choose one:") + for project in self.gl.projects.list(search="specter-desktop"): + logger.info( + f" export CI_PROJECT_ID={project.id} # {project.name_with_namespace}" + ) + exit(1) logger.info(f"Using project_id: {self.project_id}") logger.info(f"Using github_project: {self.github_project}") @@ -128,7 +132,8 @@ class ReleaseHelper: logger.info(f"Using tag: {self.tag}") if os.environ.get("CI_PIPELINE_ID"): - pipeline_id = os.environ.get("CI_PIPELINE_ID") + self.pipeline_id = os.environ.get("CI_PIPELINE_ID") + self.pipeline = self.project.pipelines.get(self.pipeline_id) else: logger.info( "no CI_PIPELINE_ID given, trying to find an appropriate one ..." @@ -138,7 +143,10 @@ class ReleaseHelper: if pipeline.ref == self.tag: self.pipeline = pipeline logger.info(f"Found matching pipeline: {pipeline}") - if not self.pipeline: + if not hasattr(self, "pipeline"): + logger.error( + f"Could not find tag {self.tag} in the pipeline-refs {[pipeline.ref for pipeline in self.project.pipelines.list()]}" + ) raise Exception("no CI_PIPELINE_ID given ( export CI_PIPELINE_ID") logger.info(f"Using pipeline_id: {self.pipeline.id}") @@ -175,7 +183,6 @@ class ReleaseHelper: zip.extract(zip_info, self.target_dir) def download_and_unpack_new_artifacts_from_github(self): - from utils import github gc = github.GithubConnection(self.github_project) release = gc.fetch_existing_release(self.tag) @@ -217,6 +224,10 @@ class ReleaseHelper: def check_all_hashes(self): for file in os.listdir(self.target_dir): if file.startswith("SHA256SUM") and not file.endswith(".asc"): + logger.info(f"Checking hashes in {file}") + if file.endswith("windows"): + logger.info(f"Converting dos2unix for {file}") + dos2unix(os.path.join("signing_dir", file)) returncode = subprocess.call( ["sha256sum", "-c", file], cwd=self.target_dir ) @@ -257,14 +268,12 @@ class ReleaseHelper: artifact = os.path.join("signing_dir", "SHA256SUMS") self.calculate_publish_params() - if self.github.artifact_exists( - self.github_project, self.tag, Path(artifact).name - ): + if github.artifact_exists(self.github_project, self.tag, Path(artifact).name): logger.info(f"Github artifact {artifact} existing. Skipping upload.") exit(0) else: logger.info(f"Github artifact {artifact} does not exist. Let's upload!") - self.github.publish_release_from_tag( + github.publish_release_from_tag( self.github_project, self.tag, [artifact], @@ -274,6 +283,17 @@ class ReleaseHelper: ) +def dos2unix(filename): + content = "" + outsize = 0 + with open(filename, "rb") as infile: + content = infile.read() + with open(filename, "wb") as output: + for line in content.splitlines(): + outsize += len(line) + 1 + output.write(line + b"\n") + + def sha256sum(filenames): sha_file = Sha256sumFile("SHA256SUMS", target_dir=".") for filename in filenames: @@ -288,6 +308,13 @@ if __name__ == "__main__": exit(0) rh = ReleaseHelper() rh.init_gitlab() + try: + from utils import github + except Exception as e: + logger.fatal(e) + logger.error("You might have called this script wrong. Execute it like:") + logger.error("python3 -m utils.release-helper ...") + if "download" in sys.argv: rh.download_and_unpack_all_artifacts() if "downloadgithub" in sys.argv: diff --git a/utils/sign_artifact.sh b/utils/sign_artifact.sh deleted file mode 100755 index 9cee55556..000000000 --- a/utils/sign_artifact.sh +++ /dev/null @@ -1,45 +0,0 @@ -#!/bin/bash - -while [[ $# -gt 0 ]] -do -key="$1" -command="main" -case $key in - --artifact) - artifact=$2 - shift - shift - ;; - --debug) - set -x - shift # past argument - ;; - *) # unknown option - POSITIONAL="$1" # save it in an array for later - shift # past argument - ;; -esac -done - -if [[ -z $artifact ]]; then - echo "no --artifact given " - exit 1 -fi - - -# We want a detached signature in cleartext. Extension: .asc (as in bitcoin) -output_file=${artifact}.asc - -if [[ -f /credentials/private.key ]]; then - echo "Importing single private key" - gpg --import --no-tty --batch --yes /credentials/private.key -fi - -if [[ -f /credentials/gnupg.tar.gz ]]; then - echo "extracting gnupg.tar.gz" - tar -xzf /credentials/gnupg.tar.gz -C /root - chown -R root:root ~/.gnupg -fi - -echo "signing ..." -echo $GPG_PASSPHRASE | gpg --detach-sign --armor --no-tty --batch --yes --passphrase-fd 0 --pinentry-mode loopback $artifact