Chore: Build Script improvements (#1612)

* some build-improvements

* build-script improvements part2

* revert unintended changes

* refactor build-scripts

* MacOS specific changes

MANIFEST.in

@@ -1,5 +1,7 @@
 recursive-include src/cryptoadvance/specter/templates *
 recursive-include src/cryptoadvance/specter/static *
+recursive-include src/*/specterext/*/templates *
+recursive-include src/*/specterext/*/static *
 recursive-include src/cryptoadvance/specter/services/templates *
 recursive-include src/cryptoadvance/specter/services/static *
 recursive-include src/cryptoadvance/specter/services/*/templates *

pyinstaller/build-common.sh

@@ -0,0 +1,107 @@
+#!/usr/bin/env bash
+
+function specify_app_name {
+    if [ -z "$app_name" ]; then
+    # activate virtualenv. This is e.g. not needed in CI
+        app_name=specter
+        specterd_filename=specterd
+        specterimg_filename=Specter
+        pkg_filename=specter_desktop
+    else
+        specterd_filename=${app_name}d # usually "specterd"
+        specterimg_filename=${app_name^} # usually "Specter"
+        pkg_filename=${app_name}
+    fi
+
+    echo specterd_filename=${specterd_filename}
+    echo specterimg_filename=${specterimg_filename}
+    echo pkg_filename=${pkg_filename}
+}
+
+
+function install_build_requirements {
+
+    echo "    --> Installing (build)-requirements"
+    pip3 install -r requirements.txt --require-hashes > /dev/null
+
+    cd ..
+    # Order is relevant here. If you flip the followng lines, the hiddenimports for services won't work anymore
+    python3 setup.py install > /dev/null
+    pip3 install -e .  > /dev/null
+    cd pyinstaller
+
+}
+
+function cleanup {
+    echo "    --> Cleaning up"
+    rm -rf build/ dist/ release/ electron/release/ electron/dist
+    rm *.dmg || true
+}
+
+function building_app {
+    echo "    --> Building ${specterd_filename}"
+    specterd_filename=${specterd_filename} pyinstaller specterd.spec > /dev/null
+}
+
+function prepare_npm {
+    echo "    --> Making us ready for building electron-app for MacOS"
+    npm ci
+}
+
+
+
+function building_electron_app {
+    echo "    --> building electron-app"
+    npm i
+    npm run dist
+}
+
+function macos_code_sign {
+    # docs:
+    # https://help.apple.com/itc/apploader/#/apdATD1E53-D1E1A1303-D1E53A1126
+    # https://keith.github.io/xcode-man-pages/altool.1.html
+    echo '    --> Attempting to code sign...'
+    ditto -c -k --keepParent "dist/mac/${specterimg_filename}.app" dist/${specterimg_filename}.zip
+    # upload
+    output_json=$(xcrun altool --notarize-app -t osx -f dist/${specterimg_filename}.zip --primary-bundle-id "solutions.specter.desktop" -u "${mail}" --password "@keychain:AC_PASSWORD" --output-format json)
+    echo "JSON-Output:"
+    # parsing the requestuuid which we'll need to track progress
+    requestuuid=$(echo $output_json | jq -r '."notarization-upload".RequestUUID')
+    mkdir -p signing_logs
+    i=1
+    while [ $i -le 6 ] ; do
+        echo "        check result in minute $i ..."
+        sign_result_json=$(xcrun altool --notarization-info $requestuuid -u "${mail}" --password "@keychain:AC_PASSWORD" --output-format json)
+        timestamp=$(date +"%Y%m%d-%H%M")
+        # If it's not json-parseable
+        if ! echo "$sign_result_json" | jq .; then
+            echo $sign_result_json > ./signing_logs/${app_name}_${timestamp}_${requestuuid}.log
+            echo "ERROR: track-json not parseable."
+            echo "$sign_result_json"
+            exit 1   
+        fi
+        # if it's no longer in progress
+        status=$(echo "$sign_result_json" | jq -e -r '.["notarization-info"].Status')
+        if [ "$status" != "in progress" ]; then
+            echo "        Finished code sign with status $status"
+            echo $sign_result_json | jq . > ./signing_logs/${app_name}_${timestamp}_${requestuuid}.log
+            break
+        fi
+        i=$(( $i + 1 ))
+        sleep 60
+    done
+    if [ "$status" != "success" ]; then
+        echo "ERROR: status $status"
+        echo $(echo $sign_result_json | jq .)
+        echo
+        exit 1
+    fi
+    # The stapler somehow "staples" the result of the notarisation in to your app
+    # see e.g. https://stackoverflow.com/questions/58817903/how-to-download-notarized-files-from-apple
+    xcrun stapler staple "dist/mac/${specterimg_filename}.app"
+}
+
+function make_release_zip {
+    echo "    --> Making the release-zip"
+    
+}

pyinstaller/build-osx.sh

@@ -1,6 +1,7 @@
 #!/usr/bin/env bash
 set -e
 
+source build-common.sh
 
 
 function sub_help {
@@ -17,6 +18,10 @@ function sub_help {
 # After installation of xcode: sudo xcode-select -s /Applications/Xcode.app/Contents/Developer
 # otherwise you get xcrun: error: unable to find utility "altool", not a developer tool or in PATH
 
+# catalina might have a a too old version of bash. You need at least 4.0 or so
+# 3.2 is too low definitely
+# brew install bash
+
 # Fill the keychain with your password like this
 # xcrun altool --store-password-in-keychain-item AC_PASSWORD -u '<your apple id>' -p apassword
 
@@ -90,32 +95,29 @@ while [[ $# -gt 0 ]]
   esac
   done
 
+
+
+
 echo "    --> This build got triggered for version $version"
 
 echo $version > version.txt
 
-echo "    --> Installing (build)-requirements"
-pip3 install -r requirements.txt --require-hashes
+specify_app_name
 
-cd ..
-# Order is relevant here. If you flip the followng lines, the hiddenimports for services won't work anymore
-python3 setup.py install
-pip3 install -e .
-cd pyinstaller
+install_build_requirements
 
-echo "    --> Cleaning up"
-rm -rf build/ dist/ release/ electron/release/ electron/dist
-rm *.dmg || true
+cleanup
+
+building_app
+
+cd electron # ./pyinstaller/electron
+
+prepare_npm
 
-echo "    --> Building specterd"
-pyinstaller specterd.spec --runtime-hook=rthooks/hook-pkgutil.py
 
-echo "    --> Making us ready for building electron-app for MacOS"
-cd electron
-npm ci
 if [[ "$make_hash" = 'True' ]]
 then
-    node ./set-version $version ../dist/specterd
+    node ./set-version $version ../dist/${specterd_filename}
 else
     node ./set-version $version
 fi
@@ -127,48 +129,41 @@ else
     echo "`jq '.build.mac.identity="'"${appleid}"'"' package.json`" > package.json
 fi
 
-echo "    --> building electron-app"
-npm run dist
+building_electron_app
 
 if [[ "$appleid" != '' ]]
 then
-    echo '    --> Attempting to code sign...'
-    ditto -c -k --keepParent "dist/mac/Specter.app" dist/Specter.zip
-    output_json=$(xcrun altool --notarize-app -t osx -f dist/Specter.zip --primary-bundle-id "solutions.specter.desktop" -u "${mail}" --password "@keychain:AC_PASSWORD" --output-format json)
-    echo "JSON-Output:"
-    requestuuid=$(echo $output_json | jq -r '."notarization-upload".RequestUUID')
-    sleep 180
-    sign_result_json=$(xcrun altool --notarization-info $requestuuid -u "${mail}" --password "@keychain:AC_PASSWORD" --output-format json)
-    mkdir -p signing_logs
-    timestamp=$(date +"%Y%m%d-%H%M")
-    echo $sign_result_json | jq . > ./signing_logs/${timestamp}_${requestuuid}.log
-    xcrun stapler staple "dist/mac/Specter.app"
+  macos_code_sign
 fi
 
-cd ..
+cd .. # ./pyinstaller
 
 echo "    --> Making the release-zip"
 mkdir release
 
-create-dmg 'electron/dist/mac/Specter.app' --identity="Developer ID Application: ${appleid}"
-mv "Specter ${version:1}.dmg" release/SpecterDesktop-${version}.dmg
+create-dmg electron/dist/mac/${specterimg_filename}.app --identity="Developer ID Application: ${appleid}"
+# create-dmg doesn't create the prepending "v" to the version
+node_comp_version=$(python3 -c "print('$version'[1:])")
+mv "electron/dist/${specterimg_filename}-${node_comp_version}.dmg" release/${specterimg_filename}-${version}.dmg
 
-cd dist
-zip ../release/specterd-${version}-osx.zip specterd
-cd ..
+cd dist # ./pyinstaller/dist
+zip ../release/${specterd_filename}-${version}-osx.zip ${specterd_filename}
+cd .. # ./pyinstaller
 
-sha256sum ./release/specterd-${version}-osx.zip
-sha256sum ./release/SpecterDesktop-${version}.dmg
+sha256sum ./release/${specterd_filename}-${version}-osx.zip
+sha256sum ./release/${specterimg_filename}-${version}.dmg
 
-echo "--------------------------------------------------------------------------"
-echo "In order to upload these artifacts to github, do:"
-echo "export CI_PROJECT_ROOT_NAMESPACE=cryptoadvance"
-echo "export CI_COMMIT_TAG=$version"
-echo "export GH_BIN_UPLOAD_PW=YourSecretHere"
-echo "python3 ../utils/github.py upload ./release/specterd-${version}-osx.zip"
-echo "python3 ../utils/github.py upload ./release/SpecterDesktop-${version}.dmg"
-echo "cd release"
-echo "sha256sum * > SHA256SUMS-macos"
-echo "python3 ../../utils/github.py upload SHA256SUMS-macos"
-echo "gpg --detach-sign --armor SHA256SUMS-macos"
-echo "python3 ../../utils/github.py upload SHA256SUMS-macos.asc"
+if [ "$app_name" == "specter" ]; then
+  echo "--------------------------------------------------------------------------"
+  echo "In order to upload these artifacts to github, do:"
+  echo "export CI_PROJECT_ROOT_NAMESPACE=cryptoadvance"
+  echo "export CI_COMMIT_TAG=$version"
+  echo "export GH_BIN_UPLOAD_PW=YourSecretHere"
+  echo "python3 ../utils/github.py upload ./release/specterd-${version}-osx.zip"
+  echo "python3 ../utils/github.py upload ./release/SpecterDesktop-${version}.dmg"
+  echo "cd release"
+  echo "sha256sum * > SHA256SUMS-macos"
+  echo "python3 ../../utils/github.py upload SHA256SUMS-macos"
+  echo "gpg --detach-sign --armor SHA256SUMS-macos"
+  echo "python3 ../../utils/github.py upload SHA256SUMS-macos.asc"
+fi

pyinstaller/build-unix.sh

@@ -2,42 +2,30 @@
 
 set -e
 
+# debug:
+set -x
+
+source build-common.sh
+
 # pass version number as an argument 
 echo "    --> This build got triggered for version $1"
 
 echo $1 > version.txt
 
-if [ -z "$app_name" ]; then
-  # activate virtualenv. This is e.g. not needed in CI
-    specterd_filename=specterd
-    specterimg_filename=Specter
-    pkg_filename=specter_desktop
-else
-    specterd_filename=${app_name}d
-    specterimg_filename=${app_name^}
-    pkg_filename=${app_name}
-    
-fi
+specify_app_name
 
-echo "    --> Installing (build)-requirements"
-pip3 install -r requirements.txt --require-hashes
-cd ..
-# Order is relevant here. If you flip the followng lines, the hiddenimports for services won't work anymore
-python3 setup.py install
-pip3 install -e .
-cd pyinstaller
 
-echo "    --> Cleaning up"
-rm -rf build/ dist/ release/ electron/release/ electron/dist
+install_build_requirements
 
-echo "    --> Building ${specterd_filename}"
-specterd_filename=${specterd_filename} pyinstaller specterd.spec
+cleanup
+
+building_app
 
-echo "    --> Making us ready for building electron-app for linux"
 cd electron
-npm ci
 
-# calculate the hash of the binary for download
+prepare_npm
+
+echo "    --> calculate the hash of the binary for download"
 if [[ "$2" == 'make-hash' ]]
 then
     node ./set-version $1 ../dist/${specterd_filename}
@@ -45,9 +33,10 @@ else
     node ./set-version $1
 fi
 
-echo "    --> building electron-app"
-npm i
-npm run dist
+echo "        Hash in version -data.json $(cat ./version-data.json | jq -r '.sha256')"
+echo "        Hash of file $(sha256sum ../dist/${specterd_filename} )"
+
+building_electron_app
 
 cd ..
 
@@ -57,8 +46,8 @@ cd dist
 cp -r ../../udev ./udev
 echo "Don't forget to set up udev rules! Check out udev folder for instructions." > README.md
 zip -r ../release/${specterd_filename}-"$1"-"$(uname -m)"-linux-gnu.zip ${specterd_filename} udev README.md
-
-cp ../electron/dist/Specter-* ./
-tar -czvf ../release/specter_desktop-"$1"-"$(uname -m)"-linux-gnu.tar.gz Specter-* udev README.md
+echo $app_name
+cp ../electron/dist/${app_name^}-* ./
+tar -czvf ../release/${pkg_filename}-"$1"-"$(uname -m)"-linux-gnu.tar.gz ${app_name^}-* udev README.md
 
 cd ..

pyinstaller/electron/downloadloc.js

@@ -1,7 +1,14 @@
+
 function getDownloadLocation(version, platformname) {
-    return `https://github.com/cryptoadvance/specter-desktop/releases/download/${version}/specterd-${version}-${platformname}.zip`
+    return `http://specterext.bitcoinops.de/user/k9ert/dice/releases/download/${version}/diced-${version}-${platformname}.zip`
+}
+
+function appName() {
+    return "Specter"
 }
 
 module.exports = {
-    getDownloadLocation: getDownloadLocation
-}
+    getDownloadLocation: getDownloadLocation, 
+    appName: appName
+}
+

pyinstaller/electron/helpers.js

@@ -2,20 +2,25 @@ const fs = require('fs')
 const path = require('path')
 const crypto = require('crypto')
 const readLastLines = require('read-last-lines');
+const downloadloc = require('./downloadloc');
+const { loggers } = require('winston');
+const appName = downloadloc.appName()
+const appNameLower = appName.toLowerCase()
 
 let versionData
 try {
     versionData = require('./version-data.json')
-} catch {
-    console.log('Could not find default version data configurations...')
+    console.log(versionData)
+} catch (e) {
+    console.log('Could not find default version data configurations...'+e)
     versionData = {
         "version": "",
         "sha256": ""
     }
 }
-const appSettingsPath = path.resolve(require('os').homedir(), '.specter/app_settings.json')
-const specterdDirPath = path.resolve(require('os').homedir(), '.specter/specterd-binaries')
-const specterAppLogPath = path.resolve(require('os').homedir(), '.specter/specterApp.log')
+const appSettingsPath = path.resolve(require('os').homedir(), `.${appNameLower}/app_settings.json`)
+const specterdDirPath = path.resolve(require('os').homedir(), `.${appNameLower}/specterd-binaries`)
+const specterAppLogPath = path.resolve(require('os').homedir(), `.${appNameLower}/specterApp.log`)
 
 function getFileHash(filename, callback) {
   let shasum = crypto.createHash('sha256')

pyinstaller/electron/main.js

@@ -16,6 +16,8 @@ const specterdDirPath = helpers.specterdDirPath
 
 const downloadloc = require('./downloadloc')
 const getDownloadLocation = downloadloc.getDownloadLocation
+const appName = downloadloc.appName()
+const appNameLower = appName.toLowerCase()
 
 // Logging
 const {transports, format, createLogger } = require('winston')
@@ -109,7 +111,12 @@ switch (process.platform) {
   case 'linux':
     platformName = 'x86_64-linux-gnu'
     break
+  default:
+      throw `Unknown platformName ${platformName}`
+
 }
+logger.info("Using version " + appSettings.specterdVersion);
+logger.info("Using platformName " + platformName);
 
 function createWindow (specterURL) {  
   if (!mainWindow) {
@@ -167,7 +174,7 @@ app.whenReady().then(() => {
     appSettings.versionInitialized = versionData.version
     fs.writeFileSync(appSettingsPath, JSON.stringify(appSettings))
   }
-  const specterdPath = specterdDirPath + '/specterd'
+  const specterdPath = specterdDirPath + '/' + appNameLower + 'd'
   if (fs.existsSync(specterdPath + (platformName == 'win64' ? '.exe' : ''))) {
     getFileHash(specterdPath + (platformName == 'win64' ? '.exe' : ''), function (specterdHash) {
       if (appSettings.specterdHash.toLowerCase() == specterdHash || appSettings.specterdHash == "") {
@@ -222,31 +229,34 @@ function initMainWindow(specterURL) {
 }
 
 function downloadSpecterd(specterdPath) {
-  updatingLoaderMsg('Fetching the Specter binary...<br>This might take a minute...')
-  updateSpecterdStatus('Fetching Specter binary...')
-  logger.info("Using version ", appSettings.specterdVersion);
-  logger.info(`https://github.com/cryptoadvance/specter-desktop/releases/download/${appSettings.specterdVersion}/specterd-${appSettings.specterdVersion}-${platformName}.zip`);
+  updatingLoaderMsg(`Fetching the ${appName} binary...<br>This might take a minute...`)
+  updateSpecterdStatus(`Fetching ${appName} binary...`)
+  logger.info("Using version " + appSettings.specterdVersion);
+  logger.info("Using platformName " + platformName);
+  
   download_location = getDownloadLocation(appSettings.specterdVersion, platformName)
+  logger.info("Downloading from "+download_location);
   download(download_location, specterdPath + '.zip', function(errored) {
     if (errored == true) {
-      updatingLoaderMsg('Fetching specter binary from the server failed, could not reach the server or the file could not have been found.')
-      updateSpecterdStatus('Fetching specterd failed...')
+      updatingLoaderMsg(`Fetching ${appNameLower} binary from the server failed, could not reach the server or the file could not have been found.`)
+      updateSpecterdStatus(`Fetching ${appNameLower}d failed...`)
       return
     }
 
     updatingLoaderMsg('Unpacking files...')
+    logger.info("Extracting "+specterdPath);
 
     extract(specterdPath + '.zip', { dir: specterdPath + '-dir' }).then(function () {
       let extraPath = ''
       switch (process.platform) {
         case 'darwin':
-          extraPath = 'specterd'
+          extraPath = appNameLower + "d"
           break
         case 'win32':
-          extraPath = 'specterd.exe'
+          extraPath = appNameLower + 'd.exe'
           break
         case 'linux':
-          extraPath = 'specterd'
+          extraPath = appNameLower + 'd'
       }
       var oldPath = specterdPath + `-dir/${extraPath}`
       var newPath = specterdPath + (platformName == 'win64' ? '.exe' : '')
@@ -260,7 +270,10 @@ function downloadSpecterd(specterdPath) {
           startSpecterd(specterdPath)
         } else {
           updatingLoaderMsg('Specterd version could not be validated.')
+          logger.error(`hash of downloaded file: ${specterdHash}`)
+          logger.error(`Expected hash: ${appSettings.specterdHash}`)
           updateSpecterdStatus('Failed to launch specterd...')
+
           // app.quit()
           // TODO: This should never happen unless the specterd file was swapped on GitHub.
           // Think of what would be the appropriate way to handle this...
@@ -285,6 +298,7 @@ function updatingLoaderMsg(msg) {
     `;
     mainWindow.webContents.executeJavaScript(code);
   } 
+  logger.info("Updated LoaderMsg: "+msg)
 }
 
 function hasSuccessfullyStarted(logs) {

pyinstaller/hooks/hook-cryptoadvance.specter.services.py

@@ -1,5 +1,7 @@
+import logging
 from cryptoadvance.specter.managers.service_manager import ServiceManager
 
+logger = logging.getLogger(__name__)
 
 # Collecting template and static files from the different services in src/cryptoadvance/specter/services
 service_template_datas = [
@@ -18,3 +20,6 @@ service_packages = ServiceManager.get_service_packages()
 datas = [*service_template_datas, *service_static_datas]
 
 hiddenimports = [*service_packages]
+
+logger.info(f"specter-hook datas: {datas}")
+logger.info(f"specter-hook hiddenimports: {hiddenimports}")

src/cryptoadvance/specter/managers/service_manager.py

@@ -26,6 +26,7 @@ from ..util.reflection import (
     get_subclasses_for_clazz,
     get_subclasses_for_clazz_in_cwd,
 )
+from ..util.reflection_fs import search_dirs_in_path
 
 logger = logging.getLogger(__name__)
 
@@ -272,14 +273,31 @@ class ServiceManager:
     @classmethod
     def get_service_x_dirs(cls, x):
         """returns a list of package-directories which represents a specific service.
-        This is used by the pyinstaller packaging specter
+        This is used EXCLUSIVELY by the pyinstaller-hook packaging specter to add templates/static
+        When this gets called, CWD is ./pyinstaller
         """
+
         arr = [
             Path(Path(_get_module_from_class(clazz).__file__).parent, x)
             for clazz in get_subclasses_for_clazz(Service)
         ]
         arr = [path for path in arr if path.is_dir()]
-        return [Path("..", *path.parts[-6:]) for path in arr]
+        # Those pathes are absolute. Let's make them relative:
+        arr = [Path(*path.parts[-6:]) for path in arr]
+
+        # ... and a as the pyinstaller is in a subdir, let's add ..
+        arr = [Path("..", path) for path in arr]
+
+        # Non cryptoadvance extensions sitting in src/org/specterext/... need to be added, too
+        src_org_specterext_exts = search_dirs_in_path(
+            "../src/", return_without_extid=False
+        )
+        src_org_specterext_exts = [Path(path, x) for path in src_org_specterext_exts]
+
+        arr.extend(src_org_specterext_exts)
+        # Not only relative, as the pyinstaller is in a subdir, let's add ..
+
+        return arr
 
     @classmethod
     def get_service_packages(cls):

tests/test_managers_service.py

@@ -1,4 +1,5 @@
 import logging
+import os
 from unittest.mock import MagicMock
 from flask import Flask
 from cryptoadvance.specter.managers.service_manager import ServiceManager
@@ -32,22 +33,32 @@ def test_ServiceManager(caplog):
     assert sm.services["bitcoinreserve"] != None
     assert sm.services["swan"] != None
 
-    # THis is usefull in the pytinstaller/specterd.spec
-    dirs = ServiceManager.get_service_x_dirs("templates")
-    assert "../src/cryptoadvance/specter/services/swan/templates" in [
-        str(dir) for dir in dirs
-    ]
-    assert len(dirs) >= 1  # Should not need constant update
 
-    dirs = ServiceManager.get_service_x_dirs("static")
-    assert "../src/cryptoadvance/specter/services/swan/static" in [
-        str(dir) for dir in dirs
-    ]
-    assert "../src/cryptoadvance/specter/services/bitcoinreserve/static" in [
-        str(dir) for dir in dirs
-    ]
-    assert len(dirs) >= 2
+def test_ServiceManager_get_service_x_dirs():
+    try:
+        os.chdir("./pyinstaller")
+        # THis is usefull in the pytinstaller/specterd.spec
+        dirs = ServiceManager.get_service_x_dirs("templates")
+        assert "../src/cryptoadvance/specter/services/swan/templates" in [
+            str(dir) for dir in dirs
+        ]
+        for path in dirs:
+            assert str(path).endswith("templates")
+        assert len(dirs) == 2  # Should not need constant update
 
+        dirs = ServiceManager.get_service_x_dirs("static")
+        assert "../src/cryptoadvance/specter/services/swan/static" in [
+            str(dir) for dir in dirs
+        ]
+        assert "../src/cryptoadvance/specter/services/bitcoinreserve/static" in [
+            str(dir) for dir in dirs
+        ]
+        assert len(dirs) >= 2
+    finally:
+        os.chdir("../")
+
+
+def test_ServiceManager_get_service_packages():
     packages = ServiceManager.get_service_packages()
     assert "cryptoadvance.specter.services.swan.service" in packages
     assert "cryptoadvance.specter.services.bitcoinreserve.service" in packages