Fixing the linux-binary-build (#396)

.gitlab-ci.yml

@@ -54,9 +54,15 @@ release_pip:
     - ls -l dist && python3 -m twine upload --verbose --user __token__ dist/*
 
 release_binary_linux:
+  image: registry.gitlab.com/cryptoadvance/specter-desktop/bionic-build:latest
   stage: releasing
   only: 
     - tags
+  before_script:
+    - python -V
+    - pip install virtualenv
+    - virtualenv --python=python3 .env
+    - source .env/bin/activate
   script:
     # Make sure the version-number is compatibe to the scheme
     - if ! [[ $CI_COMMIT_TAG =~ ^v(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)(-((0|[1-9][0-9]*|[0-9]*[a-zA-Z-][0-9a-zA-Z-]*)(\.(0|[1-9][0-9]*|[0-9]*[a-zA-Z-][0-9a-zA-Z-]*))*))?(\+([0-9a-zA-Z-]+(\.[0-9a-zA-Z-]+)*))?$ ]]; then exit 1; fi
@@ -64,6 +70,7 @@ release_binary_linux:
     - cd pyinstaller && ./build-unix.sh $CI_COMMIT_TAG
     - echo $GH_BIN_UPLOAD_PW | github-binary-upload -u gitlab_upload_release_binaries cryptoadvance/specter-desktop $CI_COMMIT_TAG ./release/specterd-${CI_COMMIT_TAG}-x86_64-linux-gnu.tar.gz ./release/specter_desktop-${CI_COMMIT_TAG}-x86_64-linux-gnu.tar.gz
 
+
 release_binary_windows:
   stage: releasing
   only: 

docker/bionic-build/Dockerfile

@@ -0,0 +1,7 @@
+FROM registry.gitlab.com/cryptoadvance/specter-desktop/python:3.8.5-bionic
+
+RUN apt-get update && apt-get install -y --no-install-recommends libusb-1.0-0-dev libudev-dev
+
+RUN apt-get install -y --no-install-recommends libgl1-mesa-dri gvfs gvfs-libs \
+        libdrm-amdgpu1 libdrm-nouveau2 libdrm-radeon1 libedit2 libelf1 libllvm10 \
+        libvulkan1 libzstd1 libtdb1 libcanberra-gtk3-0 virtualenv libcanberra-gtk3-module

docker/bionic-build/Readme.md

@@ -0,0 +1,8 @@
+This Dockerimage is manually created and uploaded:
+
+```
+docker build -t registry.gitlab.com/cryptoadvance/specter-desktop/bionic-build:latest .
+docker push registry.gitlab.com/cryptoadvance/specter-desktop/bionic-build:latest
+```
+
+The reason for this image is explained in [#356](https://github.com/cryptoadvance/specter-desktop/issues/356).

docker/python-bionic/Dockerfile

@@ -0,0 +1,101 @@
+#
+# NOTE: THIS DOCKERFILE IS GENERATED VIA "update.sh"
+#
+# PLEASE DO NOT EDIT IT DIRECTLY.
+#
+
+FROM buildpack-deps:bionic
+
+# ensure local python is preferred over distribution python
+ENV PATH /usr/local/bin:$PATH
+
+# http://bugs.python.org/issue19846
+# > At the moment, setting "LANG=C" on a Linux system *fundamentally breaks Python 3*, and that's not OK.
+ENV LANG C.UTF-8
+
+# fixing tzdata
+ENV TZ=Europe/Berlin
+RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone
+# extra dependencies (over what buildpack-deps already includes)
+RUN apt-get update && apt-get install -y --no-install-recommends \
+		libbluetooth-dev \
+		tk-dev \
+		uuid-dev \
+	&& rm -rf /var/lib/apt/lists/*
+
+ENV GPG_KEY E3FF2839C048B25C084DEBE9B26995E310250568
+ENV PYTHON_VERSION 3.8.5
+
+RUN set -ex \
+	\
+	&& wget -O python.tar.xz "https://www.python.org/ftp/python/${PYTHON_VERSION%%[a-z]*}/Python-$PYTHON_VERSION.tar.xz" \
+	&& wget -O python.tar.xz.asc "https://www.python.org/ftp/python/${PYTHON_VERSION%%[a-z]*}/Python-$PYTHON_VERSION.tar.xz.asc" \
+	&& export GNUPGHOME="$(mktemp -d)" \
+	&& gpg --batch --keyserver ha.pool.sks-keyservers.net --recv-keys "$GPG_KEY" \
+	&& gpg --batch --verify python.tar.xz.asc python.tar.xz \
+	&& { command -v gpgconf > /dev/null && gpgconf --kill all || :; } \
+	&& rm -rf "$GNUPGHOME" python.tar.xz.asc \
+	&& mkdir -p /usr/src/python \
+	&& tar -xJC /usr/src/python --strip-components=1 -f python.tar.xz \
+	&& rm python.tar.xz \
+	\
+	&& cd /usr/src/python \
+	&& gnuArch="$(dpkg-architecture --query DEB_BUILD_GNU_TYPE)" \
+	&& ./configure \
+		--build="$gnuArch" \
+		--enable-loadable-sqlite-extensions \
+		--enable-optimizations \
+		--enable-option-checking=fatal \
+		--enable-shared \
+		--with-system-expat \
+		--with-system-ffi \
+		--without-ensurepip \
+	&& make -j "$(nproc)" \
+	&& make install \
+	&& rm -rf /usr/src/python \
+	\
+	&& find /usr/local -depth \
+		\( \
+			\( -type d -a \( -name test -o -name tests -o -name idle_test \) \) \
+			-o \( -type f -a \( -name '*.pyc' -o -name '*.pyo' -o -name '*.a' \) \) \
+			-o \( -type f -a -name 'wininst-*.exe' \) \
+		\) -exec rm -rf '{}' + \
+	\
+	&& ldconfig \
+	\
+	&& python3 --version
+
+# make some useful symlinks that are expected to exist
+RUN cd /usr/local/bin \
+	&& ln -s idle3 idle \
+	&& ln -s pydoc3 pydoc \
+	&& ln -s python3 python \
+	&& ln -s python3-config python-config
+
+# if this is called "PIP_VERSION", pip explodes with "ValueError: invalid truth value '<VERSION>'"
+ENV PYTHON_PIP_VERSION 20.2.3
+# https://github.com/pypa/get-pip
+ENV PYTHON_GET_PIP_URL https://github.com/pypa/get-pip/raw/fa7dc83944936bf09a0e4cb5d5ec852c0d256599/get-pip.py
+ENV PYTHON_GET_PIP_SHA256 6e0bb0a2c2533361d7f297ed547237caf1b7507f197835974c0dd7eba998c53c
+
+RUN set -ex; \
+	\
+	wget -O get-pip.py "$PYTHON_GET_PIP_URL"; \
+	echo "$PYTHON_GET_PIP_SHA256 *get-pip.py" | sha256sum --check --strict -; \
+	\
+	python get-pip.py \
+		--disable-pip-version-check \
+		--no-cache-dir \
+		"pip==$PYTHON_PIP_VERSION" \
+	; \
+	pip --version; \
+	\
+	find /usr/local -depth \
+		\( \
+			\( -type d -a \( -name test -o -name tests -o -name idle_test \) \) \
+			-o \
+			\( -type f -a \( -name '*.pyc' -o -name '*.pyo' \) \) \
+		\) -exec rm -rf '{}' +; \
+	rm -f get-pip.py
+
+CMD ["python3"]

docker/python-bionic/Readme.md

@@ -0,0 +1,18 @@
+This Dockerimage is manually created and uploaded:
+
+```
+docker build -t registry.gitlab.com/cryptoadvance/specter-desktop/python:3.8.5-bionic .
+docker push registry.gitlab.com/cryptoadvance/specter-desktop/python:3.8.5-bionic
+```
+
+The reason for this image is explained in [#356](https://github.com/cryptoadvance/specter-desktop/issues/356). The Dockerfile is more or less created like this:
+
+```
+curl https://raw.githubusercontent.com/docker-library/python/9ff5f04241c7bcb224303ff8cea9434e9976f8af/3.8/buster/Dockerfile > Dockerfile
+sed -i -e "s/FROM buildpack-deps:buster/FROM buildpack-deps:bionic/" Dockerfile
+sed -i -e '15 a # fixing tzdata' Dockerfile
+sed -i -e '16 a ENV TZ=Europe/Berlin' Dockerfile
+sed -i -e '17 a RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone' Dockerfile
+```
+
+Check [this Directory](../python-build) for how this image is used.

pyinstaller/specter_desktop.py

@@ -385,12 +385,19 @@ def init_desktop_app():
     # fix termination ctrl+c
     signal.signal(signal.SIGINT, sigint_handler)
 
+    # This is the place to uncomment if we ever have issues like
+    # https://github.com/cryptoadvance/specter-desktop/issues/373 again
+    # So maybe let's keep it in here.
+    # import psutil
+    # print("-----------------------------------------------------------------------")
+    # print(psutil.Process().memory_maps())
+    # print("-----------------------------------------------------------------------")
+    
     # Create the icon
     icon = QIcon(os.path.join(
         resource_path('icons'),
         'icon.png'
     ))
-
     # Create the tray
     tray = QSystemTrayIcon()
     tray.setIcon(icon)

pyinstaller/specter_desktop.spec

@@ -2,6 +2,7 @@
 import platform
 import subprocess
 import mnemonic, os, sys
+import psutil
 
 mnemonic_path = os.path.join(mnemonic.__path__[0], "wordlist")
 
@@ -11,10 +12,24 @@ binaries = []
 if platform.system() == 'Windows':
     binaries = [("./windll/libusb-1.0.dll", ".")]
 elif platform.system() == 'Linux':
-    if platform.processor() == 'aarch64': #ARM 64 bit
-        binaries = [("/lib/aarch64-linux-gnu/libusb-1.0.so.0", ".")]
-    else:
-        binaries = [("/lib/x86_64-linux-gnu/libusb-1.0.so.0", ".")]
+    arch = platform.processor()
+    binaries = [(f"/lib/{arch}-linux-gnu/libusb-1.0.so.0", "."),
+                (f"/usr/lib/{arch}-linux-gnu/dri/iris_dri.so","."),
+                (f"/usr/lib/{arch}-linux-gnu/gio/modules/libgvfsdbus.so","."),
+                (f"/usr/lib/{arch}-linux-gnu/gvfs/libgvfscommon.so","."),
+                (f"/usr/lib/{arch}-linux-gnu/gtk-3.0/modules/libcanberra-gtk3-module.so","."),
+                (f"/usr/lib/{arch}-linux-gnu/libcanberra-gtk3.so.0","."),
+                (f"/usr/lib/{arch}-linux-gnu/libcanberra.so.0","."),
+                (f"/usr/lib/{arch}-linux-gnu/libdrm_amdgpu.so.1","."),
+                (f"/usr/lib/{arch}-linux-gnu/libdrm_nouveau.so.2","."),
+                (f"/usr/lib/{arch}-linux-gnu/libdrm_radeon.so.1","."),
+                (f"/usr/lib/{arch}-linux-gnu/libedit.so.2","."),
+                (f"/usr/lib/{arch}-linux-gnu/libelf.so.1","."),
+                (f"/usr/lib/{arch}-linux-gnu/libLLVM-10.so.1","."),
+                (f"/usr/lib/{arch}-linux-gnu/libltdl.so.7","."),
+                (f"/usr/lib/{arch}-linux-gnu/libsensors.so.4","."),
+                (f"/usr/lib/{arch}-linux-gnu/libtdb.so.1","."),
+    ]
 elif platform.system() == 'Darwin':
     find_brew_libusb_proc = subprocess.Popen(['brew', '--prefix', 'libusb'], stdout=subprocess.PIPE)
     libusb_path = find_brew_libusb_proc.communicate()[0]
@@ -29,7 +44,8 @@ a = Analysis(['specter_desktop.py'],
              ],
              hiddenimports=[
                 'pkg_resources.py2_warn',
-                'cryptoadvance.specter.config'
+                'cryptoadvance.specter.config',
+                'psutil'
              ],
              hookspath=['hooks/'],
              runtime_hooks=[],
@@ -88,7 +104,7 @@ if sys.platform == 'linux':
         debug=False,
         strip=False,
         upx=True,
-        runtime_tmpdir=None,
+        #runtime_tmpdir=None,
         console=False,
         icon='icons/icon.ico'
     )