diff --git a/docs/services/services.md b/docs/services/services.md index b06ff0a1f..0c1268527 100644 --- a/docs/services/services.md +++ b/docs/services/services.md @@ -79,7 +79,6 @@ It is up to each `Service` implementation to decide what data is stored; the `Se This is also where `Service`-wide configuration or other information should be stored, _**even if it is not secret**_ (see above intro about not polluting other existing data stores). - ### `ServiceEncryptedStorageManager` Because the `ServiceEncryptedStorage` is specific to each individual user, this manager provides convenient access to automatically retrieve the `current_user` from the Flask context and provide the correct user's `ServiceEncryptedStorage`. It is implemented as a `Singleton` which can be retrieved simply by importing the class and calling `get_instance()`. @@ -99,6 +98,10 @@ def get_current_user_service_data(cls) -> dict: Whenever possible, external code should not directly access these `Service`-related support classes but rather should ask for them through the `Service` class. +### `ServiceUnencryptedStorage` +A disadvantage of the `ServiceEncryptedStorage` is, that the user needs to be freshly logged in in order to be able to decrypt the secrets. If you want to avoid that login but your extension should still store data on disk, you can use the `ServiceUnencryptedStorage`. + +In parallel with the `ServiceEncryptedStorageManager` there is also a `ServiceUnencryptedStorageManager` which is used exactly the same way. ### `ServiceAnnotationsStorage` Annotations are any address-specific or transaction-specific data from a `Service` that we might want to present to the user (not yet implemented). Example: a `Service` that integrates with a onchain storefront would have product/order data associated with a utxo. That additional data could be imported by the `Service` and stored as an annotation. This annotation data could then be displayed to the user when viewing the details for that particular address or tx. @@ -107,6 +110,11 @@ Annotations are stored on a per-wallet and per-`Service` basis as _unencrypted_ _Note: current `Service` implementations have not yet needed this feature so displaying annotations is not yet implemented._ +### callback methods +Your service-class will inherit a callback-method which will get called for various reasons with the "reason" being a string as the first parameter. Checkout the `cryptoadvance.specter.services.callbacks` file for the specific callbacks. + +Some important one is the `after_serverpy_init_app` which passes a `Scheduler` class which can be used to setup regular tasks. + ### `controller.py` The minimal url routes for `Service` selection and management. diff --git a/requirements.in b/requirements.in index f0bbf9563..d094a1f89 100644 --- a/requirements.in +++ b/requirements.in @@ -23,3 +23,4 @@ pgpy==0.5.4 cbor==1.0.0 mnemonic==0.20 cryptography==3.4.7 +Flask-APScheduler==1.12.3 diff --git a/requirements.txt b/requirements.txt index 07476972b..261fc4b9d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,10 +8,32 @@ aniso8601==9.0.1 \ --hash=sha256:1d2b7ef82963909e93c4f24ce48d4de9e66009a21bf1c1e1c85bdd0812fe412f \ --hash=sha256:72e3117667eedf66951bb2d93f4296a56b94b078a8a95905a052611fb3f1b973 \ # via flask-restful +apscheduler==3.9.1 \ + --hash=sha256:65e6574b6395498d371d045f2a8a7e4f7d50c6ad21ef7313d15b1c7cf20df1e3 \ + --hash=sha256:ddc25a0ddd899de44d7f451f4375fb971887e65af51e41e5dcf681f59b8b2c9a \ + # via flask-apscheduler babel==2.9.1 \ --hash=sha256:ab49e12b91d937cd11f0b67cb259a57ab4ad2b59ac7a3b41d6c06c0ac5b0def9 \ --hash=sha256:bc0c176f9f6a994582230df350aa6e05ba2ebe4b3ac317eab29d9be5d2768da0 \ # via flask-babel +backports.zoneinfo==0.2.1 \ + --hash=sha256:17746bd546106fa389c51dbea67c8b7c8f0d14b5526a579ca6ccf5ed72c526cf \ + --hash=sha256:1b13e654a55cd45672cb54ed12148cd33628f672548f373963b0bff67b217328 \ + --hash=sha256:1c5742112073a563c81f786e77514969acb58649bcdf6cdf0b4ed31a348d4546 \ + --hash=sha256:4a0f800587060bf8880f954dbef70de6c11bbe59c673c3d818921f042f9954a6 \ + --hash=sha256:5c144945a7752ca544b4b78c8c41544cdfaf9786f25fe5ffb10e838e19a27570 \ + --hash=sha256:7b0a64cda4145548fed9efc10322770f929b944ce5cee6c0dfe0c87bf4c0c8c9 \ + --hash=sha256:8439c030a11780786a2002261569bdf362264f605dfa4d65090b64b05c9f79a7 \ + --hash=sha256:8961c0f32cd0336fb8e8ead11a1f8cd99ec07145ec2931122faaac1c8f7fd987 \ + --hash=sha256:89a48c0d158a3cc3f654da4c2de1ceba85263fafb861b98b59040a5086259722 \ + --hash=sha256:a76b38c52400b762e48131494ba26be363491ac4f9a04c1b7e92483d169f6582 \ + --hash=sha256:da6013fd84a690242c310d77ddb8441a559e9cb3d3d59ebac9aca1a57b2e18bc \ + --hash=sha256:e55b384612d93be96506932a786bbcde5a2db7a9e6a4bb4bffe8b733f5b9036b \ + --hash=sha256:e81b76cace8eda1fca50e345242ba977f9be6ae3945af8d46326d776b4cf78d1 \ + --hash=sha256:e8236383a20872c0cdf5a62b554b27538db7fa1bbec52429d8d106effbaeca08 \ + --hash=sha256:f04e857b59d9d1ccc39ce2da1021d196e47234873820cbeaad210724b1ee28ac \ + --hash=sha256:fadbfe37f74051d024037f223b8e001611eac868b5c5b06144ef4d8b799862f2 \ + # via pytz-deprecation-shim, tzlocal base58==2.1.0 \ --hash=sha256:171a547b4a3c61e1ae3807224a6f7aec75e364c4395e7562649d7335768001a2 \ --hash=sha256:8225891d501b68c843ffe30b86371f844a21c6ba00da76f52f9b998ba771fb48 \ @@ -111,6 +133,9 @@ ecdsa==0.17.0 \ embit==0.4.12 \ --hash=sha256:d340107dc1604581df59f844d4eb76ec34b0219c2ac2cbc1837c14938a4730ee \ # via -r requirements.in +flask-apscheduler==1.12.3 \ + --hash=sha256:d60948d1f2be9eb4772f68c3308ba3f973755219d13947266f89292ad6df63fc \ + # via -r requirements.in flask-babel==2.0.0 \ --hash=sha256:e6820a052a8d344e178cdd36dd4bb8aea09b4bda3d5f9fa9f008df2c7f2f5468 \ --hash=sha256:f9faf45cdb2e1a32ea2ec14403587d4295108f35017a7821a2b1acb8cfd9257d \ @@ -134,7 +159,7 @@ flask-restful==0.3.9 \ flask==1.1.4 \ --hash=sha256:0fbeb6180d383a9186d0d6ed954e0042ad9f18e0e8de088b2b419d526927d196 \ --hash=sha256:c34f04500f2cbbea882b1acb02002ad6fe6b7ffa64a6164577995657f50aed22 \ - # via -r requirements.in, flask-babel, flask-cors, flask-httpauth, flask-login, flask-restful, flask-wtf + # via -r requirements.in, flask-apscheduler, flask-babel, flask-cors, flask-httpauth, flask-login, flask-restful, flask-wtf flask_wtf==0.14.3 \ --hash=sha256:57b3faf6fe5d6168bda0c36b0df1d05770f8e205e18332d0376ddb954d17aef2 \ --hash=sha256:d417e3a0008b5ba583da1763e4db0f55a1269d9dd91dcc3eb3c026d3c5dbd720 \ @@ -301,14 +326,22 @@ pysocks==1.7.1 \ --hash=sha256:2725bd0a9925919b9b51739eea5f9e2bae91e83288108a9ad338b2e3a4435ee5 \ --hash=sha256:3f8804571ebe159c380ac6de37643bb4685970655d3bba243530d6558b799aa0 \ # via -r requirements.in +python-dateutil==2.8.2 \ + --hash=sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86 \ + --hash=sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9 \ + # via flask-apscheduler python-dotenv==0.13.0 \ --hash=sha256:25c0ff1a3e12f4bde8d592cc254ab075cfe734fc5dd989036716fd17ee7e5ec7 \ --hash=sha256:3b9909bc96b0edc6b01586e1eed05e71174ef4e04c71da5786370cebea53ad74 \ # via -r requirements.in +pytz-deprecation-shim==0.1.0.post0 \ + --hash=sha256:8314c9692a636c8eb3bda879b9f119e350e93223ae83e70e80c31675a0fdc1a6 \ + --hash=sha256:af097bae1b616dde5c5744441e2ddc69e74dfdcb0c263129610d85b87445a59d \ + # via tzlocal pytz==2021.1 \ --hash=sha256:83a4a90894bf38e243cf052c8b58f381bfe9a7a483f6a9cab140bc7f702ac4da \ --hash=sha256:eb10ce3e7736052ed3623d49975ce333bcd712c7bb19a58b9e2089d4057d0798 \ - # via babel, flask-babel, flask-restful + # via apscheduler, babel, flask-babel, flask-restful requests==2.26.0 \ --hash=sha256:6c1246513ecd5ecd4528a0906f910e8f0f9c6b8ec72030dc9fd154dc1a6efd24 \ --hash=sha256:b8aa58f8cf793ffd8782d3d8cb19e66ef36f7aba4353eec859e74678b01b07a7 \ @@ -320,7 +353,7 @@ semver==2.13.0 \ six==1.16.0 \ --hash=sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926 \ --hash=sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254 \ - # via -r requirements.in, ecdsa, flask-cors, flask-restful, pgpy, protobuf, pyopenssl + # via -r requirements.in, apscheduler, ecdsa, flask-cors, flask-restful, pgpy, protobuf, pyopenssl, python-dateutil stem==1.8.0 \ --hash=sha256:a0b48ea6224e95f22aa34c0bc3415f0eb4667ddeae3dfb5e32a6920c185568c2 \ # via -r requirements.in @@ -329,6 +362,14 @@ typing-extensions==3.10.0.0 \ --hash=sha256:50b6f157849174217d0656f99dc82fe932884fb250826c18350e159ec6cdf342 \ --hash=sha256:779383f6086d90c99ae41cf0ff39aac8a7937a9283ce0a414e5dd782f4c94a84 \ # via bitbox02, hwi +tzdata==2021.5 \ + --hash=sha256:3eee491e22ebfe1e5cfcc97a4137cd70f092ce59144d81f8924a844de05ba8f5 \ + --hash=sha256:68dbe41afd01b867894bbdfd54fa03f468cfa4f0086bfb4adcd8de8f24f3ee21 \ + # via pytz-deprecation-shim +tzlocal==4.1 \ + --hash=sha256:0f28015ac68a5c067210400a9197fc5d36ba9bc3f8eaf1da3cbd59acdfed9e09 \ + --hash=sha256:28ba8d9fcb6c9a782d6e0078b4f6627af1ea26aeaa32b4eab5324abc7df4149f \ + # via apscheduler urllib3==1.26.5 \ --hash=sha256:753a0374df26658f99d826cfe40394a686d05985786d946fbe4165b5148f5a7c \ --hash=sha256:a7acd0977125325f516bda9735fa7142b909a8d01e8b2e4c8108d0984e6e0098 \ diff --git a/src/cryptoadvance/specter/managers/service_manager.py b/src/cryptoadvance/specter/managers/service_manager.py index 9d5c13d91..3ba03139e 100644 --- a/src/cryptoadvance/specter/managers/service_manager.py +++ b/src/cryptoadvance/specter/managers/service_manager.py @@ -18,7 +18,10 @@ from flask.blueprints import Blueprint from ..services.service import Service from ..services import callbacks, ExtensionException -from ..services.service_encrypted_storage import ServiceEncryptedStorageManager +from ..services.service_encrypted_storage import ( + ServiceEncryptedStorageManager, + ServiceUnencryptedStorageManager, +) from ..util.reflection import ( _get_module_from_class, get_classlist_of_type_clazz_from_modulelist, @@ -36,6 +39,7 @@ class ServiceManager: def __init__(self, specter, devstatus_threshold): self.specter = specter + specter.ext = {} self.devstatus_threshold = devstatus_threshold # Each Service class is stored here, keyed on its Service.id str @@ -50,9 +54,12 @@ class ServiceManager: class_list = get_classlist_of_type_clazz_from_modulelist( Service, app.config.get("EXTENSION_LIST", []) ) - logger.info("----> starting service discovery Dynamic") + if app.config.get("SERVICES_LOAD_FROM_CWD", False): + logger.info("----> starting service discovery dynamic") class_list.extend(get_subclasses_for_clazz_in_cwd(Service)) + else: + logger.info("----> skipping service discovery dynamic") logger.info("----> starting service loading") class_list = set(class_list) # remove duplicates (shouldn't happen but ...) for clazz in class_list: @@ -65,6 +72,7 @@ class ServiceManager: active=clazz.id in self.specter.config.get("services", []), specter=self.specter, ) + self.specter.ext[clazz.id] = self._services[clazz.id] # maybe register the blueprint self.register_blueprint_for_ext(clazz, self._services[clazz.id]) logger.info(f"Service {clazz.__name__} activated ({clazz.devstatus})") @@ -81,6 +89,11 @@ class ServiceManager: except ConfigurableSingletonException as e: # Test suite triggers multiple calls; ignore for now. pass + + specter.service_unencrypted_storage_manager = ServiceUnencryptedStorageManager( + specter.user_manager, specter.data_folder + ) + logger.info("----> finished service processing") self.execute_ext_callbacks("afterServiceManagerInit") diff --git a/src/cryptoadvance/specter/server.py b/src/cryptoadvance/specter/server.py index d12c00aa2..bf1992d71 100644 --- a/src/cryptoadvance/specter/server.py +++ b/src/cryptoadvance/specter/server.py @@ -6,10 +6,11 @@ from pathlib import Path from cryptoadvance.specter.liquid.rpc import LiquidRPC from cryptoadvance.specter.managers.service_manager import ServiceManager from cryptoadvance.specter.rpc import BitcoinRPC +from cryptoadvance.specter.services import callbacks from cryptoadvance.specter.util.reflection import get_template_static_folder -from .services.callbacks import after_serverpy_init_app from dotenv import load_dotenv from flask import Flask, jsonify, redirect, request, session, url_for +from flask_apscheduler import APScheduler from flask_babel import Babel from flask_login import LoginManager, login_user from flask_wtf.csrf import CSRFProtect @@ -19,6 +20,7 @@ from werkzeug.middleware.proxy_fix import ProxyFix from werkzeug.wrappers import Response from .hwi_server import hwi_server +from .services.callbacks import after_serverpy_init_app from .specter import Specter from .util.specter_migrator import SpecterMigrator @@ -106,7 +108,7 @@ def create_app(config=None): return app -def init_app(app, hwibridge=False, specter=None): +def init_app(app: SpecterFlask, hwibridge=False, specter=None): """see blogpost 19nd Feb 2020""" # Configuring a prefix for the app @@ -232,7 +234,24 @@ def init_app(app, hwibridge=False, specter=None): return jsonify(success=False) # --------------------- Babel integration --------------------- - specter.service_manager.execute_ext_callbacks(after_serverpy_init_app) + + # Background Scheduler + def every5seconds(): + ctx = app.app_context() + ctx.push() + app.specter.service_manager.execute_ext_callbacks(callbacks.every5seconds) + ctx.pop() + + # initialize scheduler + from apscheduler.schedulers.background import BackgroundScheduler + + scheduler = APScheduler() + + scheduler.init_app(app) + scheduler.start() + specter.service_manager.execute_ext_callbacks( + after_serverpy_init_app, scheduler=scheduler + ) return app diff --git a/src/cryptoadvance/specter/services/service.py b/src/cryptoadvance/specter/services/service.py index 4e3b7c9d2..9eafe4013 100644 --- a/src/cryptoadvance/specter/services/service.py +++ b/src/cryptoadvance/specter/services/service.py @@ -14,6 +14,7 @@ from .service_encrypted_storage import ServiceEncryptedStorageManager from .service_annotations_storage import ServiceAnnotationsStorage from cryptoadvance.specter.addresslist import Address +from cryptoadvance.specter.services import callbacks logger = logging.getLogger(__name__) @@ -46,6 +47,11 @@ class Service: self.active = active self.specter = specter + def callback(self, callback_id, *argv, **kwargv): + if callback_id == callbacks.after_serverpy_init_app: + if hasattr(self, "callback_after_serverpy_init_app"): + self.callback_after_serverpy_init_app(kwargv["scheduler"]) + @classmethod def set_current_user_service_data(cls, service_data: dict): ServiceEncryptedStorageManager.get_instance().set_current_user_service_data( diff --git a/src/cryptoadvance/specter/services/service_encrypted_storage.py b/src/cryptoadvance/specter/services/service_encrypted_storage.py index 5ce92073a..41720d003 100644 --- a/src/cryptoadvance/specter/services/service_encrypted_storage.py +++ b/src/cryptoadvance/specter/services/service_encrypted_storage.py @@ -36,6 +36,7 @@ class ServiceEncryptedStorage(GenericDataManager): """ def __init__(self, data_folder: str, user: User, disable_decrypt: bool = False): + if not user.plaintext_user_secret and not disable_decrypt: raise ServiceEncryptedStorageError( f"User {user} must be authenticated with password before encrypted service data can be loaded" @@ -98,6 +99,26 @@ class ServiceEncryptedStorage(GenericDataManager): return service_data +class ServiceUnencryptedStorage(ServiceEncryptedStorage): + """In order to use ServiceEncryptedStorage but unencrypted, we derive from that class + and change the datafile. + """ + + def __init__(self, data_folder: str, user: User, disable_decrypt: bool = False): + if not disable_decrypt: + raise Exception( + "ServiceUnencryptedStorage needs to be initialized with disable_decrypt = True" + ) + if disable_decrypt: + super().__init__(data_folder, encryption_key=None, disable_decrypt=True) + + @property + def data_file(self): + return os.path.join( + self.data_folder, f"{self.user.username}_unencrypted_services.json" + ) + + class ServiceEncryptedStorageManager(ConfigurableSingleton): """Singleton that manages access to users' ServiceApiKeyStorage; context-aware so it knows who the current_user is for the given request context. @@ -160,3 +181,20 @@ class ServiceEncryptedStorageManager(ConfigurableSingleton): ) encrypted_storage.data = {} encrypted_storage._save() + + +class ServiceUnencryptedStorageManager(ServiceEncryptedStorageManager): + def __init__(self, user_manager, data_folder): + self.user_manager = user_manager + self.data_folder = data_folder + self.storage_by_user = {} + + def _get_current_user_service_storage(self) -> ServiceEncryptedStorage: + """Returns the storage-class for the current_user. Lazy_init if necessary""" + user = self.user_manager.get_user() + + if user not in self.storage_by_user: + self.storage_by_user[user] = ServiceUnencryptedStorage( + self.data_folder, user, disable_decrypt=True + ) + return self.storage_by_user[user] diff --git a/src/cryptoadvance/specter/templates/includes/sidebar/sidebar.jinja b/src/cryptoadvance/specter/templates/includes/sidebar/sidebar.jinja index c334e3585..934429830 100644 --- a/src/cryptoadvance/specter/templates/includes/sidebar/sidebar.jinja +++ b/src/cryptoadvance/specter/templates/includes/sidebar/sidebar.jinja @@ -303,7 +303,7 @@ } if ('{{ specter.service_manager.service_names | length }}' == '0') { - document.getElementById('toggle_services_list').innerHTML = 'Services'; + document.getElementById('toggle_services_list').innerHTML = 'Plugins'; } else { document.getElementById('toggle_services_list').addEventListener('click', (event) => { toggleList('services'); diff --git a/src/cryptoadvance/specter/templates/services/sidebar_services_list.jinja b/src/cryptoadvance/specter/templates/services/sidebar_services_list.jinja index 4add058c6..8eba07899 100644 --- a/src/cryptoadvance/specter/templates/services/sidebar_services_list.jinja +++ b/src/cryptoadvance/specter/templates/services/sidebar_services_list.jinja @@ -1,11 +1,11 @@
- Services   ▼ + Plugins   ▼
- {% for _,service in specter.service_manager.services.items() %} - {% if service.id in current_user.services %} - -  {{ service.name }} + {% for _,plugin in specter.service_manager.services.items() %} + {% if plugin.id in current_user.services %} + +  {{ plugin.name }}
{% endif %} diff --git a/src/cryptoadvance/specter/util/reflection.py b/src/cryptoadvance/specter/util/reflection.py index 78f37986a..3d5a9d718 100644 --- a/src/cryptoadvance/specter/util/reflection.py +++ b/src/cryptoadvance/specter/util/reflection.py @@ -9,6 +9,7 @@ import sys from typing import List from .common import camelcase2snake_case from ..specter_error import SpecterError +from .shell import grep from .reflection_fs import detect_extension_style_in_cwd, search_dirs_in_path @@ -100,8 +101,12 @@ def get_subclasses_for_clazz_in_cwd(clazz, cwd=".") -> List[type]: # if not testing but in a folder which looks like specter-desktop/src --> No dynamic extensions if "PYTEST_CURRENT_TEST" not in os.environ: - if Path("./src/cryptoadvance").is_dir(): - return [] + # Hmm, need a better way to detect a specter-desktop-sourcedir + try: + if grep("./setup.py", 'name="cryptoadvance.specter",'): + return [] + except FileNotFoundError: + pass # Depending on the style we either add "." or "./src" to the searchpath diff --git a/src/cryptoadvance/specter/util/shell.py b/src/cryptoadvance/specter/util/shell.py index 1863a0bb5..c58de5139 100644 --- a/src/cryptoadvance/specter/util/shell.py +++ b/src/cryptoadvance/specter/util/shell.py @@ -55,9 +55,18 @@ def run_shell(cmd): return {"code": 0xF00DBABE, "out": b"", "err": e} -def get_last_lines_from_file(file_localtion, x=50): +def get_last_lines_from_file(file_location, x=50): """returns an array of the last x lines of a file""" - with open(file_localtion, "r") as the_file: + with open(file_location, "r") as the_file: lines = the_file.readlines() last_lines = lines[-x:] return last_lines + + +def grep(file_location, search_line): + """returns true if any like in that file endswith search_line""" + with open(file_location, "r") as the_file: + for line in the_file.readlines(): + if line.strip().endswith(search_line): + return True + return False diff --git a/tests/test_util_shell.py b/tests/test_util_shell.py index 866ee1129..575b86db2 100644 --- a/tests/test_util_shell.py +++ b/tests/test_util_shell.py @@ -21,3 +21,9 @@ def test_last_lines(caplog): lines = get_last_lines_from_file("LICENSE", 30) assert lines[-2].startswith("OUT OF OR IN CONNECTION WITH THE SOFTWARE ") + + +def test_grep(): + from cryptoadvance.specter.util.shell import grep + + assert grep("./setup.py", 'name="cryptoadvance.specter",')