diff --git a/src/cryptoadvance/specter/managers/config_manager.py b/src/cryptoadvance/specter/managers/config_manager.py index 3ee602b09..f96fe6f43 100644 --- a/src/cryptoadvance/specter/managers/config_manager.py +++ b/src/cryptoadvance/specter/managers/config_manager.py @@ -22,7 +22,6 @@ class ConfigManager(GenericDataManager): with a lot of validation and computing while setting/getting """ - initial_data = {} name_of_json_file = "config.json" lock = threading.Lock() diff --git a/src/cryptoadvance/specter/managers/genericdata_manager.py b/src/cryptoadvance/specter/managers/genericdata_manager.py index 86529b58c..0e804ae4a 100644 --- a/src/cryptoadvance/specter/managers/genericdata_manager.py +++ b/src/cryptoadvance/specter/managers/genericdata_manager.py @@ -1,7 +1,9 @@ import os - +import logging from ..persistence import read_json_file, write_json_file +logger = logging.getLogger(__name__) + class GenericDataManager: """ @@ -9,7 +11,10 @@ class GenericDataManager: be derived from. See OtpManager """ - initial_data = {} + @classmethod + def initial_data(cls): + return {} + name_of_json_file = "some_data.json" # of them via json-files in an empty data folder @@ -24,13 +29,12 @@ class GenericDataManager: def load(self): # if whatever-the-name.json file exists - load from it if os.path.isfile(self.data_file): + logger.debug(f"Loading existing file {self.data_file}") self.data = read_json_file(self.data_file) # otherwise - create one and assign unique id else: - self.data = self.__class__.initial_data - # convert to stronger typed instances - # self.data = self.__class__.convert_to_list_of_type(data) - if not os.path.isfile(self.data_file): + logger.debug(f"{self.data_file} not existing. Creating ...") + self.data = self.__class__.initial_data() self._save() def _save(self): diff --git a/src/cryptoadvance/specter/managers/otp_manager.py b/src/cryptoadvance/specter/managers/otp_manager.py index 8d9a98194..0204f84e5 100644 --- a/src/cryptoadvance/specter/managers/otp_manager.py +++ b/src/cryptoadvance/specter/managers/otp_manager.py @@ -10,7 +10,6 @@ class OtpManager(GenericDataManager): The OtpManager manages otps (one time passwords) in otps.json """ - initial_data = {} name_of_json_file = "otps.json" def add_new_user_otp(self, otp_dict): diff --git a/src/cryptoadvance/specter/persistence.py b/src/cryptoadvance/specter/persistence.py index 3d55bf35a..f4e9dfa7f 100644 --- a/src/cryptoadvance/specter/persistence.py +++ b/src/cryptoadvance/specter/persistence.py @@ -3,15 +3,17 @@ call the call-back-method. """ -import os -import json import csv -import threading +import json import logging -from flask import current_app as app -from .util.shell import run_shell +import os import shutil +import threading +from flask import current_app as app + +from .specter_error import SpecterError +from .util.shell import run_shell logger = logging.getLogger(__name__) @@ -77,8 +79,10 @@ def _write_json_file(content, path, lock=None): if os.path.isfile(path): os.remove(path) shutil.copyfile(bkp, path) - logger.error(f"Failed to write to file {path}, rolled back to backup") - raise e + logger.exception(e) + raise SpecterError( + f"Error:{path} could not be saved. The old version has been restored. Check the logs for details. This is probably a bug." + ) def write_json_file(content, path, lock=None): diff --git a/src/cryptoadvance/specter/server.py b/src/cryptoadvance/specter/server.py index 085e648b2..ef41862ce 100644 --- a/src/cryptoadvance/specter/server.py +++ b/src/cryptoadvance/specter/server.py @@ -18,7 +18,7 @@ from .specter import Specter from .hwi_server import hwi_server from .user import User from .util.version import VersionChecker - +from .util.specter_migrator import SpecterMigrator from werkzeug.middleware.proxy_fix import ProxyFix from jinja2 import select_autoescape @@ -104,6 +104,10 @@ def create_app(config=None): def init_app(app, hwibridge=False, specter=None): """see blogpost 19nd Feb 2020""" + # First: Migrations + mm = SpecterMigrator(app.config["SPECTER_DATA_FOLDER"]) + mm.execute_migrations() + # Login via Flask-Login app.logger.info("Initializing LoginManager") app.secret_key = app.config["SECRET_KEY"] @@ -119,11 +123,6 @@ def init_app(app, hwibridge=False, specter=None): internal_bitcoind_version=app.config["INTERNAL_BITCOIND_VERSION"], ) - # version checker - # checks for new versions once per hour - specter.version = VersionChecker(specter=specter) - specter.version.start() - login_manager = LoginManager() login_manager.session_protection = "strong" login_manager.init_app(app) # Enable Login diff --git a/src/cryptoadvance/specter/specter.py b/src/cryptoadvance/specter/specter.py index 2eb42d827..9c7d9088a 100644 --- a/src/cryptoadvance/specter/specter.py +++ b/src/cryptoadvance/specter/specter.py @@ -41,6 +41,7 @@ from .specter_error import ExtProcTimeoutException, SpecterError from .tor_daemon import TorDaemonController from .user import User from .util.checker import Checker +from .util.version import VersionChecker from .util.price_providers import update_price from .util.setup_states import SETUP_STATES from .util.tor import get_tor_daemon_suffix @@ -66,6 +67,11 @@ class Specter: self.data_folder = data_folder + # version checker + # checks for new versions once per hour + self.version = VersionChecker(specter=self) + self.version.start() + self.user_manager = UserManager(self) self._config_manager = ConfigManager(self.data_folder, config) diff --git a/src/cryptoadvance/specter/specter_migrator.py b/src/cryptoadvance/specter/specter_migrator.py new file mode 100644 index 000000000..776bb4e09 --- /dev/null +++ b/src/cryptoadvance/specter/specter_migrator.py @@ -0,0 +1,256 @@ +""" The migrationManager handles all things necessary migrating from one specter version to another. + It uses patterns from DB-migration-frameworks like flask-migrate and the like. + The idea is to keep adding migration scripts to the SpecterMigrator and they are executed exactly + once. In order to keep track of that, the SpecterMigrator uses a MigDataManager which stores + events and migration_executions in a file called migration_data.json. + * Events are purely informational and only for troubleshooting purposes. + * migration_executions simply keeps track of which scrips has been executed. + +""" +import inspect +import logging +import os +from datetime import datetime +from importlib import import_module +from inspect import isclass +from pathlib import Path +from pkgutil import iter_modules + +from cryptoadvance import specter + +from .managers.genericdata_manager import GenericDataManager +from .specter_error import SpecterError +from .util.version import VersionChecker, compare + +logger = logging.getLogger(__name__) + + +class SpecterMigration: + """All Migrations should derive from this class. In order to create one, have a look at + the example in the migrations directory. + """ + + version = "custom" # the version this has been introduced + + def __init__(self, specter_migrator): + self.specter_migrator = specter_migrator + self.data_folder = specter_migrator.data_folder + + def should_execute(self): + """You can override this to prevent execution for specific cases + The default is that the version where this migration has been implemented needs to be + bigger than the version the user executed specter for the first time + """ + return self.__class__.should_execute_cls( + self.specter_migrator.mig.first_event["version"], self.__class__.version + ) + + @classmethod + def should_execute_cls(cls, version_first_started, migration_version): + """This is the default method and separated out for better testability""" + # We need to compare the version we have been started first with the version this migration + # is suppose to migrate. + # If version_first_started > self.version we don't need to execute this migration and will return False + try: + compare_version = compare(version_first_started, migration_version) + except SpecterError: + # if in doubt because versions are unparsable, execute! + return True + if compare_version == 1: + return True + elif compare_version == -1: + return False + elif compare_version == 0: + return False + + def execute(self): + logger.error( + f"This migration seem to have forgotten orverriding the execute-method!" + ) + + +class MigDataManager(GenericDataManager): + """Handling the data for the SpecterMigrator in migration_data.json with some convenience-methods""" + + @classmethod + def initial_data(cls): + return { + "events": [], # contains elements like {"timestamp": someTimestamp, "version": "v1.5.0"} + "migration_executions": [], # contains elements like {"timestamp": someTimestamp, "migration_id": 0} + } + + name_of_json_file = "migration_data.json" + + def __init__(self, data_folder): + # creating folders if they don't exist + # Usually Specter is doing that but in the case of tests, we're instantiated before Specter + if not os.path.isdir(data_folder): + os.makedirs(data_folder) + super().__init__(data_folder) + logger.info(f"Initiated {self}") + + @property + def events(self): + """A list of events where one event represents the initial use of a new specter-version which is a different + version than the one used before. Looks like: {"timestamp": someTimestamp, "version": "v1.5.0"} + """ + return self.data["events"] + + @property + def latest_event(self): + if self.events: + return self.events[-1] + return {"timestamp": None, "version": None} + + @property + def first_event(self): + """Effectively the binary version the data has been created with FIRST""" + if self.events: + return self.events[0] + raise SpecterError("You should not check the first_event before you've set it!") + + def create_new_event(self, version): + timestamp = datetime.now() + logger.debug(f"Creating new event with version {version} at {timestamp}") + self.events.append({"timestamp": str(timestamp), "version": version}) + self._save() + + def create_new_exec_log(self, migration_id, executing_version): + timestamp = datetime.now() + logger.debug( + f"Creating new Execution log with id {migration_id} at {timestamp}" + ) + self.migration_executions.append( + { + "timestamp": str(timestamp), + "migration_id": migration_id, + "status": "started", + "executing_version": executing_version, + } + ) + self._save() + + def set_execution_log_status(self, id, status): + logger.debug(f"Setting execution log status of {id} to {status}") + self._find_exec_log(id)["status"] = status + self._save() + + def set_Execution_log_error_msg(self, id, msg): + logger.debug(f"Setting execution log error_message of {id} to {msg}") + self._find_exec_log(id)["error_msg"] = msg + self._save() + + def _find_exec_log(self, id): + for migration in self.migration_executions: + if migration["migration_id"] == id: + return migration + raise SpecterError(f"Can't find migration_execution with id {id}") + + @property + def migration_executions(self): + return self.data["migration_executions"] + + def has_migration_executed(self, migration_id): + executed_list = [ + migration_execution["migration_id"] + for migration_execution in self.migration_executions + ] + logger.debug(f"Executed migration_classes ids: {executed_list}") + return migration_id in executed_list + + def __repr__(self): + return f"MigDataManager({self.data_file} events:{len(self.events)} execs:{len(self.migration_executions)} )" + + +class SpecterMigrator: + """Managing Migrations. Not calling it managers, as this is reserved for Instances attached to the specter instance""" + + def __init__(self, data_folder): + version = VersionChecker(specter=self) + self.current_binary_version = version.get_current_version() + self.current_data_version = "unknown" + self.data_folder = data_folder + self.mig = MigDataManager(data_folder) + if self.mig.latest_event["version"] != self.current_binary_version: + logger.info( + f"A new version has been started compared to last time: {self.current_binary_version}" + ) + self.mig.create_new_event(self.current_binary_version) + logger.debug(f"Initiated SpecterMigrator({self.mig})") + + def plan_migration(self): + """Returns a list of instances from all the migration_1234-classes which hasn't been + executed yet (according to migration_data.json) + """ + migration_objects_list = [] + # The path where all the migrations are located: + package_dir = str(Path(Path(__file__).resolve().parent, "migrations").resolve()) + for migration_class in self.get_migration_classes(): + migration_obj = migration_class(self) + + migration_id = SpecterMigrator.calculate_id(migration_obj) + if not self.mig.has_migration_executed(migration_id): + if migration_obj.should_execute(): + logger.debug( + f"Adding class {migration_class.__name__} to list of planned migrations" + ) + migration_objects_list.append(migration_obj) + else: + logger.debug( + f"Skipping class {migration_class.__name__} because of 'should_execute' False" + ) + return migration_objects_list + + def execute_migrations(self, migration_object_list=None): + if migration_object_list == None: + migration_object_list = self.plan_migration() + if not migration_object_list: + logger.info("No Migrations to execute!") + else: + for object in migration_object_list: + exec_id = SpecterMigrator.calculate_id(object) + try: + logger.info(f" --> Starting Migration {object.__class__.__name__}") + self.mig.create_new_exec_log(exec_id, self.current_binary_version) + object.execute() + logger.info( + f" --> Completed Migration {object.__class__.__name__}" + ) + self.mig.set_execution_log_status(exec_id, "completed") + except Exception as e: + logger.error( + f" --> Error in Migration {object.__class__.__name__}" + ) + logger.exception(e) + self.mig.set_execution_log_status(exec_id, "error") + self.mig.set_Execution_log_error_msg(exec_id, str(e)) + + @classmethod + def calculate_id(cls, migration_obj): + """Extract the id from ojects derived from classes like SpecterMigration_0123""" + prefix, id = migration_obj.__class__.__name__.split("_") + assert prefix == "SpecterMigration" + return int(id) + + @classmethod + def get_migration_classes(cls): + """Returns all subclasses of class SpecterMigration""" + class_list = [] + # The path where all the migrations are located: + package_dir = str(Path(Path(__file__).resolve().parent, "migrations").resolve()) + for (_, module_name, _) in iter_modules( + [package_dir] + ): # import the module and iterate through its attributes + module = import_module( + f"cryptoadvance.specter.util.migrations.{module_name}" + ) + logger.info("Collecting possible migrations ...") + for attribute_name in dir(module): + attribute = getattr(module, attribute_name) + if isclass(attribute): + if ( + issubclass(attribute, SpecterMigration) + and not attribute.__name__ == "SpecterMigration" + ): + class_list.append(attribute) + return class_list diff --git a/src/cryptoadvance/specter/util/migrations/__init__.py b/src/cryptoadvance/specter/util/migrations/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/cryptoadvance/specter/util/migrations/migration_0000.py b/src/cryptoadvance/specter/util/migrations/migration_0000.py new file mode 100644 index 000000000..0298badb8 --- /dev/null +++ b/src/cryptoadvance/specter/util/migrations/migration_0000.py @@ -0,0 +1,28 @@ +import logging +from ..specter_migrator import SpecterMigration + +logger = logging.getLogger(__name__) + + +class SpecterMigration_0000(SpecterMigration): + # the version that the stuff which is suppose to be migrated has been introduced + # If this instance has been started AFTER this version, then this migration will never + # get executed ( see the default-implementation of SpecterMigration.should_execute()) + version = "v1.5.0" # Faking it here for the unit-tests + + def __init__(self, specter_migrator): + self.specter_migrator = specter_migrator + + def execute(self): + logger.debug( + f"migration_0000 is an example method showing how to implement a migration" + ) + + @property + def description(self) -> str: + """Should return a (multiline) description of the migration which will get log.info() at execution-time""" + return """A dummy migration: + * It will do nothing + * It's just here to explain how SpecterMigration works + * It will be shown in the logs when it's executed (just like the other real migrations but doing nothing) + """ diff --git a/src/cryptoadvance/specter/util/migrations/migration_0001.py b/src/cryptoadvance/specter/util/migrations/migration_0001.py new file mode 100644 index 000000000..d53bdcf51 --- /dev/null +++ b/src/cryptoadvance/specter/util/migrations/migration_0001.py @@ -0,0 +1,129 @@ +import logging +import os +import shutil + +from urllib3.exceptions import NewConnectionError +from requests.exceptions import ConnectionError + +from cryptoadvance.specter.specter_error import SpecterError +from ...config import BaseConfig +from ..specter_migrator import SpecterMigration +from ...helpers import load_jsons +from ...managers.node_manager import NodeManager +import requests + +logger = logging.getLogger(__name__) + + +class SpecterMigration_0001(SpecterMigration): + version = "v1.6.1" # the version this migration has been rolled out + # irrelevant though because we'll execute this script in any case + # as we can't have yet a say on when specter has been started first + + def should_execute(self): + # This Migration cannot rely on the default-mechanism as the migration_framework was not + # in place when the functionality has been implemented + return True + + @property + def description(self) -> str: + return """Single-Node migration: + In v1.3.1 Single Node implementation has been implemented + Later we had multiple nodes. This migrates the single installation to one of many. + Effectively it will: + * Check whether an internal node was existing in ~/.specter/.bitcoin + * Check whether a new internal default node (bitcoin/main) is NOT existing + * Move the ~/.specter/.bitcoin to ~/.specter/nodes/specter_bitcoin/.bitcoin-main + * Creates a json-definition in ~/.specter/nodes/specter_bitcoin.json + """ + + def execute(self): + source_folder = os.path.join(self.data_folder, ".bitcoin") + if not os.path.isdir(source_folder): + logger.info( + "No .bitcoin directory found in {self.data_folder}. Nothing to do" + ) + return + if not os.path.isdir(os.path.join(self.data_folder, "bitcoin-binaries")): + raise SpecterError( + "Could not proceed with migration as bitcoin-binaries are not existing." + ) + if not self._check_port_free(): + logger.error( + "There is already a Node with the default port configured or running. Won't migrate!" + ) + return + # The version will be the version shipped with specter + bitcoin_version = BaseConfig.INTERNAL_BITCOIND_VERSION + logger.info(f".bitcoin directory detected in {self.data_folder}. Migrating ...") + recommended_name = self._find_appropriate_name() + target_folder = os.path.join(self.data_folder, "nodes", recommended_name) + logger.info(f"Migrating to folder {target_folder}") + os.makedirs(target_folder) + logger.info(f"Moving .bitcoin to folder {target_folder}") + shutil.move(source_folder, os.path.join(target_folder, ".bitcoin-main")) + if os.path.isdir(os.path.join(source_folder, "bitcoin.conf")): + logger.info("Removing bitcoin.conf file") + os.remove(os.path.join(source_folder, "bitcoin.conf")) + definition_file = os.path.join(target_folder, "specter_bitcoin.json") + logger.info( + f"Creating {definition_file}. This will cause some warnings and even errors about not being able to connect to the node which can be ignored." + ) + nm = NodeManager( + data_folder=os.path.join(self.data_folder, "nodes"), + bitcoind_path=os.path.join( + self.data_folder, "bitcoin-binaries", "bin", "bitcoind" + ), + internal_bitcoind_version=bitcoin_version, + ) + # Should create a json (see fullpath) like the one below: + node = nm.add_internal_node(recommended_name) + + # { + # "name": "Specter Bitcoin", + # "alias": "specter_bitcoin", + # "autodetect": false, + # "datadir": "/home/someuser/.specter/nodes/specter_bitcoin/.bitcoin-main", + # "user": "bitcoin", + # "password": "3ah0yc-2dDEwUSqHuuZi-w", + # "port": 8332, + # "host": "localhost", + # "protocol": "http", + # "external_node": false, + # "fullpath": "/home/someuser/.specter/nodes/specter_bitcoin.json", + # "bitcoind_path": "/home/someuser/.specter/bitcoin-binaries/bin/bitcoind", + # "bitcoind_network": "main", + # "version": "0.21.1" + # } + + def _find_appropriate_name(self): + if not os.path.isdir(os.path.join(self.data_folder, "nodes")): + return "specter_bitcoin" + if not os.path.isdir( + os.path.join(self.data_folder, "nodes", "specter_bitcoin") + ): + return "specter_bitcoin" + # Hmm, now it gets a bit trieckier + if not os.path.isdir( + os.path.join(self.data_folder, "nodes", "specter_migrated") + ): + return "specter_migrated" + # Now it's getting fishy + raise SpecterError( + "I found a node called 'specter_migrated'. This migration script should not run twice." + ) + + def _check_port_free(self, port=8332): + # For external nodes, we assume that there are already running + try: + result = requests.get(f"http://localhost:{port}") + return False + except (ConnectionRefusedError, ConnectionError, NewConnectionError): + pass + # Now let's check internal Nodes + if os.path.isfile(os.path.join(self.data_folder, "nodes")): + configs = load_jsons(os.path.join(self.data_folder, "nodes")) + ports = [node.port for node in configs.keys()] + if port in ports: + return False + return True diff --git a/src/cryptoadvance/specter/util/specter_migrator.py b/src/cryptoadvance/specter/util/specter_migrator.py new file mode 100644 index 000000000..c5e594475 --- /dev/null +++ b/src/cryptoadvance/specter/util/specter_migrator.py @@ -0,0 +1,269 @@ +""" The migrationManager handles all things necessary migrating from one specter version to another. + It uses patterns from DB-migration-frameworks like flask-migrate and the like. + The idea is to keep adding migration scripts to the SpecterMigrator and they are executed exactly + once. In order to keep track of that, the SpecterMigrator uses a MigDataManager which stores + events and migration_executions in a file called migration_data.json. + * Events are purely informational and only for troubleshooting purposes. + * migration_executions simply keeps track of which scrips has been executed. + +""" +import inspect +import logging +import os +from datetime import datetime +from importlib import import_module +from inspect import isclass +from pathlib import Path +from pkgutil import iter_modules + +from cryptoadvance import specter + +from ..managers.genericdata_manager import GenericDataManager +from ..specter_error import SpecterError +from .version import VersionChecker, compare + +logger = logging.getLogger(__name__) + + +class SpecterMigration: + """All Migrations should derive from this class. In order to create one, have a look at + the example in the migrations directory. + """ + + version = "custom" # the version this has been introduced + + def __init__(self, specter_migrator): + self.specter_migrator = specter_migrator + self.data_folder = specter_migrator.data_folder + + def should_execute(self): + """You can override this to prevent execution for specific cases + The default is that the version where this migration has been implemented needs to be + bigger than the version the user executed specter for the first time + """ + return self.__class__.should_execute_cls( + self.specter_migrator.mig.first_event["version"], self.__class__.version + ) + + @classmethod + def should_execute_cls(cls, version_first_started, migration_version): + """This is the default method and separated out for better testability""" + # We need to compare the version we have been started first with the version this migration + # is suppose to migrate. + # If version_first_started > self.version we don't need to execute this migration and will return False + try: + compare_version = compare(version_first_started, migration_version) + except SpecterError: + # if in doubt because versions are unparsable, execute! + return True + if compare_version == 1: + return True + elif compare_version == -1: + return False + elif compare_version == 0: + return False + + @property + def description(self) -> str: + """Should return a (multiline) description of the migration which will get log.info() at execution-time""" + # return """A dummy migration: + # * "foo" changed from str to list[str] to support multiple foos (PR #1234) + # * "bar" renamed to "thing" (PR #1235) + # * "blah" removed; no longer needed because... (PR #1236) + # """ + raise Exception( + "Must write a description of the changes the migration implements." + ) + + def execute(self): + logger.error( + f"This migration seem to have forgotten orverriding the execute-method!" + ) + + +class SpecterMigrator: + """A Class managing Migrations. Not calling it managers, as this is reserved for Instances attached to the specter instance""" + + def __init__(self, data_folder): + version = VersionChecker(specter=self) + self.current_binary_version = version.get_current_version() + self.current_data_version = "unknown" + self.data_folder = data_folder + self.mig = MigDataManager(data_folder) + if self.mig.latest_event["version"] != self.current_binary_version: + logger.info( + f"A new version has been started compared to last time: {self.current_binary_version}" + ) + self.mig.create_new_event(self.current_binary_version) + logger.debug(f"Initiated SpecterMigrator({self.mig})") + + def plan_migration(self): + """Returns a list of instances from all the migration_1234-classes which hasn't been + executed yet (according to migration_data.json) + """ + migration_objects_list = [] + # The path where all the migrations are located: + package_dir = str(Path(Path(__file__).resolve().parent, "migrations").resolve()) + for migration_class in self.get_migration_classes(): + migration_obj = migration_class(self) + + migration_id = SpecterMigrator.calculate_id(migration_obj) + if not self.mig.has_migration_executed(migration_id): + if migration_obj.should_execute(): + logger.debug( + f"Adding class {migration_class.__name__} to list of planned migrations" + ) + migration_objects_list.append(migration_obj) + else: + logger.debug( + f"Skipping class {migration_class.__name__} because of 'should_execute' False" + ) + return migration_objects_list + + def execute_migrations(self, migration_object_list=None): + if migration_object_list == None: + migration_object_list = self.plan_migration() + if not migration_object_list: + logger.info("No Migrations to execute!") + else: + for object in migration_object_list: + exec_id = SpecterMigrator.calculate_id(object) + try: + logger.info(f" --> Starting Migration {object.__class__.__name__}") + logger.info(object.description) + self.mig.create_new_exec_log(exec_id, self.current_binary_version) + object.execute() + logger.info( + f" --> Completed Migration {object.__class__.__name__}" + ) + self.mig.set_execution_log_status(exec_id, "completed") + except Exception as e: + logger.error( + f" --> Error in Migration {object.__class__.__name__}" + ) + logger.exception(e) + self.mig.set_execution_log_status(exec_id, "error") + self.mig.set_Execution_log_error_msg(exec_id, str(e)) + + @classmethod + def calculate_id(cls, migration_obj): + """Extract the id from ojects derived from classes like SpecterMigration_0123""" + prefix, id = migration_obj.__class__.__name__.split("_") + assert prefix == "SpecterMigration" + return int(id) + + @classmethod + def get_migration_classes(cls): + """Returns all subclasses of class SpecterMigration""" + class_list = [] + # The path where all the migrations are located: + package_dir = str(Path(Path(__file__).resolve().parent, "migrations").resolve()) + for (_, module_name, _) in iter_modules( + [package_dir] + ): # import the module and iterate through its attributes + module = import_module( + f"cryptoadvance.specter.util.migrations.{module_name}" + ) + logger.info("Collecting possible migrations ...") + for attribute_name in dir(module): + attribute = getattr(module, attribute_name) + if isclass(attribute): + if ( + issubclass(attribute, SpecterMigration) + and not attribute.__name__ == "SpecterMigration" + ): + class_list.append(attribute) + return class_list + + +class MigDataManager(GenericDataManager): + """A class handling the data for the SpecterMigrator in migration_data.json with some convenience-methods""" + + @classmethod + def initial_data(cls): + return { + "events": [], # contains elements like {"timestamp": someTimestamp, "version": "v1.5.0"} + "migration_executions": [], # contains elements like {"timestamp": someTimestamp, "migration_id": 0} + } + + name_of_json_file = "migration_data.json" + + def __init__(self, data_folder): + # creating folders if they don't exist + # Usually Specter is doing that but in the case of tests, we're instantiated before Specter + if not os.path.isdir(data_folder): + os.makedirs(data_folder) + super().__init__(data_folder) + logger.info(f"Initiated {self}") + + @property + def events(self): + """A list of events where one event represents the initial use of a new specter-version which is a different + version than the one used before. Looks like: {"timestamp": someTimestamp, "version": "v1.5.0"} + """ + return self.data["events"] + + @property + def latest_event(self): + if self.events: + return self.events[-1] + return {"timestamp": None, "version": None} + + @property + def first_event(self): + """Effectively the binary version the data has been created with FIRST""" + if self.events: + return self.events[0] + raise SpecterError("You should not check the first_event before you've set it!") + + def create_new_event(self, version): + timestamp = datetime.now() + logger.debug(f"Creating new event with version {version} at {timestamp}") + self.events.append({"timestamp": str(timestamp), "version": version}) + self._save() + + def create_new_exec_log(self, migration_id, executing_version): + timestamp = datetime.now() + logger.debug( + f"Creating new Execution log with id {migration_id} at {timestamp}" + ) + self.migration_executions.append( + { + "timestamp": str(timestamp), + "migration_id": migration_id, + "status": "started", + "executing_version": executing_version, + } + ) + self._save() + + def set_execution_log_status(self, id, status): + logger.debug(f"Setting execution log status of {id} to {status}") + self._find_exec_log(id)["status"] = status + self._save() + + def set_Execution_log_error_msg(self, id, msg): + logger.debug(f"Setting execution log error_message of {id} to {msg}") + self._find_exec_log(id)["error_msg"] = msg + self._save() + + def _find_exec_log(self, id): + for migration in self.migration_executions: + if migration["migration_id"] == id: + return migration + raise SpecterError(f"Can't find migration_execution with id {id}") + + @property + def migration_executions(self): + return self.data["migration_executions"] + + def has_migration_executed(self, migration_id): + executed_list = [ + migration_execution["migration_id"] + for migration_execution in self.migration_executions + ] + logger.debug(f"Executed migration_classes ids: {executed_list}") + return migration_id in executed_list + + def __repr__(self): + return f"MigDataManager({self.data_file} events:{len(self.events)} execs:{len(self.migration_executions)} )" diff --git a/src/cryptoadvance/specter/util/version.py b/src/cryptoadvance/specter/util/version.py index e9bf1462d..979e4cb2f 100644 --- a/src/cryptoadvance/specter/util/version.py +++ b/src/cryptoadvance/specter/util/version.py @@ -8,6 +8,8 @@ import os import requests import importlib_metadata +from cryptoadvance.specter.specter_error import SpecterError + logger = logging.getLogger(__name__) @@ -57,7 +59,8 @@ class VersionChecker: # check if it's installed from master if current == "vx.y.z-get-replaced-by-release-script": current = "custom" - except: + except Exception as e: + logger.exception(e) pass return current @@ -151,3 +154,53 @@ class VersionChecker: else: self.stop() return current, latest, False + + +def compare(version1: str, version2: str) -> int: + """Compares two version strings like v1.5.1 and v1.6.0 and returns + * 1 : version2 is bigger that version1 + * -1 : version1 is bigger than version2 + * 0 : both are the same + This is not supporting semver and it doesn't take any postfix (-pre5) + into account and is therefore a naive implementation + """ + version1 = _parse_version(version1) + version2 = _parse_version(version2) + + if version1["postfix"] != "" or version2["postfix"] != "": + raise SpecterError( + f"Cannot compare if either version has a postfix : {version1} and {version2}" + ) + if version1["major"] > version2["major"]: + return -1 + elif version1["major"] < version2["major"]: + return 1 + if version1["minor"] > version2["minor"]: + return -1 + elif version1["minor"] < version2["minor"]: + return 1 + if version1["patch"] > version2["patch"]: + return -1 + elif version1["patch"] < version2["patch"]: + return 1 + return 0 + + +def _parse_version(version: str) -> dict: + """Parses version-strings like v1.5.6-pre5 and returns a dict""" + if version[0] != "v": + raise SpecterError(f"version {version} does not have a preceding 'v'") + version = version[1:] + version_ar = version.split(".") + if len(version_ar) != 3: + raise SpecterError(f"version {version} does not have 3 separated digits") + postfix = "" + if "-" in version_ar[2]: + postfix = version_ar[2].split("-")[1] + version_ar[2] = version_ar[2].split("-")[0] + return { + "major": int(version_ar[0]), + "minor": int(version_ar[1]), + "patch": int(version_ar[2]), + "postfix": postfix, + } diff --git a/tests/conftest.py b/tests/conftest.py index 80397b79d..4f29cc534 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -26,6 +26,7 @@ from cryptoadvance.specter.specter_error import SpecterError from cryptoadvance.specter.user import User from cryptoadvance.specter.util.wallet_importer import WalletImporter from cryptoadvance.specter.util.common import str2bool +from cryptoadvance.specter.util.shell import which import code, traceback, signal logger = logging.getLogger(__name__) @@ -159,6 +160,22 @@ def instantiate_elementsd_controller(request, rpcport=18643, extra_args=[]): return elementsd_controller +# Below this point are fixtures. Fixtures have a scope. Check about scopes here: +# https://docs.pytest.org/en/6.2.x/fixture.html#scope-sharing-fixtures-across-classes-modules-packages-or-session +# possible values: function, class, module, package or session. +# The nodes are of scope session. All else is the default (function) + + +@pytest.fixture(scope="session") +def bitcoind_path(): + if os.path.isfile("tests/bitcoin/src/bitcoind"): + return "tests/bitcoin/src/bitcoind" + elif os.path.isfile("tests/bitcoin/bin/bitcoind"): + return "tests/bitcoin/bin/bitcoind" + else: + return which("bitcoind") + + @pytest.fixture(scope="session") def bitcoin_regtest(docker, request): bitcoind_regtest = instantiate_bitcoind_controller(docker, request, extra_args=None) diff --git a/tests/helpers_testdata/bitcoin_minimum_mainnet_datadir.tgz b/tests/helpers_testdata/bitcoin_minimum_mainnet_datadir.tgz new file mode 100644 index 000000000..8529222d4 Binary files /dev/null and b/tests/helpers_testdata/bitcoin_minimum_mainnet_datadir.tgz differ diff --git a/tests/test_managers_node.py b/tests/test_managers_node.py index 5e57a14d6..9e5ba9bb1 100644 --- a/tests/test_managers_node.py +++ b/tests/test_managers_node.py @@ -1,5 +1,8 @@ from enum import auto import tempfile +import time +import tarfile +import os import pytest from cryptoadvance.specter.managers.node_manager import NodeManager @@ -46,3 +49,48 @@ def test_NodeManager( assert nm.nodes_names == ["Bitcoin Core", "bitcoin_regtest", "elements_elreg"] nm.switch_node("elements_elreg") assert nm.active_node.get_rpc().getblockchaininfo()["chain"] == "elreg" + time.sleep(20) + + +""" For some reason this breaks other tests""" + + +@pytest.mark.skip() +def test_NodeManager_import(bitcoind_path): + with tempfile.TemporaryDirectory("_some_datafolder_tmp") as data_folder: + print(f"data_folder={data_folder}") + nm = NodeManager(data_folder=data_folder, bitcoind_path=bitcoind_path) + print(os.getcwd()) + # This .bitcoin folder doesn't have a config-file + btc_tar = tarfile.open( + "./tests/helpers_testdata/bitcoin_minimum_mainnet_datadir.tgz", "r:gz" + ) + btc_tar.extractall(os.path.join(data_folder, "somename", ".bitcoin-main")) + # # ... so let's create one + # with open( + # os.path.join(data_folder,"somename",".bitcoin-main","bitcoin.conf"), + # "w+", + # ) as file: + # file.write(''' + # rpcauth=bitcoin:044931c1d498b7c27080d8b981331a65$6ee7929513401c39ca1f7e376e55553c52dcb36a14e8410ac5f514fbf18bedbb + # server=1 + # listen=1 + # proxy=127.0.0.1:9050 + # bind=127.0.0.1 + # torcontrol=127.0.0.1:9051 + # torpassword=gVzREfuHso6U2OfRRvqT3w + # fallbackfee=0.0002 + # prune=1000 + # ''' + # ) + + node = nm.add_internal_node("somename", port=8339) + try: + node.start() + time.sleep(5) + # assert node.get_rpc().password == None + nm.switch_node("somename") + time.sleep(5) + assert nm.active_node.get_rpc().getblockchaininfo()["chain"] == "main" + finally: + node.stop() diff --git a/tests/test_specter_migrator.py b/tests/test_specter_migrator.py new file mode 100644 index 000000000..4d4eb334c --- /dev/null +++ b/tests/test_specter_migrator.py @@ -0,0 +1,144 @@ +import json +import logging +import os +import tarfile +import time + +import pytest +from cryptoadvance.specter.util.specter_migrator import ( + MigDataManager, + SpecterMigration, + SpecterMigrator, +) +from mock import Mock, patch + +logger = logging.getLogger(__name__) + + +def test_SpecterMigration(): + # If i started first with 1.6.1 and the migration has been implemented in 1.7.0, please do it + assert SpecterMigration.should_execute_cls("v1.6.1", "v1.7.0") + # Don't do it + assert not SpecterMigration.should_execute_cls("v1.7.0", "v1.7.0") + assert not SpecterMigration.should_execute_cls("v1.5.0", "v1.5.0") + assert not SpecterMigration.should_execute_cls("v1.6.1", "v1.5.0") + + +def test_SpecterMigrator_classnaming(): + for clazz in SpecterMigrator.get_migration_classes(): + # instantiation of a migration-class should not have any side-effects: + mig_obj: SpecterMigration = clazz(Mock()) + # will also tests the prefix implicitely + assert ( + SpecterMigrator.calculate_id(mig_obj) >= 0 + ), "a migration-class needs an id (int)" + # If implemented right, this should not throw an Exception: + assert type(mig_obj.description) == str + + +def test_SpecterMigrator_versioning(empty_data_folder, caplog): + caplog.set_level(logging.DEBUG) + with patch( + "cryptoadvance.specter.util.specter_migrator.VersionChecker" + ) as mock_version_checker_class: + # Specifiy a mock which will represent the VersionChecker Object + the_obj = Mock() + # Specify that the mock should return a funny version + the_obj.get_current_version.return_value = "v1.5.0" + # Specify that the constructor of the mock_version_checker_class should return the_obj + mock_version_checker_class.side_effect = [the_obj] + mm = SpecterMigrator(empty_data_folder) + mm.plan_migration() + assert mm.current_binary_version == "v1.5.0" + assert "Skipping class SpecterMigration_0000" in caplog.text + assert ( + "Skipping class SpecterMigration_0001" not in caplog.text + ) # this one overwrites should_execute + + +def test_SpecterMigrator_versioning2(empty_data_folder, caplog): + caplog.set_level(logging.DEBUG) + with patch( + "cryptoadvance.specter.util.specter_migrator.VersionChecker" + ) as mock_version_checker_class: + # Specifiy a mock which will represent the VersionChecker Object + the_obj = Mock() + # Specify that the mock should return a funny version + the_obj.get_current_version.return_value = "v1.7.1" + # Specify that the constructor of the mock_version_checker_class should return the_obj + mock_version_checker_class.side_effect = [the_obj] + mm = SpecterMigrator(empty_data_folder) + assert mm.current_binary_version == "v1.7.1" + assert "Skipping class SpecterMigration_0000" not in caplog.text + assert "Skipping class SpecterMigration_0001" not in caplog.text + + +def test_SpecterMigrator(empty_data_folder, caplog): + + caplog.set_level(logging.DEBUG) + assert MigDataManager.initial_data()["events"] == [] + assert MigDataManager.initial_data()["migration_executions"] == [] + assert len(os.listdir(empty_data_folder)) == 0 + # For migration1 + btc_tar = tarfile.open( + "./tests/helpers_testdata/bitcoin_minimum_mainnet_datadir.tgz", "r:gz" + ) + btc_tar.extractall(os.path.join(empty_data_folder, ".bitcoin")) + # Fake the existence of bitcoin-binaries + os.makedirs(os.path.join(empty_data_folder, "bitcoin-binaries")) + assert len(os.listdir(empty_data_folder)) == 2 + + # Patch the Class where it's used, not where it's defined + with patch( + "cryptoadvance.specter.util.specter_migrator.VersionChecker" + ) as mock_version_checker_class: + # Specifiy a mock which will represent the VersionChecker Object + the_obj = Mock() + # Specify that the mock should return a funny version + the_obj.get_current_version.return_value = "v1.6.1" + # Specify that the constructor of the mock_version_checker_class should return the_obj + mock_version_checker_class.side_effect = [the_obj] + mm = SpecterMigrator(empty_data_folder) + assert mm.current_binary_version == "v1.6.1" + + # not executed yet + assert "Setting execution log status of 1 to completed" not in caplog.text + # initally, zero executed migrations + assert len(mm.mig.migration_executions) == 0 + # With instantiation, the event get stored just right away + assert mm.mig.latest_event["version"] == "v1.6.1" + mylist = mm.plan_migration() + + # This assertion will break every time you create a new migration-script + assert len(mylist) == 1 + + mm.execute_migrations(mylist) + assert len(mm.mig.migration_executions) == 1 + assert "Setting execution log status of 1 to completed" in caplog.text + + assert mm.mig.migration_executions[0]["migration_id"] == 1 + assert os.path.isdir( + os.path.join( + empty_data_folder, + "nodes", + "specter_bitcoin", + ".bitcoin-main", + "chainstate", + ) + ) + specter_bitcoin_json = os.path.join( + empty_data_folder, "nodes", "specter_bitcoin.json" + ) + assert os.path.isfile(specter_bitcoin_json) + with open(specter_bitcoin_json) as jsonfile: + config = json.loads(jsonfile.read()) + assert config["name"] == "specter_bitcoin" + assert config["alias"] == "specter_bitcoin" + assert config["autodetect"] == False + assert config["datadir"].endswith("nodes/specter_bitcoin/.bitcoin-main") + assert config["user"] == "bitcoin" + assert config["password"] + assert config["port"] == 8332 + assert config["host"] == "localhost" + assert config["external_node"] == False + # yeah, some more but should be ok diff --git a/tests/test_util_version.py b/tests/test_util_version.py new file mode 100644 index 000000000..9c17bf37c --- /dev/null +++ b/tests/test_util_version.py @@ -0,0 +1,38 @@ +import pytest +from cryptoadvance.specter.specter_error import SpecterError +from cryptoadvance.specter.util.version import _parse_version, compare + + +def test_parse_version(): + assert _parse_version("v1.5.9") == { + "major": 1, + "minor": 5, + "patch": 9, + "postfix": "", + } + assert _parse_version("v3.4.5-pre12") == { + "major": 3, + "minor": 4, + "patch": 5, + "postfix": "pre12", + } + with pytest.raises(SpecterError): + _parse_version("3.4.5-pre12") + with pytest.raises(SpecterError): + _parse_version("3.4.5.6-pre12") + + +def test_compare(): + assert compare("v1.2.3", "v2.2.3") == 1 + assert compare("v2.2.3", "v1.2.3") == -1 + + assert compare("v2.1.3", "v2.2.3") == 1 + assert compare("v2.2.3", "v1.1.3") == -1 + + assert compare("v2.2.3", "v2.2.5") == 1 + assert compare("v2.2.5", "v2.2.2") == -1 + + assert compare("v2.2.5", "v2.2.5") == 0 + + with pytest.raises(SpecterError): + assert compare("v2.2.3-pre2", "v1.2.3") == -1