GlcoinCore/specter-desktop
1
0
Fork 0
mirror of https://github.com/cryptoadvance/specter-desktop.git synced 2026-08-13 12:33:29 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions 8

Feature: a framework for migrations and migrating single-node (#1414)

Browse source 
* a framework for migrations

* forgot testfile

* fix tests

* adding debugging for fixing tests

* troubleshoot tests

* check empty_folder

* moar logging

* some documentation

* kick

* kick

* kick

* kick (pers + logging)

* kick (assert)

* kick (reworked initial_data)

* remove False, final fix hopefully

* Preventing a clash with an existing node

* refactorings and tests

* Moving the whole stuff to util

* Some refactorings and adding description
This commit is contained in:
Kim Neunert 2021-10-13 09:05:26 +02:00 • committed by GitHub
parent 326d7c0eb5
commit 03ead8f62a
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
17 changed files with 1015 additions and 22 deletions
Download patch file Download diff file Expand all files Collapse all files

1
src/cryptoadvance/specter/managers/config_manager.py
View file

@ -22,7 +22,6 @@ class ConfigManager(GenericDataManager):
with a lot of validation and computing while setting/getting
"""
initial_data = {}
name_of_json_file = "config.json"
lock = threading.Lock()

16
src/cryptoadvance/specter/managers/genericdata_manager.py
View file

@ -1,7 +1,9 @@
import os
import logging
from ..persistence import read_json_file, write_json_file
logger = logging.getLogger(__name__)
class GenericDataManager:
"""
@ -9,7 +11,10 @@ class GenericDataManager:
be derived from. See OtpManager
"""
initial_data = {}
@classmethod
def initial_data(cls):
return {}
name_of_json_file = "some_data.json"
# of them via json-files in an empty data folder
@ -24,13 +29,12 @@ class GenericDataManager:
def load(self):
# if whatever-the-name.json file exists - load from it
if os.path.isfile(self.data_file):
logger.debug(f"Loading existing file {self.data_file}")
self.data = read_json_file(self.data_file)
# otherwise - create one and assign unique id
else:
self.data = self.__class__.initial_data
# convert to stronger typed instances
# self.data = self.__class__.convert_to_list_of_type(data)
if not os.path.isfile(self.data_file):
logger.debug(f"{self.data_file} not existing. Creating ...")
self.data = self.__class__.initial_data()
self._save()
def _save(self):

1
src/cryptoadvance/specter/managers/otp_manager.py
View file

@ -10,7 +10,6 @@ class OtpManager(GenericDataManager):
The OtpManager manages otps (one time passwords) in otps.json
"""
initial_data = {}
name_of_json_file = "otps.json"
def add_new_user_otp(self, otp_dict):

18
src/cryptoadvance/specter/persistence.py
View file

@ -3,15 +3,17 @@
call the call-back-method.
"""
import os
import json
import csv
import threading
import json
import logging
from flask import current_app as app
from .util.shell import run_shell
import os
import shutil
import threading
from flask import current_app as app
from .specter_error import SpecterError
from .util.shell import run_shell
logger = logging.getLogger(__name__)
@ -77,8 +79,10 @@ def _write_json_file(content, path, lock=None):
if os.path.isfile(path):
os.remove(path)
shutil.copyfile(bkp, path)
logger.error(f"Failed to write to file {path}, rolled back to backup")
raise e
logger.exception(e)
raise SpecterError(
f"Error:{path} could not be saved. The old version has been restored. Check the logs for details. This is probably a bug."
)
def write_json_file(content, path, lock=None):

11
src/cryptoadvance/specter/server.py
View file

@ -18,7 +18,7 @@ from .specter import Specter
from .hwi_server import hwi_server
from .user import User
from .util.version import VersionChecker
from .util.specter_migrator import SpecterMigrator
from werkzeug.middleware.proxy_fix import ProxyFix
from jinja2 import select_autoescape
@ -104,6 +104,10 @@ def create_app(config=None):
def init_app(app, hwibridge=False, specter=None):
"""see blogpost 19nd Feb 2020"""
# First: Migrations
mm = SpecterMigrator(app.config["SPECTER_DATA_FOLDER"])
mm.execute_migrations()
# Login via Flask-Login
app.logger.info("Initializing LoginManager")
app.secret_key = app.config["SECRET_KEY"]
@ -119,11 +123,6 @@ def init_app(app, hwibridge=False, specter=None):
internal_bitcoind_version=app.config["INTERNAL_BITCOIND_VERSION"],
)
# version checker
# checks for new versions once per hour
specter.version = VersionChecker(specter=specter)
specter.version.start()
login_manager = LoginManager()
login_manager.session_protection = "strong"
login_manager.init_app(app) # Enable Login

6
src/cryptoadvance/specter/specter.py
View file

@ -41,6 +41,7 @@ from .specter_error import ExtProcTimeoutException, SpecterError
from .tor_daemon import TorDaemonController
from .user import User
from .util.checker import Checker
from .util.version import VersionChecker
from .util.price_providers import update_price
from .util.setup_states import SETUP_STATES
from .util.tor import get_tor_daemon_suffix
@ -66,6 +67,11 @@ class Specter:
self.data_folder = data_folder
# version checker
# checks for new versions once per hour
self.version = VersionChecker(specter=self)
self.version.start()
self.user_manager = UserManager(self)
self._config_manager = ConfigManager(self.data_folder, config)

256
src/cryptoadvance/specter/specter_migrator.py Normal file
View file

@ -0,0 +1,256 @@
""" The migrationManager handles all things necessary migrating from one specter version to another.
It uses patterns from DB-migration-frameworks like flask-migrate and the like.
The idea is to keep adding migration scripts to the SpecterMigrator and they are executed exactly
once. In order to keep track of that, the SpecterMigrator uses a MigDataManager which stores
events and migration_executions in a file called migration_data.json.
* Events are purely informational and only for troubleshooting purposes.
* migration_executions simply keeps track of which scrips has been executed.
"""
import inspect
import logging
import os
from datetime import datetime
from importlib import import_module
from inspect import isclass
from pathlib import Path
from pkgutil import iter_modules
from cryptoadvance import specter
from .managers.genericdata_manager import GenericDataManager
from .specter_error import SpecterError
from .util.version import VersionChecker, compare
logger = logging.getLogger(__name__)
class SpecterMigration:
"""All Migrations should derive from this class. In order to create one, have a look at
the example in the migrations directory.
"""
version = "custom" # the version this has been introduced
def __init__(self, specter_migrator):
self.specter_migrator = specter_migrator
self.data_folder = specter_migrator.data_folder
def should_execute(self):
"""You can override this to prevent execution for specific cases
The default is that the version where this migration has been implemented needs to be
bigger than the version the user executed specter for the first time
"""
return self.__class__.should_execute_cls(
self.specter_migrator.mig.first_event["version"], self.__class__.version
)
@classmethod
def should_execute_cls(cls, version_first_started, migration_version):
"""This is the default method and separated out for better testability"""
# We need to compare the version we have been started first with the version this migration
# is suppose to migrate.
# If version_first_started > self.version we don't need to execute this migration and will return False
try:
compare_version = compare(version_first_started, migration_version)
except SpecterError:
# if in doubt because versions are unparsable, execute!
return True
if compare_version == 1:
return True
elif compare_version == -1:
return False
elif compare_version == 0:
return False
def execute(self):
logger.error(
f"This migration seem to have forgotten orverriding the execute-method!"
)
class MigDataManager(GenericDataManager):
"""Handling the data for the SpecterMigrator in migration_data.json with some convenience-methods"""
@classmethod
def initial_data(cls):
return {
"events": [], # contains elements like {"timestamp": someTimestamp, "version": "v1.5.0"}
"migration_executions": [], # contains elements like {"timestamp": someTimestamp, "migration_id": 0}
}
name_of_json_file = "migration_data.json"
def __init__(self, data_folder):
# creating folders if they don't exist
# Usually Specter is doing that but in the case of tests, we're instantiated before Specter
if not os.path.isdir(data_folder):
os.makedirs(data_folder)
super().__init__(data_folder)
logger.info(f"Initiated {self}")
@property
def events(self):
"""A list of events where one event represents the initial use of a new specter-version which is a different
version than the one used before. Looks like: {"timestamp": someTimestamp, "version": "v1.5.0"}
"""
return self.data["events"]
@property
def latest_event(self):
if self.events:
return self.events[-1]
return {"timestamp": None, "version": None}
@property
def first_event(self):
"""Effectively the binary version the data has been created with FIRST"""
if self.events:
return self.events[0]
raise SpecterError("You should not check the first_event before you've set it!")
def create_new_event(self, version):
timestamp = datetime.now()
logger.debug(f"Creating new event with version {version} at {timestamp}")
self.events.append({"timestamp": str(timestamp), "version": version})
self._save()
def create_new_exec_log(self, migration_id, executing_version):
timestamp = datetime.now()
logger.debug(
f"Creating new Execution log with id {migration_id} at {timestamp}"
)
self.migration_executions.append(
{
"timestamp": str(timestamp),
"migration_id": migration_id,
"status": "started",
"executing_version": executing_version,
}
)
self._save()
def set_execution_log_status(self, id, status):
logger.debug(f"Setting execution log status of {id} to {status}")
self._find_exec_log(id)["status"] = status
self._save()
def set_Execution_log_error_msg(self, id, msg):
logger.debug(f"Setting execution log error_message of {id} to {msg}")
self._find_exec_log(id)["error_msg"] = msg
self._save()
def _find_exec_log(self, id):
for migration in self.migration_executions:
if migration["migration_id"] == id:
return migration
raise SpecterError(f"Can't find migration_execution with id {id}")
@property
def migration_executions(self):
return self.data["migration_executions"]
def has_migration_executed(self, migration_id):
executed_list = [
migration_execution["migration_id"]
for migration_execution in self.migration_executions
]
logger.debug(f"Executed migration_classes ids: {executed_list}")
return migration_id in executed_list
def __repr__(self):
return f"MigDataManager({self.data_file} events:{len(self.events)} execs:{len(self.migration_executions)} )"
class SpecterMigrator:
"""Managing Migrations. Not calling it managers, as this is reserved for Instances attached to the specter instance"""
def __init__(self, data_folder):
version = VersionChecker(specter=self)
self.current_binary_version = version.get_current_version()
self.current_data_version = "unknown"
self.data_folder = data_folder
self.mig = MigDataManager(data_folder)
if self.mig.latest_event["version"] != self.current_binary_version:
logger.info(
f"A new version has been started compared to last time: {self.current_binary_version}"
)
self.mig.create_new_event(self.current_binary_version)
logger.debug(f"Initiated SpecterMigrator({self.mig})")
def plan_migration(self):
"""Returns a list of instances from all the migration_1234-classes which hasn't been
executed yet (according to migration_data.json)
"""
migration_objects_list = []
# The path where all the migrations are located:
package_dir = str(Path(Path(__file__).resolve().parent, "migrations").resolve())
for migration_class in self.get_migration_classes():
migration_obj = migration_class(self)
migration_id = SpecterMigrator.calculate_id(migration_obj)
if not self.mig.has_migration_executed(migration_id):
if migration_obj.should_execute():
logger.debug(
f"Adding class {migration_class.__name__} to list of planned migrations"
)
migration_objects_list.append(migration_obj)
else:
logger.debug(
f"Skipping class {migration_class.__name__} because of 'should_execute' False"
)
return migration_objects_list
def execute_migrations(self, migration_object_list=None):
if migration_object_list == None:
migration_object_list = self.plan_migration()
if not migration_object_list:
logger.info("No Migrations to execute!")
else:
for object in migration_object_list:
exec_id = SpecterMigrator.calculate_id(object)
try:
logger.info(f" --> Starting Migration {object.__class__.__name__}")
self.mig.create_new_exec_log(exec_id, self.current_binary_version)
object.execute()
logger.info(
f" --> Completed Migration {object.__class__.__name__}"
)
self.mig.set_execution_log_status(exec_id, "completed")
except Exception as e:
logger.error(
f" --> Error in Migration {object.__class__.__name__}"
)
logger.exception(e)
self.mig.set_execution_log_status(exec_id, "error")
self.mig.set_Execution_log_error_msg(exec_id, str(e))
@classmethod
def calculate_id(cls, migration_obj):
"""Extract the id from ojects derived from classes like SpecterMigration_0123"""
prefix, id = migration_obj.__class__.__name__.split("_")
assert prefix == "SpecterMigration"
return int(id)
@classmethod
def get_migration_classes(cls):
"""Returns all subclasses of class SpecterMigration"""
class_list = []
# The path where all the migrations are located:
package_dir = str(Path(Path(__file__).resolve().parent, "migrations").resolve())
for (_, module_name, _) in iter_modules(
[package_dir]
): # import the module and iterate through its attributes
module = import_module(
f"cryptoadvance.specter.util.migrations.{module_name}"
)
logger.info("Collecting possible migrations ...")
for attribute_name in dir(module):
attribute = getattr(module, attribute_name)
if isclass(attribute):
if (
issubclass(attribute, SpecterMigration)
and not attribute.__name__ == "SpecterMigration"
):
class_list.append(attribute)
return class_list

0
src/cryptoadvance/specter/util/migrations/__init__.py Normal file
View file

28
src/cryptoadvance/specter/util/migrations/migration_0000.py Normal file
View file

@ -0,0 +1,28 @@
import logging
from ..specter_migrator import SpecterMigration
logger = logging.getLogger(__name__)
class SpecterMigration_0000(SpecterMigration):
# the version that the stuff which is suppose to be migrated has been introduced
# If this instance has been started AFTER this version, then this migration will never
# get executed ( see the default-implementation of SpecterMigration.should_execute())
version = "v1.5.0" # Faking it here for the unit-tests
def __init__(self, specter_migrator):
self.specter_migrator = specter_migrator
def execute(self):
logger.debug(
f"migration_0000 is an example method showing how to implement a migration"
)
@property
def description(self) -> str:
"""Should return a (multiline) description of the migration which will get log.info() at execution-time"""
return """A dummy migration:
* It will do nothing
* It's just here to explain how SpecterMigration works
* It will be shown in the logs when it's executed (just like the other real migrations but doing nothing)
"""

129
src/cryptoadvance/specter/util/migrations/migration_0001.py Normal file
View file

@ -0,0 +1,129 @@
import logging
import os
import shutil
from urllib3.exceptions import NewConnectionError
from requests.exceptions import ConnectionError
from cryptoadvance.specter.specter_error import SpecterError
from ...config import BaseConfig
from ..specter_migrator import SpecterMigration
from ...helpers import load_jsons
from ...managers.node_manager import NodeManager
import requests
logger = logging.getLogger(__name__)
class SpecterMigration_0001(SpecterMigration):
version = "v1.6.1" # the version this migration has been rolled out
# irrelevant though because we'll execute this script in any case
# as we can't have yet a say on when specter has been started first
def should_execute(self):
# This Migration cannot rely on the default-mechanism as the migration_framework was not
# in place when the functionality has been implemented
return True
@property
def description(self) -> str:
return """Single-Node migration:
In v1.3.1 Single Node implementation has been implemented
Later we had multiple nodes. This migrates the single installation to one of many.
Effectively it will:
* Check whether an internal node was existing in ~/.specter/.bitcoin
* Check whether a new internal default node (bitcoin/main) is NOT existing
* Move the ~/.specter/.bitcoin to ~/.specter/nodes/specter_bitcoin/.bitcoin-main
* Creates a json-definition in ~/.specter/nodes/specter_bitcoin.json
"""
def execute(self):
source_folder = os.path.join(self.data_folder, ".bitcoin")
if not os.path.isdir(source_folder):
logger.info(
"No .bitcoin directory found in {self.data_folder}. Nothing to do"
)
return
if not os.path.isdir(os.path.join(self.data_folder, "bitcoin-binaries")):
raise SpecterError(
"Could not proceed with migration as bitcoin-binaries are not existing."
)
if not self._check_port_free():
logger.error(
"There is already a Node with the default port configured or running. Won't migrate!"
)
return
# The version will be the version shipped with specter
bitcoin_version = BaseConfig.INTERNAL_BITCOIND_VERSION
logger.info(f".bitcoin directory detected in {self.data_folder}. Migrating ...")
recommended_name = self._find_appropriate_name()
target_folder = os.path.join(self.data_folder, "nodes", recommended_name)
logger.info(f"Migrating to folder {target_folder}")
os.makedirs(target_folder)
logger.info(f"Moving .bitcoin to folder {target_folder}")
shutil.move(source_folder, os.path.join(target_folder, ".bitcoin-main"))
if os.path.isdir(os.path.join(source_folder, "bitcoin.conf")):
logger.info("Removing bitcoin.conf file")
os.remove(os.path.join(source_folder, "bitcoin.conf"))
definition_file = os.path.join(target_folder, "specter_bitcoin.json")
logger.info(
f"Creating {definition_file}. This will cause some warnings and even errors about not being able to connect to the node which can be ignored."
)
nm = NodeManager(
data_folder=os.path.join(self.data_folder, "nodes"),
bitcoind_path=os.path.join(
self.data_folder, "bitcoin-binaries", "bin", "bitcoind"
),
internal_bitcoind_version=bitcoin_version,
)
# Should create a json (see fullpath) like the one below:
node = nm.add_internal_node(recommended_name)
# {
# "name": "Specter Bitcoin",
# "alias": "specter_bitcoin",
# "autodetect": false,
# "datadir": "/home/someuser/.specter/nodes/specter_bitcoin/.bitcoin-main",
# "user": "bitcoin",
# "password": "3ah0yc-2dDEwUSqHuuZi-w",
# "port": 8332,
# "host": "localhost",
# "protocol": "http",
# "external_node": false,
# "fullpath": "/home/someuser/.specter/nodes/specter_bitcoin.json",
# "bitcoind_path": "/home/someuser/.specter/bitcoin-binaries/bin/bitcoind",
# "bitcoind_network": "main",
# "version": "0.21.1"
# }
def _find_appropriate_name(self):
if not os.path.isdir(os.path.join(self.data_folder, "nodes")):
return "specter_bitcoin"
if not os.path.isdir(
os.path.join(self.data_folder, "nodes", "specter_bitcoin")
):
return "specter_bitcoin"
# Hmm, now it gets a bit trieckier
if not os.path.isdir(
os.path.join(self.data_folder, "nodes", "specter_migrated")
):
return "specter_migrated"
# Now it's getting fishy
raise SpecterError(
"I found a node called 'specter_migrated'. This migration script should not run twice."
)
def _check_port_free(self, port=8332):
# For external nodes, we assume that there are already running
try:
result = requests.get(f"http://localhost:{port}")
return False
except (ConnectionRefusedError, ConnectionError, NewConnectionError):
pass
# Now let's check internal Nodes
if os.path.isfile(os.path.join(self.data_folder, "nodes")):
configs = load_jsons(os.path.join(self.data_folder, "nodes"))
ports = [node.port for node in configs.keys()]
if port in ports:
return False
return True

269
src/cryptoadvance/specter/util/specter_migrator.py Normal file
View file

@ -0,0 +1,269 @@
""" The migrationManager handles all things necessary migrating from one specter version to another.
It uses patterns from DB-migration-frameworks like flask-migrate and the like.
The idea is to keep adding migration scripts to the SpecterMigrator and they are executed exactly
once. In order to keep track of that, the SpecterMigrator uses a MigDataManager which stores
events and migration_executions in a file called migration_data.json.
* Events are purely informational and only for troubleshooting purposes.
* migration_executions simply keeps track of which scrips has been executed.
"""
import inspect
import logging
import os
from datetime import datetime
from importlib import import_module
from inspect import isclass
from pathlib import Path
from pkgutil import iter_modules
from cryptoadvance import specter
from ..managers.genericdata_manager import GenericDataManager
from ..specter_error import SpecterError
from .version import VersionChecker, compare
logger = logging.getLogger(__name__)
class SpecterMigration:
"""All Migrations should derive from this class. In order to create one, have a look at
the example in the migrations directory.
"""
version = "custom" # the version this has been introduced
def __init__(self, specter_migrator):
self.specter_migrator = specter_migrator
self.data_folder = specter_migrator.data_folder
def should_execute(self):
"""You can override this to prevent execution for specific cases
The default is that the version where this migration has been implemented needs to be
bigger than the version the user executed specter for the first time
"""
return self.__class__.should_execute_cls(
self.specter_migrator.mig.first_event["version"], self.__class__.version
)
@classmethod
def should_execute_cls(cls, version_first_started, migration_version):
"""This is the default method and separated out for better testability"""
# We need to compare the version we have been started first with the version this migration
# is suppose to migrate.
# If version_first_started > self.version we don't need to execute this migration and will return False
try:
compare_version = compare(version_first_started, migration_version)
except SpecterError:
# if in doubt because versions are unparsable, execute!
return True
if compare_version == 1:
return True
elif compare_version == -1:
return False
elif compare_version == 0:
return False
@property
def description(self) -> str:
"""Should return a (multiline) description of the migration which will get log.info() at execution-time"""
# return """A dummy migration:
# * "foo" changed from str to list[str] to support multiple foos (PR #1234)
# * "bar" renamed to "thing" (PR #1235)
# * "blah" removed; no longer needed because... (PR #1236)
# """
raise Exception(
"Must write a description of the changes the migration implements."
)
def execute(self):
logger.error(
f"This migration seem to have forgotten orverriding the execute-method!"
)
class SpecterMigrator:
"""A Class managing Migrations. Not calling it managers, as this is reserved for Instances attached to the specter instance"""
def __init__(self, data_folder):
version = VersionChecker(specter=self)
self.current_binary_version = version.get_current_version()
self.current_data_version = "unknown"
self.data_folder = data_folder
self.mig = MigDataManager(data_folder)
if self.mig.latest_event["version"] != self.current_binary_version:
logger.info(
f"A new version has been started compared to last time: {self.current_binary_version}"
)
self.mig.create_new_event(self.current_binary_version)
logger.debug(f"Initiated SpecterMigrator({self.mig})")
def plan_migration(self):
"""Returns a list of instances from all the migration_1234-classes which hasn't been
executed yet (according to migration_data.json)
"""
migration_objects_list = []
# The path where all the migrations are located:
package_dir = str(Path(Path(__file__).resolve().parent, "migrations").resolve())
for migration_class in self.get_migration_classes():
migration_obj = migration_class(self)
migration_id = SpecterMigrator.calculate_id(migration_obj)
if not self.mig.has_migration_executed(migration_id):
if migration_obj.should_execute():
logger.debug(
f"Adding class {migration_class.__name__} to list of planned migrations"
)
migration_objects_list.append(migration_obj)
else:
logger.debug(
f"Skipping class {migration_class.__name__} because of 'should_execute' False"
)
return migration_objects_list
def execute_migrations(self, migration_object_list=None):
if migration_object_list == None:
migration_object_list = self.plan_migration()
if not migration_object_list:
logger.info("No Migrations to execute!")
else:
for object in migration_object_list:
exec_id = SpecterMigrator.calculate_id(object)
try:
logger.info(f" --> Starting Migration {object.__class__.__name__}")
logger.info(object.description)
self.mig.create_new_exec_log(exec_id, self.current_binary_version)
object.execute()
logger.info(
f" --> Completed Migration {object.__class__.__name__}"
)
self.mig.set_execution_log_status(exec_id, "completed")
except Exception as e:
logger.error(
f" --> Error in Migration {object.__class__.__name__}"
)
logger.exception(e)
self.mig.set_execution_log_status(exec_id, "error")
self.mig.set_Execution_log_error_msg(exec_id, str(e))
@classmethod
def calculate_id(cls, migration_obj):
"""Extract the id from ojects derived from classes like SpecterMigration_0123"""
prefix, id = migration_obj.__class__.__name__.split("_")
assert prefix == "SpecterMigration"
return int(id)
@classmethod
def get_migration_classes(cls):
"""Returns all subclasses of class SpecterMigration"""
class_list = []
# The path where all the migrations are located:
package_dir = str(Path(Path(__file__).resolve().parent, "migrations").resolve())
for (_, module_name, _) in iter_modules(
[package_dir]
): # import the module and iterate through its attributes
module = import_module(
f"cryptoadvance.specter.util.migrations.{module_name}"
)
logger.info("Collecting possible migrations ...")
for attribute_name in dir(module):
attribute = getattr(module, attribute_name)
if isclass(attribute):
if (
issubclass(attribute, SpecterMigration)
and not attribute.__name__ == "SpecterMigration"
):
class_list.append(attribute)
return class_list
class MigDataManager(GenericDataManager):
"""A class handling the data for the SpecterMigrator in migration_data.json with some convenience-methods"""
@classmethod
def initial_data(cls):
return {
"events": [], # contains elements like {"timestamp": someTimestamp, "version": "v1.5.0"}
"migration_executions": [], # contains elements like {"timestamp": someTimestamp, "migration_id": 0}
}
name_of_json_file = "migration_data.json"
def __init__(self, data_folder):
# creating folders if they don't exist
# Usually Specter is doing that but in the case of tests, we're instantiated before Specter
if not os.path.isdir(data_folder):
os.makedirs(data_folder)
super().__init__(data_folder)
logger.info(f"Initiated {self}")
@property
def events(self):
"""A list of events where one event represents the initial use of a new specter-version which is a different
version than the one used before. Looks like: {"timestamp": someTimestamp, "version": "v1.5.0"}
"""
return self.data["events"]
@property
def latest_event(self):
if self.events:
return self.events[-1]
return {"timestamp": None, "version": None}
@property
def first_event(self):
"""Effectively the binary version the data has been created with FIRST"""
if self.events:
return self.events[0]
raise SpecterError("You should not check the first_event before you've set it!")
def create_new_event(self, version):
timestamp = datetime.now()
logger.debug(f"Creating new event with version {version} at {timestamp}")
self.events.append({"timestamp": str(timestamp), "version": version})
self._save()
def create_new_exec_log(self, migration_id, executing_version):
timestamp = datetime.now()
logger.debug(
f"Creating new Execution log with id {migration_id} at {timestamp}"
)
self.migration_executions.append(
{
"timestamp": str(timestamp),
"migration_id": migration_id,
"status": "started",
"executing_version": executing_version,
}
)
self._save()
def set_execution_log_status(self, id, status):
logger.debug(f"Setting execution log status of {id} to {status}")
self._find_exec_log(id)["status"] = status
self._save()
def set_Execution_log_error_msg(self, id, msg):
logger.debug(f"Setting execution log error_message of {id} to {msg}")
self._find_exec_log(id)["error_msg"] = msg
self._save()
def _find_exec_log(self, id):
for migration in self.migration_executions:
if migration["migration_id"] == id:
return migration
raise SpecterError(f"Can't find migration_execution with id {id}")
@property
def migration_executions(self):
return self.data["migration_executions"]
def has_migration_executed(self, migration_id):
executed_list = [
migration_execution["migration_id"]
for migration_execution in self.migration_executions
]
logger.debug(f"Executed migration_classes ids: {executed_list}")
return migration_id in executed_list
def __repr__(self):
return f"MigDataManager({self.data_file} events:{len(self.events)} execs:{len(self.migration_executions)} )"

55
src/cryptoadvance/specter/util/version.py
View file

@ -8,6 +8,8 @@ import os
import requests
import importlib_metadata
from cryptoadvance.specter.specter_error import SpecterError
logger = logging.getLogger(__name__)
@ -57,7 +59,8 @@ class VersionChecker:
# check if it's installed from master
if current == "vx.y.z-get-replaced-by-release-script":
current = "custom"
except:
except Exception as e:
logger.exception(e)
pass
return current
@ -151,3 +154,53 @@ class VersionChecker:
else:
self.stop()
return current, latest, False
def compare(version1: str, version2: str) -> int:
"""Compares two version strings like v1.5.1 and v1.6.0 and returns
* 1 : version2 is bigger that version1
* -1 : version1 is bigger than version2
* 0 : both are the same
This is not supporting semver and it doesn't take any postfix (-pre5)
into account and is therefore a naive implementation
"""
version1 = _parse_version(version1)
version2 = _parse_version(version2)
if version1["postfix"] != "" or version2["postfix"] != "":
raise SpecterError(
f"Cannot compare if either version has a postfix : {version1} and {version2}"
)
if version1["major"] > version2["major"]:
return -1
elif version1["major"] < version2["major"]:
return 1
if version1["minor"] > version2["minor"]:
return -1
elif version1["minor"] < version2["minor"]:
return 1
if version1["patch"] > version2["patch"]:
return -1
elif version1["patch"] < version2["patch"]:
return 1
return 0
def _parse_version(version: str) -> dict:
"""Parses version-strings like v1.5.6-pre5 and returns a dict"""
if version[0] != "v":
raise SpecterError(f"version {version} does not have a preceding 'v'")
version = version[1:]
version_ar = version.split(".")
if len(version_ar) != 3:
raise SpecterError(f"version {version} does not have 3 separated digits")
postfix = ""
if "-" in version_ar[2]:
postfix = version_ar[2].split("-")[1]
version_ar[2] = version_ar[2].split("-")[0]
return {
"major": int(version_ar[0]),
"minor": int(version_ar[1]),
"patch": int(version_ar[2]),
"postfix": postfix,
}

17
tests/conftest.py
View file

@ -26,6 +26,7 @@ from cryptoadvance.specter.specter_error import SpecterError
from cryptoadvance.specter.user import User
from cryptoadvance.specter.util.wallet_importer import WalletImporter
from cryptoadvance.specter.util.common import str2bool
from cryptoadvance.specter.util.shell import which
import code, traceback, signal
logger = logging.getLogger(__name__)
@ -159,6 +160,22 @@ def instantiate_elementsd_controller(request, rpcport=18643, extra_args=[]):
return elementsd_controller
# Below this point are fixtures. Fixtures have a scope. Check about scopes here:
# https://docs.pytest.org/en/6.2.x/fixture.html#scope-sharing-fixtures-across-classes-modules-packages-or-session
# possible values: function, class, module, package or session.
# The nodes are of scope session. All else is the default (function)
@pytest.fixture(scope="session")
def bitcoind_path():
if os.path.isfile("tests/bitcoin/src/bitcoind"):
return "tests/bitcoin/src/bitcoind"
elif os.path.isfile("tests/bitcoin/bin/bitcoind"):
return "tests/bitcoin/bin/bitcoind"
else:
return which("bitcoind")
@pytest.fixture(scope="session")
def bitcoin_regtest(docker, request):
bitcoind_regtest = instantiate_bitcoind_controller(docker, request, extra_args=None)

BIN
tests/helpers_testdata/bitcoin_minimum_mainnet_datadir.tgz Normal file
View file

Binary file not shown.

48
tests/test_managers_node.py
View file

@ -1,5 +1,8 @@
from enum import auto
import tempfile
import time
import tarfile
import os
import pytest
from cryptoadvance.specter.managers.node_manager import NodeManager
@ -46,3 +49,48 @@ def test_NodeManager(
assert nm.nodes_names == ["Bitcoin Core", "bitcoin_regtest", "elements_elreg"]
nm.switch_node("elements_elreg")
assert nm.active_node.get_rpc().getblockchaininfo()["chain"] == "elreg"
time.sleep(20)
""" For some reason this breaks other tests"""
@pytest.mark.skip()
def test_NodeManager_import(bitcoind_path):
with tempfile.TemporaryDirectory("_some_datafolder_tmp") as data_folder:
print(f"data_folder={data_folder}")
nm = NodeManager(data_folder=data_folder, bitcoind_path=bitcoind_path)
print(os.getcwd())
# This .bitcoin folder doesn't have a config-file
btc_tar = tarfile.open(
"./tests/helpers_testdata/bitcoin_minimum_mainnet_datadir.tgz", "r:gz"
)
btc_tar.extractall(os.path.join(data_folder, "somename", ".bitcoin-main"))
# # ... so let's create one
# with open(
# os.path.join(data_folder,"somename",".bitcoin-main","bitcoin.conf"),
# "w+",
# ) as file:
# file.write('''
# rpcauth=bitcoin:044931c1d498b7c27080d8b981331a65$6ee7929513401c39ca1f7e376e55553c52dcb36a14e8410ac5f514fbf18bedbb
# server=1
# listen=1
# proxy=127.0.0.1:9050
# bind=127.0.0.1
# torcontrol=127.0.0.1:9051
# torpassword=gVzREfuHso6U2OfRRvqT3w
# fallbackfee=0.0002
# prune=1000
# '''
# )
node = nm.add_internal_node("somename", port=8339)
try:
node.start()
time.sleep(5)
# assert node.get_rpc().password == None
nm.switch_node("somename")
time.sleep(5)
assert nm.active_node.get_rpc().getblockchaininfo()["chain"] == "main"
finally:
node.stop()

144
tests/test_specter_migrator.py Normal file
View file

@ -0,0 +1,144 @@
import json
import logging
import os
import tarfile
import time
import pytest
from cryptoadvance.specter.util.specter_migrator import (
MigDataManager,
SpecterMigration,
SpecterMigrator,
)
from mock import Mock, patch
logger = logging.getLogger(__name__)
def test_SpecterMigration():
# If i started first with 1.6.1 and the migration has been implemented in 1.7.0, please do it
assert SpecterMigration.should_execute_cls("v1.6.1", "v1.7.0")
# Don't do it
assert not SpecterMigration.should_execute_cls("v1.7.0", "v1.7.0")
assert not SpecterMigration.should_execute_cls("v1.5.0", "v1.5.0")
assert not SpecterMigration.should_execute_cls("v1.6.1", "v1.5.0")
def test_SpecterMigrator_classnaming():
for clazz in SpecterMigrator.get_migration_classes():
# instantiation of a migration-class should not have any side-effects:
mig_obj: SpecterMigration = clazz(Mock())
# will also tests the prefix implicitely
assert (
SpecterMigrator.calculate_id(mig_obj) >= 0
), "a migration-class needs an id (int)"
# If implemented right, this should not throw an Exception:
assert type(mig_obj.description) == str
def test_SpecterMigrator_versioning(empty_data_folder, caplog):
caplog.set_level(logging.DEBUG)
with patch(
"cryptoadvance.specter.util.specter_migrator.VersionChecker"
) as mock_version_checker_class:
# Specifiy a mock which will represent the VersionChecker Object
the_obj = Mock()
# Specify that the mock should return a funny version
the_obj.get_current_version.return_value = "v1.5.0"
# Specify that the constructor of the mock_version_checker_class should return the_obj
mock_version_checker_class.side_effect = [the_obj]
mm = SpecterMigrator(empty_data_folder)
mm.plan_migration()
assert mm.current_binary_version == "v1.5.0"
assert "Skipping class SpecterMigration_0000" in caplog.text
assert (
"Skipping class SpecterMigration_0001" not in caplog.text
) # this one overwrites should_execute
def test_SpecterMigrator_versioning2(empty_data_folder, caplog):
caplog.set_level(logging.DEBUG)
with patch(
"cryptoadvance.specter.util.specter_migrator.VersionChecker"
) as mock_version_checker_class:
# Specifiy a mock which will represent the VersionChecker Object
the_obj = Mock()
# Specify that the mock should return a funny version
the_obj.get_current_version.return_value = "v1.7.1"
# Specify that the constructor of the mock_version_checker_class should return the_obj
mock_version_checker_class.side_effect = [the_obj]
mm = SpecterMigrator(empty_data_folder)
assert mm.current_binary_version == "v1.7.1"
assert "Skipping class SpecterMigration_0000" not in caplog.text
assert "Skipping class SpecterMigration_0001" not in caplog.text
def test_SpecterMigrator(empty_data_folder, caplog):
caplog.set_level(logging.DEBUG)
assert MigDataManager.initial_data()["events"] == []
assert MigDataManager.initial_data()["migration_executions"] == []
assert len(os.listdir(empty_data_folder)) == 0
# For migration1
btc_tar = tarfile.open(
"./tests/helpers_testdata/bitcoin_minimum_mainnet_datadir.tgz", "r:gz"
)
btc_tar.extractall(os.path.join(empty_data_folder, ".bitcoin"))
# Fake the existence of bitcoin-binaries
os.makedirs(os.path.join(empty_data_folder, "bitcoin-binaries"))
assert len(os.listdir(empty_data_folder)) == 2
# Patch the Class where it's used, not where it's defined
with patch(
"cryptoadvance.specter.util.specter_migrator.VersionChecker"
) as mock_version_checker_class:
# Specifiy a mock which will represent the VersionChecker Object
the_obj = Mock()
# Specify that the mock should return a funny version
the_obj.get_current_version.return_value = "v1.6.1"
# Specify that the constructor of the mock_version_checker_class should return the_obj
mock_version_checker_class.side_effect = [the_obj]
mm = SpecterMigrator(empty_data_folder)
assert mm.current_binary_version == "v1.6.1"
# not executed yet
assert "Setting execution log status of 1 to completed" not in caplog.text
# initally, zero executed migrations
assert len(mm.mig.migration_executions) == 0
# With instantiation, the event get stored just right away
assert mm.mig.latest_event["version"] == "v1.6.1"
mylist = mm.plan_migration()
# This assertion will break every time you create a new migration-script
assert len(mylist) == 1
mm.execute_migrations(mylist)
assert len(mm.mig.migration_executions) == 1
assert "Setting execution log status of 1 to completed" in caplog.text
assert mm.mig.migration_executions[0]["migration_id"] == 1
assert os.path.isdir(
os.path.join(
empty_data_folder,
"nodes",
"specter_bitcoin",
".bitcoin-main",
"chainstate",
)
)
specter_bitcoin_json = os.path.join(
empty_data_folder, "nodes", "specter_bitcoin.json"
)
assert os.path.isfile(specter_bitcoin_json)
with open(specter_bitcoin_json) as jsonfile:
config = json.loads(jsonfile.read())
assert config["name"] == "specter_bitcoin"
assert config["alias"] == "specter_bitcoin"
assert config["autodetect"] == False
assert config["datadir"].endswith("nodes/specter_bitcoin/.bitcoin-main")
assert config["user"] == "bitcoin"
assert config["password"]
assert config["port"] == 8332
assert config["host"] == "localhost"
assert config["external_node"] == False
# yeah, some more but should be ok

38
tests/test_util_version.py Normal file
View file

@ -0,0 +1,38 @@
import pytest
from cryptoadvance.specter.specter_error import SpecterError
from cryptoadvance.specter.util.version import _parse_version, compare
def test_parse_version():
assert _parse_version("v1.5.9") == {
"major": 1,
"minor": 5,
"patch": 9,
"postfix": "",
}
assert _parse_version("v3.4.5-pre12") == {
"major": 3,
"minor": 4,
"patch": 5,
"postfix": "pre12",
}
with pytest.raises(SpecterError):
_parse_version("3.4.5-pre12")
with pytest.raises(SpecterError):
_parse_version("3.4.5.6-pre12")
def test_compare():
assert compare("v1.2.3", "v2.2.3") == 1
assert compare("v2.2.3", "v1.2.3") == -1
assert compare("v2.1.3", "v2.2.3") == 1
assert compare("v2.2.3", "v1.1.3") == -1
assert compare("v2.2.3", "v2.2.5") == 1
assert compare("v2.2.5", "v2.2.2") == -1
assert compare("v2.2.5", "v2.2.5") == 0
with pytest.raises(SpecterError):
assert compare("v2.2.3-pre2", "v1.2.3") == -1