diff --git a/src/cryptoadvance/specter/__main__.py b/src/cryptoadvance/specter/__main__.py index 9cc4ff961..8d69c6e48 100644 --- a/src/cryptoadvance/specter/__main__.py +++ b/src/cryptoadvance/specter/__main__.py @@ -1,5 +1,5 @@ from logging.config import dictConfig -from .cli import cli +from .cli import entry_point import logging if __name__ == "__main__": @@ -15,4 +15,5 @@ if __name__ == "__main__": logging.getLogger().addHandler(ch) # However initially, we'll set the root-logger to INFO: logging.getLogger("cryptoadvance").setLevel(logging.INFO) - cli() + logging.getLogger("cryptoadvance.specter.util.checker").setLevel(logging.DEBUG) + entry_point() diff --git a/src/cryptoadvance/specter/bitcoind.py b/src/cryptoadvance/specter/bitcoind.py index 5adb10fe8..759c6572a 100644 --- a/src/cryptoadvance/specter/bitcoind.py +++ b/src/cryptoadvance/specter/bitcoind.py @@ -8,11 +8,11 @@ import shutil import subprocess import tempfile import time +import json import docker from .util.shell import which -from .server import DATA_FOLDER from .rpc import RpcError from .rpc import BitcoinRPC from .helpers import load_jsons @@ -56,6 +56,21 @@ class Btcd_conn: self.rpcuser, self.rpcpassword, self.ipaddress, self.rpcport ) + def as_data(self): + """ returns a data-representation of this connection """ + me = { + "user": self.rpcuser, + "password": self.rpcpassword, + "host": self.ipaddress, + "port": self.rpcport, + "url": self.render_url(), + } + return me + + def render_json(self): + """ returns a json-representation of this connection """ + return json.dumps(self.as_data()) + def __repr__(self): return "".format(self.render_url()) @@ -66,7 +81,7 @@ class BitcoindController: def __init__(self, rpcport=18443): self.rpcconn = Btcd_conn(rpcport=rpcport) - def start_bitcoind(self, cleanup_at_exit=False, datadir=None): + def start_bitcoind(self, cleanup_at_exit=False, cleanup_hard=False, datadir=None): """starts bitcoind with a specific rpcport=18543 by default. That's not the standard in order to make pytest running while developing locally against a different regtest-instance @@ -76,7 +91,9 @@ class BitcoindController: return self.check_existing() logger.debug("Starting bitcoind") - self._start_bitcoind(cleanup_at_exit, datadir=datadir) + self._start_bitcoind( + cleanup_at_exit, cleanup_hard=cleanup_hard, datadir=datadir + ) self.wait_for_bitcoind(self.rpcconn) self.mine(block_count=100) @@ -92,7 +109,7 @@ class BitcoindController: """ wrapper for convenience """ return self.rpcconn.get_rpc() - def _start_bitcoind(self, cleanup_at_exit): + def _start_bitcoind(self, cleanup_at_exit, cleanup_hard=False): raise Exception("This should not be used in the baseclass!") def check_existing(self): @@ -196,7 +213,7 @@ class BitcoindPlainController(BitcoindController): self.bitcoind_path = bitcoind_path self.rpcconn.ipaddress = "localhost" - def _start_bitcoind(self, cleanup_at_exit=True, datadir=None): + def _start_bitcoind(self, cleanup_at_exit=True, cleanup_hard=False, datadir=None): if datadir == None: datadir = tempfile.mkdtemp(prefix="bitcoind_plain_datadir_") bitcoind_cmd = self.construct_bitcoind_cmd( @@ -213,10 +230,15 @@ class BitcoindPlainController(BitcoindController): ) def cleanup_bitcoind(): - self.bitcoind_proc.terminate() # might take a bit longer than kill but it'll preserve block-height - logger.info( - "Killed bitcoind-process with pid {}".format(self.bitcoind_proc.pid) - ) + if cleanup_hard: + self.bitcoind_proc.kill() # might be usefull for e.g. testing. We can't wait for so long + logger.info("Killed bitcoind with pid {self.bitcoind_proc.pid}") + else: + self.bitcoind_proc.terminate() # might take a bit longer than kill but it'll preserve block-height + logger.info( + f"Terminated bitcoind with pid {self.bitcoind_proc.pid}, waiting for termination ..." + ) + self.bitcoind_proc.wait() if cleanup_at_exit: logger.debug("REGISTERING EXIT FUNCTIONS") @@ -251,7 +273,7 @@ class BitcoindDockerController(BitcoindController): rpcconn, self.btcd_container = self.detect_bitcoind_container(rpcport) self.rpcconn = rpcconn - def _start_bitcoind(self, cleanup_at_exit, datadir=None): + def _start_bitcoind(self, cleanup_at_exit, cleanup_hard=False, datadir=None): if datadir != None: # ignored pass @@ -269,9 +291,7 @@ class BitcoindDockerController(BitcoindController): ) ) self.btcd_container = dclient.containers.run( - "registry.gitlab.com/cryptoadvance/specter-desktop/python-bitcoind:{}".format( - self.docker_tag - ), + image, bitcoind_path, ports=ports, detach=True, @@ -410,12 +430,10 @@ class BitcoindDockerController(BitcoindController): raise Exception("Timeout while starting bitcoind-docker-container!") -def fetch_wallet_addresses_for_mining(data_folder=None): +def fetch_wallet_addresses_for_mining(data_folder): """parses all the wallet-jsons in the folder (default ~/.specter/wallets/regtest) and returns an array with the addresses """ - if data_folder == None: - data_folder = os.path.expanduser(DATA_FOLDER) wallets = load_jsons(data_folder + "/wallets/regtest") address_array = [value["address"] for key, value in wallets.items()] # remove duplicates diff --git a/src/cryptoadvance/specter/cli.py b/src/cryptoadvance/specter/cli.py deleted file mode 100644 index f8500d9e5..000000000 --- a/src/cryptoadvance/specter/cli.py +++ /dev/null @@ -1,352 +0,0 @@ -import logging -from logging.config import dictConfig -import os -from os import path -import sys -import psutil -from pathlib import Path -import shutil -import time -from stem.control import Controller -from .util.tor import stop_hidden_services, start_hidden_service -import click - -from .server import create_app, init_app -from .helpers import set_loglevel - -import signal - -logger = logging.getLogger(__name__) - - -@click.group() -def cli(): - pass - - -@cli.command() -@click.option("--daemon", is_flag=True) -@click.option("--stop", is_flag=True) -@click.option("--restart", is_flag=True) -@click.option("--force", is_flag=True) -# options below can help to run it on a remote server, -# but better use nginx -@click.option("--port") # default - 25441 set to 80 for http, 443 for https -# set to 0.0.0.0 to make it available outside -@click.option("--host", default="127.0.0.1") -# for https: -@click.option("--cert") -@click.option("--key") -@click.option("--debug/--no-debug", default=None) -@click.option("--tor", is_flag=True) -@click.option("--hwibridge", is_flag=True) -def server(daemon, stop, restart, force, port, host, cert, key, debug, tor, hwibridge): - # create an app to get Specter instance - # and it's data folder - logger.info("Logging is hopefully configured") - if debug: - ca_logger = logging.getLogger("cryptoadvance") - ca_logger.setLevel(logging.DEBUG) - logger.debug("We're now on level DEBUG on logger cryptoadvance") - app = create_app() - app.app_context().push() - init_app(app, hwibridge=hwibridge) - - # we will store our daemon PID here - pid_file = path.join(app.specter.data_folder, "daemon.pid") - toraddr_file = path.join(app.specter.data_folder, "onion.txt") - # check if pid file exists - if path.isfile(pid_file): - # if we need to stop daemon - if stop or restart: - print("Stopping the Specter server...") - with open(pid_file) as f: - pid = int(f.read()) - os.kill(pid, signal.SIGTERM) - time.sleep(0.3) - try: - os.remove(pid_file) - except Exception: - pass - elif daemon: - if not force: - print( - f'PID file "{pid_file}" already exists. \ - Use --force to overwrite' - ) - return - else: - os.remove(pid_file) - if stop: - return - else: - if stop or restart: - print(f'Can\'t find PID file "{pid_file}"') - if stop: - return - - # watch templates folder to reload when something changes - extra_dirs = ["templates"] - extra_files = extra_dirs[:] - for extra_dir in extra_dirs: - for dirname, dirs, files in os.walk(extra_dir): - for filename in files: - filename = os.path.join(dirname, filename) - if os.path.isfile(filename): - extra_files.append(filename) - - # if port is not defined - get it from environment - if port is None: - port = int(os.getenv("PORT", 25441)) - else: - port = int(port) - - # certificates - if cert is None: - cert = os.getenv("CERT", None) - if key is None: - key = os.getenv("KEY", None) - - protocol = "http" - kwargs = {"host": host, "port": port, "extra_files": extra_files} - if cert is not None and key is not None: - cert = os.path.abspath(cert) - key = os.path.abspath(key) - kwargs["ssl_context"] = (cert, key) - protocol = "https" - - if hwibridge: - print( - " * Running HWI Bridge mode.\n" - " * You can configure access to the API " - "at: %s://%s:%d/hwi/settings" % (protocol, host, port) - ) - - # debug is false by default - def run(debug=debug): - try: - app.controller = Controller.from_port() - except Exception: - app.controller = None - try: - port = 5000 # default flask port - if "port" in kwargs: - port = kwargs["port"] - else: - kwargs["port"] = port - # if we have certificates - if "ssl_context" in kwargs: - tor_port = 443 - else: - tor_port = 80 - app.port = port - app.tor_port = tor_port - app.save_tor_address_to = toraddr_file - if debug and (tor or os.getenv("CONNECT_TOR") == "True"): - print( - " * Warning: Cannot use Tor in debug mode. \ - Starting in production mode instead." - ) - debug = False - if tor or os.getenv("CONNECT_TOR") == "True": - try: - app.tor_enabled = True - start_hidden_service(app) - except Exception as e: - print(f" * Failed to start Tor hidden service: {e}") - print(" * Continuing process with Tor disabled") - app.tor_service_id = None - app.tor_enabled = False - else: - app.tor_service_id = None - app.tor_enabled = False - app.run(debug=debug, **kwargs) - stop_hidden_services(app) - finally: - if app.controller is not None: - app.controller.close() - - # check if we should run a daemon or not - if daemon or restart: - print("Starting server in background...") - print(" * Hopefully running on %s://%s:%d/" % (protocol, host, port)) - # macOS + python3.7 is buggy - if sys.platform == "darwin" and ( - sys.version_info.major == 3 and sys.version_info.minor < 8 - ): - print( - " * WARNING: --daemon mode might not \ - work properly in python 3.7 and lower \ - on MacOS. Upgrade to python 3.8+" - ) - from daemonize import Daemonize - - d = Daemonize(app="specter", pid=pid_file, action=run) - d.start() - else: - # if not a daemon we can use DEBUG - if debug is None: - debug = app.config["DEBUG"] - run(debug=debug) - - -class Echo: - def __init__(self, quiet): - self.quiet = quiet - - def echo(self, mystring, prefix=True, **kwargs): - if self.quiet: - pass - else: - if prefix: - click.echo(f" --> ", nl=False) - click.echo(f"{mystring}", **kwargs) - - -@cli.command() -@click.option("--debug/--no-debug", default=False, help="Turns on debug-logging") -@click.option("--quiet/--no-quiet", default=False, help="as less output as possible") -@click.option( - "--nodocker", default=False, is_flag=True, help="use without docker (non-default)" -) -@click.option( - "--docker-tag", "docker_tag", default="latest", help="Use a specific docker-tag" -) -@click.option( - "--data-dir", - default="/tmp/bitcoind_plain_datadir", - help="specify a (maybe not yet existing) datadir. Works only in --nodocker (Default:/tmp/bitcoind_plain_datadir) ", -) -@click.option("--mining/--no-mining", default=True, help="Turns on mining (default)") -@click.option( - "--mining-period", - default="15", - help="Every mining-period (in seconds), a block gets mined (default 15sec)", -) -@click.option( - "--reset", - is_flag=True, - default=False, - help="Will kill the bitcoind. Datadir will get lost.", -) -def bitcoind( - debug, quiet, nodocker, docker_tag, data_dir, mining, mining_period, reset -): - """This will start a bitcoind regtest and mines a block every mining-period. - If a bitcoind is already running on port 18443, it won't start another one. If you CTRL-C this, the bitcoind will - still continue to run. You have to shut it down. - """ - # In order to avoid these dependencies for production use, we're importing them here: - import docker - from .bitcoind import ( - BitcoindDockerController, - BitcoindPlainController, - fetch_wallet_addresses_for_mining, - ) - - if debug: - logging.getLogger("cryptoadvance").setLevel(logging.DEBUG) - logger.debug("Now on debug-logging") - echo = Echo(quiet).echo - - if reset: - if not nodocker: - echo("ERROR: --reset only works in conjunction with --nodocker currently") - return - did_something = False - - for proc in psutil.process_iter(): - try: - # Get process name & pid from process object. - processName = proc.name() - pid = proc.pid - if processName.startswith("bitcoind"): - echo(f"Killing bitcoind-process with id {pid} ...") - did_something = True - os.kill(pid, signal.SIGTERM) - except (psutil.NoSuchProcess, psutil.AccessDenied, psutil.ZombieProcess): - echo(f"Pid {pid} not owned by us. Might be a docker-process? {line}") - if Path(data_dir).exists(): - echo(f"Purging Datadirectory {data_dir} ...") - did_something = True - shutil.rmtree(data_dir) - if not did_something: - echo("Nothing to do!") - return - logging.getLogger().setLevel(logging.INFO) - mining_every_x_seconds = float(mining_period) - if nodocker: - echo("starting plain bitcoind") - my_bitcoind = BitcoindPlainController() - # Make sure datadir does exist if specified: - Path(data_dir).mkdir(parents=True, exist_ok=True) - else: - echo("starting or detecting container") - my_bitcoind = BitcoindDockerController(docker_tag=docker_tag) - try: - my_bitcoind.start_bitcoind(cleanup_at_exit=True, datadir=data_dir) - except docker.errors.ImageNotFound: - echo(f"Image with tag {docker_tag} does not exist!") - echo( - f"Try to download first with docker pull \ - registry.gitlab.com/cryptoadvance/specter-desktop\ - /python-bitcoind:{docker_tag}" - ) - sys.exit(1) - if not nodocker: - tags_of_image = [ - image.split(":")[-1] for image in my_bitcoind.btcd_container.image.tags - ] - if docker_tag not in tags_of_image: - echo( - "The running docker container is not \ - the tag you requested!" - ) - echo( - "please stop first with docker stop {}".format( - my_bitcoind.btcd_container.id - ) - ) - sys.exit(1) - echo("containerImage: %s" % my_bitcoind.btcd_container.image.tags) - echo(" url: %s" % my_bitcoind.rpcconn.render_url()) - echo("user, password: bitcoin, secret") - echo(" host, port: localhost, 18443") - echo( - " bitcoin-cli: bitcoin-cli -regtest -rpcuser=bitcoin -rpcpassword=secret getblockchaininfo " - ) - if mining: - echo( - "Now, mining a block every %f seconds, avoid it via --no-mining" - % mining_every_x_seconds - ) - # Get each address some coins - try: - for address in fetch_wallet_addresses_for_mining(): - my_bitcoind.mine(address=address) - except FileNotFoundError: - # might happen if there no ~/.specter folder yet - pass - - # make them spendable - my_bitcoind.mine(block_count=100) - echo( - f"height: {my_bitcoind.rpcconn.get_rpc().getblockchaininfo()['blocks']} | ", - nl=False, - ) - i, j = 0, 0 - while True: - my_bitcoind.mine() - echo("%i" % (i % 10), prefix=False, nl=False) - if i % 10 == 9: - echo(" ", prefix=False, nl=False) - i += 1 - if i >= 50: - j = i - i = 0 - echo("", prefix=False) - echo( - f"height: {my_bitcoind.rpcconn.get_rpc().getblockchaininfo()['blocks']} | ", - nl=False, - ) - time.sleep(mining_every_x_seconds) diff --git a/src/cryptoadvance/specter/cli/__init__.py b/src/cryptoadvance/specter/cli/__init__.py new file mode 100644 index 000000000..fc82dcd1e --- /dev/null +++ b/src/cryptoadvance/specter/cli/__init__.py @@ -0,0 +1,21 @@ +import logging +import click +from .cli_server import server +from .cli_bitcoind import bitcoind + +logger = logging.getLogger(__name__) + + +@click.group() +@click.option("--debug", is_flag=True, help="Show debug information on errors.") +@click.pass_context +def entry_point(config_home, debug=False): + # ctx.obj = Repo(config_home, debug) + if debug: + logging.basicConfig(level=logging.INFO) + logging.getLogger("cryptoadvance").setLevel(logging.DEBUG) + logger + + +entry_point.add_command(server) +entry_point.add_command(bitcoind) diff --git a/src/cryptoadvance/specter/cli/cli_bitcoind.py b/src/cryptoadvance/specter/cli/cli_bitcoind.py new file mode 100644 index 000000000..7031ed0da --- /dev/null +++ b/src/cryptoadvance/specter/cli/cli_bitcoind.py @@ -0,0 +1,256 @@ +import logging +import os +import shutil +import signal +import sys +import atexit +import time +import json +from pathlib import Path + +import click +import psutil +from flask import Config + +from ..config import DEFAULT_CONFIG + +logger = logging.getLogger(__name__) + + +class Echo: + def __init__(self, quiet): + self.quiet = quiet + + def echo(self, mystring, prefix=True, **kwargs): + if self.quiet: + pass + else: + if prefix: + click.echo(f" --> ", nl=False) + click.echo(f"{mystring}", **kwargs) + + +@click.command() +@click.option("--debug/--no-debug", default=False, help="Turns on debug-logging") +@click.option("--quiet/--no-quiet", default=False, help="as less output as possible") +@click.option( + "--nodocker", default=False, is_flag=True, help="use without docker (non-default)" +) +@click.option( + "--docker-tag", "docker_tag", default="latest", help="Use a specific docker-tag" +) +@click.option( + "--data-dir", + default="/tmp/specter_btcd_regtest_plain_datadir", + help="specify a (maybe not yet existing) datadir. Works only in --nodocker (Default:/tmp/bitcoind_plain_datadir) ", +) +@click.option( + "--mining/--no-mining", + default=True, + help="Turns on mining (default). In tests it's useful to turn it off.", +) +@click.option( + "--mining-period", + default="15", + help="Every mining-period (in seconds), a block gets mined (default 15sec)", +) +@click.option( + "--reset", + is_flag=True, + default=False, + help="Will kill the bitcoind. Datadir will get lost.", +) +@click.option( + "--create-conn-json", + is_flag=True, + default=False, + help="Will create a small json-file btcd-conn.json with connection details.", +) +@click.option( + "--cleanuphard/--no-cleanuphard", + default=False, + help="Will send a SIGKILL instead of SIGTERM (default) when CTRL-C. Mostly to speedup tests.", +) +@click.option( + "--config", + default=None, + help="A class from the config.py which sets reasonable Defaults", +) +def bitcoind( + debug, + quiet, + nodocker, + docker_tag, + data_dir, + mining, + mining_period, + reset, + create_conn_json, + cleanuphard, + config, +): + """This will start a bitcoind regtest and mines a block every mining-period. + If a bitcoind is already running on port 18443, it won't start another one. If you CTRL-C this, the bitcoind will + still continue to run. You have to shut it down. + """ + # In order to avoid these dependencies for production use, we're importing them here: + import docker + + from ..bitcoind import BitcoindDockerController, BitcoindPlainController + + if config is None: + config = DEFAULT_CONFIG + else: + if not "." in config: + config = "cryptoadvance.specter.config." + config + config_obj = Config(".") + config_obj.from_object(config) + + echo = Echo(quiet).echo + + if debug: + echo( + "Sorry, --debug used this way is deprecated. This feature will get removed. Please do it like this:" + ) + echo("$ python3 -m cryptoadvance-specter --debug bitcoind") + exit(1) + + if reset: + if not nodocker: + echo("ERROR: --reset only works in conjunction with --nodocker currently") + return + did_something = False + + for proc in psutil.process_iter(): + try: + # Get process name & pid from process object. + processName = proc.name() + pid = proc.pid + if processName.startswith("bitcoind"): + echo(f"Killing bitcoind-process with id {pid} ...") + did_something = True + os.kill(pid, signal.SIGTERM) + except (psutil.NoSuchProcess, psutil.AccessDenied, psutil.ZombieProcess): + echo(f"Pid {pid} not owned by us. Might be a docker-process? {proc}") + if Path(data_dir).exists(): + echo(f"Purging Datadirectory {data_dir} ...") + did_something = True + shutil.rmtree(data_dir) + if not did_something: + echo("Nothing to do!") + return + mining_every_x_seconds = float(mining_period) + if nodocker: + echo("starting plain bitcoind") + my_bitcoind = BitcoindPlainController() + # Make sure datadir does exist if specified: + Path(data_dir).mkdir(parents=True, exist_ok=True) + else: + echo("starting or detecting container") + my_bitcoind = BitcoindDockerController(docker_tag=docker_tag) + try: + my_bitcoind.start_bitcoind( + cleanup_at_exit=True, cleanup_hard=cleanuphard, datadir=data_dir + ) + except docker.errors.ImageNotFound: + echo(f"Image with tag {docker_tag} does not exist!") + echo( + f"Try to download first with docker pull \ + registry.gitlab.com/cryptoadvance/specter-desktop\ + /python-bitcoind:{docker_tag}" + ) + sys.exit(1) + if not nodocker: + tags_of_image = [ + image.split(":")[-1] for image in my_bitcoind.btcd_container.image.tags + ] + if docker_tag not in tags_of_image: + echo( + "The running docker container is not \ + the tag you requested!" + ) + echo( + "please stop first with docker stop {}".format( + my_bitcoind.btcd_container.id + ) + ) + sys.exit(1) + echo("containerImage: %s" % my_bitcoind.btcd_container.image.tags) + echo(" url: %s" % my_bitcoind.rpcconn.render_url()) + echo("user, password: bitcoin, secret") + echo(" host, port: localhost, 18443") + echo( + " bitcoin-cli: bitcoin-cli -regtest -rpcuser=bitcoin -rpcpassword=secret getblockchaininfo " + ) + + if create_conn_json: + conn = my_bitcoind.rpcconn.as_data() + conn["pid"] = os.getpid() # usefull to sen signals + with open("btcd-conn.json", "w") as file: + file.write(json.dumps(conn)) + + def cleanup(): + os.remove("btcd-conn.json") + + atexit.register(cleanup) + + signal.signal( + signal.SIGUSR1, + lambda x, y: mine_2_specter_wallets( + my_bitcoind, config_obj["SPECTER_DATA_FOLDER"], echo + ), + ) + + if mining: + miner_loop( + my_bitcoind, config_obj["SPECTER_DATA_FOLDER"], mining_every_x_seconds, echo + ) + + +def miner_loop(my_bitcoind, data_folder, mining_every_x_seconds, echo): + " An endless loop mining bitcoin " + + echo( + "Now, mining a block every %f seconds, avoid it via --no-mining" + % mining_every_x_seconds + ) + mine_2_specter_wallets(my_bitcoind, data_folder, echo) + + # make them spendable + my_bitcoind.mine(block_count=100) + echo( + f"height: {my_bitcoind.rpcconn.get_rpc().getblockchaininfo()['blocks']} | ", + nl=False, + ) + i = 0 + while True: + my_bitcoind.mine() + echo("%i" % (i % 10), prefix=False, nl=False) + if i % 10 == 9: + echo(" ", prefix=False, nl=False) + i += 1 + if i >= 50: + i = 0 + echo("", prefix=False) + echo( + f"height: {my_bitcoind.rpcconn.get_rpc().getblockchaininfo()['blocks']} | ", + nl=False, + ) + time.sleep(mining_every_x_seconds) + + +def mine_2_specter_wallets(my_bitcoind, data_folder, echo): + """Get each specter-wallet some coins""" + + from ..bitcoind import fetch_wallet_addresses_for_mining + + try: + + for address in fetch_wallet_addresses_for_mining(data_folder): + echo("") + echo(f"Mining to address {address}") + my_bitcoind.mine(address=address) + my_bitcoind.mine(block_count=100) + except FileNotFoundError: + # might happen if there no ~/.specter folder yet + pass diff --git a/src/cryptoadvance/specter/cli/cli_server.py b/src/cryptoadvance/specter/cli/cli_server.py new file mode 100644 index 000000000..9a3de4158 --- /dev/null +++ b/src/cryptoadvance/specter/cli/cli_server.py @@ -0,0 +1,242 @@ +import logging +import os +import signal +import sys +import time +from os import path + +import click +from stem.control import Controller + +from ..server import create_app, init_app +from ..util.tor import start_hidden_service, stop_hidden_services + +logger = logging.getLogger(__name__) + + +@click.group() +def cli(): + pass + + +@cli.command() +@click.option( + "--daemon", + is_flag=True, + help="Deprecated, don't use that and prepare to see it removed", +) +@click.option( + "--stop", + is_flag=True, + help="Deprecated, don't use that and prepare to see it removed", +) +@click.option( + "--restart", + is_flag=True, + help="Deprecated, don't use that and prepare to see it removed", +) +@click.option( + "--force", + is_flag=True, + help="Deprecated, don't use that and prepare to see it removed", +) +# options below can help to run it on a remote server, +# but better use nginx +@click.option( + "--port", help="The TCP-Port to bin specter to" +) # default - 25441 set to 80 for http, 443 for https +# set to 0.0.0.0 to make it available outside +@click.option( + "--host", + default="127.0.0.1", + help="if you do --host 0.0.0.0 then specter will be available in your local lan", +) +# for https: +@click.option( + "--cert", help="--cert and --key are for using a self-signed-cert/ssl-encryption" +) +@click.option("--key") +@click.option("--debug/--no-debug", default=None) +@click.option("--tor", is_flag=True) +@click.option( + "--hwibridge", + is_flag=True, + help="Start the hwi-bridge to use your HWWs with a remote specter", +) +@click.option( + "--specter-data-folder", + default=None, + help="Enables overriding the specter-data-folder. This is usually ~/.specter", +) +@click.option( + "--config", + default=None, + help="A class from the config.py which sets reasonable Defaults", +) +def server( + daemon, + stop, + restart, + force, + port, + host, + cert, + key, + debug, + tor, + hwibridge, + specter_data_folder, + config, +): + # create an app to get Specter instance + # and it's data folder + logger.info("Logging is hopefully configured") + if debug: + ca_logger = logging.getLogger("cryptoadvance") + ca_logger.setLevel(logging.DEBUG) + logger.debug("We're now on level DEBUG on logger cryptoadvance") + if config is None: + app = create_app() + else: + if "." in config: + app = create_app(config=config) + else: + app = create_app(config="cryptoadvance.specter.config." + config) + + if specter_data_folder: + app.config["SPECTER_DATA_FOLDER"] = specter_data_folder + + if port: + app.config["PORT"] = int(port) + + # certificates + if cert: + logger.info("CERT:" + str(cert)) + app.config["CERT"] = cert + if key: + key = app.config["KEY"] = key + + app.app_context().push() + init_app(app, hwibridge=hwibridge) + + # This stuff here is deprecated + # When we remove it, we should imho keep the pid_file thing which can be very useful! + # we will store our daemon PID here + pid_file = path.join(app.specter.data_folder, "daemon.pid") + toraddr_file = path.join(app.specter.data_folder, "onion.txt") + # check if pid file exists + if path.isfile(pid_file): + # if we need to stop daemon + if stop or restart: + print("Stopping the Specter server...") + with open(pid_file) as f: + pid = int(f.read()) + os.kill(pid, signal.SIGTERM) + time.sleep(0.3) + try: + os.remove(pid_file) + except OSError: + pass + elif daemon: + if not force: + print( + f'PID file "{pid_file}" already exists. \ + Use --force to overwrite' + ) + return + else: + os.remove(pid_file) + if stop: + return + else: + if stop or restart: + print(f'Can\'t find PID file "{pid_file}"') + if stop: + return + + # watch templates folder to reload when something changes + extra_dirs = ["templates"] + extra_files = extra_dirs[:] + for extra_dir in extra_dirs: + for dirname, dirs, files in os.walk(extra_dir): + for filename in files: + filename = os.path.join(dirname, filename) + if os.path.isfile(filename): + extra_files.append(filename) + + protocol = "http" + kwargs = {"host": host, "port": app.config["PORT"], "extra_files": extra_files} + if cert is not None and key is not None: + cert = os.path.abspath(app.config["CERT"]) + key = os.path.abspath(app.config["KEY"]) + kwargs["ssl_context"] = (cert, key) + protocol = "https" + + if hwibridge: + print( + " * Running HWI Bridge mode.\n" + " * You can configure access to the API " + "at: %s://%s:%d/hwi/settings" % (protocol, host, app.config["PORT"]) + ) + + # debug is false by default + def run(debug=debug): + try: + app.controller = Controller.from_port() + except Exception: + app.controller = None + try: + # if we have certificates + if "ssl_context" in kwargs: + tor_port = 443 + else: + tor_port = 80 + app.port = kwargs["port"] + app.tor_port = tor_port + app.save_tor_address_to = toraddr_file + if debug and (tor or os.getenv("CONNECT_TOR") == "True"): + print( + " * Warning: Cannot use Tor in debug mode. \ + Starting in production mode instead." + ) + debug = False + if tor or os.getenv("CONNECT_TOR") == "True": + try: + app.tor_enabled = True + start_hidden_service(app) + except Exception as e: + print(f" * Failed to start Tor hidden service: {e}") + print(" * Continuing process with Tor disabled") + app.tor_service_id = None + app.tor_enabled = False + else: + app.tor_service_id = None + app.tor_enabled = False + app.run(debug=debug, **kwargs) + stop_hidden_services(app) + finally: + if app.controller is not None: + app.controller.close() + + # check if we should run a daemon or not + if daemon or restart: + print("Starting server in background...") + print(" * Hopefully running on %s://%s:%d/" % (protocol, host, port)) + # macOS + python3.7 is buggy + if sys.platform == "darwin" and ( + sys.version_info.major == 3 and sys.version_info.minor < 8 + ): + print( + " * WARNING: --daemon mode might not \ + work properly in python 3.7 and lower \ + on MacOS. Upgrade to python 3.8+" + ) + from daemonize import Daemonize + + d = Daemonize(app="specter", pid=pid_file, action=run) + d.start() + else: + # if not a daemon we can use DEBUG + if debug is None: + debug = app.config["DEBUG"] + run(debug=debug) diff --git a/src/cryptoadvance/specter/config.py b/src/cryptoadvance/specter/config.py index 37bb3f9a0..2ef5819df 100644 --- a/src/cryptoadvance/specter/config.py +++ b/src/cryptoadvance/specter/config.py @@ -7,8 +7,6 @@ from pathlib import Path from dotenv import load_dotenv -DATA_FOLDER = "~/.specter" - # BASEDIR = os.path.abspath(os.path.dirname(__file__)) # Loading env-vars from .flaskenv (4 levels above this file) @@ -30,10 +28,18 @@ def _get_bool_env_var(varname, default=None): return bool(value) +DEFAULT_CONFIG = "cryptoadvance.specter.config.DevelopmentConfig" + + class BaseConfig(object): PORT = os.getenv("PORT", 25441) CONNECT_TOR = _get_bool_env_var(os.getenv("CONNECT_TOR", "False")) - pass + SPECTER_DATA_FOLDER = os.path.expanduser( + os.getenv("SPECTER_DATA_FOLDER", "~/.specter") + ) + # CERT and KEY is for running self-signed-ssl-certs. Check cli_server for details + CERT = os.getenv("CERT", None) + KEY = os.getenv("KEY", None) class DevelopmentConfig(BaseConfig): @@ -45,5 +51,12 @@ class TestConfig(BaseConfig): SECRET_KEY = "test key" +class CypressTestConfig(TestConfig): + SPECTER_DATA_FOLDER = os.path.expanduser( + os.getenv("SPECTER_DATA_FOLDER", "~/.specter-cypress") + ) + PORT = os.getenv("PORT", 25444) + + class ProductionConfig(BaseConfig): pass diff --git a/src/cryptoadvance/specter/server.py b/src/cryptoadvance/specter/server.py index 91ffe096d..c26903d4f 100644 --- a/src/cryptoadvance/specter/server.py +++ b/src/cryptoadvance/specter/server.py @@ -11,7 +11,6 @@ from .helpers import hwi_get_config from .specter import Specter from .hwi_server import hwi_server from .user import User -from .config import DATA_FOLDER from .util.version import VersionChecker from werkzeug.middleware.proxy_fix import ProxyFix @@ -53,7 +52,7 @@ def init_app(app, hwibridge=False, specter=None): if specter is None: # the default. If not None, then it got injected for testing app.logger.info("Initializing Specter") - specter = Specter(DATA_FOLDER) + specter = Specter(data_folder=app.config["SPECTER_DATA_FOLDER"]) # version checker # checks for new versions once per hour diff --git a/src/cryptoadvance/specter/server_endpoints/settings.py b/src/cryptoadvance/specter/server_endpoints/settings.py index 9c831f23a..ea0e8bb87 100644 --- a/src/cryptoadvance/specter/server_endpoints/settings.py +++ b/src/cryptoadvance/specter/server_endpoints/settings.py @@ -45,6 +45,10 @@ def bitcoin_core(): if not current_user.is_admin: flash("Only an admin is allowed to access this page.", "error") return redirect("") + # The node might have been down but is now up again + # (and the checker did not realized yet) and the user clicked "Configure Node" + if app.specter.rpc is None: + app.specter.check() rpc = app.specter.config["rpc"] user = rpc["user"] password = rpc["password"] diff --git a/src/cryptoadvance/specter/specter.py b/src/cryptoadvance/specter/specter.py index a65182091..b918db052 100644 --- a/src/cryptoadvance/specter/specter.py +++ b/src/cryptoadvance/specter/specter.py @@ -29,6 +29,8 @@ def get_rpc(conf, old_rpc=None): Checks if config have changed, compares with old rpc and returns new one if necessary + If there is no working rpc-connection, + it has to return None """ if "autodetect" not in conf: conf["autodetect"] = True @@ -52,7 +54,7 @@ def get_rpc(conf, old_rpc=None): rpc = BitcoinRPC(**conf) # check if we have something to compare with if old_rpc is None: - return rpc + return rpc if rpc.test_connection() else None # check if we have something detected if rpc is None: # check if old rpc is still valid @@ -124,9 +126,11 @@ class Specter: self.check(check_all=True) except Exception as e: logger.error(e) - self.checker = Checker(lambda: self.check(check_all=True)) + self.checker = Checker(lambda: self.check(check_all=True), desc="health") self.checker.start() - self.price_checker = Checker(lambda: update_price(self, self.user)) + self.price_checker = Checker( + lambda: update_price(self, self.user), desc="price" + ) if self.price_check and self.price_provider: self.price_checker.start() @@ -154,7 +158,6 @@ class Specter: else: period = 600 if hasattr(self, "checker") and self.checker.period != period: - logger.info("Checking every %d seconds now" % period) self.checker.period = period self.rpc = rpc diff --git a/src/cryptoadvance/specter/util/checker.py b/src/cryptoadvance/specter/util/checker.py index a4e293d6f..845f0a15d 100644 --- a/src/cryptoadvance/specter/util/checker.py +++ b/src/cryptoadvance/specter/util/checker.py @@ -12,11 +12,13 @@ class Checker: set checker.last_check to 0. """ - def __init__(self, callback, period=600): + def __init__(self, callback, period=600, desc="unknown"): """Checker Contructor :param callback: a function to be called periodically :param period: defines the waiting time in seconds. If you specify values below 1, it won't sleep anymore + :param desc: specifies an optional description used in logging """ + self.desc = desc self.callback = callback self.last_check = 0 self.period = period @@ -30,10 +32,10 @@ class Checker: self.thread = threading.Thread(target=self.loop) self.thread.daemon = True self.thread.start() - logger.info(f"Checker started with period {self.period}") + logger.info(f"Checker {self.desc} started with period {self.period}") def stop(self): - logger.info("Checker stopped.") + logger.info(f"Checker {self.desc} stopped.") self.running = False def loop(self): @@ -64,6 +66,15 @@ class Checker: finally: self.last_check = time.time() + @property + def period(self): + return self._period + + @period.setter + def period(self, value): + self._period = value + logger.info(f"Checker {self.desc} Checking every {self.period} seconds now") + def _sleep(self): if self.period > 1: time.sleep(1) diff --git a/tests/conftest.py b/tests/conftest.py index 6a78233c6..c0e51ef1c 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -59,7 +59,7 @@ def bitcoin_regtest(docker, request): bitcoind_controller = ( BitcoindPlainController() ) # Alternatively take the one on the path for now - bitcoind_controller.start_bitcoind(cleanup_at_exit=True) + bitcoind_controller.start_bitcoind(cleanup_at_exit=True, cleanup_hard=True) running_version = bitcoind_controller.version() requested_version = request.config.getoption("--bitcoind-version") assert ( diff --git a/tests/test_bitcoind.py b/tests/test_bitcoind.py index 195b727d7..3e3e9c319 100644 --- a/tests/test_bitcoind.py +++ b/tests/test_bitcoind.py @@ -15,13 +15,6 @@ def test_bitcoinddocker_running(caplog, docker, request): rpcport=18999, docker_tag=requested_version ) # completly different port to not interfere else: - try: - which("bitcoind") - except: - # Skip this test as bitcoind is not available - # Doesn't make sense to print anything as this won't be shown - # for passing tests - return if os.path.isfile("tests/bitcoin/src/bitcoind"): # copied from conftest.py # always prefer the self-compiled bitcoind if existing @@ -29,16 +22,22 @@ def test_bitcoinddocker_running(caplog, docker, request): bitcoind_path="tests/bitcoin/src/bitcoind" ) else: - my_bitcoind = ( - BitcoindPlainController() - ) # Alternatively take the one on the path for now + try: + which("bitcoind") + my_bitcoind = BitcoindPlainController() + except: + # Skip this test as bitcoind is not available + # Doesn't make sense to print anything as this won't be shown + # for passing tests + raise Exception("bitcoind not available") - rpcconn = my_bitcoind.start_bitcoind(cleanup_at_exit=True) + rpcconn = my_bitcoind.start_bitcoind(cleanup_at_exit=True, cleanup_hard=True) requested_version = request.config.getoption("--bitcoind-version") assert my_bitcoind.version() == requested_version assert rpcconn.get_rpc() != None assert rpcconn.get_rpc().ipaddress != None - rpcconn.get_rpc().getblockchaininfo() + bci = rpcconn.get_rpc().getblockchaininfo() + assert bci["blocks"] == 100 # you can use the testcoin_faucet: random_address = "mruae2834buqxk77oaVpephnA5ZAxNNJ1r" my_bitcoind.testcoin_faucet(random_address, amount=25, mine_tx=True) diff --git a/tests/test_checker.py b/tests/test_checker.py index 79e6c321d..510f10fcd 100644 --- a/tests/test_checker.py +++ b/tests/test_checker.py @@ -8,15 +8,15 @@ from cryptoadvance.specter.util.checker import Checker def test_checker(caplog): callback_mock = Mock() caplog.set_level(logging.DEBUG) - checker = Checker(lambda: callback_mock(), period=0.01) + checker = Checker(lambda: callback_mock(), period=0.01, desc="test") checker.start() time.sleep( 0.1 ) # If the above assumptions are failing, you might want to increase this - assert "Checker started" in caplog.text + assert "Checker test started" in caplog.text assert "This message won't show again until stopped and started." in caplog.text checker.stop() - assert "Checker stopped" in caplog.text + assert "Checker test stopped" in caplog.text callback_mock.side_effect = Exception("someException") checker.start() time.sleep( diff --git a/tests/test_cli.py b/tests/test_cli.py deleted file mode 100644 index eaab7b8eb..000000000 --- a/tests/test_cli.py +++ /dev/null @@ -1,23 +0,0 @@ -import logging - -from cryptoadvance.specter.cli import server -from click.testing import CliRunner -import traceback -import mock -from mock import patch - - -@patch("cryptoadvance.specter.cli.create_app") -@patch("cryptoadvance.specter.cli.init_app") -def test_server_debug(init_app, create_app, caplog): - caplog.set_level(logging.DEBUG) - runner = CliRunner() - result = runner.invoke(server, ["--debug"]) - print(result.output) - if result.exception != None: - # Makes searching for issues much more convenient - traceback.print_tb(result.exception.__traceback__) - print(result.exception) - assert result.exit_code == 0 - assert "Logging is hopefully configured" in caplog.text - assert "We're now on level DEBUG on logger cryptoadvance" in caplog.text diff --git a/tests/test_cli_bitcoind.py b/tests/test_cli_bitcoind.py new file mode 100644 index 000000000..986fe4589 --- /dev/null +++ b/tests/test_cli_bitcoind.py @@ -0,0 +1,26 @@ +import logging + +from cryptoadvance.specter.cli import bitcoind +from click.testing import CliRunner +import sys +import traceback +import mock +from mock import patch, MagicMock, call + + +def test_bitcoind(caplog): + # caplog.set_level(logging.DEBUG) + + runner = CliRunner() + result = runner.invoke(bitcoind, ["--no-mining", "--nodocker", "--cleanuphard"]) + print(result.output) + if result.exception != None: + # Makes searching for issues much more convenient + traceback.print_tb(result.exception.__traceback__) + print(result.exception, file=sys.stderr) + assert result.exit_code == 0 + assert ( + "bitcoin-cli: bitcoin-cli -regtest -rpcuser=bitcoin -rpcpassword=secret getblockchaininfo" + in result.output + ) + # This might take a lot of time because we're waiting on the bitcoind to terminate diff --git a/tests/test_cli_server.py b/tests/test_cli_server.py new file mode 100644 index 000000000..30f1d6aa0 --- /dev/null +++ b/tests/test_cli_server.py @@ -0,0 +1,148 @@ +import logging + +from cryptoadvance.specter.cli import server +from click.testing import CliRunner +import sys +import traceback +import mock +from mock import patch, MagicMock, call + + +@patch("cryptoadvance.specter.cli.cli_server.create_app") +@patch("cryptoadvance.specter.cli.cli_server.init_app") +def test_server_host_and_port(init_app, create_app, caplog): + caplog.set_level(logging.DEBUG) + mock_app = MagicMock() + mock_app.config = MagicMock() + d = { + "SPECTER_DATA_FOLDER": "someValueWillGetChanged", + "PORT": "123", + "DEBUG": "WURSTBROT", + } + mock_app.config.__getitem__.side_effect = d.__getitem__ + create_app.return_value = mock_app + runner = CliRunner() + result = runner.invoke(server, ["--port", "456", "--host", "0.0.0.1"]) + print(result.output) + if result.exception != None: + # Makes searching for issues much more convenient + traceback.print_tb(result.exception.__traceback__) + print(result.exception, file=sys.stderr) + print(mock_app.config.mock_calls) + assert result.exit_code == 0 + mock_app.config.__setitem__.assert_called_with("PORT", 456) + mock_app.run.assert_called_with( + debug="WURSTBROT", host="0.0.0.1", port="123", extra_files=["templates"] + ) + + +@patch("cryptoadvance.specter.cli.cli_server.create_app") +@patch("cryptoadvance.specter.cli.cli_server.init_app") +def test_server_host_and_port(init_app, create_app, caplog): + caplog.set_level(logging.DEBUG) + mock_app = MagicMock() + mock_app.config = MagicMock() + d = { + "PORT": "123", + "DEBUG": "WURSTBROT", + # We don't want to make a more sophisticated mock, so we simply set here + # the same value we set via CMD-Line + "CERT": "bla", + "KEY": "blub", + } + mock_app.config.__getitem__.side_effect = d.__getitem__ + create_app.return_value = mock_app + runner = CliRunner() + with runner.isolated_filesystem(): + result = runner.invoke(server, ["--cert", "bla", "--key", "blub"]) + print(result.output) + if result.exception != None: + # Makes searching for issues much more convenient + traceback.print_tb(result.exception.__traceback__) + print(result.exception, file=sys.stderr) + assert result.exit_code == 0 + print(mock_app.config.mock_calls) + mock_app.config.__setitem__.call_count = 2 + mock_app.config.__setitem__.assert_called_with("KEY", "blub") + mock_app.config.__setitem__.assert_any_call("CERT", "bla") + # This doesn't work as the tmp-directory is always different + # mock_app.run.assert_called_with(debug='WURSTBROT', host='0.0.0.1', port='123', extra_files=['templates'],ssl_context=('/tmp/tmpb_2552yg/bla', '/tmp/tmpb_2552yg/blub'))) + # So let's check differently: + + print(mock_app.run.call_args.kwargs) + # results in something like: + # {'debug': 'WURSTBROT', 'host': '127.0.0.1', 'port': '123', 'extra_files': ['templates'], 'ssl_context': ('/tmp/tmpnzivft_y/bla', '/tmp/tmpnzivft_y/blub')} + assert mock_app.run.call_args.kwargs["ssl_context"][0].endswith("/bla") + assert mock_app.run.call_args.kwargs["ssl_context"][1].endswith("/blub") + + +@patch("cryptoadvance.specter.cli.cli_server.create_app") +@patch("cryptoadvance.specter.cli.cli_server.init_app") +def test_server_debug(init_app, create_app, caplog): + caplog.set_level(logging.DEBUG) + runner = CliRunner() + result = runner.invoke(server, ["--debug"]) + print(result.output) + if result.exception != None: + # Makes searching for issues much more convenient + traceback.print_tb(result.exception.__traceback__) + print(result.exception) + assert result.exit_code == 0 + assert "Logging is hopefully configured" in caplog.text + assert "We're now on level DEBUG on logger cryptoadvance" in caplog.text + + +@patch("cryptoadvance.specter.cli.cli_server.create_app") +@patch("cryptoadvance.specter.cli.cli_server.init_app") +def test_server_datafolder(init_app, create_app, caplog): + caplog.set_level(logging.DEBUG) + mock_app = MagicMock() + mock_app.config = MagicMock() + d = { + "SPECTER_DATA_FOLDER": "someValueWillGetChanged", + "PORT": "123", + "DEBUG": "WURSTBROT", + } + mock_app.config.__getitem__.side_effect = d.__getitem__ + create_app.return_value = mock_app + runner = CliRunner() + result = runner.invoke(server, ["--specter-data-folder", "~/.specter-some-folder"]) + print(result.output) + if result.exception != None: + # Makes searching for issues much more convenient + traceback.print_tb(result.exception.__traceback__) + print(result.exception, file=sys.stderr) + print(mock_app.config.mock_calls) + assert result.exit_code == 0 + mock_app.config.__setitem__.assert_called_once_with( + "SPECTER_DATA_FOLDER", "~/.specter-some-folder" + ) + + +@patch("cryptoadvance.specter.cli.cli_server.create_app") +@patch("cryptoadvance.specter.cli.cli_server.init_app") +def test_server_config(init_app, create_app, caplog): + caplog.set_level(logging.DEBUG) + mock_app = MagicMock() + mock_app.config = MagicMock() + d = { + "PORT": "123", + "DEBUG": "WURSTBROT", + # We don't want to make a more sophisticated mock, so we simply set here + # the same value we set via CMD-Line + "CERT": "bla", + "KEY": "blub", + } + mock_app.config.__getitem__.side_effect = d.__getitem__ + create_app.return_value = mock_app + runner = CliRunner() + with runner.isolated_filesystem(): + result = runner.invoke(server, ["--config", "MuhConfig"]) + print(result.output) + if result.exception != None: + # Makes searching for issues much more convenient + traceback.print_tb(result.exception.__traceback__) + print(result.exception, file=sys.stderr) + assert result.exit_code == 0 + print(mock_app.config.mock_calls) + create_app.assert_called_once_with(config="cryptoadvance.specter.config.MuhConfig") diff --git a/tests/test_config.py b/tests/test_config.py new file mode 100644 index 000000000..b3bb51310 --- /dev/null +++ b/tests/test_config.py @@ -0,0 +1,12 @@ +""" testing the config.py """ + +from flask import Config + + +def test_config(): + " how can you read the config if do not create an app? " "" + config = Config(".") + print(config) + config.from_object("cryptoadvance.specter.config.DevelopmentConfig") + print(config) + assert config["PORT"] == 25441