diff --git a/.env_sample b/.env_sample index 1fcf55b..2e4a534 100644 --- a/.env_sample +++ b/.env_sample @@ -1,24 +1,24 @@ -secret=please_please_update_me_please -algorithm=HS256 +BAPI_JWT_SECRET=please_please_update_me_please +BAPI_JWT_ALGORITHM=HS256 # expiry time in milliseconds (3600000 = 1 hour) -jwt_expiry_time=3600000 +BAPI_JWT_EXPIRY_TIME=3600000 + +# Set the ASGI root_path for applications submounted below a given URL path. +BAPI_ROOT_PATH = "/" # the log level # values [TRACE, DEBUG, INFO, SUCCESS, WARNING, ERROR, CRITICAL] # default: INFO -log_level=INFO +BAPI_LOG_LEVEL=INFO # the log file, comment or set empty to turn of file logging # default: blitz_api.log -# log_file=blitz_api.log - -# login password -login_password=12345678 +# BAPI_LOG_FILE=blitz_api.log # Enable this if you want to run blitz_gui locally. # This will create a file called ~/blitz_api/.cookie with a # JWT token. -# enable_local_cookie_auth = false +# BAPI_ENABLE_LOCAL_COOKIE_AUTH = false # Platform tells the backend on what kind of system it is running on. # Different platforms might use different data sources and might yield @@ -26,118 +26,119 @@ login_password=12345678 # information will be fetched from Redis instead of the native python implementation. # In case of native_python set other optional config vars np_* further below. # supported values: [raspiblitz, native_python] -# platform=raspiblitz +# BAPI_PLATFORM=raspiblitz # Amount of seconds the app will wait until it'll # send another hardware update # only applies when platform=native_python -gather_hw_info_interval = 2 +BAPI_GATHER_HW_INFO_INTERVAL = 2 # Amount of seconds the app will gather CPU usage data for each update # The resulting CPU usage is averaged over this period of time. # To get realistic results at least 0.1 seconds is recommended -# must be less than and not equal to gather_hw_info_interval +# must be less than and not equal to BAPI_GATHER_HW_INFO_INTERVAL # only applies when platform=native_python -cpu_usage_averaging_period = 0.5 +BAPI_CPU_USAGE_AVERAGING_PERIOD = 0.5 # Poll interval in seconds to gather lightning information and push it via SSE # only applies when platform=native_python -gather_ln_info_interval = 5.0 +BAPI_GATHER_LN_INFO_INTERVAL = 5.0 -# Path to the shell script root folder -shell_script_path = /home/admin - -# The API can push successfull forwards to SSE client. On big nodes +# The API can push successful forwards to SSE client. On big nodes # this can cause lots of traffic. Turn this on if updates are required. # default: false -# sse_notify_forward_successes=false +# BAPI_SSE_NOTIFY_FORWARD_SUCCESSES=false # If set to 0 all forward event will be sent instantly, otherwise they'll # be gathered and sent as an array of notifications. -# This also affects how often the wallet balance is updated (even if sse_notify_forward_successes is false) +# This also affects how often the wallet balance is updated (even if SSE_NOTIFY_FORWARD_SUCCESSES is false) # default: 2.0 seconds # minimum: 0.3 seconds -# forwards_gather_interval=2 +# BAPI_FORWARDS_GATHER_INTERVAL=2 # Redis - uncomment if Redis runs with non standard values (i.e. in Docker etc) -# redis_host=127.0.0.1 -# redis_port=6379 -# redis_db=0 +# BAPI_REDIS_HOST=127.0.0.1 +# BAPI_REDIS_PORT=6379 +# BAPI_REDIS_DB=0 # leave commented if no password is used -# redis_password=my_password +# BAPI_REDIS_PASSWORD=my_password # mainnet, testnet or regtest -network=testnet -bitcoind_ip_mainnet=192.168.1.18 -bitcoind_ip_testnet=192.168.1.18 -bitcoind_ip_regtest=192.168.1.18 -bitcoind_port_rpc_mainnet=8332 -bitcoind_port_rpc_testnet=18332 -bitcoind_port_rpc_regtest=28332 +BAPI_NETWORK=testnet +BAPI_BITCOIND_ADDRESS=192.168.1.18 +# Defaults: +# MAINNET=8332 (default) +# TESTNET=18332 +# REGTEST=28332 +BAPI_BITCOIND_PORT_RPC=8332 # The API can either hashblock OR rawblock to be notified of new blocks. # Hashblock is a bit faster, so it should be used if possible. -bitcoind_zmq_block_rpc="hashblock" -bitcoind_zmq_block_port_mainnet=28332 -bitcoind_zmq_block_port_testnet=28332 -bitcoind_zmq_block_port_regtest=28332 -bitcoind_user=raspibolt -bitcoind_pw=please_please_update_me_please +BAPI_BITCOIND_ZMQ_BLOCK_RPC="hashblock" +BAPI_BITCOIND_ZMQ_BLOCK_PORT=28332 +BAPI_BITCOIND_USER=raspibolt +BAPI_BITCOIND_RPC_PW=please_please_update_me_please # lnd_grpc, cln_jrpc, cln_grpc, none # Please refer to the documentation for the install procedure # for each implementation. -ln_node=lnd_grpc +BAPI_LN_NODE=lnd_grpc # Get hex string via command line: xxd -p -c2000 file.macaroon # LND macaroon in HEX format, or a path to the .macaroon file -lnd_macaroon="0201036...2211 or /path/to/admin.macaroon" -lnd_cert="2d2d2d2d2d...d2d2d2d0a or /path/to/tls.cert" -lnd_grpc_ip=192.168.1.18 -lnd_grpc_port=10009 -lnd_rest_port=8080 +BAPI_LND_MACAROON="0201036...2211 or /path/to/admin.macaroon" +BAPI_LND_CERT="2d2d2d2d2d...d2d2d2d0a or /path/to/tls.cert" +BAPI_LND_GRPC_IP=192.168.1.18 +BAPI_LND_GRPC_PORT=10009 # cln json rpc - path to the socket file -cln_jrpc_path="/mnt/hdd/app-data/.lightning/bitcoin/lightning-rpc" +BAPI_CLN_JRPC_PATH="/mnt/hdd/app-data/.lightning/bitcoin/lightning-rpc" # CLN grpc connection data, cert files are in .lightning data folder # file contents in HEX format, or a path to the file -cln_grpc_cert="2d2d2d2d2d...d2d2d2d0a or /path/to/client.pem" -cln_grpc_key="2d2d2d2d2d...d2d2d2d0a or /path/to/client-key.pem" -cln_grpc_ca="2d2d2d2d2d...d2d2d2d0a or /path/to/ca.pem" -cln_grpc_ip=127.0.0.1 -cln_grpc_port=9537 +BAPI_CLN_GRPC_CERT="2d2d2d2d2d...d2d2d2d0a or /path/to/client.pem" +BAPI_CLN_GRPC_KEY="2d2d2d2d2d...d2d2d2d0a or /path/to/client-key.pem" +BAPI_CLN_GRPC_CA="2d2d2d2d2d...d2d2d2d0a or /path/to/ca.pem" +BAPI_CLN_GRPC_IP=127.0.0.1 +BAPI_CLN_GRPC_PORT=9537 -# Tor url of this system. Ignored on platform Raspiblitz. -# Defaults to empty string -# np_tor_address="" +######################## +# RaspiBlitz env varables +######################## + +# Path to the shell script root folder +BAPI_RB_SHELL_SCRIPT_PATH="/home/admin" + +######################## +# native python env varables +######################## + +# login password for the native_python system implementation +# ATTN: https://github.com/fusion44/blitz_api/issues/255 +# BAPI_NATIVE_LOGIN_PASSWORD=12345678 # Tor url of api endpoint. Ignored on platform Raspiblitz # Defaults to empty string -# np_tor_address_api_endpoint="address.onion/api" +# BAPI_NP_TOR_ADDRESS_API_ENDPOINT="address.onion/api" # Tor url of the api docs. Ignored on platform Raspiblitz # Defaults to empty string -# np_tor_address_api_docs="address.onion/latest/docs" - -# Local LAN IP. Ignored on platform Raspiblitz -# Defaults to empty string -# np_local_ip="192.168.1.50" +# BAPI_NP_TOR_ADDRESS_API_DOCS="address.onion/latest/docs" # Local LAN url of api endpoint. Ignored on platform Raspiblitz # Defaults to empty string -# np_local_address_api_endpoint="address.onion/api" +# BAPI_NP_LOCAL_ADDRESS_API_DOCS="address.onion/api" # Local LAN of the api docs. Ignored on platform Raspiblitz # Defaults to empty string -# np_local_address_api_docs="address.onion/latest/docs" +# BAPI_NP_LOCAL_ADDRESS_API_DOCS="address.onion/latest/docs" # SSH login address. Ignored on platform Raspiblitz # Defaults to empty string -# np_ssh_address="username@192.168.1.50" +# BAPI_NP_SSH_ADDRESS="username@192.168.1.50" # Version of the platform. Ignored on platform Raspiblitz -# np_version="v0.5.1beta" +# BAPI_NP_VERSION ="v0.5.1beta" # Enable remote debugging for the server. # @@ -146,7 +147,7 @@ cln_grpc_port=9537 # Note the package debugpy must be installed manually, if set to True # More info: https://github.com/fusion44/blitz_api/issues/206 # Defaults to false -# remote_debugging=false +# BAPI_REMOTE_DEBUGGING=false # Defaults to 5678 -# remote_debugging_port=5678 +# BAPI_REMOTE_DEBUGGING_PORT=5678 diff --git a/README.md b/README.md index b5b732d..294d5ba 100644 --- a/README.md +++ b/README.md @@ -38,6 +38,9 @@ This software is still considered BETA and may contain bugs. Don't expose it to Create a `.env` file with your `bitcoind` and `lnd` configuration. See the `.env_sample` file for all configuration options. +The `.env` file is expected to be at the project root folder by default. +To use a custom path, set the `BAPI_ENV_PATH` env variable to the `.env` file path. + ### Dependencies - [Python in version 3.7](https://www.python.org/downloads/) diff --git a/app/api/config.py b/app/api/config.py new file mode 100644 index 0000000..1b17b46 --- /dev/null +++ b/app/api/config.py @@ -0,0 +1,48 @@ +import os +from decouple import ( + Config, + RepositoryEmpty, + RepositoryEnv, + Undefined, + UndefinedValueError, +) +from loguru import logger +from typing import Any + +_config: Config | None = None + + +def config( + option: str, + default: Any | Undefined = Undefined(), + cast: Any | Undefined = Undefined(), +): + if _config is None: + _setup_config() + logger.trace("Configuration was not initialized => calling setup_config()") + + try: + if _config is not None: # query again, to please the linter + return _config(option, default=default, cast=cast) + except UndefinedValueError as e: + logger.debug(f"Type of _config: {_config}") + raise e + + +def _setup_config(): + global _config + file_path = os.environ.get("BAPI_ENV_PATH") + file_path = "" if file_path is None else os.path.abspath(file_path) + try: + if os.path.isfile(file_path): + logger.info(f"Using configuration from: {file_path}") + _config = Config(RepositoryEnv(file_path)) + elif os.path.isfile(".env"): + logger.info("Using configuration from: .env") + _config = Config(RepositoryEnv(".env")) + else: + logger.info("No configuration file found, using empty repository") + _config = Config(RepositoryEmpty()) + except Exception as e: + logger.error(f"Exception {e}, using empty repository") + _config = Config(RepositoryEmpty()) diff --git a/app/apps/impl/native_python.py b/app/apps/impl/native_python.py index ea099f8..e8c7bf8 100644 --- a/app/apps/impl/native_python.py +++ b/app/apps/impl/native_python.py @@ -16,7 +16,73 @@ class NativePythonApps(AppsBase): raise _NotImplemented() async def get_app_status(self): - raise _NotImplemented() + # TODO: revert me before merge to main + return [ + { + "id": "btcpayserver", + "version": "v1.12.5", + "installed": False, + "status": "offline", + "error": "", + }, + { + "id": "lnbits", + "version": "0.11.3", + "installed": False, + "status": "offline", + "error": "", + }, + { + "id": "rtl", + "version": "v0.14.1", + "installed": False, + "status": "offline", + "error": "", + }, + { + "id": "electrs", + "installed": True, + "configured": False, + "status": "online", + "localIP": "", + "httpPort": "", + "httpsPort": "", + "httpsForced": False, + "httpsSelfsigned": False, + "hiddenService": "", + "address": "http://:", + "authMethod": "none", + "details": {}, + }, + { + "id": "btc-rpc-explorer", + "version": "v3.4.0", + "installed": False, + "status": "offline", + "error": "", + }, + { + "id": "mempool", + "version": "v2.5.0", + "installed": False, + "status": "offline", + "error": "", + }, + { + "id": "jam", + "version": "0.2.0", + "installed": False, + "status": "offline", + "error": "", + }, + { + "id": "thunderhub", + "version": "v0.13.30", + "installed": False, + "status": "offline", + "error": "", + }, + ] async def get_app_status_advanced(self, app_id: str): raise _NotImplemented() diff --git a/app/apps/impl/raspiblitz.py b/app/apps/impl/raspiblitz.py index ba2da27..f3001d2 100644 --- a/app/apps/impl/raspiblitz.py +++ b/app/apps/impl/raspiblitz.py @@ -6,11 +6,11 @@ import os import random from typing import List -from decouple import config from fastapi import HTTPException, status from fastapi.encoders import jsonable_encoder from loguru import logger as logging +from app.api.config import config from app.api.utils import SSE, broadcast_sse_msg, call_sudo_script, parse_key_value_text from app.apps.impl.apps_base import AppsBase @@ -26,13 +26,12 @@ available_app_ids = { "thunderhub", "jam", "electrs", - "albyhub", } -SHELL_SCRIPT_PATH = config("shell_script_path") +SHELL_SCRIPT_PATH = config("BAPI_RB_SHELL_SCRIPT_PATH") -node_type = config("ln_node") +node_type = config("BAPI_LN_NODE", default="none").to_lower() class RaspiBlitzApps(AppsBase): diff --git a/app/apps/service.py b/app/apps/service.py index 12faba5..c4a7ed7 100644 --- a/app/apps/service.py +++ b/app/apps/service.py @@ -1,14 +1,16 @@ -from decouple import config - +from app.api.config import config from app.system.models import APIPlatform -PLATFORM = config("platform", default=APIPlatform.RASPIBLITZ) -apps = None - +PLATFORM = config("BAPI_PLATFORM", default=APIPlatform.UNKNOWN) if PLATFORM == APIPlatform.RASPIBLITZ: from app.apps.impl.raspiblitz import RaspiBlitzApps as Apps elif PLATFORM == APIPlatform.NATIVE_PYTHON: from app.apps.impl.native_python import NativePythonApps as Apps +else: + raise RuntimeError( + f"Unsupported platform '{PLATFORM}'. Options: {APIPlatform.values_as_list()}." + ) + apps = Apps() diff --git a/app/auth/auth_handler.py b/app/auth/auth_handler.py index 52f7c3c..0c68fd4 100644 --- a/app/auth/auth_handler.py +++ b/app/auth/auth_handler.py @@ -4,12 +4,13 @@ import time from typing import Dict import jwt -from decouple import config from loguru import logger -JWT_SECRET = config("secret") -JWT_ALGORITHM = config("algorithm") -JWT_EXPIRY_TIME = config("jwt_expiry_time", default=300, cast=int) +from app.api.config import config + +JWT_SECRET = config("BAPI_JWT_SECRET") +JWT_ALGORITHM = config("BAPI_JWT_ALGORITHM") +JWT_EXPIRY_TIME = config("BAPI_JWT_EXPIRY_TIME", default=300, cast=int) def sign_jwt() -> Dict[str, str]: @@ -35,15 +36,23 @@ def handle_local_cookie(): blitz_path = os.path.join(os.path.expanduser("~"), ".blitz_api") full_cookie_file_path = os.path.join(blitz_path, ".cookie") - enabled = config("enable_local_cookie_auth", default=False, cast=bool) + enabled = config("BAPI_ENABLE_LOCAL_COOKIE_AUTH", default=False, cast=bool) + + if not enabled: + return if not os.path.exists(blitz_path): - os.makedirs(blitz_path) - - if enabled: - f = open(full_cookie_file_path, "w") - f.write(sign_jwt()) - f.close() + try: + os.makedirs(blitz_path) + except OSError as e: + logger.error( + f"""Unable to create the .blit_api folder: {e} + Please make sure that the target folder is readable. + """ + ) + f = open(full_cookie_file_path, "w") + f.write(sign_jwt()) + f.close() def remove_local_cookie(): diff --git a/app/bitcoind/models.py b/app/bitcoind/models.py index 9d6dc96..7908123 100644 --- a/app/bitcoind/models.py +++ b/app/bitcoind/models.py @@ -5,6 +5,8 @@ from typing import List, Optional, Union from fastapi import Query from pydantic.main import BaseModel +from loguru import logger + class FeeEstimationMode(str, Enum): CONSERVATIVE = "conservative" @@ -23,7 +25,9 @@ class BlockRpcFunc(str, Enum): return cls.RAWBLOCK else: raise ArgumentError( - "Function name must either be 'hashblock' or 'rawblock'" + None, + "Function name must either be 'hashblock' or 'rawblock'." + f" Actual: ${func}", ) @@ -153,14 +157,10 @@ class NetworkInfo(BaseModel): local_addresses: List[BtcLocalAddress] = Query( [], description="List of local addresses" ) - warnings: str = Query(None, description="Any network and blockchain warnings") + warnings: List[str] = Query(None, description="Any network and blockchain warnings") @classmethod def from_rpc(cls, r): - networks = [] - for n in r["networks"]: - networks.append(BtcNetwork.from_rpc(n)) - return cls( version=r["version"], subversion=r["subversion"], @@ -372,7 +372,7 @@ class BlockchainInfo(BaseModel): "enabled)" ), ) - warnings: str = Query(..., description="Any network and blockchain warnings") + warnings: List[str] = Query(..., description="Any network and blockchain warnings") softforks: List[SoftFork] = Query(..., description="Status of softforks") @classmethod diff --git a/app/bitcoind/service.py b/app/bitcoind/service.py index 20e6a95..8e26df0 100644 --- a/app/bitcoind/service.py +++ b/app/bitcoind/service.py @@ -64,7 +64,7 @@ async def initialize_bitcoin_repo() -> bool: async def get_blockchain_info() -> BlockchainInfo: result = await bitcoin_rpc_async("getblockchaininfo") - if result["error"] is not None: + if "error" in result and result["error"] is not None: raise HTTPException(result["status"], detail=result["error"]) return BlockchainInfo.from_rpc(result["result"]) @@ -77,7 +77,7 @@ async def estimate_fee( ) -> int: result = await bitcoin_rpc_async("estimatesmartfee", [target_conf, mode]) - if result["error"] is not None: + if "error" in result and result["error"] is not None: raise HTTPException(result["status"], detail=result["error"]) if "errors" in result["result"]: @@ -99,7 +99,7 @@ async def estimate_fee( async def get_network_info() -> NetworkInfo: result = await bitcoin_rpc_async("getnetworkinfo") - if result["error"] is not None: + if "error" in result and result["error"] is not None: raise HTTPException(result["status"], detail=result["error"]) return NetworkInfo.from_rpc(result["result"]) @@ -109,7 +109,7 @@ async def get_network_info() -> NetworkInfo: async def get_raw_transaction(txid: str) -> RawTransaction: result = await bitcoin_rpc_async("getrawtransaction", [txid, 1]) - if result["error"] is None: + if "error" not in result or result["error"] is None: return RawTransaction.from_rpc(result["result"]) if "No such mempool or blockchain transaction." in result["error"]: diff --git a/app/bitcoind/utils.py b/app/bitcoind/utils.py index 2f002dc..5ea6347 100644 --- a/app/bitcoind/utils.py +++ b/app/bitcoind/utils.py @@ -4,35 +4,31 @@ from types import coroutine import aiohttp import requests -from decouple import config +from loguru import logger from starlette import status +from app.api.config import config from app.bitcoind.models import BlockRpcFunc class _BitcoinConfig: def __init__(self) -> None: - self.network = config("network") - self.zmq_block_rpc = BlockRpcFunc.from_string(config("bitcoind_zmq_block_rpc")) + self.network = config("BAPI_NETWORK") + self.zmq_block_rpc = BlockRpcFunc.from_string( + str(config("BAPI_BITCOIND_ZMQ_BLOCK_RPC", default="hashblock")) + ) - if self.network == "testnet": - self.ip = config("bitcoind_ip_testnet") - self.rpc_port = config("bitcoind_port_rpc_testnet") - self.zmq_port = config("bitcoind_zmq_block_port_testnet") - elif self.network == "regtest": - self.ip = config("bitcoind_ip_regtest") - self.rpc_port = config("bitcoind_port_rpc_regtest") - self.zmq_port = config("bitcoind_zmq_block_port_regtest") - else: - self.ip = config("bitcoind_ip_mainnet") - self.rpc_port = config("bitcoind_port_rpc_mainnet") - self.zmq_port = config("bitcoind_zmq_block_port_mainnet") + self.ip = config("BAPI_BITCOIND_ADDRESS") + self.rpc_port = config("BAPI_BITCOIND_PORT_RPC") + self.zmq_port = config("BAPI_BITCOIND_ZMQ_BLOCK_PORT") self.rpc_url = f"http://{self.ip}:{self.rpc_port}" self.zmq_url = f"tcp://{self.ip}:{self.zmq_port}" - self.username = config("bitcoind_user") - self.pw = config("bitcoind_pw") + self.username = config("BAPI_BITCOIND_USER") + self.pw = config("BAPI_BITCOIND_RPC_PW") + + logger.trace(f"Built Bitcoin config: {self.rpc_url} {self.zmq_url}") bitcoin_config = _BitcoinConfig() diff --git a/app/lightning/impl/cln_grpc.py b/app/lightning/impl/cln_grpc.py index 21e2ddb..2912d3f 100644 --- a/app/lightning/impl/cln_grpc.py +++ b/app/lightning/impl/cln_grpc.py @@ -4,7 +4,6 @@ import sys from typing import AsyncGenerator, List, Optional import grpc -from decouple import config from fastapi.exceptions import HTTPException from loguru import logger from starlette import status @@ -12,6 +11,7 @@ from starlette import status import app.lightning.impl.protos.cln.node_pb2 as ln import app.lightning.impl.protos.cln.node_pb2_grpc as clnrpc import app.lightning.impl.protos.cln.primitives_pb2 as lnp +from app.api.config import config from app.api.utils import SSE, broadcast_sse_msg, config_get_hex_str, next_push_id from app.bitcoind.utils import bitcoin_rpc_async from app.lightning.exceptions import NodeNotFoundError @@ -44,7 +44,7 @@ async def _make_local_call(cmd: str): # FIXME: this is a hack because some of the commands are not exposed # in the CLN grpc interface yet. - testnet = config("network") == "testnet" + testnet = config("BAPI_NETWORK") == "testnet" cmd = f"lightning-cli -k {'--testnet ' if testnet else ''}{cmd}" proc = await asyncio.create_subprocess_shell( cmd, @@ -127,17 +127,25 @@ class LnNodeCLNgRPC(LightningNodeBase): try: cln_grpc_key = bytes.fromhex( - config_get_hex_str(config("cln_grpc_key"), name="cln_grpc_key") + config_get_hex_str( + str(config("BAPI_CLN_GRPC_KEY")), name="cln_grpc_key" + ) ) cln_grpc_cert = bytes.fromhex( - config_get_hex_str(config("cln_grpc_cert"), name="cln_grpc_cert") + config_get_hex_str( + str(config("BAPI_CLN_GRPC_CERT")), name="cln_grpc_cert" + ) ) cln_grpc_ca = bytes.fromhex( - config_get_hex_str(config("cln_grpc_ca"), name="cln_grpc_ca") + config_get_hex_str(str(config("BAPI_CLN_GRPC_CA")), name="cln_grpc_ca") + ) + cln_grpc_url = ( + str(config("BAPI_CLN_GRPC_IP")) + + ":" + + str(config("BAPI_CLN_GRPC_PORT")) ) - cln_grpc_url = config("cln_grpc_ip") + ":" + config("cln_grpc_port") except ValueError as e: - logger.critical(f"Unable to decode cln_grpc_cert: {e.args}.") + logger.critical(f"Unable to decode BAPI_CLN_GRPC_CERT: {e.args}.") sys.exit(0) self.creds = grpc.ssl_channel_credentials( @@ -816,7 +824,7 @@ class LnNodeCLNgRPC(LightningNodeBase): # CLN has no subscription to forwarded events. # We must poll instead. - interval = config("gather_ln_info_interval", default=2, cast=float) + interval = config("BAPI_GATHER_LN_INFO_INTERVAL", default=2, cast=float) # make sure we know how many forwards we have # we need to calculate the difference between each iteration diff --git a/app/lightning/impl/cln_jrpc.py b/app/lightning/impl/cln_jrpc.py index 2f55785..e7ba3b2 100644 --- a/app/lightning/impl/cln_jrpc.py +++ b/app/lightning/impl/cln_jrpc.py @@ -5,11 +5,11 @@ import sys from typing import AsyncGenerator, Dict, List, Optional, Union import decouple -from decouple import config from fastapi.exceptions import HTTPException from loguru import logger from starlette import status +from app.api.config import config from app.api.utils import SSE, broadcast_sse_msg, next_push_id from app.bitcoind.utils import bitcoin_rpc_async from app.lightning.exceptions import NodeNotFoundError @@ -78,14 +78,13 @@ class LnNodeCLNjRPC(LightningNodeBase): yield InitLnRepoUpdate(state=LnInitState.BOOTSTRAPPING) try: - self._socket_path = str(decouple.config("cln_jrpc_path")) - print(decouple.config("cln_jrpc_path")) + self._socket_path = str(config("BAPI_CLN_JRPC_PATH")) except decouple.UndefinedValueError as e: logger.debug(e) logger.error( ( - "CLN JSON-RPC implementation set, but cln_jrpc_path is missing " - "from the config file." + "CLN JSON-RPC implementation set, but BAPI_CLN_JRPC_PATH is " + "missing from the config file." ) ) sys.exit(1) @@ -699,7 +698,7 @@ class LnNodeCLNjRPC(LightningNodeBase): # CLN has no subscription to forwarded events. # We must poll instead. - interval = config("gather_ln_info_interval", default=2, cast=float) + interval = config("BAPI_GATHER_LN_INFO_INTERVAL", default=2, cast=float) # make sure we know how many forwards we have # we need to calculate the difference between each iteration diff --git a/app/lightning/impl/lnd_grpc.py b/app/lightning/impl/lnd_grpc.py index 0d96eae..6aa0db5 100644 --- a/app/lightning/impl/lnd_grpc.py +++ b/app/lightning/impl/lnd_grpc.py @@ -3,7 +3,6 @@ import os from typing import AsyncGenerator, List, Optional import grpc -from decouple import config as dconfig from fastapi.exceptions import HTTPException from loguru import logger from starlette import status @@ -15,6 +14,7 @@ import app.lightning.impl.protos.lnd.router_pb2 as router import app.lightning.impl.protos.lnd.router_pb2_grpc as routerrpc import app.lightning.impl.protos.lnd.walletunlocker_pb2 as unlocker import app.lightning.impl.protos.lnd.walletunlocker_pb2_grpc as unlockerrpc +from app.api.config import config as dconfig from app.api.utils import SSE, broadcast_sse_msg, config_get_hex_str from app.lightning.exceptions import NodeNotFoundError from app.lightning.impl.ln_base import LightningNodeBase @@ -223,17 +223,19 @@ This will show more debug information. ) yield InitLnRepoUpdate(state=LnInitState.DONE) - lnd_macaroon = config_get_hex_str(dconfig("lnd_macaroon"), name="lnd_macaroon") + lnd_macaroon = config_get_hex_str( + str(dconfig("BAPI_LND_MACAROON")), name="lnd_macaroon" + ) lnd_cert = bytes.fromhex( - config_get_hex_str(dconfig("lnd_cert"), name="lnd_cert") + config_get_hex_str(str(dconfig("BAPI_LND_CERT")), name="lnd_cert") ) def metadata_callback(context, callback): # for more info see grpc docs callback([("macaroon", lnd_macaroon)], None) - lnd_grpc_ip = dconfig("lnd_grpc_ip") - lnd_grpc_port = dconfig("lnd_grpc_port") + lnd_grpc_ip = str(dconfig("BAPI_LND_GRPC_IP")) + lnd_grpc_port = str(dconfig("BAPI_LND_GRPC_PORT")) self._lnd_grpc_url = lnd_grpc_ip + ":" + lnd_grpc_port auth_creds = grpc.metadata_call_credentials(metadata_callback) diff --git a/app/lightning/impl/specializations/cln_grpc_blitz.py b/app/lightning/impl/specializations/cln_grpc_blitz.py index 0c4189f..7c4f554 100644 --- a/app/lightning/impl/specializations/cln_grpc_blitz.py +++ b/app/lightning/impl/specializations/cln_grpc_blitz.py @@ -1,11 +1,11 @@ import asyncio from typing import AsyncGenerator, List, Optional -from decouple import config from fastapi.exceptions import HTTPException from loguru import logger from starlette import status +from app.api.config import config from app.api.utils import redis_get from app.lightning.impl.cln_grpc import LnNodeCLNgRPC from app.lightning.impl.specializations.blitz_common import blitz_cln_unlock @@ -34,7 +34,7 @@ class LnNodeCLNgRPCBlitz(LnNodeCLNgRPC): _unlocked = False - _NETWORK = config("network", default="mainnet") + _NETWORK = config("BAPI_NETWORK", default="mainnet") def get_implementation_name(self) -> str: return "CLN_GRPC_BLITZ" diff --git a/app/lightning/impl/specializations/cln_jrpc_blitz.py b/app/lightning/impl/specializations/cln_jrpc_blitz.py index cf67e54..0588e01 100644 --- a/app/lightning/impl/specializations/cln_jrpc_blitz.py +++ b/app/lightning/impl/specializations/cln_jrpc_blitz.py @@ -1,11 +1,11 @@ import asyncio from typing import AsyncGenerator, List, Optional -from decouple import config from fastapi.exceptions import HTTPException from loguru import logger from starlette import status +from app.api.config import config from app.api.utils import redis_get from app.lightning.impl.cln_jrpc import LnNodeCLNjRPC from app.lightning.impl.specializations.blitz_common import blitz_cln_unlock @@ -34,7 +34,7 @@ class LnNodeCLNjRPCBlitz(LnNodeCLNjRPC): _unlocked = False - _NETWORK = config("network", default="mainnet") + _NETWORK = config("BAPI_NETWORK", default="mainnet") def get_implementation_name(self) -> str: return "CLN_JRPC_BLITZ" diff --git a/app/lightning/service.py b/app/lightning/service.py index 58aea4f..263eb7f 100644 --- a/app/lightning/service.py +++ b/app/lightning/service.py @@ -1,11 +1,11 @@ import asyncio from typing import AsyncGenerator, List, Optional -from decouple import config from fastapi import status from fastapi.exceptions import HTTPException from loguru import logger +from app.api.config import config from app.api.utils import SSE, broadcast_sse_msg, redis_get from app.lightning.models import ( Channel, @@ -23,9 +23,9 @@ from app.lightning.models import ( ) from app.system.models import APIPlatform -PLATFORM = config("platform", cast=str) +PLATFORM = config("BAPI_PLATFORM", cast=str) -ln_node = config("ln_node").lower() +ln_node = config("BAPI_LN_NODE", default="none").lower() if ln_node == "lnd_grpc": from app.lightning.impl.lnd_grpc import LnNodeLNDgRPC as LnNode elif ln_node == "cln_jrpc" and PLATFORM == APIPlatform.RASPIBLITZ: @@ -49,19 +49,22 @@ else: logger.error(f"config: unknown lightning node: {ln_node}") raise RuntimeError(f"unknown lightning node type: {ln_node}") -GATHER_INFO_INTERVALL = config("gather_ln_info_interval", default=2, cast=float) +GATHER_INFO_INTERVALL = config("BAPI_GATHER_LN_INFO_INTERVAL", default=2, cast=float) _CACHE = {"wallet_balance": None} ENABLE_FWD_NOTIFICATIONS = config( - "sse_notify_forward_successes", default=False, cast=bool + "BAPI_SSE_NOTIFY_FORWARD_SUCCESSES", default=False, cast=bool ) -FWD_GATHER_INTERVAL = config("forwards_gather_interval", default=2.0, cast=float) +FWD_GATHER_INTERVAL = config("BAPI_FORWARDS_GATHER_INTERVAL", default=2.0, cast=float) if FWD_GATHER_INTERVAL < 0.3: - raise RuntimeError("forwards_gather_interval cannot be less than 0.3 seconds") + raise RuntimeError("BAPI_FORWARDS_GATHER_INTERVAL cannot be less than 0.3 seconds") + +if ln_node != "none": + ln = LnNode() if ln_node != "none": ln = LnNode() diff --git a/app/logging.py b/app/logging.py index 5d8a914..8c0bc0a 100644 --- a/app/logging.py +++ b/app/logging.py @@ -1,9 +1,10 @@ import logging import sys -from decouple import config as dconfig from loguru import logger +from app.api.config import config as dconfig + # Sourced from LNbits project: # https://github.com/lnbits/lnbits/blob/841e8e7bbd61fb942a776d82ca0b6d03668eb524/lnbits/app.py#L285 @@ -11,8 +12,8 @@ from loguru import logger def configure_logger() -> None: - level = dconfig("log_level", default="INFO", cast=str) - log_file = dconfig("log_file", default="", cast=str) + level = dconfig("BAPI_LOG_LEVEL", default="INFO", cast=str) + log_file = dconfig("BAPI_LOG_FILE", default="", cast=str) logger.remove() formatter = Formatter(level) @@ -84,4 +85,17 @@ class InterceptHandler(logging.Handler): level = logger.level(record.levelname).name except ValueError: level = record.levelno - logger.log(level, record.getMessage()) + + try: + logger.log(level, record.getMessage()) + except TypeError as e: + logger.error( + f"""Unable to process log message: {e} + Name of the record: + {record.name} + Message of the record: + {record.msg} + Args of the record: + {record.args} + """ + ) diff --git a/app/main.py b/app/main.py index c9037ed..69c4fb4 100644 --- a/app/main.py +++ b/app/main.py @@ -2,22 +2,19 @@ import asyncio import sys from contextlib import asynccontextmanager -from decouple import config as dconfig from fastapi import FastAPI, Request from fastapi.encoders import jsonable_encoder from fastapi.exceptions import HTTPException -from fastapi_plugins import ( - RedisSettings, - get_config, - redis_plugin, - registered_configuration, -) +from fastapi_plugins import RedisSettings +from fastapi_plugins import get_config as get_redis_config +from fastapi_plugins import redis_plugin, registered_configuration from loguru import logger from pydantic import BaseModel from starlette import status from starlette.middleware.cors import CORSMiddleware from starlette.responses import RedirectResponse +from app.api.config import config as dconfig from app.api.models import ApiStartupStatus, StartupState from app.api.utils import SSE, broadcast_sse_msg, build_sse_event, sse_mgr from app.api.warmup import ( @@ -49,7 +46,7 @@ from app.system.service import get_hardware_info, register_hardware_info_gathere configure_logger() -remote_debugging = dconfig("remote_debugging", cast=bool, default=False) +remote_debugging = dconfig("BAPI_REMOTE_DEBUGGING", cast=bool, default=False) if remote_debugging: logger.warning( ( @@ -57,7 +54,9 @@ if remote_debugging: "Only enable on development machines." ) ) - remote_debugging_port = dconfig("remote_debugging_port", cast=int, default=5678) + remote_debugging_port = dconfig( + "BAPI_REMOTE_DEBUGGING_PORT", cast=int, default=5678 + ) try: import debugpy @@ -67,7 +66,7 @@ if remote_debugging: debugpy.listen(("0.0.0.0", remote_debugging_port)) -node_type = dconfig("ln_node").lower() +node_type = dconfig("BAPI_LN_NODE", default="none").lower() if node_type == "": node_type = "none" @@ -77,7 +76,7 @@ class AppSettings(RedisSettings): api_name: str = str(__name__) -config = get_config() +config = get_redis_config() @asynccontextmanager @@ -221,8 +220,10 @@ async def _initialize_lightning(): @app.get("/") def index(req: Request): + logger.info(req.url) + p = req.scope.get("root_path") return RedirectResponse( - "/api/docs", + f"{p}/docs", status_code=status.HTTP_307_TEMPORARY_REDIRECT, ) @@ -318,6 +319,7 @@ async def warmup_new_connections(): _handle(id, SSE.HARDWARE_INFO, res[6]), ] ) + # when its bitcoin only else: res = await get_full_client_warmup_data_bitcoinonly() diff --git a/app/server.py b/app/server.py index bca163f..017932c 100644 --- a/app/server.py +++ b/app/server.py @@ -2,13 +2,31 @@ import uvicorn import click # isort:skip +from app.api.config import config + @click.command() @click.option("--port", default="5000", help="Port to run Blitz API on") @click.option("--host", default="127.0.0.1", help="Host to run Blitz API on") -def main(port, host): +@click.option( + "--root_path", + default=None, + help="Set the ASGI 'root_path' for applications submounted below a given URL path.", +) +def main(port, host, root_path): """Launched with `poetry run api` at root level""" - uvicorn.run("app.main:app", port=port, host=host) + + p = "" + if root_path is not None and root_path != "": + p = root_path + else: + p = str(config("BAPI_ROOT_PATH", default="")) + + if p == "/": + root_path = "" + + print(f"launching with {host}:{port}{p}") + uvicorn.run("app.main:app", port=port, host=host, root_path=p) if __name__ == "__main__": diff --git a/app/setup/router.py b/app/setup/router.py index b8ad9cb..47b6e7f 100644 --- a/app/setup/router.py +++ b/app/setup/router.py @@ -1,9 +1,8 @@ -from decouple import config - +from app.api.config import config from app.system.models import APIPlatform router = None -_PLATFORM = config("platform", default=APIPlatform.RASPIBLITZ) +_PLATFORM = config("BAPI_PLATFORM", default=APIPlatform.RASPIBLITZ) if _PLATFORM == APIPlatform.RASPIBLITZ: from app.setup.impl.raspiblitz.router import router # noqa: F401 diff --git a/app/system/impl/native_python.py b/app/system/impl/native_python.py index c1a025a..92c82e1 100644 --- a/app/system/impl/native_python.py +++ b/app/system/impl/native_python.py @@ -1,11 +1,11 @@ -import logging import secrets from typing import Dict import psutil -from decouple import config from fastapi import HTTPException, status +from loguru import logger +from app.api.config import config from app.api.constants import API_VERSION from app.auth.auth_handler import sign_jwt from app.lightning.service import get_ln_info @@ -19,24 +19,25 @@ from app.system.models import ( SystemInfo, ) -_SLEEP_TIME = config("gather_hw_info_interval", default=2, cast=float) -_CPU_AVG_PERIOD = config("cpu_usage_averaging_period", default=0.5, cast=float) +_SLEEP_TIME = config("BAPI_GATHER_HW_INFO_INTERVAL", default=2, cast=float) +_CPU_AVG_PERIOD = config("BAPI_CPU_USAGE_AVERAGING_PERIOD", default=0.5, cast=float) _HW_INFO_YIELD_TIME = _SLEEP_TIME + _CPU_AVG_PERIOD class NativePythonSystem(SystemBase): + @logger.catch(exclude=(HTTPException,)) async def get_system_info(self) -> SystemInfo: lninfo = await get_ln_info() - version = config("np_version", default="") + version = config("BAPI_NP_VERSION", default="") - tor_api = config("np_tor_address_api_endpoint", default="") - tor_api_docs = config("np_tor_address_api_docs", default="") + tor_api = config("BAPI_NP_TOR_ADDRESS_API_ENDPOINT", default="") + tor_api_docs = config("BAPI_NP_TOR_ADDRESS_API_DOCS", default="") - lan_api = config("np_local_address_api_endpoint", default="") - lan_api_docs = config("np_local_address_api_docs", default="") + lan_api = config("BAPI_NP_LOCAL_ADDRESS_API_ENDPOINT", default="") + lan_api_docs = config("BAPI_NP_LOCAL_ADDRESS_API_DOCS", default="") - ssh_address = config("np_ssh_address", default="") + ssh_address = config("BAPI_NP_SSH_ADDRESS", default="") return SystemInfo( alias=lninfo.alias, @@ -52,19 +53,26 @@ class NativePythonSystem(SystemBase): chain=lninfo.chains[0].network, ) + @logger.catch(exclude=(HTTPException,)) async def get_system_health(self, verbose: bool) -> SystemHealthInfo: return SystemHealthInfo(healthy=True) + @logger.catch(exclude=(HTTPException,)) async def shutdown(self, reboot: bool) -> bool: - logging.info("Shutdown / reboot not supported in native_python mode.") + logger.info("Shutdown / reboot not supported in native_python mode.") return False + @logger.catch(exclude=(HTTPException,)) async def get_connection_info(self) -> ConnectionInfo: # return an empty connection info object for now return ConnectionInfo() + @logger.catch(exclude=(HTTPException,)) async def login(self, i: LoginInput) -> Dict[str, str]: - matches = secrets.compare_digest(i.password, config("login_password", cast=str)) + # https://github.com/fusion44/blitz_api/issues/255 + matches = secrets.compare_digest( + i.password, config("BAPI_NATIVE_LOGIN_PASSWORD", cast=str) + ) if matches: return sign_jwt() @@ -72,75 +80,92 @@ class NativePythonSystem(SystemBase): status.HTTP_401_UNAUTHORIZED, detail="Password is incorrect" ) + @logger.catch(exclude=(HTTPException,)) async def change_password(self, type: str, old_password: str, new_password: str): raise NotImplementedError() + @logger.catch(exclude=(HTTPException,)) async def get_debug_logs_raw(self) -> RawDebugLogData: raise NotImplementedError() + @logger.catch(exclude=(HTTPException,)) async def get_hardware_info(self) -> map: info = {} - info["cpu_overall_percent"] = psutil.cpu_percent(interval=_CPU_AVG_PERIOD) - info["cpu_per_cpu_percent"] = psutil.cpu_percent( - interval=_CPU_AVG_PERIOD, percpu=True - ) + try: + info["cpu_overall_percent"] = psutil.cpu_percent(interval=_CPU_AVG_PERIOD) + info["cpu_per_cpu_percent"] = psutil.cpu_percent( + interval=_CPU_AVG_PERIOD, percpu=True + ) - v = psutil.virtual_memory() - info["vram_total_bytes"] = v.total - info["vram_available_bytes"] = v.available - info["vram_used_bytes"] = v.used - info["vram_usage_percent"] = v.percent + v = psutil.virtual_memory() + info["vram_total_bytes"] = v.total + info["vram_available_bytes"] = v.available + info["vram_used_bytes"] = v.used + info["vram_usage_percent"] = v.percent - s = psutil.swap_memory() - info["swap_ram_total_bytes"] = s.total - info["swap_used_bytes"] = s.used - info["swap_usage_bytes"] = s.percent + s = psutil.swap_memory() + info["swap_ram_total_bytes"] = s.total + info["swap_used_bytes"] = s.used + info["swap_usage_bytes"] = s.percent - info["temperatures_celsius"] = psutil.sensors_temperatures() - info["boot_time_timestamp"] = psutil.boot_time() + info["temperatures_celsius"] = psutil.sensors_temperatures() + info["boot_time_timestamp"] = psutil.boot_time() - disk_io = psutil.disk_io_counters() - info["disk_io_read_count"] = disk_io.read_count - info["disk_io_write_count"] = disk_io.write_count - info["disk_io_read_bytes"] = disk_io.read_bytes - info["disk_io_write_bytes"] = disk_io.write_bytes + disk_io = psutil.disk_io_counters() + info["disk_io_read_count"] = disk_io.read_count + info["disk_io_write_count"] = disk_io.write_count + info["disk_io_read_bytes"] = disk_io.read_bytes + info["disk_io_write_bytes"] = disk_io.write_bytes - disks = [] - partitions = psutil.disk_partitions() - for partition in partitions: - p = {} - p["device"] = partition.device - p["mountpoint"] = partition.mountpoint - p["filesystem_type"] = partition.fstype + disks = [] + partitions = psutil.disk_partitions() + for partition in partitions: + p = {} + p["device"] = partition.device + p["mountpoint"] = partition.mountpoint + p["filesystem_type"] = partition.fstype - try: - usage = psutil.disk_usage(partition.mountpoint) - p["partition_total_bytes"] = usage.total - p["partition_used_bytes"] = usage.used - p["partition_free_bytes"] = usage.free - p["partition_percent"] = usage.percent - except PermissionError: - continue - disks.append(p) - info["disks"] = disks + try: + usage = psutil.disk_usage(partition.mountpoint) + p["partition_total_bytes"] = usage.total + p["partition_used_bytes"] = usage.used + p["partition_free_bytes"] = usage.free + p["partition_percent"] = usage.percent + except PermissionError: + continue + disks.append(p) + info["disks"] = disks - nets = [] - addresses = psutil.net_if_addrs() - for name, address in addresses.items(): - net = {} - nets.append(net) - net["interface_name"] = name - for a in address: - if str(a.family) == "AddressFamily.AF_INET": - net["address"] = a.address - elif str(a.family) == "AddressFamily.AF_PACKET": - net["mac_address"] = a.address + nets = [] + addresses = psutil.net_if_addrs() + for name, address in addresses.items(): + net = {} + nets.append(net) + net["interface_name"] = name + for a in address: + if str(a.family) == "AddressFamily.AF_INET": + net["address"] = a.address + elif str(a.family) == "AddressFamily.AF_PACKET": + net["mac_address"] = a.address - net_io = psutil.net_io_counters() - info["networks"] = nets - info["networks_bytes_sent"] = net_io.bytes_sent - info["networks_bytes_received"] = net_io.bytes_recv + net_io = psutil.net_io_counters() + info["networks"] = nets + info["networks_bytes_sent"] = net_io.bytes_sent + info["networks_bytes_received"] = net_io.bytes_recv + + except FileNotFoundError: + logger.warning("Unable to access /proc/stat to get CPU stats") + except OSError as e: + logger.warning( + f"""Unable to query system: {e} + Check if the system is hardened against such calls. + For example in Nix you must not harden the following: + ProtectProc = "invisible"; // to get HW info + ProcSubset = "pid"; // to get HW info + RestrictAddressFamilies = "AF_UNIX AF_INET AF_INET6"; // to get network info + """ + ) return info diff --git a/app/system/impl/raspiblitz.py b/app/system/impl/raspiblitz.py index 91dcae0..18e3985 100644 --- a/app/system/impl/raspiblitz.py +++ b/app/system/impl/raspiblitz.py @@ -4,9 +4,9 @@ import os import time from typing import Dict -from decouple import config from fastapi import HTTPException, status +from app.api.config import config from app.api.constants import API_VERSION from app.api.utils import ( SSE, @@ -31,7 +31,7 @@ from app.system.models import ( _HW_INFO_YIELD_TIME = 2 -SHELL_SCRIPT_PATH = config("shell_script_path") +SHELL_SCRIPT_PATH = config("BAPI_RB_SHELL_SCRIPT_PATH") GET_DEBUG_LOG_SCRIPT = os.path.join( SHELL_SCRIPT_PATH, "config.scripts", "blitz.debug.sh" ) diff --git a/app/system/models.py b/app/system/models.py index 7a150a5..2f4a9b1 100644 --- a/app/system/models.py +++ b/app/system/models.py @@ -1,11 +1,11 @@ from enum import Enum from typing import Optional -from decouple import config from fastapi import Query from pydantic import BaseModel from pydantic.types import constr +from app.api.config import config from app.system.docs import get_debug_data_sample_str @@ -23,7 +23,7 @@ class APIPlatform(str, Enum): @staticmethod def get_current(): - p = config("platform", default="raspiblitz") + p = config("BAPI_PLATFORM", default="raspiblitz") if p == "raspiblitz": return APIPlatform.RASPIBLITZ elif p == "native_python": @@ -31,6 +31,17 @@ class APIPlatform(str, Enum): else: return APIPlatform.UNKNOWN + @staticmethod + def values_as_list(): + """ + Returns a list of all supported platform values. + + Returns: + list: A list of supported platform values. + """ + + return [e.value for e in APIPlatform if e.value != "unknown"] + class SystemInfo(BaseModel): alias: str = Query("", description="Name of the node (same as Lightning alias)") diff --git a/app/system/service.py b/app/system/service.py index 3b7e48b..f002492 100644 --- a/app/system/service.py +++ b/app/system/service.py @@ -1,9 +1,9 @@ import asyncio from typing import Dict, Optional -from decouple import config from fastapi import HTTPException, Request, status +from app.api.config import config from app.api.utils import SSE, broadcast_sse_msg from app.system.models import ( APIPlatform, @@ -14,11 +14,15 @@ from app.system.models import ( SystemInfo, ) -PLATFORM = config("platform", default=APIPlatform.RASPIBLITZ) +PLATFORM = config("BAPI_PLATFORM", default=APIPlatform.RASPIBLITZ) if PLATFORM == APIPlatform.RASPIBLITZ: from app.system.impl.raspiblitz import RaspiBlitzSystem as System elif PLATFORM == APIPlatform.NATIVE_PYTHON: from app.system.impl.native_python import NativePythonSystem as System +else: + raise RuntimeError( + f"Unsupported platform '{PLATFORM}'. Options: {APIPlatform.values_as_list()}" + ) system = System()