From ab0a8f25753b9ef8ddf05f57505dcb8f56a4468f Mon Sep 17 00:00:00 2001 From: Stefan Stammberger Date: Wed, 15 Dec 2021 21:11:46 +0100 Subject: [PATCH 1/2] feat: fetch hardware data via Redis To keep Blitz API platform independent two code paths have been implemented. A new "platform" .env variable must be set if running on a raspiblitz to fetch hardware data from Redis. See the comments in .env_sample for further explanation. --- .env_sample | 12 +++ .../hardware_impl/native_python.py | 74 ++++++++++++++ app/repositories/hardware_impl/raspiblitz.py | 63 ++++++++++++ app/repositories/system.py | 97 +++++-------------- app/routers/system.py | 4 +- 5 files changed, 174 insertions(+), 76 deletions(-) create mode 100644 app/repositories/hardware_impl/native_python.py create mode 100644 app/repositories/hardware_impl/raspiblitz.py diff --git a/.env_sample b/.env_sample index e1f846f..6059f39 100644 --- a/.env_sample +++ b/.env_sample @@ -11,17 +11,29 @@ login_password=12345678 # JWT token. # enable_local_cookie_auth = false +# Platform tells the backend on what kind of system it is running on. +# Different platforms might use different data sources and might yield +# different kinds of data. E.g.: If set to "raspiblitz",then hardware +# information will be fetched from Redis instead of the native python +# implementation and the returned data is different. +# supported values: raspiblitz, native_python +# default: native_python +# platform=native_python + # Amount of seconds the app will wait until it'll # send another hardware update +# only applies when platform=native_python gather_hw_info_interval = 2 # Amount of seconds the app will gather CPU usage data for each update # The resulting CPU usage is averaged over this period of time. # To get realistic results at least 0.1 seconds is recommended # must be less than and not equal to gather_hw_info_interval +# only applies when platform=native_python cpu_usage_averaging_period = 0.5 # Poll interval in seconds to gather lightning information and push it via SSE +# only applies when platform=native_python gather_ln_info_interval = 5.0 # Path to the shell script root folder diff --git a/app/repositories/hardware_impl/native_python.py b/app/repositories/hardware_impl/native_python.py new file mode 100644 index 0000000..73347f4 --- /dev/null +++ b/app/repositories/hardware_impl/native_python.py @@ -0,0 +1,74 @@ +import psutil +from decouple import config + + +SLEEP_TIME = config("gather_hw_info_interval", default=2, cast=float) +CPU_AVG_PERIOD = config("cpu_usage_averaging_period", default=0.5, cast=float) +HW_INFO_YIELD_TIME = SLEEP_TIME + CPU_AVG_PERIOD + + +async def get_hardware_info_impl() -> map: + info = {} + + info["cpu_overall_percent"] = psutil.cpu_percent(interval=CPU_AVG_PERIOD) + info["cpu_per_cpu_percent"] = psutil.cpu_percent( + interval=CPU_AVG_PERIOD, percpu=True + ) + + v = psutil.virtual_memory() + info["vram_total_bytes"] = v.total + info["vram_available_bytes"] = v.available + info["vram_used_bytes"] = v.used + info["vram_usage_percent"] = v.percent + + s = psutil.swap_memory() + info["swap_ram_total_bytes"] = s.total + info["swap_used_bytes"] = s.used + info["swap_usage_bytes"] = s.percent + + info["temperatures_celsius"] = psutil.sensors_temperatures() + info["boot_time_timestamp"] = psutil.boot_time() + + disk_io = psutil.disk_io_counters() + info["disk_io_read_count"] = disk_io.read_count + info["disk_io_write_count"] = disk_io.write_count + info["disk_io_read_bytes"] = disk_io.read_bytes + info["disk_io_write_bytes"] = disk_io.write_bytes + + disks = [] + partitions = psutil.disk_partitions() + for partition in partitions: + p = {} + p["device"] = partition.device + p["mountpoint"] = partition.mountpoint + p["filesystem_type"] = partition.fstype + + try: + usage = psutil.disk_usage(partition.mountpoint) + p["partition_total_bytes"] = usage.total + p["partition_used_bytes"] = usage.used + p["partition_free_bytes"] = usage.free + p["partition_percent"] = usage.percent + except PermissionError: + continue + disks.append(p) + info["disks"] = disks + + nets = [] + addresses = psutil.net_if_addrs() + for name, address in addresses.items(): + net = {} + nets.append(net) + net["interface_name"] = name + for a in address: + if str(a.family) == "AddressFamily.AF_INET": + net["address"] = a.address + elif str(a.family) == "AddressFamily.AF_PACKET": + net["mac_address"] = a.address + + net_io = psutil.net_io_counters() + info["networks"] = nets + info["networks_bytes_sent"] = net_io.bytes_sent + info["networks_bytes_received"] = net_io.bytes_recv + + return info diff --git a/app/repositories/hardware_impl/raspiblitz.py b/app/repositories/hardware_impl/raspiblitz.py new file mode 100644 index 0000000..b9c2cf6 --- /dev/null +++ b/app/repositories/hardware_impl/raspiblitz.py @@ -0,0 +1,63 @@ +import time + +HW_INFO_YIELD_TIME = 2 + + +from fastapi_plugins import redis_plugin as r + + +async def _redis_get(key: str) -> str: + v = await r.redis.get(key) + return v.decode("utf-8") + + +async def get_hardware_info_impl() -> map: + info = {} + + loads = (await _redis_get("system_cpu_load")).split(",") + iloads = [] + total = 0 + for l in loads: + value = float(l) + total += value + iloads.append(value) + info["cpu_overall_percent"] = total / len(loads) + info["cpu_per_cpu_percent"] = iloads + + info["vram_total_bytes"] = int(await _redis_get("system_ram_mb")) * 1000 * 1000 + + info["vram_available_bytes"] = ( + int(await _redis_get("system_ram_available_mb")) * 1000 * 1000 + ) + + info["vram_used_bytes"] = info["vram_total_bytes"] - info["vram_available_bytes"] + info["vram_usage_percent"] = (100 / info["vram_total_bytes"]) * info[ + "vram_used_bytes" + ] + + info["temperatures_celsius"] = { + "system_temp": float(await _redis_get("system_temp_celsius")), + "coretemp": [], + } + + now = time.time() + boot = float(await _redis_get("system_up")) + info["boot_time_timestamp"] = now - boot + + total = int(await _redis_get("hdd_capacity_bytes")) + free = int(await _redis_get("hdd_free_bytes")) + info["hdd"] = { + "hdd_capacity_bytes": total, + "hdd_free_bytes": free, + "hdd_free_percent": (100 / total) * free, + } + + info["networks"] = { + "public_ip": await _redis_get("publicIP"), + "internet_online": await _redis_get("internet_online"), + "tor_web_addr": await _redis_get("tor_web_addr"), + "internet_localip=": await _redis_get("internet_localip"), + "internet_localiprange": await _redis_get("internet_localiprange"), + } + + return info diff --git a/app/repositories/system.py b/app/repositories/system.py index ac28155..03b1dd6 100644 --- a/app/repositories/system.py +++ b/app/repositories/system.py @@ -1,16 +1,28 @@ import asyncio from os import path -import psutil from app.models.system import RawDebugLogData, SystemInfo from app.repositories.lightning import get_ln_info from app.utils import SSE, send_sse_message from decouple import config from fastapi import HTTPException, Request, status -SLEEP_TIME = config("gather_hw_info_interval", default=2, cast=float) -CPU_AVG_PERIOD = config("cpu_usage_averaging_period", default=0.5, cast=float) -HW_INFO_YIELD_TIME = SLEEP_TIME + CPU_AVG_PERIOD +PLATFORM = config("platform") +if PLATFORM == None: + PLATFORM = "native_python" + +if PLATFORM == "raspiblitz": + from app.repositories.hardware_impl.raspiblitz import ( + HW_INFO_YIELD_TIME, + get_hardware_info_impl, + ) +elif PLATFORM == "native_python": + from app.repositories.hardware_impl.native_python import ( + HW_INFO_YIELD_TIME, + get_hardware_info_impl, + ) +else: + raise RuntimeError(f"Unknown platform {PLATFORM}") SHELL_SCRIPT_PATH = config("shell_script_path") GET_DEBUG_LOG_SCRIPT = path.join(SHELL_SCRIPT_PATH, "config.scripts", "blitz.debug.sh") @@ -37,80 +49,17 @@ async def get_system_info() -> SystemInfo: raise HTTPException(status.HTTP_501_NOT_IMPLEMENTED, detail=r.args[0]) +async def get_hardware_info() -> map: + return await get_hardware_info_impl() + + async def subscribe_hardware_info(request: Request): while True: if await request.is_disconnected(): # stop if client disconnects break - yield get_hardware_info() - await asyncio.sleep(SLEEP_TIME) - - -def get_hardware_info() -> map: - info = {} - - info["cpu_overall_percent"] = psutil.cpu_percent(interval=CPU_AVG_PERIOD) - info["cpu_per_cpu_percent"] = psutil.cpu_percent( - interval=CPU_AVG_PERIOD, percpu=True - ) - - v = psutil.virtual_memory() - info["vram_total_bytes"] = v.total - info["vram_available_bytes"] = v.available - info["vram_used_bytes"] = v.used - info["vram_usage_percent"] = v.percent - - s = psutil.swap_memory() - info["swap_ram_total_bytes"] = s.total - info["swap_used_bytes"] = s.used - info["swap_usage_bytes"] = s.percent - - info["temperatures_celsius"] = psutil.sensors_temperatures() - info["boot_time_timestamp"] = psutil.boot_time() - - disk_io = psutil.disk_io_counters() - info["disk_io_read_count"] = disk_io.read_count - info["disk_io_write_count"] = disk_io.write_count - info["disk_io_read_bytes"] = disk_io.read_bytes - info["disk_io_write_bytes"] = disk_io.write_bytes - - disks = [] - partitions = psutil.disk_partitions() - for partition in partitions: - p = {} - p["device"] = partition.device - p["mountpoint"] = partition.mountpoint - p["filesystem_type"] = partition.fstype - - try: - usage = psutil.disk_usage(partition.mountpoint) - p["partition_total_bytes"] = usage.total - p["partition_used_bytes"] = usage.used - p["partition_free_bytes"] = usage.free - p["partition_percent"] = usage.percent - except PermissionError: - continue - disks.append(p) - info["disks"] = disks - - nets = [] - addresses = psutil.net_if_addrs() - for name, address in addresses.items(): - net = {} - nets.append(net) - net["interface_name"] = name - for a in address: - if str(a.family) == "AddressFamily.AF_INET": - net["address"] = a.address - elif str(a.family) == "AddressFamily.AF_PACKET": - net["mac_address"] = a.address - - net_io = psutil.net_io_counters() - info["networks"] = nets - info["networks_bytes_sent"] = net_io.bytes_sent - info["networks_bytes_received"] = net_io.bytes_recv - - return info + yield await get_hardware_info() + await asyncio.sleep(HW_INFO_YIELD_TIME) async def get_debug_logs_raw() -> RawDebugLogData: @@ -144,7 +93,7 @@ f"[{cmd!r} exited with {proc.returncode}]"\n async def _handle_gather_hardware_info(): last_info = {} while True: - info = get_hardware_info() + info = await get_hardware_info() if last_info != info: await send_sse_message(SSE.HARDWARE_INFO, info) last_info = info diff --git a/app/routers/system.py b/app/routers/system.py index 4987c33..32b745e 100644 --- a/app/routers/system.py +++ b/app/routers/system.py @@ -80,8 +80,8 @@ async def get_system_info_path(): dependencies=[Depends(JWTBearer())], status_code=status.HTTP_200_OK, ) -def hw_info() -> map: - return get_hardware_info() +async def hw_info() -> map: + return await get_hardware_info() @router.get( From 228ae2100aa9ce90d868e1ec86e7897723f2bc5a Mon Sep 17 00:00:00 2001 From: Stefan Stammberger Date: Sat, 18 Dec 2021 13:27:47 +0100 Subject: [PATCH 2/2] feat: set raspiblitz as default platform Rationale: No need to change RaspiBlitz install scripts --- .env_sample | 4 ++-- app/repositories/system.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.env_sample b/.env_sample index 6059f39..df8cd3d 100644 --- a/.env_sample +++ b/.env_sample @@ -17,8 +17,8 @@ login_password=12345678 # information will be fetched from Redis instead of the native python # implementation and the returned data is different. # supported values: raspiblitz, native_python -# default: native_python -# platform=native_python +# default: raspiblitz +# platform=raspiblitz # Amount of seconds the app will wait until it'll # send another hardware update diff --git a/app/repositories/system.py b/app/repositories/system.py index 03b1dd6..f47e73b 100644 --- a/app/repositories/system.py +++ b/app/repositories/system.py @@ -9,7 +9,7 @@ from fastapi import HTTPException, Request, status PLATFORM = config("platform") if PLATFORM == None: - PLATFORM = "native_python" + PLATFORM = "raspiblitz" if PLATFORM == "raspiblitz": from app.repositories.hardware_impl.raspiblitz import (