GlcoinCore/cln-plugins
1
0
Fork 0
mirror of https://github.com/lightningd/plugins.git synced 2026-08-13 12:33:19 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

format python files with ruff

Browse source
This commit is contained in:
daywalker90 2024-07-01 16:34:29 +02:00 • committed by Chris Guida
parent 7b3ad06a59
commit 1d3356c2e4
35 changed files with 1721 additions and 1172 deletions
Download patch file Download diff file Expand all files Collapse all files

211
.ci/test.py
View file

@ -8,24 +8,23 @@ import json
from itertools import chain
from pathlib import Path
from utils import (Plugin, configure_git, enumerate_plugins)
from utils import Plugin, configure_git, enumerate_plugins
logging.basicConfig(stream=sys.stdout, level=logging.DEBUG)
global_dependencies = [
'pytest',
'pytest-xdist',
'pytest-timeout',
"pytest",
"pytest-xdist",
"pytest-timeout",
]
pip_opts = ['-qq']
pip_opts = ["-qq"]
def prepare_env(p: Plugin, directory: Path, env: dict, workflow: str) -> bool:
""" Returns whether we can run at all. Raises error if preparing failed.
"""
subprocess.check_call(['python3', '-m', 'venv', '--clear', directory])
os.environ['PATH'] += f":{directory}"
"""Returns whether we can run at all. Raises error if preparing failed."""
subprocess.check_call(["python3", "-m", "venv", "--clear", directory])
os.environ["PATH"] += f":{directory}"
if p.framework == "pip":
return prepare_env_pip(p, directory, workflow)
@ -40,21 +39,21 @@ def prepare_env(p: Plugin, directory: Path, env: dict, workflow: str) -> bool:
def prepare_env_poetry(p: Plugin, directory: Path) -> bool:
logging.info("Installing a new poetry virtualenv")
pip3 = directory / 'bin' / 'pip3'
poetry = directory / 'bin' / 'poetry'
python3 = directory / 'bin' / 'python3'
pip3 = directory / "bin" / "pip3"
poetry = directory / "bin" / "poetry"
python3 = directory / "bin" / "python3"
subprocess.check_call(['which', 'python3'])
subprocess.check_call(["which", "python3"])
subprocess.check_call([
pip3, 'install', '-U', *pip_opts, 'pip', 'wheel', 'poetry'
], cwd=p.path.parent)
subprocess.check_call(
[pip3, "install", "-U", *pip_opts, "pip", "wheel", "poetry"], cwd=p.path.parent
)
# Install pytest (eventually we'd want plugin authors to include
# it in their requirements-dev.txt, but for now let's help them a
# bit).
subprocess.check_call(
[pip3, 'install', '-U', '-qq', *global_dependencies],
[pip3, "install", "-U", "-qq", *global_dependencies],
stderr=subprocess.STDOUT,
)
@ -64,23 +63,41 @@ def prepare_env_poetry(p: Plugin, directory: Path) -> bool:
logging.info(f"Using poetry at {poetry} ({python3}) to run tests in {workdir}")
# Now we can proceed with the actual implementation
logging.info(f"Exporting poetry {poetry} dependencies from {p.details['pyproject']}")
subprocess.check_call([
poetry, 'export', '--with=dev', '--without-hashes', '-f', 'requirements.txt',
'--output', 'requirements.txt'
], cwd=workdir)
logging.info(
f"Exporting poetry {poetry} dependencies from {p.details['pyproject']}"
)
subprocess.check_call(
[
poetry,
"export",
"--with=dev",
"--without-hashes",
"-f",
"requirements.txt",
"--output",
"requirements.txt",
],
cwd=workdir,
)
subprocess.check_call([
pip3, 'install', *pip_opts, '-r', str(workdir) + "/requirements.txt",
], stderr=subprocess.STDOUT)
subprocess.check_call(
[
pip3,
"install",
*pip_opts,
"-r",
str(workdir) + "/requirements.txt",
],
stderr=subprocess.STDOUT,
)
subprocess.check_call([pip3, 'freeze'])
subprocess.check_call([pip3, "freeze"])
return True
def prepare_env_pip(p: Plugin, directory: Path, workflow: str) -> bool:
print("Installing a new pip virtualenv")
pip_path = directory / 'bin' / 'pip3'
pip_path = directory / "bin" / "pip3"
if workflow == "nightly":
install_dev_pyln_testing(pip_path)
@ -90,24 +107,24 @@ def prepare_env_pip(p: Plugin, directory: Path, workflow: str) -> bool:
# Now install all the requirements
print(f"Installing requirements from {p.details['requirements']}")
subprocess.check_call(
[pip_path, 'install', *pip_opts, '-r', p.details['requirements']],
[pip_path, "install", *pip_opts, "-r", p.details["requirements"]],
stderr=subprocess.STDOUT,
)
if p.details['devrequirements'].exists():
if p.details["devrequirements"].exists():
print(f"Installing requirements from {p.details['devrequirements']}")
subprocess.check_call(
[pip_path, 'install', *pip_opts, '-r', p.details['devrequirements']],
[pip_path, "install", *pip_opts, "-r", p.details["devrequirements"]],
stderr=subprocess.STDOUT,
)
subprocess.check_call([pip_path, 'freeze'])
subprocess.check_call([pip_path, "freeze"])
return True
def prepare_generic(p: Plugin, directory: Path, env: dict, workflow: str) -> bool:
print("Installing a new generic virtualenv")
pip_path = directory / 'bin' / 'pip3'
pip_path = directory / "bin" / "pip3"
if workflow == "nightly":
install_dev_pyln_testing(pip_path)
@ -115,49 +132,51 @@ def prepare_generic(p: Plugin, directory: Path, env: dict, workflow: str) -> boo
install_pyln_testing(pip_path)
# Now install all the requirements
if p.details['requirements'].exists():
if p.details["requirements"].exists():
print(f"Installing requirements from {p.details['requirements']}")
subprocess.check_call(
[pip_path, 'install', '-U', *pip_opts, '-r', p.details['requirements']],
[pip_path, "install", "-U", *pip_opts, "-r", p.details["requirements"]],
stderr=subprocess.STDOUT,
)
if p.details['setup'].exists():
if p.details["setup"].exists():
print(f"Running setup script from {p.details['setup']}")
subprocess.check_call(
['bash', p.details['setup'], f'TEST_DIR={directory}'],
["bash", p.details["setup"], f"TEST_DIR={directory}"],
env=env,
stderr=subprocess.STDOUT,
)
subprocess.check_call([pip_path, 'freeze'])
subprocess.check_call([pip_path, "freeze"])
return True
def install_pyln_testing(pip_path):
# Many plugins only implicitly depend on pyln-testing, so let's help them
cln_path = os.environ['CLN_PATH']
cln_path = os.environ["CLN_PATH"]
# Install pytest (eventually we'd want plugin authors to include
# it in their requirements-dev.txt, but for now let's help them a
# bit).
subprocess.check_call(
[pip_path, 'install', *pip_opts, *global_dependencies],
[pip_path, "install", *pip_opts, *global_dependencies],
stderr=subprocess.STDOUT,
)
subprocess.check_call(
[pip_path, 'install', '-U', *pip_opts, 'pip', 'wheel'],
[pip_path, "install", "-U", *pip_opts, "pip", "wheel"],
stderr=subprocess.STDOUT,
)
subprocess.check_call(
[
pip_path, 'install', *pip_opts,
pip_path,
"install",
*pip_opts,
cln_path + "/contrib/pyln-client",
cln_path + "/contrib/pyln-testing",
"MarkupSafe>=2.0",
'itsdangerous>=2.0'
"itsdangerous>=2.0",
],
stderr=subprocess.STDOUT,
)
@ -165,11 +184,18 @@ def install_pyln_testing(pip_path):
def install_dev_pyln_testing(pip_path):
# Many plugins only implicitly depend on pyln-testing, so let's help them
cln_path = os.environ['CLN_PATH']
cln_path = os.environ["CLN_PATH"]
subprocess.check_call([
pip_path, 'install', *pip_opts, '-r', cln_path + "/requirements.txt",
], stderr=subprocess.STDOUT)
subprocess.check_call(
[
pip_path,
"install",
*pip_opts,
"-r",
cln_path + "/requirements.txt",
],
stderr=subprocess.STDOUT,
)
def run_one(p: Plugin, workflow: str) -> bool:
@ -179,24 +205,30 @@ def run_one(p: Plugin, workflow: str) -> bool:
print("No test files found, skipping plugin {p.name}".format(p=p))
return True
print("Found {ctestfiles} test files, creating virtualenv and running tests".format(ctestfiles=len(p.testfiles)))
print(
"Found {ctestfiles} test files, creating virtualenv and running tests".format(
ctestfiles=len(p.testfiles)
)
)
print("::group::{p.name}".format(p=p))
# Create a virtual env
vdir = tempfile.TemporaryDirectory()
vpath = Path(vdir.name)
bin_path = vpath / 'bin'
pytest_path = vpath / 'bin' / 'pytest'
bin_path = vpath / "bin"
pytest_path = vpath / "bin" / "pytest"
env = os.environ.copy()
env.update({
# Need to customize PATH so lightningd can find the correct python3
'PATH': "{}:{}".format(bin_path, os.environ['PATH']),
# Some plugins require a valid locale to be set
'LC_ALL': 'C.UTF-8',
'LANG': 'C.UTF-8',
})
env.update(
{
# Need to customize PATH so lightningd can find the correct python3
"PATH": "{}:{}".format(bin_path, os.environ["PATH"]),
# Some plugins require a valid locale to be set
"LC_ALL": "C.UTF-8",
"LANG": "C.UTF-8",
}
)
try:
if not prepare_env(p, vpath, env, workflow):
@ -211,11 +243,11 @@ def run_one(p: Plugin, workflow: str) -> bool:
cmd = [
str(pytest_path),
'-vvv',
'--timeout=600',
'--timeout-method=thread',
'--color=yes',
'-n=5',
"-vvv",
"--timeout=600",
"--timeout-method=thread",
"--color=yes",
"-n=5",
]
logging.info(f"Running `{' '.join(cmd)}` in directory {p.path.resolve()}")
@ -254,11 +286,11 @@ def push_gather_data(data: dict, workflow: str, python_version: str):
subprocess.run(["git", "checkout", "badges"])
filenames_to_add = []
for plugin_name, result in data.items():
filename = write_gather_data_file(
plugin_name, result, workflow, python_version
)
filename = write_gather_data_file(plugin_name, result, workflow, python_version)
filenames_to_add.append(filename)
output = subprocess.check_output(list(chain(["git", "add", "-v"], filenames_to_add))).decode("utf-8")
output = subprocess.check_output(
list(chain(["git", "add", "-v"], filenames_to_add))
).decode("utf-8")
print(f"output from git add: {output}")
if output != "":
output = subprocess.check_output(
@ -272,9 +304,9 @@ def push_gather_data(data: dict, workflow: str, python_version: str):
print(f"output from git commit: {output}")
for _ in range(10):
subprocess.run(["git", "pull", "--rebase"])
output = subprocess.run(["git", "push", "origin", "badges"],
capture_output=True,
text=True)
output = subprocess.run(
["git", "push", "origin", "badges"], capture_output=True, text=True
)
if output.returncode == 0:
print("Push successful")
break
@ -287,7 +319,9 @@ def push_gather_data(data: dict, workflow: str, python_version: str):
print("Done.")
def write_gather_data_file(plugin_name: str, result, workflow: str, python_version: str) -> str:
def write_gather_data_file(
plugin_name: str, result, workflow: str, python_version: str
) -> str:
_dir = f".badges/gather_data/{workflow}/{plugin_name}"
filename = os.path.join(_dir, f"python{python_version}.txt")
os.makedirs(_dir, exist_ok=True)
@ -307,11 +341,11 @@ def gather_old_failures(old_failures: list, workflow: str):
directory = ".badges"
for filename in os.listdir(directory):
if filename.endswith(f'_{workflow}.json'):
if filename.endswith(f"_{workflow}.json"):
file_path = os.path.join(directory, filename)
plugin_name = filename.rsplit(f'_{workflow}.json', 1)[0]
plugin_name = filename.rsplit(f"_{workflow}.json", 1)[0]
with open(file_path, 'r') as file:
with open(file_path, "r") as file:
data = json.load(file)
if data["color"] == "red":
old_failures.append(plugin_name)
@ -319,19 +353,26 @@ def gather_old_failures(old_failures: list, workflow: str):
print(f"Old failures: {old_failures}")
print("Done.")
def run_all(workflow: str, python_version: str, update_badges: bool, plugin_names: list):
root_path = subprocess.check_output([
'git',
'rev-parse',
'--show-toplevel'
]).decode('ASCII').strip()
def run_all(
workflow: str, python_version: str, update_badges: bool, plugin_names: list
):
root_path = (
subprocess.check_output(["git", "rev-parse", "--show-toplevel"])
.decode("ASCII")
.strip()
)
root = Path(root_path)
plugins = list(enumerate_plugins(root))
if plugin_names != []:
plugins = [p for p in plugins if p.name in plugin_names]
print("Testing the following plugins: {names}".format(names=[p.name for p in plugins]))
print(
"Testing the following plugins: {names}".format(
names=[p.name for p in plugins]
)
)
else:
print("Testing all plugins in {root}".format(root=root))
@ -343,7 +384,9 @@ def run_all(workflow: str, python_version: str, update_badges: bool, plugin_name
gather_old_failures(old_failures, workflow)
if update_badges:
push_gather_data(collect_gather_data(results, success), workflow, python_version)
push_gather_data(
collect_gather_data(results, success), workflow, python_version
)
if not success:
print("The following tests failed:")
@ -361,10 +404,14 @@ def run_all(workflow: str, python_version: str, update_badges: bool, plugin_name
if __name__ == "__main__":
import argparse
parser = argparse.ArgumentParser(description='Plugins test script')
parser = argparse.ArgumentParser(description="Plugins test script")
parser.add_argument("workflow", type=str, help="Name of the GitHub workflow")
parser.add_argument("python_version", type=str, help="Python version")
parser.add_argument("--update-badges", action='store_true', help="Whether badges data should be updated")
parser.add_argument(
"--update-badges",
action="store_true",
help="Whether badges data should be updated",
)
parser.add_argument("plugins", nargs="*", default=[], help="List of plugins")
args = parser.parse_args()

34
.ci/update_badges.py
View file

@ -7,7 +7,9 @@ from pathlib import Path
from utils import configure_git, enumerate_plugins
def update_and_commit_badge(plugin_name: str, passed: bool, workflow: str, has_tests: bool) -> bool:
def update_and_commit_badge(
plugin_name: str, passed: bool, workflow: str, has_tests: bool
) -> bool:
json_data = {"schemaVersion": 1, "label": "", "message": "✔", "color": "green"}
if not passed:
json_data.update({"message": "✗", "color": "red"})
@ -34,16 +36,18 @@ def update_and_commit_badge(plugin_name: str, passed: bool, workflow: str, has_t
def cleanup_old_results(plugin_name: str, file: Path) -> bool:
os.remove(file)
print(f"Removed deprecated result {file.name} for {plugin_name}, we no longer test for this version!")
print(
f"Removed deprecated result {file.name} for {plugin_name}, we no longer test for this version!"
)
subprocess.run(["git", "add", "-v", file])
subprocess.run(
[
"git",
"commit",
"-m",
f'Remove deprecated result {file.name}',
]
)
[
"git",
"commit",
"-m",
f"Remove deprecated result {file.name}",
]
)
return True
@ -96,9 +100,9 @@ def push_badges_data(workflow: str, python_versions_tested: list):
if any_changes:
for _ in range(10):
subprocess.run(["git", "pull", "--rebase"])
output = subprocess.run(["git", "push", "origin", "badges"],
capture_output=True,
text=True)
output = subprocess.run(
["git", "push", "origin", "badges"], capture_output=True, text=True
)
if output.returncode == 0:
print("Push successful")
break
@ -117,7 +121,11 @@ if __name__ == "__main__":
parser = argparse.ArgumentParser(description="Plugins completion script")
parser.add_argument("workflow", type=str, help="Name of the GitHub workflow")
parser.add_argument(
"python_versions_tested", nargs="*", type=str, default=[], help="Python versions tested"
"python_versions_tested",
nargs="*",
type=str,
default=[],
help="Python versions tested",
)
args = parser.parse_args()

10
.ci/utils.py
View file

@ -36,7 +36,15 @@ def get_testfiles(p: Path) -> List[PosixPath]:
test_files = []
for x in p.iterdir():
if x.is_dir() and x.name == "tests":
test_files.extend([y for y in x.iterdir() if y.is_file() and y.name.startswith("test_") and y.name.endswith(".py")])
test_files.extend(
[
y
for y in x.iterdir()
if y.is_file()
and y.name.startswith("test_")
and y.name.endswith(".py")
]
)
elif x.is_file() and x.name.startswith("test_") and x.name.endswith(".py"):
test_files.append(x)
return test_files

26
backup/backend.py
View file

@ -11,7 +11,7 @@ import sqlite3
# This is used by the plugin from time to time to allow the backend to compress
# the changelog and forms a new basis for the backup.
# If `Change` contains a snapshot and a transaction, they apply in that order.
Change = namedtuple('Change', ['version', 'snapshot', 'transaction'])
Change = namedtuple("Change", ["version", "snapshot", "transaction"])
class Backend(object):
@ -37,14 +37,11 @@ class Backend(object):
raise NotImplementedError
def initialize(self) -> bool:
"""Set up any resources needed by this backend.
"""
"""Set up any resources needed by this backend."""
raise NotImplementedError
def stream_changes(self) -> Iterator[Change]:
"""Retrieve changes from the backend in order to perform a restore.
"""
"""Retrieve changes from the backend in order to perform a restore."""
raise NotImplementedError
def rewind(self) -> bool:
@ -63,8 +60,7 @@ class Backend(object):
raise NotImplementedError
def compact(self):
"""Apply some incremental changes to the snapshot to reduce our size.
"""
"""Apply some incremental changes to the snapshot to reduce our size."""
raise NotImplementedError
def _db_open(self, dest: str) -> sqlite3.Connection:
@ -75,7 +71,7 @@ class Backend(object):
def _restore_snapshot(self, snapshot: bytes, dest: str):
if os.path.exists(dest):
os.unlink(dest)
with open(dest, 'wb') as f:
with open(dest, "wb") as f:
f.write(snapshot)
self.db = self._db_open(dest)
@ -87,8 +83,12 @@ class Backend(object):
re-inserts the space.
"""
stmt = re.sub(r'reserved_til=([0-9]+)WHERE', r'reserved_til=\1 WHERE', stmt)
stmt = re.sub(r'peer_id=([0-9]+)WHERE channels.id=', r'peer_id=\1 WHERE channels.id=', stmt)
stmt = re.sub(r"reserved_til=([0-9]+)WHERE", r"reserved_til=\1 WHERE", stmt)
stmt = re.sub(
r"peer_id=([0-9]+)WHERE channels.id=",
r"peer_id=\1 WHERE channels.id=",
stmt,
)
return stmt
def _restore_transaction(self, tx: Iterator[str]):
@ -109,9 +109,7 @@ class Backend(object):
if os.path.exists(dest):
if not remove_existing:
raise ValueError(
"Destination for backup restore exists: {dest}".format(
dest=dest
)
"Destination for backup restore exists: {dest}".format(dest=dest)
)
os.unlink(dest)

22
backup/backends.py
View file

@ -1,4 +1,5 @@
'''Create a backend instance based on URI scheme dispatch.'''
"""Create a backend instance based on URI scheme dispatch."""
from typing import Type
from urllib.parse import urlparse
@ -8,10 +9,9 @@ from filebackend import FileBackend
def resolve_backend_class(backend_url):
backend_map: Mapping[str, Type[Backend]] = {
'file': FileBackend,
'socket': SocketBackend,
"file": FileBackend,
"socket": SocketBackend,
}
p = urlparse(backend_url)
backend_cl = backend_map.get(p.scheme, None)
@ -21,13 +21,19 @@ def resolve_backend_class(backend_url):
def get_backend(destination, create=False, require_init=False):
backend_cl = resolve_backend_class(destination)
if backend_cl is None:
raise ValueError("No backend implementation found for {destination}".format(
destination=destination,
))
raise ValueError(
"No backend implementation found for {destination}".format(
destination=destination,
)
)
backend = backend_cl(destination, create=create)
initialized = backend.initialize()
if require_init and not initialized:
kill("Could not initialize the backup {}, please use 'backup-cli' to initialize the backup first.".format(destination))
kill(
"Could not initialize the backup {}, please use 'backup-cli' to initialize the backup first.".format(
destination
)
)
assert backend.version is not None
assert backend.prev_version is not None
return backend

42
backup/backup.py
View file

@ -17,7 +17,7 @@ root.setLevel(logging.INFO)
handler = logging.StreamHandler(sys.stdout)
handler.setLevel(logging.DEBUG)
formatter = logging.Formatter('%(message)s')
formatter = logging.Formatter("%(message)s")
handler.setFormatter(formatter)
root.addHandler(handler)
@ -37,9 +37,11 @@ def check_first_write(plugin, data_version):
"""
backend = plugin.backend
logging.info("Comparing backup version {} versus first write version {}".format(
backend.version, data_version
))
logging.info(
"Comparing backup version {} versus first write version {}".format(
backend.version, data_version
)
)
if backend.version == data_version - 1:
logging.info("Versions match up")
@ -50,13 +52,15 @@ def check_first_write(plugin, data_version):
return True
elif backend.prev_version > data_version - 1:
kill("Core-Lightning seems to have lost some state (failed restore?). Emergency shutdown.")
kill(
"Core-Lightning seems to have lost some state (failed restore?). Emergency shutdown."
)
else:
kill("Backup is out of date, we cannot continue safely. Emergency shutdown.")
@plugin.hook('db_write')
@plugin.hook("db_write")
def on_db_write(writes, data_version, plugin, **kwargs):
change = Change(data_version, None, writes)
if not plugin.initialized:
@ -85,14 +89,14 @@ def compact(plugin):
@plugin.init()
def on_init(options, **kwargs):
dest = options.get('backup-destination', 'null')
if dest != 'null':
dest = options.get("backup-destination", "null")
if dest != "null":
plugin.log(
"The `--backup-destination` option is deprecated and will be "
"removed in future versions of the backup plugin. Please remove "
"it from your configuration. The destination is now determined by "
"the `backup.lock` file in the lightning directory",
level="warn"
level="warn",
)
# IMPORTANT NOTE
@ -109,12 +113,9 @@ def kill(message: str):
# Search for lightningd in my ancestor processes:
procs = [p for p in psutil.Process(os.getpid()).parents()]
for p in procs:
if p.name() != 'lightningd':
if p.name() != "lightningd":
continue
plugin.log("Killing process {name} ({pid})".format(
name=p.name(),
pid=p.pid
))
plugin.log("Killing process {name} ({pid})".format(name=p.name(), pid=p.pid))
p.kill()
# Sleep forever, just in case the master doesn't die on us...
@ -123,8 +124,9 @@ def kill(message: str):
plugin.add_option(
'backup-destination', None,
'UNUSED. Kept for backward compatibility only. Please update your configuration to remove this option.'
"backup-destination",
None,
"UNUSED. Kept for backward compatibility only. Please update your configuration to remove this option.",
)
@ -135,10 +137,10 @@ if __name__ == "__main__":
kill("Could not find backup.lock in the lightning-dir")
try:
d = json.load(open("backup.lock", 'r'))
destination = d['backend_url']
d = json.load(open("backup.lock", "r"))
destination = d["backend_url"]
plugin.backend = get_backend(destination, require_init=True)
plugin.run()
except Exception:
logging.exception('Exception while initializing backup plugin')
kill('Exception while initializing plugin, terminating lightningd')
logging.exception("Exception while initializing backup plugin")
kill("Exception while initializing plugin, terminating lightningd")

95
backup/filebackend.py
View file

@ -18,9 +18,13 @@ class FileBackend(Backend):
self.url = urlparse(self.destination)
if os.path.exists(self.url.path) and create:
raise ValueError("Attempted to create a FileBackend, but file already exists.")
raise ValueError(
"Attempted to create a FileBackend, but file already exists."
)
if not os.path.exists(self.url.path) and not create:
raise ValueError("Attempted to open a FileBackend but file doesn't already exists, use `backup-cli init` to initialize it first.")
raise ValueError(
"Attempted to open a FileBackend but file doesn't already exists, use `backup-cli init` to initialize it first."
)
if create:
# Initialize a new backup file
self.version, self.prev_version = 0, 0
@ -32,12 +36,18 @@ class FileBackend(Backend):
return self.read_metadata()
def write_metadata(self):
blob = struct.pack("!IIQIQQ", 0x01, self.version, self.offsets[0],
self.prev_version, self.offsets[1],
self.version_count)
blob = struct.pack(
"!IIQIQQ",
0x01,
self.version,
self.offsets[0],
self.prev_version,
self.offsets[1],
self.version_count,
)
# Pad the header
blob += b'\x00' * (512 - len(blob))
blob += b"\x00" * (512 - len(blob))
mode = "rb+" if os.path.exists(self.url.path) else "wb+"
with open(self.url.path, mode) as f:
@ -46,31 +56,41 @@ class FileBackend(Backend):
f.flush()
def read_metadata(self):
with open(self.url.path, 'rb') as f:
with open(self.url.path, "rb") as f:
blob = f.read(512)
if len(blob) != 512:
logging.warn("Corrupt FileBackend header, expected 512 bytes, got {} bytes".format(len(blob)))
logging.warn(
"Corrupt FileBackend header, expected 512 bytes, got {} bytes".format(
len(blob)
)
)
return False
file_version, = struct.unpack_from("!I", blob)
(file_version,) = struct.unpack_from("!I", blob)
if file_version != 1:
logging.warn("Unknown FileBackend version {}".format(file_version))
return False
self.version, self.offsets[0], self.prev_version, self.offsets[1], self.version_count, = struct.unpack_from("!IQIQQ", blob, offset=4)
(
self.version,
self.offsets[0],
self.prev_version,
self.offsets[1],
self.version_count,
) = struct.unpack_from("!IQIQQ", blob, offset=4)
return True
def add_change(self, entry: Change) -> bool:
typ = b'\x01' if entry.snapshot is None else b'\x02'
if typ == b'\x01':
payload = b'\x00'.join([t.encode('UTF-8') for t in entry.transaction])
elif typ == b'\x02':
typ = b"\x01" if entry.snapshot is None else b"\x02"
if typ == b"\x01":
payload = b"\x00".join([t.encode("UTF-8") for t in entry.transaction])
elif typ == b"\x02":
payload = entry.snapshot
length = struct.pack("!I", len(payload))
version = struct.pack("!I", entry.version)
with open(self.url.path, 'ab') as f:
with open(self.url.path, "ab") as f:
f.seek(self.offsets[0])
f.write(length)
f.write(version)
@ -100,7 +120,7 @@ class FileBackend(Backend):
def stream_changes(self) -> Iterator[Change]:
self.read_metadata()
version = -1
with open(self.url.path, 'rb') as f:
with open(self.url.path, "rb") as f:
# Skip the header
f.seek(512)
while version < self.version:
@ -110,7 +130,7 @@ class FileBackend(Backend):
yield Change(
version=version,
snapshot=None,
transaction=[t.decode('UTF-8') for t in payload.split(b'\x00')]
transaction=[t.decode("UTF-8") for t in payload.split(b"\x00")],
)
elif typ == 2:
yield Change(version=version, snapshot=payload, transaction=None)
@ -118,7 +138,11 @@ class FileBackend(Backend):
raise ValueError("Unknown FileBackend entry type {}".format(typ))
if version != self.version:
raise ValueError("Versions do not match up: restored version {}, backend version {}".format(version, self.version))
raise ValueError(
"Versions do not match up: restored version {}, backend version {}".format(
version, self.version
)
)
assert version == self.version
def compact(self):
@ -137,9 +161,9 @@ class FileBackend(Backend):
snapshotpath = os.path.join(tmp.name, "lightningd.sqlite3")
stats = {
'before': {
'backupsize': os.stat(self.url.path).st_size,
'version_count': self.version_count,
"before": {
"backupsize": os.stat(self.url.path).st_size,
"version_count": self.version_count,
},
}
@ -178,13 +202,14 @@ class FileBackend(Backend):
snapshot = Change(
version=change.version - 1,
snapshot=open(snapshotpath, 'rb').read(),
transaction=None
snapshot=open(snapshotpath, "rb").read(),
transaction=None,
)
print(
"Adding intial snapshot with {} bytes for version {}".format(
len(snapshot.snapshot), snapshot.version
)
)
print("Adding intial snapshot with {} bytes for version {}".format(
len(snapshot.snapshot),
snapshot.version
))
clone.add_change(snapshot)
assert clone.version == change.version - 1
@ -194,15 +219,17 @@ class FileBackend(Backend):
assert self.version == clone.version
assert self.prev_version == clone.prev_version
stats['after'] = {
'version_count': clone.version_count,
'backupsize': os.stat(clonepath).st_size,
stats["after"] = {
"version_count": clone.version_count,
"backupsize": os.stat(clonepath).st_size,
}
print("Compacted {} changes, saving {} bytes, swapping backups".format(
stats['before']['version_count'] - stats['after']['version_count'],
stats['before']['backupsize'] - stats['after']['backupsize'],
))
print(
"Compacted {} changes, saving {} bytes, swapping backups".format(
stats["before"]["version_count"] - stats["after"]["version_count"],
stats["before"]["backupsize"] - stats["after"]["backupsize"],
)
)
shutil.move(clonepath, self.url.path)
# Re-initialize ourselves so we have the correct metadata

36
backup/protocol.py
View file

@ -1,6 +1,7 @@
'''
"""
Socket-based remote backup protocol. This is used to create a connection to a backup backend, and send it incremental database updates.
'''
"""
import socket
import struct
from typing import Tuple
@ -19,56 +20,59 @@ class PacketType:
NACK = 0x07
METADATA = 0x08
DONE = 0x09
COMPACT = 0x0a
COMPACT_RES = 0x0b
COMPACT = 0x0A
COMPACT_RES = 0x0B
PKT_CHANGE_TYPES = {PacketType.CHANGE, PacketType.SNAPSHOT}
def recvall(sock: socket.socket, n: int) -> bytearray:
'''Receive exactly n bytes from a socket.'''
"""Receive exactly n bytes from a socket."""
buf = bytearray(n)
view = memoryview(buf)
ptr = 0
while ptr < n:
count = sock.recv_into(view[ptr:])
if count == 0:
raise IOError('Premature end of stream')
raise IOError("Premature end of stream")
ptr += count
return buf
def send_packet(sock: socket.socket, typ: int, payload: bytes) -> None:
sock.sendall(struct.pack('!BI', typ, len(payload)))
sock.sendall(struct.pack("!BI", typ, len(payload)))
sock.sendall(payload)
def recv_packet(sock: socket.socket) -> Tuple[int, bytes]:
(typ, length) = struct.unpack('!BI', recvall(sock, 5))
(typ, length) = struct.unpack("!BI", recvall(sock, 5))
payload = recvall(sock, length)
return (typ, payload)
def change_from_packet(typ, payload):
'''Convert a network packet to a Change object.'''
"""Convert a network packet to a Change object."""
if typ == PacketType.CHANGE:
(version, ) = struct.unpack('!I', payload[0:4])
(version,) = struct.unpack("!I", payload[0:4])
payload = zlib.decompress(payload[4:])
return Change(version=version, snapshot=None,
transaction=[t.decode('UTF-8') for t in payload.split(b'\x00')])
return Change(
version=version,
snapshot=None,
transaction=[t.decode("UTF-8") for t in payload.split(b"\x00")],
)
elif typ == PacketType.SNAPSHOT:
(version, ) = struct.unpack('!I', payload[0:4])
(version,) = struct.unpack("!I", payload[0:4])
payload = zlib.decompress(payload[4:])
return Change(version=version, snapshot=payload, transaction=None)
raise ValueError('Not a change (typ {})'.format(typ))
raise ValueError("Not a change (typ {})".format(typ))
def packet_from_change(entry):
'''Convert a Change object to a network packet.'''
"""Convert a Change object to a network packet."""
if entry.snapshot is None:
typ = PacketType.CHANGE
payload = b'\x00'.join([t.encode('UTF-8') for t in entry.transaction])
payload = b"\x00".join([t.encode("UTF-8") for t in entry.transaction])
else:
typ = PacketType.SNAPSHOT
payload = entry.snapshot

73
backup/server.py
View file

@ -6,7 +6,14 @@ import sys
from typing import Tuple
from backend import Backend
from protocol import PacketType, PKT_CHANGE_TYPES, change_from_packet, packet_from_change, send_packet, recv_packet
from protocol import (
PacketType,
PKT_CHANGE_TYPES,
change_from_packet,
packet_from_change,
send_packet,
recv_packet,
)
class SystemdHandler(logging.Handler):
@ -19,7 +26,7 @@ class SystemdHandler(logging.Handler):
# NOTICE <5>
logging.INFO: "<6>",
logging.DEBUG: "<7>",
logging.NOTSET: "<7>"
logging.NOTSET: "<7>",
}
def __init__(self, stream=sys.stdout):
@ -39,12 +46,12 @@ def setup_server_logging(mode, level):
root_logger = logging.getLogger()
root_logger.setLevel(level.upper())
mode = mode.lower()
if mode == 'systemd':
if mode == "systemd":
# replace handler with systemd one
root_logger.handlers = []
root_logger.addHandler(SystemdHandler())
else:
assert mode == 'plain'
assert mode == "plain"
class SocketServer:
@ -63,67 +70,77 @@ class SocketServer:
def _handle_conn(self, conn) -> None:
# Can only handle one connection at a time
logging.info('Servicing incoming connection')
logging.info("Servicing incoming connection")
self.sock = conn
while True:
try:
(typ, payload) = self._recv_packet()
except IOError:
logging.info('Connection closed')
logging.info("Connection closed")
break
if typ in PKT_CHANGE_TYPES:
change = change_from_packet(typ, payload)
if typ == PacketType.CHANGE:
logging.debug('Received CHANGE {}'.format(change.version))
logging.debug("Received CHANGE {}".format(change.version))
else:
logging.info('Received SNAPSHOT {}'.format(change.version))
logging.info("Received SNAPSHOT {}".format(change.version))
self.backend.add_change(change)
self._send_packet(PacketType.ACK, struct.pack("!I", self.backend.version))
self._send_packet(
PacketType.ACK, struct.pack("!I", self.backend.version)
)
elif typ == PacketType.REWIND:
logging.info('Received REWIND')
to_version, = struct.unpack('!I', payload)
logging.info("Received REWIND")
(to_version,) = struct.unpack("!I", payload)
if to_version != self.backend.prev_version:
logging.info('Cannot rewind to version {}'.format(to_version))
self._send_packet(PacketType.NACK, struct.pack("!I", self.backend.version))
logging.info("Cannot rewind to version {}".format(to_version))
self._send_packet(
PacketType.NACK, struct.pack("!I", self.backend.version)
)
else:
self.backend.rewind()
self._send_packet(PacketType.ACK, struct.pack("!I", self.backend.version))
self._send_packet(
PacketType.ACK, struct.pack("!I", self.backend.version)
)
elif typ == PacketType.REQ_METADATA:
logging.debug('Received REQ_METADATA')
blob = struct.pack("!IIIQ", 0x01, self.backend.version,
self.backend.prev_version,
self.backend.version_count)
logging.debug("Received REQ_METADATA")
blob = struct.pack(
"!IIIQ",
0x01,
self.backend.version,
self.backend.prev_version,
self.backend.version_count,
)
self._send_packet(PacketType.METADATA, blob)
elif typ == PacketType.RESTORE:
logging.info('Received RESTORE')
logging.info("Received RESTORE")
for change in self.backend.stream_changes():
(typ, payload) = packet_from_change(change)
self._send_packet(typ, payload)
self._send_packet(PacketType.DONE, b'')
self._send_packet(PacketType.DONE, b"")
elif typ == PacketType.COMPACT:
logging.info('Received COMPACT')
logging.info("Received COMPACT")
stats = self.backend.compact()
self._send_packet(PacketType.COMPACT_RES, json.dumps(stats).encode())
elif typ == PacketType.ACK:
logging.debug('Received ACK')
logging.debug("Received ACK")
elif typ == PacketType.NACK:
logging.debug('Received NACK')
logging.debug("Received NACK")
elif typ == PacketType.METADATA:
logging.debug('Received METADATA')
logging.debug("Received METADATA")
elif typ == PacketType.COMPACT_RES:
logging.debug('Received COMPACT_RES')
logging.debug("Received COMPACT_RES")
else:
raise Exception('Unknown or unexpected packet type {}'.format(typ))
raise Exception("Unknown or unexpected packet type {}".format(typ))
self.conn = None
def run(self) -> None:
self.bind.listen(1)
logging.info('Waiting for connection on {}'.format(self.addr))
logging.info("Waiting for connection on {}".format(self.addr))
while True:
conn, _ = self.bind.accept()
try:
self._handle_conn(conn)
except Exception:
logging.exception('Got exception')
logging.exception("Got exception")
finally:
conn.close()

127
backup/socketbackend.py
View file

@ -9,7 +9,14 @@ from typing import Tuple, Iterator
from urllib.parse import urlparse, parse_qs
from backend import Backend, Change
from protocol import PacketType, PKT_CHANGE_TYPES, change_from_packet, packet_from_change, send_packet, recv_packet
from protocol import (
PacketType,
PKT_CHANGE_TYPES,
change_from_packet,
packet_from_change,
send_packet,
recv_packet,
)
# Total number of reconnection tries
RECONNECT_TRIES = 5
@ -20,8 +27,8 @@ RECONNECT_DELAY = 5
# Scale delay factor after each failure
RECONNECT_DELAY_BACKOFF = 1.5
HostPortInfo = namedtuple('HostPortInfo', ['host', 'port', 'addrtype'])
SocketURLInfo = namedtuple('SocketURLInfo', ['target', 'proxytype', 'proxytarget'])
HostPortInfo = namedtuple("HostPortInfo", ["host", "port", "addrtype"])
SocketURLInfo = namedtuple("SocketURLInfo", ["target", "proxytype", "proxytarget"])
# Network address type.
@ -31,6 +38,7 @@ class AddrType:
IPv6 = 1
NAME = 2
# Proxy type. Only SOCKS5 supported at the moment as this is sufficient for Tor.
@ -40,23 +48,23 @@ class ProxyType:
def parse_host_port(path: str) -> HostPortInfo:
'''Parse a host:port pair.'''
if path.startswith('['): # bracketed IPv6 address
eidx = path.find(']')
"""Parse a host:port pair."""
if path.startswith("["): # bracketed IPv6 address
eidx = path.find("]")
if eidx == -1:
raise ValueError('Unterminated bracketed host address.')
raise ValueError("Unterminated bracketed host address.")
host = path[1:eidx]
addrtype = AddrType.IPv6
eidx += 1
if eidx >= len(path) or path[eidx] != ':':
raise ValueError('Port number missing.')
if eidx >= len(path) or path[eidx] != ":":
raise ValueError("Port number missing.")
eidx += 1
else:
eidx = path.find(':')
eidx = path.find(":")
if eidx == -1:
raise ValueError('Port number missing.')
raise ValueError("Port number missing.")
host = path[0:eidx]
if re.match(r'\d+\.\d+\.\d+\.\d+$', host): # matches IPv4 address format
if re.match(r"\d+\.\d+\.\d+\.\d+$", host): # matches IPv4 address format
addrtype = AddrType.IPv4
else:
addrtype = AddrType.NAME
@ -65,16 +73,16 @@ def parse_host_port(path: str) -> HostPortInfo:
try:
port = int(path[eidx:])
except ValueError:
raise ValueError('Invalid port number')
raise ValueError("Invalid port number")
return HostPortInfo(host=host, port=port, addrtype=addrtype)
def parse_socket_url(destination: str) -> SocketURLInfo:
'''Parse a socket: URL to extract the information contained in it.'''
"""Parse a socket: URL to extract the information contained in it."""
url = urlparse(destination)
if url.scheme != 'socket':
raise ValueError('Scheme for socket backend must be socket:...')
if url.scheme != "socket":
raise ValueError("Scheme for socket backend must be socket:...")
target = parse_host_port(url.path)
@ -83,19 +91,19 @@ def parse_socket_url(destination: str) -> SocketURLInfo:
# parse query parameters
# reject unknown parameters (currently all of them)
qs = parse_qs(url.query)
for (key, values) in qs.items():
if key == 'proxy': # proxy=socks5:127.0.0.1:9050
for key, values in qs.items():
if key == "proxy": # proxy=socks5:127.0.0.1:9050
if len(values) != 1:
raise ValueError('Proxy can only have one value')
raise ValueError("Proxy can only have one value")
(ptype, ptarget) = values[0].split(':', 1)
if ptype != 'socks5':
raise ValueError('Unknown proxy type ' + ptype)
(ptype, ptarget) = values[0].split(":", 1)
if ptype != "socks5":
raise ValueError("Unknown proxy type " + ptype)
proxytype = ProxyType.SOCKS5
proxytarget = parse_host_port(ptarget)
else:
raise ValueError('Unknown query string parameter ' + key)
raise ValueError("Unknown query string parameter " + key)
return SocketURLInfo(target=target, proxytype=proxytype, proxytarget=proxytarget)
@ -117,14 +125,23 @@ class SocketBackend(Backend):
else:
assert self.url.proxytype == ProxyType.SOCKS5
import socks
self.sock = socks.socksocket()
self.sock.set_proxy(socks.SOCKS5, self.url.proxytarget.host, self.url.proxytarget.port)
logging.info('Connecting to {}:{} (addrtype {}, proxytype {}, proxytarget {})...'.format(
self.url.target.host, self.url.target.port, self.url.target.addrtype,
self.url.proxytype, self.url.proxytarget))
self.sock = socks.socksocket()
self.sock.set_proxy(
socks.SOCKS5, self.url.proxytarget.host, self.url.proxytarget.port
)
logging.info(
"Connecting to {}:{} (addrtype {}, proxytype {}, proxytarget {})...".format(
self.url.target.host,
self.url.target.port,
self.url.target.addrtype,
self.url.proxytype,
self.url.proxytarget,
)
)
self.sock.connect((self.url.target.host, self.url.target.port))
logging.info('Connected to {}'.format(self.destination))
logging.info("Connected to {}".format(self.destination))
def _send_packet(self, typ: int, payload: bytes) -> None:
send_packet(self.sock, typ, payload)
@ -133,21 +150,25 @@ class SocketBackend(Backend):
return recv_packet(self.sock)
def initialize(self) -> bool:
'''
"""
Initialize socket backend by request current metadata from server.
'''
logging.info('Initializing backend')
"""
logging.info("Initializing backend")
self._request_metadata()
logging.info('Initialized SocketBackend: protocol={}, version={}, prev_version={}, version_count={}'.format(
self.protocol, self.version, self.prev_version, self.version_count
))
logging.info(
"Initialized SocketBackend: protocol={}, version={}, prev_version={}, version_count={}".format(
self.protocol, self.version, self.prev_version, self.version_count
)
)
return True
def _request_metadata(self) -> None:
self._send_packet(PacketType.REQ_METADATA, b'')
self._send_packet(PacketType.REQ_METADATA, b"")
(typ, payload) = self._recv_packet()
assert typ == PacketType.METADATA
self.protocol, self.version, self.prev_version, self.version_count = struct.unpack("!IIIQ", payload)
self.protocol, self.version, self.prev_version, self.version_count = (
struct.unpack("!IIIQ", payload)
)
def add_change(self, entry: Change) -> bool:
typ, payload = packet_from_change(entry)
@ -172,7 +193,11 @@ class SocketBackend(Backend):
# that on the server side. Then we retry.
pass
else:
raise Exception('Unexpected backup version {} after reconnect'.format(self.version))
raise Exception(
"Unexpected backup version {} after reconnect".format(
self.version
)
)
self._send_packet(typ, payload)
# Wait for change to be acknowledged before continuing.
@ -184,11 +209,19 @@ class SocketBackend(Backend):
break
if retry == RECONNECT_TRIES:
logging.error('Connection was lost while sending change (giving up after {} retries)'.format(retry))
raise IOError('Connection was lost while sending change')
logging.error(
"Connection was lost while sending change (giving up after {} retries)".format(
retry
)
)
raise IOError("Connection was lost while sending change")
retry += 1
logging.warning('Connection was lost while sending change (retry {} of {}, will try again after {} seconds)'.format(retry, RECONNECT_TRIES, retry_delay))
logging.warning(
"Connection was lost while sending change (retry {} of {}, will try again after {} seconds)".format(
retry, RECONNECT_TRIES, retry_delay
)
)
time.sleep(retry_delay)
retry_delay *= RECONNECT_DELAY_BACKOFF
need_connect = True
@ -198,7 +231,7 @@ class SocketBackend(Backend):
return True
def rewind(self) -> bool:
'''Rewind to previous version.'''
"""Rewind to previous version."""
version = struct.pack("!I", self.prev_version)
self._send_packet(PacketType.REWIND, version)
# Wait for change to be acknowledged before continuing.
@ -207,7 +240,7 @@ class SocketBackend(Backend):
return True
def stream_changes(self) -> Iterator[Change]:
self._send_packet(PacketType.RESTORE, b'')
self._send_packet(PacketType.RESTORE, b"")
version = -1
while True:
(typ, payload) = self._recv_packet()
@ -221,11 +254,15 @@ class SocketBackend(Backend):
raise ValueError("Unknown entry type {}".format(typ))
if version != self.version:
raise ValueError("Versions do not match up: restored version {}, backend version {}".format(version, self.version))
raise ValueError(
"Versions do not match up: restored version {}, backend version {}".format(
version, self.version
)
)
assert version == self.version
def compact(self):
self._send_packet(PacketType.COMPACT, b'')
self._send_packet(PacketType.COMPACT, b"")
(typ, payload) = self._recv_packet()
assert typ == PacketType.COMPACT_RES
return json.loads(payload.decode())

176
backup/test_backup.py
View file

@ -18,42 +18,41 @@ deprecated_apis = True
def test_start(node_factory, directory):
bpath = os.path.join(directory, 'lightning-1', 'regtest')
bdest = 'file://' + os.path.join(bpath, 'backup.dbak')
bpath = os.path.join(directory, "lightning-1", "regtest")
bdest = "file://" + os.path.join(bpath, "backup.dbak")
os.makedirs(bpath)
subprocess.check_call([cli_path, "init", "--lightning-dir", bpath, bdest])
opts = {
'plugin': plugin_path,
'allow-deprecated-apis': deprecated_apis,
"plugin": plugin_path,
"allow-deprecated-apis": deprecated_apis,
}
l1 = node_factory.get_node(options=opts, cleandir=False)
plugins = [os.path.basename(p['name']) for p in l1.rpc.plugin("list")['plugins']]
plugins = [os.path.basename(p["name"]) for p in l1.rpc.plugin("list")["plugins"]]
assert "backup.py" in plugins
# Restart the node a couple of times, to check that we can resume normally
for i in range(5):
l1.restart()
plugins = [os.path.basename(p['name']) for p in l1.rpc.plugin("list")['plugins']]
plugins = [
os.path.basename(p["name"]) for p in l1.rpc.plugin("list")["plugins"]
]
assert "backup.py" in plugins
def test_start_no_init(node_factory, directory):
"""The plugin should refuse to start if we haven't initialized the backup
"""
bpath = os.path.join(directory, 'lightning-1', 'regtest')
"""The plugin should refuse to start if we haven't initialized the backup"""
bpath = os.path.join(directory, "lightning-1", "regtest")
os.makedirs(bpath)
opts = {
'plugin': plugin_path,
"plugin": plugin_path,
}
l1 = node_factory.get_node(
options=opts, cleandir=False, may_fail=True, start=False
)
l1 = node_factory.get_node(options=opts, cleandir=False, may_fail=True, start=False)
with pytest.raises(TimeoutError):
# The way we detect a failure to start is when start() is running
# into timeout looking for 'Server started with public key'.
l1.start()
assert l1.daemon.is_in_log(r'Could not find backup.lock in the lightning-dir')
assert l1.daemon.is_in_log(r"Could not find backup.lock in the lightning-dir")
def test_init_not_empty(node_factory, directory):
@ -61,20 +60,20 @@ def test_init_not_empty(node_factory, directory):
backup-cli init should start the backup with an initial snapshot.
"""
bpath = os.path.join(directory, 'lightning-1', 'regtest')
bdest = 'file://' + os.path.join(bpath, 'backup.dbak')
bpath = os.path.join(directory, "lightning-1", "regtest")
bdest = "file://" + os.path.join(bpath, "backup.dbak")
l1 = node_factory.get_node()
l1.stop()
out = subprocess.check_output([cli_path, "init", "--lightning-dir", bpath, bdest])
assert b'Found an existing database' in out
assert b'Successfully written initial snapshot' in out
assert b"Found an existing database" in out
assert b"Successfully written initial snapshot" in out
# Now restart and add the plugin
l1.daemon.opts['plugin'] = plugin_path
l1.daemon.opts['allow-deprecated-apis'] = deprecated_apis
l1.daemon.opts["plugin"] = plugin_path
l1.daemon.opts["allow-deprecated-apis"] = deprecated_apis
l1.start()
assert l1.daemon.is_in_log(r'plugin-backup.py: Versions match up')
assert l1.daemon.is_in_log(r"plugin-backup.py: Versions match up")
@flaky
@ -88,13 +87,13 @@ def test_tx_abort(node_factory, directory):
inbetween the hook call and the DB transaction.
"""
bpath = os.path.join(directory, 'lightning-1', 'regtest')
bdest = 'file://' + os.path.join(bpath, 'backup.dbak')
bpath = os.path.join(directory, "lightning-1", "regtest")
bdest = "file://" + os.path.join(bpath, "backup.dbak")
os.makedirs(bpath)
subprocess.check_call([cli_path, "init", "--lightning-dir", bpath, bdest])
opts = {
'plugin': plugin_path,
'allow-deprecated-apis': deprecated_apis,
"plugin": plugin_path,
"allow-deprecated-apis": deprecated_apis,
}
l1 = node_factory.get_node(options=opts, cleandir=False)
l1.stop()
@ -107,7 +106,7 @@ def test_tx_abort(node_factory, directory):
print(l1.db.query("SELECT * FROM vars;"))
l1.restart()
assert l1.daemon.is_in_log(r'Last changes not applied')
assert l1.daemon.is_in_log(r"Last changes not applied")
@flaky
@ -118,13 +117,13 @@ def test_failing_restore(node_factory, directory):
in the database back to n-2, which is non-recoverable.
"""
bpath = os.path.join(directory, 'lightning-1', 'regtest')
bdest = 'file://' + os.path.join(bpath, 'backup.dbak')
bpath = os.path.join(directory, "lightning-1", "regtest")
bdest = "file://" + os.path.join(bpath, "backup.dbak")
os.makedirs(bpath)
subprocess.check_call([cli_path, "init", "--lightning-dir", bpath, bdest])
opts = {
'plugin': plugin_path,
'allow-deprecated-apis': deprecated_apis,
"plugin": plugin_path,
"allow-deprecated-apis": deprecated_apis,
}
def section(comment):
@ -144,25 +143,23 @@ def test_failing_restore(node_factory, directory):
l1.daemon.proc.wait()
section("Verifying the node died with an error")
assert l1.daemon.is_in_log(r'lost some state') is not None
assert l1.daemon.is_in_log(r"lost some state") is not None
def test_intermittent_backup(node_factory, directory):
"""Simulate intermittent use of the backup, or an old file backup.
"""
bpath = os.path.join(directory, 'lightning-1', 'regtest')
bdest = 'file://' + os.path.join(bpath, 'backup.dbak')
"""Simulate intermittent use of the backup, or an old file backup."""
bpath = os.path.join(directory, "lightning-1", "regtest")
bdest = "file://" + os.path.join(bpath, "backup.dbak")
os.makedirs(bpath)
subprocess.check_call([cli_path, "init", "--lightning-dir", bpath, bdest])
opts = {
'plugin': plugin_path,
'allow-deprecated-apis': deprecated_apis,
"plugin": plugin_path,
"allow-deprecated-apis": deprecated_apis,
}
l1 = node_factory.get_node(options=opts, cleandir=False, may_fail=True)
# Now start without the plugin. This should work fine.
del l1.daemon.opts['plugin']
del l1.daemon.opts["plugin"]
l1.restart()
# Now restart adding the plugin again, and it should fail due to gaps in
@ -173,33 +170,33 @@ def test_intermittent_backup(node_factory, directory):
l1.start()
l1.daemon.proc.wait()
assert l1.daemon.is_in_log(r'Backup is out of date') is not None
assert l1.daemon.is_in_log(r"Backup is out of date") is not None
def test_restore(node_factory, directory):
bpath = os.path.join(directory, 'lightning-1', 'regtest')
bdest = 'file://' + os.path.join(bpath, 'backup.dbak')
bpath = os.path.join(directory, "lightning-1", "regtest")
bdest = "file://" + os.path.join(bpath, "backup.dbak")
os.makedirs(bpath)
subprocess.check_call([cli_path, "init", "--lightning-dir", bpath, bdest])
opts = {
'plugin': plugin_path,
'allow-deprecated-apis': deprecated_apis,
"plugin": plugin_path,
"allow-deprecated-apis": deprecated_apis,
}
l1 = node_factory.get_node(options=opts, cleandir=False)
l1.stop()
rdest = os.path.join(bpath, 'lightningd.sqlite.restore')
rdest = os.path.join(bpath, "lightningd.sqlite.restore")
subprocess.check_call([cli_path, "restore", bdest, rdest])
def test_restore_dir(node_factory, directory):
bpath = os.path.join(directory, 'lightning-1', 'regtest')
bdest = 'file://' + os.path.join(bpath, 'backup.dbak')
bpath = os.path.join(directory, "lightning-1", "regtest")
bdest = "file://" + os.path.join(bpath, "backup.dbak")
os.makedirs(bpath)
subprocess.check_call([cli_path, "init", "--lightning-dir", bpath, bdest])
opts = {
'plugin': plugin_path,
'allow-deprecated-apis': deprecated_apis,
"plugin": plugin_path,
"allow-deprecated-apis": deprecated_apis,
}
l1 = node_factory.get_node(options=opts, cleandir=False)
l1.stop()
@ -214,19 +211,21 @@ def test_restore_dir(node_factory, directory):
def test_warning(directory, node_factory):
bpath = os.path.join(directory, 'lightning-1', 'regtest')
bdest = 'file://' + os.path.join(bpath, 'backup.dbak')
bpath = os.path.join(directory, "lightning-1", "regtest")
bdest = "file://" + os.path.join(bpath, "backup.dbak")
os.makedirs(bpath)
subprocess.check_call([cli_path, "init", "--lightning-dir", bpath, bdest])
opts = {
'plugin': plugin_path,
'allow-deprecated-apis': deprecated_apis,
'backup-destination': 'somewhere/over/the/rainbox',
"plugin": plugin_path,
"allow-deprecated-apis": deprecated_apis,
"backup-destination": "somewhere/over/the/rainbox",
}
l1 = node_factory.get_node(options=opts, cleandir=False)
l1.stop()
assert l1.daemon.is_in_log(r'The `--backup-destination` option is deprecated and will be removed in future versions of the backup plugin.')
assert l1.daemon.is_in_log(
r"The `--backup-destination` option is deprecated and will be removed in future versions of the backup plugin."
)
class DummyBackend(Backend):
@ -237,8 +236,8 @@ class DummyBackend(Backend):
def test_rewrite():
tests = [
(
r'UPDATE outputs SET status=123, reserved_til=1891733WHERE prev_out_tx=1 AND prev_out_index=2',
r'UPDATE outputs SET status=123, reserved_til=1891733 WHERE prev_out_tx=1 AND prev_out_index=2',
r"UPDATE outputs SET status=123, reserved_til=1891733WHERE prev_out_tx=1 AND prev_out_index=2",
r"UPDATE outputs SET status=123, reserved_til=1891733 WHERE prev_out_tx=1 AND prev_out_index=2",
),
]
@ -249,21 +248,22 @@ def test_rewrite():
def test_restore_pre_4090(directory):
"""The prev-4090-backup.dbak contains faulty expansions, fix em.
"""
bdest = 'file://' + os.path.join(os.path.dirname(__file__), 'tests', 'pre-4090-backup.dbak')
rdest = os.path.join(directory, 'lightningd.sqlite.restore')
"""The prev-4090-backup.dbak contains faulty expansions, fix em."""
bdest = "file://" + os.path.join(
os.path.dirname(__file__), "tests", "pre-4090-backup.dbak"
)
rdest = os.path.join(directory, "lightningd.sqlite.restore")
subprocess.check_call([cli_path, "restore", bdest, rdest])
def test_compact(bitcoind, directory, node_factory):
bpath = os.path.join(directory, 'lightning-1', 'regtest')
bdest = 'file://' + os.path.join(bpath, 'backup.dbak')
bpath = os.path.join(directory, "lightning-1", "regtest")
bdest = "file://" + os.path.join(bpath, "backup.dbak")
os.makedirs(bpath)
subprocess.check_call([cli_path, "init", "--lightning-dir", bpath, bdest])
opts = {
'plugin': plugin_path,
'allow-deprecated-apis': deprecated_apis,
"plugin": plugin_path,
"allow-deprecated-apis": deprecated_apis,
}
l1 = node_factory.get_node(options=opts, cleandir=False)
l1.rpc.backup_compact()
@ -283,53 +283,57 @@ def test_compact(bitcoind, directory, node_factory):
def test_parse_socket_url():
with pytest.raises(ValueError):
# fail: invalid url scheme
socketbackend.parse_socket_url('none')
socketbackend.parse_socket_url("none")
# fail: no port number
socketbackend.parse_socket_url('socket:127.0.0.1')
socketbackend.parse_socket_url('socket:127.0.0.1:')
socketbackend.parse_socket_url("socket:127.0.0.1")
socketbackend.parse_socket_url("socket:127.0.0.1:")
# fail: unbracketed IPv6
socketbackend.parse_socket_url('socket:::1:1234')
socketbackend.parse_socket_url("socket:::1:1234")
# fail: no port number IPv6
socketbackend.parse_socket_url('socket:[::1]')
socketbackend.parse_socket_url('socket:[::1]:')
socketbackend.parse_socket_url("socket:[::1]")
socketbackend.parse_socket_url("socket:[::1]:")
# fail: invalid port number
socketbackend.parse_socket_url('socket:127.0.0.1:12bla')
socketbackend.parse_socket_url("socket:127.0.0.1:12bla")
# fail: unrecognized query string key
socketbackend.parse_socket_url('socket:127.0.0.1:1234?dummy=value')
socketbackend.parse_socket_url("socket:127.0.0.1:1234?dummy=value")
# fail: incomplete proxy spec
socketbackend.parse_socket_url('socket:127.0.0.1:1234?proxy=socks5')
socketbackend.parse_socket_url('socket:127.0.0.1:1234?proxy=socks5:')
socketbackend.parse_socket_url('socket:127.0.0.1:1234?proxy=socks5:127.0.0.1:')
socketbackend.parse_socket_url("socket:127.0.0.1:1234?proxy=socks5")
socketbackend.parse_socket_url("socket:127.0.0.1:1234?proxy=socks5:")
socketbackend.parse_socket_url("socket:127.0.0.1:1234?proxy=socks5:127.0.0.1:")
# fail: unknown proxy scheme
socketbackend.parse_socket_url('socket:127.0.0.1:1234?proxy=socks6:127.0.0.1:9050')
socketbackend.parse_socket_url(
"socket:127.0.0.1:1234?proxy=socks6:127.0.0.1:9050"
)
# IPv4
s = socketbackend.parse_socket_url('socket:127.0.0.1:1234')
assert s.target.host == '127.0.0.1'
s = socketbackend.parse_socket_url("socket:127.0.0.1:1234")
assert s.target.host == "127.0.0.1"
assert s.target.port == 1234
assert s.target.addrtype == socketbackend.AddrType.IPv4
assert s.proxytype == socketbackend.ProxyType.DIRECT
# IPv6
s = socketbackend.parse_socket_url('socket:[::1]:1235')
assert s.target.host == '::1'
s = socketbackend.parse_socket_url("socket:[::1]:1235")
assert s.target.host == "::1"
assert s.target.port == 1235
assert s.target.addrtype == socketbackend.AddrType.IPv6
assert s.proxytype == socketbackend.ProxyType.DIRECT
# Hostname
s = socketbackend.parse_socket_url('socket:backup.local:1236')
assert s.target.host == 'backup.local'
s = socketbackend.parse_socket_url("socket:backup.local:1236")
assert s.target.host == "backup.local"
assert s.target.port == 1236
assert s.target.addrtype == socketbackend.AddrType.NAME
assert s.proxytype == socketbackend.ProxyType.DIRECT
# Tor
s = socketbackend.parse_socket_url('socket:backupserver.onion:1234?proxy=socks5:127.0.0.1:9050')
assert s.target.host == 'backupserver.onion'
s = socketbackend.parse_socket_url(
"socket:backupserver.onion:1234?proxy=socks5:127.0.0.1:9050"
)
assert s.target.host == "backupserver.onion"
assert s.target.port == 1234
assert s.target.addrtype == socketbackend.AddrType.NAME
assert s.proxytype == socketbackend.ProxyType.SOCKS5
assert s.proxytarget.host == '127.0.0.1'
assert s.proxytarget.host == "127.0.0.1"
assert s.proxytarget.port == 9050
assert s.proxytarget.addrtype == socketbackend.AddrType.IPv4

61
clearnet/clearnet.py
View file

@ -1,29 +1,30 @@
#!/usr/bin/env python3
import socket
from contextlib import closing
from pyln.client import Plugin, RpcError
plugin = Plugin()
def get_address_type(addrstr: str):
""" I know this can be more sophisticated, but works """
"""I know this can be more sophisticated, but works"""
if ".onion:" in addrstr:
return 'tor'
return "tor"
if addrstr[0].isdigit():
return 'ipv4'
return "ipv4"
if addrstr.startswith("["):
return 'ipv6'
return 'dns'
return "ipv6"
return "dns"
# taken from:
# https://stackoverflow.com/questions/19196105/how-to-check-if-a-network-port-is-open
def check_socket(host: str, port: int, timeout: float = None):
""" Checks if a socket can be opened to a host """
if host.count('.') == 3:
"""Checks if a socket can be opened to a host"""
if host.count(".") == 3:
proto = socket.AF_INET
if host.count(':') > 1:
if host.count(":") > 1:
proto = socket.AF_INET6
with closing(socket.socket(proto, socket.SOCK_STREAM)) as sock:
if timeout is not None:
@ -35,30 +36,30 @@ def check_socket(host: str, port: int, timeout: float = None):
def clearnet_pid(peer: dict, messages: list):
peer_id = peer['id']
if not peer['connected']:
peer_id = peer["id"]
if not peer["connected"]:
messages += [f"Peer is not conencted: {peer_id}"]
return False
if get_address_type(peer['netaddr'][0]) != 'tor':
if get_address_type(peer["netaddr"][0]) != "tor":
messages += [f"Already connected via clearnet: {peer_id}"]
return True
# lets check what gossip knows about this peer
nodes = plugin.rpc.listnodes(peer_id)['nodes']
nodes = plugin.rpc.listnodes(peer_id)["nodes"]
if len(nodes) == 0:
messages += [f"Error: No gossip for: {peer_id}"]
return
addrs = [a for a in nodes[0]['addresses'] if not a['type'].startswith("tor")]
addrs = [a for a in nodes[0]["addresses"] if not a["type"].startswith("tor")]
if len(addrs) == 0:
messages += [f"Error: No clearnet addresses known for: {peer_id}"]
return
# now check addrs for open ports
for addr in addrs:
if addr['type'] == 'dns':
if addr["type"] == "dns":
messages += [f"TODO: DNS lookups for: {addr['address']}"]
continue
if check_socket(addr['address'], addr['port'], 2.0):
if check_socket(addr["address"], addr["port"], 2.0):
# disconnect
result = plugin.rpc.disconnect(peer_id, True)
if len(result) != 0:
@ -67,18 +68,24 @@ def clearnet_pid(peer: dict, messages: list):
# try clearnet connection
try:
result = plugin.rpc.connect(peer_id, addr['address'], addr['port'])
newtype = result['address']['type']
if not newtype.startswith('tor'):
messages += [f"Established clearnet connection for: {peer_id} with {newtype}"]
result = plugin.rpc.connect(peer_id, addr["address"], addr["port"])
newtype = result["address"]["type"]
if not newtype.startswith("tor"):
messages += [
f"Established clearnet connection for: {peer_id} with {newtype}"
]
return True
except RpcError: # we got an connection error, try reconnect
messages += [f"Error: Connection failed for: {peer_id} with {addr['type']}"]
messages += [
f"Error: Connection failed for: {peer_id} with {addr['type']}"
]
try:
result = plugin.rpc.connect(peer_id) # without address
newtype = result['address']['type']
if not newtype.startswith('tor'):
messages += [f"Established clearnet connection for: {peer_id} with {newtype}"]
newtype = result["address"]["type"]
if not newtype.startswith("tor"):
messages += [
f"Established clearnet connection for: {peer_id} with {newtype}"
]
return True
except RpcError: # we got a reconnection error
messages += [f"Error: Reconnection failed for: {peer_id}"]
@ -90,13 +97,13 @@ def clearnet_pid(peer: dict, messages: list):
@plugin.method("clearnet")
def clearnet(plugin: Plugin, peer_id: str = None):
""" Enforce a clearnet connection on all peers or a given `peer_id`."""
"""Enforce a clearnet connection on all peers or a given `peer_id`."""
if peer_id is None:
peers = plugin.rpc.listpeers(peer_id)['peers']
peers = plugin.rpc.listpeers(peer_id)["peers"]
else:
if not isinstance(peer_id, str) or len(peer_id) != 66:
return f"Error: Invalid peer_id: {peer_id}"
peers = plugin.rpc.listpeers(peer_id)['peers']
peers = plugin.rpc.listpeers(peer_id)["peers"]
if len(peers) == 0:
return f"Error: peer not found: {peer_id}"
@ -108,7 +115,7 @@ def clearnet(plugin: Plugin, peer_id: str = None):
@plugin.init()
def init(options: dict, configuration: dict, plugin: Plugin, **kwargs):
plugin.log(f"clearnet enforcer plugin initialized")
plugin.log("clearnet enforcer plugin initialized")
plugin.run()

2
clearnet/test_clearnet.py
View file

@ -17,6 +17,6 @@ def test_clearnet_starts(node_factory):
def test_clearnet_runs(node_factory):
pluginopt = {'plugin': plugin_path}
pluginopt = {"plugin": plugin_path}
l1, l2 = node_factory.line_graph(2, opts=pluginopt)
l1.rpc.clearnet()

91
currencyrate/currencyrate.py
View file

@ -10,29 +10,33 @@ import statistics
plugin = Plugin()
Source = namedtuple('Source', ['name', 'urlformat', 'replymembers'])
Source = namedtuple("Source", ["name", "urlformat", "replymembers"])
sources = [
# e.g. {"high": "18502.56", "last": "17970.41", "timestamp": "1607650787", "bid": "17961.87", "vwap": "18223.42", "volume": "7055.63066541", "low": "17815.92", "ask": "17970.41", "open": "18250.30"}
Source('bitstamp',
'https://www.bitstamp.net/api/v2/ticker/btc{currency_lc}/',
['last']),
Source(
"bitstamp", "https://www.bitstamp.net/api/v2/ticker/btc{currency_lc}/", ["last"]
),
# e.g. {"bitcoin":{"usd":17885.84}}
Source('coingecko',
'https://api.coingecko.com/api/v3/simple/price?ids=bitcoin&vs_currencies={currency_lc}',
['bitcoin', '{currency_lc}']),
Source(
"coingecko",
"https://api.coingecko.com/api/v3/simple/price?ids=bitcoin&vs_currencies={currency_lc}",
["bitcoin", "{currency_lc}"],
),
# e.g. {"time":{"updated":"Dec 16, 2020 00:58:00 UTC","updatedISO":"2020-12-16T00:58:00+00:00","updateduk":"Dec 16, 2020 at 00:58 GMT"},"disclaimer":"This data was produced from the CoinDesk Bitcoin Price Index (USD). Non-USD currency data converted using hourly conversion rate from openexchangerates.org","bpi":{"USD":{"code":"USD","rate":"19,395.1400","description":"United States Dollar","rate_float":19395.14},"AUD":{"code":"AUD","rate":"25,663.5329","description":"Australian Dollar","rate_float":25663.5329}}}
Source('coindesk',
'https://api.coindesk.com/v1/bpi/currentprice/{currency}.json',
['bpi', '{currency}', 'rate_float']),
Source(
"coindesk",
"https://api.coindesk.com/v1/bpi/currentprice/{currency}.json",
["bpi", "{currency}", "rate_float"],
),
# e.g. {"data":{"base":"BTC","currency":"USD","amount":"19414.63"}}
Source('coinbase',
'https://api.coinbase.com/v2/prices/spot?currency={currency}',
['data', 'amount']),
Source(
"coinbase",
"https://api.coinbase.com/v2/prices/spot?currency={currency}",
["data", "amount"],
),
# e.g. { "USD" : {"15m" : 6650.3, "last" : 6650.3, "buy" : 6650.3, "sell" : 6650.3, "symbol" : "$"}, "AUD" : {"15m" : 10857.19, "last" : 10857.19, "buy" : 10857.19, "sell" : 10857.19, "symbol" : "$"},...
Source('blockchain.info',
'https://blockchain.info/ticker',
['{currency}', 'last']),
Source("blockchain.info", "https://blockchain.info/ticker", ["{currency}", "last"]),
]
@ -52,8 +56,8 @@ def requests_retry_session(
status_forcelist=status_forcelist,
)
adapter = HTTPAdapter(max_retries=retry)
session.mount('http://', adapter)
session.mount('https://', adapter)
session.mount("http://", adapter)
session.mount("https://", adapter)
return session
@ -62,36 +66,43 @@ def get_currencyrate(plugin, currency, urlformat, replymembers):
# Workaround: retry up to 5 times with a delay
currency_lc = currency.lower()
url = urlformat.format(currency_lc=currency_lc, currency=currency)
r = requests_retry_session(retries=5, status_forcelist=[404]).get(url, proxies=plugin.proxies)
r = requests_retry_session(retries=5, status_forcelist=[404]).get(
url, proxies=plugin.proxies
)
if r.status_code != 200:
plugin.log(level='info', message='{}: bad response {}'.format(url, r.status_code))
plugin.log(
level="info", message="{}: bad response {}".format(url, r.status_code)
)
return None
json = r.json()
for m in replymembers:
expanded = m.format(currency_lc=currency_lc, currency=currency)
if expanded not in json:
plugin.log(level='debug', message='{}: {} not in {}'.format(url, expanded, json))
plugin.log(
level="debug", message="{}: {} not in {}".format(url, expanded, json)
)
return None
json = json[expanded]
try:
return Millisatoshi(int(10**11 / float(json)))
except Exception:
plugin.log(level='info', message='{}: could not convert {} to msat'.format(url, json))
plugin.log(
level="info", message="{}: could not convert {} to msat".format(url, json)
)
return None
def set_proxies(plugin):
config = plugin.rpc.listconfigs()
if 'always-use-proxy' in config and config['always-use-proxy']:
paddr = config['proxy']
if "always-use-proxy" in config and config["always-use-proxy"]:
paddr = config["proxy"]
# Default port in 9050
if ':' not in paddr:
paddr += ':9050'
plugin.proxies = {'https': 'socks5h://' + paddr,
'http': 'socks5h://' + paddr}
if ":" not in paddr:
paddr += ":9050"
plugin.proxies = {"https": "socks5h://" + paddr, "http": "socks5h://" + paddr}
else:
plugin.proxies = None
@ -129,31 +140,37 @@ def currencyconvert(plugin, amount, currency):
def init(options, configuration, plugin):
set_proxies(plugin)
sourceopts = options['add-source']
sourceopts = options["add-source"]
# Prior to 0.9.3, 'multi' was unsupported.
if type(sourceopts) is not list:
sourceopts = [sourceopts]
if sourceopts != ['']:
if sourceopts != [""]:
for s in sourceopts:
parts = s.split(',')
parts = s.split(",")
sources.append(Source(parts[0], parts[1], parts[2:]))
disableopts = options['disable-source']
disableopts = options["disable-source"]
# Prior to 0.9.3, 'multi' was unsupported.
if type(disableopts) is not list:
disableopts = [disableopts]
if disableopts != ['']:
if disableopts != [""]:
for s in sources[:]:
if s.name in disableopts:
sources.remove(s)
# As a bad example: binance,https://api.binance.com/api/v3/ticker/price?symbol=BTC{currency}T,price
plugin.add_option(name='add-source', default='', description='Add source name,urlformat,resultmembers...')
plugin.add_option(name='disable-source', default='', description='Disable source by name')
plugin.add_option(
name="add-source",
default="",
description="Add source name,urlformat,resultmembers...",
)
plugin.add_option(
name="disable-source", default="", description="Disable source by name"
)
# This has an effect only for recent pyln versions (0.9.3+).
plugin.options['add-source']['multi'] = True
plugin.options['disable-source']['multi'] = True
plugin.options["add-source"]["multi"] = True
plugin.options["disable-source"]["multi"] = True
plugin.run()

4
currencyrate/test_currencyrate.py
View file

@ -34,9 +34,7 @@ def test_currencyrate(node_factory):
"disable-source": "bitstamp",
}
l1 = node_factory.get_node(options=opts)
plugins = [
os.path.basename(p["name"]) for p in l1.rpc.plugin("list")["plugins"]
]
plugins = [os.path.basename(p["name"]) for p in l1.rpc.plugin("list")["plugins"]]
assert "currencyrate.py" in plugins
rates = l1.rpc.call("currencyrates", ["USD"])

72
datastore/datastore-plugin.py
View file

@ -2,6 +2,7 @@
"""This does the actual datastore work, if the main plugin says there's no
datastore support. We can't even load this if there's real datastore support.
"""
from pyln.client import Plugin, RpcException
from collections import namedtuple
import os
@ -18,7 +19,7 @@ DATASTORE_UPDATE_HAS_CHILDREN = 1205
DATASTORE_UPDATE_NO_CHILDREN = 1206
plugin = Plugin()
Entry = namedtuple('Entry', ['generation', 'data'])
Entry = namedtuple("Entry", ["generation", "data"])
# A singleton to most commands turns into a [].
@ -30,11 +31,11 @@ def normalize_key(key: Union[Sequence[str], str]) -> List[str]:
# We turn list into nul-separated hexbytes for storage (shelve needs all keys to be strings)
def key_to_hex(key: Sequence[str]) -> str:
return b'\0'.join([bytes(k, encoding='utf8') for k in key]).hex()
return b"\0".join([bytes(k, encoding="utf8") for k in key]).hex()
def hex_to_key(hexstr: str) -> List[str]:
return [b.decode() for b in bytes.fromhex(hexstr).split(b'\0')]
return [b.decode() for b in bytes.fromhex(hexstr).split(b"\0")]
def datastore_entry(key: Sequence[str], entry: Optional[Entry]):
@ -43,18 +44,18 @@ def datastore_entry(key: Sequence[str], entry: Optional[Entry]):
if isinstance(key, str):
key = [key]
ret = {'key': key}
ret = {"key": key}
if entry is not None:
# Entry may be a simple tuple; convert
entry = Entry(*entry)
ret['generation'] = entry.generation
ret['hex'] = entry.data.hex()
ret["generation"] = entry.generation
ret["hex"] = entry.data.hex()
# FFS, Python3 seems happy with \0 in UTF-8.
if 0 not in entry.data:
try:
ret['string'] = entry.data.decode('utf8')
ret["string"] = entry.data.decode("utf8")
except UnicodeDecodeError:
pass
return ret
@ -63,7 +64,7 @@ def datastore_entry(key: Sequence[str], entry: Optional[Entry]):
@plugin.method("datastore")
def datastore(plugin, key, string=None, hex=None, mode="must-create", generation=None):
"""Add/modify a {key} and {hex}/{string} data to the data store,
optionally insisting it be {generation}"""
optionally insisting it be {generation}"""
key = normalize_key(key)
khex = key_to_hex(key)
@ -78,29 +79,23 @@ optionally insisting it be {generation}"""
if mode == "must-create":
if khex in plugin.datastore:
raise RpcException("already exists",
DATASTORE_UPDATE_ALREADY_EXISTS)
raise RpcException("already exists", DATASTORE_UPDATE_ALREADY_EXISTS)
elif mode == "must-replace":
if khex not in plugin.datastore:
raise RpcException("does not exist",
DATASTORE_UPDATE_DOES_NOT_EXIST)
raise RpcException("does not exist", DATASTORE_UPDATE_DOES_NOT_EXIST)
elif mode == "create-or-replace":
if generation is not None:
raise RpcException("generation only valid with"
" must-create/must-replace")
raise RpcException("generation only valid with" " must-create/must-replace")
pass
elif mode == "must-append":
if generation is not None:
raise RpcException("generation only valid with"
" must-create/must-replace")
raise RpcException("generation only valid with" " must-create/must-replace")
if khex not in plugin.datastore:
raise RpcException("does not exist",
DATASTORE_UPDATE_DOES_NOT_EXIST)
raise RpcException("does not exist", DATASTORE_UPDATE_DOES_NOT_EXIST)
data = plugin.datastore[khex].data + data
elif mode == "create-or-append":
if generation is not None:
raise RpcException("generation only valid with"
" must-create/must-replace")
raise RpcException("generation only valid with" " must-create/must-replace")
data = plugin.datastore.get(khex, Entry(0, bytes())).data + data
else:
raise RpcException("invalid mode")
@ -109,22 +104,24 @@ optionally insisting it be {generation}"""
parent = [key[0]]
for i in range(1, len(key)):
if key_to_hex(parent) in plugin.datastore:
raise RpcException("Parent key [{}] exists".format(','.join(parent)),
DATASTORE_UPDATE_NO_CHILDREN)
raise RpcException(
"Parent key [{}] exists".format(",".join(parent)),
DATASTORE_UPDATE_NO_CHILDREN,
)
parent += [key[i]]
if khex in plugin.datastore:
entry = plugin.datastore[khex]
if generation is not None:
if entry.generation != generation:
raise RpcException("generation is different",
DATASTORE_UPDATE_WRONG_GENERATION)
raise RpcException(
"generation is different", DATASTORE_UPDATE_WRONG_GENERATION
)
gen = entry.generation + 1
else:
# Make sure child doesn't exist (grossly inefficient)
if any([hex_to_key(k)[:len(key)] == key for k in plugin.datastore]):
raise RpcException("Key has children",
DATASTORE_UPDATE_HAS_CHILDREN)
if any([hex_to_key(k)[: len(key)] == key for k in plugin.datastore]):
raise RpcException("Key has children", DATASTORE_UPDATE_HAS_CHILDREN)
gen = 0
plugin.datastore[khex] = Entry(gen, data)
@ -143,8 +140,7 @@ def deldatastore(plugin, key, generation=None):
entry = plugin.datastore[khex]
if generation is not None and entry.generation != generation:
raise RpcException("generation is different",
DATASTORE_DEL_WRONG_GENERATION)
raise RpcException("generation is different", DATASTORE_DEL_WRONG_GENERATION)
ret = datastore_entry(key, entry)
del plugin.datastore[khex]
@ -160,39 +156,39 @@ def listdatastore(plugin, key=[]):
prev = None
for khex, e in sorted(plugin.datastore.items()):
k = hex_to_key(khex)
if k[:len(key)] != key:
if k[: len(key)] != key:
continue
# Don't print sub-children
if len(k) > len(key) + 1:
if prev is None or k[:len(key)+1] != prev:
prev = k[:len(key)+1]
if prev is None or k[: len(key) + 1] != prev:
prev = k[: len(key) + 1]
ret += [datastore_entry(prev, None)]
else:
ret += [datastore_entry(k, e)]
return {'datastore': ret}
return {"datastore": ret}
def upgrade_store(plugin):
"""Initial version of this plugin had no generation numbers"""
try:
oldstore = shelve.open('datastore.dat', 'r')
oldstore = shelve.open("datastore.dat", "r")
except:
return
plugin.log("Upgrading store to have generation numbers", level='unusual')
datastore = shelve.open('datastore_v1.dat', 'c')
plugin.log("Upgrading store to have generation numbers", level="unusual")
datastore = shelve.open("datastore_v1.dat", "c")
for k, d in oldstore.items():
datastore[key_to_hex([k])] = Entry(0, d)
oldstore.close()
datastore.close()
os.unlink('datastore.dat')
os.unlink("datastore.dat")
@plugin.init()
def init(options, configuration, plugin):
upgrade_store(plugin)
plugin.datastore = shelve.open('datastore_v1.dat')
plugin.datastore = shelve.open("datastore_v1.dat")
plugin.run()

24
datastore/datastore.py
View file

@ -10,36 +10,38 @@ plugin = Plugin()
def unload_store(plugin):
"""When we have a real store, we transfer our contents into it"""
try:
datastore = shelve.open('datastore_v1.dat', 'r')
datastore = shelve.open("datastore_v1.dat", "r")
except:
return
plugin.log("Emptying store into main store (resetting generations!)", level='unusual')
plugin.log(
"Emptying store into main store (resetting generations!)", level="unusual"
)
for k, (g, data) in datastore.items():
try:
plugin.rpc.datastore(key=[k], hex=data.hex())
except RpcError as e:
plugin.log("Failed to put {} into store: {}".format(k, e),
level='broken')
plugin.log("Failed to put {} into store: {}".format(k, e), level="broken")
datastore.close()
plugin.log("Erasing our store", level='unusual')
os.unlink('datastore_v1.dat')
plugin.log("Erasing our store", level="unusual")
os.unlink("datastore_v1.dat")
@plugin.init()
def init(options, configuration, plugin):
# If we have real datastore commands, don't load plugin.
try:
plugin.rpc.help('datastore')
plugin.rpc.help("datastore")
unload_store(plugin)
return {'disable': 'there is a real datastore command'}
return {"disable": "there is a real datastore command"}
except RpcError:
pass
# Start up real plugin now
plugin.rpc.plugin_start(os.path.join(os.path.dirname(__file__),
"datastore-plugin.py"))
return {'disable': 'no builtin-datastore: plugin loaded'}
plugin.rpc.plugin_start(
os.path.join(os.path.dirname(__file__), "datastore-plugin.py")
)
return {"disable": "no builtin-datastore: plugin loaded"}
plugin.run()

310
datastore/test_datastore.py
View file

@ -11,119 +11,142 @@ plugin_path = os.path.join(os.path.dirname(__file__), "datastore.py")
# Test taken from lightning/tests/test_misc.py
def test_datastore(node_factory):
l1 = node_factory.get_node(options={'plugin': plugin_path})
l1 = node_factory.get_node(options={"plugin": plugin_path})
time.sleep(5)
# Starts empty
assert l1.rpc.listdatastore() == {'datastore': []}
assert l1.rpc.listdatastore('somekey') == {'datastore': []}
assert l1.rpc.listdatastore() == {"datastore": []}
assert l1.rpc.listdatastore("somekey") == {"datastore": []}
# Add entries.
somedata = b'somedata'.hex()
somedata_expect = {'key': ['somekey'],
'generation': 0,
'hex': somedata,
'string': 'somedata'}
assert l1.rpc.datastore(key='somekey', hex=somedata) == somedata_expect
somedata = b"somedata".hex()
somedata_expect = {
"key": ["somekey"],
"generation": 0,
"hex": somedata,
"string": "somedata",
}
assert l1.rpc.datastore(key="somekey", hex=somedata) == somedata_expect
assert l1.rpc.listdatastore() == {'datastore': [somedata_expect]}
assert l1.rpc.listdatastore('somekey') == {'datastore': [somedata_expect]}
assert l1.rpc.listdatastore('otherkey') == {'datastore': []}
assert l1.rpc.listdatastore() == {"datastore": [somedata_expect]}
assert l1.rpc.listdatastore("somekey") == {"datastore": [somedata_expect]}
assert l1.rpc.listdatastore("otherkey") == {"datastore": []}
# Cannot add by default.
with pytest.raises(RpcError, match='already exists'):
l1.rpc.datastore(key='somekey', hex=somedata)
with pytest.raises(RpcError, match="already exists"):
l1.rpc.datastore(key="somekey", hex=somedata)
with pytest.raises(RpcError, match='already exists'):
l1.rpc.datastore(key='somekey', hex=somedata, mode="must-create")
with pytest.raises(RpcError, match="already exists"):
l1.rpc.datastore(key="somekey", hex=somedata, mode="must-create")
# But can insist on replace.
l1.rpc.datastore(key='somekey', hex=somedata[:-4], mode="must-replace")
assert only_one(l1.rpc.listdatastore('somekey')['datastore'])['hex'] == somedata[:-4]
l1.rpc.datastore(key="somekey", hex=somedata[:-4], mode="must-replace")
assert (
only_one(l1.rpc.listdatastore("somekey")["datastore"])["hex"] == somedata[:-4]
)
# And append works.
l1.rpc.datastore(key='somekey', hex=somedata[-4:-2], mode="must-append")
assert only_one(l1.rpc.listdatastore('somekey')['datastore'])['hex'] == somedata[:-2]
l1.rpc.datastore(key='somekey', hex=somedata[-2:], mode="create-or-append")
assert only_one(l1.rpc.listdatastore('somekey')['datastore'])['hex'] == somedata
l1.rpc.datastore(key="somekey", hex=somedata[-4:-2], mode="must-append")
assert (
only_one(l1.rpc.listdatastore("somekey")["datastore"])["hex"] == somedata[:-2]
)
l1.rpc.datastore(key="somekey", hex=somedata[-2:], mode="create-or-append")
assert only_one(l1.rpc.listdatastore("somekey")["datastore"])["hex"] == somedata
# Generation will have increased due to three ops above.
somedata_expect['generation'] += 3
assert l1.rpc.listdatastore() == {'datastore': [somedata_expect]}
somedata_expect["generation"] += 3
assert l1.rpc.listdatastore() == {"datastore": [somedata_expect]}
# Can't replace or append non-existing records if we say not to
with pytest.raises(RpcError, match='does not exist'):
l1.rpc.datastore(key='otherkey', hex=somedata, mode="must-replace")
with pytest.raises(RpcError, match="does not exist"):
l1.rpc.datastore(key="otherkey", hex=somedata, mode="must-replace")
with pytest.raises(RpcError, match='does not exist'):
l1.rpc.datastore(key='otherkey', hex=somedata, mode="must-append")
with pytest.raises(RpcError, match="does not exist"):
l1.rpc.datastore(key="otherkey", hex=somedata, mode="must-append")
otherdata = b'otherdata'.hex()
otherdata_expect = {'key': ['otherkey'],
'generation': 0,
'hex': otherdata,
'string': 'otherdata'}
assert l1.rpc.datastore(key='otherkey', string='otherdata', mode="create-or-append") == otherdata_expect
otherdata = b"otherdata".hex()
otherdata_expect = {
"key": ["otherkey"],
"generation": 0,
"hex": otherdata,
"string": "otherdata",
}
assert (
l1.rpc.datastore(key="otherkey", string="otherdata", mode="create-or-append")
== otherdata_expect
)
assert l1.rpc.listdatastore('somekey') == {'datastore': [somedata_expect]}
assert l1.rpc.listdatastore('otherkey') == {'datastore': [otherdata_expect]}
assert l1.rpc.listdatastore('badkey') == {'datastore': []}
assert l1.rpc.listdatastore("somekey") == {"datastore": [somedata_expect]}
assert l1.rpc.listdatastore("otherkey") == {"datastore": [otherdata_expect]}
assert l1.rpc.listdatastore("badkey") == {"datastore": []}
ds = l1.rpc.listdatastore()
# Order is undefined!
assert (ds == {'datastore': [somedata_expect, otherdata_expect]}
or ds == {'datastore': [otherdata_expect, somedata_expect]})
assert ds == {"datastore": [somedata_expect, otherdata_expect]} or ds == {
"datastore": [otherdata_expect, somedata_expect]
}
assert l1.rpc.deldatastore('somekey') == somedata_expect
assert l1.rpc.listdatastore() == {'datastore': [otherdata_expect]}
assert l1.rpc.listdatastore('somekey') == {'datastore': []}
assert l1.rpc.listdatastore('otherkey') == {'datastore': [otherdata_expect]}
assert l1.rpc.listdatastore('badkey') == {'datastore': []}
assert l1.rpc.listdatastore() == {'datastore': [otherdata_expect]}
assert l1.rpc.deldatastore("somekey") == somedata_expect
assert l1.rpc.listdatastore() == {"datastore": [otherdata_expect]}
assert l1.rpc.listdatastore("somekey") == {"datastore": []}
assert l1.rpc.listdatastore("otherkey") == {"datastore": [otherdata_expect]}
assert l1.rpc.listdatastore("badkey") == {"datastore": []}
assert l1.rpc.listdatastore() == {"datastore": [otherdata_expect]}
# if it's not a string, won't print
badstring_expect = {'key': ['badstring'],
'generation': 0,
'hex': '00'}
assert l1.rpc.datastore(key='badstring', hex='00') == badstring_expect
assert l1.rpc.listdatastore('badstring') == {'datastore': [badstring_expect]}
assert l1.rpc.deldatastore('badstring') == badstring_expect
badstring_expect = {"key": ["badstring"], "generation": 0, "hex": "00"}
assert l1.rpc.datastore(key="badstring", hex="00") == badstring_expect
assert l1.rpc.listdatastore("badstring") == {"datastore": [badstring_expect]}
assert l1.rpc.deldatastore("badstring") == badstring_expect
# It's persistent
l1.restart()
assert l1.rpc.listdatastore() == {'datastore': [otherdata_expect]}
assert l1.rpc.listdatastore() == {"datastore": [otherdata_expect]}
# We can insist generation match on update.
with pytest.raises(RpcError, match='generation is different'):
l1.rpc.datastore(key='otherkey', hex='00', mode='must-replace',
generation=otherdata_expect['generation'] + 1)
with pytest.raises(RpcError, match="generation is different"):
l1.rpc.datastore(
key="otherkey",
hex="00",
mode="must-replace",
generation=otherdata_expect["generation"] + 1,
)
otherdata_expect['generation'] += 1
otherdata_expect['string'] += 'a'
otherdata_expect['hex'] += '61'
assert (l1.rpc.datastore(key='otherkey', string='otherdataa',
mode='must-replace',
generation=otherdata_expect['generation'] - 1)
== otherdata_expect)
assert l1.rpc.listdatastore() == {'datastore': [otherdata_expect]}
otherdata_expect["generation"] += 1
otherdata_expect["string"] += "a"
otherdata_expect["hex"] += "61"
assert (
l1.rpc.datastore(
key="otherkey",
string="otherdataa",
mode="must-replace",
generation=otherdata_expect["generation"] - 1,
)
== otherdata_expect
)
assert l1.rpc.listdatastore() == {"datastore": [otherdata_expect]}
# We can insist generation match on delete.
with pytest.raises(RpcError, match='generation is different'):
l1.rpc.deldatastore(key='otherkey',
generation=otherdata_expect['generation'] + 1)
with pytest.raises(RpcError, match="generation is different"):
l1.rpc.deldatastore(
key="otherkey", generation=otherdata_expect["generation"] + 1
)
assert (l1.rpc.deldatastore(key='otherkey',
generation=otherdata_expect['generation'])
== otherdata_expect)
assert l1.rpc.listdatastore() == {'datastore': []}
assert (
l1.rpc.deldatastore(key="otherkey", generation=otherdata_expect["generation"])
== otherdata_expect
)
assert l1.rpc.listdatastore() == {"datastore": []}
def test_upgrade(node_factory):
l1 = node_factory.get_node()
datastore = shelve.open(os.path.join(l1.daemon.lightning_dir, 'regtest', 'datastore.dat'), 'c')
datastore['foo'] = b'foodata'
datastore['bar'] = b'bardata'
datastore = shelve.open(
os.path.join(l1.daemon.lightning_dir, "regtest", "datastore.dat"), "c"
)
datastore["foo"] = b"foodata"
datastore["bar"] = b"bardata"
datastore.close()
# This "fails" because it unloads itself.
@ -133,83 +156,108 @@ def test_upgrade(node_factory):
pass
# There's no upgrade if there's a real datastore.
if l1.daemon.is_in_log('there is a real datastore command'):
if l1.daemon.is_in_log("there is a real datastore command"):
return
l1.daemon.wait_for_log('Upgrading store to have generation numbers')
wait_for(lambda: not os.path.exists(os.path.join(l1.daemon.lightning_dir,
'regtest',
'datastore.dat')))
l1.daemon.wait_for_log("Upgrading store to have generation numbers")
wait_for(
lambda: not os.path.exists(
os.path.join(l1.daemon.lightning_dir, "regtest", "datastore.dat")
)
)
vals = l1.rpc.listdatastore()['datastore']
assert vals == [{'key': ['bar'],
'generation': 0,
'hex': b'bardata'.hex(),
'string': 'bardata'},
{'key': ['foo'],
'generation': 0,
'hex': b'foodata'.hex(),
'string': 'foodata'}]
vals = l1.rpc.listdatastore()["datastore"]
assert vals == [
{"key": ["bar"], "generation": 0, "hex": b"bardata".hex(), "string": "bardata"},
{"key": ["foo"], "generation": 0, "hex": b"foodata".hex(), "string": "foodata"},
]
def test_datastore_keylist(node_factory):
l1 = node_factory.get_node(options={'plugin': plugin_path})
l1 = node_factory.get_node(options={"plugin": plugin_path})
time.sleep(5)
# Starts empty
assert l1.rpc.listdatastore() == {'datastore': []}
assert l1.rpc.listdatastore(['a']) == {'datastore': []}
assert l1.rpc.listdatastore(['a', 'b']) == {'datastore': []}
assert l1.rpc.listdatastore() == {"datastore": []}
assert l1.rpc.listdatastore(["a"]) == {"datastore": []}
assert l1.rpc.listdatastore(["a", "b"]) == {"datastore": []}
# Cannot add child to existing!
l1.rpc.datastore(key='a', string='aval')
with pytest.raises(RpcError, match=r'1206.*Parent key \[a\] exists'):
l1.rpc.datastore(key=['a', 'b'], string='abval',
mode='create-or-replace')
l1.rpc.datastore(key="a", string="aval")
with pytest.raises(RpcError, match=r"1206.*Parent key \[a\] exists"):
l1.rpc.datastore(key=["a", "b"], string="abval", mode="create-or-replace")
# Listing subkey gives DNE.
assert l1.rpc.listdatastore(['a', 'b']) == {'datastore': []}
l1.rpc.deldatastore(key=['a'])
assert l1.rpc.listdatastore(["a", "b"]) == {"datastore": []}
l1.rpc.deldatastore(key=["a"])
# Create child key.
l1.rpc.datastore(key=['a', 'b'], string='abval')
assert l1.rpc.listdatastore() == {'datastore': [{'key': ['a']}]}
assert l1.rpc.listdatastore(key=['a']) == {'datastore': [{'key': ['a', 'b'],
'generation': 0,
'string': 'abval',
'hex': b'abval'.hex()}]}
l1.rpc.datastore(key=["a", "b"], string="abval")
assert l1.rpc.listdatastore() == {"datastore": [{"key": ["a"]}]}
assert l1.rpc.listdatastore(key=["a"]) == {
"datastore": [
{
"key": ["a", "b"],
"generation": 0,
"string": "abval",
"hex": b"abval".hex(),
}
]
}
# Cannot create key over that
with pytest.raises(RpcError, match='has children'):
l1.rpc.datastore(key='a', string='aval', mode='create-or-replace')
with pytest.raises(RpcError, match="has children"):
l1.rpc.datastore(key="a", string="aval", mode="create-or-replace")
# Can create another key.
l1.rpc.datastore(key=['a', 'b2'], string='ab2val')
assert l1.rpc.listdatastore() == {'datastore': [{'key': ['a']}]}
assert l1.rpc.listdatastore(key=['a']) == {'datastore': [{'key': ['a', 'b'],
'string': 'abval',
'generation': 0,
'hex': b'abval'.hex()},
{'key': ['a', 'b2'],
'string': 'ab2val',
'generation': 0,
'hex': b'ab2val'.hex()}]}
l1.rpc.datastore(key=["a", "b2"], string="ab2val")
assert l1.rpc.listdatastore() == {"datastore": [{"key": ["a"]}]}
assert l1.rpc.listdatastore(key=["a"]) == {
"datastore": [
{
"key": ["a", "b"],
"string": "abval",
"generation": 0,
"hex": b"abval".hex(),
},
{
"key": ["a", "b2"],
"string": "ab2val",
"generation": 0,
"hex": b"ab2val".hex(),
},
]
}
# Can create subkey.
l1.rpc.datastore(key=['a', 'b3', 'c'], string='ab2val')
assert l1.rpc.listdatastore() == {'datastore': [{'key': ['a']}]}
assert l1.rpc.listdatastore(key=['a']) == {'datastore': [{'key': ['a', 'b'],
'string': 'abval',
'generation': 0,
'hex': b'abval'.hex()},
{'key': ['a', 'b2'],
'string': 'ab2val',
'generation': 0,
'hex': b'ab2val'.hex()},
{'key': ['a', 'b3']}]}
l1.rpc.datastore(key=["a", "b3", "c"], string="ab2val")
assert l1.rpc.listdatastore() == {"datastore": [{"key": ["a"]}]}
assert l1.rpc.listdatastore(key=["a"]) == {
"datastore": [
{
"key": ["a", "b"],
"string": "abval",
"generation": 0,
"hex": b"abval".hex(),
},
{
"key": ["a", "b2"],
"string": "ab2val",
"generation": 0,
"hex": b"ab2val".hex(),
},
{"key": ["a", "b3"]},
]
}
# Can update subkey
l1.rpc.datastore(key=['a', 'b3', 'c'], string='2', mode='must-append')
assert l1.rpc.listdatastore(key=['a', 'b3', 'c']) == {'datastore': [{'key': ['a', 'b3', 'c'],
'string': 'ab2val2',
'generation': 1,
'hex': b'ab2val2'.hex()}]}
l1.rpc.datastore(key=["a", "b3", "c"], string="2", mode="must-append")
assert l1.rpc.listdatastore(key=["a", "b3", "c"]) == {
"datastore": [
{
"key": ["a", "b3", "c"],
"string": "ab2val2",
"generation": 1,
"hex": b"ab2val2".hex(),
}
]
}

59
donations/donations.py
View file

@ -1,5 +1,5 @@
#!/usr/bin/env python3
""" A small donation service so that users can request ln invoices
"""A small donation service so that users can request ln invoices
This plugin spins up a small flask server that provides a form to
users who wish to donate some money to the owner of the lightning
@ -15,6 +15,7 @@ you can see a demo of the plugin (and leave a tip) directly at:
LICENSE: MIT / APACHE
"""
import base64
import multiprocessing
import qrcode
@ -34,11 +35,14 @@ plugin = Plugin()
class DonationForm(FlaskForm):
"""Form for donations """
amount = IntegerField("Enter how many Satoshis you want to donate!",
validators=[DataRequired(), NumberRange(min=1, max=16666666)])
"""Form for donations"""
amount = IntegerField(
"Enter how many Satoshis you want to donate!",
validators=[DataRequired(), NumberRange(min=1, max=16666666)],
)
description = StringField("Leave a comment (displayed publically)")
submit = SubmitField('Donate')
submit = SubmitField("Donate")
def make_base64_qr_code(bolt11):
@ -94,18 +98,26 @@ def donation_form():
donations.append((ts, satoshis, description))
if b11 is not None:
return render_template("donation.html", donations=sorted(donations, reverse=True), form=form, bolt11=b11, qr=qr, label=label)
return render_template(
"donation.html",
donations=sorted(donations, reverse=True),
form=form,
bolt11=b11,
qr=qr,
label=label,
)
else:
return render_template("donation.html", donations=sorted(donations, reverse=True), form=form)
return render_template(
"donation.html", donations=sorted(donations, reverse=True), form=form
)
def worker(port):
app = Flask(__name__)
# FIXME: use hexlified hsm secret or something else
app.config['SECRET_KEY'] = 'you-will-never-guess-this'
app.add_url_rule('/donation', 'donation',
donation_form, methods=["GET", "POST"])
app.add_url_rule('/is_invoice_paid/<label>', 'ajax', ajax)
app.config["SECRET_KEY"] = "you-will-never-guess-this"
app.add_url_rule("/donation", "donation", donation_form, methods=["GET", "POST"])
app.add_url_rule("/is_invoice_paid/<label>", "ajax", ajax)
Bootstrap(app)
app.run(host="0.0.0.0", port=port)
return
@ -119,7 +131,8 @@ def start_server(port):
return False, "server already running"
p = multiprocessing.Process(
target=worker, args=[port], name="server on port {}".format(port))
target=worker, args=[port], name="server on port {}".format(port)
)
p.daemon = True
jobs[port] = p
@ -138,7 +151,7 @@ def stop_server(port):
return False
@plugin.method('donationserver')
@plugin.method("donationserver")
def donationserver(command="start", port=8088):
"""Starts a donationserver with {start/stop/restart} on {port}.
@ -159,14 +172,18 @@ def donationserver(command="start", port=8088):
try:
port = int(port)
except Exception:
port = int(plugin.options['donations-web-port']['value'])
port = int(plugin.options["donations-web-port"]["value"])
if command == "list":
return "servers running on the following ports: {}".format(list(jobs.keys()))
if command == "start":
if port in jobs:
return "Server already running on port {}. Maybe restart the server?".format(port)
return (
"Server already running on port {}. Maybe restart the server?".format(
port
)
)
suc = start_server(port)
if suc:
return "started server successfully on port {}".format(port)
@ -189,23 +206,19 @@ def donationserver(command="start", port=8088):
plugin.add_option(
'donations-autostart',
'true',
'Should the donation server start automatically'
"donations-autostart", "true", "Should the donation server start automatically"
)
plugin.add_option(
'donations-web-port',
'8088',
'Which port should the donation server listen to?'
"donations-web-port", "8088", "Which port should the donation server listen to?"
)
@plugin.init()
def init(options, configuration, plugin):
port = int(options['donations-web-port'])
port = int(options["donations-web-port"])
if options['donations-autostart'].lower() in ['true', '1']:
if options["donations-autostart"].lower() in ["true", "1"]:
start_server(port)

6
donations/test_donations.py
View file

@ -18,11 +18,11 @@ def test_donation_starts(node_factory):
def test_donation_server(node_factory):
pluginopt = {'plugin': plugin_path, 'donations-autostart': False}
pluginopt = {"plugin": plugin_path, "donations-autostart": False}
l1 = node_factory.get_node(options=pluginopt, allow_warning=True)
port = reserve()
l1.rpc.donationserver('start', port)
l1.rpc.donationserver("start", port)
l1.daemon.wait_for_log("plugin-donations.py:.*Serving Flask app 'donations'")
l1.daemon.wait_for_log("plugin-donations.py:.*Running on all addresses")
msg = l1.rpc.donationserver("stop", port)
assert msg == f'stopped server on port {port}'
assert msg == f"stopped server on port {port}"

16
feeadjuster/clnutils.py
View file

@ -10,14 +10,14 @@ def cln_parse_rpcversion(string):
make sure we can read all of them for (the next 80 years).
"""
rpcversion = string
if rpcversion.startswith('v'): # strip leading 'v'
if rpcversion.startswith("v"): # strip leading 'v'
rpcversion = rpcversion[1:]
if rpcversion.find('-') != -1: # strip mods
rpcversion = rpcversion[:rpcversion.find('-')]
if re.search('.*(rc[\\d]*)$', rpcversion): # strip release candidates
rpcversion = rpcversion[:rpcversion.find('rc')]
if rpcversion.count('.') == 1: # imply patch version 0 if not given
rpcversion = rpcversion + '.0'
if rpcversion.find("-") != -1: # strip mods
rpcversion = rpcversion[: rpcversion.find("-")]
if re.search(".*(rc[\\d]*)$", rpcversion): # strip release candidates
rpcversion = rpcversion[: rpcversion.find("rc")]
if rpcversion.count(".") == 1: # imply patch version 0 if not given
rpcversion = rpcversion + ".0"
# split and convert numeric string parts to actual integers
return list(map(int, rpcversion.split('.')))
return list(map(int, rpcversion.split(".")))

220
feeadjuster/feeadjuster.py
View file

@ -25,12 +25,14 @@ plugin.mutex.acquire()
def read_excludelist():
try:
with open('feeadjuster-exclude.list') as file:
with open("feeadjuster-exclude.list") as file:
exclude_list = [l.rstrip("\n") for l in file]
print("Excluding the channels with the nodes:", exclude_list)
except FileNotFoundError:
exclude_list = []
print("There is no feeadjuster-exclude.list given, applying the options to the channels with all peers.")
print(
"There is no feeadjuster-exclude.list given, applying the options to the channels with all peers."
)
return exclude_list
@ -59,7 +61,7 @@ def get_ratio_soft(our_percentage):
"""
Basic algorithm: lesser difference than default
"""
return 10**(0.5 - our_percentage)
return 10 ** (0.5 - our_percentage)
def get_ratio(our_percentage):
@ -67,38 +69,46 @@ def get_ratio(our_percentage):
Basic algorithm: the farther we are from the optimal case, the more we
bump/lower.
"""
return 50**(0.5 - our_percentage)
return 50 ** (0.5 - our_percentage)
def get_ratio_hard(our_percentage):
"""
Return value is between 0 and 20: 0 -> 20; 0.5 -> 1; 1 -> 0
"""
return 100**(0.5 - our_percentage) * (1 - our_percentage) * 2
return 100 ** (0.5 - our_percentage) * (1 - our_percentage) * 2
def get_peerchannels(plugin: Plugin):
""" Helper to reconstruct `listpeerchannels` for older CLN versions """
"""Helper to reconstruct `listpeerchannels` for older CLN versions"""
# first the good case
if plugin.rpcversion[0] > 23 or plugin.rpcversion[0] == 23 and plugin.rpcversion[1] >= 2:
if (
plugin.rpcversion[0] > 23
or plugin.rpcversion[0] == 23
and plugin.rpcversion[1] >= 2
):
return plugin.rpc.listpeerchannels()["channels"]
# now the workaround
channels = []
peers = plugin.rpc.listpeers()['peers']
peers = plugin.rpc.listpeers()["peers"]
for peer in peers:
newchans = peer['channels']
newchans = peer["channels"]
for ch in newchans:
ch['peer_id'] = peer['id'] # all we need is to set the 'peer_id'
ch["peer_id"] = peer["id"] # all we need is to set the 'peer_id'
channels.extend(newchans)
return channels
def get_config(plugin: Plugin, config: str):
""" Helper to reconstruct `listconfigs` for older CLN versions """
"""Helper to reconstruct `listconfigs` for older CLN versions"""
# versions >=23.08 return a configs object and value_* fields
if plugin.rpcversion[0] > 23 or plugin.rpcversion[0] == 23 and plugin.rpcversion[1] >= 8:
if (
plugin.rpcversion[0] > 23
or plugin.rpcversion[0] == 23
and plugin.rpcversion[1] >= 8
):
result = plugin.rpc.listconfigs(config)["configs"]
assert len(result)>0
assert len(result) > 0
conf_obj = result[config]
if "value_str" in conf_obj:
return conf_obj["value_str"]
@ -120,8 +130,8 @@ def get_config(plugin: Plugin, config: str):
def get_peer_id_for_scid(plugin: Plugin, scid: str):
for ch in plugin.peerchannels:
if ch.get('short_channel_id') == scid:
return ch['peer_id']
if ch.get("short_channel_id") == scid:
return ch["peer_id"]
return None
@ -135,7 +145,10 @@ def get_peerchannel(plugin: Plugin, scid: str):
def get_chan_fees(plugin: Plugin, scid: str):
channel = get_peerchannel(plugin, scid)
assert channel is not None
return {"base": channel["fee_base_msat"], "ppm": channel["fee_proportional_millionths"]}
return {
"base": channel["fee_base_msat"],
"ppm": channel["fee_proportional_millionths"],
}
def get_fees_global(plugin: Plugin, scid: str):
@ -143,36 +156,56 @@ def get_fees_global(plugin: Plugin, scid: str):
def get_fees_median(plugin: Plugin, scid: str):
""" Median fees from peers or peer.
"""Median fees from peers or peer.
The assumption is that our node competes in fees to other peers of a peer.
"""
peer_id = get_peer_id_for_scid(plugin, scid)
assert peer_id is not None
if plugin.listchannels_by_dst:
plugin.channels = plugin.rpc.call("listchannels",
{"destination": peer_id})['channels']
channels_to_peer = [ch for ch in plugin.channels
if ch['destination'] == peer_id
and ch['source'] != plugin.our_node_id]
plugin.channels = plugin.rpc.call("listchannels", {"destination": peer_id})[
"channels"
]
channels_to_peer = [
ch
for ch in plugin.channels
if ch["destination"] == peer_id and ch["source"] != plugin.our_node_id
]
if len(channels_to_peer) == 0:
return None
# fees > ~5000 (base and ppm) are currently about top 2% of network fee extremists
fees_ppm = [ch['fee_per_millionth'] for ch in channels_to_peer if 0 < ch['fee_per_millionth'] < 5000]
fees_base = [ch['base_fee_millisatoshi'] for ch in channels_to_peer if 0 < ch['base_fee_millisatoshi'] < 5000]
fees_ppm = [
ch["fee_per_millionth"]
for ch in channels_to_peer
if 0 < ch["fee_per_millionth"] < 5000
]
fees_base = [
ch["base_fee_millisatoshi"]
for ch in channels_to_peer
if 0 < ch["base_fee_millisatoshi"] < 5000
]
# if lists are emtpy use default values, otherwise statistics.median will fail.
if len(fees_ppm) == 0:
fees_ppm = [int(plugin.adj_ppmfee / plugin.median_multiplier)]
if len(fees_base) == 0:
fees_base = [int(plugin.adj_basefee / plugin.median_multiplier)]
return {"base": statistics.median(fees_base) * plugin.median_multiplier,
"ppm": statistics.median(fees_ppm) * plugin.median_multiplier}
return {
"base": statistics.median(fees_base) * plugin.median_multiplier,
"ppm": statistics.median(fees_ppm) * plugin.median_multiplier,
}
def setchannelfee(plugin: Plugin, scid: str, base: int, ppm: int, min_htlc: int = None, max_htlc: int = None):
def setchannelfee(
plugin: Plugin,
scid: str,
base: int,
ppm: int,
min_htlc: int = None,
max_htlc: int = None,
):
fees = get_chan_fees(plugin, scid)
if fees is None or base == fees['base'] and ppm == fees['ppm']:
if fees is None or base == fees["base"] and ppm == fees["ppm"]:
return False
try:
plugin.rpc.setchannel(scid, base, ppm, min_htlc, max_htlc)
@ -196,8 +229,10 @@ def significant_update(plugin: Plugin, scid: str):
update_threshold_abs += update_threshold_abs * random.uniform(-0.015, 0.015)
last_percentage = last_liquidity / channel["total"]
percentage = channel["our"] / channel["total"]
if (abs(last_percentage - percentage) > update_threshold
or abs(last_liquidity - channel["our"]) > update_threshold_abs):
if (
abs(last_percentage - percentage) > update_threshold
or abs(last_liquidity - channel["our"]) > update_threshold_abs
):
return True
return False
@ -205,7 +240,10 @@ def significant_update(plugin: Plugin, scid: str):
def maybe_adjust_fees(plugin: Plugin, scids: list):
channels_adjusted = 0
for scid in scids:
if scid in plugin.exclude_list or get_peer_id_for_scid(plugin, scid) in plugin.exclude_list:
if (
scid in plugin.exclude_list
or get_peer_id_for_scid(plugin, scid) in plugin.exclude_list
):
continue
our = plugin.adj_balances[scid]["our"]
total = plugin.adj_balances[scid]["total"]
@ -216,14 +254,16 @@ def maybe_adjust_fees(plugin: Plugin, scids: list):
# select ideal values per channel
fees = plugin.fee_strategy(plugin, scid)
if fees is not None:
ppm = int(fees['ppm'])
ppm = int(fees["ppm"])
if plugin.basefee:
base = int(fees['base'])
base = int(fees["base"])
# reset to normal fees if imbalance is not high enough
if (percentage > plugin.imbalance and percentage < 1 - plugin.imbalance):
if percentage > plugin.imbalance and percentage < 1 - plugin.imbalance:
if setchannelfee(plugin, scid, base, ppm):
plugin.log(f"Set default fees as imbalance is too low for {scid}: ppm {ppm} base {base}msat")
plugin.log(
f"Set default fees as imbalance is too low for {scid}: ppm {ppm} base {base}msat"
)
plugin.adj_balances[scid]["last_liquidity"] = our
channels_adjusted += 1
continue
@ -235,11 +275,17 @@ def maybe_adjust_fees(plugin: Plugin, scids: list):
assert 0 <= percentage and percentage <= 1
ratio = plugin.get_ratio(percentage)
if plugin.max_htlc_steps >= 1:
max_htlc = int(total * math.ceil(plugin.max_htlc_steps * percentage) / plugin.max_htlc_steps)
max_htlc = int(
total
* math.ceil(plugin.max_htlc_steps * percentage)
/ plugin.max_htlc_steps
)
else:
max_htlc = None
if setchannelfee(plugin, scid, base, int(ppm * ratio), None, max_htlc):
plugin.log(f"Adjusted fees of {scid} with a ratio of {ratio}: ppm {int(ppm * ratio)} base {base}msat max_htlc {max_htlc}")
plugin.log(
f"Adjusted fees of {scid} with a ratio of {ratio}: ppm {int(ppm * ratio)} base {base}msat max_htlc {max_htlc}"
)
plugin.adj_balances[scid]["last_liquidity"] = our
channels_adjusted += 1
plugin.log("maybe_adjust_fees done", "debug")
@ -281,7 +327,7 @@ def forward_event(plugin: Plugin, forward_event: dict, **kwargs):
plugin.mutex.acquire(blocking=True)
plugin.peerchannels = get_peerchannels(plugin)
if plugin.fee_strategy == get_fees_median and not plugin.listchannels_by_dst:
plugin.channels = plugin.rpc.listchannels()['channels']
plugin.channels = plugin.rpc.listchannels()["channels"]
in_scid = forward_event["in_channel"]
out_scid = forward_event["out_channel"]
get_new_balance(plugin, in_scid)
@ -311,7 +357,7 @@ def feeadjust(plugin: Plugin, scid: str = None):
plugin.mutex.acquire(blocking=True)
plugin.peerchannels = get_peerchannels(plugin)
if plugin.fee_strategy == get_fees_median and not plugin.listchannels_by_dst:
plugin.channels = plugin.rpc.listchannels()['channels']
plugin.channels = plugin.rpc.listchannels()["channels"]
channels_adjusted = 0
plugin.exclude_list = read_excludelist()
@ -324,7 +370,7 @@ def feeadjust(plugin: Plugin, scid: str = None):
continue
plugin.adj_balances[_scid] = {
"our": int(chan["to_us_msat"]),
"total": int(chan["total_msat"])
"total": int(chan["total_msat"]),
}
channels_adjusted += maybe_adjust_fees(plugin, [_scid])
msg = f"{channels_adjusted} channel(s) adjusted"
@ -339,7 +385,9 @@ def feeadjuster_toggle(plugin: Plugin, value: bool = None):
The status will be set to value.
"""
msg = {"forward_event_subscription": {"previous": plugin.forward_event_subscription}}
msg = {
"forward_event_subscription": {"previous": plugin.forward_event_subscription}
}
if value is None:
plugin.forward_event_subscription = not plugin.forward_event_subscription
else:
@ -352,27 +400,32 @@ def feeadjuster_toggle(plugin: Plugin, value: bool = None):
def init(options: dict, configuration: dict, plugin: Plugin, **kwargs):
# do all the stuff that needs to be done just once ...
plugin.getinfo = plugin.rpc.getinfo()
plugin.rpcversion = cln_parse_rpcversion(plugin.getinfo.get('version'))
plugin.rpcversion = cln_parse_rpcversion(plugin.getinfo.get("version"))
plugin.our_node_id = plugin.getinfo["id"]
plugin.deactivate_fuzz = options.get("feeadjuster-deactivate-fuzz")
plugin.forward_event_subscription = not options.get("feeadjuster-deactivate-fee-update")
plugin.forward_event_subscription = not options.get(
"feeadjuster-deactivate-fee-update"
)
plugin.update_threshold = float(options.get("feeadjuster-threshold"))
plugin.update_threshold_abs = Millisatoshi(options.get("feeadjuster-threshold-abs"))
plugin.big_enough_liquidity = Millisatoshi(options.get("feeadjuster-enough-liquidity"))
plugin.big_enough_liquidity = Millisatoshi(
options.get("feeadjuster-enough-liquidity")
)
plugin.imbalance = float(options.get("feeadjuster-imbalance"))
plugin.max_htlc_steps = int(options.get("feeadjuster-max-htlc-steps"))
plugin.basefee = bool(options.get("feeadjuster-basefee"))
adjustment_switch = {
"soft": get_ratio_soft,
"hard": get_ratio_hard,
"default": get_ratio
"default": get_ratio,
}
plugin.get_ratio = adjustment_switch.get(options.get("feeadjuster-adjustment-method"), get_ratio)
fee_strategy_switch = {
"global": get_fees_global,
"median": get_fees_median
}
plugin.fee_strategy = fee_strategy_switch.get(options.get("feeadjuster-feestrategy"), get_fees_global)
plugin.get_ratio = adjustment_switch.get(
options.get("feeadjuster-adjustment-method"), get_ratio
)
fee_strategy_switch = {"global": get_fees_global, "median": get_fees_median}
plugin.fee_strategy = fee_strategy_switch.get(
options.get("feeadjuster-feestrategy"), get_fees_global
)
plugin.median_multiplier = float(options.get("feeadjuster-median-multiplier"))
plugin.adj_basefee = get_config(plugin, "fee-base")
if plugin.adj_basefee is None:
@ -389,9 +442,18 @@ def init(options: dict, configuration: dict, plugin: Plugin, **kwargs):
# detect if server supports the new listchannels by `destination` (#4614)
plugin.listchannels_by_dst = False
rpchelp = plugin.rpc.help().get('help')
if len([c for c in rpchelp if c["command"].startswith("listchannels ")
and "destination" in c["command"]]) == 1:
rpchelp = plugin.rpc.help().get("help")
if (
len(
[
c
for c in rpchelp
if c["command"].startswith("listchannels ")
and "destination" in c["command"]
]
)
== 1
):
plugin.listchannels_by_dst = True
# Detect if server supports new 'setchannel' command over setchannelfee.
@ -399,19 +461,21 @@ def init(options: dict, configuration: dict, plugin: Plugin, **kwargs):
if len([c for c in rpchelp if c["command"].startswith("setchannel ")]) == 0:
plugin.rpc.setchannel = plugin.rpc.setchannelfee
plugin.log(f"Plugin feeadjuster initialized "
f"({plugin.adj_basefee} base / {plugin.adj_ppmfee} ppm) with an "
f"imbalance of {int(100 * plugin.imbalance)}%/{int(100 * ( 1 - plugin.imbalance))}%, "
f"update_threshold: {int(100 * plugin.update_threshold)}%, "
f"update_threshold_abs: {plugin.update_threshold_abs}, "
f"enough_liquidity: {plugin.big_enough_liquidity}, "
f"deactivate_fuzz: {plugin.deactivate_fuzz}, "
f"forward_event_subscription: {plugin.forward_event_subscription}, "
f"adjustment_method: {plugin.get_ratio.__name__}, "
f"fee_strategy: {plugin.fee_strategy.__name__}, "
f"listchannels_by_dst: {plugin.listchannels_by_dst},"
f"max_htlc_steps: {plugin.max_htlc_steps},"
f"basefee: {plugin.basefee}")
plugin.log(
f"Plugin feeadjuster initialized "
f"({plugin.adj_basefee} base / {plugin.adj_ppmfee} ppm) with an "
f"imbalance of {int(100 * plugin.imbalance)}%/{int(100 * ( 1 - plugin.imbalance))}%, "
f"update_threshold: {int(100 * plugin.update_threshold)}%, "
f"update_threshold_abs: {plugin.update_threshold_abs}, "
f"enough_liquidity: {plugin.big_enough_liquidity}, "
f"deactivate_fuzz: {plugin.deactivate_fuzz}, "
f"forward_event_subscription: {plugin.forward_event_subscription}, "
f"adjustment_method: {plugin.get_ratio.__name__}, "
f"fee_strategy: {plugin.fee_strategy.__name__}, "
f"listchannels_by_dst: {plugin.listchannels_by_dst},"
f"max_htlc_steps: {plugin.max_htlc_steps},"
f"basefee: {plugin.basefee}"
)
plugin.mutex.release()
feeadjust(plugin)
@ -420,27 +484,27 @@ plugin.add_option(
"feeadjuster-deactivate-fuzz",
False,
"Deactivate update threshold randomization and hysterisis.",
"flag"
"flag",
)
plugin.add_option(
"feeadjuster-deactivate-fee-update",
False,
"Deactivate automatic fee updates for forward events.",
"flag"
"flag",
)
plugin.add_option(
"feeadjuster-threshold",
"0.05",
"Relative channel balance delta at which to trigger an update. Default 0.05 means 5%. "
"Note: it's also fuzzed by 1.5%",
"string"
"string",
)
plugin.add_option(
"feeadjuster-threshold-abs",
"0.001btc",
"Absolute channel balance delta at which to always trigger an update. "
"Note: it's also fuzzed by 1.5%",
"string"
"string",
)
plugin.add_option(
"feeadjuster-enough-liquidity",
@ -448,14 +512,14 @@ plugin.add_option(
"Beyond this liquidity do not adjust fees. "
"This also modifies the fee curve to achieve having this amount of liquidity. "
"Default: '0msat' (turned off).",
"string"
"string",
)
plugin.add_option(
"feeadjuster-adjustment-method",
"default",
"Adjustment method to calculate channel fee"
"Can be 'default', 'soft' for less difference or 'hard' for higher difference"
"string"
"string",
)
plugin.add_option(
"feeadjuster-imbalance",
@ -464,7 +528,7 @@ plugin.add_option(
"Default: 0.5 (always). Set higher or lower values to limit feeadjuster's "
"activity to more imbalanced channels. "
"E.g. 0.3 for '70/30'% or 0.6 for '40/60'%.",
"string"
"string",
)
plugin.add_option(
"feeadjuster-feestrategy",
@ -473,7 +537,7 @@ plugin.add_option(
"Can be 'global' to use global config or default values, "
"or 'median' to use the median fees from peers of peer "
"Default: 'global'.",
"string"
"string",
)
plugin.add_option(
"feeadjuster-median-multiplier",
@ -481,7 +545,7 @@ plugin.add_option(
"Sets the factor with which the median fee is multiplied if using the fee strategy 'median'. "
"This allows over or underbidding other nodes by a constant factor"
"Default: '1.0'.",
"string"
"string",
)
plugin.add_option(
"feeadjuster-max-htlc-steps",
@ -490,12 +554,12 @@ plugin.add_option(
"This will reduce the max htlc according to available "
"liquidity, which can reduce local routing channel failures."
"A value of 0 disables the stepping.",
"string"
"string",
)
plugin.add_option(
"feeadjuster-basefee",
False,
"Also adjust base fee dynamically. Currently only affects median strategy.",
"bool"
"bool",
)
plugin.run()

48
feeadjuster/test_clnutils.py
View file

@ -3,41 +3,41 @@ from clnutils import cln_parse_rpcversion
def test_rpcversion():
foo = cln_parse_rpcversion("0.11.2")
assert(foo[0] == 0)
assert(foo[1] == 11)
assert(foo[2] == 2)
assert foo[0] == 0
assert foo[1] == 11
assert foo[2] == 2
foo = cln_parse_rpcversion("0.11.2rc2-modded")
assert(foo[0] == 0)
assert(foo[1] == 11)
assert(foo[2] == 2)
assert foo[0] == 0
assert foo[1] == 11
assert foo[2] == 2
foo = cln_parse_rpcversion("22.11")
assert(foo[0] == 22)
assert(foo[1] == 11)
assert(foo[2] == 0)
assert foo[0] == 22
assert foo[1] == 11
assert foo[2] == 0
foo = cln_parse_rpcversion("22.11rc1")
assert(foo[0] == 22)
assert(foo[1] == 11)
assert(foo[2] == 0)
assert foo[0] == 22
assert foo[1] == 11
assert foo[2] == 0
foo = cln_parse_rpcversion("22.11rc1-modded")
assert(foo[0] == 22)
assert(foo[1] == 11)
assert(foo[2] == 0)
assert foo[0] == 22
assert foo[1] == 11
assert foo[2] == 0
foo = cln_parse_rpcversion("22.11-modded")
assert(foo[0] == 22)
assert(foo[1] == 11)
assert(foo[2] == 0)
assert foo[0] == 22
assert foo[1] == 11
assert foo[2] == 0
foo = cln_parse_rpcversion("22.11.0")
assert(foo[0] == 22)
assert(foo[1] == 11)
assert(foo[2] == 0)
assert foo[0] == 22
assert foo[1] == 11
assert foo[2] == 0
foo = cln_parse_rpcversion("22.11.1")
assert(foo[0] == 22)
assert(foo[1] == 11)
assert(foo[2] == 1)
assert foo[0] == 22
assert foo[1] == 11
assert foo[2] == 1

170
feeadjuster/test_feeadjuster.py
View file

@ -2,7 +2,6 @@ import os
import random
import string
import unittest
from pyln.testing.fixtures import * # noqa: F401,F403
from pyln.testing.utils import wait_for
@ -25,9 +24,13 @@ def test_feeadjuster_starts(node_factory):
l1.start()
# Start at 0 and 're-await' the two inits above. Otherwise this is flaky.
l1.daemon.logsearch_start = 0
l1.daemon.wait_for_logs(["Plugin feeadjuster initialized.*",
"Plugin feeadjuster initialized.*",
"Plugin feeadjuster initialized.*"])
l1.daemon.wait_for_logs(
[
"Plugin feeadjuster initialized.*",
"Plugin feeadjuster initialized.*",
"Plugin feeadjuster initialized.*",
]
)
l1.rpc.plugin_stop(plugin_path)
# We adjust fees in init
@ -35,8 +38,9 @@ def test_feeadjuster_starts(node_factory):
scid_A = l2.rpc.listpeerchannels(l1.info["id"])["channels"][0]["short_channel_id"]
scid_B = l2.rpc.listpeerchannels(l3.info["id"])["channels"][0]["short_channel_id"]
l2.rpc.plugin_start(plugin_path)
l2.daemon.wait_for_logs([f"Adjusted fees of {scid_A}.*",
f"Adjusted fees of {scid_B}.*"])
l2.daemon.wait_for_logs(
[f"Adjusted fees of {scid_A}.*", f"Adjusted fees of {scid_B}.*"]
)
def get_chan_fees(l, scid):
@ -56,10 +60,14 @@ def wait_for_not_fees(l, scids, fees):
def pay(l, ll, amount):
label = ''.join(random.choices(string.ascii_letters, k=20))
label = "".join(random.choices(string.ascii_letters, k=20))
invoice = ll.rpc.invoice(amount, label, "desc")
route = l.rpc.getroute(ll.info["id"], amount, riskfactor=0, fuzzpercent=0)
l.rpc.sendpay(route["route"], invoice["payment_hash"], payment_secret=invoice.get('payment_secret'))
l.rpc.sendpay(
route["route"],
invoice["payment_hash"],
payment_secret=invoice.get("payment_secret"),
)
l.rpc.waitsendpay(invoice["payment_hash"])
l.wait_for_htlcs()
@ -89,10 +97,13 @@ def test_feeadjuster_adjusts(node_factory):
"fee-per-satoshi": ppm_fee,
"plugin": plugin_path,
"feeadjuster-deactivate-fuzz": None,
'may_reconnect': True,
"may_reconnect": True,
}
l1, l2, l3 = node_factory.line_graph(3, opts=[{'may_reconnect': True}, l2_opts, {'may_reconnect': True}],
wait_for_announce=True)
l1, l2, l3 = node_factory.line_graph(
3,
opts=[{"may_reconnect": True}, l2_opts, {"may_reconnect": True}],
wait_for_announce=True,
)
chan_A = l2.rpc.listpeerchannels(l1.info["id"])["channels"][0]
chan_B = l2.rpc.listpeerchannels(l3.info["id"])["channels"][0]
@ -102,8 +113,7 @@ def test_feeadjuster_adjusts(node_factory):
scids = [scid_A, scid_B]
# Fees don't get updated until there is a forwarding event!
assert all([get_chan_fees(l2, scid) == (base_fee, ppm_fee)
for scid in scids])
assert all([get_chan_fees(l2, scid) == (base_fee, ppm_fee) for scid in scids])
chan_total = int(chan_A["total_msat"])
assert chan_total == int(chan_B["total_msat"])
@ -111,15 +121,20 @@ def test_feeadjuster_adjusts(node_factory):
# The first payment will trigger fee adjustment, no matter its value
amount = int(chan_total * 0.04)
pay(l1, l3, amount)
wait_for(lambda: all([get_chan_fees(l2, scid) != (base_fee, ppm_fee)
for scid in scids]))
wait_for(
lambda: all([get_chan_fees(l2, scid) != (base_fee, ppm_fee) for scid in scids])
)
wait_for(lambda: l2.daemon.is_in_log("maybe_adjust_fees done"))
# Send most of the balance to the other side..
amount = int(chan_total * 0.8)
pay(l1, l3, amount)
l2.daemon.wait_for_logs([f'Adjusted fees of {scid_A} with a ratio of 0.2',
f'Adjusted fees of {scid_B} with a ratio of 3.'])
l2.daemon.wait_for_logs(
[
f"Adjusted fees of {scid_A} with a ratio of 0.2",
f"Adjusted fees of {scid_B} with a ratio of 3.",
]
)
# ..And back, but first reconnect so old cln nodes gossip properly
l2.rpc.disconnect(l1.info["id"], True)
@ -128,8 +143,12 @@ def test_feeadjuster_adjusts(node_factory):
l2.rpc.connect(l3.info["id"], "localhost", l3.port)
sync_gossip(nodes, scids)
pay(l3, l1, amount)
l2.daemon.wait_for_logs([f'Adjusted fees of {scid_A} with a ratio of 6.',
f'Adjusted fees of {scid_B} with a ratio of 0.1'])
l2.daemon.wait_for_logs(
[
f"Adjusted fees of {scid_A} with a ratio of 6.",
f"Adjusted fees of {scid_B} with a ratio of 0.1",
]
)
# Sending a payment worth 3% of the channel balance should not trigger
# fee adjustment
@ -151,8 +170,12 @@ def test_feeadjuster_adjusts(node_factory):
# But sending another 3%-worth payment does trigger adjustment (total sent
# since last adjustment is >5%)
pay(l1, l3, amount)
l2.daemon.wait_for_logs([f'Adjusted fees of {scid_A} with a ratio of 4.',
f'Adjusted fees of {scid_B} with a ratio of 0.2'])
l2.daemon.wait_for_logs(
[
f"Adjusted fees of {scid_A} with a ratio of 4.",
f"Adjusted fees of {scid_B} with a ratio of 0.2",
]
)
def test_feeadjuster_imbalance(node_factory):
@ -174,8 +197,9 @@ def test_feeadjuster_imbalance(node_factory):
"feeadjuster-deactivate-fuzz": None,
"feeadjuster-imbalance": 0.7, # should be normalized to 30/70
}
l1, l2, l3 = node_factory.line_graph(3, opts=[{}, l2_opts, {}],
wait_for_announce=True)
l1, l2, l3 = node_factory.line_graph(
3, opts=[{}, l2_opts, {}], wait_for_announce=True
)
chan_A = l2.rpc.listpeerchannels(l1.info["id"])["channels"][0]
chan_B = l2.rpc.listpeerchannels(l3.info["id"])["channels"][0]
@ -187,25 +211,24 @@ def test_feeadjuster_imbalance(node_factory):
chan_total = int(chan_A["total_msat"])
assert chan_total == int(chan_B["total_msat"])
l2.daemon.logsearch_start = 0
l2.daemon.wait_for_log('imbalance of 30%/70%')
l2.daemon.wait_for_log("imbalance of 30%/70%")
# we force feeadjust initially to test this method and check if it applies
# default fees when balancing the channel below
l2.rpc.feeadjust()
l2.daemon.wait_for_logs([
f"Adjusted fees.*{scid_A}",
f"Adjusted fees.*{scid_B}"
])
l2.daemon.wait_for_logs([f"Adjusted fees.*{scid_A}", f"Adjusted fees.*{scid_B}"])
log_offset = len(l2.daemon.logs)
wait_for_not_fees(l2, scids, default_fees[0])
# First bring channel to somewhat of a balance
amount = int(chan_total * 0.5)
pay(l1, l3, amount)
l2.daemon.wait_for_logs([
f'Set default fees as imbalance is too low for {scid_A}',
f'Set default fees as imbalance is too low for {scid_B}'
])
l2.daemon.wait_for_logs(
[
f"Set default fees as imbalance is too low for {scid_A}",
f"Set default fees as imbalance is too low for {scid_B}",
]
)
wait_for_fees(l2, scids, default_fees[0])
# Because of the 70/30 imbalance limiter, a 15% payment must not yet trigger
@ -216,18 +239,17 @@ def test_feeadjuster_imbalance(node_factory):
# Sending another 20% must now trigger because the imbalance
pay(l1, l3, amount)
l2.daemon.wait_for_logs([
f"Adjusted fees.*{scid_A}",
f"Adjusted fees.*{scid_B}"
])
l2.daemon.wait_for_logs([f"Adjusted fees.*{scid_A}", f"Adjusted fees.*{scid_B}"])
wait_for_not_fees(l2, scids, default_fees[0])
# Bringing it back must cause default fees
pay(l3, l1, amount)
l2.daemon.wait_for_logs([
f'Set default fees as imbalance is too low for {scid_A}',
f'Set default fees as imbalance is too low for {scid_B}'
])
l2.daemon.wait_for_logs(
[
f"Set default fees as imbalance is too low for {scid_A}",
f"Set default fees as imbalance is too low for {scid_B}",
]
)
wait_for_fees(l2, scids, default_fees[0])
@ -254,8 +276,9 @@ def test_feeadjuster_big_enough_liquidity(node_factory):
}
# channels' size: 0.01btc
# between 0.001btc and 0.009btc the liquidity is big enough
l1, l2, l3 = node_factory.line_graph(3, fundamount=10**6, opts=[{}, l2_opts, {}],
wait_for_announce=True)
l1, l2, l3 = node_factory.line_graph(
3, fundamount=10**6, opts=[{}, l2_opts, {}], wait_for_announce=True
)
chan_A = l2.rpc.listpeerchannels(l1.info["id"])["channels"][0]
chan_B = l2.rpc.listpeerchannels(l3.info["id"])["channels"][0]
@ -267,24 +290,23 @@ def test_feeadjuster_big_enough_liquidity(node_factory):
chan_total = int(chan_A["total_msat"])
assert chan_total == int(chan_B["total_msat"])
l2.daemon.logsearch_start = 0
l2.daemon.wait_for_log('enough_liquidity: 100000000msat')
l2.daemon.wait_for_log("enough_liquidity: 100000000msat")
# we force feeadjust initially to test this method and check if it applies
# default fees when balancing the channel below
l2.rpc.feeadjust()
l2.daemon.wait_for_logs([
f"Adjusted fees.*{scid_A}",
f"Adjusted fees.*{scid_B}"
])
l2.daemon.wait_for_logs([f"Adjusted fees.*{scid_A}", f"Adjusted fees.*{scid_B}"])
wait_for_not_fees(l2, scids, default_fees[0])
# Bring channels to beyond big enough liquidity with 0.003btc
amount = 300000000
pay(l1, l3, amount)
l2.daemon.wait_for_logs([
f"Adjusted fees of {scid_A} with a ratio of 1.0",
f"Adjusted fees of {scid_B} with a ratio of 1.0"
])
l2.daemon.wait_for_logs(
[
f"Adjusted fees of {scid_A} with a ratio of 1.0",
f"Adjusted fees of {scid_B} with a ratio of 1.0",
]
)
log_offset = len(l2.daemon.logs)
wait_for_fees(l2, scids, default_fees[0])
@ -298,10 +320,7 @@ def test_feeadjuster_big_enough_liquidity(node_factory):
# It must trigger because the remaining liquidity is not big enough
amount = 330000000
pay(l1, l3, amount)
l2.daemon.wait_for_logs([
f"Adjusted fees.*{scid_A}",
f"Adjusted fees.*{scid_B}"
])
l2.daemon.wait_for_logs([f"Adjusted fees.*{scid_A}", f"Adjusted fees.*{scid_B}"])
wait_for_not_fees(l2, scids, default_fees[0])
@ -329,45 +348,52 @@ def test_feeadjuster_median(node_factory):
"feeadjuster-feestrategy": "median",
"feeadjuster-basefee": True,
}
l1, l2, l3, _ = node_factory.line_graph(4, opts=[opts, l2_opts, opts, opts],
wait_for_announce=True)
l1, l2, l3, _ = node_factory.line_graph(
4, opts=[opts, l2_opts, opts, opts], wait_for_announce=True
)
scid_a = l2.rpc.listpeerchannels(l1.info["id"])["channels"][0]["short_channel_id"]
scid_b = l2.rpc.listpeerchannels(l3.info["id"])["channels"][0]["short_channel_id"]
# we do a manual feeadjust
l2.rpc.feeadjust()
l2.daemon.wait_for_logs([
f"Adjusted fees.*{scid_a}",
f"Adjusted fees.*{scid_b}"
])
l2.daemon.wait_for_logs([f"Adjusted fees.*{scid_a}", f"Adjusted fees.*{scid_b}"])
# since there is only l4 with channel c towards l3, l2 should take that value
chan_b = l2.rpc.listpeerchannels(l3.info['id'])['channels'][0]
assert chan_b['fee_base_msat'] == 1337
assert chan_b['fee_proportional_millionths'] < 42 # we could do the actual ratio math, but meh
chan_b = l2.rpc.listpeerchannels(l3.info["id"])["channels"][0]
assert chan_b["fee_base_msat"] == 1337
assert (
chan_b["fee_proportional_millionths"] < 42
) # we could do the actual ratio math, but meh
def test_excludelist(node_factory, directory):
opts1 = {'may_reconnect': True}
opts2 = {'may_reconnect': True, "plugin": plugin_path,
opts1 = {"may_reconnect": True}
opts2 = {
"may_reconnect": True,
"plugin": plugin_path,
"feeadjuster-deactivate-fee-update": None,
"feeadjuster-deactivate-fuzz": None,
"feeadjuster-imbalance": 0.5}
l1, l2, l3 = node_factory.line_graph(3, opts=[opts1, opts2, opts1], wait_for_announce=True)
"feeadjuster-imbalance": 0.5,
}
l1, l2, l3 = node_factory.line_graph(
3, opts=[opts1, opts2, opts1], wait_for_announce=True
)
scid_a = l2.rpc.listpeerchannels(l1.info["id"])["channels"][0]["short_channel_id"]
scid_b = l2.rpc.listpeerchannels(l3.info["id"])["channels"][0]["short_channel_id"]
# without exclude list a notification is printed
assert l2.rpc.feeadjust(scid_a) == "1 channel(s) adjusted"
assert l2.daemon.is_in_log("There is no feeadjuster-exclude.list given, applying the options to the channels with all peers.")
assert l2.daemon.is_in_log(
"There is no feeadjuster-exclude.list given, applying the options to the channels with all peers."
)
# stop l2, create a exlude list file containing [l1_id] and restart l2
l2.stop()
l2path = os.path.join(directory, 'lightning-2', 'regtest')
file = open(os.path.join(l2path, 'feeadjuster-exclude.list'), 'w+')
file.write(l1.info['id'])
l2path = os.path.join(directory, "lightning-2", "regtest")
file = open(os.path.join(l2path, "feeadjuster-exclude.list"), "w+")
file.write(l1.info["id"])
file.close()
l2.start()
l2.daemon.is_in_log(f"Excluding the channels with the nodes: ['{l1.info['id']}']")

56
monitor/monitor.py
View file

@ -22,55 +22,63 @@ except Exception:
def monitor(plugin):
"""Monitors channels of this node."""
reply = {}
reply['num_connected'] = 0
reply['num_channels'] = 0
reply['format-hint'] = 'simple'
reply["num_connected"] = 0
reply["num_channels"] = 0
reply["format-hint"] = "simple"
peers = plugin.rpc.listpeers()
info = plugin.rpc.getinfo()
nid = info["id"]
chans = {}
states = {}
for p in peers['peers']:
for p in peers["peers"]:
channels = []
if 'channels' in p:
channels = p['channels']
elif 'num_channels' in p and p['num_channels'] > 0:
channels = plugin.rpc.listpeerchannels(p['id'])['channels']
if "channels" in p:
channels = p["channels"]
elif "num_channels" in p and p["num_channels"] > 0:
channels = plugin.rpc.listpeerchannels(p["id"])["channels"]
for c in channels:
if p['connected']:
reply['num_connected'] += 1
reply['num_channels'] += 1
state = c['state']
if p["connected"]:
reply["num_connected"] += 1
reply["num_channels"] += 1
state = c["state"]
if state in states:
states[state] += 1
else:
states[state] = 1
connected = 'connected' if p['connected'] else 'disconnected'
connected = "connected" if p["connected"] else "disconnected"
fees = "unknown onchain fees"
funding = c.get('funding_msat', None)
funding = c.get("funding_msat", None)
if funding is not None:
our_funding = funding[nid]
their_funding = funding[p['id']]
their_funding = funding[p["id"]]
if int(our_funding) == 0:
fees = "their onchain fees"
elif int(their_funding) == 0:
fees = "our onchain fees"
else:
fees = "shared onchain fees"
total = int(c['total_msat'])
ours = int(c['our_reserve_msat']) + int(c['spendable_msat'])
our_fraction = '{:4.2f}% owned by us'.format(ours * 100 / total)
tmp = "\t".join([p['id'], connected, fees, our_fraction,
c['short_channel_id'] if 'short_channel_id' in c
else 'unknown scid'])
total = int(c["total_msat"])
ours = int(c["our_reserve_msat"]) + int(c["spendable_msat"])
our_fraction = "{:4.2f}% owned by us".format(ours * 100 / total)
tmp = "\t".join(
[
p["id"],
connected,
fees,
our_fraction,
c["short_channel_id"]
if "short_channel_id" in c
else "unknown scid",
]
)
if state in chans:
chans[state].append(tmp)
else:
chans[state] = [tmp]
reply['states'] = []
reply["states"] = []
for key, value in states.items():
reply['states'].append(key + ": " + str(value))
reply['channels'] = json.dumps(chans)
reply["states"].append(key + ": " + str(value))
reply["channels"] = json.dumps(chans)
return reply

2
monitor/test_monitor.py
View file

@ -17,6 +17,6 @@ def test_monitor_starts(node_factory):
def test_monitor(node_factory):
pluginopt = {'plugin': plugin_path}
pluginopt = {"plugin": plugin_path}
l1, l2 = node_factory.line_graph(2, opts=pluginopt)
assert l1.rpc.monitor()

62
persistent-channels/persistent-channels.py
View file

@ -11,18 +11,17 @@ plugin = Plugin()
def load_state(path):
try:
state = json.loads(open(path, 'r').read())
state = json.loads(open(path, "r").read())
except Exception:
print("Could not read state file, creating a new one.")
return {'channels': {}}
return {"channels": {}}
return state
def save_state(path, state):
"""Atomically save the new state to the state_file.
"""
tmppath = path + '.tmp'
with open(tmppath, 'w') as f:
"""Atomically save the new state to the state_file."""
tmppath = path + ".tmp"
with open(tmppath, "w") as f:
f.write(json.dumps(state, indent=2))
os.rename(tmppath, path)
@ -37,11 +36,11 @@ def is_connectable(rpc, node_id):
def maybe_open_channel(desired, rpc):
peers = rpc.listpeers(desired['node_id'])['peers']
peers = rpc.listpeers(desired["node_id"])["peers"]
if 'satoshi' in desired:
desired['amount'] = "{}sat".format(desired['satoshi'])
del desired['satoshi']
if "satoshi" in desired:
desired["amount"] = "{}sat".format(desired["satoshi"])
del desired["satoshi"]
if peers == []:
# Need to connect first, and then open a channel
@ -49,37 +48,36 @@ def maybe_open_channel(desired, rpc):
# print("No address known for {}, cannot connect.".format(desired['node_id']))
try:
rpc.connect(desired['node_id'])
rpc.connect(desired["node_id"])
except RpcError as re:
print("Could not connect to peer: ".format(re.error))
print("Could not connect to peer: {}".format(re.error))
return
peer = rpc.listpeers(desired['node_id'])['peers'][0]
peer = rpc.listpeers(desired["node_id"])["peers"][0]
else:
peer = peers[0]
channel_states = [c['state'] for c in peer['channels']]
channel_states = [c["state"] for c in peer["channels"]]
if peer is None or len(peer['channels']) == 0:
if peer is None or len(peer["channels"]) == 0:
# Just open it, we don't have one yet
# TODO(cdecker) Check balance before actually opening
rpc.fundchannel(**desired)
elif 'CHANNELD_NORMAL' in channel_states:
elif "CHANNELD_NORMAL" in channel_states:
# Already in the desired state, nothing to do.
return
elif channel_states == ['ONCHAIND']:
elif channel_states == ["ONCHAIND"]:
# If our only channel is in state ONCHAIND it's probably time
# to open a new one
rpc.connect(desired['node_id'])
rpc.connect(desired["node_id"])
rpc.fundchannel(**desired)
def check_channels(plugin):
"""Load actual and desired states, and try to reconcile.
"""
"""Load actual and desired states, and try to reconcile."""
state = load_state(plugin.state_file)
print(state)
for c in state['channels'].values():
for c in state["channels"].values():
try:
maybe_open_channel(c, plugin.rpc)
except Exception:
@ -88,9 +86,8 @@ def check_channels(plugin):
Timer(30, check_channels, args=[plugin]).start()
@plugin.method('addpersistentchannel')
def add_persistent_channel(node_id, satoshi, plugin, feerate='normal',
announce=True):
@plugin.method("addpersistentchannel")
def add_persistent_channel(node_id, satoshi, plugin, feerate="normal", announce=True):
"""Add a persistent channel to the state map.
The persistent-channels plugin will ensure that the channel is
@ -99,22 +96,23 @@ def add_persistent_channel(node_id, satoshi, plugin, feerate='normal',
"""
state = load_state(plugin.state_file)
state['channels'][node_id] = {
'node_id': node_id,
'satoshi': satoshi,
'feerate': feerate,
'announce': announce,
state["channels"][node_id] = {
"node_id": node_id,
"satoshi": satoshi,
"feerate": feerate,
"announce": announce,
}
save_state(plugin.state_file, state)
maybe_open_channel(state['channels'][node_id], plugin.rpc)
maybe_open_channel(state["channels"][node_id], plugin.rpc)
@plugin.init()
def init(options, configuration, plugin):
# This is the file in which we'll store all of our state (mostly
# desired channels for now)
plugin.state_file = os.path.join(configuration['lightning-dir'],
"persistent-channels.json")
plugin.state_file = os.path.join(
configuration["lightning-dir"], "persistent-channels.json"
)
check_channels(plugin)

8
pyproject.toml
View file

@ -8,9 +8,17 @@ license = "MIT"
[tool.poetry.dependencies]
python = "^3.8"
virtualenv = "^20.25.0"
ruff = "^0.5.0"
[tool.poetry.dev-dependencies]
[tool.ruff]
include = ["pyproject.toml", ".ci/**/*.py", "backup/**/*.py",
"clearnet/**/*.py", "currencyrate/**/*.py", "datastore/**/*.py",
"donations/**/*.py", "feeadjuster/**/*.py", "monitor/**/*.py",
"persistent-channels/**/*.py", "rebalance/**/*.py",
"sauron/**/*.py", "zmq/**/*.py", "sitecustomize.py"]
[build-system]
requires = ["poetry-core"]
build-backend = "poetry.core.masonry.api"

16
rebalance/clnutils.py
View file

@ -10,14 +10,14 @@ def cln_parse_rpcversion(string):
make sure we can read all of them for (the next 80 years).
"""
rpcversion = string
if rpcversion.startswith('v'): # strip leading 'v'
if rpcversion.startswith("v"): # strip leading 'v'
rpcversion = rpcversion[1:]
if rpcversion.find('-') != -1: # strip mods
rpcversion = rpcversion[:rpcversion.find('-')]
if re.search('.*(rc[\\d]*)$', rpcversion): # strip release candidates
rpcversion = rpcversion[:rpcversion.find('rc')]
if rpcversion.count('.') == 1: # imply patch version 0 if not given
rpcversion = rpcversion + '.0'
if rpcversion.find("-") != -1: # strip mods
rpcversion = rpcversion[: rpcversion.find("-")]
if re.search(".*(rc[\\d]*)$", rpcversion): # strip release candidates
rpcversion = rpcversion[: rpcversion.find("rc")]
if rpcversion.count(".") == 1: # imply patch version 0 if not given
rpcversion = rpcversion + ".0"
# split and convert numeric string parts to actual integers
return list(map(int, rpcversion.split('.')))
return list(map(int, rpcversion.split(".")))

525
rebalance/rebalance.py
View file

@ -16,9 +16,11 @@ plugin.threadids = {}
def rebalance_stopping():
return (plugin.rebalance_stop_by_user or
plugin.rebalance_stop_by_thread or
plugin.rebalance_stop_by_event)
return (
plugin.rebalance_stop_by_user
or plugin.rebalance_stop_by_thread
or plugin.rebalance_stop_by_event
)
def get_thread_id_str():
@ -31,7 +33,7 @@ def get_thread_id_str():
def route_set_msat(obj, msat):
if plugin.rpcversion[0] == 0 and plugin.rpcversion[1] < 12:
obj[plugin.msatfield] = msat.millisatoshis
obj['amount_msat'] = Millisatoshi(msat)
obj["amount_msat"] = Millisatoshi(msat)
else:
obj[plugin.msatfield] = Millisatoshi(msat)
@ -44,57 +46,85 @@ def setup_routing_fees(route, msat):
delay = plugin.cltv_final
for r in reversed(route):
route_set_msat(r, msat)
r['delay'] = delay
channels = plugin.rpc.listchannels(r['channel'])
ch = next(c for c in channels.get('channels') if c['destination'] == r['id'])
fee = Millisatoshi(ch['base_fee_millisatoshi'])
r["delay"] = delay
channels = plugin.rpc.listchannels(r["channel"])
ch = next(c for c in channels.get("channels") if c["destination"] == r["id"])
fee = Millisatoshi(ch["base_fee_millisatoshi"])
# BOLT #7 requires fee >= fee_base_msat + ( amount_to_forward * fee_proportional_millionths / 1000000 )
fee += (msat * ch['fee_per_millionth'] + 10**6 - 1) // 10**6 # integer math trick to round up
fee += (
msat * ch["fee_per_millionth"] + 10**6 - 1
) // 10**6 # integer math trick to round up
msat += fee
delay += ch['delay']
delay += ch["delay"]
def get_channel(payload, peer_id, scid, check_state: bool = False):
if plugin.listpeerchannels:
channels = plugin.rpc.listpeerchannels(peer_id)['channels']
channel = next(c for c in channels if c.get('short_channel_id') == scid)
channels = plugin.rpc.listpeerchannels(peer_id)["channels"]
channel = next(c for c in channels if c.get("short_channel_id") == scid)
if check_state:
if channel['state'] != "CHANNELD_NORMAL":
raise RpcError('rebalance', payload, {'message': 'Channel %s not in state CHANNELD_NORMAL, but: %s' % (scid, channel['state'])})
if not channel['peer_connected']:
raise RpcError('rebalance', payload, {'message': 'Channel %s peer is not connected.' % scid})
if channel["state"] != "CHANNELD_NORMAL":
raise RpcError(
"rebalance",
payload,
{
"message": "Channel %s not in state CHANNELD_NORMAL, but: %s"
% (scid, channel["state"])
},
)
if not channel["peer_connected"]:
raise RpcError(
"rebalance",
payload,
{"message": "Channel %s peer is not connected." % scid},
)
return channel
peer = plugin.rpc.listpeers(peer_id).get('peers')[0]
channel = next(c for c in peer['channels'] if c.get('short_channel_id') == scid)
peer = plugin.rpc.listpeers(peer_id).get("peers")[0]
channel = next(c for c in peer["channels"] if c.get("short_channel_id") == scid)
if check_state:
if channel['state'] != "CHANNELD_NORMAL":
raise RpcError('rebalance', payload, {'message': 'Channel %s not in state CHANNELD_NORMAL, but: %s' % (scid, channel['state'])})
if not peer['connected']:
raise RpcError('rebalance', payload, {'message': 'Channel %s peer is not connected.' % scid})
if channel["state"] != "CHANNELD_NORMAL":
raise RpcError(
"rebalance",
payload,
{
"message": "Channel %s not in state CHANNELD_NORMAL, but: %s"
% (scid, channel["state"])
},
)
if not peer["connected"]:
raise RpcError(
"rebalance",
payload,
{"message": "Channel %s peer is not connected." % scid},
)
return channel
def amounts_from_scid(scid):
channels = plugin.rpc.listfunds().get('channels')
channel = next(c for c in channels if c.get('short_channel_id') == scid)
our_msat = Millisatoshi(channel['our_amount_msat'])
total_msat = Millisatoshi(channel['amount_msat'])
channels = plugin.rpc.listfunds().get("channels")
channel = next(c for c in channels if c.get("short_channel_id") == scid)
our_msat = Millisatoshi(channel["our_amount_msat"])
total_msat = Millisatoshi(channel["amount_msat"])
return our_msat, total_msat
def peer_from_scid(short_channel_id, my_node_id, payload):
channels = plugin.rpc.listchannels(short_channel_id).get('channels')
channels = plugin.rpc.listchannels(short_channel_id).get("channels")
for ch in channels:
if ch['source'] == my_node_id:
return ch['destination']
raise RpcError("rebalance", payload, {'message': 'Cannot find peer for channel: ' + short_channel_id})
if ch["source"] == my_node_id:
return ch["destination"]
raise RpcError(
"rebalance",
payload,
{"message": "Cannot find peer for channel: " + short_channel_id},
)
def get_node_alias(node_id):
node = plugin.rpc.listnodes(node_id)['nodes']
node = plugin.rpc.listnodes(node_id)["nodes"]
s = ""
if len(node) != 0 and 'alias' in node[0]:
s += node[0]['alias']
if len(node) != 0 and "alias" in node[0]:
s += node[0]["alias"]
else:
s += node_id[0:7]
return s
@ -116,18 +146,20 @@ def find_worst_channel(route):
def cleanup(label, payload, rpc_result, error=None):
try:
plugin.rpc.delinvoice(label, 'unpaid')
plugin.rpc.delinvoice(label, "unpaid")
except RpcError as e:
# race condition: waitsendpay timed out, but invoice get paid
if 'status is paid' in e.error.get('message', ""):
if "status is paid" in e.error.get("message", ""):
return rpc_result
if error is not None:
if isinstance(error, RpcError):
# unwrap rebalance errors as 'normal' RPC result
if error.method == "rebalance":
return {"status": "exception",
"message": error.error.get('message', "error not given")}
return {
"status": "exception",
"message": error.error.get("message", "error not given"),
}
raise error
return rpc_result
@ -175,7 +207,11 @@ def calc_optimal_amount(out_ours, out_total, in_ours, in_total, payload):
if vo > 0 and vo < in_theirs and vi > 0 and vi < out_ours:
return Millisatoshi(min(vi, vo))
raise RpcError("rebalance", payload, {'message': 'rebalancing these channels will make things worse'})
raise RpcError(
"rebalance",
payload,
{"message": "rebalancing these channels will make things worse"},
)
class NoRouteException(Exception):
@ -187,34 +223,40 @@ def getroute_basic(targetid, fromid, excludes, amount_msat: Millisatoshi):
""" This does not make special assumptions and tries all routes
it gets. Uses less CPU and does not filter any routes.
"""
return plugin.rpc.getroute(targetid,
fromid=fromid,
exclude=excludes,
amount_msat=amount_msat,
maxhops=plugin.maxhops,
riskfactor=10, cltv=9)
return plugin.rpc.getroute(
targetid,
fromid=fromid,
exclude=excludes,
amount_msat=amount_msat,
maxhops=plugin.maxhops,
riskfactor=10,
cltv=9,
)
except RpcError as e:
# could not find route -> change params and restart loop
if e.method == "getroute" and e.error.get('code') == 205:
if e.method == "getroute" and e.error.get("code") == 205:
raise NoRouteException
raise e
def getroute_iterative(targetid, fromid, excludes, amount_msat: Millisatoshi):
""" This searches for 'shorter and bigger pipes' first in order
to increase likelyhood of success on short timeout.
Can be useful for manual `rebalance`.
"""This searches for 'shorter and bigger pipes' first in order
to increase likelyhood of success on short timeout.
Can be useful for manual `rebalance`.
"""
try:
return plugin.rpc.getroute(targetid,
fromid=fromid,
exclude=excludes,
amount_msat=amount_msat * plugin.msatfactoridx,
maxhops=plugin.maxhopidx,
riskfactor=10, cltv=9)
return plugin.rpc.getroute(
targetid,
fromid=fromid,
exclude=excludes,
amount_msat=amount_msat * plugin.msatfactoridx,
maxhops=plugin.maxhopidx,
riskfactor=10,
cltv=9,
)
except RpcError as e:
# could not find route -> change params and restart loop
if e.method == "getroute" and e.error.get('code') == 205:
if e.method == "getroute" and e.error.get("code") == 205:
# reduce _msatfactor to look for smaller channels now
plugin.msatfactoridx -= 1
if plugin.msatfactoridx < 1:
@ -229,10 +271,7 @@ def getroute_iterative(targetid, fromid, excludes, amount_msat: Millisatoshi):
def getroute_switch(method_name):
switch = {
"basic": getroute_basic,
"iterative": getroute_iterative
}
switch = {"basic": getroute_basic, "iterative": getroute_iterative}
return switch.get(method_name, getroute_iterative)
@ -243,7 +282,7 @@ def waitsendpay(payment_hash, start_ts, retry_for):
result = plugin.rpc.waitsendpay(payment_hash, timeout)
return result
except RpcError as e:
if e.method == "waitsendpay" and e.error.get('code') == 200:
if e.method == "waitsendpay" and e.error.get("code") == 200:
if rebalance_stopping():
raise e
if int(time.time()) - start_ts >= retry_for:
@ -253,10 +292,16 @@ def waitsendpay(payment_hash, start_ts, retry_for):
@plugin.method("rebalance")
def rebalance(plugin, outgoing_scid, incoming_scid, msatoshi: Millisatoshi = None,
retry_for: int = 60, maxfeepercent: float = 0.5,
exemptfee: Millisatoshi = Millisatoshi(5000),
getroute_method=None):
def rebalance(
plugin,
outgoing_scid,
incoming_scid,
msatoshi: Millisatoshi = None,
retry_for: int = 60,
maxfeepercent: float = 0.5,
exemptfee: Millisatoshi = Millisatoshi(5000),
getroute_method=None,
):
"""Rebalancing channel liquidity with circular payments.
This tool helps to move some msatoshis between your channels.
@ -276,9 +321,9 @@ def rebalance(plugin, outgoing_scid, incoming_scid, msatoshi: Millisatoshi = Non
"msatoshi": msatoshi,
"retry_for": retry_for,
"maxfeepercent": maxfeepercent,
"exemptfee": exemptfee
"exemptfee": exemptfee,
}
my_node_id = plugin.getinfo.get('id')
my_node_id = plugin.getinfo.get("id")
outgoing_node_id = peer_from_scid(outgoing_scid, my_node_id, payload)
incoming_node_id = peer_from_scid(incoming_scid, my_node_id, payload)
get_channel(payload, outgoing_node_id, outgoing_scid, True)
@ -293,24 +338,35 @@ def rebalance(plugin, outgoing_scid, incoming_scid, msatoshi: Millisatoshi = Non
# Check requested amounts are selected channels
if msatoshi > out_ours or msatoshi > in_total - in_ours:
raise RpcError("rebalance", payload, {'message': 'Channel capacities too low'})
raise RpcError("rebalance", payload, {"message": "Channel capacities too low"})
plugin.log(f"starting rebalance out_scid:{outgoing_scid} in_scid:{incoming_scid} amount:{msatoshi}", 'debug')
plugin.log(
f"starting rebalance out_scid:{outgoing_scid} in_scid:{incoming_scid} amount:{msatoshi}",
"debug",
)
route_out = {'id': outgoing_node_id, 'channel': outgoing_scid, 'direction': int(not my_node_id < outgoing_node_id)}
route_in = {'id': my_node_id, 'channel': incoming_scid, 'direction': int(not incoming_node_id < my_node_id)}
route_out = {
"id": outgoing_node_id,
"channel": outgoing_scid,
"direction": int(not my_node_id < outgoing_node_id),
}
route_in = {
"id": my_node_id,
"channel": incoming_scid,
"direction": int(not incoming_node_id < my_node_id),
}
start_ts = int(time.time())
label = "Rebalance-" + str(uuid.uuid4())
description = "%s to %s" % (outgoing_scid, incoming_scid)
invoice = plugin.rpc.invoice(msatoshi, label, description, retry_for + 60)
payment_hash = invoice['payment_hash']
payment_hash = invoice["payment_hash"]
# The requirement for payment_secret coincided with its addition to the invoice output.
payment_secret = invoice.get('payment_secret')
payment_secret = invoice.get("payment_secret")
rpc_result = None
excludes = [my_node_id] # excude all own channels to prevent shortcuts
nodes = {} # here we store erring node counts
plugin.maxhopidx = 1 # start with short routes and increase
excludes = [my_node_id] # excude all own channels to prevent shortcuts
nodes = {} # here we store erring node counts
plugin.maxhopidx = 1 # start with short routes and increase
plugin.msatfactoridx = plugin.msatfactor # start with high capacity factor
# and decrease to reduce WIRE_TEMPORARY failures because of imbalances
@ -330,23 +386,25 @@ def rebalance(plugin, outgoing_scid, incoming_scid, msatoshi: Millisatoshi = Non
count += 1
try:
time_start = time.time()
r = getroute(targetid=incoming_node_id,
fromid=outgoing_node_id,
excludes=excludes,
amount_msat=msatoshi)
r = getroute(
targetid=incoming_node_id,
fromid=outgoing_node_id,
excludes=excludes,
amount_msat=msatoshi,
)
time_getroute += time.time() - time_start
except NoRouteException:
# no more chance for a successful getroute
rpc_result = {'status': 'error', 'message': 'No suitable routes found'}
rpc_result = {"status": "error", "message": "No suitable routes found"}
return cleanup(label, payload, rpc_result)
except RpcError as e:
# getroute can be successful next time with different parameters
if e.method == "getroute" and e.error.get('code') == 205:
if e.method == "getroute" and e.error.get("code") == 205:
continue
else:
raise e
route_mid = r['route']
route_mid = r["route"]
route = [route_out] + route_mid + [route_in]
setup_routing_fees(route, msatoshi)
fees = route_get_msat(route[0]) - msatoshi
@ -356,8 +414,12 @@ def rebalance(plugin, outgoing_scid, incoming_scid, msatoshi: Millisatoshi = Non
if fees > exemptfee and int(fees) > int(msatoshi) * maxfeepercent / 100:
worst_channel = find_worst_channel(route)
if worst_channel is None:
raise RpcError("rebalance", payload, {'message': 'Insufficient fee'})
excludes.append(worst_channel['channel'] + '/' + str(worst_channel['direction']))
raise RpcError(
"rebalance", payload, {"message": "Insufficient fee"}
)
excludes.append(
worst_channel["channel"] + "/" + str(worst_channel["direction"])
)
continue
rpc_result = {
@ -370,11 +432,24 @@ def rebalance(plugin, outgoing_scid, incoming_scid, msatoshi: Millisatoshi = Non
"status": "complete",
"message": f"{msatoshi + fees} sent over {len(route)} hops to rebalance {msatoshi}",
}
midroute_str = reduce(lambda x, y: x + " -> " + y, map(lambda r: get_node_alias(r['id']), route_mid))
full_route_str = "%s -> %s -> %s -> %s" % (get_node_alias(my_node_id), get_node_alias(outgoing_node_id), midroute_str, get_node_alias(my_node_id))
plugin.log(f"Thread{get_thread_id_str()} {len(route)} hops and {fees.to_satoshi_str()} fees for {msatoshi.to_satoshi_str()} along route: {full_route_str}")
midroute_str = reduce(
lambda x, y: x + " -> " + y,
map(lambda r: get_node_alias(r["id"]), route_mid),
)
full_route_str = "%s -> %s -> %s -> %s" % (
get_node_alias(my_node_id),
get_node_alias(outgoing_node_id),
midroute_str,
get_node_alias(my_node_id),
)
plugin.log(
f"Thread{get_thread_id_str()} {len(route)} hops and {fees.to_satoshi_str()} fees for {msatoshi.to_satoshi_str()} along route: {full_route_str}"
)
for r in route:
plugin.log(" - %s %14s %s" % (r['id'], r['channel'], route_get_msat(r)), 'debug')
plugin.log(
" - %s %14s %s" % (r["id"], r["channel"], route_get_msat(r)),
"debug",
)
time_start = time.time()
count_sendpay += 1
@ -382,28 +457,37 @@ def rebalance(plugin, outgoing_scid, incoming_scid, msatoshi: Millisatoshi = Non
plugin.rpc.sendpay(route, payment_hash, payment_secret=payment_secret)
result = waitsendpay(payment_hash, start_ts, retry_for)
time_sendpay += time.time() - time_start
if result.get('status') == "complete":
rpc_result["stats"] = f"running_for:{int(time.time()) - start_ts} count_getroute:{count} time_getroute:{time_getroute} time_getroute_avg:{time_getroute / count} count_sendpay:{count_sendpay} time_sendpay:{time_sendpay} time_sendpay_avg:{time_sendpay / count_sendpay}"
if result.get("status") == "complete":
rpc_result["stats"] = (
f"running_for:{int(time.time()) - start_ts} count_getroute:{count} time_getroute:{time_getroute} time_getroute_avg:{time_getroute / count} count_sendpay:{count_sendpay} time_sendpay:{time_sendpay} time_sendpay_avg:{time_sendpay / count_sendpay}"
)
return cleanup(label, payload, rpc_result)
except RpcError as e:
time_sendpay += time.time() - time_start
plugin.log(f"maxhops:{plugin.maxhopidx} msatfactor:{plugin.msatfactoridx} running_for:{int(time.time()) - start_ts} count_getroute:{count} time_getroute:{time_getroute} time_getroute_avg:{time_getroute / count} count_sendpay:{count_sendpay} time_sendpay:{time_sendpay} time_sendpay_avg:{time_sendpay / count_sendpay}", 'debug')
plugin.log(
f"maxhops:{plugin.maxhopidx} msatfactor:{plugin.msatfactoridx} running_for:{int(time.time()) - start_ts} count_getroute:{count} time_getroute:{time_getroute} time_getroute_avg:{time_getroute / count} count_sendpay:{count_sendpay} time_sendpay:{time_sendpay} time_sendpay_avg:{time_sendpay / count_sendpay}",
"debug",
)
# plugin.log(f"RpcError: {str(e)}", 'debug')
# check if we ran into the `rpc.waitsendpay` timeout
if e.method == "waitsendpay" and e.error.get('code') == 200:
raise RpcError("rebalance", payload, {'message': 'Timeout reached'})
if e.method == "waitsendpay" and e.error.get("code") == 200:
raise RpcError("rebalance", payload, {"message": "Timeout reached"})
# check if we have problems with our own channels
erring_node = e.error.get('data', {}).get('erring_node')
erring_channel = e.error.get('data', {}).get('erring_channel')
erring_direction = e.error.get('data', {}).get('erring_direction')
erring_node = e.error.get("data", {}).get("erring_node")
erring_channel = e.error.get("data", {}).get("erring_channel")
erring_direction = e.error.get("data", {}).get("erring_direction")
if erring_channel == incoming_scid:
raise RpcError("rebalance", payload, {'message': 'Error with incoming channel'})
raise RpcError(
"rebalance", payload, {"message": "Error with incoming channel"}
)
if erring_channel == outgoing_scid:
raise RpcError("rebalance", payload, {'message': 'Error with outgoing channel'})
raise RpcError(
"rebalance", payload, {"message": "Error with outgoing channel"}
)
# exclude other erroring channels
if erring_channel is not None and erring_direction is not None:
excludes.append(erring_channel + '/' + str(erring_direction))
excludes.append(erring_channel + "/" + str(erring_direction))
# count and exclude nodes that produce a lot of errors
if erring_node and plugin.erringnodes > 0:
if nodes.get(erring_node) is None:
@ -414,7 +498,7 @@ def rebalance(plugin, outgoing_scid, incoming_scid, msatoshi: Millisatoshi = Non
except Exception as e:
return cleanup(label, payload, rpc_result, e)
rpc_result = {'status': 'error', 'message': 'Timeout reached'}
rpc_result = {"status": "error", "message": "Timeout reached"}
return cleanup(label, payload, rpc_result)
@ -456,7 +540,7 @@ def could_receive(liquidity):
def get_open_channels(plugin: Plugin):
result = []
if plugin.listpeerchannels:
channels = plugin.rpc.listpeerchannels()['channels']
channels = plugin.rpc.listpeerchannels()["channels"]
for ch in channels:
if ch["state"] == "CHANNELD_NORMAL" and not ch["private"]:
result.append(ch)
@ -504,9 +588,14 @@ def get_ideal_ratio(channels: list, enough_liquidity: Millisatoshi):
while len(chs) > 0:
ratio = int(our) / int(total)
smallest_channel = min(chs, key=lambda ch: ch["total_msat"])
if Millisatoshi(smallest_channel["total_msat"]) * min(ratio, 1 - ratio) > enough_liquidity:
if (
Millisatoshi(smallest_channel["total_msat"]) * min(ratio, 1 - ratio)
> enough_liquidity
):
break
min_liquidity = min(Millisatoshi(smallest_channel["total_msat"]) / 2, enough_liquidity)
min_liquidity = min(
Millisatoshi(smallest_channel["total_msat"]) / 2, enough_liquidity
)
diff = Millisatoshi(smallest_channel["total_msat"]) * ratio
diff = max(diff, min_liquidity)
diff = min(diff, Millisatoshi(smallest_channel["total_msat"]) - min_liquidity)
@ -518,7 +607,11 @@ def get_ideal_ratio(channels: list, enough_liquidity: Millisatoshi):
def feeadjust_would_be_nice():
commands = [c for c in plugin.rpc.help().get("help") if c["command"].split()[0] == "feeadjust"]
commands = [
c
for c in plugin.rpc.help().get("help")
if c["command"].split()[0] == "feeadjust"
]
if len(commands) == 1:
msg = plugin.rpc.feeadjust()
plugin.log(f"Feeadjust succeeded: {msg}")
@ -537,7 +630,7 @@ def get_max_fee(msat: Millisatoshi):
def get_chan(scid: str):
if plugin.listpeerchannels:
channels = plugin.rpc.listpeerchannels()['channels']
channels = plugin.rpc.listpeerchannels()["channels"]
for chan in channels:
if chan.get("short_channel_id") == scid:
return chan
@ -555,11 +648,20 @@ def liquidity_info(channel, enough_liquidity: Millisatoshi, ideal_ratio: float):
"our": Millisatoshi(channel["to_us_msat"]),
"their": Millisatoshi(channel["total_msat"] - channel["to_us_msat"]),
"min": min(enough_liquidity, Millisatoshi(channel["total_msat"]) / 2),
"max": max(a_minus_b(Millisatoshi(channel["total_msat"]), enough_liquidity), Millisatoshi(channel["total_msat"]) / 2),
"ideal": {}
"max": max(
a_minus_b(Millisatoshi(channel["total_msat"]), enough_liquidity),
Millisatoshi(channel["total_msat"]) / 2,
),
"ideal": {},
}
liquidity["ideal"]["our"] = min(max(Millisatoshi(channel["total_msat"]) * ideal_ratio, liquidity["min"]), liquidity["max"])
liquidity["ideal"]["their"] = min(max(Millisatoshi(channel["total_msat"]) * (1 - ideal_ratio), liquidity["min"]), liquidity["max"])
liquidity["ideal"]["our"] = min(
max(Millisatoshi(channel["total_msat"]) * ideal_ratio, liquidity["min"]),
liquidity["max"],
)
liquidity["ideal"]["their"] = min(
max(Millisatoshi(channel["total_msat"]) * (1 - ideal_ratio), liquidity["min"]),
liquidity["max"],
)
return liquidity
@ -583,28 +685,40 @@ def wait_for_htlcs(failed_channels: list, scids: list = None):
# HTLC settlement helper
# taken and modified from pyln-testing/pyln/testing/utils.py
result = True
peers = plugin.rpc.listpeers()['peers']
peers = plugin.rpc.listpeers()["peers"]
for p, peer in enumerate(peers):
pid = peer['id']
pid = peer["id"]
channels = []
if 'channels' in peer:
channels = peer['channels']
elif 'num_channels' in peer and peer['num_channels'] > 0:
channels = plugin.rpc.listpeerchannels(peer['id'])['channels']
if "channels" in peer:
channels = peer["channels"]
elif "num_channels" in peer and peer["num_channels"] > 0:
channels = plugin.rpc.listpeerchannels(peer["id"])["channels"]
for c, channel in enumerate(channels):
scid = channel.get('short_channel_id')
scid = channel.get("short_channel_id")
if scids is not None and scid not in scids:
continue
if scid in failed_channels:
result = False
continue
if 'htlcs' in channel:
lam = lambda: len(plugin.rpc.listpeers()['peers'][p]['channels'][c]['htlcs']) == 0
if "htlcs" in channel:
lam = (
lambda: len(
plugin.rpc.listpeers()["peers"][p]["channels"][c]["htlcs"]
)
== 0
)
if plugin.listpeerchannels:
lam = lambda: len(plugin.rpc.listpeerchannels(pid)['channels'][c]['htlcs']) == 0
lam = (
lambda: len(
plugin.rpc.listpeerchannels(pid)["channels"][c]["htlcs"]
)
== 0
)
if not wait_for(lam):
failed_channels.append(scid)
plugin.log(f"Thread{get_thread_id_str()} timeout while waiting for htlc settlement in channel {scid}")
plugin.log(
f"Thread{get_thread_id_str()} timeout while waiting for htlc settlement in channel {scid}"
)
result = False
return result
@ -630,19 +744,28 @@ def maybe_rebalance_pairs(ch1, ch2, failed_channels: list):
return result
amount = min(amount, get_max_amount(i, plugin))
maxfee = get_max_fee(amount)
plugin.log(f"Thread{get_thread_id_str()} tries to rebalance: {scid1} -> {scid2}; amount={amount.to_satoshi_str()}; maxfee={maxfee.to_satoshi_str()}")
plugin.log(
f"Thread{get_thread_id_str()} tries to rebalance: {scid1} -> {scid2}; amount={amount.to_satoshi_str()}; maxfee={maxfee.to_satoshi_str()}"
)
start_ts = time.time()
try:
res = rebalance(plugin, outgoing_scid=scid1, incoming_scid=scid2,
msatoshi=amount, retry_for=1200, maxfeepercent=0,
exemptfee=maxfee)
if not res.get('status') == 'complete':
res = rebalance(
plugin,
outgoing_scid=scid1,
incoming_scid=scid2,
msatoshi=amount,
retry_for=1200,
maxfeepercent=0,
exemptfee=maxfee,
)
if not res.get("status") == "complete":
raise Exception # fall into exception handler below
except Exception:
failed_channels.append(scid1 + ":" + scid2)
# rebalance failed, let's try with a smaller amount
while (get_max_amount(i, plugin) >= amount and
get_max_amount(i, plugin) != get_max_amount(i + 1, plugin)):
while get_max_amount(i, plugin) >= amount and get_max_amount(
i, plugin
) != get_max_amount(i + 1, plugin):
i += 1
if amount > get_max_amount(i, plugin):
continue
@ -687,14 +810,18 @@ def rebalance_pair_picker(threadid, channel_pairs: list, failed_channels: list):
ch2["lock"].release()
ch1["lock"].release()
if result["success"]:
plugin.log(f"Thread{get_thread_id_str()} restarts rebalance threads after successful rebalance")
plugin.log(
f"Thread{get_thread_id_str()} restarts rebalance threads after successful rebalance"
)
plugin.rebalance_stop_by_thread = True
return result
unprocessed = [p for p in channel_pairs if not p[2]]
if len(unprocessed) == 0:
return result
if idle_count == 1:
plugin.log(f"Thread{get_thread_id_str()} is idle, {len(unprocessed)} possible channel pairs remained")
plugin.log(
f"Thread{get_thread_id_str()} is idle, {len(unprocessed)} possible channel pairs remained"
)
if idle_count > 0:
time.sleep(10)
return result
@ -715,7 +842,11 @@ def maybe_rebalance_once(failed_channels: list):
executor = concurrent.futures.ThreadPoolExecutor(max_workers=plugin.threads)
futures = set()
for threadid in range(plugin.threads):
futures.add(executor.submit(rebalance_pair_picker, threadid, channel_pairs, failed_channels))
futures.add(
executor.submit(
rebalance_pair_picker, threadid, channel_pairs, failed_channels
)
)
result = {"success": False, "fee_spent": Millisatoshi(0)}
for future in concurrent.futures.as_completed(futures):
r2 = future.result()
@ -730,7 +861,11 @@ def maybe_rebalance_once(failed_channels: list):
def feeadjuster_toggle(new_value: bool):
commands = [c for c in plugin.rpc.help().get("help") if c["command"].split()[0] == "feeadjuster-toggle"]
commands = [
c
for c in plugin.rpc.help().get("help")
if c["command"].split()[0] == "feeadjuster-toggle"
]
if len(commands) == 1:
msg = plugin.rpc.feeadjuster_toggle(new_value)
return msg["forward_event_subscription"]["previous"]
@ -742,10 +877,12 @@ def refresh_parameters():
channels = get_open_channels(plugin)
plugin.enough_liquidity = get_enough_liquidity_threshold(channels)
plugin.ideal_ratio = get_ideal_ratio(channels, plugin.enough_liquidity)
plugin.log(f"Automatic rebalance is running with enough liquidity threshold: {plugin.enough_liquidity.to_satoshi_str()}, "
f"ideal liquidity ratio: {plugin.ideal_ratio * 100:.2f}%, "
f"min rebalancable amount: {plugin.min_amount.to_satoshi_str()}, "
f"feeratio: {plugin.feeratio}")
plugin.log(
f"Automatic rebalance is running with enough liquidity threshold: {plugin.enough_liquidity.to_satoshi_str()}, "
f"ideal liquidity ratio: {plugin.ideal_ratio * 100:.2f}%, "
f"min rebalancable amount: {plugin.min_amount.to_satoshi_str()}, "
f"feeratio: {plugin.feeratio}"
)
def rebalanceall_thread():
@ -754,7 +891,7 @@ def rebalanceall_thread():
try:
start_ts = time.time()
feeadjuster_state = feeadjuster_toggle(False)
plugin.log(f"Automatic rebalance started")
plugin.log("Automatic rebalance started")
failed_channels = []
success = 0
fee_spent = Millisatoshi(0)
@ -772,8 +909,10 @@ def rebalanceall_thread():
feeadjust_would_be_nice()
feeadjuster_toggle(feeadjuster_state)
elapsed_time = timedelta(seconds=time.time() - start_ts)
plugin.rebalanceall_msg = (f"Automatic rebalance finished: {success} successful rebalance, "
f"{fee_spent.to_satoshi_str()} fee spent, it took {str(elapsed_time)[:-3]}")
plugin.rebalanceall_msg = (
f"Automatic rebalance finished: {success} successful rebalance, "
f"{fee_spent.to_satoshi_str()} fee spent, it took {str(elapsed_time)[:-3]}"
)
plugin.log(plugin.rebalanceall_msg)
finally:
plugin.mutex.release()
@ -792,7 +931,7 @@ def forward_event(plugin: Plugin, forward_event: dict, **kwargs):
def invoice_payment(plugin: Plugin, invoice_payment: dict, **kwargs):
if not plugin.mutex.locked():
return
if invoice_payment.get('label').startswith("Rebalance"):
if invoice_payment.get("label").startswith("Rebalance"):
return
plugin.log("Invoice payment restarts rebalance threads")
plugin.rebalance_stop_by_event = True
@ -802,8 +941,8 @@ def invoice_payment(plugin: Plugin, invoice_payment: dict, **kwargs):
def sendpay_success(plugin: Plugin, sendpay_success: dict, **kwargs):
if not plugin.mutex.locked():
return
my_node_id = plugin.getinfo.get('id')
if sendpay_success.get('destination') == my_node_id:
my_node_id = plugin.getinfo.get("id")
if sendpay_success.get("destination") == my_node_id:
return
plugin.log("Sendpay success restarts rebalance threads")
plugin.rebalance_stop_by_event = True
@ -813,14 +952,21 @@ def sendpay_success(plugin: Plugin, sendpay_success: dict, **kwargs):
def channel_state_changed(plugin: Plugin, channel_state_changed: dict, **kwargs):
if not plugin.mutex.locked():
return
if channel_state_changed.get('old_state') != 'CHANNELD_NORMAL' and channel_state_changed.get('new_state') != 'CHANNELD_NORMAL':
if (
channel_state_changed.get("old_state") != "CHANNELD_NORMAL"
and channel_state_changed.get("new_state") != "CHANNELD_NORMAL"
):
return
plugin.log("Channel state changed restarts rebalance threads")
plugin.rebalance_stop_by_event = True
@plugin.method("rebalanceall")
def rebalanceall(plugin: Plugin, min_amount: Millisatoshi = Millisatoshi("50000sat"), feeratio: float = 0.5):
def rebalanceall(
plugin: Plugin,
min_amount: Millisatoshi = Millisatoshi("50000sat"),
feeratio: float = 0.5,
):
"""Rebalance all unbalanced channels if possible for a very low fee.
Default minimum rebalancable amount is 50000sat. Default feeratio = 0.5, half of our node's default fee.
To be economical, it tries to fix the liquidity cheaper than it can be ruined by transaction forwards.
@ -828,7 +974,9 @@ def rebalanceall(plugin: Plugin, min_amount: Millisatoshi = Millisatoshi("50000s
"""
# some early checks before we start the async thread
if plugin.mutex.locked():
return {"message": "Rebalance is already running, this may take a while. To stop it use the cli method 'rebalancestop'."}
return {
"message": "Rebalance is already running, this may take a while. To stop it use the cli method 'rebalancestop'."
}
channels = get_open_channels(plugin)
if len(channels) <= 1:
return {"message": "Error: Not enough open channels to rebalance anything"}
@ -845,18 +993,21 @@ def rebalanceall(plugin: Plugin, min_amount: Millisatoshi = Millisatoshi("50000s
# run the job
t = threading.Thread(target=rebalanceall_thread, args=())
t.start()
return {"message": f"Rebalance started with min rebalancable amount: {plugin.min_amount}, feeratio: {plugin.feeratio}"}
return {
"message": f"Rebalance started with min rebalancable amount: {plugin.min_amount}, feeratio: {plugin.feeratio}"
}
@plugin.method("rebalancestop")
def rebalancestop(plugin: Plugin):
"""It stops the ongoing rebalanceall.
"""
"""It stops the ongoing rebalanceall."""
if not plugin.mutex.locked():
if plugin.rebalanceall_msg is None:
return {"message": "No rebalance is running, nothing to stop."}
return {"message": f"No rebalance is running, nothing to stop. "
f"Last 'rebalanceall' gave: {plugin.rebalanceall_msg}"}
return {
"message": f"No rebalance is running, nothing to stop. "
f"Last 'rebalanceall' gave: {plugin.rebalanceall_msg}"
}
start_ts = time.time()
plugin.rebalance_stop_by_user = True
plugin.mutex.acquire(blocking=True)
@ -868,7 +1019,11 @@ def rebalancestop(plugin: Plugin):
def health_score(liquidity):
if int(liquidity["ideal"]["our"]) == 0 or int(liquidity["ideal"]["their"]) == 0 or int(liquidity["min"]) == 0:
if (
int(liquidity["ideal"]["our"]) == 0
or int(liquidity["ideal"]["their"]) == 0
or int(liquidity["min"]) == 0
):
return 0
score_our = int(liquidity["our"]) / int(liquidity["ideal"]["our"])
score_their = int(liquidity["their"]) / int(liquidity["ideal"]["their"])
@ -887,10 +1042,13 @@ def get_avg_forward_fees(intervals):
total = [0] * len(intervals)
fees = [0] * len(intervals)
res = [0] * len(intervals)
all_forwards = list(filter(lambda fwd: fwd.get("status") == "settled"
and fwd.get("resolved_time", 0)
+ max_interval * 60 * 60 * 24 > now,
plugin.rpc.listforwards()["forwards"]))
all_forwards = list(
filter(
lambda fwd: fwd.get("status") == "settled"
and fwd.get("resolved_time", 0) + max_interval * 60 * 60 * 24 > now,
plugin.rpc.listforwards()["forwards"],
)
)
# build intermediate result per interval
for fwd in all_forwards:
@ -911,8 +1069,7 @@ def get_avg_forward_fees(intervals):
@plugin.method("rebalancereport")
def rebalancereport(plugin: Plugin, include_avg_fees: bool = True):
"""Show information about rebalance
"""
"""Show information about rebalance"""
res = {}
res["rebalanceall_is_running"] = plugin.mutex.locked()
res["getroute_method"] = plugin.getroute.__name__
@ -934,16 +1091,24 @@ def rebalancereport(plugin: Plugin, include_avg_fees: bool = True):
res["enough_liquidity_threshold"] = Millisatoshi(0)
res["ideal_liquidity_ratio"] = "0%"
res["liquidity_health"] = f"{health_percent:.2f}%"
invoices = plugin.rpc.listinvoices()['invoices']
rebalances = [i for i in invoices if i.get('status') == 'paid' and i.get('label').startswith("Rebalance")]
invoices = plugin.rpc.listinvoices()["invoices"]
rebalances = [
i
for i in invoices
if i.get("status") == "paid" and i.get("label").startswith("Rebalance")
]
total_fee = Millisatoshi(0)
total_amount = Millisatoshi(0)
res["total_successful_rebalances"] = len(rebalances)
# iterate if cln doesn't already support `status` on listpays since v0.10.2
if plugin.rpcversion[0] == 0 and plugin.rpcversion[1] <= 10 and plugin.rpcversion[2] < 2:
if (
plugin.rpcversion[0] == 0
and plugin.rpcversion[1] <= 10
and plugin.rpcversion[2] < 2
):
pays = plugin.rpc.listpays()["pays"]
pays = [p for p in pays if p.get('status') == 'complete']
pays = [p for p in pays if p.get("status") == "complete"]
else:
pays = plugin.rpc.listpays(status="complete")["pays"]
@ -963,16 +1128,16 @@ def rebalancereport(plugin: Plugin, include_avg_fees: bool = True):
if include_avg_fees:
avg_forward_fees = get_avg_forward_fees([1, 7, 30])
res['average_forward_fee_ppm_1d'] = avg_forward_fees[0]
res['average_forward_fee_ppm_7d'] = avg_forward_fees[1]
res['average_forward_fee_ppm_30d'] = avg_forward_fees[2]
res["average_forward_fee_ppm_1d"] = avg_forward_fees[0]
res["average_forward_fee_ppm_7d"] = avg_forward_fees[1]
res["average_forward_fee_ppm_30d"] = avg_forward_fees[2]
return res
@plugin.init()
def init(options: dict, configuration: dict, plugin: Plugin, **kwargs):
rpchelp = plugin.rpc.help().get('help')
rpchelp = plugin.rpc.help().get("help")
# detect if server cli has moved `listpeers.channels[]` to `listpeerchannels`
# See https://github.com/ElementsProject/lightning/pull/5825
# TODO: replace by rpc version check once v23 is released
@ -982,7 +1147,7 @@ def init(options: dict, configuration: dict, plugin: Plugin, **kwargs):
# do all the stuff that needs to be done just once ...
plugin.getinfo = plugin.rpc.getinfo()
plugin.rpcversion = cln_parse_rpcversion(plugin.getinfo.get('version'))
plugin.rpcversion = cln_parse_rpcversion(plugin.getinfo.get("version"))
config = plugin.rpc.listconfigs()["configs"]
plugin.cltv_final = config["cltv-final"]["value_int"]
plugin.fee_base = Millisatoshi(config["fee-base"]["value_int"])
@ -996,17 +1161,19 @@ def init(options: dict, configuration: dict, plugin: Plugin, **kwargs):
plugin.rebalanceall_msg = None
# use getroute amount_msat/msatoshi field depending on version
plugin.msatfield = 'amount_msat'
plugin.msatfield = "amount_msat"
if plugin.rpcversion[0] == 0 and plugin.rpcversion[1] < 12:
plugin.msatfield = 'msatoshi'
plugin.msatfield = "msatoshi"
plugin.log(f"Plugin rebalance initialized with {plugin.fee_base.to_satoshi_str()} base / {plugin.fee_ppm} ppm fee "
f"cltv_final:{plugin.cltv_final} "
f"maxhops:{plugin.maxhops} "
f"msatfactor:{plugin.msatfactor} "
f"erringnodes:{plugin.erringnodes} "
f"getroute:{plugin.getroute.__name__} "
f"threads:{plugin.threads} ")
plugin.log(
f"Plugin rebalance initialized with {plugin.fee_base.to_satoshi_str()} base / {plugin.fee_ppm} ppm fee "
f"cltv_final:{plugin.cltv_final} "
f"maxhops:{plugin.maxhops} "
f"msatfactor:{plugin.msatfactor} "
f"erringnodes:{plugin.erringnodes} "
f"getroute:{plugin.getroute.__name__} "
f"threads:{plugin.threads} "
)
plugin.add_option(
@ -1015,7 +1182,7 @@ plugin.add_option(
"Getroute method for route search can be 'basic' or 'iterative'."
"'basic': Tries all routes sequentially. "
"'iterative': Tries shorter and bigger routes first.",
"string"
"string",
)
plugin.add_option(
"rebalance-maxhops",
@ -1023,7 +1190,7 @@ plugin.add_option(
"Maximum number of hops for `getroute` call. Set to 0 to disable. "
"Note: Two hops are added for own nodes input and output channel. "
"Note: Routes with a 8 or more hops have less than 3% success rate.",
"string"
"string",
)
plugin.add_option(
@ -1031,7 +1198,7 @@ plugin.add_option(
"4",
"Will instruct `getroute` call to use higher requested capacity first. "
"Note: This will decrease to 1 when no routes can be found.",
"string"
"string",
)
plugin.add_option(
@ -1039,7 +1206,7 @@ plugin.add_option(
"5",
"Exclude nodes from routing that raised N or more errors. "
"Note: Use 0 to disable.",
"string"
"string",
)
plugin.add_option(
@ -1047,7 +1214,7 @@ plugin.add_option(
"8",
"Number of threads used parallelly by `rebalanceall` "
"Higher numbers increase speed and CPU consumption",
"string"
"string",
)
plugin.run()

91
rebalance/test_rebalance.py
View file

@ -3,7 +3,7 @@ from pyln.testing.fixtures import * # noqa: F401,F403
from pyln.client import Millisatoshi
plugin_path = os.path.join(os.path.dirname(__file__), "rebalance.py")
plugin_opt = {'plugin': plugin_path}
plugin_opt = {"plugin": plugin_path}
# waits for a bunch of nodes HTLCs to settle
@ -33,9 +33,13 @@ def test_rebalance_starts(node_factory):
l1.start()
# Start at 0 and 're-await' the two inits above. Otherwise this is flaky.
l1.daemon.logsearch_start = 0
l1.daemon.wait_for_logs(["Plugin rebalance initialized.*",
"Plugin rebalance initialized.*",
"Plugin rebalance initialized.*"])
l1.daemon.wait_for_logs(
[
"Plugin rebalance initialized.*",
"Plugin rebalance initialized.*",
"Plugin rebalance initialized.*",
]
)
def test_rebalance_manual(node_factory, bitcoind):
@ -63,33 +67,39 @@ def test_rebalance_manual(node_factory, bitcoind):
# check we can do an auto amount rebalance
result = l1.rpc.rebalance(scid12, scid31)
print(result)
assert result['status'] == 'complete'
assert result['outgoing_scid'] == scid12
assert result['incoming_scid'] == scid31
assert result['hops'] == 3
assert result['received'] == '500000000msat'
assert result["status"] == "complete"
assert result["outgoing_scid"] == scid12
assert result["incoming_scid"] == scid31
assert result["hops"] == 3
assert result["received"] == "500000000msat"
# wait until listpeers is up2date
wait_for_all_htlcs(nodes)
# check that channels are now balanced
c12 = l1.rpc.listpeerchannels(l2.info['id'])['channels'][0]
c13 = l1.rpc.listpeerchannels(l3.info['id'])['channels'][0]
assert abs(0.5 - (Millisatoshi(c12['to_us_msat']) / Millisatoshi(c12['total_msat']))) < 0.01
assert abs(0.5 - (Millisatoshi(c13['to_us_msat']) / Millisatoshi(c13['total_msat']))) < 0.01
c12 = l1.rpc.listpeerchannels(l2.info["id"])["channels"][0]
c13 = l1.rpc.listpeerchannels(l3.info["id"])["channels"][0]
assert (
abs(0.5 - (Millisatoshi(c12["to_us_msat"]) / Millisatoshi(c12["total_msat"])))
< 0.01
)
assert (
abs(0.5 - (Millisatoshi(c13["to_us_msat"]) / Millisatoshi(c13["total_msat"])))
< 0.01
)
# check we can do a manual amount rebalance in the other direction
result = l1.rpc.rebalance(scid31, scid12, '250000000msat')
assert result['status'] == 'complete'
assert result['outgoing_scid'] == scid31
assert result['incoming_scid'] == scid12
assert result['hops'] == 3
assert result['received'] == '250000000msat'
result = l1.rpc.rebalance(scid31, scid12, "250000000msat")
assert result["status"] == "complete"
assert result["outgoing_scid"] == scid31
assert result["incoming_scid"] == scid12
assert result["hops"] == 3
assert result["received"] == "250000000msat"
# briefly check rebalancereport works
report = l1.rpc.rebalancereport()
assert report.get('rebalanceall_is_running') is False
assert report.get('total_successful_rebalances') == 2
assert report.get("rebalanceall_is_running") is False
assert report.get("total_successful_rebalances") == 2
def test_rebalance_all(node_factory, bitcoind):
@ -100,7 +110,7 @@ def test_rebalance_all(node_factory, bitcoind):
# check we get an error if theres just one channel
result = l1.rpc.rebalanceall()
assert result['message'] == 'Error: Not enough open channels to rebalance anything'
assert result["message"] == "Error: Not enough open channels to rebalance anything"
# now we add another 100% outgoing liquidity to l1 which does not help
l4 = node_factory.get_node()
@ -109,7 +119,7 @@ def test_rebalance_all(node_factory, bitcoind):
# test this is still not possible
result = l1.rpc.rebalanceall()
assert result['message'] == 'Error: Not enough liquidity to rebalance anything'
assert result["message"] == "Error: Not enough liquidity to rebalance anything"
# remove l4 it does not distort further testing
l1.rpc.close(l1.get_channel_scid(l4))
@ -130,29 +140,38 @@ def test_rebalance_all(node_factory, bitcoind):
# check that theres nothing to stop when theres nothing to stop
result = l1.rpc.rebalancestop()
assert result['message'] == "No rebalance is running, nothing to stop."
assert result["message"] == "No rebalance is running, nothing to stop."
# check the rebalanceall starts
result = l1.rpc.rebalanceall(feeratio=5.0) # we need high fees to work
assert result['message'].startswith('Rebalance started')
l1.daemon.wait_for_logs([f"tries to rebalance: {scid12} -> {scid31}",
f"Automatic rebalance finished"])
assert result["message"].startswith("Rebalance started")
l1.daemon.wait_for_logs(
[f"tries to rebalance: {scid12} -> {scid31}", "Automatic rebalance finished"]
)
# check additional calls to stop return 'nothing to stop' + last message
result = l1.rpc.rebalancestop()['message']
assert result.startswith("No rebalance is running, nothing to stop. "
"Last 'rebalanceall' gave: Automatic rebalance finished")
result = l1.rpc.rebalancestop()["message"]
assert result.startswith(
"No rebalance is running, nothing to stop. "
"Last 'rebalanceall' gave: Automatic rebalance finished"
)
# wait until listpeers is up2date
wait_for_all_htlcs(nodes)
# check that channels are now balanced
c12 = l1.rpc.listpeerchannels(l2.info['id'])['channels'][0]
c13 = l1.rpc.listpeerchannels(l3.info['id'])['channels'][0]
assert abs(0.5 - (Millisatoshi(c12['to_us_msat']) / Millisatoshi(c12['total_msat']))) < 0.01
assert abs(0.5 - (Millisatoshi(c13['to_us_msat']) / Millisatoshi(c13['total_msat']))) < 0.01
c12 = l1.rpc.listpeerchannels(l2.info["id"])["channels"][0]
c13 = l1.rpc.listpeerchannels(l3.info["id"])["channels"][0]
assert (
abs(0.5 - (Millisatoshi(c12["to_us_msat"]) / Millisatoshi(c12["total_msat"])))
< 0.01
)
assert (
abs(0.5 - (Millisatoshi(c13["to_us_msat"]) / Millisatoshi(c13["total_msat"])))
< 0.01
)
# briefly check rebalancereport works
report = l1.rpc.rebalancereport()
assert report.get('rebalanceall_is_running') is False
assert report.get('total_successful_rebalances') == 2
assert report.get("rebalanceall_is_running") is False
assert report.get("total_successful_rebalances") == 2

58
sauron/sauron.py
View file

@ -29,7 +29,7 @@ def fetch(url):
backoff_factor=1,
total=10,
status_forcelist=[429, 500, 502, 503, 504],
method_whitelist=["HEAD", "GET", "OPTIONS"]
method_whitelist=["HEAD", "GET", "OPTIONS"],
)
adapter = HTTPAdapter(max_retries=retry_strategy)
@ -64,29 +64,28 @@ def getchaininfo(plugin, **kwargs):
blockhash_url = "{}/block-height/0".format(plugin.api_endpoint)
blockcount_url = "{}/blocks/tip/height".format(plugin.api_endpoint)
chains = {
"000000000019d6689c085ae165831e934ff763ae46a2a6c172b3f1b60a8ce26f":
"main",
"000000000933ea01ad0ee984209779baaec3ced90fa3f408719526f8d77f4943":
"test",
"0f9188f13cb7b2c71f2a335e3a4fc328bf5beb436012afca590b1a11466e2206":
"regtest",
"00000008819873e925422c1ff0f99f7cc9bbb232af63a077a480a3633bee1ef6":
"signet"
"000000000019d6689c085ae165831e934ff763ae46a2a6c172b3f1b60a8ce26f": "main",
"000000000933ea01ad0ee984209779baaec3ced90fa3f408719526f8d77f4943": "test",
"0f9188f13cb7b2c71f2a335e3a4fc328bf5beb436012afca590b1a11466e2206": "regtest",
"00000008819873e925422c1ff0f99f7cc9bbb232af63a077a480a3633bee1ef6": "signet",
}
genesis_req = fetch(blockhash_url)
if not genesis_req.status_code == 200:
raise SauronError("Endpoint at {} returned {} ({}) when trying to "
"get genesis block hash."
.format(blockhash_url, genesis_req.status_code,
genesis_req.text))
raise SauronError(
"Endpoint at {} returned {} ({}) when trying to "
"get genesis block hash.".format(
blockhash_url, genesis_req.status_code, genesis_req.text
)
)
blockcount_req = fetch(blockcount_url)
if not blockcount_req.status_code == 200:
raise SauronError("Endpoint at {} returned {} ({}) when trying to "
"get blockcount.".format(blockcount_url,
blockcount_req.status_code,
blockcount_req.text))
raise SauronError(
"Endpoint at {} returned {} ({}) when trying to " "get blockcount.".format(
blockcount_url, blockcount_req.status_code, blockcount_req.text
)
)
if genesis_req.text not in chains.keys():
raise SauronError("Unsupported network")
plugin.sauron_network = chains[genesis_req.text]
@ -128,8 +127,7 @@ def getrawblock(plugin, height, **kwargs):
break
if int(content_len) == len(block_req.content):
break
plugin.log("Esplora gave us an incomplete block, retrying in 2s",
level="error")
plugin.log("Esplora gave us an incomplete block, retrying in 2s", level="error")
time.sleep(2)
return {
@ -162,16 +160,18 @@ def getutxout(plugin, txid, vout, **kwargs):
gettx_req = fetch(gettx_url)
if not gettx_req.status_code == 200:
raise SauronError("Endpoint at {} returned {} ({}) when trying to "
"get transaction.".format(gettx_url,
gettx_req.status_code,
gettx_req.text))
raise SauronError(
"Endpoint at {} returned {} ({}) when trying to " "get transaction.".format(
gettx_url, gettx_req.status_code, gettx_req.text
)
)
status_req = fetch(status_url)
if not status_req.status_code == 200:
raise SauronError("Endpoint at {} returned {} ({}) when trying to "
"get utxo status.".format(status_url,
status_req.status_code,
status_req.text))
raise SauronError(
"Endpoint at {} returned {} ({}) when trying to " "get utxo status.".format(
status_url, status_req.status_code, status_req.text
)
)
if status_req.json()["spent"]:
return {
@ -219,7 +219,7 @@ def estimatefees(plugin, **kwargs):
plugin.add_option(
"sauron-api-endpoint",
"",
"The URL of the esplora instance to hit (including '/api')."
"The URL of the esplora instance to hit (including '/api').",
)
plugin.add_option(
@ -227,7 +227,7 @@ plugin.add_option(
"",
"Tor's SocksPort address in the form address:port, don't specify the"
" protocol. If you didn't modify your torrc you want to put"
"'localhost:9050' here."
"'localhost:9050' here.",
)

1
sitecustomize.py
View file

@ -1,4 +1,5 @@
import coverage
cov = coverage.process_startup()
if cov is not None:

108
zmq/cl-zmq.py
View file

@ -40,20 +40,22 @@ from pyln.client import Plugin
###############################################################################
NOTIFICATION_TYPE_NAMES = ['channel_opened',
'connect',
'disconnect',
'invoice_payment',
'warning',
'forward_event',
'sendpay_success',
'sendpay_failure']
NOTIFICATION_TYPE_NAMES = [
"channel_opened",
"connect",
"disconnect",
"invoice_payment",
"warning",
"forward_event",
"sendpay_success",
"sendpay_failure",
]
class NotificationType():
""" Wrapper for notification type string to generate the corresponding
plugin option strings. By convention of lightningd, the cli options
use dashes in place of rather than underscores or no spaces."""
class NotificationType:
"""Wrapper for notification type string to generate the corresponding
plugin option strings. By convention of lightningd, the cli options
use dashes in place of rather than underscores or no spaces."""
def __init__(self, notification_type_name):
self.notification_type_name = notification_type_name
@ -73,11 +75,11 @@ NOTIFICATION_TYPES = [NotificationType(n) for n in NOTIFICATION_TYPE_NAMES]
###############################################################################
class Publisher():
""" Holds the connection state and accepts incoming notifications that
come from the subscription. If there is an associated publishing
endpoint connected, it will encode and pass the contents of the
notification. """
class Publisher:
"""Holds the connection state and accepts incoming notifications that
come from the subscription. If there is an associated publishing
endpoint connected, it will encode and pass the contents of the
notification."""
def __init__(self):
self.factory = ZmqFactory()
@ -86,9 +88,9 @@ class Publisher():
def load_setup(self, setup):
for e, s in setup.items():
endpoint = ZmqEndpoint(ZmqEndpointType.bind, e)
ZmqPubConnection.highWaterMark = s['high_water_mark']
ZmqPubConnection.highWaterMark = s["high_water_mark"]
connection = ZmqPubConnection(self.factory, endpoint)
for n in s['notification_type_names']:
for n in s["notification_type_names"]:
self.connection_map[n] = connection
def publish_notification(self, notification_type_name, *args, **kwargs):
@ -104,15 +106,17 @@ publisher = Publisher()
###############################################################################
ZMQ_TRANSPORT_PREFIXES = ['tcp://', "ipc://", 'inproc://', "pgm://", "epgm://"]
ZMQ_TRANSPORT_PREFIXES = ["tcp://", "ipc://", "inproc://", "pgm://", "epgm://"]
class Setup():
""" Does some light validation of the plugin option input and generates a
dictionary to configure the Twisted and ZeroMQ setup """
class Setup:
"""Does some light validation of the plugin option input and generates a
dictionary to configure the Twisted and ZeroMQ setup"""
def _at_least_one_binding(options):
n_bindings = sum(1 for o, v in options.items() if
not o.endswith("-hwm") and v != "null")
n_bindings = sum(
1 for o, v in options.items() if not o.endswith("-hwm") and v != "null"
)
return n_bindings > 0
def _iter_endpoints_not_ok(options):
@ -120,18 +124,29 @@ class Setup():
endpoint_opt = nt.endpoint_option()
endpoint = options[endpoint_opt]
if endpoint != "null":
if len([1 for prefix in ZMQ_TRANSPORT_PREFIXES if
endpoint.startswith(prefix)]) != 0:
if (
len(
[
1
for prefix in ZMQ_TRANSPORT_PREFIXES
if endpoint.startswith(prefix)
]
)
!= 0
):
continue
yield endpoint
def check_option_warnings(options, plugin):
if not Setup._at_least_one_binding(options):
plugin.log("No zmq publish sockets are bound as per launch args",
level='warn')
plugin.log(
"No zmq publish sockets are bound as per launch args", level="warn"
)
for endpoint in Setup._iter_endpoints_not_ok(options):
plugin.log(("Endpoint option {} doesn't appear to be recognized"
).format(endpoint), level='warn')
plugin.log(
("Endpoint option {} doesn't appear to be recognized").format(endpoint),
level="warn",
)
###########################################################################
@ -149,21 +164,21 @@ class Setup():
setup = {}
for e, nt, hwm in Setup._iter_endpoint_setup(options):
if e not in setup:
setup[e] = {'notification_type_names': [],
'high_water_mark': hwm}
setup[e]['notification_type_names'].append(str(nt))
setup[e] = {"notification_type_names": [], "high_water_mark": hwm}
setup[e]["notification_type_names"].append(str(nt))
# use the lowest high water mark given for the endpoint
setup[e]['high_water_mark'] = min(
setup[e]['high_water_mark'], hwm)
setup[e]["high_water_mark"] = min(setup[e]["high_water_mark"], hwm)
return setup
###########################################################################
def log_setup_dict(setup, plugin):
for e, s in setup.items():
m = ("Endpoint {} will get events from {} subscriptions "
"published with high water mark {}")
m = m.format(e, s['notification_type_names'], s['high_water_mark'])
m = (
"Endpoint {} will get events from {} subscriptions "
"published with high water mark {}"
)
m = m.format(e, s["notification_type_names"], s["high_water_mark"])
plugin.log(m)
@ -183,8 +198,9 @@ def init(options, configuration, plugin, **kwargs):
def on_notification(notification_type_name, plugin, *args, **kwargs):
if len(args) != 0:
plugin.log("got unexpected args: {}".format(args), level="warn")
reactor.callFromThread(publisher.publish_notification,
notification_type_name, *args, **kwargs)
reactor.callFromThread(
publisher.publish_notification, notification_type_name, *args, **kwargs
)
DEFAULT_HIGH_WATER_MARK = 1000
@ -197,13 +213,13 @@ for nt in NOTIFICATION_TYPES:
# zmq socket binding option
endpoint_opt = nt.endpoint_option()
endpoint_desc = "Enable publish {} info to ZMQ socket endpoint".format(nt)
plugin.add_option(endpoint_opt, None, endpoint_desc, opt_type='string')
plugin.add_option(endpoint_opt, None, endpoint_desc, opt_type="string")
# high water mark option
hwm_opt = nt.hwm_option()
hwm_desc = ("Set publish {} info message high water mark "
"(default: {})".format(nt, DEFAULT_HIGH_WATER_MARK))
plugin.add_option(hwm_opt, DEFAULT_HIGH_WATER_MARK, hwm_desc,
opt_type='int')
hwm_desc = "Set publish {} info message high water mark " "(default: {})".format(
nt, DEFAULT_HIGH_WATER_MARK
)
plugin.add_option(hwm_opt, DEFAULT_HIGH_WATER_MARK, hwm_desc, opt_type="int")
###############################################################################

31
zmq/example-subscriber.py
View file

@ -42,17 +42,19 @@ from txzmq import ZmqSubConnection
###############################################################################
NOTIFICATION_TYPE_NAMES = ['channel_opened',
'connect',
'disconnect',
'invoice_payment',
'warning',
'forward_event',
'sendpay_success',
'sendpay_failure']
NOTIFICATION_TYPE_NAMES = [
"channel_opened",
"connect",
"disconnect",
"invoice_payment",
"warning",
"forward_event",
"sendpay_success",
"sendpay_failure",
]
class NotificationType():
class NotificationType:
def __init__(self, notification_type_name):
self.notification_type_name = notification_type_name
@ -71,15 +73,16 @@ NOTIFICATION_TYPES = [NotificationType(n) for n in NOTIFICATION_TYPE_NAMES]
###############################################################################
class Subscriber():
class Subscriber:
def __init__(self):
self.factory = ZmqFactory()
def _log_message(self, message, tag):
tag = tag.decode("utf8")
message = json.dumps(json.loads(message.decode("utf8")), indent=1,
sort_keys=True)
current_time = time.strftime('%X %x %Z')
message = json.dumps(
json.loads(message.decode("utf8")), indent=1, sort_keys=True
)
current_time = time.strftime("%X %x %Z")
print("{} - {}\n{}".format(current_time, tag, message))
def _load_setup(self, setup):
@ -109,7 +112,7 @@ class Subscriber():
parser = argparse.ArgumentParser(prog="example-subscriber.py")
for nt in NOTIFICATION_TYPES:
h = "subscribe to {} events published from this endpoint".format(nt)
parser.add_argument('--' + nt.endpoint_option(), type=str, help=h)
parser.add_argument("--" + nt.endpoint_option(), type=str, help=h)
settings = parser.parse_args()
subscriber = Subscriber()