GlcoinCore/blitz_api
1
0
Fork 0
mirror of https://github.com/fusion44/blitz_api.git synced 2026-08-13 11:52:45 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

feat: implement Core Lightning gRPC backend (#98)

Browse source 
Some remaining issues with current implementation:

Some remaining issues with current implementation:

* Some calls require shell access to lightning-cli as some gRPC calls are not yet implemented
* Getting on-chain fund flows is not as nice as it is with LND and required access to the Core Lightning SQlite database
* With current API you can only get current outputs
* High probability of bugs arising

closes #45
This commit is contained in:
fusion44 2022-06-04 14:10:44 +02:00 • committed by GitHub
parent bb6d6a6bff
commit de9a96e621
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
24 changed files with 6240 additions and 463 deletions
Download patch file Download diff file Expand all files Collapse all files

16
.env_sample
View file

@ -71,8 +71,10 @@ bitcoind_zmq_block_port_testnet=28332
bitcoind_user=raspibolt
bitcoind_pw=please_please_update_me_please
# lnd or clightning (clightning is not yet implemented!)
ln_node=lnd
# lnd_grpc, cln_grpc
# Please refer to the documentation for the install procedure
# for each implementation.
ln_node=lnd_grpc
# LND macaroon in HEX format
lnd_macaroon=0201036...2211
# LND certificate in HEX format
@ -81,6 +83,16 @@ lnd_grpc_ip=192.168.1.18
lnd_grpc_port=10009
lnd_rest_port=8080
# cln grpc connection data, cert files are in .lightning data folder
# xxd -p -c2000 client.pem
cln_grpc_cert="2d2d2d2d2d...d2d2d2d0a"
# xxd -p -c2000 client-key.pem
cln_grpc_key="2d2d2d2d2d...d2d2d2d0a"
# xxd -p -c2000 ca.pem
cln_grpc_ca="2d2d2d2d2d...d2d2d2d0a"
cln_grpc_ip=127.0.0.1
cln_grpc_port=9537
# Tor url of this system. Ignored on platform Raspiblitz.
# Defaults to empty string
# np_tor_address=""

1017
app/models/lightning.py
View file

File diff suppressed because it is too large Load diff

98
app/repositories/lightning.py
View file

@ -24,47 +24,11 @@ from app.models.system import APIPlatform
from app.utils import SSE, lightning_config, redis_get, send_sse_message
if lightning_config.ln_node == "lnd":
from app.repositories.ln_impl.lnd import (
add_invoice_impl,
channel_close_impl,
channel_list_impl,
channel_open_impl,
decode_pay_request_impl,
get_fee_revenue_impl,
get_ln_info_impl,
get_wallet_balance_impl,
list_all_tx_impl,
list_invoices_impl,
list_on_chain_tx_impl,
list_payments_impl,
listen_forward_events,
listen_invoices,
new_address_impl,
send_coins_impl,
send_payment_impl,
unlock_wallet_impl,
)
else:
from app.repositories.ln_impl.clightning import (
add_invoice_impl,
channel_close_impl,
channel_list_impl,
channel_open_impl,
decode_pay_request_impl,
get_fee_revenue_impl,
get_ln_info_impl,
get_wallet_balance_impl,
list_all_tx_impl,
list_invoices_impl,
list_on_chain_tx_impl,
list_payments_impl,
listen_forward_events,
listen_invoices,
new_address_impl,
send_coins_impl,
send_payment_impl,
unlock_wallet_impl,
)
import app.repositories.ln_impl.lnd_grpc as ln
elif lightning_config.ln_node == "cln_grpc":
import app.repositories.ln_impl.cln_grpc as ln
elif lightning_config.ln_node == "cln_unix_socket":
import app.repositories.ln_impl.cln_unix_socket as ln
GATHER_INFO_INTERVALL = config("gather_ln_info_interval", default=2, cast=float)
@ -84,24 +48,24 @@ if FWD_GATHER_INTERVAL < 0.3:
async def get_ln_info_lite() -> LightningInfoLite:
ln_info = await get_ln_info_impl()
return LightningInfoLite.from_grpc(ln_info)
ln_info = await ln.get_ln_info_impl()
return LightningInfoLite.from_lninfo(ln_info)
async def get_wallet_balance():
return await get_wallet_balance_impl()
return await ln.get_wallet_balance_impl()
async def list_all_tx(
successfull_only: bool, index_offset: int, max_tx: int, reversed: bool
) -> List[GenericTx]:
return await list_all_tx_impl(successfull_only, index_offset, max_tx, reversed)
return await ln.list_all_tx_impl(successfull_only, index_offset, max_tx, reversed)
async def list_invoices(
pending_only: bool, index_offset: int, num_max_invoices: int, reversed: bool
) -> List[Invoice]:
return await list_invoices_impl(
return await ln.list_invoices_impl(
pending_only,
index_offset,
num_max_invoices,
@ -110,13 +74,13 @@ async def list_invoices(
async def list_on_chain_tx() -> List[OnChainTransaction]:
return await list_on_chain_tx_impl()
return await ln.list_on_chain_tx_impl()
async def list_payments(
include_incomplete: bool, index_offset: int, max_payments: int, reversed: bool
) -> List[Payment]:
return await list_payments_impl(
return await ln.list_payments_impl(
include_incomplete, index_offset, max_payments, reversed
)
@ -124,19 +88,19 @@ async def list_payments(
async def add_invoice(
value_msat: int, memo: str = "", expiry: int = 3600, is_keysend: bool = False
) -> Invoice:
return await add_invoice_impl(memo, value_msat, expiry, is_keysend)
return await ln.add_invoice_impl(memo, value_msat, expiry, is_keysend)
async def decode_pay_request(pay_req: str) -> PaymentRequest:
return await decode_pay_request_impl(pay_req)
return await ln.decode_pay_request_impl(pay_req)
async def new_address(input: NewAddressInput) -> str:
return await new_address_impl(input)
return await ln.new_address_impl(input)
async def send_coins(input: SendCoinsInput) -> SendCoinsResponse:
res = await send_coins_impl(input)
res = await ln.send_coins_impl(input)
_schedule_wallet_balance_update()
return res
@ -147,7 +111,9 @@ async def send_payment(
fee_limit_msat: int,
amount_msat: Optional[int] = None,
) -> Payment:
res = await send_payment_impl(pay_req, timeout_seconds, fee_limit_msat, amount_msat)
res = await ln.send_payment_impl(
pay_req, timeout_seconds, fee_limit_msat, amount_msat
)
_schedule_wallet_balance_update()
return res
@ -168,29 +134,29 @@ async def channel_open(
if not "@" in node_URI:
raise ValueError("node_URI must contain @ with node physical address")
res = await channel_open_impl(local_funding_amount, node_URI, target_confs)
res = await ln.channel_open_impl(local_funding_amount, node_URI, target_confs)
return res
async def channel_list() -> List[Channel]:
res = await channel_list_impl()
res = await ln.channel_list_impl()
return res
async def channel_close(channel_id: int, force_close: bool) -> str:
res = await channel_close_impl(channel_id, force_close)
res = await ln.channel_close_impl(channel_id, force_close)
return res
async def get_ln_info() -> LnInfo:
ln_info = await get_ln_info_impl()
ln_info = await ln.get_ln_info_impl()
if PLATFORM == APIPlatform.RASPIBLITZ:
ln_info.identity_uri = await redis_get("ln_default_address")
return ln_info
async def unlock_wallet(password: str) -> bool:
res = await unlock_wallet_impl(password)
res = await ln.unlock_wallet_impl(password)
if res:
for l in _WALLET_UNLOCK_LISTENERS:
await l.put("unlocked")
@ -198,7 +164,7 @@ async def unlock_wallet(password: str) -> bool:
async def get_fee_revenue() -> FeeRevenue:
return await get_fee_revenue_impl()
return await ln.get_fee_revenue_impl()
async def register_lightning_listener():
@ -211,7 +177,7 @@ async def register_lightning_listener():
"""
try:
await get_ln_info_impl()
await ln.get_ln_info_impl()
loop = asyncio.get_event_loop()
loop.create_task(_handle_info_listener())
@ -245,13 +211,13 @@ async def _handle_info_listener():
last_info = None
last_info_lite = None
while True:
info = await get_ln_info_impl()
info = await ln.get_ln_info_impl()
if last_info != info:
await send_sse_message(SSE.LN_INFO, info.dict())
last_info = info
info_lite = LightningInfoLite.from_grpc(info)
info_lite = LightningInfoLite.from_lninfo(info)
if last_info_lite != info_lite:
await send_sse_message(SSE.LN_INFO_LITE, info_lite.dict())
@ -261,7 +227,7 @@ async def _handle_info_listener():
async def _handle_invoice_listener():
async for i in listen_invoices():
async for i in ln.listen_invoices():
await send_sse_message(SSE.LN_INVOICE_STATUS, i.dict())
_schedule_wallet_balance_update()
@ -290,7 +256,7 @@ async def _handle_forward_event_listener():
_fwd_update_scheduled = False
async for i in listen_forward_events():
async for i in ln.listen_forward_events():
if ENABLE_FWD_NOTIFICATIONS:
_fwd_successes.append(i.dict())
@ -307,7 +273,7 @@ def _schedule_wallet_balance_update():
global _wallet_balance_update_scheduled
_wallet_balance_update_scheduled = True
await asyncio.sleep(1.1)
wb = await get_wallet_balance_impl()
wb = await ln.get_wallet_balance_impl()
if _CACHE["wallet_balance"] != wb:
await send_sse_message(SSE.WALLET_BALANCE, wb.dict())
_CACHE["wallet_balance"] = wb
@ -339,7 +305,7 @@ def listen_for_ssh_unlock():
async def _do_check_unlock():
while True:
try:
_ = await get_ln_info_impl()
_ = await ln.get_ln_info_impl()
for l in _WALLET_UNLOCK_LISTENERS:
await l.put("unlocked")
break

107
app/repositories/ln_impl/clightning.py
View file

@ -1,107 +0,0 @@
from typing import List, Optional
from app.models.lightning import (
Channel,
FeeRevenue,
ForwardSuccessEvent,
GenericTx,
Invoice,
LnInfo,
NewAddressInput,
OnChainTransaction,
Payment,
PaymentRequest,
SendCoinsInput,
SendCoinsResponse,
)
def get_implementation_name() -> str:
return "c-lightning"
async def get_wallet_balance_impl():
raise NotImplementedError("c-lightning not yet implemented")
async def list_all_tx_impl(
successful_only: bool, index_offset: int, max_tx: int, reversed: bool
) -> List[GenericTx]:
raise NotImplementedError("c-lightning not yet implemented")
async def list_invoices_impl(
pending_only: bool, index_offset: int, num_max_invoices: int, reversed: bool
):
raise NotImplementedError("c-lightning not yet implemented")
async def list_on_chain_tx_impl() -> List[OnChainTransaction]:
raise NotImplementedError("c-lightning not yet implemented")
async def list_payments_impl(
include_incomplete: bool, index_offset: int, max_payments: int, reversed: bool
):
raise NotImplementedError("c-lightning not yet implemented")
async def add_invoice_impl(
value_msat: int, memo: str = "", expiry: int = 3600, is_keysend: bool = False
) -> Invoice:
raise NotImplementedError("c-lightning not yet implemented")
async def decode_pay_request_impl(pay_req: str) -> PaymentRequest:
raise NotImplementedError("c-lightning not yet implemented")
async def get_fee_revenue_impl() -> FeeRevenue:
raise NotImplementedError("c-lightning not yet implemented")
async def new_address_impl(input: NewAddressInput) -> str:
raise NotImplementedError("c-lightning not yet implemented")
async def send_coins_impl(input: SendCoinsInput) -> SendCoinsResponse:
raise NotImplementedError("c-lightning not yet implemented")
async def send_payment_impl(
pay_req: str,
timeout_seconds: int,
fee_limit_msat: int,
amount_msat: Optional[int] = None,
) -> Payment:
raise NotImplementedError("c-lightning not yet implemented")
async def get_ln_info_impl() -> LnInfo:
raise NotImplementedError("c-lightning not yet implemented")
async def unlock_wallet_impl(password: str) -> bool:
raise NotImplementedError("c-lightning not yet implemented")
async def listen_invoices() -> Invoice:
raise NotImplementedError("c-lightning not yet implemented")
async def listen_forward_events() -> ForwardSuccessEvent:
raise NotImplementedError("c-lightning not yet implemented")
async def channel_open_impl(
local_funding_amount: int, node_URI: str, target_confs: int
) -> str:
raise NotImplementedError("not yet implemented")
async def channel_list_impl() -> List[Channel]:
raise NotImplementedError("not yet implemented")
async def channel_close_impl(channel_id: int, force_close: bool) -> str:
raise NotImplementedError("not yet implemented")

666
app/repositories/ln_impl/cln_grpc.py Normal file
View file

@ -0,0 +1,666 @@
import asyncio
import json
import logging
import shutil
import sqlite3
import time
from typing import AsyncGenerator, List, Optional
import grpc
from decouple import config
from fastapi.exceptions import HTTPException
from starlette import status
import app.repositories.ln_impl.protos.cln.node_pb2 as ln
import app.repositories.ln_impl.protos.cln.primitives_pb2 as lnp
from app.models.lightning import (
Channel,
FeeRevenue,
ForwardSuccessEvent,
GenericTx,
Invoice,
InvoiceState,
LnInfo,
NewAddressInput,
OnchainAddressType,
OnChainTransaction,
Payment,
PaymentRequest,
SendCoinsInput,
SendCoinsResponse,
TxStatus,
WalletBalance,
)
from app.utils import bitcoin_rpc_async
from app.utils import lightning_config as lncfg
from app.utils import next_push_id
async def _make_local_call(cmd: str):
# FIXME: this is a hack because some of the commands are not exposed
# in the CLN grpc interface yet.
testnet = config("network") == "testnet"
cmd = f"lightning-cli -k {'--testnet ' if testnet else ''}{cmd}"
proc = await asyncio.create_subprocess_shell(
cmd,
stdout=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.PIPE,
)
return await proc.communicate()
def get_implementation_name() -> str:
return "CLN_GRPC"
async def get_wallet_balance_impl() -> WalletBalance:
req = ln.ListfundsRequest()
res = await lncfg.cln_stub.ListFunds(req)
onchain_confirmed = onchain_unconfirmed = onchain_total = 0
for o in res.outputs:
sat = o.amount_msat.msat
onchain_total += sat
if o.status == 0:
onchain_unconfirmed += sat
elif o.status == 1:
onchain_confirmed += sat
# 2 is spent => ignore
chan_local = chan_remote = chan_pending_local = chan_pending_remote = 0
for c in res.channels:
our_msat = c.our_amount_msat.msat
their_msat = c.amount_msat.msat - our_msat
if c.state == 2: # ChanneldNormal
chan_local += our_msat
chan_remote += their_msat
else:
# treat everything else as pending for now
chan_pending_local += our_msat
chan_pending_remote += their_msat
return WalletBalance(
onchain_confirmed_balance=onchain_confirmed,
onchain_total_balance=onchain_total,
onchain_unconfirmed_balance=onchain_unconfirmed,
channel_local_balance=chan_local,
channel_remote_balance=chan_remote,
# TODO: find out how to get these values with CLN
channel_unsettled_local_balance=0,
channel_unsettled_remote_balance=0,
channel_pending_open_local_balance=chan_pending_local,
channel_pending_open_remote_balance=chan_pending_remote,
)
# Decoding the payment request take a long time,
# hence we build a simple cache here.
memo_cache = {}
block_cache = {}
async def _get_block_time(block_height: int) -> tuple:
if block_height is None or block_height < 0:
raise ValueError("block_height cannot be None or negative")
if block_height in block_cache:
return block_cache[block_height]
res = await bitcoin_rpc_async("getblockstats", params=[block_height])
hash = res["result"]["blockhash"]
block = await bitcoin_rpc_async("getblock", params=[hash])
block_cache[block_height] = (block["result"]["time"], block["result"]["mediantime"])
return block_cache[block_height]
# Decoding the payment request take a long time,
# hence we build a simple cache here.
memo_cache = {}
async def list_all_tx_impl(
successfull_only: bool, index_offset: int, max_tx: int, reversed: bool
) -> List[GenericTx]:
list_invoice_req = ln.ListinvoicesRequest()
list_payments_req = ln.ListpaysRequest()
try:
res = await asyncio.gather(
*[
lncfg.cln_stub.ListInvoices(list_invoice_req),
list_on_chain_tx_impl(),
lncfg.cln_stub.ListPays(list_payments_req),
get_ln_info_impl(),
]
)
tx = []
for invoice in res[0].invoices:
i = GenericTx.from_cln_grpc_invoice(invoice)
if successfull_only and i.status == TxStatus.SUCCEEDED:
tx.append(i)
continue
tx.append(i)
for transaction in res[1]:
t = GenericTx.from_cln_grpc_onchain_tx(transaction, res[3].block_height)
if successfull_only and t.status == TxStatus.SUCCEEDED:
tx.append(t)
continue
tx.append(t)
for pay in res[2].pays:
comment = ""
if pay.bolt11 in memo_cache:
comment = memo_cache[pay.bolt11]
else:
pr = await decode_pay_request_impl(pay.bolt11)
comment = pr.description
memo_cache[pay.bolt11] = pr.description
p = GenericTx.from_cln_grpc_payment(pay, comment)
if successfull_only and p.status == TxStatus.SUCCEEDED:
tx.append(p)
continue
tx.append(p)
def sortKey(e: GenericTx):
return e.time_stamp
tx.sort(key=sortKey)
if reversed:
tx.reverse()
l = len(tx)
for invoice in range(l):
tx[invoice].index = invoice
if max_tx == 0:
max_tx = l
return tx[index_offset : index_offset + max_tx]
except grpc.aio._call.AioRpcError as error:
raise HTTPException(
status.HTTP_500_INTERNAL_SERVER_ERROR, detail=error.details()
)
async def list_invoices_impl(
pending_only: bool, index_offset: int, num_max_invoices: int, reversed: bool
) -> List[Invoice]:
req = ln.ListinvoicesRequest()
res = await lncfg.cln_stub.ListInvoices(req)
tx = []
for i in res.invoices:
if pending_only:
if i.status == 0:
tx.append(Invoice.from_cln_grpc(i))
else:
tx.append(Invoice.from_cln_grpc(i))
if reversed:
tx.reverse()
if num_max_invoices == 0 or num_max_invoices is None:
return tx
return tx[index_offset : index_offset + num_max_invoices]
async def list_on_chain_tx_impl() -> List[OnChainTransaction]:
# Make a temporary copy of the file to avoid locking the db.
# CLN might want to write while we read.
info = await get_ln_info_impl()
# FIXME(#87): Once Core Lightnings accountability plugin is available
src = "/home/admin/.lightning/testnet/lightningd.sqlite3"
dest = "/tmp/lightningd.sqlite3"
shutil.copyfile(src, dest)
conn = sqlite3.connect(dest, uri=True)
cur = conn.execute("select * from outputs")
res = cur.fetchall()
conn.close()
txs = []
for o in res:
prev_out_tx = o[0].hex()
amount = o[2]
conf_block = o[9]
spent_block = o[10]
conf_time = (await _get_block_time(conf_block))[0]
txs.append(
OnChainTransaction(
tx_hash=f"prev_out_tx {prev_out_tx}",
amount=amount,
num_confirmations=info.block_height - conf_block,
block_height=conf_block,
time_stamp=conf_time,
total_fees=0,
)
)
if spent_block is not None:
spent_time = (await _get_block_time(spent_block))[0]
txs.append(
OnChainTransaction(
tx_hash=f"prev_out_tx {prev_out_tx}",
amount=-amount,
num_confirmations=info.block_height - spent_block,
block_height=spent_block,
time_stamp=spent_time,
total_fees=0,
),
)
return txs
async def list_payments_impl(
include_incomplete: bool, index_offset: int, max_payments: int, reversed: bool
):
req = ln.ListpaysRequest()
res = await lncfg.cln_stub.ListPays(req)
pays = []
for p in res.pays:
if p.status == 2:
# always include completed payments
pays.append(Payment.from_cln_grpc(p))
continue
if include_incomplete:
pays.append(Payment.from_cln_grpc(p))
if reversed:
pays.reverse()
if max_payments == 0 or max_payments is None:
return pays
return pays[index_offset : index_offset + max_payments]
async def add_invoice_impl(
value_msat: int, memo: str = "", expiry: int = 3600, is_keysend: bool = False
) -> Invoice:
if value_msat < 0:
raise ValueError("value_msat cannot be negative")
msat = None
if value_msat == 0:
msat = lnp.AmountOrAny(any=True)
elif value_msat > 0:
msat = lnp.AmountOrAny(amount=lnp.Amount(msat=value_msat))
id = next_push_id()
req = ln.InvoiceRequest(
msatoshi=msat,
description=memo,
label=id,
expiry=expiry,
)
res = await lncfg.cln_stub.Invoice(req)
return Invoice(
payment_request=res.bolt11,
memo=memo,
value_msat=value_msat,
expiry_date=res.expires_at,
add_index=id,
state=InvoiceState.OPEN,
)
async def decode_pay_request_impl(pay_req: str) -> PaymentRequest:
res = await _make_local_call(f"decodepay bolt11={pay_req}")
if not res:
raise HTTPException(
status.HTTP_500_INTERNAL_SERVER_ERROR,
detail="Unknown CLN error decoding pay request",
)
if len(res) == 0:
raise HTTPException(
status.HTTP_500_INTERNAL_SERVER_ERROR,
detail="No response from CLN decoding pay request",
)
decoded = res[0].decode()
if "Invalid bolt11: Bad bech32 string" in decoded:
raise HTTPException(
status.HTTP_400_BAD_REQUEST, detail="Invalid bolt11: Bad bech32 string"
)
return PaymentRequest.from_cln_json(json.loads(decoded))
async def get_fee_revenue_impl() -> FeeRevenue:
# status 1 == "settled"
req = ln.ListforwardsRequest(status=1)
res = await lncfg.cln_stub.ListForwards(req)
day = week = month = year = total = 0
now = time.time()
t_day = now - 86400.0 # 1 day
t_week = now - 604800.0 # 1 week
t_month = now - 2592000.0 # 1 month
t_year = now - 31536000.0 # 1 year
# TODO: performance: cache this in redis
for f in res.forwards:
received_time = f.received_time
fee = f.fee_msat.msat
total += fee
if received_time > t_day:
day += fee
week += fee
month += fee
year += fee
elif received_time > t_week:
week += fee
month += fee
year += fee
elif received_time > t_month:
month += fee
year += fee
elif received_time > t_year:
year += fee
return FeeRevenue(day=day, week=week, month=month, year=year, total=total)
async def new_address_impl(input: NewAddressInput) -> str:
if input.type == OnchainAddressType.P2WKH:
req = ln.NewaddrRequest(addresstype=2)
res = await lncfg.cln_stub.NewAddr(req)
return res.bech32
req = ln.NewaddrRequest(addresstype=1)
res = await lncfg.cln_stub.NewAddr(req)
return res.p2sh_segwit
async def send_coins_impl(input: SendCoinsInput) -> SendCoinsResponse:
fee_rate: lnp.Feerate = None
if input.sat_per_vbyte != None and input.sat_per_vbyte > 0:
fee_rate = lnp.Feerate(perkw=input.sat_per_vbyte)
elif input.target_conf != None and input.target_conf == 1:
fee_rate = lnp.Feerate(urgent=True)
elif input.target_conf != None and input.target_conf >= 2:
fee_rate = lnp.Feerate(normal=True)
elif input.target_conf != None and input.target_conf >= 10:
fee_rate = lnp.Feerate(slow=True)
try:
funds = await lncfg.cln_stub.ListFunds(ln.ListfundsRequest())
if len(funds.outputs) == 0:
raise HTTPException(
status.HTTP_412_PRECONDITION_FAILED,
detail=f"Could not afford {input.amount}sat. No UTXOs available at all",
)
utxos = []
max_amt = 0
for o in funds.outputs:
utxos.append(lnp.Outpoint(txid=o.txid, outnum=o.output))
max_amt += o.amount_msat.msat
if max_amt <= input.amount:
raise HTTPException(
status.HTTP_412_PRECONDITION_FAILED,
detail=f"Could not afford {input.amount}sat. Not enough funds available",
)
req = ln.WithdrawRequest(
destination=input.address,
satoshi=lnp.AmountOrAll(amount=lnp.Amount(msat=input.amount), all=False),
minconf=input.min_confs,
feerate=fee_rate,
utxos=utxos,
)
res = await lncfg.cln_stub.Withdraw(req)
return SendCoinsResponse.from_cln_grpc(res, input)
except grpc.aio._call.AioRpcError as error:
details = error.details()
if details and details.find("Could not parse destination address") > -1:
raise HTTPException(
status.HTTP_400_BAD_REQUEST,
detail="Could not parse destination address, destination should be a valid address.",
)
elif (
details
and details.find("UTXO") > -1
and details.find("already reserved") > -1
):
raise HTTPException(
status.HTTP_500_INTERNAL_SERVER_ERROR,
detail="Server tried to use a reserved UTXO. Please submit an issue to the BlitzAPI repository.",
)
elif details and details.find("insufficient funds available") > -1:
raise HTTPException(status.HTTP_412_PRECONDITION_FAILED, detail=details)
else:
raise HTTPException(status.HTTP_500_INTERNAL_SERVER_ERROR, detail=details)
async def send_payment_impl(
pay_req: str,
timeout_seconds: int,
fee_limit_msat: int,
amount_msat: Optional[int] = None,
) -> Payment:
amt = lnp.Amount(msat=amount_msat)
fee_limit = lnp.Amount(msat=fee_limit_msat)
req = ln.PayRequest(
bolt11=pay_req,
msatoshi=amt,
maxfee=fee_limit,
retry_for=timeout_seconds,
)
res = await lncfg.cln_stub.Pay(req)
return Payment.from_cln_grpc(res)
async def get_ln_info_impl() -> LnInfo:
req = ln.GetinfoRequest()
res = await lncfg.cln_stub.Getinfo(req)
return LnInfo.from_cln_grpc(get_implementation_name(), res)
async def unlock_wallet_impl(password: str) -> bool:
# Core Lightning doesn't lock wallets,
# so we don't need to do anything here
return True
async def listen_invoices() -> AsyncGenerator[Invoice, None]:
lastpay_index = 0
invoices = await list_invoices_impl(
pending_only=False,
index_offset=0,
num_max_invoices=9999999999999,
reversed=False,
)
for i in invoices: # type Invoice
if i.state == InvoiceState.SETTLED and i.settle_index > lastpay_index:
lastpay_index = i.settle_index
while True:
req = ln.WaitanyinvoiceRequest(lastpay_index=lastpay_index)
i = await lncfg.cln_stub.WaitAnyInvoice(req)
i = Invoice.from_cln_grpc(i)
lastpay_index = i.settle_index
yield i
async def listen_forward_events() -> ForwardSuccessEvent:
# CLN has no subscription to forwarded events.
# We must poll instead.
interval = config("gather_ln_info_interval", default=2, cast=float)
# make sure we know how many forewards we have
# we need to calculate the difference between each iteration
# status=1 == "settled"
req = ln.ListforwardsRequest(status=1)
res = await lncfg.cln_stub.ListForwards(req)
num_fwd_last_poll = len(res.forwards)
while True:
res = await lncfg.cln_stub.ListForwards(req)
if len(res.forwards) > num_fwd_last_poll:
fwds = res.forwards[num_fwd_last_poll:]
for fwd in fwds:
yield ForwardSuccessEvent.from_cln_grpc(fwd)
num_fwd_last_poll = len(res.forwards)
await asyncio.sleep(interval - 0.1)
async def connect_peer_impl(node_URI: str) -> bool:
try:
id = node_URI.split("@")[0]
stdout, stderr = await _make_local_call(f"connect id={node_URI}")
if stdout:
if id in stdout.decode():
return True
if "Connection timed out" in stdout.decode():
raise HTTPException(
status.HTTP_504_GATEWAY_TIMEOUT,
detail="Connection establishment: Connection timed out.",
)
if "Connection refused" in stdout.decode():
raise HTTPException(
status.HTTP_504_GATEWAY_TIMEOUT,
detail="Connection establishment: Connection refused.",
)
if stderr:
logging.error(stderr.decode())
return False
except grpc.aio._call.AioRpcError as error:
raise HTTPException(
status.HTTP_500_INTERNAL_SERVER_ERROR, detail=error.details()
)
async def channel_open_impl(
local_funding_amount: int, node_URI: str, target_confs: int
) -> str:
fee_rate = None
if target_confs == 1:
fee_rate = "urgent"
elif target_confs >= 2 and target_confs <= 9:
fee_rate = "normal"
elif target_confs >= 10:
fee_rate = "slow"
try:
res = await connect_peer_impl(node_URI)
if not res:
raise HTTPException(
status.HTTP_408_REQUEST_TIMEOUT,
detail="Unknown error while trying to connect to peer",
)
cmd = f"fundchannel id={node_URI} amount={local_funding_amount} feerate={fee_rate}"
stdout, stderr = await _make_local_call(cmd)
if stdout:
o = stdout.decode()
j = json.loads(o)
if "txid" in o and "channel_id" in o:
return j["txid"]
if "Unknown peer" in o:
raise HTTPException(
status.HTTP_500_INTERNAL_SERVER_ERROR,
detail="We where able to connect to the peer but CLN can't find it when opening a channel.",
)
if "Owning subdaemon openingd died" in o:
# https://github.com/ElementsProject/lightning/issues/2798#issuecomment-511205719
raise HTTPException(
status.HTTP_400_BAD_REQUEST,
detail="Likely the peer didn't like our channel opening proposal and disconnected from us.",
)
if "Could not afford " in o:
raise HTTPException(status.HTTP_400_BAD_REQUEST, detail=j["message"])
if "Number of pending channels exceed maximum" in o:
raise HTTPException(status.HTTP_400_BAD_REQUEST, detail=j["message"])
raise HTTPException(
status.HTTP_500_INTERNAL_SERVER_ERROR, detail=j["message"]
)
if stderr:
logging.error(stderr.decode())
return False
except grpc.aio._call.AioRpcError as error:
raise HTTPException(
status.HTTP_500_INTERNAL_SERVER_ERROR, detail=error.details()
)
async def channel_list_impl() -> List[Channel]:
try:
i = await get_ln_info_impl()
req = ln.ListchannelsRequest(source=bytes.fromhex(i.identity_pubkey))
res = await lncfg.cln_stub.ListChannels(req)
channels = []
for c in res.channels:
chan = Channel.from_cln_grpc(c)
channels.append(chan)
return channels
except grpc.aio._call.AioRpcError as error:
raise HTTPException(
status.HTTP_500_INTERNAL_SERVER_ERROR, detail=error.details()
)
async def channel_close_impl(channel_id: int, force_close: bool) -> str:
try:
# on CLN we wait for 2 minutes to negotiate a channel close
# if peer doesn't respond we force close
wait_time_before_unilateral_close = 120 if force_close else 0
req = ln.CloseRequest(
id=channel_id,
unilateraltimeout=wait_time_before_unilateral_close,
feerange=[lnp.Feerate(slow=True), lnp.Feerate(urgent=True)],
)
res = await lncfg.cln_stub.Close(req)
# “mutual”, “unilateral”, “unopened”
t = res.item_type
if t == 0 or t == 1: # mutual, unilateral
return res.txid.hex()
elif t == 2: # unopened
raise HTTPException(
status.HTTP_400_BAD_REQUEST, detail="Channel is not open yet."
)
raise HTTPException(
status.HTTP_400_BAD_REQUEST,
detail=f"CLN returned unknown close type: {t}",
)
except grpc.aio._call.AioRpcError as error:
if "Channel is in state AWAITING_UNILATERAL" in error.details():
raise HTTPException(
status.HTTP_400_BAD_REQUEST,
detail="Channel is awaiting an unilateral close.",
)
raise HTTPException(
status.HTTP_500_INTERNAL_SERVER_ERROR, detail=error.details()
)

435
app/repositories/ln_impl/cln_unix_socket.py Normal file
View file

@ -0,0 +1,435 @@
import asyncio
import functools
import shutil
import sqlite3
import time
from typing import AsyncGenerator, List, Optional
from decouple import config
from fastapi.exceptions import HTTPException
from starlette import status
from app.models.lightning import (
FeeRevenue,
ForwardSuccessEvent,
GenericTx,
Invoice,
InvoiceState,
LnInfo,
NewAddressInput,
OnChainTransaction,
Payment,
PaymentRequest,
SendCoinsInput,
SendCoinsResponse,
TxCategory,
TxStatus,
TxType,
WalletBalance,
)
from app.utils import bitcoin_rpc
from app.utils import lightning_config as lncfg
# https://gist.github.com/phizaz/20c36c6734878c6ec053245a477572ec
# pyln does not yet support asyncio, so we need to force wrap them
# with an async function.
def force_async(fn):
"""
turns a sync function to async function using threads
"""
from concurrent.futures import ThreadPoolExecutor
pool = ThreadPoolExecutor()
@functools.wraps(fn)
def wrapper(*args, **kwargs):
future = pool.submit(fn, *args, **kwargs)
return asyncio.wrap_future(future) # make it awaitable
return wrapper
def get_implementation_name() -> str:
return "CLN_UNIX_SOCKET"
async def get_wallet_balance_impl():
@force_async
def _list_funds() -> WalletBalance:
res = lncfg.cln_sock.listfunds()
onchain_confirmed = onchain_unconfirmed = onchain_total = 0
for o in res["outputs"]:
sat = o["value"]
onchain_total += sat
if o["status"] == "confirmed":
onchain_confirmed += sat
else:
onchain_unconfirmed += sat
chan_local = chan_remote = chan_pending_local = chan_pending_remote = 0
for c in res["channels"]:
our_msat = c["our_amount_msat"].millisatoshis
their_msat = c["amount_msat"].millisatoshis - our_msat
if c["state"] == "CHANNELD_NORMAL":
chan_local += our_msat
chan_remote += their_msat
else:
chan_pending_local += our_msat
chan_pending_remote += their_msat
return WalletBalance(
onchain_confirmed_balance=onchain_confirmed,
onchain_total_balance=onchain_total,
onchain_unconfirmed_balance=onchain_unconfirmed,
channel_local_balance=chan_local,
channel_remote_balance=chan_remote,
# TODO: find out how to get these values with CLN
channel_unsettled_local_balance=0,
channel_unsettled_remote_balance=0,
channel_pending_open_local_balance=chan_pending_local,
channel_pending_open_remote_balance=chan_pending_remote,
)
return await _list_funds()
# Decoding the payment request take a long time,
# hence we build a simple cache here.
memo_cache = {}
block_cache = {}
class CLNOutput:
prev_out_tx: str
prev_out_index: int
value: int
type: int
status: int
keyindex: int
channel_id: int
peer_id: str
commitment_point: str
confirmation_height: int
spend_height: int
scriptpubkey: str
reserved_til: int
option_anchor_output: int
csv_lock: int
@classmethod
def from_db_entry(cls, entry):
pass
def _get_block_time(block_height: int) -> tuple:
if block_height is None or block_height < 0:
raise ValueError("block_height cannot be None or negative")
if block_height in block_cache:
print("cache hit")
return block_cache[block_height]
res = bitcoin_rpc("getblockstats", params=[block_height]).json()
hash = res["result"]["blockhash"]
block = bitcoin_rpc("getblock", params=[hash]).json()["result"]
block_cache[block_height] = (block["time"], block["mediantime"])
return block_cache[block_height]
async def list_all_tx_impl(
successfull_only: bool, index_offset: int, max_tx: int, reversed: bool
) -> List[GenericTx]:
@force_async
def _list_invoices():
return lncfg.cln_sock.listinvoices()
@force_async
def _list_payments():
return lncfg.cln_sock.listpays()
@force_async
def _list_transactions(current_block_height: int):
# Make a temporary copy of the file to avoid locking the db.
# CLN might want to write while we read.
src = "/home/fusion44/.lightning/testnet/lightningd.sqlite3"
dest = "/tmp/lightningd.sqlite3"
shutil.copyfile(src, dest)
conn = sqlite3.connect(dest, uri=True)
cur = conn.execute("select * from outputs")
res = cur.fetchall()
conn.close()
txs = []
for o in res:
amount = o[2]
conf_block = o[9]
spent_block = o[10]
conf_time = _get_block_time(conf_block)[0]
txs.append(
GenericTx(
id="my id",
category=TxCategory.ONCHAIN,
type=TxType.RECEIVE,
amount=amount,
time_stamp=conf_time,
status=TxStatus.SUCCEEDED,
comment="",
block_height=conf_block,
num_confs=current_block_height - conf_block,
)
)
if spent_block is not None:
spent_time = _get_block_time(conf_block)[0]
txs.append(
GenericTx(
id="my id",
category=TxCategory.ONCHAIN,
type=TxType.SEND,
amount=amount,
time_stamp=spent_time,
status=TxStatus.SUCCEEDED,
comment="",
block_height=spent_block,
num_confs=current_block_height - spent_block,
)
)
return txs
try:
start = time.time()
info = await get_ln_info_impl() # for the current block height
res = await asyncio.gather(
*[
_list_invoices(),
_list_transactions(info.block_height),
_list_payments(),
]
)
tx = []
for i in res[0]["invoices"]:
tx.append(GenericTx.from_cln_json_invoice(i))
# add all transactions
tx = tx + res[1]
for p in res[2]["pays"]:
bolt11 = p["bolt11"]
comment = ""
if bolt11 in memo_cache:
comment = memo_cache[bolt11]
else:
pr = await decode_pay_request_impl(bolt11)
comment = pr.description
memo_cache[bolt11] = pr.description
tx.append(GenericTx.from_cln_json_payment(p, comment))
def sortKey(e: GenericTx):
return e.time_stamp
tx.sort(key=sortKey)
if reversed:
tx.reverse()
l = len(tx)
for i in range(l):
tx[i].index = i
if max_tx == 0:
max_tx = l
end = time.time()
print("The time of execution of above program is :", end - start)
return tx[index_offset : index_offset + max_tx]
except sqlite3.OperationalError as e:
print("Error while trying to open the database:", e)
async def list_invoices_impl(
pending_only: bool, index_offset: int, num_max_invoices: int, reversed: bool
) -> List[Invoice]:
# TODO: Core Lightning returns way less information about
# the invoice compared to LND. Only way to extract the data is to
# decode the pay request... seems inefficient.
# TODO: Core Lightning does not yet allow for proper paging. Cache this?
@force_async
def _list_invoices():
return lncfg.cln_sock.listinvoices()
res = await _list_invoices()
tx = []
for i in res["invoices"]:
if pending_only:
if i["status"] == "unpaid":
tx.append(Invoice.from_cln_json(i))
else:
tx.append(Invoice.from_cln_json(i))
if reversed:
tx.reverse()
if num_max_invoices == 0 or num_max_invoices is None:
return tx
return tx[index_offset : index_offset + num_max_invoices]
async def list_on_chain_tx_impl() -> List[OnChainTransaction]:
raise NotImplementedError("c-lightning not yet implemented")
async def list_payments_impl(
include_incomplete: bool, index_offset: int, max_payments: int, reversed: bool
):
raise NotImplementedError("c-lightning not yet implemented")
async def add_invoice_impl(
value_msat: int, memo: str = "", expiry: int = 3600, is_keysend: bool = False
) -> Invoice:
raise NotImplementedError("c-lightning not yet implemented")
async def decode_pay_request_impl(pay_req: str) -> PaymentRequest:
@force_async
def _decode() -> PaymentRequest:
return PaymentRequest.from_cln_json(lncfg.cln_sock.decodepay(pay_req))
return await _decode()
async def get_fee_revenue_impl() -> FeeRevenue:
@force_async
def _get_fee_revenue() -> FeeRevenue:
res = lncfg.cln_sock.listforwards(status="settled")
day = week = month = year = total = 0
now = time.time()
t_day = now - 86400.0 # 1 day
t_week = now - 604800.0 # 1 week
t_month = now - 2592000.0 # 1 month
t_year = now - 31536000.0 # 1 year
# TODO: performance: cache this in redis
for f in res["forwards"]:
resolved_time = f["resolved_time"]
fee = f["fee"]
total += fee
if resolved_time > t_day:
day += fee
week += fee
month += fee
year += fee
elif resolved_time > t_week:
week += fee
month += fee
year += fee
elif resolved_time > t_month:
month += fee
year += fee
elif resolved_time > t_year:
year += fee
return FeeRevenue(day=day, week=week, month=month, year=year, total=total)
return await _get_fee_revenue()
async def new_address_impl(input: NewAddressInput) -> str:
raise NotImplementedError("c-lightning not yet implemented")
async def send_coins_impl(input: SendCoinsInput) -> SendCoinsResponse:
raise NotImplementedError("c-lightning not yet implemented")
async def send_payment_impl(
pay_req: str,
timeout_seconds: int,
fee_limit_msat: int,
amount_msat: Optional[int] = None,
) -> Payment:
raise NotImplementedError("c-lightning not yet implemented")
async def get_ln_info_impl() -> LnInfo:
@force_async
def _get_info() -> LnInfo:
res = lncfg.cln_sock.getinfo()
return LnInfo.from_cln_json(get_implementation_name(), res)
return await _get_info()
async def unlock_wallet_impl(password: str) -> bool:
raise NotImplementedError("c-lightning not yet implemented")
async def listen_invoices() -> AsyncGenerator[Invoice, None]:
@force_async
def _wrapper(ln, last_pay_index):
"async wrapper for waitanyinvoice"
return ln.waitanyinvoice(last_pay_index)
lastpay_index = 0
invoices = await list_invoices_impl(
pending_only=False,
index_offset=0,
num_max_invoices=9999999999999,
reversed=False,
)
for i in invoices: # type Invoice
if i.state == InvoiceState.SETTLED and i.settle_index > lastpay_index:
lastpay_index = i.settle_index
# wait for the invoices
try:
while True:
r = await _wrapper(lncfg.cln_sock, last_pay_index=lastpay_index)
r = Invoice.from_cln_json(r)
lastpay_index = r.settle_index
yield r
except TypeError as e:
raise HTTPException(status.HTTP_500_INTERNAL_SERVER_ERROR, detail=e)
except AttributeError as ae:
raise HTTPException(status.HTTP_500_INTERNAL_SERVER_ERROR, detail=ae)
async def listen_forward_events() -> ForwardSuccessEvent:
# CLN has no subscription to forwarded events.
# We must poll instead.
interval = config("gather_ln_info_interval", default=2, cast=float)
if interval > 0.2:
# We don't want to poll too often, as it will slow down the
# server but we still want to be a bit quicker than the
# routine that sends the SSE messages in the lightning
# repository.
interval - 0.1
# make sure we know how many forewards we have
# we need to calculate the difference between each iteration
res = lncfg.cln_sock.listforwards(status="settled")
num_fwd_last_poll = len(res["forwards"])
while True:
res = lncfg.cln_sock.listforwards(status="settled")
if len(res["forwards"]) > num_fwd_last_poll:
fwds = res["forwards"][num_fwd_last_poll:]
for fwd in fwds:
yield ForwardSuccessEvent.from_cln_json(fwd)
num_fwd_last_poll = len(res["forwards"])
await asyncio.sleep(interval - 0.1)

49
app/repositories/ln_impl/lnd.py → app/repositories/ln_impl/lnd_grpc.py
View file

@ -5,9 +5,9 @@ import grpc
from fastapi.exceptions import HTTPException
from starlette import status
import app.repositories.ln_impl.protos.lightning_pb2 as ln
import app.repositories.ln_impl.protos.router_pb2 as router
import app.repositories.ln_impl.protos.walletunlocker_pb2 as unlocker
import app.repositories.ln_impl.protos.lnd.lightning_pb2 as ln
import app.repositories.ln_impl.protos.lnd.router_pb2 as router
import app.repositories.ln_impl.protos.lnd.walletunlocker_pb2 as unlocker
from app.models.lightning import (
Channel,
FeeRevenue,
@ -42,7 +42,7 @@ async def get_wallet_balance_impl() -> WalletBalance:
c_req = ln.ChannelBalanceRequest()
channel = await lncfg.lnd_stub.ChannelBalance(c_req)
return WalletBalance.from_grpc(onchain, channel)
return WalletBalance.from_lnd_grpc(onchain, channel)
except grpc.aio._call.AioRpcError as error:
_check_if_locked(error)
raise HTTPException(
@ -86,9 +86,9 @@ async def list_all_tx_impl(
tx = []
for i in res[0].invoices:
tx.append(GenericTx.from_grpc_invoice(i))
tx.append(GenericTx.from_lnd_grpc_invoice(i))
for t in res[1].transactions:
tx.append(GenericTx.from_grpc_onchain_tx(t))
tx.append(GenericTx.from_lnd_grpc_onchain_tx(t))
for p in res[2].payments:
comment = ""
if p.payment_request in memo_cache:
@ -97,7 +97,7 @@ async def list_all_tx_impl(
pr = await decode_pay_request_impl(p.payment_request)
comment = pr.description
memo_cache[p.payment_request] = pr.description
tx.append(GenericTx.from_grpc_payment(p, comment))
tx.append(GenericTx.from_lnd_grpc_payment(p, comment))
def sortKey(e: GenericTx):
return e.time_stamp
@ -133,7 +133,7 @@ async def list_invoices_impl(
reversed=reversed,
)
response = await lncfg.lnd_stub.ListInvoices(req)
return [Invoice.from_grpc(i) for i in response.invoices]
return [Invoice.from_lnd_grpc(i) for i in response.invoices]
except grpc.aio._call.AioRpcError as error:
_check_if_locked(error)
raise HTTPException(
@ -145,7 +145,7 @@ async def list_on_chain_tx_impl() -> List[OnChainTransaction]:
try:
req = ln.GetTransactionsRequest()
response = await lncfg.lnd_stub.GetTransactions(req)
return [OnChainTransaction.from_grpc(t) for t in response.transactions]
return [OnChainTransaction.from_lnd_grpc(t) for t in response.transactions]
except grpc.aio._call.AioRpcError as error:
_check_if_locked(error)
raise HTTPException(
@ -164,7 +164,7 @@ async def list_payments_impl(
reversed=reversed,
)
response = await lncfg.lnd_stub.ListPayments(req)
return [Payment.from_grpc(p) for p in response.payments]
return [Payment.from_lnd_grpc(p) for p in response.payments]
except grpc.aio._call.AioRpcError as error:
_check_if_locked(error)
raise HTTPException(
@ -185,7 +185,7 @@ async def add_invoice_impl(
response = await lncfg.lnd_stub.AddInvoice(i)
# Can't use Invoice.from_grpc() here because
# Can't use Invoice.from_lnd_grpc() here because
# the response is not a standard invoice
invoice = Invoice(
memo=memo,
@ -210,7 +210,7 @@ async def decode_pay_request_impl(pay_req: str) -> PaymentRequest:
try:
req = ln.PayReqString(pay_req=pay_req)
res = await lncfg.lnd_stub.DecodePayReq(req)
return PaymentRequest.from_grpc(res)
return PaymentRequest.from_lnd_grpc(res)
except grpc.aio._call.AioRpcError as error:
_check_if_locked(error)
if error.details() != None and error.details().find("checksum failed.") > -1:
@ -226,7 +226,7 @@ async def decode_pay_request_impl(pay_req: str) -> PaymentRequest:
async def get_fee_revenue_impl() -> FeeRevenue:
req = ln.FeeReportRequest()
res = await lncfg.lnd_stub.FeeReport(req)
return FeeRevenue.from_grpc(res)
return FeeRevenue.from_lnd_grpc(res)
async def new_address_impl(input: NewAddressInput) -> str:
@ -254,7 +254,7 @@ async def send_coins_impl(input: SendCoinsInput) -> SendCoinsResponse:
)
response = await lncfg.lnd_stub.SendCoins(r)
r = SendCoinsResponse.from_grpc(response, input)
r = SendCoinsResponse.from_lnd_grpc(response, input)
await send_sse_message(SSE.LN_ONCHAIN_PAYMENT_STATUS, r.dict())
return r
except grpc.aio._call.AioRpcError as error:
@ -262,16 +262,13 @@ async def send_coins_impl(input: SendCoinsInput) -> SendCoinsResponse:
details = error.details()
if details and details.find("invalid bech32 string") > -1:
raise HTTPException(
status.HTTP_400_BAD_REQUEST, detail="Invalid payment request string"
status.HTTP_400_BAD_REQUEST,
detail="Could not parse destination address, destination should be a valid address.",
)
elif details and details.find("insufficient funds available") > -1:
raise HTTPException(
status.HTTP_412_PRECONDITION_FAILED, detail=error.details()
)
raise HTTPException(status.HTTP_412_PRECONDITION_FAILED, detail=details)
else:
raise HTTPException(
status.HTTP_500_INTERNAL_SERVER_ERROR, detail=error.details()
)
raise HTTPException(status.HTTP_500_INTERNAL_SERVER_ERROR, detail=details)
async def send_payment_impl(
@ -290,7 +287,7 @@ async def send_payment_impl(
p = None
async for response in lncfg.router_stub.SendPaymentV2(r):
p = Payment.from_grpc(response)
p = Payment.from_lnd_grpc(response)
await send_sse_message(SSE.LN_PAYMENT_STATUS, p.dict())
return p
except grpc.aio._call.AioRpcError as error:
@ -341,7 +338,7 @@ async def get_ln_info_impl() -> LnInfo:
try:
req = ln.GetInfoRequest()
response = await lncfg.lnd_stub.GetInfo(req)
return LnInfo.from_grpc(get_implementation_name(), response)
return LnInfo.from_lnd_grpc(get_implementation_name(), response)
except grpc.aio._call.AioRpcError as error:
_check_if_locked(error)
raise HTTPException(
@ -371,7 +368,7 @@ async def listen_invoices() -> Invoice:
request = ln.InvoiceSubscription()
try:
async for r in lncfg.lnd_stub.SubscribeInvoices(request):
yield Invoice.from_grpc(r)
yield Invoice.from_lnd_grpc(r)
except grpc.aio._call.AioRpcError as error:
_check_if_locked(error)
raise HTTPException(
@ -493,14 +490,14 @@ async def channel_list_impl() -> List[Channel]:
channels = []
for channel_grpc in response.channels:
channel = Channel.from_grpc(channel_grpc)
channel = Channel.from_lnd_grpc(channel_grpc)
channel.peer_alias = await peer_resolve_alias(channel.peer_publickey)
channels.append(channel)
request = ln.PendingChannelsRequest()
response = await lncfg.lnd_stub.PendingChannels(request)
for channel_grpc in response.pending_open_channels:
channel = Channel.from_grpc_pending(channel_grpc.channel)
channel = Channel.from_lnd_grpc_pending(channel_grpc.channel)
channel.peer_alias = await peer_resolve_alias(channel.peer_publickey)
channels.append(channel)

0
app/repositories/ln_impl/protos/__init__.py
View file

12
app/repositories/ln_impl/protos/cln/README.md Normal file
View file

@ -0,0 +1,12 @@
# Build the Python gRPC files
Build for lightningd v0.11.0.1
```sh
cd ~/dev/lightning/clightning/cln-grpc/proto
poetry shell
pip install grpcio grpcio-tools googleapis-common-protos
git clone https://github.com/googleapis/googleapis.git
python -m grpc_tools.protoc --proto_path=googleapis:. --python_out=. --grpc_python_out=. primitives.proto
python -m grpc_tools.protoc --proto_path=googleapis:. --python_out=. --grpc_python_out=. node.proto
```

1929
app/repositories/ln_impl/protos/cln/node_pb2.py Normal file
View file

File diff suppressed because one or more lines are too long

2014
app/repositories/ln_impl/protos/cln/node_pb2_grpc.py Normal file
View file

File diff suppressed because it is too large Load diff

187
app/repositories/ln_impl/protos/cln/primitives_pb2.py Normal file
View file

@ -0,0 +1,187 @@
# -*- coding: utf-8 -*-
# Generated by the protocol buffer compiler. DO NOT EDIT!
# source: primitives.proto
"""Generated protocol buffer code."""
from google.protobuf import descriptor as _descriptor
from google.protobuf import descriptor_pool as _descriptor_pool
from google.protobuf import message as _message
from google.protobuf import reflection as _reflection
from google.protobuf import symbol_database as _symbol_database
from google.protobuf.internal import enum_type_wrapper
# @@protoc_insertion_point(imports)
_sym_db = _symbol_database.Default()
DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(
b'\n\x10primitives.proto\x12\x03\x63ln"\x16\n\x06\x41mount\x12\x0c\n\x04msat\x18\x01 \x01(\x04"D\n\x0b\x41mountOrAll\x12\x1d\n\x06\x61mount\x18\x01 \x01(\x0b\x32\x0b.cln.AmountH\x00\x12\r\n\x03\x61ll\x18\x02 \x01(\x08H\x00\x42\x07\n\x05value"D\n\x0b\x41mountOrAny\x12\x1d\n\x06\x61mount\x18\x01 \x01(\x0b\x32\x0b.cln.AmountH\x00\x12\r\n\x03\x61ny\x18\x02 \x01(\x08H\x00\x42\x07\n\x05value"\x19\n\x17\x43hannelStateChangeCause"(\n\x08Outpoint\x12\x0c\n\x04txid\x18\x01 \x01(\x0c\x12\x0e\n\x06outnum\x18\x02 \x01(\r"h\n\x07\x46\x65\x65rate\x12\x0e\n\x04slow\x18\x01 \x01(\x08H\x00\x12\x10\n\x06normal\x18\x02 \x01(\x08H\x00\x12\x10\n\x06urgent\x18\x03 \x01(\x08H\x00\x12\x0f\n\x05perkb\x18\x04 \x01(\rH\x00\x12\x0f\n\x05perkw\x18\x05 \x01(\rH\x00\x42\x07\n\x05style":\n\nOutputDesc\x12\x0f\n\x07\x61\x64\x64ress\x18\x01 \x01(\t\x12\x1b\n\x06\x61mount\x18\x02 \x01(\x0b\x32\x0b.cln.Amount"t\n\x08RouteHop\x12\n\n\x02id\x18\x01 \x01(\x0c\x12\x18\n\x10short_channel_id\x18\x02 \x01(\t\x12\x1c\n\x07\x66\x65\x65\x62\x61se\x18\x03 \x01(\x0b\x32\x0b.cln.Amount\x12\x0f\n\x07\x66\x65\x65prop\x18\x04 \x01(\r\x12\x13\n\x0b\x65xpirydelta\x18\x05 \x01(\r"(\n\tRoutehint\x12\x1b\n\x04hops\x18\x01 \x03(\x0b\x32\r.cln.RouteHop".\n\rRoutehintList\x12\x1d\n\x05hints\x18\x02 \x03(\x0b\x32\x0e.cln.Routehint*\x1e\n\x0b\x43hannelSide\x12\x06\n\x02IN\x10\x00\x12\x07\n\x03OUT\x10\x01*\x84\x02\n\x0c\x43hannelState\x12\x0c\n\x08Openingd\x10\x00\x12\x1a\n\x16\x43hanneldAwaitingLockin\x10\x01\x12\x12\n\x0e\x43hanneldNormal\x10\x02\x12\x18\n\x14\x43hanneldShuttingDown\x10\x03\x12\x17\n\x13\x43losingdSigexchange\x10\x04\x12\x14\n\x10\x43losingdComplete\x10\x05\x12\x16\n\x12\x41waitingUnilateral\x10\x06\x12\x14\n\x10\x46undingSpendSeen\x10\x07\x12\x0b\n\x07Onchain\x10\x08\x12\x15\n\x11\x44ualopendOpenInit\x10\t\x12\x1b\n\x17\x44ualopendAwaitingLockin\x10\nb\x06proto3'
)
_CHANNELSIDE = DESCRIPTOR.enum_types_by_name["ChannelSide"]
ChannelSide = enum_type_wrapper.EnumTypeWrapper(_CHANNELSIDE)
_CHANNELSTATE = DESCRIPTOR.enum_types_by_name["ChannelState"]
ChannelState = enum_type_wrapper.EnumTypeWrapper(_CHANNELSTATE)
IN = 0
OUT = 1
Openingd = 0
ChanneldAwaitingLockin = 1
ChanneldNormal = 2
ChanneldShuttingDown = 3
ClosingdSigexchange = 4
ClosingdComplete = 5
AwaitingUnilateral = 6
FundingSpendSeen = 7
Onchain = 8
DualopendOpenInit = 9
DualopendAwaitingLockin = 10
_AMOUNT = DESCRIPTOR.message_types_by_name["Amount"]
_AMOUNTORALL = DESCRIPTOR.message_types_by_name["AmountOrAll"]
_AMOUNTORANY = DESCRIPTOR.message_types_by_name["AmountOrAny"]
_CHANNELSTATECHANGECAUSE = DESCRIPTOR.message_types_by_name["ChannelStateChangeCause"]
_OUTPOINT = DESCRIPTOR.message_types_by_name["Outpoint"]
_FEERATE = DESCRIPTOR.message_types_by_name["Feerate"]
_OUTPUTDESC = DESCRIPTOR.message_types_by_name["OutputDesc"]
_ROUTEHOP = DESCRIPTOR.message_types_by_name["RouteHop"]
_ROUTEHINT = DESCRIPTOR.message_types_by_name["Routehint"]
_ROUTEHINTLIST = DESCRIPTOR.message_types_by_name["RoutehintList"]
Amount = _reflection.GeneratedProtocolMessageType(
"Amount",
(_message.Message,),
{
"DESCRIPTOR": _AMOUNT,
"__module__": "primitives_pb2"
# @@protoc_insertion_point(class_scope:cln.Amount)
},
)
_sym_db.RegisterMessage(Amount)
AmountOrAll = _reflection.GeneratedProtocolMessageType(
"AmountOrAll",
(_message.Message,),
{
"DESCRIPTOR": _AMOUNTORALL,
"__module__": "primitives_pb2"
# @@protoc_insertion_point(class_scope:cln.AmountOrAll)
},
)
_sym_db.RegisterMessage(AmountOrAll)
AmountOrAny = _reflection.GeneratedProtocolMessageType(
"AmountOrAny",
(_message.Message,),
{
"DESCRIPTOR": _AMOUNTORANY,
"__module__": "primitives_pb2"
# @@protoc_insertion_point(class_scope:cln.AmountOrAny)
},
)
_sym_db.RegisterMessage(AmountOrAny)
ChannelStateChangeCause = _reflection.GeneratedProtocolMessageType(
"ChannelStateChangeCause",
(_message.Message,),
{
"DESCRIPTOR": _CHANNELSTATECHANGECAUSE,
"__module__": "primitives_pb2"
# @@protoc_insertion_point(class_scope:cln.ChannelStateChangeCause)
},
)
_sym_db.RegisterMessage(ChannelStateChangeCause)
Outpoint = _reflection.GeneratedProtocolMessageType(
"Outpoint",
(_message.Message,),
{
"DESCRIPTOR": _OUTPOINT,
"__module__": "primitives_pb2"
# @@protoc_insertion_point(class_scope:cln.Outpoint)
},
)
_sym_db.RegisterMessage(Outpoint)
Feerate = _reflection.GeneratedProtocolMessageType(
"Feerate",
(_message.Message,),
{
"DESCRIPTOR": _FEERATE,
"__module__": "primitives_pb2"
# @@protoc_insertion_point(class_scope:cln.Feerate)
},
)
_sym_db.RegisterMessage(Feerate)
OutputDesc = _reflection.GeneratedProtocolMessageType(
"OutputDesc",
(_message.Message,),
{
"DESCRIPTOR": _OUTPUTDESC,
"__module__": "primitives_pb2"
# @@protoc_insertion_point(class_scope:cln.OutputDesc)
},
)
_sym_db.RegisterMessage(OutputDesc)
RouteHop = _reflection.GeneratedProtocolMessageType(
"RouteHop",
(_message.Message,),
{
"DESCRIPTOR": _ROUTEHOP,
"__module__": "primitives_pb2"
# @@protoc_insertion_point(class_scope:cln.RouteHop)
},
)
_sym_db.RegisterMessage(RouteHop)
Routehint = _reflection.GeneratedProtocolMessageType(
"Routehint",
(_message.Message,),
{
"DESCRIPTOR": _ROUTEHINT,
"__module__": "primitives_pb2"
# @@protoc_insertion_point(class_scope:cln.Routehint)
},
)
_sym_db.RegisterMessage(Routehint)
RoutehintList = _reflection.GeneratedProtocolMessageType(
"RoutehintList",
(_message.Message,),
{
"DESCRIPTOR": _ROUTEHINTLIST,
"__module__": "primitives_pb2"
# @@protoc_insertion_point(class_scope:cln.RoutehintList)
},
)
_sym_db.RegisterMessage(RoutehintList)
if _descriptor._USE_C_DESCRIPTORS == False:
DESCRIPTOR._options = None
_CHANNELSIDE._serialized_start = 632
_CHANNELSIDE._serialized_end = 662
_CHANNELSTATE._serialized_start = 665
_CHANNELSTATE._serialized_end = 925
_AMOUNT._serialized_start = 25
_AMOUNT._serialized_end = 47
_AMOUNTORALL._serialized_start = 49
_AMOUNTORALL._serialized_end = 117
_AMOUNTORANY._serialized_start = 119
_AMOUNTORANY._serialized_end = 187
_CHANNELSTATECHANGECAUSE._serialized_start = 189
_CHANNELSTATECHANGECAUSE._serialized_end = 214
_OUTPOINT._serialized_start = 216
_OUTPOINT._serialized_end = 256
_FEERATE._serialized_start = 258
_FEERATE._serialized_end = 362
_OUTPUTDESC._serialized_start = 364
_OUTPUTDESC._serialized_end = 422
_ROUTEHOP._serialized_start = 424
_ROUTEHOP._serialized_end = 540
_ROUTEHINT._serialized_start = 542
_ROUTEHINT._serialized_end = 582
_ROUTEHINTLIST._serialized_start = 584
_ROUTEHINTLIST._serialized_end = 630
# @@protoc_insertion_point(module_scope)

3
app/repositories/ln_impl/protos/cln/primitives_pb2_grpc.py Normal file
View file

@ -0,0 +1,3 @@
# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT!
"""Client and server classes corresponding to protobuf-defined services."""
import grpc

0
app/repositories/ln_impl/protos/lightning_pb2.py → app/repositories/ln_impl/protos/lnd/lightning_pb2.py
View file

2
app/repositories/ln_impl/protos/lightning_pb2_grpc.py → app/repositories/ln_impl/protos/lnd/lightning_pb2_grpc.py
View file

@ -2,7 +2,7 @@
"""Client and server classes corresponding to protobuf-defined services."""
import grpc
import app.repositories.ln_impl.protos.lightning_pb2 as lightning__pb2
import app.repositories.ln_impl.protos.lnd.lightning_pb2 as lightning__pb2
class LightningStub(object):

2
app/repositories/ln_impl/protos/router_pb2.py → app/repositories/ln_impl/protos/lnd/router_pb2.py
View file

@ -8,7 +8,7 @@ from google.protobuf import reflection as _reflection
from google.protobuf import symbol_database as _symbol_database
from google.protobuf.internal import enum_type_wrapper
import app.repositories.ln_impl.protos.lightning_pb2 as lightning__pb2
import app.repositories.ln_impl.protos.lnd.lightning_pb2 as lightning__pb2
# @@protoc_insertion_point(imports)

4
app/repositories/ln_impl/protos/router_pb2_grpc.py → app/repositories/ln_impl/protos/lnd/router_pb2_grpc.py
View file

@ -2,8 +2,8 @@
"""Client and server classes corresponding to protobuf-defined services."""
import grpc
import app.repositories.ln_impl.protos.lightning_pb2 as lightning__pb2
import app.repositories.ln_impl.protos.router_pb2 as router__pb2
import app.repositories.ln_impl.protos.lnd.lightning_pb2 as lightning__pb2
import app.repositories.ln_impl.protos.lnd.router_pb2 as router__pb2
class RouterStub(object):

0
app/repositories/ln_impl/protos/signer_pb2.py → app/repositories/ln_impl/protos/lnd/signer_pb2.py
View file

0
app/repositories/ln_impl/protos/signer_pb2_grpc.py → app/repositories/ln_impl/protos/lnd/signer_pb2_grpc.py
View file

2
app/repositories/ln_impl/protos/walletunlocker_pb2.py → app/repositories/ln_impl/protos/lnd/walletunlocker_pb2.py
View file

@ -12,7 +12,7 @@ from google.protobuf import symbol_database as _symbol_database
_sym_db = _symbol_database.Default()
import app.repositories.ln_impl.protos.lightning_pb2 as lightning__pb2
import app.repositories.ln_impl.protos.lnd.lightning_pb2 as lightning__pb2
DESCRIPTOR = _descriptor.FileDescriptor(
name="walletunlocker.proto",

2
app/repositories/ln_impl/protos/walletunlocker_pb2_grpc.py → app/repositories/ln_impl/protos/lnd/walletunlocker_pb2_grpc.py
View file

@ -2,7 +2,7 @@
"""Client and server classes corresponding to protobuf-defined services."""
import grpc
import app.repositories.ln_impl.protos.walletunlocker_pb2 as walletunlocker__pb2
import app.repositories.ln_impl.protos.lnd.walletunlocker_pb2 as walletunlocker__pb2
class WalletUnlockerStub(object):

18
app/routers/lightning.py
View file

@ -42,6 +42,7 @@ from app.repositories.lightning import (
from app.routers.lightning_docs import (
get_balance_response_desc,
new_address_desc,
open_channel_desc,
send_coins_desc,
send_payment_desc,
)
@ -51,7 +52,9 @@ _PREFIX = "lightning"
router = APIRouter(prefix=f"/{_PREFIX}", tags=["Lightning"])
responses = {
423: {"description": "Wallet is locked. Unlock via /lightning/unlock-wallet"}
423: {
"description": "LND only: Wallet is locked. Unlock via /lightning/unlock-wallet."
}
}
@ -273,7 +276,10 @@ async def new_address_path(input: NewAddressInput):
response_description="Either an error or a SendCoinsResponse object on success",
dependencies=[Depends(JWTBearer())],
response_model=SendCoinsResponse,
responses=responses,
responses={
412: {"description": "When not enough funds are available."},
423: responses[423],
},
)
async def send_coins_path(input: SendCoinsInput):
try:
@ -288,10 +294,14 @@ async def send_coins_path(input: SendCoinsInput):
"/open-channel",
name=f"{_PREFIX}.open-channel",
summary="open a new lightning channel",
description="For additional information see [LND docs](https://api.lightning.community/#openchannel)",
description=open_channel_desc,
dependencies=[Depends(JWTBearer())],
response_model=str,
responses=responses,
responses={
412: {"description": "When not enough funds are available."},
423: responses[423],
504: {"description": "When the peer is not reachable."},
},
)
async def channelopen(local_funding_amount: int, node_URI: str, target_confs: int = 3):
try:

24
app/routers/lightning_docs.py
View file

@ -1,3 +1,13 @@
add_invoice_desc = """
Adds a new invoice to the database.
LND is generating a unique auto-incrementing `add_index` for the invoice.
CLN will receive a [Firebase-like PushID](https://firebase.blog/posts/2015/02/the-2120-ways-to-ensure-unique_68) from the backend for the `label` when creating the invoice.
Please refer to the response schema docs for more information.
"""
get_balance_response_desc = """
A JSON String with on chain wallet balances with on-chain balances in
**sat** and channel balances in **msat**. Detailed description is in
@ -31,3 +41,17 @@ This endpoints attempts to pay a payment request.
Intermediate status updates will be sent via the SSE channel. This endpoint returns the last success or error message from the node.
"""
open_channel_desc = """
__open-channel__ attempts to open a channel with a peer.
### LND:
__target_conf__: The target number of blocks that the funding transaction should be confirmed by.
### c-lightning:
* Set __target_conf__ ==1: interpreted as urgent (aim for next block)
* Set __target_conf__ >=2: interpreted as normal (next 4 blocks or so, **default**)
* Set __target_cont__ >=10: interpreted as slow (next 100 blocks or so)
> 👉 See [https://lightning.readthedocs.io/lightning-txprepare.7.html](https://lightning.readthedocs.io/lightning-txprepare.7.html)
"""

116
app/utils.py
View file

@ -1,8 +1,11 @@
import array
import asyncio
import json
import logging
import os
import random
import re
import time
from types import coroutine
from typing import Dict
@ -14,9 +17,19 @@ from fastapi.encoders import jsonable_encoder
from fastapi_plugins import redis_plugin
from starlette import status
import app.repositories.ln_impl.protos.lightning_pb2_grpc as lnrpc
import app.repositories.ln_impl.protos.router_pb2_grpc as routerrpc
import app.repositories.ln_impl.protos.walletunlocker_pb2_grpc as unlockerrpc
node_type = config("ln_node")
if node_type == "lnd":
import app.repositories.ln_impl.protos.lnd.lightning_pb2_grpc as lnrpc
import app.repositories.ln_impl.protos.lnd.router_pb2_grpc as routerrpc
import app.repositories.ln_impl.protos.lnd.walletunlocker_pb2_grpc as unlockerrpc
elif node_type == "cln_grpc":
import app.repositories.ln_impl.protos.cln.node_pb2_grpc as clnrpc
elif node_type == "cln_unix_socket":
from pyln.client import LightningRpc
else:
raise ValueError(f"Unknown node type: {node_type}")
from app.models.bitcoind import BlockRpcFunc
@ -48,8 +61,9 @@ class LightningConfig:
def __init__(self) -> None:
self.network = config("network")
self.ln_node = config("ln_node")
self.cln_sock: "LightningRpc" = None
if self.ln_node == "lnd":
if self.ln_node == "lnd_grpc":
# Due to updated ECDSA generated tls.cert we need to let gprc know that
# we need to use that cipher suite otherwise there will be a handshake
# error when we communicate with the lnd rpc server.
@ -74,9 +88,22 @@ class LightningConfig:
self.lnd_stub = lnrpc.LightningStub(self._channel)
self.router_stub = routerrpc.RouterStub(self._channel)
self.wallet_unlocker = unlockerrpc.WalletUnlockerStub(self._channel)
elif self.ln_node == "clightning":
# TODO: implement c-lightning
pass
elif self.ln_node == "cln_unix_socket":
self._cln_socket_path = config("cln_socket_path")
self.cln_sock = LightningRpc(self._cln_socket_path) # type: LightningRpc
elif self.ln_node == "cln_grpc":
cln_grpc_cert = bytes.fromhex(config("cln_grpc_cert"))
cln_grpc_key = bytes.fromhex(config("cln_grpc_key"))
cln_grpc_ca = bytes.fromhex(config("cln_grpc_ca"))
cln_grpc_url = config("cln_grpc_ip") + ":" + config("cln_grpc_port")
creds = grpc.ssl_channel_credentials(
root_certificates=cln_grpc_ca,
private_key=cln_grpc_key,
certificate_chain=cln_grpc_cert,
)
opts = (("grpc.ssl_target_name_override", "cln"),)
self._channel = grpc.aio.secure_channel(cln_grpc_url, creds, options=opts)
self.cln_stub = clnrpc.NodeStub(self._channel)
elif self.ln_node == "":
# its ok to run raspiblitz also without lightning
pass
@ -238,3 +265,78 @@ def parse_key_value_lines(lines: list) -> dict:
def parse_key_value_text(text: str) -> dict:
return parse_key_value_lines(text.splitlines())
# https://gist.github.com/risent/4cab3878d995bec7d1c2
# https://firebase.blog/posts/2015/02/the-2120-ways-to-ensure-unique_68
# https://gist.github.com/mikelehen/3596a30bd69384624c11
class _PushID(object):
# Modeled after base64 web-safe chars, but ordered by ASCII.
PUSH_CHARS = (
"-0123456789" "ABCDEFGHIJKLMNOPQRSTUVWXYZ" "_abcdefghijklmnopqrstuvwxyz"
)
def __init__(self):
# Timestamp of last push, used to prevent local collisions if you
# pushtwice in one ms.
self.last_push_time = 0
# We generate 72-bits of randomness which get turned into 12
# characters and appended to the timestamp to prevent
# collisions with other clients. We store the last characters
# we generated because in the event of a collision, we'll use
# those same characters except "incremented" by one.
self.last_rand_chars = array.array("i", [i for i in range(12)])
def next_id(self):
now = int(time.time() * 1000)
duplicate_time = now == self.last_push_time
self.last_push_time = now
time_stamp_chars = array.array("u", "12345678")
for i in range(7, -1, -1):
time_stamp_chars[i] = self.PUSH_CHARS[now % 64]
now = int(now / 64)
if now != 0:
raise ValueError("We should have converted the entire timestamp.")
uid = "".join(time_stamp_chars)
if not duplicate_time:
for i in range(12):
self.last_rand_chars[i] = int(random.random() * 64)
else:
# If the timestamp hasn't changed since last push, use the
# same random number, except incremented by 1.
for i in range(11, -1, -1):
if self.last_rand_chars[i] == 63:
self.last_rand_chars[i] = 0
else:
break
self.last_rand_chars[i] += 1
for i in range(12):
uid += self.PUSH_CHARS[self.last_rand_chars[i]]
if len(uid) != 20:
raise ValueError("Length should be 20.")
return uid
pid_gen = _PushID()
def next_push_id() -> str:
"""Generates a unique random 20 character long string id
* They're based on timestamp so that they sort *after* any existing ids.
* They contain 72-bits of random data after the timestamp so that IDs won't collide with other clients' IDs.
* They sort *lexicographically* (so the timestamp is converted to characters that will sort properly).
* They're monotonically increasing. Even if you generate more than one in the same timestamp, the
latter ones will sort after the former ones. We do this by using the previous random bits
but "incrementing" them by 1 (only in the case of a timestamp collision).
"""
return pid_gen.next_id()