add inbound formulas to avoid flags

CHANGELOG.md

@@ -1,5 +1,14 @@
 # Versions
 
+## Version 10.11.0
+
+- `accounting`: Remove non-fee sweep amounts
+- `open`: Wait for chain fee entry before attempting channel open when using internal funds
+- `open`: Increase speed of internal UTXO lock canceling when internal funding open fails
+- `rebalance`: Add formula support to avoids for in-policy avoiding
+- `rebalance`: Automatically retry on liquidity contention and other remote errors
+- `transfer`: Add tag icons to route evaluation output
+
 ## Version 10.10.2
 
 - `balance`: For off-chain balances, switch commitment fee adjustment to credit for commit fees

README.md

@@ -64,9 +64,6 @@ bos cert-validity-days
 # Receive on-chain funds via a regular address
 bos chain-deposit
 
-# Receive funds via swap on-chain
-bos chain-receive "amount"
-
 # See the current fee estimates confirmation targets
 bos chainfees
 
@@ -88,9 +85,6 @@ bos closed
 # Export credentials
 bos credentials
 
-# Create a utxo fan-out with on-chain funds
-bos fanout "amount" "count"
-
 # View outbound fee rates and update outbound fee rates to peers
 bos fees
 
@@ -127,7 +121,7 @@ bos increase-outbound-liquidity
 # View and adjust list of saved nodes
 bos nodes
 
-# Open channels to public keys using external funding
+# Open channels to public keys in a batch transaction
 bos open "public_keys..."
 
 # Outputs the sum total of local channel liquidity
@@ -145,9 +139,6 @@ bos price
 # Test if funds can be sent to a destination
 bos probe "payment_request/public_key"
 
-# Make a ping keysend-request to a node
-bos purchase-ping "public_key"
-
 # Rebalance funds between peers
 bos rebalance
 
@@ -160,9 +151,6 @@ bos report
 # Send funds using keysend and an optional message to a node
 bos send
 
-# Service keysend-requests
-bos service-keysend-requests
-
 # Connect up to a Telegram bot
 bos telegram
 

bos

@@ -1176,9 +1176,15 @@ prog
   // Rebalance between peers
   .command('rebalance', 'Rebalance funds between peers')
   .help('Change the liquidity profile of two peers')
-  .help('When specifying amount you can use formulas or *k *m for *1e3, *1e6')
   .help('Specifying target liquidity you can use CAPACITY/2, other formulas')
-  .help('You can specify tags for --avoid, --in, --out')
+  .help('You can specify tags for --avoid, --in, --out (see help tags)')
+  .help('--amount can take m/k variables: 5*m for 5 million, 250*k = 0.0025')
+  .help('--avoid can take a channel id or a public key to avoid')
+  .help('--avoid can take a public_key/public_key to avoid a directed pair')
+  .help('--avoid can take a FORMULA/public_key to avoid inbound peers')
+  .help('--avoid FORMULA variables: FEE_RATE, BASE_FEE, HEIGHT')
+  .help('--in decreases the inbound liquidity with a specific peer/tag')
+  .help('--out increases the inbound liquidity with a specific peer/tag')
   .option('--amount <amount>', 'Maximum amount to rebalance')
   .option('--avoid <pubkey_or_chanid>', 'Avoid forwarding through', REPEATABLE)
   .option('--in <pubkey_or_alias>', 'Route in through a specific peer')
@@ -1194,25 +1200,24 @@ prog
   .action((args, options, logger) => {
     return new Promise(async (resolve, reject) => {
       try {
-        return swaps.rebalance({
+        return swaps.manageRebalance({
           logger,
           avoid: flatten([options.avoid].filter(n => !!n)),
           fs: {getFile: readFile},
-          in_through: options.in || undefined,
           in_outbound: options.inTargetOutbound || undefined,
-          lnd: (await lnd.authenticatedLnd({logger, node: options.node})).lnd,
+          in_through: options.in || undefined,
+          lnd: (await lndForNode(logger, options.node)).lnd,
           max_fee: options.maxFee,
           max_fee_rate: options.maxFeeRate,
           max_rebalance: options.amount,
-          node: options.node || undefined,
           out_channels: flatten([options.outChannel].filter(n => !!n)),
           out_inbound: options.outTargetInbound,
           out_through: options.out || undefined,
           timeout_minutes: options.minutes || undefined,
         },
-        responses.returnObject({exit, logger, reject, resolve}));
+        responses.returnObject({logger, reject, resolve}));
       } catch (err) {
-        return reject(err);
+        return reject(logger.error({err}));
       }
     });
   })
@@ -1559,6 +1564,7 @@ prog
           logger,
           amount: args.amount,
           description: options.description || undefined,
+          fs: {getFile: readFile},
           in_through: options.in,
           is_dry_run: options.dryrun,
           lnd: (await lndForNode(logger, options.node)).lnd,

display/constants.json

@@ -0,0 +1,10 @@
+{
+  "parseErrors": {
+    "#DIV/0!": "CannotDivideByZeroInFormula",
+    "#ERROR!":  "FailedToParseFormula",
+    "#N/A": "UnrecognizedVariableOrFunctionInFormula",
+    "#NAME?": "UnrecognizedVariableOrFunctionInFormula",
+    "#NUM!": "InvalidNumberFoundInFormula",
+    "#VALUE!": "UnexpectedValueTypeInFormula"
+  }
+}

display/describe_parse_error.js

@@ -0,0 +1,16 @@
+const {parseErrors} = require('./constants');
+
+const defaultError = '#ERROR!';
+
+/** Describe a hot formula parser error
+
+  {
+    error: <Error Type String>
+  }
+
+  @returns
+  <Display String>
+*/
+module.exports = ({error}) => {
+  return parseErrors[error] || parseErrors[defaultError];
+};

display/index.js

@@ -1,5 +1,6 @@
 const chartAliasForPeer = require('./chart_alias_for_peer');
 const describeConfidence = require('./describe_confidence');
+const describeParseError = require('./describe_parse_error');
 const describeRoute = require('./describe_route');
 const describeRoutingFailure = require('./describe_routing_failure');
 const formatFeeRate = require('./format_fee_rate');
@@ -11,6 +12,7 @@ const sumsForSegment = require('./sums_for_segment');
 module.exports = {
   chartAliasForPeer,
   describeConfidence,
+  describeParseError,
   describeRoute,
   describeRoutingFailure,
   formatFeeRate,

network/open_channels.js

@@ -25,6 +25,7 @@ const moment = require('moment');
 const {returnResult} = require('asyncjs-util');
 const {Transaction} = require('bitcoinjs-lib');
 const {transactionAsPsbt} = require('psbt');
+const {unlockUtxo} = require('ln-service');
 
 const adjustFees = require('./../routing/adjust_fees');
 const {getAddressUtxo} = require('./../chain');
@@ -304,6 +305,7 @@ module.exports = (args, cbk) => {
 
       // Initiate open requests
       openChannels: [
+        'askForFeeRate',
         'capacities',
         'connect',
         'getWalletVersion',
@@ -522,7 +524,43 @@ module.exports = (args, cbk) => {
         },
         () => {
           // Return the original error that canceled the finalization
-          return cbk(error);
+          return cbk(null, error);
+        });
+      }],
+
+      // Cancel UTXO locks if they are present
+      cancelLocks: [
+        'cancelPending',
+        'getFunding',
+        ({cancelPending, getFunding}, cbk) =>
+      {
+        // Exit early when there is no error that caused a cancel
+        if (!cancelPending) {
+          return cbk();
+        }
+
+        // Exit early when there is no UTXOs to unlock
+        if (!isArray(getFunding.inputs)) {
+          return cbk(cancelPending);
+        }
+
+        // Unlock UTXOs locked from internal funding
+        return asyncEach(getFunding.inputs, (input, cbk) => {
+          // Potentially the UTXO will be relocked with a new id, but attempt
+          return unlockUtxo({
+            id: input.lock_id,
+            lnd: args.lnd,
+            transaction_id: input.transaction_id,
+            transaction_vout: input.transaction_vout,
+          },
+          () => {
+            //Ignore errors when trying to cancel a locked UTXO, it'll timeout
+            return cbk();
+          });
+        },
+        () => {
+          // Return the original error that caused the cancel
+          return cbk(cancelPending);
         });
       }],
 

network/probe_destination.js

@@ -30,6 +30,7 @@ const fromKeyType = '34349339';
 const {isArray} = Array;
 const keySendPreimageType = '5482373484';
 const makeNonce = () => randomBytes(32).toString('hex');
+const {max} = Math;
 const messageType = '34349334';
 const nodeKeyFamily = 6;
 const preimageByteLength = 32;
@@ -374,8 +375,14 @@ module.exports = (args, cbk) => {
         });
 
         sub.once('error', err => cbk(err));
-        sub.once('failure', () => cbk(null, {maximum: Number()}));
-        sub.once('success', ({maximum}) => cbk(null, {maximum}));
+
+        // Did not find any higher value routes
+        sub.once('failure', () => cbk(null, {maximum: probe.route.tokens}));
+
+        // Found a successful high value route
+        sub.once('success', ({maximum}) => {
+          return cbk(null, {maximum: max(maximum, probe.route.tokens)});
+        });
 
         return;
       }],

network/remove_peer.js

@@ -9,6 +9,7 @@ const {returnResult} = require('asyncjs-util');
 const {getMempoolSize} = require('./../chain');
 const getPeers = require('./get_peers');
 
+const arrayWithEntries = arr => !!arr.length ? arr : undefined;
 const asOutpoint = n => `${n.transaction_id}:${n.transaction_vout}`;
 const fastConf = 6;
 const {floor} = Math;
@@ -158,6 +159,11 @@ module.exports = (args, cbk) => {
           return args.outpoints.includes(asOutpoint(channel));
         });
 
+        const costToClose = selectedChannels
+          .filter(n => n.is_partner_initiated === false)
+          .map(n => n.commit_transaction_fee)
+          .reduce((sum, n) => sum + n, Number());
+
         const [cannotCoopClose] = selectedChannels.filter(channel => {
           // Inactive channels cannot be cooperatively closed
           if (!channel.is_active) {
@@ -177,7 +183,8 @@ module.exports = (args, cbk) => {
         if (!!cannotCoopClose) {
           return cbk([400, 'CannotCurrentlyCooperativelyCloseWithPeer', {
             is_active: cannotCoopClose.is_active,
-            pending: cannotCoopClose.pending_payments,
+            pending: arrayWithEntries(cannotCoopClose.pending_payments),
+            cost_to_force_close: costToClose,
           }]);
         }
 

network/transfer_funds.js

@@ -23,6 +23,9 @@ const tokAsBigTok = tokens => !tokens ? undefined : (tokens / 1e8).toFixed(8);
   {
     amount: <Amount to Transfer Tokens String>
     [description]: <Description String>
+    [fs]: {
+      getFile: <Read File Contents Function> (path, cbk) => {}
+    }
     [in_through]: <Transfer In Through Peer String>
     [is_dry_run]: <Do Not Transfer Bool>
     lnd: <Authenticated LND API Object>
@@ -42,6 +45,10 @@ module.exports = (args, cbk) => {
           return cbk([400, 'ExpectedAmountToTransferFundsToSavedNode']);
         }
 
+        if (!args.fs) {
+          return cbk([400, 'ExpectedFileSystemMethodsToTransferFundsToNode']);
+        }
+
         if (!!args.in_through && !!isArray(args.in_through)) {
           return cbk([400, 'MultipleInThroughPeersNotSupported']);
         }
@@ -221,6 +228,7 @@ module.exports = (args, cbk) => {
 
         return probeDestination({
           destination: getToKey.public_key,
+          fs: args.fs,
           lnd: args.lnd,
           logger: args.logger,
           in_through: getInKey.public_key,

package.json

@@ -26,14 +26,14 @@
     "cert-info": "1.5.1",
     "colorette": "1.3.0",
     "crypto-js": "4.1.1",
-    "csv-parse": "4.16.2",
+    "csv-parse": "4.16.3",
     "goldengate": "10.4.0",
     "hot-formula-parser": "4.0.0",
     "import-lazy": "4.0.0",
     "ini": "2.0.0",
     "inquirer": "8.1.2",
     "invoices": "2.0.0",
-    "ln-accounting": "5.0.0",
+    "ln-accounting": "5.0.1",
     "ln-service": "52.1.0",
     "ln-sync": "0.4.7",
     "ln-telegram": "3.2.11",
@@ -79,5 +79,5 @@
     "postpublish": "docker buildx build --platform linux/amd64,linux/arm64,linux/arm/v7 -t alexbosworth/balanceofsatoshis --push .",
     "test": "tap --branches=1 --functions=1 --lines=1 --statements=1 -t 60 test/arrays/*.js test/balances/*.js test/chain/*.js test/display/*.js test/encryption/*.js test/fiat/*.js test/lnd/*.js test/network/*.js test/nodes/*.js test/peers/*.js test/responses/*.js test/routing/*.js test/services/*.js test/swaps/*.js test/tags/*.js test/wallets/*.js"
   },
-  "version": "10.10.2"
+  "version": "10.11.0"
 }

routing/get_ignores.js

@@ -2,14 +2,22 @@ const asyncAuto = require('async/auto');
 const asyncMap = require('async/map');
 const {findKey} = require('ln-sync');
 const {getChannel} = require('ln-service');
+const {getNode} = require('ln-service');
+const {Parser} = require('hot-formula-parser');
 const {returnResult} = require('asyncjs-util');
 
+const {describeParseError} = require('./../display');
+
+const asFormula = n => ({formula: n.slice(0, n.length-67), key: n.slice(-66)});
 const decodePair = n => n.split('/');
 const flatten = arr => [].concat(...arr);
+const heightFromId = id => Number(id.split('x').shift());
 const {isArray} = Array;
 const isChannel = n => /^\d*x\d*x\d*$/.test(n);
+const isFormula = n => /(.*)\/0[2-3][0-9A-F]{64}$/gim.test(n);
 const isPair = n => !!n && /^0[2-3][0-9A-F]{64}\/0[2-3][0-9A-F]{64}$/i.test(n);
 const isPublicKey = n => !!n && /^0[2-3][0-9A-F]{64}$/i.test(n);
+const {keys} = Object;
 const pairAsIgnore = (a, b) => ({from_public_key: a, to_public_key: b});
 const uniq = arr => Array.from(new Set(arr));
 
@@ -90,6 +98,11 @@ module.exports = (args, cbk) => {
       // Avoids sorted by type
       sortedAvoids: ['avoids', ({avoids}, cbk) => {
         const withKeys = avoids.map(id => {
+          // Exit early when the id is a formula
+          if (isFormula(id)) {
+            return asFormula(id);
+          }
+
           // Exit early when the id is a public key
           if (isPublicKey(id)) {
             return {node: {from_public_key: id}};
@@ -146,6 +159,68 @@ module.exports = (args, cbk) => {
         cbk);
       }],
 
+      // Get formula avoids
+      getFormulaIgnores: ['sortedAvoids', ({sortedAvoids}, cbk) => {
+        const formulas = sortedAvoids.filter(n => n.formula);
+
+        return asyncMap(formulas, ({formula, key}, cbk) => {
+          return getNode({lnd: args.lnd, public_key: key}, (err, res) => {
+            if (!!err) {
+              return cbk(err);
+            }
+
+            const inboundAvoids = res.channels
+              .map(({id, policies}) => {
+                const height = heightFromId(id);
+                const inPolicy = policies.find(n => n.public_key !== key);
+
+                if (!inPolicy) {
+                  return;
+                }
+
+                const parser = new Parser();
+
+                const variables = {
+                  height,
+                  base_fee: Number(inPolicy.base_fee_mtokens) || Number(),
+                  fee_rate: inPolicy.fee_rate || Number(),
+                };
+
+                keys(variables).forEach(key => {
+                  parser.setVariable(key.toLowerCase(), variables[key]);
+                  parser.setVariable(key.toUpperCase(), variables[key]);
+
+                  return;
+                });
+
+                const parsed = parser.parse(formula);
+
+                if (!!parsed.error) {
+                  return {error: describeParseError({error: parsed.error})};
+                }
+
+                if (parsed.result === false) {
+                  return;
+                }
+
+                return {
+                  from_public_key: inPolicy.public_key,
+                  to_public_key: key,
+                };
+              });
+
+            const {error} = inboundAvoids.find(n => !!n && !!n.error) || {};
+
+            if (!!error) {
+              return cbk([400, 'InvalidAvoidDirective', {error, formula}]);
+            }
+
+            return cbk(null, inboundAvoids.filter(n => !!n));
+          });
+        },
+        cbk);
+      }],
+
       // Resolve referenced queries
       getQueryIgnores: ['sortedAvoids', ({sortedAvoids}, cbk) => {
         const queries = sortedAvoids.map(n => n.query).filter(n => !!n);
@@ -170,14 +245,23 @@ module.exports = (args, cbk) => {
       // Combine ignores together
       combinedIgnores: [
         'getChannelIgnores',
+        'getFormulaIgnores',
         'getQueryIgnores',
         'sortedAvoids',
-        ({getChannelIgnores, getQueryIgnores, sortedAvoids}, cbk) =>
+        ({
+          getChannelIgnores,
+          getFormulaIgnores,
+          getQueryIgnores,
+          sortedAvoids,
+        },
+        cbk) =>
       {
-        const chanIgnores = flatten(getChannelIgnores);
-        const nodeIgnores = sortedAvoids.map(n => n.node).filter(n => !!n);
-
-        const ignore = [chanIgnores, getQueryIgnores, nodeIgnores];
+        const ignore = [
+          flatten(getChannelIgnores),
+          flatten(getFormulaIgnores),
+          getQueryIgnores,
+          sortedAvoids.map(n => n.node).filter(n => !!n),
+        ];
 
         const allIgnores = flatten(ignore).filter(avoid => {
           const isFromInThrough = avoid.from_public_key === args.in_through;

swaps/index.js

@@ -1,6 +1,7 @@
 const getPaidService = require('./get_paid_service');
 const getSwapCost = require('./get_swap_cost');
 const getSwapService = require('./get_swap_service');
+const manageRebalance = require('./manage_rebalance');
 const rebalance = require('./rebalance');
 const swapApiKey = require('./swap_api_key');
 const swapIn = require('./swap_in');
@@ -11,6 +12,7 @@ module.exports = {
   getPaidService,
   getSwapCost,
   getSwapService,
+  manageRebalance,
   rebalance,
   swapApiKey,
   swapIn,

swaps/manage_rebalance.js

@@ -0,0 +1,96 @@
+const asyncAuto = require('async/auto');
+const asyncRetry = require('async/retry');
+const {returnResult} = require('asyncjs-util');
+
+const rebalance = require('./rebalance');
+
+const {isArray} = Array;
+
+/** Manage rebalance attempts
+
+  {
+    [avoid]: [<Avoid Forwarding Through Node With Public Key Hex String>]
+    fs: {
+      getFile: <Read File Contents Function> (path, cbk) => {}
+    }
+    [in_outbound]: <Inbound Target Outbound Liquidity Tokens Number>
+    [in_through]: <Pay In Through Peer String>
+    lnd: <Authenticated LND API Object>
+    logger: <Winston Logger Object>
+    [max_fee]: <Maximum Fee Tokens Number>
+    [max_fee_rate]: <Max Fee Rate Tokens Per Million Number>
+    [max_rebalance]: <Maximum Amount to Rebalance Tokens String>
+    [node]: <Node Name String>
+    [out_channels]: [<Exclusively Rebalance Through Channel Ids String>]
+    [out_inbound]: <Outbound Target Inbound Liquidity Tokens Number>
+    [out_through]: <Pay Out Through Peer String>
+    [timeout_minutes]: <Deadline To Stop Rebalance Minutes Number>
+  }
+
+  @returns via cbk or Promise
+*/
+module.exports = (args, cbk) => {
+  return new Promise((resolve, reject) => {
+    return asyncAuto({
+      // Check arguments
+      validate: cbk => {
+        if (!args.fs) {
+          return cbk([400, 'ExpectedFsToManageRebalance']);
+        }
+
+        if (!args.logger) {
+          return cbk([400, 'ExpectedLoggerToManageRebalance'])
+        }
+
+        if (!args.lnd) {
+          return cbk([400, 'ExpectedLndToManageRebalance']);
+        }
+
+        return cbk();
+      },
+
+      // Run the rebalance
+      rebalance: ['validate', ({}, cbk) => {
+        return asyncRetry({
+          errorFilter: err => {
+            // Do not retry on invalid errors
+            if (!isArray(err)) {
+              return false;
+            }
+
+            const [code] = err;
+
+            // Do not retry on client errors
+            if (code === 400) {
+              return false;
+            }
+
+            args.logger.error({err});
+
+            return true;
+          },
+        },
+        cbk => {
+          return rebalance({
+            avoid: args.avoid,
+            fs: args.fs,
+            in_outbound: args.in_outbound,
+            in_through: args.in_through,
+            lnd: args.lnd,
+            logger: args.logger,
+            max_fee: args.max_fee,
+            max_fee_rate: args.max_fee_rate,
+            max_rebalance: args.max_rebalance,
+            out_channels: args.out_channels,
+            out_inbound: args.out_inbound,
+            out_through: args.out_through,
+            timeout_minutes: args.timeout_minutes,
+          },
+          cbk);
+        },
+        cbk);
+      }],
+    },
+    returnResult({reject, resolve, of: 'rebalance'}, cbk));
+  });
+};

swaps/rebalance.js

@@ -38,6 +38,7 @@ const defaultMaxFeeTotal = Math.floor(5e6 * 0.0025);
 const flatten = arr => [].concat(...arr);
 const highInbound = 4500000;
 const initialProbeTokens = size => Math.round((Math.random() * size) + size);
+const interval = 1000 * 10;
 const {isArray} = Array;
 const isPublicKey = n => /^[0-9A-F]{66}$/i.test(n);
 const legacyMaxRebalanceTokens = 4294967;
@@ -60,6 +61,7 @@ const rateDivisor = 1e6;
 const sample = a => !!a.length ? a[Math.floor(Math.random()*a.length)] : null;
 const sumOf = arr => arr.reduce((sum, n) => sum + n);
 const tagFilePath = () => join(...[homedir(), '.bos', 'tags.json']);
+const times = 6;
 const tokAsBigTok = tokens => !tokens ? undefined : (tokens / 1e8).toFixed(8);
 const topOf = arr => arr.slice(0, Math.ceil(arr.length / 2));
 const uniq = arr => Array.from(new Set(arr));
@@ -588,7 +590,6 @@ module.exports = (args, cbk) => {
           logger: args.logger,
           lnd: args.lnd,
           max_fee: defaultMaxFeeTotal,
-          node: args.node,
           out_through: getOutbound.public_key,
           timeout_minutes: args.timeout_minutes,
         },
@@ -728,20 +729,17 @@ module.exports = (args, cbk) => {
 
       // Execute the rebalance
       pay: ['invoice', 'lnd', 'routes', ({invoice, lnd, routes}, cbk) => {
-        return asyncRetry({}, cbk => {
-          return payViaRoutes({lnd, routes, id: invoice.id}, (err, res) => {
-            if (!!err) {
-              return cbk([503, 'UnexpectedErrExecutingRebalance', {err}]);
-            }
+        return payViaRoutes({lnd, routes, id: invoice.id}, (err, res) => {
+          if (!!err) {
+            return cbk([503, 'UnexpectedErrExecutingRebalance', {err}]);
+          }
 
-            return cbk(null, {
-              fee: res.fee,
-              id: invoice.id,
-              tokens: res.tokens,
-            });
+          return cbk(null, {
+            fee: res.fee,
+            id: invoice.id,
+            tokens: res.tokens,
           });
-        },
-        cbk);
+        });
       }],
 
       // Get adjusted inbound liquidity after rebalance