add avoid append and lnd 0.20.2

CHANGELOG.md

@@ -1,5 +1,10 @@
 # Versions
 
+## 23.1.0
+
+- Add `--avoid-append` to add failing edges to an avoid list based on formula
+- Add support for LND 0.20.2
+
 ## 23.0.0
 
 ### Breaking Changes

README.md

@@ -8,7 +8,7 @@ Commands for working with LND balances.
 Supported LND versions:
 
 - v0.21.0-beta to v0.21.1-beta
-- v0.20.0-beta to v0.20.1-beta
+- v0.20.0-beta to v0.20.2-beta
 
 ## Install
 

bos

@@ -1,4 +1,5 @@
 #!/usr/bin/env node
+const {appendFile} = require('fs');
 const {lstat} = require('fs');
 const {mkdir} = require('fs');
 const {readdir} = require('fs');
@@ -1719,6 +1720,7 @@ prog
   .help('--avoid can take a FORMULA/public_key to avoid inbound peers')
   .help('--avoid can take a public_key/FORMULA to avoid outbound peers')
   .help('--avoid FORMULA variables: FEE_RATE, BASE_FEE, HEIGHT, AGE')
+  .help('--avoid-append vars: FAILURE_REASON, FAILURE_INDEX, ROUTE_HOPS_COUNT')
   .help('--avoid-list allows passing a file of newline separated avoids')
   .help('--avoid-list will delete edges from the file unknown to the node')
   .help('--in decreases the inbound liquidity with a specific peer/tag')
@@ -1728,6 +1730,7 @@ prog
   .help('--out increases the inbound liquidity with a specific peer/tag')
   .option('--amount <amount>', 'Maximum amount to rebalance')
   .option('--avoid <pubkey_or_chanid>', 'Avoid forwarding through', REPEATABLE)
+  .option('--avoid-append', 'Formula for appending failing edges to list')
   .option('--avoid-high-fee-routes', 'Avoid trying routes above max-fee-rate')
   .option('--avoid-list', 'Path to file with newline separated list of avoids')
   .option('--in <pubkey_or_alias>', 'Route in through a specific peer')
@@ -1747,8 +1750,9 @@ prog
         return swaps.manageRebalance({
           logger,
           avoid: flatten([options.avoid].filter(n => !!n)),
+          avoid_append: options.avoidAppend || undefined,
           avoid_list: options.avoidList || undefined,
-          fs: {writeFile, getFile: readFile},
+          fs: {appendFile, writeFile, getFile: readFile},
           in_filters: flatten([options.inFilter].filter(n => !!n)),
           in_outbound: options.inTargetOutbound || undefined,
           in_through: options.in || undefined,

network/execute_probe.js

@@ -29,6 +29,7 @@ const tokensAsMillitokens = tok => (BigInt(tok) * BigInt(1e3)).toString();
     [in_through]: <In Through Public Key Hex String>
     [is_strict_max_fee]: <Avoid Probing Too-High Fee Routes Bool>
     lnd: <Authenticated LND API Object>
+    [log_failure]: <Failure Details Log Callback Function>
     logger: <Winston Logger Object>
     [max_fee]: <Maximum Fee Tokens Number>
     [max_fee_mtokens]: <Maximum Fee Millitokens Number String>
@@ -180,6 +181,10 @@ module.exports = (args, cbk) => {
 
         // Log failures encountered while trying to find a route
         sub.on('routing_failure', async failure => {
+          if (!!args.log_failure) {
+            args.log_failure(null, failure);
+          }
+
           const {description} = await describeRoutingFailure({
             index: failure.index,
             lnd: args.lnd,

network/probe_destination.js

@@ -62,7 +62,8 @@ const tokAsMtok = tokens => (BigInt(tokens || 0) * BigInt(1e3)).toString();
     [is_push]: <Is Push Payment Bool>
     [is_real_payment]: <Pay the Request after Probing Bool> // default: false
     [is_strict_max_fee]: <Avoid Probing Too-High Fee Routes Bool>
-    lnd: <Authenticated LND gRPC API Object>
+    lnd: <Authenticated LND API Object>
+    [log_failure]: <Failure Details Log Callback Function>
     logger: <Winston Logger Object>
     [max_fee]: <Maximum Fee Tokens Number>
     [max_fee_mtokens]: <Maximum Fee Millitokens Number>
@@ -411,6 +412,7 @@ module.exports = (args, cbk) => {
           in_through: args.in_through,
           is_strict_max_fee: args.is_strict_max_fee,
           lnd: args.lnd,
+          log_failure: args.log_failure,
           logger: args.logger,
           max_fee: args.max_fee,
           max_fee_mtokens: args.max_fee_mtokens,

package-lock.json

@@ -1,12 +1,12 @@
 {
   "name": "balanceofsatoshis",
-  "version": "23.0.0",
+  "version": "23.1.0",
   "lockfileVersion": 2,
   "requires": true,
   "packages": {
     "": {
       "name": "balanceofsatoshis",
-      "version": "23.0.0",
+      "version": "23.1.0",
       "license": "MIT",
       "dependencies": {
         "@alexbosworth/blockchain": "3.2.0",
@@ -31,12 +31,11 @@
         "csv-parse": "7.0.1",
         "goldengate": "16.0.4",
         "grammy": "1.45.1",
-        "hot-formula-parser": "4.0.0",
         "import-lazy": "4.0.0",
         "ini": "7.0.0",
         "inquirer": "14.0.2",
         "ln-accounting": "10.0.4",
-        "ln-service": "59.2.2",
+        "ln-service": "59.2.3",
         "ln-sync": "8.0.3",
         "ln-telegram": "8.0.4",
         "minimist": "1.2.8",
@@ -2627,13 +2626,13 @@
       }
     },
     "node_modules/ln-service": {
-      "version": "59.2.2",
-      "resolved": "https://registry.npmjs.org/ln-service/-/ln-service-59.2.2.tgz",
-      "integrity": "sha512-fb0zvOd4BamK/2TP4fRele1nHMr336KItcN+fJU353bdr8T6HQTMHvSqmzHIT3R8DAZkQj1IZRHoMc73CbzkhQ==",
+      "version": "59.2.3",
+      "resolved": "https://registry.npmjs.org/ln-service/-/ln-service-59.2.3.tgz",
+      "integrity": "sha512-h2Db7FvZfl9ts0O/64pcbEfeSwkoruaktut06733bOSiYr0sctDQs3FMKUhgYh8OHagprvqWESIFhyInCbmK6A==",
       "dependencies": {
         "bolt07": "1.9.5",
         "invoices": "5.0.2",
-        "lightning": "12.2.2",
+        "lightning": "12.2.3",
         "macaroon": "3.0.4"
       },
       "engines": {
@@ -2654,9 +2653,9 @@
       "integrity": "sha512-Vq886eXykuP5E6HcKSSStP3bJgrE6In5WKxVUvJ8XGpWWYs2xZHWqUwzCtGgEtBcxyd57KBFDPFoUfNzdaHCNg=="
     },
     "node_modules/ln-service/node_modules/lightning": {
-      "version": "12.2.2",
-      "resolved": "https://registry.npmjs.org/lightning/-/lightning-12.2.2.tgz",
-      "integrity": "sha512-GXg/cSbD97bLnzmtTHpIIyjT041kJWC7lpn3PmoO8bYChks0HghZxS0d35dRGoWRFFONAwESzTPSt7Fr+ArUBQ==",
+      "version": "12.2.3",
+      "resolved": "https://registry.npmjs.org/lightning/-/lightning-12.2.3.tgz",
+      "integrity": "sha512-+kUcEGZ84BdHtYrTJi3fqDyAWp8WYJcdmyQ089YhN7GuOdj9zof/hiujZwGFCqN0fDUiaP2UBOjElIRygtAzWg==",
       "dependencies": {
         "@alexbosworth/blockchain": "3.2.0",
         "@grpc/grpc-js": "1.14.4",
@@ -6037,13 +6036,13 @@
       }
     },
     "ln-service": {
-      "version": "59.2.2",
-      "resolved": "https://registry.npmjs.org/ln-service/-/ln-service-59.2.2.tgz",
-      "integrity": "sha512-fb0zvOd4BamK/2TP4fRele1nHMr336KItcN+fJU353bdr8T6HQTMHvSqmzHIT3R8DAZkQj1IZRHoMc73CbzkhQ==",
+      "version": "59.2.3",
+      "resolved": "https://registry.npmjs.org/ln-service/-/ln-service-59.2.3.tgz",
+      "integrity": "sha512-h2Db7FvZfl9ts0O/64pcbEfeSwkoruaktut06733bOSiYr0sctDQs3FMKUhgYh8OHagprvqWESIFhyInCbmK6A==",
       "requires": {
         "bolt07": "1.9.5",
         "invoices": "5.0.2",
-        "lightning": "12.2.2",
+        "lightning": "12.2.3",
         "macaroon": "3.0.4"
       },
       "dependencies": {
@@ -6061,9 +6060,9 @@
           "integrity": "sha512-Vq886eXykuP5E6HcKSSStP3bJgrE6In5WKxVUvJ8XGpWWYs2xZHWqUwzCtGgEtBcxyd57KBFDPFoUfNzdaHCNg=="
         },
         "lightning": {
-          "version": "12.2.2",
-          "resolved": "https://registry.npmjs.org/lightning/-/lightning-12.2.2.tgz",
-          "integrity": "sha512-GXg/cSbD97bLnzmtTHpIIyjT041kJWC7lpn3PmoO8bYChks0HghZxS0d35dRGoWRFFONAwESzTPSt7Fr+ArUBQ==",
+          "version": "12.2.3",
+          "resolved": "https://registry.npmjs.org/lightning/-/lightning-12.2.3.tgz",
+          "integrity": "sha512-+kUcEGZ84BdHtYrTJi3fqDyAWp8WYJcdmyQ089YhN7GuOdj9zof/hiujZwGFCqN0fDUiaP2UBOjElIRygtAzWg==",
           "requires": {
             "@alexbosworth/blockchain": "3.2.0",
             "@grpc/grpc-js": "1.14.4",

package.json

@@ -36,7 +36,7 @@
     "ini": "7.0.0",
     "inquirer": "14.0.2",
     "ln-accounting": "10.0.4",
-    "ln-service": "59.2.2",
+    "ln-service": "59.2.3",
     "ln-sync": "8.0.3",
     "ln-telegram": "8.0.4",
     "minimist": "1.2.8",
@@ -80,5 +80,5 @@
     "postpublish": "docker buildx build --platform linux/amd64,linux/arm64 -t alexbosworth/balanceofsatoshis -t alexbosworth/balanceofsatoshis:$npm_package_version --push .",
     "test": "node --experimental-test-coverage --test test/arrays/*.js test/balances/*.js test/chain/*.js test/display/*.js test/encryption/*.js test/lnd/*.js test/network/*.js test/nodes/*.js test/peers/*.js test/responses/*.js test/routing/*.js test/services/*.js test/swaps/*.js test/tags/*.js test/telegram/*.js test/wallets/*.js"
   },
-  "version": "23.0.0"
+  "version": "23.1.0"
 }

swaps/append_failing_edge.js

@@ -0,0 +1,108 @@
+const asyncAuto = require('async/auto');
+const {evaluateFormula} = require('@alexbosworth/formulas');
+const {returnResult} = require('asyncjs-util');
+
+const pairEdgeIndex = (pair, key) => `x${Number(!pair.indexOf(key))}`;
+
+/** Append failing edge based on formula match
+
+  {
+    avoid: <Formula For Edge To Avoid String>
+    failure: {
+      index: <Failure Hop Index Number>
+      reason: <Failure Reason String>
+      route: [{
+        hops: [{
+          channel: <Standard Format Channel Id String>
+          public_key: <Node Public Key Hex String>
+        }]
+      }]
+    }
+    fs: {
+      appendFile: <Append to File Function> (path, content, cbk) => {}
+    }
+    list: <File Path To Avoid List String>
+  }
+
+  @returns via cbk
+  {
+    edge: <Appended Failing Edge Id String>
+  }
+*/
+module.exports = ({avoid, failure, fs, list}, cbk) => {
+  return new Promise((resolve, reject) => {
+    return asyncAuto({
+      // Check arguments
+      validate: cbk => {
+        if (!avoid) {
+          return cbk([400, 'ExpectedAvoidanceFormulaToAppendFailingEdge']);
+        }
+
+        if (!failure) {
+          return cbk([400, 'ExpectedFailureDetailsToAppendFailingEdge']);
+        }
+
+        if (!fs) {
+          return cbk([400, 'ExpectedFilesystemMethodsToAppendFailingEdge']);
+        }
+
+        if (!list) {
+          return cbk([400, 'ExpectedPathToAvoidListToAppendFailingEdge']);
+        }
+
+        return cbk();
+      },
+
+      // Derive the set of edges
+      edges: ['validate', ({}, cbk) => {
+        return cbk(null, failure.route.hops.map((hop, i, hops) => {
+          const pair = [(hops[i - 1] || {}).public_key, hop.public_key].sort();
+
+          return `${hop.channel}${pairEdgeIndex(pair, hop.public_key)}`;
+        }));
+      }],
+
+      // Determine if there is a formula matching edge
+      failing: ['edges', ({edges}, cbk) => {
+        const edge = edges[failure.index];
+
+        try {
+          const {result} = evaluateFormula({
+            constants: {
+              FAILURE_INDEX: failure.index,
+              FAILURE_REASON: failure.reason,
+              ROUTE_HOPS_COUNT: failure.route.hops.length,
+            },
+            formula: avoid,
+          });
+
+          // Exit early when there is no append edge formula match
+          if (!result) {
+            return cbk(null, {});
+          }
+
+          return cbk(null, {edge});
+        } catch (err) {
+          return cbk([400, 'ExpectedValidAppendEdgeFormula', err.message]);
+        }
+      }],
+
+      // Append the failing edge to the avoid list
+      append: ['failing', ({failing}, cbk) => {
+        // Exit early when there is no relevant failing edge to append
+        if (!failing.edge) {
+          return cbk();
+        }
+
+        return fs.appendFile(list, `\n${failing.edge}`, err => {
+          if (!!err) {
+            return cbk([500, 'UnexpectedErrorAppendingFailEdge', err.message]);
+          }
+
+          return cbk()
+        });
+      }],
+    },
+    returnResult({reject, resolve, of: 'failing'}, cbk));
+  });
+};

swaps/manage_rebalance.js

@@ -4,6 +4,7 @@ const asyncRetry = require('async/retry');
 const {getChannel} = require('ln-service');
 const {returnResult} = require('asyncjs-util');
 
+const appendFailingEdge = require('./append_failing_edge');
 const rebalance = require('./rebalance');
 
 const channelFromEdge = edge => edge.slice(0, -2);
@@ -18,7 +19,10 @@ const uniq = arr => Array.from(new Set(arr));
 
   {
     [avoid]: [<Avoid Forwarding Through Node With Public Key Hex String>]
+    [avoid_append]: <Append Avoid Edges To Avoid List Matching Formula String>
+    [avoid_list]: <Use Avoid Directives From File At Path String>
     fs: {
+      appendFile: <Append to File Function> (path, content, cbk) => {}
       getFile: <Read File Contents Function> (path, cbk) => {}
       writeFile: <Write File Contents Function> (path, contents, cbk) => {}
     }
@@ -45,6 +49,10 @@ module.exports = (args, cbk) => {
     return asyncAuto({
       // Check arguments
       validate: cbk => {
+        if (!!args.avoid_append && !args.avoid_list) {
+          return cbk([400, 'ExpectedAvoidListToAppendAvoidsTo']);
+        }
+
         if (!args.fs) {
           return cbk([400, 'ExpectedFsToManageRebalance']);
         }
@@ -95,6 +103,35 @@ module.exports = (args, cbk) => {
         });
       }],
 
+      // Create failing edge logger for avoid appending
+      logFail: ['validate', ({}, cbk) => {
+        // Exit early when there is no avoid appending
+        if (!args.avoid_append) {
+          return cbk();
+        }
+
+        return cbk(null, (err, failure) => {
+          return appendFailingEdge({
+            failure,
+            avoid: args.avoid_append,
+            fs: args.fs,
+            list: args.avoid_list
+          },
+          (err, res) => {
+            if (!!err) {
+              return args.logger.error({append_failure_error: err});
+            }
+
+            // Exit early when there was no append
+            if (!res.edge) {
+              return;
+            }
+
+            return args.logger.info({appended_failing_edge: res.edge});
+          });
+        });
+      }],
+
       // Look at all of the lines in the file and clean them up
       getCleanAvoids: ['getAvoids', ({getAvoids}, cbk) => {
         return asyncFilter(getAvoids.file, (line, cbk) => {
@@ -119,7 +156,7 @@ module.exports = (args, cbk) => {
       }],
 
       // Run the rebalance
-      rebalance: ['getAvoids', ({getAvoids}, cbk) => {
+      rebalance: ['getAvoids', 'logFail', ({getAvoids, logFail}, cbk) => {
         const start = new Date().toISOString();
 
         return asyncRetry({
@@ -156,6 +193,7 @@ module.exports = (args, cbk) => {
             in_through: args.in_through,
             is_strict_max_fee_rate: args.is_strict_max_fee_rate,
             lnd: args.lnd,
+            log_failure: logFail || undefined,
             logger: args.logger,
             max_fee: Number(args.max_fee) || undefined,
             max_fee_rate: Number(args.max_fee_rate) || undefined,

swaps/rebalance.js

@@ -77,6 +77,7 @@ const uniq = arr => Array.from(new Set(arr));
     [in_through]: <Pay In Through Peer String>
     [is_strict_max_fee_rate]: <Avoid Probing Too-High Fee Rate Routes Bool>
     lnd: <Authenticated LND API Object>
+    [log_failure]: <Failure Details Log Callback Function>
     logger: <Winston Logger Object>
     [max_fee]: <Maximum Fee Tokens Number>
     [max_fee_rate]: <Max Fee Rate Tokens Per Million Number>
@@ -643,6 +644,7 @@ module.exports = (args, cbk) => {
           is_strict_max_fee: args.is_strict_max_fee_rate || undefined,
           logger: args.logger,
           lnd: args.lnd,
+          log_failure: args.log_failure,
           max_fee_mtokens: maxFeeMtokens,
           out_through: getOutbound.public_key,
           timeout_minutes: args.timeout_minutes,

test/swaps/test_append_failing_edge.js

@@ -0,0 +1,127 @@
+const {deepEqual} = require('node:assert').strict;
+const {rejects} = require('node:assert').strict;
+const test = require('node:test');
+
+const appendFailingEdge = require('./../../swaps/append_failing_edge.js');
+
+const makeFailure = override => {
+  const failure = {
+    index: 1,
+    reason: 'TemporaryChannelFailure',
+    route: {
+      hops: [
+        {
+          channel: '1x1x1',
+          public_key: '02',
+        },
+        {
+          channel: '2x2x2',
+          public_key: '03',
+        },
+      ],
+    },
+  };
+
+  Object.keys(override).forEach(key => failure[key] = override[key]);
+
+  return failure;
+};
+
+const makeFs = ({err} = {}) => {
+  const fs = {
+    calls: [],
+    appendFile: (path, content, cbk) => {
+      fs.calls.push({content, path});
+
+      return cbk(err);
+    },
+  };
+
+  return fs;
+};
+
+const makeArgs = override => {
+  const args = {
+    avoid: [
+      'AND(',
+      'FAILURE_INDEX = 1,',
+      'FAILURE_REASON = "TemporaryChannelFailure",',
+      'ROUTE_HOPS_COUNT = 2',
+      ')',
+    ].join(''),
+    failure: makeFailure({}),
+    fs: makeFs({}),
+    list: '/tmp/avoid',
+  };
+
+  Object.keys(override).forEach(key => args[key] = override[key]);
+
+  return args;
+};
+
+const tests = [
+  {
+    args: makeArgs({avoid: undefined}),
+    description: 'An avoidance formula is required',
+    error: [400, 'ExpectedAvoidanceFormulaToAppendFailingEdge'],
+  },
+  {
+    args: makeArgs({failure: undefined}),
+    description: 'Failure details are required',
+    error: [400, 'ExpectedFailureDetailsToAppendFailingEdge'],
+  },
+  {
+    args: makeArgs({fs: undefined}),
+    description: 'Filesystem methods are required',
+    error: [400, 'ExpectedFilesystemMethodsToAppendFailingEdge'],
+  },
+  {
+    args: makeArgs({list: undefined}),
+    description: 'An avoid list path is required',
+    error: [400, 'ExpectedPathToAvoidListToAppendFailingEdge'],
+  },
+  {
+    args: makeArgs({avoid: '('}),
+    description: 'A valid avoidance formula is required',
+    error: [400, 'ExpectedValidAppendEdgeFormula'],
+  },
+  {
+    appended: [],
+    args: makeArgs({avoid: 'FALSE'}),
+    description: 'An edge is not appended when the formula does not match',
+    expected: {},
+  },
+  {
+    appended: [
+      {
+        content: '\n2x2x2x0',
+        path: '/tmp/avoid',
+      },
+    ],
+    args: makeArgs({}),
+    description: 'A failing edge is appended',
+    expected: {edge: '2x2x2x0'},
+  },
+  {
+    args: makeArgs({
+      fs: makeFs({err: new Error('Failed to append')}),
+    }),
+    description: 'An append error is returned',
+    error: [500, 'UnexpectedErrorAppendingFailEdge'],
+  },
+];
+
+tests.forEach(({appended, args, description, error, expected}) => {
+  return test(description, async () => {
+    if (!!error) {
+      await rejects(appendFailingEdge(args), error, 'Got expected error');
+    } else {
+      const result = await appendFailingEdge(args);
+
+      deepEqual(result, expected, 'Got expected result');
+      deepEqual(args.fs.calls, appended, 'Got expected appended data');
+    }
+
+    return;
+  });
+});