Add pagination for sent payments (#1220)

* Add pagination to get sent payments rpc method

* Got frontend mostly working for paginating sent payments

* Fix warning for missing arg in load sent payments effect

* Update frontend build

frontend/src/pages/sentpayments/SentPayments.js

@@ -1,25 +1,38 @@
-import React, { useState, useEffect } from 'react';
+import React, { useState, useEffect, useCallback } from 'react';
 import {
   Grid,
   Tabs,
   Tab,
   AppBar,
   Box,
+  Button,
+  CircularProgress,
 } from '@material-ui/core';
 
+import ReplayIcon from '@material-ui/icons/Replay';
+
 // components
 import Widget from '../../components/Widget';
 import SentPayment from '../../components/SentPayment';
 
+// styles
+import useStyles from './styles';
+
 // data
 
 import {
   getSentPaymentsRequest,
 } from '../../squeakclient/requests';
 
+const SENT_PAYMENTS_PER_PAGE = 10;
+
+
 export default function SentPayments() {
+  const classes = useStyles();
   const [value, setValue] = useState(0);
   const [sentPayments, setSentPayments] = useState([]);
+  const [waitingForSentPayments, setWaitingForSentPayments] = React.useState(false);
+
 
   function a11yProps(index) {
     return {
@@ -32,15 +45,31 @@ export default function SentPayments() {
     setValue(newValue);
   };
 
-  const loadSentPayments = () => {
-    getSentPaymentsRequest((sentPaymentsReply) => {
-      setSentPayments(sentPaymentsReply.getSentPaymentsList());
+  // const loadSentPayments = () => {
+  //   getSentPaymentsRequest((sentPaymentsReply) => {
+  //     setSentPayments(sentPaymentsReply.getSentPaymentsList());
+  //   });
+  // };
+  const loadSentPayments = useCallback((limit, lastSentPayment) => {
+    setWaitingForSentPayments(true);
+    getSentPaymentsRequest(limit, lastSentPayment, handleLoadedSentPayments);
+  },
+  []);
+
+  const handleLoadedSentPayments = (reply) => {
+    const loadedSentPayments = reply.getSentPaymentsList();
+    setWaitingForSentPayments(false);
+    setSentPayments((prevSentPayments) => {
+      if (!prevSentPayments) {
+        return loadedSentPayments;
+      }
+      return prevSentPayments.concat(loadedSentPayments);
     });
   };
 
   useEffect(() => {
-    loadSentPayments();
-  }, []);
+    loadSentPayments(SENT_PAYMENTS_PER_PAGE, null);
+  }, [loadSentPayments]);
 
   function TabPanel(props) {
     const {
@@ -109,10 +138,38 @@ export default function SentPayments() {
           {PaymentsTabs()}
         </Grid>
         <Grid item xs={12} sm={3} />
+        {ViewMoreSentPaymentsButton()}
       </Grid>
     );
   }
 
+  function ViewMoreSentPaymentsButton() {
+    return (
+      <>
+        <Grid item xs={12}>
+          <div className={classes.wrapper}>
+            {!waitingForSentPayments
+            && (
+            <Button
+              variant="contained"
+              color="primary"
+              disabled={waitingForSentPayments}
+              onClick={() => {
+                const latestSentPayment = sentPayments.slice(-1).pop();
+                loadSentPayments(SENT_PAYMENTS_PER_PAGE, latestSentPayment);
+              }}
+            >
+              <ReplayIcon />
+              View more squeaks
+            </Button>
+            )}
+            {waitingForSentPayments && <CircularProgress size={48} className={classes.buttonProgress} />}
+          </div>
+        </Grid>
+      </>
+    );
+  }
+
   return (
     <>
       {GridContent()}

frontend/src/pages/sentpayments/styles.js

@@ -0,0 +1,56 @@
+import { makeStyles } from '@material-ui/styles';
+import { green } from '@material-ui/core/colors';
+
+export default makeStyles((theme) => ({
+  dashedBorder: {
+    border: '1px dashed',
+    borderColor: theme.palette.primary.main,
+    padding: theme.spacing(2),
+    paddingTop: theme.spacing(4),
+    paddingBottom: theme.spacing(4),
+    marginTop: theme.spacing(1),
+  },
+  text: {
+    marginBottom: theme.spacing(2),
+  },
+  fab: {
+    position: 'fixed',
+    bottom: theme.spacing(4),
+    right: theme.spacing(4),
+  },
+  refreshFab: {
+    position: 'fixed',
+    top: theme.spacing(14),
+    margin: 'auto',
+    justifyContent: 'center',
+  },
+  root: {
+    display: 'flex',
+    alignItems: 'center',
+  },
+  wrapper: {
+    margin: theme.spacing(1),
+    position: 'relative',
+  },
+  buttonSuccess: {
+    backgroundColor: green[500],
+    '&:hover': {
+      backgroundColor: green[700],
+    },
+  },
+  fabProgress: {
+    color: green[500],
+    position: 'absolute',
+    top: -6,
+    left: -6,
+    zIndex: 1,
+  },
+  buttonProgress: {
+    color: green[500],
+    position: 'absolute',
+    top: '50%',
+    left: '50%',
+    marginTop: -12,
+    marginLeft: -12,
+  },
+}));

frontend/src/squeakclient/requests.js

@@ -522,8 +522,10 @@ export function getSqueakDetailsRequest(hash, handleResponse) {
   });
 }
 
-export function getSentPaymentsRequest(handleResponse) {
+export function getSentPaymentsRequest(limit, lastSentPayment, handleResponse) {
   const request = new GetSentPaymentsRequest();
+  request.setLimit(limit);
+  request.setLastSentPayment(lastSentPayment);
   client.getSentPayments(request, {}, (err, response) => {
     handleResponse(response);
   });

itests/tests/test_squeak_node.py

@@ -639,7 +639,9 @@ def test_buy_squeak(
 
         # Get all sent payments
         get_sent_payments_response = other_admin_stub.GetSentPayments(
-            squeak_admin_pb2.GetSentPaymentsRequest(),
+            squeak_admin_pb2.GetSentPaymentsRequest(
+                limit=10,
+            ),
         )
         squeak_hashes = [
             sent_payment.squeak_hash

proto/squeak_admin.proto

@@ -809,6 +809,11 @@ message PayOfferReply {
 }
 
 message GetSentPaymentsRequest {
+    /// Limit number of results
+    int32 limit = 1;
+
+    /// Last entry
+    SentPayment last_sent_payment = 2;
 }
 
 message GetSentPaymentsReply {

squeaknode/admin/messages.py

@@ -20,6 +20,8 @@
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 # SOFTWARE.
 import logging
+from datetime import datetime
+from datetime import timezone
 
 from squeak.core import CSqueak
 
@@ -220,3 +222,21 @@ def message_to_squeak_entry(squeak_entry: squeak_admin_pb2.SqueakDisplayEntry) -
         liked_time=squeak_entry.liked_time_s,
         content=squeak_entry.content_str,
     )
+
+
+def message_to_sent_payment(sent_payment: squeak_admin_pb2.SentPayment) -> SentPayment:
+    return SentPayment(
+        sent_payment_id=sent_payment.sent_payment_id,
+        created=timestamp_to_datetime(sent_payment.time_s),
+        peer_address=message_to_peer_address(sent_payment.peer_address),
+        squeak_hash=bytes.fromhex(sent_payment.squeak_hash),
+        payment_hash=bytes.fromhex(sent_payment.payment_hash),
+        secret_key=b'',  # TODO: why does this field exist?
+        price_msat=sent_payment.price_msat,
+        node_pubkey=sent_payment.node_pubkey,
+        valid=sent_payment.valid,
+    )
+
+
+def timestamp_to_datetime(timestamp: int) -> datetime:
+    return datetime.fromtimestamp(timestamp, timezone.utc)

squeaknode/admin/squeak_admin_server_handler.py

@@ -24,6 +24,7 @@ import logging
 from proto import squeak_admin_pb2
 from squeaknode.admin.messages import connected_peer_to_message
 from squeaknode.admin.messages import message_to_peer_address
+from squeaknode.admin.messages import message_to_sent_payment
 from squeaknode.admin.messages import message_to_squeak_entry
 from squeaknode.admin.messages import offer_entry_to_message
 from squeaknode.admin.messages import payment_summary_to_message
@@ -626,8 +627,25 @@ class SqueakAdminServerHandler(object):
         )
 
     def handle_get_sent_payments(self, request):
-        logger.info("Handle get sent payments")
-        sent_payments = self.squeak_controller.get_sent_payments()
+        limit = request.limit
+        last_sent_payment = message_to_sent_payment(request.last_sent_payment) if request.HasField(
+            "last_sent_payment") else None
+        logger.info("""Handle get sent payments with
+        limit: {}
+        last_sent_payment: {}
+        """.format(
+            limit,
+            last_sent_payment,
+        ))
+        sent_payments = self.squeak_controller.get_sent_payments(
+            limit,
+            last_sent_payment,
+        )
+        logger.info(
+            "Got number of sent payments: {}".format(
+                len(sent_payments)
+            )
+        )
         sent_payment_msgs = [
             sent_payment_to_message(sent_payment)
             for sent_payment in sent_payments

squeaknode/admin/webapp/static/build/asset-manifest.json

@@ -1,17 +1,17 @@
 {
   "files": {
-    "main.js": "/static/js/main.1364ecd2.chunk.js",
-    "main.js.map": "/static/js/main.1364ecd2.chunk.js.map",
+    "main.js": "/static/js/main.c017f06a.chunk.js",
+    "main.js.map": "/static/js/main.c017f06a.chunk.js.map",
     "runtime-main.js": "/static/js/runtime-main.9f0ba400.js",
     "runtime-main.js.map": "/static/js/runtime-main.9f0ba400.js.map",
     "static/css/2.ea4ba2f0.chunk.css": "/static/css/2.ea4ba2f0.chunk.css",
-    "static/js/2.ed4620a0.chunk.js": "/static/js/2.ed4620a0.chunk.js",
-    "static/js/2.ed4620a0.chunk.js.map": "/static/js/2.ed4620a0.chunk.js.map",
+    "static/js/2.73e108c6.chunk.js": "/static/js/2.73e108c6.chunk.js",
+    "static/js/2.73e108c6.chunk.js.map": "/static/js/2.73e108c6.chunk.js.map",
     "index.html": "/index.html",
-    "precache-manifest.52936837c4e646d976a01212b0f006e2.js": "/precache-manifest.52936837c4e646d976a01212b0f006e2.js",
+    "precache-manifest.98527034402653794ed933617b9ff8d5.js": "/precache-manifest.98527034402653794ed933617b9ff8d5.js",
     "service-worker.js": "/service-worker.js",
     "static/css/2.ea4ba2f0.chunk.css.map": "/static/css/2.ea4ba2f0.chunk.css.map",
-    "static/js/2.ed4620a0.chunk.js.LICENSE.txt": "/static/js/2.ed4620a0.chunk.js.LICENSE.txt",
+    "static/js/2.73e108c6.chunk.js.LICENSE.txt": "/static/js/2.73e108c6.chunk.js.LICENSE.txt",
     "static/media/font-awesome.min.css": "/static/media/fontawesome-webfont.fee66e71.woff",
     "static/media/google.svg": "/static/media/google.695a3160.svg",
     "static/media/logo.svg": "/static/media/logo.a0185b04.svg"
@@ -19,7 +19,7 @@
   "entrypoints": [
     "static/js/runtime-main.9f0ba400.js",
     "static/css/2.ea4ba2f0.chunk.css",
-    "static/js/2.ed4620a0.chunk.js",
-    "static/js/main.1364ecd2.chunk.js"
+    "static/js/2.73e108c6.chunk.js",
+    "static/js/main.c017f06a.chunk.js"
   ]
 }

squeaknode/admin/webapp/static/build/index.html

@@ -1 +1 @@
-<!doctype html><html lang="en"><head><meta charset="utf-8"/><link rel="shortcut icon" href="/favicon.ico"/><meta name="viewport" content="width=device-width,initial-scale=1,shrink-to-fit=no"/><meta name="theme-color" content="#000000"/><link rel="manifest" href="/manifest.json"/><title>Squeak Node</title><meta name="description" content="Squeak Node is a frontend for accessing a squeak node"><meta name="keywords" content="squeak, bitcoin, lightning"><meta name="author" content="Flatlogic LLC."><link href="/static/css/2.ea4ba2f0.chunk.css" rel="stylesheet"></head><body style="font-family:Roboto,sans-serif"><noscript>You need to enable JavaScript to run this app.</noscript><div id="root"></div><script>!function(e){function r(r){for(var n,f,l=r[0],a=r[1],i=r[2],c=0,s=[];c<l.length;c++)f=l[c],Object.prototype.hasOwnProperty.call(o,f)&&o[f]&&s.push(o[f][0]),o[f]=0;for(n in a)Object.prototype.hasOwnProperty.call(a,n)&&(e[n]=a[n]);for(p&&p(r);s.length;)s.shift()();return u.push.apply(u,i||[]),t()}function t(){for(var e,r=0;r<u.length;r++){for(var t=u[r],n=!0,l=1;l<t.length;l++){var a=t[l];0!==o[a]&&(n=!1)}n&&(u.splice(r--,1),e=f(f.s=t[0]))}return e}var n={},o={1:0},u=[];function f(r){if(n[r])return n[r].exports;var t=n[r]={i:r,l:!1,exports:{}};return e[r].call(t.exports,t,t.exports,f),t.l=!0,t.exports}f.m=e,f.c=n,f.d=function(e,r,t){f.o(e,r)||Object.defineProperty(e,r,{enumerable:!0,get:t})},f.r=function(e){"undefined"!=typeof Symbol&&Symbol.toStringTag&&Object.defineProperty(e,Symbol.toStringTag,{value:"Module"}),Object.defineProperty(e,"__esModule",{value:!0})},f.t=function(e,r){if(1&r&&(e=f(e)),8&r)return e;if(4&r&&"object"==typeof e&&e&&e.__esModule)return e;var t=Object.create(null);if(f.r(t),Object.defineProperty(t,"default",{enumerable:!0,value:e}),2&r&&"string"!=typeof e)for(var n in e)f.d(t,n,function(r){return e[r]}.bind(null,n));return t},f.n=function(e){var r=e&&e.__esModule?function(){return e.default}:function(){return e};return f.d(r,"a",r),r},f.o=function(e,r){return Object.prototype.hasOwnProperty.call(e,r)},f.p="/";var l=this["webpackJsonpsqueak-node-frontend"]=this["webpackJsonpsqueak-node-frontend"]||[],a=l.push.bind(l);l.push=r,l=l.slice();for(var i=0;i<l.length;i++)r(l[i]);var p=a;t()}([])</script><script src="/static/js/2.ed4620a0.chunk.js"></script><script src="/static/js/main.1364ecd2.chunk.js"></script></body></html>
+<!doctype html><html lang="en"><head><meta charset="utf-8"/><link rel="shortcut icon" href="/favicon.ico"/><meta name="viewport" content="width=device-width,initial-scale=1,shrink-to-fit=no"/><meta name="theme-color" content="#000000"/><link rel="manifest" href="/manifest.json"/><title>Squeak Node</title><meta name="description" content="Squeak Node is a frontend for accessing a squeak node"><meta name="keywords" content="squeak, bitcoin, lightning"><meta name="author" content="Flatlogic LLC."><link href="/static/css/2.ea4ba2f0.chunk.css" rel="stylesheet"></head><body style="font-family:Roboto,sans-serif"><noscript>You need to enable JavaScript to run this app.</noscript><div id="root"></div><script>!function(e){function r(r){for(var n,f,l=r[0],a=r[1],i=r[2],c=0,s=[];c<l.length;c++)f=l[c],Object.prototype.hasOwnProperty.call(o,f)&&o[f]&&s.push(o[f][0]),o[f]=0;for(n in a)Object.prototype.hasOwnProperty.call(a,n)&&(e[n]=a[n]);for(p&&p(r);s.length;)s.shift()();return u.push.apply(u,i||[]),t()}function t(){for(var e,r=0;r<u.length;r++){for(var t=u[r],n=!0,l=1;l<t.length;l++){var a=t[l];0!==o[a]&&(n=!1)}n&&(u.splice(r--,1),e=f(f.s=t[0]))}return e}var n={},o={1:0},u=[];function f(r){if(n[r])return n[r].exports;var t=n[r]={i:r,l:!1,exports:{}};return e[r].call(t.exports,t,t.exports,f),t.l=!0,t.exports}f.m=e,f.c=n,f.d=function(e,r,t){f.o(e,r)||Object.defineProperty(e,r,{enumerable:!0,get:t})},f.r=function(e){"undefined"!=typeof Symbol&&Symbol.toStringTag&&Object.defineProperty(e,Symbol.toStringTag,{value:"Module"}),Object.defineProperty(e,"__esModule",{value:!0})},f.t=function(e,r){if(1&r&&(e=f(e)),8&r)return e;if(4&r&&"object"==typeof e&&e&&e.__esModule)return e;var t=Object.create(null);if(f.r(t),Object.defineProperty(t,"default",{enumerable:!0,value:e}),2&r&&"string"!=typeof e)for(var n in e)f.d(t,n,function(r){return e[r]}.bind(null,n));return t},f.n=function(e){var r=e&&e.__esModule?function(){return e.default}:function(){return e};return f.d(r,"a",r),r},f.o=function(e,r){return Object.prototype.hasOwnProperty.call(e,r)},f.p="/";var l=this["webpackJsonpsqueak-node-frontend"]=this["webpackJsonpsqueak-node-frontend"]||[],a=l.push.bind(l);l.push=r,l=l.slice();for(var i=0;i<l.length;i++)r(l[i]);var p=a;t()}([])</script><script src="/static/js/2.73e108c6.chunk.js"></script><script src="/static/js/main.c017f06a.chunk.js"></script></body></html>

squeaknode/admin/webapp/static/build/precache-manifest.98527034402653794ed933617b9ff8d5.js

@@ -1,23 +1,23 @@
 self.__precacheManifest = (self.__precacheManifest || []).concat([
   {
-    "revision": "3506e159d44fb551de01dcbd2e22ea24",
+    "revision": "bc8b0db3dd20a409a07d5a51f9d8271f",
     "url": "/index.html"
   },
   {
-    "revision": "d19b38cc282fe93ba8a7",
+    "revision": "734b8e8587fed7afde5b",
     "url": "/static/css/2.ea4ba2f0.chunk.css"
   },
   {
-    "revision": "d19b38cc282fe93ba8a7",
-    "url": "/static/js/2.ed4620a0.chunk.js"
+    "revision": "734b8e8587fed7afde5b",
+    "url": "/static/js/2.73e108c6.chunk.js"
   },
   {
     "revision": "279b8724b89bf7d504758b3fa917239f",
-    "url": "/static/js/2.ed4620a0.chunk.js.LICENSE.txt"
+    "url": "/static/js/2.73e108c6.chunk.js.LICENSE.txt"
   },
   {
-    "revision": "25695dcd9d15decd3d41",
-    "url": "/static/js/main.1364ecd2.chunk.js"
+    "revision": "e9f6eb5e38da474d1d0a",
+    "url": "/static/js/main.c017f06a.chunk.js"
   },
   {
     "revision": "cc9816de5a8639d377ea",

squeaknode/admin/webapp/static/build/service-worker.js

@@ -14,7 +14,7 @@
 importScripts("https://storage.googleapis.com/workbox-cdn/releases/4.3.1/workbox-sw.js");
 
 importScripts(
-  "/precache-manifest.52936837c4e646d976a01212b0f006e2.js"
+  "/precache-manifest.98527034402653794ed933617b9ff8d5.js"
 );
 
 self.addEventListener('message', (event) => {

squeaknode/db/squeak_db.py

@@ -1042,13 +1042,39 @@ class SqueakDb:
             sent_payment_id = res.inserted_primary_key[0]
             return sent_payment_id
 
-    def get_sent_payments(self) -> List[SentPayment]:
+    def get_sent_payments(
+            self,
+            limit: int,
+            last_sent_payment: Optional[SentPayment],
+    ) -> List[SentPayment]:
         """ Get all sent payments. """
+        last_created_time = last_sent_payment.created if last_sent_payment else self.datetime_now
+        last_payment_hash = last_sent_payment.payment_hash if last_sent_payment else MAX_HASH
+        logger.info("""Get sent payments db query with
+        limit: {}
+        last_created_time: {}
+        last_payment_hash: {}
+        """.format(
+            limit,
+            last_created_time,
+            last_payment_hash.hex(),
+        ))
         s = (
             select([self.sent_payments])
+            .where(
+                tuple_(
+                    self.sent_payments.c.created,
+                    self.sent_payments.c.payment_hash,
+                ) < tuple_(
+                    last_created_time,
+                    last_payment_hash,
+                )
+            )
             .order_by(
                 self.sent_payments.c.created.desc(),
+                self.sent_payments.c.payment_hash.desc(),
             )
+            .limit(limit)
         )
         with self.get_connection() as connection:
             result = connection.execute(s)

squeaknode/node/squeak_controller.py

@@ -415,8 +415,15 @@ class SqueakController:
         )
         return sent_payment_id
 
-    def get_sent_payments(self) -> List[SentPayment]:
-        return self.squeak_db.get_sent_payments()
+    def get_sent_payments(
+            self,
+            limit: int,
+            last_sent_payment: Optional[SentPayment],
+    ) -> List[SentPayment]:
+        return self.squeak_db.get_sent_payments(
+            limit,
+            last_sent_payment,
+        )
 
     def get_sent_payment(self, sent_payment_id: int) -> Optional[SentPayment]:
         return self.squeak_db.get_sent_payment(sent_payment_id)