From b93578546975c2d7204d5ca6f08d0e82188b115a Mon Sep 17 00:00:00 2001 From: Jonathan Zernik Date: Fri, 4 Jun 2021 18:28:42 -0700 Subject: [PATCH] Use timestamp for like field (#896) * Use a timestamp for like field in database * Use query to get liked squeaks in order of like --- itests/tests/test_squeak_node.py | 16 ++++- proto/squeak_admin.proto | 4 +- squeaknode/admin/messages.py | 4 +- .../admin/squeak_admin_server_handler.py | 2 +- squeaknode/core/squeak_entry.py | 3 +- ...4732c09b6_use_timestamp_for_liked_field.py | 38 ++++++++++++ squeaknode/db/models.py | 3 +- squeaknode/db/squeak_db.py | 58 +++++++++++++------ squeaknode/node/squeak_controller.py | 12 ++-- squeaknode/sync/network_sync.py | 6 +- 10 files changed, 111 insertions(+), 35 deletions(-) create mode 100644 squeaknode/db/alembic/versions/c874732c09b6_use_timestamp_for_liked_field.py diff --git a/itests/tests/test_squeak_node.py b/itests/tests/test_squeak_node.py index 518ff8a1..d48ab993 100644 --- a/itests/tests/test_squeak_node.py +++ b/itests/tests/test_squeak_node.py @@ -1363,6 +1363,18 @@ def test_get_squeak_details(server_stub, admin_stub, saved_squeak_hash): def test_like_squeak(server_stub, admin_stub, saved_squeak_hash): + # Get the squeak display item + get_squeak_display_response = admin_stub.GetSqueakDisplay( + squeak_admin_pb2.GetSqueakDisplayRequest( + squeak_hash=saved_squeak_hash, + ) + ) + print("get_squeak_display_response.squeak_display_entry:") + print(get_squeak_display_response.squeak_display_entry) + assert ( + get_squeak_display_response.squeak_display_entry.liked_time_s == 0 + ) + # Like the squeak admin_stub.LikeSqueak( squeak_admin_pb2.LikeSqueakRequest( @@ -1377,7 +1389,7 @@ def test_like_squeak(server_stub, admin_stub, saved_squeak_hash): ) ) assert ( - get_squeak_display_response.squeak_display_entry.liked + get_squeak_display_response.squeak_display_entry.liked_time_s > 0 ) # Unlike the squeak @@ -1394,5 +1406,5 @@ def test_like_squeak(server_stub, admin_stub, saved_squeak_hash): ) ) assert ( - not get_squeak_display_response.squeak_display_entry.liked + get_squeak_display_response.squeak_display_entry.liked_time_s == 0 ) diff --git a/proto/squeak_admin.proto b/proto/squeak_admin.proto index 56919f3b..22a759c8 100644 --- a/proto/squeak_admin.proto +++ b/proto/squeak_admin.proto @@ -488,8 +488,8 @@ message SqueakDisplayEntry { /// The author name SqueakProfile author = 11; - /// Is liked - bool liked = 12; + /// Liked time + int64 liked_time_s = 12; } message GetTimelineSqueakDisplaysRequest { diff --git a/squeaknode/admin/messages.py b/squeaknode/admin/messages.py index 090c6d15..5761c3bd 100644 --- a/squeaknode/admin/messages.py +++ b/squeaknode/admin/messages.py @@ -25,7 +25,7 @@ def squeak_entry_to_message(squeak_entry_with_profile: SqueakEntryWithProfile) - squeak_entry = squeak_entry_with_profile.squeak_entry squeak = squeak_entry.squeak block_header = squeak_entry.block_header - liked = squeak_entry.liked + liked_time_s = squeak_entry.liked_time or 0 is_unlocked = squeak.HasDecryptionKey() content_str = squeak.GetDecryptedContentStr() if is_unlocked else None is_reply = squeak.is_reply @@ -49,7 +49,7 @@ def squeak_entry_to_message(squeak_entry_with_profile: SqueakEntryWithProfile) - author_address=author_address, is_author_known=is_author_known, author=profile_msg, - liked=liked, + liked_time_s=liked_time_s, ) diff --git a/squeaknode/admin/squeak_admin_server_handler.py b/squeaknode/admin/squeak_admin_server_handler.py index 34133afd..70ad5eff 100644 --- a/squeaknode/admin/squeak_admin_server_handler.py +++ b/squeaknode/admin/squeak_admin_server_handler.py @@ -274,7 +274,7 @@ class SqueakAdminServerHandler(object): ) ) logger.info("Returning squeak display entry with like value: {}".format( - squeak_entry_with_profile.squeak_entry.liked, + squeak_entry_with_profile.squeak_entry.liked_time, )) display_message = squeak_entry_to_message( squeak_entry_with_profile) diff --git a/squeaknode/core/squeak_entry.py b/squeaknode/core/squeak_entry.py index fd62cbe6..7e8953be 100644 --- a/squeaknode/core/squeak_entry.py +++ b/squeaknode/core/squeak_entry.py @@ -1,4 +1,5 @@ from typing import NamedTuple +from typing import Optional from bitcoin.core import CBlockHeader from squeak.core import CSqueak @@ -7,4 +8,4 @@ from squeak.core import CSqueak class SqueakEntry(NamedTuple): squeak: CSqueak block_header: CBlockHeader - liked: bool = False + liked_time: Optional[int] = None diff --git a/squeaknode/db/alembic/versions/c874732c09b6_use_timestamp_for_liked_field.py b/squeaknode/db/alembic/versions/c874732c09b6_use_timestamp_for_liked_field.py new file mode 100644 index 00000000..1032bc6f --- /dev/null +++ b/squeaknode/db/alembic/versions/c874732c09b6_use_timestamp_for_liked_field.py @@ -0,0 +1,38 @@ +"""Use timestamp for liked field + +Revision ID: c874732c09b6 +Revises: f62215acb987 +Create Date: 2021-06-04 17:03:56.626831 + +""" +import sqlalchemy as sa +from alembic import op + +import squeaknode.db.models + + +# revision identifiers, used by Alembic. +revision = 'c874732c09b6' +down_revision = 'f62215acb987' +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + with op.batch_alter_table('squeak', schema=None) as batch_op: + batch_op.add_column( + sa.Column('liked_time', squeaknode.db.models.TZDateTime(), nullable=True)) + batch_op.drop_column('liked') + + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + with op.batch_alter_table('squeak', schema=None) as batch_op: + batch_op.add_column(sa.Column('liked', sa.BOOLEAN(), + server_default=sa.text('0'), nullable=False)) + batch_op.drop_column('liked_time') + + # ### end Alembic commands ### diff --git a/squeaknode/db/models.py b/squeaknode/db/models.py index 0fa3fab3..89c0e3f3 100644 --- a/squeaknode/db/models.py +++ b/squeaknode/db/models.py @@ -70,7 +70,8 @@ class Models: Column("author_address", String(35), index=True, nullable=False), Column("secret_key", String(64), nullable=True), Column("block_header", Binary, nullable=False), - Column("liked", Boolean, nullable=False, default=False), + Column("liked_time", TZDateTime, + default=None, nullable=True), ) self.profiles = Table( diff --git a/squeaknode/db/squeak_db.py b/squeaknode/db/squeak_db.py index bbee7325..70a5a899 100644 --- a/squeaknode/db/squeak_db.py +++ b/squeaknode/db/squeak_db.py @@ -90,11 +90,11 @@ class SqueakDb: @property def squeak_is_liked(self): - return self.squeaks.c.liked + return self.squeaks.c.liked_time != None # noqa: E711 @property def squeak_is_not_liked(self): - return self.squeaks.c.liked == False # noqa: E711 + return self.squeaks.c.liked_time == None # noqa: E711 @property def squeak_has_no_secret_key(self): @@ -218,10 +218,7 @@ class SqueakDb: return None return self._parse_squeak_entry_with_profile(row) - def get_timeline_squeak_entries_with_profile( - self, - only_liked: bool = False, - ) -> List[SqueakEntryWithProfile]: + def get_timeline_squeak_entries_with_profile(self) -> List[SqueakEntryWithProfile]: """ Get all followed squeaks. """ s = ( select([self.squeaks, self.profiles]) @@ -231,12 +228,6 @@ class SqueakDb: self.profiles.c.address == self.squeaks.c.author_address, ) ) - .where( - or_( - self.squeak_is_liked, - not only_liked, - ) - ) .order_by( self.squeaks.c.n_block_height.desc(), self.squeaks.c.n_time.desc(), @@ -247,6 +238,28 @@ class SqueakDb: rows = result.fetchall() return [self._parse_squeak_entry_with_profile(row) for row in rows] + def get_liked_squeak_entries_with_profile(self) -> List[SqueakEntryWithProfile]: + """ Get liked squeaks. """ + s = ( + select([self.squeaks, self.profiles]) + .select_from( + self.squeaks.outerjoin( + self.profiles, + self.profiles.c.address == self.squeaks.c.author_address, + ) + ) + .where( + self.squeak_is_liked, + ) + .order_by( + self.squeaks.c.liked_time.desc(), + ) + ) + with self.get_connection() as connection: + result = connection.execute(s) + rows = result.fetchall() + return [self._parse_squeak_entry_with_profile(row) for row in rows] + def get_squeak_entries_with_profile_for_address( self, address: str, min_block: int, max_block: int ) -> List[SqueakEntryWithProfile]: @@ -622,12 +635,22 @@ class SqueakDb: with self.get_connection() as connection: connection.execute(stmt) - def set_squeak_liked(self, squeak_hash: bytes, liked: bool) -> None: - """ Set the liked value of a squeak. """ + def set_squeak_liked(self, squeak_hash: bytes) -> None: + """ Set the squeak to be liked. """ stmt = ( self.squeaks.update() .where(self.squeaks.c.hash == squeak_hash.hex()) - .values(liked=liked) + .values(liked_time=self.datetime_now) + ) + with self.get_connection() as connection: + connection.execute(stmt) + + def set_squeak_unliked(self, squeak_hash: bytes) -> None: + """ Set the squeak to be unliked. """ + stmt = ( + self.squeaks.update() + .where(self.squeaks.c.hash == squeak_hash.hex()) + .values(liked_time=None) ) with self.get_connection() as connection: connection.execute(stmt) @@ -1121,11 +1144,12 @@ class SqueakDb: parse_block_header(block_header_bytes) if block_header_bytes else None ) - liked = row["liked"] + liked_time = row["liked_time"] + liked_time_s = int(liked_time.timestamp()) if liked_time else None return SqueakEntry( squeak=squeak, block_header=block_header, - liked=liked, + liked_time=liked_time_s, ) def _parse_squeak_profile(self, row) -> SqueakProfile: diff --git a/squeaknode/node/squeak_controller.py b/squeaknode/node/squeak_controller.py index 4d10f355..035748e2 100644 --- a/squeaknode/node/squeak_controller.py +++ b/squeaknode/node/squeak_controller.py @@ -450,9 +450,7 @@ class SqueakController: return self.squeak_db.get_timeline_squeak_entries_with_profile() def get_liked_squeak_entries_with_profile(self): - return self.squeak_db.get_timeline_squeak_entries_with_profile( - only_liked=True, - ) + return self.squeak_db.get_liked_squeak_entries_with_profile() def get_squeak_entries_with_profile_for_address( self, address: str, min_block: int, max_block: int @@ -503,9 +501,13 @@ class SqueakController: self.payment_processor.start_processing() def delete_old_squeaks(self): + logger.info("Deleting old squeaks...") squeaks_to_delete = self.squeak_db.get_old_squeaks_to_delete( self.config.core.squeak_retention_s, ) + logger.info("Got old squeakd to delete: {}".format( + squeaks_to_delete, + )) for squeak_entry_with_profile in squeaks_to_delete: squeak = squeak_entry_with_profile.squeak_entry.squeak squeak_hash = get_hash(squeak) @@ -522,14 +524,12 @@ class SqueakController: )) self.squeak_db.set_squeak_liked( squeak_hash, - True, ) def unlike_squeak(self, squeak_hash: bytes): logger.info("Unliking squeak: {}".format( squeak_hash.hex(), )) - self.squeak_db.set_squeak_liked( + self.squeak_db.set_squeak_unliked( squeak_hash, - False, ) diff --git a/squeaknode/sync/network_sync.py b/squeaknode/sync/network_sync.py index e579fe22..5d55887c 100644 --- a/squeaknode/sync/network_sync.py +++ b/squeaknode/sync/network_sync.py @@ -83,7 +83,7 @@ class TimelineDownloadSync(DownloadSync): try: peer_connection.download(self.block_interval) except Exception: - logger.info("Failed to download timeline from peer: {}".format( + logger.exception("Failed to download timeline from peer: {}".format( peer_connection.peer_address, )) @@ -106,11 +106,10 @@ class SingleSqueakDownloadSync(DownloadSync): try: peer_connection.download_single_squeak(self.squeak_hash) except Exception: - logger.info("Failed to download single squeak: {} from peer: {}".format( + logger.exception("Failed to download single squeak: {} from peer: {}".format( self.squeak_hash.hex(), peer_connection.peer_address, )) - logger.error("Download single squeak error:", exc_info=True) class TimelineUploadSync(UploadSync): @@ -132,6 +131,7 @@ class TimelineUploadSync(UploadSync): logger.info("Failed to upload timeline to peer: {}".format( peer_connection.peer_address, )) + logger.exception("Failed to upload timeline to peer") class SingleSqueakUploadSync(UploadSync):