From 508d3ab539f94939f6fa6125ec8ba431ea459268 Mon Sep 17 00:00:00 2001 From: Jonathan Zernik Date: Sun, 10 Jan 2021 18:57:36 -0800 Subject: [PATCH] Remove squeak store (#592) * Stop using save_squeak method of squeak_store * Stop using get_squeak method of squeak_store * Stop using delete_squeak method of squeak store * Stop using unlock_squeak in squeak_store * Remove squeak_store usage from peer task * Remove some usages of squeak_db from peer_task * Remove all squeak_db usage from peer_task * Remove squeak_store from main module * Delete squeak store class --- squeaknode/core/squeak_controller.py | 133 ++++++++++++++++++++------- squeaknode/main.py | 9 +- squeaknode/node/squeak_store.py | 90 ------------------ squeaknode/sync/peer_task.py | 45 ++++----- tests/core/test_squeak_controller.py | 14 +-- 5 files changed, 126 insertions(+), 165 deletions(-) delete mode 100644 squeaknode/node/squeak_store.py diff --git a/squeaknode/core/squeak_controller.py b/squeaknode/core/squeak_controller.py index 4992436c..d5542b0f 100644 --- a/squeaknode/core/squeak_controller.py +++ b/squeaknode/core/squeak_controller.py @@ -25,27 +25,48 @@ class SqueakController: self, squeak_db, squeak_core, - squeak_store, squeak_whitelist, + squeak_rate_limiter, config, ): self.squeak_db = squeak_db self.squeak_core = squeak_core - self.squeak_store = squeak_store self.squeak_whitelist = squeak_whitelist + self.squeak_rate_limiter = squeak_rate_limiter self.config = config - def save_uploaded_squeak(self, squeak: CSqueak): - return self.squeak_store.save_squeak(squeak) + def save_uploaded_squeak(self, squeak: CSqueak) -> bytes: + if not self.squeak_whitelist.should_allow_squeak(squeak): + raise Exception("Squeak upload not allowed by whitelist.") + if not self.squeak_rate_limiter.should_rate_limit_allow(squeak): + raise Exception( + "Excedeed allowed number of squeaks per block.") + # TODO: Only allow uploaded squeak if decryption key included. + squeak_entry = self.squeak_core.validate_squeak(squeak) + inserted_squeak_hash = self.squeak_db.insert_squeak( + squeak, squeak_entry.block_header) + return inserted_squeak_hash - def save_created_squeak(self, squeak: CSqueak): - return self.squeak_store.save_squeak(squeak, skip_whitelist_check=True) + def save_downloaded_squeak(self, squeak: CSqueak) -> bytes: + if not self.squeak_rate_limiter.should_rate_limit_allow(squeak): + raise Exception( + "Excedeed allowed number of squeaks per block.") + squeak_entry = self.squeak_core.validate_squeak(squeak) + inserted_squeak_hash = self.squeak_db.insert_squeak( + squeak, squeak_entry.block_header) + return inserted_squeak_hash + + def get_squeak(self, squeak_hash: bytes, clear_decryption_key: bool = False): + squeak_entry = self.squeak_db.get_squeak_entry(squeak_hash) + if squeak_entry is None: + return None + squeak = squeak_entry.squeak + if clear_decryption_key: + squeak.ClearDecryptionKey() + return squeak def get_public_squeak(self, squeak_hash: bytes): - return self.squeak_store.get_squeak(squeak_hash, clear_decryption_key=True) - - def lookup_squeaks(self, addresses: str, min_block: int, max_block: int): - return self.squeak_store.lookup_squeaks(addresses, min_block, max_block) + return self.get_squeak(squeak_hash, clear_decryption_key=True) def lookup_allowed_addresses(self, addresses: List[str]): return self.squeak_whitelist.get_allowed_addresses(addresses) @@ -67,7 +88,7 @@ class SqueakController: ) if sent_offer: return sent_offer - squeak = self.squeak_store.get_squeak(squeak_hash) + squeak = self.get_squeak(squeak_hash) # sent_offer = self.create_offer( # squeak, client_addr, self.config.core.price_msat) sent_offer = self.squeak_core.create_offer( @@ -138,33 +159,16 @@ class SqueakController: squeak_profile = self.squeak_db.get_profile(profile_id) squeak_entry = self.squeak_core.make_squeak( squeak_profile, content_str, replyto_hash) - return self.save_created_squeak(squeak_entry.squeak) - - def get_squeak_entry_with_profile(self, squeak_hash: bytes): - return self.squeak_store.get_squeak_entry_with_profile(squeak_hash) - - def get_timeline_squeak_entries_with_profile(self): - return self.squeak_store.get_timeline_squeak_entries_with_profile() - - def get_squeak_entries_with_profile_for_address( - self, address: str, min_block: int, max_block: int - ): - return self.squeak_store.get_squeak_entries_with_profile_for_address( - address, - min_block, - max_block, - ) - - def get_ancestor_squeak_entries_with_profile(self, squeak_hash_str: str): - return self.squeak_store.get_ancestor_squeak_entries_with_profile( - squeak_hash_str, - ) + # return self.save_created_squeak(squeak_entry.squeak) + inserted_squeak_hash = self.squeak_db.insert_squeak( + squeak_entry.squeak, squeak_entry.block_header) + return inserted_squeak_hash def delete_squeak(self, squeak_hash: bytes): num_deleted_offers = self.squeak_db.delete_offers_for_squeak( squeak_hash) logger.info("Deleted number of offers : {}".format(num_deleted_offers)) - return self.squeak_store.delete_squeak(squeak_hash) + return self.squeak_db.delete_squeak(squeak_hash) def create_peer(self, peer_name: str, host: str, port: int): port = port or self.config.core.default_peer_rpc_port @@ -213,12 +217,18 @@ class SqueakController: squeak.SetDecryptionKey(secret_key) CheckSqueak(squeak) # Set the decryption key in the database - self.squeak_store.unlock_squeak( + self.unlock_squeak( offer.squeak_hash, secret_key, ) return sent_payment_id + def unlock_squeak(self, squeak_hash: bytes, secret_key: bytes): + self.squeak_db.set_squeak_decryption_key( + squeak_hash, + secret_key, + ) + def get_sent_payments(self): return self.squeak_db.get_sent_payments() @@ -283,3 +293,58 @@ class SqueakController: def get_offer(self, squeak: CSqueak, offer_msg: squeak_server_pb2.SqueakBuyOffer, peer: SqueakPeer) -> Offer: return self.squeak_core.get_offer(squeak, offer_msg, peer) + + def get_squeak_entry_with_profile(self, squeak_hash: bytes): + return self.squeak_db.get_squeak_entry_with_profile(squeak_hash) + + def get_timeline_squeak_entries_with_profile(self): + return self.squeak_db.get_timeline_squeak_entries_with_profile() + + def get_squeak_entries_with_profile_for_address( + self, address: str, min_block: int, max_block: int + ): + return self.squeak_db.get_squeak_entries_with_profile_for_address( + address, + min_block, + max_block, + ) + + def get_ancestor_squeak_entries_with_profile(self, squeak_hash: bytes): + return self.squeak_db.get_thread_ancestor_squeak_entries_with_profile( + squeak_hash, + ) + + def lookup_squeaks(self, addresses: List[str], min_block: int, max_block: int): + return self.squeak_db.lookup_squeaks( + addresses, + min_block, + max_block, + ) + + def lookup_squeaks_include_locked(self, addresses: List[str], min_block: int, max_block: int): + return self.squeak_db.lookup_squeaks( + addresses, + min_block, + max_block, + include_locked=True, + ) + + def lookup_squeaks_needing_offer(self, addresses: List[str], min_block, max_block, peer_id): + return self.squeak_db.lookup_squeaks_needing_offer( + addresses, + min_block, + max_block, + peer_id, + ) + + def save_offer(self, offer: Offer): + logger.info("Saving offer: {}".format(offer)) + self.squeak_db.insert_offer(offer) + + def get_followed_addresses(self): + followed_profiles = self.squeak_db.get_following_profiles() + return [profile.address for profile in followed_profiles] + + def get_sharing_addresses(self): + sharing_profiles = self.squeak_db.get_sharing_profiles() + return [profile.address for profile in sharing_profiles] diff --git a/squeaknode/main.py b/squeaknode/main.py index 47c9a915..cc0727e5 100644 --- a/squeaknode/main.py +++ b/squeaknode/main.py @@ -23,7 +23,6 @@ from squeaknode.node.received_payments_subscription_client import ( from squeaknode.node.squeak_memory_whitelist import SqueakMemoryWhitelist from squeaknode.node.squeak_node import SqueakNode from squeaknode.node.squeak_rate_limiter import SqueakRateLimiter -from squeaknode.node.squeak_store import SqueakStore from squeaknode.server.squeak_server_handler import SqueakServerHandler from squeaknode.server.squeak_server_servicer import SqueakServerServicer from squeaknode.sync.squeak_peer_sync_worker import SqueakPeerSyncWorker @@ -246,18 +245,12 @@ def run_node(config): squeak_whitelist = SqueakMemoryWhitelist( squeak_db, ) - squeak_store = SqueakStore( - squeak_db, - squeak_core, - squeak_rate_limiter, - squeak_whitelist, - ) squeak_controller = SqueakController( squeak_db, squeak_core, - squeak_store, squeak_whitelist, + squeak_rate_limiter, config, ) diff --git a/squeaknode/node/squeak_store.py b/squeaknode/node/squeak_store.py deleted file mode 100644 index bbe84986..00000000 --- a/squeaknode/node/squeak_store.py +++ /dev/null @@ -1,90 +0,0 @@ -import logging -from typing import List - -from squeak.core import CSqueak - -logger = logging.getLogger(__name__) - - -class SqueakStore: - def __init__( - self, squeak_db, squeak_core, squeak_rate_limiter, squeak_whitelist - ): - self.squeak_db = squeak_db - self.squeak_core = squeak_core - self.squeak_rate_limiter = squeak_rate_limiter - self.squeak_whitelist = squeak_whitelist - - def save_squeak(self, squeak: CSqueak, skip_whitelist_check: bool = False): - if not skip_whitelist_check: - if not self.squeak_whitelist.should_allow_squeak(squeak): - raise Exception("Squeak upload not allowed by whitelist.") - - if not self.squeak_rate_limiter.should_rate_limit_allow(squeak): - raise Exception( - "Excedeed allowed number of squeaks per block.") - block_info = self.squeak_core.validate_squeak(squeak) - inserted_squeak_hash = self.squeak_db.insert_squeak( - squeak, block_info.block_header) - return inserted_squeak_hash - - def get_squeak(self, squeak_hash: bytes, clear_decryption_key: bool = False): - squeak_entry = self.squeak_db.get_squeak_entry(squeak_hash) - if squeak_entry is None: - return None - squeak = squeak_entry.squeak - if clear_decryption_key: - squeak.ClearDecryptionKey() - return squeak - - def get_squeak_entry_with_profile(self, squeak_hash: bytes): - return self.squeak_db.get_squeak_entry_with_profile(squeak_hash) - - def get_timeline_squeak_entries_with_profile(self): - return self.squeak_db.get_timeline_squeak_entries_with_profile() - - def get_squeak_entries_with_profile_for_address( - self, address: str, min_block: int, max_block: int - ): - return self.squeak_db.get_squeak_entries_with_profile_for_address( - address, - min_block, - max_block, - ) - - def get_ancestor_squeak_entries_with_profile(self, squeak_hash: bytes): - return self.squeak_db.get_thread_ancestor_squeak_entries_with_profile( - squeak_hash, - ) - - def delete_squeak(self, squeak_hash: bytes): - return self.squeak_db.delete_squeak(squeak_hash) - - def lookup_squeaks(self, addresses: List[str], min_block: int, max_block: int): - return self.squeak_db.lookup_squeaks( - addresses, - min_block, - max_block, - ) - - def lookup_squeaks_include_locked(self, addresses: List[str], min_block: int, max_block: int): - return self.squeak_db.lookup_squeaks( - addresses, - min_block, - max_block, - include_locked=True, - ) - - def lookup_squeaks_needing_offer(self, addresses: List[str], min_block, max_block, peer_id): - return self.squeak_db.lookup_squeaks_needing_offer( - addresses, - min_block, - max_block, - peer_id, - ) - - def unlock_squeak(self, squeak_hash: bytes, secret_key: bytes): - self.squeak_db.set_squeak_decryption_key( - squeak_hash, - secret_key, - ) diff --git a/squeaknode/sync/peer_task.py b/squeaknode/sync/peer_task.py index 46720dca..9b955cdd 100644 --- a/squeaknode/sync/peer_task.py +++ b/squeaknode/sync/peer_task.py @@ -18,14 +18,6 @@ class PeerSyncTask: def peer(self): return self.peer_connection.peer - @property - def squeak_store(self): - return self.squeak_controller.squeak_store - - @property - def squeak_db(self): - return self.squeak_controller.squeak_db - @property def peer_client(self): return self.peer_connection.peer_client @@ -36,7 +28,7 @@ class PeerSyncTask: max_block, ): # Get list of followed addresses. - addresses = self._get_followed_addresses() + addresses = self.squeak_controller.get_followed_addresses() logger.debug("Followed addresses: {}".format(addresses)) # Get remote hashes @@ -93,7 +85,7 @@ class PeerSyncTask: max_block, ): # Get list of sharing addresses. - addresses = self._get_sharing_addresses() + addresses = self.squeak_controller.get_sharing_addresses() logger.debug("Sharing addresses: {}".format(addresses)) # Get remote hashes @@ -166,20 +158,20 @@ class PeerSyncTask: squeak, offer_msg, self.peer) # Save the offer - self._save_offer(decoded_offer) + self.squeak_controller.save_offer(decoded_offer) def _get_local_hashes(self, addresses, min_block, max_block): - return self.squeak_store.lookup_squeaks_include_locked( + return self.squeak_controller.lookup_squeaks_include_locked( addresses, min_block, max_block, ) def _get_local_unlocked_hashes(self, addresses, min_block, max_block): - return self.squeak_store.lookup_squeaks(addresses, min_block, max_block) + return self.squeak_controller.lookup_squeaks(addresses, min_block, max_block) def _get_locked_hashes(self, addresses, min_block, max_block): - return self.squeak_store.lookup_squeaks_needing_offer( + return self.squeak_controller.lookup_squeaks_needing_offer( addresses, min_block, max_block, @@ -190,12 +182,13 @@ class PeerSyncTask: return self.peer_client.lookup_squeaks(addresses, min_block, max_block) def _save_squeak(self, squeak): - self.squeak_store.save_squeak(squeak, skip_whitelist_check=True) + self.squeak_controller.save_downloaded_squeak(squeak) def _get_saved_offer(self, squeak_hash: bytes): logger.info("Getting saved offer for hash: {}".format( squeak_hash.hex())) - offers = self.squeak_db.get_offers_with_peer(squeak_hash) + # offers = self.squeak_db.get_offers_with_peer(squeak_hash) + offers = self.squeak_controller.get_buy_offers_with_peer(squeak_hash) for offer_with_peer in offers: if offer_with_peer.offer.peer_id == self.peer.peer_id: return offer_with_peer @@ -209,16 +202,16 @@ class PeerSyncTask: squeak = self.peer_client.get_squeak(squeak_hash) self._save_squeak(squeak) - def _get_followed_addresses(self): - followed_profiles = self.squeak_db.get_following_profiles() - return [profile.address for profile in followed_profiles] + # def _get_followed_addresses(self): + # followed_profiles = self.squeak_db.get_following_profiles() + # return [profile.address for profile in followed_profiles] def _download_offer(self, squeak_hash: bytes): logger.info("Downloading offer for hash: {}".format(squeak_hash.hex())) self.get_offer(squeak_hash) def _get_local_squeak(self, squeak_hash: bytes): - return self.squeak_store.get_squeak(squeak_hash) + return self.squeak_controller.get_squeak(squeak_hash) def _try_upload_squeak(self, squeak_hash: bytes, allowed_addresses): squeak = self._get_local_squeak(squeak_hash) @@ -232,15 +225,15 @@ class PeerSyncTask: )) self.peer_client.post_squeak(squeak) - def _get_sharing_addresses(self): - sharing_profiles = self.squeak_db.get_sharing_profiles() - return [profile.address for profile in sharing_profiles] + # def _get_sharing_addresses(self): + # sharing_profiles = self.squeak_db.get_sharing_profiles() + # return [profile.address for profile in sharing_profiles] def _download_offer_msg(self, squeak_hash: bytes): logger.info( "Downloading buy offer for squeak hash: {}".format(squeak_hash.hex())) return self.peer_client.buy_squeak(squeak_hash) - def _save_offer(self, offer): - logger.info("Saving offer: {}".format(offer)) - self.squeak_db.insert_offer(offer) + # def _save_offer(self, offer): + # logger.info("Saving offer: {}".format(offer)) + # self.squeak_db.insert_offer(offer) diff --git a/tests/core/test_squeak_controller.py b/tests/core/test_squeak_controller.py index 42afbdf2..519c9782 100644 --- a/tests/core/test_squeak_controller.py +++ b/tests/core/test_squeak_controller.py @@ -7,7 +7,7 @@ from squeaknode.core.squeak_controller import SqueakController from squeaknode.core.squeak_core import SqueakCore from squeaknode.core.squeak_peer import SqueakPeer from squeaknode.db.squeak_db import SqueakDb -from squeaknode.node.squeak_store import SqueakStore +from squeaknode.node.squeak_rate_limiter import SqueakRateLimiter from squeaknode.node.squeak_whitelist import SqueakWhitelist @@ -59,23 +59,23 @@ def squeak_whitelist(): @pytest.fixture -def squeak_store(): - return mock.Mock(spec=SqueakStore) +def squeak_rate_limiter(): + return mock.Mock(spec=SqueakRateLimiter) @pytest.fixture def squeak_controller( squeak_db, squeak_core, - squeak_store, squeak_whitelist, + squeak_rate_limiter, config, ): return SqueakController( squeak_db, squeak_core, - squeak_store, squeak_whitelist, + squeak_rate_limiter, config, ) @@ -84,15 +84,15 @@ def squeak_controller( def regtest_squeak_controller( squeak_db, squeak_core, - squeak_store, squeak_whitelist, + squeak_rate_limiter, regtest_config, ): return SqueakController( squeak_db, squeak_core, - squeak_store, squeak_whitelist, + squeak_rate_limiter, regtest_config, )