diff --git a/itests/config.ini b/itests/config.ini index 74a60325..8298c254 100644 --- a/itests/config.ini +++ b/itests/config.ini @@ -18,7 +18,7 @@ rpc_ssl_cert=/rpc/rpc.cert [server] external_address=myexternaladdress.com -port=8765 +external_port=8765 [postgresql] host=db diff --git a/itests/tests/test_squeak_node.py b/itests/tests/test_squeak_node.py index 01cbe2e8..6f12f0ad 100644 --- a/itests/tests/test_squeak_node.py +++ b/itests/tests/test_squeak_node.py @@ -39,7 +39,6 @@ from tests.util import delete_profile from tests.util import delete_squeak from tests.util import download_squeak from tests.util import free_price -from tests.util import get_default_peer_port from tests.util import get_external_address from tests.util import get_hash from tests.util import get_network @@ -94,14 +93,6 @@ def test_get_external_address(admin_stub): assert external_address.port == 8765 -def test_get_default_peer_port(admin_stub): - # Get the default peer port - default_peer_port = get_default_peer_port(admin_stub) - - print(default_peer_port) - assert default_peer_port == 18777 - - def test_reprocess_received_payments(admin_stub): # Reprocess received payments reprocess_received_payments_response = admin_stub.ReprocessReceivedPayments( diff --git a/itests/tests/util.py b/itests/tests/util.py index e6e62b4e..a359686b 100644 --- a/itests/tests/util.py +++ b/itests/tests/util.py @@ -296,13 +296,6 @@ def get_external_address(node_stub): return get_external_address_response.peer_address -def get_default_peer_port(node_stub): - get_default_peer_port_response = node_stub.GetDefaultPeerPort( - squeak_admin_pb2.GetDefaultPeerPortRequest() - ) - return get_default_peer_port_response.port - - def make_squeak(node_stub, profile_id, squeak_content, reply_to_hash=None, recipient_profile_id=None): make_squeak_response = node_stub.MakeSqueak( squeak_admin_pb2.MakeSqueakRequest( diff --git a/squeaknode/admin/squeak_admin_server_handler.py b/squeaknode/admin/squeak_admin_server_handler.py index afc60e41..09ea0b86 100644 --- a/squeaknode/admin/squeak_admin_server_handler.py +++ b/squeaknode/admin/squeak_admin_server_handler.py @@ -703,7 +703,7 @@ class SqueakAdminServerHandler(object): author_profile_id, ), ) - self.squeak_controller.unlock_squeak( + self.squeak_controller.decrypt_private_squeak( squeak_hash=squeak_hash, author_profile_id=author_profile_id, recipient_profile_id=recipient_profile_id, diff --git a/squeaknode/client/network_controller.py b/squeaknode/client/network_controller.py index ec666a2e..4375cec4 100644 --- a/squeaknode/client/network_controller.py +++ b/squeaknode/client/network_controller.py @@ -20,12 +20,15 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. import logging +from concurrent.futures import ThreadPoolExecutor +from concurrent.futures import wait from typing import Optional from squeak.core.keys import SqueakPublicKey from squeaknode.client.peer_downloader import RangeDownloader from squeaknode.client.peer_downloader import SingleDownloader +from squeaknode.node.squeak_store import SqueakStore logger = logging.getLogger(__name__) @@ -37,11 +40,11 @@ class NetworkController: def __init__( self, - squeak_controller, + squeak_store: SqueakStore, proxy_host: Optional[str], proxy_port: Optional[int], ): - self.squeak_controller = squeak_controller + self.squeak_store = squeak_store self.proxy_host = proxy_host self.proxy_port = proxy_port @@ -49,14 +52,14 @@ class NetworkController: self, interest_block_interval: int, ) -> None: - max_block = self.squeak_controller.get_latest_block() + max_block = self.squeak_store.get_latest_block() min_block = max(0, max_block - interest_block_interval) - followed_public_keys = self.squeak_controller.get_followed_public_keys() - peers = self.squeak_controller.get_autoconnect_peers() + followed_public_keys = self.squeak_store.get_followed_public_keys() + peers = self.squeak_store.get_autoconnect_peers() for peer in peers: downloader = RangeDownloader( peer, - self.squeak_controller, + self.squeak_store, self.proxy_host, self.proxy_port, min_block, @@ -68,11 +71,11 @@ class NetworkController: def download_pubkey_squeaks_async(self, pubkey: SqueakPublicKey) -> None: min_block = 0 # TODO max_block = 999999999999 # TODO - peers = self.squeak_controller.get_autoconnect_peers() + peers = self.squeak_store.get_autoconnect_peers() for peer in peers: downloader = RangeDownloader( peer, - self.squeak_controller, + self.squeak_store, self.proxy_host, self.proxy_port, min_block, @@ -82,13 +85,20 @@ class NetworkController: downloader.download_async() def download_single_squeak(self, squeak_hash: bytes) -> None: - peers = self.squeak_controller.get_autoconnect_peers() - for peer in peers: - downloader = SingleDownloader( + peers = self.squeak_store.get_autoconnect_peers() + downloaders = [ + SingleDownloader( peer, - self.squeak_controller, + self.squeak_store, self.proxy_host, self.proxy_port, squeak_hash, - ) - downloader.download_async() + ) for peer in peers + ] + with ThreadPoolExecutor(50) as executor: + # submit tasks and collect futures + futures = [executor.submit(downloader.download) + for downloader in downloaders] + # wait for all tasks to complete + wait(futures) + logger.info('All downloads are done!') diff --git a/squeaknode/client/peer_downloader.py b/squeaknode/client/peer_downloader.py index cc5b5100..7e874c6b 100644 --- a/squeaknode/client/peer_downloader.py +++ b/squeaknode/client/peer_downloader.py @@ -32,6 +32,7 @@ from squeak.core.keys import SqueakPublicKey from squeaknode.client.peer_client import PeerClient from squeaknode.core.squeak_peer import SqueakPeer from squeaknode.core.squeaks import get_hash +from squeaknode.node.squeak_store import SqueakStore logger = logging.getLogger(__name__) @@ -44,7 +45,7 @@ class PeerDownloader(ABC): def __init__( self, peer: SqueakPeer, - squeak_controller, + squeak_store: SqueakStore, proxy_host: Optional[str], proxy_port: Optional[int], ): @@ -52,7 +53,7 @@ class PeerDownloader(ABC): self.proxy_host = proxy_host self.proxy_port = proxy_port self.client = PeerClient(peer, proxy_host, proxy_port) - self.squeak_controller = squeak_controller + self.squeak_store = squeak_store @abstractmethod def get_hashes(self) -> List[bytes]: @@ -83,33 +84,33 @@ class PeerDownloader(ABC): def get_squeak(self, squeak_hash: bytes) -> None: # Download the squeak if not already owned. - if self.squeak_controller.get_squeak(squeak_hash): + if self.squeak_store.get_squeak(squeak_hash): return squeak = self.client.get_squeak(squeak_hash) if squeak and self.is_squeak_wanted(squeak): - self.squeak_controller.save_squeak(squeak) + self.squeak_store.save_squeak(squeak) def get_secret_key(self, squeak_hash: bytes) -> None: # Get the squeak from the database. - squeak = self.squeak_controller.get_squeak(squeak_hash) + squeak = self.squeak_store.get_squeak(squeak_hash) if squeak and self.is_squeak_wanted(squeak): # Download the secret key is not already unlocked. - if self.squeak_controller.get_squeak_secret_key(squeak_hash): + if self.squeak_store.get_squeak_secret_key(squeak_hash): return secret_key = self.client.get_secret_key(squeak_hash) if secret_key: - self.squeak_controller.save_secret_key(squeak_hash, secret_key) + self.squeak_store.save_secret_key(squeak_hash, secret_key) def get_offer(self, squeak_hash: bytes) -> None: # Get the squeak from the database. - squeak = self.squeak_controller.get_squeak(squeak_hash) + squeak = self.squeak_store.get_squeak(squeak_hash) if squeak and self.is_squeak_wanted(squeak): # Download the secret key is not already unlocked. - if self.squeak_controller.get_squeak_secret_key(squeak_hash): + if self.squeak_store.get_squeak_secret_key(squeak_hash): return offer = self.client.get_offer(squeak_hash) if offer: - self.squeak_controller.handle_offer( + self.squeak_store.handle_offer( squeak, offer, self.peer.address, @@ -121,14 +122,14 @@ class RangeDownloader(PeerDownloader): def __init__( self, peer: SqueakPeer, - squeak_controller, + squeak_store: SqueakStore, proxy_host: Optional[str], proxy_port: Optional[int], min_block: int, max_block: int, pubkeys: List[SqueakPublicKey], ): - super().__init__(peer, squeak_controller, proxy_host, proxy_port) + super().__init__(peer, squeak_store, proxy_host, proxy_port) self.min_block = min_block self.max_block = max_block self.pubkeys = pubkeys @@ -151,12 +152,12 @@ class SingleDownloader(PeerDownloader): def __init__( self, peer: SqueakPeer, - squeak_controller, + squeak_store: SqueakStore, proxy_host: Optional[str], proxy_port: Optional[int], squeak_hash: bytes, ): - super().__init__(peer, squeak_controller, proxy_host, proxy_port) + super().__init__(peer, squeak_store, proxy_host, proxy_port) self.squeak_hash = squeak_hash def get_hashes(self) -> List[bytes]: diff --git a/squeaknode/node/squeak_controller.py b/squeaknode/node/squeak_controller.py index 35bd1742..e7e0b255 100644 --- a/squeaknode/node/squeak_controller.py +++ b/squeaknode/node/squeak_controller.py @@ -28,7 +28,6 @@ from squeak.core import CSqueak from squeak.core.keys import SqueakPrivateKey from squeak.core.keys import SqueakPublicKey -from squeaknode.client.network_controller import NetworkController from squeaknode.core.download_result import DownloadResult from squeaknode.core.lightning_address import LightningAddressHostPort from squeaknode.core.offer import Offer @@ -37,10 +36,8 @@ from squeaknode.core.peer_address import PeerAddress from squeaknode.core.received_offer import ReceivedOffer from squeaknode.core.received_payment import ReceivedPayment from squeaknode.core.received_payment_summary import ReceivedPaymentSummary -from squeaknode.core.sent_offer import SentOffer from squeaknode.core.sent_payment import SentPayment from squeaknode.core.sent_payment_summary import SentPaymentSummary -from squeaknode.core.squeak_core import SqueakCore from squeaknode.core.squeak_entry import SqueakEntry from squeaknode.core.squeak_peer import SqueakPeer from squeaknode.core.squeak_profile import SqueakProfile @@ -61,22 +58,19 @@ class SqueakController: def __init__( self, squeak_store: SqueakStore, - squeak_core: SqueakCore, payment_processor, + tweet_forwarder, + network_controller, node_settings, config, ): self.squeak_store = squeak_store - self.squeak_core = squeak_core self.payment_processor = payment_processor + self.tweet_forwarder = tweet_forwarder + self.network_controller = network_controller self.node_settings = node_settings self.config = config - def save_squeak(self, squeak: CSqueak) -> Optional[bytes]: - # Get the block header. - block_header = self.squeak_core.get_block_header(squeak) - return self.squeak_store.save_squeak(squeak, block_header) - def make_squeak( self, profile_id: int, @@ -84,118 +78,15 @@ class SqueakController: replyto_hash: Optional[bytes], recipient_profile_id: Optional[int], ) -> Optional[bytes]: - squeak_profile = self.squeak_store.get_squeak_profile(profile_id) - if squeak_profile is None: - raise Exception("Profile with id {} not found.".format( - profile_id, - )) - if recipient_profile_id: - recipient_profile = self.squeak_store.get_squeak_profile( - recipient_profile_id) - if recipient_profile is None: - raise Exception("Recipient profile with id {} not found.".format( - recipient_profile_id, - )) - squeak, secret_key = self.squeak_core.make_squeak( - squeak_profile, + return self.squeak_store.make_squeak( + profile_id, content_str, replyto_hash, - recipient_profile=recipient_profile if recipient_profile_id else None, + recipient_profile_id, ) - inserted_squeak_hash = self.save_squeak(squeak) - if inserted_squeak_hash is None: - raise Exception("Failed to save squeak.") - self.squeak_store.save_secret_key(inserted_squeak_hash, secret_key) - if squeak.is_private_message: - self.unlock_squeak( - inserted_squeak_hash, - author_profile_id=profile_id, - ) - else: - self.unlock_squeak( - inserted_squeak_hash, - ) - return inserted_squeak_hash - - def save_secret_key(self, squeak_hash: bytes, secret_key: bytes): - squeak = self.squeak_store.get_squeak(squeak_hash) - if squeak is None: - raise Exception("Squeakdoes not exist.") - self.squeak_store.save_secret_key(squeak_hash, secret_key) - if not squeak.is_private_message: - self.unlock_squeak(squeak_hash) - - def unlock_squeak( - self, - squeak_hash: bytes, - author_profile_id: Optional[int] = None, - recipient_profile_id: Optional[int] = None, - ): - squeak = self.squeak_store.get_squeak(squeak_hash) - secret_key = self.squeak_store.get_squeak_secret_key(squeak_hash) - if squeak is None: - raise Exception("Squeakdoes not exist.") - if secret_key is None: - raise Exception("Secret key does not exist.") - if recipient_profile_id: - recipient_profile = self.squeak_store.get_squeak_profile( - recipient_profile_id) - if recipient_profile is None: - raise Exception("Recipient profile does not exist.") - decrypted_content = self.squeak_core.get_decrypted_content( - squeak, - secret_key, - recipient_profile=recipient_profile, - ) - elif author_profile_id: - author_profile = self.squeak_store.get_squeak_profile( - author_profile_id) - if author_profile is None: - raise Exception("Author profile does not exist.") - decrypted_content = self.squeak_core.get_decrypted_content( - squeak, - secret_key, - author_profile=author_profile, - ) - else: - decrypted_content = self.squeak_core.get_decrypted_content( - squeak, - secret_key, - ) - self.squeak_store.set_decrypted_content( - squeak_hash, - decrypted_content, - ) - logger.info("Unlocked squeak content: {}".format( - squeak_hash.hex(), - )) def pay_offer(self, received_offer_id: int) -> int: - received_offer = self.squeak_store.get_received_offer( - received_offer_id, - ) - if received_offer is None: - raise Exception("Received offer with id {} not found.".format( - received_offer_id, - )) - squeak = self.squeak_store.get_squeak(received_offer.squeak_hash) - if squeak is None: - raise Exception("Squeak for received offer with id {} not found.".format( - received_offer_id, - )) - logger.info("Paying received offer: {}".format(received_offer)) - sent_payment = self.squeak_core.pay_offer(received_offer) - sent_payment_id = self.squeak_store.save_sent_payment(sent_payment) - self.squeak_store.mark_received_offer_paid( - sent_payment.payment_hash, - ) - self.squeak_store.save_secret_key( - received_offer.squeak_hash, - sent_payment.secret_key, - ) - if not squeak.is_private_message: - self.unlock_squeak(received_offer.squeak_hash) - return sent_payment_id + return self.squeak_store.pay_offer(received_offer_id) def get_packaged_offer( self, @@ -212,53 +103,24 @@ class SqueakController: price_msat = self.get_sell_price_msat() if price_msat == 0: return None - # return self.squeak_store.get_packaged_offer( - # squeak_hash, - # peer_address, - # price_msat, - # lnd_external_address, - # ) - sent_offer = self.get_sent_offer_for_peer( + return self.squeak_store.get_packaged_offer( squeak_hash, peer_address, price_msat, - ) - if sent_offer is None: - return None - return self.squeak_core.package_offer( - sent_offer, lnd_external_address, ) - def get_sent_offer_for_peer( + def decrypt_private_squeak( self, squeak_hash: bytes, - peer_address: PeerAddress, - price_msat: int, - ) -> Optional[SentOffer]: - # Check if there is an existing offer for the hash/peer_address combination - sent_offer = self.squeak_store.get_sent_offer_by_squeak_hash_and_peer( + author_profile_id: Optional[int], + recipient_profile_id: Optional[int], + ): + self.squeak_store.unlock_squeak( squeak_hash, - peer_address, + author_profile_id=author_profile_id, + recipient_profile_id=recipient_profile_id, ) - if sent_offer: - return sent_offer - squeak = self.squeak_store.get_squeak(squeak_hash) - secret_key = self.squeak_store.get_squeak_secret_key(squeak_hash) - if squeak is None or secret_key is None: - return None - try: - sent_offer = self.squeak_core.create_offer( - squeak, - secret_key, - peer_address, - price_msat, - ) - except Exception: - logger.exception("Failed to create offer.") - return None - self.squeak_store.save_sent_offer(sent_offer) - return sent_offer def get_squeak(self, squeak_hash: bytes) -> Optional[CSqueak]: return self.squeak_store.get_squeak(squeak_hash) @@ -314,9 +176,6 @@ class SqueakController: def get_squeak_profile_private_key(self, profile_id: int) -> bytes: return self.squeak_store.get_squeak_profile_private_key(profile_id) - def get_followed_public_keys(self) -> List[SqueakPublicKey]: - return self.squeak_store.get_followed_public_keys() - def create_peer(self, peer_name: str, peer_address: PeerAddress): return self.squeak_store.create_peer(peer_name, peer_address) @@ -344,14 +203,6 @@ class SqueakController: def delete_peer(self, peer_id: int): return self.squeak_store.delete_peer(peer_id) - def handle_offer(self, squeak: CSqueak, offer: Offer, peer_address: PeerAddress): - received_offer = self.squeak_core.unpack_offer( - squeak, - offer, - peer_address, - ) - self.squeak_store.save_received_offer(received_offer) - def get_received_offers(self, squeak_hash: bytes) -> List[ReceivedOffer]: return self.squeak_store.get_received_offers(squeak_hash) @@ -402,12 +253,7 @@ class SqueakController: return self.squeak_store.get_squeak_entry(squeak_hash) def download_single_squeak(self, squeak_hash: bytes) -> DownloadResult: - network_controller = NetworkController( - self, - self.config.tor.proxy_ip, - self.config.tor.proxy_port, - ) - network_controller.download_single_squeak(squeak_hash) + self.network_controller.download_single_squeak(squeak_hash) return DownloadResult(1, 1, 0, 9999) def get_timeline_squeak_entries( @@ -450,12 +296,7 @@ class SqueakController: ) -> List[SqueakEntry]: # TODO: remove this temporary hack, after converting this to websockets. logger.info('Start downloading pubkey squeaks...') - network_controller = NetworkController( - self, - self.config.tor.proxy_ip, - self.config.tor.proxy_port, - ) - network_controller.download_pubkey_squeaks_async(public_key) + self.network_controller.download_pubkey_squeaks_async(public_key) logger.info('Finished downloading pubkey squeaks.') return self.squeak_store.get_squeak_entries_for_public_key( public_key, @@ -566,7 +407,7 @@ class SqueakController: ) def get_default_peer_port(self) -> int: - return 0 # TODO: maybe remove this method. + return 0 # TODO: remove default port method. def set_sell_price_msat(self, sell_price_msat: int) -> None: self.node_settings.set_sell_price_msat(sell_price_msat) @@ -589,7 +430,7 @@ class SqueakController: profile_id, bearer_token, ) - # self.update_twitter_stream() + self.update_twitter_stream() return twitter_account_id def get_twitter_accounts(self) -> List[TwitterAccountEntry]: @@ -597,10 +438,7 @@ class SqueakController: def delete_twitter_account(self, twitter_account_id: int) -> None: self.squeak_store.delete_twitter_account(twitter_account_id) - # self.update_twitter_stream() + self.update_twitter_stream() - # def update_twitter_stream(self) -> None: - # self.tweet_forwarder.start_processing() - - def get_latest_block(self) -> int: - return self.squeak_core.get_best_block_height() + def update_twitter_stream(self) -> None: + self.tweet_forwarder.start_processing() diff --git a/squeaknode/node/squeak_node.py b/squeaknode/node/squeak_node.py index 5b808814..5a62b3fc 100644 --- a/squeaknode/node/squeak_node.py +++ b/squeaknode/node/squeak_node.py @@ -63,9 +63,9 @@ class SqueakNode: self.create_squeak_core() self.create_squeak_store() self.create_payment_processor() - self.create_squeak_controller() - self.create_network_controller() self.create_twitter_forwarder() + self.create_network_controller() + self.create_squeak_controller() self.create_peer_handler() self.create_peer_web_server() @@ -144,6 +144,7 @@ class SqueakNode: def create_squeak_store(self): self.squeak_store = SqueakStore( self.squeak_db, + self.squeak_core, self.config.node.max_squeaks, self.config.node.max_squeaks_per_public_key_per_block, self.config.node.squeak_retention_s, @@ -160,27 +161,28 @@ class SqueakNode: def create_twitter_forwarder(self): self.twitter_forwarder = TwitterForwarder( - self.squeak_controller, + self.squeak_store, self.squeak_core, self.config.twitter.forward_tweets_retry_s, ) + def create_network_controller(self): + self.network_controller = NetworkController( + self.squeak_store, + self.config.tor.proxy_ip, + self.config.tor.proxy_port, + ) + def create_squeak_controller(self): self.squeak_controller = SqueakController( self.squeak_store, - self.squeak_core, self.payment_processor, + self.twitter_forwarder, + self.network_controller, self.node_settings, self.config, ) - def create_network_controller(self): - self.network_controller = NetworkController( - self.squeak_controller, - self.config.tor.proxy_ip, - self.config.tor.proxy_port, - ) - def create_admin_handler(self): self.admin_handler = SqueakAdminServerHandler( self.lightning_client, diff --git a/squeaknode/node/squeak_store.py b/squeaknode/node/squeak_store.py index 2432ded7..d953a9b3 100644 --- a/squeaknode/node/squeak_store.py +++ b/squeaknode/node/squeak_store.py @@ -25,13 +25,14 @@ from typing import Iterator from typing import List from typing import Optional -from bitcoin.core import CBlockHeader from squeak.core import CheckSqueak from squeak.core import CheckSqueakSecretKey from squeak.core import CSqueak from squeak.core.keys import SqueakPrivateKey from squeak.core.keys import SqueakPublicKey +from squeaknode.core.lightning_address import LightningAddressHostPort +from squeaknode.core.offer import Offer from squeaknode.core.peer_address import PeerAddress from squeaknode.core.peers import create_saved_peer from squeaknode.core.profiles import create_contact_profile @@ -43,6 +44,7 @@ from squeaknode.core.received_payment_summary import ReceivedPaymentSummary from squeaknode.core.sent_offer import SentOffer from squeaknode.core.sent_payment import SentPayment from squeaknode.core.sent_payment_summary import SentPaymentSummary +from squeaknode.core.squeak_core import SqueakCore from squeaknode.core.squeak_entry import SqueakEntry from squeaknode.core.squeak_peer import SqueakPeer from squeaknode.core.squeak_profile import SqueakProfile @@ -61,6 +63,7 @@ class SqueakStore: def __init__( self, squeak_db: SqueakDb, + squeak_core: SqueakCore, max_squeaks, max_squeaks_per_public_key_per_block, squeak_retention_s, @@ -68,6 +71,7 @@ class SqueakStore: sent_offer_retention_s, ): self.squeak_db = squeak_db + self.squeak_core = squeak_core self.max_squeaks = max_squeaks self.max_squeaks_per_public_key_per_block = max_squeaks_per_public_key_per_block self.squeak_retention_s = squeak_retention_s @@ -79,12 +83,51 @@ class SqueakStore: self.new_follow_listener = EventListener() self.twitter_stream_change_listener = EventListener() - def save_squeak(self, squeak: CSqueak, block_header: CBlockHeader) -> Optional[bytes]: + def make_squeak( + self, + profile_id: int, + content_str: str, + replyto_hash: Optional[bytes], + recipient_profile_id: Optional[int], + ) -> Optional[bytes]: + squeak_profile = self.get_squeak_profile(profile_id) + if squeak_profile is None: + raise Exception("Profile with id {} not found.".format( + profile_id, + )) + if recipient_profile_id: + recipient_profile = self.get_squeak_profile( + recipient_profile_id) + if recipient_profile is None: + raise Exception("Recipient profile with id {} not found.".format( + recipient_profile_id, + )) + squeak, secret_key = self.squeak_core.make_squeak( + squeak_profile, + content_str, + replyto_hash, + recipient_profile=recipient_profile if recipient_profile_id else None, + ) + inserted_squeak_hash = self.save_squeak(squeak) + if inserted_squeak_hash is None: + raise Exception("Failed to save squeak.") + self.save_secret_key(inserted_squeak_hash, secret_key) + if squeak.is_private_message: + self.unlock_squeak( + inserted_squeak_hash, + author_profile_id=profile_id, + ) + return inserted_squeak_hash + + def save_squeak(self, squeak: CSqueak) -> Optional[bytes]: # Check if the squeak is valid context free. CheckSqueak(squeak) + # Get the block header. + block_header = self.squeak_core.get_block_header(squeak) + # Check if limit exceeded. if self.squeak_db.get_number_of_squeaks() >= self.max_squeaks: raise Exception("Exceeded max number of squeaks.") - # Check if limit per public key per block is exceeded. + # TODO: Check if limit per public key per block is exceeded. if self.squeak_db.number_of_squeaks_with_public_key_with_block_height( squeak.GetPubKey(), squeak.nBlockHeight, @@ -117,12 +160,74 @@ class SqueakStore: squeak_hash.hex(), )) self.new_secret_key_listener.handle_new_item(squeak) + # Unlock the squeak if it is not private. + if not squeak.is_private_message: + self.unlock_squeak(squeak_hash) - def set_decrypted_content(self, squeak_hash: bytes, content: str): + def unlock_squeak( + self, + squeak_hash: bytes, + author_profile_id: Optional[int] = None, + recipient_profile_id: Optional[int] = None, + ): + squeak = self.squeak_db.get_squeak(squeak_hash) + secret_key = self.squeak_db.get_squeak_secret_key(squeak_hash) + if squeak is None: + raise Exception("Squeakdoes not exist.") + if secret_key is None: + raise Exception("Secret key does not exist.") + if recipient_profile_id: + recipient_profile = self.squeak_db.get_profile( + recipient_profile_id) + if recipient_profile is None: + raise Exception("Recipient profile does not exist.") + decrypted_content = self.squeak_core.get_decrypted_content( + squeak, + secret_key, + recipient_profile=recipient_profile, + ) + elif author_profile_id: + author_profile = self.squeak_db.get_profile( + author_profile_id) + if author_profile is None: + raise Exception("Author profile does not exist.") + decrypted_content = self.squeak_core.get_decrypted_content( + squeak, + secret_key, + author_profile=author_profile, + ) + else: + decrypted_content = self.squeak_core.get_decrypted_content( + squeak, + secret_key, + ) self.squeak_db.set_squeak_decrypted_content( squeak_hash, - content, + decrypted_content, ) + logger.info("Unlocked squeak content: {}".format( + squeak_hash.hex(), + )) + + def pay_offer(self, received_offer_id: int) -> int: + received_offer = self.get_received_offer( + received_offer_id, + ) + if received_offer is None: + raise Exception("Received offer with id {} not found.".format( + received_offer_id, + )) + logger.info("Paying received offer: {}".format(received_offer)) + sent_payment = self.squeak_core.pay_offer(received_offer) + sent_payment_id = self.save_sent_payment(sent_payment) + self.mark_received_offer_paid( + sent_payment.payment_hash, + ) + self.save_secret_key( + received_offer.squeak_hash, + sent_payment.secret_key, + ) + return sent_payment_id def get_squeak(self, squeak_hash: bytes) -> Optional[CSqueak]: return self.squeak_db.get_squeak(squeak_hash) @@ -142,6 +247,56 @@ class SqueakStore: def save_sent_offer(self, sent_offer: SentOffer) -> int: return self.squeak_db.insert_sent_offer(sent_offer) + def get_sent_offer_for_peer( + self, + squeak_hash: bytes, + peer_address: PeerAddress, + price_msat: int, + ) -> Optional[SentOffer]: + # Check if there is an existing offer for the hash/peer_address combination + sent_offer = self.get_sent_offer_by_squeak_hash_and_peer( + squeak_hash, + peer_address, + ) + if sent_offer: + return sent_offer + squeak = self.get_squeak(squeak_hash) + secret_key = self.get_squeak_secret_key(squeak_hash) + if squeak is None or secret_key is None: + return None + try: + sent_offer = self.squeak_core.create_offer( + squeak, + secret_key, + peer_address, + price_msat, + ) + except Exception: + logger.exception("Failed to create offer.") + return None + self.save_sent_offer(sent_offer) + return sent_offer + + # TODO: remove this method. Do this logic in squeakcontroller. + def get_packaged_offer( + self, + squeak_hash: bytes, + peer_address: PeerAddress, + price_msat: int, + lnd_external_address: Optional[LightningAddressHostPort], + ) -> Optional[Offer]: + sent_offer = self.get_sent_offer_for_peer( + squeak_hash, + peer_address, + price_msat, + ) + if sent_offer is None: + return None + return self.squeak_core.package_offer( + sent_offer, + lnd_external_address, + ) + def create_signing_profile(self, profile_name: str) -> int: squeak_profile = create_signing_profile( profile_name, @@ -380,6 +535,14 @@ class SqueakStore: self.new_received_offer_listener.handle_new_item(received_offer) return received_offer_id + def handle_offer(self, squeak: CSqueak, offer: Offer, peer_address: PeerAddress): + received_offer = self.squeak_core.unpack_offer( + squeak, + offer, + peer_address, + ) + self.save_received_offer(received_offer) + def get_followed_public_keys(self) -> List[SqueakPublicKey]: followed_profiles = self.squeak_db.get_following_profiles() return [profile.public_key for profile in followed_profiles] @@ -481,3 +644,6 @@ class SqueakStore: def delete_twitter_account(self, twitter_account_id: int) -> None: self.squeak_db.delete_twitter_account(twitter_account_id) + + def get_latest_block(self) -> int: + return self.squeak_core.get_best_block_height() diff --git a/squeaknode/twitter/twitter_forwarder.py b/squeaknode/twitter/twitter_forwarder.py index 934b8281..cc10ceca 100644 --- a/squeaknode/twitter/twitter_forwarder.py +++ b/squeaknode/twitter/twitter_forwarder.py @@ -25,7 +25,7 @@ from typing import Dict from squeaknode.core.squeak_core import SqueakCore from squeaknode.core.twitter_account_entry import TwitterAccountEntry -from squeaknode.node.squeak_controller import SqueakController +from squeaknode.node.squeak_store import SqueakStore from squeaknode.twitter.twitter_stream import TwitterStream @@ -36,11 +36,11 @@ class TwitterForwarder: def __init__( self, - squeak_controller: SqueakController, + squeak_store: SqueakStore, squeak_core: SqueakCore, retry_s: int, ): - self.squeak_controller = squeak_controller + self.squeak_store = squeak_store self.squeak_core = squeak_core self.retry_s = retry_s self.lock = threading.Lock() @@ -54,9 +54,9 @@ class TwitterForwarder: del self.current_tasks[handle] # Start new tasks. - for account in self.squeak_controller.get_twitter_accounts(): + for account in self.squeak_store.get_twitter_accounts(): task = TwitterForwarderTask( - self.squeak_controller, + self.squeak_store, self.squeak_core, account, self.retry_s, @@ -83,12 +83,12 @@ class TwitterForwarderTask: def __init__( self, - squeak_controller: SqueakController, + squeak_store: SqueakStore, squeak_core: SqueakCore, twitter_account: TwitterAccountEntry, retry_s: int, ): - self.squeak_controller = squeak_controller + self.squeak_store = squeak_store self.squeak_core = squeak_core self.twitter_account = twitter_account self.retry_s = retry_s @@ -166,7 +166,7 @@ class TwitterForwarderTask: self.forward_tweet(tweet) def make_squeak(self, profile_id: int, content_str: str): - squeak_profile = self.squeak_controller.get_squeak_profile(profile_id) + squeak_profile = self.squeak_store.get_squeak_profile(profile_id) if squeak_profile is None: raise Exception("Profile with id {} not found.".format( profile_id, @@ -175,10 +175,9 @@ class TwitterForwarderTask: squeak_profile, content_str, ) - inserted_squeak_hash = self.squeak_controller.save_squeak( + inserted_squeak_hash = self.squeak_store.save_squeak( squeak, ) if inserted_squeak_hash is None: return None - self.squeak_controller.save_secret_key( - inserted_squeak_hash, secret_key) + self.squeak_store.save_secret_key(inserted_squeak_hash, secret_key) diff --git a/tests/node/test_squeak_controller.py b/tests/node/test_squeak_controller.py index a50e3882..2715e215 100644 --- a/tests/node/test_squeak_controller.py +++ b/tests/node/test_squeak_controller.py @@ -22,15 +22,16 @@ import mock import pytest +from squeaknode.client.network_controller import NetworkController from squeaknode.config.config import SqueaknodeConfig from squeaknode.core.lightning_address import LightningAddressHostPort from squeaknode.core.peer_address import Network from squeaknode.core.peer_address import PeerAddress -from squeaknode.core.squeak_core import SqueakCore from squeaknode.node.node_settings import NodeSettings from squeaknode.node.payment_processor import PaymentProcessor from squeaknode.node.squeak_controller import SqueakController from squeaknode.node.squeak_store import SqueakStore +from squeaknode.twitter.twitter_forwarder import TwitterForwarder @pytest.fixture @@ -59,11 +60,6 @@ def node_settings(): return mock.Mock(spec=NodeSettings) -@pytest.fixture -def squeak_core(): - return mock.Mock(spec=SqueakCore) - - @pytest.fixture def lightning_host_port(): return LightningAddressHostPort(host="my_lightning_host", port=8765) @@ -97,18 +93,30 @@ def payment_processor(): return mock.Mock(spec=PaymentProcessor) +@pytest.fixture +def twitter_forwarder(): + return mock.Mock(spec=TwitterForwarder) + + +@pytest.fixture +def network_controller(): + return mock.Mock(spec=NetworkController) + + @pytest.fixture def squeak_controller( squeak_store, - squeak_core, payment_processor, + twitter_forwarder, + network_controller, node_settings, config, ): return SqueakController( squeak_store, - squeak_core, payment_processor, + twitter_forwarder, + network_controller, node_settings, config, ) @@ -117,15 +125,17 @@ def squeak_controller( @pytest.fixture def regtest_squeak_controller( squeak_store, - squeak_core, payment_processor, + twitter_forwarder, + network_controller, node_settings, regtest_config, ): return SqueakController( squeak_store, - squeak_core, payment_processor, + twitter_forwarder, + network_controller, node_settings, regtest_config, ) diff --git a/tests/node/test_squeak_store.py b/tests/node/test_squeak_store.py index d7ba905a..9250296a 100644 --- a/tests/node/test_squeak_store.py +++ b/tests/node/test_squeak_store.py @@ -23,6 +23,7 @@ import mock import pytest from squeaknode.core.lightning_address import LightningAddressHostPort +from squeaknode.core.squeak_core import SqueakCore from squeaknode.db.squeak_db import SqueakDb from squeaknode.node.squeak_store import SqueakStore @@ -32,6 +33,11 @@ def squeak_db(): return mock.Mock(spec=SqueakDb) +@pytest.fixture +def squeak_core(): + return mock.Mock(spec=SqueakCore) + + @pytest.fixture def lightning_host_port(): return LightningAddressHostPort(host="my_lightning_host", port=8765) @@ -75,6 +81,7 @@ def inserted_signing_profile_id(squeak_db, signing_profile): @pytest.fixture def squeak_store( squeak_db, + squeak_core, max_squeaks, max_squeaks_per_public_key_per_block, squeak_retention_s, @@ -83,6 +90,7 @@ def squeak_store( ): return SqueakStore( squeak_db, + squeak_core, max_squeaks, max_squeaks_per_public_key_per_block, squeak_retention_s, @@ -91,27 +99,31 @@ def squeak_store( ) -def test_save_squeak(squeak_store, squeak_db, block_header, squeak, squeak_hash): +def test_save_squeak(squeak_store, squeak_db, squeak_core, block_header, squeak, squeak_hash): with mock.patch.object(squeak_db, 'get_number_of_squeaks', autospec=True) as mock_get_number_of_squeaks, \ mock.patch.object(squeak_db, 'number_of_squeaks_with_public_key_with_block_height', autospec=True) as mock_number_of_squeaks_with_public_key_with_block_height, \ mock.patch.object(squeak_db, 'insert_squeak', autospec=True) as mock_insert_squeak, \ - mock.patch.object(squeak_store.new_squeak_listener, 'handle_new_item', autospec=True) as mock_handle_new_squeak: + mock.patch.object(squeak_store.new_squeak_listener, 'handle_new_item', autospec=True) as mock_handle_new_squeak, \ + mock.patch.object(squeak_core, 'get_block_header', autospec=True) as mock_get_block_header: mock_get_number_of_squeaks.return_value = 0 mock_number_of_squeaks_with_public_key_with_block_height.return_value = 0 + mock_get_block_header.return_value = block_header mock_insert_squeak.return_value = squeak_hash - squeak_store.save_squeak(squeak, block_header) + squeak_store.save_squeak(squeak) mock_insert_squeak.assert_called_once_with(squeak, block_header) mock_handle_new_squeak.assert_called_once_with(squeak) -def test_save_squeak_above_max(squeak_store, squeak_db, block_header, squeak, squeak_hash, max_squeaks): +def test_save_squeak_above_max(squeak_store, squeak_db, squeak_core, block_header, squeak, squeak_hash, max_squeaks): with mock.patch.object(squeak_db, 'get_number_of_squeaks', autospec=True) as mock_get_number_of_squeaks, \ mock.patch.object(squeak_db, 'number_of_squeaks_with_public_key_with_block_height', autospec=True) as mock_number_of_squeaks_with_public_key_with_block_height, \ mock.patch.object(squeak_db, 'insert_squeak', autospec=True) as mock_insert_squeak, \ - mock.patch.object(squeak_store.new_squeak_listener, 'handle_new_item', autospec=True) as mock_handle_new_squeak: + mock.patch.object(squeak_store.new_squeak_listener, 'handle_new_item', autospec=True) as mock_handle_new_squeak, \ + mock.patch.object(squeak_core, 'get_block_header', autospec=True) as mock_get_block_header: mock_get_number_of_squeaks.return_value = max_squeaks + 1 mock_number_of_squeaks_with_public_key_with_block_height.return_value = 0 + mock_get_block_header.return_value = block_header mock_insert_squeak.return_value = squeak_hash with pytest.raises(Exception): @@ -121,13 +133,15 @@ def test_save_squeak_above_max(squeak_store, squeak_db, block_header, squeak, sq assert mock_handle_new_squeak.call_count == 0 -def test_save_squeak_above_max_per_pubkey(squeak_store, squeak_db, block_header, squeak, squeak_hash, max_squeaks_per_public_key_per_block): +def test_save_squeak_above_max_per_pubkey(squeak_store, squeak_db, squeak_core, block_header, squeak, squeak_hash, max_squeaks_per_public_key_per_block): with mock.patch.object(squeak_db, 'get_number_of_squeaks', autospec=True) as mock_get_number_of_squeaks, \ mock.patch.object(squeak_db, 'number_of_squeaks_with_public_key_with_block_height', autospec=True) as mock_number_of_squeaks_with_public_key_with_block_height, \ mock.patch.object(squeak_db, 'insert_squeak', autospec=True) as mock_insert_squeak, \ - mock.patch.object(squeak_store.new_squeak_listener, 'handle_new_item', autospec=True) as mock_handle_new_squeak: + mock.patch.object(squeak_store.new_squeak_listener, 'handle_new_item', autospec=True) as mock_handle_new_squeak, \ + mock.patch.object(squeak_core, 'get_block_header', autospec=True) as mock_get_block_header: mock_get_number_of_squeaks.return_value = 0 mock_number_of_squeaks_with_public_key_with_block_height.return_value = max_squeaks_per_public_key_per_block + 1 + mock_get_block_header.return_value = block_header mock_insert_squeak.return_value = squeak_hash with pytest.raises(Exception): @@ -137,9 +151,10 @@ def test_save_squeak_above_max_per_pubkey(squeak_store, squeak_db, block_header, assert mock_handle_new_squeak.call_count == 0 -def test_save_secret_key(squeak_store, squeak_db, squeak, squeak_hash, secret_key): +def test_save_secret_key(squeak_store, squeak_db, squeak_core, squeak, squeak_hash, secret_key): with mock.patch.object(squeak_db, 'get_squeak', autospec=True) as mock_get_squeak, \ mock.patch.object(squeak_db, 'set_squeak_secret_key', autospec=True) as mock_set_squeak_secret_key, \ + mock.patch.object(squeak_store, 'unlock_squeak', autospec=True) as mock_unlock_squeak, \ mock.patch.object(squeak_store.new_secret_key_listener, 'handle_new_item', autospec=True) as mock_handle_new_secret_key: mock_get_squeak.return_value = squeak squeak_store.save_secret_key(squeak_hash, secret_key) @@ -147,6 +162,21 @@ def test_save_secret_key(squeak_store, squeak_db, squeak, squeak_hash, secret_ke mock_set_squeak_secret_key.assert_called_once_with( squeak_hash, secret_key) mock_handle_new_secret_key.assert_called_once_with(squeak) + mock_unlock_squeak.assert_called_once_with(squeak_hash) + + +def test_unlock_squeak(squeak_store, squeak_db, squeak_core, squeak, squeak_hash, secret_key, squeak_content): + with mock.patch.object(squeak_db, 'get_squeak', autospec=True) as mock_get_squeak, \ + mock.patch.object(squeak_db, 'get_squeak_secret_key', autospec=True) as mock_get_squeak_secret_key, \ + mock.patch.object(squeak_db, 'set_squeak_decrypted_content', autospec=True) as mock_set_squeak_decrypted_content, \ + mock.patch.object(squeak_core, 'get_decrypted_content', autospec=True) as mock_get_decrypted_content: + mock_get_squeak.return_value = squeak + mock_get_squeak_secret_key.return_value = secret_key + mock_get_decrypted_content.return_value = squeak_content + squeak_store.unlock_squeak(squeak_hash) + + mock_set_squeak_decrypted_content.assert_called_once_with( + squeak_hash, squeak_content) # @pytest.fixture @@ -215,3 +245,91 @@ def test_get_received_offer(squeak_store, squeak_db, received_offer): assert retrieved_received_offer == received_offer mock_get_received_offer.assert_called_once_with(789) + + +# def test_get_free_secret_key(squeak_store, squeak_core, unlocked_squeak, secret_key, peer_address): +# unlocked_squeak_hash = get_hash(unlocked_squeak) +# secret_key_reply = squeak_store.get_secret_key_reply( +# unlocked_squeak_hash, peer_address, 0, None) + +# assert secret_key_reply.squeak_hash == unlocked_squeak_hash +# assert secret_key_reply.secret_key == secret_key + + +# def test_get_offer_secret_key(squeak_store, squeak_core, unlocked_squeak, secret_key, peer_address, sent_offer, offer): +# with mock.patch.object(squeak_core, 'create_offer', autospec=True) as mock_create_offer, \ +# mock.patch.object(squeak_core, 'package_offer', autospec=True) as mock_package_offer: +# mock_create_offer.return_value = sent_offer +# mock_package_offer.return_value = offer +# unlocked_squeak_hash = get_hash(unlocked_squeak) +# secret_key_reply = squeak_store.get_secret_key_reply( +# unlocked_squeak_hash, peer_address, 1000, None) + +# assert secret_key_reply.squeak_hash == unlocked_squeak_hash +# assert secret_key_reply.offer == offer + + +# def test_pay_offer( +# squeak_store, +# squeak_db, +# squeak_core, +# unlocked_squeak, +# block_header, +# squeak_content, +# secret_key, +# peer_address, +# inserted_received_offer_id, +# sent_payment, +# ): +# with mock.patch.object(squeak_core, 'pay_offer', autospec=True) as mock_pay_offer, \ +# mock.patch.object(squeak_core, 'get_block_header', autospec=True) as mock_get_block_header, \ +# mock.patch.object(squeak_core, 'get_decrypted_content', autospec=True) as mock_get_decrypted_content: +# mock_pay_offer.return_value = sent_payment +# mock_get_block_header.return_value = block_header +# mock_get_decrypted_content.return_value = squeak_content +# sent_payment_id = squeak_store.pay_offer(inserted_received_offer_id) + +# retrieved_sent_payment = squeak_db.get_sent_payment( +# sent_payment_id, +# ) + +# assert sent_payment_id is not None +# assert retrieved_sent_payment is not None + + +# def test_save_received_offer_already_unlocked( +# squeak_store, +# unlocked_squeak, +# offer, +# peer_address, +# ): +# received_offer_id = squeak_store.save_received_offer( +# offer, +# peer_address, +# ) + +# assert received_offer_id is None + + +# def test_save_received_offer( +# squeak_store, +# squeak_db, +# squeak_core, +# saved_squeak, +# offer, +# received_offer, +# peer_address, +# ): +# with mock.patch.object(squeak_core, 'unpack_offer', autospec=True) as mock_unpack_offer: +# mock_unpack_offer.return_value = received_offer +# received_offer_id = squeak_store.save_received_offer( +# offer, +# peer_address, +# ) + +# assert received_offer_id is not None +# retrieved_received_offer = squeak_db.get_received_offer(received_offer_id) + +# assert retrieved_received_offer == received_offer._replace( +# received_offer_id=received_offer_id, +# )