From 38a7a1af877ec9c7aa3a048ddcb84dc3650d8693 Mon Sep 17 00:00:00 2001 From: Jonathan Zernik Date: Tue, 14 Sep 2021 20:47:11 -0700 Subject: [PATCH] Got itest working for text search (#1287) * Got itest working for text search * Use util function to get search results in itest --- itests/tests/test_squeak_node.py | 22 +++++++ itests/tests/util.py | 10 ++++ proto/squeak_admin.proto | 20 +++++++ .../admin/squeak_admin_server_handler.py | 32 ++++++++++ .../admin/squeak_admin_server_servicer.py | 3 + squeaknode/admin/webapp/app.py | 8 +++ squeaknode/db/squeak_db.py | 59 ++++++++++++++++++- squeaknode/node/squeak_controller.py | 12 ++++ 8 files changed, 165 insertions(+), 1 deletion(-) diff --git a/itests/tests/test_squeak_node.py b/itests/tests/test_squeak_node.py index a6625995..5d52615a 100644 --- a/itests/tests/test_squeak_node.py +++ b/itests/tests/test_squeak_node.py @@ -45,6 +45,7 @@ from tests.util import get_connected_peers from tests.util import get_external_address from tests.util import get_hash from tests.util import get_network +from tests.util import get_search_squeaks from tests.util import get_squeak_display from tests.util import get_squeak_profile from tests.util import import_signing_profile @@ -1060,3 +1061,24 @@ def test_subscribe_squeaks( assert ( get_squeak_display_entry.content_str == make_squeak_content ) + + +def test_search(admin_stub, signing_profile_id): + make_squeak_content = "Just some random weird text." + make_squeak_hash = make_squeak( + admin_stub, signing_profile_id, make_squeak_content) + assert len(make_squeak_hash) == 32 * 2 + + # Get all squeak displays for the given search text. + search_results = get_search_squeaks( + admin_stub, + "Weird", + ) + assert len(search_results) == 1 + + # Get all squeak displays for other search text that shouldn't be there. + missing_search_results = get_search_squeaks( + admin_stub, + "strange", + ) + assert len(missing_search_results) == 0 diff --git a/itests/tests/util.py b/itests/tests/util.py index 8e5f2c12..d1d49240 100644 --- a/itests/tests/util.py +++ b/itests/tests/util.py @@ -205,6 +205,16 @@ def create_saved_peer(node_stub, name, host, port): return create_peer_response.peer_id +def get_search_squeaks(node_stub, search_text): + get_search_squeak_display_response = node_stub.GetSearchSqueakDisplays( + squeak_admin_pb2.GetSearchSqueakDisplaysRequest( + search_text=search_text, + limit=100, + ), + ) + return get_search_squeak_display_response.squeak_display_entries + + @contextmanager def subscribe_connected_peers(node_stub): q = queue.Queue() diff --git a/proto/squeak_admin.proto b/proto/squeak_admin.proto index 0906f8a8..9f40571b 100644 --- a/proto/squeak_admin.proto +++ b/proto/squeak_admin.proto @@ -148,6 +148,10 @@ service SqueakAdmin { */ rpc GetAddressSqueakDisplays (GetAddressSqueakDisplaysRequest) returns (GetAddressSqueakDisplaysReply) {} + /** sqkadmin: `getsearchsqueakdisplays` + */ + rpc GetSearchSqueakDisplays (GetSearchSqueakDisplaysRequest) returns (GetSearchSqueakDisplaysReply) {} + /** sqkadmin: `getancestorsqueakdisplays` */ rpc GetAncestorSqueakDisplays (GetAncestorSqueakDisplaysRequest) returns (GetAncestorSqueakDisplaysReply) {} @@ -618,6 +622,22 @@ message GetAddressSqueakDisplaysReply { repeated SqueakDisplayEntry squeak_display_entries = 1; } +message GetSearchSqueakDisplaysRequest { + /// The search text + string search_text = 1; + + /// Limit number of results + int32 limit = 2; + + /// Last entry + SqueakDisplayEntry last_entry = 3; +} + +message GetSearchSqueakDisplaysReply { + /// Multiple squeak display entries + repeated SqueakDisplayEntry squeak_display_entries = 1; +} + message GetAncestorSqueakDisplaysRequest { /// Hash of the squeak. string squeak_hash = 1; diff --git a/squeaknode/admin/squeak_admin_server_handler.py b/squeaknode/admin/squeak_admin_server_handler.py index 36d6adc9..2b171cac 100644 --- a/squeaknode/admin/squeak_admin_server_handler.py +++ b/squeaknode/admin/squeak_admin_server_handler.py @@ -401,6 +401,38 @@ class SqueakAdminServerHandler(object): squeak_display_entries=squeak_display_msgs ) + def handle_get_squeak_display_entries_for_text_search(self, request): + search_text = request.search_text + limit = request.limit + last_entry = message_to_squeak_entry(request.last_entry) if request.HasField( + "last_entry") else None + logger.info("""Handle get squeak display entries for search_text: {} with + limit: {} + last_entry: {} + """.format( + search_text, + limit, + last_entry, + )) + squeak_entries = ( + self.squeak_controller.get_squeak_entries_for_text_search( + search_text, + limit, + last_entry, + ) + ) + logger.info( + "Got number of squeak entries for text search: {}".format( + len(squeak_entries) + ) + ) + squeak_display_msgs = [ + squeak_entry_to_message(entry) for entry in squeak_entries + ] + return squeak_admin_pb2.GetAddressSqueakDisplaysReply( + squeak_display_entries=squeak_display_msgs + ) + def handle_get_ancestor_squeak_display_entries(self, request): squeak_hash_str = request.squeak_hash squeak_hash = bytes.fromhex(squeak_hash_str) diff --git a/squeaknode/admin/squeak_admin_server_servicer.py b/squeaknode/admin/squeak_admin_server_servicer.py index 5554b078..34368b8b 100644 --- a/squeaknode/admin/squeak_admin_server_servicer.py +++ b/squeaknode/admin/squeak_admin_server_servicer.py @@ -161,6 +161,9 @@ class SqueakAdminServerServicer(squeak_admin_pb2_grpc.SqueakAdminServicer): def GetAddressSqueakDisplays(self, request, context): return self.handler.handle_get_squeak_display_entries_for_address(request) + def GetSearchSqueakDisplays(self, request, context): + return self.handler.handle_get_squeak_display_entries_for_text_search(request) + def GetAncestorSqueakDisplays(self, request, context): return self.handler.handle_get_ancestor_squeak_display_entries(request) diff --git a/squeaknode/admin/webapp/app.py b/squeaknode/admin/webapp/app.py index 91960c4a..ebadff19 100644 --- a/squeaknode/admin/webapp/app.py +++ b/squeaknode/admin/webapp/app.py @@ -433,6 +433,14 @@ def create_app(handler, username, password): handler.handle_get_squeak_display_entries_for_address, ) + @app.route("/getsearchsqueakdisplays", methods=["POST"]) + @login_required + def getsearchsqueakdisplays(): + return handle_request( + squeak_admin_pb2.GetSearchSqueakDisplaysRequest(), + handler.handle_get_squeak_display_entries_for_text_search, + ) + @app.route("/createcontactprofile", methods=["POST"]) @login_required def createcontactprofile(): diff --git a/squeaknode/db/squeak_db.py b/squeaknode/db/squeak_db.py index f1ccd308..e6477af8 100644 --- a/squeaknode/db/squeak_db.py +++ b/squeaknode/db/squeak_db.py @@ -357,12 +357,14 @@ class SqueakDb: last_block_height = last_entry.block_height if last_entry else MAX_INT last_squeak_time = last_entry.squeak_time if last_entry else MAX_INT last_squeak_hash = last_entry.squeak_hash if last_entry else MAX_HASH - logger.info("""Timeline db query with + logger.info("""Address db query with + address: {} limit: {} block_height: {} squeak_time: {} squeak_hash: {} """.format( + address, limit, last_block_height, last_squeak_time, @@ -400,6 +402,61 @@ class SqueakDb: rows = result.fetchall() return [self._parse_squeak_entry(row) for row in rows] + def get_squeak_entries_for_text_search( + self, + search_text: str, + limit: int, + last_entry: Optional[SqueakEntry], + ) -> List[SqueakEntry]: + """ Get a squeak. """ + last_block_height = last_entry.block_height if last_entry else MAX_INT + last_squeak_time = last_entry.squeak_time if last_entry else MAX_INT + last_squeak_hash = last_entry.squeak_hash if last_entry else MAX_HASH + logger.info("""Search db query with + search_text: {} + limit: {} + block_height: {} + squeak_time: {} + squeak_hash: {} + """.format( + search_text, + limit, + last_block_height, + last_squeak_time, + last_squeak_hash.hex(), + )) + s = ( + select([self.squeaks, self.profiles]) + .select_from( + self.squeaks.outerjoin( + self.profiles, + self.profiles.c.address == self.squeaks.c.author_address, + ) + ) + .where(self.squeaks.c.content.ilike(f'%{search_text}%')) + .where( + tuple_( + self.squeaks.c.n_block_height, + self.squeaks.c.n_time, + self.squeaks.c.hash, + ) < tuple_( + last_block_height, + last_squeak_time, + last_squeak_hash, + ) + ) + .order_by( + self.squeaks.c.n_block_height.desc(), + self.squeaks.c.n_time.desc(), + self.squeaks.c.hash.desc(), + ) + .limit(limit) + ) + with self.get_connection() as connection: + result = connection.execute(s) + rows = result.fetchall() + return [self._parse_squeak_entry(row) for row in rows] + def get_thread_ancestor_squeak_entries(self, squeak_hash: bytes) -> List[SqueakEntry]: """ Get all reply ancestors of squeak hash. """ ancestors = ( diff --git a/squeaknode/node/squeak_controller.py b/squeaknode/node/squeak_controller.py index 1c8c3a17..4c20c000 100644 --- a/squeaknode/node/squeak_controller.py +++ b/squeaknode/node/squeak_controller.py @@ -513,6 +513,18 @@ class SqueakController: last_entry, ) + def get_squeak_entries_for_text_search( + self, + search_text: str, + limit: int, + last_entry: Optional[SqueakEntry], + ) -> List[SqueakEntry]: + return self.squeak_db.get_squeak_entries_for_text_search( + search_text, + limit, + last_entry, + ) + def get_ancestor_squeak_entries(self, squeak_hash: bytes) -> List[SqueakEntry]: return self.squeak_db.get_thread_ancestor_squeak_entries( squeak_hash,