From 387aa13893744e8240893365d552eef09f651e4f Mon Sep 17 00:00:00 2001 From: Jonathan Zernik Date: Sat, 6 Feb 2021 01:24:31 -0800 Subject: [PATCH] Use autoincrement for primary keys (#788) * Change db model to use bigint with autoincrement * Add constraint on host and port in peer table * Use autoincrement in database model --- itests/tests/conftest.py | 44 +++++++++++++- itests/tests/test_squeak_node.py | 39 +----------- itests/tests/util.py | 1 + ...e3_add_profile_image_column_to_profile_.py | 33 ---------- ..._all.py => 86f7ad5667c9_initialize_all.py} | 60 ++++++++++++------- ..._add_unique_constraint_on_payment_hash_.py | 33 ---------- ...d3e_add_composite_unique_constraint_on_.py | 33 ---------- ..._add_valid_column_to_sent_payment_table.py | 34 ----------- squeaknode/db/models.py | 41 ++++++++++--- 9 files changed, 116 insertions(+), 202 deletions(-) delete mode 100644 squeaknode/db/alembic/versions/6851569c46e3_add_profile_image_column_to_profile_.py rename squeaknode/db/alembic/versions/{afb1f38de2cb_initialize_all.py => 86f7ad5667c9_initialize_all.py} (76%) delete mode 100644 squeaknode/db/alembic/versions/cd3f18ee33c8_add_unique_constraint_on_payment_hash_.py delete mode 100644 squeaknode/db/alembic/versions/d5a06570ed3e_add_composite_unique_constraint_on_.py delete mode 100644 squeaknode/db/alembic/versions/df59995f322c_add_valid_column_to_sent_payment_table.py diff --git a/itests/tests/conftest.py b/itests/tests/conftest.py index 6b8d9f6e..734c5d57 100644 --- a/itests/tests/conftest.py +++ b/itests/tests/conftest.py @@ -130,17 +130,23 @@ def saved_squeak_hash(server_stub, admin_stub, signing_profile_id): @pytest.fixture -def peer_id(server_stub, admin_stub): +def peer_id(server_stub, admin_stub, random_peer_name): # Create a new peer create_peer_response = admin_stub.CreatePeer( squeak_admin_pb2.CreatePeerRequest( - peer_name="fake_peer_name", - host="fake_host", + peer_name=random_peer_name, + host=random_peer_name, port=1234, ) ) peer_id = create_peer_response.peer_id yield peer_id + # Delete the peer + admin_stub.DeletePeer( + squeak_admin_pb2.DeletePeerRequest( + peer_id=peer_id, + ) + ) @pytest.fixture @@ -148,6 +154,11 @@ def random_name(): yield "random_name_{}".format(uuid.uuid1()) +@pytest.fixture +def random_peer_name(): + yield "random_peer_name_{}".format(uuid.uuid1()) + + @pytest.fixture def random_image(): yield os.urandom(567) @@ -156,3 +167,30 @@ def random_image(): @pytest.fixture def random_image_base64_string(random_image): yield bytes_to_base64_string(random_image) + + +@pytest.fixture +def connected_peer_id(server_stub, other_admin_stub): + # Add the main node as a peer + create_peer_response = other_admin_stub.CreatePeer( + squeak_admin_pb2.CreatePeerRequest( + peer_name="test_peer", + host="squeaknode", + port=8774, + ) + ) + peer_id = create_peer_response.peer_id + # Set the peer to be downloading + other_admin_stub.SetPeerDownloading( + squeak_admin_pb2.SetPeerDownloadingRequest( + peer_id=peer_id, + downloading=True, + ) + ) + yield peer_id + # Delete the peer + other_admin_stub.DeletePeer( + squeak_admin_pb2.DeletePeerRequest( + peer_id=peer_id, + ) + ) diff --git a/itests/tests/test_squeak_node.py b/itests/tests/test_squeak_node.py index 38652f78..426f13ec 100644 --- a/itests/tests/test_squeak_node.py +++ b/itests/tests/test_squeak_node.py @@ -1025,6 +1025,7 @@ def test_connect_other_node( admin_stub, other_server_stub, other_admin_stub, + connected_peer_id, lightning_client, signing_profile_id, saved_squeak_hash, @@ -1035,24 +1036,6 @@ def test_connect_other_node( ) assert len(get_timeline_squeak_display_response.squeak_display_entries) == 0 - # Add the main node as a peer - create_peer_response = other_admin_stub.CreatePeer( - squeak_admin_pb2.CreatePeerRequest( - peer_name="test_peer", - host="squeaknode", - port=8774, - ) - ) - peer_id = create_peer_response.peer_id - - # Set the peer to be downloading - other_admin_stub.SetPeerDownloading( - squeak_admin_pb2.SetPeerDownloadingRequest( - peer_id=peer_id, - downloading=True, - ) - ) - # Get the squeak profile get_squeak_profile_response = admin_stub.GetSqueakProfile( squeak_admin_pb2.GetSqueakProfileRequest( @@ -1232,29 +1215,11 @@ def test_download_single_squeak( admin_stub, other_server_stub, other_admin_stub, + connected_peer_id, lightning_client, signing_profile_id, saved_squeak_hash, ): - - # Add the main node as a peer - create_peer_response = other_admin_stub.CreatePeer( - squeak_admin_pb2.CreatePeerRequest( - peer_name="test_peer", - host="squeaknode", - port=8774, - ) - ) - peer_id = create_peer_response.peer_id - - # Set the peer to be downloading - other_admin_stub.SetPeerDownloading( - squeak_admin_pb2.SetPeerDownloadingRequest( - peer_id=peer_id, - downloading=True, - ) - ) - # Get the squeak profile get_squeak_profile_response = admin_stub.GetSqueakProfile( squeak_admin_pb2.GetSqueakProfileRequest( diff --git a/itests/tests/util.py b/itests/tests/util.py index 7ec140d6..6e9b968d 100644 --- a/itests/tests/util.py +++ b/itests/tests/util.py @@ -117,6 +117,7 @@ def connect_peer(lightning_client, lightning_host, remote_pubkey): lightning_client.disconnect_peer( remote_pubkey, ) + time.sleep(2) @contextmanager diff --git a/squeaknode/db/alembic/versions/6851569c46e3_add_profile_image_column_to_profile_.py b/squeaknode/db/alembic/versions/6851569c46e3_add_profile_image_column_to_profile_.py deleted file mode 100644 index 282d8fcb..00000000 --- a/squeaknode/db/alembic/versions/6851569c46e3_add_profile_image_column_to_profile_.py +++ /dev/null @@ -1,33 +0,0 @@ -"""Add profile image column to profile table - -Revision ID: 6851569c46e3 -Revises: d5a06570ed3e -Create Date: 2021-01-28 23:25:31.015184 - -""" -import sqlalchemy as sa -from alembic import op - - -# revision identifiers, used by Alembic. -revision = '6851569c46e3' -down_revision = 'd5a06570ed3e' -branch_labels = None -depends_on = None - - -def upgrade(): - # ### commands auto generated by Alembic - please adjust! ### - with op.batch_alter_table('profile', schema=None) as batch_op: - batch_op.add_column( - sa.Column('profile_image', sa.Binary(), nullable=True)) - - # ### end Alembic commands ### - - -def downgrade(): - # ### commands auto generated by Alembic - please adjust! ### - with op.batch_alter_table('profile', schema=None) as batch_op: - batch_op.drop_column('profile_image') - - # ### end Alembic commands ### diff --git a/squeaknode/db/alembic/versions/afb1f38de2cb_initialize_all.py b/squeaknode/db/alembic/versions/86f7ad5667c9_initialize_all.py similarity index 76% rename from squeaknode/db/alembic/versions/afb1f38de2cb_initialize_all.py rename to squeaknode/db/alembic/versions/86f7ad5667c9_initialize_all.py index 9175dfc5..aad8d88c 100644 --- a/squeaknode/db/alembic/versions/afb1f38de2cb_initialize_all.py +++ b/squeaknode/db/alembic/versions/86f7ad5667c9_initialize_all.py @@ -1,16 +1,19 @@ """Initialize all -Revision ID: afb1f38de2cb +Revision ID: 86f7ad5667c9 Revises: -Create Date: 2021-01-11 16:10:46.633311 +Create Date: 2021-02-06 00:29:45.755247 """ import sqlalchemy as sa from alembic import op +from squeaknode.db.models import SLBigInteger +from squeaknode.db.models import TZDateTime + # revision identifiers, used by Alembic. -revision = 'afb1f38de2cb' +revision = '86f7ad5667c9' down_revision = None branch_labels = None depends_on = None @@ -20,32 +23,37 @@ def upgrade(): # ### commands auto generated by Alembic - please adjust! ### op.create_table('peer', sa.Column('peer_id', sa.Integer(), nullable=False), - sa.Column('created', sa.DateTime(), server_default=sa.text( + sa.Column('created', TZDateTime(), server_default=sa.text( '(CURRENT_TIMESTAMP)'), nullable=False), - sa.Column('peer_name', sa.String(), nullable=True), + sa.Column('peer_name', sa.String(), nullable=False), sa.Column('server_host', sa.String(), nullable=False), sa.Column('server_port', sa.Integer(), nullable=False), sa.Column('uploading', sa.Boolean(), nullable=False), sa.Column('downloading', sa.Boolean(), nullable=False), - sa.PrimaryKeyConstraint('peer_id') + sa.PrimaryKeyConstraint('peer_id'), + sa.UniqueConstraint('server_host', 'server_port', + name='uq_peer_server_host_server_port'), + sqlite_autoincrement=True ) op.create_table('profile', sa.Column('profile_id', sa.Integer(), nullable=False), - sa.Column('created', sa.DateTime(), server_default=sa.text( + sa.Column('created', TZDateTime(), server_default=sa.text( '(CURRENT_TIMESTAMP)'), nullable=False), sa.Column('profile_name', sa.String(), nullable=False), sa.Column('private_key', sa.Binary(), nullable=True), sa.Column('address', sa.String(length=35), nullable=False), sa.Column('sharing', sa.Boolean(), nullable=False), sa.Column('following', sa.Boolean(), nullable=False), + sa.Column('profile_image', sa.Binary(), nullable=True), sa.PrimaryKeyConstraint('profile_id'), sa.UniqueConstraint('address'), - sa.UniqueConstraint('profile_name') + sa.UniqueConstraint('profile_name'), + sqlite_autoincrement=True ) op.create_table('received_offer', sa.Column('received_offer_id', - sa.Integer(), nullable=False), - sa.Column('created', sa.DateTime(), server_default=sa.text( + SLBigInteger(), nullable=False), + sa.Column('created', TZDateTime(), server_default=sa.text( '(CURRENT_TIMESTAMP)'), nullable=False), sa.Column('squeak_hash', sa.String( length=64), nullable=False), @@ -64,12 +72,14 @@ def upgrade(): sa.Column('node_host', sa.String(), nullable=False), sa.Column('node_port', sa.Integer(), nullable=False), sa.Column('peer_id', sa.Integer(), nullable=False), - sa.PrimaryKeyConstraint('received_offer_id') + sa.PrimaryKeyConstraint('received_offer_id'), + sa.UniqueConstraint('payment_hash'), + sqlite_autoincrement=True ) op.create_table('received_payment', sa.Column('received_payment_id', - sa.Integer(), nullable=False), - sa.Column('created', sa.DateTime(), server_default=sa.text( + SLBigInteger(), nullable=False), + sa.Column('created', TZDateTime(), server_default=sa.text( '(CURRENT_TIMESTAMP)'), nullable=False), sa.Column('squeak_hash', sa.String( length=64), nullable=False), @@ -80,11 +90,12 @@ def upgrade(): sa.Column('client_addr', sa.String( length=64), nullable=False), sa.PrimaryKeyConstraint('received_payment_id'), - sa.UniqueConstraint('payment_hash') + sa.UniqueConstraint('payment_hash'), + sqlite_autoincrement=True ) op.create_table('sent_offer', - sa.Column('sent_offer_id', sa.Integer(), nullable=False), - sa.Column('created', sa.DateTime(), server_default=sa.text( + sa.Column('sent_offer_id', SLBigInteger(), nullable=False), + sa.Column('created', TZDateTime(), server_default=sa.text( '(CURRENT_TIMESTAMP)'), nullable=False), sa.Column('squeak_hash', sa.String( length=64), nullable=False), @@ -101,11 +112,15 @@ def upgrade(): sa.Column('client_addr', sa.String( length=64), nullable=False), sa.PrimaryKeyConstraint('sent_offer_id'), - sa.UniqueConstraint('payment_hash') + sa.UniqueConstraint('payment_hash'), + sa.UniqueConstraint('squeak_hash', 'client_addr', + name='uq_sent_offer_squeak_hash_client_addr'), + sqlite_autoincrement=True ) op.create_table('sent_payment', - sa.Column('sent_payment_id', sa.Integer(), nullable=False), - sa.Column('created', sa.DateTime(), server_default=sa.text( + sa.Column('sent_payment_id', + SLBigInteger(), nullable=False), + sa.Column('created', TZDateTime(), server_default=sa.text( '(CURRENT_TIMESTAMP)'), nullable=False), sa.Column('peer_id', sa.Integer(), nullable=False), sa.Column('squeak_hash', sa.String( @@ -117,11 +132,14 @@ def upgrade(): sa.Column('price_msat', sa.Integer(), nullable=False), sa.Column('node_pubkey', sa.String( length=66), nullable=False), - sa.PrimaryKeyConstraint('sent_payment_id') + sa.Column('valid', sa.Boolean(), nullable=False), + sa.PrimaryKeyConstraint('sent_payment_id'), + sa.UniqueConstraint('payment_hash'), + sqlite_autoincrement=True ) op.create_table('squeak', sa.Column('hash', sa.String(length=64), nullable=False), - sa.Column('created', sa.DateTime(), server_default=sa.text( + sa.Column('created', TZDateTime(), server_default=sa.text( '(CURRENT_TIMESTAMP)'), nullable=False), sa.Column('squeak', sa.Binary(), nullable=False), sa.Column('hash_reply_sqk', sa.String( diff --git a/squeaknode/db/alembic/versions/cd3f18ee33c8_add_unique_constraint_on_payment_hash_.py b/squeaknode/db/alembic/versions/cd3f18ee33c8_add_unique_constraint_on_payment_hash_.py deleted file mode 100644 index f5eed1c8..00000000 --- a/squeaknode/db/alembic/versions/cd3f18ee33c8_add_unique_constraint_on_payment_hash_.py +++ /dev/null @@ -1,33 +0,0 @@ -"""Add unique constraint on payment hash of received offer - -Revision ID: cd3f18ee33c8 -Revises: df59995f322c -Create Date: 2021-01-27 17:40:06.995928 - -""" -from alembic import op - - -# revision identifiers, used by Alembic. -revision = 'cd3f18ee33c8' -down_revision = 'df59995f322c' -branch_labels = None -depends_on = None - - -def upgrade(): - # ### commands auto generated by Alembic - please adjust! ### - with op.batch_alter_table('received_offer', schema=None) as batch_op: - batch_op.create_unique_constraint( - 'uq_received_offer_payment_hash', ['payment_hash']) - - # ### end Alembic commands ### - - -def downgrade(): - # ### commands auto generated by Alembic - please adjust! ### - with op.batch_alter_table('received_offer', schema=None) as batch_op: - batch_op.drop_constraint( - 'uq_received_offer_payment_hash', type_='unique') - - # ### end Alembic commands ### diff --git a/squeaknode/db/alembic/versions/d5a06570ed3e_add_composite_unique_constraint_on_.py b/squeaknode/db/alembic/versions/d5a06570ed3e_add_composite_unique_constraint_on_.py deleted file mode 100644 index 4e3fb60b..00000000 --- a/squeaknode/db/alembic/versions/d5a06570ed3e_add_composite_unique_constraint_on_.py +++ /dev/null @@ -1,33 +0,0 @@ -"""Add composite unique constraint on squeak hash and client addr of sent offer - -Revision ID: d5a06570ed3e -Revises: cd3f18ee33c8 -Create Date: 2021-01-28 17:49:17.615748 - -""" -from alembic import op - - -# revision identifiers, used by Alembic. -revision = 'd5a06570ed3e' -down_revision = 'cd3f18ee33c8' -branch_labels = None -depends_on = None - - -def upgrade(): - # ### commands auto generated by Alembic - please adjust! ### - with op.batch_alter_table('sent_offer', schema=None) as batch_op: - batch_op.create_unique_constraint('uq_sent_offer_squeak_hash_client_addr', [ - 'squeak_hash', 'client_addr']) - - # ### end Alembic commands ### - - -def downgrade(): - # ### commands auto generated by Alembic - please adjust! ### - with op.batch_alter_table('sent_offer', schema=None) as batch_op: - batch_op.drop_constraint( - 'uq_sent_offer_squeak_hash_client_addr', type_='unique') - - # ### end Alembic commands ### diff --git a/squeaknode/db/alembic/versions/df59995f322c_add_valid_column_to_sent_payment_table.py b/squeaknode/db/alembic/versions/df59995f322c_add_valid_column_to_sent_payment_table.py deleted file mode 100644 index 7e7ea282..00000000 --- a/squeaknode/db/alembic/versions/df59995f322c_add_valid_column_to_sent_payment_table.py +++ /dev/null @@ -1,34 +0,0 @@ -"""Add valid column to sent_payment table - -Revision ID: df59995f322c -Revises: afb1f38de2cb -Create Date: 2021-01-21 18:18:22.589806 - -""" -import sqlalchemy as sa -from alembic import op -from sqlalchemy.sql.expression import literal - - -# revision identifiers, used by Alembic. -revision = 'df59995f322c' -down_revision = 'afb1f38de2cb' -branch_labels = None -depends_on = None - - -def upgrade(): - # ### commands auto generated by Alembic - please adjust! ### - with op.batch_alter_table('sent_payment', schema=None) as batch_op: - batch_op.add_column(sa.Column('valid', sa.Boolean(), - server_default=literal(True), nullable=False)) - - # ### end Alembic commands ### - - -def downgrade(): - # ### commands auto generated by Alembic - please adjust! ### - with op.batch_alter_table('sent_payment', schema=None) as batch_op: - batch_op.drop_column('valid') - - # ### end Alembic commands ### diff --git a/squeaknode/db/models.py b/squeaknode/db/models.py index 1ab49f93..447f675e 100644 --- a/squeaknode/db/models.py +++ b/squeaknode/db/models.py @@ -1,6 +1,7 @@ import datetime import logging +from sqlalchemy import BigInteger from sqlalchemy import Binary from sqlalchemy import Boolean from sqlalchemy import Column @@ -11,8 +12,10 @@ from sqlalchemy import MetaData from sqlalchemy import String from sqlalchemy import Table from sqlalchemy import UniqueConstraint +from sqlalchemy.ext.compiler import compiles from sqlalchemy.types import TypeDecorator + logger = logging.getLogger(__name__) @@ -34,6 +37,20 @@ class TZDateTime(TypeDecorator): return value +class SLBigInteger(BigInteger): + pass + + +@compiles(SLBigInteger, 'sqlite') +def bi_c_sqlite(element, compiler, **kw): + return "INTEGER" + + +@compiles(SLBigInteger) +def bi_c(element, compiler, **kw): + return compiler.visit_BIGINT(element, **kw) + + class Models: def __init__(self, schema=None): self.schema = schema @@ -62,11 +79,12 @@ class Models: Column("created", TZDateTime, server_default=func.now(), nullable=False), Column("profile_name", String, unique=True, nullable=False), - Column("private_key", Binary), + Column("private_key", Binary, nullable=True), Column("address", String(35), unique=True, nullable=False), Column("sharing", Boolean, nullable=False), Column("following", Boolean, nullable=False), Column("profile_image", Binary, nullable=True), + sqlite_autoincrement=True, ) self.peers = Table( @@ -75,17 +93,20 @@ class Models: Column("peer_id", Integer, primary_key=True), Column("created", TZDateTime, server_default=func.now(), nullable=False), - Column("peer_name", String), + Column("peer_name", String, nullable=False), Column("server_host", String, nullable=False), Column("server_port", Integer, nullable=False), Column("uploading", Boolean, nullable=False), Column("downloading", Boolean, nullable=False), + UniqueConstraint('server_host', 'server_port', + name='uq_peer_server_host_server_port'), + sqlite_autoincrement=True, ) self.received_offers = Table( "received_offer", self.metadata, - Column("received_offer_id", Integer, primary_key=True), + Column("received_offer_id", SLBigInteger, primary_key=True), Column("created", TZDateTime, server_default=func.now(), nullable=False), Column("squeak_hash", String(64), nullable=False), @@ -100,27 +121,29 @@ class Models: Column("node_host", String, nullable=False), Column("node_port", Integer, nullable=False), Column("peer_id", Integer, nullable=False), + sqlite_autoincrement=True, ) self.sent_payments = Table( "sent_payment", self.metadata, - Column("sent_payment_id", Integer, primary_key=True), + Column("sent_payment_id", SLBigInteger, primary_key=True), Column("created", TZDateTime, server_default=func.now(), nullable=False), Column("peer_id", Integer, nullable=False), Column("squeak_hash", String(64), nullable=False), - Column("payment_hash", String(64), nullable=False), + Column("payment_hash", String(64), unique=True, nullable=False), Column("secret_key", String(64), nullable=False), Column("price_msat", Integer, nullable=False, default=0), Column("node_pubkey", String(66), nullable=False), Column("valid", Boolean, nullable=False), + sqlite_autoincrement=True, ) self.sent_offers = Table( "sent_offer", self.metadata, - Column("sent_offer_id", Integer, primary_key=True), + Column("sent_offer_id", SLBigInteger, primary_key=True), Column("created", TZDateTime, server_default=func.now(), nullable=False), Column("squeak_hash", String(64), nullable=False), @@ -133,13 +156,14 @@ class Models: Column("invoice_expiry", Integer, nullable=False), Column("client_addr", String(64), nullable=False), UniqueConstraint('squeak_hash', 'client_addr', - name='uq_sent_offer_squeak_hash_client_addr') + name='uq_sent_offer_squeak_hash_client_addr'), + sqlite_autoincrement=True, ) self.received_payments = Table( "received_payment", self.metadata, - Column("received_payment_id", Integer, primary_key=True), + Column("received_payment_id", SLBigInteger, primary_key=True), Column("created", TZDateTime, server_default=func.now(), nullable=False), Column("squeak_hash", String(64), nullable=False), @@ -147,4 +171,5 @@ class Models: Column("price_msat", Integer, nullable=False), Column("settle_index", Integer, nullable=False), Column("client_addr", String(64), nullable=False), + sqlite_autoincrement=True, )