From e966f73db13b2a583ec1c45e93b97384d04ef7b7 Mon Sep 17 00:00:00 2001 From: Jonathan Zernik Date: Sun, 27 Dec 2020 23:32:02 -0800 Subject: [PATCH] Update db to use Timestamp for times (#519) --- docker/docker-compose.yml | 1 - docs/DEVELOPMENT.md | 3 +- .../versions/56e24c42b105_initialize_all.py | 129 ------------------ .../versions/b92797939348_initialize_all.py | 129 ++++++++++++++++++ squeaknode/db/models.py | 15 +- squeaknode/db/squeak_db.py | 7 +- 6 files changed, 143 insertions(+), 141 deletions(-) delete mode 100644 squeaknode/db/alembic/versions/56e24c42b105_initialize_all.py create mode 100644 squeaknode/db/alembic/versions/b92797939348_initialize_all.py diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index abe0deff..38f883da 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -9,7 +9,6 @@ services: ports: - 8332:8332 - 18332:18332 - - 18556:18556 lnd: image: lnd diff --git a/docs/DEVELOPMENT.md b/docs/DEVELOPMENT.md index 3b436e12..86ec6386 100644 --- a/docs/DEVELOPMENT.md +++ b/docs/DEVELOPMENT.md @@ -18,5 +18,6 @@ To change any database model, follow these steps. $ virtuelenv venv $ pip install -r requirements.txt $ pip install -e . - $ alembic revision --autogenerate -m "" + $ alembic -c squeaknode/db/alembic.ini revision --autogenerate -m "" ``` +- To generate a new initial version revision, follow the same steps as above, but the `/home//.sqk/data/testnet/data.db` file should be an empty file. diff --git a/squeaknode/db/alembic/versions/56e24c42b105_initialize_all.py b/squeaknode/db/alembic/versions/56e24c42b105_initialize_all.py deleted file mode 100644 index df71c6e0..00000000 --- a/squeaknode/db/alembic/versions/56e24c42b105_initialize_all.py +++ /dev/null @@ -1,129 +0,0 @@ -"""Initialize all - -Revision ID: 56e24c42b105 -Revises: -Create Date: 2020-12-22 19:30:23.837560 - -""" -import sqlalchemy as sa - -from alembic import op - -# revision identifiers, used by Alembic. -revision = '56e24c42b105' -down_revision = None -branch_labels = None -depends_on = None - - -def upgrade(): - # ### commands auto generated by Alembic - please adjust! ### - op.create_table('offer', - sa.Column('offer_id', sa.Integer(), nullable=False), - sa.Column('created', sa.DateTime(), server_default=sa.text('(CURRENT_TIMESTAMP)'), nullable=False), - sa.Column('squeak_hash', sa.String(length=64), nullable=False), - sa.Column('payment_hash', sa.String(length=64), nullable=False), - sa.Column('nonce', sa.String(length=64), nullable=False), - sa.Column('payment_point', sa.String(length=66), nullable=False), - sa.Column('invoice_timestamp', sa.Integer(), nullable=False), - sa.Column('invoice_expiry', sa.Integer(), nullable=False), - sa.Column('price_msat', sa.Integer(), nullable=False), - sa.Column('payment_request', sa.String(), nullable=False), - sa.Column('destination', sa.String(length=66), nullable=False), - sa.Column('node_host', sa.String(), nullable=False), - sa.Column('node_port', sa.Integer(), nullable=False), - sa.Column('peer_id', sa.Integer(), nullable=False), - sa.PrimaryKeyConstraint('offer_id') - ) - op.create_table('peer', - sa.Column('id', sa.Integer(), nullable=False), - sa.Column('created', sa.DateTime(), server_default=sa.text('(CURRENT_TIMESTAMP)'), nullable=False), - sa.Column('peer_name', sa.String(), nullable=True), - sa.Column('server_host', sa.String(), nullable=False), - sa.Column('server_port', sa.Integer(), nullable=False), - sa.Column('uploading', sa.Boolean(), nullable=False), - sa.Column('downloading', sa.Boolean(), nullable=False), - sa.PrimaryKeyConstraint('id') - ) - op.create_table('profile', - sa.Column('profile_id', sa.Integer(), nullable=False), - sa.Column('created', sa.DateTime(), server_default=sa.text('(CURRENT_TIMESTAMP)'), nullable=False), - sa.Column('profile_name', sa.String(), nullable=False), - sa.Column('private_key', sa.Binary(), nullable=True), - sa.Column('address', sa.String(length=35), nullable=False), - sa.Column('sharing', sa.Boolean(), nullable=False), - sa.Column('following', sa.Boolean(), nullable=False), - sa.PrimaryKeyConstraint('profile_id'), - sa.UniqueConstraint('address'), - sa.UniqueConstraint('profile_name') - ) - op.create_table('received_payment', - sa.Column('received_payment_id', sa.Integer(), nullable=False), - sa.Column('created', sa.DateTime(), server_default=sa.text('(CURRENT_TIMESTAMP)'), nullable=False), - sa.Column('squeak_hash', sa.String(length=64), nullable=False), - sa.Column('payment_hash', sa.String(length=64), nullable=False), - sa.Column('price_msat', sa.Integer(), nullable=False), - sa.Column('settle_index', sa.Integer(), nullable=False), - sa.Column('client_addr', sa.String(length=64), nullable=False), - sa.PrimaryKeyConstraint('received_payment_id'), - sa.UniqueConstraint('payment_hash') - ) - op.create_table('sent_offer', - sa.Column('sent_offer_id', sa.Integer(), nullable=False), - sa.Column('created', sa.DateTime(), server_default=sa.text('(CURRENT_TIMESTAMP)'), nullable=False), - sa.Column('squeak_hash', sa.String(length=64), nullable=False), - sa.Column('payment_hash', sa.String(length=64), nullable=False), - sa.Column('secret_key', sa.String(length=64), nullable=False), - sa.Column('nonce', sa.String(length=64), nullable=False), - sa.Column('price_msat', sa.Integer(), nullable=False), - sa.Column('payment_request', sa.String(), nullable=False), - sa.Column('invoice_timestamp', sa.Integer(), nullable=False), - sa.Column('invoice_expiry', sa.Integer(), nullable=False), - sa.Column('client_addr', sa.String(length=64), nullable=False), - sa.PrimaryKeyConstraint('sent_offer_id'), - sa.UniqueConstraint('payment_hash') - ) - op.create_table('sent_payment', - sa.Column('sent_payment_id', sa.Integer(), nullable=False), - sa.Column('created', sa.DateTime(), server_default=sa.text('(CURRENT_TIMESTAMP)'), nullable=False), - sa.Column('offer_id', sa.Integer(), nullable=False), - sa.Column('peer_id', sa.Integer(), nullable=False), - sa.Column('squeak_hash', sa.String(length=64), nullable=False), - sa.Column('payment_hash', sa.String(length=64), nullable=False), - sa.Column('secret_key', sa.String(length=64), nullable=False), - sa.Column('price_msat', sa.Integer(), nullable=False), - sa.Column('node_pubkey', sa.String(length=66), nullable=False), - sa.PrimaryKeyConstraint('sent_payment_id') - ) - op.create_table('squeak', - sa.Column('hash', sa.String(length=64), nullable=False), - sa.Column('created', sa.DateTime(), server_default=sa.text('(CURRENT_TIMESTAMP)'), nullable=False), - sa.Column('squeak', sa.Binary(), nullable=False), - sa.Column('hash_reply_sqk', sa.String(length=64), nullable=False), - sa.Column('hash_block', sa.String(length=64), nullable=False), - sa.Column('n_block_height', sa.Integer(), nullable=False), - sa.Column('n_time', sa.Integer(), nullable=False), - sa.Column('author_address', sa.String(length=35), nullable=False), - sa.Column('secret_key', sa.String(length=64), nullable=True), - sa.Column('block_header', sa.Binary(), nullable=False), - sa.PrimaryKeyConstraint('hash') - ) - with op.batch_alter_table('squeak', schema=None) as batch_op: - batch_op.create_index(batch_op.f('ix_squeak_author_address'), ['author_address'], unique=False) - - # ### end Alembic commands ### - - -def downgrade(): - # ### commands auto generated by Alembic - please adjust! ### - with op.batch_alter_table('squeak', schema=None) as batch_op: - batch_op.drop_index(batch_op.f('ix_squeak_author_address')) - - op.drop_table('squeak') - op.drop_table('sent_payment') - op.drop_table('sent_offer') - op.drop_table('received_payment') - op.drop_table('profile') - op.drop_table('peer') - op.drop_table('offer') - # ### end Alembic commands ### diff --git a/squeaknode/db/alembic/versions/b92797939348_initialize_all.py b/squeaknode/db/alembic/versions/b92797939348_initialize_all.py new file mode 100644 index 00000000..06856fa0 --- /dev/null +++ b/squeaknode/db/alembic/versions/b92797939348_initialize_all.py @@ -0,0 +1,129 @@ +"""Initialize all + +Revision ID: b92797939348 +Revises: +Create Date: 2020-12-27 22:22:09.374933 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = 'b92797939348' +down_revision = None +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.create_table('offer', + sa.Column('offer_id', sa.Integer(), nullable=False), + sa.Column('created', sa.DateTime(), server_default=sa.text('(CURRENT_TIMESTAMP)'), nullable=False), + sa.Column('squeak_hash', sa.String(length=64), nullable=False), + sa.Column('payment_hash', sa.String(length=64), nullable=False), + sa.Column('nonce', sa.String(length=64), nullable=False), + sa.Column('payment_point', sa.String(length=66), nullable=False), + sa.Column('invoice_timestamp', sa.Integer(), nullable=False), + sa.Column('invoice_expiry', sa.Integer(), nullable=False), + sa.Column('price_msat', sa.Integer(), nullable=False), + sa.Column('payment_request', sa.String(), nullable=False), + sa.Column('destination', sa.String(length=66), nullable=False), + sa.Column('node_host', sa.String(), nullable=False), + sa.Column('node_port', sa.Integer(), nullable=False), + sa.Column('peer_id', sa.Integer(), nullable=False), + sa.PrimaryKeyConstraint('offer_id') + ) + op.create_table('peer', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('created', sa.DateTime(), server_default=sa.text('(CURRENT_TIMESTAMP)'), nullable=False), + sa.Column('peer_name', sa.String(), nullable=True), + sa.Column('server_host', sa.String(), nullable=False), + sa.Column('server_port', sa.Integer(), nullable=False), + sa.Column('uploading', sa.Boolean(), nullable=False), + sa.Column('downloading', sa.Boolean(), nullable=False), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('profile', + sa.Column('profile_id', sa.Integer(), nullable=False), + sa.Column('created', sa.DateTime(), server_default=sa.text('(CURRENT_TIMESTAMP)'), nullable=False), + sa.Column('profile_name', sa.String(), nullable=False), + sa.Column('private_key', sa.Binary(), nullable=True), + sa.Column('address', sa.String(length=35), nullable=False), + sa.Column('sharing', sa.Boolean(), nullable=False), + sa.Column('following', sa.Boolean(), nullable=False), + sa.PrimaryKeyConstraint('profile_id'), + sa.UniqueConstraint('address'), + sa.UniqueConstraint('profile_name') + ) + op.create_table('received_payment', + sa.Column('received_payment_id', sa.Integer(), nullable=False), + sa.Column('created', sa.DateTime(), server_default=sa.text('(CURRENT_TIMESTAMP)'), nullable=False), + sa.Column('squeak_hash', sa.String(length=64), nullable=False), + sa.Column('payment_hash', sa.String(length=64), nullable=False), + sa.Column('price_msat', sa.Integer(), nullable=False), + sa.Column('settle_index', sa.Integer(), nullable=False), + sa.Column('client_addr', sa.String(length=64), nullable=False), + sa.PrimaryKeyConstraint('received_payment_id'), + sa.UniqueConstraint('payment_hash') + ) + op.create_table('sent_offer', + sa.Column('sent_offer_id', sa.Integer(), nullable=False), + sa.Column('created', sa.DateTime(), server_default=sa.text('(CURRENT_TIMESTAMP)'), nullable=False), + sa.Column('squeak_hash', sa.String(length=64), nullable=False), + sa.Column('payment_hash', sa.String(length=64), nullable=False), + sa.Column('secret_key', sa.String(length=64), nullable=False), + sa.Column('nonce', sa.String(length=64), nullable=False), + sa.Column('price_msat', sa.Integer(), nullable=False), + sa.Column('payment_request', sa.String(), nullable=False), + sa.Column('invoice_timestamp', sa.Integer(), nullable=False), + sa.Column('invoice_expiry', sa.Integer(), nullable=False), + sa.Column('client_addr', sa.String(length=64), nullable=False), + sa.PrimaryKeyConstraint('sent_offer_id'), + sa.UniqueConstraint('payment_hash') + ) + op.create_table('sent_payment', + sa.Column('sent_payment_id', sa.Integer(), nullable=False), + sa.Column('created', sa.DateTime(), server_default=sa.text('(CURRENT_TIMESTAMP)'), nullable=False), + sa.Column('offer_id', sa.Integer(), nullable=False), + sa.Column('peer_id', sa.Integer(), nullable=False), + sa.Column('squeak_hash', sa.String(length=64), nullable=False), + sa.Column('payment_hash', sa.String(length=64), nullable=False), + sa.Column('secret_key', sa.String(length=64), nullable=False), + sa.Column('price_msat', sa.Integer(), nullable=False), + sa.Column('node_pubkey', sa.String(length=66), nullable=False), + sa.PrimaryKeyConstraint('sent_payment_id') + ) + op.create_table('squeak', + sa.Column('hash', sa.String(length=64), nullable=False), + sa.Column('created', sa.DateTime(), server_default=sa.text('(CURRENT_TIMESTAMP)'), nullable=False), + sa.Column('squeak', sa.Binary(), nullable=False), + sa.Column('hash_reply_sqk', sa.String(length=64), nullable=False), + sa.Column('hash_block', sa.String(length=64), nullable=False), + sa.Column('n_block_height', sa.Integer(), nullable=False), + sa.Column('n_time', sa.Integer(), nullable=False), + sa.Column('author_address', sa.String(length=35), nullable=False), + sa.Column('secret_key', sa.String(length=64), nullable=True), + sa.Column('block_header', sa.Binary(), nullable=False), + sa.PrimaryKeyConstraint('hash') + ) + with op.batch_alter_table('squeak', schema=None) as batch_op: + batch_op.create_index(batch_op.f('ix_squeak_author_address'), ['author_address'], unique=False) + + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + with op.batch_alter_table('squeak', schema=None) as batch_op: + batch_op.drop_index(batch_op.f('ix_squeak_author_address')) + + op.drop_table('squeak') + op.drop_table('sent_payment') + op.drop_table('sent_offer') + op.drop_table('received_payment') + op.drop_table('profile') + op.drop_table('peer') + op.drop_table('offer') + # ### end Alembic commands ### diff --git a/squeaknode/db/models.py b/squeaknode/db/models.py index 3c20b21d..6998cdea 100644 --- a/squeaknode/db/models.py +++ b/squeaknode/db/models.py @@ -11,6 +11,7 @@ from sqlalchemy import ( Table, func, ) +from sqlalchemy.types import TIMESTAMP logger = logging.getLogger(__name__) @@ -24,7 +25,7 @@ class Models: "squeak", self.metadata, Column("hash", String(64), primary_key=True), - Column("created", DateTime, server_default=func.now(), nullable=False), + Column("created", TIMESTAMP, server_default=func.now(), nullable=False), Column("squeak", Binary, nullable=False), Column("hash_reply_sqk", String(64), nullable=False), Column("hash_block", String(64), nullable=False), @@ -39,7 +40,7 @@ class Models: "profile", self.metadata, Column("profile_id", Integer, primary_key=True), - Column("created", DateTime, server_default=func.now(), nullable=False), + Column("created", TIMESTAMP, server_default=func.now(), nullable=False), Column("profile_name", String, unique=True, nullable=False), Column("private_key", Binary), Column("address", String(35), unique=True, nullable=False), @@ -51,7 +52,7 @@ class Models: "peer", self.metadata, Column("id", Integer, primary_key=True), - Column("created", DateTime, server_default=func.now(), nullable=False), + Column("created", TIMESTAMP, server_default=func.now(), nullable=False), Column("peer_name", String), Column("server_host", String, nullable=False), Column("server_port", Integer, nullable=False), @@ -63,7 +64,7 @@ class Models: "offer", self.metadata, Column("offer_id", Integer, primary_key=True), - Column("created", DateTime, server_default=func.now(), nullable=False), + Column("created", TIMESTAMP, server_default=func.now(), nullable=False), Column("squeak_hash", String(64), nullable=False), Column("payment_hash", String(64), nullable=False), Column("nonce", String(64), nullable=False), @@ -82,7 +83,7 @@ class Models: "sent_payment", self.metadata, Column("sent_payment_id", Integer, primary_key=True), - Column("created", DateTime, server_default=func.now(), nullable=False), + Column("created", TIMESTAMP, server_default=func.now(), nullable=False), Column("offer_id", Integer, nullable=False), Column("peer_id", Integer, nullable=False), Column("squeak_hash", String(64), nullable=False), @@ -96,7 +97,7 @@ class Models: "sent_offer", self.metadata, Column("sent_offer_id", Integer, primary_key=True), - Column("created", DateTime, server_default=func.now(), nullable=False), + Column("created", TIMESTAMP, server_default=func.now(), nullable=False), Column("squeak_hash", String(64), nullable=False), Column("payment_hash", String(64), unique=True, nullable=False), Column("secret_key", String(64), nullable=False), @@ -112,7 +113,7 @@ class Models: "received_payment", self.metadata, Column("received_payment_id", Integer, primary_key=True), - Column("created", DateTime, server_default=func.now(), nullable=False), + Column("created", TIMESTAMP, server_default=func.now(), nullable=False), Column("squeak_hash", String(64), nullable=False), Column("payment_hash", String(64), unique=True, nullable=False), Column("price_msat", Integer, nullable=False), diff --git a/squeaknode/db/squeak_db.py b/squeaknode/db/squeak_db.py index 58efc8f6..90f4cac8 100644 --- a/squeaknode/db/squeak_db.py +++ b/squeaknode/db/squeak_db.py @@ -1,6 +1,7 @@ import logging from contextlib import contextmanager from datetime import datetime, timedelta +import time import sqlalchemy from sqlalchemy import func, literal @@ -307,7 +308,7 @@ class SqueakDb: select([self.squeaks.c.hash]) .where(self.squeaks.c.author_address.in_(addresses)) .where( - self.squeaks.c.created > datetime.utcnow() - timedelta(seconds=interval_seconds) + self.squeaks.c.created > time.time() - interval_seconds ) .where( or_( @@ -833,7 +834,7 @@ class SqueakDb: def delete_expired_offers(self): """ Delete all expired offers. """ s = self.offers.delete().where( - datetime.utcnow().timestamp() > self.offers.c.invoice_timestamp + self.offers.c.invoice_expiry + time.time() > self.offers.c.invoice_timestamp + self.offers.c.invoice_expiry ) with self.get_connection() as connection: res = connection.execute(s) @@ -967,7 +968,7 @@ class SqueakDb: def delete_expired_sent_offers(self): """ Delete all expired sent offers. """ s = self.sent_offers.delete().where( - datetime.utcnow().timestamp() > self.sent_offers.c.invoice_timestamp + self.sent_offers.c.invoice_expiry + time.time() > self.sent_offers.c.invoice_timestamp + self.sent_offers.c.invoice_expiry ) with self.get_connection() as connection: res = connection.execute(s)