Add alembic (#418)

* Add alembic with init command

* Create separate models class

* Add target metadata to alembic config

* Got initial migration working

* Move migration code into separate module

* Add documentation for migrations

alembic.ini

@@ -0,0 +1,85 @@
+# A generic, single database configuration.
+
+[alembic]
+# path to migration scripts
+script_location = alembic
+
+# template used to generate migration files
+# file_template = %%(rev)s_%%(slug)s
+
+# timezone to use when rendering the date
+# within the migration file as well as the filename.
+# string value is passed to dateutil.tz.gettz()
+# leave blank for localtime
+# timezone =
+
+# max length of characters to apply to the
+# "slug" field
+# truncate_slug_length = 40
+
+# set to 'true' to run the environment during
+# the 'revision' command, regardless of autogenerate
+# revision_environment = false
+
+# set to 'true' to allow .pyc and .pyo files without
+# a source .py file to be detected as revisions in the
+# versions/ directory
+# sourceless = false
+
+# version location specification; this defaults
+# to alembic/versions.  When using multiple version
+# directories, initial revisions must be specified with --version-path
+# version_locations = %(here)s/bar %(here)s/bat alembic/versions
+
+# the output encoding used when revision files
+# are written from script.py.mako
+# output_encoding = utf-8
+
+sqlalchemy.url = sqlite://///home/yzernik/.sqk/data/testnet/data.db
+
+
+[post_write_hooks]
+# post_write_hooks defines scripts or Python functions that are run
+# on newly generated revision scripts.  See the documentation for further
+# detail and examples
+
+# format using "black" - use the console_scripts runner, against the "black" entrypoint
+# hooks=black
+# black.type=console_scripts
+# black.entrypoint=black
+# black.options=-l 79
+
+# Logging configuration
+[loggers]
+keys = root,sqlalchemy,alembic
+
+[handlers]
+keys = console
+
+[formatters]
+keys = generic
+
+[logger_root]
+level = WARN
+handlers = console
+qualname =
+
+[logger_sqlalchemy]
+level = WARN
+handlers =
+qualname = sqlalchemy.engine
+
+[logger_alembic]
+level = INFO
+handlers =
+qualname = alembic
+
+[handler_console]
+class = StreamHandler
+args = (sys.stderr,)
+level = NOTSET
+formatter = generic
+
+[formatter_generic]
+format = %(levelname)-5.5s [%(name)s] %(message)s
+datefmt = %H:%M:%S

alembic/README

@@ -0,0 +1 @@
+Generic single-database configuration.

alembic/env.py

@@ -0,0 +1,86 @@
+from logging.config import fileConfig
+
+from sqlalchemy import engine_from_config
+from sqlalchemy import pool
+
+from alembic import context
+
+# this is the Alembic Config object, which provides
+# access to the values within the .ini file in use.
+config = context.config
+
+# Interpret the config file for Python logging.
+# This line sets up loggers basically.
+fileConfig(config.config_file_name)
+
+# add your model's MetaData object here
+# for 'autogenerate' support
+# from myapp import mymodel
+# target_metadata = mymodel.Base.metadata
+from squeaknode.db.models import Models
+target_metadata = Models().metadata
+
+# other values from the config, defined by the needs of env.py,
+# can be acquired:
+# my_important_option = config.get_main_option("my_important_option")
+# ... etc.
+
+
+def run_migrations_offline():
+    """Run migrations in 'offline' mode.
+
+    This configures the context with just a URL
+    and not an Engine, though an Engine is acceptable
+    here as well.  By skipping the Engine creation
+    we don't even need a DBAPI to be available.
+
+    Calls to context.execute() here emit the given string to the
+    script output.
+
+    """
+    url = config.get_main_option("sqlalchemy.url")
+    context.configure(
+        url=url,
+        target_metadata=target_metadata,
+        literal_binds=True,
+        dialect_opts={"paramstyle": "named"},
+    )
+
+    with context.begin_transaction():
+        context.run_migrations()
+
+
+def run_migrations_online():
+    """Run migrations in 'online' mode.
+
+    In this scenario we need to create an Engine
+    and associate a connection with the context.
+
+    """
+    connectable = config.attributes.get('connection', None)
+
+    if connectable is None:
+        # only create Engine if we don't have a Connection
+        # from the outside
+        connectable = engine_from_config(
+            config.get_section(config.config_ini_section),
+            prefix='sqlalchemy.',
+            poolclass=pool.NullPool)
+
+    # when connectable is already a Connection object, calling
+    # connect() gives us a *branched connection*.
+
+    with connectable.connect() as connection:
+        context.configure(
+            connection=connection,
+            target_metadata=target_metadata
+        )
+
+        with context.begin_transaction():
+            context.run_migrations()
+
+
+if context.is_offline_mode():
+    run_migrations_offline()
+else:
+    run_migrations_online()

alembic/script.py.mako

@@ -0,0 +1,24 @@
+"""${message}
+
+Revision ID: ${up_revision}
+Revises: ${down_revision | comma,n}
+Create Date: ${create_date}
+
+"""
+from alembic import op
+import sqlalchemy as sa
+${imports if imports else ""}
+
+# revision identifiers, used by Alembic.
+revision = ${repr(up_revision)}
+down_revision = ${repr(down_revision)}
+branch_labels = ${repr(branch_labels)}
+depends_on = ${repr(depends_on)}
+
+
+def upgrade():
+    ${upgrades if upgrades else "pass"}
+
+
+def downgrade():
+    ${downgrades if downgrades else "pass"}

alembic/versions/e771283d1959_initialize_all.py

@@ -0,0 +1,106 @@
+"""Initialize all
+
+Revision ID: e771283d1959
+Revises: 
+Create Date: 2020-11-08 19:23:46.430021
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = 'e771283d1959'
+down_revision = None
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table('offer',
+    sa.Column('offer_id', sa.Integer(), nullable=False),
+    sa.Column('created', sa.DateTime(), server_default=sa.text('(CURRENT_TIMESTAMP)'), nullable=False),
+    sa.Column('squeak_hash', sa.String(length=64), nullable=False),
+    sa.Column('key_cipher', sa.Binary(), nullable=False),
+    sa.Column('iv', sa.Binary(), nullable=False),
+    sa.Column('payment_hash', sa.String(length=64), nullable=False),
+    sa.Column('invoice_timestamp', sa.Integer(), nullable=False),
+    sa.Column('invoice_expiry', sa.Integer(), nullable=False),
+    sa.Column('price_msat', sa.Integer(), nullable=False),
+    sa.Column('payment_request', sa.String(), nullable=False),
+    sa.Column('destination', sa.String(length=66), nullable=False),
+    sa.Column('node_host', sa.String(), nullable=False),
+    sa.Column('node_port', sa.Integer(), nullable=False),
+    sa.Column('peer_id', sa.Integer(), nullable=False),
+    sa.PrimaryKeyConstraint('offer_id')
+    )
+    op.create_table('peer',
+    sa.Column('id', sa.Integer(), nullable=False),
+    sa.Column('created', sa.DateTime(), server_default=sa.text('(CURRENT_TIMESTAMP)'), nullable=False),
+    sa.Column('peer_name', sa.String(), nullable=True),
+    sa.Column('server_host', sa.String(), nullable=False),
+    sa.Column('server_port', sa.Integer(), nullable=False),
+    sa.Column('uploading', sa.Boolean(), nullable=False),
+    sa.Column('downloading', sa.Boolean(), nullable=False),
+    sa.PrimaryKeyConstraint('id')
+    )
+    op.create_table('profile',
+    sa.Column('profile_id', sa.Integer(), nullable=False),
+    sa.Column('created', sa.DateTime(), server_default=sa.text('(CURRENT_TIMESTAMP)'), nullable=False),
+    sa.Column('profile_name', sa.String(), nullable=False),
+    sa.Column('private_key', sa.Binary(), nullable=True),
+    sa.Column('address', sa.String(length=35), nullable=False),
+    sa.Column('sharing', sa.Boolean(), nullable=False),
+    sa.Column('following', sa.Boolean(), nullable=False),
+    sa.PrimaryKeyConstraint('profile_id'),
+    sa.UniqueConstraint('address'),
+    sa.UniqueConstraint('profile_name')
+    )
+    op.create_table('sent_payment',
+    sa.Column('sent_payment_id', sa.Integer(), nullable=False),
+    sa.Column('created', sa.DateTime(), server_default=sa.text('(CURRENT_TIMESTAMP)'), nullable=False),
+    sa.Column('offer_id', sa.Integer(), nullable=False),
+    sa.Column('peer_id', sa.Integer(), nullable=False),
+    sa.Column('squeak_hash', sa.String(length=64), nullable=False),
+    sa.Column('preimage_hash', sa.String(length=64), nullable=False),
+    sa.Column('preimage', sa.String(length=64), nullable=False),
+    sa.Column('amount', sa.Integer(), nullable=False),
+    sa.Column('node_pubkey', sa.String(length=66), nullable=False),
+    sa.Column('preimage_is_valid', sa.Boolean(), nullable=False),
+    sa.PrimaryKeyConstraint('sent_payment_id')
+    )
+    op.create_table('squeak',
+    sa.Column('hash', sa.String(length=64), nullable=False),
+    sa.Column('created', sa.DateTime(), server_default=sa.text('(CURRENT_TIMESTAMP)'), nullable=False),
+    sa.Column('n_version', sa.Integer(), nullable=False),
+    sa.Column('hash_enc_content', sa.String(length=64), nullable=False),
+    sa.Column('hash_reply_sqk', sa.String(length=64), nullable=False),
+    sa.Column('hash_block', sa.String(length=64), nullable=False),
+    sa.Column('n_block_height', sa.Integer(), nullable=False),
+    sa.Column('vch_script_pub_key', sa.Binary(), nullable=False),
+    sa.Column('vch_encryption_key', sa.Binary(), nullable=False),
+    sa.Column('enc_data_key', sa.String(), nullable=False),
+    sa.Column('iv', sa.String(length=64), nullable=False),
+    sa.Column('n_time', sa.Integer(), nullable=False),
+    sa.Column('n_nonce', sa.BigInteger(), nullable=False),
+    sa.Column('enc_content', sa.String(length=2272), nullable=False),
+    sa.Column('vch_script_sig', sa.Binary(), nullable=False),
+    sa.Column('author_address', sa.String(length=35), nullable=False),
+    sa.Column('vch_decryption_key', sa.Binary(), nullable=True),
+    sa.Column('block_header', sa.Binary(), nullable=True),
+    sa.PrimaryKeyConstraint('hash')
+    )
+    op.create_index(op.f('ix_squeak_author_address'), 'squeak', ['author_address'], unique=False)
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_index(op.f('ix_squeak_author_address'), table_name='squeak')
+    op.drop_table('squeak')
+    op.drop_table('sent_payment')
+    op.drop_table('profile')
+    op.drop_table('peer')
+    op.drop_table('offer')
+    # ### end Alembic commands ###

docs/DEVELOPMENT.md

@@ -0,0 +1,22 @@
+# development
+
+### DB migration:
+
+Alembic is used for database migrations.
+
+To change any database model, follow these steps.
+
+- Make sure that you have an up-to-date squeaknode with a sqlite database.
+- Make a note of the path to the ".db" sqlite database file. (Usually `~/.sqk/data/testnet/data.db` by default)
+- Make the changes to database models in `squeaknode/db/models.py`
+- Update the `alembic.ini` file to point to the sqlite file from before:
+	```
+	sqlalchemy.url = sqlite://///home/<USER>/.sqk/data/testnet/data.db
+	```
+- Run the command to generate a new alembic migration:
+	```
+	$ virtuelenv venv
+	$ pip install -r requirements.txt
+	$ pip install -e .
+	$ alembic revision --autogenerate -m "<YOUR_MESSAGE>"
+	```

requirements.txt

@@ -12,3 +12,4 @@ protobuf
 flask-login
 Flask-WTF
 flask-cors
+alembic

squeaknode/db/migrations.py

@@ -0,0 +1,10 @@
+from alembic.config import Config
+from alembic import command
+
+
+def run_migrations(engine):
+    """ Run migrations. """
+    alembic_cfg = Config("alembic.ini")
+    with engine.begin() as connection:
+        alembic_cfg.attributes['connection'] = connection
+        command.upgrade(alembic_cfg, "head")

squeaknode/db/models.py

@@ -0,0 +1,119 @@
+import logging
+from contextlib import contextmanager
+from datetime import datetime, timedelta
+
+import sqlalchemy
+from sqlalchemy import (
+    BigInteger,
+    Binary,
+    Boolean,
+    Column,
+    DateTime,
+    Integer,
+    MetaData,
+    String,
+    Table,
+    func,
+    literal,
+)
+from sqlalchemy.sql import and_, or_, select
+from squeak.core import CSqueak
+
+from squeaknode.blockchain.util import parse_block_header
+from squeaknode.core.offer import Offer
+from squeaknode.core.offer_with_peer import OfferWithPeer
+from squeaknode.core.squeak_entry import SqueakEntry
+from squeaknode.core.squeak_entry_with_profile import SqueakEntryWithProfile
+from squeaknode.server.squeak_peer import SqueakPeer
+from squeaknode.server.squeak_profile import SqueakProfile
+from squeaknode.server.sent_payment import SentPayment
+from squeaknode.server.util import get_hash
+
+logger = logging.getLogger(__name__)
+
+
+class Models:
+    def __init__(self, schema=None):
+        self.schema = schema
+        self.metadata = MetaData(schema=schema)
+
+        self.squeaks = Table(
+            "squeak",
+            self.metadata,
+            Column("hash", String(64), primary_key=True),
+            Column("created", DateTime, server_default=func.now(), nullable=False),
+            Column("n_version", Integer, nullable=False),
+            Column("hash_enc_content", String(64), nullable=False),
+            Column("hash_reply_sqk", String(64), nullable=False),
+            Column("hash_block", String(64), nullable=False),
+            Column("n_block_height", Integer, nullable=False),
+            Column("vch_script_pub_key", Binary, nullable=False),
+            Column("vch_encryption_key", Binary, nullable=False),
+            Column("enc_data_key", String, nullable=False),
+            Column("iv", String(64), nullable=False),
+            Column("n_time", Integer, nullable=False),
+            Column("n_nonce", BigInteger, nullable=False),
+            Column("enc_content", String(2272), nullable=False),
+            Column("vch_script_sig", Binary, nullable=False),
+            Column("author_address", String(35), index=True, nullable=False),
+            Column("vch_decryption_key", Binary, nullable=True),
+            Column("block_header", Binary, nullable=True),
+        )
+
+        self.profiles = Table(
+            "profile",
+            self.metadata,
+            Column("profile_id", Integer, primary_key=True),
+            Column("created", DateTime, server_default=func.now(), nullable=False),
+            Column("profile_name", String, unique=True, nullable=False),
+            Column("private_key", Binary),
+            Column("address", String(35), unique=True, nullable=False),
+            Column("sharing", Boolean, nullable=False),
+            Column("following", Boolean, nullable=False),
+        )
+
+        self.peers = Table(
+            "peer",
+            self.metadata,
+            Column("id", Integer, primary_key=True),
+            Column("created", DateTime, server_default=func.now(), nullable=False),
+            Column("peer_name", String),
+            Column("server_host", String, nullable=False),
+            Column("server_port", Integer, nullable=False),
+            Column("uploading", Boolean, nullable=False),
+            Column("downloading", Boolean, nullable=False),
+        )
+
+        self.offers = Table(
+            "offer",
+            self.metadata,
+            Column("offer_id", Integer, primary_key=True),
+            Column("created", DateTime, server_default=func.now(), nullable=False),
+            Column("squeak_hash", String(64), nullable=False),
+            Column("key_cipher", Binary, nullable=False),
+            Column("iv", Binary, nullable=False),
+            Column("payment_hash", String(64), nullable=False),
+            Column("invoice_timestamp", Integer, nullable=False),
+            Column("invoice_expiry", Integer, nullable=False),
+            Column("price_msat", Integer, nullable=False),
+            Column("payment_request", String, nullable=False),
+            Column("destination", String(66), nullable=False),
+            Column("node_host", String, nullable=False),
+            Column("node_port", Integer, nullable=False),
+            Column("peer_id", Integer, nullable=False),
+        )
+
+        self.sent_payments = Table(
+            "sent_payment",
+            self.metadata,
+            Column("sent_payment_id", Integer, primary_key=True),
+            Column("created", DateTime, server_default=func.now(), nullable=False),
+            Column("offer_id", Integer, nullable=False),
+            Column("peer_id", Integer, nullable=False),
+            Column("squeak_hash", String(64), nullable=False),
+            Column("preimage_hash", String(64), nullable=False),
+            Column("preimage", String(64), nullable=False),
+            Column("amount", Integer, nullable=False),
+            Column("node_pubkey", String(66), nullable=False),
+            Column("preimage_is_valid", Boolean, nullable=False),
+        )

squeaknode/db/squeak_db.py

@@ -28,116 +28,56 @@ from squeaknode.server.squeak_peer import SqueakPeer
 from squeaknode.server.squeak_profile import SqueakProfile
 from squeaknode.server.sent_payment import SentPayment
 from squeaknode.server.util import get_hash
+from squeaknode.db.models import Models
+from squeaknode.db.migrations import run_migrations
+
 
 logger = logging.getLogger(__name__)
 
 
+# def run_migrations(script_location: str, dsn: str) -> None:
+#     LOG.info('Running DB migrations in %r on %r', script_location, dsn)
+#     alembic_cfg = Config()
+#     #alembic_cfg.set_main_option('script_location', script_location)
+#     alembic_cfg.set_main_option('sqlalchemy.url', dsn)
+#     command.upgrade(alembic_cfg, 'head')
+
+
 class SqueakDb:
     def __init__(self, engine, schema=None):
         self.engine = engine
         self.schema = schema
-        self.metadata = MetaData(schema=schema)
-
-        self.squeaks = Table(
-            "squeak",
-            self.metadata,
-            Column("hash", String(64), primary_key=True),
-            Column("created", DateTime, server_default=func.now(), nullable=False),
-            Column("n_version", Integer, nullable=False),
-            Column("hash_enc_content", String(64), nullable=False),
-            Column("hash_reply_sqk", String(64), nullable=False),
-            Column("hash_block", String(64), nullable=False),
-            Column("n_block_height", Integer, nullable=False),
-            Column("vch_script_pub_key", Binary, nullable=False),
-            Column("vch_encryption_key", Binary, nullable=False),
-            Column("enc_data_key", String, nullable=False),
-            Column("iv", String(64), nullable=False),
-            Column("n_time", Integer, nullable=False),
-            Column("n_nonce", BigInteger, nullable=False),
-            Column("enc_content", String(2272), nullable=False),
-            Column("vch_script_sig", Binary, nullable=False),
-            Column("author_address", String(35), index=True, nullable=False),
-            Column("vch_decryption_key", Binary, nullable=True),
-            Column("block_header", Binary, nullable=True),
-        )
-
-        self.profiles = Table(
-            "profile",
-            self.metadata,
-            Column("profile_id", Integer, primary_key=True),
-            Column("created", DateTime, server_default=func.now(), nullable=False),
-            Column("profile_name", String, unique=True, nullable=False),
-            Column("private_key", Binary),
-            Column("address", String(35), unique=True, nullable=False),
-            Column("sharing", Boolean, nullable=False),
-            Column("following", Boolean, nullable=False),
-        )
-
-        self.peers = Table(
-            "peer",
-            self.metadata,
-            Column("id", Integer, primary_key=True),
-            Column("created", DateTime, server_default=func.now(), nullable=False),
-            Column("peer_name", String),
-            Column("server_host", String, nullable=False),
-            Column("server_port", Integer, nullable=False),
-            Column("uploading", Boolean, nullable=False),
-            Column("downloading", Boolean, nullable=False),
-        )
-
-        self.offers = Table(
-            "offer",
-            self.metadata,
-            Column("offer_id", Integer, primary_key=True),
-            Column("created", DateTime, server_default=func.now(), nullable=False),
-            Column("squeak_hash", String(64), nullable=False),
-            Column("key_cipher", Binary, nullable=False),
-            Column("iv", Binary, nullable=False),
-            Column("payment_hash", String(64), nullable=False),
-            Column("invoice_timestamp", Integer, nullable=False),
-            Column("invoice_expiry", Integer, nullable=False),
-            Column("price_msat", Integer, nullable=False),
-            Column("payment_request", String, nullable=False),
-            Column("destination", String(66), nullable=False),
-            Column("node_host", String, nullable=False),
-            Column("node_port", Integer, nullable=False),
-            Column("peer_id", Integer, nullable=False),
-        )
-
-        self.sent_payments = Table(
-            "sent_payment",
-            self.metadata,
-            Column("sent_payment_id", Integer, primary_key=True),
-            Column("created", DateTime, server_default=func.now(), nullable=False),
-            Column("offer_id", Integer, nullable=False),
-            Column("peer_id", Integer, nullable=False),
-            Column("squeak_hash", String(64), nullable=False),
-            Column("preimage_hash", String(64), nullable=False),
-            Column("preimage", String(64), nullable=False),
-            Column("amount", Integer, nullable=False),
-            Column("node_pubkey", String(66), nullable=False),
-            Column("preimage_is_valid", Boolean, nullable=False),
-        )
+        self.models = Models(schema=schema)
 
     @contextmanager
     def get_connection(self):
         with self.engine.connect() as connection:
             yield connection
 
-    def create_tables(self):
-        logger.debug("Creating tables...")
-        self.metadata.create_all(self.engine)
-        self.show_tables()
-
-    def show_tables(self):
-        self.metadata.reflect(bind=self.engine)
-        tables = self.metadata.tables.keys()
-        logger.debug("Database tables: {}".format(tables))
-
     def init(self):
         """ Create the tables and indices in the database. """
         logger.debug("SqlAlchemy version: {}".format(sqlalchemy.__version__))
-        self.create_tables()
+        run_migrations(self.engine)
+
+    @property
+    def squeaks(self):
+        return self.models.squeaks
+
+    @property
+    def profiles(self):
+        return self.models.profiles
+
+    @property
+    def peers(self):
+        return self.models.peers
+
+    @property
+    def offers(self):
+        return self.models.offers
+
+    @property
+    def sent_payments(self):
+        return self.models.sent_payments
 
     def insert_squeak(self, squeak):
         """ Insert a new squeak. """