From 34ec5deee85214927eec5288049442fb90571b97 Mon Sep 17 00:00:00 2001 From: Jonathan Zernik Date: Sun, 27 Dec 2020 04:59:59 -0800 Subject: [PATCH] Use python slim docker base image (#516) * Use python slim image for both squeaknode and test * Clean up dockerfiles * Reorder dockerfile for squeaknode * Load alembic config file from packaged module * Alembic directory is now loaded from packaged module * Got multi-stage docker build working * Removed old app directory from docker config * Remove commented package data arg from setup.py * Remove unused alembic graft from manifest * Fix extra space in alembic revision --- MANIFEST.in | 2 ++ docker/docker-compose.yml | 2 +- docker/squeaknode/Dockerfile | 30 +++++++++++-------- docker/test/Dockerfile | 23 +++++--------- install-rpc.sh | 11 ------- itests/docker-compose.yml | 4 +-- alembic.ini => squeaknode/db/alembic.ini | 4 ++- {alembic => squeaknode/db/alembic}/README | 0 {alembic => squeaknode/db/alembic}/env.py | 0 .../db/alembic}/script.py.mako | 0 .../versions/56e24c42b105_initialize_all.py | 2 +- squeaknode/db/migrations.py | 10 ++++++- 12 files changed, 42 insertions(+), 46 deletions(-) delete mode 100755 install-rpc.sh rename alembic.ini => squeaknode/db/alembic.ini (94%) rename {alembic => squeaknode/db/alembic}/README (100%) rename {alembic => squeaknode/db/alembic}/env.py (100%) rename {alembic => squeaknode/db/alembic}/script.py.mako (100%) rename {alembic => squeaknode/db/alembic}/versions/56e24c42b105_initialize_all.py (99%) diff --git a/MANIFEST.in b/MANIFEST.in index 4e7d4843..3cd2dd97 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -2,3 +2,5 @@ include README.md include createdb.sql graft squeaknode/admin/webapp/templates graft squeaknode/admin/webapp/static +include squeaknode/db/alembic.ini +graft squeaknode/db/alembic diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index aae3cedc..d886fba5 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -40,7 +40,7 @@ services: - SQUEAKNODE_SYNC_INTERVAL_S volumes: - ~/.lnd:/root/.lnd - - ./config.ini:/app/config.ini + - ./config.ini:/config.ini - ~/.sqk:/root/.sqk ports: - 8774:8774 diff --git a/docker/squeaknode/Dockerfile b/docker/squeaknode/Dockerfile index 9071d446..27f7b75f 100644 --- a/docker/squeaknode/Dockerfile +++ b/docker/squeaknode/Dockerfile @@ -1,31 +1,32 @@ -FROM ubuntu:20.04 - -ENV DEBIAN_FRONTEND=noninteractive -RUN apt-get update && apt-get install -y \ - python3-pip \ - python3-psycopg2 \ - git +FROM python:3.8-slim AS compile-image WORKDIR / +RUN apt-get update && apt-get install -y libpq-dev gcc libffi-dev + +RUN python -m venv /opt/venv +# Make sure we use the virtualenv: +ENV PATH="/opt/venv/bin:$PATH" + COPY requirements.txt / COPY libs /libs -RUN ls -l - -RUN pip3 install --upgrade pip && \ - pip3 install -r requirements.txt +RUN pip install psycopg2 && \ + pip install -r requirements.txt WORKDIR /app # Copy the source code. -COPY alembic ./alembic COPY squeaknode ./squeaknode COPY proto ./proto -COPY alembic.ini LICENSE MANIFEST.in README.md requirements.txt setup.cfg setup.py ./ +COPY LICENSE MANIFEST.in README.md requirements.txt setup.cfg setup.py ./ RUN python3 setup.py install +FROM python:3.8-slim + +COPY --from=compile-image /opt/venv /opt/venv + EXPOSE 8774 EXPOSE 8994 EXPOSE 18774 @@ -33,6 +34,9 @@ EXPOSE 18994 # Web server EXPOSE 12994 +# Make sure we use the virtualenv: +ENV PATH="/opt/venv/bin:$PATH" + # Copy the entrypoint script. COPY "docker/squeaknode/start-squeaknode.sh" . RUN chmod +x start-squeaknode.sh diff --git a/docker/test/Dockerfile b/docker/test/Dockerfile index 88655436..900a63d6 100644 --- a/docker/test/Dockerfile +++ b/docker/test/Dockerfile @@ -1,29 +1,20 @@ -FROM ubuntu:20.04 - -ENV DEBIAN_FRONTEND=noninteractive -RUN apt-get update && apt-get install -y \ - python3-pip \ - git +FROM python:3.8-slim COPY requirements-itest.txt / COPY libs /libs -RUN pip3 install --upgrade pip && \ - pip3 install -r requirements-itest.txt +RUN pip3 install -r requirements-itest.txt WORKDIR /app -# Download the rpc files. -RUN git clone https://github.com/googleapis/googleapis.git - COPY proto ./proto -COPY install-rpc.sh itests/test.sh squeaknode/lightning/lnd_lightning_client.py ./ +COPY itests/test.sh squeaknode/lightning/lnd_lightning_client.py ./ COPY itests/tests ./tests -# Install the gRPC files. -RUN chmod +x install-rpc.sh && \ - ./install-rpc.sh +RUN python3 -m grpc_tools.protoc --proto_path=. --python_out=. --grpc_python_out=. \ + proto/lnd.proto \ + proto/squeak_server.proto \ + proto/squeak_admin.proto RUN chmod +x test.sh - CMD ["bash", "test.sh"] diff --git a/install-rpc.sh b/install-rpc.sh deleted file mode 100755 index 712c3c68..00000000 --- a/install-rpc.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/usr/bin/env bash - -if [ ! -d "googleapis" ]; then - git clone https://github.com/googleapis/googleapis.git -fi - -echo "Installing RPC protocol files" -python3 -m grpc_tools.protoc --proto_path=googleapis:. --python_out=. --grpc_python_out=. \ - proto/lnd.proto \ - proto/squeak_server.proto \ - proto/squeak_admin.proto diff --git a/itests/docker-compose.yml b/itests/docker-compose.yml index 09939774..bfca808d 100644 --- a/itests/docker-compose.yml +++ b/itests/docker-compose.yml @@ -79,7 +79,7 @@ services: volumes: - test_shared:/rpc - test_lnd_server_dir:/root/.lnd - - ./config.ini:/app/config.ini + - ./config.ini:/config.ini links: - "btcd:btcd" - "lnd_server:lnd" @@ -96,7 +96,7 @@ services: volumes: - test_shared:/rpc - test_lnd_client_dir:/root/.lnd - - ./config.ini:/app/config.ini + - ./config.ini:/config.ini links: - "btcd:btcd" - "lnd_client:lnd" diff --git a/alembic.ini b/squeaknode/db/alembic.ini similarity index 94% rename from alembic.ini rename to squeaknode/db/alembic.ini index 0c81e8a2..794cb14b 100644 --- a/alembic.ini +++ b/squeaknode/db/alembic.ini @@ -2,7 +2,8 @@ [alembic] # path to migration scripts -script_location = alembic +script_location = %(here)s/alembic +#script_location = alembic # template used to generate migration files # file_template = %%(rev)s_%%(slug)s @@ -30,6 +31,7 @@ script_location = alembic # to alembic/versions. When using multiple version # directories, initial revisions must be specified with --version-path # version_locations = %(here)s/bar %(here)s/bat alembic/versions +version_locations = %(here)s/alembic/versions # the output encoding used when revision files # are written from script.py.mako diff --git a/alembic/README b/squeaknode/db/alembic/README similarity index 100% rename from alembic/README rename to squeaknode/db/alembic/README diff --git a/alembic/env.py b/squeaknode/db/alembic/env.py similarity index 100% rename from alembic/env.py rename to squeaknode/db/alembic/env.py diff --git a/alembic/script.py.mako b/squeaknode/db/alembic/script.py.mako similarity index 100% rename from alembic/script.py.mako rename to squeaknode/db/alembic/script.py.mako diff --git a/alembic/versions/56e24c42b105_initialize_all.py b/squeaknode/db/alembic/versions/56e24c42b105_initialize_all.py similarity index 99% rename from alembic/versions/56e24c42b105_initialize_all.py rename to squeaknode/db/alembic/versions/56e24c42b105_initialize_all.py index 5edeccf9..df71c6e0 100644 --- a/alembic/versions/56e24c42b105_initialize_all.py +++ b/squeaknode/db/alembic/versions/56e24c42b105_initialize_all.py @@ -1,7 +1,7 @@ """Initialize all Revision ID: 56e24c42b105 -Revises: +Revises: Create Date: 2020-12-22 19:30:23.837560 """ diff --git a/squeaknode/db/migrations.py b/squeaknode/db/migrations.py index dbdb56ed..2827259a 100644 --- a/squeaknode/db/migrations.py +++ b/squeaknode/db/migrations.py @@ -1,10 +1,18 @@ +import logging + +from pkg_resources import resource_filename + from alembic import command from alembic.config import Config +logger = logging.getLogger(__name__) + + def run_migrations(engine): """ Run migrations. """ - alembic_cfg = Config("alembic.ini") + alembic_conf_path = resource_filename(__name__, 'alembic.ini') + alembic_cfg = Config(alembic_conf_path) alembic_cfg.attributes["configure_logger"] = False with engine.begin() as connection: alembic_cfg.attributes["connection"] = connection