From 45ffeb78796544349de7b2e83a4c33d434e47578 Mon Sep 17 00:00:00 2001 From: Matt Jia Date: Nov 21 2017 11:47:52 +0000 Subject: Adds database migration support --- diff --git a/MANIFEST.in b/MANIFEST.in index 50e52f6..b121087 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -5,10 +5,10 @@ include requirements.txt include pytest.ini include tests/*.py tests/client_secrets.json include tox.ini -include run-dev-server.py include fedmsg.d/*.py recursive-include systemd * recursive-include conf * exclude conf/settings.py exclude conf/client.conf recursive-include docs *.py *.rst *.inv Makefile +recursive-include waiverdb/migrations *.ini *mako README diff --git a/README.md b/README.md index e4d43aa..0cf7b31 100644 --- a/README.md +++ b/README.md @@ -10,9 +10,14 @@ Install dependencies: $ sudo dnf builddep waiverdb.spec -Run the server: +Run the server:: - $ python run-dev-server.py + $ cp conf/settings.py.example conf/settings.py + $ DEV=true python waiverdb/manage.py run -h localhost -p 5004 --debugger + +Migrate the db:: + + $ DEV=true python waiverdb/manage.py db upgrade The server is now running at and API calls can be sent to . All data is stored inside `/var/tmp/waiverdb_db.sqlite`. diff --git a/docs/developer-guide.rst b/docs/developer-guide.rst index 0299f60..41bd0b2 100644 --- a/docs/developer-guide.rst +++ b/docs/developer-guide.rst @@ -22,7 +22,12 @@ Install the project:: Run the server:: - $ python run-dev-server.py + $ cp conf/settings.py.example conf/settings.py + $ DEV=true python waiverdb/manage.py run -h localhost -p 5004 --debugger + +Migrate the db:: + + $ DEV=true python waiverdb/manage.py db upgrade The server is now running at on `localhost port 5004`_. Consult the :ref:`rest-api` for available API calls. All data is stored inside diff --git a/openshift/waiverdb-test-template.yaml b/openshift/waiverdb-test-template.yaml index 541d1d3..20b1903 100644 --- a/openshift/waiverdb-test-template.yaml +++ b/openshift/waiverdb-test-template.yaml @@ -1,6 +1,6 @@ -# Template to produce a new test environment in OpenShift. Uses OpenID Connect -# against iddev.fedorainfracloud.org for authentication, and ephemeral storage +# Template to produce a new test environment in OpenShift. Uses OpenID Connect +# against iddev.fedorainfracloud.org for authentication, and ephemeral storage # for Postgres data. # # To create an environment from the template, process and apply it: @@ -180,6 +180,10 @@ objects: image: "docker-registry.engineering.redhat.com/factory2/waiverdb:${WAIVERDB_APP_VERSION}" ports: - containerPort: 8080 + lifecycle: + postStart: + exec: + command: [ /bin/sh, -i, -c, "waiverdb db upgrade" ] volumeMounts: - name: config-volume mountPath: /etc/waiverdb @@ -210,8 +214,8 @@ objects: httpGet: path: /healthcheck port: 8080 - # Limit to 384MB memory. This is probably *not* enough but it is - # necessary in the current environment to allow for 2 replicas and + # Limit to 384MB memory. This is probably *not* enough but it is + # necessary in the current environment to allow for 2 replicas and # rolling updates, without hitting the (very aggressive) memory quota. resources: limits: diff --git a/requirements.txt b/requirements.txt index 8dd7afb..14951b7 100644 --- a/requirements.txt +++ b/requirements.txt @@ -15,6 +15,7 @@ systemd pytest >= 2.4.2 mock stomp.py +Flask-Migrate # Documentation requirements sphinx diff --git a/run-dev-server.py b/run-dev-server.py deleted file mode 100644 index bf92062..0000000 --- a/run-dev-server.py +++ /dev/null @@ -1,14 +0,0 @@ -#!/usr/bin/python - -# SPDX-License-Identifier: GPL-2.0+ - -from waiverdb.app import create_app, init_db - -if __name__ == '__main__': - app = create_app('waiverdb.config.DevelopmentConfig') - init_db(app) - app.run( - host=app.config['HOST'], - port=app.config['PORT'], - debug=app.config['DEBUG'], - ) diff --git a/setup.py b/setup.py index 4d71a76..c2950fc 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ import os import re -from setuptools import setup +from setuptools import setup, find_packages here = os.path.abspath(os.path.dirname(__file__)) with open(os.path.join(here, 'README.md')) as fd: @@ -34,9 +34,10 @@ setup(name='waiverdb', author='Red Hat, Inc.', author_email='qa-devel@lists.fedoraproject.org', license='GPLv2+', - packages=['waiverdb', 'waiverdb.models'], - package_dir={'waiverdb': 'waiverdb'}, + packages=find_packages(exclude=['tests']), + include_package_data=True, entry_points={ - 'console_scripts': ['waiverdb-cli=waiverdb.cli:cli'], + 'console_scripts': ['waiverdb-cli=waiverdb.cli:cli', + 'waiverdb=waiverdb.manage:cli'], }, ) diff --git a/tox.ini b/tox.ini index d4c5c88..ffe318a 100644 --- a/tox.ini +++ b/tox.ini @@ -33,7 +33,7 @@ commands = [flake8] show-source = True max-line-length = 100 -exclude = .git,.tox,dist,*egg,env_waiverdb,*fedmsg.d,docs +exclude = .git,.tox,dist,*egg,env_waiverdb,*fedmsg.d,docs,conf,waiverdb/migrations # E124: closing bracket does not match visual indentation ignore = E124 diff --git a/waiverdb.spec b/waiverdb.spec index de6dedc..642aa75 100644 --- a/waiverdb.spec +++ b/waiverdb.spec @@ -30,6 +30,7 @@ BuildRequires: python2-mock BuildRequires: python2-flask-oidc BuildRequires: python2-configparser BuildRequires: python2-click +BuildRequires: python2-flask-migrate BuildRequires: stomppy %else # EPEL7 uses python- naming BuildRequires: python-setuptools @@ -44,6 +45,7 @@ BuildRequires: python-mock BuildRequires: python-flask-oidc BuildRequires: python-click BuildRequires: python-configparser +BuildRequires: python-flask-migrate BuildRequires: stomppy %endif BuildRequires: fedmsg @@ -65,6 +67,7 @@ Requires: python2-mock Requires: python2-flask-oidc Requires: python2-click Requires: python2-configparser +Requires: python2-flask-migrate Requires: stomppy %else Requires: python-flask @@ -77,6 +80,7 @@ Requires: python-mock Requires: python-flask-oidc Requires: python-click Requires: python-configparser +Requires: python-flask-migrate Requires: stomppy %endif Requires: fedmsg @@ -145,6 +149,7 @@ py.test tests/ %{python2_sitelib}/%{name}*.egg-info %{_unitdir}/%{name}.service %{_unitdir}/%{name}.socket +%attr(755,root,root) %{_bindir}/waiverdb %files common %license COPYING diff --git a/waiverdb/app.py b/waiverdb/app.py index 789f061..9588b78 100644 --- a/waiverdb/app.py +++ b/waiverdb/app.py @@ -4,6 +4,7 @@ import os import urlparse from flask import Flask +from flask_migrate import Migrate from sqlalchemy import event from waiverdb.events import publish_new_waiver @@ -74,6 +75,10 @@ def create_app(config_obj=None): app.oidc = OpenIDConnect(app) # initialize db db.init_app(app) + # initialize db migrations + migrations_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)), + 'migrations') + Migrate(app, db, directory=migrations_dir) # initialize logging init_logging(app) # register blueprints diff --git a/waiverdb/config.py b/waiverdb/config.py index e21b262..6c46c46 100644 --- a/waiverdb/config.py +++ b/waiverdb/config.py @@ -29,6 +29,7 @@ class Config(object): MESSAGE_BUS_PUBLISH = True # Specify fedmsg or stomp for publishing messages MESSAGE_PUBLISHER = 'fedmsg' + SQLALCHEMY_TRACK_MODIFICATIONS = True class ProductionConfig(Config): diff --git a/waiverdb/manage.py b/waiverdb/manage.py new file mode 100644 index 0000000..c69b971 --- /dev/null +++ b/waiverdb/manage.py @@ -0,0 +1,18 @@ +# SPDX-License-Identifier: GPL-2.0+ + +import click +from flask.cli import FlaskGroup + + +def create_waiver_app(_): + from waiverdb.app import create_app # noqa: F401 + return create_app() + + +@click.group(cls=FlaskGroup, create_app=create_waiver_app) +def cli(): + pass + + +if __name__ == '__main__': + cli() # pylint: disable=E1120 diff --git a/waiverdb/migrations/README b/waiverdb/migrations/README new file mode 100644 index 0000000..98e4f9c --- /dev/null +++ b/waiverdb/migrations/README @@ -0,0 +1 @@ +Generic single-database configuration. \ No newline at end of file diff --git a/waiverdb/migrations/__init__.py b/waiverdb/migrations/__init__.py new file mode 100644 index 0000000..1016702 --- /dev/null +++ b/waiverdb/migrations/__init__.py @@ -0,0 +1 @@ +# SPDX-License-Identifier: GPL-2.0+ diff --git a/waiverdb/migrations/alembic.ini b/waiverdb/migrations/alembic.ini new file mode 100644 index 0000000..f8ed480 --- /dev/null +++ b/waiverdb/migrations/alembic.ini @@ -0,0 +1,45 @@ +# A generic, single database configuration. + +[alembic] +# template used to generate migration files +# file_template = %%(rev)s_%%(slug)s + +# set to 'true' to run the environment during +# the 'revision' command, regardless of autogenerate +# revision_environment = false + + +# Logging configuration +[loggers] +keys = root,sqlalchemy,alembic + +[handlers] +keys = console + +[formatters] +keys = generic + +[logger_root] +level = WARN +handlers = console +qualname = + +[logger_sqlalchemy] +level = WARN +handlers = +qualname = sqlalchemy.engine + +[logger_alembic] +level = INFO +handlers = +qualname = alembic + +[handler_console] +class = StreamHandler +args = (sys.stderr,) +level = NOTSET +formatter = generic + +[formatter_generic] +format = %(levelname)-5.5s [%(name)s] %(message)s +datefmt = %H:%M:%S diff --git a/waiverdb/migrations/env.py b/waiverdb/migrations/env.py new file mode 100644 index 0000000..8d12f2e --- /dev/null +++ b/waiverdb/migrations/env.py @@ -0,0 +1,87 @@ +from __future__ import with_statement +from alembic import context +from sqlalchemy import engine_from_config, pool +from logging.config import fileConfig +import logging + +# this is the Alembic Config object, which provides +# access to the values within the .ini file in use. +config = context.config + +# Interpret the config file for Python logging. +# This line sets up loggers basically. +fileConfig(config.config_file_name) +logger = logging.getLogger('alembic.env') + +# add your model's MetaData object here +# for 'autogenerate' support +# from myapp import mymodel +# target_metadata = mymodel.Base.metadata +from flask import current_app +config.set_main_option('sqlalchemy.url', + current_app.config.get('SQLALCHEMY_DATABASE_URI')) +target_metadata = current_app.extensions['migrate'].db.metadata + +# other values from the config, defined by the needs of env.py, +# can be acquired: +# my_important_option = config.get_main_option("my_important_option") +# ... etc. + + +def run_migrations_offline(): + """Run migrations in 'offline' mode. + + This configures the context with just a URL + and not an Engine, though an Engine is acceptable + here as well. By skipping the Engine creation + we don't even need a DBAPI to be available. + + Calls to context.execute() here emit the given string to the + script output. + + """ + url = config.get_main_option("sqlalchemy.url") + context.configure(url=url) + + with context.begin_transaction(): + context.run_migrations() + + +def run_migrations_online(): + """Run migrations in 'online' mode. + + In this scenario we need to create an Engine + and associate a connection with the context. + + """ + + # this callback is used to prevent an auto-migration from being generated + # when there are no changes to the schema + # reference: http://alembic.readthedocs.org/en/latest/cookbook.html + def process_revision_directives(context, revision, directives): # pylint: disable=W0621,W0613 + if getattr(config.cmd_opts, 'autogenerate', False): + script = directives[0] + if script.upgrade_ops.is_empty(): + directives[:] = [] + logger.info('No changes in schema detected.') + + engine = engine_from_config(config.get_section(config.config_ini_section), + prefix='sqlalchemy.', + poolclass=pool.NullPool) + + connection = engine.connect() + context.configure(connection=connection, + target_metadata=target_metadata, + process_revision_directives=process_revision_directives, + **current_app.extensions['migrate'].configure_args) + + try: + with context.begin_transaction(): + context.run_migrations() + finally: + connection.close() + +if context.is_offline_mode(): + run_migrations_offline() +else: + run_migrations_online() diff --git a/waiverdb/migrations/script.py.mako b/waiverdb/migrations/script.py.mako new file mode 100644 index 0000000..9570201 --- /dev/null +++ b/waiverdb/migrations/script.py.mako @@ -0,0 +1,22 @@ +"""${message} + +Revision ID: ${up_revision} +Revises: ${down_revision} +Create Date: ${create_date} + +""" + +# revision identifiers, used by Alembic. +revision = ${repr(up_revision)} +down_revision = ${repr(down_revision)} + +from alembic import op +import sqlalchemy as sa +${imports if imports else ""} + +def upgrade(): + ${upgrades if upgrades else "pass"} + + +def downgrade(): + ${downgrades if downgrades else "pass"} diff --git a/waiverdb/migrations/versions/0a27a8ad723a_initial_migration.py b/waiverdb/migrations/versions/0a27a8ad723a_initial_migration.py new file mode 100644 index 0000000..ec3137c --- /dev/null +++ b/waiverdb/migrations/versions/0a27a8ad723a_initial_migration.py @@ -0,0 +1,35 @@ +"""Initializes migration + +Revision ID: 0a27a8ad723a +Revises: None +Create Date: 2017-11-21 10:57:17.056587 + +""" + +# revision identifiers, used by Alembic. +revision = '0a27a8ad723a' +down_revision = None + +from alembic import op +import sqlalchemy as sa + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.create_table('waiver', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('result_id', sa.Integer(), nullable=False), + sa.Column('username', sa.String(length=255), nullable=False), + sa.Column('product_version', sa.String(length=200), nullable=False), + sa.Column('waived', sa.Boolean(), nullable=False), + sa.Column('comment', sa.Text(), nullable=True), + sa.Column('timestamp', sa.DateTime(), nullable=True), + sa.PrimaryKeyConstraint('id') + ) + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_table('waiver') + # ### end Alembic commands ### diff --git a/waiverdb/migrations/versions/__init__.py b/waiverdb/migrations/versions/__init__.py new file mode 100644 index 0000000..1016702 --- /dev/null +++ b/waiverdb/migrations/versions/__init__.py @@ -0,0 +1 @@ +# SPDX-License-Identifier: GPL-2.0+ diff --git a/waiverdb/wsgi.py b/waiverdb/wsgi.py index 709c003..c79533a 100644 --- a/waiverdb/wsgi.py +++ b/waiverdb/wsgi.py @@ -1,5 +1,4 @@ # SPDX-License-Identifier: GPL-2.0+ -from waiverdb.app import create_app, init_db +from waiverdb.app import create_app app = create_app() -init_db(app)