From 6079ad1e062de552c87fa02c911f19216abeb396 Mon Sep 17 00:00:00 2001 From: Fabio Teixeira Date: Oct 19 2017 20:15:31 +0000 Subject: Add suport for database migrations with alembic Signed-off-by: Daniel Moura Signed-off-by: Fabio Teixeira --- diff --git a/README.md b/README.md index 4a9012e..81c2031 100644 --- a/README.md +++ b/README.md @@ -40,6 +40,21 @@ Install the python dependencies using pip pip install -e . pip install "fedmsg[consumers]" +### Environment variables + +Kiskadee database migration tool(alembic) get its database configuration +from a environment variable named DATABASE_TYPE. +If this variable is not defined, then it will assume its running on a developemnt +environment, but for others environments such as test, homologation or +production be sure to set which one are being used. + +Only set DATABASE_TYPE if kiskadee is running on a non development environment. +```bash +export DATABASE_TYPE=db_test +``` + +> To see which data each one of those alembic will use take a look on: util/kiskadee.conf + ### Docker Images To run the static analyzers, you must have @@ -223,6 +238,41 @@ The events that comes to the anitya fetcher are published by Anitya, on this [page](https://apps.fedoraproject.org/datagrepper/raw?category=anitya.) For more info about the Anitya service, read kiskadee documentation. +## Migrations + +Kiskadee uses alembic as its tool for database migration, it has a solid +documentation on: http://alembic.zzzcomputing.com/en/latest + +For short, the most used commands are: + +**To create a new migration** +```bash +alembic revision -m "migration description" +``` + +**To autogenerate a new migration** +```bash +alembic revision --autogenerate +``` + +or + +```bash +alembic revision --autogenerate -m "some migration description" +``` + +**To execute the migrations** +```bash +alembic upgrade head +alembic upgrade +2 +alembic upgrade -1 +alembic upgrade some_revision_id+2 +``` + +**Downgrading** +```bash +alembic downgrade base +``` ## License Copyright (C) 2017 the AUTHORS (see the AUTHORS file) diff --git a/alembic.ini b/alembic.ini new file mode 100644 index 0000000..bc56c59 --- /dev/null +++ b/alembic.ini @@ -0,0 +1,74 @@ +# A generic, single database configuration. + +[alembic] +# path to migration scripts +script_location = alembic + +# template used to generate migration files +# file_template = %%(rev)s_%%(slug)s + +# timezone to use when rendering the date +# within the migration file as well as the filename. +# string value is passed to dateutil.tz.gettz() +# leave blank for localtime +# timezone = + +# max length of characters to apply to the +# "slug" field +#truncate_slug_length = 40 + +# set to 'true' to run the environment during +# the 'revision' command, regardless of autogenerate +# revision_environment = false + +# set to 'true' to allow .pyc and .pyo files without +# a source .py file to be detected as revisions in the +# versions/ directory +# sourceless = false + +# version location specification; this defaults +# to alembic/versions. When using multiple version +# directories, initial revisions must be specified with --version-path +# version_locations = %(here)s/bar %(here)s/bat alembic/versions + +# the output encoding used when revision files +# are written from script.py.mako +# output_encoding = utf-8 + +# sqlalchemy.url = driver://user:pass@localhost/dbname + + +# Logging configuration +[loggers] +keys = root,sqlalchemy,alembic + +[handlers] +keys = console + +[formatters] +keys = generic + +[logger_root] +level = WARN +handlers = console +qualname = + +[logger_sqlalchemy] +level = WARN +handlers = +qualname = sqlalchemy.engine + +[logger_alembic] +level = INFO +handlers = +qualname = alembic + +[handler_console] +class = StreamHandler +args = (sys.stderr,) +level = NOTSET +formatter = generic + +[formatter_generic] +format = %(levelname)-5.5s [%(name)s] %(message)s +datefmt = %H:%M:%S diff --git a/alembic/README b/alembic/README new file mode 100644 index 0000000..98e4f9c --- /dev/null +++ b/alembic/README @@ -0,0 +1 @@ +Generic single-database configuration. \ No newline at end of file diff --git a/alembic/env.py b/alembic/env.py new file mode 100644 index 0000000..5dea161 --- /dev/null +++ b/alembic/env.py @@ -0,0 +1,85 @@ +"""This file is used to setup and override some of alembic configurations.""" + +from __future__ import with_statement +from alembic import context +from sqlalchemy import engine_from_config, pool +from logging.config import fileConfig +from os import environ + +from kiskadee.database import get_database_uri +from kiskadee.model import Base + +# this is the Alembic Config object, which provides +# access to the values within the .ini file in use. +config = context.config + +# Interpret the config file for Python logging. +# This line sets up loggers basically. +fileConfig(config.config_file_name) + +# add your model's MetaData object here +# for 'autogenerate' support +# from myapp import mymodel +# target_metadata = mymodel.Base.metadata +target_metadata = Base.metadata + +# other values from the config, defined by the needs of env.py, +# can be acquired: +# my_important_option = config.get_main_option("my_important_option") +# ... etc. + +try: + database_type = environ['DATABASE_TYPE'] +except KeyError: + database_type = 'db_development' + +db_uri = get_database_uri(database_type) +config.set_main_option('sqlalchemy.url', db_uri) + + +def run_migrations_offline(): + """Run migrations in 'offline' mode. + + This configures the context with just a URL + and not an Engine, though an Engine is acceptable + here as well. By skipping the Engine creation + we don't even need a DBAPI to be available. + + Calls to context.execute() here emit the given string to the + script output. + + """ + url = config.get_main_option("sqlalchemy.url") + context.configure( + url=url, target_metadata=target_metadata, literal_binds=True) + + with context.begin_transaction(): + context.run_migrations() + + +def run_migrations_online(): + """Run migrations in 'online' mode. + + In this scenario we need to create an Engine + and associate a connection with the context. + + """ + connectable = engine_from_config( + config.get_section(config.config_ini_section), + prefix='sqlalchemy.', + poolclass=pool.NullPool) + + with connectable.connect() as connection: + context.configure( + connection=connection, + target_metadata=target_metadata + ) + + with context.begin_transaction(): + context.run_migrations() + + +if context.is_offline_mode(): + run_migrations_offline() +else: + run_migrations_online() diff --git a/alembic/script.py.mako b/alembic/script.py.mako new file mode 100644 index 0000000..bea906f --- /dev/null +++ b/alembic/script.py.mako @@ -0,0 +1,26 @@ +"""${message}. + +Revision ID: ${up_revision} +Revises: ${down_revision | comma,n} +Create Date: ${create_date} + +""" +from alembic import op +import sqlalchemy as sa +${imports if imports else ""} + +# revision identifiers, used by Alembic. +revision = ${repr(up_revision)} +down_revision = ${repr(down_revision)} +branch_labels = ${repr(branch_labels)} +depends_on = ${repr(depends_on)} + + +def upgrade(): + """TODO: Add an upgrade description.""" + ${upgrades if upgrades else "pass"} + + +def downgrade(): + """TODO: Add a downgrade description.""" + ${downgrades if downgrades else "pass"} diff --git a/alembic/versions/48dd292b5a80_.py b/alembic/versions/48dd292b5a80_.py new file mode 100644 index 0000000..fe789bf --- /dev/null +++ b/alembic/versions/48dd292b5a80_.py @@ -0,0 +1,72 @@ +"""Auto generate a migration to the following tables. + +* reports +* analysis +* versions +* analyzers + +Revision ID: 48dd292b5a80 +Revises: d2989db85795 +Create Date: 2017-10-19 10:32:42.008412 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = '48dd292b5a80' +down_revision = 'd2989db85795' +branch_labels = None +depends_on = None + + +def upgrade(): + """Create tables: reports, analysis, versions, analyzers.""" + # ### commands auto generated by Alembic - please adjust! ### + op.create_table( + 'analyzers', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('name', sa.Unicode(length=255), nullable=False), + sa.Column('version', sa.Unicode(length=255), nullable=True), + sa.PrimaryKeyConstraint('id'), + sa.UniqueConstraint('name') + ) + op.create_table( + 'versions', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('number', sa.Unicode(length=100), nullable=False), + sa.Column('package_id', sa.Integer(), nullable=False), + sa.ForeignKeyConstraint(['package_id'], ['packages.id'], ), + sa.PrimaryKeyConstraint('id'), + sa.UniqueConstraint('number', 'package_id') + ) + op.create_table( + 'analysis', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('version_id', sa.Integer(), nullable=False), + sa.Column('analyzer_id', sa.Integer(), nullable=False), + sa.Column('raw', sa.JSON(), nullable=True), + sa.ForeignKeyConstraint(['analyzer_id'], ['analyzers.id'], ), + sa.ForeignKeyConstraint(['version_id'], ['versions.id'], ), + sa.PrimaryKeyConstraint('id') + ) + op.create_table( + 'reports', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('analysis_id', sa.Integer(), nullable=False), + sa.Column('results', sa.JSON(), nullable=True), + sa.ForeignKeyConstraint(['analysis_id'], ['analysis.id'], ), + sa.PrimaryKeyConstraint('id') + ) + # ### end Alembic commands ### + + +def downgrade(): + """Drop the tables created at upgrade.""" + # ### commands auto generated by Alembic - please adjust! ### + op.drop_table('reports') + op.drop_table('analysis') + op.drop_table('versions') + op.drop_table('analyzers') + # ### end Alembic commands ### diff --git a/alembic/versions/50988af48b09_create_fetcher_table.py b/alembic/versions/50988af48b09_create_fetcher_table.py new file mode 100644 index 0000000..54fd6c4 --- /dev/null +++ b/alembic/versions/50988af48b09_create_fetcher_table.py @@ -0,0 +1,40 @@ +"""create fetcher table. + +Revision ID: 50988af48b09 +Revises: 6daaf1d5cfee +Create Date: 2017-10-17 16:48:26.940294 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = '50988af48b09' +down_revision = '6daaf1d5cfee' +branch_labels = None +depends_on = None + + +def upgrade(): + """Create table fetchers.""" + op.create_table( + 'fetchers', + sa.Column( + 'id', + sa.Integer, + sa.Sequence( + 'fetchers_id_seq', + optional=True + ), + primary_key=True + ), + sa.Column('name', sa.Unicode(255), nullable=False, unique=True), + sa.Column('target', sa.Unicode(255), nullable=True), + sa.Column('description', sa.UnicodeText) + ) + + +def downgrade(): + """Drop table fetchers.""" + op.drop_table('fetchers') diff --git a/alembic/versions/6daaf1d5cfee_create_package_table.py b/alembic/versions/6daaf1d5cfee_create_package_table.py new file mode 100644 index 0000000..f3159f6 --- /dev/null +++ b/alembic/versions/6daaf1d5cfee_create_package_table.py @@ -0,0 +1,34 @@ +"""create Package table. + +Revision ID: 6daaf1d5cfee +Revises: +Create Date: 2017-10-17 15:48:05.193085 + +""" +from alembic import op +import sqlalchemy as sa + +# revision identifiers, used by Alembic. +revision = '6daaf1d5cfee' +down_revision = None +branch_labels = None +depends_on = None + + +def upgrade(): + """Create table packages.""" + op.create_table( + 'packages', + sa.Column( + 'id', + sa.Integer, + sa.Sequence('packages_id_seq', optional=True), + primary_key=True + ), + sa.Column('name', sa.Unicode(255), nullable=False) + ) + + +def downgrade(): + """Drop table packages.""" + op.drop_table('packages') diff --git a/alembic/versions/d2989db85795_add_relation_between_fetchers_and_.py b/alembic/versions/d2989db85795_add_relation_between_fetchers_and_.py new file mode 100644 index 0000000..53dac7a --- /dev/null +++ b/alembic/versions/d2989db85795_add_relation_between_fetchers_and_.py @@ -0,0 +1,40 @@ +"""add relation between fetchers and packages tables. + +Revision ID: d2989db85795 +Revises: 50988af48b09 +Create Date: 2017-10-17 16:59:09.495015 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = 'd2989db85795' +down_revision = '50988af48b09' +branch_labels = None +depends_on = None + + +def upgrade(): + """Add fetch_id to packages and set a unique constraint for it.""" + with op.batch_alter_table("packages") as batch_op: + batch_op.add_column( + sa.Column( + 'fetcher_id', + sa.Integer, + sa.ForeignKey('fetchers.id'), + nullable=False + ) + ) + batch_op.create_unique_constraint( + 'name_and_fetcher_id_unique', + ['name', 'fetcher_id'] + ) + + +def downgrade(): + """Undo add fetcher_id to packages.""" + with op.batch_alter_table("packages") as batch_op: + op.drop_constraint('name_and_fetcher_id_unique', 'packages') + batch_op.drop_column('fetcher_id') diff --git a/kiskadee/database.py b/kiskadee/database.py index d7de516..60dd0a0 100644 --- a/kiskadee/database.py +++ b/kiskadee/database.py @@ -16,19 +16,24 @@ class Database: Base.metadata.bind = self.engine def _create_engine(self, db): - driver = kiskadee.config[db]['driver'] - username = kiskadee.config[db]['username'] - password = kiskadee.config[db]['password'] - hostname = kiskadee.config[db]['hostname'] - port = kiskadee.config[db]['port'] - dbname = kiskadee.config[db]['dbname'] - return create_engine('%s://%s:%s@%s:%s/%s' % (driver, - username, - password, - hostname, - port, - dbname)) + uri = get_database_uri(db) + return create_engine(uri) def _create_session(self, engine): DBSession = orm.sessionmaker(bind=engine) return DBSession() + + +def get_database_uri(db): + """Return the Database URI of the current session.""" + config = kiskadee.config[db] + + driver = config['driver'] + username = config['username'] + password = config['password'] + hostname = config['hostname'] + port = config['port'] + dbname = config['dbname'] + + return '%s://%s:%s@%s:%s/%s' % (driver, username, password, + hostname, port, dbname) diff --git a/requirements.txt b/requirements.txt index 97a784f..79e4720 100644 --- a/requirements.txt +++ b/requirements.txt @@ -19,3 +19,4 @@ flask-cors coverage nose ansible +alembic