From d0a3e63c8ec74c9313f407c25c5944d57600dd90 Mon Sep 17 00:00:00 2001 From: Aurélien Bompard Date: Mar 04 2017 08:46:52 +0000 Subject: [PATCH 1/5] Move the alembic conf and migrations inside the package They must be distributed with the package to allow admins to upgrade their databases. --- diff --git a/MANIFEST.in b/MANIFEST.in index 44ebdab..caa39a2 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,2 +1,5 @@ include requirements.txt include test-requirements.txt +include hubs/alembic.ini +recursive-include hubs/migrations *.py *.mako *.rst +include hubs/migrations/versions/.keep diff --git a/alembic.ini b/alembic.ini deleted file mode 100644 index 305672c..0000000 --- a/alembic.ini +++ /dev/null @@ -1,68 +0,0 @@ -# A generic, single database configuration. - -[alembic] -# path to migration scripts -script_location = alembic - -# template used to generate migration files -# file_template = %%(rev)s_%%(slug)s - -# max length of characters to apply to the -# "slug" field -#truncate_slug_length = 40 - -# set to 'true' to run the environment during -# the 'revision' command, regardless of autogenerate -# revision_environment = false - -# set to 'true' to allow .pyc and .pyo files without -# a source .py file to be detected as revisions in the -# versions/ directory -# sourceless = false - -# version location specification; this defaults -# to alembic/versions. When using multiple version -# directories, initial revisions must be specified with --version-path -# version_locations = %(here)s/bar %(here)s/bat alembic/versions - -# the output encoding used when revision files -# are written from script.py.mako -output_encoding = utf-8 - -sqlalchemy.url = driver://user:pass@localhost/dbname - - -# Logging configuration -[loggers] -keys = root,sqlalchemy,alembic - -[handlers] -keys = console - -[formatters] -keys = generic - -[logger_root] -level = WARN -handlers = console -qualname = - -[logger_sqlalchemy] -level = WARN -handlers = -qualname = sqlalchemy.engine - -[logger_alembic] -level = INFO -handlers = -qualname = alembic - -[handler_console] -class = StreamHandler -args = (sys.stderr,) -level = NOTSET -formatter = generic - -[formatter_generic] -format = %(levelname)-5.5s [%(name)s] %(message)s -datefmt = %H:%M:%S diff --git a/alembic/README.rst b/alembic/README.rst deleted file mode 100644 index 5b7a3d1..0000000 --- a/alembic/README.rst +++ /dev/null @@ -1,7 +0,0 @@ -Generic single-database configuration of Alembic. - -Consult `the documentation `_ for -details on how to write new database migrations. In particular, you should be -aware of the Alembic `autogenerate -`_ feature which -you can use with ``alembic revision --autogenerate -m ""`` diff --git a/alembic/env.py b/alembic/env.py deleted file mode 100644 index 058378b..0000000 --- a/alembic/env.py +++ /dev/null @@ -1,70 +0,0 @@ -from __future__ import with_statement -from alembic import context -from sqlalchemy import engine_from_config, pool -from logging.config import fileConfig - -# this is the Alembic Config object, which provides -# access to the values within the .ini file in use. -config = context.config - -# Interpret the config file for Python logging. -# This line sets up loggers basically. -fileConfig(config.config_file_name) - -# add your model's MetaData object here -# for 'autogenerate' support -# from myapp import mymodel -# target_metadata = mymodel.Base.metadata -target_metadata = None - -# other values from the config, defined by the needs of env.py, -# can be acquired: -# my_important_option = config.get_main_option("my_important_option") -# ... etc. - - -def run_migrations_offline(): - """Run migrations in 'offline' mode. - - This configures the context with just a URL - and not an Engine, though an Engine is acceptable - here as well. By skipping the Engine creation - we don't even need a DBAPI to be available. - - Calls to context.execute() here emit the given string to the - script output. - - """ - url = config.get_main_option("sqlalchemy.url") - context.configure( - url=url, target_metadata=target_metadata, literal_binds=True) - - with context.begin_transaction(): - context.run_migrations() - - -def run_migrations_online(): - """Run migrations in 'online' mode. - - In this scenario we need to create an Engine - and associate a connection with the context. - - """ - connectable = engine_from_config( - config.get_section(config.config_ini_section), - prefix='sqlalchemy.', - poolclass=pool.NullPool) - - with connectable.connect() as connection: - context.configure( - connection=connection, - target_metadata=target_metadata - ) - - with context.begin_transaction(): - context.run_migrations() - -if context.is_offline_mode(): - run_migrations_offline() -else: - run_migrations_online() diff --git a/alembic/script.py.mako b/alembic/script.py.mako deleted file mode 100644 index 7df1d1e..0000000 --- a/alembic/script.py.mako +++ /dev/null @@ -1,40 +0,0 @@ -# -*- coding: utf-8 -*- -# This Alembic database migration is part of the Fedora Hubs project. -# Copyright (C) -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see . -""" -${message} - -Revision ID: ${up_revision} -Revises: ${down_revision | comma,n} -Create Date: ${create_date} -""" -from alembic import op -import sqlalchemy as sa -${imports if imports else ""} - -# revision identifiers, used by Alembic. -revision = ${repr(up_revision)} -down_revision = ${repr(down_revision)} -branch_labels = ${repr(branch_labels)} -depends_on = ${repr(depends_on)} - - -def upgrade(): - ${upgrades if upgrades else "pass"} - - -def downgrade(): - ${downgrades if downgrades else "pass"} diff --git a/hubs/alembic.ini b/hubs/alembic.ini new file mode 100644 index 0000000..8b365d8 --- /dev/null +++ b/hubs/alembic.ini @@ -0,0 +1,68 @@ +# A generic, single database configuration. + +[alembic] +# path to migration scripts +script_location = %(here)s/migrations + +# template used to generate migration files +# file_template = %%(rev)s_%%(slug)s + +# max length of characters to apply to the +# "slug" field +#truncate_slug_length = 40 + +# set to 'true' to run the environment during +# the 'revision' command, regardless of autogenerate +# revision_environment = false + +# set to 'true' to allow .pyc and .pyo files without +# a source .py file to be detected as revisions in the +# versions/ directory +# sourceless = false + +# version location specification; this defaults +# to alembic/versions. When using multiple version +# directories, initial revisions must be specified with --version-path +# version_locations = %(here)s/bar %(here)s/bat alembic/versions + +# the output encoding used when revision files +# are written from script.py.mako +output_encoding = utf-8 + +#sqlalchemy.url = driver://user:pass@localhost/dbname + + +# Logging configuration +[loggers] +keys = root,sqlalchemy,alembic + +[handlers] +keys = console + +[formatters] +keys = generic + +[logger_root] +level = WARN +handlers = console +qualname = + +[logger_sqlalchemy] +level = WARN +handlers = +qualname = sqlalchemy.engine + +[logger_alembic] +level = INFO +handlers = +qualname = alembic + +[handler_console] +class = StreamHandler +args = (sys.stderr,) +level = NOTSET +formatter = generic + +[formatter_generic] +format = %(levelname)-5.5s [%(name)s] %(message)s +datefmt = %H:%M:%S diff --git a/hubs/migrations/README.rst b/hubs/migrations/README.rst new file mode 100644 index 0000000..06c25e5 --- /dev/null +++ b/hubs/migrations/README.rst @@ -0,0 +1,9 @@ +Generic single-database configuration of Alembic. + +Consult `the documentation `_ for +details on how to write new database migrations. In particular, you should be +aware of the Alembic `autogenerate +`_ feature which +you can use with:: + + alembic -c hubs/alembic.ini revision --autogenerate -m "" diff --git a/hubs/migrations/env.py b/hubs/migrations/env.py new file mode 100644 index 0000000..058378b --- /dev/null +++ b/hubs/migrations/env.py @@ -0,0 +1,70 @@ +from __future__ import with_statement +from alembic import context +from sqlalchemy import engine_from_config, pool +from logging.config import fileConfig + +# this is the Alembic Config object, which provides +# access to the values within the .ini file in use. +config = context.config + +# Interpret the config file for Python logging. +# This line sets up loggers basically. +fileConfig(config.config_file_name) + +# add your model's MetaData object here +# for 'autogenerate' support +# from myapp import mymodel +# target_metadata = mymodel.Base.metadata +target_metadata = None + +# other values from the config, defined by the needs of env.py, +# can be acquired: +# my_important_option = config.get_main_option("my_important_option") +# ... etc. + + +def run_migrations_offline(): + """Run migrations in 'offline' mode. + + This configures the context with just a URL + and not an Engine, though an Engine is acceptable + here as well. By skipping the Engine creation + we don't even need a DBAPI to be available. + + Calls to context.execute() here emit the given string to the + script output. + + """ + url = config.get_main_option("sqlalchemy.url") + context.configure( + url=url, target_metadata=target_metadata, literal_binds=True) + + with context.begin_transaction(): + context.run_migrations() + + +def run_migrations_online(): + """Run migrations in 'online' mode. + + In this scenario we need to create an Engine + and associate a connection with the context. + + """ + connectable = engine_from_config( + config.get_section(config.config_ini_section), + prefix='sqlalchemy.', + poolclass=pool.NullPool) + + with connectable.connect() as connection: + context.configure( + connection=connection, + target_metadata=target_metadata + ) + + with context.begin_transaction(): + context.run_migrations() + +if context.is_offline_mode(): + run_migrations_offline() +else: + run_migrations_online() diff --git a/hubs/migrations/script.py.mako b/hubs/migrations/script.py.mako new file mode 100644 index 0000000..46b3b54 --- /dev/null +++ b/hubs/migrations/script.py.mako @@ -0,0 +1,43 @@ +# -*- coding: utf-8 -*- +# This Alembic database migration is part of the Fedora Hubs project. +# Copyright (C) +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +""" +${message} + +Revision ID: ${up_revision} +Revises: ${down_revision | comma,n} +Create Date: ${create_date} +""" + +from __future__ import absolute_import, unicode_literals + +from alembic import op +import sqlalchemy as sa +${imports if imports else ""} + +# revision identifiers, used by Alembic. +revision = ${repr(up_revision)} +down_revision = ${repr(down_revision)} +branch_labels = ${repr(branch_labels)} +depends_on = ${repr(depends_on)} + + +def upgrade(): + ${upgrades if upgrades else "pass"} + + +def downgrade(): + ${downgrades if downgrades else "pass"} diff --git a/hubs/migrations/versions/.keep b/hubs/migrations/versions/.keep new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/hubs/migrations/versions/.keep From c90f8593a505a39840642a67a84cce0b20d00789 Mon Sep 17 00:00:00 2001 From: Aurélien Bompard Date: Mar 04 2017 08:46:52 +0000 Subject: [PATCH 2/5] Read the database config and metadata from hubs itself --- diff --git a/hubs/migrations/env.py b/hubs/migrations/env.py index 058378b..7605ce5 100644 --- a/hubs/migrations/env.py +++ b/hubs/migrations/env.py @@ -1,8 +1,25 @@ -from __future__ import with_statement +from __future__ import absolute_import, unicode_literals + from alembic import context -from sqlalchemy import engine_from_config, pool +from sqlalchemy import create_engine, pool from logging.config import fileConfig + +import fedmsg.config + +fedmsg_config = fedmsg.config.load_config() + + +try: + import hubs +except ImportError: + import os, sys + if "hubs" not in os.listdir("."): + raise + sys.path.append(os.getcwd()) + +from hubs.models import BASE + # this is the Alembic Config object, which provides # access to the values within the .ini file in use. config = context.config @@ -11,11 +28,8 @@ config = context.config # This line sets up loggers basically. fileConfig(config.config_file_name) -# add your model's MetaData object here -# for 'autogenerate' support -# from myapp import mymodel -# target_metadata = mymodel.Base.metadata -target_metadata = None +url = fedmsg_config['hubs.sqlalchemy.uri'] +target_metadata = BASE.metadata # other values from the config, defined by the needs of env.py, # can be acquired: @@ -35,7 +49,6 @@ def run_migrations_offline(): script output. """ - url = config.get_main_option("sqlalchemy.url") context.configure( url=url, target_metadata=target_metadata, literal_binds=True) @@ -50,11 +63,7 @@ def run_migrations_online(): and associate a connection with the context. """ - connectable = engine_from_config( - config.get_section(config.config_ini_section), - prefix='sqlalchemy.', - poolclass=pool.NullPool) - + connectable = create_engine(url, poolclass=pool.NullPool) with connectable.connect() as connection: context.configure( connection=connection, From 3596487a50ea7a92106ae56b072b03922b3e9ee1 Mon Sep 17 00:00:00 2001 From: Aurélien Bompard Date: Mar 04 2017 08:46:53 +0000 Subject: [PATCH 3/5] Adjust the MANIFEST.in file so that sdist is valid --- diff --git a/.gitignore b/.gitignore index b3b67a9..ce06a7d 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,7 @@ fedmsg.d/fas_credentials.py fedmsg.d/credentials.py node* build/ +dist/ scratch/ npm-debug.log diff --git a/MANIFEST.in b/MANIFEST.in index caa39a2..c533ed4 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,5 +1,12 @@ -include requirements.txt -include test-requirements.txt +include *.txt LICENSE *.py *.sh *.ini include hubs/alembic.ini +graft hubs/static +prune hubs/static/client/node_modules +recursive-include hubs/templates *.html recursive-include hubs/migrations *.py *.mako *.rst include hubs/migrations/versions/.keep +include hubs/actions.json +graft ansible +graft docs +prune docs/_build +recursive-include fedmsg.d *.py From b4052c2617f7f202300ebbf6e4db23341ba15d63 Mon Sep 17 00:00:00 2001 From: Aurélien Bompard Date: Mar 04 2017 08:54:36 +0000 Subject: [PATCH 4/5] Stamp the DB with the Alembic revision on creation --- diff --git a/hubs/migrations/env.py b/hubs/migrations/env.py index 7605ce5..e00d82b 100644 --- a/hubs/migrations/env.py +++ b/hubs/migrations/env.py @@ -20,13 +20,13 @@ except ImportError: from hubs.models import BASE -# this is the Alembic Config object, which provides +# This is the Alembic Config object, which provides # access to the values within the .ini file in use. -config = context.config - -# Interpret the config file for Python logging. -# This line sets up loggers basically. -fileConfig(config.config_file_name) +# The config_file_name property may be None when used programatically. +if context.config.config_file_name is not None: + # Interpret the config file for Python logging. + # This line sets up loggers basically. + fileConfig(context.config.config_file_name) url = fedmsg_config['hubs.sqlalchemy.uri'] target_metadata = BASE.metadata diff --git a/hubs/migrations/versions/.keep b/hubs/migrations/versions/.keep deleted file mode 100644 index e69de29..0000000 --- a/hubs/migrations/versions/.keep +++ /dev/null diff --git a/hubs/migrations/versions/26cc11577009_initial_empty_revision.py b/hubs/migrations/versions/26cc11577009_initial_empty_revision.py new file mode 100644 index 0000000..d142330 --- /dev/null +++ b/hubs/migrations/versions/26cc11577009_initial_empty_revision.py @@ -0,0 +1,39 @@ +# This Alembic database migration is part of the Fedora Hubs project. +# Copyright (C) 2017 Fedora project +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +""" +Initial empty revision. It does nothing, to the database except allowing +Alembic to record a revision. + +Revision ID: 26cc11577009 +Revises: None +Create Date: 2017-03-04 09:20:55.915068 +""" + +from __future__ import absolute_import, unicode_literals + + +revision = '26cc11577009' +down_revision = None +branch_labels = None +depends_on = None + + +def upgrade(): + pass + + +def downgrade(): + pass diff --git a/hubs/models.py b/hubs/models.py index 2b7b6d5..23d4f14 100644 --- a/hubs/models.py +++ b/hubs/models.py @@ -32,6 +32,9 @@ import random import bleach import dogpile import sqlalchemy as sa + +from alembic import command as alembic_command +from alembic.config import Config as AlembicConfig from sqlalchemy import create_engine from sqlalchemy.ext.declarative import declarative_base from sqlalchemy.orm import sessionmaker @@ -93,6 +96,11 @@ def init(db_url, debug=False, create=False): if create: BASE.metadata.create_all(engine) + # Stamp the database with the lastest Alembic revision: + # http://alembic.zzzcomputing.com/en/latest/cookbook.html#building-an-up-to-date-database-from-scratch + alembic_cfg = AlembicConfig() + alembic_cfg.set_main_option("script_location", "hubs:migrations") + alembic_command.stamp(alembic_cfg, "head") session = scoped_session(sessionmaker(bind=engine)) BASE.query = session.query_property() return session From 336f277e14456a403cb52202a25da891dbcd70de Mon Sep 17 00:00:00 2001 From: Aurélien Bompard Date: Mar 04 2017 08:54:36 +0000 Subject: [PATCH 5/5] Enable foreign keys constraints on SQLite --- diff --git a/hubs/models.py b/hubs/models.py index 23d4f14..d9f7a00 100644 --- a/hubs/models.py +++ b/hubs/models.py @@ -93,6 +93,8 @@ def init(db_url, debug=False, create=False): """ engine = create_engine(db_url, echo=debug) + if engine.name == "sqlite": + engine.execute("PRAGMA foreign_keys=ON") if create: BASE.metadata.create_all(engine)