From 6a877e0affc405c4ae432bdf263a57acd36c870d Mon Sep 17 00:00:00 2001 From: Jeremy Cline Date: Mar 27 2017 15:35:09 +0000 Subject: [PATCH 1/2] Announce new Waivers via fedmsg This adds an SQLAlchemy event which emits fedmsgs when a Waiver is created. Signed-off-by: Jeremy Cline --- diff --git a/README.md b/README.md index 0f1c4e9..7faa043 100644 --- a/README.md +++ b/README.md @@ -48,3 +48,10 @@ You can view the docs locally with:: $ cd docs $ make html $ firefox _build/html/index.html + +## Viewing published fedmsgs + +You can view fedmsgs published when new waivers get created by doing:: + + $ fedmsg-relay --config-filename fedmsg.d/config.py & + $ fedmsg-tail --config fedmsg.d/config.py --no-validate --really-pretty diff --git a/fedmsg.d/config.py b/fedmsg.d/config.py new file mode 100644 index 0000000..ca3953f --- /dev/null +++ b/fedmsg.d/config.py @@ -0,0 +1,146 @@ +# -*- coding: utf-8 -*- +# This file is part of WaiverDB. +# Copyright (C) 2017 Red Hat, Inc. + +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, +# USA. + +import os +import socket + +hostname = socket.gethostname() + +config = dict( + active=True, + # Set this to dev if you're hacking on fedmsg or an app. + # Set to stg or prod if running in the Fedora Infrastructure + environment="dev", + + # Default is 0 + high_water_mark=0, + io_threads=1, + + ## For the fedmsg-hub and fedmsg-relay. ## + + # This is a status dir to keep a record of the last processed message + #status_directory=os.getcwd() + "/status", + #status_directory='/var/run/fedmsg/status', + + # This is the URL of a datagrepper instance that we can query for backlog. + #datagrepper_url="https://apps.fedoraproject.org/datagrepper/raw", + + # We almost always want the fedmsg-hub to be sending messages with zmq as + # opposed to amqp or stomp. You can send with only *one* of the messaging + # backends: zeromq or amqp or stomp. You cannot send with two or more at + # the same time. Here, zmq is either enabled, or it is not. If it is not, + # see the options below for how to configure stomp or amqp. + zmq_enabled=True, + + # On the other hand, if you wanted to use STOMP *instead* of zeromq, you + # could do the following... + #zmq_enabled=False, + #stomp_uri='localhost:59597,localhost:59598', + #stomp_user='username', + #stomp_pass='password', + #stomp_ssl_crt='/path/to/an/optional.crt', + #stomp_ssl_key='/path/to/an/optional.key', + + # When subscribing to messages, we want to allow splats ('*') so we tell + # the hub to not be strict when comparing messages topics to subscription + # topics. + zmq_strict=False, + + # Number of seconds to sleep after initializing waiting for sockets to sync + post_init_sleep=0.5, + + # Wait a whole second to kill all the last io threads for messages to + # exit our outgoing queue (if we have any). This is in milliseconds. + zmq_linger=1000, + + # See the following + # - http://tldp.org/HOWTO/TCP-Keepalive-HOWTO/overview.html + # - http://api.zeromq.org/3-2:zmq-setsockopt + zmq_tcp_keepalive=1, + zmq_tcp_keepalive_cnt=3, + zmq_tcp_keepalive_idle=60, + zmq_tcp_keepalive_intvl=5, + + # Number of miliseconds that zeromq will wait to reconnect until it gets + # a connection if an endpoint is unavailable. + zmq_reconnect_ivl=100, + # Max delay that you can reconfigure to reduce reconnect storm spam. This + # is in miliseconds. + zmq_reconnect_ivl_max=1000, + + # This is a dict of possible addresses from which fedmsg can send + # messages. fedmsg.init(...) requires that a 'name' argument be passed + # to it which corresponds with one of the keys in this dict. + endpoints={ + "waiverdb.%s" % hostname: [ + "tcp://127.0.0.1:5011", + ], + "relay_outbound": [ + "tcp://127.0.0.1:4001", + ], + }, + # This is the address of an active->passive relay. It is used for the + # fedmsg-logger command which requires another service with a stable + # listening address for it to send messages to. + # It is also used by the git-hook, for the same reason. + # It is also used by the mediawiki php plugin which, due to the oddities of + # php, can't maintain a single passive-bind endpoint of it's own. + relay_inbound=[ + "tcp://127.0.0.1:2003", + ], + sign_messages=False, + validate_signatures=False, + + # Use these implementations to sign and validate messages + crypto_backend='x509', + crypto_validate_backends=['x509'], + + ssldir="/etc/pki/fedmsg", + crl_location="https://fedoraproject.org/fedmsg/crl.pem", + crl_cache="/var/run/fedmsg/crl.pem", + crl_cache_expiry=10, + + ca_cert_location="https://fedoraproject.org/fedmsg/ca.crt", + ca_cert_cache="/var/run/fedmsg/ca.crt", + ca_cert_cache_expiry=0, # Never expires + + certnames={ + # In prod/stg, map hostname to the name of the cert in ssldir. + # Unfortunately, we can't use socket.getfqdn() + #"app01.stg": "app01.stg.phx2.fedoraproject.org", + }, + + # A mapping of fully qualified topics to a list of cert names for which + # a valid signature is to be considered authorized. Messages on topics not + # listed here are considered automatically authorized. + routing_policy={ + # Only allow announcements from production if they're signed by a + # certain certificate. + "org.fedoraproject.prod.announce.announcement": [ + "announce-lockbox.phx2.fedoraproject.org", + ], + }, + + # Set this to True if you want messages to be dropped that aren't + # explicitly whitelisted in the routing_policy. + # When this is False, only messages that have a topic in the routing_policy + # but whose cert names aren't in the associated list are dropped; messages + # whose topics do not appear in the routing_policy are not dropped. + routing_nitpicky=False, +) diff --git a/requirements.txt b/requirements.txt index 33e8f36..a201a83 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,7 @@ # This is a list of pypi packages to be installed into virtualenv. Alternatively, # you can install these as RPMs instead of pypi packages. +fedmsg[consumers,commands] Flask Flask-RESTful Flask-SQLAlchemy diff --git a/tests/test_app.py b/tests/test_app.py new file mode 100644 index 0000000..4474139 --- /dev/null +++ b/tests/test_app.py @@ -0,0 +1,47 @@ +# -*- coding: utf-8 -*- +# +# This file is part of WaiverDB. +# Copyright © 2017 Red Hat, Inc. +# +# This copyrighted material is made available to anyone wishing to use, +# modify, copy, or redistribute it subject to the terms and conditions +# of the GNU General Public License v.2, or (at your option) any later +# version. This program is distributed in the hope that it will be +# useful, but WITHOUT ANY WARRANTY expressed or implied, including the +# implied warranties of MERCHANTABILITY or FITNESS FOR A PARTICULAR +# PURPOSE. See the GNU General Public License for more details. You +# should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, +# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# Any Red Hat trademarks that are incorporated in the source +# code or documentation are not subject to the GNU General Public +# License and may only be used or replicated with the express permission +# of Red Hat, Inc. +"""This module contains tests for :mod:`waiverdb.app`.""" +from __future__ import unicode_literals + +import mock + +from waiverdb import app, config + + +class NoZmqConfig(config.Config): + ZEROMQ_PUBLISH = False + + +class ZmqConfig(config.Config): + ZEROMQ_PUBLISH = True + + +@mock.patch('waiverdb.app.event.listen') +def test_register_events_no_zmq(mock_listen): + app.create_app(NoZmqConfig) + assert 0 == mock_listen.call_count + + +@mock.patch('waiverdb.app.event.listen') +def test_register_events_zmq(mock_listen): + app.create_app(ZmqConfig) + mock_listen.assert_called_once_with( + app.db.session, 'after_commit', app.fedmsg_new_waiver) diff --git a/tests/test_events.py b/tests/test_events.py new file mode 100644 index 0000000..080f531 --- /dev/null +++ b/tests/test_events.py @@ -0,0 +1,53 @@ +# -*- coding: utf-8 -*- +# +# This file is part of WaiverDB. +# Copyright © 2017 Red Hat, Inc. +# +# This copyrighted material is made available to anyone wishing to use, +# modify, copy, or redistribute it subject to the terms and conditions +# of the GNU General Public License v.2, or (at your option) any later +# version. This program is distributed in the hope that it will be +# useful, but WITHOUT ANY WARRANTY expressed or implied, including the +# implied warranties of MERCHANTABILITY or FITNESS FOR A PARTICULAR +# PURPOSE. See the GNU General Public License for more details. You +# should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, +# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# Any Red Hat trademarks that are incorporated in the source +# code or documentation are not subject to the GNU General Public +# License and may only be used or replicated with the express permission +# of Red Hat, Inc. +"""This module contains tests for :mod:`waiverdb.events`.""" +from __future__ import unicode_literals + +import pytest + +import mock + +from waiverdb import events +from waiverdb.models import Waiver + + +@mock.patch('waiverdb.events.fedmsg', None) +def test_fedmsg_new_waiver_missing_fedmsg(): + with pytest.raises(RuntimeError): + events.fedmsg_new_waiver(None) + + +@mock.patch('waiverdb.events.fedmsg') +def test_fedmsg_new_waiver(mock_fedmsg, session): + waiver = Waiver( + result_id=1, + username='jcline', + product_version='something', + waived=True, + comment='This is a comment', + ) + sesh = session() + sesh.add(waiver) + sesh.commit() + mock_fedmsg.publish.assert_called_once_with( + topic='waiver.new', + msg=waiver.__json__(), + ) diff --git a/waiverdb/app.py b/waiverdb/app.py index bf3f4a7..537c69e 100644 --- a/waiverdb/app.py +++ b/waiverdb/app.py @@ -10,7 +10,11 @@ # GNU General Public License for more details. import os + from flask import Flask +from sqlalchemy import event + +from waiverdb.events import fedmsg_new_waiver from waiverdb.logger import init_logging from waiverdb.api_v1 import api_v1 from waiverdb.models import db @@ -50,6 +54,7 @@ def create_app(config_obj=None): init_logging(app) # register blueprints app.register_blueprint(api_v1, url_prefix="/api/v1.0") + register_event_handlers(app) return app @@ -57,3 +62,15 @@ def init_db(app): with app.app_context(): db.create_all() return db + + +def register_event_handlers(app): + """ + Register SQLAlchemy event handlers with the application's session factory. + + Args: + app (flask.Flask): The Flask object with the configured scoped session + attached as the ``session`` attribute. + """ + if app.config['ZEROMQ_PUBLISH']: + event.listen(db.session, 'after_commit', fedmsg_new_waiver) diff --git a/waiverdb/config.py b/waiverdb/config.py index 4c8a7df..dd03a3c 100644 --- a/waiverdb/config.py +++ b/waiverdb/config.py @@ -11,6 +11,13 @@ class Config(object): + """ + A WaiverDB Flask configuration. + + Attributes: + ZEROMQ_PUBLISH (bool): When true, ZeroMQ messages will be emitted via + fedmsg when new waivers are created. + """ DEBUG = True SQLALCHEMY_DATABASE_URI = 'sqlite://' JOURNAL_LOGGING = False @@ -24,6 +31,7 @@ class Config(object): ERROR_404_HELP = False # Change it if the Kerberos service is not running on which the waiverdb is run. KERBEROS_HTTP_HOST = None + ZEROMQ_PUBLISH = True class ProductionConfig(Config): diff --git a/waiverdb/events.py b/waiverdb/events.py new file mode 100644 index 0000000..eb4e9cf --- /dev/null +++ b/waiverdb/events.py @@ -0,0 +1,92 @@ +# -*- coding: utf-8 -*- +# +# This file is part of WaiverDB. +# Copyright © 2017 Red Hat, Inc. +# +# This copyrighted material is made available to anyone wishing to use, +# modify, copy, or redistribute it subject to the terms and conditions +# of the GNU General Public License v.2, or (at your option) any later +# version. This program is distributed in the hope that it will be +# useful, but WITHOUT ANY WARRANTY expressed or implied, including the +# implied warranties of MERCHANTABILITY or FITNESS FOR A PARTICULAR +# PURPOSE. See the GNU General Public License for more details. You +# should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, +# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# Any Red Hat trademarks that are incorporated in the source +# code or documentation are not subject to the GNU General Public +# License and may only be used or replicated with the express permission +# of Red Hat, Inc. +""" +This module contains a set of `SQLAlchemy event`_ hooks. + +To use these hooks, you must register them with SQLAlchemy +using the :func:`sqlalchemy.event.listen` function. + +.. _SQLALchemy events: + https://docs.sqlalchemy.org/en/latest/orm/events.html +""" +from __future__ import unicode_literals + +from gettext import gettext as _ +import logging + +# fedmsg is an optional dependency and may not be present +try: + import fedmsg +except ImportError: + fedmsg = None + +from waiverdb.models import Waiver + + +_log = logging.getLogger(__name__) + + +def fedmsg_new_waiver(session): + """ + A post-commit event hook that emits fedmsgs. + + This event is designed to be registered with a session factory:: + + >>> from sqlalchemy.event import listen + >>> listen(MyScopedSession, 'after_commit', fedmsg_new_waiver) + + The emitted fedmsg will look like:: + + { + "username": "jcline", + "i": 4, + "timestamp": 1489686124, + "msg_id": "2017-80e46243-e6f5-46df-8dcd-4d17809eb298", + "topic": "org.fedoraproject.dev.waiverdb.waiver.new", + "msg": { + "comment": "Because I said so", + "username": "http://jcline.id.fedoraproject.org/", + "waived": true, + "timestamp": "2017-03-16T17:42:04.209638", + "product_version": "Satellite 6.3", + "result_id": 1, + "id": 15 + } + } + + Args: + session (sqlalchemy.orm.Session): The session that was committed to the + database. This session is not active and cannot emit SQL. + + Raises: + RuntimeError: If fedmsg is not installed. + """ + _log.debug('The fedmsg_new_waiver SQLAlchemy event has been activated.') + if fedmsg is None: + msg = _('The application has been configured to publish fedmsgs, but ' + 'fedmsg is not installed. Please install fedmsg or remove the ' + 'fedmsg SQLAlchemy event handler.') + raise RuntimeError(msg) + + for row in session.identity_map.values(): + if isinstance(row, Waiver): + _log.debug('Publishing fedmsg for %r', row) + fedmsg.publish(topic='waiver.new', msg=row.__json__()) diff --git a/waiverdb/models/waivers.py b/waiverdb/models/waivers.py index 1fde440..2d488af 100644 --- a/waiverdb/models/waivers.py +++ b/waiverdb/models/waivers.py @@ -33,3 +33,14 @@ class Waiver(db.Model): return '%s(result_id=%r, username=%r, product_version=%r, waived=%r)' % ( self.__class__.__name__, self.result_id, self.username, self.product_version, self.waived) + + def __json__(self): + return { + 'id': self.id, + 'result_id': self.result_id, + 'username': self.username, + 'product_version': self.product_version, + 'waived': self.waived, + 'comment': self.comment, + 'timestamp': self.timestamp.isoformat(), + } From e667d0ff179550cfd5492635b41a4d0daaf40123 Mon Sep 17 00:00:00 2001 From: Jeremy Cline Date: Mar 27 2017 15:35:11 +0000 Subject: [PATCH 2/2] Drop __json__ and fix the license headers Signed-off-by: Jeremy Cline --- diff --git a/fedmsg.d/config.py b/fedmsg.d/config.py index ca3953f..f25c0ba 100644 --- a/fedmsg.d/config.py +++ b/fedmsg.d/config.py @@ -1,21 +1,17 @@ # -*- coding: utf-8 -*- +# # This file is part of WaiverDB. -# Copyright (C) 2017 Red Hat, Inc. - -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License -# as published by the Free Software Foundation; either version 2 -# of the License, or (at your option) any later version. +# Copyright © 2017 Red Hat, Inc. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software -# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, -# USA. import os import socket diff --git a/tests/test_app.py b/tests/test_app.py index 4474139..a065711 100644 --- a/tests/test_app.py +++ b/tests/test_app.py @@ -3,21 +3,15 @@ # This file is part of WaiverDB. # Copyright © 2017 Red Hat, Inc. # -# This copyrighted material is made available to anyone wishing to use, -# modify, copy, or redistribute it subject to the terms and conditions -# of the GNU General Public License v.2, or (at your option) any later -# version. This program is distributed in the hope that it will be -# useful, but WITHOUT ANY WARRANTY expressed or implied, including the -# implied warranties of MERCHANTABILITY or FITNESS FOR A PARTICULAR -# PURPOSE. See the GNU General Public License for more details. You -# should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, -# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. # -# Any Red Hat trademarks that are incorporated in the source -# code or documentation are not subject to the GNU General Public -# License and may only be used or replicated with the express permission -# of Red Hat, Inc. +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. """This module contains tests for :mod:`waiverdb.app`.""" from __future__ import unicode_literals diff --git a/tests/test_events.py b/tests/test_events.py index 080f531..89af138 100644 --- a/tests/test_events.py +++ b/tests/test_events.py @@ -3,21 +3,15 @@ # This file is part of WaiverDB. # Copyright © 2017 Red Hat, Inc. # -# This copyrighted material is made available to anyone wishing to use, -# modify, copy, or redistribute it subject to the terms and conditions -# of the GNU General Public License v.2, or (at your option) any later -# version. This program is distributed in the hope that it will be -# useful, but WITHOUT ANY WARRANTY expressed or implied, including the -# implied warranties of MERCHANTABILITY or FITNESS FOR A PARTICULAR -# PURPOSE. See the GNU General Public License for more details. You -# should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, -# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. # -# Any Red Hat trademarks that are incorporated in the source -# code or documentation are not subject to the GNU General Public -# License and may only be used or replicated with the express permission -# of Red Hat, Inc. +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. """This module contains tests for :mod:`waiverdb.events`.""" from __future__ import unicode_literals @@ -49,5 +43,13 @@ def test_fedmsg_new_waiver(mock_fedmsg, session): sesh.commit() mock_fedmsg.publish.assert_called_once_with( topic='waiver.new', - msg=waiver.__json__(), + msg={ + 'id': waiver.id, + 'result_id': 1, + 'username': 'jcline', + 'product_version': 'something', + 'waived': True, + 'comment': 'This is a comment', + 'timestamp': waiver.timestamp.isoformat(), + } ) diff --git a/waiverdb/events.py b/waiverdb/events.py index eb4e9cf..513055c 100644 --- a/waiverdb/events.py +++ b/waiverdb/events.py @@ -3,21 +3,15 @@ # This file is part of WaiverDB. # Copyright © 2017 Red Hat, Inc. # -# This copyrighted material is made available to anyone wishing to use, -# modify, copy, or redistribute it subject to the terms and conditions -# of the GNU General Public License v.2, or (at your option) any later -# version. This program is distributed in the hope that it will be -# useful, but WITHOUT ANY WARRANTY expressed or implied, including the -# implied warranties of MERCHANTABILITY or FITNESS FOR A PARTICULAR -# PURPOSE. See the GNU General Public License for more details. You -# should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, -# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. # -# Any Red Hat trademarks that are incorporated in the source -# code or documentation are not subject to the GNU General Public -# License and may only be used or replicated with the express permission -# of Red Hat, Inc. +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. """ This module contains a set of `SQLAlchemy event`_ hooks. @@ -32,12 +26,14 @@ from __future__ import unicode_literals from gettext import gettext as _ import logging +from flask_restful import marshal # fedmsg is an optional dependency and may not be present try: import fedmsg except ImportError: fedmsg = None +from waiverdb.fields import waiver_fields from waiverdb.models import Waiver @@ -89,4 +85,4 @@ def fedmsg_new_waiver(session): for row in session.identity_map.values(): if isinstance(row, Waiver): _log.debug('Publishing fedmsg for %r', row) - fedmsg.publish(topic='waiver.new', msg=row.__json__()) + fedmsg.publish(topic='waiver.new', msg=marshal(row, waiver_fields)) diff --git a/waiverdb/models/waivers.py b/waiverdb/models/waivers.py index 2d488af..1fde440 100644 --- a/waiverdb/models/waivers.py +++ b/waiverdb/models/waivers.py @@ -33,14 +33,3 @@ class Waiver(db.Model): return '%s(result_id=%r, username=%r, product_version=%r, waived=%r)' % ( self.__class__.__name__, self.result_id, self.username, self.product_version, self.waived) - - def __json__(self): - return { - 'id': self.id, - 'result_id': self.result_id, - 'username': self.username, - 'product_version': self.product_version, - 'waived': self.waived, - 'comment': self.comment, - 'timestamp': self.timestamp.isoformat(), - }