From e6bf065969090d7f89cf5a2c139c0094576cf50c Mon Sep 17 00:00:00 2001 From: Clement Verna Date: Mar 06 2019 20:56:06 +0000 Subject: Drop support for fedmsg and replace by fedora-messaging Signed-off-by: Clement Verna --- diff --git a/conf/fedora-messaging-example.toml b/conf/fedora-messaging-example.toml new file mode 100644 index 0000000..72d9719 --- /dev/null +++ b/conf/fedora-messaging-example.toml @@ -0,0 +1,18 @@ +# A sample configuration for fedora-messaging. This file is in the TOML format. +# For complete details on all configuration options, see the documentation. + +amqp_url = "amqp://" + +publish_exchange = "amq.topic" + +# The topic_prefix configuration value will add a prefix to the topics of every sent message. +# This is used for migrating from fedmsg, and should not be used afterwards. +topic_prefix = "" + +[tls] +ca_cert = "/etc/pki/tls/certs/ca-bundle.crt" +keyfile = "/my/client/key.pem" +certfile = "/my/client/cert.pem" + +[client_properties] +app = "WaiverDB" diff --git a/fedmsg.d/config.py b/fedmsg.d/config.py deleted file mode 100644 index 6f8dbbf..0000000 --- a/fedmsg.d/config.py +++ /dev/null @@ -1,129 +0,0 @@ -# -*- coding: utf-8 -*- -# SPDX-License-Identifier: GPL-2.0+ - -import socket - -hostname = socket.gethostname() - -config = dict( - active=True, - # Set this to dev if you're hacking on fedmsg or an app. - # Set to stg or prod if running in the Fedora Infrastructure - environment="dev", - - # Default is 0 - high_water_mark=0, - io_threads=1, - - ## For the fedmsg-hub and fedmsg-relay. ## - - # This is a status dir to keep a record of the last processed message - #status_directory=os.getcwd() + "/status", - #status_directory='/var/run/fedmsg/status', - - # This is the URL of a datagrepper instance that we can query for backlog. - #datagrepper_url="https://apps.fedoraproject.org/datagrepper/raw", - - # We almost always want the fedmsg-hub to be sending messages with zmq as - # opposed to amqp or stomp. You can send with only *one* of the messaging - # backends: zeromq or amqp or stomp. You cannot send with two or more at - # the same time. Here, zmq is either enabled, or it is not. If it is not, - # see the options below for how to configure stomp or amqp. - zmq_enabled=True, - - # On the other hand, if you wanted to use STOMP *instead* of zeromq, you - # could do the following... - #zmq_enabled=False, - #stomp_uri='localhost:59597,localhost:59598', - #stomp_user='username', - #stomp_pass='password', - #stomp_ssl_crt='/path/to/an/optional.crt', - #stomp_ssl_key='/path/to/an/optional.key', - - # When subscribing to messages, we want to allow splats ('*') so we tell - # the hub to not be strict when comparing messages topics to subscription - # topics. - zmq_strict=False, - - # Number of seconds to sleep after initializing waiting for sockets to sync - post_init_sleep=0.5, - - # Wait a whole second to kill all the last io threads for messages to - # exit our outgoing queue (if we have any). This is in milliseconds. - zmq_linger=1000, - - # See the following - # - http://tldp.org/HOWTO/TCP-Keepalive-HOWTO/overview.html - # - http://api.zeromq.org/3-2:zmq-setsockopt - zmq_tcp_keepalive=1, - zmq_tcp_keepalive_cnt=3, - zmq_tcp_keepalive_idle=60, - zmq_tcp_keepalive_intvl=5, - - # Number of miliseconds that zeromq will wait to reconnect until it gets - # a connection if an endpoint is unavailable. - zmq_reconnect_ivl=100, - # Max delay that you can reconfigure to reduce reconnect storm spam. This - # is in miliseconds. - zmq_reconnect_ivl_max=1000, - - # This is a dict of possible addresses from which fedmsg can send - # messages. fedmsg.init(...) requires that a 'name' argument be passed - # to it which corresponds with one of the keys in this dict. - endpoints={ - "waiverdb.%s" % hostname: [ - "tcp://127.0.0.1:5011", - ], - "relay_outbound": [ - "tcp://127.0.0.1:4001", - ], - }, - # This is the address of an active->passive relay. It is used for the - # fedmsg-logger command which requires another service with a stable - # listening address for it to send messages to. - # It is also used by the git-hook, for the same reason. - # It is also used by the mediawiki php plugin which, due to the oddities of - # php, can't maintain a single passive-bind endpoint of it's own. - relay_inbound=[ - "tcp://127.0.0.1:2003", - ], - sign_messages=False, - validate_signatures=False, - - # Use these implementations to sign and validate messages - crypto_backend='x509', - crypto_validate_backends=['x509'], - - ssldir="/etc/pki/fedmsg", - crl_location="https://fedoraproject.org/fedmsg/crl.pem", - crl_cache="/var/run/fedmsg/crl.pem", - crl_cache_expiry=10, - - ca_cert_location="https://fedoraproject.org/fedmsg/ca.crt", - ca_cert_cache="/var/run/fedmsg/ca.crt", - ca_cert_cache_expiry=0, # Never expires - - certnames={ - # In prod/stg, map hostname to the name of the cert in ssldir. - # Unfortunately, we can't use socket.getfqdn() - #"app01.stg": "app01.stg.phx2.fedoraproject.org", - }, - - # A mapping of fully qualified topics to a list of cert names for which - # a valid signature is to be considered authorized. Messages on topics not - # listed here are considered automatically authorized. - routing_policy={ - # Only allow announcements from production if they're signed by a - # certain certificate. - "org.fedoraproject.prod.announce.announcement": [ - "announce-lockbox.phx2.fedoraproject.org", - ], - }, - - # Set this to True if you want messages to be dropped that aren't - # explicitly whitelisted in the routing_policy. - # When this is False, only messages that have a topic in the routing_policy - # but whose cert names aren't in the associated list are dropped; messages - # whose topics do not appear in the routing_policy are not dropped. - routing_nitpicky=False, -) diff --git a/requirements.txt b/requirements.txt index a16a131..f805c98 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,7 @@ # This is a list of pypi packages to be installed into virtualenv. Alternatively, # you can install these as RPMs instead of pypi packages. -fedmsg[consumers,commands] +fedora_messaging Flask Flask-RESTful!=0.3.6 Flask-SQLAlchemy diff --git a/tests/conftest.py b/tests/conftest.py index d9d07a5..251fbc8 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -2,6 +2,7 @@ import os from copy import copy +from mock import patch import pytest from sqlalchemy import create_engine from waiverdb.app import create_app @@ -63,7 +64,8 @@ def client(app): by default. """ with app.test_client() as client: - yield client + with patch('fedora_messaging.api._session_cache'): + yield client @pytest.fixture() diff --git a/tests/test_events.py b/tests/test_events.py index f248572..1aabf42 100644 --- a/tests/test_events.py +++ b/tests/test_events.py @@ -2,12 +2,13 @@ """This module contains tests for :mod:`waiverdb.events`.""" from __future__ import unicode_literals -import mock +from fedora_messaging import api, testing +from flask_restful import marshal from waiverdb.models import Waiver +from waiverdb.fields import waiver_fields -@mock.patch('waiverdb.events.fedmsg') -def test_publish_new_waiver_with_fedmsg(mock_fedmsg, session): +def test_publish_new_waiver_with_fedmsg(session): waiver = Waiver( subject_type='koji_build', subject_identifier='glibc-2.26-27.fc27', @@ -17,29 +18,21 @@ def test_publish_new_waiver_with_fedmsg(mock_fedmsg, session): waived=True, comment='This is a comment', ) + sesh = session() sesh.add(waiver) - sesh.commit() - mock_fedmsg.publish.assert_called_once_with( - topic='waiver.new', - msg={ - 'id': waiver.id, - 'subject_type': 'koji_build', - 'subject_identifier': 'glibc-2.26-27.fc27', - 'subject': {'type': 'koji_build', 'item': 'glibc-2.26-27.fc27'}, - 'testcase': 'testcase1', - 'username': 'jcline', - 'proxied_by': None, - 'product_version': 'something', - 'waived': True, - 'comment': 'This is a comment', - 'timestamp': waiver.timestamp.isoformat(), - } + sesh.flush() + + expected_msg = api.Message( + topic='waiverdb.waiver.new', + body=marshal(waiver, waiver_fields) ) + with testing.mock_sends(expected_msg): + sesh.commit() -@mock.patch('waiverdb.events.fedmsg') -def test_publish_new_waiver_with_fedmsg_for_proxy_user(mock_fedmsg, session): + +def test_publish_new_waiver_with_fedmsg_for_proxy_user(session): waiver = Waiver( subject_type='koji_build', subject_identifier='glibc-2.26-27.fc27', @@ -52,20 +45,11 @@ def test_publish_new_waiver_with_fedmsg_for_proxy_user(mock_fedmsg, session): ) sesh = session() sesh.add(waiver) - sesh.commit() - mock_fedmsg.publish.assert_called_once_with( - topic='waiver.new', - msg={ - 'id': waiver.id, - 'subject_type': 'koji_build', - 'subject_identifier': 'glibc-2.26-27.fc27', - 'subject': {'type': 'koji_build', 'item': 'glibc-2.26-27.fc27'}, - 'testcase': 'testcase1', - 'username': 'jcline', - 'proxied_by': 'bodhi', - 'product_version': 'something', - 'waived': True, - 'comment': 'This is a comment', - 'timestamp': waiver.timestamp.isoformat(), - } + sesh.flush() + + expected_msg = api.Message( + topic='waiverdb.waiver.new', + body=marshal(waiver, waiver_fields) ) + with testing.mock_sends(expected_msg): + sesh.commit() diff --git a/tox.ini b/tox.ini index 0256e61..aba4b9a 100644 --- a/tox.ini +++ b/tox.ini @@ -42,7 +42,7 @@ commands = [flake8] show-source = True max-line-length = 100 -exclude = .git,.tox,dist,*egg,env_waiverdb,*fedmsg.d,docs,conf,waiverdb/migrations +exclude = .git,.tox,dist,*egg,env_waiverdb,docs,conf,waiverdb/migrations # E124: closing bracket does not match visual indentation # W503 line break before binary operator diff --git a/waiverdb.spec b/waiverdb.spec index bc90e95..32c890a 100644 --- a/waiverdb.spec +++ b/waiverdb.spec @@ -42,7 +42,7 @@ BuildRequires: python3-flask-oidc BuildRequires: python3-click BuildRequires: python3-flask-migrate BuildRequires: python3-stomppy -BuildRequires: python3-fedmsg +BuildRequires: python3-fedora-messaging BuildRequires: python3-prometheus_client BuildRequires: python3-six Requires: python3-flask @@ -57,7 +57,7 @@ Requires: python3-flask-oidc Requires: python3-click Requires: python3-flask-migrate Requires: python3-stomppy -Requires: python3-fedmsg +Requires: python3-fedora-messaging Requires: python3-prometheus_client Requires: waiverdb-common = %{version}-%{release} %endif diff --git a/waiverdb/config.py b/waiverdb/config.py index add3ec6..ac1fe14 100644 --- a/waiverdb/config.py +++ b/waiverdb/config.py @@ -6,10 +6,6 @@ import os class Config(object): """ A WaiverDB Flask configuration. - - Attributes: - ZEROMQ_PUBLISH (bool): When true, ZeroMQ messages will be emitted via - fedmsg when new waivers are created. """ DEBUG = True DATABASE_URI = 'postgresql+psycopg2:///waiverdb' diff --git a/waiverdb/events.py b/waiverdb/events.py index f9b521e..4ecfc31 100644 --- a/waiverdb/events.py +++ b/waiverdb/events.py @@ -12,11 +12,12 @@ using the :func:`sqlalchemy.event.listen` function. import logging from flask_restful import marshal -import fedmsg import stomp import json import waiverdb.monitor as monitor +from fedora_messaging.api import Message, publish +from fedora_messaging.exceptions import PublishReturned, ConnectionException from flask import current_app from waiverdb.fields import waiver_fields from waiverdb.models import Waiver @@ -28,7 +29,7 @@ _log = logging.getLogger(__name__) def publish_new_waiver(session): """ A post-commit event hook that emits messages to a message bus. The messages - can be published by either fedmsg-hub with zmq or stomp. + can be published by either fedora-messaging or stomp. This event is designed to be registered with a session factory:: @@ -91,10 +92,17 @@ def publish_new_waiver(session): continue _log.debug('Publishing a message for %r', row) try: - fedmsg.publish(topic='waiver.new', msg=marshal(row, waiver_fields)) + msg = Message( + topic='waiverdb.waiver.new', + body=marshal(row, waiver_fields) + ) + publish(msg) monitor.messaging_tx_sent_ok_counter.inc() - except Exception: - _log.exception('Couldn\'t publish message via fedmsg') + except PublishReturned as e: + _log.exception('Fedora Messaging broker rejected message %s: %s', msg.id, e) + monitor.messaging_tx_failed_counter.inc() + except ConnectionException as e: + _log.exception('Error sending message %s: %s', msg.id, e) monitor.messaging_tx_failed_counter.inc() raise