From aa6f9f35b6f9f82d7a06070fac51e61d28f8c626 Mon Sep 17 00:00:00 2001 From: Adam Miller Date: Apr 12 2017 20:04:46 +0000 Subject: add setup.py, README.rst, requirements, gitignore make python package, fix setup.py add import logging to consumer add test/hack to README remove .pyc files Signed-off-by: Adam Miller --- diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..ff65a43 --- /dev/null +++ b/.gitignore @@ -0,0 +1,97 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +env/ +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +*.egg-info/ +.installed.cfg +*.egg + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*,cover +.hypothesis/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# pyenv +.python-version + +# celery beat schedule file +celerybeat-schedule + +# SageMath parsed files +*.sage.py + +# dotenv +.env + +# virtualenv +.venv +venv/ +ENV/ + +# Spyder project settings +.spyderproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site diff --git a/README.rst b/README.rst new file mode 100644 index 0000000..d345bf4 --- /dev/null +++ b/README.rst @@ -0,0 +1,43 @@ +fedmsg-rabbitmq-serializer +========================== + +``fedmsg-rabbitmq-serializer`` is a small utility that will serialize fedmsg +data into a RabbitMQ worker queue. + +Testing/Hacking +--------------- + +It is recommended that you are running this on the same system you have +``rabbitmq`` installed on for simplicity. + +For Fedora, this will suffice: + +:: + + $ dnf -y install rabbitmq-server + $ systemctl start rabbitmq-server + + +Next up you're going to want to setup a virtualenv for this, if you don't have +the virtualenv wrapper utils installed, go ahead and do that. + +:: + + $ dnf -y install python2-virtualenvwrapper + + $ mkvirtualenv f-r-s + +Now run ``setup.py`` to get things going ... this will take a while. + +:: + + $(f-r-s) python setup.py develop + + +With all the deps installed as needed and rabbitmq all setup, we can go ahead +and run the ``fedmsg-hub`` (which this is a consumer plugin for). + +:: + + $(f-r-s) fedmsg-hub + diff --git a/consumer.py b/consumer.py deleted file mode 100644 index 784fc46..0000000 --- a/consumer.py +++ /dev/null @@ -1,65 +0,0 @@ -""" -This is a `fedmsg consumer`_ that subscribes to every topic on the message bus -it is connected to. It just places all messages into a RabbitMQ message queue. -""" - -import fedmsg.consumers -import json -import pika - - -log = logging.getLogger("fedmsg-rabbitmq-serializer") - -OPTS = pika.ConnectionParameters( - heartbeat_interval=0, - retry_delay=2, -) - - -class RabbitMQSerializerConsumer(fedmsg.consumers.FedmsgConsumer): - """ - A `fedmsg consumer`_ that subscribes to all topics and re-publishes all - messages to the ``workers`` exchange. - - Attributes: - topic (str): The topics this consumer is subscribed to. Set to ``*`` - (all topics). - config_key (str): The key to set to ``True`` in the fedmsg config to - enable this consumer. The key is ``rabbitmq.serializer.enabled``. - """ - topic = '*' - config_key = 'rabbitmq.serializer.enabled' - - def __init__(self, *args, **kwargs): - log.debug("RabbitMQ Serializer initializing") - super(RabbitMQSerializerConsumer, self).__init__(*args, **kwargs) - log.debug("RabbitMQ Serializer initialized") - - def consume(self, raw_msg): - """ - This method is called when a message arrives on the fedmsg bus. - - Args: - raw_msg (dict): The raw fedmsg deserialized to a Python dictionary. - """ - connection = pika.BlockingConnection(OPTS) - channel = connection.channel() - channel.exchange_declare(exchange='workers') - channel.queue_declare('workers', durable=True) - channel.basic_publish( - exchange='', - routing_key='workers', - body=json.dumps(raw_msg), - properties=pika.BasicProperties( - delivery_mode=2 - ) - ) - channel.close() - connection.close() - - def stop(self): - """ - Gracefully halt this fedmsg consumer. - """ - log.info("Cleaning up RabbitMQ Serializer.") - super(RabbitMQSerializerConsumer, self).stop() diff --git a/fedmsg_rabbitmq_serializer/__init__.py b/fedmsg_rabbitmq_serializer/__init__.py new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/fedmsg_rabbitmq_serializer/__init__.py diff --git a/fedmsg_rabbitmq_serializer/consumer.py b/fedmsg_rabbitmq_serializer/consumer.py new file mode 100644 index 0000000..76b0f5b --- /dev/null +++ b/fedmsg_rabbitmq_serializer/consumer.py @@ -0,0 +1,66 @@ +""" +This is a `fedmsg consumer`_ that subscribes to every topic on the message bus +it is connected to. It just places all messages into a RabbitMQ message queue. +""" + +import logging +import fedmsg.consumers +import json +import pika + + +log = logging.getLogger("fedmsg-rabbitmq-serializer") + +OPTS = pika.ConnectionParameters( + heartbeat_interval=0, + retry_delay=2, +) + + +class RabbitMQSerializerConsumer(fedmsg.consumers.FedmsgConsumer): + """ + A `fedmsg consumer`_ that subscribes to all topics and re-publishes all + messages to the ``workers`` exchange. + + Attributes: + topic (str): The topics this consumer is subscribed to. Set to ``*`` + (all topics). + config_key (str): The key to set to ``True`` in the fedmsg config to + enable this consumer. The key is ``rabbitmq.serializer.enabled``. + """ + topic = '*' + config_key = 'rabbitmq.serializer.enabled' + + def __init__(self, *args, **kwargs): + log.debug("RabbitMQ Serializer initializing") + super(RabbitMQSerializerConsumer, self).__init__(*args, **kwargs) + log.debug("RabbitMQ Serializer initialized") + + def consume(self, raw_msg): + """ + This method is called when a message arrives on the fedmsg bus. + + Args: + raw_msg (dict): The raw fedmsg deserialized to a Python dictionary. + """ + connection = pika.BlockingConnection(OPTS) + channel = connection.channel() + channel.exchange_declare(exchange='workers') + channel.queue_declare('workers', durable=True) + channel.basic_publish( + exchange='', + routing_key='workers', + body=json.dumps(raw_msg), + properties=pika.BasicProperties( + delivery_mode=2 + ) + ) + channel.close() + connection.close() + + def stop(self): + """ + Gracefully halt this fedmsg consumer. + """ + log.info("Cleaning up RabbitMQ Serializer.") + super(RabbitMQSerializerConsumer, self).stop() diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..bcaca98 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,5 @@ +fedmsg[consumers] +fedmsg_meta_fedora_infrastructure +moksha.hub +pika +python-fedora diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..e22f21d --- /dev/null +++ b/setup.py @@ -0,0 +1,59 @@ +"""Setup file for fedmsg-rabbitmq-serializer""" + +import sys + +from setuptools import setup, find_packages + + +def get_description(): + with open('README.rst', 'r') as f: + return ''.join(f.readlines()[2:]) + + +def get_requirements(filename='requirements.txt'): + """ + Get the contents of a file listing the requirements. + + :param filename: path to a requirements file + :type filename: str + + :returns: the list of requirements + :return type: list + """ + with open(filename) as f: + return [ + line.rstrip().split('#')[0] + for line in f.readlines() + if not line.startswith('#') + ] + + +requires = get_requirements() +if sys.version_info[0] == 2 and sys.version_info[1] <= 6: + requires.append("ordereddict") + +setup( + name='fedmsg-rabbitmq-serializer', + version='0.0.1', + description='Small utility to serialize fedmsgs into rabbitmq worker queue', + long_description=get_description(), + author='Fedora Infrastructure Team', + author_email='infrastructure@lists.fedoraproject.org', + url="https://pagure.io/fedmsg-rabbitmq-serializer", + download_url="https://pypi.python.org/pypi/fedmsg-rabbitmq-serializer/", + license='LGPLv2+', + install_requires=requires, + zip_safe=False, + classifiers=[ + 'Development Status :: 3 - Alpha', + 'License :: OSI Approved :: GNU Lesser General Public License v2 or later (LGPLv2+)', + 'Programming Language :: Python', + 'Programming Language :: Python :: 2', + 'Programming Language :: Python :: 2.7', + ], + entry_points={ + 'moksha.consumer': [ + "fedmsg_rabbitmq_serializer_consumer = fedmsg_rabbitmq_serializer.consumer:RabbitMQSerializerConsumer", + ], + }, +)