From 44fceef22e13975a01c52bfdac42af94abe07be2 Mon Sep 17 00:00:00 2001 From: Sayan Chowdhury Date: Jun 14 2019 13:07:17 +0000 Subject: Add the tests and refactor code Signed-off-by: Sayan Chowdhury --- diff --git a/.gitignore b/.gitignore index c8db487..e309bc4 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ +__pycache__ fedora-messaging.toml dist diff --git a/MANIFEST.in b/MANIFEST.in index 81dfe9b..e007a4d 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -3,3 +3,4 @@ include README.md include fedora-messaging.toml.example recursive-include scripts *.py +recursive-include tests *.py diff --git a/README.md b/README.md index afffb12..d5a6343 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,20 @@ ## Joystick +Joystick is a fedora-messaging consumer that listens to `org.fedoraproject.prod.pungi.compose.status.change`. The application processes the messages, the extract out the required metadata and call `plume` with the required arguments. The project is in active development state + +### Development + +``` +git clone https://pagure.io/joystick +cd joystick +python3 setup.py develop +fedora-messaging --conf joystick.toml consume +``` + +### Tests + +``` +pytest tests/ +``` + diff --git a/joystick/consumers/fedora_messaging_consumer.py b/joystick/consumers/fedora_messaging_consumer.py index 6ed9315..acd58f3 100644 --- a/joystick/consumers/fedora_messaging_consumer.py +++ b/joystick/consumers/fedora_messaging_consumer.py @@ -1,3 +1,5 @@ +# -*- coding: utf-8 -*- + import boto3 import fedfind import fedfind.release @@ -17,18 +19,27 @@ class JoyStickController(object): """ JoyStick controller for plume. """ - def __init__(self): + """ This method initializes all the configs needed for the process the + images. + """ self.config = conf["consumer_config"] self.aws_access_key_id = self.config['aws_access_key_id'] self.aws_secret_access_key = self.config['aws_secret_access_key'] - self.valid_status = ('FINISHED_INCOMPLETE', 'FINISHED') - self.regions = self.config['regions'] + self.channel = "" self.environment = self.config['environment'] + self.regions = self.config['regions'] self.topic = "org.fedoraproject.%s.pungi.compose.status.change" % ( self.environment) + self.valid_status = ('FINISHED_INCOMPLETE', 'FINISHED') def run_command(self, command): + """ This method takes in the command and runs the command using + `subprocess.run` method. + + :args command: `str` the command to execute. + """ + _log.info("Starting the command: %r" % command) process = subprocess.run(command, capture_output=True) @@ -37,6 +48,8 @@ class JoyStickController(object): output = process.stdout.decode('utf-8') _log.info("Finished executing the command: %r" % command) + # If the return code is a non-zero number the command has failed, log + # the error and return the output, error, returncode if returncode != 0: _log.error("Command failed during run") _log.error("(output) %s, (error) %s, (retcode) %s" % ( @@ -47,81 +60,125 @@ class JoyStickController(object): return output, error, returncode + def process_upload(self, image_type, board): + """ This process accepts the one of the combinations of image_type and + board. For example it would accept, "Cloud" and "aarch64" to generate + the name of the image and fetch the image from koji. + + :args image_type: `str` type of the image. + :args board: `str` name of the board. + """ + self.image_type = image_type + self.board = board + + # channel args does not have image type as "AtomicHost" + if self.channel == 'cloud' and self.image_type == 'AtomicHost': + return + + # Run the 4-step process. The first step is the `pre-release` step, + # where plume downloads the image, and uploads the image to various + # cloud providers. During this step, the artifacts created are kept + # private. + output, error, retcode = self.run_pre_release() + if retcode != 0: + _log.debug("There was an issue with pre-release") + _log.debug(error) + return + + # If `pre-release` step is success, then the messages are published + # with to message bus via fedora-messaging. + output, error, retcode = self.push_upload_messages() + if retcode != 0: + _log.debug("There was an issue with publishing messages") + _log.debug(error) + + # During this step, the private artifacts are made public. + output, error, retcode = self.run_release() + if retcode != 0: + _log.debug("There was an issue with release") + _log.debug(error) + return + + # If the process of making the artifacts public is success, then the + # messages are pushed to message bus. + output, error, retcode = self.push_publish_messages() + if retcode != 0: + _log.debug("There was an issue with publishing messages") + _log.debug(error) + + return + + def process_joystick_topic(self, msg_info): + """ Process the message info to extract information needed by plume + binary and trigger the upload process. + + :args msg_info: `dict` fedora-messaging message data + """ + _log.debug("Processing %r" % (msg_info['id'])) + if msg_info['status'] not in self.valid_status: + _log.debug("%s is not a valid status" % msg_info["status"]) + return + + compose_id = msg_info.get('compose_id') + try: + compose_metadata = fedfind.release.get_release( + cid=compose_id).metadata + except fedfind.exceptions.UnsupportedComposeError: + _log.error("%s is unsupported composes" % compose_id) + return + + self.location = msg_info['location'] + + # Filter the channel from the location extracted from the message + self.channel = self.location.rsplit('/', 3) + if not self.channel: + _log.debug('channel cannot be empty') + return + + self.channel = self.channel[1] + if self.channel not in ['cloud', 'branched', 'updates', 'rawhide']: + _log.debug('%s is not a valid channel' % self.channel) + return + + self.compose_id = msg_info['compose_id'] + self.respin = msg_info['compose_respin'] + self.timestamp = msg_info['compose_date'] + self.release_version = msg_info['release_version'] + self.aws_crendentials = '~/.aws/credentials' + for image_type, board in itertools.product(['Cloud-Base', 'AtomicHost'], ['x86_64', 'aarch64']): + self.process_upload(image_type, board) + def __call__(self, msg): + """ This method is called by `fedora-messaging` to process the messages + received. + + :args msg: `dict` fedora-messaging message + """ + _log.info("Received message: %s", msg.id) topic = msg.topic body = msg.body - msg_info = {} + msg_info = { + 'id': msg.id + } if 'msg' not in body: + _log.debug("Invalid message body: %r" % msg.id) return else: msg_info = body['msg'] + # Matches if the current topic is the intended one and needs to be + # processed if topic == self.topic: - _log.debug("Processing %r" % (msg.id)) - if msg_info['status'] not in self.valid_status: - _log.debug("%s is not a valid status" % msg_info["status"]) - return - - compose_id = msg_info.get('compose_id') - try: - compose_metadata = fedfind.release.get_release( - cid=compose_id).metadata - except fedfind.exceptions.UnsupportedComposeError: - _log.error("%s is unsupported composes" % compose_id) - return - - self.location = msg_info['location'] - - self.channel = self.location.rsplit('/', 3) - if not self.channel: - _log.debug('channel cannot be empty') - return - - self.channel = self.channel[1] - if self.channel not in ['cloud', 'branched', 'updates', 'rawhide']: - _log.debug('%s is not a valid channel' % self.channel) - return - - self.compose_id = msg_info['compose_id'] - self.respin = msg_info['compose_respin'] - self.timestamp = msg_info['compose_date'] - self.release_version = msg_info['release_version'] - self.aws_crendentials = '~/.aws/credentials' - for image_type, board in itertools.product(['Cloud-Base', 'AtomicHost'], ['x86_64', 'aarch64']): - self.image_type = image_type - self.board = board - - if self.channel == 'cloud' and self.image_type == 'AtomicHost': - continue - - output, error, retcode = self._run_pre_release() - if not retcode != 0: - _log.debug("There was an issue with pre-release") - _log.debug(error) - continue - - output, error, retcode = self._publish_upload_messages() - if not retcode != 0: - _log.debug("There was an issue with publishing messages") - _log.debug(error) - - output, error, retcode = self._run_release() - if not retcode != 0: - _log.debug("There was an issue with release") - _log.debug(error) - continue - - output, error, retcode = self._publish_publish_messages() - if not retcode != 0: - _log.debug("There was an issue with publishing messages") - _log.debug(error) + self.process_joystick_topic(msg_info) else: _log.debug("Dropping %r: %r" % (topic, msg.id)) - pass + return + + def run_pre_release(self): + """ Runs the pre release step of the plume process.""" - def _run_pre_release(self): output, error, retcode = self.run_command([ 'plume', 'pre-release', @@ -138,7 +195,8 @@ class JoyStickController(object): return output, error, retcode - def _run_release(self): + def run_release(self): + """ Runs the release step of the plume process.""" output, error, retcode = self.run_command([ 'plume', 'release', @@ -155,7 +213,10 @@ class JoyStickController(object): return output, error, retcode - def _generate_ami_upload_list(self): + def generate_ami_upload_list(self): + """Iterates through the regions, and fetches the AMI information + using the compose id.""" + for region in self.regions: conn = boto3.client( "ec2", @@ -186,8 +247,11 @@ class JoyStickController(object): } for image in images['Images'] ] - def _publish_messages(self): - self._generate_ami_upload_list() + def push_upload_messages(self): + """ Publishes the messages to the fedora message bus with the + `image.upload` topic. + """ + self.generate_ami_upload_list() for image in self.uploaded_images: msg = Message( @@ -205,8 +269,11 @@ class JoyStickController(object): ) api.publish(msg) - def _publish_upload_messages(self): - self._generate_ami_upload_list() + def push_publish_messages(self): + """ Publishes the messages to the fedora message bus with the + `image.publish` topic. + """ + self.generate_ami_upload_list() for image in self.uploaded_images: msg = Message( diff --git a/setup.py b/setup.py index 4e2d619..2791be6 100644 --- a/setup.py +++ b/setup.py @@ -12,6 +12,11 @@ setup( packages=['joystick', 'joystick.consumers'], package_dir={'joystick': 'joystick'}, package_data={'joystick': ['config/joystick.toml']}, + tests_require=[ + 'pytest', + 'pytest-cov', + 'mock' + ], classifiers={ 'Development Status :: 2 - Pre-Alpha', 'Environment :: Console', diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..a977ed8 --- /dev/null +++ b/tests/__init__.py @@ -0,0 +1,90 @@ +class Client(): + def describe_images(self, *args, **kwargs): + return { + 'Images': [{ + 'ImageId': 'ami-foobar12', + 'Architecture': 'x86_64', + 'VirtualizationType': 'hvm', + 'BlockDeviceMappings':[{ + 'Ebs': { + 'VolumeType': 'gp2', + } + }] + }, + { + 'ImageId': 'ami-foobar23', + 'Architecture': 'aarch64', + 'VirtualizationType': 'hvm', + 'BlockDeviceMappings':[{ + 'Ebs': { + 'VolumeType': 'gp2', + } + }] + } + ]} + +class Message(object): + def __init__(self): + self.topic = "org.fedoraproject.random.topic" + self.id = '2019-random-msg-id' + + self.body = { + 'msg': { + "status": "FINISHED", + "release_type": "ga", + "compose_label": "RC-20190611.0", + "compose_respin": 0, + "compose_date": "20190611", + "release_version": "30", + "location": "https://kojipkgs.fedoraproject.org/compose/cloud/Fedora-Cloud-30-20190611.0/compose", + "compose_type": "production", + "release_is_layered": False, + "release_name": "Fedora-Cloud", + "release_short": "Fedora-Cloud", + "compose_id": "Fedora-Cloud-30-20190611.0" + } + } + + +class NonMatchingTopicMessage(object): + def __init__(self): + self.topic = "org.fedoraproject.random.topic" + self.id = '2019-random-msg-id' + + self.body = { + 'msg': { + "status": "FINISHED", + "release_type": "ga", + "compose_label": "RC-20190611.0", + "compose_respin": 0, + "compose_date": "20190611", + "release_version": "30", + "location": "https://kojipkgs.fedoraproject.org/compose/cloud/Fedora-Cloud-30-20190611.0/compose", + "compose_type": "production", + "release_is_layered": False, + "release_name": "Fedora-Cloud", + "release_short": "Fedora-Cloud", + "compose_id": "Fedora-Cloud-30-20190611.0" + } + } + + +class NoMsgInBodyMessage(object): + def __init__(self): + self.topic = "org.fedoraproject.random.topic" + self.id = '2019-random-msg-id' + self.body = {} + + +class ProcessSuccess(object): + def __init__(self): + self.returncode = 0 + self.stderr = b"" + self.stdout = b"Success" + + +class ProcessFailure(object): + def __init__(self): + self.returncode = 1 + self.stderr = b"Command Failed" + self.stdout = b"Failure" diff --git a/tests/test_consumer.py b/tests/test_consumer.py new file mode 100644 index 0000000..2264d19 --- /dev/null +++ b/tests/test_consumer.py @@ -0,0 +1,172 @@ +import mock +import os +import unittest + +from fedora_messaging.config import conf +from joystick.consumers.fedora_messaging_consumer import JoyStickController +from tests import Client, Message, NoMsgInBodyMessage, NonMatchingTopicMessage +from tests import ProcessSuccess, ProcessFailure + +RUN_COMMAND_OUTPUT_SUCCESS = ProcessSuccess() +RUN_COMMAND_OUTPUT_FAILURE = ProcessFailure() +RUN_COMMAND_OUTPUT_SUCCESS_T = ('Success', '', 0) +RUN_COMMAND_OUTPUT_FAILURE_T = ('Failure', 'Command Failed', 1) +BOTO_RETURN_VALUE = Client() + +@mock.patch('boto3.client', return_value=BOTO_RETURN_VALUE) +def test_generate_ami_upload_list(mock_boto_client): + config_path = os.path.abspath("tests/testjoystick.toml") + conf.load_config(config_path=config_path) + js = JoyStickController() + setattr(js, 'compose_id', 'Fedora-Compose-ID') + js.generate_ami_upload_list() + + +@mock.patch('joystick.consumers.fedora_messaging_consumer._log.debug') +def test_process_joystick_test_call_no_msg_in_body(mock_log): + msg = NoMsgInBodyMessage() + js = JoyStickController() + js.__call__(msg) + + mock_log.assert_called_with("Invalid message body: '2019-random-msg-id'") + +@mock.patch('joystick.consumers.fedora_messaging_consumer._log.debug') +def test_process_joystick_test_call_non_matching_topic(mock_log): + msg = NonMatchingTopicMessage() + + js = JoyStickController() + # Set the topic to the pungi topic we intend to listen + setattr(js, 'topic', 'org.fedoraproject.pungi.compose.status.change') + js.__call__(msg=msg) + + mock_log.assert_called_with( + "Dropping 'org.fedoraproject.random.topic': '2019-random-msg-id'" + ) + +@mock.patch('subprocess.run', return_value=RUN_COMMAND_OUTPUT_SUCCESS) +@mock.patch('joystick.consumers.fedora_messaging_consumer._log.debug') +def test_run_command_success(mock_log, mock_subprocess_run): + js = JoyStickController() + output = js.run_command('ls -l') + mock_log.assert_called_with( + "(output) Success, (error) , (retcode) 0" + ) + + +@mock.patch('subprocess.run', return_value=RUN_COMMAND_OUTPUT_FAILURE) +@mock.patch('joystick.consumers.fedora_messaging_consumer._log.error') +def test_run_command_success(mock_log, mock_subprocess_run): + js = JoyStickController() + output = js.run_command('ls -l') + mock_log.assert_called_with( + "(output) Failure, (error) Command Failed, (retcode) 1" + ) + +@mock.patch('joystick.consumers.fedora_messaging_consumer.JoyStickController.push_upload_messages', + return_value=RUN_COMMAND_OUTPUT_SUCCESS_T) +@mock.patch('joystick.consumers.fedora_messaging_consumer.JoyStickController.run_release', + return_value=RUN_COMMAND_OUTPUT_SUCCESS_T) +@mock.patch('joystick.consumers.fedora_messaging_consumer.JoyStickController.push_publish_messages', + return_value=RUN_COMMAND_OUTPUT_SUCCESS_T) +@mock.patch('joystick.consumers.fedora_messaging_consumer.JoyStickController.run_pre_release', + return_value=RUN_COMMAND_OUTPUT_FAILURE_T) +@mock.patch('joystick.consumers.fedora_messaging_consumer._log.debug') +def test_process_upload_pre_release_fail(mock_log, mock_run_pre_release, + mock_push_publish_messages, mock_run_release, + mock_push_upload_messages): + js = JoyStickController() + js.process_upload('AtomicHost', 'x86_64') + + mock_log.assert_called_with('Command Failed') + + +@mock.patch('joystick.consumers.fedora_messaging_consumer.JoyStickController.push_upload_messages', + return_value=RUN_COMMAND_OUTPUT_SUCCESS_T) +@mock.patch('joystick.consumers.fedora_messaging_consumer.JoyStickController.run_release', + return_value=RUN_COMMAND_OUTPUT_SUCCESS_T) +@mock.patch('joystick.consumers.fedora_messaging_consumer.JoyStickController.push_publish_messages', + return_value=RUN_COMMAND_OUTPUT_FAILURE_T) +@mock.patch('joystick.consumers.fedora_messaging_consumer.JoyStickController.run_pre_release', + return_value=RUN_COMMAND_OUTPUT_SUCCESS_T) +@mock.patch('joystick.consumers.fedora_messaging_consumer._log.debug') +def test_process_upload_push_publish_msg_fail(mock_log, mock_run_pre_release, + mock_push_publish_messages, mock_run_release, + mock_push_upload_messages): + js = JoyStickController() + js.process_upload('AtomicHost', 'x86_64') + + mock_log.assert_called_with('Command Failed') + + +@mock.patch('joystick.consumers.fedora_messaging_consumer.JoyStickController.push_upload_messages', + return_value=RUN_COMMAND_OUTPUT_SUCCESS_T) +@mock.patch('joystick.consumers.fedora_messaging_consumer.JoyStickController.run_release', + return_value=RUN_COMMAND_OUTPUT_FAILURE_T) +@mock.patch('joystick.consumers.fedora_messaging_consumer.JoyStickController.push_publish_messages', + return_value=RUN_COMMAND_OUTPUT_SUCCESS_T) +@mock.patch('joystick.consumers.fedora_messaging_consumer.JoyStickController.run_pre_release', + return_value=RUN_COMMAND_OUTPUT_SUCCESS_T) +@mock.patch('joystick.consumers.fedora_messaging_consumer._log.debug') +def test_process_upload_release_fail(mock_log, mock_run_pre_release, + mock_push_publish_messages, mock_run_release, + mock_push_upload_messages): + js = JoyStickController() + js.process_upload('AtomicHost', 'x86_64') + + mock_log.assert_called_with('Command Failed') + + +@mock.patch('joystick.consumers.fedora_messaging_consumer.JoyStickController.push_upload_messages', + return_value=RUN_COMMAND_OUTPUT_FAILURE_T) +@mock.patch('joystick.consumers.fedora_messaging_consumer.JoyStickController.run_release', + return_value=RUN_COMMAND_OUTPUT_FAILURE_T) +@mock.patch('joystick.consumers.fedora_messaging_consumer.JoyStickController.push_publish_messages', + return_value=RUN_COMMAND_OUTPUT_SUCCESS_T) +@mock.patch('joystick.consumers.fedora_messaging_consumer.JoyStickController.run_pre_release', + return_value=RUN_COMMAND_OUTPUT_SUCCESS_T) +@mock.patch('joystick.consumers.fedora_messaging_consumer._log.debug') +def test_process_upload_push_upload_msg_fail(mock_log, mock_run_pre_release, + mock_push_publish_messages, mock_run_release, + mock_push_upload_messages): + js = JoyStickController() + js.process_upload('AtomicHost', 'x86_64') + + mock_log.assert_called_with('Command Failed') + + + +""" +class TestJoystickController(unittest.TestCase): + @mock.patch('subprocess.run', + mock.MagicMock(return_value=RUN_COMMAND_OUTPUT_SUCCESS)) + def test_run_command_success(self): + output = self.joystick.run_command('ls -l') + + @mock.patch('subprocess.run', + mock.MagicMock(return_value=RUN_COMMAND_OUTPUT_FAIL)) + def test_run_command_fail(self): + output = self.joystick.run_command('sl -l') + + @mock.patch('JoystickController.process_upload', + return_value=True) + def test_process_joystick_topic_empty_channel(self): + self.joystick._process_joystick_topic(msg_info) + + @mock.patch('JoystickController.process_upload', + return_value=True) + @mock.patch('fedfind.release.get_release', + return_value=True) + def test_process_joystick_topic_invalid_channel(self, mock_process_upload, + mock_get_release): + self.joystick._process_joystick_topic(msg_info) + + @mock.patch('JoystickController.process_joystick_topic', + return_value=True) + def test_cls_call_method(self): + self.JoystickController.__call__(msg) + + @mock.patch('boto3.client', return_value=BOTO_RETURN_VALUE) + def test_generate_ami_upload_list(self, mock_boto_client): + js = JoyStickController() + js.generate_ami_upload_list() + """ diff --git a/tests/testjoystick.toml b/tests/testjoystick.toml new file mode 100644 index 0000000..f64f1e9 --- /dev/null +++ b/tests/testjoystick.toml @@ -0,0 +1,68 @@ +# Example configuration for Fedora Messaging +# For complete documentation check +# https://fedora-messaging.readthedocs.io/en/latest/configuration.html. + +amqp_url = "amqps://fedora:@rabbitmq.fedoraproject.org/%2Fpublic_pubsub" + +publish_exchange = "amq.topic" +passive_declares = false + +callback = "joystick.consumers.fedora_messaging_consumer:JoyStickController" + +[tls] +ca_cert = "/etc/fedora-messaging/cacert.pem" +keyfile = "/etc/fedora-messaging/fedora-key.pem" +certfile = "/etc/fedora-messaging/fedora-cert.pem" + +[client_properties] +app = "joystick" + +[exchanges."amq.topic"] +type = "topic" +durable = true +auto_delete = false +arguments = {} + +[queues."6d0c4921-a865-4b10-8d13-47196a66289c"] +durable = true +auto_delete = false +exclusive = false +arguments = {} + +[[bindings]] +queue = "6d0c4921-a865-4b10-8d13-47196a66289c" +exchange = "amq.topic" +routing_keys = ["#"] + +[qos] +prefetch_size = 0 +prefetch_count = 25 + +[log_config] +version = 1 +disable_existing_loggers = true + +[log_config.formatters.simple] +format = "[%(name)s %(levelname)s] %(message)s" + +[log_config.handlers.console] +class = "logging.StreamHandler" +formatter = "simple" +stream = "ext://sys.stdout" + +[log_config.loggers.fedora_messaging] +level = "INFO" +propagate = false +handlers = ["console"] + +[log_config.root] +level = "DEBUG" +handlers = ["console"] + +# joystick consumer configuration +[consumer_config] +topic_prefix = 'org.fedoraproject' +environment = 'prod' +regions = ["us-east-2"] +aws_access_key_id = '' +aws_secret_access_key = ''