From e512d453dd3e144581140d263bdd8322671971bf Mon Sep 17 00:00:00 2001 From: Chenxiong Qi Date: May 08 2017 04:17:19 +0000 Subject: Rebuild docker image when bodhi update moves to stable This is also for original issue that is to rebuild docker image when RPMs are built/tagged. Since building docker image should get RPMs from official Fedora repositories, freshmaker has to wait for message of topic bodhi.update.complete.stable. Signed-off-by: Chenxiong Qi --- diff --git a/conf/config.py b/conf/config.py index 89b7adc..4dd572c 100644 --- a/conf/config.py +++ b/conf/config.py @@ -1,5 +1,6 @@ from os import path + # FIXME: workaround for this moment till confdir, dbdir (installdir etc.) are # declared properly somewhere/somehow confdir = path.abspath(path.dirname(__file__)) @@ -43,6 +44,7 @@ class BaseConfiguration(object): HANDLERS = [ "freshmaker.handlers.mbs:MBS", # Module Build Service "freshmaker.handlers.image_builder:DockerImageRebuildHandler", + "freshmaker.handlers.image_builder:DockerImageRebuildHandlerForBodhi", ] # Base URL of git repository with source artifacts. diff --git a/freshmaker/consumer.py b/freshmaker/consumer.py index 1deb528..b2ef291 100644 --- a/freshmaker/consumer.py +++ b/freshmaker/consumer.py @@ -30,6 +30,7 @@ import moksha.hub import freshmaker.handlers import freshmaker.parsers.mbsmodule import freshmaker.parsers.gitreceive +import freshmaker.parsers.bodhiupdate from freshmaker import log, conf, messaging, events @@ -63,6 +64,7 @@ class FreshmakerConsumer(fedmsg.consumers.FedmsgConsumer): def register_parsers(self): events.BaseEvent.register_parser(freshmaker.parsers.mbsmodule.MBSModuleParser) events.BaseEvent.register_parser(freshmaker.parsers.gitreceive.GitReceiveParser) + events.BaseEvent.register_parser(freshmaker.parsers.bodhiupdate.UpdateCompleteStableParser) log.debug("Parser classes: %r", events.BaseEvent._parsers) self.topic = events.BaseEvent.get_parsed_topics() diff --git a/freshmaker/events.py b/freshmaker/events.py index b6eee8a..33cb0fa 100644 --- a/freshmaker/events.py +++ b/freshmaker/events.py @@ -182,3 +182,31 @@ class DockerfileChanged(BaseEvent): self.namespace = namespace self.repo = repo self.rev = rev + + +class BodhiUpdateCompleteStable(BaseEvent): + """Event when RPMs are available in Fedora master mirrors + + Refer to an example in datagrepper: + + https://apps.fedoraproject.org/datagrepper/raw?delta=572800& \ + topic=org.fedoraproject.prod.bodhi.update.complete.stable + """ + + def __init__(self, msg_id, update_id, builds, release): + """Initiate event with data from message got from fedmsg + + Not complete data is required, only part of attributes that are useful + for rebuild are stored in this event. + + :param str update_id: the Bodhi update ID got from message. + :param list builds: a list of maps, each of them contains build NVRs + that are useful for getting RPMs for the rebuild. + :param dist release: a map contains release information, e.g. name and + branch. Refer to the example given above to see all available + attributes in a message. + """ + super(BodhiUpdateCompleteStable, self).__init__(msg_id) + self.update_id = update_id + self.builds = builds + self.release = release diff --git a/freshmaker/handlers/image_builder.py b/freshmaker/handlers/image_builder.py index 81993e1..5503b40 100644 --- a/freshmaker/handlers/image_builder.py +++ b/freshmaker/handlers/image_builder.py @@ -21,9 +21,19 @@ # # Written by Chenxiong Qi +import os + +from itertools import chain + +import freshmaker.pdc as pdc + from freshmaker import log, conf from freshmaker.handlers import BaseHandler from freshmaker.events import DockerfileChanged +from freshmaker.events import BodhiUpdateCompleteStable +from freshmaker.utils import temp_dir +from freshmaker.utils import _run_command +from freshmaker.utils import get_commit_hash from freshmaker.kojiservice import koji_service @@ -36,29 +46,102 @@ class DockerImageRebuildHandler(BaseHandler): """Rebuild docker image""" import koji + log.info('Start to rebuild docker image %s', event.repo) + try: - self.build_image(event) + self.build_image(repo_url=event.repo_url, + rev=event.rev, + branch=event.branch, + namespace=event.namespace) except koji.krbV.Krb5Error as e: log.exception('Failed to login Koji via Kerberos using GSSAPI. %s', e.args[1]) except: log.exception('Could not create task to build docker image %s', event.repo) - def build_image(self, event): + def build_image(self, repo_url, rev, branch, namespace=None): with koji_service(profile=conf.koji_profile, logger=log) as service: log.debug('Logging into {0} with Kerberos authentication.'.format(service.server)) proxyuser = conf.koji_build_owner if conf.koji_proxyuser else None - service.krb_login(proxyuser=proxyuser) if not service.logged_in: log.error('Could not login server %s', service.server) return - build_source = '{}#{}'.format(event.repo_url, event.rev) + build_source = '{}?#{}'.format(repo_url, rev) - log.info('Start to build docker image %s', event.repo) log.debug('Build from source: %s', build_source) - return service.build_container(build_source, event.branch, - namespace=event.namespace, + return service.build_container(build_source, + branch, + namespace=namespace, scratch=conf.koji_container_scratch_build) + + +class DockerImageRebuildHandlerForBodhi(DockerImageRebuildHandler): + """Rebuild docker images when RPMs are synced by Bodhi""" + + def __init__(self): + self.pdc_session = pdc.get_client_session(conf) + + def can_handle(self, event): + return isinstance(event, BodhiUpdateCompleteStable) + + def handle(self, event): + log.info('Rebuild docker images for event %s, msgid: %s', + BodhiUpdateCompleteStable.__name__, event.msg_id) + + rpms = self.get_rpms_included_in_bodhi_update(event.builds) + containers = self.get_containers_including_rpms(rpms) + + log.info('Found docker images to rebuild: %s', containers) + + for container in containers: + try: + self.handle_image_build(container) + except: + log.exception('Error when rebuild %s', container) + + def handle_image_build(self, container_info): + container_detail = pdc.get_release_component(self.pdc_session, + container_info['id']) + + branch = container_detail['dist_git_branch'] + image_name = container_detail['name'] + repo_url = '{}/{}/{}'.format(conf.git_base_url, + 'container', + image_name) + + log.info('Start to rebuild docker image %s from branch %s', + image_name, branch) + + with temp_dir(suffix='-rebuild-docker-image') as working_dir: + self.clone_repository(repo_url, branch, working_dir) + + last_commit_hash = get_commit_hash( + os.path.join(working_dir, image_name)) + + return self.build_image(repo_url=repo_url, + branch=branch, + rev=last_commit_hash) + + def clone_repository(self, url, branch, working_dir): + cmd = ['git', 'clone', '-b', branch, url] + log.debug('Clone repository: %s', cmd) + _run_command(cmd, rundir=working_dir) + + def get_rpms_included_in_bodhi_update(self, builds): + build_nvrs = (build['nvr'] for build in builds) + with koji_service(profile=conf.koji_profile, logger=log) as service: + return chain(*[service.get_build_rpms(nvr) for nvr in build_nvrs]) + + def get_containers_including_rpms(self, rpms): + containers = {} + for rpm in rpms: + found = pdc.find_containers_by_rpm_name(self.pdc_session, rpm['name']) + for container in found: + id = container['id'] + if id not in containers: + containers[id] = container + + return containers.values() diff --git a/freshmaker/kojiservice.py b/freshmaker/kojiservice.py index af9f0fd..edce528 100644 --- a/freshmaker/kojiservice.py +++ b/freshmaker/kojiservice.py @@ -95,6 +95,10 @@ class KojiService(object): return task_id + def get_build_rpms(self, build_nvr): + build_info = self.session.getBuild(build_nvr) + return self.session.listRPMs(buildID=build_info['id']) + @contextlib.contextmanager def koji_service(profile=None, logger=None): diff --git a/freshmaker/parsers/bodhiupdate.py b/freshmaker/parsers/bodhiupdate.py new file mode 100644 index 0000000..646b269 --- /dev/null +++ b/freshmaker/parsers/bodhiupdate.py @@ -0,0 +1,50 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2016 Red Hat, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +# Written by Chenxiong Qi + +from freshmaker import log +from freshmaker.parsers import BaseParser +from freshmaker.events import BodhiUpdateCompleteStable + + +class UpdateCompleteStableParser(BaseParser): + """Parse Bodhi message from topic bodhi.update.complete.stable""" + + name = 'UpdateCompleteStableParser' + topic_suffixes = ['bodhi.update.complete.stable'] + + def can_parse(self, topic, msg): + return any([topic.endswith(suffix) for suffix in self.topic_suffixes]) + + def parse(self, topic, msg): + msg_id = msg.get('msg_id') + msg_inner_msg = msg.get('msg') + + if not msg_inner_msg: + log.debug('Skipping message without any content with the topic "%s"', topic) + return None + + update = msg_inner_msg['update'] + return BodhiUpdateCompleteStable(msg_id, + update_id=update['updateid'], + builds=update['builds'], + release=update['release']) diff --git a/freshmaker/pdc.py b/freshmaker/pdc.py index 3e65fc8..9f90b76 100644 --- a/freshmaker/pdc.py +++ b/freshmaker/pdc.py @@ -24,7 +24,6 @@ import inspect import requests from pdc_client import PDCClient -import freshmaker import freshmaker.utils @@ -79,3 +78,14 @@ def get_latest_modules(pdc_session, **kwargs): mods = get_modules(pdc_session, variant_name=name, variant_version=version, active=active) latest_modules.append(sorted(mods, key=lambda x: x['variant_release']).pop()) return list(filter(lambda x: x in latest_modules, modules)) + + +@freshmaker.utils.retry(wait_on=(requests.ConnectTimeout, requests.ConnectionError), logger=freshmaker.log) +def find_containers_by_rpm_name(pdc_session, rpm_name): + rels = pdc_session['release-component-relationships'](type='ContainerIncludesRPM', + to_component_name=rpm_name) + return [rel['from_component'] for rel in rels['results']] + + +def get_release_component(pdc_session, id): + return pdc_session['release-components/{}/'.format(id)]() diff --git a/tests/fedmsgs/bodhi_update_stable b/tests/fedmsgs/bodhi_update_stable new file mode 100644 index 0000000..07362ef --- /dev/null +++ b/tests/fedmsgs/bodhi_update_stable @@ -0,0 +1,129 @@ +{ + "source_name": "datanommer", + "certificate": "LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVYakNDQThlZ0F3SUJBZ0lDQWhRd0RRWUpL\nb1pJaHZjTkFRRUZCUUF3Z2FBeEN6QUpCZ05WQkFZVEFsVlQKTVFzd0NRWURWUVFJRXdKT1F6RVFN\nQTRHQTFVRUJ4TUhVbUZzWldsbmFERVhNQlVHQTFVRUNoTU9SbVZrYjNKaApJRkJ5YjJwbFkzUXhE\nekFOQmdOVkJBc1RCbVpsWkcxelp6RVBNQTBHQTFVRUF4TUdabVZrYlhObk1ROHdEUVlEClZRUXBF\nd1ptWldSdGMyY3hKakFrQmdrcWhraUc5dzBCQ1FFV0YyRmtiV2x1UUdabFpHOXlZWEJ5YjJwbFkz\nUXUKYjNKbk1CNFhEVEUxTURneE9ESXhNakUwT0ZvWERUSTFNRGd4TlRJeE1qRTBPRm93Z2V3eEN6\nQUpCZ05WQkFZVApBbFZUTVFzd0NRWURWUVFJRXdKT1F6RVFNQTRHQTFVRUJ4TUhVbUZzWldsbmFE\nRVhNQlVHQTFVRUNoTU9SbVZrCmIzSmhJRkJ5YjJwbFkzUXhEekFOQmdOVkJBc1RCbVpsWkcxelp6\nRTFNRE1HQTFVRUF4TXNZbTlrYUdrdFltOWsKYUdrdFltRmphMlZ1WkRBeExuQm9lREl1Wm1Wa2Iz\nSmhjSEp2YW1WamRDNXZjbWN4TlRBekJnTlZCQ2tUTEdKdgpaR2hwTFdKdlpHaHBMV0poWTJ0bGJt\nUXdNUzV3YUhneUxtWmxaRzl5WVhCeWIycGxZM1F1YjNKbk1TWXdKQVlKCktvWklodmNOQVFrQkZo\nZGhaRzFwYmtCbVpXUnZjbUZ3Y205cVpXTjBMbTl5WnpDQm56QU5CZ2txaGtpRzl3MEIKQVFFRkFB\nT0JqUUF3Z1lrQ2dZRUF1b3NVcXpXM010RlpkRTdEelFCT0VzVzY3ZThsZnQ5S3QzMFVJbGZSVUhS\nVwpVMGpWSEoyaWtic1oxQmZMckh3Ly8yd1lUa2E3cE1Xblh2KzZnRVhhYTNFZDRqbk8zTDNVallj\nQ1pBUzBDYW93CmZad1VVTWwwNFN6N1ZSZnVRdElJbG5ocjFxSjhKTTJqZXZNQkpyaTQyZDZZKzdF\nTEFNZjgvTkVjQ2p6NERuY0MKQXdFQUFhT0NBVmN3Z2dGVE1Ba0dBMVVkRXdRQ01BQXdMUVlKWUla\nSUFZYjRRZ0VOQkNBV0hrVmhjM2t0VWxOQgpJRWRsYm1WeVlYUmxaQ0JEWlhKMGFXWnBZMkYwWlRB\nZEJnTlZIUTRFRmdRVXVBb1E0T05pRlFERXpvSmE1ZG9CCnA4NTJoeEV3Z2RVR0ExVWRJd1NCelRD\nQnlvQVVhMEJhK1JJSWlWbm5XZVVGOVFJZENrNS9GQUNoZ2Fha2dhTXcKZ2FBeEN6QUpCZ05WQkFZ\nVEFsVlRNUXN3Q1FZRFZRUUlFd0pPUXpFUU1BNEdBMVVFQnhNSFVtRnNaV2xuYURFWApNQlVHQTFV\nRUNoTU9SbVZrYjNKaElGQnliMnBsWTNReER6QU5CZ05WQkFzVEJtWmxaRzF6WnpFUE1BMEdBMVVF\nCkF4TUdabVZrYlhObk1ROHdEUVlEVlFRcEV3Wm1aV1J0YzJjeEpqQWtCZ2txaGtpRzl3MEJDUUVX\nRjJGa2JXbHUKUUdabFpHOXlZWEJ5YjJwbFkzUXViM0puZ2drQTQxQWVSMDhYSGtVd0V3WURWUjBs\nQkF3d0NnWUlLd1lCQlFVSApBd0l3Q3dZRFZSMFBCQVFEQWdlQU1BMEdDU3FHU0liM0RRRUJCUVVB\nQTRHQkFHazBJVGhUT2xzV29RdncxQzdCCjNINnh6cEp0Znd4NHNVdnk4cXVXN2FHOE9VNXdscmNF\ndmZzZENJbmdKdEwxZndWTGgwT3M3SmhyaGxkdzZBUkoKRWdIS2NtSnVjbkJEZW9tck1DdTVaM3U1\nNWp4SWlHcG5wMThRcUhlSHVmZUEzLysvWlhTL3F6bGRxdjNrMG9kMQpIbFNweFZxWUQ5OGs2eHRD\nQXo5YWh0Q20KLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQo=\n", + "i": 12, + "timestamp": 1493430570.0, + "msg_id": "2017-0c7f37f3-d0b5-474f-b600-198d9b3be748", + "topic": "org.fedoraproject.prod.bodhi.update.complete.stable", + "source_version": "0.6.5", + "signature": "n3DUWAJMCMF3ipgcxUnb7EaCNQydfKvjr+AQA4S3nuTqUT/0wpuOAqTnOOXhW6wL6LVmzleOQdo7\nWH/tR5qPkemPTs6wWTNIlon1QJWaMWpNHzYn3O1yzcFiTizX98Fi+p5oBKzyafHHqAiTN5Nj+RjR\nT+xz/rkmRy+Ql+96m04=\n", + "msg": { + "update": { + "date_testing": "2017-04-20 17:22:23", + "old_updateid": null, + "pushed": true, + "require_testcases": false, + "date_stable": "2017-04-28 19:59:31", + "critpath": false, + "date_approved": null, + "stable_karma": 3, + "date_pushed": "2017-04-28 19:59:31", + "requirements": "", + "severity": "medium", + "autokarma": true, + "title": "community-mysql-5.7.18-2.fc25", + "suggest": "unspecified", + "require_bugs": false, + "comments": [ + { + "bug_feedback": [], + "user_id": 91, + "timestamp": "2017-04-19 23:31:30", + "text": "This update has been submitted for testing by mschorm. ", + "karma_critpath": 0, + "update_id": 85679, + "karma": 0, + "anonymous": false, + "testcase_feedback": [], + "id": 596151, + "user": { + "openid": null, + "name": "bodhi", + "show_popups": true, + "email": null, + "avatar": null, + "groups": [], + "id": 91 + } + } + ], + "updateid": "FEDORA-2017-fe6e14dcf9", + "test_cases": [], + "close_bugs": false, + "meets_testing_requirements": true, + "date_locked": "2017-04-28 19:59:21", + "date_submitted": "2017-04-19 23:31:30", + "unstable_karma": -2, + "submitter": "mschorm", + "user": { + "openid": null, + "name": "mschorm", + "show_popups": true, + "email": "mschorm@redhat.com", + "avatar": null, + "groups": [ + { + "name": "packager" + } + ], + "id": 3283 + }, + "locked": true, + "builds": [ + { + "epoch": 0, + "nvr": "community-mysql-5.7.18-2.fc25", + "signed": true + } + ], + "date_modified": null, + "url": "https://bodhi.fedoraproject.org/updates/FEDORA-2017-fe6e14dcf9", + "type": "security", + "notes": "Update to 5.7.18\n\nCVEs fixed by this update can be found here:\nhttp://www.oracle.com/technetwork/security-advisory/cpuapr2017-3236618.html\n\n", + "request": null, + "bugs": [ + { + "bug_id": 1414386, + "security": true, + "feedback": [], + "parent": false, + "title": "CVE-2017-3265 community-mysql: various flaws [fedora-all]" + }, + { + "bug_id": 1443407, + "security": true, + "feedback": [], + "parent": false, + "title": "CVE-2017-3308 CVE-2017-3309 CVE-2017-3450 CVE-2017-3453 CVE-2017-3456 CVE-2017-3461 CVE-2017-3462 CVE-2017-3463 CVE-2017-3464 CVE-2017-3599 community-mysql: various flaws [fedora-all]" + }, + { + "bug_id": 1441001, + "security": false, + "feedback": [], + "parent": false, + "title": "community-mysql-5.7.18 is available" + } + ], + "alias": "FEDORA-2017-fe6e14dcf9", + "status": "stable", + "karma": 0, + "release": { + "dist_tag": "f25", + "name": "F25", + "testing_tag": "f25-updates-testing", + "pending_stable_tag": "f25-updates-pending", + "pending_signing_tag": "f25-signing-pending", + "long_name": "Fedora 25", + "state": "current", + "version": "25", + "override_tag": "f25-override", + "branch": "f25", + "id_prefix": "FEDORA", + "pending_testing_tag": "f25-updates-testing-pending", + "stable_tag": "f25-updates", + "candidate_tag": "f25-updates-candidate" + } + }, + "agent": "masher" + } +} \ No newline at end of file diff --git a/tests/handlers/test_image_builder.py b/tests/handlers/test_image_builder.py index 2b823ca..6dfc7fe 100644 --- a/tests/handlers/test_image_builder.py +++ b/tests/handlers/test_image_builder.py @@ -20,19 +20,25 @@ # # Written by Chenxiong Qi +import shutil +import tempfile +import unittest + import fedmsg.config import pytest import six -import unittest from mock import patch from mock import MagicMock +from mock import call + +from freshmaker import conf from freshmaker.consumer import FreshmakerConsumer +from freshmaker.handlers.image_builder import DockerImageRebuildHandlerForBodhi from tests import get_fedmsg -@pytest.mark.skipif(six.PY3, reason='koji does not work in Python 3') -class TestImageBuilderHandler(unittest.TestCase): +class BaseTestCase(unittest.TestCase): def create_consumer(self): hub = MagicMock() @@ -41,32 +47,37 @@ class TestImageBuilderHandler(unittest.TestCase): @patch("requests.request") @patch('freshmaker.consumer.get_global_consumer') - def consume_git_receive_msg(self, msg, global_consumer, request): + def consume_fedmsg(self, msg, global_consumer, request): consumer = self.create_consumer() global_consumer.return_value = consumer consumer.consume(msg) + +@pytest.mark.skipif(six.PY3, reason='koji does not work in Python 3') +class TestImageBuilderHandler(BaseTestCase): + @patch('koji.read_config') @patch('koji.ClientSession') def test_rebuild_if_Dockerfile_changed(self, ClientSession, read_config): read_config.return_value = { 'server': 'https://localhost/kojihub', 'krb_rdns': False, + 'weburl': 'https://localhost/koji', } - self.consume_git_receive_msg(get_fedmsg("git_receive_dockerfile_changed")) + self.consume_fedmsg(get_fedmsg('git_receive_dockerfile_changed')) mock_session = ClientSession.return_value mock_session.krb_login.assert_called_once_with(proxyuser=None) mock_session.buildContainer.assert_called_once_with( - 'git://pkgs.fedoraproject.org/container/testimage.git#e1f39d43471fc37ec82616f76a119da4eddec787', + 'git://pkgs.fedoraproject.org/container/testimage.git?#e1f39d43471fc37ec82616f76a119da4eddec787', 'rawhide-container-candidate', {'scratch': True, 'git_branch': 'master'}) mock_session.logout.assert_called_once() @patch('freshmaker.handlers.image_builder.DockerImageRebuildHandler.build_image') def test_not_rebuild_if_Dockerfile_not_changed(self, build_image): - self.consume_git_receive_msg(get_fedmsg("git_receive_dockerfile_not_changed")) + self.consume_fedmsg(get_fedmsg('git_receive_dockerfile_not_changed')) build_image.assert_not_called() @patch('koji.read_config') @@ -76,9 +87,10 @@ class TestImageBuilderHandler(unittest.TestCase): read_config.return_value = { 'server': 'https://localhost/kojihub', 'krb_rdns': False, + 'weburl': 'https://localhost/koji', } - self.consume_git_receive_msg(get_fedmsg("git_receive_dockerfile_changed")) + self.consume_fedmsg(get_fedmsg('git_receive_dockerfile_changed')) ClientSession.return_value.logout.assert_called_once() @@ -89,8 +101,256 @@ class TestImageBuilderHandler(unittest.TestCase): read_config.return_value = { 'server': 'https://localhost/kojihub', 'krb_rdns': False, + 'weburl': 'https://localhost/koji', } - self.consume_git_receive_msg(get_fedmsg("git_receive_dockerfile_changed")) + self.consume_fedmsg(get_fedmsg('git_receive_dockerfile_changed')) ClientSession.return_value.buildContainer.assert_not_called() + + +mock_found_containers = [ + { + 'release': 'fedora-25-updates', + 'id': 5430, + 'name': 'testimage1' + }, + { + 'release': 'fedora-25-updates', + 'id': 5431, + 'name': 'testimage2' + }, +] + +mock_release_components = { + 5430: { + 'id': 5430, + 'release': { + 'active': True, + 'release_id': 'fedora-25-updates' + }, + 'bugzilla_component': None, + 'brew_package': None, + 'global_component': 'testimage1', + 'name': 'testimage1', + 'dist_git_branch': 'f25', + 'dist_git_web_url': 'http://pkgs.example.com/cgit/container/testimage1', + 'active': True, + 'type': 'container', + 'srpm': None, + }, + 5431: { + 'id': 5431, + 'release': { + 'active': True, + 'release_id': 'fedora-25-updates' + }, + 'bugzilla_component': None, + 'brew_package': None, + 'global_component': 'testimage2', + 'name': 'testimage2', + 'dist_git_branch': 'f25', + 'dist_git_web_url': 'http://pkgs.example.com/cgit/container/testimage2', + 'active': True, + 'type': 'container', + 'srpm': None, + } +} + +def mock_get_release_component(pdc_session, id): + return mock_release_components[id] + + +@pytest.mark.skipif(six.PY3, reason='koji does not work in Python 3') +class TestRebuildWhenBodhiUpdateStable(BaseTestCase): + + def setUp(self): + # Use to return a temporary directory from temp_dir method. So, no need + # to delete this directory, since temp_dir ensures to do that. + self.working_dir = tempfile.mkdtemp(prefix='test-image-rebuild-') + + @patch('koji.ClientSession') + @patch('tempfile.mkdtemp') + @patch('freshmaker.handlers.image_builder._run_command') + @patch('freshmaker.handlers.image_builder.get_commit_hash') + @patch('freshmaker.handlers.image_builder.' + 'DockerImageRebuildHandlerForBodhi.get_rpms_included_in_bodhi_update') + @patch('freshmaker.handlers.image_builder.' + 'DockerImageRebuildHandlerForBodhi.get_containers_including_rpms') + @patch('freshmaker.pdc.get_release_component', new=mock_get_release_component) + def test_rebuild(self, + get_containers_including_rpms, + get_rpms_included_in_bodhi_update, + get_commit_hash, + _run_command, + mkdtemp, + ClientSession): + last_commit_hash = 'dea19c748434ec962f13d680682eee87393a4d8e' + + # A repository is not cloned actually, so just use a fake commit hash + # to construct build source URL. + get_commit_hash.return_value = last_commit_hash + + # temp_dir creates a temporary file using mkdtemp that is used for + # working directory for everything related to rebuild docker image. + # It is difficult to catch that temporary directory, so mock mkdtemp + # and use this test's own directory. + mkdtemp.return_value = self.working_dir + + get_containers_including_rpms.return_value = mock_found_containers + + self.consume_fedmsg(get_fedmsg('bodhi_update_stable')) + + self.assertEqual(2, _run_command.call_count) + + _run_command.assert_has_calls([ + call(['git', 'clone', '-b', 'f25', + '{}/container/{}'.format(conf.git_base_url, 'testimage1')], + rundir=self.working_dir), + call(['git', 'clone', '-b', 'f25', + '{}/container/{}'.format(conf.git_base_url, 'testimage2')], + rundir=self.working_dir) + ]) + + session = ClientSession.return_value + + self.assertEqual(2, session.krb_login.call_count) + + buildContainer = session.buildContainer + self.assertEqual(2, buildContainer.call_count) + buildContainer.assert_has_calls([ + call('{}/container/{}?#{}'.format(conf.git_base_url, + 'testimage1', + last_commit_hash), + 'f25-container-candidate', + {'scratch': True, 'git_branch': 'f25'}), + call('{}/container/{}?#{}'.format(conf.git_base_url, + 'testimage2', + last_commit_hash), + 'f25-container-candidate', + {'scratch': True, 'git_branch': 'f25'}), + ], + any_order=True) + + +class TestContainersIncludingRPMs(unittest.TestCase): + + @patch('freshmaker.handlers.image_builder.pdc.find_containers_by_rpm_name') + def test_get_containers(self, find_containers_by_rpm_name): + expected_found_containers = [ + { + 'release': 'fedora-24-updates', + 'id': 5430, + 'name': 'testimage1', + }, + { + 'release': 'fedora-24-updates', + 'id': 5432, + 'name': 'testimage2', + }, + ] + find_containers_by_rpm_name.return_value = expected_found_containers + + handler = DockerImageRebuildHandlerForBodhi() + rpms = [ + {'id': 9515683, + 'name': 'community-mysql-devel', + 'nvr': 'community-mysql-devel-5.7.18-2.fc25', + 'release': '2.fc25', + 'version': '5.7.18'}, + {'id': 9515682, + 'name': 'community-mysql-libs', + 'nvr': 'community-mysql-libs-5.7.18-2.fc25', + 'release': '2.fc25', + 'version': '5.7.18'}, + {'id': 9515681, + 'name': 'community-mysql-server', + 'nvr': 'community-mysql-server-5.7.18-2.fc25', + 'release': '2.fc25', + 'version': '5.7.18'}, + ] + containers = handler.get_containers_including_rpms(rpms) + + self.assertEqual(3, find_containers_by_rpm_name.call_count) + found_containers = sorted(containers, key=lambda item: item['id']) + self.assertEqual(expected_found_containers, found_containers) + + +def mock_get_build_rpms(self, nvr): + """Used to patch KojiService.get_build_rpms""" + + rpms = { + 'community-mysql-5.7.18-2.fc25': [ + { + 'id': 9515683, + 'name': 'community-mysql-devel', + 'nvr': 'community-mysql-devel-5.7.18-2.fc25', + 'release': '2.fc25', + 'version': '5.7.18', + }, + { + 'id': 9515682, + 'name': 'community-mysql-libs', + 'nvr': 'community-mysql-libs-5.7.18-2.fc25', + 'release': '2.fc25', + 'version': '5.7.18', + }, + { + 'id': 9515681, + 'name': 'community-mysql-server', + 'nvr': 'community-mysql-server-5.7.18-2.fc25', + 'release': '2.fc25', + 'version': '5.7.18', + }, + ], + 'qt5-qtwebengine-5.8.0-11.fc25': [ + { + 'id': 9571317, + 'name': 'qt5-qtwebengine-devel', + 'nvr': 'qt5-qtwebengine-devel-5.8.0-11.fc25', + 'release': '11.fc25', + 'version': '5.8.0', + }, + { + 'id': 9571316, + 'name': 'qt5-qtwebengine-examples', + 'nvr': 'qt5-qtwebengine-examples-5.8.0-11.fc25', + 'release': '11.fc25', + 'version': '5.8.0', + } + ], + } + + return rpms[nvr] + + +@pytest.mark.skipif(six.PY3, reason='koji does not work in Python 3') +class TestGetRpmsIncludedInBodhiUpdate(unittest.TestCase): + """Test case for get_rpms_included_in_bodhi_update""" + + @patch('freshmaker.kojiservice.KojiService.get_build_rpms', + new=mock_get_build_rpms) + def test_get_rpms(self): + builds = [ + { + 'build_id': 884455, + 'name': 'qt5-qtwebengine', + 'nvr': 'qt5-qtwebengine-5.8.0-11.fc25', + 'release': '11.fc25', + 'version': '5.8.0', + }, + { + 'build_id': 881597, + 'name': 'community-mysql', + 'nvr': 'community-mysql-5.7.18-2.fc25', + 'release': '2.fc25', + 'version': '5.7.18', + } + ] + handler = DockerImageRebuildHandlerForBodhi() + rpms = list(handler.get_rpms_included_in_bodhi_update(builds)) + + self.assertEqual(5, len(rpms)) + + rpm = filter(lambda item: item['id'] == 9515681, rpms) + self.assertEqual(1, len(rpm))