From 763928d6297989e2c933e94a4e7e08af218dc862 Mon Sep 17 00:00:00 2001 From: Qixiang Wan Date: Jan 16 2018 04:54:47 +0000 Subject: Verify the rebuilt container has new version of RPMs from advisory When a container is built on brew, and the build is triggered by an advisory, check whether the container contains latest version of RPMs from the advisory, if true, mark the build as DONE, or FAILED if some RPMs are not the latest ones from advisory. --- diff --git a/freshmaker/handlers/brew/container_task_state_change.py b/freshmaker/handlers/brew/container_task_state_change.py index 00b1e66..9fb473b 100644 --- a/freshmaker/handlers/brew/container_task_state_change.py +++ b/freshmaker/handlers/brew/container_task_state_change.py @@ -19,12 +19,18 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. +from kobo import rpmlib + +from freshmaker import conf from freshmaker import log from freshmaker import db -from freshmaker.events import BrewContainerTaskStateChangeEvent -from freshmaker.models import ArtifactBuild +from freshmaker.errata import Errata +from freshmaker.events import ( + BrewContainerTaskStateChangeEvent, ErrataAdvisoryRPMsSignedEvent) +from freshmaker.models import ArtifactBuild, EVENT_TYPES from freshmaker.handlers import ( ContainerBuildHandler, fail_event_on_handler_exception) +from freshmaker.kojiservice import koji_service from freshmaker.types import ArtifactType, ArtifactBuildState, EventState @@ -55,9 +61,23 @@ class BrewContainerTaskStateChangeHandler(ContainerBuildHandler): self.set_context(found_build) # update build state in db if event.new_state == 'CLOSED': - found_build.transition( - ArtifactBuildState.DONE.value, - "Built successfully.") + # if build is triggered by an advisory, verify the container + # contains latest RPMs from the advisory + if found_build.event.event_type_id == EVENT_TYPES[ErrataAdvisoryRPMsSignedEvent]: + errata_id = found_build.event.search_key + # build_id is actually task id in build system, find out the actual build first + with koji_service(conf.koji_profile, log, login=False) as session: + container_build_id = session.get_container_build_id_from_task(build_id) + + ret, msg = self._verify_advisory_rpms_in_container_build(errata_id, container_build_id) + if ret: + found_build.transition(ArtifactBuildState.DONE.value, "Built successfully.") + else: + found_build.transition(ArtifactBuildState.FAILED.value, msg) + + # for other builds, mark them as DONE + else: + found_build.transition(ArtifactBuildState.DONE.value, "Built successfully.") if event.new_state == 'FAILED': found_build.transition( ArtifactBuildState.FAILED.value, @@ -105,3 +125,40 @@ class BrewContainerTaskStateChangeHandler(ContainerBuildHandler): if all_builds_done: db_event.transition( EventState.COMPLETE, 'All docker images have been rebuilt.') + + def _verify_advisory_rpms_in_container_build(self, errata_id, container_build_id): + """ + verify container built on brew has the latest rpms from an advisory + """ + if conf.dry_run: + return (True, '') + + # get rpms in advisory + advisory_rpms = set() + e = Errata() + build_nvrs = e.get_builds(errata_id) + if build_nvrs: + with koji_service(conf.koji_profile, log, login=False) as session: + for build_nvr in build_nvrs: + build_rpms = session.get_build_rpms(build_nvr) + for rpm in build_rpms: + advisory_rpms.add(rpm['nvr']) + + # get rpms in container + with koji_service(conf.koji_profile, log, login=False) as session: + components = session.get_rpms_in_container(container_build_id) + + # compare rpms from advisory and container + unmatched_rpms = [] + container_rpm_names = [rpmlib.parse_nvr(x)['name'] for x in components] + for rpm in advisory_rpms: + rpm_name = rpmlib.parse_nvr(rpm)['name'] + if rpm_name in container_rpm_names and rpm not in components: + unmatched_rpms.append(rpm_name) + + if unmatched_rpms: + msg = ("The following RPMs in container build (%s) do not match " + "with the latest RPMs in advisory (%s):\n%s" % + (container_build_id, errata_id, unmatched_rpms)) + return (False, msg) + return (True, "") diff --git a/freshmaker/kojiservice.py b/freshmaker/kojiservice.py index 44546f6..0edfc25 100644 --- a/freshmaker/kojiservice.py +++ b/freshmaker/kojiservice.py @@ -28,13 +28,15 @@ import koji # it would import freshmaker.handlers.koji, so instead, we import it here # and in freshmaker.handler do "from freshmaker.kojiservice import parse_NVR". from koji import parse_NVR # noqa +from kobo import rpmlib import contextlib import re +import requests +import freshmaker.utils from freshmaker import log, conf from freshmaker.consumer import work_queue_put from freshmaker.events import BrewContainerTaskStateChangeEvent -from freshmaker.utils import krb_context class KojiService(object): @@ -62,6 +64,10 @@ class KojiService(object): return self.config['weburl'] @property + def topurl(self): + return self.config['topurl'] + + @property def server(self): return self.config['server'] @@ -158,8 +164,17 @@ class KojiService(object): return self.session.listRPMs(buildID=build_info['id'], arches=arches) - def get_build(self, build_nvr): - return self.session.getBuild(build_nvr) + def get_build(self, buildinfo): + """ + Return information about a build. + + buildinfo may be either a int ID, a string NVR, or a map containing + 'name', 'version' and 'release. + """ + return self.session.getBuild(buildinfo) + + def get_build_id(self, build_nvr): + return self.session.findBuildID(build_nvr) def get_task_request(self, task_id): return self.session.getTaskRequest(task_id) @@ -167,6 +182,82 @@ class KojiService(object): def get_build_target(self, target_name): return self.session.getBuildTarget(target_name) + def get_container_build_id_from_task(self, task_id): + """ + Return container build id by check 'koji_builds' in build + task result. If not found, return None. + """ + build_id = None + subtasks = self.session.getTaskChildren(task_id) + if subtasks: + for task in subtasks: + task_result = self.session.getTaskResult(task['id']) + builds = task_result.get('koji_builds', None) + if builds: + build_id = int(builds.pop()) + break + else: + task_result = self.session.getTaskResult(task_id) + builds = task_result.get('koji_builds', None) + if builds: + build_id = int(builds.pop()) + return build_id + + def get_cg_metadata_url(self, buildinfo): + """ + Return url of the CG metadata.json + + buildinfo may be either a int ID, a string NVR, or a map containing + 'name', 'version' and 'release. + + Note: it doesn't check whether the metadata.json exists or not. + """ + build_info = self.get_build(buildinfo) + return koji.PathInfo(topdir=self.topurl).build(build_info) + '/metadata.json' + + @freshmaker.utils.retry(wait_on=(requests.Timeout, requests.ConnectionError), logger=log) + def load_cg_metadata(self, buildinfo): + """ + Fetch CG metadata.json and load the json. + + buildinfo may be either a int ID, a string NVR, or a map containing + 'name', 'version' and 'release. + """ + try: + cg_metadata_url = self.get_cg_metadata_url(buildinfo) + resp = requests.get(cg_metadata_url) + # url is redirected + if resp.history: + cg_metadata_url = resp.url + return requests.get(cg_metadata_url).json() + except requests.ConnectionError: + raise + except Exception as e: + if cg_metadata_url: + log.error("Unable to load CG metadata for build (%r) from url (%s): %s", + buildinfo, cg_metadata_url, str(e)) + else: + log.error("Unable to load CG metadata for build (%r): %s", str(e)) + raise + + def get_rpms_in_container(self, buildinfo): + """ + Get rpms in a koji container build. + + buildinfo may be either a int ID, a string NVR, or a map containing + 'name', 'version' and 'release. + + Return a set of rpm NVRs. + """ + rpms = set() + cg_metadata = self.load_cg_metadata(buildinfo) + outputs = cg_metadata['output'] + for out in outputs: + if out['type'] == 'docker-image': + components = out['components'] + rpms = set([rpmlib.make_nvr(rpm) for rpm in components if rpm['type'] == 'rpm']) + return rpms + @contextlib.contextmanager def koji_service(profile=None, logger=None, login=True): @@ -194,7 +285,7 @@ def koji_service(profile=None, logger=None, login=True): log.debug('Logging into %s with Kerberos authentication.', service.server) - with krb_context(): + with freshmaker.utils.krb_context(): service.krb_login() # We are not logged in in dry run mode... diff --git a/tests/test_brew_container_task_state_change_handler.py b/tests/test_brew_container_task_state_change_handler.py index 68b1782..1f10552 100644 --- a/tests/test_brew_container_task_state_change_handler.py +++ b/tests/test_brew_container_task_state_change_handler.py @@ -201,6 +201,57 @@ class TestBrewContainerTaskStateChangeHandler(helpers.FreshmakerTestCase): self.assertEqual(EventState.BUILDING.value, self.db_advisory_rpm_signed_event.state) + @mock.patch('freshmaker.kojiservice.KojiService') + @mock.patch('freshmaker.errata.Errata.get_builds') + def test_mark_build_done_when_container_has_latest_rpms_from_advisory(self, errata_get_builds, KojiService): + """ + Tests when dependency container build task failed in brew, only update build state in db. + """ + errata_get_builds.return_value = set(['foo-1.2.1-22.el7']) + + koji_service = KojiService.return_value + koji_service.get_build_rpms.return_value = [ + {'build_id': 634904, 'nvr': 'foo-debuginfo-1.2.1-22.el7', 'name': 'foo-debuginfo'}, + {'build_id': 634904, 'nvr': 'foo-1.2.1-22.el7', 'name': 'foo'} + ] + koji_service.get_rpms_in_container.return_value = set( + ['foo-1.2.1-22.el7', 'bar-1.2.3-1.el7'] + ) + + e1 = models.Event.create(db.session, "test_msg_id", "2018001", events.ErrataAdvisoryRPMsSignedEvent) + event = self.get_event_from_msg(get_fedmsg('brew_container_task_closed')) + build = models.ArtifactBuild.create(db.session, e1, 'test-product-docker', ArtifactType.IMAGE, event.task_id) + + self.handler.handle(event) + + self.assertEqual(build.state, ArtifactBuildState.DONE.value) + self.assertEqual(build.state_reason, 'Built successfully.') + + @mock.patch('freshmaker.kojiservice.KojiService') + @mock.patch('freshmaker.errata.Errata.get_builds') + def test_mark_build_fail_when_container_not_has_latest_rpms_from_advisory(self, errata_get_builds, KojiService): + """ + Tests when dependency container build task failed in brew, only update build state in db. + """ + errata_get_builds.return_value = set(['foo-1.2.1-23.el7']) + + koji_service = KojiService.return_value + koji_service.get_build_rpms.return_value = [ + {'build_id': 634904, 'nvr': 'foo-debuginfo-1.2.1-23.el7', 'name': 'foo-debuginfo'}, + {'build_id': 634904, 'nvr': 'foo-1.2.1-23.el7', 'name': 'foo'} + ] + koji_service.get_rpms_in_container.return_value = set( + ['foo-1.2.1-22.el7', 'bar-1.2.3-1.el7'] + ) + + e1 = models.Event.create(db.session, "test_msg_id", "2018001", events.ErrataAdvisoryRPMsSignedEvent) + event = self.get_event_from_msg(get_fedmsg('brew_container_task_closed')) + build = models.ArtifactBuild.create(db.session, e1, 'test-product-docker', ArtifactType.IMAGE, event.task_id) + + self.handler.handle(event) + self.assertEqual(build.state, ArtifactBuildState.FAILED.value) + self.assertRegexpMatches(build.state_reason, r"The following RPMs in container build.*") + if __name__ == '__main__': unittest.main() diff --git a/tests/test_errata_advisory_state_changed.py b/tests/test_errata_advisory_state_changed.py index 1057de2..e5a0e91 100644 --- a/tests/test_errata_advisory_state_changed.py +++ b/tests/test_errata_advisory_state_changed.py @@ -676,7 +676,7 @@ class TestPrepareYumRepo(unittest.TestCase): 'ErrataAdvisoryRPMsSignedHandler._get_compose_source') @patch('time.sleep') @patch('freshmaker.handlers.errata.errata_advisory_rpms_signed.Errata') - @patch('freshmaker.kojiservice.krb_context', + @patch('freshmaker.utils.krb_context', new_callable=PropertyMock) def test_get_repo_url_packages_in_multiple_tags( self, krb_context, errata, sleep, _get_compose_source, @@ -708,7 +708,7 @@ class TestPrepareYumRepo(unittest.TestCase): 'ErrataAdvisoryRPMsSignedHandler._get_compose_source') @patch('time.sleep') @patch('freshmaker.handlers.errata.errata_advisory_rpms_signed.Errata') - @patch('freshmaker.kojiservice.krb_context', + @patch('freshmaker.utils.krb_context', new_callable=PropertyMock) def test_get_repo_url_packages_not_found_in_tag( self, krb_context, errata, sleep, _get_compose_source,