From 3d0eb8f41946c21ae684e8d439c959fabbb73311 Mon Sep 17 00:00:00 2001 From: Jan Kaluza Date: Sep 26 2017 15:05:49 +0000 Subject: [PATCH 1/2] Add DRY_RUN option to not submit real builds, but instead fake them. Also fix bug when build_id of submitted build was not recorded in db. --- diff --git a/freshmaker/config.py b/freshmaker/config.py index 7ea20b1..fa72576 100644 --- a/freshmaker/config.py +++ b/freshmaker/config.py @@ -187,6 +187,14 @@ class Config(object): 'type': str, 'default': '', 'desc': 'Build owner.'}, + 'dry_run': { + 'type': bool, + 'default': False, + 'desc': 'When True, no builds will be submitted and only log ' + 'messages will be logged instead. Freshmaker will also ' + 'generate fake "build succeeded" events to mark fake ' + 'artifact rebuild as done.', + }, 'handler_build_whitelist': { 'type': dict, 'default': {}, diff --git a/freshmaker/handlers/__init__.py b/freshmaker/handlers/__init__.py index 65ebaa0..fea6ee5 100644 --- a/freshmaker/handlers/__init__.py +++ b/freshmaker/handlers/__init__.py @@ -197,10 +197,12 @@ class ContainerBuildHandler(BaseHandler): proxyuser = conf.koji_build_owner if conf.koji_proxyuser else None + with self.krb_context: service.krb_login(proxyuser=proxyuser) - if not service.logged_in: + # We are not logged in in dry run mode... + if not conf.dry_run and not service.logged_in: log.error('Could not login server %s', service.server) return None diff --git a/freshmaker/handlers/brew/container_task_state_change.py b/freshmaker/handlers/brew/container_task_state_change.py index 676faee..999bfce 100644 --- a/freshmaker/handlers/brew/container_task_state_change.py +++ b/freshmaker/handlers/brew/container_task_state_change.py @@ -62,4 +62,6 @@ class BrewContainerTaskStateChangeHandler(ContainerBuildHandler): repo_urls = self.get_repo_urls(found_build.event) for build in planned_builds: log.info("Build %r depends on build %r" % (build, found_build)) - self.build_image_artifact_build(build, repo_urls) + build.build_id = self.build_image_artifact_build(build, repo_urls) + build.state = ArtifactBuildState.BUILD.value + db.session.commit() diff --git a/freshmaker/handlers/errata/errata_advisory_rpms_signed.py b/freshmaker/handlers/errata/errata_advisory_rpms_signed.py index ec99658..3485cb4 100644 --- a/freshmaker/handlers/errata/errata_advisory_rpms_signed.py +++ b/freshmaker/handlers/errata/errata_advisory_rpms_signed.py @@ -29,6 +29,7 @@ from freshmaker import conf from freshmaker import log from freshmaker import db from freshmaker.events import ErrataAdvisoryRPMsSignedEvent +from freshmaker.events import ODCSComposeStateChangeEvent from freshmaker.handlers import BaseHandler from freshmaker.kojiservice import koji_service from freshmaker.lightblue import LightBlue @@ -36,9 +37,11 @@ from freshmaker.pulp import Pulp from freshmaker.errata import Errata from freshmaker.types import ArtifactType, ArtifactBuildState from freshmaker.models import Event +from freshmaker.consumer import work_queue_put from odcs.client.odcs import ODCS from odcs.client.odcs import AuthMech +from odcs.common.types import COMPOSE_STATES class ErrataAdvisoryRPMsSignedHandler(BaseHandler): @@ -49,6 +52,9 @@ class ErrataAdvisoryRPMsSignedHandler(BaseHandler): name = 'ErrataAdvisoryRPMsSignedHandler' + # Used to generate incremental compose id in dry run mode. + _FAKE_COMPOSE_ID = 1 + def can_handle(self, event): return isinstance(event, ErrataAdvisoryRPMsSignedEvent) @@ -160,9 +166,27 @@ class ErrataAdvisoryRPMsSignedHandler(BaseHandler): odcs = ODCS(conf.odcs_server_url, auth_mech=AuthMech.Kerberos, verify_ssl=conf.odcs_verify_ssl) - with self.krb_context: - new_compose = odcs.new_compose( - compose_source, 'tag', packages=packages) + if not conf.dry_run: + with self.krb_context: + new_compose = odcs.new_compose( + compose_source, 'tag', packages=packages) + else: + log.info("DRY RUN: Calling fake odcs.new_compose with args: %r", + (compose_source, 'tag', packages)) + + ErrataAdvisoryRPMsSignedHandler._FAKE_COMPOSE_ID += 1 + + new_compose = {} + new_compose['id'] = ErrataAdvisoryRPMsSignedHandler._FAKE_COMPOSE_ID + new_compose['result_repofile'] = "http://localhost/%d.repo" % ( + new_compose['id']) + new_compose['state'] = COMPOSE_STATES['done'] + + event = ODCSComposeStateChangeEvent( + "fake_compose_msg", new_compose) + log.info("Injecting fake event: %r", event) + work_queue_put(event) + compose_id = new_compose['id'] yum_repourl = new_compose['result_repofile'] diff --git a/freshmaker/kojiservice.py b/freshmaker/kojiservice.py index c1efa68..4347195 100644 --- a/freshmaker/kojiservice.py +++ b/freshmaker/kojiservice.py @@ -24,7 +24,10 @@ import koji import contextlib -from freshmaker import log +import re +from freshmaker import log, conf +from freshmaker.consumer import work_queue_put +from freshmaker.events import BrewContainerTaskStateChangeEvent class KojiService(object): @@ -37,9 +40,11 @@ class KojiService(object): As a wrapper of Koji API, new APIs could be added as well. """ - def __init__(self, profile=None, logger=None): + # Used to generate incremental task id in dry run mode. + _FAKE_TASK_ID = 1 + + def __init__(self, profile=None): self._config = koji.read_config(profile or 'koji') - self._logger = logger @property def config(self): @@ -61,7 +66,11 @@ class KojiService(object): return self._session def krb_login(self, proxyuser=None): - self.session.krb_login(proxyuser=proxyuser) + # No need to login on dry run, this makes dry run much faster. + if not conf.dry_run: + self.session.krb_login(proxyuser=proxyuser) + else: + log.info("DRY RUN: Skipping login in dry run mode.") @property def logged_in(self): @@ -90,16 +99,30 @@ class KojiService(object): if release: build_opts['release'] = release - if self._logger: - self._logger.debug('Build from target: %s', build_target) - self._logger.debug('Build options: %s', build_opts) + log.debug('Build from target: %s', build_target) + log.debug('Build options: %s', build_opts) + + if not conf.dry_run: + task_id = self.session.buildContainer(source_url, build_target, build_opts) + else: + log.info("DRY RUN: Calling fake buildContainer with args: %r", + (source_url, build_target, build_opts)) + KojiService._FAKE_TASK_ID += 1 + task_id = KojiService._FAKE_TASK_ID + + m = re.match(r".*/(?P[^#]*)", source_url) + container = m.group('container') + event = BrewContainerTaskStateChangeEvent( + "fake_koji_msg_%d" % task_id, container, branch, target, + task_id, "BUILDING", "CLOSED") + + log.info("DRY RUN: Injecting fake event: %r", event) - task_id = self.session.buildContainer(source_url, build_target, build_opts) + work_queue_put(event) - if self._logger: - self._logger.info('Task %s is created to build docker image for %s', - task_id, source_url) - self._logger.info('Task info: %s/taskinfo?taskID=%s', self.weburl, task_id) + log.info('Task %s is created to build docker image for %s', + task_id, source_url) + log.info('Task info: %s/taskinfo?taskID=%s', self.weburl, task_id) return task_id @@ -135,7 +158,7 @@ def koji_service(profile=None, logger=None): with KojiService(koji='stg', logger=logger) as service: ... """ - service = KojiService(profile=profile, logger=logger) + service = KojiService(profile=profile) try: yield service finally: diff --git a/tests/test_brew_container_task_state_change_handler.py b/tests/test_brew_container_task_state_change_handler.py index 2a60890..6cae225 100644 --- a/tests/test_brew_container_task_state_change_handler.py +++ b/tests/test_brew_container_task_state_change_handler.py @@ -68,6 +68,7 @@ class TestBrewContainerTaskStateChangeHandler(helpers.FreshmakerTestCase): """ Tests when dependency container is built, rebuild containers depend on it. """ + build_image.side_effect = [1, 2, 3] repo_urls.return_value = ["url"] e1 = models.Event.create(db.session, "test_msg_id", "RHSA-2018-001", events.TestingEvent) event = self.get_event_from_msg(get_fedmsg('brew_container_task_closed')) @@ -83,12 +84,15 @@ class TestBrewContainerTaskStateChangeHandler(helpers.FreshmakerTestCase): self.handler.handle(event) self.assertEqual(base_build.state, ArtifactBuildState.DONE.value) - # we only log the builds at this moment build_image.assert_has_calls([ mock.call(build_0, ['url']), mock.call(build_1, ['url']), mock.call(build_2, ['url']), ]) + self.assertEqual(build_0.build_id, 1) + self.assertEqual(build_1.build_id, 2) + self.assertEqual(build_2.build_id, 3) + @mock.patch('freshmaker.handlers.ContainerBuildHandler.build_image_artifact_build') @mock.patch('freshmaker.handlers.ContainerBuildHandler.get_repo_urls') def test_not_build_containers_when_dependency_container_build_task_failed(self, repo_urls, build_image): From 2ef8871eef05fb2dcab1d3175fa53cd01b75cf96 Mon Sep 17 00:00:00 2001 From: Jan Kaluza Date: Sep 27 2017 08:47:30 +0000 Subject: [PATCH 2/2] Move dry run mode fake calls to separate methods. --- diff --git a/fedmsg.d/freshmaker-logging.py b/fedmsg.d/freshmaker-logging.py index 682e867..8a87382 100644 --- a/fedmsg.d/freshmaker-logging.py +++ b/fedmsg.d/freshmaker-logging.py @@ -1,12 +1,17 @@ config = dict( logging=dict( loggers=dict( - # Quiet this guy down... + # Quiet these guys down... requests={ "level": "WARNING", "propagate": True, "handlers": ["console"], }, + requests_kerberos={ + "level": "WARNING", + "propagate": True, + "handlers": ["console"], + }, # freshmaker={ # "level": "INFO", # "propagate": True, diff --git a/freshmaker/handlers/__init__.py b/freshmaker/handlers/__init__.py index fea6ee5..2887860 100644 --- a/freshmaker/handlers/__init__.py +++ b/freshmaker/handlers/__init__.py @@ -197,7 +197,6 @@ class ContainerBuildHandler(BaseHandler): proxyuser = conf.koji_build_owner if conf.koji_proxyuser else None - with self.krb_context: service.krb_login(proxyuser=proxyuser) diff --git a/freshmaker/handlers/errata/errata_advisory_rpms_signed.py b/freshmaker/handlers/errata/errata_advisory_rpms_signed.py index 3485cb4..ad19a0a 100644 --- a/freshmaker/handlers/errata/errata_advisory_rpms_signed.py +++ b/freshmaker/handlers/errata/errata_advisory_rpms_signed.py @@ -130,6 +130,35 @@ class ErrataAdvisoryRPMsSignedHandler(BaseHandler): return [] + def _fake_odcs_new_compose(self, compose_source, tag, packages=None): + """ + Fake KojiSession.buildContainer method used dry run mode. + + Logs the arguments and emits ErrataAdvisoryRPMsSignedHandler of + "done" state. + + :rtype: dict + :return: Fake odcs.new_compose dict. + """ + log.info("DRY RUN: Calling fake odcs.new_compose with args: %r", + (compose_source, tag, packages)) + + # Generate the new_compose dict. + ErrataAdvisoryRPMsSignedHandler._FAKE_COMPOSE_ID += 1 + new_compose = {} + new_compose['id'] = ErrataAdvisoryRPMsSignedHandler._FAKE_COMPOSE_ID + new_compose['result_repofile'] = "http://localhost/%d.repo" % ( + new_compose['id']) + new_compose['state'] = COMPOSE_STATES['done'] + + # Generate and inject the ODCSComposeStateChangeEvent event. + event = ODCSComposeStateChangeEvent( + "fake_compose_msg", new_compose) + log.info("Injecting fake event: %r", event) + work_queue_put(event) + + return new_compose + def _prepare_yum_repo(self, db_event): """ Prepare a yum repo for rebuild @@ -171,21 +200,8 @@ class ErrataAdvisoryRPMsSignedHandler(BaseHandler): new_compose = odcs.new_compose( compose_source, 'tag', packages=packages) else: - log.info("DRY RUN: Calling fake odcs.new_compose with args: %r", - (compose_source, 'tag', packages)) - - ErrataAdvisoryRPMsSignedHandler._FAKE_COMPOSE_ID += 1 - - new_compose = {} - new_compose['id'] = ErrataAdvisoryRPMsSignedHandler._FAKE_COMPOSE_ID - new_compose['result_repofile'] = "http://localhost/%d.repo" % ( - new_compose['id']) - new_compose['state'] = COMPOSE_STATES['done'] - - event = ODCSComposeStateChangeEvent( - "fake_compose_msg", new_compose) - log.info("Injecting fake event: %r", event) - work_queue_put(event) + new_compose = self._fake_odcs_new_compose( + compose_source, 'tag', packages=packages) compose_id = new_compose['id'] yum_repourl = new_compose['result_repofile'] diff --git a/freshmaker/kojiservice.py b/freshmaker/kojiservice.py index 4347195..5b28081 100644 --- a/freshmaker/kojiservice.py +++ b/freshmaker/kojiservice.py @@ -79,6 +79,37 @@ class KojiService(object): def logout(self): self.session.logout() + def _fake_build_container(self, source_url, build_target, build_opts): + """ + Fake KojiSession.buildContainer method used dry run mode. + + Logs the arguments and emits BrewContainerTaskStateChangeEvent of + CLOSED state. + + :rtype: number + :return: Fake task_id. + """ + log.info("DRY RUN: Calling fake buildContainer with args: %r", + (source_url, build_target, build_opts)) + + # Get the task_id + KojiService._FAKE_TASK_ID += 1 + task_id = KojiService._FAKE_TASK_ID + + # Parse the source_url to get the name of container and generate + # fake event. + m = re.match(r".*/(?P[^#]*)", source_url) + container = m.group('container') + event = BrewContainerTaskStateChangeEvent( + "fake_koji_msg_%d" % task_id, container, build_opts["git_branch"], + build_target, task_id, "BUILDING", "CLOSED") + + # Inject the fake event. + log.info("DRY RUN: Injecting fake event: %r", event) + work_queue_put(event) + + return task_id + def build_container(self, source_url, branch, target, scratch=None, repo_urls=None, isolated=False, release=None, koji_parent_build=None): @@ -103,22 +134,11 @@ class KojiService(object): log.debug('Build options: %s', build_opts) if not conf.dry_run: - task_id = self.session.buildContainer(source_url, build_target, build_opts) + task_id = self.session.buildContainer(source_url, build_target, + build_opts) else: - log.info("DRY RUN: Calling fake buildContainer with args: %r", - (source_url, build_target, build_opts)) - KojiService._FAKE_TASK_ID += 1 - task_id = KojiService._FAKE_TASK_ID - - m = re.match(r".*/(?P[^#]*)", source_url) - container = m.group('container') - event = BrewContainerTaskStateChangeEvent( - "fake_koji_msg_%d" % task_id, container, branch, target, - task_id, "BUILDING", "CLOSED") - - log.info("DRY RUN: Injecting fake event: %r", event) - - work_queue_put(event) + task_id = self._fake_build_container(source_url, build_target, + build_opts) log.info('Task %s is created to build docker image for %s', task_id, source_url) diff --git a/tests/test_consumer.py b/tests/test_consumer.py index 0903793..eb07adf 100644 --- a/tests/test_consumer.py +++ b/tests/test_consumer.py @@ -148,5 +148,6 @@ class ParseBrewSignRPMEventTest(ConsumerBaseTest): self.assertRaises(ValueError, consumer.get_abstracted_msg, msg) + if __name__ == '__main__': unittest.main()