From 47544e918e5be7f3bf4f05ac9e1eb0d83c2e2ab2 Mon Sep 17 00:00:00 2001 From: Jan Kaluza Date: Sep 15 2017 12:46:33 +0000 Subject: Add way how to express dependencies between Events using Event.add_event_dependency and Event.event_dependencies. --- diff --git a/freshmaker/handlers/__init__.py b/freshmaker/handlers/__init__.py index 82df05d..8cca2c7 100644 --- a/freshmaker/handlers/__init__.py +++ b/freshmaker/handlers/__init__.py @@ -33,6 +33,9 @@ from freshmaker.models import ArtifactBuildState from freshmaker.models import Event from krbcontext import krbContext +from freshmaker.odcsclient import ODCS +from freshmaker.odcsclient import AuthMech + class BaseHandler(object): """ @@ -218,7 +221,19 @@ class ContainerBuildHandler(BaseHandler): """ rebuild_event = Event.get(db.session, db_event.message_id) - # TODO: Add other repofiles from "extra events" + + # Get compose ids of ODCS composes of all event dependencies. + compose_ids = [rebuild_event.compose_id] + for event in rebuild_event.event_dependencies: + compose_ids.append(event.compose_id) + + # Use compose ids to get the repofile URLs. + repo_urls = [] + for compose_id in compose_ids: + odcs = ODCS(conf.odcs_server_url, auth_mech=AuthMech.Kerberos, + verify_ssl=conf.odcs_verify_ssl) + compose = odcs.get_compose(compose_id) + repo_urls.append(compose["result_repofile"]) for build in rebuild_event.builds: if build.dep_on: @@ -252,7 +267,7 @@ class ContainerBuildHandler(BaseHandler): target = args["target"] build.build_id = self.build_container( - scm_url, branch, target, repo_urls=[args['yum_repourl']], + scm_url, branch, target, repo_urls=repo_urls, isolated=True, release=release, koji_parent_build=parent) build.state = ArtifactBuildState.BUILD.value db.session.add(build) diff --git a/freshmaker/handlers/errata/errata_advisory_rpms_signed.py b/freshmaker/handlers/errata/errata_advisory_rpms_signed.py index e0cf0cd..9e11cf2 100644 --- a/freshmaker/handlers/errata/errata_advisory_rpms_signed.py +++ b/freshmaker/handlers/errata/errata_advisory_rpms_signed.py @@ -107,10 +107,12 @@ class ErrataAdvisoryRPMsSignedHandler(BaseHandler): if ev in seen_extra_events: continue seen_extra_events.append(ev) + db_event.add_event_dependency(db.session, ev) builds = self._find_and_record_images_to_rebuild( ev, event, builds) repo_urls.append(self._prepare_yum_repo(ev)) + db.session.commit() # Remove duplicates from repo_urls. repo_urls = list(set(repo_urls)) @@ -169,12 +171,6 @@ class ErrataAdvisoryRPMsSignedHandler(BaseHandler): rebuild_event = Event.get(db.session, db_event.msg_id) rebuild_event.compose_id = compose_id - # Save YUM repo URL for saving time to retrieve compose from ODCS again - # when start to rebuild images. - for build in rebuild_event.builds: - data = json.loads(build.build_args) - data['yum_repourl'] = yum_repourl - build.build_args = json.dumps(data) db.session.commit() return yum_repourl diff --git a/freshmaker/handlers/odcs/__init__.py b/freshmaker/handlers/odcs/__init__.py index 86b1b04..b29af86 100644 --- a/freshmaker/handlers/odcs/__init__.py +++ b/freshmaker/handlers/odcs/__init__.py @@ -19,4 +19,4 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. -from .compose_state_change import ComposeStateChangeHandler # noqa \ No newline at end of file +from .compose_state_change import ComposeStateChangeHandler # noqa diff --git a/freshmaker/migrations/versions/300b86758bb1_.py b/freshmaker/migrations/versions/300b86758bb1_.py new file mode 100644 index 0000000..e4aa2b8 --- /dev/null +++ b/freshmaker/migrations/versions/300b86758bb1_.py @@ -0,0 +1,33 @@ +"""empty message + +Revision ID: 300b86758bb1 +Revises: bfc0e0d2eea6 +Create Date: 2017-09-14 07:41:45.501531 + +""" + +# revision identifiers, used by Alembic. +revision = '300b86758bb1' +down_revision = 'bfc0e0d2eea6' + +from alembic import op +import sqlalchemy as sa + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.create_table('event_dependencies', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('event_id', sa.Integer(), nullable=False), + sa.Column('event_dependency_id', sa.Integer(), nullable=False), + sa.ForeignKeyConstraint(['event_dependency_id'], ['events.id'], ), + sa.ForeignKeyConstraint(['event_id'], ['events.id'], ), + sa.PrimaryKeyConstraint('id') + ) + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_table('event_dependencies') + # ### end Alembic commands ### diff --git a/freshmaker/models.py b/freshmaker/models.py index 37107dc..f4de93b 100644 --- a/freshmaker/models.py +++ b/freshmaker/models.py @@ -111,6 +111,20 @@ class Event(FreshmakerBase): def event_type(self): return INVERSE_EVENT_TYPES[self.event_type_id] + def add_event_dependency(self, session, event): + dep = EventDependency(event_id=self.id, + event_dependency_id=event.id) + session.add(dep) + + @property + def event_dependencies(self): + events = [] + deps = EventDependency.query.filter_by(event_id=self.id).all() + for dep in deps: + events.append(Event.query.filter_by( + id=dep.event_dependency_id).first()) + return events + def __repr__(self): return "" % (self.message_id, self.event_type, self.search_key) @@ -124,6 +138,13 @@ class Event(FreshmakerBase): } +class EventDependency(FreshmakerBase): + __tablename__ = "event_dependencies" + id = db.Column(db.Integer, primary_key=True) + event_id = db.Column(db.Integer, db.ForeignKey('events.id'), nullable=False) + event_dependency_id = db.Column(db.Integer, db.ForeignKey('events.id'), nullable=False) + + class ArtifactBuild(FreshmakerBase): __tablename__ = "artifact_builds" id = db.Column(db.Integer, primary_key=True) diff --git a/freshmaker/odcsclient.py b/freshmaker/odcsclient.py new file mode 100644 index 0000000..4eaf69a --- /dev/null +++ b/freshmaker/odcsclient.py @@ -0,0 +1,32 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2017 Red Hat, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +# Written by Jan Kaluza + +# We have name conflict between two modules here: +# - "odcs" module provided by python2-odcs-client +# - "odcs" submodule in freshmaker.handlers.odcs +# +# Unfortunatelly we want to use "odcs" provided by python2-odcs-client +# in freshmaker.handlers __init__.py. We cannot "import odcs" there, because +# it would import freshmaker.handlers.odcs, so instead, we import it here +# and in freshmaker.handler do "from freshmaker.odcsclient import ODCS". +from odcs.client.odcs import * # noqa diff --git a/tests/test_handler.py b/tests/test_handler.py index dd61cb2..2214b8e 100644 --- a/tests/test_handler.py +++ b/tests/test_handler.py @@ -135,13 +135,23 @@ class TestBuildFirstBatch(TestCase): db.drop_all() db.session.commit() + @patch('freshmaker.handlers.ODCS') @patch('koji.ClientSession') @patch('freshmaker.handlers.krbContext') - def test_build_first_batch(self, krb, ClientSession): + def test_build_first_batch(self, krb, ClientSession, ODCS): """ Tests that only PLANNED images without a parent are submitted to build system. """ + ODCS.return_value.get_compose.return_value = { + "id": 3, + "result_repo": "http://localhost/composes/latest-odcs-3-1/compose/Temporary", + "result_repofile": "http://localhost/composes/latest-odcs-3-1/compose/Temporary/odcs-3.repo", + "source": "f26", + "source_type": 1, + "state": 2, + "state_name": "done", + } mock_session = ClientSession.return_value mock_session.buildContainer.return_value = 123 @@ -162,3 +172,58 @@ class TestBuildFirstBatch(TestCase): self.assertEqual(build.build_id, 123) else: self.assertEqual(build.build_id, None) + + @patch('freshmaker.handlers.ODCS') + @patch('koji.ClientSession') + @patch('freshmaker.handlers.krbContext') + def test_build_first_batch_extra_events(self, krb, ClientSession, ODCS): + """ + Tests that only PLANNED images without a parent are submitted to + build system. + """ + ODCS.return_value.get_compose.side_effect = [{ + "id": 3, + "result_repo": "http://localhost/composes/latest-odcs-3-1/compose/Temporary", + "result_repofile": "http://localhost/composes/latest-odcs-3-1/compose/Temporary/odcs-3.repo", + "source": "f26", + "source_type": 1, + "state": 2, + "state_name": "done", + }, { + "id": 4, + "result_repo": "http://localhost/composes/latest-odcs-4-1/compose/Temporary", + "result_repofile": "http://localhost/composes/latest-odcs-4-1/compose/Temporary/odcs-4.repo", + "source": "f26", + "source_type": 1, + "state": 2, + "state_name": "done", + }] + mock_session = ClientSession.return_value + mock_session.buildContainer.return_value = 123 + + db_event2 = Event.get_or_create( + db.session, "msg2", "current_event", ErrataAdvisoryRPMsSignedEvent, + released=False) + db_event2.compose_id = 4 + db.session.commit() + self.db_event.add_event_dependency(db.session, db_event2) + db.session.commit() + + handler = MyHandler() + handler._build_first_batch(self.db_event) + + mock_session.buildContainer.assert_called_once_with( + 'git://pkgs.fedoraproject.org/repo#hash', + 'target', + {'scratch': True, 'isolated': True, 'koji_parent_build': u'nvr', + 'git_branch': 'unknown', 'release': AnyStringWith('4.'), + 'yum_repourls': [ + 'http://localhost/composes/latest-odcs-3-1/compose/Temporary/odcs-3.repo', + 'http://localhost/composes/latest-odcs-4-1/compose/Temporary/odcs-4.repo']}) + + db.session.refresh(self.db_event) + for build in self.db_event.builds: + if build.name == "parent1-1-4": + self.assertEqual(build.build_id, 123) + else: + self.assertEqual(build.build_id, None) diff --git a/tests/test_lightblue.py b/tests/test_lightblue.py index ea3731a..29b1f4d 100644 --- a/tests/test_lightblue.py +++ b/tests/test_lightblue.py @@ -632,8 +632,14 @@ class TestQueryEntityFromLightBlue(unittest.TestCase): @patch('freshmaker.lightblue.LightBlue.find_container_images') @patch('os.path.exists') - def test_parent_images_with_package(self, exists, cont_images): + @patch('freshmaker.kojiservice.KojiService.get_build') + @patch('freshmaker.kojiservice.KojiService.get_task_request') + def test_parent_images_with_package(self, get_task_request, get_build, + exists, cont_images): + get_build.return_value = {"task_id": 123456} + get_task_request.return_value = [ + "git://example.com/rpms/repo-1#commit_hash1", "target1"] exists.return_value = True cont_images.side_effect = [self.fake_container_images, [], self.fake_container_images] diff --git a/tests/test_models.py b/tests/test_models.py index 0d21ba0..3d563cd 100644 --- a/tests/test_models.py +++ b/tests/test_models.py @@ -75,3 +75,16 @@ class TestModels(unittest.TestCase): self.assertEqual(build2.get_root_dep_on(), build1) self.assertEqual(build3.get_root_dep_on(), build1) self.assertEqual(build4.get_root_dep_on(), build1) + + def test_event_dependencies(self): + event = Event.create(db.session, "test_msg_id", "test", events.TestingEvent) + db.session.commit() + self.assertEqual(event.event_dependencies, []) + + event1 = Event.create(db.session, "test_msg_id2", "test2", events.TestingEvent) + db.session.commit() + event.add_event_dependency(db.session, event1) + db.session.commit() + self.assertEqual(event.event_dependencies, [event1]) + self.assertEqual(event.event_dependencies[0].search_key, "test2") + self.assertEqual(event1.event_dependencies, [])