From 245c59b8b59aa9b156f3706849d5e29ab1ea4a9d Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Jan 13 2020 13:18:54 +0000 Subject: [PATCH 1/8] Move the list of package to build to its own list Signed-off-by: Pierre-Yves Chibon --- diff --git a/monitor_gating_multi_builds.py b/monitor_gating_multi_builds.py index 08d000e..abf3ac5 100644 --- a/monitor_gating_multi_builds.py +++ b/monitor_gating_multi_builds.py @@ -152,13 +152,14 @@ def main(): conf = toml.load(args.conf) fas_username = conf["fas_username"] + pkgslist = [conf["name_multi_1"], conf["name_multi_2"]] with tempfile.TemporaryDirectory(prefix="ci-test-") as folder: print(f"Working in {folder}\n") nevrs = {} # Bump the release on both packages: - for idx, name in enumerate([conf["name_multi_1"], conf["name_multi_2"]]): + for idx, name in enumerate(pkgslist): if idx < 1: nevrs, side_tag_name = clone_to_build( folder, nevrs, conf, name, new_side_tag=True) From 619eac7a7e60746401f8a757f268115426fe03b5 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Jan 13 2020 13:19:19 +0000 Subject: [PATCH 2/8] Re-enable the checks on the CI pipeline now that it has been enabled Signed-off-by: Pierre-Yves Chibon --- diff --git a/monitor_gating_multi_builds.py b/monitor_gating_multi_builds.py index abf3ac5..11c8f2a 100644 --- a/monitor_gating_multi_builds.py +++ b/monitor_gating_multi_builds.py @@ -186,63 +186,72 @@ def main(): print(f" Update created : {updateid}") # Check the tag of the build - # get_build_tags( - # conf.get("koji_hub"), - # nevr, - # expected_ends=["signing-pending", "testing-pending"], - # ) + get_build_tags( + conf.get("koji_hub"), + nevr, + expected_ends=["signing-pending", "testing-pending"], + ) if not updateid: finalize(start) return + # Check that bodhi notified the pipeline it can run + lookup_results_datagrepper( + base_url=conf["datagrepper"], + name="bodhi to CI", + topic=f"org.fedoraproject.{conf['_env']}.bodhi.update.status." + "testing.koji-build-group.build.complete", + bodhi_id=updateid, + ) + nevr_names = [] for name in nevrs: nevr = nevrs[name] nevr_names.append(nevr) # Check that the CI pipeline is running - # lookup_results_datagrepper( - # base_url=conf["datagrepper"], - # name="CI (running)", - # topic=f"org.centos.{conf['_ci_env']}.ci.pipeline.allpackages-build.package.running", - # nevr=nevr, - # ) - # # Check at the CI pipeline has completed - # lookup_results_datagrepper( - # base_url=conf["datagrepper"], - # name="CI (complete)", - # topic=f"org.centos.{conf['_ci_env']}.ci.pipeline.allpackages-build.complete", - # nevr=nevr, - # ) - - # # Check the tag of the build - # get_build_tags( - # conf.get("koji_hub"), - # nevr, - # expected_ends=["testing-pending"], - # ) + lookup_results_datagrepper( + base_url=conf["datagrepper"], + name="CI (running)", + topic=f"org.centos.{conf['_ci_env']}.ci.koji-build.test.running", + nevr=nevr, + ) + # Check at the CI pipeline has completed + lookup_results_datagrepper( + base_url=conf["datagrepper"], + name="CI (complete)", + topic=f"org.centos.{conf['_ci_env']}.ci.koji-build.test.complete", + nevr=nevr, + ) + + # Check the tag of the build + get_build_tags( + conf.get("koji_hub"), + nevr, + expected_ends=["testing-pending"], + ) # Check that the CI results made it to resultsdb - # lookup_ci_resultsdb( - # nevr=nevr, name="resultsdb(phx)", url=conf["resultsdb"] - # ) + lookup_ci_resultsdb( + nevr=nevr, name="resultsdb(phx)", url=conf["resultsdb"] + ) # Check that resultsdb announced the new results - # lookup_results_datagrepper( - # base_url=conf["datagrepper"], - # name="resultsdb", - # topic=f"org.fedoraproject.{conf['_env']}.resultsdb.result.new", - # nevr=nevr, - # ) + lookup_results_datagrepper( + base_url=conf["datagrepper"], + name="resultsdb", + topic=f"org.fedoraproject.{conf['_env']}.resultsdb.result.new", + nevr=nevr, + ) # Check that greenwave reacted to resultsdb's new results - # lookup_results_datagrepper( - # base_url=conf["datagrepper"], - # name="greenwave", - # topic=f"org.fedoraproject.{conf['_env']}.greenwave.decision.update", - # nevr=nevr, - # ) + lookup_results_datagrepper( + base_url=conf["datagrepper"], + name="greenwave", + topic=f"org.fedoraproject.{conf['_env']}.greenwave.decision.update", + nevr=nevr, + ) # Check the tag of the build -- build is blocked but should be signed get_build_tags( From a9f4d4ec4031fd295f38b11aa62371f6d3904500 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Jan 13 2020 15:56:39 +0000 Subject: [PATCH 3/8] Move to use chain-build instead of doing separate builds Signed-off-by: Pierre-Yves Chibon --- diff --git a/monitor_gating_multi_builds.py b/monitor_gating_multi_builds.py index 11c8f2a..e660ebd 100644 --- a/monitor_gating_multi_builds.py +++ b/monitor_gating_multi_builds.py @@ -40,6 +40,7 @@ from utils import( switch_branch, bump_release, commit_changes, + chain_build_packages, push_changes, pull_changes, open_pullrequest, @@ -114,8 +115,8 @@ def create_side_tag(command, folder): return side_tag_name -def clone_to_build(folder, nevrs, conf, name, target=None, new_side_tag=False): - """Clone the repo, bump the release, commit and build.""" +def clone_and_bump(folder, nevrs, conf, name, target=None, new_side_tag=False): + """Clone the repo, bump the release, commit and push.""" namespace = conf["namespace"] branch = conf["branch"] @@ -134,12 +135,6 @@ def clone_to_build(folder, nevrs, conf, name, target=None, new_side_tag=False): # Push to the main repo push_changes(gitfolder, "origin", branch) print(f" Upcoming build : {nevr}") - # Build the package - build_package( - conf["fedpkg"], - folder=gitfolder, - target=target - ) return (nevrs, target) @@ -161,12 +156,20 @@ def main(): # Bump the release on both packages: for idx, name in enumerate(pkgslist): if idx < 1: - nevrs, side_tag_name = clone_to_build( + nevrs, side_tag_name = clone_and_bump( folder, nevrs, conf, name, new_side_tag=True) else: - nevrs, _ = clone_to_build( + nevrs, _ = clone_and_bump( folder, nevrs, conf, name, target=side_tag_name) + # Chain-build the packages + chain_build_packages( + conf["fedpkg"], + packages=pkgslist[0], + folder=os.path.join(folder, pkgslist[1]), + target=side_tag_name + ) + if args.conflict: clone_to_build(folder, nevrs, conf, name, target=None) diff --git a/utils.py b/utils.py index f5aef7a..1f641b2 100644 --- a/utils.py +++ b/utils.py @@ -219,6 +219,24 @@ def build_package(command, folder, target=None): print_user(info_log, success=False) +def chain_build_packages(command, packages, folder, target=None): + """ Chain-build the packages in the current branch + """ + if not isinstance(packages, list): + packages = [packages] + info_log = f"Chain-building the packages: {packages + [os.path.basename(folder)]}" + print_user(info_log) + command = [command, "chain-build"] + command.extend(packages) + if target: + command.extend(["--target", target]) + try: + run_command(command, cwd=folder) + print_user(info_log, success=True) + except MonitoringException: + print_user(info_log, success=False) + + def get_build_tags(koji_url, nevr, expected_ends): """ List the tags associated with the specified build. """ From 95b46227e8c63820c61736c798b2aab3bbd94f6b Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Jan 13 2020 15:57:08 +0000 Subject: [PATCH 4/8] Fix the undefined nevr variable by picking the first one in the list of nevrs Signed-off-by: Pierre-Yves Chibon --- diff --git a/monitor_gating_multi_builds.py b/monitor_gating_multi_builds.py index e660ebd..1c598f7 100644 --- a/monitor_gating_multi_builds.py +++ b/monitor_gating_multi_builds.py @@ -191,7 +191,7 @@ def main(): # Check the tag of the build get_build_tags( conf.get("koji_hub"), - nevr, + nevrs[pkgslist[0]], expected_ends=["signing-pending", "testing-pending"], ) From 6240a030c151cf369b2a1695c425bc6bb2cdea2c Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Jan 14 2020 08:52:28 +0000 Subject: [PATCH 5/8] Fixed the time from which we look for results in datagrepper Otherwise, as it may take a little time to check them, we may end up in a situation where we are looking for messages with a "start" time which is more recent than the time at which the messages we are looking for were sent/received. Signed-off-by: Pierre-Yves Chibon --- diff --git a/monitor_gating_multi_builds.py b/monitor_gating_multi_builds.py index 1c598f7..1f70270 100644 --- a/monitor_gating_multi_builds.py +++ b/monitor_gating_multi_builds.py @@ -208,6 +208,8 @@ def main(): bodhi_id=updateid, ) + start_dg = datetime.datetime.utcnow() + nevr_names = [] for name in nevrs: nevr = nevrs[name] @@ -219,6 +221,7 @@ def main(): name="CI (running)", topic=f"org.centos.{conf['_ci_env']}.ci.koji-build.test.running", nevr=nevr, + start=start_dg, ) # Check at the CI pipeline has completed lookup_results_datagrepper( @@ -226,6 +229,7 @@ def main(): name="CI (complete)", topic=f"org.centos.{conf['_ci_env']}.ci.koji-build.test.complete", nevr=nevr, + start=start_dg, ) # Check the tag of the build @@ -246,6 +250,7 @@ def main(): name="resultsdb", topic=f"org.fedoraproject.{conf['_env']}.resultsdb.result.new", nevr=nevr, + start=start_dg, ) # Check that greenwave reacted to resultsdb's new results @@ -254,6 +259,7 @@ def main(): name="greenwave", topic=f"org.fedoraproject.{conf['_env']}.greenwave.decision.update", nevr=nevr, + start=start_dg, ) # Check the tag of the build -- build is blocked but should be signed diff --git a/utils.py b/utils.py index 1f641b2..4c3e8dd 100644 --- a/utils.py +++ b/utils.py @@ -358,12 +358,13 @@ def get_update_id(nevr, url): def lookup_results_datagrepper( base_url, name, topic, nevr=None, nevrs=None, rev=None, bodhi_id=None, - duration=15 + start=None, duration=15 ): """ Check the CI results in datagrepper for results about our specified build. """ - start = datetime.datetime.utcnow() + if start is None: + start = datetime.datetime.utcnow() info_log = f"Checking datagrepper for {name} messages" print_user(info_log) # Start pulling messages 10 minutes before now From 4a8b3e06a5618b77513917262c65494ed06a6d0c Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Jan 14 2020 08:53:55 +0000 Subject: [PATCH 6/8] Fix the topic when looking for the CI results in datagrepper The tests we have for all our test packages are all always erroring Signed-off-by: Pierre-Yves Chibon --- diff --git a/monitor_gating_multi_builds.py b/monitor_gating_multi_builds.py index 1f70270..41a0880 100644 --- a/monitor_gating_multi_builds.py +++ b/monitor_gating_multi_builds.py @@ -227,7 +227,7 @@ def main(): lookup_results_datagrepper( base_url=conf["datagrepper"], name="CI (complete)", - topic=f"org.centos.{conf['_ci_env']}.ci.koji-build.test.complete", + topic=f"org.centos.{conf['_ci_env']}.ci.koji-build.test.error", nevr=nevr, start=start_dg, ) From 616bf6946630722050a46ad2309fbd6e508b7edd Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Jan 14 2020 08:54:35 +0000 Subject: [PATCH 7/8] Document two more variables in the example configuration file These two variables being bodhi-user and bodhi-password both used when calling bodhi's CLI and allowing to authenticate without interaction. Signed-off-by: Pierre-Yves Chibon --- diff --git a/monitor_gating.cfg b/monitor_gating.cfg index aa1aa93..c446482 100644 --- a/monitor_gating.cfg +++ b/monitor_gating.cfg @@ -7,6 +7,8 @@ distgit_host = "pkgs.fedoraproject.org" koji_hub = "https://koji.fedoraproject.org/kojihub" bodhi = "https://bodhi.fedoraproject.org" bodhi-cli = "bodhi" +bodhi-user = "pingou" +bodhi-password = None _env = "prod" _ci_env = "prod" From 2619f1765b729afb5d9988e799242fe4a0a7be42 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Jan 14 2020 11:22:39 +0000 Subject: [PATCH 8/8] Drop the use of the pkgslist since there are only two packages considered Signed-off-by: Pierre-Yves Chibon --- diff --git a/monitor_gating_multi_builds.py b/monitor_gating_multi_builds.py index 41a0880..0fd3f71 100644 --- a/monitor_gating_multi_builds.py +++ b/monitor_gating_multi_builds.py @@ -147,26 +147,22 @@ def main(): conf = toml.load(args.conf) fas_username = conf["fas_username"] - pkgslist = [conf["name_multi_1"], conf["name_multi_2"]] with tempfile.TemporaryDirectory(prefix="ci-test-") as folder: print(f"Working in {folder}\n") nevrs = {} # Bump the release on both packages: - for idx, name in enumerate(pkgslist): - if idx < 1: - nevrs, side_tag_name = clone_and_bump( - folder, nevrs, conf, name, new_side_tag=True) - else: - nevrs, _ = clone_and_bump( - folder, nevrs, conf, name, target=side_tag_name) + nevrs, side_tag_name = clone_and_bump( + folder, nevrs, conf, conf["name_multi_1"], new_side_tag=True) + nevrs, _ = clone_and_bump( + folder, nevrs, conf, conf["name_multi_2"], target=side_tag_name) # Chain-build the packages chain_build_packages( conf["fedpkg"], - packages=pkgslist[0], - folder=os.path.join(folder, pkgslist[1]), + packages=conf["name_multi_1"], + folder=os.path.join(folder, conf["name_multi_2"]), target=side_tag_name ) @@ -191,7 +187,7 @@ def main(): # Check the tag of the build get_build_tags( conf.get("koji_hub"), - nevrs[pkgslist[0]], + nevrs[conf["name_multi_1"]], expected_ends=["signing-pending", "testing-pending"], )