From 6d2792a6de92e81422064a037b500361a01b3bc6 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Apr 07 2020 08:21:33 +0000 Subject: [PATCH 1/29] Always try to clean up the side-tag at the end of the run This should fail if the tests all passed and will prevent dangling side-tags which are only consuming resources if the tests failed. Signed-off-by: Pierre-Yves Chibon --- diff --git a/monitor_gating_multi_builds.py b/monitor_gating_multi_builds.py index cef5dcd..c4b88ae 100644 --- a/monitor_gating_multi_builds.py +++ b/monitor_gating_multi_builds.py @@ -218,6 +218,15 @@ def main(args): conf.get("koji_hub"), nevr, expected_ends=conf["koji_end_tag"], ) + try: + print(" Removing side-tag: %s" % side_tag_name) + cmd = [ + conf["fedpkg"], "remove-side-tag", side_tag_name + ] + output = run_command(cmd) + except Exception: + pass + utils.finalize(start) return utils.logs From d4298884499a486a170b765ad38551f7fd43da56 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Apr 15 2020 07:54:35 +0000 Subject: [PATCH 2/29] Do not split the kerberos principal, it is needed to list the side-tags Signed-off-by: Pierre-Yves Chibon --- diff --git a/clean_up_side_tags.py b/clean_up_side_tags.py index 607ba4c..59d7afd 100644 --- a/clean_up_side_tags.py +++ b/clean_up_side_tags.py @@ -51,7 +51,7 @@ def main(args): # list side-tags: cmd = [ - conf["fedpkg"], "list-side-tags", "--user", conf["kb_principal"].rsplit('@', 1)[0] + conf["fedpkg"], "list-side-tags", "--user", conf["kb_principal"] ] output = run_command(cmd) From f8d2d2db729be9e3c3d8d4a6573da00c0169b278 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Apr 15 2020 08:00:22 +0000 Subject: [PATCH 3/29] Clean up the side-tag in the runner rather than the test This way if an exception occurs before we get to the check if the tests passed or failed, we will still clean the side-tag. Signed-off-by: Pierre-Yves Chibon --- diff --git a/monitor_gating_multi_builds.py b/monitor_gating_multi_builds.py index c4b88ae..441b015 100644 --- a/monitor_gating_multi_builds.py +++ b/monitor_gating_multi_builds.py @@ -66,14 +66,15 @@ def get_arguments(args): return parser.parse_args(args) -def main(args): +def main(args, utils=None): """ Main method used by this script. """ start = datetime.datetime.utcnow() args = get_arguments(args) conf = toml.load(args.conf) - utils = MonitoringUtils() + if utils is None: + utils = MonitoringUtils() with tempfile.TemporaryDirectory(prefix="ci-test-") as folder: print(f"Working in {folder}\n") @@ -83,6 +84,9 @@ def main(args): nevrs, side_tag_name = utils.clone_and_bump( folder, nevrs, conf, conf["name_multi_1"], new_side_tag=True ) + # Store the side_tag_name so we can use it in the runner + utils.side_tag_name = side_tag_name + nevrs, _ = utils.clone_and_bump( folder, nevrs, conf, conf["name_multi_2"], target=side_tag_name ) @@ -218,15 +222,6 @@ def main(args): conf.get("koji_hub"), nevr, expected_ends=conf["koji_end_tag"], ) - try: - print(" Removing side-tag: %s" % side_tag_name) - cmd = [ - conf["fedpkg"], "remove-side-tag", side_tag_name - ] - output = run_command(cmd) - except Exception: - pass - utils.finalize(start) return utils.logs diff --git a/runner.py b/runner.py index 81ea2c3..f6032a0 100644 --- a/runner.py +++ b/runner.py @@ -18,7 +18,7 @@ import toml import monitor_gating_single_build import monitor_gating_multi_builds -from utils import run_command +from utils import MonitoringUtils, run_command s = sched.scheduler(time.time, time.sleep) conf = toml.load @@ -48,6 +48,17 @@ def notify(topic, message): print(f"Error sending fedora-messaging message: {err}") +def _clean_up_side_tags(utils): + try: + print(" Removing side-tag: %s" % utils.side_tag_name) + cmd = [ + conf["fedpkg"], "remove-side-tag", utils.side_tag_name + ] + output = run_command(cmd) + except Exception: + pass + + def schedule(conf): """ Run the test and schedules the next one. """ @@ -86,11 +97,16 @@ def schedule(conf): # Multi Build Gating multi_args = conf["workflow_multi_gating_args"].split() + utils = MonitoringUtils() notify( topic=f"multi-build.start", message={"arguments": multi_args, "runid": runid,}, ) - output = monitor_gating_multi_builds.main(multi_args) + try: + output = monitor_gating_multi_builds.main(multi_args, utils=utils) + finally: + _clean_up_side_tags(utils) + output_text = "\n".join(output) if "[FAILED]" not in output_text: result = "succeeded" From 06584432fc3d8c6386f750caaaba3a02bbda13ee Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Apr 15 2020 08:01:11 +0000 Subject: [PATCH 4/29] Send the a notification when the script itself crashed So we do not know if the test passed or failed, because an exception happened in the script. Signed-off-by: Pierre-Yves Chibon --- diff --git a/runner.py b/runner.py index f6032a0..f6211d0 100644 --- a/runner.py +++ b/runner.py @@ -127,6 +127,14 @@ def schedule(conf): print(f"Tests failed with: {err}", flush=True) print(sys.exc_info()[0]) + notify( + topic=f"multi-build.end.error", + message={ + "runid": runid, + "exception": err, + }, + ) + blocking_issues = utils.blocking_issues(blocker_tags) now = datetime.datetime.utcnow().strftime("%H:%M:%S") if blocking_issues: From fcbd4399a09f3fa962b48947efedac405ea20121 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Apr 20 2020 13:06:08 +0000 Subject: [PATCH 5/29] Rename the utils variable to not conflict with the utils module Signed-off-by: Pierre-Yves Chibon --- diff --git a/runner.py b/runner.py index f6211d0..e9da11e 100644 --- a/runner.py +++ b/runner.py @@ -97,15 +97,15 @@ def schedule(conf): # Multi Build Gating multi_args = conf["workflow_multi_gating_args"].split() - utils = MonitoringUtils() + monit_utils = MonitoringUtils() notify( topic=f"multi-build.start", message={"arguments": multi_args, "runid": runid,}, ) try: - output = monitor_gating_multi_builds.main(multi_args, utils=utils) + output = monitor_gating_multi_builds.main(multi_args, utils=monit_utils) finally: - _clean_up_side_tags(utils) + _clean_up_side_tags(monit_utils) output_text = "\n".join(output) if "[FAILED]" not in output_text: From 2f90b908cc5a638e54c6fd3534229bc13118222e Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Apr 20 2020 13:12:51 +0000 Subject: [PATCH 6/29] Refactor the code to make explicit what is imported from utils With this, we are also following the principal of stdlib imports first, then 3rd party lib, then application-specific lib. We're also renaming the list of blocking_issues so it does not conflict with the function. Signed-off-by: Pierre-Yves Chibon --- diff --git a/runner.py b/runner.py index e9da11e..a4d30b0 100644 --- a/runner.py +++ b/runner.py @@ -9,7 +9,6 @@ import datetime import sched import sys import time -import utils import uuid import fedora_messaging.api @@ -18,7 +17,7 @@ import toml import monitor_gating_single_build import monitor_gating_multi_builds -from utils import MonitoringUtils, run_command +from utils import MonitoringUtils, blocking_issues, run_command s = sched.scheduler(time.time, time.sleep) conf = toml.load @@ -135,12 +134,12 @@ def schedule(conf): }, ) - blocking_issues = utils.blocking_issues(blocker_tags) + blocking_issues_list = blocking_issues(blocker_tags) now = datetime.datetime.utcnow().strftime("%H:%M:%S") - if blocking_issues: + if blocking_issues_list: print( f"{now} Next run in: {delay_when_failing} seconds because of " - f"{len(blocking_issues)} open issues", + f"{len(blocking_issues_list)} open issues", flush=True, ) s.enter(delay_when_failing, 1, schedule, argument=(conf,)) From 542bb3ed83a28eca8ed1f9686dbe4c29dc9c24de Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Apr 20 2020 13:15:12 +0000 Subject: [PATCH 7/29] Run black on the entire project Signed-off-by: Pierre-Yves Chibon --- diff --git a/clean_up_side_tags.py b/clean_up_side_tags.py index 59d7afd..982cc06 100644 --- a/clean_up_side_tags.py +++ b/clean_up_side_tags.py @@ -14,12 +14,12 @@ _log = logging.getLogger(__name__) def get_cli_args(args): parser = argparse.ArgumentParser( - prog="clean up side-tags", formatter_class=argparse.ArgumentDefaultsHelpFormatter + prog="clean up side-tags", + formatter_class=argparse.ArgumentDefaultsHelpFormatter, ) parser.add_argument( - "conf", - help="Configuration file used by the runner", + "conf", help="Configuration file used by the runner", ) return parser.parse_args(args) @@ -33,7 +33,9 @@ def run_command(command) -> bytes: try: output = subprocess.check_output(command, stderr=subprocess.PIPE) except subprocess.CalledProcessError as e: - _log.error("Command `{}` return code: `{}`".format(" ".join(command), e.returncode)) + _log.error( + "Command `{}` return code: `{}`".format(" ".join(command), e.returncode) + ) _log.error("stdout:\n-------\n{}".format(e.stdout)) _log.error("stderr:\n-------\n{}".format(e.stderr)) pass @@ -45,14 +47,14 @@ def main(args): conf = toml.load(args.conf) if conf.get("kb_principal") and conf.get("kb_keytab_file"): - print(f"Logging as {conf['kb_principal']} into kerberos using: {conf['kb_keytab_file']}") + print( + f"Logging as {conf['kb_principal']} into kerberos using: {conf['kb_keytab_file']}" + ) cmd = ["kinit", conf["kb_principal"], "-kt", conf["kb_keytab_file"]] run_command(cmd) # list side-tags: - cmd = [ - conf["fedpkg"], "list-side-tags", "--user", conf["kb_principal"] - ] + cmd = [conf["fedpkg"], "list-side-tags", "--user", conf["kb_principal"]] output = run_command(cmd) if not output: @@ -63,15 +65,12 @@ def main(args): # Remove all the side-tags but the last one (which is the latest one), which # we keep in case there is a run ongoing for line in output[:-1]: - side_tag = line.split('\t')[0] + side_tag = line.split("\t")[0] print("Removing side-tag: %s" % side_tag) - cmd = [ - conf["fedpkg"], "remove-side-tag", side_tag - ] + cmd = [conf["fedpkg"], "remove-side-tag", side_tag] output = run_command(cmd) - if __name__ == "__main__": """ Main method. """ diff --git a/runner.py b/runner.py index a4d30b0..20c99f6 100644 --- a/runner.py +++ b/runner.py @@ -50,9 +50,7 @@ def notify(topic, message): def _clean_up_side_tags(utils): try: print(" Removing side-tag: %s" % utils.side_tag_name) - cmd = [ - conf["fedpkg"], "remove-side-tag", utils.side_tag_name - ] + cmd = [conf["fedpkg"], "remove-side-tag", utils.side_tag_name] output = run_command(cmd) except Exception: pass @@ -127,11 +125,7 @@ def schedule(conf): print(sys.exc_info()[0]) notify( - topic=f"multi-build.end.error", - message={ - "runid": runid, - "exception": err, - }, + topic=f"multi-build.end.error", message={"runid": runid, "exception": err,}, ) blocking_issues_list = blocking_issues(blocker_tags) From 315b954a1a35f9f03ab2258caf416c01525173a5 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Apr 20 2020 13:17:43 +0000 Subject: [PATCH 8/29] flake8 and black fixes for the entire project Signed-off-by: Pierre-Yves Chibon --- diff --git a/runner.py b/runner.py index 20c99f6..5c68a87 100644 --- a/runner.py +++ b/runner.py @@ -13,10 +13,13 @@ import uuid import fedora_messaging.api import fedora_messaging.exceptions -import toml -import monitor_gating_single_build import monitor_gating_multi_builds + +import monitor_gating_single_build + +import toml + from utils import MonitoringUtils, blocking_issues, run_command s = sched.scheduler(time.time, time.sleep) @@ -51,7 +54,7 @@ def _clean_up_side_tags(utils): try: print(" Removing side-tag: %s" % utils.side_tag_name) cmd = [conf["fedpkg"], "remove-side-tag", utils.side_tag_name] - output = run_command(cmd) + run_command(cmd) except Exception: pass @@ -74,7 +77,7 @@ def schedule(conf): single_args = conf["workflow_single_gating_args"].split() notify( topic=f"single-build.start", - message={"arguments": single_args, "runid": runid,}, + message={"arguments": single_args, "runid": runid}, ) output = monitor_gating_single_build.main(single_args) output_text = "\n".join(output) @@ -97,7 +100,7 @@ def schedule(conf): monit_utils = MonitoringUtils() notify( topic=f"multi-build.start", - message={"arguments": multi_args, "runid": runid,}, + message={"arguments": multi_args, "runid": runid}, ) try: output = monitor_gating_multi_builds.main(multi_args, utils=monit_utils) @@ -125,7 +128,7 @@ def schedule(conf): print(sys.exc_info()[0]) notify( - topic=f"multi-build.end.error", message={"runid": runid, "exception": err,}, + topic=f"multi-build.end.error", message={"runid": runid, "exception": err}, ) blocking_issues_list = blocking_issues(blocker_tags) diff --git a/utils.py b/utils.py index 265b2e2..411a7a2 100644 --- a/utils.py +++ b/utils.py @@ -172,7 +172,7 @@ class MonitoringUtils: info_log = f"Creating PR from forks/{username}/{namespace}/{name}" self.print_user(info_log) url = "/".join( - [base_url.rstrip("/"), "api/0", namespace, name, "pull-request/new",] + [base_url.rstrip("/"), "api/0", namespace, name, "pull-request/new"] ) data = { "branch_to": branch, @@ -258,9 +258,7 @@ class MonitoringUtils: "koji", ] if koji_url: - command.extend( - ["-s", koji_url,] - ) + command.extend(["-s", koji_url]) command.extend(["call", "listTags", nevr]) success = False From 041aee09d8fbf1910efdec12e08ce4daa68ee722 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: May 18 2020 12:26:00 +0000 Subject: [PATCH 9/29] Have the tests return the MonitorUtils object instead of the logs This allows to store more things in the object and access them from the runner. Signed-off-by: Pierre-Yves Chibon --- diff --git a/monitor_gating_multi_builds.py b/monitor_gating_multi_builds.py index 441b015..18648d3 100644 --- a/monitor_gating_multi_builds.py +++ b/monitor_gating_multi_builds.py @@ -223,7 +223,7 @@ def main(args, utils=None): ) utils.finalize(start) - return utils.logs + return utils if __name__ == "__main__": diff --git a/monitor_gating_single_build.py b/monitor_gating_single_build.py index 1156f11..29425b0 100644 --- a/monitor_gating_single_build.py +++ b/monitor_gating_single_build.py @@ -303,7 +303,7 @@ def main(args): ) utils.finalize(start) - return utils.logs + return utils if __name__ == "__main__": diff --git a/runner.py b/runner.py index 5c68a87..08c3247 100644 --- a/runner.py +++ b/runner.py @@ -79,8 +79,8 @@ def schedule(conf): topic=f"single-build.start", message={"arguments": single_args, "runid": runid}, ) - output = monitor_gating_single_build.main(single_args) - output_text = "\n".join(output) + monit_utils = monitor_gating_single_build.main(single_args) + output_text = "\n".join(monit_utils.logs) if "[FAILED]" not in output_text: result = "succeeded" else: @@ -88,7 +88,7 @@ def schedule(conf): notify( topic=f"single-build.end.{result}", message={ - "output": output, + "output": monit_utils.logs, "output_text": output_text, "result": result, "runid": runid, @@ -103,11 +103,13 @@ def schedule(conf): message={"arguments": multi_args, "runid": runid}, ) try: - output = monitor_gating_multi_builds.main(multi_args, utils=monit_utils) + monit_utils = monitor_gating_multi_builds.main( + multi_args, utils=monit_utils + ) finally: _clean_up_side_tags(monit_utils) - output_text = "\n".join(output) + output_text = "\n".join(monit_utils.logs) if "[FAILED]" not in output_text: result = "succeeded" else: @@ -115,7 +117,7 @@ def schedule(conf): notify( topic=f"multi-build.end.{result}", message={ - "output": output, + "output": monit_utils.logs, "output_text": output_text, "result": result, "runid": runid, From c7d1c8fa7bfed8bab604dbe523701e09ae69fd92 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: May 18 2020 12:27:12 +0000 Subject: [PATCH 10/29] For failed each operation in MonitorUtils record what failed Basically in MonitorUtils, for each utility function we know what would have failed for this step to fail, so we record this in the `failed` list and that should tell us at the end which tool failed. Signed-off-by: Pierre-Yves Chibon --- diff --git a/utils.py b/utils.py index 411a7a2..8f331ca 100644 --- a/utils.py +++ b/utils.py @@ -49,6 +49,7 @@ class MonitoringUtils: def __init__(self): """ Instanciate the object. """ self.logs = [] + self.failed = [] def print_user(self, content, success=None): """ Prints the specified content to the user. @@ -81,17 +82,24 @@ class MonitoringUtils: [command, "--user", username, "clone", f"{namespace}/{name}"], cwd=folder, ) - clone_folder = os.path.join(folder, name) - run_command( - ["git", "config", "user.name", "packagerbot"], cwd=clone_folder, - ) - run_command( - ["git", "config", "user.email", "admin@fedoraproject.org"], - cwd=clone_folder, - ) - self.print_user(info_log, success=True) except MonitoringException: + self.failed.append("git/dist-git") self.print_user(info_log, success=False) + else: + try: + # Assume these commands can't fail + clone_folder = os.path.join(folder, name) + run_command( + ["git", "config", "user.name", "packagerbot"], cwd=clone_folder, + ) + run_command( + ["git", "config", "user.email", "admin@fedoraproject.org"], + cwd=clone_folder, + ) + self.print_user(info_log, success=True) + except MonitoringException: + self.failed.append("git") + self.print_user(info_log, success=False) def add_remote(self, name, url, folder): """ Add the specified remote to the git repo in the folder with the @@ -103,6 +111,7 @@ class MonitoringUtils: run_command(["git", "remote", "add", name, url], cwd=folder) self.print_user(info_log, success=True) except MonitoringException: + self.failed.append("git") self.print_user(info_log, success=False) def switch_branch(self, command, name, folder): @@ -114,6 +123,7 @@ class MonitoringUtils: run_command([command, "switch-branch", f"{name}"], cwd=folder) self.print_user(info_log, success=True) except MonitoringException: + self.failed("fedpkg") self.print_user(info_log, success=False) def bump_release(self, name, folder): @@ -125,6 +135,7 @@ class MonitoringUtils: run_command(["rpmdev-bumpspec", f"{name}.spec"], cwd=folder) self.print_user(info_log, success=True) except MonitoringException: + self.failed("rpmdev-bumspec") self.print_user(info_log, success=False) def commit_changes(self, commit_log, folder): @@ -137,6 +148,7 @@ class MonitoringUtils: run_command(["git", "commit", "-asm", commit_log], cwd=folder) self.print_user(info_log, success=True) except MonitoringException: + self.failed("git") self.print_user(info_log, success=False) def push_changes(self, folder, target, branch, force=False): @@ -151,6 +163,7 @@ class MonitoringUtils: run_command(cmd, cwd=folder) self.print_user(info_log, success=True) except MonitoringException: + self.failed("git/dist-git") self.print_user(info_log, success=False) def pull_changes(self, folder, target, branch): @@ -163,6 +176,7 @@ class MonitoringUtils: run_command(cmd, cwd=folder) self.print_user(info_log, success=True) except MonitoringException: + self.failed("git/dist-git") self.print_user(info_log, success=False) def open_pullrequest(self, base_url, username, namespace, name, branch, token): @@ -190,6 +204,7 @@ class MonitoringUtils: success = False pr_id = None pr_uid = None + self.failed("dist-git") else: output = req.json() pr_id = str(output["id"]) @@ -212,6 +227,7 @@ class MonitoringUtils: self.print_user(info_log, success=True) return nevr.strip().decode("utf-8") except MonitoringException: + self.failed("fedpkg") self.print_user(info_log, success=False) def build_package(self, command, folder, target=None): @@ -226,6 +242,7 @@ class MonitoringUtils: run_command(command, cwd=folder) self.print_user(info_log, success=True) except MonitoringException: + self.failed("koji") self.print_user(info_log, success=False) def chain_build_packages(self, command, packages, folder, target=None): @@ -245,6 +262,7 @@ class MonitoringUtils: run_command(command, cwd=folder) self.print_user(info_log, success=True) except MonitoringException: + self.failed("koji") self.print_user(info_log, success=False) def get_build_tags(self, koji_url, nevr, expected_ends): @@ -301,6 +319,8 @@ class MonitoringUtils: break info_log = f"Retrieving koji tags: {tags}" + if not success: + self.failed("koji") self.print_user(info_log, success=success) def create_update( @@ -334,6 +354,7 @@ class MonitoringUtils: run_command(command) self.print_user(info_log, success=True) except MonitoringException: + self.failed("bodhi") self.print_user(info_log, success=False) def get_update_id(self, nevr, url): @@ -354,6 +375,7 @@ class MonitoringUtils: if (datetime.datetime.utcnow() - start).seconds > (15 * 60): success = False + self.failed("bodhi") info_log = f"Update for {nevr} not created within 15 minutes" break @@ -512,6 +534,8 @@ class MonitoringUtils: end = datetime.datetime.utcnow() info_log += f" - ran for: {(end - start_lookup).seconds}s" self.print_user(info_log, success=success) + if not success: + self.failed("datagrepper") def lookup_ci_resultsdb(self, nevr, name, url): """ Check the CI results in the specified resultsdb for results about @@ -561,6 +585,8 @@ class MonitoringUtils: end = datetime.datetime.utcnow() info_log += f" - ran for: {(end - start).seconds}s" self.print_user(info_log, success=success) + if not success: + self.failed("resultsdb") def waive_update(self, command, updateid, prod=True, username=None, password=None): """ Waive all the tests results for the specified update using bodhi's @@ -586,6 +612,7 @@ class MonitoringUtils: run_command(command) self.print_user(info_log, success=True) except MonitoringException: + self.failed("waiverdb") self.print_user(info_log, success=False) def get_pr_flag( @@ -640,6 +667,8 @@ class MonitoringUtils: break self.print_user(info_log, success=success) + if not success: + self.failed("dist-git") def merge_pr(self, base_url, username, namespace, name, pr_id, token): """ Merge the specified PR @@ -659,6 +688,8 @@ class MonitoringUtils: success = True self.print_user(info_log, success=success) + if not success: + self.failed("dist-git") def finalize(self, start): """ End data returned. """ @@ -681,6 +712,7 @@ class MonitoringUtils: self.print_user(info_log, success=True) except (MonitoringException, Exception) as err: print(err) + self.failed("dist-git") self.print_user(info_log, success=False) return side_tag_name From fe94d7f288810c880cc932f0a5fa260715cf5fe0 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: May 18 2020 12:36:29 +0000 Subject: [PATCH 11/29] Include in the notifications sent on the bus the tools that failed In the messages sent over via fedora-messaging, include the tools that failed during the run (if any). Signed-off-by: Pierre-Yves Chibon --- diff --git a/runner.py b/runner.py index 08c3247..c8bd05c 100644 --- a/runner.py +++ b/runner.py @@ -92,6 +92,7 @@ def schedule(conf): "output_text": output_text, "result": result, "runid": runid, + "failed": monit_utils.failed, }, ) @@ -121,6 +122,7 @@ def schedule(conf): "output_text": output_text, "result": result, "runid": runid, + "failed": monit_utils.failed, }, ) From 4a797f53ec07d6722c77d288e6f1ff7211e1e3f3 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: May 18 2020 13:03:43 +0000 Subject: [PATCH 12/29] Make configurable the project which slows down the subsequent runs Signed-off-by: Pierre-Yves Chibon --- diff --git a/runner.cfg b/runner.cfg index 1be6a44..3879950 100644 --- a/runner.cfg +++ b/runner.cfg @@ -4,8 +4,6 @@ delay = 3600 # Time between two blocked runs in second delay_when_failing = 43200 -# blocker issue tags, issue has to have all of them -blocker_tags = ['packager_workflow_blocker', 'staging'] # CLI arguments to give to the script testing the single build gating workflow workflow_single_gating_args = "--conf monitor_gating_stg.cfg --auto-update --no-pr" @@ -20,3 +18,8 @@ workflow_multi_gating_args = "--conf monitor_gating_stg.cfg" # kb_keytab_file = "/etc/keytabs/monitor-gating-keytab" fedpkg = "fedpkg" +# Project whose issue will slow down the subsequent runs (delay defined +# above). +pagure_blocking_project = "fedora-infrastructure" +# blocker issue tags, issue has to have all of them. +blocker_tags = ['packager_workflow_blocker', 'staging'] diff --git a/runner.py b/runner.py index c8bd05c..069366a 100644 --- a/runner.py +++ b/runner.py @@ -68,8 +68,6 @@ def schedule(conf): run_command(cmd) delay = conf["delay"] - delay_when_failing = conf["delay_when_failing"] - blocker_tags = conf["blocker_tags"] print("Tests started:", datetime.datetime.utcnow(), flush=True) runid = f"{datetime.datetime.utcnow().year}-{uuid.uuid4()}" try: @@ -135,7 +133,11 @@ def schedule(conf): topic=f"multi-build.end.error", message={"runid": runid, "exception": err}, ) - blocking_issues_list = blocking_issues(blocker_tags) + delay_when_failing = conf["delay_when_failing"] + blocker_tags = conf["blocker_tags"] + blocking_project = conf["pagure_blocking_project"] + + blocking_issues_list = blocking_issues(blocking_project, blocker_tags) now = datetime.datetime.utcnow().strftime("%H:%M:%S") if blocking_issues_list: print( diff --git a/utils.py b/utils.py index 8f331ca..e63a5ed 100644 --- a/utils.py +++ b/utils.py @@ -17,25 +17,23 @@ import requests _log = logging.getLogger(__name__) -def blocking_issues(tags): +def blocking_issues(project, tags): """Lists blocking issues we track in the fedora-infrastructure project. """ if not tags: print(f"No tags to filter blocking issues by, returning empty.") return [] - api = f"https://pagure.io/api/0/fedora-infrastructure/issues" - q = f"?status=Open&tags={tags[0]}" + + api = f"https://pagure.io/api/0/{project}/issues?status=Open&tags={tags[0]}" issues = [] try: - r = requests.get(api + q) + r = requests.get(api) issues = r.json()["issues"] if tags: t = set(tags[1:]) issues = [i for i in issues if t & set(i["tags"])] for i in issues: - print( - f"Found blocking issue https://pagure.io/fedora-infrastructure/issue/{i['id']}" - ) + print(f"Found blocking issue https://pagure.io/{project}/issue/{i['id']}") except Exception as e: print(f"Error when querying pagure for blocking issues: {e}") return issues From 94c121d0c59f56643e19cb2c76425bc1d2a9a5f0 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: May 25 2020 07:48:53 +0000 Subject: [PATCH 13/29] Add support for opening a ticket on a specific project when a test fails When a test fails, it will now automatically open a ticket on a specified project hosted on pagure.io with the information it has about why the run failed. Signed-off-by: Pierre-Yves Chibon --- diff --git a/runner.cfg b/runner.cfg index 3879950..a1c8c31 100644 --- a/runner.cfg +++ b/runner.cfg @@ -18,8 +18,22 @@ workflow_multi_gating_args = "--conf monitor_gating_stg.cfg" # kb_keytab_file = "/etc/keytabs/monitor-gating-keytab" fedpkg = "fedpkg" + +# The configuration key below are used when interacting with pagure projects +# There are two ways monitor-gating interacts with them. +# a) it monitors a specific project to slows down its run in case a known issue +# prevents the workflow from working (so as to now increase the load on a +# known broken system). +# b) it reports to a specific project (but not necessarily the same) when a +# run failed to run properly end to end. + # Project whose issue will slow down the subsequent runs (delay defined # above). pagure_blocking_project = "fedora-infrastructure" # blocker issue tags, issue has to have all of them. blocker_tags = ['packager_workflow_blocker', 'staging'] + +# Project against which failed runs report their failure. +pagure_report_project = "fedora-infra/packaging_workflow_health" +pagure_api_token = "" +env = "prod" diff --git a/runner.py b/runner.py index 069366a..d8ae268 100644 --- a/runner.py +++ b/runner.py @@ -68,6 +68,10 @@ def schedule(conf): run_command(cmd) delay = conf["delay"] + report_project = conf["pagure_report_project"] + report_api_token = conf["pagure_api_token"] + report_env = conf["env"] + print("Tests started:", datetime.datetime.utcnow(), flush=True) runid = f"{datetime.datetime.utcnow().year}-{uuid.uuid4()}" try: @@ -83,6 +87,14 @@ def schedule(conf): result = "succeeded" else: result = "failed" + report_failure( + report_project, + report_api_token, + report_env, + "single-package", + monit_utils, + ) + notify( topic=f"single-build.end.{result}", message={ @@ -113,6 +125,14 @@ def schedule(conf): result = "succeeded" else: result = "failed" + report_failure( + report_project, + report_api_token, + report_env, + "multi-package", + monit_utils, + ) + notify( topic=f"multi-build.end.{result}", message={ diff --git a/utils.py b/utils.py index e63a5ed..6b0c027 100644 --- a/utils.py +++ b/utils.py @@ -17,6 +17,38 @@ import requests _log = logging.getLogger(__name__) +def report_failure(project, token, env, workflow, monit_utils): + """ Open a pagure ticket against the instance specified in the + configuration file when something does not work. + """ + url = f"https://pagure.io/api/0/{project}/new_issue" + title = f"Failure in {env} of the {workflow} packager workflow" + logs = "\n".join(monit_utils.logs) + content = f"""A run of monitor-gating has just failed in {env} for the {workflow} workflow. + +The suspects are '{", ".join(monit_utils.failed)}'. + +Full log: +```` +{logs} +```` +""" + tag = env + + data = { + "title": title, + "content": content, + "tag": tag, + } + headers = { + "Authorization": f"token {token}", + } + + req = requests.post(url, data=data, headers=headers) + if not req.ok: + print(f"Error when trying to open a ticket at: {url} to report the failure") + + def blocking_issues(project, tags): """Lists blocking issues we track in the fedora-infrastructure project. """ From 8b2b6a062d838474a437db2f803f2af1a9fbd454 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: May 25 2020 07:48:53 +0000 Subject: [PATCH 14/29] Expand the documentation for those of us that don't measure time in seconds Signed-off-by: Pierre-Yves Chibon --- diff --git a/runner.cfg b/runner.cfg index a1c8c31..9bd81a6 100644 --- a/runner.cfg +++ b/runner.cfg @@ -1,7 +1,9 @@ # Time between two runs in second +# 3600 = 1h delay = 3600 # Time between two blocked runs in second +# 43200 = 12h delay_when_failing = 43200 From 17662ad01f19bba9deb3a75398d95ffcde682b34 Mon Sep 17 00:00:00 2001 From: Nils Philippsen Date: May 25 2020 07:53:02 +0000 Subject: [PATCH 15/29] Move code into its own `monitor_gating` package Signed-off-by: Nils Philippsen --- diff --git a/clean_up_side_tags.py b/clean_up_side_tags.py deleted file mode 100644 index 982cc06..0000000 --- a/clean_up_side_tags.py +++ /dev/null @@ -1,78 +0,0 @@ -#!/usr/bin/python - -import argparse -import logging -import subprocess -import sys - -import toml - - -_log = logging.getLogger(__name__) - - -def get_cli_args(args): - - parser = argparse.ArgumentParser( - prog="clean up side-tags", - formatter_class=argparse.ArgumentDefaultsHelpFormatter, - ) - - parser.add_argument( - "conf", help="Configuration file used by the runner", - ) - - return parser.parse_args(args) - - -def run_command(command) -> bytes: - """ Run the specified command in a specific working directory if one - is specified. - """ - output = None - try: - output = subprocess.check_output(command, stderr=subprocess.PIPE) - except subprocess.CalledProcessError as e: - _log.error( - "Command `{}` return code: `{}`".format(" ".join(command), e.returncode) - ) - _log.error("stdout:\n-------\n{}".format(e.stdout)) - _log.error("stderr:\n-------\n{}".format(e.stderr)) - pass - - return output - - -def main(args): - - conf = toml.load(args.conf) - if conf.get("kb_principal") and conf.get("kb_keytab_file"): - print( - f"Logging as {conf['kb_principal']} into kerberos using: {conf['kb_keytab_file']}" - ) - cmd = ["kinit", conf["kb_principal"], "-kt", conf["kb_keytab_file"]] - run_command(cmd) - - # list side-tags: - cmd = [conf["fedpkg"], "list-side-tags", "--user", conf["kb_principal"]] - output = run_command(cmd) - - if not output: - print("No side-tags to clean found.") - return - - output = output.decode("UTF-8").strip().split("\n") - # Remove all the side-tags but the last one (which is the latest one), which - # we keep in case there is a run ongoing - for line in output[:-1]: - side_tag = line.split("\t")[0] - print("Removing side-tag: %s" % side_tag) - cmd = [conf["fedpkg"], "remove-side-tag", side_tag] - output = run_command(cmd) - - -if __name__ == "__main__": - """ Main method. """ - - args = get_cli_args(sys.argv[1:]) - main(args) diff --git a/monitor_gating/__init__.py b/monitor_gating/__init__.py new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/monitor_gating/__init__.py diff --git a/monitor_gating/clean_up_side_tags.py b/monitor_gating/clean_up_side_tags.py new file mode 100644 index 0000000..982cc06 --- /dev/null +++ b/monitor_gating/clean_up_side_tags.py @@ -0,0 +1,78 @@ +#!/usr/bin/python + +import argparse +import logging +import subprocess +import sys + +import toml + + +_log = logging.getLogger(__name__) + + +def get_cli_args(args): + + parser = argparse.ArgumentParser( + prog="clean up side-tags", + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + ) + + parser.add_argument( + "conf", help="Configuration file used by the runner", + ) + + return parser.parse_args(args) + + +def run_command(command) -> bytes: + """ Run the specified command in a specific working directory if one + is specified. + """ + output = None + try: + output = subprocess.check_output(command, stderr=subprocess.PIPE) + except subprocess.CalledProcessError as e: + _log.error( + "Command `{}` return code: `{}`".format(" ".join(command), e.returncode) + ) + _log.error("stdout:\n-------\n{}".format(e.stdout)) + _log.error("stderr:\n-------\n{}".format(e.stderr)) + pass + + return output + + +def main(args): + + conf = toml.load(args.conf) + if conf.get("kb_principal") and conf.get("kb_keytab_file"): + print( + f"Logging as {conf['kb_principal']} into kerberos using: {conf['kb_keytab_file']}" + ) + cmd = ["kinit", conf["kb_principal"], "-kt", conf["kb_keytab_file"]] + run_command(cmd) + + # list side-tags: + cmd = [conf["fedpkg"], "list-side-tags", "--user", conf["kb_principal"]] + output = run_command(cmd) + + if not output: + print("No side-tags to clean found.") + return + + output = output.decode("UTF-8").strip().split("\n") + # Remove all the side-tags but the last one (which is the latest one), which + # we keep in case there is a run ongoing + for line in output[:-1]: + side_tag = line.split("\t")[0] + print("Removing side-tag: %s" % side_tag) + cmd = [conf["fedpkg"], "remove-side-tag", side_tag] + output = run_command(cmd) + + +if __name__ == "__main__": + """ Main method. """ + + args = get_cli_args(sys.argv[1:]) + main(args) diff --git a/monitor_gating/multi_builds.py b/monitor_gating/multi_builds.py new file mode 100644 index 0000000..9cbe830 --- /dev/null +++ b/monitor_gating/multi_builds.py @@ -0,0 +1,233 @@ +#!/usr/bin/env python3 + +""" +This is a script to test how dist-git, koji, bodhi, resultsdb, fedora-ci, +greenwave and waiverdb act together and if any piece of them is failing to +do its part. + +Requirements: + - Have installed on your machine + - python3-requests + - bodhi-client >= 5.0 + - fedpkg + - fedpkg-stage + - Ensure git is configured correctly in your system (username, email...) + - You should have tests and gating setup in the package's repo you're + playing with + - Fork that repo in your name + - Make sure the repo has the f30 branch + - Ensure your ssh key is unlocked + - A valid kerberos ticket for staging and/or production + +""" + +import argparse +import datetime +import logging +import os +import sys +import tempfile + +import toml + +from .utils import MonitoringUtils + +_log = logging.getLogger(__name__) + + +def get_arguments(args): + """ Parse and return the CLI arguments. + """ + parser = argparse.ArgumentParser(description="Test the CI workflow in Fedora.") + parser.add_argument( + "--staging", + action="store_true", + default=False, + help="Changes to environment in which this runs from prod to staging", + ) + parser.add_argument( + "--no-waive", + action="store_true", + default=False, + help="Skip the end of the workflow when the failing tests are waived", + ) + parser.add_argument( + "--conf", + default="monitor_gating.cfg", + help="Configuration file to use, specifying the URLs and all", + ) + parser.add_argument( + "--conflict", + action="store_true", + default=False, + help="Make a build, conflicting in the main tag", + ) + + return parser.parse_args(args) + + +def main(args, utils=None): + """ Main method used by this script. """ + start = datetime.datetime.utcnow() + + args = get_arguments(args) + + conf = toml.load(args.conf) + if utils is None: + utils = MonitoringUtils() + + with tempfile.TemporaryDirectory(prefix="ci-test-") as folder: + print(f"Working in {folder}\n") + nevrs = {} + + # Bump the release on both packages: + nevrs, side_tag_name = utils.clone_and_bump( + folder, nevrs, conf, conf["name_multi_1"], new_side_tag=True + ) + # Store the side_tag_name so we can use it in the runner + utils.side_tag_name = side_tag_name + + nevrs, _ = utils.clone_and_bump( + folder, nevrs, conf, conf["name_multi_2"], target=side_tag_name + ) + + # Chain-build the packages + utils.chain_build_packages( + conf["fedpkg"], + packages=conf["name_multi_1"], + folder=os.path.join(folder, conf["name_multi_2"]), + target=side_tag_name, + ) + + if args.conflict: + utils.clone_to_build(folder, nevrs, conf, conf["name_multi_1"], target=None) + + # Create the update + utils.create_update( + conf["bodhi-cli"], + side_tag_name, + prod=conf["_env"] == "prod", + username=conf.get("bodhi-user"), + password=conf.get("bodhi-password"), + from_tag=True, + ) + updateid = utils.get_update_id(nevrs[list(nevrs.keys())[0]], conf["bodhi"],) + print(f" Update created : {updateid}") + + # Check the tag of the build + utils.get_build_tags( + conf.get("koji_hub"), + nevrs[conf["name_multi_1"]], + expected_ends=["signing-pending", "testing-pending"], + ) + + if not updateid: + utils.finalize(start) + return + + # Check that bodhi notified the pipeline it can run + utils.lookup_results_datagrepper( + base_url=conf["datagrepper"], + name="bodhi to CI", + topic=f"org.fedoraproject.{conf['_env']}.bodhi.update.status." + "testing.koji-build-group.build.complete", + bodhi_id=updateid, + ) + + start_dg = datetime.datetime.utcnow() + + nevr_names = [] + for name in nevrs: + nevr = nevrs[name] + nevr_names.append(nevr) + + # Check that the CI pipeline is running + utils.lookup_results_datagrepper( + base_url=conf["datagrepper"], + name="CI (running)", + topic=f"org.centos.{conf['_ci_env']}.ci.koji-build.test.running", + nevr=nevr, + start=start_dg, + ) + # Check at the CI pipeline has completed + utils.lookup_results_datagrepper( + base_url=conf["datagrepper"], + name="CI (complete)", + topic=f"org.centos.{conf['_ci_env']}.ci.koji-build.test.error", + nevr=nevr, + start=start_dg, + ) + + # Check the tag of the build + utils.get_build_tags( + conf.get("koji_hub"), nevr, expected_ends=["testing-pending"], + ) + + # Check that the CI results made it to resultsdb + utils.lookup_ci_resultsdb( + nevr=nevr, name="resultsdb(phx)", url=conf["resultsdb"] + ) + + # Check that resultsdb announced the new results + utils.lookup_results_datagrepper( + base_url=conf["datagrepper"], + name="resultsdb", + topic=f"org.fedoraproject.{conf['_env']}.resultsdb.result.new", + nevr=nevr, + start=start_dg, + ) + + # Check that greenwave reacted to resultsdb's new results + utils.lookup_results_datagrepper( + base_url=conf["datagrepper"], + name="greenwave", + topic=f"org.fedoraproject.{conf['_env']}.greenwave.decision.update", + nevr=nevr, + start=start_dg, + ) + + # Check the tag of the build -- build is blocked but should be signed + utils.get_build_tags( + conf.get("koji_hub"), nevr, expected_ends=["testing-pending"], + ) + + if not args.no_waive: + nevr = nevrs[list(nevrs.keys())[0]] + utils.waive_update( + conf["bodhi-cli"], + updateid, + prod=conf["_env"] == "prod", + username=conf.get("bodhi-user"), + password=conf.get("bodhi-password"), + ) + + # Check that waiverdb announced the new waiver + utils.lookup_results_datagrepper( + base_url=conf["datagrepper"], + name="waiverdb", + topic=f"org.fedoraproject.{conf['_env']}.waiverdb.waiver.new", + nevrs=nevr_names, + ) + + # Check that greenwave reacted to the new waiver + utils.lookup_results_datagrepper( + base_url=conf["datagrepper"], + name="greenwave", + topic=f"org.fedoraproject.{conf['_env']}.greenwave.decision.update", + nevrs=nevr_names, + ) + + # Check the tag of the build -- build was waived, let is through + utils.get_build_tags( + conf.get("koji_hub"), nevr, expected_ends=conf["koji_end_tag"], + ) + + utils.finalize(start) + return utils + + +if __name__ == "__main__": + try: + main(sys.argv[1:]) + except KeyboardInterrupt: + print(" -- Interupted --") diff --git a/monitor_gating/runner.py b/monitor_gating/runner.py new file mode 100644 index 0000000..924605d --- /dev/null +++ b/monitor_gating/runner.py @@ -0,0 +1,188 @@ +""" +This script is meant to run the different tests that we have sequentially, with +a scheduler, ie: after running all the tests, it will wait for a specified +amount of time and then run them again, until it's stopped. +""" + +import argparse +import datetime +import sched +import sys +import time +import uuid + +import fedora_messaging.api +import fedora_messaging.exceptions + +import toml + +from . import single_build +from . import multi_builds +from .utils import MonitoringUtils, blocking_issues, run_command + + +s = sched.scheduler(time.time, time.sleep) +conf = toml.load + + +def get_arguments(args): + """ Load and parse the CLI arguments.""" + parser = argparse.ArgumentParser(description="Runner for the CI canary tests.") + parser.add_argument( + "conf", help="Configuration file for the different tests", + ) + + return parser.parse_args(args) + + +def notify(topic, message): + try: + msg = fedora_messaging.api.Message( + topic="monitor-gating.{}".format(topic), body=message + ) + fedora_messaging.api.publish(msg) + except fedora_messaging.exceptions.PublishReturned as err: + print(f"Fedora Messaging broker rejected message {msg.id}: {err}") + except fedora_messaging.exceptions.ConnectionException as err: + print(f"Error sending message {msg.id}: {err}") + except Exception as err: + print(f"Error sending fedora-messaging message: {err}") + + +def _clean_up_side_tags(utils): + try: + print(" Removing side-tag: %s" % utils.side_tag_name) + cmd = [conf["fedpkg"], "remove-side-tag", utils.side_tag_name] + run_command(cmd) + except Exception: + pass + + +def schedule(conf): + """ Run the test and schedules the next one. """ + + if conf.get("kb_principal") and conf.get("kb_keytab_file"): + print(f"Logging into kerberos using: {conf['kb_keytab_file']}") + cmd = ["kinit", conf["kb_principal"], "-kt", conf["kb_keytab_file"]] + run_command(cmd) + + delay = conf["delay"] + report_project = conf["pagure_report_project"] + report_api_token = conf["pagure_api_token"] + report_env = conf["env"] + + print("Tests started:", datetime.datetime.utcnow(), flush=True) + runid = f"{datetime.datetime.utcnow().year}-{uuid.uuid4()}" + try: + # Single Build Gating + single_args = conf["workflow_single_gating_args"].split() + notify( + topic=f"single-build.start", + message={"arguments": single_args, "runid": runid}, + ) + monit_utils = single_build.main(single_args) + output_text = "\n".join(monit_utils.logs) + if "[FAILED]" not in output_text: + result = "succeeded" + else: + result = "failed" + report_failure( + report_project, + report_api_token, + report_env, + "single-package", + monit_utils, + ) + + notify( + topic=f"single-build.end.{result}", + message={ + "output": monit_utils.logs, + "output_text": output_text, + "result": result, + "runid": runid, + "failed": monit_utils.failed, + }, + ) + + # Multi Build Gating + multi_args = conf["workflow_multi_gating_args"].split() + monit_utils = MonitoringUtils() + notify( + topic=f"multi-build.start", + message={"arguments": multi_args, "runid": runid}, + ) + try: + monit_utils = multi_builds.main(multi_args, utils=monit_utils) + finally: + _clean_up_side_tags(monit_utils) + + output_text = "\n".join(monit_utils.logs) + if "[FAILED]" not in output_text: + result = "succeeded" + else: + result = "failed" + report_failure( + report_project, + report_api_token, + report_env, + "multi-package", + monit_utils, + ) + + notify( + topic=f"multi-build.end.{result}", + message={ + "output": monit_utils.logs, + "output_text": output_text, + "result": result, + "runid": runid, + "failed": monit_utils.failed, + }, + ) + + print("Tests finished:", datetime.datetime.utcnow(), flush=True) + except Exception as err: + print(f"Tests failed with: {err}", flush=True) + print(sys.exc_info()[0]) + + notify( + topic=f"multi-build.end.error", message={"runid": runid, "exception": err}, + ) + + delay_when_failing = conf["delay_when_failing"] + blocker_tags = conf["blocker_tags"] + blocking_project = conf["pagure_blocking_project"] + + blocking_issues_list = blocking_issues(blocking_project, blocker_tags) + now = datetime.datetime.utcnow().strftime("%H:%M:%S") + if blocking_issues_list: + print( + f"{now} Next run in: {delay_when_failing} seconds because of " + f"{len(blocking_issues_list)} open issues", + flush=True, + ) + s.enter(delay_when_failing, 1, schedule, argument=(conf,)) + else: + print(f"{now} Next run in: {delay} seconds", flush=True) + s.enter(delay, 1, schedule, argument=(conf,)) + + +def main(args): + """ Schedule the first test and run the scheduler. """ + args = get_arguments(args) + conf = toml.load(args.conf) + s.enter(0, 1, schedule, argument=(conf,)) + s.run() + + +if __name__ == "__main__": + try: + main(sys.argv[1:]) + except KeyboardInterrupt: + from code import InteractiveConsole + + InteractiveConsole(locals={"s": s}).interact( + "ENTERING THE DEBUG CONSOLE:\n s is the scheduler\n ^d to quit", + "LEAVING THE DEBUG CONSOLE", + ) diff --git a/monitor_gating/single_build.py b/monitor_gating/single_build.py new file mode 100644 index 0000000..d1e613c --- /dev/null +++ b/monitor_gating/single_build.py @@ -0,0 +1,313 @@ +#!/usr/bin/env python3 + +""" +This is a script to test how dist-git, koji, bodhi, resultsdb, fedora-ci, +greenwave and waiverdb act together and if any piece of them is failing to +do its part. + +Requirements: + - Have installed on your machine + - python3-requests + - bodhi-client + - fedpkg + - fedpkg-stage + - Ensure git is configured correctly in your system (username, email...) + - You should have tests and gating setup in the package's repo you're + playing with + - Fork that repo in your name + - Make sure the repo has the f30 branch + - Ensure your ssh key is unlocked + - A valid kerberos ticket for staging and/or production + +""" + +import argparse +import datetime +import logging +import os +import sys +import tempfile + +import toml + +from .utils import MonitoringUtils + +_log = logging.getLogger(__name__) + + +def get_arguments(args): + """ Parse and return the CLI arguments. + """ + parser = argparse.ArgumentParser(description="Test the CI workflow in Fedora.") + parser.add_argument( + "--nevr", help="NEVR of the build, allows by-passing: commit, push, build", + ) + parser.add_argument( + "--update", + help="Alias for the update, allows by-passing creating the udpate " "", + ) + parser.add_argument( + "--no-pr", + action="store_true", + default=False, + help="Skip testing the testing of pull-requests", + ) + parser.add_argument( + "--staging", + action="store_true", + default=False, + help="Changes to environment in which this runs from prod to staging", + ) + parser.add_argument( + "--auto-update", + action="store_true", + default=False, + help="Wait for the update to be created automatically instead of " + "doing it manually", + ) + parser.add_argument( + "--no-waive", + action="store_true", + default=False, + help="Skip the end of the workflow when the failing tests are waived", + ) + parser.add_argument( + "--conf", + default="monitor_gating.cfg", + help="Configuration file to use, specifying the URLs and all", + ) + + return parser.parse_args(args) + + +def main(args): + """ Main method used by this script. """ + start = datetime.datetime.utcnow() + + args = get_arguments(args) + + conf = toml.load(args.conf) + utils = MonitoringUtils() + + name = conf["name_single"] + namespace = conf["namespace"] + fas_username = conf["fas_username"] + branch = conf["branch"] + + with tempfile.TemporaryDirectory(prefix="ci-test-") as folder: + print(f"Working in {folder}\n") + if not args.nevr: + utils.clone_repo( + conf["fedpkg"], conf["fas_username"], namespace, name, folder=folder, + ) + gitfolder = os.path.join(folder, name) + utils.switch_branch(conf["fedpkg"], branch, folder=gitfolder) + utils.bump_release(name, folder=gitfolder) + utils.commit_changes("Bump release", folder=gitfolder) + nevr = utils.get_nevr(conf["fedpkg"], folder=gitfolder) + print(f" Upcoming build : {nevr}") + + if args.no_pr: + # Push to the main repo + utils.push_changes(gitfolder, "origin", branch) + else: + # Add the fork as remote, push to the it, open the PR, + # wait for CI to flag the PR, twice, merge the PR + utils.add_remote( + f"{fas_username}", + f"ssh://{fas_username}@{conf['distgit_host']}/" + f"forks/{fas_username}/{namespace}/{name}.git", + folder=gitfolder, + ) + utils.push_changes(gitfolder, fas_username, branch, force=True) + pr_created, pr_id, pr_uid = utils.open_pullrequest( + base_url=conf["pagure_dist_git"], + username=fas_username, + namespace=namespace, + name=name, + branch=branch, + token=conf["pagure_token"], + ) + if pr_created: + # Check that pr pipeline is running + utils.lookup_results_datagrepper( + base_url=conf["datagrepper"], + name="CI (running)", + topic=f"org.centos.{conf['_ci_env']}.ci.dist-git-pr.test.running", + rev=pr_uid, + ) + # Check that CI flag pending was set + utils.get_pr_flag( + base_url=conf["pagure_dist_git"], + username=fas_username, + namespace=namespace, + name=name, + pr_id=pr_id, + flag_username="Fedora CI", + flag_status="pending", + ) + # Check that pr pipeline has finished + utils.lookup_results_datagrepper( + base_url=conf["datagrepper"], + name="CI (complete)", + topic=f"org.centos.{conf['_ci_env']}.ci.dist-git-pr.test.error", + rev=pr_uid, + ) + # Check that CI flag failure was set + utils.get_pr_flag( + base_url=conf["pagure_dist_git"], + username=fas_username, + namespace=namespace, + name=name, + pr_id=pr_id, + flag_username="Fedora CI", + flag_status="error", + duration=25, + ) + # Merge the PR: TODO + utils.merge_pr( + base_url=conf["pagure_dist_git"], + username=fas_username, + namespace=namespace, + name=name, + pr_id=pr_id, + token=conf["pagure_token"], + ) + utils.pull_changes(gitfolder, "origin", branch) + else: + return + + # Build the package + utils.build_package(conf["fedpkg"], folder=gitfolder) + + # Check the tag of the build + utils.get_build_tags( + conf.get("koji_hub"), + nevr, + expected_ends=["updates-candidate", "signing-pending"], + ) + else: + nevr = args.nevr + + # Retrieve or create the update + updateid = utils.get_update_id(nevr, conf["bodhi"]) + if not args.update and not args.auto_update: + utils.create_update( + conf["bodhi-cli"], + nevr, + prod=conf["_env"] == "prod", + username=conf.get("bodhi-user"), + password=conf.get("bodhi-password"), + ) + print(f" Update created : {updateid}") + elif args.auto_update: + print(f" Update automatically created : {updateid}") + else: + updateid = args.update + + # Check the tag of the build + utils.get_build_tags( + conf.get("koji_hub"), + nevr, + expected_ends=["signing-pending", "testing-pending"], + ) + + if not updateid: + utils.finalize(start) + return + + # Check that bodhi notified the pipeline it can run + utils.lookup_results_datagrepper( + base_url=conf["datagrepper"], + name="bodhi to CI", + topic=f"org.fedoraproject.{conf['_env']}.bodhi.update.status." + "testing.koji-build-group.build.complete", + bodhi_id=updateid, + ) + + # Check that the CI pipeline is running + utils.lookup_results_datagrepper( + base_url=conf["datagrepper"], + name="CI (running)", + topic=f"org.centos.{conf['_ci_env']}.ci.koji-build.test.running", + nevr=nevr, + ) + # Check at the CI pipeline has completed + utils.lookup_results_datagrepper( + base_url=conf["datagrepper"], + name="CI (complete)", + topic=f"org.centos.{conf['_ci_env']}.ci.koji-build.test.error", + nevr=nevr, + duration=30, + ) + + # Check the tag of the build + utils.get_build_tags( + conf.get("koji_hub"), nevr, expected_ends=["testing-pending"], + ) + + # Check that the CI results made it to resultsdb + utils.lookup_ci_resultsdb( + nevr=nevr, name="resultsdb(phx)", url=conf["resultsdb"] + ) + + # Check that resultsdb announced the new results + utils.lookup_results_datagrepper( + base_url=conf["datagrepper"], + name="resultsdb", + topic=f"org.fedoraproject.{conf['_env']}.resultsdb.result.new", + nevr=nevr, + ) + + # Check that greenwave reacted to resultsdb's new results + utils.lookup_results_datagrepper( + base_url=conf["datagrepper"], + name="greenwave", + topic=f"org.fedoraproject.{conf['_env']}.greenwave.decision.update", + nevr=nevr, + ) + + # Check the tag of the build -- build is blocked but should be signed + utils.get_build_tags( + conf.get("koji_hub"), nevr, expected_ends=["testing-pending"], + ) + + if not args.no_waive: + utils.waive_update( + conf["bodhi-cli"], + updateid, + prod=conf["_env"] == "prod", + username=conf.get("bodhi-user"), + password=conf.get("bodhi-password"), + ) + + # Check that waiverdb announced the new waiver + utils.lookup_results_datagrepper( + base_url=conf["datagrepper"], + name="waiverdb", + topic=f"org.fedoraproject.{conf['_env']}.waiverdb.waiver.new", + nevr=nevr, + ) + + # Check that greenwave reacted to the new waiver + utils.lookup_results_datagrepper( + base_url=conf["datagrepper"], + name="greenwave", + topic=f"org.fedoraproject.{conf['_env']}.greenwave.decision.update", + nevr=nevr, + ) + + # Check the tag of the build -- build was waived, let is through + utils.get_build_tags( + conf.get("koji_hub"), nevr, expected_ends=conf["koji_end_tag"], + ) + + utils.finalize(start) + return utils + + +if __name__ == "__main__": + try: + main(sys.argv[1:]) + except KeyboardInterrupt: + print(" -- Interupted --") diff --git a/monitor_gating/utils.py b/monitor_gating/utils.py new file mode 100644 index 0000000..6b0c027 --- /dev/null +++ b/monitor_gating/utils.py @@ -0,0 +1,794 @@ +#!/usr/bin/env python3 + +""" +This is a small library of utility methods used by the monitoring scripts. + +""" + +import ast +import datetime +import logging +import os +import subprocess +import time + +import requests + +_log = logging.getLogger(__name__) + + +def report_failure(project, token, env, workflow, monit_utils): + """ Open a pagure ticket against the instance specified in the + configuration file when something does not work. + """ + url = f"https://pagure.io/api/0/{project}/new_issue" + title = f"Failure in {env} of the {workflow} packager workflow" + logs = "\n".join(monit_utils.logs) + content = f"""A run of monitor-gating has just failed in {env} for the {workflow} workflow. + +The suspects are '{", ".join(monit_utils.failed)}'. + +Full log: +```` +{logs} +```` +""" + tag = env + + data = { + "title": title, + "content": content, + "tag": tag, + } + headers = { + "Authorization": f"token {token}", + } + + req = requests.post(url, data=data, headers=headers) + if not req.ok: + print(f"Error when trying to open a ticket at: {url} to report the failure") + + +def blocking_issues(project, tags): + """Lists blocking issues we track in the fedora-infrastructure project. + """ + if not tags: + print(f"No tags to filter blocking issues by, returning empty.") + return [] + + api = f"https://pagure.io/api/0/{project}/issues?status=Open&tags={tags[0]}" + issues = [] + try: + r = requests.get(api) + issues = r.json()["issues"] + if tags: + t = set(tags[1:]) + issues = [i for i in issues if t & set(i["tags"])] + for i in issues: + print(f"Found blocking issue https://pagure.io/{project}/issue/{i['id']}") + except Exception as e: + print(f"Error when querying pagure for blocking issues: {e}") + return issues + + +class MonitoringException(Exception): + """The base class for all exceptions raised by this script.""" + + +class MonitoringUtils: + def __init__(self): + """ Instanciate the object. """ + self.logs = [] + self.failed = [] + + def print_user(self, content, success=None): + """ Prints the specified content to the user. + """ + spaces = 90 + if success is not None: + end = None + if success: + content = "{} {}".format(content.ljust(spaces), "[DONE]") + else: + content = "{} {}".format(content.ljust(spaces), "[FAILED]") + else: + if os.environ.get("OPENSHIFT"): + end = None + else: + end = "\r" + + now = datetime.datetime.utcnow() + time = now.strftime("%H:%M:%S") + self.logs.append(f"{time} - {content}") + print(f"{time} - {content}", end=end, flush=True) + + def clone_repo(self, command, username, namespace, name, folder): + """ Clone the specified git repo into the specified folder. + """ + info_log = f"Cloning as {username} the git repo: {namespace}/{name}" + self.print_user(info_log) + try: + run_command( + [command, "--user", username, "clone", f"{namespace}/{name}"], + cwd=folder, + ) + except MonitoringException: + self.failed.append("git/dist-git") + self.print_user(info_log, success=False) + else: + try: + # Assume these commands can't fail + clone_folder = os.path.join(folder, name) + run_command( + ["git", "config", "user.name", "packagerbot"], cwd=clone_folder, + ) + run_command( + ["git", "config", "user.email", "admin@fedoraproject.org"], + cwd=clone_folder, + ) + self.print_user(info_log, success=True) + except MonitoringException: + self.failed.append("git") + self.print_user(info_log, success=False) + + def add_remote(self, name, url, folder): + """ Add the specified remote to the git repo in the folder with the + specified url. + """ + info_log = f"Adding remote: {name}" + self.print_user(info_log) + try: + run_command(["git", "remote", "add", name, url], cwd=folder) + self.print_user(info_log, success=True) + except MonitoringException: + self.failed.append("git") + self.print_user(info_log, success=False) + + def switch_branch(self, command, name, folder): + """ Switch to the specified git branch in the specified git repo. + """ + info_log = f"Switching to branch: {name}" + self.print_user(info_log) + try: + run_command([command, "switch-branch", f"{name}"], cwd=folder) + self.print_user(info_log, success=True) + except MonitoringException: + self.failed("fedpkg") + self.print_user(info_log, success=False) + + def bump_release(self, name, folder): + """ Bump the release of the spec file the specified git repo. + """ + info_log = f"Bumping release of: {name}.spec" + self.print_user(info_log) + try: + run_command(["rpmdev-bumpspec", f"{name}.spec"], cwd=folder) + self.print_user(info_log, success=True) + except MonitoringException: + self.failed("rpmdev-bumspec") + self.print_user(info_log, success=False) + + def commit_changes(self, commit_log, folder): + """ Commit all the changes made to *tracked* files in the git repo + with the specified commit log. + """ + info_log = f"Commiting changes" + self.print_user(info_log) + try: + run_command(["git", "commit", "-asm", commit_log], cwd=folder) + self.print_user(info_log, success=True) + except MonitoringException: + self.failed("git") + self.print_user(info_log, success=False) + + def push_changes(self, folder, target, branch, force=False): + """ Push all changes using git. + """ + info_log = f"Pushing changes" + self.print_user(info_log) + try: + cmd = ["git", "push", target, branch] + if force: + cmd.append("-f") + run_command(cmd, cwd=folder) + self.print_user(info_log, success=True) + except MonitoringException: + self.failed("git/dist-git") + self.print_user(info_log, success=False) + + def pull_changes(self, folder, target, branch): + """ Pull all changes using git. + """ + info_log = f"Pulling changes" + self.print_user(info_log) + try: + cmd = ["git", "pull", "--rebase", target, branch] + run_command(cmd, cwd=folder) + self.print_user(info_log, success=True) + except MonitoringException: + self.failed("git/dist-git") + self.print_user(info_log, success=False) + + def open_pullrequest(self, base_url, username, namespace, name, branch, token): + """ Open a pull-request from the user's fork to the main project for + the specified branch. + """ + info_log = f"Creating PR from forks/{username}/{namespace}/{name}" + self.print_user(info_log) + url = "/".join( + [base_url.rstrip("/"), "api/0", namespace, name, "pull-request/new"] + ) + data = { + "branch_to": branch, + "branch_from": branch, + "repo_from": name, + "repo_from_username": username, + "repo_from_namespace": namespace, + "initial_comment": "Testing PR", + "title": "Test PR for monitoring", + } + headers = {"Authorization": f"token {token}"} + req = requests.post(url=url, data=data, headers=headers) + if not req.ok: + print(req.text) + success = False + pr_id = None + pr_uid = None + self.failed("dist-git") + else: + output = req.json() + pr_id = str(output["id"]) + pr_uid = output["uid"] + url = "/".join( + [base_url.rstrip("/"), namespace, name, "pull-request", pr_id] + ) + info_log = f"PR created {url}" + success = True + self.print_user(info_log, success=success) + return (success, pr_id, pr_uid) + + def get_nevr(self, command, folder): + """ Get the name-epoch-version-release presently in git + """ + info_log = f"Getting nevr" + self.print_user(info_log) + try: + nevr = run_command([command, "verrel"], cwd=folder) + self.print_user(info_log, success=True) + return nevr.strip().decode("utf-8") + except MonitoringException: + self.failed("fedpkg") + self.print_user(info_log, success=False) + + def build_package(self, command, folder, target=None): + """ Build the package in the current branch + """ + info_log = f"Building the package" + self.print_user(info_log) + command = [command, "build"] + if target: + command.extend(["--target", target]) + try: + run_command(command, cwd=folder) + self.print_user(info_log, success=True) + except MonitoringException: + self.failed("koji") + self.print_user(info_log, success=False) + + def chain_build_packages(self, command, packages, folder, target=None): + """ Chain-build the packages in the current branch + """ + if not isinstance(packages, list): + packages = [packages] + info_log = ( + f"Chain-building the packages: {packages + [os.path.basename(folder)]}" + ) + self.print_user(info_log) + command = [command, "chain-build"] + command.extend(packages) + if target: + command.extend(["--target", target]) + try: + run_command(command, cwd=folder) + self.print_user(info_log, success=True) + except MonitoringException: + self.failed("koji") + self.print_user(info_log, success=False) + + def get_build_tags(self, koji_url, nevr, expected_ends): + """ List the tags associated with the specified build. + """ + # return + start = datetime.datetime.utcnow() + info_log = f"Retrieving koji tags" + self.print_user(info_log) + command = [ + "koji", + ] + if koji_url: + command.extend(["-s", koji_url]) + command.extend(["call", "listTags", nevr]) + + success = False + tags = None + broke = False + while True: + try: + output = run_command(command) + output = output.decode("utf-8") + try: + data = ast.literal_eval(output.strip()) + except Exception: + print("Could not decode JSON in:") + print(command) + print(output) + broke = True + break + tags = [tag.get("name") for tag in data] + for tag_name in tags: + for expectation in expected_ends: + if tag_name.endswith(expectation): + success = True + broke = True + break + if success: + broke = True + break + if broke: + break + + if (datetime.datetime.utcnow() - start).seconds > (15 * 60): + success = False + info_log = f"Update for {nevr} not created within 15 minutes" + break + + # Only query koji every 30 seconds + time.sleep(30) + except MonitoringException: + success = False + break + + info_log = f"Retrieving koji tags: {tags}" + if not success: + self.failed("koji") + self.print_user(info_log, success=success) + + def create_update( + self, command, item, prod=True, username=None, password=None, from_tag=False, + ): + """ Create the update for the package built. + """ + info_log = f"Creating a bodhi update" + self.print_user(info_log) + command = [ + command, + "updates", + "new", + "--notes", + "Bump release to test CI", + "--type", + "bugfix", + "--autotime", + ] + if from_tag: + command.append("--from-tag") + command.append(item) + + if not prod: + command.append("--staging") + if username: + command.extend(["--user", username]) + if password: + command.extend(["--password", password]) + try: + run_command(command) + self.print_user(info_log, success=True) + except MonitoringException: + self.failed("bodhi") + self.print_user(info_log, success=False) + + def get_update_id(self, nevr, url): + """ Retrieve the update identifier from bodhi for the given nevr. """ + start = datetime.datetime.utcnow() + info_log = f"Retrieving update created" + self.print_user(info_log) + url = f"{url}/updates/?builds={nevr}" + updateid = None + success = True + while True: + req = requests.get(url) + data = req.json() + if data["updates"]: + updateid = data["updates"][0]["updateid"] + if updateid: + break + + if (datetime.datetime.utcnow() - start).seconds > (15 * 60): + success = False + self.failed("bodhi") + info_log = f"Update for {nevr} not created within 15 minutes" + break + + # Only query bodhi every 30 seconds + time.sleep(30) + + self.print_user(info_log, success=success) + return updateid + + def lookup_results_datagrepper( + self, + base_url, + name, + topic, + nevr=None, + nevrs=None, + rev=None, + bodhi_id=None, + start=None, + duration=15, + ): + """ Check the CI results in datagrepper for results about our specified + build. + """ + start_lookup = datetime.datetime.utcnow() + if start is None: + start = start_lookup + info_log = f"Checking datagrepper for {name} messages" + self.print_user(info_log) + # Start pulling messages 10 minutes before now + start_time = start - datetime.timedelta(minutes=10) + # Limiting the number of row per page to 10 allows for quicker results + url = ( + base_url + f"?topic={topic}" + f"&start={start_time.timestamp()}&row_per_page=10" + ) + + success = None + returned_status = None + info_log = None + nevrs = nevrs or [] + while True: + # We're assuming here that there won't be more than 100 messages for + # that topic coming in between the one we're interested in and when we + # are looking for it (10*10 == 100) + for page in range(1, 11): + end_url = url + end_url += f"&page={page}" + data = requests.get(end_url).json() + if "raw_messages" not in data: + nomsg_log = f"No messages in data-grepper on {end_url} " + if "error" in data: + nomsg_log += data["error"] + self.print_user(nomsg_log) + break + + for message in data["raw_messages"]: + + # Old message format from the CI pipeline + if "ci.pipeline" in message["topic"] and ( + message["msg"]["nvr"] == nevr + or message["msg"]["nvr"] in nevrs + or message["msg"]["rev"] == rev + ): + success = True + returned_status = message["msg"]["status"] + break + + # New message format from the CI pipeline for koji builds + if ( + "ci.koji-build" in message["topic"] + and message["msg"]["artifact"]["nvr"] == nevr + ): + if message["topic"].endswith("test.complete"): + success = True + returned_status = message["msg"]["test"]["result"] + elif message["topic"].endswith("test.error"): + success = True + returned_status = "error" + elif message["topic"].endswith("test.running"): + success = True + returned_status = "running" + break + + # New message format from the CI pipeline for dist-git PR + if ( + "ci.dist-git-pr" in message["topic"] + and message["msg"]["artifact"]["type"] == "pull-request" + and message["msg"]["artifact"]["uid"] == rev + ): + if message["topic"].endswith("test.complete"): + success = True + returned_status = message["msg"]["test"]["result"] + elif message["topic"].endswith("test.error"): + success = True + returned_status = "error" + elif message["topic"].endswith("test.running"): + success = True + returned_status = "running" + break + + # resultsdb messages + if ( + "resultsdb" in message["topic"] + and "nvr" in message["msg"]["data"] + and ( + nevr in message["msg"]["data"]["nvr"] + or message["msg"]["data"]["nvr"] in nevrs + ) + ): + success = True + returned_status = message["msg"]["outcome"] + break + + # greenwave messages + if "greenwave" in message["topic"] and ( + message["msg"]["subject_identifier"] == nevr + or message["msg"]["subject_identifier"] in nevrs + ): + success = True + returned_status = message["msg"]["policies_satisfied"] + break + + # waiverdb messages + if "waiverdb" in message["topic"] and ( + message["msg"]["subject_identifier"] == nevr + or message["msg"]["subject_identifier"] in nevrs + ): + success = True + returned_status = "" + break + + # bodhi messages + if "bodhi.update.status.testing" in message["topic"] and message[ + "msg" + ]["artifact"]["id"].startswith(bodhi_id): + success = True + returned_status = "" + break + if success is not None: + break + if success is not None: + break + + if (datetime.datetime.utcnow() - start).seconds > (duration * 60): + success = False + info_log = f"{name} results not found in datagrepper" + break + + # Only query datagrepper every 30 seconds + time.sleep(30) + + if info_log is None: + info_log = f"{name} results in datagrepper returned {returned_status}" + + end = datetime.datetime.utcnow() + info_log += f" - ran for: {(end - start_lookup).seconds}s" + self.print_user(info_log, success=success) + if not success: + self.failed("datagrepper") + + def lookup_ci_resultsdb(self, nevr, name, url): + """ Check the CI results in the specified resultsdb for results about + our specified build. + """ + start = datetime.datetime.utcnow() + info_log = f"Checking {name} for CI results " + self.print_user(info_log) + topic = "org.centos.prod.ci.pipeline.allpackages-build.complete" + if ".stg" in url: + topic = "org.centos.stage.ci.pipeline.allpackages-build.complete" + url = f"{url}?testcases={topic}" + + success = False + returned_status = None + info_log = None + while True: + # Assume we won't have more than 3 pages of results coming in b/w + # our checks + for page in [0, 1, 2]: + end_url = url + end_url += f"&page={page}" + data = requests.get(end_url).json() + for result in data["data"]: + if nevr in result["data"]["nvr"]: + success = True + returned_status = result["data"]["status"][0] + break + if success: + break + if success: + break + + if (datetime.datetime.utcnow() - start).seconds > (15 * 60): + success = False + info_log = ( + f"CI results did not show in {name} for {nevr} within 15 minutes" + ) + break + + # Only query datagrepper every 30 seconds + time.sleep(30) + + if info_log is None: + info_log = f"CI results in {name} returned {returned_status}" + + end = datetime.datetime.utcnow() + info_log += f" - ran for: {(end - start).seconds}s" + self.print_user(info_log, success=success) + if not success: + self.failed("resultsdb") + + def waive_update(self, command, updateid, prod=True, username=None, password=None): + """ Waive all the tests results for the specified update using bodhi's + CLI. + """ + info_log = f"Waiving test results for bodhi update" + self.print_user(info_log) + command = [ + command, + "updates", + "waive", + updateid, + "'This is fine, we are testing the workflow'", + "--debug", + ] + if not prod: + command.append("--staging") + if username: + command.extend(["--user", username]) + if password: + command.extend(["--password", password]) + try: + run_command(command) + self.print_user(info_log, success=True) + except MonitoringException: + self.failed("waiverdb") + self.print_user(info_log, success=False) + + def get_pr_flag( + self, + base_url, + username, + namespace, + name, + pr_id, + flag_username, + flag_status, + duration=10, + ): + """ Retrieve the flags of the PR and assert the last one from the + specified flag_username has the given status. + """ + pr = "/".join([namespace, name, "pull-request", pr_id]) + info_log = f"Retreiving flags for PR: {pr}" + self.print_user(info_log) + url = "/".join([base_url.rstrip("/"), "api/0", pr, "flag"]) + + start = datetime.datetime.utcnow() + success = False + + while True: + try: + req = requests.get(url=url) + except requests.exceptions.ConnectionError: + continue + + if req.ok: + break + + if (datetime.datetime.utcnow() - start).seconds > (duration * 60): + success = False + info_log = f"Failed to retrieve flags for PR: {pr}" + break + + # Only query pagure every 30 seconds + time.sleep(30) + + if not req.ok: + print(req.text) + self.logs.append(f"Error retrieving PR flags: {req.text}") + raise MonitoringException("Error retrieving PR flags") + else: + output = req.json() + for flag in output["flags"]: + if flag["username"] == flag_username: + info_log = f"Retreived flag {flag['status']} on PR" + success = flag["status"] == flag_status + break + + self.print_user(info_log, success=success) + if not success: + self.failed("dist-git") + + def merge_pr(self, base_url, username, namespace, name, pr_id, token): + """ Merge the specified PR + """ + pr = "/".join([namespace, name, "pull-request", pr_id]) + info_log = f"Merge PR: {pr}" + self.print_user(info_log) + url = "/".join([base_url.rstrip("/"), "api/0", pr, "merge"]) + headers = {"Authorization": f"token {token}"} + req = requests.post(url=url, data={"wait": True}, headers=headers) + success = False + if not req.ok: + print(req.text) + self.logs(f"Error Merging flags: {req.text}") + raise MonitoringException("Error merging flags") + else: + success = True + + self.print_user(info_log, success=success) + if not success: + self.failed("dist-git") + + def finalize(self, start): + """ End data returned. """ + end = datetime.datetime.utcnow() + delta = (end - start).seconds + self.logs.append(f"Ran for {delta} seconds ({delta/60:.2f} minutes)") + print(f"Ran for {delta} seconds ({delta/60:.2f} minutes)") + + def create_side_tag(self, command, folder): + """ Create a side-tag to build packages in it. """ + info_log = f"Creating the side-tag" + self.print_user(info_log) + command = [command, "request-side-tag"] + side_tag_name = None + try: + output = run_command(command, cwd=folder) + lines = output.decode("utf-8").split("\n") + _, side_tag_name, _ = lines[0].split("'") + info_log = f"Created side-tag {side_tag_name}" + self.print_user(info_log, success=True) + except (MonitoringException, Exception) as err: + print(err) + self.failed("dist-git") + self.print_user(info_log, success=False) + return side_tag_name + + def clone_and_bump( + self, folder, nevrs, conf, name, target=None, new_side_tag=False + ): + """Clone the repo, bump the release, commit and push.""" + namespace = conf["namespace"] + branch = conf["branch"] + + self.clone_repo( + conf["fedpkg"], conf["fas_username"], namespace, name, folder=folder, + ) + gitfolder = os.path.join(folder, name) + self.switch_branch(conf["fedpkg"], branch, folder=gitfolder) + side_tag_name = None + # Create a side-tag + if new_side_tag: + side_tag_name = self.create_side_tag(conf["fedpkg"], folder=gitfolder) + target = side_tag_name + self.bump_release(name, folder=gitfolder) + self.commit_changes("Bump release", folder=gitfolder) + nevr = self.get_nevr(conf["fedpkg"], folder=gitfolder) + nevrs[name] = nevr + # Push to the main repo + self.push_changes(gitfolder, "origin", branch) + print(f" Upcoming build : {nevr}") + return (nevrs, target) + + +def run_command(command, cwd=None): + """ Run the specified command in a specific working directory if one + is specified. + """ + output = None + try: + output = subprocess.check_output(command, cwd=cwd, stderr=subprocess.PIPE) + except subprocess.CalledProcessError as e: + if "--password" in command: + idx = command.index("--password") + command[idx + 1] = "" + _log.error( + "Command `{}` return code: `{}`".format(" ".join(command), e.returncode) + ) + _log.error("stdout:\n-------\n{}".format(e.stdout)) + _log.error("stderr:\n-------\n{}".format(e.stderr)) + raise MonitoringException("Command failed to run") + + return output diff --git a/monitor_gating_multi_builds.py b/monitor_gating_multi_builds.py deleted file mode 100644 index 18648d3..0000000 --- a/monitor_gating_multi_builds.py +++ /dev/null @@ -1,233 +0,0 @@ -#!/usr/bin/env python3 - -""" -This is a script to test how dist-git, koji, bodhi, resultsdb, fedora-ci, -greenwave and waiverdb act together and if any piece of them is failing to -do its part. - -Requirements: - - Have installed on your machine - - python3-requests - - bodhi-client >= 5.0 - - fedpkg - - fedpkg-stage - - Ensure git is configured correctly in your system (username, email...) - - You should have tests and gating setup in the package's repo you're - playing with - - Fork that repo in your name - - Make sure the repo has the f30 branch - - Ensure your ssh key is unlocked - - A valid kerberos ticket for staging and/or production - -""" - -import argparse -import datetime -import logging -import os -import sys -import tempfile - -import toml - -from utils import MonitoringUtils - -_log = logging.getLogger(__name__) - - -def get_arguments(args): - """ Parse and return the CLI arguments. - """ - parser = argparse.ArgumentParser(description="Test the CI workflow in Fedora.") - parser.add_argument( - "--staging", - action="store_true", - default=False, - help="Changes to environment in which this runs from prod to staging", - ) - parser.add_argument( - "--no-waive", - action="store_true", - default=False, - help="Skip the end of the workflow when the failing tests are waived", - ) - parser.add_argument( - "--conf", - default="monitor_gating.cfg", - help="Configuration file to use, specifying the URLs and all", - ) - parser.add_argument( - "--conflict", - action="store_true", - default=False, - help="Make a build, conflicting in the main tag", - ) - - return parser.parse_args(args) - - -def main(args, utils=None): - """ Main method used by this script. """ - start = datetime.datetime.utcnow() - - args = get_arguments(args) - - conf = toml.load(args.conf) - if utils is None: - utils = MonitoringUtils() - - with tempfile.TemporaryDirectory(prefix="ci-test-") as folder: - print(f"Working in {folder}\n") - nevrs = {} - - # Bump the release on both packages: - nevrs, side_tag_name = utils.clone_and_bump( - folder, nevrs, conf, conf["name_multi_1"], new_side_tag=True - ) - # Store the side_tag_name so we can use it in the runner - utils.side_tag_name = side_tag_name - - nevrs, _ = utils.clone_and_bump( - folder, nevrs, conf, conf["name_multi_2"], target=side_tag_name - ) - - # Chain-build the packages - utils.chain_build_packages( - conf["fedpkg"], - packages=conf["name_multi_1"], - folder=os.path.join(folder, conf["name_multi_2"]), - target=side_tag_name, - ) - - if args.conflict: - utils.clone_to_build(folder, nevrs, conf, conf["name_multi_1"], target=None) - - # Create the update - utils.create_update( - conf["bodhi-cli"], - side_tag_name, - prod=conf["_env"] == "prod", - username=conf.get("bodhi-user"), - password=conf.get("bodhi-password"), - from_tag=True, - ) - updateid = utils.get_update_id(nevrs[list(nevrs.keys())[0]], conf["bodhi"],) - print(f" Update created : {updateid}") - - # Check the tag of the build - utils.get_build_tags( - conf.get("koji_hub"), - nevrs[conf["name_multi_1"]], - expected_ends=["signing-pending", "testing-pending"], - ) - - if not updateid: - utils.finalize(start) - return - - # Check that bodhi notified the pipeline it can run - utils.lookup_results_datagrepper( - base_url=conf["datagrepper"], - name="bodhi to CI", - topic=f"org.fedoraproject.{conf['_env']}.bodhi.update.status." - "testing.koji-build-group.build.complete", - bodhi_id=updateid, - ) - - start_dg = datetime.datetime.utcnow() - - nevr_names = [] - for name in nevrs: - nevr = nevrs[name] - nevr_names.append(nevr) - - # Check that the CI pipeline is running - utils.lookup_results_datagrepper( - base_url=conf["datagrepper"], - name="CI (running)", - topic=f"org.centos.{conf['_ci_env']}.ci.koji-build.test.running", - nevr=nevr, - start=start_dg, - ) - # Check at the CI pipeline has completed - utils.lookup_results_datagrepper( - base_url=conf["datagrepper"], - name="CI (complete)", - topic=f"org.centos.{conf['_ci_env']}.ci.koji-build.test.error", - nevr=nevr, - start=start_dg, - ) - - # Check the tag of the build - utils.get_build_tags( - conf.get("koji_hub"), nevr, expected_ends=["testing-pending"], - ) - - # Check that the CI results made it to resultsdb - utils.lookup_ci_resultsdb( - nevr=nevr, name="resultsdb(phx)", url=conf["resultsdb"] - ) - - # Check that resultsdb announced the new results - utils.lookup_results_datagrepper( - base_url=conf["datagrepper"], - name="resultsdb", - topic=f"org.fedoraproject.{conf['_env']}.resultsdb.result.new", - nevr=nevr, - start=start_dg, - ) - - # Check that greenwave reacted to resultsdb's new results - utils.lookup_results_datagrepper( - base_url=conf["datagrepper"], - name="greenwave", - topic=f"org.fedoraproject.{conf['_env']}.greenwave.decision.update", - nevr=nevr, - start=start_dg, - ) - - # Check the tag of the build -- build is blocked but should be signed - utils.get_build_tags( - conf.get("koji_hub"), nevr, expected_ends=["testing-pending"], - ) - - if not args.no_waive: - nevr = nevrs[list(nevrs.keys())[0]] - utils.waive_update( - conf["bodhi-cli"], - updateid, - prod=conf["_env"] == "prod", - username=conf.get("bodhi-user"), - password=conf.get("bodhi-password"), - ) - - # Check that waiverdb announced the new waiver - utils.lookup_results_datagrepper( - base_url=conf["datagrepper"], - name="waiverdb", - topic=f"org.fedoraproject.{conf['_env']}.waiverdb.waiver.new", - nevrs=nevr_names, - ) - - # Check that greenwave reacted to the new waiver - utils.lookup_results_datagrepper( - base_url=conf["datagrepper"], - name="greenwave", - topic=f"org.fedoraproject.{conf['_env']}.greenwave.decision.update", - nevrs=nevr_names, - ) - - # Check the tag of the build -- build was waived, let is through - utils.get_build_tags( - conf.get("koji_hub"), nevr, expected_ends=conf["koji_end_tag"], - ) - - utils.finalize(start) - return utils - - -if __name__ == "__main__": - try: - main(sys.argv[1:]) - except KeyboardInterrupt: - print(" -- Interupted --") diff --git a/monitor_gating_single_build.py b/monitor_gating_single_build.py deleted file mode 100644 index 29425b0..0000000 --- a/monitor_gating_single_build.py +++ /dev/null @@ -1,313 +0,0 @@ -#!/usr/bin/env python3 - -""" -This is a script to test how dist-git, koji, bodhi, resultsdb, fedora-ci, -greenwave and waiverdb act together and if any piece of them is failing to -do its part. - -Requirements: - - Have installed on your machine - - python3-requests - - bodhi-client - - fedpkg - - fedpkg-stage - - Ensure git is configured correctly in your system (username, email...) - - You should have tests and gating setup in the package's repo you're - playing with - - Fork that repo in your name - - Make sure the repo has the f30 branch - - Ensure your ssh key is unlocked - - A valid kerberos ticket for staging and/or production - -""" - -import argparse -import datetime -import logging -import os -import sys -import tempfile - -import toml - -from utils import MonitoringUtils - -_log = logging.getLogger(__name__) - - -def get_arguments(args): - """ Parse and return the CLI arguments. - """ - parser = argparse.ArgumentParser(description="Test the CI workflow in Fedora.") - parser.add_argument( - "--nevr", help="NEVR of the build, allows by-passing: commit, push, build", - ) - parser.add_argument( - "--update", - help="Alias for the update, allows by-passing creating the udpate " "", - ) - parser.add_argument( - "--no-pr", - action="store_true", - default=False, - help="Skip testing the testing of pull-requests", - ) - parser.add_argument( - "--staging", - action="store_true", - default=False, - help="Changes to environment in which this runs from prod to staging", - ) - parser.add_argument( - "--auto-update", - action="store_true", - default=False, - help="Wait for the update to be created automatically instead of " - "doing it manually", - ) - parser.add_argument( - "--no-waive", - action="store_true", - default=False, - help="Skip the end of the workflow when the failing tests are waived", - ) - parser.add_argument( - "--conf", - default="monitor_gating.cfg", - help="Configuration file to use, specifying the URLs and all", - ) - - return parser.parse_args(args) - - -def main(args): - """ Main method used by this script. """ - start = datetime.datetime.utcnow() - - args = get_arguments(args) - - conf = toml.load(args.conf) - utils = MonitoringUtils() - - name = conf["name_single"] - namespace = conf["namespace"] - fas_username = conf["fas_username"] - branch = conf["branch"] - - with tempfile.TemporaryDirectory(prefix="ci-test-") as folder: - print(f"Working in {folder}\n") - if not args.nevr: - utils.clone_repo( - conf["fedpkg"], conf["fas_username"], namespace, name, folder=folder, - ) - gitfolder = os.path.join(folder, name) - utils.switch_branch(conf["fedpkg"], branch, folder=gitfolder) - utils.bump_release(name, folder=gitfolder) - utils.commit_changes("Bump release", folder=gitfolder) - nevr = utils.get_nevr(conf["fedpkg"], folder=gitfolder) - print(f" Upcoming build : {nevr}") - - if args.no_pr: - # Push to the main repo - utils.push_changes(gitfolder, "origin", branch) - else: - # Add the fork as remote, push to the it, open the PR, - # wait for CI to flag the PR, twice, merge the PR - utils.add_remote( - f"{fas_username}", - f"ssh://{fas_username}@{conf['distgit_host']}/" - f"forks/{fas_username}/{namespace}/{name}.git", - folder=gitfolder, - ) - utils.push_changes(gitfolder, fas_username, branch, force=True) - pr_created, pr_id, pr_uid = utils.open_pullrequest( - base_url=conf["pagure_dist_git"], - username=fas_username, - namespace=namespace, - name=name, - branch=branch, - token=conf["pagure_token"], - ) - if pr_created: - # Check that pr pipeline is running - utils.lookup_results_datagrepper( - base_url=conf["datagrepper"], - name="CI (running)", - topic=f"org.centos.{conf['_ci_env']}.ci.dist-git-pr.test.running", - rev=pr_uid, - ) - # Check that CI flag pending was set - utils.get_pr_flag( - base_url=conf["pagure_dist_git"], - username=fas_username, - namespace=namespace, - name=name, - pr_id=pr_id, - flag_username="Fedora CI", - flag_status="pending", - ) - # Check that pr pipeline has finished - utils.lookup_results_datagrepper( - base_url=conf["datagrepper"], - name="CI (complete)", - topic=f"org.centos.{conf['_ci_env']}.ci.dist-git-pr.test.error", - rev=pr_uid, - ) - # Check that CI flag failure was set - utils.get_pr_flag( - base_url=conf["pagure_dist_git"], - username=fas_username, - namespace=namespace, - name=name, - pr_id=pr_id, - flag_username="Fedora CI", - flag_status="error", - duration=25, - ) - # Merge the PR: TODO - utils.merge_pr( - base_url=conf["pagure_dist_git"], - username=fas_username, - namespace=namespace, - name=name, - pr_id=pr_id, - token=conf["pagure_token"], - ) - utils.pull_changes(gitfolder, "origin", branch) - else: - return - - # Build the package - utils.build_package(conf["fedpkg"], folder=gitfolder) - - # Check the tag of the build - utils.get_build_tags( - conf.get("koji_hub"), - nevr, - expected_ends=["updates-candidate", "signing-pending"], - ) - else: - nevr = args.nevr - - # Retrieve or create the update - updateid = utils.get_update_id(nevr, conf["bodhi"]) - if not args.update and not args.auto_update: - utils.create_update( - conf["bodhi-cli"], - nevr, - prod=conf["_env"] == "prod", - username=conf.get("bodhi-user"), - password=conf.get("bodhi-password"), - ) - print(f" Update created : {updateid}") - elif args.auto_update: - print(f" Update automatically created : {updateid}") - else: - updateid = args.update - - # Check the tag of the build - utils.get_build_tags( - conf.get("koji_hub"), - nevr, - expected_ends=["signing-pending", "testing-pending"], - ) - - if not updateid: - utils.finalize(start) - return - - # Check that bodhi notified the pipeline it can run - utils.lookup_results_datagrepper( - base_url=conf["datagrepper"], - name="bodhi to CI", - topic=f"org.fedoraproject.{conf['_env']}.bodhi.update.status." - "testing.koji-build-group.build.complete", - bodhi_id=updateid, - ) - - # Check that the CI pipeline is running - utils.lookup_results_datagrepper( - base_url=conf["datagrepper"], - name="CI (running)", - topic=f"org.centos.{conf['_ci_env']}.ci.koji-build.test.running", - nevr=nevr, - ) - # Check at the CI pipeline has completed - utils.lookup_results_datagrepper( - base_url=conf["datagrepper"], - name="CI (complete)", - topic=f"org.centos.{conf['_ci_env']}.ci.koji-build.test.error", - nevr=nevr, - duration=30, - ) - - # Check the tag of the build - utils.get_build_tags( - conf.get("koji_hub"), nevr, expected_ends=["testing-pending"], - ) - - # Check that the CI results made it to resultsdb - utils.lookup_ci_resultsdb( - nevr=nevr, name="resultsdb(phx)", url=conf["resultsdb"] - ) - - # Check that resultsdb announced the new results - utils.lookup_results_datagrepper( - base_url=conf["datagrepper"], - name="resultsdb", - topic=f"org.fedoraproject.{conf['_env']}.resultsdb.result.new", - nevr=nevr, - ) - - # Check that greenwave reacted to resultsdb's new results - utils.lookup_results_datagrepper( - base_url=conf["datagrepper"], - name="greenwave", - topic=f"org.fedoraproject.{conf['_env']}.greenwave.decision.update", - nevr=nevr, - ) - - # Check the tag of the build -- build is blocked but should be signed - utils.get_build_tags( - conf.get("koji_hub"), nevr, expected_ends=["testing-pending"], - ) - - if not args.no_waive: - utils.waive_update( - conf["bodhi-cli"], - updateid, - prod=conf["_env"] == "prod", - username=conf.get("bodhi-user"), - password=conf.get("bodhi-password"), - ) - - # Check that waiverdb announced the new waiver - utils.lookup_results_datagrepper( - base_url=conf["datagrepper"], - name="waiverdb", - topic=f"org.fedoraproject.{conf['_env']}.waiverdb.waiver.new", - nevr=nevr, - ) - - # Check that greenwave reacted to the new waiver - utils.lookup_results_datagrepper( - base_url=conf["datagrepper"], - name="greenwave", - topic=f"org.fedoraproject.{conf['_env']}.greenwave.decision.update", - nevr=nevr, - ) - - # Check the tag of the build -- build was waived, let is through - utils.get_build_tags( - conf.get("koji_hub"), nevr, expected_ends=conf["koji_end_tag"], - ) - - utils.finalize(start) - return utils - - -if __name__ == "__main__": - try: - main(sys.argv[1:]) - except KeyboardInterrupt: - print(" -- Interupted --") diff --git a/runner.py b/runner.py deleted file mode 100644 index d8ae268..0000000 --- a/runner.py +++ /dev/null @@ -1,191 +0,0 @@ -""" -This script is meant to run the different tests that we have sequentially, with -a scheduler, ie: after running all the tests, it will wait for a specified -amount of time and then run them again, until it's stopped. -""" - -import argparse -import datetime -import sched -import sys -import time -import uuid - -import fedora_messaging.api -import fedora_messaging.exceptions - -import monitor_gating_multi_builds - -import monitor_gating_single_build - -import toml - -from utils import MonitoringUtils, blocking_issues, run_command - -s = sched.scheduler(time.time, time.sleep) -conf = toml.load - - -def get_arguments(args): - """ Load and parse the CLI arguments.""" - parser = argparse.ArgumentParser(description="Runner for the CI canary tests.") - parser.add_argument( - "conf", help="Configuration file for the different tests", - ) - - return parser.parse_args(args) - - -def notify(topic, message): - try: - msg = fedora_messaging.api.Message( - topic="monitor-gating.{}".format(topic), body=message - ) - fedora_messaging.api.publish(msg) - except fedora_messaging.exceptions.PublishReturned as err: - print(f"Fedora Messaging broker rejected message {msg.id}: {err}") - except fedora_messaging.exceptions.ConnectionException as err: - print(f"Error sending message {msg.id}: {err}") - except Exception as err: - print(f"Error sending fedora-messaging message: {err}") - - -def _clean_up_side_tags(utils): - try: - print(" Removing side-tag: %s" % utils.side_tag_name) - cmd = [conf["fedpkg"], "remove-side-tag", utils.side_tag_name] - run_command(cmd) - except Exception: - pass - - -def schedule(conf): - """ Run the test and schedules the next one. """ - - if conf.get("kb_principal") and conf.get("kb_keytab_file"): - print(f"Logging into kerberos using: {conf['kb_keytab_file']}") - cmd = ["kinit", conf["kb_principal"], "-kt", conf["kb_keytab_file"]] - run_command(cmd) - - delay = conf["delay"] - report_project = conf["pagure_report_project"] - report_api_token = conf["pagure_api_token"] - report_env = conf["env"] - - print("Tests started:", datetime.datetime.utcnow(), flush=True) - runid = f"{datetime.datetime.utcnow().year}-{uuid.uuid4()}" - try: - # Single Build Gating - single_args = conf["workflow_single_gating_args"].split() - notify( - topic=f"single-build.start", - message={"arguments": single_args, "runid": runid}, - ) - monit_utils = monitor_gating_single_build.main(single_args) - output_text = "\n".join(monit_utils.logs) - if "[FAILED]" not in output_text: - result = "succeeded" - else: - result = "failed" - report_failure( - report_project, - report_api_token, - report_env, - "single-package", - monit_utils, - ) - - notify( - topic=f"single-build.end.{result}", - message={ - "output": monit_utils.logs, - "output_text": output_text, - "result": result, - "runid": runid, - "failed": monit_utils.failed, - }, - ) - - # Multi Build Gating - multi_args = conf["workflow_multi_gating_args"].split() - monit_utils = MonitoringUtils() - notify( - topic=f"multi-build.start", - message={"arguments": multi_args, "runid": runid}, - ) - try: - monit_utils = monitor_gating_multi_builds.main( - multi_args, utils=monit_utils - ) - finally: - _clean_up_side_tags(monit_utils) - - output_text = "\n".join(monit_utils.logs) - if "[FAILED]" not in output_text: - result = "succeeded" - else: - result = "failed" - report_failure( - report_project, - report_api_token, - report_env, - "multi-package", - monit_utils, - ) - - notify( - topic=f"multi-build.end.{result}", - message={ - "output": monit_utils.logs, - "output_text": output_text, - "result": result, - "runid": runid, - "failed": monit_utils.failed, - }, - ) - - print("Tests finished:", datetime.datetime.utcnow(), flush=True) - except Exception as err: - print(f"Tests failed with: {err}", flush=True) - print(sys.exc_info()[0]) - - notify( - topic=f"multi-build.end.error", message={"runid": runid, "exception": err}, - ) - - delay_when_failing = conf["delay_when_failing"] - blocker_tags = conf["blocker_tags"] - blocking_project = conf["pagure_blocking_project"] - - blocking_issues_list = blocking_issues(blocking_project, blocker_tags) - now = datetime.datetime.utcnow().strftime("%H:%M:%S") - if blocking_issues_list: - print( - f"{now} Next run in: {delay_when_failing} seconds because of " - f"{len(blocking_issues_list)} open issues", - flush=True, - ) - s.enter(delay_when_failing, 1, schedule, argument=(conf,)) - else: - print(f"{now} Next run in: {delay} seconds", flush=True) - s.enter(delay, 1, schedule, argument=(conf,)) - - -def main(args): - """ Schedule the first test and run the scheduler. """ - args = get_arguments(args) - conf = toml.load(args.conf) - s.enter(0, 1, schedule, argument=(conf,)) - s.run() - - -if __name__ == "__main__": - try: - main(sys.argv[1:]) - except KeyboardInterrupt: - from code import InteractiveConsole - - InteractiveConsole(locals={"s": s}).interact( - "ENTERING THE DEBUG CONSOLE:\n s is the scheduler\n ^d to quit", - "LEAVING THE DEBUG CONSOLE", - ) diff --git a/utils.py b/utils.py deleted file mode 100644 index 6b0c027..0000000 --- a/utils.py +++ /dev/null @@ -1,794 +0,0 @@ -#!/usr/bin/env python3 - -""" -This is a small library of utility methods used by the monitoring scripts. - -""" - -import ast -import datetime -import logging -import os -import subprocess -import time - -import requests - -_log = logging.getLogger(__name__) - - -def report_failure(project, token, env, workflow, monit_utils): - """ Open a pagure ticket against the instance specified in the - configuration file when something does not work. - """ - url = f"https://pagure.io/api/0/{project}/new_issue" - title = f"Failure in {env} of the {workflow} packager workflow" - logs = "\n".join(monit_utils.logs) - content = f"""A run of monitor-gating has just failed in {env} for the {workflow} workflow. - -The suspects are '{", ".join(monit_utils.failed)}'. - -Full log: -```` -{logs} -```` -""" - tag = env - - data = { - "title": title, - "content": content, - "tag": tag, - } - headers = { - "Authorization": f"token {token}", - } - - req = requests.post(url, data=data, headers=headers) - if not req.ok: - print(f"Error when trying to open a ticket at: {url} to report the failure") - - -def blocking_issues(project, tags): - """Lists blocking issues we track in the fedora-infrastructure project. - """ - if not tags: - print(f"No tags to filter blocking issues by, returning empty.") - return [] - - api = f"https://pagure.io/api/0/{project}/issues?status=Open&tags={tags[0]}" - issues = [] - try: - r = requests.get(api) - issues = r.json()["issues"] - if tags: - t = set(tags[1:]) - issues = [i for i in issues if t & set(i["tags"])] - for i in issues: - print(f"Found blocking issue https://pagure.io/{project}/issue/{i['id']}") - except Exception as e: - print(f"Error when querying pagure for blocking issues: {e}") - return issues - - -class MonitoringException(Exception): - """The base class for all exceptions raised by this script.""" - - -class MonitoringUtils: - def __init__(self): - """ Instanciate the object. """ - self.logs = [] - self.failed = [] - - def print_user(self, content, success=None): - """ Prints the specified content to the user. - """ - spaces = 90 - if success is not None: - end = None - if success: - content = "{} {}".format(content.ljust(spaces), "[DONE]") - else: - content = "{} {}".format(content.ljust(spaces), "[FAILED]") - else: - if os.environ.get("OPENSHIFT"): - end = None - else: - end = "\r" - - now = datetime.datetime.utcnow() - time = now.strftime("%H:%M:%S") - self.logs.append(f"{time} - {content}") - print(f"{time} - {content}", end=end, flush=True) - - def clone_repo(self, command, username, namespace, name, folder): - """ Clone the specified git repo into the specified folder. - """ - info_log = f"Cloning as {username} the git repo: {namespace}/{name}" - self.print_user(info_log) - try: - run_command( - [command, "--user", username, "clone", f"{namespace}/{name}"], - cwd=folder, - ) - except MonitoringException: - self.failed.append("git/dist-git") - self.print_user(info_log, success=False) - else: - try: - # Assume these commands can't fail - clone_folder = os.path.join(folder, name) - run_command( - ["git", "config", "user.name", "packagerbot"], cwd=clone_folder, - ) - run_command( - ["git", "config", "user.email", "admin@fedoraproject.org"], - cwd=clone_folder, - ) - self.print_user(info_log, success=True) - except MonitoringException: - self.failed.append("git") - self.print_user(info_log, success=False) - - def add_remote(self, name, url, folder): - """ Add the specified remote to the git repo in the folder with the - specified url. - """ - info_log = f"Adding remote: {name}" - self.print_user(info_log) - try: - run_command(["git", "remote", "add", name, url], cwd=folder) - self.print_user(info_log, success=True) - except MonitoringException: - self.failed.append("git") - self.print_user(info_log, success=False) - - def switch_branch(self, command, name, folder): - """ Switch to the specified git branch in the specified git repo. - """ - info_log = f"Switching to branch: {name}" - self.print_user(info_log) - try: - run_command([command, "switch-branch", f"{name}"], cwd=folder) - self.print_user(info_log, success=True) - except MonitoringException: - self.failed("fedpkg") - self.print_user(info_log, success=False) - - def bump_release(self, name, folder): - """ Bump the release of the spec file the specified git repo. - """ - info_log = f"Bumping release of: {name}.spec" - self.print_user(info_log) - try: - run_command(["rpmdev-bumpspec", f"{name}.spec"], cwd=folder) - self.print_user(info_log, success=True) - except MonitoringException: - self.failed("rpmdev-bumspec") - self.print_user(info_log, success=False) - - def commit_changes(self, commit_log, folder): - """ Commit all the changes made to *tracked* files in the git repo - with the specified commit log. - """ - info_log = f"Commiting changes" - self.print_user(info_log) - try: - run_command(["git", "commit", "-asm", commit_log], cwd=folder) - self.print_user(info_log, success=True) - except MonitoringException: - self.failed("git") - self.print_user(info_log, success=False) - - def push_changes(self, folder, target, branch, force=False): - """ Push all changes using git. - """ - info_log = f"Pushing changes" - self.print_user(info_log) - try: - cmd = ["git", "push", target, branch] - if force: - cmd.append("-f") - run_command(cmd, cwd=folder) - self.print_user(info_log, success=True) - except MonitoringException: - self.failed("git/dist-git") - self.print_user(info_log, success=False) - - def pull_changes(self, folder, target, branch): - """ Pull all changes using git. - """ - info_log = f"Pulling changes" - self.print_user(info_log) - try: - cmd = ["git", "pull", "--rebase", target, branch] - run_command(cmd, cwd=folder) - self.print_user(info_log, success=True) - except MonitoringException: - self.failed("git/dist-git") - self.print_user(info_log, success=False) - - def open_pullrequest(self, base_url, username, namespace, name, branch, token): - """ Open a pull-request from the user's fork to the main project for - the specified branch. - """ - info_log = f"Creating PR from forks/{username}/{namespace}/{name}" - self.print_user(info_log) - url = "/".join( - [base_url.rstrip("/"), "api/0", namespace, name, "pull-request/new"] - ) - data = { - "branch_to": branch, - "branch_from": branch, - "repo_from": name, - "repo_from_username": username, - "repo_from_namespace": namespace, - "initial_comment": "Testing PR", - "title": "Test PR for monitoring", - } - headers = {"Authorization": f"token {token}"} - req = requests.post(url=url, data=data, headers=headers) - if not req.ok: - print(req.text) - success = False - pr_id = None - pr_uid = None - self.failed("dist-git") - else: - output = req.json() - pr_id = str(output["id"]) - pr_uid = output["uid"] - url = "/".join( - [base_url.rstrip("/"), namespace, name, "pull-request", pr_id] - ) - info_log = f"PR created {url}" - success = True - self.print_user(info_log, success=success) - return (success, pr_id, pr_uid) - - def get_nevr(self, command, folder): - """ Get the name-epoch-version-release presently in git - """ - info_log = f"Getting nevr" - self.print_user(info_log) - try: - nevr = run_command([command, "verrel"], cwd=folder) - self.print_user(info_log, success=True) - return nevr.strip().decode("utf-8") - except MonitoringException: - self.failed("fedpkg") - self.print_user(info_log, success=False) - - def build_package(self, command, folder, target=None): - """ Build the package in the current branch - """ - info_log = f"Building the package" - self.print_user(info_log) - command = [command, "build"] - if target: - command.extend(["--target", target]) - try: - run_command(command, cwd=folder) - self.print_user(info_log, success=True) - except MonitoringException: - self.failed("koji") - self.print_user(info_log, success=False) - - def chain_build_packages(self, command, packages, folder, target=None): - """ Chain-build the packages in the current branch - """ - if not isinstance(packages, list): - packages = [packages] - info_log = ( - f"Chain-building the packages: {packages + [os.path.basename(folder)]}" - ) - self.print_user(info_log) - command = [command, "chain-build"] - command.extend(packages) - if target: - command.extend(["--target", target]) - try: - run_command(command, cwd=folder) - self.print_user(info_log, success=True) - except MonitoringException: - self.failed("koji") - self.print_user(info_log, success=False) - - def get_build_tags(self, koji_url, nevr, expected_ends): - """ List the tags associated with the specified build. - """ - # return - start = datetime.datetime.utcnow() - info_log = f"Retrieving koji tags" - self.print_user(info_log) - command = [ - "koji", - ] - if koji_url: - command.extend(["-s", koji_url]) - command.extend(["call", "listTags", nevr]) - - success = False - tags = None - broke = False - while True: - try: - output = run_command(command) - output = output.decode("utf-8") - try: - data = ast.literal_eval(output.strip()) - except Exception: - print("Could not decode JSON in:") - print(command) - print(output) - broke = True - break - tags = [tag.get("name") for tag in data] - for tag_name in tags: - for expectation in expected_ends: - if tag_name.endswith(expectation): - success = True - broke = True - break - if success: - broke = True - break - if broke: - break - - if (datetime.datetime.utcnow() - start).seconds > (15 * 60): - success = False - info_log = f"Update for {nevr} not created within 15 minutes" - break - - # Only query koji every 30 seconds - time.sleep(30) - except MonitoringException: - success = False - break - - info_log = f"Retrieving koji tags: {tags}" - if not success: - self.failed("koji") - self.print_user(info_log, success=success) - - def create_update( - self, command, item, prod=True, username=None, password=None, from_tag=False, - ): - """ Create the update for the package built. - """ - info_log = f"Creating a bodhi update" - self.print_user(info_log) - command = [ - command, - "updates", - "new", - "--notes", - "Bump release to test CI", - "--type", - "bugfix", - "--autotime", - ] - if from_tag: - command.append("--from-tag") - command.append(item) - - if not prod: - command.append("--staging") - if username: - command.extend(["--user", username]) - if password: - command.extend(["--password", password]) - try: - run_command(command) - self.print_user(info_log, success=True) - except MonitoringException: - self.failed("bodhi") - self.print_user(info_log, success=False) - - def get_update_id(self, nevr, url): - """ Retrieve the update identifier from bodhi for the given nevr. """ - start = datetime.datetime.utcnow() - info_log = f"Retrieving update created" - self.print_user(info_log) - url = f"{url}/updates/?builds={nevr}" - updateid = None - success = True - while True: - req = requests.get(url) - data = req.json() - if data["updates"]: - updateid = data["updates"][0]["updateid"] - if updateid: - break - - if (datetime.datetime.utcnow() - start).seconds > (15 * 60): - success = False - self.failed("bodhi") - info_log = f"Update for {nevr} not created within 15 minutes" - break - - # Only query bodhi every 30 seconds - time.sleep(30) - - self.print_user(info_log, success=success) - return updateid - - def lookup_results_datagrepper( - self, - base_url, - name, - topic, - nevr=None, - nevrs=None, - rev=None, - bodhi_id=None, - start=None, - duration=15, - ): - """ Check the CI results in datagrepper for results about our specified - build. - """ - start_lookup = datetime.datetime.utcnow() - if start is None: - start = start_lookup - info_log = f"Checking datagrepper for {name} messages" - self.print_user(info_log) - # Start pulling messages 10 minutes before now - start_time = start - datetime.timedelta(minutes=10) - # Limiting the number of row per page to 10 allows for quicker results - url = ( - base_url + f"?topic={topic}" - f"&start={start_time.timestamp()}&row_per_page=10" - ) - - success = None - returned_status = None - info_log = None - nevrs = nevrs or [] - while True: - # We're assuming here that there won't be more than 100 messages for - # that topic coming in between the one we're interested in and when we - # are looking for it (10*10 == 100) - for page in range(1, 11): - end_url = url - end_url += f"&page={page}" - data = requests.get(end_url).json() - if "raw_messages" not in data: - nomsg_log = f"No messages in data-grepper on {end_url} " - if "error" in data: - nomsg_log += data["error"] - self.print_user(nomsg_log) - break - - for message in data["raw_messages"]: - - # Old message format from the CI pipeline - if "ci.pipeline" in message["topic"] and ( - message["msg"]["nvr"] == nevr - or message["msg"]["nvr"] in nevrs - or message["msg"]["rev"] == rev - ): - success = True - returned_status = message["msg"]["status"] - break - - # New message format from the CI pipeline for koji builds - if ( - "ci.koji-build" in message["topic"] - and message["msg"]["artifact"]["nvr"] == nevr - ): - if message["topic"].endswith("test.complete"): - success = True - returned_status = message["msg"]["test"]["result"] - elif message["topic"].endswith("test.error"): - success = True - returned_status = "error" - elif message["topic"].endswith("test.running"): - success = True - returned_status = "running" - break - - # New message format from the CI pipeline for dist-git PR - if ( - "ci.dist-git-pr" in message["topic"] - and message["msg"]["artifact"]["type"] == "pull-request" - and message["msg"]["artifact"]["uid"] == rev - ): - if message["topic"].endswith("test.complete"): - success = True - returned_status = message["msg"]["test"]["result"] - elif message["topic"].endswith("test.error"): - success = True - returned_status = "error" - elif message["topic"].endswith("test.running"): - success = True - returned_status = "running" - break - - # resultsdb messages - if ( - "resultsdb" in message["topic"] - and "nvr" in message["msg"]["data"] - and ( - nevr in message["msg"]["data"]["nvr"] - or message["msg"]["data"]["nvr"] in nevrs - ) - ): - success = True - returned_status = message["msg"]["outcome"] - break - - # greenwave messages - if "greenwave" in message["topic"] and ( - message["msg"]["subject_identifier"] == nevr - or message["msg"]["subject_identifier"] in nevrs - ): - success = True - returned_status = message["msg"]["policies_satisfied"] - break - - # waiverdb messages - if "waiverdb" in message["topic"] and ( - message["msg"]["subject_identifier"] == nevr - or message["msg"]["subject_identifier"] in nevrs - ): - success = True - returned_status = "" - break - - # bodhi messages - if "bodhi.update.status.testing" in message["topic"] and message[ - "msg" - ]["artifact"]["id"].startswith(bodhi_id): - success = True - returned_status = "" - break - if success is not None: - break - if success is not None: - break - - if (datetime.datetime.utcnow() - start).seconds > (duration * 60): - success = False - info_log = f"{name} results not found in datagrepper" - break - - # Only query datagrepper every 30 seconds - time.sleep(30) - - if info_log is None: - info_log = f"{name} results in datagrepper returned {returned_status}" - - end = datetime.datetime.utcnow() - info_log += f" - ran for: {(end - start_lookup).seconds}s" - self.print_user(info_log, success=success) - if not success: - self.failed("datagrepper") - - def lookup_ci_resultsdb(self, nevr, name, url): - """ Check the CI results in the specified resultsdb for results about - our specified build. - """ - start = datetime.datetime.utcnow() - info_log = f"Checking {name} for CI results " - self.print_user(info_log) - topic = "org.centos.prod.ci.pipeline.allpackages-build.complete" - if ".stg" in url: - topic = "org.centos.stage.ci.pipeline.allpackages-build.complete" - url = f"{url}?testcases={topic}" - - success = False - returned_status = None - info_log = None - while True: - # Assume we won't have more than 3 pages of results coming in b/w - # our checks - for page in [0, 1, 2]: - end_url = url - end_url += f"&page={page}" - data = requests.get(end_url).json() - for result in data["data"]: - if nevr in result["data"]["nvr"]: - success = True - returned_status = result["data"]["status"][0] - break - if success: - break - if success: - break - - if (datetime.datetime.utcnow() - start).seconds > (15 * 60): - success = False - info_log = ( - f"CI results did not show in {name} for {nevr} within 15 minutes" - ) - break - - # Only query datagrepper every 30 seconds - time.sleep(30) - - if info_log is None: - info_log = f"CI results in {name} returned {returned_status}" - - end = datetime.datetime.utcnow() - info_log += f" - ran for: {(end - start).seconds}s" - self.print_user(info_log, success=success) - if not success: - self.failed("resultsdb") - - def waive_update(self, command, updateid, prod=True, username=None, password=None): - """ Waive all the tests results for the specified update using bodhi's - CLI. - """ - info_log = f"Waiving test results for bodhi update" - self.print_user(info_log) - command = [ - command, - "updates", - "waive", - updateid, - "'This is fine, we are testing the workflow'", - "--debug", - ] - if not prod: - command.append("--staging") - if username: - command.extend(["--user", username]) - if password: - command.extend(["--password", password]) - try: - run_command(command) - self.print_user(info_log, success=True) - except MonitoringException: - self.failed("waiverdb") - self.print_user(info_log, success=False) - - def get_pr_flag( - self, - base_url, - username, - namespace, - name, - pr_id, - flag_username, - flag_status, - duration=10, - ): - """ Retrieve the flags of the PR and assert the last one from the - specified flag_username has the given status. - """ - pr = "/".join([namespace, name, "pull-request", pr_id]) - info_log = f"Retreiving flags for PR: {pr}" - self.print_user(info_log) - url = "/".join([base_url.rstrip("/"), "api/0", pr, "flag"]) - - start = datetime.datetime.utcnow() - success = False - - while True: - try: - req = requests.get(url=url) - except requests.exceptions.ConnectionError: - continue - - if req.ok: - break - - if (datetime.datetime.utcnow() - start).seconds > (duration * 60): - success = False - info_log = f"Failed to retrieve flags for PR: {pr}" - break - - # Only query pagure every 30 seconds - time.sleep(30) - - if not req.ok: - print(req.text) - self.logs.append(f"Error retrieving PR flags: {req.text}") - raise MonitoringException("Error retrieving PR flags") - else: - output = req.json() - for flag in output["flags"]: - if flag["username"] == flag_username: - info_log = f"Retreived flag {flag['status']} on PR" - success = flag["status"] == flag_status - break - - self.print_user(info_log, success=success) - if not success: - self.failed("dist-git") - - def merge_pr(self, base_url, username, namespace, name, pr_id, token): - """ Merge the specified PR - """ - pr = "/".join([namespace, name, "pull-request", pr_id]) - info_log = f"Merge PR: {pr}" - self.print_user(info_log) - url = "/".join([base_url.rstrip("/"), "api/0", pr, "merge"]) - headers = {"Authorization": f"token {token}"} - req = requests.post(url=url, data={"wait": True}, headers=headers) - success = False - if not req.ok: - print(req.text) - self.logs(f"Error Merging flags: {req.text}") - raise MonitoringException("Error merging flags") - else: - success = True - - self.print_user(info_log, success=success) - if not success: - self.failed("dist-git") - - def finalize(self, start): - """ End data returned. """ - end = datetime.datetime.utcnow() - delta = (end - start).seconds - self.logs.append(f"Ran for {delta} seconds ({delta/60:.2f} minutes)") - print(f"Ran for {delta} seconds ({delta/60:.2f} minutes)") - - def create_side_tag(self, command, folder): - """ Create a side-tag to build packages in it. """ - info_log = f"Creating the side-tag" - self.print_user(info_log) - command = [command, "request-side-tag"] - side_tag_name = None - try: - output = run_command(command, cwd=folder) - lines = output.decode("utf-8").split("\n") - _, side_tag_name, _ = lines[0].split("'") - info_log = f"Created side-tag {side_tag_name}" - self.print_user(info_log, success=True) - except (MonitoringException, Exception) as err: - print(err) - self.failed("dist-git") - self.print_user(info_log, success=False) - return side_tag_name - - def clone_and_bump( - self, folder, nevrs, conf, name, target=None, new_side_tag=False - ): - """Clone the repo, bump the release, commit and push.""" - namespace = conf["namespace"] - branch = conf["branch"] - - self.clone_repo( - conf["fedpkg"], conf["fas_username"], namespace, name, folder=folder, - ) - gitfolder = os.path.join(folder, name) - self.switch_branch(conf["fedpkg"], branch, folder=gitfolder) - side_tag_name = None - # Create a side-tag - if new_side_tag: - side_tag_name = self.create_side_tag(conf["fedpkg"], folder=gitfolder) - target = side_tag_name - self.bump_release(name, folder=gitfolder) - self.commit_changes("Bump release", folder=gitfolder) - nevr = self.get_nevr(conf["fedpkg"], folder=gitfolder) - nevrs[name] = nevr - # Push to the main repo - self.push_changes(gitfolder, "origin", branch) - print(f" Upcoming build : {nevr}") - return (nevrs, target) - - -def run_command(command, cwd=None): - """ Run the specified command in a specific working directory if one - is specified. - """ - output = None - try: - output = subprocess.check_output(command, cwd=cwd, stderr=subprocess.PIPE) - except subprocess.CalledProcessError as e: - if "--password" in command: - idx = command.index("--password") - command[idx + 1] = "" - _log.error( - "Command `{}` return code: `{}`".format(" ".join(command), e.returncode) - ) - _log.error("stdout:\n-------\n{}".format(e.stdout)) - _log.error("stderr:\n-------\n{}".format(e.stderr)) - raise MonitoringException("Command failed to run") - - return output From 0a81be65d4b6974d1ead45e405566287c84127f4 Mon Sep 17 00:00:00 2001 From: Nils Philippsen Date: May 25 2020 07:53:02 +0000 Subject: [PATCH 16/29] Make container entrypoint script more flexible Previously, the container entrypoint (which is hard to override) invariably called the Python script. Now the entrypoint script will run anything passed as arguments to it, and the container will default to running the monitor-gating script. This is to let users e.g. run a shell to inspect things more easily. Signed-off-by: Nils Philippsen --- diff --git a/Dockerfile b/Dockerfile index 34f29c4..61b17ec 100644 --- a/Dockerfile +++ b/Dockerfile @@ -22,4 +22,5 @@ RUN echo "packagerbot" > /.fedora.upn \ USER 1000 WORKDIR / -ENTRYPOINT ["sh", "/opt/code/entrypoint.sh"] +ENTRYPOINT ["/opt/code/entrypoint.sh"] +CMD ["python3", "/opt/code/runner.py", "/opt/config/runner.cfg"] diff --git a/entrypoint.sh b/entrypoint.sh index 5c2d603..28a65af 100755 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -17,4 +17,4 @@ echo "Running in production" ssh-keyscan pkgs.fedoraproject.org >> /.ssh/known_hosts fi -python3 /opt/code/runner.py /opt/config/runner.cfg +exec "$@" From f7a4628135b87803a5a0892c6101dfd1dcab7a9c Mon Sep 17 00:00:00 2001 From: Nils Philippsen Date: May 25 2020 07:53:02 +0000 Subject: [PATCH 17/29] Dockerfile: install default configuration file Signed-off-by: Nils Philippsen --- diff --git a/Dockerfile b/Dockerfile index 61b17ec..1cadbb3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -9,9 +9,12 @@ RUN dnf -y install python3-requests bodhi-client fedpkg fedpkg-stage \ COPY . /opt/code -RUN echo "packagerbot" > /.fedora.upn \ +RUN cd /opt/code && mkdir /opt/config && cp runner.cfg /opt/config \ + && echo "packagerbot" > /.fedora.upn \ && chgrp -R 0 /opt/code \ && chmod -R g=u /opt/code \ + && chgrp -R 0 /opt/config \ + && chmod -R g=u /opt/config \ && chgrp -R 0 /.ssh \ && chmod -R g=u /.ssh \ && chgrp -R 0 /.fedora \ From 5d921d3add473b28fcc96da51f7e786472899f7f Mon Sep 17 00:00:00 2001 From: Nils Philippsen Date: May 25 2020 07:53:02 +0000 Subject: [PATCH 18/29] Add and use setup.py, requirements.txt In the course, designate the team as maintainers of the container image, too. Signed-off-by: Nils Philippsen --- diff --git a/Dockerfile b/Dockerfile index 1cadbb3..c63ef54 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,7 @@ # This Dockerfile is used to build and run monitor-gating on Openshift FROM fedora:31 -LABEL maintainer "Pierre-Yves Chibon " +LABEL maintainer "Fedora Infrastructure Team " RUN dnf -y install python3-requests bodhi-client fedpkg fedpkg-stage \ python3-toml git python3-koji python3-fedora-messaging cracklib-dicts \ @@ -9,7 +9,7 @@ RUN dnf -y install python3-requests bodhi-client fedpkg fedpkg-stage \ COPY . /opt/code -RUN cd /opt/code && mkdir /opt/config && cp runner.cfg /opt/config \ +RUN cd /opt/code && python3 setup.py install && mkdir /opt/config && cp runner.cfg /opt/config \ && echo "packagerbot" > /.fedora.upn \ && chgrp -R 0 /opt/code \ && chmod -R g=u /opt/code \ @@ -26,4 +26,4 @@ RUN cd /opt/code && mkdir /opt/config && cp runner.cfg /opt/config \ USER 1000 WORKDIR / ENTRYPOINT ["/opt/code/entrypoint.sh"] -CMD ["python3", "/opt/code/runner.py", "/opt/config/runner.cfg"] +CMD ["monitor-gating", "/opt/config/runner.cfg"] diff --git a/monitor_gating/clean_up_side_tags.py b/monitor_gating/clean_up_side_tags.py index 982cc06..9c3e994 100644 --- a/monitor_gating/clean_up_side_tags.py +++ b/monitor_gating/clean_up_side_tags.py @@ -43,7 +43,9 @@ def run_command(command) -> bytes: return output -def main(args): +def main(): + """ Main method. """ + args = get_cli_args(sys.argv[1:]) conf = toml.load(args.conf) if conf.get("kb_principal") and conf.get("kb_keytab_file"): @@ -72,7 +74,4 @@ def main(args): if __name__ == "__main__": - """ Main method. """ - - args = get_cli_args(sys.argv[1:]) - main(args) + main() diff --git a/monitor_gating/runner.py b/monitor_gating/runner.py index 924605d..413b94a 100644 --- a/monitor_gating/runner.py +++ b/monitor_gating/runner.py @@ -16,8 +16,8 @@ import fedora_messaging.exceptions import toml -from . import single_build from . import multi_builds +from . import single_build from .utils import MonitoringUtils, blocking_issues, run_command @@ -168,17 +168,13 @@ def schedule(conf): s.enter(delay, 1, schedule, argument=(conf,)) -def main(args): - """ Schedule the first test and run the scheduler. """ - args = get_arguments(args) - conf = toml.load(args.conf) - s.enter(0, 1, schedule, argument=(conf,)) - s.run() - - -if __name__ == "__main__": +def main(): + """ Main method. """ try: - main(sys.argv[1:]) + args = get_arguments(sys.argv[1:]) + conf = toml.load(args.conf) + s.enter(0, 1, schedule, argument=(conf,)) + s.run() except KeyboardInterrupt: from code import InteractiveConsole @@ -186,3 +182,7 @@ if __name__ == "__main__": "ENTERING THE DEBUG CONSOLE:\n s is the scheduler\n ^d to quit", "LEAVING THE DEBUG CONSOLE", ) + + +if __name__ == "__main__": + main() diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..778e4a6 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,3 @@ +fedora_messaging +requests +toml diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..7638255 --- /dev/null +++ b/setup.py @@ -0,0 +1,66 @@ +import os + +from setuptools import setup + + +def get_requirements(): + requirements = [] + with open("requirements.txt", "r") as f: + for line in f: + line = line.strip() + before, _, after = line.partition("#") + # Allow source control references for development. + if before.startswith("git+"): + if not after: + # The name is in the fragment + continue + _, _, requirement = after.rpartition("=") + requirement, _, _ = requirement.partition("#") + else: + requirement = before + + requirement = requirement.strip() + + if requirement: + requirements.append(requirement) + return requirements + + +here = os.path.abspath(os.path.dirname(__file__)) +with open(os.path.join(here, "README.rst"), "r") as f: + README = f.read() + + +setup( + name="monitor-gating", + version="0.0.1", + # Possible options are at https://pypi.python.org/pypi?%3Aaction=list_classifiers + classifiers=[ + "Development Status :: 5 - Alpha", + "Intended Audience :: Developers", + "Intended Audience :: System Administrators", + # 'License :: OSI Approved :: GNU General Public License v2 or later (GPLv2+)', + "Operating System :: POSIX :: Linux", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.6", + "Programming Language :: Python :: 3.7", + "Topic :: System :: Software Distribution", + ], + # license=LICENSE, + maintainer="Fedora Infrastructure Team", + maintainer_email="infrastructure@lists.fedoraproject.org", + platforms=["Fedora", "GNU/Linux"], + url="https://pagure.io/fedora-ci/monitor-gating", + description="Monitor the health of gating in Fedora", + long_description=README, + keywords="fedora", + packages=["monitor_gating"], + include_package_data=True, + zip_safe=False, + install_requires=get_requirements(), + entry_points=""" + [console_scripts] + monitor-gating = monitor_gating.runner:main + monitor-gating-clean-up-side-tags = monitor_gating.clean_up_side_tags:main + """, +) From 66a3b0c2ae753e138715451fa0a724cba005e1fb Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Jun 12 2020 16:41:02 +0000 Subject: [PATCH 19/29] Add some debugging for the keytab file Signed-off-by: Pierre-Yves Chibon --- diff --git a/entrypoint.sh b/entrypoint.sh index 28a65af..a699fc3 100755 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -6,6 +6,9 @@ if ! whoami &> /dev/null; then fi fi + +klist -A -k /etc/keytabs/monitor-gating-keytab + ln -s /opt/ssh/id_rsa /.ssh/id_rsa || true if [ -z ${PRODUCTION+x} ]; then # Staging info From 801fdee76de19fa0a771dc2263ddfd6a04fe0400 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Aug 18 2020 11:48:33 +0000 Subject: [PATCH 20/29] Add more debugging Signed-off-by: Pierre-Yves Chibon --- diff --git a/monitor_gating/runner.py b/monitor_gating/runner.py index 413b94a..9e6cfee 100644 --- a/monitor_gating/runner.py +++ b/monitor_gating/runner.py @@ -45,8 +45,12 @@ def notify(topic, message): print(f"Fedora Messaging broker rejected message {msg.id}: {err}") except fedora_messaging.exceptions.ConnectionException as err: print(f"Error sending message {msg.id}: {err}") + print(f"topic: {topic}") + print(f"message: {message}") except Exception as err: print(f"Error sending fedora-messaging message: {err}") + print(f"topic: {topic}") + print(f"message: {message}") def _clean_up_side_tags(utils): @@ -145,6 +149,8 @@ def schedule(conf): except Exception as err: print(f"Tests failed with: {err}", flush=True) print(sys.exc_info()[0]) + print("-"*20) + print(sys.exc_info()) notify( topic=f"multi-build.end.error", message={"runid": runid, "exception": err}, From 77e00c479bf9462857dd6c0e25fc9a7cc18576b3 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Aug 18 2020 11:48:33 +0000 Subject: [PATCH 21/29] Check that the request return a 2xx code before proceeding Signed-off-by: Pierre-Yves Chibon --- diff --git a/monitor_gating/utils.py b/monitor_gating/utils.py index 6b0c027..0018f09 100644 --- a/monitor_gating/utils.py +++ b/monitor_gating/utils.py @@ -60,12 +60,15 @@ def blocking_issues(project, tags): issues = [] try: r = requests.get(api) - issues = r.json()["issues"] - if tags: - t = set(tags[1:]) - issues = [i for i in issues if t & set(i["tags"])] - for i in issues: - print(f"Found blocking issue https://pagure.io/{project}/issue/{i['id']}") + if not r.ok: + print(f"Failed to query: {api} -- returned : {r.status_code}") + else: + issues = r.json()["issues"] + if tags: + t = set(tags[1:]) + issues = [i for i in issues if t & set(i["tags"])] + for i in issues: + print(f"Found blocking issue https://pagure.io/{project}/issue/{i['id']}") except Exception as e: print(f"Error when querying pagure for blocking issues: {e}") return issues From 765c212b18236af4a0c434c34dc90b57473d061b Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Aug 18 2020 11:48:33 +0000 Subject: [PATCH 22/29] Add missing import to the runner.py Signed-off-by: Pierre-Yves Chibon --- diff --git a/monitor_gating/runner.py b/monitor_gating/runner.py index 9e6cfee..579b250 100644 --- a/monitor_gating/runner.py +++ b/monitor_gating/runner.py @@ -18,7 +18,7 @@ import toml from . import multi_builds from . import single_build -from .utils import MonitoringUtils, blocking_issues, run_command +from .utils import MonitoringUtils, blocking_issues, run_command, report_failure s = sched.scheduler(time.time, time.sleep) From 67e9a66324fe7ead17a9e30e901da944bcd4cb53 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Aug 18 2020 11:48:33 +0000 Subject: [PATCH 23/29] When notifying on errors, don't send the raw error object, send its text version Signed-off-by: Pierre-Yves Chibon --- diff --git a/monitor_gating/runner.py b/monitor_gating/runner.py index 579b250..e00a675 100644 --- a/monitor_gating/runner.py +++ b/monitor_gating/runner.py @@ -153,7 +153,7 @@ def schedule(conf): print(sys.exc_info()) notify( - topic=f"multi-build.end.error", message={"runid": runid, "exception": err}, + topic=f"multi-build.end.error", message={"runid": runid, "exception": str(err)}, ) delay_when_failing = conf["delay_when_failing"] From 7de62ba5ce0cfb381eb76ace845ea04cb2486bfc Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Aug 18 2020 11:48:34 +0000 Subject: [PATCH 24/29] Flush some prints Signed-off-by: Pierre-Yves Chibon --- diff --git a/monitor_gating/utils.py b/monitor_gating/utils.py index 0018f09..210282f 100644 --- a/monitor_gating/utils.py +++ b/monitor_gating/utils.py @@ -46,7 +46,7 @@ Full log: req = requests.post(url, data=data, headers=headers) if not req.ok: - print(f"Error when trying to open a ticket at: {url} to report the failure") + print(f"Error when trying to open a ticket at: {url} to report the failure", flush=True) def blocking_issues(project, tags): @@ -61,16 +61,16 @@ def blocking_issues(project, tags): try: r = requests.get(api) if not r.ok: - print(f"Failed to query: {api} -- returned : {r.status_code}") + print(f"Failed to query: {api} -- returned : {r.status_code}", flush=True) else: issues = r.json()["issues"] if tags: t = set(tags[1:]) issues = [i for i in issues if t & set(i["tags"])] for i in issues: - print(f"Found blocking issue https://pagure.io/{project}/issue/{i['id']}") + print(f"Found blocking issue https://pagure.io/{project}/issue/{i['id']}", flush=True) except Exception as e: - print(f"Error when querying pagure for blocking issues: {e}") + print(f"Error when querying pagure for blocking issues: {e}", flush=True) return issues From 258daf1d3b8787a55169269eed63c531097947e4 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Aug 18 2020 11:48:34 +0000 Subject: [PATCH 25/29] Add some more debugging Signed-off-by: Pierre-Yves Chibon --- diff --git a/monitor_gating/runner.py b/monitor_gating/runner.py index e00a675..2ce25bd 100644 --- a/monitor_gating/runner.py +++ b/monitor_gating/runner.py @@ -149,8 +149,14 @@ def schedule(conf): except Exception as err: print(f"Tests failed with: {err}", flush=True) print(sys.exc_info()[0]) - print("-"*20) + print("-"*60) print(sys.exc_info()) + print("="*60) + import traceback + traceback.print_exc(file=sys.stdout) + print("-"*60) + traceback.print_stack() + notify( topic=f"multi-build.end.error", message={"runid": runid, "exception": str(err)}, From 69e327b66f0a5b81f32fdfa0b1708e2fe6d3dc4b Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Aug 18 2020 11:48:34 +0000 Subject: [PATCH 26/29] Fix appending an element to a list rather than trying to call the list Signed-off-by: Pierre-Yves Chibon --- diff --git a/monitor_gating/utils.py b/monitor_gating/utils.py index 210282f..68f6332 100644 --- a/monitor_gating/utils.py +++ b/monitor_gating/utils.py @@ -156,7 +156,7 @@ class MonitoringUtils: run_command([command, "switch-branch", f"{name}"], cwd=folder) self.print_user(info_log, success=True) except MonitoringException: - self.failed("fedpkg") + self.failed.append("fedpkg") self.print_user(info_log, success=False) def bump_release(self, name, folder): @@ -168,7 +168,7 @@ class MonitoringUtils: run_command(["rpmdev-bumpspec", f"{name}.spec"], cwd=folder) self.print_user(info_log, success=True) except MonitoringException: - self.failed("rpmdev-bumspec") + self.failed.append("rpmdev-bumspec") self.print_user(info_log, success=False) def commit_changes(self, commit_log, folder): @@ -181,7 +181,7 @@ class MonitoringUtils: run_command(["git", "commit", "-asm", commit_log], cwd=folder) self.print_user(info_log, success=True) except MonitoringException: - self.failed("git") + self.failed.append("git") self.print_user(info_log, success=False) def push_changes(self, folder, target, branch, force=False): @@ -196,7 +196,7 @@ class MonitoringUtils: run_command(cmd, cwd=folder) self.print_user(info_log, success=True) except MonitoringException: - self.failed("git/dist-git") + self.failed.append("git/dist-git") self.print_user(info_log, success=False) def pull_changes(self, folder, target, branch): @@ -209,7 +209,7 @@ class MonitoringUtils: run_command(cmd, cwd=folder) self.print_user(info_log, success=True) except MonitoringException: - self.failed("git/dist-git") + self.failed.append("git/dist-git") self.print_user(info_log, success=False) def open_pullrequest(self, base_url, username, namespace, name, branch, token): @@ -237,7 +237,7 @@ class MonitoringUtils: success = False pr_id = None pr_uid = None - self.failed("dist-git") + self.failed.append("dist-git") else: output = req.json() pr_id = str(output["id"]) @@ -260,7 +260,7 @@ class MonitoringUtils: self.print_user(info_log, success=True) return nevr.strip().decode("utf-8") except MonitoringException: - self.failed("fedpkg") + self.failed.append("fedpkg") self.print_user(info_log, success=False) def build_package(self, command, folder, target=None): @@ -275,7 +275,7 @@ class MonitoringUtils: run_command(command, cwd=folder) self.print_user(info_log, success=True) except MonitoringException: - self.failed("koji") + self.failed.append("koji") self.print_user(info_log, success=False) def chain_build_packages(self, command, packages, folder, target=None): @@ -295,7 +295,7 @@ class MonitoringUtils: run_command(command, cwd=folder) self.print_user(info_log, success=True) except MonitoringException: - self.failed("koji") + self.failed.append("koji") self.print_user(info_log, success=False) def get_build_tags(self, koji_url, nevr, expected_ends): @@ -353,7 +353,7 @@ class MonitoringUtils: info_log = f"Retrieving koji tags: {tags}" if not success: - self.failed("koji") + self.failed.append("koji") self.print_user(info_log, success=success) def create_update( @@ -387,7 +387,7 @@ class MonitoringUtils: run_command(command) self.print_user(info_log, success=True) except MonitoringException: - self.failed("bodhi") + self.failed.append("bodhi") self.print_user(info_log, success=False) def get_update_id(self, nevr, url): @@ -408,7 +408,7 @@ class MonitoringUtils: if (datetime.datetime.utcnow() - start).seconds > (15 * 60): success = False - self.failed("bodhi") + self.failed.append("bodhi") info_log = f"Update for {nevr} not created within 15 minutes" break @@ -568,7 +568,7 @@ class MonitoringUtils: info_log += f" - ran for: {(end - start_lookup).seconds}s" self.print_user(info_log, success=success) if not success: - self.failed("datagrepper") + self.failed.append("datagrepper") def lookup_ci_resultsdb(self, nevr, name, url): """ Check the CI results in the specified resultsdb for results about @@ -619,7 +619,7 @@ class MonitoringUtils: info_log += f" - ran for: {(end - start).seconds}s" self.print_user(info_log, success=success) if not success: - self.failed("resultsdb") + self.failed.append("resultsdb") def waive_update(self, command, updateid, prod=True, username=None, password=None): """ Waive all the tests results for the specified update using bodhi's @@ -645,7 +645,7 @@ class MonitoringUtils: run_command(command) self.print_user(info_log, success=True) except MonitoringException: - self.failed("waiverdb") + self.failed.append("waiverdb") self.print_user(info_log, success=False) def get_pr_flag( @@ -701,7 +701,7 @@ class MonitoringUtils: self.print_user(info_log, success=success) if not success: - self.failed("dist-git") + self.failed.append("dist-git") def merge_pr(self, base_url, username, namespace, name, pr_id, token): """ Merge the specified PR @@ -722,7 +722,7 @@ class MonitoringUtils: self.print_user(info_log, success=success) if not success: - self.failed("dist-git") + self.failed.append("dist-git") def finalize(self, start): """ End data returned. """ @@ -745,7 +745,7 @@ class MonitoringUtils: self.print_user(info_log, success=True) except (MonitoringException, Exception) as err: print(err) - self.failed("dist-git") + self.failed.append("dist-git") self.print_user(info_log, success=False) return side_tag_name From 09034426ffd252d8707a551839f26c8b0ea21f44 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Aug 18 2020 11:56:02 +0000 Subject: [PATCH 27/29] Add some debugging for the keytab file Signed-off-by: Pierre-Yves Chibon --- diff --git a/entrypoint.sh b/entrypoint.sh index a699fc3..96bfa6d 100755 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -7,8 +7,6 @@ if ! whoami &> /dev/null; then fi -klist -A -k /etc/keytabs/monitor-gating-keytab - ln -s /opt/ssh/id_rsa /.ssh/id_rsa || true if [ -z ${PRODUCTION+x} ]; then # Staging info @@ -20,4 +18,6 @@ echo "Running in production" ssh-keyscan pkgs.fedoraproject.org >> /.ssh/known_hosts fi -exec "$@" +klist -t -k /etc/keytabs/monitor-gating-keytab + +python3 /opt/code/runner.py /opt/config/runner.cfg From 90056dc671baa8b840a95d5eaab326324c350b0c Mon Sep 17 00:00:00 2001 From: Adam Saleh Date: Aug 18 2020 11:56:05 +0000 Subject: [PATCH 28/29] Attempting to solve some problems by adding explicit start for the datanommer searches in single-build. --- diff --git a/monitor_gating/single_build.py b/monitor_gating/single_build.py index d1e613c..94f3dbc 100644 --- a/monitor_gating/single_build.py +++ b/monitor_gating/single_build.py @@ -232,6 +232,9 @@ def main(args): topic=f"org.centos.{conf['_ci_env']}.ci.koji-build.test.running", nevr=nevr, ) + + start_dg = datetime.datetime.utcnow() + # Check at the CI pipeline has completed utils.lookup_results_datagrepper( base_url=conf["datagrepper"], @@ -239,6 +242,7 @@ def main(args): topic=f"org.centos.{conf['_ci_env']}.ci.koji-build.test.error", nevr=nevr, duration=30, + start=start_dg, ) # Check the tag of the build @@ -257,6 +261,8 @@ def main(args): name="resultsdb", topic=f"org.fedoraproject.{conf['_env']}.resultsdb.result.new", nevr=nevr, + duration=45, + start=start_dg, ) # Check that greenwave reacted to resultsdb's new results @@ -265,6 +271,8 @@ def main(args): name="greenwave", topic=f"org.fedoraproject.{conf['_env']}.greenwave.decision.update", nevr=nevr, + duration=60, + start=start_dg, ) # Check the tag of the build -- build is blocked but should be signed From d76d35c730dee174919403edba47a3648eac4cf1 Mon Sep 17 00:00:00 2001 From: Stephen Coady Date: Aug 18 2020 11:56:05 +0000 Subject: [PATCH 29/29] add the ability to sync the nevr of both packages. the logic is: if the releases are not in sync then we use rpmdev-bumpspec to increase the version of both packages. this is the easiest way to make sure they both end up with the same release as the default behaviour when a release is bumped is to set version to 1. Signed-off-by: Stephen Coady --- diff --git a/monitor_gating/multi_builds.py b/monitor_gating/multi_builds.py index 9cbe830..bb1e306 100644 --- a/monitor_gating/multi_builds.py +++ b/monitor_gating/multi_builds.py @@ -82,15 +82,26 @@ def main(args, utils=None): # Bump the release on both packages: nevrs, side_tag_name = utils.clone_and_bump( - folder, nevrs, conf, conf["name_multi_1"], new_side_tag=True + folder, nevrs, conf, conf["name_multi_1"], version=0, new_side_tag=True ) # Store the side_tag_name so we can use it in the runner utils.side_tag_name = side_tag_name nevrs, _ = utils.clone_and_bump( - folder, nevrs, conf, conf["name_multi_2"], target=side_tag_name + folder, nevrs, conf, conf["name_multi_2"], version=0, target=side_tag_name ) + version, synced = utils.nevrs_synced(nevrs, conf) + + if not synced: + nevrs, side_tag_name = utils.clone_and_bump( + folder, nevrs, conf, conf["name_multi_1"], version=version, new_side_tag=True + ) + + nevrs, _ = utils.clone_and_bump( + folder, nevrs, conf, conf["name_multi_2"], version=version, target=side_tag_name + ) + # Chain-build the packages utils.chain_build_packages( conf["fedpkg"], diff --git a/monitor_gating/single_build.py b/monitor_gating/single_build.py index 94f3dbc..e7665d2 100644 --- a/monitor_gating/single_build.py +++ b/monitor_gating/single_build.py @@ -102,7 +102,7 @@ def main(args): ) gitfolder = os.path.join(folder, name) utils.switch_branch(conf["fedpkg"], branch, folder=gitfolder) - utils.bump_release(name, folder=gitfolder) + utils.bump_release(name, version=0, folder=gitfolder) utils.commit_changes("Bump release", folder=gitfolder) nevr = utils.get_nevr(conf["fedpkg"], folder=gitfolder) print(f" Upcoming build : {nevr}") diff --git a/monitor_gating/utils.py b/monitor_gating/utils.py index 68f6332..a02fe3c 100644 --- a/monitor_gating/utils.py +++ b/monitor_gating/utils.py @@ -11,6 +11,7 @@ import logging import os import subprocess import time +import re import requests @@ -159,14 +160,18 @@ class MonitoringUtils: self.failed.append("fedpkg") self.print_user(info_log, success=False) - def bump_release(self, name, folder): + def bump_release(self, name, version, folder): """ Bump the release of the spec file the specified git repo. """ info_log = f"Bumping release of: {name}.spec" self.print_user(info_log) try: - run_command(["rpmdev-bumpspec", f"{name}.spec"], cwd=folder) - self.print_user(info_log, success=True) + if version: + run_command(["rpmdev-bumpspec", f"{name}.spec", "-n", f"{version}"], cwd=folder) + self.print_user(info_log, success=True) + else: + run_command(["rpmdev-bumpspec", f"{name}.spec"], cwd=folder) + self.print_user(info_log, success=True) except MonitoringException: self.failed.append("rpmdev-bumspec") self.print_user(info_log, success=False) @@ -750,7 +755,7 @@ class MonitoringUtils: return side_tag_name def clone_and_bump( - self, folder, nevrs, conf, name, target=None, new_side_tag=False + self, folder, nevrs, conf, name, version, target=None, new_side_tag=False ): """Clone the repo, bump the release, commit and push.""" namespace = conf["namespace"] @@ -766,7 +771,11 @@ class MonitoringUtils: if new_side_tag: side_tag_name = self.create_side_tag(conf["fedpkg"], folder=gitfolder) target = side_tag_name - self.bump_release(name, folder=gitfolder) + if version: + version += version + self.bump_release(name, version, folder=gitfolder) + else: + self.bump_release(name, version=0, folder=gitfolder) self.commit_changes("Bump release", folder=gitfolder) nevr = self.get_nevr(conf["fedpkg"], folder=gitfolder) nevrs[name] = nevr @@ -775,6 +784,25 @@ class MonitoringUtils: print(f" Upcoming build : {nevr}") return (nevrs, target) + def nevrs_synced(self, nevrs, conf): + package_one = conf["name_multi_1"] + package_two = conf["name_multi_2"] + + nevr_one = nevrs[package_one] + nevr_two = nevrs[package_two] + + if nevr_one.startswith(package_one): + release_one = nevr_one[len(package_one):] + version = int(re.search(r'\d+', nevr_one).group()) + + if nevr_two.startswith(package_two): + release_two = nevr_one[len(package_two):] + + if release_one != release_two: + return version, False + + return version, True + def run_command(command, cwd=None): """ Run the specified command in a specific working directory if one