From 17662ad01f19bba9deb3a75398d95ffcde682b34 Mon Sep 17 00:00:00 2001 From: Nils Philippsen Date: May 25 2020 07:53:02 +0000 Subject: [PATCH 1/4] Move code into its own `monitor_gating` package Signed-off-by: Nils Philippsen --- diff --git a/clean_up_side_tags.py b/clean_up_side_tags.py deleted file mode 100644 index 982cc06..0000000 --- a/clean_up_side_tags.py +++ /dev/null @@ -1,78 +0,0 @@ -#!/usr/bin/python - -import argparse -import logging -import subprocess -import sys - -import toml - - -_log = logging.getLogger(__name__) - - -def get_cli_args(args): - - parser = argparse.ArgumentParser( - prog="clean up side-tags", - formatter_class=argparse.ArgumentDefaultsHelpFormatter, - ) - - parser.add_argument( - "conf", help="Configuration file used by the runner", - ) - - return parser.parse_args(args) - - -def run_command(command) -> bytes: - """ Run the specified command in a specific working directory if one - is specified. - """ - output = None - try: - output = subprocess.check_output(command, stderr=subprocess.PIPE) - except subprocess.CalledProcessError as e: - _log.error( - "Command `{}` return code: `{}`".format(" ".join(command), e.returncode) - ) - _log.error("stdout:\n-------\n{}".format(e.stdout)) - _log.error("stderr:\n-------\n{}".format(e.stderr)) - pass - - return output - - -def main(args): - - conf = toml.load(args.conf) - if conf.get("kb_principal") and conf.get("kb_keytab_file"): - print( - f"Logging as {conf['kb_principal']} into kerberos using: {conf['kb_keytab_file']}" - ) - cmd = ["kinit", conf["kb_principal"], "-kt", conf["kb_keytab_file"]] - run_command(cmd) - - # list side-tags: - cmd = [conf["fedpkg"], "list-side-tags", "--user", conf["kb_principal"]] - output = run_command(cmd) - - if not output: - print("No side-tags to clean found.") - return - - output = output.decode("UTF-8").strip().split("\n") - # Remove all the side-tags but the last one (which is the latest one), which - # we keep in case there is a run ongoing - for line in output[:-1]: - side_tag = line.split("\t")[0] - print("Removing side-tag: %s" % side_tag) - cmd = [conf["fedpkg"], "remove-side-tag", side_tag] - output = run_command(cmd) - - -if __name__ == "__main__": - """ Main method. """ - - args = get_cli_args(sys.argv[1:]) - main(args) diff --git a/monitor_gating/__init__.py b/monitor_gating/__init__.py new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/monitor_gating/__init__.py diff --git a/monitor_gating/clean_up_side_tags.py b/monitor_gating/clean_up_side_tags.py new file mode 100644 index 0000000..982cc06 --- /dev/null +++ b/monitor_gating/clean_up_side_tags.py @@ -0,0 +1,78 @@ +#!/usr/bin/python + +import argparse +import logging +import subprocess +import sys + +import toml + + +_log = logging.getLogger(__name__) + + +def get_cli_args(args): + + parser = argparse.ArgumentParser( + prog="clean up side-tags", + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + ) + + parser.add_argument( + "conf", help="Configuration file used by the runner", + ) + + return parser.parse_args(args) + + +def run_command(command) -> bytes: + """ Run the specified command in a specific working directory if one + is specified. + """ + output = None + try: + output = subprocess.check_output(command, stderr=subprocess.PIPE) + except subprocess.CalledProcessError as e: + _log.error( + "Command `{}` return code: `{}`".format(" ".join(command), e.returncode) + ) + _log.error("stdout:\n-------\n{}".format(e.stdout)) + _log.error("stderr:\n-------\n{}".format(e.stderr)) + pass + + return output + + +def main(args): + + conf = toml.load(args.conf) + if conf.get("kb_principal") and conf.get("kb_keytab_file"): + print( + f"Logging as {conf['kb_principal']} into kerberos using: {conf['kb_keytab_file']}" + ) + cmd = ["kinit", conf["kb_principal"], "-kt", conf["kb_keytab_file"]] + run_command(cmd) + + # list side-tags: + cmd = [conf["fedpkg"], "list-side-tags", "--user", conf["kb_principal"]] + output = run_command(cmd) + + if not output: + print("No side-tags to clean found.") + return + + output = output.decode("UTF-8").strip().split("\n") + # Remove all the side-tags but the last one (which is the latest one), which + # we keep in case there is a run ongoing + for line in output[:-1]: + side_tag = line.split("\t")[0] + print("Removing side-tag: %s" % side_tag) + cmd = [conf["fedpkg"], "remove-side-tag", side_tag] + output = run_command(cmd) + + +if __name__ == "__main__": + """ Main method. """ + + args = get_cli_args(sys.argv[1:]) + main(args) diff --git a/monitor_gating/multi_builds.py b/monitor_gating/multi_builds.py new file mode 100644 index 0000000..9cbe830 --- /dev/null +++ b/monitor_gating/multi_builds.py @@ -0,0 +1,233 @@ +#!/usr/bin/env python3 + +""" +This is a script to test how dist-git, koji, bodhi, resultsdb, fedora-ci, +greenwave and waiverdb act together and if any piece of them is failing to +do its part. + +Requirements: + - Have installed on your machine + - python3-requests + - bodhi-client >= 5.0 + - fedpkg + - fedpkg-stage + - Ensure git is configured correctly in your system (username, email...) + - You should have tests and gating setup in the package's repo you're + playing with + - Fork that repo in your name + - Make sure the repo has the f30 branch + - Ensure your ssh key is unlocked + - A valid kerberos ticket for staging and/or production + +""" + +import argparse +import datetime +import logging +import os +import sys +import tempfile + +import toml + +from .utils import MonitoringUtils + +_log = logging.getLogger(__name__) + + +def get_arguments(args): + """ Parse and return the CLI arguments. + """ + parser = argparse.ArgumentParser(description="Test the CI workflow in Fedora.") + parser.add_argument( + "--staging", + action="store_true", + default=False, + help="Changes to environment in which this runs from prod to staging", + ) + parser.add_argument( + "--no-waive", + action="store_true", + default=False, + help="Skip the end of the workflow when the failing tests are waived", + ) + parser.add_argument( + "--conf", + default="monitor_gating.cfg", + help="Configuration file to use, specifying the URLs and all", + ) + parser.add_argument( + "--conflict", + action="store_true", + default=False, + help="Make a build, conflicting in the main tag", + ) + + return parser.parse_args(args) + + +def main(args, utils=None): + """ Main method used by this script. """ + start = datetime.datetime.utcnow() + + args = get_arguments(args) + + conf = toml.load(args.conf) + if utils is None: + utils = MonitoringUtils() + + with tempfile.TemporaryDirectory(prefix="ci-test-") as folder: + print(f"Working in {folder}\n") + nevrs = {} + + # Bump the release on both packages: + nevrs, side_tag_name = utils.clone_and_bump( + folder, nevrs, conf, conf["name_multi_1"], new_side_tag=True + ) + # Store the side_tag_name so we can use it in the runner + utils.side_tag_name = side_tag_name + + nevrs, _ = utils.clone_and_bump( + folder, nevrs, conf, conf["name_multi_2"], target=side_tag_name + ) + + # Chain-build the packages + utils.chain_build_packages( + conf["fedpkg"], + packages=conf["name_multi_1"], + folder=os.path.join(folder, conf["name_multi_2"]), + target=side_tag_name, + ) + + if args.conflict: + utils.clone_to_build(folder, nevrs, conf, conf["name_multi_1"], target=None) + + # Create the update + utils.create_update( + conf["bodhi-cli"], + side_tag_name, + prod=conf["_env"] == "prod", + username=conf.get("bodhi-user"), + password=conf.get("bodhi-password"), + from_tag=True, + ) + updateid = utils.get_update_id(nevrs[list(nevrs.keys())[0]], conf["bodhi"],) + print(f" Update created : {updateid}") + + # Check the tag of the build + utils.get_build_tags( + conf.get("koji_hub"), + nevrs[conf["name_multi_1"]], + expected_ends=["signing-pending", "testing-pending"], + ) + + if not updateid: + utils.finalize(start) + return + + # Check that bodhi notified the pipeline it can run + utils.lookup_results_datagrepper( + base_url=conf["datagrepper"], + name="bodhi to CI", + topic=f"org.fedoraproject.{conf['_env']}.bodhi.update.status." + "testing.koji-build-group.build.complete", + bodhi_id=updateid, + ) + + start_dg = datetime.datetime.utcnow() + + nevr_names = [] + for name in nevrs: + nevr = nevrs[name] + nevr_names.append(nevr) + + # Check that the CI pipeline is running + utils.lookup_results_datagrepper( + base_url=conf["datagrepper"], + name="CI (running)", + topic=f"org.centos.{conf['_ci_env']}.ci.koji-build.test.running", + nevr=nevr, + start=start_dg, + ) + # Check at the CI pipeline has completed + utils.lookup_results_datagrepper( + base_url=conf["datagrepper"], + name="CI (complete)", + topic=f"org.centos.{conf['_ci_env']}.ci.koji-build.test.error", + nevr=nevr, + start=start_dg, + ) + + # Check the tag of the build + utils.get_build_tags( + conf.get("koji_hub"), nevr, expected_ends=["testing-pending"], + ) + + # Check that the CI results made it to resultsdb + utils.lookup_ci_resultsdb( + nevr=nevr, name="resultsdb(phx)", url=conf["resultsdb"] + ) + + # Check that resultsdb announced the new results + utils.lookup_results_datagrepper( + base_url=conf["datagrepper"], + name="resultsdb", + topic=f"org.fedoraproject.{conf['_env']}.resultsdb.result.new", + nevr=nevr, + start=start_dg, + ) + + # Check that greenwave reacted to resultsdb's new results + utils.lookup_results_datagrepper( + base_url=conf["datagrepper"], + name="greenwave", + topic=f"org.fedoraproject.{conf['_env']}.greenwave.decision.update", + nevr=nevr, + start=start_dg, + ) + + # Check the tag of the build -- build is blocked but should be signed + utils.get_build_tags( + conf.get("koji_hub"), nevr, expected_ends=["testing-pending"], + ) + + if not args.no_waive: + nevr = nevrs[list(nevrs.keys())[0]] + utils.waive_update( + conf["bodhi-cli"], + updateid, + prod=conf["_env"] == "prod", + username=conf.get("bodhi-user"), + password=conf.get("bodhi-password"), + ) + + # Check that waiverdb announced the new waiver + utils.lookup_results_datagrepper( + base_url=conf["datagrepper"], + name="waiverdb", + topic=f"org.fedoraproject.{conf['_env']}.waiverdb.waiver.new", + nevrs=nevr_names, + ) + + # Check that greenwave reacted to the new waiver + utils.lookup_results_datagrepper( + base_url=conf["datagrepper"], + name="greenwave", + topic=f"org.fedoraproject.{conf['_env']}.greenwave.decision.update", + nevrs=nevr_names, + ) + + # Check the tag of the build -- build was waived, let is through + utils.get_build_tags( + conf.get("koji_hub"), nevr, expected_ends=conf["koji_end_tag"], + ) + + utils.finalize(start) + return utils + + +if __name__ == "__main__": + try: + main(sys.argv[1:]) + except KeyboardInterrupt: + print(" -- Interupted --") diff --git a/monitor_gating/runner.py b/monitor_gating/runner.py new file mode 100644 index 0000000..924605d --- /dev/null +++ b/monitor_gating/runner.py @@ -0,0 +1,188 @@ +""" +This script is meant to run the different tests that we have sequentially, with +a scheduler, ie: after running all the tests, it will wait for a specified +amount of time and then run them again, until it's stopped. +""" + +import argparse +import datetime +import sched +import sys +import time +import uuid + +import fedora_messaging.api +import fedora_messaging.exceptions + +import toml + +from . import single_build +from . import multi_builds +from .utils import MonitoringUtils, blocking_issues, run_command + + +s = sched.scheduler(time.time, time.sleep) +conf = toml.load + + +def get_arguments(args): + """ Load and parse the CLI arguments.""" + parser = argparse.ArgumentParser(description="Runner for the CI canary tests.") + parser.add_argument( + "conf", help="Configuration file for the different tests", + ) + + return parser.parse_args(args) + + +def notify(topic, message): + try: + msg = fedora_messaging.api.Message( + topic="monitor-gating.{}".format(topic), body=message + ) + fedora_messaging.api.publish(msg) + except fedora_messaging.exceptions.PublishReturned as err: + print(f"Fedora Messaging broker rejected message {msg.id}: {err}") + except fedora_messaging.exceptions.ConnectionException as err: + print(f"Error sending message {msg.id}: {err}") + except Exception as err: + print(f"Error sending fedora-messaging message: {err}") + + +def _clean_up_side_tags(utils): + try: + print(" Removing side-tag: %s" % utils.side_tag_name) + cmd = [conf["fedpkg"], "remove-side-tag", utils.side_tag_name] + run_command(cmd) + except Exception: + pass + + +def schedule(conf): + """ Run the test and schedules the next one. """ + + if conf.get("kb_principal") and conf.get("kb_keytab_file"): + print(f"Logging into kerberos using: {conf['kb_keytab_file']}") + cmd = ["kinit", conf["kb_principal"], "-kt", conf["kb_keytab_file"]] + run_command(cmd) + + delay = conf["delay"] + report_project = conf["pagure_report_project"] + report_api_token = conf["pagure_api_token"] + report_env = conf["env"] + + print("Tests started:", datetime.datetime.utcnow(), flush=True) + runid = f"{datetime.datetime.utcnow().year}-{uuid.uuid4()}" + try: + # Single Build Gating + single_args = conf["workflow_single_gating_args"].split() + notify( + topic=f"single-build.start", + message={"arguments": single_args, "runid": runid}, + ) + monit_utils = single_build.main(single_args) + output_text = "\n".join(monit_utils.logs) + if "[FAILED]" not in output_text: + result = "succeeded" + else: + result = "failed" + report_failure( + report_project, + report_api_token, + report_env, + "single-package", + monit_utils, + ) + + notify( + topic=f"single-build.end.{result}", + message={ + "output": monit_utils.logs, + "output_text": output_text, + "result": result, + "runid": runid, + "failed": monit_utils.failed, + }, + ) + + # Multi Build Gating + multi_args = conf["workflow_multi_gating_args"].split() + monit_utils = MonitoringUtils() + notify( + topic=f"multi-build.start", + message={"arguments": multi_args, "runid": runid}, + ) + try: + monit_utils = multi_builds.main(multi_args, utils=monit_utils) + finally: + _clean_up_side_tags(monit_utils) + + output_text = "\n".join(monit_utils.logs) + if "[FAILED]" not in output_text: + result = "succeeded" + else: + result = "failed" + report_failure( + report_project, + report_api_token, + report_env, + "multi-package", + monit_utils, + ) + + notify( + topic=f"multi-build.end.{result}", + message={ + "output": monit_utils.logs, + "output_text": output_text, + "result": result, + "runid": runid, + "failed": monit_utils.failed, + }, + ) + + print("Tests finished:", datetime.datetime.utcnow(), flush=True) + except Exception as err: + print(f"Tests failed with: {err}", flush=True) + print(sys.exc_info()[0]) + + notify( + topic=f"multi-build.end.error", message={"runid": runid, "exception": err}, + ) + + delay_when_failing = conf["delay_when_failing"] + blocker_tags = conf["blocker_tags"] + blocking_project = conf["pagure_blocking_project"] + + blocking_issues_list = blocking_issues(blocking_project, blocker_tags) + now = datetime.datetime.utcnow().strftime("%H:%M:%S") + if blocking_issues_list: + print( + f"{now} Next run in: {delay_when_failing} seconds because of " + f"{len(blocking_issues_list)} open issues", + flush=True, + ) + s.enter(delay_when_failing, 1, schedule, argument=(conf,)) + else: + print(f"{now} Next run in: {delay} seconds", flush=True) + s.enter(delay, 1, schedule, argument=(conf,)) + + +def main(args): + """ Schedule the first test and run the scheduler. """ + args = get_arguments(args) + conf = toml.load(args.conf) + s.enter(0, 1, schedule, argument=(conf,)) + s.run() + + +if __name__ == "__main__": + try: + main(sys.argv[1:]) + except KeyboardInterrupt: + from code import InteractiveConsole + + InteractiveConsole(locals={"s": s}).interact( + "ENTERING THE DEBUG CONSOLE:\n s is the scheduler\n ^d to quit", + "LEAVING THE DEBUG CONSOLE", + ) diff --git a/monitor_gating/single_build.py b/monitor_gating/single_build.py new file mode 100644 index 0000000..d1e613c --- /dev/null +++ b/monitor_gating/single_build.py @@ -0,0 +1,313 @@ +#!/usr/bin/env python3 + +""" +This is a script to test how dist-git, koji, bodhi, resultsdb, fedora-ci, +greenwave and waiverdb act together and if any piece of them is failing to +do its part. + +Requirements: + - Have installed on your machine + - python3-requests + - bodhi-client + - fedpkg + - fedpkg-stage + - Ensure git is configured correctly in your system (username, email...) + - You should have tests and gating setup in the package's repo you're + playing with + - Fork that repo in your name + - Make sure the repo has the f30 branch + - Ensure your ssh key is unlocked + - A valid kerberos ticket for staging and/or production + +""" + +import argparse +import datetime +import logging +import os +import sys +import tempfile + +import toml + +from .utils import MonitoringUtils + +_log = logging.getLogger(__name__) + + +def get_arguments(args): + """ Parse and return the CLI arguments. + """ + parser = argparse.ArgumentParser(description="Test the CI workflow in Fedora.") + parser.add_argument( + "--nevr", help="NEVR of the build, allows by-passing: commit, push, build", + ) + parser.add_argument( + "--update", + help="Alias for the update, allows by-passing creating the udpate " "", + ) + parser.add_argument( + "--no-pr", + action="store_true", + default=False, + help="Skip testing the testing of pull-requests", + ) + parser.add_argument( + "--staging", + action="store_true", + default=False, + help="Changes to environment in which this runs from prod to staging", + ) + parser.add_argument( + "--auto-update", + action="store_true", + default=False, + help="Wait for the update to be created automatically instead of " + "doing it manually", + ) + parser.add_argument( + "--no-waive", + action="store_true", + default=False, + help="Skip the end of the workflow when the failing tests are waived", + ) + parser.add_argument( + "--conf", + default="monitor_gating.cfg", + help="Configuration file to use, specifying the URLs and all", + ) + + return parser.parse_args(args) + + +def main(args): + """ Main method used by this script. """ + start = datetime.datetime.utcnow() + + args = get_arguments(args) + + conf = toml.load(args.conf) + utils = MonitoringUtils() + + name = conf["name_single"] + namespace = conf["namespace"] + fas_username = conf["fas_username"] + branch = conf["branch"] + + with tempfile.TemporaryDirectory(prefix="ci-test-") as folder: + print(f"Working in {folder}\n") + if not args.nevr: + utils.clone_repo( + conf["fedpkg"], conf["fas_username"], namespace, name, folder=folder, + ) + gitfolder = os.path.join(folder, name) + utils.switch_branch(conf["fedpkg"], branch, folder=gitfolder) + utils.bump_release(name, folder=gitfolder) + utils.commit_changes("Bump release", folder=gitfolder) + nevr = utils.get_nevr(conf["fedpkg"], folder=gitfolder) + print(f" Upcoming build : {nevr}") + + if args.no_pr: + # Push to the main repo + utils.push_changes(gitfolder, "origin", branch) + else: + # Add the fork as remote, push to the it, open the PR, + # wait for CI to flag the PR, twice, merge the PR + utils.add_remote( + f"{fas_username}", + f"ssh://{fas_username}@{conf['distgit_host']}/" + f"forks/{fas_username}/{namespace}/{name}.git", + folder=gitfolder, + ) + utils.push_changes(gitfolder, fas_username, branch, force=True) + pr_created, pr_id, pr_uid = utils.open_pullrequest( + base_url=conf["pagure_dist_git"], + username=fas_username, + namespace=namespace, + name=name, + branch=branch, + token=conf["pagure_token"], + ) + if pr_created: + # Check that pr pipeline is running + utils.lookup_results_datagrepper( + base_url=conf["datagrepper"], + name="CI (running)", + topic=f"org.centos.{conf['_ci_env']}.ci.dist-git-pr.test.running", + rev=pr_uid, + ) + # Check that CI flag pending was set + utils.get_pr_flag( + base_url=conf["pagure_dist_git"], + username=fas_username, + namespace=namespace, + name=name, + pr_id=pr_id, + flag_username="Fedora CI", + flag_status="pending", + ) + # Check that pr pipeline has finished + utils.lookup_results_datagrepper( + base_url=conf["datagrepper"], + name="CI (complete)", + topic=f"org.centos.{conf['_ci_env']}.ci.dist-git-pr.test.error", + rev=pr_uid, + ) + # Check that CI flag failure was set + utils.get_pr_flag( + base_url=conf["pagure_dist_git"], + username=fas_username, + namespace=namespace, + name=name, + pr_id=pr_id, + flag_username="Fedora CI", + flag_status="error", + duration=25, + ) + # Merge the PR: TODO + utils.merge_pr( + base_url=conf["pagure_dist_git"], + username=fas_username, + namespace=namespace, + name=name, + pr_id=pr_id, + token=conf["pagure_token"], + ) + utils.pull_changes(gitfolder, "origin", branch) + else: + return + + # Build the package + utils.build_package(conf["fedpkg"], folder=gitfolder) + + # Check the tag of the build + utils.get_build_tags( + conf.get("koji_hub"), + nevr, + expected_ends=["updates-candidate", "signing-pending"], + ) + else: + nevr = args.nevr + + # Retrieve or create the update + updateid = utils.get_update_id(nevr, conf["bodhi"]) + if not args.update and not args.auto_update: + utils.create_update( + conf["bodhi-cli"], + nevr, + prod=conf["_env"] == "prod", + username=conf.get("bodhi-user"), + password=conf.get("bodhi-password"), + ) + print(f" Update created : {updateid}") + elif args.auto_update: + print(f" Update automatically created : {updateid}") + else: + updateid = args.update + + # Check the tag of the build + utils.get_build_tags( + conf.get("koji_hub"), + nevr, + expected_ends=["signing-pending", "testing-pending"], + ) + + if not updateid: + utils.finalize(start) + return + + # Check that bodhi notified the pipeline it can run + utils.lookup_results_datagrepper( + base_url=conf["datagrepper"], + name="bodhi to CI", + topic=f"org.fedoraproject.{conf['_env']}.bodhi.update.status." + "testing.koji-build-group.build.complete", + bodhi_id=updateid, + ) + + # Check that the CI pipeline is running + utils.lookup_results_datagrepper( + base_url=conf["datagrepper"], + name="CI (running)", + topic=f"org.centos.{conf['_ci_env']}.ci.koji-build.test.running", + nevr=nevr, + ) + # Check at the CI pipeline has completed + utils.lookup_results_datagrepper( + base_url=conf["datagrepper"], + name="CI (complete)", + topic=f"org.centos.{conf['_ci_env']}.ci.koji-build.test.error", + nevr=nevr, + duration=30, + ) + + # Check the tag of the build + utils.get_build_tags( + conf.get("koji_hub"), nevr, expected_ends=["testing-pending"], + ) + + # Check that the CI results made it to resultsdb + utils.lookup_ci_resultsdb( + nevr=nevr, name="resultsdb(phx)", url=conf["resultsdb"] + ) + + # Check that resultsdb announced the new results + utils.lookup_results_datagrepper( + base_url=conf["datagrepper"], + name="resultsdb", + topic=f"org.fedoraproject.{conf['_env']}.resultsdb.result.new", + nevr=nevr, + ) + + # Check that greenwave reacted to resultsdb's new results + utils.lookup_results_datagrepper( + base_url=conf["datagrepper"], + name="greenwave", + topic=f"org.fedoraproject.{conf['_env']}.greenwave.decision.update", + nevr=nevr, + ) + + # Check the tag of the build -- build is blocked but should be signed + utils.get_build_tags( + conf.get("koji_hub"), nevr, expected_ends=["testing-pending"], + ) + + if not args.no_waive: + utils.waive_update( + conf["bodhi-cli"], + updateid, + prod=conf["_env"] == "prod", + username=conf.get("bodhi-user"), + password=conf.get("bodhi-password"), + ) + + # Check that waiverdb announced the new waiver + utils.lookup_results_datagrepper( + base_url=conf["datagrepper"], + name="waiverdb", + topic=f"org.fedoraproject.{conf['_env']}.waiverdb.waiver.new", + nevr=nevr, + ) + + # Check that greenwave reacted to the new waiver + utils.lookup_results_datagrepper( + base_url=conf["datagrepper"], + name="greenwave", + topic=f"org.fedoraproject.{conf['_env']}.greenwave.decision.update", + nevr=nevr, + ) + + # Check the tag of the build -- build was waived, let is through + utils.get_build_tags( + conf.get("koji_hub"), nevr, expected_ends=conf["koji_end_tag"], + ) + + utils.finalize(start) + return utils + + +if __name__ == "__main__": + try: + main(sys.argv[1:]) + except KeyboardInterrupt: + print(" -- Interupted --") diff --git a/monitor_gating/utils.py b/monitor_gating/utils.py new file mode 100644 index 0000000..6b0c027 --- /dev/null +++ b/monitor_gating/utils.py @@ -0,0 +1,794 @@ +#!/usr/bin/env python3 + +""" +This is a small library of utility methods used by the monitoring scripts. + +""" + +import ast +import datetime +import logging +import os +import subprocess +import time + +import requests + +_log = logging.getLogger(__name__) + + +def report_failure(project, token, env, workflow, monit_utils): + """ Open a pagure ticket against the instance specified in the + configuration file when something does not work. + """ + url = f"https://pagure.io/api/0/{project}/new_issue" + title = f"Failure in {env} of the {workflow} packager workflow" + logs = "\n".join(monit_utils.logs) + content = f"""A run of monitor-gating has just failed in {env} for the {workflow} workflow. + +The suspects are '{", ".join(monit_utils.failed)}'. + +Full log: +```` +{logs} +```` +""" + tag = env + + data = { + "title": title, + "content": content, + "tag": tag, + } + headers = { + "Authorization": f"token {token}", + } + + req = requests.post(url, data=data, headers=headers) + if not req.ok: + print(f"Error when trying to open a ticket at: {url} to report the failure") + + +def blocking_issues(project, tags): + """Lists blocking issues we track in the fedora-infrastructure project. + """ + if not tags: + print(f"No tags to filter blocking issues by, returning empty.") + return [] + + api = f"https://pagure.io/api/0/{project}/issues?status=Open&tags={tags[0]}" + issues = [] + try: + r = requests.get(api) + issues = r.json()["issues"] + if tags: + t = set(tags[1:]) + issues = [i for i in issues if t & set(i["tags"])] + for i in issues: + print(f"Found blocking issue https://pagure.io/{project}/issue/{i['id']}") + except Exception as e: + print(f"Error when querying pagure for blocking issues: {e}") + return issues + + +class MonitoringException(Exception): + """The base class for all exceptions raised by this script.""" + + +class MonitoringUtils: + def __init__(self): + """ Instanciate the object. """ + self.logs = [] + self.failed = [] + + def print_user(self, content, success=None): + """ Prints the specified content to the user. + """ + spaces = 90 + if success is not None: + end = None + if success: + content = "{} {}".format(content.ljust(spaces), "[DONE]") + else: + content = "{} {}".format(content.ljust(spaces), "[FAILED]") + else: + if os.environ.get("OPENSHIFT"): + end = None + else: + end = "\r" + + now = datetime.datetime.utcnow() + time = now.strftime("%H:%M:%S") + self.logs.append(f"{time} - {content}") + print(f"{time} - {content}", end=end, flush=True) + + def clone_repo(self, command, username, namespace, name, folder): + """ Clone the specified git repo into the specified folder. + """ + info_log = f"Cloning as {username} the git repo: {namespace}/{name}" + self.print_user(info_log) + try: + run_command( + [command, "--user", username, "clone", f"{namespace}/{name}"], + cwd=folder, + ) + except MonitoringException: + self.failed.append("git/dist-git") + self.print_user(info_log, success=False) + else: + try: + # Assume these commands can't fail + clone_folder = os.path.join(folder, name) + run_command( + ["git", "config", "user.name", "packagerbot"], cwd=clone_folder, + ) + run_command( + ["git", "config", "user.email", "admin@fedoraproject.org"], + cwd=clone_folder, + ) + self.print_user(info_log, success=True) + except MonitoringException: + self.failed.append("git") + self.print_user(info_log, success=False) + + def add_remote(self, name, url, folder): + """ Add the specified remote to the git repo in the folder with the + specified url. + """ + info_log = f"Adding remote: {name}" + self.print_user(info_log) + try: + run_command(["git", "remote", "add", name, url], cwd=folder) + self.print_user(info_log, success=True) + except MonitoringException: + self.failed.append("git") + self.print_user(info_log, success=False) + + def switch_branch(self, command, name, folder): + """ Switch to the specified git branch in the specified git repo. + """ + info_log = f"Switching to branch: {name}" + self.print_user(info_log) + try: + run_command([command, "switch-branch", f"{name}"], cwd=folder) + self.print_user(info_log, success=True) + except MonitoringException: + self.failed("fedpkg") + self.print_user(info_log, success=False) + + def bump_release(self, name, folder): + """ Bump the release of the spec file the specified git repo. + """ + info_log = f"Bumping release of: {name}.spec" + self.print_user(info_log) + try: + run_command(["rpmdev-bumpspec", f"{name}.spec"], cwd=folder) + self.print_user(info_log, success=True) + except MonitoringException: + self.failed("rpmdev-bumspec") + self.print_user(info_log, success=False) + + def commit_changes(self, commit_log, folder): + """ Commit all the changes made to *tracked* files in the git repo + with the specified commit log. + """ + info_log = f"Commiting changes" + self.print_user(info_log) + try: + run_command(["git", "commit", "-asm", commit_log], cwd=folder) + self.print_user(info_log, success=True) + except MonitoringException: + self.failed("git") + self.print_user(info_log, success=False) + + def push_changes(self, folder, target, branch, force=False): + """ Push all changes using git. + """ + info_log = f"Pushing changes" + self.print_user(info_log) + try: + cmd = ["git", "push", target, branch] + if force: + cmd.append("-f") + run_command(cmd, cwd=folder) + self.print_user(info_log, success=True) + except MonitoringException: + self.failed("git/dist-git") + self.print_user(info_log, success=False) + + def pull_changes(self, folder, target, branch): + """ Pull all changes using git. + """ + info_log = f"Pulling changes" + self.print_user(info_log) + try: + cmd = ["git", "pull", "--rebase", target, branch] + run_command(cmd, cwd=folder) + self.print_user(info_log, success=True) + except MonitoringException: + self.failed("git/dist-git") + self.print_user(info_log, success=False) + + def open_pullrequest(self, base_url, username, namespace, name, branch, token): + """ Open a pull-request from the user's fork to the main project for + the specified branch. + """ + info_log = f"Creating PR from forks/{username}/{namespace}/{name}" + self.print_user(info_log) + url = "/".join( + [base_url.rstrip("/"), "api/0", namespace, name, "pull-request/new"] + ) + data = { + "branch_to": branch, + "branch_from": branch, + "repo_from": name, + "repo_from_username": username, + "repo_from_namespace": namespace, + "initial_comment": "Testing PR", + "title": "Test PR for monitoring", + } + headers = {"Authorization": f"token {token}"} + req = requests.post(url=url, data=data, headers=headers) + if not req.ok: + print(req.text) + success = False + pr_id = None + pr_uid = None + self.failed("dist-git") + else: + output = req.json() + pr_id = str(output["id"]) + pr_uid = output["uid"] + url = "/".join( + [base_url.rstrip("/"), namespace, name, "pull-request", pr_id] + ) + info_log = f"PR created {url}" + success = True + self.print_user(info_log, success=success) + return (success, pr_id, pr_uid) + + def get_nevr(self, command, folder): + """ Get the name-epoch-version-release presently in git + """ + info_log = f"Getting nevr" + self.print_user(info_log) + try: + nevr = run_command([command, "verrel"], cwd=folder) + self.print_user(info_log, success=True) + return nevr.strip().decode("utf-8") + except MonitoringException: + self.failed("fedpkg") + self.print_user(info_log, success=False) + + def build_package(self, command, folder, target=None): + """ Build the package in the current branch + """ + info_log = f"Building the package" + self.print_user(info_log) + command = [command, "build"] + if target: + command.extend(["--target", target]) + try: + run_command(command, cwd=folder) + self.print_user(info_log, success=True) + except MonitoringException: + self.failed("koji") + self.print_user(info_log, success=False) + + def chain_build_packages(self, command, packages, folder, target=None): + """ Chain-build the packages in the current branch + """ + if not isinstance(packages, list): + packages = [packages] + info_log = ( + f"Chain-building the packages: {packages + [os.path.basename(folder)]}" + ) + self.print_user(info_log) + command = [command, "chain-build"] + command.extend(packages) + if target: + command.extend(["--target", target]) + try: + run_command(command, cwd=folder) + self.print_user(info_log, success=True) + except MonitoringException: + self.failed("koji") + self.print_user(info_log, success=False) + + def get_build_tags(self, koji_url, nevr, expected_ends): + """ List the tags associated with the specified build. + """ + # return + start = datetime.datetime.utcnow() + info_log = f"Retrieving koji tags" + self.print_user(info_log) + command = [ + "koji", + ] + if koji_url: + command.extend(["-s", koji_url]) + command.extend(["call", "listTags", nevr]) + + success = False + tags = None + broke = False + while True: + try: + output = run_command(command) + output = output.decode("utf-8") + try: + data = ast.literal_eval(output.strip()) + except Exception: + print("Could not decode JSON in:") + print(command) + print(output) + broke = True + break + tags = [tag.get("name") for tag in data] + for tag_name in tags: + for expectation in expected_ends: + if tag_name.endswith(expectation): + success = True + broke = True + break + if success: + broke = True + break + if broke: + break + + if (datetime.datetime.utcnow() - start).seconds > (15 * 60): + success = False + info_log = f"Update for {nevr} not created within 15 minutes" + break + + # Only query koji every 30 seconds + time.sleep(30) + except MonitoringException: + success = False + break + + info_log = f"Retrieving koji tags: {tags}" + if not success: + self.failed("koji") + self.print_user(info_log, success=success) + + def create_update( + self, command, item, prod=True, username=None, password=None, from_tag=False, + ): + """ Create the update for the package built. + """ + info_log = f"Creating a bodhi update" + self.print_user(info_log) + command = [ + command, + "updates", + "new", + "--notes", + "Bump release to test CI", + "--type", + "bugfix", + "--autotime", + ] + if from_tag: + command.append("--from-tag") + command.append(item) + + if not prod: + command.append("--staging") + if username: + command.extend(["--user", username]) + if password: + command.extend(["--password", password]) + try: + run_command(command) + self.print_user(info_log, success=True) + except MonitoringException: + self.failed("bodhi") + self.print_user(info_log, success=False) + + def get_update_id(self, nevr, url): + """ Retrieve the update identifier from bodhi for the given nevr. """ + start = datetime.datetime.utcnow() + info_log = f"Retrieving update created" + self.print_user(info_log) + url = f"{url}/updates/?builds={nevr}" + updateid = None + success = True + while True: + req = requests.get(url) + data = req.json() + if data["updates"]: + updateid = data["updates"][0]["updateid"] + if updateid: + break + + if (datetime.datetime.utcnow() - start).seconds > (15 * 60): + success = False + self.failed("bodhi") + info_log = f"Update for {nevr} not created within 15 minutes" + break + + # Only query bodhi every 30 seconds + time.sleep(30) + + self.print_user(info_log, success=success) + return updateid + + def lookup_results_datagrepper( + self, + base_url, + name, + topic, + nevr=None, + nevrs=None, + rev=None, + bodhi_id=None, + start=None, + duration=15, + ): + """ Check the CI results in datagrepper for results about our specified + build. + """ + start_lookup = datetime.datetime.utcnow() + if start is None: + start = start_lookup + info_log = f"Checking datagrepper for {name} messages" + self.print_user(info_log) + # Start pulling messages 10 minutes before now + start_time = start - datetime.timedelta(minutes=10) + # Limiting the number of row per page to 10 allows for quicker results + url = ( + base_url + f"?topic={topic}" + f"&start={start_time.timestamp()}&row_per_page=10" + ) + + success = None + returned_status = None + info_log = None + nevrs = nevrs or [] + while True: + # We're assuming here that there won't be more than 100 messages for + # that topic coming in between the one we're interested in and when we + # are looking for it (10*10 == 100) + for page in range(1, 11): + end_url = url + end_url += f"&page={page}" + data = requests.get(end_url).json() + if "raw_messages" not in data: + nomsg_log = f"No messages in data-grepper on {end_url} " + if "error" in data: + nomsg_log += data["error"] + self.print_user(nomsg_log) + break + + for message in data["raw_messages"]: + + # Old message format from the CI pipeline + if "ci.pipeline" in message["topic"] and ( + message["msg"]["nvr"] == nevr + or message["msg"]["nvr"] in nevrs + or message["msg"]["rev"] == rev + ): + success = True + returned_status = message["msg"]["status"] + break + + # New message format from the CI pipeline for koji builds + if ( + "ci.koji-build" in message["topic"] + and message["msg"]["artifact"]["nvr"] == nevr + ): + if message["topic"].endswith("test.complete"): + success = True + returned_status = message["msg"]["test"]["result"] + elif message["topic"].endswith("test.error"): + success = True + returned_status = "error" + elif message["topic"].endswith("test.running"): + success = True + returned_status = "running" + break + + # New message format from the CI pipeline for dist-git PR + if ( + "ci.dist-git-pr" in message["topic"] + and message["msg"]["artifact"]["type"] == "pull-request" + and message["msg"]["artifact"]["uid"] == rev + ): + if message["topic"].endswith("test.complete"): + success = True + returned_status = message["msg"]["test"]["result"] + elif message["topic"].endswith("test.error"): + success = True + returned_status = "error" + elif message["topic"].endswith("test.running"): + success = True + returned_status = "running" + break + + # resultsdb messages + if ( + "resultsdb" in message["topic"] + and "nvr" in message["msg"]["data"] + and ( + nevr in message["msg"]["data"]["nvr"] + or message["msg"]["data"]["nvr"] in nevrs + ) + ): + success = True + returned_status = message["msg"]["outcome"] + break + + # greenwave messages + if "greenwave" in message["topic"] and ( + message["msg"]["subject_identifier"] == nevr + or message["msg"]["subject_identifier"] in nevrs + ): + success = True + returned_status = message["msg"]["policies_satisfied"] + break + + # waiverdb messages + if "waiverdb" in message["topic"] and ( + message["msg"]["subject_identifier"] == nevr + or message["msg"]["subject_identifier"] in nevrs + ): + success = True + returned_status = "" + break + + # bodhi messages + if "bodhi.update.status.testing" in message["topic"] and message[ + "msg" + ]["artifact"]["id"].startswith(bodhi_id): + success = True + returned_status = "" + break + if success is not None: + break + if success is not None: + break + + if (datetime.datetime.utcnow() - start).seconds > (duration * 60): + success = False + info_log = f"{name} results not found in datagrepper" + break + + # Only query datagrepper every 30 seconds + time.sleep(30) + + if info_log is None: + info_log = f"{name} results in datagrepper returned {returned_status}" + + end = datetime.datetime.utcnow() + info_log += f" - ran for: {(end - start_lookup).seconds}s" + self.print_user(info_log, success=success) + if not success: + self.failed("datagrepper") + + def lookup_ci_resultsdb(self, nevr, name, url): + """ Check the CI results in the specified resultsdb for results about + our specified build. + """ + start = datetime.datetime.utcnow() + info_log = f"Checking {name} for CI results " + self.print_user(info_log) + topic = "org.centos.prod.ci.pipeline.allpackages-build.complete" + if ".stg" in url: + topic = "org.centos.stage.ci.pipeline.allpackages-build.complete" + url = f"{url}?testcases={topic}" + + success = False + returned_status = None + info_log = None + while True: + # Assume we won't have more than 3 pages of results coming in b/w + # our checks + for page in [0, 1, 2]: + end_url = url + end_url += f"&page={page}" + data = requests.get(end_url).json() + for result in data["data"]: + if nevr in result["data"]["nvr"]: + success = True + returned_status = result["data"]["status"][0] + break + if success: + break + if success: + break + + if (datetime.datetime.utcnow() - start).seconds > (15 * 60): + success = False + info_log = ( + f"CI results did not show in {name} for {nevr} within 15 minutes" + ) + break + + # Only query datagrepper every 30 seconds + time.sleep(30) + + if info_log is None: + info_log = f"CI results in {name} returned {returned_status}" + + end = datetime.datetime.utcnow() + info_log += f" - ran for: {(end - start).seconds}s" + self.print_user(info_log, success=success) + if not success: + self.failed("resultsdb") + + def waive_update(self, command, updateid, prod=True, username=None, password=None): + """ Waive all the tests results for the specified update using bodhi's + CLI. + """ + info_log = f"Waiving test results for bodhi update" + self.print_user(info_log) + command = [ + command, + "updates", + "waive", + updateid, + "'This is fine, we are testing the workflow'", + "--debug", + ] + if not prod: + command.append("--staging") + if username: + command.extend(["--user", username]) + if password: + command.extend(["--password", password]) + try: + run_command(command) + self.print_user(info_log, success=True) + except MonitoringException: + self.failed("waiverdb") + self.print_user(info_log, success=False) + + def get_pr_flag( + self, + base_url, + username, + namespace, + name, + pr_id, + flag_username, + flag_status, + duration=10, + ): + """ Retrieve the flags of the PR and assert the last one from the + specified flag_username has the given status. + """ + pr = "/".join([namespace, name, "pull-request", pr_id]) + info_log = f"Retreiving flags for PR: {pr}" + self.print_user(info_log) + url = "/".join([base_url.rstrip("/"), "api/0", pr, "flag"]) + + start = datetime.datetime.utcnow() + success = False + + while True: + try: + req = requests.get(url=url) + except requests.exceptions.ConnectionError: + continue + + if req.ok: + break + + if (datetime.datetime.utcnow() - start).seconds > (duration * 60): + success = False + info_log = f"Failed to retrieve flags for PR: {pr}" + break + + # Only query pagure every 30 seconds + time.sleep(30) + + if not req.ok: + print(req.text) + self.logs.append(f"Error retrieving PR flags: {req.text}") + raise MonitoringException("Error retrieving PR flags") + else: + output = req.json() + for flag in output["flags"]: + if flag["username"] == flag_username: + info_log = f"Retreived flag {flag['status']} on PR" + success = flag["status"] == flag_status + break + + self.print_user(info_log, success=success) + if not success: + self.failed("dist-git") + + def merge_pr(self, base_url, username, namespace, name, pr_id, token): + """ Merge the specified PR + """ + pr = "/".join([namespace, name, "pull-request", pr_id]) + info_log = f"Merge PR: {pr}" + self.print_user(info_log) + url = "/".join([base_url.rstrip("/"), "api/0", pr, "merge"]) + headers = {"Authorization": f"token {token}"} + req = requests.post(url=url, data={"wait": True}, headers=headers) + success = False + if not req.ok: + print(req.text) + self.logs(f"Error Merging flags: {req.text}") + raise MonitoringException("Error merging flags") + else: + success = True + + self.print_user(info_log, success=success) + if not success: + self.failed("dist-git") + + def finalize(self, start): + """ End data returned. """ + end = datetime.datetime.utcnow() + delta = (end - start).seconds + self.logs.append(f"Ran for {delta} seconds ({delta/60:.2f} minutes)") + print(f"Ran for {delta} seconds ({delta/60:.2f} minutes)") + + def create_side_tag(self, command, folder): + """ Create a side-tag to build packages in it. """ + info_log = f"Creating the side-tag" + self.print_user(info_log) + command = [command, "request-side-tag"] + side_tag_name = None + try: + output = run_command(command, cwd=folder) + lines = output.decode("utf-8").split("\n") + _, side_tag_name, _ = lines[0].split("'") + info_log = f"Created side-tag {side_tag_name}" + self.print_user(info_log, success=True) + except (MonitoringException, Exception) as err: + print(err) + self.failed("dist-git") + self.print_user(info_log, success=False) + return side_tag_name + + def clone_and_bump( + self, folder, nevrs, conf, name, target=None, new_side_tag=False + ): + """Clone the repo, bump the release, commit and push.""" + namespace = conf["namespace"] + branch = conf["branch"] + + self.clone_repo( + conf["fedpkg"], conf["fas_username"], namespace, name, folder=folder, + ) + gitfolder = os.path.join(folder, name) + self.switch_branch(conf["fedpkg"], branch, folder=gitfolder) + side_tag_name = None + # Create a side-tag + if new_side_tag: + side_tag_name = self.create_side_tag(conf["fedpkg"], folder=gitfolder) + target = side_tag_name + self.bump_release(name, folder=gitfolder) + self.commit_changes("Bump release", folder=gitfolder) + nevr = self.get_nevr(conf["fedpkg"], folder=gitfolder) + nevrs[name] = nevr + # Push to the main repo + self.push_changes(gitfolder, "origin", branch) + print(f" Upcoming build : {nevr}") + return (nevrs, target) + + +def run_command(command, cwd=None): + """ Run the specified command in a specific working directory if one + is specified. + """ + output = None + try: + output = subprocess.check_output(command, cwd=cwd, stderr=subprocess.PIPE) + except subprocess.CalledProcessError as e: + if "--password" in command: + idx = command.index("--password") + command[idx + 1] = "" + _log.error( + "Command `{}` return code: `{}`".format(" ".join(command), e.returncode) + ) + _log.error("stdout:\n-------\n{}".format(e.stdout)) + _log.error("stderr:\n-------\n{}".format(e.stderr)) + raise MonitoringException("Command failed to run") + + return output diff --git a/monitor_gating_multi_builds.py b/monitor_gating_multi_builds.py deleted file mode 100644 index 18648d3..0000000 --- a/monitor_gating_multi_builds.py +++ /dev/null @@ -1,233 +0,0 @@ -#!/usr/bin/env python3 - -""" -This is a script to test how dist-git, koji, bodhi, resultsdb, fedora-ci, -greenwave and waiverdb act together and if any piece of them is failing to -do its part. - -Requirements: - - Have installed on your machine - - python3-requests - - bodhi-client >= 5.0 - - fedpkg - - fedpkg-stage - - Ensure git is configured correctly in your system (username, email...) - - You should have tests and gating setup in the package's repo you're - playing with - - Fork that repo in your name - - Make sure the repo has the f30 branch - - Ensure your ssh key is unlocked - - A valid kerberos ticket for staging and/or production - -""" - -import argparse -import datetime -import logging -import os -import sys -import tempfile - -import toml - -from utils import MonitoringUtils - -_log = logging.getLogger(__name__) - - -def get_arguments(args): - """ Parse and return the CLI arguments. - """ - parser = argparse.ArgumentParser(description="Test the CI workflow in Fedora.") - parser.add_argument( - "--staging", - action="store_true", - default=False, - help="Changes to environment in which this runs from prod to staging", - ) - parser.add_argument( - "--no-waive", - action="store_true", - default=False, - help="Skip the end of the workflow when the failing tests are waived", - ) - parser.add_argument( - "--conf", - default="monitor_gating.cfg", - help="Configuration file to use, specifying the URLs and all", - ) - parser.add_argument( - "--conflict", - action="store_true", - default=False, - help="Make a build, conflicting in the main tag", - ) - - return parser.parse_args(args) - - -def main(args, utils=None): - """ Main method used by this script. """ - start = datetime.datetime.utcnow() - - args = get_arguments(args) - - conf = toml.load(args.conf) - if utils is None: - utils = MonitoringUtils() - - with tempfile.TemporaryDirectory(prefix="ci-test-") as folder: - print(f"Working in {folder}\n") - nevrs = {} - - # Bump the release on both packages: - nevrs, side_tag_name = utils.clone_and_bump( - folder, nevrs, conf, conf["name_multi_1"], new_side_tag=True - ) - # Store the side_tag_name so we can use it in the runner - utils.side_tag_name = side_tag_name - - nevrs, _ = utils.clone_and_bump( - folder, nevrs, conf, conf["name_multi_2"], target=side_tag_name - ) - - # Chain-build the packages - utils.chain_build_packages( - conf["fedpkg"], - packages=conf["name_multi_1"], - folder=os.path.join(folder, conf["name_multi_2"]), - target=side_tag_name, - ) - - if args.conflict: - utils.clone_to_build(folder, nevrs, conf, conf["name_multi_1"], target=None) - - # Create the update - utils.create_update( - conf["bodhi-cli"], - side_tag_name, - prod=conf["_env"] == "prod", - username=conf.get("bodhi-user"), - password=conf.get("bodhi-password"), - from_tag=True, - ) - updateid = utils.get_update_id(nevrs[list(nevrs.keys())[0]], conf["bodhi"],) - print(f" Update created : {updateid}") - - # Check the tag of the build - utils.get_build_tags( - conf.get("koji_hub"), - nevrs[conf["name_multi_1"]], - expected_ends=["signing-pending", "testing-pending"], - ) - - if not updateid: - utils.finalize(start) - return - - # Check that bodhi notified the pipeline it can run - utils.lookup_results_datagrepper( - base_url=conf["datagrepper"], - name="bodhi to CI", - topic=f"org.fedoraproject.{conf['_env']}.bodhi.update.status." - "testing.koji-build-group.build.complete", - bodhi_id=updateid, - ) - - start_dg = datetime.datetime.utcnow() - - nevr_names = [] - for name in nevrs: - nevr = nevrs[name] - nevr_names.append(nevr) - - # Check that the CI pipeline is running - utils.lookup_results_datagrepper( - base_url=conf["datagrepper"], - name="CI (running)", - topic=f"org.centos.{conf['_ci_env']}.ci.koji-build.test.running", - nevr=nevr, - start=start_dg, - ) - # Check at the CI pipeline has completed - utils.lookup_results_datagrepper( - base_url=conf["datagrepper"], - name="CI (complete)", - topic=f"org.centos.{conf['_ci_env']}.ci.koji-build.test.error", - nevr=nevr, - start=start_dg, - ) - - # Check the tag of the build - utils.get_build_tags( - conf.get("koji_hub"), nevr, expected_ends=["testing-pending"], - ) - - # Check that the CI results made it to resultsdb - utils.lookup_ci_resultsdb( - nevr=nevr, name="resultsdb(phx)", url=conf["resultsdb"] - ) - - # Check that resultsdb announced the new results - utils.lookup_results_datagrepper( - base_url=conf["datagrepper"], - name="resultsdb", - topic=f"org.fedoraproject.{conf['_env']}.resultsdb.result.new", - nevr=nevr, - start=start_dg, - ) - - # Check that greenwave reacted to resultsdb's new results - utils.lookup_results_datagrepper( - base_url=conf["datagrepper"], - name="greenwave", - topic=f"org.fedoraproject.{conf['_env']}.greenwave.decision.update", - nevr=nevr, - start=start_dg, - ) - - # Check the tag of the build -- build is blocked but should be signed - utils.get_build_tags( - conf.get("koji_hub"), nevr, expected_ends=["testing-pending"], - ) - - if not args.no_waive: - nevr = nevrs[list(nevrs.keys())[0]] - utils.waive_update( - conf["bodhi-cli"], - updateid, - prod=conf["_env"] == "prod", - username=conf.get("bodhi-user"), - password=conf.get("bodhi-password"), - ) - - # Check that waiverdb announced the new waiver - utils.lookup_results_datagrepper( - base_url=conf["datagrepper"], - name="waiverdb", - topic=f"org.fedoraproject.{conf['_env']}.waiverdb.waiver.new", - nevrs=nevr_names, - ) - - # Check that greenwave reacted to the new waiver - utils.lookup_results_datagrepper( - base_url=conf["datagrepper"], - name="greenwave", - topic=f"org.fedoraproject.{conf['_env']}.greenwave.decision.update", - nevrs=nevr_names, - ) - - # Check the tag of the build -- build was waived, let is through - utils.get_build_tags( - conf.get("koji_hub"), nevr, expected_ends=conf["koji_end_tag"], - ) - - utils.finalize(start) - return utils - - -if __name__ == "__main__": - try: - main(sys.argv[1:]) - except KeyboardInterrupt: - print(" -- Interupted --") diff --git a/monitor_gating_single_build.py b/monitor_gating_single_build.py deleted file mode 100644 index 29425b0..0000000 --- a/monitor_gating_single_build.py +++ /dev/null @@ -1,313 +0,0 @@ -#!/usr/bin/env python3 - -""" -This is a script to test how dist-git, koji, bodhi, resultsdb, fedora-ci, -greenwave and waiverdb act together and if any piece of them is failing to -do its part. - -Requirements: - - Have installed on your machine - - python3-requests - - bodhi-client - - fedpkg - - fedpkg-stage - - Ensure git is configured correctly in your system (username, email...) - - You should have tests and gating setup in the package's repo you're - playing with - - Fork that repo in your name - - Make sure the repo has the f30 branch - - Ensure your ssh key is unlocked - - A valid kerberos ticket for staging and/or production - -""" - -import argparse -import datetime -import logging -import os -import sys -import tempfile - -import toml - -from utils import MonitoringUtils - -_log = logging.getLogger(__name__) - - -def get_arguments(args): - """ Parse and return the CLI arguments. - """ - parser = argparse.ArgumentParser(description="Test the CI workflow in Fedora.") - parser.add_argument( - "--nevr", help="NEVR of the build, allows by-passing: commit, push, build", - ) - parser.add_argument( - "--update", - help="Alias for the update, allows by-passing creating the udpate " "", - ) - parser.add_argument( - "--no-pr", - action="store_true", - default=False, - help="Skip testing the testing of pull-requests", - ) - parser.add_argument( - "--staging", - action="store_true", - default=False, - help="Changes to environment in which this runs from prod to staging", - ) - parser.add_argument( - "--auto-update", - action="store_true", - default=False, - help="Wait for the update to be created automatically instead of " - "doing it manually", - ) - parser.add_argument( - "--no-waive", - action="store_true", - default=False, - help="Skip the end of the workflow when the failing tests are waived", - ) - parser.add_argument( - "--conf", - default="monitor_gating.cfg", - help="Configuration file to use, specifying the URLs and all", - ) - - return parser.parse_args(args) - - -def main(args): - """ Main method used by this script. """ - start = datetime.datetime.utcnow() - - args = get_arguments(args) - - conf = toml.load(args.conf) - utils = MonitoringUtils() - - name = conf["name_single"] - namespace = conf["namespace"] - fas_username = conf["fas_username"] - branch = conf["branch"] - - with tempfile.TemporaryDirectory(prefix="ci-test-") as folder: - print(f"Working in {folder}\n") - if not args.nevr: - utils.clone_repo( - conf["fedpkg"], conf["fas_username"], namespace, name, folder=folder, - ) - gitfolder = os.path.join(folder, name) - utils.switch_branch(conf["fedpkg"], branch, folder=gitfolder) - utils.bump_release(name, folder=gitfolder) - utils.commit_changes("Bump release", folder=gitfolder) - nevr = utils.get_nevr(conf["fedpkg"], folder=gitfolder) - print(f" Upcoming build : {nevr}") - - if args.no_pr: - # Push to the main repo - utils.push_changes(gitfolder, "origin", branch) - else: - # Add the fork as remote, push to the it, open the PR, - # wait for CI to flag the PR, twice, merge the PR - utils.add_remote( - f"{fas_username}", - f"ssh://{fas_username}@{conf['distgit_host']}/" - f"forks/{fas_username}/{namespace}/{name}.git", - folder=gitfolder, - ) - utils.push_changes(gitfolder, fas_username, branch, force=True) - pr_created, pr_id, pr_uid = utils.open_pullrequest( - base_url=conf["pagure_dist_git"], - username=fas_username, - namespace=namespace, - name=name, - branch=branch, - token=conf["pagure_token"], - ) - if pr_created: - # Check that pr pipeline is running - utils.lookup_results_datagrepper( - base_url=conf["datagrepper"], - name="CI (running)", - topic=f"org.centos.{conf['_ci_env']}.ci.dist-git-pr.test.running", - rev=pr_uid, - ) - # Check that CI flag pending was set - utils.get_pr_flag( - base_url=conf["pagure_dist_git"], - username=fas_username, - namespace=namespace, - name=name, - pr_id=pr_id, - flag_username="Fedora CI", - flag_status="pending", - ) - # Check that pr pipeline has finished - utils.lookup_results_datagrepper( - base_url=conf["datagrepper"], - name="CI (complete)", - topic=f"org.centos.{conf['_ci_env']}.ci.dist-git-pr.test.error", - rev=pr_uid, - ) - # Check that CI flag failure was set - utils.get_pr_flag( - base_url=conf["pagure_dist_git"], - username=fas_username, - namespace=namespace, - name=name, - pr_id=pr_id, - flag_username="Fedora CI", - flag_status="error", - duration=25, - ) - # Merge the PR: TODO - utils.merge_pr( - base_url=conf["pagure_dist_git"], - username=fas_username, - namespace=namespace, - name=name, - pr_id=pr_id, - token=conf["pagure_token"], - ) - utils.pull_changes(gitfolder, "origin", branch) - else: - return - - # Build the package - utils.build_package(conf["fedpkg"], folder=gitfolder) - - # Check the tag of the build - utils.get_build_tags( - conf.get("koji_hub"), - nevr, - expected_ends=["updates-candidate", "signing-pending"], - ) - else: - nevr = args.nevr - - # Retrieve or create the update - updateid = utils.get_update_id(nevr, conf["bodhi"]) - if not args.update and not args.auto_update: - utils.create_update( - conf["bodhi-cli"], - nevr, - prod=conf["_env"] == "prod", - username=conf.get("bodhi-user"), - password=conf.get("bodhi-password"), - ) - print(f" Update created : {updateid}") - elif args.auto_update: - print(f" Update automatically created : {updateid}") - else: - updateid = args.update - - # Check the tag of the build - utils.get_build_tags( - conf.get("koji_hub"), - nevr, - expected_ends=["signing-pending", "testing-pending"], - ) - - if not updateid: - utils.finalize(start) - return - - # Check that bodhi notified the pipeline it can run - utils.lookup_results_datagrepper( - base_url=conf["datagrepper"], - name="bodhi to CI", - topic=f"org.fedoraproject.{conf['_env']}.bodhi.update.status." - "testing.koji-build-group.build.complete", - bodhi_id=updateid, - ) - - # Check that the CI pipeline is running - utils.lookup_results_datagrepper( - base_url=conf["datagrepper"], - name="CI (running)", - topic=f"org.centos.{conf['_ci_env']}.ci.koji-build.test.running", - nevr=nevr, - ) - # Check at the CI pipeline has completed - utils.lookup_results_datagrepper( - base_url=conf["datagrepper"], - name="CI (complete)", - topic=f"org.centos.{conf['_ci_env']}.ci.koji-build.test.error", - nevr=nevr, - duration=30, - ) - - # Check the tag of the build - utils.get_build_tags( - conf.get("koji_hub"), nevr, expected_ends=["testing-pending"], - ) - - # Check that the CI results made it to resultsdb - utils.lookup_ci_resultsdb( - nevr=nevr, name="resultsdb(phx)", url=conf["resultsdb"] - ) - - # Check that resultsdb announced the new results - utils.lookup_results_datagrepper( - base_url=conf["datagrepper"], - name="resultsdb", - topic=f"org.fedoraproject.{conf['_env']}.resultsdb.result.new", - nevr=nevr, - ) - - # Check that greenwave reacted to resultsdb's new results - utils.lookup_results_datagrepper( - base_url=conf["datagrepper"], - name="greenwave", - topic=f"org.fedoraproject.{conf['_env']}.greenwave.decision.update", - nevr=nevr, - ) - - # Check the tag of the build -- build is blocked but should be signed - utils.get_build_tags( - conf.get("koji_hub"), nevr, expected_ends=["testing-pending"], - ) - - if not args.no_waive: - utils.waive_update( - conf["bodhi-cli"], - updateid, - prod=conf["_env"] == "prod", - username=conf.get("bodhi-user"), - password=conf.get("bodhi-password"), - ) - - # Check that waiverdb announced the new waiver - utils.lookup_results_datagrepper( - base_url=conf["datagrepper"], - name="waiverdb", - topic=f"org.fedoraproject.{conf['_env']}.waiverdb.waiver.new", - nevr=nevr, - ) - - # Check that greenwave reacted to the new waiver - utils.lookup_results_datagrepper( - base_url=conf["datagrepper"], - name="greenwave", - topic=f"org.fedoraproject.{conf['_env']}.greenwave.decision.update", - nevr=nevr, - ) - - # Check the tag of the build -- build was waived, let is through - utils.get_build_tags( - conf.get("koji_hub"), nevr, expected_ends=conf["koji_end_tag"], - ) - - utils.finalize(start) - return utils - - -if __name__ == "__main__": - try: - main(sys.argv[1:]) - except KeyboardInterrupt: - print(" -- Interupted --") diff --git a/runner.py b/runner.py deleted file mode 100644 index d8ae268..0000000 --- a/runner.py +++ /dev/null @@ -1,191 +0,0 @@ -""" -This script is meant to run the different tests that we have sequentially, with -a scheduler, ie: after running all the tests, it will wait for a specified -amount of time and then run them again, until it's stopped. -""" - -import argparse -import datetime -import sched -import sys -import time -import uuid - -import fedora_messaging.api -import fedora_messaging.exceptions - -import monitor_gating_multi_builds - -import monitor_gating_single_build - -import toml - -from utils import MonitoringUtils, blocking_issues, run_command - -s = sched.scheduler(time.time, time.sleep) -conf = toml.load - - -def get_arguments(args): - """ Load and parse the CLI arguments.""" - parser = argparse.ArgumentParser(description="Runner for the CI canary tests.") - parser.add_argument( - "conf", help="Configuration file for the different tests", - ) - - return parser.parse_args(args) - - -def notify(topic, message): - try: - msg = fedora_messaging.api.Message( - topic="monitor-gating.{}".format(topic), body=message - ) - fedora_messaging.api.publish(msg) - except fedora_messaging.exceptions.PublishReturned as err: - print(f"Fedora Messaging broker rejected message {msg.id}: {err}") - except fedora_messaging.exceptions.ConnectionException as err: - print(f"Error sending message {msg.id}: {err}") - except Exception as err: - print(f"Error sending fedora-messaging message: {err}") - - -def _clean_up_side_tags(utils): - try: - print(" Removing side-tag: %s" % utils.side_tag_name) - cmd = [conf["fedpkg"], "remove-side-tag", utils.side_tag_name] - run_command(cmd) - except Exception: - pass - - -def schedule(conf): - """ Run the test and schedules the next one. """ - - if conf.get("kb_principal") and conf.get("kb_keytab_file"): - print(f"Logging into kerberos using: {conf['kb_keytab_file']}") - cmd = ["kinit", conf["kb_principal"], "-kt", conf["kb_keytab_file"]] - run_command(cmd) - - delay = conf["delay"] - report_project = conf["pagure_report_project"] - report_api_token = conf["pagure_api_token"] - report_env = conf["env"] - - print("Tests started:", datetime.datetime.utcnow(), flush=True) - runid = f"{datetime.datetime.utcnow().year}-{uuid.uuid4()}" - try: - # Single Build Gating - single_args = conf["workflow_single_gating_args"].split() - notify( - topic=f"single-build.start", - message={"arguments": single_args, "runid": runid}, - ) - monit_utils = monitor_gating_single_build.main(single_args) - output_text = "\n".join(monit_utils.logs) - if "[FAILED]" not in output_text: - result = "succeeded" - else: - result = "failed" - report_failure( - report_project, - report_api_token, - report_env, - "single-package", - monit_utils, - ) - - notify( - topic=f"single-build.end.{result}", - message={ - "output": monit_utils.logs, - "output_text": output_text, - "result": result, - "runid": runid, - "failed": monit_utils.failed, - }, - ) - - # Multi Build Gating - multi_args = conf["workflow_multi_gating_args"].split() - monit_utils = MonitoringUtils() - notify( - topic=f"multi-build.start", - message={"arguments": multi_args, "runid": runid}, - ) - try: - monit_utils = monitor_gating_multi_builds.main( - multi_args, utils=monit_utils - ) - finally: - _clean_up_side_tags(monit_utils) - - output_text = "\n".join(monit_utils.logs) - if "[FAILED]" not in output_text: - result = "succeeded" - else: - result = "failed" - report_failure( - report_project, - report_api_token, - report_env, - "multi-package", - monit_utils, - ) - - notify( - topic=f"multi-build.end.{result}", - message={ - "output": monit_utils.logs, - "output_text": output_text, - "result": result, - "runid": runid, - "failed": monit_utils.failed, - }, - ) - - print("Tests finished:", datetime.datetime.utcnow(), flush=True) - except Exception as err: - print(f"Tests failed with: {err}", flush=True) - print(sys.exc_info()[0]) - - notify( - topic=f"multi-build.end.error", message={"runid": runid, "exception": err}, - ) - - delay_when_failing = conf["delay_when_failing"] - blocker_tags = conf["blocker_tags"] - blocking_project = conf["pagure_blocking_project"] - - blocking_issues_list = blocking_issues(blocking_project, blocker_tags) - now = datetime.datetime.utcnow().strftime("%H:%M:%S") - if blocking_issues_list: - print( - f"{now} Next run in: {delay_when_failing} seconds because of " - f"{len(blocking_issues_list)} open issues", - flush=True, - ) - s.enter(delay_when_failing, 1, schedule, argument=(conf,)) - else: - print(f"{now} Next run in: {delay} seconds", flush=True) - s.enter(delay, 1, schedule, argument=(conf,)) - - -def main(args): - """ Schedule the first test and run the scheduler. """ - args = get_arguments(args) - conf = toml.load(args.conf) - s.enter(0, 1, schedule, argument=(conf,)) - s.run() - - -if __name__ == "__main__": - try: - main(sys.argv[1:]) - except KeyboardInterrupt: - from code import InteractiveConsole - - InteractiveConsole(locals={"s": s}).interact( - "ENTERING THE DEBUG CONSOLE:\n s is the scheduler\n ^d to quit", - "LEAVING THE DEBUG CONSOLE", - ) diff --git a/utils.py b/utils.py deleted file mode 100644 index 6b0c027..0000000 --- a/utils.py +++ /dev/null @@ -1,794 +0,0 @@ -#!/usr/bin/env python3 - -""" -This is a small library of utility methods used by the monitoring scripts. - -""" - -import ast -import datetime -import logging -import os -import subprocess -import time - -import requests - -_log = logging.getLogger(__name__) - - -def report_failure(project, token, env, workflow, monit_utils): - """ Open a pagure ticket against the instance specified in the - configuration file when something does not work. - """ - url = f"https://pagure.io/api/0/{project}/new_issue" - title = f"Failure in {env} of the {workflow} packager workflow" - logs = "\n".join(monit_utils.logs) - content = f"""A run of monitor-gating has just failed in {env} for the {workflow} workflow. - -The suspects are '{", ".join(monit_utils.failed)}'. - -Full log: -```` -{logs} -```` -""" - tag = env - - data = { - "title": title, - "content": content, - "tag": tag, - } - headers = { - "Authorization": f"token {token}", - } - - req = requests.post(url, data=data, headers=headers) - if not req.ok: - print(f"Error when trying to open a ticket at: {url} to report the failure") - - -def blocking_issues(project, tags): - """Lists blocking issues we track in the fedora-infrastructure project. - """ - if not tags: - print(f"No tags to filter blocking issues by, returning empty.") - return [] - - api = f"https://pagure.io/api/0/{project}/issues?status=Open&tags={tags[0]}" - issues = [] - try: - r = requests.get(api) - issues = r.json()["issues"] - if tags: - t = set(tags[1:]) - issues = [i for i in issues if t & set(i["tags"])] - for i in issues: - print(f"Found blocking issue https://pagure.io/{project}/issue/{i['id']}") - except Exception as e: - print(f"Error when querying pagure for blocking issues: {e}") - return issues - - -class MonitoringException(Exception): - """The base class for all exceptions raised by this script.""" - - -class MonitoringUtils: - def __init__(self): - """ Instanciate the object. """ - self.logs = [] - self.failed = [] - - def print_user(self, content, success=None): - """ Prints the specified content to the user. - """ - spaces = 90 - if success is not None: - end = None - if success: - content = "{} {}".format(content.ljust(spaces), "[DONE]") - else: - content = "{} {}".format(content.ljust(spaces), "[FAILED]") - else: - if os.environ.get("OPENSHIFT"): - end = None - else: - end = "\r" - - now = datetime.datetime.utcnow() - time = now.strftime("%H:%M:%S") - self.logs.append(f"{time} - {content}") - print(f"{time} - {content}", end=end, flush=True) - - def clone_repo(self, command, username, namespace, name, folder): - """ Clone the specified git repo into the specified folder. - """ - info_log = f"Cloning as {username} the git repo: {namespace}/{name}" - self.print_user(info_log) - try: - run_command( - [command, "--user", username, "clone", f"{namespace}/{name}"], - cwd=folder, - ) - except MonitoringException: - self.failed.append("git/dist-git") - self.print_user(info_log, success=False) - else: - try: - # Assume these commands can't fail - clone_folder = os.path.join(folder, name) - run_command( - ["git", "config", "user.name", "packagerbot"], cwd=clone_folder, - ) - run_command( - ["git", "config", "user.email", "admin@fedoraproject.org"], - cwd=clone_folder, - ) - self.print_user(info_log, success=True) - except MonitoringException: - self.failed.append("git") - self.print_user(info_log, success=False) - - def add_remote(self, name, url, folder): - """ Add the specified remote to the git repo in the folder with the - specified url. - """ - info_log = f"Adding remote: {name}" - self.print_user(info_log) - try: - run_command(["git", "remote", "add", name, url], cwd=folder) - self.print_user(info_log, success=True) - except MonitoringException: - self.failed.append("git") - self.print_user(info_log, success=False) - - def switch_branch(self, command, name, folder): - """ Switch to the specified git branch in the specified git repo. - """ - info_log = f"Switching to branch: {name}" - self.print_user(info_log) - try: - run_command([command, "switch-branch", f"{name}"], cwd=folder) - self.print_user(info_log, success=True) - except MonitoringException: - self.failed("fedpkg") - self.print_user(info_log, success=False) - - def bump_release(self, name, folder): - """ Bump the release of the spec file the specified git repo. - """ - info_log = f"Bumping release of: {name}.spec" - self.print_user(info_log) - try: - run_command(["rpmdev-bumpspec", f"{name}.spec"], cwd=folder) - self.print_user(info_log, success=True) - except MonitoringException: - self.failed("rpmdev-bumspec") - self.print_user(info_log, success=False) - - def commit_changes(self, commit_log, folder): - """ Commit all the changes made to *tracked* files in the git repo - with the specified commit log. - """ - info_log = f"Commiting changes" - self.print_user(info_log) - try: - run_command(["git", "commit", "-asm", commit_log], cwd=folder) - self.print_user(info_log, success=True) - except MonitoringException: - self.failed("git") - self.print_user(info_log, success=False) - - def push_changes(self, folder, target, branch, force=False): - """ Push all changes using git. - """ - info_log = f"Pushing changes" - self.print_user(info_log) - try: - cmd = ["git", "push", target, branch] - if force: - cmd.append("-f") - run_command(cmd, cwd=folder) - self.print_user(info_log, success=True) - except MonitoringException: - self.failed("git/dist-git") - self.print_user(info_log, success=False) - - def pull_changes(self, folder, target, branch): - """ Pull all changes using git. - """ - info_log = f"Pulling changes" - self.print_user(info_log) - try: - cmd = ["git", "pull", "--rebase", target, branch] - run_command(cmd, cwd=folder) - self.print_user(info_log, success=True) - except MonitoringException: - self.failed("git/dist-git") - self.print_user(info_log, success=False) - - def open_pullrequest(self, base_url, username, namespace, name, branch, token): - """ Open a pull-request from the user's fork to the main project for - the specified branch. - """ - info_log = f"Creating PR from forks/{username}/{namespace}/{name}" - self.print_user(info_log) - url = "/".join( - [base_url.rstrip("/"), "api/0", namespace, name, "pull-request/new"] - ) - data = { - "branch_to": branch, - "branch_from": branch, - "repo_from": name, - "repo_from_username": username, - "repo_from_namespace": namespace, - "initial_comment": "Testing PR", - "title": "Test PR for monitoring", - } - headers = {"Authorization": f"token {token}"} - req = requests.post(url=url, data=data, headers=headers) - if not req.ok: - print(req.text) - success = False - pr_id = None - pr_uid = None - self.failed("dist-git") - else: - output = req.json() - pr_id = str(output["id"]) - pr_uid = output["uid"] - url = "/".join( - [base_url.rstrip("/"), namespace, name, "pull-request", pr_id] - ) - info_log = f"PR created {url}" - success = True - self.print_user(info_log, success=success) - return (success, pr_id, pr_uid) - - def get_nevr(self, command, folder): - """ Get the name-epoch-version-release presently in git - """ - info_log = f"Getting nevr" - self.print_user(info_log) - try: - nevr = run_command([command, "verrel"], cwd=folder) - self.print_user(info_log, success=True) - return nevr.strip().decode("utf-8") - except MonitoringException: - self.failed("fedpkg") - self.print_user(info_log, success=False) - - def build_package(self, command, folder, target=None): - """ Build the package in the current branch - """ - info_log = f"Building the package" - self.print_user(info_log) - command = [command, "build"] - if target: - command.extend(["--target", target]) - try: - run_command(command, cwd=folder) - self.print_user(info_log, success=True) - except MonitoringException: - self.failed("koji") - self.print_user(info_log, success=False) - - def chain_build_packages(self, command, packages, folder, target=None): - """ Chain-build the packages in the current branch - """ - if not isinstance(packages, list): - packages = [packages] - info_log = ( - f"Chain-building the packages: {packages + [os.path.basename(folder)]}" - ) - self.print_user(info_log) - command = [command, "chain-build"] - command.extend(packages) - if target: - command.extend(["--target", target]) - try: - run_command(command, cwd=folder) - self.print_user(info_log, success=True) - except MonitoringException: - self.failed("koji") - self.print_user(info_log, success=False) - - def get_build_tags(self, koji_url, nevr, expected_ends): - """ List the tags associated with the specified build. - """ - # return - start = datetime.datetime.utcnow() - info_log = f"Retrieving koji tags" - self.print_user(info_log) - command = [ - "koji", - ] - if koji_url: - command.extend(["-s", koji_url]) - command.extend(["call", "listTags", nevr]) - - success = False - tags = None - broke = False - while True: - try: - output = run_command(command) - output = output.decode("utf-8") - try: - data = ast.literal_eval(output.strip()) - except Exception: - print("Could not decode JSON in:") - print(command) - print(output) - broke = True - break - tags = [tag.get("name") for tag in data] - for tag_name in tags: - for expectation in expected_ends: - if tag_name.endswith(expectation): - success = True - broke = True - break - if success: - broke = True - break - if broke: - break - - if (datetime.datetime.utcnow() - start).seconds > (15 * 60): - success = False - info_log = f"Update for {nevr} not created within 15 minutes" - break - - # Only query koji every 30 seconds - time.sleep(30) - except MonitoringException: - success = False - break - - info_log = f"Retrieving koji tags: {tags}" - if not success: - self.failed("koji") - self.print_user(info_log, success=success) - - def create_update( - self, command, item, prod=True, username=None, password=None, from_tag=False, - ): - """ Create the update for the package built. - """ - info_log = f"Creating a bodhi update" - self.print_user(info_log) - command = [ - command, - "updates", - "new", - "--notes", - "Bump release to test CI", - "--type", - "bugfix", - "--autotime", - ] - if from_tag: - command.append("--from-tag") - command.append(item) - - if not prod: - command.append("--staging") - if username: - command.extend(["--user", username]) - if password: - command.extend(["--password", password]) - try: - run_command(command) - self.print_user(info_log, success=True) - except MonitoringException: - self.failed("bodhi") - self.print_user(info_log, success=False) - - def get_update_id(self, nevr, url): - """ Retrieve the update identifier from bodhi for the given nevr. """ - start = datetime.datetime.utcnow() - info_log = f"Retrieving update created" - self.print_user(info_log) - url = f"{url}/updates/?builds={nevr}" - updateid = None - success = True - while True: - req = requests.get(url) - data = req.json() - if data["updates"]: - updateid = data["updates"][0]["updateid"] - if updateid: - break - - if (datetime.datetime.utcnow() - start).seconds > (15 * 60): - success = False - self.failed("bodhi") - info_log = f"Update for {nevr} not created within 15 minutes" - break - - # Only query bodhi every 30 seconds - time.sleep(30) - - self.print_user(info_log, success=success) - return updateid - - def lookup_results_datagrepper( - self, - base_url, - name, - topic, - nevr=None, - nevrs=None, - rev=None, - bodhi_id=None, - start=None, - duration=15, - ): - """ Check the CI results in datagrepper for results about our specified - build. - """ - start_lookup = datetime.datetime.utcnow() - if start is None: - start = start_lookup - info_log = f"Checking datagrepper for {name} messages" - self.print_user(info_log) - # Start pulling messages 10 minutes before now - start_time = start - datetime.timedelta(minutes=10) - # Limiting the number of row per page to 10 allows for quicker results - url = ( - base_url + f"?topic={topic}" - f"&start={start_time.timestamp()}&row_per_page=10" - ) - - success = None - returned_status = None - info_log = None - nevrs = nevrs or [] - while True: - # We're assuming here that there won't be more than 100 messages for - # that topic coming in between the one we're interested in and when we - # are looking for it (10*10 == 100) - for page in range(1, 11): - end_url = url - end_url += f"&page={page}" - data = requests.get(end_url).json() - if "raw_messages" not in data: - nomsg_log = f"No messages in data-grepper on {end_url} " - if "error" in data: - nomsg_log += data["error"] - self.print_user(nomsg_log) - break - - for message in data["raw_messages"]: - - # Old message format from the CI pipeline - if "ci.pipeline" in message["topic"] and ( - message["msg"]["nvr"] == nevr - or message["msg"]["nvr"] in nevrs - or message["msg"]["rev"] == rev - ): - success = True - returned_status = message["msg"]["status"] - break - - # New message format from the CI pipeline for koji builds - if ( - "ci.koji-build" in message["topic"] - and message["msg"]["artifact"]["nvr"] == nevr - ): - if message["topic"].endswith("test.complete"): - success = True - returned_status = message["msg"]["test"]["result"] - elif message["topic"].endswith("test.error"): - success = True - returned_status = "error" - elif message["topic"].endswith("test.running"): - success = True - returned_status = "running" - break - - # New message format from the CI pipeline for dist-git PR - if ( - "ci.dist-git-pr" in message["topic"] - and message["msg"]["artifact"]["type"] == "pull-request" - and message["msg"]["artifact"]["uid"] == rev - ): - if message["topic"].endswith("test.complete"): - success = True - returned_status = message["msg"]["test"]["result"] - elif message["topic"].endswith("test.error"): - success = True - returned_status = "error" - elif message["topic"].endswith("test.running"): - success = True - returned_status = "running" - break - - # resultsdb messages - if ( - "resultsdb" in message["topic"] - and "nvr" in message["msg"]["data"] - and ( - nevr in message["msg"]["data"]["nvr"] - or message["msg"]["data"]["nvr"] in nevrs - ) - ): - success = True - returned_status = message["msg"]["outcome"] - break - - # greenwave messages - if "greenwave" in message["topic"] and ( - message["msg"]["subject_identifier"] == nevr - or message["msg"]["subject_identifier"] in nevrs - ): - success = True - returned_status = message["msg"]["policies_satisfied"] - break - - # waiverdb messages - if "waiverdb" in message["topic"] and ( - message["msg"]["subject_identifier"] == nevr - or message["msg"]["subject_identifier"] in nevrs - ): - success = True - returned_status = "" - break - - # bodhi messages - if "bodhi.update.status.testing" in message["topic"] and message[ - "msg" - ]["artifact"]["id"].startswith(bodhi_id): - success = True - returned_status = "" - break - if success is not None: - break - if success is not None: - break - - if (datetime.datetime.utcnow() - start).seconds > (duration * 60): - success = False - info_log = f"{name} results not found in datagrepper" - break - - # Only query datagrepper every 30 seconds - time.sleep(30) - - if info_log is None: - info_log = f"{name} results in datagrepper returned {returned_status}" - - end = datetime.datetime.utcnow() - info_log += f" - ran for: {(end - start_lookup).seconds}s" - self.print_user(info_log, success=success) - if not success: - self.failed("datagrepper") - - def lookup_ci_resultsdb(self, nevr, name, url): - """ Check the CI results in the specified resultsdb for results about - our specified build. - """ - start = datetime.datetime.utcnow() - info_log = f"Checking {name} for CI results " - self.print_user(info_log) - topic = "org.centos.prod.ci.pipeline.allpackages-build.complete" - if ".stg" in url: - topic = "org.centos.stage.ci.pipeline.allpackages-build.complete" - url = f"{url}?testcases={topic}" - - success = False - returned_status = None - info_log = None - while True: - # Assume we won't have more than 3 pages of results coming in b/w - # our checks - for page in [0, 1, 2]: - end_url = url - end_url += f"&page={page}" - data = requests.get(end_url).json() - for result in data["data"]: - if nevr in result["data"]["nvr"]: - success = True - returned_status = result["data"]["status"][0] - break - if success: - break - if success: - break - - if (datetime.datetime.utcnow() - start).seconds > (15 * 60): - success = False - info_log = ( - f"CI results did not show in {name} for {nevr} within 15 minutes" - ) - break - - # Only query datagrepper every 30 seconds - time.sleep(30) - - if info_log is None: - info_log = f"CI results in {name} returned {returned_status}" - - end = datetime.datetime.utcnow() - info_log += f" - ran for: {(end - start).seconds}s" - self.print_user(info_log, success=success) - if not success: - self.failed("resultsdb") - - def waive_update(self, command, updateid, prod=True, username=None, password=None): - """ Waive all the tests results for the specified update using bodhi's - CLI. - """ - info_log = f"Waiving test results for bodhi update" - self.print_user(info_log) - command = [ - command, - "updates", - "waive", - updateid, - "'This is fine, we are testing the workflow'", - "--debug", - ] - if not prod: - command.append("--staging") - if username: - command.extend(["--user", username]) - if password: - command.extend(["--password", password]) - try: - run_command(command) - self.print_user(info_log, success=True) - except MonitoringException: - self.failed("waiverdb") - self.print_user(info_log, success=False) - - def get_pr_flag( - self, - base_url, - username, - namespace, - name, - pr_id, - flag_username, - flag_status, - duration=10, - ): - """ Retrieve the flags of the PR and assert the last one from the - specified flag_username has the given status. - """ - pr = "/".join([namespace, name, "pull-request", pr_id]) - info_log = f"Retreiving flags for PR: {pr}" - self.print_user(info_log) - url = "/".join([base_url.rstrip("/"), "api/0", pr, "flag"]) - - start = datetime.datetime.utcnow() - success = False - - while True: - try: - req = requests.get(url=url) - except requests.exceptions.ConnectionError: - continue - - if req.ok: - break - - if (datetime.datetime.utcnow() - start).seconds > (duration * 60): - success = False - info_log = f"Failed to retrieve flags for PR: {pr}" - break - - # Only query pagure every 30 seconds - time.sleep(30) - - if not req.ok: - print(req.text) - self.logs.append(f"Error retrieving PR flags: {req.text}") - raise MonitoringException("Error retrieving PR flags") - else: - output = req.json() - for flag in output["flags"]: - if flag["username"] == flag_username: - info_log = f"Retreived flag {flag['status']} on PR" - success = flag["status"] == flag_status - break - - self.print_user(info_log, success=success) - if not success: - self.failed("dist-git") - - def merge_pr(self, base_url, username, namespace, name, pr_id, token): - """ Merge the specified PR - """ - pr = "/".join([namespace, name, "pull-request", pr_id]) - info_log = f"Merge PR: {pr}" - self.print_user(info_log) - url = "/".join([base_url.rstrip("/"), "api/0", pr, "merge"]) - headers = {"Authorization": f"token {token}"} - req = requests.post(url=url, data={"wait": True}, headers=headers) - success = False - if not req.ok: - print(req.text) - self.logs(f"Error Merging flags: {req.text}") - raise MonitoringException("Error merging flags") - else: - success = True - - self.print_user(info_log, success=success) - if not success: - self.failed("dist-git") - - def finalize(self, start): - """ End data returned. """ - end = datetime.datetime.utcnow() - delta = (end - start).seconds - self.logs.append(f"Ran for {delta} seconds ({delta/60:.2f} minutes)") - print(f"Ran for {delta} seconds ({delta/60:.2f} minutes)") - - def create_side_tag(self, command, folder): - """ Create a side-tag to build packages in it. """ - info_log = f"Creating the side-tag" - self.print_user(info_log) - command = [command, "request-side-tag"] - side_tag_name = None - try: - output = run_command(command, cwd=folder) - lines = output.decode("utf-8").split("\n") - _, side_tag_name, _ = lines[0].split("'") - info_log = f"Created side-tag {side_tag_name}" - self.print_user(info_log, success=True) - except (MonitoringException, Exception) as err: - print(err) - self.failed("dist-git") - self.print_user(info_log, success=False) - return side_tag_name - - def clone_and_bump( - self, folder, nevrs, conf, name, target=None, new_side_tag=False - ): - """Clone the repo, bump the release, commit and push.""" - namespace = conf["namespace"] - branch = conf["branch"] - - self.clone_repo( - conf["fedpkg"], conf["fas_username"], namespace, name, folder=folder, - ) - gitfolder = os.path.join(folder, name) - self.switch_branch(conf["fedpkg"], branch, folder=gitfolder) - side_tag_name = None - # Create a side-tag - if new_side_tag: - side_tag_name = self.create_side_tag(conf["fedpkg"], folder=gitfolder) - target = side_tag_name - self.bump_release(name, folder=gitfolder) - self.commit_changes("Bump release", folder=gitfolder) - nevr = self.get_nevr(conf["fedpkg"], folder=gitfolder) - nevrs[name] = nevr - # Push to the main repo - self.push_changes(gitfolder, "origin", branch) - print(f" Upcoming build : {nevr}") - return (nevrs, target) - - -def run_command(command, cwd=None): - """ Run the specified command in a specific working directory if one - is specified. - """ - output = None - try: - output = subprocess.check_output(command, cwd=cwd, stderr=subprocess.PIPE) - except subprocess.CalledProcessError as e: - if "--password" in command: - idx = command.index("--password") - command[idx + 1] = "" - _log.error( - "Command `{}` return code: `{}`".format(" ".join(command), e.returncode) - ) - _log.error("stdout:\n-------\n{}".format(e.stdout)) - _log.error("stderr:\n-------\n{}".format(e.stderr)) - raise MonitoringException("Command failed to run") - - return output From 0a81be65d4b6974d1ead45e405566287c84127f4 Mon Sep 17 00:00:00 2001 From: Nils Philippsen Date: May 25 2020 07:53:02 +0000 Subject: [PATCH 2/4] Make container entrypoint script more flexible Previously, the container entrypoint (which is hard to override) invariably called the Python script. Now the entrypoint script will run anything passed as arguments to it, and the container will default to running the monitor-gating script. This is to let users e.g. run a shell to inspect things more easily. Signed-off-by: Nils Philippsen --- diff --git a/Dockerfile b/Dockerfile index 34f29c4..61b17ec 100644 --- a/Dockerfile +++ b/Dockerfile @@ -22,4 +22,5 @@ RUN echo "packagerbot" > /.fedora.upn \ USER 1000 WORKDIR / -ENTRYPOINT ["sh", "/opt/code/entrypoint.sh"] +ENTRYPOINT ["/opt/code/entrypoint.sh"] +CMD ["python3", "/opt/code/runner.py", "/opt/config/runner.cfg"] diff --git a/entrypoint.sh b/entrypoint.sh index 5c2d603..28a65af 100755 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -17,4 +17,4 @@ echo "Running in production" ssh-keyscan pkgs.fedoraproject.org >> /.ssh/known_hosts fi -python3 /opt/code/runner.py /opt/config/runner.cfg +exec "$@" From f7a4628135b87803a5a0892c6101dfd1dcab7a9c Mon Sep 17 00:00:00 2001 From: Nils Philippsen Date: May 25 2020 07:53:02 +0000 Subject: [PATCH 3/4] Dockerfile: install default configuration file Signed-off-by: Nils Philippsen --- diff --git a/Dockerfile b/Dockerfile index 61b17ec..1cadbb3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -9,9 +9,12 @@ RUN dnf -y install python3-requests bodhi-client fedpkg fedpkg-stage \ COPY . /opt/code -RUN echo "packagerbot" > /.fedora.upn \ +RUN cd /opt/code && mkdir /opt/config && cp runner.cfg /opt/config \ + && echo "packagerbot" > /.fedora.upn \ && chgrp -R 0 /opt/code \ && chmod -R g=u /opt/code \ + && chgrp -R 0 /opt/config \ + && chmod -R g=u /opt/config \ && chgrp -R 0 /.ssh \ && chmod -R g=u /.ssh \ && chgrp -R 0 /.fedora \ From 5d921d3add473b28fcc96da51f7e786472899f7f Mon Sep 17 00:00:00 2001 From: Nils Philippsen Date: May 25 2020 07:53:02 +0000 Subject: [PATCH 4/4] Add and use setup.py, requirements.txt In the course, designate the team as maintainers of the container image, too. Signed-off-by: Nils Philippsen --- diff --git a/Dockerfile b/Dockerfile index 1cadbb3..c63ef54 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,7 @@ # This Dockerfile is used to build and run monitor-gating on Openshift FROM fedora:31 -LABEL maintainer "Pierre-Yves Chibon " +LABEL maintainer "Fedora Infrastructure Team " RUN dnf -y install python3-requests bodhi-client fedpkg fedpkg-stage \ python3-toml git python3-koji python3-fedora-messaging cracklib-dicts \ @@ -9,7 +9,7 @@ RUN dnf -y install python3-requests bodhi-client fedpkg fedpkg-stage \ COPY . /opt/code -RUN cd /opt/code && mkdir /opt/config && cp runner.cfg /opt/config \ +RUN cd /opt/code && python3 setup.py install && mkdir /opt/config && cp runner.cfg /opt/config \ && echo "packagerbot" > /.fedora.upn \ && chgrp -R 0 /opt/code \ && chmod -R g=u /opt/code \ @@ -26,4 +26,4 @@ RUN cd /opt/code && mkdir /opt/config && cp runner.cfg /opt/config \ USER 1000 WORKDIR / ENTRYPOINT ["/opt/code/entrypoint.sh"] -CMD ["python3", "/opt/code/runner.py", "/opt/config/runner.cfg"] +CMD ["monitor-gating", "/opt/config/runner.cfg"] diff --git a/monitor_gating/clean_up_side_tags.py b/monitor_gating/clean_up_side_tags.py index 982cc06..9c3e994 100644 --- a/monitor_gating/clean_up_side_tags.py +++ b/monitor_gating/clean_up_side_tags.py @@ -43,7 +43,9 @@ def run_command(command) -> bytes: return output -def main(args): +def main(): + """ Main method. """ + args = get_cli_args(sys.argv[1:]) conf = toml.load(args.conf) if conf.get("kb_principal") and conf.get("kb_keytab_file"): @@ -72,7 +74,4 @@ def main(args): if __name__ == "__main__": - """ Main method. """ - - args = get_cli_args(sys.argv[1:]) - main(args) + main() diff --git a/monitor_gating/runner.py b/monitor_gating/runner.py index 924605d..413b94a 100644 --- a/monitor_gating/runner.py +++ b/monitor_gating/runner.py @@ -16,8 +16,8 @@ import fedora_messaging.exceptions import toml -from . import single_build from . import multi_builds +from . import single_build from .utils import MonitoringUtils, blocking_issues, run_command @@ -168,17 +168,13 @@ def schedule(conf): s.enter(delay, 1, schedule, argument=(conf,)) -def main(args): - """ Schedule the first test and run the scheduler. """ - args = get_arguments(args) - conf = toml.load(args.conf) - s.enter(0, 1, schedule, argument=(conf,)) - s.run() - - -if __name__ == "__main__": +def main(): + """ Main method. """ try: - main(sys.argv[1:]) + args = get_arguments(sys.argv[1:]) + conf = toml.load(args.conf) + s.enter(0, 1, schedule, argument=(conf,)) + s.run() except KeyboardInterrupt: from code import InteractiveConsole @@ -186,3 +182,7 @@ if __name__ == "__main__": "ENTERING THE DEBUG CONSOLE:\n s is the scheduler\n ^d to quit", "LEAVING THE DEBUG CONSOLE", ) + + +if __name__ == "__main__": + main() diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..778e4a6 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,3 @@ +fedora_messaging +requests +toml diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..7638255 --- /dev/null +++ b/setup.py @@ -0,0 +1,66 @@ +import os + +from setuptools import setup + + +def get_requirements(): + requirements = [] + with open("requirements.txt", "r") as f: + for line in f: + line = line.strip() + before, _, after = line.partition("#") + # Allow source control references for development. + if before.startswith("git+"): + if not after: + # The name is in the fragment + continue + _, _, requirement = after.rpartition("=") + requirement, _, _ = requirement.partition("#") + else: + requirement = before + + requirement = requirement.strip() + + if requirement: + requirements.append(requirement) + return requirements + + +here = os.path.abspath(os.path.dirname(__file__)) +with open(os.path.join(here, "README.rst"), "r") as f: + README = f.read() + + +setup( + name="monitor-gating", + version="0.0.1", + # Possible options are at https://pypi.python.org/pypi?%3Aaction=list_classifiers + classifiers=[ + "Development Status :: 5 - Alpha", + "Intended Audience :: Developers", + "Intended Audience :: System Administrators", + # 'License :: OSI Approved :: GNU General Public License v2 or later (GPLv2+)', + "Operating System :: POSIX :: Linux", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.6", + "Programming Language :: Python :: 3.7", + "Topic :: System :: Software Distribution", + ], + # license=LICENSE, + maintainer="Fedora Infrastructure Team", + maintainer_email="infrastructure@lists.fedoraproject.org", + platforms=["Fedora", "GNU/Linux"], + url="https://pagure.io/fedora-ci/monitor-gating", + description="Monitor the health of gating in Fedora", + long_description=README, + keywords="fedora", + packages=["monitor_gating"], + include_package_data=True, + zip_safe=False, + install_requires=get_requirements(), + entry_points=""" + [console_scripts] + monitor-gating = monitor_gating.runner:main + monitor-gating-clean-up-side-tags = monitor_gating.clean_up_side_tags:main + """, +)