From 4a797f53ec07d6722c77d288e6f1ff7211e1e3f3 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: May 18 2020 13:03:43 +0000 Subject: [PATCH 1/3] Make configurable the project which slows down the subsequent runs Signed-off-by: Pierre-Yves Chibon --- diff --git a/runner.cfg b/runner.cfg index 1be6a44..3879950 100644 --- a/runner.cfg +++ b/runner.cfg @@ -4,8 +4,6 @@ delay = 3600 # Time between two blocked runs in second delay_when_failing = 43200 -# blocker issue tags, issue has to have all of them -blocker_tags = ['packager_workflow_blocker', 'staging'] # CLI arguments to give to the script testing the single build gating workflow workflow_single_gating_args = "--conf monitor_gating_stg.cfg --auto-update --no-pr" @@ -20,3 +18,8 @@ workflow_multi_gating_args = "--conf monitor_gating_stg.cfg" # kb_keytab_file = "/etc/keytabs/monitor-gating-keytab" fedpkg = "fedpkg" +# Project whose issue will slow down the subsequent runs (delay defined +# above). +pagure_blocking_project = "fedora-infrastructure" +# blocker issue tags, issue has to have all of them. +blocker_tags = ['packager_workflow_blocker', 'staging'] diff --git a/runner.py b/runner.py index c8bd05c..069366a 100644 --- a/runner.py +++ b/runner.py @@ -68,8 +68,6 @@ def schedule(conf): run_command(cmd) delay = conf["delay"] - delay_when_failing = conf["delay_when_failing"] - blocker_tags = conf["blocker_tags"] print("Tests started:", datetime.datetime.utcnow(), flush=True) runid = f"{datetime.datetime.utcnow().year}-{uuid.uuid4()}" try: @@ -135,7 +133,11 @@ def schedule(conf): topic=f"multi-build.end.error", message={"runid": runid, "exception": err}, ) - blocking_issues_list = blocking_issues(blocker_tags) + delay_when_failing = conf["delay_when_failing"] + blocker_tags = conf["blocker_tags"] + blocking_project = conf["pagure_blocking_project"] + + blocking_issues_list = blocking_issues(blocking_project, blocker_tags) now = datetime.datetime.utcnow().strftime("%H:%M:%S") if blocking_issues_list: print( diff --git a/utils.py b/utils.py index 8f331ca..e63a5ed 100644 --- a/utils.py +++ b/utils.py @@ -17,25 +17,23 @@ import requests _log = logging.getLogger(__name__) -def blocking_issues(tags): +def blocking_issues(project, tags): """Lists blocking issues we track in the fedora-infrastructure project. """ if not tags: print(f"No tags to filter blocking issues by, returning empty.") return [] - api = f"https://pagure.io/api/0/fedora-infrastructure/issues" - q = f"?status=Open&tags={tags[0]}" + + api = f"https://pagure.io/api/0/{project}/issues?status=Open&tags={tags[0]}" issues = [] try: - r = requests.get(api + q) + r = requests.get(api) issues = r.json()["issues"] if tags: t = set(tags[1:]) issues = [i for i in issues if t & set(i["tags"])] for i in issues: - print( - f"Found blocking issue https://pagure.io/fedora-infrastructure/issue/{i['id']}" - ) + print(f"Found blocking issue https://pagure.io/{project}/issue/{i['id']}") except Exception as e: print(f"Error when querying pagure for blocking issues: {e}") return issues From 94c121d0c59f56643e19cb2c76425bc1d2a9a5f0 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: May 25 2020 07:48:53 +0000 Subject: [PATCH 2/3] Add support for opening a ticket on a specific project when a test fails When a test fails, it will now automatically open a ticket on a specified project hosted on pagure.io with the information it has about why the run failed. Signed-off-by: Pierre-Yves Chibon --- diff --git a/runner.cfg b/runner.cfg index 3879950..a1c8c31 100644 --- a/runner.cfg +++ b/runner.cfg @@ -18,8 +18,22 @@ workflow_multi_gating_args = "--conf monitor_gating_stg.cfg" # kb_keytab_file = "/etc/keytabs/monitor-gating-keytab" fedpkg = "fedpkg" + +# The configuration key below are used when interacting with pagure projects +# There are two ways monitor-gating interacts with them. +# a) it monitors a specific project to slows down its run in case a known issue +# prevents the workflow from working (so as to now increase the load on a +# known broken system). +# b) it reports to a specific project (but not necessarily the same) when a +# run failed to run properly end to end. + # Project whose issue will slow down the subsequent runs (delay defined # above). pagure_blocking_project = "fedora-infrastructure" # blocker issue tags, issue has to have all of them. blocker_tags = ['packager_workflow_blocker', 'staging'] + +# Project against which failed runs report their failure. +pagure_report_project = "fedora-infra/packaging_workflow_health" +pagure_api_token = "" +env = "prod" diff --git a/runner.py b/runner.py index 069366a..d8ae268 100644 --- a/runner.py +++ b/runner.py @@ -68,6 +68,10 @@ def schedule(conf): run_command(cmd) delay = conf["delay"] + report_project = conf["pagure_report_project"] + report_api_token = conf["pagure_api_token"] + report_env = conf["env"] + print("Tests started:", datetime.datetime.utcnow(), flush=True) runid = f"{datetime.datetime.utcnow().year}-{uuid.uuid4()}" try: @@ -83,6 +87,14 @@ def schedule(conf): result = "succeeded" else: result = "failed" + report_failure( + report_project, + report_api_token, + report_env, + "single-package", + monit_utils, + ) + notify( topic=f"single-build.end.{result}", message={ @@ -113,6 +125,14 @@ def schedule(conf): result = "succeeded" else: result = "failed" + report_failure( + report_project, + report_api_token, + report_env, + "multi-package", + monit_utils, + ) + notify( topic=f"multi-build.end.{result}", message={ diff --git a/utils.py b/utils.py index e63a5ed..6b0c027 100644 --- a/utils.py +++ b/utils.py @@ -17,6 +17,38 @@ import requests _log = logging.getLogger(__name__) +def report_failure(project, token, env, workflow, monit_utils): + """ Open a pagure ticket against the instance specified in the + configuration file when something does not work. + """ + url = f"https://pagure.io/api/0/{project}/new_issue" + title = f"Failure in {env} of the {workflow} packager workflow" + logs = "\n".join(monit_utils.logs) + content = f"""A run of monitor-gating has just failed in {env} for the {workflow} workflow. + +The suspects are '{", ".join(monit_utils.failed)}'. + +Full log: +```` +{logs} +```` +""" + tag = env + + data = { + "title": title, + "content": content, + "tag": tag, + } + headers = { + "Authorization": f"token {token}", + } + + req = requests.post(url, data=data, headers=headers) + if not req.ok: + print(f"Error when trying to open a ticket at: {url} to report the failure") + + def blocking_issues(project, tags): """Lists blocking issues we track in the fedora-infrastructure project. """ From 8b2b6a062d838474a437db2f803f2af1a9fbd454 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: May 25 2020 07:48:53 +0000 Subject: [PATCH 3/3] Expand the documentation for those of us that don't measure time in seconds Signed-off-by: Pierre-Yves Chibon --- diff --git a/runner.cfg b/runner.cfg index a1c8c31..9bd81a6 100644 --- a/runner.cfg +++ b/runner.cfg @@ -1,7 +1,9 @@ # Time between two runs in second +# 3600 = 1h delay = 3600 # Time between two blocked runs in second +# 43200 = 12h delay_when_failing = 43200