From ad0f320d8481659c24a2b22bfbba12c1eec6d3df Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Jul 07 2020 13:28:54 +0000 Subject: [PATCH 1/11] Add more debugging Signed-off-by: Pierre-Yves Chibon --- diff --git a/monitor_gating/runner.py b/monitor_gating/runner.py index 413b94a..9e6cfee 100644 --- a/monitor_gating/runner.py +++ b/monitor_gating/runner.py @@ -45,8 +45,12 @@ def notify(topic, message): print(f"Fedora Messaging broker rejected message {msg.id}: {err}") except fedora_messaging.exceptions.ConnectionException as err: print(f"Error sending message {msg.id}: {err}") + print(f"topic: {topic}") + print(f"message: {message}") except Exception as err: print(f"Error sending fedora-messaging message: {err}") + print(f"topic: {topic}") + print(f"message: {message}") def _clean_up_side_tags(utils): @@ -145,6 +149,8 @@ def schedule(conf): except Exception as err: print(f"Tests failed with: {err}", flush=True) print(sys.exc_info()[0]) + print("-"*20) + print(sys.exc_info()) notify( topic=f"multi-build.end.error", message={"runid": runid, "exception": err}, From c0848e574e405377599f2f20f6309a31d4214f11 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Jul 07 2020 13:28:54 +0000 Subject: [PATCH 2/11] Check that the request return a 2xx code before proceeding Signed-off-by: Pierre-Yves Chibon --- diff --git a/monitor_gating/utils.py b/monitor_gating/utils.py index 6b0c027..0018f09 100644 --- a/monitor_gating/utils.py +++ b/monitor_gating/utils.py @@ -60,12 +60,15 @@ def blocking_issues(project, tags): issues = [] try: r = requests.get(api) - issues = r.json()["issues"] - if tags: - t = set(tags[1:]) - issues = [i for i in issues if t & set(i["tags"])] - for i in issues: - print(f"Found blocking issue https://pagure.io/{project}/issue/{i['id']}") + if not r.ok: + print(f"Failed to query: {api} -- returned : {r.status_code}") + else: + issues = r.json()["issues"] + if tags: + t = set(tags[1:]) + issues = [i for i in issues if t & set(i["tags"])] + for i in issues: + print(f"Found blocking issue https://pagure.io/{project}/issue/{i['id']}") except Exception as e: print(f"Error when querying pagure for blocking issues: {e}") return issues From 7f4103058675eb9c3652e2c3286d5e00beb6b1d8 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Jul 07 2020 13:28:54 +0000 Subject: [PATCH 3/11] Add missing import to the runner.py Signed-off-by: Pierre-Yves Chibon --- diff --git a/monitor_gating/runner.py b/monitor_gating/runner.py index 9e6cfee..579b250 100644 --- a/monitor_gating/runner.py +++ b/monitor_gating/runner.py @@ -18,7 +18,7 @@ import toml from . import multi_builds from . import single_build -from .utils import MonitoringUtils, blocking_issues, run_command +from .utils import MonitoringUtils, blocking_issues, run_command, report_failure s = sched.scheduler(time.time, time.sleep) From 3f41fd5afa58c143ab72de2a2a3fda29e76ed957 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Jul 07 2020 13:28:54 +0000 Subject: [PATCH 4/11] When notifying on errors, don't send the raw error object, send its text version Signed-off-by: Pierre-Yves Chibon --- diff --git a/monitor_gating/runner.py b/monitor_gating/runner.py index 579b250..e00a675 100644 --- a/monitor_gating/runner.py +++ b/monitor_gating/runner.py @@ -153,7 +153,7 @@ def schedule(conf): print(sys.exc_info()) notify( - topic=f"multi-build.end.error", message={"runid": runid, "exception": err}, + topic=f"multi-build.end.error", message={"runid": runid, "exception": str(err)}, ) delay_when_failing = conf["delay_when_failing"] From d01a6881be52e341dbbe9cf5d42c16993bcf4f9c Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Jul 07 2020 13:28:54 +0000 Subject: [PATCH 5/11] Flush some prints Signed-off-by: Pierre-Yves Chibon --- diff --git a/monitor_gating/utils.py b/monitor_gating/utils.py index 0018f09..210282f 100644 --- a/monitor_gating/utils.py +++ b/monitor_gating/utils.py @@ -46,7 +46,7 @@ Full log: req = requests.post(url, data=data, headers=headers) if not req.ok: - print(f"Error when trying to open a ticket at: {url} to report the failure") + print(f"Error when trying to open a ticket at: {url} to report the failure", flush=True) def blocking_issues(project, tags): @@ -61,16 +61,16 @@ def blocking_issues(project, tags): try: r = requests.get(api) if not r.ok: - print(f"Failed to query: {api} -- returned : {r.status_code}") + print(f"Failed to query: {api} -- returned : {r.status_code}", flush=True) else: issues = r.json()["issues"] if tags: t = set(tags[1:]) issues = [i for i in issues if t & set(i["tags"])] for i in issues: - print(f"Found blocking issue https://pagure.io/{project}/issue/{i['id']}") + print(f"Found blocking issue https://pagure.io/{project}/issue/{i['id']}", flush=True) except Exception as e: - print(f"Error when querying pagure for blocking issues: {e}") + print(f"Error when querying pagure for blocking issues: {e}", flush=True) return issues From e719e33d996f86cb37e3dfe25e62c436d968c432 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Jul 07 2020 13:28:54 +0000 Subject: [PATCH 6/11] Add some more debugging Signed-off-by: Pierre-Yves Chibon --- diff --git a/monitor_gating/runner.py b/monitor_gating/runner.py index e00a675..2ce25bd 100644 --- a/monitor_gating/runner.py +++ b/monitor_gating/runner.py @@ -149,8 +149,14 @@ def schedule(conf): except Exception as err: print(f"Tests failed with: {err}", flush=True) print(sys.exc_info()[0]) - print("-"*20) + print("-"*60) print(sys.exc_info()) + print("="*60) + import traceback + traceback.print_exc(file=sys.stdout) + print("-"*60) + traceback.print_stack() + notify( topic=f"multi-build.end.error", message={"runid": runid, "exception": str(err)}, From f6380764467df3254ffa60853240a64944de9ee1 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Jul 07 2020 13:28:54 +0000 Subject: [PATCH 7/11] Fix appending an element to a list rather than trying to call the list Signed-off-by: Pierre-Yves Chibon --- diff --git a/monitor_gating/utils.py b/monitor_gating/utils.py index 210282f..68f6332 100644 --- a/monitor_gating/utils.py +++ b/monitor_gating/utils.py @@ -156,7 +156,7 @@ class MonitoringUtils: run_command([command, "switch-branch", f"{name}"], cwd=folder) self.print_user(info_log, success=True) except MonitoringException: - self.failed("fedpkg") + self.failed.append("fedpkg") self.print_user(info_log, success=False) def bump_release(self, name, folder): @@ -168,7 +168,7 @@ class MonitoringUtils: run_command(["rpmdev-bumpspec", f"{name}.spec"], cwd=folder) self.print_user(info_log, success=True) except MonitoringException: - self.failed("rpmdev-bumspec") + self.failed.append("rpmdev-bumspec") self.print_user(info_log, success=False) def commit_changes(self, commit_log, folder): @@ -181,7 +181,7 @@ class MonitoringUtils: run_command(["git", "commit", "-asm", commit_log], cwd=folder) self.print_user(info_log, success=True) except MonitoringException: - self.failed("git") + self.failed.append("git") self.print_user(info_log, success=False) def push_changes(self, folder, target, branch, force=False): @@ -196,7 +196,7 @@ class MonitoringUtils: run_command(cmd, cwd=folder) self.print_user(info_log, success=True) except MonitoringException: - self.failed("git/dist-git") + self.failed.append("git/dist-git") self.print_user(info_log, success=False) def pull_changes(self, folder, target, branch): @@ -209,7 +209,7 @@ class MonitoringUtils: run_command(cmd, cwd=folder) self.print_user(info_log, success=True) except MonitoringException: - self.failed("git/dist-git") + self.failed.append("git/dist-git") self.print_user(info_log, success=False) def open_pullrequest(self, base_url, username, namespace, name, branch, token): @@ -237,7 +237,7 @@ class MonitoringUtils: success = False pr_id = None pr_uid = None - self.failed("dist-git") + self.failed.append("dist-git") else: output = req.json() pr_id = str(output["id"]) @@ -260,7 +260,7 @@ class MonitoringUtils: self.print_user(info_log, success=True) return nevr.strip().decode("utf-8") except MonitoringException: - self.failed("fedpkg") + self.failed.append("fedpkg") self.print_user(info_log, success=False) def build_package(self, command, folder, target=None): @@ -275,7 +275,7 @@ class MonitoringUtils: run_command(command, cwd=folder) self.print_user(info_log, success=True) except MonitoringException: - self.failed("koji") + self.failed.append("koji") self.print_user(info_log, success=False) def chain_build_packages(self, command, packages, folder, target=None): @@ -295,7 +295,7 @@ class MonitoringUtils: run_command(command, cwd=folder) self.print_user(info_log, success=True) except MonitoringException: - self.failed("koji") + self.failed.append("koji") self.print_user(info_log, success=False) def get_build_tags(self, koji_url, nevr, expected_ends): @@ -353,7 +353,7 @@ class MonitoringUtils: info_log = f"Retrieving koji tags: {tags}" if not success: - self.failed("koji") + self.failed.append("koji") self.print_user(info_log, success=success) def create_update( @@ -387,7 +387,7 @@ class MonitoringUtils: run_command(command) self.print_user(info_log, success=True) except MonitoringException: - self.failed("bodhi") + self.failed.append("bodhi") self.print_user(info_log, success=False) def get_update_id(self, nevr, url): @@ -408,7 +408,7 @@ class MonitoringUtils: if (datetime.datetime.utcnow() - start).seconds > (15 * 60): success = False - self.failed("bodhi") + self.failed.append("bodhi") info_log = f"Update for {nevr} not created within 15 minutes" break @@ -568,7 +568,7 @@ class MonitoringUtils: info_log += f" - ran for: {(end - start_lookup).seconds}s" self.print_user(info_log, success=success) if not success: - self.failed("datagrepper") + self.failed.append("datagrepper") def lookup_ci_resultsdb(self, nevr, name, url): """ Check the CI results in the specified resultsdb for results about @@ -619,7 +619,7 @@ class MonitoringUtils: info_log += f" - ran for: {(end - start).seconds}s" self.print_user(info_log, success=success) if not success: - self.failed("resultsdb") + self.failed.append("resultsdb") def waive_update(self, command, updateid, prod=True, username=None, password=None): """ Waive all the tests results for the specified update using bodhi's @@ -645,7 +645,7 @@ class MonitoringUtils: run_command(command) self.print_user(info_log, success=True) except MonitoringException: - self.failed("waiverdb") + self.failed.append("waiverdb") self.print_user(info_log, success=False) def get_pr_flag( @@ -701,7 +701,7 @@ class MonitoringUtils: self.print_user(info_log, success=success) if not success: - self.failed("dist-git") + self.failed.append("dist-git") def merge_pr(self, base_url, username, namespace, name, pr_id, token): """ Merge the specified PR @@ -722,7 +722,7 @@ class MonitoringUtils: self.print_user(info_log, success=success) if not success: - self.failed("dist-git") + self.failed.append("dist-git") def finalize(self, start): """ End data returned. """ @@ -745,7 +745,7 @@ class MonitoringUtils: self.print_user(info_log, success=True) except (MonitoringException, Exception) as err: print(err) - self.failed("dist-git") + self.failed.append("dist-git") self.print_user(info_log, success=False) return side_tag_name From d0bd9a0b5f6667fca6ad0e8e2f00318b2d2ef368 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Jul 07 2020 13:28:54 +0000 Subject: [PATCH 8/11] Fix typo in the rpmdev-bumpspec name Signed-off-by: Pierre-Yves Chibon --- diff --git a/monitor_gating/utils.py b/monitor_gating/utils.py index 68f6332..5a04128 100644 --- a/monitor_gating/utils.py +++ b/monitor_gating/utils.py @@ -168,7 +168,7 @@ class MonitoringUtils: run_command(["rpmdev-bumpspec", f"{name}.spec"], cwd=folder) self.print_user(info_log, success=True) except MonitoringException: - self.failed.append("rpmdev-bumspec") + self.failed.append("rpmdev-bumpspec") self.print_user(info_log, success=False) def commit_changes(self, commit_log, folder): From e320728649f1591b60da434dc2819f9cf48edc85 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Jul 07 2020 13:28:54 +0000 Subject: [PATCH 9/11] Be more specific about the error Signed-off-by: Pierre-Yves Chibon --- diff --git a/monitor_gating/utils.py b/monitor_gating/utils.py index 5a04128..753ba69 100644 --- a/monitor_gating/utils.py +++ b/monitor_gating/utils.py @@ -47,6 +47,8 @@ Full log: req = requests.post(url, data=data, headers=headers) if not req.ok: print(f"Error when trying to open a ticket at: {url} to report the failure", flush=True) + print(f"Request to {url} with data={data} returned: {req.status_code}", flush=True) + print(f"Output: {req.text}", flush=True) def blocking_issues(project, tags): From 9e59d09e7bc1cf787d7511402bd402dff5c458d9 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Jul 07 2020 13:28:54 +0000 Subject: [PATCH 10/11] Fix field name when opening an issue on pagure Signed-off-by: Pierre-Yves Chibon --- diff --git a/monitor_gating/utils.py b/monitor_gating/utils.py index 753ba69..3dddb04 100644 --- a/monitor_gating/utils.py +++ b/monitor_gating/utils.py @@ -37,7 +37,7 @@ Full log: data = { "title": title, - "content": content, + "issue_content": content, "tag": tag, } headers = { From 6391facdd7ff99ce054fad9be31339c70c8f6dde Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Jul 07 2020 13:28:55 +0000 Subject: [PATCH 11/11] Fix returning the MonitorUtils object Signed-off-by: Pierre-Yves Chibon --- diff --git a/monitor_gating/multi_builds.py b/monitor_gating/multi_builds.py index 9cbe830..e740330 100644 --- a/monitor_gating/multi_builds.py +++ b/monitor_gating/multi_builds.py @@ -123,7 +123,7 @@ def main(args, utils=None): if not updateid: utils.finalize(start) - return + return utils # Check that bodhi notified the pipeline it can run utils.lookup_results_datagrepper(