From 801fdee76de19fa0a771dc2263ddfd6a04fe0400 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Aug 18 2020 11:48:33 +0000 Subject: [PATCH 1/7] Add more debugging Signed-off-by: Pierre-Yves Chibon --- diff --git a/monitor_gating/runner.py b/monitor_gating/runner.py index 413b94a..9e6cfee 100644 --- a/monitor_gating/runner.py +++ b/monitor_gating/runner.py @@ -45,8 +45,12 @@ def notify(topic, message): print(f"Fedora Messaging broker rejected message {msg.id}: {err}") except fedora_messaging.exceptions.ConnectionException as err: print(f"Error sending message {msg.id}: {err}") + print(f"topic: {topic}") + print(f"message: {message}") except Exception as err: print(f"Error sending fedora-messaging message: {err}") + print(f"topic: {topic}") + print(f"message: {message}") def _clean_up_side_tags(utils): @@ -145,6 +149,8 @@ def schedule(conf): except Exception as err: print(f"Tests failed with: {err}", flush=True) print(sys.exc_info()[0]) + print("-"*20) + print(sys.exc_info()) notify( topic=f"multi-build.end.error", message={"runid": runid, "exception": err}, From 77e00c479bf9462857dd6c0e25fc9a7cc18576b3 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Aug 18 2020 11:48:33 +0000 Subject: [PATCH 2/7] Check that the request return a 2xx code before proceeding Signed-off-by: Pierre-Yves Chibon --- diff --git a/monitor_gating/utils.py b/monitor_gating/utils.py index 6b0c027..0018f09 100644 --- a/monitor_gating/utils.py +++ b/monitor_gating/utils.py @@ -60,12 +60,15 @@ def blocking_issues(project, tags): issues = [] try: r = requests.get(api) - issues = r.json()["issues"] - if tags: - t = set(tags[1:]) - issues = [i for i in issues if t & set(i["tags"])] - for i in issues: - print(f"Found blocking issue https://pagure.io/{project}/issue/{i['id']}") + if not r.ok: + print(f"Failed to query: {api} -- returned : {r.status_code}") + else: + issues = r.json()["issues"] + if tags: + t = set(tags[1:]) + issues = [i for i in issues if t & set(i["tags"])] + for i in issues: + print(f"Found blocking issue https://pagure.io/{project}/issue/{i['id']}") except Exception as e: print(f"Error when querying pagure for blocking issues: {e}") return issues From 765c212b18236af4a0c434c34dc90b57473d061b Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Aug 18 2020 11:48:33 +0000 Subject: [PATCH 3/7] Add missing import to the runner.py Signed-off-by: Pierre-Yves Chibon --- diff --git a/monitor_gating/runner.py b/monitor_gating/runner.py index 9e6cfee..579b250 100644 --- a/monitor_gating/runner.py +++ b/monitor_gating/runner.py @@ -18,7 +18,7 @@ import toml from . import multi_builds from . import single_build -from .utils import MonitoringUtils, blocking_issues, run_command +from .utils import MonitoringUtils, blocking_issues, run_command, report_failure s = sched.scheduler(time.time, time.sleep) From 67e9a66324fe7ead17a9e30e901da944bcd4cb53 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Aug 18 2020 11:48:33 +0000 Subject: [PATCH 4/7] When notifying on errors, don't send the raw error object, send its text version Signed-off-by: Pierre-Yves Chibon --- diff --git a/monitor_gating/runner.py b/monitor_gating/runner.py index 579b250..e00a675 100644 --- a/monitor_gating/runner.py +++ b/monitor_gating/runner.py @@ -153,7 +153,7 @@ def schedule(conf): print(sys.exc_info()) notify( - topic=f"multi-build.end.error", message={"runid": runid, "exception": err}, + topic=f"multi-build.end.error", message={"runid": runid, "exception": str(err)}, ) delay_when_failing = conf["delay_when_failing"] From 7de62ba5ce0cfb381eb76ace845ea04cb2486bfc Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Aug 18 2020 11:48:34 +0000 Subject: [PATCH 5/7] Flush some prints Signed-off-by: Pierre-Yves Chibon --- diff --git a/monitor_gating/utils.py b/monitor_gating/utils.py index 0018f09..210282f 100644 --- a/monitor_gating/utils.py +++ b/monitor_gating/utils.py @@ -46,7 +46,7 @@ Full log: req = requests.post(url, data=data, headers=headers) if not req.ok: - print(f"Error when trying to open a ticket at: {url} to report the failure") + print(f"Error when trying to open a ticket at: {url} to report the failure", flush=True) def blocking_issues(project, tags): @@ -61,16 +61,16 @@ def blocking_issues(project, tags): try: r = requests.get(api) if not r.ok: - print(f"Failed to query: {api} -- returned : {r.status_code}") + print(f"Failed to query: {api} -- returned : {r.status_code}", flush=True) else: issues = r.json()["issues"] if tags: t = set(tags[1:]) issues = [i for i in issues if t & set(i["tags"])] for i in issues: - print(f"Found blocking issue https://pagure.io/{project}/issue/{i['id']}") + print(f"Found blocking issue https://pagure.io/{project}/issue/{i['id']}", flush=True) except Exception as e: - print(f"Error when querying pagure for blocking issues: {e}") + print(f"Error when querying pagure for blocking issues: {e}", flush=True) return issues From 258daf1d3b8787a55169269eed63c531097947e4 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Aug 18 2020 11:48:34 +0000 Subject: [PATCH 6/7] Add some more debugging Signed-off-by: Pierre-Yves Chibon --- diff --git a/monitor_gating/runner.py b/monitor_gating/runner.py index e00a675..2ce25bd 100644 --- a/monitor_gating/runner.py +++ b/monitor_gating/runner.py @@ -149,8 +149,14 @@ def schedule(conf): except Exception as err: print(f"Tests failed with: {err}", flush=True) print(sys.exc_info()[0]) - print("-"*20) + print("-"*60) print(sys.exc_info()) + print("="*60) + import traceback + traceback.print_exc(file=sys.stdout) + print("-"*60) + traceback.print_stack() + notify( topic=f"multi-build.end.error", message={"runid": runid, "exception": str(err)}, From 69e327b66f0a5b81f32fdfa0b1708e2fe6d3dc4b Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Aug 18 2020 11:48:34 +0000 Subject: [PATCH 7/7] Fix appending an element to a list rather than trying to call the list Signed-off-by: Pierre-Yves Chibon --- diff --git a/monitor_gating/utils.py b/monitor_gating/utils.py index 210282f..68f6332 100644 --- a/monitor_gating/utils.py +++ b/monitor_gating/utils.py @@ -156,7 +156,7 @@ class MonitoringUtils: run_command([command, "switch-branch", f"{name}"], cwd=folder) self.print_user(info_log, success=True) except MonitoringException: - self.failed("fedpkg") + self.failed.append("fedpkg") self.print_user(info_log, success=False) def bump_release(self, name, folder): @@ -168,7 +168,7 @@ class MonitoringUtils: run_command(["rpmdev-bumpspec", f"{name}.spec"], cwd=folder) self.print_user(info_log, success=True) except MonitoringException: - self.failed("rpmdev-bumspec") + self.failed.append("rpmdev-bumspec") self.print_user(info_log, success=False) def commit_changes(self, commit_log, folder): @@ -181,7 +181,7 @@ class MonitoringUtils: run_command(["git", "commit", "-asm", commit_log], cwd=folder) self.print_user(info_log, success=True) except MonitoringException: - self.failed("git") + self.failed.append("git") self.print_user(info_log, success=False) def push_changes(self, folder, target, branch, force=False): @@ -196,7 +196,7 @@ class MonitoringUtils: run_command(cmd, cwd=folder) self.print_user(info_log, success=True) except MonitoringException: - self.failed("git/dist-git") + self.failed.append("git/dist-git") self.print_user(info_log, success=False) def pull_changes(self, folder, target, branch): @@ -209,7 +209,7 @@ class MonitoringUtils: run_command(cmd, cwd=folder) self.print_user(info_log, success=True) except MonitoringException: - self.failed("git/dist-git") + self.failed.append("git/dist-git") self.print_user(info_log, success=False) def open_pullrequest(self, base_url, username, namespace, name, branch, token): @@ -237,7 +237,7 @@ class MonitoringUtils: success = False pr_id = None pr_uid = None - self.failed("dist-git") + self.failed.append("dist-git") else: output = req.json() pr_id = str(output["id"]) @@ -260,7 +260,7 @@ class MonitoringUtils: self.print_user(info_log, success=True) return nevr.strip().decode("utf-8") except MonitoringException: - self.failed("fedpkg") + self.failed.append("fedpkg") self.print_user(info_log, success=False) def build_package(self, command, folder, target=None): @@ -275,7 +275,7 @@ class MonitoringUtils: run_command(command, cwd=folder) self.print_user(info_log, success=True) except MonitoringException: - self.failed("koji") + self.failed.append("koji") self.print_user(info_log, success=False) def chain_build_packages(self, command, packages, folder, target=None): @@ -295,7 +295,7 @@ class MonitoringUtils: run_command(command, cwd=folder) self.print_user(info_log, success=True) except MonitoringException: - self.failed("koji") + self.failed.append("koji") self.print_user(info_log, success=False) def get_build_tags(self, koji_url, nevr, expected_ends): @@ -353,7 +353,7 @@ class MonitoringUtils: info_log = f"Retrieving koji tags: {tags}" if not success: - self.failed("koji") + self.failed.append("koji") self.print_user(info_log, success=success) def create_update( @@ -387,7 +387,7 @@ class MonitoringUtils: run_command(command) self.print_user(info_log, success=True) except MonitoringException: - self.failed("bodhi") + self.failed.append("bodhi") self.print_user(info_log, success=False) def get_update_id(self, nevr, url): @@ -408,7 +408,7 @@ class MonitoringUtils: if (datetime.datetime.utcnow() - start).seconds > (15 * 60): success = False - self.failed("bodhi") + self.failed.append("bodhi") info_log = f"Update for {nevr} not created within 15 minutes" break @@ -568,7 +568,7 @@ class MonitoringUtils: info_log += f" - ran for: {(end - start_lookup).seconds}s" self.print_user(info_log, success=success) if not success: - self.failed("datagrepper") + self.failed.append("datagrepper") def lookup_ci_resultsdb(self, nevr, name, url): """ Check the CI results in the specified resultsdb for results about @@ -619,7 +619,7 @@ class MonitoringUtils: info_log += f" - ran for: {(end - start).seconds}s" self.print_user(info_log, success=success) if not success: - self.failed("resultsdb") + self.failed.append("resultsdb") def waive_update(self, command, updateid, prod=True, username=None, password=None): """ Waive all the tests results for the specified update using bodhi's @@ -645,7 +645,7 @@ class MonitoringUtils: run_command(command) self.print_user(info_log, success=True) except MonitoringException: - self.failed("waiverdb") + self.failed.append("waiverdb") self.print_user(info_log, success=False) def get_pr_flag( @@ -701,7 +701,7 @@ class MonitoringUtils: self.print_user(info_log, success=success) if not success: - self.failed("dist-git") + self.failed.append("dist-git") def merge_pr(self, base_url, username, namespace, name, pr_id, token): """ Merge the specified PR @@ -722,7 +722,7 @@ class MonitoringUtils: self.print_user(info_log, success=success) if not success: - self.failed("dist-git") + self.failed.append("dist-git") def finalize(self, start): """ End data returned. """ @@ -745,7 +745,7 @@ class MonitoringUtils: self.print_user(info_log, success=True) except (MonitoringException, Exception) as err: print(err) - self.failed("dist-git") + self.failed.append("dist-git") self.print_user(info_log, success=False) return side_tag_name