From 6e5d1c2bda74987f3755850e8ec1874b289a746c Mon Sep 17 00:00:00 2001 From: Adam Saleh Date: Sep 09 2020 12:22:01 +0000 Subject: Fixed the errors in both multi and single build scripts. Namely, the test-case that is used in results-db has changed its name, meaning that we stopped seeing the old one in resultsdb around may Second, the final tag still expected f33, even though f34 is already in rawhide. Third, there are more log outputs, with links to the systems we query, which should improve the debugging of future failures. --- diff --git a/monitor_gating/multi_builds.py b/monitor_gating/multi_builds.py index bb1e306..7e7f66d 100644 --- a/monitor_gating/multi_builds.py +++ b/monitor_gating/multi_builds.py @@ -94,11 +94,14 @@ def main(args, utils=None): version, synced = utils.nevrs_synced(nevrs, conf) if not synced: - nevrs, side_tag_name = utils.clone_and_bump( + nevr_1 = nevrs[conf["name_multi_1"]] + nevr_2 = nevrs[conf["name_multi_2"]] + utils.print_user(f"Nevrs {nevr_1} and {nevr_2} not in sync, attempting to sync", success=True) + nevrs, side_tag_name = utils.bump( folder, nevrs, conf, conf["name_multi_1"], version=version, new_side_tag=True ) - nevrs, _ = utils.clone_and_bump( + nevrs, _ = utils.bump( folder, nevrs, conf, conf["name_multi_2"], version=version, target=side_tag_name ) @@ -123,7 +126,7 @@ def main(args, utils=None): from_tag=True, ) updateid = utils.get_update_id(nevrs[list(nevrs.keys())[0]], conf["bodhi"],) - print(f" Update created : {updateid}") + utils.print_user(f"Update {updateid} created : {conf['bodhi']}/updates/{updateid}", success=True) # Check the tag of the build utils.get_build_tags( @@ -176,7 +179,10 @@ def main(args, utils=None): # Check that the CI results made it to resultsdb utils.lookup_ci_resultsdb( - nevr=nevr, name="resultsdb(phx)", url=conf["resultsdb"] + nevr=nevr, + name="resultsdb(phx)", + url=conf["resultsdb"], + start=start_dg, ) # Check that resultsdb announced the new results diff --git a/monitor_gating/single_build.py b/monitor_gating/single_build.py index e7665d2..735bbac 100644 --- a/monitor_gating/single_build.py +++ b/monitor_gating/single_build.py @@ -105,7 +105,7 @@ def main(args): utils.bump_release(name, version=0, folder=gitfolder) utils.commit_changes("Bump release", folder=gitfolder) nevr = utils.get_nevr(conf["fedpkg"], folder=gitfolder) - print(f" Upcoming build : {nevr}") + utils.print_user(f"Upcoming build : {nevr}", success=True) if args.no_pr: # Push to the main repo @@ -178,6 +178,9 @@ def main(args): return # Build the package + koji_search_uri = "https://koji.fedoraproject.org/koji/search?match=glob" + utils.print_user(f"The build should be found on: {koji_search_uri}&type=build&terms={nevr}", success=True) + utils.print_user(f"The build should be found in: {koji_search_uri}&type=package&terms={name}", success=True) utils.build_package(conf["fedpkg"], folder=gitfolder) # Check the tag of the build @@ -199,9 +202,9 @@ def main(args): username=conf.get("bodhi-user"), password=conf.get("bodhi-password"), ) - print(f" Update created : {updateid}") + utils.print_user(f"Update created : {conf['bodhi']}/updates/{updateid}", success=True) elif args.auto_update: - print(f" Update automatically created : {updateid}") + utils.print_user(f"Update automatically created : {conf['bodhi']}/updates/{updateid}", success=True) else: updateid = args.update @@ -252,7 +255,11 @@ def main(args): # Check that the CI results made it to resultsdb utils.lookup_ci_resultsdb( - nevr=nevr, name="resultsdb(phx)", url=conf["resultsdb"] + nevr=nevr, + name="resultsdb(phx)", + url=conf["resultsdb"], + start=None, + duration=5 * 60, ) # Check that resultsdb announced the new results diff --git a/monitor_gating/utils.py b/monitor_gating/utils.py index 9451dd5..ccf6b0d 100644 --- a/monitor_gating/utils.py +++ b/monitor_gating/utils.py @@ -356,7 +356,7 @@ class MonitoringUtils: success = False break - info_log = f"Retrieving koji tags: {tags}" + info_log = f"Retrieving koji tags: {tags} found while expected {expected_ends}" if not success: self.failed.append("koji") self.print_user(info_log, success=success) @@ -398,9 +398,9 @@ class MonitoringUtils: def get_update_id(self, nevr, url): """ Retrieve the update identifier from bodhi for the given nevr. """ start = datetime.datetime.utcnow() - info_log = f"Retrieving update created" - self.print_user(info_log) url = f"{url}/updates/?builds={nevr}" + info_log = f"Retrieving update created from {url}" + self.print_user(info_log) updateid = None success = True while True: @@ -575,42 +575,46 @@ class MonitoringUtils: if not success: self.failed.append("datagrepper") - def lookup_ci_resultsdb(self, nevr, name, url): + def lookup_ci_resultsdb(self, nevr, name, url, start=None, duration=(15 * 60)): """ Check the CI results in the specified resultsdb for results about our specified build. """ - start = datetime.datetime.utcnow() - info_log = f"Checking {name} for CI results " - self.print_user(info_log) - topic = "org.centos.prod.ci.pipeline.allpackages-build.complete" + if start is None: + start = datetime.datetime.utcnow() + # previously "org.centos.prod.ci.pipeline.allpackages-build.complete" + topic = "fedora-ci.koji-build.tier0.functional" if ".stg" in url: topic = "org.centos.stage.ci.pipeline.allpackages-build.complete" url = f"{url}?testcases={topic}" + self.print_user(f"CI results query from {url}&page=0", success=True) + info_log = f"Checking {name} for CI results" + self.print_user(info_log) + success = False returned_status = None info_log = None while True: - # Assume we won't have more than 3 pages of results coming in b/w + # Assume we won't have more than 10 pages of results coming in b/w # our checks - for page in [0, 1, 2]: + for page in range(0, 9): end_url = url end_url += f"&page={page}" data = requests.get(end_url).json() for result in data["data"]: if nevr in result["data"]["nvr"]: success = True - returned_status = result["data"]["status"][0] + returned_status = result["data"]["topic"][0] break if success: break if success: break - if (datetime.datetime.utcnow() - start).seconds > (15 * 60): + if (datetime.datetime.utcnow() - start).seconds > duration: success = False info_log = ( - f"CI results did not show in {name} for {nevr} within 15 minutes" + f"CI results did not show in {name} for {nevr} within {duration} minutes since {start}" ) break @@ -759,11 +763,16 @@ class MonitoringUtils: ): """Clone the repo, bump the release, commit and push.""" namespace = conf["namespace"] - branch = conf["branch"] - self.clone_repo( conf["fedpkg"], conf["fas_username"], namespace, name, folder=folder, ) + return self.bump(folder, nevrs, conf, name, version, target, new_side_tag) + + def bump( + self, folder, nevrs, conf, name, version, target=None, new_side_tag=False + ): + """Clone the repo, bump the release, commit and push.""" + branch = conf["branch"] gitfolder = os.path.join(folder, name) self.switch_branch(conf["fedpkg"], branch, folder=gitfolder) side_tag_name = None @@ -781,7 +790,7 @@ class MonitoringUtils: nevrs[name] = nevr # Push to the main repo self.push_changes(gitfolder, "origin", branch) - print(f" Upcoming build : {nevr}") + self.print_user(f"Upcoming build : {nevr}") return (nevrs, target) def nevrs_synced(self, nevrs, conf):