From 3899daccdca742e2d663a232bfd0e950064f558f Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Mar 25 2020 14:19:34 +0000 Subject: Run black without argument on the entire project Signed-off-by: Pierre-Yves Chibon --- diff --git a/monitor_gating_multi_builds.py b/monitor_gating_multi_builds.py index fbb5b61..cef5dcd 100644 --- a/monitor_gating_multi_builds.py +++ b/monitor_gating_multi_builds.py @@ -38,9 +38,7 @@ _log = logging.getLogger(__name__) def get_arguments(args): """ Parse and return the CLI arguments. """ - parser = argparse.ArgumentParser( - description="Test the CI workflow in Fedora." - ) + parser = argparse.ArgumentParser(description="Test the CI workflow in Fedora.") parser.add_argument( "--staging", action="store_true", @@ -83,16 +81,18 @@ def main(args): # Bump the release on both packages: nevrs, side_tag_name = utils.clone_and_bump( - folder, nevrs, conf, conf["name_multi_1"], new_side_tag=True) + folder, nevrs, conf, conf["name_multi_1"], new_side_tag=True + ) nevrs, _ = utils.clone_and_bump( - folder, nevrs, conf, conf["name_multi_2"], target=side_tag_name) + folder, nevrs, conf, conf["name_multi_2"], target=side_tag_name + ) # Chain-build the packages utils.chain_build_packages( conf["fedpkg"], packages=conf["name_multi_1"], folder=os.path.join(folder, conf["name_multi_2"]), - target=side_tag_name + target=side_tag_name, ) if args.conflict: @@ -107,10 +107,7 @@ def main(args): password=conf.get("bodhi-password"), from_tag=True, ) - updateid = utils.get_update_id( - nevrs[list(nevrs.keys())[0]], - conf["bodhi"], - ) + updateid = utils.get_update_id(nevrs[list(nevrs.keys())[0]], conf["bodhi"],) print(f" Update created : {updateid}") # Check the tag of the build @@ -159,9 +156,7 @@ def main(args): # Check the tag of the build utils.get_build_tags( - conf.get("koji_hub"), - nevr, - expected_ends=["testing-pending"], + conf.get("koji_hub"), nevr, expected_ends=["testing-pending"], ) # Check that the CI results made it to resultsdb @@ -189,9 +184,7 @@ def main(args): # Check the tag of the build -- build is blocked but should be signed utils.get_build_tags( - conf.get("koji_hub"), - nevr, - expected_ends=["testing-pending"], + conf.get("koji_hub"), nevr, expected_ends=["testing-pending"], ) if not args.no_waive: @@ -222,9 +215,7 @@ def main(args): # Check the tag of the build -- build was waived, let is through utils.get_build_tags( - conf.get("koji_hub"), - nevr, - expected_ends=conf["koji_end_tag"], + conf.get("koji_hub"), nevr, expected_ends=conf["koji_end_tag"], ) utils.finalize(start) diff --git a/monitor_gating_single_build.py b/monitor_gating_single_build.py index 263da9f..1156f11 100644 --- a/monitor_gating_single_build.py +++ b/monitor_gating_single_build.py @@ -38,12 +38,9 @@ _log = logging.getLogger(__name__) def get_arguments(args): """ Parse and return the CLI arguments. """ - parser = argparse.ArgumentParser( - description="Test the CI workflow in Fedora." - ) + parser = argparse.ArgumentParser(description="Test the CI workflow in Fedora.") parser.add_argument( - "--nevr", - help="NEVR of the build, allows by-passing: commit, push, build", + "--nevr", help="NEVR of the build, allows by-passing: commit, push, build", ) parser.add_argument( "--update", @@ -101,11 +98,7 @@ def main(args): print(f"Working in {folder}\n") if not args.nevr: utils.clone_repo( - conf["fedpkg"], - conf["fas_username"], - namespace, - name, - folder=folder, + conf["fedpkg"], conf["fas_username"], namespace, name, folder=folder, ) gitfolder = os.path.join(folder, name) utils.switch_branch(conf["fedpkg"], branch, folder=gitfolder) @@ -199,11 +192,13 @@ def main(args): # Retrieve or create the update updateid = utils.get_update_id(nevr, conf["bodhi"]) if not args.update and not args.auto_update: - utils.create_update(conf["bodhi-cli"], + utils.create_update( + conf["bodhi-cli"], nevr, prod=conf["_env"] == "prod", username=conf.get("bodhi-user"), - password=conf.get("bodhi-password"),) + password=conf.get("bodhi-password"), + ) print(f" Update created : {updateid}") elif args.auto_update: print(f" Update automatically created : {updateid}") @@ -248,9 +243,7 @@ def main(args): # Check the tag of the build utils.get_build_tags( - conf.get("koji_hub"), - nevr, - expected_ends=["testing-pending"], + conf.get("koji_hub"), nevr, expected_ends=["testing-pending"], ) # Check that the CI results made it to resultsdb @@ -276,9 +269,7 @@ def main(args): # Check the tag of the build -- build is blocked but should be signed utils.get_build_tags( - conf.get("koji_hub"), - nevr, - expected_ends=["testing-pending"], + conf.get("koji_hub"), nevr, expected_ends=["testing-pending"], ) if not args.no_waive: @@ -308,9 +299,7 @@ def main(args): # Check the tag of the build -- build was waived, let is through utils.get_build_tags( - conf.get("koji_hub"), - nevr, - expected_ends=conf["koji_end_tag"], + conf.get("koji_hub"), nevr, expected_ends=conf["koji_end_tag"], ) utils.finalize(start) diff --git a/runner.py b/runner.py index 26e6379..d1c7233 100644 --- a/runner.py +++ b/runner.py @@ -25,12 +25,9 @@ conf = toml.load def get_arguments(args): """ Load and parse the CLI arguments.""" - parser = argparse.ArgumentParser( - description="Runner for the CI canary tests." - ) + parser = argparse.ArgumentParser(description="Runner for the CI canary tests.") parser.add_argument( - "conf", - help="Configuration file for the different tests", + "conf", help="Configuration file for the different tests", ) return parser.parse_args(args) @@ -39,8 +36,7 @@ def get_arguments(args): def notify(topic, message): try: msg = fedora_messaging.api.Message( - topic="monitor-gating.{}".format(topic), - body=message + topic="monitor-gating.{}".format(topic), body=message ) fedora_messaging.api.publish(msg) except fedora_messaging.exceptions.PublishReturned as err: @@ -66,36 +62,22 @@ def schedule(conf): try: # Single Build Gating single_args = conf["workflow_single_gating_args"].split() - notify( - topic=f"single-build.start", - message={ - "arguments": single_args, - } - ) + notify(topic=f"single-build.start", message={"arguments": single_args,}) output = monitor_gating_single_build.main(single_args) output_text = "\n".join(output) - output_text="\n".join(output) + output_text = "\n".join(output) if "[FAILED]" not in output_text: result = "succeeded" else: result = "failed" notify( topic=f"single-build.end.{result}", - message={ - "output": output, - "output_text": output_text, - "result": result, - } + message={"output": output, "output_text": output_text, "result": result,}, ) # Multi Build Gating multi_args = conf["workflow_multi_gating_args"].split() - notify( - topic=f"multi-build.start", - message={ - "arguments": multi_args, - } - ) + notify(topic=f"multi-build.start", message={"arguments": multi_args,}) output = monitor_gating_multi_builds.main(multi_args) output_text = "\n".join(output) if "[FAILED]" not in output_text: @@ -104,11 +86,7 @@ def schedule(conf): result = "failed" notify( topic=f"multi-build.end.{result}", - message={ - "output": output, - "output_text": output_text, - "result": result, - } + message={"output": output, "output_text": output_text, "result": result,}, ) print("Tests finished:", datetime.datetime.utcnow(), flush=True) @@ -119,8 +97,11 @@ def schedule(conf): blocking_issues = utils.blocking_issues(blocker_tags) now = datetime.datetime.utcnow().strftime("%H:%M:%S") if blocking_issues: - print(f"{now} Next run in: {delay_when_failing} seconds because of " - f"{len(blocking_issues)} open issues", flush=True) + print( + f"{now} Next run in: {delay_when_failing} seconds because of " + f"{len(blocking_issues)} open issues", + flush=True, + ) s.enter(delay_when_failing, 1, schedule, argument=(conf,)) else: print(f"{now} Next run in: {delay} seconds", flush=True) @@ -140,8 +121,8 @@ if __name__ == "__main__": main(sys.argv[1:]) except KeyboardInterrupt: from code import InteractiveConsole - InteractiveConsole( - locals={"s": s}).interact( - "ENTERING THE DEBUG CONSOLE:\n s is the scheduler\n ^d to quit", - "LEAVING THE DEBUG CONSOLE" + + InteractiveConsole(locals={"s": s}).interact( + "ENTERING THE DEBUG CONSOLE:\n s is the scheduler\n ^d to quit", + "LEAVING THE DEBUG CONSOLE", ) diff --git a/utils.py b/utils.py index 67ebf53..265b2e2 100644 --- a/utils.py +++ b/utils.py @@ -28,12 +28,14 @@ def blocking_issues(tags): issues = [] try: r = requests.get(api + q) - issues = r.json()['issues'] + issues = r.json()["issues"] if tags: t = set(tags[1:]) - issues = [i for i in issues if t & set(i['tags'])] + issues = [i for i in issues if t & set(i["tags"])] for i in issues: - print(f"Found blocking issue https://pagure.io/fedora-infrastructure/issue/{i['id']}") + print( + f"Found blocking issue https://pagure.io/fedora-infrastructure/issue/{i['id']}" + ) except Exception as e: print(f"Error when querying pagure for blocking issues: {e}") return issues @@ -44,7 +46,6 @@ class MonitoringException(Exception): class MonitoringUtils: - def __init__(self): """ Instanciate the object. """ self.logs = [] @@ -76,11 +77,13 @@ class MonitoringUtils: info_log = f"Cloning as {username} the git repo: {namespace}/{name}" self.print_user(info_log) try: - run_command([command, "--user", username, "clone", f"{namespace}/{name}"], cwd=folder) + run_command( + [command, "--user", username, "clone", f"{namespace}/{name}"], + cwd=folder, + ) clone_folder = os.path.join(folder, name) run_command( - ["git", "config", "user.name", "packagerbot"], - cwd=clone_folder, + ["git", "config", "user.name", "packagerbot"], cwd=clone_folder, ) run_command( ["git", "config", "user.email", "admin@fedoraproject.org"], @@ -169,7 +172,7 @@ class MonitoringUtils: info_log = f"Creating PR from forks/{username}/{namespace}/{name}" self.print_user(info_log) url = "/".join( - [base_url.rstrip("/"), "api/0", namespace, name, "pull-request/new"] + [base_url.rstrip("/"), "api/0", namespace, name, "pull-request/new",] ) data = { "branch_to": branch, @@ -230,7 +233,9 @@ class MonitoringUtils: """ if not isinstance(packages, list): packages = [packages] - info_log = f"Chain-building the packages: {packages + [os.path.basename(folder)]}" + info_log = ( + f"Chain-building the packages: {packages + [os.path.basename(folder)]}" + ) self.print_user(info_log) command = [command, "chain-build"] command.extend(packages) @@ -253,15 +258,10 @@ class MonitoringUtils: "koji", ] if koji_url: - command.extend([ - "-s", - koji_url, - ]) - command.extend([ - "call", - "listTags", - nevr - ]) + command.extend( + ["-s", koji_url,] + ) + command.extend(["call", "listTags", nevr]) success = False tags = None @@ -269,7 +269,7 @@ class MonitoringUtils: while True: try: output = run_command(command) - output = output.decode('utf-8') + output = output.decode("utf-8") try: data = ast.literal_eval(output.strip()) except Exception: @@ -305,7 +305,9 @@ class MonitoringUtils: info_log = f"Retrieving koji tags: {tags}" self.print_user(info_log, success=success) - def create_update(self, command, item, prod=True, username=None, password=None, from_tag=False): + def create_update( + self, command, item, prod=True, username=None, password=None, from_tag=False, + ): """ Create the update for the package built. """ info_log = f"Creating a bodhi update" @@ -364,8 +366,16 @@ class MonitoringUtils: return updateid def lookup_results_datagrepper( - self, base_url, name, topic, nevr=None, nevrs=None, rev=None, - bodhi_id=None, start=None, duration=15 + self, + base_url, + name, + topic, + nevr=None, + nevrs=None, + rev=None, + bodhi_id=None, + start=None, + duration=15, ): """ Check the CI results in datagrepper for results about our specified build. @@ -397,8 +407,8 @@ class MonitoringUtils: data = requests.get(end_url).json() if "raw_messages" not in data: nomsg_log = f"No messages in data-grepper on {end_url} " - if 'error' in data: - nomsg_log += data['error'] + if "error" in data: + nomsg_log += data["error"] self.print_user(nomsg_log) break @@ -451,42 +461,37 @@ class MonitoringUtils: if ( "resultsdb" in message["topic"] and "nvr" in message["msg"]["data"] - and (nevr in message["msg"]["data"]["nvr"] - or message["msg"]["data"]["nvr"] in nevrs) + and ( + nevr in message["msg"]["data"]["nvr"] + or message["msg"]["data"]["nvr"] in nevrs + ) ): success = True returned_status = message["msg"]["outcome"] break # greenwave messages - if ( - "greenwave" in message["topic"] - and ( - message["msg"]["subject_identifier"] == nevr - or message["msg"]["subject_identifier"] in nevrs - ) + if "greenwave" in message["topic"] and ( + message["msg"]["subject_identifier"] == nevr + or message["msg"]["subject_identifier"] in nevrs ): success = True returned_status = message["msg"]["policies_satisfied"] break # waiverdb messages - if ( - "waiverdb" in message["topic"] - and ( - message["msg"]["subject_identifier"] == nevr - or message["msg"]["subject_identifier"] in nevrs - ) + if "waiverdb" in message["topic"] and ( + message["msg"]["subject_identifier"] == nevr + or message["msg"]["subject_identifier"] in nevrs ): success = True returned_status = "" break # bodhi messages - if ( - "bodhi.update.status.testing" in message["topic"] - and message["msg"]["artifact"]["id"].startswith(bodhi_id) - ): + if "bodhi.update.status.testing" in message["topic"] and message[ + "msg" + ]["artifact"]["id"].startswith(bodhi_id): success = True returned_status = "" break @@ -544,7 +549,9 @@ class MonitoringUtils: if (datetime.datetime.utcnow() - start).seconds > (15 * 60): success = False - info_log = f"CI results did not show in {name} for {nevr} within 15 minutes" + info_log = ( + f"CI results did not show in {name} for {nevr} within 15 minutes" + ) break # Only query datagrepper every 30 seconds @@ -679,18 +686,16 @@ class MonitoringUtils: self.print_user(info_log, success=False) return side_tag_name - def clone_and_bump(self, folder, nevrs, conf, name, target=None, new_side_tag=False): + def clone_and_bump( + self, folder, nevrs, conf, name, target=None, new_side_tag=False + ): """Clone the repo, bump the release, commit and push.""" namespace = conf["namespace"] branch = conf["branch"] self.clone_repo( - conf["fedpkg"], - conf["fas_username"], - namespace, - name, - folder=folder, - ) + conf["fedpkg"], conf["fas_username"], namespace, name, folder=folder, + ) gitfolder = os.path.join(folder, name) self.switch_branch(conf["fedpkg"], branch, folder=gitfolder) side_tag_name = None @@ -714,17 +719,13 @@ def run_command(command, cwd=None): """ output = None try: - output = subprocess.check_output( - command, cwd=cwd, stderr=subprocess.PIPE - ) + output = subprocess.check_output(command, cwd=cwd, stderr=subprocess.PIPE) except subprocess.CalledProcessError as e: if "--password" in command: idx = command.index("--password") command[idx + 1] = "" _log.error( - "Command `{}` return code: `{}`".format( - " ".join(command), e.returncode - ) + "Command `{}` return code: `{}`".format(" ".join(command), e.returncode) ) _log.error("stdout:\n-------\n{}".format(e.stdout)) _log.error("stderr:\n-------\n{}".format(e.stderr))