From 2cc4f94c59d23bdf7a65336084c6c38b46c9f15d Mon Sep 17 00:00:00 2001
From: Fabien Boucher <fboucher@redhat.com>
Date: Apr 06 2021 10:43:25 +0000
Subject: Setup a periodic job to clean too old koji tasks


---

diff --git a/files/clean-koji-stalled-tasks.py b/files/clean-koji-stalled-tasks.py
new file mode 100755
index 0000000..569c33a
--- /dev/null
+++ b/files/clean-koji-stalled-tasks.py
@@ -0,0 +1,125 @@
+#!/bin/env python3
+
+from typing import List, Optional
+from subprocess import run, CalledProcessError
+from datetime import datetime, timedelta
+import unittest
+import argparse
+import sys
+
+KOJI_CMD = ["koji"]
+MAX_HR = 12
+
+mocked_list_tasks_ids = """
+64642854 20   zuul                 OPEN     noarch     build (rawhide, libkcapi-1.2.1-1.fc35.src.rpm)
+64642908 19   zuul                 OPEN     ppc64le     +buildArch (libkcapi-1.2.1-1.fc35.src.rpm, ppc64le)
+64642862 20   zuul                 OPEN     noarch     build (rawhide, libkcapi-1.2.1-1.fc35.src.rpm)
+"""
+
+mocked_task_info_1 = """
+Task: 64642801
+Type: build
+Owner: zuul
+State: open
+Created: Fri Mar 26 15:43:39 2021
+Started: Thu Apr  1 21:39:23 2021
+Host: buildvm-s390x-11.s390.fedoraproject.org
+"""
+
+now = datetime.strftime(datetime.now(), "%c")
+mocked_task_info_2 = """
+Task: 64642802
+Type: build
+Owner: zuul
+State: open
+Created: %s
+Started: %s
+Host: buildvm-s390x-11.s390.fedoraproject.org
+""" % (
+    now,
+    now,
+)
+
+
+def call_koji_cmd(args: List[str], mock: bool = False) -> Optional[str]:
+    if mock and args == ["list-tasks", "--quiet", "--mine"]:
+        return mocked_list_tasks_ids
+    if mock and args == ["taskinfo", "64642801"]:
+        return mocked_task_info_1
+    if mock and args == ["taskinfo", "64642802"]:
+        return mocked_task_info_2
+    cmd = KOJI_CMD + args
+    compp = run(cmd, capture_output=True, text=True)
+    try:
+        compp.check_returncode()
+    except CalledProcessError as exc:
+        print("Command %s did not success: %s" % (" ".join(cmd), exc))
+        return None
+    return compp.stdout
+
+
+def get_tasks_ids_list(mocked: bool = False) -> Optional[List[str]]:
+    cmd_output = call_koji_cmd(["list-tasks", "--quiet", "--mine"], mock=mocked)
+    if not cmd_output:
+        return None
+    return list(
+        map(lambda l: l.split()[0], filter(lambda x: x, cmd_output.splitlines()))
+    )
+
+
+def check_stalled_task(task_id: str, mocked: bool = False) -> bool:
+    cmd_output = call_koji_cmd(["taskinfo", task_id], mock=mocked)
+    if not cmd_output:
+        return False
+    started_line = list(
+        filter(
+            lambda l: l.startswith("Started: "),
+            filter(lambda x: x, cmd_output.splitlines()),
+        )
+    )
+    if not started_line:
+        return False
+    started_date = datetime.strptime(started_line[0].replace("Started: ", ""), "%c")
+    stalled = started_date < datetime.now() - timedelta(hours=MAX_HR)
+    print("Tasks %s started at %s [stalled: %s]" % (task_id, started_date, stalled))
+    return stalled
+
+
+def run_check(noop: bool) -> None:
+    tasks_ids = get_tasks_ids_list()
+    if tasks_ids:
+        for task_id in tasks_ids:
+            if not check_stalled_task(task_id):
+                continue
+            if not noop:
+                print("Cancelling task %s ..." % task_id)
+                call_koji_cmd(["cancel", task_id])
+            else:
+                print("Will cancel task %s ..." % task_id)
+
+
+### Run tests with python -m unittest clean-koji-stalled-tasks.py ###
+class ToolTests(unittest.TestCase):
+    def test_get_tasks_ids_list(self):
+        tasks_ids = get_tasks_ids_list(mocked=True)
+        self.assertListEqual(["64642854", "64642908", "64642862"], tasks_ids)
+
+    def test_check_stalled_tasks(self):
+        self.assertTrue(check_stalled_task("64642801", mocked=True))
+        self.assertFalse(check_stalled_task("64642802", mocked=True))
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--noop", help="Do not cancel tasks (only report)", action="store_true"
+    )
+    if "-m unittest" in sys.argv[0]:
+        # Skip main if running local tests
+        pass
+    else:
+        args = parser.parse_args()
+        run_check(args.noop)
+
+
+main()
diff --git a/playbooks/koji/clean-stalled-koji-tasks.yaml b/playbooks/koji/clean-stalled-koji-tasks.yaml
new file mode 100644
index 0000000..5432c72
--- /dev/null
+++ b/playbooks/koji/clean-stalled-koji-tasks.yaml
@@ -0,0 +1,6 @@
+- hosts: all
+  tasks:
+    - include_role:
+        name: krb-secret
+    - name: Run cleaning script
+      script: files/clean-koji-stalled-tasks.py
diff --git a/zuul.d/jobs.dhall b/zuul.d/jobs.dhall
index 62a5512..101c64b 100644
--- a/zuul.d/jobs.dhall
+++ b/zuul.d/jobs.dhall
@@ -107,6 +107,18 @@ let check_for_arches =
       , nodeset = Some default_nodeset
       }
 
+let clean_stalled_koji_tasks =
+      Zuul.Job::{
+      , name = "${job-prefix}clean-stalled-koji-tasks"
+      , protected = Some True
+      , description = Some "Cancel Koji tasks running for more than 12 hours"
+      , nodeset = Some default_nodeset
+      , roles = Some [ { zuul = "zuul-distro-jobs" } ]
+      , run = Some "playbooks/koji/clean-stalled-koji-tasks.yaml"
+      , secrets = Some
+        [ Zuul.Job.Secret::{ name = "krb_keytab", secret = "krb_keytab" } ]
+      }
+
 let common_koji_rpm_build =
       Zuul.Job::{
       , name = "${job-prefix}common-koji-rpm-build"
@@ -199,7 +211,11 @@ let generateRpmBuildJobs
               Branches.all
 
 let Jobs =
-        [ check_for_tests, check_for_arches, common_koji_rpm_build ]
+        [ clean_stalled_koji_tasks
+        , check_for_tests
+        , check_for_arches
+        , common_koji_rpm_build
+        ]
       # generateRpmBuildJobs KojiBuild.Type.Scratch
       # generateRpmBuildJobs KojiBuild.Type.Final
 
diff --git a/zuul.d/jobs.yaml b/zuul.d/jobs.yaml
index 79996e0..406ac4c 100644
--- a/zuul.d/jobs.yaml
+++ b/zuul.d/jobs.yaml
@@ -1,6 +1,18 @@
 # Code generated by dhall-to-yaml.  DO NOT EDIT.
 
 - job:
+    name: clean-stalled-koji-tasks
+    description: Cancel Koji tasks running for more than 12 hours
+    run: playbooks/koji/clean-stalled-koji-tasks.yaml
+    nodeset: fedora-33-container
+    protected: true
+    roles:
+      - zuul: zuul-distro-jobs
+    secrets:
+      - name: krb_keytab
+        secret: krb_keytab
+
+- job:
     name: check-for-tests
     description: Check the project has a tests/tests.yml
     run: playbooks/rpm/check-for-tests.yaml
diff --git a/zuul.d/projects.yaml b/zuul.d/projects.yaml
index ab8f008..880181c 100644
--- a/zuul.d/projects.yaml
+++ b/zuul.d/projects.yaml
@@ -99,6 +99,9 @@
 
 # Linter job for the current repository
 - project:
+    periodic:
+      jobs:
+        - clean-stalled-koji-tasks
     check:
       jobs:
         - linters:

