From 9cbd07890d6e1706f263572e20057bdaed750627 Mon Sep 17 00:00:00 2001
From: Dusty Mabe <dusty@dustymabe.com>
Date: May 22 2019 13:59:30 +0000
Subject: [PATCH 1/3] compose-tracker.py -> compose_tracker.py


Fixes syntax error when trying to `import compose-tracker` in python.

---

diff --git a/compose-tracker.py b/compose-tracker.py
deleted file mode 100755
index 46bf4c9..0000000
--- a/compose-tracker.py
+++ /dev/null
@@ -1,147 +0,0 @@
-#!/usr/bin/python3
-import datetime
-import fedmsg
-import os
-import re
-import requests
-from libpagure import Pagure
-
-# Set fedmsg logging to not print warnings
-import logging
-logger = logging.getLogger('fedmsg')
-logger.setLevel(logging.ERROR)
-
-# Set local logging 
-logger = logging.getLogger(__name__)
-sh = logging.StreamHandler()
-sh.setFormatter(logging.Formatter('%(asctime)s %(message)s'))
-logger.addHandler(sh)
-logger.setLevel(logging.INFO)
-
-
-# Connect to pagure and set it to point to our repo
-PAGURE_REPO='dusty/failed-composes'
-
-# URL for linking to koji tasks by ID
-KOJI_TASK_URL='https://koji.fedoraproject.org/koji/taskinfo?taskID='
-
-########import json
-########msg = json.loads("""{
-########                "msg": {
-########                    "status": "DOOMED",
-########                    "release_type": "ga",
-########                    "compose_label": null,
-########                    "compose_respin": 0,
-########                    "compose_date": "20180215",
-########                    "release_version": "Bikeshed",
-########                    "location": "http://kojipkgs.fedoraproject.org/compose/Fedora-Modular-Bikeshed-20180215.n.0/compose",
-########                    "compose_type": "nightly",
-########                    "release_is_layered": false,
-########                    "release_name": "Fedora-Modular",
-########                    "release_short": "Fedora-Modular",
-########                    "compose_id": "Fedora-Modular-Bikeshed-20180215.n.0"
-########                  }}
-########"""
-########)
-
-
-def get_supporting_text(line):
-    """ given a log file line determine if it has a koji task ID in it
-        or not and give back an appropriate message
-    """
-    r = re.search('.*failed: (\d{8}).*', line)
-    if r:
-        taskid = r.group(1)
-        text = "- [%s](%s%s)\n" % (taskid, KOJI_TASK_URL, taskid)
-    else:
-        text = "- No Task ID, look at log statement\n"
-    return text
-
-def main():
-    # grab token and connec to pagure
-    token = os.getenv('PAGURE_TOKEN')
-    if token:
-        logger.info("Using detected token to talk to pagure.") 
-        pg = Pagure(pagure_token=token)
-    else:
-        logger.info("No pagure token was detected.") 
-        logger.info("This script will run but won't be able to create new issues.")
-        pg = Pagure()
-
-    # Set the repo to create new issues against
-    pg.repo=PAGURE_REPO
-
-    # Used for printing out a value when the day has changed
-    date = datetime.date.today()
-
-    # Grab messages from fedmsg and process them as we go
-    logger.info("Starting listening for fedmsgs..") 
-    for name, endpoint, topic, msg in fedmsg.tail_messages():
-        logger.debug(topic)
-
-        # Print out a log statement if the day has changed
-        today = datetime.date.today()
-        if today != date:
-            date = today
-            logger.info('mark')
-
-        # https://apps.fedoraproject.org/datagrepper/raw?topic=org.fedoraproject.prod.pungi.compose.status.change&delta=100000
-        if "pungi.compose.status.change" in topic:
-            print('.', end='') # some sort of indicator of progress
-            print('.') # some sort of indicator of progress
-
-            status = msg['msg']['status']
-
-            # If we are in good states then continue
-            if status in ['FINISHED', 'STARTED']:
-                continue
-
-            # We have a compose that either failed or had missing artifacts
-            # create a new issue.
-            title = msg['msg']['compose_id'] + ' ' + status
-            logfileurl = msg['msg']['location'] + '/../logs/global/pungi.global.log'
-            logger.info("%s\t%s" % (title, logfileurl))
-
-            # variable to hold description for issue
-            content = "[pungi.global.log](%s)\n\n" % logfileurl
-
-            # If we fail to get the log file contents then we'll just
-            # best effort put a message in the issue.
-            try:
-                lines = requests.get(logfileurl).text.splitlines()
-            except:
-                logger.info("Failed to retrieve log contents from server.. skipping analysis")
-                content+= "Failed to retrieve log contents from server.. skipping analysis"
-                lines = []
-                pass
-
-            for x in range(1, len(lines)):
-                line = lines[x-1][20:]   # trim date off log lines
-                nextline = lines[x][20:] # trim date off log lines
-
-                # If this is a [FAIL] line then we take it and the
-                # next line and add them in markdown format. Also grab
-                # the taskid if we can and print a hyperlink to koji
-                if re.search('\[FAIL\]', line):
-                    content+= get_supporting_text(nextline)
-                    content+= "```\n%s\n%s\n```\n\n" % (line, nextline)
-
-                # If this is the Compose run failed line, then add it
-                # to the description too
-                if re.search('.*Compose run failed.*', line):
-                    content+= ("- Compose run failed because: %s\n" %
-                                            get_supporting_text(line))
-                    content+= "```\n%s\n```\n" % (line)
-
-            logger.debug(content)
-
-            # pull only part of the compose ID for the tag to set
-            tag = re.search('(.*)-\d{8}', msg['msg']['compose_id']).group(1)
-            #TODO figure out how to set tag on an issue
-            # Should be able to do this now https://pagure.io/libpagure/issue/31
-
-            if token:
-                pg.create_issue(title=title, content=content)
-
-if __name__ == '__main__':
-    main()
diff --git a/compose_tracker.py b/compose_tracker.py
new file mode 100755
index 0000000..46bf4c9
--- /dev/null
+++ b/compose_tracker.py
@@ -0,0 +1,147 @@
+#!/usr/bin/python3
+import datetime
+import fedmsg
+import os
+import re
+import requests
+from libpagure import Pagure
+
+# Set fedmsg logging to not print warnings
+import logging
+logger = logging.getLogger('fedmsg')
+logger.setLevel(logging.ERROR)
+
+# Set local logging 
+logger = logging.getLogger(__name__)
+sh = logging.StreamHandler()
+sh.setFormatter(logging.Formatter('%(asctime)s %(message)s'))
+logger.addHandler(sh)
+logger.setLevel(logging.INFO)
+
+
+# Connect to pagure and set it to point to our repo
+PAGURE_REPO='dusty/failed-composes'
+
+# URL for linking to koji tasks by ID
+KOJI_TASK_URL='https://koji.fedoraproject.org/koji/taskinfo?taskID='
+
+########import json
+########msg = json.loads("""{
+########                "msg": {
+########                    "status": "DOOMED",
+########                    "release_type": "ga",
+########                    "compose_label": null,
+########                    "compose_respin": 0,
+########                    "compose_date": "20180215",
+########                    "release_version": "Bikeshed",
+########                    "location": "http://kojipkgs.fedoraproject.org/compose/Fedora-Modular-Bikeshed-20180215.n.0/compose",
+########                    "compose_type": "nightly",
+########                    "release_is_layered": false,
+########                    "release_name": "Fedora-Modular",
+########                    "release_short": "Fedora-Modular",
+########                    "compose_id": "Fedora-Modular-Bikeshed-20180215.n.0"
+########                  }}
+########"""
+########)
+
+
+def get_supporting_text(line):
+    """ given a log file line determine if it has a koji task ID in it
+        or not and give back an appropriate message
+    """
+    r = re.search('.*failed: (\d{8}).*', line)
+    if r:
+        taskid = r.group(1)
+        text = "- [%s](%s%s)\n" % (taskid, KOJI_TASK_URL, taskid)
+    else:
+        text = "- No Task ID, look at log statement\n"
+    return text
+
+def main():
+    # grab token and connec to pagure
+    token = os.getenv('PAGURE_TOKEN')
+    if token:
+        logger.info("Using detected token to talk to pagure.") 
+        pg = Pagure(pagure_token=token)
+    else:
+        logger.info("No pagure token was detected.") 
+        logger.info("This script will run but won't be able to create new issues.")
+        pg = Pagure()
+
+    # Set the repo to create new issues against
+    pg.repo=PAGURE_REPO
+
+    # Used for printing out a value when the day has changed
+    date = datetime.date.today()
+
+    # Grab messages from fedmsg and process them as we go
+    logger.info("Starting listening for fedmsgs..") 
+    for name, endpoint, topic, msg in fedmsg.tail_messages():
+        logger.debug(topic)
+
+        # Print out a log statement if the day has changed
+        today = datetime.date.today()
+        if today != date:
+            date = today
+            logger.info('mark')
+
+        # https://apps.fedoraproject.org/datagrepper/raw?topic=org.fedoraproject.prod.pungi.compose.status.change&delta=100000
+        if "pungi.compose.status.change" in topic:
+            print('.', end='') # some sort of indicator of progress
+            print('.') # some sort of indicator of progress
+
+            status = msg['msg']['status']
+
+            # If we are in good states then continue
+            if status in ['FINISHED', 'STARTED']:
+                continue
+
+            # We have a compose that either failed or had missing artifacts
+            # create a new issue.
+            title = msg['msg']['compose_id'] + ' ' + status
+            logfileurl = msg['msg']['location'] + '/../logs/global/pungi.global.log'
+            logger.info("%s\t%s" % (title, logfileurl))
+
+            # variable to hold description for issue
+            content = "[pungi.global.log](%s)\n\n" % logfileurl
+
+            # If we fail to get the log file contents then we'll just
+            # best effort put a message in the issue.
+            try:
+                lines = requests.get(logfileurl).text.splitlines()
+            except:
+                logger.info("Failed to retrieve log contents from server.. skipping analysis")
+                content+= "Failed to retrieve log contents from server.. skipping analysis"
+                lines = []
+                pass
+
+            for x in range(1, len(lines)):
+                line = lines[x-1][20:]   # trim date off log lines
+                nextline = lines[x][20:] # trim date off log lines
+
+                # If this is a [FAIL] line then we take it and the
+                # next line and add them in markdown format. Also grab
+                # the taskid if we can and print a hyperlink to koji
+                if re.search('\[FAIL\]', line):
+                    content+= get_supporting_text(nextline)
+                    content+= "```\n%s\n%s\n```\n\n" % (line, nextline)
+
+                # If this is the Compose run failed line, then add it
+                # to the description too
+                if re.search('.*Compose run failed.*', line):
+                    content+= ("- Compose run failed because: %s\n" %
+                                            get_supporting_text(line))
+                    content+= "```\n%s\n```\n" % (line)
+
+            logger.debug(content)
+
+            # pull only part of the compose ID for the tag to set
+            tag = re.search('(.*)-\d{8}', msg['msg']['compose_id']).group(1)
+            #TODO figure out how to set tag on an issue
+            # Should be able to do this now https://pagure.io/libpagure/issue/31
+
+            if token:
+                pg.create_issue(title=title, content=content)
+
+if __name__ == '__main__':
+    main()

From 57c62ddad4e848aef54f6ee892547676bbe6cf27 Mon Sep 17 00:00:00 2001
From: Dusty Mabe <dusty@dustymabe.com>
Date: May 23 2019 16:42:21 +0000
Subject: [PATCH 2/3] move to fedora-messaging


This required the following changes:

- migrate to calling fedora-messaging CLI and providing a callback
- creating a new ComposeTracker class and implementing __call__ func
- removing main()

---

diff --git a/Dockerfile b/Dockerfile
index 60e13db..6abc1e3 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,8 +1,30 @@
 FROM registry.fedoraproject.org/fedora:30
 
+# set PYTHONUNBUFFERED env var to non-empty string so that our
+# periods with no newline get printed immediately to the screen
+ENV PYTHONUNBUFFERED=true
+
 # Install pagure/fedmsg libraries
-RUN dnf -y install python3-libpagure python3-fedmsg && dnf clean all
+RUN dnf -y install python3-libpagure fedora-messaging && dnf clean all
+
+RUN mkdir /work
+WORKDIR /work
+
+# Copy the fedora config for fedora-messaging and also generate a random UUID
+# https://fedora-messaging.readthedocs.io/en/latest/fedora-broker.html#getting-connected
+# Note this will mean that if there is more than one container running
+# using this image they will be reading from the same queue. Generally
+# I expect this to only be running in one place.
+RUN sed -e "s/[0-9a-f]\{8\}-[0-9a-f]\{4\}-[0-9a-f]\{4\}-[0-9a-f]\{4\}-[0-9a-f]\{12\}/$(uuidgen)/g" /etc/fedora-messaging/fedora.toml > /work/my_config.toml
+
+# Lower log levels to WARNING level
+RUN sed -i 's/INFO/WARNING/' /work/my_config.toml
+# We only care about pungi.compose.status.change messages
+RUN sed -i 's/^routing_keys.*$/routing_keys = ["org.fedoraproject.prod.pungi.compose.status.change"]/' /work/my_config.toml
 
-ADD compose-tracker.py /usr/local/bin/
+# Put compose-tracker into a location that can be imported
+ADD compose_tracker.py /usr/lib/python3.7/site-packages/
 
-CMD /usr/local/bin/compose-tracker.py
+# Call fedora-messaging CLI and tell it to use the ComposeTracker
+# class from the compose-tracker module.
+CMD fedora-messaging --conf /work/my_config.toml consume --callback=compose_tracker:Consumer
diff --git a/compose_tracker.py b/compose_tracker.py
index 46bf4c9..114748c 100755
--- a/compose_tracker.py
+++ b/compose_tracker.py
@@ -1,15 +1,11 @@
 #!/usr/bin/python3
 import datetime
-import fedmsg
+import fedora_messaging
 import os
 import re
 import requests
 from libpagure import Pagure
-
-# Set fedmsg logging to not print warnings
 import logging
-logger = logging.getLogger('fedmsg')
-logger.setLevel(logging.ERROR)
 
 # Set local logging 
 logger = logging.getLogger(__name__)
@@ -25,6 +21,8 @@ PAGURE_REPO='dusty/failed-composes'
 # URL for linking to koji tasks by ID
 KOJI_TASK_URL='https://koji.fedoraproject.org/koji/taskinfo?taskID='
 
+# We are processing the org.fedoraproject.prod.pungi.compose.status.change topic
+# https://apps.fedoraproject.org/datagrepper/raw?topic=org.fedoraproject.prod.pungi.compose.status.change&delta=100000
 ########import json
 ########msg = json.loads("""{
 ########                "msg": {
@@ -44,104 +42,104 @@ KOJI_TASK_URL='https://koji.fedoraproject.org/koji/taskinfo?taskID='
 ########"""
 ########)
 
-
-def get_supporting_text(line):
-    """ given a log file line determine if it has a koji task ID in it
-        or not and give back an appropriate message
-    """
-    r = re.search('.*failed: (\d{8}).*', line)
-    if r:
-        taskid = r.group(1)
-        text = "- [%s](%s%s)\n" % (taskid, KOJI_TASK_URL, taskid)
-    else:
-        text = "- No Task ID, look at log statement\n"
-    return text
-
-def main():
-    # grab token and connec to pagure
-    token = os.getenv('PAGURE_TOKEN')
-    if token:
-        logger.info("Using detected token to talk to pagure.") 
-        pg = Pagure(pagure_token=token)
-    else:
-        logger.info("No pagure token was detected.") 
-        logger.info("This script will run but won't be able to create new issues.")
-        pg = Pagure()
-
-    # Set the repo to create new issues against
-    pg.repo=PAGURE_REPO
-
-    # Used for printing out a value when the day has changed
-    date = datetime.date.today()
-
-    # Grab messages from fedmsg and process them as we go
-    logger.info("Starting listening for fedmsgs..") 
-    for name, endpoint, topic, msg in fedmsg.tail_messages():
-        logger.debug(topic)
+class Consumer(object):
+    def __init__(self):
+        self.token = os.getenv('PAGURE_TOKEN')
+        if self.token:
+            logger.info("Using detected token to talk to pagure.") 
+            self.pg = Pagure(pagure_token=self.token)
+        else:
+            logger.info("No pagure token was detected.") 
+            logger.info("This script will run but won't be able to create new issues.")
+            self.pg = Pagure()
+
+        # Set the repo to create new issues against
+        self.pg.repo=PAGURE_REPO
+
+        # Used for printing out a value when the day has changed
+        self.date = datetime.date.today()
+
+    def get_supporting_text(self, line):
+        """ given a log file line determine if it has a koji task ID in it
+            or not and give back an appropriate message
+        """
+        r = re.search('.*failed: (\d{8}).*', line)
+        if r:
+            taskid = r.group(1)
+            text = "- [%s](%s%s)\n" % (taskid, KOJI_TASK_URL, taskid)
+        else:
+            text = "- No Task ID, look at log statement\n"
+        return text
+
+    def __call__(self, message: fedora_messaging.api.Message):
+        logger.debug(message.topic)
+        logger.debug(message.body)
+
+        # Grab the raw message body and the status from that
+        msg = message.body
+        status = msg['msg']['status']
 
         # Print out a log statement if the day has changed
         today = datetime.date.today()
-        if today != date:
-            date = today
+        if today != self.date:
+            self.date = today
+            print('') # get to the next line
             logger.info('mark')
 
-        # https://apps.fedoraproject.org/datagrepper/raw?topic=org.fedoraproject.prod.pungi.compose.status.change&delta=100000
-        if "pungi.compose.status.change" in topic:
+        # If we are in good states then continue
+        if status in ['FINISHED', 'STARTED']:
+            # Print out an indicator of progress (i.e. composes have been
+            # finishing and are successful. Note this won't print out
+            # right unless we are setting PYTHONUNBUFFERED env var
+            # to a non-empty string
             print('.', end='') # some sort of indicator of progress
-            print('.') # some sort of indicator of progress
-
-            status = msg['msg']['status']
-
-            # If we are in good states then continue
-            if status in ['FINISHED', 'STARTED']:
-                continue
-
-            # We have a compose that either failed or had missing artifacts
-            # create a new issue.
-            title = msg['msg']['compose_id'] + ' ' + status
-            logfileurl = msg['msg']['location'] + '/../logs/global/pungi.global.log'
-            logger.info("%s\t%s" % (title, logfileurl))
-
-            # variable to hold description for issue
-            content = "[pungi.global.log](%s)\n\n" % logfileurl
-
-            # If we fail to get the log file contents then we'll just
-            # best effort put a message in the issue.
-            try:
-                lines = requests.get(logfileurl).text.splitlines()
-            except:
-                logger.info("Failed to retrieve log contents from server.. skipping analysis")
-                content+= "Failed to retrieve log contents from server.. skipping analysis"
-                lines = []
-                pass
-
-            for x in range(1, len(lines)):
-                line = lines[x-1][20:]   # trim date off log lines
-                nextline = lines[x][20:] # trim date off log lines
-
-                # If this is a [FAIL] line then we take it and the
-                # next line and add them in markdown format. Also grab
-                # the taskid if we can and print a hyperlink to koji
-                if re.search('\[FAIL\]', line):
-                    content+= get_supporting_text(nextline)
-                    content+= "```\n%s\n%s\n```\n\n" % (line, nextline)
-
-                # If this is the Compose run failed line, then add it
-                # to the description too
-                if re.search('.*Compose run failed.*', line):
-                    content+= ("- Compose run failed because: %s\n" %
-                                            get_supporting_text(line))
-                    content+= "```\n%s\n```\n" % (line)
-
-            logger.debug(content)
-
-            # pull only part of the compose ID for the tag to set
-            tag = re.search('(.*)-\d{8}', msg['msg']['compose_id']).group(1)
-            #TODO figure out how to set tag on an issue
-            # Should be able to do this now https://pagure.io/libpagure/issue/31
-
-            if token:
-                pg.create_issue(title=title, content=content)
-
-if __name__ == '__main__':
-    main()
+            return
+
+        print('') # get to the next line
+
+        # We have a compose that either failed or had missing artifacts
+        # create a new issue.
+        title = msg['msg']['compose_id'] + ' ' + status
+        logfileurl = msg['msg']['location'] + '/../logs/global/pungi.global.log'
+        logger.info("%s\t%s" % (title, logfileurl))
+
+        # variable to hold description for issue
+        content = "[pungi.global.log](%s)\n\n" % logfileurl
+
+        # If we fail to get the log file contents then we'll just
+        # best effort put a message in the issue.
+        try:
+            lines = requests.get(logfileurl).text.splitlines()
+        except:
+            logger.info("Failed to retrieve log contents from server.. skipping analysis")
+            content+= "Failed to retrieve log contents from server.. skipping analysis"
+            lines = []
+            pass
+
+        for x in range(1, len(lines)):
+            line = lines[x-1][20:]   # trim date off log lines
+            nextline = lines[x][20:] # trim date off log lines
+
+            # If this is a [FAIL] line then we take it and the
+            # next line and add them in markdown format. Also grab
+            # the taskid if we can and print a hyperlink to koji
+            if re.search('\[FAIL\]', line):
+                content+= self.get_supporting_text(nextline)
+                content+= "```\n%s\n%s\n```\n\n" % (line, nextline)
+
+            # If this is the Compose run failed line, then add it
+            # to the description too
+            if re.search('.*Compose run failed.*', line):
+                content+= ("- Compose run failed because: %s\n" %
+                                        self.get_supporting_text(line))
+                content+= "```\n%s\n```\n" % (line)
+
+        logger.debug(content)
+
+        # pull only part of the compose ID for the tag to set
+        tag = re.search('(.*)-\d{8}', msg['msg']['compose_id']).group(1)
+        #TODO figure out how to set tag on an issue
+        # Should be able to do this now https://pagure.io/libpagure/issue/31
+
+        if self.token:
+            self.pg.create_issue(title=title, content=content)

From 791efef546076f4f91a9290f3ca337def3faa2da Mon Sep 17 00:00:00 2001
From: Dusty Mabe <dusty@dustymabe.com>
Date: May 23 2019 20:01:17 +0000
Subject: [PATCH 3/3] compose-tracker: fixup logging


- remove duplicate log messages
- apply a format that's easier to read

---

diff --git a/Dockerfile b/Dockerfile
index 6abc1e3..15c9197 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -19,6 +19,10 @@ RUN sed -e "s/[0-9a-f]\{8\}-[0-9a-f]\{4\}-[0-9a-f]\{4\}-[0-9a-f]\{4\}-[0-9a-f]\{
 
 # Lower log levels to WARNING level
 RUN sed -i 's/INFO/WARNING/' /work/my_config.toml
+
+# Set the format for the log messages
+RUN sed -i 's/format =.*$/format = "%(asctime)s %(levelname)s %(name)s - %(message)s"/' /work/my_config.toml
+
 # We only care about pungi.compose.status.change messages
 RUN sed -i 's/^routing_keys.*$/routing_keys = ["org.fedoraproject.prod.pungi.compose.status.change"]/' /work/my_config.toml
 
diff --git a/compose_tracker.py b/compose_tracker.py
index 114748c..4600d12 100755
--- a/compose_tracker.py
+++ b/compose_tracker.py
@@ -9,9 +9,6 @@ import logging
 
 # Set local logging 
 logger = logging.getLogger(__name__)
-sh = logging.StreamHandler()
-sh.setFormatter(logging.Formatter('%(asctime)s %(message)s'))
-logger.addHandler(sh)
 logger.setLevel(logging.INFO)
 
 

