From b76d12d4875b96e14c087deffbbf5c9b99285071 Mon Sep 17 00:00:00 2001
From: Adam Miller <maxamillion@fedoraproject.org>
Date: Jun 15 2017 19:52:23 +0000
Subject: Exclude incorrect user events, better pagination


This patch fixes the issue where data is inconsistent because it
will no longer silently timeout on paginated queries to datagrepper
and subsequently ignore that container, this scenario is now handled
gracefully and failed attempts to query are reported to Ansible.

Also resolved in this patch is the exclusion of an tag operation
that wasn't done by the correct koji user (containerbuild at this
time).

Signed-off-by: Adam Miller <maxamillion@fedoraproject.org>

---

diff --git a/ansible/library/fedcontainer_rc.py b/ansible/library/fedcontainer_rc.py
index affc126..a75244e 100755
--- a/ansible/library/fedcontainer_rc.py
+++ b/ansible/library/fedcontainer_rc.py
@@ -113,9 +113,13 @@ def get_lists(release_num, cntr_list, dg_api_url, dg_req_params):
     :return:        tuple(list, list), rclist and rebuildlist
     """
 
+    # Container build user in koji
+    cntr_user = dg_req_params['user']
+
     # The lists we will return, initialized here so we can append to them
     rclist = []
     rebuildlist = []
+    errors = []
 
     # Set the FGC prefix as the "repo" inside the registry as per documented
     # layered image naming guidelines:
@@ -131,6 +135,7 @@ def get_lists(release_num, cntr_list, dg_api_url, dg_req_params):
             #
             # Check for both the f##docker and f##container disttags for
             # backwards compat after the namespace migration
+
             nvr_request = requests.get(
                 dg_api_url,
                 params=dict(
@@ -138,13 +143,54 @@ def get_lists(release_num, cntr_list, dg_api_url, dg_req_params):
                     **dg_req_params
                 )
             )
+
             if nvr_request.status_code == 200:
+
+                # Go ahead and just grab the json, we don't need the other bits
+                # anymore
+                nvr_request = nvr_request.json()
+
+                # NOTE:
+                #
+                # This is a bit of a hack to work around the oscar tagging
+                # events logged in datagrepper. This is also dependent on
+                # rows_per_page = 1 being set in dg_req_params
+                #
+                # If we didn't get what we were looking for on the first hit,
+                # then run through the pagination.
+                if nvr_request[u'raw_messages'][0][u'msg'][u'user'] != cntr_user:
+                    nvr_list = []
+                    nvr_range = range(2, nvr_request[u'pages']+1)
+                    for nvr_page in nvr_range:
+                        tmp_request = requests.get(
+                            dg_api_url,
+                            params=dict(
+                                page=nvr_page,
+                                package=cntr[u'name'],
+                                **dg_req_params
+                            )
+                        )
+                        if tmp_request.status_code == 200:
+                            nvr_list.append(tmp_request.json())
+                        else:
+                            errors.append(
+                                "Error attempting to query datagrepper for {}: {}".format(
+                                    cntr[u'name'],
+                                    tmp_request.request.url
+                                )
+                            )
+
+                    nvr_request = [
+                        nvr_msg for nvr_msg in nvr_list
+                        if nvr_msg[u'raw_messages'][0][u'msg'][u'user'] == cntr_user
+                    ][0]
+
                 full_nvr = [
                     "{}-{}-{}".format(
                         cntr_msg[u'msg'][u'name'],
                         cntr_msg[u'msg'][u'version'],
                         cntr_msg[u'msg'][u'release']
-                    ) for cntr_msg in nvr_request.json()[u'raw_messages']
+                    ) for cntr_msg in nvr_request[u'raw_messages']
                     if "f{}docker".format(release_num) in cntr_msg[u'msg'][u'release']
                         or "f{}container".format(release_num) in cntr_msg[u'msg'][u'release']
                 ][0]
@@ -156,15 +202,22 @@ def get_lists(release_num, cntr_list, dg_api_url, dg_req_params):
                 rclist.append("{}/{}:{}".format(fgc, '-'.join(fnvr_split[:-2]),fnvr_split[-2:][0]))
                 rclist.append("{}/{}".format(fgc, '-'.join(fnvr_split[:-2])))
                 rebuildlist.append("{}".format('-'.join(fnvr_split[:-2])))
+            else:
+                errors.append(
+                    "Error attempting to query datagrepper for {}: {}".format(
+                        cntr[u'name'],
+                        nvr_request.request.url
+                    )
+                )
         except IndexError:
             # This happens when we have a container branched in pkgdb but nobody
             # has done a build in koji yet.
             pass
 
-    return (rclist, rebuildlist)
+    return (rclist, rebuildlist, errors)
 
 def get_releasecandidates(release_num):
-    # type: (int) -> list
+    # type: (int) -> tuple
     """
     Logic to actually get the release candidate, this is likely to change over
     time (PDC?) so we want it split out from the main control of the module
@@ -186,11 +239,11 @@ def get_releasecandidates(release_num):
     dg_req_params = {
         "topic": "org.fedoraproject.prod.buildsys.tag",
         "user": "containerbuild",
-        "rows_per_page": 100
+        "rows_per_page": 1
     }
-    rclist = get_lists(release_num, cntr_list, dg_api_url, dg_req_params)
+    rc_data_tuple = get_lists(release_num, cntr_list, dg_api_url, dg_req_params)
 
-    return rclist
+    return rc_data_tuple
 
 
 def main():
@@ -204,7 +257,7 @@ def main():
         supports_check_mode=False
     )
 
-    rclist, rebuildlist = get_releasecandidates(module.params['release'])
+    rclist, rebuildlist, errors = get_releasecandidates(module.params['release'])
 
     if rclist and rebuildlist:
         # Technically this never changes anything on the system, so always mark
@@ -214,6 +267,7 @@ def main():
             msg="Successfully gathered container release candidates, results in 'rclist'",
             rclist=rclist,
             rebuildlist=rebuildlist,
+            errors=errors
         )
     else:
         module.fail_json(