From b671e5089842a11d8f8a6d5823a989cccd23f12c Mon Sep 17 00:00:00 2001 From: Adam Miller Date: Jul 25 2017 17:17:41 +0000 Subject: [PATCH 1/2] switch from datagrepper to koji as source of truth for release candidate builds Signed-off-by: Adam Miller --- diff --git a/ansible/library/fedcontainer_rc.py b/ansible/library/fedcontainer_rc.py index d923471..0cc7272 100755 --- a/ansible/library/fedcontainer_rc.py +++ b/ansible/library/fedcontainer_rc.py @@ -52,6 +52,24 @@ EXAMPLES = ''' import requests +################################################## +# import module snippets +from ansible.module_utils.basic import * + +# Make the module global +module = AnsibleModule( + argument_spec=dict( + release=dict(required=True, type="int", default=None), + query_source=dict(default="pkgdb", choices=["pkgdb", "pdc"]) + ), + supports_check_mode=False +) + +try: + import koji +except ImportError: + module.fail_json("koji python module not found on target system") + # Set default retried for python request queries # (sometimes datagrepper queries time out) requests.adapters.DEFAULT_RETRIES = 5 @@ -64,11 +82,6 @@ requests.adapters.DEFAULT_RETRIES = 5 # dg == datagrepper # fgc == Fedora Generational Core - -# Make the module global -module = None - - def get_cntr_list(pkgdb_url, release_num): # type: (str, int) -> list """ @@ -164,23 +177,18 @@ def get_cntr_list_pdc(pdc_url, release_num): return sanitized_cntr_list -def get_lists(release_num, cntr_list, dg_api_url, dg_req_params): - # type: (int, list, str, dict) -> tuple +def get_lists(release_num, cntr_list): + # type: (int, list) -> tuple """ - Query datagrepper to get a list of Release Candidates and Rebuild Candidates + Query koji to get a list of Release Candidates and Rebuild Candidates for a specified Fedora Release based on request params :release_num: str, Fedora Release Number to get RC list for :cntr_list: list, list of all containers to query for - :dg_api_url: str, datagrepper REST API URL - :dg_req_params: dict, python requests params to use in datagrepper query :return: tuple(list, list), rclist and rebuildlist """ - # Container build user in koji - cntr_user = dg_req_params['user'] - # The lists we will return, initialized here so we can append to them rclist = [] rebuildlist = [] @@ -191,75 +199,40 @@ def get_lists(release_num, cntr_list, dg_api_url, dg_req_params): # https://fedoraproject.org/wiki/Container:Guidelines fgc = "f{}".format(release_num) + # full Name-Version-Release + full_nvr = "" - # Query Datagrepper for latest container build + # Query koji for latest container build for cntr in cntr_list: try: - # Query datagrepper, extract raw_messages, list comprehension - # filter, grab only the latest (index 0) - # - # Check for both the f##docker and f##container disttags for - # backwards compat after the namespace migration - - nvr_request = requests.get( - dg_api_url, - params=dict( - package=cntr, - **dg_req_params - ) + kc = koji.ClientSession("https://koji.fedoraproject.org/kojihub") + + # Little bit of a hack because have to check both for backwards + # compat due to namespace migration + container_builds = kc.getLatestBuilds( + "f{}-container-build".format(release_num), + package=cntr ) + if container_builds: + full_nvr = container_builds[0]['nvr'] - if nvr_request.status_code == 200: - - # Go ahead and just grab the json, we don't need the other bits - # anymore - nvr_request = nvr_request.json() - - # NOTE: - # - # This is a bit of a hack to work around the oscar tagging - # events logged in datagrepper. This is also dependent on - # rows_per_page = 1 being set in dg_req_params - # - # If we didn't get what we were looking for on the first hit, - # then run through the pagination. - if nvr_request[u'raw_messages'][0][u'msg'][u'user'] != cntr_user: - nvr_list = [] - nvr_range = range(2, nvr_request[u'pages']+1) - for nvr_page in nvr_range: - tmp_request = requests.get( - dg_api_url, - params=dict( - page=nvr_page, - package=cntr, - **dg_req_params - ) - ) - if tmp_request.status_code == 200: - nvr_list.append(tmp_request.json()) - else: - errors.append( - "Error attempting to query datagrepper for {}: {}".format( - cntr, - tmp_request.request.url - ) - ) - - nvr_request = [ - nvr_msg for nvr_msg in nvr_list - if nvr_msg[u'raw_messages'][0][u'msg'][u'user'] == cntr_user - ][0] - - full_nvr = [ - "{}-{}-{}".format( - cntr_msg[u'msg'][u'name'], - cntr_msg[u'msg'][u'version'], - cntr_msg[u'msg'][u'release'] - ) for cntr_msg in nvr_request[u'raw_messages'] - if "f{}docker".format(release_num) in cntr_msg[u'msg'][u'release'] - or "f{}container".format(release_num) in cntr_msg[u'msg'][u'release'] - ][0] + else: + # Consider it an error case when it's not found in the container + # tag because it should be + errors.append("{} not found in f{}-container-build tag".format( + cntr, + release_num + ) + ) + docker_builds = kc.getLatestBuilds( + "f{}-docker-build".format(release_num), + package=cntr + ) + if docker_builds: + full_nvr = docker_builds[0]['nvr'] + + if full_nvr: # We want the full Name-Version-Release, but we also want the # shorthand Name and Name-Version references as well fnvr_split = full_nvr.split('-') @@ -268,16 +241,16 @@ def get_lists(release_num, cntr_list, dg_api_url, dg_req_params): rclist.append("{}/{}".format(fgc, '-'.join(fnvr_split[:-2]))) rebuildlist.append("{}".format('-'.join(fnvr_split[:-2]))) else: - errors.append( - "Error attempting to query datagrepper for {}: {}".format( + errors.append("{0} not found in f{1}-container-build or f{1}-docker-build tag".format( cntr, - nvr_request.request.url + release_num ) ) - except IndexError: - # This happens when we have a container branched in pkgdb but nobody - # has done a build in koji yet. - pass + + except Exception, e: + module.exit_json( + msg="Exception occurred attempting to query koji:\n{}".format(e) + ) return (rclist, rebuildlist, errors) @@ -298,14 +271,7 @@ def get_releasecandidates(release_num, query_source): pdc_api_url = "https://pdc.fedoraproject.org" cntr_list = get_cntr_list_pdc(pdc_api_url, release_num) - # datagrepper api vars - dg_api_url = "https://apps.fedoraproject.org/datagrepper/raw" - dg_req_params = { - "topic": "org.fedoraproject.prod.buildsys.tag", - "user": "containerbuild", - "rows_per_page": 1 - } - rc_data_tuple = get_lists(release_num, cntr_list, dg_api_url, dg_req_params) + rc_data_tuple = get_lists(release_num, cntr_list) return rc_data_tuple @@ -314,14 +280,6 @@ def main(): global module - module = AnsibleModule( - argument_spec=dict( - release=dict(required=True, type="int", default=None), - query_source=dict(default="pkgdb", choices=["pkgdb", "pdc"]) - ), - supports_check_mode=False - ) - rclist, rebuildlist, errors = get_releasecandidates( module.params['release'], module.params['query_source']) @@ -342,8 +300,5 @@ def main(): rclist=rclist ) -################################################## -# import module snippets -from ansible.module_utils.basic import * main() # vim: set expandtab sw=4 sts=4 ts=4 From 184849b44f53c2e22ad0567e2d4d2fef9e179569 Mon Sep 17 00:00:00 2001 From: Adam Miller Date: Jul 25 2017 21:58:59 +0000 Subject: [PATCH 2/2] re-use the koji session, make sure to clear full_nvr Signed-off-by: Adam Miller --- diff --git a/ansible/library/fedcontainer_rc.py b/ansible/library/fedcontainer_rc.py index 0cc7272..b237866 100755 --- a/ansible/library/fedcontainer_rc.py +++ b/ansible/library/fedcontainer_rc.py @@ -199,13 +199,14 @@ def get_lists(release_num, cntr_list): # https://fedoraproject.org/wiki/Container:Guidelines fgc = "f{}".format(release_num) - # full Name-Version-Release - full_nvr = "" + + kc = koji.ClientSession("https://koji.fedoraproject.org/kojihub") # Query koji for latest container build - for cntr in cntr_list: + for cntr in set(cntr_list): try: - kc = koji.ClientSession("https://koji.fedoraproject.org/kojihub") + # full Name-Version-Release + full_nvr = "" # Little bit of a hack because have to check both for backwards # compat due to namespace migration