From f0a4e56c40eabd7ff3b2a8e1800e931dd7c0cf34 Mon Sep 17 00:00:00 2001 From: Adam Miller Date: May 19 2017 20:09:52 +0000 Subject: Add rebuildlist to fedcontainer_rc At the moment, the Factory 2.0 and OSBS work for layered image rebuilds is not ready to production use so we're going to have to identify what images to rebuild (right now, all of them) and then actually rebuild them. This patch introduces that data collection so we can perform builds. Also, a previously known bug was found and resolved in this patch. There was a requests call in the list comprehension which did not allow for any amount of error checking. Signed-off-by: Adam Miller --- diff --git a/ansible/fedcontainerrcdemo.yml b/ansible/fedcontainerrcdemo.yml index ad26d07..0d71f03 100644 --- a/ansible/fedcontainerrcdemo.yml +++ b/ansible/fedcontainerrcdemo.yml @@ -8,6 +8,7 @@ register: query_data - debug: var=query_data.rclist + - debug: var=query_data.rebuildlist - include: "include/release-containers-demo.yml" with_items: "{{query_data.rclist}}" diff --git a/ansible/library/fedcontainer_rc.py b/ansible/library/fedcontainer_rc.py index e789d14..affc126 100755 --- a/ansible/library/fedcontainer_rc.py +++ b/ansible/library/fedcontainer_rc.py @@ -20,7 +20,10 @@ short_description: Query Container Image Release Candidates description: - Query Container Image Release Candidates, will return a list of current candidate images. - - Returns dict entry 'rclist' in return JSON + - Returns dict entry 'rclist' and 'rebuildlist' in return JSON + - rclist - list of release candidate image tags + - rebuildlist - list of containers that are candidates for rebuild + (pending a rebuild is necessary) options: release: description: @@ -96,20 +99,23 @@ def get_cntr_list(pkgdb_api_url, pkgdb_req_params, release_num): return sanitized_cntr_list -def get_rclist(release_num, cntr_list, dg_api_url, dg_req_params): - # type: (str, list, str, dict) -> list +def get_lists(release_num, cntr_list, dg_api_url, dg_req_params): + # type: (str, list, str, dict) -> tuple """ - Query datagrepper to get a list of Release Candidates for a specified - Fedora Release based on request params + Query datagrepper to get a list of Release Candidates and Rebuild Candidates + for a specified Fedora Release based on request params :release_num: str, Fedora Release Number to get RC list for :cntr_list: list, list of all containers to query for :dg_api_url: str, datagrepper REST API URL :dg_req_params: dict, python requests params to use in datagrepper query + + :return: tuple(list, list), rclist and rebuildlist """ - # The list we will return, initialized here so we can append to it + # The lists we will return, initialized here so we can append to them rclist = [] + rebuildlist = [] # Set the FGC prefix as the "repo" inside the registry as per documented # layered image naming guidelines: @@ -122,33 +128,40 @@ def get_rclist(release_num, cntr_list, dg_api_url, dg_req_params): try: # Query datagrepper, extract raw_messages, list comprehension # filter, grab only the latest (index 0) - full_nvr = [ - "{}-{}-{}".format( - cntr_msg[u'msg'][u'name'], - cntr_msg[u'msg'][u'version'], - cntr_msg[u'msg'][u'release'] - ) for cntr_msg in requests.get( - dg_api_url, - params=dict( - package=cntr[u'name'], - **dg_req_params - ) - ).json()[u'raw_messages'] - if "f{}docker".format(release_num) in cntr_msg[u'msg'][u'release'] - ][0] - - # We want the full Name-Version-Release, but we also want the - # shorthand Name and Name-Version references as well - fnvr_split = full_nvr.split('-') - rclist.append("{}/{}:{}".format(fgc, '-'.join(fnvr_split[:-2]), '-'.join(fnvr_split[-2:]))) - rclist.append("{}/{}:{}".format(fgc, '-'.join(fnvr_split[:-2]),fnvr_split[-2:][0])) - rclist.append("{}/{}".format(fgc, '-'.join(fnvr_split[:-2]))) + # + # Check for both the f##docker and f##container disttags for + # backwards compat after the namespace migration + nvr_request = requests.get( + dg_api_url, + params=dict( + package=cntr[u'name'], + **dg_req_params + ) + ) + if nvr_request.status_code == 200: + full_nvr = [ + "{}-{}-{}".format( + cntr_msg[u'msg'][u'name'], + cntr_msg[u'msg'][u'version'], + cntr_msg[u'msg'][u'release'] + ) for cntr_msg in nvr_request.json()[u'raw_messages'] + if "f{}docker".format(release_num) in cntr_msg[u'msg'][u'release'] + or "f{}container".format(release_num) in cntr_msg[u'msg'][u'release'] + ][0] + + # We want the full Name-Version-Release, but we also want the + # shorthand Name and Name-Version references as well + fnvr_split = full_nvr.split('-') + rclist.append("{}/{}:{}".format(fgc, '-'.join(fnvr_split[:-2]), '-'.join(fnvr_split[-2:]))) + rclist.append("{}/{}:{}".format(fgc, '-'.join(fnvr_split[:-2]),fnvr_split[-2:][0])) + rclist.append("{}/{}".format(fgc, '-'.join(fnvr_split[:-2]))) + rebuildlist.append("{}".format('-'.join(fnvr_split[:-2]))) except IndexError: # This happens when we have a container branched in pkgdb but nobody # has done a build in koji yet. pass - return rclist + return (rclist, rebuildlist) def get_releasecandidates(release_num): # type: (int) -> list @@ -175,7 +188,7 @@ def get_releasecandidates(release_num): "user": "containerbuild", "rows_per_page": 100 } - rclist = get_rclist(release_num, cntr_list, dg_api_url, dg_req_params) + rclist = get_lists(release_num, cntr_list, dg_api_url, dg_req_params) return rclist @@ -191,15 +204,16 @@ def main(): supports_check_mode=False ) - rclist = get_releasecandidates(module.params['release']) + rclist, rebuildlist = get_releasecandidates(module.params['release']) - if rclist: + if rclist and rebuildlist: # Technically this never changes anything on the system, so always mark # changed=False module.exit_json( changed=False, msg="Successfully gathered container release candidates, results in 'rclist'", - rclist=rclist + rclist=rclist, + rebuildlist=rebuildlist, ) else: module.fail_json(