From d93662092593f38e2571475bbf8bee52cd929ce1 Mon Sep 17 00:00:00 2001 From: Adam Miller Date: Mar 24 2017 17:28:45 +0000 Subject: filter retired/blacklisted cntrs - fedcontainer_rc Refactoring the container release candidate code a bit and include a blacklist functionality for images that should never have existed in the first place (mostly due to growing pains in the Atomic WG around container best practices and guidelines, we likely won't need this in the future). Also add filtering for Retired packages. One thing to note is that right now this is querying pkgdb and datagrepper but in the future it should be asking koji about the list of containers in the fXX-docker-pending tag. Unfortunately at this time we don't have the ingress point (CI/CD test pipeline) to trigger an action to move containers into that tag. We will switch to that ASAP but this will get us moving for the time being. Signed-off-by: Adam Miller --- diff --git a/ansible/library/fedcontainer_rc.py b/ansible/library/fedcontainer_rc.py index 64204cd..dae467f 100755 --- a/ansible/library/fedcontainer_rc.py +++ b/ansible/library/fedcontainer_rc.py @@ -42,57 +42,83 @@ EXAMPLES = ''' import requests -def get_releasecandidates(release_num): - # type: (int) -> list - """ - Logic to actually get the release candidate, this is likely to change over - time (PDC?) so we want it split out from the main control of the module - """ +# Set default retried for python request queries +# (sometimes datagrepper queries time out) +requests.adapters.DEFAULT_RETRIES = 5 - # NOTE: the following abbreviations are true for the vars below - # - # cntr == container - # pkgdb == package db - # dg == datagrepper - # fgc == Fedora Generational Core +# NOTE: the following abbreviations are true for the vars through out the code +# +# cntr == container +# pkgdb == package db +# dg == datagrepper +# fgc == Fedora Generational Core - # Set the FGC prefix as the "repo" inside the registry as per documented - # layered image naming guidelines: - # https://fedoraproject.org/wiki/Container:Guidelines - fgc = "f{}".format(release_num) - # The list we will return, initialized here so we can append to it - rclist = [] +# Make the module global +module = None - # pkgdb2 api vars - pkgdb_api_url = "https://admin.fedoraproject.org/pkgdb/api/packages/*" - pkgdb_req_params = {"namespace": "docker"} +def get_cntr_list(pkgdb_api_url, pkgdb_req_params, release_num): + # type: (str, dict) -> list + """ + Query PackageDB (pkgdb) for container list, sanitize out containers that + are marketed dead or blacklisted. - # datagrepper api vars - dg_api_url = "https://apps.fedoraproject.org/datagrepper/raw" - dg_req_params = { - "topic": "org.fedoraproject.prod.buildsys.tag", - "user": "containerbuild", - "rows_per_page": 100 - } - # FIXME - not sure which topic to be looking for + :pkgdb_api_url: str, PackageDB REST API URL + :pkgdb_req_params: dict, python requests params to use in pkgdb query + """ + + # Get blacklist from https://pagure.io/releng/container-blacklist + blacklist_url = "https://pagure.io/releng/container-blacklist/raw/master/f/blacklist.json" # First query pkgdb for all container images we currently have cntr_request = requests.get(pkgdb_api_url, params=pkgdb_req_params) # Start with page 1 response from pkgdb, grab names and then iterate # through pagination - cntr_data = cntr_request.json()[u'packages'] + cntr_list = cntr_request.json()[u'packages'] cpage_range = range(2, cntr_request.json()[u'page_total']+1) for cntr_page in cpage_range: - cntr_data += requests.get( + cntr_list += requests.get( pkgdb_api_url, params=dict(page=cntr_page, **pkgdb_req_params) ).json()[u'packages'] + + blacklist = requests.get(blacklist_url).json() + # sanitize the container list + sanitized_cntr_list = [ + cntr for cntr in cntr_list + if cntr[u'acls'][0][u'status'] == 'Approved' + and cntr[u'name'] not in blacklist.get("f{}".format(release_num),[]) + ] + + return sanitized_cntr_list + + +def get_rclist(release_num, cntr_list, dg_api_url, dg_req_params): + # type: (str, list, str, dict) -> list + """ + Query datagrepper to get a list of Release Candidates for a specified + Fedora Release based on request params + + :release_num: str, Fedora Release Number to get RC list for + :cntr_list: list, list of all containers to query for + :dg_api_url: str, datagrepper REST API URL + :dg_req_params: dict, python requests params to use in datagrepper query + """ + + # The list we will return, initialized here so we can append to it + rclist = [] + + # Set the FGC prefix as the "repo" inside the registry as per documented + # layered image naming guidelines: + # https://fedoraproject.org/wiki/Container:Guidelines + fgc = "f{}".format(release_num) + + # Query Datagrepper for latest container build - for cntr in cntr_data: + for cntr in cntr_list: try: # Query datagrepper, extract raw_messages, list comprehension # filter, grab only the latest (index 0) @@ -124,9 +150,40 @@ def get_releasecandidates(release_num): return rclist +def get_releasecandidates(release_num): + # type: (int) -> list + """ + Logic to actually get the release candidate, this is likely to change over + time (PDC?) so we want it split out from the main control of the module + """ + + global module + + # pkgdb2 api vars + pkgdb_api_url = "https://admin.fedoraproject.org/pkgdb/api/packages/*" + pkgdb_req_params = { + "namespace": "docker", + "branches": ["f{}".format(release_num)], + "acls": True + } + cntr_list = get_cntr_list(pkgdb_api_url, pkgdb_req_params, release_num) + + # datagrepper api vars + dg_api_url = "https://apps.fedoraproject.org/datagrepper/raw" + dg_req_params = { + "topic": "org.fedoraproject.prod.buildsys.tag", + "user": "containerbuild", + "rows_per_page": 100 + } + rclist = get_rclist(release_num, cntr_list, dg_api_url, dg_req_params) + + return rclist + def main(): + global module + module = AnsibleModule( argument_spec=dict( release=dict(required=True, type="int", default=None), @@ -134,7 +191,6 @@ def main(): supports_check_mode=False ) - # Technically this never changes anything on the system, so always mark # changed=False module.exit_json(