From 74e6fee5cf91ae44519468b874716679f8285035 Mon Sep 17 00:00:00 2001 From: Adam Miller Date: Oct 31 2017 20:33:17 +0000 Subject: add module to identify layered image inheritance Created a module that will take fedora release and a list of layered images (meant to be provided by the fedcontainer_rc module but can originate from anywhere) and will provide a dictionary where they keys are the parent images and the value is a list of children images. This will assist in automating the layered image rebuild process. dep graph rewrite of fedcontainer_inheritance The original implementation didn't properly determine the order in which parent images and their children need to be rebuilt. This implementation does. Signed-off-by: Adam Miller --- diff --git a/ansible/fedcontainer_inheritance_demo.yml b/ansible/fedcontainer_inheritance_demo.yml new file mode 100644 index 0000000..4ed2224 --- /dev/null +++ b/ansible/fedcontainer_inheritance_demo.yml @@ -0,0 +1,19 @@ +--- +- name: fedcontainer_inheritance demo + hosts: automator + vars: + fedora_release: 26 + tasks: + - name: Gather list of releases + fedcontainer_rc: + release: "{{fedora_release}}" + register: query_data + + - name: Determine parent image inheritance + fedcontainer_inheritance: + release: "{{fedora_release}}" + layered_images: "{{query_data.rebuildlist}}" + register: image_inheritance + + - debug: var=image_inheritance["parent_children"] + - debug: var=image_inheritance["errors"] diff --git a/ansible/library/fedcontainer_inheritance.py b/ansible/library/fedcontainer_inheritance.py new file mode 100755 index 0000000..699b16b --- /dev/null +++ b/ansible/library/fedcontainer_inheritance.py @@ -0,0 +1,317 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# fedcontainer_inheritance.py - Ansible module to gather Fedora container image +# inheritance. This will return a dict. +# +# Copyright (C) 2017 Red Hat, Inc. +# SPDX-License-Identifier: GPL-2.0+ +# +# Authors: +# Adam Miller +# + +DOCUMENTATION = ''' +--- +author: + - "Adam Miller " +module: fedcontainer_inheritance +short_description: Query Container Parent Image Inheritance +description: + - Query Container Parent Image Inheritance + - Returns list entry 'parent_children' in return JSON, which will contain a + list of lists where the first item of the list is the parent image and + the second is the list of child images. + + This is for two reasons: + 1) The list is ordered + 2) Allows for easy iteration in a playbook +options: + release: + description: + - Fedora Release number + required: true + layered_images: + description: + - List of layered images that need their inheritance identified +''' + +EXAMPLES = ''' +# Generally meant to be used with other tasks as input, registry the output var +- fedcontainer_inheritance: + release: 26 + register: image_inheritance + +- debug: var=image_inheritance["parent_children"] + + +# More "full featured" example using the fedcontainer_rc module as input +- name: Gather list of releases + fedcontainer_rc: + release: 26 + register: query_data + +- name: Determine parent image inheritance + fedcontainer_inheritance: + release: 26 + layered_images: "{{query_data.rebuildlist}}" + register: image_inheritance + +- debug: var=image_inheritance["parent_children"] +''' + + +################################################## +# import module snippets +from ansible.module_utils.basic import * + +import re +import copy + +class DepGraph(object): + """ + Simple representation of the dependency graph, this is a graph-as-a-list + representation with helper functions. It is not meant to be a full featured + graph implementation and does not map verticies or edges. + + It should be noted, this is a relatively rudamentary implementation, my + apologies to the academically minded graph theorists. + """ + + def __init__(self): + + # Our graph representation will be similar to this example: + # + # Assume we have the following graph + # key -> value1 + # key -> value2 + # + # It would be represented as the following in self.graph: + # + # self.graph = [ + # [ "key", ["value1", "value2"] ], + # ] + # + # The graph itself it a list of lists, each list in the graph list is + # made up of two items: a string and a list. The string is the "key" + # and the list are it's child items. + self.graph = [] + + def __len__(self): + return len(self.graph) + + def __repr__(self): + return self.graph.__repr__() + + def __iter__(self): + return self.graph.__iter__() + + def __next__(self): + return self.graph.__next__() + + def _get_index(self, fake_key): + """ + Find an index so we can append to the "key" (fake_key) which is the + string at the first item of the subarray. + + If the fake_key is not found, then return -1 + """ + + try: + return self.graph.index( + [inlist for inlist in self.graph + if inlist[0] == fake_key + ][0] + ) + except ValueError: + # The item wasn't found by graph.index() + return -1 + except IndexError: + # self.graph is empty and we got nothing so + return -1 + + def _get_parents(self, key, listgraph=None): + """ + Find if this vertex's "key" has a parent, if so return a list of + parents. Otherwise return an empty list. + + This is used to determine order of sorted graph + """ + + # sublist is the graph list minus the vertex we're looking into + sublist = self._get_sublist(key, listgraph=listgraph) + + # Note: the hacky splits on vertex are for the sake of stripping away + # potential registry URI formatting + return [ + value[0] for value in sublist + if key.split("/")[-1].split(":")[0] in value[1] + ] + + def _get_sublist(self, key, listgraph=None): + """ + Get a copy of the graph list, minus a list who's key matches the + provided "key" + """ + + localgraph = listgraph if listgraph is not None else self.graph + + sublist = copy.deepcopy(localgraph) + try: + sublist.remove([sl for sl in localgraph if sl[0] == key][0]) + except IndexError: + # If we get an IndexError here, there was nothing to remove because + # the key wasn't a vertex in the graph + pass + except ValueError: + # If we get an ValueError here, there was nothing in the listgraph + # passed in (it was an empty list, which is fine ... just return it) + pass + + return sublist + + def _get_top_order(self, listgraph=None): + """ + determine the top order members of the graph, these have no dependencies + themselves and can be built first. They will be used as starting points + to establish order to the dep graph + """ + localgraph = listgraph if listgraph is not None else self.graph + + spare_copy = copy.deepcopy(localgraph) + + # FIXME DEBUG - find_base_image + + # Go through localgraph vertex + for vertex in localgraph: + if vertex[0].split('/')[-1].split(":")[0] in reduce( + lambda x,y: x+y, + [ + vlist[1] for vlist in + self._get_sublist(vertex[0], listgraph=localgraph) + ] + ): + spare_copy.remove(vertex) + + return spare_copy + + def insert(self, parent, child): + """ + insert a parent/child pair to the graph + """ + key_index = self._get_index(parent) + + try: + if key_index >= 0: + # This works because the structure is as such: + # + # self.graph = [ + # [ "key", ["value1", "value2"] ], + # ] + # + self.graph[key_index][1].append(child) + else: + self.graph.append([parent, [child]]) + except IndexError: + return False + + return True + + def keys(self): + """ + return a list of the "keys" + """ + + return [inlist[0] for inlist in self.graph] + + def depsolve(self): + """ + We're going to sort the graph such that we have the parents in the + correct order to be iterated upon for rebuild and release. + """ + + ordered_depgraph = [] + working_graph = copy.deepcopy(self.graph) + + while True: + base_images = self._get_top_order(listgraph=working_graph) + if base_images: + ordered_depgraph = ordered_depgraph + base_images + for graph in ordered_depgraph: + working_graph = self._get_sublist( + graph[0], listgraph=working_graph + ) + else: + break + + return ordered_depgraph + + +def main(): + + depgraph = DepGraph() + errors = [] + query_url = "https://src.fedoraproject.org/container/{}/raw/f{}/f/Dockerfile" + + # Define the module and parameters + module = AnsibleModule( + argument_spec=dict( + release=dict(required=True, type="int", default=None), + layered_images=dict(required=True, type="list", default=None), + ), + supports_check_mode=False + ) + + try: + import requests + except ImportError: + module.fail_json("requests python module not found but is required") + + for container in module.params["layered_images"]: + + # Grab the contents of the Dockerfile + r = requests.get(query_url.format(container, module.params["release"])) + + # list comprehension to filter out the FROM line, it's a little overkill + # and a little hacky, but it is considerably faster than alternatives. + # + try: + # The logic here is that the list comprehension returns a list + # which contains a single item (which we know it will because a + # Dockerfile can only have one FROM line), then we look at the + # string at index 0 (which is the only thing in the list) and split + # it over whitespace and reference the last item, which will be + # the parent image of this layered image's Dockerfile. + parent_image = [ + item for item in r.content.split("\n") + if 'FROM' in item + ][0].split()[-1] + + # Now we will filter out to make sure that the base images are all + # grouped together since they can technically be referenced + # short-hand and long form in the FROM line. + base_img_key = "fedora:{}".format(module.params["release"]) + pattern = re.compile(".*{}$".format(base_img_key)) + if pattern.match(parent_image): + parent_base_img_key = [ + key for key in depgraph.keys() + if pattern.match(key) + ] + if parent_base_img_key: + parent_image = parent_base_img_key[0] + + depgraph.insert(parent_image, container) + + except IndexError: + errors.append(r.content) + + + module.exit_json( + changed=False, + msg="Successfully identified container image inheritance, results in 'parent_children'", + parent_children=depgraph.depsolve(), + errors=errors + ) + +main() +# vim: set expandtab sw=4 sts=4 ts=4