From 13f69500fd7741e806b7619e235ff5f4fa80f525 Mon Sep 17 00:00:00 2001 From: Adam Miller Date: Dec 01 2017 17:47:16 +0000 Subject: Fully automated container build playbooks Custom modules and docs included Signed-off-by: Adam Miller --- diff --git a/ansible/.gitignore b/ansible/.gitignore new file mode 100644 index 0000000..2d50efe --- /dev/null +++ b/ansible/.gitignore @@ -0,0 +1 @@ +*.retry \ No newline at end of file diff --git a/ansible/container_rebuild_and_release.yml b/ansible/container_rebuild_and_release.yml new file mode 100644 index 0000000..6827abf --- /dev/null +++ b/ansible/container_rebuild_and_release.yml @@ -0,0 +1,72 @@ +--- +- name: Layered Image Rebuild and Release + hosts: automator + + # The base_images list is unfortunately including N, N-1 and N-2 purely for + # the scenario in which containers are branched and nobody updates their + # base image. If this isn't included, it leads to trouble filtering out base + # images during rebuilds. + # + vars: + fedora_release: 27 + base_images: + - "registry.fedoraproject.org/fedora:{{fedora_release}}" + - "registry.fedoraproject.org/fedora:{{fedora_release|int - 1}}" + - "registry.fedoraproject.org/fedora:{{fedora_release|int - 2}}" + - "registry.fedoraproject.org/fedora:{{fedora_release}}-modular" + - "registry.fedoraproject.org/fedora:{{fedora_release|int - 1}}-modular" + - "registry.fedoraproject.org/fedora:{{fedora_release|int - 2}}-modular" + rebuild_in_stage: True + fedora_distgit_branch: "f{{fedora_release}}" + koji_tag: "f{{fedora_release}}-container" + + # This service account and it's credentials are provided by Fedora + # Infrastructure Team and are provisioned using Fedora Infrastructure's + # Ansible playbooks. + # + # https://infrastructure.fedoraproject.org/cgit/ansible.git/ + rebuild_fas_user: "releng" + rebuild_ssh_key: "/etc/pki/releng" + + # This is provided by Fedora Infrastructure to facilitate the sudo-less + # pushing of commits to DistGit. This is passed into a call to the + # fedcontainer_rebuild module + # This is used as an environment to the rebuild as: + # GIT_SSH="{{git_ssh_environment}}" + git_ssh_environment: "/usr/local/bin/relengpush" + + # This is a koji profile that is created by Fedora Infrastructure for use by + # the RelEng Team for various compose related tasks. We're re-using it here. + rebuild_koji_profile: "compose_koji" + + tasks: + - name: Set production variables + set_fact: + container_registry: "registry.fedoraproject.org" + compose_host: "compose-x86-01.phx2.fedoraproject.org" + when: rebuild_in_stage != True + + - name: Set stage variables + set_fact: + container_registry: "registry.stg.fedoraproject.org" + compose_host: "composer.stg.phx2.fedoraproject.org" + when: rebuild_in_stage == True + + - name: Gather list of releases + fedcontainer_rc: + release: "{{fedora_release}}" + stage: "{{ rebuild_in_stage }}" + register: query_data + + - name: Determine parent image inheritance + fedcontainer_inheritance: + release: "{{fedora_release}}" + layered_images: "{{query_data.rebuildlist}}" + stage: "{{ rebuild_in_stage }}" + register: image_inheritance + + - include_tasks: "include/rebuild-containers.yml" + with_list: "{{ image_inheritance.parent_children }}" + loop_control: + loop_var: outer_item + diff --git a/ansible/fedcontainer_inheritance_demo.yml b/ansible/fedcontainer_inheritance_demo.yml new file mode 100644 index 0000000..4ed2224 --- /dev/null +++ b/ansible/fedcontainer_inheritance_demo.yml @@ -0,0 +1,19 @@ +--- +- name: fedcontainer_inheritance demo + hosts: automator + vars: + fedora_release: 26 + tasks: + - name: Gather list of releases + fedcontainer_rc: + release: "{{fedora_release}}" + register: query_data + + - name: Determine parent image inheritance + fedcontainer_inheritance: + release: "{{fedora_release}}" + layered_images: "{{query_data.rebuildlist}}" + register: image_inheritance + + - debug: var=image_inheritance["parent_children"] + - debug: var=image_inheritance["errors"] diff --git a/ansible/fedcontainerrcdemo.yml b/ansible/fedcontainerrcdemo.yml index 0d71f03..f4e0ff3 100644 --- a/ansible/fedcontainerrcdemo.yml +++ b/ansible/fedcontainerrcdemo.yml @@ -4,7 +4,7 @@ tasks: - name: Gather list of releases fedcontainer_rc: - release: 25 + release: 26 register: query_data - debug: var=query_data.rclist diff --git a/ansible/include/nested-rebuild-containers.yml b/ansible/include/nested-rebuild-containers.yml new file mode 100644 index 0000000..bda0549 --- /dev/null +++ b/ansible/include/nested-rebuild-containers.yml @@ -0,0 +1,31 @@ +# This is meant to be included in a playbook to be used along with the +# include/rebuild-containers.yml taskset using Loops and Includes: +# http://docs.ansible.com/ansible/playbooks_loops.html#loops-and-includes-in-2-0 +# +# The intent here is that we are iterating over what is effectively a nested +# loop and therefore we will handle the actual logic here. +# +# This should be called similar to: +# +# - include_tasks: "include/nested-rebuild-containers.yml" +# with_list: "{{ outer_item }}" +# loop_control: +# loop_var: nested_item +# +# +# The loop_var above is important, we make an assumption here that outer_item +# exists. + +# PARENT IMAGE CASE +# If the 'nested_item' is a string, it's a parent image +- include_tasks: "include/rebuild-parent-container-images.yml" + when: nested_item is string + +# CHILD IMAGE CASE +# If the 'nested_item' is a list (and therefore iterable), it's a list of +# child images +- include_tasks: "include/rebuild-child-container-images.yml" + when: nested_item is iterable and nested_item is not string + + + diff --git a/ansible/include/rebuild-child-container-images.yml b/ansible/include/rebuild-child-container-images.yml new file mode 100644 index 0000000..dfa87a0 --- /dev/null +++ b/ansible/include/rebuild-child-container-images.yml @@ -0,0 +1,42 @@ +# This include taskset is meant to perform the rebuild and release process +# specific to child container layered images +# +# This is meant to be included in a playbook to be used along with the +# include/rebuild-containers.yml taskset using Loops and Includes: +# http://docs.ansible.com/ansible/playbooks_loops.html#loops-and-includes-in-2-0 +# +# We assume the following variables exist in scope: +# - nested_item (list) +# - base_images (list) +# - rebuild_fas_user (str) +# - rebuild_in_stage (bool) +# - koji_tag (str) +# - fedora_distgit_branch (str) + +- name: Rebuild the Child images + fedcontainer_rebuild: + branch: "{{fedora_distgit_branch}}" + containers: '{{nested_item|join(" ")}}' + fas_user: "{{rebuild_fas_user}}" + stage: "{{rebuild_in_stage}}" + koji_bin: "/usr/bin/compose-koji" + environment: + GIT_SSH: "{{ git_ssh_environment }}" + register: rebuild_info + +- debug: var=rebuild_info + +- name: Get latest Child image build N-V-Rs from the rebuilds + koji_latestbuild: + tag: "{{koji_tag}}" + package: "{{nested_item}}" + stage: "{{rebuild_in_stage}}" + register: query_output + +- debug: var=query_output.latest_builds + +- name: Include sync task set for container images + include_tasks: "include/sync-container-images.yml" + with_items: "{{ query_output.latest_builds }}" + loop_control: + loop_var: latest_build diff --git a/ansible/include/rebuild-containers.yml b/ansible/include/rebuild-containers.yml new file mode 100644 index 0000000..1025c90 --- /dev/null +++ b/ansible/include/rebuild-containers.yml @@ -0,0 +1,24 @@ +# This is meant to be included in a playbook to be used along with the +# fedcontainers_release module and it's data be passed into this include +# taskset using Loops and Includes: +# http://docs.ansible.com/ansible/playbooks_loops.html#loops-and-includes-in-2-0 +# +# This should be called similar to: +# +# - include_tasks: "include/rebuild-containers.yml" +# with_list: "{{ image_inheritance['parent_children'] }}" +# loop_control: +# loop_var: outer_item +# +# +# The loop_var above is important, we make an assumption here that outer_item +# exists. + +- include_tasks: "include/nested-rebuild-containers.yml" + with_list: "{{outer_item}}" + loop_control: + loop_var: nested_item + + + + diff --git a/ansible/include/rebuild-parent-container-images.yml b/ansible/include/rebuild-parent-container-images.yml new file mode 100644 index 0000000..5ef7139 --- /dev/null +++ b/ansible/include/rebuild-parent-container-images.yml @@ -0,0 +1,37 @@ +# This include taskset is meant to perform the rebuild and release process +# specific to parent container layered images +# +# This is meant to be included in a playbook to be used along with the +# include/rebuild-containers.yml taskset using Loops and Includes: +# http://docs.ansible.com/ansible/playbooks_loops.html#loops-and-includes-in-2-0 +# +# We assume the following variables exist in scope: +# - nested_item (string) +# - fedora_release (int) +# - base_images (list) +# - rebuild_fas_user (str) +# - rebuild_in_stage (bool) +# - koji_tag (str) + +# Base images come from koji and we will not be rebuilding them here. + +- name: Block to handle non-base parent images + block: + + # NOTE: Right now this is just an information gathering task that's more or + # less a place holder in case the non-base parent images ever need + # some special consideration for rebuild or release as this is the + # construct we'd want to use in the playbook "framework" as it exists + # today + + - name: Get latest Parent image build N-V-Rs from the rebuilds + koji_latestbuild: + tag: "{{koji_tag}}" + package: '{{nested_item.split("/")[-1].split(":")[0]}}' + kojihub_url: "https://koji.stg.fedoraproject.org/kojihub" + register: query_output + + - debug: var=query_output["latest_builds"] + + when: nested_item not in base_images + diff --git a/ansible/include/sync-container-images.yml b/ansible/include/sync-container-images.yml new file mode 100644 index 0000000..2a3cfac --- /dev/null +++ b/ansible/include/sync-container-images.yml @@ -0,0 +1,51 @@ +# This include taskset is meant to perform the sync operation of container +# images +# +# This is meant to be included in a playbook to be used along with the +# include/rebuild-child-container-images.yml and +# include/rebuild-parent-container-images.yml taskset using Loops and Includes: +# http://docs.ansible.com/ansible/playbooks_loops.html#loops-and-includes-in-2-0 +# +# We assume the following variables exist in scope: +# - container_registry (str) +# - latest_build (dict) +# - an example of this dictionary's structure is as follows: +# +# { +# "build_id": 999101, +# "completion_time": "2017-11-14 20:43:52", +# "creation_event_id": 28732043, +# "creation_time": "2017-11-14 20:44:12.168314", +# "epoch": null, +# "id": 999101, +# "name": "etcd", +# "nvr": "etcd-0-9.f26container", +# "owner_id": 3538, +# "owner_name": "containerbuild", +# "package_id": 17028, +# "package_name": "etcd", +# "release": "9.f26container", +# "start_time": "2017-11-14 20:17:29", +# "state": 1, +# "tag_id": 741, +# "tag_name": "f26-container", +# "task_id": null, +# "version": "0", +# "volume_id": 0, +# "volume_name": "DEFAULT" +# } + +- name: Sync image to registry + block: + - shell: 'skopeo copy --src-cert-dir /etc/docker/certs.d/candidate-{{container_registry}}/ --dest-cert-dir /etc/docker/certs.d/{{container_registry}}/ docker://candidate-{{container_registry}}/{{fedora_distgit_branch}}/{{latest_build.name}} docker://{{container_registry}}/{{fedora_distgit_branch}}/{{latest_build.name}}' + + - shell: 'skopeo copy --src-cert-dir /etc/docker/certs.d/candidate-{{container_registry}}/ --dest-cert-dir /etc/docker/certs.d/{{container_registry}}/ docker://candidate-{{container_registry}}/{{fedora_distgit_branch}}/{{latest_build.name}}:{{latest_build.version}} docker://{{container_registry}}/{{fedora_distgit_branch}}/{{latest_build.name}}:{{latest_build.version}}' + + - shell: 'skopeo copy --src-cert-dir /etc/docker/certs.d/candidate-{{container_registry}}/ --dest-cert-dir /etc/docker/certs.d/{{container_registry}}/ docker://candidate-{{container_registry}}/{{fedora_distgit_branch}}/{{latest_build.name}}:{{latest_build.version}}-{{latest_build.release}} docker://{{container_registry}}/{{fedora_distgit_branch}}/{{latest_build.name}}:{{latest_build.version}}-{{latest_build.release}}' + + when: fedora_distgit_branch in latest_build.nvr + +- debug: msg="{{latest_build.nvr}} not synchronized to the registry, FGC did not match" + when: fedora_distgit_branch not in latest_build.nvr + + diff --git a/ansible/inventory/group_vars/all b/ansible/inventory/group_vars/all new file mode 100644 index 0000000..e3a6e68 --- /dev/null +++ b/ansible/inventory/group_vars/all @@ -0,0 +1,17 @@ +################################################################################ +# +# +# This file contains what is effectively "global" variables for all playbooks +# contained within this repository. +# +# +# Guidelines: +# +# - If you are going to define a variable for both the Fedora Infrastructure +# staging environment and the production environment they should contain +# the following postfixes for uniformity, respectively: +# +# foo_prod +# foo_stg +# +# diff --git a/ansible/library/fedcontainer_inheritance.py b/ansible/library/fedcontainer_inheritance.py new file mode 100755 index 0000000..db53246 --- /dev/null +++ b/ansible/library/fedcontainer_inheritance.py @@ -0,0 +1,347 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# fedcontainer_inheritance.py - Ansible module to gather Fedora container image +# inheritance. This will return a dict. +# +# Copyright (C) 2017 Red Hat, Inc. +# SPDX-License-Identifier: GPL-2.0+ +# +# Authors: +# Adam Miller +# + +DOCUMENTATION = ''' +--- +author: + - "Adam Miller " +module: fedcontainer_inheritance +short_description: Query Container Parent Image Inheritance +description: + - Query Container Parent Image Inheritance + - Returns list entry 'parent_children' in return JSON, which will contain a + list of lists where the first item of the list is the parent image and + the second is the list of child images. + + This is for two reasons: + 1) The list is ordered + 2) Allows for easy iteration in a playbook +options: + release: + description: + - Fedora Release number + required: true + layered_images: + description: + - List of layered images that need their inheritance identified + required: true + stage: + description: + - Boolean: If set to True, query against Fedora Stage Infrastructure. + - Default: False + required: false + registry: + description: + - The fqdn of the authoritative container registry + - Default: registry.fedoraproject.org + required: false +''' + +EXAMPLES = ''' +# Generally meant to be used with other tasks as input, registry the output var +- fedcontainer_inheritance: + release: 26 + register: image_inheritance + +- debug: var=image_inheritance["parent_children"] + + +# More "full featured" example using the fedcontainer_rc module as input +- name: Gather list of releases + fedcontainer_rc: + release: 26 + register: query_data + +- name: Determine parent image inheritance + fedcontainer_inheritance: + release: 26 + layered_images: "{{query_data.rebuildlist}}" + register: image_inheritance + +- debug: var=image_inheritance["parent_children"] +''' + + +################################################## +# import module snippets +from ansible.module_utils.basic import * + +import re +import copy + +class DepGraph(object): + """ + Simple representation of the dependency graph, this is a graph-as-a-list + representation with helper functions. It is not meant to be a full featured + graph implementation and does not map verticies or edges. + + It should be noted, this is a relatively rudamentary implementation, my + apologies to the academically minded graph theorists. + """ + + def __init__(self): + + # Our graph representation will be similar to this example: + # + # Assume we have the following graph + # key -> value1 + # key -> value2 + # + # It would be represented as the following in self.graph: + # + # self.graph = [ + # [ "key", ["value1", "value2"] ], + # ] + # + # The graph itself it a list of lists, each list in the graph list is + # made up of two items: a string and a list. The string is the "key" + # and the list are it's child items. + self.graph = [] + + def __len__(self): + return len(self.graph) + + def __repr__(self): + return self.graph.__repr__() + + def __iter__(self): + return self.graph.__iter__() + + def __next__(self): + return self.graph.__next__() + + def _get_index(self, fake_key): + """ + Find an index so we can append to the "key" (fake_key) which is the + string at the first item of the subarray. + + If the fake_key is not found, then return -1 + """ + + try: + return self.graph.index( + [inlist for inlist in self.graph + if inlist[0] == fake_key + ][0] + ) + except ValueError: + # The item wasn't found by graph.index() + return -1 + except IndexError: + # self.graph is empty and we got nothing so + return -1 + + def _get_parents(self, key, listgraph=None): + """ + Find if this vertex's "key" has a parent, if so return a list of + parents. Otherwise return an empty list. + + This is used to determine order of sorted graph + """ + + # sublist is the graph list minus the vertex we're looking into + sublist = self._get_sublist(key, listgraph=listgraph) + + # Note: the hacky splits on vertex are for the sake of stripping away + # potential registry URI formatting + return [ + value[0] for value in sublist + if key.split("/")[-1].split(":")[0] in value[1] + ] + + def _get_sublist(self, key, listgraph=None): + """ + Get a copy of the graph list, minus a list who's key matches the + provided "key" + """ + + localgraph = listgraph if listgraph is not None else self.graph + + sublist = copy.deepcopy(localgraph) + try: + sublist.remove([sl for sl in localgraph if sl[0] == key][0]) + except IndexError: + # If we get an IndexError here, there was nothing to remove because + # the key wasn't a vertex in the graph + pass + except ValueError: + # If we get an ValueError here, there was nothing in the listgraph + # passed in (it was an empty list, which is fine ... just return it) + pass + + return sublist + + def _get_top_order(self, listgraph=None): + """ + determine the top order members of the graph, these have no dependencies + themselves and can be built first. They will be used as starting points + to establish order to the dep graph + """ + localgraph = listgraph if listgraph is not None else self.graph + + spare_copy = copy.deepcopy(localgraph) + + # Go through localgraph vertex + for vertex in localgraph: + sublist = [ + vlist[1] for vlist in + self._get_sublist(vertex[0], listgraph=localgraph) + ] + + if sublist: + if vertex[0].split('/')[-1].split(":")[0] in reduce( + lambda x, y: x+y, sublist): + spare_copy.remove(vertex) + + return spare_copy + + def insert(self, parent, child): + """ + insert a parent/child pair to the graph + """ + key_index = self._get_index(parent) + + try: + if key_index >= 0: + # This works because the structure is as such: + # + # self.graph = [ + # [ "key", ["value1", "value2"] ], + # ] + # + self.graph[key_index][1].append(child) + else: + self.graph.append([parent, [child]]) + except IndexError: + return False + + return True + + def keys(self): + """ + return a list of the "keys" + """ + + return [inlist[0] for inlist in self.graph] + + def depsolve(self): + """ + We're going to sort the graph such that we have the parents in the + correct order to be iterated upon for rebuild and release. + """ + + ordered_depgraph = [] + working_graph = copy.deepcopy(self.graph) + + while True: + base_images = self._get_top_order(listgraph=working_graph) + if base_images: + ordered_depgraph = ordered_depgraph + base_images + for graph in ordered_depgraph: + working_graph = self._get_sublist( + graph[0], listgraph=working_graph + ) + else: + break + + return ordered_depgraph + + +def main(): + + depgraph = DepGraph() + errors = [] + + # Define the module and parameters + module = AnsibleModule( + argument_spec=dict( + release=dict(required=True, type="int", default=None), + layered_images=dict(required=True, type="list", default=None), + stage=dict(required=False, type="bool", default=False), + registry=dict(required=False, default="registry.fedoraproject.org") + ), + supports_check_mode=False + ) + + if module.params["stage"]: + query_url = "https://src.stg.fedoraproject.org/container/{}/raw/f{}/f/Dockerfile" + else: + query_url = "https://src.fedoraproject.org/container/{}/raw/f{}/f/Dockerfile" + + try: + import requests + except ImportError: + module.fail_json(msg="requests python module not found but is required") + + for container in module.params["layered_images"]: + + # Grab the contents of the Dockerfile + dockerfile_url = query_url.format(container, module.params["release"]) + r = requests.get(dockerfile_url) + + if r.status_code == 404: + errors.append("Dockerfile not found at {}".format(dockerfile_url)) + elif r.status_code > 400: + module.fail_json(msg="Failed to query {}".format(dockerfile_url)) + else: + # list comprehension to filter out the FROM line, it's a little + # overkill and a little hacky, but it is considerably faster than + # alternatives. + # + try: + # The logic here is that the list comprehension returns a list + # which contains a single item (which we know it will because a + # Dockerfile can only have one FROM line), then we look at the + # string at index 0 (which is the only thing in the list) and + # split it over whitespace and reference the last item, which + # will be the parent image of this layered image's Dockerfile. + parent_image = [ + item for item in r.content.split("\n") + if 'FROM' in item + ][0].split()[-1] + + # Make sure we have the fully qualified name + if module.params["registry"] not in parent_image: + parent_image = "{}/{}".format( + module.params["registry"], + parent_image + ) + + # Now we will filter out to make sure that the base images are all + # grouped together since they can technically be referenced + # short-hand and long form in the FROM line. + base_img_key = "fedora:{}".format(module.params["release"]) + pattern = re.compile(".*{}$".format(base_img_key)) + if pattern.match(parent_image): + parent_base_img_key = [ + key for key in depgraph.keys() + if pattern.match(key) + ] + if parent_base_img_key: + parent_image = parent_base_img_key[0] + + depgraph.insert(parent_image, container) + + except IndexError: + errors.append(r.content) + + + module.exit_json( + changed=False, + msg="Successfully identified container image inheritance, results in 'parent_children'", + parent_children=depgraph.depsolve(), + errors=errors + ) + +main() +# vim: set expandtab sw=4 sts=4 ts=4 diff --git a/ansible/library/fedcontainer_rc.py b/ansible/library/fedcontainer_rc.py index b237866..f744e7c 100755 --- a/ansible/library/fedcontainer_rc.py +++ b/ansible/library/fedcontainer_rc.py @@ -35,7 +35,12 @@ options: choices: - pkgdb - pdc - default: pkgdb + - src + default: src + stage: + description: + - Boolean: When set to True, Fedora's Stage Infra is queried. + default: False ''' @@ -60,7 +65,8 @@ from ansible.module_utils.basic import * module = AnsibleModule( argument_spec=dict( release=dict(required=True, type="int", default=None), - query_source=dict(default="pkgdb", choices=["pkgdb", "pdc"]) + query_source=dict(default="src", choices=["src", "pdc", "pkgdb"]), + stage=dict(required=False, type="bool", default=False) ), supports_check_mode=False ) @@ -68,7 +74,7 @@ module = AnsibleModule( try: import koji except ImportError: - module.fail_json("koji python module not found on target system") + module.fail_json(msg="koji python module not found on target system") # Set default retried for python request queries # (sometimes datagrepper queries time out) @@ -82,7 +88,7 @@ requests.adapters.DEFAULT_RETRIES = 5 # dg == datagrepper # fgc == Fedora Generational Core -def get_cntr_list(pkgdb_url, release_num): +def get_cntr_list_pkgdb(pkgdb_url, release_num): # type: (str, int) -> list """ Query PackageDB (pkgdb) for container list, sanitize out containers that @@ -176,6 +182,64 @@ def get_cntr_list_pdc(pdc_url, release_num): return sanitized_cntr_list +def get_cntr_list_src(src_url, release_num): + # type: (str, int) -> list + """ + Query src.fedoraproject.org for container list, sanitize out containers + that are inactive or are blacklisted. + + :arg src_url: str, src URL + :arg release_num: int, Fedora release to query for + :return: list of container names + """ + + global module + + # Get blacklist from https://pagure.io/releng/container-blacklist + blacklist_url = ('https://pagure.io/releng/container-blacklist/raw/master' + '/f/blacklist.json') + + # First query src for all container images in DistGit we currently have + src_all_cntrs_url = \ + "{}/api/0/projects?namespace=container&short=1&fork=0".format(src_url) + + branch_name = "f{}".format(release_num) + + try: + projects = requests.get( + src_all_cntrs_url, + timeout=60 + ).json()["projects"] + except Exception as e: + module.fail_json(msg="Failed to query {}\n{}".format(src_url, e)) + + try: + cntr_list = [ + project["name"] for project in projects + if branch_name in requests.get( + "{}/api/0/container/{}/git/branches".format( + src_url, + project["name"] + ) + ).json()["branches"] + + ] + except Exception as e: + module.fail_json(msg="Failed to compile container list\n {}".format(e)) + + try: + blacklist = requests.get(blacklist_url, timeout=60).json() + except Exception as e: + module.fail_json(msg="Failed to query {}\n{}".format(blacklist_url, e)) + + # sanitize the container list + sanitized_cntr_list = [ + cntr for cntr in cntr_list + if cntr not in blacklist.get('f{}'.format(release_num), []) + ] + + return sanitized_cntr_list + def get_lists(release_num, cntr_list): # type: (int, list) -> tuple @@ -256,7 +320,7 @@ def get_lists(release_num, cntr_list): return (rclist, rebuildlist, errors) -def get_releasecandidates(release_num, query_source): +def get_releasecandidates(release_num, query_source, stage=False): # type: (int) -> tuple """ Logic to actually get the release candidate, this is likely to change over @@ -265,12 +329,41 @@ def get_releasecandidates(release_num, query_source): global module - if query_source == 'pkgdb': - pkgdb_url = "https://admin.fedoraproject.org/pkgdb" - cntr_list = get_cntr_list(pkgdb_url, release_num) + if query_source == "src": + if not stage: + cntr_list = get_cntr_list_src( + "https://src.fedoraproject.org/", + release_num + ) + else: + cntr_list = get_cntr_list_src( + "https://src.stg.fedoraproject.org/", + release_num + ) + elif query_source == "pdb": + if not stage: + cntr_list = get_cntr_list_pdc( + "https://pdc.fedoraproject.org", + release_num + ) + else: + cntr_list = get_cntr_list_pdc( + "https://pdc.stg.fedoraproject.org", + release_num + ) + elif query_source == 'pkgdb': + if not stage: + cntr_list = get_cntr_list_pkgdb( + "https://admin.fedoraproject.org/pkgdb", + release_num + ) + else: + cntr_list = get_cntr_list_pkgdb( + "https://admin.stg.fedoraproject.org/pkgdb", + release_num + ) else: - pdc_api_url = "https://pdc.fedoraproject.org" - cntr_list = get_cntr_list_pdc(pdc_api_url, release_num) + module.fail_json(msg="invalid option for query_source provided") rc_data_tuple = get_lists(release_num, cntr_list) @@ -282,7 +375,10 @@ def main(): global module rclist, rebuildlist, errors = get_releasecandidates( - module.params['release'], module.params['query_source']) + module.params['release'], + module.params['query_source'], + stage=module.params['stage'] + ) if rclist and rebuildlist: # Technically this never changes anything on the system, so always mark diff --git a/ansible/library/fedcontainer_rebuild.py b/ansible/library/fedcontainer_rebuild.py new file mode 100644 index 0000000..927eb0f --- /dev/null +++ b/ansible/library/fedcontainer_rebuild.py @@ -0,0 +1,303 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# fedcontainer_rebuild.py - Ansible module to rebuild a list of containers. +# +# Copyright (C) 2017 Red Hat, Inc. +# SPDX-License-Identifier: GPL-2.0+ +# +# Authors: +# Adam Miller +# + +DOCUMENTATION = ''' +--- +author: + - "Adam Miller " +module: fedcontainer_rebuild +short_description: Rebuild a list of containers +description: + - Perform rebuilds on a set of container images + +options: + branch: + description: + - DistGit branch + containers: + description: + - Space delimited string of containers to rebuild + user: + description: + - FAS user to perform the builds as + stage: + description: + - Boolean value, True == use stage, False = use prod + default: False + koji_bin: + description: + - Optional path to the koji binary + default: None + koji_profile: + description: + - Optional name of koji profile to use + default: None + +''' + +EXAMPLES = ''' +# Perform a rebuild in production koji +- fedcontainer_rebuild: + branch: f27 + containers: "etcd kubernetes-master kubernetes-node" + user: relenguser + +# Perform a rebuild in stage koji +- fedcontainer_rebuild: + branch: f27 + containers: "etcd kubernetes-master kubernetes-node" + user: relenguser + stage: True +''' + +import os +import shutil +import tempfile + +from ansible.module_utils.basic import AnsibleModule + +# Base koji command +KOJI_CMD = "/usr/bin/koji" +STG_KOJI_CMD = "/usr/bin/stg-koji" +FEDPKG_CMD = "/usr/bin/fedpkg" +STG_FEDPKG_CMD = "/usr/bin/fedpkg-stage" + +# Make the module global +module = None + + +def run_cmd_ary(cmd): + + global module + + returncode, stdout, stderr = module.run_command(cmd) + + if returncode: + module.warn("Failed to run command: {}".format(" ".join(cmd))) + + return (returncode, stdout, stderr) + + +def rebuild(branch, containers, user, stage, koji_bin=None, koji_profile=None): + """ + rebuild containers + + :param branch: str, DistGit branch + :param containers: list, artifact names (container names) + :param user: str, FAS username + :param stage: bool, True == use stage, False = use prod + :param koji_bin: str, optional named arg, path to koji binary + :param koji_profile: str, optional named arg, koji profile name + + """ + global module + try: + from dockerfile_parse import DockerfileParser + except ImportError: + module.fail_json( + msg="Python dockerfile-parse not found and is required" + ) + + # Set username if one is provided + if user: + user_opt = ("--user", user) + else: + user_opt = ("", "") + + # Set fedpkg command based on stage or not + if stage: + git_url = "src.stg.fedoraproject.org" + fedpkg_prefix = [STG_FEDPKG_CMD, user_opt[0], user_opt[1]] + if koji_bin: + koji_prefix = [koji_bin, user_opt[0], user_opt[1]] + else: + koji_prefix = [STG_KOJI_CMD, user_opt[0], user_opt[1]] + else: + git_url = "src.fedoraproject.org" + fedpkg_prefix = [FEDPKG_CMD, user_opt[0], user_opt[1]] + if koji_bin: + koji_prefix = [koji_bin, user_opt[0], user_opt[1]] + else: + koji_prefix = [KOJI_CMD, user_opt[0], user_opt[1]] + + if koji_profile: + koji_prefix + ['--profile', koji_profile] + + # Currently the container namespace in DistGit is called "container" + cntr_ns = 'container' + + work_dir = tempfile.mkdtemp() + + # List of koji tasks to watch at the end + koji_tasks = [] + + # List of containers actually sent to koji for rebuild + containers_rebuilt = [] + + # Steps to build: + # + # - Clone the DistGit Repo + # - Increment the Release + # - Write the Dockerfile out + # - fepkg commit + # - Open the Dockerfile + # + + # Change directories to where we can do some work + os.chdir(work_dir) + + for container in containers: + + # Clone the DistGit repo + cmd = fedpkg_prefix + ['clone', '{}/{}'.format(cntr_ns, container)] + # Execute the koji command + returncode, stdout, stderr = run_cmd_ary(cmd) + if returncode: + # FIXME - Need to handle this somehow, but for now at least WARN + module.warn("Failed to fedmsg clone {}/{}".format(cntr_ns, container)) + break + + os.chdir(os.path.join(work_dir, container)) + + # Switch branch to requested DistGit branch to operate on + cmd = fedpkg_prefix + ['switch-branch', branch] + run_cmd_ary(cmd) + + # Load the Dockerfile content + dfp = DockerfileParser() + + try: + with open('Dockerfile', 'r') as dkr_file: + dfp.content = ''.join(dkr_file.readlines()) + except IOError: + # FIXME - No Dockerfile was found, we probably want to handle this + # properly somehow. + continue + + # Need to support both old and new style + if 'RELEASE' in dfp.envs: + release_str = 'RELEASE' + elif 'release' in dfp.envs: + release_str = 'release' + else: + release_str = None + + if release_str: + dfp_release = dfp.envs[release_str] + # This is a weird hack, but it works and takes into account + # that we don't want 0.9 + 0.1 to "wrap" to 1.0 + dfp_relsplit = dfp_release.split('.') + dfp_newrel = dfp_relsplit[:-1] + dfp_newrel.append(str(int(dfp_relsplit[-1]) + 1)) + dfp.envs[release_str] = u'{}'.format('.'.join(dfp_newrel)) + + # commit changes + cmd = fedpkg_prefix + [ + 'commit', '-m', '\"Bump RELEASE for automatic rebuild\"' + ] + returncode, stdout, stderr = run_cmd_ary(cmd) + + # push changes + cmd = fedpkg_prefix + ['push'] + run_cmd_ary(cmd) + + # Get the commit hash + cmd = ["git", "rev-parse", "HEAD"] + returncode, stdout, stderr = run_cmd_ary(cmd) + commit_hash = stdout + + # Build the image by calling 'call buildContainer'. We have to do + # this instead of with fedpkg because we need to handle the case + # of an alternative koji binary as well as a koji profile, neither + # are supported in fedpkg + cmd = koji_prefix + [ + "call", + "buildContainer", + "git+https://{}/{}/{}.git?#{}".format(git_url, cntr_ns, container, commit_hash).strip(), + branch, + ] + returncode, stdout, stderr = run_cmd_ary(cmd) + + if not returncode: + # Parse output for task id so we can wait for them later + if len(stdout.split()) == 1: + koji_tasks.append(stdout) + containers_rebuilt.append(container) + else: + # FIXME - Need to handle this better, but at least warn for now + module.warn( + "FAILED TO BUMP RELEASE - SKIPPING: {}/{}".format( + cntr_ns, + container + ) + ) + continue + + # Clean up after ourselves + os.chdir(work_dir) + shutil.rmtree(os.path.join(work_dir, container)) + + # Wait for the koji tasks + if koji_tasks: + cmd = koji_prefix + ['watch-task'] + cmd.extend(koji_tasks) + returncode, stdout, stderr = run_cmd_ary(cmd) + module.exit_json( + msg="Rebuild koji tasks complete: {}".format(" ".join(koji_tasks)), + koji_tasks=koji_tasks, + koji_watch_task_stdout=stdout, + koji_watch_task_stderr=stderr, + rebuild_list=containers_rebuilt + ) + else: + module.fail_json( + msg="No containers rebuilt" + ) + + +def main(): + + global module + + module = AnsibleModule( + argument_spec=dict( + containers=dict(required=True, default=None), + fas_user=dict(required=True, default=None), + branch=dict(required=True, default=None), + stage=dict(required=False, type="bool", default=False), + koji_bin=dict(required=False, default=None), + koji_profile=dict(required=False, default=None), + + ), + supports_check_mode=False + ) + + # Make sure koji exists on the machine trying to execute this module + if not os.path.isfile(KOJI_CMD): + module.fail_json( + msg="Unable to find {} on target host".format(KOJI_CMD) + ) + + rebuild( + module.params["branch"], + module.params["containers"].split(), + module.params["fas_user"], + module.params["stage"], + koji_bin=module.params["koji_bin"], + koji_profile=module.params["koji_profile"] + ) + + +if __name__ == '__main__': + main() + +# vim: tabstop=8 expandtab shiftwidth=4 softtabstop=4 diff --git a/ansible/library/koji_latestbuild.py b/ansible/library/koji_latestbuild.py new file mode 100755 index 0000000..71eac9c --- /dev/null +++ b/ansible/library/koji_latestbuild.py @@ -0,0 +1,165 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# koji_latestbuild.py - Ansible module to query koji for the latest build +# +# Copyright (C) 2017 Red Hat, Inc. +# SPDX-License-Identifier: GPL-2.0+ +# +# Authors: +# Adam Miller +# + +DOCUMENTATION = ''' +--- +author: + - "Adam Miller " +module: koji_latestbuild +short_description: Query koji to determine latest build +description: + - Query koji to determine latest build of a specific tag and/or package + - Returns dict entry 'builds' in return JSON + - latest_builds - list containing the build(s) requested +options: + tag: + description: + - Koji tag + required: True + package: + description + - Package to query for or a list of packages to query for + required: False + arch: + description: + - Architecture to query for + required: False + type: + description: + - Type of build + required: False + all_builds: + description: + - If set to True, the module will return a list of all latest builds in + the supplied tag. + - Assumed True if `tag` is the only provided argument + required: False + default: False + stage: + description: + - To perform query against stage or not. This is mutually exclusive with + kojihub_url as it will assume "https://koji.stg.fedoraproject.org/kojihub" + required: False + default: False + kojihub_url: + description: + - Koji Hub URL to query against + required: False + default: "https://koji.fedoraproject.org/kojihub" + +''' + +EXAMPLES = ''' +# Generally meant to be used with other tasks as input as follows: +- koji_latestbuild: + tag: f27-container + package: etcd + register: query_data + +- koji_latestbuild: + tag: f27-container + package: + - etcd + - cockpit + register: query_data + +- debug: var=query_data["latest_builds"] + + +''' + + +def main(): + + from ansible.module_utils.basic import * + + # Make the module global + module = AnsibleModule( + argument_spec=dict( + tag=dict(required=True, default=None), + package=dict(required=False, type="list", default=None), + arch=dict(required=False, default=None), + build_type=dict(required=False, default=None), + all_builds=dict(required=False, type="bool", default=False), + stage=dict(required=False, type="bool", default=False), + kojihub_url=dict( + required=False, + default="https://koji.fedoraproject.org/kojihub" + ) + ), + supports_check_mode=False, + mutually_exclusive=( + ["package", "all"], + ["stage", "kojihub_url"], + ), + ) + + try: + import koji + except ImportError: + module.fail_json(msg="koji python module not found on target system") + + if module.params["stage"]: + kc = koji.ClientSession("https://koji.stg.fedoraproject.org/kojihub") + else: + kc = koji.ClientSession(module.params["kojihub_url"]) + + pkgs_without_latest_build = [] + + + # The getLatestRPMS and getLatestBuilds Koji Hub API calls both use the + # same internal API call on the backend but the getLatestRPMS has more + # options available to it so we'll use that for everything. + if module.params["all_builds"]: + build_list = kc.getLatestRPMS(module.params["tag"]) + else: + build_list = [] + for pkg in module.params["package"]: + latest_pkg_build = kc.getLatestRPMS( + module.params["tag"], + package=pkg, + arch=module.params["arch"], + type=module.params["build_type"] + ) + filtered_build = [ + build for build in latest_pkg_build if build != [] + ] + if filtered_build: + # Because the koji API doesn't seem to return a consistent + # ADT structure we have to parse a little to provide a + # consistent list back to the user + if len(module.params["package"]) > 1: + build_list.append(filtered_build[0][0]) + else: + build_list.append(filtered_build[0]) + else: + pkgs_without_latest_build.append(pkg) + + # For whatever reason, the koji API will return a list of lists and some of + # the list entries are emtpy. Filter them out here and then only take the + # first one because after we filter out the empty lists, there's only one + # list left which is a list of dict types + filtered_builds = [build for build in build_list if build != []] + return_builds = filtered_builds[0] if len(filtered_builds) == 1 else filtered_builds + + if pkgs_without_latest_build: + module.fail_json( + msg="Latest Build not found for packages: {}".format( + ", ".join(pkgs_without_latest_build) + ) + ) + else: + module.exit_json(latest_builds=return_builds) + + +main() +# vim: set expandtab sw=4 sts=4 ts=4 diff --git a/ansible/roles/releng-shell/README.md b/ansible/roles/releng-shell/README.md index 1e13d46..94f1c48 100644 --- a/ansible/roles/releng-shell/README.md +++ b/ansible/roles/releng-shell/README.md @@ -6,7 +6,7 @@ This is a simple wrapper around the shell ansible module for error handling. Role Variables -------------- -`shell-cmd` - The shell command to pass to ansible's shell module +`shell_cmd` - The shell command to pass to ansible's shell module Example Playbook ---------------- @@ -17,7 +17,7 @@ Including an example of how to use your role (for instance, with variables passe roles: - { role: releng-shell, - shell-cmd: "echo true" + shell_cmd: "echo true" } diff --git a/ansible/test_koji_latestbuild.yml b/ansible/test_koji_latestbuild.yml new file mode 100644 index 0000000..4ebaa87 --- /dev/null +++ b/ansible/test_koji_latestbuild.yml @@ -0,0 +1,26 @@ +--- +- name: koji_latestbuild + hosts: automator + tasks: + - name: Get latest build + koji_latestbuild: + stage: True + tag: "f27-container" + package: + - etcd + - cockpit + register: query_data + + - debug: var=query_data["latest_builds"] + +- name: koji_latestbuild + hosts: automator + tasks: + - name: Get latest build + koji_latestbuild: + stage: True + tag: "f27-container" + package: "cockpit" + register: query_data + + - debug: var=query_data["latest_builds"] diff --git a/docs/Makefile b/docs/Makefile new file mode 100644 index 0000000..ca4940d --- /dev/null +++ b/docs/Makefile @@ -0,0 +1,20 @@ +# Minimal makefile for Sphinx documentation +# + +# You can set these variables from the command line. +SPHINXOPTS = +SPHINXBUILD = sphinx-build +SPHINXPROJ = FedoraReleaseEngineeringAutomation +SOURCEDIR = source +BUILDDIR = build + +# Put it first so that "make" without argument is like "make help". +help: + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +.PHONY: help Makefile + +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +%: Makefile + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) \ No newline at end of file diff --git a/docs/source/conf.py b/docs/source/conf.py new file mode 100644 index 0000000..429bb49 --- /dev/null +++ b/docs/source/conf.py @@ -0,0 +1,157 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# +# Fedora Release Engineering Automation documentation build configuration file, created by +# sphinx-quickstart on Thu Nov 16 09:33:29 2017. +# +# This file is execfile()d with the current directory set to its +# containing dir. +# +# Note that not all possible configuration values are present in this +# autogenerated file. +# +# All configuration values have a default; values that are commented out +# serve to show the default. + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +# +# import os +# import sys +# sys.path.insert(0, os.path.abspath('.')) + + +# -- General configuration ------------------------------------------------ + +# If your documentation needs a minimal Sphinx version, state it here. +# +# needs_sphinx = '1.0' + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. +extensions = [] + +# Add any paths that contain templates here, relative to this directory. +templates_path = ['_templates'] + +# The suffix(es) of source filenames. +# You can specify multiple suffix as a list of string: +# +# source_suffix = ['.rst', '.md'] +source_suffix = '.rst' + +# The master toctree document. +master_doc = 'index' + +# General information about the project. +project = 'Fedora Release Engineering Automation' +copyright = '2017, Fedora Project Release Engineering' +author = 'Fedora Project Release Engineering' + +# The version info for the project you're documenting, acts as replacement for +# |version| and |release|, also used in various other places throughout the +# built documents. +# +# The short X.Y version. +version = '0.0.1' +# The full version, including alpha/beta/rc tags. +release = '0.0.1' + +# The language for content autogenerated by Sphinx. Refer to documentation +# for a list of supported languages. +# +# This is also used if you do content translation via gettext catalogs. +# Usually you set "language" from the command line for these cases. +language = None + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +# This patterns also effect to html_static_path and html_extra_path +exclude_patterns = [] + +# The name of the Pygments (syntax highlighting) style to use. +pygments_style = 'sphinx' + +# If true, `todo` and `todoList` produce output, else they produce nothing. +todo_include_todos = False + + +# -- Options for HTML output ---------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +# +html_theme = 'alabaster' + +# Theme options are theme-specific and customize the look and feel of a theme +# further. For a list of options available for each theme, see the +# documentation. +# +# html_theme_options = {} + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = ['_static'] + + +# -- Options for HTMLHelp output ------------------------------------------ + +# Output file base name for HTML help builder. +htmlhelp_basename = 'FedoraReleaseEngineeringAutomationdoc' + + +# -- Options for LaTeX output --------------------------------------------- + +latex_elements = { + # The paper size ('letterpaper' or 'a4paper'). + # + # 'papersize': 'letterpaper', + + # The font size ('10pt', '11pt' or '12pt'). + # + # 'pointsize': '10pt', + + # Additional stuff for the LaTeX preamble. + # + # 'preamble': '', + + # Latex figure (float) alignment + # + # 'figure_align': 'htbp', +} + +# Grouping the document tree into LaTeX files. List of tuples +# (source start file, target name, title, +# author, documentclass [howto, manual, or own class]). +latex_documents = [ + (master_doc, 'FedoraReleaseEngineeringAutomation.tex', 'Fedora Release Engineering Automation Documentation', + 'Fedora Project Release Engineering', 'manual'), +] + + +# -- Options for manual page output --------------------------------------- + +# One entry per manual page. List of tuples +# (source start file, name, description, authors, manual section). +man_pages = [ + (master_doc, 'fedorareleaseengineeringautomation', 'Fedora Release Engineering Automation Documentation', + [author], 1) +] + + +# -- Options for Texinfo output ------------------------------------------- + +# Grouping the document tree into Texinfo files. List of tuples +# (source start file, target name, title, author, +# dir menu entry, description, category) +texinfo_documents = [ + (master_doc, 'FedoraReleaseEngineeringAutomation', 'Fedora Release Engineering Automation Documentation', + author, 'FedoraReleaseEngineeringAutomation', 'One line description of project.', + 'Miscellaneous'), +] + + + diff --git a/docs/source/container_release.rst b/docs/source/container_release.rst new file mode 100644 index 0000000..a6bcc6c --- /dev/null +++ b/docs/source/container_release.rst @@ -0,0 +1,198 @@ +.. SPDX-License-Identifier: CC-BY-SA-3.0 + +Container Layered Images Release +================================ + +At the time of this writing, the Fedora Layered Image Build System (`FLIBS`_) +does not handle automatic rebuilds of container images, nor does `Freshmaker`_ +yet initiate them based on RPM content manifests. As such, the playbook to +release the container images currently handles what is equivalent to a mass +rebuild of RPM content. + +Technical Implementation +------------------------ + +For this procedure there are a few custom `Ansible`_ modules that were written +in order to satisfy specific tasks needed to rebuild and release the container +images. These custom modules live in ``library/`` directory of this repository. + +fedcontainer_rc +~~~~~~~~~~~~~~~ + +The ``fedcontainer_rc`` module exists for the sake of identifying which +containers are release candidates for any specific release. This can be +performed against either stage or production for testing purposes. This playbook +will return two lists that are meant to be used as data inputs to other parts of +the playbook: + +* ``rclist``: This is the release candidate list that will feed into the actual + release of containers when syncing them to the registries containing + an URL structure similar to ``FGC/NAME:VERSION-RELEASE`` where ``FGC`` is + "Fedora Generational Core" which is a `Fedora Modularity`_ notation and the + ``NAME:VERSION-RELEASE`` is similar to that of RPMs. However, we push multiple + iterations of that for content streams to users. The following is an example: + + * ``f27/cockpit:0-1.f27container`` + * ``f27/cockpit:0`` + * ``f27/cockpit`` + +* ``rebuildlist``: This is a list of containers to be rebuilt, it is effectively + just a listing of the ``name`` entries from the ``rclist`` since we want to + rebuild all release candidates (until `FLIBS`_ has that capability built in + and will do that automatically pre-release time). + +fedcontainer_inheritance +~~~~~~~~~~~~~~~~~~~~~~~~ + +The ``fedcontainer_inheritance`` module will query ``src.fedoraproject.org`` or +``src.stg.fedoraproject.org`` depending on prod vs stage request to the module. +It will pull the ``Dockerfile`` contents from all container images passed to it +in a list (effectively expecting the ``rebuild`` list from ``fedcontainer_rc`` +as input). It will use the collection of ``Dockerfile`` data ``FROM`` line to +constract a rudamentary dependency graph in order to build the containers in the +correct order. + +This module will return a list called ``parent_children`` which is a list of +lists, the sublists will be two-element, the first is the parent image, the +latter being a list of child container images. We can then process this nested +list structure using the Ansible `Loops and Includes`_ strategy. + +koji_latestbuild +~~~~~~~~~~~~~~~~ + +The ``koji_latestbuild`` is effectively just an Ansible Module version of the +command ``koji lates-build`` using the koji python API and providing an return +variable of ``latest_builds`` which is a list of python dictionaries that +contains the same set of information provided by the command and can then be +iterated upon and fields referenced as needed in later tasks of the playbook. +This can be optionally run against Fedora's production or stage koji instances. + +fedcontainer_rebuild +~~~~~~~~~~~~~~~~~~~~ + +The ``fedcontainer_rebuild`` module is what handles all the rebuild work, it +will clone the DistGit repo, bump the release of in the Dockerfile, commit to +the DistGit repo, perform a build in koji, wait for koji's builds to complete +and report results. This can also be executed against stage or prod if desired. + +The information returned by ``fedcontainer_rebuild`` is below. + +* ``koji_tasks``: This is a list of the koji tasks that were executed. +* ``koji_watch_task_stdout``: Once the koji tasks have been initiated, there is + a ``watch-task`` initiated to wait for them to complete. This is the stdout of + that command. This is mostly for logging purposes. +* ``koji_watch_task_stderr``: The stdout of the ``watch-task`` described above. +* ``rebuild_list``: list of containers successfully submitted to koji for + rebuild. This information is used to then query koji again for the latest + builds to know which images to sync. + + +Execution +--------- + +The following is a flow diagram of how the ansible playbooks operate. + +:: + + +---------------------------------------------+ + | | + | container_rebuild_and_release_POC.yml | + | | + +--------------------+------------------------+ + | + | + | + V + +--------------------------------------+ + | | + | | + | include/rebuild-containers.yml | + +----------------+---+-----------------+ + | + | + V + +------------------------------------------+ + | | + | include/nested-rebuild-containers.yml |<--------+ + | | | + +-----+------------------------------+-----+ | + | | | + | | | + V | | + +-----------------------------------------------+ | | + | | | | + | include/rebuild-parent-container-images.yml | | | + | | | | + +-----------------------------------------------+ | | + | | | + | V | + | +--------------------------------------------+ | + | | | | + | | include/rebuild-child-container-images.yml | | + | | | | + | +--------------------------------------------+ | + | | | + | | | + V V | + +-----------------------------------+ | + | | | + | include/sync-container-images.yml +-------------------+ + | | + +-----------------------------------+ + +In the above diagram is that we will start execution by setting specific +variables for the run, including the Fedora Release (this could come from +somewhere like PDC) and gather information using the ``fedcontainer_rc`` and +``fecontainer_inheritance`` modules, we will then use the +``include/rebuild-containers.yml`` and ``include/nested-build-containers.yml`` +to process the list of lists in a nested-loop structure as described by the +Ansible `Loops and Includes`_ strategy. In the nested loop we will process first +the parents which are effectively a no-op at this time because the dependency +graph returned by ``fedcontainer_inheritance`` is ordered and therefore once +the children of the base images are built, the effect will cascade down as +parents for the remaining layered image groups, so on and so forth. However, we +have the framework in place to handle the parents differently if that ever +becomes necessary. The child images are then rebuilt and synchronized out to the +registries in order to allow for the next group of images that could be based on +this group can be built on the new content. + +Running the Playbook +-------------------- + +The ultimately goal is to run the playbook from `loopabull`_ but there is still +work to be done in the infrastructure so for the time being you will need to ssh +out to either ``composer.stg.phx2.fedoraproject.org`` for stage or +``compose-x86-01.phx2.fedoraproject.org`` for production, clone this git +repository, change directory to ``ansible/``, and then run the following command: + +.. note:: You will need to forward your ssh agent for the sake of fedpkg until + the service account is setup. + + Also, This command will ask you for your sudo password. + +:: + + localhost$ ssh -A composer.stg.phx2.fedoraproject.org + + composer$ git clone://https://pagure.io/releng-automation.git + + composer$ cd releng-automation/ansible + + composer$ ansible-playbook container_rebuild_and_release.yml -i inventory/inventory.txt + + +.. _FLIBS: https://docs.pagure.org/releng/layered_image_build_service.html +.. _Freshmaker: + https://fedoraproject.org/wiki/Infrastructure/Factory2/Focus/Freshmaker +.. _Fedora Modularity: https://docs.pagure.org/modularity/ +.. _Loops and Includes: + http://docs.ansible.com/ansible/playbooks_loops.html#loops-and-includes-in-2-0 +.. _Ansible: https://github.com/ansible/ansible +.. _loopabull: https://github.com/maxamillion/loopabull +.. _Ansible Role: http://docs.ansible.com/ansible/playbooks_roles.html#roles +.. _Fedora Release Engineering: https://pagure.io/releng +.. _SPDX Unique License Identifiers: http://spdx.org/licenses/ +.. _Release Engineering Automation Workflow Engine: + https://fedoraproject.org/wiki/Changes/ReleaseEngineeringAutomationWorkflowEngine +.. _Fedora Container Guidelines: + https://fedoraproject.org/wiki/Container:Guidelines diff --git a/docs/source/index.rst b/docs/source/index.rst new file mode 100644 index 0000000..4c57348 --- /dev/null +++ b/docs/source/index.rst @@ -0,0 +1,167 @@ +.. Fedora Release Engineering Automation documentation master file, created by + sphinx-quickstart on Thu Nov 16 09:33:29 2017. + You can adapt this file completely to your liking, but it should at least + contain the root `toctree` directive. + +Welcome to Fedora Release Engineering Automation's documentation! +================================================================= + +.. toctree:: + :maxdepth: 2 + :caption: Contents: + + container_release + + +Indices and tables +================== + +* :ref:`genindex` +* :ref:`modindex` +* :ref:`search` + +============================== +Release Engineering Automation +============================== + +This is the `Fedora Release Engineering`_ repository for Automation Workflow +Tooling here. + +One of the primary goals of the Fedora Release Engineering Team moving forward +into the future is to continue to automate more and more components of the +process that actually creates Fedora, and to do so in a way that is maintainable +and can be collaborated upon. This repository is meant to be the focal point of +that effort. + +Release Engineering Automation Workflow +======================================= + +This effort has been put together by the collaboration of many involved in +Fedora and the original public write up of this effort and future goals was the +`Release Engineering Automation Workflow Engine`_. The pertinant details the +original proposal that have lived on are were iterated upon are now in this +repository. + +Currently the goal is to automate or make self-service options for as much as +humanly possible within the Fedora RelEng group using `Ansible`_. The technical +implementation of how this is being accomplished can be seen below. + +Technical Implementation +------------------------ + +Everything will be powered by `Ansible`_ as this is a toolchain that both Fedora +Infrastucture and Fedora Release Engineering is familiar with and has been using +heavily for automation tasks. We are simply aiming to solve a new automation +problem space with the same tool and a different set of rules/policy. + +Tasks or sets of tasks should be in an "Include Playbook" such that they are not +meant to stand on their own but should be included by other Playbooks or an +`Ansible Role`_. + +Workflow Playbooks should effectively be "glue" that supply necessary variables +to make the "Include Playbooks" and Roles useful for the Workflow at hand. + +Execution +--------- + +In the past (or still currently, depending on the specific task and state of +migration to the new workflow) Fedora Release Engineering Automation tasks are +performed by various scripts run on various machines within the Fedora +Infrastructure with no real centralized logging. Some of these are automated by +chron jobs and some run by hand by request of various members within the Fedora +Community, normally around Fedora Test Days. Finding information about old tasks +is not always the easiest of things to do and the delegation of tasks is +currently not available. The goal here is to provide a solution that removes +those barriers. + +Workflows will be executed and potentially orchestrate actions between multiple +other systems or tools such as bodhi, pungi, and koji. Fedmsgs will be emitted +with information about the start and completion of workflows along with metadata +about them. + +In the event of a compose, certain fedmsg output will be picked up by taskotron +and autocloud to perform various levels of testing. + +:: + + +--------------+ +----------------+ + | | +------------+ | | + | AutoCloud |<--------------+ +----------->+ Taskotron | + | | | fedmsg | | | + | +-------------->| |<-----------+ | + +--------------+ | | +----------------+ + +----+-------+ + | ^ + | | + | | + | | + | | + | | + V | + +------------------+-----------------+ + | | + | Release Engineering +-----------+ + | Workflow Automation Engine | | + | (loopabull) | | + | | | + +------------------+-----------------+ | + | | | + | | | + +-----------------+ | | + | | | + | | | + V V | + +-------------+ +--------------+ | + | | | | | + | bodhi | | | | + | | | pungi | | + +-------------+ | | | + | | | + +----------+---+ | + ^ | V + | | +---------------+ + | | | | + | +----------->| koji | + | | | + +------------------+ | + +---------------+ + +Licensing +========= + +Everything is copyrighted by the respective authors. You can use and +redistribute the code under the terms of version 3 or later of the GNU Public +License as published by the Free Software Foundation. + +To make licensing easier, license headers in the source files will be a single +line reference to Unique License Identifiers as defined by the Linux +Foundation's `SPDX`_ project. For example, in a source file the full "GPL v3.0 +or later" header text will be replaced by a single line: + +:: + + SPDX-License-Identifier: GPL-3.0+ + +The license terms of all files in the source tree should be defined by such +License Identifiers; in no case a file can contain more than one such License +Identifier list. + +If a ``SPDX-License-Identifier:`` line references more than one Unique License +Identifier, then this means that the respective file can be used under the terms +of either of these licenses, i. e. with + +:: + + SPDX-License-Identifier: GPL-3.0+ LGPL-3.1+ + +you can chose between GPL-2.0+ and LGPL-2.1+ licensing. + +We use the `SPDX Unique License Identifiers`_ here; + +.. _SPDX: http://spdx.org/ +.. _Ansible: https://github.com/ansible/ansible +.. _Ansible Role: http://docs.ansible.com/ansible/playbooks_roles.html#roles +.. _Fedora Release Engineering: https://pagure.io/releng +.. _SPDX Unique License Identifiers: http://spdx.org/licenses/ +.. _Release Engineering Automation Workflow Engine: + https://fedoraproject.org/wiki/Changes/ReleaseEngineeringAutomationWorkflowEngine