From 7f18809733eb41e9e72d78653fac920eafd05e74 Mon Sep 17 00:00:00 2001 From: Jan Kaluza Date: Apr 20 2017 10:38:00 +0000 Subject: Auto-sign modules. Signed-off-by: Jan Kaluza --- diff --git a/fedmsg.d/example-config.py b/fedmsg.d/example-config.py index c6f165c..a82ddd5 100644 --- a/fedmsg.d/example-config.py +++ b/fedmsg.d/example-config.py @@ -4,6 +4,8 @@ hostname = socket.gethostname() config = { 'robosignatory.enabled.tagsigner': True, 'robosignatory.enabled.atomicsigner': True, + 'robosignatory.pdc_url': 'https://pdc.fedoraproject.org/rest_api/v1', + 'robosignatory.module_prefixes': ['module-'], 'robosignatory.signing': { # This should be the name of an entrypoint plugin that provides @@ -32,6 +34,13 @@ config = { 'key': 'fedora26', 'keyid': 'xxxxxxxx' }, + ], + 'module_streams': [ + { + 'stream': 'master', + 'key': 'fedora26', + 'keyid': 'xxxxxxxx' + } ] }, }, diff --git a/robosignatory/tagconsumer.py b/robosignatory/tagconsumer.py index c52fe48..2348713 100644 --- a/robosignatory/tagconsumer.py +++ b/robosignatory/tagconsumer.py @@ -3,6 +3,7 @@ import koji import fedmsg import fedmsg.consumers import robosignatory.utils as utils +from pdc_client import PDCClient import logging log = logging.getLogger("robosignatory.tagconsumer") @@ -26,6 +27,12 @@ class TagSignerConsumer(fedmsg.consumers.FedmsgConsumer): '%s.%s.buildsys.tag' % (prefix, env) ] + self.module_prefixes = \ + tuple(self.config['robosignatory.module_prefixes']) + + self.pdc_client = PDCClient( + server=self.config['robosignatory.pdc_url'], develop=True) + signing_config = self.hub.config['robosignatory.signing'] self.signer = utils.get_signing_helper(**signing_config) @@ -50,7 +57,8 @@ class TagSignerConsumer(fedmsg.consumers.FedmsgConsumer): raise Exception('Only SSL and kerberos authmethods supported') instance_obj = {'client': client, - 'tags': {}} + 'tags': {}, + 'module_streams': {}} for tag in instance_info['tags']: if tag['from'] in instance_obj['tags']: raise Exception('From detected twice: %s' % tag['from']) @@ -58,6 +66,13 @@ class TagSignerConsumer(fedmsg.consumers.FedmsgConsumer): 'key': tag['key'], 'keyid': tag['keyid']} + for stream in instance_info['module_streams']: + if stream['stream'] in instance_obj['module_streams']: + raise Exception('Module stream detected twice: %s' + % stream['stream']) + instance_obj['module_streams'][stream['stream']] = { + 'key': tag['key'], 'keyid': tag['keyid']} + self.koji_clients[instance] = instance_obj log.info('TagSignerConsumer ready for service') @@ -84,11 +99,6 @@ class TagSignerConsumer(fedmsg.consumers.FedmsgConsumer): tag = msg['tag'] koji_instance = msg['instance'] - self.dowork(build_nvr, build_id, tag, koji_instance, - skip_tagging=False) - - def dowork(self, build_nvr, build_id, tag, koji_instance, - skip_tagging=False): log.info('Build %s (%s) tagged into %s on %s', build_nvr, build_id, tag, koji_instance) @@ -97,14 +107,187 @@ class TagSignerConsumer(fedmsg.consumers.FedmsgConsumer): return instance = self.koji_clients[koji_instance] - if tag not in instance['tags']: - log.info('Tag not autosigned, skipping') + if tag in instance['tags']: + self.dowork(build_nvr, build_id, tag, koji_instance, + skip_tagging=False) + elif tag.startswith(self.module_prefixes): + self.sign_modular_rpms(build_nvr, build_id, tag, koji_instance) + + + def verify_base_runtime_tag(self, tag): + """ + Verifies that the base-runtime tag is valid. Sets the tag['stream'] + and tag['verified']. + """ + query = {} + query["koji_tag"] = tag["name"] + query["active"] = True + retval = self.pdc_client.unreleasedvariants(page_size=-1, **query) + + if (not retval or len(retval) != 1 + or retval[0]["variant_name"] != "base-runtime"): + tag["verified"] = False + return tag + + tag["verified"] = True + tag["stream"] = retval[0]["variant_version"] + return tag + + def get_base_runtime_tag(self, session, info, parent_tags=None): + """ + Recursively traverse the inheritance hiearchy of tags in Koji to find + out the base-runtime tag. + """ + + # Handle only tags with modular prefix. + if not info['name'].startswith(self.module_prefixes): + return None + + # Find all targets pointing to our current tag, there should be just + # single target for our tag. + targets = session.getBuildTargets(destTagID=info['id']) + if not targets: + return None + + # TODO: For now skip the -repo target. We are creating these targets + # for modules only to force kojira to create repo for the modules. + # We can remove these targets and this code once the composes will be + # be running, but for that, we need robosignatory to sign packages + # (chicken and egg)... + targets = [target for target in targets + if target['name'] != info['name'] + "-repo"] + + if len(targets) != 1: + log.info("Expected exactly 1 target for tag %s, skipping." + % info["name"]) + return None + + target = targets[0] + + # Get the build tag of this target. + build_tag_id = target["build_tag"] + build_tag_info = session.getTag(build_tag_id) + + # Store the dest_tag as a possible tag with base-runtime. + base_runtime_tag = {"id": target["dest_tag"], + "name": target["dest_tag_name"], + "verified": False, + "stream": None} + + # Get the inheritance data and filter out tags from parent_tags set. + # Following those tags would bring us back to the already seen target. + inheritance_data = session.getInheritanceData(build_tag_info['name']) + inheritance_data = [data for data in inheritance_data + if data['parent_id'] not in parent_tags] + + # Iterate over all the tags this tag inherits from. There may be many of + # them, because single module can build-require multiple other modules. + for inherited in inheritance_data: + # Make a note to ourselves that we have seen this parent_tag. + parent_tag_id = inherited['parent_id'] + parent_tags.add(parent_tag_id) + + # Get tag info for the parent_tag. + info = session.getTag(parent_tag_id) + if info is None: + return base_runtime_tag + + # Try to recursively find all the parents of this parent tag. + maybe_tag = self.get_base_runtime_tag(session, info, parent_tags) + if not maybe_tag: + continue + + # Verify that the found tag is really valid base-runtime tag. + maybe_tag = self.verify_base_runtime_tag(maybe_tag) + if maybe_tag['verified']: + # In case we have already found valid tag in the previous subtree + # and right now we have another one, compare that their streams + # are matching. + if (base_runtime_tag['verified'] + and maybe_tag['stream'] != base_runtime_tag['stream']): + log.info("Multiple base-runtime streams found in " + "inheritance tree.") + return None + else: + base_runtime_tag = maybe_tag + + return base_runtime_tag + + def sign_modular_rpms(self, build_nvr, build_id, tag, koji_instance): + # Skip the -build tag. + if tag.endswith("-build"): + log.info("Skipping build tag %s" % tag) return + instance = self.koji_clients[koji_instance] + session = instance["client"] + + # Get the tag to find out its id. + info = session.getTag(tag) + if info is None: + log.info("Koji tag %s not known, skipping" % tag) + return + + # Try to find out if the current tag is base-runtime before traversing + # the tag inheritance tree. + maybe_tag = {"id": info["id"], "name": info['name'], "verified": False, + "stream": None} + maybe_tag = self.verify_base_runtime_tag(maybe_tag) + if maybe_tag["verified"]: + base_runtime_tag = maybe_tag + else: + # build tag inherits from the main tag, so when we are evaluating + # which tag is the right one to check when examining tag inheritance + # we have to know that we do not want to go back to the main tag. + # Therefore we have to track the set of parent tags. + parent_tags = set([info['id']]) + + # Resulting base-runtime tag according to which we will use the right key. + base_runtime_tag = self.get_base_runtime_tag( + session, info, parent_tags=parent_tags) + + if not base_runtime_tag: + log.info("No base-runtime tag found in inheritance tree for " + "%s, skipping" % tag) + return + + if not base_runtime_tag["verified"]: + log.info("No verified base-runtime tag found in inheritance tree for " + "%s, skipping. Found tag: %r" % (tag, base_runtime_tag)) + return + + if base_runtime_tag["stream"] not in instance['module_streams']: + log.info("Base-runtime stream %s not allowed for " + "auto-sign" % base_runtime_tag["stream"]) + return + + stream_info = instance['module_streams'][base_runtime_tag["stream"]] + # We are not moving to any tag after signing. + stream_info["to"] = tag + + # Cache the stream_info (which is in the same format as tag_info), so + # for next package in this tag, we know what key to use without + # traversing the Koji tag inheritance tree. + instance["tags"][tag] = stream_info + + self.dowork(build_nvr, build_id, tag, koji_instance, + tag_info=stream_info) + + + def dowork(self, build_nvr, build_id, tag, koji_instance, + skip_tagging=False, tag_info=None): + instance = self.koji_clients[koji_instance] + if not build_id: build_id = instance['client'].findBuildID(build_nvr) - tag_info = instance['tags'][tag] + if not tag_info: + if tag not in instance['tags']: + log.info('Tag not autosigned, skipping') + return + + tag_info = instance['tags'][tag] + log.info('Going to sign with %s (%s) and move to %s', tag_info['key'], tag_info['keyid'], tag_info['to'])