From 43ba5a6071f14db296eed41b377bcbf6be0680d9 Mon Sep 17 00:00:00 2001 From: Ralph Bean Date: Apr 08 2016 14:05:28 +0000 Subject: [PATCH 1/3] Install both hub and builder plugins. This is a different take at https://pagure.io/koji/pull-request/45 Instead of installing just the runroot builder plugin explicitly, here we separate the plugins out into hub and builder plugins explicitly and install each type in turn. --- diff --git a/plugins/Makefile b/plugins/Makefile index 0bbf748..f570286 100644 --- a/plugins/Makefile +++ b/plugins/Makefile @@ -1,8 +1,12 @@ PYTHON=python PLUGINDIR = /usr/lib/koji-hub-plugins -FILES = $(wildcard *.py) +BUILDERPLUGINDIR = /usr/lib/koji-builder-plugins +FILES = $(wildcard hub/*.py) +BUILDERFILES = $(wildcard builder/*.py) CONFDIR = /etc/koji-hub/plugins -CONFFILES = $(wildcard *.conf) +BUILDERCONFDIR = /etc/kojid +CONFFILES = $(wildcard hub/*.conf) +BUILDERCONFFILES = $(wildcard builder/*.conf) _default: @echo "nothing to make. try make install" @@ -18,7 +22,12 @@ install: fi mkdir -p $(DESTDIR)/$(PLUGINDIR) + mkdir -p $(DESTDIR)/$(BUILDERPLUGINDIR) install -p -m 644 $(FILES) $(DESTDIR)/$(PLUGINDIR) + install -p -m 644 $(BUILDERFILES) $(DESTDIR)/$(BUILDERPLUGINDIR) $(PYTHON) -c "import compileall; compileall.compile_dir('$(DESTDIR)/$(PLUGINDIR)', 1, '$(PLUGINDIR)', 1)" + $(PYTHON) -c "import compileall; compileall.compile_dir('$(DESTDIR)/$(BUILDERPLUGINDIR)', 1, '$(BUILDERPLUGINDIR)', 1)" mkdir -p $(DESTDIR)/$(CONFDIR) + mkdir -p $(DESTDIR)/$(BUILDERCONFDIR) install -p -m 644 $(CONFFILES) $(DESTDIR)/$(CONFDIR) + install -p -m 644 $(BUILDERCONFFILES) $(DESTDIR)/$(BUILDERCONFDIR) diff --git a/plugins/builder/runroot.conf b/plugins/builder/runroot.conf new file mode 100644 index 0000000..d3d222b --- /dev/null +++ b/plugins/builder/runroot.conf @@ -0,0 +1,25 @@ +[paths] +; comma-delimited list of default mountpoints +; They will be mounted during each run. It is suggested, that these +; paths has readonly options and are made writable via extra_mounts +; parameter for individual calls. +; default_mounts = /mnt/archive,/mnt/workdir + +; comma-delimited list of safe roots. +; Each extra_mount need to start with some of these prefixes. Other paths are +; not allowed for mounting. Only absolute paths are allowed here, no +; wildcards. +; safe_roots = /mnt/workdir/tmp + +; path substitutions is tuple per line, delimited by comma, order is +; important. +; Path prefixes which can be substituted for other mountpoints. +; Usable for locations symlinked from other mounts. +; path_subs = /mnt/archive/prehistory/,/mnt/prehistoric_disk/archive/prehistory + +; mount origins, order is important here, ordered by best catch +; [path0] +; mountpoint = /mnt/archive +; path = archive.org:/vol/archive +; fstype = nfs +; options = ro,hard,intr,nosuid,nodev,noatime,tcp diff --git a/plugins/builder/runroot.py b/plugins/builder/runroot.py new file mode 100644 index 0000000..9a71c91 --- /dev/null +++ b/plugins/builder/runroot.py @@ -0,0 +1,322 @@ +# kojid plugin + +import commands +import koji +import ConfigParser +import os +import platform +compat_mode = False +try: + import koji.tasks as tasks + from koji.tasks import scan_mounts + from koji.util import isSuccess as _isSuccess + from koji.util import parseStatus as _parseStatus + from koji.daemon import log_output + from __main__ import BuildRoot +except ImportError: + compat_mode = True + #old way + import tasks + #XXX - stuff we need from kojid + from __main__ import BuildRoot, log_output, scan_mounts, _isSuccess, _parseStatus + + +__all__ = ('RunRootTask',) + +CONFIG_FILE = '/etc/kojid/runroot.conf' + + +class RunRootTask(tasks.BaseTaskHandler): + + Methods = ['runroot'] + + _taskWeight = 2.0 + + def __init__(self, *args, **kwargs): + self._read_config() + return super(RunRootTask, self).__init__(*args, **kwargs) + + def _get_path_params(self, path, rw=False): + found = False + for mount_data in self.config['paths']: + if path.startswith(mount_data['mountpoint']): + found = True + break + if not found: + raise koji.GenericError("bad config: missing corresponding mountpoint") + options = [] + for o in mount_data['options'].split(','): + if rw and o == 'ro': + options.append('rw') + else: + options.append(o) + rel_path = path[len(mount_data['mountpoint']):] + rel_path = rel_path[1:] if rel_path.startswith('/') else rel_path + res = (os.path.join(mount_data['path'], rel_path), path, mount_data['fstype'], ','.join(options)) + return res + + def _read_config(self): + cp = ConfigParser.SafeConfigParser() + cp.read(CONFIG_FILE) + self.config = { + 'default_mounts': [], + 'safe_roots': [], + 'path_subs': [], + 'paths': [], + } + + if cp.has_option('paths', 'default_mounts'): + self.config['default_mounts'] = cp.get('paths', 'default_mounts').split(',') + if cp.has_option('paths', 'safe_roots'): + self.config['safe_roots'] = cp.get('paths', 'safe_roots').split(',') + if cp.has_option('paths', 'path_subs'): + self.config['path_subs'] = [x.split(',') for x in cp.get('paths', 'path_subs').split('\n')] + + count = 0 + while True: + section_name = 'path%d' % count + if not cp.has_section(section_name): + break + try: + self.config['paths'].append({ + 'mountpoint': cp.get(section_name, 'mountpoint'), + 'path': cp.get(section_name, 'path'), + 'fstype': cp.get(section_name, 'fstype'), + 'options': cp.get(section_name, 'options'), + }) + except ConfigParser.NoOptionError: + raise koji.GenericError("bad config: missing options in %s section" % section_name) + count += 1 + + for path in self.config['default_mounts'] + self.config['safe_roots'] + [x[0] for x in self.config['path_subs']]: + if not path.startswith('/'): + raise koji.GenericError("bad config: all paths (default_mounts, safe_roots, path_subs) needs to be absolute: %s" % path) + + def handler(self, root, arch, command, keep=False, packages=[], mounts=[], repo_id=None, skip_setarch=False, weight=None, upload_logs=None): + """Create a buildroot and run a command (as root) inside of it + + Command may be a string or a list. + + Returns a message indicating success if the command was successful, and + raises an error otherwise. Command output will be available in + runroot.log in the task output directory on the hub. + + skip_setarch is a rough approximation of an old hack + + the keep option is not used. keeping for compatibility for now... + + upload_logs is list of absolute paths which will be uploaded for + archiving on hub. It always consists of /tmp/runroot.log, but can be + used for additional logs (pungi.log, etc.) + """ + if weight is not None: + weight = max(weight, 0.5) + self.session.host.setTaskWeight(self.id, weight) + #noarch is funny + if arch == "noarch": + #we need a buildroot arch. Pick one that: + # a) this host can handle + # b) the build tag can support + # c) is canonical + host_arches = self.session.host.getHost()['arches'] + if not host_arches: + raise koji.BuildError, "No arch list for this host" + tag_arches = self.session.getBuildConfig(root)['arches'] + if not tag_arches: + raise koji.BuildError, "No arch list for tag: %s" % root + #index canonical host arches + host_arches = dict([(koji.canonArch(a),1) for a in host_arches.split()]) + #pick the first suitable match from tag's archlist + for br_arch in tag_arches.split(): + br_arch = koji.canonArch(br_arch) + if host_arches.has_key(br_arch): + #we're done + break + else: + #no overlap + raise koji.BuildError, "host does not match tag arches: %s (%s)" % (root, tag_arches) + else: + br_arch = arch + if repo_id: + repo_info = self.session.repoInfo(repo_id, strict=True) + if repo_info['tag_name'] != root: + raise koji.BuildError, "build tag (%s) does not match repo tag (%s)" % (root, repo_info['tag_name']) + if repo_info['state'] not in (koji.REPO_STATES['READY'], koji.REPO_STATES['EXPIRED']): + raise koji.BuildError, "repos in the %s state may not be used by runroot" % koji.REPO_STATES[repo_info['state']] + else: + repo_info = self.session.getRepo(root) + if not repo_info: + #wait for it + task_id = self.session.host.subtask(method='waitrepo', + arglist=[root, None, None], + parent=self.id) + repo_info = self.wait(task_id)[task_id] + if compat_mode: + broot = BuildRoot(root, br_arch, self.id, repo_id=repo_info['id']) + else: + broot = BuildRoot(self.session, self.options, root, br_arch, self.id, repo_id=repo_info['id']) + broot.workdir = self.workdir + broot.init() + rootdir = broot.rootdir() + #workaround for rpm oddness + os.system('rm -f "%s"/var/lib/rpm/__db.*' % rootdir) + #update buildroot state (so that updateBuildRootList() will work) + self.session.host.setBuildRootState(broot.id, 'BUILDING') + try: + if packages: + #pkglog = '%s/%s' % (broot.resultdir(), 'packages.log') + pkgcmd = ['--install'] + packages + status = broot.mock(pkgcmd) + self.session.host.updateBuildRootList(broot.id, broot.getPackageList()) + if not _isSuccess(status): + raise koji.BuildrootError, _parseStatus(status, pkgcmd) + + if isinstance(command, str): + cmdstr = command + else: + #we were passed an arglist + #we still have to run this through the shell (for redirection) + #but we can preserve the list structure precisely with careful escaping + cmdstr = ' '.join(["'%s'" % arg.replace("'", r"'\''") for arg in command]) + # A nasty hack to put command output into its own file until mock can be + # patched to do something more reasonable than stuff everything into build.log + cmdargs = ['/bin/sh', '-c', "{ %s; } < /dev/null 2>&1 | /usr/bin/tee /tmp/runroot.log; exit ${PIPESTATUS[0]}" % cmdstr] + + # always mount /mnt/redhat (read-only) + # always mount /mnt/iso (read-only) + # also need /dev bind mount + self.do_mounts(rootdir, [self._get_path_params(x) for x in self.config['default_mounts']]) + self.do_extra_mounts(rootdir, mounts) + mock_cmd = ['chroot'] + if skip_setarch: + #we can't really skip it, but we can set it to the current one instead of of the chroot one + myarch = platform.uname()[5] + mock_cmd.extend(['--arch', myarch]) + mock_cmd.append('--') + mock_cmd.extend(cmdargs) + rv = broot.mock(mock_cmd) + log_paths = ['/tmp/runroot.log'] + if upload_logs is not None: + log_paths += upload_logs + for log_path in log_paths: + self.uploadFile(rootdir + log_path) + finally: + # mock should umount its mounts, but it will not handle ours + self.undo_mounts(rootdir, fatal=False) + broot.expire() + if isinstance(command, str): + cmdlist = command.split() + else: + cmdlist = command + cmdlist = [param for param in cmdlist if '=' not in param] + if cmdlist: + cmd = os.path.basename(cmdlist[0]) + else: + cmd = '(none)' + if _isSuccess(rv): + return '%s completed successfully' % cmd + else: + raise koji.BuildrootError, _parseStatus(rv, cmd) + + def do_extra_mounts(self, rootdir, mounts): + mnts = [] + for mount in mounts: + mount = os.path.normpath(mount) + for safe_root in self.config['safe_roots']: + if mount.startswith(safe_root): + break + else: + #no match + raise koji.GenericError("read-write mount point is not safe: %s" % mount) + #normpath should have removed any .. dirs, but just in case... + if mount.find('/../') != -1: + raise koji.GenericError("read-write mount point is not safe: %s" % mount) + + for re, sub in self.config['path_subs']: + mount = mount.replace(re, sub) + + mnts.append(self._get_path_params(mount, rw=True)) + self.do_mounts(rootdir, mnts) + + def do_mounts(self, rootdir, mounts): + if not mounts: + return + self.logger.info('New runroot') + self.logger.info("Runroot mounts: %s" % mounts) + fn = '%s/tmp/runroot_mounts' % rootdir + fslog = file(fn, 'a') + logfile = "%s/do_mounts.log" % self.workdir + uploadpath = self.getUploadDir() + error = None + for dev,path,type,opts in mounts: + if not path.startswith('/'): + raise koji.GenericError("invalid mount point: %s" % path) + mpoint = "%s%s" % (rootdir,path) + if opts is None: + opts = [] + else: + opts = opts.split(',') + if 'bind' in opts: + #make sure dir exists + if not os.path.isdir(dev): + error = koji.GenericError("No such directory or mount: %s" % dev) + break + type = 'none' + if path is None: + #shorthand for "same path" + path = dev + if 'bg' in opts: + error = koji.GenericError("bad config: background mount not allowed") + break + opts = ','.join(opts) + cmd = ['mount', '-t', type, '-o', opts, dev, mpoint] + self.logger.info("Mount command: %r" % cmd) + koji.ensuredir(mpoint) + if compat_mode: + status = log_output(cmd[0], cmd, logfile, uploadpath, logerror=True, append=True) + else: + status = log_output(self.session, cmd[0], cmd, logfile, uploadpath, logerror=True, append=True) + if not _isSuccess(status): + error = koji.GenericError("Unable to mount %s: %s" \ + % (mpoint, _parseStatus(status, cmd))) + break + fslog.write("%s\n" % mpoint) + fslog.flush() + fslog.close() + if error is not None: + self.undo_mounts(rootdir, fatal=False) + raise error + + def undo_mounts(self, rootdir, fatal=True): + self.logger.debug("Unmounting runroot mounts") + mounts = {} + fn = '%s/tmp/runroot_mounts' % rootdir + if os.path.exists(fn): + fslog = file(fn,'r') + for line in fslog: + mounts.setdefault(line.strip(), 1) + fslog.close() + #also, check /proc/mounts just in case + for dir in scan_mounts(rootdir): + mounts.setdefault(dir, 1) + mounts = mounts.keys() + # deeper directories first + mounts.sort() + mounts.reverse() + failed = [] + self.logger.info("Unmounting (runroot): %s" % mounts) + for dir in mounts: + (rv, output) = commands.getstatusoutput("umount -l '%s'" % dir) + if rv != 0: + failed.append("%s: %s" % (dir, output)) + if failed: + msg = "Unable to unmount: %s" % ', '.join(failed) + self.logger.warn(msg) + if fatal: + raise koji.GenericError, msg + else: + # remove the mount list when everything is unmounted + try: + os.unlink(fn) + except OSError: + pass diff --git a/plugins/echo.py b/plugins/echo.py deleted file mode 100644 index 6727d41..0000000 --- a/plugins/echo.py +++ /dev/null @@ -1,15 +0,0 @@ -# Example Koji callback -# Copyright (c) 2009-2014 Red Hat, Inc. -# This callback simply logs all of its args using the logging module -# -# Authors: -# Mike Bonnet - -from koji.plugin import callbacks, callback, ignore_error -import logging - -@callback(*callbacks.keys()) -@ignore_error -def echo(cbtype, *args, **kws): - logging.getLogger('koji.plugin.echo').info('Called the %s callback, args: %s; kws: %s', - cbtype, str(args), str(kws)) diff --git a/plugins/hub/echo.py b/plugins/hub/echo.py new file mode 100644 index 0000000..6727d41 --- /dev/null +++ b/plugins/hub/echo.py @@ -0,0 +1,15 @@ +# Example Koji callback +# Copyright (c) 2009-2014 Red Hat, Inc. +# This callback simply logs all of its args using the logging module +# +# Authors: +# Mike Bonnet + +from koji.plugin import callbacks, callback, ignore_error +import logging + +@callback(*callbacks.keys()) +@ignore_error +def echo(cbtype, *args, **kws): + logging.getLogger('koji.plugin.echo').info('Called the %s callback, args: %s; kws: %s', + cbtype, str(args), str(kws)) diff --git a/plugins/hub/messagebus.conf b/plugins/hub/messagebus.conf new file mode 100644 index 0000000..fe18a1c --- /dev/null +++ b/plugins/hub/messagebus.conf @@ -0,0 +1,24 @@ +# config file for the Koji messagebus plugin + +[broker] +host = amqp.example.com +port = 5671 +ssl = true +timeout = 10 +heartbeat = 60 +# PLAIN options +auth = PLAIN +username = guest +password = guest +# GSSAPI options +# auth = GSSAPI +# keytab = /etc/koji-hub/plugins/koji-messagebus.keytab +# principal = messagebus/koji.example.com@EXAMPLE.COM + +[exchange] +name = koji.events +type = topic +durable = true + +[topic] +prefix = koji.event diff --git a/plugins/hub/messagebus.py b/plugins/hub/messagebus.py new file mode 100644 index 0000000..3f3dc6f --- /dev/null +++ b/plugins/hub/messagebus.py @@ -0,0 +1,226 @@ +# Koji callback for sending notifications about events to a messagebus (amqp broker) +# Copyright (c) 2009-2014 Red Hat, Inc. +# +# Authors: +# Mike Bonnet + +from koji.plugin import callbacks, callback, ignore_error +import ConfigParser +import logging +import qpid.messaging +import qpid.messaging.transports +from ssl import wrap_socket +import socket +import os +import krbV + +MAX_KEY_LENGTH = 255 +CONFIG_FILE = '/etc/koji-hub/plugins/messagebus.conf' + +config = None +session = None +target = None + +def connect_timeout(host, port, timeout): + for res in socket.getaddrinfo(host, port, 0, socket.SOCK_STREAM): + af, socktype, proto, canonname, sa = res + sock = socket.socket(af, socktype, proto) + sock.settimeout(timeout) + try: + sock.connect(sa) + break + except socket.error, msg: + sock.close() + else: + # If we got here then we couldn't connect (yet) + raise + return sock + +class tlstimeout(qpid.messaging.transports.tls): + def __init__(self, conn, host, port): + self.socket = connect_timeout(host, port, getattr(conn, '_timeout')) + if conn.tcp_nodelay: + self.socket.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) + self.tls = wrap_socket(self.socket, keyfile=conn.ssl_keyfile, certfile=conn.ssl_certfile, ca_certs=conn.ssl_trustfile) + self.socket.setblocking(0) + self.state = None + +qpid.messaging.transports.TRANSPORTS['tls+timeout'] = tlstimeout + +class Connection(qpid.messaging.Connection): + """ + A connection class which supports a timeout option + to the establish() method. Only necessary until + upstream Apache Qpid commit 1487578 is available in + a supported release. + """ + @staticmethod + def establish(url=None, timeout=None, **options): + conn = Connection(url, **options) + conn._timeout = timeout + conn.open() + return conn + + def _wait(self, predicate, timeout=None): + if timeout is None and hasattr(self, '_timeout'): + timeout = self._timeout + return qpid.messaging.Connection._wait(self, predicate, timeout) + +def get_sender(): + global config, session, target + if session and target: + try: + return session.sender(target) + except: + logging.getLogger('koji.plugin.messagebus').warning('Error getting session, will retry', exc_info=True) + session = None + target = None + + config = ConfigParser.SafeConfigParser() + config.read(CONFIG_FILE) + if not config.has_option('broker', 'timeout'): + config.set('broker', 'timeout', '60') + if not config.has_option('broker', 'heartbeat'): + config.set('broker', 'heartbeat', '60') + + if config.getboolean('broker', 'ssl'): + url = 'amqps://' + else: + url = 'amqp://' + auth = config.get('broker', 'auth') + if auth == 'PLAIN': + url += config.get('broker', 'username') + '/' + url += config.get('broker', 'password') + '@' + elif auth == 'GSSAPI': + ccname = 'MEMORY:messagebus' + os.environ['KRB5CCNAME'] = ccname + ctx = krbV.default_context() + ccache = krbV.CCache(name=ccname, context=ctx) + cprinc = krbV.Principal(name=config.get('broker', 'principal'), context=ctx) + ccache.init(principal=cprinc) + keytab = krbV.Keytab(name='FILE:' + config.get('broker', 'keytab'), context=ctx) + ccache.init_creds_keytab(principal=cprinc, keytab=keytab) + else: + raise koji.PluginError, 'unsupported auth type: %s' % auth + + url += config.get('broker', 'host') + ':' + url += config.get('broker', 'port') + + conn = Connection.establish(url, + sasl_mechanisms=config.get('broker', 'auth'), + transport='tls+timeout', + timeout=config.getfloat('broker', 'timeout'), + heartbeat=config.getint('broker', 'heartbeat')) + sess = conn.session() + tgt = """%s; + { create: sender, + assert: always, + node: { type: topic, + durable: %s, + x-declare: { exchange: "%s", + type: %s } } }""" % \ + (config.get('exchange', 'name'), config.getboolean('exchange', 'durable'), + config.get('exchange', 'name'), config.get('exchange', 'type')) + sender = sess.sender(tgt) + session = sess + target = tgt + + return sender + +def _token_append(tokenlist, val): + # Replace any periods with underscores so we have a deterministic number of tokens + val = val.replace('.', '_') + tokenlist.append(val) + +def get_message_subject(msgtype, *args, **kws): + key = [config.get('topic', 'prefix'), msgtype] + + if msgtype == 'PackageListChange': + _token_append(key, kws['tag']['name']) + _token_append(key, kws['package']['name']) + elif msgtype == 'TaskStateChange': + _token_append(key, kws['info']['method']) + _token_append(key, kws['attribute']) + elif msgtype == 'BuildStateChange': + info = kws['info'] + _token_append(key, kws['attribute']) + _token_append(key, info['name']) + elif msgtype == 'Import': + _token_append(key, kws['type']) + elif msgtype in ('Tag', 'Untag'): + _token_append(key, kws['tag']['name']) + build = kws['build'] + _token_append(key, build['name']) + _token_append(key, kws['user']['name']) + elif msgtype == 'RepoInit': + _token_append(key, kws['tag']['name']) + elif msgtype == 'RepoDone': + _token_append(key, kws['repo']['tag_name']) + + key = '.'.join(key) + key = key[:MAX_KEY_LENGTH] + return key + +def get_message_headers(msgtype, *args, **kws): + headers = {'type': msgtype} + + if msgtype == 'PackageListChange': + headers['tag'] = kws['tag']['name'] + headers['package'] = kws['package']['name'] + elif msgtype == 'TaskStateChange': + headers['id'] = kws['info']['id'] + headers['parent'] = kws['info']['parent'] + headers['method'] = kws['info']['method'] + headers['attribute'] = kws['attribute'] + headers['old'] = kws['old'] + headers['new'] = kws['new'] + elif msgtype == 'BuildStateChange': + info = kws['info'] + headers['name'] = info['name'] + headers['version'] = info['version'] + headers['release'] = info['release'] + headers['attribute'] = kws['attribute'] + headers['old'] = kws['old'] + headers['new'] = kws['new'] + elif msgtype == 'Import': + headers['importType'] = kws['type'] + elif msgtype in ('Tag', 'Untag'): + headers['tag'] = kws['tag']['name'] + build = kws['build'] + headers['name'] = build['name'] + headers['version'] = build['version'] + headers['release'] = build['release'] + headers['user'] = kws['user']['name'] + elif msgtype == 'RepoInit': + headers['tag'] = kws['tag']['name'] + elif msgtype == 'RepoDone': + headers['tag'] = kws['repo']['tag_name'] + + return headers + +@callback(*[c for c in callbacks.keys() if c.startswith('post')]) +@ignore_error +def send_message(cbtype, *args, **kws): + global config + sender = get_sender() + if cbtype.startswith('post'): + msgtype = cbtype[4:] + else: + msgtype = cbtype[3:] + + data = kws.copy() + if args: + data['args'] = list(args) + + exchange_type = config.get('exchange', 'type') + if exchange_type == 'topic': + subject = get_message_subject(msgtype, *args, **kws) + message = qpid.messaging.Message(subject=subject, content=data) + elif exchange_type == 'headers': + headers = get_message_headers(msgtype, *args, **kws) + message = qpid.messaging.Message(properties=headers, content=data) + else: + raise koji.PluginError, 'unsupported exchange type: %s' % exchange_type + + sender.send(message, sync=True, timeout=config.getfloat('broker', 'timeout')) + sender.close(timeout=config.getfloat('broker', 'timeout')) diff --git a/plugins/hub/rpm2maven.conf b/plugins/hub/rpm2maven.conf new file mode 100644 index 0000000..900bf13 --- /dev/null +++ b/plugins/hub/rpm2maven.conf @@ -0,0 +1,5 @@ +# config file for the Koji rpm2maven plugin + +[patterns] +rpm_names = *-repolib +artifact_paths = /usr/share/java/repository/maven2/* diff --git a/plugins/hub/rpm2maven.py b/plugins/hub/rpm2maven.py new file mode 100644 index 0000000..484a319 --- /dev/null +++ b/plugins/hub/rpm2maven.py @@ -0,0 +1,107 @@ +# Koji callback for extracting Maven artifacts (.pom and .jar files) +# from an rpm and making them available via the Koji-managed Maven repo. +# Copyright (c) 2010-2014 Red Hat, Inc. +# +# Authors: +# Mike Bonnet + +import koji +from koji.context import context +from koji.plugin import callback +import ConfigParser +import fnmatch +import os +import shutil +import subprocess + +CONFIG_FILE = '/etc/koji-hub/plugins/rpm2maven.conf' + +config = None + +@callback('postImport') +def maven_import(cbtype, *args, **kws): + global config + if not context.opts.get('EnableMaven', False): + return + if kws.get('type') != 'rpm': + return + buildinfo = kws['build'] + rpminfo = kws['rpm'] + filepath = kws['filepath'] + + if not config: + config = ConfigParser.SafeConfigParser() + config.read(CONFIG_FILE) + name_patterns = config.get('patterns', 'rpm_names').split() + for pattern in name_patterns: + if fnmatch.fnmatch(rpminfo['name'], pattern): + break + else: + return + + tmpdir = os.path.join(koji.pathinfo.work(), 'rpm2maven', koji.buildLabel(buildinfo)) + try: + if os.path.exists(tmpdir): + shutil.rmtree(tmpdir) + koji.ensuredir(tmpdir) + expand_rpm(filepath, tmpdir) + scan_and_import(buildinfo, rpminfo, tmpdir) + finally: + if os.path.exists(tmpdir): + shutil.rmtree(tmpdir) + +def expand_rpm(filepath, tmpdir): + devnull = file('/dev/null', 'r+') + rpm2cpio = subprocess.Popen(['/usr/bin/rpm2cpio', filepath], + stdout=subprocess.PIPE, + stdin=devnull, stderr=devnull, + close_fds=True) + cpio = subprocess.Popen(['/bin/cpio', '-id'], + stdin=rpm2cpio.stdout, + cwd=tmpdir, + stdout=devnull, stderr=devnull, + close_fds=True) + if rpm2cpio.wait() != 0 or cpio.wait() != 0: + raise koji.CallbackError, 'error extracting files from %s, ' \ + 'rpm2cpio returned %s, cpio returned %s' % \ + (filepath, rpm2cpio.wait(), cpio.wait()) + devnull.close() + +def scan_and_import(buildinfo, rpminfo, tmpdir): + global config + path_patterns = config.get('patterns', 'artifact_paths').split() + + maven_archives = [] + for dirpath, dirnames, filenames in os.walk(tmpdir): + relpath = dirpath[len(tmpdir):] + for pattern in path_patterns: + if fnmatch.fnmatch(relpath, pattern): + break + else: + continue + + poms = [f for f in filenames if f.endswith('.pom')] + if len(poms) != 1: + continue + + pom_info = koji.parse_pom(os.path.join(dirpath, poms[0])) + maven_info = koji.pom_to_maven_info(pom_info) + maven_archives.append({'maven_info': maven_info, + 'files': [os.path.join(dirpath, f) for f in filenames]}) + + if not maven_archives: + return + + # We don't know which pom is the top-level pom, so we don't know what Maven + # metadata to associate with the build. So we make something up. + maven_build = {'group_id': buildinfo['name'], 'artifact_id': rpminfo['name'], + 'version': '%(version)s-%(release)s' % buildinfo} + context.handlers.call('host.createMavenBuild', buildinfo, maven_build) + + for entry in maven_archives: + maven_info = entry['maven_info'] + for filepath in entry['files']: + if not context.handlers.call('getArchiveType', filename=filepath): + # unsupported archive type, skip it + continue + context.handlers.call('host.importArchive', filepath, buildinfo, 'maven', maven_info) diff --git a/plugins/hub/runroot_hub.py b/plugins/hub/runroot_hub.py new file mode 100644 index 0000000..a666919 --- /dev/null +++ b/plugins/hub/runroot_hub.py @@ -0,0 +1,61 @@ +#koji hub plugin +# There is a kojid plugin that goes with this hub plugin. The kojid builder +# plugin has a config file. This hub plugin has no config file. + + +from koji.context import context +from koji.plugin import export +import koji +import random +import sys + +#XXX - have to import kojihub for mktask +sys.path.insert(0, '/usr/share/koji-hub/') +from kojihub import mktask, get_tag, get_all_arches + +__all__ = ('runroot',) + + +def get_channel_arches(channel): + """determine arches available in channel""" + chan = context.handlers.call('getChannel', channel, strict=True) + ret = {} + for host in context.handlers.call('listHosts', channelID=chan['id'], enabled=True): + for a in host['arches'].split(): + ret[koji.canonArch(a)] = 1 + return ret + +@export +def runroot(tagInfo, arch, command, channel=None, **opts): + """ Create a runroot task """ + context.session.assertPerm('runroot') + taskopts = { + 'priority': 15, + 'arch': arch, + } + + taskopts['channel'] = channel or 'runroot' + + if arch == 'noarch': + #not all arches can generate a proper buildroot for all tags + tag = get_tag(tagInfo) + if not tag['arches']: + raise koji.GenericError, 'no arches defined for tag %s' % tag['name'] + + #get all known arches for the system + fullarches = get_all_arches() + + tagarches = tag['arches'].split() + + # If our tag can't do all arches, then we need to + # specify one of the arches it can do. + if set(fullarches) - set(tagarches): + chanarches = get_channel_arches(taskopts['channel']) + choices = [x for x in tagarches if x in chanarches] + if not choices: + raise koji.GenericError, 'no common arches for tag/channel: %s/%s' \ + % (tagInfo, taskopts['channel']) + taskopts['arch'] = koji.canonArch(random.choice(choices)) + + return mktask(taskopts,'runroot', tagInfo, arch, command, **opts) + diff --git a/plugins/messagebus.conf b/plugins/messagebus.conf deleted file mode 100644 index fe18a1c..0000000 --- a/plugins/messagebus.conf +++ /dev/null @@ -1,24 +0,0 @@ -# config file for the Koji messagebus plugin - -[broker] -host = amqp.example.com -port = 5671 -ssl = true -timeout = 10 -heartbeat = 60 -# PLAIN options -auth = PLAIN -username = guest -password = guest -# GSSAPI options -# auth = GSSAPI -# keytab = /etc/koji-hub/plugins/koji-messagebus.keytab -# principal = messagebus/koji.example.com@EXAMPLE.COM - -[exchange] -name = koji.events -type = topic -durable = true - -[topic] -prefix = koji.event diff --git a/plugins/messagebus.py b/plugins/messagebus.py deleted file mode 100644 index 3f3dc6f..0000000 --- a/plugins/messagebus.py +++ /dev/null @@ -1,226 +0,0 @@ -# Koji callback for sending notifications about events to a messagebus (amqp broker) -# Copyright (c) 2009-2014 Red Hat, Inc. -# -# Authors: -# Mike Bonnet - -from koji.plugin import callbacks, callback, ignore_error -import ConfigParser -import logging -import qpid.messaging -import qpid.messaging.transports -from ssl import wrap_socket -import socket -import os -import krbV - -MAX_KEY_LENGTH = 255 -CONFIG_FILE = '/etc/koji-hub/plugins/messagebus.conf' - -config = None -session = None -target = None - -def connect_timeout(host, port, timeout): - for res in socket.getaddrinfo(host, port, 0, socket.SOCK_STREAM): - af, socktype, proto, canonname, sa = res - sock = socket.socket(af, socktype, proto) - sock.settimeout(timeout) - try: - sock.connect(sa) - break - except socket.error, msg: - sock.close() - else: - # If we got here then we couldn't connect (yet) - raise - return sock - -class tlstimeout(qpid.messaging.transports.tls): - def __init__(self, conn, host, port): - self.socket = connect_timeout(host, port, getattr(conn, '_timeout')) - if conn.tcp_nodelay: - self.socket.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) - self.tls = wrap_socket(self.socket, keyfile=conn.ssl_keyfile, certfile=conn.ssl_certfile, ca_certs=conn.ssl_trustfile) - self.socket.setblocking(0) - self.state = None - -qpid.messaging.transports.TRANSPORTS['tls+timeout'] = tlstimeout - -class Connection(qpid.messaging.Connection): - """ - A connection class which supports a timeout option - to the establish() method. Only necessary until - upstream Apache Qpid commit 1487578 is available in - a supported release. - """ - @staticmethod - def establish(url=None, timeout=None, **options): - conn = Connection(url, **options) - conn._timeout = timeout - conn.open() - return conn - - def _wait(self, predicate, timeout=None): - if timeout is None and hasattr(self, '_timeout'): - timeout = self._timeout - return qpid.messaging.Connection._wait(self, predicate, timeout) - -def get_sender(): - global config, session, target - if session and target: - try: - return session.sender(target) - except: - logging.getLogger('koji.plugin.messagebus').warning('Error getting session, will retry', exc_info=True) - session = None - target = None - - config = ConfigParser.SafeConfigParser() - config.read(CONFIG_FILE) - if not config.has_option('broker', 'timeout'): - config.set('broker', 'timeout', '60') - if not config.has_option('broker', 'heartbeat'): - config.set('broker', 'heartbeat', '60') - - if config.getboolean('broker', 'ssl'): - url = 'amqps://' - else: - url = 'amqp://' - auth = config.get('broker', 'auth') - if auth == 'PLAIN': - url += config.get('broker', 'username') + '/' - url += config.get('broker', 'password') + '@' - elif auth == 'GSSAPI': - ccname = 'MEMORY:messagebus' - os.environ['KRB5CCNAME'] = ccname - ctx = krbV.default_context() - ccache = krbV.CCache(name=ccname, context=ctx) - cprinc = krbV.Principal(name=config.get('broker', 'principal'), context=ctx) - ccache.init(principal=cprinc) - keytab = krbV.Keytab(name='FILE:' + config.get('broker', 'keytab'), context=ctx) - ccache.init_creds_keytab(principal=cprinc, keytab=keytab) - else: - raise koji.PluginError, 'unsupported auth type: %s' % auth - - url += config.get('broker', 'host') + ':' - url += config.get('broker', 'port') - - conn = Connection.establish(url, - sasl_mechanisms=config.get('broker', 'auth'), - transport='tls+timeout', - timeout=config.getfloat('broker', 'timeout'), - heartbeat=config.getint('broker', 'heartbeat')) - sess = conn.session() - tgt = """%s; - { create: sender, - assert: always, - node: { type: topic, - durable: %s, - x-declare: { exchange: "%s", - type: %s } } }""" % \ - (config.get('exchange', 'name'), config.getboolean('exchange', 'durable'), - config.get('exchange', 'name'), config.get('exchange', 'type')) - sender = sess.sender(tgt) - session = sess - target = tgt - - return sender - -def _token_append(tokenlist, val): - # Replace any periods with underscores so we have a deterministic number of tokens - val = val.replace('.', '_') - tokenlist.append(val) - -def get_message_subject(msgtype, *args, **kws): - key = [config.get('topic', 'prefix'), msgtype] - - if msgtype == 'PackageListChange': - _token_append(key, kws['tag']['name']) - _token_append(key, kws['package']['name']) - elif msgtype == 'TaskStateChange': - _token_append(key, kws['info']['method']) - _token_append(key, kws['attribute']) - elif msgtype == 'BuildStateChange': - info = kws['info'] - _token_append(key, kws['attribute']) - _token_append(key, info['name']) - elif msgtype == 'Import': - _token_append(key, kws['type']) - elif msgtype in ('Tag', 'Untag'): - _token_append(key, kws['tag']['name']) - build = kws['build'] - _token_append(key, build['name']) - _token_append(key, kws['user']['name']) - elif msgtype == 'RepoInit': - _token_append(key, kws['tag']['name']) - elif msgtype == 'RepoDone': - _token_append(key, kws['repo']['tag_name']) - - key = '.'.join(key) - key = key[:MAX_KEY_LENGTH] - return key - -def get_message_headers(msgtype, *args, **kws): - headers = {'type': msgtype} - - if msgtype == 'PackageListChange': - headers['tag'] = kws['tag']['name'] - headers['package'] = kws['package']['name'] - elif msgtype == 'TaskStateChange': - headers['id'] = kws['info']['id'] - headers['parent'] = kws['info']['parent'] - headers['method'] = kws['info']['method'] - headers['attribute'] = kws['attribute'] - headers['old'] = kws['old'] - headers['new'] = kws['new'] - elif msgtype == 'BuildStateChange': - info = kws['info'] - headers['name'] = info['name'] - headers['version'] = info['version'] - headers['release'] = info['release'] - headers['attribute'] = kws['attribute'] - headers['old'] = kws['old'] - headers['new'] = kws['new'] - elif msgtype == 'Import': - headers['importType'] = kws['type'] - elif msgtype in ('Tag', 'Untag'): - headers['tag'] = kws['tag']['name'] - build = kws['build'] - headers['name'] = build['name'] - headers['version'] = build['version'] - headers['release'] = build['release'] - headers['user'] = kws['user']['name'] - elif msgtype == 'RepoInit': - headers['tag'] = kws['tag']['name'] - elif msgtype == 'RepoDone': - headers['tag'] = kws['repo']['tag_name'] - - return headers - -@callback(*[c for c in callbacks.keys() if c.startswith('post')]) -@ignore_error -def send_message(cbtype, *args, **kws): - global config - sender = get_sender() - if cbtype.startswith('post'): - msgtype = cbtype[4:] - else: - msgtype = cbtype[3:] - - data = kws.copy() - if args: - data['args'] = list(args) - - exchange_type = config.get('exchange', 'type') - if exchange_type == 'topic': - subject = get_message_subject(msgtype, *args, **kws) - message = qpid.messaging.Message(subject=subject, content=data) - elif exchange_type == 'headers': - headers = get_message_headers(msgtype, *args, **kws) - message = qpid.messaging.Message(properties=headers, content=data) - else: - raise koji.PluginError, 'unsupported exchange type: %s' % exchange_type - - sender.send(message, sync=True, timeout=config.getfloat('broker', 'timeout')) - sender.close(timeout=config.getfloat('broker', 'timeout')) diff --git a/plugins/rpm2maven.conf b/plugins/rpm2maven.conf deleted file mode 100644 index 900bf13..0000000 --- a/plugins/rpm2maven.conf +++ /dev/null @@ -1,5 +0,0 @@ -# config file for the Koji rpm2maven plugin - -[patterns] -rpm_names = *-repolib -artifact_paths = /usr/share/java/repository/maven2/* diff --git a/plugins/rpm2maven.py b/plugins/rpm2maven.py deleted file mode 100644 index 484a319..0000000 --- a/plugins/rpm2maven.py +++ /dev/null @@ -1,107 +0,0 @@ -# Koji callback for extracting Maven artifacts (.pom and .jar files) -# from an rpm and making them available via the Koji-managed Maven repo. -# Copyright (c) 2010-2014 Red Hat, Inc. -# -# Authors: -# Mike Bonnet - -import koji -from koji.context import context -from koji.plugin import callback -import ConfigParser -import fnmatch -import os -import shutil -import subprocess - -CONFIG_FILE = '/etc/koji-hub/plugins/rpm2maven.conf' - -config = None - -@callback('postImport') -def maven_import(cbtype, *args, **kws): - global config - if not context.opts.get('EnableMaven', False): - return - if kws.get('type') != 'rpm': - return - buildinfo = kws['build'] - rpminfo = kws['rpm'] - filepath = kws['filepath'] - - if not config: - config = ConfigParser.SafeConfigParser() - config.read(CONFIG_FILE) - name_patterns = config.get('patterns', 'rpm_names').split() - for pattern in name_patterns: - if fnmatch.fnmatch(rpminfo['name'], pattern): - break - else: - return - - tmpdir = os.path.join(koji.pathinfo.work(), 'rpm2maven', koji.buildLabel(buildinfo)) - try: - if os.path.exists(tmpdir): - shutil.rmtree(tmpdir) - koji.ensuredir(tmpdir) - expand_rpm(filepath, tmpdir) - scan_and_import(buildinfo, rpminfo, tmpdir) - finally: - if os.path.exists(tmpdir): - shutil.rmtree(tmpdir) - -def expand_rpm(filepath, tmpdir): - devnull = file('/dev/null', 'r+') - rpm2cpio = subprocess.Popen(['/usr/bin/rpm2cpio', filepath], - stdout=subprocess.PIPE, - stdin=devnull, stderr=devnull, - close_fds=True) - cpio = subprocess.Popen(['/bin/cpio', '-id'], - stdin=rpm2cpio.stdout, - cwd=tmpdir, - stdout=devnull, stderr=devnull, - close_fds=True) - if rpm2cpio.wait() != 0 or cpio.wait() != 0: - raise koji.CallbackError, 'error extracting files from %s, ' \ - 'rpm2cpio returned %s, cpio returned %s' % \ - (filepath, rpm2cpio.wait(), cpio.wait()) - devnull.close() - -def scan_and_import(buildinfo, rpminfo, tmpdir): - global config - path_patterns = config.get('patterns', 'artifact_paths').split() - - maven_archives = [] - for dirpath, dirnames, filenames in os.walk(tmpdir): - relpath = dirpath[len(tmpdir):] - for pattern in path_patterns: - if fnmatch.fnmatch(relpath, pattern): - break - else: - continue - - poms = [f for f in filenames if f.endswith('.pom')] - if len(poms) != 1: - continue - - pom_info = koji.parse_pom(os.path.join(dirpath, poms[0])) - maven_info = koji.pom_to_maven_info(pom_info) - maven_archives.append({'maven_info': maven_info, - 'files': [os.path.join(dirpath, f) for f in filenames]}) - - if not maven_archives: - return - - # We don't know which pom is the top-level pom, so we don't know what Maven - # metadata to associate with the build. So we make something up. - maven_build = {'group_id': buildinfo['name'], 'artifact_id': rpminfo['name'], - 'version': '%(version)s-%(release)s' % buildinfo} - context.handlers.call('host.createMavenBuild', buildinfo, maven_build) - - for entry in maven_archives: - maven_info = entry['maven_info'] - for filepath in entry['files']: - if not context.handlers.call('getArchiveType', filename=filepath): - # unsupported archive type, skip it - continue - context.handlers.call('host.importArchive', filepath, buildinfo, 'maven', maven_info) diff --git a/plugins/runroot.conf b/plugins/runroot.conf deleted file mode 100644 index d3d222b..0000000 --- a/plugins/runroot.conf +++ /dev/null @@ -1,25 +0,0 @@ -[paths] -; comma-delimited list of default mountpoints -; They will be mounted during each run. It is suggested, that these -; paths has readonly options and are made writable via extra_mounts -; parameter for individual calls. -; default_mounts = /mnt/archive,/mnt/workdir - -; comma-delimited list of safe roots. -; Each extra_mount need to start with some of these prefixes. Other paths are -; not allowed for mounting. Only absolute paths are allowed here, no -; wildcards. -; safe_roots = /mnt/workdir/tmp - -; path substitutions is tuple per line, delimited by comma, order is -; important. -; Path prefixes which can be substituted for other mountpoints. -; Usable for locations symlinked from other mounts. -; path_subs = /mnt/archive/prehistory/,/mnt/prehistoric_disk/archive/prehistory - -; mount origins, order is important here, ordered by best catch -; [path0] -; mountpoint = /mnt/archive -; path = archive.org:/vol/archive -; fstype = nfs -; options = ro,hard,intr,nosuid,nodev,noatime,tcp diff --git a/plugins/runroot.py b/plugins/runroot.py deleted file mode 100644 index 9a71c91..0000000 --- a/plugins/runroot.py +++ /dev/null @@ -1,322 +0,0 @@ -# kojid plugin - -import commands -import koji -import ConfigParser -import os -import platform -compat_mode = False -try: - import koji.tasks as tasks - from koji.tasks import scan_mounts - from koji.util import isSuccess as _isSuccess - from koji.util import parseStatus as _parseStatus - from koji.daemon import log_output - from __main__ import BuildRoot -except ImportError: - compat_mode = True - #old way - import tasks - #XXX - stuff we need from kojid - from __main__ import BuildRoot, log_output, scan_mounts, _isSuccess, _parseStatus - - -__all__ = ('RunRootTask',) - -CONFIG_FILE = '/etc/kojid/runroot.conf' - - -class RunRootTask(tasks.BaseTaskHandler): - - Methods = ['runroot'] - - _taskWeight = 2.0 - - def __init__(self, *args, **kwargs): - self._read_config() - return super(RunRootTask, self).__init__(*args, **kwargs) - - def _get_path_params(self, path, rw=False): - found = False - for mount_data in self.config['paths']: - if path.startswith(mount_data['mountpoint']): - found = True - break - if not found: - raise koji.GenericError("bad config: missing corresponding mountpoint") - options = [] - for o in mount_data['options'].split(','): - if rw and o == 'ro': - options.append('rw') - else: - options.append(o) - rel_path = path[len(mount_data['mountpoint']):] - rel_path = rel_path[1:] if rel_path.startswith('/') else rel_path - res = (os.path.join(mount_data['path'], rel_path), path, mount_data['fstype'], ','.join(options)) - return res - - def _read_config(self): - cp = ConfigParser.SafeConfigParser() - cp.read(CONFIG_FILE) - self.config = { - 'default_mounts': [], - 'safe_roots': [], - 'path_subs': [], - 'paths': [], - } - - if cp.has_option('paths', 'default_mounts'): - self.config['default_mounts'] = cp.get('paths', 'default_mounts').split(',') - if cp.has_option('paths', 'safe_roots'): - self.config['safe_roots'] = cp.get('paths', 'safe_roots').split(',') - if cp.has_option('paths', 'path_subs'): - self.config['path_subs'] = [x.split(',') for x in cp.get('paths', 'path_subs').split('\n')] - - count = 0 - while True: - section_name = 'path%d' % count - if not cp.has_section(section_name): - break - try: - self.config['paths'].append({ - 'mountpoint': cp.get(section_name, 'mountpoint'), - 'path': cp.get(section_name, 'path'), - 'fstype': cp.get(section_name, 'fstype'), - 'options': cp.get(section_name, 'options'), - }) - except ConfigParser.NoOptionError: - raise koji.GenericError("bad config: missing options in %s section" % section_name) - count += 1 - - for path in self.config['default_mounts'] + self.config['safe_roots'] + [x[0] for x in self.config['path_subs']]: - if not path.startswith('/'): - raise koji.GenericError("bad config: all paths (default_mounts, safe_roots, path_subs) needs to be absolute: %s" % path) - - def handler(self, root, arch, command, keep=False, packages=[], mounts=[], repo_id=None, skip_setarch=False, weight=None, upload_logs=None): - """Create a buildroot and run a command (as root) inside of it - - Command may be a string or a list. - - Returns a message indicating success if the command was successful, and - raises an error otherwise. Command output will be available in - runroot.log in the task output directory on the hub. - - skip_setarch is a rough approximation of an old hack - - the keep option is not used. keeping for compatibility for now... - - upload_logs is list of absolute paths which will be uploaded for - archiving on hub. It always consists of /tmp/runroot.log, but can be - used for additional logs (pungi.log, etc.) - """ - if weight is not None: - weight = max(weight, 0.5) - self.session.host.setTaskWeight(self.id, weight) - #noarch is funny - if arch == "noarch": - #we need a buildroot arch. Pick one that: - # a) this host can handle - # b) the build tag can support - # c) is canonical - host_arches = self.session.host.getHost()['arches'] - if not host_arches: - raise koji.BuildError, "No arch list for this host" - tag_arches = self.session.getBuildConfig(root)['arches'] - if not tag_arches: - raise koji.BuildError, "No arch list for tag: %s" % root - #index canonical host arches - host_arches = dict([(koji.canonArch(a),1) for a in host_arches.split()]) - #pick the first suitable match from tag's archlist - for br_arch in tag_arches.split(): - br_arch = koji.canonArch(br_arch) - if host_arches.has_key(br_arch): - #we're done - break - else: - #no overlap - raise koji.BuildError, "host does not match tag arches: %s (%s)" % (root, tag_arches) - else: - br_arch = arch - if repo_id: - repo_info = self.session.repoInfo(repo_id, strict=True) - if repo_info['tag_name'] != root: - raise koji.BuildError, "build tag (%s) does not match repo tag (%s)" % (root, repo_info['tag_name']) - if repo_info['state'] not in (koji.REPO_STATES['READY'], koji.REPO_STATES['EXPIRED']): - raise koji.BuildError, "repos in the %s state may not be used by runroot" % koji.REPO_STATES[repo_info['state']] - else: - repo_info = self.session.getRepo(root) - if not repo_info: - #wait for it - task_id = self.session.host.subtask(method='waitrepo', - arglist=[root, None, None], - parent=self.id) - repo_info = self.wait(task_id)[task_id] - if compat_mode: - broot = BuildRoot(root, br_arch, self.id, repo_id=repo_info['id']) - else: - broot = BuildRoot(self.session, self.options, root, br_arch, self.id, repo_id=repo_info['id']) - broot.workdir = self.workdir - broot.init() - rootdir = broot.rootdir() - #workaround for rpm oddness - os.system('rm -f "%s"/var/lib/rpm/__db.*' % rootdir) - #update buildroot state (so that updateBuildRootList() will work) - self.session.host.setBuildRootState(broot.id, 'BUILDING') - try: - if packages: - #pkglog = '%s/%s' % (broot.resultdir(), 'packages.log') - pkgcmd = ['--install'] + packages - status = broot.mock(pkgcmd) - self.session.host.updateBuildRootList(broot.id, broot.getPackageList()) - if not _isSuccess(status): - raise koji.BuildrootError, _parseStatus(status, pkgcmd) - - if isinstance(command, str): - cmdstr = command - else: - #we were passed an arglist - #we still have to run this through the shell (for redirection) - #but we can preserve the list structure precisely with careful escaping - cmdstr = ' '.join(["'%s'" % arg.replace("'", r"'\''") for arg in command]) - # A nasty hack to put command output into its own file until mock can be - # patched to do something more reasonable than stuff everything into build.log - cmdargs = ['/bin/sh', '-c', "{ %s; } < /dev/null 2>&1 | /usr/bin/tee /tmp/runroot.log; exit ${PIPESTATUS[0]}" % cmdstr] - - # always mount /mnt/redhat (read-only) - # always mount /mnt/iso (read-only) - # also need /dev bind mount - self.do_mounts(rootdir, [self._get_path_params(x) for x in self.config['default_mounts']]) - self.do_extra_mounts(rootdir, mounts) - mock_cmd = ['chroot'] - if skip_setarch: - #we can't really skip it, but we can set it to the current one instead of of the chroot one - myarch = platform.uname()[5] - mock_cmd.extend(['--arch', myarch]) - mock_cmd.append('--') - mock_cmd.extend(cmdargs) - rv = broot.mock(mock_cmd) - log_paths = ['/tmp/runroot.log'] - if upload_logs is not None: - log_paths += upload_logs - for log_path in log_paths: - self.uploadFile(rootdir + log_path) - finally: - # mock should umount its mounts, but it will not handle ours - self.undo_mounts(rootdir, fatal=False) - broot.expire() - if isinstance(command, str): - cmdlist = command.split() - else: - cmdlist = command - cmdlist = [param for param in cmdlist if '=' not in param] - if cmdlist: - cmd = os.path.basename(cmdlist[0]) - else: - cmd = '(none)' - if _isSuccess(rv): - return '%s completed successfully' % cmd - else: - raise koji.BuildrootError, _parseStatus(rv, cmd) - - def do_extra_mounts(self, rootdir, mounts): - mnts = [] - for mount in mounts: - mount = os.path.normpath(mount) - for safe_root in self.config['safe_roots']: - if mount.startswith(safe_root): - break - else: - #no match - raise koji.GenericError("read-write mount point is not safe: %s" % mount) - #normpath should have removed any .. dirs, but just in case... - if mount.find('/../') != -1: - raise koji.GenericError("read-write mount point is not safe: %s" % mount) - - for re, sub in self.config['path_subs']: - mount = mount.replace(re, sub) - - mnts.append(self._get_path_params(mount, rw=True)) - self.do_mounts(rootdir, mnts) - - def do_mounts(self, rootdir, mounts): - if not mounts: - return - self.logger.info('New runroot') - self.logger.info("Runroot mounts: %s" % mounts) - fn = '%s/tmp/runroot_mounts' % rootdir - fslog = file(fn, 'a') - logfile = "%s/do_mounts.log" % self.workdir - uploadpath = self.getUploadDir() - error = None - for dev,path,type,opts in mounts: - if not path.startswith('/'): - raise koji.GenericError("invalid mount point: %s" % path) - mpoint = "%s%s" % (rootdir,path) - if opts is None: - opts = [] - else: - opts = opts.split(',') - if 'bind' in opts: - #make sure dir exists - if not os.path.isdir(dev): - error = koji.GenericError("No such directory or mount: %s" % dev) - break - type = 'none' - if path is None: - #shorthand for "same path" - path = dev - if 'bg' in opts: - error = koji.GenericError("bad config: background mount not allowed") - break - opts = ','.join(opts) - cmd = ['mount', '-t', type, '-o', opts, dev, mpoint] - self.logger.info("Mount command: %r" % cmd) - koji.ensuredir(mpoint) - if compat_mode: - status = log_output(cmd[0], cmd, logfile, uploadpath, logerror=True, append=True) - else: - status = log_output(self.session, cmd[0], cmd, logfile, uploadpath, logerror=True, append=True) - if not _isSuccess(status): - error = koji.GenericError("Unable to mount %s: %s" \ - % (mpoint, _parseStatus(status, cmd))) - break - fslog.write("%s\n" % mpoint) - fslog.flush() - fslog.close() - if error is not None: - self.undo_mounts(rootdir, fatal=False) - raise error - - def undo_mounts(self, rootdir, fatal=True): - self.logger.debug("Unmounting runroot mounts") - mounts = {} - fn = '%s/tmp/runroot_mounts' % rootdir - if os.path.exists(fn): - fslog = file(fn,'r') - for line in fslog: - mounts.setdefault(line.strip(), 1) - fslog.close() - #also, check /proc/mounts just in case - for dir in scan_mounts(rootdir): - mounts.setdefault(dir, 1) - mounts = mounts.keys() - # deeper directories first - mounts.sort() - mounts.reverse() - failed = [] - self.logger.info("Unmounting (runroot): %s" % mounts) - for dir in mounts: - (rv, output) = commands.getstatusoutput("umount -l '%s'" % dir) - if rv != 0: - failed.append("%s: %s" % (dir, output)) - if failed: - msg = "Unable to unmount: %s" % ', '.join(failed) - self.logger.warn(msg) - if fatal: - raise koji.GenericError, msg - else: - # remove the mount list when everything is unmounted - try: - os.unlink(fn) - except OSError: - pass diff --git a/plugins/runroot_hub.py b/plugins/runroot_hub.py deleted file mode 100644 index a666919..0000000 --- a/plugins/runroot_hub.py +++ /dev/null @@ -1,61 +0,0 @@ -#koji hub plugin -# There is a kojid plugin that goes with this hub plugin. The kojid builder -# plugin has a config file. This hub plugin has no config file. - - -from koji.context import context -from koji.plugin import export -import koji -import random -import sys - -#XXX - have to import kojihub for mktask -sys.path.insert(0, '/usr/share/koji-hub/') -from kojihub import mktask, get_tag, get_all_arches - -__all__ = ('runroot',) - - -def get_channel_arches(channel): - """determine arches available in channel""" - chan = context.handlers.call('getChannel', channel, strict=True) - ret = {} - for host in context.handlers.call('listHosts', channelID=chan['id'], enabled=True): - for a in host['arches'].split(): - ret[koji.canonArch(a)] = 1 - return ret - -@export -def runroot(tagInfo, arch, command, channel=None, **opts): - """ Create a runroot task """ - context.session.assertPerm('runroot') - taskopts = { - 'priority': 15, - 'arch': arch, - } - - taskopts['channel'] = channel or 'runroot' - - if arch == 'noarch': - #not all arches can generate a proper buildroot for all tags - tag = get_tag(tagInfo) - if not tag['arches']: - raise koji.GenericError, 'no arches defined for tag %s' % tag['name'] - - #get all known arches for the system - fullarches = get_all_arches() - - tagarches = tag['arches'].split() - - # If our tag can't do all arches, then we need to - # specify one of the arches it can do. - if set(fullarches) - set(tagarches): - chanarches = get_channel_arches(taskopts['channel']) - choices = [x for x in tagarches if x in chanarches] - if not choices: - raise koji.GenericError, 'no common arches for tag/channel: %s/%s' \ - % (tagInfo, taskopts['channel']) - taskopts['arch'] = koji.canonArch(random.choice(choices)) - - return mktask(taskopts,'runroot', tagInfo, arch, command, **opts) - From 69e77dd693b22ae9489ddd56872a6ee29863a803 Mon Sep 17 00:00:00 2001 From: Ralph Bean Date: Apr 08 2016 16:10:59 +0000 Subject: [PATCH 2/3] Make the other makefile vars here explicit now. --- diff --git a/plugins/Makefile b/plugins/Makefile index f570286..dd59f76 100644 --- a/plugins/Makefile +++ b/plugins/Makefile @@ -1,11 +1,11 @@ PYTHON=python -PLUGINDIR = /usr/lib/koji-hub-plugins +HUBPLUGINDIR = /usr/lib/koji-hub-plugins BUILDERPLUGINDIR = /usr/lib/koji-builder-plugins -FILES = $(wildcard hub/*.py) +HUBFILES = $(wildcard hub/*.py) BUILDERFILES = $(wildcard builder/*.py) -CONFDIR = /etc/koji-hub/plugins +HUBCONFDIR = /etc/koji-hub/plugins BUILDERCONFDIR = /etc/kojid -CONFFILES = $(wildcard hub/*.conf) +HUBCONFFILES = $(wildcard hub/*.conf) BUILDERCONFFILES = $(wildcard builder/*.conf) _default: @@ -21,13 +21,13 @@ install: exit 1; \ fi - mkdir -p $(DESTDIR)/$(PLUGINDIR) + mkdir -p $(DESTDIR)/$(HUBPLUGINDIR) mkdir -p $(DESTDIR)/$(BUILDERPLUGINDIR) - install -p -m 644 $(FILES) $(DESTDIR)/$(PLUGINDIR) + install -p -m 644 $(HUBFILES) $(DESTDIR)/$(HUBPLUGINDIR) install -p -m 644 $(BUILDERFILES) $(DESTDIR)/$(BUILDERPLUGINDIR) - $(PYTHON) -c "import compileall; compileall.compile_dir('$(DESTDIR)/$(PLUGINDIR)', 1, '$(PLUGINDIR)', 1)" + $(PYTHON) -c "import compileall; compileall.compile_dir('$(DESTDIR)/$(HUBPLUGINDIR)', 1, '$(HUBPLUGINDIR)', 1)" $(PYTHON) -c "import compileall; compileall.compile_dir('$(DESTDIR)/$(BUILDERPLUGINDIR)', 1, '$(BUILDERPLUGINDIR)', 1)" - mkdir -p $(DESTDIR)/$(CONFDIR) + mkdir -p $(DESTDIR)/$(HUBCONFDIR) mkdir -p $(DESTDIR)/$(BUILDERCONFDIR) - install -p -m 644 $(CONFFILES) $(DESTDIR)/$(CONFDIR) + install -p -m 644 $(HUBCONFFILES) $(DESTDIR)/$(HUBCONFDIR) install -p -m 644 $(BUILDERCONFFILES) $(DESTDIR)/$(BUILDERCONFDIR) From 7fd256cde15ea00f6ddb4428d3ad7313d5ba27ba Mon Sep 17 00:00:00 2001 From: Ralph Bean Date: Apr 08 2016 16:15:07 +0000 Subject: [PATCH 3/3] Add subdirectory for builder plugin configuration. --- diff --git a/plugins/Makefile b/plugins/Makefile index dd59f76..d4b1861 100644 --- a/plugins/Makefile +++ b/plugins/Makefile @@ -4,7 +4,7 @@ BUILDERPLUGINDIR = /usr/lib/koji-builder-plugins HUBFILES = $(wildcard hub/*.py) BUILDERFILES = $(wildcard builder/*.py) HUBCONFDIR = /etc/koji-hub/plugins -BUILDERCONFDIR = /etc/kojid +BUILDERCONFDIR = /etc/kojid/plugins HUBCONFFILES = $(wildcard hub/*.conf) BUILDERCONFFILES = $(wildcard builder/*.conf) diff --git a/plugins/builder/runroot.py b/plugins/builder/runroot.py index 9a71c91..b303890 100644 --- a/plugins/builder/runroot.py +++ b/plugins/builder/runroot.py @@ -23,7 +23,7 @@ except ImportError: __all__ = ('RunRootTask',) -CONFIG_FILE = '/etc/kojid/runroot.conf' +CONFIG_FILE = '/etc/kojid/plugins/runroot.conf' class RunRootTask(tasks.BaseTaskHandler):