From d31306e2f673a89a7791cf209720f52318502816 Mon Sep 17 00:00:00 2001 From: Tomas Kopecek Date: Nov 18 2021 10:12:48 +0000 Subject: [PATCH 1/15] basic kiwi support --- diff --git a/plugins/builder/kiwi.py b/plugins/builder/kiwi.py new file mode 100644 index 0000000..42f55fd --- /dev/null +++ b/plugins/builder/kiwi.py @@ -0,0 +1,387 @@ +import glob +import pickle +import os +import xml.dom.minidom +from fnmatch import fnmatch + +import koji +from koji.util import joinpath, to_list +from koji.tasks import ServerExit +from __main__ import BaseBuildTask, BuildImageTask, BuildRoot, SCM + + +class KiwiBuildTask(BuildImageTask): + Methods = ['kiwiBuild'] + _taskWeight = 4.0 + + def get_nvr(self, desc_path): + # TODO: update release in desc + kiwi_files = glob.glob('%s/*.kiwi' % desc_path) + if len(kiwi_files) != 1: + raise koji.GenericError("Repo must contain only one .kiwi file.") + + cfg = kiwi_files[0] + + newxml = xml.dom.minidom.parse(cfg) + image = newxml.getElementsByTagName('image')[0] + + name = image.getAttribute('name') + version = None + release = None + for preferences in image.getElementsByTagName('preferences'): + try: + version = preferences.getElementsByTagName('version')[0].childNodes[0].data + except Exception: + pass + try: + release = preferences.getElementsByTagName('release')[0].childNodes[0].data + except Exception: + release = None + if not version: + raise koji.BuildError("Description file doesn't contain preferences/version") + return name, version, release + + def handler(self, target, arches, desc_url, desc_path, opts=None): + target_info = self.session.getBuildTarget(target, strict=True) + build_tag = target_info['build_tag'] + repo_info = self.getRepo(build_tag) + # check requested arches against build tag + buildconfig = self.session.getBuildConfig(build_tag) + if not buildconfig['arches']: + raise koji.BuildError("No arches for tag %(name)s [%(id)s]" % buildconfig) + tag_archlist = [koji.canonArch(a) for a in buildconfig['arches'].split()] + if arches: + for arch in arches: + if koji.canonArch(arch) not in tag_archlist: + raise koji.BuildError("Invalid arch for build tag: %s" % arch) + else: + arches = tag_archlist + + if not opts: + opts = {} + if not opts.get('scratch'): + opts['scratch'] = False + if not opts.get('optional_arches'): + opts['optional_arches'] = [] + self.opts = opts + + # get configuration + scm = SCM(desc_url) + scm.assert_allowed(allowed=self.options.allowed_scms, + session=self.session, + by_config=self.options.allowed_scms_use_config, + by_policy=self.options.allowed_scms_use_policy, + policy_data={ + 'user_id': self.taskinfo['owner'], + 'channel': self.session.getChannel(self.taskinfo['channel_id'], + strict=True)['name'], + 'scratch': opts['scratch'], + }) + logfile = os.path.join(self.workdir, 'checkout.log') + self.run_callbacks('preSCMCheckout', scminfo=scm.get_info(), + build_tag=build_tag, scratch=opts['scratch']) + scmdir = self.workdir + koji.ensuredir(scmdir) + scmsrcdir = scm.checkout(scmdir, self.session, + self.getUploadDir(), logfile) + self.run_callbacks("postSCMCheckout", + scminfo=scm.get_info(), + build_tag=build_tag, + scratch=opts['scratch'], + srcdir=scmsrcdir) + + path = os.path.join(scmsrcdir, desc_path) + name, version, release = self.get_nvr(path) + + bld_info = {} + if not opts['scratch']: + bld_info = self.initImageBuild(name, version, release, target_info, opts) + release = bld_info['release'] + elif not release: + release = self.session.getNextRelease({'name': name, 'version': version}) + + try: + subtasks = {} + canfail = [] + self.logger.debug("Spawning jobs for image arches: %r" % (arches)) + for arch in arches: + subtasks[arch] = self.session.host.subtask( + method='createKiwiImage', + arglist=[name, version, release, arch, + target_info, build_tag, repo_info, + desc_url, desc_path, opts], + label=arch, parent=self.id, arch=arch) + if arch in self.opts['optional_arches']: + canfail.append(subtasks[arch]) + self.logger.debug("Got image subtasks: %r" % (subtasks)) + self.logger.debug("Waiting on image subtasks (%s can fail)..." % canfail) + results = self.wait(to_list(subtasks.values()), all=True, + failany=True, canfail=canfail) + + # if everything failed, fail even if all subtasks are in canfail + self.logger.debug('subtask results: %r', results) + all_failed = True + for result in results.values(): + if not isinstance(result, dict) or 'faultCode' not in result: + all_failed = False + break + if all_failed: + raise koji.GenericError("all subtasks failed") + + # determine ignored arch failures + ignored_arches = set() + for arch in arches: + if arch in self.opts['optional_arches']: + task_id = subtasks[arch] + result = results[task_id] + if isinstance(result, dict) and 'faultCode' in result: + ignored_arches.add(arch) + + self.logger.debug('Image Results for hub: %s' % results) + results = dict([(str(k), v) for k, v in results.items()]) + if opts['scratch']: + self.session.host.moveImageBuildToScratch(self.id, results) + else: + self.session.host.completeImageBuild(self.id, bld_info['id'], results) + except (SystemExit, ServerExit, KeyboardInterrupt): + # we do not trap these + raise + except Exception: + if not opts['scratch']: + if bld_info: + self.session.host.failBuild(self.id, bld_info['id']) + raise + + # tag it + if not opts['scratch'] and not opts.get('skip_tag'): + tag_task_id = self.session.host.subtask(method='tagBuild', + arglist=[target_info['dest_tag'], + bld_info['id'], False, None, True], + label='tag', parent=self.id, arch='noarch') + self.wait(tag_task_id) + + # report results + report = '' + if opts['scratch']: + respath = ', '.join( + [os.path.join(koji.pathinfo.work(), + koji.pathinfo.taskrelpath(tid)) for tid in subtasks.values()]) + report += 'Scratch ' + else: + respath = koji.pathinfo.imagebuild(bld_info) + report += 'image build results in: %s' % respath + return report + + +class KiwiCreateImageTask(BaseBuildTask): + Methods = ['createKiwiImage'] + _taskWeight = 2.0 + + def prepareDescription(self, desc_path, release, repos): + # TODO: update release in desc + kiwi_files = glob.glob('%s/*.kiwi' % desc_path) + if len(kiwi_files) != 1: + raise koji.GenericError("Repo must contain only one .kiwi file.") + + cfg = kiwi_files[0] + + newxml = xml.dom.minidom.parse(cfg) + image = newxml.getElementsByTagName('image')[0] + + # remove old repos + for old_repo in image.getElementsByTagName('repository'): + image.removeChild(old_repo) + + # add new ones + for repo in sorted(set(repos)): + repo_node = newxml.createElement('repository') + repo_node.setAttribute('type', 'rpm-md') + source = newxml.createElement('source') + source.setAttribute('path', repo) + repo_node.appendChild(source) + image.appendChild(repo_node) + + # TODO: release is part of version (major.minor.release) + # preferences = image.getElementsByTagName('preferences')[0] + # try: + # preferences.getElementsByTagName('release')[0].childNodes[0].data = release + # except Exception: + # rel_node = newxml.createElement('release') + # rel_node.data = release + # preferences.appendChild(rel_node) + + types = [] + for pref in image.getElementsByTagName('preferences'): + for type in pref.getElementsByTagName('type'): + # TODO: if type.getAttribute('primary') == 'true': + types.append(type.getAttribute('image')) + + # write file back + with open(cfg, 'wt') as f: + f.write(newxml.toprettyxml()) + + return cfg, types + + def getImagePackagesFromCache(self, cachepath): + """ + Read RPM header information from the yum cache available in the + given path. Returns a list of dictionaries for each RPM included. + """ + found = False + hdrlist = {} + fields = ['name', 'version', 'release', 'epoch', 'arch', + 'buildtime', 'sigmd5'] + for root, dirs, files in os.walk(cachepath): + for f in files: + if fnmatch(f, '*.rpm'): + pkgfile = os.path.join(root, f) + hdr = koji.get_header_fields(pkgfile, fields) + hdr['size'] = os.path.getsize(pkgfile) + hdr['payloadhash'] = koji.hex_string(hdr['sigmd5']) + del hdr['sigmd5'] + hdrlist[os.path.basename(pkgfile)] = hdr + found = True + if not found: + raise koji.LiveCDError('No repos found in yum cache!') + return list(hdrlist.values()) + + def getImagePackages(self, result): + """Proper handler for getting rpminfo from result list, + it need result list to contain payloadhash, etc. to work correctly""" + hdrlist = [] + for line in open(result, 'rt'): + line = line.strip() + name, epoch, version, release, arch, disturl, license = line.split('|') + try: + # "(none)" for None epochs + epoch = int(epoch) + except ValueError: + epoch = None + hdrlist.append({ + 'name': name, + 'epoch': epoch, + 'version': version, + 'release': release, + 'arch': arch, + 'payloadhash': '', + 'size': 0, + 'buildtime': 0, + }) + + return hdrlist + + def handler(self, name, version, release, arch, + target_info, build_tag, repo_info, + desc_url, desc_path, opts=None): + self.opts = opts + build_tag = target_info['build_tag'] + broot = BuildRoot(self.session, self.options, + tag=build_tag, + arch=arch, + task_id=self.id, + repo_id=repo_info['id'], + install_group='kiwi', + setup_dns=True, + bind_opts={'dirs': {'/dev': '/dev', }}) + broot.workdir = self.workdir + + # create the mock chroot + self.logger.debug("Initializing kiwi buildroot") + broot.init() + self.logger.debug("Kiwi buildroot ready: " + broot.rootdir()) + + # get configuration + scm = SCM(desc_url) + scm.assert_allowed(allowed=self.options.allowed_scms, + session=self.session, + by_config=self.options.allowed_scms_use_config, + by_policy=self.options.allowed_scms_use_policy, + policy_data={ + 'user_id': self.taskinfo['owner'], + 'channel': self.session.getChannel(self.taskinfo['channel_id'], + strict=True)['name'], + 'scratch': self.opts.get('scratch') + }) + logfile = os.path.join(self.workdir, 'checkout-%s.log' % arch) + self.run_callbacks('preSCMCheckout', scminfo=scm.get_info(), + build_tag=build_tag, scratch=self.opts.get('scratch')) + scmdir = broot.tmpdir() + koji.ensuredir(scmdir) + scmsrcdir = scm.checkout(scmdir, self.session, + self.getUploadDir(), logfile) + self.run_callbacks("postSCMCheckout", + scminfo=scm.get_info(), + build_tag=build_tag, + scratch=self.opts.get('scratch'), + srcdir=scmsrcdir) + + # user repos + repos = self.opts.get('repos', []) + # buildroot repo + path_info = koji.PathInfo(topdir=self.options.topurl) + repopath = path_info.repo(repo_info['id'], target_info['build_tag_name']) + baseurl = '%s/%s' % (repopath, arch) + self.logger.debug('BASEURL: %s' % baseurl) + repos.append(baseurl) + + path = os.path.join(scmsrcdir, desc_path) + desc, types = self.prepareDescription(path, release, repos) + self.uploadFile(desc) + + cmd = ['kiwi-ng'] + if self.opts.get('profile'): + cmd.extend(['--profile', self.opts['profile']]) + target_dir = '/builddir/result/image' + cmd.extend([ + 'system', 'build', + '--description', os.path.join(os.path.basename(scmsrcdir), desc_path), + '--target-dir', target_dir, + ]) + rv = broot.mock(['--cwd', broot.tmpdir(within=True), '--chroot', '--'] + cmd) + if rv: + raise koji.GenericError("Kiwi failed") + + result = pickle.load(open(joinpath(broot.rootdir(), target_dir[1:], 'kiwi.result'), 'rb')) + + imgdata = { + 'arch': arch, + 'task_id': self.id, + 'logs': [ + os.path.basename(desc) + ], + 'name': name, + 'version': version, + 'release': release, + 'rpmlist': [], + 'files': [], + } + # TODO: upload detailed log? + # build/image-root.log + # os.path.join(broot.tmpdir(), target_dir[1:], "build/image-root.log") + + # for type in types: + # img_file = '%s.%s-%s.%s' % (name, version, arch, type) + # self.uploadFile(os.path.join(broot.rootdir()), remoteName=img_file) + # imgdata['files'].append(img_file) + fpath = os.path.join(broot.rootdir(), + result.result_files['disk_format_image'].filename[1:]) + img_file = os.path.basename(fpath) + self.uploadFile(fpath, remoteName=os.path.basename(img_file)) + imgdata['files'].append(img_file) + + if not self.opts.get('scratch'): + if False: + # should be used after kiwi update + fpath = os.path.join(broot.rootdir(), + result.result_files['image_packages'].filename[1:]) + hdrlist = self.getImagePackages(fpath) + else: + cachepath = os.path.join(broot.rootdir(), 'var/cache/kiwi/dnf') + hdrlist = self.getImagePackagesFromCache(cachepath) + broot.markExternalRPMs(hdrlist) + imgdata['rpmlist'] = hdrlist + + broot.expire() + + self.logger.error("Uploading image data: %s", imgdata) + return imgdata diff --git a/plugins/cli/kiwi.py b/plugins/cli/kiwi.py new file mode 100644 index 0000000..8076124 --- /dev/null +++ b/plugins/cli/kiwi.py @@ -0,0 +1,75 @@ +import os +from optparse import OptionParser + +from koji import canonArch + +from koji.plugin import export_cli +from koji_cli.lib import ( + _, + _running_in_bg, + activate_session, + watch_tasks, +) + + +@export_cli +def handle_kiwi_build(goptions, session, args): + "[build] Run a command in a buildroot" + usage = _("usage: %prog kiwi-build [options] " + " ") + usage += _("\n(Specify the --help global option for a list of other help options)") + parser = OptionParser(usage=usage) + parser.add_option("--scratch", action="store_true", default=False, + help=_("Perform a scratch build")) + parser.add_option("--repo", action="append", + help=_("Specify a repo that will override the repo used to install " + "RPMs in the image. May be used multiple times. The " + "build tag repo associated with the target is the default.")) + parser.add_option("--noprogress", action="store_true", + help=_("Do not display progress of the upload")) + parser.add_option("--kiwi-profile", action="store", default=None, + help=_("Select profile from description file")) + parser.add_option("--can-fail", action="store", dest="optional_arches", + metavar="ARCH1,ARCH2,...", default="", + help=_("List of archs which are not blocking for build " + "(separated by commas.")) + parser.add_option("--arch", action="append", dest="arches", default=[], + help=_("Limit arches to this subset")) + parser.add_option("--nowait", action="store_false", dest="wait", default=True) + parser.add_option("--wait", action="store_true", + help=_("Wait on the image creation, even if running in the background")) + (options, args) = parser.parse_args(args) + + if len(args) != 3: + parser.error(_("Incorrect number of arguments")) + assert False # pragma: no cover + target, scm, path = args + + activate_session(session, goptions) + + kwargs = { + 'scratch': options.scratch, + 'optional_arches': [canonArch(arch) + for arch in options.optional_arches.split(',') + if arch], + 'profile': options.kiwi_profile, + } + + arches = [] + if options.arches: + arches = [canonArch(arch) for arch in options.arches] + + task_id = session.kiwiBuild( + target=target, + arches=arches, + desc_url=scm, + desc_path=path, + **kwargs) + + if not goptions.quiet: + print("Created task: %d" % task_id) + print("Task info: %s/taskinfo?taskID=%s" % (goptions.weburl, task_id)) + if options.wait or (options.wait is None and not _running_in_bg()): + session.logout() + return watch_tasks(session, [task_id], quiet=goptions.quiet, + poll_interval=goptions.poll_interval, topurl=goptions.topurl) diff --git a/plugins/hub/kiwi.py b/plugins/hub/kiwi.py new file mode 100644 index 0000000..911612f --- /dev/null +++ b/plugins/hub/kiwi.py @@ -0,0 +1,38 @@ +import koji +import koji.tasks +import kojihub + +from koji.context import context +from koji.plugin import export + +koji.tasks.LEGACY_SIGNATURES['kiwiBuild'] = [ + [['target', 'arches', 'desc_url', 'desc_path', 'opts'], + None, None, (None,)]] +koji.tasks.LEGACY_SIGNATURES['createKiwiImage'] = [ + [['name', 'version', 'release', 'arch', + 'target_info', 'build_tag', 'repo_info', 'desc_url', 'opts'], + None, None, (None,)]] + + +@export +def kiwiBuild(target, arches, desc_url, desc_path, optional_arches=None, profile=None, + scratch=False, priority=None): + context.session.assertPerm('image') + taskOpts = { + 'channel': 'image', + } + if priority: + if priority < 0: + if not context.session.hasPerm('admin'): + raise koji.ActionNotAllowed( + 'only admins may create high-priority tasks') + taskOpts['priority'] = koji.PRIO_DEFAULT + priority + + opts = { + 'optional_arches': optional_arches, + 'profile': profile, + 'scratch': scratch, + } + return kojihub.make_task('kiwiBuild', + [target, arches, desc_url, desc_path, opts], + **taskOpts) From 86ee135dd182d9354a91ebcf0abb635d8688cac5 Mon Sep 17 00:00:00 2001 From: Tomas Kopecek Date: Nov 18 2021 10:12:48 +0000 Subject: [PATCH 2/15] json results --- diff --git a/plugins/builder/kiwi.py b/plugins/builder/kiwi.py index 42f55fd..3cfe632 100644 --- a/plugins/builder/kiwi.py +++ b/plugins/builder/kiwi.py @@ -1,4 +1,5 @@ import glob +#import json import pickle import os import xml.dom.minidom @@ -342,6 +343,7 @@ class KiwiCreateImageTask(BaseBuildTask): raise koji.GenericError("Kiwi failed") result = pickle.load(open(joinpath(broot.rootdir(), target_dir[1:], 'kiwi.result'), 'rb')) + #result = json.load(open(joinpath(broot.rootdir(), target_dir[1:], 'kiwi.result'), 'rb')) imgdata = { 'arch': arch, From d07c0d400eef60aa8b8c28ede7bf9d34fb092879 Mon Sep 17 00:00:00 2001 From: Tomas Kopecek Date: Nov 18 2021 10:12:48 +0000 Subject: [PATCH 3/15] basic kiwi docs --- diff --git a/docs/source/plugins.rst b/docs/source/plugins.rst index 1ec332f..4e4d999 100644 --- a/docs/source/plugins.rst +++ b/docs/source/plugins.rst @@ -226,3 +226,41 @@ sent on the bus (e.g. if the amqps server is offline). Admins should consider the balance between the ``batch_size`` and ``extra_limit`` options, as both can affect the total amount of data that the plugin could attempt to send during a single call. + + +Kiwi images +=========== + +**This is just a tech-preview. API/usage can drastically change in upcoming +releases** + +Plugin for creating images via `kiwi `_ +project. + +All three parts (cli/hub/builder) needs to be installed. There is currently no +configuration except allowing the plugins. + +Builders have to be part of ``image`` channel and don't need to have any +specific library installed (kiwi invocation/usage is only in buildroots not on +builder itself). + +Buildtag needs to be configured by adding special group ``kiwi`` which should +contain at least ``kiwi-cli``, potentially ``jing`` for better description files +validation and any ``kiwi-systemdeps-*`` packages for requested image types. So, +most simple configuration will look like: + +.. code-block:: shell + + $ koji add-group kiwi-build-tag kiwi + $ koji add-group-pkg kiwi-build-tag kiwi-cli + +Calling the build itself is a matter of simple CLI call: + +.. code-block: shell + + $ koji kiwi-build kiwi-target git+https://my.git/image-descriptions#master my_image_path + +Selecting other than default kiwi profile can be done by ``--kiwi-profile`` +option. Similarly to other image tasks, alternative architecture failures can be +ignored for successful build by ``--can-fail`` option. ``--arch`` can be used to +limit build tag architectures. From 8c1a7337abc8476af40ccc412db908a1941cdc71 Mon Sep 17 00:00:00 2001 From: Tomas Kopecek Date: Nov 18 2021 10:12:48 +0000 Subject: [PATCH 4/15] kiwi: check include directive --- diff --git a/plugins/builder/kiwi.py b/plugins/builder/kiwi.py index 3cfe632..3403423 100644 --- a/plugins/builder/kiwi.py +++ b/plugins/builder/kiwi.py @@ -189,11 +189,22 @@ class KiwiCreateImageTask(BaseBuildTask): newxml = xml.dom.minidom.parse(cfg) image = newxml.getElementsByTagName('image')[0] - # remove old repos + # apply includes - kiwi can include only top-level nodes, so we can simply + # go through "include" elements and replace them with referred content (without + # doing it recursively) + for inc_node in image.getElementsByTagName('include'): + path = inc_node.getAttribute('from') + inc = xml.dom.minidom.parse(path) + # every included xml has image root element again + for node in inc.getElementsByTagName('image').childNodes: + if node.nodeName != 'repository': + image.appendChild(node) + + # remove remaining old repos for old_repo in image.getElementsByTagName('repository'): image.removeChild(old_repo) - # add new ones + # add koji ones for repo in sorted(set(repos)): repo_node = newxml.createElement('repository') repo_node.setAttribute('type', 'rpm-md') @@ -220,7 +231,7 @@ class KiwiCreateImageTask(BaseBuildTask): # write file back with open(cfg, 'wt') as f: f.write(newxml.toprettyxml()) - + return cfg, types def getImagePackagesFromCache(self, cachepath): From 3c15c7a34f7db08b2a4e5c445e2bb39a10f7b558 Mon Sep 17 00:00:00 2001 From: Tomas Kopecek Date: Nov 18 2021 10:12:48 +0000 Subject: [PATCH 5/15] update docs --- diff --git a/docs/source/plugins.rst b/docs/source/plugins.rst index 4e4d999..39de555 100644 --- a/docs/source/plugins.rst +++ b/docs/source/plugins.rst @@ -228,17 +228,17 @@ Admins should consider the balance between the ``batch_size`` and plugin could attempt to send during a single call. -Kiwi images -=========== +Image builds using Kiwi +======================= **This is just a tech-preview. API/usage can drastically change in upcoming releases** Plugin for creating images via `kiwi `_ -project. +project. Minimal supported version of kiwi is ``kiwi-9.24.2``. All three parts (cli/hub/builder) needs to be installed. There is currently no -configuration except allowing the plugins. +configuration except allowing the plugins (name is 'kiwi' for all components). Builders have to be part of ``image`` channel and don't need to have any specific library installed (kiwi invocation/usage is only in buildroots not on From 946a24cca8dd5619ac948a61bc2b060dec37c8fc Mon Sep 17 00:00:00 2001 From: Tomas Kopecek Date: Nov 18 2021 10:12:48 +0000 Subject: [PATCH 6/15] remove translation --- diff --git a/plugins/cli/kiwi.py b/plugins/cli/kiwi.py index 8076124..0de6559 100644 --- a/plugins/cli/kiwi.py +++ b/plugins/cli/kiwi.py @@ -5,7 +5,6 @@ from koji import canonArch from koji.plugin import export_cli from koji_cli.lib import ( - _, _running_in_bg, activate_session, watch_tasks, @@ -15,33 +14,32 @@ from koji_cli.lib import ( @export_cli def handle_kiwi_build(goptions, session, args): "[build] Run a command in a buildroot" - usage = _("usage: %prog kiwi-build [options] " - " ") - usage += _("\n(Specify the --help global option for a list of other help options)") + usage = "usage: %prog kiwi-build [options] " + usage += "\n(Specify the --help global option for a list of other help options)" parser = OptionParser(usage=usage) parser.add_option("--scratch", action="store_true", default=False, - help=_("Perform a scratch build")) + help="Perform a scratch build") parser.add_option("--repo", action="append", - help=_("Specify a repo that will override the repo used to install " - "RPMs in the image. May be used multiple times. The " - "build tag repo associated with the target is the default.")) + help="Specify a repo that will override the repo used to install " + "RPMs in the image. May be used multiple times. The " + "build tag repo associated with the target is the default.") parser.add_option("--noprogress", action="store_true", - help=_("Do not display progress of the upload")) + help="Do not display progress of the upload") parser.add_option("--kiwi-profile", action="store", default=None, - help=_("Select profile from description file")) + help="Select profile from description file") parser.add_option("--can-fail", action="store", dest="optional_arches", metavar="ARCH1,ARCH2,...", default="", - help=_("List of archs which are not blocking for build " - "(separated by commas.")) + help="List of archs which are not blocking for build " + "(separated by commas.") parser.add_option("--arch", action="append", dest="arches", default=[], - help=_("Limit arches to this subset")) + help="Limit arches to this subset") parser.add_option("--nowait", action="store_false", dest="wait", default=True) parser.add_option("--wait", action="store_true", - help=_("Wait on the image creation, even if running in the background")) + help="Wait on the image creation, even if running in the background") (options, args) = parser.parse_args(args) if len(args) != 3: - parser.error(_("Incorrect number of arguments")) + parser.error("Incorrect number of arguments") assert False # pragma: no cover target, scm, path = args From 86c8fdc17720eccf915c4031cc349bcbfa0e0d04 Mon Sep 17 00:00:00 2001 From: Tomas Kopecek Date: Nov 18 2021 10:12:48 +0000 Subject: [PATCH 7/15] use json results --- diff --git a/plugins/builder/kiwi.py b/plugins/builder/kiwi.py index 3403423..58f25d1 100644 --- a/plugins/builder/kiwi.py +++ b/plugins/builder/kiwi.py @@ -1,6 +1,5 @@ import glob -#import json -import pickle +import json import os import xml.dom.minidom from fnmatch import fnmatch @@ -353,8 +352,7 @@ class KiwiCreateImageTask(BaseBuildTask): if rv: raise koji.GenericError("Kiwi failed") - result = pickle.load(open(joinpath(broot.rootdir(), target_dir[1:], 'kiwi.result'), 'rb')) - #result = json.load(open(joinpath(broot.rootdir(), target_dir[1:], 'kiwi.result'), 'rb')) + result = json.load(open(joinpath(broot.rootdir(), target_dir[1:], 'kiwi.result'), 'rb')) imgdata = { 'arch': arch, @@ -377,7 +375,7 @@ class KiwiCreateImageTask(BaseBuildTask): # self.uploadFile(os.path.join(broot.rootdir()), remoteName=img_file) # imgdata['files'].append(img_file) fpath = os.path.join(broot.rootdir(), - result.result_files['disk_format_image'].filename[1:]) + result['result_files']['disk_format_image'].filename[1:]) img_file = os.path.basename(fpath) self.uploadFile(fpath, remoteName=os.path.basename(img_file)) imgdata['files'].append(img_file) @@ -386,7 +384,7 @@ class KiwiCreateImageTask(BaseBuildTask): if False: # should be used after kiwi update fpath = os.path.join(broot.rootdir(), - result.result_files['image_packages'].filename[1:]) + result['result_files']['image_packages'].filename[1:]) hdrlist = self.getImagePackages(fpath) else: cachepath = os.path.join(broot.rootdir(), 'var/cache/kiwi/dnf') diff --git a/plugins/cli/kiwi.py b/plugins/cli/kiwi.py index 0de6559..c2fb6af 100644 --- a/plugins/cli/kiwi.py +++ b/plugins/cli/kiwi.py @@ -1,4 +1,3 @@ -import os from optparse import OptionParser from koji import canonArch From 336573c0fd430d37b87d99a7bc92dc02973e6a9c Mon Sep 17 00:00:00 2001 From: Tomas Kopecek Date: Nov 18 2021 10:12:48 +0000 Subject: [PATCH 8/15] code simplification --- diff --git a/plugins/builder/kiwi.py b/plugins/builder/kiwi.py index 58f25d1..7310a28 100644 --- a/plugins/builder/kiwi.py +++ b/plugins/builder/kiwi.py @@ -138,7 +138,7 @@ class KiwiBuildTask(BuildImageTask): ignored_arches.add(arch) self.logger.debug('Image Results for hub: %s' % results) - results = dict([(str(k), v) for k, v in results.items()]) + results = {str(k): v for k, v in results.items()} if opts['scratch']: self.session.host.moveImageBuildToScratch(self.id, results) else: @@ -263,11 +263,10 @@ class KiwiCreateImageTask(BaseBuildTask): for line in open(result, 'rt'): line = line.strip() name, epoch, version, release, arch, disturl, license = line.split('|') - try: - # "(none)" for None epochs - epoch = int(epoch) - except ValueError: + if epoch == '(none)': epoch = None + else: + epoch = int(epoch) hdrlist.append({ 'name': name, 'epoch': epoch, From 2d6b03eebaa1a9aeb7a0150f3a85e40a2fca380b Mon Sep 17 00:00:00 2001 From: Tomas Kopecek Date: Nov 18 2021 10:12:48 +0000 Subject: [PATCH 9/15] fix signature --- diff --git a/plugins/hub/kiwi.py b/plugins/hub/kiwi.py index 911612f..be51bb0 100644 --- a/plugins/hub/kiwi.py +++ b/plugins/hub/kiwi.py @@ -10,7 +10,7 @@ koji.tasks.LEGACY_SIGNATURES['kiwiBuild'] = [ None, None, (None,)]] koji.tasks.LEGACY_SIGNATURES['createKiwiImage'] = [ [['name', 'version', 'release', 'arch', - 'target_info', 'build_tag', 'repo_info', 'desc_url', 'opts'], + 'target_info', 'build_tag', 'repo_info', 'desc_url', 'desc_path', 'opts'], None, None, (None,)]] From 2d6a14845f8ac8207697f3969aeb90e0508ddc93 Mon Sep 17 00:00:00 2001 From: Tomas Kopecek Date: Nov 18 2021 10:12:48 +0000 Subject: [PATCH 10/15] alter name by selected profile --- diff --git a/plugins/builder/kiwi.py b/plugins/builder/kiwi.py index 7310a28..3f06163 100644 --- a/plugins/builder/kiwi.py +++ b/plugins/builder/kiwi.py @@ -14,7 +14,7 @@ class KiwiBuildTask(BuildImageTask): Methods = ['kiwiBuild'] _taskWeight = 4.0 - def get_nvr(self, desc_path): + def get_nvrp(self, desc_path): # TODO: update release in desc kiwi_files = glob.glob('%s/*.kiwi' % desc_path) if len(kiwi_files) != 1: @@ -37,9 +37,17 @@ class KiwiBuildTask(BuildImageTask): release = preferences.getElementsByTagName('release')[0].childNodes[0].data except Exception: release = None + profile = None + try: + for p in image.getElementsByTagName('profiles')[0].getElementsByTagName('profile'): + if p.getAttribute('image') == 'true': + profile = p.getAttribute('name') + except IndexError: + # missing profiles section + pass if not version: raise koji.BuildError("Description file doesn't contain preferences/version") - return name, version, release + return name, version, release, profile def handler(self, target, arches, desc_url, desc_path, opts=None): target_info = self.session.getBuildTarget(target, strict=True) @@ -91,7 +99,12 @@ class KiwiBuildTask(BuildImageTask): srcdir=scmsrcdir) path = os.path.join(scmsrcdir, desc_path) - name, version, release = self.get_nvr(path) + + name, version, release, default_profile = self.get_nvrp(path) + if opts.get('profile') or default_profile: + # package name is a combination of name + profile + # in case profiles are not used, let's use the standalone name + name = "%s-%s" % (name, opts.get('profile', default_profile)) bld_info = {} if not opts['scratch']: @@ -177,7 +190,7 @@ class KiwiCreateImageTask(BaseBuildTask): Methods = ['createKiwiImage'] _taskWeight = 2.0 - def prepareDescription(self, desc_path, release, repos): + def prepareDescription(self, desc_path, name, release, repos): # TODO: update release in desc kiwi_files = glob.glob('%s/*.kiwi' % desc_path) if len(kiwi_files) != 1: @@ -212,6 +225,7 @@ class KiwiCreateImageTask(BaseBuildTask): repo_node.appendChild(source) image.appendChild(repo_node) + image.setAttribute('name', name) # TODO: release is part of version (major.minor.release) # preferences = image.getElementsByTagName('preferences')[0] # try: @@ -229,7 +243,10 @@ class KiwiCreateImageTask(BaseBuildTask): # write file back with open(cfg, 'wt') as f: - f.write(newxml.toprettyxml()) + s = newxml.toprettyxml() + # toprettyxml adds too many whitespaces/newlines + s = '\n'.join([x for x in s.splitlines() if x.strip()]) + f.write(s) return cfg, types @@ -335,7 +352,7 @@ class KiwiCreateImageTask(BaseBuildTask): repos.append(baseurl) path = os.path.join(scmsrcdir, desc_path) - desc, types = self.prepareDescription(path, release, repos) + desc, types = self.prepareDescription(path, name, release, repos) self.uploadFile(desc) cmd = ['kiwi-ng'] @@ -351,13 +368,15 @@ class KiwiCreateImageTask(BaseBuildTask): if rv: raise koji.GenericError("Kiwi failed") - result = json.load(open(joinpath(broot.rootdir(), target_dir[1:], 'kiwi.result'), 'rb')) + #result = json.load(open(joinpath(broot.rootdir(), target_dir[1:], 'kiwi.result'), 'rb')) + import pickle + result = pickle.load(open(joinpath(broot.rootdir(), target_dir[1:], 'kiwi.result'), 'rb')) imgdata = { 'arch': arch, 'task_id': self.id, 'logs': [ - os.path.basename(desc) + os.path.basename(desc), ], 'name': name, 'version': version, @@ -365,19 +384,25 @@ class KiwiCreateImageTask(BaseBuildTask): 'rpmlist': [], 'files': [], } + # TODO: upload detailed log? # build/image-root.log - # os.path.join(broot.tmpdir(), target_dir[1:], "build/image-root.log") + root_log_path = os.path.join(broot.tmpdir(), target_dir[1:], "build/image-root.log") + if os.path.exists(root_log_path): + self.uploadFile(root_log_path, remoteName="image-root.log") # for type in types: # img_file = '%s.%s-%s.%s' % (name, version, arch, type) # self.uploadFile(os.path.join(broot.rootdir()), remoteName=img_file) # imgdata['files'].append(img_file) - fpath = os.path.join(broot.rootdir(), - result['result_files']['disk_format_image'].filename[1:]) - img_file = os.path.basename(fpath) - self.uploadFile(fpath, remoteName=os.path.basename(img_file)) - imgdata['files'].append(img_file) + for ftype in ('disk_format_image', 'installation_image'): + fdata = result.result_files.get(ftype) + if not fdata: + continue + fpath = os.path.join(broot.rootdir(), fdata.filename[1:]) + img_file = os.path.basename(fpath) + self.uploadFile(fpath, remoteName=os.path.basename(img_file)) + imgdata['files'].append(img_file) if not self.opts.get('scratch'): if False: From 1e971affef300e41c2d3ff3db535ef1efbba865e Mon Sep 17 00:00:00 2001 From: Tomas Kopecek Date: Nov 18 2021 10:12:48 +0000 Subject: [PATCH 11/15] fix doc typo --- diff --git a/docs/source/plugins.rst b/docs/source/plugins.rst index 39de555..c2fbb87 100644 --- a/docs/source/plugins.rst +++ b/docs/source/plugins.rst @@ -252,7 +252,7 @@ most simple configuration will look like: .. code-block:: shell $ koji add-group kiwi-build-tag kiwi - $ koji add-group-pkg kiwi-build-tag kiwi-cli + $ koji add-group-pkg kiwi-build-tag kiwi kiwi-cli Calling the build itself is a matter of simple CLI call: From 79e1841c96a1aa2a3de31f8de08b9b400960d0ce Mon Sep 17 00:00:00 2001 From: Tomas Kopecek Date: Nov 18 2021 10:12:48 +0000 Subject: [PATCH 12/15] fix bandit --- diff --git a/plugins/builder/kiwi.py b/plugins/builder/kiwi.py index 3f06163..64d6bcf 100644 --- a/plugins/builder/kiwi.py +++ b/plugins/builder/kiwi.py @@ -22,7 +22,7 @@ class KiwiBuildTask(BuildImageTask): cfg = kiwi_files[0] - newxml = xml.dom.minidom.parse(cfg) + newxml = xml.dom.minidom.parse(cfg) # nosec image = newxml.getElementsByTagName('image')[0] name = image.getAttribute('name') @@ -198,7 +198,7 @@ class KiwiCreateImageTask(BaseBuildTask): cfg = kiwi_files[0] - newxml = xml.dom.minidom.parse(cfg) + newxml = xml.dom.minidom.parse(cfg) # nosec image = newxml.getElementsByTagName('image')[0] # apply includes - kiwi can include only top-level nodes, so we can simply @@ -206,7 +206,7 @@ class KiwiCreateImageTask(BaseBuildTask): # doing it recursively) for inc_node in image.getElementsByTagName('include'): path = inc_node.getAttribute('from') - inc = xml.dom.minidom.parse(path) + inc = xml.dom.minidom.parse(path) # nosec # every included xml has image root element again for node in inc.getElementsByTagName('image').childNodes: if node.nodeName != 'repository': @@ -369,8 +369,10 @@ class KiwiCreateImageTask(BaseBuildTask): raise koji.GenericError("Kiwi failed") #result = json.load(open(joinpath(broot.rootdir(), target_dir[1:], 'kiwi.result'), 'rb')) + # nosec comment - we will replace it with json ASAP import pickle - result = pickle.load(open(joinpath(broot.rootdir(), target_dir[1:], 'kiwi.result'), 'rb')) + result = pickle.load(open(joinpath(broot.rootdir(), target_dir[1:], # nosec + 'kiwi.result'), 'rb')) imgdata = { 'arch': arch, From eb56f2bec437f4f7a102ff4153be5e69ac4c8c69 Mon Sep 17 00:00:00 2001 From: Tomas Kopecek Date: Nov 18 2021 10:12:48 +0000 Subject: [PATCH 13/15] expand docs --- diff --git a/docs/source/plugins.rst b/docs/source/plugins.rst index c2fbb87..c6c8f43 100644 --- a/docs/source/plugins.rst +++ b/docs/source/plugins.rst @@ -254,6 +254,13 @@ most simple configuration will look like: $ koji add-group kiwi-build-tag kiwi $ koji add-group-pkg kiwi-build-tag kiwi kiwi-cli +Another thing we need to ensure is that we're building in chroot and not in +container. + +.. code-block:: shell + + $ koji edit-tag kiwi-build-tag -x mock.new_chroot=False + Calling the build itself is a matter of simple CLI call: .. code-block: shell From f0f227f0ab7fbf9643f0533c585e9f585575ee9e Mon Sep 17 00:00:00 2001 From: Tomas Kopecek Date: Nov 18 2021 10:12:48 +0000 Subject: [PATCH 14/15] fix flake8 --- diff --git a/plugins/builder/kiwi.py b/plugins/builder/kiwi.py index 64d6bcf..83cffc1 100644 --- a/plugins/builder/kiwi.py +++ b/plugins/builder/kiwi.py @@ -1,5 +1,5 @@ import glob -import json +# import json import os import xml.dom.minidom from fnmatch import fnmatch @@ -368,7 +368,7 @@ class KiwiCreateImageTask(BaseBuildTask): if rv: raise koji.GenericError("Kiwi failed") - #result = json.load(open(joinpath(broot.rootdir(), target_dir[1:], 'kiwi.result'), 'rb')) + # result = json.load(open(joinpath(broot.rootdir(), target_dir[1:], 'kiwi.result'), 'rb')) # nosec comment - we will replace it with json ASAP import pickle result = pickle.load(open(joinpath(broot.rootdir(), target_dir[1:], # nosec From 468656d8a384fcf08e1a83beb9ed3864ee5a7b8b Mon Sep 17 00:00:00 2001 From: Tomas Kopecek Date: Nov 18 2021 10:12:48 +0000 Subject: [PATCH 15/15] fix docs --- diff --git a/docs/source/plugins.rst b/docs/source/plugins.rst index c6c8f43..8e5aa5d 100644 --- a/docs/source/plugins.rst +++ b/docs/source/plugins.rst @@ -242,7 +242,9 @@ configuration except allowing the plugins (name is 'kiwi' for all components). Builders have to be part of ``image`` channel and don't need to have any specific library installed (kiwi invocation/usage is only in buildroots not on -builder itself). +builder itself). (Temporarily ``python3-kiwi`` needs to be installed on builder +for kojid to be able to parse kiwi output. It will be changed to json in next +version and this requirement will be dropped.) Buildtag needs to be configured by adding special group ``kiwi`` which should contain at least ``kiwi-cli``, potentially ``jing`` for better description files