From def681d2185708553a363927d478292d050b6d4d Mon Sep 17 00:00:00 2001 From: djuarezg Date: Apr 07 2022 12:42:38 +0000 Subject: [PATCH 1/8] Adding initial Driver Update Disk building support --- diff --git a/docs/source/plugins.rst b/docs/source/plugins.rst index 839678b..ff1ec52 100644 --- a/docs/source/plugins.rst +++ b/docs/source/plugins.rst @@ -278,3 +278,52 @@ Selecting other than default kiwi profile can be done by ``--kiwi-profile`` option. Similarly to other image tasks, alternative architecture failures can be ignored for successful build by ``--can-fail`` option. ``--arch`` can be used to limit build tag architectures. + + +Driver Update Disks building +=========================== + +**This is just a tech-preview. API/usage can drastically change in upcoming +releases** + +Plugin for creating Driver Update Disks with ``mkisofs``. + +All three parts (cli/hub/builder) needs to be installed. There is currently no +configuration except allowing the plugins (name is 'dud' for all components). + +Builders have to be part of ``image`` channel and don't need to have any +specific library installed (mkisofs invocation/usage is only in buildroots not on +builder itself) + +Buildtag needs to be configured by adding special group ``dud`` which should contain +the following packages: + +.. code-block:: shell + + + $ koji add-group dud-build-tag dud + $ koji add-group-pkg dud-build-tag dud genisoimage + $ koji add-group-pkg dud-build-tag dud createrepo_c + $ koji add-group-pkg dud-build-tag dud dnf + $ koji add-group-pkg dud-build-tag dud dnf-plugins-core + +Another thing we need to ensure is that we're building in chroot and not in +container. + +.. code-block:: shell + + $ koji edit-tag dud-build-tag -x mock.new_chroot=False + +Calling the build itself is a matter of simple CLI call: + +.. code-block: shell + + $ koji dud-build dud-build-tag --scmurl=git+https://my.git/image-descriptions#master myamazingdud 1 package1 package2 + +The command options allows to bring all the package dependencies into the DUD +ISO with ``--alldeps``. ``--scmurl`` allows to include non-RPM related content +inside the produced ISO. + +Similarly to other image tasks, alternative architecture failures can be +ignored for successful build by ``--can-fail`` option. ``--arch`` can be used +to limit build tag architectures. diff --git a/plugins/builder/dud.py b/plugins/builder/dud.py new file mode 100644 index 0000000..b1aa8dd --- /dev/null +++ b/plugins/builder/dud.py @@ -0,0 +1,319 @@ +import glob +import json +import os +import rpm +import xml.dom.minidom +from fnmatch import fnmatch + +import koji +import time +import re +from koji.util import joinpath, to_list +from koji.tasks import ServerExit +from __main__ import BaseBuildTask, BuildImageTask, BuildRoot, SCM + +# /usr/lib/koji-builder-plugins/ + +class DudBuildTask(BuildImageTask): + Methods = ['dudBuild'] + _taskWeight = 1.0 + + def handler(self, dud_name, dud_version, arches, target, pkg_list, opts=None): + target_info = self.session.getBuildTarget(target, strict=True) + build_tag = target_info['build_tag'] + repo_info = self.getRepo(build_tag) + # check requested arches against build tag + buildconfig = self.session.getBuildConfig(build_tag) + if not buildconfig['arches']: + raise koji.BuildError("No arches for tag %(name)s [%(id)s]" % buildconfig) + tag_archlist = [koji.canonArch(a) for a in buildconfig['arches'].split()] + if arches: + for arch in arches: + if koji.canonArch(arch) not in tag_archlist: + raise koji.BuildError("Invalid arch for build tag: %s" % arch) + else: + arches = tag_archlist + + if not opts: + opts = {} + if not opts.get('scratch'): + opts['scratch'] = False + if not opts.get('alldeps'): + opts['alldeps'] = False + if not opts.get('scmurl'): + opts['scmurl'] = None + if not opts.get('optional_arches'): + opts['optional_arches'] = [] + self.opts = opts + + name, version, release = dud_name, dud_version, None + + bld_info = None + if '-' in version: + raise koji.ApplianceError('The Version may not have a hyphen') + if not opts['scratch']: + bld_info = self.initImageBuild(name, version, release, target_info, opts) + release = bld_info['release'] + elif not release: + release = self.session.getNextRelease({'name': name, 'version': version}) + + try: + subtasks = {} + canfail = [] + self.logger.info("Spawning jobs for image arches: %r" % (arches)) + for arch in arches: + subtasks[arch] = self.session.host.subtask( + method='createDudIso', + arglist=[name, version, release, arch, + target_info, build_tag, repo_info, + pkg_list, opts], label=arch, + parent=self.id, arch=arch) + if arch in self.opts['optional_arches']: + canfail.append(subtasks[arch]) + self.logger.info("Got image subtasks: %r" % (subtasks)) + self.logger.info("Waiting on image subtasks (%s can fail)..." % canfail) + results = self.wait(to_list(subtasks.values()), all=True, failany=True, canfail=canfail) + + # if everything failed, fail even if all subtasks are in canfail + self.logger.info('subtask results: %r', results) + all_failed = True + for result in results.values(): + if not isinstance(result, dict) or 'faultCode' not in result: + all_failed = False + break + if all_failed: + raise koji.GenericError("all subtasks failed") + + # determine ignored arch failures + ignored_arches = set() + for arch in arches: + if arch in self.opts['optional_arches']: + task_id = subtasks[arch] + result = results[task_id] + if isinstance(result, dict) and 'faultCode' in result: + ignored_arches.add(arch) + + self.logger.info('Image Results for hub: %s' % results) + results = {str(k): v for k, v in results.items()} + if opts['scratch']: + self.session.host.moveImageBuildToScratch(self.id, results) + else: + self.session.host.completeImageBuild(self.id, bld_info['id'], results) + except (SystemExit, ServerExit, KeyboardInterrupt): + # we do not trap these + raise + except Exception: + if not opts['scratch']: + if bld_info: + self.session.host.failBuild(self.id, bld_info['id']) + raise + + # tag it + if not opts['scratch'] and not opts.get('skip_tag'): + tag_task_id = self.session.host.subtask(method='tagBuild', + arglist=[target_info['dest_tag'], + bld_info['id'], False, None, True], + label='tag', parent=self.id, arch='noarch') + self.wait(tag_task_id) + + # report results + report = '' + + if opts['scratch']: + respath = ', '.join( + [os.path.join(koji.pathinfo.work(), + koji.pathinfo.taskrelpath(tid)) for tid in subtasks.values()]) + report += 'Scratch ' + + else: + respath = koji.pathinfo.imagebuild(bld_info) + report += 'image build results in: %s' % respath + return report + + +class DudCreateImageTask(BaseBuildTask): + Methods = ['createDudIso'] + _taskWeight = 1.0 + + def getImagePackagesFromPath(self, path): + """ + Read RPM header information from the yum cache available in the + given path. Returns a list of dictionaries for each RPM included. + """ + found = False + hdrlist = {} + # For non scratch builds this is a must or it will not work + fields = ['name', 'version', 'release', 'epoch', 'arch', + 'buildtime', 'sigmd5'] + for root, dirs, files in os.walk(path): + for f in files: + if fnmatch(f, '*.rpm'): + pkgfile = os.path.join(root, f) + hdr = koji.get_header_fields(pkgfile, fields) + hdr['size'] = os.path.getsize(pkgfile) + hdr['payloadhash'] = koji.hex_string(hdr['sigmd5']) + del hdr['sigmd5'] + hdrlist[os.path.basename(pkgfile)] = hdr + found = True + if not found: + raise koji.LiveCDError('No rpms found in root dir!') + return list(hdrlist.values()) + + def handler(self, dud_name, dud_version, dud_release, arch, + target_info, build_tag, repo_info, + pkg_list, opts=None): + self.opts = opts + self.logger.info("Running my dud task...") + build_tag = target_info['build_tag'] + broot = BuildRoot(self.session, self.options, + tag=build_tag, + arch=arch, + task_id=self.id, + repo_id=repo_info['id'], + install_group='dud', # Replace with a group that includes createrepo and mkisofs + setup_dns=True, + bind_opts={'dirs': {'/dev': '/dev', }}) + broot.workdir = self.workdir + + # create the mock chroot + self.logger.info("Initializing dud buildroot") + broot.init() + self.logger.info("dud buildroot ready: " + broot.rootdir()) + + # user repos + repos = self.opts.get('repos', []) + # buildroot repo + path_info = koji.PathInfo(topdir=self.options.topurl) + repopath = path_info.repo(repo_info['id'], target_info['build_tag_name']) + baseurl = '%s/%s' % (repopath, arch) + self.logger.info('BASEURL: %s' % baseurl) + repos.append(baseurl) + + imgdata = { + 'arch': arch, + 'task_id': self.id, + 'name': '', + 'version': '', + 'release': '', + 'logs': [], + 'rpmlist': [], + 'files': [], + } + + # Donwload each and every one of the packages on the list. We allow more than one + # rpms per DUD ISO. Do them one by one to report which one may fail + for rpm in pkg_list: + cmd = ['/usr/bin/dnf'] + if self.opts.get('alldeps'): + cmd.extend([ + 'download', '--resolve', '--alldeps', rpm, + ]) + else: + cmd.extend([ + 'download', rpm, + ]) + + rv = broot.mock(['--cwd', broot.tmpdir(within=True), '--chroot', '--'] + cmd) + if rv: + raise koji.GenericError("DUD build failed while getting the involved rpm '{}': {}".format(rpm, str(rv))) + + # Create the dd directory structure. + cmd = ['/usr/bin/mkdir'] + cmd.extend([ + '-p', './dd/rpms/x86_64/repodata/', + '-p', './dd/src/', + ]) + rv = broot.mock(['--cwd', broot.tmpdir(within=True), '--chroot', '--'] + cmd) + if rv: + raise koji.GenericError("DUD build failed while preparing the dir struct for the ISO: " + str(rv)) + + # Inspiration from https://pagure.io/koji/blob/master/f/plugins/builder/runroot.py#_201 for this dirty hack + cmd = ['/usr/bin/sh', '-c'] + cmd.extend([ + '/usr/bin/echo -e "Driver Update Disk version 3\c" > ./dd/rhdd3', + ]) + rv = broot.mock(['--cwd', broot.tmpdir(within=True), '--chroot', '--'] + cmd) + if rv: + raise koji.GenericError("DUD build failed while writing the rhdd3 file in the ISO: " + str(rv)) + + imgdata['name'] = dud_name + imgdata['version'] = dud_version + imgdata['release'] = dud_release + + # Get the SCM content into the ISO root + # Retrieve SCM content if it exists + if self.opts.get('scmurl'): + # get configuration + scm = SCM(self.opts.get('scmurl')) + scm.assert_allowed(allowed=self.options.allowed_scms, + session=self.session, + by_config=self.options.allowed_scms_use_config, + by_policy=self.options.allowed_scms_use_policy, + policy_data={ + 'user_id': self.taskinfo['owner'], + 'channel': self.session.getChannel(self.taskinfo['channel_id'], + strict=True)['name'], + 'scratch': self.opts.get('scratch') + }) + logfile = os.path.join(self.workdir, 'checkout-%s.log' % arch) + self.run_callbacks('preSCMCheckout', scminfo=scm.get_info(), + build_tag=build_tag, scratch=self.opts.get('scratch')) + scmdir = broot.tmpdir() + koji.ensuredir(scmdir) + scmsrcdir = scm.checkout(scmdir, self.session, + self.getUploadDir(), logfile) + self.run_callbacks("postSCMCheckout", + scminfo=scm.get_info(), + build_tag=build_tag, + scratch=self.opts.get('scratch'), + srcdir=scmsrcdir) + cmd = ['/usr/bin/cp'] + cmd.extend([ + '-aR', os.path.basename(scmsrcdir), './dd/', + ]) + rv = broot.mock(['--cwd', broot.tmpdir(within=True), '--chroot', '--'] + cmd) + if rv: + raise koji.GenericError("DUD build failed while copying SCM repo content into dir struct: " + str(rv)) + + # Get the RPMs inside the corresponding dir struct for the ISO + cmd = ['/usr/bin/sh', '-c'] + # Could not get it to work with a more elegant syntax, as it would not find the *.rpm files otherwise + cmd.extend([ + '/usr/bin/cp *.rpm ./dd/rpms/x86_64/', + ]) + rv = broot.mock(['--cwd', broot.tmpdir(within=True), '--chroot', '--'] + cmd) + if rv: + raise koji.GenericError("DUD build failed while copying RPMs into dir struct: " + str(rv)) + + cmd = ['/usr/bin/createrepo'] + cmd.extend([ + '-q', '--workers=1', './dd/rpms/x86_64/', + ]) + rv = broot.mock(['--cwd', broot.tmpdir(within=True), '--chroot', '--'] + cmd) + if rv: + raise koji.GenericError("DUD build failed while creating ISO repodata: " + str(rv)) + + # mkisofs -quiet -lR -V OEMDRV -input-charset utf8 -o $PACKNAME ./dd + cmd = ['/usr/bin/mkisofs'] + iso_name = 'dd-{name}-{version}-{release}.{arch}.iso'.format(name=dud_name, version=dud_version, release=dud_release, arch=arch) + cmd.extend([ + '-quiet', '-lR', '-V OEMDRV', '-input-charset', 'utf8', '-o', iso_name, './dd', + ]) + rv = broot.mock(['--cwd', broot.tmpdir(within=True), '--chroot', '--'] + cmd) + if rv: + raise koji.GenericError("DUD build failed while mkisofs: " + str(rv)) + + fpath = os.path.join(broot.tmpdir(), iso_name) + img_file = os.path.basename(fpath) + self.uploadFile(fpath, remoteName=os.path.basename(img_file)) + imgdata['files'].append(img_file) + + if not self.opts.get('scratch'): + hdrlist = self.getImagePackagesFromPath(broot.tmpdir()) + broot.markExternalRPMs(hdrlist) + imgdata['rpmlist'] = hdrlist + + broot.expire() + self.logger.error("Uploading image data: %s", imgdata) + return imgdata + diff --git a/plugins/cli/dud.py b/plugins/cli/dud.py new file mode 100644 index 0000000..fb70cf2 --- /dev/null +++ b/plugins/cli/dud.py @@ -0,0 +1,74 @@ +from optparse import OptionParser + +from koji import canonArch + +from koji.plugin import export_cli +from koji_cli.lib import ( + _running_in_bg, + activate_session, + watch_tasks, +) + +# All client related stuff, to be located in ~/.koji/plugins/dud.py +@export_cli +def handle_dud_build(goptions, session, args): + "[build] Run a command in a buildroot" + usage = "usage: %prog dud-build [options] --scmurl= [ ...]" + usage += "\n(Specify the --help global option for a list of other help options)" + parser = OptionParser(usage=usage) + parser.add_option("--scratch", action="store_true", default=False, + help="Perform a scratch build") + parser.add_option("--scmurl", metavar="SCMURL", default=None, + help="SCM repository URL for non-rpm related content to be included in the ISO") + parser.add_option("--alldeps", action="store_true", default=False, + help="Download all involved rpm dependencies and put them inside the DUD ISO as well") + parser.add_option("--arch", action="append", dest="arches", default=[], + help="Limit arches to this subset") + parser.add_option("--can-fail", action="store", dest="optional_arches", + metavar="ARCH1,ARCH2,...", default="", + help="List of archs which are not blocking for build " + "(separated by commas.") + parser.add_option("--nowait", action="store_false", dest="wait", default=True) + parser.add_option("--wait", action="store_true", + help="Wait on the image creation, even if running in the background") + (options, args) = parser.parse_args(args) + + if len(args) < 4: + parser.error("Incorrect number of arguments") + assert False # pragma: no cover + pkg_list = [] + + # Can't use * operator with unpacking in Python 2.7, but this works for both Python 2 and 3 + target, dud_name, dud_version, pkg_list = args[0], args[1], args[2], args[3:] + + activate_session(session, goptions) + + kwargs = { + 'scratch': options.scratch, + 'alldeps': options.alldeps, + 'scmurl': options.scmurl, + 'optional_arches': [canonArch(arch) + for arch in options.optional_arches.split(',') + if arch], + } + + arches = [] + if options.arches: + arches = [canonArch(arch) for arch in options.arches] + + task_id = session.dudBuild( + target=target, + arches=arches, + dud_version=dud_version, + dud_name=dud_name, + pkg_list=pkg_list, + **kwargs) + + if not goptions.quiet: + print("Created task: %d" % task_id) + print("Task info: %s/taskinfo?taskID=%s" % (goptions.weburl, task_id)) + if options.wait or (options.wait is None and not _running_in_bg()): + session.logout() + return watch_tasks(session, [task_id], quiet=goptions.quiet, + poll_interval=goptions.poll_interval, topurl=goptions.topurl) + diff --git a/plugins/hub/dud.py b/plugins/hub/dud.py new file mode 100644 index 0000000..954e168 --- /dev/null +++ b/plugins/hub/dud.py @@ -0,0 +1,40 @@ +import koji +import koji.tasks +import kojihub + +from koji.context import context +from koji.plugin import export + +koji.tasks.LEGACY_SIGNATURES['dudBuild'] = [ + [['dud_name', 'dud_version', 'arches', 'target', 'pkg_list', 'opts'], + None, None, (None,)]] +koji.tasks.LEGACY_SIGNATURES['createDudIso'] = [ + [['dud_name', 'dud_version', 'dud_release', 'arch', + 'target_info', 'build_tag', 'repo_info', 'pkg_list', 'opts'], + None, None, (None,)]] + +# /usr/lib/koji-hub-plugins/ + +@export +def dudBuild(dud_name, dud_version, arches, target, pkg_list, optional_arches=None, scratch=False, alldeps=False, scmurl=None, priority=None): + context.session.assertPerm('image') + taskOpts = { + 'channel': 'image', + } + if priority: + if priority < 0: + if not context.session.hasPerm('admin'): + raise koji.ActionNotAllowed( + 'only admins may create high-priority tasks') + taskOpts['priority'] = koji.PRIO_DEFAULT + priority + + opts = { + 'scratch': scratch, + 'alldeps': alldeps, + 'scmurl': scmurl, + 'optional_arches': optional_arches, + } + + return kojihub.make_task('dudBuild', + [dud_name, dud_version, arches, target, pkg_list, opts], + **taskOpts) From b368db6972d2fc32bb76dbc4690e5b946fe2e571 Mon Sep 17 00:00:00 2001 From: djuarezg Date: Apr 07 2022 12:42:38 +0000 Subject: [PATCH 2/8] Apply xorriso and flake8 suggestions --- diff --git a/plugins/builder/dud.py b/plugins/builder/dud.py index b1aa8dd..3227324 100644 --- a/plugins/builder/dud.py +++ b/plugins/builder/dud.py @@ -1,19 +1,13 @@ -import glob -import json import os -import rpm -import xml.dom.minidom -from fnmatch import fnmatch - import koji -import time -import re -from koji.util import joinpath, to_list +from fnmatch import fnmatch +from koji.util import to_list from koji.tasks import ServerExit from __main__ import BaseBuildTask, BuildImageTask, BuildRoot, SCM # /usr/lib/koji-builder-plugins/ + class DudBuildTask(BuildImageTask): Methods = ['dudBuild'] _taskWeight = 1.0 @@ -66,13 +60,13 @@ class DudBuildTask(BuildImageTask): method='createDudIso', arglist=[name, version, release, arch, target_info, build_tag, repo_info, - pkg_list, opts], label=arch, - parent=self.id, arch=arch) + pkg_list, opts], label=arch, parent=self.id, arch=arch) if arch in self.opts['optional_arches']: canfail.append(subtasks[arch]) self.logger.info("Got image subtasks: %r" % (subtasks)) self.logger.info("Waiting on image subtasks (%s can fail)..." % canfail) - results = self.wait(to_list(subtasks.values()), all=True, failany=True, canfail=canfail) + results = self.wait(to_list(subtasks.values()), all=True, failany=True, + canfail=canfail) # if everything failed, fail even if all subtasks are in canfail self.logger.info('subtask results: %r', results) @@ -170,7 +164,8 @@ class DudCreateImageTask(BaseBuildTask): arch=arch, task_id=self.id, repo_id=repo_info['id'], - install_group='dud', # Replace with a group that includes createrepo and mkisofs + # Replace with a group that includes createrepo and xorrisofs + install_group='dud', setup_dns=True, bind_opts={'dirs': {'/dev': '/dev', }}) broot.workdir = self.workdir @@ -215,26 +210,30 @@ class DudCreateImageTask(BaseBuildTask): rv = broot.mock(['--cwd', broot.tmpdir(within=True), '--chroot', '--'] + cmd) if rv: - raise koji.GenericError("DUD build failed while getting the involved rpm '{}': {}".format(rpm, str(rv))) + raise koji.GenericError("DUD build failed while getting the involved rpm '{}': {}" + .format(rpm, str(rv))) # Create the dd directory structure. cmd = ['/usr/bin/mkdir'] cmd.extend([ - '-p', './dd/rpms/x86_64/repodata/', + '-p', './dd/rpms/{arch}/repodata/'.format(arch=arch), '-p', './dd/src/', ]) rv = broot.mock(['--cwd', broot.tmpdir(within=True), '--chroot', '--'] + cmd) if rv: - raise koji.GenericError("DUD build failed while preparing the dir struct for the ISO: " + str(rv)) + raise koji.GenericError("DUD build failed while preparing the dir struct for " + "the ISO: " + str(rv)) - # Inspiration from https://pagure.io/koji/blob/master/f/plugins/builder/runroot.py#_201 for this dirty hack + # Inspiration from https://pagure.io/koji/blob/master/f/plugins/builder/runroot.py#_201 + # for this dirty hack cmd = ['/usr/bin/sh', '-c'] cmd.extend([ '/usr/bin/echo -e "Driver Update Disk version 3\c" > ./dd/rhdd3', ]) rv = broot.mock(['--cwd', broot.tmpdir(within=True), '--chroot', '--'] + cmd) if rv: - raise koji.GenericError("DUD build failed while writing the rhdd3 file in the ISO: " + str(rv)) + raise koji.GenericError("DUD build failed while writing the rhdd3 file in " + "the ISO: " + str(rv)) imgdata['name'] = dud_name imgdata['version'] = dud_version @@ -246,62 +245,69 @@ class DudCreateImageTask(BaseBuildTask): # get configuration scm = SCM(self.opts.get('scmurl')) scm.assert_allowed(allowed=self.options.allowed_scms, - session=self.session, - by_config=self.options.allowed_scms_use_config, - by_policy=self.options.allowed_scms_use_policy, - policy_data={ - 'user_id': self.taskinfo['owner'], - 'channel': self.session.getChannel(self.taskinfo['channel_id'], + session=self.session, + by_config=self.options.allowed_scms_use_config, + by_policy=self.options.allowed_scms_use_policy, + policy_data={ + 'user_id': self.taskinfo['owner'], + 'channel': self.session.getChannel(self.taskinfo['channel_id'], strict=True)['name'], - 'scratch': self.opts.get('scratch') - }) + 'scratch': self.opts.get('scratch') + }) logfile = os.path.join(self.workdir, 'checkout-%s.log' % arch) self.run_callbacks('preSCMCheckout', scminfo=scm.get_info(), - build_tag=build_tag, scratch=self.opts.get('scratch')) + build_tag=build_tag, scratch=self.opts.get('scratch')) scmdir = broot.tmpdir() koji.ensuredir(scmdir) scmsrcdir = scm.checkout(scmdir, self.session, - self.getUploadDir(), logfile) + self.getUploadDir(), logfile) self.run_callbacks("postSCMCheckout", - scminfo=scm.get_info(), - build_tag=build_tag, - scratch=self.opts.get('scratch'), - srcdir=scmsrcdir) + scminfo=scm.get_info(), + build_tag=build_tag, + scratch=self.opts.get('scratch'), + srcdir=scmsrcdir) cmd = ['/usr/bin/cp'] cmd.extend([ '-aR', os.path.basename(scmsrcdir), './dd/', ]) rv = broot.mock(['--cwd', broot.tmpdir(within=True), '--chroot', '--'] + cmd) if rv: - raise koji.GenericError("DUD build failed while copying SCM repo content into dir struct: " + str(rv)) + raise koji.GenericError("DUD build failed while copying SCM repo content into dir" + "struct: " + str(rv)) # Get the RPMs inside the corresponding dir struct for the ISO cmd = ['/usr/bin/sh', '-c'] - # Could not get it to work with a more elegant syntax, as it would not find the *.rpm files otherwise + # Could not get it to work with a more elegant syntax, as it would not find the *.rpm + # files otherwise cmd.extend([ - '/usr/bin/cp *.rpm ./dd/rpms/x86_64/', + '/usr/bin/cp *.rpm ./dd/rpms/{arch}/'.format(arch=arch), ]) rv = broot.mock(['--cwd', broot.tmpdir(within=True), '--chroot', '--'] + cmd) if rv: - raise koji.GenericError("DUD build failed while copying RPMs into dir struct: " + str(rv)) + raise koji.GenericError("DUD build failed while copying RPMs into dir struct: " + + str(rv)) cmd = ['/usr/bin/createrepo'] cmd.extend([ - '-q', '--workers=1', './dd/rpms/x86_64/', + '-q', '--workers=1', './dd/rpms/{arch}/'.format(arch=arch), ]) rv = broot.mock(['--cwd', broot.tmpdir(within=True), '--chroot', '--'] + cmd) if rv: raise koji.GenericError("DUD build failed while creating ISO repodata: " + str(rv)) - # mkisofs -quiet -lR -V OEMDRV -input-charset utf8 -o $PACKNAME ./dd - cmd = ['/usr/bin/mkisofs'] - iso_name = 'dd-{name}-{version}-{release}.{arch}.iso'.format(name=dud_name, version=dud_version, release=dud_release, arch=arch) + # xorrisofs -quiet -lR -V OEMDRV -input-charset utf8 -o $PACKNAME ./dd + cmd = ['/usr/bin/sh', '-c'] + iso_name = 'dd-{name}-{version}-{release}.{arch}.iso'.format(name=dud_name, + version=dud_version, + release=dud_release, + arch=arch) cmd.extend([ - '-quiet', '-lR', '-V OEMDRV', '-input-charset', 'utf8', '-o', iso_name, './dd', + "/usr/bin/xorrisofs -quiet -lR -V OEMDRV -input-charset utf8 -o {} ".format(iso_name) + + "./dd -v" ]) rv = broot.mock(['--cwd', broot.tmpdir(within=True), '--chroot', '--'] + cmd) if rv: - raise koji.GenericError("DUD build failed while mkisofs: " + str(rv)) + raise koji.GenericError("DUD build failed while xorrisofs: " + str(rv)) fpath = os.path.join(broot.tmpdir(), iso_name) img_file = os.path.basename(fpath) @@ -316,4 +322,3 @@ class DudCreateImageTask(BaseBuildTask): broot.expire() self.logger.error("Uploading image data: %s", imgdata) return imgdata - diff --git a/plugins/cli/dud.py b/plugins/cli/dud.py index fb70cf2..e09ebce 100644 --- a/plugins/cli/dud.py +++ b/plugins/cli/dud.py @@ -10,18 +10,23 @@ from koji_cli.lib import ( ) # All client related stuff, to be located in ~/.koji/plugins/dud.py + + @export_cli def handle_dud_build(goptions, session, args): "[build] Run a command in a buildroot" - usage = "usage: %prog dud-build [options] --scmurl= [ ...]" + usage = ("usage: %prog dud-build [options] --scmurl= " + " [ ...]") usage += "\n(Specify the --help global option for a list of other help options)" parser = OptionParser(usage=usage) parser.add_option("--scratch", action="store_true", default=False, - help="Perform a scratch build") + help="Perform a scratch build") parser.add_option("--scmurl", metavar="SCMURL", default=None, - help="SCM repository URL for non-rpm related content to be included in the ISO") + help="SCM repository URL for non-rpm related content to be included " + "in the ISO") parser.add_option("--alldeps", action="store_true", default=False, - help="Download all involved rpm dependencies and put them inside the DUD ISO as well") + help="Download all involved rpm dependencies and put them inside " + "the DUD ISO as well") parser.add_option("--arch", action="append", dest="arches", default=[], help="Limit arches to this subset") parser.add_option("--can-fail", action="store", dest="optional_arches", @@ -71,4 +76,3 @@ def handle_dud_build(goptions, session, args): session.logout() return watch_tasks(session, [task_id], quiet=goptions.quiet, poll_interval=goptions.poll_interval, topurl=goptions.topurl) - diff --git a/plugins/hub/dud.py b/plugins/hub/dud.py index 954e168..e31f241 100644 --- a/plugins/hub/dud.py +++ b/plugins/hub/dud.py @@ -15,18 +15,20 @@ koji.tasks.LEGACY_SIGNATURES['createDudIso'] = [ # /usr/lib/koji-hub-plugins/ + @export -def dudBuild(dud_name, dud_version, arches, target, pkg_list, optional_arches=None, scratch=False, alldeps=False, scmurl=None, priority=None): +def dudBuild(dud_name, dud_version, arches, target, pkg_list, optional_arches=None, scratch=False, + alldeps=False, scmurl=None, priority=None): context.session.assertPerm('image') taskOpts = { 'channel': 'image', } if priority: - if priority < 0: - if not context.session.hasPerm('admin'): - raise koji.ActionNotAllowed( - 'only admins may create high-priority tasks') - taskOpts['priority'] = koji.PRIO_DEFAULT + priority + if priority < 0: + if not context.session.hasPerm('admin'): + raise koji.ActionNotAllowed( + 'only admins may create high-priority tasks') + taskOpts['priority'] = koji.PRIO_DEFAULT + priority opts = { 'scratch': scratch, From c16ad0c272b7e4b41733e189821abf79d9250ff3 Mon Sep 17 00:00:00 2001 From: djuarezg Date: Apr 07 2022 12:42:38 +0000 Subject: [PATCH 3/8] Rename pkg group to dud-build --- diff --git a/docs/source/plugins.rst b/docs/source/plugins.rst index ff1ec52..e7167b0 100644 --- a/docs/source/plugins.rst +++ b/docs/source/plugins.rst @@ -295,17 +295,17 @@ Builders have to be part of ``image`` channel and don't need to have any specific library installed (mkisofs invocation/usage is only in buildroots not on builder itself) -Buildtag needs to be configured by adding special group ``dud`` which should contain +Buildtag needs to be configured by adding special group ``dud-build`` which should contain the following packages: .. code-block:: shell - $ koji add-group dud-build-tag dud - $ koji add-group-pkg dud-build-tag dud genisoimage - $ koji add-group-pkg dud-build-tag dud createrepo_c - $ koji add-group-pkg dud-build-tag dud dnf - $ koji add-group-pkg dud-build-tag dud dnf-plugins-core + $ koji add-group dud-build-tag dud-build + $ koji add-group-pkg dud-build-tag dud-build xorriso + $ koji add-group-pkg dud-build-tag dud-build createrepo_c + $ koji add-group-pkg dud-build-tag dud-build dnf + $ koji add-group-pkg dud-build-tag dud-build dnf-plugins-core Another thing we need to ensure is that we're building in chroot and not in container. diff --git a/plugins/builder/dud.py b/plugins/builder/dud.py index 3227324..fb42cb4 100644 --- a/plugins/builder/dud.py +++ b/plugins/builder/dud.py @@ -165,7 +165,7 @@ class DudCreateImageTask(BaseBuildTask): task_id=self.id, repo_id=repo_info['id'], # Replace with a group that includes createrepo and xorrisofs - install_group='dud', + install_group='dud-build', setup_dns=True, bind_opts={'dirs': {'/dev': '/dev', }}) broot.workdir = self.workdir From 1e9a235a468936d255c43758e3fdbecc59450676 Mon Sep 17 00:00:00 2001 From: djuarezg Date: Apr 07 2022 12:42:38 +0000 Subject: [PATCH 4/8] Change dud plugin doc to mention xorrisofs --- diff --git a/docs/source/plugins.rst b/docs/source/plugins.rst index e7167b0..4481659 100644 --- a/docs/source/plugins.rst +++ b/docs/source/plugins.rst @@ -286,7 +286,7 @@ Driver Update Disks building **This is just a tech-preview. API/usage can drastically change in upcoming releases** -Plugin for creating Driver Update Disks with ``mkisofs``. +Plugin for creating Driver Update Disks with ``xorrisofs``. All three parts (cli/hub/builder) needs to be installed. There is currently no configuration except allowing the plugins (name is 'dud' for all components). From 7a34265aab89acc6ed8e6c8502e4f64a2d29242d Mon Sep 17 00:00:00 2001 From: djuarezg Date: Apr 07 2022 12:42:38 +0000 Subject: [PATCH 5/8] Update NVR option checking to match kiwi's --- diff --git a/plugins/builder/dud.py b/plugins/builder/dud.py index fb42cb4..6841430 100644 --- a/plugins/builder/dud.py +++ b/plugins/builder/dud.py @@ -43,13 +43,15 @@ class DudBuildTask(BuildImageTask): name, version, release = dud_name, dud_version, None bld_info = None + if opts.get('release'): + release = opts['release'] + else: + release = self.session.getNextRelease({'name': name, 'version': version}) if '-' in version: raise koji.ApplianceError('The Version may not have a hyphen') if not opts['scratch']: bld_info = self.initImageBuild(name, version, release, target_info, opts) release = bld_info['release'] - elif not release: - release = self.session.getNextRelease({'name': name, 'version': version}) try: subtasks = {} @@ -187,9 +189,9 @@ class DudCreateImageTask(BaseBuildTask): imgdata = { 'arch': arch, 'task_id': self.id, - 'name': '', - 'version': '', - 'release': '', + 'name': dud_name, + 'version': dud_version, + 'release': dud_release, 'logs': [], 'rpmlist': [], 'files': [], @@ -235,10 +237,6 @@ class DudCreateImageTask(BaseBuildTask): raise koji.GenericError("DUD build failed while writing the rhdd3 file in " "the ISO: " + str(rv)) - imgdata['name'] = dud_name - imgdata['version'] = dud_version - imgdata['release'] = dud_release - # Get the SCM content into the ISO root # Retrieve SCM content if it exists if self.opts.get('scmurl'): From eebd629306f61623e67a7321cd570b2179e01095 Mon Sep 17 00:00:00 2001 From: Daniel Juarez Date: Apr 07 2022 12:42:38 +0000 Subject: [PATCH 6/8] Remove trailing whitespaces --- diff --git a/docs/source/plugins.rst b/docs/source/plugins.rst index 4481659..5a784d3 100644 --- a/docs/source/plugins.rst +++ b/docs/source/plugins.rst @@ -320,10 +320,10 @@ Calling the build itself is a matter of simple CLI call: $ koji dud-build dud-build-tag --scmurl=git+https://my.git/image-descriptions#master myamazingdud 1 package1 package2 -The command options allows to bring all the package dependencies into the DUD -ISO with ``--alldeps``. ``--scmurl`` allows to include non-RPM related content +The command options allows to bring all the package dependencies into the DUD +ISO with ``--alldeps``. ``--scmurl`` allows to include non-RPM related content inside the produced ISO. Similarly to other image tasks, alternative architecture failures can be -ignored for successful build by ``--can-fail`` option. ``--arch`` can be used +ignored for successful build by ``--can-fail`` option. ``--arch`` can be used to limit build tag architectures. From 7c7dfc75cd0a864bdcf43266792ac792cfcafa65 Mon Sep 17 00:00:00 2001 From: djuarezg Date: Apr 07 2022 12:55:03 +0000 Subject: [PATCH 7/8] Do not confuse dud-build-tag with dud-target --- diff --git a/docs/source/plugins.rst b/docs/source/plugins.rst index 5a784d3..e432fe0 100644 --- a/docs/source/plugins.rst +++ b/docs/source/plugins.rst @@ -318,7 +318,7 @@ Calling the build itself is a matter of simple CLI call: .. code-block: shell - $ koji dud-build dud-build-tag --scmurl=git+https://my.git/image-descriptions#master myamazingdud 1 package1 package2 + $ koji dud-build dud-target --scmurl=git+https://my.git/image-descriptions#master myamazingdud 1 package1 package2 The command options allows to bring all the package dependencies into the DUD ISO with ``--alldeps``. ``--scmurl`` allows to include non-RPM related content From 85da1036fae0b3916e7e448d394004bb307d3dd7 Mon Sep 17 00:00:00 2001 From: djuarezg Date: Apr 19 2022 15:13:30 +0000 Subject: [PATCH 8/8] better description for dud channel requirements --- diff --git a/docs/source/plugins.rst b/docs/source/plugins.rst index e432fe0..2a09d2f 100644 --- a/docs/source/plugins.rst +++ b/docs/source/plugins.rst @@ -291,9 +291,8 @@ Plugin for creating Driver Update Disks with ``xorrisofs``. All three parts (cli/hub/builder) needs to be installed. There is currently no configuration except allowing the plugins (name is 'dud' for all components). -Builders have to be part of ``image`` channel and don't need to have any -specific library installed (mkisofs invocation/usage is only in buildroots not on -builder itself) +Builders don't need to have any specific library installed (xorrisofs +invocation/usage is only in buildroots not on builder itself). Buildtag needs to be configured by adding special group ``dud-build`` which should contain the following packages: