From e935d2f8c15f8d49615b998973f622c779fcbd1e Mon Sep 17 00:00:00 2001 From: Jay Greguske Date: Mar 16 2016 00:08:42 +0000 Subject: [PATCH 1/5] Merge remote branch 'github/signed' --- diff --git a/builder/kojid b/builder/kojid index a7bc06d..73235ff 100755 --- a/builder/kojid +++ b/builder/kojid @@ -35,12 +35,14 @@ import logging.handlers from koji.daemon import incremental_upload, log_output, TaskManager, SCM from koji.tasks import ServerExit, ServerRestart, BaseTaskHandler, MultiPlatformTask from koji.util import parseStatus, isSuccess, dslice, dslice_ex +import multilib import os import pwd import grp import random import re import rpm +import rpmUtils.arch import shutil import signal import smtplib @@ -59,6 +61,8 @@ from gzip import GzipFile from optparse import OptionParser, SUPPRESS_HELP from StringIO import StringIO from yum import repoMDObject +import yum.packages +import yum.Errors #imports for LiveCD and Appliance handler image_enabled = False @@ -4292,8 +4296,8 @@ class CreaterepoTask(BaseTaskHandler): if os.path.getsize(pkglist) == 0: pkglist = None self.create_local_repo(rinfo, arch, pkglist, groupdata, oldrepo) - - external_repos = self.session.getExternalRepoList(rinfo['tag_id'], event=rinfo['create_event']) + external_repos = self.session.getExternalRepoList( + rinfo['tag_id'], event=rinfo['create_event']) if external_repos: self.merge_repos(external_repos, arch, groupdata) elif pkglist is None: @@ -4306,10 +4310,9 @@ class CreaterepoTask(BaseTaskHandler): for f in os.listdir(self.datadir): files.append(f) self.session.uploadWrapper('%s/%s' % (self.datadir, f), uploadpath, f) - return [uploadpath, files] - def create_local_repo(self, rinfo, arch, pkglist, groupdata, oldrepo): + def create_local_repo(self, rinfo, arch, pkglist, groupdata, oldrepo, drpms=False): koji.ensuredir(self.outdir) if self.options.use_createrepo_c: cmd = ['/usr/bin/createrepo_c'] @@ -4321,7 +4324,9 @@ class CreaterepoTask(BaseTaskHandler): if os.path.isfile(groupdata): cmd.extend(['-g', groupdata]) #attempt to recycle repodata from last repo - if pkglist and oldrepo and self.options.createrepo_update: + if pkglist and oldrepo and self.options.createrepo_update and not drpms: + # signed repos overload the use of "oldrepo", so the conditional + # explicitly make sure this does not get executed with that on oldpath = self.pathinfo.repo(oldrepo['id'], rinfo['tag_name']) olddatadir = '%s/%s/repodata' % (oldpath, arch) if not os.path.isdir(olddatadir): @@ -4336,6 +4341,11 @@ class CreaterepoTask(BaseTaskHandler): cmd.append('--update') if self.options.createrepo_skip_stat: cmd.append('--skip-stat') + if drpms: + # generate delta-rpms + cmd.append('--deltas') + for repo in oldrepo: + cmd.extend(['--oldpackagedirs', repo]) # note: we can't easily use a cachedir because we do not have write # permission. The good news is that with --update we won't need to # be scanning many rpms. @@ -4380,6 +4390,319 @@ class CreaterepoTask(BaseTaskHandler): raise koji.GenericError, 'failed to merge repos: %s' \ % parseStatus(status, ' '.join(cmd)) + +class NewSignedRepoTask(BaseTaskHandler): + Methods = ['signedRepo'] + _taskWeight = 0.1 + + def handler(self, tag, repo_id, keys, task_opts): + tinfo = self.session.getTag(tag, strict=True, event=task_opts['event']) + path = koji.pathinfo.signedrepo(repo_id, tinfo['name']) + if len(task_opts['arch']) == 0: + task_opts['arch'] = tinfo['arches'].split() + if len(task_opts['arch']) == 0: + raise koji.GenericError('No arches specified nor for the tag!') + subtasks = {} + arch32s = set() + for arch in task_opts['arch']: + if not rpmUtils.arch.isMultiLibArch(arch): + arch32s.add(arch) + for arch in arch32s: + # we do 32-bit multilib arches first so the 64-bit ones can + # get a task ID and wait for them to complete + arglist = [tag, repo_id, arch, keys, task_opts] + subtasks[arch] = self.session.host.subtask( + method='createsignedrepo', arglist=arglist, label=arch, + parent=self.id, arch='noarch') + if len(subtasks) > 0 and task_opts['multilib']: + results = self.wait(subtasks.values(), all=True, failany=True) + for arch in arch32s: + # move the 32-bit task output to the final resting place + # so the 64-bit arches can use it + upload, files, keypaths = results[subtasks[arch]] + self.session.host.signedRepoMove( + repo_id, upload, files, arch, keypaths) + for arch in task_opts['arch']: + # do the other arches + if arch not in arch32s: + arglist = [tag, repo_id, arch, keys, task_opts] + subtasks[arch] = self.session.host.subtask( + method='createsignedrepo', arglist=arglist, label=arch, + parent=self.id, arch='noarch') + # wait for 64-bit subtasks to finish + data = {} + results = self.wait(subtasks.values(), all=True, failany=True) + for (arch, task_id) in subtasks.iteritems(): + data[arch] = results[task_id] + self.logger.debug("DEBUG: %r : %r " % (arch, data[arch])) + if arch not in arch32s: + # we moved the 32-bit results before, do the 64-bit + upload, files, keypaths = results[subtasks[arch]] + self.session.host.signedRepoMove( + repo_id, upload, files, arch, keypaths) + self.session.host.repoDone(repo_id, data, expire=False, signed=True) + return 'Signed repository #%s successfully generated' % repo_id + + +class createSignedRepoTask(CreaterepoTask): + Methods = ['createsignedrepo'] + _taskWeight = 1.5 + + archmap = {'s390x': 's390', 'ppc64': 'ppc', 'x86_64': 'i686'} + compat = {"i386": ("athlon", "i686", "i586", "i486", "i386", "noarch"), + "x86_64": ("amd64", "ia32e", "x86_64", "noarch"), + "ia64": ("ia64", "noarch"), + "ppc": ("ppc", "noarch"), + "ppc64": ("ppc64p7", "ppc64pseries", "ppc64iseries", "ppc64", "noarch"), + "ppc64le": ("ppc64le", "noarch"), + "s390": ("s390", "noarch"), + "s390x": ("s390x", "noarch"), + "sparc": ("sparcv9v", "sparcv9", "sparcv8", "sparc", "noarch"), + "sparc64": ("sparc64v", "sparc64", "noarch"), + "alpha": ("alphaev6", "alphaev56", "alphaev5", "alpha", "noarch"), + "arm": ("arm", "armv4l", "armv4tl", "armv5tel", "armv5tejl", "armv6l", "armv7l", "noarch"), + "armhfp": ("armv7hl", "armv7hnl", "noarch"), + "aarch64": ("aarch64", "noarch"), + } + + biarch = {"ppc": "ppc64", "x86_64": "i386", "sparc": + "sparc64", "s390x": "s390", "ppc64": "ppc"} + + def handler(self, tag, repo_id, arch, keys, opts): + #arch is the arch of the repo, not the task + self.rinfo = self.session.repoInfo(repo_id, strict=True) + if self.rinfo['state'] != koji.REPO_INIT: + raise koji.GenericError, "Repo %(id)s not in INIT state (got %(state)s)" % self.rinfo + self.repo_id = self.rinfo['id'] + self.pathinfo = koji.PathInfo(self.options.topdir) + groupdata = os.path.join( + self.pathinfo.signedrepo(repo_id, self.rinfo['tag_name']), + 'groups', 'comps.xml') + #set up our output dir + self.repodir = '%s/repo' % self.workdir + koji.ensuredir(self.repodir) + self.outdir = self.repodir # workaround create_local_repo use + self.datadir = '%s/repodata' % self.repodir + self.keypaths = {} + if len(opts['delta']) > 0: + for path in opts['delta']: + if not os.path.exists(path): + raise koji.GenericError( + 'drpm path %s does not exist!' % path) + self.uploadpath = self.getUploadDir() + self.pkglist = self.make_pkglist(tag, arch, keys, opts) + if opts['multilib'] and rpmUtils.arch.isMultiLibArch(arch): + self.do_multilib(arch, self.archmap[arch], opts['multilib']) + self.logger.debug('package list is %s' % self.pkglist) + self.session.uploadWrapper(self.pkglist, self.uploadpath, + os.path.basename(self.pkglist)) + if os.path.getsize(self.pkglist) == 0: + self.pkglist = None + if len(opts['delta']) > 0: + do_drpms = True + else: + do_drpms = False + self.create_local_repo(self.rinfo, arch, self.pkglist, groupdata, + opts['delta'], drpms=do_drpms) + if self.pkglist is None: + fo = file(os.path.join(self.datadir, "EMPTY_REPO"), 'w') + fo.write("This repo is empty because its tag has no content for this arch\n") + fo.close() + files = ['pkglist'] + for f in os.listdir(self.datadir): + files.append(f) + self.session.uploadWrapper('%s/%s' % (self.datadir, f), + self.uploadpath, f) + if opts['delta']: + ddir = os.path.join(self.repodir, 'drpms') + for f in os.listdir(ddir): + files.append(f) + self.session.uploadWrapper('%s/%s' % (ddir, f), + self.uploadpath, f) + return [self.uploadpath, files, self.keypaths] + + def do_multilib(self, arch, ml_arch, conf): + self.repo_id = self.rinfo['id'] + pathinfo = koji.PathInfo(self.options.topdir) + repodir = pathinfo.signedrepo(self.rinfo['id'], self.rinfo['tag_name']) + mldir = os.path.join(repodir, koji.canonArch(ml_arch)) + ml_true = set() # multilib packages we need to include before depsolve + ml_conf = os.path.join(self.pathinfo.work(), conf) + + # step 1: figure out which packages are multlib (should already exist) + mlm = multilib.DevelMultilibMethod(ml_conf) + fs_missing = set() + with open(self.pkglist) as pkglist: + for pkg in pkglist: + ppath = os.path.join(self.repodir, pkg.strip()) + po = yum.packages.YumLocalPackage(filename=ppath) + if mlm.select(po) and self.archmap.has_key(arch): + # we need a multilib package to be included + # we assume the same signature level is available + pl_path = pkg.replace(arch, self.archmap[arch]).strip() + # assume this exists in the task results for the ml arch + real_path = os.path.join(mldir, pl_path) + ml_true.add(real_path) + if not os.path.exists(real_path): + self.logger.error('%s (multilib) is not on the filesystem' % real_path) + fs_missing.add(real_path) + + # step 2: set up architectures for yum configuration + self.logger.info("Resolving multilib for %s using method devel" % arch) + yumbase = yum.YumBase() + yumbase.verbose_logger.setLevel(logging.ERROR) + yumdir = os.path.join(self.workdir, 'yum') + # TODO: unwind this arch mess + archlist = (arch, 'noarch') + transaction_arch = arch + archlist = archlist + self.compat[self.biarch[arch]] + best_compat = self.compat[self.biarch[arch]][0] + if rpmUtils.arch.archDifference(best_compat, arch) > 0: + transaction_arch = best_compat + if hasattr(rpmUtils.arch, 'ArchStorage'): + yumbase.preconf.arch = transaction_arch + else: + rpmUtils.arch.canonArch = transaction_arch + + yconfig = """ +[main] +debuglevel=2 +pkgpolicy=newest +exactarch=1 +gpgcheck=0 +reposdir=/dev/null +cachedir=/yumcache +installroot=%s +logfile=/yum.log + +[koji-%s] +name=koji multilib task +baseurl=file://%s +enabled=1 + +""" % (yumdir, self.id, mldir) + os.makedirs(os.path.join(yumdir, "yumcache")) + os.makedirs(os.path.join(yumdir, 'var/lib/rpm')) + + # step 3: proceed with yum config and set up + yconfig_path = os.path.join(yumdir, 'yum.conf-koji-%s' % arch) + f = open(yconfig_path, 'w') + f.write(yconfig) + f.close() + self.session.uploadWrapper(yconfig_path, self.uploadpath, + os.path.basename(yconfig_path)) + yumbase.doConfigSetup(fn=yconfig_path) + yumbase.conf.cache = 0 + yumbase.doRepoSetup() + yumbase.doTsSetup() + yumbase.doRpmDBSetup() + # we trust Koji's files, so skip verifying sigs and digests + yumbase.ts.pushVSFlags( + (rpm._RPMVSF_NOSIGNATURES | rpm._RPMVSF_NODIGESTS)) + yumbase.doSackSetup(archlist=archlist, thisrepo='koji-%s' % arch) + yumbase.doSackFilelistPopulate() + for pkg in ml_true: + # TODO: store packages by first letter + # ppath = os.path.join(pkgdir, pkg.name[0].lower(), pname) + po = yum.packages.YumLocalPackage(filename=pkg) + yumbase.tsInfo.addInstall(po) + + # step 4: execute yum transaction to get dependencies + self.logger.info("Resolving depenencies for arch %s" % arch) + rc, errors = yumbase.resolveDeps() + ml_needed = set() + for f in yumbase.tsInfo.getMembers(): + bnp = os.path.basename(f.po.localPkg()) + dep_path = os.path.join(mldir, bnp[0], bnp) + ml_needed.add(dep_path) + self.logger.debug("added %s" % dep_path) + if not os.path.exists(dep_path): + self.logger.error('%s (multilib dep) not on filesystem' % dep_path) + fs_missing.add(dep_path) + self.logger.info('yum return code: %s' % rc) + if not rc: + self.logger.error('yum depsolve was unsuccessful') + raise koji.GenericError(errors) + if len(fs_missing) > 0: + raise koji.GenericError('multilib packages missing:\n' + + '\n'.join(fs_missing)) + + # step 5: add dependencies to our package list + pkgwriter = open(self.pkglist, 'a') + for ml_pkg in ml_needed: + bnp = os.path.basename(ml_pkg) + pkgwriter.write(bnp[0] + '/' + bnp + '\n') + koji.ensuredir(os.path.join(self.repodir, bnp[0])) + os.symlink(ml_pkg, os.path.join(self.repodir, bnp[0], bnp)) + self.keypaths[bnp] = ml_pkg + + + def make_pkglist(self, tag_id, arch, keys, opts): + + # Need to pass event_id because even though this is a single trans, + # it is possible to see the results of other committed transactions + rpms = [] + builddirs = {} + for a in (arch, 'noarch'): + rpm_iter, builds = self.session.listTaggedRPMS(tag_id, + event=opts['event'], arch=a, + inherit=opts['inherit'], rpmsigs=True) + for build in builds: + builddirs[build['id']] = self.pathinfo.build(build) + rpms += list(rpm_iter) + #get build dirs + need = set(['%(name)s-%(version)s-%(release)s.%(arch)s.rpm' % r for r in rpms]) + #generate pkglist files + pkgfile = os.path.join(self.repodir, 'pkglist') + pkglist = file(pkgfile, 'w') + preferred = {} + if opts['unsigned']: + keys.append('') # make unsigned rpms the least preferred + for rpminfo in rpms: + if rpminfo['sigkey'] == '' and not opts['unsigned']: + # skip, this is the unsigned rpminfo + continue + if rpminfo['sigkey'] not in keys: + # skip, not a key we are looking for + continue + idx = keys.index(rpminfo['sigkey']) + if preferred.has_key(rpminfo['id']): + if keys.index(preferred[rpminfo['id']]['sigkey']) <= idx: + # key for this is not as preferable as what has been seen + continue + preferred[rpminfo['id']] = rpminfo + seen = set() + fs_missing = set() + for rpminfo in preferred.values(): + if rpminfo['sigkey'] == '': + # we're taking an unsigned rpm (--allow-unsigned) + pkgpath = '%s/%s' % (builddirs[rpminfo['build_id']], + self.pathinfo.rpm(rpminfo)) + else: + pkgpath = '%s/%s' % (builddirs[rpminfo['build_id']], + self.pathinfo.signed(rpminfo, rpminfo['sigkey'])) + seen.add(os.path.basename(pkgpath)) + if not os.path.exists(pkgpath): + fs_missing.add(pkgpath) + else: + bnp = os.path.basename(pkgpath) + pkglist.write(bnp[0] + '/' + bnp + '\n') + koji.ensuredir(os.path.join(self.repodir, bnp[0])) + self.keypaths[bnp] = pkgpath + os.symlink(pkgpath, os.path.join(self.repodir, bnp[0], bnp)) + pkglist.close() + if len(fs_missing) > 0: + raise koji.GenericError('Packages missing from the filesystem:\n' + + '\n'.join(fs_missing)) + if not opts['skip']: + missing = list(need - seen) + if len(missing) != 0: + missing.sort() + raise koji.GenericError('Unsigned packages found: ' + + '\n'.join(missing)) + return pkgfile + + class WaitrepoTask(BaseTaskHandler): Methods = ['waitrepo'] diff --git a/cli/koji b/cli/koji index 8c63e2a..8789fb8 100755 --- a/cli/koji +++ b/cli/koji @@ -6713,6 +6713,103 @@ def handle_regen_repo(options, session, args): session.logout() return watch_tasks(session, [task_id], quiet=options.quiet) +def handle_signed_repo(options, session, args): + """create a yum repo of GPG signed RPMs""" + usage = _("usage: %prog signed-repo [options] tag keyID [keyID...]") + usage += _("\n(Specify the --help option for a list of other options)") + parser = OptionParser(usage=usage) + parser.add_option('--allow-unsigned', action='store_true', default=False, + help=_('Use unsigned RPMs if none are available with the right key')) + parser.add_option("--arch", action='append', default=[], + help=_("Indicate an architecture to consider. The default is all " + + "architectures associated with the given tag. This option may " + + "be specified multiple times.")) + parser.add_option('--comps', help='Include a comps file in the repodata') + parser.add_option('--delta-rpms', metavar='PATH',default=[], + action='append', + help=_('Create delta-rpms. PATH points to (older) rpms to generate against. May be specified multiple times. These have to be reachable by the builder too, so the path needs to reach shared storage.')) + parser.add_option('--event', type='int', + help=_('create a signed repository based on a Brew event')) + parser.add_option('--multilib', default=None, + help=_('Include multilib packages in the repository using a config')) + parser.add_option("--noinherit", action='store_true', default=False, + help=_('Do not consider tag inheritance')) + parser.add_option("--nowait", action='store_true', default=False, + help=_('Do not wait for the task to complete')) + parser.add_option('--skip-unsigned', action='store_true', default=False, + help=_('Skip RPMs not signed with the desired key(s)')) + task_opts, args = parser.parse_args(args) + if len(args) < 2: + parser.error(_('You must provide a tag and 1 or more GPG key IDs')) + if task_opts.allow_unsigned and task_opts.skip_unsigned: + parser.error(_('allow_signed and skip_unsigned are mutually exclusive')) + activate_session(session) + stuffdir = _unique_path('cli-signed') + if task_opts.comps: + if not os.path.exists(task_opts.comps): + parser.error(_('could not find %s' % task_opts.comps)) + session.uploadWrapper(task_opts.comps, stuffdir, + callback=_progress_callback) + print + task_opts.comps = os.path.join(stuffdir, + os.path.basename(task_opts.comps)) + if len(task_opts.delta_rpms) > 0: + for path in task_opts.delta_rpms: + if not os.path.exists(path): + print _("Warning: %s is not reachable locally. If this" % path) + print _(" host does not have access to Koji's shared storage") + print _(" this can be ignored.") + tag = args[0] + keys = args[1:] + taginfo = session.getTag(tag) + if not taginfo: + parser.error(_('unknown tag %s' % tag)) + if len(task_opts.arch) == 0: + task_opts.arch = taginfo['arches'] + if task_opts.arch == None: + parser.error(_('No arches given and no arches associated with tag')) + else: + for a in task_opts.arch: + if not taginfo['arches'] or a not in taginfo['arches']: + print _('Warning: %s is not in the list of tag arches' % a) + if task_opts.multilib: + if not os.path.exists(task_opts.multilib): + parser.error(_('could not find %s' % task_opts.multilib)) + if 'x86_64' in task_opts.arch and not 'i686' in task_opts.arch: + parser.error(_('The multilib arch (i686) must be included')) + if 's390x' in task_opts.arch and not 's390' in task_opts.arch: + parser.error(_('The multilib arch (s390) must be included')) + if 'ppc64' in task_opts.arch and not 'ppc' in task_opts.arch: + parser.error(_('The multilib arch (ppc) must be included')) + session.uploadWrapper(task_opts.multilib, stuffdir, + callback=_progress_callback) + task_opts.multilib = os.path.join(stuffdir, + os.path.basename(task_opts.multilib)) + print + try: + task_opts.arch.remove('noarch') # handled specifically + task_opts.arch.remove('src') # ditto + except ValueError: + pass + opts = { + 'arch': task_opts.arch, + 'comps': task_opts.comps, + 'event': task_opts.event, + 'delta': task_opts.delta_rpms, + 'multilib': task_opts.multilib, + 'inherit': not task_opts.noinherit, + 'skip': task_opts.skip_unsigned, + 'unsigned': task_opts.allow_unsigned + } + task_id = session.signedRepo(tag, keys, **opts) + print "Creating signed repo for tag " + tag + if _running_in_bg() or task_opts.nowait: + return + else: + session.logout() + return watch_tasks(session, [task_id], quiet=options.quiet) + + def anon_handle_search(options, session, args): "Search the system" usage = _("usage: %prog search [options] search_type pattern") diff --git a/docs/schema.sql b/docs/schema.sql index 3d57d33..ec6fb19 100644 --- a/docs/schema.sql +++ b/docs/schema.sql @@ -51,6 +51,7 @@ CREATE TABLE permissions ( INSERT INTO permissions (name) VALUES ('admin'); INSERT INTO permissions (name) VALUES ('build'); INSERT INTO permissions (name) VALUES ('repo'); +INSERT INTO permissions (name) VALUES ('image'); INSERT INTO permissions (name) VALUES ('livecd'); INSERT INTO permissions (name) VALUES ('maven-import'); INSERT INTO permissions (name) VALUES ('win-import'); @@ -387,7 +388,8 @@ CREATE TABLE repo ( id SERIAL NOT NULL PRIMARY KEY, create_event INTEGER NOT NULL REFERENCES events(id) DEFAULT get_event(), tag_id INTEGER NOT NULL REFERENCES tag(id), - state INTEGER + state INTEGER, + signed BOOLEAN DEFAULT 'false' ) WITHOUT OIDS; -- external yum repos diff --git a/hub/kojihub.py b/hub/kojihub.py index c84db08..f1ca43a 100644 --- a/hub/kojihub.py +++ b/hub/kojihub.py @@ -2345,6 +2345,38 @@ def _write_maven_repo_metadata(destdir, artifacts): mdfile.close() _generate_maven_metadata(destdir) +def signed_repo_init(tag, keys, task_opts): + """Create a new repo entry in the INIT state, return full repo data""" + logger = logging.getLogger("koji.hub.signed_repo_init") + state = koji.REPO_INIT + tinfo = get_tag(tag, strict=True) + koji.plugin.run_callbacks('preRepoInit', tag=tinfo, keys=keys, repo_id=None) + tag_id = tinfo['id'] + repo_id = _singleValue("SELECT nextval('repo_id_seq')") + repo_arches = task_opts['arch'] + arches = set([]) + for arch in repo_arches: + arches.add(koji.canonArch(arch)) + if not task_opts['event']: + task_opts['event'] = _singleValue("SELECT get_event()") + insert = InsertProcessor('repo') + insert.set(id=repo_id, create_event=task_opts['event'], tag_id=tag_id, + state=state, signed=True) + insert.execute() + repodir = koji.pathinfo.signedrepo(repo_id, tinfo['name']) + for arch in arches: + koji.ensuredir(os.path.join(repodir, arch)) + # handle comps + if task_opts['comps']: + groupsdir = os.path.join(repodir, 'groups') + koji.ensuredir(groupsdir) + shutil.copyfile(os.path.join(koji.pathinfo.work(), + task_opts['comps']), groupsdir + '/comps.xml') + koji.plugin.run_callbacks('postRepoInit', tag=tinfo, + event=task_opts['event'], repo_id=repo_id) + return repo_id, task_opts['event'] + + def repo_set_state(repo_id, state, check=True): """Set repo state""" if check: @@ -2366,6 +2398,7 @@ def repo_info(repo_id, strict=False): ('EXTRACT(EPOCH FROM events.time)','create_ts'), ('repo.tag_id', 'tag_id'), ('tag.name', 'tag_name'), + ('repo.signed', 'signed'), ) q = """SELECT %s FROM repo JOIN tag ON tag_id=tag.id @@ -9498,16 +9531,20 @@ class RootExports(object): taginfo['extra'][key] = ancestor['extra'][key] return taginfo - def getRepo(self,tag,state=None,event=None): - if isinstance(tag,int): + def getRepo(self, tag, state=None, event=None, signed=False): + if isinstance(tag, int): id = tag else: - id = get_tag_id(tag,strict=True) + id = get_tag_id(tag, strict=True) - fields = ['repo.id', 'repo.state', 'repo.create_event', 'events.time', 'EXTRACT(EPOCH FROM events.time)'] - aliases = ['id', 'state', 'create_event', 'creation_time', 'create_ts'] + fields = ['repo.id', 'repo.state', 'repo.create_event', 'events.time', 'EXTRACT(EPOCH FROM events.time)', 'repo.signed'] + aliases = ['id', 'state', 'create_event', 'creation_time', 'create_ts', 'signed'] joins = ['events ON repo.create_event = events.id'] clauses = ['repo.tag_id = %(id)i'] + if signed: + clauses.append('repo.signed is true') + else: + clauses.append('repo.signed is false') if event: # the repo table doesn't have all the fields of a _config table, just create_event clauses.append('create_event <= %(event)i') @@ -9525,6 +9562,13 @@ class RootExports(object): repoInfo = staticmethod(repo_info) getActiveRepos = staticmethod(get_active_repos) + def signedRepo(self, tag, keys, **task_opts): + """Create a signed-repo task. returns task id""" + context.session.assertPerm('signed-repo') + repo_id, event_id = signed_repo_init(tag, keys, task_opts) + task_opts['event'] = event_id + return make_task('signedRepo', [tag, repo_id, keys, task_opts], priority=15) + def newRepo(self, tag, event=None, src=False, debuginfo=False): """Create a newRepo task. returns task id""" if context.session.hasPerm('regen-repo'): @@ -11572,12 +11616,13 @@ class HostExports(object): else: os.link(filepath, dst) - def repoDone(self, repo_id, data, expire=False): + def repoDone(self, repo_id, data, expire=False, signed=False): """Move repo data into place, mark as ready, and expire earlier repos repo_id: the id of the repo data: a dictionary of the form { arch: (uploadpath, files), ...} expire(optional): if set to true, mark the repo expired immediately* + signed(optional): if true, hardlink signed rpms in the final directory * This is used when a repo from an older event is generated """ @@ -11589,19 +11634,23 @@ class HostExports(object): raise koji.GenericError, "Repo %(id)s not in INIT state (got %(state)s)" % rinfo repodir = koji.pathinfo.repo(repo_id, rinfo['tag_name']) workdir = koji.pathinfo.work() - for arch, (uploadpath, files) in data.iteritems(): - archdir = "%s/%s" % (repodir, arch) - if not os.path.isdir(archdir): - raise koji.GenericError, "Repo arch directory missing: %s" % archdir - datadir = "%s/repodata" % archdir - koji.ensuredir(datadir) - for fn in files: - src = "%s/%s/%s" % (workdir,uploadpath, fn) - dst = "%s/%s" % (datadir, fn) - if not os.path.exists(src): - raise koji.GenericError, "uploaded file missing: %s" % src - os.link(src, dst) - os.unlink(src) + if not signed: + for arch, (uploadpath, files) in data.iteritems(): + archdir = "%s/%s" % (repodir, koji.canonArch(arch)) + if not os.path.isdir(archdir): + raise koji.GenericError, "Repo arch directory missing: %s" % archdir + datadir = "%s/repodata" % archdir + koji.ensuredir(datadir) + for fn in files: + src = "%s/%s/%s" % (workdir,uploadpath, fn) + if fn.endswith('pkglist'): + dst = '%s/%s' % (archdir, fn) + else: + dst = "%s/%s" % (datadir, fn) + if not os.path.exists(src): + raise koji.GenericError, "uploaded file missing: %s" % src + os.link(src, dst) + os.unlink(src) if expire: repo_expire(repo_id) koji.plugin.run_callbacks('postRepoDone', repo=rinfo, data=data, expire=expire) @@ -11621,6 +11670,41 @@ class HostExports(object): log_error("Unable to create latest link for repo: %s" % repodir) koji.plugin.run_callbacks('postRepoDone', repo=rinfo, data=data, expire=expire) + def signedRepoMove(self, repo_id, uploadpath, files, arch, fullpaths): + """ + Very similar to repoDone, except only the uploads are completed. + fullpaths is a dict like so: rpm file name -> sig""" + workdir = koji.pathinfo.work() + rinfo = repo_info(repo_id, strict=True) + repodir = koji.pathinfo.signedrepo(repo_id, rinfo['tag_name']) + archdir = "%s/%s" % (repodir, koji.canonArch(arch)) + if not os.path.isdir(archdir): + raise koji.GenericError, "Repo arch directory missing: %s" % archdir + datadir = "%s/repodata" % archdir + koji.ensuredir(datadir) + for fn in files: + src = "%s/%s/%s" % (workdir, uploadpath, fn) + if fn.endswith('.drpm'): + koji.ensuredir(os.path.join(archdir, 'drpms')) + dst = "%s/drpms/%s" % (archdir, fn) + elif fn.endswith('pkglist'): + dst = '%s/%s' % (archdir, fn) + else: + dst = "%s/%s" % (datadir, fn) + if not os.path.exists(src): + raise koji.GenericError, "uploaded file missing: %s" % src + os.link(src, dst) + if fn.endswith('pkglist'): + # hardlink the found rpms into the final repodir + with open(src) as pkgfile: + for pkg in pkgfile: + pkg = os.path.basename(pkg.strip()) + rpmpath = fullpaths[pkg] + bnp = os.path.basename(rpmpath) + koji.ensuredir(os.path.join(archdir, bnp[0])) + os.link(rpmpath, os.path.join(archdir, bnp[0], bnp)) + os.unlink(src) + def isEnabled(self): host = Host() host.verify() diff --git a/koji/__init__.py b/koji/__init__.py index 4b2f7ee..e644887 100644 --- a/koji/__init__.py +++ b/koji/__init__.py @@ -1533,6 +1533,10 @@ class PathInfo(object): """Return the directory where a repo belongs""" return self.topdir + ("/repos/%(tag_str)s/%(repo_id)s" % locals()) + def signedrepo(self, repo_id, tag): + """Return the directory with a signed repo lives""" + return os.path.join(self.topdir, 'repos', 'signed', tag, str(repo_id)) + def repocache(self,tag_str): """Return the directory where a repo belongs""" return self.topdir + ("/repos/%(tag_str)s/cache" % locals()) @@ -2349,7 +2353,7 @@ def _taskLabel(taskInfo): if taskInfo.has_key('request'): build = taskInfo['request'][1] extra = buildLabel(build) - elif method == 'newRepo': + elif method in ('newRepo', 'signedRepo'): if taskInfo.has_key('request'): extra = str(taskInfo['request'][0]) elif method in ('tagBuild', 'tagNotification'): @@ -2360,10 +2364,15 @@ def _taskLabel(taskInfo): if taskInfo.has_key('request'): tagInfo = taskInfo['request'][0] extra = tagInfo['name'] - elif method == 'createrepo': + elif method in ('createrepo'): if taskInfo.has_key('request'): arch = taskInfo['request'][1] extra = arch + elif method in ('createsignedrepo'): + if taskInfo.has_key('request'): + repo_id = taskInfo['request'][1] + arch = taskInfo['request'][2] + extra = '%s, %s' % (repo_id, arch) elif method == 'dependantTask': if taskInfo.has_key('request'): extra = ', '.join([subtask[0] for subtask in taskInfo['request'][1]]) diff --git a/util/kojira b/util/kojira index 006b10f..93d0f59 100755 --- a/util/kojira +++ b/util/kojira @@ -134,7 +134,11 @@ class ManagedRepo(object): (self.tag_id, self.repo_id)) return False tag_name = tag_info['name'] - path = pathinfo.repo(self.repo_id, tag_name) + rinfo = self.session.repoInfo(self.repo_id, strict=True) + if rinfo['signed']: + path = pathinfo.signedrepo(self.repo_id, tag_name) + else: + path = pathinfo.repo(self.repo_id, tag_name) try: #also check dir age. We do this because a repo can be created from an older event #and should not be removed based solely on that event's timestamp. @@ -333,40 +337,51 @@ class RepoManager(object): finally: session.logout() - def pruneLocalRepos(self): + def pruneLocalRepos(self, topdir, timername): """Scan filesystem for repos and remove any deleted ones Also, warn about any oddities""" if self.delete_pids: #skip return - self.logger.debug("Scanning filesystem for repos") - topdir = "%s/repos" % pathinfo.topdir + self.logger.debug("Scanning %s for repos" % topdir) + self.logger.debug('max age allowed: %s seconds (from %s)' % + (getattr(self.options, timername), timername)) for tag in os.listdir(topdir): tagdir = "%s/%s" % (topdir, tag) if not os.path.isdir(tagdir): + self.logger.debug("%s is not a directory, skipping" % tagdir) continue for repo_id in os.listdir(tagdir): try: repo_id = int(repo_id) except ValueError: + self.logger.debug("%s not an int, skipping" % tagdir) + # This condition is how signed repos are not removed by + # the first call to this method. Although, if someone has + # tags that are just integers, that could be a problem. continue repodir = "%s/%s" % (tagdir, repo_id) if not os.path.isdir(repodir): + self.logger.debug("%s not a directory, skipping" % repodir) continue if self.repos.has_key(repo_id): #we're already managing it, no need to deal with it here + self.logger.debug("seen %s already, skipping" % repodir) continue try: dir_ts = os.stat(repodir).st_mtime except OSError: #just in case something deletes the repo out from under us + self.logger.debug("%s deleted already?!" % repodir) continue rinfo = self.session.repoInfo(repo_id) if rinfo is None: if not self.options.ignore_stray_repos: age = time.time() - dir_ts - if age > self.options.deleted_repo_lifetime: + self.logger.debug("did not expect %s; age: %s" % + (repodir, age)) + if age > getattr(self.options, timername): self.logger.info("Removing unexpected directory (no such repo): %s" % repodir) self.rmtree(repodir) continue @@ -375,11 +390,11 @@ class RepoManager(object): continue if rinfo['state'] in (koji.REPO_DELETED, koji.REPO_PROBLEM): age = time.time() - max(rinfo['create_ts'], dir_ts) - if age > self.options.deleted_repo_lifetime: + self.logger.debug("potential removal candidate: %s; age: %s" % (repodir, age)) + if age > getattr(self.options, timername): #XXX should really be called expired_repo_lifetime logger.info("Removing stray repo (state=%s): %s" % (koji.REPO_STATES[rinfo['state']], repodir)) self.rmtree(repodir) - pass def tagUseStats(self, tag_id): stats = self.tag_use_stats.get(tag_id) @@ -632,7 +647,9 @@ def main(options, session): repomgr.updateRepos() repomgr.checkQueue() repomgr.printState() - repomgr.pruneLocalRepos() + repodir = "%s/repos" % pathinfo.topdir + repomgr.pruneLocalRepos(repodir, 'deleted_repo_lifetime') + repomgr.pruneLocalRepos(repodir + '/signed', 'signed_repo_lifetime') if not curr_chk_thread.isAlive(): logger.error("Currency checker thread died. Restarting it.") curr_chk_thread = start_currency_checker(session, repomgr) @@ -725,6 +742,7 @@ def get_options(): 'delete_batch_size' : 3, 'deleted_repo_lifetime': 7*24*3600, #XXX should really be called expired_repo_lifetime + 'signed_repo_lifetime': 7*24*3600, 'sleeptime' : 15, 'cert': '/etc/kojira/client.crt', 'ca': '', # FIXME: unused, remove in next major release @@ -733,7 +751,8 @@ def get_options(): if config.has_section(section): int_opts = ('deleted_repo_lifetime', 'max_repo_tasks', 'repo_tasks_limit', 'retry_interval', 'max_retries', 'offline_retry_interval', - 'max_delete_processes', 'max_repo_tasks_maven', 'delete_batch_size', ) + 'max_delete_processes', 'max_repo_tasks_maven', + 'delete_batch_size', 'signed_repo_lifetime') str_opts = ('topdir', 'server', 'user', 'password', 'logfile', 'principal', 'keytab', 'krbservice', 'cert', 'ca', 'serverca', 'debuginfo_tags', 'source_tags') # FIXME: remove ca here bool_opts = ('with_src','verbose','debug','ignore_stray_repos', 'offline_retry') diff --git a/util/kojira.conf b/util/kojira.conf index def5370..1d361b3 100644 --- a/util/kojira.conf +++ b/util/kojira.conf @@ -39,3 +39,12 @@ with_src=no ;certificate of the CA that issued the HTTP server certificate ;serverca = /etc/kojira/serverca.crt + +;how soon (in seconds) to clean up expired repositories. 1 week default +;deleted_repo_lifetime = 604800 + +;how soon (in seconds) to clean up signed repositories. 1 week default here too +;signed_repo_lifetime = 604800 + +;turn on debugging statements in the log +;debug = false diff --git a/www/conf/web.conf b/www/conf/web.conf index 3bd6f91..b6300be 100644 --- a/www/conf/web.conf +++ b/www/conf/web.conf @@ -28,3 +28,8 @@ LibPath = /usr/share/koji-web/lib # If False, then the footer will be included as another Kid Template. # Defaults to True LiteralFooter = True + +# Uncommenting this will show python tracebacks in the webUI, but they are the +# same as what you will see in apache's error_log. +# Not for production use +#PythonDebug = True diff --git a/www/kojiweb/index.py b/www/kojiweb/index.py index 5d7b09e..9fc35dd 100644 --- a/www/kojiweb/index.py +++ b/www/kojiweb/index.py @@ -424,6 +424,8 @@ _TASKS = ['build', 'tagBuild', 'newRepo', 'createrepo', + 'signedRepo', + 'createsignedrepo', 'buildNotification', 'tagNotification', 'dependantTask', @@ -435,9 +437,9 @@ _TASKS = ['build', 'createImage', 'runroot'] # Tasks that can exist without a parent -_TOPLEVEL_TASKS = ['build', 'buildNotification', 'chainbuild', 'maven', 'chainmaven', 'wrapperRPM', 'winbuild', 'newRepo', 'tagBuild', 'tagNotification', 'waitrepo', 'livecd', 'appliance', 'image'] +_TOPLEVEL_TASKS = ['build', 'buildNotification', 'chainbuild', 'maven', 'chainmaven', 'wrapperRPM', 'winbuild', 'newRepo', 'signedRepo', 'tagBuild', 'tagNotification', 'waitrepo', 'livecd', 'appliance', 'image'] # Tasks that can have children -_PARENT_TASKS = ['build', 'chainbuild', 'maven', 'chainmaven', 'winbuild', 'newRepo', 'wrapperRPM', 'livecd', 'appliance', 'image'] +_PARENT_TASKS = ['build', 'chainbuild', 'maven', 'chainmaven', 'winbuild', 'newRepo', 'signedRepo', 'wrapperRPM', 'livecd', 'appliance', 'image'] def tasks(environ, owner=None, state='active', view='tree', method='all', hostID=None, channelID=None, start=None, order='-id'): values = _initValues(environ, 'Tasks', 'tasks') @@ -614,7 +616,7 @@ def taskinfo(environ, taskID): build = server.getBuild(params[1]) values['destTag'] = destTag values['build'] = build - elif task['method'] == 'newRepo': + elif task['method'] in ('newRepo', 'signedRepo', 'createsignedrepo'): tag = server.getTag(params[0]) values['tag'] = tag elif task['method'] == 'tagNotification': diff --git a/www/kojiweb/taskinfo.chtml b/www/kojiweb/taskinfo.chtml index 8db3571..3650837 100644 --- a/www/kojiweb/taskinfo.chtml +++ b/www/kojiweb/taskinfo.chtml @@ -216,11 +216,11 @@ $value #if $len($params) > 2 $printOpts($params[2]) #end if - #elif $task.method == 'newRepo' + #elif $task.method == 'signedRepo' Tag: $tag.name
- #if $len($params) > 1 - $printOpts($params[1]) - #end if + Repo ID: $params[1]
+ Keys: $printValue(0, $params[2])
+ $printOpts($params[3]) #elif $task.method == 'prepRepo' Tag: $params[0].name #elif $task.method == 'createrepo' @@ -228,12 +228,18 @@ $value Arch: $params[1]
#set $oldrepo = $params[2] #if $oldrepo - Old Repo ID: $oldrepo.id
- Old Repo Creation: $koji.formatTimeLong($oldrepo.creation_time)
+ Old Repo ID: $oldrepo.id
+ Old Repo Creation: $koji.formatTimeLong($oldrepo.creation_time)
#end if - #if $len($params) > 3 - External Repos: $printValue(None, [ext['external_repo_name'] for ext in $params[3]])
+ #if $len($params) > 4 and $params[4] + External Repos: $printValue(None, [ext['external_repo_name'] for ext in $params[3]])
#end if + #elif $task.method == 'createsignedrepo' + Tag: $tag.name
+ Repo ID: $params[1]
+ Arch: $printValue(0, $params[2])
+ Keys: $printValue(0, $params[3])
+ Options: $printMap($params[4], '    ') #elif $task.method == 'dependantTask' Dependant Tasks:
#for $dep in $deps From a18b1f1754c5653a60a8a252e502a4bf2ad921d4 Mon Sep 17 00:00:00 2001 From: Jay Greguske Date: Mar 29 2016 14:22:14 +0000 Subject: [PATCH 2/5] fix newRepo in webui --- diff --git a/www/kojiweb/taskinfo.chtml b/www/kojiweb/taskinfo.chtml index 3650837..06aa186 100644 --- a/www/kojiweb/taskinfo.chtml +++ b/www/kojiweb/taskinfo.chtml @@ -216,6 +216,11 @@ $value #if $len($params) > 2 $printOpts($params[2]) #end if + #elif $task.method == 'newRepo' + Tag: $tag.name
+ #if $len($params) > 1 + $printOpts($params[1]) + #end if #elif $task.method == 'signedRepo' Tag: $tag.name
Repo ID: $params[1]
From 1ab7ddafbc0e55013181fb3131f2962353dad4ef Mon Sep 17 00:00:00 2001 From: Jay Greguske Date: Apr 04 2016 18:03:13 +0000 Subject: [PATCH 3/5] Make the signedRepo tasks happen in the createrepo channe --- diff --git a/hub/kojihub.py b/hub/kojihub.py index f1ca43a..ff2ff8a 100644 --- a/hub/kojihub.py +++ b/hub/kojihub.py @@ -9567,7 +9567,7 @@ class RootExports(object): context.session.assertPerm('signed-repo') repo_id, event_id = signed_repo_init(tag, keys, task_opts) task_opts['event'] = event_id - return make_task('signedRepo', [tag, repo_id, keys, task_opts], priority=15) + return make_task('signedRepo', [tag, repo_id, keys, task_opts], priority=15, channel='createrepo') def newRepo(self, tag, event=None, src=False, debuginfo=False): """Create a newRepo task. returns task id""" From ca4af2d124f4ef5c819886ac9e2fe01a7091f119 Mon Sep 17 00:00:00 2001 From: Jay Greguske Date: Apr 04 2016 18:14:05 +0000 Subject: [PATCH 4/5] fall back to a copy if we cannot hardlink --- diff --git a/hub/kojihub.py b/hub/kojihub.py index ff2ff8a..6e132c5 100644 --- a/hub/kojihub.py +++ b/hub/kojihub.py @@ -11696,13 +11696,21 @@ class HostExports(object): os.link(src, dst) if fn.endswith('pkglist'): # hardlink the found rpms into the final repodir + # TODO: properly consider split-volume functionality with open(src) as pkgfile: for pkg in pkgfile: pkg = os.path.basename(pkg.strip()) rpmpath = fullpaths[pkg] bnp = os.path.basename(rpmpath) koji.ensuredir(os.path.join(archdir, bnp[0])) - os.link(rpmpath, os.path.join(archdir, bnp[0], bnp)) + try: + os.link(rpmpath, os.path.join(archdir, bnp[0], bnp)) + except OSError, ose: + if ose.error == 18: + shutil.copy2( + rpmpath, os.path.join(archdir, bnp[0], bnp)) + else: + raise ose os.unlink(src) def isEnabled(self): From 6c8576e6e546dadc0841dd4649a693e3a13f0bf1 Mon Sep 17 00:00:00 2001 From: Jay Greguske Date: Apr 04 2016 19:26:30 +0000 Subject: [PATCH 5/5] errno not error --- diff --git a/hub/kojihub.py b/hub/kojihub.py index 6e132c5..7aa3289 100644 --- a/hub/kojihub.py +++ b/hub/kojihub.py @@ -11706,7 +11706,7 @@ class HostExports(object): try: os.link(rpmpath, os.path.join(archdir, bnp[0], bnp)) except OSError, ose: - if ose.error == 18: + if ose.errno == 18: shutil.copy2( rpmpath, os.path.join(archdir, bnp[0], bnp)) else: