From 0a06ec131888473e5a681450f8764dbc7604a64a Mon Sep 17 00:00:00 2001 From: Jay Greguske Date: Mar 30 2017 13:37:56 +0000 Subject: [PATCH 1/77] implement CLI for signed-repos --- diff --git a/cli/koji b/cli/koji index 5a234fd..4eb49bc 100755 --- a/cli/koji +++ b/cli/koji @@ -7074,6 +7074,45 @@ def handle_regen_repo(options, session, args): session.logout() return watch_tasks(session, [task_id], quiet=options.quiet) +def handle_signed_repo(options, session, args): + """create a yum repo of GPG signed RPMs""" + usage = _("usage: %prog signed-repo [options] tag keyID [keyID...]") + usage += _("\n(Specify the --help option for a list of other options)") + parser = OptionParser(usage=usage) + parser.add_option("--arch", action='append', default=[], + help=_("Indicate an architecture to consider. The default is all architectures associated with the given tag. This option may be specified multiple times.")) + parser.add_option('--multilib', action='store_true', default=False, + help=_('Include multilib packages in the repository')) + parser.add_option("--noinherit", action='store_true', default=False, + help=_('Do not consider tag inheritance')) + parser.add_option("--nowait", action='store_true', default=False, + help=_('Do not wait for the task to complete')) + task_opts, args = parser.parse_args(args) + if len(args) < 2: + parser.error(_('You must provide a tag and 1 or more GPG key IDs')) + activate_session(session) + tag = args[0] + keys = args[1:] + taginfo = session.getTag(tag) + if not taginfo: + parser.error(_('unknown tag %s' % tag)) + if len(task_opts.arch) == 0: + task_opts.arch = taginfo['arches'] + if task_opts.arch == None: + parser.error(_('No arches given and no arches associated with tag')) + else: + for a in task_opts.arch: + if a not in taginfo['arches']: + print _('Warning: %s is not in the list of tag arches' % a) + task_id = session.signedRepo(tag, keys, **task_opts) + print "Creating signed repo for tag " + tag + if _running_in_bg() or task_opts.nowait: + return + else: + session.logout() + return watch_tasks(session, [task_id], quiet=options.quiet) + + def anon_handle_search(options, session, args): "[search] Search the system" usage = _("usage: %prog search [options] search_type pattern") From 53c379b6f874b90f51fa6327de00e7bca6dec650 Mon Sep 17 00:00:00 2001 From: Jay Greguske Date: Mar 30 2017 13:37:56 +0000 Subject: [PATCH 2/77] initial hub implementation for signed-repos --- diff --git a/cli/koji b/cli/koji index 4eb49bc..bb4e006 100755 --- a/cli/koji +++ b/cli/koji @@ -7085,6 +7085,10 @@ def handle_signed_repo(options, session, args): help=_('Include multilib packages in the repository')) parser.add_option("--noinherit", action='store_true', default=False, help=_('Do not consider tag inheritance')) + # TODO: accept comps + # TODO: accept events + # TODO: sources or no? + # TODO: latest? parser.add_option("--nowait", action='store_true', default=False, help=_('Do not wait for the task to complete')) task_opts, args = parser.parse_args(args) @@ -7102,9 +7106,14 @@ def handle_signed_repo(options, session, args): parser.error(_('No arches given and no arches associated with tag')) else: for a in task_opts.arch: - if a not in taginfo['arches']: + if not taginfo['arches'] or a not in taginfo['arches']: print _('Warning: %s is not in the list of tag arches' % a) - task_id = session.signedRepo(tag, keys, **task_opts) + opts = { + 'arch': task_opts.arch, + 'multilib': task_opts.multilib, + 'inherit': not task_opts.noinherit + } + task_id = session.signedRepo(tag, keys, **opts) print "Creating signed repo for tag " + tag if _running_in_bg() or task_opts.nowait: return diff --git a/hub/kojihub.py b/hub/kojihub.py index bd7fa89..934d71e 100644 --- a/hub/kojihub.py +++ b/hub/kojihub.py @@ -2442,6 +2442,79 @@ def _write_maven_repo_metadata(destdir, artifacts): mdfile.close() _generate_maven_metadata(destdir) +def signed_repo_init(tag, keys, task_opts): + """Create a new repo entry in the INIT state, return full repo data""" + logger = logging.getLogger("koji.hub.signed_repo_init") + state = koji.REPO_INIT + tinfo = get_tag(tag, strict=True) + koji.plugin.run_callbacks('preRepoInit', tag=tinfo, keys=keys, repo_id=None) + tag_id = tinfo['id'] + repo_arches = task_opts['arch'] + arches = set([]) + for arch in repo_arches: + arches.add(koji.canonArch(arch)) + repo_id = _singleValue("SELECT nextval('repo_id_seq')") + event_id = _singleValue("SELECT get_event()") + insert = InsertProcessor('repo') + insert.set(id=repo_id, create_event=event_id, tag_id=tag_id, state=state) + insert.execute() + # Need to pass event_id because even though this is a single transaction, + # it is possible to see the results of other committed transactions + rpms, builds = readTaggedRPMS(tag_id, event=event_id, + inherit=task_opts['inherit'], rpmsigs=True) + repodir = koji.pathinfo.signedrepo(tag, str(repo_id)) + os.makedirs(repodir) # should not already exist + + #get build dirs + relpathinfo = koji.PathInfo(topdir='toplink') + builddirs = {} + for build in builds: + relpath = relpathinfo.build(build) + builddirs[build['id']] = relpath.lstrip('/') + #generate pkglist files + pkglist = {} + for repoarch in arches: + archdir = os.path.join(repodir, repoarch) + koji.ensuredir(archdir) + # Make a symlink to our topdir + top_relpath = koji.util.relpath(koji.pathinfo.topdir, archdir) + top_link = os.path.join(archdir, 'toplink') + os.symlink(top_relpath, top_link) + pkglist[repoarch] = file(os.path.join(archdir, 'pkglist'), 'w') + #NOTE - rpms is now an iterator + preferred = {} + for rpminfo in rpms: + if rpminfo['sigkey'] == '': + # skip, this is the unsigned rpminfo + continue + if rpminfo['sigkey'] not in keys: + # skip, not a key we are looking for + continue + arch = koji.canonArch(rpminfo['arch']) + if arch not in arches and arch != 'noarch': + # not an architecture we care about + continue + idx = keys.index(rpminfo['sigkey']) + if preferred.has_key(rpminfo['id']): + if keys.index(preferred[rpminfo['id']]['sigkey']) <= idx: + # key for this is not as preferable as what we have seen before + continue + preferred[rpminfo['id']] = rpminfo + for rpminfo in preferred.values(): + relpath = "%s/%s\n" % (builddirs[rpminfo['build_id']], + relpathinfo.signed(rpminfo, rpminfo['sigkey'])) + if rpminfo['arch'] == 'noarch': + for repoarch in arches: + pkglist[repoarch].write(relpath) + else: + pkglist[rpminfo['arch']].write(relpath) + for repoarch in arches: + pkglist[repoarch].close() + koji.plugin.run_callbacks('postRepoInit', tag=tinfo, event=event_id, + repo_id=repo_id) + return [repo_id, event_id] + + def repo_set_state(repo_id, state, check=True): """Set repo state""" if check: @@ -10096,6 +10169,12 @@ class RootExports(object): repoInfo = staticmethod(repo_info) getActiveRepos = staticmethod(get_active_repos) + def signedRepo(self, tag, keys, **task_opts): + """Create a signed-repo task. returns task id""" + context.session.assertPerm('signed-repo') + repo_id = signed_repo_init(tag, keys, task_opts) + return make_task('signedRepo', repo_id, priority=15) + def newRepo(self, tag, event=None, src=False, debuginfo=False): """Create a newRepo task. returns task id""" if context.session.hasPerm('regen-repo'): diff --git a/koji/__init__.py b/koji/__init__.py index 014c457..c9ddf49 100644 --- a/koji/__init__.py +++ b/koji/__init__.py @@ -1815,6 +1815,10 @@ class PathInfo(object): """Return the directory where a repo belongs""" return self.topdir + ("/repos/%(tag_str)s/%(repo_id)s" % locals()) + def signedrepo(self, repo_id, tag): + """Return the directory with a signed repo lives""" + return os.path.join(self.topdir, 'repos', 'signed', tag, repo_id) + def repocache(self, tag_str): """Return the directory where a repo belongs""" return self.topdir + ("/repos/%(tag_str)s/cache" % locals()) From 76d8caf33b9a259184db0c0e4150813d3509cf94 Mon Sep 17 00:00:00 2001 From: Jay Greguske Date: Mar 30 2017 13:37:56 +0000 Subject: [PATCH 3/77] initial builder implementation for signed-repos --- diff --git a/builder/kojid b/builder/kojid index f76da1c..2e80efa 100755 --- a/builder/kojid +++ b/builder/kojid @@ -4812,14 +4812,17 @@ class CreaterepoTask(BaseTaskHandler): Methods = ['createrepo'] _taskWeight = 1.5 - def handler(self, repo_id, arch, oldrepo): + def getRepoPath(self, repo_id, tag): + return self.pathinfo.repo(repo_id, tag) + + def handler(self, repo_id, arch, oldrepo, do_external): #arch is the arch of the repo, not the task rinfo = self.session.repoInfo(repo_id, strict=True) if rinfo['state'] != koji.REPO_INIT: raise koji.GenericError("Repo %(id)s not in INIT state (got %(state)s)" % rinfo) self.repo_id = rinfo['id'] self.pathinfo = koji.PathInfo(self.options.topdir) - toprepodir = self.pathinfo.repo(repo_id, rinfo['tag_name']) + toprepodir = self.getRepoPath(repo_id, rinfo['tag_name']) self.repodir = '%s/%s' % (toprepodir, arch) if not os.path.isdir(self.repodir): raise koji.GenericError("Repo directory missing: %s" % self.repodir) @@ -4833,7 +4836,7 @@ class CreaterepoTask(BaseTaskHandler): self.create_local_repo(rinfo, arch, pkglist, groupdata, oldrepo) external_repos = self.session.getExternalRepoList(rinfo['tag_id'], event=rinfo['create_event']) - if external_repos: + if external_repos and do_external: self.merge_repos(external_repos, arch, groupdata) elif pkglist is None: fo = file(os.path.join(self.datadir, "EMPTY_REPO"), 'w') @@ -4920,6 +4923,45 @@ class CreaterepoTask(BaseTaskHandler): raise koji.GenericError('failed to merge repos: %s' \ % parseStatus(status, ' '.join(cmd))) + +class NewSignedRepoTask(BaseTaskHandler): + Methods = ['signedRepo'] + _taskWeight = 0.1 + + def handler(self, repo_id, tag): + # TODO: remember to use an event here + tinfo = self.session.getTag(tag, strict=True) + kwargs = {} + path = koji.pathinfo.signedrepo(repo_id, tinfo['name']) + if not os.path.isdir(path): + raise koji.GenericError, "Repo directory missing: %s" % path + arches = [] + for fn in os.listdir(path): + if os.path.isfile("%s/%s/pkglist" % (path, fn)): + arches.append(fn) + subtasks = {} + for arch in arches: + arglist = [repo_id, arch, None, False] # no old repo or external + subtasks[arch] = self.session.host.subtask( + method='createsignedrepo', arglist=arglist, label=arch, + parent=self.id, arch='noarch') + # wait for subtasks to finish + results = self.wait(subtasks.values(), all=True, failany=True) + data = {} + for (arch, task_id) in subtasks.iteritems(): + data[arch] = results[task_id] + self.logger.debug("DEBUG: %r : %r " % (arch, data[arch],)) + self.session.host.repoDone(repo_id, data, expire=True, signed=True) + return repo_id + + +class createSignedRepoTask(CreaterepoTask): + Methods = ['createsignedrepo'] + _taskWeight = 1.5 + + def getRepoPath(self, repo_id, tag): + return self.pathinfo.signedrepo(repo_id, tag) + class WaitrepoTask(BaseTaskHandler): Methods = ['waitrepo'] diff --git a/hub/kojihub.py b/hub/kojihub.py index 934d71e..bd9a8ab 100644 --- a/hub/kojihub.py +++ b/hub/kojihub.py @@ -2462,14 +2462,14 @@ def signed_repo_init(tag, keys, task_opts): # it is possible to see the results of other committed transactions rpms, builds = readTaggedRPMS(tag_id, event=event_id, inherit=task_opts['inherit'], rpmsigs=True) - repodir = koji.pathinfo.signedrepo(tag, str(repo_id)) + repodir = koji.pathinfo.signedrepo(repo_id, tinfo['name']) os.makedirs(repodir) # should not already exist #get build dirs - relpathinfo = koji.PathInfo(topdir='toplink') + pathinfo = koji.PathInfo() builddirs = {} for build in builds: - relpath = relpathinfo.build(build) + relpath = pathinfo.build(build) builddirs[build['id']] = relpath.lstrip('/') #generate pkglist files pkglist = {} @@ -2501,18 +2501,26 @@ def signed_repo_init(tag, keys, task_opts): continue preferred[rpminfo['id']] = rpminfo for rpminfo in preferred.values(): - relpath = "%s/%s\n" % (builddirs[rpminfo['build_id']], - relpathinfo.signed(rpminfo, rpminfo['sigkey'])) - if rpminfo['arch'] == 'noarch': + pkgpath = '%s/%s' % (builddirs[rpminfo['build_id']], + pathinfo.signed(rpminfo, rpminfo['sigkey'])) + repopath = '/' + pkgpath + repopath = repopath.replace(koji.pathinfo.topdir, 'toplink') + '\n' + arch = koji.canonArch(rpminfo['arch']) + if arch == 'noarch': for repoarch in arches: - pkglist[repoarch].write(relpath) + pkglist[repoarch].write(repopath) + archdir = os.path.join(repodir, repoarch) + os.link(pkgpath, + os.path.join(archdir, os.path.basename(pkgpath))) else: - pkglist[rpminfo['arch']].write(relpath) + pkglist[arch].write(repopath) + dest = os.path.join(repodir, arch, os.path.basename(pkgpath)) + os.link(pkgpath, dest) for repoarch in arches: pkglist[repoarch].close() koji.plugin.run_callbacks('postRepoInit', tag=tinfo, event=event_id, repo_id=repo_id) - return [repo_id, event_id] + return repo_id, event_id def repo_set_state(repo_id, state, check=True): @@ -10172,8 +10180,8 @@ class RootExports(object): def signedRepo(self, tag, keys, **task_opts): """Create a signed-repo task. returns task id""" context.session.assertPerm('signed-repo') - repo_id = signed_repo_init(tag, keys, task_opts) - return make_task('signedRepo', repo_id, priority=15) + repo_id, event_id = signed_repo_init(tag, keys, task_opts) + return make_task('signedRepo', [repo_id, tag], priority=15) def newRepo(self, tag, event=None, src=False, debuginfo=False): """Create a newRepo task. returns task id""" @@ -12298,7 +12306,7 @@ class HostExports(object): else: safer_move(filepath, dst) - def repoDone(self, repo_id, data, expire=False): + def repoDone(self, repo_id, data, expire=False, signed=False): """Move repo data into place, mark as ready, and expire earlier repos repo_id: the id of the repo @@ -12313,7 +12321,10 @@ class HostExports(object): koji.plugin.run_callbacks('preRepoDone', repo=rinfo, data=data, expire=expire) if rinfo['state'] != koji.REPO_INIT: raise koji.GenericError("Repo %(id)s not in INIT state (got %(state)s)" % rinfo) - repodir = koji.pathinfo.repo(repo_id, rinfo['tag_name']) + if signed: + repodir = koji.pathinfo.signedrepo(repo_id, rinfo['tag_name']) + else: + repodir = koji.pathinfo.repo(repo_id, rinfo['tag_name']) workdir = koji.pathinfo.work() for arch, (uploadpath, files) in data.iteritems(): archdir = "%s/%s" % (repodir, arch) diff --git a/koji/__init__.py b/koji/__init__.py index c9ddf49..22b494b 100644 --- a/koji/__init__.py +++ b/koji/__init__.py @@ -1817,7 +1817,7 @@ class PathInfo(object): def signedrepo(self, repo_id, tag): """Return the directory with a signed repo lives""" - return os.path.join(self.topdir, 'repos', 'signed', tag, repo_id) + return os.path.join(self.topdir, 'repos', 'signed', tag, str(repo_id)) def repocache(self, tag_str): """Return the directory where a repo belongs""" From f1a45e00244fefefeb452e232ec4e7be23831c9e Mon Sep 17 00:00:00 2001 From: Jay Greguske Date: Mar 30 2017 13:37:56 +0000 Subject: [PATCH 4/77] signed-repo kojiweb tweaks --- diff --git a/builder/kojid b/builder/kojid index 2e80efa..80519d2 100755 --- a/builder/kojid +++ b/builder/kojid @@ -4928,7 +4928,7 @@ class NewSignedRepoTask(BaseTaskHandler): Methods = ['signedRepo'] _taskWeight = 0.1 - def handler(self, repo_id, tag): + def handler(self, tag, repo_id): # TODO: remember to use an event here tinfo = self.session.getTag(tag, strict=True) kwargs = {} diff --git a/hub/kojihub.py b/hub/kojihub.py index bd9a8ab..0c5be1d 100644 --- a/hub/kojihub.py +++ b/hub/kojihub.py @@ -10181,7 +10181,7 @@ class RootExports(object): """Create a signed-repo task. returns task id""" context.session.assertPerm('signed-repo') repo_id, event_id = signed_repo_init(tag, keys, task_opts) - return make_task('signedRepo', [repo_id, tag], priority=15) + return make_task('signedRepo', [tag, repo_id], priority=15) def newRepo(self, tag, event=None, src=False, debuginfo=False): """Create a newRepo task. returns task id""" diff --git a/koji/__init__.py b/koji/__init__.py index 22b494b..849c7a9 100644 --- a/koji/__init__.py +++ b/koji/__init__.py @@ -2792,7 +2792,7 @@ def _taskLabel(taskInfo): if 'request' in taskInfo: build = taskInfo['request'][1] extra = buildLabel(build) - elif method == 'newRepo': + elif method in ('newRepo', 'signedRepo'): if 'request' in taskInfo: extra = str(taskInfo['request'][0]) elif method in ('tagBuild', 'tagNotification'): @@ -2803,7 +2803,7 @@ def _taskLabel(taskInfo): if 'request' in taskInfo: tagInfo = taskInfo['request'][0] extra = tagInfo['name'] - elif method == 'createrepo': + elif method in ('createrepo', 'createsignedrepo'): if 'request' in taskInfo: arch = taskInfo['request'][1] extra = arch diff --git a/www/conf/web.conf b/www/conf/web.conf index 2ec0959..db73a1e 100644 --- a/www/conf/web.conf +++ b/www/conf/web.conf @@ -42,3 +42,8 @@ LiteralFooter = True # ToplevelTasks = # Tasks that can have children # ParentTasks = + +# Uncommenting this will show python tracebacks in the webUI, but they are the +# same as what you will see in apache's error_log. +# Not for production use +# PythonDebug = True diff --git a/www/kojiweb/index.py b/www/kojiweb/index.py index 5ea585d..569e95b 100644 --- a/www/kojiweb/index.py +++ b/www/kojiweb/index.py @@ -431,6 +431,8 @@ _TASKS = ['build', 'tagBuild', 'newRepo', 'createrepo', + 'signedRepo', + 'createsignedrepo', 'buildNotification', 'tagNotification', 'dependantTask', @@ -444,9 +446,9 @@ _TASKS = ['build', 'livemedia', 'createLiveMedia'] # Tasks that can exist without a parent -_TOPLEVEL_TASKS = ['build', 'buildNotification', 'chainbuild', 'maven', 'chainmaven', 'wrapperRPM', 'winbuild', 'newRepo', 'tagBuild', 'tagNotification', 'waitrepo', 'livecd', 'appliance', 'image', 'livemedia'] +_TOPLEVEL_TASKS = ['build', 'buildNotification', 'chainbuild', 'maven', 'chainmaven', 'wrapperRPM', 'winbuild', 'newRepo', 'signedRepo', 'tagBuild', 'tagNotification', 'waitrepo', 'livecd', 'appliance', 'image', 'livemedia'] # Tasks that can have children -_PARENT_TASKS = ['build', 'chainbuild', 'maven', 'chainmaven', 'winbuild', 'newRepo', 'wrapperRPM', 'livecd', 'appliance', 'image', 'livemedia'] +_PARENT_TASKS = ['build', 'chainbuild', 'maven', 'chainmaven', 'winbuild', 'newRepo', 'signedRepo', 'wrapperRPM', 'livecd', 'appliance', 'image', 'livemedia'] def tasks(environ, owner=None, state='active', view='tree', method='all', hostID=None, channelID=None, start=None, order='-id'): values = _initValues(environ, 'Tasks', 'tasks') @@ -623,7 +625,7 @@ def taskinfo(environ, taskID): build = server.getBuild(params[1]) values['destTag'] = destTag values['build'] = build - elif task['method'] == 'newRepo': + elif task['method'] in ('newRepo', 'signedRepo'): tag = server.getTag(params[0]) values['tag'] = tag elif task['method'] == 'tagNotification': diff --git a/www/kojiweb/taskinfo.chtml b/www/kojiweb/taskinfo.chtml index e12a64b..8f22eac 100644 --- a/www/kojiweb/taskinfo.chtml +++ b/www/kojiweb/taskinfo.chtml @@ -218,23 +218,27 @@ $value #if $len($params) > 2 $printOpts($params[2]) #end if - #elif $task.method == 'newRepo' + #elif $task.method in ('newRepo', 'signedRepo') Tag: $tag.name
- #if $len($params) > 1 - $printOpts($params[1]) + #if $task.method == 'signedRepo' + Repo ID: $params[1]
+ #elif $len($params) > 1 + $printOpts($params[1]) #end if #elif $task.method == 'prepRepo' Tag: $params[0].name - #elif $task.method == 'createrepo' + #elif $task.method in ('createrepo', 'createsignedrepo') Repo ID: $params[0]
Arch: $params[1]
- #set $oldrepo = $params[2] - #if $oldrepo - Old Repo ID: $oldrepo.id
- Old Repo Creation: $koji.formatTimeLong($oldrepo.creation_time)
+ #if $len($params) > 2 + #set $oldrepo = $params[2] + #if $oldrepo + Old Repo ID: $oldrepo.id
+ Old Repo Creation: $koji.formatTimeLong($oldrepo.creation_time)
+ #end if #end if - #if $len($params) > 3 - External Repos: $printValue(None, [ext['external_repo_name'] for ext in $params[3]])
+ #if $len($params) > 3 and $params[3] + External Repos: $printValue(None, [ext['external_repo_name'] for ext in $params[3]])
#end if #elif $task.method == 'dependantTask' Dependant Tasks:
From 08259b426e0b149e150eaffff584f0e6cc0d4813 Mon Sep 17 00:00:00 2001 From: Jay Greguske Date: Mar 30 2017 13:37:56 +0000 Subject: [PATCH 5/77] implement --allow-unsigned and --skip-unsigned --- diff --git a/builder/kojid b/builder/kojid index 80519d2..47803d3 100755 --- a/builder/kojid +++ b/builder/kojid @@ -4928,10 +4928,9 @@ class NewSignedRepoTask(BaseTaskHandler): Methods = ['signedRepo'] _taskWeight = 0.1 - def handler(self, tag, repo_id): + def handler(self, tag, repo_id, task_opts): # TODO: remember to use an event here tinfo = self.session.getTag(tag, strict=True) - kwargs = {} path = koji.pathinfo.signedrepo(repo_id, tinfo['name']) if not os.path.isdir(path): raise koji.GenericError, "Repo directory missing: %s" % path diff --git a/cli/koji b/cli/koji index bb4e006..fb8a521 100755 --- a/cli/koji +++ b/cli/koji @@ -7079,8 +7079,12 @@ def handle_signed_repo(options, session, args): usage = _("usage: %prog signed-repo [options] tag keyID [keyID...]") usage += _("\n(Specify the --help option for a list of other options)") parser = OptionParser(usage=usage) + parser.add_option('--allow-unsigned', action='store_true', default=False, + help=_('Use unsigned RPMs if none are available with the right key')) parser.add_option("--arch", action='append', default=[], - help=_("Indicate an architecture to consider. The default is all architectures associated with the given tag. This option may be specified multiple times.")) + help=_("Indicate an architecture to consider. The default is all " + + "architectures associated with the given tag. This option may " + + "be specified multiple times.")) parser.add_option('--multilib', action='store_true', default=False, help=_('Include multilib packages in the repository')) parser.add_option("--noinherit", action='store_true', default=False, @@ -7089,11 +7093,16 @@ def handle_signed_repo(options, session, args): # TODO: accept events # TODO: sources or no? # TODO: latest? + # TODO: delta-rpms ugh parser.add_option("--nowait", action='store_true', default=False, help=_('Do not wait for the task to complete')) + parser.add_option('--skip-unsigned', action='store_true', default=False, + help=_('Skip RPMs not signed with the desired key(s)')) task_opts, args = parser.parse_args(args) if len(args) < 2: parser.error(_('You must provide a tag and 1 or more GPG key IDs')) + if task_opts.allow_unsigned and task_opts.skip_unsigned: + parser.error(_('allow_signed and skip_unsigned are mutually exclusive')) activate_session(session) tag = args[0] keys = args[1:] @@ -7108,10 +7117,17 @@ def handle_signed_repo(options, session, args): for a in task_opts.arch: if not taginfo['arches'] or a not in taginfo['arches']: print _('Warning: %s is not in the list of tag arches' % a) + try: + task_opts.arch.remove('noarch') # handled specifically + task_opts.arch.remove('src') # ditto + except ValueError: + pass opts = { 'arch': task_opts.arch, 'multilib': task_opts.multilib, - 'inherit': not task_opts.noinherit + 'inherit': not task_opts.noinherit, + 'skip': task_opts.skip_unsigned, + 'unsigned': task_opts.allow_unsigned } task_id = session.signedRepo(tag, keys, **opts) print "Creating signed repo for tag " + tag diff --git a/hub/kojihub.py b/hub/kojihub.py index 0c5be1d..6724bd2 100644 --- a/hub/kojihub.py +++ b/hub/kojihub.py @@ -2460,8 +2460,15 @@ def signed_repo_init(tag, keys, task_opts): insert.execute() # Need to pass event_id because even though this is a single transaction, # it is possible to see the results of other committed transactions - rpms, builds = readTaggedRPMS(tag_id, event=event_id, + rpm_iter, builds = readTaggedRPMS(tag_id, event=event_id, inherit=task_opts['inherit'], rpmsigs=True) + rpms = list(rpm_iter) + for rpm_copy in list(rpms): + arch = koji.canonArch(rpm_copy['arch']) + if arch not in arches: + # not an architecture we care about + rpms.remove(rpm_copy) + need = set(['%(name)s-%(version)s-%(release)s.%(arch)s.rpm' % r for r in rpms]) repodir = koji.pathinfo.signedrepo(repo_id, tinfo['name']) os.makedirs(repodir) # should not already exist @@ -2481,28 +2488,32 @@ def signed_repo_init(tag, keys, task_opts): top_link = os.path.join(archdir, 'toplink') os.symlink(top_relpath, top_link) pkglist[repoarch] = file(os.path.join(archdir, 'pkglist'), 'w') - #NOTE - rpms is now an iterator preferred = {} + if task_opts['unsigned']: + keys.append('') # make unsigned rpms the least preferred for rpminfo in rpms: - if rpminfo['sigkey'] == '': + if rpminfo['sigkey'] == '' and not task_opts['unsigned']: # skip, this is the unsigned rpminfo continue if rpminfo['sigkey'] not in keys: # skip, not a key we are looking for continue - arch = koji.canonArch(rpminfo['arch']) - if arch not in arches and arch != 'noarch': - # not an architecture we care about - continue idx = keys.index(rpminfo['sigkey']) if preferred.has_key(rpminfo['id']): if keys.index(preferred[rpminfo['id']]['sigkey']) <= idx: # key for this is not as preferable as what we have seen before continue preferred[rpminfo['id']] = rpminfo + seen = set() for rpminfo in preferred.values(): - pkgpath = '%s/%s' % (builddirs[rpminfo['build_id']], - pathinfo.signed(rpminfo, rpminfo['sigkey'])) + if rpminfo['sigkey'] == '': + # we're taking an unsigned rpm (--allow-unsigned) + pkgpath = '%s/%s' % (builddirs[rpminfo['build_id']], + pathinfo.rpm(rpminfo)) + else: + pkgpath = '%s/%s' % (builddirs[rpminfo['build_id']], + pathinfo.signed(rpminfo, rpminfo['sigkey'])) + seen.add(os.path.basename(pkgpath)) repopath = '/' + pkgpath repopath = repopath.replace(koji.pathinfo.topdir, 'toplink') + '\n' arch = koji.canonArch(rpminfo['arch']) @@ -2518,6 +2529,12 @@ def signed_repo_init(tag, keys, task_opts): os.link(pkgpath, dest) for repoarch in arches: pkglist[repoarch].close() + if not task_opts['skip']: + missing = list(need - seen) + if len(missing) != 0: + missing.sort() + raise koji.GenericError('Unsigned packages found: ' + + '\n'.join(missing)) koji.plugin.run_callbacks('postRepoInit', tag=tinfo, event=event_id, repo_id=repo_id) return repo_id, event_id @@ -10181,7 +10198,7 @@ class RootExports(object): """Create a signed-repo task. returns task id""" context.session.assertPerm('signed-repo') repo_id, event_id = signed_repo_init(tag, keys, task_opts) - return make_task('signedRepo', [tag, repo_id], priority=15) + return make_task('signedRepo', [tag, repo_id, task_opts], priority=15) def newRepo(self, tag, event=None, src=False, debuginfo=False): """Create a newRepo task. returns task id""" diff --git a/www/kojiweb/taskinfo.chtml b/www/kojiweb/taskinfo.chtml index 8f22eac..8b5b41d 100644 --- a/www/kojiweb/taskinfo.chtml +++ b/www/kojiweb/taskinfo.chtml @@ -222,6 +222,7 @@ $value Tag: $tag.name
#if $task.method == 'signedRepo' Repo ID: $params[1]
+ $printOpts($params[2]) #elif $len($params) > 1 $printOpts($params[1]) #end if From e611febbcd11a159576a730b373a17b659de205a Mon Sep 17 00:00:00 2001 From: Jay Greguske Date: Mar 30 2017 13:37:56 +0000 Subject: [PATCH 6/77] implement --event --- diff --git a/builder/kojid b/builder/kojid index 47803d3..f0e6067 100755 --- a/builder/kojid +++ b/builder/kojid @@ -4929,8 +4929,7 @@ class NewSignedRepoTask(BaseTaskHandler): _taskWeight = 0.1 def handler(self, tag, repo_id, task_opts): - # TODO: remember to use an event here - tinfo = self.session.getTag(tag, strict=True) + tinfo = self.session.getTag(tag, strict=True, event=task_opts['event']) path = koji.pathinfo.signedrepo(repo_id, tinfo['name']) if not os.path.isdir(path): raise koji.GenericError, "Repo directory missing: %s" % path @@ -4951,7 +4950,7 @@ class NewSignedRepoTask(BaseTaskHandler): data[arch] = results[task_id] self.logger.debug("DEBUG: %r : %r " % (arch, data[arch],)) self.session.host.repoDone(repo_id, data, expire=True, signed=True) - return repo_id + return repo_id, task_opts['event'] class createSignedRepoTask(CreaterepoTask): diff --git a/cli/koji b/cli/koji index fb8a521..2a2925b 100755 --- a/cli/koji +++ b/cli/koji @@ -7085,13 +7085,13 @@ def handle_signed_repo(options, session, args): help=_("Indicate an architecture to consider. The default is all " + "architectures associated with the given tag. This option may " + "be specified multiple times.")) + parser.add_option('--event', type='int', + help=_('create a signed repository based on a Brew event')) parser.add_option('--multilib', action='store_true', default=False, help=_('Include multilib packages in the repository')) parser.add_option("--noinherit", action='store_true', default=False, help=_('Do not consider tag inheritance')) # TODO: accept comps - # TODO: accept events - # TODO: sources or no? # TODO: latest? # TODO: delta-rpms ugh parser.add_option("--nowait", action='store_true', default=False, @@ -7124,6 +7124,7 @@ def handle_signed_repo(options, session, args): pass opts = { 'arch': task_opts.arch, + 'event': task_opts.event, 'multilib': task_opts.multilib, 'inherit': not task_opts.noinherit, 'skip': task_opts.skip_unsigned, diff --git a/hub/kojihub.py b/hub/kojihub.py index 6724bd2..5c58fb5 100644 --- a/hub/kojihub.py +++ b/hub/kojihub.py @@ -2454,13 +2454,15 @@ def signed_repo_init(tag, keys, task_opts): for arch in repo_arches: arches.add(koji.canonArch(arch)) repo_id = _singleValue("SELECT nextval('repo_id_seq')") - event_id = _singleValue("SELECT get_event()") + if not task_opts['event']: + task_opts['event'] = _singleValue("SELECT get_event()") insert = InsertProcessor('repo') - insert.set(id=repo_id, create_event=event_id, tag_id=tag_id, state=state) + insert.set(id=repo_id, create_event=task_opts['event'], tag_id=tag_id, + state=state) insert.execute() # Need to pass event_id because even though this is a single transaction, # it is possible to see the results of other committed transactions - rpm_iter, builds = readTaggedRPMS(tag_id, event=event_id, + rpm_iter, builds = readTaggedRPMS(tag_id, event=task_opts['event'], inherit=task_opts['inherit'], rpmsigs=True) rpms = list(rpm_iter) for rpm_copy in list(rpms): @@ -2535,9 +2537,9 @@ def signed_repo_init(tag, keys, task_opts): missing.sort() raise koji.GenericError('Unsigned packages found: ' + '\n'.join(missing)) - koji.plugin.run_callbacks('postRepoInit', tag=tinfo, event=event_id, - repo_id=repo_id) - return repo_id, event_id + koji.plugin.run_callbacks('postRepoInit', tag=tinfo, + event=task_opts['event'], repo_id=repo_id) + return repo_id, task_opts['event'] def repo_set_state(repo_id, state, check=True): From 506ca5418062fa7e646c49e07590c6f23e5f6d8d Mon Sep 17 00:00:00 2001 From: Jay Greguske Date: Mar 30 2017 13:37:56 +0000 Subject: [PATCH 7/77] kojira policy for signed repos --- diff --git a/util/kojira b/util/kojira index 2d72f67..611edd1 100755 --- a/util/kojira +++ b/util/kojira @@ -333,40 +333,51 @@ class RepoManager(object): finally: session.logout() - def pruneLocalRepos(self): + def pruneLocalRepos(self, topdir, timername): """Scan filesystem for repos and remove any deleted ones Also, warn about any oddities""" if self.delete_pids: #skip return - self.logger.debug("Scanning filesystem for repos") - topdir = "%s/repos" % pathinfo.topdir + self.logger.debug("Scanning %s for repos" % topdir) + self.logger.debug('max age allowed: %s seconds (from %s)' % + (getattr(self.options, timername), timername)) for tag in os.listdir(topdir): tagdir = "%s/%s" % (topdir, tag) if not os.path.isdir(tagdir): + self.logger.debug("%s is not a directory, skipping" % tagdir) continue for repo_id in os.listdir(tagdir): try: repo_id = int(repo_id) except ValueError: + self.logger.debug("%s not an int, skipping" % tagdir) + # This condition is how signed repos are not removed by + # the first call to this method. Although, if someone has + # tags that are just integers, that could be a problem. continue repodir = "%s/%s" % (tagdir, repo_id) if not os.path.isdir(repodir): + self.logger.debug("%s not a directory, skipping" % repodir) continue if repo_id in self.repos: #we're already managing it, no need to deal with it here + self.logger.debug("seen %s already, skipping" % repodir) continue try: dir_ts = os.stat(repodir).st_mtime except OSError: #just in case something deletes the repo out from under us + self.logger.debug("%s deleted already?!" % repodir) continue rinfo = self.session.repoInfo(repo_id) if rinfo is None: if not self.options.ignore_stray_repos: age = time.time() - dir_ts - if age > self.options.deleted_repo_lifetime: + self.logger.debug("did not expect %s; age: %s" % + (repodir, age)) + if age > getattr(self.options, timername): self.logger.info("Removing unexpected directory (no such repo): %s" % repodir) self.rmtree(repodir) continue @@ -375,11 +386,11 @@ class RepoManager(object): continue if rinfo['state'] in (koji.REPO_DELETED, koji.REPO_PROBLEM): age = time.time() - max(rinfo['create_ts'], dir_ts) - if age > self.options.deleted_repo_lifetime: + self.logger.debug("potential removal candidate: %s; age: %s" % (repodir, age)) + if age > getattr(self.options, timername): #XXX should really be called expired_repo_lifetime logger.info("Removing stray repo (state=%s): %s" % (koji.REPO_STATES[rinfo['state']], repodir)) self.rmtree(repodir) - pass def tagUseStats(self, tag_id): stats = self.tag_use_stats.get(tag_id) @@ -632,7 +643,9 @@ def main(options, session): repomgr.updateRepos() repomgr.checkQueue() repomgr.printState() - repomgr.pruneLocalRepos() + repodir = "%s/repos" % pathinfo.topdir + repomgr.pruneLocalRepos(repodir, 'deleted_repo_lifetime') + repomgr.pruneLocalRepos(repodir + '/signed', 'signed_repo_lifetime') if not curr_chk_thread.isAlive(): logger.error("Currency checker thread died. Restarting it.") curr_chk_thread = start_currency_checker(session, repomgr) @@ -728,6 +741,7 @@ def get_options(): 'delete_batch_size' : 3, 'deleted_repo_lifetime': 7*24*3600, #XXX should really be called expired_repo_lifetime + 'signed_repo_lifetime': 7*24*3600, 'sleeptime' : 15, 'cert': None, 'ca': '', # FIXME: unused, remove in next major release @@ -736,7 +750,8 @@ def get_options(): if config.has_section(section): int_opts = ('deleted_repo_lifetime', 'max_repo_tasks', 'repo_tasks_limit', 'retry_interval', 'max_retries', 'offline_retry_interval', - 'max_delete_processes', 'max_repo_tasks_maven', 'delete_batch_size', ) + 'max_delete_processes', 'max_repo_tasks_maven', + 'delete_batch_size', 'signed_repo_lifetime') str_opts = ('topdir', 'server', 'user', 'password', 'logfile', 'principal', 'keytab', 'krbservice', 'cert', 'ca', 'serverca', 'debuginfo_tags', 'source_tags') # FIXME: remove ca here bool_opts = ('with_src','verbose','debug','ignore_stray_repos', 'offline_retry', diff --git a/util/kojira.conf b/util/kojira.conf index def5370..1d361b3 100644 --- a/util/kojira.conf +++ b/util/kojira.conf @@ -39,3 +39,12 @@ with_src=no ;certificate of the CA that issued the HTTP server certificate ;serverca = /etc/kojira/serverca.crt + +;how soon (in seconds) to clean up expired repositories. 1 week default +;deleted_repo_lifetime = 604800 + +;how soon (in seconds) to clean up signed repositories. 1 week default here too +;signed_repo_lifetime = 604800 + +;turn on debugging statements in the log +;debug = false From 43da89749f4749ed3e1a90890e37012e3032c92c Mon Sep 17 00:00:00 2001 From: Jay Greguske Date: Mar 30 2017 13:37:56 +0000 Subject: [PATCH 8/77] fix createrepo task breakage in webui --- diff --git a/www/kojiweb/taskinfo.chtml b/www/kojiweb/taskinfo.chtml index 8b5b41d..c68b5af 100644 --- a/www/kojiweb/taskinfo.chtml +++ b/www/kojiweb/taskinfo.chtml @@ -228,19 +228,21 @@ $value #end if #elif $task.method == 'prepRepo' Tag: $params[0].name - #elif $task.method in ('createrepo', 'createsignedrepo') + #elif $task.method == 'createrepo' Repo ID: $params[0]
Arch: $params[1]
- #if $len($params) > 2 - #set $oldrepo = $params[2] - #if $oldrepo - Old Repo ID: $oldrepo.id
- Old Repo Creation: $koji.formatTimeLong($oldrepo.creation_time)
- #end if + #set $oldrepo = $params[2] + #if $oldrepo + Old Repo ID: $oldrepo.id
+ Old Repo Creation: $koji.formatTimeLong($oldrepo.creation_time)
#end if - #if $len($params) > 3 and $params[3] + #if $len($params) > 4 and $params[4] External Repos: $printValue(None, [ext['external_repo_name'] for ext in $params[3]])
#end if + #elif $task.method == 'createsignedrepo' + Repo ID: $params[0]
+ Arch: $params[1]
+ Options: $printMap($params[3], '    ') #elif $task.method == 'dependantTask' Dependant Tasks:
#for $dep in $deps From 77bdd1e239589bf98b85ba35b9a2c1d620b6a32f Mon Sep 17 00:00:00 2001 From: Jay Greguske Date: Mar 30 2017 13:37:56 +0000 Subject: [PATCH 9/77] implement --delta-rpms --- diff --git a/builder/kojid b/builder/kojid index f0e6067..58ab107 100755 --- a/builder/kojid +++ b/builder/kojid @@ -4783,8 +4783,9 @@ class NewRepoTask(BaseTaskHandler): else: oldrepo = self.session.getRepo(tinfo['id'], state=koji.REPO_READY) subtasks = {} + opts = {'do_external': True, 'deltas': False} for arch in arches: - arglist = [repo_id, arch, oldrepo] + arglist = [repo_id, arch, oldrepo, opts] subtasks[arch] = self.session.host.subtask(method='createrepo', arglist=arglist, label=arch, @@ -4815,7 +4816,7 @@ class CreaterepoTask(BaseTaskHandler): def getRepoPath(self, repo_id, tag): return self.pathinfo.repo(repo_id, tag) - def handler(self, repo_id, arch, oldrepo, do_external): + def handler(self, repo_id, arch, oldrepo, opts): #arch is the arch of the repo, not the task rinfo = self.session.repoInfo(repo_id, strict=True) if rinfo['state'] != koji.REPO_INIT: @@ -4833,11 +4834,13 @@ class CreaterepoTask(BaseTaskHandler): pkglist = os.path.join(self.repodir, 'pkglist') if os.path.getsize(pkglist) == 0: pkglist = None - self.create_local_repo(rinfo, arch, pkglist, groupdata, oldrepo) - - external_repos = self.session.getExternalRepoList(rinfo['tag_id'], event=rinfo['create_event']) - if external_repos and do_external: - self.merge_repos(external_repos, arch, groupdata) + self.create_local_repo(rinfo, arch, pkglist, groupdata, oldrepo, + opts['deltas']) + if opts['do_external']: + external_repos = self.session.getExternalRepoList( + rinfo['tag_id'], event=rinfo['create_event']) + if external_repos: + self.merge_repos(external_repos, arch, groupdata) elif pkglist is None: fo = file(os.path.join(self.datadir, "EMPTY_REPO"), 'w') fo.write("This repo is empty because its tag has no content for this arch\n") @@ -4848,10 +4851,14 @@ class CreaterepoTask(BaseTaskHandler): for f in os.listdir(self.datadir): files.append(f) self.session.uploadWrapper('%s/%s' % (self.datadir, f), uploadpath, f) - + if opts['deltas']: + ddir = os.path.join(self.outdir, 'drpms') + for f in os.listdir(ddir): + files.append(f) + self.session.uploadWrapper('%s/%s' % (ddir, f), uploadpath, f) return [uploadpath, files] - def create_local_repo(self, rinfo, arch, pkglist, groupdata, oldrepo): + def create_local_repo(self, rinfo, arch, pkglist, groupdata, oldrepo, drpms): koji.ensuredir(self.outdir) if self.options.use_createrepo_c: cmd = ['/usr/bin/createrepo_c'] @@ -4863,7 +4870,9 @@ class CreaterepoTask(BaseTaskHandler): if os.path.isfile(groupdata): cmd.extend(['-g', groupdata]) #attempt to recycle repodata from last repo - if pkglist and oldrepo and self.options.createrepo_update: + if pkglist and oldrepo and self.options.createrepo_update and not drpms: + # signed repos overload the use of "oldrepo", so the conditional + # explicitly make sure this does not get executed with that on oldpath = self.pathinfo.repo(oldrepo['id'], rinfo['tag_name']) olddatadir = '%s/%s/repodata' % (oldpath, arch) if not os.path.isdir(olddatadir): @@ -4878,6 +4887,11 @@ class CreaterepoTask(BaseTaskHandler): cmd.append('--update') if self.options.createrepo_skip_stat: cmd.append('--skip-stat') + if drpms: + # generate delta-rpms + cmd.append('--deltas') + for repo in oldrepo: + cmd.extend(['--oldpackagedirs', repo]) # note: we can't easily use a cachedir because we do not have write # permission. The good news is that with --update we won't need to # be scanning many rpms. @@ -4938,8 +4952,15 @@ class NewSignedRepoTask(BaseTaskHandler): if os.path.isfile("%s/%s/pkglist" % (path, fn)): arches.append(fn) subtasks = {} + if task_opts['delta']: + make_drpms = True + oldrepo = task_opts['delta'] + else: + make_drpms = False + oldrepo = None for arch in arches: - arglist = [repo_id, arch, None, False] # no old repo or external + opts = {'do_external': False, 'deltas': make_drpms} + arglist = [repo_id, arch, oldrepo, opts] subtasks[arch] = self.session.host.subtask( method='createsignedrepo', arglist=arglist, label=arch, parent=self.id, arch='noarch') diff --git a/cli/koji b/cli/koji index 2a2925b..375a7e5 100755 --- a/cli/koji +++ b/cli/koji @@ -7085,6 +7085,9 @@ def handle_signed_repo(options, session, args): help=_("Indicate an architecture to consider. The default is all " + "architectures associated with the given tag. This option may " + "be specified multiple times.")) + parser.add_option('--delta-rpms', metavar='PATH',default=[], + action='append', + help=_('Create delta-rpms. PATH points to (older) rpms to generate against. May be specified multiple times.')) parser.add_option('--event', type='int', help=_('create a signed repository based on a Brew event')) parser.add_option('--multilib', action='store_true', default=False, @@ -7093,7 +7096,6 @@ def handle_signed_repo(options, session, args): help=_('Do not consider tag inheritance')) # TODO: accept comps # TODO: latest? - # TODO: delta-rpms ugh parser.add_option("--nowait", action='store_true', default=False, help=_('Do not wait for the task to complete')) parser.add_option('--skip-unsigned', action='store_true', default=False, @@ -7125,6 +7127,7 @@ def handle_signed_repo(options, session, args): opts = { 'arch': task_opts.arch, 'event': task_opts.event, + 'delta': task_opts.delta_rpms, 'multilib': task_opts.multilib, 'inherit': not task_opts.noinherit, 'skip': task_opts.skip_unsigned, diff --git a/hub/kojihub.py b/hub/kojihub.py index 5c58fb5..d3ae5cb 100644 --- a/hub/kojihub.py +++ b/hub/kojihub.py @@ -12353,7 +12353,11 @@ class HostExports(object): koji.ensuredir(datadir) for fn in files: src = "%s/%s/%s" % (workdir, uploadpath, fn) - dst = "%s/%s" % (datadir, fn) + if fn.endswith('.drpm'): + koji.ensuredir(os.path.join(archdir, 'drpms')) + dst = "%s/drpms/%s" % (archdir, fn) + else: + dst = "%s/%s" % (datadir, fn) if not os.path.exists(src): raise koji.GenericError("uploaded file missing: %s" % src) safer_move(src, dst) From e0e1dc15ee3f54ed128374d49f7cda9d6065733b Mon Sep 17 00:00:00 2001 From: Jay Greguske Date: Mar 30 2017 13:37:56 +0000 Subject: [PATCH 10/77] implement comps --- diff --git a/cli/koji b/cli/koji index 375a7e5..5a34e02 100755 --- a/cli/koji +++ b/cli/koji @@ -7085,6 +7085,7 @@ def handle_signed_repo(options, session, args): help=_("Indicate an architecture to consider. The default is all " + "architectures associated with the given tag. This option may " + "be specified multiple times.")) + parser.add_option('--comps', help='Include a comps file in the repodata') parser.add_option('--delta-rpms', metavar='PATH',default=[], action='append', help=_('Create delta-rpms. PATH points to (older) rpms to generate against. May be specified multiple times.')) @@ -7094,7 +7095,6 @@ def handle_signed_repo(options, session, args): help=_('Include multilib packages in the repository')) parser.add_option("--noinherit", action='store_true', default=False, help=_('Do not consider tag inheritance')) - # TODO: accept comps # TODO: latest? parser.add_option("--nowait", action='store_true', default=False, help=_('Do not wait for the task to complete')) @@ -7106,6 +7106,13 @@ def handle_signed_repo(options, session, args): if task_opts.allow_unsigned and task_opts.skip_unsigned: parser.error(_('allow_signed and skip_unsigned are mutually exclusive')) activate_session(session) + if task_opts.comps: + if not os.path.exists(task_opts.comps): + parser.error(_('could not find %s' % task_opts.comps)) + compsdir = _unique_path('cli-signed') + session.uploadWrapper(task_opts.comps, compsdir, + callback=_progress_callback) + task_opts.comps = os.path.join(compsdir, os.path.basename(task_opts.comps)) tag = args[0] keys = args[1:] taginfo = session.getTag(tag) @@ -7126,6 +7133,7 @@ def handle_signed_repo(options, session, args): pass opts = { 'arch': task_opts.arch, + 'comps': task_opts.comps, 'event': task_opts.event, 'delta': task_opts.delta_rpms, 'multilib': task_opts.multilib, diff --git a/hub/kojihub.py b/hub/kojihub.py index d3ae5cb..30879f3 100644 --- a/hub/kojihub.py +++ b/hub/kojihub.py @@ -2537,6 +2537,13 @@ def signed_repo_init(tag, keys, task_opts): missing.sort() raise koji.GenericError('Unsigned packages found: ' + '\n'.join(missing)) + + # handle comps + if task_opts['comps']: + groupsdir = os.path.join(repodir, 'groups') + koji.ensuredir(groupsdir) + shutil.copyfile(os.path.join(koji.pathinfo.work(), task_opts['comps']), + groupsdir + '/comps.xml') koji.plugin.run_callbacks('postRepoInit', tag=tinfo, event=task_opts['event'], repo_id=repo_id) return repo_id, task_opts['event'] From 8f638935b6c7bfe0123b887cba5833e9d74a8fc4 Mon Sep 17 00:00:00 2001 From: Jay Greguske Date: Mar 30 2017 13:37:56 +0000 Subject: [PATCH 11/77] move package addition logic to builder from hub --- diff --git a/builder/kojid b/builder/kojid index 58ab107..e6fdb95 100755 --- a/builder/kojid +++ b/builder/kojid @@ -35,6 +35,8 @@ import logging.handlers from koji.daemon import incremental_upload, log_output, TaskManager, SCM from koji.tasks import ServerExit, ServerRestart, BaseTaskHandler, MultiPlatformTask from koji.util import parseStatus, isSuccess, dslice, dslice_ex +import multilib +import multilib.fakepo import os import pwd import grp @@ -58,6 +60,7 @@ from fnmatch import fnmatch from gzip import GzipFile from optparse import OptionParser, SUPPRESS_HELP from yum import repoMDObject +import yum.packages #imports for LiveCD, LiveMedia, and Appliance handler image_enabled = False @@ -4783,9 +4786,8 @@ class NewRepoTask(BaseTaskHandler): else: oldrepo = self.session.getRepo(tinfo['id'], state=koji.REPO_READY) subtasks = {} - opts = {'do_external': True, 'deltas': False} for arch in arches: - arglist = [repo_id, arch, oldrepo, opts] + arglist = [repo_id, arch, oldrepo] subtasks[arch] = self.session.host.subtask(method='createrepo', arglist=arglist, label=arch, @@ -4813,17 +4815,14 @@ class CreaterepoTask(BaseTaskHandler): Methods = ['createrepo'] _taskWeight = 1.5 - def getRepoPath(self, repo_id, tag): - return self.pathinfo.repo(repo_id, tag) - - def handler(self, repo_id, arch, oldrepo, opts): + def handler(self, repo_id, arch, oldrepo): #arch is the arch of the repo, not the task rinfo = self.session.repoInfo(repo_id, strict=True) if rinfo['state'] != koji.REPO_INIT: raise koji.GenericError("Repo %(id)s not in INIT state (got %(state)s)" % rinfo) self.repo_id = rinfo['id'] self.pathinfo = koji.PathInfo(self.options.topdir) - toprepodir = self.getRepoPath(repo_id, rinfo['tag_name']) + toprepodir = self.pathinfo.repo(repo_id, rinfo['tag_name']) self.repodir = '%s/%s' % (toprepodir, arch) if not os.path.isdir(self.repodir): raise koji.GenericError("Repo directory missing: %s" % self.repodir) @@ -4834,13 +4833,11 @@ class CreaterepoTask(BaseTaskHandler): pkglist = os.path.join(self.repodir, 'pkglist') if os.path.getsize(pkglist) == 0: pkglist = None - self.create_local_repo(rinfo, arch, pkglist, groupdata, oldrepo, - opts['deltas']) - if opts['do_external']: - external_repos = self.session.getExternalRepoList( - rinfo['tag_id'], event=rinfo['create_event']) - if external_repos: - self.merge_repos(external_repos, arch, groupdata) + self.create_local_repo(rinfo, arch, pkglist, groupdata, oldrepo) + external_repos = self.session.getExternalRepoList( + rinfo['tag_id'], event=rinfo['create_event']) + if external_repos: + self.merge_repos(external_repos, arch, groupdata) elif pkglist is None: fo = file(os.path.join(self.datadir, "EMPTY_REPO"), 'w') fo.write("This repo is empty because its tag has no content for this arch\n") @@ -4851,14 +4848,9 @@ class CreaterepoTask(BaseTaskHandler): for f in os.listdir(self.datadir): files.append(f) self.session.uploadWrapper('%s/%s' % (self.datadir, f), uploadpath, f) - if opts['deltas']: - ddir = os.path.join(self.outdir, 'drpms') - for f in os.listdir(ddir): - files.append(f) - self.session.uploadWrapper('%s/%s' % (ddir, f), uploadpath, f) return [uploadpath, files] - def create_local_repo(self, rinfo, arch, pkglist, groupdata, oldrepo, drpms): + def create_local_repo(self, rinfo, arch, pkglist, groupdata, oldrepo, baseurl=None, drpms=False): koji.ensuredir(self.outdir) if self.options.use_createrepo_c: cmd = ['/usr/bin/createrepo_c'] @@ -4869,6 +4861,8 @@ class CreaterepoTask(BaseTaskHandler): cmd.extend(['-i', pkglist]) if os.path.isfile(groupdata): cmd.extend(['-g', groupdata]) + if baseurl: + cmd.extend(['-u', baseurl]) #attempt to recycle repodata from last repo if pkglist and oldrepo and self.options.createrepo_update and not drpms: # signed repos overload the use of "oldrepo", so the conditional @@ -4942,44 +4936,161 @@ class NewSignedRepoTask(BaseTaskHandler): Methods = ['signedRepo'] _taskWeight = 0.1 - def handler(self, tag, repo_id, task_opts): + def handler(self, tag, repo_id, keys, task_opts): tinfo = self.session.getTag(tag, strict=True, event=task_opts['event']) path = koji.pathinfo.signedrepo(repo_id, tinfo['name']) - if not os.path.isdir(path): - raise koji.GenericError, "Repo directory missing: %s" % path - arches = [] - for fn in os.listdir(path): - if os.path.isfile("%s/%s/pkglist" % (path, fn)): - arches.append(fn) + if len(task_opts['arch']) == 0: + task_opts['arch'] = tinfo['arches'].split() + if len(task_opts['arch']) == 0: + raise koji.GenericError('No arches specified nor for the tag!') subtasks = {} - if task_opts['delta']: - make_drpms = True - oldrepo = task_opts['delta'] - else: - make_drpms = False - oldrepo = None - for arch in arches: - opts = {'do_external': False, 'deltas': make_drpms} - arglist = [repo_id, arch, oldrepo, opts] + for arch in task_opts['arch']: + # call canonArch? + arglist = [tag, repo_id, arch, keys, task_opts] # no mergerepo subtasks[arch] = self.session.host.subtask( method='createsignedrepo', arglist=arglist, label=arch, parent=self.id, arch='noarch') # wait for subtasks to finish + self.logger.warn("5: %s" % subtasks.values()) results = self.wait(subtasks.values(), all=True, failany=True) + self.logger.warn("6") data = {} for (arch, task_id) in subtasks.iteritems(): data[arch] = results[task_id] self.logger.debug("DEBUG: %r : %r " % (arch, data[arch],)) self.session.host.repoDone(repo_id, data, expire=True, signed=True) - return repo_id, task_opts['event'] + return 'Signed repository #%s successfully generated' % repo_id class createSignedRepoTask(CreaterepoTask): Methods = ['createsignedrepo'] _taskWeight = 1.5 + archmap = {'s390x': 's390', 'ppc64': 'ppc', 'x86_64': 'i686'} + + def handler(self, tag, repo_id, arch, keys, opts): + #arch is the arch of the repo, not the task + rinfo = self.session.repoInfo(repo_id, strict=True) + if rinfo['state'] != koji.REPO_INIT: + raise koji.GenericError, "Repo %(id)s not in INIT state (got %(state)s)" % rinfo + self.repo_id = rinfo['id'] + self.pathinfo = koji.PathInfo(self.options.topdir) + groupdata = os.path.join( + self.pathinfo.signedrepo(repo_id, rinfo['tag_name']), + 'groups', 'comps.xml') + self.repodir = self.options.topdir # workaround for create_local_repo + #set up our output dir + self.outdir = '%s/repo' % self.workdir + self.datadir = '%s/repodata' % self.outdir + if len(opts['delta']) > 0: + for path in opts['delta']: + if not os.path.exists(path): + raise koji.GenericError( + 'drpm path %s does not exist!' % path) + pkglist = self.make_pkglist(tag, arch, keys, opts) + uploadpath = self.getUploadDir() + self.session.uploadWrapper(pkglist, uploadpath, + os.path.basename(pkglist)) + if os.path.getsize(pkglist) == 0: + pkglist = None + if len(opts['delta']) > 0: + do_drpms = True + else: + do_drpms = False + self.create_local_repo(rinfo, arch, pkglist, groupdata, opts['delta'], + drpms=do_drpms, baseurl='toplink') + if pkglist is None: + fo = file(os.path.join(self.datadir, "EMPTY_REPO"), 'w') + fo.write("This repo is empty because its tag has no content for this arch\n") + fo.close() + files = ['pkglist'] + for f in os.listdir(self.datadir): + files.append(f) + self.session.uploadWrapper('%s/%s' % (self.datadir, f), uploadpath, f) + if opts['delta']: + ddir = os.path.join(self.outdir, 'drpms') + for f in os.listdir(ddir): + files.append(f) + self.session.uploadWrapper('%s/%s' % (ddir, f), uploadpath, f) + return [uploadpath, files] + + def get_po(self, rpmpath): + """create a fake yum-like package object given an rpminfo dictionary""" + po = yum.packages.YumLocalPackage(filename=rpmpath) + return multilib.fakepo.FakePackageObject(po=po) + + def make_pkglist(self, tag_id, arch, keys, opts): + + def write_pkg(pkgpath): + self.logger.info('incoming: %s' % pkgpath) + self.logger.info('topdir: %s' % self.options.topdir) + newpath = pkgpath.replace(self.options.topdir, '') + '\n' + self.logger.info('outgoing: %s' % newpath) + pkglist.write(newpath) + + # Need to pass event_id because even though this is a single trans, + # it is possible to see the results of other committed transactions + rpm_iter, builds = self.session.listTaggedRPMS(tag_id, + event=opts['event'], arch=arch, + inherit=opts['inherit'], rpmsigs=True) + rpms = list(rpm_iter) + if opts['multilib']: + mlm = multilib.MultilibDevelMethod(opts['multilib']) + else: + # this method always returns False, no multilib packages added + mlm = multilib.NoMultilibMethod(opts['multilib']) + need = set(['%(name)s-%(version)s-%(release)s.%(arch)s.rpm' % r for r in rpms]) + #get build dirs + builddirs = {} + for build in builds: + builddirs[build['id']] = self.pathinfo.build(build) + #generate pkglist files + archdir = os.path.join(self.outdir, arch) + koji.ensuredir(archdir) + pkgfile = os.path.join(archdir, 'pkglist') + pkglist = file(pkgfile, 'w') + preferred = {} + if opts['unsigned']: + keys.append('') # make unsigned rpms the least preferred + for rpminfo in rpms: + if rpminfo['sigkey'] == '' and not opts['unsigned']: + # skip, this is the unsigned rpminfo + continue + if rpminfo['sigkey'] not in keys: + # skip, not a key we are looking for + continue + idx = keys.index(rpminfo['sigkey']) + if preferred.has_key(rpminfo['id']): + if keys.index(preferred[rpminfo['id']]['sigkey']) <= idx: + # key for this is not as preferable as what has been seen + continue + preferred[rpminfo['id']] = rpminfo + seen = set() + for rpminfo in preferred.values(): + if rpminfo['sigkey'] == '': + # we're taking an unsigned rpm (--allow-unsigned) + pkgpath = '%s/%s' % (builddirs[rpminfo['build_id']], + self.pathinfo.rpm(rpminfo)) + else: + pkgpath = '%s/%s' % (builddirs[rpminfo['build_id']], + self.pathinfo.signed(rpminfo, rpminfo['sigkey'])) + seen.add(os.path.basename(pkgpath)) + po = self.get_po(pkgpath) + mlppath = None # multilib package path + if mlm.select(po): + # we need a multilib package to be included + # we assume the same signature level is available + write_pkg(pkgpath.replace(arch, archmap[arch])) + write_pkg(pkgpath) + pkglist.close() + if not opts['skip']: + missing = list(need - seen) + if len(missing) != 0: + missing.sort() + raise koji.GenericError('Unsigned packages found: ' + + '\n'.join(missing)) + # TODO: needs to not be in /var/tmp... + return pkgfile - def getRepoPath(self, repo_id, tag): - return self.pathinfo.signedrepo(repo_id, tag) class WaitrepoTask(BaseTaskHandler): diff --git a/cli/koji b/cli/koji index 5a34e02..ed8169c 100755 --- a/cli/koji +++ b/cli/koji @@ -7088,14 +7088,13 @@ def handle_signed_repo(options, session, args): parser.add_option('--comps', help='Include a comps file in the repodata') parser.add_option('--delta-rpms', metavar='PATH',default=[], action='append', - help=_('Create delta-rpms. PATH points to (older) rpms to generate against. May be specified multiple times.')) + help=_('Create delta-rpms. PATH points to (older) rpms to generate against. May be specified multiple times. These have to be reachable by the builder too, so the path needs to reach shared storage.')) parser.add_option('--event', type='int', help=_('create a signed repository based on a Brew event')) - parser.add_option('--multilib', action='store_true', default=False, - help=_('Include multilib packages in the repository')) + parser.add_option('--multilib', action='store_true', default=None, + help=_('Include multilib packages in the repository using a config')) parser.add_option("--noinherit", action='store_true', default=False, help=_('Do not consider tag inheritance')) - # TODO: latest? parser.add_option("--nowait", action='store_true', default=False, help=_('Do not wait for the task to complete')) parser.add_option('--skip-unsigned', action='store_true', default=False, @@ -7106,13 +7105,27 @@ def handle_signed_repo(options, session, args): if task_opts.allow_unsigned and task_opts.skip_unsigned: parser.error(_('allow_signed and skip_unsigned are mutually exclusive')) activate_session(session) + stuffdir = _unique_path('cli-signed') if task_opts.comps: if not os.path.exists(task_opts.comps): parser.error(_('could not find %s' % task_opts.comps)) - compsdir = _unique_path('cli-signed') - session.uploadWrapper(task_opts.comps, compsdir, + session.uploadWrapper(task_opts.comps, stuffdir, callback=_progress_callback) - task_opts.comps = os.path.join(compsdir, os.path.basename(task_opts.comps)) + task_opts.comps = os.path.join(stuffdir, + os.path.basename(task_opts.comps)) + if len(task_opts.delta_rpms) > 0: + for path in task_opts.delta_rpms: + if not os.path.exists(path): + print _("Warning: %s is not reachable locally. If this" % path) + print _(" host does not have access to Koji's shared storage") + print _(" this can be ignored.") + if task_opts.multilib: + if not os.path.exists(task_opts.multilib): + parser.error(_('could not find %s' % task_opts.multilib)) + session.uploadWrapper(task_opts.multilib, stuffdir, + callback=_progress_callback) + task_opts.comps = os.path.join(stuffdir, + os.path.basename(task_opts.multilib)) tag = args[0] keys = args[1:] taginfo = session.getTag(tag) diff --git a/hub/kojihub.py b/hub/kojihub.py index 30879f3..6f4c9a0 100644 --- a/hub/kojihub.py +++ b/hub/kojihub.py @@ -2449,101 +2449,31 @@ def signed_repo_init(tag, keys, task_opts): tinfo = get_tag(tag, strict=True) koji.plugin.run_callbacks('preRepoInit', tag=tinfo, keys=keys, repo_id=None) tag_id = tinfo['id'] + repo_id = _singleValue("SELECT nextval('repo_id_seq')") repo_arches = task_opts['arch'] arches = set([]) for arch in repo_arches: arches.add(koji.canonArch(arch)) - repo_id = _singleValue("SELECT nextval('repo_id_seq')") if not task_opts['event']: task_opts['event'] = _singleValue("SELECT get_event()") insert = InsertProcessor('repo') insert.set(id=repo_id, create_event=task_opts['event'], tag_id=tag_id, state=state) insert.execute() - # Need to pass event_id because even though this is a single transaction, - # it is possible to see the results of other committed transactions - rpm_iter, builds = readTaggedRPMS(tag_id, event=task_opts['event'], - inherit=task_opts['inherit'], rpmsigs=True) - rpms = list(rpm_iter) - for rpm_copy in list(rpms): - arch = koji.canonArch(rpm_copy['arch']) - if arch not in arches: - # not an architecture we care about - rpms.remove(rpm_copy) - need = set(['%(name)s-%(version)s-%(release)s.%(arch)s.rpm' % r for r in rpms]) repodir = koji.pathinfo.signedrepo(repo_id, tinfo['name']) - os.makedirs(repodir) # should not already exist - - #get build dirs - pathinfo = koji.PathInfo() - builddirs = {} - for build in builds: - relpath = pathinfo.build(build) - builddirs[build['id']] = relpath.lstrip('/') - #generate pkglist files - pkglist = {} - for repoarch in arches: - archdir = os.path.join(repodir, repoarch) - koji.ensuredir(archdir) + for arch in arches: + koji.ensuredir(os.path.join(repodir, arch)) # Make a symlink to our topdir + archdir = os.path.join(repodir, arch) top_relpath = koji.util.relpath(koji.pathinfo.topdir, archdir) top_link = os.path.join(archdir, 'toplink') os.symlink(top_relpath, top_link) - pkglist[repoarch] = file(os.path.join(archdir, 'pkglist'), 'w') - preferred = {} - if task_opts['unsigned']: - keys.append('') # make unsigned rpms the least preferred - for rpminfo in rpms: - if rpminfo['sigkey'] == '' and not task_opts['unsigned']: - # skip, this is the unsigned rpminfo - continue - if rpminfo['sigkey'] not in keys: - # skip, not a key we are looking for - continue - idx = keys.index(rpminfo['sigkey']) - if preferred.has_key(rpminfo['id']): - if keys.index(preferred[rpminfo['id']]['sigkey']) <= idx: - # key for this is not as preferable as what we have seen before - continue - preferred[rpminfo['id']] = rpminfo - seen = set() - for rpminfo in preferred.values(): - if rpminfo['sigkey'] == '': - # we're taking an unsigned rpm (--allow-unsigned) - pkgpath = '%s/%s' % (builddirs[rpminfo['build_id']], - pathinfo.rpm(rpminfo)) - else: - pkgpath = '%s/%s' % (builddirs[rpminfo['build_id']], - pathinfo.signed(rpminfo, rpminfo['sigkey'])) - seen.add(os.path.basename(pkgpath)) - repopath = '/' + pkgpath - repopath = repopath.replace(koji.pathinfo.topdir, 'toplink') + '\n' - arch = koji.canonArch(rpminfo['arch']) - if arch == 'noarch': - for repoarch in arches: - pkglist[repoarch].write(repopath) - archdir = os.path.join(repodir, repoarch) - os.link(pkgpath, - os.path.join(archdir, os.path.basename(pkgpath))) - else: - pkglist[arch].write(repopath) - dest = os.path.join(repodir, arch, os.path.basename(pkgpath)) - os.link(pkgpath, dest) - for repoarch in arches: - pkglist[repoarch].close() - if not task_opts['skip']: - missing = list(need - seen) - if len(missing) != 0: - missing.sort() - raise koji.GenericError('Unsigned packages found: ' + - '\n'.join(missing)) - # handle comps if task_opts['comps']: groupsdir = os.path.join(repodir, 'groups') koji.ensuredir(groupsdir) - shutil.copyfile(os.path.join(koji.pathinfo.work(), task_opts['comps']), - groupsdir + '/comps.xml') + shutil.copyfile(os.path.join(koji.pathinfo.work(), + task_opts['comps']), groupsdir + '/comps.xml') koji.plugin.run_callbacks('postRepoInit', tag=tinfo, event=task_opts['event'], repo_id=repo_id) return repo_id, task_opts['event'] @@ -10207,7 +10137,8 @@ class RootExports(object): """Create a signed-repo task. returns task id""" context.session.assertPerm('signed-repo') repo_id, event_id = signed_repo_init(tag, keys, task_opts) - return make_task('signedRepo', [tag, repo_id, task_opts], priority=15) + task_opts['event'] = event_id + return make_task('signedRepo', [tag, repo_id, keys, task_opts], priority=15) def newRepo(self, tag, event=None, src=False, debuginfo=False): """Create a newRepo task. returns task id""" @@ -12338,6 +12269,7 @@ class HostExports(object): repo_id: the id of the repo data: a dictionary of the form { arch: (uploadpath, files), ...} expire(optional): if set to true, mark the repo expired immediately* + signed(optional): if true, hardlink signed rpms in the final directory * This is used when a repo from an older event is generated """ @@ -12363,11 +12295,20 @@ class HostExports(object): if fn.endswith('.drpm'): koji.ensuredir(os.path.join(archdir, 'drpms')) dst = "%s/drpms/%s" % (archdir, fn) + elif fn.endswith('pkglist'): + dst = '%s/%s' % (archdir, fn) else: dst = "%s/%s" % (datadir, fn) if not os.path.exists(src): raise koji.GenericError("uploaded file missing: %s" % src) safer_move(src, dst) + if fn.endswith('pkglist') and signed: + # hardlink the found rpms into the final repodir + with open(src) as pkgfile: + for pkg in pkgfile: + pkg = pkg.strip() + rpm = os.path.basename(pkg) + os.link(koji.pathinfo.topdir + pkg, os.path.join(archdir, rpm)) if expire: repo_expire(repo_id) koji.plugin.run_callbacks('postRepoDone', repo=rinfo, data=data, expire=expire) diff --git a/www/kojiweb/index.py b/www/kojiweb/index.py index 569e95b..8fc8248 100644 --- a/www/kojiweb/index.py +++ b/www/kojiweb/index.py @@ -625,7 +625,7 @@ def taskinfo(environ, taskID): build = server.getBuild(params[1]) values['destTag'] = destTag values['build'] = build - elif task['method'] in ('newRepo', 'signedRepo'): + elif task['method'] in ('newRepo', 'signedRepo', 'createsignedrepo'): tag = server.getTag(params[0]) values['tag'] = tag elif task['method'] == 'tagNotification': diff --git a/www/kojiweb/taskinfo.chtml b/www/kojiweb/taskinfo.chtml index c68b5af..50cec39 100644 --- a/www/kojiweb/taskinfo.chtml +++ b/www/kojiweb/taskinfo.chtml @@ -218,14 +218,11 @@ $value #if $len($params) > 2 $printOpts($params[2]) #end if - #elif $task.method in ('newRepo', 'signedRepo') + #elif $task.method == 'signedRepo' Tag: $tag.name
- #if $task.method == 'signedRepo' - Repo ID: $params[1]
- $printOpts($params[2]) - #elif $len($params) > 1 - $printOpts($params[1]) - #end if + Repo ID: $params[1]
+ Keys: $printValue(0, $params[2])
+ $printOpts($params[3]) #elif $task.method == 'prepRepo' Tag: $params[0].name #elif $task.method == 'createrepo' @@ -240,9 +237,11 @@ $value External Repos: $printValue(None, [ext['external_repo_name'] for ext in $params[3]])
#end if #elif $task.method == 'createsignedrepo' - Repo ID: $params[0]
- Arch: $params[1]
- Options: $printMap($params[3], '    ') + Tag: $tag.name
+ Repo ID: $params[1]
+ Arch: $printValue(0, $params[2])
+ Keys: $printValue(0, $params[3])
+ Options: $printMap($params[4], '    ') #elif $task.method == 'dependantTask' Dependant Tasks:
#for $dep in $deps From 4f4c7e3d4e63329f7d07846a76f9a76982ecdbba Mon Sep 17 00:00:00 2001 From: Jay Greguske Date: Mar 30 2017 13:37:56 +0000 Subject: [PATCH 12/77] implement multilib --- diff --git a/builder/kojid b/builder/kojid index e6fdb95..e94e4c1 100755 --- a/builder/kojid +++ b/builder/kojid @@ -36,13 +36,13 @@ from koji.daemon import incremental_upload, log_output, TaskManager, SCM from koji.tasks import ServerExit, ServerRestart, BaseTaskHandler, MultiPlatformTask from koji.util import parseStatus, isSuccess, dslice, dslice_ex import multilib -import multilib.fakepo import os import pwd import grp import random import re import rpm +import rpmUtils.arch import shutil import signal import smtplib @@ -61,6 +61,7 @@ from gzip import GzipFile from optparse import OptionParser, SUPPRESS_HELP from yum import repoMDObject import yum.packages +import yum.Errors #imports for LiveCD, LiveMedia, and Appliance handler image_enabled = False @@ -4944,20 +4945,41 @@ class NewSignedRepoTask(BaseTaskHandler): if len(task_opts['arch']) == 0: raise koji.GenericError('No arches specified nor for the tag!') subtasks = {} + arch32s = set() for arch in task_opts['arch']: - # call canonArch? - arglist = [tag, repo_id, arch, keys, task_opts] # no mergerepo + if not rpmUtils.arch.isMultiLibArch(arch): + arch32s.add(arch) + for arch in arch32s: + # we do 32-bit multilib arches first so the 64-bit ones can + # get a task ID and wait for them to complete + arglist = [tag, repo_id, arch, keys, task_opts] subtasks[arch] = self.session.host.subtask( method='createsignedrepo', arglist=arglist, label=arch, parent=self.id, arch='noarch') - # wait for subtasks to finish - self.logger.warn("5: %s" % subtasks.values()) - results = self.wait(subtasks.values(), all=True, failany=True) - self.logger.warn("6") + if len(subtasks) > 0 and task_opts['multilib']: + results = self.wait(subtasks.values(), all=True, failany=True) + for arch in arch32s: + # move the 32-bit task output to the final resting place + # so the 64-bit arches can use it + upload, files = results[subtasks[arch]] + self.session.host.signedRepoMove(repo_id, upload, files, arch) + for arch in task_opts['arch']: + # do the other arches + if arch not in arch32s: + arglist = [tag, repo_id, arch, keys, task_opts] + subtasks[arch] = self.session.host.subtask( + method='createsignedrepo', arglist=arglist, label=arch, + parent=self.id, arch='noarch') + # wait for 64-bit subtasks to finish data = {} + results = self.wait(subtasks.values(), all=True, failany=True) for (arch, task_id) in subtasks.iteritems(): data[arch] = results[task_id] - self.logger.debug("DEBUG: %r : %r " % (arch, data[arch],)) + self.logger.debug("DEBUG: %r : %r " % (arch, data[arch])) + if arch not in arch32s: + # we moved the 32-bit results before, do the 64-bit + upload, files = results[subtasks[arch]] + self.session.host.signedRepoMove(repo_id, upload, files, arch) self.session.host.repoDone(repo_id, data, expire=True, signed=True) return 'Signed repository #%s successfully generated' % repo_id @@ -4965,17 +4987,36 @@ class NewSignedRepoTask(BaseTaskHandler): class createSignedRepoTask(CreaterepoTask): Methods = ['createsignedrepo'] _taskWeight = 1.5 + archmap = {'s390x': 's390', 'ppc64': 'ppc', 'x86_64': 'i686'} + compat = {"i386": ("athlon", "i686", "i586", "i486", "i386", "noarch"), + "x86_64": ("amd64", "ia32e", "x86_64", "noarch"), + "ia64": ("ia64", "noarch"), + "ppc": ("ppc", "noarch"), + "ppc64": ("ppc64p7", "ppc64pseries", "ppc64iseries", "ppc64", "noarch"), + "ppc64le": ("ppc64le", "noarch"), + "s390": ("s390", "noarch"), + "s390x": ("s390x", "noarch"), + "sparc": ("sparcv9v", "sparcv9", "sparcv8", "sparc", "noarch"), + "sparc64": ("sparc64v", "sparc64", "noarch"), + "alpha": ("alphaev6", "alphaev56", "alphaev5", "alpha", "noarch"), + "arm": ("arm", "armv4l", "armv4tl", "armv5tel", "armv5tejl", "armv6l", "armv7l", "noarch"), + "armhfp": ("armv7hl", "armv7hnl", "noarch"), + "aarch64": ("aarch64", "noarch"), + } + + biarch = {"ppc": "ppc64", "x86_64": "i386", "sparc": + "sparc64", "s390x": "s390", "ppc64": "ppc"} def handler(self, tag, repo_id, arch, keys, opts): #arch is the arch of the repo, not the task - rinfo = self.session.repoInfo(repo_id, strict=True) - if rinfo['state'] != koji.REPO_INIT: - raise koji.GenericError, "Repo %(id)s not in INIT state (got %(state)s)" % rinfo - self.repo_id = rinfo['id'] + self.rinfo = self.session.repoInfo(repo_id, strict=True) + if self.rinfo['state'] != koji.REPO_INIT: + raise koji.GenericError, "Repo %(id)s not in INIT state (got %(state)s)" % self.rinfo + self.repo_id = self.rinfo['id'] self.pathinfo = koji.PathInfo(self.options.topdir) groupdata = os.path.join( - self.pathinfo.signedrepo(repo_id, rinfo['tag_name']), + self.pathinfo.signedrepo(repo_id, self.rinfo['tag_name']), 'groups', 'comps.xml') self.repodir = self.options.topdir # workaround for create_local_repo #set up our output dir @@ -4986,65 +5027,173 @@ class createSignedRepoTask(CreaterepoTask): if not os.path.exists(path): raise koji.GenericError( 'drpm path %s does not exist!' % path) - pkglist = self.make_pkglist(tag, arch, keys, opts) - uploadpath = self.getUploadDir() - self.session.uploadWrapper(pkglist, uploadpath, - os.path.basename(pkglist)) - if os.path.getsize(pkglist) == 0: - pkglist = None + self.uploadpath = self.getUploadDir() + self.pkglist = self.make_pkglist(tag, arch, keys, opts) + if opts['multilib'] and rpmUtils.arch.isMultiLibArch(arch): + self.do_multilib(arch, self.archmap[arch], opts['multilib']) + self.logger.debug('package list is %s' % self.pkglist) + self.session.uploadWrapper(self.pkglist, self.uploadpath, + os.path.basename(self.pkglist)) + if os.path.getsize(self.pkglist) == 0: + self.pkglist = None if len(opts['delta']) > 0: do_drpms = True else: do_drpms = False - self.create_local_repo(rinfo, arch, pkglist, groupdata, opts['delta'], - drpms=do_drpms, baseurl='toplink') - if pkglist is None: + self.create_local_repo(self.rinfo, arch, self.pkglist, groupdata, + opts['delta'], drpms=do_drpms, baseurl='toplink') + if self.pkglist is None: fo = file(os.path.join(self.datadir, "EMPTY_REPO"), 'w') fo.write("This repo is empty because its tag has no content for this arch\n") fo.close() files = ['pkglist'] for f in os.listdir(self.datadir): files.append(f) - self.session.uploadWrapper('%s/%s' % (self.datadir, f), uploadpath, f) + self.session.uploadWrapper('%s/%s' % (self.datadir, f), + self.uploadpath, f) if opts['delta']: ddir = os.path.join(self.outdir, 'drpms') for f in os.listdir(ddir): files.append(f) - self.session.uploadWrapper('%s/%s' % (ddir, f), uploadpath, f) - return [uploadpath, files] - - def get_po(self, rpmpath): - """create a fake yum-like package object given an rpminfo dictionary""" - po = yum.packages.YumLocalPackage(filename=rpmpath) - return multilib.fakepo.FakePackageObject(po=po) + self.session.uploadWrapper('%s/%s' % (ddir, f), + self.uploadpath, f) + return [self.uploadpath, files] + + def do_multilib(self, arch, ml_arch, conf): + self.repo_id = self.rinfo['id'] + pathinfo = koji.PathInfo(self.options.topdir) + repodir = pathinfo.signedrepo(self.rinfo['id'], self.rinfo['tag_name']) + archdir = os.path.join(repodir, arch) + mldir = os.path.join(repodir, koji.canonArch(ml_arch)) + ml_true = set() + ml_conf = os.path.join(self.pathinfo.work(), conf) + + # step 1: figure out which packages are multlib (should already exist) + mlm = multilib.DevelMultilibMethod(ml_conf) + fs_missing = set() + with open(self.pkglist) as pkglist: + for pkg in pkglist: + pkg = pkg.strip() + rpmpath = self.options.topdir + pkg + try: + po = yum.packages.YumLocalPackage(filename=rpmpath) + except yum.Errors.MiscError: + self.logger.error('%s is not on the filesystem' % rpmpath) + fs_missing.add(rpmpath) + continue + if mlm.select(po) and self.archmap.has_key(arch): + # we need a multilib package to be included + # we assume the same signature level is available + pl_path = pkg.replace(arch, self.archmap[arch]) + real_path = rpmpath.replace(arch, self.archmap[arch]) + ml_true.add(pl_path) + if not os.path.exists(real_path): + self.logger.error('%s (multilib) is not on the filesystem' % ml_path) + fs_missing.add(real_path) + + # step 2: set up architectures for yum configuration + self.logger.info("Resolving multilib for %s using method devel" % arch) + yumbase = yum.YumBase() + yumbase.verbose_logger.setLevel(logging.ERROR) + yumdir = os.path.join(self.workdir, 'yum') + # TODO: unwind this arch mess + archlist = (arch, 'noarch') + transaction_arch = arch + archlist = archlist + self.compat[self.biarch[arch]] + best_compat = self.compat[self.biarch[arch]][0] + if rpmUtils.arch.archDifference(best_compat, arch) > 0: + transaction_arch = best_compat + if hasattr(rpmUtils.arch, 'ArchStorage'): + yumbase.preconf.arch = transaction_arch + else: + rpmUtils.arch.canonArch = transaction_arch + + yconfig = """ +[main] +debuglevel=2 +pkgpolicy=newest +exactarch=1 +gpgcheck=0 +reposdir=/dev/null +cachedir=/yumcache +installroot=%s +logfile=/yum.log + +[koji-%s] +name=koji multilib task +baseurl=file://%s +enabled=1 + +""" % (yumdir, self.id, mldir) + os.makedirs(os.path.join(yumdir, "yumcache")) + os.makedirs(os.path.join(yumdir, 'var/lib/rpm')) + + # step 3: proceed with yum config and set up + yconfig_path = os.path.join(yumdir, 'yum.conf-koji-%s' % arch) + f = open(yconfig_path, 'w') + f.write(yconfig) + f.close() + self.session.uploadWrapper(yconfig_path, self.uploadpath, + os.path.basename(yconfig_path)) + yumbase.doConfigSetup(fn=yconfig_path) + yumbase.conf.cache = 0 + yumbase.doRepoSetup() + yumbase.doTsSetup() + yumbase.doRpmDBSetup() + # we trust Koji's files, so skip verifying sigs and digests + yumbase.ts.pushVSFlags( + (rpm._RPMVSF_NOSIGNATURES | rpm._RPMVSF_NODIGESTS)) + yumbase.doSackSetup(archlist=archlist, thisrepo='koji-%s' % arch) + yumbase.doSackFilelistPopulate() + for pkg in ml_true: + # TODO: store packages by first letter + # ppath = os.path.join(pkgdir, pkg.name[0].lower(), pname) + real_path = self.options.topdir + pkg + po = yum.packages.YumLocalPackage(filename=real_path) + yumbase.tsInfo.addInstall(po) + + # step 4: execute yum transaction to get dependencies + self.logger.info("Resolving depenencies for arch %s" % arch) + rc, errors = yumbase.resolveDeps() + ml_needed = set() + for f in yumbase.tsInfo.getMembers(): + dep_path = os.path.join(mldir, os.path.basename(f.po.localPkg())) + rel_path = dep_path.replace(self.options.topdir, '') + ml_needed.add(rel_path) + self.logger.debug("added %s" % rel_path) + if not os.path.exists(dep_path): + self.logger.error('%s (multilib dep) not on filesystem' % dep_path) + fs_missing.add(dep_path) + self.logger.info('yum return code: %s' % rc) + if not rc: + self.logger.error('yum depsolve was unsuccessful') + raise koji.GenericError(errors) + if len(fs_missing) > 0: + raise koji.GenericError('multilib packages missing:\n' + + '\n'.join(fs_missing)) + + # step 5: add dependencies to our package list + pkgwriter = open(self.pkglist, 'a') + for ml_pkg in ml_needed: + pkgwriter.write(ml_pkg + '\n') def make_pkglist(self, tag_id, arch, keys, opts): - def write_pkg(pkgpath): - self.logger.info('incoming: %s' % pkgpath) - self.logger.info('topdir: %s' % self.options.topdir) - newpath = pkgpath.replace(self.options.topdir, '') + '\n' - self.logger.info('outgoing: %s' % newpath) - pkglist.write(newpath) - # Need to pass event_id because even though this is a single trans, # it is possible to see the results of other committed transactions - rpm_iter, builds = self.session.listTaggedRPMS(tag_id, - event=opts['event'], arch=arch, - inherit=opts['inherit'], rpmsigs=True) - rpms = list(rpm_iter) - if opts['multilib']: - mlm = multilib.MultilibDevelMethod(opts['multilib']) - else: - # this method always returns False, no multilib packages added - mlm = multilib.NoMultilibMethod(opts['multilib']) - need = set(['%(name)s-%(version)s-%(release)s.%(arch)s.rpm' % r for r in rpms]) - #get build dirs + rpms = [] builddirs = {} - for build in builds: - builddirs[build['id']] = self.pathinfo.build(build) + for a in (arch, 'noarch'): + rpm_iter, builds = self.session.listTaggedRPMS(tag_id, + event=opts['event'], arch=a, + inherit=opts['inherit'], rpmsigs=True) + for build in builds: + builddirs[build['id']] = self.pathinfo.build(build) + rpms += list(rpm_iter) + #get build dirs + need = set(['%(name)s-%(version)s-%(release)s.%(arch)s.rpm' % r for r in rpms]) #generate pkglist files - archdir = os.path.join(self.outdir, arch) + archdir = os.path.join(self.outdir, koji.canonArch(arch)) koji.ensuredir(archdir) pkgfile = os.path.join(archdir, 'pkglist') pkglist = file(pkgfile, 'w') @@ -5065,6 +5214,7 @@ class createSignedRepoTask(CreaterepoTask): continue preferred[rpminfo['id']] = rpminfo seen = set() + fs_missing = set() for rpminfo in preferred.values(): if rpminfo['sigkey'] == '': # we're taking an unsigned rpm (--allow-unsigned) @@ -5074,21 +5224,19 @@ class createSignedRepoTask(CreaterepoTask): pkgpath = '%s/%s' % (builddirs[rpminfo['build_id']], self.pathinfo.signed(rpminfo, rpminfo['sigkey'])) seen.add(os.path.basename(pkgpath)) - po = self.get_po(pkgpath) - mlppath = None # multilib package path - if mlm.select(po): - # we need a multilib package to be included - # we assume the same signature level is available - write_pkg(pkgpath.replace(arch, archmap[arch])) - write_pkg(pkgpath) + pkglist.write(pkgpath.replace(self.options.topdir, '') + '\n') + if not os.path.exists(pkgpath): + fs_missing.add(pkgpath) pkglist.close() + if len(fs_missing) > 0: + raise koji.GenericError('Packages missing from the filesystem:\n' + + '\n'.join(fs_missing)) if not opts['skip']: missing = list(need - seen) if len(missing) != 0: missing.sort() raise koji.GenericError('Unsigned packages found: ' + '\n'.join(missing)) - # TODO: needs to not be in /var/tmp... return pkgfile diff --git a/cli/koji b/cli/koji index ed8169c..f10e144 100755 --- a/cli/koji +++ b/cli/koji @@ -7091,7 +7091,7 @@ def handle_signed_repo(options, session, args): help=_('Create delta-rpms. PATH points to (older) rpms to generate against. May be specified multiple times. These have to be reachable by the builder too, so the path needs to reach shared storage.')) parser.add_option('--event', type='int', help=_('create a signed repository based on a Brew event')) - parser.add_option('--multilib', action='store_true', default=None, + parser.add_option('--multilib', default=None, help=_('Include multilib packages in the repository using a config')) parser.add_option("--noinherit", action='store_true', default=False, help=_('Do not consider tag inheritance')) @@ -7111,6 +7111,7 @@ def handle_signed_repo(options, session, args): parser.error(_('could not find %s' % task_opts.comps)) session.uploadWrapper(task_opts.comps, stuffdir, callback=_progress_callback) + print task_opts.comps = os.path.join(stuffdir, os.path.basename(task_opts.comps)) if len(task_opts.delta_rpms) > 0: @@ -7119,13 +7120,6 @@ def handle_signed_repo(options, session, args): print _("Warning: %s is not reachable locally. If this" % path) print _(" host does not have access to Koji's shared storage") print _(" this can be ignored.") - if task_opts.multilib: - if not os.path.exists(task_opts.multilib): - parser.error(_('could not find %s' % task_opts.multilib)) - session.uploadWrapper(task_opts.multilib, stuffdir, - callback=_progress_callback) - task_opts.comps = os.path.join(stuffdir, - os.path.basename(task_opts.multilib)) tag = args[0] keys = args[1:] taginfo = session.getTag(tag) @@ -7139,6 +7133,20 @@ def handle_signed_repo(options, session, args): for a in task_opts.arch: if not taginfo['arches'] or a not in taginfo['arches']: print _('Warning: %s is not in the list of tag arches' % a) + if task_opts.multilib: + if not os.path.exists(task_opts.multilib): + parser.error(_('could not find %s' % task_opts.multilib)) + if 'x86_64' in task_opts.arch and not 'i686' in task_opts.arch: + parser.error(_('The multilib arch (i686) must be included')) + if 's390x' in task_opts.arch and not 's390' in task_opts.arch: + parser.error(_('The multilib arch (s390) must be included')) + if 'ppc64' in task_opts.arch and not 'ppc' in task_opts.arch: + parser.error(_('The multilib arch (ppc) must be included')) + session.uploadWrapper(task_opts.multilib, stuffdir, + callback=_progress_callback) + task_opts.multilib = os.path.join(stuffdir, + os.path.basename(task_opts.multilib)) + print try: task_opts.arch.remove('noarch') # handled specifically task_opts.arch.remove('src') # ditto diff --git a/hub/kojihub.py b/hub/kojihub.py index 6f4c9a0..7f090fd 100644 --- a/hub/kojihub.py +++ b/hub/kojihub.py @@ -12279,36 +12279,24 @@ class HostExports(object): koji.plugin.run_callbacks('preRepoDone', repo=rinfo, data=data, expire=expire) if rinfo['state'] != koji.REPO_INIT: raise koji.GenericError("Repo %(id)s not in INIT state (got %(state)s)" % rinfo) - if signed: - repodir = koji.pathinfo.signedrepo(repo_id, rinfo['tag_name']) - else: - repodir = koji.pathinfo.repo(repo_id, rinfo['tag_name']) + repodir = koji.pathinfo.repo(repo_id, rinfo['tag_name']) workdir = koji.pathinfo.work() - for arch, (uploadpath, files) in data.iteritems(): - archdir = "%s/%s" % (repodir, arch) - if not os.path.isdir(archdir): - raise koji.GenericError("Repo arch directory missing: %s" % archdir) - datadir = "%s/repodata" % archdir - koji.ensuredir(datadir) - for fn in files: - src = "%s/%s/%s" % (workdir, uploadpath, fn) - if fn.endswith('.drpm'): - koji.ensuredir(os.path.join(archdir, 'drpms')) - dst = "%s/drpms/%s" % (archdir, fn) - elif fn.endswith('pkglist'): - dst = '%s/%s' % (archdir, fn) - else: - dst = "%s/%s" % (datadir, fn) - if not os.path.exists(src): - raise koji.GenericError("uploaded file missing: %s" % src) - safer_move(src, dst) - if fn.endswith('pkglist') and signed: - # hardlink the found rpms into the final repodir - with open(src) as pkgfile: - for pkg in pkgfile: - pkg = pkg.strip() - rpm = os.path.basename(pkg) - os.link(koji.pathinfo.topdir + pkg, os.path.join(archdir, rpm)) + if not signed: + for arch, (uploadpath, files) in data.iteritems(): + archdir = "%s/%s" % (repodir, koji.canonArch(arch)) + if not os.path.isdir(archdir): + raise koji.GenericError("Repo arch directory missing: %s" % archdir) + datadir = "%s/repodata" % archdir + koji.ensuredir(datadir) + for fn in files: + src = "%s/%s/%s" % (workdir, uploadpath, fn) + if fn.endswith('pkglist'): + dst = '%s/%s' % (archdir, fn) + else: + dst = "%s/%s" % (datadir, fn) + if not os.path.exists(src): + raise koji.GenericError("uploaded file missing: %s" % src) + safer_move(src, dst) if expire: repo_expire(repo_id) koji.plugin.run_callbacks('postRepoDone', repo=rinfo, data=data, expire=expire) @@ -12328,6 +12316,37 @@ class HostExports(object): log_error("Unable to create latest link for repo: %s" % repodir) koji.plugin.run_callbacks('postRepoDone', repo=rinfo, data=data, expire=expire) + def signedRepoMove(self, repo_id, uploadpath, files, arch): + """very similar to repoDone, except only the uploads are completed""" + workdir = koji.pathinfo.work() + rinfo = repo_info(repo_id, strict=True) + repodir = koji.pathinfo.signedrepo(repo_id, rinfo['tag_name']) + archdir = "%s/%s" % (repodir, koji.canonArch(arch)) + if not os.path.isdir(archdir): + raise koji.GenericError, "Repo arch directory missing: %s" % archdir + datadir = "%s/repodata" % archdir + koji.ensuredir(datadir) + for fn in files: + src = "%s/%s/%s" % (workdir, uploadpath, fn) + if fn.endswith('.drpm'): + koji.ensuredir(os.path.join(archdir, 'drpms')) + dst = "%s/drpms/%s" % (archdir, fn) + elif fn.endswith('pkglist'): + dst = '%s/%s' % (archdir, fn) + else: + dst = "%s/%s" % (datadir, fn) + if not os.path.exists(src): + raise koji.GenericError, "uploaded file missing: %s" % src + os.link(src, dst) + if fn.endswith('pkglist'): + # hardlink the found rpms into the final repodir + with open(src) as pkgfile: + for pkg in pkgfile: + pkg = pkg.strip() + rpm = os.path.basename(pkg) + os.link(koji.pathinfo.topdir + pkg, os.path.join(archdir, rpm)) + os.unlink(src) + def isEnabled(self): host = Host() host.verify() diff --git a/koji/__init__.py b/koji/__init__.py index 849c7a9..bf628fe 100644 --- a/koji/__init__.py +++ b/koji/__init__.py @@ -2803,10 +2803,15 @@ def _taskLabel(taskInfo): if 'request' in taskInfo: tagInfo = taskInfo['request'][0] extra = tagInfo['name'] - elif method in ('createrepo', 'createsignedrepo'): + elif method in ('createrepo'): if 'request' in taskInfo: arch = taskInfo['request'][1] extra = arch + elif method in ('createsignedrepo'): + if 'request' in taskInfo: + repo_id = taskInfo['request'][1] + arch = taskInfo['request'][2] + extra = '%s, %s' % (repo_id, arch) elif method == 'dependantTask': if 'request' in taskInfo: extra = ', '.join([subtask[0] for subtask in taskInfo['request'][1]]) From 1a6feb5070d09a5a983dda1a54cf16afcb557cba Mon Sep 17 00:00:00 2001 From: Jay Greguske Date: Mar 30 2017 13:37:56 +0000 Subject: [PATCH 13/77] encapsulate repodata references --- diff --git a/builder/kojid b/builder/kojid index e94e4c1..fd544f5 100755 --- a/builder/kojid +++ b/builder/kojid @@ -4851,7 +4851,7 @@ class CreaterepoTask(BaseTaskHandler): self.session.uploadWrapper('%s/%s' % (self.datadir, f), uploadpath, f) return [uploadpath, files] - def create_local_repo(self, rinfo, arch, pkglist, groupdata, oldrepo, baseurl=None, drpms=False): + def create_local_repo(self, rinfo, arch, pkglist, groupdata, oldrepo, drpms=False): koji.ensuredir(self.outdir) if self.options.use_createrepo_c: cmd = ['/usr/bin/createrepo_c'] @@ -4862,8 +4862,6 @@ class CreaterepoTask(BaseTaskHandler): cmd.extend(['-i', pkglist]) if os.path.isfile(groupdata): cmd.extend(['-g', groupdata]) - if baseurl: - cmd.extend(['-u', baseurl]) #attempt to recycle repodata from last repo if pkglist and oldrepo and self.options.createrepo_update and not drpms: # signed repos overload the use of "oldrepo", so the conditional @@ -4961,8 +4959,9 @@ class NewSignedRepoTask(BaseTaskHandler): for arch in arch32s: # move the 32-bit task output to the final resting place # so the 64-bit arches can use it - upload, files = results[subtasks[arch]] - self.session.host.signedRepoMove(repo_id, upload, files, arch) + upload, files, keypaths = results[subtasks[arch]] + self.session.host.signedRepoMove( + repo_id, upload, files, arch, keypaths) for arch in task_opts['arch']: # do the other arches if arch not in arch32s: @@ -4978,8 +4977,9 @@ class NewSignedRepoTask(BaseTaskHandler): self.logger.debug("DEBUG: %r : %r " % (arch, data[arch])) if arch not in arch32s: # we moved the 32-bit results before, do the 64-bit - upload, files = results[subtasks[arch]] - self.session.host.signedRepoMove(repo_id, upload, files, arch) + upload, files, keypaths = results[subtasks[arch]] + self.session.host.signedRepoMove( + repo_id, upload, files, arch, keypaths) self.session.host.repoDone(repo_id, data, expire=True, signed=True) return 'Signed repository #%s successfully generated' % repo_id @@ -5018,10 +5018,12 @@ class createSignedRepoTask(CreaterepoTask): groupdata = os.path.join( self.pathinfo.signedrepo(repo_id, self.rinfo['tag_name']), 'groups', 'comps.xml') - self.repodir = self.options.topdir # workaround for create_local_repo #set up our output dir - self.outdir = '%s/repo' % self.workdir - self.datadir = '%s/repodata' % self.outdir + self.repodir = '%s/repo' % self.workdir + koji.ensuredir(self.repodir) + self.outdir = self.repodir # workaround create_local_repo use + self.datadir = '%s/repodata' % self.repodir + self.keypaths = {} if len(opts['delta']) > 0: for path in opts['delta']: if not os.path.exists(path): @@ -5041,7 +5043,7 @@ class createSignedRepoTask(CreaterepoTask): else: do_drpms = False self.create_local_repo(self.rinfo, arch, self.pkglist, groupdata, - opts['delta'], drpms=do_drpms, baseurl='toplink') + opts['delta'], drpms=do_drpms) if self.pkglist is None: fo = file(os.path.join(self.datadir, "EMPTY_REPO"), 'w') fo.write("This repo is empty because its tag has no content for this arch\n") @@ -5052,20 +5054,19 @@ class createSignedRepoTask(CreaterepoTask): self.session.uploadWrapper('%s/%s' % (self.datadir, f), self.uploadpath, f) if opts['delta']: - ddir = os.path.join(self.outdir, 'drpms') + ddir = os.path.join(self.repodir, 'drpms') for f in os.listdir(ddir): files.append(f) self.session.uploadWrapper('%s/%s' % (ddir, f), self.uploadpath, f) - return [self.uploadpath, files] + return [self.uploadpath, files, self.keypaths] def do_multilib(self, arch, ml_arch, conf): self.repo_id = self.rinfo['id'] pathinfo = koji.PathInfo(self.options.topdir) repodir = pathinfo.signedrepo(self.rinfo['id'], self.rinfo['tag_name']) - archdir = os.path.join(repodir, arch) mldir = os.path.join(repodir, koji.canonArch(ml_arch)) - ml_true = set() + ml_true = set() # multilib packages we need to include before depsolve ml_conf = os.path.join(self.pathinfo.work(), conf) # step 1: figure out which packages are multlib (should already exist) @@ -5073,22 +5074,17 @@ class createSignedRepoTask(CreaterepoTask): fs_missing = set() with open(self.pkglist) as pkglist: for pkg in pkglist: - pkg = pkg.strip() - rpmpath = self.options.topdir + pkg - try: - po = yum.packages.YumLocalPackage(filename=rpmpath) - except yum.Errors.MiscError: - self.logger.error('%s is not on the filesystem' % rpmpath) - fs_missing.add(rpmpath) - continue + ppath = os.path.join(self.repodir, pkg.strip()) + po = yum.packages.YumLocalPackage(filename=ppath) if mlm.select(po) and self.archmap.has_key(arch): # we need a multilib package to be included # we assume the same signature level is available - pl_path = pkg.replace(arch, self.archmap[arch]) - real_path = rpmpath.replace(arch, self.archmap[arch]) - ml_true.add(pl_path) + pl_path = pkg.replace(arch, self.archmap[arch]).strip() + # assume this exists in the task results for the ml arch + real_path = os.path.join(mldir, pl_path) + ml_true.add(real_path) if not os.path.exists(real_path): - self.logger.error('%s (multilib) is not on the filesystem' % ml_path) + self.logger.error('%s (multilib) is not on the filesystem' % real_path) fs_missing.add(real_path) # step 2: set up architectures for yum configuration @@ -5148,8 +5144,7 @@ enabled=1 for pkg in ml_true: # TODO: store packages by first letter # ppath = os.path.join(pkgdir, pkg.name[0].lower(), pname) - real_path = self.options.topdir + pkg - po = yum.packages.YumLocalPackage(filename=real_path) + po = yum.packages.YumLocalPackage(filename=pkg) yumbase.tsInfo.addInstall(po) # step 4: execute yum transaction to get dependencies @@ -5158,9 +5153,8 @@ enabled=1 ml_needed = set() for f in yumbase.tsInfo.getMembers(): dep_path = os.path.join(mldir, os.path.basename(f.po.localPkg())) - rel_path = dep_path.replace(self.options.topdir, '') - ml_needed.add(rel_path) - self.logger.debug("added %s" % rel_path) + ml_needed.add(dep_path) + self.logger.debug("added %s" % dep_path) if not os.path.exists(dep_path): self.logger.error('%s (multilib dep) not on filesystem' % dep_path) fs_missing.add(dep_path) @@ -5175,7 +5169,11 @@ enabled=1 # step 5: add dependencies to our package list pkgwriter = open(self.pkglist, 'a') for ml_pkg in ml_needed: - pkgwriter.write(ml_pkg + '\n') + bnp = os.path.basename(ml_pkg) + pkgwriter.write(bnp + '\n') + os.symlink(ml_pkg, os.path.join(self.repodir, bnp)) + self.keypaths[bnp] = ml_pkg + def make_pkglist(self, tag_id, arch, keys, opts): @@ -5193,9 +5191,7 @@ enabled=1 #get build dirs need = set(['%(name)s-%(version)s-%(release)s.%(arch)s.rpm' % r for r in rpms]) #generate pkglist files - archdir = os.path.join(self.outdir, koji.canonArch(arch)) - koji.ensuredir(archdir) - pkgfile = os.path.join(archdir, 'pkglist') + pkgfile = os.path.join(self.repodir, 'pkglist') pkglist = file(pkgfile, 'w') preferred = {} if opts['unsigned']: @@ -5224,9 +5220,13 @@ enabled=1 pkgpath = '%s/%s' % (builddirs[rpminfo['build_id']], self.pathinfo.signed(rpminfo, rpminfo['sigkey'])) seen.add(os.path.basename(pkgpath)) - pkglist.write(pkgpath.replace(self.options.topdir, '') + '\n') if not os.path.exists(pkgpath): fs_missing.add(pkgpath) + else: + bnp = os.path.basename(pkgpath) + pkglist.write(bnp + '\n') + self.keypaths[bnp] = pkgpath + os.symlink(pkgpath, os.path.join(self.repodir, bnp)) pkglist.close() if len(fs_missing) > 0: raise koji.GenericError('Packages missing from the filesystem:\n' + diff --git a/hub/kojihub.py b/hub/kojihub.py index 7f090fd..40a8973 100644 --- a/hub/kojihub.py +++ b/hub/kojihub.py @@ -2463,11 +2463,6 @@ def signed_repo_init(tag, keys, task_opts): repodir = koji.pathinfo.signedrepo(repo_id, tinfo['name']) for arch in arches: koji.ensuredir(os.path.join(repodir, arch)) - # Make a symlink to our topdir - archdir = os.path.join(repodir, arch) - top_relpath = koji.util.relpath(koji.pathinfo.topdir, archdir) - top_link = os.path.join(archdir, 'toplink') - os.symlink(top_relpath, top_link) # handle comps if task_opts['comps']: groupsdir = os.path.join(repodir, 'groups') @@ -12316,8 +12311,10 @@ class HostExports(object): log_error("Unable to create latest link for repo: %s" % repodir) koji.plugin.run_callbacks('postRepoDone', repo=rinfo, data=data, expire=expire) - def signedRepoMove(self, repo_id, uploadpath, files, arch): - """very similar to repoDone, except only the uploads are completed""" + def signedRepoMove(self, repo_id, uploadpath, files, arch, fullpaths): + """ + Very similar to repoDone, except only the uploads are completed. + fullpaths is a dict like so: rpm file name -> sig""" workdir = koji.pathinfo.work() rinfo = repo_info(repo_id, strict=True) repodir = koji.pathinfo.signedrepo(repo_id, rinfo['tag_name']) @@ -12343,8 +12340,8 @@ class HostExports(object): with open(src) as pkgfile: for pkg in pkgfile: pkg = pkg.strip() - rpm = os.path.basename(pkg) - os.link(koji.pathinfo.topdir + pkg, os.path.join(archdir, rpm)) + rpmpath = fullpaths[pkg] + os.link(rpmpath, os.path.join(archdir, os.path.basename(rpmpath))) os.unlink(src) def isEnabled(self): From 97b6d4f866bd731890a984beb6363aff8de5e0e6 Mon Sep 17 00:00:00 2001 From: Jay Greguske Date: Mar 30 2017 13:37:56 +0000 Subject: [PATCH 14/77] lay out rpms by first character --- diff --git a/builder/kojid b/builder/kojid index fd544f5..fa007fa 100755 --- a/builder/kojid +++ b/builder/kojid @@ -5152,7 +5152,8 @@ enabled=1 rc, errors = yumbase.resolveDeps() ml_needed = set() for f in yumbase.tsInfo.getMembers(): - dep_path = os.path.join(mldir, os.path.basename(f.po.localPkg())) + bnp = os.path.basename(f.po.localPkg()) + dep_path = os.path.join(mldir, bnp[0], bnp) ml_needed.add(dep_path) self.logger.debug("added %s" % dep_path) if not os.path.exists(dep_path): @@ -5170,8 +5171,9 @@ enabled=1 pkgwriter = open(self.pkglist, 'a') for ml_pkg in ml_needed: bnp = os.path.basename(ml_pkg) - pkgwriter.write(bnp + '\n') - os.symlink(ml_pkg, os.path.join(self.repodir, bnp)) + pkgwriter.write(bnp[0] + '/' + bnp + '\n') + koji.ensuredir(os.path.join(self.repodir, bnp[0])) + os.symlink(ml_pkg, os.path.join(self.repodir, bnp[0], bnp)) self.keypaths[bnp] = ml_pkg @@ -5224,9 +5226,10 @@ enabled=1 fs_missing.add(pkgpath) else: bnp = os.path.basename(pkgpath) - pkglist.write(bnp + '\n') + pkglist.write(bnp[0] + '/' + bnp + '\n') + koji.ensuredir(os.path.join(self.repodir, bnp[0])) self.keypaths[bnp] = pkgpath - os.symlink(pkgpath, os.path.join(self.repodir, bnp)) + os.symlink(pkgpath, os.path.join(self.repodir, bnp[0], bnp)) pkglist.close() if len(fs_missing) > 0: raise koji.GenericError('Packages missing from the filesystem:\n' + diff --git a/hub/kojihub.py b/hub/kojihub.py index 40a8973..f0d9691 100644 --- a/hub/kojihub.py +++ b/hub/kojihub.py @@ -12339,9 +12339,11 @@ class HostExports(object): # hardlink the found rpms into the final repodir with open(src) as pkgfile: for pkg in pkgfile: - pkg = pkg.strip() + pkg = os.path.basename(pkg.strip()) rpmpath = fullpaths[pkg] - os.link(rpmpath, os.path.join(archdir, os.path.basename(rpmpath))) + bnp = os.path.basename(rpmpath) + koji.ensuredir(os.path.join(archdir, bnp[0])) + os.link(rpmpath, os.path.join(archdir, bnp[0], bnp)) os.unlink(src) def isEnabled(self): From 9b504a280c4444321b546ad93b1c6602b371881c Mon Sep 17 00:00:00 2001 From: Jay Greguske Date: Mar 30 2017 13:37:56 +0000 Subject: [PATCH 15/77] add signed flag to repo table --- diff --git a/builder/kojid b/builder/kojid index fa007fa..42a631b 100755 --- a/builder/kojid +++ b/builder/kojid @@ -4980,7 +4980,7 @@ class NewSignedRepoTask(BaseTaskHandler): upload, files, keypaths = results[subtasks[arch]] self.session.host.signedRepoMove( repo_id, upload, files, arch, keypaths) - self.session.host.repoDone(repo_id, data, expire=True, signed=True) + self.session.host.repoDone(repo_id, data, expire=False, signed=True) return 'Signed repository #%s successfully generated' % repo_id diff --git a/docs/schema.sql b/docs/schema.sql index 021cc4e..2edaab8 100644 --- a/docs/schema.sql +++ b/docs/schema.sql @@ -51,6 +51,7 @@ CREATE TABLE permissions ( INSERT INTO permissions (name) VALUES ('admin'); INSERT INTO permissions (name) VALUES ('build'); INSERT INTO permissions (name) VALUES ('repo'); +INSERT INTO permissions (name) VALUES ('image'); INSERT INTO permissions (name) VALUES ('livecd'); INSERT INTO permissions (name) VALUES ('maven-import'); INSERT INTO permissions (name) VALUES ('win-import'); @@ -409,7 +410,8 @@ CREATE TABLE repo ( id SERIAL NOT NULL PRIMARY KEY, create_event INTEGER NOT NULL REFERENCES events(id) DEFAULT get_event(), tag_id INTEGER NOT NULL REFERENCES tag(id), - state INTEGER + state INTEGER, + signed BOOLEAN DEFAULT 'false' ) WITHOUT OIDS; -- external yum repos diff --git a/hub/kojihub.py b/hub/kojihub.py index f0d9691..0f9834f 100644 --- a/hub/kojihub.py +++ b/hub/kojihub.py @@ -2458,7 +2458,7 @@ def signed_repo_init(tag, keys, task_opts): task_opts['event'] = _singleValue("SELECT get_event()") insert = InsertProcessor('repo') insert.set(id=repo_id, create_event=task_opts['event'], tag_id=tag_id, - state=state) + state=state, signed=True) insert.execute() repodir = koji.pathinfo.signedrepo(repo_id, tinfo['name']) for arch in arches: @@ -2495,6 +2495,7 @@ def repo_info(repo_id, strict=False): ('EXTRACT(EPOCH FROM events.time)', 'create_ts'), ('repo.tag_id', 'tag_id'), ('tag.name', 'tag_name'), + ('repo.signed', 'signed'), ) q = """SELECT %s FROM repo JOIN tag ON tag_id=tag.id @@ -10101,16 +10102,20 @@ class RootExports(object): taginfo['extra'][key] = ancestor['extra'][key] return taginfo - def getRepo(self, tag, state=None, event=None): + def getRepo(self, tag, state=None, event=None, signed=False): if isinstance(tag, (int, long)): id = tag else: id = get_tag_id(tag, strict=True) - fields = ['repo.id', 'repo.state', 'repo.create_event', 'events.time', 'EXTRACT(EPOCH FROM events.time)'] - aliases = ['id', 'state', 'create_event', 'creation_time', 'create_ts'] + fields = ['repo.id', 'repo.state', 'repo.create_event', 'events.time', 'EXTRACT(EPOCH FROM events.time)', 'repo.signed'] + aliases = ['id', 'state', 'create_event', 'creation_time', 'create_ts', 'signed'] joins = ['events ON repo.create_event = events.id'] clauses = ['repo.tag_id = %(id)i'] + if signed: + clauses.append('repo.signed is true') + else: + clauses.append('repo.signed is false') if event: # the repo table doesn't have all the fields of a _config table, just create_event clauses.append('create_event <= %(event)i') diff --git a/util/kojira b/util/kojira index 611edd1..edd4b90 100755 --- a/util/kojira +++ b/util/kojira @@ -134,7 +134,11 @@ class ManagedRepo(object): (self.tag_id, self.repo_id)) return False tag_name = tag_info['name'] - path = pathinfo.repo(self.repo_id, tag_name) + rinfo = self.session.repoInfo(self.repo_id, strict=True) + if rinfo['signed']: + path = pathinfo.signedrepo(self.repo_id, tag_name) + else: + path = pathinfo.repo(self.repo_id, tag_name) try: #also check dir age. We do this because a repo can be created from an older event #and should not be removed based solely on that event's timestamp. From 211ed4bcaefc7c0a7d50893d4d82c34f765faceb Mon Sep 17 00:00:00 2001 From: Jay Greguske Date: Mar 30 2017 13:37:56 +0000 Subject: [PATCH 16/77] fix newRepo in webui --- diff --git a/www/kojiweb/taskinfo.chtml b/www/kojiweb/taskinfo.chtml index 50cec39..613e2f9 100644 --- a/www/kojiweb/taskinfo.chtml +++ b/www/kojiweb/taskinfo.chtml @@ -218,6 +218,11 @@ $value #if $len($params) > 2 $printOpts($params[2]) #end if + #elif $task.method == 'newRepo' + Tag: $tag.name
+ #if $len($params) > 1 + $printOpts($params[1]) + #end if #elif $task.method == 'signedRepo' Tag: $tag.name
Repo ID: $params[1]
From 0b19e890c6c393d2b3a53ca185246668e8e251b2 Mon Sep 17 00:00:00 2001 From: Jay Greguske Date: Mar 30 2017 13:37:56 +0000 Subject: [PATCH 17/77] Make the signedRepo tasks happen in the createrepo channe --- diff --git a/hub/kojihub.py b/hub/kojihub.py index 0f9834f..f7c6c7e 100644 --- a/hub/kojihub.py +++ b/hub/kojihub.py @@ -10138,7 +10138,7 @@ class RootExports(object): context.session.assertPerm('signed-repo') repo_id, event_id = signed_repo_init(tag, keys, task_opts) task_opts['event'] = event_id - return make_task('signedRepo', [tag, repo_id, keys, task_opts], priority=15) + return make_task('signedRepo', [tag, repo_id, keys, task_opts], priority=15, channel='createrepo') def newRepo(self, tag, event=None, src=False, debuginfo=False): """Create a newRepo task. returns task id""" From bed61eec2356f3b448325aea29f19433730f3297 Mon Sep 17 00:00:00 2001 From: Jay Greguske Date: Mar 30 2017 13:37:56 +0000 Subject: [PATCH 18/77] fall back to a copy if we cannot hardlink --- diff --git a/hub/kojihub.py b/hub/kojihub.py index f7c6c7e..b684807 100644 --- a/hub/kojihub.py +++ b/hub/kojihub.py @@ -12342,13 +12342,21 @@ class HostExports(object): os.link(src, dst) if fn.endswith('pkglist'): # hardlink the found rpms into the final repodir + # TODO: properly consider split-volume functionality with open(src) as pkgfile: for pkg in pkgfile: pkg = os.path.basename(pkg.strip()) rpmpath = fullpaths[pkg] bnp = os.path.basename(rpmpath) koji.ensuredir(os.path.join(archdir, bnp[0])) - os.link(rpmpath, os.path.join(archdir, bnp[0], bnp)) + try: + os.link(rpmpath, os.path.join(archdir, bnp[0], bnp)) + except OSError, ose: + if ose.error == 18: + shutil.copy2( + rpmpath, os.path.join(archdir, bnp[0], bnp)) + else: + raise ose os.unlink(src) def isEnabled(self): From 1c0c991f025464bddf09772dd5cf0b814da9ae96 Mon Sep 17 00:00:00 2001 From: Jay Greguske Date: Mar 30 2017 13:37:56 +0000 Subject: [PATCH 19/77] errno not error --- diff --git a/hub/kojihub.py b/hub/kojihub.py index b684807..bc64faf 100644 --- a/hub/kojihub.py +++ b/hub/kojihub.py @@ -12352,7 +12352,7 @@ class HostExports(object): try: os.link(rpmpath, os.path.join(archdir, bnp[0], bnp)) except OSError, ose: - if ose.error == 18: + if ose.errno == 18: shutil.copy2( rpmpath, os.path.join(archdir, bnp[0], bnp)) else: From 47daa0aad814166539e1c1c27f7b3d76ea105ed5 Mon Sep 17 00:00:00 2001 From: Jay Greguske Date: Mar 30 2017 13:37:56 +0000 Subject: [PATCH 20/77] add --non-latest --- diff --git a/builder/kojid b/builder/kojid index 42a631b..820662c 100755 --- a/builder/kojid +++ b/builder/kojid @@ -5185,7 +5185,7 @@ enabled=1 builddirs = {} for a in (arch, 'noarch'): rpm_iter, builds = self.session.listTaggedRPMS(tag_id, - event=opts['event'], arch=a, + event=opts['event'], arch=a, latest=opts['latest'], inherit=opts['inherit'], rpmsigs=True) for build in builds: builddirs[build['id']] = self.pathinfo.build(build) diff --git a/cli/koji b/cli/koji index f10e144..3170e27 100755 --- a/cli/koji +++ b/cli/koji @@ -7091,6 +7091,8 @@ def handle_signed_repo(options, session, args): help=_('Create delta-rpms. PATH points to (older) rpms to generate against. May be specified multiple times. These have to be reachable by the builder too, so the path needs to reach shared storage.')) parser.add_option('--event', type='int', help=_('create a signed repository based on a Brew event')) + parser.add_option('--non-latest', dest='latest', default=True, + action='store_false', help='Include older builds, not just the latest') parser.add_option('--multilib', default=None, help=_('Include multilib packages in the repository using a config')) parser.add_option("--noinherit", action='store_true', default=False, @@ -7155,10 +7157,11 @@ def handle_signed_repo(options, session, args): opts = { 'arch': task_opts.arch, 'comps': task_opts.comps, - 'event': task_opts.event, 'delta': task_opts.delta_rpms, - 'multilib': task_opts.multilib, + 'event': task_opts.event, 'inherit': not task_opts.noinherit, + 'latest': task_opts.latest, + 'multilib': task_opts.multilib, 'skip': task_opts.skip_unsigned, 'unsigned': task_opts.allow_unsigned } From 31b98eefc4f7ad716a7712107b41cfa24ed35f8b Mon Sep 17 00:00:00 2001 From: Jay Greguske Date: Mar 30 2017 13:37:56 +0000 Subject: [PATCH 21/77] fixes from testing and upstream comments --- diff --git a/builder/kojid b/builder/kojid index 820662c..51c0cd0 100755 --- a/builder/kojid +++ b/builder/kojid @@ -4943,8 +4943,12 @@ class NewSignedRepoTask(BaseTaskHandler): if len(task_opts['arch']) == 0: raise koji.GenericError('No arches specified nor for the tag!') subtasks = {} - arch32s = set() + # weed out subarchitectures + canonArches = set() for arch in task_opts['arch']: + canonArches.add(koji.canonArch(arch)) + arch32s = set() + for arch in canonArches: if not rpmUtils.arch.isMultiLibArch(arch): arch32s.add(arch) for arch in arch32s: @@ -4958,11 +4962,11 @@ class NewSignedRepoTask(BaseTaskHandler): results = self.wait(subtasks.values(), all=True, failany=True) for arch in arch32s: # move the 32-bit task output to the final resting place - # so the 64-bit arches can use it + # so the 64-bit arches can use it for multilib upload, files, keypaths = results[subtasks[arch]] self.session.host.signedRepoMove( repo_id, upload, files, arch, keypaths) - for arch in task_opts['arch']: + for arch in canonArches: # do the other arches if arch not in arch32s: arglist = [tag, repo_id, arch, keys, task_opts] @@ -4975,8 +4979,13 @@ class NewSignedRepoTask(BaseTaskHandler): for (arch, task_id) in subtasks.iteritems(): data[arch] = results[task_id] self.logger.debug("DEBUG: %r : %r " % (arch, data[arch])) - if arch not in arch32s: + if task_opts['multilib']: # we moved the 32-bit results before, do the 64-bit + if arch not in arch32s: + upload, files, keypaths = results[subtasks[arch]] + self.session.host.signedRepoMove( + repo_id, upload, files, arch, keypaths) + else: upload, files, keypaths = results[subtasks[arch]] self.session.host.signedRepoMove( repo_id, upload, files, arch, keypaths) @@ -5079,6 +5088,7 @@ class createSignedRepoTask(CreaterepoTask): if mlm.select(po) and self.archmap.has_key(arch): # we need a multilib package to be included # we assume the same signature level is available + # XXX: what is a subarchitecture is the right answer? pl_path = pkg.replace(arch, self.archmap[arch]).strip() # assume this exists in the task results for the ml arch real_path = os.path.join(mldir, pl_path) @@ -5171,9 +5181,9 @@ enabled=1 pkgwriter = open(self.pkglist, 'a') for ml_pkg in ml_needed: bnp = os.path.basename(ml_pkg) - pkgwriter.write(bnp[0] + '/' + bnp + '\n') - koji.ensuredir(os.path.join(self.repodir, bnp[0])) - os.symlink(ml_pkg, os.path.join(self.repodir, bnp[0], bnp)) + pkgwriter.write(bnp[0].lower() + '/' + bnp + '\n') + koji.ensuredir(os.path.join(self.repodir, bnp[0].lower())) + os.symlink(ml_pkg, os.path.join(self.repodir, bnp[0].lower(), bnp)) self.keypaths[bnp] = ml_pkg @@ -5183,7 +5193,7 @@ enabled=1 # it is possible to see the results of other committed transactions rpms = [] builddirs = {} - for a in (arch, 'noarch'): + for a in self.compat[arch] + ('noarch',): rpm_iter, builds = self.session.listTaggedRPMS(tag_id, event=opts['event'], arch=a, latest=opts['latest'], inherit=opts['inherit'], rpmsigs=True) @@ -5226,10 +5236,11 @@ enabled=1 fs_missing.add(pkgpath) else: bnp = os.path.basename(pkgpath) - pkglist.write(bnp[0] + '/' + bnp + '\n') - koji.ensuredir(os.path.join(self.repodir, bnp[0])) + pkglist.write(bnp[0].lower() + '/' + bnp + '\n') + koji.ensuredir(os.path.join(self.repodir, bnp[0].lower())) self.keypaths[bnp] = pkgpath - os.symlink(pkgpath, os.path.join(self.repodir, bnp[0], bnp)) + os.symlink(pkgpath, os.path.join(self.repodir, bnp[0].lower(), + bnp)) pkglist.close() if len(fs_missing) > 0: raise koji.GenericError('Packages missing from the filesystem:\n' + From 0a19bee1038e75430cc0b81d720a8feae1077ff2 Mon Sep 17 00:00:00 2001 From: Jay Greguske Date: Mar 30 2017 13:37:56 +0000 Subject: [PATCH 22/77] make the src arch work in signed repos --- diff --git a/builder/kojid b/builder/kojid index 51c0cd0..80a02a0 100755 --- a/builder/kojid +++ b/builder/kojid @@ -5012,6 +5012,7 @@ class createSignedRepoTask(CreaterepoTask): "arm": ("arm", "armv4l", "armv4tl", "armv5tel", "armv5tejl", "armv6l", "armv7l", "noarch"), "armhfp": ("armv7hl", "armv7hnl", "noarch"), "aarch64": ("aarch64", "noarch"), + "src": ("src",) } biarch = {"ppc": "ppc64", "x86_64": "i386", "sparc": From f35d3aad7bd3fabcdd6430fe9004d910484ede82 Mon Sep 17 00:00:00 2001 From: Jay Greguske Date: Mar 30 2017 13:37:56 +0000 Subject: [PATCH 23/77] lowercase directories in signed repos --- diff --git a/builder/kojid b/builder/kojid index 80a02a0..76f12df 100755 --- a/builder/kojid +++ b/builder/kojid @@ -5164,7 +5164,7 @@ enabled=1 ml_needed = set() for f in yumbase.tsInfo.getMembers(): bnp = os.path.basename(f.po.localPkg()) - dep_path = os.path.join(mldir, bnp[0], bnp) + dep_path = os.path.join(mldir, bnp[0].lower(), bnp) ml_needed.add(dep_path) self.logger.debug("added %s" % dep_path) if not os.path.exists(dep_path): @@ -5182,9 +5182,10 @@ enabled=1 pkgwriter = open(self.pkglist, 'a') for ml_pkg in ml_needed: bnp = os.path.basename(ml_pkg) - pkgwriter.write(bnp[0].lower() + '/' + bnp + '\n') - koji.ensuredir(os.path.join(self.repodir, bnp[0].lower())) - os.symlink(ml_pkg, os.path.join(self.repodir, bnp[0].lower(), bnp)) + bnplet = bnp[0].lower() + pkgwriter.write(bnplet + '/' + bnp + '\n') + koji.ensuredir(os.path.join(self.repodir, bnplet)) + os.symlink(ml_pkg, os.path.join(self.repodir, bnplet, bnp)) self.keypaths[bnp] = ml_pkg @@ -5237,11 +5238,11 @@ enabled=1 fs_missing.add(pkgpath) else: bnp = os.path.basename(pkgpath) - pkglist.write(bnp[0].lower() + '/' + bnp + '\n') - koji.ensuredir(os.path.join(self.repodir, bnp[0].lower())) + bnplet = bnp[0].lower() + pkglist.write(bnplet + '/' + bnp + '\n') + koji.ensuredir(os.path.join(self.repodir, bnplet)) self.keypaths[bnp] = pkgpath - os.symlink(pkgpath, os.path.join(self.repodir, bnp[0].lower(), - bnp)) + os.symlink(pkgpath, os.path.join(self.repodir, bnplet, bnp)) pkglist.close() if len(fs_missing) > 0: raise koji.GenericError('Packages missing from the filesystem:\n' + diff --git a/hub/kojihub.py b/hub/kojihub.py index bc64faf..71efe39 100644 --- a/hub/kojihub.py +++ b/hub/kojihub.py @@ -12348,13 +12348,14 @@ class HostExports(object): pkg = os.path.basename(pkg.strip()) rpmpath = fullpaths[pkg] bnp = os.path.basename(rpmpath) - koji.ensuredir(os.path.join(archdir, bnp[0])) + bnplet = bnp[0].lower() + koji.ensuredir(os.path.join(archdir, bnplet)) try: - os.link(rpmpath, os.path.join(archdir, bnp[0], bnp)) + os.link(rpmpath, os.path.join(archdir, bnplet, bnp)) except OSError, ose: if ose.errno == 18: shutil.copy2( - rpmpath, os.path.join(archdir, bnp[0], bnp)) + rpmpath, os.path.join(archdir, bnplet, bnp)) else: raise ose os.unlink(src) From 6a6f79dec3b4e74225ed0661ba2b0efb6527f227 Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Mar 30 2017 13:37:56 +0000 Subject: [PATCH 24/77] remove incorrect singleton syntax in _taskLabel() --- diff --git a/koji/__init__.py b/koji/__init__.py index bf628fe..593e7c9 100644 --- a/koji/__init__.py +++ b/koji/__init__.py @@ -2803,11 +2803,11 @@ def _taskLabel(taskInfo): if 'request' in taskInfo: tagInfo = taskInfo['request'][0] extra = tagInfo['name'] - elif method in ('createrepo'): + elif method == 'createrepo': if 'request' in taskInfo: arch = taskInfo['request'][1] extra = arch - elif method in ('createsignedrepo'): + elif method == 'createsignedrepo': if 'request' in taskInfo: repo_id = taskInfo['request'][1] arch = taskInfo['request'][2] From b6bc8f561b5475139a399df4fd86d32a897ba2aa Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Mar 30 2017 13:37:56 +0000 Subject: [PATCH 25/77] remove stray comment copied from hub code --- diff --git a/builder/kojid b/builder/kojid index 76f12df..f9e24de 100755 --- a/builder/kojid +++ b/builder/kojid @@ -5191,8 +5191,6 @@ enabled=1 def make_pkglist(self, tag_id, arch, keys, opts): - # Need to pass event_id because even though this is a single trans, - # it is possible to see the results of other committed transactions rpms = [] builddirs = {} for a in self.compat[arch] + ('noarch',): From f383f41a5586b74fe4345a66adf73594a9b1a58b Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Mar 30 2017 13:37:56 +0000 Subject: [PATCH 26/77] clean up signed_repo_init - remove unused logger - keep pre/postRepoInit callbacks compatible with existing ones --- diff --git a/hub/kojihub.py b/hub/kojihub.py index 71efe39..0a8e826 100644 --- a/hub/kojihub.py +++ b/hub/kojihub.py @@ -2444,18 +2444,17 @@ def _write_maven_repo_metadata(destdir, artifacts): def signed_repo_init(tag, keys, task_opts): """Create a new repo entry in the INIT state, return full repo data""" - logger = logging.getLogger("koji.hub.signed_repo_init") state = koji.REPO_INIT tinfo = get_tag(tag, strict=True) - koji.plugin.run_callbacks('preRepoInit', tag=tinfo, keys=keys, repo_id=None) tag_id = tinfo['id'] - repo_id = _singleValue("SELECT nextval('repo_id_seq')") - repo_arches = task_opts['arch'] - arches = set([]) - for arch in repo_arches: - arches.add(koji.canonArch(arch)) + arches = set([koji.canonArch(a) for a in task_opts['arch']]) + # note: we need to match args from the other preRepoInit callback + koji.plugin.run_callbacks('preRepoInit', tag=tinfo, with_src=False, + with_debuginfo=False, event=task_opts['event'], repo_id=None, + signed=True, keys=keys, arches=arches, task_opts=task_opts) if not task_opts['event']: task_opts['event'] = _singleValue("SELECT get_event()") + repo_id = _singleValue("SELECT nextval('repo_id_seq')") insert = InsertProcessor('repo') insert.set(id=repo_id, create_event=task_opts['event'], tag_id=tag_id, state=state, signed=True) @@ -2469,8 +2468,9 @@ def signed_repo_init(tag, keys, task_opts): koji.ensuredir(groupsdir) shutil.copyfile(os.path.join(koji.pathinfo.work(), task_opts['comps']), groupsdir + '/comps.xml') - koji.plugin.run_callbacks('postRepoInit', tag=tinfo, - event=task_opts['event'], repo_id=repo_id) + # note: we need to match args from the other postRepoInit callback + koji.plugin.run_callbacks('postRepoInit', tag=tinfo, with_src=False, + with_debuginfo=False, event=task_opts['event'], repo_id=repo_id) return repo_id, task_opts['event'] From 7c22cf4825847b8064d4c57a75c673c8aebcb92d Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Mar 30 2017 13:37:56 +0000 Subject: [PATCH 27/77] clean up strings in signed-repo cli --- diff --git a/cli/koji b/cli/koji index 3170e27..c0d6619 100755 --- a/cli/koji +++ b/cli/koji @@ -7088,7 +7088,10 @@ def handle_signed_repo(options, session, args): parser.add_option('--comps', help='Include a comps file in the repodata') parser.add_option('--delta-rpms', metavar='PATH',default=[], action='append', - help=_('Create delta-rpms. PATH points to (older) rpms to generate against. May be specified multiple times. These have to be reachable by the builder too, so the path needs to reach shared storage.')) + help=_('Create delta-rpms. PATH points to (older) rpms to generate ' + 'against. May be specified multiple times. These have to be ' + 'reachable by the builder too, so the path needs to reach shared ' + 'storage.')) parser.add_option('--event', type='int', help=_('create a signed repository based on a Brew event')) parser.add_option('--non-latest', dest='latest', default=True, @@ -7105,12 +7108,12 @@ def handle_signed_repo(options, session, args): if len(args) < 2: parser.error(_('You must provide a tag and 1 or more GPG key IDs')) if task_opts.allow_unsigned and task_opts.skip_unsigned: - parser.error(_('allow_signed and skip_unsigned are mutually exclusive')) + parser.error(_('allow_unsigned and skip_unsigned are mutually exclusive')) activate_session(session) stuffdir = _unique_path('cli-signed') if task_opts.comps: if not os.path.exists(task_opts.comps): - parser.error(_('could not find %s' % task_opts.comps)) + parser.error(_('could not find %s') % task_opts.comps) session.uploadWrapper(task_opts.comps, stuffdir, callback=_progress_callback) print @@ -7119,14 +7122,14 @@ def handle_signed_repo(options, session, args): if len(task_opts.delta_rpms) > 0: for path in task_opts.delta_rpms: if not os.path.exists(path): - print _("Warning: %s is not reachable locally. If this" % path) - print _(" host does not have access to Koji's shared storage") - print _(" this can be ignored.") + print _("Warning: %s is not reachable locally. If this\n" + " host does not have access to Koji's shared storage\n" + " this can be ignored.") % path tag = args[0] keys = args[1:] taginfo = session.getTag(tag) if not taginfo: - parser.error(_('unknown tag %s' % tag)) + parser.error(_('unknown tag %s') % tag) if len(task_opts.arch) == 0: task_opts.arch = taginfo['arches'] if task_opts.arch == None: @@ -7134,10 +7137,10 @@ def handle_signed_repo(options, session, args): else: for a in task_opts.arch: if not taginfo['arches'] or a not in taginfo['arches']: - print _('Warning: %s is not in the list of tag arches' % a) + print _('Warning: %s is not in the list of tag arches') % a if task_opts.multilib: if not os.path.exists(task_opts.multilib): - parser.error(_('could not find %s' % task_opts.multilib)) + parser.error(_('could not find %s') % task_opts.multilib) if 'x86_64' in task_opts.arch and not 'i686' in task_opts.arch: parser.error(_('The multilib arch (i686) must be included')) if 's390x' in task_opts.arch and not 's390' in task_opts.arch: From f30d5ba75877574cd6b7e7daa55d4d133796cd03 Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Mar 30 2017 13:37:56 +0000 Subject: [PATCH 28/77] repoDone knows if a repo is signed --- diff --git a/builder/kojid b/builder/kojid index f9e24de..32b716e 100755 --- a/builder/kojid +++ b/builder/kojid @@ -4989,7 +4989,7 @@ class NewSignedRepoTask(BaseTaskHandler): upload, files, keypaths = results[subtasks[arch]] self.session.host.signedRepoMove( repo_id, upload, files, arch, keypaths) - self.session.host.repoDone(repo_id, data, expire=False, signed=True) + self.session.host.repoDone(repo_id, data, expire=False) return 'Signed repository #%s successfully generated' % repo_id diff --git a/hub/kojihub.py b/hub/kojihub.py index 0a8e826..f2b521c 100644 --- a/hub/kojihub.py +++ b/hub/kojihub.py @@ -12263,13 +12263,15 @@ class HostExports(object): else: safer_move(filepath, dst) - def repoDone(self, repo_id, data, expire=False, signed=False): + def repoDone(self, repo_id, data, expire=False): """Move repo data into place, mark as ready, and expire earlier repos repo_id: the id of the repo data: a dictionary of the form { arch: (uploadpath, files), ...} expire(optional): if set to true, mark the repo expired immediately* - signed(optional): if true, hardlink signed rpms in the final directory + + If this is a signed repo, also hardlink signed rpms in the final + directory. * This is used when a repo from an older event is generated """ @@ -12281,7 +12283,7 @@ class HostExports(object): raise koji.GenericError("Repo %(id)s not in INIT state (got %(state)s)" % rinfo) repodir = koji.pathinfo.repo(repo_id, rinfo['tag_name']) workdir = koji.pathinfo.work() - if not signed: + if not rinfo['signed']: for arch, (uploadpath, files) in data.iteritems(): archdir = "%s/%s" % (repodir, koji.canonArch(arch)) if not os.path.isdir(archdir): From f58790091d78190a1bec14b45a49fc04aed7e626 Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Mar 30 2017 13:37:56 +0000 Subject: [PATCH 29/77] use safer_move in repoDone --- diff --git a/hub/kojihub.py b/hub/kojihub.py index f2b521c..17f89d1 100644 --- a/hub/kojihub.py +++ b/hub/kojihub.py @@ -12341,7 +12341,6 @@ class HostExports(object): dst = "%s/%s" % (datadir, fn) if not os.path.exists(src): raise koji.GenericError, "uploaded file missing: %s" % src - os.link(src, dst) if fn.endswith('pkglist'): # hardlink the found rpms into the final repodir # TODO: properly consider split-volume functionality @@ -12360,7 +12359,7 @@ class HostExports(object): rpmpath, os.path.join(archdir, bnplet, bnp)) else: raise ose - os.unlink(src) + safer_move(src, dst) def isEnabled(self): host = Host() From 0957834e0cc062c088658d34091cbc765f8a1977 Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Mar 30 2017 13:45:59 +0000 Subject: [PATCH 30/77] update data for list-commands test --- diff --git a/tests/test_cli/data/list-commands.txt b/tests/test_cli/data/list-commands.txt index 6249a73..8c5a54f 100644 --- a/tests/test_cli/data/list-commands.txt +++ b/tests/test_cli/data/list-commands.txt @@ -119,6 +119,7 @@ miscellaneous commands: import-comps Import group/package information from a comps file moshimoshi Introduce yourself save-failed-tree Create tarball with whole buildtree + signed-repo create a yum repo of GPG signed RPMs monitor commands: wait-repo Wait for a repo to be regenerated From cc19dfb1222f0c2fb7d128d78ade41324a669524 Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Mar 30 2017 13:45:59 +0000 Subject: [PATCH 31/77] fix default arches for signed-repo cli --- diff --git a/cli/koji b/cli/koji index c0d6619..3149cdf 100755 --- a/cli/koji +++ b/cli/koji @@ -7131,7 +7131,7 @@ def handle_signed_repo(options, session, args): if not taginfo: parser.error(_('unknown tag %s') % tag) if len(task_opts.arch) == 0: - task_opts.arch = taginfo['arches'] + task_opts.arch = taginfo['arches'].split() if task_opts.arch == None: parser.error(_('No arches given and no arches associated with tag')) else: From 1f29246f26d1a2a539141fa44f62713885087372 Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Mar 30 2017 13:45:59 +0000 Subject: [PATCH 32/77] schema update script for signed repos feature --- diff --git a/docs/schema-update-signed-repos.sql b/docs/schema-update-signed-repos.sql new file mode 100644 index 0000000..e67abb4 --- /dev/null +++ b/docs/schema-update-signed-repos.sql @@ -0,0 +1,7 @@ +# schema updates for signed repo feature +# to be merged into schema upgrade script for next release + +INSERT INTO permissions (name) VALUES ('image'); + +ALTER TABLE repo ADD COLUMN signed BOOLEAN DEFAULT 'false'; + From d161a1b4a323119f2ddd32893faa397f73029fb3 Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Mar 30 2017 13:45:59 +0000 Subject: [PATCH 33/77] support nvra as well as nvr in write-signed-rpm --- diff --git a/cli/koji b/cli/koji index 3149cdf..e3e396a 100755 --- a/cli/koji +++ b/cli/koji @@ -1922,6 +1922,7 @@ def handle_import_sig(options, session, args): if not options.test: session.addRPMSig(rinfo['id'], base64.encodestring(sighdr)) + def handle_write_signed_rpm(options, session, args): "[admin] Write signed RPMs to disk" usage = _("usage: %prog write-signed-rpm [options] n-v-r [n-v-r...]") @@ -1940,21 +1941,31 @@ def handle_write_signed_rpm(options, session, args): activate_session(session) if options.all: rpms = session.queryRPMSigs(sigkey=key) - count = 1 - for rpm in rpms: - print("%d/%d" % (count, len(rpms))) - count += 1 - session.writeSignedRPM(rpm['rpm_id'], key) + rpms = [session.getRPM(r['rpm_id']) for r in rpms] elif options.buildid: rpms = session.listRPMs(int(options.buildid)) - for rpm in rpms: - session.writeSignedRPM(rpm['id'], key) else: - for nvr in args: + rpms = [] + bad = [] + for nvra in args: + try: + koji.parse_NVRA(nvra) + rinfo = session.getRPM(nvra, strict=True) + if rinfo: + rpms.append(rinfo) + except koji.GenericError: + bad.append(nvra) + # for historical reasons, we also accept nvrs + for nvr in bad: build = session.getBuild(nvr) - rpms = session.listRPMs(buildID=build['id']) - for rpm in rpms: - session.writeSignedRPM(rpm['id'], key) + if not build: + raise koji.GenericError("No such rpm or build: %s" % nvr) + rpms.extend(session.listRPMs(buildID=build['id'])) + for i, rpminfo in enumerate(rpms): + nvra = "%(name)s-%(version)s-%(release)s.%(arch)s" % rpminfo + print "[%d/%d] %s" % (i+1, len(rpms), nvra) + session.writeSignedRPM(rpminfo['id'], key) + def handle_prune_signed_copies(options, session, args): "[admin] Prune signed copies" From f5707043a59b03f43bd01d381d9a8cbd5fd8be79 Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Mar 30 2017 13:45:59 +0000 Subject: [PATCH 34/77] option to write signed copy when importing signature --- diff --git a/cli/koji b/cli/koji index e3e396a..e2f9508 100755 --- a/cli/koji +++ b/cli/koji @@ -1872,6 +1872,8 @@ def handle_import_sig(options, session, args): parser = OptionParser(usage=usage) parser.add_option("--with-unsigned", action="store_true", help=_("Also import unsigned sig headers")) + parser.add_option("--write", action="store_true", + help=_("Also write the signed copies")) parser.add_option("--test", action="store_true", help=_("Test mode -- don't actually import")) (options, args) = parser.parse_args(args) @@ -1921,6 +1923,9 @@ def handle_import_sig(options, session, args): print(_("Importing signature [key %s] from %s...") % (sigkey, path)) if not options.test: session.addRPMSig(rinfo['id'], base64.encodestring(sighdr)) + print _("Writing signed copy") + if not options.test: + session.writeSignedRPM(rinfo['id'], sigkey) def handle_write_signed_rpm(options, session, args): From 75ac8be3b7a845b9bab591bfa5af095f0af1b066 Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Mar 30 2017 13:45:59 +0000 Subject: [PATCH 35/77] avoid directory namespace conflicts with signed repos --- diff --git a/koji/__init__.py b/koji/__init__.py index 593e7c9..4ac9a68 100644 --- a/koji/__init__.py +++ b/koji/__init__.py @@ -1817,7 +1817,7 @@ class PathInfo(object): def signedrepo(self, repo_id, tag): """Return the directory with a signed repo lives""" - return os.path.join(self.topdir, 'repos', 'signed', tag, str(repo_id)) + return os.path.join(self.topdir, 'repos-signed', tag, str(repo_id)) def repocache(self, tag_str): """Return the directory where a repo belongs""" diff --git a/util/kojira b/util/kojira index edd4b90..c240ead 100755 --- a/util/kojira +++ b/util/kojira @@ -357,9 +357,6 @@ class RepoManager(object): repo_id = int(repo_id) except ValueError: self.logger.debug("%s not an int, skipping" % tagdir) - # This condition is how signed repos are not removed by - # the first call to this method. Although, if someone has - # tags that are just integers, that could be a problem. continue repodir = "%s/%s" % (tagdir, repo_id) if not os.path.isdir(repodir): @@ -642,14 +639,15 @@ def main(options, session): curr_chk_thread = start_currency_checker(session, repomgr) # TODO also move rmtree jobs to threads logger.info("Entering main loop") + repodir = "%s/repos" % pathinfo.topdir + signedrepodir = "%s/repos-signed" % pathinfo.topdir while True: try: repomgr.updateRepos() repomgr.checkQueue() repomgr.printState() - repodir = "%s/repos" % pathinfo.topdir repomgr.pruneLocalRepos(repodir, 'deleted_repo_lifetime') - repomgr.pruneLocalRepos(repodir + '/signed', 'signed_repo_lifetime') + repomgr.pruneLocalRepos(signedrepodir, 'signed_repo_lifetime') if not curr_chk_thread.isAlive(): logger.error("Currency checker thread died. Restarting it.") curr_chk_thread = start_currency_checker(session, repomgr) From ec5d71016c4e401281c51e72fe53d9d1c1119390 Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Mar 30 2017 13:45:59 +0000 Subject: [PATCH 36/77] logging adjustments --- diff --git a/util/kojira b/util/kojira index c240ead..7fced68 100755 --- a/util/kojira +++ b/util/kojira @@ -344,42 +344,42 @@ class RepoManager(object): if self.delete_pids: #skip return - self.logger.debug("Scanning %s for repos" % topdir) - self.logger.debug('max age allowed: %s seconds (from %s)' % - (getattr(self.options, timername), timername)) + self.logger.debug("Scanning %s for repos", topdir) + self.logger.debug('max age allowed: %s seconds (from %s)', + getattr(self.options, timername), timername) for tag in os.listdir(topdir): tagdir = "%s/%s" % (topdir, tag) if not os.path.isdir(tagdir): - self.logger.debug("%s is not a directory, skipping" % tagdir) + self.logger.debug("%s is not a directory, skipping", tagdir) continue for repo_id in os.listdir(tagdir): try: repo_id = int(repo_id) except ValueError: - self.logger.debug("%s not an int, skipping" % tagdir) + self.logger.debug("%s not an int, skipping", tagdir) continue repodir = "%s/%s" % (tagdir, repo_id) if not os.path.isdir(repodir): - self.logger.debug("%s not a directory, skipping" % repodir) + self.logger.debug("%s not a directory, skipping", repodir) continue if repo_id in self.repos: #we're already managing it, no need to deal with it here - self.logger.debug("seen %s already, skipping" % repodir) + self.logger.debug("seen %s already, skipping", repodir) continue try: dir_ts = os.stat(repodir).st_mtime except OSError: #just in case something deletes the repo out from under us - self.logger.debug("%s deleted already?!" % repodir) + self.logger.debug("%s deleted already?!", repodir) continue rinfo = self.session.repoInfo(repo_id) if rinfo is None: if not self.options.ignore_stray_repos: age = time.time() - dir_ts - self.logger.debug("did not expect %s; age: %s" % - (repodir, age)) + self.logger.debug("did not expect %s; age: %s", + repodir, age) if age > getattr(self.options, timername): - self.logger.info("Removing unexpected directory (no such repo): %s" % repodir) + self.logger.info("Removing unexpected directory (no such repo): %s", repodir) self.rmtree(repodir) continue if rinfo['tag_name'] != tag: From da540262789cd922245f84e27a9b1a781ec92311 Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Mar 30 2017 13:45:59 +0000 Subject: [PATCH 37/77] use correct lifetime option for deleting signed repos --- diff --git a/util/kojira b/util/kojira index 7fced68..3e73ce3 100755 --- a/util/kojira +++ b/util/kojira @@ -137,8 +137,11 @@ class ManagedRepo(object): rinfo = self.session.repoInfo(self.repo_id, strict=True) if rinfo['signed']: path = pathinfo.signedrepo(self.repo_id, tag_name) + lifetime = self.options.signed_repo_lifetime else: path = pathinfo.repo(self.repo_id, tag_name) + lifetime = self.options.deleted_repo_lifetime + # (should really be called expired_repo_lifetime) try: #also check dir age. We do this because a repo can be created from an older event #and should not be removed based solely on that event's timestamp. @@ -156,8 +159,7 @@ class ManagedRepo(object): times = [self.event_ts, mtime, self.first_seen, self.expire_ts] times = [ts for ts in times if ts is not None] age = time.time() - max(times) - if age < self.options.deleted_repo_lifetime: - #XXX should really be called expired_repo_lifetime + if age < lifetime: return False self.logger.debug("Attempting to delete repo %s.." % self.repo_id) if self.state != koji.REPO_EXPIRED: @@ -389,7 +391,6 @@ class RepoManager(object): age = time.time() - max(rinfo['create_ts'], dir_ts) self.logger.debug("potential removal candidate: %s; age: %s" % (repodir, age)) if age > getattr(self.options, timername): - #XXX should really be called expired_repo_lifetime logger.info("Removing stray repo (state=%s): %s" % (koji.REPO_STATES[rinfo['state']], repodir)) self.rmtree(repodir) From 87a44ac86372f7e83451941d227fbbddd3265974 Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Mar 30 2017 13:45:59 +0000 Subject: [PATCH 38/77] debug logging --- diff --git a/util/kojira b/util/kojira index 3e73ce3..38e02b3 100755 --- a/util/kojira +++ b/util/kojira @@ -159,6 +159,7 @@ class ManagedRepo(object): times = [self.event_ts, mtime, self.first_seen, self.expire_ts] times = [ts for ts in times if ts is not None] age = time.time() - max(times) + self.logger.debug("Repo %s (%s) age: %i sec", self.repo_id, path, age) if age < lifetime: return False self.logger.debug("Attempting to delete repo %s.." % self.repo_id) From a7bff7c7fe36a82032172bad09756ce7dd636e02 Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Mar 30 2017 13:45:59 +0000 Subject: [PATCH 39/77] fix multilib import --- diff --git a/builder/kojid b/builder/kojid index 32b716e..4c6665c 100755 --- a/builder/kojid +++ b/builder/kojid @@ -35,7 +35,7 @@ import logging.handlers from koji.daemon import incremental_upload, log_output, TaskManager, SCM from koji.tasks import ServerExit, ServerRestart, BaseTaskHandler, MultiPlatformTask from koji.util import parseStatus, isSuccess, dslice, dslice_ex -import multilib +import multilib.multilib as multilib import os import pwd import grp From d133806c841be6ea854dfb141c12db17538e89d6 Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Mar 30 2017 13:45:59 +0000 Subject: [PATCH 40/77] avoid duplicate hard linking --- diff --git a/builder/kojid b/builder/kojid index 4c6665c..63adc3a 100755 --- a/builder/kojid +++ b/builder/kojid @@ -5185,7 +5185,12 @@ enabled=1 bnplet = bnp[0].lower() pkgwriter.write(bnplet + '/' + bnp + '\n') koji.ensuredir(os.path.join(self.repodir, bnplet)) - os.symlink(ml_pkg, os.path.join(self.repodir, bnplet, bnp)) + dst = os.path.join(self.repodir, bnplet, bnp) + if os.path.exists(dst): + self.logger.warning("Path exists: %r", dst) + continue + self.logger.debug("os.symlink(%r, %r)", ml_pkg, dst) + os.symlink(ml_pkg, dst) self.keypaths[bnp] = ml_pkg @@ -5240,7 +5245,9 @@ enabled=1 pkglist.write(bnplet + '/' + bnp + '\n') koji.ensuredir(os.path.join(self.repodir, bnplet)) self.keypaths[bnp] = pkgpath - os.symlink(pkgpath, os.path.join(self.repodir, bnplet, bnp)) + dst = os.path.join(self.repodir, bnplet, bnp) + self.logger.debug("os.symlink(%r, %r(", pkgpath, dst) + os.symlink(pkgpath, dst) pkglist.close() if len(fs_missing) > 0: raise koji.GenericError('Packages missing from the filesystem:\n' + diff --git a/hub/kojihub.py b/hub/kojihub.py index 17f89d1..bd96cb9 100644 --- a/hub/kojihub.py +++ b/hub/kojihub.py @@ -12351,14 +12351,19 @@ class HostExports(object): bnp = os.path.basename(rpmpath) bnplet = bnp[0].lower() koji.ensuredir(os.path.join(archdir, bnplet)) + l_dst = os.path.join(archdir, bnplet, bnp) + if os.path.exists(l_dst): + logger.warning("Path exists: %s", l_dst) + continue + logger.debug("os.link(%r, %r)", rpmpath, l_dst) try: - os.link(rpmpath, os.path.join(archdir, bnplet, bnp)) + os.link(rpmpath, l_dst) except OSError, ose: if ose.errno == 18: shutil.copy2( rpmpath, os.path.join(archdir, bnplet, bnp)) else: - raise ose + raise safer_move(src, dst) def isEnabled(self): From 576885de1c3901764ea5c1e119cda7499a2fe9b2 Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Mar 30 2017 13:45:59 +0000 Subject: [PATCH 41/77] avoid noarch duplication --- diff --git a/builder/kojid b/builder/kojid index 63adc3a..39369cb 100755 --- a/builder/kojid +++ b/builder/kojid @@ -5164,6 +5164,9 @@ enabled=1 ml_needed = set() for f in yumbase.tsInfo.getMembers(): bnp = os.path.basename(f.po.localPkg()) + if f.arch == 'noarch': + # noarch packages should already be there + continue dep_path = os.path.join(mldir, bnp[0].lower(), bnp) ml_needed.add(dep_path) self.logger.debug("added %s" % dep_path) @@ -5183,12 +5186,12 @@ enabled=1 for ml_pkg in ml_needed: bnp = os.path.basename(ml_pkg) bnplet = bnp[0].lower() - pkgwriter.write(bnplet + '/' + bnp + '\n') koji.ensuredir(os.path.join(self.repodir, bnplet)) dst = os.path.join(self.repodir, bnplet, bnp) if os.path.exists(dst): self.logger.warning("Path exists: %r", dst) continue + pkgwriter.write(bnplet + '/' + bnp + '\n') self.logger.debug("os.symlink(%r, %r)", ml_pkg, dst) os.symlink(ml_pkg, dst) self.keypaths[bnp] = ml_pkg From aa18a1f77ae8ede918ae73a3b7fda7240d407506 Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Mar 30 2017 13:45:59 +0000 Subject: [PATCH 42/77] rework noarch filter --- diff --git a/builder/kojid b/builder/kojid index 39369cb..9438829 100755 --- a/builder/kojid +++ b/builder/kojid @@ -5161,14 +5161,11 @@ enabled=1 # step 4: execute yum transaction to get dependencies self.logger.info("Resolving depenencies for arch %s" % arch) rc, errors = yumbase.resolveDeps() - ml_needed = set() - for f in yumbase.tsInfo.getMembers(): - bnp = os.path.basename(f.po.localPkg()) - if f.arch == 'noarch': - # noarch packages should already be there - continue + ml_needed = {} + for tspkg in yumbase.tsInfo.getMembers(): + bnp = os.path.basename(tspkg.po.localPkg()) dep_path = os.path.join(mldir, bnp[0].lower(), bnp) - ml_needed.add(dep_path) + ml_needed[dep_path] = tspkg self.logger.debug("added %s" % dep_path) if not os.path.exists(dep_path): self.logger.error('%s (multilib dep) not on filesystem' % dep_path) @@ -5183,18 +5180,21 @@ enabled=1 # step 5: add dependencies to our package list pkgwriter = open(self.pkglist, 'a') - for ml_pkg in ml_needed: - bnp = os.path.basename(ml_pkg) + for dep_path in ml_needed: + tspkg = ml_needed[dep_path] + bnp = os.path.basename(dep_path) bnplet = bnp[0].lower() koji.ensuredir(os.path.join(self.repodir, bnplet)) dst = os.path.join(self.repodir, bnplet, bnp) if os.path.exists(dst): - self.logger.warning("Path exists: %r", dst) + # we expect duplication with noarch, but not other arches + if tspkg.arch != 'noarch': + self.logger.warning("Path exists: %r", dst) continue pkgwriter.write(bnplet + '/' + bnp + '\n') - self.logger.debug("os.symlink(%r, %r)", ml_pkg, dst) - os.symlink(ml_pkg, dst) - self.keypaths[bnp] = ml_pkg + self.logger.debug("os.symlink(%r, %r)", dep_path, dst) + os.symlink(dep_path, dst) + self.keypaths[bnp] = dep_path def make_pkglist(self, tag_id, arch, keys, opts): From b79f90eda4ca0d24915cfc13a601c04f9b5bd1a1 Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Mar 30 2017 13:45:59 +0000 Subject: [PATCH 43/77] avoid print statement --- diff --git a/cli/koji b/cli/koji index e2f9508..a604ae1 100755 --- a/cli/koji +++ b/cli/koji @@ -1923,7 +1923,7 @@ def handle_import_sig(options, session, args): print(_("Importing signature [key %s] from %s...") % (sigkey, path)) if not options.test: session.addRPMSig(rinfo['id'], base64.encodestring(sighdr)) - print _("Writing signed copy") + print(_("Writing signed copy")) if not options.test: session.writeSignedRPM(rinfo['id'], sigkey) @@ -1968,7 +1968,7 @@ def handle_write_signed_rpm(options, session, args): rpms.extend(session.listRPMs(buildID=build['id'])) for i, rpminfo in enumerate(rpms): nvra = "%(name)s-%(version)s-%(release)s.%(arch)s" % rpminfo - print "[%d/%d] %s" % (i+1, len(rpms), nvra) + print("[%d/%d] %s" % (i+1, len(rpms), nvra)) session.writeSignedRPM(rpminfo['id'], key) @@ -7138,9 +7138,9 @@ def handle_signed_repo(options, session, args): if len(task_opts.delta_rpms) > 0: for path in task_opts.delta_rpms: if not os.path.exists(path): - print _("Warning: %s is not reachable locally. If this\n" + print(_("Warning: %s is not reachable locally. If this\n" " host does not have access to Koji's shared storage\n" - " this can be ignored.") % path + " this can be ignored.") % path) tag = args[0] keys = args[1:] taginfo = session.getTag(tag) @@ -7153,7 +7153,7 @@ def handle_signed_repo(options, session, args): else: for a in task_opts.arch: if not taginfo['arches'] or a not in taginfo['arches']: - print _('Warning: %s is not in the list of tag arches') % a + print(_('Warning: %s is not in the list of tag arches') % a) if task_opts.multilib: if not os.path.exists(task_opts.multilib): parser.error(_('could not find %s') % task_opts.multilib) @@ -7185,7 +7185,7 @@ def handle_signed_repo(options, session, args): 'unsigned': task_opts.allow_unsigned } task_id = session.signedRepo(tag, keys, **opts) - print "Creating signed repo for tag " + tag + print("Creating signed repo for tag " + tag) if _running_in_bg() or task_opts.nowait: return else: From 531e3db1f2a0782ae5d27e0790cf0e4f7a385c5b Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Mar 30 2017 13:45:59 +0000 Subject: [PATCH 44/77] use nextval function in signed_repo_init() --- diff --git a/hub/kojihub.py b/hub/kojihub.py index bd96cb9..c461568 100644 --- a/hub/kojihub.py +++ b/hub/kojihub.py @@ -2453,8 +2453,8 @@ def signed_repo_init(tag, keys, task_opts): with_debuginfo=False, event=task_opts['event'], repo_id=None, signed=True, keys=keys, arches=arches, task_opts=task_opts) if not task_opts['event']: - task_opts['event'] = _singleValue("SELECT get_event()") - repo_id = _singleValue("SELECT nextval('repo_id_seq')") + task_opts['event'] = get_event() + repo_id = nextval('repo_id_seq') insert = InsertProcessor('repo') insert.set(id=repo_id, create_event=task_opts['event'], tag_id=tag_id, state=state, signed=True) @@ -7381,6 +7381,12 @@ def get_event(): return event_id +def nextval(sequence): + """Get the next value for the given sequence""" + data = {'sequence': sequence} + return _singleValue("SELECT nextval(%(sequence)s)", data, strict=True) + + def parse_json(value, desc=None, errstr=None): if value is None: return value From e5d7990308612e171ae4ef4b4665076f3cf66bf4 Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Mar 30 2017 13:45:59 +0000 Subject: [PATCH 45/77] clean up handling of task_opts in signed_repo_init --- diff --git a/hub/kojihub.py b/hub/kojihub.py index c461568..dbc73a0 100644 --- a/hub/kojihub.py +++ b/hub/kojihub.py @@ -2447,31 +2447,32 @@ def signed_repo_init(tag, keys, task_opts): state = koji.REPO_INIT tinfo = get_tag(tag, strict=True) tag_id = tinfo['id'] + event = task_opts.get('event') arches = set([koji.canonArch(a) for a in task_opts['arch']]) # note: we need to match args from the other preRepoInit callback koji.plugin.run_callbacks('preRepoInit', tag=tinfo, with_src=False, - with_debuginfo=False, event=task_opts['event'], repo_id=None, + with_debuginfo=False, event=event, repo_id=None, signed=True, keys=keys, arches=arches, task_opts=task_opts) - if not task_opts['event']: - task_opts['event'] = get_event() + if not event: + event = get_event() repo_id = nextval('repo_id_seq') insert = InsertProcessor('repo') - insert.set(id=repo_id, create_event=task_opts['event'], tag_id=tag_id, + insert.set(id=repo_id, create_event=event, tag_id=tag_id, state=state, signed=True) insert.execute() repodir = koji.pathinfo.signedrepo(repo_id, tinfo['name']) for arch in arches: koji.ensuredir(os.path.join(repodir, arch)) # handle comps - if task_opts['comps']: + if task_opts.get('comps'): groupsdir = os.path.join(repodir, 'groups') koji.ensuredir(groupsdir) shutil.copyfile(os.path.join(koji.pathinfo.work(), task_opts['comps']), groupsdir + '/comps.xml') # note: we need to match args from the other postRepoInit callback koji.plugin.run_callbacks('postRepoInit', tag=tinfo, with_src=False, - with_debuginfo=False, event=task_opts['event'], repo_id=repo_id) - return repo_id, task_opts['event'] + with_debuginfo=False, event=event, repo_id=repo_id) + return repo_id, event def repo_set_state(repo_id, state, check=True): From a5fbfa392c041223775a5a696976de421b86e71c Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Mar 30 2017 13:45:59 +0000 Subject: [PATCH 46/77] unit test for signed_repo_init() --- diff --git a/tests/test_hub/test_signed_repo.py b/tests/test_hub/test_signed_repo.py new file mode 100644 index 0000000..4d5f477 --- /dev/null +++ b/tests/test_hub/test_signed_repo.py @@ -0,0 +1,56 @@ + +import unittest +import mock + +import koji +import kojihub +from koji.util import dslice_ex + +IP = kojihub.InsertProcessor + + +class TestSignedRepoInit(unittest.TestCase): + + + def getInsert(self, *args, **kwargs): + insert = IP(*args, **kwargs) + insert.execute = mock.MagicMock() + self.inserts.append(insert) + return insert + + + def setUp(self): + self.InsertProcessor = mock.patch('kojihub.InsertProcessor', + side_effect=self.getInsert).start() + self.inserts = [] + + self.get_tag = mock.patch('kojihub.get_tag').start() + self.get_event = mock.patch('kojihub.get_event').start() + self.nextval = mock.patch('kojihub.nextval').start() + self.ensuredir = mock.patch('koji.ensuredir').start() + self.copyfile = mock.patch('shutil.copyfile').start() + + self.get_tag.return_value = {'id': 42, 'name': 'tag'} + self.get_event.return_value = 12345 + self.nextval.return_value = 99 + + + def tearDown(self): + mock.patch.stopall() + + + def test_simple_signed_repo_init(self): + + # simple case + kojihub.signed_repo_init('tag', ['key'], {'arch': ['x86_64']}) + self.InsertProcessor.assert_called_once() + + ip = self.inserts[0] + self.assertEquals(ip.table, 'repo') + data = {'signed': True, 'create_event': 12345, 'tag_id': 42, 'id': 99, + 'state': koji.REPO_STATES['INIT']} + self.assertEquals(ip.data, data) + self.assertEquals(ip.rawdata, {}) + + # no comps option + self.copyfile.assert_not_called() From 1371b188863489bf51e44b54299ef7fb3755f2c8 Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Mar 30 2017 13:45:59 +0000 Subject: [PATCH 47/77] extend unit test for signed_repo_init() --- diff --git a/tests/test_hub/test_signed_repo.py b/tests/test_hub/test_signed_repo.py index 4d5f477..6e2e491 100644 --- a/tests/test_hub/test_signed_repo.py +++ b/tests/test_hub/test_signed_repo.py @@ -54,3 +54,21 @@ class TestSignedRepoInit(unittest.TestCase): # no comps option self.copyfile.assert_not_called() + + + def test_signed_repo_init_with_comps(self): + + # simple case + kojihub.signed_repo_init('tag', ['key'], {'arch': ['x86_64'], + 'comps': 'COMPSFILE'}) + self.InsertProcessor.assert_called_once() + + ip = self.inserts[0] + self.assertEquals(ip.table, 'repo') + data = {'signed': True, 'create_event': 12345, 'tag_id': 42, 'id': 99, + 'state': koji.REPO_STATES['INIT']} + self.assertEquals(ip.data, data) + self.assertEquals(ip.rawdata, {}) + + # no comps option + self.copyfile.assert_called_once() From cea00501a60326d30dca45ac243dc3a8f909e221 Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Mar 30 2017 13:45:59 +0000 Subject: [PATCH 48/77] clean up logic to avoid duplicate code --- diff --git a/builder/kojid b/builder/kojid index 9438829..d34ff22 100755 --- a/builder/kojid +++ b/builder/kojid @@ -4979,16 +4979,13 @@ class NewSignedRepoTask(BaseTaskHandler): for (arch, task_id) in subtasks.iteritems(): data[arch] = results[task_id] self.logger.debug("DEBUG: %r : %r " % (arch, data[arch])) - if task_opts['multilib']: - # we moved the 32-bit results before, do the 64-bit - if arch not in arch32s: - upload, files, keypaths = results[subtasks[arch]] - self.session.host.signedRepoMove( - repo_id, upload, files, arch, keypaths) - else: - upload, files, keypaths = results[subtasks[arch]] - self.session.host.signedRepoMove( - repo_id, upload, files, arch, keypaths) + if task_opts['multilib'] and arch in arch32s: + # already moved above + continue + #else + upload, files, keypaths = results[subtasks[arch]] + self.session.host.signedRepoMove( + repo_id, upload, files, arch, keypaths) self.session.host.repoDone(repo_id, data, expire=False) return 'Signed repository #%s successfully generated' % repo_id From e0826a3145f9bc3e2ab0d690aced9c29e715bdad Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Mar 30 2017 13:45:59 +0000 Subject: [PATCH 49/77] more unit tests --- diff --git a/tests/test_hub/test_signed_repo.py b/tests/test_hub/test_signed_repo.py index 6e2e491..17c72a6 100644 --- a/tests/test_hub/test_signed_repo.py +++ b/tests/test_hub/test_signed_repo.py @@ -1,6 +1,9 @@ import unittest import mock +import os +import shutil +import tempfile import koji import kojihub @@ -72,3 +75,107 @@ class TestSignedRepoInit(unittest.TestCase): # no comps option self.copyfile.assert_called_once() + + +class TestSignedRepo(unittest.TestCase): + + @mock.patch('kojihub.signed_repo_init') + @mock.patch('kojihub.make_task') + def test_SignedRepo(self, make_task, signed_repo_init): + session = kojihub.context.session = mock.MagicMock() + # It seems MagicMock will not automatically handle attributes that + # start with "assert" + session.assertPerm = mock.MagicMock() + signed_repo_init.return_value = ('repo_id', 'event_id') + make_task.return_value = 'task_id' + + exports = kojihub.RootExports() + ret = exports.signedRepo('tag', 'keys') + session.assertPerm.assert_called_once_with('signed-repo') + signed_repo_init.assert_called_once() + make_task.assert_called_once() + self.assertEquals(ret, make_task.return_value) + + +class TestSignedRepoMove(unittest.TestCase): + + def setUp(self): + self.topdir = tempfile.mkdtemp() + self.rinfo = { + 'create_event': 2915, + 'create_ts': 1487256924.72718, + 'creation_time': '2017-02-16 14:55:24.727181', + 'id': 47, + 'state': 1, + 'tag_id': 2, + 'tag_name': 'my-tag'} + self.arch = 'x86_64' + + # set up a fake koji topdir + # koji.pathinfo._topdir = self.topdir + mock.patch('koji.pathinfo._topdir', new=self.topdir).start() + repodir = koji.pathinfo.signedrepo(self.rinfo['id'], self.rinfo['tag_name']) + archdir = "%s/%s" % (repodir, koji.canonArch(self.arch)) + os.makedirs(archdir) + self.uploadpath = 'UNITTEST' + workdir = koji.pathinfo.work() + uploaddir = "%s/%s" % (workdir, self.uploadpath) + os.makedirs(uploaddir) + + # place some test files + self.files = ['foo.drpm', 'repomd.xml'] + self.expected = ['x86_64/drpms/foo.drpm', 'x86_64/repodata/repomd.xml'] + for fn in self.files: + path = os.path.join(uploaddir, fn) + koji.ensuredir(os.path.dirname(path)) + with open(path, 'w') as fo: + fo.write('%s' % fn) + + # also a pkglist file + self.files.append('pkglist') + plist = os.path.join(uploaddir, 'pkglist') + # crap this is terrible -- code needs fixing + nvrs = ['aaa-1.0-2', 'bbb-3.0-5', 'ccc-8.0-13','ddd-21.0-34'] + self.fullpaths = {} # XXX + with open(plist, 'w') as f_pkglist: + for nvr in nvrs: + binfo = koji.parse_NVR(nvr) + rpminfo = binfo.copy() + rpminfo['arch'] = 'x86_64' + builddir = koji.pathinfo.build(binfo) + relpath = koji.pathinfo.rpm(rpminfo) + path = os.path.join(builddir, relpath) + koji.ensuredir(os.path.dirname(path)) + basename = os.path.basename(path) + with open(path, 'w') as fo: + fo.write('%s' % basename) + f_pkglist.write(path) + f_pkglist.write('\n') + self.expected.append('x86_64/%s/%s' % (basename[0], basename)) + self.fullpaths[basename] = path # XXX + + # mocks + self.repo_info = mock.patch('kojihub.repo_info').start() + self.repo_info.return_value = self.rinfo.copy() + + + def tearDown(self): + mock.patch.stopall() + shutil.rmtree(self.topdir) + + + def test_signedRepoMove(self): + exports = kojihub.HostExports() + exports.signedRepoMove(self.rinfo['id'], self.uploadpath, + list(self.files), self.arch, self.fullpaths) + # check result + repodir = self.topdir + '/repos-signed/%(tag_name)s/%(id)s' % self.rinfo + for relpath in self.expected: + path = os.path.join(repodir, relpath) + basename = os.path.basename(path) + if not os.path.exists(path): + raise Exception, "Missing file: %s" % path + data = open(path).read() + data.strip() + self.assertEquals(data, basename) + From 1bfa815b16d9ae4eb5eb9813f36dcb85ed4e162d Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Mar 30 2017 13:45:59 +0000 Subject: [PATCH 50/77] update exception syntax in signed-repo code --- diff --git a/builder/kojid b/builder/kojid index d34ff22..6e5dfcc 100755 --- a/builder/kojid +++ b/builder/kojid @@ -5019,7 +5019,7 @@ class createSignedRepoTask(CreaterepoTask): #arch is the arch of the repo, not the task self.rinfo = self.session.repoInfo(repo_id, strict=True) if self.rinfo['state'] != koji.REPO_INIT: - raise koji.GenericError, "Repo %(id)s not in INIT state (got %(state)s)" % self.rinfo + raise koji.GenericError("Repo %(id)s not in INIT state (got %(state)s)" % self.rinfo) self.repo_id = self.rinfo['id'] self.pathinfo = koji.PathInfo(self.options.topdir) groupdata = os.path.join( diff --git a/hub/kojihub.py b/hub/kojihub.py index dbc73a0..29f7d93 100644 --- a/hub/kojihub.py +++ b/hub/kojihub.py @@ -12334,7 +12334,7 @@ class HostExports(object): repodir = koji.pathinfo.signedrepo(repo_id, rinfo['tag_name']) archdir = "%s/%s" % (repodir, koji.canonArch(arch)) if not os.path.isdir(archdir): - raise koji.GenericError, "Repo arch directory missing: %s" % archdir + raise koji.GenericError("Repo arch directory missing: %s" % archdir) datadir = "%s/repodata" % archdir koji.ensuredir(datadir) for fn in files: @@ -12347,7 +12347,7 @@ class HostExports(object): else: dst = "%s/%s" % (datadir, fn) if not os.path.exists(src): - raise koji.GenericError, "uploaded file missing: %s" % src + raise koji.GenericError("uploaded file missing: %s" % src) if fn.endswith('pkglist'): # hardlink the found rpms into the final repodir # TODO: properly consider split-volume functionality diff --git a/tests/test_hub/test_signed_repo.py b/tests/test_hub/test_signed_repo.py index 17c72a6..4a20ba7 100644 --- a/tests/test_hub/test_signed_repo.py +++ b/tests/test_hub/test_signed_repo.py @@ -174,7 +174,7 @@ class TestSignedRepoMove(unittest.TestCase): path = os.path.join(repodir, relpath) basename = os.path.basename(path) if not os.path.exists(path): - raise Exception, "Missing file: %s" % path + raise Exception("Missing file: %s" % path) data = open(path).read() data.strip() self.assertEquals(data, basename) From cb6a425d7fc492957623afeb999c381321c99703 Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Mar 30 2017 13:45:59 +0000 Subject: [PATCH 51/77] rework signedRepoMove api a bit --- diff --git a/builder/kojid b/builder/kojid index 6e5dfcc..6ca9c87 100755 --- a/builder/kojid +++ b/builder/kojid @@ -30,6 +30,10 @@ import koji.plugin import koji.util import koji.tasks import glob +try: + import json +except ImportError: # pragma: no cover + import simplejson as json import logging import logging.handlers from koji.daemon import incremental_upload, log_output, TaskManager, SCM @@ -4963,9 +4967,9 @@ class NewSignedRepoTask(BaseTaskHandler): for arch in arch32s: # move the 32-bit task output to the final resting place # so the 64-bit arches can use it for multilib - upload, files, keypaths = results[subtasks[arch]] + upload, files, sigmap = results[subtasks[arch]] self.session.host.signedRepoMove( - repo_id, upload, files, arch, keypaths) + repo_id, upload, files, arch, sigmap) for arch in canonArches: # do the other arches if arch not in arch32s: @@ -4983,9 +4987,9 @@ class NewSignedRepoTask(BaseTaskHandler): # already moved above continue #else - upload, files, keypaths = results[subtasks[arch]] + upload, files, sigmap = results[subtasks[arch]] self.session.host.signedRepoMove( - repo_id, upload, files, arch, keypaths) + repo_id, upload, files, arch, sigmap) self.session.host.repoDone(repo_id, data, expire=False) return 'Signed repository #%s successfully generated' % repo_id @@ -5030,8 +5034,9 @@ class createSignedRepoTask(CreaterepoTask): koji.ensuredir(self.repodir) self.outdir = self.repodir # workaround create_local_repo use self.datadir = '%s/repodata' % self.repodir - self.keypaths = {} + self.sigmap = {} if len(opts['delta']) > 0: + # XXX raw path in options for path in opts['delta']: if not os.path.exists(path): raise koji.GenericError( @@ -5040,6 +5045,7 @@ class createSignedRepoTask(CreaterepoTask): self.pkglist = self.make_pkglist(tag, arch, keys, opts) if opts['multilib'] and rpmUtils.arch.isMultiLibArch(arch): self.do_multilib(arch, self.archmap[arch], opts['multilib']) + self.write_kojipkgs() self.logger.debug('package list is %s' % self.pkglist) self.session.uploadWrapper(self.pkglist, self.uploadpath, os.path.basename(self.pkglist)) @@ -5066,7 +5072,7 @@ class createSignedRepoTask(CreaterepoTask): files.append(f) self.session.uploadWrapper('%s/%s' % (ddir, f), self.uploadpath, f) - return [self.uploadpath, files, self.keypaths] + return [self.uploadpath, files, self.sigmap] def do_multilib(self, arch, ml_arch, conf): self.repo_id = self.rinfo['id'] @@ -5076,7 +5082,7 @@ class createSignedRepoTask(CreaterepoTask): ml_true = set() # multilib packages we need to include before depsolve ml_conf = os.path.join(self.pathinfo.work(), conf) - # step 1: figure out which packages are multlib (should already exist) + # step 1: figure out which packages are multilib (should already exist) mlm = multilib.DevelMultilibMethod(ml_conf) fs_missing = set() with open(self.pkglist) as pkglist: @@ -5175,6 +5181,10 @@ enabled=1 raise koji.GenericError('multilib packages missing:\n' + '\n'.join(fs_missing)) + # get rpm ids for ml pkgs + kpkgfile = os.path.join(mldir, 'kojipkgs') + kojipkgs = json.load(open(kpkgfile, 'r')) + # step 5: add dependencies to our package list pkgwriter = open(self.pkglist, 'a') for dep_path in ml_needed: @@ -5191,11 +5201,10 @@ enabled=1 pkgwriter.write(bnplet + '/' + bnp + '\n') self.logger.debug("os.symlink(%r, %r)", dep_path, dst) os.symlink(dep_path, dst) - self.keypaths[bnp] = dep_path + self.sigmap[bnp] = kojipkgs[bnp]['sigkey'] def make_pkglist(self, tag_id, arch, keys, opts): - rpms = [] builddirs = {} for a in self.compat[arch] + ('noarch',): @@ -5228,6 +5237,7 @@ enabled=1 preferred[rpminfo['id']] = rpminfo seen = set() fs_missing = set() + kojipkgs = {} for rpminfo in preferred.values(): if rpminfo['sigkey'] == '': # we're taking an unsigned rpm (--allow-unsigned) @@ -5239,16 +5249,19 @@ enabled=1 seen.add(os.path.basename(pkgpath)) if not os.path.exists(pkgpath): fs_missing.add(pkgpath) + # we'll raise an error below else: bnp = os.path.basename(pkgpath) bnplet = bnp[0].lower() pkglist.write(bnplet + '/' + bnp + '\n') koji.ensuredir(os.path.join(self.repodir, bnplet)) - self.keypaths[bnp] = pkgpath + self.sigmap[rpminfo['id']] = rpminfo['sigkey'] dst = os.path.join(self.repodir, bnplet, bnp) self.logger.debug("os.symlink(%r, %r(", pkgpath, dst) os.symlink(pkgpath, dst) + kojipkgs[bnp] = rpminfo pkglist.close() + self.kojipkgs = kojipkgs if len(fs_missing) > 0: raise koji.GenericError('Packages missing from the filesystem:\n' + '\n'.join(fs_missing)) @@ -5261,6 +5274,13 @@ enabled=1 return pkgfile + def write_kojipkgs(self): + datafile = file(os.path.join(self.repodir, 'kojipkgs'), 'w') + json.dump(self.kojipkgs, datafile, indent=4) + datafile.close() + + + class WaitrepoTask(BaseTaskHandler): Methods = ['waitrepo'] diff --git a/hub/kojihub.py b/hub/kojihub.py index 29f7d93..9f17c8f 100644 --- a/hub/kojihub.py +++ b/hub/kojihub.py @@ -12325,10 +12325,27 @@ class HostExports(object): log_error("Unable to create latest link for repo: %s" % repodir) koji.plugin.run_callbacks('postRepoDone', repo=rinfo, data=data, expire=expire) - def signedRepoMove(self, repo_id, uploadpath, files, arch, fullpaths): + + def signedRepoMove(self, repo_id, uploadpath, files, arch, sigmap): + """ + Move a signed repo into its final location + + + Unlike normal repos (which are moved into place by repoDone), signed + repos have all their content linked (or copied) into place. + + repo_id - the repo to move + uploadpath - where the uploaded files are + files - a list of the uploaded file names + arch - the arch of the repo + sigmap - a dictionary rpm_id -> sig + + The rpms from sigmap should match the contents of the uploaded pkglist + file. + + In sigmap, use sig=None to use the primary copy of the rpm instead of a + signed copy. """ - Very similar to repoDone, except only the uploads are completed. - fullpaths is a dict like so: rpm file name -> sig""" workdir = koji.pathinfo.work() rinfo = repo_info(repo_id, strict=True) repodir = koji.pathinfo.signedrepo(repo_id, rinfo['tag_name']) @@ -12337,6 +12354,8 @@ class HostExports(object): raise koji.GenericError("Repo arch directory missing: %s" % archdir) datadir = "%s/repodata" % archdir koji.ensuredir(datadir) + + pkglist = set() for fn in files: src = "%s/%s/%s" % (workdir, uploadpath, fn) if fn.endswith('.drpm'): @@ -12349,30 +12368,60 @@ class HostExports(object): if not os.path.exists(src): raise koji.GenericError("uploaded file missing: %s" % src) if fn.endswith('pkglist'): - # hardlink the found rpms into the final repodir - # TODO: properly consider split-volume functionality with open(src) as pkgfile: for pkg in pkgfile: pkg = os.path.basename(pkg.strip()) - rpmpath = fullpaths[pkg] - bnp = os.path.basename(rpmpath) - bnplet = bnp[0].lower() - koji.ensuredir(os.path.join(archdir, bnplet)) - l_dst = os.path.join(archdir, bnplet, bnp) - if os.path.exists(l_dst): - logger.warning("Path exists: %s", l_dst) - continue - logger.debug("os.link(%r, %r)", rpmpath, l_dst) - try: - os.link(rpmpath, l_dst) - except OSError, ose: - if ose.errno == 18: - shutil.copy2( - rpmpath, os.path.join(archdir, bnplet, bnp)) - else: - raise + pkglist.add(pkg) safer_move(src, dst) + # get rpms + build_dirs = {} + rpmdata = {} + for rpm_id in sigmap: + sigkey = sigmap[rpm_id] + rpminfo = get_rpm(rpm_id, strict=True) + relpath = koji.pathinfo.signed(rpminfo, sigkey) + rpminfo['_relpath'] = relpath + if rpminfo['build_id'] in build_dirs: + builddir = build_dirs[rpminfo['build_id']] + else: + binfo = get_build(rpminfo['build_id']) + builddir = koji.pathinfo.build(binfo) + build_dirs[rpminfo['build_id']] = builddir + rpminfo['_fullpath'] = os.path.join(builddir, relpath) + basename = os.path.basename(relpath) + rpmdata[basename] = rpminfo + + # sanity check + for fn in rpmdata: + if fn not in pkglist: + raise koji.GenericError("No signature data for: %s" % fn) + for fn in pkglist: + if fn not in rpmdata: + raise koji.GenericError("RPM missing from pkglist: %s" % fn) + + for fn in rpmdata: + # hardlink or copy the rpms into the final repodir + # TODO: properly consider split-volume functionality + rpminfo = rpmdata[fn] + rpmpath = rpminfo['_fullpath'] + bnp = fn + bnplet = bnp[0].lower() + koji.ensuredir(os.path.join(archdir, bnplet)) + l_dst = os.path.join(archdir, bnplet, bnp) + if os.path.exists(l_dst): + raise koji.GenericError("File already in repo: %s", l_dst) + logger.debug("os.link(%r, %r)", rpmpath, l_dst) + try: + os.link(rpmpath, l_dst) + except OSError, ose: + if ose.errno == 18: + shutil.copy2( + rpmpath, os.path.join(archdir, bnplet, bnp)) + else: + raise + + def isEnabled(self): host = Host() host.verify() diff --git a/tests/test_hub/test_signed_repo.py b/tests/test_hub/test_signed_repo.py index 4a20ba7..0d4295c 100644 --- a/tests/test_hub/test_signed_repo.py +++ b/tests/test_hub/test_signed_repo.py @@ -131,19 +131,21 @@ class TestSignedRepoMove(unittest.TestCase): with open(path, 'w') as fo: fo.write('%s' % fn) - # also a pkglist file + # generate pkglist file and sigmap self.files.append('pkglist') plist = os.path.join(uploaddir, 'pkglist') - # crap this is terrible -- code needs fixing nvrs = ['aaa-1.0-2', 'bbb-3.0-5', 'ccc-8.0-13','ddd-21.0-34'] - self.fullpaths = {} # XXX + self.sigmap = {} + self.rpms = {} + self.builds ={} + self.key = '4c8da725' with open(plist, 'w') as f_pkglist: for nvr in nvrs: binfo = koji.parse_NVR(nvr) rpminfo = binfo.copy() rpminfo['arch'] = 'x86_64' builddir = koji.pathinfo.build(binfo) - relpath = koji.pathinfo.rpm(rpminfo) + relpath = koji.pathinfo.signed(rpminfo, self.key) path = os.path.join(builddir, relpath) koji.ensuredir(os.path.dirname(path)) basename = os.path.basename(path) @@ -152,11 +154,22 @@ class TestSignedRepoMove(unittest.TestCase): f_pkglist.write(path) f_pkglist.write('\n') self.expected.append('x86_64/%s/%s' % (basename[0], basename)) - self.fullpaths[basename] = path # XXX + build_id = len(self.builds) + 10000 + rpm_id = len(self.rpms) + 20000 + binfo['id'] = build_id + rpminfo['build_id'] = build_id + rpminfo['id'] = rpm_id + self.builds[build_id] = binfo + self.rpms[rpm_id] = rpminfo + self.sigmap[rpm_id] = self.key # mocks self.repo_info = mock.patch('kojihub.repo_info').start() self.repo_info.return_value = self.rinfo.copy() + self.get_rpm = mock.patch('kojihub.get_rpm').start() + self.get_build = mock.patch('kojihub.get_build').start() + self.get_rpm.side_effect = self.our_get_rpm + self.get_build.side_effect = self.our_get_build def tearDown(self): @@ -164,10 +177,18 @@ class TestSignedRepoMove(unittest.TestCase): shutil.rmtree(self.topdir) + def our_get_rpm(self, rpminfo, strict=False, multi=False): + return self.rpms[rpminfo] + + + def our_get_build(self, buildInfo, strict=False): + return self.builds[buildInfo] + + def test_signedRepoMove(self): exports = kojihub.HostExports() exports.signedRepoMove(self.rinfo['id'], self.uploadpath, - list(self.files), self.arch, self.fullpaths) + list(self.files), self.arch, self.sigmap) # check result repodir = self.topdir + '/repos-signed/%(tag_name)s/%(id)s' % self.rinfo for relpath in self.expected: From 21fda8805e405121d8eb12124441f0475ad0674c Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Mar 30 2017 13:45:59 +0000 Subject: [PATCH 52/77] no integer keys in xmlrpc --- diff --git a/builder/kojid b/builder/kojid index 6ca9c87..ec06dbc 100755 --- a/builder/kojid +++ b/builder/kojid @@ -5072,7 +5072,7 @@ class createSignedRepoTask(CreaterepoTask): files.append(f) self.session.uploadWrapper('%s/%s' % (ddir, f), self.uploadpath, f) - return [self.uploadpath, files, self.sigmap] + return [self.uploadpath, files, self.sigmap.items()] def do_multilib(self, arch, ml_arch, conf): self.repo_id = self.rinfo['id'] diff --git a/hub/kojihub.py b/hub/kojihub.py index 9f17c8f..c5452b6 100644 --- a/hub/kojihub.py +++ b/hub/kojihub.py @@ -12338,7 +12338,7 @@ class HostExports(object): uploadpath - where the uploaded files are files - a list of the uploaded file names arch - the arch of the repo - sigmap - a dictionary rpm_id -> sig + sigmap - a list of [rpm_id, sig] pairs The rpms from sigmap should match the contents of the uploaded pkglist file. @@ -12377,8 +12377,7 @@ class HostExports(object): # get rpms build_dirs = {} rpmdata = {} - for rpm_id in sigmap: - sigkey = sigmap[rpm_id] + for rpm_id, sigkey in sigmap: rpminfo = get_rpm(rpm_id, strict=True) relpath = koji.pathinfo.signed(rpminfo, sigkey) rpminfo['_relpath'] = relpath diff --git a/tests/test_hub/test_signed_repo.py b/tests/test_hub/test_signed_repo.py index 0d4295c..cea8a4a 100644 --- a/tests/test_hub/test_signed_repo.py +++ b/tests/test_hub/test_signed_repo.py @@ -135,7 +135,7 @@ class TestSignedRepoMove(unittest.TestCase): self.files.append('pkglist') plist = os.path.join(uploaddir, 'pkglist') nvrs = ['aaa-1.0-2', 'bbb-3.0-5', 'ccc-8.0-13','ddd-21.0-34'] - self.sigmap = {} + self.sigmap = [] self.rpms = {} self.builds ={} self.key = '4c8da725' @@ -161,7 +161,7 @@ class TestSignedRepoMove(unittest.TestCase): rpminfo['id'] = rpm_id self.builds[build_id] = binfo self.rpms[rpm_id] = rpminfo - self.sigmap[rpm_id] = self.key + self.sigmap.append([rpm_id, self.key]) # mocks self.repo_info = mock.patch('kojihub.repo_info').start() From eb330165cf8e9c981df7ee7e7ff4762eb4a7ba60 Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Mar 30 2017 13:45:59 +0000 Subject: [PATCH 53/77] handle unsigned rpms in signedRepoMove --- diff --git a/hub/kojihub.py b/hub/kojihub.py index c5452b6..46fed71 100644 --- a/hub/kojihub.py +++ b/hub/kojihub.py @@ -12379,7 +12379,10 @@ class HostExports(object): rpmdata = {} for rpm_id, sigkey in sigmap: rpminfo = get_rpm(rpm_id, strict=True) - relpath = koji.pathinfo.signed(rpminfo, sigkey) + if sigkey is None or sigkey == '': + relpath = koji.pathinfo.rpm(rpminfo) + else: + relpath = koji.pathinfo.signed(rpminfo, sigkey) rpminfo['_relpath'] = relpath if rpminfo['build_id'] in build_dirs: builddir = build_dirs[rpminfo['build_id']] From 09ed16e532e8413212c595977a4d4b6bdbbdf03a Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Mar 30 2017 13:45:59 +0000 Subject: [PATCH 54/77] upload kojipkgs data --- diff --git a/builder/kojid b/builder/kojid index ec06dbc..4bef0f6 100755 --- a/builder/kojid +++ b/builder/kojid @@ -5061,7 +5061,7 @@ class createSignedRepoTask(CreaterepoTask): fo = file(os.path.join(self.datadir, "EMPTY_REPO"), 'w') fo.write("This repo is empty because its tag has no content for this arch\n") fo.close() - files = ['pkglist'] + files = ['pkglist', 'kojipkgs'] for f in os.listdir(self.datadir): files.append(f) self.session.uploadWrapper('%s/%s' % (self.datadir, f), @@ -5201,7 +5201,8 @@ enabled=1 pkgwriter.write(bnplet + '/' + bnp + '\n') self.logger.debug("os.symlink(%r, %r)", dep_path, dst) os.symlink(dep_path, dst) - self.sigmap[bnp] = kojipkgs[bnp]['sigkey'] + rpminfo = kojipkgs[bnp] + self.sigmap[rpminfo['id']] = rpminfo['sigkey'] def make_pkglist(self, tag_id, arch, keys, opts): @@ -5275,9 +5276,14 @@ enabled=1 def write_kojipkgs(self): - datafile = file(os.path.join(self.repodir, 'kojipkgs'), 'w') - json.dump(self.kojipkgs, datafile, indent=4) - datafile.close() + filename = os.path.join(self.repodir, 'kojipkgs') + datafile = file(filename, 'w') + try: + json.dump(self.kojipkgs, datafile, indent=4) + finally: + datafile.close() + # and upload too + self.session.uploadWrapper(filename, self.uploadpath, 'kojipkgs') diff --git a/hub/kojihub.py b/hub/kojihub.py index 46fed71..b35ec33 100644 --- a/hub/kojihub.py +++ b/hub/kojihub.py @@ -12361,7 +12361,7 @@ class HostExports(object): if fn.endswith('.drpm'): koji.ensuredir(os.path.join(archdir, 'drpms')) dst = "%s/drpms/%s" % (archdir, fn) - elif fn.endswith('pkglist'): + elif fn.endswith('pkglist') or fn.endswith('kojipkgs'): dst = '%s/%s' % (archdir, fn) else: dst = "%s/%s" % (datadir, fn) From c2b48823231b4010f0cf8d1ff3c3e94fb0adb1ef Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Mar 30 2017 13:45:59 +0000 Subject: [PATCH 55/77] require python2-multilib on builder --- diff --git a/koji.spec b/koji.spec index 7c97f29..f70cd06 100644 --- a/koji.spec +++ b/koji.spec @@ -98,6 +98,7 @@ Requires: %{name} = %{version}-%{release} Requires: mock >= 0.9.14 Requires(pre): /usr/sbin/useradd Requires: squashfs-tools +Requires: python2-multilib %if %{use_systemd} Requires(post): systemd Requires(preun): systemd From 1557292fe02a3aadfadeaa249697dbe111b1083c Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Mar 30 2017 13:45:59 +0000 Subject: [PATCH 56/77] add builder json requirement for rhel5 --- diff --git a/koji.spec b/koji.spec index f70cd06..250c1a2 100644 --- a/koji.spec +++ b/koji.spec @@ -117,6 +117,7 @@ Requires: python-cheetah Requires: createrepo >= 0.4.11-2 Requires: python-hashlib Requires: python-createrepo +Requires: python-simplejson %endif %if 0%{?fedora} >= 9 Requires: createrepo >= 0.9.2 From d301fa964b085e281b952efb36b15ec3accb6b0f Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Mar 30 2017 13:45:59 +0000 Subject: [PATCH 57/77] first stab at fixing delta rpm behavior for signed repos --- diff --git a/builder/kojid b/builder/kojid index 4bef0f6..5e3dea9 100755 --- a/builder/kojid +++ b/builder/kojid @@ -4855,7 +4855,7 @@ class CreaterepoTask(BaseTaskHandler): self.session.uploadWrapper('%s/%s' % (self.datadir, f), uploadpath, f) return [uploadpath, files] - def create_local_repo(self, rinfo, arch, pkglist, groupdata, oldrepo, drpms=False): + def create_local_repo(self, rinfo, arch, pkglist, groupdata, oldrepo, oldpkgs=None): koji.ensuredir(self.outdir) if self.options.use_createrepo_c: cmd = ['/usr/bin/createrepo_c'] @@ -4867,9 +4867,7 @@ class CreaterepoTask(BaseTaskHandler): if os.path.isfile(groupdata): cmd.extend(['-g', groupdata]) #attempt to recycle repodata from last repo - if pkglist and oldrepo and self.options.createrepo_update and not drpms: - # signed repos overload the use of "oldrepo", so the conditional - # explicitly make sure this does not get executed with that on + if pkglist and oldrepo and self.options.createrepo_update: oldpath = self.pathinfo.repo(oldrepo['id'], rinfo['tag_name']) olddatadir = '%s/%s/repodata' % (oldpath, arch) if not os.path.isdir(olddatadir): @@ -4884,11 +4882,11 @@ class CreaterepoTask(BaseTaskHandler): cmd.append('--update') if self.options.createrepo_skip_stat: cmd.append('--skip-stat') - if drpms: + if oldpkgs is not None: # generate delta-rpms cmd.append('--deltas') - for repo in oldrepo: - cmd.extend(['--oldpackagedirs', repo]) + for op_dir in oldpkgs: + cmd.extend(['--oldpackagedirs', op_dir]) # note: we can't easily use a cachedir because we do not have write # permission. The good news is that with --update we won't need to # be scanning many rpms. @@ -5035,12 +5033,18 @@ class createSignedRepoTask(CreaterepoTask): self.outdir = self.repodir # workaround create_local_repo use self.datadir = '%s/repodata' % self.repodir self.sigmap = {} - if len(opts['delta']) > 0: - # XXX raw path in options - for path in opts['delta']: + oldpkgs = [] + if opts.get('delta'): + # should be a list of repo ids to delta against + for repo_id in opts['delta']: + oldrepo = self.session.repoInfo(repo_id, strict=True) + if not oldrepo['signed']: + raise koji.GenericError("Base repo for deltas must be signed") + # regular repos don't actually have rpms, just pkglist + path = koji.pathinfo.signedrepo(repo_id, oldrepo['tag_name']) if not os.path.exists(path): - raise koji.GenericError( - 'drpm path %s does not exist!' % path) + raise koji.GenericError('Base drpm repo missing: %s' % path) + oldpkgs.append(path) self.uploadpath = self.getUploadDir() self.pkglist = self.make_pkglist(tag, arch, keys, opts) if opts['multilib'] and rpmUtils.arch.isMultiLibArch(arch): @@ -5051,12 +5055,7 @@ class createSignedRepoTask(CreaterepoTask): os.path.basename(self.pkglist)) if os.path.getsize(self.pkglist) == 0: self.pkglist = None - if len(opts['delta']) > 0: - do_drpms = True - else: - do_drpms = False - self.create_local_repo(self.rinfo, arch, self.pkglist, groupdata, - opts['delta'], drpms=do_drpms) + self.create_local_repo(self.rinfo, arch, self.pkglist, groupdata, None, oldpkgs=oldpkgs) if self.pkglist is None: fo = file(os.path.join(self.datadir, "EMPTY_REPO"), 'w') fo.write("This repo is empty because its tag has no content for this arch\n") diff --git a/cli/koji b/cli/koji index a604ae1..549b264 100755 --- a/cli/koji +++ b/cli/koji @@ -7102,12 +7102,11 @@ def handle_signed_repo(options, session, args): "architectures associated with the given tag. This option may " + "be specified multiple times.")) parser.add_option('--comps', help='Include a comps file in the repodata') - parser.add_option('--delta-rpms', metavar='PATH',default=[], + parser.add_option('--delta-rpms', metavar='REPO',default=[], action='append', - help=_('Create delta-rpms. PATH points to (older) rpms to generate ' - 'against. May be specified multiple times. These have to be ' - 'reachable by the builder too, so the path needs to reach shared ' - 'storage.')) + help=_('Create delta-rpms. REPO can be the id of another signed repo ' + 'or the name of a tag that has a signed repo. May be specified ' + 'multiple times.')) parser.add_option('--event', type='int', help=_('create a signed repository based on a Brew event')) parser.add_option('--non-latest', dest='latest', default=True, From 528cef065ff3dc469cd8ccde622496fe4abc0f75 Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Mar 30 2017 13:45:59 +0000 Subject: [PATCH 58/77] adjust cli handling of --delta-rpms arg --- diff --git a/cli/koji b/cli/koji index 549b264..446e45d 100755 --- a/cli/koji +++ b/cli/koji @@ -7134,12 +7134,21 @@ def handle_signed_repo(options, session, args): print task_opts.comps = os.path.join(stuffdir, os.path.basename(task_opts.comps)) + old_repos = [] if len(task_opts.delta_rpms) > 0: - for path in task_opts.delta_rpms: - if not os.path.exists(path): - print(_("Warning: %s is not reachable locally. If this\n" - " host does not have access to Koji's shared storage\n" - " this can be ignored.") % path) + for repo in task_opts.delta_rpms: + if repo.isdigit(): + rinfo = session.repoInfo(int(repo), strict=True) + else: + # get signed repo for tag + rinfo = session.getRepo(repo, signed=True) + if not rinfo: + # maybe there is an expired one + rinfo = session.getRepo(repo, + state=koji.REPO_STATES['EXPIRED'], signed=True) + if not rinfo: + parser.errpr(_("Can't find repo for tag: %s") % repo) + old_repos.append(rinfo['id']) tag = args[0] keys = args[1:] taginfo = session.getTag(tag) @@ -7175,7 +7184,7 @@ def handle_signed_repo(options, session, args): opts = { 'arch': task_opts.arch, 'comps': task_opts.comps, - 'delta': task_opts.delta_rpms, + 'delta': old_repos, 'event': task_opts.event, 'inherit': not task_opts.noinherit, 'latest': task_opts.latest, From 3d551dae5e4d4dbd38a3aa0ac67ff4c7f2ef181d Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Mar 30 2017 13:45:59 +0000 Subject: [PATCH 59/77] command help adjustments --- diff --git a/cli/koji b/cli/koji index 446e45d..f50a0a1 100755 --- a/cli/koji +++ b/cli/koji @@ -7104,15 +7104,16 @@ def handle_signed_repo(options, session, args): parser.add_option('--comps', help='Include a comps file in the repodata') parser.add_option('--delta-rpms', metavar='REPO',default=[], action='append', - help=_('Create delta-rpms. REPO can be the id of another signed repo ' + help=_('Create delta rpms. REPO can be the id of another signed repo ' 'or the name of a tag that has a signed repo. May be specified ' 'multiple times.')) parser.add_option('--event', type='int', help=_('create a signed repository based on a Brew event')) parser.add_option('--non-latest', dest='latest', default=True, action='store_false', help='Include older builds, not just the latest') - parser.add_option('--multilib', default=None, - help=_('Include multilib packages in the repository using a config')) + parser.add_option('--multilib', default=None, metavar="CONFIG", + help=_('Include multilib packages in the repository using the given ' + 'config file')) parser.add_option("--noinherit", action='store_true', default=False, help=_('Do not consider tag inheritance')) parser.add_option("--nowait", action='store_true', default=False, From c86b5c3ac0d1e6499f846ba5febe09c7fa834b3e Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Mar 30 2017 13:47:20 +0000 Subject: [PATCH 60/77] first stab at renaming signed repos to dist repos sed -i -e 's/signed\(.\?[Rr]epo\)/dist\1/g' sed -i -e 's/Signed\(.\?[Rr]epo\)/Dist\1/g' --- diff --git a/builder/kojid b/builder/kojid index 5e3dea9..74a0d55 100755 --- a/builder/kojid +++ b/builder/kojid @@ -4933,13 +4933,13 @@ class CreaterepoTask(BaseTaskHandler): % parseStatus(status, ' '.join(cmd))) -class NewSignedRepoTask(BaseTaskHandler): - Methods = ['signedRepo'] +class NewDistRepoTask(BaseTaskHandler): + Methods = ['distRepo'] _taskWeight = 0.1 def handler(self, tag, repo_id, keys, task_opts): tinfo = self.session.getTag(tag, strict=True, event=task_opts['event']) - path = koji.pathinfo.signedrepo(repo_id, tinfo['name']) + path = koji.pathinfo.distrepo(repo_id, tinfo['name']) if len(task_opts['arch']) == 0: task_opts['arch'] = tinfo['arches'].split() if len(task_opts['arch']) == 0: @@ -4958,7 +4958,7 @@ class NewSignedRepoTask(BaseTaskHandler): # get a task ID and wait for them to complete arglist = [tag, repo_id, arch, keys, task_opts] subtasks[arch] = self.session.host.subtask( - method='createsignedrepo', arglist=arglist, label=arch, + method='createdistrepo', arglist=arglist, label=arch, parent=self.id, arch='noarch') if len(subtasks) > 0 and task_opts['multilib']: results = self.wait(subtasks.values(), all=True, failany=True) @@ -4966,14 +4966,14 @@ class NewSignedRepoTask(BaseTaskHandler): # move the 32-bit task output to the final resting place # so the 64-bit arches can use it for multilib upload, files, sigmap = results[subtasks[arch]] - self.session.host.signedRepoMove( + self.session.host.distRepoMove( repo_id, upload, files, arch, sigmap) for arch in canonArches: # do the other arches if arch not in arch32s: arglist = [tag, repo_id, arch, keys, task_opts] subtasks[arch] = self.session.host.subtask( - method='createsignedrepo', arglist=arglist, label=arch, + method='createdistrepo', arglist=arglist, label=arch, parent=self.id, arch='noarch') # wait for 64-bit subtasks to finish data = {} @@ -4986,14 +4986,14 @@ class NewSignedRepoTask(BaseTaskHandler): continue #else upload, files, sigmap = results[subtasks[arch]] - self.session.host.signedRepoMove( + self.session.host.distRepoMove( repo_id, upload, files, arch, sigmap) self.session.host.repoDone(repo_id, data, expire=False) - return 'Signed repository #%s successfully generated' % repo_id + return 'Dist repository #%s successfully generated' % repo_id -class createSignedRepoTask(CreaterepoTask): - Methods = ['createsignedrepo'] +class createDistRepoTask(CreaterepoTask): + Methods = ['createdistrepo'] _taskWeight = 1.5 archmap = {'s390x': 's390', 'ppc64': 'ppc', 'x86_64': 'i686'} @@ -5025,7 +5025,7 @@ class createSignedRepoTask(CreaterepoTask): self.repo_id = self.rinfo['id'] self.pathinfo = koji.PathInfo(self.options.topdir) groupdata = os.path.join( - self.pathinfo.signedrepo(repo_id, self.rinfo['tag_name']), + self.pathinfo.distrepo(repo_id, self.rinfo['tag_name']), 'groups', 'comps.xml') #set up our output dir self.repodir = '%s/repo' % self.workdir @@ -5041,7 +5041,7 @@ class createSignedRepoTask(CreaterepoTask): if not oldrepo['signed']: raise koji.GenericError("Base repo for deltas must be signed") # regular repos don't actually have rpms, just pkglist - path = koji.pathinfo.signedrepo(repo_id, oldrepo['tag_name']) + path = koji.pathinfo.distrepo(repo_id, oldrepo['tag_name']) if not os.path.exists(path): raise koji.GenericError('Base drpm repo missing: %s' % path) oldpkgs.append(path) @@ -5076,7 +5076,7 @@ class createSignedRepoTask(CreaterepoTask): def do_multilib(self, arch, ml_arch, conf): self.repo_id = self.rinfo['id'] pathinfo = koji.PathInfo(self.options.topdir) - repodir = pathinfo.signedrepo(self.rinfo['id'], self.rinfo['tag_name']) + repodir = pathinfo.distrepo(self.rinfo['id'], self.rinfo['tag_name']) mldir = os.path.join(repodir, koji.canonArch(ml_arch)) ml_true = set() # multilib packages we need to include before depsolve ml_conf = os.path.join(self.pathinfo.work(), conf) diff --git a/cli/koji b/cli/koji index f50a0a1..3506884 100755 --- a/cli/koji +++ b/cli/koji @@ -7090,9 +7090,9 @@ def handle_regen_repo(options, session, args): session.logout() return watch_tasks(session, [task_id], quiet=options.quiet) -def handle_signed_repo(options, session, args): +def handle_dist_repo(options, session, args): """create a yum repo of GPG signed RPMs""" - usage = _("usage: %prog signed-repo [options] tag keyID [keyID...]") + usage = _("usage: %prog dist-repo [options] tag keyID [keyID...]") usage += _("\n(Specify the --help option for a list of other options)") parser = OptionParser(usage=usage) parser.add_option('--allow-unsigned', action='store_true', default=False, @@ -7104,11 +7104,11 @@ def handle_signed_repo(options, session, args): parser.add_option('--comps', help='Include a comps file in the repodata') parser.add_option('--delta-rpms', metavar='REPO',default=[], action='append', - help=_('Create delta rpms. REPO can be the id of another signed repo ' - 'or the name of a tag that has a signed repo. May be specified ' + help=_('Create delta rpms. REPO can be the id of another dist repo ' + 'or the name of a tag that has a dist repo. May be specified ' 'multiple times.')) parser.add_option('--event', type='int', - help=_('create a signed repository based on a Brew event')) + help=_('create a dist repository based on a Brew event')) parser.add_option('--non-latest', dest='latest', default=True, action='store_false', help='Include older builds, not just the latest') parser.add_option('--multilib', default=None, metavar="CONFIG", @@ -7141,7 +7141,7 @@ def handle_signed_repo(options, session, args): if repo.isdigit(): rinfo = session.repoInfo(int(repo), strict=True) else: - # get signed repo for tag + # get dist repo for tag rinfo = session.getRepo(repo, signed=True) if not rinfo: # maybe there is an expired one @@ -7193,8 +7193,8 @@ def handle_signed_repo(options, session, args): 'skip': task_opts.skip_unsigned, 'unsigned': task_opts.allow_unsigned } - task_id = session.signedRepo(tag, keys, **opts) - print("Creating signed repo for tag " + tag) + task_id = session.distRepo(tag, keys, **opts) + print("Creating dist repo for tag " + tag) if _running_in_bg() or task_opts.nowait: return else: diff --git a/docs/schema-update-signed-repos.sql b/docs/schema-update-signed-repos.sql index e67abb4..fa38ef1 100644 --- a/docs/schema-update-signed-repos.sql +++ b/docs/schema-update-signed-repos.sql @@ -1,4 +1,4 @@ -# schema updates for signed repo feature +# schema updates for dist repo feature # to be merged into schema upgrade script for next release INSERT INTO permissions (name) VALUES ('image'); diff --git a/hub/kojihub.py b/hub/kojihub.py index b35ec33..d35ce54 100644 --- a/hub/kojihub.py +++ b/hub/kojihub.py @@ -2442,7 +2442,7 @@ def _write_maven_repo_metadata(destdir, artifacts): mdfile.close() _generate_maven_metadata(destdir) -def signed_repo_init(tag, keys, task_opts): +def dist_repo_init(tag, keys, task_opts): """Create a new repo entry in the INIT state, return full repo data""" state = koji.REPO_INIT tinfo = get_tag(tag, strict=True) @@ -2460,7 +2460,7 @@ def signed_repo_init(tag, keys, task_opts): insert.set(id=repo_id, create_event=event, tag_id=tag_id, state=state, signed=True) insert.execute() - repodir = koji.pathinfo.signedrepo(repo_id, tinfo['name']) + repodir = koji.pathinfo.distrepo(repo_id, tinfo['name']) for arch in arches: koji.ensuredir(os.path.join(repodir, arch)) # handle comps @@ -10140,12 +10140,12 @@ class RootExports(object): repoInfo = staticmethod(repo_info) getActiveRepos = staticmethod(get_active_repos) - def signedRepo(self, tag, keys, **task_opts): - """Create a signed-repo task. returns task id""" - context.session.assertPerm('signed-repo') - repo_id, event_id = signed_repo_init(tag, keys, task_opts) + def distRepo(self, tag, keys, **task_opts): + """Create a dist-repo task. returns task id""" + context.session.assertPerm('dist-repo') + repo_id, event_id = dist_repo_init(tag, keys, task_opts) task_opts['event'] = event_id - return make_task('signedRepo', [tag, repo_id, keys, task_opts], priority=15, channel='createrepo') + return make_task('distRepo', [tag, repo_id, keys, task_opts], priority=15, channel='createrepo') def newRepo(self, tag, event=None, src=False, debuginfo=False): """Create a newRepo task. returns task id""" @@ -12277,7 +12277,7 @@ class HostExports(object): data: a dictionary of the form { arch: (uploadpath, files), ...} expire(optional): if set to true, mark the repo expired immediately* - If this is a signed repo, also hardlink signed rpms in the final + If this is a dist repo, also hardlink signed rpms in the final directory. * This is used when a repo from an older event is generated @@ -12326,9 +12326,9 @@ class HostExports(object): koji.plugin.run_callbacks('postRepoDone', repo=rinfo, data=data, expire=expire) - def signedRepoMove(self, repo_id, uploadpath, files, arch, sigmap): + def distRepoMove(self, repo_id, uploadpath, files, arch, sigmap): """ - Move a signed repo into its final location + Move a dist repo into its final location Unlike normal repos (which are moved into place by repoDone), signed @@ -12348,7 +12348,7 @@ class HostExports(object): """ workdir = koji.pathinfo.work() rinfo = repo_info(repo_id, strict=True) - repodir = koji.pathinfo.signedrepo(repo_id, rinfo['tag_name']) + repodir = koji.pathinfo.distrepo(repo_id, rinfo['tag_name']) archdir = "%s/%s" % (repodir, koji.canonArch(arch)) if not os.path.isdir(archdir): raise koji.GenericError("Repo arch directory missing: %s" % archdir) diff --git a/koji.next.md b/koji.next.md index 5f30e42..255359e 100644 --- a/koji.next.md +++ b/koji.next.md @@ -64,7 +64,7 @@ Warning to the reader: - refactor uploads - more flexible gc - introduce an ORM to do away with raw SQL queries. -- know how to manage signed repositories of RPMs +- know how to manage dist repositories of RPMs - know how to build installation media - more granular access control/groups - things like Read, Execute, Execute scratch, Delete, Tag, so we can delegate diff --git a/koji/__init__.py b/koji/__init__.py index 4ac9a68..2f06b75 100644 --- a/koji/__init__.py +++ b/koji/__init__.py @@ -1815,8 +1815,8 @@ class PathInfo(object): """Return the directory where a repo belongs""" return self.topdir + ("/repos/%(tag_str)s/%(repo_id)s" % locals()) - def signedrepo(self, repo_id, tag): - """Return the directory with a signed repo lives""" + def distrepo(self, repo_id, tag): + """Return the directory with a dist repo lives""" return os.path.join(self.topdir, 'repos-signed', tag, str(repo_id)) def repocache(self, tag_str): @@ -2792,7 +2792,7 @@ def _taskLabel(taskInfo): if 'request' in taskInfo: build = taskInfo['request'][1] extra = buildLabel(build) - elif method in ('newRepo', 'signedRepo'): + elif method in ('newRepo', 'distRepo'): if 'request' in taskInfo: extra = str(taskInfo['request'][0]) elif method in ('tagBuild', 'tagNotification'): @@ -2807,7 +2807,7 @@ def _taskLabel(taskInfo): if 'request' in taskInfo: arch = taskInfo['request'][1] extra = arch - elif method == 'createsignedrepo': + elif method == 'createdistrepo': if 'request' in taskInfo: repo_id = taskInfo['request'][1] arch = taskInfo['request'][2] diff --git a/tests/test_cli/data/list-commands.txt b/tests/test_cli/data/list-commands.txt index 8c5a54f..5a21320 100644 --- a/tests/test_cli/data/list-commands.txt +++ b/tests/test_cli/data/list-commands.txt @@ -119,7 +119,7 @@ miscellaneous commands: import-comps Import group/package information from a comps file moshimoshi Introduce yourself save-failed-tree Create tarball with whole buildtree - signed-repo create a yum repo of GPG signed RPMs + dist-repo create a yum repo of GPG signed RPMs monitor commands: wait-repo Wait for a repo to be regenerated diff --git a/tests/test_hub/test_signed_repo.py b/tests/test_hub/test_signed_repo.py index cea8a4a..e62306f 100644 --- a/tests/test_hub/test_signed_repo.py +++ b/tests/test_hub/test_signed_repo.py @@ -12,7 +12,7 @@ from koji.util import dslice_ex IP = kojihub.InsertProcessor -class TestSignedRepoInit(unittest.TestCase): +class TestDistRepoInit(unittest.TestCase): def getInsert(self, *args, **kwargs): @@ -42,10 +42,10 @@ class TestSignedRepoInit(unittest.TestCase): mock.patch.stopall() - def test_simple_signed_repo_init(self): + def test_simple_dist_repo_init(self): # simple case - kojihub.signed_repo_init('tag', ['key'], {'arch': ['x86_64']}) + kojihub.dist_repo_init('tag', ['key'], {'arch': ['x86_64']}) self.InsertProcessor.assert_called_once() ip = self.inserts[0] @@ -59,10 +59,10 @@ class TestSignedRepoInit(unittest.TestCase): self.copyfile.assert_not_called() - def test_signed_repo_init_with_comps(self): + def test_dist_repo_init_with_comps(self): # simple case - kojihub.signed_repo_init('tag', ['key'], {'arch': ['x86_64'], + kojihub.dist_repo_init('tag', ['key'], {'arch': ['x86_64'], 'comps': 'COMPSFILE'}) self.InsertProcessor.assert_called_once() @@ -77,27 +77,27 @@ class TestSignedRepoInit(unittest.TestCase): self.copyfile.assert_called_once() -class TestSignedRepo(unittest.TestCase): +class TestDistRepo(unittest.TestCase): - @mock.patch('kojihub.signed_repo_init') + @mock.patch('kojihub.dist_repo_init') @mock.patch('kojihub.make_task') - def test_SignedRepo(self, make_task, signed_repo_init): + def test_DistRepo(self, make_task, dist_repo_init): session = kojihub.context.session = mock.MagicMock() # It seems MagicMock will not automatically handle attributes that # start with "assert" session.assertPerm = mock.MagicMock() - signed_repo_init.return_value = ('repo_id', 'event_id') + dist_repo_init.return_value = ('repo_id', 'event_id') make_task.return_value = 'task_id' exports = kojihub.RootExports() - ret = exports.signedRepo('tag', 'keys') - session.assertPerm.assert_called_once_with('signed-repo') - signed_repo_init.assert_called_once() + ret = exports.distRepo('tag', 'keys') + session.assertPerm.assert_called_once_with('dist-repo') + dist_repo_init.assert_called_once() make_task.assert_called_once() self.assertEquals(ret, make_task.return_value) -class TestSignedRepoMove(unittest.TestCase): +class TestDistRepoMove(unittest.TestCase): def setUp(self): self.topdir = tempfile.mkdtemp() @@ -114,7 +114,7 @@ class TestSignedRepoMove(unittest.TestCase): # set up a fake koji topdir # koji.pathinfo._topdir = self.topdir mock.patch('koji.pathinfo._topdir', new=self.topdir).start() - repodir = koji.pathinfo.signedrepo(self.rinfo['id'], self.rinfo['tag_name']) + repodir = koji.pathinfo.distrepo(self.rinfo['id'], self.rinfo['tag_name']) archdir = "%s/%s" % (repodir, koji.canonArch(self.arch)) os.makedirs(archdir) self.uploadpath = 'UNITTEST' @@ -185,9 +185,9 @@ class TestSignedRepoMove(unittest.TestCase): return self.builds[buildInfo] - def test_signedRepoMove(self): + def test_distRepoMove(self): exports = kojihub.HostExports() - exports.signedRepoMove(self.rinfo['id'], self.uploadpath, + exports.distRepoMove(self.rinfo['id'], self.uploadpath, list(self.files), self.arch, self.sigmap) # check result repodir = self.topdir + '/repos-signed/%(tag_name)s/%(id)s' % self.rinfo diff --git a/util/kojira b/util/kojira index 38e02b3..45ff365 100755 --- a/util/kojira +++ b/util/kojira @@ -136,8 +136,8 @@ class ManagedRepo(object): tag_name = tag_info['name'] rinfo = self.session.repoInfo(self.repo_id, strict=True) if rinfo['signed']: - path = pathinfo.signedrepo(self.repo_id, tag_name) - lifetime = self.options.signed_repo_lifetime + path = pathinfo.distrepo(self.repo_id, tag_name) + lifetime = self.options.dist_repo_lifetime else: path = pathinfo.repo(self.repo_id, tag_name) lifetime = self.options.deleted_repo_lifetime @@ -642,14 +642,14 @@ def main(options, session): # TODO also move rmtree jobs to threads logger.info("Entering main loop") repodir = "%s/repos" % pathinfo.topdir - signedrepodir = "%s/repos-signed" % pathinfo.topdir + distrepodir = "%s/repos-signed" % pathinfo.topdir while True: try: repomgr.updateRepos() repomgr.checkQueue() repomgr.printState() repomgr.pruneLocalRepos(repodir, 'deleted_repo_lifetime') - repomgr.pruneLocalRepos(signedrepodir, 'signed_repo_lifetime') + repomgr.pruneLocalRepos(distrepodir, 'dist_repo_lifetime') if not curr_chk_thread.isAlive(): logger.error("Currency checker thread died. Restarting it.") curr_chk_thread = start_currency_checker(session, repomgr) @@ -745,7 +745,7 @@ def get_options(): 'delete_batch_size' : 3, 'deleted_repo_lifetime': 7*24*3600, #XXX should really be called expired_repo_lifetime - 'signed_repo_lifetime': 7*24*3600, + 'dist_repo_lifetime': 7*24*3600, 'sleeptime' : 15, 'cert': None, 'ca': '', # FIXME: unused, remove in next major release @@ -755,7 +755,7 @@ def get_options(): int_opts = ('deleted_repo_lifetime', 'max_repo_tasks', 'repo_tasks_limit', 'retry_interval', 'max_retries', 'offline_retry_interval', 'max_delete_processes', 'max_repo_tasks_maven', - 'delete_batch_size', 'signed_repo_lifetime') + 'delete_batch_size', 'dist_repo_lifetime') str_opts = ('topdir', 'server', 'user', 'password', 'logfile', 'principal', 'keytab', 'krbservice', 'cert', 'ca', 'serverca', 'debuginfo_tags', 'source_tags') # FIXME: remove ca here bool_opts = ('with_src','verbose','debug','ignore_stray_repos', 'offline_retry', diff --git a/util/kojira.conf b/util/kojira.conf index 1d361b3..fc8f4c0 100644 --- a/util/kojira.conf +++ b/util/kojira.conf @@ -43,8 +43,8 @@ with_src=no ;how soon (in seconds) to clean up expired repositories. 1 week default ;deleted_repo_lifetime = 604800 -;how soon (in seconds) to clean up signed repositories. 1 week default here too -;signed_repo_lifetime = 604800 +;how soon (in seconds) to clean up dist repositories. 1 week default here too +;dist_repo_lifetime = 604800 ;turn on debugging statements in the log ;debug = false diff --git a/www/kojiweb/index.py b/www/kojiweb/index.py index 8fc8248..49ea24b 100644 --- a/www/kojiweb/index.py +++ b/www/kojiweb/index.py @@ -431,8 +431,8 @@ _TASKS = ['build', 'tagBuild', 'newRepo', 'createrepo', - 'signedRepo', - 'createsignedrepo', + 'distRepo', + 'createdistrepo', 'buildNotification', 'tagNotification', 'dependantTask', @@ -446,9 +446,9 @@ _TASKS = ['build', 'livemedia', 'createLiveMedia'] # Tasks that can exist without a parent -_TOPLEVEL_TASKS = ['build', 'buildNotification', 'chainbuild', 'maven', 'chainmaven', 'wrapperRPM', 'winbuild', 'newRepo', 'signedRepo', 'tagBuild', 'tagNotification', 'waitrepo', 'livecd', 'appliance', 'image', 'livemedia'] +_TOPLEVEL_TASKS = ['build', 'buildNotification', 'chainbuild', 'maven', 'chainmaven', 'wrapperRPM', 'winbuild', 'newRepo', 'distRepo', 'tagBuild', 'tagNotification', 'waitrepo', 'livecd', 'appliance', 'image', 'livemedia'] # Tasks that can have children -_PARENT_TASKS = ['build', 'chainbuild', 'maven', 'chainmaven', 'winbuild', 'newRepo', 'signedRepo', 'wrapperRPM', 'livecd', 'appliance', 'image', 'livemedia'] +_PARENT_TASKS = ['build', 'chainbuild', 'maven', 'chainmaven', 'winbuild', 'newRepo', 'distRepo', 'wrapperRPM', 'livecd', 'appliance', 'image', 'livemedia'] def tasks(environ, owner=None, state='active', view='tree', method='all', hostID=None, channelID=None, start=None, order='-id'): values = _initValues(environ, 'Tasks', 'tasks') @@ -625,7 +625,7 @@ def taskinfo(environ, taskID): build = server.getBuild(params[1]) values['destTag'] = destTag values['build'] = build - elif task['method'] in ('newRepo', 'signedRepo', 'createsignedrepo'): + elif task['method'] in ('newRepo', 'distRepo', 'createdistrepo'): tag = server.getTag(params[0]) values['tag'] = tag elif task['method'] == 'tagNotification': diff --git a/www/kojiweb/taskinfo.chtml b/www/kojiweb/taskinfo.chtml index 613e2f9..a4e050f 100644 --- a/www/kojiweb/taskinfo.chtml +++ b/www/kojiweb/taskinfo.chtml @@ -223,7 +223,7 @@ $value #if $len($params) > 1 $printOpts($params[1]) #end if - #elif $task.method == 'signedRepo' + #elif $task.method == 'distRepo' Tag: $tag.name
Repo ID: $params[1]
Keys: $printValue(0, $params[2])
@@ -241,7 +241,7 @@ $value #if $len($params) > 4 and $params[4] External Repos: $printValue(None, [ext['external_repo_name'] for ext in $params[3]])
#end if - #elif $task.method == 'createsignedrepo' + #elif $task.method == 'createdistrepo' Tag: $tag.name
Repo ID: $params[1]
Arch: $printValue(0, $params[2])
From 8346a60976b8f79762f2bdeecacba70b4cf98e00 Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Mar 30 2017 13:47:20 +0000 Subject: [PATCH 61/77] more renaming --- diff --git a/builder/kojid b/builder/kojid index 74a0d55..3d88110 100755 --- a/builder/kojid +++ b/builder/kojid @@ -5038,8 +5038,9 @@ class createDistRepoTask(CreaterepoTask): # should be a list of repo ids to delta against for repo_id in opts['delta']: oldrepo = self.session.repoInfo(repo_id, strict=True) - if not oldrepo['signed']: - raise koji.GenericError("Base repo for deltas must be signed") + if not oldrepo['dist']: + raise koji.GenericError("Base repo for deltas must also " + "be a dist repo") # regular repos don't actually have rpms, just pkglist path = koji.pathinfo.distrepo(repo_id, oldrepo['tag_name']) if not os.path.exists(path): diff --git a/cli/koji b/cli/koji index 3506884..842b5d0 100755 --- a/cli/koji +++ b/cli/koji @@ -7091,7 +7091,7 @@ def handle_regen_repo(options, session, args): return watch_tasks(session, [task_id], quiet=options.quiet) def handle_dist_repo(options, session, args): - """create a yum repo of GPG signed RPMs""" + """Create a yum repo with distribution options""" usage = _("usage: %prog dist-repo [options] tag keyID [keyID...]") usage += _("\n(Specify the --help option for a list of other options)") parser = OptionParser(usage=usage) @@ -7126,7 +7126,7 @@ def handle_dist_repo(options, session, args): if task_opts.allow_unsigned and task_opts.skip_unsigned: parser.error(_('allow_unsigned and skip_unsigned are mutually exclusive')) activate_session(session) - stuffdir = _unique_path('cli-signed') + stuffdir = _unique_path('cli-dist-repo') if task_opts.comps: if not os.path.exists(task_opts.comps): parser.error(_('could not find %s') % task_opts.comps) @@ -7142,11 +7142,11 @@ def handle_dist_repo(options, session, args): rinfo = session.repoInfo(int(repo), strict=True) else: # get dist repo for tag - rinfo = session.getRepo(repo, signed=True) + rinfo = session.getRepo(repo, dist=True) if not rinfo: # maybe there is an expired one rinfo = session.getRepo(repo, - state=koji.REPO_STATES['EXPIRED'], signed=True) + state=koji.REPO_STATES['EXPIRED'], dist=True) if not rinfo: parser.errpr(_("Can't find repo for tag: %s") % repo) old_repos.append(rinfo['id']) diff --git a/docs/schema-update-signed-repos.sql b/docs/schema-update-signed-repos.sql index fa38ef1..ef71423 100644 --- a/docs/schema-update-signed-repos.sql +++ b/docs/schema-update-signed-repos.sql @@ -3,5 +3,5 @@ INSERT INTO permissions (name) VALUES ('image'); -ALTER TABLE repo ADD COLUMN signed BOOLEAN DEFAULT 'false'; +ALTER TABLE repo ADD COLUMN dist BOOLEAN DEFAULT 'false'; diff --git a/docs/schema.sql b/docs/schema.sql index 2edaab8..18bff30 100644 --- a/docs/schema.sql +++ b/docs/schema.sql @@ -411,7 +411,7 @@ CREATE TABLE repo ( create_event INTEGER NOT NULL REFERENCES events(id) DEFAULT get_event(), tag_id INTEGER NOT NULL REFERENCES tag(id), state INTEGER, - signed BOOLEAN DEFAULT 'false' + dist BOOLEAN DEFAULT 'false' ) WITHOUT OIDS; -- external yum repos diff --git a/hub/kojihub.py b/hub/kojihub.py index d35ce54..3e936e2 100644 --- a/hub/kojihub.py +++ b/hub/kojihub.py @@ -2452,13 +2452,13 @@ def dist_repo_init(tag, keys, task_opts): # note: we need to match args from the other preRepoInit callback koji.plugin.run_callbacks('preRepoInit', tag=tinfo, with_src=False, with_debuginfo=False, event=event, repo_id=None, - signed=True, keys=keys, arches=arches, task_opts=task_opts) + dist=True, keys=keys, arches=arches, task_opts=task_opts) if not event: event = get_event() repo_id = nextval('repo_id_seq') insert = InsertProcessor('repo') insert.set(id=repo_id, create_event=event, tag_id=tag_id, - state=state, signed=True) + state=state, dist=True) insert.execute() repodir = koji.pathinfo.distrepo(repo_id, tinfo['name']) for arch in arches: @@ -10109,20 +10109,20 @@ class RootExports(object): taginfo['extra'][key] = ancestor['extra'][key] return taginfo - def getRepo(self, tag, state=None, event=None, signed=False): + def getRepo(self, tag, state=None, event=None, dist=False): if isinstance(tag, (int, long)): id = tag else: id = get_tag_id(tag, strict=True) - fields = ['repo.id', 'repo.state', 'repo.create_event', 'events.time', 'EXTRACT(EPOCH FROM events.time)', 'repo.signed'] - aliases = ['id', 'state', 'create_event', 'creation_time', 'create_ts', 'signed'] + fields = ['repo.id', 'repo.state', 'repo.create_event', 'events.time', 'EXTRACT(EPOCH FROM events.time)', 'repo.dist'] + aliases = ['id', 'state', 'create_event', 'creation_time', 'create_ts', 'dist'] joins = ['events ON repo.create_event = events.id'] clauses = ['repo.tag_id = %(id)i'] - if signed: - clauses.append('repo.signed is true') + if dist: + clauses.append('repo.dist is true') else: - clauses.append('repo.signed is false') + clauses.append('repo.dist is false') if event: # the repo table doesn't have all the fields of a _config table, just create_event clauses.append('create_event <= %(event)i') @@ -12277,7 +12277,7 @@ class HostExports(object): data: a dictionary of the form { arch: (uploadpath, files), ...} expire(optional): if set to true, mark the repo expired immediately* - If this is a dist repo, also hardlink signed rpms in the final + If this is a dist repo, also hardlink the rpms in the final directory. * This is used when a repo from an older event is generated @@ -12290,7 +12290,7 @@ class HostExports(object): raise koji.GenericError("Repo %(id)s not in INIT state (got %(state)s)" % rinfo) repodir = koji.pathinfo.repo(repo_id, rinfo['tag_name']) workdir = koji.pathinfo.work() - if not rinfo['signed']: + if not rinfo['dist']: for arch, (uploadpath, files) in data.iteritems(): archdir = "%s/%s" % (repodir, koji.canonArch(arch)) if not os.path.isdir(archdir): @@ -12331,7 +12331,7 @@ class HostExports(object): Move a dist repo into its final location - Unlike normal repos (which are moved into place by repoDone), signed + Unlike normal repos (which are moved into place by repoDone), dist repos have all their content linked (or copied) into place. repo_id - the repo to move diff --git a/koji/__init__.py b/koji/__init__.py index 2f06b75..8509f80 100644 --- a/koji/__init__.py +++ b/koji/__init__.py @@ -1817,7 +1817,7 @@ class PathInfo(object): def distrepo(self, repo_id, tag): """Return the directory with a dist repo lives""" - return os.path.join(self.topdir, 'repos-signed', tag, str(repo_id)) + return os.path.join(self.topdir, 'repos-dist', tag, str(repo_id)) def repocache(self, tag_str): """Return the directory where a repo belongs""" diff --git a/util/kojira b/util/kojira index 45ff365..4a21332 100755 --- a/util/kojira +++ b/util/kojira @@ -135,7 +135,7 @@ class ManagedRepo(object): return False tag_name = tag_info['name'] rinfo = self.session.repoInfo(self.repo_id, strict=True) - if rinfo['signed']: + if rinfo['dist']: path = pathinfo.distrepo(self.repo_id, tag_name) lifetime = self.options.dist_repo_lifetime else: @@ -642,7 +642,7 @@ def main(options, session): # TODO also move rmtree jobs to threads logger.info("Entering main loop") repodir = "%s/repos" % pathinfo.topdir - distrepodir = "%s/repos-signed" % pathinfo.topdir + distrepodir = "%s/repos-dist" % pathinfo.topdir while True: try: repomgr.updateRepos() From 9c2564da85008ade61fed5c6dc51c6e951a128f2 Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Mar 30 2017 13:47:20 +0000 Subject: [PATCH 62/77] rename files --- diff --git a/docs/schema-update-dist-repos.sql b/docs/schema-update-dist-repos.sql new file mode 100644 index 0000000..ef71423 --- /dev/null +++ b/docs/schema-update-dist-repos.sql @@ -0,0 +1,7 @@ +# schema updates for dist repo feature +# to be merged into schema upgrade script for next release + +INSERT INTO permissions (name) VALUES ('image'); + +ALTER TABLE repo ADD COLUMN dist BOOLEAN DEFAULT 'false'; + diff --git a/docs/schema-update-signed-repos.sql b/docs/schema-update-signed-repos.sql deleted file mode 100644 index ef71423..0000000 --- a/docs/schema-update-signed-repos.sql +++ /dev/null @@ -1,7 +0,0 @@ -# schema updates for dist repo feature -# to be merged into schema upgrade script for next release - -INSERT INTO permissions (name) VALUES ('image'); - -ALTER TABLE repo ADD COLUMN dist BOOLEAN DEFAULT 'false'; - diff --git a/tests/test_hub/test_dist_repo.py b/tests/test_hub/test_dist_repo.py new file mode 100644 index 0000000..e62306f --- /dev/null +++ b/tests/test_hub/test_dist_repo.py @@ -0,0 +1,202 @@ + +import unittest +import mock +import os +import shutil +import tempfile + +import koji +import kojihub +from koji.util import dslice_ex + +IP = kojihub.InsertProcessor + + +class TestDistRepoInit(unittest.TestCase): + + + def getInsert(self, *args, **kwargs): + insert = IP(*args, **kwargs) + insert.execute = mock.MagicMock() + self.inserts.append(insert) + return insert + + + def setUp(self): + self.InsertProcessor = mock.patch('kojihub.InsertProcessor', + side_effect=self.getInsert).start() + self.inserts = [] + + self.get_tag = mock.patch('kojihub.get_tag').start() + self.get_event = mock.patch('kojihub.get_event').start() + self.nextval = mock.patch('kojihub.nextval').start() + self.ensuredir = mock.patch('koji.ensuredir').start() + self.copyfile = mock.patch('shutil.copyfile').start() + + self.get_tag.return_value = {'id': 42, 'name': 'tag'} + self.get_event.return_value = 12345 + self.nextval.return_value = 99 + + + def tearDown(self): + mock.patch.stopall() + + + def test_simple_dist_repo_init(self): + + # simple case + kojihub.dist_repo_init('tag', ['key'], {'arch': ['x86_64']}) + self.InsertProcessor.assert_called_once() + + ip = self.inserts[0] + self.assertEquals(ip.table, 'repo') + data = {'signed': True, 'create_event': 12345, 'tag_id': 42, 'id': 99, + 'state': koji.REPO_STATES['INIT']} + self.assertEquals(ip.data, data) + self.assertEquals(ip.rawdata, {}) + + # no comps option + self.copyfile.assert_not_called() + + + def test_dist_repo_init_with_comps(self): + + # simple case + kojihub.dist_repo_init('tag', ['key'], {'arch': ['x86_64'], + 'comps': 'COMPSFILE'}) + self.InsertProcessor.assert_called_once() + + ip = self.inserts[0] + self.assertEquals(ip.table, 'repo') + data = {'signed': True, 'create_event': 12345, 'tag_id': 42, 'id': 99, + 'state': koji.REPO_STATES['INIT']} + self.assertEquals(ip.data, data) + self.assertEquals(ip.rawdata, {}) + + # no comps option + self.copyfile.assert_called_once() + + +class TestDistRepo(unittest.TestCase): + + @mock.patch('kojihub.dist_repo_init') + @mock.patch('kojihub.make_task') + def test_DistRepo(self, make_task, dist_repo_init): + session = kojihub.context.session = mock.MagicMock() + # It seems MagicMock will not automatically handle attributes that + # start with "assert" + session.assertPerm = mock.MagicMock() + dist_repo_init.return_value = ('repo_id', 'event_id') + make_task.return_value = 'task_id' + + exports = kojihub.RootExports() + ret = exports.distRepo('tag', 'keys') + session.assertPerm.assert_called_once_with('dist-repo') + dist_repo_init.assert_called_once() + make_task.assert_called_once() + self.assertEquals(ret, make_task.return_value) + + +class TestDistRepoMove(unittest.TestCase): + + def setUp(self): + self.topdir = tempfile.mkdtemp() + self.rinfo = { + 'create_event': 2915, + 'create_ts': 1487256924.72718, + 'creation_time': '2017-02-16 14:55:24.727181', + 'id': 47, + 'state': 1, + 'tag_id': 2, + 'tag_name': 'my-tag'} + self.arch = 'x86_64' + + # set up a fake koji topdir + # koji.pathinfo._topdir = self.topdir + mock.patch('koji.pathinfo._topdir', new=self.topdir).start() + repodir = koji.pathinfo.distrepo(self.rinfo['id'], self.rinfo['tag_name']) + archdir = "%s/%s" % (repodir, koji.canonArch(self.arch)) + os.makedirs(archdir) + self.uploadpath = 'UNITTEST' + workdir = koji.pathinfo.work() + uploaddir = "%s/%s" % (workdir, self.uploadpath) + os.makedirs(uploaddir) + + # place some test files + self.files = ['foo.drpm', 'repomd.xml'] + self.expected = ['x86_64/drpms/foo.drpm', 'x86_64/repodata/repomd.xml'] + for fn in self.files: + path = os.path.join(uploaddir, fn) + koji.ensuredir(os.path.dirname(path)) + with open(path, 'w') as fo: + fo.write('%s' % fn) + + # generate pkglist file and sigmap + self.files.append('pkglist') + plist = os.path.join(uploaddir, 'pkglist') + nvrs = ['aaa-1.0-2', 'bbb-3.0-5', 'ccc-8.0-13','ddd-21.0-34'] + self.sigmap = [] + self.rpms = {} + self.builds ={} + self.key = '4c8da725' + with open(plist, 'w') as f_pkglist: + for nvr in nvrs: + binfo = koji.parse_NVR(nvr) + rpminfo = binfo.copy() + rpminfo['arch'] = 'x86_64' + builddir = koji.pathinfo.build(binfo) + relpath = koji.pathinfo.signed(rpminfo, self.key) + path = os.path.join(builddir, relpath) + koji.ensuredir(os.path.dirname(path)) + basename = os.path.basename(path) + with open(path, 'w') as fo: + fo.write('%s' % basename) + f_pkglist.write(path) + f_pkglist.write('\n') + self.expected.append('x86_64/%s/%s' % (basename[0], basename)) + build_id = len(self.builds) + 10000 + rpm_id = len(self.rpms) + 20000 + binfo['id'] = build_id + rpminfo['build_id'] = build_id + rpminfo['id'] = rpm_id + self.builds[build_id] = binfo + self.rpms[rpm_id] = rpminfo + self.sigmap.append([rpm_id, self.key]) + + # mocks + self.repo_info = mock.patch('kojihub.repo_info').start() + self.repo_info.return_value = self.rinfo.copy() + self.get_rpm = mock.patch('kojihub.get_rpm').start() + self.get_build = mock.patch('kojihub.get_build').start() + self.get_rpm.side_effect = self.our_get_rpm + self.get_build.side_effect = self.our_get_build + + + def tearDown(self): + mock.patch.stopall() + shutil.rmtree(self.topdir) + + + def our_get_rpm(self, rpminfo, strict=False, multi=False): + return self.rpms[rpminfo] + + + def our_get_build(self, buildInfo, strict=False): + return self.builds[buildInfo] + + + def test_distRepoMove(self): + exports = kojihub.HostExports() + exports.distRepoMove(self.rinfo['id'], self.uploadpath, + list(self.files), self.arch, self.sigmap) + # check result + repodir = self.topdir + '/repos-signed/%(tag_name)s/%(id)s' % self.rinfo + for relpath in self.expected: + path = os.path.join(repodir, relpath) + basename = os.path.basename(path) + if not os.path.exists(path): + raise Exception("Missing file: %s" % path) + data = open(path).read() + data.strip() + self.assertEquals(data, basename) + diff --git a/tests/test_hub/test_signed_repo.py b/tests/test_hub/test_signed_repo.py deleted file mode 100644 index e62306f..0000000 --- a/tests/test_hub/test_signed_repo.py +++ /dev/null @@ -1,202 +0,0 @@ - -import unittest -import mock -import os -import shutil -import tempfile - -import koji -import kojihub -from koji.util import dslice_ex - -IP = kojihub.InsertProcessor - - -class TestDistRepoInit(unittest.TestCase): - - - def getInsert(self, *args, **kwargs): - insert = IP(*args, **kwargs) - insert.execute = mock.MagicMock() - self.inserts.append(insert) - return insert - - - def setUp(self): - self.InsertProcessor = mock.patch('kojihub.InsertProcessor', - side_effect=self.getInsert).start() - self.inserts = [] - - self.get_tag = mock.patch('kojihub.get_tag').start() - self.get_event = mock.patch('kojihub.get_event').start() - self.nextval = mock.patch('kojihub.nextval').start() - self.ensuredir = mock.patch('koji.ensuredir').start() - self.copyfile = mock.patch('shutil.copyfile').start() - - self.get_tag.return_value = {'id': 42, 'name': 'tag'} - self.get_event.return_value = 12345 - self.nextval.return_value = 99 - - - def tearDown(self): - mock.patch.stopall() - - - def test_simple_dist_repo_init(self): - - # simple case - kojihub.dist_repo_init('tag', ['key'], {'arch': ['x86_64']}) - self.InsertProcessor.assert_called_once() - - ip = self.inserts[0] - self.assertEquals(ip.table, 'repo') - data = {'signed': True, 'create_event': 12345, 'tag_id': 42, 'id': 99, - 'state': koji.REPO_STATES['INIT']} - self.assertEquals(ip.data, data) - self.assertEquals(ip.rawdata, {}) - - # no comps option - self.copyfile.assert_not_called() - - - def test_dist_repo_init_with_comps(self): - - # simple case - kojihub.dist_repo_init('tag', ['key'], {'arch': ['x86_64'], - 'comps': 'COMPSFILE'}) - self.InsertProcessor.assert_called_once() - - ip = self.inserts[0] - self.assertEquals(ip.table, 'repo') - data = {'signed': True, 'create_event': 12345, 'tag_id': 42, 'id': 99, - 'state': koji.REPO_STATES['INIT']} - self.assertEquals(ip.data, data) - self.assertEquals(ip.rawdata, {}) - - # no comps option - self.copyfile.assert_called_once() - - -class TestDistRepo(unittest.TestCase): - - @mock.patch('kojihub.dist_repo_init') - @mock.patch('kojihub.make_task') - def test_DistRepo(self, make_task, dist_repo_init): - session = kojihub.context.session = mock.MagicMock() - # It seems MagicMock will not automatically handle attributes that - # start with "assert" - session.assertPerm = mock.MagicMock() - dist_repo_init.return_value = ('repo_id', 'event_id') - make_task.return_value = 'task_id' - - exports = kojihub.RootExports() - ret = exports.distRepo('tag', 'keys') - session.assertPerm.assert_called_once_with('dist-repo') - dist_repo_init.assert_called_once() - make_task.assert_called_once() - self.assertEquals(ret, make_task.return_value) - - -class TestDistRepoMove(unittest.TestCase): - - def setUp(self): - self.topdir = tempfile.mkdtemp() - self.rinfo = { - 'create_event': 2915, - 'create_ts': 1487256924.72718, - 'creation_time': '2017-02-16 14:55:24.727181', - 'id': 47, - 'state': 1, - 'tag_id': 2, - 'tag_name': 'my-tag'} - self.arch = 'x86_64' - - # set up a fake koji topdir - # koji.pathinfo._topdir = self.topdir - mock.patch('koji.pathinfo._topdir', new=self.topdir).start() - repodir = koji.pathinfo.distrepo(self.rinfo['id'], self.rinfo['tag_name']) - archdir = "%s/%s" % (repodir, koji.canonArch(self.arch)) - os.makedirs(archdir) - self.uploadpath = 'UNITTEST' - workdir = koji.pathinfo.work() - uploaddir = "%s/%s" % (workdir, self.uploadpath) - os.makedirs(uploaddir) - - # place some test files - self.files = ['foo.drpm', 'repomd.xml'] - self.expected = ['x86_64/drpms/foo.drpm', 'x86_64/repodata/repomd.xml'] - for fn in self.files: - path = os.path.join(uploaddir, fn) - koji.ensuredir(os.path.dirname(path)) - with open(path, 'w') as fo: - fo.write('%s' % fn) - - # generate pkglist file and sigmap - self.files.append('pkglist') - plist = os.path.join(uploaddir, 'pkglist') - nvrs = ['aaa-1.0-2', 'bbb-3.0-5', 'ccc-8.0-13','ddd-21.0-34'] - self.sigmap = [] - self.rpms = {} - self.builds ={} - self.key = '4c8da725' - with open(plist, 'w') as f_pkglist: - for nvr in nvrs: - binfo = koji.parse_NVR(nvr) - rpminfo = binfo.copy() - rpminfo['arch'] = 'x86_64' - builddir = koji.pathinfo.build(binfo) - relpath = koji.pathinfo.signed(rpminfo, self.key) - path = os.path.join(builddir, relpath) - koji.ensuredir(os.path.dirname(path)) - basename = os.path.basename(path) - with open(path, 'w') as fo: - fo.write('%s' % basename) - f_pkglist.write(path) - f_pkglist.write('\n') - self.expected.append('x86_64/%s/%s' % (basename[0], basename)) - build_id = len(self.builds) + 10000 - rpm_id = len(self.rpms) + 20000 - binfo['id'] = build_id - rpminfo['build_id'] = build_id - rpminfo['id'] = rpm_id - self.builds[build_id] = binfo - self.rpms[rpm_id] = rpminfo - self.sigmap.append([rpm_id, self.key]) - - # mocks - self.repo_info = mock.patch('kojihub.repo_info').start() - self.repo_info.return_value = self.rinfo.copy() - self.get_rpm = mock.patch('kojihub.get_rpm').start() - self.get_build = mock.patch('kojihub.get_build').start() - self.get_rpm.side_effect = self.our_get_rpm - self.get_build.side_effect = self.our_get_build - - - def tearDown(self): - mock.patch.stopall() - shutil.rmtree(self.topdir) - - - def our_get_rpm(self, rpminfo, strict=False, multi=False): - return self.rpms[rpminfo] - - - def our_get_build(self, buildInfo, strict=False): - return self.builds[buildInfo] - - - def test_distRepoMove(self): - exports = kojihub.HostExports() - exports.distRepoMove(self.rinfo['id'], self.uploadpath, - list(self.files), self.arch, self.sigmap) - # check result - repodir = self.topdir + '/repos-signed/%(tag_name)s/%(id)s' % self.rinfo - for relpath in self.expected: - path = os.path.join(repodir, relpath) - basename = os.path.basename(path) - if not os.path.exists(path): - raise Exception("Missing file: %s" % path) - data = open(path).read() - data.strip() - self.assertEquals(data, basename) - From 1ceb366f9d4c512453e7fea6c640e6bae43478d5 Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Mar 30 2017 13:49:58 +0000 Subject: [PATCH 63/77] last bit of renaming --- diff --git a/hub/kojihub.py b/hub/kojihub.py index 3e936e2..9f3149c 100644 --- a/hub/kojihub.py +++ b/hub/kojihub.py @@ -2496,7 +2496,7 @@ def repo_info(repo_id, strict=False): ('EXTRACT(EPOCH FROM events.time)', 'create_ts'), ('repo.tag_id', 'tag_id'), ('tag.name', 'tag_name'), - ('repo.signed', 'signed'), + ('repo.dist', 'dist'), ) q = """SELECT %s FROM repo JOIN tag ON tag_id=tag.id diff --git a/tests/test_cli/data/list-commands.txt b/tests/test_cli/data/list-commands.txt index 5a21320..f2d6a25 100644 --- a/tests/test_cli/data/list-commands.txt +++ b/tests/test_cli/data/list-commands.txt @@ -116,10 +116,10 @@ info commands: miscellaneous commands: call Execute an arbitrary XML-RPC call + dist-repo Create a yum repo with distribution options import-comps Import group/package information from a comps file moshimoshi Introduce yourself save-failed-tree Create tarball with whole buildtree - dist-repo create a yum repo of GPG signed RPMs monitor commands: wait-repo Wait for a repo to be regenerated diff --git a/tests/test_hub/test_dist_repo.py b/tests/test_hub/test_dist_repo.py index e62306f..33f1b6c 100644 --- a/tests/test_hub/test_dist_repo.py +++ b/tests/test_hub/test_dist_repo.py @@ -50,7 +50,7 @@ class TestDistRepoInit(unittest.TestCase): ip = self.inserts[0] self.assertEquals(ip.table, 'repo') - data = {'signed': True, 'create_event': 12345, 'tag_id': 42, 'id': 99, + data = {'dist': True, 'create_event': 12345, 'tag_id': 42, 'id': 99, 'state': koji.REPO_STATES['INIT']} self.assertEquals(ip.data, data) self.assertEquals(ip.rawdata, {}) @@ -68,7 +68,7 @@ class TestDistRepoInit(unittest.TestCase): ip = self.inserts[0] self.assertEquals(ip.table, 'repo') - data = {'signed': True, 'create_event': 12345, 'tag_id': 42, 'id': 99, + data = {'dist': True, 'create_event': 12345, 'tag_id': 42, 'id': 99, 'state': koji.REPO_STATES['INIT']} self.assertEquals(ip.data, data) self.assertEquals(ip.rawdata, {}) @@ -190,7 +190,7 @@ class TestDistRepoMove(unittest.TestCase): exports.distRepoMove(self.rinfo['id'], self.uploadpath, list(self.files), self.arch, self.sigmap) # check result - repodir = self.topdir + '/repos-signed/%(tag_name)s/%(id)s' % self.rinfo + repodir = self.topdir + '/repos-dist/%(tag_name)s/%(id)s' % self.rinfo for relpath in self.expected: path = os.path.join(repodir, relpath) basename = os.path.basename(path) From 33e1c3ac4fd8f1749761b7374ad18e0c43acb970 Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Mar 30 2017 13:49:58 +0000 Subject: [PATCH 64/77] cleanup: has_key and print 2to3 -pvwn --fix has_key 2to3 -pvwn --fix print --- diff --git a/builder/kojid b/builder/kojid index 3d88110..d833d2d 100755 --- a/builder/kojid +++ b/builder/kojid @@ -5089,7 +5089,7 @@ class createDistRepoTask(CreaterepoTask): for pkg in pkglist: ppath = os.path.join(self.repodir, pkg.strip()) po = yum.packages.YumLocalPackage(filename=ppath) - if mlm.select(po) and self.archmap.has_key(arch): + if mlm.select(po) and arch in self.archmap: # we need a multilib package to be included # we assume the same signature level is available # XXX: what is a subarchitecture is the right answer? @@ -5231,7 +5231,7 @@ enabled=1 # skip, not a key we are looking for continue idx = keys.index(rpminfo['sigkey']) - if preferred.has_key(rpminfo['id']): + if rpminfo['id'] in preferred: if keys.index(preferred[rpminfo['id']]['sigkey']) <= idx: # key for this is not as preferable as what has been seen continue diff --git a/cli/koji b/cli/koji index 842b5d0..969d04d 100755 --- a/cli/koji +++ b/cli/koji @@ -7132,7 +7132,7 @@ def handle_dist_repo(options, session, args): parser.error(_('could not find %s') % task_opts.comps) session.uploadWrapper(task_opts.comps, stuffdir, callback=_progress_callback) - print + print() task_opts.comps = os.path.join(stuffdir, os.path.basename(task_opts.comps)) old_repos = [] @@ -7176,7 +7176,7 @@ def handle_dist_repo(options, session, args): callback=_progress_callback) task_opts.multilib = os.path.join(stuffdir, os.path.basename(task_opts.multilib)) - print + print() try: task_opts.arch.remove('noarch') # handled specifically task_opts.arch.remove('src') # ditto From dda2549d9c96e804ac71e3b812b23ea41ae4c132 Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Mar 30 2017 13:49:58 +0000 Subject: [PATCH 65/77] handle case where tag archlist is None --- diff --git a/builder/kojid b/builder/kojid index d833d2d..3a72366 100755 --- a/builder/kojid +++ b/builder/kojid @@ -4941,7 +4941,8 @@ class NewDistRepoTask(BaseTaskHandler): tinfo = self.session.getTag(tag, strict=True, event=task_opts['event']) path = koji.pathinfo.distrepo(repo_id, tinfo['name']) if len(task_opts['arch']) == 0: - task_opts['arch'] = tinfo['arches'].split() + arches = tinfo['arches'] or '' + task_opts['arch'] = arches.split() if len(task_opts['arch']) == 0: raise koji.GenericError('No arches specified nor for the tag!') subtasks = {} diff --git a/cli/koji b/cli/koji index 969d04d..853c988 100755 --- a/cli/koji +++ b/cli/koji @@ -7156,7 +7156,8 @@ def handle_dist_repo(options, session, args): if not taginfo: parser.error(_('unknown tag %s') % tag) if len(task_opts.arch) == 0: - task_opts.arch = taginfo['arches'].split() + arches = taginfo['arches'] or '' + task_opts.arch = arches.split() if task_opts.arch == None: parser.error(_('No arches given and no arches associated with tag')) else: From b7f301f33c162c6580cbf91c551509eeec7f8508 Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Mar 30 2017 13:49:58 +0000 Subject: [PATCH 66/77] log missing files and signatures for dist repos --- diff --git a/builder/kojid b/builder/kojid index 3a72366..34e0a1e 100755 --- a/builder/kojid +++ b/builder/kojid @@ -5222,12 +5222,15 @@ enabled=1 pkgfile = os.path.join(self.repodir, 'pkglist') pkglist = file(pkgfile, 'w') preferred = {} + rpm_keys = {} if opts['unsigned']: keys.append('') # make unsigned rpms the least preferred for rpminfo in rpms: if rpminfo['sigkey'] == '' and not opts['unsigned']: # skip, this is the unsigned rpminfo continue + fname = '%(name)s-%(version)s-%(release)s.%(arch)s.rpm' % rpminfo + rpm_keys.setdefault(fname, []).append(rpminfo['sigkey']) if rpminfo['sigkey'] not in keys: # skip, not a key we are looking for continue @@ -5265,14 +5268,33 @@ enabled=1 pkglist.close() self.kojipkgs = kojipkgs if len(fs_missing) > 0: + missing_log = os.path.join(self.workdir, 'missing_files.log') + outfile = open(missing_log, 'w') + outfile.write('Some rpm files were missing.\n' + 'Most likely, you want to create these signed copies.\n\n' + 'Missing files:\n') + for pkgpath in sorted(fs_missing): + outfile.write(pkgpath) + outfile.write('\n') + outfile.close() + self.session.uploadWrapper(missing_log, self.uploadpath) raise koji.GenericError('Packages missing from the filesystem:\n' + '\n'.join(fs_missing)) - if not opts['skip']: - missing = list(need - seen) - if len(missing) != 0: - missing.sort() - raise koji.GenericError('Unsigned packages found: ' + - '\n'.join(missing)) + missing = need - seen + if not opts['skip'] and missing: + # log missing signatures and error + missing_log = os.path.join(self.workdir, 'missing_signatures.log') + outfile = open(missing_log, 'w') + outfile.write('Some rpms were missing required signatures.\n') + outfile.write('Acceptable keys: %r\n\n' % keys) + outfile.write('# RPM name: available keys\n') + for fname in sorted(missing): + avail = rpm_keys.get(fname, []) + outfile.write('%s: %r\n' % (fname, avail)) + outfile.close() + self.session.uploadWrapper(missing_log, self.uploadpath) + raise koji.GenericError('Unsigned packages found. See ' + 'missing_signatures.log') return pkgfile From 216c21d89a763aee5a166db711849ec03cfc56a5 Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Mar 30 2017 13:49:58 +0000 Subject: [PATCH 67/77] log missing signatures even if allowing unsigned --- diff --git a/builder/kojid b/builder/kojid index 34e0a1e..4b3976d 100755 --- a/builder/kojid +++ b/builder/kojid @@ -5281,11 +5281,11 @@ enabled=1 raise koji.GenericError('Packages missing from the filesystem:\n' + '\n'.join(fs_missing)) missing = need - seen - if not opts['skip'] and missing: - # log missing signatures and error + if missing: + # log missing signatures and possibly error missing_log = os.path.join(self.workdir, 'missing_signatures.log') outfile = open(missing_log, 'w') - outfile.write('Some rpms were missing required signatures.\n') + outfile.write('Some rpms were missing requested signatures.\n') outfile.write('Acceptable keys: %r\n\n' % keys) outfile.write('# RPM name: available keys\n') for fname in sorted(missing): @@ -5293,8 +5293,9 @@ enabled=1 outfile.write('%s: %r\n' % (fname, avail)) outfile.close() self.session.uploadWrapper(missing_log, self.uploadpath) - raise koji.GenericError('Unsigned packages found. See ' - 'missing_signatures.log') + if not opts['skip']: + raise koji.GenericError('Unsigned packages found. See ' + 'missing_signatures.log') return pkgfile From 884a20c0cd37c54278db663ba1ce58fe3fa02350 Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Mar 30 2017 13:49:58 +0000 Subject: [PATCH 68/77] propagate the full name of the skip_unsigned option --- diff --git a/builder/kojid b/builder/kojid index 4b3976d..1d34f86 100755 --- a/builder/kojid +++ b/builder/kojid @@ -5278,14 +5278,17 @@ enabled=1 outfile.write('\n') outfile.close() self.session.uploadWrapper(missing_log, self.uploadpath) - raise koji.GenericError('Packages missing from the filesystem:\n' + - '\n'.join(fs_missing)) + raise koji.GenericError('Packages missing from the filesystem. ' + 'See missing_files.log.') missing = need - seen if missing: # log missing signatures and possibly error missing_log = os.path.join(self.workdir, 'missing_signatures.log') outfile = open(missing_log, 'w') outfile.write('Some rpms were missing requested signatures.\n') + if opts['skip_unsigned']: + outfile.write('The skip_unsigned option was specified, so ' + 'these files were excluded.\n') outfile.write('Acceptable keys: %r\n\n' % keys) outfile.write('# RPM name: available keys\n') for fname in sorted(missing): @@ -5293,7 +5296,7 @@ enabled=1 outfile.write('%s: %r\n' % (fname, avail)) outfile.close() self.session.uploadWrapper(missing_log, self.uploadpath) - if not opts['skip']: + if not opts['skip_unsigned']: raise koji.GenericError('Unsigned packages found. See ' 'missing_signatures.log') return pkgfile diff --git a/cli/koji b/cli/koji index 853c988..f0c2d58 100755 --- a/cli/koji +++ b/cli/koji @@ -7191,7 +7191,7 @@ def handle_dist_repo(options, session, args): 'inherit': not task_opts.noinherit, 'latest': task_opts.latest, 'multilib': task_opts.multilib, - 'skip': task_opts.skip_unsigned, + 'skip_unsigned': task_opts.skip_unsigned, 'unsigned': task_opts.allow_unsigned } task_id = session.distRepo(tag, keys, **opts) From 2a1d9678cc29071331e95fe9d6cf6e44bb16db76 Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Mar 30 2017 13:49:58 +0000 Subject: [PATCH 69/77] saner error on missing multilib files --- diff --git a/builder/kojid b/builder/kojid index 1d34f86..2498fa7 100755 --- a/builder/kojid +++ b/builder/kojid @@ -5097,10 +5097,12 @@ class createDistRepoTask(CreaterepoTask): pl_path = pkg.replace(arch, self.archmap[arch]).strip() # assume this exists in the task results for the ml arch real_path = os.path.join(mldir, pl_path) - ml_true.add(real_path) if not os.path.exists(real_path): self.logger.error('%s (multilib) is not on the filesystem' % real_path) fs_missing.add(real_path) + # we defer failure so can report all the missing deps + continue + ml_true.add(real_path) # step 2: set up architectures for yum configuration self.logger.info("Resolving multilib for %s using method devel" % arch) From 5555a14dd167075dde6fa4303380577c67deb9b8 Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Mar 30 2017 13:49:58 +0000 Subject: [PATCH 70/77] log missing multilib files --- diff --git a/builder/kojid b/builder/kojid index 2498fa7..0cf2fe4 100755 --- a/builder/kojid +++ b/builder/kojid @@ -5181,8 +5181,16 @@ enabled=1 self.logger.error('yum depsolve was unsuccessful') raise koji.GenericError(errors) if len(fs_missing) > 0: - raise koji.GenericError('multilib packages missing:\n' + - '\n'.join(fs_missing)) + missing_log = os.path.join(self.workdir, 'missing_multilib.log') + outfile = open(missing_log, 'w') + outfile.write('The following multilib files were missing:\n') + for ml_path in fs_missing: + outfile.write(ml_path) + outfile.write('\n') + outfile.close() + self.session.uploadWrapper(missing_log, self.uploadpath) + raise koji.GenericError('multilib packages missing. ' + 'See missing_multilib.log') # get rpm ids for ml pkgs kpkgfile = os.path.join(mldir, 'kojipkgs') From dc58f50fc6e711b38c4a2bfede853555298a114a Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Mar 30 2017 13:49:58 +0000 Subject: [PATCH 71/77] print('') is more 2.4 friendly --- diff --git a/cli/koji b/cli/koji index f0c2d58..fe0c85c 100755 --- a/cli/koji +++ b/cli/koji @@ -7132,7 +7132,7 @@ def handle_dist_repo(options, session, args): parser.error(_('could not find %s') % task_opts.comps) session.uploadWrapper(task_opts.comps, stuffdir, callback=_progress_callback) - print() + print('') task_opts.comps = os.path.join(stuffdir, os.path.basename(task_opts.comps)) old_repos = [] @@ -7177,7 +7177,7 @@ def handle_dist_repo(options, session, args): callback=_progress_callback) task_opts.multilib = os.path.join(stuffdir, os.path.basename(task_opts.multilib)) - print() + print('') try: task_opts.arch.remove('noarch') # handled specifically task_opts.arch.remove('src') # ditto From a0ed3a7a5e738de934d6fa58e838bc7c453cfa17 Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Mar 30 2017 13:49:58 +0000 Subject: [PATCH 72/77] fix latest links for dist repos --- diff --git a/hub/kojihub.py b/hub/kojihub.py index 9f3149c..e845037 100644 --- a/hub/kojihub.py +++ b/hub/kojihub.py @@ -12313,9 +12313,13 @@ class HostExports(object): #else: repo_ready(repo_id) repo_expire_older(rinfo['tag_id'], rinfo['create_event']) + #make a latest link - latestrepolink = koji.pathinfo.repo('latest', rinfo['tag_name']) - #XXX - this is a slight abuse of pathinfo + if rinfo['dist']: + latestrepolink = koji.pathinfo.distrepo('latest', rinfo['tag_name']) + else: + latestrepolink = koji.pathinfo.repo('latest', rinfo['tag_name']) + #XXX - this is a slight abuse of pathinfo try: if os.path.lexists(latestrepolink): os.unlink(latestrepolink) From 3bdd64f4ff094d68ddce91a2f71e649b13fa9233 Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Mar 30 2017 13:49:58 +0000 Subject: [PATCH 73/77] Don't require specifying a key if --allow-unsigned is given --- diff --git a/cli/koji b/cli/koji index fe0c85c..958c8c5 100755 --- a/cli/koji +++ b/cli/koji @@ -7121,8 +7121,10 @@ def handle_dist_repo(options, session, args): parser.add_option('--skip-unsigned', action='store_true', default=False, help=_('Skip RPMs not signed with the desired key(s)')) task_opts, args = parser.parse_args(args) - if len(args) < 2: - parser.error(_('You must provide a tag and 1 or more GPG key IDs')) + if len(args) < 1: + parser.error(_('You must provide a tag to generate the repo from')) + if not task_opts.allow_unsigned: + parser.error(_('Please specify one or more GPG key IDs (or --allow-unsigned)')) if task_opts.allow_unsigned and task_opts.skip_unsigned: parser.error(_('allow_unsigned and skip_unsigned are mutually exclusive')) activate_session(session) From a758374fdf10c6de048b35689f73ff4d1be32799 Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Mar 30 2017 13:49:58 +0000 Subject: [PATCH 74/77] deal with missing signatures more correctly --- diff --git a/builder/kojid b/builder/kojid index 0cf2fe4..caf36d2 100755 --- a/builder/kojid +++ b/builder/kojid @@ -5216,7 +5216,23 @@ enabled=1 self.sigmap[rpminfo['id']] = rpminfo['sigkey'] + def pick_key(self, keys, avail_keys): + best = None + best_idx = None + for sigkey in avail_keys: + if sigkey not in keys: + # skip, not a key we are looking for + continue + idx = keys.index(sigkey) + # lower idx (earlier in list) is more preferrable + if best is None or best_idx > idx: + best = sigkey + best_idx = idx + return best + + def make_pkglist(self, tag_id, arch, keys, opts): + # get the rpm data rpms = [] builddirs = {} for a in self.compat[arch] + ('noarch',): @@ -5226,44 +5242,49 @@ enabled=1 for build in builds: builddirs[build['id']] = self.pathinfo.build(build) rpms += list(rpm_iter) - #get build dirs - need = set(['%(name)s-%(version)s-%(release)s.%(arch)s.rpm' % r for r in rpms]) + + # index by id and key + preferred = {} + rpm_idx = {} + for rpminfo in rpms: + sigidx = rpm_idx.setdefault(rpminfo['id'], {}) + sigidx[rpminfo['sigkey']] = rpminfo + + # select our rpms + selected = {} + for rpm_id in rpm_idx: + avail_keys = rpm_idx[rpm_id].keys() + best_key = self.pick_key(keys, avail_keys) + if best_key is None: + # we lack a matching key for this rpm + fallback = avail_keys[0] + rpminfo = rpm_idx[rpm_id][fallback].copy() + rpminfo['sigkey'] = None + selected[rpm_id] = rpminfo + else: + selected[rpm_id] = rpm_idx[rpm_id][best_key] + #generate pkglist files pkgfile = os.path.join(self.repodir, 'pkglist') pkglist = file(pkgfile, 'w') - preferred = {} - rpm_keys = {} - if opts['unsigned']: - keys.append('') # make unsigned rpms the least preferred - for rpminfo in rpms: - if rpminfo['sigkey'] == '' and not opts['unsigned']: - # skip, this is the unsigned rpminfo - continue - fname = '%(name)s-%(version)s-%(release)s.%(arch)s.rpm' % rpminfo - rpm_keys.setdefault(fname, []).append(rpminfo['sigkey']) - if rpminfo['sigkey'] not in keys: - # skip, not a key we are looking for - continue - idx = keys.index(rpminfo['sigkey']) - if rpminfo['id'] in preferred: - if keys.index(preferred[rpminfo['id']]['sigkey']) <= idx: - # key for this is not as preferable as what has been seen - continue - preferred[rpminfo['id']] = rpminfo - seen = set() - fs_missing = set() + fs_missing = [] + sig_missing = [] kojipkgs = {} - for rpminfo in preferred.values(): - if rpminfo['sigkey'] == '': - # we're taking an unsigned rpm (--allow-unsigned) + for rpm_id in selected: + rpminfo = selected[rpm_id] + if rpminfo['sigkey'] is None: + if opts['skip_unsigned']: + continue + sig_missing.append(rpm_id) + # use the primary copy, if allowed (checked below) pkgpath = '%s/%s' % (builddirs[rpminfo['build_id']], self.pathinfo.rpm(rpminfo)) else: + # use the signed copy pkgpath = '%s/%s' % (builddirs[rpminfo['build_id']], self.pathinfo.signed(rpminfo, rpminfo['sigkey'])) - seen.add(os.path.basename(pkgpath)) if not os.path.exists(pkgpath): - fs_missing.add(pkgpath) + fs_missing.append(pkgpath) # we'll raise an error below else: bnp = os.path.basename(pkgpath) @@ -5277,6 +5298,8 @@ enabled=1 kojipkgs[bnp] = rpminfo pkglist.close() self.kojipkgs = kojipkgs + + # report problems if len(fs_missing) > 0: missing_log = os.path.join(self.workdir, 'missing_files.log') outfile = open(missing_log, 'w') @@ -5290,8 +5313,7 @@ enabled=1 self.session.uploadWrapper(missing_log, self.uploadpath) raise koji.GenericError('Packages missing from the filesystem. ' 'See missing_files.log.') - missing = need - seen - if missing: + if sig_missing: # log missing signatures and possibly error missing_log = os.path.join(self.workdir, 'missing_signatures.log') outfile = open(missing_log, 'w') @@ -5301,12 +5323,14 @@ enabled=1 'these files were excluded.\n') outfile.write('Acceptable keys: %r\n\n' % keys) outfile.write('# RPM name: available keys\n') - for fname in sorted(missing): - avail = rpm_keys.get(fname, []) + fmt = '%(name)s-%(version)s-%(release)s.%(arch)s' + filenames = [[fmt % selected[r], r] for r in sig_missing] + for fname, rpm_id in sorted(filenames): + avail = rpm_idx.get(rpm_id, {}).keys() outfile.write('%s: %r\n' % (fname, avail)) outfile.close() self.session.uploadWrapper(missing_log, self.uploadpath) - if not opts['skip_unsigned']: + if not opts['skip_unsigned'] and not opts['unsigned']: raise koji.GenericError('Unsigned packages found. See ' 'missing_signatures.log') return pkgfile From 63029e3d378fc0884173a13efb256a37a3515119 Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Mar 30 2017 13:49:58 +0000 Subject: [PATCH 75/77] fix arg sanity check --- diff --git a/cli/koji b/cli/koji index 958c8c5..a3724f6 100755 --- a/cli/koji +++ b/cli/koji @@ -7123,7 +7123,7 @@ def handle_dist_repo(options, session, args): task_opts, args = parser.parse_args(args) if len(args) < 1: parser.error(_('You must provide a tag to generate the repo from')) - if not task_opts.allow_unsigned: + if len(args) < 2 and not task_opts.allow_unsigned: parser.error(_('Please specify one or more GPG key IDs (or --allow-unsigned)')) if task_opts.allow_unsigned and task_opts.skip_unsigned: parser.error(_('allow_unsigned and skip_unsigned are mutually exclusive')) From 96d5d0f73b4477c9b53775c1de7449c6bc3de170 Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Mar 30 2017 13:49:58 +0000 Subject: [PATCH 76/77] rename some options for clarity --- diff --git a/builder/kojid b/builder/kojid index caf36d2..a9b6a86 100755 --- a/builder/kojid +++ b/builder/kojid @@ -5273,7 +5273,7 @@ enabled=1 for rpm_id in selected: rpminfo = selected[rpm_id] if rpminfo['sigkey'] is None: - if opts['skip_unsigned']: + if opts['skip_missing_signatures']: continue sig_missing.append(rpm_id) # use the primary copy, if allowed (checked below) @@ -5318,8 +5318,8 @@ enabled=1 missing_log = os.path.join(self.workdir, 'missing_signatures.log') outfile = open(missing_log, 'w') outfile.write('Some rpms were missing requested signatures.\n') - if opts['skip_unsigned']: - outfile.write('The skip_unsigned option was specified, so ' + if opts['skip_missing_signatures']: + outfile.write('The skip_missing_signatures option was specified, so ' 'these files were excluded.\n') outfile.write('Acceptable keys: %r\n\n' % keys) outfile.write('# RPM name: available keys\n') @@ -5330,7 +5330,8 @@ enabled=1 outfile.write('%s: %r\n' % (fname, avail)) outfile.close() self.session.uploadWrapper(missing_log, self.uploadpath) - if not opts['skip_unsigned'] and not opts['unsigned']: + if (not opts['skip_missing_signatures'] + and not opts['allow_missing_signatures']): raise koji.GenericError('Unsigned packages found. See ' 'missing_signatures.log') return pkgfile diff --git a/cli/koji b/cli/koji index a3724f6..9ac3bac 100755 --- a/cli/koji +++ b/cli/koji @@ -7095,8 +7095,10 @@ def handle_dist_repo(options, session, args): usage = _("usage: %prog dist-repo [options] tag keyID [keyID...]") usage += _("\n(Specify the --help option for a list of other options)") parser = OptionParser(usage=usage) - parser.add_option('--allow-unsigned', action='store_true', default=False, - help=_('Use unsigned RPMs if none are available with the right key')) + parser.add_option('--allow-missing-signatures', action='store_true', + default=False, + help=_('For RPMs not signed with a desired key, fall back to the ' + 'primary copy')) parser.add_option("--arch", action='append', default=[], help=_("Indicate an architecture to consider. The default is all " + "architectures associated with the given tag. This option may " + @@ -7118,15 +7120,17 @@ def handle_dist_repo(options, session, args): help=_('Do not consider tag inheritance')) parser.add_option("--nowait", action='store_true', default=False, help=_('Do not wait for the task to complete')) - parser.add_option('--skip-unsigned', action='store_true', default=False, + parser.add_option('--skip-missing-signatures', action='store_true', default=False, help=_('Skip RPMs not signed with the desired key(s)')) task_opts, args = parser.parse_args(args) if len(args) < 1: parser.error(_('You must provide a tag to generate the repo from')) - if len(args) < 2 and not task_opts.allow_unsigned: - parser.error(_('Please specify one or more GPG key IDs (or --allow-unsigned)')) - if task_opts.allow_unsigned and task_opts.skip_unsigned: - parser.error(_('allow_unsigned and skip_unsigned are mutually exclusive')) + if len(args) < 2 and not task_opts.allow_missing_signatures: + parser.error(_('Please specify one or more GPG key IDs (or ' + '--allow-missing-signatures)')) + if task_opts.allow_missing_signatures and task_opts.skip_missing_signatures: + parser.error(_('allow_missing_signatures and skip_missing_signatures ' + 'are mutually exclusive')) activate_session(session) stuffdir = _unique_path('cli-dist-repo') if task_opts.comps: @@ -7193,8 +7197,8 @@ def handle_dist_repo(options, session, args): 'inherit': not task_opts.noinherit, 'latest': task_opts.latest, 'multilib': task_opts.multilib, - 'skip_unsigned': task_opts.skip_unsigned, - 'unsigned': task_opts.allow_unsigned + 'skip_missing_signatures': task_opts.skip_missing_signatures, + 'allow_missing_signatures': task_opts.allow_missing_signatures } task_id = session.distRepo(tag, keys, **opts) print("Creating dist repo for tag " + tag) From dad02ba6adf9bdd71ead4055d7185b93ddc7e1d9 Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Mar 30 2017 13:54:28 +0000 Subject: [PATCH 77/77] record missing signatures even if skipping them --- diff --git a/builder/kojid b/builder/kojid index a9b6a86..6f3837a 100755 --- a/builder/kojid +++ b/builder/kojid @@ -5273,9 +5273,9 @@ enabled=1 for rpm_id in selected: rpminfo = selected[rpm_id] if rpminfo['sigkey'] is None: + sig_missing.append(rpm_id) if opts['skip_missing_signatures']: continue - sig_missing.append(rpm_id) # use the primary copy, if allowed (checked below) pkgpath = '%s/%s' % (builddirs[rpminfo['build_id']], self.pathinfo.rpm(rpminfo))