From 2b7447ef3232a562543f134804b37c8ce98e5f6f Mon Sep 17 00:00:00 2001 From: Mike McLean Date: May 03 2018 20:05:16 +0000 Subject: [PATCH 1/36] misc cleanup --- diff --git a/builder/kojid b/builder/kojid index 42d6269..c94ff04 100755 --- a/builder/kojid +++ b/builder/kojid @@ -5139,10 +5139,9 @@ class NewDistRepoTask(BaseTaskHandler): def handler(self, tag, repo_id, keys, task_opts): tinfo = self.session.getTag(tag, strict=True, event=task_opts['event']) - path = koji.pathinfo.distrepo(repo_id, tinfo['name']) if len(task_opts['arch']) == 0: - arches = tinfo['arches'] or '' - task_opts['arch'] = arches.split() + arches = tinfo['arches'] or '' + task_opts['arch'] = arches.split() if len(task_opts['arch']) == 0: raise koji.GenericError('No arches specified nor for the tag!') subtasks = {} @@ -5231,7 +5230,7 @@ class createDistRepoTask(CreaterepoTask): #set up our output dir self.repodir = '%s/repo' % self.workdir koji.ensuredir(self.repodir) - self.outdir = self.repodir # workaround create_local_repo use + self.outdir = self.repodir # workaround create_local_repo use self.datadir = '%s/repodata' % self.repodir self.sigmap = {} oldpkgs = [] @@ -5280,7 +5279,7 @@ class createDistRepoTask(CreaterepoTask): pathinfo = koji.PathInfo(self.options.topdir) repodir = pathinfo.distrepo(self.rinfo['id'], self.rinfo['tag_name']) mldir = os.path.join(repodir, koji.canonArch(ml_arch)) - ml_true = set() # multilib packages we need to include before depsolve + ml_true = set() # multilib packages we need to include before depsolve ml_conf = os.path.join(self.pathinfo.work(), conf) # step 1: figure out which packages are multilib (should already exist) @@ -5444,7 +5443,6 @@ enabled=1 rpms += list(rpm_iter) # index by id and key - preferred = {} rpm_idx = {} for rpminfo in rpms: sigidx = rpm_idx.setdefault(rpminfo['id'], {}) From c23210374d6a219159cc369aed06c3fc48c704ae Mon Sep 17 00:00:00 2001 From: Mike McLean Date: May 03 2018 20:05:16 +0000 Subject: [PATCH 2/36] hub: use UpdateProcessor in repo_expire_older() --- diff --git a/hub/kojihub.py b/hub/kojihub.py index 3f281d1..e1bfe07 100644 --- a/hub/kojihub.py +++ b/hub/kojihub.py @@ -2550,15 +2550,19 @@ def repo_delete(repo_id): repo_set_state(repo_id, koji.REPO_DELETED) return len(references) + def repo_expire_older(tag_id, event_id): """Expire repos for tag older than event""" st_ready = koji.REPO_READY - st_expired = koji.REPO_EXPIRED - q = """UPDATE repo SET state=%(st_expired)i - WHERE tag_id = %(tag_id)i - AND create_event < %(event_id)i - AND state = %(st_ready)i""" - _dml(q, locals()) + update = UpdateProcessor( + 'repo', + clauses=['tag_id = %(tag_id)s', + 'create_event < %(event_id)s', + 'state = %(st_ready)s'], + values=locals()) + update.set(state=koji.REPO_EXPIRED) + update.execute() + def repo_references(repo_id): """Return a list of buildroots that reference the repo""" From 72955e7dd54f388cc86c37d312e15b921eb4f1e5 Mon Sep 17 00:00:00 2001 From: Mike McLean Date: May 03 2018 20:05:16 +0000 Subject: [PATCH 3/36] unit test for repo_expire_older() --- diff --git a/tests/test_hub/test_repos.py b/tests/test_hub/test_repos.py new file mode 100644 index 0000000..b8cd801 --- /dev/null +++ b/tests/test_hub/test_repos.py @@ -0,0 +1,57 @@ +import mock +import unittest + +import koji +import kojihub + + +QP = kojihub.QueryProcessor +IP = kojihub.InsertProcessor +UP = kojihub.UpdateProcessor + + +class TestRepoFunctions(unittest.TestCase): + + def setUp(self): + self.QueryProcessor = mock.patch('kojihub.QueryProcessor', + side_effect=self.getQuery).start() + self.queries = [] + self.InsertProcessor = mock.patch('kojihub.InsertProcessor', + side_effect=self.getInsert).start() + self.inserts = [] + self.UpdateProcessor = mock.patch('kojihub.UpdateProcessor', + side_effect=self.getUpdate).start() + self.updates = [] + self._dml = mock.patch('kojihub._dml').start() + + def tearDown(self): + mock.patch.stopall() + + def getQuery(self, *args, **kwargs): + query = QP(*args, **kwargs) + query.execute = mock.MagicMock() + self.queries.append(query) + return query + + def getInsert(self, *args, **kwargs): + insert = IP(*args, **kwargs) + insert.execute = mock.MagicMock() + self.inserts.append(insert) + return insert + + def getUpdate(self, *args, **kwargs): + update = UP(*args, **kwargs) + update.execute = mock.MagicMock() + self.updates.append(update) + return update + + def test_repo_expire_older(self): + kojihub.repo_expire_older(mock.sentinel.tag_id, mock.sentinel.event_id) + self.assertEqual(len(self.updates), 1) + update = self.updates[0] + query = str(update) + self.assertEqual(update.table, 'repo') + self.assertEqual(update.data, {'state': koji.REPO_EXPIRED}) + self.assertEqual(update.rawdata, {}) + self.assertEqual(update.values, {'event_id': mock.sentinel.event_id, + 'st_ready': koji.REPO_READY, 'tag_id': mock.sentinel.tag_id}) From 0fe113096cf5b4b960d5a48defaa8e320bc375a7 Mon Sep 17 00:00:00 2001 From: Mike McLean Date: May 03 2018 20:05:16 +0000 Subject: [PATCH 4/36] only expire older repos of matching type Fixes: https://pagure.io/koji/issue/457 --- diff --git a/hub/kojihub.py b/hub/kojihub.py index e1bfe07..7ea1df1 100644 --- a/hub/kojihub.py +++ b/hub/kojihub.py @@ -2551,15 +2551,19 @@ def repo_delete(repo_id): return len(references) -def repo_expire_older(tag_id, event_id): - """Expire repos for tag older than event""" +def repo_expire_older(tag_id, event_id, dist=None): + """Expire repos for tag older than event + + If dist is not None, then only expire repos with the given dist value + """ st_ready = koji.REPO_READY - update = UpdateProcessor( - 'repo', - clauses=['tag_id = %(tag_id)s', - 'create_event < %(event_id)s', - 'state = %(st_ready)s'], - values=locals()) + clauses=['tag_id = %(tag_id)s', + 'create_event < %(event_id)s', + 'state = %(st_ready)s'] + if dist is not None: + dist = bool(dist) + clauses.append('dist = %(dist)s') + update = UpdateProcessor('repo', values=locals(), clauses=clauses) update.set(state=koji.REPO_EXPIRED) update.execute() @@ -12609,7 +12613,7 @@ class HostExports(object): return #else: repo_ready(repo_id) - repo_expire_older(rinfo['tag_id'], rinfo['create_event']) + repo_expire_older(rinfo['tag_id'], rinfo['create_event'], rinfo['dist']) #make a latest link if rinfo['dist']: diff --git a/tests/test_hub/test_repos.py b/tests/test_hub/test_repos.py index b8cd801..8bda536 100644 --- a/tests/test_hub/test_repos.py +++ b/tests/test_hub/test_repos.py @@ -49,9 +49,27 @@ class TestRepoFunctions(unittest.TestCase): kojihub.repo_expire_older(mock.sentinel.tag_id, mock.sentinel.event_id) self.assertEqual(len(self.updates), 1) update = self.updates[0] - query = str(update) self.assertEqual(update.table, 'repo') self.assertEqual(update.data, {'state': koji.REPO_EXPIRED}) self.assertEqual(update.rawdata, {}) - self.assertEqual(update.values, {'event_id': mock.sentinel.event_id, - 'st_ready': koji.REPO_READY, 'tag_id': mock.sentinel.tag_id}) + self.assertEqual(update.values['event_id'], mock.sentinel.event_id) + self.assertEqual(update.values['tag_id'], mock.sentinel.tag_id) + self.assertEqual(update.values['dist'], None) + if 'dist = %(dist)s' in update.clauses: + raise Exception('Unexpected dist condition') + + # and with dist specified + for dist in True, False: + self.updates = [] + kojihub.repo_expire_older(mock.sentinel.tag_id, mock.sentinel.event_id, + dist=dist) + self.assertEqual(len(self.updates), 1) + update = self.updates[0] + self.assertEqual(update.table, 'repo') + self.assertEqual(update.data, {'state': koji.REPO_EXPIRED}) + self.assertEqual(update.rawdata, {}) + self.assertEqual(update.values['event_id'], mock.sentinel.event_id) + self.assertEqual(update.values['tag_id'], mock.sentinel.tag_id) + self.assertEqual(update.values['dist'], dist) + if 'dist = %(dist)s' not in update.clauses: + raise Exception('Missing dist condition') From eba0c85a93bf5d9e14e491a8f121a921cdebe8c4 Mon Sep 17 00:00:00 2001 From: Mike McLean Date: May 03 2018 20:05:16 +0000 Subject: [PATCH 5/36] make postRepoInit data consistent with preRepoInit data for dist repos --- diff --git a/hub/kojihub.py b/hub/kojihub.py index 7ea1df1..31622d5 100644 --- a/hub/kojihub.py +++ b/hub/kojihub.py @@ -2493,7 +2493,9 @@ def dist_repo_init(tag, keys, task_opts): task_opts['comps']), groupsdir + '/comps.xml') # note: we need to match args from the other postRepoInit callback koji.plugin.run_callbacks('postRepoInit', tag=tinfo, with_src=False, - with_debuginfo=False, event=event, repo_id=repo_id) + with_debuginfo=False, event=event, repo_id=repo_id, + dist=True, keys=keys, arches=arches, task_opts=task_opts, + repodir=repodir) return repo_id, event From 534d8db8ec50b2e2a2a0360041d4364b404f928a Mon Sep 17 00:00:00 2001 From: Mike McLean Date: May 03 2018 20:05:16 +0000 Subject: [PATCH 6/36] add some comments --- diff --git a/builder/kojid b/builder/kojid index c94ff04..4cd547f 100755 --- a/builder/kojid +++ b/builder/kojid @@ -5227,12 +5227,15 @@ class createDistRepoTask(CreaterepoTask): groupdata = os.path.join( self.pathinfo.distrepo(repo_id, self.rinfo['tag_name']), 'groups', 'comps.xml') - #set up our output dir + + # set up our output dir self.repodir = '%s/repo' % self.workdir koji.ensuredir(self.repodir) self.outdir = self.repodir # workaround create_local_repo use self.datadir = '%s/repodata' % self.repodir self.sigmap = {} + + # gather oldpkgs data if delta option in use oldpkgs = [] if opts.get('delta'): # should be a list of repo ids to delta against @@ -5246,6 +5249,8 @@ class createDistRepoTask(CreaterepoTask): if not os.path.exists(path): raise koji.GenericError('Base drpm repo missing: %s' % path) oldpkgs.append(path) + + # sort out our package list self.uploadpath = self.getUploadDir() self.pkglist = self.make_pkglist(tag, arch, keys, opts) if opts['multilib'] and rpmUtils.arch.isMultiLibArch(arch): @@ -5256,11 +5261,15 @@ class createDistRepoTask(CreaterepoTask): os.path.basename(self.pkglist)) if os.path.getsize(self.pkglist) == 0: self.pkglist = None + + # generate the repodata self.create_local_repo(self.rinfo, arch, self.pkglist, groupdata, None, oldpkgs=oldpkgs) if self.pkglist is None: fo = file(os.path.join(self.datadir, "EMPTY_REPO"), 'w') fo.write("This repo is empty because its tag has no content for this arch\n") fo.close() + + # upload repo files files = ['pkglist', 'kojipkgs'] for f in os.listdir(self.datadir): files.append(f) @@ -5272,6 +5281,7 @@ class createDistRepoTask(CreaterepoTask): files.append(f) self.session.uploadWrapper('%s/%s' % (ddir, f), self.uploadpath, f) + return [self.uploadpath, files, self.sigmap.items()] def do_multilib(self, arch, ml_arch, conf): From e28b7a80c8e7b59d610607a8b0dd8c102bb086de Mon Sep 17 00:00:00 2001 From: Mike McLean Date: May 03 2018 20:05:16 +0000 Subject: [PATCH 7/36] avoid passing a set as a callback param --- diff --git a/hub/kojihub.py b/hub/kojihub.py index 31622d5..d1ba9bf 100644 --- a/hub/kojihub.py +++ b/hub/kojihub.py @@ -2470,7 +2470,7 @@ def dist_repo_init(tag, keys, task_opts): tinfo = get_tag(tag, strict=True) tag_id = tinfo['id'] event = task_opts.get('event') - arches = set([koji.canonArch(a) for a in task_opts['arch']]) + arches = list(set([koji.canonArch(a) for a in task_opts['arch']])) # note: we need to match args from the other preRepoInit callback koji.plugin.run_callbacks('preRepoInit', tag=tinfo, with_src=False, with_debuginfo=False, event=event, repo_id=None, From 651f5e05d5be8632b3e4470b3f08bba0f73471dc Mon Sep 17 00:00:00 2001 From: Mike McLean Date: May 03 2018 20:05:16 +0000 Subject: [PATCH 8/36] short option -a/--arch for dist-repo command --- diff --git a/cli/koji_cli/commands.py b/cli/koji_cli/commands.py index ef0ceac..2b287cd 100644 --- a/cli/koji_cli/commands.py +++ b/cli/koji_cli/commands.py @@ -6911,7 +6911,7 @@ def handle_dist_repo(options, session, args): default=False, help=_('For RPMs not signed with a desired key, fall back to the ' 'primary copy')) - parser.add_option("--arch", action='append', default=[], + parser.add_option("-a", "--arch", action='append', default=[], help=_("Indicate an architecture to consider. The default is all " + "architectures associated with the given tag. This option may " + "be specified multiple times.")) From a2d5f63b6068452511ba10980629319a3c6a50c5 Mon Sep 17 00:00:00 2001 From: Mike McLean Date: May 03 2018 20:05:16 +0000 Subject: [PATCH 9/36] avoid including noarch in src repos --- diff --git a/builder/kojid b/builder/kojid index 4cd547f..b36c2c9 100755 --- a/builder/kojid +++ b/builder/kojid @@ -5444,7 +5444,8 @@ enabled=1 # get the rpm data rpms = [] builddirs = {} - for a in self.compat[arch] + ('noarch',): + for a in self.compat[arch]: + # note: self.compat includes noarch for non-src already rpm_iter, builds = self.session.listTaggedRPMS(tag_id, event=opts['event'], arch=a, latest=opts['latest'], inherit=opts['inherit'], rpmsigs=True) From 10e1963f758ed7b77ea44a65c007f2b23ecacfc4 Mon Sep 17 00:00:00 2001 From: Mike McLean Date: May 03 2018 20:05:16 +0000 Subject: [PATCH 10/36] dist-repo --with-src option --- diff --git a/cli/koji_cli/commands.py b/cli/koji_cli/commands.py index 2b287cd..3c5206e 100644 --- a/cli/koji_cli/commands.py +++ b/cli/koji_cli/commands.py @@ -6915,6 +6915,7 @@ def handle_dist_repo(options, session, args): help=_("Indicate an architecture to consider. The default is all " + "architectures associated with the given tag. This option may " + "be specified multiple times.")) + parser.add_option("--with-src", action='store_true', help='Also generate a src repo') parser.add_option('--comps', help='Include a comps file in the repodata') parser.add_option('--delta-rpms', metavar='REPO',default=[], action='append', @@ -6996,9 +6997,10 @@ def handle_dist_repo(options, session, args): task_opts.multilib = os.path.join(stuffdir, os.path.basename(task_opts.multilib)) print('') - for f in ('noarch', 'src'): - if f in task_opts.arch: - task_opts.arch.remove(f) + if 'noarch' in task_opts.arch: + task_opts.arch.remove('noarch') + if task_opts.with_src and 'src' not in task_opts.arch: + task_opts.arch.append('src') opts = { 'arch': task_opts.arch, 'comps': task_opts.comps, From 8fd08cce7e244aa62472c338bbf4a2a23a23ce8c Mon Sep 17 00:00:00 2001 From: Mike McLean Date: May 03 2018 20:05:16 +0000 Subject: [PATCH 11/36] disconnect createdistrepo and createrepo task handlers --- diff --git a/builder/kojid b/builder/kojid index b36c2c9..9e22fec 100755 --- a/builder/kojid +++ b/builder/kojid @@ -5055,7 +5055,7 @@ class CreaterepoTask(BaseTaskHandler): self.session.uploadWrapper('%s/%s' % (self.datadir, f), uploadpath, f) return [uploadpath, files] - def create_local_repo(self, rinfo, arch, pkglist, groupdata, oldrepo, oldpkgs=None): + def create_local_repo(self, rinfo, arch, pkglist, groupdata, oldrepo): koji.ensuredir(self.outdir) if self.options.use_createrepo_c: cmd = ['/usr/bin/createrepo_c'] @@ -5082,11 +5082,6 @@ class CreaterepoTask(BaseTaskHandler): cmd.append('--update') if self.options.createrepo_skip_stat: cmd.append('--skip-stat') - if oldpkgs: - # generate delta-rpms - cmd.append('--deltas') - for op_dir in oldpkgs: - cmd.extend(['--oldpackagedirs', op_dir]) # note: we can't easily use a cachedir because we do not have write # permission. The good news is that with --update we won't need to # be scanning many rpms. @@ -5192,7 +5187,7 @@ class NewDistRepoTask(BaseTaskHandler): return 'Dist repository #%s successfully generated' % repo_id -class createDistRepoTask(CreaterepoTask): +class createDistRepoTask(BaseTaskHandler): Methods = ['createdistrepo'] _taskWeight = 1.5 @@ -5263,7 +5258,7 @@ class createDistRepoTask(CreaterepoTask): self.pkglist = None # generate the repodata - self.create_local_repo(self.rinfo, arch, self.pkglist, groupdata, None, oldpkgs=oldpkgs) + self.do_createrepo(self.rinfo, arch, self.pkglist, groupdata, oldpkgs=oldpkgs) if self.pkglist is None: fo = file(os.path.join(self.datadir, "EMPTY_REPO"), 'w') fo.write("This repo is empty because its tag has no content for this arch\n") @@ -5284,6 +5279,42 @@ class createDistRepoTask(CreaterepoTask): return [self.uploadpath, files, self.sigmap.items()] + def do_createrepo(self, rinfo, arch, pkglist, groupdata, oldpkgs=None): + """Run createrepo + + This is derived from CreaterepoTask.create_local_repo, but adapted to + our requirements here + """ + koji.ensuredir(self.outdir) + if self.options.use_createrepo_c: + cmd = ['/usr/bin/createrepo_c'] + else: + cmd = ['/usr/bin/createrepo'] + cmd.extend(['-vd', '-o', self.outdir]) + if pkglist is not None: + cmd.extend(['-i', pkglist]) + if os.path.isfile(groupdata): + cmd.extend(['-g', groupdata]) + # TODO: can we recycle data (with --update) as in create_local_repo? + if oldpkgs: + # generate delta-rpms + cmd.append('--deltas') + for op_dir in oldpkgs: + cmd.extend(['--oldpackagedirs', op_dir]) + # note: we can't easily use a cachedir because we do not have write + # permission. The good news is that with --update we won't need to + # be scanning many rpms. + if pkglist is None: + cmd.append(self.outdir) + else: + cmd.append(self.repodir) + + logfile = '%s/createrepo.log' % self.workdir + status = log_output(self.session, cmd[0], cmd, logfile, self.getUploadDir(), logerror=True) + if not isSuccess(status): + raise koji.GenericError('failed to create repo: %s' \ + % parseStatus(status, ' '.join(cmd))) + def do_multilib(self, arch, ml_arch, conf): self.repo_id = self.rinfo['id'] pathinfo = koji.PathInfo(self.options.topdir) From 01a47750d10ec7a8d43f0f57d09f626f714d481c Mon Sep 17 00:00:00 2001 From: Mike McLean Date: May 03 2018 20:05:16 +0000 Subject: [PATCH 12/36] simplify do_createrepo a bit --- diff --git a/builder/kojid b/builder/kojid index 9e22fec..7c552c7 100755 --- a/builder/kojid +++ b/builder/kojid @@ -5258,8 +5258,9 @@ class createDistRepoTask(BaseTaskHandler): self.pkglist = None # generate the repodata - self.do_createrepo(self.rinfo, arch, self.pkglist, groupdata, oldpkgs=oldpkgs) + self.do_createrepo(self.repodir, self.pkglist, groupdata, oldpkgs=oldpkgs) if self.pkglist is None: + # can this happen? fo = file(os.path.join(self.datadir, "EMPTY_REPO"), 'w') fo.write("This repo is empty because its tag has no content for this arch\n") fo.close() @@ -5279,20 +5280,18 @@ class createDistRepoTask(BaseTaskHandler): return [self.uploadpath, files, self.sigmap.items()] - def do_createrepo(self, rinfo, arch, pkglist, groupdata, oldpkgs=None): + def do_createrepo(self, repodir, pkglist, groupdata, oldpkgs=None, logname=None): """Run createrepo This is derived from CreaterepoTask.create_local_repo, but adapted to our requirements here """ - koji.ensuredir(self.outdir) + koji.ensuredir(repodir) if self.options.use_createrepo_c: cmd = ['/usr/bin/createrepo_c'] else: cmd = ['/usr/bin/createrepo'] - cmd.extend(['-vd', '-o', self.outdir]) - if pkglist is not None: - cmd.extend(['-i', pkglist]) + cmd.extend(['-vd', '-i', pkglist]) if os.path.isfile(groupdata): cmd.extend(['-g', groupdata]) # TODO: can we recycle data (with --update) as in create_local_repo? @@ -5301,15 +5300,11 @@ class createDistRepoTask(BaseTaskHandler): cmd.append('--deltas') for op_dir in oldpkgs: cmd.extend(['--oldpackagedirs', op_dir]) - # note: we can't easily use a cachedir because we do not have write - # permission. The good news is that with --update we won't need to - # be scanning many rpms. - if pkglist is None: - cmd.append(self.outdir) - else: - cmd.append(self.repodir) + cmd.append(repodir) - logfile = '%s/createrepo.log' % self.workdir + if logname is None: + logname = 'createrepo' + logfile = '%s/%s.log' % (self.workdir, logname) status = log_output(self.session, cmd[0], cmd, logfile, self.getUploadDir(), logerror=True) if not isSuccess(status): raise koji.GenericError('failed to create repo: %s' \ From df0a6b28fa3ca82f777b2a60482b4c626c6b7cd2 Mon Sep 17 00:00:00 2001 From: Mike McLean Date: May 03 2018 20:05:16 +0000 Subject: [PATCH 13/36] attempt to handle empty tag more gracefully --- diff --git a/builder/kojid b/builder/kojid index 7c552c7..765feee 100755 --- a/builder/kojid +++ b/builder/kojid @@ -5254,13 +5254,10 @@ class createDistRepoTask(BaseTaskHandler): self.logger.debug('package list is %s' % self.pkglist) self.session.uploadWrapper(self.pkglist, self.uploadpath, os.path.basename(self.pkglist)) - if os.path.getsize(self.pkglist) == 0: - self.pkglist = None # generate the repodata self.do_createrepo(self.repodir, self.pkglist, groupdata, oldpkgs=oldpkgs) - if self.pkglist is None: - # can this happen? + if os.path.getsize(self.pkglist) == 0: fo = file(os.path.join(self.datadir, "EMPTY_REPO"), 'w') fo.write("This repo is empty because its tag has no content for this arch\n") fo.close() From e128d7596efb080558f295401500ae0910607cc9 Mon Sep 17 00:00:00 2001 From: Mike McLean Date: May 03 2018 20:05:16 +0000 Subject: [PATCH 14/36] cli: dist-repo --split-debuginfo option --- diff --git a/cli/koji_cli/commands.py b/cli/koji_cli/commands.py index 3c5206e..7ef15ba 100644 --- a/cli/koji_cli/commands.py +++ b/cli/koji_cli/commands.py @@ -6916,6 +6916,8 @@ def handle_dist_repo(options, session, args): "architectures associated with the given tag. This option may " + "be specified multiple times.")) parser.add_option("--with-src", action='store_true', help='Also generate a src repo') + parser.add_option("--split-debuginfo", action='store_true', default=False, + help='Split debuginfo info a separate repo for each arch') parser.add_option('--comps', help='Include a comps file in the repodata') parser.add_option('--delta-rpms', metavar='REPO',default=[], action='append', @@ -7009,6 +7011,7 @@ def handle_dist_repo(options, session, args): 'inherit': not task_opts.noinherit, 'latest': task_opts.latest, 'multilib': task_opts.multilib, + 'split_debuginfo': task_opts.split_debuginfo, 'skip_missing_signatures': task_opts.skip_missing_signatures, 'allow_missing_signatures': task_opts.allow_missing_signatures } From 7862bd94d0ca886552ec6650cea7d9fac42faede Mon Sep 17 00:00:00 2001 From: Mike McLean Date: May 03 2018 20:05:16 +0000 Subject: [PATCH 15/36] drop unused var --- diff --git a/builder/kojid b/builder/kojid index 765feee..7733032 100755 --- a/builder/kojid +++ b/builder/kojid @@ -5217,7 +5217,6 @@ class createDistRepoTask(BaseTaskHandler): self.rinfo = self.session.repoInfo(repo_id, strict=True) if self.rinfo['state'] != koji.REPO_INIT: raise koji.GenericError("Repo %(id)s not in INIT state (got %(state)s)" % self.rinfo) - self.repo_id = self.rinfo['id'] self.pathinfo = koji.PathInfo(self.options.topdir) groupdata = os.path.join( self.pathinfo.distrepo(repo_id, self.rinfo['tag_name']), @@ -5308,7 +5307,6 @@ class createDistRepoTask(BaseTaskHandler): % parseStatus(status, ' '.join(cmd))) def do_multilib(self, arch, ml_arch, conf): - self.repo_id = self.rinfo['id'] pathinfo = koji.PathInfo(self.options.topdir) repodir = pathinfo.distrepo(self.rinfo['id'], self.rinfo['tag_name']) mldir = os.path.join(repodir, koji.canonArch(ml_arch)) From 356a64386e617f06e803272aac922dd90d73445e Mon Sep 17 00:00:00 2001 From: Mike McLean Date: May 03 2018 20:05:16 +0000 Subject: [PATCH 16/36] standardize pathinfo usage in createDistRepoTask --- diff --git a/builder/kojid b/builder/kojid index 7733032..c7fb6f6 100755 --- a/builder/kojid +++ b/builder/kojid @@ -5217,9 +5217,8 @@ class createDistRepoTask(BaseTaskHandler): self.rinfo = self.session.repoInfo(repo_id, strict=True) if self.rinfo['state'] != koji.REPO_INIT: raise koji.GenericError("Repo %(id)s not in INIT state (got %(state)s)" % self.rinfo) - self.pathinfo = koji.PathInfo(self.options.topdir) groupdata = os.path.join( - self.pathinfo.distrepo(repo_id, self.rinfo['tag_name']), + koji.pathinfo.distrepo(repo_id, self.rinfo['tag_name']), 'groups', 'comps.xml') # set up our output dir @@ -5307,11 +5306,10 @@ class createDistRepoTask(BaseTaskHandler): % parseStatus(status, ' '.join(cmd))) def do_multilib(self, arch, ml_arch, conf): - pathinfo = koji.PathInfo(self.options.topdir) - repodir = pathinfo.distrepo(self.rinfo['id'], self.rinfo['tag_name']) + repodir = koji.pathinfo.distrepo(self.rinfo['id'], self.rinfo['tag_name']) mldir = os.path.join(repodir, koji.canonArch(ml_arch)) ml_true = set() # multilib packages we need to include before depsolve - ml_conf = os.path.join(self.pathinfo.work(), conf) + ml_conf = os.path.join(koji.pathinfo.work(), conf) # step 1: figure out which packages are multilib (should already exist) mlm = multilib.DevelMultilibMethod(ml_conf) @@ -5471,7 +5469,7 @@ enabled=1 event=opts['event'], arch=a, latest=opts['latest'], inherit=opts['inherit'], rpmsigs=True) for build in builds: - builddirs[build['id']] = self.pathinfo.build(build) + builddirs[build['id']] = koji.pathinfo.build(build) rpms += list(rpm_iter) # index by id and key @@ -5508,11 +5506,11 @@ enabled=1 continue # use the primary copy, if allowed (checked below) pkgpath = '%s/%s' % (builddirs[rpminfo['build_id']], - self.pathinfo.rpm(rpminfo)) + koji.pathinfo.rpm(rpminfo)) else: # use the signed copy pkgpath = '%s/%s' % (builddirs[rpminfo['build_id']], - self.pathinfo.signed(rpminfo, rpminfo['sigkey'])) + koji.pathinfo.signed(rpminfo, rpminfo['sigkey'])) if not os.path.exists(pkgpath): fs_missing.append(pkgpath) # we'll raise an error below From 8158613e4a2170bb9ab45b5185ba5fa6db9d027d Mon Sep 17 00:00:00 2001 From: Mike McLean Date: May 03 2018 20:05:16 +0000 Subject: [PATCH 17/36] trying to separate pkglist calculation from output generation --- diff --git a/builder/kojid b/builder/kojid index c7fb6f6..23fab62 100755 --- a/builder/kojid +++ b/builder/kojid @@ -5245,7 +5245,9 @@ class createDistRepoTask(BaseTaskHandler): # sort out our package list self.uploadpath = self.getUploadDir() - self.pkglist = self.make_pkglist(tag, arch, keys, opts) + self.make_pkglist(tag, arch, keys, opts) + self.pkglist = self.write_pkglist() + self.link_pkgs() if opts['multilib'] and rpmUtils.arch.isMultiLibArch(arch): self.do_multilib(arch, self.archmap[arch], opts['multilib']) self.write_kojipkgs() @@ -5458,7 +5460,6 @@ enabled=1 best_idx = idx return best - def make_pkglist(self, tag_id, arch, keys, opts): # get the rpm data rpms = [] @@ -5492,9 +5493,7 @@ enabled=1 else: selected[rpm_id] = rpm_idx[rpm_id][best_key] - #generate pkglist files - pkgfile = os.path.join(self.repodir, 'pkglist') - pkglist = file(pkgfile, 'w') + # generate kojipkgs data and note missing files fs_missing = [] sig_missing = [] kojipkgs = {} @@ -5516,15 +5515,9 @@ enabled=1 # we'll raise an error below else: bnp = os.path.basename(pkgpath) - bnplet = bnp[0].lower() - pkglist.write(bnplet + '/' + bnp + '\n') - koji.ensuredir(os.path.join(self.repodir, bnplet)) self.sigmap[rpminfo['id']] = rpminfo['sigkey'] - dst = os.path.join(self.repodir, bnplet, bnp) - self.logger.debug("os.symlink(%r, %r(", pkgpath, dst) - os.symlink(pkgpath, dst) + rpminfo['_pkgpath'] = pkgpath kojipkgs[bnp] = rpminfo - pkglist.close() self.kojipkgs = kojipkgs # report problems @@ -5562,8 +5555,24 @@ enabled=1 and not opts['allow_missing_signatures']): raise koji.GenericError('Unsigned packages found. See ' 'missing_signatures.log') - return pkgfile + def link_pkgs(self): + for bnp in self.kojipkgs: + bnplet = bnp[0].lower() + koji.ensuredir(os.path.join(self.repodir, bnplet)) + dst = os.path.join(self.repodir, bnplet, bnp) + pkgpath = self.kojipkgs[bnp]['_pkgpath'] + self.logger.debug("os.symlink(%r, %r(", pkgpath, dst) + os.symlink(pkgpath, dst) + + def write_pkglist(self): + pkgfile = os.path.join(self.repodir, 'pkglist') + pkglist = file(pkgfile, 'w') + for bnp in self.kojipkgs: + bnplet = bnp[0].lower() + pkglist.write(bnplet + '/' + bnp + '\n') + pkglist.close() + return pkgfile def write_kojipkgs(self): filename = os.path.join(self.repodir, 'kojipkgs') From 6e3482ea2290e39cf794171d42fd812b51e1ece2 Mon Sep 17 00:00:00 2001 From: Mike McLean Date: May 03 2018 20:05:16 +0000 Subject: [PATCH 18/36] trying to use kojipkgs data instead of fs lookups --- diff --git a/builder/kojid b/builder/kojid index 23fab62..5caa09b 100755 --- a/builder/kojid +++ b/builder/kojid @@ -5313,26 +5313,29 @@ class createDistRepoTask(BaseTaskHandler): ml_true = set() # multilib packages we need to include before depsolve ml_conf = os.path.join(koji.pathinfo.work(), conf) + # read pkgs data from multilib repo + ml_pkgfile = os.path.join(mldir, 'kojipkgs') + ml_pkgs = json.load(open(ml_pkgfile, 'r')) + # step 1: figure out which packages are multilib (should already exist) mlm = multilib.DevelMultilibMethod(ml_conf) fs_missing = set() - with open(self.pkglist) as pkglist: - for pkg in pkglist: - ppath = os.path.join(self.repodir, pkg.strip()) + for bnp in self.kojipkgs: + rpminfo = self.kojipkgs[bnp] + ppath = rpminfo['_pkgpath'] po = yum.packages.YumLocalPackage(filename=ppath) - if mlm.select(po) and arch in self.archmap: + if mlm.select(po): # we need a multilib package to be included - # we assume the same signature level is available - # XXX: what is a subarchitecture is the right answer? - pl_path = pkg.replace(arch, self.archmap[arch]).strip() - # assume this exists in the task results for the ml arch - real_path = os.path.join(mldir, pl_path) - if not os.path.exists(real_path): - self.logger.error('%s (multilib) is not on the filesystem' % real_path) - fs_missing.add(real_path) + ml_bnp = bnp.replace(arch, self.archmap[arch]) + ml_path = os.path.join(mldir, ml_bnp[0].lower(), ml_bnp) + # ^ XXX - should actually generate this + if ml_bnp not in ml_pkgs: + # not in our multilib repo + self.logger.error('%s (multilib) is not on the filesystem' % ml_path) + fs_missing.add(ml_path) # we defer failure so can report all the missing deps continue - ml_true.add(real_path) + ml_true.add(ml_path) # step 2: set up architectures for yum configuration self.logger.info("Resolving multilib for %s using method devel" % arch) @@ -5422,10 +5425,6 @@ enabled=1 raise koji.GenericError('multilib packages missing. ' 'See missing_multilib.log') - # get rpm ids for ml pkgs - kpkgfile = os.path.join(mldir, 'kojipkgs') - kojipkgs = json.load(open(kpkgfile, 'r')) - # step 5: add dependencies to our package list pkgwriter = open(self.pkglist, 'a') for dep_path in ml_needed: @@ -5442,7 +5441,7 @@ enabled=1 pkgwriter.write(bnplet + '/' + bnp + '\n') self.logger.debug("os.symlink(%r, %r)", dep_path, dst) os.symlink(dep_path, dst) - rpminfo = kojipkgs[bnp] + rpminfo = ml_pkgs[bnp] self.sigmap[rpminfo['id']] = rpminfo['sigkey'] From a93554139e58936e109c49d21a8527d8fde06a50 Mon Sep 17 00:00:00 2001 From: Mike McLean Date: May 03 2018 20:05:16 +0000 Subject: [PATCH 19/36] update unit test for new options --- diff --git a/tests/test_cli/test_dist_repo.py b/tests/test_cli/test_dist_repo.py index 9b176e5..9a55403 100644 --- a/tests/test_cli/test_dist_repo.py +++ b/tests/test_cli/test_dist_repo.py @@ -252,9 +252,11 @@ Options: --allow-missing-signatures For RPMs not signed with a desired key, fall back to the primary copy - --arch=ARCH Indicate an architecture to consider. The default is + -a ARCH, --arch=ARCH Indicate an architecture to consider. The default is all architectures associated with the given tag. This option may be specified multiple times. + --with-src Also generate a src repo + --split-debuginfo Split debuginfo info a separate repo for each arch --comps=COMPS Include a comps file in the repodata --delta-rpms=REPO Create delta rpms. REPO can be the id of another dist repo or the name of a tag that has a dist repo. May be From f26eb7560cf8c47120a9f1d4166624c8dfbacbff Mon Sep 17 00:00:00 2001 From: Mike McLean Date: May 03 2018 20:05:16 +0000 Subject: [PATCH 20/36] actually do the debuginfo split --- diff --git a/builder/kojid b/builder/kojid index 5caa09b..cead161 100755 --- a/builder/kojid +++ b/builder/kojid @@ -5225,8 +5225,11 @@ class createDistRepoTask(BaseTaskHandler): self.repodir = '%s/repo' % self.workdir koji.ensuredir(self.repodir) self.outdir = self.repodir # workaround create_local_repo use - self.datadir = '%s/repodata' % self.repodir + datadir = '%s/repodata' % self.repodir self.sigmap = {} + if opts['split_debuginfo']: + debugdir = '%s/debug' % self.repodir + koji.ensuredir(debugdir) # gather oldpkgs data if delta option in use oldpkgs = [] @@ -5241,32 +5244,42 @@ class createDistRepoTask(BaseTaskHandler): path = koji.pathinfo.distrepo(repo_id, oldrepo['tag_name']) if not os.path.exists(path): raise koji.GenericError('Base drpm repo missing: %s' % path) + # note: since we're using the top level dir, this will handle + # split debuginfo as well oldpkgs.append(path) # sort out our package list self.uploadpath = self.getUploadDir() - self.make_pkglist(tag, arch, keys, opts) - self.pkglist = self.write_pkglist() - self.link_pkgs() + self.get_rpms(tag, arch, keys, opts) if opts['multilib'] and rpmUtils.arch.isMultiLibArch(arch): self.do_multilib(arch, self.archmap[arch], opts['multilib']) + self.write_pkglist(opts) self.write_kojipkgs() - self.logger.debug('package list is %s' % self.pkglist) - self.session.uploadWrapper(self.pkglist, self.uploadpath, - os.path.basename(self.pkglist)) + self.link_pkgs() + self.session.uploadWrapper('%s/pkglist' % self.repodir, + self.uploadpath, 'pkglist') + if opts['split_debuginfo']: + self.session.uploadWrapper('%s/debug/pkglist' % self.repodir, + self.uploadpath, 'debug_pkglist') # generate the repodata - self.do_createrepo(self.repodir, self.pkglist, groupdata, oldpkgs=oldpkgs) - if os.path.getsize(self.pkglist) == 0: - fo = file(os.path.join(self.datadir, "EMPTY_REPO"), 'w') + self.do_createrepo(self.repodir, '%s/pkglist' % self.repodir, + groupdata, oldpkgs=oldpkgs) + if opts['split_debuginfo']: + self.do_createrepo(debugdir, '%s/pkglist' % debugdir, None, + oldpkgs=oldpkgs) + if len(self.kojipkgs) == 0: + fo = file(os.path.join(datadir, "EMPTY_REPO"), 'w') fo.write("This repo is empty because its tag has no content for this arch\n") fo.close() # upload repo files files = ['pkglist', 'kojipkgs'] - for f in os.listdir(self.datadir): + if opts['split_debuginfo']: + files.append('debug_pkglist') + for f in os.listdir(datadir): files.append(f) - self.session.uploadWrapper('%s/%s' % (self.datadir, f), + self.session.uploadWrapper('%s/%s' % (datadir, f), self.uploadpath, f) if opts['delta']: ddir = os.path.join(self.repodir, 'drpms') @@ -5289,7 +5302,7 @@ class createDistRepoTask(BaseTaskHandler): else: cmd = ['/usr/bin/createrepo'] cmd.extend(['-vd', '-i', pkglist]) - if os.path.isfile(groupdata): + if groupdata and os.path.isfile(groupdata): cmd.extend(['-g', groupdata]) # TODO: can we recycle data (with --update) as in create_local_repo? if oldpkgs: @@ -5425,23 +5438,22 @@ enabled=1 raise koji.GenericError('multilib packages missing. ' 'See missing_multilib.log') - # step 5: add dependencies to our package list - pkgwriter = open(self.pkglist, 'a') + # step 5: update kojipkgs for dep_path in ml_needed: tspkg = ml_needed[dep_path] bnp = os.path.basename(dep_path) - bnplet = bnp[0].lower() - koji.ensuredir(os.path.join(self.repodir, bnplet)) - dst = os.path.join(self.repodir, bnplet, bnp) - if os.path.exists(dst): + if bnp in self.kojipkgs: # we expect duplication with noarch, but not other arches if tspkg.arch != 'noarch': - self.logger.warning("Path exists: %r", dst) + self.logger.warning("Multilib duplicate: %s", bnp) continue - pkgwriter.write(bnplet + '/' + bnp + '\n') - self.logger.debug("os.symlink(%r, %r)", dep_path, dst) - os.symlink(dep_path, dst) - rpminfo = ml_pkgs[bnp] + rpminfo = ml_pkgs[bnp].copy() + # fix _pkgpath, which comes from another task and could be wrong + # for us + # TODO: would be better if we could use the proper path here + rpminfo['_pkgpath'] = dep_path + rpminfo['_multilib'] = True + self.kojipkgs[bnp] = rpminfo self.sigmap[rpminfo['id']] = rpminfo['sigkey'] @@ -5459,7 +5471,7 @@ enabled=1 best_idx = idx return best - def make_pkglist(self, tag_id, arch, keys, opts): + def get_rpms(self, tag_id, arch, keys, opts): # get the rpm data rpms = [] builddirs = {} @@ -5564,20 +5576,31 @@ enabled=1 self.logger.debug("os.symlink(%r, %r(", pkgpath, dst) os.symlink(pkgpath, dst) - def write_pkglist(self): - pkgfile = os.path.join(self.repodir, 'pkglist') - pkglist = file(pkgfile, 'w') + def write_pkglist(self, opts): + pkgs = [] + debug_pkgs = [] for bnp in self.kojipkgs: + rpminfo = self.kojipkgs[bnp] bnplet = bnp[0].lower() - pkglist.write(bnplet + '/' + bnp + '\n') - pkglist.close() - return pkgfile + if opts['split_debuginfo'] and rpminfo.get('_multilib'): + # note the ../ + debug_pkgs.append('../%s/%s\n' % (bnplet, bnp)) + else: + pkgs.append('%s/%s\n' % (bnplet, bnp)) + + with open('%s/pkglist' % self.repodir, 'w') as fo: + for line in pkgs: + fo.write(line) + if opts['split_debuginfo']: + with open('%s/debug/pkglist' % self.repodir, 'w') as fo: + for line in debug_pkgs: + fo.write(line) def write_kojipkgs(self): filename = os.path.join(self.repodir, 'kojipkgs') datafile = file(filename, 'w') try: - json.dump(self.kojipkgs, datafile, indent=4) + json.dump(self.kojipkgs, datafile, indent=4, sort_keys=True) finally: datafile.close() # and upload too From 869dc3b57b88c0ca7134ff79b2a1be11a8bdab62 Mon Sep 17 00:00:00 2001 From: Mike McLean Date: May 03 2018 20:05:16 +0000 Subject: [PATCH 21/36] fix debuginfo check and logfile name --- diff --git a/builder/kojid b/builder/kojid index cead161..887fbf8 100755 --- a/builder/kojid +++ b/builder/kojid @@ -5267,7 +5267,7 @@ class createDistRepoTask(BaseTaskHandler): groupdata, oldpkgs=oldpkgs) if opts['split_debuginfo']: self.do_createrepo(debugdir, '%s/pkglist' % debugdir, None, - oldpkgs=oldpkgs) + oldpkgs=oldpkgs, logname='createrepo_debug') if len(self.kojipkgs) == 0: fo = file(os.path.join(datadir, "EMPTY_REPO"), 'w') fo.write("This repo is empty because its tag has no content for this arch\n") @@ -5582,7 +5582,7 @@ enabled=1 for bnp in self.kojipkgs: rpminfo = self.kojipkgs[bnp] bnplet = bnp[0].lower() - if opts['split_debuginfo'] and rpminfo.get('_multilib'): + if opts['split_debuginfo'] and koji.is_debuginfo(rpminfo['name']): # note the ../ debug_pkgs.append('../%s/%s\n' % (bnplet, bnp)) else: From 1a971f39e18a0782fb515b73e94a43133b2cd30e Mon Sep 17 00:00:00 2001 From: Mike McLean Date: May 03 2018 20:05:16 +0000 Subject: [PATCH 22/36] handle uploaded debug_pkglist --- diff --git a/hub/kojihub.py b/hub/kojihub.py index d1ba9bf..2ca9962 100644 --- a/hub/kojihub.py +++ b/hub/kojihub.py @@ -12665,6 +12665,7 @@ class HostExports(object): koji.ensuredir(datadir) pkglist = set() + debuglist = set() for fn in files: src = "%s/%s/%s" % (workdir, uploadpath, fn) if fn.endswith('.drpm'): @@ -12677,10 +12678,14 @@ class HostExports(object): if not os.path.exists(src): raise koji.GenericError("uploaded file missing: %s" % src) if fn.endswith('pkglist'): + if fn.endswith('debug_pkglist'): + tracker = debuglist + else: + tracker = pkglist with open(src) as pkgfile: for pkg in pkgfile: pkg = os.path.basename(pkg.strip()) - pkglist.add(pkg) + tracker.add(pkg) safer_move(src, dst) # get rpms @@ -12705,9 +12710,9 @@ class HostExports(object): # sanity check for fn in rpmdata: - if fn not in pkglist: + if fn not in pkglist and fn not in debuglist: raise koji.GenericError("No signature data for: %s" % fn) - for fn in pkglist: + for fn in pkglist.union(debuglist): if fn not in rpmdata: raise koji.GenericError("RPM missing from pkglist: %s" % fn) From 86043223c27e22aead4b835ddc79822db238c349 Mon Sep 17 00:00:00 2001 From: Mike McLean Date: May 03 2018 20:05:16 +0000 Subject: [PATCH 23/36] record dist subrepo in kojipkgs --- diff --git a/builder/kojid b/builder/kojid index 887fbf8..064b80f 100755 --- a/builder/kojid +++ b/builder/kojid @@ -5227,9 +5227,7 @@ class createDistRepoTask(BaseTaskHandler): self.outdir = self.repodir # workaround create_local_repo use datadir = '%s/repodata' % self.repodir self.sigmap = {} - if opts['split_debuginfo']: - debugdir = '%s/debug' % self.repodir - koji.ensuredir(debugdir) + self.subrepos = set() # gather oldpkgs data if delta option in use oldpkgs = [] @@ -5253,14 +5251,11 @@ class createDistRepoTask(BaseTaskHandler): self.get_rpms(tag, arch, keys, opts) if opts['multilib'] and rpmUtils.arch.isMultiLibArch(arch): self.do_multilib(arch, self.archmap[arch], opts['multilib']) - self.write_pkglist(opts) + if opts['split_debuginfo']: + self.split_debuginfo() self.write_kojipkgs() + self.write_pkglist() self.link_pkgs() - self.session.uploadWrapper('%s/pkglist' % self.repodir, - self.uploadpath, 'pkglist') - if opts['split_debuginfo']: - self.session.uploadWrapper('%s/debug/pkglist' % self.repodir, - self.uploadpath, 'debug_pkglist') # generate the repodata self.do_createrepo(self.repodir, '%s/pkglist' % self.repodir, @@ -5276,7 +5271,7 @@ class createDistRepoTask(BaseTaskHandler): # upload repo files files = ['pkglist', 'kojipkgs'] if opts['split_debuginfo']: - files.append('debug_pkglist') + files.append('debug/pkglist') for f in os.listdir(datadir): files.append(f) self.session.uploadWrapper('%s/%s' % (datadir, f), @@ -5576,26 +5571,42 @@ enabled=1 self.logger.debug("os.symlink(%r, %r(", pkgpath, dst) os.symlink(pkgpath, dst) - def write_pkglist(self, opts): + def split_debuginfo(self): + for rpminfo in self.kojipkgs.values(): + if koji.is_debuginfo(rpminfo['name']): + rpminfo['_subrepo'] = 'debug' + + def write_pkglist(self): pkgs = [] - debug_pkgs = [] + subrepo_pkgs = {} for bnp in self.kojipkgs: rpminfo = self.kojipkgs[bnp] bnplet = bnp[0].lower() - if opts['split_debuginfo'] and koji.is_debuginfo(rpminfo['name']): + subrepo = rpminfo.get('_subrepo') + if subrepo: # note the ../ - debug_pkgs.append('../%s/%s\n' % (bnplet, bnp)) + subrepo_pkgs.setdefault(subrepo, []).append( + '../%s/%s\n' % (bnplet, bnp)) else: pkgs.append('%s/%s\n' % (bnplet, bnp)) with open('%s/pkglist' % self.repodir, 'w') as fo: for line in pkgs: fo.write(line) - if opts['split_debuginfo']: - with open('%s/debug/pkglist' % self.repodir, 'w') as fo: - for line in debug_pkgs: + for subrepo in subrepo_pkgs: + koji.ensuredir('%s/%s' % (self.repodir, subrepo)) + with open('%s/%s/pkglist' % (self.repodir, subrepo), 'w') as fo: + for line in subrepo_pkgs[subrepo]: fo.write(line) + # and upload too + self.session.uploadWrapper('%s/pkglist' % self.repodir, + self.uploadpath, 'pkglist') + for subrepo in subrepo_pkgs: + self.session.uploadWrapper( + '%s/%s/pkglist' % (self.repodir, subrepo), + '%s/%s' % (self.uploadpath, subrepo), 'pkglist') + def write_kojipkgs(self): filename = os.path.join(self.repodir, 'kojipkgs') datafile = file(filename, 'w') From c5f126a7b01c63a725eca026dfc371247f4089ad Mon Sep 17 00:00:00 2001 From: Mike McLean Date: May 03 2018 20:05:16 +0000 Subject: [PATCH 24/36] streamline subrepo logic a bit --- diff --git a/builder/kojid b/builder/kojid index 064b80f..236cd9b 100755 --- a/builder/kojid +++ b/builder/kojid @@ -5243,16 +5243,15 @@ class createDistRepoTask(BaseTaskHandler): if not os.path.exists(path): raise koji.GenericError('Base drpm repo missing: %s' % path) # note: since we're using the top level dir, this will handle - # split debuginfo as well + # split repos as well oldpkgs.append(path) - # sort out our package list + # sort out our package list(s) self.uploadpath = self.getUploadDir() self.get_rpms(tag, arch, keys, opts) if opts['multilib'] and rpmUtils.arch.isMultiLibArch(arch): self.do_multilib(arch, self.archmap[arch], opts['multilib']) - if opts['split_debuginfo']: - self.split_debuginfo() + self.split_pkgs(opts) self.write_kojipkgs() self.write_pkglist() self.link_pkgs() @@ -5260,9 +5259,12 @@ class createDistRepoTask(BaseTaskHandler): # generate the repodata self.do_createrepo(self.repodir, '%s/pkglist' % self.repodir, groupdata, oldpkgs=oldpkgs) - if opts['split_debuginfo']: - self.do_createrepo(debugdir, '%s/pkglist' % debugdir, None, - oldpkgs=oldpkgs, logname='createrepo_debug') + for subrepo in self.subrepos: + self.do_createrepo( + '%s/%s' % (self.repodir, subrepo), + '%s/%s/pkglist' % (self.repodir, subrepo), + groupdata, oldpkgs=oldpkgs, + logname='createrepo_%s' % subrepo) if len(self.kojipkgs) == 0: fo = file(os.path.join(datadir, "EMPTY_REPO"), 'w') fo.write("This repo is empty because its tag has no content for this arch\n") @@ -5270,9 +5272,10 @@ class createDistRepoTask(BaseTaskHandler): # upload repo files files = ['pkglist', 'kojipkgs'] - if opts['split_debuginfo']: - files.append('debug/pkglist') + for subrepo in self.subrepos: + files.append('%s/pkglist' % subrepo) for f in os.listdir(datadir): + # repo metadata files files.append(f) self.session.uploadWrapper('%s/%s' % (datadir, f), self.uploadpath, f) @@ -5571,10 +5574,12 @@ enabled=1 self.logger.debug("os.symlink(%r, %r(", pkgpath, dst) os.symlink(pkgpath, dst) - def split_debuginfo(self): + def split_pkgs(self, opts): + '''Direct rpms to subrepos if needed''' for rpminfo in self.kojipkgs.values(): - if koji.is_debuginfo(rpminfo['name']): + if opts['split_debuginfo'] and koji.is_debuginfo(rpminfo['name']): rpminfo['_subrepo'] = 'debug' + self.subrepos.add('debug') def write_pkglist(self): pkgs = [] From 8e08e66d9abdb4f386389b0863e8c1d43eb4e4f7 Mon Sep 17 00:00:00 2001 From: Mike McLean Date: May 03 2018 20:05:16 +0000 Subject: [PATCH 25/36] partial refactor --- diff --git a/hub/kojihub.py b/hub/kojihub.py index 2ca9962..a2a7142 100644 --- a/hub/kojihub.py +++ b/hub/kojihub.py @@ -12635,8 +12635,7 @@ class HostExports(object): def distRepoMove(self, repo_id, uploadpath, files, arch, sigmap): """ - Move a dist repo into its final location - + Move one arch of a dist repo into its final location Unlike normal repos (which are moved into place by repoDone), dist repos have all their content linked (or copied) into place. @@ -12661,11 +12660,28 @@ class HostExports(object): archdir = "%s/%s" % (repodir, koji.canonArch(arch)) if not os.path.isdir(archdir): raise koji.GenericError("Repo arch directory missing: %s" % archdir) - datadir = "%s/repodata" % archdir - koji.ensuredir(datadir) + repo_state = koji.REPO_STATES[rinfo['state']] + if repo_state != 'INIT': + raise koji.GenericError('Repo is in state: %s' % repo_state) - pkglist = set() - debuglist = set() + # Read package data + fn = '%s/%s/kojipkgs' % (workdir, uploadpath) + if not os.path.isfile(fn): + raise koji.GenericError('Missing kojipkgs file') + with open(fn) as fp: + kojipkgs = json.load(fp) + + # Figure out subrepos + subrepos = set() + for bnp in kojipkgs: + rpminfo = kojipkgs[bnp] + subrepo = rpminfo.get('_subrepo') + if subrepo: + subrepos.add(subrepo) + + # Figure out where to send the uploaded files + file_moves = [] + datadir = "%s/repodata" % archdir for fn in files: src = "%s/%s/%s" % (workdir, uploadpath, fn) if fn.endswith('.drpm'): @@ -12677,17 +12693,16 @@ class HostExports(object): dst = "%s/%s" % (datadir, fn) if not os.path.exists(src): raise koji.GenericError("uploaded file missing: %s" % src) - if fn.endswith('pkglist'): - if fn.endswith('debug_pkglist'): - tracker = debuglist - else: - tracker = pkglist - with open(src) as pkgfile: - for pkg in pkgfile: - pkg = os.path.basename(pkg.strip()) - tracker.add(pkg) + file_moves.append([src, dst]) + + dirnames = set([os.path.dirname(fm[1]) for fm in file_moves]) + for dirname in dirnames: + koji.ensuredir(dirname) + for src, dst in file_moves: safer_move(src, dst) + koji.ensuredir(datadir) + # get rpms build_dirs = {} rpmdata = {} From fb5da35edfb59e1be666fedaf9baad98e3aafa07 Mon Sep 17 00:00:00 2001 From: Mike McLean Date: May 03 2018 20:05:16 +0000 Subject: [PATCH 26/36] refactor uploaded file data --- diff --git a/builder/kojid b/builder/kojid index 236cd9b..945659d 100755 --- a/builder/kojid +++ b/builder/kojid @@ -5160,9 +5160,9 @@ class NewDistRepoTask(BaseTaskHandler): for arch in arch32s: # move the 32-bit task output to the final resting place # so the 64-bit arches can use it for multilib - upload, files, sigmap = results[subtasks[arch]] + upload, files = results[subtasks[arch]] self.session.host.distRepoMove( - repo_id, upload, files, arch, sigmap) + repo_id, upload, files, arch) for arch in canonArches: # do the other arches if arch not in arch32s: @@ -5180,9 +5180,9 @@ class NewDistRepoTask(BaseTaskHandler): # already moved above continue #else - upload, files, sigmap = results[subtasks[arch]] + upload, files = results[subtasks[arch]] self.session.host.distRepoMove( - repo_id, upload, files, arch, sigmap) + repo_id, upload, files, arch) self.session.host.repoDone(repo_id, data, expire=False) return 'Dist repository #%s successfully generated' % repo_id @@ -5223,10 +5223,8 @@ class createDistRepoTask(BaseTaskHandler): # set up our output dir self.repodir = '%s/repo' % self.workdir + self.repo_files = [] koji.ensuredir(self.repodir) - self.outdir = self.repodir # workaround create_local_repo use - datadir = '%s/repodata' % self.repodir - self.sigmap = {} self.subrepos = set() # gather oldpkgs data if delta option in use @@ -5266,27 +5264,36 @@ class createDistRepoTask(BaseTaskHandler): groupdata, oldpkgs=oldpkgs, logname='createrepo_%s' % subrepo) if len(self.kojipkgs) == 0: - fo = file(os.path.join(datadir, "EMPTY_REPO"), 'w') - fo.write("This repo is empty because its tag has no content for this arch\n") - fo.close() + fn = os.path.join(self.repodir, "repodata", "EMPTY_REPO") + with open(fn, 'w') as fp: + fp.write("This repo is empty because its tag has no content " + "for this arch\n") # upload repo files - files = ['pkglist', 'kojipkgs'] - for subrepo in self.subrepos: - files.append('%s/pkglist' % subrepo) - for f in os.listdir(datadir): + for f in os.listdir('%s/repodata' % self.repodir): # repo metadata files - files.append(f) - self.session.uploadWrapper('%s/%s' % (datadir, f), - self.uploadpath, f) + self.upload_repo_file("repodata/%s" % f) if opts['delta']: - ddir = os.path.join(self.repodir, 'drpms') - for f in os.listdir(ddir): - files.append(f) - self.session.uploadWrapper('%s/%s' % (ddir, f), - self.uploadpath, f) + for f in os.listdir('%s/drpms' % self.repodir): + self.upload_repo_file("drpms/%s" % f) + + return [self.uploadpath, self.repo_files] - return [self.uploadpath, files, self.sigmap.items()] + def upload_repo_file(self, relpath): + """Upload a file from the repo + + relpath should be relative to self.repodir + """ + localpath = '%s/%s' % (self.repodir, relpath) + reldir = os.path.dirname(relpath) + if reldir: + uploadpath = "%s/%s" % (self.uploadpath, reldir) + fn = os.path.basename(relpath) + else: + uploadpath = self.uploadpath + fn = relpath + self.session.uploadWrapper(localpath, uploadpath, fn) + self.repo_files.append(relpath) def do_createrepo(self, repodir, pkglist, groupdata, oldpkgs=None, logname=None): """Run createrepo @@ -5452,8 +5459,6 @@ enabled=1 rpminfo['_pkgpath'] = dep_path rpminfo['_multilib'] = True self.kojipkgs[bnp] = rpminfo - self.sigmap[rpminfo['id']] = rpminfo['sigkey'] - def pick_key(self, keys, avail_keys): best = None @@ -5524,7 +5529,6 @@ enabled=1 # we'll raise an error below else: bnp = os.path.basename(pkgpath) - self.sigmap[rpminfo['id']] = rpminfo['sigkey'] rpminfo['_pkgpath'] = pkgpath kojipkgs[bnp] = rpminfo self.kojipkgs = kojipkgs @@ -5605,12 +5609,9 @@ enabled=1 fo.write(line) # and upload too - self.session.uploadWrapper('%s/pkglist' % self.repodir, - self.uploadpath, 'pkglist') + self.upload_repo_file('pkglist') for subrepo in subrepo_pkgs: - self.session.uploadWrapper( - '%s/%s/pkglist' % (self.repodir, subrepo), - '%s/%s' % (self.uploadpath, subrepo), 'pkglist') + self.upload_repo_file('%s/pkglist' % subrepo) def write_kojipkgs(self): filename = os.path.join(self.repodir, 'kojipkgs') @@ -5620,8 +5621,7 @@ enabled=1 finally: datafile.close() # and upload too - self.session.uploadWrapper(filename, self.uploadpath, 'kojipkgs') - + self.upload_repo_file('kojipkgs') class WaitrepoTask(BaseTaskHandler): diff --git a/hub/kojihub.py b/hub/kojihub.py index a2a7142..cb7ccec 100644 --- a/hub/kojihub.py +++ b/hub/kojihub.py @@ -12633,7 +12633,7 @@ class HostExports(object): koji.plugin.run_callbacks('postRepoDone', repo=rinfo, data=data, expire=expire) - def distRepoMove(self, repo_id, uploadpath, files, arch, sigmap): + def distRepoMove(self, repo_id, uploadpath, files, arch): """ Move one arch of a dist repo into its final location @@ -12644,13 +12644,10 @@ class HostExports(object): uploadpath - where the uploaded files are files - a list of the uploaded file names arch - the arch of the repo - sigmap - a list of [rpm_id, sig] pairs - The rpms from sigmap should match the contents of the uploaded pkglist - file. - - In sigmap, use sig=None to use the primary copy of the rpm instead of a - signed copy. + The uploaded files should include: + - kojipkgs: json file with information about the component rpms + - repo metadata files """ host = Host() host.verify() @@ -12671,43 +12668,28 @@ class HostExports(object): with open(fn) as fp: kojipkgs = json.load(fp) - # Figure out subrepos - subrepos = set() - for bnp in kojipkgs: - rpminfo = kojipkgs[bnp] - subrepo = rpminfo.get('_subrepo') - if subrepo: - subrepos.add(subrepo) - # Figure out where to send the uploaded files file_moves = [] - datadir = "%s/repodata" % archdir - for fn in files: - src = "%s/%s/%s" % (workdir, uploadpath, fn) - if fn.endswith('.drpm'): - koji.ensuredir(os.path.join(archdir, 'drpms')) - dst = "%s/drpms/%s" % (archdir, fn) - elif fn.endswith('pkglist') or fn.endswith('kojipkgs'): - dst = '%s/%s' % (archdir, fn) - else: - dst = "%s/%s" % (datadir, fn) + for relpath in files: + src = "%s/%s/%s" % (workdir, uploadpath, relpath) + dst = "%s/%s" % (archdir, relpath) if not os.path.exists(src): raise koji.GenericError("uploaded file missing: %s" % src) file_moves.append([src, dst]) - dirnames = set([os.path.dirname(fm[1]) for fm in file_moves]) - for dirname in dirnames: - koji.ensuredir(dirname) - for src, dst in file_moves: - safer_move(src, dst) - - koji.ensuredir(datadir) - # get rpms build_dirs = {} rpmdata = {} - for rpm_id, sigkey in sigmap: - rpminfo = get_rpm(rpm_id, strict=True) + for bnp in kojipkgs: + rpminfo = kojipkgs[bnp] + rpm_id = rpminfo['id'] + sigkey = rpminfo['sigkey'] + _rpminfo = get_rpm(rpm_id, strict=True) + for key in _rpminfo: + if key not in rpminfo or rpminfo[key] != _rpminfo[key]: + raise koji.GenericError( + 'kojipkgs entry does not match db: file %s, key %s' + % (bnp, key)) if sigkey is None or sigkey == '': relpath = koji.pathinfo.rpm(rpminfo) else: @@ -12720,20 +12702,18 @@ class HostExports(object): builddir = koji.pathinfo.build(binfo) build_dirs[rpminfo['build_id']] = builddir rpminfo['_fullpath'] = os.path.join(builddir, relpath) - basename = os.path.basename(relpath) - rpmdata[basename] = rpminfo + rpmdata[bnp] = rpminfo - # sanity check - for fn in rpmdata: - if fn not in pkglist and fn not in debuglist: - raise koji.GenericError("No signature data for: %s" % fn) - for fn in pkglist.union(debuglist): - if fn not in rpmdata: - raise koji.GenericError("RPM missing from pkglist: %s" % fn) + # move the uploaded files + dirnames = set([os.path.dirname(fm[1]) for fm in file_moves]) + for dirname in dirnames: + koji.ensuredir(dirname) + for src, dst in file_moves: + safer_move(src, dst) + # hardlink or copy the rpms into the final repodir + # TODO: properly consider split-volume functionality for fn in rpmdata: - # hardlink or copy the rpms into the final repodir - # TODO: properly consider split-volume functionality rpminfo = rpmdata[fn] rpmpath = rpminfo['_fullpath'] bnp = fn From 60cccdbcfaee9d5f5477c6217a2a1eb1fb212485 Mon Sep 17 00:00:00 2001 From: Mike McLean Date: May 03 2018 20:05:16 +0000 Subject: [PATCH 27/36] limit rpm keys to check --- diff --git a/hub/kojihub.py b/hub/kojihub.py index cb7ccec..506a5c9 100644 --- a/hub/kojihub.py +++ b/hub/kojihub.py @@ -12680,12 +12680,14 @@ class HostExports(object): # get rpms build_dirs = {} rpmdata = {} + rpm_check_keys = ['name', 'version', 'release', 'arch', 'epoch', + 'size', 'payloadhash', 'build_id'] for bnp in kojipkgs: rpminfo = kojipkgs[bnp] rpm_id = rpminfo['id'] sigkey = rpminfo['sigkey'] _rpminfo = get_rpm(rpm_id, strict=True) - for key in _rpminfo: + for key in rpm_check_keys: if key not in rpminfo or rpminfo[key] != _rpminfo[key]: raise koji.GenericError( 'kojipkgs entry does not match db: file %s, key %s' From 72a57611f1bd8edf435d5852f393b98c43a755a5 Mon Sep 17 00:00:00 2001 From: Mike McLean Date: May 03 2018 20:05:16 +0000 Subject: [PATCH 28/36] also upload subrepo metadata --- diff --git a/builder/kojid b/builder/kojid index 945659d..8e2251e 100755 --- a/builder/kojid +++ b/builder/kojid @@ -5271,8 +5271,10 @@ class createDistRepoTask(BaseTaskHandler): # upload repo files for f in os.listdir('%s/repodata' % self.repodir): - # repo metadata files self.upload_repo_file("repodata/%s" % f) + for subrepo in self.subrepos: + for f in os.listdir('%s/%s/repodata' % (self.repodir, subrepo)): + self.upload_repo_file("%s/repodata/%s" % (subrepo, f)) if opts['delta']: for f in os.listdir('%s/drpms' % self.repodir): self.upload_repo_file("drpms/%s" % f) From c032bdf6bcce17eada3fc318c345d2718dea55d1 Mon Sep 17 00:00:00 2001 From: Mike McLean Date: May 03 2018 20:05:16 +0000 Subject: [PATCH 29/36] upload a repo manifest --- diff --git a/builder/kojid b/builder/kojid index 8e2251e..3f98135 100755 --- a/builder/kojid +++ b/builder/kojid @@ -5160,9 +5160,8 @@ class NewDistRepoTask(BaseTaskHandler): for arch in arch32s: # move the 32-bit task output to the final resting place # so the 64-bit arches can use it for multilib - upload, files = results[subtasks[arch]] - self.session.host.distRepoMove( - repo_id, upload, files, arch) + upload = results[subtasks[arch]] + self.session.host.distRepoMove(repo_id, upload, arch) for arch in canonArches: # do the other arches if arch not in arch32s: @@ -5180,9 +5179,8 @@ class NewDistRepoTask(BaseTaskHandler): # already moved above continue #else - upload, files = results[subtasks[arch]] - self.session.host.distRepoMove( - repo_id, upload, files, arch) + upload = results[subtasks[arch]] + self.session.host.distRepoMove(repo_id, upload, arch) self.session.host.repoDone(repo_id, data, expire=False) return 'Dist repository #%s successfully generated' % repo_id @@ -5270,23 +5268,18 @@ class createDistRepoTask(BaseTaskHandler): "for this arch\n") # upload repo files - for f in os.listdir('%s/repodata' % self.repodir): - self.upload_repo_file("repodata/%s" % f) - for subrepo in self.subrepos: - for f in os.listdir('%s/%s/repodata' % (self.repodir, subrepo)): - self.upload_repo_file("%s/repodata/%s" % (subrepo, f)) - if opts['delta']: - for f in os.listdir('%s/drpms' % self.repodir): - self.upload_repo_file("drpms/%s" % f) + self.upload_repo() + self.upload_repo_manifest() - return [self.uploadpath, self.repo_files] + return self.uploadpath - def upload_repo_file(self, relpath): + def upload_repo_file(self, relpath, localpath=None, record=True): """Upload a file from the repo relpath should be relative to self.repodir """ - localpath = '%s/%s' % (self.repodir, relpath) + if localpath is None: + localpath = '%s/%s' % (self.repodir, relpath) reldir = os.path.dirname(relpath) if reldir: uploadpath = "%s/%s" % (self.uploadpath, reldir) @@ -5295,7 +5288,29 @@ class createDistRepoTask(BaseTaskHandler): uploadpath = self.uploadpath fn = relpath self.session.uploadWrapper(localpath, uploadpath, fn) - self.repo_files.append(relpath) + if record: + self.repo_files.append(relpath) + + def upload_repo(self): + """Traverse the repo and upload needed files + + We omit the symlinks we made for the rpms + """ + for dirpath, dirs, files in os.walk(self.repodir): + reldir = os.path.relpath(dirpath, self.repodir) + for filename in files: + path = "%s/%s" % (dirpath, filename) + if os.path.islink(path): + continue + relpath = "%s/%s" % (reldir, filename) + self.upload_repo_file(relpath) + + def upload_repo_manifest(self): + """Upload a list of the repo files we've uploaded""" + fn = '%s/repo_manifest' % self.workdir + with open(fn, 'w') as fp: + json.dump(self.repo_files, fp, indent=4) + self.session.uploadWrapper(fn, self.uploadpath) def do_createrepo(self, repodir, pkglist, groupdata, oldpkgs=None, logname=None): """Run createrepo @@ -5610,11 +5625,6 @@ enabled=1 for line in subrepo_pkgs[subrepo]: fo.write(line) - # and upload too - self.upload_repo_file('pkglist') - for subrepo in subrepo_pkgs: - self.upload_repo_file('%s/pkglist' % subrepo) - def write_kojipkgs(self): filename = os.path.join(self.repodir, 'kojipkgs') datafile = file(filename, 'w') @@ -5622,8 +5632,6 @@ enabled=1 json.dump(self.kojipkgs, datafile, indent=4, sort_keys=True) finally: datafile.close() - # and upload too - self.upload_repo_file('kojipkgs') class WaitrepoTask(BaseTaskHandler): diff --git a/hub/kojihub.py b/hub/kojihub.py index 506a5c9..bc21789 100644 --- a/hub/kojihub.py +++ b/hub/kojihub.py @@ -12633,7 +12633,7 @@ class HostExports(object): koji.plugin.run_callbacks('postRepoDone', repo=rinfo, data=data, expire=expire) - def distRepoMove(self, repo_id, uploadpath, files, arch): + def distRepoMove(self, repo_id, uploadpath, arch): """ Move one arch of a dist repo into its final location @@ -12642,9 +12642,10 @@ class HostExports(object): repo_id - the repo to move uploadpath - where the uploaded files are - files - a list of the uploaded file names arch - the arch of the repo + uploadpath should contain a repo_manifest file + The uploaded files should include: - kojipkgs: json file with information about the component rpms - repo metadata files @@ -12661,6 +12662,13 @@ class HostExports(object): if repo_state != 'INIT': raise koji.GenericError('Repo is in state: %s' % repo_state) + # read manifest + fn = '%s/%s/repo_manifest' % (workdir, uploadpath) + if not os.path.isfile(fn): + raise koji.GenericError('Missing repo manifest') + with open(fn) as fp: + files = json.load(fp) + # Read package data fn = '%s/%s/kojipkgs' % (workdir, uploadpath) if not os.path.isfile(fn): From 12129dd7f1fb33d6164cd01bfbdc32390ff787bc Mon Sep 17 00:00:00 2001 From: Mike McLean Date: May 03 2018 20:05:16 +0000 Subject: [PATCH 30/36] fix help string --- diff --git a/cli/koji_cli/commands.py b/cli/koji_cli/commands.py index 7ef15ba..a9c14ac 100644 --- a/cli/koji_cli/commands.py +++ b/cli/koji_cli/commands.py @@ -6925,7 +6925,7 @@ def handle_dist_repo(options, session, args): 'or the name of a tag that has a dist repo. May be specified ' 'multiple times.')) parser.add_option('--event', type='int', - help=_('create a dist repository based on a Brew event')) + help=_('Use tag content at event')) parser.add_option('--non-latest', dest='latest', default=True, action='store_false', help='Include older builds, not just the latest') parser.add_option('--multilib', default=None, metavar="CONFIG", From 387f92b5dd4920ae32a65ad1cf37e1d1a0ef3c3f Mon Sep 17 00:00:00 2001 From: Mike McLean Date: May 03 2018 20:05:16 +0000 Subject: [PATCH 31/36] drop unused params --- diff --git a/builder/kojid b/builder/kojid index 3f98135..4a45853 100755 --- a/builder/kojid +++ b/builder/kojid @@ -5273,13 +5273,13 @@ class createDistRepoTask(BaseTaskHandler): return self.uploadpath - def upload_repo_file(self, relpath, localpath=None, record=True): + def upload_repo_file(self, relpath): """Upload a file from the repo relpath should be relative to self.repodir """ - if localpath is None: - localpath = '%s/%s' % (self.repodir, relpath) + + localpath = '%s/%s' % (self.repodir, relpath) reldir = os.path.dirname(relpath) if reldir: uploadpath = "%s/%s" % (self.uploadpath, reldir) @@ -5288,8 +5288,7 @@ class createDistRepoTask(BaseTaskHandler): uploadpath = self.uploadpath fn = relpath self.session.uploadWrapper(localpath, uploadpath, fn) - if record: - self.repo_files.append(relpath) + self.repo_files.append(relpath) def upload_repo(self): """Traverse the repo and upload needed files From ddf120a4d33bc6328dcd16bd5f9e8252b16e258b Mon Sep 17 00:00:00 2001 From: Mike McLean Date: May 03 2018 20:05:16 +0000 Subject: [PATCH 32/36] update docstrings --- diff --git a/hub/kojihub.py b/hub/kojihub.py index bc21789..3304778 100644 --- a/hub/kojihub.py +++ b/hub/kojihub.py @@ -12574,14 +12574,21 @@ class HostExports(object): safer_move(filepath, dst) def repoDone(self, repo_id, data, expire=False): - """Move repo data into place, mark as ready, and expire earlier repos + """Finalize a repo repo_id: the id of the repo - data: a dictionary of the form { arch: (uploadpath, files), ...} - expire(optional): if set to true, mark the repo expired immediately* + data: a dictionary of repo files in the form: + { arch: [uploadpath, [file1, file2, ...]], ...} + expire: if set to true, mark the repo expired immediately [*] - If this is a dist repo, also hardlink the rpms in the final - directory. + Actions: + * Move uploaded repo files into place + * Mark repo ready + * Expire earlier repos + * Move/create 'latest' symlink + + For dist repos, the move step is skipped (that is handled in + distRepoMove). * This is used when a repo from an older event is generated """ @@ -12637,8 +12644,8 @@ class HostExports(object): """ Move one arch of a dist repo into its final location - Unlike normal repos (which are moved into place by repoDone), dist - repos have all their content linked (or copied) into place. + Unlike normal repos, dist repos have all their content linked (or + copied) into place. repo_id - the repo to move uploadpath - where the uploaded files are From d60bdddcf40c37ee4285f3a0c97171de9c3d9e9e Mon Sep 17 00:00:00 2001 From: Mike McLean Date: May 03 2018 20:05:16 +0000 Subject: [PATCH 33/36] link dist repo on task page --- diff --git a/www/kojiweb/taskinfo.chtml b/www/kojiweb/taskinfo.chtml index a4e050f..5ee634a 100644 --- a/www/kojiweb/taskinfo.chtml +++ b/www/kojiweb/taskinfo.chtml @@ -225,7 +225,7 @@ $value #end if #elif $task.method == 'distRepo' Tag: $tag.name
- Repo ID: $params[1]
+ Repo ID: $params[1]
Keys: $printValue(0, $params[2])
$printOpts($params[3]) #elif $task.method == 'prepRepo' @@ -243,7 +243,7 @@ $value #end if #elif $task.method == 'createdistrepo' Tag: $tag.name
- Repo ID: $params[1]
+ Repo ID: $params[1]
Arch: $printValue(0, $params[2])
Keys: $printValue(0, $params[3])
Options: $printMap($params[4], '    ') From cb714b742703d54e7aace5d266dd035117156aca Mon Sep 17 00:00:00 2001 From: Mike McLean Date: May 03 2018 20:05:16 +0000 Subject: [PATCH 34/36] no need for subtask to return its upload dir --- diff --git a/builder/kojid b/builder/kojid index 4a45853..d989cda 100755 --- a/builder/kojid +++ b/builder/kojid @@ -5156,12 +5156,12 @@ class NewDistRepoTask(BaseTaskHandler): method='createdistrepo', arglist=arglist, label=arch, parent=self.id, arch='noarch') if len(subtasks) > 0 and task_opts['multilib']: - results = self.wait(subtasks.values(), all=True, failany=True) + self.wait(subtasks.values(), all=True, failany=True) for arch in arch32s: # move the 32-bit task output to the final resting place # so the 64-bit arches can use it for multilib - upload = results[subtasks[arch]] - self.session.host.distRepoMove(repo_id, upload, arch) + upload_dir = koji.pathinfo.taskrelpath(subtasks[arch]) + self.session.host.distRepoMove(repo_id, upload_dir, arch) for arch in canonArches: # do the other arches if arch not in arch32s: @@ -5170,18 +5170,14 @@ class NewDistRepoTask(BaseTaskHandler): method='createdistrepo', arglist=arglist, label=arch, parent=self.id, arch='noarch') # wait for 64-bit subtasks to finish - data = {} - results = self.wait(subtasks.values(), all=True, failany=True) + self.wait(subtasks.values(), all=True, failany=True) for (arch, task_id) in subtasks.iteritems(): - data[arch] = results[task_id] - self.logger.debug("DEBUG: %r : %r " % (arch, data[arch])) if task_opts['multilib'] and arch in arch32s: # already moved above continue - #else - upload = results[subtasks[arch]] - self.session.host.distRepoMove(repo_id, upload, arch) - self.session.host.repoDone(repo_id, data, expire=False) + upload_dir = koji.pathinfo.taskrelpath(subtasks[arch]) + self.session.host.distRepoMove(repo_id, upload_dir, arch) + self.session.host.repoDone(repo_id, {}, expire=False) return 'Dist repository #%s successfully generated' % repo_id @@ -5271,8 +5267,6 @@ class createDistRepoTask(BaseTaskHandler): self.upload_repo() self.upload_repo_manifest() - return self.uploadpath - def upload_repo_file(self, relpath): """Upload a file from the repo From 0171027224522a4a8f96fd92e1ea93e7d69f830c Mon Sep 17 00:00:00 2001 From: Mike McLean Date: May 03 2018 20:05:16 +0000 Subject: [PATCH 35/36] fix unit tests --- diff --git a/tests/test_cli/test_dist_repo.py b/tests/test_cli/test_dist_repo.py index 9a55403..527cd81 100644 --- a/tests/test_cli/test_dist_repo.py +++ b/tests/test_cli/test_dist_repo.py @@ -261,7 +261,7 @@ Options: --delta-rpms=REPO Create delta rpms. REPO can be the id of another dist repo or the name of a tag that has a dist repo. May be specified multiple times. - --event=EVENT create a dist repository based on a Brew event + --event=EVENT Use tag content at event --non-latest Include older builds, not just the latest --multilib=CONFIG Include multilib packages in the repository using the given config file diff --git a/tests/test_hub/test_dist_repo.py b/tests/test_hub/test_dist_repo.py index 33f1b6c..df4388d 100644 --- a/tests/test_hub/test_dist_repo.py +++ b/tests/test_hub/test_dist_repo.py @@ -1,5 +1,6 @@ import unittest +import json import mock import os import shutil @@ -106,7 +107,7 @@ class TestDistRepoMove(unittest.TestCase): 'create_ts': 1487256924.72718, 'creation_time': '2017-02-16 14:55:24.727181', 'id': 47, - 'state': 1, + 'state': 0, # INIT 'tag_id': 2, 'tag_name': 'my-tag'} self.arch = 'x86_64' @@ -123,19 +124,18 @@ class TestDistRepoMove(unittest.TestCase): os.makedirs(uploaddir) # place some test files - self.files = ['foo.drpm', 'repomd.xml'] + self.files = ['drpms/foo.drpm', 'repodata/repomd.xml'] self.expected = ['x86_64/drpms/foo.drpm', 'x86_64/repodata/repomd.xml'] for fn in self.files: path = os.path.join(uploaddir, fn) koji.ensuredir(os.path.dirname(path)) with open(path, 'w') as fo: - fo.write('%s' % fn) + fo.write('%s' % os.path.basename(fn)) - # generate pkglist file and sigmap + # generate pkglist file self.files.append('pkglist') plist = os.path.join(uploaddir, 'pkglist') nvrs = ['aaa-1.0-2', 'bbb-3.0-5', 'ccc-8.0-13','ddd-21.0-34'] - self.sigmap = [] self.rpms = {} self.builds ={} self.key = '4c8da725' @@ -159,9 +159,24 @@ class TestDistRepoMove(unittest.TestCase): binfo['id'] = build_id rpminfo['build_id'] = build_id rpminfo['id'] = rpm_id + rpminfo['sigkey'] = self.key + rpminfo['size'] = 1024 + rpminfo['payloadhash'] = 'helloworld' self.builds[build_id] = binfo self.rpms[rpm_id] = rpminfo - self.sigmap.append([rpm_id, self.key]) + + # write kojipkgs + kojipkgs = {} + for rpminfo in self.rpms.values(): + bnp = '%(name)s-%(version)s-%(release)s.%(arch)s.rpm' % rpminfo + kojipkgs[bnp] = rpminfo + with open("%s/kojipkgs" % uploaddir, "w") as fp: + json.dump(kojipkgs, fp, indent=4) + self.files.append('kojipkgs') + + # write manifest + with open("%s/repo_manifest" % uploaddir, "w") as fp: + json.dump(self.files, fp, indent=4) # mocks self.repo_info = mock.patch('kojihub.repo_info').start() @@ -187,8 +202,7 @@ class TestDistRepoMove(unittest.TestCase): def test_distRepoMove(self): exports = kojihub.HostExports() - exports.distRepoMove(self.rinfo['id'], self.uploadpath, - list(self.files), self.arch, self.sigmap) + exports.distRepoMove(self.rinfo['id'], self.uploadpath, self.arch) # check result repodir = self.topdir + '/repos-dist/%(tag_name)s/%(id)s' % self.rinfo for relpath in self.expected: From 7a0acc00cba4266227e1de5d16b8aac14f592983 Mon Sep 17 00:00:00 2001 From: Mike McLean Date: May 03 2018 20:05:16 +0000 Subject: [PATCH 36/36] put dist repo rpms under Packages/ --- diff --git a/builder/kojid b/builder/kojid index d989cda..eb78d94 100755 --- a/builder/kojid +++ b/builder/kojid @@ -5582,8 +5582,9 @@ enabled=1 def link_pkgs(self): for bnp in self.kojipkgs: bnplet = bnp[0].lower() - koji.ensuredir(os.path.join(self.repodir, bnplet)) - dst = os.path.join(self.repodir, bnplet, bnp) + ddir = os.path.join(self.repodir, 'Packages', bnplet) + koji.ensuredir(ddir) + dst = os.path.join(ddir, bnp) pkgpath = self.kojipkgs[bnp]['_pkgpath'] self.logger.debug("os.symlink(%r, %r(", pkgpath, dst) os.symlink(pkgpath, dst) @@ -5605,9 +5606,9 @@ enabled=1 if subrepo: # note the ../ subrepo_pkgs.setdefault(subrepo, []).append( - '../%s/%s\n' % (bnplet, bnp)) + '../Packages/%s/%s\n' % (bnplet, bnp)) else: - pkgs.append('%s/%s\n' % (bnplet, bnp)) + pkgs.append('Packages/%s/%s\n' % (bnplet, bnp)) with open('%s/pkglist' % self.repodir, 'w') as fo: for line in pkgs: diff --git a/hub/kojihub.py b/hub/kojihub.py index 3304778..af87e77 100644 --- a/hub/kojihub.py +++ b/hub/kojihub.py @@ -12735,8 +12735,9 @@ class HostExports(object): rpmpath = rpminfo['_fullpath'] bnp = fn bnplet = bnp[0].lower() - koji.ensuredir(os.path.join(archdir, bnplet)) - l_dst = os.path.join(archdir, bnplet, bnp) + ddir = os.path.join(archdir, 'Packages', bnplet) + koji.ensuredir(ddir) + l_dst = os.path.join(ddir, bnp) if os.path.exists(l_dst): raise koji.GenericError("File already in repo: %s", l_dst) logger.debug("os.link(%r, %r)", rpmpath, l_dst) @@ -12744,8 +12745,7 @@ class HostExports(object): os.link(rpmpath, l_dst) except OSError as ose: if ose.errno == 18: - shutil.copy2( - rpmpath, os.path.join(archdir, bnplet, bnp)) + shutil.copy2(rpmpath, l_dst) else: raise diff --git a/tests/test_hub/test_dist_repo.py b/tests/test_hub/test_dist_repo.py index df4388d..90f9d54 100644 --- a/tests/test_hub/test_dist_repo.py +++ b/tests/test_hub/test_dist_repo.py @@ -153,7 +153,7 @@ class TestDistRepoMove(unittest.TestCase): fo.write('%s' % basename) f_pkglist.write(path) f_pkglist.write('\n') - self.expected.append('x86_64/%s/%s' % (basename[0], basename)) + self.expected.append('x86_64/Packages/%s/%s' % (basename[0], basename)) build_id = len(self.builds) + 10000 rpm_id = len(self.rpms) + 20000 binfo['id'] = build_id